diff --git a/.bumpversion.toml b/.bumpversion.toml index 6840e4f3138..608cabad906 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "9.0.0-beta.16" +current_version = "12.0.0-beta.11" parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)(-(?P(beta|rc))\\.(?P\\d+))?" serialize = [ "{major}.{minor}.{patch}-{prerelease}.{prerelease_num}", @@ -80,6 +80,11 @@ filename = "Cargo.toml" search = 'lance-index = {{ version = "={current_version}"' replace = 'lance-index = {{ version = "={new_version}"' +[[tool.bumpversion.files]] +filename = "Cargo.toml" +search = 'lance-index-core = {{ version = "={current_version}"' +replace = 'lance-index-core = {{ version = "={new_version}"' + [[tool.bumpversion.files]] filename = "Cargo.toml" search = 'lance-io = {{ version = "={current_version}"' diff --git a/.cargo/config.toml b/.cargo/config.toml index c455c4a978d..4fef1d7e0fa 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -22,6 +22,14 @@ lto = "thin" codegen-units = 16 [target.x86_64-unknown-linux-gnu] +# The default target is haswell. This is an old enough target +# that portability is high but a new enough target that we gain +# some of the most common SIMD optimizations. +# +# On certain paths we use explicit SIMD and runtime dispatch to +# opt-in to even higher CPU targets. The target specified here +# is for all code that is NOT using explicit SIMD (still the large +# majority of code). rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"] [target.aarch64-apple-darwin] diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 00000000000..9779b0ff8e4 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,17 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +language: en-US +early_access: false +reviews: + profile: quiet + high_level_summary: false + poem: false + review_status: true + auto_review: + enabled: true + auto_incremental_review: true + ignore_title_keywords: + - WIP + - Draft + drafts: false + base_branches: + - main diff --git a/.gitattributes b/.gitattributes index f3351d043eb..a447781cffc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ notebooks/** linguist-vendored Cargo.lock linguist-generated -**/Cargo.lock linguist-generated \ No newline at end of file +**/Cargo.lock linguist-generated +rust/lance-file/test_data/**/*.lance binary diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000000..0e7363697b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,79 @@ +name: Bug report +description: Report incorrect, unexpected, or crashing behavior. +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a bug report! Please search + [existing issues](https://github.com/lance-format/lance/issues) first to + avoid filing a duplicate. + - type: textarea + id: description + attributes: + label: Description + description: A clear and concise description of what the bug is. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Steps to reproduce + description: >- + A minimal, self-contained code snippet or sequence of steps that + reproduces the problem. The more we can copy-paste and run, the faster + we can fix it. + placeholder: | + 1. Write a dataset with `...` + 2. Scan with filter `...` + 3. Observe `...` + render: python + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + description: What did you expect to happen? + validations: + required: false + - type: input + id: lance-version + attributes: + label: Lance version + description: >- + Output of `python -c "import lance; print(lance.__version__)"`, or the + `lance`/`pylance` version from your `Cargo.toml` / `pyproject.toml`. + placeholder: "e.g. 0.40.0" + validations: + required: true + - type: dropdown + id: language + attributes: + label: Language binding + multiple: true + options: + - Python + - Rust + - Java + - Other / not sure + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + description: OS, architecture, and storage backend (local, S3, GCS, Azure, ...). + placeholder: "e.g. Ubuntu 22.04, x86_64, S3" + validations: + required: false + - type: textarea + id: logs + attributes: + label: Logs / traceback + description: >- + Any relevant log output or stack trace. This will be automatically + formatted as code, so no need for backticks. + render: shell + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000..32389799971 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +blank_issues_enabled: true +contact_links: + - name: Question / usage help + url: https://discord.gg/lance + about: >- + For questions and general discussion, please ask in the Lance Discord + rather than opening an issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..41d6278fc02 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,11 @@ +--- +name: Feature request +about: Suggest a new capability or an improvement to an existing one. +labels: feature +--- + + diff --git a/.github/ISSUE_TEMPLATE/performance_issue.md b/.github/ISSUE_TEMPLATE/performance_issue.md new file mode 100644 index 00000000000..65fcd0be6f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/performance_issue.md @@ -0,0 +1,12 @@ +--- +name: Performance issue +about: Report slow operations, high memory use, or a performance regression. +labels: performance +--- + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a7b7d11c92b..5c5f1aa6113 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,45 +1,86 @@ version: 2 updates: + # Rust — one grouped version-bump PR per directory each week, and security + # advisories that land together in a directory batched into a single PR + # instead of one PR per advisory. - package-ecosystem: "cargo" - directory: "/" + directories: + - "/" + - "/python" + - "/java/lance-jni" versioning-strategy: lockfile-only schedule: interval: "weekly" day: "wednesday" groups: cargo: + applies-to: version-updates + patterns: + - "*" + cargo-security: + applies-to: security-updates patterns: - "*" - - package-ecosystem: "cargo" + # Python (uv) — same weekly-grouped + grouped-security treatment. + - package-ecosystem: "uv" directory: "/python" versioning-strategy: lockfile-only schedule: interval: "weekly" day: "wednesday" groups: - cargo: + uv: + applies-to: version-updates + patterns: + - "*" + uv-security: + applies-to: security-updates patterns: - "*" + # Standalone Rust lockfiles for dev/test fixtures. Dependabot security updates + # scan these regardless of config, so we group them; open-pull-requests-limit: 0 + # disables the (unwanted) weekly version bumps for these crates. - package-ecosystem: "cargo" - directory: "/java/lance-jni" - versioning-strategy: lockfile-only + directories: + - "/memtest" + - "/test_data/fri_straddle_pre_6610/datagen" + open-pull-requests-limit: 0 schedule: interval: "weekly" day: "wednesday" groups: - cargo: + cargo-fixtures-security: + applies-to: security-updates patterns: - "*" - - package-ecosystem: "uv" - directory: "/python" - versioning-strategy: lockfile-only + # Java (Maven). Security updates fire regardless of config; group them and + # skip version bumps (limit 0) — we bump Java deps manually. + - package-ecosystem: "maven" + directory: "/java" + open-pull-requests-limit: 0 schedule: interval: "weekly" day: "wednesday" groups: - uv: + maven-security: + applies-to: security-updates + patterns: + - "*" + + # Python benchmark scripts (pip / requirements.txt). Group security updates + # and skip version bumps (limit 0). + - package-ecosystem: "pip" + directories: + - "/benchmarks/*" + open-pull-requests-limit: 0 + schedule: + interval: "weekly" + day: "wednesday" + groups: + pip-security: + applies-to: security-updates patterns: - "*" diff --git a/.github/labeler-area.yml b/.github/labeler-area.yml index 9afb49172af..998e752fba6 100644 --- a/.github/labeler-area.yml +++ b/.github/labeler-area.yml @@ -29,12 +29,40 @@ A-encoding: - "rust/lance-io/**" - "rust/lance-file/**" -# On-disk format: the proto definitions and the format spec docs. Format docs +# On-disk format: persisted proto definitions and the format spec docs. Keep the +# explicit proto lists here in sync. Execution-plan schemas (ann, +# filtered_read, and table_identifier) are intentionally excluded. Format docs # are excluded from A-docs (see below) so format changes get A-format only. A-format: - changed-files: - any-glob-to-any-file: - - "protos/**" + - "protos/encodings_v2_0.proto" + - "protos/encodings_v2_1.proto" + - "protos/file.proto" + - "protos/file2.proto" + - "protos/index.proto" + - "protos/index_old.proto" + - "protos/rowids.proto" + - "protos/table.proto" + - "protos/transaction.proto" + - "docs/src/format/**" + +# Drives the format-spec vote gate (.github/workflows/format-vote-gate.yml): +# any change to persisted proto definitions or the spec docs requires a PMC +# vote. Execution-only wire schemas such as filtered_read.proto are not format +# changes. A PMC member waives a trivial edit with the `format-waived` label. +format-change: + - changed-files: + - any-glob-to-any-file: + - "protos/encodings_v2_0.proto" + - "protos/encodings_v2_1.proto" + - "protos/file.proto" + - "protos/file2.proto" + - "protos/index.proto" + - "protos/index_old.proto" + - "protos/rowids.proto" + - "protos/table.proto" + - "protos/transaction.proto" - "docs/src/format/**" # Lockfiles are intentionally not excluded: a pure dependency bump gets both diff --git a/.github/labeler-issues.yml b/.github/labeler-issues.yml new file mode 100644 index 00000000000..625b321cd6c --- /dev/null +++ b/.github/labeler-issues.yml @@ -0,0 +1,28 @@ +version: 1 +# Never remove labels a template or a human already applied. +appendOnly: true +# Content-based labels for issues, applied by srvaroa/labeler via +# .github/workflows/issue-labeler.yml. This covers issues that bypass the +# .github/ISSUE_TEMPLATE forms entirely — those filed via `gh issue create`, +# the REST API, or an agent, which never see the web-UI template chooser. +# +# The primary signal is a leading marker in the title (e.g. "bug:", "[feature]", +# "perf -"), which many people type by hand; a modest body keyword match is a +# secondary fallback. Rules are OR'd (one label per matching entry), so an +# unmatched issue gets no label and is left for human / LLM triage rather than +# guessed at. appendOnly means matches stack harmlessly with template labels. +labels: +# --- Primary signal: a leading bug/feature/perf marker in the title --- +- label: bug + title: "(?i)^\\s*[\\[(]?\\s*bug\\b" +- label: feature + title: "(?i)^\\s*[\\[(]?\\s*(feature|feat)\\b" +- label: performance + title: "(?i)^\\s*[\\[(]?\\s*(perf|performance)\\b" +# --- Secondary fallback: keywords anywhere in the body --- +- label: bug + body: "(?i)(panic|segfault|traceback|stack ?trace|crash|corrupt|incorrect result|wrong result)" +- label: performance + body: "(?i)(regression|latency|throughput|\\bOOM\\b|out of memory|memory usage|too slow)" +- label: feature + body: "(?i)(feature request|would be (nice|great|useful)|it would be nice|support for|add .+ support)" diff --git a/.github/nextest/daily-code-coverage.toml b/.github/nextest/daily-code-coverage.toml new file mode 100644 index 00000000000..ee49194bd2e --- /dev/null +++ b/.github/nextest/daily-code-coverage.toml @@ -0,0 +1,3 @@ +[profile.daily-code-coverage] +slow-timeout = { period = "60s", terminate-after = 30, grace-period = "10s" } +global-timeout = "5h30m" diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml deleted file mode 100644 index aa6e73ae74f..00000000000 --- a/.github/workflows/ci-benchmarks.yml +++ /dev/null @@ -1,78 +0,0 @@ -name: Run Regression Benchmarks - -on: - workflow_dispatch: - push: - branches: - - main - -permissions: - contents: read - -jobs: - bench_regress: - timeout-minutes: 120 - runs-on: warp-custom-gcp-storage-benchmark - env: - # Need up-to-date compilers for kernels - CC: clang-18 - CXX: clang++-18 - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - lfs: true - - name: Authenticate with GCS - uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2 - with: - credentials_json: "${{ secrets.GCLOUD_BENCH_STORAGE_USER_KEY }}" - - name: Install bencher - uses: bencherdev/bencher@8151077aa7b1bceaac11c4b308265417cae60e2b # v0.5.10 - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: 3.11 # Ray does not support 3.12 yet. - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Build - run: | - python -m venv venv - source venv/bin/activate - pip install maturin==1.13.3 uv duckdb requests pytest pytest-benchmark datasets - maturin develop --uv --locked --release --features datagen - - name: Build memtest - run: | - source venv/bin/activate - make -C ../memtest build-release - - name: Generate datasets - run: | - source venv/bin/activate - python python/ci_benchmarks/datagen/gen_all.py - - name: Run benchmarks - run: | - source venv/bin/activate - bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} --adapter python_pytest \ - --branch main --testbed google-genoa --err --file results.json "python -mpytest --benchmark-json \ - results.json python/ci_benchmarks" - - name: Run IO/memory benchmarks - run: | - source venv/bin/activate - LIB_PATH=$(lance-memtest) - LD_PRELOAD=$LIB_PATH pytest python/ci_benchmarks \ - -k "io_mem_" \ - --benchmark-stats-json io_mem_stats.json - - name: Upload IO/memory stats to bencher - run: | - source venv/bin/activate - bencher run --project weston-lancedb --token ${{ secrets.LANCE_BENCHER_TOKEN }} \ - --adapter json --branch main --testbed google-genoa \ - --err --file io_mem_stats.json diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml new file mode 100644 index 00000000000..7b1cc2ee220 --- /dev/null +++ b/.github/workflows/ci-scripts.yml @@ -0,0 +1,40 @@ +name: CI scripts + +# Tests for helper scripts under ci/ that aren't covered by the language test +# suites (e.g. the format-spec vote gate logic). + +on: + push: + branches: + - main + - release/** + pull_request: + branches: + - main + - release/** + paths: + - ci/format_vote_gate.py + - ci/test_format_vote_gate.py + - ci/test_labeler_area.py + - .github/labeler-area.yml + - .github/workflows/format-vote-gate.yml + - .github/workflows/ci-scripts.yml + +permissions: + contents: read + +jobs: + format-vote-gate: + name: Format vote gate unit tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Install test dependencies + run: pip install pytest PyYAML + - name: Run tests + run: pytest ci/test_format_vote_gate.py ci/test_labeler_area.py diff --git a/.github/workflows/compat-pair.yml b/.github/workflows/compat-pair.yml index ba3332d8d7b..07bb0f49741 100644 --- a/.github/workflows/compat-pair.yml +++ b/.github/workflows/compat-pair.yml @@ -21,7 +21,7 @@ on: required: false default: "" kinds: - description: "Comma-separated index kinds (INVERTED,BTREE,...) or 'all'." + description: "Comma-separated index kinds (INVERTED,BTREE,IVF_PQ,...) or 'all'." required: false default: "all" max_length: @@ -51,10 +51,10 @@ jobs: # Toolchain for the build-from-source provisioning path (refs without a wheel). - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - - name: Install build deps - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Install host deps run: pip install pytest pytest-xdist pyarrow packaging maturin - name: Resolve kinds @@ -63,7 +63,7 @@ jobs: KINDS_IN: ${{ inputs.kinds }} run: | if [ -z "$KINDS_IN" ] || [ "$KINDS_IN" = "all" ]; then - echo "value=INVERTED,BTREE,BITMAP,LABEL_LIST,NGRAM,ZONEMAP,BLOOMFILTER" >> "$GITHUB_OUTPUT" + echo "value=INVERTED,BTREE,BITMAP,LABEL_LIST,NGRAM,ZONEMAP,BLOOMFILTER,IVF_PQ" >> "$GITHUB_OUTPUT" else echo "value=$KINDS_IN" >> "$GITHUB_OUTPUT" fi diff --git a/.github/workflows/daily-code-coverage.yml b/.github/workflows/daily-code-coverage.yml new file mode 100644 index 00000000000..62871834b62 --- /dev/null +++ b/.github/workflows/daily-code-coverage.yml @@ -0,0 +1,74 @@ +name: Daily Code Coverage + +on: + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: daily-code-coverage + cancel-in-progress: false + +jobs: + coverage: + runs-on: ubuntu-24.04-8x + timeout-minutes: 360 + env: + CC: clang + CXX: clang++ + RUSTFLAGS: "-D warnings" + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup Rust toolchain + run: | + rustup toolchain install nightly-2026-07-13 --component llvm-tools-preview + rustup default nightly-2026-07-13 + - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + key: daily-code-coverage + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov + with: + tool: cargo-llvm-cov + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + - name: Start DynamoDB and S3 + run: docker compose -f docker-compose.yml up -d --wait + - name: Run coverage tests + env: + NEXTEST_PROFILE: daily-code-coverage + run: | + ALL_FEATURES=$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -) + cargo +nightly-2026-07-13 llvm-cov nextest \ + --codecov \ + --output-path coverage.codecov \ + --cargo-profile ci \ + --locked \ + --workspace \ + --features "${ALL_FEATURES}" \ + --config-file .github/nextest/daily-code-coverage.toml + - name: Upload coverage artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: daily-rust-coverage + path: coverage.codecov + retention-days: 14 + if-no-files-found: error + - name: Upload coverage to Codecov + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.codecov + flags: unittests + name: daily-code-coverage + fail_ci_if_error: true diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml deleted file mode 100644 index 2eda3de66b4..00000000000 --- a/.github/workflows/docs-check.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Check docs - -on: - push: - branches: - - main - - release/** - pull_request: - branches: - - main - - release/** - paths: - - docs/** - - .github/workflows/docs-check.yml - -permissions: - contents: read - -env: - RUSTFLAGS: "-C debuginfo=0" - # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html - # CI builds are faster with incremental disabled. - CARGO_INCREMENTAL: "0" - -jobs: - # Single deploy job since we're just deploying - check-docs: - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: "Set up Python" - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version-file: "docs/pyproject.toml" - - name: Install uv - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 - with: - enable-cache: true - - name: Check links - working-directory: docs - run: | - uv run mkdocs-linkcheck src diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 0733e5ce32c..aeaea0bbcba 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -62,6 +62,11 @@ jobs: with: repository: lance-format/lance-trino path: lance-trino + - name: Checkout lance-context + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: lance-format/lance-context + path: lance-context - name: Configure Git Credentials run: | git config user.name github-actions[bot] @@ -83,6 +88,7 @@ jobs: LANCE_TRINO_REPO: ${{ github.workspace }}/lance-trino LANCE_DUCKDB_REPO: ${{ github.workspace }}/lance-duckdb LANCE_HUGGINGFACE_REPO: ${{ github.workspace }}/lance-huggingface + LANCE_CONTEXT_REPO: ${{ github.workspace }}/lance-context run: | docs/make-full-website.sh - name: Deploy diff --git a/.github/workflows/docs-link-check.yml b/.github/workflows/docs-link-check.yml new file mode 100644 index 00000000000..a92bf4812d7 --- /dev/null +++ b/.github/workflows/docs-link-check.yml @@ -0,0 +1,166 @@ +name: Check doc links + +# Checking external links is inherently noisy: third-party sites rate-limit +# automated clients, reject non-browser user agents, and go down temporarily. +# Blocking pull requests on that trades a lot of false failures for very little +# signal, so this runs on a schedule and reports findings in a single tracking +# issue instead of failing anyone's build. +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + +# The report lives in one repository-global issue, so runs must not overlap: a +# lookup racing a create produces duplicate issues, and a healthy run closing +# the issue while a failing run only rewrites its body would leave a broken +# report closed. The group is deliberately ref-independent so that a manual +# dispatch serializes against the scheduled run. +concurrency: + group: docs-link-check + cancel-in-progress: false + +permissions: {} + +env: + REPORT_TITLE: "Docs link checker report" + +jobs: + scan: + name: Scan links + runs-on: ubuntu-24.04 + # lychee-action is pinned by SHA, but its wrapper downloads the lychee + # release tarball at run time without verifying a digest, and hands the + # resulting binary a GitHub token. Release assets remain replaceable, so + # that binary is confined to a job whose token can only read public + # content; everything that writes runs in the report job below. + permissions: + contents: read + outputs: + exit_code: ${{ steps.lychee.outputs.exit_code }} + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + # workflow_dispatch can run from any ref, but the report is + # repository-global. Always measure the default branch so a manual + # run from a topic branch cannot close a report that main warrants, + # or overwrite it with branch-only findings. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + + - name: Check links + id: lychee + uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # v2.9.0 + with: + # Restricted to http(s) on purpose. Relative links resolve against + # the assembled multi-repo site (docs/make-full-website.sh pulls + # docs/src/format/catalog, format/namespace, integrations/spark and + # friends from other repositories), so they cannot be verified from + # this checkout alone and would report as broken on every run. + # + # crates.io answers automated clients with 403/404 no matter the user + # agent, so it is excluded rather than reported every day. + args: >- + --scheme https + --scheme http + --no-progress + --max-retries 3 + --timeout 20 + --exclude '^https://crates\.io/' + 'docs/src/**/*.md' + format: markdown + output: ./lychee/out.md + jobSummary: true + # The report, not a red build, is the signal for broken links. The + # report job below still fails the run if the check itself breaks. + fail: false + + - name: Upload report + if: steps.lychee.outputs.exit_code == 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: link-report + path: ./lychee/out.md + retention-days: 7 + + report: + name: Update report issue + needs: scan + runs-on: ubuntu-24.04 + # Deliberately no checkout: this job needs the report artifact and the + # issues API, not the repository contents. + permissions: + issues: write + env: + EXIT_CODE: ${{ needs.scan.outputs.exit_code }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Classify checker result + # lychee exits 0 when every link resolves and 2 when links fail. + # Anything else (1 runtime, 3 bad config) means the check never + # produced a link verdict, which must surface as a failed run rather + # than be published as "broken documentation links". + run: | + case "$EXIT_CODE" in + 0|2) + echo "lychee exit code $EXIT_CODE" + ;; + *) + echo "::error::lychee exited with '$EXIT_CODE': the link check did not complete. Leaving the report issue untouched." + exit 1 + ;; + esac + + - name: Find existing report issue + id: report + # Matched on title alone, and through search rather than a listing: + # the issue action applies labels in a separate call after creating the + # issue, so a label filter misses a half-created report, and this + # repository has far more open issues than one listing page holds. + run: | + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open \ + --search "in:title \"$REPORT_TITLE\" author:app/github-actions" \ + --limit 50 --json number,title \ + --jq "[.[] | select(.title == \"$REPORT_TITLE\") | .number] | first // empty") + echo "number=$number" >> "$GITHUB_OUTPUT" + + - name: Download report + if: env.EXIT_CODE == 2 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: link-report + path: ./lychee + + - name: Compose report + if: env.EXIT_CODE == 2 + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + { + echo "Broken documentation links found by [\`$GITHUB_WORKFLOW\`]($run_url)." + echo + echo "This issue is rewritten by every scheduled run and closed automatically once all links resolve." + echo + echo "Entries can be false positives: some sites rate-limit or block automated clients while working fine in a browser. Confirm before editing the docs, and add persistent offenders to \`--exclude\` in \`.github/workflows/docs-link-check.yml\`." + echo + cat ./lychee/out.md + } > ./lychee/issue.md + + - name: Report broken links + if: env.EXIT_CODE == 2 + uses: peter-evans/create-issue-from-file@fca9117c27cdc29c6c4db3b86c48e4115a786710 # v6.0.0 + with: + # Empty on the first failing run, which creates the issue; afterwards + # the same issue is updated in place. + issue-number: ${{ steps.report.outputs.number }} + title: ${{ env.REPORT_TITLE }} + content-filepath: ./lychee/issue.md + labels: documentation + + - name: Close report issue once links are healthy + if: env.EXIT_CODE == 0 && steps.report.outputs.number != '' + env: + ISSUE_NUMBER: ${{ steps.report.outputs.number }} + run: | + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + gh issue close "$ISSUE_NUMBER" --repo "$GITHUB_REPOSITORY" \ + --comment "All documentation links resolved in [the latest run]($run_url)." diff --git a/.github/workflows/file_verification.yml b/.github/workflows/file_verification.yml index 41c7883aa83..425cfc12196 100644 --- a/.github/workflows/file_verification.yml +++ b/.github/workflows/file_verification.yml @@ -18,10 +18,10 @@ jobs: with: python-version: "3.11" - - name: Install Build Requirements - run: | - sudo apt update - sudo apt install -y protobuf-compiler + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Validate AWS Credentials run: | diff --git a/.github/workflows/format-vote-gate.yml b/.github/workflows/format-vote-gate.yml new file mode 100644 index 00000000000..d120057ac18 --- /dev/null +++ b/.github/workflows/format-vote-gate.yml @@ -0,0 +1,86 @@ +name: Format spec vote gate + +# Structurally enforces the PMC vote required for Lance format-specification +# changes (see https://lance.org/community/voting/). The path labeler +# (.github/labeler-area.yml) applies the `format-change` label to PRs that touch +# the persisted format protos or `docs/src/format/**`; execution-only wire +# schemas are excluded. This gate reads that label and blocks merging until the +# PR has 3 binding +1 votes from PMC members (PR approvals, excluding the +# author), has no outstanding veto (a PMC "Request changes" review), and the +# 72-hour voting period has elapsed. That period starts once the PR is labeled +# and ready for review, and pauses over weekends. +# +# The gate publishes its verdict as the `format-spec-vote` commit status on the +# PR head. To make it a merge blocker, an org admin must add `format-spec-vote` +# as a required status check in the branch protection rules for `main` (and any +# release branches). The status is posted on *every* PR — success immediately +# for non-format PRs — so a required check is never left pending forever. +# +# A PMC member may waive a trivial edit (typo, wording, formatting) by applying +# the `format-waived` label. +# +# Uses pull_request_target so the token can post statuses/comments on fork-based +# PRs. It never checks out or executes PR code: it reads the trusted base +# checkout (for the PMC roster and this script) and queries the API only. +# +# There is deliberately no `pull_request_review` trigger. A run triggered by a +# fork PR's review event gets a read-only GITHUB_TOKEN no matter what the +# `permissions` block below asks for, so it cannot post the status — and format +# proposals from non-committers are exactly the fork-PR case. Approvals are +# therefore picked up by the `schedule` sweep instead, or by a manual +# `workflow_dispatch` run for a voter who does not want to wait for it. The +# gate's PR comment links to that dispatch page. + +on: + pull_request_target: + types: + - opened + - reopened + - synchronize + - labeled + - unlabeled + # The voting clock starts when a PR leaves draft, so both draft + # transitions have to re-evaluate the gate. + - ready_for_review + - converted_to_draft + schedule: + # Re-evaluate open format-change PRs so votes and the voting-period clock are + # re-checked when no PR event fires: reviews land without one (see the note + # above), and the period routinely elapses days after the third approval. + # Every 15 minutes, because this is the only path that observes a vote. + - cron: "*/15 * * * *" + workflow_dispatch: + inputs: + pr: + description: "PR number to re-check (blank = every open format-change PR)" + required: false + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + statuses: write + +jobs: + gate: + name: Evaluate format spec vote + runs-on: ubuntu-latest + steps: + - name: Checkout base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: Install dependencies + run: pip install PyGithub PyYAML + - name: Evaluate vote + run: python ci/format_vote_gate.py + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GATE_PR: ${{ inputs.pr }} diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml new file mode 100644 index 00000000000..36f5a9cdbca --- /dev/null +++ b/.github/workflows/issue-labeler.yml @@ -0,0 +1,29 @@ +name: Issue Labeler + +# Applies bug / feature / performance labels to issues based on their title and +# body. This complements the .github/ISSUE_TEMPLATE forms, which only apply +# labels for issues opened through the web UI; this catches issues opened via +# `gh issue create`, the REST API, or an agent, which bypass templates. +# +# Issues never originate from a fork, so unlike the PR labelers this uses a +# plain `issues` trigger (no pull_request_target) with a scoped GITHUB_TOKEN. +on: + issues: + types: [opened, edited] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number }} + cancel-in-progress: true + +jobs: + label: + name: Apply issue labels + permissions: + issues: write + runs-on: ubuntu-latest + steps: + - uses: srvaroa/labeler@bf262763a8a8e191f5847873aecc0f29df84f957 # v1.14.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + config_path: .github/labeler-issues.yml diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 85b50e10836..07b5a9a86aa 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -35,13 +35,11 @@ jobs: workspaces: | lance java/lance-jni -> ../target/rust-maven-plugin/lance-jni - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov - name: Run cargo fmt working-directory: java/lance-jni run: cargo fmt --check @@ -59,17 +57,15 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc # pin the toolchain version to avoid surprises - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 with: toolchain: stable - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: workspaces: java/lance-jni -> ../target/rust-maven-plugin/lance-jni diff --git a/.github/workflows/nightly_run.yml b/.github/workflows/nightly_run.yml index 228b0b7de22..b980d5ef1d5 100644 --- a/.github/workflows/nightly_run.yml +++ b/.github/workflows/nightly_run.yml @@ -11,7 +11,7 @@ permissions: jobs: run: runs-on: ubuntu-24.04 - if: github.repository == 'lancedb/lance' + if: github.repository == 'lance-format/lance' permissions: actions: write steps: @@ -25,16 +25,16 @@ jobs: jumbo-tests: # jumbo tests need more resources runs-on: ubuntu-24.04-8x - if: github.repository == 'lancedb/lance' + if: github.repository == 'lance-format/lance' timeout-minutes: 60 permissions: contents: read steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Run Jumbo String/Binary Tests run: | echo "Running jumbo tests for Lance 2.0 and 2.1..." @@ -51,7 +51,7 @@ jobs: # slow and unbounded in depth, so it runs nightly with a generous timeout rather than blocking # merges; the manual escape hatch for arbitrary ref pairs lives in compat-pair.yml. compat-sequence: - if: github.repository == 'lancedb/lance' + if: github.repository == 'lance-format/lance' timeout-minutes: 360 runs-on: ubuntu-24.04 name: Index Sequence Compat @@ -71,10 +71,10 @@ jobs: - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: workspaces: python - - name: Install build deps - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Install host deps run: pip install pytest pytest-xdist pyarrow packaging maturin # Build HEAD once and feed it to the suite as the prebuilt reader, so the two writer diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index a9d339e36ce..51c899f56a3 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -22,69 +22,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: fail_on_error: true - format-vote-reminder: - permissions: - pull-requests: write - name: Remind about format spec vote - runs-on: ubuntu-latest - # Comments on PRs that touch the Lance format specification (*.proto files - # and docs/src/format/**) to remind the author that substantive format - # changes require a PMC vote. Re-checks the full PR diff on every push, so a - # format change introduced by a later commit is still caught; a hidden - # marker keeps it to at most one comment. - steps: - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 - with: - script: | - const { owner, repo } = context.repo; - const prNumber = context.payload.pull_request.number; - const MARKER = ''; - - // The Lance format specification is the proto definitions plus the - // spec docs. Changes to either require a PMC vote. - const isFormatFile = (path) => - path.endsWith('.proto') || path.startsWith('docs/src/format/'); - - const files = await github.paginate(github.rest.pulls.listFiles, { - owner, repo, pull_number: prNumber, per_page: 100, - }); - const formatFiles = files.map((f) => f.filename).filter(isFormatFile); - if (formatFiles.length === 0) { - core.info('No format specification files changed; nothing to do.'); - return; - } - - // Best effort to comment only once: skip if our marker is present. - const comments = await github.paginate(github.rest.issues.listComments, { - owner, repo, issue_number: prNumber, per_page: 100, - }); - if (comments.some((c) => c.body && c.body.includes(MARKER))) { - core.info('Reminder already posted; skipping.'); - return; - } - - const body = [ - MARKER, - '> [!IMPORTANT]', - '> **This PR touches the Lance format specification.**', - '>', - '> Substantive changes to the format specification — the `.proto` definitions', - '> and the spec docs under `docs/src/format/` — require a PMC vote before merge.', - '> Minor edits such as typo fixes, wording, or formatting are excluded; use your', - '> judgment.', - '>', - '> If this is a meaningful format change:', - '> - Start a vote following the [Lance community voting process](https://lance.org/community/voting/).', - '> Format specification modifications need **3 binding +1 votes** (excluding the', - '> proposer), held on GitHub Discussions, with a minimum voting period of **1 week**.', - '> - Once the vote passes, **link the completed vote in this PR**. It should not be', - '> merged until the vote is linked.', - ].join('\n'); - - await github.rest.issues.createComment({ - owner, repo, issue_number: prNumber, body, - }); - core.info(`Posted format vote reminder (changed: ${formatFiles.join(', ')}).`); commitlint: permissions: pull-requests: write diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 601f2dd9dfc..7503b048a46 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -73,10 +73,12 @@ jobs: ruff format --check --diff python ruff check python pyright - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc + - name: Test Rust reader concurrency + run: cargo test --profile ci --locked --no-default-features reader::tests - name: Lint Rust run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -` @@ -93,11 +95,66 @@ jobs: run: | source venv/bin/activate pytest --doctest-modules python/lance + linux-wheel: + timeout-minutes: 45 + name: Python Linux x86_64 wheel + runs-on: "ubuntu-24.04-4x" + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + lfs: true + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.10" + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + workspaces: python + prefix-key: ${{ env.CACHE_PREFIX }} + # python/Cargo.toml enables abi3-py310, so this wheel is reusable by + # every CPython version in the Linux test matrix. + - name: Build cp310 ABI3 wheel + uses: ./.github/workflows/build_linux_wheel + with: + python-minor-version: "10" + args: "--profile ci" + - name: Upload wheel + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: linux-x86_64-wheel + path: python/target/wheels/pylance-*.whl + if-no-files-found: error + + linux-memtest: + timeout-minutes: 15 + name: Python Linux x86_64 memtest + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 + with: + cache: "false" + rustflags: "" + - name: Build memtest + working-directory: memtest + run: cargo build --release + - name: Upload memtest + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: linux-x86_64-memtest + path: memtest/target/release/libmemtest.so + if-no-files-found: error + linux: + needs: [linux-wheel, linux-memtest] timeout-minutes: 45 strategy: matrix: - python-minor-version: ["10", "13"] + python-minor-version: ["10", "13", "14"] name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64" runs-on: "ubuntu-24.04-4x" defaults: @@ -113,27 +170,24 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: 3.${{ matrix.python-minor-version }} - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + - name: Download wheel + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - workspaces: python - prefix-key: ${{ env.CACHE_PREFIX }} - - uses: ./.github/workflows/build_linux_wheel + name: linux-x86_64-wheel + path: python/target/wheels + - name: Download memtest + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - args: "--profile ci" + name: linux-x86_64-memtest + path: memtest/python/memtest - uses: ./.github/workflows/run_tests with: memtest: true - - name: Upload wheels as artifacts - if: ${{ matrix.python-minor-version == '13' }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: linux-wheels - path: python/target/wheels/pylance-*.whl compat: - needs: linux + needs: linux-wheel timeout-minutes: 60 - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-8x name: Compatibility Tests defaults: run: @@ -147,25 +201,29 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: 3.13 - - name: Download wheels + python-version: 3.14 + - name: Download wheel uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: linux-wheels - path: python/wheels + name: linux-x86_64-wheel + path: python/target/wheels - name: Install dependencies run: | - pip install $(ls wheels/pylance-*.whl)[tests,ray] + pip install $(ls target/wheels/pylance-*.whl)[tests,ray] + # Nearly every test here shells out to an old pylance in its own venv, so the + # job spends most of its time waiting on subprocesses rather than on CPU. + # Oversubscribing cores follows what nightly_run.yml already does with this + # suite. Leaving COMPAT_TEMP_VENV unset is what makes that pay off: the venvs + # then land in the shared cache directory, where VenvFactory's flock lets all + # workers reuse one venv per version instead of each building its own. - name: Run compatibility tests run: | - make compattest - env: - COMPAT_TEMP_VENV: 1 + make compattest PYTEST_WORKERS=$(( $(nproc) * 2 )) linux-arm: timeout-minutes: 45 runs-on: ubuntu-24.04-arm64-4x - name: Python Linux 3.13 ARM + name: Python Linux 3.14 ARM defaults: run: shell: bash @@ -178,7 +236,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: 3.13 + python-version: 3.14 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: workspaces: python @@ -195,7 +253,7 @@ jobs: - uses: ./.github/workflows/run_tests mac: timeout-minutes: 45 - name: Python macOS 3.13 ARM + name: Python macOS 3.14 ARM runs-on: "warp-macos-14-arm64-6x" defaults: run: @@ -209,7 +267,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: "3.13" + python-version: "3.14" - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: workspaces: python @@ -242,6 +300,7 @@ jobs: args: "--profile ci" - uses: ./.github/workflows/run_tests aws-integtest: + needs: linux-wheel timeout-minutes: 45 runs-on: "ubuntu-24.04-4x" defaults: @@ -257,13 +316,11 @@ jobs: uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.11" # TODO: upgrade when ray supports 3.12 - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - prefix-key: ${{ env.CACHE_PREFIX }} - - uses: ./.github/workflows/build_linux_wheel + - name: Download wheel + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - args: "--profile ci" + name: linux-x86_64-wheel + path: python/target/wheels - name: Install dependencies run: | pip install ray[data] diff --git a/.github/workflows/recurring-tests.yml b/.github/workflows/recurring-tests.yml deleted file mode 100644 index dd8205a9567..00000000000 --- a/.github/workflows/recurring-tests.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Recurring Tests - -on: - schedule: - - cron: "0 0 * * 0" # Runs at 00:00 UTC every Sunday - workflow_dispatch: - -permissions: - contents: read - -jobs: - get-pylance-versions: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Fetch latest 2 stable pylance versions - id: set-matrix - run: | - # Get all versions from PyPI - pypi_versions=$(curl -s https://pypi.org/pypi/pylance/json | jq -r '.releases | keys_unsorted | .[]') - - # Use only PyPI versions - all_versions=$(echo -e "$pypi_versions" | sort -u -V) - - # Get latest 2 stable versions - stable_versions=$(echo "$all_versions" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 2) - - # Create matrix array - matrix_versions=() - while IFS= read -r version; do - if [ -n "$version" ]; then - matrix_versions+=("$version") - fi - done <<< "$stable_versions" - - # Create JSON array manually - json_array="[" - for i in "${!matrix_versions[@]}"; do - if [ $i -gt 0 ]; then - json_array="$json_array," - fi - json_array="$json_array\"${matrix_versions[$i]}\"" - done - json_array="$json_array]" - - matrix="{\"pylance-version\": $json_array}" - echo "matrix=$matrix" >> $GITHUB_OUTPUT - - # This job is used to test the recurring tests on the latest 2 stable versions of Lance, - # and the back-compat of the recurring tests. - recurring-linux: - needs: get-pylance-versions - name: "Recurring: Linux (Pylance ${{ matrix.pylance-version }})" - runs-on: ubuntu-24.04 - timeout-minutes: 7200 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.get-pylance-versions.outputs.matrix) }} - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - lfs: true - - name: Install protobuf - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.13" - - name: Install dependencies - working-directory: python - shell: bash - run: | - pip install -e ".[tests]" - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - - name: Install Pylance - run: | - pip install pylance==${{ matrix.pylance-version }} - - name: Run recurring tests - id: run_recurring_tests - run: pytest -vvv -s python/tests/recurring/test_recurring.py - - name: Upgrade Pylance - run: pip install -e ".[tests]" - - name: Run recurring tests again - run: pytest -vvv -s python/tests/recurring/test_recurring.py - - # This job is used to test the recurring tests on the main branch. - recurring-linux-fresh-main: - name: "Recurring: Linux (main)" - runs-on: ubuntu-24.04 - timeout-minutes: 7200 - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - fetch-depth: 0 - lfs: true - - name: Install protobuf - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.13" - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - workspaces: python - - name: Install Lance - run: pip install -e ".[tests]" - - name: Run recurring tests - run: pytest -vvv -s python/tests/recurring/test_recurring.py diff --git a/.github/workflows/run_tests/action.yml b/.github/workflows/run_tests/action.yml index ab761532eeb..1a6786f84e2 100644 --- a/.github/workflows/run_tests/action.yml +++ b/.github/workflows/run_tests/action.yml @@ -11,8 +11,12 @@ inputs: default: "false" memtest: required: false - description: "Run memtest" + description: "Install and preload the prebuilt memtest library" default: "false" + pytest-workers: + required: false + description: "pytest-xdist worker count; 'auto' matches the runner's cores" + default: "auto" runs: using: "composite" steps: @@ -36,9 +40,10 @@ runs: if: inputs.memtest == 'true' shell: bash run: | - make build-release - echo "LD_PRELOAD=$(lance-memtest)" >> $GITHUB_ENV + test -f python/memtest/libmemtest.so + pip install -e . + echo "LD_PRELOAD=$(lance-memtest)" >> "$GITHUB_ENV" - name: Run python tests shell: bash working-directory: python - run: make test + run: make test PYTEST_WORKERS=${{ inputs.pytest-workers }} diff --git a/.github/workflows/rust-benchmark.yml b/.github/workflows/rust-benchmark.yml index bb0960148a9..6877fc89d7b 100644 --- a/.github/workflows/rust-benchmark.yml +++ b/.github/workflows/rust-benchmark.yml @@ -35,10 +35,10 @@ jobs: runs-on: warp-ubuntu-latest-arm64-8x timeout-minutes: 120 steps: - - name: Apt-get - run: | - sudo apt update - sudo apt install -y protobuf-compiler + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Checkout uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Run linalg benchmarks diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e3c17671ce3..00755179e36 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -42,15 +42,34 @@ jobs: - name: Check formatting run: cargo fmt -- --check + package: + name: Check crates are publishable + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1 + # Catches manifest problems that only surface at publish time (missing + # readme/license files, path dependencies without a version, excluded + # files a crate needs). `--no-verify` skips building each packaged crate, + # which the other jobs already cover. Keep the excludes in sync with + # .github/workflows/cargo-publish.yml. + - name: Package crates + run: | + cargo package --workspace --no-verify \ + --exclude lance-arrow-stats \ + --exclude lance-arrow-scalar \ + --all-features + rustdoc: runs-on: ubuntu-24.04 timeout-minutes: 30 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Check documentation run: RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps @@ -62,10 +81,10 @@ jobs: steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Get features run: | ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | sort | uniq | paste -s -d "," -` @@ -85,7 +104,7 @@ jobs: linux-build: runs-on: "ubuntu-24.04-8x" - timeout-minutes: 60 + timeout-minutes: 75 env: # Need up-to-date compilers for kernels CC: clang @@ -96,30 +115,38 @@ jobs: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Setup rust toolchain run: | - rustup toolchain install nightly - rustup default nightly + rustup toolchain install stable + rustup default stable - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - - name: Install dependencies + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + - name: Build tests run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev - - name: Start DynamodDB and S3 + ALL_FEATURES=$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -) + cargo nextest run \ + --cargo-profile ci \ + --locked \ + --workspace \ + --features "${ALL_FEATURES}" \ + --no-run + - name: Start DynamoDB and S3 run: docker compose -f docker-compose.yml up -d --wait - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # cargo-llvm-cov - name: Run tests run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` - cargo +nightly llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES} - - name: Upload coverage to Codecov - uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4 - with: - token: ${{ secrets.CODECOV_TOKEN }} - codecov_yml_path: codecov.yml - files: coverage.codecov - flags: unittests - fail_ci_if_error: false + ALL_FEATURES=$(cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -) + cargo nextest run \ + --cargo-profile ci \ + --locked \ + --workspace \ + --features "${ALL_FEATURES}" + linux-arm: runs-on: ubuntu-24.04-arm64-8x timeout-minutes: 75 @@ -131,47 +158,20 @@ jobs: rustup default stable - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - - name: Install dependencies - run: | - sudo apt -y -qq update - sudo apt install -y protobuf-compiler libssl-dev pkg-config + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Build tests run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo test --profile ci --locked --features ${ALL_FEATURES} --no-run - name: Start DynamodDB and S3 run: docker compose -f docker-compose.yml up -d --wait - name: Run tests run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo test --profile ci --locked --features ${ALL_FEATURES} - query-integration-tests: - runs-on: ubuntu-24.04-4x - timeout-minutes: 75 - env: - # We use opt-level 1 which makes some tests 5x faster to run. - RUSTFLAGS: "-C debuginfo=1 -C opt-level=1" - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup rust toolchain - run: | - rustup toolchain install stable - rustup default stable - - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - with: - cache-targets: false - cache-workspace-crates: true - - name: Install dependencies - run: | - sudo apt -y -qq update - sudo apt install -y protobuf-compiler libssl-dev pkg-config - - name: Build query integration tests - run: | - cargo build --locked -p lance --no-default-features --features fp16kernels,slow_tests --tests --test integration_tests - - name: Run query integration tests - run: | - cargo test --locked -p lance --no-default-features --features fp16kernels,slow_tests --test integration_tests build-no-lock: runs-on: ubuntu-24.04-8x timeout-minutes: 30 @@ -189,13 +189,13 @@ jobs: - name: Remove Cargo.lock run: rm -f Cargo.lock - uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1 - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Build all run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo build --profile ci --benches --features ${ALL_FEATURES} --tests mac-build: runs-on: warp-macos-14-arm64-6x @@ -220,15 +220,21 @@ jobs: run: | rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + # macOS runner capacity is the scarcest in CI, so this job is kept as short + # as it can be. nextest runs each test in its own process and schedules + # across all cores, rather than one test binary at a time. Benchmarks are + # only compile-checked here, which build-no-lock already does on Linux; + # nothing about bench compilation is macOS-specific. - name: Build tests run: | - cargo test --profile ci --locked --features fp16kernels,cli,dynamodb,substrait --no-run + cargo nextest run --cargo-profile ci --locked --features fp16kernels,cli,dynamodb,substrait --no-run - name: Run tests run: | - cargo test --profile ci --features fp16kernels,cli,dynamodb,substrait - - name: Check benchmarks - run: | - cargo check --profile ci --benches --features fp16kernels,cli,dynamodb,substrait + cargo nextest run --cargo-profile ci --features fp16kernels,cli,dynamodb,substrait windows-build: runs-on: windows-latest-4x defaults: @@ -246,12 +252,108 @@ jobs: 7z x protoc.zip Add-Content $env:GITHUB_PATH "C:\protoc\bin" shell: powershell + - name: Install cargo-nextest + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # nextest + with: + tool: nextest + # Same reasoning as mac-build: nextest schedules across all cores instead + # of running one test binary at a time, and build-no-lock already + # compile-checks the benchmarks on Linux. - name: Build tests - run: cargo test --profile ci --locked --no-run + run: cargo nextest run --cargo-profile ci --locked --no-run - name: Run tests - run: cargo test --profile ci - - name: Check benchmarks - run: cargo check --profile ci --benches + run: cargo nextest run --cargo-profile ci + + qemu-pre-haswell: + # Verifies that lance-linalg's runtime SIMD dispatch still works + # correctly when the binary is built with the lower x86-64-v2 baseline + # (the legacy build path documented in CONTRIBUTING.md). Emulates a + # Nehalem CPU under qemu-user; catches any accidental AVX2/FMA + # instructions that leak past the runtime dispatch. + # + # The published-wheel default baseline (`target-cpu=haswell`) is set in + # `.cargo/config.toml`; this job overrides RUSTFLAGS for one job to + # exercise the legacy path without affecting any other build. + name: pre-Haswell SIGILL check (qemu Nehalem) + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + CC: clang + CXX: clang++ + RUSTFLAGS: "-C target-cpu=x86-64-v2" + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER: "qemu-x86_64 -cpu Nehalem" + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup rust toolchain + run: | + rustup toolchain install stable + rustup default stable + - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + - name: Restore QEMU 8.2.10 + id: qemu-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/lance-qemu/8.2.10/qemu-x86_64 + key: qemu-user-8.2.10-x86_64-linux-user-${{ runner.os }}-${{ runner.arch }}-v1 + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc + - name: Install QEMU build dependencies + if: steps.qemu-cache.outputs.cache-hit != 'true' + run: | + sudo apt update + sudo apt install -y \ + ninja-build \ + pkg-config \ + libglib2.0-dev \ + python3-venv \ + curl \ + xz-utils + - name: Build QEMU 8.2.10 + if: steps.qemu-cache.outputs.cache-hit != 'true' + run: | + QEMU_VERSION=8.2.10 + QEMU_ARCHIVE="qemu-${QEMU_VERSION}.tar.xz" + QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/${QEMU_VERSION}" + QEMU_SOURCE_DIR="${RUNNER_TEMP}/qemu-${QEMU_VERSION}" + + # Ubuntu 24.04 ships QEMU 8.2.2, which is affected by + # https://gitlab.com/qemu-project/qemu/-/issues/2170. + curl --fail --location --silent --show-error \ + "https://download.qemu.org/${QEMU_ARCHIVE}" \ + --output "${RUNNER_TEMP}/${QEMU_ARCHIVE}" + echo "37b4a643da8ed6015eef35f5d7f06e7259d9c95359965a0a98e9667c621ab2bb ${RUNNER_TEMP}/${QEMU_ARCHIVE}" \ + | sha256sum --check - + tar --extract \ + --file "${RUNNER_TEMP}/${QEMU_ARCHIVE}" \ + --directory "${RUNNER_TEMP}" + mkdir "${QEMU_SOURCE_DIR}/build" + ( + cd "${QEMU_SOURCE_DIR}/build" + ../configure --target-list=x86_64-linux-user --disable-docs + ninja qemu-x86_64 + ) + mkdir -p "${QEMU_BIN_DIR}" + install -m 0755 \ + "${QEMU_SOURCE_DIR}/build/qemu-x86_64" \ + "${QEMU_BIN_DIR}/qemu-x86_64" + - name: Add QEMU to PATH + run: | + QEMU_BIN_DIR="${RUNNER_TEMP}/lance-qemu/8.2.10" + echo "${QEMU_BIN_DIR}" >> "${GITHUB_PATH}" + QEMU_VERSION_OUTPUT="$("${QEMU_BIN_DIR}/qemu-x86_64" --version)" + echo "${QEMU_VERSION_OUTPUT}" + if [[ "${QEMU_VERSION_OUTPUT}" != *"version 8.2.10"* ]]; then + echo "Expected QEMU 8.2.10, got: ${QEMU_VERSION_OUTPUT}" >&2 + exit 1 + fi + - name: Build lance-linalg lib tests in release mode + run: | + cargo test --release -p lance-linalg --lib --no-run + - name: Run lance-linalg lib tests under qemu Nehalem + run: | + cargo test --release -p lance-linalg --lib msrv: # Check the minimum supported Rust version @@ -269,10 +371,10 @@ jobs: with: submodules: true - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler libssl-dev + - name: Install protoc + uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc + with: + tool: protoc - name: Install ${{ matrix.msrv }} run: | rustup toolchain install ${{ matrix.msrv }} @@ -280,5 +382,5 @@ jobs: env: RUSTUP_TOOLCHAIN: ${{ matrix.msrv }} run: | - ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc -e slow_tests | sort | uniq | paste -s -d "," -` + ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v -e protoc | sort | uniq | paste -s -d "," -` cargo check --profile ci --workspace --tests --benches --features ${ALL_FEATURES} diff --git a/.gitignore b/.gitignore index dcc9c5089ff..b916613b9ce 100644 --- a/.gitignore +++ b/.gitignore @@ -105,6 +105,8 @@ test_data/venv **/*.profraw *.lance +!test_data/**/*.lance +!rust/lance-file/test_data/**/*.lance # Pytest benchmarks .benchmarks/ diff --git a/.typos.toml b/.typos.toml index 9285dd52a23..30f5fb4965f 100644 --- a/.typos.toml +++ b/.typos.toml @@ -23,6 +23,7 @@ nprob = "nprobe" extend-exclude = [ "notebooks/*.ipynb", "*_THIRD_PARTY_LICENSES.*", + "rust/lance-file/test_data/exact_versions/*.lance", "rust/lance-tokenizer/src/stop_word_filter/stopwords.rs", ] # If a line ends with # or // and has spellchecker:disable-line, ignore it diff --git a/AGENTS.md b/AGENTS.md index ee9b3b07e9d..23a1b976b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,31 +4,17 @@ Lance is a modern columnar data format optimized for ML workflows and datasets, Also see directory-specific guidelines: [rust/](rust/AGENTS.md) | [python/](python/AGENTS.md) | [java/](java/AGENTS.md) | [protos/](protos/AGENTS.md) | [docs/src/format/](docs/src/format/AGENTS.md) -## Architecture - -Rust workspace with Python and Java bindings: - -- `rust/lance/` - Main library implementing the columnar format -- `rust/lance-core/` - Core types, traits, and utilities -- `rust/lance-arrow/` - Apache Arrow integration layer -- `rust/lance-encoding/` - Data encoding and compression algorithms -- `rust/lance-file/` - File format reading/writing -- `rust/lance-index/` - Vector and scalar indexing -- `rust/lance-io/` - I/O operations and object store integration -- `rust/lance-linalg/` - Linear algebra for vector search -- `rust/lance-table/` - Table format and operations -- `rust/lance-geo/` - Geospatial data support -- `rust/lance-datagen/` - Data generation for tests and benchmarks -- `rust/lance-namespace/` / `rust/lance-namespace-impls/` - Namespace/catalog interfaces -- `rust/lance-test-macros/` / `rust/lance-testing/` - Test infrastructure -- `rust/lance-tools/` - CLI and developer tooling -- `rust/examples/` - Sample binaries and demonstrations -- `rust/compression/bitpacking/` / `rust/compression/fsst/` - Compression codecs -- `rust/lance-datafusion/` - DataFusion integration (built separately) -- `python/` - Python bindings (PyO3/maturin) -- `java/` - Java bindings (JNI) - -Key technical traits: async-first (tokio), Arrow-native, versioned writes with manifest tracking, custom ML-optimized encodings, unified object store interface (local/S3/Azure/GCS). +## File Format Stability and Compatibility + +- Treat every file format marked stable as a durable compatibility contract. All changes to a stable format must preserve both backward and forward compatibility. +- Treat every file format marked unstable as disposable. It may change freely; do not add compatibility code, migrations, fallbacks, or tests for files written by earlier unstable revisions. +- Evaluate compatibility against the latest released stable version while continuing to honor all stable format contracts. Changes that exist only on the current branch or `main` are not compatibility constraints; do not compromise a cleaner or more complete design to preserve those intermediate states. + +### Legacy Compatibility Boundaries + +- Treat formats and code paths that current writers no longer emit as frozen compatibility surfaces. Preserve their existing read behavior, but exclude them from new feature design unless legacy support is explicitly required. +- Implement new features in the current format and write paths. Do not extend legacy writers, retrofit new capabilities into legacy readers, or reuse legacy implementations as the foundation for new code. +- Avoid refactoring or otherwise modifying legacy code during feature work. If a shared boundary makes a legacy change unavoidable, isolate the change, preserve existing behavior, and add targeted regression coverage using released historical fixtures. ## Development Commands @@ -45,23 +31,12 @@ Key technical traits: async-first (tokio), Arrow-native, versioned writes with m * Use `release-with-debug` for benchmarks and profiling so optimized builds keep debug symbols without a rebuild. * Use `release-no-lto` only for local debugging, IO-bound benchmarks, or compile-time-sensitive performance investigation where LTO would not affect the measured bottleneck. -### Python / Java - -See [python/AGENTS.md](python/AGENTS.md) and [java/AGENTS.md](java/AGENTS.md). - ## Language-Specific Environment Contract - For language-specific tasks, always follow the environment and command rules in the corresponding subdirectory guide before running build, test, lint, format, or tooling commands. - Do not substitute a different environment manager or toolchain just because a command appears missing, unavailable, or slow. - If a language-specific command fails outside the documented workflow, treat that as an environment usage mistake first. Fix the environment usage, rerun with the prescribed commands, and only then conclude that a dependency or tool is unavailable. -### Integration Testing - -```bash -cd test_data && docker compose up -d -AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.py -``` - ## Coding Standards ### General @@ -70,7 +45,6 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p - Code is for readability, not just execution. Only add meaningful comments and tests. - Comments should explain non-obvious "why" reasoning, not restate what the code does. - Remove debug prints (`println!`, `dbg!`, `print()`) before merging — use `tracing` or logging frameworks. -- Extract logic repeated in 2+ places into a shared helper; inline single-use logic at its call site. - Think carefully before adding a helper: only introduce one when it materially reduces cognitive load or eliminates substantial duplication, and do not add thin wrappers that only rename or forward existing calls. - Keep PRs focused — no drive-by refactors, reformatting, or cosmetic changes. - Be mindful of memory use: avoid collecting streams of `RecordBatch` into memory; use `RoaringBitmap` instead of `HashSet`. @@ -106,6 +80,7 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p ## Testing Standards - **All bugfixes and features must have corresponding tests. We do not merge code without tests.** +- Keep local unit tests lightweight: each test case should finish within one second on typical developer hardware. Split independent parameter matrices and use the smallest fixture or model that preserves the asserted behavior; do not relax assertions, coverage, or recall thresholds to meet the budget. - Use `rstest` (Rust) or `@pytest.mark.parametrize` (Python) for tests that differ only in inputs. Use `#[case::{name}(...)]` for readable case names. - Replace `print()` in tests with `assert` — prints don't catch regressions. - Extend existing tests instead of adding overlapping new ones. Add to existing test files. @@ -134,8 +109,14 @@ AWS_DEFAULT_REGION=us-east-1 pytest --run-integration python/tests/test_s3_ddb.p - Indent content under MkDocs admonition directives (`!!! note`, etc.) with 4 spaces. - Proofread comments and docs for typos before committing. +## Filing Issues + +- When opening an issue with `gh issue create` or the API, classify it and pass the matching label: `--label bug`, `--label feature`, or `--label performance`. These paths bypass the `.github/ISSUE_TEMPLATE` forms, so the label is not applied automatically. +- Prefix the title to match, e.g. `bug: ...`, `feature: ...`, or `perf: ...`. A content-based labeler (`.github/workflows/issue-labeler.yml`) uses this as a fallback signal, but an explicit `--label` is the reliable path. + ## Pull Requests +- Before creating a PR, search for similar PRs and inspect any PRs linked to the issue being addressed. If a matching PR exists, verify its current status and scope before proceeding to avoid creating duplicate work. - PR titles must follow the Conventional Commits specification because `.github/workflows/pr-title.yml` validates the PR title and body with commitlint. Use prefixes like `feat:`, `fix:`, `docs:`, `perf:`, `ci:`, `test:`, `build:`, `style:`, or `chore:`; add a scope when useful. - Before creating or updating a PR, run the lint checks for every touched language surface, even when they are expensive. For Rust changes, run `cargo fmt --all` and `cargo clippy --all --tests --benches -- -D warnings`. For Python changes, follow the environment workflow in `python/AGENTS.md` and run `uv run make lint` from `python/`. If a required lint check cannot be run, state the blocker explicitly in the PR summary. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f3ec285f31..0c946ba2a48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Guide for New Contributors This is a guide for new contributors to the Lance project. -Even if you have no previous experience with python, rust, and open source, you can still make an non-trivial +Even if you have no previous experience with python, rust, and open source, you can still make a non-trivial impact by helping us improve documentation, examples, and more. For experienced developers, the issues you can work on run the gamut from warm-ups to serious challenges in python and rust. @@ -11,7 +11,7 @@ If you have any questions, please join our [Discord](https://discord.gg/zMM32dvN 1. Join our Discord and say hi 2. Setup your development environment -3. Pick an issue to work on. See https://github.com/lancedb/lance/contribute for good first issues. +3. Pick an issue to work on. See https://github.com/lance-format/lance/contribute for good first issues. 4. Have fun! ## Development Environment @@ -20,18 +20,28 @@ Currently Lance is implemented in Rust and comes with a Python wrapper. So you'l 1. Install Rust: https://www.rust-lang.org/tools/install 2. Install Python 3.10+: https://www.python.org/downloads/ -3. Install protoctol buffers: https://grpc.io/docs/protoc-installation/ (make sure you have version 3.20 or higher) +3. Install protocol buffers: https://grpc.io/docs/protoc-installation/ (make sure you have version 3.20 or higher) 4. Install commit hooks: a. Install pre-commit: https://pre-commit.com/#install b. Run `pre-commit install` in the root of the repo +## Building for legacy x86_64 hosts (pre-Haswell) + +The default workspace build targets `haswell` (AVX2 + FMA + F16C), matching the published wheels. To build a binary that runs on pre-Haswell silicon (Sandy Bridge / Ivy Bridge / Westmere on Intel, Bulldozer / Piledriver / Steamroller on AMD — i.e. CPUs without AVX2), set the baseline yourself at build time: + +```sh +RUSTFLAGS="-C target-cpu=x86-64-v2" cargo build --release +``` + +Runtime SIMD dispatch in `lance-linalg::distance` will then pick the appropriate tier (scalar / AVX / AVX+FMA / AVX2+FMA / AVX-512) based on the host. From Python, use `lance.simd_info()` to verify which tier was selected. + ## Sample Workflow 1. Fork the repo -2. Pick [Github issue](https://github.com/lancedb/lance/issues) +2. Pick [Github issue](https://github.com/lance-format/lance/issues) 3. Create a branch for the issue 4. Make your changes -5. Create a pull request from your fork to lancedb/lance +5. Create a pull request from your fork to lance-format/lance 6. Get feedback and iterate 7. Merge! 8. Go back to step 2 diff --git a/Cargo.lock b/Cargo.lock index f537f382479..41b73d96d47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,9 +148,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -205,9 +205,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -219,9 +219,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -250,9 +250,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -300,9 +300,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -354,9 +354,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -367,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -378,9 +378,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -460,18 +460,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -647,7 +647,7 @@ dependencies = [ "http 0.2.12", "http 1.4.2", "http-body 1.0.1", - "lru", + "lru 0.16.4", "percent-encoding", "regex-lite", "sha2 0.10.9", @@ -1052,6 +1052,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1225,7 +1231,7 @@ checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1236,9 +1242,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "byteorder" @@ -1248,9 +1268,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1279,9 +1299,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -1390,9 +1410,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1400,9 +1420,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1412,14 +1432,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -1656,16 +1676,7 @@ dependencies = [ "crc", "digest 0.10.7", "rustversion", - "spin 0.10.0", -] - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", + "spin 0.10.1", ] [[package]] @@ -1734,18 +1745,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1908,7 +1919,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1921,7 +1932,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1943,7 +1954,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1954,7 +1965,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1973,14 +1984,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -2007,12 +2017,11 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "log", "object_store", "parking_lot", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -2022,9 +2031,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -2047,9 +2056,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -2070,32 +2079,33 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", "object_store", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -2104,9 +2114,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-trait", @@ -2126,16 +2136,17 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.4", + "parking_lot", + "rand 0.9.5", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -2157,9 +2168,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -2180,9 +2191,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2197,27 +2208,25 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -2226,18 +2235,19 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2248,29 +2258,27 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", @@ -2285,26 +2293,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -2314,19 +2321,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2335,9 +2341,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2351,34 +2357,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2389,14 +2395,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2404,20 +2409,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2434,11 +2439,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2446,20 +2450,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2472,26 +2475,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2507,12 +2510,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2527,7 +2531,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2539,9 +2543,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2550,15 +2554,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2570,9 +2573,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2588,9 +2591,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -2650,7 +2653,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -2693,7 +2695,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2713,7 +2715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core 0.20.2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2775,7 +2777,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2860,6 +2862,12 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "env_filter" version = "2.0.0" @@ -2900,7 +2908,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2936,11 +2944,10 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -3068,6 +3075,12 @@ dependencies = [ "futures-core", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -3076,10 +3089,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", - "rand 0.9.4", + "rand 0.9.5", "test-log", "tokio", ] @@ -3155,7 +3168,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3312,12 +3325,13 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-schema", "datafusion", "geo", @@ -3369,11 +3383,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if 1.0.4", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -3413,7 +3425,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3436,17 +3448,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru 0.12.5", + "memmap2", + "moka", "prost", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "thiserror 2.0.18", @@ -3456,13 +3476,14 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -3483,6 +3504,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if 1.0.4", "crunchy", "num-traits", @@ -3516,6 +3538,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -3535,6 +3559,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapify" @@ -3584,7 +3613,7 @@ dependencies = [ "log", "native-tls", "num_cpus", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "serde_json", @@ -3714,9 +3743,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -4198,18 +4227,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jieba-macros" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5" dependencies = [ "phf_codegen", ] [[package]] name = "jieba-rs" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d" dependencies = [ "bytecount", "cedarwood", @@ -4244,7 +4273,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4289,7 +4318,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4308,7 +4337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4346,7 +4375,7 @@ dependencies = [ "nom 8.0.0", "num-traits", "ordered-float 5.3.0", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "zmij", @@ -4380,7 +4409,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "all_asserts", "approx", @@ -4457,7 +4486,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rayon", "reqwest 0.12.28", "roaring", @@ -4483,7 +4512,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", @@ -4492,13 +4521,14 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", + "bytemuck", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -4531,7 +4561,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrayref", "bitpacking", @@ -4542,19 +4572,19 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", - "byteorder", + "blake3", "bytes", + "criterion", "datafusion-common", "datafusion-sql", "futures", - "itertools 0.14.0", "lance-arrow", "lance-derive", "libc", @@ -4564,9 +4594,9 @@ dependencies = [ "num_cpus", "object_store", "pin-project", - "proptest", "prost", - "rand 0.9.4", + "quick_cache", + "rand 0.9.5", "roaring", "rstest", "serde_json", @@ -4582,7 +4612,7 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4599,6 +4629,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-substrait", "futures", + "half", "jsonb", "lance-arrow", "lance-core", @@ -4609,13 +4640,14 @@ dependencies = [ "prost", "prost-build", "protobuf-src", + "rstest", "tokio", "tracing", ] [[package]] name = "lance-datagen" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4627,23 +4659,24 @@ dependencies = [ "half", "hex", "lance-testing", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rand_xoshiro", + "rstest", ] [[package]] name = "lance-derive" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "lance-encoding" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-arith", "arrow-array", @@ -4674,11 +4707,10 @@ dependencies = [ "prost", "prost-build", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rand_xoshiro", "rstest", "serial_test", - "strum 0.26.3", "test-log", "tokio", "tracing", @@ -4688,7 +4720,7 @@ dependencies = [ [[package]] name = "lance-examples" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "all_asserts", "arrow", @@ -4706,7 +4738,7 @@ dependencies = [ "lance-linalg", "object_store", "parquet", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "tokenizers", "tokio", @@ -4714,11 +4746,12 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -4745,6 +4778,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", + "rand 0.9.5", "rstest", "test-log", "tokio", @@ -4753,7 +4787,7 @@ dependencies = [ [[package]] name = "lance-geo" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "datafusion", "geo-traits", @@ -4767,13 +4801,14 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "approx", "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -4810,12 +4845,14 @@ dependencies = [ "lance-encoding", "lance-file", "lance-geo", + "lance-index-core", "lance-io", "lance-linalg", "lance-select", "lance-table", "lance-testing", "lance-tokenizer", + "libc", "libsais-rs", "log", "ndarray", @@ -4825,7 +4862,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rangemap", "rayon", @@ -4834,6 +4871,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "serial_test", "smallvec", "tempfile", "test-log", @@ -4842,19 +4880,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "lance-index-core" +version = "12.0.0-beta.11" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", +] + [[package]] name = "lance-io" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4865,11 +4919,12 @@ dependencies = [ "futures", "http 1.4.2", "io-uring", - "lance-arrow", "lance-core", "lance-namespace", "lance-testing", "log", + "metrics", + "metrics-util", "mock_instant", "mockall", "moka", @@ -4879,20 +4934,27 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.4", + "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "rstest", "serde", + "serde_json", + "serial_test", "tempfile", "test-log", "tokio", "tracing", "tracing-mock", "url", + "uuid", + "wiremock", ] [[package]] name = "lance-linalg" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "approx", "arrow-array", @@ -4906,25 +4968,28 @@ dependencies = [ "lance-testing", "num-traits", "proptest", - "rand 0.9.4", + "rand 0.9.5", "rayon", + "rstest", ] [[package]] name = "lance-namespace" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", + "serde_json", "snafu", ] [[package]] name = "lance-namespace-datafusion" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-schema", @@ -4940,10 +5005,9 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", - "arrow-array", "arrow-ipc", "arrow-schema", "async-trait", @@ -4958,7 +5022,6 @@ dependencies = [ "futures", "hmac 0.12.1", "lance", - "lance-arrow", "lance-core", "lance-index", "lance-io", @@ -4967,9 +5030,8 @@ dependencies = [ "lance-table", "log", "object_store", - "opendal", - "quick-xml 0.38.4", - "rand 0.9.4", + "quick-xml 0.40.1", + "rand 0.9.5", "reqwest 0.12.28", "ring", "roaring", @@ -4979,7 +5041,6 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tempfile", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -4990,9 +5051,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.8.6" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af" dependencies = [ "reqwest 0.12.28", "serde", @@ -5004,25 +5065,23 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", "byteorder", - "bytes", "criterion", "itertools 0.14.0", "lance-core", "proptest", "roaring", - "rstest", "tracing", ] [[package]] name = "lance-table" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -5032,6 +5091,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-dynamodb", + "blake3", "byteorder", "bytes", "chrono", @@ -5052,7 +5112,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "roaring", "rstest", @@ -5068,16 +5128,16 @@ dependencies = [ [[package]] name = "lance-test-macros" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "lance-testing" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-schema", @@ -5085,17 +5145,17 @@ dependencies = [ "lance-arrow", "num-traits", "pprof", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -5103,7 +5163,7 @@ dependencies = [ [[package]] name = "lance-tools" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "clap", "lance-core", @@ -5120,7 +5180,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin 0.9.8", + "spin 0.9.9", ] [[package]] @@ -5182,9 +5242,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -5229,8 +5289,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "unicode-blocks", "unicode-normalization", "unicode-segmentation", @@ -5259,8 +5319,8 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "thiserror 2.0.18", ] @@ -5316,6 +5376,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru" version = "0.16.4" @@ -5451,13 +5520,43 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "metrics", + "ordered-float 4.6.0", + "quanta", + "radix_trie", + "rand 0.9.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -5530,7 +5629,7 @@ dependencies = [ "cfg-if 1.0.4", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5572,7 +5671,7 @@ checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5604,7 +5703,7 @@ checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5639,6 +5738,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.26.4" @@ -5797,7 +5905,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5885,9 +5993,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -5948,12 +6056,13 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -5971,24 +6080,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64 0.22.1", + "base64 0.23.0", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -5997,11 +6104,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.4.2", @@ -6011,9 +6132,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -6021,9 +6142,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -6032,9 +6153,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -6042,17 +6163,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6063,17 +6184,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -6083,9 +6205,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.4.2", "opendal-core", @@ -6093,15 +6215,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -6110,9 +6232,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -6120,7 +6242,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -6131,9 +6253,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -6145,9 +6267,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -6155,22 +6277,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -6179,18 +6300,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64 0.22.1", + "base64 0.23.0", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -6200,14 +6321,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "7841a1a09485bd08eeac34d67804321b62456c855027c580bce12a75564f4609" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -6237,7 +6358,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6273,6 +6394,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.3.0" @@ -6348,9 +6478,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", "arrow-array", @@ -6557,7 +6687,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6692,7 +6822,7 @@ dependencies = [ "nix", "once_cell", "smallvec", - "spin 0.10.0", + "spin 0.10.1", "symbolic-demangle", "tempfile", "thiserror 2.0.18", @@ -6750,7 +6880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6764,9 +6894,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -6781,7 +6911,7 @@ dependencies = [ "bit-vec", "bitflags 2.13.0", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -6815,7 +6945,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.118", + "syn 2.0.119", "tempfile", ] @@ -6829,7 +6959,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6867,7 +6997,22 @@ checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", ] [[package]] @@ -6887,33 +7032,45 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", + "serde", ] [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", - "serde", ] [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6936,15 +7093,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -6972,9 +7130,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -6997,6 +7155,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rancor" version = "0.1.1" @@ -7019,9 +7187,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -7089,7 +7257,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -7116,6 +7293,24 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -7199,7 +7394,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7264,9 +7459,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f" dependencies = [ "anyhow", "form_urlencoded", @@ -7281,9 +7476,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2" dependencies = [ "anyhow", "bytes", @@ -7292,7 +7487,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", @@ -7303,9 +7498,9 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" dependencies = [ "anyhow", "base64 0.22.1", @@ -7324,14 +7519,13 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb" dependencies = [ "anyhow", "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -7349,9 +7543,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b" dependencies = [ "anyhow", "reqsign-core", @@ -7360,9 +7554,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -7378,9 +7572,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644" dependencies = [ "anyhow", "http 1.4.2", @@ -7393,9 +7587,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "4335f949a3fd8b53867fd716dac97fbd545e45bcaed44343796517f2e19cd609" dependencies = [ "anyhow", "http 1.4.2", @@ -7532,9 +7726,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck", "bytes", @@ -7551,13 +7745,13 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7633,7 +7827,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", "unicode-ident", ] @@ -7647,16 +7841,6 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-demangle" version = "0.1.27" @@ -7665,9 +7849,9 @@ checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -7870,7 +8054,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7937,9 +8121,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -7947,22 +8131,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -7973,15 +8157,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -8008,7 +8193,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8020,7 +8205,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8064,7 +8249,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8115,7 +8300,7 @@ checksum = "94e153fc76e1c6a068703d6d29c508a0b15c061c4b7e43da59cc097bc342673c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8246,6 +8431,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -8260,23 +8451,23 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8314,15 +8505,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" dependencies = [ "lock_api", ] @@ -8351,9 +8542,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -8367,7 +8558,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8431,35 +8622,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.118", + "strum_macros", ] [[package]] @@ -8471,16 +8640,17 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -8494,7 +8664,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.118", + "syn 2.0.119", "typify", "walkdir", ] @@ -8547,9 +8717,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -8573,7 +8754,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8661,7 +8842,7 @@ checksum = "c26ef8b00e4d382e59f6a8ddb3cd790b3a5bb29f21a358a9a69ea2f29f13f27b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8670,7 +8851,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "944ad38adcbb71eaa682c56bceeb079e4ca82b4b3edc2a0fde5cb297b77dac8d" dependencies = [ - "syn 2.0.118", + "syn 2.0.119", "test-log-core", ] @@ -8700,7 +8881,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8711,7 +8892,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8745,12 +8926,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -8760,15 +8940,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -8854,9 +9034,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -8877,7 +9057,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -8913,9 +9093,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -8925,13 +9105,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -9109,7 +9290,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9203,11 +9384,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] [[package]] @@ -9247,7 +9428,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.118", + "syn 2.0.119", "thiserror 2.0.18", "unicode-ident", ] @@ -9265,7 +9446,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.118", + "syn 2.0.119", "typify-impl", ] @@ -9399,9 +9580,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -9536,7 +9717,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -9719,7 +9900,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -9730,7 +9911,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10185,9 +10366,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yansi" @@ -10214,7 +10395,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10235,7 +10416,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -10255,7 +10436,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "synstructure", ] @@ -10297,7 +10478,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 486aa1e56a2..2f6046faa05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "rust/lance-file", "rust/lance-geo", "rust/lance-index", + "rust/lance-index-core", "rust/lance-io", "rust/lance-linalg", "rust/lance-namespace", @@ -32,7 +33,7 @@ resolver = "3" [workspace.package] -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" edition = "2024" authors = ["Lance Devs "] license = "Apache-2.0" @@ -57,27 +58,28 @@ rust-version = "1.91.0" [workspace.dependencies] arc-swap = "1.7" libc = "0.2.176" -lance = { version = "=9.0.0-beta.16", path = "./rust/lance", default-features = false } -lance-arrow = { version = "=9.0.0-beta.16", path = "./rust/lance-arrow" } -lance-core = { version = "=9.0.0-beta.16", path = "./rust/lance-core" } -lance-datafusion = { version = "=9.0.0-beta.16", path = "./rust/lance-datafusion" } -lance-datagen = { version = "=9.0.0-beta.16", path = "./rust/lance-datagen" } -lance-derive = { version = "=9.0.0-beta.16", path = "./rust/lance-derive" } -lance-encoding = { version = "=9.0.0-beta.16", path = "./rust/lance-encoding" } -lance-file = { version = "=9.0.0-beta.16", path = "./rust/lance-file" } -lance-geo = { version = "=9.0.0-beta.16", path = "./rust/lance-geo" } -lance-index = { version = "=9.0.0-beta.16", path = "./rust/lance-index" } -lance-io = { version = "=9.0.0-beta.16", path = "./rust/lance-io", default-features = false } -lance-linalg = { version = "=9.0.0-beta.16", path = "./rust/lance-linalg" } -lance-namespace = { version = "=9.0.0-beta.16", path = "./rust/lance-namespace" } -lance-namespace-impls = { version = "=9.0.0-beta.16", path = "./rust/lance-namespace-impls" } -lance-namespace-datafusion = { version = "=7.0.0-beta.9", path = "./rust/lance-namespace-datafusion" } -lance-namespace-reqwest-client = "0.8.6" -lance-select = { version = "=9.0.0-beta.16", path = "./rust/lance-select" } -lance-tokenizer = { version = "=9.0.0-beta.16", path = "./rust/lance-tokenizer" } -lance-table = { version = "=9.0.0-beta.16", path = "./rust/lance-table" } -lance-test-macros = { version = "=9.0.0-beta.16", path = "./rust/lance-test-macros" } -lance-testing = { version = "=9.0.0-beta.16", path = "./rust/lance-testing" } +lance = { version = "=12.0.0-beta.11", path = "./rust/lance", default-features = false } +lance-arrow = { version = "=12.0.0-beta.11", path = "./rust/lance-arrow" } +lance-core = { version = "=12.0.0-beta.11", path = "./rust/lance-core" } +lance-datafusion = { version = "=12.0.0-beta.11", path = "./rust/lance-datafusion" } +lance-datagen = { version = "=12.0.0-beta.11", path = "./rust/lance-datagen" } +lance-derive = { version = "=12.0.0-beta.11", path = "./rust/lance-derive" } +lance-encoding = { version = "=12.0.0-beta.11", path = "./rust/lance-encoding" } +lance-file = { version = "=12.0.0-beta.11", path = "./rust/lance-file" } +lance-geo = { version = "=12.0.0-beta.11", path = "./rust/lance-geo" } +lance-index = { version = "=12.0.0-beta.11", path = "./rust/lance-index" } +lance-index-core = { version = "=12.0.0-beta.11", path = "./rust/lance-index-core" } +lance-io = { version = "=12.0.0-beta.11", path = "./rust/lance-io", default-features = false } +lance-linalg = { version = "=12.0.0-beta.11", path = "./rust/lance-linalg" } +lance-namespace = { version = "=12.0.0-beta.11", path = "./rust/lance-namespace" } +lance-namespace-impls = { version = "=12.0.0-beta.11", path = "./rust/lance-namespace-impls" } +lance-namespace-reqwest-client = "0.12.0" +lance-select = { version = "=12.0.0-beta.11", path = "./rust/lance-select" } +lance-tokenizer = { version = "=12.0.0-beta.11", path = "./rust/lance-tokenizer" } +lance-table = { version = "=12.0.0-beta.11", path = "./rust/lance-table" } +lance-test-macros = { version = "=12.0.0-beta.11", path = "./rust/lance-test-macros" } +lance-testing = { version = "=12.0.0-beta.11", path = "./rust/lance-testing" } +all_asserts = "2.3.1" approx = "0.5.1" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false, features = ["prettyprint"] } @@ -103,10 +105,15 @@ aws-sdk-s3 = { version = "1.38.0", default-features = false } half = { "version" = "2.1", default-features = false, features = [ "num-traits", "std", + "bytemuck", ] } -lance-bitpacking = { version = "=9.0.0-beta.16", path = "./rust/compression/bitpacking" } +lance-bitpacking = { version = "=12.0.0-beta.11", path = "./rust/compression/bitpacking" } bitpacking = "0.9" bitvec = "1" +blake3 = "1.8.5" +bytemuck = { version = "1", default-features = false, features = [ + "extern_crate_alloc", +] } bytes = "1.11.1" byteorder = "1.5" clap = { version = "4", features = ["derive"] } @@ -122,7 +129,8 @@ criterion = { version = "0.8.2", features = [ ] } crossbeam-queue = "0.3" crossbeam-skiplist = "0.1" -datafusion = { version = "53.0.0", default-features = false, features = [ +dashmap = "6" +datafusion = { version = "54.0.0", default-features = false, features = [ "crypto_expressions", "datetime_expressions", "encoding_expressions", @@ -132,24 +140,25 @@ datafusion = { version = "53.0.0", default-features = false, features = [ "string_expressions", "unicode_expressions", ] } -datafusion-common = "53.0.0" -datafusion-functions = { version = "53.0.0", default-features = false, features = ["regex_expressions"] } -datafusion-sql = "53.0.0" -datafusion-expr = "53.0.0" -datafusion-ffi = "53.0.0" -datafusion-physical-expr = "53.0.0" -datafusion-physical-plan = "53.0.0" -datafusion-substrait = { version = "53.0.0", default-features = false } +datafusion-common = "54.0.0" +datafusion-functions = { version = "54.0.0", default-features = false, features = ["regex_expressions"] } +datafusion-sql = "54.0.0" +datafusion-expr = "54.0.0" +datafusion-physical-expr = "54.0.0" +datafusion-physical-plan = "54.0.0" +datafusion-substrait = { version = "54.0.0", default-features = false } dirs = "6.0.0" either = "1.0" +env_logger = "0.11.7" fst = { version = "0.4.7", features = ["levenshtein"] } -fsst = { version = "=9.0.0-beta.16", path = "./rust/compression/fsst" } +fsst = { version = "=12.0.0-beta.11", path = "./rust/compression/fsst" } futures = "0.3" geoarrow-array = "0.8" geoarrow-schema = "0.8" -geodatafusion = "0.4.0" +geodatafusion = "0.5.0" geo-traits = "0.3.0" geo-types = "0.7.16" +hex = "0.4.3" http = "1.1.0" humantime = "2.2.0" hyperloglogplus = { version = "0.4.1", features = ["const-loop"] } @@ -160,27 +169,37 @@ jieba-rs = { version = "0.10.0", default-features = false } jsonb = { version = "0.5.3", default-features = false, features = ["databend"] } libm = "0.2.15" log = "0.4" +metrics = { version = "0.24" } +metrics-util = { version = "0.19" } mockall = { version = "0.14.0" } mock_instant = { version = "0.6.0" } moka = { version = "0.12", features = ["future", "sync"] } ndarray = { version = "0.16.1", features = ["matrixmultiply-threading"] } num-traits = "0.2" object_store = { version = "0.13.2" } -opendal = { version = "0.57" } -object_store_opendal = { version = "0.57" } +opendal = { version = "0.58.1" } +object_store_opendal = { version = "0.58" } +parquet = { version = "58.0.0", default-features = false, features = [ + "arrow", + "async", +] } pin-project = "1.0" path_abs = "0.5" pprof = { version = "0.15.0", features = ["flamegraph"] } +proc-macro2 = "1.0.67" proptest = "1.3.1" prost = "0.14.1" prost-build = "0.14.1" prost-types = "0.14.1" +protobuf-src = "2.1" +quote = "1.0.33" rand = { version = "0.9.1", features = ["small_rng"] } rand_distr = { version = "0.5.1" } rand_xoshiro = "0.7.0" rangemap = { version = "1.0" } rayon = "1.10" regex-syntax = "0.8.10" +reqwest = { version = "0.12", default-features = false, features = ["json"] } roaring = "0.11.4" rstest = "0.26.1" serde = { version = "^1" } @@ -188,9 +207,9 @@ serde_json = { version = "1" } semver = "1.0" serial_test = "3" snafu = "0.9" -strum = "0.26" +syn = { version = "2.0.37", features = ["full"] } lindera = { version = "3.0.7" } -tempfile = "3" +tempfile = "3.10" test-log = { version = "0.2.15" } tokio = { version = "1.23", features = [ "rt-multi-thread", diff --git a/README.md b/README.md index 886fd70425e..08716c84ead 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,17 @@ For more details, see the full [Lance format specification](https://lance.org/fo > [!TIP] > Lance is in active development and we welcome contributions. Please see our [contributing guide](https://lance.org/community/contributing/) for more information. +## File format stability + +Lance releases frequently because the SDKs, integrations, and performance work are moving quickly. This does not mean the Lance file format changes incompatibly in every release. The Lance file format is identified by the `data_storage_version` stored in each dataset, and stable storage versions are a long-term compatibility contract. + +* Once a dataset is written with a stable `data_storage_version`, future Lance releases will continue to support reading that storage version. +* SDK and API compatibility is separate from file format compatibility. SDK/API changes follow semantic versioning and are documented in the [migration guide](https://lance.org/guide/migration/). +* Older Lance releases may not understand file format versions introduced later. If you run mixed Lance versions, pin `data_storage_version` for deterministic writes. +* The `next` file format alias is unstable and should only be used for experimentation, never for production data. + +For production, write data with a stable `data_storage_version`. See the [format versioning guide](https://lance.org/format/file/versioning/) for the current compatibility matrix. + ## Quick Start **Installation** diff --git a/ci/format_vote_gate.py b/ci/format_vote_gate.py new file mode 100644 index 00000000000..f4ed1a972d0 --- /dev/null +++ b/ci/format_vote_gate.py @@ -0,0 +1,385 @@ +"""Format-specification vote gate (see `.github/workflows/format-vote-gate.yml`). + +Structurally enforces the PMC vote required for Lance format-specification +changes (https://lance.org/community/voting/). The `format-change` label is +applied by the path labeler (`.github/labeler-area.yml`); this script reads it +and publishes the `format-spec-vote` commit status, which blocks merging until: + + * 3 PMC members have approved the PR (excluding the author), counted only on + the head commit so new pushes invalidate stale approvals; + * no PMC member has an outstanding "Request changes" review (a veto); and + * the 72-hour voting period has elapsed. The clock starts once the PR is both + labeled and out of draft, and pauses over weekends. + +A PMC member can waive a trivial edit by applying the `format-waived` label. +Non-format PRs get a passing status immediately and are otherwise left alone. +Drafts get a blocking status but no comment: the vote has not opened yet. + +The vote-counting and deadline rules are pure functions (`tally_reviews`, +`decide_verdict`, `vote_opened_at`, `weekday_deadline`) unit tested in +`test_format_vote_gate.py`; `main` wires them to the GitHub API. +""" + +import json +import os +from collections import namedtuple +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +STATUS_CONTEXT = "format-spec-vote" +FORMAT_LABEL = "format-change" +WAIVED_LABEL = "format-waived" +COMMENT_MARKER = "" +REQUIRED_APPROVALS = 3 +PERIOD_HOURS = 72 +VOTING_URL = "https://lance.org/community/voting/" +WORKFLOW_FILE = "format-vote-gate.yml" +# Keep in sync with the `cron` in the workflow; only used in the comment text. +SWEEP_MINUTES = 15 + +# The weekend boundary is fixed in UTC rather than a local zone: it has no DST +# transitions to reason about, and no PMC member's timezone gets to define when +# everyone else's clock pauses. Deadlines are *displayed* in UTC and Pacific. +WEEKEND_TZ = timezone.utc +DISPLAY_TZ = ZoneInfo("America/Los_Angeles") + +# datetime.weekday() numbers Monday 0 .. Sunday 6, so the weekend is >= 5. +_SATURDAY = 5 + +# Review states that express a stance; COMMENTED/PENDING are ignored. +_STANCE_STATES = ("APPROVED", "CHANGES_REQUESTED", "DISMISSED") + +TimelineFacts = namedtuple("TimelineFacts", "labeled_at waived ready_at") + + +def tally_reviews(reviews, head_sha, author, is_pmc): + """Tally PMC votes from a PR's reviews. + + `reviews` is an ordered list of dicts with `login`, `state`, `commit_id`. + A member's stance is their most recent stance review. Approvals only count + on the head commit; earlier ones are stale. A "changes requested" review is + a veto regardless of commit. The PR author never counts. + """ + latest = {} + for review in reviews: + login = review["login"] + if not login or not is_pmc(login) or login == author: + continue + if review["state"] not in _STANCE_STATES: + continue + latest[login.lower()] = review + + approvals, stale_approvals, vetoes = [], [], [] + for review in latest.values(): + if review["state"] == "APPROVED": + target = approvals if review["commit_id"] == head_sha else stale_approvals + target.append(review["login"]) + elif review["state"] == "CHANGES_REQUESTED": + vetoes.append(review["login"]) + return approvals, stale_approvals, vetoes + + +def decide_verdict(veto_count, approval_count, period_elapsed, required): + """Return the blocking condition (if any), in priority order.""" + if veto_count > 0: + return "veto" + if approval_count < required: + return "insufficient" + if not period_elapsed: + return "waiting_period" + return "pass" + + +def vote_opened_at(labeled_at, ready_at): + """When the voting period starts, or None if it hasn't. + + A vote opens only once the proposal is both identified as a format change + and offered for review, so the clock starts at the later of the two. A draft + is still being drafted; time spent there shouldn't count toward the period. + """ + if labeled_at is None or ready_at is None: + return None + return max(labeled_at, ready_at) + + +def _start_of_day(dt): + return dt.replace(hour=0, minute=0, second=0, microsecond=0) + + +# Advance to Monday 00:00 if `dt` lands on a weekend; otherwise leave it alone. +def _skip_weekend(dt): + while dt.weekday() >= _SATURDAY: + dt = _start_of_day(dt) + timedelta(days=1) + return dt + + +# Saturday 00:00 following `dt`, which must already be a weekday. +def _next_weekend(dt): + return _start_of_day(dt) + timedelta(days=_SATURDAY - dt.weekday()) + + +def weekday_deadline(start, hours): + """When `hours` of non-weekend time have elapsed after `start`. + + Weekends don't count toward the voting period, so a proposal opened on a + Friday afternoon doesn't burn most of its period while nobody is reading it. + Both `start` and the result are aware datetimes; the arithmetic happens in + `WEEKEND_TZ`, which decides where each weekend begins and ends. + """ + cursor = _skip_weekend(start.astimezone(WEEKEND_TZ)) + remaining = timedelta(hours=hours) + while True: + until_weekend = _next_weekend(cursor) - cursor + if remaining <= until_weekend: + return cursor + remaining + remaining -= until_weekend + cursor = _skip_weekend(_next_weekend(cursor)) + + +def _fmt_list(logins): + return ", ".join(f"@{login}" for login in logins) if logins else "none" + + +# Renders as `Wed 2026-08-05 17:00 UTC (10:00 PDT)` — the PMC spans both zones. +# The Pacific weekday is spelled out only when the deadline falls on a different +# day there, which is the case that actually trips people up. +def _fmt_deadline(dt): + local = dt.astimezone(DISPLAY_TZ) + local_day = "" if local.date() == dt.date() else f"{local:%a }" + return f"{dt:%a %Y-%m-%d %H:%M} UTC ({local_day}{local:%H:%M %Z})" + + +def _as_utc(dt): + return dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt + + +def _build_comment(headline, approval_cell, vetoes, period_cell, rerun_url): + return "\n".join( + [ + COMMENT_MARKER, + "> [!IMPORTANT]", + "> ## Format specification vote", + "", + "This PR modifies the Lance format specification, so it requires " + f"**{REQUIRED_APPROVALS} binding +1 votes from PMC members** " + "(excluding the proposer) and a minimum " + f"**{PERIOD_HOURS}-hour** voting period, weekends excluded, before " + "it can merge. " + "Vote by approving this PR (+1) or requesting changes (−1, a veto). " + f"See the [voting process]({VOTING_URL}).", + "", + f"**Status: {headline}**", + "", + "| | |", + "|---|---|", + f"| Approvals (this commit) | {approval_cell} |", + f"| Vetoes | {_fmt_list(vetoes)} |", + f"| Voting period | {period_cell} |", + "", + "Updated automatically by the format-spec vote gate, which " + f"re-checks every {SWEEP_MINUTES} minutes — just voted? " + f"[Re-check now]({rerun_url}) (press Run workflow; leave the input " + "blank to re-check every open format PR). A PMC member may apply " + f"the `{WAIVED_LABEL}` label to waive the vote for a trivial edit " + "(typo, wording, formatting).", + ] + ) + + +def _load_pmc(workspace): + import yaml + + roster_path = os.path.join(workspace, "docs", "src", "community", "pmc.yaml") + with open(roster_path) as handle: + roster = yaml.safe_load(handle) + return {member["handle"].lower() for member in roster["members"]} + + +class Gate: + def __init__(self, repo, pmc, run_url, rerun_url): + self.repo = repo + self.pmc = pmc + self.run_url = run_url + self.rerun_url = rerun_url + + def is_pmc(self, login): + return login is not None and login.lower() in self.pmc + + def set_status(self, sha, state, description): + self.repo.get_commit(sha).create_status( + state=state, + context=STATUS_CONTEXT, + description=description[:140], + target_url=self.run_url, + ) + + def upsert_comment(self, issue, body): + for comment in issue.get_comments(): + if COMMENT_MARKER in (comment.body or ""): + if comment.body != body: + comment.edit(body) + return + issue.create_comment(body) + + def timeline_facts(self, issue): + """Read the vote-clock inputs off the PR timeline in one pass.""" + labeled_at = None + waived_by_pmc = False + ready_at = None + for event in issue.get_events(): + actor = event.actor.login if event.actor else None + # A PR converted back to draft and re-opened for review restarts the + # clock, so the *last* ready_for_review wins. + if event.event == "ready_for_review": + ready_at = _as_utc(event.created_at) + continue + if event.event != "labeled" or event.label is None: + continue + if event.label.name == FORMAT_LABEL and labeled_at is None: + labeled_at = _as_utc(event.created_at) + elif event.label.name == WAIVED_LABEL and self.is_pmc(actor): + waived_by_pmc = True + return TimelineFacts(labeled_at, waived_by_pmc, ready_at) + + def evaluate(self, number): + pr = self.repo.get_pull(number) + if pr.state != "open": + print(f"PR #{number} is {pr.state}; skipping.") + return + head_sha = pr.head.sha + labels = {label.name for label in pr.labels} + + # Non-format PRs get a passing status and are otherwise left alone. + if FORMAT_LABEL not in labels: + self.set_status( + head_sha, "success", "No format-spec change; vote not required." + ) + print(f"PR #{number}: not a format change.") + return + + issue = self.repo.get_issue(number) + facts = self.timeline_facts(issue) + + if WAIVED_LABEL in labels and facts.waived: + self.set_status( + head_sha, "success", "Format-spec vote waived by a PMC member." + ) + print(f"PR #{number}: vote waived.") + return + + # Stay quiet on drafts: the proposal isn't up for a vote yet, so there is + # nothing for the PMC to act on and no deadline to announce. + if pr.draft: + self.set_status( + head_sha, + "failure", + "Draft; voting period starts when marked ready for review.", + ) + print(f"PR #{number}: draft, vote not open.") + return + + reviews = [ + { + "login": review.user.login if review.user else None, + "state": review.state, + "commit_id": review.commit_id, + } + for review in pr.get_reviews() + ] + approvals, stale, vetoes = tally_reviews( + reviews, head_sha, pr.user.login, self.is_pmc + ) + + now = datetime.now(timezone.utc) + # `pr.created_at` covers a PR opened ready for review, which never emits a + # ready_for_review event. + opened_at = vote_opened_at( + facts.labeled_at, facts.ready_at or _as_utc(pr.created_at) + ) + period_ends = weekday_deadline(opened_at or now, PERIOD_HOURS) + period_elapsed = now >= period_ends + verdict = decide_verdict( + len(vetoes), len(approvals), period_elapsed, REQUIRED_APPROVALS + ) + + deadline = _fmt_deadline(period_ends) + if verdict == "veto": + state, summary = "failure", f"Vetoed by {len(vetoes)} PMC member(s)." + headline = f"❌ Blocked — vetoed by {_fmt_list(vetoes)}" + elif verdict == "insufficient": + state = "failure" + summary = ( + f"{len(approvals)}/{REQUIRED_APPROVALS} PMC approvals on this commit." + ) + headline = f"❌ Blocked — {len(approvals)} of {REQUIRED_APPROVALS} required approvals" + elif verdict == "waiting_period": + state, summary = "failure", f"Approved; voting period ends {deadline}." + headline = ( + f"⏳ Approvals met ({len(approvals)}/{REQUIRED_APPROVALS}); " + f"voting period ends {deadline}" + ) + else: + state = "success" + summary = f"Passed — {len(approvals)} PMC approvals, period elapsed." + headline = f"✅ Vote passed — {len(approvals)} PMC approvals, voting period elapsed" + + period_cell = ( + f"elapsed — ended {deadline}" if period_elapsed else f"ends {deadline}" + ) + approval_cell = ( + f"{_fmt_list(approvals)} ({len(approvals)}/{REQUIRED_APPROVALS})" + ) + if stale: + approval_cell += f" — stale, re-approve needed: {_fmt_list(stale)}" + + self.set_status(head_sha, state, summary) + self.upsert_comment( + issue, + _build_comment( + headline, approval_cell, vetoes, period_cell, self.rerun_url + ), + ) + print(f"PR #{number}: {summary}") + + +def main(): + from github import Github + + workspace = os.environ["GITHUB_WORKSPACE"] + token = os.environ["GITHUB_TOKEN"] + repo_name = os.environ["GITHUB_REPOSITORY"] + event_name = os.environ["GITHUB_EVENT_NAME"] + actions_url = f"{os.environ['GITHUB_SERVER_URL']}/{repo_name}/actions" + run_url = f"{actions_url}/runs/{os.environ['GITHUB_RUN_ID']}" + rerun_url = f"{actions_url}/workflows/{WORKFLOW_FILE}" + + repo = Github(token).get_repo(repo_name) + gate = Gate(repo, _load_pmc(workspace), run_url, rerun_url) + print(f"Re-check on demand: {rerun_url}") + + if event_name in ("schedule", "workflow_dispatch"): + # Neither trigger carries PR context. A manual run may name one PR; + # otherwise sweep every open format-change PR. + requested = os.environ.get("GATE_PR", "").strip() + if requested: + gate.evaluate(int(requested)) + return + pulls = [ + pr + for pr in repo.get_pulls(state="open") + if any(label.name == FORMAT_LABEL for label in pr.labels) + ] + print(f"Sweep: {len(pulls)} open {FORMAT_LABEL} PR(s).") + for pr in pulls: + try: + gate.evaluate(pr.number) + except Exception as err: # noqa: BLE001 - keep sweeping other PRs + print(f"PR #{pr.number}: {err}") + else: + with open(os.environ["GITHUB_EVENT_PATH"]) as handle: + event = json.load(handle) + gate.evaluate(event["pull_request"]["number"]) + + +if __name__ == "__main__": + main() diff --git a/ci/test_format_vote_gate.py b/ci/test_format_vote_gate.py new file mode 100644 index 00000000000..2186b5deacd --- /dev/null +++ b/ci/test_format_vote_gate.py @@ -0,0 +1,153 @@ +"""Unit tests for the format-spec vote gate logic. + +Run with: pytest ci/test_format_vote_gate.py +""" + +from datetime import datetime, timedelta, timezone + +import pytest + +from format_vote_gate import ( + PERIOD_HOURS, + decide_verdict, + tally_reviews, + vote_opened_at, + weekday_deadline, +) + +HEAD = "sha_head" +PMC = {"alice", "bob", "carol", "dave"} + + +def is_pmc(login): + return login is not None and login.lower() in PMC + + +def review(login, state, commit_id=HEAD): + return {"login": login, "state": state, "commit_id": commit_id} + + +def test_counts_distinct_pmc_approvals_on_head_commit(): + approvals, stale, vetoes = tally_reviews( + [ + review("alice", "APPROVED"), + review("bob", "APPROVED"), + review("carol", "APPROVED"), + ], + HEAD, + "author", + is_pmc, + ) + assert sorted(approvals) == ["alice", "bob", "carol"] + assert stale == [] + assert vetoes == [] + + +def test_only_latest_review_per_member_counts(): + # Alice approved, then later requested changes -> she is a veto, not approval. + approvals, _, vetoes = tally_reviews( + [review("alice", "APPROVED"), review("alice", "CHANGES_REQUESTED")], + HEAD, + "author", + is_pmc, + ) + assert approvals == [] + assert vetoes == ["alice"] + + +def test_approvals_on_earlier_commit_are_stale(): + approvals, stale, _ = tally_reviews( + [review("alice", "APPROVED", "old_sha"), review("bob", "APPROVED")], + HEAD, + "author", + is_pmc, + ) + assert approvals == ["bob"] + assert stale == ["alice"] + + +def test_ignores_author_non_pmc_and_dismissed(): + approvals, _, vetoes = tally_reviews( + [ + review("author", "APPROVED"), # PR author, even if PMC, never counts + review("eve", "APPROVED"), # not on the PMC + review("dave", "DISMISSED"), # withdrawn + review("carol", "COMMENTED"), # a comment is not a vote + ], + HEAD, + "author", + is_pmc, + ) + assert approvals == [] + assert vetoes == [] + + +@pytest.mark.parametrize( + ("veto_count", "approval_count", "period_elapsed", "expected"), + [ + (1, 5, True, "veto"), # veto wins even with enough approvals + elapsed + (0, 2, True, "insufficient"), + (0, 3, False, "waiting_period"), + (0, 3, True, "pass"), + ], +) +def test_decide_verdict_priority(veto_count, approval_count, period_elapsed, expected): + assert decide_verdict(veto_count, approval_count, period_elapsed, 3) == expected + + +def utc(text): + return datetime.fromisoformat(text).replace(tzinfo=timezone.utc) + + +# 2026-08-03 is a Monday, so this week runs Mon 03 .. Sun 09 August. +@pytest.mark.parametrize( + ("opened", "expected"), + [ + # Fully inside a work week: a plain 72-hour offset. + ("2026-08-03T09:00", "2026-08-06T09:00"), + # Opened Friday afternoon: 7h accrue before Saturday, the remaining 65h + # resume Monday 00:00 and land Wednesday afternoon. + ("2026-08-07T17:00", "2026-08-12T17:00"), + # Opened during a weekend: the clock only starts on Monday. + ("2026-08-08T12:00", "2026-08-13T00:00"), + # Opened the instant a weekend ends. + ("2026-08-10T00:00", "2026-08-13T00:00"), + # The deadline itself lands exactly on the weekend boundary. + ("2026-08-05T00:00", "2026-08-08T00:00"), + ], +) +def test_weekday_deadline_excludes_weekends(opened, expected): + assert weekday_deadline(utc(opened), PERIOD_HOURS) == utc(expected) + + +def test_weekday_deadline_spans_multiple_weekends(): + # A period longer than one work week has to skip more than one weekend. + assert weekday_deadline(utc("2026-08-03T00:00"), 24 * 6) == utc("2026-08-11T00:00") + + +def test_weekday_deadline_converts_to_weekend_tz(): + # Late Friday in a UTC+X zone is already Saturday in UTC, so the clock waits. + friday_evening_tokyo = utc("2026-08-08T01:00").astimezone( + timezone(timedelta(hours=9)) + ) + assert weekday_deadline(friday_evening_tokyo, PERIOD_HOURS) == utc( + "2026-08-13T00:00" + ) + + +def test_vote_opens_at_the_later_of_label_and_ready(): + labeled, ready = utc("2026-08-03T09:00"), utc("2026-08-04T09:00") + assert vote_opened_at(labeled, ready) == ready + assert vote_opened_at(ready, labeled) == ready + + +@pytest.mark.parametrize( + ("labeled", "ready"), + [ + (None, utc("2026-08-03T09:00")), # not a format change (yet) + (utc("2026-08-03T09:00"), None), # still a draft + (None, None), + ], +) +def test_vote_does_not_open_until_both_conditions_hold(labeled, ready): + assert vote_opened_at(labeled, ready) is None diff --git a/ci/test_labeler_area.py b/ci/test_labeler_area.py new file mode 100644 index 00000000000..ae4b74bd806 --- /dev/null +++ b/ci/test_labeler_area.py @@ -0,0 +1,34 @@ +"""Regression tests for format-spec path classification.""" + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).parent.parent +LABELER_CONFIG = ROOT / ".github" / "labeler-area.yml" +EXECUTION_PROTO_PATHS = { + "protos/ann.proto", + "protos/filtered_read.proto", + "protos/table_identifier.proto", +} + + +def paths_for(label): + config = yaml.safe_load(LABELER_CONFIG.read_text()) + return set(config[label][0]["changed-files"][0]["any-glob-to-any-file"]) + + +def test_format_labels_use_the_same_paths(): + assert paths_for("A-format") == paths_for("format-change") + + +def test_only_persisted_protos_are_format_changes(): + detected_proto_paths = { + path for path in paths_for("format-change") if path.startswith("protos/") + } + all_proto_paths = { + path.relative_to(ROOT).as_posix() for path in (ROOT / "protos").glob("*.proto") + } + + assert detected_proto_paths == all_proto_paths - EXECUTION_PROTO_PATHS diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index 301f0663d39..00000000000 --- a/codecov.yml +++ /dev/null @@ -1,13 +0,0 @@ -# make all status informational -- pass no matter what -coverage: - status: - project: - default: - target: 50% - threshold: 10% - informational: true - patch: - default: - target: 50% - threshold: 10% - informational: true diff --git a/deny.toml b/deny.toml index 75b92e53447..17d4b546f2d 100644 --- a/deny.toml +++ b/deny.toml @@ -85,6 +85,7 @@ ignore = [ { id = "RUSTSEC-2025-0119", reason = "`number_prefix` used by hf-hub in examples" }, { id = "RUSTSEC-2026-0194", reason = "`quick-xml` <0.41 pulled transitively via object_store (datafusion) and reqsign-aws-v4 (opendal); upstream must upgrade first" }, { id = "RUSTSEC-2026-0195", reason = "`quick-xml` <0.41 pulled transitively via object_store (datafusion) and reqsign-aws-v4 (opendal); upstream must upgrade first" }, + { id = "RUSTSEC-2026-0002", reason = "`lru` 0.12.x pulled transitively via goosefs-sdk; fixed in >=0.16.3, goosefs-sdk must upgrade first" }, ] # If this is true, then cargo deny will use the git executable to fetch advisory database. # If this is false, then it uses a built-in git library. @@ -165,6 +166,10 @@ registries = [ # More documentation about the 'bans' section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/bans/cfg.html [bans] +# Lint every dependency declared by a workspace member against the shared +# `[workspace.dependencies]` table: any crate used by more than one member must +# go through `workspace = true`, and entries nothing uses are an error. +workspace-dependencies = { duplicates = "deny", unused = "deny" } # Lint level for when multiple versions of the same crate are detected multiple-versions = "warn" # Lint level for when a crate version requirement is `*` diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index f592df6966c..bd822264494 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -18,7 +18,7 @@ uv run mkdocs serve ### Python Generated Doc -Python code documentation is built using Sphinx in [lance-python-doc](https://github.com/lancedb/lance-python-doc), +Python code documentation is built using Sphinx in [lance-python-doc](https://github.com/lance-format/lance-python-doc), and published through [Github Pages](https://lance-format.github.io/lance-python-doc/) in ReadTheDocs style. ### Rust Generated Doc diff --git a/docs/Makefile b/docs/Makefile index 4f70f375142..08f5108b770 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,7 +6,7 @@ help: @echo " make build - Build documentation (auto-installs deps)" @echo " make make-full-website - Assemble the full website from local repo checkouts" @echo " make clean-full-website - Remove generated website content" - @echo " Run `make make-full-website` before `make build` or `make serve` for the full multi-repo site" + @echo " Run 'make make-full-website' before 'make build' or 'make serve' for the full multi-repo site" @echo " Override any repo with env vars like LANCE_NAMESPACE_REPO=..., LANCE_SPARK_REPO=..., LANCE_RAY_REPO=..." @echo " make clean - Clean build artifacts" @echo " make check-links - Check for broken links" diff --git a/docs/README.md b/docs/README.md index 80092b157c7..8cb33f86387 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,7 @@ # Lance Documentation -This directory contains the documentation for Lance, built with MkDocs and Material theme. +This directory contains the documentation for Lance, built with MkDocs and a +custom theme (`theme/`) implementing the Lance Docs design. ## Getting Started with uv @@ -62,5 +63,18 @@ uv sync --upgrade ## Project Structure - `src/` - Source markdown files for documentation +- `theme/` - Custom MkDocs theme implementing the Lance Docs design + (templates in `*.html`, styles/behaviour in `theme/assets/`) - `mkdocs.yml` - MkDocs configuration - `pyproject.toml` - Python project configuration (uv compatible) + +## Theme Notes + +- Light/dark mode follows `prefers-color-scheme`, is toggleable from the + header, and persists in `localStorage` (`ld-theme`). +- The GitHub star count is fetched from the public GitHub API and cached in + `localStorage` for an hour; the button degrades to a plain link offline. +- Search is a lightweight overlay (`/` or `Cmd/Ctrl+K`) over the standard + `search` plugin index — no external search dependencies. +- Mermaid diagrams render client-side; the library is loaded from a CDN only + on pages that contain a diagram. diff --git a/docs/clean-full-website.sh b/docs/clean-full-website.sh index db8013cd744..3ffa8ffe15b 100755 --- a/docs/clean-full-website.sh +++ b/docs/clean-full-website.sh @@ -15,6 +15,7 @@ rm -rf "$docs_src/integrations/duckdb" rm -rf "$docs_src/integrations/spark" rm -rf "$docs_src/integrations/ray" rm -rf "$docs_src/integrations/trino" +rm -rf "$docs_src/integrations/context" rm -f "$docs_src/community/project-specific/.pages" rm -rf "$docs_src/community/project-specific/lance" rm -f "$docs_src/community/project-specific/namespace.md" @@ -22,6 +23,7 @@ rm -f "$docs_src/community/project-specific/namespace-impls.md" rm -f "$docs_src/community/project-specific/ray.md" rm -f "$docs_src/community/project-specific/spark.md" rm -f "$docs_src/community/project-specific/trino.md" +rm -f "$docs_src/community/project-specific/context.md" cat > "$docs_src/format/.pages" <<'EOF' nav: @@ -37,9 +39,9 @@ cat > "$docs_src/integrations/.pages" <<'EOF' nav: - Overview: index.md - Apache DataFusion: datafusion.md - - PostgreSQL: https://github.com/lancedb/pglance + - PostgreSQL: https://github.com/lance-format/pglance - PyTorch: pytorch.md - - Tensorflow: tensorflow.md + - TensorFlow: tensorflow.md EOF mkdir -p "$docs_src/format/catalog/dir" diff --git a/docs/hooks/pmc_roster.py b/docs/hooks/pmc_roster.py new file mode 100644 index 00000000000..3a56daa2b5d --- /dev/null +++ b/docs/hooks/pmc_roster.py @@ -0,0 +1,46 @@ +"""MkDocs hook: render the PMC roster table from `pmc.yaml` at build time. + +`docs/src/community/pmc.yaml` is the source of truth for the PMC roster (it also +drives the format-spec vote gate). The roster page contains the placeholder +``; this hook expands it into a Markdown table when the +docs are built, so the table never has to be maintained by hand. + +Registered via `hooks:` in `mkdocs.yml`. +""" + +import pathlib + +import yaml + +PLACEHOLDER = "" + +COLUMNS = [ + ("Name", "name"), + ("GitHub Handle", "handle"), + ("Affiliation", "affiliation"), + ("Ecosystem Roles", "ecosystem_roles"), +] + + +def _render_table(members): + headers = [title for title, _ in COLUMNS] + rows = [[str(m.get(key, "") or "") for _, key in COLUMNS] for m in members] + widths = [ + max([len(headers[i])] + [len(row[i]) for row in rows]) + for i in range(len(COLUMNS)) + ] + + def row(cells): + return "| " + " | ".join(c.ljust(widths[i]) for i, c in enumerate(cells)) + " |" + + lines = [row(headers), "|" + "|".join("-" * (w + 2) for w in widths) + "|"] + lines.extend(row(r) for r in rows) + return "\n".join(lines) + + +def on_page_markdown(markdown, page, config, files): + if PLACEHOLDER not in markdown: + return markdown + roster_path = pathlib.Path(config["docs_dir"]) / "community" / "pmc.yaml" + roster = yaml.safe_load(roster_path.read_text()) + return markdown.replace(PLACEHOLDER, _render_table(roster["members"])) diff --git a/docs/make-full-website.sh b/docs/make-full-website.sh index bb446a2f070..414b3831fb0 100755 --- a/docs/make-full-website.sh +++ b/docs/make-full-website.sh @@ -13,6 +13,7 @@ Override any repo path with the matching environment variable: LANCE_TRINO_REPO LANCE_DUCKDB_REPO LANCE_HUGGINGFACE_REPO + LANCE_CONTEXT_REPO Defaults: LANCE_NAMESPACE_REPO=$HOME/oss/lance-namespace LANCE_NAMESPACE_IMPLS_REPO=$HOME/oss/lance-namespace-impls @@ -21,6 +22,7 @@ Defaults: LANCE_TRINO_REPO=$HOME/oss/lance-trino LANCE_DUCKDB_REPO=$HOME/oss/lance-duckdb LANCE_HUGGINGFACE_REPO=$HOME/oss/lance-huggingface + LANCE_CONTEXT_REPO=$HOME/oss/lance-context EOF } @@ -60,6 +62,7 @@ ray_repo_input=${LANCE_RAY_REPO:-$HOME/oss/lance-ray} trino_repo_input=${LANCE_TRINO_REPO:-$HOME/oss/lance-trino} duckdb_repo_input=${LANCE_DUCKDB_REPO:-$HOME/oss/lance-duckdb} huggingface_repo_input=${LANCE_HUGGINGFACE_REPO:-$HOME/oss/lance-huggingface} +context_repo_input=${LANCE_CONTEXT_REPO:-$HOME/oss/lance-context} copy_docs_dir() { local source_dir="$1" @@ -134,6 +137,7 @@ ray_repo=$(resolve_repo_dir "$ray_repo_input") trino_repo=$(resolve_repo_dir "$trino_repo_input") duckdb_repo=$(resolve_repo_dir "$duckdb_repo_input") huggingface_repo=$(resolve_repo_dir "$huggingface_repo_input") +context_repo=$(resolve_repo_dir "$context_repo_input") "$script_dir/clean-full-website.sh" @@ -265,6 +269,12 @@ else warn_missing_repo "Lance Trino docs" "$trino_repo/docs/src" fi +if copy_docs_dir "$context_repo/docs/src" "$docs_src/integrations/context"; then + integration_entries+=(" - Lance Context: context") +else + warn_missing_repo "Lance Context docs" "$context_repo/docs/src" +fi + { echo "nav:" for entry in "${integration_entries[@]}"; do @@ -305,6 +315,10 @@ if copy_file_if_exists "$trino_repo/CONTRIBUTING.md" "$docs_src/community/projec project_entries+=(" - Lance Trino: trino.md") fi +if copy_file_if_exists "$context_repo/CONTRIBUTING.md" "$docs_src/community/project-specific/context.md"; then + project_entries+=(" - Lance Context: context.md") +fi + { echo "nav:" for entry in "${project_entries[@]}"; do diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 8144a42e5f5..d872d749ff2 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -6,37 +6,13 @@ docs_dir: src repo_name: lance-format/lance repo_url: https://github.com/lance-format/lance +# Custom theme implementing the "Lance Docs" design (see theme/). theme: - name: material - custom_dir: overrides - logo: logo/white.png - favicon: logo/logo.png - palette: - - scheme: default - primary: custom - accent: custom - toggle: - icon: material/brightness-7 - name: Switch to dark mode - - scheme: slate - primary: custom - accent: custom - toggle: - icon: material/brightness-4 - name: Switch to light mode - features: - - navigation.tabs - - navigation.sections - - navigation.instant - - navigation.indexes - - navigation.tracking - - navigation.top - - search.highlight - - search.share - - content.code.copy - - content.code.annotate - icon: - repo: fontawesome/brands/github + name: null + custom_dir: theme + locale: en + static_templates: + - 404.html markdown_extensions: - admonition @@ -51,9 +27,16 @@ markdown_extensions: line_spans: __span pygments_lang_class: true - pymdownx.inlinehilite - - pymdownx.snippets + - pymdownx.snippets: + # Allow snippets to pull in files from the repo root (e.g. shared docs + # kept alongside Rust source). Paths are relative to the docs/ dir. + base_path: + - . + - .. - pymdownx.tabbed: alternate_style: true + # Autolink bare URLs (e.g. repository tables on the community pages). + - pymdownx.magiclink - attr_list - md_in_html - tables @@ -66,18 +49,7 @@ plugins: - mkdocs_protobuf: proto_dir: ../protos -extra: - generator: false - social: - - icon: fontawesome/brands/github - link: https://github.com/lance-format/lance - - icon: fontawesome/brands/discord - link: https://discord.gg/lance +hooks: + - hooks/pmc_roster.py copyright: © 2025 Lance Format. All rights reserved. - -extra_css: - - assets/stylesheets/home.css -extra_javascript: - - assets/javascripts/nav-expand.js - diff --git a/docs/overrides/home.html b/docs/overrides/home.html deleted file mode 100644 index 26eb288aee6..00000000000 --- a/docs/overrides/home.html +++ /dev/null @@ -1,295 +0,0 @@ -{% extends "main.html" %} - -{% block tabs %} - {{ super() }} - - - - - - -
-
-
- -

The Open Lakehouse Format for Multimodal AI

-
- -
-
-
- - -
-
-
-

What is Lance?

-

- Lance is a modern, open source lakehouse format for multimodal AI. It contains a file format, table format, and catalog spec, - allowing you to build a complete open lakehouse on top of object storage to power your AI workflows. - Lance brings high-performance vector search, full-text search, random access, and feature - engineering capabilities to the lakehouse, while you can still get all the existing lakehouse benefits - like SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB, etc.) - and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore, etc.) -

-

- Learn more about Lance's technical details by reading our - research paper - published at VLDB 2025. -

- Read the Docs -
-
-
- - -
-
-
-
-

Expressive Hybrid Search

-

- Lance enables powerful hybrid search combining vector similarity, full-text search, - and SQL analytics on the same dataset. All query types are accelerated by corresponding - secondary indexes as part of the Lance specification. -

-

- Run semantic search on embeddings, BM25 search on keywords, and apply complex SQL predicates - - all using a single table with a unified interface. -

- Learn More -
-
- Hybrid Search Example -
-
-
-
- - -
-
-
-
-

Lightning-fast Random Access

-

- Lance delivers 100x faster random access compared to Parquet or Iceberg. - Unlike traditional formats, Lance maintains high performance even when - randomly accessing scattered rows across your entire dataset. -

-

- With a highly optimized file format plus efficient row-addressing and secondary indexes at table level, - you can access individual records across multiple files instantly, - making it perfect for real-time ML serving, random sampling, and interactive applications. -

- Learn More -
-
- Random Access Example -
-
-
-
- - -
-
-
-
-

Native Multimodal Data Support

-

- Store images, videos, audio, text, and embeddings alongside your traditional tabular data in a single unified format. - Lance's blob encoding efficiently handles large binary objects with lazy loading, - while optimized vector storage accelerates similarity search. -

-

- Perfect for AI/ML workloads where you need to store raw data, ML features, generated captions and embeddings - all together for multimodal retrieval and genAI workflows. -

- Learn More -
-
- Multimodal Data Example -
-
-
-
- - -
-
-
-
-

Data Evolution > Schema Evolution

-

- Schema evolution in most open table formats are metadata only and fast. - But when trying to backfill column values in existing rows, a full table rewrite is typically required. - Lance supports data evolution (efficient schema evolution with backfill), making it perfect for ML - feature engineering, embedding and media content management. -

-

- Adding a new column with data is as simple as writing new Lance files to the Lance table - - no need to rewrite your entire dataset. -

- Learn More -
-
- Data Evolution Example -
-
-
-
- - -
-
-
-
-

Rich Ecosystem Integrations

-

- As an open format, Lance integrates seamlessly with the Python data ecosystem and modern data platforms. - Work with your favorite tools including Pandas, Polars, Ray and PyTorch for data processing and machine learning. -

-

- Connect with leading query engines like Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink/Fluss - to run SQL analytics and distributed processing on your Lance datasets. -

- View Integrations -
-
- Lance Ecosystem Integrations -
-
-
-
- - -{% endblock %} - -{% block content %}{% endblock %} -{% block footer %} - {{ super() }} -{% endblock %} diff --git a/docs/pyproject.toml b/docs/pyproject.toml index 4112230aec5..3aa6847e4f3 100644 --- a/docs/pyproject.toml +++ b/docs/pyproject.toml @@ -6,10 +6,12 @@ readme = "README.md" requires-python = ">=3.10,<3.11" dependencies = [ "mkdocs>=1.5.0", - "mkdocs-material>=9.4.0", + "pymdown-extensions>=10.0", + "pygments>=2.16", "mkdocs-protobuf>=0.1.0", "mkdocs-linkcheck>=1.0.0", - "mkdocs-awesome-pages-plugin>=2.10.1" + "mkdocs-awesome-pages-plugin>=2.10.1", + "requests>=2.31.0" # mkdocs-linkcheck imports it but doesn't declare it ] [tool.uv] diff --git a/docs/src/assets/javascripts/nav-expand.js b/docs/src/assets/javascripts/nav-expand.js deleted file mode 100644 index 17171f5a295..00000000000 --- a/docs/src/assets/javascripts/nav-expand.js +++ /dev/null @@ -1,16 +0,0 @@ -// Auto-expand sidebar navigation to 2 levels on page load. -// Level 1 sections are already expanded by navigation.sections. -// This expands level 2 (e.g. Operations, Models become visible) -// but leaves level 3+ collapsed. -document.addEventListener("DOMContentLoaded", function () { - // In mkdocs-material with navigation.sections, the top-level items - // are rendered as non-collapsible sections. The collapsible items - // start at the next level. We want to expand one more level. - // - // Selector: inside the primary nav, find toggle checkboxes that are - // exactly 2 nesting levels deep (the second-level sections). - var toggles = document.querySelectorAll( - ".md-sidebar--primary .md-nav--primary > .md-nav__list > .md-nav__item > .md-nav > .md-nav__list > .md-nav__item > .md-nav__toggle" - ); - toggles.forEach(function (t) { t.checked = true; }); -}); diff --git a/docs/src/assets/stylesheets/home.css b/docs/src/assets/stylesheets/home.css deleted file mode 100644 index eafdd5afbf9..00000000000 --- a/docs/src/assets/stylesheets/home.css +++ /dev/null @@ -1,244 +0,0 @@ -/* Lance Homepage Styles */ - -/* Override with custom color #625EFF site-wide */ -:root > * { - --md-primary-fg-color: #625EFF; - --md-primary-fg-color--light: #8481FF; - --md-primary-fg-color--dark: #4A46CC; - --md-accent-fg-color: #625EFF; - --md-accent-fg-color--transparent: rgba(98, 94, 255, 0.1); -} - -* { - box-sizing: border-box; -} - -.container { - width: 100%; - max-width: 1140px; - margin-right: auto; - margin-left: auto; - padding-right: 15px; - padding-left: 15px; -} - -/* Hero Section - Fullscreen with Background Image */ -.mdx-container { - text-align: center; - color: #f8f8f8; - background: url("../images/lance-mj.png") no-repeat center center; - background-size: cover; - min-height: 100vh; - height: 100vh; - display: flex; - align-items: center; - justify-content: center; -} - -.intro-message { - position: relative; - padding: 40px 20px; - font-family: "Lato", -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; - max-width: 1000px; - margin: 0 auto; -} - -.hero-logo { - display: inline-flex; - align-items: center; - margin-bottom: 16px; -} - -.hero-logo img { - height: 120px; - width: auto; - margin-right: 24px; - margin-top: 12px; - filter: drop-shadow(3px 3px 8px rgba(0, 0, 0, 0.9)); -} - -.intro-message h1 { - font-weight: 400; - margin: 0; - display: inline-block; - text-shadow: 3px 3px 8px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1); - font-size: 8em; - line-height: 1.2; - color: #ffffff; - vertical-align: middle; -} - -.intro-message h1 sup { - font-size: 2rem; - text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9); -} - -.intro-message h3 { - font-size: 1.1rem; - text-shadow: 2px 2px 6px rgba(0, 0, 0, 0.9), 1px 1px 3px rgba(0, 0, 0, 1); - font-weight: 600; - margin-bottom: 32px; - color: #ffffff; -} - -.intro-divider { - width: 400px; - max-width: 80%; - border-top: 1px solid rgba(255, 255, 255, 0.8); - border-bottom: 1px solid rgba(0, 0, 0, 0.2); - margin: 24px auto; -} - -.list-inline { - padding-left: 0; - margin-left: -5px; - list-style: none; - margin-bottom: 0; -} - -.list-inline li { - display: inline-block; - padding-right: 5px; - padding-left: 5px; -} - -.intro-message .md-button { - margin: 8px; - padding: 14px 36px; - font-size: 1.1rem; - font-weight: 600; - text-shadow: 2px 2px 4px rgba(0, 0, 0, 0.8); - transition: all 0.3s ease; -} - -.intro-message .md-button:hover { - transform: translateY(-2px); - box-shadow: 0 8px 16px rgba(0, 0, 0, 0.3); -} - -.intro-message .md-button--primary:hover { - box-shadow: 0 8px 20px rgba(98, 94, 255, 0.5); -} - -.intro-message .md-button:not(.md-button--primary):hover { - box-shadow: 0 8px 20px rgba(255, 255, 255, 0.4); -} - -/* What is Lance Section */ -.lance-intro-section { - padding: 80px 0; - background-color: rgba(128, 128, 128, 0.03); - border-bottom: 1px solid rgba(128, 128, 128, 0.1); -} - -.lance-intro-content { - max-width: 900px; - margin: 0 auto; - text-align: center; -} - -.lance-intro-content h2 { - font-size: 36px; - font-weight: 500; - margin-bottom: 32px; - color: var(--md-primary-fg-color); -} - -.lance-intro-content p { - font-size: 16px; - line-height: 1.8; - margin-bottom: 32px; - opacity: 0.9; - text-align: left; -} - -.lance-paper-link { - color: var(--md-primary-fg-color); - text-decoration: none; -} - -.lance-paper-link:hover { - color: var(--md-primary-fg-color); - text-decoration: none; -} - -.lance-intro-content a:hover { - color: #757575; - text-decoration: none; -} - -.lance-intro-content .md-button { - margin-top: 16px; - padding: 10px 28px; - font-size: 14px; - border: 2px solid currentColor; - background-color: transparent; - transition: all 0.3s ease; -} - -.lance-intro-content .md-button:hover { - color: var(--md-primary-fg-color); - background-color: transparent; -} - -/* Feature Sections */ -.lance-feature-section { - padding: 80px 0; - border-bottom: 1px solid rgba(128, 128, 128, 0.1); -} - -.lance-feature-section:last-child { - border-bottom: none; -} - -.lance-feature-content { - display: flex; - flex-wrap: wrap; - align-items: center; - gap: 60px; -} - -.lance-feature-text { - flex: 1; - min-width: 300px; -} - -.lance-feature-text h2 { - font-size: 30px; - font-weight: 500; - margin-bottom: 16px; - color: var(--md-primary-fg-color); -} - -.lance-feature-text p { - font-size: 15px; - line-height: 1.6; - opacity: 0.85; - margin-bottom: 16px; -} - -.lance-feature-text .md-button { - font-size: 0.6rem; - padding: 0; - transition: all 0.3s ease; -} - -.lance-feature-text .md-button:hover { - transform: translateX(4px); - color: var(--md-primary-fg-color); -} - -.lance-feature-demo { - flex: 1; - min-width: 400px; - display: flex; - justify-content: center; - overflow: hidden; -} - -/* Alternating layout */ -.lance-feature-section.reverse .lance-feature-content { - flex-direction: row-reverse; -} - - diff --git a/docs/src/community/contributing.md b/docs/src/community/contributing.md index 7e2e681b4d2..fab3df40c21 100644 --- a/docs/src/community/contributing.md +++ b/docs/src/community/contributing.md @@ -21,7 +21,18 @@ Major technical changes are discussed organically through the following approach - **Iterate on Design**: Engage with the community to refine the approach based on their input and expertise - **Draft PRs for Details**: Once the general direction is acceptable to the community, publish draft PRs to help hash out implementation details. Draft PRs are encouraged as they facilitate concrete discussions - **Break Down Changes**: Split large draft PRs into smaller, incremental PRs for easier review and to demonstrate progress -- **Formal Voting**: Maintainers with write access can approve code modifications related to the design. If the design requires Lance format spec changes, a separate vote will be conducted on GitHub Discussions following the [voting requirements](./voting.md#voting-requirements) +- **Formal Voting**: Maintainers with write access can approve code modifications related to the design. If the design requires Lance format spec changes, those changes go in their own PR and the PMC votes on that PR following the [voting requirements](./voting.md#voting-requirements) + +## Format Specification Changes + +Changes to the Lance format specification — the protobuf definitions and the docs +under `docs/src/format/` — are proposed as a pull request and voted on there by the +PMC. The PR is the proposal; there is no separate discussion thread to open first. + +Scope such a PR to the specification itself, plus the minimum library changes needed +to keep the build green, and put the implementation in follow-up PRs. See +[Lance Format Specification Changes](./voting.md#lance-format-specification-changes) +for what the vote requires and how it is counted. ## AI Tooling Integrations diff --git a/docs/src/community/index.md b/docs/src/community/index.md index 4bf1ca30842..5b9c9bdcbe4 100644 --- a/docs/src/community/index.md +++ b/docs/src/community/index.md @@ -72,7 +72,7 @@ Here is the list of current subprojects: | lance-huggingface | https://github.com/lance-format/lance-huggingface | Hugging Face integration for Lance | | lance-namespace | https://github.com/lance-format/lance-namespace | Lance namespace format specification, Rust/Python/Java Codegen SDKs | | lance-namespace-impls | https://github.com/lance-format/lance-namespace-impls | Lance Namespace Implementations - Apache Hive, Apache Polaris, Apache Gravitino, Unity Catalog, AWS Glue and more | -| lance-python-docs | https://github.com/lance-format/lance-python-docs | Lance Python SDK generated docs and integration hook with readthedocs | +| lance-python-doc | https://github.com/lance-format/lance-python-doc | Lance Python SDK generated docs and integration hook with readthedocs | | lance-ray | https://github.com/lance-format/lance-ray | Ray integration for Lance | | lance-spark | https://github.com/lance-format/lance-spark | Apache Spark connector for Lance | diff --git a/docs/src/community/maintainers.md b/docs/src/community/maintainers.md index f3ba6e70304..a772f4739bd 100644 --- a/docs/src/community/maintainers.md +++ b/docs/src/community/maintainers.md @@ -54,7 +54,7 @@ Maintainers with GitHub write access are additionally encouraged to: | Bryan Keller | bryanck | Netflix | | Apache Iceberg Committer | | Aman Kishore | AmanKishore | Harvey.ai | | | | Sangwu Lee | RE-N-Y | Krea.ai | | | -| Jeremy Leibs | jleibs | Rerun.io | | | +| Jeremy Leibs | jleibs | Genesis AI | | | | Haocheng Liu | HaochengLIU | Seven Research | ✓ | | | Nathan Ma | majin1102 | ByteDance | ✓ | Apache Amoro (incubating) PPMC Member | | ChanChan Mao | ccmao1130 | LanceDB | | | @@ -107,4 +107,4 @@ To be granted GitHub write access, the maintainer should: - Have a history of high-quality contributions - Have earned trust from the community for code reviews - Get nominated by a PMC member and approval through a passing vote -- Sign the Contributor License Agreement (CLA) \ No newline at end of file +- Sign the Contributor License Agreement (CLA) diff --git a/docs/src/community/pmc.md b/docs/src/community/pmc.md index 3d9daeaebff..5088e45d6ae 100644 --- a/docs/src/community/pmc.md +++ b/docs/src/community/pmc.md @@ -27,27 +27,9 @@ In addition to the [activities of maintainers](./maintainers.md#activities), PMC ## Roster -| Name | GitHub Handle | Affiliation | Ecosystem Roles | -|-----------------|-----------------|--------------|---------------------------------------------------------------------------------------------------------------| -| Yang Cen | BubbleCal | LanceDB | Milvus Contributor | -| Pablo Delgado | pablete | Netflix | | -| Hao Ding | Xuanwo | LanceDB | Apache OpenDAL PMC Chair, Apache Iceberg Committer, Apache Member and [more](https://xuanwo.io/about/) | -| Zhaowei Huang | SaintBacchus | Alibaba | Apache Doris Committer | -| Will Jones | wjones127 | LanceDB | Apache Arrow PMC Member, Apache DataFusion PMC Member, Delta Lake Maintainer | -| Matt Kafonek | kafonek | Runway AI | | -| Denny Lee | dennyglee | Databricks | Unity Catalog Maintainer, Delta Lake Maintainer, Apache Spark Contributor, MLflow Contributor | -| Rob Meng | chebbyChefNEQ | Jump Trading | | -| Dao Mi | dowjones226 | Netflix | | -| Weston Pace | westonpace | LanceDB | Apache Arrow PMC Member, Substrait SMC Member | -| Calvin Qi | calvinqi | Harvey.ai | | -| Prashanth Rao | prrao87 | LanceDB | | -| Ethan Rosenthal | EthanRosenthal | Runway AI | | | -| Tim Saucer | timsaucer | Rerun.io | Apache DataFusion PMC Member | -| Chang She | changhiskhan | LanceDB | Pandas Co-Author | -| Jasmine Wang | onigiriisabunny | LanceDB | Alluxio PMC Community Manager | -| Lei Xu | eddyxu | LanceDB | Apache Hadoop PMC Member | -| Vino Yang | yanghua | Bytedance | Apache Hudi PMC Member, Apache Kyuubi PMC Member, Apache Kylin Committer, Apache Incubation Program Committer | -| Jack Ye | jackye1995 | LanceDB | Apache Iceberg PMC Member, Apache Polaris (incubating) PPMC Member, Apache Incubation Program Committer | + + ## Becoming a PMC Member diff --git a/docs/src/community/pmc.yaml b/docs/src/community/pmc.yaml new file mode 100644 index 00000000000..34819652c00 --- /dev/null +++ b/docs/src/community/pmc.yaml @@ -0,0 +1,90 @@ +# Source of truth for the Project Management Committee (PMC) roster. +# +# This file drives two things, so keep it accurate: +# 1. The roster table in `pmc.md`, rendered at docs build time by the +# `docs/hooks/pmc_roster.py` MkDocs hook. +# 2. The format-specification vote gate, which only counts PR approvals from +# the `handle`s listed here (see `.github/workflows/format-vote-gate.yml`). +# +# Adding or removing a member is itself a PMC vote (roster change). After the +# vote passes, edit this file; the docs table updates automatically. +# +# `handle` must match the member's GitHub login exactly (case-insensitive when +# matched). `ecosystem_roles` is free-form markdown and may be empty. +members: + - name: Yang Cen + handle: BubbleCal + affiliation: LanceDB + ecosystem_roles: Milvus Contributor + - name: Pablo Delgado + handle: pablete + affiliation: Netflix + ecosystem_roles: "" + - name: Hao Ding + handle: Xuanwo + affiliation: LanceDB + ecosystem_roles: Apache OpenDAL PMC Chair, Apache Iceberg Committer, Apache Member and [more](https://xuanwo.io/about/) + - name: Zhaowei Huang + handle: SaintBacchus + affiliation: Alibaba + ecosystem_roles: Apache Doris Committer + - name: Will Jones + handle: wjones127 + affiliation: LanceDB + ecosystem_roles: Apache Arrow PMC Member, Apache DataFusion PMC Member, Delta Lake Maintainer + - name: Matt Kafonek + handle: kafonek + affiliation: Runway AI + ecosystem_roles: "" + - name: Denny Lee + handle: dennyglee + affiliation: Databricks + ecosystem_roles: Unity Catalog Maintainer, Delta Lake Maintainer, Apache Spark Contributor, MLflow Contributor + - name: Rob Meng + handle: chebbyChefNEQ + affiliation: Jump Trading + ecosystem_roles: "" + - name: Dao Mi + handle: dowjones226 + affiliation: Netflix + ecosystem_roles: "" + - name: Weston Pace + handle: westonpace + affiliation: LanceDB + ecosystem_roles: Apache Arrow PMC Member, Substrait SMC Member + - name: Calvin Qi + handle: calvinqi + affiliation: Harvey.ai + ecosystem_roles: "" + - name: Prashanth Rao + handle: prrao87 + affiliation: LanceDB + ecosystem_roles: "" + - name: Ethan Rosenthal + handle: EthanRosenthal + affiliation: Runway AI + ecosystem_roles: "" + - name: Tim Saucer + handle: timsaucer + affiliation: Rerun.io + ecosystem_roles: Apache DataFusion PMC Member + - name: Chang She + handle: changhiskhan + affiliation: LanceDB + ecosystem_roles: Pandas Co-Author + - name: Jasmine Wang + handle: onigiriisabunny + affiliation: LanceDB + ecosystem_roles: Alluxio PMC Community Manager + - name: Lei Xu + handle: eddyxu + affiliation: LanceDB + ecosystem_roles: Apache Hadoop PMC Member + - name: Vino Yang + handle: yanghua + affiliation: Bytedance + ecosystem_roles: Apache Hudi PMC Member, Apache Kyuubi PMC Member, Apache Kylin Committer, Apache Incubation Program Committer + - name: Jack Ye + handle: jackye1995 + affiliation: LanceDB + ecosystem_roles: Apache Iceberg PMC Member, Apache Polaris (incubating) PPMC Member, Apache Incubation Program Committer diff --git a/docs/src/community/voting.md b/docs/src/community/voting.md index 8c5ac341e67..0a2218752c3 100644 --- a/docs/src/community/voting.md +++ b/docs/src/community/voting.md @@ -22,6 +22,10 @@ each vote should be cast as an independent comment instead of as a reply within This ensures that people can discuss the vote as replies to that specific comment if needed (e.g., to discuss **-1** vetoes or address concerns). +For votes conducted on a pull request, cast **+1** by approving the PR and **-1** by +requesting changes. These votes are counted automatically, so a **+1** written only as +a comment does not count. + ## Binding Votes Only votes from the binding voters are counted for each decision, @@ -47,7 +51,65 @@ A **-1** binding vote is considered a veto for all decision types. Vetoes: | Release a new stable major version of the core project | 3 | PMC | GitHub Discussions | 3 days | | Release a new stable minor version of the core project | 3 | PMC | GitHub Discussions | 3 days | | Release a new stable patch version of the core project | 3 | PMC | GitHub Discussions | N/A | -| Lance Format Specification modifications | 3 (excluding proposer) | PMC | GitHub Discussions (with a GitHub PR) | 1 week | +| Lance Format Specification modifications | 3 (excluding proposer) | PMC | GitHub PR (see [below](#lance-format-specification-changes)) | 72 hours, excluding weekends | | Code modifications in the core project (except changes to format specifications) | 1 (excluding proposer) | Maintainers with write access | GitHub PR | N/A | | Release a new stable version of subprojects | 1 | PMC | GitHub Discussions | N/A | | Code modifications in subprojects | 1 (excluding proposer) | Contributors with write access | GitHub PR | N/A | + +## Lance Format Specification Changes + +The pull request *is* the proposal. Open a PR with the specification change, +and the PMC votes on it there — there is no separate design document or +discussion thread to write first, and the requirement is enforced structurally +in CI rather than by convention. + +### Proposing a Change + +Keep a format-specification PR to the specification itself: the protobuf +definitions and the spec documentation, plus the minimum library changes needed +to keep the build green (for example, matching a renamed generated field). +Implement the behavior behind the change in follow-up PRs. + +This is not just a tidiness preference. The vote is on the format — a durable +compatibility contract that outlives any one implementation — and PMC members +should be able to read the whole of what they are voting on. A PR that also +carries the reader, writer, and test changes buries the contract in +implementation detail, and it drags an ordinary code review through a 72-hour +voting period it does not need. + +Discussion happens as review comments on the PR, so reviewers can respond to +specific lines of the specification. Open the PR as a draft while it is still +taking shape; the voting period starts when you mark it ready for review. + +### How the Vote is Counted + +A PR counts as a format-specification change when it modifies the protobuf +definitions (`protos/**/*.proto`) or the spec documentation (`docs/src/format/**`); +such PRs are labeled `format-change` automatically. The +[format spec vote gate](https://github.com/lance-format/lance/blob/main/.github/workflows/format-vote-gate.yml) +blocks merging a `format-change` PR until all of the following hold: + +- **Three binding +1 votes.** Three PMC members have approved the PR, excluding + the proposer. Cast +1 by approving the PR. Only approvals on the latest commit + count — pushing new commits invalidates earlier approvals, since the proposal + has changed. +- **No veto.** No PMC member has an outstanding "Request changes" review. A `-1` + binding vote (cast by requesting changes) is a veto and blocks the merge until + withdrawn. +- **Minimum voting period.** At least 72 hours have elapsed since the vote + opened. Weekends do not count toward the 72 hours, so a proposal opened on a + Friday afternoon still gets three working days of attention. The voting period + opens once the PR is both labeled `format-change` and marked ready for + review, whichever comes last. Weekends are delimited in UTC; the gate comments + on the PR with the exact closing time in both UTC and Pacific Time. + +The gate is the `format-spec-vote` required status check on protected branches. +The PMC roster used to count votes is read from +[`docs/src/community/pmc.yaml`](./pmc.md). It re-evaluates on a 15-minute +schedule, so the tally comment and the status check trail a review by a few +minutes; the comment links to a "Run workflow" page for anyone who would rather +re-check immediately. + +For a trivial edit that does not change the format — a typo, wording, or +formatting fix — a PMC member may apply the `format-waived` label to waive the +vote. diff --git a/docs/src/format/AGENTS.md b/docs/src/format/AGENTS.md index c47277c052e..faa482f687c 100644 --- a/docs/src/format/AGENTS.md +++ b/docs/src/format/AGENTS.md @@ -2,6 +2,11 @@ Also see [root AGENTS.md](../../../AGENTS.md) for cross-language standards. +## Change Process + +- Changes here require a PMC vote on the pull request, enforced by the `format-spec-vote` CI gate. See [Lance Format Specification Changes](../community/voting.md#lance-format-specification-changes). +- Keep a spec change in its own PR, together with the matching `protos/` change and only the library edits needed to compile. Put the implementation in a follow-up PR — voters need to read the contract, not its implementation. + ## Style - Keep format docs as concise, text-only reference — no code examples (put those in user guide sections). diff --git a/docs/src/format/CLAUDE.md b/docs/src/format/CLAUDE.md new file mode 120000 index 00000000000..47dc3e3d863 --- /dev/null +++ b/docs/src/format/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/docs/src/format/file/encoding.md b/docs/src/format/file/encoding.md index 4ca053d4fa6..1cb83d581e9 100644 --- a/docs/src/format/file/encoding.md +++ b/docs/src/format/file/encoding.md @@ -327,6 +327,148 @@ The protobuf for the full zip layout describes the compression of the data buffe size of the control words and how many bits we have per value (for fixed-width data) or how many bits we have per offset (for variable-width data). +### Sparse Page Layout + +Sparse pages require Lance 2.3. They represent flat or nested Arrow structure directly as slot-domain mappings instead +of dense repetition and definition events. Writers emit this layout only in files declared as 2.3 or above. The layout is +identified only by `PageLayout`; field metadata does not identify the layout of an existing page. + +A domain is a layer-local integer coordinate space `[0, num_slots)`, and a slot is one element in that space. The +outer-most domain contains the page's top-level rows. Each layer maps its parent domain to the next layer's parent +domain, and the terminal child domain contains the leaf value slots stored in value chunks. + +Structural layers are ordered from outer-most to inner-most: + +- validity maps a nullable item or struct slot to valid or null +- list maps non-empty parent slots to variable-size child ranges +- fixed-size-list maps each parent slot to a child range of a fixed dimension + +The layer list may be empty for a flat, non-nullable leaf page. In that case the scheduling domain and +`num_visible_items` must be equal. The explicit writer currently emits its normalized all-valid layer even when a flat +page could use this shorter wire representation. + +A list slot that is valid and absent from `non_empty_positions` is an empty list. Maps use the same structural contract +as lists. The terminal child-domain size equals `SparseLayout.num_visible_items`. + +`SparseLayout.num_visible_items` is the number of leaf value slots encoded in value chunks. Null leaf slots count +because they still occupy positions in Arrow's leaf value buffer; a nullable primitive with 100 slots, including 30 +nulls, has 100 visible items. `SparseLayout.num_items` is the number of entries in the equivalent dense repetition and +definition stream. It equals `num_visible_items` plus one structural placeholder for every list slot without children. +The first layer's `num_slots` is the logical top-level row count used for projection. + +Position sets have four semantic representations: `empty`, `all`, one non-empty `range`, or an `explicit` +delta-compressed `u64` buffer. Count sets are `empty`, one positive `constant` value, or an `explicit` compressed `u64` +buffer. Every layer has a `SparseValiditySet` whose meaning is explicit: + +- `SPARSE_VALIDITY_NULL_POSITIONS`: stored positions are null and all other positions are valid +- `SPARSE_VALIDITY_VALID_POSITIONS`: stored positions are valid and all other positions are null + +The unspecified validity meaning is invalid. Both polarities are part of the wire contract and have identical Arrow +semantics after normalization. + +#### Writer Selection + +Writers may emit this layout only for Lance 2.3+ fields. A field can request it explicitly with +`lance-encoding:structural-encoding=sparse`; the same request is an input error for earlier file versions. Without an +explicit structural encoding, the Lance 2.3 writer selects sparse only when the dense mini-block repetition/definition +budget would split the page or one top-level row exceeds that budget, and only when the value path is supported by the +sparse writer. Explicit `miniblock`, `fullzip`, and `sparse` requests are not changed by this automatic policy. Lance +2.2 and earlier writers never select sparse. + +Unsupported sparse value paths, including dictionary values and variable-width packed structs, retain their dense +behavior. Writers normalize Arrow validity and list structure once. Within-budget dense pages do not build sparse +position/count plans. All-valid layers use null positions plus `empty`; all-null layers use valid positions plus +`empty`. Other layers choose the validity polarity with the lower semantic encoded cost, with ties using null +positions. Field metadata controls writer selection only: readers always use `PageLayout` to determine the layout of +an encoded page and must not use field metadata for that decision. + +Pages without a value payload keep the existing canonical `ConstantLayout`: structural-only types such as an empty +struct, and leaf pages whose visible values are all null, do not emit `SparseLayout`. An explicitly sparse page with +at least one non-null visible value does emit `SparseLayout`, even when all non-null values are equal. This boundary +avoids introducing a second structural-only representation without evidence that it improves the existing constant +encoding. + +#### Buffers and Selective Reads + +A sparse page contains the following physical buffers: + +| Buffer | Contents | +| ------ | -------- | +| 0 | Value chunk metadata, one 8-byte entry per chunk | +| 1 | Mini-block compressed value chunks without repetition or definition levels | +| 2+ | One buffer for each explicit position or count set, in structural-layer field order | + +Each value chunk metadata entry stores `(chunk_size / 8) - 1` as little-endian `u32`, followed by its visible value +count as little-endian `u32`. Chunk sizes must be positive multiples of 8 and fit this representation. The sum of +chunk sizes must equal buffer 1 exactly and the sum of chunk value counts must equal `num_visible_items`. A value chunk +contains at most 32,768 visible values. `num_buffers` describes the number of value buffers inside every chunk and +excludes the structural buffers. + +General-compressed sparse buffers use the existing length-prefixed LZ4 or Zstd representation and must not contain +another general-compression wrapper. SparseLayout does not impose additional size or descriptor-complexity limits on +otherwise representable buffers. + +Readers normalize structural metadata once, project requested top-level ranges through each layer, and read only value +chunks that intersect the resulting leaf ranges. When no leaf range remains, readers rebuild offsets and validity from +the structural plan without reading buffer 1. + +#### Caching and Point Reads + +Reader initialization loads buffer 0 and every explicit structural buffer, then validates and normalizes them into a +cached page plan. The cached state contains parsed value-chunk descriptors and prefix offsets, decoded semantic +position/count sets, validity, and the ordered structural layers. It does not contain value payload bytes from buffer +1. The plan is cached per field and page and reused by later scans, range reads, and takes. + +After that plan is cached, reading one primitive leaf value reads only the value chunk that contains it. A cold read +first loads the structural metadata and then the intersecting value chunk. Reading one top-level list or +fixed-size-list value may intersect multiple leaf chunks and reads each intersecting chunk. A selection whose projected +structure contains no leaf slots reads no value chunk. + +#### Validation + +Readers must reject malformed sparse metadata instead of inferring or repairing it. Required checks include: + +- physical buffer count, chunk-count bounds, and every checked offset/size range +- first-layer row domain, adjacent parent/child domain chaining, and terminal visible-value domain +- semantic set cardinality, explicit position ordering and bounds, and validity meaning +- exact `num_items` +- list non-empty positions being valid, count cardinality, positive counts, and child-count sum +- fixed-size-list dimension and checked child-domain multiplication +- value chunk byte/value sums, size representation and alignment, general-compression headers, descriptor buffer + count, and complete chunk consumption + +```protobuf +%%% proto.message.SparseLayout %%% +``` + +```protobuf +%%% proto.message.SparseStructuralLayer %%% +``` + +```protobuf +%%% proto.message.SparseValidityLayer %%% +``` + +```protobuf +%%% proto.message.SparseListLayer %%% +``` + +```protobuf +%%% proto.message.SparseFixedSizeListLayer %%% +``` + +```protobuf +%%% proto.message.SparseValiditySet %%% +``` + +```protobuf +%%% proto.message.SparsePositionSet %%% +``` + +```protobuf +%%% proto.message.SparseCountSet %%% +``` + ### Constant Page Layout This layout is used when all (visible) values in the page are the same scalar value. @@ -549,7 +691,7 @@ options. However, they can also be set in the field metadata in the schema. | `lance-encoding:dict-values-compression-level` | Integers (scheme dependent) | Varies by scheme | Compression level for dictionary values general compression | | `lance-encoding:general` | `off`, `on` | `off` | Whether to apply general compression. | | `lance-encoding:packed` | Any string | Not set | Whether to apply packed struct encoding (see above). | -| `lance-encoding:structural-encoding` | `miniblock`, `fullzip` | Not set | Force a particular structural encoding to be applied (only useful for testing purposes) | +| `lance-encoding:structural-encoding` | `miniblock`, `fullzip`, `sparse` | Not set | Force a structural encoding; `sparse` requires Lance 2.3. | ### Configuration Details diff --git a/docs/src/format/file/versioning.md b/docs/src/format/file/versioning.md index 2add82ef2fa..8024a46fbc8 100644 --- a/docs/src/format/file/versioning.md +++ b/docs/src/format/file/versioning.md @@ -5,10 +5,10 @@ major number is changed when the file format itself is modified while the minor strategy is modified. Newer versions will typically have better performance and compression but may not be readable by older versions of Lance. -In addition, the `next` alias points to an unstable format version and should not be used for production use cases. -Breaking changes could be made to unstable encodings and that would mean that files written with these encodings are -no longer readable by any newer versions of Lance. The `next` version should only be used for experimentation and -benchmarking upcoming features. +Any version explicitly labeled unstable, including the current 2.3 format and the `next` alias, should not be used for +production use cases. Unstable formats have no compatibility guarantee: breaking encoding changes may make files +written by one Lance build unreadable by later builds. They should only be used for experimentation and benchmarking +upcoming features. The `stable` and `next` aliases are resolved by the specific Lance release you are using. During a format rollout (for example, 2.3), prefer explicit version pinning for deterministic behavior across environments. @@ -21,7 +21,7 @@ The following values are supported: | 2.0 | 0.16.0 | Any | Rework of the Lance file format that removed row groups and introduced null support for lists, fixed size lists, and primitives | | 2.1 | 0.38.1 | Any | Enhances integer and string compression, adds support for nulls in struct fields, and improves random access performance with nested fields. | | 2.2 | None | Any | Adds support for newer nested type/encoding capabilities (including map support) and 2.2-era storage features. | -| 2.3 (unstable) | None | Any | Adds experimental encodings for upcoming features. | +| 2.3 (unstable) | None | Unspecified | Adds sparse structural pages and other experimental encodings. | | legacy | N/A | N/A | Alias for 0.1 | | stable | N/A | N/A | Alias for the default version for new datasets in the Lance release you are running. | | next | N/A | N/A | Alias for the latest unstable version in the Lance release you are running.| diff --git a/docs/src/format/index/index.md b/docs/src/format/index/index.md index 4a2e33c60a4..01970133208 100644 --- a/docs/src/format/index/index.md +++ b/docs/src/format/index/index.md @@ -98,7 +98,12 @@ Index segments are created and updated through a transactional process: 2. **Prepare the metadata**: Create an `IndexMetadata` message with: - `uuid`: The newly generated UUID - `name`: The index name (must match existing segments if adding to an existing index) - - `fields`: The column(s) being indexed + - `fields`: The columns the index depends on: the keyed column(s) it is searched on, followed + by any merely-carried columns named in `covering_fields`. `fields[0]` is always a keyed column. + - `covering_fields`: The trailing subset of `fields` whose values the index carries but is not + keyed on, letting a query that only projects those columns be answered without a fragment take. + Empty for an index that carries no extra columns. Declaring a column here does not by itself + make it servable -- see [Serving carried columns](#serving-carried-columns). - `fragment_bitmap`: The set of fragment IDs covered by this segment - `index_details`: Index-specific configuration and parameters - `version`: The format version of this index type @@ -108,10 +113,11 @@ Index segments are created and updated through a transactional process: in its `IndexSection`. This is done atomically using the same transaction mechanism as data writes. -When updating an indexed column in place (without deleting the row), the engine must -remove the affected fragment IDs from the `fragment_bitmap` field of any index segments -that cover those fragments. This marks those fragments as needing re-indexing without -invalidating the entire segment and prevents invalid data from being read from the index. +When updating a column in place (without deleting the row), the engine must remove the +affected fragment IDs from the `fragment_bitmap` field of any index segment whose `fields` +include that column — whether the index is keyed on it or merely carries it. This marks +those fragments as needing re-indexing without invalidating the entire segment and prevents +invalid data from being read from the index. ## Index Compatibility @@ -129,6 +135,25 @@ Before using an index segment, engines must verify they support it: When an engine cannot use an index segment, it should fall back to scanning the fragments that would have been covered by that segment. +### Serving carried columns + +`IndexMetadata.covering_fields` records the columns an index segment *declares* it +carries. It does not establish that the segment's storage holds their values. + +**The segment's storage schema is authoritative.** Before answering a query from a +carried column, an engine must confirm that column is present in the storage it opened, +and fall back to a take against the base table when it is not. A segment whose +declaration names a column its storage does not hold is a legal state, not corruption: +a maintenance operation that cannot carry the payload through a rebuild is permitted to +withdraw it and leave the declaration standing. + +!!! note "Current state" + + No index builder writes carried values yet, so today every declaration is ahead of + its storage. Engines that read `covering_fields` must therefore treat it purely as a + declaration and serve every column from the base table until they have verified the + storage themselves. This is transitional; the rule above is not. + ## Loading an index When loading an index: @@ -147,7 +172,12 @@ When loading an index: The `IndexMetadata` message contains important information about the index segment: - `uuid`: the unique identifier of the index segment. -- `fields`: the column(s) the index is built on. +- `fields`: the columns the index depends on: the keyed column(s) the index is searched on, followed + by any columns it merely carries, as named in `covering_fields`. `fields[0]` is always a keyed column. +- `covering_fields`: the trailing subset of `fields` whose values the index carries alongside its own + data but is not keyed on. Empty for an index that carries no extra columns. This declaration is + not authoritative for what the segment can serve -- see + [Serving carried columns](#serving-carried-columns). - `fragment_bitmap`: the set of fragment IDs covered by this index segment. - `index_details`: a protobuf `Any` message that contains index-specific details, such as index type, parameters, and storage format. This allows different index types to store their own metadata. @@ -177,7 +207,7 @@ or updated. These should be filtered out during query execution. -There are three situations to consider: +There are four situations to consider: 1. **A fragment has some deleted rows.** A few of the rows in the fragment have been marked as deleted, but some of the rows are still present. The row addresses from the deletion @@ -185,9 +215,27 @@ There are three situations to consider: 2. **A fragment has been completely deleted.** This can be detected by checking if a fragment ID present in the fragment bitmap is missing from the dataset. Any row addresses from this fragment should be filtered out. -3. **A fragment has had the indexed column updated in place.** This cannot be detected just - by examining metadata. To prevent reading invalid data, the engine should filter out any +3. **A fragment has had one of the index's columns updated in place.** This cannot be detected + just by examining metadata. To prevent reading invalid data, the engine should filter out any row addresses that are not in the index's current `fragment_bitmap`. + The column need not be one the index is keyed on: every column in `fields` counts, including + the merely-carried ones named in `covering_fields`. A carried column can be updated while the + keyed column is untouched, and a segment left covering that fragment would answer from an + obsolete carried value. +4. **A fragment has an updated value in an [overlay file](../table/data_overlay_file.md).** + This can be detected by checking if any of the fragments in the index's `fragment_bitmap` + have overlay files. For each overlay whose `committed_version` is greater than the index + segment's `dataset_version`, the overlay carries updated values not reflected in the index, + so its covered rows must be excluded from index results. Excluded rows are re-evaluated + against their current (overlaid) values on the flat path — dropping them without + re-evaluation would silently lose rows that match under the new value. Exclusion is + field-aware: only overlays covering a column in the index's `fields` matter — keyed or + merely carried. Restricting this to the keyed column would leave a fragment covered after + an overlay updated a carried one, and the index would then serve a stale carried value. + You may exclude just the affected rows or the whole fragment; the latter is simpler and + safer but re-evaluates more rows than necessary. + See [Data Overlay Files](../table/data_overlay_file.md#index-integration) + for the exclusion set, re-evaluation, and correctness invariant. ## Compaction and remapping @@ -220,7 +268,8 @@ logical identifier that remains constant even when rows are moved during compact **Benefits:** - No remapping needed after compaction -- Updates only invalidate the index if the indexed column data changes +- Updates only invalidate the index if data in one of its `fields` changes — the keyed + column(s) or any column named in `covering_fields` **Tradeoffs:** diff --git a/docs/src/format/index/scalar/.pages b/docs/src/format/index/scalar/.pages index ba297222b07..98e5ebda5cb 100644 --- a/docs/src/format/index/scalar/.pages +++ b/docs/src/format/index/scalar/.pages @@ -7,4 +7,5 @@ nav: - Bloom Filter: bloom_filter.md - Full Text Search: fts.md - N-gram: ngram.md + - FM-Index: fmindex.md - RTree: rtree.md diff --git a/docs/src/format/index/scalar/bloom_filter.md b/docs/src/format/index/scalar/bloom_filter.md index 5a0e08e4228..6d924f8705d 100644 --- a/docs/src/format/index/scalar/bloom_filter.md +++ b/docs/src/format/index/scalar/bloom_filter.md @@ -4,6 +4,9 @@ Bloom filters are probabilistic data structures that allow for fast membership t They are space-efficient and can test whether an element is a member of a set. It's an inexact filter - they may include false positives but never false negatives. +In addition, since finding NULLs is a common query pattern, the index also maintains a +bitmap of null rows which allows it to return exact results for IS NULL queries. + ## Index Details ```protobuf @@ -32,6 +35,13 @@ The bloom filter index stores zone-based bloom filters in a single file: |---------------------------|--------|-------------------------------------------------------------| | `bloomfilter_item` | String | Expected number of items per zone (default: "8192") | | `bloomfilter_probability` | String | False positive probability (default: "0.00057", ~1 in 1754) | +| `null_bitmap` | UInt32 | Index of null bitmap global buffer | + +### Global Buffers + +| Metadata Key | Description | +|---------------------|------------------------------------------------------------| +| `null_bitmap` | A serialized RowAddrTreeMap specifying which rows are null | ## Bloom Filter Spec @@ -122,10 +132,11 @@ Offset 60-63: Block 1, Word 7 (32-bit LE) ## Accelerated Queries -The bloom filter index provides inexact results for the following query types: +The bloom filter index provides inexact results for the following query types (nullability queries +return exact results): | Query Type | Description | Operation | Result Type | |------------|---------------------------|-------------------------------------------|-------------| | **Equals** | `column = value` | Tests if value exists in bloom filter | AtMost | | **IsIn** | `column IN (v1, v2, ...)` | Tests if any value exists in bloom filter | AtMost | -| **IsNull** | `column IS NULL` | Returns zones where has_null is true | AtMost | \ No newline at end of file +| **IsNull** | `column IS NULL` | Returns zones where has_null is true | Exact | diff --git a/docs/src/format/index/scalar/fts.md b/docs/src/format/index/scalar/fts.md index adc7f94d65e..563d7ebf994 100644 --- a/docs/src/format/index/scalar/fts.md +++ b/docs/src/format/index/scalar/fts.md @@ -34,6 +34,15 @@ An FTS index may contain multiple partitions. Each partition has its own set of | `_rowid` | UInt64 | false | Document row ID | | `_num_tokens` | UInt32 | false | Number of tokens in the document | +Partitioned `docs.lance` files may include the optional schema metadata key +`total_tokens`. Its decimal `UInt64` value is the sum of `_num_tokens` in that +file. `_num_tokens` remains the canonical per-document data. Readers use the +metadata value to construct exact corpus statistics without scanning the column; +when the key is absent, they compute the sum from `_num_tokens`. Writers produce +the key from the same document table in the same file commit. A present value +that cannot be parsed, or that differs when `_num_tokens` is subsequently +loaded, is file corruption. + ### FTS List File Schema | Column | Type | Nullable | Description | @@ -43,6 +52,8 @@ An FTS index may contain multiple partitions. Each partition has its own set of | `_length` | UInt32 | false | Number of documents containing the token | | `_compressed_position` | List> | true | Optional compressed position lists for phrase queries | +The posting-list file schema metadata includes `posting_block_size`, the number of documents encoded per compressed posting block. Older indexes that do not have this metadata use the legacy block size `128`. + ### Metadata File Schema The metadata file contains JSON-serialized configuration and partition information: @@ -67,6 +78,7 @@ The metadata file contains JSON-serialized configuration and partition informati | `min_gram` | UInt32 | 2 | Minimum n-gram length (only for ngram tokenizer) | | `max_gram` | UInt32 | 15 | Maximum n-gram length (only for ngram tokenizer) | | `prefix_only` | Boolean | false | Generate only prefix n-grams (only for ngram tokenizer) | +| `block_size` | UInt32 | 128 | Documents per compressed posting block. Must be 128 or 256. Missing values from older indexes read as 128. `256` is experimental and may introduce breaking changes. | ## Tokenizers diff --git a/docs/src/format/index/scalar/zonemap.md b/docs/src/format/index/scalar/zonemap.md index 256edc2671d..552d476cc86 100644 --- a/docs/src/format/index/scalar/zonemap.md +++ b/docs/src/format/index/scalar/zonemap.md @@ -8,6 +8,9 @@ zones that cannot contain matching values. Zone maps are "inexact" filters - they can definitively exclude zones but may include false positives that require rechecking. +In addition, since finding NULLs is a common query pattern, the index also maintains a +bitmap of null rows which allows it to return exact results for IS NULL queries. + ## Index Details ```protobuf @@ -34,17 +37,25 @@ The zone map index stores zone statistics in a single file: ### Schema Metadata -| Key | Type | Description | -|-----------------|--------|-------------------------------------------| -| `rows_per_zone` | String | Number of rows per zone (default: "8192") | +| Key | Type | Description | +|---------------------|--------|-------------------------------------------| +| `rows_per_zone` | String | Number of rows per zone (default: "8192") | +| `null_bitmap` | UInt32 | Index of null bitmap global buffer | + +### Global Buffers + +| Metadata Key | Description | +|---------------------|------------------------------------------------------------| +| `null_bitmap` | A serialized RowAddrTreeMap specifying which rows are null | ## Accelerated Queries -The zone map index provides inexact results for the following query types: +The zone map index provides inexact results for the following query types (nullability queries +return exact results): | Query Type | Description | Operation | Result Type | |------------|---------------------------|---------------------------------------------|-------------| | **Equals** | `column = value` | Includes zones where min ≤ value ≤ max | AtMost | | **Range** | `column BETWEEN a AND b` | Includes zones where ranges overlap | AtMost | | **IsIn** | `column IN (v1, v2, ...)` | Includes zones that could contain any value | AtMost | -| **IsNull** | `column IS NULL` | Includes zones where null_count > 0 | AtMost | \ No newline at end of file +| **IsNull** | `column IS NULL` | Includes zones where null_count > 0 | Exact | diff --git a/docs/src/format/index/system/mem_wal.md b/docs/src/format/index/system/mem_wal.md index 44515f0af28..2f0a5cb1fa5 100644 --- a/docs/src/format/index/system/mem_wal.md +++ b/docs/src/format/index/system/mem_wal.md @@ -1,7 +1,7 @@ # MemWAL Index The MemWAL Index is a system index that serves as the centralized structure for all MemWAL metadata. -It stores configuration (shard specs, indexes to maintain), merge progress, and shard state snapshots. +It stores configuration (shard specs, indexes to maintain), SSTable compaction progress, and shard state snapshots. A table has at most one MemWAL index. The table may be a primary-key table or an append-only table without primary-key metadata. diff --git a/docs/src/format/index/vector/index.md b/docs/src/format/index/vector/index.md index 7aaf9b55996..91d73a23a16 100644 --- a/docs/src/format/index/vector/index.md +++ b/docs/src/format/index/vector/index.md @@ -68,7 +68,7 @@ The index file stores the search structure with graph or flat organization. The Arrow schema of the Lance file varies depending on the sub-index type used. !!! note -All partitions are stored in the same file, and partitions must be written in order. + All partitions are stored in the same file, and partitions must be written in order. ##### FLAT @@ -84,12 +84,12 @@ HNSW (Hierarchical Navigable Small World) indices provide fast approximate searc | Column | Type | Nullable | Description | | ------------- | ------------- | -------- | ---------------------- | -| `__vector_id` | uint64 | false | Vector identifier | -| `__neighbors` | list | false | Neighbor node IDs | -| `_distance` | list | false | Distances to neighbors | +| `__vector_id` | uint32 | true | Vector identifier | +| `__neighbors` | list | true | Neighbor node IDs | +| `_distance` | list | true | Distances to neighbors | !!! note -HNSW consists of multiple levels, and all levels must be written in order starting from level 0. + HNSW consists of multiple levels, and all levels must be written in order starting from level 0. #### Arrow Schema Metadata @@ -111,8 +111,8 @@ References the IVF metadata stored in the Lance file global buffer. This value records the global buffer index, currently this is always "1". !!! note -Global buffer indices in Lance files are 1-based, -so you need to subtract 1 when accessing them through code. + Global buffer indices in Lance files are 1-based, + so you need to subtract 1 when accessing them through code. ##### "lance:flat" @@ -159,7 +159,7 @@ Since the auxiliary file stores the actual (quantized) vectors, the Arrow schema of the Lance file varies depending on the quantization method used. !!! note -All partitions are stored in the same file, and partitions must be written in order. + All partitions are stored in the same file, and partitions must be written in order. ##### FLAT @@ -167,17 +167,17 @@ No quantization applied - stores original vectors in their full precision: | Column | Type | Nullable | Description | | -------- | ------------------------ | -------- | ----------------------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `flat` | list[dimension] | false | Original vector values (list_size = vector dimension) | +| `_rowid` | uint64 | true | Row identifier | +| `flat` | list[dimension] | true | Original vector values (list_size = vector dimension) | ##### PQ Compresses vectors using product quantization for significant memory savings: -| Column | Type | Nullable | Description | -| ----------- | -------------- | -------- | ------------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `__pq_code` | list[m] | false | PQ codes (list_size = number of subvectors) | +| Column | Type | Nullable | Description | +| ----------- | ----------------------------------------------- | -------- | --------------------------------------------- | +| `_rowid` | uint64 | true | Row identifier | +| `__pq_code` | list[num_sub_vectors * num_bits / 8] | true | PQ codes, packed to `num_bits` per subvector | ##### SQ @@ -185,23 +185,28 @@ Compresses vectors using scalar quantization for moderate memory savings: | Column | Type | Nullable | Description | | ----------- | ---------------------- | -------- | --------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `__sq_code` | list[dimension] | false | SQ codes (list_size = vector dimension) | +| `_rowid` | uint64 | true | Row identifier | +| `__sq_code` | list[dimension] | true | SQ codes (list_size = vector dimension) | ##### RQ Compresses vectors using RabitQ with random rotation and binary quantization for extreme compression: -| Column | Type | Nullable | Description | -| -------------------- | ------------------------------------------------ | ------------------------ | --------------------------------------------------------------- | -| `_rowid` | uint64 | false | Row identifier | -| `_rabit_codes` | list[dimension / 8] | false | Binary quantized codes (1 bit per dimension, packed into bytes) | -| `__add_factors` | float32 | false | Additive correction factors for distance computation | -| `__scale_factors` | float32 | false | Scale correction factors for distance computation | -| `__error_factors` | float32 | false for `raw_query` | Error factors for raw-query lower-bound pruning | -| `__ex_codes` | list[ceil(dimension * (num_bits - 1) / 8)] | false for `num_bits > 1` | Extra RabitQ code bits for multi-bit RQ | -| `__add_factors_ex` | float32 | false for `num_bits > 1` | Additive correction factors for ex-code distance computation | -| `__scale_factors_ex` | float32 | false for `num_bits > 1` | Scale correction factors for ex-code distance computation | +| Column | Type | Nullable | Present when | Description | +| -------------------- | ------------------------------------------------ | -------- | --------------------------- | --------------------------------------------------------------- | +| `_rowid` | uint64 | true | always | Row identifier | +| `_rabit_codes` | list[ceil(code_dim / 8)] | true | always | Binary quantized codes (1 bit per dimension, packed into bytes) | +| `__add_factors` | float32 | true | always | Additive correction factors for distance computation | +| `__scale_factors` | float32 | true | always | Scale correction factors for distance computation | +| `__error_factors` | float32 | true | `raw_query` estimator | Error factors for raw-query lower-bound pruning | +| `__blocked_ex_codes` | list[next_multiple_of(code_dim, 64) * (num_bits - 1) / 8] | true | `num_bits > 1` | Extra RabitQ code bits for multi-bit RQ, in the blocked layout | +| `__add_factors_ex` | float32 | true | `num_bits > 1` | Additive correction factors for ex-code distance computation | +| `__scale_factors_ex` | float32 | true | `num_bits > 1` | Scale correction factors for ex-code distance computation | + +!!! note + Indexes written before the blocked ex-code layout store the same bits in + `__ex_codes`, sized `ceil(dimension * (num_bits - 1) / 8)`. Readers still + accept that column and repack it at load time; writers no longer emit it. #### Arrow Schema Metadata @@ -280,7 +285,7 @@ to rotate vectors before binary quantization: The rotation matrix has shape `[code_dim, code_dim]` where `code_dim` is the rotated vector dimension. IVF_RQ always stores the 1-bit binary sign code in `_rabit_codes`; for `num_bits > 1`, -the remaining `num_bits - 1` ex-code bits are stored in `__ex_codes` instead of widening the +the remaining `num_bits - 1` ex-code bits are stored in `__blocked_ex_codes` instead of widening the binary code path. New IVF_RQ indexes store raw-query estimator factors. `num_bits=1` indexes only store the binary-code factor columns; multi-bit indexes also store separate ex-code additive and scale factors. @@ -319,7 +324,7 @@ PQ uses 16 num_sub_vectors (m=16) with 8 num_bits per subvector, and distance ty ```python pa.schema([ pa.field("_rowid", pa.uint64()), - pa.field("__pq_code", pa.list(pa.uint8(), list_size=16)), # m subvector codes + pa.field("__pq_code", pa.list_(pa.uint8(), list_size=16)), # num_sub_vectors * num_bits / 8 = 16 * 8 / 8 ]) ``` @@ -327,7 +332,7 @@ pa.schema([ This example shows how an `IVF_RQ` index is physically laid out. Assume vectors have dimension 128, RQ uses 1 bit per dimension (`num_bits=1`), and distance type is "l2". For `num_bits > 1`, the -auxiliary schema also includes `__ex_codes`, `__add_factors_ex`, and `__scale_factors_ex`. +auxiliary schema also includes `__blocked_ex_codes`, `__add_factors_ex`, and `__scale_factors_ex`. #### Index File @@ -356,7 +361,7 @@ auxiliary schema also includes `__ex_codes`, `__add_factors_ex`, and `__scale_fa ```python pa.schema([ pa.field("_rowid", pa.uint64()), - pa.field("_rabit_codes", pa.list(pa.uint8(), list_size=16)), # dimension/8 = 128/8 = 16 bytes + pa.field("_rabit_codes", pa.list_(pa.uint8(), list_size=16)), # ceil(code_dim / 8) = ceil(128 / 8) pa.field("__add_factors", pa.float32()), pa.field("__scale_factors", pa.float32()), pa.field("__error_factors", pa.float32()), diff --git a/docs/src/format/table/.pages b/docs/src/format/table/.pages index 16c20058608..5b0cb0e95e6 100644 --- a/docs/src/format/table/.pages +++ b/docs/src/format/table/.pages @@ -6,4 +6,5 @@ nav: - Layout: layout.md - Branch & Tag: branch_tag.md - Row ID & Lineage: row_id_lineage.md + - Data Overlay Files: data_overlay_file.md - MemTable & WAL: mem_wal.md diff --git a/docs/src/format/table/data_overlay_file.md b/docs/src/format/table/data_overlay_file.md new file mode 100644 index 00000000000..f6c4a5ed452 --- /dev/null +++ b/docs/src/format/table/data_overlay_file.md @@ -0,0 +1,395 @@ +# Data Overlay Files + +!!! warning "Experimental" + + This feature is currently experimental and not yet supported in any library. + + + +!!! note "Overlay files require feature flag 64 (data overlay files)" + + A reader or writer that does not understand overlay files must refuse a + dataset that uses them. Silently ignoring an overlay would return stale base + values, which is a correctness bug rather than a degraded experience. + +Overlay files supply new values for a subset of `(row offset, field)` cells +within a fragment **without rewriting the fragment's base data files**. They make +updates cheap when only a small fraction of rows and/or columns change: instead +of rewriting whole columns or moving rows to a new fragment, a writer appends a +small file carrying just the changed cells. + +This is Lance's third mechanism for changing data in place, alongside +[deletion files](index.md#deletion-files) (which remove rows) and +[data evolution](index.md#data-evolution) (which adds or rewrites whole columns). +An overlay changes individual cells. + +## Concepts + +### Coverage and resolution + +Each overlay declares which cells it provides through a **coverage** bitmap (or, +for sparse overlays, one bitmap per field). The bitmaps index **physical row +offsets**. They include deleted rows and are stable even as deletion vectors change. + +To resolve a cell `(offset, field)` on read, walk the fragment's overlays from +**newest to oldest**. The first overlay that covers `(offset, field)` wins; its +value is used. If no overlay covers the cell, the value falls through to the base +data file (or is `NULL` if no base data file holds that field). + +Precedence among overlays is determined by: + +1. `committed_version` — higher wins (see [Versioning](#versioning-and-ordering)). +2. Position in `DataFragment.overlays` as a tiebreaker — a later entry is newer. + +A covered offset whose value is `NULL` overrides the cell **to** `NULL`. This is +distinct from an offset that is simply absent from the bitmap, which falls +through to the base. Coverage, not value-nullness, decides whether an overlay +applies. + +### Interaction with deletions + +Deletions take precedence over overlays. If a row offset is marked deleted in the +fragment's deletion file, any overlay value for that offset is dead and is +ignored, regardless of commit order. + +### Physical layout + +An overlay's data file stores **one value column per field**, in the order of +`data_file.fields`. It does **not** store a row-offset key column. The position of +a covered offset's value within its column is the **rank** of that offset in the +field's coverage bitmap — the number of set bits below it. Resolving a cell is a +rank lookup plus one value fetch, with no separate offset column to store or +search. + +Because different fields may cover different offset sets, the value columns of a +single sparse overlay may have **different lengths**. The Lance file format +permits columns of differing item counts within one file, so a sparse overlay is +representable as a single file. (See [Writer support](#writer-support) for the +current implementation status.) + +### Dense vs. sparse overlays + +A single overlay is one of two shapes: + +- **Dense (rectangular).** One `shared_offset_bitmap` applies to every field. Every + covered offset has a value for every field. This is the common case for a plain + `UPDATE`, where one `SET` list is applied to one set of rows. +- **Sparse.** A `FieldCoverage` carries one bitmap per field, used when different + fields cover different offset sets — for example a `MERGE` with multiple + `WHEN MATCHED` branches, where different rows update different columns. A dense + overlay would have to widen to the bounding rectangle and fill the untouched + cells with their current values (post-images), which for wide columns such as + embeddings means re-storing data that did not change. A sparse overlay stores + exactly the changed cells. + +## Protobuf + +
+DataOverlayFile protobuf message + +```protobuf +%%% proto.message.DataOverlayFile %%% +``` + +
+ +
+FieldCoverage protobuf message + +```protobuf +%%% proto.message.FieldCoverage %%% +``` + +
+ +## Versioning and ordering + +Overlays reuse the dataset version as their ordering clock rather than +introducing a separate generation counter. + +`committed_version` is the dataset version at which an overlay **became +effective** — the version of the commit that introduced it, **not** the version +it was read from. It is stamped at commit time and re-stamped if the commit is +retried, in the same way as the created-at / last-updated-at version sequences. + +This single value drives every ordering decision: + +- **Overlay vs. overlay** (read precedence): higher `committed_version` wins. +- **Overlay vs. index** (query correctness): an index records the + `dataset_version` it was built from. An index whose `dataset_version >= + committed_version` already incorporates the overlay. An overlay whose + `committed_version > index.dataset_version` is newer than the index and its + cells must be excluded from index results and re-evaluated. +- **Scheduler signal**: the gap between an overlay's `committed_version` and an + index's `dataset_version`, or between an overlay and the base, is a staleness + measure the compaction scheduler can use. + +!!! note "Why effective version, not read version" + + Suppose an overlay reads version 5 and commits at version 6, while an index + is built reading version 5 (before the overlay) and commits at version 7 with + `dataset_version = 5`. If the overlay stored its *read* version (5), the test + `5 > 5` is false, the row would not be excluded, and the index — which never + saw the overlay — would return a stale result. Storing the *effective* + version (6) makes `6 > 5` true, the cell is excluded and re-evaluated, and the + result is correct. + +## Index integration + +Building an index over a fragment that has overlays does **not** require dropping +the fragment from the index's coverage. The fragment stays indexed, and the query +path reconciles overlays at query time using an **exclusion set**. + +The exclusion set for an index on field `F` is the union of the coverage bitmaps, +restricted to field `F`, of every overlay whose `committed_version > +index.dataset_version`. The exclusion is **field-aware**: an overlay that touches +only unrelated columns does not exclude anything from the index on `F`. + +`F` here ranges over every field in the index's `fields`, not only the ones it is +keyed on. An index that carries columns it is not keyed on (see +[`covering_fields`](../index/index.md#serving-carried-columns)) depends on those +columns too: an overlay updating a merely-carried column leaves the keyed value +correct while making the carried value stale, so it must exclude those rows just +the same. + +The query then proceeds as: + +1. Run the index search as usual, producing candidate rows. +2. Remove any candidate in the exclusion set. (Its indexed value may be stale.) +3. **Re-evaluate** the excluded rows against their current values — the same flat + path already used for the unindexed tail of fragments. For a scalar predicate + this re-applies the filter; for a vector query it re-scores the row's current + vector. Rows that still match are added back to the result. + +Step 3 is what makes exclusion correct rather than merely safe: removing a row +from index candidates without re-evaluating it would silently drop a row that +should match under its new value. + +Exclusion is always *sufficient* because a write changes a cell only by adding an +overlay, and that overlay's `committed_version` — the version of the commit that +adds it — necessarily exceeds the `dataset_version` of any pre-existing index. So +every cell a write changes is guaranteed to fall in that index's exclusion set. +Compaction may remove an overlay only if no index still relies on it for exclusion +(see [Compaction](#compaction)). + +## Compaction + +Overlays accumulate read cost — every overlay is a bitmap to test, a possible +file to open, and additional work to interleave values. Compaction bounds that cost in two modes: + +- **Overlay → overlay.** Merge several overlays into fewer, computing the + post-image per `(offset, field)` by walking the merged overlays newest-first. + The merged overlay takes the **maximum** `committed_version` of its inputs, so + the exclusion semantics are preserved. The merged overlays must be **contiguous + in `committed_version`** — with overlays at v10, v30, and v50 you cannot merge + just v10 and v50, because stamping the result v50 would incorrectly promote + v10's values above the intervening v30 for any cell v30 also covers. Indexes can + still be re-used, but they may now need to exclude more rows. This is cheap to + write and does not touch the base. +- **Overlay → base.** Fold overlays into a fresh base data file, computing the + post-image for every covered cell, then clear the overlays. The base is + complete, so every post-image is well defined. Overlay offsets are physical, so + they cannot survive a rewrite that reorders rows; folding therefore materializes + values rather than carrying overlays forward. + +!!! warning "Folding an indexed field must update its index" + + An overlay→base fold removes the overlay, which removes the exclusion signal + that kept an index correct. Folding an overlay that covers an indexed field + `F` is therefore equivalent to a column rewrite of `F` and must, in the same + commit, either rebuild the index to a `dataset_version` at least the folded + overlay's `committed_version`, or remove the fragment from the index's + coverage so the rows fall to the flat path. Otherwise the index would serve + stale values with no overlay to exclude them. This is the same rule that + already governs rewriting a column that an index is built on. + +When a fragment with overlays is compacted by a row-rewriting operation +(`RewriteRows`, which produces new fragments with new row addresses), the +overlays are folded into the new base as part of the rewrite, and existing +[fragment-reuse remapping](row_id_lineage.md) handles the row-address changes as +it does today. + +## Row lineage + +An overlay write updates the `last_updated_at_version` of every covered row, so +change-data-feed and time-travel queries observe the update. Because overlays are +addressed by physical offset, they do **not** require stable row IDs to be +enabled; lineage updates apply only when those features are on. + +## Worked example + +The following example illustrates how overlays function across their lifecycle, to make the rules above concrete. + +A table `users` with stable row IDs enabled and these fields: + +| field id | name | type | +|----------|-----------|-------------------------| +| 1 | id | `int32` (primary key) | +| 2 | name | `utf8` | +| 3 | age | `int32` | +| 4 | embedding | `fixed_size_list`| + +Created at version 1 as a single fragment `0` with one base data file +`data/file0.lance` holding all four columns. `physical_rows = 4`: + +| offset | id | name | age | embedding | +|--------|----|-------|-----|------------------| +| 0 | 1 | Alice | 30 | … | +| 1 | 2 | Bob | 25 | … | +| 2 | 3 | Carol | 40 | … | +| 3 | 4 | Dave | 22 | … | + +A BTree scalar index on `age` is built at version 1, covering fragment `0` +(`dataset_version = 1`). + +### Step 1 — write an overlay + +```sql +UPDATE users SET age = age + 1 WHERE id IN (2, 4); -- Bob (offset 1), Dave (offset 3) +``` + +This touches one field (`age`) for two rows, so the writer emits a dense overlay — +one shared bitmap covering both offsets — and commits it as version 2. Fragment +`0` gains: + +```text +DataOverlayFile { + data_file: { path: "data/overlay-.lance", fields: [3], column_indices: [0] } + coverage: shared_offset_bitmap = {1, 3} + committed_version: 2 +} +``` + +The overlay file stores a single `age` column with two values, `[26, 23]`, at +ranks `{1,3}.rank(1) = 0` and `{1,3}.rank(3) = 1`. `last_updated_at_version` is +set to 2 for offsets 1 and 3. + +### Step 2 — read + +`SELECT id, age FROM users` reads base ages `[30, 25, 40, 22]`. For `age` +(field 3), the overlay covers offsets 1 and 3, so `age[1]` is replaced with the +overlay value at rank `{1,3}.rank(1) = 0` → `26`, and `age[3]` with the value at +rank `{1,3}.rank(3) = 1` → `23`. Result ages: `[30, 26, 40, 23]`. + +### Step 3 — index query + +```sql +SELECT * FROM users WHERE age = 26; +``` + +The `age` index was built at `dataset_version = 1`; the overlay's +`committed_version` is 2. Since `2 > 1`, the overlay's coverage for `age`, `{1, 3}`, +is the exclusion set for this query. + +- The index (built at v1) holds Bob's *old* `age = 25`, so a lookup for `26` + returns nothing from the index. +- The whole exclusion set is re-evaluated on the flat path, not just the rows the + index returned. Offset 1's current `age` (26, via the overlay) matches, so Bob + is returned; offset 3's current `age` (23) does not match and is dropped. + +The mirror case `WHERE age = 25` shows exclusion preventing a stale hit: the index +returns offset 1 (stale `25`), but offset 1 is excluded, re-evaluated to `26`, and +correctly dropped. + +### Step 4 — a second, non-rectangular write + +```sql +MERGE INTO users USING staged ON users.id = staged.id +WHEN MATCHED AND staged.kind = 'rename' THEN UPDATE SET name = staged.name -- Carol(2), Dave(3) +WHEN MATCHED AND staged.kind = 'embed' THEN UPDATE SET embedding = staged.embedding -- Bob(1) +``` + +`name` is updated for offsets `{2, 3}` and `embedding` for offset `{1}` — different +fields over different rows. This is a sparse overlay, committed as version 3: + +```text +DataOverlayFile { + data_file: { path: "data/overlay-.lance", fields: [2, 4], column_indices: [0, 1] } + coverage: field_coverage { offset_bitmaps: [ {2,3}, {1} ] } + // name (field 2) ^ ^ embedding (field 4) + committed_version: 3 +} +``` + +The file's `name` column has **two** values (`["Caroline", "David"]`, at +ranks 0 and 1 of `{2,3}`) and its `embedding` column has **one** value (at rank 0 +of `{1}`) — columns of different lengths in one file. + +### Step 5 — read after the second write + +`SELECT name, age, embedding FROM users` resolves each field independently, +newest overlay first: + +- `name`: the v3 overlay covers `{2,3}` → `["Alice", "Bob", "Caroline", "David"]`. +- `age`: the v3 overlay does not cover `age`; the v2 overlay still applies at + offsets 1 and 3 → `[30, 26, 40, 23]`. +- `embedding`: the v3 overlay covers `{1}` → Bob's vector is the new one, others + from base. + +Overlays from different versions coexist and apply per field. + +### Step 6 — compaction (overlay → base) + +The scheduler folds both overlays into fragment `0` at version 4, computing +post-images for `age`, `name`, and `embedding`, and writing a new base data file +`data/file1.lance` with those columns. In the old file, fields 2, 3, and 4 are +marked with a tombstone (`-2`); field 1 (`id`) remains. The fragment's `overlays` list is +cleared. Row addresses are preserved (a column rewrite, not a row rewrite), so +stable row IDs and the deletion vector are untouched. + +Because the fold removed the overlay that was excluding offsets 1 and 3 from the +`age` index, the commit must drop fragment `0` from its coverage so `age` queries +fall to the flat path. + +## Guidance + +!!! note "This section is a stub." + + The following are implementation considerations, not part of the on-disk + specification. + +### When to overlay vs. rewrite a column vs. move rows + + + +*(To be expanded.)* The choice between appending an overlay, rewriting a full +column (data evolution), and moving updated rows to a new fragment depends on the +fraction of rows changed, the fraction of columns changed, column width, the +presence of indexes on the changed columns, and the accumulated overlay read +cost. Roughly: few rows changed favors overlays; most rows in a few columns +favors a column rewrite; most columns changed favors moving rows to a new +fragment. + +### Writer support + + + +*(To be expanded.)* Dense (rectangular) overlays write with the existing +equal-length file writer today. Sparse overlays stored as a **single** file +require the writer to emit columns of independent lengths, which the current v2 +writer does not yet do (it advances all columns from one global row counter). +Until that support lands, a writer can express a sparse update as multiple dense +overlays in one transaction. + +### Scheduling compaction + + + +*(To be expanded.)* The overlay→overlay and overlay→base modes have very +different costs; a cost/benefit scheduler decides when each is worthwhile, using +the version gap as a staleness signal. + +## Related specifications + +- [Table format overview](index.md) +- [Transactions: DataOverlay operation](transaction.md#dataoverlay) — write path + and conflict semantics +- [Row ID & Lineage](row_id_lineage.md) +- [Index Formats: handling overlay rows](../index/index.md#handling-deleted-and-invalidated-rows) +- [Format Versioning](versioning.md) diff --git a/docs/src/format/table/index.md b/docs/src/format/table/index.md index 94ea4b90dc9..f9da132cf3b 100644 --- a/docs/src/format/table/index.md +++ b/docs/src/format/table/index.md @@ -168,6 +168,29 @@ However, this invalidates row addresses and requires rebuilding indices, which c +## Data Overlay Files + +!!! warning "Experimental" + + This feature is currently experimental and not yet supported in any library. + + + +!!! note "Overlay files require feature flag 64 (data overlay files)" + +Overlay files supply new values for a subset of cells within +a fragment without rewriting the base data files. They make updates cheap when only +a small percentage of rows and/or columns change: a writer appends a small file +carrying just the changed cells instead of rewriting whole columns or moving rows +to a new fragment. + +For the full specification — coverage and resolution rules, dense vs. sparse layout, +versioning, index integration, compaction, and a worked example — see the +[Data Overlay Files Specification](data_overlay_file.md). + + + ## Related Specifications ### Storage Layout diff --git a/docs/src/format/table/mem_wal.md b/docs/src/format/table/mem_wal.md index 8a228721123..e8a5f5fce35 100644 --- a/docs/src/format/table/mem_wal.md +++ b/docs/src/format/table/mem_wal.md @@ -7,684 +7,692 @@ scan, point lookup, vector search and full-text search. ![MemWAL Overview](../../images/mem_wal_overview.png) -A Lance table is called a **base table** under the context of the MemWAL spec. -It may have an [unenforced primary key](index.md#unenforced-primary-key) defined in the table schema. -Primary keys are required for primary-key lookups and last-write-wins upsert semantics, -but append-only MemWAL tables may omit them. +A Lance table is called the **base table** in this document. +The base table may have an [unenforced primary key](index.md#unenforced-primary-key) in its schema. +Primary keys are required for primary-key lookups and last-write-wins upsert semantics. +Append-only MemWAL tables may omit a primary key. -On top of the base table, the MemWAL spec defines a set of shards. -Writers write to shards, and data in each shard is merged into the base table asynchronously. -An index is kept in the base table for readers to quickly discover the state of all shards at a point of time. +MemWAL adds a set of shards on top of the base table. +Writers append to shards. +Each shard keeps recent data in an in-memory MemTable, persists writes to a per-shard WAL, flushes MemTables as small Lance datasets, and later compacts those SSTables into the base table. + +The base table manifest contains one MemWAL system index entry named `__lance_mem_wal`. +This index stores MemWAL configuration and global progress metadata inline in `IndexMetadata.index_details`. +Each shard's own manifest remains authoritative for shard-local mutable state. ### MemWAL Shard -A **MemWAL Shard** is the main unit to horizontally scale out writes. +A **MemWAL shard** is the unit of horizontal write scaling. +Each shard has exactly one active writer epoch at a time. +Writers claim a shard, append WAL entries, update the in-memory MemTable, and publish SSTable generations by updating the shard manifest. -Each shard has exactly one active writer at any time. -Writers claim a shard and then write data to that shard. -Data in each shard is expected to be merged into the base table asynchronously. +For primary-key tables, all rows for the same primary key must map to the same shard. +If one primary key can appear in multiple shards, asynchronous compaction order between shards can make an older row overwrite a newer row. +Append-only tables without a primary key do not rely on last-write-wins conflict resolution and may use any deterministic shard assignment suitable for the workload. -For tables with a primary key, rows of the same primary key must be written to one and only one shard. -If two shards contain rows with the same primary key, the following scenario can cause data corruption: +### MemWAL Index -1. Shard A receives a write with primary key `pk=1` at time T1 -2. Shard B receives a write with primary key `pk=1` at time T2 (T2 > T1) -3. The row in shard B is merged into the base table first -4. The row in shard A is merged into the base table second -5. The row from Shard A (older) now overwrites the row from Shard B (newer) +The MemWAL index is a system index entry on the base table. +It has `name = "__lance_mem_wal"`, no indexed fields, and no index files. +`IndexMetadata.files` is `None`. +All MemWAL index data is stored in the `MemWalIndexDetails` protobuf message in `IndexMetadata.index_details`. -This violates the expected "last write wins" semantics. -By ensuring each primary key is assigned to exactly one shard via the sharding spec, -merge order between shards becomes irrelevant for correctness. -Append-only tables without a primary key do not rely on last-write-wins conflict resolution -and may shard by any deterministic append key or partitioning column. +The index stores: -See [MemWAL Shard Architecture](#shard-architecture) for the complete shard architecture. +- **Configuration**: `sharding_specs`, `maintained_indexes`, and `writer_config_defaults`. +- **Compaction progress**: `compacted_sstables`, the last SSTable compacted into the base table for each shard. +- **Index catchup progress**: `index_catchup`, the compacted SSTable generation covered by each base-table index. +- **Shard snapshots**: optional point-in-time snapshot fields for read optimization. -### MemWAL Index +Shard snapshots are not authoritative. +Readers that need the latest shard set list `_mem_wal/` and read each shard's latest manifest. -A **MemWAL Index** is the centralized structure for all MemWAL metadata on top of a base table. -A table has at most one MemWAL index. It stores: +## Shard Architecture -- **Configuration**: Sharding specs defining how rows map to shards, and which indexes to maintain -- **Merge progress**: Last generation merged to base table for each shard -- **Index catchup progress**: Which merged generation each base table index has been rebuilt to cover -- **Shard snapshots**: Point-in-time snapshot of shard states for read optimization +![Shard Architecture](../../images/mem_wal_shard.png) -The index is the source of truth for **configuration**, **merge progress** and **index catchup progress** -Writers and mergers read the MemWAL index to get these configurations before writing. +Within a shard, writes first enter an in-memory **MemTable** and are durably appended to the shard **write-ahead log (WAL)**. +The MemTable is periodically **flushed** to storage as a Lance dataset. +SSTables are asynchronously **compacted** into the base table. -Each [shard's manifest](#shard-manifest) is authoritative for its own state. -Readers may use **shard snapshots** as a read-only optimization to see a point-in-time view of shards without opening each shard manifest. -Readers that need the latest shard set must discover shard directories in storage and read each shard's latest manifest. +### MemTable -See [MemWAL Index Details](#memwal-index-details) for the complete structure. +A MemTable holds rows inserted into a shard before those rows are flushed to storage. +It serves two purposes: -## Shard Architecture +1. It buffers data and per-MemTable indexes before an SSTable is written. +2. It lets readers access data that has not been flushed yet when strong consistency is required. -![Shard Architecture](../../images/mem_wal_regional.png) +The storage format does not prescribe the in-memory MemTable layout. +Conceptually, a MemTable is an append log of Arrow record batches. +Later appends have larger in-memory row positions. +For primary-key tables, in-memory reads use the largest visible row position as the newest row for a key. -Within a shard, writes are stored in an **in-memory table (MemTable)**. -It is also written to the shard's **Write-Ahead Log (WAL)** for durability guarantee. -The MemTable is periodically **flushed** to storage based on memory pressure and other conditions. -**Flushed MemTables** in storage are then asynchronously **merged** into the base table. +### SSTable Generation -### MemTable +Within each shard, SSTables have monotonically increasing generation numbers starting from 1. +When a MemTable is flushed, the resulting SSTable is assigned the shard manifest's `current_generation`, and `current_generation` advances to the next SSTable generation. +A MemTable does not have a generation. -A MemTable holds rows inserted into the shard before flushing to storage. -It serves 2 purposes: +SSTable generation numbers order persisted data freshness within one shard: -1. build up data and related indexes to be flushed to storage as a flushed MemTable -2. allow a reader to potentially access data that is not flushed to storage yet +- Base table data is modeled as generation 0. +- Higher SSTable generations are newer. +- The active MemTable is newer than every published SSTable. +- Within the active MemTable, higher row positions are newer. +- Within an SSTable, flush-time deletion vectors hide older duplicate primary-key rows, so readers see at most the newest row for each primary key. -#### MemTable Format +## WAL -The complete in-memory format of a MemTable is implementation-specific and out of the scope of this spec. -The Lance core Rust SDK maintains one default implementation and is available through all its language binding SDKs, -but integrations are free to build their own MemTable format depending on the specific use cases, -as long as it follows the MemWAL storage layout, reader and writer requirements when flushing MemTable. +The WAL is the durable append log for a shard. +Every durable WAL append creates one **WAL entry**. -Conceptually, because Lance uses [Arrow as its in-memory data exchange format](https://arrow.apache.org/docs/format/index.html), -for the ease of explanation in this spec, we will treat MemTable as a list of Arrow record batches, -and each write into the MemTable is a new Arrow record batch. +### WAL Entry Positions -#### MemTable Generation +WAL entry positions are 1-based. +The first data entry is position 1. +Position 0 is reserved as the sentinel value meaning no WAL entry has been covered. -Based on conditions like memory limit and durability requirements, -a MemTable needs to be **flushed** to storage and discarded. -When that happens, new writes go to a new MemTable and the cycle repeats. -Each MemTable is assigned a monotonically increasing generation number starting from 1. -When MemTable of generation `N` is discarded, the next MemTable gets assigned generation `N+1`. +Writers append WAL entries in increasing position order. +If entry `N` is not fully written, entry `N + 1` must not exist. +Recovery replays from `replay_after_wal_entry_position + 1`. -### WAL +### WAL Entry Format -WAL serves as the durable storage of all MemTables in a shard. -It consists of data in MemTables ordered by generation. -Every time we write to the WAL, we call it a **WAL Flush**. +Each WAL entry is an Apache Arrow IPC stream file. +The Arrow schema metadata includes: -#### WAL Durability +- `writer_epoch`: decimal string containing the writer epoch that created the entry. +- `fence_sentinel`: optional marker for a data-less fence sentinel entry. -When a write is flushed to WAL, the specific write becomes durable. -Otherwise, if the MemTable is lost, data is also lost. +A normal WAL entry contains one or more record batches. +A fence sentinel entry contains no batches and is skipped during replay. +Sentinels are used so an older writer collides on the next WAL position and discovers that it has been fenced. -Multiple writes can be batched together in a single WAL flush to reduce WAL flush frequency and improve throughput. -The more writes a single WAL flush batches, the longer it takes for a write to be durable. +### WAL Storage Layout -The whole LSM tree's durability is determined by the durability of the WAL. -For example, if WAL is stored in Amazon S3, it has 99.999999999% durability. -If it is stored in local disk, the data will be lost if the local disk is damaged. +WAL entries live under `_mem_wal/{shard_id}/wal/`. +Filenames use bit-reversed 64-bit binary names with the `.arrow` suffix: -#### WAL Entry +```text +_mem_wal/{shard_id}/wal/{bit_reversed_position}.arrow +``` -Each time a WAL flush happens, it adds a new **WAL Entry** to the WAL. -In other words, a WAL consists of an ordered list of WAL entries starting from position 0. -Writer must flush WAL entries in sequential order from lower to higher position. -If WAL entry `N` is not flushed fully, WAL entry `N+1` must not exist in storage. +The bit-reversal spreads sequential positions across object-store keyspace. +For example, position 5 is encoded as: -#### WAL Replay +```text +1010000000000000000000000000000000000000000000000000000000000000.arrow +``` -**Replaying** a WAL means to read data in the WAL from a lower to a higher position. -This is commonly used to recover the latest MemTable after it is lost, -by reading from the start position of the latest MemTable generation till the highest position in the WAL, -assuming proper fencing to guard against multiple writers to the same shard. +## SSTable -See [Writer Fencing](#writer-fencing) for the full fencing mechanism. +An SSTable is the immutable result of flushing a MemTable. +It is stored as a Lance dataset under its shard directory. -#### WAL Entry Format +!!! note + Unlike a classic LSM sorted string table, a MemWAL SSTable is not sorted by key; random access is instead served by its BTree primary-key sidecar. It is called an SSTable because it is an immutable, persisted, indexed run. -Each WAL entry is a file in storage following the [Apache Arrow IPC stream format](https://arrow.apache.org/docs/format/Columnar.html#ipc-streaming-format) to store the batch of writes in the MemTable. -The writer epoch is stored in the stream's Arrow schema metadata with key `writer_epoch` for fencing validation during replay. +### SSTable Storage Layout -#### WAL Storage Layout +An SSTable with generation `i` is written to: -Each WAL entry is stored within the WAL directory of the shard located at `_mem_wal/{shard_id}/wal`. +```text +_mem_wal/{shard_id}/{random8}_gen_{i}/ +``` -WAL files use bit-reversed 64-bit binary naming to distribute files evenly across the directory keyspace. -This optimizes S3 throughput by spreading sequential writes across S3's internal partitions, minimizing throttling. -The filename is the bit-reversed binary representation of the entry ID with suffix `.arrow`. -For example, entry ID 5 (binary `000...101`) becomes `1010000000000000000000000000000000000000000000000000000000000000.arrow`. +`{random8}` is an 8-character random hex value generated for each flush attempt. +If a flush attempt fails, a retry writes a different directory instead of reusing a partially written one. +The shard manifest records the successful directory name in the SSTable's `path`. + +The SSTable directory is a standard Lance dataset written with the base table's data storage version. +Each SSTable is written as one fragment. +Additional MemWAL sidecars may be present: + +```text +{random8}_gen_{i}/ +├── _versions/ +│ └── {version}.manifest +├── _deletions/ # Present when within-SSTable dedup deletes rows +├── _indices/ # Present when maintained user indexes are built +│ └── {index_uuid}/ +├── _pk_index/ # Primary-key sidecar BTree, not a manifest index +└── bloom_filter.bin # Primary-key bloom filter +``` -### Flushed MemTable +The exact Lance dataset internals follow the [Lance table storage layout](layout.md). -A flushed MemTable is created by flushing the MemTable to storage. -In Lance MemWAL spec, a flushed MemTable must be a Lance table following the Lance table format spec. +### SSTable Row Order -!!!note -This is called Sorted String Table (SSTable) or Sorted Run in many LSM-tree literatures and implementations. -However, since our MemTable is not sorted, we just use the term flushed MemTable to avoid confusion. +SSTable rows are written in forward insert order. +Physical row offsets increase with write time. +For a duplicate primary key within one SSTable, the newest row has the largest physical offset. -#### Flushed MemTable Storage Layout +Primary-key SSTables use a deletion vector to expose last-write-wins semantics. +During flush, the writer scans rows in forward order, keeps the last occurrence of each primary key, and marks all earlier duplicate offsets deleted. +The deletion vector is attached to fragment 0 in the SSTable's Lance manifest. -The MemTable of generation `i` is flushed to `_mem_wal/{shard_id}/{random_hex}_gen_{i}/` directory, -where `{random_hex}` is a random 8-character hex value generated at flush time. -The random hex value is necessary to ensure if one MemTable flush attempt fails, -The retry can use another directory. -The content within the generation directory follows the [Lance table storage layout](layout.md). +Append-only SSTables without a primary key do not perform primary-key deduplication and retain every row. -#### Merging MemTable to Base Table +### Tombstone Rows -Generation numbers determine merge order of flushed MemTable into base table: -lower numbers represent older data and must be merged to the base table first to preserve correct upsert semantics. +Delete operations are represented as rows with the internal `_tombstone` column. +Tombstone rows follow the same forward row ordering and deletion-vector rules as ordinary rows. +If the newest row for a primary key is a tombstone, the deletion vector keeps that tombstone row and hides older rows for the key. +Read planning then filters `_tombstone = false`, so the key is absent from query results. -Within a single flushed MemTable for a primary-key table, -if there are multiple rows of the same primary key, the row that is last inserted wins. -Append-only tables without a primary key retain all inserted rows. +### SSTable Primary-Key Sidecars -### Shard Manifest +Primary-key MemTables maintain an implicit BTree for primary-key deduplication, independent of `maintained_indexes`. +When a primary-key MemTable is flushed, the SSTable writes two primary-key sidecars: -Each shard has a manifest file. This is the source of truth for the state of a shard. +- `bloom_filter.bin` stores the SSTable's primary-key bloom filter and lets point lookups skip SSTables that cannot contain the queried key. +- `_pk_index/` stores a standalone BTree over primary-key values to forward row ids. -#### Shard Manifest Contents +The `_pk_index/` sidecar is not a maintained user index, is not registered in the SSTable's Lance manifest, and has no manifest UUID. +Its identity is its immutable SSTable path. +Readers open it directly from `{sstable_path}/_pk_index`. -The manifest contains: +The `_pk_index/` directory is a Lance scalar BTree index store: -- **Fencing state**: `writer_epoch` as the latest writer fencing token, see [Writer Fencing](#writer-fencing) for more details. -- **Shard assignment**: `shard_spec_id` and `shard_field_values` record how this shard maps to its sharding spec. `shard_field_values` is a map from shard field id to the raw Arrow scalar bytes of the computed value; the matching `ShardingField.result_type` in the `ShardingSpec` determines how to interpret each entry (e.g., 4 little-endian bytes for int32, raw UTF-8 bytes for utf8). -- **WAL pointers**: `replay_after_wal_entry_position` (last entry position flushed to MemTable, 0-based), `wal_entry_position_last_seen` (last entry position seen at manifest update, 0-based) -- **Generation trackers**: `current_generation` (next generation to flush), `flushed_generations` list of generation number and directory path pairs (e.g., generation 1 at `a1b2c3d4_gen_1`) +```text +_pk_index/ +├── page_data.lance +└── page_lookup.lance +``` -Note: `wal_entry_position_last_seen` is a hint that may be stale since it's not updated on WAL write. -It is updated opportunistically by any reader that can update the shard manifest. -The manifest itself is atomically written, but recovery must try to get newer WAL files to find the actual state beyond this hint. +Readers load this directory as a BTree index using `BTreeIndexDetails` with default parameters. +The primary-key index type is the Arrow type of the primary-key column for a single-column primary key, or `Binary` for a composite primary key. -The manifest is serialized as a protobuf binary file using the `ShardManifest` message. +The `page_lookup.lance` file has the following schema: -
-ShardManifest protobuf message +| Column | Type | Nullable | Description | +|--------------|-------------------------|----------|------------------------------------------------| +| `min` | {PrimaryKeyIndexType} | true | Minimum primary-key index value in the page | +| `max` | {PrimaryKeyIndexType} | true | Maximum primary-key index value in the page | +| `null_count` | UInt32 | false | Number of null values in the page | +| `page_idx` | UInt32 | false | Page number pointing into `page_data.lance` | -```protobuf -%%% mem_wal.message.ShardManifest %%% -``` +The `page_data.lance` file has the following schema: -
+| Column | Type | Nullable | Description | +|----------|-----------------------|----------|-------------------------------------------------------------------| +| `values` | {PrimaryKeyIndexType} | true | Sorted primary-key index values | +| `ids` | UInt64 | false | Forward row ids corresponding to each primary-key index value | -#### Shard Manifest Versioning +For a single-column primary key, the indexed value stores the primary-key scalar directly. +For a composite primary key, the indexed value stores an order-preserving binary tuple encoding of all primary-key columns in primary-key column order. +Each tuple column is encoded as: -Manifests are versioned starting from 1 and immutable. -Each update creates a new manifest file at the next version number. -Updates use put-if-not-exists or file rename to ensure atomicity depending on the storage system. -If two processes compete, one wins and the other retries. +- `0x00` for null. +- `0x01` followed by the non-null value encoding otherwise. -To commit a manifest version: +Supported non-null value encodings are: -1. Compute the next version number -2. Write the manifest to `{bit_reversed_version}.binpb` using put-if-not-exists -3. In parallel best-effort write to `version_hint.json` with `{"version": }` (failure is acceptable) +- Signed integers and date values: sign-flipped 8-byte big-endian integer bytes. +- Unsigned integers: 8-byte big-endian unsigned integer bytes. +- Boolean: one byte, `0x00` for false and `0x01` for true. +- UTF-8 and binary values: raw bytes, with each `0x00` byte escaped as `0x00 0xff`, followed by a `0x00 0x00` terminator. -To read the latest manifest version: +This encoding is injective and preserves primary-key tuple ordering under lexicographic byte comparison. +Composite primary-key columns must use one of the supported encodings above. -1. Read `version_hint.json` to get the latest version hint. If not found, start from version 1 -2. Check existence for subsequent versions from the starting version -3. Continue until a version is not found -4. The latest version is the last found version +The sidecar row ids are in the same forward row-position space as the data files, deletion vector, and maintained user indexes. +The sidecar is used for cross-generation membership and block-list checks. +It is not used to choose the newest row inside the same SSTable; the deletion vector has already hidden older same-generation duplicates. -!!!note -This works because the write rate to shard manifests is significantly lower than read rates. Shard manifests are only updated when shard metadata changes (MemTable flush), not on every write. This ensures HEAD requests will eventually terminate and find the latest version. +### Maintained User Indexes -#### Shard Manifest Storage Layout +When the MemWAL index lists `maintained_indexes`, flush may build matching indexes inside the SSTable. +These index files live in the SSTable's `_indices/{index_uuid}/` directory and are recorded in the SSTable's Lance manifest. +The implicit primary-key BTree sidecar is not included in `maintained_indexes` and does not live under `_indices/`. -All shard manifest versions are stored in `_mem_wal/{shard_id}/manifest` directory. +These indexes use the same row-position space as the forward-written data files. +If the SSTable has a primary key, its deletion vector masks stale duplicate rows for indexed reads as well. -Each shard manifest version file uses bit-reversed 64-bit binary naming, the same scheme as WAL files. -For example, version 5 becomes `1010000000000000000000000000000000000000000000000000000000000000.binpb`. +### SSTable Compaction -## MemWAL Index Details +SSTables are compacted into the base table in ascending generation order within each shard. +Lower generation numbers are older and must be compacted before higher generation numbers. +SSTable compaction uses merge-insert semantics so newer rows overwrite older rows for the same primary key. + +## Shard Manifest + +Each shard has a versioned manifest. +The latest shard manifest is the source of truth for shard-local state. + +### Shard Manifest Contents + +The manifest contains: + +- **Identity**: `shard_id`, `shard_spec_id`, and `shard_field_entries`. +- **Fencing state**: `writer_epoch`. +- **WAL pointers**: `replay_after_wal_entry_position` and `wal_entry_position_last_seen`. +- **SSTable generation state**: `current_generation` and `sstables`. +- **Lifecycle state**: `status`, either `ACTIVE` or `SEALED`. -The MemWAL Index uses the [standard index storage](../index/index.md#index-storage) at `_indices/{UUID}/`. +`shard_field_entries` stores computed shard field values as raw Arrow scalar bytes keyed by `ShardingField.field_id`. +The matching `ShardingField.result_type` determines how to decode each value. +For example, `int32` values are four little-endian bytes and `utf8` values are raw UTF-8 bytes. -The index stores its data in two parts: +`replay_after_wal_entry_position` is the most recent 1-based WAL position covered by an SSTable. +The default value 0 means no WAL entry has been covered and recovery starts at position 1. -1. **Index details** (`index_details` in `IndexMetadata`): Contains configuration, merge progress, and snapshot metadata -2. **Shard snapshots**: Stored as a Lance file or inline, depending on shard count +`wal_entry_position_last_seen` is a best-effort hint for the most recent WAL position observed at manifest update time. +It is not authoritative because it is not updated on every WAL write. +Recovery must still probe or list WAL files to find the actual tail. -### Index Details +`current_generation` is the generation number to assign to the next SSTable created by flushing the MemTable. +Each entry in `sstables` records a published SSTable's `generation` and `path`. -The `index_details` field in `IndexMetadata` contains a `MemWalIndexDetails` protobuf message with the following key fields: +`status = SEALED` marks a reversible in-flight drop-table operation. +Sealed shards refuse new writer claims. -- **Configuration fields** (`sharding_specs`, `maintained_indexes`) are the source of truth for MemWAL configuration. - Writers read these fields to determine how to partition data and which indexes to maintain. -- **Merge progress** (`merged_generations`) tracks the last generation merged to the base table for each shard. - This field is updated atomically with merge-insert data commits, enabling conflict resolution when multiple mergers operate concurrently. - Each entry contains the shard UUID and generation number. -- **Index catchup progress** (`index_catchup`) tracks which merged generation each base table index has been rebuilt to cover. - When data is merged from a flushed MemTable to the base table, the base table's indexes may be rebuilt asynchronously. - During this window, queries should use the flushed MemTable's pre-built indexes instead of scanning unindexed data in the base table. - See [Indexed Read Plan](#indexed-read-plan) for details. -- **Shard snapshot fields** (`snapshot_ts_millis`, `num_shards`, `inline_snapshots`) provide a snapshot of shard states. - The actual shard manifests remain authoritative for shard state. - When `num_shards` is 0, the `inline_snapshots` field may be `None` or an empty Lance file with 0 rows but proper schema. +The manifest is serialized as the `ShardManifest` protobuf message.
-MemWalIndexDetails protobuf message +ShardManifest protobuf message ```protobuf -%%% mem_wal.message.MemWalIndexDetails %%% +%%% mem_wal.message.ShardManifest %%% ```
-### Shard Identifier +### Shard Manifest Versioning -Each shard has a unique UUID identifier within the table. -When a new shard is created, implementations may assign either a random UUID or -a deterministic UUID derived from the shard assignment when deterministic -writer fencing is required. +Manifest versions start at 1. +Each update writes a new immutable protobuf file: -### Shard Discovery +```text +_mem_wal/{shard_id}/manifest/{bit_reversed_version}.binpb +``` -The MemWAL index can store shard snapshots for read optimization, but those snapshots may lag the latest shard set. -Implementations that need to discover the current shard set should list `_mem_wal/` shard directories and read each shard's latest [shard manifest](#shard-manifest). +Writers use put-if-not-exists or atomic rename, depending on storage support. +If two processes race to write the same next version, one wins and the other reloads and retries. -Each shard manifest records the shard UUID, sharding spec ID, and computed shard field values needed to map the shard back to a sharding spec assignment. +After a successful version write, the writer best-effort updates: -### Sharding Spec +```json +{"version": } +``` -A **Sharding Spec** defines how all rows in a table are logically divided into different shards, -enabling automatic shard assignment and query-time shard pruning. +in: -Each sharding spec has: +```text +_mem_wal/{shard_id}/manifest/version_hint.json +``` -- **Spec ID**: A positive integer that uniquely identifies this spec within the MemWAL index. IDs are never reused. -- **Sharding fields**: An array of field definitions that determine how to compute shard values. +Readers use `version_hint.json` as a starting point and then probe subsequent versions until a version is missing. +The latest manifest is the last existing version. -Each shard is bound to a specific sharding spec ID, recorded in its [manifest](#shard-manifest). -Shards without a spec ID (`spec_id = 0`) are manually-created shards not governed by any spec. +## MemWAL Index Details -A sharding spec's field array consists of **sharding field** definitions. -Each sharding field has the following properties: +The MemWAL index is stored inline in the base table's `IndexMetadata`. +It is a system index with no file directory. +The `index_details` field contains a `MemWalIndexDetails` protobuf message. -| Property | Description | -| ------------- | ------------------------------------------------------------------------- | -| `field_id` | Unique string identifier for this sharding field | -| `source_ids` | Array of field IDs referencing source columns in the schema | -| `transform` | A well-known shard expression, specify this or `expression` | -| `expression` | A DataFusion SQL expression for custom logic, specify this or `transform` | -| `result_type` | The output type of the shard value | +Important fields: -#### Shard Expression +- `sharding_specs`: sharding configuration used by writers and shard pruning. +- `maintained_indexes`: names of base-table indexes to maintain in MemTables and SSTables. +- `writer_config_defaults`: string map of default writer configuration values persisted for all writers. +- `compacted_sstables`: per-shard compaction progress, updated atomically with base-table compaction commits. +- `index_catchup`: per-index coverage progress after data has been compacted into the base table. +- `snapshot_ts_millis`, `num_shards`, and `inline_snapshots`: optional shard snapshot fields for read optimization. -A **Shard Expression** is a [DataFusion SQL expression](https://datafusion.apache.org/user-guide/sql/index.html) that derives a shard value from source column(s). -Source columns are referenced as `col0`, `col1`, etc., corresponding to the order of field IDs in `source_ids`. +A shard absent from `index_catchup` for an index means that index is *not* known +to have caught up, so the shard's SSTables must be retained until some commit +records that it has. -Shard expressions must satisfy the following requirements: +Catch-up is derived at commit time, not reported by the writer. An index whose segments together span every fragment live at the transaction's read version holds every row compaction had copied into the base table by then, so the commit records it as caught up to that version's `compacted_sstables`. That is the only proof available — nothing maps a compaction generation to the fragments its rows landed in — so covering the table as the transaction read it is how an index shows it covered those rows. Fragments appended since that read are a later catch-up gap and are not required. -1. **Deterministic**: The same input value must always produce the same output value. -2. **Stateless**: The expression must not depend on external state (e.g., current time, random values, session variables). -3. **Type-promotion resistant**: The expression must produce the same result for equivalent values regardless of their numeric type (e.g., `int32(5)` and `int64(5)` must yield the same shard value). -4. **Column removal resistant**: If a source field ID is not found in the schema, the column should be interpreted as NULL. -5. **NULL-safe**: The expression should properly handle NULL inputs and have defined behavior (e.g., return NULL if input is NULL for single-column expressions). -6. **Consistent with result type**: The expression's return type must be consistent with `result_type` in non-NULL cases. +Two rules bound what a commit may record. It never credits more than its own `compacted_sstables`, and it clamps to that value, so a position can only describe generations the base table has actually taken in. Otherwise it never lowers a position an index already held, provided that index is unchanged by this commit. "Unchanged" compares each segment's UUID together with its fragment bitmap, not the UUID alone, because an operation can prune the bitmap in place while keeping the UUID; the remaining metadata does not affect which rows the index answers for. An index this commit changes keeps no position it cannot re-earn. -#### Shard Transform +Because the position is derived rather than transmitted, it cannot go stale between inspection and commit, and it survives rebase — `read_version` is fixed for a transaction's life, so what a commit can prove does not move, though a rebased attempt may record a different result because the head it commits against has changed. Any commit can earn a position, so an ordinary index build that happens to cover the table records catch-up as a side effect. A dedicated repair is still needed where no such commit occurs, or where an index does not yet span the table. -A **Shard Transform** is a well-known shard expression with a predefined name. -When a transform is specified, the expression is derived automatically. +A read version with no fragments proves nothing, even though an index trivially covers an empty table. An empty fragment list is also what a manifest written before the `UpdateMemWalState` fragment fix looks like, where the SSTables are the last copy of those rows; crediting coverage there would retire them. The cost is that a table whose rows have all been deleted keeps its SSTables. -| Transform | Parameters | Shard Expression | Result Type | -| -------------- | ------------- | --------------------------------------------------------- | -------------- | -| `identity` | (none) | `col0` | same as source | -| `year` | (none) | `date_part('year', col0)` | `int32` | -| `month` | (none) | `date_part('month', col0)` | `int32` | -| `day` | (none) | `date_part('day', col0)` | `int32` | -| `hour` | (none) | `date_part('hour', col0)` | `int32` | -| `bucket` | `num_buckets` | `abs(murmur3(col0)) % N` | `int32` | -| `multi_bucket` | `num_buckets` | `abs(murmur3_multi(col0, col1, ...)) % N` | `int32` | -| `truncate` | `width` | `left(col0, W)` (string) or `col0 - (col0 % W)` (numeric) | same as source | +Shard snapshots, when present, use the following Lance file schema: -The `bucket` and `multi_bucket` transforms use Murmur3 hash functions: +| Column | Type | Nullable | Description | +|----------------------------|------------------------------|----------|--------------------------------------------------------| +| `shard_id` | Utf8 | false | Shard UUID string | +| `shard_spec_id` | UInt32 | false | Sharding spec that produced the shard | +| `shard_field_{field_id}` | `ShardingField.result_type` | false | Computed shard field value for the given sharding field | -- **`murmur3(col)`**: Computes the 32-bit Murmur3 hash (x86 variant, seed 0) of a single column. Returns a signed 32-bit integer. Returns NULL if input is NULL. -- **`murmur3_multi(col0, col1, ...)`**: Computes the Murmur3 hash across multiple columns. Returns a signed 32-bit integer. NULL fields are ignored during hashing; returns NULL only if all inputs are NULL. +The MemWAL index data is stored inline. +Readers discover the latest shard set by listing `_mem_wal/` shard directories and reading shard manifests. -The hash result is wrapped with `abs()` and modulo `N` to produce a non-negative bucket number in the range `[0, N)`. +
+MemWalIndexDetails protobuf message -### Shard Snapshot Storage +```protobuf +%%% mem_wal.message.MemWalIndexDetails %%% +``` -Shard snapshots are stored using one of two strategies based on the number of shards: +
-| Shard Count | Storage Strategy | Location | -| ------------------ | ------------------- | ----------------------------------------- | -| <= 100 (threshold) | Inline | `inline_snapshots` field in index details | -| > 100 | External Lance file | `_indices/{UUID}/index.lance` | +## Sharding -The threshold (100 shards) is implementation-defined and may vary. +A **ShardingSpec** defines how rows map to shards. +Each spec has a positive `spec_id` and one or more `ShardingField` entries. +Each shard manifest records the `shard_spec_id` and the computed shard field values for that shard. +`spec_id = 0` means the shard was manually created and is not governed by a sharding spec. -**Inline storage**: For small shard counts, snapshots are serialized as a Lance file and stored in the `inline_snapshots` field. -This keeps the index metadata compact while avoiding an additional file read for common cases. +Each `ShardingField` contains: -**External Lance file**: For large shard counts, snapshots are stored as a Lance file at `_indices/{UUID}/index.lance`. -This file uses standard Lance format with the shard snapshot schema, enabling efficient columnar access and compression. +- `field_id`: stable identifier for the computed shard field. +- `source_ids`: field IDs of source columns in the Lance schema. +- `transform`: well-known transform name, when using built-in transform evaluation. +- `expression`: reserved custom expression text, mutually exclusive with `transform`. +- `result_type`: Arrow type name for the computed value. +- `parameters`: transform-specific string parameters. -### Shard Snapshot Arrow Schema +The supported built-in transforms are: -Shard snapshots are stored as a Lance file with one row per shard. -The snapshot schema is optimized for shard discovery. Full mutable shard state -remains in the authoritative shard manifest files. +- `unsharded`: takes no source columns, always returns `int32` value 0, and creates one shard. +- `bucket`: takes one source column and `num_buckets`, hashes the value, and returns an `int32` bucket id in `[0, num_buckets)`. +- `identity`: takes one source column and returns the raw scalar value as the shard value. -| Column | Type | Description | -| ------------------------ | ------------- | ---------------------------------------------------------------------------------------------------------- | -| `shard_id` | `utf8` | Shard UUID string | -| `shard_spec_id` | `uint32` | Sharding spec ID (0 if manual) | -| `shard_field_{field_id}` | varies | One column per sharding field defined in the sharding spec, typed to match the field's `ShardingField.result_type`. | +`bucket` computes a deterministic 32-bit hash with seed 0 and then computes: -For example, with a sharding spec containing a field `user_bucket` of type `int32`: +```text +(hash & i32::MAX) % num_buckets +``` -| Column | Type | Description | -| -------------------------- | ------- | ---------------------------- | -| ... | ... | (base columns above) | -| `shard_field_user_bucket` | `int32` | Bucket value for this shard | +`num_buckets` must be in `[1, 1024]`. +Null bucket values hash to 0 and therefore map to bucket 0. +See [Appendix 3: Bucket Hashing](#appendix-3-bucket-hashing) for the exact hash algorithm and test vectors. -This schema records the fields needed to map each shard back to its sharding spec -assignment. Readers that need fencing epochs, WAL positions, or flushed -generation state must read the latest shard manifests directly. +The `bucket` transform supports scalar boolean, integer, floating-point, date32, time, timestamp, utf8, and large_utf8 source types. +The `identity` transform supports scalar boolean, integer, utf8, and large_utf8 source types. + +The `year`, `month`, `day`, `hour`, `multi_bucket`, and `truncate` transform names are not supported MemWAL sharding transforms and must not be used in `ShardingSpec.transform`. ## Storage Layout -Here is a recap of the storage layout with all the files and concepts defined so far: +The MemWAL storage layout is: -``` +```text {table_path}/ +├── _versions/ +│ └── ... # Base table manifests, including __lance_mem_wal index metadata ├── _indices/ -│ └── {index_uuid}/ # MemWAL Index (uses standard index storage) -│ └── index.lance # Serialized shard snapshots (Lance file) -│ +│ └── ... # Ordinary base table index files; MemWAL index has no files └── _mem_wal/ - └── {shard_id}/ # Shard directory (UUID v4) + └── {shard_id}/ ├── manifest/ - │ ├── {bit_reversed_version}.binpb # Serialized shard manifest (bit-reversed naming) - │ └── version_hint.json # Version hint file + │ ├── {bit_reversed_version}.binpb + │ └── version_hint.json ├── wal/ - │ ├── {bit_reversed_entry_id}.arrow # WAL data files (bit-reversed naming) + │ ├── {bit_reversed_position}.arrow │ └── ... - └── {random_hash}_gen_{i}/ # Flushed MemTable (generation i, random prefix) + └── {random8}_gen_{generation}/ ├── _versions/ - │ └── {version}.manifest # Table manifest (V2 naming scheme) - ├── _indices/ # Indexes - │ ├── {vector_index}/ - │ └── {scalar_index}/ - └── bloom_filter.bin # Primary key bloom filter + │ └── {version}.manifest + ├── _deletions/ + ├── _indices/ + │ └── {index_uuid}/ + ├── _pk_index/ + └── bloom_filter.bin ``` -## Implementation Expectation - -This specification describes the storage layout for the LSM tree architecture. Implementations are free to use any approach to fulfill the storage layout requirements. Once data is written to the expected storage layout, the reader and writer expectations apply. - -The specification defines: +Some SSTable subdirectories are conditional. +For example, `_deletions/` is present only when the SSTable's Lance manifest references a deletion vector, `_indices/` is present only when maintained user indexes are built, and `_pk_index/` plus `bloom_filter.bin` are meaningful for primary-key tables. -- **Storage layout**: The directory structure, file formats, and naming conventions for WAL entries, flushed MemTables, shard manifests, and the MemWAL index -- **Durability guarantees**: How data is persisted through WAL entries and flushed MemTables -- **Consistency model**: How readers and writers coordinate through manifests and epoch-based fencing +## Implementation Expectation -Implementations may choose different approaches for: +This document specifies the storage layout and observable reader and writer invariants. +Implementations may choose different in-memory structures, buffering policies, background scheduling, and query execution plans. -- In-memory data structures and indexing -- Buffering strategies before WAL flush -- Background task scheduling and concurrency -- Query execution strategies +An implementation is compatible when it: -As long as the storage layout is correct and the documented invariants are maintained, implementations can optimize for their specific use cases. +1. Writes WAL entries, shard manifests, SSTables, and MemWAL index metadata using the documented layout. +2. Preserves WAL position, writer fencing, and manifest versioning invariants. +3. Exposes last-write-wins semantics for primary-key tables. +4. Preserves append-only semantics for tables without primary keys. +5. Maintains generation ordering when compacting SSTables into the base table. ## Writer Expectations -A writer operates on a single shard and is responsible for: +A writer operates on one shard and is responsible for: -1. Claiming the shard using epoch-based fencing -2. Writing data to WAL entries and flushed MemTables following the [storage layout](#storage-layout) -3. Maintaining the shard manifest to track WAL and generation progress +1. Claiming the shard with epoch-based fencing. +2. Appending WAL entries in sequential 1-based positions. +3. Maintaining in-memory MemTable state. +4. Flushing MemTables to SSTable Lance datasets. +5. Updating the shard manifest after an SSTable is durably written. ### Writer Fencing -Writers use epoch-based fencing to ensure single-writer semantics per shard. +Writers use `writer_epoch` to enforce single-writer semantics per shard. To claim a shard: -1. Load the latest shard manifest -2. Increment `writer_epoch` by one -3. Atomically write a new manifest version -4. If the write fails (another writer claimed the epoch), reload and retry with a higher epoch +1. Load the latest shard manifest. +2. Verify the shard is `ACTIVE`. +3. Increment `writer_epoch`. +4. Atomically write a new manifest version. +5. If the manifest write loses a race, reload and retry. -Before any manifest update, a writer must verify its `writer_epoch` remains valid: +Before a manifest update, a writer verifies its local epoch is still current: -- If `local_writer_epoch == stored_writer_epoch`: The writer is still active and may proceed -- If `local_writer_epoch < stored_writer_epoch`: The writer has been fenced and must abort +- If `local_writer_epoch == stored_writer_epoch`, the writer may proceed. +- If `local_writer_epoch < stored_writer_epoch`, the writer has been fenced and must abort. -For a concrete example, see [Appendix 1: Writer Fencing Example](#appendix-1-writer-fencing-example). +WAL append conflicts also detect fencing. +If an older writer collides with a newer writer's WAL entry at the same position, it reloads the manifest and observes the higher epoch. +Fence sentinel entries make this collision path explicit without storing data batches. ## Background Job Expectations -Background jobs handle merging flushed MemTables to the base table and garbage collection. - -### MemTable Merger +Background jobs compact SSTables into the base table and remove obsolete shard data. -Flushed MemTables must be merged to the base table in **ascending generation order** within each shard. This ordering is essential for correct upsert semantics: newer generations must overwrite older ones. +### SSTable Compactor -The merge uses Lance's merge-insert operation with atomic transaction semantics: +SSTables must be compacted into the base table in ascending generation order within each shard. +The compaction uses Lance merge-insert semantics and updates `compacted_sstables[shard_id]` atomically with the base-table commit. -- `merged_generations[shard_id]` is updated atomically with the data commit -- On commit conflict, check the conflicting commit's `merged_generations` to determine if the generation was already merged +On commit conflict, a compactor reloads the conflicting base-table version: -For a concrete example, see [Appendix 2: Concurrent Merger Example](#appendix-2-concurrent-merger-example). +- If the committed `compacted_sstables[shard_id]` is already greater than or equal to the generation being compacted, the compactor skips that generation. +- Otherwise, the compactor retries from the latest base-table version. ### Garbage Collector -The garbage collector removes obsolete data from shard directories. Flushed MemTables and their referenced WAL files may be deleted after: +The garbage collector may remove obsolete SSTables after: -1. The generation has been merged to the base table (`generation <= merged_generations[shard_id]`) -2. All maintained indexes have caught up (`generation <= min(index_catchup[I].caught_up_generation)`) -3. No retained base table version references the generation for time travel +1. The SSTable has been compacted into the base table. +2. Every index a query may rely on has caught up to cover the SSTable's generation, or the SSTable is no longer needed for indexed reads. An index absent from `index_catchup` has *not* caught up, so this condition is not met for it. +3. No retained base-table version needs the SSTable for time travel or consistency. -!!!warning - Deleting WAL files weakens [writer fencing](#writer-fencing) and can lead to silent acknowledgement of lost writes. +!!! warning + Deleting WAL files can weaken writer fencing. - Fencing detects a stalled writer when its `put-if-not-exists` for the next WAL entry collides with a newer writer's entry at the same position — only that collision triggers the epoch check. If GC has already removed the WAL file at that position, the stalled writer's PUT lands on empty space and succeeds against its old `writer_epoch`. The entry is acknowledged to the client, but the new manifest's `replay_after_wal_entry_position` has already advanced past it, so the data is never replayed. - - Implementations that GC WAL files must compensate, for example by re-checking fence state after each successful WAL write, encoding the writer epoch into the WAL filename so positions are partitioned by epoch, or otherwise guaranteeing a stalled writer cannot land at a position that has been or will be GC'd. + Fencing detects a stalled writer when its put-if-not-exists for the next WAL entry collides with a newer writer's entry at the same position. + If garbage collection has removed that WAL file, the stalled writer may write into empty space with an old `writer_epoch`. + Implementations that garbage collect WAL files must compensate by re-checking fence state after WAL writes, partitioning WAL positions by epoch, or otherwise preventing stale writers from landing at positions that have been garbage collected. ## Reader Expectations ### LSM Tree Merging Read -For tables with a primary key, readers **MUST** merge results from multiple data sources -(base table, flushed MemTables, in-memory MemTables) by primary key to ensure correctness. - -When the same primary key exists in multiple sources, the reader must keep only the newest version based on: - -1. **Generation number** (`_gen`): Higher generation wins. The base table has generation 0, MemTables have positive integers starting from 1. -2. **Row address** (`_rowaddr`): Within the same generation, higher row address wins (later writes within a batch overwrite earlier ones). +For primary-key tables, readers merge rows from the base table, SSTables, and optionally in-memory MemTables by primary key. +The newest row wins. -The ordering for "newest" is: highest `_gen` first, then highest `_rowaddr`. +Freshness ordering within one shard is: -This deduplication is essential because: +1. The active MemTable wins over every SSTable and the base table. +2. Among SSTables, higher generation wins. +3. Any uncompacted SSTable wins over the base table. +4. Within the active MemTable, higher row position wins. +5. Within an SSTable, its deletion vector has already hidden older duplicate primary-key rows. -- A row updated in a MemTable also exists (with older data) in the base table -- A flushed MemTable that has been merged to the base table may not yet be garbage collected, causing the same row to appear in both -- A single write batch may contain multiple updates to the same primary key - -Without proper merging, queries would return duplicate or stale rows. +For freshness comparisons, the base table uses the sentinel generation 0. +SSTable generations are positive. +This ordering applies only to sources selected for the same read plan. +Readers must not include an SSTable that is already covered by the base table according to `compacted_sstables[shard_id]`, because otherwise the positive SSTable generation would incorrectly outrank base-table rows during deduplication. +Rows from different shards do not need primary-key deduplication if the sharding spec guarantees that each primary key maps to exactly one shard. Append-only tables without a primary key do not perform primary-key deduplication. -Readers should include the relevant base table, flushed MemTables, and in-memory MemTables -according to the requested consistency level; duplicate values are treated as distinct appended rows. +Rows from all selected sources are distinct appended rows. -### Reader Consistency +### Tombstones -Reader consistency depends on two factors: +Readers must treat `_tombstone = true` rows as delete markers. +In SSTables, deletion vectors first resolve same-generation duplicate primary keys. +Then query planning filters tombstone rows from user-visible results. +In active in-memory MemTables, the newest visible row position for a primary key wins; if that row is a tombstone, the key is absent. -1. access to in-memory MemTables -2. the source of shard metadata (either through MemWAL index or shard manifests) - -Strong consistency requires access to in-memory MemTables for all shards involved in the query and reading shard manifests directly. -Otherwise, the query is eventually consistent due to missing unflushed data or stale MemWAL Index snapshots. - -!!!note -Reading a stale MemWAL Index does not impact correctness, only freshness: +### Reader Consistency - - **Merged MemTable still in index**: If a flushed MemTable has been merged to the base table but still shows in the MemWAL index, readers query both. This results in some inefficiency for querying the same data twice, but [LSM-tree merging](#lsm-tree-merging-read) ensures correct results since both contain the same data. The inefficiency is also compensated by the fact that the data is covered by index and we rarely end up scanning both data. - - **Garbage collected MemTable still in index**: If a flushed MemTable has been garbage collected, but is still in the MemWAL index, readers would fail to open it and skip it. This is also safe because if it is garbage collected, the data must already exist in the base table. - - **Newly flushed MemTable not in index**: If a newly flushed MemTable is added after the snapshot was built, it is not queried. The result is eventually consistent but correct for the snapshot's point in time. +Reader consistency depends on: -### Query Planning +1. Whether the reader can access active in-memory MemTables. +2. Whether shard metadata comes from latest shard manifests or from an older MemWAL index snapshot. -#### MemTable Collection +Strong consistency requires active in-memory MemTable access for relevant shards and direct reads of latest shard manifests. +Otherwise, reads are eventually consistent because unflushed data or newly-created shards may be absent from the read plan. -The query planner collects datasets from multiple sources and assembles them for unified query execution. -Datasets come from: +Reading a stale MemWAL index snapshot does not corrupt last-write-wins ordering, but it can reduce freshness: -1. base table (representing already-merged data) -2. flushed MemTables (persisted but not yet merged) -3. optionally in-memory MemTables (if accessible). +- If a compacted SSTable is still listed, readers must skip it when `generation <= compacted_sstables[shard_id]`. + For primary-key tables, including it would let an older SSTable row outrank newer base-table contents because SSTable generations are positive and the base table is modeled as generation 0. + For append-only tables, including it would return the same append twice. +- If a garbage-collected SSTable is still listed, readers may skip it after failing to open it because its data must already be in the base table or be filtered out by `compacted_sstables`. +- If a new SSTable is not listed, the read is consistent with the older snapshot but may miss fresher data. -Each dataset is tagged with a generation number: 0 for the base table, and positive integers for MemTable generations. -Within a shard, the generation number determines data freshness, with higher numbers representing newer data. -For primary-key tables, rows from different shards do not need deduplication -since each primary key maps to exactly one shard. -Append-only tables without a primary key do not require cross-shard primary-key deduplication. +Readers that require latest shard membership should list `_mem_wal/` and read shard manifests instead of relying only on snapshots. -The planner also collects bloom filters from each generation for staleness detection during search queries. +### Query Planning -#### Shard Pruning +A query planner collects sources from: -Before executing queries, if sharding spec is available, -the planner evaluates filter predicates against sharding specs to determine which shards may contain matching data. -This pruning step reduces the number of shards to scan. +1. The base table. +2. SSTables that are not yet safely replaceable by base-table indexed reads. +3. Active in-memory MemTables, when available and required by the requested consistency level. -For each filter predicate: +Each source is tagged with its shard and freshness tier. +SSTable sources are also tagged with their generation. +For primary-key reads, the planner applies LSM deduplication across selected sources. +For append-only reads, the planner concatenates selected sources without primary-key deduplication. -1. Extract predicates on columns used in sharding specs -2. Evaluate which shard values can satisfy the predicate -3. Prune shards whose values cannot match +Bloom filters and `_pk_index/` sidecars help prune SSTables during point lookups and cross-generation deduplication. -For example, with a sharding spec using `bucket(user_id, 10)` and a filter `user_id = 123`: +### Shard Pruning -1. Compute `bucket(123, 10) = 3` -2. Only scan shards with bucket value 3 -3. Skip all other shards +When sharding specs are available, the planner evaluates query predicates against shard fields and skips shards whose computed shard values cannot match. -Shard pruning applies to both scan queries and prefilters in search queries. +For example, with `bucket(user_id, 10)` and predicate `user_id = 123`: -#### Indexed Read Plan +1. Compute the bucket id for `123`. +2. Scan only shards whose manifest has the same computed bucket value. +3. Skip all other bucket shards. -When data is merged from a flushed MemTable to the base table, the base table's indexes are rebuilt asynchronously by the base table index builders. -During this window, the merged data exists in the base table but is not yet covered by the base table's indexes. +### Indexed Read Plan -Without special handling, indexed queries would fall back to expensive full scans for the unindexed part of the base table. -To maintain indexed read performance, the query planner should use `index_catchup` progress to determine the optimal data source for each query. +When data is compacted from an SSTable into the base table, base-table indexes may lag behind the data commit. +`index_catchup` records which compacted generation each base-table index covers. -The key insight is that flushed MemTables serve as a bridge between the base table's index catchup and the current merged state. -For a query that requires a specific index for acceleration, when `index_gen < merged_gen`, -the generations in the gap `(index_gen, merged_gen]` have data already merged in the base table but are not covered by the base table's index. -Since flushed MemTables contain pre-built indexes (created during [MemTable flush](#flushed-memtable)), queries can use these indexes instead of scanning unindexed data in the base table. -This ensures all reads remain indexed regardless of how far behind the async index builder is. +If an indexed query needs index `I` and `I` has only caught up to generation `G` while `compacted_sstables[shard_id]` is higher, the planner should read the gap from SSTable indexes instead of scanning unindexed base-table rows. +Once index `I` catches up, the planner can use the base-table index for those compacted rows. ## Appendices ### Appendix 1: Writer Fencing Example -This example demonstrates how epoch-based fencing prevents data corruption when two writers compete for the same shard. - -#### Initial State +Initial shard manifest: +```text +version: 1 +writer_epoch: 5 +replay_after_wal_entry_position: 10 +wal_entry_position_last_seen: 12 +status: ACTIVE ``` -Shard manifest (version 1): - writer_epoch: 5 - replay_after_wal_entry_position: 10 - wal_entry_position_last_seen: 12 -``` - -#### Scenario - -| Step | Writer A | Writer B | Manifest State | -| ---- | --------------------------------------------- | ----------------------------------------- | ------------------ | -| 1 | Loads manifest, sees epoch=5 | | epoch=5, version=1 | -| 2 | Increments to epoch=6, writes manifest v2 | | epoch=6, version=2 | -| 3 | Starts writing WAL entries 13, 14, 15 | | | -| 4 | | Loads manifest v2, sees epoch=6 | epoch=6, version=2 | -| 5 | | Increments to epoch=7, writes manifest v3 | epoch=7, version=3 | -| 6 | | Starts writing WAL entries 16, 17 | | -| 7 | Tries to flush MemTable, loads manifest | | | -| 8 | Sees epoch=7, but local epoch=6 | | | -| 9 | **Writer A is fenced!** Aborts all operations | | | -| 10 | | Continues writing normally | epoch=7, version=3 | - -#### What Happens to Writer A's WAL Entries? - -Writer A wrote WAL entries 13, 14, 15 with `writer_epoch=6` in their schema metadata. - -When Writer B performs crash recovery or MemTable flush: - -1. Reads WAL entries sequentially starting from `replay_after_wal_entry_position + 1` (entry 11, since positions are 0-based) -2. For each entry, checks existence using HEAD request on the bit-reversed filename -3. Continues until an entry is not found (e.g., entry 18 doesn't exist) -4. Finds entries 13, 14, 15, 16, 17 -5. Reads each file's `writer_epoch` from schema metadata -6. Entries 13, 14, 15 have `writer_epoch=6` which is <= current epoch (7) -> **valid, will be replayed** -7. Entries 16, 17 have `writer_epoch=7` -> **valid, will be replayed** -#### Key Points +Writer A loads version 1, claims epoch 6, and writes manifest version 2. +It appends WAL entries 13, 14, and 15 with `writer_epoch = 6`. -1. **No data loss**: Writer A's entries are not discarded. They were written with a valid epoch at the time and will be included in recovery. +Writer B then loads version 2, claims epoch 7, and writes manifest version 3. +It appends WAL entries 16 and 17 with `writer_epoch = 7`. -2. **Consistency preserved**: Writer A is prevented from making further writes that could conflict with Writer B. +When Writer A later tries to flush or update the shard manifest, it reloads the manifest and sees stored epoch 7 while its local epoch is 6. +Writer A is fenced and must abort. -3. **Orphaned files are safe**: WAL files from fenced writers remain on storage and are replayed by the new writer. They are only garbage collected after being included in a flushed MemTable that has been merged. +Recovery starts from `replay_after_wal_entry_position + 1`, which is entry 11. +Entries 13, 14, 15, 16, and 17 are valid replay inputs because they were written by epochs that were valid at write time and are not greater than the current shard epoch. -4. **Epoch validation timing**: Writers check their epoch before manifest updates (MemTable flush), not on every WAL write. This keeps the hot path fast while ensuring consistency at commit boundaries. +### Appendix 2: Concurrent Compactor Example -### Appendix 2: Concurrent Merger Example +Initial state: -This example demonstrates how MemWAL Index and conflict resolution handle concurrent mergers safely. +```text +MemWAL index: + compacted_sstables: {shard: 5} -#### Initial State - -``` -MemWAL Index: - merged_generations: {shard: 5} - -Shard manifest (version 1): +Shard manifest: current_generation: 8 - flushed_generations: [(6, "abc123_gen_6"), (7, "def456_gen_7")] + sstables: + - generation: 6, path: "abc12345_gen_6" + - generation: 7, path: "def67890_gen_7" ``` -#### Scenario 1: Racing on the Same Generation - -Two mergers both try to merge generation 6 concurrently. - -| Step | Merger A | Merger B | MemWAL Index | -| ---- | ------------------------- | ------------------------------ | ---------------- | -| 1 | Reads index: merged_gen=5 | | merged_gen=5 | -| 2 | Reads shard manifest | | | -| 3 | Starts merging gen 6 | | | -| 4 | | Reads index: merged_gen=5 | merged_gen=5 | -| 5 | | Reads shard manifest | | -| 6 | | Starts merging gen 6 | | -| 7 | Commits (merged_gen=6) | | **merged_gen=6** | -| 8 | | Tries to commit | | -| 9 | | **Conflict**: reads new index | | -| 10 | | Sees merged_gen=6 >= 6, aborts | | -| 11 | | Reloads, continues to gen 7 | | +Two compactors both try to compact the SSTable at generation 6. +Compactor A commits first and updates `compacted_sstables[shard]` to 6 in the same base-table commit as the data. +Compactor B then hits a commit conflict, reloads the latest MemWAL index, sees `compacted_sstables[shard] >= 6`, skips generation 6, and continues with generation 7. -Merger B's conflict resolution detected that generation 6 was already merged by checking the MemWAL Index in the conflicting commit. +The MemWAL index is the authoritative compaction-progress record because it is committed atomically with the base-table data changes. -#### Scenario 2: Crash After Table Commit +### Appendix 3: Bucket Hashing -Merger A crashes after committing to the table. +The bucket transform hash uses 32-bit wrapping arithmetic with these mixing functions. +Right shifts in `fmix` are logical shifts of the `u32` bit pattern. -| Step | Merger A | Merger B | MemWAL Index | -| ---- | ------------------------- | -------------------------------- | ---------------- | -| 1 | Reads index: merged_gen=5 | | merged_gen=5 | -| 2 | Merges gen 6, commits | | **merged_gen=6** | -| 3 | **CRASH** | | merged_gen=6 | -| 4 | | Reads index: merged_gen=6 | merged_gen=6 | -| 5 | | Reads shard manifest | | -| 6 | | **Skips gen 6** (already merged) | | -| 7 | | Merges gen 7, commits | **merged_gen=7** | - -The MemWAL Index is the single source of truth. Merger B correctly used it to determine that generation 6 was already merged. - -#### Key Points +```text +mix_k1(k) = rotl32(k * 0xcc9e2d51, 15) * 0x1b873593 +mix_h1(h, k) = rotl32(h ^ k, 13) * 5 + 0xe6546b64 +fmix(h, len) = + h = h ^ len + h = (h ^ (h >> 16)) * 0x85ebca6b + h = (h ^ (h >> 13)) * 0xc2b2ae35 + h ^ (h >> 16) +``` -1. **Single source of truth**: `merged_generations` is the authoritative source for merge progress, updated atomically with data. +Signed and unsigned casts use two's-complement wrapping. +Values are normalized and hashed as follows: + +- `bool`: `false` as `0`, `true` as `1`, then `hash_i32`. +- `int8`, `int16`, `int32`, `uint8`, `uint16`, `uint32`, `date32`, `time32`: cast to `i32`, then `hash_i32`. +- `int64`, `uint64`, `timestamp`, `time64`: cast to `i64`, then `hash_i64`. +- `float32`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7fc00000`; other values use IEEE 754 bits cast to `i32`, then `hash_i32`. +- `float64`: `-0.0` and `+0.0` normalize to bits `0`; all NaNs normalize to `0x7ff8000000000000`; other values use IEEE 754 bits cast to `i64`, then `hash_i64`. +- `utf8` and `large_utf8`: hash the UTF-8 bytes with `hash_bytes`. + +The helper hashes are: + +```text +hash_i32(v) = fmix(mix_h1(0, mix_k1(v)), 4) + +hash_i64(v) = + low = low 32 bits of v as i32 + high = high 32 bits of v as i32 + fmix(mix_h1(mix_h1(0, mix_k1(low)), mix_k1(high)), 8) + +hash_bytes(bytes) = + h = 0 + for each complete 4-byte little-endian chunk: + h = mix_h1(h, mix_k1(chunk_as_i32)) + for each remaining byte: + h = mix_h1(h, mix_k1(sign_extend_i8(byte))) + fmix(h, byte_length) +``` -2. **Conflict resolution uses MemWAL Index**: When a commit conflicts, the merger checks the conflicting commit's MemWAL Index. +Test vectors for `num_buckets = 8`: -3. **No progress regression**: Because MemWAL Index is updated atomically with data, concurrent mergers cannot regress the merge progress. +- `int32` or `date32`: `1 -> 2`, `2 -> 7`, `null -> 0`, `3 -> 1`. +- `utf8`: `"a" -> 1`, `"b" -> 5`, `null -> 0`. +- `bool`: `true -> 2`. +- `float32`: `1.25 -> 0`. +- `float64`: `1.25 -> 0`. diff --git a/docs/src/format/table/transaction.md b/docs/src/format/table/transaction.md index 78dd5301fb8..c88c3170988 100644 --- a/docs/src/format/table/transaction.md +++ b/docs/src/format/table/transaction.md @@ -466,6 +466,64 @@ The following operations are retryable conflicts with DataReplacement: A concurrent Delete or Update that only adds a deletion vector to a target fragment (without removing it) is compatible: the positional column file stays aligned and the rebase preserves the deletion vector. +### DataOverlay + +Attaches [overlay files](data_overlay_file.md) to fragments, supplying new values +for a subset of `(row offset, field)` cells without rewriting the fragments' base +data files. The overlays are appended to each fragment's existing `overlays` list, +so overlays written by concurrent commits are preserved. Each overlay's +`committed_version` is stamped to the new dataset version at commit time (and +re-stamped on retry), like the created-at / last-updated-at version sequences. + +
+DataOverlay protobuf message + +```protobuf +%%% proto.message.DataOverlay %%% + +%%% proto.message.DataOverlayGroup %%% +``` + +
+ +#### DataOverlay Compatibility + +A DataOverlay operation only changes cells within existing fragments and preserves +physical row addresses, so — like DataReplacement — it is intentionally permissive. +Because overlays stack and the higher `committed_version` wins each covered cell, +independent backfills never conflict, and a concurrent Delete simply makes the +overlay value for a deleted offset inert. Here are the operations that conflict +with DataOverlay: + +- Overwrite +- Restore +- UpdateMemWalState + +The following operations are retryable conflicts with DataOverlay: + +- Rewrite (only if overlapping fragments) — row-rewriting compaction or an + overlay→base fold changes physical row addresses or consumes the overlays, so + the overlay's offsets are no longer valid; the writer must re-read the new + fragment, recompute, and retry. +- Merge (always). +- A row-moving Update that touches an overlaid fragment — a delete-and-reinsert + update (any update that is not a `REWRITE_COLUMNS` column rewrite) relocates the + updated rows into new fragments, so the overlay's physical offsets no longer + address them; the writer must re-read and retry. + +DataOverlay is compatible with another DataOverlay (any fields), Append, Delete, a +`REWRITE_COLUMNS` column rewrite, and DataReplacement, because all of these +preserve physical row addresses: overlay offsets stay valid, the overlay is newer +and wins its covered cells, and the version gate excludes those cells from any +rebuilt index. + +When a DataReplacement or a `REWRITE_COLUMNS` update writes new base values for a +field, it supersedes any older overlay on that field: the writer tombstones the +overlay's entry for the rewritten field — replacing the field id with the obsolete +sentinel, as with obsolete base columns — so the fresh base values are not silently +shadowed. Overlay entries for other fields are preserved, and an overlay left with +no live fields is dropped. + ### UpdateMemWalState Updates the state of MemWal indices (write-ahead log based indices). @@ -627,7 +685,9 @@ In this scenario: If the backing object store does not support atomic operations (rename-if-not-exists or put-if-not-exists), an external manifest store can be used to enable concurrent writers. An external manifest store is a key-value store that supports put-if-not-exists operations. -The external manifest store supplements but does not replace the manifests in object storage. +It is the concurrency coordinator and fast version index: its conditional write selects one +immutable staging manifest for each version. The canonical manifest bytes in object storage +remain authoritative, so the external store supplements but does not replace them. A reader unaware of the external manifest store can still read the table, but may observe a version up to one commit behind the true latest version. ### Commit Process with External Store @@ -640,24 +700,41 @@ The commit process follows a four-step protocol: - Write the new manifest to object storage under a unique path determined by a new UUID - This staged manifest is not yet visible to readers -2. **Commit to external store**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest-{uuid}` - - Atomically commit the path of the staged manifest to the external store using put-if-not-exists - - The commit is effectively complete after this step - - If this operation fails due to conflict, another writer has committed this version +2. **Reserve version in external store**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest-{uuid}` + - Atomically reserve the version for this staged manifest using put-if-not-exists + - The reservation selects one immutable staging object; it is not yet the canonical commit + - If this operation fails due to conflict, another writer reserved this version 3. **Finalize in object store**: `COPY_OBJECT_STORE {dataset}/_versions/{version}.manifest-{uuid} → {dataset}/_versions/{version}.manifest` - Copy the staged manifest to the final path + - Successful materialization at this deterministic path is the commit point - This makes the manifest discoverable by readers unaware of the external store 4. **Update external store pointer**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest` - Update the external store to point to the finalized manifest path + - After copying, read the canonical object's current metadata. Return its ETag to the caller as + an opaque physical-generation observation so runtime caches do not collapse a newly committed + Dataset into an older cached Dataset at the same URI and version + - Do not persist that ETag in the external store. Concurrent finalizers can copy the same selected + immutable bytes into different physical generations, and COPY plus external-store publication + is not atomic. Every helper therefore publishes the same stable path-and-size tuple - Completes the synchronization between external store and object storage **Fault Tolerance:** -If the writer fails after step 2 but before step 4, the external store and object store are temporarily out of sync. -Readers detect this condition and attempt to complete the synchronization. -If synchronization fails, the reader refuses to load to ensure dataset portability. +If the writer fails after step 2 but before step 3, the external store contains a pending +reservation. Readers that use the external store detect this state and retry materialization. +If step 3 succeeds but step 4 fails, the canonical object remains committed; readers use it and +may repair the external index. Staging deletion is garbage collection and does not affect the +commit outcome. + +**Rolling Upgrade:** + +Roll this behavior out normally across the fleet. New readers ignore legacy stored +ETags, and legacy readers already accept finalized rows without an ETag, so mixed-version rows +remain compatible. While both legacy finalizers and legacy readers remain, the pre-existing race +can still republish a stale ETag that a legacy reader rejects. Full protection takes effect when +the rolling upgrade converges; no row migration or quiesced cutover is required. ### Reader Process with External Store @@ -667,7 +744,9 @@ The reader follows a validation and synchronization protocol: 1. **Query external store**: `GET_EXTERNAL_STORE base_uri, version` → `path` - Retrieve the manifest path for the requested version - - If the path does not end with a UUID, return it directly (synchronization complete) + - If the path does not end with a UUID, validate the canonical object's size. Ignore any legacy + stored ETag because it is neither content identity nor dataset-incarnation identity; the + validation HEAD still returns the current canonical ETag to the caller - If the path ends with a UUID, synchronization is required 2. **Synchronize to object store**: `COPY_OBJECT_STORE {dataset}/_versions/{version}.manifest-{uuid} → {dataset}/_versions/{version}.manifest` @@ -675,11 +754,12 @@ The reader follows a validation and synchronization protocol: - This operation is idempotent 3. **Update external store**: `PUT_EXTERNAL_STORE base_uri, version, {dataset}/_versions/{version}.manifest` - - Update the external store to reflect the finalized path - - Future readers will see the synchronized state + - Best-effort record the finalized path and size without an ETag while returning the observed + destination ETag to the current caller + - If this index repair fails, retain staging so a future reader can retry it 4. **Return finalized path**: Return `{dataset}/_versions/{version}.manifest` - - Always return the finalized path - - If synchronization fails, return an error to prevent reading inconsistent state + - Return once canonical materialization succeeds, even if index repair or staging cleanup fails + - If canonical materialization cannot be established, or an observed size differs, return an error This protocol ensures that datasets using external manifest stores remain portable: copying the dataset directory preserves all data without requiring the external store. diff --git a/docs/src/format/table/versioning.md b/docs/src/format/table/versioning.md index 745dd1ccd87..fa899451583 100644 --- a/docs/src/format/table/versioning.md +++ b/docs/src/format/table/versioning.md @@ -28,7 +28,10 @@ they should return an "unsupported" error on any read or write operation. | 4 | `FLAG_USE_V2_FORMAT_DEPRECATED` | No | No | Files are written with the new v2 format. This flag is deprecated and no longer used. | | 8 | `FLAG_TABLE_CONFIG` | No | Yes | Table config is present in the manifest. | | 16 | `FLAG_BASE_PATHS` | Yes | Yes | Dataset uses multiple base paths (for shallow clones or multi-base datasets). | +| 32 | `FLAG_DISABLE_TRANSACTION_FILE` | No | Yes | Transactions are recorded in the manifest rather than in a separate transaction file. | +| 64 | `FLAG_UNSTABLE_DATA_OVERLAY_FILES` | Yes | Yes | Fragments may carry data overlay files. Unstable: release builds reject it unless explicitly opted in. | +| 128 | `FLAG_COVERED_INDEX_METADATA` | Yes | Yes | Some index declares covering columns (`IndexMetadata.covering_fields`), so `fields` means keyed columns followed by carried ones. An implementation without this flag selects an index by membership of `fields` and would answer a query on a merely-carried column with an index keyed on a different one. | -Flags with bit values 32 and above are unknown and will cause implementations to reject the dataset with an "unsupported" error. +Flags with bit values 256 and above are unknown and will cause implementations to reject the dataset with an "unsupported" error. diff --git a/docs/src/guide/.pages b/docs/src/guide/.pages index 46ddd475799..7a0fd817b1c 100644 --- a/docs/src/guide/.pages +++ b/docs/src/guide/.pages @@ -6,6 +6,7 @@ nav: - JSON Support: json.md - Tags and Branches: tags_and_branches.md - Object Store Configuration: object_store.md + - Observability: observability.md - Distributed Write: distributed_write.md - Distributed Indexing: distributed_indexing.md - Migration Guide: migration.md diff --git a/docs/src/guide/arrays.md b/docs/src/guide/arrays.md index f824dd641c1..5765ca7675b 100644 --- a/docs/src/guide/arrays.md +++ b/docs/src/guide/arrays.md @@ -119,11 +119,11 @@ calling `lance.arrow.ImageTensorArray.to_encoded`. A `lance.arrow.EncodedImageArray.to_tensor` method is provided to decode encoded images and return them as `lance.arrow.FixedShapeImageTensorArray`, from -which they can be converted to numpy arrays or TensorFlow tensors. +which they can be converted to numpy arrays. For decoding images, it will first attempt to use a decoder provided via the optional function parameter. If decoder is not provided it will attempt to use -[Pillow](https://pillow.readthedocs.io/en/stable/) and [tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) in that -order. If neither library or custom decoder is available an exception will be raised. +[Pillow](https://pillow.readthedocs.io/en/stable/). If neither Pillow nor a custom +decoder is available an exception will be raised. ```python from lance.arrow import ImageURIArray @@ -132,13 +132,16 @@ uris = [os.path.join(os.path.dirname(__file__), "images/1.png")] encoded_images = ImageURIArray.from_uris(uris).read_uris() print(encoded_images.to_tensor()) -def tensorflow_decoder(images): - import tensorflow as tf +def pillow_decoder(images): + import io import numpy as np + from PIL import Image - return np.stack(tf.io.decode_png(img.as_py(), channels=3) for img in images.storage) + return np.stack( + np.asarray(Image.open(io.BytesIO(img.as_py()))) for img in images.storage + ) -print(encoded_images.to_tensor(tensorflow_decoder)) +print(encoded_images.to_tensor(pillow_decoder)) # # [[42, 42, 42, 255]] # @@ -164,8 +167,8 @@ created by calling `lance.arrow.ImageArray.from_array` and passing in a It can be encoded into to `lance.arrow.EncodedImageArray` by calling `lance.arrow.FixedShapeImageTensorArray.to_encoded` and passing custom encoder If encoder is not provided it will attempt to use -[tensorflow](https://www.tensorflow.org/api_docs/python/tf/io/encode_png) and [Pillow](https://pillow.readthedocs.io/en/stable/) in that order. Default encoders will -encode to PNG. If neither library is available it will raise an exception. +[Pillow](https://pillow.readthedocs.io/en/stable/). The default encoder will encode +to PNG. If neither Pillow nor a custom encoder is available it will raise an exception. ```python from lance.arrow import ImageURIArray @@ -176,4 +179,4 @@ tensor_images.to_encoded() # # [... # b'\x89PNG\r\n\x1a...' -``` \ No newline at end of file +``` diff --git a/docs/src/guide/blob.md b/docs/src/guide/blob.md index 00bd5d086e3..d34557e5384 100644 --- a/docs/src/guide/blob.md +++ b/docs/src/guide/blob.md @@ -65,6 +65,25 @@ source of truth for which scheme is supported at each `data_storage_version`. Use `blob_field` and `blob_array` to build blob v2 columns. +### Logical Arrow schema + +A blob v2 field is tagged with `ARROW:extension:name = "lance.blob.v2"`. Writers +accept these logical struct shapes: + +| Shape | Children | Use | +|---|---|---| +| Minimal | `data: LargeBinary?`, `uri: Utf8?` | Inline bytes or a complete external object | +| Complete | Minimal fields plus `position: UInt64?`, `size: UInt64?` | An optional byte range within an external object | + +Every non-null row must set exactly one of `data` and `uri`. For the complete +shape, `position` and `size` must either both be set or both be null, a range +requires `uri`, and an explicit range must have `size > 0`. Use inline `b""` for +an empty blob; a URI without range fields still represents the complete external +object, including an empty object. Python's `blob_field` and `BlobType` use the +complete shape. Lance preserves an accepted logical shape, including child +fields, nullability, and metadata, across create, append, and merge-insert +writes; descriptor scans still return the compact stored descriptor shape. + ```python import lance import pyarrow as pa @@ -115,6 +134,9 @@ Note: metadata for the same column are rejected. - `blob_pack_file_size_threshold` is a write option for rolling packed `.blob` sidecar files. It does not control inline-vs-packed placement. +- Blob v2 fields can be nested inside structs and variable-length lists. Blob-aware + scans preserve the surrounding nested layout; use `blob_handling="all_binary"` + to materialize nested blob payloads as bytes. ### Example: packed external blobs (single container file) @@ -174,8 +196,9 @@ Choose the read API based on the payload shape you want: | API | Returns | Use When | |---|---|---| -| `read_blobs` | `List[Tuple[int, bytes]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. | -| `take_blobs` | `List[BlobFile]` | You need file-like objects for streaming, seeking, or partial reads. | +| `read_blobs` | `List[Tuple[int, Optional[bytes]]]` | You need complete blob payloads in memory, such as training loaders or batch preprocessing. | +| `read_blob_ranges` | `List[Tuple[int, int, Optional[bytes]]]` | You need selected byte ranges from multiple rows without materializing complete blobs. | +| `take_blobs` | `List[Optional[BlobFile]]` | You need file-like objects for streaming, seeking, or partial reads. | | `scanner(..., blob_handling="all_binary")` | Arrow binary columns | You want blob columns in a scan result or `pyarrow.Table`. | Do not wrap `take_blobs` in your own thread pool just to call `read()` or @@ -183,7 +206,8 @@ Do not wrap `take_blobs` in your own thread pool just to call `read()` or batched blob reads through Lance's scheduler. Exactly one selector must be provided to `read_blobs` or `take_blobs`: `ids`, -`indices`, or `addresses`. +`indices`, or `addresses`. `read_blob_ranges` accepts the same selector kinds +through its required `selector` argument. | Selector | Typical Use | Stability | |---|---|---| @@ -223,6 +247,48 @@ row_addrs = ds.to_table(columns=[], with_row_address=True).column("_rowaddr").to rows = ds.read_blobs("blob", addresses=row_addrs[:2]) ``` +Blob selection APIs preserve logical result cardinality. `read_blobs()` and +`take_blobs()` return one element per selected row, and `read_blob_ranges()` +returns one element per request. A null blob is returned as `None`; a valid +empty blob remains a non-null empty payload or zero-length `BlobFile`. + +### Read row-specific byte ranges + +Use `read_blob_ranges` to read multiple blob-local ranges with one planned API +call. Each request is a `(row, offset, length)` tuple, and `selector` determines +whether every `row` is interpreted as a row ID, row address, or dataset index. + +```python +import lance + +ds = lance.dataset("./blobs_v22.lance") +results = ds.read_blob_ranges( + "blob", + requests=[ + (7, 0, 1024), + (7, 4096, 1024), + (12, 0, 0), + ], + selector="indices", +) + +for request_index, row_address, data in results: + if data is None: + # The selected blob is null. + continue + print(request_index, row_address, len(data)) +``` + +Each result contains the zero-based `request_index`, the resolved physical row +address, and the requested bytes. `request_index` identifies the original +request when the same row appears more than once. + +A request on a null blob returns `None`, including when its range is empty. An +empty range on a non-null blob returns `b""` without payload I/O. For every +request, `offset + length` must fit in an unsigned 64-bit integer. A range on a +non-null blob must not extend beyond its logical size; blob-local bounds are not +evaluated for null blobs because they have no logical payload length. + ### Read blob columns as Arrow binary ```python @@ -241,8 +307,10 @@ import lance ds = lance.dataset("./blobs_v22.lance") blobs = ds.take_blobs("blob", indices=[0, 1]) -with blobs[0] as f: - header = f.read(1024) +blob = blobs[0] +if blob is not None: + with blob as f: + header = f.read(1024) ``` ### Example: decode video frames lazily @@ -253,6 +321,8 @@ import lance ds = lance.dataset("./videos_v22.lance") blob = ds.take_blobs("video", indices=[0])[0] +if blob is None: + raise ValueError("video blob is null") start_ms, end_ms = 500, 1000 @@ -347,7 +417,7 @@ lance.write_dataset( Not every binary column needs to be a blob column. Plain Arrow `binary`/`large_binary` stores bytes *inline*, interleaved with your other columns, which is simplest and fastest for really small blobs (e.g., thumbnail images). Using a blob column to store the binary payload makes sense when either of these holds: -- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `take_blobs` → `BlobFile` handles that seek and range-read, so you pay only for the bytes you touch. +- **You need partial or streaming reads.** Inline binary is always read in full; there is no way to fetch a byte range without materializing the entire value. Blob columns expose `read_blob_ranges` for planned row-specific range reads and `take_blobs` → `BlobFile` handles for caller-driven seeks, so you pay only for the bytes you touch. - **Your values are large (roughly 1 MB or more on average).** Operations that rewrite entire rows, such as compaction or some updates, must copy the large inline payloads forward into the new version — even when those bytes never changed. The bigger the payload, the more bytes you rewrite per logical change (write amplification). A blob column keeps large payloads in separate `.blob` files that are referenced rather than re-copied, so these operations don't rewrite the heavy bytes. !!! tip diff --git a/docs/src/guide/data_types.md b/docs/src/guide/data_types.md index 06f26da1db3..c3f222d36ba 100644 --- a/docs/src/guide/data_types.md +++ b/docs/src/guide/data_types.md @@ -408,7 +408,7 @@ This maps to Lance's `FixedSizeList(Float32, 384)` type, which is optimized for: 3. **Align dimensions for SIMD**: Vector dimensions divisible by 8 enable optimal SIMD acceleration. Common dimensions: 128, 256, 384, 512, 768, 1024, 1536. -4. **Create indexes for large datasets**: For datasets with more than ~10,000 vectors, create an ANN index for fast search: +4. **Create indices for large datasets**: For datasets with more than ~10,000 vectors, create an ANN index for fast search: ```python # IVF_PQ is recommended for most use cases diff --git a/docs/src/guide/distributed_write.md b/docs/src/guide/distributed_write.md index 4fbc43a1058..13137b5de8e 100644 --- a/docs/src/guide/distributed_write.md +++ b/docs/src/guide/distributed_write.md @@ -263,3 +263,114 @@ Output: 6 7 Gracie 88 7 8 Henry 82 ``` + +### Handling stable row id + +On a dataset created with `enable_stable_row_ids=True`, each row keeps the same +`_rowid` for its lifetime, even when an update rewrites it into a different +fragment. Lance cannot infer which new row replaces which old one, so when you +assemble the transaction yourself, carrying those ids across is your job: read +the rows you are rewriting with `with_row_id=True` and attach their ids to the +new fragment with `lance.fragment.RowIdSequence`. + +Rows you leave without an id are treated as newly inserted. That is not an error, +so a fragment written without `row_id_meta` commits successfully while silently +giving every rewritten row a fresh identity, breaking `_rowid` for anything +downstream that relies on it. + +You do **not** need to supply `created_at_version_meta` or +`last_updated_at_version_meta`. Leave them as `None`. Lance derives both while +building the manifest: `last_updated_at_version_meta` becomes the version being +committed, and `created_at_version_meta` is copied from whichever existing row +carries the same stable row id, so a rewritten row keeps the version it first +appeared in. + +#### Mixing updated and new rows + +A single fragment may hold both rewritten rows and brand new ones. Order it so +that **the rewritten rows come first and the new rows last**, then pass only the +row ids of the rewritten rows. The row ids bind to the leading rows in fragment +order, and the commit generates new ids for the remaining rows. + +Do not generate ids for the new rows yourself. Row ids are handed out from a counter +in the manifest, and a commit that loses a race is retried against the version +that won, which may have consumed the very ids you picked. Only the commit knows +which values are free, so it assigns them after conflict resolution has settled. +Supplying more row ids than the fragment has rows is rejected. + +```python +import lance +import pyarrow as pa +import pyarrow.compute as pc +from lance.fragment import RowIdSequence, write_fragments + +schema = pa.schema([("id", pa.int64()), ("score", pa.int64())]) +dataset_uri = "./stable_row_ids.lance" +dataset = lance.write_dataset( + pa.table({"id": [1, 2, 3, 4], "score": [85, 90, 75, 80]}, schema=schema), + dataset_uri, + enable_stable_row_ids=True, +) + +# On a worker: read the rows to rewrite, keeping their stable row ids. +rows = dataset.to_table(columns=["id", "score"], with_row_id=True) +rewritten = rows.filter(pc.field("id").isin([2, 3])) + +# Rewritten rows first, then the row that did not exist before. +new_data = pa.table( + { + "id": rewritten["id"].to_pylist() + [5], + "score": [95, 70, 60], + }, + schema=schema, +) +fragments = write_fragments(new_data, dataset_uri, schema=schema) +assert len(fragments) == 1 + +# Only the rewritten rows have ids. The trailing row gets one at commit time. +fragments[0].row_id_meta = RowIdSequence(rewritten["_rowid"]).to_inline_metadata() + +# On the committing worker: tombstone the old copies of the rewritten rows. +updated_fragment = dataset.get_fragments()[0].delete("id in (2, 3)") + +op = lance.LanceOperation.Update( + updated_fragments=[updated_fragment], + new_fragments=fragments, +) +dataset = lance.LanceDataset.commit(dataset_uri, op, read_version=dataset.version) + +print(dataset.to_table(with_row_id=True).to_pandas()) +``` + +Output: +``` + id score _rowid +0 1 85 0 +1 4 80 3 +2 2 95 1 +3 3 70 2 +4 5 60 4 +``` + +Row ids 1 and 2 followed their rows into the new fragment, and the inserted row +received the next unused id. Reading the lineage columns shows that the rewritten +rows kept their original creation version while the inserted row is stamped with +the version that added it: + +```python +print( + dataset.to_table( + columns=["id", "_row_created_at_version", "_row_last_updated_at_version"] + ).to_pandas() +) +``` + +Output: +``` + id _row_created_at_version _row_last_updated_at_version +0 1 1 1 +1 4 1 1 +2 2 1 2 +3 3 1 2 +4 5 2 2 +``` diff --git a/docs/src/guide/migration.md b/docs/src/guide/migration.md index 5efd7b26b7f..56d7737941f 100644 --- a/docs/src/guide/migration.md +++ b/docs/src/guide/migration.md @@ -8,24 +8,19 @@ migrate. ## 9.0.0 -* Newly created FTS / inverted indexes now default to format v2 instead of v1. - The `LANCE_FTS_FORMAT_VERSION` environment variable no longer controls the - format used for newly created indexes. Users who need a specific index layout - should pass the index creation parameter `format_version` explicitly. - -* This affects users who create FTS / inverted indexes and need those indexes to - be readable by older Lance versions, or who depend on the v1 index layout. In - those cases, pass `format_version=1` when creating the index. Otherwise, newly - created indexes will use v2 by default, and older Lance readers may not be able - to read them. - - ```python - dataset.create_scalar_index("text", "INVERTED", format_version=1) - ``` - -* Existing v1 FTS indexes remain queryable. Operations that maintain an existing - v1 index, including append, incremental indexing, optimize, and mem-wal - maintained-index flush, should continue preserving the v1 format. +* Unless overridden, newly created FTS indexes use format v2. The code analyzer + and `block_size=256` require format v3, so readers must support v3 before an + index using either option is created. `document_granularity="list_element"` + also requires v3 reader capability, independently of the posting format. + +* To keep new indexes readable by nodes that support at most format v1 or v2, + set `format_version` in the index creation parameters, or set + `LANCE_FTS_FORMAT_VERSION` for a rollout-wide override. Formats v1 and v2 + require the text analyzer and `block_size=128`. + +* Operations that maintain an existing FTS index, including append, incremental + indexing, optimize, and mem-wal maintained-index flush, preserve its format + version. ## 7.2.0 diff --git a/docs/src/guide/object_store.md b/docs/src/guide/object_store.md index 9ce4800b800..2e1d5ac8ed9 100644 --- a/docs/src/guide/object_store.md +++ b/docs/src/guide/object_store.md @@ -30,10 +30,10 @@ These options apply to all object stores. | Key | Description | |------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `allow_http` | Allow non-TLS, i.e. non-HTTPS connections. Default, `False`. | -| `download_retry_count` | Number of times to retry a download. Default, `3`. This limit is applied when the HTTP request succeeds but the response is not fully downloaded, typically due to a violation of `request_timeout`. | +| `download_retry_count` | Number of times to retry a download. Default, `3`. This limit is applied when the HTTP request succeeds but the response is not fully downloaded, typically due to a violation of `timeout`. | | `allow_invalid_certificates` | Skip certificate validation on https connections. Default, `False`. Warning: This is insecure and should only be used for testing. | | `connect_timeout` | Timeout for only the connect phase of a Client. Default, `5s`. | -| `request_timeout` | Timeout for the entire request, from connection until the response body has finished. Default, `30s`. | +| `timeout` | Timeout for the entire request, from connection until the response body has finished. Default, `30s`. This applies to each individual request, so on a large write it must cover one complete multipart part upload; raise it alongside `LANCE_INITIAL_UPLOAD_SIZE`. | | `user_agent` | User agent string to use in requests. | | `proxy_url` | URL of a proxy server to use for requests. Default, `None`. | | `proxy_ca_certificate` | PEM-formatted CA certificate for proxy connections | @@ -41,6 +41,26 @@ These options apply to all object stores. | `client_max_retries` | Number of times for the object store client to retry the request. Default, `3`. | | `client_retry_timeout` | Timeout for the object store client to retry the request in seconds. Default, `180`. | +### Bulk copy strategy + +Lance streams bulk index-file movement and dataset deep-clone files through +read and write APIs by default. This avoids requiring a provider-native copy +operation and works across different object stores. + +Set `LANCE_IO_SERVER_SIDE_COPY_ENABLED` to a truthy value (`1`, `true`, `on`, +`yes`, or `y`, case-insensitive) to opt cloud copies whose source and destination +share the same object-store client into the provider-native server-side copy +operation. Cross-client, cross-store, and local copies do not use this setting. +Native copy can reduce client bandwidth and transfer cost, but it requires copy +support from the object-store integration and is subject to the provider +request's timeout and retry behavior. + +Deep clone bounds non-local file movement to four concurrent files by default. +Set `LANCE_DEEP_CLONE_STREAM_CONCURRENCY` to a positive integer to override this +operation-specific limit. The bound also applies when server-side copy is +enabled because S3 and GCS copies above the provider's single-copy size limit +fall back to streaming through Lance. + ## Per-Base Configuration A dataset can register additional base paths that store part of its data, and each @@ -110,6 +130,38 @@ The following keys can be used as both environment variables or keys in the | `aws_sse_kms_key_id` | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. | | `aws_sse_bucket_key_enabled` | Whether to use bucket keys for server-side encryption. | +### Credential provider selection + +By default, Lance uses the standard AWS credential provider chain (environment +variables, shared config file, web identity tokens, ECS, EC2 instance metadata). + +The `aws_provider_scheme` storage option pins a dataset to a specific credential +provider, which is useful when two datasets in the same process need different +AWS auth (for example, one bucket using IRSA and another using ECS container +credentials). + +| Value | Behavior | +|-------|----------| +| `token` | Use static access-key credentials. Returns an error if `aws_access_key_id` and `aws_secret_access_key` are not set. | +| `ecs` | Use the ECS/Pod Identity container credential endpoint. Reads `AWS_CONTAINER_CREDENTIALS_FULL_URI` or `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` from the environment. | +| `irsa` | Use IRSA (IAM Roles for Service Accounts) web identity token credentials. Reads `AWS_WEB_IDENTITY_TOKEN_FILE` and `AWS_ROLE_ARN` from the environment. | + +```python +import lance + +# Bucket A — use IRSA (web identity token from the environment) +ds_a = lance.dataset( + "s3://bucket-a/path", + storage_options={"aws_provider_scheme": "irsa"}, +) + +# Bucket B — use ECS container credentials +ds_b = lance.dataset( + "s3://bucket-b/path", + storage_options={"aws_provider_scheme": "ecs"}, +) +``` + ### S3-compatible stores Lance can also connect to S3-compatible stores, such as MinIO. To do so, you must @@ -326,6 +378,15 @@ parameter; explicit `storage_options` override environment variables: | `cos_secret_key` | Secret key used for COS authentication. Optional if credentials are provided by environment. | | `cos_enable_versioning` | Whether to enable object versioning on the bucket. Optional. | +!!! warning + + Tencent COS does not reliably enforce put-if-not-exists on buckets that have + ever had versioning enabled, even if versioning is now suspended. To prevent + silent manifest overwrites, Lance requires a custom distributed commit lock + for COS writes. Pass the same `commit_lock` implementation to every Python + writer, or provide a custom `CommitHandler` in Rust. Reads do not require a + commit lock. + !!! note The OpenDAL `CosConfig` currently exposes a limited set of options. Additional @@ -340,6 +401,31 @@ is `goosefs://host:port/path`, where `host:port` is the GooseFS Master address (default port: `9200`, may be omitted, e.g. `goosefs://10.0.0.1/path`) and `/path` is the filesystem path within GooseFS. +Manifest commits on `goosefs://` use `ConditionalPutCommitHandler` +(`PutMode::Create` / if-not-exists), backed by GooseFS master's atomic +no-replace rename so concurrent writers cannot clobber each other's +versioned manifests. + +!!! warning "Mixed-version writers are NOT safe" + + The `if-not-exists` guarantee only holds when **every** writer for a + dataset routes through this new handler. A writer running an older + Lance release still selects `UnsafeCommitHandler` for `goosefs://` + and writes the version path unconditionally, which can overwrite a + manifest that an upgraded writer has already won. Safe concurrent + commits therefore require: + + - all writers for the dataset run a Lance release that includes this + routing change, **or** + - writers share an external coordination boundary (e.g. a single-writer + queue, table-level lock, or a gateway that serializes commits) that + prevents the old code path from racing the new one. + + When upgrading in place, quiesce all writers (drain jobs, scale + clients to zero, or route traffic through a writer coordinator) before + rolling out the new Lance version, then bring writers back on the new + version together. + !!! note "About the dataset path" `/path` is just an arbitrary directory inside GooseFS — Lance does **not** diff --git a/docs/src/guide/observability.md b/docs/src/guide/observability.md new file mode 100644 index 00000000000..9cec3154e82 --- /dev/null +++ b/docs/src/guide/observability.md @@ -0,0 +1,100 @@ +# Observability + +Lance can publish operational metrics to your monitoring stack. The table below +is the authoritative catalogue of the metrics Lance emits, shared verbatim with +the Rust [`lance::metrics`](https://github.com/lance-format/lance/blob/main/rust/lance/src/metrics.md) +module documentation. + +--8<-- "rust/lance/src/metrics.md" + +## Collecting metrics + +Lance emits through the [`metrics`](https://docs.rs/metrics) crate facade, so it +is not tied to a specific backend — you install a recorder/exporter and route +the metrics wherever you like. Metrics are available from the Rust, Python, and +Java APIs. + +### Rust + +Enable the `metrics` feature on the `lance` crate: + +```toml +lance = { version = "...", features = ["metrics"] } +``` + +Then install any `metrics`-compatible recorder once at startup, before opening +datasets. For example, with +[`metrics-exporter-prometheus`](https://docs.rs/metrics-exporter-prometheus): + +```rust +metrics_exporter_prometheus::PrometheusBuilder::new() + .install() + .expect("install Prometheus recorder"); +``` + +Any recorder works — Prometheus, StatsD, an OpenTelemetry bridge, and so on. +When no recorder is installed, emission is a cheap no-op. + +### Python + +Unlike Rust, the Python bindings do not let you plug in an arbitrary recorder: +bridging one across the FFI boundary into the Rust `metrics` facade would be +complicated and inefficient. Instead `pylance` standardizes on OpenTelemetry, +which has good Python support, as its recorder. + +The `pylance` wheels are built with the `metrics` feature enabled. Install the +OpenTelemetry extra and call `instrument_lance_metrics`, which registers Lance's +metrics as observable instruments on your OpenTelemetry `MeterProvider`: + +```bash +pip install "pylance[otel]" +``` + +```python +from lance.otel import instrument_lance_metrics + +# Uses the global MeterProvider; pass meter_provider=... to target a specific one. +instrument_lance_metrics() +``` + +### Java + +The Java SDK includes an OpenTelemetry bridge in `org.lance.otel`. Register it +before opening datasets or performing Lance IO so the process-global Rust +recorder sees every emitted metric: + +```java +import org.lance.otel.LanceMetrics; + +LanceMetrics.instrument(); +``` + +The OpenTelemetry API is a dependency of the Java SDK. The application must +still configure an OpenTelemetry SDK, metric reader, and exporter for collection +and delivery. + +The no-argument method uses OpenTelemetry's global `MeterProvider`. To register +with an explicitly configured provider, pass it directly: + +```java +SdkMeterProvider provider = SdkMeterProvider.builder() + .registerMetricReader(metricReader) + .build(); +LanceMetrics.instrument(provider); +``` + +Repeated calls with the same provider are idempotent. Passing a different +provider unregisters the existing callback before registering the new one. Call +`LanceMetrics.close()` to stop exporting while retaining the process-global Rust +metric state. + +From there the metrics flow through whatever OpenTelemetry pipeline you have +configured (OTLP, Prometheus, console, …). Because OpenTelemetry has no +asynchronous histogram instrument, histograms are exported Prometheus-style as +three observable counters: `_bucket`, `_count`, and `_sum`. +Each `_bucket` sample carries an `le` ("less than or equal") attribute +giving that bucket's inclusive upper bound in the metric's unit; the bucket +count is cumulative, covering every observation at or below `le`. For example, a +`lance_object_store_request_duration_seconds_bucket` sample with `le="0.5"` +counts all requests that completed in 0.5 seconds or less, while `le="+Inf"` is +the total count. diff --git a/docs/src/guide/performance.md b/docs/src/guide/performance.md index 14181eb69af..8d7f0c5401f 100644 --- a/docs/src/guide/performance.md +++ b/docs/src/guide/performance.md @@ -141,7 +141,7 @@ Keys are often a composite of multiple fields and all keys are scoped to the dat | Deletion Files | Dataset URI, fragment_id, version, id, file_type | The deletion vector for a frag | | Row Id Mask | Dataset URI, version | The row id sequence for the dataset | | Row Id Index | Dataset URI, version | The row id index for the dataset | -| Row Id Sequence | Dataset URI, fragment_id | The row id sequence for a fragment | +| Row Id Sequence | Dataset URI, fragment_id, row_id_meta | The row id sequence for a fragment | | Index Metadata | Dataset URI, version | The index metadata for the dataset | | Index Details¹ | Dataset URI, index uuid | The index details for an index | | File Global Meta | Dataset URI, file path | The global metadata for a file | @@ -189,6 +189,41 @@ working with 1024-dimensional vector embeddings (e.g. 32-bit floats) then 8192 r spread that across 16 CPU threads then you would need 512MB of compute memory per scan. You might find working with 1024 rows per batch is more appropriate. +#### Tuning remote scans + +An ordered dataset scan still overlaps I/O from multiple fragments. `scan_in_order=True` controls the order in +which batches are returned; it does not make fragment reads sequential. This is why a dataset scan can issue +more concurrent requests than scanning one fragment directly. The following controls tune different parts of +the scan: + +* `fragment_readahead` limits how many fragments may have reads scheduled concurrently. Set it to `1` to match + the fragment-level I/O pattern, then increase it if the storage connection has spare bandwidth. +* `LANCE_IO_THREADS` limits concurrent storage requests for the process. Cloud stores default to 64, which is + intended for high-bandwidth, in-region access and can be too aggressive across regions or over the public + internet. +* `io_buffer_size` limits buffered I/O bytes and applies backpressure when decoding falls behind. +* `batch_readahead` limits concurrent batch decoding. It does not control the size of storage range requests. + +For a bandwidth-constrained remote connection, start with conservative settings and tune upward: + +```shell +LANCE_IO_THREADS=8 python scan.py +``` + +```python +scanner = dataset.scanner( + fragment_readahead=1, + batch_readahead=2, + io_buffer_size=64 * 1024 * 1024, +) +for batch in scanner.to_batches(): + process(batch) +``` + +Lance reads encoded pages from storage, so reducing `batch_size` changes the returned and decoded batch sizes +but may not reduce the initial range request. The first batch can require loading one encoded page for each +selected column. + In summary, scans could use up to `(2 * io_buffer_size) + (batch_size * num_compute_threads)` bytes of memory. Keep in mind that `io_buffer_size` is a soft limit (e.g. we cannot read less than one page at a time right now) and so it is not necessarily a bug if you see memory usage exceed this limit by a small margin. @@ -219,6 +254,37 @@ use cases. For example, S3 can typically get up to 5000 req/s and with these settings we should get there in about 10 seconds. +## Fragment Sizing + +A Lance table is a collection of fragments tracked by a manifest. How you size those fragments +trades off two classes of work: + +- **Manifest-level operations** scale with the *number* of fragments. Every dataset mutation + (appends, metadata updates, schema changes, compactions, etc.) rewrites the manifest, so a + larger fragment list makes every write slower. Reads pay a similar cost up front: opening a + dataset, listing fragments, planning a scan, and resolving transaction conflicts at the + dataset level all walk the manifest. +- **Fragment-level operations** scale with the *size* of a fragment. These include scans + against a matching fragment, compaction, updates, deletes, and `merge_insert`. Conflict + detection for these operations is also done at the fragment level. + +Fewer, larger fragments make manifest-level operations cheap but make each fragment-level +operation heavier and increase the chance of conflicts when many writers target the same +fragment. More, smaller fragments do the reverse. + +Practical guidance: + +- The default of 1M rows per fragment works well up to ~1B rows. Past that, bumping toward + ~100M rows per fragment is reasonable, though fragment-count limits are rarely the bottleneck + in practice. +- Tens of thousands of fragments per table is generally fine. +- Keep individual fragments well under object-store object-size limits (S3 caps at 5 TB, and + stores tend to misbehave well before that). 10 GB–100 GB per fragment is a reasonable upper + range; 1 TB is a hard ceiling. +- If you run many concurrent updates, deletes, or `merge_insert` operations, err toward more + fragments — conflict detection is per-fragment, so too few fragments leads to excess + retries. + ## Conflict Handling Lance supports concurrent operations on the same table using optimistic concurrency control. When two @@ -414,11 +480,52 @@ exact size depends on the quantizer: 100M * (768 + 8) = ~72.3 GiB ``` -**RQ (RaBitQ):** Vectors are currently quantized to 1-bit binary codes. Each row also stores per-row -scale and offset factors (4 bytes each) used for distance correction. Each row requires -`dimension / 8 + 16` bytes (8 bytes for the row ID plus 8 bytes for the factors). For example, 100M -rows with 768 dimensions and 1 bit per dimension: +**RQ (RaBitQ):** New indexes default to 5 bits per dimension. Every bit width stores a 1-bit sign +code plus three 4-byte correction factors. Multi-bit indexes also store the remaining bits in +64-dimension-padded blocks and two additional 4-byte correction factors. Including the 8-byte row +ID, the approximate size per row is: + +- 1-bit: `dimension / 8 + 20` bytes +- Multi-bit: `dimension / 8 + round_up(dimension, 64) * (num_bits - 1) / 8 + 28` bytes + +For example, the default 5-bit index for 100M rows with 768 dimensions requires: ``` -100M * (768 / 8 + 16) = ~10.8 GiB +100M * (768 / 8 + 768 * 4 / 8 + 28) = ~47.3 GiB ``` + +The 5-bit default retains more information for the higher-fidelity distance estimates used by +`Normal` and `Accurate` search modes, at the cost of more quantization work and index I/O during +the build and a larger index. `Fast` search mode uses only the 1-bit sign code even when the index +stores additional bits. Set `num_bits=1` explicitly to minimize index size and build I/O; the same +100M-row example uses about 10.8 GiB, but searches cannot use the multi-bit distance estimate and +may have lower recall. + +#### AMX Acceleration + +On Linux x86_64 with an AMX-FP16 CPU (Intel Granite Rapids / Xeon 6 and newer), a `float16` +vector column indexed with `dot` distance uses the AMX tile instructions, provided the build +machine had clang >= 16 or gcc >= 13 to compile the kernel. There is nothing to enable — +Lance checks the CPU at run time and falls back to the previous implementation everywhere else. + +The accelerated paths are also shape-gated, because below these sizes a tile pass costs more +than it saves and the kernel declines the work: + +| Condition | Why | +|---|---| +| `float16` vectors, `dot` distance | The kernel is fp16-specific; other types and metrics keep their existing paths | +| `dimension >= 32` | One tile pass covers 32 dimensions; a shorter vector would be all scalar cleanup | +| `num_centroids >= 32` | The GEMM steps its centroid loop by 32 and has no partial-tile path | + +Anything outside them behaves exactly as it does today, so a small dataset or a low-dimensional +column simply keeps the previous implementation rather than changing behaviour. + +Index build also changes algorithm where all of the above hold: comparing every vector against +every centroid becomes affordable, so partition assignment is exact instead of approximated with +a graph search over the centroids. Recall improves, and partition assignments differ from what an +older build produced. + +Set `LANCE_DISABLE_AMX=1` to take the AMX paths out of service without rebuilding — for +A/B measurement, or to get the previous behaviour back. Because it also moves partition +assignment back to the approximate path, an index built with it set is not equivalent to one +built without it; compare recall, not just build time. diff --git a/docs/src/guide/read_and_write.md b/docs/src/guide/read_and_write.md index ec6cde5173a..c7feb69144b 100644 --- a/docs/src/guide/read_and_write.md +++ b/docs/src/guide/read_and_write.md @@ -456,3 +456,146 @@ affected files are no longer part of any ANN index if they were before. Because of this, it's recommended to rewrite files before re-building indices. + +### Cleanup old versions + +Lance is an immutable format — every write creates a new version. The new version +only writes the data that changed, so an insert writes the new rows and an update +rewrites the affected columns for the affected rows. Even a delete creates a +small deletion file. However, old versions still reference the previous data +files, so those files are kept on disk until explicitly removed. Over time this +means storage grows with each operation — inserts, updates, and deletes alike. + +Keeping old versions has important benefits: readers that opened an older version +can continue reading it without interference from concurrent writers, providing +snapshot isolation. Old versions also enable time travel queries, letting you +read the dataset as it existed at any prior point in time. + +`cleanup_old_versions` deletes old version metadata and any data files that are +no longer referenced by any version, reclaiming the accumulated storage. + +!!! warning + + Once old versions are cleaned up, time travel queries to those versions are + no longer possible. Choose your retention window (`older_than`) accordingly — + any version removed by cleanup cannot be recovered. + +```python +import lance + +dataset = lance.dataset("./my_dataset.lance") +dataset.cleanup_old_versions() +``` + +By default, versions older than 7 days are removed. You can override this with +the `older_than` parameter (a `timedelta`): + +```python +from datetime import timedelta + +dataset.cleanup_old_versions(older_than=timedelta(days=1)) +``` + +!!! note + + Tagged versions are exempt from cleanup. See [Tags and Branches](tags_and_branches.md) + for details. + +By default, Lance only removes files that it can **verify** are no longer needed. +A file is verified when Lance can see that it was referenced by an older version +and is no longer referenced by any newer version. However, some orphaned files +cannot be verified this way — for example, files left behind by aborted or failed +commits that were never recorded in any version. These files are +indistinguishable from files being written by an in-progress operation. + +Cleanup will never delete the current (active) version. This means passing +`older_than=timedelta(0)` is safe and will delete all versions except the current +one. + +The `delete_unverified` flag enables a more aggressive strategy that will also +delete these unverified files: + +```python +dataset.cleanup_old_versions( + older_than=timedelta(hours=2), + delete_unverified=True, +) +``` + +!!! danger + + Only use `delete_unverified=True` when you are confident that no other + concurrent operation has been in-progress for longer than the `older_than` + duration. Lance uses the file's age to decide whether an unverified file is + safe to remove, so any operation that is still running past the `older_than` + window risks having its files deleted out from under it. + + In particular, combining `delete_unverified=True` with `older_than=timedelta(0)` + is **extremely dangerous** — if any other operation is in-progress at all, + its data files may be deleted, leading to dataset corruption. + +### Automatic cleanup + +Instead of calling `cleanup_old_versions` manually, you can configure Lance to +clean up old versions automatically during writes. When auto cleanup is enabled, +Lance will run cleanup every *N* commits (the **interval**), removing versions +older than a specified duration. + +Auto cleanup can be enabled when creating a new dataset: + +```python +import lance +import pyarrow as pa +from lance.dataset import AutoCleanupConfig + +table = pa.table({"id": range(100)}) +ds = lance.write_dataset( + table, + "./my_dataset.lance", + auto_cleanup_options=AutoCleanupConfig( + interval=20, # run cleanup every 20 commits + older_than_seconds=3600, # remove versions older than 1 hour + ), +) +``` + +Or enabled on an existing dataset: + +```python +ds = lance.dataset("./my_dataset.lance") +ds.optimize.enable_auto_cleanup( + AutoCleanupConfig( + interval=20, + older_than_seconds=3600, + ) +) +``` + +And disabled again: + +```python +ds.optimize.disable_auto_cleanup() +``` + +Auto cleanup parameters can also be set directly via dataset config keys: + +```python +ds.update_config({ + "lance.auto_cleanup.interval": "20", + "lance.auto_cleanup.older_than": "3600s", +}) +``` + +!!! warning + + Auto cleanup runs as part of the commit path. If your writer does not have + delete permissions, or you are doing high-frequency writes where the extra + latency matters, pass `skip_auto_cleanup=True` to `write_dataset` to skip it + on a per-write basis. + +### Other cleanup strategies + +It is common to run cleanup as a periodic background task on a dedicated server +(for example, via a cron job or scheduled workflow). This keeps cleanup off the +write path entirely, avoiding any impact to write latency, but requires setting +up and maintaining additional infrastructure. diff --git a/docs/src/guide/tags_and_branches.md b/docs/src/guide/tags_and_branches.md index 8af2302bfb0..4b371ea6dae 100644 --- a/docs/src/guide/tags_and_branches.md +++ b/docs/src/guide/tags_and_branches.md @@ -87,7 +87,7 @@ import pyarrow as pa # Open dataset ds = lance.dataset("/tmp/test.lance") -# Create branch from latest version (default: current branch's latest) +# Create branch from the currently checked-out version experiment_branch = ds.create_branch("experiment") experimental_data = pa.Table.from_pydict({"a": [11], "b": [12]}) lance.write_dataset(experimental_data, experiment_branch, mode="append") diff --git a/docs/src/guide/tokenizer.md b/docs/src/guide/tokenizer.md index eedac585fe9..0d3605df30b 100644 --- a/docs/src/guide/tokenizer.md +++ b/docs/src/guide/tokenizer.md @@ -12,6 +12,39 @@ ${system data directory}/lance/language_models It also supports configuring user dictionaries, which makes it convenient for users to expand their own dictionaries without retraining the language models. +## Inspect Query Tokenization + +Use `lance.tokenize` to inspect the tokens that a full-text query will produce +without creating a dataset or index: + +```python +import lance + +tokens = lance.tokenize("the Cats and Dogs") +[(token.text, token.position) for token in tokens] +# [("cat", 0), ("dog", 2)] +``` + +Positions start at the first retained query token and preserve gaps left by stop +word removal and other filters. This is the same representation used for phrase +matching. The function accepts the tokenizer-related options supported by +`LanceDataset.create_scalar_index`, including custom stop words, n-grams, and the +code analyzer: + +```python +tokens = lance.tokenize( + "getUserName::value42", + analyzer="code", + split_identifiers=True, + index_operators=True, +) +``` + +Options set to `None` use the selected analyzer profile's default. For example, +the code analyzer disables stemming and stop-word removal unless explicitly +overridden. The exception is `max_token_length`: omitting it keeps the default +length limit of 40, while `max_token_length=None` disables the limit. + ## ICU Tokenizer ICU uses Unicode word boundary rules and bundled dictionary data for complex scripts. It is useful for mixed-language text and does not require downloading a language model. @@ -54,7 +87,7 @@ Create a file named config.json in the root directory of the current model. ``` - The "main" field is optional. If not filled, the default is "dict.txt". -- "users" is the path of the user dictionary. For the format of the user dictionary, please refer to https://github.com/messense/jieba-rs/blob/main/src/data/dict.txt. +- "users" is the path of the user dictionary. For the format of the user dictionary, please refer to https://github.com/messense/jieba-rs/blob/main/jieba/src/data/dict.txt. ## Language Models of Lindera diff --git a/docs/src/images/mem_wal_overview.png b/docs/src/images/mem_wal_overview.png index 008c84d0724..1c9f64bc647 100644 Binary files a/docs/src/images/mem_wal_overview.png and b/docs/src/images/mem_wal_overview.png differ diff --git a/docs/src/images/mem_wal_regional.png b/docs/src/images/mem_wal_regional.png deleted file mode 100644 index 5681fa27b8b..00000000000 Binary files a/docs/src/images/mem_wal_regional.png and /dev/null differ diff --git a/docs/src/images/mem_wal_shard.png b/docs/src/images/mem_wal_shard.png new file mode 100644 index 00000000000..ad8bed9e4b9 Binary files /dev/null and b/docs/src/images/mem_wal_shard.png differ diff --git a/docs/src/integrations/.pages b/docs/src/integrations/.pages index 62feffae067..ab9c2c0dcee 100644 --- a/docs/src/integrations/.pages +++ b/docs/src/integrations/.pages @@ -1,9 +1,6 @@ nav: - Overview: index.md - Apache DataFusion: datafusion.md - - PostgreSQL: https://github.com/lancedb/pglance + - PostgreSQL: https://github.com/lance-format/pglance - PyTorch: pytorch.md - - Tensorflow: tensorflow.md - - Apache Spark: spark - - Ray: ray - - Trino: trino + - TensorFlow: tensorflow.md diff --git a/docs/src/integrations/index.md b/docs/src/integrations/index.md index 0304f8f5277..eb3b25051cc 100644 --- a/docs/src/integrations/index.md +++ b/docs/src/integrations/index.md @@ -27,7 +27,7 @@ GitHub organization. | Integration | Description | Source | |---|---|---| | [PyTorch](pytorch.md) | Use `lance.torch.data.LanceDataset` as a `torch.utils.data.IterableDataset` for training and inference. | Built-in | -| [TensorFlow](tensorflow.md) | Use `lance.tf.data.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | Built-in | +| [TensorFlow](tensorflow.md) | Use `lance_tensorflow.from_lance` to stream Lance data into `tf.data.Dataset` pipelines. | [lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow) | | [Ray](ray) | Distributed read/write of Lance datasets with Ray Data. | [lance-format/lance-ray](https://github.com/lance-format/lance-ray) | | [Hugging Face](huggingface) | Convert and load Hugging Face datasets to and from Lance in a single call. | [lance-format/lance-huggingface](https://github.com/lance-format/lance-huggingface) | diff --git a/docs/src/integrations/pytorch.md b/docs/src/integrations/pytorch.md index 543b9978064..ae5c9f6286e 100644 --- a/docs/src/integrations/pytorch.md +++ b/docs/src/integrations/pytorch.md @@ -4,7 +4,7 @@ Machine learning users can use `lance.torch.data.LanceDataset`, a subclass of `torch.utils.data.IterableDataset`, that to use Lance data directly PyTorch training and inference loops. -It starts with creating a ML dataset for training. With the [HuggingFace integration](huggingface.md), +It starts with creating a ML dataset for training. With the [HuggingFace integration](huggingface), it takes just one line of Python to convert a HuggingFace dataset to a Lance dataset. ```python diff --git a/docs/src/integrations/tensorflow.md b/docs/src/integrations/tensorflow.md index 1c5d6b87157..03a5c5c5eca 100644 --- a/docs/src/integrations/tensorflow.md +++ b/docs/src/integrations/tensorflow.md @@ -1,92 +1,71 @@ -# Tensorflow Integration +--- +title: TensorFlow +description: Stream Lance datasets into TensorFlow tf.data pipelines with lance-tensorflow. +--- -Lance can be used as a regular [tf.data.Dataset](https://www.tensorflow.org/api_docs/python/tf/data/Dataset) -in [Tensorflow](https://www.tensorflow.org/). +# TensorFlow Integration -!!! warning +The TensorFlow integration is maintained in the +[lance-format/lance-tensorflow](https://github.com/lance-format/lance-tensorflow) +project. - This feature is experimental and the APIs may change in the future. +The main Lance Python package no longer includes `lance.tf`. Install +`lance-tensorflow` and import `lance_tensorflow` instead. + +```bash +pip install lance-tensorflow +``` ## Reading from Lance -Using `lance.tf.data.from_lance`, you can create an `tf.data.Dataset` easily. +Use `lance_tensorflow.from_lance` to create a `tf.data.Dataset` from a Lance +dataset. ```python -import tensorflow as tf -import lance +from lance_tensorflow import from_lance -# Create tf dataset -ds = lance.tf.data.from_lance("s3://my-bucket/my-dataset") - -# Chain tf dataset with other tf primitives +ds = from_lance( + "s3://my-bucket/my-dataset", + columns=["image", "label"], + filter="split = 'train'", + batch_size=256, +) -for batch in ds.shuffling(32).map(lambda x: tf.io.decode_png(x["image"])): - print(batch) +for batch in ds: + print(batch["label"]) ``` -Backed by the Lance [columnar format](../format/index.md), using `lance.tf.data.from_lance` supports -efficient column selection, filtering, and more. +## Dataset Convenience Methods + +If you want `tf.data.Dataset.from_lance`, register the convenience methods +explicitly after importing `lance_tensorflow`. ```python -ds = lance.tf.data.from_lance( - "s3://my-bucket/my-dataset", - columns=["image", "label"], - filter="split = 'train' AND collected_time > timestamp '2020-01-01'", - batch_size=256) -``` +import tensorflow as tf +import lance_tensorflow -By default, Lance will infer the Tensor spec from the projected columns. You can also specify `tf.TensorSpec` manually. +lance_tensorflow.register_tensorflow_dataset() -```python -batch_size = 256 -ds = lance.tf.data.from_lance( - "s3://my-bucket/my-dataset", - columns=["image", "labels"], - batch_size=batch_size, - output_signature={ - "image": tf.TensorSpec(shape=(), dtype=tf.string), - "labels": tf.RaggedTensorSpec( - dtype=tf.int32, shape=(batch_size, None), ragged_rank=1), - }, +ds = tf.data.Dataset.from_lance("s3://my-bucket/my-dataset") ``` -## Distributed Training and Shuffling +## Migration -Since [a Lance Dataset is a set of Fragments](../format/index.md), we can distribute and shuffle Fragments to different -workers. +Replace old imports: ```python -import tensorflow as tf -from lance.tf.data import from_lance, lance_fragments +import lance.tf.data -world_size = 32 -rank = 10 -seed = 123 # -epoch = 100 +ds = lance.tf.data.from_lance(uri) +``` -dataset_uri = "s3://my-bucket/my-dataset" +with: -# Shuffle fragments distributedly. -fragments = - lance_fragments("s3://my-bucket/my-dataset") - .shuffling(32, seed=seed) - .repeat(epoch) - .enumerate() - .filter(lambda i, _: i % world_size == rank) - .map(lambda _, fid: fid) +```python +from lance_tensorflow import from_lance -ds = from_lance( - uri, - columns=["image", "label"], - fragments=fragments, - batch_size=32 - ) -for batch in ds: - print(batch) +ds = from_lance(uri) ``` -!!! warning - - For multiprocessing you should probably not use fork as lance is - multi-threaded internally and fork and multi-thread do not work well. - Refer to [this discussion](https://discuss.python.org/t/concerns-regarding-deprecation-of-fork-with-alive-threads/33555). \ No newline at end of file +See the [lance-tensorflow README](https://github.com/lance-format/lance-tensorflow) +for the current installation and compatibility details. diff --git a/docs/src/quickstart/full-text-search.md b/docs/src/quickstart/full-text-search.md index f990b2bd589..de6bec7dd2c 100644 --- a/docs/src/quickstart/full-text-search.md +++ b/docs/src/quickstart/full-text-search.md @@ -98,6 +98,7 @@ ds.create_scalar_index( remove_stop_words=True, # Remove stop words (language-dependent) custom_stop_words=None, # Optional additional stop words (only used if remove_stop_words=True) ascii_folding=True, # Fold accents to ASCII when possible (e.g., "é" -> "e") + block_size=128, # Posting block size: 128 or 256; 256 is experimental ) ``` @@ -241,6 +242,9 @@ query_result = ds.to_table(full_text_query=(q1 & q2)) To combine `OR` queries via operators, use the pattern `q1 | q2`. +Every query combined with `AND` becomes a scoring `MUST` clause: all clauses must match, +and every matching clause contributes to the final `_score`. + #### Exclude terms: `NOT` Queries that exclude specific keywords are explicitly written using `BooleanQuery`/`Occur` diff --git a/docs/src/quickstart/index.md b/docs/src/quickstart/index.md index 34367c7177f..f10c5749014 100644 --- a/docs/src/quickstart/index.md +++ b/docs/src/quickstart/index.md @@ -105,4 +105,4 @@ dataset.to_table().to_pandas() Now that you've mastered the basics of creating Lance datasets, here's what you can explore next: - **[Versioning Your Datasets with Lance](versioning.md)** - Learn how to track changes over time with native versioning -- **[Vector Indexing and Vector Search With Lance](vector-search.md)** - Build high-performance vector search capabilities with ANN indexes +- **[Vector Indexing and Vector Search With Lance](vector-search.md)** - Build high-performance vector search capabilities with ANN indices diff --git a/docs/src/quickstart/vector-search.md b/docs/src/quickstart/vector-search.md index 6b1f6a5e516..ac77e12b752 100644 --- a/docs/src/quickstart/vector-search.md +++ b/docs/src/quickstart/vector-search.md @@ -1,13 +1,13 @@ --- title: Vector Search -description: High-performance vector search with ANN indexes, including IVF_PQ, IVF_HNSW_PQ, and IVF_HNSW_SQ +description: High-performance vector search with ANN indices, including IVF_PQ, IVF_HNSW_PQ, and IVF_HNSW_SQ --- # Vector Indexing and Vector Search With Lance -Lance provides high-performance vector search capabilities with ANN (Approximate Nearest Neighbor) indexes. +Lance provides high-performance vector search capabilities with ANN (Approximate Nearest Neighbor) indices. -By the end of this tutorial, you'll be able to build and use ANN indexes to dramatically speed up vector search operations while maintaining high accuracy. You'll also learn how to tune search parameters for optimal performance and combine vector search with metadata queries in a single operation. +By the end of this tutorial, you'll be able to build and use ANN indices to dramatically speed up vector search operations while maintaining high accuracy. You'll also learn how to tune search parameters for optimal performance and combine vector search with metadata queries in a single operation. ## Install the Python SDK @@ -210,8 +210,6 @@ The latency vs recall is tunable via: - **refine_factor**: determines how many vectors are retrieved during re-ranking ```python -%%time - sift1m.to_table( nearest={ "column": "vector", diff --git a/docs/src/quickstart/versioning.md b/docs/src/quickstart/versioning.md index 8cdf1cb35ea..5b339abd9e4 100644 --- a/docs/src/quickstart/versioning.md +++ b/docs/src/quickstart/versioning.md @@ -63,6 +63,16 @@ List all versions of a dataset with this request: dataset.versions() ``` +If you only need version numbers, use the lightweight reference API. It lists manifest +locations without reading and deserializing every manifest: + +```python +dataset.version_refs() +``` + +Use `dataset.latest_version` instead when only the latest version of the current branch +is needed. + You can also access any available version: ```python diff --git a/docs/theme/404.html b/docs/theme/404.html new file mode 100644 index 00000000000..51897bd0805 --- /dev/null +++ b/docs/theme/404.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block container %} +
+
404 · Page not found
+

This page doesn't exist.

+

The page you're looking for may have moved.

+
+ Back to home + Getting started +
+
+{% endblock %} diff --git a/docs/theme/assets/site.css b/docs/theme/assets/site.css new file mode 100644 index 00000000000..a9d5286244c --- /dev/null +++ b/docs/theme/assets/site.css @@ -0,0 +1,792 @@ +/* Lance docs site chrome — faithful port of the "Lance Docs" design prototype. + Layout/nav classes (ld-*) come from the theme templates; article content + styles target python-markdown + pymdownx output. */ + +[hidden] { display: none !important; } + +.ld-shell { + min-height: 100vh; + display: flex; + flex-direction: column; + background: var(--surface-page); + font-family: var(--font-body); + color: var(--text-body); +} + +/* ---------- header ---------- */ +.ld-header { + position: sticky; + top: 0; + z-index: 60; + display: flex; + align-items: center; + gap: 20px; + height: 60px; + padding: 0 clamp(16px, 3vw, 32px); + min-width: 0; + background: var(--surface-header); + backdrop-filter: blur(12px); + border-bottom: 1px solid var(--line-1); +} +.ld-brand { + display: inline-flex; + align-items: center; + text-decoration: none; + user-select: none; + flex-shrink: 0; +} +.ld-brand img { display: block; height: 24px; width: auto; } +.ld-topnav { + display: flex; + gap: 2px; + margin-right: auto; + overflow-x: auto; + scrollbar-width: none; + min-width: 0; + flex: 1; +} +.ld-topnav::-webkit-scrollbar { display: none; } +.ld-toptab { + font-family: var(--font-body); + font-size: 14px; + font-weight: 500; + color: var(--text-muted); + text-decoration: none; + padding: 7px 12px; + cursor: pointer; + transition: color var(--dur-fast) var(--ease-out); + position: relative; + white-space: nowrap; +} +.ld-toptab:hover { color: var(--text-body); } +.ld-toptab.active { color: var(--fg-1); } +.ld-toptab.active::after { + content: ""; + position: absolute; + left: 12px; + right: 12px; + bottom: -10px; + height: 2px; + background: var(--beam-400); +} +.ld-header__actions { display: flex; gap: 10px; align-items: center; flex-shrink: 0; } +/* All header controls share one 34px height regardless of content (icon/text). */ +.ld-header__actions .ld-btn { height: 34px; padding-top: 0; padding-bottom: 0; } +.ld-header__actions .ld-btn--icon { width: 34px; padding: 0; justify-content: center; } + +/* ---------- buttons ---------- */ +.ld-btn { + display: inline-flex; + align-items: center; + gap: 8px; + text-decoration: none; + white-space: nowrap; + flex-shrink: 0; +} +.ld-btn--primary { + font-family: var(--font-body); + font-size: 13.5px; + font-weight: 500; + color: #ffffff; + background: var(--beam-400); + padding: 8px 16px; +} +.ld-btn--primary:hover { background: var(--beam-600); color: #ffffff; } +.ld-btn--ghost { + font-family: var(--font-mono); + font-size: 12.5px; + color: var(--text-secondary); + border: 1px solid var(--line-1); + padding: 7px 14px; +} +.ld-btn--ghost:hover { border-color: var(--beam-400); color: var(--fg-1); } +.ld-btn--icon { padding: 7px 9px; background: none; cursor: pointer; } +.ld-btn__count { + border-left: 1px solid var(--line-2); + padding-left: 8px; + color: var(--fg-1); + font-weight: 600; +} +:root[data-theme="light"] .ld-icon-sun { display: none; } +:root[data-theme="dark"] .ld-icon-moon { display: none; } +.ld-btn--outline { + font-size: 14px; + font-weight: 500; + color: var(--fg-1); + border: 1px solid var(--line-1); + padding: 11px 24px; +} +.ld-btn--outline:hover { border-color: var(--beam-400); color: var(--fg-1); } +.ld-btn--text { + font-size: 14px; + font-weight: 500; + color: var(--text-secondary); + padding: 11px 12px; +} +.ld-btn--text:hover { color: var(--fg-1); } +.ld-btn--lg.ld-btn--primary { font-size: 14px; padding: 12px 24px; } + +/* ---------- home: hero ---------- */ +.ld-hero { max-width: var(--container-max); margin: 0 auto; padding: 96px 32px 72px; width: 100%; } +.ld-kicker { + font-family: var(--font-mono); + font-size: 12.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--beam-600); + margin-bottom: 24px; +} +/* Wrap only between segments (at the · separators), never inside one. */ +.ld-kicker span, .ld-kicker a { white-space: nowrap; } +.ld-kicker a { color: inherit; text-decoration: underline; text-underline-offset: 3px; } +.ld-kicker a:hover { color: var(--beam-700); } +.ld-hero h1 { + font-family: var(--font-display); + font-size: 72px; + font-weight: 600; + letter-spacing: var(--tracking-display); + line-height: 0.97; + color: var(--fg-1); + margin: 0 0 28px; + max-width: 900px; + text-wrap: balance; +} +.ld-hero__lead { + font-size: 17px; + line-height: 1.6; + color: var(--text-secondary); + max-width: 640px; + margin: 0 0 36px; + text-wrap: pretty; +} +.ld-hero__cta { display: flex; gap: 12px; flex-wrap: wrap; } + +/* ---------- home: stats band ---------- */ +.ld-band { max-width: var(--container-max); margin: 0 auto; padding: 0 32px; width: 100%; } +.ld-stats { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + border-top: 1px solid var(--line-1); + border-bottom: 1px solid var(--line-1); +} +.ld-stat { padding: 28px 32px; } +.ld-stat:first-child { padding-left: 0; } +.ld-stat:last-child { padding-right: 0; } +.ld-stat + .ld-stat { border-left: 1px solid var(--line-2); } +.ld-stat--link { display: block; text-decoration: none; } +.ld-stat--link .ld-stat__label { transition: color var(--dur-fast) var(--ease-out); } +.ld-stat--link:hover .ld-stat__label { color: var(--beam-600); } +.ld-stat__value { + font-family: var(--font-display); + font-size: 40px; + font-weight: 600; + letter-spacing: -0.03em; + color: var(--fg-1); +} +.ld-stat__value--beam { color: var(--beam-400); } +.ld-stat__label { + font-family: var(--font-mono); + font-size: 12px; + letter-spacing: 0.1em; + text-transform: uppercase; + color: var(--text-muted); + margin-top: 6px; +} + +/* ---------- home: what is lance ---------- */ +.ld-what { max-width: var(--container-max); margin: 0 auto; padding: 72px 32px 40px; width: 100%; } +.ld-what h2 { + font-family: var(--font-display); + font-size: 34px; + font-weight: 600; + letter-spacing: -0.03em; + color: var(--fg-1); + margin: 0 0 16px; +} +.ld-what__body { + font-size: 16px; + line-height: 1.65; + color: var(--text-secondary); + max-width: 760px; + margin: 0 0 12px; + text-wrap: pretty; +} +.ld-what__more { font-size: 15px; line-height: 1.6; color: var(--text-secondary); margin: 0; } +.ld-inline-link { + color: var(--beam-600); + text-decoration: underline; + text-decoration-thickness: 2px; + text-underline-offset: 3px; +} + +/* ---------- home: features ---------- */ +.ld-features { max-width: var(--container-max); margin: 0 auto; padding: 24px 32px 96px; width: 100%; } +.ld-feature { + display: grid; + grid-template-columns: 90px minmax(0, 1fr) minmax(0, 460px); + gap: 40px; + align-items: center; + border-top: 1px solid var(--line-1); + padding: 40px 0; +} +.ld-feature__num { + font-family: var(--font-mono); + font-size: 15px; + color: var(--beam-600); + align-self: start; + padding-top: 6px; +} +.ld-feature__body h3 { + font-family: var(--font-display); + font-size: 24px; + font-weight: 600; + letter-spacing: -0.02em; + color: var(--fg-1); + margin: 0 0 12px; +} +.ld-feature__body p { + font-size: 15px; + line-height: 1.65; + color: var(--text-secondary); + margin: 0 0 16px; + text-wrap: pretty; +} +.ld-more { + font-family: var(--font-mono); + font-size: 13px; + color: var(--beam-600); + text-decoration: none; +} +.ld-more:hover { text-decoration: underline; } +/* Logo grids stay on white in both themes — the artwork assumes a light background. */ +.ld-feature__img { border: 1px solid var(--line-1); padding: 16px; background: #ffffff; } +.ld-feature__img img { display: block; width: 100%; height: auto; } + +/* homepage code windows (always ink-dark, like docs code blocks) */ +.ld-feature__code { border: 1px solid var(--line-2); background: var(--ink-900); min-width: 0; border-radius: var(--radius-box); overflow: hidden; } +.ld-win { display: flex; gap: 6px; padding: 12px 16px 0; } +.ld-win i { width: 10px; height: 10px; border-radius: 999px; background: var(--ink-600); } +.ld-feature__code pre { + margin: 0; + padding: 14px 20px 18px; + overflow-x: auto; + font-family: var(--font-mono); + font-size: 12.5px; + line-height: 1.7; + color: #EDEBF5; +} +.ld-feature__code code { font-family: inherit; } +.tok-kw { color: #B8A8FF; } +.tok-str { color: #7CC0FF; } +.tok-com { color: #6F6A85; font-style: italic; } +.tok-num { color: #F2CE6B; } +.tok-arg { color: #FF95A8; } +.tok-fn { color: #8FE8CE; } + +/* ---------- docs layout ---------- */ +.ld-docs { + flex: 1; + display: grid; + grid-template-columns: 240px minmax(0, 1fr) 190px; + max-width: 1280px; + width: 100%; + margin: 0 auto; + gap: 44px; + padding: 0 32px; +} +.ld-sidenav { + border-right: 1px solid var(--line-2); + padding: 36px 24px 48px 0; + position: sticky; + top: 60px; + align-self: start; + height: calc(100vh - 60px); + overflow-y: auto; +} +/* Mobile-only disclosure for the sidenav; hidden on desktop where the + sidenav is a sticky column. */ +.ld-sidenav-toggle { + display: none; + align-items: center; + justify-content: space-between; + gap: 8px; + width: 100%; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 600; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-secondary); + background: none; + border: 1px solid var(--line-2); + padding: 9px 12px; + margin: 16px 0 0; + cursor: pointer; +} +.ld-sidenav-toggle:hover { color: var(--fg-1); border-color: var(--beam-400); } +.ld-sidenav-toggle svg { transition: transform var(--dur-fast) var(--ease-out); } +.ld-sidenav-toggle.open svg { transform: rotate(180deg); } +.ld-sidenav__group { margin-bottom: 6px; } +.ld-sidenav__label { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin: 22px 0 8px; +} +.ld-sidenav a { + display: block; + font-size: 13.5px; + color: var(--text-secondary); + text-decoration: none; + padding: 5px 12px; + cursor: pointer; + border-left: 1px solid var(--line-2); + line-height: 1.45; +} +.ld-sidenav a:hover { color: var(--text-body); background: var(--surface-overlay); } +.ld-sidenav a.active { + color: var(--beam-600); + border-left: 2px solid var(--beam-400); + padding-left: 11px; + font-weight: 500; +} +.ld-sidenav a .ext { color: var(--text-muted); font-size: 11px; margin-left: 4px; } + +/* ---------- right-hand page TOC ---------- */ +.ld-toc { + position: sticky; + top: 60px; + align-self: start; + max-height: calc(100vh - 60px); + overflow-y: auto; + padding: 44px 0 48px; +} +.ld-toc__label { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin: 0 0 10px; +} +.ld-toc a { + display: block; + font-size: 12.5px; + line-height: 1.45; + color: var(--text-muted); + text-decoration: none; + border-left: 1px solid var(--line-2); + padding: 4px 0 4px 12px; +} +.ld-toc a.ld-toc__h3 { padding-left: 24px; } +.ld-toc a:hover { color: var(--text-body); } +.ld-toc a.active { + color: var(--beam-600); + border-left: 2px solid var(--beam-400); + padding-left: 11px; + font-weight: 500; +} +.ld-toc a.ld-toc__h3.active { padding-left: 23px; } + +.ld-crumbs { + font-family: var(--font-mono); + font-size: 11.5px; + color: var(--text-muted); + display: flex; + gap: 8px; + flex-wrap: wrap; + margin-bottom: 24px; +} +.ld-crumbs b { color: var(--beam-600); font-weight: 500; } + +.ld-pagenav { + display: flex; + justify-content: space-between; + gap: 16px; + border-top: 1px solid var(--line-1); + margin-top: 56px; + padding-top: 20px; +} +.ld-pagenav a { text-decoration: none; color: var(--fg-1); } +.ld-pagenav a:hover { color: var(--beam-600); } +.ld-pagenav a.next { text-align: right; } +.ld-pagenav__k { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin-bottom: 4px; +} +.ld-pagenav__t { font-size: 14.5px; font-weight: 500; } + +/* ---------- article prose (python-markdown output) ---------- */ +.ld-article-col { max-width: 760px; min-width: 0; padding: 40px 0 96px; } +.ld-article { font-family: var(--font-body); color: var(--text-secondary); } +/* Keep anchor targets clear of the sticky header. */ +.ld-article :is(h1, h2, h3, h4, h5, h6) { scroll-margin-top: 76px; } +.ld-article h1 { font-family: var(--font-display); font-size: 42px; font-weight: 600; letter-spacing: -0.03em; line-height: 1.05; color: var(--fg-1); margin: 0 0 20px; text-wrap: balance; } +.ld-article h2 { font-family: var(--font-display); font-size: 26px; font-weight: 600; letter-spacing: -0.02em; color: var(--fg-1); margin: 48px 0 14px; padding-top: 28px; border-top: 1px solid var(--line-2); } +.ld-article h3 { font-family: var(--font-display); font-size: 19px; font-weight: 600; letter-spacing: -0.01em; color: var(--fg-1); margin: 32px 0 10px; } +.ld-article h4, .ld-article h5, .ld-article h6 { font-family: var(--font-mono); font-size: 12.5px; font-weight: 600; letter-spacing: .14em; text-transform: uppercase; color: var(--fg-1); margin: 28px 0 8px; } +.ld-article p { font-size: 15.5px; line-height: 1.65; margin: 0 0 16px; text-wrap: pretty; } +.ld-article a { color: var(--beam-600); text-decoration: underline; text-decoration-thickness: 2px; text-underline-offset: 3px; text-decoration-color: rgba(98, 94, 255, .35); } +.ld-article a:hover { text-decoration-color: var(--beam-600); } +.ld-article code { font-family: var(--font-mono); font-size: 13px; background: var(--surface-inline-code); border-radius: var(--radius-chip); padding: 1px 5px; color: var(--fg-1); } +.ld-article ul, .ld-article ol { margin: 0 0 16px; padding-left: 22px; font-size: 15.5px; line-height: 1.65; } +.ld-article li { margin-bottom: 6px; } +.ld-article li p { margin-bottom: 8px; } +.ld-article hr { border: 0; border-top: 1px solid var(--line-1); margin: 40px 0; } +.ld-article blockquote { margin: 0 0 16px; padding: 4px 0 4px 20px; border-left: 2px solid var(--beam-400); } +.ld-article blockquote p:last-child { margin-bottom: 0; } + +/* heading permalinks (toc: permalink) */ +.ld-article .headerlink { + margin-left: 8px; + color: var(--beam-400); + text-decoration: none; + opacity: 0; + transition: opacity var(--dur-fast) var(--ease-out); +} +.ld-article :is(h1, h2, h3, h4, h5, h6):hover .headerlink { opacity: 1; } + +/* figures: lone images sit on a white plate in both themes */ +.ld-article img { max-width: 100%; height: auto; } +.ld-article p > img:only-child { + display: block; + margin: 24px auto; + border: 1px solid var(--line-1); + padding: 16px; + background: #ffffff; +} + +/* tables (JS wraps them in .ld-tablewrap for overflow) */ +.ld-tablewrap { overflow-x: auto; margin: 0 0 20px; border: 1px solid var(--line-1); } +.ld-article table { border-collapse: collapse; width: 100%; font-size: 14px; } +.ld-article th { font-family: var(--font-mono); font-size: 11px; font-weight: 600; letter-spacing: .12em; text-transform: uppercase; text-align: left; color: var(--fg-1); background: var(--surface-overlay); padding: 10px 14px; border-bottom: 1px solid var(--line-1); } +.ld-article td { padding: 10px 14px; border-bottom: 1px solid var(--line-2); vertical-align: top; line-height: 1.55; color: var(--text-secondary); } +.ld-article tr:last-child td { border-bottom: 0; } + +/* ---------- code blocks (pymdownx.highlight output, JS adds the bar) ---------- */ +.ld-code { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; } +.ld-code__bar { display: flex; justify-content: space-between; align-items: center; padding: 7px 14px; border-bottom: 1px solid var(--line-2); background: var(--surface-card); } +.ld-code__bar span { display: inline-flex; align-items: center; gap: 7px; font-family: var(--font-mono); font-size: 11px; letter-spacing: .12em; text-transform: uppercase; color: var(--text-muted); } +.ld-code__bar span svg { width: 13px; height: 13px; display: block; flex-shrink: 0; } +.ld-copy { font-family: var(--font-mono); font-size: 11px; letter-spacing: .08em; text-transform: uppercase; background: none; border: none; color: var(--beam-600); cursor: pointer; padding: 2px 0; } +.ld-copy:hover { color: var(--beam-700); } +.ld-code .highlight { margin: 0; } +.ld-code pre { margin: 0; background: var(--surface-code); padding: 16px 18px; overflow-x: auto; } +.ld-code pre code { background: none; border: none; padding: 0; font-size: 13px; line-height: 1.6; color: var(--code-fg); } +/* Ink code surfaces (dark-theme docs blocks + the always-dark homepage windows) + need an explicit light selection color; light-theme docs blocks use the default. */ +:root[data-theme="dark"] .ld-code pre ::selection, :root[data-theme="dark"] .ld-code pre::selection, +.ld-feature__code pre ::selection, .ld-feature__code pre::selection { + background: rgba(98, 94, 255, 0.55); + color: #ffffff; +} + +/* Pygments tokens — colors resolve per theme via the --code-* palette */ +.highlight .k, .highlight .kn, .highlight .kd, .highlight .kt, .highlight .kr, .highlight .kp, .highlight .ow { color: var(--code-kw); } +.highlight .kc { color: var(--code-kw); } +.highlight .s, .highlight .s1, .highlight .s2, .highlight .sb, .highlight .sd, .highlight .sa, .highlight .se, .highlight .si, .highlight .sx, .highlight .sr, .highlight .ss, .highlight .sh { color: var(--code-str); } +.highlight .c, .highlight .c1, .highlight .cm, .highlight .ch, .highlight .cs, .highlight .cp, .highlight .cpf { color: var(--code-com); font-style: italic; } +.highlight .m, .highlight .mi, .highlight .mf, .highlight .mh, .highlight .mo, .highlight .mb, .highlight .il { color: var(--code-num); } +.highlight .nf, .highlight .fm, .highlight .nd, .highlight .ne { color: var(--code-fn); } +.highlight .nc, .highlight .nn { color: var(--code-name); font-weight: 500; } +.highlight .nb, .highlight .bp { color: var(--code-kw); } +.highlight .nt { color: var(--code-tag); } +.highlight .na { color: var(--code-tag); } +.highlight .o { color: var(--code-punct); } +.highlight .p { color: var(--code-punct); } +.highlight .gp { color: var(--code-prompt); } +.highlight .go { color: var(--code-punct); } +.highlight .gh, .highlight .gu { color: var(--code-name); font-weight: 600; } +.highlight .hll { background: var(--code-hll); display: block; } + +/* mermaid diagrams: white plate (rendered by JS) */ +.ld-article pre.mermaid, .ld-article div.mermaid { + border: 1px solid var(--line-1); + padding: 16px; + background: #ffffff; + margin: 0 0 20px; + text-align: center; + overflow-x: auto; +} + +/* ---------- admonitions ---------- */ +.ld-article .admonition { + border: 1px solid var(--line-2); + border-radius: var(--radius-box); + overflow: hidden; + margin: 0 0 20px; + background: var(--surface-card); + padding: 0; + font-size: 14.5px; +} +.ld-article .admonition > .admonition-title { + display: flex; + align-items: center; + gap: 8px; + font-family: var(--font-mono); + font-size: 11px; + font-weight: 600; + letter-spacing: .14em; + text-transform: uppercase; + color: var(--beam-600); + padding: 9px 16px; + margin: 0; + border-bottom: 1px solid var(--line-2); + background: none; +} +/* Material-style type icon, drawn in the title color via a CSS mask */ +.ld-article .admonition > .admonition-title::before { + content: ""; + width: 14px; + height: 14px; + flex-shrink: 0; + background-color: currentColor; + -webkit-mask: var(--ld-adm-icon) no-repeat center / contain; + mask: var(--ld-adm-icon) no-repeat center / contain; +} +/* Icon shapes are Material Design Icons paths (Apache 2.0), matching mkdocs-material's defaults. */ +.ld-article .admonition { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20.71,7.04C21.1,6.65 21.1,6 20.71,5.63L18.37,3.29C18,2.9 17.35,2.9 16.96,3.29L15.12,5.12L18.87,8.87M3,17.25V21H6.75L17.81,9.93L14.06,6.18L3,17.25Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.abstract, .admonition.summary, .admonition.tldr) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17,9H7V7H17M17,13H7V11H17M14,17H7V15H14M12,3A1,1 0 0,1 13,4A1,1 0 0,1 12,5A1,1 0 0,1 11,4A1,1 0 0,1 12,3M19,3H14.82C14.4,1.84 13.3,1 12,1C10.7,1 9.6,1.84 9.18,3H5A2,2 0 0,0 3,5V19A2,2 0 0,0 5,21H19A2,2 0 0,0 21,19V5A2,2 0 0,0 19,3Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.info, .admonition.todo) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,9H11V7H13M13,17H11V11H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12A10,10 0 0,0 12,2Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.tip, .admonition.hint, .admonition.important) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M17.66,11.2C17.43,10.9 17.15,10.64 16.89,10.38C16.22,9.78 15.46,9.35 14.82,8.72C13.33,7.26 13,4.85 13.95,3C13,3.23 12.17,3.75 11.46,4.32C8.87,6.4 7.85,10.07 9.07,13.22C9.11,13.32 9.15,13.42 9.15,13.55C9.15,13.77 9,13.97 8.8,14.05C8.57,14.15 8.33,14.09 8.14,13.93C8.08,13.88 8.04,13.83 8,13.76C6.87,12.33 6.69,10.28 7.45,8.64C5.78,10 4.87,12.3 5,14.47C5.06,14.97 5.12,15.47 5.29,15.97C5.43,16.57 5.7,17.17 6,17.7C7.08,19.43 8.95,20.67 10.96,20.92C13.1,21.19 15.39,20.8 17.03,19.32C18.86,17.66 19.5,15 18.56,12.72L18.43,12.46C18.22,12 17.66,11.2 17.66,11.2M14.5,17.5C14.22,17.74 13.76,18 13.4,18.1C12.28,18.5 11.16,17.94 10.5,17.28C11.69,17 12.4,16.12 12.61,15.23C12.78,14.43 12.46,13.77 12.33,13C12.21,12.26 12.23,11.63 12.5,10.94C12.69,11.32 12.89,11.7 13.13,12C13.9,13 15.11,13.44 15.37,14.8C15.41,14.94 15.43,15.08 15.43,15.23C15.46,16.05 15.1,16.95 14.5,17.5H14.5Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.success, .admonition.check, .admonition.done) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M9,20.42L2.79,14.21L5.62,11.38L9,14.77L18.88,4.88L21.71,7.71L9,20.42Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.question, .admonition.help, .admonition.faq) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M15.07,11.25L14.17,12.17C13.45,12.89 13,13.5 13,15H11V14.5C11,13.39 11.45,12.39 12.17,11.67L13.41,10.41C13.78,10.05 14,9.55 14,9C14,7.89 13.1,7 12,7A2,2 0 0,0 10,9H8A4,4 0 0,1 12,5A4,4 0 0,1 16,9C16,9.88 15.64,10.67 15.07,11.25M13,19H11V17H13M12,2A10,10 0 0,0 2,12A10,10 0 0,0 12,22A10,10 0 0,0 22,12C22,6.47 17.5,2 12,2Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.warning, .admonition.caution, .admonition.attention) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M13,14H11V9H13M13,18H11V16H13M1,21H23L12,2L1,21Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.failure, .admonition.fail, .admonition.missing) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M20 6.91L17.09 4L12 9.09L6.91 4L4 6.91L9.09 12L4 17.09L6.91 20L12 14.91L17.09 20L20 17.09L14.91 12L20 6.91Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.danger, .admonition.error) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M11,15H6L13,1V9H18L11,23V15Z"/%3E%3C/svg%3E'); } +.ld-article .admonition.bug { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,12H10V10H14M14,16H10V14H14M20,8H17.19C16.74,7.22 16.12,6.55 15.37,6.04L17,4.41L15.59,3L13.42,5.17C12.96,5.06 12.5,5 12,5C11.5,5 11.04,5.06 10.59,5.17L8.41,3L7,4.41L8.62,6.04C7.88,6.55 7.26,7.22 6.81,8H4V10H6.09C6.04,10.33 6,10.66 6,11V12H4V14H6V15C6,15.34 6.04,15.67 6.09,16H4V18H6.81C7.85,19.79 9.78,21 12,21C14.22,21 16.15,19.79 17.19,18H20V16H17.91C17.96,15.67 18,15.34 18,15V14H20V12H18V11C18,10.66 17.96,10.33 17.91,10H20V8Z"/%3E%3C/svg%3E'); } +.ld-article .admonition.example { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M7,13V11H21V13H7M7,19V17H21V19H7M7,7V5H21V7H7M3,8V5H2V4H4V8H3M2,17V16H5V20H2V19H4V18.5H3V17.5H4V17H2M4.25,10A0.75,0.75 0 0,1 5,10.75C5,10.95 4.92,11.14 4.79,11.27L3.12,13H5V14H2V13.08L4,11H2V10H4.25Z"/%3E%3C/svg%3E'); } +.ld-article :is(.admonition.quote, .admonition.cite) { --ld-adm-icon: url('data:image/svg+xml;charset=utf-8,%3Csvg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"%3E%3Cpath d="M14,17H17L19,13V7H13V13H16M6,17H9L11,13V7H5V13H8L6,17Z"/%3E%3C/svg%3E'); } +.ld-article .admonition > :not(.admonition-title) { margin: 14px 16px; font-size: 14.5px; } +.ld-article .admonition > .highlight, .ld-article .admonition > .ld-code { margin: 14px 16px; } +.ld-article .admonition.warning > .admonition-title, +.ld-article .admonition.caution > .admonition-title, +.ld-article .admonition.attention > .admonition-title { color: var(--warn-500); } +.ld-article .admonition.danger > .admonition-title, +.ld-article .admonition.error > .admonition-title, +.ld-article .admonition.bug > .admonition-title, +.ld-article .admonition.failure > .admonition-title { color: var(--danger-500); } +.ld-article .admonition.success > .admonition-title, +.ld-article .admonition.check > .admonition-title { color: var(--ok-500); } + +/* ---------- details / summary (pymdownx.details) ---------- */ +.ld-article details { + border: 1px solid var(--line-2); + border-radius: var(--radius-box); + overflow: hidden; + margin: 0 0 20px; + background: var(--surface-card); +} +.ld-article details > summary { + font-family: var(--font-mono); + font-size: 12px; + font-weight: 600; + letter-spacing: .1em; + text-transform: uppercase; + color: var(--beam-600); + padding: 10px 16px; + cursor: pointer; + user-select: none; + list-style: none; +} +.ld-article details > summary::-webkit-details-marker { display: none; } +.ld-article details > summary:hover { background: var(--surface-overlay); } +.ld-article details[open] > summary { border-bottom: 1px solid var(--line-2); } +.ld-article details > :not(summary) { margin: 14px 16px; } + +/* ---------- content tabs (pymdownx.tabbed, alternate style) ---------- */ +.ld-article .tabbed-set { margin: 0 0 20px; border: 1px solid var(--line-2); border-radius: var(--radius-box); overflow: hidden; position: relative; } +.ld-article .tabbed-set > input { position: absolute; opacity: 0; pointer-events: none; } +.ld-article .tabbed-labels { + display: flex; + border-bottom: 1px solid var(--line-2); + background: var(--surface-card); + overflow-x: auto; + scrollbar-width: none; +} +.ld-article .tabbed-labels > label { + font-family: var(--font-mono); + font-size: 12px; + letter-spacing: .06em; + padding: 9px 18px; + border-right: 1px solid var(--line-2); + color: var(--text-muted); + cursor: pointer; + white-space: nowrap; +} +.ld-article .tabbed-labels > label:hover { color: var(--fg-1); } +.ld-article .tabbed-content { display: block; } +.ld-article .tabbed-block { display: none; padding: 16px 16px 0; } +.ld-article .tabbed-block > :last-child { margin-bottom: 16px; } +.ld-article .tabbed-block > .ld-code:last-child { margin-bottom: 16px; } +/* nth-input → nth-label/nth-block mapping (supports up to 8 tabs) */ +.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-labels > label:nth-child(1), +.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-labels > label:nth-child(2), +.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-labels > label:nth-child(3), +.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-labels > label:nth-child(4), +.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-labels > label:nth-child(5), +.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-labels > label:nth-child(6), +.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-labels > label:nth-child(7), +.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-labels > label:nth-child(8) { + color: var(--fg-1); + box-shadow: inset 0 -2px 0 var(--beam-400); +} +.ld-article .tabbed-set > input:nth-child(1):checked ~ .tabbed-content > .tabbed-block:nth-child(1), +.ld-article .tabbed-set > input:nth-child(2):checked ~ .tabbed-content > .tabbed-block:nth-child(2), +.ld-article .tabbed-set > input:nth-child(3):checked ~ .tabbed-content > .tabbed-block:nth-child(3), +.ld-article .tabbed-set > input:nth-child(4):checked ~ .tabbed-content > .tabbed-block:nth-child(4), +.ld-article .tabbed-set > input:nth-child(5):checked ~ .tabbed-content > .tabbed-block:nth-child(5), +.ld-article .tabbed-set > input:nth-child(6):checked ~ .tabbed-content > .tabbed-block:nth-child(6), +.ld-article .tabbed-set > input:nth-child(7):checked ~ .tabbed-content > .tabbed-block:nth-child(7), +.ld-article .tabbed-set > input:nth-child(8):checked ~ .tabbed-content > .tabbed-block:nth-child(8) { + display: block; +} + +/* ---------- search overlay ---------- */ +.ld-search { position: fixed; inset: 0; z-index: 100; } +.ld-search__scrim { position: absolute; inset: 0; background: rgba(14, 12, 20, 0.5); } +.ld-search__panel { + position: relative; + max-width: 640px; + margin: 96px auto 0; + background: var(--surface-page); + border: 1px solid var(--line-1); +} +.ld-search__bar { display: flex; align-items: center; border-bottom: 1px solid var(--line-1); } +.ld-search__input { + flex: 1; + font-family: var(--font-body); + font-size: 15px; + color: var(--fg-1); + background: none; + border: none; + outline: none; + padding: 14px 16px; +} +.ld-search__input::placeholder { color: var(--text-muted); } +.ld-search__close { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: .1em; + text-transform: uppercase; + color: var(--text-muted); + background: none; + border: 1px solid var(--line-2); + margin-right: 12px; + padding: 3px 8px; + cursor: pointer; +} +.ld-search__close:hover { color: var(--fg-1); border-color: var(--beam-400); } +.ld-search__results { max-height: 55vh; overflow-y: auto; } +.ld-search__hit { display: block; padding: 12px 16px; text-decoration: none; border-top: 1px solid var(--line-2); } +.ld-search__hit:first-child { border-top: 0; } +.ld-search__hit:hover, .ld-search__hit.active { background: var(--surface-overlay); } +.ld-search__hit-title { font-size: 14px; font-weight: 500; color: var(--fg-1); } +.ld-search__hit-title mark, .ld-search__hit-text mark { background: none; color: var(--beam-600); } +.ld-search__hit-crumb { font-family: var(--font-mono); font-size: 10.5px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); margin-bottom: 2px; } +.ld-search__hit-text { font-size: 12.5px; color: var(--text-muted); line-height: 1.5; margin-top: 2px; } +.ld-search__empty { font-family: var(--font-mono); font-size: 12px; letter-spacing: .08em; text-transform: uppercase; color: var(--text-muted); padding: 20px 16px; } + +/* ---------- 404 ---------- */ +.ld-notfound { max-width: var(--container-max); margin: 0 auto; padding: 120px 32px 160px; width: 100%; flex: 1; } +.ld-notfound h1 { + font-family: var(--font-display); + font-size: 56px; + font-weight: 600; + letter-spacing: var(--tracking-display); + color: var(--fg-1); + margin: 0 0 16px; +} +.ld-notfound p { color: var(--text-secondary); margin: 0 0 32px; } + +/* ---------- footer ---------- */ +.ld-footer { + border-top: 1px solid var(--line-1); + padding: 40px 32px; + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 32px; + flex-wrap: wrap; +} +.ld-footer__brand img { display: block; height: 24px; width: auto; } +.ld-footer__brand p { + font-size: 12.5px; + color: var(--text-muted); + max-width: 300px; + line-height: 1.6; + margin: 12px 0 0; +} +.ld-footer__cols { display: flex; gap: 64px; flex-wrap: wrap; } +.ld-footer__col { display: flex; flex-direction: column; gap: 8px; } +.ld-footer__col h5 { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: var(--tracking-caps); + text-transform: uppercase; + color: var(--text-muted); + margin: 0 0 4px; +} +.ld-footer__col a { font-size: 13.5px; color: var(--text-secondary); text-decoration: none; } +.ld-footer__col a:hover { color: var(--fg-1); } + +/* ---------- responsive ---------- */ +@media (max-width: 1180px) { + .ld-docs { grid-template-columns: 240px minmax(0, 1fr); } + .ld-toc { display: none; } +} +@media (max-width: 960px) { + /* Header folds into two rows: brand + icon actions, then a scrollable + section-tab row. Text-heavy controls collapse to icons. */ + .ld-header { flex-wrap: wrap; height: auto; gap: 0 10px; padding: 10px 16px 0; } + .ld-header__actions { margin-left: auto; } + .ld-header__actions .ld-btn--primary, + .ld-header__actions .ld-btn__label, + .ld-header__actions .ld-btn__count { display: none; } + .ld-topnav { order: 1; flex-basis: 100%; margin: 4px 0 0; padding-bottom: 10px; } + .ld-toptab { padding: 7px 10px; } + .ld-toptab:first-child { margin-left: -10px; } + .ld-toptab.active::after { left: 10px; right: 10px; } + + .ld-article :is(h1, h2, h3, h4, h5, h6) { scroll-margin-top: 104px; } + .ld-article h1 { font-size: 32px; } + + .ld-hero h1 { font-size: clamp(38px, 9vw, 56px); } + .ld-stats { grid-template-columns: 1fr; } + .ld-stat { padding: 20px 0; } + .ld-stat + .ld-stat { border-left: 0; border-top: 1px solid var(--line-2); } + .ld-feature { grid-template-columns: 1fr; gap: 16px; } + .ld-feature__num { padding-top: 0; } + + /* Docs: sidenav collapses behind a disclosure button; article leads. */ + .ld-docs { display: block; padding: 0 20px; } + .ld-sidenav-toggle { display: flex; } + .ld-sidenav { + display: none; + position: static; + height: auto; + border-right: 0; + border-bottom: 1px solid var(--line-2); + padding: 0 0 20px; + } + .ld-sidenav.open { display: block; } + .ld-sidenav__group:first-child .ld-sidenav__label { margin-top: 16px; } + .ld-article-col { padding-top: 24px; } + + .ld-hero, .ld-band, .ld-what, .ld-features { padding-left: 20px; padding-right: 20px; } + .ld-hero { padding-top: 56px; padding-bottom: 48px; } + .ld-notfound { padding: 72px 20px 96px; } + .ld-footer { padding: 32px 20px; } + .ld-footer__cols { gap: 28px 40px; } +} +@media (max-width: 700px) { + /* Search becomes an edge-to-edge sheet under the top of the screen. */ + .ld-search__panel { margin: 0; max-width: none; border-left: 0; border-right: 0; border-top: 0; } + .ld-search__results { max-height: calc(100dvh - 110px); } +} diff --git a/docs/theme/assets/site.js b/docs/theme/assets/site.js new file mode 100644 index 00000000000..34378c820b2 --- /dev/null +++ b/docs/theme/assets/site.js @@ -0,0 +1,263 @@ +/* Lance docs theme behaviours: theme toggle, GitHub stars, code block chrome, + TOC scroll-spy, search overlay, mermaid rendering. */ +(function () { + "use strict"; + + var BASE = (window.LD_BASE || ".").replace(/\/$/, ""); + + /* ---------- theme toggle ---------- */ + var themeBtn = document.getElementById("theme-toggle"); + if (themeBtn) { + themeBtn.addEventListener("click", function () { + var next = document.documentElement.dataset.theme === "dark" ? "light" : "dark"; + document.documentElement.dataset.theme = next; + try { localStorage.setItem("ld-theme", next); } catch (e) { /* private mode */ } + }); + } + + /* ---------- GitHub stars ---------- */ + (function loadStars() { + var el = document.getElementById("gh-stars"); + if (!el) return; + var TTL = 3600e3; + function show(count) { + if (!(count > 0)) return; + var label = count >= 1000 ? (count / 1000).toFixed(1).replace(/\.0$/, "") + "k" : String(count); + el.textContent = "★ " + label; + el.hidden = false; + } + try { + var cached = JSON.parse(localStorage.getItem("ld-gh-stars") || "null"); + if (cached && Date.now() - cached.t < TTL) { show(cached.v); return; } + } catch (e) { /* fall through to fetch */ } + fetch("https://api.github.com/repos/lance-format/lance") + .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); }) + .then(function (d) { + show(d.stargazers_count); + try { localStorage.setItem("ld-gh-stars", JSON.stringify({ v: d.stargazers_count, t: Date.now() })); } catch (e) { /* ignore */ } + }) + .catch(function () { /* rate-limited or offline: button still works without the count */ }); + })(); + + /* ---------- mobile sidenav toggle ---------- */ + var sidenavToggle = document.querySelector(".ld-sidenav-toggle"); + var sidenav = document.querySelector(".ld-sidenav"); + if (sidenavToggle && sidenav) { + sidenavToggle.addEventListener("click", function () { + var open = sidenav.classList.toggle("open"); + sidenavToggle.classList.toggle("open", open); + sidenavToggle.setAttribute("aria-expanded", open ? "true" : "false"); + }); + } + + /* ---------- article enhancements ---------- */ + var article = document.querySelector(".ld-article"); + + if (article) { + // Language logos for code-bar labels (Simple Icons + Devicon paths, drawn in currentColor). + var LANG_ICONS = { + python: ["0 0 24 24", "M14.25.18l.9.2.73.26.59.3.45.32.34.34.25.34.16.33.1.3.04.26.02.2-.01.13V8.5l-.05.63-.13.55-.21.46-.26.38-.3.31-.33.25-.35.19-.35.14-.33.1-.3.07-.26.04-.21.02H8.77l-.69.05-.59.14-.5.22-.41.27-.33.32-.27.35-.2.36-.15.37-.1.35-.07.32-.04.27-.02.21v3.06H3.17l-.21-.03-.28-.07-.32-.12-.35-.18-.36-.26-.36-.36-.35-.46-.32-.59-.28-.73-.21-.88-.14-1.05-.05-1.23.06-1.22.16-1.04.24-.87.32-.71.36-.57.4-.44.42-.33.42-.24.4-.16.36-.1.32-.05.24-.01h.16l.06.01h8.16v-.83H6.18l-.01-2.75-.02-.37.05-.34.11-.31.17-.28.25-.26.31-.23.38-.2.44-.18.51-.15.58-.12.64-.1.71-.06.77-.04.84-.02 1.27.05zm-6.3 1.98l-.23.33-.08.41.08.41.23.34.33.22.41.09.41-.09.33-.22.23-.34.08-.41-.08-.41-.23-.33-.33-.22-.41-.09-.41.09zm13.09 3.95l.28.06.32.12.35.18.36.27.36.35.35.47.32.59.28.73.21.88.14 1.04.05 1.23-.06 1.23-.16 1.04-.24.86-.32.71-.36.57-.4.45-.42.33-.42.24-.4.16-.36.09-.32.05-.24.02-.16-.01h-8.22v.82h5.84l.01 2.76.02.36-.05.34-.11.31-.17.29-.25.25-.31.24-.38.2-.44.17-.51.15-.58.13-.64.09-.71.07-.77.04-.84.01-1.27-.04-1.07-.14-.9-.2-.73-.25-.59-.3-.45-.33-.34-.34-.25-.34-.16-.33-.1-.3-.04-.25-.02-.2.01-.13v-5.34l.05-.64.13-.54.21-.46.26-.38.3-.32.33-.24.35-.2.35-.14.33-.1.3-.06.26-.04.21-.02.13-.01h5.84l.69-.05.59-.14.5-.21.41-.28.33-.32.27-.35.2-.36.15-.36.1-.35.07-.32.04-.28.02-.21V6.07h2.09l.14.01zm-6.47 14.25l-.23.33-.08.41.08.41.23.33.33.23.41.08.41-.08.33-.23.23-.33.08-.41-.08-.41-.23-.33-.33-.23-.41-.08-.41.08z"], + rust: ["0 0 24 24", "M23.8346 11.7033l-1.0073-.6236a13.7268 13.7268 0 00-.0283-.2936l.8656-.8069a.3483.3483 0 00-.1154-.578l-1.1066-.414a8.4958 8.4958 0 00-.087-.2856l.6904-.9587a.3462.3462 0 00-.2257-.5446l-1.1663-.1894a9.3574 9.3574 0 00-.1407-.2622l.49-1.0761a.3437.3437 0 00-.0274-.3361.3486.3486 0 00-.3006-.154l-1.1845.0416a6.7444 6.7444 0 00-.1873-.2268l.2723-1.153a.3472.3472 0 00-.417-.4172l-1.1532.2724a14.0183 14.0183 0 00-.2278-.1873l.0415-1.1845a.3442.3442 0 00-.49-.328l-1.076.491c-.0872-.0476-.1742-.0952-.2623-.1407l-.1903-1.1673A.3483.3483 0 0016.256.955l-.9597.6905a8.4867 8.4867 0 00-.2855-.086l-.414-1.1066a.3483.3483 0 00-.5781-.1154l-.8069.8666a9.2936 9.2936 0 00-.2936-.0284L12.2946.1683a.3462.3462 0 00-.5892 0l-.6236 1.0073a13.7383 13.7383 0 00-.2936.0284L9.9803.3374a.3462.3462 0 00-.578.1154l-.4141 1.1065c-.0962.0274-.1903.0567-.2855.086L7.744.955a.3483.3483 0 00-.5447.2258L7.009 2.348a9.3574 9.3574 0 00-.2622.1407l-1.0762-.491a.3462.3462 0 00-.49.328l.0416 1.1845a7.9826 7.9826 0 00-.2278.1873L3.8413 3.425a.3472.3472 0 00-.4171.4171l.2713 1.1531c-.0628.075-.1255.1509-.1863.2268l-1.1845-.0415a.3462.3462 0 00-.328.49l.491 1.0761a9.167 9.167 0 00-.1407.2622l-1.1662.1894a.3483.3483 0 00-.2258.5446l.6904.9587a13.303 13.303 0 00-.087.2855l-1.1065.414a.3483.3483 0 00-.1155.5781l.8656.807a9.2936 9.2936 0 00-.0283.2935l-1.0073.6236a.3442.3442 0 000 .5892l1.0073.6236c.008.0982.0182.1964.0283.2936l-.8656.8079a.3462.3462 0 00.1155.578l1.1065.4141c.0273.0962.0567.1914.087.2855l-.6904.9587a.3452.3452 0 00.2268.5447l1.1662.1893c.0456.088.0922.1751.1408.2622l-.491 1.0762a.3462.3462 0 00.328.49l1.1834-.0415c.0618.0769.1235.1528.1873.2277l-.2713 1.1541a.3462.3462 0 00.4171.4161l1.153-.2713c.075.0638.151.1255.2279.1863l-.0415 1.1845a.3442.3442 0 00.49.327l1.0761-.49c.087.0486.1741.0951.2622.1407l.1903 1.1662a.3483.3483 0 00.5447.2268l.9587-.6904a9.299 9.299 0 00.2855.087l.414 1.1066a.3452.3452 0 00.5781.1154l.8079-.8656c.0972.0111.1954.0203.2936.0294l.6236 1.0073a.3472.3472 0 00.5892 0l.6236-1.0073c.0982-.0091.1964-.0183.2936-.0294l.8069.8656a.3483.3483 0 00.578-.1154l.4141-1.1066a8.4626 8.4626 0 00.2855-.087l.9587.6904a.3452.3452 0 00.5447-.2268l.1903-1.1662c.088-.0456.1751-.0931.2622-.1407l1.0762.49a.3472.3472 0 00.49-.327l-.0415-1.1845a6.7267 6.7267 0 00.2267-.1863l1.1531.2713a.3472.3472 0 00.4171-.416l-.2713-1.1542c.0628-.0749.1255-.1508.1863-.2278l1.1845.0415a.3442.3442 0 00.328-.49l-.49-1.076c.0475-.0872.0951-.1742.1407-.2623l1.1662-.1893a.3483.3483 0 00.2258-.5447l-.6904-.9587.087-.2855 1.1066-.414a.3462.3462 0 00.1154-.5781l-.8656-.8079c.0101-.0972.0202-.1954.0283-.2936l1.0073-.6236a.3442.3442 0 000-.5892zm-6.7413 8.3551a.7138.7138 0 01.2986-1.396.714.714 0 11-.2997 1.396zm-.3422-2.3142a.649.649 0 00-.7715.5l-.3573 1.6685c-1.1035.501-2.3285.7795-3.6193.7795a8.7368 8.7368 0 01-3.6951-.814l-.3574-1.6684a.648.648 0 00-.7714-.499l-1.473.3158a8.7216 8.7216 0 01-.7613-.898h7.1676c.081 0 .1356-.0141.1356-.088v-2.536c0-.074-.0536-.0881-.1356-.0881h-2.0966v-1.6077h2.2677c.2065 0 1.1065.0587 1.394 1.2088.0901.3533.2875 1.5044.4232 1.8729.1346.413.6833 1.2381 1.2685 1.2381h3.5716a.7492.7492 0 00.1296-.0131 8.7874 8.7874 0 01-.8119.9526zM6.8369 20.024a.714.714 0 11-.2997-1.396.714.714 0 01.2997 1.396zM4.1177 8.9972a.7137.7137 0 11-1.304.5791.7137.7137 0 011.304-.579zm-.8352 1.9813l1.5347-.6824a.65.65 0 00.33-.8585l-.3158-.7147h1.2432v5.6025H3.5669a8.7753 8.7753 0 01-.2834-3.348zm6.7343-.5437V8.7836h2.9601c.153 0 1.0792.1772 1.0792.8697 0 .575-.7107.7815-1.2948.7815zm10.7574 1.4862c0 .2187-.008.4363-.0243.651h-.9c-.09 0-.1265.0586-.1265.1477v.413c0 .973-.5487 1.1846-1.0296 1.2382-.4576.0517-.9648-.1913-1.0275-.4717-.2704-1.5186-.7198-1.8436-1.4305-2.4034.8817-.5599 1.799-1.386 1.799-2.4915 0-1.1936-.819-1.9458-1.3769-2.3153-.7825-.5163-1.6491-.6195-1.883-.6195H5.4682a8.7651 8.7651 0 014.907-2.7699l1.0974 1.151a.648.648 0 00.9182.0213l1.227-1.1743a8.7753 8.7753 0 016.0044 4.2762l-.8403 1.8982a.652.652 0 00.33.8585l1.6178.7188c.0283.2875.0425.577.0425.8717zm-9.3006-9.5993a.7128.7128 0 11.984 1.0316.7137.7137 0 01-.984-1.0316zm8.3389 6.71a.7107.7107 0 01.9395-.3625.7137.7137 0 11-.9405.3635z"], + java: ["0 0 128 128", "M47.617 98.12c-19.192 5.362 11.677 16.439 36.115 5.969-4.003-1.556-6.874-3.351-6.874-3.351-10.897 2.06-15.952 2.222-25.844 1.092-8.164-.935-3.397-3.71-3.397-3.71zm33.189-10.46c-14.444 2.779-22.787 2.69-33.354 1.6-8.171-.845-2.822-4.805-2.822-4.805-21.137 7.016 11.767 14.977 41.309 6.336-3.14-1.106-5.133-3.131-5.133-3.131zm11.319-60.575c.001 0-42.731 10.669-22.323 34.187 6.024 6.935-1.58 13.17-1.58 13.17s15.289-7.891 8.269-17.777c-6.559-9.215-11.587-13.793 15.634-29.58zm9.998 81.144s3.529 2.91-3.888 5.159c-14.102 4.272-58.706 5.56-71.095.171-4.45-1.938 3.899-4.625 6.526-5.192 2.739-.593 4.303-.485 4.303-.485-4.952-3.487-32.013 6.85-13.742 9.815 49.821 8.076 90.817-3.637 77.896-9.468zM85 77.896c2.395-1.634 5.703-3.053 5.703-3.053s-9.424 1.685-18.813 2.474c-11.494.964-23.823 1.154-30.012.326-14.652-1.959 8.033-7.348 8.033-7.348s-8.812-.596-19.644 4.644C17.455 81.134 61.958 83.958 85 77.896zm5.609 15.145c-.108.29-.468.616-.468.616 31.273-8.221 19.775-28.979 4.822-23.725-1.312.464-2 1.543-2 1.543s.829-.334 2.678-.72c7.559-1.575 18.389 10.119-5.032 22.286zM64.181 70.069c-4.614-10.429-20.26-19.553.007-35.559C89.459 14.563 76.492 1.587 76.492 1.587c5.23 20.608-18.451 26.833-26.999 39.667-5.821 8.745 2.857 18.142 14.688 28.815zm27.274 51.748c-19.187 3.612-42.854 3.191-56.887.874 0 0 2.874 2.38 17.646 3.331 22.476 1.437 57-.8 57.816-11.436.001 0-1.57 4.032-18.575 7.231z"], + bash: ["0 0 24 24", "M21.038,4.9l-7.577-4.498C13.009,0.134,12.505,0,12,0c-0.505,0-1.009,0.134-1.462,0.403L2.961,4.9 C2.057,5.437,1.5,6.429,1.5,7.503v8.995c0,1.073,0.557,2.066,1.462,2.603l7.577,4.497C10.991,23.866,11.495,24,12,24 c0.505,0,1.009-0.134,1.461-0.402l7.577-4.497c0.904-0.537,1.462-1.529,1.462-2.603V7.503C22.5,6.429,21.943,5.437,21.038,4.9z M15.17,18.946l0.013,0.646c0.001,0.078-0.05,0.167-0.111,0.198l-0.383,0.22c-0.061,0.031-0.111-0.007-0.112-0.085L14.57,19.29 c-0.328,0.136-0.66,0.169-0.872,0.084c-0.04-0.016-0.057-0.075-0.041-0.142l0.139-0.584c0.011-0.046,0.036-0.092,0.069-0.121 c0.012-0.011,0.024-0.02,0.036-0.026c0.022-0.011,0.043-0.014,0.062-0.006c0.229,0.077,0.521,0.041,0.802-0.101 c0.357-0.181,0.596-0.545,0.592-0.907c-0.003-0.328-0.181-0.465-0.613-0.468c-0.55,0.001-1.064-0.107-1.072-0.917 c-0.007-0.667,0.34-1.361,0.889-1.8l-0.007-0.652c-0.001-0.08,0.048-0.168,0.111-0.2l0.37-0.236 c0.061-0.031,0.111,0.007,0.112,0.087l0.006,0.653c0.273-0.109,0.511-0.138,0.726-0.088c0.047,0.012,0.067,0.076,0.048,0.151 l-0.144,0.578c-0.011,0.044-0.036,0.088-0.065,0.116c-0.012,0.012-0.025,0.021-0.038,0.028c-0.019,0.01-0.038,0.013-0.057,0.009 c-0.098-0.022-0.332-0.073-0.699,0.113c-0.385,0.195-0.52,0.53-0.517,0.778c0.003,0.297,0.155,0.387,0.681,0.396 c0.7,0.012,1.003,0.318,1.01,1.023C16.105,17.747,15.736,18.491,15.17,18.946z M19.143,17.859c0,0.06-0.008,0.116-0.058,0.145 l-1.916,1.164c-0.05,0.029-0.09,0.004-0.09-0.056v-0.494c0-0.06,0.037-0.093,0.087-0.122l1.887-1.129 c0.05-0.029,0.09-0.004,0.09,0.056V17.859z M20.459,6.797l-7.168,4.427c-0.894,0.523-1.553,1.109-1.553,2.187v8.833 c0,0.645,0.26,1.063,0.66,1.184c-0.131,0.023-0.264,0.039-0.398,0.039c-0.42,0-0.833-0.114-1.197-0.33L3.226,18.64 c-0.741-0.44-1.201-1.261-1.201-2.142V7.503c0-0.881,0.46-1.702,1.201-2.142l7.577-4.498c0.363-0.216,0.777-0.33,1.197-0.33 c0.419,0,0.833,0.114,1.197,0.33l7.577,4.498c0.624,0.371,1.046,1.013,1.164,1.732C21.686,6.557,21.12,6.411,20.459,6.797z"], + }; + var LANG_ALIASES = { py: "python", python3: "python", rs: "rust", sh: "bash", shell: "bash", console: "bash", zsh: "bash" }; + + // Code blocks: wrap .highlight in a window with a language bar + copy button. + article.querySelectorAll("div.highlight").forEach(function (hl) { + if (hl.closest(".ld-code")) return; + var code = hl.querySelector("code"); + var m = (hl.className + " " + (code ? code.className : "")).match(/language-([\w+-]+)/); + var lang = m ? m[1] : "text"; + var wrap = document.createElement("div"); + wrap.className = "ld-code"; + var bar = document.createElement("div"); + bar.className = "ld-code__bar"; + bar.innerHTML = ""; + var span = bar.querySelector("span"); + var icon = LANG_ICONS[LANG_ALIASES[lang] || lang]; + if (icon) span.innerHTML = ''; + span.appendChild(document.createTextNode(lang)); + hl.parentNode.insertBefore(wrap, hl); + wrap.appendChild(bar); + wrap.appendChild(hl); + }); + + document.addEventListener("click", function (e) { + var copy = e.target.closest(".ld-copy"); + if (!copy) return; + var pre = copy.closest(".ld-code").querySelector("pre"); + if (navigator.clipboard && pre) navigator.clipboard.writeText(pre.textContent); + copy.textContent = "Copied"; + setTimeout(function () { copy.textContent = "Copy"; }, 1400); + }); + + // Tables: wrap for horizontal overflow. + article.querySelectorAll("table").forEach(function (t) { + if (t.closest(".ld-tablewrap")) return; + var wrap = document.createElement("div"); + wrap.className = "ld-tablewrap"; + t.parentNode.insertBefore(wrap, t); + wrap.appendChild(t); + }); + + // Mermaid: render fenced diagrams on demand. + var mermaidNodes = article.querySelectorAll("pre.mermaid, div.mermaid"); + if (mermaidNodes.length) { + import("https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs").then(function (mod) { + var mermaid = mod.default; + mermaidNodes.forEach(function (node) { + if (node.tagName === "PRE") { + var div = document.createElement("div"); + div.className = "mermaid"; + div.textContent = node.textContent; + node.replaceWith(div); + } + }); + mermaid.initialize({ startOnLoad: false, securityLevel: "loose", theme: "neutral" }); + mermaid.run({ querySelector: ".ld-article div.mermaid" }); + }).catch(function () { /* offline: leave the diagram source visible */ }); + } + } + + /* ---------- TOC scroll-spy ---------- */ + var tocLinks = Array.prototype.slice.call(document.querySelectorAll(".ld-toc a")); + if (tocLinks.length && article) { + var targets = tocLinks.map(function (a) { + var id = decodeURIComponent((a.getAttribute("href") || "").replace(/^#/, "")); + return document.getElementById(id); + }); + var update = function () { + var active = 0; + for (var i = 0; i < targets.length; i++) { + if (targets[i] && targets[i].getBoundingClientRect().top <= 90) active = i; + } + tocLinks.forEach(function (a, i) { a.classList.toggle("active", i === active); }); + }; + window.addEventListener("scroll", update, { passive: true }); + update(); + } + + /* ---------- search overlay ---------- */ + var overlay = document.getElementById("search-overlay"); + var input = document.getElementById("search-input"); + var results = document.getElementById("search-results"); + var indexPromise = null; + + function loadIndex() { + if (!indexPromise) { + indexPromise = fetch(BASE + "/search/search_index.json") + .then(function (r) { if (!r.ok) throw new Error(r.status); return r.json(); }) + .then(function (d) { + return d.docs.map(function (doc) { + return { + location: doc.location, + title: doc.title || "", + text: (doc.text || "").replace(/\s+/g, " "), + }; + }); + }); + } + return indexPromise; + } + + function esc(s) { + return s.replace(/&/g, "&").replace(//g, ">"); + } + + function highlight(text, terms) { + var out = esc(text); + terms.forEach(function (t) { + out = out.replace(new RegExp("(" + t.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + ")", "ig"), "$1"); + }); + return out; + } + + function search(docs, query) { + var terms = query.toLowerCase().split(/\s+/).filter(Boolean); + if (!terms.length) return []; + var scored = []; + docs.forEach(function (doc) { + var title = doc.title.toLowerCase(); + var text = doc.text.toLowerCase(); + var score = 0; + for (var i = 0; i < terms.length; i++) { + var t = terms[i]; + var inTitle = title.indexOf(t) !== -1; + var inText = text.indexOf(t) !== -1; + if (!inTitle && !inText) { score = 0; break; } + score += (inTitle ? 10 : 0) + (inText ? 1 : 0); + } + // Prefer page-level entries slightly over deep anchors. + if (score > 0) scored.push({ doc: doc, score: score + (doc.location.indexOf("#") === -1 ? 2 : 0) }); + }); + scored.sort(function (a, b) { return b.score - a.score; }); + return scored.slice(0, 12).map(function (s) { return s.doc; }); + } + + function snippet(text, terms) { + var lower = text.toLowerCase(); + var pos = -1; + for (var i = 0; i < terms.length; i++) { + pos = lower.indexOf(terms[i]); + if (pos !== -1) break; + } + if (pos === -1) pos = 0; + var start = Math.max(0, pos - 60); + var s = (start > 0 ? "…" : "") + text.slice(start, start + 160) + (start + 160 < text.length ? "…" : ""); + return s; + } + + function render(docs, query) { + var terms = query.toLowerCase().split(/\s+/).filter(Boolean); + if (!docs.length) { + results.innerHTML = "
No results
"; + return; + } + results.innerHTML = docs.map(function (doc) { + var crumb = doc.location.split("#")[0].replace(/\/$/, "").replace(/\//g, " / ") || "home"; + return "" + + "
" + esc(crumb) + "
" + + "
" + highlight(doc.title, terms) + "
" + + "
" + highlight(snippet(doc.text, terms), terms) + "
" + + ""; + }).join(""); + } + + function openSearch() { + overlay.hidden = false; + input.value = ""; + results.innerHTML = ""; + input.focus(); + loadIndex(); + } + function closeSearch() { overlay.hidden = true; } + + if (overlay && input && results) { + var openBtn = document.getElementById("search-open"); + if (openBtn) openBtn.addEventListener("click", openSearch); + overlay.addEventListener("click", function (e) { + if (e.target.closest("[data-search-close]")) closeSearch(); + }); + document.addEventListener("keydown", function (e) { + if (e.key === "Escape" && !overlay.hidden) { closeSearch(); return; } + var typing = /^(INPUT|TEXTAREA|SELECT)$/.test((document.activeElement || {}).tagName || ""); + if ((e.key === "/" && !typing) || ((e.metaKey || e.ctrlKey) && e.key.toLowerCase() === "k")) { + e.preventDefault(); + if (overlay.hidden) openSearch(); else closeSearch(); + } + }); + var pending = 0; + input.addEventListener("input", function () { + var q = input.value.trim(); + var seq = ++pending; + if (q.length < 2) { results.innerHTML = ""; return; } + loadIndex().then(function (docs) { + if (seq !== pending) return; + render(search(docs, q), q); + }).catch(function () { + results.innerHTML = "
Search index unavailable
"; + }); + }); + } +})(); diff --git a/docs/theme/assets/tokens.css b/docs/theme/assets/tokens.css new file mode 100644 index 00000000000..29c08f2711e --- /dev/null +++ b/docs/theme/assets/tokens.css @@ -0,0 +1,142 @@ +/* Lance Design System — tokens (Swiss editorial: white, ink, brand purple #625EFF). + Merged from the design project's ds/tokens set; dark theme mirrors the same + structure with paper rules on ink. */ +@import url("https://fonts.googleapis.com/css2?family=Space+Grotesk:wght@400;500;600;700&family=IBM+Plex+Sans:wght@400;500;600&family=Roboto+Mono:wght@400;500;600&display=swap"); + +:root { + color-scheme: light; + + /* ---- Typography ---- */ + --font-display: "Space Grotesk", "Segoe UI", sans-serif; + --font-body: "IBM Plex Sans", "Segoe UI", sans-serif; + --font-mono: "Roboto Mono", "SF Mono", monospace; + --text-base: 16px; + --leading-body: 1.6; + --tracking-display: -0.045em; + --tracking-caps: 0.14em; + + /* ---- Ink scale (dark values — code surfaces & tooltips only) ---- */ + --ink-950: #0E0C14; + --ink-900: #14111D; + --ink-850: #1A1626; + --ink-800: #221D30; + --ink-700: #2C2640; + --ink-600: #3A3352; + + /* ---- Foreground scale (on white) ---- */ + --fg-1: #0E0C14; + --fg-2: #4A4459; + --fg-3: #8A8499; + --fg-inverse: #ffffff; + + /* ---- Brand purple ramp (official #625EFF) ---- */ + --beam-300: #B3B1FF; + --beam-400: #625EFF; + --beam-500: #625EFF; + --beam-600: #4B47E0; + --beam-700: #3936B4; + --beam-glow: rgba(98, 94, 255, 0.25); + --beam-dim: rgba(98, 94, 255, 0.09); + + /* ---- Semantic status ---- */ + --ok-500: #0E8F63; + --warn-500: #B26205; + --danger-500: #D92D20; + + /* ---- Lines: ink rules carry the structure ---- */ + --line-1: #14111D; + --line-2: rgba(20, 17, 29, 0.16); + + /* ---- Surfaces ---- */ + --surface-page: #FFFFFF; + --surface-card: #FFFFFF; + --surface-overlay: #F5F4F8; + --surface-header: rgba(255, 255, 255, 0.88); + --surface-code: #F5F4F8; /* light code surface */ + --surface-inline-code: rgba(20, 17, 29, 0.05); + + --text-body: var(--fg-1); + --text-secondary: var(--fg-2); + --text-muted: var(--fg-3); + + /* ---- Syntax highlighting (Pygments), light theme ---- */ + --code-fg: #2A2635; + --code-kw: #7C3AED; /* keywords, builtins */ + --code-str: #0B62C4; /* strings */ + --code-com: #6E6A80; /* comments */ + --code-num: #9A6700; /* numbers */ + --code-fn: #0F766E; /* functions, decorators */ + --code-name: #2A2635; /* class/module names, headings */ + --code-tag: #BE123C; /* HTML tags, attributes */ + --code-punct: #57534E; /* operators, punctuation, output */ + --code-prompt: #8A8499; /* REPL prompts */ + --code-hll: rgba(98, 94, 255, 0.12); /* highlighted line */ + + /* ---- Layout ---- */ + --container-max: 1200px; + --radius-box: 6px; /* content boxes: code, admonitions, tabs */ + --radius-chip: 4px; /* inline code chips */ + + /* ---- Motion — fast, precise, no bounce ---- */ + --ease-out: cubic-bezier(0.2, 0.8, 0.2, 1); + --dur-fast: 120ms; +} + +/* ---- Dark theme: same structure, paper rules on ink ---- */ +:root[data-theme="dark"] { + color-scheme: dark; + + --fg-1: #F2F0FA; + --fg-2: #B7B1C6; + --fg-3: #837D96; + --fg-inverse: #0E0C14; + + /* text accents need a lighter purple to stay readable on ink */ + --beam-600: #8E8BFF; + --beam-700: #A9A6FF; + + --ok-500: #34C08E; + --warn-500: #E8A13C; + --danger-500: #FF6B5E; + + --line-1: rgba(237, 235, 245, 0.7); + --line-2: rgba(237, 235, 245, 0.15); + + --surface-page: #0E0C14; + --surface-card: #14111D; + --surface-overlay: #1A1626; + --surface-header: rgba(14, 12, 20, 0.85); + --surface-code: #14111D; + --surface-inline-code: rgba(237, 235, 245, 0.08); + + /* ---- Syntax highlighting (Pygments), dark theme (on ink) ---- */ + --code-fg: #EDEBF5; + --code-kw: #B8A8FF; + --code-str: #7CC0FF; + --code-com: #6F6A85; + --code-num: #F2CE6B; + --code-fn: #8FE8CE; + --code-name: #EDEBF5; + --code-tag: #FF95A8; + --code-punct: #B7B1C6; + --code-prompt: #6F6A85; + --code-hll: rgba(98, 94, 255, 0.22); +} + +/* ---- Minimal base ---- */ +* { box-sizing: border-box; } + +body { + margin: 0; + background: var(--surface-page); + color: var(--text-body); + font-family: var(--font-body); + font-size: var(--text-base); + line-height: var(--leading-body); + -webkit-font-smoothing: antialiased; +} + +/* No `color` here: forcing ink text breaks selection inside dark code blocks. */ +::selection { background: rgba(98, 94, 255, 0.3); } + +code, kbd, pre { font-family: var(--font-mono); } diff --git a/docs/theme/base.html b/docs/theme/base.html new file mode 100644 index 00000000000..d90b2ae02c4 --- /dev/null +++ b/docs/theme/base.html @@ -0,0 +1,118 @@ +{#- Lance Docs custom theme — shared skeleton: head, header, footer, scripts. -#} +{%- macro first_url(item) -%} + {%- if item.is_link or item.is_page -%} + {{- item.url -}} + {%- elif item.children -%} + {{- first_url(item.children[0]) -}} + {%- endif -%} +{%- endmacro -%} + + + + + +{% if page and page.title and not page.is_homepage %} +{{ page.title }} — {{ config.site_name }} +{% else %} +{{ config.site_name }} — The open lakehouse format for multimodal AI +{% endif %} +{% if page and page.meta and page.meta.description %} + +{% elif config.site_description %} + +{% endif %} +{% if page and page.canonical_url %}{% endif %} + + + + + + +
+
+ + {{ config.site_name }} + + +
+ + + + GitHub + + + + + + + Get started +
+
+ + {% block container %}{% endblock %} + + +
+ + + + + + + + diff --git a/docs/theme/home.html b/docs/theme/home.html new file mode 100644 index 00000000000..1b361168d46 --- /dev/null +++ b/docs/theme/home.html @@ -0,0 +1,147 @@ +{% extends "base.html" %} + +{% block container %} +
+
+
Lance format · Open source · Apache-2.0 · VLDB '25 paper ↗
+

The open lakehouse format for multimodal AI.

+

A file format, table format, and catalog spec for building a complete lakehouse on object storage — powering vector and full-text search, feature engineering, and model training with the fast random access and scans that AI workloads need.

+
+ Get started + Read the spec + Train an LLM + Join Discord → +
+
+ +
+
+
+
100×
+
Faster random access than Parquet
+
+
+
1 line
+
To convert Parquet to Lance
+
+ +
VLDB '25
+
Peer-reviewed research paper →
+ +
+
+ +
+

What is Lance?

+

Lance is a modern, open source lakehouse format for multimodal AI. It brings high-performance vector and full-text search, feature engineering, and model training to the lakehouse, powered by fast random access and scans — while keeping SQL analytics, ACID transactions, time travel, and integrations with open engines (Apache Spark, Ray, PyTorch, Trino, DuckDB) and open catalogs (Apache Polaris, Unity Catalog, Apache Gravitino, Hive Metastore).

+

Learn more in the research paper published at VLDB 2025.

+
+ +
+
+
01
+
+

Expressive hybrid search

+

Combine vector similarity, full-text search (BM25), and SQL analytics on the same dataset. All query types are accelerated by secondary indexes that are part of the Lance specification.

+ Learn more → +
+
+ + 
import lance
+
+ds = lance.dataset("s3://my-bucket/docs")
+
+# Full text search
+ds.to_table(full_text_query="machine learning")
+
+# Hybrid search
+ds.to_table(
+    nearest={
+        "column": "embedding", "q": query_vec, "k": 10
+    },
+    filter="year > 2020",
+)
+
+
+ +
+
02
+
+

Lightning-fast random access

+

100x faster random access than Parquet or Iceberg. An optimized file format plus row addressing and secondary indexes let you fetch individual records across files instantly — for ML serving, sampling, and interactive apps.

+ Learn more → +
+
+ + 
import lance
+
+ds = lance.dataset("s3://my-bucket/embeddings.lance")
+
+# Access the 2nd & 51st rows
+ds.take([2, 51], columns=["id", "vec_gemma3"])
+
+# Take 1000 random samples
+ds.sample(1000, columns=["id", "vec_llama"])
+
+
+ +
+
03
+
+

Native multimodal data

+

Store images, videos, audio, text, and embeddings alongside tabular data in one format. Blob encoding handles large binary objects with lazy loading; optimized vector storage accelerates similarity search.

+ Learn more → +
+
+ + 
import lance
+import av
+
+ds = lance.dataset("s3://my-bucket/videos.lance")
+
+# Get blobs from the 2nd and 51st rows
+blobs = ds.take_blobs("video", ids=[2, 51])
+
+for blob in blobs:
+    with av.open(blob) as container:
+        stream = container.streams.video[0]
+        container.seek(start_time=500, stream=stream)
+
+
+ +
+
04
+
+

Data evolution > schema evolution

+

Backfilling column values normally forces a full table rewrite. Lance supports efficient schema evolution with backfill — adding a column with data is just writing new Lance files to the table.

+ Learn more → +
+
+ + 
import lance
+
+dataset = lance.dataset("my_data.lance")
+
+@lance.batch_udf()
+def add_embeddings(batch):
+    vectors = model.encode(batch["text"])
+    return {"embedding": vectors}
+
+dataset.add_columns(add_embeddings)
+
+
+ +
+
05
+
+

Rich ecosystem integrations

+

Works with Pandas, Polars, Ray, and PyTorch for processing and ML. Connects to Apache DataFusion, DuckDB, Apache Spark, Trino, and Apache Flink for SQL analytics and distributed processing.

+ View integrations → +
+
+ Lance ecosystem integrations +
+
+
+
+{% endblock %} diff --git a/docs/theme/main.html b/docs/theme/main.html new file mode 100644 index 00000000000..f9a4a7e6bd9 --- /dev/null +++ b/docs/theme/main.html @@ -0,0 +1,92 @@ +{% extends "base.html" %} + +{#- Render one nav section as sidenav groups: its direct pages/links under the + section's own label, then each child section as a further group. -#} +{%- macro sidenav_section(sec, label) -%} + {%- set direct = sec.children | selectattr("is_section", "ne", true) | list -%} + {%- if direct -%} +
+
{{ label }}
+ {%- for child in direct %} + {%- if child.is_link %} + {{ child.title }} ↗ + {%- else %} + {{ child.title }} + {%- endif %} + {%- endfor %} +
+ {%- endif -%} + {%- for child in sec.children if child.is_section -%} + {{ sidenav_section(child, child.title) }} + {%- endfor -%} +{%- endmacro -%} + +{% block container %} + {%- set active_section = namespace(item=none) -%} + {%- for item in nav %}{% if item.active %}{% set active_section.item = item %}{% endif %}{% endfor %} + +
+ + + +
+
+ Docs + {%- for crumb in page.ancestors | reverse %} + /{{ crumb.title }} + {%- endfor %} + /{{ page.title }} +
+ +
+ {{ page.content }} +
+ +
+ {%- if page.previous_page %} + +
← Previous
+
{{ page.previous_page.title }}
+ + {%- else %}{% endif %} + {%- if page.next_page %} + + {%- endif %} +
+
+ + +
+{% endblock %} diff --git a/java/lance-jni/Cargo.lock b/java/lance-jni/Cargo.lock index e392fd82c76..9b24703dc7f 100644 --- a/java/lance-jni/Cargo.lock +++ b/java/lance-jni/Cargo.lock @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -145,9 +145,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" dependencies = [ "arrow-arith", "arrow-array", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -222,7 +222,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ "arrow-array", "arrow-cast", @@ -248,9 +248,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -261,9 +261,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -277,9 +277,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ "arrow-array", "arrow-buffer", @@ -302,9 +302,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -315,9 +315,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags", "serde_core", @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ "arrow-array", "arrow-buffer", @@ -411,18 +411,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -883,6 +883,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1008,9 +1014,23 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "byteorder" @@ -1020,9 +1040,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1045,9 +1065,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -1126,9 +1146,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1136,9 +1156,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1148,14 +1168,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -1347,12 +1367,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1385,18 +1406,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1518,7 +1539,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", ] [[package]] @@ -1529,7 +1550,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1548,14 +1569,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1582,12 +1602,11 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "log", "object_store", "parking_lot", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -1597,9 +1616,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -1622,9 +1641,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -1645,32 +1664,33 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", "object_store", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -1679,9 +1699,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-trait", @@ -1701,16 +1721,17 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.4", + "parking_lot", + "rand 0.9.5", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -1732,9 +1753,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -1755,9 +1776,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -1772,27 +1793,25 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -1801,18 +1820,19 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -1823,33 +1843,31 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -1860,26 +1878,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -1889,19 +1906,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -1910,9 +1926,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -1926,34 +1942,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -1964,14 +1980,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1979,20 +1994,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2009,11 +2024,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2021,20 +2035,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2047,26 +2060,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2082,12 +2095,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2102,7 +2116,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2114,9 +2128,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2125,15 +2139,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2145,9 +2158,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2163,9 +2176,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -2255,7 +2268,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2366,11 +2379,10 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2462,6 +2474,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -2470,10 +2488,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -2547,7 +2565,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2698,12 +2716,13 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-schema", "datafusion", "geo", @@ -2755,11 +2774,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if 1.0.4", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -2793,7 +2810,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2816,17 +2833,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru", + "memmap2", + "moka", "prost", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "thiserror 2.0.18", @@ -2836,13 +2861,14 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -2863,6 +2889,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if 1.0.4", "crunchy", "num-traits", @@ -2896,6 +2923,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -2915,6 +2944,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapify" @@ -3141,7 +3175,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3495,7 +3529,7 @@ checksum = "782d32378dddf207193ac91cefb848ad41abb58195c95168e1291227a0832b47" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3556,7 +3590,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.118", ] [[package]] @@ -3584,7 +3618,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3622,7 +3656,7 @@ dependencies = [ "nom", "num-traits", "ordered-float", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "zmij", @@ -3647,7 +3681,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arc-swap", "arrow", @@ -3663,7 +3697,6 @@ dependencies = [ "async-recursion", "async-trait", "async_cell", - "aws-credential-types", "byteorder", "bytes", "chrono", @@ -3678,7 +3711,6 @@ dependencies = [ "either", "fst", "futures", - "half", "humantime", "itertools 0.14.0", "lance-arrow", @@ -3702,7 +3734,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rayon", "roaring", "rustc-hash", @@ -3720,7 +3752,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", @@ -3729,13 +3761,14 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", + "bytemuck", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -3762,7 +3795,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrayref", "crunchy", @@ -3772,19 +3805,18 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", - "byteorder", + "blake3", "bytes", "datafusion-common", "datafusion-sql", "futures", - "itertools 0.14.0", "lance-arrow", "lance-derive", "libc", @@ -3795,13 +3827,13 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.4", + "quick_cache", + "rand 0.9.5", "roaring", "serde_json", "snafu", "tempfile", "tokio", - "tokio-stream", "tokio-util", "tracing", "twox-hash", @@ -3810,7 +3842,7 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -3827,10 +3859,10 @@ dependencies = [ "datafusion-physical-expr", "datafusion-substrait", "futures", + "half", "jsonb", "lance-arrow", "lance-core", - "lance-datagen", "lance-geo", "log", "pin-project", @@ -3840,35 +3872,18 @@ dependencies = [ "tracing", ] -[[package]] -name = "lance-datagen" -version = "9.0.0-beta.16" -dependencies = [ - "arrow", - "arrow-array", - "arrow-cast", - "arrow-schema", - "chrono", - "futures", - "half", - "hex", - "rand 0.9.4", - "rand_distr", - "rand_xoshiro", -] - [[package]] name = "lance-derive" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "lance-encoding" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-arith", "arrow-array", @@ -3893,8 +3908,6 @@ dependencies = [ "num-traits", "prost", "prost-build", - "rand 0.9.4", - "strum", "tokio", "tracing", "xxhash-rust", @@ -3903,11 +3916,12 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -3933,7 +3947,7 @@ dependencies = [ [[package]] name = "lance-geo" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "datafusion", "geo-traits", @@ -3947,12 +3961,13 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -3961,7 +3976,6 @@ dependencies = [ "async-trait", "bitvec", "bytes", - "chrono", "crossbeam-queue", "datafusion", "datafusion-common", @@ -3981,10 +3995,10 @@ dependencies = [ "lance-bitpacking", "lance-core", "lance-datafusion", - "lance-datagen", "lance-encoding", "lance-file", "lance-geo", + "lance-index-core", "lance-io", "lance-linalg", "lance-select", @@ -3998,7 +4012,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rangemap", "rayon", @@ -4010,22 +4024,37 @@ dependencies = [ "tempfile", "tokio", "tracing", - "uuid", +] + +[[package]] +name = "lance-index-core" +version = "12.0.0-beta.11" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", ] [[package]] name = "lance-io" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4035,10 +4064,10 @@ dependencies = [ "futures", "http 1.4.2", "io-uring", - "lance-arrow", "lance-core", "lance-namespace", "log", + "metrics", "moka", "object_store", "object_store_opendal", @@ -4046,17 +4075,22 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.4", + "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", + "serde_json", "tempfile", "tokio", "tracing", "url", + "uuid", ] [[package]] name = "lance-jni" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4080,6 +4114,8 @@ dependencies = [ "lance-namespace-impls", "lance-table", "log", + "metrics", + "metrics-util", "object_store", "prost", "prost-types", @@ -4092,35 +4128,35 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", - "arrow-buffer", "arrow-schema", "cc", "half", "lance-arrow", "lance-core", "num-traits", - "rand 0.9.4", "rayon", ] [[package]] name = "lance-namespace" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", + "serde_json", "snafu", ] [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-ipc", @@ -4140,12 +4176,11 @@ dependencies = [ "lance-table", "log", "object_store", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "roaring", "serde", "serde_json", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -4155,9 +4190,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.8.6" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af" dependencies = [ "reqwest 0.12.28", "serde", @@ -4169,13 +4204,12 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", "byteorder", - "bytes", "itertools 0.14.0", "lance-core", "roaring", @@ -4184,7 +4218,7 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4192,6 +4226,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "async-trait", + "blake3", "byteorder", "bytes", "chrono", @@ -4206,7 +4241,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "roaring", "semver", @@ -4221,10 +4256,10 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ + "frostem", "icu_segmenter", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -4236,7 +4271,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] [[package]] @@ -4298,9 +4333,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libm" @@ -4378,6 +4413,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4475,6 +4519,40 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "metrics", + "rand 0.9.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -4686,7 +4764,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4724,7 +4802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -4759,9 +4837,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -4794,12 +4872,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -4817,24 +4896,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64", + "base64 0.23.0", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -4843,11 +4920,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.4.2", @@ -4857,9 +4948,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -4867,9 +4958,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -4878,9 +4969,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -4888,17 +4979,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4909,17 +5000,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -4929,9 +5021,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.4.2", "opendal-core", @@ -4939,15 +5031,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -4956,9 +5048,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -4966,7 +5058,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -4977,9 +5069,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -4991,9 +5083,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -5001,22 +5093,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5025,18 +5116,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64", + "base64 0.23.0", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5046,14 +5137,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "7841a1a09485bd08eeac34d67804321b62456c855027c580bce12a75564f4609" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -5160,7 +5251,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -5207,7 +5298,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -5279,7 +5370,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5386,7 +5477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", ] [[package]] @@ -5432,7 +5523,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.118", "tempfile", ] @@ -5446,7 +5537,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5470,14 +5561,26 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -5500,15 +5603,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -5573,9 +5677,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -5643,7 +5747,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -5661,6 +5774,15 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -5733,7 +5855,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5783,9 +5905,9 @@ dependencies = [ [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f" dependencies = [ "anyhow", "form_urlencoded", @@ -5800,9 +5922,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2" dependencies = [ "anyhow", "bytes", @@ -5811,7 +5933,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", @@ -5822,12 +5944,12 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "http 1.4.2", @@ -5843,14 +5965,13 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -5868,9 +5989,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b" dependencies = [ "anyhow", "reqsign-core", @@ -5879,9 +6000,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -5897,9 +6018,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644" dependencies = [ "anyhow", "http 1.4.2", @@ -5912,9 +6033,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "4335f949a3fd8b53867fd716dac97fbd545e45bcaed44343796517f2e19cd609" dependencies = [ "anyhow", "http 1.4.2", @@ -5929,7 +6050,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -5975,7 +6096,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -6095,21 +6216,11 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -6299,7 +6410,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.118", ] [[package]] @@ -6366,9 +6477,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -6376,22 +6487,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.2", ] [[package]] @@ -6402,15 +6513,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6437,7 +6549,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6449,7 +6561,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.118", ] [[package]] @@ -6470,7 +6582,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -6493,7 +6605,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6626,6 +6738,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -6640,23 +6758,23 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6683,9 +6801,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -6699,9 +6823,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -6715,7 +6839,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6767,35 +6891,14 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn", -] - [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -6809,7 +6912,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn", + "syn 2.0.118", "typify", "walkdir", ] @@ -6837,6 +6940,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -6854,7 +6968,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6943,7 +7057,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6954,7 +7068,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7043,9 +7157,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -7066,7 +7180,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7092,9 +7206,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -7104,13 +7218,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -7152,7 +7267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.2", @@ -7288,7 +7403,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7351,11 +7466,11 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] [[package]] @@ -7395,7 +7510,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.118", "thiserror 2.0.18", "unicode-ident", ] @@ -7413,7 +7528,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 2.0.118", "typify-impl", ] @@ -7500,9 +7615,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -7622,7 +7737,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -7796,7 +7911,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -7807,7 +7922,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -8149,7 +8264,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "clap", "crc32fast", @@ -8186,7 +8301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", @@ -8296,9 +8411,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yoke" @@ -8319,7 +8434,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -8340,7 +8455,7 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -8360,7 +8475,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] @@ -8402,7 +8517,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] diff --git a/java/lance-jni/Cargo.toml b/java/lance-jni/Cargo.toml index d8f839b273a..62828ed855a 100644 --- a/java/lance-jni/Cargo.toml +++ b/java/lance-jni/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lance-jni" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" edition = "2024" authors = ["Lance Devs "] rust-version = "1.91" @@ -20,14 +20,15 @@ crate-type = ["cdylib"] [features] default = [] +backtrace = ["lance/backtrace", "lance-core/backtrace"] [dependencies] -lance = { path = "../../rust/lance", features = ["substrait"] } +lance = { path = "../../rust/lance", features = ["substrait", "metrics"] } lance-datafusion = { path = "../../rust/lance-datafusion" } lance-encoding = { path = "../../rust/lance-encoding" } lance-linalg = { path = "../../rust/lance-linalg" } lance-index = { path = "../../rust/lance-index" } -lance-io = { path = "../../rust/lance-io" } +lance-io = { path = "../../rust/lance-io", features = ["metrics"] } lance-namespace = { path = "../../rust/lance-namespace" } lance-namespace-impls = { path = "../../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] } lance-core = { path = "../../rust/lance-core" } @@ -36,8 +37,8 @@ lance-table = { path = "../../rust/lance-table" } arrow = { version = "58.0.0", features = ["ffi"] } arrow-array = "58.0.0" arrow-schema = "58.0.0" -datafusion = { version = "53.0.0", default-features = false } -datafusion-common = "53.0.0" +datafusion = { version = "54.0.0", default-features = false } +datafusion-common = "54.0.0" object_store = { version = "0.13.2" } tokio = { version = "1.23", features = [ "rt-multi-thread", @@ -51,6 +52,8 @@ serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1" } bytes = "1.11" log = "0.4" +metrics = "0.24" +metrics-util = { version = "0.19", default-features = false, features = ["registry"] } env_logger = "0.11.7" uuid = { version = "1.17.0", features = ["v4"] } prost = "0.14.1" @@ -71,5 +74,8 @@ debug-assertions = false strip = "debuginfo" incremental = false +[profile.release] +strip = true + [lints.clippy] disallowed_macros = "deny" diff --git a/java/lance-jni/src/async_scanner.rs b/java/lance-jni/src/async_scanner.rs index 6da10479266..4f65deda0d4 100644 --- a/java/lance-jni/src/async_scanner.rs +++ b/java/lance-jni/src/async_scanner.rs @@ -3,12 +3,12 @@ use std::sync::Arc; -use crate::RT; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::blocking_scanner::{ScannerOptions, build_scanner_with_options}; use crate::dispatcher::{DISPATCHER, DispatcherMessage}; use crate::error::Result; use crate::task_tracker::{TASK_TRACKER, TaskInfo}; +use crate::{RT, block_on}; use arrow::ffi::FFI_ArrowSchema; use jni::JNIEnv; use jni::objects::JObject; @@ -82,7 +82,7 @@ impl AsyncScanner { // Will be aborted when real handle is registered }); - RT.block_on(async { + block_on(async { TASK_TRACKER .register( task_id, @@ -163,7 +163,7 @@ impl AsyncScanner { }); // Step 3: Update registration with real handle - RT.block_on(async { + block_on(async { TASK_TRACKER.update_handle(task_id, handle).await; }); } @@ -181,6 +181,8 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local substrait_filter_obj: JObject<'local>, filter_obj: JObject<'local>, batch_size_obj: JObject<'local>, + batch_size_bytes_obj: JObject<'local>, + io_buffer_size_obj: JObject<'local>, limit_obj: JObject<'local>, offset_obj: JObject<'local>, query_obj: JObject<'local>, @@ -189,6 +191,9 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local with_row_id: jboolean, with_row_address: jboolean, batch_readahead: jint, + fragment_readahead_obj: JObject<'local>, + scan_in_order: jboolean, + late_materialization_obj: JObject<'local>, column_orderings: JObject<'local>, use_scalar_index: jboolean, fast_search: jboolean, @@ -207,6 +212,8 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -215,6 +222,9 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_createAsyncScanner<'local with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -235,6 +245,8 @@ fn inner_create_async_scanner<'local>( substrait_filter_obj: JObject<'local>, filter_obj: JObject<'local>, batch_size_obj: JObject<'local>, + batch_size_bytes_obj: JObject<'local>, + io_buffer_size_obj: JObject<'local>, limit_obj: JObject<'local>, offset_obj: JObject<'local>, query_obj: JObject<'local>, @@ -243,6 +255,9 @@ fn inner_create_async_scanner<'local>( with_row_id: jboolean, with_row_address: jboolean, batch_readahead: jint, + fragment_readahead_obj: JObject<'local>, + scan_in_order: jboolean, + late_materialization_obj: JObject<'local>, column_orderings: JObject<'local>, use_scalar_index: jboolean, fast_search: jboolean, @@ -262,6 +277,8 @@ fn inner_create_async_scanner<'local>( substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -270,6 +287,9 @@ fn inner_create_async_scanner<'local>( with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -323,7 +343,7 @@ pub extern "system" fn Java_org_lance_ipc_AsyncScanner_nativeCancelTask( _j_scanner: JObject, task_id: jlong, ) { - RT.block_on(async { + block_on(async { TASK_TRACKER.cancel(task_id as u64).await; }); } @@ -361,7 +381,7 @@ fn inner_import_async_ffi_schema( let scanner_guard = unsafe { env.get_rust_field::<_, _, AsyncScanner>(j_scanner, NATIVE_ASYNC_SCANNER)? }; - let schema = RT.block_on(scanner_guard.inner.schema())?; + let schema = block_on(scanner_guard.inner.schema())?; let ffi_schema = FFI_ArrowSchema::try_from(&*schema)?; unsafe { std::ptr::write_unaligned(schema_addr as *mut FFI_ArrowSchema, ffi_schema) } Ok(()) diff --git a/java/lance-jni/src/blocking_blob.rs b/java/lance-jni/src/blocking_blob.rs index 002fa817cf6..bdddd9b9039 100755 --- a/java/lance-jni/src/blocking_blob.rs +++ b/java/lance-jni/src/blocking_blob.rs @@ -4,18 +4,23 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::{FromJString, IntoJava}; -use crate::{JNIEnvExt, RT}; +use crate::{JNIEnvExt, block_on}; use jni::JNIEnv; use jni::objects::{JByteArray, JObject, JString, JValueGen}; -use jni::sys::{jbyteArray, jint, jlong}; +use jni::sys::{jbyte, jbyteArray, jint, jlong}; use lance::dataset::BlobFile; -use std::mem::transmute; use std::sync::Arc; const BLOB_FILE_CLASS: &str = "org/lance/BlobFile"; const BLOB_FILE_CTOR_SIG: &str = "()V"; const NATIVE_BLOB: &str = "nativeBlobHandle"; +fn as_jbytes(bytes: &[u8]) -> &[jbyte] { + // SAFETY: jbyte is i8; u8 and i8 have identical size and alignment, and + // both permit every bit pattern. The returned slice retains the input lifetime. + unsafe { std::slice::from_raw_parts(bytes.as_ptr().cast(), bytes.len()) } +} + pub struct BlockingBlobFile { pub(crate) inner: BlobFile, } @@ -59,23 +64,26 @@ fn inner_take_blobs<'local>( let blobs = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?; - RT.block_on(Arc::new(dataset.inner.clone()).take_blobs(&row_ids_u64, col_name))? + block_on(Arc::new(dataset.inner.clone()).take_blobs(&row_ids_u64, col_name))? }; let j_blobs = blobs .into_iter() - .map(BlockingBlobFile::from) - .collect::>(); + .map(|blob| blob.map(BlockingBlobFile::from)) + .collect::>(); transform_vec(env, j_blobs) } fn transform_vec<'local>( env: &mut JNIEnv<'local>, - vec: Vec, + vec: Vec>, ) -> Result> { let array_list_class = env.find_class("java/util/ArrayList")?; let array_list = env.new_object(array_list_class, "()V", &[])?; for blob_file in vec { - let blob_file_obj = blob_file.into_java(env)?; + let blob_file_obj = match blob_file { + Some(blob_file) => blob_file.into_java(env)?, + None => JObject::null(), + }; env.call_method( &array_list, "add", @@ -111,14 +119,12 @@ fn inner_take_blobs_by_indices<'local>( let blobs = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?; - RT.block_on( - Arc::new(dataset.inner.clone()).take_blobs_by_indices(&row_indices_u64, col_name), - )? + block_on(Arc::new(dataset.inner.clone()).take_blobs_by_indices(&row_indices_u64, col_name))? }; let j_blobs = blobs .into_iter() - .map(BlockingBlobFile::from) - .collect::>(); + .map(|blob| blob.map(BlockingBlobFile::from)) + .collect::>(); transform_vec(env, j_blobs) } @@ -137,13 +143,10 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeRead( fn inner_blob_read<'local>(env: &mut JNIEnv<'local>, jblob: JObject) -> Result> { let bytes = { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.read())? + block_on(blob.inner.read())? }; let arr = env.new_byte_array(bytes.len() as jint)?; - let u8_slice: &[u8] = bytes.as_ref(); - let i8_slice: &[i8] = unsafe { transmute(u8_slice) }; - - env.set_byte_array_region(&arr, 0, i8_slice)?; + env.set_byte_array_region(&arr, 0, as_jbytes(bytes.as_ref()))?; Ok(arr) } @@ -167,13 +170,10 @@ fn inner_blob_read_up_to<'local>( ) -> Result> { let bytes = { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.read_up_to(len as usize))? + block_on(blob.inner.read_up_to(len as usize))? }; let arr = env.new_byte_array(bytes.len() as jint)?; - let u8_slice: &[u8] = bytes.as_ref(); - let i8_slice: &[i8] = unsafe { transmute(u8_slice) }; - - env.set_byte_array_region(&arr, 0, i8_slice)?; + env.set_byte_array_region(&arr, 0, as_jbytes(bytes.as_ref()))?; Ok(arr) } @@ -202,13 +202,10 @@ fn inner_blob_read_range<'local>( .ok_or_else(|| lance_core::Error::invalid_input("offset + len overflowed".to_string()))?; let bytes = { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.read_range(offset as u64..end))? + block_on(blob.inner.read_range(offset as u64..end))? }; let arr = env.new_byte_array(bytes.len() as jint)?; - let u8_slice: &[u8] = bytes.as_ref(); - let i8_slice: &[i8] = unsafe { transmute(u8_slice) }; - - env.set_byte_array_region(&arr, 0, i8_slice)?; + env.set_byte_array_region(&arr, 0, as_jbytes(bytes.as_ref()))?; Ok(arr) } @@ -223,7 +220,7 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeSeek( fn inner_blob_seek(env: &mut JNIEnv, jblob: JObject, new_cursor: jlong) -> Result<()> { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - RT.block_on(blob.inner.seek(new_cursor as u64))?; + block_on(blob.inner.seek(new_cursor as u64))?; Ok(()) } @@ -237,7 +234,7 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeTell( fn inner_blob_tell(env: &mut JNIEnv, jblob: JObject) -> Result { let blob = unsafe { env.get_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB) }?; - Ok(RT.block_on(blob.inner.tell())?) + Ok(block_on(blob.inner.tell())?) } #[unsafe(no_mangle)] @@ -260,6 +257,17 @@ pub extern "system" fn Java_org_lance_BlobFile_nativeClose(mut env: JNIEnv, jblo fn inner_blob_close(env: &mut JNIEnv, jblob: JObject) -> Result<()> { let blob = unsafe { env.take_rust_field::<_, _, BlockingBlobFile>(jblob, NATIVE_BLOB)? }; - RT.block_on(blob.inner.close())?; + block_on(blob.inner.close())?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::as_jbytes; + + #[test] + fn byte_slice_cast_preserves_bits() { + assert_eq!(as_jbytes(&[0, 127, 128, 255]), &[0, 127, -128, -1]); + assert!(as_jbytes(&[]).is_empty()); + } +} diff --git a/java/lance-jni/src/blocking_dataset.rs b/java/lance-jni/src/blocking_dataset.rs index caf837b371a..b1613c5f458 100644 --- a/java/lance-jni/src/blocking_dataset.rs +++ b/java/lance-jni/src/blocking_dataset.rs @@ -3,6 +3,7 @@ use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; +use crate::index_progress::JavaIndexBuildProgress; use crate::namespace::{ BlockingDirectoryNamespace, BlockingRestNamespace, create_java_lance_namespace, }; @@ -13,7 +14,7 @@ use crate::utils::{ extract_write_params, get_scalar_index_params, get_vector_index_params, to_java_map, to_rust_map, }; -use crate::{RT, traits::IntoJava}; +use crate::{block_on, traits::IntoJava}; use arrow::array::RecordBatchReader; use arrow::datatypes::Schema; use arrow::ffi::FFI_ArrowSchema; @@ -21,7 +22,6 @@ use arrow::ffi_stream::ArrowArrayStreamReader; use arrow::ffi_stream::FFI_ArrowArrayStream; use arrow::ipc::writer::StreamWriter; use arrow::record_batch::RecordBatchIterator; -use arrow_schema::DataType; use arrow_schema::Schema as ArrowSchema; use chrono::{DateTime, Utc}; use jni::objects::{JMap, JString, JValue}; @@ -41,12 +41,12 @@ use lance::dataset::{ ColumnAlteration, CommitBuilder, Dataset, NewColumnTransform, ProjectionRequest, ReadParams, Version, WriteParams, }; -use lance::index::{DatasetIndexExt, IndexSegment}; +use lance::index::{DatasetIndexExt, IndexSegment, IntoIndexSegment}; use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore; use lance::io::{ObjectStore, ObjectStoreParams}; use lance::session::Session as LanceSession; use lance::table::format::IndexMetadata; -use lance::table::format::{BasePath, Fragment}; +use lance::table::format::{BasePath, Fragment, WriterVersion}; use lance_core::datatypes::Schema as LanceSchema; use lance_file::version::LanceFileVersion; use lance_index::IndexCriteria as RustIndexCriteria; @@ -58,10 +58,10 @@ use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOption use lance_namespace::LanceNamespace; use lance_table::io::commit::CommitHandler; use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; -use std::collections::HashMap; +use lance_table::io::commit::{ManifestLocation, ManifestNamingScheme}; +use std::collections::{HashMap, HashSet}; use std::future::IntoFuture; use std::iter::empty; -use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; use uuid::Uuid; @@ -103,13 +103,13 @@ impl BlockingDataset { /// If a storage options provider was configured and credentials are expiring, /// this will refresh them. pub fn latest_storage_options(&self) -> Result>> { - RT.block_on(async { self.inner.latest_storage_options().await }) + block_on(async { self.inner.latest_storage_options().await }) .map(|opt| opt.map(|opts| opts.0)) .map_err(|e| Error::io_error(e.to_string())) } pub fn drop(uri: &str, storage_options: HashMap) -> Result<()> { - RT.block_on(async move { + block_on(async move { let registry = Arc::new(ObjectStoreRegistry::default()); let object_store_params = ObjectStoreParams { storage_options_accessor: Some(Arc::new( @@ -121,18 +121,45 @@ impl BlockingDataset { ObjectStore::from_uri_and_params(registry, uri, &object_store_params) .await .map_err(|e| Error::io_error(e.to_string()))?; + lance::dataset::validate_dataset_root_for_drop(&object_store, &path) + .await + .map_err(|e| Error::input_error(e.to_string()))?; object_store .remove_dir_all(path) .await .map_err(|e| Error::io_error(e.to_string())) }) } + + pub fn list_manifest_locations( + uri: &str, + storage_options: HashMap, + ) -> Result> { + let accessor = (!storage_options.is_empty()).then(|| { + Arc::new(lance::io::StorageOptionsAccessor::with_static_options( + storage_options, + )) + }); + let params = ReadParams { + store_options: Some(ObjectStoreParams { + storage_options_accessor: accessor, + ..Default::default() + }), + ..Default::default() + }; + Ok(block_on( + DatasetBuilder::from_uri(uri) + .with_read_params(params) + .list_manifest_locations(), + )?) + } + pub fn write( reader: impl RecordBatchReader + Send + 'static, uri: &str, params: Option, ) -> Result { - let inner = RT.block_on(Dataset::write(reader, uri, params))?; + let inner = block_on(Dataset::write(reader, uri, params))?; Ok(Self { inner }) } @@ -212,7 +239,7 @@ impl BlockingDataset { builder = builder.with_commit_handler(commit_handler); } - let inner = RT.block_on(builder.load())?; + let inner = block_on(builder.load())?; Ok(Self { inner }) } @@ -229,7 +256,7 @@ impl BlockingDataset { lance::io::StorageOptionsAccessor::with_static_options(storage_options), )) }; - let inner = RT.block_on(Dataset::commit( + let inner = block_on(Dataset::commit( uri, operation, read_version, @@ -245,51 +272,55 @@ impl BlockingDataset { } pub fn latest_version(&self) -> Result { - let version = RT.block_on(self.inner.latest_version_id())?; + let version = block_on(self.inner.latest_version_id())?; Ok(version) } pub fn list_versions(&self) -> Result> { - let versions = RT.block_on(self.inner.versions())?; + let versions = block_on(self.inner.versions())?; Ok(versions) } + pub fn count_versions(&self) -> Result { + Ok(block_on(self.inner.count_versions())?) + } + pub fn version(&self) -> Result { Ok(self.inner.version()) } pub fn checkout_version(&mut self, version: u64) -> Result { - let inner = RT.block_on(self.inner.checkout_version(version))?; + let inner = block_on(self.inner.checkout_version(version))?; Ok(Self { inner }) } pub fn checkout_tag(&mut self, tag: &str) -> Result { - let inner = RT.block_on(self.inner.checkout_version(tag))?; + let inner = block_on(self.inner.checkout_version(tag))?; Ok(Self { inner }) } pub fn checkout_latest(&mut self) -> Result<()> { - RT.block_on(self.inner.checkout_latest())?; + block_on(self.inner.checkout_latest())?; Ok(()) } pub fn restore(&mut self) -> Result<()> { - RT.block_on(self.inner.restore())?; + block_on(self.inner.restore())?; Ok(()) } pub fn list_tags(&self) -> Result> { - let tags = RT.block_on(self.inner.tags().list())?; + let tags = block_on(self.inner.tags().list())?; Ok(tags) } pub fn list_branches(&self) -> Result> { - let branches = RT.block_on(self.inner.branches().list())?; + let branches = block_on(self.inner.branches().list())?; Ok(branches) } pub fn delete_branch(&mut self, branch: &str) -> Result<()> { - RT.block_on(self.inner.branches().delete(branch, true))?; + block_on(self.inner.branches().delete(branch, true))?; Ok(()) } @@ -304,22 +335,22 @@ impl BlockingDataset { } else { Ref::Version(branch, version) }; - let inner = RT.block_on(self.inner.checkout_version(reference))?; + let inner = block_on(self.inner.checkout_version(reference))?; Ok(Self { inner }) } pub fn create_tag(&mut self, tag: &str, reference: Ref) -> Result<()> { - RT.block_on(self.inner.tags().create(tag, reference))?; + block_on(self.inner.tags().create(tag, reference))?; Ok(()) } pub fn delete_tag(&mut self, tag: &str) -> Result<()> { - RT.block_on(self.inner.tags().delete(tag))?; + block_on(self.inner.tags().delete(tag))?; Ok(()) } pub fn update_tag(&mut self, tag: &str, reference: Ref) -> Result<()> { - RT.block_on(self.inner.tags().update(tag, reference))?; + block_on(self.inner.tags().update(tag, reference))?; Ok(()) } @@ -328,7 +359,7 @@ impl BlockingDataset { tag: &str, metadata: HashMap, ) -> Result<()> { - RT.block_on(self.inner.tags().replace_metadata(tag, metadata))?; + block_on(self.inner.tags().replace_metadata(tag, metadata))?; Ok(()) } @@ -337,27 +368,27 @@ impl BlockingDataset { branch: &str, metadata: HashMap, ) -> Result<()> { - RT.block_on(self.inner.branches().replace_metadata(branch, metadata))?; + block_on(self.inner.branches().replace_metadata(branch, metadata))?; Ok(()) } pub fn get_version(&self, tag: &str) -> Result { - let version = RT.block_on(self.inner.tags().get_version(tag))?; + let version = block_on(self.inner.tags().get_version(tag))?; Ok(version) } pub fn count_rows(&self, filter: Option) -> Result { - let rows = RT.block_on(self.inner.count_rows(filter))?; + let rows = block_on(self.inner.count_rows(filter))?; Ok(rows) } pub fn calculate_data_stats(&self) -> Result { - let stats = RT.block_on(Arc::new(self.clone().inner).calculate_data_stats())?; + let stats = block_on(Arc::new(self.clone().inner).calculate_data_stats())?; Ok(stats) } pub fn list_indexes(&self) -> Result>> { - let indexes = RT.block_on(self.inner.load_indices())?; + let indexes = block_on(self.inner.load_indices())?; Ok(indexes) } @@ -395,12 +426,12 @@ impl BlockingDataset { if let Some(handler) = commit_handler { builder = builder.with_commit_handler(handler); } - let new_dataset = RT.block_on(builder.execute(transaction))?; + let new_dataset = block_on(builder.execute(transaction))?; Ok(BlockingDataset { inner: new_dataset }) } pub fn read_transaction(&self) -> Result> { - let transaction = RT.block_on(self.inner.read_transaction())?; + let transaction = block_on(self.inner.read_transaction())?; Ok(transaction) } @@ -409,12 +440,12 @@ impl BlockingDataset { } pub fn compact(&mut self, options: RustCompactionOptions) -> Result<()> { - RT.block_on(compact_files(&mut self.inner, options, None))?; + block_on(compact_files(&mut self.inner, options, None))?; Ok(()) } pub fn cleanup_with_policy(&mut self, policy: CleanupPolicy) -> Result { - Ok(RT.block_on(self.inner.cleanup_with_policy(policy))?) + Ok(block_on(self.inner.cleanup_with_policy(policy))?) } pub fn explain_cleanup_with_policy( @@ -426,7 +457,7 @@ impl BlockingDataset { if let Some(limit) = max_candidate_files { op = op.with_max_candidate_files(limit); } - Ok(RT.block_on(op.explain())?) + Ok(block_on(op.explain())?) } pub fn close(&self) {} @@ -552,7 +583,7 @@ pub extern "system" fn Java_org_lance_Dataset_nativeMigrateManifestPathsV2( fn inner_native_migrate_manifest_paths_v2(env: &mut JNIEnv, java_dataset: JObject) -> Result<()> { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.migrate_manifest_paths_v2())?; + block_on(dataset_guard.inner.migrate_manifest_paths_v2())?; Ok(()) } @@ -786,6 +817,60 @@ impl IntoJava for Version { } } +impl IntoJava for WriterVersion { + fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { + let library = env.new_string(self.library)?; + let version = env.new_string(self.version)?; + let prerelease = match self.prerelease { + Some(value) => JObject::from(env.new_string(value)?), + None => JObject::null(), + }; + let build_metadata = match self.build_metadata { + Some(value) => JObject::from(env.new_string(value)?), + None => JObject::null(), + }; + + Ok(env.new_object( + "org/lance/WriterVersion", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V", + &[ + JValue::Object(&library), + JValue::Object(&version), + JValue::Object(&prerelease), + JValue::Object(&build_metadata), + ], + )?) + } +} + +impl IntoJava for ManifestLocation { + fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { + let size = self.size.ok_or_else(|| { + Error::runtime_error(format!("Manifest size is unavailable for {}", self.path)) + })?; + let path = env.new_string(self.path.to_string())?; + let naming_scheme = env.new_string(match self.naming_scheme { + ManifestNamingScheme::V1 => "V1", + ManifestNamingScheme::V2 => "V2", + })?; + let e_tag = match self.e_tag { + Some(value) => JObject::from(env.new_string(value)?), + None => JObject::null(), + }; + Ok(env.new_object( + "org/lance/ManifestLocation", + "(JLjava/lang/String;JLjava/lang/String;Ljava/lang/String;)V", + &[ + JValue::Long(self.version as i64), + JValue::Object(&path), + JValue::Long(size as i64), + JValue::Object(&naming_scheme), + JValue::Object(&e_tag), + ], + )?) + } +} + fn attach_native_dataset<'local>( env: &mut JNIEnv<'local>, dataset: BlockingDataset, @@ -1062,9 +1147,9 @@ fn inner_create_index<'local>( } if skip_commit { - RT.block_on(index_builder.execute_uncommitted())? + block_on(index_builder.execute_uncommitted())? } else { - RT.block_on(index_builder.into_future())? + block_on(index_builder.into_future())? } }; @@ -1084,7 +1169,7 @@ fn inner_drop_index(env: &mut JNIEnv, java_dataset: JObject, name: JString) -> R let name = name.extract(env)?; let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.drop_index(&name))?; + block_on(dataset_guard.inner.drop_index(&name))?; Ok(()) } @@ -1115,6 +1200,92 @@ fn inner_merge_index_metadata( index_type_code_jobj: jint, batch_readhead_jobj: JObject, // Optional ) -> Result<()> { + let (index_uuid, index_type, batch_readhead) = parse_merge_index_metadata_args( + env, + index_uuid, + index_type_code_jobj, + batch_readhead_jobj, + )?; + + // Clone the inner Dataset out of the `get_rust_field` guard and drop the + // guard before the long-lived merge. Otherwise nested JNI callbacks that + // touch the same Dataset would deadlock on the native field mutex. + let inner_dataset = unsafe { + let dataset_guard = + env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET)?; + dataset_guard.inner.clone() + }; + + block_on(async { + inner_dataset + .merge_index_metadata(&index_uuid, index_type, batch_readhead, noop_progress()) + .await + })?; + Ok(()) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_innerMergeIndexMetadataWithProgress<'local>( + mut env: JNIEnv<'local>, + java_dataset: JObject, + index_uuid: JString, + index_type_code_jobj: jint, + batch_readhead_jobj: JObject, + progress_jobj: JObject, +) { + ok_or_throw_without_return!( + env, + inner_merge_index_metadata_with_progress( + &mut env, + java_dataset, + index_uuid, + index_type_code_jobj, + batch_readhead_jobj, + progress_jobj, + ) + ); +} + +fn inner_merge_index_metadata_with_progress( + env: &mut JNIEnv, + java_dataset: JObject, + index_uuid: JString, + index_type_code_jobj: jint, + batch_readhead_jobj: JObject, + progress_jobj: JObject, +) -> Result<()> { + let (index_uuid, index_type, batch_readhead) = parse_merge_index_metadata_args( + env, + index_uuid, + index_type_code_jobj, + batch_readhead_jobj, + )?; + let progress = Arc::new(JavaIndexBuildProgress::new(env, &progress_jobj)?); + + // Clone the inner Dataset out of the `get_rust_field` guard and drop the + // guard before the long-lived merge. Progress callbacks are allowed to + // re-enter Dataset JNI methods; holding the guard across those callbacks + // would deadlock on the native field mutex (see update.rs). + let inner_dataset = unsafe { + let dataset_guard = + env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET)?; + dataset_guard.inner.clone() + }; + + block_on(async { + inner_dataset + .merge_index_metadata(&index_uuid, index_type, batch_readhead, progress) + .await + })?; + Ok(()) +} + +fn parse_merge_index_metadata_args( + env: &mut JNIEnv, + index_uuid: JString, + index_type_code_jobj: jint, + batch_readhead_jobj: JObject, +) -> Result<(Uuid, IndexType, Option)> { let index_uuid_str = index_uuid.extract(env)?; let index_uuid = Uuid::parse_str(&index_uuid_str) .map_err(|e| Error::input_error(format!("Invalid UUID string for index_uuid: {e}")))?; @@ -1122,17 +1293,7 @@ fn inner_merge_index_metadata( let batch_readhead = env .get_int_opt(&batch_readhead_jobj)? .map(|val| val as usize); - - let dataset_guard = - unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - - RT.block_on(async { - dataset_guard - .inner - .merge_index_metadata(&index_uuid, index_type, batch_readhead, noop_progress()) - .await - })?; - Ok(()) + Ok((index_uuid, index_type, batch_readhead)) } #[unsafe(no_mangle)] @@ -1156,7 +1317,7 @@ fn inner_merge_existing_index_segments<'local>( let merged_segment = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.merge_existing_index_segments(segments))? + block_on(dataset_guard.inner.merge_existing_index_segments(segments))? }; (&merged_segment).into_java(env) } @@ -1200,37 +1361,19 @@ fn inner_commit_existing_index_segments<'local>( let committed = { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.commit_existing_index_segments( + block_on(dataset_guard.inner.commit_existing_index_segments( &index_name, &column, segments, ))?; - RT.block_on(dataset_guard.inner.load_indices_by_name(&index_name))? + block_on(dataset_guard.inner.load_indices_by_name(&index_name))? }; export_vec(env, &committed) } fn index_metadata_to_segment(metadata: &IndexMetadata) -> Result { - let fragment_bitmap = metadata.fragment_bitmap.clone().ok_or_else(|| { - Error::input_error(format!( - "Segment '{}' is missing fragment coverage metadata", - metadata.uuid - )) - })?; - let index_details = metadata.index_details.clone().ok_or_else(|| { - Error::input_error(format!( - "Segment '{}' is missing index details metadata", - metadata.uuid - )) - })?; - - Ok(IndexSegment::new( - metadata.uuid, - fragment_bitmap, - index_details, - metadata.index_version, - )) + Ok(metadata.clone().into_index_segment()?) } #[unsafe(no_mangle)] @@ -1277,7 +1420,7 @@ fn inner_optimize_indices( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.optimize_indices(&options))?; + block_on(dataset_guard.inner.optimize_indices(&options))?; Ok(()) } @@ -1321,6 +1464,44 @@ pub extern "system" fn Java_org_lance_Dataset_openNative<'local>( ) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_listManifestLocationsNative<'local>( + mut env: JNIEnv<'local>, + _obj: JObject, + path: JString, + storage_options_obj: JObject, +) -> JObject<'local> { + ok_or_throw!( + env, + inner_list_manifest_locations(&mut env, path, storage_options_obj) + ) +} + +fn inner_list_manifest_locations<'local>( + env: &mut JNIEnv<'local>, + path: JString, + storage_options_obj: JObject, +) -> Result> { + let path: String = path.extract(env)?; + let storage_options = JMap::from_env(env, &storage_options_obj)?; + let storage_options = to_rust_map(env, &storage_options)?; + let locations = BlockingDataset::list_manifest_locations(&path, storage_options)?; + let list = env.new_object("java/util/ArrayList", "()V", &[])?; + for location in locations { + env.with_local_frame(8, |env| { + let java_location = location.into_java(env)?; + env.call_method( + &list, + "add", + "(Ljava/lang/Object;)Z", + &[JValue::Object(&java_location)], + )?; + Ok::<(), Error>(()) + })?; + } + Ok(list) +} + #[allow(clippy::too_many_arguments)] fn inner_open_native<'local>( env: &mut JNIEnv<'local>, @@ -1466,6 +1647,70 @@ fn inner_get_fragments<'local>( export_vec(env, &fragments) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_nativeGetFragmentStatistics<'a>( + mut env: JNIEnv<'a>, + jdataset: JObject, +) -> JObject<'a> { + ok_or_throw!(env, inner_get_fragment_statistics(&mut env, jdataset)) +} + +/// Returns per-fragment statistics in their final Java primitive arrays. +/// +/// Row count semantics match Java `FragmentMetadata.getNumRows()`: +/// physical rows minus deleted rows, with absent values treated as 0. +/// Data file count is the number of data files in the fragment. +fn inner_get_fragment_statistics<'local>( + env: &mut JNIEnv<'local>, + jdataset: JObject, +) -> Result> { + let fragments = { + let dataset = + unsafe { env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET) }?; + dataset.inner.fragments().clone() + }; + let fragment_count = i32::try_from(fragments.len()).map_err(|_| { + Error::runtime_error(format!( + "Fragment statistics contain {} fragments, exceeding the Java array limit of {}", + fragments.len(), + i32::MAX + )) + })?; + let ids = env.new_int_array(fragment_count)?; + let row_counts = env.new_long_array(fragment_count)?; + let data_file_nums = env.new_int_array(fragment_count)?; + + let mut id_values = Vec::with_capacity(fragments.len()); + let mut row_count_values = Vec::with_capacity(fragments.len()); + let mut data_file_num_values = Vec::with_capacity(fragments.len()); + + for fragment in fragments.iter() { + let physical_rows = fragment.physical_rows.unwrap_or(0) as i64; + let deleted_rows = fragment + .deletion_file + .as_ref() + .and_then(|deletion_file| deletion_file.num_deleted_rows) + .unwrap_or(0) as i64; + id_values.push(fragment.id as i32); + row_count_values.push(physical_rows - deleted_rows); + data_file_num_values.push(fragment.files.len() as i32); + } + + env.set_int_array_region(&ids, 0, &id_values)?; + env.set_long_array_region(&row_counts, 0, &row_count_values)?; + env.set_int_array_region(&data_file_nums, 0, &data_file_num_values)?; + + Ok(env.new_object( + "org/lance/FragmentStatistics", + "([I[J[I)V", + &[ + JValue::Object(&ids), + JValue::Object(&row_counts), + JValue::Object(&data_file_nums), + ], + )?) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_Dataset_getFragmentNative<'a>( mut env: JNIEnv<'a>, @@ -1571,6 +1816,20 @@ pub extern "system" fn Java_org_lance_Dataset_nativeListVersions<'local>( ok_or_throw!(env, inner_list_versions(&mut env, java_dataset)) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_nativeGetVersionCount( + mut env: JNIEnv, + java_dataset: JObject, +) -> jlong { + ok_or_throw_with_return!(env, inner_get_version_count(&mut env, java_dataset), -1) as jlong +} + +fn inner_get_version_count(env: &mut JNIEnv, java_dataset: JObject) -> Result { + let dataset_guard = + unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; + dataset_guard.count_versions() +} + fn inner_list_versions<'local>( env: &mut JNIEnv<'local>, java_dataset: JObject, @@ -1793,7 +2052,7 @@ fn inner_shallow_clone<'local>( let new_ds = { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.shallow_clone( + block_on(dataset_guard.inner.shallow_clone( target_path_str.as_str(), reference, storage_opts, @@ -1951,6 +2210,29 @@ pub extern "system" fn Java_org_lance_Dataset_nativeHasStableRowIds( ok_or_throw_with_return!(env, inner_has_stable_row_ids(&mut env, java_dataset), 0u8) } +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Dataset_nativeGetWriterVersion<'local>( + mut env: JNIEnv<'local>, + java_dataset: JObject, +) -> JObject<'local> { + ok_or_throw!(env, inner_get_writer_version(&mut env, java_dataset)) +} + +fn inner_get_writer_version<'local>( + env: &mut JNIEnv<'local>, + java_dataset: JObject, +) -> Result> { + let writer_version = { + let dataset_guard = + unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; + dataset_guard.inner.manifest().writer_version.clone() + }; + match writer_version { + Some(writer_version) => writer_version.into_java(env), + None => Ok(JObject::null()), + } +} + fn inner_has_stable_row_ids(env: &mut JNIEnv, java_dataset: JObject) -> Result { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; @@ -1976,12 +2258,13 @@ fn inner_get_lance_file_format_version<'local>( let version_string = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - let version = dataset_guard + dataset_guard .inner .manifest() .data_storage_format - .lance_file_version()?; - version.to_string() + .lance_file_format() + .to_manifest_string() + .to_string() }; Ok(env @@ -2023,7 +2306,7 @@ fn inner_take( let projection = ProjectionRequest::from_columns(columns, dataset.schema()); - match RT.block_on(dataset.take(indices_slice, projection)) { + match block_on(dataset.take(indices_slice, projection)) { Ok(res) => res, Err(e) => { return Err(e.into()); @@ -2075,7 +2358,7 @@ fn inner_take_rows( let projection = ProjectionRequest::from_columns(columns, dataset.schema()); - match RT.block_on(dataset.take_rows(&row_ids_u64, projection)) { + match block_on(dataset.take_rows(&row_ids_u64, projection)) { Ok(res) => res, Err(e) => { return Err(e.into()); @@ -2133,7 +2416,7 @@ fn inner_sample( .project_preserve_system_columns(&columns) .map_err(|e| Error::runtime_error(e.to_string()))?; - match RT.block_on(dataset.sample(n as usize, &projection, fragment_ids_u32.as_deref())) { + match block_on(dataset.sample(n as usize, &projection, fragment_ids_u32.as_deref())) { Ok(res) => res, Err(e) => { return Err(e.into()); @@ -2165,7 +2448,7 @@ fn inner_delete(env: &mut JNIEnv, java_dataset: JObject, predicate: JString) -> let predicate_str = predicate.extract(env)?; let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.delete(&predicate_str))?; + block_on(dataset_guard.inner.delete(&predicate_str))?; Ok(()) } @@ -2180,7 +2463,7 @@ pub extern "system" fn Java_org_lance_Dataset_nativeTruncateTable( fn inner_truncate_table(env: &mut JNIEnv, java_dataset: JObject) -> Result<()> { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.truncate_table())?; + block_on(dataset_guard.inner.truncate_table())?; Ok(()) } @@ -2205,7 +2488,7 @@ fn inner_drop_columns( let columns_slice: Vec<&str> = columns.iter().map(AsRef::as_ref).collect(); let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.drop_columns(&columns_slice))?; + block_on(dataset_guard.inner.drop_columns(&columns_slice))?; Ok(()) } @@ -2214,17 +2497,23 @@ pub extern "system" fn Java_org_lance_Dataset_nativeAlterColumns( mut env: JNIEnv, java_dataset: JObject, column_alterations_obj: JObject, // List + cast_schema_addr: jlong, ) { ok_or_throw_without_return!( env, - inner_alter_columns(&mut env, java_dataset, column_alterations_obj) + inner_alter_columns( + &mut env, + java_dataset, + column_alterations_obj, + cast_schema_addr + ) ) } fn create_column_alteration( env: &mut JNIEnv, column_alteration_jobj: JObject, // ColumnAlteration -) -> Result { +) -> Result<(ColumnAlteration, bool)> { let path_obj = env .get_field(&column_alteration_jobj, "path", "Ljava/lang/String;")? .l()?; @@ -2263,54 +2552,61 @@ fn create_column_alteration( None }; + // The cast target type (if any) is not read here: it is transferred separately through the + // Arrow C Data Interface (see inner_alter_columns), because ArrowType#toString() does not + // round-trip through DataType::from_str for parameterized types. This flag records whether a + // cast was requested so the caller can attach the imported type in order. let data_type_obj = env .get_field(&column_alteration_jobj, "dataType", "Ljava/util/Optional;")? .l()?; - let data_type = if env + let wants_cast = env .call_method(&data_type_obj, "isPresent", "()Z", &[])? - .z()? - { - let j_data_type: JObject = env - .call_method(data_type_obj, "get", "()Ljava/lang/Object;", &[])? - .l()?; - let jstring: JString = env - .call_method(j_data_type, "toString", "()Ljava/lang/String;", &[])? - .l()? - .into(); - let data_type_str: String = env.get_string(&jstring)?.into(); // Intermediate variable - DataType::from_str(&data_type_str) - .map_err(|e| Error::input_error(e.to_string())) - .ok() - } else { - None - }; + .z()?; - Ok(ColumnAlteration { + let alteration = ColumnAlteration { path, rename, nullable, - data_type, - }) + data_type: None, + }; + Ok((alteration, wants_cast)) } fn inner_alter_columns( env: &mut JNIEnv, java_dataset: JObject, column_alterations_obj: JObject, // List + cast_schema_addr: jlong, ) -> Result<()> { let list = env.get_list(&column_alterations_obj)?; let mut iter = list.iter(env)?; let mut column_alterations = Vec::new(); + let mut cast_flags = Vec::new(); while let Some(elem) = iter.next(env)? { - let alteration = create_column_alteration(env, elem)?; + let (alteration, wants_cast) = create_column_alteration(env, elem)?; column_alterations.push(alteration); + cast_flags.push(wants_cast); + } + + // Cast target types arrive as one Arrow schema field per requested cast, in the same order + // as the alterations that requested one. + let cast_schema = unsafe { FFI_ArrowSchema::from_raw(cast_schema_addr as *mut _) }; + let cast_schema = ArrowSchema::try_from(&cast_schema) + .map_err(|_| Error::input_error("ArrowSchema conversion error".to_string()))?; + let mut cast_types = cast_schema.fields.iter().map(|f| f.data_type().clone()); + for (alteration, wants_cast) in column_alterations.iter_mut().zip(cast_flags) { + if wants_cast { + alteration.data_type = Some(cast_types.next().ok_or_else(|| { + Error::input_error("Missing cast type for column alteration".to_string()) + })?); + } } let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.alter_columns(&column_alterations))?; + block_on(dataset_guard.inner.alter_columns(&column_alterations))?; Ok(()) } @@ -2368,7 +2664,7 @@ fn inner_add_columns_by_sql_expressions( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on( + block_on( dataset_guard .inner .add_columns(rust_transform, None, batch_size), @@ -2413,7 +2709,7 @@ fn inner_add_columns_by_reader( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.add_columns(transform, None, batch_size))?; + block_on(dataset_guard.inner.add_columns(transform, None, batch_size))?; Ok(()) } @@ -2444,7 +2740,7 @@ fn inner_add_columns_by_schema( let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.add_columns(transform, None, None))?; + block_on(dataset_guard.inner.add_columns(transform, None, None))?; Ok(()) } @@ -2744,7 +3040,7 @@ fn inner_create_branch<'local>( let new_blocking_dataset = { let mut dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - let inner = RT.block_on(dataset_guard.inner.create_branch( + let inner = block_on(dataset_guard.inner.create_branch( branch_name.as_str(), reference, storage_opts, @@ -3046,6 +3342,30 @@ fn convert_java_compaction_options_to_rust( &[], )? .l()?; + let max_source_rows = env + .call_method( + &java_options, + "getMaxSourceRows", + "()Ljava/util/Optional;", + &[], + )? + .l()?; + let max_source_bytes = env + .call_method( + &java_options, + "getMaxSourceBytes", + "()Ljava/util/Optional;", + &[], + )? + .l()?; + let excluded_fragment_ids = env + .call_method( + &java_options, + "getExcludedFragmentIds", + "()Ljava/util/List;", + &[], + )? + .l()?; build_compaction_options( env, @@ -3060,6 +3380,9 @@ fn convert_java_compaction_options_to_rust( &compaction_mode, &binary_copy_read_batch_bytes, &max_source_fragments, + &max_source_rows, + &max_source_bytes, + &excluded_fragment_ids, config, ) } @@ -3141,6 +3464,14 @@ fn extract_cleanup_policy(env: &mut JNIEnv<'_>, jpolicy: &JObject) -> Result, jpolicy: &JObject) -> Result( let stats_json = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.index_statistics(&index_name))? + block_on(dataset_guard.inner.index_statistics(&index_name))? }; let jstats = env.new_string(stats_json)?; Ok(jstats) @@ -3402,6 +3734,7 @@ fn inner_describe_indices<'local>( for_column: for_column.as_deref(), has_name: has_name.as_deref(), must_support_fts, + fts_document_granularity: None, must_support_exact_equality, }) })?; @@ -3409,7 +3742,7 @@ fn inner_describe_indices<'local>( let descriptions = { let dataset_guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(dataset_guard.inner.describe_indices(index_criteria))? + block_on(dataset_guard.inner.describe_indices(index_criteria))? }; export_vec(env, &descriptions) @@ -3467,7 +3800,7 @@ fn inner_count_indexed_rows( // This ensures we only count rows in the specified fragments let inner = dataset_guard.inner.clone(); - RT.block_on(async { + block_on(async { let mut scanner = inner.scan(); // Apply filter @@ -3572,13 +3905,14 @@ fn inner_get_zonemap_stats<'local>( })?; // Do all async work in a single block_on call to avoid nested runtime issues - RT.block_on(async { + block_on(async { // Find the zonemap index for this column using describe_indices let descriptions = dataset .describe_indices(Some(lance_index::IndexCriteria { for_column: Some(&column_name), has_name: None, must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, })) .await diff --git a/java/lance-jni/src/blocking_scanner.rs b/java/lance-jni/src/blocking_scanner.rs index 335cb2a4fa3..fb0cce7c3e0 100644 --- a/java/lance-jni/src/blocking_scanner.rs +++ b/java/lance-jni/src/blocking_scanner.rs @@ -15,19 +15,22 @@ use jni::sys::{JNI_TRUE, jboolean, jint}; use jni::{JNIEnv, sys::jlong}; use lance::dataset::scanner::{ AggregateExpr, ColumnOrdering, DatasetRecordBatchStream, ExecutionStatsCallback, - ExecutionSummaryCounts, Scanner, + ExecutionSummaryCounts, MaterializationStyle, Scanner, }; use lance_index::scalar::FullTextSearchQuery; -use lance_index::scalar::inverted::query::{ - BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery, - MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur, - PhraseQuery as FtsPhraseQuery, +use lance_index::scalar::inverted::{ + DocumentGranularity, + query::{ + BooleanQuery as FtsBooleanQuery, BoostQuery as FtsBoostQuery, FtsQuery, + MatchQuery as FtsMatchQuery, MultiMatchQuery as FtsMultiMatchQuery, Occur as FtsOccur, + PhraseQuery as FtsPhraseQuery, + }, }; use lance_io::ffi::to_ffi_arrow_array_stream; use lance_linalg::distance::DistanceType; use crate::{ - RT, + RT, block_on, blocking_dataset::{BlockingDataset, NATIVE_DATASET}, traits::IntoJava, utils::parse_approx_mode, @@ -66,17 +69,17 @@ impl BlockingScanner { pub fn open_stream(&self) -> Result { self.reset_stats(); - let res = RT.block_on(self.inner.try_into_stream())?; + let res = block_on(self.inner.try_into_stream())?; Ok(res) } pub fn schema(&self) -> Result { - let res = RT.block_on(self.inner.schema())?; + let res = block_on(self.inner.schema())?; Ok(res) } pub fn count_rows(&self) -> Result { - let res = RT.block_on(self.inner.count_rows())?; + let res = block_on(self.inner.count_rows())?; Ok(res) } @@ -114,6 +117,7 @@ pub(crate) fn build_full_text_search_query<'a>( let max_expansions = env.get_int_as_usize_from_method(&java_obj, "getMaxExpansions")?; let operator = env.get_fts_operator_from_method(&java_obj)?; let prefix_length = env.get_u32_from_method(&java_obj, "getPrefixLength")?; + let document_granularity = get_document_granularity(env, &java_obj)?; let mut query = FtsMatchQuery::new(query_text); query = query.with_column(Some(column)); @@ -123,6 +127,9 @@ pub(crate) fn build_full_text_search_query<'a>( .with_max_expansions(max_expansions) .with_operator(operator) .with_prefix_length(prefix_length); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } Ok(FtsQuery::Match(query)) } @@ -130,10 +137,14 @@ pub(crate) fn build_full_text_search_query<'a>( let query_text = env.get_string_from_method(&java_obj, "getQueryText")?; let column = env.get_string_from_method(&java_obj, "getColumn")?; let slop = env.get_u32_from_method(&java_obj, "getSlop")?; + let document_granularity = get_document_granularity(env, &java_obj)?; let mut query = FtsPhraseQuery::new(query_text); query = query.with_column(Some(column)); query = query.with_slop(slop); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } Ok(FtsQuery::Phrase(query)) } @@ -229,6 +240,20 @@ pub(crate) fn build_full_text_search_query<'a>( } } +fn get_document_granularity( + env: &mut JNIEnv<'_>, + java_obj: &JObject, +) -> Result> { + env.get_optional_from_method( + java_obj, + "getDocumentGranularity", + |env, granularity_obj| { + let value = env.get_string_from_method(&granularity_obj, "toRustString")?; + DocumentGranularity::try_from(value.as_str()).map_err(Error::from) + }, + ) +} + /// Scanner options passed from JNI - shared between blocking and async scanners pub(crate) struct ScannerOptions<'a> { pub fragment_ids_obj: JObject<'a>, @@ -236,6 +261,8 @@ pub(crate) struct ScannerOptions<'a> { pub substrait_filter_obj: JObject<'a>, pub filter_obj: JObject<'a>, pub batch_size_obj: JObject<'a>, + pub batch_size_bytes_obj: JObject<'a>, + pub io_buffer_size_obj: JObject<'a>, pub limit_obj: JObject<'a>, pub offset_obj: JObject<'a>, pub query_obj: JObject<'a>, @@ -244,6 +271,9 @@ pub(crate) struct ScannerOptions<'a> { pub with_row_id: jboolean, pub with_row_address: jboolean, pub batch_readahead: jint, + pub fragment_readahead_obj: JObject<'a>, + pub scan_in_order: jboolean, + pub late_materialization_obj: JObject<'a>, pub column_orderings: JObject<'a>, pub use_scalar_index: jboolean, pub fast_search: jboolean, @@ -283,7 +313,7 @@ pub(crate) fn build_scanner_with_options<'a>( let substrait_opt = env.get_bytes_opt(&options.substrait_filter_obj)?; if let Some(substrait) = substrait_opt { - RT.block_on(async { scanner.filter_substrait(substrait) })?; + block_on(async { scanner.filter_substrait(substrait) })?; } let filter_opt = env.get_string_opt(&options.filter_obj)?; @@ -296,6 +326,26 @@ pub(crate) fn build_scanner_with_options<'a>( scanner.batch_size(batch_size as usize); } + let batch_size_bytes_opt = env.get_long_opt(&options.batch_size_bytes_obj)?; + if let Some(batch_size_bytes) = batch_size_bytes_opt { + let batch_size_bytes = u64::try_from(batch_size_bytes).map_err(|_| { + Error::input_error(format!( + "batchSizeBytes must be non-negative, got {batch_size_bytes}" + )) + })?; + scanner.batch_size_bytes(batch_size_bytes); + } + + let io_buffer_size_opt = env.get_long_opt(&options.io_buffer_size_obj)?; + if let Some(io_buffer_size) = io_buffer_size_opt { + let io_buffer_size = u64::try_from(io_buffer_size).map_err(|_| { + Error::input_error(format!( + "ioBufferSize must be non-negative, got {io_buffer_size}" + )) + })?; + scanner.io_buffer_size(io_buffer_size); + } + let limit_opt = env.get_long_opt(&options.limit_obj)?; let offset_opt = env.get_long_opt(&options.offset_obj)?; scanner @@ -377,6 +427,47 @@ pub(crate) fn build_scanner_with_options<'a>( scanner.batch_readahead(options.batch_readahead as usize); + let fragment_readahead_opt = env.get_int_opt(&options.fragment_readahead_obj)?; + if let Some(fragment_readahead) = fragment_readahead_opt { + let fragment_readahead = usize::try_from(fragment_readahead).map_err(|_| { + Error::input_error(format!( + "fragmentReadahead must be greater than 0, got {fragment_readahead}" + )) + })?; + if fragment_readahead == 0 { + return Err(Error::input_error( + "fragmentReadahead must be greater than 0, got 0".to_string(), + )); + } + scanner.fragment_readahead(fragment_readahead); + } + + scanner.scan_in_order(options.scan_in_order == JNI_TRUE); + + env.get_optional(&options.late_materialization_obj, |env, java_obj| { + let style_name = env.get_string_from_method(&java_obj, "toRustString")?; + let style = match style_name.as_str() { + "heuristic" => MaterializationStyle::Heuristic, + "all_late" => MaterializationStyle::AllLate, + "all_early" => MaterializationStyle::AllEarly, + "all_early_except" => { + let columns: Vec = + import_vec_from_method(env, &java_obj, "getColumns", |env, elem| { + let jstr = JString::from(elem); + Ok(env.get_string(&jstr)?.into()) + })?; + MaterializationStyle::all_early_except(&columns, dataset.schema())? + } + other => { + return Err(Error::input_error(format!( + "Unsupported materialization style: {other}" + ))); + } + }; + scanner.materialization_style(style); + Ok(()) + })?; + env.get_optional(&options.column_orderings, |env, java_obj| { let list = env.get_list(&java_obj)?; let mut iter = list.iter(env)?; @@ -427,6 +518,8 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( substrait_filter_obj: JObject<'local>, // Optional filter_obj: JObject<'local>, // Optional batch_size_obj: JObject<'local>, // Optional + batch_size_bytes_obj: JObject<'local>, // Optional + io_buffer_size_obj: JObject<'local>, // Optional limit_obj: JObject<'local>, // Optional offset_obj: JObject<'local>, // Optional query_obj: JObject<'local>, // Optional @@ -435,6 +528,9 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( with_row_id: jboolean, // boolean with_row_address: jboolean, // boolean batch_readahead: jint, // int + fragment_readahead_obj: JObject<'local>, // Optional + scan_in_order: jboolean, // boolean + late_materialization_obj: JObject<'local>, // Optional column_orderings: JObject<'local>, // Optional> use_scalar_index: jboolean, // boolean fast_search: jboolean, // boolean @@ -454,6 +550,8 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -462,6 +560,9 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_createScanner<'local>( with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -483,6 +584,8 @@ fn inner_create_scanner<'local>( substrait_filter_obj: JObject<'local>, filter_obj: JObject<'local>, batch_size_obj: JObject<'local>, + batch_size_bytes_obj: JObject<'local>, + io_buffer_size_obj: JObject<'local>, limit_obj: JObject<'local>, offset_obj: JObject<'local>, query_obj: JObject<'local>, @@ -491,6 +594,9 @@ fn inner_create_scanner<'local>( with_row_id: jboolean, with_row_address: jboolean, batch_readahead: jint, + fragment_readahead_obj: JObject<'local>, + scan_in_order: jboolean, + late_materialization_obj: JObject<'local>, column_orderings: JObject<'local>, use_scalar_index: jboolean, fast_search: jboolean, @@ -511,6 +617,8 @@ fn inner_create_scanner<'local>( substrait_filter_obj, filter_obj, batch_size_obj, + batch_size_bytes_obj, + io_buffer_size_obj, limit_obj, offset_obj, query_obj, @@ -519,6 +627,9 @@ fn inner_create_scanner<'local>( with_row_id, with_row_address, batch_readahead, + fragment_readahead_obj, + scan_in_order, + late_materialization_obj, column_orderings, use_scalar_index, fast_search, @@ -589,6 +700,38 @@ pub extern "system" fn Java_org_lance_ipc_LanceScanner_openStream( } fn inner_open_stream(env: &mut JNIEnv, j_scanner: JObject, stream_addr: jlong) -> Result<()> { + if stream_addr == 0 { + return Err(Error::input_error( + "ArrowArrayStream address must not be null".to_string(), + )); + } + + // Reject a stream that already holds a producer. We write the C struct in place below with + // `ptr::write_unaligned`, which does not run any destructor on the previous contents. If the + // caller passed a stream whose `release` callback is already set (e.g. it was populated by an + // earlier export and not yet released), overwriting it would drop that callback and leak the + // first producer's resources. A freshly-allocated `ArrowArrayStream` has a null `release`, per + // the Arrow C Data Interface, so requiring `release == None` is the contract for "empty". + // + // The struct is allocated by Arrow Java inside an ArrowBuf and is not guaranteed to be aligned + // (hence `write_unaligned` below), so we must not form a reference to it. We read only the + // `release` field through an unaligned read: `addr_of!` computes the field address without + // creating an intermediate, possibly-unaligned reference, and the field is an `Option` + // which is `Copy` with no destructor, so reading a copy of it leaves the caller's stream + // untouched. + let release_is_set = unsafe { + let stream_ptr = stream_addr as *const FFI_ArrowArrayStream; + let release = std::ptr::read_unaligned(std::ptr::addr_of!((*stream_ptr).release)); + release.is_some() + }; + if release_is_set { + return Err(Error::input_error( + "ArrowArrayStream is already populated; exporting into it would leak the existing \ + producer. Pass a freshly-allocated, empty stream." + .to_string(), + )); + } + let record_batch_stream = { let scanner_guard = unsafe { env.get_rust_field::<_, _, BlockingScanner>(j_scanner, NATIVE_SCANNER) }?; @@ -637,7 +780,7 @@ fn inner_count_rows(env: &mut JNIEnv, j_scanner: JObject) -> Result { } const SCAN_STATS_CLASS: &str = "org/lance/ipc/ScanStats"; -const SCAN_STATS_CONSTRUCTOR_SIG: &str = "(JJJJJJLjava/util/Map;Ljava/util/Map;)V"; +const SCAN_STATS_CONSTRUCTOR_SIG: &str = "(JJJJJJJJLjava/util/Map;Ljava/util/Map;)V"; fn export_usize_map<'a>(env: &mut JNIEnv<'a>, map: &HashMap) -> Result> { let hash_map = env.new_object("java/util/HashMap", "()V", &[])?; @@ -669,6 +812,8 @@ impl IntoJava for &ExecutionSummaryCounts { JValueGen::Long(self.indices_loaded as i64), JValueGen::Long(self.parts_loaded as i64), JValueGen::Long(self.index_comparisons as i64), + JValueGen::Long(self.index_cache_hits() as i64), + JValueGen::Long(self.index_cache_misses() as i64), JValueGen::Object(&all_counts), JValueGen::Object(&all_times), ], diff --git a/java/lance-jni/src/delta.rs b/java/lance-jni/src/delta.rs index d5a6b0f3a27..5cd5300787f 100755 --- a/java/lance-jni/src/delta.rs +++ b/java/lance-jni/src/delta.rs @@ -1,11 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use crate::RT; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::ffi::JNIEnvExt; use crate::transaction::convert_to_java_transaction; +use crate::{RT, block_on}; use arrow::ffi_stream::FFI_ArrowArrayStream; use jni::JNIEnv; use jni::objects::{JObject, JValue}; @@ -123,7 +123,7 @@ fn inner_list_transactions<'local>( let txs: Vec = { let delta_guard = unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; - RT.block_on(delta_guard.inner.list_transactions())? + block_on(delta_guard.inner.list_transactions())? }; let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; @@ -156,7 +156,7 @@ fn inner_get_inserted_rows<'local>( let delta_guard = unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; - let stream: DatasetRecordBatchStream = RT.block_on(delta_guard.inner.get_inserted_rows())?; + let stream: DatasetRecordBatchStream = block_on(delta_guard.inner.get_inserted_rows())?; let ffi_stream = to_ffi_arrow_array_stream(stream, RT.handle().clone())?; unsafe { std::ptr::write_unaligned(stream_addr as *mut FFI_ArrowArrayStream, ffi_stream) } @@ -180,7 +180,34 @@ fn inner_get_updated_rows<'local>( let delta_guard = unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; - let stream: DatasetRecordBatchStream = RT.block_on(delta_guard.inner.get_updated_rows())?; + let stream: DatasetRecordBatchStream = block_on(delta_guard.inner.get_updated_rows())?; + let ffi_stream = to_ffi_arrow_array_stream(stream, RT.handle().clone())?; + + unsafe { std::ptr::write_unaligned(stream_addr as *mut FFI_ArrowArrayStream, ffi_stream) } + Ok(()) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_delta_DatasetDelta_nativeGetDeletedRowIds<'local>( + mut env: JNIEnv<'local>, + j_delta: JObject<'local>, + stream_addr: jlong, +) { + ok_or_throw_without_return!( + env, + inner_get_deleted_row_ids(&mut env, j_delta, stream_addr) + ) +} + +fn inner_get_deleted_row_ids<'local>( + env: &mut JNIEnv, + j_delta: JObject<'local>, + stream_addr: jlong, +) -> Result<()> { + let delta_guard = + unsafe { env.get_rust_field::<_, _, BlockingDatasetDelta>(&j_delta, NATIVE_DELTA) }?; + + let stream: DatasetRecordBatchStream = block_on(delta_guard.inner.get_deleted_row_ids())?; let ffi_stream = to_ffi_arrow_array_stream(stream, RT.handle().clone())?; unsafe { std::ptr::write_unaligned(stream_addr as *mut FFI_ArrowArrayStream, ffi_stream) } diff --git a/java/lance-jni/src/error.rs b/java/lance-jni/src/error.rs index cdb922a3cef..0affdca8f98 100644 --- a/java/lance-jni/src/error.rs +++ b/java/lance-jni/src/error.rs @@ -181,29 +181,36 @@ impl std::fmt::Display for Error { impl From for Error { fn from(err: LanceError) -> Self { + let backtrace_suffix = err + .backtrace() + .map(|bt| format!("\n\nRust backtrace:\n{}", bt)) + .unwrap_or_default(); + let message = format!("{}{}", err, backtrace_suffix); + match &err { LanceError::DatasetNotFound { .. } | LanceError::DatasetAlreadyExists { .. } | LanceError::CommitConflict { .. } - | LanceError::InvalidInput { .. } => Self::input_error(err.to_string()), - LanceError::IO { .. } => Self::io_error(err.to_string()), - LanceError::Timeout { .. } => Self::timeout_error(err.to_string()), - LanceError::NotSupported { .. } => Self::unsupported_error(err.to_string()), - LanceError::NotFound { .. } => Self::io_error(err.to_string()), + | LanceError::InvalidInput { .. } => Self::input_error(message), + LanceError::IO { .. } => Self::io_error(message), + LanceError::Timeout { .. } => Self::timeout_error(message), + LanceError::NotSupported { .. } => Self::unsupported_error(message), + LanceError::NotFound { .. } => Self::io_error(message), LanceError::Namespace { source, .. } => { // Try to downcast to NamespaceError and get the error code if let Some(ns_err) = source.downcast_ref::() { - Self::namespace_error(ns_err.code().as_u32(), ns_err.to_string()) + let ns_message = format!("{}{}", ns_err, backtrace_suffix); + Self::namespace_error(ns_err.code().as_u32(), ns_message) } else { log::warn!( "Failed to downcast NamespaceError source, falling back to runtime error. \ This may indicate a version mismatch. Source type: {:?}", source ); - Self::runtime_error(err.to_string()) + Self::runtime_error(message) } } - _ => Self::runtime_error(err.to_string()), + _ => Self::runtime_error(message), } } } @@ -241,3 +248,104 @@ impl From for Error { Self::input_error(err.to_string()) } } + +#[cfg(test)] +mod tests { + use super::*; + + // Helper: extract the java_class from an Error via Display output + fn java_class(err: &Error) -> &JavaExceptionClass { + &err.java_class + } + + #[test] + fn test_invalid_input_maps_to_illegal_argument() { + let lance_err = LanceError::invalid_input("bad input"); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + assert!(jni_err.message.contains("bad input")); + } + + #[test] + fn test_dataset_not_found_maps_to_illegal_argument() { + let lance_err = LanceError::dataset_not_found("my_dataset", "not found".to_string().into()); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + assert!(jni_err.message.contains("my_dataset")); + } + + #[test] + fn test_dataset_already_exists_maps_to_illegal_argument() { + let lance_err = LanceError::dataset_already_exists("my_dataset"); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + assert!(jni_err.message.contains("my_dataset")); + } + + #[test] + fn test_commit_conflict_maps_to_illegal_argument() { + let lance_err = LanceError::commit_conflict_source(42, "conflict".to_string().into()); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::IllegalArgumentException + ); + } + + #[test] + fn test_io_maps_to_ioexception() { + let lance_err = LanceError::io("disk failure"); + let jni_err: Error = lance_err.into(); + assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException); + assert!(jni_err.message.contains("disk failure")); + } + + #[test] + fn test_not_supported_maps_to_unsupported() { + let lance_err = LanceError::not_supported("nope"); + let jni_err: Error = lance_err.into(); + assert_eq!( + *java_class(&jni_err), + JavaExceptionClass::UnsupportedOperationException + ); + assert!(jni_err.message.contains("nope")); + } + + #[test] + fn test_not_found_maps_to_ioexception() { + let lance_err = LanceError::not_found("missing_uri"); + let jni_err: Error = lance_err.into(); + assert_eq!(*java_class(&jni_err), JavaExceptionClass::IOException); + assert!(jni_err.message.contains("missing_uri")); + } + + #[test] + fn test_fallthrough_maps_to_runtime() { + let lance_err = LanceError::internal("internal oops"); + let jni_err: Error = lance_err.into(); + assert_eq!(*java_class(&jni_err), JavaExceptionClass::RuntimeException); + assert!(jni_err.message.contains("internal oops")); + } + + #[test] + fn test_no_backtrace_suffix_when_backtrace_is_none() { + // Without the backtrace feature enabled in lance-core default tests, + // backtrace() returns None, so no suffix should be appended. + let lance_err = LanceError::io("clean message"); + let jni_err: Error = lance_err.into(); + assert!( + !jni_err.message.contains("Rust backtrace:"), + "Expected no backtrace suffix, got: {}", + jni_err.message + ); + } +} diff --git a/java/lance-jni/src/file_reader.rs b/java/lance-jni/src/file_reader.rs index 3df9766d066..35d2ecd8f77 100644 --- a/java/lance-jni/src/file_reader.rs +++ b/java/lance-jni/src/file_reader.rs @@ -7,7 +7,7 @@ use std::sync::{Arc, Mutex}; use crate::utils::to_rust_map; use crate::{ - JNIEnvExt, RT, + JNIEnvExt, RT, block_on, error::{Error, Result}, traits::IntoJava, }; @@ -23,8 +23,8 @@ use lance::io::ObjectStore; use lance_core::cache::LanceCache; use lance_core::datatypes::{BlobHandling, OnMissing, Projection, Schema}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; -use lance_encoding::version::LanceFileVersion; use lance_file::reader::{FileReader, FileReaderOptions, ReaderProjection}; +use lance_file::versions as file_versions; use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry}; use lance_io::{ ReadBatchParams, @@ -53,16 +53,15 @@ impl BlockingFileReader { filter_expression: FilterExpression, ) -> Result> { let reader = self.inner.clone(); - Ok(RT - .block_on(RT.spawn_blocking(move || { - reader.read_stream_projected_blocking( - read_batch_params, - batch_size, - reader_projection, - filter_expression, - ) - })) - .unwrap()?) + Ok(block_on(RT.spawn_blocking(move || { + reader.read_stream_projected_blocking( + read_batch_params, + batch_size, + reader_projection, + filter_expression, + ) + })) + .unwrap()?) } pub fn schema(&self) -> Result { @@ -112,7 +111,7 @@ fn inner_open<'local>( let file_uri_str: String = env.get_string(&file_uri)?.into(); let jmap = JMap::from_env(env, &storage_options_obj)?; let storage_options = to_rust_map(env, &jmap)?; - let reader = RT.block_on(async move { + let reader = block_on(async move { let object_params = ObjectStoreParams { storage_options_accessor: Some(Arc::new( lance::io::StorageOptionsAccessor::with_static_options(storage_options), @@ -262,18 +261,17 @@ pub extern "system" fn Java_org_lance_file_LanceFileReader_readAllNative( let transformed_schema = projection.to_bare_schema(); - let field_id_to_column_index = base_schema - .fields_pre_order() - .filter(|field| { - file_version < LanceFileVersion::V2_1 - || field.is_leaf() - || field.is_packed_struct() + let (field_ids, column_indices) = + file_versions::data_file_columns(file_version, &base_schema); + let field_id_to_column_index = field_ids + .into_iter() + .zip(column_indices) + .filter_map(|(field_id, column_index)| { + (column_index >= 0).then_some((field_id as u32, column_index as u32)) }) - .enumerate() - .map(|(idx, field)| (field.id as u32, idx as u32)) .collect::>(); - Some(ReaderProjection::from_field_ids( + Some(file_versions::reader_projection_from_field_ids( file_version, &transformed_schema, &field_id_to_column_index, diff --git a/java/lance-jni/src/file_writer.rs b/java/lance-jni/src/file_writer.rs index 40b48bd686b..c8cffe8b642 100644 --- a/java/lance-jni/src/file_writer.rs +++ b/java/lance-jni/src/file_writer.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Mutex}; use crate::utils::to_rust_map; use crate::{ - JNIEnvExt, RT, + JNIEnvExt, block_on, error::{Error, Result}, traits::IntoJava, }; @@ -23,6 +23,7 @@ use jni::{ use lance::io::ObjectStore; use lance_file::{ version::LanceFileVersion, + versions as file_versions, writer::{FileWriter, FileWriterOptions}, }; use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry}; @@ -92,7 +93,7 @@ fn inner_open<'local>( let data_storage_version_opt = env.get_string_opt(&data_storage_version)?; let storage_options = to_rust_map(env, &jmap)?; - let writer = RT.block_on(async move { + let writer = block_on(async move { let object_params = ObjectStoreParams { storage_options_accessor: Some(Arc::new( lance::io::StorageOptionsAccessor::with_static_options(storage_options), @@ -108,15 +109,12 @@ fn inner_open<'local>( let obj_store = Arc::new(obj_store); let obj_writer = obj_store.create(&path).await?; - Result::Ok(FileWriter::new_lazy( - obj_writer, - FileWriterOptions { - format_version: data_storage_version_opt - .map(|v| v.parse::()) - .transpose()?, - ..Default::default() - }, - )) + let version = data_storage_version_opt + .map(|value| value.parse::()) + .transpose()? + .unwrap_or_default() + .resolve(); + file_versions::create_lazy_writer(version, obj_writer, FileWriterOptions::default()) })?; let writer = BlockingFileWriter::create(writer); @@ -141,7 +139,7 @@ pub extern "system" fn Java_org_lance_file_LanceFileWriter_closeNative<'local>( } }; if let Some(writer) = writer { - match RT.block_on(writer.inner.lock().unwrap().finish()) { + match block_on(writer.inner.lock().unwrap().finish()) { Ok(_) => {} Err(e) => { Error::from(e).throw(&mut env); @@ -213,6 +211,6 @@ fn inner_write_batch( let writer = unsafe { env.get_rust_field::<_, _, BlockingFileWriter>(writer, NATIVE_WRITER) }?; let mut writer = writer.inner.lock().unwrap(); - RT.block_on(writer.write_batch(&record_batch))?; + block_on(writer.write_batch(&record_batch))?; Ok(()) } diff --git a/java/lance-jni/src/fragment.rs b/java/lance-jni/src/fragment.rs index d6603925947..69cbf04a013 100644 --- a/java/lance-jni/src/fragment.rs +++ b/java/lance-jni/src/fragment.rs @@ -5,7 +5,7 @@ use arrow::array::{RecordBatch, RecordBatchIterator, StructArray}; use arrow::ffi::{FFI_ArrowArray, FFI_ArrowSchema, from_ffi_and_data_type}; use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use arrow_schema::{DataType, Schema as ArrowSchema}; -use jni::objects::{JIntArray, JValue, JValueGen}; +use jni::objects::{JByteArray, JIntArray, JValue, JValueGen}; use jni::{ JNIEnv, objects::{JClass, JLongArray, JObject, JString}, @@ -19,6 +19,8 @@ use lance_io::utils::CachedFileSize; use lance_table::rowids::{RowIdSequence, write_row_ids}; use std::iter::once; +use roaring::RoaringBitmap; + use lance::dataset::fragment::write::FragmentCreateBuilder; use lance::io::ObjectStoreParams; use lance_datafusion::utils::StreamingWriteSource; @@ -29,10 +31,11 @@ use std::sync::Arc; use crate::blocking_dataset::extract_namespace_info; use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; +use crate::session::session_from_handle; use crate::traits::{FromJObjectWithEnv, IntoJava, JLance, export_vec, import_vec}; use crate::utils::extract_storage_options; use crate::{ - RT, + block_on, blocking_dataset::{BlockingDataset, NATIVE_DATASET}, traits::FromJString, utils::extract_write_params, @@ -48,8 +51,8 @@ pub(crate) struct FragmentMergeResult { pub(crate) struct FragmentUpdateResult { updated_fragment: Fragment, fields_modified: Vec, - /// Physical row offsets that received column updates (from `_rowaddr` low bits). - updated_row_offsets: Vec, + /// Matched row offsets serialized as portable RoaringBitmap bytes. + updated_row_offset_bytes: Vec, } ////////////////// @@ -80,7 +83,7 @@ fn inner_count_rows_native( "Fragment not found: {fragment_id}" ))); }; - let res = RT.block_on(fragment.count_rows(None))?; + let res = block_on(fragment.count_rows(None))?; Ok(res) } @@ -109,6 +112,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> JObject<'local> { ok_or_throw_with_return!( env, @@ -132,6 +136,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiArray<'local>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, ), JObject::default() ) @@ -158,6 +163,7 @@ fn inner_create_with_ffi_array<'local>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> Result> { let c_array_ptr = arrow_array_addr as *mut FFI_ArrowArray; let c_schema_ptr = arrow_schema_addr as *mut FFI_ArrowSchema; @@ -190,6 +196,7 @@ fn inner_create_with_ffi_array<'local>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, reader, ) } @@ -215,6 +222,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> JObject<'a> { ok_or_throw_with_return!( env, @@ -237,6 +245,7 @@ pub extern "system" fn Java_org_lance_Fragment_createWithFfiStream<'a>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, ), JObject::null() ) @@ -262,6 +271,7 @@ fn inner_create_with_ffi_stream<'local>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session ) -> Result> { let stream_ptr = arrow_array_stream_addr as *mut FFI_ArrowArrayStream; let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; @@ -284,6 +294,7 @@ fn inner_create_with_ffi_stream<'local>( allow_external_blob_outside_bases, blob_pack_file_size_threshold, schema_addr, + session_handle, reader, ) } @@ -307,6 +318,7 @@ fn create_fragment<'a>( allow_external_blob_outside_bases: JObject, // Optional blob_pack_file_size_threshold: JObject, // Optional schema_addr: jlong, + session_handle: jlong, // Session handle, 0 means no session source: impl StreamingWriteSource, ) -> Result> { let path_str = dataset_uri.extract(env)?; @@ -328,6 +340,8 @@ fn create_fragment<'a>( &blob_pack_file_size_threshold, )?; + write_params.session = session_from_handle(session_handle); + // Set up storage options provider if namespace is provided let namespace_info = extract_namespace_info(env, &namespace_obj, &table_id_obj)?; if let Some((namespace, table_id)) = namespace_info { @@ -366,7 +380,7 @@ fn create_fragment<'a>( builder = builder.schema(&schema); } - let fragments = RT.block_on(builder.write_fragments(source))?; + let fragments = block_on(builder.write_fragments(source))?; export_vec(env, &fragments) } @@ -408,7 +422,7 @@ fn inner_delete_rows<'local>( .map(|x| x as u32) .collect(); - let res = RT.block_on(async move { fragment.extend_deletions(indexes).await }); + let res = block_on(async move { fragment.extend_deletions(indexes).await }); let obj = match res { Ok(Some(f)) => f.metadata().into_java(env)?, @@ -480,7 +494,7 @@ fn inner_merge_column<'local>( let right_on_str: String = right_on.extract(env)?; let (new_frag, new_schema) = - RT.block_on(fragment.merge_columns(reader, &left_on_str, &right_on_str, max_field_id))?; + block_on(fragment.merge_columns(reader, &left_on_str, &right_on_str, max_field_id))?; let result = FragmentMergeResult { fragment: new_frag, schema: new_schema, @@ -537,17 +551,112 @@ fn inner_update_column<'local>( let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; let left_on_str: String = left_on.extract(env)?; let right_on_str: String = right_on.extract(env)?; - let r = - RT.block_on(fragment.update_columns_with_offsets(reader, &left_on_str, &right_on_str))?; - let updated_row_offsets: Vec = r.matched_offsets.iter().map(|o| o as i64).collect(); + let r = block_on(fragment.update_columns_with_offsets(reader, &left_on_str, &right_on_str))?; + let updated_row_offset_bytes = serialize_matched_offsets(&r.matched_offsets)?; let result = FragmentUpdateResult { updated_fragment: r.fragment, fields_modified: r.fields_modified, - updated_row_offsets, + updated_row_offset_bytes, }; result.into_java(env) } +fn serialize_matched_offsets(bitmap: &RoaringBitmap) -> Result> { + let mut buf = Vec::new(); + bitmap.serialize_into(&mut buf).map_err(|e| { + Error::runtime_error(format!( + "failed to serialize matched row offsets RoaringBitmap: {e}" + )) + })?; + Ok(buf) +} + +fn deserialize_row_offset_bytes(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Ok(RoaringBitmap::new()); + } + RoaringBitmap::deserialize_from(bytes).map_err(|e| { + Error::input_error(format!( + "invalid updatedRowOffsetBytes RoaringBitmap bytes: {e}" + )) + }) +} + +fn expand_row_offset_bytes_to_i64(bitmap: &RoaringBitmap) -> Vec { + bitmap.iter().map(|o| o as i64).collect() +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_fragment_FragmentUpdateResult_expandRowOffsetsFromBytes< + 'local, +>( + mut env: JNIEnv<'local>, + _cls: JClass, + jbytes: JByteArray, +) -> JLongArray<'local> { + ok_or_throw_with_return!( + env, + inner_expand_updated_row_offset_bytes(&mut env, jbytes), + unsafe { JLongArray::from_raw(std::ptr::null_mut()) } + ) +} + +fn inner_expand_updated_row_offset_bytes<'local>( + env: &mut JNIEnv<'local>, + jbytes: JByteArray, +) -> Result> { + let buf = env.convert_byte_array(&jbytes)?; + let bitmap = deserialize_row_offset_bytes(&buf)?; + let offsets = expand_row_offset_bytes_to_i64(&bitmap); + let arr = env.new_long_array(offsets.len() as i32)?; + if !offsets.is_empty() { + env.set_long_array_region(&arr, 0, &offsets)?; + } + Ok(arr) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_fragment_FragmentUpdateResult_encodeRowOffsetsToBytes< + 'local, +>( + mut env: JNIEnv<'local>, + _cls: JClass, + joffsets: JLongArray, +) -> JByteArray<'local> { + ok_or_throw_with_return!( + env, + inner_encode_updated_row_offset_bytes(&mut env, joffsets), + unsafe { JByteArray::from_raw(std::ptr::null_mut()) } + ) +} + +fn inner_encode_updated_row_offset_bytes<'local>( + env: &mut JNIEnv<'local>, + joffsets: JLongArray, +) -> Result> { + let len = env.get_array_length(&joffsets)?; + let mut buf: Vec = vec![0; len as usize]; + if len > 0 { + env.get_long_array_region(&joffsets, 0, buf.as_mut_slice())?; + } + let mut bitmap = RoaringBitmap::new(); + for offset in buf { + if offset < 0 { + return Err(Error::input_error(format!( + "updatedRowOffsets must be non-negative, got {offset}" + ))); + } + if offset > u32::MAX as i64 { + return Err(Error::input_error(format!( + "updatedRowOffsets value {offset} exceeds u32::MAX" + ))); + } + bitmap.insert(offset as u32); + } + let bytes = serialize_matched_offsets(&bitmap)?; + Ok(env.byte_array_from_slice(&bytes)?) +} + #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_fragment_RowIdMeta_nativeEncodeRowIds( mut env: JNIEnv, @@ -569,7 +678,7 @@ fn inner_encode_row_ids(env: &mut JNIEnv, row_ids: &JLongArray) -> Result = buf.into_iter().map(|x| x as u64).collect(); let seq = RowIdSequence::from(ids.as_slice()); - let meta = RowIdMeta::Inline(write_row_ids(&seq)); + let meta = RowIdMeta::Inline(write_row_ids(&seq).into()); let json = serde_json::to_string(&meta)?; Ok(json) } @@ -591,7 +700,7 @@ const FRAGMENT_MERGE_RESULT_CLASS: &str = "org/lance/fragment/FragmentMergeResul const FRAGMENT_MERGE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/FragmentMetadata;Lorg/lance/schema/LanceSchema;)V"; const FRAGMENT_UPDATE_RESULT_CLASS: &str = "org/lance/fragment/FragmentUpdateResult"; -const FRAGMENT_UPDATE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/FragmentMetadata;[J[J)V"; +const FRAGMENT_UPDATE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/FragmentMetadata;[J[B)V"; impl IntoJava for &FragmentMergeResult { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { @@ -612,14 +721,15 @@ impl IntoJava for &FragmentUpdateResult { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { let java_updated_fragment = self.updated_fragment.into_java(env)?; let java_fields_modified = JLance(self.fields_modified.clone()).into_java(env)?; - let java_updated_row_offsets = JLance(self.updated_row_offsets.clone()).into_java(env)?; + let java_updated_row_offset_bytes = + env.byte_array_from_slice(&self.updated_row_offset_bytes)?; Ok(env.new_object( FRAGMENT_UPDATE_RESULT_CLASS, FRAGMENT_UPDATE_RESULT_CONSTRUCTOR_SIG, &[ JValueGen::Object(&java_updated_fragment), JValueGen::Object(&java_fields_modified), - JValueGen::Object(&java_updated_row_offsets), + JValueGen::Object(&java_updated_row_offset_bytes), ], )?) } @@ -828,6 +938,9 @@ impl FromJObjectWithEnv for JObject<'_> { row_id_meta, created_at_version_meta, last_updated_at_version_meta, + // Overlays are not exposed to Java yet, and the reverse conversion + // does not export them, so this round-trip is overlay-free. + overlays: vec![], }) } } diff --git a/java/lance-jni/src/index.rs b/java/lance-jni/src/index.rs index 6cb64a05a81..479a69a144d 100644 --- a/java/lance-jni/src/index.rs +++ b/java/lance-jni/src/index.rs @@ -12,22 +12,27 @@ use prost::Message; use prost_types::Any; use std::sync::Arc; +/// Build a `java.util.List`. +/// +/// Not `JLance>`'s `IntoJava`: that produces a primitive `int[]`, while +/// the Java constructors here take `List`. +fn int_list<'a>(env: &mut JNIEnv<'a>, ids: impl IntoIterator) -> Result> { + let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; + for id in ids { + let id_obj = env.new_object("java/lang/Integer", "(I)V", &[JValue::Int(id)])?; + env.call_method( + &array_list, + "add", + "(Ljava/lang/Object;)Z", + &[JValue::Object(&id_obj)], + )?; + } + Ok(array_list) +} + impl IntoJava for &Arc { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { - let field_ids_list = { - let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; - for id in self.field_ids() { - let int_obj = - env.new_object("java/lang/Integer", "(I)V", &[JValue::Int(*id as i32)])?; - env.call_method( - &array_list, - "add", - "(Ljava/lang/Object;)Z", - &[JValue::Object(&int_obj)], - )?; - } - array_list - }; + let field_ids_list = int_list(env, self.field_ids().iter().map(|id| *id as i32))?; let name = env.new_string(self.name())?; let type_url = env.new_string(self.type_url())?; let index_type = env.new_string(self.index_type())?; @@ -35,10 +40,15 @@ impl IntoJava for &Arc { let metadata_list = export_vec(env, self.metadata())?; let details_json = self.details()?; let details = env.new_string(details_json)?; + let total_size_bytes = if let Some(size) = self.total_size_bytes() { + env.new_object("java/lang/Long", "(J)V", &[JValue::Long(size as i64)])? + } else { + JObject::null() + }; let j_index_desc = env.new_object( "org/lance/index/IndexDescription", - "(Ljava/lang/String;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;)V", + "(Ljava/lang/String;Ljava/util/List;Ljava/lang/String;Ljava/lang/String;JLjava/util/List;Ljava/lang/String;Ljava/lang/Long;)V", &[ JValue::Object(&name), JValue::Object(&field_ids_list), @@ -47,6 +57,7 @@ impl IntoJava for &Arc { JValue::Long(rows_indexed), JValue::Object(&metadata_list), JValue::Object(&details), + JValue::Object(&total_size_bytes), ], )?; Ok(j_index_desc) @@ -57,37 +68,13 @@ impl IntoJava for &IndexMetadata { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { let uuid = self.uuid.into_java(env)?; - let fields = { - let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; - for field in &self.fields { - let field_obj = - env.new_object("java/lang/Integer", "(I)V", &[JValue::Int(*field)])?; - env.call_method( - &array_list, - "add", - "(Ljava/lang/Object;)Z", - &[JValue::Object(&field_obj)], - )?; - } - array_list - }; + let fields = int_list(env, self.fields.iter().copied())?; + let covering_fields = int_list(env, self.covering_fields.iter().copied())?; let name = env.new_string(&self.name)?; - let fragments = if let Some(bitmap) = &self.fragment_bitmap { - let array_list = env.new_object("java/util/ArrayList", "()V", &[])?; - for frag_id in bitmap.iter() { - let id_obj = - env.new_object("java/lang/Integer", "(I)V", &[JValue::Int(frag_id as i32)])?; - env.call_method( - &array_list, - "add", - "(Ljava/lang/Object;)Z", - &[JValue::Object(&id_obj)], - )?; - } - array_list - } else { - JObject::null() + let fragments = match &self.fragment_bitmap { + Some(bitmap) => int_list(env, bitmap.iter().map(|id| id as i32))?, + None => JObject::null(), }; // Convert index_details to byte array @@ -125,16 +112,23 @@ impl IntoJava for &IndexMetadata { JObject::null() }; + let size_bytes = if let Some(size) = self.total_size_bytes() { + env.new_object("java/lang/Long", "(J)V", &[JValue::Long(size as i64)])? + } else { + JObject::null() + }; + // Determine index type from index_details type_url let index_type = determine_index_type(env, &self.index_details)?; // Create Index object Ok(env.new_object( "org/lance/index/Index", - "(Ljava/util/UUID;Ljava/util/List;Ljava/lang/String;JLjava/util/List;[BILjava/time/Instant;Ljava/lang/Integer;Lorg/lance/index/IndexType;)V", + "(Ljava/util/UUID;Ljava/util/List;Ljava/util/List;Ljava/lang/String;JLjava/util/List;[BILjava/time/Instant;Ljava/lang/Integer;Ljava/lang/Long;Lorg/lance/index/IndexType;)V", &[ JValue::Object(&uuid), JValue::Object(&fields), + JValue::Object(&covering_fields), JValue::Object(&name), JValue::Long(self.dataset_version as i64), JValue::Object(&fragments), @@ -142,6 +136,7 @@ impl IntoJava for &IndexMetadata { JValue::Int(self.index_version), JValue::Object(&created_at), JValue::Object(&base_id), + JValue::Object(&size_bytes), JValue::Object(&index_type), ], )?) diff --git a/java/lance-jni/src/index_progress.rs b/java/lance-jni/src/index_progress.rs new file mode 100644 index 00000000000..a9275668db1 --- /dev/null +++ b/java/lance-jni/src/index_progress.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use async_trait::async_trait; +use jni::objects::{GlobalRef, JObject, JString, JValue}; +use jni::{JNIEnv, JavaVM}; +use lance_index::progress::IndexBuildProgress; + +use crate::error::{Error, Result}; + +/// Bridges Rust index progress events to a Java callback. +pub(crate) struct JavaIndexBuildProgress { + callback: GlobalRef, + jvm: Arc, +} + +impl std::fmt::Debug for JavaIndexBuildProgress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("JavaIndexBuildProgress") + } +} + +impl JavaIndexBuildProgress { + pub(crate) fn new(env: &mut JNIEnv, callback: &JObject) -> Result { + if callback.is_null() { + return Err(Error::input_error( + "IndexBuildProgress callback cannot be null".to_string(), + )); + } + Ok(Self { + callback: env.new_global_ref(callback)?, + jvm: Arc::new(env.get_java_vm()?), + }) + } + + fn call_stage_start( + &self, + stage: &str, + total: Option, + unit: &str, + ) -> lance_core::Result<()> { + let total = total + .map(|value| { + i64::try_from(value).map_err(|_| { + lance_core::Error::invalid_input(format!( + "IndexBuildProgress total exceeds Java long for stage '{stage}': {value}" + )) + }) + }) + .transpose()?; + let mut env = self.jvm.attach_current_thread().map_err(|error| { + lance_core::Error::internal(format!( + "IndexBuildProgress.stageStart failed to attach JVM thread for stage '{stage}': {error}" + )) + })?; + + env.with_local_frame(16, |env| { + let stage_obj = env + .new_string(stage) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + let unit_obj = env + .new_string(unit) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + let total_obj = match total { + Some(value) => env + .new_object("java/lang/Long", "(J)V", &[JValue::Long(value)]) + .map_err(|error| callback_error(env, "stageStart", stage, error))?, + None => JObject::null(), + }; + let total_optional = env + .call_static_method( + "java/util/Optional", + "ofNullable", + "(Ljava/lang/Object;)Ljava/util/Optional;", + &[JValue::Object(&total_obj)], + ) + .and_then(|value| value.l()) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + + env.call_method( + &self.callback, + "stageStart", + "(Ljava/lang/String;Ljava/util/Optional;Ljava/lang/String;)V", + &[ + JValue::Object(&stage_obj), + JValue::Object(&total_optional), + JValue::Object(&unit_obj), + ], + ) + .map_err(|error| callback_error(env, "stageStart", stage, error))?; + Ok::<(), Error>(()) + }) + .map_err(|error| lance_core::Error::internal(error.to_string())) + } + + fn call_stage_progress(&self, stage: &str, completed: u64) -> lance_core::Result<()> { + let completed = i64::try_from(completed).map_err(|_| { + lance_core::Error::invalid_input(format!( + "IndexBuildProgress completed value exceeds Java long for stage '{stage}': {completed}" + )) + })?; + let mut env = self.jvm.attach_current_thread().map_err(|error| { + lance_core::Error::internal(format!( + "IndexBuildProgress.stageProgress failed to attach JVM thread for stage '{stage}': {error}" + )) + })?; + + env.with_local_frame(16, |env| { + let stage_obj = env + .new_string(stage) + .map_err(|error| callback_error(env, "stageProgress", stage, error))?; + + env.call_method( + &self.callback, + "stageProgress", + "(Ljava/lang/String;J)V", + &[JValue::Object(&stage_obj), JValue::Long(completed)], + ) + .map_err(|error| callback_error(env, "stageProgress", stage, error))?; + Ok::<(), Error>(()) + }) + .map_err(|error| lance_core::Error::internal(error.to_string())) + } + + fn call_stage_complete(&self, stage: &str) -> lance_core::Result<()> { + let mut env = self.jvm.attach_current_thread().map_err(|error| { + lance_core::Error::internal(format!( + "IndexBuildProgress.stageComplete failed to attach JVM thread for stage '{stage}': {error}" + )) + })?; + + env.with_local_frame(16, |env| { + let stage_obj = env + .new_string(stage) + .map_err(|error| callback_error(env, "stageComplete", stage, error))?; + + env.call_method( + &self.callback, + "stageComplete", + "(Ljava/lang/String;)V", + &[JValue::Object(&stage_obj)], + ) + .map_err(|error| callback_error(env, "stageComplete", stage, error))?; + Ok::<(), Error>(()) + }) + .map_err(|error| lance_core::Error::internal(error.to_string())) + } +} + +fn callback_error(env: &mut JNIEnv, method: &str, stage: &str, error: jni::errors::Error) -> Error { + let java_exception = take_pending_java_exception(env); + let detail = java_exception.unwrap_or_else(|| error.to_string()); + Error::runtime_error(format!( + "IndexBuildProgress.{method} callback failed for stage '{stage}': {detail}" + )) +} + +fn take_pending_java_exception(env: &mut JNIEnv) -> Option { + if !env.exception_check().unwrap_or(false) { + return None; + } + + let throwable = env.exception_occurred().ok(); + let _ = env.exception_clear(); + + let description = throwable.and_then(|throwable| { + if throwable.is_null() { + return None; + } + let description = env + .call_method(&throwable, "toString", "()Ljava/lang/String;", &[]) + .and_then(|value| value.l()) + .ok()?; + if description.is_null() { + return None; + } + let description = JString::from(description); + env.get_string(&description).ok().map(|value| value.into()) + }); + + if env.exception_check().unwrap_or(false) { + let _ = env.exception_clear(); + } + description +} + +#[async_trait] +impl IndexBuildProgress for JavaIndexBuildProgress { + async fn stage_start( + &self, + stage: &str, + total: Option, + unit: &str, + ) -> lance_core::Result<()> { + self.call_stage_start(stage, total, unit) + } + + async fn stage_progress(&self, stage: &str, completed: u64) -> lance_core::Result<()> { + self.call_stage_progress(stage, completed) + } + + async fn stage_complete(&self, stage: &str) -> lance_core::Result<()> { + if let Err(error) = self.call_stage_complete(stage) { + log::warn!( + "Ignoring IndexBuildProgress.stageComplete callback failure for stage '{}': {}", + stage, + error + ); + } + Ok(()) + } +} diff --git a/java/lance-jni/src/lib.rs b/java/lance-jni/src/lib.rs index 37eeff66693..3e342a3eb9a 100644 --- a/java/lance-jni/src/lib.rs +++ b/java/lance-jni/src/lib.rs @@ -51,10 +51,12 @@ mod file_reader; mod file_writer; mod fragment; mod index; +mod index_progress; mod mem_wal; mod merge_insert; mod namespace; mod optimize; +mod otel; mod schema; mod session; mod sql; @@ -85,6 +87,22 @@ pub static RT: LazyLock = LazyLock::new(|| { .expect("Failed to create tokio runtime") }); +/// Drive a future on the shared JNI runtime, including nested calls. +/// +/// Progress callbacks (and similar JNI re-entry) may invoke Dataset methods while +/// already inside `RT.block_on`. Calling `Runtime::block_on` again panics with +/// "Cannot start a runtime from within a runtime". When a Tokio handle is already +/// available, use `block_in_place` + `Handle::block_on` instead. +/// +/// JNI entry points should use this helper instead of calling `RT.block_on` +/// directly so they remain safe when invoked from a callback. +pub fn block_on(future: F) -> F::Output { + match tokio::runtime::Handle::try_current() { + Ok(handle) => tokio::task::block_in_place(|| handle.block_on(future)), + Err(_) => RT.block_on(future), + } +} + fn set_timestamp_precision(builder: &mut env_logger::Builder) { if let Ok(timestamp_precision) = env::var("LANCE_LOG_TS_PRECISION") { match timestamp_precision.as_str() { diff --git a/java/lance-jni/src/mem_wal.rs b/java/lance-jni/src/mem_wal.rs index 37fe377ed17..95755d6635b 100644 --- a/java/lance-jni/src/mem_wal.rs +++ b/java/lance-jni/src/mem_wal.rs @@ -26,10 +26,10 @@ use jni::objects::{JClass, JMap, JObject, JString, JValueGen}; use jni::sys::{jdouble, jint, jlong}; use lance::dataset::Dataset as LanceDataset; use lance::dataset::mem_wal::scanner::{ - FlushedGeneration, LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, + LsmDataSourceCollector, LsmPointLookupPlanner, LsmVectorSearchPlanner, SsTable, parse_filter_expr as parse_lsm_filter_expr, write_pk_sidecar, }; -use lance::dataset::mem_wal::write::{MemTableStats, WriteStatsSnapshot}; +use lance::dataset::mem_wal::write::{MemTableStats, ShardMemory, WriteStatsSnapshot}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardSnapshot, ShardWriter, ShardWriterConfig, evaluate_sharding_spec_with_source_columns, @@ -41,7 +41,6 @@ use lance_io::ffi::to_ffi_arrow_array_stream; use lance_linalg::distance::DistanceType; use uuid::Uuid; -use crate::RT; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; @@ -49,6 +48,7 @@ use crate::traits::{ FromJString, IntoJava, export_vec, import_vec, import_vec_from_method, import_vec_to_rust, }; use crate::utils::to_rust_map; +use crate::{RT, block_on}; const NATIVE_SHARD_WRITER: &str = "nativeShardWriterHandle"; const NATIVE_LSM_SCANNER: &str = "nativeLsmScannerHandle"; @@ -126,7 +126,7 @@ fn inner_create_shard_writer<'local>( Arc::new(guard.inner.clone()) }; - let writer = RT.block_on(dataset.mem_wal_writer(uuid, writer_config))?; + let writer = block_on(dataset.mem_wal_writer(uuid, writer_config))?; let blocking = BlockingShardWriter { writer, shard_id: uuid, @@ -177,14 +177,37 @@ fn inner_put(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()> let guard = unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?; - RT.block_on(guard.writer.put(batches))?; + block_on(guard.writer.put(batches))?; Ok(()) } -/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for a -/// flushed-generation dataset already staged at `gen_path`, mirroring what -/// production flush emits. Lets Java tests stage a *faithful* flushed -/// generation (dataset + sidecar); production always writes the sidecar during +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_memwal_ShardWriter_nativeDelete( + mut env: JNIEnv, + this: JObject, + stream_addr: jlong, +) { + ok_or_throw_without_return!(env, inner_delete(&mut env, this, stream_addr)); +} + +fn inner_delete(env: &mut JNIEnv, this: JObject, stream_addr: jlong) -> Result<()> { + let stream_ptr = stream_addr as *mut FFI_ArrowArrayStream; + let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; + let batches: Vec = reader.collect::>()?; + if batches.is_empty() { + return Ok(()); + } + + let guard = + unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?; + block_on(guard.writer.delete(batches))?; + Ok(()) +} + +/// Test-support: write a primary-key dedup sidecar (`_pk_index/`) for an +/// SSTable dataset already staged at `gen_path`, mirroring what +/// production flush emits. Lets Java tests stage a *faithful* SSTable +/// (dataset + sidecar); production always writes the sidecar during /// flush, so a dataset-without-sidecar is not a state the system produces. /// Mirrors the Python `_write_pk_sidecar` binding. #[unsafe(no_mangle)] @@ -213,7 +236,7 @@ fn inner_write_pk_sidecar( let reader = unsafe { ArrowArrayStreamReader::from_raw(stream_ptr) }?; let batches: Vec = reader.collect::>()?; let pk_refs: Vec<&str> = pk_columns.iter().map(String::as_str).collect(); - RT.block_on(write_pk_sidecar(&gen_path, &batches, &pk_refs))?; + block_on(write_pk_sidecar(&gen_path, &batches, &pk_refs))?; Ok(()) } @@ -249,12 +272,16 @@ fn inner_memtable_stats<'local>( env: &mut JNIEnv<'local>, this: JObject<'local>, ) -> Result> { - let stats = { + let (stats, memory) = { let guard = unsafe { env.get_rust_field::<_, _, BlockingShardWriter>(&this, NATIVE_SHARD_WRITER) }?; - RT.block_on(guard.writer.memtable_stats())? + // Byte totals live on `memory()` now, not on `MemTableStats`. + ( + block_on(guard.writer.memtable_stats())?, + guard.writer.memory(), + ) }; - memtable_stats_to_java(env, &stats) + memtable_stats_to_java(env, &stats, &memory) } #[unsafe(no_mangle)] @@ -283,9 +310,8 @@ fn inner_writer_lsm_scanner<'local>( // Capture the active memtable *and* any frozen-awaiting-flush memtables // so a concurrent flush rollover cannot hide acknowledged writes from // this read-your-writes scanner. - let in_memory_memtables = RT.block_on(guard.writer.in_memory_memtable_refs())?; - let writer_snapshot = RT - .block_on(guard.writer.manifest())? + let in_memory_memtables = block_on(guard.writer.in_memory_memtable_refs())?; + let writer_snapshot = block_on(guard.writer.manifest())? .map(shard_snapshot_from_manifest) .unwrap_or_else(|| ShardSnapshot::new(guard.shard_id)); ( @@ -316,7 +342,7 @@ pub extern "system" fn Java_org_lance_memwal_ShardWriter_releaseNativeShardWrite fn inner_release_shard_writer(env: &mut JNIEnv, this: JObject) -> Result<()> { let blocking: BlockingShardWriter = unsafe { env.take_rust_field(&this, NATIVE_SHARD_WRITER) }?; - RT.block_on(blocking.writer.close())?; + block_on(blocking.writer.close())?; Ok(()) } @@ -475,7 +501,7 @@ fn inner_scanner_open_stream(env: &mut JNIEnv, this: JObject, stream_addr: jlong let scanner = guard.inner.as_ref().ok_or_else(|| { Error::runtime_error("LsmScanner is no longer usable because an earlier builder call (e.g. filter) failed; create a new scanner".to_string()) })?; - RT.block_on(scanner.try_into_stream())? + block_on(scanner.try_into_stream())? }; let ffi_stream = to_ffi_arrow_array_stream(DatasetRecordBatchStream::new(stream), RT.handle().clone())?; @@ -498,7 +524,7 @@ fn inner_scanner_count_rows(env: &mut JNIEnv, this: JObject) -> Result { .inner .as_ref() .ok_or_else(|| Error::runtime_error("LsmScanner is no longer usable because an earlier builder call (e.g. filter) failed; create a new scanner".to_string()))?; - Ok(RT.block_on(scanner.count_rows())?) + Ok(block_on(scanner.count_rows())?) } #[unsafe(no_mangle)] @@ -625,12 +651,11 @@ fn inner_plan_open_stream(env: &mut JNIEnv, this: JObject, stream_addr: jlong) - guard.plan.clone() }; let schema = plan.schema(); - let batches = RT - .block_on(async move { - let ctx = SessionContext::new(); - collect(plan, ctx.task_ctx()).await - }) - .map_err(|e| Error::io_error(format!("Plan execution failed: {}", e)))?; + let batches = block_on(async move { + let ctx = SessionContext::new(); + collect(plan, ctx.task_ctx()).await + }) + .map_err(|e| Error::io_error(format!("Plan execution failed: {}", e)))?; let reader: Box = Box::new(RecordBatchIterator::new( batches.into_iter().map(Ok), @@ -790,7 +815,7 @@ fn inner_plan_lookup<'local>( env.get_rust_field::<_, _, BlockingLsmPointLookupPlanner>(&this, NATIVE_LOOKUP_PLANNER) }?; let pk_values = scalar_values_from_pk_value(pk_value.as_ref(), &guard.pk_columns)?; - let plan = RT.block_on(guard.planner.plan_lookup(&pk_values, columns.as_deref()))?; + let plan = block_on(guard.planner.plan_lookup(&pk_values, columns.as_deref()))?; (plan, guard.dataset_schema.clone()) }; attach_execution_plan(env, plan, dataset_schema) @@ -961,7 +986,7 @@ fn inner_plan_search<'local>( Arc::new(float32_array.clone()), None, )?; - let plan = RT.block_on(guard.planner.plan_search( + let plan = block_on(guard.planner.plan_search( &fsl, k as usize, nprobes as usize, @@ -1049,7 +1074,7 @@ fn inner_initialize_mem_wal(env: &mut JNIEnv, jdataset: JObject, params: JObject if let Some(config) = writer_config { builder = builder.writer_config_defaults(config); } - RT.block_on(builder.execute())?; + block_on(builder.execute())?; Ok(()) } @@ -1068,7 +1093,7 @@ fn inner_mem_wal_index_details<'local>( let details = { let guard = unsafe { env.get_rust_field::<_, _, BlockingDataset>(&jdataset, NATIVE_DATASET) }?; - RT.block_on(guard.inner.mem_wal_index_details())? + block_on(guard.inner.mem_wal_index_details())? }; match details { Some(details) => index_details_to_java(env, &details), @@ -1096,13 +1121,13 @@ fn read_shard_snapshots(env: &mut JNIEnv, list_obj: &JObject) -> Result ShardSnapshot { shard_id: manifest.shard_id, spec_id: manifest.shard_spec_id, current_generation: manifest.current_generation, - flushed_generations: manifest - .flushed_generations + sstables: manifest + .sstables .into_iter() - .map(|generation| FlushedGeneration { - generation: generation.generation, - path: generation.path, + .map(|sstable| SsTable { + generation: sstable.generation, + path: sstable.path, }) .collect(), } @@ -1242,9 +1267,6 @@ fn build_writer_config(env: &mut JNIEnv, config: &JObject) -> Result Result( )?) } -fn memtable_stats_to_java<'a>(env: &mut JNIEnv<'a>, stats: &MemTableStats) -> Result> { +fn memtable_stats_to_java<'a>( + env: &mut JNIEnv<'a>, + stats: &MemTableStats, + memory: &ShardMemory, +) -> Result> { let max_buffered = box_u64_opt(env, stats.max_buffered_batch_position)?; - let max_flushed = box_u64_opt(env, stats.max_flushed_batch_position)?; let pending_start = box_u64_opt(env, stats.pending_wal_start_batch_position)?; let pending_end = box_u64_opt(env, stats.pending_wal_end_batch_position)?; Ok(env.new_object( "org/lance/memwal/MemTableStats", - "(JJJJLjava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;Ljava/lang/Long;JJJ)V", + "(JJJJLjava/lang/Long;JJLjava/lang/Long;Ljava/lang/Long;JJJJJJ)V", &[ JValueGen::Long(stats.row_count as i64), JValueGen::Long(stats.batch_count as i64), - JValueGen::Long(stats.estimated_size as i64), + JValueGen::Long(memory.row_bytes() as i64), JValueGen::Long(stats.generation as i64), JValueGen::Object(&max_buffered), - JValueGen::Object(&max_flushed), + JValueGen::Long(stats.durable_batch_count as i64), + JValueGen::Long(stats.global_offset as i64), JValueGen::Object(&pending_start), JValueGen::Object(&pending_end), JValueGen::Long(stats.pending_wal_batch_count as i64), JValueGen::Long(stats.pending_wal_row_count as i64), JValueGen::Long(stats.pending_wal_estimated_bytes as i64), + JValueGen::Long(memory.index_bytes() as i64), + JValueGen::Long(memory.grace_bytes() as i64), + JValueGen::Long(memory.retained_bytes() as i64), ], )?) } diff --git a/java/lance-jni/src/merge_insert.rs b/java/lance-jni/src/merge_insert.rs index df4d63bd2f6..2e7f869757a 100644 --- a/java/lance-jni/src/merge_insert.rs +++ b/java/lance-jni/src/merge_insert.rs @@ -5,7 +5,7 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::import_vec_to_rust; use crate::traits::{FromJString, IntoJava}; -use crate::{Error, JNIEnvExt, RT}; +use crate::{Error, JNIEnvExt, block_on}; use arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream}; use jni::JNIEnv; use jni::objects::{JObject, JString, JValueGen}; @@ -15,7 +15,7 @@ use lance::dataset::{ MergeInsertBuilder, MergeStats, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, }; use lance_core::datatypes::Schema; -use lance_index::mem_wal::MergedGeneration; +use lance_index::mem_wal::CompactedSsTable; use std::sync::Arc; use std::time::Duration; use uuid::Uuid; @@ -52,7 +52,7 @@ fn inner_merge_insert<'local>( let retry_timeout_ms = extract_retry_timeout_ms(env, &jparam)?; let skip_auto_cleanup = extract_skip_auto_cleanup(env, &jparam)?; let use_index = extract_use_index(env, &jparam)?; - let marked_generations = extract_marked_generations(env, &jparam)?; + let compacted_sstables = extract_compacted_sstables(env, &jparam)?; let (new_ds, merge_stats) = unsafe { let dataset = env.get_rust_field::<_, _, BlockingDataset>(jdataset, NATIVE_DATASET)?; @@ -71,13 +71,13 @@ fn inner_merge_insert<'local>( .retry_timeout(Duration::from_millis(retry_timeout_ms as u64)) .skip_auto_cleanup(skip_auto_cleanup) .use_index(use_index) - .mark_generations_as_merged(marked_generations) + .mark_sstables_as_compacted(compacted_sstables) .try_build()?; let stream_ptr = batch_address as *mut FFI_ArrowArrayStream; let source_stream = ArrowArrayStreamReader::from_raw(stream_ptr)?; - RT.block_on(async move { merge_insert_job.execute_reader(source_stream).await })? + block_on(async move { merge_insert_job.execute_reader(source_stream).await })? }; MergeResult( @@ -241,23 +241,23 @@ fn extract_use_index<'local>(env: &mut JNIEnv<'local>, jparam: &JObject) -> Resu Ok(use_index) } -fn extract_marked_generations<'local>( +fn extract_compacted_sstables<'local>( env: &mut JNIEnv<'local>, jparam: &JObject, -) -> Result> { +) -> Result> { let list = env - .call_method(jparam, "markedGenerations", "()Ljava/util/List;", &[])? + .call_method(jparam, "getCompactedSstables", "()Ljava/util/List;", &[])? .l()?; import_vec_to_rust(env, &list, |env, obj| { let shard_id: JString = env - .call_method(&obj, "shardId", "()Ljava/lang/String;", &[])? + .call_method(&obj, "getShardId", "()Ljava/lang/String;", &[])? .l()? .into(); let shard_id = shard_id.extract(env)?; - let generation = env.call_method(&obj, "generation", "()J", &[])?.j()? as u64; + let generation = env.call_method(&obj, "getGeneration", "()J", &[])?.j()? as u64; let uuid = Uuid::parse_str(&shard_id) .map_err(|e| Error::input_error(format!("Invalid shard_id UUID: {}", e)))?; - Ok(MergedGeneration::new(uuid, generation)) + Ok(CompactedSsTable::new(uuid, generation)) }) } diff --git a/java/lance-jni/src/namespace.rs b/java/lance-jni/src/namespace.rs index f0da7ff79ae..9a670af0e2a 100644 --- a/java/lance-jni/src/namespace.rs +++ b/java/lance-jni/src/namespace.rs @@ -10,6 +10,7 @@ use jni::JNIEnv; use jni::objects::{GlobalRef, JByteArray, JMap, JObject, JString, JValue}; use jni::sys::{jbyteArray, jlong, jobject, jstring}; use lance_namespace::LanceNamespace as LanceNamespaceTrait; +use lance_namespace::compat::merge_insert_request_from_json; use lance_namespace::models::*; use lance_namespace_impls::{ ConnectBuilder, DirectoryNamespace, DirectoryNamespaceBuilder, DynamicContextProvider, @@ -17,7 +18,7 @@ use lance_namespace_impls::{ }; use serde::{Deserialize, Serialize}; -use crate::RT; +use crate::block_on; use crate::error::{Error, Result}; use crate::utils::to_rust_map; @@ -1581,8 +1582,7 @@ fn create_directory_namespace_internal( builder = builder.context_provider(Arc::new(java_provider)); } - let namespace = RT - .block_on(builder.build()) + let namespace = block_on(builder.build()) .map_err(|e| Error::runtime_error(format!("Failed to build DirectoryNamespace: {}", e)))?; let blocking_namespace = BlockingDirectoryNamespace { @@ -1631,7 +1631,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listNamespace ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_namespaces(req)) + block_on(namespace_client.inner.list_namespaces(req)) }), std::ptr::null_mut() ) @@ -1648,7 +1648,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeNames ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_namespace(req)) + block_on(namespace_client.inner.describe_namespace(req)) }), std::ptr::null_mut() ) @@ -1665,7 +1665,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createNamespa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_namespace(req)) + block_on(namespace_client.inner.create_namespace(req)) }), std::ptr::null_mut() ) @@ -1682,7 +1682,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropNamespace ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_namespace(req)) + block_on(namespace_client.inner.drop_namespace(req)) }), std::ptr::null_mut() ) @@ -1699,7 +1699,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_namespaceExis ok_or_throw_without_return!( env, call_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.namespace_exists(req)) + block_on(namespace_client.inner.namespace_exists(req)) }) ) } @@ -1714,7 +1714,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTablesNat ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_tables(req)) + block_on(namespace_client.inner.list_tables(req)) }), std::ptr::null_mut() ) @@ -1731,7 +1731,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table(req)) + block_on(namespace_client.inner.describe_table(req)) }), std::ptr::null_mut() ) @@ -1748,7 +1748,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_registerTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.register_table(req)) + block_on(namespace_client.inner.register_table(req)) }), std::ptr::null_mut() ) @@ -1765,7 +1765,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_tableExistsNa ok_or_throw_without_return!( env, call_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.table_exists(req)) + block_on(namespace_client.inner.table_exists(req)) }) ) } @@ -1780,7 +1780,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropTableNati ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table(req)) + block_on(namespace_client.inner.drop_table(req)) }), std::ptr::null_mut() ) @@ -1797,7 +1797,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deregisterTab ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.deregister_table(req)) + block_on(namespace_client.inner.deregister_table(req)) }), std::ptr::null_mut() ) @@ -1834,7 +1834,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableNa request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.create_table(req, data)) + block_on(namespace_client.inner.create_table(req, data)) } ), std::ptr::null_mut() @@ -1852,7 +1852,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_declareTableN ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.declare_table(req)) + block_on(namespace_client.inner.declare_table(req)) }), std::ptr::null_mut() ) @@ -1869,7 +1869,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_renameTableNa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.rename_table(req)) + block_on(namespace_client.inner.rename_table(req)) }), std::ptr::null_mut() ) @@ -1892,7 +1892,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_insertIntoTab request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.insert_into_table(req, data)) + block_on(namespace_client.inner.insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -1915,8 +1915,9 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_mergeInsertIn handle, request_json, request_data, - |namespace_client, req, data| { - RT.block_on(namespace_client.inner.merge_insert_into_table(req, data)) + |namespace_client, req: serde_json::Value, data| { + let req = merge_insert_request_from_json(req)?; + block_on(namespace_client.inner.merge_insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -1934,7 +1935,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableNa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table(req)) + block_on(namespace_client.inner.update_table(req)) }), std::ptr::null_mut() ) @@ -1951,7 +1952,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deleteFromTab ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_from_table(req)) + block_on(namespace_client.inner.delete_from_table(req)) }), std::ptr::null_mut() ) @@ -1983,7 +1984,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableIn ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_index(req)) + block_on(namespace_client.inner.create_table_index(req)) }), std::ptr::null_mut() ) @@ -2000,7 +2001,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableIndi ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_indices(req)) + block_on(namespace_client.inner.list_table_indices(req)) }), std::ptr::null_mut() ) @@ -2017,7 +2018,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_index_stats(req)) + block_on(namespace_client.inner.describe_table_index_stats(req)) }), std::ptr::null_mut() ) @@ -2034,7 +2035,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTrans ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_transaction(req)) + block_on(namespace_client.inner.describe_transaction(req)) }), std::ptr::null_mut() ) @@ -2051,7 +2052,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTransact ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_transaction(req)) + block_on(namespace_client.inner.alter_transaction(req)) }), std::ptr::null_mut() ) @@ -2068,7 +2069,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableVers ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_versions(req)) + block_on(namespace_client.inner.list_table_versions(req)) }), std::ptr::null_mut() ) @@ -2085,7 +2086,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableVe ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_version(req)) + block_on(namespace_client.inner.create_table_version(req)) }), std::ptr::null_mut() ) @@ -2102,7 +2103,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_describeTable ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_version(req)) + block_on(namespace_client.inner.describe_table_version(req)) }), std::ptr::null_mut() ) @@ -2119,7 +2120,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_batchDeleteTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.batch_delete_table_versions(req)) + block_on(namespace_client.inner.batch_delete_table_versions(req)) }), std::ptr::null_mut() ) @@ -2136,7 +2137,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableSc ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_scalar_index(req)) + block_on(namespace_client.inner.create_table_scalar_index(req)) }), std::ptr::null_mut() ) @@ -2153,7 +2154,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_dropTableInde ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table_index(req)) + block_on(namespace_client.inner.drop_table_index(req)) }), std::ptr::null_mut() ) @@ -2170,7 +2171,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listAllTables ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_all_tables(req)) + block_on(namespace_client.inner.list_all_tables(req)) }), std::ptr::null_mut() ) @@ -2187,7 +2188,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_restoreTableN ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.restore_table(req)) + block_on(namespace_client.inner.restore_table(req)) }), std::ptr::null_mut() ) @@ -2204,7 +2205,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableSc ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_schema_metadata(req)) + block_on(namespace_client.inner.update_table_schema_metadata(req)) }), std::ptr::null_mut() ) @@ -2221,7 +2222,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_getTableStats ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_stats(req)) + block_on(namespace_client.inner.get_table_stats(req)) }), std::ptr::null_mut() ) @@ -2238,7 +2239,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_explainTableQ ok_or_throw_with_return!( env, call_namespace_string_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.explain_table_query_plan(req)) + block_on(namespace_client.inner.explain_table_query_plan(req)) }), std::ptr::null_mut() ) @@ -2255,7 +2256,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_analyzeTableQ ok_or_throw_with_return!( env, call_namespace_string_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.analyze_table_query_plan(req)) + block_on(namespace_client.inner.analyze_table_query_plan(req)) }), std::ptr::null_mut() ) @@ -2272,7 +2273,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableAdd ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_add_columns(req)) + block_on(namespace_client.inner.alter_table_add_columns(req)) }), std::ptr::null_mut() ) @@ -2289,7 +2290,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableAlt ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_alter_columns(req)) + block_on(namespace_client.inner.alter_table_alter_columns(req)) }), std::ptr::null_mut() ) @@ -2306,7 +2307,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableDro ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_drop_columns(req)) + block_on(namespace_client.inner.alter_table_drop_columns(req)) }), std::ptr::null_mut() ) @@ -2323,7 +2324,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_alterTableBac ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_backfill_columns(req)) + block_on(namespace_client.inner.alter_table_backfill_columns(req)) }), std::ptr::null_mut() ) @@ -2340,7 +2341,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_refreshMateri ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.refresh_materialized_view(req)) + block_on(namespace_client.inner.refresh_materialized_view(req)) }), std::ptr::null_mut() ) @@ -2357,7 +2358,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_listTableTags ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_tags(req)) + block_on(namespace_client.inner.list_table_tags(req)) }), std::ptr::null_mut() ) @@ -2374,7 +2375,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_getTableTagVe ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_tag_version(req)) + block_on(namespace_client.inner.get_table_tag_version(req)) }), std::ptr::null_mut() ) @@ -2391,7 +2392,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createTableTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_tag(req)) + block_on(namespace_client.inner.create_table_tag(req)) }), std::ptr::null_mut() ) @@ -2408,7 +2409,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_deleteTableTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_table_tag(req)) + block_on(namespace_client.inner.delete_table_tag(req)) }), std::ptr::null_mut() ) @@ -2425,7 +2426,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_updateTableTa ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_tag(req)) + block_on(namespace_client.inner.update_table_tag(req)) }), std::ptr::null_mut() ) @@ -2442,7 +2443,7 @@ pub extern "system" fn Java_org_lance_namespace_DirectoryNamespace_createMateria ok_or_throw_with_return!( env, call_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_materialized_view(req)) + block_on(namespace_client.inner.create_materialized_view(req)) }), std::ptr::null_mut() ) @@ -2581,7 +2582,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listNamespacesNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_namespaces(req)) + block_on(namespace_client.inner.list_namespaces(req)) }), std::ptr::null_mut() ) @@ -2598,7 +2599,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeNamespaceN ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_namespace(req)) + block_on(namespace_client.inner.describe_namespace(req)) }), std::ptr::null_mut() ) @@ -2615,7 +2616,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createNamespaceNat ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_namespace(req)) + block_on(namespace_client.inner.create_namespace(req)) }), std::ptr::null_mut() ) @@ -2632,7 +2633,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropNamespaceNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_namespace(req)) + block_on(namespace_client.inner.drop_namespace(req)) }), std::ptr::null_mut() ) @@ -2649,7 +2650,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_namespaceExistsNat ok_or_throw_without_return!( env, call_rest_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.namespace_exists(req)) + block_on(namespace_client.inner.namespace_exists(req)) }) ) } @@ -2664,7 +2665,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTablesNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_tables(req)) + block_on(namespace_client.inner.list_tables(req)) }), std::ptr::null_mut() ) @@ -2681,7 +2682,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table(req)) + block_on(namespace_client.inner.describe_table(req)) }), std::ptr::null_mut() ) @@ -2698,7 +2699,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_registerTableNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.register_table(req)) + block_on(namespace_client.inner.register_table(req)) }), std::ptr::null_mut() ) @@ -2715,7 +2716,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_tableExistsNative( ok_or_throw_without_return!( env, call_rest_namespace_void_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.table_exists(req)) + block_on(namespace_client.inner.table_exists(req)) }) ) } @@ -2730,7 +2731,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table(req)) + block_on(namespace_client.inner.drop_table(req)) }), std::ptr::null_mut() ) @@ -2747,7 +2748,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deregisterTableNat ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.deregister_table(req)) + block_on(namespace_client.inner.deregister_table(req)) }), std::ptr::null_mut() ) @@ -2784,7 +2785,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableNative( request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.create_table(req, data)) + block_on(namespace_client.inner.create_table(req, data)) } ), std::ptr::null_mut() @@ -2802,7 +2803,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_declareTableNative ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.declare_table(req)) + block_on(namespace_client.inner.declare_table(req)) }), std::ptr::null_mut() ) @@ -2819,7 +2820,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_renameTableNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.rename_table(req)) + block_on(namespace_client.inner.rename_table(req)) }), std::ptr::null_mut() ) @@ -2842,7 +2843,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_insertIntoTableNat request_json, request_data, |namespace_client, req, data| { - RT.block_on(namespace_client.inner.insert_into_table(req, data)) + block_on(namespace_client.inner.insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -2865,8 +2866,9 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_mergeInsertIntoTab handle, request_json, request_data, - |namespace_client, req, data| { - RT.block_on(namespace_client.inner.merge_insert_into_table(req, data)) + |namespace_client, req: serde_json::Value, data| { + let req = merge_insert_request_from_json(req)?; + block_on(namespace_client.inner.merge_insert_into_table(req, data)) } ), std::ptr::null_mut() @@ -2884,7 +2886,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableNative( ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table(req)) + block_on(namespace_client.inner.update_table(req)) }), std::ptr::null_mut() ) @@ -2901,7 +2903,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteFromTableNat ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_from_table(req)) + block_on(namespace_client.inner.delete_from_table(req)) }), std::ptr::null_mut() ) @@ -2933,7 +2935,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableIndexNa ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_index(req)) + block_on(namespace_client.inner.create_table_index(req)) }), std::ptr::null_mut() ) @@ -2950,7 +2952,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableIndicesNa ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_indices(req)) + block_on(namespace_client.inner.list_table_indices(req)) }), std::ptr::null_mut() ) @@ -2967,7 +2969,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableIndex ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_index_stats(req)) + block_on(namespace_client.inner.describe_table_index_stats(req)) }), std::ptr::null_mut() ) @@ -2984,7 +2986,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTransactio ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_transaction(req)) + block_on(namespace_client.inner.describe_transaction(req)) }), std::ptr::null_mut() ) @@ -3001,7 +3003,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTransactionNa ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_transaction(req)) + block_on(namespace_client.inner.alter_transaction(req)) }), std::ptr::null_mut() ) @@ -3018,7 +3020,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableVersionsN ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_versions(req)) + block_on(namespace_client.inner.list_table_versions(req)) }), std::ptr::null_mut() ) @@ -3035,7 +3037,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableVersion ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_version(req)) + block_on(namespace_client.inner.create_table_version(req)) }), std::ptr::null_mut() ) @@ -3052,7 +3054,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_describeTableVersi ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.describe_table_version(req)) + block_on(namespace_client.inner.describe_table_version(req)) }), std::ptr::null_mut() ) @@ -3069,7 +3071,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_batchDeleteTableVe ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.batch_delete_table_versions(req)) + block_on(namespace_client.inner.batch_delete_table_versions(req)) }), std::ptr::null_mut() ) @@ -3086,7 +3088,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableScalarI ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_scalar_index(req)) + block_on(namespace_client.inner.create_table_scalar_index(req)) }), std::ptr::null_mut() ) @@ -3103,7 +3105,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_dropTableIndexNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.drop_table_index(req)) + block_on(namespace_client.inner.drop_table_index(req)) }), std::ptr::null_mut() ) @@ -3120,7 +3122,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listAllTablesNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_all_tables(req)) + block_on(namespace_client.inner.list_all_tables(req)) }), std::ptr::null_mut() ) @@ -3137,7 +3139,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_restoreTableNative ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.restore_table(req)) + block_on(namespace_client.inner.restore_table(req)) }), std::ptr::null_mut() ) @@ -3154,7 +3156,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableSchemaM ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_schema_metadata(req)) + block_on(namespace_client.inner.update_table_schema_metadata(req)) }), std::ptr::null_mut() ) @@ -3171,7 +3173,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_getTableStatsNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_stats(req)) + block_on(namespace_client.inner.get_table_stats(req)) }), std::ptr::null_mut() ) @@ -3192,7 +3194,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_explainTableQueryP handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.explain_table_query_plan(req)) + block_on(namespace_client.inner.explain_table_query_plan(req)) } ), std::ptr::null_mut() @@ -3214,7 +3216,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_analyzeTableQueryP handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.analyze_table_query_plan(req)) + block_on(namespace_client.inner.analyze_table_query_plan(req)) } ), std::ptr::null_mut() @@ -3232,7 +3234,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableAddColum ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_add_columns(req)) + block_on(namespace_client.inner.alter_table_add_columns(req)) }), std::ptr::null_mut() ) @@ -3249,7 +3251,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableAlterCol ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_alter_columns(req)) + block_on(namespace_client.inner.alter_table_alter_columns(req)) }), std::ptr::null_mut() ) @@ -3266,7 +3268,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableDropColu ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_drop_columns(req)) + block_on(namespace_client.inner.alter_table_drop_columns(req)) }), std::ptr::null_mut() ) @@ -3283,7 +3285,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_alterTableBackfill ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.alter_table_backfill_columns(req)) + block_on(namespace_client.inner.alter_table_backfill_columns(req)) }), std::ptr::null_mut() ) @@ -3300,7 +3302,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_refreshMaterialize ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.refresh_materialized_view(req)) + block_on(namespace_client.inner.refresh_materialized_view(req)) }), std::ptr::null_mut() ) @@ -3317,7 +3319,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_listTableTagsNativ ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.list_table_tags(req)) + block_on(namespace_client.inner.list_table_tags(req)) }), std::ptr::null_mut() ) @@ -3334,7 +3336,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_getTableTagVersion ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.get_table_tag_version(req)) + block_on(namespace_client.inner.get_table_tag_version(req)) }), std::ptr::null_mut() ) @@ -3351,7 +3353,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createTableTagNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_table_tag(req)) + block_on(namespace_client.inner.create_table_tag(req)) }), std::ptr::null_mut() ) @@ -3368,7 +3370,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_deleteTableTagNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.delete_table_tag(req)) + block_on(namespace_client.inner.delete_table_tag(req)) }), std::ptr::null_mut() ) @@ -3385,7 +3387,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_updateTableTagNati ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.update_table_tag(req)) + block_on(namespace_client.inner.update_table_tag(req)) }), std::ptr::null_mut() ) @@ -3402,7 +3404,7 @@ pub extern "system" fn Java_org_lance_namespace_RestNamespace_createMaterialized ok_or_throw_with_return!( env, call_rest_namespace_method(&mut env, handle, request_json, |namespace_client, req| { - RT.block_on(namespace_client.inner.create_materialized_view(req)) + block_on(namespace_client.inner.create_materialized_view(req)) }), std::ptr::null_mut() ) @@ -3522,9 +3524,7 @@ fn call_namespace_count_method( let request: CountTableRowsRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let count = RT - .block_on(namespace.inner.count_table_rows(request)) - .map_err(Error::from)?; + let count = block_on(namespace.inner.count_table_rows(request)).map_err(Error::from)?; Ok(count) } @@ -3569,9 +3569,7 @@ fn call_namespace_query_method<'local>( let request: QueryTableRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let result_bytes = RT - .block_on(namespace.inner.query_table(request)) - .map_err(Error::from)?; + let result_bytes = block_on(namespace.inner.query_table(request)).map_err(Error::from)?; let byte_array = env.byte_array_from_slice(&result_bytes)?; Ok(byte_array) @@ -3655,9 +3653,7 @@ fn call_rest_namespace_count_method( let request: CountTableRowsRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let count = RT - .block_on(namespace.inner.count_table_rows(request)) - .map_err(Error::from)?; + let count = block_on(namespace.inner.count_table_rows(request)).map_err(Error::from)?; Ok(count) } @@ -3702,9 +3698,7 @@ fn call_rest_namespace_query_method<'local>( let request: QueryTableRequest = serde_json::from_str(&request_str) .map_err(|e| Error::input_error(format!("Failed to parse request JSON: {}", e)))?; - let result_bytes = RT - .block_on(namespace.inner.query_table(request)) - .map_err(Error::from)?; + let result_bytes = block_on(namespace.inner.query_table(request)).map_err(Error::from)?; let byte_array = env.byte_array_from_slice(&result_bytes)?; Ok(byte_array) @@ -3756,8 +3750,7 @@ fn create_rest_adapter_internal( builder = builder.property(k, v); } - let backend = RT - .block_on(builder.connect()) + let backend = block_on(builder.connect()) .map_err(|e| Error::runtime_error(format!("Failed to build backend namespace: {}", e)))?; // Build config with defaults, overriding if values provided @@ -3799,7 +3792,7 @@ pub extern "system" fn Java_org_lance_namespace_RestAdapter_start( fn start_internal(handle: jlong) -> Result<()> { let adapter = unsafe { &mut *(handle as *mut BlockingRestAdapter) }; let rest_adapter = RestAdapter::new(adapter.backend.clone(), adapter.config.clone()); - let server_handle = RT.block_on(rest_adapter.start())?; + let server_handle = block_on(rest_adapter.start())?; adapter.server_handle = Some(server_handle); Ok(()) } diff --git a/java/lance-jni/src/optimize.rs b/java/lance-jni/src/optimize.rs index 0ce92baeec8..5e34f622c93 100644 --- a/java/lance-jni/src/optimize.rs +++ b/java/lance-jni/src/optimize.rs @@ -17,14 +17,14 @@ use lance::dataset::{ }; use crate::{ - RT, + block_on, blocking_dataset::{BlockingDataset, NATIVE_DATASET}, traits::{ FromJObjectWithEnv, IntoJava, export_vec, import_vec_from_method, import_vec_to_rust, }, utils::{ - build_compaction_options, to_java_boolean_obj, to_java_float_obj, to_java_long_obj, - to_java_optional, + build_compaction_options, to_java_boolean_obj, to_java_float_obj, to_java_list, + to_java_long_obj, to_java_optional, }, }; @@ -46,6 +46,9 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativePlanCompaction compaction_mode: JObject, // Optional binary_copy_read_batch_bytes: JObject, // Optional max_source_fragments: JObject, // Optional + max_source_rows: JObject, // Optional + max_source_bytes: JObject, // Optional + excluded_fragment_ids: JObject, // List ) -> JObject<'local> { ok_or_throw_with_return!( env, @@ -62,7 +65,10 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativePlanCompaction defer_index_remap, compaction_mode, binary_copy_read_batch_bytes, - max_source_fragments + max_source_fragments, + max_source_rows, + max_source_bytes, + excluded_fragment_ids ), JObject::null() ) @@ -83,6 +89,9 @@ fn inner_plan_compaction<'local>( compaction_mode: JObject, // Optional binary_copy_read_batch_bytes: JObject, // Optional max_source_fragments: JObject, // Optional + max_source_rows: JObject, // Optional + max_source_bytes: JObject, // Optional + excluded_fragment_ids: JObject, // List ) -> Result> { let config = { let dataset = @@ -102,19 +111,22 @@ fn inner_plan_compaction<'local>( &compaction_mode, &binary_copy_read_batch_bytes, &max_source_fragments, + &max_source_rows, + &max_source_bytes, + &excluded_fragment_ids, &config, )?; let plan = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(plan_compaction(&dataset.inner, &compaction_options))? + block_on(plan_compaction(&dataset.inner, &compaction_options))? }; plan.into_java(env) } #[unsafe(no_mangle)] -pub extern "system" fn Java_org_lance_compaction_Compaction_nativeCommitCompaction<'local>( +pub extern "system" fn Java_org_lance_compaction_Compaction_commitCompactionNative<'local>( mut env: JNIEnv<'local>, _obj: JObject, java_dataset: JObject, // Dataset @@ -130,6 +142,9 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativeCommitCompacti compaction_mode: JObject, // Optional binary_copy_read_batch_bytes: JObject, // Optional max_source_fragments: JObject, // Optional + max_source_rows: JObject, // Optional + max_source_bytes: JObject, // Optional + excluded_fragment_ids: JObject, // List ) -> JObject<'local> { ok_or_throw_with_return!( env, @@ -148,6 +163,9 @@ pub extern "system" fn Java_org_lance_compaction_Compaction_nativeCommitCompacti compaction_mode, binary_copy_read_batch_bytes, max_source_fragments, + max_source_rows, + max_source_bytes, + excluded_fragment_ids, ), JObject::null() ) @@ -169,6 +187,9 @@ fn inner_commit_compaction<'local>( compaction_mode: JObject, // Optional binary_copy_read_batch_bytes: JObject, // Optional max_source_fragments: JObject, // Optional + max_source_rows: JObject, // Optional + max_source_bytes: JObject, // Optional + excluded_fragment_ids: JObject, // List ) -> Result> { let config = { let dataset = @@ -188,6 +209,9 @@ fn inner_commit_compaction<'local>( &compaction_mode, &binary_copy_read_batch_bytes, &max_source_fragments, + &max_source_rows, + &max_source_bytes, + &excluded_fragment_ids, &config, )?; let completed_tasks = import_vec_to_rust(env, &rewrite_results, |env, rewrite_result| { @@ -197,7 +221,7 @@ fn inner_commit_compaction<'local>( let committed_metrics = { let mut dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(commit_compaction( + block_on(commit_compaction( &mut dataset.inner, completed_tasks, remap_options, @@ -225,6 +249,9 @@ pub extern "system" fn Java_org_lance_compaction_CompactionTask_nativeExecute<'l compaction_mode: JObject, // Optional binary_copy_read_batch_bytes: JObject, // Optional max_source_fragments: JObject, // Optional + max_source_rows: JObject, // Optional + max_source_bytes: JObject, // Optional + excluded_fragment_ids: JObject, // List ) -> JObject<'local> { ok_or_throw_with_return!( env, @@ -243,7 +270,10 @@ pub extern "system" fn Java_org_lance_compaction_CompactionTask_nativeExecute<'l defer_index_remap, compaction_mode, binary_copy_read_batch_bytes, - max_source_fragments + max_source_fragments, + max_source_rows, + max_source_bytes, + excluded_fragment_ids ), JObject::null() ) @@ -266,6 +296,9 @@ fn inner_execute_task<'local>( compaction_mode: JObject, // Optional binary_copy_read_batch_bytes: JObject, // Optional max_source_fragments: JObject, // Optional + max_source_rows: JObject, // Optional + max_source_bytes: JObject, // Optional + excluded_fragment_ids: JObject, // List ) -> Result> { let task_data: TaskData = task_data.extract_object(env)?; let config = { @@ -286,6 +319,9 @@ fn inner_execute_task<'local>( &compaction_mode, &binary_copy_read_batch_bytes, &max_source_fragments, + &max_source_rows, + &max_source_bytes, + &excluded_fragment_ids, &config, )?; let compaction_task = CompactionTask { @@ -296,7 +332,7 @@ fn inner_execute_task<'local>( let rewrite_result = { let dataset = unsafe { env.get_rust_field::<_, _, BlockingDataset>(java_dataset, NATIVE_DATASET) }?; - RT.block_on(compaction_task.execute(&dataset.inner))? + block_on(compaction_task.execute(&dataset.inner))? }; rewrite_result.into_java(env) } @@ -312,7 +348,8 @@ const REWRITE_RESULT_CLASS: &str = "org/lance/compaction/RewriteResult"; const REWRITE_RESULT_CONSTRUCTOR_SIG: &str = "(Lorg/lance/compaction/CompactionMetrics;Ljava/util/List;Ljava/util/List;J[B)V"; const COMPACTION_OPTIONS_CLASS: &str = "org/lance/compaction/CompactionOptions"; -const COMPACTION_OPTIONS_CONSTRUCTOR_SIG: &str = "(Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;)V"; +const COMPACTION_MODE_CLASS: &str = "org/lance/compaction/CompactionMode"; +const COMPACTION_OPTIONS_CONSTRUCTOR_SIG: &str = "(Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/Optional;Ljava/util/List;)V"; impl IntoJava for &TaskData { fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> Result> { @@ -361,13 +398,20 @@ impl IntoJava for &CompactionOptions { let batch_size_opt = to_java_optional(env, batch_size)?; let defer_index_remap = to_java_boolean_obj(env, Some(self.defer_index_remap))?; let defer_index_remap_opt = to_java_optional(env, defer_index_remap)?; - let compaction_mode_str = self.compaction_mode.as_ref().map(|mode| match mode { - CompactionMode::Reencode => "reencode", - CompactionMode::TryBinaryCopy => "try_binary_copy", - CompactionMode::ForceBinaryCopy => "force_binary_copy", - }); - let compaction_mode_obj = match compaction_mode_str { - Some(s) => env.new_string(s)?.into(), + let compaction_mode_obj = match self.compaction_mode { + Some(mode) => { + let name = match mode { + CompactionMode::Reencode => "REENCODE", + CompactionMode::TryBinaryCopy => "TRY_BINARY_COPY", + CompactionMode::ForceBinaryCopy => "FORCE_BINARY_COPY", + }; + env.get_static_field( + COMPACTION_MODE_CLASS, + name, + format!("L{};", COMPACTION_MODE_CLASS), + )? + .l()? + } None => JObject::null(), }; let compaction_mode_opt = to_java_optional(env, compaction_mode_obj)?; @@ -377,6 +421,16 @@ impl IntoJava for &CompactionOptions { let max_source_fragments = to_java_long_obj(env, self.max_source_fragments.map(|v| v as i64))?; let max_source_fragments_opt = to_java_optional(env, max_source_fragments)?; + let max_source_rows = to_java_long_obj(env, self.max_source_rows.map(|v| v as i64))?; + let max_source_rows_opt = to_java_optional(env, max_source_rows)?; + let max_source_bytes = to_java_long_obj(env, self.max_source_bytes.map(|v| v as i64))?; + let max_source_bytes_opt = to_java_optional(env, max_source_bytes)?; + let excluded_fragment_ids = self + .excluded_fragment_ids + .iter() + .map(|fragment_id| to_java_long_obj(env, Some(*fragment_id as i64))) + .collect::>>()?; + let excluded_fragment_ids = to_java_list(env, &excluded_fragment_ids)?; Ok(env.new_object( COMPACTION_OPTIONS_CLASS, @@ -393,6 +447,9 @@ impl IntoJava for &CompactionOptions { JValueGen::Object(&compaction_mode_opt), JValueGen::Object(&binary_copy_read_batch_bytes_opt), JValueGen::Object(&max_source_fragments_opt), + JValueGen::Object(&max_source_rows_opt), + JValueGen::Object(&max_source_bytes_opt), + JValueGen::Object(&excluded_fragment_ids), ], )?) } diff --git a/java/lance-jni/src/otel.rs b/java/lance-jni/src/otel.rs new file mode 100644 index 00000000000..54f46c7955a --- /dev/null +++ b/java/lance-jni/src/otel.rs @@ -0,0 +1,541 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Bridge from the [`metrics`] crate facade to Java OpenTelemetry. +//! +//! The Java layer owns the actual OpenTelemetry instruments. This module +//! installs the process-global Rust [`metrics::Recorder`], keeps cumulative +//! metric state, and exposes catalog/snapshot calls over JNI. + +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock}; + +use jni::JNIEnv; +use jni::objects::{JClass, JObject, JValue}; +use jni::sys::{jboolean, jobject}; +use metrics::atomics::AtomicU64; +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; +use metrics_util::registry::{Registry, Storage}; + +use crate::error::Result; + +const DEFAULT_BOUNDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, +]; + +#[derive(Clone, Copy)] +enum MetricKind { + Counter, + Gauge, + Histogram, +} + +impl MetricKind { + fn as_str(self) -> &'static str { + match self { + Self::Counter => "counter", + Self::Gauge => "gauge", + Self::Histogram => "histogram", + } + } +} + +struct MetricDescription { + kind: MetricKind, + unit: Option, + description: String, +} + +static CATALOG: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +static HISTOGRAM_BOUNDS: LazyLock>>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +static REGISTRY: OnceLock>> = OnceLock::new(); + +fn bounds_for(name: &str) -> Arc<[f64]> { + HISTOGRAM_BOUNDS + .read() + .unwrap() + .get(name) + .cloned() + .unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS)) +} + +struct BucketedHistogram { + bounds: Arc<[f64]>, + counts: Box<[AtomicU64]>, + count: AtomicU64, + sum_bits: AtomicU64, +} + +impl BucketedHistogram { + fn new(bounds: Arc<[f64]>) -> Self { + let counts = (0..=bounds.len()) + .map(|_| AtomicU64::new(0)) + .collect::>(); + Self { + bounds, + counts, + count: AtomicU64::new(0), + sum_bits: AtomicU64::new(0), + } + } + + fn add_to_sum(&self, value: f64) { + let mut current = self.sum_bits.load(Ordering::Relaxed); + loop { + let updated = (f64::from_bits(current) + value).to_bits(); + match self.sum_bits.compare_exchange_weak( + current, + updated, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + fn snapshot(&self) -> MetricValue { + let mut cumulative = 0u64; + let mut buckets = Vec::with_capacity(self.bounds.len() + 1); + for (i, bound) in self.bounds.iter().enumerate() { + cumulative += self.counts[i].load(Ordering::Relaxed); + buckets.push((bound.to_string(), cumulative)); + } + cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed); + buckets.push(("+Inf".to_string(), cumulative)); + MetricValue::Histogram { + buckets, + count: self.count.load(Ordering::Relaxed), + sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)), + } + } +} + +impl metrics::HistogramFn for BucketedHistogram { + fn record(&self, value: f64) { + let idx = self.bounds.partition_point(|&bound| bound < value); + self.counts[idx].fetch_add(1, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.add_to_sum(value); + } +} + +struct LanceStorage; + +impl Storage for LanceStorage { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc; + + fn counter(&self, _key: &Key) -> Self::Counter { + Arc::new(AtomicU64::new(0)) + } + + fn gauge(&self, _key: &Key) -> Self::Gauge { + Arc::new(AtomicU64::new(0)) + } + + fn histogram(&self, key: &Key) -> Self::Histogram { + Arc::new(BucketedHistogram::new(bounds_for(key.name()))) + } +} + +struct LanceRecorder { + registry: Arc>, +} + +impl LanceRecorder { + fn describe( + &self, + key: KeyName, + kind: MetricKind, + unit: Option, + description: SharedString, + ) { + CATALOG.lock().unwrap().insert( + key.as_str().to_string(), + MetricDescription { + kind, + unit: unit.map(|u| u.as_canonical_label().to_string()), + description: description.into_owned(), + }, + ); + } +} + +impl Recorder for LanceRecorder { + fn describe_counter(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Counter, unit, description); + } + + fn describe_gauge(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Gauge, unit, description); + } + + fn describe_histogram(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Histogram, unit, description); + } + + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { + self.registry + .get_or_create_counter(key, |counter| Counter::from_arc(counter.clone())) + } + + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { + self.registry + .get_or_create_gauge(key, |gauge| Gauge::from_arc(gauge.clone())) + } + + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { + self.registry + .get_or_create_histogram(key, |histogram| Histogram::from_arc(histogram.clone())) + } +} + +fn register_bounds() { + let mut bounds = HISTOGRAM_BOUNDS.write().unwrap(); + for (name, values) in lance_io::object_store::metrics::histogram_bounds() { + bounds.insert((*name).to_string(), Arc::from(*values)); + } +} + +fn describe_all() { + lance_io::object_store::metrics::describe_metrics(); +} + +enum MetricValue { + Scalar(f64), + Histogram { + buckets: Vec<(String, u64)>, + count: u64, + sum: f64, + }, +} + +struct MetricPoint { + name: String, + kind: &'static str, + attributes: HashMap, + value: MetricValue, +} + +fn labels(key: &Key) -> HashMap { + key.labels() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect() +} + +fn collect_points(registry: &Registry) -> Vec { + let mut points = Vec::new(); + for (key, handle) in registry.get_counter_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "counter", + attributes: labels(&key), + value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64), + }); + } + for (key, handle) in registry.get_gauge_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "gauge", + attributes: labels(&key), + value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))), + }); + } + for (key, handle) in registry.get_histogram_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "histogram", + attributes: labels(&key), + value: handle.snapshot(), + }); + } + points +} + +fn register_lance_metrics_recorder() -> bool { + if REGISTRY.get().is_some() { + return true; + } + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + register_bounds(); + match metrics::set_global_recorder(recorder) { + Ok(()) => { + let _ = REGISTRY.set(registry); + describe_all(); + true + } + Err(_) => false, + } +} + +fn object_list<'local>(env: &mut JNIEnv<'local>) -> Result> { + Ok(env.new_object("java/util/ArrayList", "()V", &[])?) +} + +fn add_to_list(env: &mut JNIEnv, list: &JObject, item: &JObject) -> Result<()> { + env.call_method( + list, + "add", + "(Ljava/lang/Object;)Z", + &[JValue::Object(item)], + )?; + Ok(()) +} + +fn string_object<'local>(env: &mut JNIEnv<'local>, value: Option<&str>) -> Result> { + match value { + Some(value) => Ok(env.new_string(value)?.into()), + None => Ok(JObject::null()), + } +} + +fn attributes_to_java<'local>( + env: &mut JNIEnv<'local>, + attributes: &HashMap, +) -> Result> { + let java_map = env.new_object("java/util/HashMap", "()V", &[])?; + for (key, value) in attributes { + let java_key = env.new_string(key)?; + let java_value = env.new_string(value)?; + env.call_method( + &java_map, + "put", + "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;", + &[JValue::Object(&java_key), JValue::Object(&java_value)], + )?; + env.delete_local_ref(java_key)?; + env.delete_local_ref(java_value)?; + } + Ok(java_map) +} + +fn buckets_to_java<'local>( + env: &mut JNIEnv<'local>, + buckets: Option<&[(String, u64)]>, +) -> Result> { + let Some(buckets) = buckets else { + return Ok(JObject::null()); + }; + let list = object_list(env)?; + for (le, cumulative_count) in buckets { + let le = env.new_string(le)?; + let bucket = env.new_object( + "org/lance/otel/MetricBucket", + "(Ljava/lang/String;J)V", + &[JValue::Object(&le), JValue::Long(*cumulative_count as i64)], + )?; + add_to_list(env, &list, &bucket)?; + env.delete_local_ref(le)?; + env.delete_local_ref(bucket)?; + } + Ok(list) +} + +fn boxed_double<'local>(env: &mut JNIEnv<'local>, value: Option) -> Result> { + match value { + Some(value) => Ok(env.new_object("java/lang/Double", "(D)V", &[JValue::Double(value)])?), + None => Ok(JObject::null()), + } +} + +fn boxed_long<'local>(env: &mut JNIEnv<'local>, value: Option) -> Result> { + match value { + Some(value) => { + Ok(env.new_object("java/lang/Long", "(J)V", &[JValue::Long(value as i64)])?) + } + None => Ok(JObject::null()), + } +} + +fn metric_description_to_java<'local>( + env: &mut JNIEnv<'local>, + name: &str, + desc: &MetricDescription, +) -> Result> { + let name = env.new_string(name)?; + let kind = env.new_string(desc.kind.as_str())?; + let unit = string_object(env, desc.unit.as_deref())?; + let description = env.new_string(&desc.description)?; + Ok(env.new_object( + "org/lance/otel/MetricDescription", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;)V", + &[ + JValue::Object(&name), + JValue::Object(&kind), + JValue::Object(&unit), + JValue::Object(&description), + ], + )?) +} + +fn metric_point_to_java<'local>( + env: &mut JNIEnv<'local>, + point: MetricPoint, +) -> Result> { + let (value, buckets, count, sum) = match point.value { + MetricValue::Scalar(value) => (Some(value), None, None, None), + MetricValue::Histogram { + buckets, + count, + sum, + } => (None, Some(buckets), Some(count), Some(sum)), + }; + let name = env.new_string(&point.name)?; + let kind = env.new_string(point.kind)?; + let attributes = attributes_to_java(env, &point.attributes)?; + let value = boxed_double(env, value)?; + let buckets = buckets_to_java(env, buckets.as_deref())?; + let count = boxed_long(env, count)?; + let sum = boxed_double(env, sum)?; + Ok(env.new_object( + "org/lance/otel/MetricPoint", + "(Ljava/lang/String;Ljava/lang/String;Ljava/util/Map;Ljava/lang/Double;Ljava/util/List;Ljava/lang/Long;Ljava/lang/Double;)V", + &[ + JValue::Object(&name), + JValue::Object(&kind), + JValue::Object(&attributes), + JValue::Object(&value), + JValue::Object(&buckets), + JValue::Object(&count), + JValue::Object(&sum), + ], + )?) +} + +fn lance_metrics_catalog_native<'local>(env: &mut JNIEnv<'local>) -> Result> { + let catalog = CATALOG.lock().unwrap(); + let list = object_list(env)?; + for (name, desc) in catalog.iter() { + env.with_local_frame(16, |env| { + let item = metric_description_to_java(env, name, desc)?; + add_to_list(env, &list, &item) + })?; + } + Ok(list) +} + +fn snapshot_lance_metrics_native<'local>(env: &mut JNIEnv<'local>) -> Result> { + let list = object_list(env)?; + if let Some(registry) = REGISTRY.get() { + for point in collect_points(registry) { + env.with_local_frame(64, |env| { + let item = metric_point_to_java(env, point)?; + add_to_list(env, &list, &item) + })?; + } + } + Ok(list) +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_otel_LanceMetrics_registerLanceMetricsRecorderNative( + _env: JNIEnv, + _class: JClass, +) -> jboolean { + register_lance_metrics_recorder() as jboolean +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_otel_LanceMetrics_lanceMetricsCatalogNative( + mut env: JNIEnv, + _class: JClass, +) -> jobject { + ok_or_throw_with_return!( + env, + lance_metrics_catalog_native(&mut env), + std::ptr::null_mut() + ) + .into_raw() +} + +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_otel_LanceMetrics_snapshotLanceMetricsNative( + mut env: JNIEnv, + _class: JClass, +) -> jobject { + ok_or_throw_with_return!( + env, + snapshot_lance_metrics_native(&mut env), + std::ptr::null_mut() + ) + .into_raw() +} + +#[cfg(test)] +mod tests { + use super::*; + use metrics::HistogramFn; + + fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 { + buckets + .iter() + .find(|(bound, _)| bound == le) + .map(|(_, count)| *count) + .unwrap_or_else(|| panic!("no bucket with le={le}")) + } + + #[test] + fn test_bucketed_histogram_records_cumulative_buckets() { + let histogram = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice())); + histogram.record(0.05); + histogram.record(0.1); + histogram.record(0.5); + histogram.record(1.0); + histogram.record(5.0); + histogram.record(50.0); + + let MetricValue::Histogram { + buckets, + count, + sum, + } = histogram.snapshot() + else { + panic!("expected histogram"); + }; + + assert_eq!(bucket_count(&buckets, "0.1"), 2); + assert_eq!(bucket_count(&buckets, "1"), 4); + assert_eq!(bucket_count(&buckets, "10"), 5); + assert_eq!(bucket_count(&buckets, "+Inf"), 6); + assert_eq!(bucket_count(&buckets, "+Inf"), count); + assert!((sum - 56.65).abs() < 1e-9); + } + + #[test] + fn test_registry_aggregates_counters_and_gauges() { + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::counter!("test_counter", "operation" => "get").increment(2); + metrics::counter!("test_counter", "operation" => "get").increment(3); + metrics::gauge!("test_gauge", "operation" => "get").increment(3.5); + metrics::gauge!("test_gauge", "operation" => "get").decrement(1.25); + }); + + let points = collect_points(®istry); + let counter = points + .iter() + .find(|point| point.name == "test_counter") + .expect("counter recorded"); + let gauge = points + .iter() + .find(|point| point.name == "test_gauge") + .expect("gauge recorded"); + + assert!(matches!(counter.value, MetricValue::Scalar(value) if value == 5.0)); + assert!(matches!(gauge.value, MetricValue::Scalar(value) if (value - 2.25).abs() < 1e-9)); + } +} diff --git a/java/lance-jni/src/session.rs b/java/lance-jni/src/session.rs index 17b59c12ecb..6afae56f43a 100644 --- a/java/lance-jni/src/session.rs +++ b/java/lance-jni/src/session.rs @@ -4,13 +4,15 @@ use std::sync::Arc; use jni::JNIEnv; -use jni::objects::JObject; +use jni::objects::{JMap, JObject, JString, JValue}; use jni::sys::jlong; -use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE}; -use lance::session::Session as LanceSession; +use lance::session::{CacheSpec, Session as LanceSession}; +use lance_core::cache::{BackendConfig, build_from_config, build_from_uri}; use lance_io::object_store::ObjectStoreRegistry; +use crate::block_on; use crate::error::{Error, Result}; +use crate::utils::to_rust_map; /// Creates a new Session and returns a handle to it. /// @@ -22,33 +24,64 @@ pub extern "system" fn Java_org_lance_Session_createNative( _obj: JObject, index_cache_size_bytes: jlong, metadata_cache_size_bytes: jlong, + index_cache_backend_uri: JString, + index_cache_backend_kind: JString, + index_cache_backend_options: JObject, + metadata_cache_backend_uri: JString, + metadata_cache_backend_kind: JString, + metadata_cache_backend_options: JObject, ) -> jlong { ok_or_throw_with_return!( env, - create_session(index_cache_size_bytes, metadata_cache_size_bytes), + create_session( + &mut env, + index_cache_size_bytes, + metadata_cache_size_bytes, + index_cache_backend_uri, + index_cache_backend_kind, + index_cache_backend_options, + metadata_cache_backend_uri, + metadata_cache_backend_kind, + metadata_cache_backend_options, + ), 0 ) } +#[allow(clippy::too_many_arguments)] fn create_session( + env: &mut JNIEnv, index_cache_size_bytes: jlong, metadata_cache_size_bytes: jlong, + index_cache_backend_uri: JString, + index_cache_backend_kind: JString, + index_cache_backend_options: JObject, + metadata_cache_backend_uri: JString, + metadata_cache_backend_kind: JString, + metadata_cache_backend_options: JObject, ) -> Result { - let index_cache_size = if index_cache_size_bytes >= 0 { - index_cache_size_bytes as usize - } else { - DEFAULT_INDEX_CACHE_SIZE - }; - - let metadata_cache_size = if metadata_cache_size_bytes >= 0 { - metadata_cache_size_bytes as usize - } else { - DEFAULT_METADATA_CACHE_SIZE - }; + let index_cache = resolve_cache_spec( + env, + "indexCacheBackend", + "indexCacheSizeBytes", + index_cache_size_bytes, + index_cache_backend_uri, + index_cache_backend_kind, + index_cache_backend_options, + )?; + let metadata_cache = resolve_cache_spec( + env, + "metadataCacheBackend", + "metadataCacheSizeBytes", + metadata_cache_size_bytes, + metadata_cache_backend_uri, + metadata_cache_backend_kind, + metadata_cache_backend_options, + )?; - let session = LanceSession::new( - index_cache_size, - metadata_cache_size, + let session = LanceSession::with_cache_backends( + index_cache, + metadata_cache, Arc::new(ObjectStoreRegistry::default()), ); @@ -58,6 +91,63 @@ fn create_session( Ok(handle) } +#[allow(clippy::too_many_arguments)] +fn resolve_cache_spec( + env: &mut JNIEnv, + backend_field: &str, + size_field: &str, + size_bytes: jlong, + backend_uri: JString, + backend_kind: JString, + backend_options: JObject, +) -> Result { + let has_uri = !backend_uri.is_null(); + let has_kind = !backend_kind.is_null(); + if has_uri && has_kind { + return Err(Error::input_error(format!( + "{} must use either a URI or a structured config, not both", + backend_field + ))); + } + if size_bytes >= 0 && (has_uri || has_kind) { + return Err(Error::input_error(format!( + "{} and {} are mutually exclusive; set one or the other", + size_field, backend_field + ))); + } + + if has_uri { + let uri: String = env.get_string(&backend_uri)?.into(); + return build_from_uri(&uri) + .map(CacheSpec::Backend) + .map_err(Error::from); + } + + if has_kind { + let kind: String = env.get_string(&backend_kind)?.into(); + let mut config = BackendConfig::new(&kind)?; + if !backend_options.is_null() { + let options = JMap::from_env(env, &backend_options)?; + config.options = to_rust_map(env, &options)?; + } + return build_from_config(&config) + .map(CacheSpec::Backend) + .map_err(Error::from); + } + + if size_bytes >= 0 { + let size = usize::try_from(size_bytes).map_err(|_| { + Error::input_error(format!( + "{} value {} does not fit in usize", + size_field, size_bytes + )) + })?; + Ok(CacheSpec::Size(size)) + } else { + Ok(CacheSpec::Default) + } +} + /// Returns the current size of the session in bytes. #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_Session_sizeBytesNative( @@ -78,6 +168,41 @@ fn size_bytes_native(env: &mut JNIEnv, obj: JObject) -> Result { Ok(session_arc.size_bytes() as jlong) } +/// Returns statistics for the session's metadata cache as an org.lance.CacheStats object. +#[unsafe(no_mangle)] +pub extern "system" fn Java_org_lance_Session_metadataCacheStatsNative<'local>( + mut env: JNIEnv<'local>, + obj: JObject, +) -> JObject<'local> { + ok_or_throw!(env, metadata_cache_stats_native(&mut env, obj)) +} + +fn metadata_cache_stats_native<'local>( + env: &mut JNIEnv<'local>, + obj: JObject, +) -> Result> { + let handle = get_session_handle(env, &obj)?; + if handle == 0 { + return Err(Error::input_error("Session is closed".to_string())); + } + + // Safety: We trust that the handle is valid and was created by createNative + let session_arc = unsafe { &*(handle as *const Arc) }; + let stats = block_on(session_arc.metadata_cache_stats()); + + let stats_obj = env.new_object( + "org/lance/CacheStats", + "(JJJJ)V", + &[ + JValue::Long(stats.hits as jlong), + JValue::Long(stats.misses as jlong), + JValue::Long(stats.num_entries as jlong), + JValue::Long(stats.size_bytes as jlong), + ], + )?; + Ok(stats_obj) +} + /// Releases the native session handle. #[unsafe(no_mangle)] pub extern "system" fn Java_org_lance_Session_releaseNative( diff --git a/java/lance-jni/src/sql.rs b/java/lance-jni/src/sql.rs index f378577eedf..2111d02c9fd 100644 --- a/java/lance-jni/src/sql.rs +++ b/java/lance-jni/src/sql.rs @@ -4,7 +4,7 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::FromJString; -use crate::{Error, JNIEnvExt, RT}; +use crate::{Error, JNIEnvExt, RT, block_on}; use arrow::ffi_stream::FFI_ArrowArrayStream; use jni::JNIEnv; use jni::objects::{JClass, JObject, JString}; @@ -57,7 +57,7 @@ fn inner_into_batch_records( with_row_addr, )?; - let stream = RT.block_on(async move { + let stream = block_on(async move { let query = builder.build().await?; query.into_stream().await })?; diff --git a/java/lance-jni/src/transaction.rs b/java/lance-jni/src/transaction.rs index 4f899f56ff2..9d048f41c3f 100644 --- a/java/lance-jni/src/transaction.rs +++ b/java/lance-jni/src/transaction.rs @@ -3,7 +3,7 @@ use crate::Error; use crate::JNIEnvExt; -use crate::RT; +use crate::block_on; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET, extract_namespace_info}; use crate::error::Result; use crate::traits::{ @@ -19,14 +19,14 @@ use jni::sys::{jboolean, jint, jlong}; use lance::dataset::CommitBuilder; use lance::dataset::transaction::{ DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, TransactionBuilder, - UpdateMap, UpdateMapEntry, UpdateMode, + UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, }; use lance::io::ObjectStoreParams; use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore; use lance::table::format::{Fragment, IndexMetadata}; use lance_core::datatypes::Field; use lance_core::datatypes::Schema as LanceSchema; -use lance_file::version::LanceFileVersion; +use lance_file::version::{LanceFileVersion, V2_FORMAT_2_0, V2_FORMAT_2_1, V2_FORMAT_2_2}; use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider}; use lance_table::io::commit::CommitHandler; use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; @@ -167,6 +167,10 @@ impl FromJObjectWithEnv for JObject<'_> { let fields: Vec = import_vec_from_method(env, self, "fields", |env, field_id| { field_id.extract_object(env) })?; + let covering_fields: Vec = + import_vec_from_method(env, self, "coveringFields", |env, field_id| { + field_id.extract_object(env) + })?; let name = env.get_string_from_method(self, "name")?; let dataset_version = env.get_field(self, "datasetVersion", "J")?.j()? as u64; @@ -207,6 +211,7 @@ impl FromJObjectWithEnv for JObject<'_> { Ok(IndexMetadata { uuid, fields, + covering_fields, name, dataset_version, fragment_bitmap, @@ -411,6 +416,7 @@ fn convert_to_java_operation_inner<'local>( Operation::CreateIndex { new_indices, removed_indices, + .. } => { let java_new_indices = export_vec(env, &new_indices)?; let java_removed_indices = export_vec(env, &removed_indices)?; @@ -429,11 +435,11 @@ fn convert_to_java_operation_inner<'local>( updated_fragments, new_fragments, fields_modified, - merged_generations: _, + compacted_sstables: _, fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter: _, - updated_fragment_offsets: _, + updated_fragment_offsets, } => { let removed_ids: Vec> = removed_fragment_ids .iter() @@ -457,9 +463,48 @@ fn convert_to_java_operation_inner<'local>( &[JValue::Object(&update_mode)], )? .l()?; + // Serialize updated_fragment_offsets to Java Map. + // Values are portable RoaringBitmap bytes so the JNI boundary stays O(bitmap size) + // rather than O(n rows). Empty HashMap when None so the Java constructor always + // receives a non-null map. + let java_offsets_map = { + let java_map = env.new_object("java/util/HashMap", "()V", &[])?; + if let Some(UpdatedFragmentOffsets(ref map)) = updated_fragment_offsets { + for (frag_id, bitmap) in map { + let mut buf: Vec = Vec::new(); + bitmap.serialize_into(&mut buf).map_err(|e| { + Error::runtime_error(format!( + "failed to serialize updatedFragmentOffsets for fragment \ + {frag_id}: {e}" + )) + })?; + // JNI byte arrays are signed i8; reinterpret without copying. + let buf_i8: &[i8] = unsafe { + std::slice::from_raw_parts(buf.as_ptr() as *const i8, buf.len()) + }; + env.with_local_frame(4, |env| { + let java_key = env.new_object( + "java/lang/Long", + "(J)V", + &[JValue::Long(*frag_id as i64)], + )?; + let java_arr = env.new_byte_array(buf_i8.len() as i32)?; + env.set_byte_array_region(&java_arr, 0, buf_i8)?; + env.call_method( + &java_map, + "put", + "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;", + &[JValue::Object(&java_key), JValue::Object(&*java_arr)], + )?; + Ok::(JObject::null()) + })?; + } + } + java_map + }; Ok(env.new_object( "org/lance/operation/Update", - "(Ljava/util/List;Ljava/util/List;Ljava/util/List;[J[JLjava/util/Optional;)V", + "(Ljava/util/List;Ljava/util/List;Ljava/util/List;[J[JLjava/util/Optional;Ljava/util/Map;)V", &[ JValue::Object(&removed_fragment_ids_obj), JValue::Object(&updated_fragments_obj), @@ -467,16 +512,23 @@ fn convert_to_java_operation_inner<'local>( JValueGen::Object(&fields_modified), JValueGen::Object(&fields_for_preserving_frag_bitmap), JValue::Object(&update_mode_optional), + JValue::Object(&java_offsets_map), ], )?) } - Operation::Project { schema } => { + Operation::Project { + schema, + preserves_nullability, + } => { let java_schema = convert_to_java_schema(env, schema)?; Ok(env.new_object( "org/lance/operation/Project", - "(Lorg/apache/arrow/vector/types/pojo/Schema;)V", - &[JValue::Object(&java_schema)], + "(Lorg/apache/arrow/vector/types/pojo/Schema;Z)V", + &[ + JValue::Object(&java_schema), + JValue::Bool(preserves_nullability as u8), + ], )?) } Operation::Rewrite { @@ -552,16 +604,18 @@ fn convert_to_java_operation_inner<'local>( Operation::Merge { fragments: rust_fragments, schema, + preserves_nullability, } => { let java_fragments = export_vec(env, &rust_fragments)?; let java_schema = convert_to_java_schema(env, schema)?; Ok(env.new_object( "org/lance/operation/Merge", - "(Ljava/util/List;Lorg/apache/arrow/vector/types/pojo/Schema;)V", + "(Ljava/util/List;Lorg/apache/arrow/vector/types/pojo/Schema;Z)V", &[ JValue::Object(&java_fragments), JValue::Object(&java_schema), + JValue::Bool(preserves_nullability as u8), ], )?) } @@ -594,19 +648,38 @@ pub(crate) fn convert_to_java_schema<'local>( .l()?) } +/// Parse a `CommitBuilder.storageFormat` string into a [`LanceFileVersion`]. +/// +/// The canonical spellings ("2.1", "stable", ...) are the ones every other Lance +/// binding accepts and the ones [`LanceFileVersion`]'s `Display` emits. +/// +/// The `v`-prefixed spellings are a Java-only accident: this function originally +/// hand-rolled its match by walking the `LanceFileVersion` variant identifiers +/// (`V2_1` -> `"v2_1"`) instead of delegating to `FromStr`, so it accepted those +/// identifiers and rejected the canonical "2.1". They were documented on +/// `CommitBuilder.storageFormat` and shipped from 3.0.0, so they are translated +/// here for compatibility. The set is deliberately frozen to what shipped — +/// newer versions are reachable only by their canonical name. fn parse_storage_format(name: &str) -> Result { - match name.to_lowercase().as_str() { - "legacy" => Ok(LanceFileVersion::Legacy), - "v2_0" | "v2.0" => Ok(LanceFileVersion::V2_0), - "stable" => Ok(LanceFileVersion::Stable), - "v2_1" | "v2.1" => Ok(LanceFileVersion::V2_1), - "next" => Ok(LanceFileVersion::Next), - "v2_2" | "v2.2" => Ok(LanceFileVersion::V2_2), - _ => Err(Error::input_error(format!( - "Unknown storage format: {}", - name - ))), + let requested = name.to_lowercase(); + let canonical = match requested.as_str() { + "v2_0" | "v2.0" => V2_FORMAT_2_0, + "v2_1" | "v2.1" => V2_FORMAT_2_1, + "v2_2" | "v2.2" => V2_FORMAT_2_2, + _ => requested.as_str(), + }; + + if canonical != requested { + log::warn!( + "Storage format \"{}\" is deprecated and will be removed in a future release; use \"{}\" instead", + name, + canonical + ); } + + canonical + .parse::() + .map_err(|_| Error::input_error(format!("Unknown storage format: {}", name))) } /// Translate the Java `commitTimeoutNanos` sentinel into an @@ -1025,6 +1098,8 @@ fn convert_to_rust_operation( let op_name = env.get_string_from_method(java_operation, "name")?; let op = match op_name.as_str() { "Project" => Operation::Project { + preserves_nullability: env + .get_boolean_from_method(java_operation, "preservesNullability")?, schema: convert_schema_from_operation( env, java_operation, @@ -1238,16 +1313,69 @@ fn convert_to_rust_operation( update_mode.extract_object(env) })?; + let updated_fragment_offsets = { + let offsets_obj = env + .call_method( + java_operation, + "updatedFragmentOffsets", + "()Ljava/util/Map;", + &[], + )? + .l()?; + if offsets_obj.is_null() { + None + } else { + let jmap = JMap::from_env(env, &offsets_obj)?; + let mut iter = jmap.iter(env)?; + let mut offsets: HashMap = HashMap::new(); + // Per-iteration local frame: iterator key/value JNI refs are released each + // loop so large multi-fragment maps cannot exhaust the local reference table. + loop { + let entry = env.with_local_frame( + 8, + |env| -> Result> { + let Some((key, value)) = iter.next(env)? else { + return Ok(None); + }; + let frag_id = + env.call_method(&key, "longValue", "()J", &[])?.j()? as u64; + let buf: Vec = + env.convert_byte_array(JByteArray::from(value))?; + let bitmap = RoaringBitmap::deserialize_from(buf.as_slice()) + .map_err(|e| { + Error::input_error(format!( + "invalid updatedFragmentOffsets RoaringBitmap bytes \ + for fragment {frag_id}: {e}" + )) + })?; + Ok(Some((frag_id, bitmap))) + }, + )?; + match entry { + None => break, + Some((frag_id, bitmap)) => { + offsets.insert(frag_id, bitmap); + } + } + } + if offsets.is_empty() { + None + } else { + Some(UpdatedFragmentOffsets(offsets)) + } + } + }; + Operation::Update { removed_fragment_ids, updated_fragments, new_fragments, fields_modified, - merged_generations: vec![], + compacted_sstables: vec![], fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter: None, - updated_fragment_offsets: None, + updated_fragment_offsets, } } "DataReplacement" => { @@ -1264,6 +1392,8 @@ fn convert_to_rust_operation( })?; Operation::Merge { fragments, + preserves_nullability: env + .get_boolean_from_method(java_operation, "preservesNullability")?, schema: convert_schema_from_operation( env, java_operation, @@ -1574,7 +1704,7 @@ fn inner_commit_to_uri<'local>( builder = builder.with_commit_handler(commit_handler); } - let dataset = RT.block_on(builder.execute(transaction))?; + let dataset = block_on(builder.execute(transaction))?; let blocking_ds = BlockingDataset { inner: dataset }; blocking_ds.into_java(env) } @@ -1836,4 +1966,80 @@ mod tests { HashMap::from([("new_schema_k".to_string(), "new_schema_v".to_string())]) ); } + + #[test] + fn test_parse_storage_format_canonical_forms() { + let cases = [ + ("2.0", LanceFileVersion::V2_0), + ("2.1", LanceFileVersion::V2_1), + ("2.2", LanceFileVersion::V2_2), + ("2.3", LanceFileVersion::V2_3), + ("0.1", LanceFileVersion::Legacy), + ("legacy", LanceFileVersion::Legacy), + ("stable", LanceFileVersion::Stable), + ("next", LanceFileVersion::Next), + ]; + for (input, expected) in cases { + assert_eq!( + parse_storage_format(input).unwrap(), + expected, + "parse_storage_format({:?}) failed", + input + ); + } + } + + /// The `v`-prefixed spellings shipped in the `CommitBuilder.storageFormat` + /// Javadoc and must keep working for existing Java callers. + #[test] + fn test_parse_storage_format_deprecated_aliases() { + let cases = [ + ("v2_0", LanceFileVersion::V2_0), + ("v2.0", LanceFileVersion::V2_0), + ("v2_1", LanceFileVersion::V2_1), + ("v2.1", LanceFileVersion::V2_1), + ("v2_2", LanceFileVersion::V2_2), + ("v2.2", LanceFileVersion::V2_2), + ]; + for (input, expected) in cases { + assert_eq!( + parse_storage_format(input).unwrap(), + expected, + "parse_storage_format({:?}) failed", + input + ); + } + } + + /// The alias set is frozen to what shipped, so versions added after the + /// aliases were deprecated are reachable only by their canonical name. + #[test] + fn test_parse_storage_format_does_not_extend_aliases_to_new_versions() { + assert!(parse_storage_format("v2_3").is_err()); + assert!(parse_storage_format("v2.3").is_err()); + assert_eq!(parse_storage_format("2.3").unwrap(), LanceFileVersion::V2_3); + } + + #[test] + fn test_parse_storage_format_case_insensitive() { + assert_eq!( + parse_storage_format("LEGACY").unwrap(), + LanceFileVersion::Legacy + ); + assert_eq!( + parse_storage_format("Stable").unwrap(), + LanceFileVersion::Stable + ); + assert_eq!( + parse_storage_format("V2_1").unwrap(), + LanceFileVersion::V2_1 + ); + } + + #[test] + fn test_parse_storage_format_rejects_invalid() { + assert!(parse_storage_format("v3.0").is_err()); + assert!(parse_storage_format("").is_err()); + assert!(parse_storage_format("foo").is_err()); + } } diff --git a/java/lance-jni/src/update.rs b/java/lance-jni/src/update.rs index 3544ce9c9f2..3a6f9360062 100644 --- a/java/lance-jni/src/update.rs +++ b/java/lance-jni/src/update.rs @@ -5,7 +5,7 @@ use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::Result; use crate::traits::IntoJava; use crate::utils::to_rust_map; -use crate::{JNIEnvExt, RT}; +use crate::{JNIEnvExt, block_on}; use jni::JNIEnv; use jni::objects::{JMap, JObject, JValueGen}; use lance::dataset::UpdateBuilder; @@ -53,7 +53,7 @@ fn inner_update<'local>( } let job = builder.build()?; - let update_result = RT.block_on(job.execute())?; + let update_result = block_on(job.execute())?; // Avoid panicking if Lance core retains a clone of the Arc; fall back to a // deep clone so the JNI boundary stays panic-free. diff --git a/java/lance-jni/src/utils.rs b/java/lance-jni/src/utils.rs index 94372ef27cc..c9d2d2005b1 100644 --- a/java/lance-jni/src/utils.rs +++ b/java/lance-jni/src/utils.rs @@ -12,7 +12,7 @@ use lance::dataset::optimize::{CompactionMode, CompactionOptions}; use lance::dataset::{WriteMode, WriteParams}; use lance::index::vector::{IndexFileVersion, StageParams, VectorIndexParams}; use lance::io::ObjectStoreParams; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use lance_index::IndexParams; use lance_index::vector::bq::RQBuildParams; use lance_index::vector::hnsw::builder::HnswBuildParams; @@ -191,6 +191,9 @@ pub fn build_compaction_options( compaction_mode: &JObject, // Optional binary_copy_read_batch_bytes: &JObject, // Optional max_source_fragments: &JObject, // Optional + max_source_rows: &JObject, // Optional + max_source_bytes: &JObject, // Optional + excluded_fragment_ids: &JObject, // List config: &std::collections::HashMap, ) -> Result { let mut compaction_options = CompactionOptions::from_dataset_config(config)?; @@ -234,6 +237,25 @@ pub fn build_compaction_options( if let Some(max_source_fragments_val) = env.get_long_opt(max_source_fragments)? { compaction_options.max_source_fragments = Some(max_source_fragments_val as usize); } + if let Some(max_source_rows_val) = env.get_long_opt(max_source_rows)? { + compaction_options.max_source_rows = Some(max_source_rows_val as usize); + } + if let Some(max_source_bytes_val) = env.get_long_opt(max_source_bytes)? { + compaction_options.max_source_bytes = Some(max_source_bytes_val as u64); + } + compaction_options.excluded_fragment_ids = env + .get_longs(excluded_fragment_ids)? + .into_iter() + .map(|fragment_id| { + u32::try_from(fragment_id).map_err(|_| { + Error::input_error(format!( + "excluded_fragment_ids must contain values between 0 and {}, got {}", + u32::MAX, + fragment_id + )) + }) + }) + .collect::>>()?; Ok(compaction_options) } diff --git a/java/lance-jni/src/vector_trainer.rs b/java/lance-jni/src/vector_trainer.rs index 798b88baaff..d988aebd468 100755 --- a/java/lance-jni/src/vector_trainer.rs +++ b/java/lance-jni/src/vector_trainer.rs @@ -3,7 +3,7 @@ use std::sync::Arc; -use crate::RT; +use crate::block_on; use crate::blocking_dataset::{BlockingDataset, NATIVE_DATASET}; use crate::error::{Error, Result}; use crate::ffi::JNIEnvExt; @@ -122,7 +122,7 @@ fn inner_train_ivf_centroids<'local>( let dim = get_vector_dim(dataset.schema(), &column)?; - let ivf_model = RT.block_on(lance::index::vector::ivf::build_ivf_model( + let ivf_model = block_on(lance::index::vector::ivf::build_ivf_model( dataset, &column, dim, @@ -185,7 +185,7 @@ fn inner_train_pq_codebook<'local>( let dim = get_vector_dim(dataset.schema(), &column)?; - let pq = RT.block_on(lance::index::vector::pq::build_pq_model( + let pq = block_on(lance::index::vector::pq::build_pq_model( dataset, &column, dim, diff --git a/java/pom.xml b/java/pom.xml index 89b567c5f3e..5a89e1a4fb9 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -7,7 +7,7 @@ org.lance lance-core Lance Core - 9.0.0-beta.16 + 12.0.0-beta.11 jar Lance Format Java API @@ -29,6 +29,7 @@ UTF-8 18.3.0 + 1.64.0 0.28.1 false 2.43.0 @@ -39,6 +40,7 @@ 3.7.5 package false + false org.lance.shaded @@ -109,18 +111,23 @@ org.lance lance-namespace-core - 0.7.7 + 0.11.1 org.lance lance-namespace-apache-client - 0.7.7 + 0.11.1 com.fasterxml.jackson.core jackson-databind 2.15.2 + + io.opentelemetry + opentelemetry-api + ${opentelemetry.version} + software.amazon.awssdk @@ -134,6 +141,18 @@ 2.20.26 test + + io.opentelemetry + opentelemetry-sdk + ${opentelemetry.version} + test + + + io.opentelemetry + opentelemetry-sdk-testing + ${opentelemetry.version} + test + @@ -396,6 +415,9 @@ lance-jni ${rust.release.build} + + ${rust.features} + ${project.build.directory}/classes/nativelib true @@ -409,6 +431,9 @@ lance-jni ${rust.release.build} + + ${rust.features} + -v diff --git a/java/src/main/java/org/lance/CacheBackendConfig.java b/java/src/main/java/org/lance/CacheBackendConfig.java new file mode 100644 index 00000000000..b43a49e9404 --- /dev/null +++ b/java/src/main/java/org/lance/CacheBackendConfig.java @@ -0,0 +1,113 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +import org.apache.arrow.util.Preconditions; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * Structured configuration for a cache backend registered with the native Lance runtime. + * + *

The {@code kind} selects a registered backend constructor, while {@code options} contains + * backend-specific string settings. For example: + * + * 

{@code
+ * CacheBackendConfig config = CacheBackendConfig.builder("moka")
+ *     .option("capacity", "1048576")
+ *     .build();
+ * }
+ * + *

Third-party native backend crates register their constructors with Lance at application + * startup. This class selects and configures one of those registered constructors; it does not + * register a Java implementation as a native cache backend. + */ +public final class CacheBackendConfig { + private final String kind; + private final Map options; + + private CacheBackendConfig(Builder builder) { + this.kind = builder.kind; + this.options = Collections.unmodifiableMap(new HashMap<>(builder.options)); + } + + /** + * Creates a builder for a registered backend kind. + * + * @param kind registered backend identifier, such as {@code moka} + * @return a new builder + */ + public static Builder builder(String kind) { + return new Builder(kind); + } + + /** Returns the registered backend identifier. */ + public String getKind() { + return kind; + } + + /** Returns an immutable map of backend-specific options. */ + public Map getOptions() { + return options; + } + + /** Builder for {@link CacheBackendConfig}. */ + public static final class Builder { + private final String kind; + private final Map options = new HashMap<>(); + + private Builder(String kind) { + Preconditions.checkNotNull(kind, "kind must not be null"); + Preconditions.checkArgument(!kind.isEmpty(), "kind must not be empty"); + this.kind = kind; + } + + /** + * Adds a backend-specific option. + * + * @param key option name + * @param value option value + * @return this builder + */ + public Builder option(String key, String value) { + Preconditions.checkNotNull(key, "cache backend option key must not be null"); + Preconditions.checkNotNull(value, "cache backend option value must not be null"); + Preconditions.checkArgument(!key.isEmpty(), "cache backend option key must not be empty"); + options.put(key, value); + return this; + } + + /** + * Replaces the current backend options. + * + * @param options backend-specific options + * @return this builder + */ + public Builder options(Map options) { + Preconditions.checkNotNull(options, "options must not be null"); + this.options.clear(); + for (Map.Entry option : options.entrySet()) { + option(option.getKey(), option.getValue()); + } + return this; + } + + /** Builds the immutable backend configuration. */ + public CacheBackendConfig build() { + return new CacheBackendConfig(this); + } + } +} diff --git a/java/src/main/java/org/lance/CacheStats.java b/java/src/main/java/org/lance/CacheStats.java new file mode 100644 index 00000000000..9df3388dabb --- /dev/null +++ b/java/src/main/java/org/lance/CacheStats.java @@ -0,0 +1,105 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** + * Statistics for a session cache. + * + *

A snapshot of cache activity, useful for monitoring hit/miss rates over time. + */ +public class CacheStats { + private final long hits; + private final long misses; + private final long numEntries; + private final long sizeBytes; + + /** + * Constructs cache statistics. Instances are created by the native layer. + * + * @param hits number of cache lookups that found an item + * @param misses number of cache lookups that did not find an item + * @param numEntries number of entries currently in the cache + * @param sizeBytes total size in bytes of all entries in the cache + */ + public CacheStats(long hits, long misses, long numEntries, long sizeBytes) { + if (hits < 0 || misses < 0 || numEntries < 0 || sizeBytes < 0) { + throw new IllegalArgumentException( + String.format( + "Cache statistics must be non-negative: " + + "hits=%d, misses=%d, numEntries=%d, sizeBytes=%d", + hits, misses, numEntries, sizeBytes)); + } + this.hits = hits; + this.misses = misses; + this.numEntries = numEntries; + this.sizeBytes = sizeBytes; + } + + /** + * Returns the number of cache lookups that found an item in the cache. + * + * @return the number of cache hits + */ + public long getHits() { + return hits; + } + + /** + * Returns the number of cache lookups that did not find an item in the cache. + * + * @return the number of cache misses + */ + public long getMisses() { + return misses; + } + + /** + * Returns the number of entries currently in the cache. + * + * @return the number of cache entries + */ + public long getNumEntries() { + return numEntries; + } + + /** + * Returns the total size in bytes of all entries in the cache. + * + * @return the cache size in bytes + */ + public long getSizeBytes() { + return sizeBytes; + } + + /** + * Returns the ratio of hits to total lookups, or 0 if there have been no lookups. + * + * @return the cache hit ratio in the range [0, 1] + */ + public double getHitRatio() { + // Sum in double to avoid long overflow for very large counters + double total = (double) hits + (double) misses; + if (total == 0) { + return 0.0; + } + return hits / total; + } + + @Override + public String toString() { + return String.format( + "CacheStats(hits=%d, misses=%d, numEntries=%d, sizeBytes=%d)", + hits, misses, numEntries, sizeBytes); + } +} diff --git a/java/src/main/java/org/lance/CommitBuilder.java b/java/src/main/java/org/lance/CommitBuilder.java index 62861b7ef73..b0e29d45ac2 100644 --- a/java/src/main/java/org/lance/CommitBuilder.java +++ b/java/src/main/java/org/lance/CommitBuilder.java @@ -200,8 +200,14 @@ public CommitBuilder useStableRowIds(boolean useStableRowIds) { * Set the storage format to use for the dataset. * *

This is only needed when creating a new empty table. If any data files are passed, the - * storage format will be inferred from the data files. Valid values: "legacy", "v2_0", "stable", - * "v2_1", "next", "v2_2". + * storage format will be inferred from the data files. Valid values are the numeric versions + * ("0.1", "2.0", "2.1", "2.2", "2.3") and the release selectors ("legacy", "stable", "next"), + * matching {@link WriteParams.Builder#withDataStorageVersion(String)}. Parsing is + * case-insensitive. + * + *

The {@code v}-prefixed spellings ("v2_0", "v2.0", "v2_1", "v2.1", "v2_2", "v2.2") are + * deprecated. They were accepted only by this method, never by the rest of Lance, and will be + * removed in a future release — use the numeric version instead ("v2_1" becomes "2.1"). * * @param storageFormat the storage format name * @return this builder instance @@ -273,23 +279,25 @@ public CommitBuilder commitTimeout(Duration timeout) { public Dataset execute(Transaction transaction) { Preconditions.checkNotNull(transaction, "Transaction must not be null"); if (dataset != null) { - Dataset result = - nativeCommitToDataset( - dataset, - transaction, - detached, - enableV2ManifestPaths, - writeParams, - useStableRowIds, - storageFormat, - maxRetries, - skipAutoCleanup, - namespaceClient, - tableId, - namespaceClientManagedVersioning, - commitTimeoutNanos); - result.setAllocator(dataset.allocator()); - return result; + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + Dataset result = + nativeCommitToDataset( + dataset, + transaction, + detached, + enableV2ManifestPaths, + writeParams, + useStableRowIds, + storageFormat, + maxRetries, + skipAutoCleanup, + namespaceClient, + tableId, + namespaceClientManagedVersioning, + commitTimeoutNanos); + result.setAllocator(dataset.allocator()); + return result; + } } if (uri != null) { Dataset result = diff --git a/java/src/main/java/org/lance/Dataset.java b/java/src/main/java/org/lance/Dataset.java index 8f79d7cbaea..65b084e0dea 100644 --- a/java/src/main/java/org/lance/Dataset.java +++ b/java/src/main/java/org/lance/Dataset.java @@ -19,6 +19,7 @@ import org.lance.compaction.CompactionOptions; import org.lance.delta.DatasetDelta; import org.lance.index.Index; +import org.lance.index.IndexBuildProgress; import org.lance.index.IndexCriteria; import org.lance.index.IndexDescription; import org.lance.index.IndexOptions; @@ -54,6 +55,7 @@ import org.apache.arrow.vector.ipc.ArrowReader; import org.apache.arrow.vector.ipc.ArrowStreamReader; import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; import org.apache.arrow.vector.types.pojo.Schema; import java.io.ByteArrayInputStream; @@ -62,6 +64,7 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -480,6 +483,47 @@ private static native Dataset openNative( List tableId, boolean namespaceClientManagedVersioning); + /** + * List manifest locations without reading or deserializing the manifest contents. + * + *

The returned locations are not guaranteed to be ordered. This operation may list and + * materialize the full manifest history. + * + *

This method is for datasets whose committed manifests can be listed authoritatively from the + * object store. Namespace-managed tables, external version stores such as {@code s3+ddb}, and + * tables using a custom commit handler are not supported. + * + * @param uri dataset URI + * @return manifest locations + */ + public static List listManifestLocations(String uri) { + return listManifestLocations(uri, new HashMap<>()); + } + + /** + * List manifest locations without reading or deserializing the manifest contents. + * + *

The returned locations are not guaranteed to be ordered. This operation may list and + * materialize the full manifest history. + * + *

This method is for datasets whose committed manifests can be listed authoritatively from the + * object store. Namespace-managed tables, external version stores such as {@code s3+ddb}, and + * tables using a custom commit handler are not supported. + * + * @param uri dataset URI + * @param storageOptions object-store credentials and connection options + * @return manifest locations + */ + public static List listManifestLocations( + String uri, Map storageOptions) { + Preconditions.checkNotNull(uri, "uri must not be null"); + Preconditions.checkNotNull(storageOptions, "storageOptions must not be null"); + return listManifestLocationsNative(uri, storageOptions); + } + + private static native List listManifestLocationsNative( + String uri, Map storageOptions); + /** * Creates a builder for opening a dataset. * @@ -621,10 +665,19 @@ public Dataset commitTransaction( } /** - * Drop a Dataset. + * Drop a Dataset, deleting everything under {@code path} recursively. + * + *

To limit the damage a mistyped or misconfigured path can do, {@code path} must be a dataset + * root, meaning it holds a manifest that can be read, or a namespace declare/deregister marker. + * Anything else throws {@link IllegalArgumentException}, including a path that holds only data + * files or only unreadable manifests: such leftovers need an explicit storage-level delete. + * + *

Note that a path which passes this check is deleted in full, including any unmanaged files + * kept next to the dataset. * * @param path The file path of the dataset * @param storageOptions Storage options + * @throws IllegalArgumentException if {@code path} is not a Lance dataset root */ public static native void drop(String path, Map storageOptions); @@ -732,11 +785,31 @@ public void dropColumns(List columns) { public void alterColumns(List columnAlterations) { try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) { Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); - nativeAlterColumns(columnAlterations); + // Cast target types are carried across the FFI boundary through the Arrow C Data + // Interface rather than ArrowType#toString(), which does not round-trip reliably on + // the native side (parameterized types such as Int(64, true) fail to parse and the + // cast would otherwise be silently dropped). One field is exported per alteration that + // requests a type change, in the same order as {@code columnAlterations}. + List castFields = new ArrayList<>(); + int castIndex = 0; + for (ColumnAlteration alteration : columnAlterations) { + if (alteration.getDataType().isPresent()) { + castFields.add(new Field("f" + castIndex++, castFieldType(alteration), null)); + } + } + try (ArrowSchema castSchema = ArrowSchema.allocateNew(allocator)) { + Data.exportSchema(allocator, new Schema(castFields), null, castSchema); + nativeAlterColumns(columnAlterations, castSchema.memoryAddress()); + } } } - private native void nativeAlterColumns(List columnAlterations); + private static FieldType castFieldType(ColumnAlteration alteration) { + boolean nullable = alteration.getNullable().orElse(true); + return new FieldType(nullable, alteration.getDataType().get(), null); + } + + private native void nativeAlterColumns(List columnAlterations, long castAddr); /** * Create a new Dataset Scanner. @@ -955,6 +1028,23 @@ public List listVersions() { private native List nativeListVersions(); + /** + * Get the number of versions in the current version history. + * + *

Unlike {@link #listVersions()}, this method does not read or deserialize every manifest. + * Detached versions are not included. + * + * @return the number of versions + */ + public long getVersionCount() { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); + return nativeGetVersionCount(); + } + } + + private native long nativeGetVersionCount(); + /** * @return the latest version of the dataset. */ @@ -1024,13 +1114,7 @@ public Dataset checkoutVersion(long version) { Preconditions.checkArgument(version > 0, "version number must be greater than 0"); try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); - Dataset newDataset = nativeCheckoutVersion(version); - if (selfManagedAllocator) { - newDataset.allocator = new RootAllocator(Long.MAX_VALUE); - } else { - newDataset.allocator = allocator; - } - return newDataset; + return initializeCheckoutDataset(nativeCheckoutVersion(version)); } } @@ -1047,18 +1131,23 @@ public Dataset checkoutTag(String tag) { Preconditions.checkArgument(tag != null, "Tag can not be null"); try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); - Dataset newDataset = nativeCheckoutTag(tag); - if (selfManagedAllocator) { - newDataset.allocator = new RootAllocator(Long.MAX_VALUE); - } else { - newDataset.allocator = allocator; - } - return newDataset; + return initializeCheckoutDataset(nativeCheckoutTag(tag)); } } private native Dataset nativeCheckoutTag(String tag); + private Dataset initializeCheckoutDataset(Dataset checkedOutDataset) { + if (selfManagedAllocator) { + checkedOutDataset.allocator = new RootAllocator(Long.MAX_VALUE); + } else { + checkedOutDataset.allocator = allocator; + } + checkedOutDataset.session = Session.fromHandle(checkedOutDataset.nativeGetSessionHandle()); + checkedOutDataset.ownsSession = true; + return checkedOutDataset; + } + /** * Restore the currently checked out version of the dataset as the latest version. This operation * produces a new version and doesn't influence any old versions and tags. @@ -1153,6 +1242,29 @@ public void mergeIndexMetadata( private native void innerMergeIndexMetadata( String indexUUID, int indexType, Optional batchReadHead); + /** + * Merge distributed index metadata while reporting stage-level progress. + * + * @param indexUUID shared UUID used by the distributed index parts + * @param indexType type of index metadata to merge + * @param batchReadHead optional limit for metadata read concurrency + * @param progress thread-safe progress callback + */ + public void mergeIndexMetadata( + String indexUUID, + IndexType indexType, + Optional batchReadHead, + IndexBuildProgress progress) { + Preconditions.checkNotNull(progress, "progress cannot be null"); + innerMergeIndexMetadataWithProgress(indexUUID, indexType.getValue(), batchReadHead, progress); + } + + private native void innerMergeIndexMetadataWithProgress( + String indexUUID, + int indexType, + Optional batchReadHead, + IndexBuildProgress progress); + /** Merge one caller-defined group of existing uncommitted vector index segments. */ public Index mergeExistingIndexSegments(List segments) { Preconditions.checkNotNull(segments, "segments cannot be null"); @@ -1303,6 +1415,25 @@ public List getFragments() { private native List getFragmentsNative(); + /** + * Get per-fragment statistics for all fragments in this dataset version. + * + *

Unlike {@link #getFragments()}, this is a metadata-only bulk operation: no per-fragment Java + * objects are materialized, and native code fills the returned primitive arrays directly. This + * makes it suitable for planning over datasets with a very large number of fragments. Row counts + * match {@link FragmentMetadata#getNumRows()} (physical rows minus deleted rows). + * + * @return per-fragment statistics as parallel arrays, in manifest order + */ + public FragmentStatistics getFragmentStatistics() { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); + return nativeGetFragmentStatistics(); + } + } + + private native FragmentStatistics nativeGetFragmentStatistics(); + /** * Gets the arrow schema of the dataset. * @@ -1491,6 +1622,22 @@ public boolean hasStableRowIds() { private native boolean nativeHasStableRowIds(); + /** + * Get the library version that wrote the current manifest. + * + *

Older manifests may not contain writer version metadata. + * + * @return the current manifest writer version, or empty if unavailable + */ + public Optional getWriterVersion() { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); + return Optional.ofNullable(nativeGetWriterVersion()); + } + } + + private native WriterVersion nativeGetWriterVersion(); + /** * Get the Lance file format version of this dataset. * @@ -1586,6 +1733,26 @@ private void updateToNewDataset(Dataset newDataset) { newDataset.nativeDatasetHandle = 0; } + /** + * Acquires a shared read lock that pins the native dataset handle, blocking a concurrent {@link + * #close()} until the lock is released. + * + *

Any code that passes this {@link Dataset} into a native method must hold this lock for the + * whole native call; otherwise {@code close()} can release the native dataset mid-call and crash + * the JVM. The lock is reentrant and intended for try-with-resources use. + * + * @return the acquired read lock + * @throws IllegalArgumentException if the dataset is already closed + */ + public LockManager.ReadLock acquireReadLock() { + LockManager.ReadLock readLock = lockManager.acquireReadLock(); + if (nativeDatasetHandle == 0) { + readLock.close(); + throw new IllegalArgumentException("Dataset is closed"); + } + return readLock; + } + /** * Closes this dataset and releases any system resources associated with it. If the dataset is * already closed, then invoking this method has no effect. @@ -1621,11 +1788,27 @@ public void close() { private native List nativeTakeBlobsByIndices(List rowIndices, String column); /** - * Open blob files for given row ids on a blob column. Names and semantics align with Rust/Python. + * Open {@link BlobFile} handles for given row IDs on a blob column. Names and semantics align + * with Rust/Python. + * + *

Pass logical row IDs read from {@code _rowid}, not physical row addresses from {@code + * _rowaddr}. * - * @param rowIds stable row ids (row addresses) + * 

{@code
+   * long rowId = 42L; // Example value from the _rowid column.
+   * List blobs = dataset.takeBlobs(List.of(rowId), "images");
+   * for (BlobFile blob : blobs) {
+   *   if (blob != null) {
+   *     try (BlobFile file = blob) {
+   *       byte[] data = file.read();
+   *     }
+   *   }
+   * }
+   * }
+ * + * @param rowIds logical row IDs from the {@code _rowid} column * @param column blob column name - * @return list of BlobFile objects + * @return one {@link BlobFile} per row ID; null blob values are represented by null elements */ public List takeBlobs(List rowIds, String column) { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { @@ -1639,11 +1822,22 @@ public List takeBlobs(List rowIds, String column) { } /** - * Open blob files for given row indices on a blob column. + * Open {@link BlobFile} handles for given row indices on a blob column. + * + * 
{@code
+   * List blobs = dataset.takeBlobsByIndices(List.of(0L), "images");
+   * for (BlobFile blob : blobs) {
+   *   if (blob != null) {
+   *     try (BlobFile file = blob) {
+   *       byte[] data = file.read();
+   *     }
+   *   }
+   * }
+   * }
* * @param rowIndices row offsets within dataset * @param column blob column name - * @return list of BlobFile objects + * @return one {@link BlobFile} per row index; null blob values are represented by null elements */ public List takeBlobsByIndices(List rowIndices, String column) { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { @@ -1738,13 +1932,7 @@ public Dataset checkout(Ref ref) { Preconditions.checkNotNull(ref); try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { Preconditions.checkArgument(nativeDatasetHandle != 0, "Dataset is closed"); - Dataset newDataset = nativeCheckout(ref); - if (selfManagedAllocator) { - newDataset.allocator = new RootAllocator(Long.MAX_VALUE); - } else { - newDataset.allocator = allocator; - } - return newDataset; + return initializeCheckoutDataset(nativeCheckout(ref)); } } diff --git a/java/src/main/java/org/lance/DocumentGranularity.java b/java/src/main/java/org/lance/DocumentGranularity.java new file mode 100644 index 00000000000..c540cc1ff9c --- /dev/null +++ b/java/src/main/java/org/lance/DocumentGranularity.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** The unit treated as one full-text-search document. */ +public enum DocumentGranularity { + /** All text selected from one dataset row belongs to one document. */ + ROW("row"), + + /** Each element of the deepest list on the field path is one document. */ + LIST_ELEMENT("list_element"); + + private final String rustString; + + DocumentGranularity(String rustString) { + this.rustString = rustString; + } + + /** Return the stable value understood by the Rust API and serialized index parameters. */ + public String toRustString() { + return rustString; + } +} diff --git a/java/src/main/java/org/lance/Fragment.java b/java/src/main/java/org/lance/Fragment.java index 3b12e158617..dfb4652cf58 100644 --- a/java/src/main/java/org/lance/Fragment.java +++ b/java/src/main/java/org/lance/Fragment.java @@ -113,7 +113,9 @@ public LanceScanner newScan(ScanOptions options) { * returns a new fragment with the updated deletion vector. */ public FragmentMetadata deleteRows(List rowIndexes) { - return nativeDeleteRows(dataset, fragmentMetadata.getId(), rowIndexes); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeDeleteRows(dataset, fragmentMetadata.getId(), rowIndexes); + } } private static native FragmentMetadata nativeDeleteRows( @@ -129,7 +131,9 @@ public int getId() { * @return row counts in this Fragment */ public int countRows() { - return countRowsNative(dataset, fragmentMetadata.getId()); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return countRowsNative(dataset, fragmentMetadata.getId()); + } } /** @@ -153,8 +157,10 @@ public int countRows() { * @return the fragment metadata and new schema. */ public FragmentMergeResult mergeColumns(ArrowArrayStream stream, String leftOn, String rightOn) { - return nativeMergeColumns( - dataset, fragmentMetadata.getId(), stream.memoryAddress(), leftOn, rightOn); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeMergeColumns( + dataset, fragmentMetadata.getId(), stream.memoryAddress(), leftOn, rightOn); + } } private native FragmentMergeResult nativeMergeColumns( @@ -186,8 +192,10 @@ private native FragmentMergeResult nativeMergeColumns( */ public FragmentUpdateResult updateColumns( ArrowArrayStream stream, String leftOn, String rightOn) { - return nativeUpdateColumns( - dataset, fragmentMetadata.getId(), stream.memoryAddress(), leftOn, rightOn); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeUpdateColumns( + dataset, fragmentMetadata.getId(), stream.memoryAddress(), leftOn, rightOn); + } } public FragmentUpdateResult updateColumns(ArrowArrayStream stream) { @@ -261,7 +269,7 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId) { - return create(datasetUri, allocator, root, params, namespaceClient, tableId, null); + return create(datasetUri, allocator, root, params, namespaceClient, tableId, null, null); } /** Create a fragment from the given arrow array and schema. */ @@ -272,11 +280,13 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId, - LanceSchema schema) { + LanceSchema schema, + Session session) { Preconditions.checkNotNull(datasetUri); Preconditions.checkNotNull(allocator); Preconditions.checkNotNull(root); Preconditions.checkNotNull(params); + long sessionHandle = getSessionHandle(session); try (ArrowSchema arrowSchema = ArrowSchema.allocateNew(allocator); ArrowArray arrowArray = ArrowArray.allocateNew(allocator)) { Data.exportVectorSchemaRoot(allocator, root, null, arrowArray, arrowSchema); @@ -301,7 +311,8 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - lanceSchema.memoryAddress()); + lanceSchema.memoryAddress(), + sessionHandle); } } return createWithFfiArray( @@ -322,7 +333,8 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - 0L); + 0L, + sessionHandle); } } @@ -333,7 +345,7 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId) { - return create(datasetUri, null, stream, params, namespaceClient, tableId, null); + return create(datasetUri, null, stream, params, namespaceClient, tableId, null, null); } /** Create a fragment from the given arrow stream. */ @@ -344,10 +356,12 @@ static List create( WriteParams params, LanceNamespace namespaceClient, List tableId, - LanceSchema schema) { + LanceSchema schema, + Session session) { Preconditions.checkNotNull(datasetUri); Preconditions.checkNotNull(stream); Preconditions.checkNotNull(params); + long sessionHandle = getSessionHandle(session); if (schema != null) { Preconditions.checkNotNull(allocator, "allocator is required with schema"); try (ArrowSchema lanceSchema = ArrowSchema.allocateNew(allocator)) { @@ -369,7 +383,8 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - lanceSchema.memoryAddress()); + lanceSchema.memoryAddress(), + sessionHandle); } } return createWithFfiStream( @@ -389,7 +404,16 @@ static List create( tableId, params.getAllowExternalBlobOutsideBases(), params.getBlobPackFileSizeThreshold(), - 0L); + 0L, + sessionHandle); + } + + /** + * Resolves the native handle of an optional session. A closed session has a zero handle and is + * treated as absent, matching how Dataset handles closed sessions. + */ + private static long getSessionHandle(Session session) { + return session == null ? 0L : session.getNativeHandle(); } /** Create a fragment from the given arrow array and schema. */ @@ -411,7 +435,8 @@ private static native List createWithFfiArray( List tableId, Optional allowExternalBlobOutsideBases, Optional blobPackFileSizeThreshold, - long schemaMemoryAddress); + long schemaMemoryAddress, + long sessionHandle); /** Create a fragment from the given arrow stream. */ private static native List createWithFfiStream( @@ -431,5 +456,6 @@ private static native List createWithFfiStream( List tableId, Optional allowExternalBlobOutsideBases, Optional blobPackFileSizeThreshold, - long schemaMemoryAddress); + long schemaMemoryAddress, + long sessionHandle); } diff --git a/java/src/main/java/org/lance/FragmentStatistics.java b/java/src/main/java/org/lance/FragmentStatistics.java new file mode 100644 index 00000000000..8e4cd53bc54 --- /dev/null +++ b/java/src/main/java/org/lance/FragmentStatistics.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** + * Per-fragment statistics of a dataset version as parallel primitive arrays: {@code ids[i]}, {@code + * rowCounts[i]} and {@code dataFileNums[i]} describe the same fragment. Returned by {@link + * Dataset#getFragmentStatistics()} as a lightweight alternative to materializing full {@link + * Fragment} objects. + */ +public final class FragmentStatistics { + private final int[] ids; + private final long[] rowCounts; + private final int[] dataFileNums; + + FragmentStatistics(int[] ids, long[] rowCounts, int[] dataFileNums) { + this.ids = ids; + this.rowCounts = rowCounts; + this.dataFileNums = dataFileNums; + } + + /** Fragment IDs in manifest order. */ + public int[] getIds() { + return ids; + } + + /** + * Row count per fragment, aligned with {@link #getIds()}. Matches {@link + * FragmentMetadata#getNumRows()}: physical rows minus deleted rows. + */ + public long[] getRowCounts() { + return rowCounts; + } + + /** Number of data files per fragment, aligned with {@link #getIds()}. */ + public int[] getDataFileNums() { + return dataFileNums; + } + + /** Number of fragments described. */ + public int size() { + return ids.length; + } +} diff --git a/java/src/main/java/org/lance/LanceException.java b/java/src/main/java/org/lance/LanceException.java new file mode 100644 index 00000000000..0b5c180e319 --- /dev/null +++ b/java/src/main/java/org/lance/LanceException.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** + * Thrown when a Lance operation fails and its API cannot expose a checked exception. + * + *

For example, scanner I/O failures can be handled separately from programming errors: + * + * 

{@code
+ * try (ArrowReader reader = scanner.scanBatches()) {
+ *   // Consume batches.
+ * } catch (LanceException e) {
+ *   // Handle the failed Lance operation.
+ * }
+ * }
+ */ +public class LanceException extends RuntimeException { + /** + * Creates an exception with a message describing the failed operation. + * + * @param message description of the failure + */ + public LanceException(String message) { + super(message); + } + + /** + * Creates an exception with a message and its underlying cause. + * + * @param message description of the failure + * @param cause underlying cause of the failure + */ + public LanceException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/java/src/main/java/org/lance/ManifestLocation.java b/java/src/main/java/org/lance/ManifestLocation.java new file mode 100644 index 00000000000..37a5273abd0 --- /dev/null +++ b/java/src/main/java/org/lance/ManifestLocation.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +import java.util.Optional; + +/** Metadata describing the location of a dataset manifest. */ +public final class ManifestLocation { + private final long version; + private final String path; + private final long sizeBytes; + private final ManifestNamingScheme namingScheme; + private final String eTag; + + ManifestLocation(long version, String path, long sizeBytes, String namingScheme, String eTag) { + this.version = version; + this.path = path; + this.sizeBytes = sizeBytes; + this.namingScheme = ManifestNamingScheme.valueOf(namingScheme); + this.eTag = eTag; + } + + /** Dataset version represented by the manifest. */ + public long getVersion() { + return version; + } + + /** Manifest path relative to the object-store namespace or root. */ + public String getPath() { + return path; + } + + /** Manifest object size in bytes. */ + public long getSizeBytes() { + return sizeBytes; + } + + /** Naming scheme used by the manifest path. */ + public ManifestNamingScheme getNamingScheme() { + return namingScheme; + } + + /** Object-store entity tag, when available. */ + public Optional getETag() { + return Optional.ofNullable(eTag); + } +} diff --git a/java/src/main/java/org/lance/ManifestNamingScheme.java b/java/src/main/java/org/lance/ManifestNamingScheme.java new file mode 100644 index 00000000000..ec6b371af63 --- /dev/null +++ b/java/src/main/java/org/lance/ManifestNamingScheme.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +/** Naming scheme used by a Lance manifest path. */ +public enum ManifestNamingScheme { + /** Manifest names based directly on the dataset version. */ + V1, + /** Zero-padded, inverted version names optimized for latest-version lookup. */ + V2 +} diff --git a/java/src/main/java/org/lance/Session.java b/java/src/main/java/org/lance/Session.java index 0fe4a59736c..95a92cc8b57 100644 --- a/java/src/main/java/org/lance/Session.java +++ b/java/src/main/java/org/lance/Session.java @@ -16,6 +16,8 @@ import org.apache.arrow.util.Preconditions; import java.io.Closeable; +import java.util.Collections; +import java.util.Map; /** * A user session that holds runtime state for Lance datasets. @@ -35,6 +37,14 @@ * .metadataCacheSizeBytes(512L * 1024 * 1024) // 512 MiB * .build(); * + * // Select registered cache backends using URIs or structured configuration + * Session backendSession = Session.builder() + * .indexCacheBackend("moka://?capacity=1073741824") + * .metadataCacheBackend(CacheBackendConfig.builder("moka") + * .option("capacity", "268435456") + * .build()) + * .build(); + * * // Open multiple datasets with shared session * Dataset ds1 = Dataset.open() * .uri("s3://bucket/table1.lance") @@ -110,8 +120,12 @@ public static Session create(long indexCacheSizeBytes, long metadataCacheSizeByt /** Builder for creating Session instances with custom configuration. */ public static class Builder { - private long indexCacheSizeBytes = DEFAULT_INDEX_CACHE_SIZE_BYTES; - private long metadataCacheSizeBytes = DEFAULT_METADATA_CACHE_SIZE_BYTES; + private Long indexCacheSizeBytes; + private Long metadataCacheSizeBytes; + private String indexCacheBackendUri; + private CacheBackendConfig indexCacheBackendConfig; + private String metadataCacheBackendUri; + private CacheBackendConfig metadataCacheBackendConfig; private Builder() {} @@ -127,6 +141,37 @@ public Builder indexCacheSizeBytes(long indexCacheSizeBytes) { return this; } + /** + * Selects a registered index cache backend using a backend URI. + * + *

For example, {@code moka://?capacity=1048576}. This option is mutually exclusive with + * {@link #indexCacheSizeBytes(long)}. + * + * @param backendUri backend URI whose scheme identifies the registered backend + * @return this builder instance + */ + public Builder indexCacheBackend(String backendUri) { + Preconditions.checkNotNull(backendUri, "backendUri must not be null"); + this.indexCacheBackendUri = backendUri; + this.indexCacheBackendConfig = null; + return this; + } + + /** + * Selects a registered index cache backend using structured configuration. + * + *

This option is mutually exclusive with {@link #indexCacheSizeBytes(long)}. + * + * @param backendConfig backend kind and backend-specific options + * @return this builder instance + */ + public Builder indexCacheBackend(CacheBackendConfig backendConfig) { + Preconditions.checkNotNull(backendConfig, "backendConfig must not be null"); + this.indexCacheBackendConfig = backendConfig; + this.indexCacheBackendUri = null; + return this; + } + /** * Sets the size of the metadata cache in bytes. * @@ -140,15 +185,76 @@ public Builder metadataCacheSizeBytes(long metadataCacheSizeBytes) { return this; } + /** + * Selects a registered metadata cache backend using a backend URI. + * + *

For example, {@code moka://?capacity=1048576}. This option is mutually exclusive with + * {@link #metadataCacheSizeBytes(long)}. + * + * @param backendUri backend URI whose scheme identifies the registered backend + * @return this builder instance + */ + public Builder metadataCacheBackend(String backendUri) { + Preconditions.checkNotNull(backendUri, "backendUri must not be null"); + this.metadataCacheBackendUri = backendUri; + this.metadataCacheBackendConfig = null; + return this; + } + + /** + * Selects a registered metadata cache backend using structured configuration. + * + *

This option is mutually exclusive with {@link #metadataCacheSizeBytes(long)}. + * + * @param backendConfig backend kind and backend-specific options + * @return this builder instance + */ + public Builder metadataCacheBackend(CacheBackendConfig backendConfig) { + Preconditions.checkNotNull(backendConfig, "backendConfig must not be null"); + this.metadataCacheBackendConfig = backendConfig; + this.metadataCacheBackendUri = null; + return this; + } + /** * Builds the Session with the configured settings. * * @return a new Session instance */ public Session build() { - long handle = createNative(indexCacheSizeBytes, metadataCacheSizeBytes); + validateCacheConfiguration(); + long handle = + createNative( + indexCacheSizeBytes == null ? -1 : indexCacheSizeBytes, + metadataCacheSizeBytes == null ? -1 : metadataCacheSizeBytes, + indexCacheBackendUri, + backendKind(indexCacheBackendConfig), + backendOptions(indexCacheBackendConfig), + metadataCacheBackendUri, + backendKind(metadataCacheBackendConfig), + backendOptions(metadataCacheBackendConfig)); return new Session(handle); } + + private void validateCacheConfiguration() { + Preconditions.checkArgument( + indexCacheSizeBytes == null + || (indexCacheBackendUri == null && indexCacheBackendConfig == null), + "indexCacheSizeBytes and indexCacheBackend are mutually exclusive; set one or the other"); + Preconditions.checkArgument( + metadataCacheSizeBytes == null + || (metadataCacheBackendUri == null && metadataCacheBackendConfig == null), + "metadataCacheSizeBytes and metadataCacheBackend are mutually exclusive; " + + "set one or the other"); + } + + private static String backendKind(CacheBackendConfig config) { + return config == null ? null : config.getKind(); + } + + private static Map backendOptions(CacheBackendConfig config) { + return config == null ? Collections.emptyMap() : config.getOptions(); + } } /** @@ -176,6 +282,19 @@ public long sizeBytes() { return sizeBytesNative(); } + /** + * Returns statistics for the metadata cache of this session. + * + *

The returned statistics are a snapshot; call this method periodically to observe how hits + * and misses evolve over time. + * + * @return the metadata cache statistics + */ + public CacheStats metadataCacheStats() { + Preconditions.checkArgument(nativeSessionHandle != 0, "Session is closed"); + return metadataCacheStatsNative(); + } + /** * Returns whether the other session is the same as this one. * @@ -238,10 +357,20 @@ public String toString() { return String.format("Session(sizeBytes=%d)", sizeBytes()); } - private static native long createNative(long indexCacheSizeBytes, long metadataCacheSizeBytes); + private static native long createNative( + long indexCacheSizeBytes, + long metadataCacheSizeBytes, + String indexCacheBackendUri, + String indexCacheBackendKind, + Map indexCacheBackendOptions, + String metadataCacheBackendUri, + String metadataCacheBackendKind, + Map metadataCacheBackendOptions); private native long sizeBytesNative(); + private native CacheStats metadataCacheStatsNative(); + private static native void releaseNative(long handle); private static native boolean isSameAsNative(long handle1, long handle2); diff --git a/java/src/main/java/org/lance/SqlQuery.java b/java/src/main/java/org/lance/SqlQuery.java index cce6d939222..cb149a7fa6f 100644 --- a/java/src/main/java/org/lance/SqlQuery.java +++ b/java/src/main/java/org/lance/SqlQuery.java @@ -51,7 +51,8 @@ public SqlQuery withRowAddr(boolean withAddr) { } public ArrowReader intoBatchRecords() throws IOException { - try (ArrowArrayStream s = ArrowArrayStream.allocateNew(dataset.allocator())) { + try (LockManager.ReadLock readLock = dataset.acquireReadLock(); + ArrowArrayStream s = ArrowArrayStream.allocateNew(dataset.allocator())) { intoBatchRecords( dataset, sql, Optional.ofNullable(table), withRowId, withRowAddr, s.memoryAddress()); return Data.importArrayStream(dataset.allocator(), s); diff --git a/java/src/main/java/org/lance/WriteDatasetBuilder.java b/java/src/main/java/org/lance/WriteDatasetBuilder.java index 96a894fc944..9d406406291 100644 --- a/java/src/main/java/org/lance/WriteDatasetBuilder.java +++ b/java/src/main/java/org/lance/WriteDatasetBuilder.java @@ -70,6 +70,7 @@ public class WriteDatasetBuilder { private WriteParams.WriteMode mode = WriteParams.WriteMode.CREATE; private Schema schema; private Map storageOptions = new HashMap<>(); + private Map properties = new HashMap<>(); private Map> baseStoreParams = new HashMap<>(); private boolean ignoreNamespaceStorageOptions = false; private Optional maxRowsPerFile = Optional.empty(); @@ -207,6 +208,27 @@ public WriteDatasetBuilder storageOptions(Map storageOptions) { return this; } + /** + * Sets the table properties to forward to the namespace on table creation. + * + *

These are Lance-namespace properties: catalog-level key-value metadata stored by the + * namespace outside the Lance table (available even if the table manifest does not exist), as + * distinct from the manifest-stored {@code config} (read/write behavior) and {@code metadata} + * (business metadata), and from non-persisted {@code storageOptions}. They are attached to the + * underlying declareTable request via its {@code properties} field. + * + *

Only used when a namespace client is configured via namespaceClient()+tableId() and the + * write creates the table (CREATE mode). Ignored for direct-URI writes and for APPEND/OVERWRITE + * modes. + * + * @param properties Table properties to forward on declareTable + * @return this builder instance + */ + public WriteDatasetBuilder properties(Map properties) { + this.properties = new HashMap<>(properties); + return this; + } + /** * Sets runtime-only object store parameters for registered base paths. * @@ -410,6 +432,9 @@ private Dataset executeWithNamespaceClient() { if (mode == WriteParams.WriteMode.CREATE) { DeclareTableRequest declareRequest = new DeclareTableRequest(); declareRequest.setId(tableId); + if (properties != null && !properties.isEmpty()) { + declareRequest.setProperties(properties); + } DeclareTableResponse declareResponse = namespaceClient.declareTable(declareRequest); tableUri = declareResponse.getLocation(); diff --git a/java/src/main/java/org/lance/WriteFragmentBuilder.java b/java/src/main/java/org/lance/WriteFragmentBuilder.java index 2dbef873849..6e4a6b007df 100644 --- a/java/src/main/java/org/lance/WriteFragmentBuilder.java +++ b/java/src/main/java/org/lance/WriteFragmentBuilder.java @@ -51,6 +51,7 @@ public class WriteFragmentBuilder { private WriteParams.Builder writeParamsBuilder; private LanceNamespace namespaceClient; private List tableId; + private Session session; WriteFragmentBuilder() {} @@ -185,6 +186,19 @@ public WriteFragmentBuilder tableId(List tableId) { return this; } + /** + * Set a session to reuse across operations. + * + *

The session holds shared caches (metadata and index) and the object store registry. + * + * @param session the session to share + * @return this builder + */ + public WriteFragmentBuilder session(Session session) { + this.session = session; + return this; + } + /** * Set the maximum number of rows per file. * @@ -302,7 +316,8 @@ public List execute() { finalWriteParams, namespaceClient, tableId, - schema); + schema, + session); } else { return Fragment.create( datasetUri, @@ -311,7 +326,8 @@ public List execute() { finalWriteParams, namespaceClient, tableId, - schema); + schema, + session); } } diff --git a/java/src/main/java/org/lance/WriterVersion.java b/java/src/main/java/org/lance/WriterVersion.java new file mode 100644 index 00000000000..7e4fda48cf1 --- /dev/null +++ b/java/src/main/java/org/lance/WriterVersion.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +import java.util.Optional; + +/** Version metadata for the library that wrote a dataset manifest. */ +public final class WriterVersion { + private final String library; + private final String version; + private final String prerelease; + private final String buildMetadata; + + WriterVersion(String library, String version, String prerelease, String buildMetadata) { + this.library = library; + this.version = version; + this.prerelease = prerelease; + this.buildMetadata = buildMetadata; + } + + /** Name of the writer library, such as {@code lance}. */ + public String getLibrary() { + return library; + } + + /** + * Version string reported by the writer library. + * + *

This value is opaque because writer libraries are not required to use semantic versioning. + * When a writer does use semantic versioning, newer writers store the core version here and + * expose prerelease and build metadata separately. + */ + public String getVersion() { + return version; + } + + /** Optional semantic-version prerelease component, when supplied by the writer. */ + public Optional getPrerelease() { + return Optional.ofNullable(prerelease); + } + + /** Optional semantic-version build metadata component, when supplied by the writer. */ + public Optional getBuildMetadata() { + return Optional.ofNullable(buildMetadata); + } +} diff --git a/java/src/main/java/org/lance/cleanup/CleanupPolicy.java b/java/src/main/java/org/lance/cleanup/CleanupPolicy.java index 6316f70f1a6..25da8763747 100644 --- a/java/src/main/java/org/lance/cleanup/CleanupPolicy.java +++ b/java/src/main/java/org/lance/cleanup/CleanupPolicy.java @@ -13,6 +13,9 @@ */ package org.lance.cleanup; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Optional; /** @@ -24,6 +27,7 @@ public class CleanupPolicy { private final Optional beforeTimestampMillis; private final Optional beforeVersion; + private final Optional> versions; private final Optional deleteUnverified; private final Optional errorIfTaggedOldVersions; private final Optional cleanReferencedBranches; @@ -32,12 +36,14 @@ public class CleanupPolicy { private CleanupPolicy( Optional beforeTimestampMillis, Optional beforeVersion, + Optional> versions, Optional deleteUnverified, Optional errorIfTaggedOldVersions, Optional cleanReferencedBranches, Optional deleteRateLimit) { this.beforeTimestampMillis = beforeTimestampMillis; this.beforeVersion = beforeVersion; + this.versions = versions; this.deleteUnverified = deleteUnverified; this.errorIfTaggedOldVersions = errorIfTaggedOldVersions; this.cleanReferencedBranches = cleanReferencedBranches; @@ -56,6 +62,10 @@ public Optional getBeforeVersion() { return beforeVersion; } + public Optional> getVersions() { + return versions; + } + public Optional getDeleteUnverified() { return deleteUnverified; } @@ -76,6 +86,7 @@ public Optional getDeleteRateLimit() { public static class Builder { private Optional beforeTimestampMillis = Optional.empty(); private Optional beforeVersion = Optional.empty(); + private Optional> versions = Optional.empty(); private Optional deleteUnverified = Optional.empty(); private Optional errorIfTaggedOldVersions = Optional.empty(); private Optional cleanReferencedBranches = Optional.empty(); @@ -95,6 +106,12 @@ public Builder withBeforeVersion(long beforeVersion) { return this; } + /** Set the exact dataset versions to clean. */ + public Builder withVersions(List versions) { + this.versions = Optional.of(Collections.unmodifiableList(new ArrayList<>(versions))); + return this; + } + /** If true, delete unverified data files even if they are recent. */ public Builder withDeleteUnverified(boolean deleteUnverified) { this.deleteUnverified = Optional.of(deleteUnverified); @@ -123,6 +140,7 @@ public CleanupPolicy build() { return new CleanupPolicy( beforeTimestampMillis, beforeVersion, + versions, deleteUnverified, errorIfTaggedOldVersions, cleanReferencedBranches, diff --git a/java/src/main/java/org/lance/compaction/Compaction.java b/java/src/main/java/org/lance/compaction/Compaction.java index 0ce7050900c..5f7b99c666d 100644 --- a/java/src/main/java/org/lance/compaction/Compaction.java +++ b/java/src/main/java/org/lance/compaction/Compaction.java @@ -15,6 +15,7 @@ import org.lance.Dataset; import org.lance.JniLoader; +import org.lance.LockManager; import com.google.common.base.Preconditions; @@ -32,19 +33,24 @@ public static CompactionPlan planCompaction( Preconditions.checkNotNull(dataset); Preconditions.checkNotNull(compactionOptions); - return nativePlanCompaction( - dataset, - compactionOptions.getTargetRowsPerFragment(), - compactionOptions.getMaxRowsPerGroup(), - compactionOptions.getMaxBytesPerFile(), - compactionOptions.getMaterializeDeletions(), - compactionOptions.getMaterializeDeletionsThreshold(), - compactionOptions.getNumThreads(), - compactionOptions.getBatchSize(), - compactionOptions.getDeferIndexRemap(), - compactionOptions.getCompactionMode(), - compactionOptions.getBinaryCopyReadBatchBytes(), - compactionOptions.getMaxSourceFragments()); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativePlanCompaction( + dataset, + compactionOptions.getTargetRowsPerFragment(), + compactionOptions.getMaxRowsPerGroup(), + compactionOptions.getMaxBytesPerFile(), + compactionOptions.getMaterializeDeletions(), + compactionOptions.getMaterializeDeletionsThreshold(), + compactionOptions.getNumThreads(), + compactionOptions.getBatchSize(), + compactionOptions.getDeferIndexRemap(), + compactionOptions.getCompactionMode(), + compactionOptions.getBinaryCopyReadBatchBytes(), + compactionOptions.getMaxSourceFragments(), + compactionOptions.getMaxSourceRows(), + compactionOptions.getMaxSourceBytes(), + compactionOptions.getExcludedFragmentIds()); + } } public static CompactionMetrics commitCompaction( @@ -65,10 +71,56 @@ public static CompactionMetrics commitCompaction( compactionOptions.getDeferIndexRemap(), compactionOptions.getCompactionMode(), compactionOptions.getBinaryCopyReadBatchBytes(), - compactionOptions.getMaxSourceFragments()); + compactionOptions.getMaxSourceFragments(), + compactionOptions.getMaxSourceRows(), + compactionOptions.getMaxSourceBytes(), + compactionOptions.getExcludedFragmentIds()); + } + + /** + * Java wrapper around the raw commit-compaction JNI call. It acquires the dataset read lock so + * the native call cannot race with {@link Dataset#close()}; keep the raw native method private so + * no caller can bypass this lock. + */ + public static CompactionMetrics nativeCommitCompaction( + Dataset dataset, + List rewriteResults, + Optional targetRowsPerFragment, + Optional maxRowsPerGroup, + Optional maxBytesPerFile, + Optional materializeDeletions, + Optional materializeDeletionsThreshold, + Optional numThreads, + Optional batchSize, + Optional deferIndexRemap, + Optional compactionMode, + Optional binaryCopyReadBatchBytes, + Optional maxSourceFragments, + Optional maxSourceRows, + Optional maxSourceBytes, + List excludedFragmentIds) { + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return commitCompactionNative( + dataset, + rewriteResults, + targetRowsPerFragment, + maxRowsPerGroup, + maxBytesPerFile, + materializeDeletions, + materializeDeletionsThreshold, + numThreads, + batchSize, + deferIndexRemap, + compactionMode, + binaryCopyReadBatchBytes, + maxSourceFragments, + maxSourceRows, + maxSourceBytes, + excludedFragmentIds); + } } - public static native CompactionMetrics nativeCommitCompaction( + private static native CompactionMetrics commitCompactionNative( Dataset dataset, List rewriteResults, Optional targetRowsPerFragment, @@ -81,7 +133,10 @@ public static native CompactionMetrics nativeCommitCompaction( Optional deferIndexRemap, Optional compactionMode, Optional binaryCopyReadBatchBytes, - Optional maxSourceFragments); + Optional maxSourceFragments, + Optional maxSourceRows, + Optional maxSourceBytes, + List excludedFragmentIds); private static native CompactionPlan nativePlanCompaction( Dataset dataset, @@ -95,5 +150,8 @@ private static native CompactionPlan nativePlanCompaction( Optional deferIndexRemap, Optional compactionMode, Optional binaryCopyReadBatchBytes, - Optional maxSourceFragments); + Optional maxSourceFragments, + Optional maxSourceRows, + Optional maxSourceBytes, + List excludedFragmentIds); } diff --git a/java/src/main/java/org/lance/compaction/CompactionOptions.java b/java/src/main/java/org/lance/compaction/CompactionOptions.java index 7c3d65ffc3f..df1fd8628a4 100644 --- a/java/src/main/java/org/lance/compaction/CompactionOptions.java +++ b/java/src/main/java/org/lance/compaction/CompactionOptions.java @@ -18,7 +18,11 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import java.io.OptionalDataException; import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -28,6 +32,10 @@ * default values. */ public class CompactionOptions implements Serializable { + // Pinned to the UID generated before maxSourceRows/maxSourceBytes were added, so that + // CompactionTask streams queued by older workers still deserialize during a rolling upgrade. + private static final long serialVersionUID = 3114922060085417942L; + // these fields are effectively final, but not marked as final for de/ser private Optional targetRowsPerFragment; private Optional maxRowsPerGroup; @@ -40,6 +48,9 @@ public class CompactionOptions implements Serializable { private Optional compactionMode; private Optional binaryCopyReadBatchBytes; private Optional maxSourceFragments; + private Optional maxSourceRows; + private Optional maxSourceBytes; + private List excludedFragmentIds; private CompactionOptions( Optional targetRowsPerFragment, @@ -52,7 +63,10 @@ private CompactionOptions( Optional deferIndexRemap, Optional compactionMode, Optional binaryCopyReadBatchBytes, - Optional maxSourceFragments) { + Optional maxSourceFragments, + Optional maxSourceRows, + Optional maxSourceBytes, + List excludedFragmentIds) { this.targetRowsPerFragment = targetRowsPerFragment; this.maxRowsPerGroup = maxRowsPerGroup; this.maxBytesPerFile = maxBytesPerFile; @@ -64,6 +78,9 @@ private CompactionOptions( this.compactionMode = compactionMode; this.binaryCopyReadBatchBytes = binaryCopyReadBatchBytes; this.maxSourceFragments = maxSourceFragments; + this.maxSourceRows = maxSourceRows; + this.maxSourceBytes = maxSourceBytes; + this.excludedFragmentIds = List.copyOf(excludedFragmentIds); } public Optional getDeferIndexRemap() { @@ -83,6 +100,18 @@ public Optional getMaxSourceFragments() { return maxSourceFragments; } + public Optional getMaxSourceRows() { + return maxSourceRows; + } + + public Optional getMaxSourceBytes() { + return maxSourceBytes; + } + + public List getExcludedFragmentIds() { + return excludedFragmentIds; + } + public Optional getMaterializeDeletions() { return materializeDeletions; } @@ -129,6 +158,9 @@ public String toString() { .add("compactionMode", compactionMode.orElse(null)) .add("binaryCopyReadBatchBytes", binaryCopyReadBatchBytes.orElse(null)) .add("maxSourceFragments", maxSourceFragments.orElse(null)) + .add("maxSourceRows", maxSourceRows.orElse(null)) + .add("maxSourceBytes", maxSourceBytes.orElse(null)) + .add("excludedFragmentIds", excludedFragmentIds) .toString(); } @@ -144,6 +176,9 @@ private void writeObject(ObjectOutputStream output) throws IOException { output.writeObject(compactionMode.map(CompactionMode::getValue).orElse(null)); output.writeObject(binaryCopyReadBatchBytes.orElse(null)); output.writeObject(maxSourceFragments.orElse(null)); + output.writeObject(maxSourceRows.orElse(null)); + output.writeObject(maxSourceBytes.orElse(null)); + output.writeObject(excludedFragmentIds); } private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException { @@ -167,6 +202,40 @@ private void readObject(ObjectInputStream input) throws IOException, ClassNotFou } this.binaryCopyReadBatchBytes = Optional.ofNullable((Long) input.readObject()); this.maxSourceFragments = Optional.ofNullable((Long) input.readObject()); + this.maxSourceRows = readTrailingLong(input); + this.maxSourceBytes = readTrailingLong(input); + this.excludedFragmentIds = readTrailingLongList(input); + } + + /** + * Reads a trailing Long field that older writers did not emit. Streams written before the field + * was added end here, which surfaces as an {@link OptionalDataException} with {@code eof} set; in + * that case the field is treated as unset. + */ + private static Optional readTrailingLong(ObjectInputStream input) + throws IOException, ClassNotFoundException { + try { + return Optional.ofNullable((Long) input.readObject()); + } catch (OptionalDataException e) { + if (!e.eof) { + throw e; + } + return Optional.empty(); + } + } + + @SuppressWarnings("unchecked") + private static List readTrailingLongList(ObjectInputStream input) + throws IOException, ClassNotFoundException { + try { + List fragmentIds = (List) input.readObject(); + return fragmentIds == null ? Collections.emptyList() : List.copyOf(fragmentIds); + } catch (OptionalDataException e) { + if (!e.eof) { + throw e; + } + return Collections.emptyList(); + } } /** Builder for CompactionOptions. */ @@ -182,6 +251,9 @@ public static class Builder { private Optional compactionMode = Optional.empty(); private Optional binaryCopyReadBatchBytes = Optional.empty(); private Optional maxSourceFragments = Optional.empty(); + private Optional maxSourceRows = Optional.empty(); + private Optional maxSourceBytes = Optional.empty(); + private List excludedFragmentIds = Collections.emptyList(); private Builder() {} @@ -239,12 +311,73 @@ public Builder withBinaryCopyReadBatchBytes(long binaryCopyReadBatchBytes) { * Maximum number of source fragments to compact in a single run. Tasks are included until * adding the next task would exceed this limit, allowing for incremental compaction. Fragments * are processed oldest first. + * + * @throws IllegalArgumentException if {@code maxSourceFragments} is not positive */ public Builder withMaxSourceFragments(long maxSourceFragments) { - this.maxSourceFragments = Optional.of(maxSourceFragments); + this.maxSourceFragments = + Optional.of(positiveBudget("maxSourceFragments", maxSourceFragments)); return this; } + /** + * Maximum number of source rows to compact in a single run. Rows are counted as live rows + * (physical rows minus soft-deleted rows). Tasks are included until adding the next task would + * exceed this limit. + * + * @throws IllegalArgumentException if {@code maxSourceRows} is not positive + */ + public Builder withMaxSourceRows(long maxSourceRows) { + this.maxSourceRows = Optional.of(positiveBudget("maxSourceRows", maxSourceRows)); + return this; + } + + /** + * Maximum number of source bytes to compact in a single run, measured as the total size of the + * source fragments' data and overlay files. Tasks are included until adding the next task would + * exceed this limit. Blob v2 payloads live in separate blob files and are not counted, so this + * is not a cap on total compaction I/O for datasets with blob columns. + * + * @throws IllegalArgumentException if {@code maxSourceBytes} is not positive + */ + public Builder withMaxSourceBytes(long maxSourceBytes) { + this.maxSourceBytes = Optional.of(positiveBudget("maxSourceBytes", maxSourceBytes)); + return this; + } + + /** + * Fragment IDs to exclude from compaction planning. Excluded fragments remain unchanged and act + * as boundaries, so fragments on opposite sides are not combined into the same task. Duplicate + * and unknown IDs are ignored. + * + * @throws IllegalArgumentException if an ID is negative or exceeds the unsigned 32-bit range + */ + public Builder withExcludedFragmentIds(List excludedFragmentIds) { + Objects.requireNonNull(excludedFragmentIds, "excludedFragmentIds"); + for (Long fragmentId : excludedFragmentIds) { + if (fragmentId == null || fragmentId < 0 || fragmentId > 0xFFFF_FFFFL) { + throw new IllegalArgumentException( + "excludedFragmentIds must contain values between 0 and 4294967295, got " + + fragmentId); + } + } + this.excludedFragmentIds = List.copyOf(excludedFragmentIds); + return this; + } + + /** + * A max source budget of zero admits no work and a negative value would wrap around to an + * effectively unlimited budget on the Rust side, so both are rejected here. Leave the option + * unset for no limit. + */ + private static long positiveBudget(String name, long value) { + if (value <= 0) { + throw new IllegalArgumentException( + name + " must be greater than 0, got " + value + " (leave unset for no limit)"); + } + return value; + } + public CompactionOptions build() { return new CompactionOptions( targetRowsPerFragment, @@ -257,7 +390,10 @@ public CompactionOptions build() { deferIndexRemap, compactionMode, binaryCopyReadBatchBytes, - maxSourceFragments); + maxSourceFragments, + maxSourceRows, + maxSourceBytes, + excludedFragmentIds); } } } diff --git a/java/src/main/java/org/lance/compaction/CompactionTask.java b/java/src/main/java/org/lance/compaction/CompactionTask.java index 89ec364e980..571d4fd503b 100644 --- a/java/src/main/java/org/lance/compaction/CompactionTask.java +++ b/java/src/main/java/org/lance/compaction/CompactionTask.java @@ -14,10 +14,12 @@ package org.lance.compaction; import org.lance.Dataset; +import org.lance.LockManager; import com.google.common.base.MoreObjects; import java.io.Serializable; +import java.util.List; import java.util.Optional; /** The compaction task which can be sent across network and executed individually. */ @@ -42,21 +44,26 @@ public String toString() { } public RewriteResult execute(Dataset dataset) { - return nativeExecute( - dataset, - taskData, - readVersion, - compactionOptions.getTargetRowsPerFragment(), - compactionOptions.getMaxRowsPerGroup(), - compactionOptions.getMaxBytesPerFile(), - compactionOptions.getMaterializeDeletions(), - compactionOptions.getMaterializeDeletionsThreshold(), - compactionOptions.getNumThreads(), - compactionOptions.getBatchSize(), - compactionOptions.getDeferIndexRemap(), - compactionOptions.getCompactionMode(), - compactionOptions.getBinaryCopyReadBatchBytes(), - compactionOptions.getMaxSourceFragments()); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeExecute( + dataset, + taskData, + readVersion, + compactionOptions.getTargetRowsPerFragment(), + compactionOptions.getMaxRowsPerGroup(), + compactionOptions.getMaxBytesPerFile(), + compactionOptions.getMaterializeDeletions(), + compactionOptions.getMaterializeDeletionsThreshold(), + compactionOptions.getNumThreads(), + compactionOptions.getBatchSize(), + compactionOptions.getDeferIndexRemap(), + compactionOptions.getCompactionMode(), + compactionOptions.getBinaryCopyReadBatchBytes(), + compactionOptions.getMaxSourceFragments(), + compactionOptions.getMaxSourceRows(), + compactionOptions.getMaxSourceBytes(), + compactionOptions.getExcludedFragmentIds()); + } } private native RewriteResult nativeExecute( @@ -73,7 +80,10 @@ private native RewriteResult nativeExecute( Optional deferIndexRemap, Optional compactionMode, Optional binaryCopyReadBatchBytes, - Optional maxSourceFragments); + Optional maxSourceFragments, + Optional maxSourceRows, + Optional maxSourceBytes, + List excludedFragmentIds); public CompactionOptions getCompactionOptions() { return compactionOptions; diff --git a/java/src/main/java/org/lance/delta/DatasetDelta.java b/java/src/main/java/org/lance/delta/DatasetDelta.java index 1c0eb4e9a73..c02e4e90e47 100755 --- a/java/src/main/java/org/lance/delta/DatasetDelta.java +++ b/java/src/main/java/org/lance/delta/DatasetDelta.java @@ -91,6 +91,24 @@ public ArrowReader getUpdatedRows() throws IOException { private native void nativeGetUpdatedRows(long streamAddress) throws IOException; + /** + * Return a streaming ArrowReader of the row ids deleted in the range. + * + *

The batches carry a single {@code _rowid} column. Requires stable row ids. + */ + public ArrowReader getDeletedRowIds() throws IOException { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeDeltaHandle != 0, "DatasetDelta is closed"); + BufferAllocator allocator = dataset.allocator(); + try (ArrowArrayStream s = ArrowArrayStream.allocateNew(allocator)) { + nativeGetDeletedRowIds(s.memoryAddress()); + return Data.importArrayStream(allocator, s); + } + } + } + + private native void nativeGetDeletedRowIds(long streamAddress) throws IOException; + @Override public void close() { try (LockManager.WriteLock writeLock = lockManager.acquireWriteLock()) { diff --git a/java/src/main/java/org/lance/delta/DatasetDeltaBuilder.java b/java/src/main/java/org/lance/delta/DatasetDeltaBuilder.java index 9084da2ab9c..9813b8aeba1 100755 --- a/java/src/main/java/org/lance/delta/DatasetDeltaBuilder.java +++ b/java/src/main/java/org/lance/delta/DatasetDeltaBuilder.java @@ -15,6 +15,7 @@ import org.lance.Dataset; import org.lance.JniLoader; +import org.lance.LockManager; import java.util.Optional; @@ -71,7 +72,9 @@ public DatasetDeltaBuilder withEndVersion(long version) { /** Build the DatasetDelta after validating builder state. */ public DatasetDelta build() { - return nativeBuild(dataset, comparedAgainst, beginVersion, endVersion); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeBuild(dataset, comparedAgainst, beginVersion, endVersion); + } } private static native DatasetDelta nativeBuild( diff --git a/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java b/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java index cadb22b0e6a..34dbbe3908c 100644 --- a/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java +++ b/java/src/main/java/org/lance/fragment/FragmentUpdateResult.java @@ -14,6 +14,7 @@ package org.lance.fragment; import org.lance.FragmentMetadata; +import org.lance.JniLoader; import com.google.common.base.MoreObjects; import org.apache.arrow.c.ArrowArrayStream; @@ -23,22 +24,51 @@ * Fragment.updateColumns()}. */ public class FragmentUpdateResult { + static { + JniLoader.ensureLoaded(); + } + private final FragmentMetadata updatedFragment; private final long[] fieldsModified; - /** Local physical row offsets within the fragment that received updates (see RowAddress). */ - private final long[] updatedRowOffsets; + /** + * Matched physical row offsets within the fragment, serialized as portable RoaringBitmap bytes + * (little-endian, same format as {@link org.lance.operation.Update#updatedFragmentOffsets()}). + */ + private final byte[] updatedRowOffsetBytes; + + /** Primary public API for constructing a result with portable RoaringBitmap offset bytes. */ + public static FragmentUpdateResult create( + FragmentMetadata updatedFragment, long[] updatedFieldIds, byte[] updatedRowOffsetBytes) { + return new FragmentUpdateResult(updatedFragment, updatedFieldIds, updatedRowOffsetBytes); + } /** Two-argument form for callers that do not track per-row offsets; offsets default to empty. */ public FragmentUpdateResult(FragmentMetadata updatedFragment, long[] updatedFieldIds) { - this(updatedFragment, updatedFieldIds, new long[0]); + this(updatedFragment, updatedFieldIds, new byte[0]); } - public FragmentUpdateResult( - FragmentMetadata updatedFragment, long[] updatedFieldIds, long[] updatedRowOffsets) { + private FragmentUpdateResult( + FragmentMetadata updatedFragment, long[] updatedFieldIds, byte[] updatedRowOffsetBytes) { this.updatedFragment = updatedFragment; this.fieldsModified = updatedFieldIds; - this.updatedRowOffsets = updatedRowOffsets; + this.updatedRowOffsetBytes = + updatedRowOffsetBytes != null ? updatedRowOffsetBytes : new byte[0]; + } + + /** + * @deprecated Use {@link #create(FragmentMetadata, long[], byte[])} instead. This constructor + * encodes the expanded {@code long[]} offsets into portable RoaringBitmap bytes via JNI and + * is retained for backward compatibility with callers compiled against the prior long[]-based + * API. + */ + @Deprecated + public FragmentUpdateResult( + FragmentMetadata updatedFragment, long[] updatedFieldIds, long[] updatedRowOffsets) { + this( + updatedFragment, + updatedFieldIds, + encodeRowOffsetsToBytes(updatedRowOffsets != null ? updatedRowOffsets : new long[0])); } public FragmentMetadata getUpdatedFragment() { @@ -49,17 +79,34 @@ public long[] getFieldsModified() { return fieldsModified; } - /** Physical row offsets (0-based within the fragment) whose columns were rewritten. */ + /** + * Physical row offsets (0-based within the fragment) whose columns were rewritten, as portable + * RoaringBitmap bytes. + */ + public byte[] getUpdatedRowOffsetBytes() { + return updatedRowOffsetBytes; + } + + /** + * Physical row offsets (0-based within the fragment) whose columns were rewritten. + * + * @deprecated Use {@link #getUpdatedRowOffsetBytes()} instead. + */ + @Deprecated public long[] getUpdatedRowOffsets() { - return updatedRowOffsets; + return expandRowOffsetsFromBytes(updatedRowOffsetBytes); } + private static native byte[] encodeRowOffsetsToBytes(long[] rowOffsets); + + private static native long[] expandRowOffsetsFromBytes(byte[] rowOffsetBytes); + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("fragmentMetadata", updatedFragment) .add("updatedFieldIds", fieldsModified) - .add("updatedRowOffsets", updatedRowOffsets) + .add("updatedRowOffsetBytesLength", updatedRowOffsetBytes.length) .toString(); } } diff --git a/java/src/main/java/org/lance/index/Index.java b/java/src/main/java/org/lance/index/Index.java index 955835496ed..620474ff787 100644 --- a/java/src/main/java/org/lance/index/Index.java +++ b/java/src/main/java/org/lance/index/Index.java @@ -17,6 +17,7 @@ import java.time.Instant; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -29,6 +30,7 @@ public class Index { private final UUID uuid; private final List fields; + private final List coveringFields; private final String name; private final long datasetVersion; private final List fragments; @@ -36,11 +38,13 @@ public class Index { private final int indexVersion; private final Instant createdAt; private final Integer baseId; + private final Long sizeBytes; private final IndexType indexType; private Index( UUID uuid, List fields, + List coveringFields, String name, long datasetVersion, List fragments, @@ -48,9 +52,11 @@ private Index( int indexVersion, Instant createdAt, Integer baseId, + Long sizeBytes, IndexType indexType) { this.uuid = uuid; this.fields = fields; + this.coveringFields = coveringFields; this.name = name; this.datasetVersion = datasetVersion; this.fragments = fragments; @@ -58,6 +64,7 @@ private Index( this.indexVersion = indexVersion; this.createdAt = createdAt; this.baseId = baseId; + this.sizeBytes = sizeBytes; this.indexType = indexType; } @@ -74,6 +81,20 @@ public List fields() { return fields; } + /** + * Fields whose values this index carries but is not keyed on. Always a suffix of {@link + * #fields()}, and never all of it, so the first entry of {@code fields()} is always a column the + * index is keyed on. Empty for an index that carries no extra columns. + * + *

These ids also appear in {@link #fields()} — that is deliberate, so that every consumer + * reading {@code fields()} as the index's dependency set also covers them with no change. + * + * @return the covering field IDs + */ + public List coveringFields() { + return coveringFields; + } + /** * Human readable index name * @@ -104,6 +125,17 @@ public Optional baseId() { return Optional.ofNullable(baseId); } + /** + * Get the total size of all files in this physical index segment. + * + *

The size is unavailable for indices created before index file sizes were tracked. + * + * @return the segment size in bytes, or empty if unavailable + */ + public Optional getSizeBytes() { + return Optional.ofNullable(sizeBytes); + } + /** * Get the index version. * @@ -140,11 +172,13 @@ public boolean equals(Object o) { && indexVersion == index.indexVersion && Objects.equals(uuid, index.uuid) && Objects.equals(fields, index.fields) + && Objects.equals(coveringFields, index.coveringFields) && Objects.equals(name, index.name) && Objects.equals(fragments, index.fragments) && Arrays.equals(indexDetails, index.indexDetails) && Objects.equals(createdAt, index.createdAt) && Objects.equals(baseId, index.baseId) + && Objects.equals(sizeBytes, index.sizeBytes) && indexType == index.indexType; } @@ -154,11 +188,13 @@ public int hashCode() { Objects.hash( uuid, fields, + coveringFields, name, datasetVersion, indexVersion, createdAt, baseId, + sizeBytes, fragments, indexType); result = 31 * result + Arrays.hashCode(indexDetails); @@ -170,12 +206,14 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("uuid", uuid) .add("fields", fields) + .add("coveringFields", coveringFields) .add("name", name) .add("datasetVersion", datasetVersion) .add("indexVersion", indexVersion) .add("indexType", indexType) .add("createdAt", createdAt) .add("baseId", baseId) + .add("sizeBytes", sizeBytes) .toString(); } @@ -192,6 +230,7 @@ public static class Builder { private UUID uuid; private List fields; + private List coveringFields = Collections.emptyList(); private String name; private long datasetVersion; private List fragments; @@ -199,6 +238,7 @@ public static class Builder { private int indexVersion; private Instant createdAt; private Integer baseId; + private Long sizeBytes; private IndexType indexType; private Builder() {} @@ -213,6 +253,11 @@ public Builder fields(List fields) { return this; } + public Builder coveringFields(List coveringFields) { + this.coveringFields = coveringFields; + return this; + } + public Builder name(String name) { this.name = name; return this; @@ -248,6 +293,11 @@ public Builder baseId(Integer baseId) { return this; } + public Builder sizeBytes(Long sizeBytes) { + this.sizeBytes = sizeBytes; + return this; + } + public Builder indexType(IndexType indexType) { this.indexType = indexType; return this; @@ -257,6 +307,7 @@ public Index build() { return new Index( uuid, fields, + coveringFields, name, datasetVersion, fragments, @@ -264,6 +315,7 @@ public Index build() { indexVersion, createdAt, baseId, + sizeBytes, indexType); } } diff --git a/java/src/main/java/org/lance/index/IndexBuildProgress.java b/java/src/main/java/org/lance/index/IndexBuildProgress.java new file mode 100644 index 00000000000..20cd0dd6d8d --- /dev/null +++ b/java/src/main/java/org/lance/index/IndexBuildProgress.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.index; + +import java.util.Optional; + +/** Receives stage-level progress while Lance builds or finalizes an index. */ +public interface IndexBuildProgress { + + /** + * Reports that a stage has started. + * + *

Implementations must be thread-safe. Lance may invoke callbacks concurrently from native + * runtime threads. Callbacks may re-enter the same {@code Dataset} through JNI methods. An + * exception thrown by this method terminates the index operation. + * + * @param stage stable, index-type-specific stage name + * @param total number of work units, or empty when the total is unknown + * @param unit description of the work unit, such as {@code files} or {@code partitions} + */ + void stageStart(String stage, Optional total, String unit); + + /** + * Reports completed work within a stage. + * + *

An exception thrown by this method terminates the index operation. + * + * @param stage stage name previously reported to {@link #stageStart} + * @param completed number of completed work units + */ + void stageProgress(String stage, long completed); + + /** + * Reports that a stage has completed. + * + *

The stage work has already succeeded when this method is called. Lance therefore logs and + * ignores exceptions thrown by this callback instead of failing the completed operation. + * + * @param stage completed stage name + */ + void stageComplete(String stage); +} diff --git a/java/src/main/java/org/lance/index/IndexDescription.java b/java/src/main/java/org/lance/index/IndexDescription.java index 1b5e5a3a8f8..b1e2689a00c 100755 --- a/java/src/main/java/org/lance/index/IndexDescription.java +++ b/java/src/main/java/org/lance/index/IndexDescription.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; /** * High-level description of an index, aggregating metadata across all segments. @@ -31,6 +32,7 @@ public final class IndexDescription { private final long rowsIndexed; private final List metadata; private final String detailsJson; + private final Long totalSizeBytes; public IndexDescription( String name, @@ -40,6 +42,18 @@ public IndexDescription( long rowsIndexed, List metadata, String detailsJson) { + this(name, fieldIds, typeUrl, indexType, rowsIndexed, metadata, detailsJson, null); + } + + public IndexDescription( + String name, + List fieldIds, + String typeUrl, + String indexType, + long rowsIndexed, + List metadata, + String detailsJson, + Long totalSizeBytes) { this.name = Objects.requireNonNull(name, "name must not be null"); this.fieldIds = Objects.requireNonNull(fieldIds, "fieldIds must not be null"); this.typeUrl = Objects.requireNonNull(typeUrl, "typeUrl must not be null"); @@ -47,6 +61,7 @@ public IndexDescription( this.rowsIndexed = rowsIndexed; this.metadata = Objects.requireNonNull(metadata, "metadata must not be null"); this.detailsJson = detailsJson; + this.totalSizeBytes = totalSizeBytes; } /** The logical name of the index. */ @@ -100,4 +115,15 @@ public List getSegments() { public String getDetailsJson() { return detailsJson; } + + /** + * Total size of all files across all physical index segments. + * + *

The size is unavailable if any segment predates index file size tracking. + * + * @return the logical index size in bytes, or empty if unavailable + */ + public Optional getTotalSizeBytes() { + return Optional.ofNullable(totalSizeBytes); + } } diff --git a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java index 9b29d7a0795..6928a78c4d9 100755 --- a/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java +++ b/java/src/main/java/org/lance/index/scalar/InvertedIndexParams.java @@ -13,6 +13,7 @@ */ package org.lance.index.scalar; +import org.lance.DocumentGranularity; import org.lance.util.JsonUtils; import com.google.common.base.Preconditions; @@ -53,8 +54,10 @@ public static final class Builder { private Integer minNgramLength; private Integer maxNgramLength; private Boolean prefixOnly; + private Integer blockSize = 128; private Boolean skipMerge; private Integer formatVersion; + private DocumentGranularity documentGranularity = DocumentGranularity.ROW; /** * Configure the base tokenizer. @@ -66,6 +69,7 @@ public static final class Builder { *

  • {@code "whitespace"}: splits tokens on whitespace *
  • {@code "raw"}: no tokenization *
  • {@code "ngram"}: N-Gram tokenizer + *
  • {@code "code"}: code-aware tokenizer *
  • {@code "icu"}: ICU dictionary-based Unicode word segmentation *
  • {@code "icu/split"}: ICU segmentation with simple-style delimiter splitting *
  • {@code "lindera/*"}: Lindera tokenizer @@ -226,6 +230,27 @@ public Builder prefixOnly(boolean prefixOnly) { return this; } + /** + * Configure the number of documents in each compressed posting block. + * + *

    Supported values are {@code 128} and {@code 256}. New indexes default to {@code 128} when + * this is not set. + * + *

    {@code blockSize = 256} requires FTS format v3. Format v3 also supports the default {@code + * blockSize = 128}. + * + * @param blockSize posting block size + * @return this builder + * @throws IllegalArgumentException if {@code blockSize} is unsupported + */ + public Builder blockSize(int blockSize) { + if (blockSize != 128 && blockSize != 256) { + throw new IllegalArgumentException("blockSize must be one of 128 or 256"); + } + this.blockSize = blockSize; + return this; + } + /** * Configure whether to skip the partition merge stage after indexing. If true, skip the * partition merge stage after indexing. This can be useful for distributed indexing where merge @@ -242,22 +267,47 @@ public Builder skipMerge(boolean skipMerge) { /** * Configure the on-disk FTS format version to write when creating a new index. * - *

    If unset, Lance chooses the current default format. + *

    If unset, Lance uses {@code LANCE_FTS_FORMAT_VERSION} when present and otherwise selects + * v3 for the code analyzer or {@code blockSize = 256}, and v2 for other indexes. Format v3 + * supports both posting block sizes. Formats v1 and v2 support only {@code blockSize = 128} and + * cannot be used with the code analyzer. * - * @param formatVersion FTS format version, must be 1 or 2 + * @param formatVersion FTS format version, must be 1, 2, or 3 * @return this builder * @throws IllegalArgumentException */ public Builder formatVersion(int formatVersion) { - if (formatVersion != 1 && formatVersion != 2) { - throw new IllegalArgumentException("formatVersion must be 1 or 2"); + if (formatVersion != 1 && formatVersion != 2 && formatVersion != 3) { + throw new IllegalArgumentException("formatVersion must be 1, 2, or 3"); } this.formatVersion = formatVersion; return this; } + /** + * Configure the unit treated as one FTS document. + * + *

    {@link DocumentGranularity#LIST_ELEMENT} uses each element of the deepest list on the + * indexed field path as one document. The default is {@link DocumentGranularity#ROW}. + * + * @param documentGranularity document boundary semantics + * @return this builder + */ + public Builder documentGranularity(DocumentGranularity documentGranularity) { + this.documentGranularity = + Objects.requireNonNull(documentGranularity, "documentGranularity must not be null"); + return this; + } + /** Build a {@link ScalarIndexParams} instance for an inverted index. */ public ScalarIndexParams build() { + if (formatVersion != null) { + Preconditions.checkArgument( + formatVersion == 3 || blockSize == 128, "formatVersion 1 and 2 require blockSize 128"); + Preconditions.checkArgument( + !"code".equals(baseTokenizer) || formatVersion == 3, + "baseTokenizer 'code' requires formatVersion 3"); + } Map params = new HashMap<>(); if (baseTokenizer != null) { params.put("base_tokenizer", baseTokenizer); @@ -300,12 +350,16 @@ public ScalarIndexParams build() { if (prefixOnly != null) { params.put("prefix_only", prefixOnly); } + if (blockSize != null) { + params.put("block_size", blockSize); + } if (skipMerge != null) { params.put("skip_merge", skipMerge); } if (formatVersion != null) { params.put("format_version", formatVersion); } + params.put("document_granularity", documentGranularity.toRustString()); String json = JsonUtils.toJson(params); return ScalarIndexParams.create(INDEX_TYPE, json); diff --git a/java/src/main/java/org/lance/index/vector/RQBuildParams.java b/java/src/main/java/org/lance/index/vector/RQBuildParams.java index 3898f674dab..49db5d9567e 100755 --- a/java/src/main/java/org/lance/index/vector/RQBuildParams.java +++ b/java/src/main/java/org/lance/index/vector/RQBuildParams.java @@ -15,7 +15,7 @@ import com.google.common.base.MoreObjects; -/** Parameters for building a Rabit Quantizer (RQ) index stage. */ +/** Parameters for building a Rabit Quantizer (RQ) index stage. Defaults to 5 bits per dimension. */ public class RQBuildParams { private final byte numBits; @@ -24,7 +24,7 @@ private RQBuildParams(Builder builder) { } public static class Builder { - private byte numBits = 1; + private byte numBits = 5; public Builder() {} diff --git a/java/src/main/java/org/lance/index/vector/VectorTrainer.java b/java/src/main/java/org/lance/index/vector/VectorTrainer.java index 9514c356fe7..640398ffcf6 100755 --- a/java/src/main/java/org/lance/index/vector/VectorTrainer.java +++ b/java/src/main/java/org/lance/index/vector/VectorTrainer.java @@ -15,6 +15,7 @@ import org.lance.Dataset; import org.lance.JniLoader; +import org.lance.LockManager; import org.lance.index.DistanceType; import org.apache.arrow.util.Preconditions; @@ -64,7 +65,9 @@ public static float[] trainIvfCentroids( column != null && !column.isEmpty(), "column cannot be null or empty"); Preconditions.checkArgument(params != null, "params cannot be null"); Preconditions.checkArgument(distanceType != null, "distanceType cannot be null"); - return nativeTrainIvfCentroids(dataset, column, params, distanceType.toString()); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeTrainIvfCentroids(dataset, column, params, distanceType.toString()); + } } /** @@ -98,7 +101,9 @@ public static float[] trainPqCodebook( column != null && !column.isEmpty(), "column cannot be null or empty"); Preconditions.checkArgument(params != null, "params cannot be null"); Preconditions.checkArgument(distanceType != null, "distanceType cannot be null"); - return nativeTrainPqCodebook(dataset, column, params, distanceType.toString()); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + return nativeTrainPqCodebook(dataset, column, params, distanceType.toString()); + } } private static native float[] nativeTrainIvfCentroids( diff --git a/java/src/main/java/org/lance/ipc/AsyncScanner.java b/java/src/main/java/org/lance/ipc/AsyncScanner.java index 6e515e3546c..193622f51bc 100644 --- a/java/src/main/java/org/lance/ipc/AsyncScanner.java +++ b/java/src/main/java/org/lance/ipc/AsyncScanner.java @@ -14,6 +14,7 @@ package org.lance.ipc; import org.lance.Dataset; +import org.lance.LanceException; import org.lance.LockManager; import org.apache.arrow.c.ArrowArrayStream; @@ -61,29 +62,37 @@ public static AsyncScanner create( Preconditions.checkNotNull(dataset); Preconditions.checkNotNull(options); Preconditions.checkNotNull(allocator); - AsyncScanner scanner = - createAsyncScanner( - dataset, - options.getFragmentIds(), - options.getColumns(), - options.getSubstraitFilter(), - options.getFilter(), - options.getBatchSize(), - options.getLimit(), - options.getOffset(), - options.getNearest(), - options.getFullTextQuery(), - options.isPrefilter(), - options.isWithRowId(), - options.isWithRowAddress(), - options.getBatchReadahead(), - options.getColumnOrderings(), - options.isUseScalarIndex(), - options.isFastSearch(), - options.getSubstraitAggregate(), - options.isIncludeDeletedRows(), - options.isStrictBatchSize(), - options.isDisableScoringAutoprojection()); + AsyncScanner scanner; + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + scanner = + createAsyncScanner( + dataset, + options.getFragmentIds(), + options.getColumns(), + options.getSubstraitFilter(), + options.getFilter(), + options.getBatchSize(), + options.getBatchSizeBytes(), + options.getIoBufferSize(), + options.getLimit(), + options.getOffset(), + options.getNearest(), + options.getFullTextQuery(), + options.isPrefilter(), + options.isWithRowId(), + options.isWithRowAddress(), + options.getBatchReadahead(), + options.getFragmentReadahead(), + options.isScanInOrder(), + options.getLateMaterialization(), + options.getColumnOrderings(), + options.isUseScalarIndex(), + options.isFastSearch(), + options.getSubstraitAggregate(), + options.isIncludeDeletedRows(), + options.isStrictBatchSize(), + options.isDisableScoringAutoprojection()); + } scanner.allocator = allocator; return scanner; } @@ -95,6 +104,8 @@ static native AsyncScanner createAsyncScanner( Optional substraitFilter, Optional filter, Optional batchSize, + Optional batchSizeBytes, + Optional ioBufferSize, Optional limit, Optional offset, Optional query, @@ -103,6 +114,9 @@ static native AsyncScanner createAsyncScanner( boolean withRowId, boolean withRowAddress, int batchReadahead, + Optional fragmentReadahead, + boolean scanInOrder, + Optional lateMaterialization, Optional> columnOrderings, boolean useScalarIndex, boolean fastSearch, @@ -137,18 +151,18 @@ public CompletableFuture scanBatchesAsync() { pendingTasks.remove(taskId); if (error != null) { - throw new RuntimeException("Scan failed", error); + throw new LanceException("Scan failed", error); } if (streamPtr < 0) { - throw new RuntimeException("Native scan error"); + throw new LanceException("Native scan returned an invalid stream pointer"); } try { ArrowArrayStream stream = ArrowArrayStream.wrap(streamPtr); return Data.importArrayStream(allocator, stream); } catch (Exception e) { - throw new RuntimeException(e); + throw new LanceException("Failed to import scan stream", e); } }); } @@ -166,7 +180,7 @@ private void completeTask(long taskId, long resultPtr) { private void failTask(long taskId, String errorMessage) { CompletableFuture future = pendingTasks.get(taskId); if (future != null) { - future.completeExceptionally(new RuntimeException(errorMessage)); + future.completeExceptionally(new LanceException(errorMessage)); } } diff --git a/java/src/main/java/org/lance/ipc/FullTextQuery.java b/java/src/main/java/org/lance/ipc/FullTextQuery.java index 0163def2f76..73badab9a49 100755 --- a/java/src/main/java/org/lance/ipc/FullTextQuery.java +++ b/java/src/main/java/org/lance/ipc/FullTextQuery.java @@ -13,6 +13,8 @@ */ package org.lance.ipc; +import org.lance.DocumentGranularity; + import com.google.common.base.MoreObjects; import org.apache.arrow.util.Preconditions; @@ -21,7 +23,13 @@ import java.util.Objects; import java.util.Optional; -/** Base type for full text search queries used by Lance scanner. */ +/** + * Base type for full text search queries used by Lance scanner. + * + *

    Match and phrase overloads without a {@link DocumentGranularity} infer the unique indexed + * granularity for the field. They use row documents when no index exists and fail as ambiguous when + * row and list-element indexes coexist. + */ public abstract class FullTextQuery { public enum Type { MATCH, @@ -37,8 +45,11 @@ public enum Operator { } public enum Occur { + /** The clause may match and contributes its score when it does. */ SHOULD, + /** The clause must match and contributes its score. */ MUST, + /** The clause must not match and never contributes to the score. */ MUST_NOT } @@ -63,7 +74,14 @@ public FullTextQuery getQuery() { public abstract Type getType(); public static FullTextQuery match(String queryText, String column) { - return match(queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0); + return new MatchQuery( + queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0, Optional.empty()); + } + + public static FullTextQuery match( + String queryText, String column, DocumentGranularity documentGranularity) { + return match( + queryText, column, 1.0f, Optional.empty(), 50, Operator.OR, 0, documentGranularity); } public static FullTextQuery match( @@ -75,15 +93,58 @@ public static FullTextQuery match( Operator operator, int prefixLength) { return new MatchQuery( - queryText, column, boost, fuzziness, maxExpansions, operator, prefixLength); + queryText, + column, + boost, + fuzziness, + maxExpansions, + operator, + prefixLength, + Optional.empty()); + } + + public static FullTextQuery match( + String queryText, + String column, + float boost, + Optional fuzziness, + int maxExpansions, + Operator operator, + int prefixLength, + DocumentGranularity documentGranularity) { + return new MatchQuery( + queryText, + column, + boost, + fuzziness, + maxExpansions, + operator, + prefixLength, + Optional.of( + Objects.requireNonNull(documentGranularity, "documentGranularity must not be null"))); } public static FullTextQuery phrase(String queryText, String column) { - return phrase(queryText, column, 0); + return new PhraseQuery(queryText, column, 0, Optional.empty()); + } + + public static FullTextQuery phrase( + String queryText, String column, DocumentGranularity documentGranularity) { + return phrase(queryText, column, 0, documentGranularity); } public static FullTextQuery phrase(String queryText, String column, int slop) { - return new PhraseQuery(queryText, column, slop); + return new PhraseQuery(queryText, column, slop, Optional.empty()); + } + + public static FullTextQuery phrase( + String queryText, String column, int slop, DocumentGranularity documentGranularity) { + return new PhraseQuery( + queryText, + column, + slop, + Optional.of( + Objects.requireNonNull(documentGranularity, "documentGranularity must not be null"))); } public static FullTextQuery multiMatch(String queryText, List columns) { @@ -117,6 +178,7 @@ public static final class MatchQuery extends FullTextQuery { private final int maxExpansions; private final Operator operator; private final int prefixLength; + private final Optional documentGranularity; MatchQuery( String queryText, @@ -125,7 +187,8 @@ public static final class MatchQuery extends FullTextQuery { Optional fuzziness, int maxExpansions, Operator operator, - int prefixLength) { + int prefixLength, + Optional documentGranularity) { Preconditions.checkArgument( queryText != null && !queryText.isEmpty(), "queryText must not be null or empty"); Preconditions.checkArgument( @@ -140,6 +203,7 @@ public static final class MatchQuery extends FullTextQuery { this.maxExpansions = maxExpansions; this.operator = operator == null ? Operator.OR : operator; this.prefixLength = prefixLength; + this.documentGranularity = Objects.requireNonNull(documentGranularity); } @Override @@ -175,6 +239,11 @@ public int getPrefixLength() { return prefixLength; } + /** Returns the explicit granularity, or empty when query planning should infer it. */ + public Optional getDocumentGranularity() { + return documentGranularity; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -184,6 +253,7 @@ public boolean equals(Object o) { && maxExpansions == other.maxExpansions && prefixLength == other.prefixLength && operator == other.operator + && Objects.equals(documentGranularity, other.documentGranularity) && Objects.equals(queryText, other.queryText) && Objects.equals(column, other.column) && Objects.equals(fuzziness, other.fuzziness); @@ -192,7 +262,14 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hash( - queryText, column, boost, fuzziness, maxExpansions, operator, prefixLength); + queryText, + column, + boost, + fuzziness, + maxExpansions, + operator, + prefixLength, + documentGranularity); } @Override @@ -206,6 +283,7 @@ public String toString() { .add("maxExpansions", maxExpansions) .add("operator", operator) .add("prefixLength", prefixLength) + .add("documentGranularity", documentGranularity) .toString(); } } @@ -215,8 +293,13 @@ public static final class PhraseQuery extends FullTextQuery { private final String queryText; private final String column; private final int slop; + private final Optional documentGranularity; - PhraseQuery(String queryText, String column, int slop) { + PhraseQuery( + String queryText, + String column, + int slop, + Optional documentGranularity) { Preconditions.checkArgument( queryText != null && !queryText.isEmpty(), "queryText must not be null or empty"); Preconditions.checkArgument( @@ -226,6 +309,7 @@ public static final class PhraseQuery extends FullTextQuery { this.queryText = queryText; this.column = column; this.slop = slop; + this.documentGranularity = Objects.requireNonNull(documentGranularity); } @Override @@ -245,19 +329,25 @@ public int getSlop() { return slop; } + /** Returns the explicit granularity, or empty when query planning should infer it. */ + public Optional getDocumentGranularity() { + return documentGranularity; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (!(o instanceof PhraseQuery)) return false; PhraseQuery other = (PhraseQuery) o; return slop == other.slop + && Objects.equals(documentGranularity, other.documentGranularity) && Objects.equals(queryText, other.queryText) && Objects.equals(column, other.column); } @Override public int hashCode() { - return Objects.hash(queryText, column, slop); + return Objects.hash(queryText, column, slop, documentGranularity); } @Override @@ -267,6 +357,7 @@ public String toString() { .add("queryText", queryText) .add("column", column) .add("slop", slop) + .add("documentGranularity", documentGranularity) .toString(); } } diff --git a/java/src/main/java/org/lance/ipc/LanceScanner.java b/java/src/main/java/org/lance/ipc/LanceScanner.java index 3a413e0ccfd..edb0a36f71a 100644 --- a/java/src/main/java/org/lance/ipc/LanceScanner.java +++ b/java/src/main/java/org/lance/ipc/LanceScanner.java @@ -14,6 +14,7 @@ package org.lance.ipc; import org.lance.Dataset; +import org.lance.LanceException; import org.lance.LockManager; import org.apache.arrow.c.ArrowArrayStream; @@ -57,30 +58,38 @@ public static LanceScanner create( Preconditions.checkNotNull(dataset); Preconditions.checkNotNull(options); Preconditions.checkNotNull(allocator); - LanceScanner scanner = - createScanner( - dataset, - options.getFragmentIds(), - options.getColumns(), - options.getSubstraitFilter(), - options.getFilter(), - options.getBatchSize(), - options.getLimit(), - options.getOffset(), - options.getNearest(), - options.getFullTextQuery(), - options.isPrefilter(), - options.isWithRowId(), - options.isWithRowAddress(), - options.getBatchReadahead(), - options.getColumnOrderings(), - options.isUseScalarIndex(), - options.isFastSearch(), - options.getSubstraitAggregate(), - options.isCollectStats(), - options.isIncludeDeletedRows(), - options.isStrictBatchSize(), - options.isDisableScoringAutoprojection()); + LanceScanner scanner; + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + scanner = + createScanner( + dataset, + options.getFragmentIds(), + options.getColumns(), + options.getSubstraitFilter(), + options.getFilter(), + options.getBatchSize(), + options.getBatchSizeBytes(), + options.getIoBufferSize(), + options.getLimit(), + options.getOffset(), + options.getNearest(), + options.getFullTextQuery(), + options.isPrefilter(), + options.isWithRowId(), + options.isWithRowAddress(), + options.getBatchReadahead(), + options.getFragmentReadahead(), + options.isScanInOrder(), + options.getLateMaterialization(), + options.getColumnOrderings(), + options.isUseScalarIndex(), + options.isFastSearch(), + options.getSubstraitAggregate(), + options.isCollectStats(), + options.isIncludeDeletedRows(), + options.isStrictBatchSize(), + options.isDisableScoringAutoprojection()); + } scanner.allocator = allocator; scanner.dataset = dataset; scanner.options = options; @@ -94,6 +103,8 @@ static native LanceScanner createScanner( Optional substraitFilter, Optional filter, Optional batchSize, + Optional batchSizeBytes, + Optional ioBufferSize, Optional limit, Optional offset, Optional query, @@ -102,6 +113,9 @@ static native LanceScanner createScanner( boolean withRowId, boolean withRowAddress, int batchReadahead, + Optional fragmentReadahead, + boolean scanInOrder, + Optional lateMaterialization, Optional> columnOrderings, boolean useScalarIndex, boolean fastSearch, @@ -140,12 +154,71 @@ public ArrowReader scanBatches() { openStream(s.memoryAddress()); return Data.importArrayStream(allocator, s); } catch (IOException e) { - // TODO: handle IO exception? - throw new RuntimeException(e); + throw new LanceException("Failed to open scan stream", e); } } } + /** + * Export this scan's results into a caller-owned Arrow C stream identified by its memory address, + * using the Arrow C Data Interface release callback to transfer ownership. + * + *

    This method intentionally takes a raw {@code streamAddress} (an {@code ArrowArrayStream} + * memory address) rather than a Java {@link ArrowArrayStream} object. A typed parameter would be + * an {@code org.apache.arrow.c.ArrowArrayStream} loaded by Lance's classloader / Arrow + * version; a caller running a different Arrow version (or under a different classloader, e.g. + * Spark + a native engine bundling its own Arrow) cannot construct that exact type and would hit + * a {@code ClassCastException}/{@code NoSuchMethodError} at the very boundary this method exists + * to cross. The C Data Interface ABI is stable across Arrow versions, so passing the C struct's + * address keeps the two sides fully decoupled: the caller allocates the stream with its + * own Arrow runtime and only the {@code long} address crosses into Lance. See gluten#12263 + * for the cross-Arrow-version integration that motivated this. + * + *

    Unlike {@link #scanBatches()}, no Java Arrow {@link ArrowReader} is created on Lance's side: + * Lance writes the C struct directly at {@code streamAddress} and the caller drives the read loop + * with its own Arrow runtime. + * + *

    The {@code streamAddress} must point to a freshly-allocated, empty {@code ArrowArrayStream} + * (its {@code release} callback must be null). Exporting into a stream that already holds a + * producer is rejected with an {@link IllegalArgumentException}, because overwriting the struct + * would drop the existing {@code release} callback and leak the first producer. The caller owns + * the stream and is responsible for closing it; the release callback installed by this call + * routes back through Lance's native side. + * + *

    The provided stream must not be shared across concurrent exports. An {@code + * ArrowArrayStream} is a plain C struct in caller-owned memory with no internal synchronization, + * so a single stream must be exported into, then drained, by one thread at a time. The + * already-populated check above guards the sequential "export twice" mistake, but it cannot make + * two concurrent exports into the same struct safe — that is a caller-side data race on + * caller-owned memory, the same contract as Arrow's C Data Interface itself. Use a separate + * stream per concurrent export. + * + *

    Example (caller on its own Arrow version / allocator): + * + * 

    {@code
+   * try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(callerAllocator)) {
+   *   scanner.exportArrowStream(stream.memoryAddress());
+   *   try (ArrowReader reader = Data.importArrayStream(callerAllocator, stream)) {
+   *     while (reader.loadNextBatch()) {
+   *       VectorSchemaRoot batch = reader.getVectorSchemaRoot();
+   *       // ...
+   *     }
+   *   }
+   * }
+   * }
    + * + * @param streamAddress the memory address of a freshly-allocated, empty {@code ArrowArrayStream} + * to populate + * @throws IllegalArgumentException if the scanner is closed or the stream is already populated + * @throws IOException if the native scan fails to start + */ + public void exportArrowStream(long streamAddress) throws IOException { + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeScannerHandle != 0, "Scanner is closed"); + openStream(streamAddress); + } + } + private native void openStream(long streamAddress) throws IOException; @Override diff --git a/java/src/main/java/org/lance/ipc/MaterializationStyle.java b/java/src/main/java/org/lance/ipc/MaterializationStyle.java new file mode 100644 index 00000000000..e0ab095e06c --- /dev/null +++ b/java/src/main/java/org/lance/ipc/MaterializationStyle.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.ipc; + +import org.apache.arrow.util.Preconditions; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Controls whether projected columns are fetched before or after filtering. + * + *

    For example, a selective filter can fetch a large payload column only for matching rows: + * + * 

    {@code
+ * ScanOptions options = new ScanOptions.Builder()
+ *     .filter("id = 42")
+ *     .lateMaterialization(MaterializationStyle.allEarlyExcept(
+ *         Collections.singletonList("payload")))
+ *     .build();
+ * }
    + * + *

    This policy only affects projected columns that are not used by a filter in a regular scan. + * Vector search and full-text search always use late materialization. + */ +public final class MaterializationStyle { + /** Available materialization policies. */ + public enum Mode { + /** Let Lance choose based on the storage system and column size. */ + HEURISTIC("heuristic"), + /** Fetch all eligible projected columns after filtering. */ + ALL_LATE("all_late"), + /** Fetch all projected columns before filtering. */ + ALL_EARLY("all_early"), + /** Fetch only the specified projected columns after filtering. */ + ALL_EARLY_EXCEPT("all_early_except"); + + private final String rustValue; + + Mode(String rustValue) { + this.rustValue = rustValue; + } + + /** Returns the explicit value expected by the Rust binding. */ + public String toRustString() { + return rustValue; + } + } + + private final Mode mode; + private final List columns; + + private MaterializationStyle(Mode mode, List columns) { + this.mode = mode; + this.columns = Collections.unmodifiableList(new ArrayList<>(columns)); + } + + /** Use Lance's storage-aware heuristic. This is the default when no style is specified. */ + public static MaterializationStyle heuristic() { + return new MaterializationStyle(Mode.HEURISTIC, Collections.emptyList()); + } + + /** Fetch all eligible projected columns after filtering. */ + public static MaterializationStyle allLate() { + return new MaterializationStyle(Mode.ALL_LATE, Collections.emptyList()); + } + + /** Fetch all projected columns before filtering. */ + public static MaterializationStyle allEarly() { + return new MaterializationStyle(Mode.ALL_EARLY, Collections.emptyList()); + } + + /** + * Fetch only the specified projected columns after filtering; all other columns are fetched + * before filtering. An empty list is equivalent to {@link #allEarly()}. + */ + public static MaterializationStyle allEarlyExcept(List lateColumns) { + Preconditions.checkNotNull(lateColumns, "lateColumns must not be null"); + for (String column : lateColumns) { + Preconditions.checkArgument( + column != null && !column.isEmpty(), "lateColumns must not contain null or empty names"); + } + return new MaterializationStyle(Mode.ALL_EARLY_EXCEPT, lateColumns); + } + + /** Returns the selected materialization mode. */ + public Mode getMode() { + return mode; + } + + /** Returns the explicit value expected by the Rust binding. */ + public String toRustString() { + return mode.toRustString(); + } + + /** Columns to fetch after filtering when using {@link Mode#ALL_EARLY_EXCEPT}. */ + public List getColumns() { + return columns; + } + + @Override + public String toString() { + return "MaterializationStyle{" + "mode=" + mode + ", columns=" + columns + '}'; + } +} diff --git a/java/src/main/java/org/lance/ipc/ScanOptions.java b/java/src/main/java/org/lance/ipc/ScanOptions.java index a9aad590c2b..1287652d37b 100644 --- a/java/src/main/java/org/lance/ipc/ScanOptions.java +++ b/java/src/main/java/org/lance/ipc/ScanOptions.java @@ -24,6 +24,8 @@ public class ScanOptions { private final Optional> fragmentIds; private final Optional batchSize; + private final Optional batchSizeBytes; + private final Optional ioBufferSize; private final Optional> columns; private final Optional filter; private final Optional substraitFilter; @@ -35,6 +37,9 @@ public class ScanOptions { private final boolean withRowId; private final boolean withRowAddress; private final int batchReadahead; + private final Optional fragmentReadahead; + private final boolean scanInOrder; + private final Optional lateMaterialization; private final Optional> columnOrderings; private final boolean useScalarIndex; private final Optional substraitAggregate; @@ -131,11 +136,86 @@ public ScanOptions( boolean includeDeletedRows, boolean strictBatchSize, boolean disableScoringAutoprojection) { + this( + fragmentIds, + batchSize, + Optional.empty(), + Optional.empty(), + columns, + filter, + substraitFilter, + limit, + offset, + nearest, + fullTextQuery, + prefilter, + withRowId, + withRowAddress, + batchReadahead, + Optional.empty(), + true, + Optional.empty(), + columnOrderings, + useScalarIndex, + substraitAggregate, + collectStats, + fastSearch, + includeDeletedRows, + strictBatchSize, + disableScoringAutoprojection); + } + + private ScanOptions( + Optional> fragmentIds, + Optional batchSize, + Optional batchSizeBytes, + Optional ioBufferSize, + Optional> columns, + Optional filter, + Optional substraitFilter, + Optional limit, + Optional offset, + Optional nearest, + Optional fullTextQuery, + boolean prefilter, + boolean withRowId, + boolean withRowAddress, + int batchReadahead, + Optional fragmentReadahead, + boolean scanInOrder, + Optional lateMaterialization, + Optional> columnOrderings, + boolean useScalarIndex, + Optional substraitAggregate, + boolean collectStats, + boolean fastSearch, + boolean includeDeletedRows, + boolean strictBatchSize, + boolean disableScoringAutoprojection) { Preconditions.checkArgument( !(filter.isPresent() && substraitFilter.isPresent()), "cannot set both substrait filter and string filter"); + Preconditions.checkArgument( + batchReadahead > 0, "batchReadahead must be greater than 0, got %s", batchReadahead); + batchSizeBytes.ifPresent( + value -> + Preconditions.checkArgument( + value > 0, "batchSizeBytes must be greater than 0, got %s", value)); + ioBufferSize.ifPresent( + value -> + Preconditions.checkArgument( + value > 0, "ioBufferSize must be greater than 0, got %s", value)); + fragmentReadahead.ifPresent( + value -> + Preconditions.checkArgument( + value > 0, "fragmentReadahead must be greater than 0, got %s", value)); + Preconditions.checkArgument( + !(strictBatchSize && batchSizeBytes.isPresent()), + "strictBatchSize=true cannot be combined with batchSizeBytes"); this.fragmentIds = fragmentIds; this.batchSize = batchSize; + this.batchSizeBytes = batchSizeBytes; + this.ioBufferSize = ioBufferSize; this.columns = columns; this.filter = filter; this.substraitFilter = substraitFilter; @@ -147,6 +227,9 @@ public ScanOptions( this.withRowId = withRowId; this.withRowAddress = withRowAddress; this.batchReadahead = batchReadahead; + this.fragmentReadahead = fragmentReadahead; + this.scanInOrder = scanInOrder; + this.lateMaterialization = lateMaterialization; this.columnOrderings = columnOrderings; this.useScalarIndex = useScalarIndex; this.substraitAggregate = substraitAggregate; @@ -175,6 +258,24 @@ public Optional getBatchSize() { return batchSize; } + /** + * Get the target batch size in bytes. + * + * @return Optional containing the target batch size in bytes if specified, otherwise empty. + */ + public Optional getBatchSizeBytes() { + return batchSizeBytes; + } + + /** + * Get the I/O buffer size in bytes. + * + * @return Optional containing the I/O buffer size if specified, otherwise empty. + */ + public Optional getIoBufferSize() { + return ioBufferSize; + } + /** * Get the columns. * @@ -274,6 +375,33 @@ public int getBatchReadahead() { return batchReadahead; } + /** + * Get the number of fragments to read ahead concurrently. + * + * @return Optional containing the fragment readahead if specified, otherwise empty. + */ + public Optional getFragmentReadahead() { + return fragmentReadahead; + } + + /** + * Get whether batches should be returned in fragment and batch order. + * + * @return true to preserve scan order, false to return batches as soon as they are ready. + */ + public boolean isScanInOrder() { + return scanInOrder; + } + + /** + * Get the materialization policy for projected columns. + * + * @return Optional containing the materialization policy if specified, otherwise empty. + */ + public Optional getLateMaterialization() { + return lateMaterialization; + } + public Optional> getColumnOrderings() { return columnOrderings; } @@ -341,6 +469,8 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("fragmentIds", fragmentIds.orElse(null)) .add("batchSize", batchSize.orElse(null)) + .add("batchSizeBytes", batchSizeBytes.orElse(null)) + .add("ioBufferSize", ioBufferSize.orElse(null)) .add("columns", columns.orElse(null)) .add("filter", filter.orElse(null)) .add( @@ -354,6 +484,9 @@ public String toString() { .add("withRowId", withRowId) .add("WithRowAddress", withRowAddress) .add("batchReadahead", batchReadahead) + .add("fragmentReadahead", fragmentReadahead.orElse(null)) + .add("scanInOrder", scanInOrder) + .add("lateMaterialization", lateMaterialization.orElse(null)) .add("columnOrdering", columnOrderings) .add("useScalarIndex", useScalarIndex) .add("fastSearch", fastSearch) @@ -371,6 +504,8 @@ public String toString() { public static class Builder { private Optional> fragmentIds = Optional.empty(); private Optional batchSize = Optional.empty(); + private Optional batchSizeBytes = Optional.empty(); + private Optional ioBufferSize = Optional.empty(); private Optional> columns = Optional.empty(); private Optional filter = Optional.empty(); private Optional substraitFilter = Optional.empty(); @@ -382,6 +517,9 @@ public static class Builder { private boolean withRowId = false; private boolean withRowAddress = false; private int batchReadahead = 16; + private Optional fragmentReadahead = Optional.empty(); + private boolean scanInOrder = true; + private Optional lateMaterialization = Optional.empty(); private Optional> columnOrderings = Optional.empty(); private boolean useScalarIndex = true; private boolean fastSearch = false; @@ -401,6 +539,8 @@ public Builder() {} public Builder(ScanOptions options) { this.fragmentIds = options.getFragmentIds(); this.batchSize = options.getBatchSize(); + this.batchSizeBytes = options.getBatchSizeBytes(); + this.ioBufferSize = options.getIoBufferSize(); this.columns = options.getColumns(); this.filter = options.getFilter(); this.substraitFilter = options.getSubstraitFilter(); @@ -412,6 +552,9 @@ public Builder(ScanOptions options) { this.withRowId = options.isWithRowId(); this.withRowAddress = options.isWithRowAddress(); this.batchReadahead = options.getBatchReadahead(); + this.fragmentReadahead = options.getFragmentReadahead(); + this.scanInOrder = options.isScanInOrder(); + this.lateMaterialization = options.getLateMaterialization(); this.columnOrderings = options.getColumnOrderings(); this.useScalarIndex = options.isUseScalarIndex(); this.fastSearch = options.isFastSearch(); @@ -444,6 +587,34 @@ public Builder batchSize(long batchSize) { return this; } + /** + * Set the approximate target size of each returned Arrow record batch in bytes. + * + *

    If {@link #batchSize(long)} is also set, the limit reached first determines the batch + * size. This option cannot be combined with {@link #strictBatchSize(boolean)}. + * + * @param batchSizeBytes target batch size in bytes; must be greater than zero. + * @return Builder instance for method chaining. + */ + public Builder batchSizeBytes(long batchSizeBytes) { + this.batchSizeBytes = Optional.of(batchSizeBytes); + return this; + } + + /** + * Set the amount of memory reserved for buffering storage I/O. + * + *

    This controls scanner backpressure but is not a hard upper bound on total scanner memory. + * It currently applies only to v2 files. + * + * @param ioBufferSize I/O buffer size in bytes; must be greater than zero. + * @return Builder instance for method chaining. + */ + public Builder ioBufferSize(long ioBufferSize) { + this.ioBufferSize = Optional.of(ioBufferSize); + return this; + } + /** * Set the columns. * @@ -565,6 +736,46 @@ public Builder batchReadahead(int batchReadahead) { return this; } + /** + * Set the number of fragments to read ahead concurrently. + * + * @param fragmentReadahead number of fragments to read ahead; must be greater than zero. + * @return Builder instance for method chaining. + */ + public Builder fragmentReadahead(int fragmentReadahead) { + this.fragmentReadahead = Optional.of(fragmentReadahead); + return this; + } + + /** + * Set whether batches are returned in fragment and batch order. + * + *

    Disabling ordering can increase scan concurrency at the cost of additional memory. This + * option is ignored for v2 files, which always scan in order. It is also ignored when a column + * ordering or nearest-neighbor query is configured; the query determines the result order. + * + * @param scanInOrder true to preserve scan order, false to return ready batches immediately. + * @return Builder instance for method chaining. + */ + public Builder scanInOrder(boolean scanInOrder) { + this.scanInOrder = scanInOrder; + return this; + } + + /** + * Set when projected columns that are not used by the filter should be materialized. + * + *

    This option only affects regular scans. Vector search and full-text search always use late + * materialization. + * + * @param lateMaterialization materialization policy. + * @return Builder instance for method chaining. + */ + public Builder lateMaterialization(MaterializationStyle lateMaterialization) { + this.lateMaterialization = Optional.of(lateMaterialization); + return this; + } + public Builder setColumnOrderings(List columnOrderings) { this.columnOrderings = Optional.of(columnOrderings); return this; @@ -667,6 +878,8 @@ public ScanOptions build() { return new ScanOptions( fragmentIds, batchSize, + batchSizeBytes, + ioBufferSize, columns, filter, substraitFilter, @@ -678,6 +891,9 @@ public ScanOptions build() { withRowId, withRowAddress, batchReadahead, + fragmentReadahead, + scanInOrder, + lateMaterialization, columnOrderings, useScalarIndex, substraitAggregate, diff --git a/java/src/main/java/org/lance/ipc/ScanStats.java b/java/src/main/java/org/lance/ipc/ScanStats.java index 24926d00911..e2160d0f371 100755 --- a/java/src/main/java/org/lance/ipc/ScanStats.java +++ b/java/src/main/java/org/lance/ipc/ScanStats.java @@ -30,6 +30,13 @@ public final class ScanStats { private final long indicesLoaded; private final long partsLoaded; private final long indexComparisons; + + /** Number of index cache page lookups served from memory in this scan. */ + private final long indexCacheHits; + + /** Number of index cache page lookups that had to load from storage in this scan. */ + private final long indexCacheMisses; + private final Map allCounts; private final Map allTimes; @@ -40,6 +47,8 @@ public ScanStats( long indicesLoaded, long partsLoaded, long indexComparisons, + long indexCacheHits, + long indexCacheMisses, Map allCounts, Map allTimes) { this.iops = iops; @@ -48,10 +57,42 @@ public ScanStats( this.indicesLoaded = indicesLoaded; this.partsLoaded = partsLoaded; this.indexComparisons = indexComparisons; + this.indexCacheHits = indexCacheHits; + this.indexCacheMisses = indexCacheMisses; this.allCounts = freezeMap(allCounts); this.allTimes = freezeMap(allTimes); } + /** + * Backwards-compatible constructor kept for existing callers that predate the addition of + * per-query index cache statistics. New code should use the 10-argument constructor that also + * accepts {@code indexCacheHits} and {@code indexCacheMisses}. + * + * @deprecated Use {@link #ScanStats(long, long, long, long, long, long, long, long, Map, Map)}. + */ + @Deprecated + public ScanStats( + long iops, + long requests, + long bytesRead, + long indicesLoaded, + long partsLoaded, + long indexComparisons, + Map allCounts, + Map allTimes) { + this( + iops, + requests, + bytesRead, + indicesLoaded, + partsLoaded, + indexComparisons, + 0L, + 0L, + allCounts, + allTimes); + } + private static Map freezeMap(Map map) { if (map == null || map.isEmpty()) { return Collections.emptyMap(); @@ -83,6 +124,41 @@ public long getIndexComparisons() { return indexComparisons; } + /** + * Number of index cache page lookups where the loader was not executed in this scan. + * + *

    Counts both true cache hits on already-populated entries and coalesced concurrent loads (a + * follower attached to another caller's in-flight load). + * + *

    Instrumented boundaries in this release: BTree, IVF v2 (write-cache scan path), inverted + * posting list (grouped and per-token) and its per-token metadata, inverted phrase positions, + * bitmap (Equals / Range / IsIn), ngram, rtree. + * + *

    Caveats: + * + *

      + *
    • IVF v2 streaming scans and legacy v1 IVF partitions bypass the cache by design and are + * therefore reported as a miss on every call. + *
    • A cold posting-list lookup on the grouped inverted layout can record up to two misses + * (group + per-token metadata) for a single term. + *
    + * + *

    Uninstrumented paths (HNSW graph pages, quantizer codebooks) do not contribute to either + * counter. See the sibling {@link #getIndexCacheMisses()} for the paired counter. + */ + public long getIndexCacheHits() { + return indexCacheHits; + } + + /** + * Number of index cache page lookups where the loader ran in this scan (the page was not resident + * and had to be materialised, typically from storage). See {@link #getIndexCacheHits()} for the + * paired counter and the list of instrumented boundaries. + */ + public long getIndexCacheMisses() { + return indexCacheMisses; + } + public Map getAllCounts() { return allCounts; } @@ -106,6 +182,8 @@ public boolean equals(Object o) { && indicesLoaded == that.indicesLoaded && partsLoaded == that.partsLoaded && indexComparisons == that.indexComparisons + && indexCacheHits == that.indexCacheHits + && indexCacheMisses == that.indexCacheMisses && Objects.equals(allCounts, that.allCounts) && Objects.equals(allTimes, that.allTimes); } @@ -119,6 +197,8 @@ public int hashCode() { indicesLoaded, partsLoaded, indexComparisons, + indexCacheHits, + indexCacheMisses, allCounts, allTimes); } @@ -138,6 +218,10 @@ public String toString() { + partsLoaded + ", indexComparisons=" + indexComparisons + + ", indexCacheHits=" + + indexCacheHits + + ", indexCacheMisses=" + + indexCacheMisses + ", allCounts=" + allCounts + ", allTimes=" diff --git a/java/src/main/java/org/lance/memwal/MergedGeneration.java b/java/src/main/java/org/lance/memwal/CompactedSsTable.java similarity index 77% rename from java/src/main/java/org/lance/memwal/MergedGeneration.java rename to java/src/main/java/org/lance/memwal/CompactedSsTable.java index 481e79ba3c0..cb703b42641 100644 --- a/java/src/main/java/org/lance/memwal/MergedGeneration.java +++ b/java/src/main/java/org/lance/memwal/CompactedSsTable.java @@ -17,32 +17,32 @@ import com.google.common.base.Preconditions; /** - * Identifies a flushed MemWAL generation that has been merged into the base table. + * Points to an SSTable compacted into the base table. * - *

    Pass a list of these to {@link org.lance.merge.MergeInsertParams#markGenerationsAsMerged} so - * Lance knows which generations are now part of the base table. + *

    Pass a list of these to {@link org.lance.merge.MergeInsertParams#markSstablesAsCompacted} so + * Lance can record compaction progress. */ -public class MergedGeneration { +public class CompactedSsTable { private final String shardId; private final long generation; /** * @param shardId UUID string for the write shard - * @param generation generation number from {@link ShardSnapshot#flushedGenerations()} + * @param generation generation number from {@link ShardSnapshot#sstables()} */ - public MergedGeneration(String shardId, long generation) { + public CompactedSsTable(String shardId, long generation) { Preconditions.checkNotNull(shardId, "shardId must not be null"); this.shardId = shardId; this.generation = generation; } /** UUID string for the write shard. */ - public String shardId() { + public String getShardId() { return shardId; } - /** The merged generation number. */ - public long generation() { + /** The compacted SSTable's generation number. */ + public long getGeneration() { return generation; } diff --git a/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java b/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java index 0488f9e4530..941a68a1606 100644 --- a/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java +++ b/java/src/main/java/org/lance/memwal/LsmPointLookupPlanner.java @@ -44,7 +44,7 @@ public class LsmPointLookupPlanner implements AutoCloseable { /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include */ public LsmPointLookupPlanner(Dataset dataset, List shardSnapshots) { this(dataset, shardSnapshots, null); @@ -52,7 +52,7 @@ public LsmPointLookupPlanner(Dataset dataset, List shardSnapshots /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param pkColumns primary key column names; inferred from schema metadata when {@code null} */ public LsmPointLookupPlanner( @@ -60,7 +60,9 @@ public LsmPointLookupPlanner( Preconditions.checkNotNull(dataset, "dataset must not be null"); Preconditions.checkNotNull(shardSnapshots, "shardSnapshots must not be null"); this.allocator = dataset.allocator(); - nativeCreate(dataset, shardSnapshots, Optional.ofNullable(pkColumns)); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + nativeCreate(dataset, shardSnapshots, Optional.ofNullable(pkColumns)); + } } private native void nativeCreate( diff --git a/java/src/main/java/org/lance/memwal/LsmScanner.java b/java/src/main/java/org/lance/memwal/LsmScanner.java index 29bfb097375..308d8009d71 100644 --- a/java/src/main/java/org/lance/memwal/LsmScanner.java +++ b/java/src/main/java/org/lance/memwal/LsmScanner.java @@ -31,8 +31,8 @@ * LSM-aware scanner covering all MemWAL data levels. * *

    Results are deduplicated by primary key, always returning the newest version of each row - * across the base table, flushed MemTables, and (when created from a {@link ShardWriter}) the - * active MemTable. + * across the base table, SSTables, and (when created from a {@link ShardWriter}) the active + * MemTable. * *

    The builder methods ({@link #project}, {@link #filter}, {@link #limit}, {@link * #withRowAddress}, {@link #withMemtableGen}) mutate this scanner and return it for chaining. @@ -55,15 +55,17 @@ private LsmScanner() {} * read-your-writes consistency. * * @param dataset the base dataset to scan - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @return an LSM scanner */ public static LsmScanner fromSnapshots(Dataset dataset, List shardSnapshots) { Preconditions.checkNotNull(dataset, "dataset must not be null"); Preconditions.checkNotNull(shardSnapshots, "shardSnapshots must not be null"); - LsmScanner scanner = createFromSnapshots(dataset, shardSnapshots); - scanner.allocator = dataset.allocator(); - return scanner; + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + LsmScanner scanner = createFromSnapshots(dataset, shardSnapshots); + scanner.allocator = dataset.allocator(); + return scanner; + } } static native LsmScanner createFromSnapshots(Dataset dataset, List shardSnapshots); diff --git a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java index 75c0d86c475..b91e5ae4a7d 100644 --- a/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java +++ b/java/src/main/java/org/lance/memwal/LsmVectorSearchPlanner.java @@ -44,7 +44,7 @@ public class LsmVectorSearchPlanner implements AutoCloseable { /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param vectorColumn name of the {@code FixedSizeList} vector column */ public LsmVectorSearchPlanner( @@ -54,7 +54,7 @@ public LsmVectorSearchPlanner( /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param vectorColumn name of the {@code FixedSizeList} vector column * @param pkColumns primary key column names; inferred from schema metadata when {@code null} * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"}, @@ -71,7 +71,7 @@ public LsmVectorSearchPlanner( /** * @param dataset the base dataset - * @param shardSnapshots shard snapshots specifying the flushed generations to include + * @param shardSnapshots shard snapshots specifying the SSTables to include * @param vectorColumn name of the {@code FixedSizeList} vector column * @param pkColumns primary key column names; inferred from schema metadata when {@code null} * @param distanceType distance metric, one of {@code "l2"}, {@code "cosine"}, {@code "dot"}, @@ -90,13 +90,15 @@ public LsmVectorSearchPlanner( Preconditions.checkNotNull(shardSnapshots, "shardSnapshots must not be null"); Preconditions.checkNotNull(vectorColumn, "vectorColumn must not be null"); this.allocator = dataset.allocator(); - nativeCreate( - dataset, - shardSnapshots, - vectorColumn, - Optional.ofNullable(pkColumns), - Optional.ofNullable(distanceType), - Optional.ofNullable(filter)); + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + nativeCreate( + dataset, + shardSnapshots, + vectorColumn, + Optional.ofNullable(pkColumns), + Optional.ofNullable(distanceType), + Optional.ofNullable(filter)); + } } private native void nativeCreate( diff --git a/java/src/main/java/org/lance/memwal/MemTableStats.java b/java/src/main/java/org/lance/memwal/MemTableStats.java index a993d771b42..55370ad101a 100644 --- a/java/src/main/java/org/lance/memwal/MemTableStats.java +++ b/java/src/main/java/org/lance/memwal/MemTableStats.java @@ -24,12 +24,16 @@ public class MemTableStats { private final long estimatedSizeBytes; private final long generation; private final Optional maxBufferedBatchPosition; - private final Optional maxFlushedBatchPosition; + private final long durableBatchCount; + private final long globalOffset; private final Optional pendingWalStartBatchPosition; private final Optional pendingWalEndBatchPosition; private final long pendingWalBatchCount; private final long pendingWalRowCount; private final long pendingWalEstimatedBytes; + private final long indexBytes; + private final long graceBytes; + private final long retainedBytes; public MemTableStats( long rowCount, @@ -37,23 +41,31 @@ public MemTableStats( long estimatedSizeBytes, long generation, Long maxBufferedBatchPosition, - Long maxFlushedBatchPosition, + long durableBatchCount, + long globalOffset, Long pendingWalStartBatchPosition, Long pendingWalEndBatchPosition, long pendingWalBatchCount, long pendingWalRowCount, - long pendingWalEstimatedBytes) { + long pendingWalEstimatedBytes, + long indexBytes, + long graceBytes, + long retainedBytes) { this.rowCount = rowCount; this.batchCount = batchCount; this.estimatedSizeBytes = estimatedSizeBytes; this.generation = generation; this.maxBufferedBatchPosition = Optional.ofNullable(maxBufferedBatchPosition); - this.maxFlushedBatchPosition = Optional.ofNullable(maxFlushedBatchPosition); + this.durableBatchCount = durableBatchCount; + this.globalOffset = globalOffset; this.pendingWalStartBatchPosition = Optional.ofNullable(pendingWalStartBatchPosition); this.pendingWalEndBatchPosition = Optional.ofNullable(pendingWalEndBatchPosition); this.pendingWalBatchCount = pendingWalBatchCount; this.pendingWalRowCount = pendingWalRowCount; this.pendingWalEstimatedBytes = pendingWalEstimatedBytes; + this.indexBytes = indexBytes; + this.graceBytes = graceBytes; + this.retainedBytes = retainedBytes; } /** Number of rows currently buffered in the active MemTable. */ @@ -66,7 +78,10 @@ public long batchCount() { return batchCount; } - /** Estimated in-memory size of the active MemTable, in bytes. */ + /** + * Row-data bytes of the active MemTable: the unit the flush trigger measures. Its in-memory + * indexes are reported separately by {@link #indexBytes()} and are not included here. + */ public long estimatedSizeBytes() { return estimatedSizeBytes; } @@ -81,9 +96,17 @@ public Optional maxBufferedBatchPosition() { return maxBufferedBatchPosition; } - /** Highest WAL batch position flushed from the MemTable, if any. */ - public Optional maxFlushedBatchPosition() { - return maxFlushedBatchPosition; + /** + * Writer-global count of WAL-durable batches (exclusive; 0 means none). Compare against {@code + * globalOffset() + batchCount()} to see what this MemTable still owes the WAL. + */ + public long durableBatchCount() { + return durableBatchCount; + } + + /** Writer-global coordinate of this MemTable's batch 0. */ + public long globalOffset() { + return globalOffset; } /** First WAL batch position pending flush, if any. */ @@ -111,6 +134,31 @@ public long pendingWalEstimatedBytes() { return pendingWalEstimatedBytes; } + /** + * Bytes held by the active MemTable's in-memory indexes, its primary-key bloom filter included. + * Usually what explains a shard near its ceiling with few rows in it: an HNSW graph is + * pre-allocated in full from the configured row capacity. + */ + public long indexBytes() { + return indexBytes; + } + + /** + * Bytes held by generations that have flushed but are lingering out the configured + * frozen-MemTable grace. Resident, but no flush reclaims them — the sweeper does, on a timer. + */ + public long graceBytes() { + return graceBytes; + } + + /** + * Every resident byte this shard holds. The figure a process-wide budget meters, as opposed to + * what a flush can still give back. + */ + public long retainedBytes() { + return retainedBytes; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) @@ -119,12 +167,16 @@ public String toString() { .add("estimatedSizeBytes", estimatedSizeBytes) .add("generation", generation) .add("maxBufferedBatchPosition", maxBufferedBatchPosition.orElse(null)) - .add("maxFlushedBatchPosition", maxFlushedBatchPosition.orElse(null)) + .add("durableBatchCount", durableBatchCount) + .add("globalOffset", globalOffset) .add("pendingWalStartBatchPosition", pendingWalStartBatchPosition.orElse(null)) .add("pendingWalEndBatchPosition", pendingWalEndBatchPosition.orElse(null)) .add("pendingWalBatchCount", pendingWalBatchCount) .add("pendingWalRowCount", pendingWalRowCount) .add("pendingWalEstimatedBytes", pendingWalEstimatedBytes) + .add("indexBytes", indexBytes) + .add("graceBytes", graceBytes) + .add("retainedBytes", retainedBytes) .toString(); } } diff --git a/java/src/main/java/org/lance/memwal/ShardSnapshot.java b/java/src/main/java/org/lance/memwal/ShardSnapshot.java index 493b2fce496..42e20bb5177 100644 --- a/java/src/main/java/org/lance/memwal/ShardSnapshot.java +++ b/java/src/main/java/org/lance/memwal/ShardSnapshot.java @@ -24,13 +24,13 @@ * Snapshot of a MemWAL shard's state, used when constructing scanners and planners. * *

    The builder methods ({@link #withSpecId}, {@link #withCurrentGeneration}, {@link - * #withFlushedGeneration}) mutate this instance and return it for chaining. + * #withSsTable}) mutate this instance and return it for chaining. */ public class ShardSnapshot { private final String shardId; private int specId = 0; private long currentGeneration = 0; - private final List flushedGenerations = new ArrayList<>(); + private final List sstables = new ArrayList<>(); /** * @param shardId UUID string for the write shard @@ -52,10 +52,10 @@ public ShardSnapshot withCurrentGeneration(long currentGeneration) { return this; } - /** Add a flushed generation with its storage path. */ - public ShardSnapshot withFlushedGeneration(long generation, String path) { + /** Add an SSTable with its storage path. */ + public ShardSnapshot withSsTable(long generation, String path) { Preconditions.checkNotNull(path, "path must not be null"); - this.flushedGenerations.add(new FlushedGeneration(generation, path)); + this.sstables.add(new SsTable(generation, path)); return this; } @@ -74,9 +74,9 @@ public long currentGeneration() { return currentGeneration; } - /** The flushed generations included in this snapshot. */ - public List flushedGenerations() { - return Collections.unmodifiableList(flushedGenerations); + /** The SSTables included in this snapshot. */ + public List sstables() { + return Collections.unmodifiableList(sstables); } @Override @@ -85,7 +85,7 @@ public String toString() { .add("shardId", shardId) .add("specId", specId) .add("currentGeneration", currentGeneration) - .add("flushedGenerations", flushedGenerations) + .add("sstables", sstables) .toString(); } } diff --git a/java/src/main/java/org/lance/memwal/ShardWriter.java b/java/src/main/java/org/lance/memwal/ShardWriter.java index a5a3000bdba..290b4c8f3c2 100644 --- a/java/src/main/java/org/lance/memwal/ShardWriter.java +++ b/java/src/main/java/org/lance/memwal/ShardWriter.java @@ -36,6 +36,7 @@ * 

    {@code
  * try (ShardWriter writer = dataset.memWalWriter(shardId)) {
  *   writer.put(reader);
+ *   writer.delete(keys);
  * }
  * }
    * @@ -64,9 +65,11 @@ private ShardWriter() {} public static ShardWriter create(Dataset dataset, String shardId, ShardWriterConfig config) { Preconditions.checkNotNull(dataset, "dataset must not be null"); Preconditions.checkNotNull(shardId, "shardId must not be null"); - ShardWriter writer = createNative(dataset, shardId, config); - writer.allocator = dataset.allocator(); - return writer; + try (LockManager.ReadLock readLock = dataset.acquireReadLock()) { + ShardWriter writer = createNative(dataset, shardId, config); + writer.allocator = dataset.allocator(); + return writer; + } } static native ShardWriter createNative(Dataset dataset, String shardId, ShardWriterConfig config); @@ -99,6 +102,34 @@ public void put(ArrowReader reader) { private native void nativePut(long streamAddress); + /** + * Delete rows from the MemWAL by primary key. + * + *

    Each batch in {@code reader} must carry this shard's primary key column(s); other columns + * are ignored. Lance builds a tombstone row per key — the primary key plus {@code _tombstone = + * true} and null in every other column — and appends it like an ordinary write. The tombstone is + * the newest value for its key: it wins newest-per-PK resolution (suppressing the older real row) + * and is then dropped from query results. + * + *

    Only supported in memtable mode. Because a tombstone nulls every non-PK column, those + * columns must be nullable in the base schema; deleting against a schema with a non-nullable + * non-PK column errors. Deleting on a shard with no primary key columns also errors. + * + * @param reader the keys to delete; consumed fully by this call + */ + public void delete(ArrowReader reader) { + Preconditions.checkNotNull(reader, "reader must not be null"); + try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { + Preconditions.checkArgument(nativeShardWriterHandle != 0, "ShardWriter is closed"); + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + Data.exportArrayStream(allocator, reader, stream); + nativeDelete(stream.memoryAddress()); + } + } + } + + private native void nativeDelete(long streamAddress); + /** Return a snapshot of cumulative write statistics. */ public WriteStats stats() { try (LockManager.ReadLock readLock = lockManager.acquireReadLock()) { @@ -122,9 +153,8 @@ public MemTableStats memtableStats() { /** * Create an LSM scanner that includes this writer's active MemTable. * - *

    The scanner covers the base table, the given flushed generations, and the current active - * MemTable, providing read-your-writes consistency. This writer's own shard is included - * automatically. + *

    The scanner covers the base table, the given SSTables, and the current active MemTable, + * providing read-your-writes consistency. This writer's own shard is included automatically. * * @param shardSnapshots snapshots of other shards to include * @return an LSM scanner diff --git a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java index 40310907d3e..d6bf2aeb383 100644 --- a/java/src/main/java/org/lance/memwal/ShardWriterConfig.java +++ b/java/src/main/java/org/lance/memwal/ShardWriterConfig.java @@ -28,7 +28,6 @@ */ public class ShardWriterConfig { private Optional durableWrite = Optional.empty(); - private Optional syncIndexedWrite = Optional.empty(); private Optional maxWalBufferSize = Optional.empty(); private Optional maxWalFlushIntervalMs = Optional.empty(); private Optional maxMemtableSize = Optional.empty(); @@ -36,8 +35,6 @@ public class ShardWriterConfig { private Optional maxMemtableBatches = Optional.empty(); private Optional maxUnflushedMemtableBytes = Optional.empty(); private Optional manifestScanBatchSize = Optional.empty(); - private Optional asyncIndexBufferRows = Optional.empty(); - private Optional asyncIndexIntervalMs = Optional.empty(); private Optional backpressureLogIntervalMs = Optional.empty(); private Optional statsLogIntervalMs = Optional.empty(); private List hnswParams = Collections.emptyList(); @@ -48,12 +45,6 @@ public ShardWriterConfig withDurableWrite(boolean durableWrite) { return this; } - /** Whether indexed writes are applied synchronously. */ - public ShardWriterConfig withSyncIndexedWrite(boolean syncIndexedWrite) { - this.syncIndexedWrite = Optional.of(syncIndexedWrite); - return this; - } - /** Maximum size of the in-memory WAL buffer, in bytes. */ public ShardWriterConfig withMaxWalBufferSize(long maxWalBufferSize) { Preconditions.checkArgument( @@ -118,26 +109,6 @@ public ShardWriterConfig withManifestScanBatchSize(long manifestScanBatchSize) { return this; } - /** Number of rows buffered before an asynchronous index update is triggered. */ - public ShardWriterConfig withAsyncIndexBufferRows(long asyncIndexBufferRows) { - Preconditions.checkArgument( - asyncIndexBufferRows >= 0, - "asyncIndexBufferRows must not be negative, got %s", - asyncIndexBufferRows); - this.asyncIndexBufferRows = Optional.of(asyncIndexBufferRows); - return this; - } - - /** Interval between asynchronous index updates, in milliseconds. */ - public ShardWriterConfig withAsyncIndexIntervalMs(long asyncIndexIntervalMs) { - Preconditions.checkArgument( - asyncIndexIntervalMs >= 0, - "asyncIndexIntervalMs must not be negative, got %s", - asyncIndexIntervalMs); - this.asyncIndexIntervalMs = Optional.of(asyncIndexIntervalMs); - return this; - } - /** Interval between backpressure log messages, in milliseconds. */ public ShardWriterConfig withBackpressureLogIntervalMs(long backpressureLogIntervalMs) { Preconditions.checkArgument( @@ -172,10 +143,6 @@ public Optional durableWrite() { return durableWrite; } - public Optional syncIndexedWrite() { - return syncIndexedWrite; - } - public Optional maxWalBufferSize() { return maxWalBufferSize; } @@ -204,14 +171,6 @@ public Optional manifestScanBatchSize() { return manifestScanBatchSize; } - public Optional asyncIndexBufferRows() { - return asyncIndexBufferRows; - } - - public Optional asyncIndexIntervalMs() { - return asyncIndexIntervalMs; - } - public Optional backpressureLogIntervalMs() { return backpressureLogIntervalMs; } diff --git a/java/src/main/java/org/lance/memwal/FlushedGeneration.java b/java/src/main/java/org/lance/memwal/SsTable.java similarity index 78% rename from java/src/main/java/org/lance/memwal/FlushedGeneration.java rename to java/src/main/java/org/lance/memwal/SsTable.java index 66288161438..55cb2e63f77 100644 --- a/java/src/main/java/org/lance/memwal/FlushedGeneration.java +++ b/java/src/main/java/org/lance/memwal/SsTable.java @@ -15,22 +15,22 @@ import com.google.common.base.MoreObjects; -/** A flushed MemWAL generation and the storage path of its Lance files. */ -public class FlushedGeneration { +/** An SSTable and the storage path of its Lance files. */ +public class SsTable { private final long generation; private final String path; - public FlushedGeneration(long generation, String path) { + public SsTable(long generation, String path) { this.generation = generation; this.path = path; } - /** The generation number of this flushed MemTable. */ + /** The generation number of this SSTable. */ public long generation() { return generation; } - /** The storage path of the flushed Lance files. */ + /** The storage path of the SSTable Lance files. */ public String path() { return path; } diff --git a/java/src/main/java/org/lance/merge/MergeInsertParams.java b/java/src/main/java/org/lance/merge/MergeInsertParams.java index 2ae27b67cba..a9a696ead73 100644 --- a/java/src/main/java/org/lance/merge/MergeInsertParams.java +++ b/java/src/main/java/org/lance/merge/MergeInsertParams.java @@ -13,7 +13,7 @@ */ package org.lance.merge; -import org.lance.memwal.MergedGeneration; +import org.lance.memwal.CompactedSsTable; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; @@ -39,7 +39,7 @@ public class MergeInsertParams { private long retryTimeoutMs = 30 * 1000; private boolean skipAutoCleanup = false; private boolean useIndex = true; - private List markedGenerations = Collections.emptyList(); + private List compactedSstables = Collections.emptyList(); public MergeInsertParams(List on) { this.on = on; @@ -245,17 +245,23 @@ public MergeInsertParams withUseIndex(boolean useIndex) { } /** - * Mark MemWAL generations as merged into the base table. + * Mark MemWAL SSTables as compacted into the base table. * - *

    Use this when the merge insert incorporates data from MemWAL flushed generations. It updates - * the MemWAL generation tracking to prevent the same generations from being merged again. + *

    Use this when merge insert compacts MemWAL SSTables. It updates MemWAL compaction progress + * to prevent the same SSTables from being compacted again, in the same commit as the data. * - * @param generations the flushed generations being merged + *

    For multi-pass compaction, call this only on the final successful data-changing pass. + * Intermediate passes must not carry compaction progress. Lance cannot tell whether a caller has + * another pass planned, so it cannot enforce this: if a delete pass carried the marker and the + * process then died before the matching upsert, the recorded progress would claim rows were + * copied in that never were. + * + * @param sstables the SSTables being compacted * @return This MergeInsertParams instance */ - public MergeInsertParams markGenerationsAsMerged(List generations) { - Preconditions.checkNotNull(generations, "generations must not be null"); - this.markedGenerations = generations; + public MergeInsertParams markSstablesAsCompacted(List sstables) { + Preconditions.checkNotNull(sstables, "sstables must not be null"); + this.compactedSstables = sstables; return this; } @@ -263,8 +269,8 @@ public List on() { return on; } - public List markedGenerations() { - return markedGenerations; + public List getCompactedSstables() { + return compactedSstables; } public WhenMatched whenMatched() { diff --git a/java/src/main/java/org/lance/namespace/DirectoryNamespace.java b/java/src/main/java/org/lance/namespace/DirectoryNamespace.java index d26bbff9135..e9e8618c763 100644 --- a/java/src/main/java/org/lance/namespace/DirectoryNamespace.java +++ b/java/src/main/java/org/lance/namespace/DirectoryNamespace.java @@ -26,6 +26,7 @@ import org.lance.namespace.model.BatchDeleteTableVersionsRequest; import org.lance.namespace.model.BatchDeleteTableVersionsResponse; import org.lance.namespace.model.CountTableRowsRequest; +import org.lance.namespace.model.CountTableRowsResponse; import org.lance.namespace.model.CreateMaterializedViewRequest; import org.lance.namespace.model.CreateMaterializedViewResponse; import org.lance.namespace.model.CreateNamespaceRequest; @@ -83,7 +84,9 @@ import org.lance.namespace.model.MergeInsertIntoTableRequest; import org.lance.namespace.model.MergeInsertIntoTableResponse; import org.lance.namespace.model.NamespaceExistsRequest; +import org.lance.namespace.model.NamespaceExistsResponse; import org.lance.namespace.model.QueryTableRequest; +import org.lance.namespace.model.QueryTableResponse; import org.lance.namespace.model.RegisterTableRequest; import org.lance.namespace.model.RegisterTableResponse; import org.lance.namespace.model.RenameTableRequest; @@ -91,6 +94,7 @@ import org.lance.namespace.model.RestoreTableRequest; import org.lance.namespace.model.RestoreTableResponse; import org.lance.namespace.model.TableExistsRequest; +import org.lance.namespace.model.TableExistsResponse; import org.lance.namespace.model.UpdateTableRequest; import org.lance.namespace.model.UpdateTableResponse; import org.lance.namespace.model.UpdateTableSchemaMetadataRequest; @@ -101,6 +105,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.apache.arrow.memory.BufferAllocator; import java.io.Closeable; @@ -239,6 +244,7 @@ public class DirectoryNamespace implements LanceNamespace, Closeable { private static ObjectMapper createObjectMapper() { ObjectMapper mapper = new ObjectMapper(); + mapper.registerModule(new JavaTimeModule()); mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); return mapper; } @@ -329,10 +335,11 @@ public DropNamespaceResponse dropNamespace(DropNamespaceRequest request) { } @Override - public void namespaceExists(NamespaceExistsRequest request) { + public NamespaceExistsResponse namespaceExists(NamespaceExistsRequest request) { ensureInitialized(); String requestJson = toJson(request); namespaceExistsNative(nativeDirectoryNamespaceHandle, requestJson); + return new NamespaceExistsResponse(); } @Override @@ -360,10 +367,11 @@ public RegisterTableResponse registerTable(RegisterTableRequest request) { } @Override - public void tableExists(TableExistsRequest request) { + public TableExistsResponse tableExists(TableExistsRequest request) { ensureInitialized(); String requestJson = toJson(request); tableExistsNative(nativeDirectoryNamespaceHandle, requestJson); + return new TableExistsResponse(); } @Override @@ -383,10 +391,11 @@ public DeregisterTableResponse deregisterTable(DeregisterTableRequest request) { } @Override - public Long countTableRows(CountTableRowsRequest request) { + public CountTableRowsResponse countTableRows(CountTableRowsRequest request) { ensureInitialized(); String requestJson = toJson(request); - return countTableRowsNative(nativeDirectoryNamespaceHandle, requestJson); + Long count = countTableRowsNative(nativeDirectoryNamespaceHandle, requestJson); + return new CountTableRowsResponse().count(count); } @Override @@ -451,10 +460,11 @@ public DeleteFromTableResponse deleteFromTable(DeleteFromTableRequest request) { } @Override - public byte[] queryTable(QueryTableRequest request) { + public QueryTableResponse queryTable(QueryTableRequest request) { ensureInitialized(); String requestJson = toJson(request); - return queryTableNative(nativeDirectoryNamespaceHandle, requestJson); + byte[] data = queryTableNative(nativeDirectoryNamespaceHandle, requestJson); + return new QueryTableResponse().data(data); } @Override diff --git a/java/src/main/java/org/lance/namespace/RestNamespace.java b/java/src/main/java/org/lance/namespace/RestNamespace.java index 9cbbc588660..c477c5470ef 100644 --- a/java/src/main/java/org/lance/namespace/RestNamespace.java +++ b/java/src/main/java/org/lance/namespace/RestNamespace.java @@ -26,6 +26,7 @@ import org.lance.namespace.model.BatchDeleteTableVersionsRequest; import org.lance.namespace.model.BatchDeleteTableVersionsResponse; import org.lance.namespace.model.CountTableRowsRequest; +import org.lance.namespace.model.CountTableRowsResponse; import org.lance.namespace.model.CreateMaterializedViewRequest; import org.lance.namespace.model.CreateMaterializedViewResponse; import org.lance.namespace.model.CreateNamespaceRequest; @@ -83,7 +84,9 @@ import org.lance.namespace.model.MergeInsertIntoTableRequest; import org.lance.namespace.model.MergeInsertIntoTableResponse; import org.lance.namespace.model.NamespaceExistsRequest; +import org.lance.namespace.model.NamespaceExistsResponse; import org.lance.namespace.model.QueryTableRequest; +import org.lance.namespace.model.QueryTableResponse; import org.lance.namespace.model.RegisterTableRequest; import org.lance.namespace.model.RegisterTableResponse; import org.lance.namespace.model.RenameTableRequest; @@ -91,6 +94,7 @@ import org.lance.namespace.model.RestoreTableRequest; import org.lance.namespace.model.RestoreTableResponse; import org.lance.namespace.model.TableExistsRequest; +import org.lance.namespace.model.TableExistsResponse; import org.lance.namespace.model.UpdateTableRequest; import org.lance.namespace.model.UpdateTableResponse; import org.lance.namespace.model.UpdateTableSchemaMetadataRequest; @@ -100,6 +104,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import org.apache.arrow.memory.BufferAllocator; import java.io.Closeable; @@ -149,7 +154,8 @@ public class RestNamespace implements LanceNamespace, Closeable { JniLoader.ensureLoaded(); } - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final ObjectMapper OBJECT_MAPPER = + new ObjectMapper().registerModule(new JavaTimeModule()); private long nativeRestNamespaceHandle; private BufferAllocator allocator; @@ -241,10 +247,11 @@ public DropNamespaceResponse dropNamespace(DropNamespaceRequest request) { } @Override - public void namespaceExists(NamespaceExistsRequest request) { + public NamespaceExistsResponse namespaceExists(NamespaceExistsRequest request) { ensureInitialized(); String requestJson = toJson(request); namespaceExistsNative(nativeRestNamespaceHandle, requestJson); + return new NamespaceExistsResponse(); } @Override @@ -272,10 +279,11 @@ public RegisterTableResponse registerTable(RegisterTableRequest request) { } @Override - public void tableExists(TableExistsRequest request) { + public TableExistsResponse tableExists(TableExistsRequest request) { ensureInitialized(); String requestJson = toJson(request); tableExistsNative(nativeRestNamespaceHandle, requestJson); + return new TableExistsResponse(); } @Override @@ -295,10 +303,11 @@ public DeregisterTableResponse deregisterTable(DeregisterTableRequest request) { } @Override - public Long countTableRows(CountTableRowsRequest request) { + public CountTableRowsResponse countTableRows(CountTableRowsRequest request) { ensureInitialized(); String requestJson = toJson(request); - return countTableRowsNative(nativeRestNamespaceHandle, requestJson); + Long count = countTableRowsNative(nativeRestNamespaceHandle, requestJson); + return new CountTableRowsResponse().count(count); } @Override @@ -362,10 +371,11 @@ public DeleteFromTableResponse deleteFromTable(DeleteFromTableRequest request) { } @Override - public byte[] queryTable(QueryTableRequest request) { + public QueryTableResponse queryTable(QueryTableRequest request) { ensureInitialized(); String requestJson = toJson(request); - return queryTableNative(nativeRestNamespaceHandle, requestJson); + byte[] data = queryTableNative(nativeRestNamespaceHandle, requestJson); + return new QueryTableResponse().data(data); } @Override diff --git a/java/src/main/java/org/lance/operation/Merge.java b/java/src/main/java/org/lance/operation/Merge.java index bd83657384b..07ecb880d90 100644 --- a/java/src/main/java/org/lance/operation/Merge.java +++ b/java/src/main/java/org/lance/operation/Merge.java @@ -27,16 +27,26 @@ */ public class Merge extends SchemaOperation { private final List fragments; + // True when this merge makes no nullability-affecting schema change: it + // introduces no field that data staged against an earlier schema could not + // safely omit. Without the assertion the merge conservatively conflicts with + // concurrent appends, whose fragments would omit new columns and read as null. + private final boolean preservesNullability; - protected Merge(List fragments, Schema schema) { + protected Merge(List fragments, Schema schema, boolean preservesNullability) { super(schema); this.fragments = fragments; + this.preservesNullability = preservesNullability; } public List fragments() { return fragments; } + public boolean preservesNullability() { + return preservesNullability; + } + @Override public String name() { return "Merge"; @@ -47,6 +57,7 @@ public String toString() { return MoreObjects.toStringHelper(this) .add("fragments", fragments) .add("schema", schema()) + .add("preservesNullability", preservesNullability) .toString(); } @@ -56,12 +67,13 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; Merge that = (Merge) o; - return Objects.equals(fragments, that.fragments); + return Objects.equals(fragments, that.fragments) + && preservesNullability == that.preservesNullability; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), fragments); + return Objects.hash(super.hashCode(), fragments, preservesNullability); } public static Builder builder() { @@ -71,6 +83,8 @@ public static Builder builder() { public static class Builder { private List fragments; private Schema schema; + // No assertion by default, which conservatively conflicts. + private boolean preservesNullability = false; private Builder() {} @@ -84,8 +98,13 @@ public Builder schema(Schema schema) { return this; } + public Builder preservesNullability(boolean preservesNullability) { + this.preservesNullability = preservesNullability; + return this; + } + public Merge build() { - return new Merge(fragments, schema); + return new Merge(fragments, schema, preservesNullability); } } } diff --git a/java/src/main/java/org/lance/operation/Project.java b/java/src/main/java/org/lance/operation/Project.java index a4c718c4c4b..8c79431a7d1 100644 --- a/java/src/main/java/org/lance/operation/Project.java +++ b/java/src/main/java/org/lance/operation/Project.java @@ -16,14 +16,26 @@ import com.google.common.base.MoreObjects; import org.apache.arrow.vector.types.pojo.Schema; +import java.util.Objects; + /** * Project to a new schema. This Operation only changes the schema, not the data. Note: 1. For * removing columns. The data will be removed after compaction. 2. Project will modify column * positions, not ids(a.k.a. field id) */ public class Project extends SchemaOperation { - private Project(Schema schema) { + // True when this projection makes no nullability-affecting schema change, + // as a rename or a drop does not. Without the assertion the projection + // conservatively conflicts with concurrent value writes. + private final boolean preservesNullability; + + private Project(Schema schema, boolean preservesNullability) { super(schema); + this.preservesNullability = preservesNullability; + } + + public boolean preservesNullability() { + return preservesNullability; } @Override @@ -33,7 +45,23 @@ public String name() { @Override public String toString() { - return MoreObjects.toStringHelper(this).add("schema", schema()).toString(); + return MoreObjects.toStringHelper(this) + .add("schema", schema()) + .add("preservesNullability", preservesNullability) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (!super.equals(o)) { + return false; + } + return preservesNullability == ((Project) o).preservesNullability; + } + + @Override + public int hashCode() { + return Objects.hash(schema(), preservesNullability); } public static Builder builder() { @@ -42,6 +70,7 @@ public static Builder builder() { public static class Builder { private Schema schema; + private boolean preservesNullability; public Builder() {} @@ -50,8 +79,13 @@ public Builder schema(Schema schema) { return this; } + public Builder preservesNullability(boolean preservesNullability) { + this.preservesNullability = preservesNullability; + return this; + } + public Project build() { - return new Project(schema); + return new Project(schema, preservesNullability); } } } diff --git a/java/src/main/java/org/lance/operation/Update.java b/java/src/main/java/org/lance/operation/Update.java index f886942b4b9..721bbd84b47 100644 --- a/java/src/main/java/org/lance/operation/Update.java +++ b/java/src/main/java/org/lance/operation/Update.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -31,19 +32,30 @@ public class Update implements Operation { private final long[] fieldsForPreservingFragBitmap; private final Optional updateMode; + /** + * Per-fragment matched row offsets serialized as portable RoaringBitmap bytes (little-endian, + * spec-compliant). Keys are fragment ids; values are the serialized bitmap for the local physical + * row offsets (0-based) within the fragment whose columns were rewritten. Empty map means the + * caller did not supply offsets and the partial last_updated refresh in build_manifest will not + * activate. + */ + private final Map updatedFragmentOffsets; + private Update( List removedFragmentIds, List updatedFragments, List newFragments, long[] fieldsModified, long[] fieldsForPreservingFragBitmap, - Optional updateMode) { + Optional updateMode, + Map updatedFragmentOffsets) { this.removedFragmentIds = removedFragmentIds; this.updatedFragments = updatedFragments; this.newFragments = newFragments; this.fieldsModified = fieldsModified; this.fieldsForPreservingFragBitmap = fieldsForPreservingFragBitmap; this.updateMode = updateMode; + this.updatedFragmentOffsets = updatedFragmentOffsets; } public static Builder builder() { @@ -74,6 +86,10 @@ public Optional updateMode() { return updateMode; } + public Map updatedFragmentOffsets() { + return updatedFragmentOffsets; + } + @Override public String name() { return "Update"; @@ -87,6 +103,7 @@ public String toString() { .add("fieldsModified", fieldsModified) .add("fieldsForPreservingFragBitmap", fieldsForPreservingFragBitmap) .add("updateMode", updateMode) + .add("updatedFragmentOffsets", updatedFragmentOffsets) .toString(); } @@ -100,7 +117,32 @@ public boolean equals(Object o) { && Objects.equals(newFragments, that.newFragments) && Arrays.equals(fieldsModified, that.fieldsModified) && Arrays.equals(fieldsForPreservingFragBitmap, that.fieldsForPreservingFragBitmap) - && Objects.equals(updateMode, that.updateMode); + && Objects.equals(updateMode, that.updateMode) + && offsetMapsEqual(updatedFragmentOffsets, that.updatedFragmentOffsets); + } + + /** Deep-equality for {@code Map}: keys by value, arrays by content. */ + private static boolean offsetMapsEqual(Map a, Map b) { + if (a == b) return true; + if (a.size() != b.size()) return false; + for (Map.Entry entry : a.entrySet()) { + if (!Arrays.equals(entry.getValue(), b.get(entry.getKey()))) return false; + } + return true; + } + + @Override + public int hashCode() { + int h = Objects.hash(removedFragmentIds, updatedFragments, newFragments, updateMode); + h = 31 * h + Arrays.hashCode(fieldsModified); + h = 31 * h + Arrays.hashCode(fieldsForPreservingFragBitmap); + // Sum entry hashes (XOR key ^ array-content hash) so result is insertion-order-independent. + int mapHash = 0; + for (Map.Entry entry : updatedFragmentOffsets.entrySet()) { + mapHash += Long.hashCode(entry.getKey()) ^ Arrays.hashCode(entry.getValue()); + } + h = 31 * h + mapHash; + return h; } public enum UpdateMode { @@ -115,6 +157,7 @@ public static class Builder { private long[] fieldsModified = new long[0]; private long[] fieldsForPreservingFragBitmap = new long[0]; private Optional updateMode = Optional.empty(); + private Map updatedFragmentOffsets = Collections.emptyMap(); private Builder() {} @@ -148,6 +191,20 @@ public Builder updateMode(Optional updateMode) { return this; } + /** + * Set the per-fragment matched row offsets for a RewriteColumns commit. + * + *

    Keys are fragment ids; values are portable RoaringBitmap bytes (little-endian, + * spec-compliant serialization) encoding the local physical row offsets (0-based) within the + * fragment that matched the update_columns hash join. When non-empty and update mode is + * RewriteColumns with stable row IDs enabled, build_manifest will call the partial last_updated + * refresh for those offsets only. + */ + public Builder updatedFragmentOffsets(Map updatedFragmentOffsets) { + this.updatedFragmentOffsets = updatedFragmentOffsets; + return this; + } + public Update build() { return new Update( removedFragmentIds, @@ -155,7 +212,8 @@ public Update build() { newFragments, fieldsModified, fieldsForPreservingFragBitmap, - updateMode); + updateMode, + updatedFragmentOffsets); } } } diff --git a/java/src/main/java/org/lance/otel/LanceMetrics.java b/java/src/main/java/org/lance/otel/LanceMetrics.java new file mode 100644 index 00000000000..07b8595ff62 --- /dev/null +++ b/java/src/main/java/org/lance/otel/LanceMetrics.java @@ -0,0 +1,283 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +import org.lance.JniLoader; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.metrics.BatchCallback; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.metrics.ObservableDoubleMeasurement; +import io.opentelemetry.api.metrics.ObservableMeasurement; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.logging.Logger; + +/** Bridges Lance's internal Rust metrics into OpenTelemetry observable instruments. */ +public final class LanceMetrics { + private static final Logger LOGGER = Logger.getLogger(LanceMetrics.class.getName()); + private static final String METER_NAME = "lance"; + private static final List INSTRUMENTS = new ArrayList<>(); + private static boolean instrumented; + private static MeterProvider instrumentedProvider; + + private LanceMetrics() {} + + /** + * Register Lance metrics on the global OpenTelemetry {@link MeterProvider}. + * + * @return true if the recorder is installed and instruments are registered, false if a different + * Rust metrics recorder is already installed in this process + */ + public static synchronized boolean instrument() { + return instrument(GlobalOpenTelemetry.get().getMeterProvider()); + } + + /** + * Register Lance metrics as OpenTelemetry observable instruments. + * + *

    This installs a process-global Rust metrics recorder. If a different Rust metrics recorder + * is already installed, this returns false and does not create instruments. Repeated successful + * calls with the same provider are safe and return true without creating duplicate instruments. + * Calls with a different provider close the existing instruments and register new callbacks on + * the supplied provider. + * + * @param meterProvider the OpenTelemetry meter provider that owns the instruments + * @return true if the recorder is installed and instruments are registered, false otherwise + */ + public static synchronized boolean instrument(MeterProvider meterProvider) { + Objects.requireNonNull(meterProvider, "meterProvider"); + JniLoader.ensureLoaded(); + + if (!registerLanceMetricsRecorderNative()) { + LOGGER.warning( + "Could not install the Lance metrics recorder: another Rust metrics recorder is already" + + " installed in this process. Lance metrics will not be exported via" + + " OpenTelemetry."); + return false; + } + + if (instrumented) { + if (instrumentedProvider == meterProvider) { + return true; + } + closeInstruments(); + } + + Meter meter = meterProvider.meterBuilder(METER_NAME).build(); + Map registeredMetrics = new HashMap<>(); + List measurements = new ArrayList<>(); + + for (MetricDescription desc : supportedMetrics(catalog())) { + String unit = desc.getUnit() == null ? "" : desc.getUnit(); + String description = desc.getDescription(); + switch (desc.getKind()) { + case "counter": + ObservableDoubleMeasurement counter = + meter + .counterBuilder(desc.getName()) + .ofDoubles() + .setUnit(unit) + .setDescription(description) + .buildObserver(); + registeredMetrics.put(desc.getName(), RegisteredMetric.scalar(counter)); + measurements.add(counter); + break; + case "gauge": + ObservableDoubleMeasurement gauge = + meter + .gaugeBuilder(desc.getName()) + .setUnit(unit) + .setDescription(description) + .buildObserver(); + registeredMetrics.put(desc.getName(), RegisteredMetric.scalar(gauge)); + measurements.add(gauge); + break; + case "histogram": + ObservableDoubleMeasurement buckets = + meter + .counterBuilder(desc.getName() + "_bucket") + .ofDoubles() + .setDescription(description + " (cumulative buckets)") + .buildObserver(); + ObservableDoubleMeasurement count = + meter + .counterBuilder(desc.getName() + "_count") + .ofDoubles() + .setDescription(description + " (count)") + .buildObserver(); + ObservableDoubleMeasurement sum = + meter + .counterBuilder(desc.getName() + "_sum") + .ofDoubles() + .setUnit(unit) + .setDescription(description + " (sum)") + .buildObserver(); + registeredMetrics.put(desc.getName(), RegisteredMetric.histogram(buckets, count, sum)); + measurements.add(buckets); + measurements.add(count); + measurements.add(sum); + break; + default: + LOGGER.warning( + "Skipping Lance metric " + desc.getName() + " with unknown kind: " + desc.getKind()); + continue; + } + } + + if (!measurements.isEmpty()) { + ObservableMeasurement first = measurements.get(0); + ObservableMeasurement[] rest = + measurements.subList(1, measurements.size()).toArray(new ObservableMeasurement[0]); + BatchCallback callback = + meter.batchCallback(() -> recordSnapshot(registeredMetrics), first, rest); + INSTRUMENTS.add(callback); + } + + instrumented = true; + instrumentedProvider = meterProvider; + return true; + } + + /** Close registered OpenTelemetry callbacks. Rust metric state remains process-global. */ + public static synchronized void close() { + closeInstruments(); + } + + /** Return the catalog of Lance metrics described by the native recorder. */ + public static List catalog() { + JniLoader.ensureLoaded(); + return Collections.unmodifiableList(lanceMetricsCatalogNative()); + } + + /** Return a point-in-time snapshot of all recorded Lance metric series. */ + public static List snapshot() { + JniLoader.ensureLoaded(); + return Collections.unmodifiableList(snapshotLanceMetricsNative()); + } + + static List supportedMetrics(List catalog) { + List supported = new ArrayList<>(); + for (MetricDescription desc : catalog) { + switch (desc.getKind()) { + case "counter": + case "gauge": + case "histogram": + supported.add(desc); + break; + default: + LOGGER.warning( + "Skipping Lance metric " + desc.getName() + " with unknown kind: " + desc.getKind()); + } + } + return supported; + } + + private static void recordSnapshot(Map registeredMetrics) { + for (MetricPoint point : snapshot()) { + RegisteredMetric metric = registeredMetrics.get(point.getName()); + if (metric == null) { + continue; + } + metric.record(point); + } + } + + private static void closeInstruments() { + for (AutoCloseable instrument : INSTRUMENTS) { + try { + instrument.close(); + } catch (Exception e) { + LOGGER.warning("Failed to close Lance OpenTelemetry instrument: " + e.getMessage()); + } + } + INSTRUMENTS.clear(); + instrumented = false; + instrumentedProvider = null; + } + + private static AttributesBuilder attributesBuilder(Map values) { + AttributesBuilder builder = Attributes.builder(); + for (Map.Entry entry : values.entrySet()) { + builder.put(entry.getKey(), entry.getValue()); + } + return builder; + } + + private static Attributes attributes(Map values) { + return attributesBuilder(values).build(); + } + + private static final class RegisteredMetric { + private final ObservableDoubleMeasurement scalar; + private final ObservableDoubleMeasurement buckets; + private final ObservableDoubleMeasurement count; + private final ObservableDoubleMeasurement sum; + + private RegisteredMetric( + ObservableDoubleMeasurement scalar, + ObservableDoubleMeasurement buckets, + ObservableDoubleMeasurement count, + ObservableDoubleMeasurement sum) { + this.scalar = scalar; + this.buckets = buckets; + this.count = count; + this.sum = sum; + } + + private static RegisteredMetric scalar(ObservableDoubleMeasurement measurement) { + return new RegisteredMetric(measurement, null, null, null); + } + + private static RegisteredMetric histogram( + ObservableDoubleMeasurement buckets, + ObservableDoubleMeasurement count, + ObservableDoubleMeasurement sum) { + return new RegisteredMetric(null, buckets, count, sum); + } + + private void record(MetricPoint point) { + if (scalar != null && point.getValue() != null) { + scalar.record(point.getValue(), attributes(point.getAttributes())); + } + if (buckets != null && point.getBuckets() != null) { + for (MetricBucket bucket : point.getBuckets()) { + AttributesBuilder attributes = attributesBuilder(point.getAttributes()); + attributes.put("le", bucket.getLe()); + buckets.record(bucket.getCumulativeCount(), attributes.build()); + } + } + if (count != null && point.getCount() != null) { + count.record(point.getCount().doubleValue(), attributes(point.getAttributes())); + } + if (sum != null && point.getSum() != null) { + sum.record(point.getSum(), attributes(point.getAttributes())); + } + } + } + + private static native boolean registerLanceMetricsRecorderNative(); + + private static native List lanceMetricsCatalogNative(); + + private static native List snapshotLanceMetricsNative(); +} diff --git a/java/src/main/java/org/lance/otel/MetricBucket.java b/java/src/main/java/org/lance/otel/MetricBucket.java new file mode 100644 index 00000000000..4efa1058a30 --- /dev/null +++ b/java/src/main/java/org/lance/otel/MetricBucket.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +/** A cumulative histogram bucket exported with Prometheus-style {@code le} semantics. */ +public final class MetricBucket { + private final String le; + private final long cumulativeCount; + + public MetricBucket(String le, long cumulativeCount) { + this.le = le; + this.cumulativeCount = cumulativeCount; + } + + public String getLe() { + return le; + } + + public long getCumulativeCount() { + return cumulativeCount; + } +} diff --git a/java/src/main/java/org/lance/otel/MetricDescription.java b/java/src/main/java/org/lance/otel/MetricDescription.java new file mode 100644 index 00000000000..a350e2be963 --- /dev/null +++ b/java/src/main/java/org/lance/otel/MetricDescription.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +/** Metadata for one Lance metric available through the OpenTelemetry bridge. */ +public final class MetricDescription { + private final String name; + private final String kind; + private final String unit; + private final String description; + + public MetricDescription(String name, String kind, String unit, String description) { + this.name = name; + this.kind = kind; + this.unit = unit; + this.description = description; + } + + public String getName() { + return name; + } + + public String getKind() { + return kind; + } + + public String getUnit() { + return unit; + } + + public String getDescription() { + return description; + } +} diff --git a/java/src/main/java/org/lance/otel/MetricPoint.java b/java/src/main/java/org/lance/otel/MetricPoint.java new file mode 100644 index 00000000000..f9a44b95f20 --- /dev/null +++ b/java/src/main/java/org/lance/otel/MetricPoint.java @@ -0,0 +1,75 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** A cumulative Lance metric data point captured by the native metrics recorder. */ +public final class MetricPoint { + private final String name; + private final String kind; + private final Map attributes; + private final Double value; + private final List buckets; + private final Long count; + private final Double sum; + + public MetricPoint( + String name, + String kind, + Map attributes, + Double value, + List buckets, + Long count, + Double sum) { + this.name = name; + this.kind = kind; + this.attributes = + attributes == null ? Collections.emptyMap() : Collections.unmodifiableMap(attributes); + this.value = value; + this.buckets = buckets == null ? null : Collections.unmodifiableList(buckets); + this.count = count; + this.sum = sum; + } + + public String getName() { + return name; + } + + public String getKind() { + return kind; + } + + public Map getAttributes() { + return attributes; + } + + public Double getValue() { + return value; + } + + public List getBuckets() { + return buckets; + } + + public Long getCount() { + return count; + } + + public Double getSum() { + return sum; + } +} diff --git a/java/src/test/java/org/lance/AsyncScannerTest.java b/java/src/test/java/org/lance/AsyncScannerTest.java index fc786ff57c2..5b75a22cee2 100644 --- a/java/src/test/java/org/lance/AsyncScannerTest.java +++ b/java/src/test/java/org/lance/AsyncScannerTest.java @@ -18,6 +18,7 @@ import org.lance.index.IndexType; import org.lance.index.scalar.ScalarIndexParams; import org.lance.ipc.AsyncScanner; +import org.lance.ipc.MaterializationStyle; import org.lance.ipc.ScanOptions; import org.apache.arrow.memory.BufferAllocator; @@ -32,6 +33,9 @@ import java.io.BufferedReader; import java.io.InputStreamReader; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Path; @@ -40,6 +44,8 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -47,6 +53,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -68,6 +75,29 @@ static void tearDown() { } } + @Test + @SuppressWarnings("unchecked") + void testNativeScanFailureUsesLanceException() throws Exception { + Constructor constructor = AsyncScanner.class.getDeclaredConstructor(); + constructor.setAccessible(true); + AsyncScanner scanner = constructor.newInstance(); + + Field pendingTasksField = AsyncScanner.class.getDeclaredField("pendingTasks"); + pendingTasksField.setAccessible(true); + ConcurrentHashMap> pendingTasks = + (ConcurrentHashMap>) pendingTasksField.get(scanner); + CompletableFuture pendingTask = new CompletableFuture<>(); + pendingTasks.put(1L, pendingTask); + + Method failTask = AsyncScanner.class.getDeclaredMethod("failTask", long.class, String.class); + failTask.setAccessible(true); + failTask.invoke(scanner, 1L, "scan I/O failed"); + + CompletionException failure = assertThrows(CompletionException.class, pendingTask::join); + assertEquals(LanceException.class, failure.getCause().getClass()); + assertEquals("scan I/O failed", failure.getCause().getMessage()); + } + /** * Example 1: Basic async scan with CompletableFuture. * @@ -142,6 +172,35 @@ void testAsyncScanWithFilter(@TempDir Path tempDir) throws Exception { } } + @Test + void testAsyncScanWithPerformanceOptions(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("async_scanner_performance_options").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + try (Dataset dataset = testDataset.write(1, 40)) { + ScanOptions options = + new ScanOptions.Builder() + .filter("id < 20") + .batchSize(10) + .batchSizeBytes(1024) + .ioBufferSize(1024 * 1024) + .batchReadahead(2) + .fragmentReadahead(2) + .scanInOrder(false) + .lateMaterialization(MaterializationStyle.allEarly()) + .build(); + + try (AsyncScanner scanner = AsyncScanner.create(dataset, options, allocator); + ArrowReader reader = scanner.scanBatchesAsync().get(10, TimeUnit.SECONDS)) { + assertEquals(20, countRows(reader)); + } + } + } + } + @Test void testFastSearchSkipsUnindexedFragments(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("async_scanner_fast_search_scalar_index").toString(); diff --git a/java/src/test/java/org/lance/CleanupTest.java b/java/src/test/java/org/lance/CleanupTest.java index 5fc8ceeaa3f..f6f3f0ef043 100644 --- a/java/src/test/java/org/lance/CleanupTest.java +++ b/java/src/test/java/org/lance/CleanupTest.java @@ -54,6 +54,31 @@ public void testCleanupBeforeVersion(@TempDir Path tempDir) { } } + @Test + public void testCleanupSpecificVersions(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("test_dataset_for_cleanup").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + + testDataset.createEmptyDataset().close(); + + testDataset.write(1, 10).close(); + testDataset.write(2, 10).close(); + + try (Dataset dataset = testDataset.write(3, 10)) { + assertEquals(4, dataset.listVersions().size()); + + RemovalStats stats = + dataset.cleanupWithPolicy(CleanupPolicy.builder().withVersions(List.of(2L)).build()); + + assertEquals(1L, stats.getOldVersions()); + assertEquals(3, dataset.listVersions().size()); + assertTrue(dataset.listVersions().stream().noneMatch(version -> version.getId() == 2L)); + } + } + } + @Test public void testExplainCleanupBeforeVersion(@TempDir Path tempDir) { String datasetPath = tempDir.resolve("test_dataset_for_cleanup").toString(); diff --git a/java/src/test/java/org/lance/CommitBuilderStorageFormatTest.java b/java/src/test/java/org/lance/CommitBuilderStorageFormatTest.java new file mode 100644 index 00000000000..b500369cbb9 --- /dev/null +++ b/java/src/test/java/org/lance/CommitBuilderStorageFormatTest.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance; + +import org.lance.operation.Append; +import org.lance.operation.Delete; +import org.lance.operation.OperationTestBase; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class CommitBuilderStorageFormatTest extends OperationTestBase { + + /** + * Append to a freshly created (2.1) dataset with the given storage format and return the + * committed dataset's format version. + */ + private String commitWithStorageFormat(String datasetPath, String storageFormat) + throws Exception { + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + FragmentMetadata fragment = testDataset.createNewFragment(10); + try (Transaction txn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Append.builder().fragments(Collections.singletonList(fragment)).build()) + .build()) { + try (Dataset committed = + new CommitBuilder(dataset).storageFormat(storageFormat).execute(txn)) { + assertEquals(2, committed.version()); + return committed.getLanceFileFormatVersion(); + } + } + } + } + + /** + * The numeric versions are what {@link Dataset#getLanceFileFormatVersion()} returns and what + * {@link WriteParams.Builder#withDataStorageVersion(String)} accepts, so they must work here too + * — a caller that encodes fragments as "2.1" has to be able to commit them as "2.1". + */ + @Test + void testCanonicalVersionAccepted(@TempDir Path tempDir) throws Exception { + assertEquals( + LanceConstants.FILE_FORMAT_VERSION_2_1, + commitWithStorageFormat( + tempDir.resolve("canonical").toString(), LanceConstants.FILE_FORMAT_VERSION_2_1)); + } + + /** The "v"-prefixed spelling shipped in this method's Javadoc and stays accepted. */ + @Test + void testDeprecatedAliasAccepted(@TempDir Path tempDir) throws Exception { + assertEquals( + LanceConstants.FILE_FORMAT_VERSION_2_1, + commitWithStorageFormat(tempDir.resolve("alias").toString(), "v2_1")); + } + + /** + * A delete adds no data files, so nothing about it depends on the storage format — but {@link + * CommitBuilder#storageFormat(String)} is still validated against the existing dataset for any + * operation other than overwrite. A caller that forwards a configured format on every commit hits + * this on row-level operations against a table written in a different version, so the failure is + * a mismatch error rather than anything to do with parsing. + */ + @Test + void testMismatchedFormatRejectedOnRowLevelOperation(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("mismatch").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + FragmentMetadata fragment = testDataset.createNewFragment(10); + try (Transaction appendTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Append.builder().fragments(Collections.singletonList(fragment)).build()) + .build()) { + dataset = new CommitBuilder(dataset).execute(appendTxn); + } + assertEquals(LanceConstants.FILE_FORMAT_VERSION_2_1, dataset.getLanceFileFormatVersion()); + + List fragmentIds = + dataset.getFragments().stream() + .map(f -> Long.valueOf(f.getId())) + .collect(Collectors.toList()); + + // "2.2" parses fine, so a failure here is the mismatch guard and not the parser. + try (Transaction deleteTxn = deleteAll(fragmentIds)) { + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, + () -> + new CommitBuilder(dataset) + .storageFormat(LanceConstants.FILE_FORMAT_VERSION_2_2) + .execute(deleteTxn)); + assertTrue(error.getMessage().contains("Storage format mismatch"), error.getMessage()); + } + + // The same delete succeeds when the format agrees with the dataset. + try (Transaction deleteTxn = deleteAll(fragmentIds)) { + try (Dataset deleted = + new CommitBuilder(dataset) + .storageFormat(LanceConstants.FILE_FORMAT_VERSION_2_1) + .execute(deleteTxn)) { + assertEquals(0, deleted.countRows()); + } + } + } + } + + private Transaction deleteAll(List fragmentIds) { + return new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Delete.builder().deletedFragmentIds(fragmentIds).predicate("1=1").build()) + .build(); + } + + @Test + void testUnknownFormatRejected(@TempDir Path tempDir) { + assertThrows( + IllegalArgumentException.class, + () -> commitWithStorageFormat(tempDir.resolve("bogus").toString(), "bogus")); + // The alias set is frozen at what shipped, so it does not extend to newer versions. + assertThrows( + IllegalArgumentException.class, + () -> commitWithStorageFormat(tempDir.resolve("v23").toString(), "v2_3")); + } +} diff --git a/java/src/test/java/org/lance/CompactionTest.java b/java/src/test/java/org/lance/CompactionTest.java index 9ac96804a61..a0d58612f79 100644 --- a/java/src/test/java/org/lance/CompactionTest.java +++ b/java/src/test/java/org/lance/CompactionTest.java @@ -15,6 +15,7 @@ import org.lance.compaction.Compaction; import org.lance.compaction.CompactionMetrics; +import org.lance.compaction.CompactionMode; import org.lance.compaction.CompactionOptions; import org.lance.compaction.CompactionPlan; import org.lance.compaction.CompactionTask; @@ -23,6 +24,8 @@ import org.apache.arrow.memory.RootAllocator; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -30,7 +33,10 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Base64; import java.util.Collections; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -49,9 +55,21 @@ public void testBasicCompaction(@TempDir Path tempDir) throws Exception { testDataset.write(1, 10).close(); try (Dataset dataset = testDataset.write(2, 10)) { CompactionOptions compactionOptions = - CompactionOptions.builder().withTargetRowsPerFragment(100).withNumThreads(1).build(); + CompactionOptions.builder() + .withTargetRowsPerFragment(100) + .withNumThreads(1) + .withMaxSourceRows(1000) + .withMaxSourceBytes(10L * 1024 * 1024) + .build(); CompactionPlan compactionPlan = Compaction.planCompaction(dataset, compactionOptions); + // The source budgets are loose, so the plan is unaffected and the + // options must survive the JNI round trip. + assertEquals(Optional.of(1000L), compactionPlan.getCompactionOptions().getMaxSourceRows()); + assertEquals( + Optional.of(10L * 1024 * 1024), + compactionPlan.getCompactionOptions().getMaxSourceBytes()); + // will plan to compact two fragments into one. assertEquals(1, compactionPlan.getCompactionTasks().size()); CompactionTask task = compactionPlan.getCompactionTasks().get(0); @@ -130,6 +148,109 @@ public void testDeletionCompaction(@TempDir Path tempDir) throws Exception { } } + @Test + public void testExcludedFragmentIds(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("test_excluded_fragment_ids").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + testDataset.write(2, 10).close(); + testDataset.write(3, 10).close(); + try (Dataset dataset = testDataset.write(4, 10)) { + CompactionOptions options = + CompactionOptions.builder() + .withTargetRowsPerFragment(100) + .withExcludedFragmentIds(Arrays.asList(1L, 1L, 999L)) + .build(); + + CompactionPlan plan = Compaction.planCompaction(dataset, options); + + assertEquals( + Arrays.asList(1L, 1L, 999L), plan.getCompactionOptions().getExcludedFragmentIds()); + assertEquals(1, plan.getCompactionTasks().size()); + assertEquals(2, plan.getCompactionTasks().get(0).getTaskData().getFragments().size()); + assertEquals( + 2, plan.getCompactionTasks().get(0).getTaskData().getFragments().get(0).getId()); + assertEquals( + 3, plan.getCompactionTasks().get(0).getTaskData().getFragments().get(1).getId()); + + CompactionTask task = serializeAndDeserialize(plan.getCompactionTasks().get(0)); + assertEquals( + Arrays.asList(1L, 1L, 999L), task.getCompactionOptions().getExcludedFragmentIds()); + } + } + } + + @ParameterizedTest + @EnumSource(CompactionMode.class) + public void testCompactionModeRoundTrip(CompactionMode mode, @TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("test_dataset_for_compaction").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + CompactionOptions compactionOptions = + CompactionOptions.builder() + .withTargetRowsPerFragment(100) + .withNumThreads(1) + .withCompactionMode(mode) + .build(); + CompactionPlan compactionPlan = Compaction.planCompaction(dataset, compactionOptions); + + // The plan's options are rebuilt by the native layer; the mode must come + // back as a CompactionMode enum, not a raw String. + assertEquals( + Optional.of(mode.getValue()), + compactionPlan.getCompactionOptions().getCompactionMode()); + + CompactionTask task = serializeAndDeserialize(compactionPlan.getCompactionTasks().get(0)); + RewriteResult result = task.execute(dataset); + assertEquals(2, result.getMetrics().getFragmentsRemoved()); + assertEquals(1, result.getMetrics().getFragmentsAdded()); + } + } + } + + /** + * A serialized CompactionOptions produced by the class as it existed before maxSourceRows and + * maxSourceBytes were added (no declared serialVersionUID, stream ends after maxSourceFragments), + * built with targetRowsPerFragment=1024, materializeDeletions=true, + * compactionMode=TRY_BINARY_COPY, maxSourceFragments=4. + */ + private static final String PRE_SOURCE_BUDGET_OPTIONS_BASE64 = + "rO0ABXNyACZvcmcubGFuY2UuY29tcGFjdGlvbi5Db21wYWN0aW9uT3B0aW9ucys6bRwua1fWAwALTAAJYmF0Y2hTaXpl" + + "dAAUTGphdmEvdXRpbC9PcHRpb25hbDtMABhiaW5hcnlDb3B5UmVhZEJhdGNoQnl0ZXNxAH4AAUwADmNvbXBhY3Rpb25N" + + "b2RlcQB+AAFMAA9kZWZlckluZGV4UmVtYXBxAH4AAUwAFG1hdGVyaWFsaXplRGVsZXRpb25zcQB+AAFMAB1tYXRlcmlh" + + "bGl6ZURlbGV0aW9uc1RocmVzaG9sZHEAfgABTAAPbWF4Qnl0ZXNQZXJGaWxlcQB+AAFMAA9tYXhSb3dzUGVyR3JvdXBx" + + "AH4AAUwAEm1heFNvdXJjZUZyYWdtZW50c3EAfgABTAAKbnVtVGhyZWFkc3EAfgABTAAVdGFyZ2V0Um93c1BlckZyYWdt" + + "ZW50cQB+AAF4cHNyAA5qYXZhLmxhbmcuTG9uZzuL5JDMjyPfAgABSgAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoas" + + "lR0LlOCLAgAAeHAAAAAAAAAEAHBwc3IAEWphdmEubGFuZy5Cb29sZWFuzSBygNWc+u4CAAFaAAV2YWx1ZXhwAXBwcHB0" + + "AA90cnlfYmluYXJ5X2NvcHlwc3EAfgADAAAAAAAAAAR4"; + + @Test + public void testDeserializeOptionsFromOlderVersion() throws Exception { + byte[] serialized = Base64.getDecoder().decode(PRE_SOURCE_BUDGET_OPTIONS_BASE64); + CompactionOptions options; + try (ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(serialized))) { + options = (CompactionOptions) in.readObject(); + } + assertEquals(Optional.of(1024L), options.getTargetRowsPerFragment()); + assertEquals(Optional.of(true), options.getMaterializeDeletions()); + assertEquals( + Optional.of(CompactionMode.TRY_BINARY_COPY.getValue()), options.getCompactionMode()); + assertEquals(Optional.of(4L), options.getMaxSourceFragments()); + // Fields absent from the old stream deserialize as unset. + assertEquals(Optional.empty(), options.getMaxSourceRows()); + assertEquals(Optional.empty(), options.getMaxSourceBytes()); + assertEquals(Collections.emptyList(), options.getExcludedFragmentIds()); + } + private static T serializeAndDeserialize(T object) throws IOException, ClassNotFoundException { ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); diff --git a/java/src/test/java/org/lance/DatasetTest.java b/java/src/test/java/org/lance/DatasetTest.java index 45466a0367c..ff78d3cf74f 100644 --- a/java/src/test/java/org/lance/DatasetTest.java +++ b/java/src/test/java/org/lance/DatasetTest.java @@ -80,6 +80,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -128,6 +129,9 @@ void testGetLanceFileFormatVersion(@TempDir Path tempDir) { new TestUtils.SimpleTestDataset(allocator, defaultPath); try (Dataset dataset = testDataset.createEmptyDataset()) { assertEquals(LanceConstants.FILE_FORMAT_VERSION_2_1, dataset.getLanceFileFormatVersion()); + WriterVersion writerVersion = dataset.getWriterVersion().orElseThrow(AssertionError::new); + assertEquals("lance", writerVersion.getLibrary()); + assertFalse(writerVersion.getVersion().isEmpty()); } // Test LEGACY version @@ -143,6 +147,67 @@ void testGetLanceFileFormatVersion(@TempDir Path tempDir) { assertEquals( LanceConstants.FILE_FORMAT_VERSION_0_1, legacyDataset.getLanceFileFormatVersion()); } + + // This dataset was written before writer_version was added to the manifest. + String historicalPath = + Path.of("..", "test_data", "v0.7.5", "with_deletions") + .toAbsolutePath() + .normalize() + .toString(); + try (Dataset historicalDataset = Dataset.open(historicalPath, allocator)) { + assertTrue(historicalDataset.getWriterVersion().isEmpty()); + } + + // This fixture was written by lance 2.0.0-beta.1. Reading it through Dataset verifies + // that the manifest's prerelease qualifier survives the Rust-to-Java JNI mapping. + String prereleasePath = + Path.of("..", "test_data", "pre_file_sizes", "index_without_file_sizes") + .toAbsolutePath() + .normalize() + .toString(); + try (Dataset prereleaseDataset = Dataset.open(prereleasePath, allocator)) { + WriterVersion writerVersion = + prereleaseDataset.getWriterVersion().orElseThrow(AssertionError::new); + assertEquals("lance", writerVersion.getLibrary()); + assertEquals("2.0.0", writerVersion.getVersion()); + assertEquals("beta.1", writerVersion.getPrerelease().orElseThrow(AssertionError::new)); + assertTrue(writerVersion.getBuildMetadata().isEmpty()); + } + } + } + + @Test + void testWriterVersionPreservesOpaqueAndOptionalFields() { + WriterVersion writerVersion = + new WriterVersion("custom-writer", "release-2026", "preview.1", "build.42"); + + assertEquals("custom-writer", writerVersion.getLibrary()); + assertEquals("release-2026", writerVersion.getVersion()); + assertEquals("preview.1", writerVersion.getPrerelease().orElseThrow(AssertionError::new)); + assertEquals("build.42", writerVersion.getBuildMetadata().orElseThrow(AssertionError::new)); + } + + @Test + void testListManifestLocations(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("manifest_locations").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 1).close(); + + List manifests = Dataset.listManifestLocations(datasetPath); + assertEquals(2, manifests.size()); + assertEquals( + Set.of(1L, 2L), + manifests.stream().map(ManifestLocation::getVersion).collect(Collectors.toSet())); + for (ManifestLocation manifest : manifests) { + assertTrue(manifest.getPath().contains("manifest_locations/_versions/")); + assertFalse(manifest.getPath().startsWith("_versions/")); + assertTrue(manifest.getPath().endsWith(".manifest")); + assertTrue(manifest.getSizeBytes() > 0); + assertNotNull(manifest.getNamingScheme()); + } } } @@ -240,6 +305,9 @@ void testDatasetVersion(@TempDir Path tempDir) { List versions = dataset.listVersions(); assertEquals(3, versions.size()); + assertEquals(3, dataset.getVersionCount()); + assertEquals(3, dataset2.getVersionCount()); + assertEquals(3, dataset3.getVersionCount()); assertEquals(1, versions.get(0).getId()); assertEquals(2, versions.get(1).getId()); assertEquals(3, versions.get(2).getId()); @@ -659,6 +727,40 @@ void testAlterColumns(@TempDir Path tempDir) { } } + @Test + void testAlterColumnsCastType(@TempDir Path tempDir) { + String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName(); + String datasetPath = tempDir.resolve(testMethodName).toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Widen "id" from Int32 to Int64. The cast target type is a parameterized ArrowType, which + // must survive the trip to the native side; regression test for a dropped cast that left the + // stored type unchanged. + ColumnAlteration widenId = + new ColumnAlteration.Builder("id").castTo(new ArrowType.Int(64, true)).build(); + dataset.alterColumns(Collections.singletonList(widenId)); + + assertEquals(new ArrowType.Int(64, true), dataset.getSchema().findField("id").getType()); + + // A cast combined with rename must apply both. + ColumnAlteration renameAndWiden = + new ColumnAlteration.Builder("id") + .rename("id_long") + .castTo(new ArrowType.Int(64, true)) + .build(); + dataset.alterColumns(Collections.singletonList(renameAndWiden)); + + List fieldNames = + dataset.getSchema().getFields().stream().map(Field::getName).collect(Collectors.toList()); + assertFalse(fieldNames.contains("id")); + assertTrue(fieldNames.contains("id_long")); + assertEquals(new ArrowType.Int(64, true), dataset.getSchema().findField("id_long").getType()); + } + } + @Test void testAddColumnBySqlExpressions(@TempDir Path tempDir) { String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName(); @@ -879,6 +981,23 @@ void testDropPath(@TempDir Path tempDir) { } } + @Test + void testDropRejectsNonDatasetPath(@TempDir Path tempDir) { + Path warehouse = tempDir.resolve("warehouse"); + Path tablePath = warehouse.resolve("table.lance"); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, tablePath.toString()); + dataset = testDataset.createEmptyDataset(); + + // Pointing at the parent of a dataset must not wipe out the whole warehouse. + assertThrows( + IllegalArgumentException.class, + () -> Dataset.drop(warehouse.toString(), new HashMap<>())); + assertTrue(Files.exists(tablePath)); + } + } + @Test void testTake(@TempDir Path tempDir) throws IOException, ClosedChannelException { String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName(); @@ -2026,6 +2145,16 @@ void testReadZeroLengthBlob(@TempDir Path tempDir) throws Exception { } } + @Test + void testTakeNullBlobPreservesSelection(@TempDir Path tempDir) throws Exception { + String base = tempDir.resolve("testTakeNullBlobPreservesSelection").toString(); + try (Dataset ds = TestUtils.createBlobDataset(base, 128, 8)) { + List blobs = ds.takeBlobsByIndices(Collections.singletonList(15L), "blobs"); + assertEquals(1, blobs.size()); + assertNull(blobs.get(0)); + } + } + @Test void testReadLargeBlobAndRanges(@TempDir Path tempDir) throws Exception { String base = tempDir.resolve("testReadLargeBlobAndRanges").toString(); @@ -2142,6 +2271,17 @@ public void testDescribeIndicesByName(@TempDir Path tempDir) throws Exception { assertEquals(1, desc.getSegments().size(), "Expected exactly one physical segment"); assertEquals("index1", desc.getSegments().get(0).name()); + assertEquals( + Collections.emptyList(), + desc.getSegments().get(0).coveringFields(), + "no covering columns are declared yet"); + assertTrue( + desc.getSegments().get(0).getSizeBytes().orElse(0L) > 0, + "segment size should be positive"); + assertEquals( + desc.getSegments().get(0).getSizeBytes(), + desc.getTotalSizeBytes(), + "single-segment size should equal the logical index size"); descriptions = dataset.describeIndices(); assertEquals(2, descriptions.size(), "Expected exactly one matching index"); @@ -2154,6 +2294,8 @@ public void testDescribeIndicesByName(@TempDir Path tempDir) throws Exception { indexDesc.getSegments(), "segments alias should match metadata"); assertNotNull(indexDesc.getDetailsJson(), "Details JSON should not be null"); + assertTrue( + indexDesc.getTotalSizeBytes().orElse(0L) > 0, "total index size should be positive"); } } } diff --git a/java/src/test/java/org/lance/DeltaTest.java b/java/src/test/java/org/lance/DeltaTest.java index ac7056840e4..da4fd512466 100755 --- a/java/src/test/java/org/lance/DeltaTest.java +++ b/java/src/test/java/org/lance/DeltaTest.java @@ -188,6 +188,57 @@ public void testListTransactionsExplicitRange(@TempDir Path tempDir) throws IOEx } } + @Test + public void testGetDeletedRowIds(@TempDir Path tempDir) throws IOException { + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + String uri = tempDir.resolve("delta_demo_delete").toString(); + Schema schema = + new Schema( + Arrays.asList( + Field.notNullable( + "id", new org.apache.arrow.vector.types.pojo.ArrowType.Int(32, true)), + Field.nullable( + "val", org.apache.arrow.vector.types.pojo.ArrowType.Utf8.INSTANCE))); + + // v1: create with three rows, keeping stable row ids so deletions are reportable. + byte[] batch1 = + writeBatch(allocator, schema, new int[] {1, 2, 3}, new String[] {"a", "b", "c"}); + try (ArrowStreamReader reader1 = + new ArrowStreamReader(new ByteArrayReadableSeekableByteChannel(batch1), allocator); + ArrowArrayStream stream1 = ArrowArrayStream.allocateNew(allocator)) { + Data.exportArrayStream(allocator, reader1, stream1); + Dataset.write().stream(stream1) + .uri(uri) + .mode(WriteParams.WriteMode.CREATE) + .enableStableRowIds(true) + .execute() + .close(); + } + + // v2: delete one row. + try (Dataset ds = Dataset.open(uri, allocator)) { + ds.delete("id = 2"); + } + + try (Dataset ds2 = Dataset.open(uri, allocator)) { + DatasetDelta delta = ds2.delta(1L); + try (ArrowReader deleted = delta.getDeletedRowIds()) { + int total = 0; + while (deleted.loadNextBatch()) { + VectorSchemaRoot outRoot = deleted.getVectorSchemaRoot(); + List names = + outRoot.getSchema().getFields().stream() + .map(Field::getName) + .collect(Collectors.toList()); + Assertions.assertEquals(Arrays.asList("_rowid"), names); + total += outRoot.getRowCount(); + } + Assertions.assertEquals(1, total, "exactly one row was deleted"); + } + } + } + } + /** Helper: serialize a single Arrow batch with the given schema and (id, val) pairs. */ private static byte[] writeBatch(RootAllocator allocator, Schema schema, int[] ids, String[] vals) throws IOException { diff --git a/java/src/test/java/org/lance/FileReaderWriterTest.java b/java/src/test/java/org/lance/FileReaderWriterTest.java index a849a87c576..85c7430f087 100644 --- a/java/src/test/java/org/lance/FileReaderWriterTest.java +++ b/java/src/test/java/org/lance/FileReaderWriterTest.java @@ -256,7 +256,9 @@ void testInvalidPath() { LanceFileReader.open("/tmp/does_not_exist.lance", allocator); fail("Expected LanceException to be thrown"); } catch (IOException e) { - assertTrue(e.getMessage().contains("Object at location /tmp/does_not_exist.lance not found")); + String message = e.getMessage(); + assertTrue(message.contains("/tmp/does_not_exist.lance")); + assertTrue(message.toLowerCase().contains("not found")); } try { LanceFileReader.open("", allocator); diff --git a/java/src/test/java/org/lance/FragmentTest.java b/java/src/test/java/org/lance/FragmentTest.java index 29a21b5258a..b97cee9241a 100644 --- a/java/src/test/java/org/lance/FragmentTest.java +++ b/java/src/test/java/org/lance/FragmentTest.java @@ -39,6 +39,11 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertArrayEquals; @@ -150,6 +155,64 @@ void testWriteFragmentWithSchemaOverride(@TempDir Path tempDir) throws Exception } } + @Test + void testWriteFragmentWithSession(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("fragment_with_session").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + Session session = Session.builder().build()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + long sizeBefore = session.sizeBytes(); + try (VectorSchemaRoot root = VectorSchemaRoot.create(testDataset.getSchema(), allocator)) { + root.allocateNew(); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + IntVector idVector = (IntVector) root.getVector("id"); + nameVector.setSafe(0, "Person 1".getBytes(StandardCharsets.UTF_8)); + idVector.setSafe(0, 1); + root.setRowCount(1); + + // First append in APPEND mode without an explicit schema: the + // manifest load for schema inference populates the shared session's + // metadata cache. + List firstFragments = + appendWithSession(datasetPath, allocator, root, session); + assertEquals(1, firstFragments.size()); + assertEquals(1, firstFragments.get(0).getPhysicalRows()); + assertTrue(session.sizeBytes() > sizeBefore); + long hitsAfterFirst = session.metadataCacheStats().getHits(); + + // Second append through the same session: schema inference reads the + // manifest cached by the first write, so cache hits must increase. + List secondFragments = + appendWithSession(datasetPath, allocator, root, session); + assertEquals(1, secondFragments.size()); + assertEquals(1, secondFragments.get(0).getPhysicalRows()); + assertTrue(session.metadataCacheStats().getHits() > hitsAfterFirst); + + // A closed session has a zero native handle and degrades to "no + // session", matching Dataset's behavior for closed sessions. + Session closedSession = Session.builder().build(); + closedSession.close(); + List fragmentsWithClosedSession = + appendWithSession(datasetPath, allocator, root, closedSession); + assertEquals(1, fragmentsWithClosedSession.size()); + } + } + } + + private static List appendWithSession( + String datasetPath, RootAllocator allocator, VectorSchemaRoot root, Session session) { + return Fragment.write() + .datasetUri(datasetPath) + .allocator(allocator) + .data(root) + .mode(WriteParams.WriteMode.APPEND) + .session(session) + .execute(); + } + @Test void commitWithoutVersion(@TempDir Path tempDir) { String datasetPath = tempDir.resolve("commit_without_version").toString(); @@ -413,4 +476,114 @@ void testMergeColumns(@TempDir Path tempDir) throws Exception { } } } + + @Test + void testFragmentStatistics(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("fragment_statistics").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + // Two fragments with different row counts + FragmentMetadata frag1 = testDataset.createNewFragment(21); + FragmentMetadata frag2 = testDataset.createNewFragment(9); + FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(frag1, frag2)); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L))) { + List fragments = dataset.getFragments(); + FragmentStatistics stats = dataset.getFragmentStatistics(); + assertEquals(fragments.size(), stats.size()); + + // Parity with getFragments across all three arrays + assertArrayEquals(fragments.stream().mapToInt(Fragment::getId).toArray(), stats.getIds()); + assertArrayEquals( + fragments.stream().mapToLong(f -> f.metadata().getNumRows()).toArray(), + stats.getRowCounts()); + assertArrayEquals( + fragments.stream().mapToInt(f -> f.metadata().getFiles().size()).toArray(), + stats.getDataFileNums()); + + assertEquals(30, Arrays.stream(stats.getRowCounts()).sum()); + + dataset.delete("id < 5"); + assertArrayEquals(new long[] {16, 4}, dataset.getFragmentStatistics().getRowCounts()); + } + } + } + + @Test + void testFragmentStatisticsOnEmptyDataset(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("fragment_statistics_empty").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + try (Dataset dataset = testDataset.createEmptyDataset()) { + assertEquals(0, dataset.getFragmentStatistics().size()); + } + } + } + + @Test + void testFragmentStatisticsPreservesLegacyMissingRowCount() { + String historicalPath = + Path.of("..", "test_data", "v0.7.5", "with_deletions") + .toAbsolutePath() + .normalize() + .toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE); + Dataset dataset = Dataset.open(historicalPath, allocator)) { + FragmentStatistics stats = dataset.getFragmentStatistics(); + assertArrayEquals(new int[] {0}, stats.getIds()); + assertArrayEquals(new long[] {0}, stats.getRowCounts()); + assertArrayEquals(new int[] {1}, stats.getDataFileNums()); + } + } + + @Test + void testCountRowsConcurrentWithClose(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("count_rows_close_race").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + FragmentMetadata fragmentMeta = testDataset.createNewFragment(100); + FragmentOperation.Append appendOp = new FragmentOperation.Append(Arrays.asList(fragmentMeta)); + Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L)); + Fragment fragment = dataset.getFragments().get(0); + + int threadCount = 8; + ExecutorService executor = Executors.newFixedThreadPool(threadCount); + try { + CountDownLatch start = new CountDownLatch(1); + List> futures = new ArrayList<>(); + for (int i = 0; i < threadCount; i++) { + futures.add( + executor.submit( + () -> { + start.await(); + // Hammer countRows until close() wins the race. The only acceptable + // failure is the "Dataset is closed" rejection; anything else (a native + // crash or "Null pointer in rust value from Java") means the native + // handle was released while still in use. + while (true) { + try { + fragment.countRows(); + } catch (IllegalArgumentException e) { + assertEquals("Dataset is closed", e.getMessage()); + return null; + } + } + })); + } + start.countDown(); + Thread.sleep(50); + dataset.close(); + for (Future future : futures) { + future.get(30, TimeUnit.SECONDS); + } + } finally { + executor.shutdownNow(); + } + } + } } diff --git a/java/src/test/java/org/lance/JNITest.java b/java/src/test/java/org/lance/JNITest.java index daa123b3200..94db13d6dea 100644 --- a/java/src/test/java/org/lance/JNITest.java +++ b/java/src/test/java/org/lance/JNITest.java @@ -18,6 +18,7 @@ import org.lance.index.vector.HnswBuildParams; import org.lance.index.vector.IvfBuildParams; import org.lance.index.vector.PQBuildParams; +import org.lance.index.vector.RQBuildParams; import org.lance.index.vector.SQBuildParams; import org.lance.index.vector.VectorIndexParams; import org.lance.ipc.ApproxMode; @@ -78,6 +79,11 @@ public void testIvfFlatIndexParams() { .build()); } + @Test + public void testRqBuildParamsDefaultNumBits() { + assertEquals((byte) 5, new RQBuildParams.Builder().build().getNumBits()); + } + @Test public void testIvfPqIndexParams() { JniTestHelper.parseIndexParams( diff --git a/java/src/test/java/org/lance/ScannerTest.java b/java/src/test/java/org/lance/ScannerTest.java index 00434034b64..dd4998b63cd 100644 --- a/java/src/test/java/org/lance/ScannerTest.java +++ b/java/src/test/java/org/lance/ScannerTest.java @@ -19,9 +19,12 @@ import org.lance.index.scalar.ScalarIndexParams; import org.lance.ipc.ColumnOrdering; import org.lance.ipc.LanceScanner; +import org.lance.ipc.MaterializationStyle; import org.lance.ipc.ScanOptions; import org.lance.ipc.ScanStats; +import org.apache.arrow.c.ArrowArrayStream; +import org.apache.arrow.c.Data; import org.apache.arrow.dataset.scanner.Scanner; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; @@ -37,8 +40,11 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -46,9 +52,11 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class ScannerTest { @@ -65,6 +73,90 @@ static void tearDown() { } } + @Test + void testScannerPerformanceOptions() { + MaterializationStyle materialization = + MaterializationStyle.allEarlyExcept(Collections.singletonList("name")); + ScanOptions options = + new ScanOptions.Builder() + .batchSize(1024) + .batchSizeBytes(64 * 1024) + .ioBufferSize(8 * 1024 * 1024) + .batchReadahead(4) + .fragmentReadahead(2) + .scanInOrder(false) + .lateMaterialization(materialization) + .build(); + + assertEquals(1024L, options.getBatchSize().orElseThrow()); + assertEquals(64 * 1024L, options.getBatchSizeBytes().orElseThrow()); + assertEquals(8 * 1024 * 1024L, options.getIoBufferSize().orElseThrow()); + assertEquals(4, options.getBatchReadahead()); + assertEquals(2, options.getFragmentReadahead().orElseThrow()); + assertFalse(options.isScanInOrder()); + assertEquals(materialization, options.getLateMaterialization().orElseThrow()); + assertEquals("heuristic", MaterializationStyle.heuristic().toRustString()); + assertEquals("all_late", MaterializationStyle.allLate().toRustString()); + assertEquals("all_early", MaterializationStyle.allEarly().toRustString()); + assertEquals("all_early_except", materialization.toRustString()); + assertEquals(Collections.singletonList("name"), materialization.getColumns()); + + ScanOptions defaults = new ScanOptions.Builder().build(); + assertTrue(defaults.getBatchSizeBytes().isEmpty()); + assertTrue(defaults.getIoBufferSize().isEmpty()); + assertTrue(defaults.getFragmentReadahead().isEmpty()); + assertTrue(defaults.isScanInOrder()); + assertTrue(defaults.getLateMaterialization().isEmpty()); + + assertThrows( + IllegalArgumentException.class, () -> new ScanOptions.Builder().batchSizeBytes(0).build()); + assertThrows( + IllegalArgumentException.class, () -> new ScanOptions.Builder().ioBufferSize(0).build()); + assertThrows( + IllegalArgumentException.class, + () -> new ScanOptions.Builder().fragmentReadahead(0).build()); + assertThrows( + IllegalArgumentException.class, + () -> new ScanOptions.Builder().batchSizeBytes(1024).strictBatchSize(true).build()); + assertTrue(MaterializationStyle.allEarlyExcept(Collections.emptyList()).getColumns().isEmpty()); + } + + static Stream materializationStyles() { + return Stream.of( + MaterializationStyle.heuristic(), + MaterializationStyle.allLate(), + MaterializationStyle.allEarly(), + MaterializationStyle.allEarlyExcept(Collections.emptyList())); + } + + @ParameterizedTest + @MethodSource("materializationStyles") + void testMaterializationStylesAcrossJni( + MaterializationStyle materializationStyle, @TempDir Path tempDir) throws Exception { + String datasetPath = + tempDir.resolve("materialization_" + materializationStyle.getMode().name()).toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + try (Dataset dataset = testDataset.write(1, 40); + LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .filter("id < 20") + .lateMaterialization(materializationStyle) + .build()); + ArrowReader reader = scanner.scanBatches()) { + int rowCount = 0; + while (reader.loadNextBatch()) { + rowCount += reader.getVectorSchemaRoot().getRowCount(); + } + assertEquals(20, rowCount); + } + } + } + @Test void testDatasetScanner(@TempDir Path tempDir) throws IOException { String datasetPath = tempDir.resolve("dataset_scanner").toString(); @@ -158,6 +250,458 @@ void testDatasetScannerSchema(@TempDir Path tempDir) throws Exception { } } + /** + * Imports a caller-owned C stream populated by {@link LanceScanner#exportArrowStream(long)} and + * returns the {@code id} values in the order the stream produced them. + * + *

    The projected schema is asserted to be exactly a single {@code id: int32} field, and the + * assertion is made on the imported reader before the first {@code loadNextBatch()} call + * so that it still runs for an empty (zero-batch) result — a regression that exported the wrong + * schema for an empty scan would otherwise slip through. See {@code + * org.apache.arrow.vector.ipc.ArrowReader#getVectorSchemaRoot()}, which exposes the schema as + * soon as the stream is imported. + * + *

    This helper intentionally makes no assertion about per-batch row counts. The + * scanner's {@code batchSize} is only a hint unless {@code strictBatchSize(true)} is set, so the + * number of batches and the rows per batch are not part of the contract being tested here; that + * dimension is covered separately by {@link #testExportArrowStreamStrictBatchSize}. Row ordering + * and exact values are asserted by the callers against the returned list. + */ + private static List drainIdStream(BufferAllocator allocator, ArrowArrayStream stream) + throws IOException { + List ids = new ArrayList<>(); + try (ArrowReader reader = Data.importArrayStream(allocator, stream)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + List fields = root.getSchema().getFields(); + assertEquals(1, fields.size()); + Field idField = fields.get(0); + assertEquals("id", idField.getName()); + // Pin the exact type, not just the ArrowTypeID family: the projected column is a nullable + // signed int32. ArrowTypeID.Int alone also matches int8/16/64 and unsigned, and the + // (IntVector) cast below only guards the width on non-empty results — an empty scan that + // exported e.g. int64 or a non-nullable id would otherwise slip through this helper. + assertTrue(idField.isNullable()); + ArrowType.Int idType = (ArrowType.Int) idField.getType(); + assertEquals(32, idType.getBitWidth()); + assertTrue(idType.getIsSigned()); + while (reader.loadNextBatch()) { + IntVector vector = (IntVector) root.getVector("id"); + int rowsInBatch = vector.getValueCount(); + for (int i = 0; i < rowsInBatch; i++) { + ids.add(vector.get(i)); + } + } + } + return ids; + } + + /** + * Happy path: a single-fragment ordered scan exported through a caller-owned C stream returns + * every row exactly once, in scan order. The caller allocates the {@link ArrowArrayStream} from + * its own allocator and passes only the memory address; the scanner fills the C struct in place. + * This is the cross-Arrow-version / cross-classloader boundary the API exists to serve. + */ + @Test + void testExportArrowStream(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_basic").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + int totalRows = 40; + int batchRows = 20; + try (Dataset dataset = testDataset.write(1, totalRows)) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .batchSize(batchRows) + .columns(Arrays.asList("id")) + .build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + // SimpleTestDataset writes id = 0..totalRows-1; an ordered scan must return them in + // exactly that sequence, so assert the exact ordering (no sort). + List ids = drainIdStream(allocator, stream); + assertEquals(totalRows, ids.size()); + for (int i = 0; i < totalRows; i++) { + assertEquals(i, ids.get(i)); + } + } + } + } + } + } + + /** + * A scan that spans multiple fragments is exported as a single C stream that concatenates the + * fragments in fragment order. {@code createNewFragment(40, 10)} produces 4 fragments of 10 rows + * (ids 0-9, 10-19, 20-29, 30-39), and an ordered scan must return 0..39 in exactly that order. + * + *

    The expected ids are asserted in stream order without sorting: sorting would mask a + * regression that returned fragments out of order, which is exactly the kind of bug this test + * exists to catch. A non-divisor batch size (7) is used so batch boundaries do not line up with + * fragment boundaries, exercising the stream's batch stitching across fragments. + */ + @Test + void testExportArrowStreamMultipleFragments(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_multi_fragment").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + int totalRows = 40; + // maxRowsPerFile < totalRows forces multiple fragments (4 fragments of 10 rows). + List fragments = testDataset.createNewFragment(totalRows, 10); + assertEquals(4, fragments.size()); + FragmentOperation.Append appendOp = new FragmentOperation.Append(fragments); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, appendOp, Optional.of(1L))) { + int batchRows = 7; // deliberately not a divisor of any fragment size + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .batchSize(batchRows) + .columns(Arrays.asList("id")) + .build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + List ids = drainIdStream(allocator, stream); + assertEquals(totalRows, ids.size()); + // Assert exact scan order (no sort) so out-of-order fragments would fail. + for (int i = 0; i < totalRows; i++) { + assertEquals(i, ids.get(i), "row " + i + " out of expected scan order"); + } + } + } + } + } + } + + /** + * A pushed-down filter is honored by the exported stream: only matching rows cross the C-data + * boundary. {@code id < 20} over ids 0..39 must yield exactly 0..19 in order. Asserted in scan + * order without sorting so a filter/ordering regression cannot hide behind a sort. + */ + @Test + void testExportArrowStreamWithFilter(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_filter").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 40)) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .batchSize(50) + .columns(Arrays.asList("id")) + .filter("id < 20") + .build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + List ids = drainIdStream(allocator, stream); + assertEquals(20, ids.size()); + for (int i = 0; i < 20; i++) { + assertEquals(i, ids.get(i)); + } + } + } + } + } + } + + /** + * Pushed-down limit and offset are honored by the exported stream. Over ids 0..39, {@code + * offset(10).limit(5)} must yield exactly [10, 11, 12, 13, 14] in order — asserted as an exact + * ordered list so both the window bounds and the ordering are checked. + */ + @Test + void testExportArrowStreamWithLimitOffset(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_limit_offset").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 40)) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .batchSize(50) + .columns(Arrays.asList("id")) + .limit(5) + .offset(10) + .build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + List ids = drainIdStream(allocator, stream); + assertEquals(Arrays.asList(10, 11, 12, 13, 14), ids); + } + } + } + } + } + + /** + * Column projection is reflected in the exported stream's schema. {@code SimpleTestDataset} has + * columns {@code (id, name)}; projecting only {@code name} must produce a stream whose schema is + * exactly that one column. The schema is checked on the imported reader before draining, and the + * full row count is verified after. + */ + @Test + void testExportArrowStreamProjectsRequestedColumnsOnly(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_projection").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 10)) { + // Project only "name"; the exported stream's schema must contain exactly that column. + try (LanceScanner scanner = + dataset.newScan(new ScanOptions.Builder().columns(Arrays.asList("name")).build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + try (ArrowReader reader = Data.importArrayStream(allocator, stream)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertEquals(1, root.getSchema().getFields().size()); + assertEquals("name", root.getSchema().getFields().get(0).getName()); + int rows = 0; + while (reader.loadNextBatch()) { + rows += root.getRowCount(); + } + assertEquals(10, rows); + } + } + } + } + } + } + + /** + * A scan that matches no rows ({@code id < 0}) still exports a valid, well-formed stream that + * yields zero rows. {@link #drainIdStream} asserts the projected schema ({@code id: int32}) on + * the imported reader before any {@code loadNextBatch()}, so this case also guards the empty-scan + * schema — a regression that exported a wrong or absent schema for zero-row results would fail + * here even though no batch is ever produced. + */ + @Test + void testExportArrowStreamEmptyResult(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_empty").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 40)) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder().columns(Arrays.asList("id")).filter("id < 0").build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + List ids = drainIdStream(allocator, stream); + assertTrue(ids.isEmpty()); + } + } + } + } + } + + /** + * Guards against the sequential "export twice into the same stream" mistake. After the first + * export installs a producer (non-null {@code release} callback), a second export into the same + * stream must be rejected with {@link IllegalArgumentException} rather than overwriting the C + * struct in place — overwriting would drop the first producer's release callback and leak it. + * + *

    The test also verifies the rejection is non-destructive: the first producer is still intact + * and fully drainable (all 40 rows) after the rejected second call. This is the single-threaded + * misuse case; concurrent exports into one caller-owned stream are the caller's responsibility, + * as documented on {@link LanceScanner#exportArrowStream(long)}. + */ + @Test + void testExportArrowStreamRejectsPopulatedStream(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_reject_populated").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 40)) { + try (LanceScanner scanner = + dataset.newScan(new ScanOptions.Builder().columns(Arrays.asList("id")).build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + // First export populates the stream and installs a release callback. + scanner.exportArrowStream(stream.memoryAddress()); + // Exporting again into the same (already-populated) stream must be rejected rather + // than silently overwriting and leaking the first producer's release callback. + IllegalArgumentException ex = + assertThrows( + IllegalArgumentException.class, + () -> scanner.exportArrowStream(stream.memoryAddress())); + assertTrue(ex.getMessage().toLowerCase().contains("already populated")); + // The first producer is still intact and drainable. + try (ArrowReader reader = Data.importArrayStream(allocator, stream)) { + int rows = 0; + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + while (reader.loadNextBatch()) { + rows += root.getRowCount(); + } + assertEquals(40, rows); + } + } + } + } + } + } + + /** + * A null (0) stream address is rejected with {@link IllegalArgumentException} before any native + * dereference, so a caller mistake cannot turn into a native null-pointer write. + */ + @Test + void testExportArrowStreamRejectsNullAddress(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_reject_null").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 10)) { + try (LanceScanner scanner = + dataset.newScan(new ScanOptions.Builder().columns(Arrays.asList("id")).build())) { + assertThrows(IllegalArgumentException.class, () -> scanner.exportArrowStream(0L)); + } + } + } + } + + /** + * Exporting from a closed scanner is rejected with {@link IllegalArgumentException} (the native + * scanner handle is zero after {@code close()}), rather than dereferencing a freed handle. The + * scanner is closed explicitly here, so it is intentionally not in a try-with-resources. + */ + @Test + void testExportArrowStreamRejectsClosedScanner(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_reject_closed").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.write(1, 10)) { + LanceScanner scanner = + dataset.newScan(new ScanOptions.Builder().columns(Arrays.asList("id")).build()); + scanner.close(); + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + assertThrows( + IllegalArgumentException.class, + () -> scanner.exportArrowStream(stream.memoryAddress())); + } + } + } + } + + /** + * Null values survive the C-data export round-trip. {@code writeSortByDataset} writes 10 rows + * (insertion order) in which {@code id} is null at rows 2 and 5 and {@code name} is null at rows + * 0 and 6. An unordered scan returns rows in insertion order, so the exported stream must + * reproduce both the non-null values and the null positions exactly — null/validity bitmaps are a + * common casualty of an incorrect C-data export, so this guards them explicitly. + */ + @Test + void testExportArrowStreamPreservesNulls(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_nulls").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + try (Dataset dataset = testDataset.writeSortByDataset(1)) { + // Insertion order, row -> (id, name): + // 0 -> (0, null) 3 -> (2, "P2") 6 -> (3, null) 9 -> (5, "P5") + // 1 -> (1, "P0") 4 -> (2, "P3") 7 -> (4, "P4") + // 2 -> (null,"P1") 5 -> (null,"P3") 8 -> (4, "P5") + Integer[] expectedIds = {0, 1, null, 2, 2, null, 3, 4, 4, 5}; + String[] expectedNames = {null, "P0", "P1", "P2", "P3", "P3", null, "P4", "P5", "P5"}; + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder().columns(Arrays.asList("id", "name")).build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + try (ArrowReader reader = Data.importArrayStream(allocator, stream)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertEquals(2, root.getSchema().getFields().size()); + int row = 0; + while (reader.loadNextBatch()) { + IntVector idVector = (IntVector) root.getVector("id"); + VarCharVector nameVector = (VarCharVector) root.getVector("name"); + for (int i = 0; i < root.getRowCount(); i++, row++) { + if (expectedIds[row] == null) { + assertTrue(idVector.isNull(i), "id should be null at row " + row); + } else { + assertEquals( + expectedIds[row].intValue(), idVector.get(i), "id mismatch at row " + row); + } + if (expectedNames[row] == null) { + assertTrue(nameVector.isNull(i), "name should be null at row " + row); + } else { + assertEquals( + expectedNames[row], + new String(nameVector.get(i), StandardCharsets.UTF_8), + "name mismatch at row " + row); + } + } + } + assertEquals(expectedIds.length, row); + } + } + } + } + } + } + + /** + * With {@code strictBatchSize(true)}, the exported stream must split into batches no larger than + * the requested batch size, and still reproduce every row in order. This is the one place the + * per-batch size is part of the contract; the other export tests deliberately leave batch sizing + * unasserted because it is only a hint by default. Mirrors {@link #testStrictBatchSize} but over + * the C-data export path. A batch size of 10 over 25 rows yields batches of at most 10. + */ + @Test + void testExportArrowStreamStrictBatchSize(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("export_stream_strict_batch").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + int totalRows = 25; + int batchSize = 10; + try (Dataset dataset = testDataset.write(1, totalRows)) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .batchSize(batchSize) + .strictBatchSize(true) + .columns(Arrays.asList("id")) + .build())) { + try (ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator)) { + scanner.exportArrowStream(stream.memoryAddress()); + try (ArrowReader reader = Data.importArrayStream(allocator, stream)) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + List ids = new ArrayList<>(); + while (reader.loadNextBatch()) { + int rowsInBatch = root.getRowCount(); + assertTrue( + rowsInBatch <= batchSize, + "strict: batch of " + rowsInBatch + " should be <= " + batchSize); + IntVector idVector = (IntVector) root.getVector("id"); + for (int i = 0; i < rowsInBatch; i++) { + ids.add(idVector.get(i)); + } + } + assertEquals(totalRows, ids.size()); + for (int i = 0; i < totalRows; i++) { + assertEquals(i, ids.get(i)); + } + } + } + } + } + } + } + @Test void testDatasetScannerCountRows(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("dataset_scanner_count").toString(); @@ -212,6 +756,10 @@ void testDatasetScannerStats(@TempDir Path tempDir) throws Exception { assertTrue(statsOpt.isPresent()); ScanStats stats = statsOpt.get(); assertTrue(stats.getBytesRead() > 0 || !stats.getAllCounts().isEmpty()); + // Even without an index on this dataset, the two new counters must + // still marshal through JNI and default to zero rather than throwing. + assertTrue(stats.getIndexCacheHits() >= 0); + assertTrue(stats.getIndexCacheMisses() >= 0); } } } @@ -422,31 +970,99 @@ void testDatasetScannerBatchReadahead(@TempDir Path tempDir) throws Exception { TestUtils.SimpleTestDataset testDataset = new TestUtils.SimpleTestDataset(allocator, datasetPath); testDataset.createEmptyDataset().close(); - int totalRows = 1000; - int batchSize = 100; - int batchReadahead = 5; - try (Dataset dataset = testDataset.write(1, totalRows)) { + + int totalRows = 2000; + int maxRowsPerFile = 100; // ~20 fragments + List fragments = testDataset.createNewFragment(totalRows, maxRowsPerFile); + assertTrue(fragments.size() > 1, "expected multiple fragments, got " + fragments.size()); + + FragmentOperation.Append append = new FragmentOperation.Append(fragments); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, append, Optional.of(1L))) { + int batchReadahead = 2; // far below the default (num compute CPUs) try (LanceScanner scanner = dataset.newScan( - new ScanOptions.Builder() - .batchSize(batchSize) - .batchReadahead(batchReadahead) - .build())) { - // This test is more about ensuring that the batchReadahead parameter is accepted - // and doesn't cause errors. The actual effect of batchReadahead might not be - // directly observable in this test. + new ScanOptions.Builder().batchSize(50).batchReadahead(batchReadahead).build())) { try (ArrowReader reader = scanner.scanBatches()) { int rowCount = 0; + long idSum = 0; while (reader.loadNextBatch()) { - rowCount += reader.getVectorSchemaRoot().getRowCount(); + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + IntVector ids = (IntVector) root.getVector("id"); + for (int i = 0; i < root.getRowCount(); i++) { + idSum += ids.get(i); + } + rowCount += root.getRowCount(); } assertEquals(totalRows, rowCount); + // ids are the contiguous range [0, totalRows) + assertEquals((long) totalRows * (totalRows - 1) / 2, idSum); } } } } } + @Test + void testDatasetScannerPerformanceOptions(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("dataset_scanner_performance_options").toString(); + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + int totalRows = 2000; + int matchingRows = 1000; + WriteParams writeParams = + new WriteParams.Builder() + .withMaxRowsPerFile(100) + .withDataStorageVersion(LanceConstants.FILE_FORMAT_VERSION_STABLE) + .build(); + List fragments = testDataset.createNewFragment(totalRows, writeParams); + FragmentOperation.Append append = new FragmentOperation.Append(fragments); + try (Dataset dataset = Dataset.commit(allocator, datasetPath, append, Optional.of(1L))) { + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .filter("id < " + matchingRows) + .batchSize(200) + .batchSizeBytes(256) + .ioBufferSize(1024 * 1024) + .batchReadahead(2) + .fragmentReadahead(2) + .scanInOrder(false) + .lateMaterialization(MaterializationStyle.allEarly()) + .build()); + ArrowReader reader = scanner.scanBatches()) { + int rowCount = 0; + long idSum = 0; + while (reader.loadNextBatch()) { + VectorSchemaRoot root = reader.getVectorSchemaRoot(); + assertTrue( + root.getRowCount() < 200, + "batchSizeBytes should split batches before the row limit"); + IntVector ids = (IntVector) root.getVector("id"); + for (int i = 0; i < root.getRowCount(); i++) { + idSum += ids.get(i); + } + rowCount += root.getRowCount(); + } + assertEquals(matchingRows, rowCount); + assertEquals((long) matchingRows * (matchingRows - 1) / 2, idSum); + } + + try (LanceScanner scanner = + dataset.newScan( + new ScanOptions.Builder() + .filter("id < " + matchingRows) + .lateMaterialization( + MaterializationStyle.allEarlyExcept(Collections.singletonList("name"))) + .build())) { + assertEquals(matchingRows, scanner.countRows()); + } + } + } + } + @Test void testDatasetScannerSortBy(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("testDatasetScannerSortBy").toString(); diff --git a/java/src/test/java/org/lance/SessionTest.java b/java/src/test/java/org/lance/SessionTest.java index b2ed3baa343..7828d79f06b 100644 --- a/java/src/test/java/org/lance/SessionTest.java +++ b/java/src/test/java/org/lance/SessionTest.java @@ -19,6 +19,8 @@ import org.junit.jupiter.api.io.TempDir; import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -68,6 +70,135 @@ void testCreateSessionWithPartialCustomCacheSizes() { } } + @Test + void testCreateSessionWithCacheBackendUris() { + try (Session session = + Session.builder() + .indexCacheBackend("moka://?capacity=1048576") + .metadataCacheBackend("moka://?capacity=524288") + .build()) { + assertNotNull(session); + assertFalse(session.isClosed()); + assertEquals(0, session.metadataCacheStats().getNumEntries()); + } + } + + @Test + void testCreateSessionWithStructuredCacheBackendConfigs() { + CacheBackendConfig indexBackend = + CacheBackendConfig.builder("moka").option("capacity", "1048576").build(); + CacheBackendConfig metadataBackend = + CacheBackendConfig.builder("moka") + .options(Collections.singletonMap("capacity", "524288")) + .build(); + + assertEquals("moka", indexBackend.getKind()); + assertEquals(Collections.singletonMap("capacity", "1048576"), indexBackend.getOptions()); + assertThrows( + UnsupportedOperationException.class, () -> indexBackend.getOptions().put("capacity", "1")); + + try (Session session = + Session.builder() + .indexCacheBackend(indexBackend) + .metadataCacheBackend(metadataBackend) + .build()) { + assertNotNull(session); + assertFalse(session.isClosed()); + } + } + + @Test + void testCacheBackendReplacesPreviousDescriptorForSameTier() { + CacheBackendConfig structuredBackend = + CacheBackendConfig.builder("moka").option("capacity", "1048576").build(); + + try (Session session = + Session.builder() + .indexCacheBackend("missing://") + .indexCacheBackend(structuredBackend) + .build()) { + assertNotNull(session); + } + + try (Session session = + Session.builder() + .metadataCacheBackend(structuredBackend) + .metadataCacheBackend("moka://?capacity=1048576") + .build()) { + assertNotNull(session); + } + } + + @Test + void testCacheBackendRejectsSizeAndBackend() { + IllegalArgumentException indexError = + assertThrows( + IllegalArgumentException.class, + () -> + Session.builder() + .indexCacheSizeBytes(1024) + .indexCacheBackend("moka://?capacity=1048576") + .build()); + assertTrue( + indexError + .getMessage() + .contains("indexCacheSizeBytes and indexCacheBackend are mutually exclusive")); + + IllegalArgumentException metadataError = + assertThrows( + IllegalArgumentException.class, + () -> + Session.builder() + .metadataCacheBackend( + CacheBackendConfig.builder("moka").option("capacity", "1048576").build()) + .metadataCacheSizeBytes(1024) + .build()); + assertTrue( + metadataError + .getMessage() + .contains("metadataCacheSizeBytes and metadataCacheBackend are mutually exclusive")); + } + + @Test + void testCacheBackendRejectsUnknownKind() { + IllegalArgumentException error = + assertThrows( + IllegalArgumentException.class, + () -> Session.builder().indexCacheBackend("missing://").build()); + assertTrue(error.getMessage().contains("unknown cache backend kind")); + } + + @Test + void testCacheBackendRejectsInvalidMokaConfig() { + IllegalArgumentException missingCapacity = + assertThrows( + IllegalArgumentException.class, + () -> Session.builder().indexCacheBackend("moka://").build()); + assertTrue(missingCapacity.getMessage().contains("capacity is required")); + + IllegalArgumentException unknownOption = + assertThrows( + IllegalArgumentException.class, + () -> + Session.builder() + .metadataCacheBackend( + CacheBackendConfig.builder("moka").option("unknown", "value").build()) + .build()); + assertTrue(unknownOption.getMessage().contains("unknown option")); + } + + @Test + void testCacheBackendConfigValidatesInputs() { + assertThrows(NullPointerException.class, () -> CacheBackendConfig.builder(null)); + assertThrows(IllegalArgumentException.class, () -> CacheBackendConfig.builder("")); + assertThrows( + NullPointerException.class, + () -> CacheBackendConfig.builder("moka").options((Map) null)); + assertThrows( + NullPointerException.class, + () -> CacheBackendConfig.builder("moka").option("capacity", null)); + } + @Test void testSessionClose() { Session session = Session.builder().build(); @@ -246,6 +377,39 @@ void testUserProvidedSessionNotClosedWithDataset(@TempDir Path tempDir) { } } + @Test + void testCheckedOutDatasetsShareInternalSession(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("dataset_checkout_session").toString(); + + try (BufferAllocator allocator = new RootAllocator()) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + try (Dataset source = Dataset.open().allocator(allocator).uri(datasetPath).build()) { + source.tags().create("version-one", Ref.ofMain(1)); + Session sourceSession = source.session(); + + try (Dataset byVersion = source.checkoutVersion(1); + Dataset byTag = source.checkoutTag("version-one"); + Dataset byRef = source.checkout(Ref.ofMain(1))) { + assertNotNull(byVersion.session()); + assertNotNull(byTag.session()); + assertNotNull(byRef.session()); + assertTrue(byVersion.session().isSameAs(sourceSession)); + assertTrue(byTag.session().isSameAs(sourceSession)); + assertTrue(byRef.session().isSameAs(sourceSession)); + + source.close(); + assertTrue(sourceSession.isClosed()); + assertFalse(byVersion.session().isClosed()); + assertFalse(byTag.session().isClosed()); + assertFalse(byRef.session().isClosed()); + } + } + } + } + @Test void testSessionToString() { try (Session session = Session.builder().build()) { @@ -259,6 +423,49 @@ void testSessionToString() { assertEquals("Session(closed)", closedSession.toString()); } + @Test + void testMetadataCacheStats(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("dataset_metadata_cache_stats").toString(); + + try (BufferAllocator allocator = new RootAllocator(); + Session session = Session.builder().build()) { + CacheStats initialStats = session.metadataCacheStats(); + assertEquals(0, initialStats.getHits()); + assertEquals(0, initialStats.getMisses()); + assertEquals(0, initialStats.getNumEntries()); + assertEquals(0, initialStats.getSizeBytes()); + assertEquals(0.0, initialStats.getHitRatio()); + + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + + // First open populates the metadata cache, producing misses + try (Dataset ds = + Dataset.open().allocator(allocator).uri(datasetPath).session(session).build()) { + ds.countRows(); + } + CacheStats statsAfterFirstOpen = session.metadataCacheStats(); + assertTrue(statsAfterFirstOpen.getMisses() > 0); + assertTrue(statsAfterFirstOpen.getNumEntries() > 0); + assertTrue(statsAfterFirstOpen.getSizeBytes() > 0); + + // Reopening the same dataset should hit the shared metadata cache + try (Dataset ds = + Dataset.open().allocator(allocator).uri(datasetPath).session(session).build()) { + ds.countRows(); + } + CacheStats statsAfterSecondOpen = session.metadataCacheStats(); + assertTrue(statsAfterSecondOpen.getHits() > statsAfterFirstOpen.getHits()); + assertTrue(statsAfterSecondOpen.getHitRatio() > 0.0); + + // Stats are not accessible once the session is closed (close is idempotent, + // so the implicit close from try-with-resources remains safe) + session.close(); + assertThrows(IllegalArgumentException.class, session::metadataCacheStats); + } + } + @Test void testInvalidCacheSizes() { assertThrows( diff --git a/java/src/test/java/org/lance/TestUtils.java b/java/src/test/java/org/lance/TestUtils.java index c9033361103..1989a5243bc 100644 --- a/java/src/test/java/org/lance/TestUtils.java +++ b/java/src/test/java/org/lance/TestUtils.java @@ -111,6 +111,11 @@ public FragmentMetadata createNewFragment(int rowCount) { } public List createNewFragment(int rowCount, int maxRowsPerFile) { + return createNewFragment( + rowCount, new WriteParams.Builder().withMaxRowsPerFile(maxRowsPerFile).build()); + } + + public List createNewFragment(int rowCount, WriteParams writeParams) { List fragmentMetas; try (VectorSchemaRoot root = VectorSchemaRoot.create(getSchema(), allocator)) { root.allocateNew(); @@ -124,12 +129,7 @@ public List createNewFragment(int rowCount, int maxRowsPerFile } root.setRowCount(rowCount); - fragmentMetas = - Fragment.create( - datasetPath, - allocator, - root, - new WriteParams.Builder().withMaxRowsPerFile(maxRowsPerFile).build()); + fragmentMetas = Fragment.create(datasetPath, allocator, root, writeParams); } return fragmentMetas; } @@ -694,7 +694,8 @@ public Dataset createEmptyDataset() { /** * Create a single fragment with given row count and return its metadata. The fragment contains * deterministic blob payloads: - Every 16th row starting at 0 has zero-length blob - Every 16th - * row starting at 1 has a ~1 MiB payload - Others have small variable blobs (128..383 bytes) + * row starting at 1 has a ~1 MiB payload - Every 16th row starting at 15 is null - Others have + * small variable blobs (128..383 bytes) */ public FragmentMetadata createBlobFragment(int rowCount, int maxRowsPerFile) { Preconditions.checkArgument(rowCount >= 0, "rowCount must be non-negative"); @@ -716,6 +717,8 @@ public FragmentMetadata createBlobFragment(int rowCount, int maxRowsPerFile) { byte[] big = new byte[1024 * 1024]; Arrays.fill(big, (byte) 0xAB); blobsVec.setSafe(i, big); + } else if (i % 16 == 15) { + blobsVec.setNull(i); } else { // small variable blob int sz = 128 + (i % 256); diff --git a/java/src/test/java/org/lance/fragment/FragmentUpdateResultTest.java b/java/src/test/java/org/lance/fragment/FragmentUpdateResultTest.java new file mode 100644 index 00000000000..1161f18b415 --- /dev/null +++ b/java/src/test/java/org/lance/fragment/FragmentUpdateResultTest.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.fragment; + +import org.lance.CommitBuilder; +import org.lance.Dataset; +import org.lance.Fragment; +import org.lance.FragmentMetadata; +import org.lance.TestUtils; +import org.lance.Transaction; +import org.lance.operation.Append; + +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class FragmentUpdateResultTest { + + /** Portable RoaringBitmap bytes for offsets {1, 3, 5} (see UpdateTest round-trip fixture). */ + private static final byte[] PORTABLE_ROARING_BYTES_135 = + new byte[] { + (byte) 0x3A, + (byte) 0x30, + (byte) 0x00, + (byte) 0x00, + (byte) 0x01, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x02, + (byte) 0x00, + (byte) 0x10, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, + (byte) 0x01, + (byte) 0x00, + (byte) 0x03, + (byte) 0x00, + (byte) 0x05, + (byte) 0x00 + }; + + @Test + void testGetUpdatedRowOffsetBytesRoundTripViaDeprecatedGetter() { + FragmentUpdateResult result = + FragmentUpdateResult.create(null, new long[0], PORTABLE_ROARING_BYTES_135); + assertArrayEquals(PORTABLE_ROARING_BYTES_135, result.getUpdatedRowOffsetBytes()); + assertArrayEquals(new long[] {1, 3, 5}, result.getUpdatedRowOffsets()); + } + + @Test + void testDeprecatedLongArrayConstructorEncodesToBytes() { + FragmentUpdateResult result = new FragmentUpdateResult(null, new long[0], new long[] {1, 3, 5}); + assertArrayEquals(new long[] {1, 3, 5}, result.getUpdatedRowOffsets()); + + // Stored bytes from the deprecated long[] constructor decode to the same offsets. + FragmentUpdateResult fromEncodedBytes = + FragmentUpdateResult.create(null, new long[0], result.getUpdatedRowOffsetBytes()); + assertArrayEquals(new long[] {1, 3, 5}, fromEncodedBytes.getUpdatedRowOffsets()); + } + + @Test + void testUpdateColumnsReturnsMatchedRowOffsetBytes(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testUpdateColumnsRowOffsetBytes").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.UpdateColumnTestDataset testDataset = + new TestUtils.UpdateColumnTestDataset(allocator, datasetPath); + try (Dataset dataset = testDataset.createEmptyDataset()) { + FragmentMetadata fragmentMeta = testDataset.createNewFragment(6); + try (Transaction appendTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build()) { + try (Dataset appended = new CommitBuilder(dataset).execute(appendTxn)) { + Fragment fragment = appended.getFragments().get(0); + FragmentUpdateResult updateResult = testDataset.updateColumn(fragment, 4); + assertTrue(updateResult.getUpdatedRowOffsetBytes().length > 0); + assertArrayEquals(new long[] {0, 1, 2, 3}, updateResult.getUpdatedRowOffsets()); + } + } + } + } + } +} diff --git a/java/src/test/java/org/lance/index/ScalarIndexTest.java b/java/src/test/java/org/lance/index/ScalarIndexTest.java index cb090e7c955..6e3e309d681 100644 --- a/java/src/test/java/org/lance/index/ScalarIndexTest.java +++ b/java/src/test/java/org/lance/index/ScalarIndexTest.java @@ -40,6 +40,7 @@ import org.apache.arrow.vector.types.pojo.Schema; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import java.io.ByteArrayInputStream; @@ -52,6 +53,8 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -60,6 +63,113 @@ public class ScalarIndexTest { + private static final class RecordingIndexBuildProgress implements IndexBuildProgress { + private final List events = Collections.synchronizedList(new ArrayList<>()); + + @Override + public void stageStart(String stage, Optional total, String unit) { + events.add( + "start:" + stage + ":" + total.map(String::valueOf).orElse("unknown") + ":" + unit); + } + + @Override + public void stageProgress(String stage, long completed) { + events.add("progress:" + stage + ":" + completed); + } + + @Override + public void stageComplete(String stage) { + events.add("complete:" + stage); + } + + private List snapshot() { + synchronized (events) { + return new ArrayList<>(events); + } + } + } + + private static final class FailingProgressIndexBuildProgress implements IndexBuildProgress { + private final RecordingIndexBuildProgress recorder = new RecordingIndexBuildProgress(); + + @Override + public void stageStart(String stage, Optional total, String unit) { + recorder.stageStart(stage, total, unit); + } + + @Override + public void stageProgress(String stage, long completed) { + recorder.stageProgress(stage, completed); + throw new IllegalStateException("progress callback failure"); + } + + @Override + public void stageComplete(String stage) { + recorder.stageComplete(stage); + } + } + + private static final class FailingCompleteIndexBuildProgress implements IndexBuildProgress { + private final RecordingIndexBuildProgress recorder = new RecordingIndexBuildProgress(); + + @Override + public void stageStart(String stage, Optional total, String unit) { + recorder.stageStart(stage, total, unit); + } + + @Override + public void stageProgress(String stage, long completed) { + recorder.stageProgress(stage, completed); + } + + @Override + public void stageComplete(String stage) { + recorder.stageComplete(stage); + throw new IllegalStateException("complete callback failure"); + } + } + + /** + * Progress callback that re-enters the same Dataset via JNI. Without releasing the native field + * lock before merge starts, these calls would deadlock. + */ + private static final class ReentrantDatasetIndexBuildProgress implements IndexBuildProgress { + private final Dataset dataset; + private final RecordingIndexBuildProgress recorder = new RecordingIndexBuildProgress(); + private final AtomicInteger reentries = new AtomicInteger(); + + private ReentrantDatasetIndexBuildProgress(Dataset dataset) { + this.dataset = dataset; + } + + @Override + public void stageStart(String stage, Optional total, String unit) { + recorder.stageStart(stage, total, unit); + touchDataset(); + } + + @Override + public void stageProgress(String stage, long completed) { + recorder.stageProgress(stage, completed); + touchDataset(); + } + + @Override + public void stageComplete(String stage) { + recorder.stageComplete(stage); + touchDataset(); + } + + private void touchDataset() { + assertNotNull(dataset.uri()); + assertTrue(dataset.version() > 0); + assertTrue(dataset.countRows() > 0); + assertFalse(dataset.getFragments().isEmpty()); + assertFalse(dataset.memWalIndexDetails().isPresent()); + reentries.incrementAndGet(); + } + } + @Test public void testCreateBTreeIndex(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("btree_test").toString(); @@ -276,6 +386,175 @@ public void testBtreeMergeIndexMetadataSoftBreak(@TempDir Path tempDir) throws E } } + @Test + public void testMergeInvertedIndexMetadataReportsProgress(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_progress").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + RecordingIndexBuildProgress progress = new RecordingIndexBuildProgress(); + + dataset.mergeIndexMetadata(indexUuid, IndexType.INVERTED, Optional.empty(), progress); + + List events = progress.snapshot(); + assertEventsInOrder( + events, + "start:read_partition_metadata:", + "complete:read_partition_metadata", + "start:remap_partition_files:", + "complete:remap_partition_files", + "start:write_merged_metadata:", + "complete:write_merged_metadata"); + assertTrue( + events.contains("progress:read_partition_metadata:2"), + "Expected metadata progress to reach both fragments, got: " + events); + assertTrue( + events.stream().anyMatch(event -> event.startsWith("progress:remap_partition_files:")), + "Expected remap progress, got: " + events); + assertTrue( + events.contains("progress:write_merged_metadata:1"), + "Expected merged metadata write progress, got: " + events); + } + } + } + + @Test + public void testMergeInvertedIndexMetadataPropagatesProgressFailure(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_progress_failure").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + + RuntimeException failure = + Assertions.assertThrows( + RuntimeException.class, + () -> + dataset.mergeIndexMetadata( + indexUuid, + IndexType.INVERTED, + Optional.empty(), + new FailingProgressIndexBuildProgress())); + + assertFalse( + failure instanceof IllegalArgumentException, + "Progress callback failures should not be reported as invalid input: " + failure); + assertTrue( + causeChainContains(failure, "stageProgress") + && causeChainContains(failure, "read_partition_metadata") + && causeChainContains(failure, "java.lang.IllegalStateException") + && causeChainContains(failure, "progress callback failure"), + "Expected callback context and original Java exception details, got: " + failure); + } + } + } + + @Test + public void testMergeInvertedIndexMetadataIgnoresCompleteFailure(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_complete_failure").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + FailingCompleteIndexBuildProgress progress = new FailingCompleteIndexBuildProgress(); + + dataset.mergeIndexMetadata(indexUuid, IndexType.INVERTED, Optional.empty(), progress); + + assertTrue( + progress.recorder.snapshot().contains("complete:write_merged_metadata"), + "Expected merge to continue after stageComplete callback failures"); + } + } + } + + @Test + @Timeout(value = 60, unit = TimeUnit.SECONDS) + public void testMergeInvertedIndexMetadataAllowsReentrantDatasetAccess(@TempDir Path tempDir) + throws Exception { + String datasetPath = tempDir.resolve("inverted_merge_reentrant_dataset").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + testDataset.createEmptyDataset().close(); + testDataset.write(1, 10).close(); + try (Dataset dataset = testDataset.write(2, 10)) { + String indexUuid = createDistributedInvertedIndex(dataset); + ReentrantDatasetIndexBuildProgress progress = + new ReentrantDatasetIndexBuildProgress(dataset); + + dataset.mergeIndexMetadata(indexUuid, IndexType.INVERTED, Optional.empty(), progress); + + assertTrue( + progress.reentries.get() > 0, + "Expected progress callbacks to re-enter Dataset JNI methods"); + assertTrue( + progress.recorder.snapshot().contains("complete:write_merged_metadata"), + "Expected merge to finish after re-entrant Dataset access, got: " + + progress.recorder.snapshot()); + } + } + } + + private static String createDistributedInvertedIndex(Dataset dataset) { + ScalarIndexParams scalarParams = + ScalarIndexParams.create( + "inverted", + "{\"base_tokenizer\":\"simple\",\"language\":\"English\"," + + "\"max_token_length\":40,\"lower_case\":true,\"stem\":false," + + "\"remove_stop_words\":false}"); + IndexParams indexParams = IndexParams.builder().setScalarIndexParams(scalarParams).build(); + String indexUuid = UUID.randomUUID().toString(); + for (Fragment fragment : dataset.getFragments()) { + dataset.createIndex( + IndexOptions.builder(Collections.singletonList("name"), IndexType.INVERTED, indexParams) + .replace(true) + .withIndexName("inverted_progress_idx") + .withIndexUUID(indexUuid) + .withFragmentIds(Collections.singletonList(fragment.getId())) + .build()); + } + return indexUuid; + } + + private static void assertEventsInOrder(List events, String... prefixes) { + int previous = -1; + for (String prefix : prefixes) { + int current = -1; + for (int i = previous + 1; i < events.size(); i++) { + if (events.get(i).startsWith(prefix)) { + current = i; + break; + } + } + assertTrue( + current >= 0, + "Missing event '" + prefix + "' after position " + previous + ": " + events); + previous = current; + } + } + + private static boolean causeChainContains(Throwable failure, String expected) { + for (Throwable current = failure; current != null; current = current.getCause()) { + if (current.getMessage() != null && current.getMessage().contains(expected)) { + return true; + } + } + return false; + } + @Test public void testCreateZonemapIndex(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("zonemap_test").toString(); diff --git a/java/src/test/java/org/lance/index/VectorIndexTest.java b/java/src/test/java/org/lance/index/VectorIndexTest.java index 81c3554eaf1..7a9e066a1c5 100755 --- a/java/src/test/java/org/lance/index/VectorIndexTest.java +++ b/java/src/test/java/org/lance/index/VectorIndexTest.java @@ -25,10 +25,13 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.nio.file.Path; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -37,6 +40,37 @@ public class VectorIndexTest { + @ParameterizedTest + @EnumSource( + value = IndexType.class, + names = {"IVF_FLAT", "IVF_PQ"}) + @SuppressWarnings("deprecation") + public void testCreateIndexWithConcreteVectorType(IndexType indexType, @TempDir Path tempDir) + throws Exception { + try (TestVectorDataset testVectorDataset = + new TestVectorDataset(tempDir.resolve(indexType.name()))) { + try (Dataset dataset = testVectorDataset.create()) { + VectorIndexParams vectorIndexParams = + indexType == IndexType.IVF_FLAT + ? VectorIndexParams.ivfFlat(2, DistanceType.L2) + : VectorIndexParams.ivfPq(2, 8, 2, DistanceType.L2, 2); + IndexParams indexParams = + IndexParams.builder().setVectorIndexParams(vectorIndexParams).build(); + + Index index = + dataset.createIndex( + Collections.singletonList(TestVectorDataset.vectorColumnName), + indexType, + Optional.empty(), + indexParams, + false); + + assertNotNull(index); + assertTrue(dataset.listIndexes().contains(index.name())); + } + } + } + @Test public void testCreateIvfFlatIndexDistributively(@TempDir Path tempDir) throws Exception { try (TestVectorDataset testVectorDataset = diff --git a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java index e5024a95c2a..9fb8de4375b 100644 --- a/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java +++ b/java/src/test/java/org/lance/index/scalar/InvertedIndexParamsTest.java @@ -13,19 +13,93 @@ */ package org.lance.index.scalar; +import org.lance.DocumentGranularity; +import org.lance.util.JsonUtils; + import org.junit.jupiter.api.Test; +import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -public class InvertedIndexParamsTest { +class InvertedIndexParamsTest { @Test - public void testIcuSplitTokenizerVariant() { + void testIcuSplitTokenizerVariant() { ScalarIndexParams params = InvertedIndexParams.builder().baseTokenizer("icu/split").build(); assertEquals("inverted", params.getIndexType()); String jsonParams = params.getJsonParams().orElseThrow(AssertionError::new); assertTrue(jsonParams.contains("\"base_tokenizer\":\"icu/split\"")); } + + @Test + void defaultBlockSizeIsSerialized() { + ScalarIndexParams params = InvertedIndexParams.builder().build(); + + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(128, ((Number) json.get("block_size")).intValue()); + assertEquals("row", json.get("document_granularity")); + } + + @Test + void documentGranularityIsSerialized() { + ScalarIndexParams params = + InvertedIndexParams.builder().documentGranularity(DocumentGranularity.LIST_ELEMENT).build(); + + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals("list_element", json.get("document_granularity")); + } + + @Test + void blockSizeIsSerialized() { + ScalarIndexParams params = InvertedIndexParams.builder().blockSize(128).build(); + + assertEquals("inverted", params.getIndexType()); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(128, ((Number) json.get("block_size")).intValue()); + } + + @Test + void invalidBlockSizeIsRejected() { + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(129)); + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().blockSize(512)); + } + + @Test + void formatVersionThreeSupportsBothBlockSizes() { + for (int blockSize : new int[] {128, 256}) { + ScalarIndexParams params = + InvertedIndexParams.builder().blockSize(blockSize).formatVersion(3).build(); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(blockSize, ((Number) json.get("block_size")).intValue()); + assertEquals(3, ((Number) json.get("format_version")).intValue()); + } + + assertThrows( + IllegalArgumentException.class, + () -> InvertedIndexParams.builder().blockSize(256).formatVersion(2).build()); + } + + @Test + void formatVersionFourIsRejected() { + assertThrows( + IllegalArgumentException.class, () -> InvertedIndexParams.builder().formatVersion(4)); + } + + @Test + void codeAnalyzerRequiresFormatVersionThreeWhenExplicit() { + ScalarIndexParams params = + InvertedIndexParams.builder().baseTokenizer("code").formatVersion(3).build(); + Map json = JsonUtils.fromJson(params.getJsonParams().orElseThrow()); + assertEquals(3, ((Number) json.get("format_version")).intValue()); + + assertThrows( + IllegalArgumentException.class, + () -> InvertedIndexParams.builder().baseTokenizer("code").formatVersion(2).build()); + } } diff --git a/java/src/test/java/org/lance/ipc/FullTextQueryTest.java b/java/src/test/java/org/lance/ipc/FullTextQueryTest.java index 3dcf4276277..b84a19db4c8 100755 --- a/java/src/test/java/org/lance/ipc/FullTextQueryTest.java +++ b/java/src/test/java/org/lance/ipc/FullTextQueryTest.java @@ -13,6 +13,8 @@ */ package org.lance.ipc; +import org.lance.DocumentGranularity; + import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -39,6 +41,7 @@ void testMatchQueryDefaults() { assertEquals(50, q.getMaxExpansions()); assertEquals(FullTextQuery.Operator.OR, q.getOperator()); assertEquals(0, q.getPrefixLength()); + assertEquals(Optional.empty(), q.getDocumentGranularity()); } @Test @@ -46,7 +49,14 @@ void testMatchQueryCustomParameters() { FullTextQuery.MatchQuery q = (FullTextQuery.MatchQuery) FullTextQuery.match( - "hello", "title", 2.0f, Optional.of(1), 10, FullTextQuery.Operator.AND, 3); + "hello", + "title", + 2.0f, + Optional.of(1), + 10, + FullTextQuery.Operator.AND, + 3, + DocumentGranularity.LIST_ELEMENT); assertEquals(FullTextQuery.Type.MATCH, q.getType()); assertEquals("hello", q.getQueryText()); @@ -56,6 +66,7 @@ void testMatchQueryCustomParameters() { assertEquals(10, q.getMaxExpansions()); assertEquals(FullTextQuery.Operator.AND, q.getOperator()); assertEquals(3, q.getPrefixLength()); + assertEquals(Optional.of(DocumentGranularity.LIST_ELEMENT), q.getDocumentGranularity()); } @Test @@ -67,17 +78,20 @@ void testPhraseQueryDefaults() { assertEquals("exact match", q.getQueryText()); assertEquals("content", q.getColumn()); assertEquals(0, q.getSlop()); + assertEquals(Optional.empty(), q.getDocumentGranularity()); } @Test void testPhraseQueryCustomSlop() { FullTextQuery.PhraseQuery q = - (FullTextQuery.PhraseQuery) FullTextQuery.phrase("ordered terms", "content", 2); + (FullTextQuery.PhraseQuery) + FullTextQuery.phrase("ordered terms", "content", 2, DocumentGranularity.LIST_ELEMENT); assertEquals(FullTextQuery.Type.MATCH_PHRASE, q.getType()); assertEquals("ordered terms", q.getQueryText()); assertEquals("content", q.getColumn()); assertEquals(2, q.getSlop()); + assertEquals(Optional.of(DocumentGranularity.LIST_ELEMENT), q.getDocumentGranularity()); } @Test diff --git a/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java b/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java index 1c46b399195..9098084ac9f 100755 --- a/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java +++ b/java/src/test/java/org/lance/ipc/LanceScannerFullTextSearchTest.java @@ -14,6 +14,7 @@ package org.lance.ipc; import org.lance.Dataset; +import org.lance.DocumentGranularity; import org.lance.WriteParams; import org.lance.index.IndexOptions; import org.lance.index.IndexParams; @@ -41,12 +42,31 @@ import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; class LanceScannerFullTextSearchTest { @Test void testMatchQuery() throws Exception { - runFtsQuery("memory://fts_java_match", FullTextQuery.match("hello", "doc"), 2L); + runFtsQuery( + "memory://fts_java_match", + FullTextQuery.match("hello", "doc", DocumentGranularity.ROW), + 2L); + } + + @Test + void testExplicitListElementGranularityReachesRustRouting() { + RuntimeException error = + assertThrows( + RuntimeException.class, + () -> + runFtsQuery( + "memory://fts_java_list_element_validation", + FullTextQuery.match("hello", "doc", DocumentGranularity.LIST_ELEMENT), + 0L)); + assertTrue(error.getMessage().contains("requested ListElement"), error.getMessage()); + assertTrue(error.getMessage().contains("'doc_idx' (Row)"), error.getMessage()); } @Test diff --git a/java/src/test/java/org/lance/memwal/MemWalTest.java b/java/src/test/java/org/lance/memwal/MemWalTest.java index de0124bb1b3..e9bd053f298 100644 --- a/java/src/test/java/org/lance/memwal/MemWalTest.java +++ b/java/src/test/java/org/lance/memwal/MemWalTest.java @@ -97,6 +97,22 @@ private static VectorSchemaRoot lookupRoot(BufferAllocator allocator, long[] ids return root; } + /** Build a single-batch root carrying only the {@code id} primary key, for deletes. */ + private static VectorSchemaRoot keysRoot(BufferAllocator allocator, long[] ids) { + VectorSchemaRoot root = + VectorSchemaRoot.create( + new Schema( + Collections.singletonList(Field.nullable("id", new ArrowType.Int(64, true)))), + allocator); + BigIntVector idVector = (BigIntVector) root.getVector("id"); + idVector.allocateNew(ids.length); + for (int i = 0; i < ids.length; i++) { + idVector.set(i, ids[i]); + } + root.setRowCount(ids.length); + return root; + } + /** Build a single-batch append-only root without primary-key metadata. */ private static VectorSchemaRoot appendOnlyRoot( BufferAllocator allocator, long[] ids, String prefix) { @@ -144,12 +160,12 @@ private static Dataset writeAppendOnlyDataset( } /** - * Stage a faithful flushed generation at {@code genPath}: the Lance dataset plus its - * primary-key dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM - * scanner's cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not - * a state production produces. Mirrors the Python {@code _write_flushed_gen} test helper. + * Stage a faithful SSTable at {@code genPath}: the Lance dataset plus its primary-key + * dedup sidecar ({@code _pk_index/}), mirroring what production flush emits. The LSM scanner's + * cross-generation block-list opens the sidecar, so a dataset alone (no sidecar) is not a state + * production produces. Mirrors the Python {@code _write_sstable} test helper. */ - private static void writeFlushedGen( + private static void writeSsTable( BufferAllocator allocator, String genPath, long[] ids, String prefix) throws Exception { writeLookupDataset(allocator, genPath, ids, prefix).close(); try (VectorSchemaRoot root = lookupRoot(allocator, ids, prefix); @@ -161,8 +177,8 @@ private static void writeFlushedGen( } /** - * Test-support native: write the primary-key dedup sidecar for a flushed-generation dataset - * already staged at {@code genPath}. See {@link #writeFlushedGen}. + * Test-support native: write the primary-key dedup sidecar for an SSTable dataset already staged + * at {@code genPath}. See {@link #writeSsTable}. */ private static native void nativeWritePkSidecar( String genPath, long streamAddress, List pkColumns); @@ -382,6 +398,51 @@ void testShardWriterPutAndLsmScanner(@TempDir Path tempDir) throws Exception { } } + @Test + void testShardWriterDeleteMasksBaseRow(@TempDir Path tempDir) throws Exception { + String path = tempDir.resolve("base").toString(); + String shardId = UUID.randomUUID().toString(); + try (BufferAllocator allocator = new RootAllocator(); + Dataset dataset = writeLookupDataset(allocator, path, new long[] {1, 2, 3}, "base")) { + dataset.initializeMemWal(new InitializeMemWalParams()); + + ShardWriterConfig config = + new ShardWriterConfig() + .withDurableWrite(true) + .withMaxWalBufferSize(1) + .withMaxWalFlushIntervalMs(10); + + try (ShardWriter writer = dataset.memWalWriter(shardId, config)) { + try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {4}, "writer"); + ArrowReader reader = toReader(allocator, root)) { + writer.put(reader); + } + try (VectorSchemaRoot keys = keysRoot(allocator, new long[] {2}); + ArrowReader reader = toReader(allocator, keys)) { + writer.delete(reader); + } + + Map byId = Collections.emptyMap(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline) { + try (LsmScanner scanner = writer.lsmScanner(); + ArrowReader reader = scanner.scanBatches()) { + byId = readByName(reader); + } + if (!byId.containsKey(2L) && "writer_4".equals(byId.get(4L))) { + break; + } + Thread.sleep(50); + } + + assertEquals("base_1", byId.get(1L)); + assertFalse(byId.containsKey(2L), "deleted base row should be masked by the tombstone"); + assertEquals("base_3", byId.get(3L)); + assertEquals("writer_4", byId.get(4L)); + } + } + } + @Test void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception { String basePath = tempDir.resolve("base").toString(); @@ -390,12 +451,12 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception { Dataset dataset = writeLookupDataset(allocator, basePath, new long[] {1, 2, 3}, "base")) { dataset.initializeMemWal(new InitializeMemWalParams()); - // Flushed generation overwrites id=2. + // SSTable overwrites id=2. String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1"; - writeFlushedGen(allocator, genPath, new long[] {2}, "gen1"); + writeSsTable(allocator, genPath, new long[] {2}, "gen1"); ShardSnapshot snapshot = - new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2); + new ShardSnapshot(shardId).withSsTable(1, "gen_1").withCurrentGeneration(2); try (LsmScanner scanner = LsmScanner.fromSnapshots(dataset, Collections.singletonList(snapshot)); @@ -403,7 +464,7 @@ void testLsmScannerFromSnapshots(@TempDir Path tempDir) throws Exception { Map byId = readByName(reader); assertEquals(3, byId.size(), "Expected 3 deduplicated rows"); assertEquals("base_1", byId.get(1L)); - assertEquals("gen1_2", byId.get(2L), "Flushed generation must win over base"); + assertEquals("gen1_2", byId.get(2L), "SSTable must win over base"); assertEquals("base_3", byId.get(3L)); } @@ -426,14 +487,14 @@ void testPointLookup(@TempDir Path tempDir) throws Exception { dataset.initializeMemWal(new InitializeMemWalParams()); String genPath = basePath + "/_mem_wal/" + shardId + "/gen_1"; - writeFlushedGen(allocator, genPath, new long[] {2}, "gen1"); + writeSsTable(allocator, genPath, new long[] {2}, "gen1"); ShardSnapshot snapshot = - new ShardSnapshot(shardId).withFlushedGeneration(1, "gen_1").withCurrentGeneration(2); + new ShardSnapshot(shardId).withSsTable(1, "gen_1").withCurrentGeneration(2); try (LsmPointLookupPlanner planner = new LsmPointLookupPlanner(dataset, Collections.singletonList(snapshot))) { - // id=2 must resolve to the flushed-generation value. + // id=2 must resolve to the SSTable value. assertEquals("gen1_2", lookup(planner, allocator, 2L)); // id=1 only exists in the base table. assertEquals("base_1", lookup(planner, allocator, 1L)); @@ -459,7 +520,7 @@ private static String lookup(LsmPointLookupPlanner planner, BufferAllocator allo } @Test - void testMergeInsertMarkGenerationsAsMerged(@TempDir Path tempDir) throws Exception { + void testMergeInsertMarkSstablesAsCompacted(@TempDir Path tempDir) throws Exception { String path = tempDir.resolve("base").toString(); String shardId = UUID.randomUUID().toString(); try (BufferAllocator allocator = new RootAllocator()) { @@ -471,8 +532,8 @@ void testMergeInsertMarkGenerationsAsMerged(@TempDir Path tempDir) throws Except new MergeInsertParams(Collections.singletonList("id")) .withMatchedUpdateAll() .withNotMatched(MergeInsertParams.WhenNotMatched.InsertAll) - .markGenerationsAsMerged( - Collections.singletonList(new MergedGeneration(shardId, 1))); + .markSstablesAsCompacted( + Collections.singletonList(new CompactedSsTable(shardId, 1))); try (VectorSchemaRoot root = lookupRoot(allocator, new long[] {2, 4}, "merged"); ArrowReader reader = toReader(allocator, root); diff --git a/java/src/test/java/org/lance/namespace/CustomNamespace.java b/java/src/test/java/org/lance/namespace/CustomNamespace.java index e12489936b2..1d353756695 100644 --- a/java/src/test/java/org/lance/namespace/CustomNamespace.java +++ b/java/src/test/java/org/lance/namespace/CustomNamespace.java @@ -29,6 +29,7 @@ import org.lance.namespace.model.BatchDeleteTableVersionsRequest; import org.lance.namespace.model.BatchDeleteTableVersionsResponse; import org.lance.namespace.model.CountTableRowsRequest; +import org.lance.namespace.model.CountTableRowsResponse; import org.lance.namespace.model.CreateNamespaceRequest; import org.lance.namespace.model.CreateNamespaceResponse; import org.lance.namespace.model.CreateTableIndexRequest; @@ -84,7 +85,9 @@ import org.lance.namespace.model.MergeInsertIntoTableRequest; import org.lance.namespace.model.MergeInsertIntoTableResponse; import org.lance.namespace.model.NamespaceExistsRequest; +import org.lance.namespace.model.NamespaceExistsResponse; import org.lance.namespace.model.QueryTableRequest; +import org.lance.namespace.model.QueryTableResponse; import org.lance.namespace.model.RegisterTableRequest; import org.lance.namespace.model.RegisterTableResponse; import org.lance.namespace.model.RenameTableRequest; @@ -92,6 +95,7 @@ import org.lance.namespace.model.RestoreTableRequest; import org.lance.namespace.model.RestoreTableResponse; import org.lance.namespace.model.TableExistsRequest; +import org.lance.namespace.model.TableExistsResponse; import org.lance.namespace.model.UpdateTableRequest; import org.lance.namespace.model.UpdateTableResponse; import org.lance.namespace.model.UpdateTableSchemaMetadataRequest; @@ -174,8 +178,8 @@ public DropNamespaceResponse dropNamespace(DropNamespaceRequest request) { } @Override - public void namespaceExists(NamespaceExistsRequest request) { - inner.namespaceExists(request); + public NamespaceExistsResponse namespaceExists(NamespaceExistsRequest request) { + return inner.namespaceExists(request); } // Table operations @@ -196,8 +200,8 @@ public RegisterTableResponse registerTable(RegisterTableRequest request) { } @Override - public void tableExists(TableExistsRequest request) { - inner.tableExists(request); + public TableExistsResponse tableExists(TableExistsRequest request) { + return inner.tableExists(request); } @Override @@ -211,7 +215,7 @@ public DeregisterTableResponse deregisterTable(DeregisterTableRequest request) { } @Override - public Long countTableRows(CountTableRowsRequest request) { + public CountTableRowsResponse countTableRows(CountTableRowsRequest request) { return inner.countTableRows(request); } @@ -250,7 +254,7 @@ public DeleteFromTableResponse deleteFromTable(DeleteFromTableRequest request) { } @Override - public byte[] queryTable(QueryTableRequest request) { + public QueryTableResponse queryTable(QueryTableRequest request) { return inner.queryTable(request); } diff --git a/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java b/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java index c622bac9fcd..9a15063029c 100644 --- a/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java +++ b/java/src/test/java/org/lance/namespace/DirectoryNamespaceTest.java @@ -1161,7 +1161,7 @@ void testCountTableRows() throws Exception { // Count rows CountTableRowsRequest countReq = new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table")); - long count = namespaceClient.countTableRows(countReq); + long count = namespaceClient.countTableRows(countReq).getCount(); assertEquals(3, count); } @@ -1183,7 +1183,7 @@ void testCountTableRowsWithFilter() throws Exception { new CountTableRowsRequest() .id(Arrays.asList("workspace", "test_table")) .predicate("age > 28"); - long count = namespaceClient.countTableRows(countReq); + long count = namespaceClient.countTableRows(countReq).getCount(); assertEquals(2, count); // Alice (30) and Charlie (35) } @@ -1210,7 +1210,7 @@ void testInsertIntoTable() throws Exception { // Verify row count increased CountTableRowsRequest countReq = new CountTableRowsRequest().id(Arrays.asList("workspace", "test_table")); - long count = namespaceClient.countTableRows(countReq); + long count = namespaceClient.countTableRows(countReq).getCount(); assertEquals(6, count); } @@ -1233,7 +1233,7 @@ void testQueryTable() throws Exception { .id(Arrays.asList("workspace", "test_table")) .k(10) .vector(new QueryTableRequestVector()); - byte[] resultBytes = namespaceClient.queryTable(queryReq); + byte[] resultBytes = namespaceClient.queryTable(queryReq).getData(); assertNotNull(resultBytes); assertTrue(resultBytes.length > 0); } diff --git a/java/src/test/java/org/lance/operation/MergeTest.java b/java/src/test/java/org/lance/operation/MergeTest.java index 841ad41d5ea..1e037285217 100644 --- a/java/src/test/java/org/lance/operation/MergeTest.java +++ b/java/src/test/java/org/lance/operation/MergeTest.java @@ -104,10 +104,14 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { Merge.builder() .fragments(Collections.singletonList(evolvedFragment)) .schema(evolvedSchema) + .preservesNullability(true) .build()) .build()) { try (Dataset evolvedDataset = new CommitBuilder(initialDataset).execute(mergeTxn)) { Assertions.assertEquals(3, evolvedDataset.version()); + // The explicit non-default assertion must survive the JNI round trip. + Transaction readBack = evolvedDataset.readTransaction().orElseThrow(); + Assertions.assertTrue(((Merge) readBack.operation()).preservesNullability()); Assertions.assertEquals(rowCount, evolvedDataset.countRows()); Assertions.assertEquals(evolvedSchema, evolvedDataset.getSchema()); Assertions.assertEquals(3, evolvedDataset.getSchema().getFields().size()); @@ -136,6 +140,27 @@ void testMergeNewColumn(@TempDir Path tempDir) throws Exception { } } + @Test + void testPreservesNullabilityEquality() { + Schema schema = + new Schema( + Collections.singletonList(Field.nullable("id", new ArrowType.Int(32, true))), null); + // No assertion by default, and the assertion is part of the operation's identity. + Assertions.assertFalse( + Merge.builder() + .fragments(Collections.emptyList()) + .schema(schema) + .build() + .preservesNullability()); + Assertions.assertNotEquals( + Merge.builder().fragments(Collections.emptyList()).schema(schema).build(), + Merge.builder() + .fragments(Collections.emptyList()) + .schema(schema) + .preservesNullability(true) + .build()); + } + @Test void testMergeNewColumnWithNonContiguousFieldId(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("testMergeNewColumnWithNonContiguousFieldId").toString(); diff --git a/java/src/test/java/org/lance/operation/ProjectTest.java b/java/src/test/java/org/lance/operation/ProjectTest.java index fa0c92cc15f..bd3dd7d2960 100644 --- a/java/src/test/java/org/lance/operation/ProjectTest.java +++ b/java/src/test/java/org/lance/operation/ProjectTest.java @@ -30,6 +30,8 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class ProjectTest extends OperationTestBase { @@ -69,4 +71,36 @@ void testProjection(@TempDir Path tempDir) { } } } + + @Test + void testPreservesNullabilityEqualityAndRoundTrip(@TempDir Path tempDir) { + String datasetPath = tempDir.resolve("testAssertsNonNull").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + Schema schema = testDataset.getSchema(); + + // The assertion is part of the operation's identity. + assertNotEquals( + Project.builder().schema(schema).preservesNullability(true).build(), + Project.builder().schema(schema).preservesNullability(false).build()); + assertEquals( + Project.builder().schema(schema).preservesNullability(true).build(), + Project.builder().schema(schema).preservesNullability(true).build()); + + // The explicit non-default assertion must survive the JNI round trip. + try (Transaction txn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation(Project.builder().schema(schema).preservesNullability(true).build()) + .build()) { + try (Dataset committed = new CommitBuilder(dataset).execute(txn)) { + Transaction readBack = committed.readTransaction().orElseThrow(); + Project project = (Project) readBack.operation(); + assertTrue(project.preservesNullability()); + } + } + } + } } diff --git a/java/src/test/java/org/lance/operation/UpdateTest.java b/java/src/test/java/org/lance/operation/UpdateTest.java index bb39a5f4d12..0e69707c972 100644 --- a/java/src/test/java/org/lance/operation/UpdateTest.java +++ b/java/src/test/java/org/lance/operation/UpdateTest.java @@ -36,10 +36,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertThrows; public class UpdateTest extends OperationTestBase { @@ -104,6 +108,88 @@ void testUpdate(@TempDir Path tempDir) throws Exception { } } + @Test + void testUpdatedFragmentOffsetsRoundTrip(@TempDir Path tempDir) throws Exception { + String datasetPath = tempDir.resolve("testUpdatedFragmentOffsetsRoundTrip").toString(); + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath); + dataset = testDataset.createEmptyDataset(); + + // Append an initial fragment so we have a real fragment id. + FragmentMetadata fragmentMeta = testDataset.createNewFragment(10); + try (Transaction appendTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation( + Append.builder().fragments(Collections.singletonList(fragmentMeta)).build()) + .build()) { + new CommitBuilder(dataset).execute(appendTxn).close(); + } + + dataset = Dataset.open(datasetPath, allocator); + Fragment existingFragment = dataset.getFragments().get(0); + long fragmentId = existingFragment.getId(); + // Use the committed fragment's own metadata as the updatedFragment so that + // the updatedFragmentOffsets key is valid (must match a fragment in updatedFragments). + FragmentMetadata existingFragmentMeta = existingFragment.metadata(); + + // Build Update with non-empty updatedFragmentOffsets. Values are portable RoaringBitmap + // bytes encoding {1, 3, 5}: cookie(4) + containerCount(4) + key(2) + card-1(2) + + // offset(4) + elems(6). Offset = 16 (start of container data from beginning of stream). + Map offsets = new HashMap<>(); + offsets.put( + fragmentId, + new byte[] { + (byte) 0x3A, + (byte) 0x30, + (byte) 0x00, + (byte) 0x00, // cookie = 12346 + (byte) 0x01, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, // 1 container + (byte) 0x00, + (byte) 0x00, // container key 0 + (byte) 0x02, + (byte) 0x00, // cardinality - 1 = 2 + (byte) 0x10, + (byte) 0x00, + (byte) 0x00, + (byte) 0x00, // offset = 16 + (byte) 0x01, + (byte) 0x00, // element 1 + (byte) 0x03, + (byte) 0x00, // element 3 + (byte) 0x05, + (byte) 0x00 // element 5 + }); + + try (Transaction updateTxn = + new Transaction.Builder() + .readVersion(dataset.version()) + .operation( + Update.builder() + .updatedFragments(Collections.singletonList(existingFragmentMeta)) + .updateMode(Optional.of(UpdateMode.RewriteColumns)) + .updatedFragmentOffsets(offsets) + .build()) + .build()) { + try (Dataset committed = new CommitBuilder(dataset).execute(updateTxn)) { + // Read the committed transaction back (exercises the IntoJava JNI path). + try (Transaction readTx = committed.readTransaction().orElseThrow()) { + assertInstanceOf(Update.class, readTx.operation()); + Update readOp = (Update) readTx.operation(); + + Map readOffsets = readOp.updatedFragmentOffsets(); + assertEquals(1, readOffsets.size()); + assertArrayEquals(offsets.get(fragmentId), readOffsets.get(fragmentId)); + } + } + } + } + } + @Test void testUpdateColumns(@TempDir Path tempDir) throws Exception { String datasetPath = tempDir.resolve("testUpdateColumns").toString(); diff --git a/java/src/test/java/org/lance/otel/LanceMetricsTest.java b/java/src/test/java/org/lance/otel/LanceMetricsTest.java new file mode 100644 index 00000000000..b165bec3d36 --- /dev/null +++ b/java/src/test/java/org/lance/otel/LanceMetricsTest.java @@ -0,0 +1,172 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.lance.otel; + +import org.lance.Dataset; +import org.lance.TestUtils; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.DoublePointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import org.apache.arrow.memory.RootAllocator; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class LanceMetricsTest { + private static final String REQUESTS = "lance_object_store_requests_total"; + private static final String DURATION = "lance_object_store_request_duration_seconds"; + private static final String RETRYABLE = "lance_object_store_retryable_responses_total"; + private static final String IN_FLIGHT = "lance_object_store_in_flight_requests"; + + @AfterEach + void closeLanceMetrics() { + LanceMetrics.close(); + } + + @Test + void testInstrumentLanceMetricsExportsObjectStoreMetrics(@TempDir Path tempDir) { + InMemoryMetricReader reader = InMemoryMetricReader.create(); + SdkMeterProvider provider = SdkMeterProvider.builder().registerMetricReader(reader).build(); + + try { + assertTrue(LanceMetrics.instrument(provider)); + + Map catalog = + LanceMetrics.catalog().stream() + .collect(Collectors.toMap(MetricDescription::getName, Function.identity())); + assertTrue(catalog.containsKey(REQUESTS)); + assertEquals("histogram", catalog.get(DURATION).getKind()); + assertEquals("counter", catalog.get(RETRYABLE).getKind()); + assertEquals("gauge", catalog.get(IN_FLIGHT).getKind()); + + generateObjectStoreMetrics(tempDir.resolve("otel_metrics.lance")); + + MetricPoint requests = + LanceMetrics.snapshot().stream() + .filter(point -> REQUESTS.equals(point.getName())) + .findFirst() + .orElseThrow(() -> new AssertionError("expected object store request metric")); + assertNotNull(requests.getValue()); + assertTrue(requests.getValue() > 0); + assertTrue(requests.getAttributes().containsKey("operation")); + assertTrue(requests.getAttributes().containsKey("base")); + + Collection metrics = reader.collectAllMetrics(); + Map metricsByName = + metrics.stream().collect(Collectors.toMap(MetricData::getName, Function.identity())); + Set names = metricsByName.keySet(); + assertTrue(names.contains(REQUESTS)); + assertTrue(names.contains(DURATION + "_bucket")); + assertTrue(names.contains(DURATION + "_count")); + assertTrue(names.contains(DURATION + "_sum")); + + Collection bucketPoints = + metricsByName.get(DURATION + "_bucket").getDoubleSumData().getPoints(); + Collection countPoints = + metricsByName.get(DURATION + "_count").getDoubleSumData().getPoints(); + Collection sumPoints = + metricsByName.get(DURATION + "_sum").getDoubleSumData().getPoints(); + + assertFalse(bucketPoints.isEmpty()); + assertTrue( + bucketPoints.stream() + .allMatch(point -> point.getAttributes().get(AttributeKey.stringKey("le")) != null)); + + Map countsByAttributes = + countPoints.stream() + .collect(Collectors.toMap(DoublePointData::getAttributes, DoublePointData::getValue)); + Map infiniteBucketsByAttributes = new HashMap<>(); + for (DoublePointData bucketPoint : bucketPoints) { + if ("+Inf".equals(bucketPoint.getAttributes().get(AttributeKey.stringKey("le")))) { + Attributes attributes = + bucketPoint.getAttributes().toBuilder().remove(AttributeKey.stringKey("le")).build(); + infiniteBucketsByAttributes.put(attributes, bucketPoint.getValue()); + } + } + assertEquals(countsByAttributes, infiniteBucketsByAttributes); + assertTrue(sumPoints.stream().anyMatch(point -> point.getValue() > 0)); + } finally { + provider.close(); + } + } + + @Test + void testInstrumentReRegistersWithNewProvider(@TempDir Path tempDir) { + InMemoryMetricReader firstReader = InMemoryMetricReader.create(); + InMemoryMetricReader secondReader = InMemoryMetricReader.create(); + SdkMeterProvider firstProvider = + SdkMeterProvider.builder().registerMetricReader(firstReader).build(); + SdkMeterProvider secondProvider = + SdkMeterProvider.builder().registerMetricReader(secondReader).build(); + + try { + assertTrue(LanceMetrics.instrument(firstProvider)); + generateObjectStoreMetrics(tempDir.resolve("first_provider.lance")); + assertTrue(metricNames(firstReader).contains(REQUESTS)); + + assertTrue(LanceMetrics.instrument(secondProvider)); + assertTrue(metricNames(secondReader).contains(REQUESTS)); + assertFalse(metricNames(firstReader).contains(REQUESTS)); + } finally { + firstProvider.close(); + secondProvider.close(); + } + } + + @Test + void testSupportedMetricsSkipsUnknownKinds() { + MetricDescription counter = new MetricDescription("counter", "counter", null, "counter"); + MetricDescription unknown = new MetricDescription("unknown", "summary", null, "unknown"); + + List supported = + LanceMetrics.supportedMetrics(Arrays.asList(counter, unknown)); + + assertEquals(1, supported.size()); + assertEquals("counter", supported.get(0).getName()); + } + + private static void generateObjectStoreMetrics(Path datasetPath) { + try (RootAllocator allocator = new RootAllocator(Long.MAX_VALUE)) { + TestUtils.SimpleTestDataset testDataset = + new TestUtils.SimpleTestDataset(allocator, datasetPath.toString()); + try (Dataset dataset = testDataset.createEmptyDataset()) { + assertEquals(0, dataset.countRows()); + } + } + } + + private static Set metricNames(InMemoryMetricReader reader) { + Collection metrics = reader.collectAllMetrics(); + return metrics.stream().map(MetricData::getName).collect(Collectors.toSet()); + } +} diff --git a/protos/AGENTS.md b/protos/AGENTS.md index 23aef9fc196..290affd3e61 100644 --- a/protos/AGENTS.md +++ b/protos/AGENTS.md @@ -2,9 +2,17 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. +## Change Process + +- Changes to protobuf schemas that define a persisted Lance format require a PMC vote on the pull request, enforced by the `format-spec-vote` CI gate. See [Lance Format Specification Changes](../docs/src/community/voting.md#lance-format-specification-changes). +- Keep a persisted-format proto change in its own PR, together with the matching `docs/src/format/` change and only the library edits needed to compile. Put the implementation in a follow-up PR — voters need to read the contract, not its implementation. +- Execution-plan schemas (`ann.proto`, `filtered_read.proto`, and `table_identifier.proto`) are wire contracts, not persisted Lance formats. Changes to them belong with their implementation and do not require a format vote or a `docs/src/format/` change. + ## Compatibility -- All changes must be backwards compatible. Never re-use or change field numbers of existing fields. +- Protobuf schemas that are part of a stable file format or any other stable persisted contract must remain backwards compatible. Never reuse or change their existing field numbers. +- Protobuf schemas used exclusively by an unstable file format follow the root file-format stability contract: do not preserve compatibility with prior unstable revisions. Before making a breaking protobuf change, verify that the schema is not shared with a stable format or another persisted contract. +- Execution wire schemas may still cross process or version boundaries. Preserve their field-number compatibility unless all producers and consumers are upgraded atomically. ## Schema Design diff --git a/protos/encodings_v2_1.proto b/protos/encodings_v2_1.proto index 46fd012fb58..51427332063 100644 --- a/protos/encodings_v2_1.proto +++ b/protos/encodings_v2_1.proto @@ -147,6 +147,128 @@ message FullZipLayout { repeated RepDefLayer layers = 8; } +// A layout used for sparse flat or nested pages where Arrow structure is represented directly +// in layer-local slot domains instead of as dense repetition / definition events. +// +// Structural layers are ordered from outer-most to inner-most. Values remain mini-block +// compressed and are split into independently readable chunks. +message SparseLayout { + // Description of the compression of values. + CompressiveEncoding value_compression = 1; + // Number of value buffers in each mini-block chunk. This does not include structural buffers. + uint64 num_buffers = 2; + // Number of entries in the equivalent dense repetition / definition stream. This equals + // num_visible_items plus one structural placeholder for every list slot without children. + // Null leaf slots count as visible items because they still occupy positions in Arrow's + // leaf value buffer. For example, a nullable primitive with 100 slots, 30 of them null, + // has num_items = num_visible_items = 100. + uint64 num_items = 3; + // Number of leaf value slots encoded in the value chunks, including null leaf slots. + uint64 num_visible_items = 4; + // If true, chunk-local value buffer sizes use u32. Otherwise they use u16. + bool has_large_chunk = 5; + // Structural layers ordered from outer-most to inner-most. This may be empty for a flat, + // non-nullable leaf page whose scheduling domain equals num_visible_items. + repeated SparseStructuralLayer structural_layers = 6; +} + +// A domain is a layer-local integer coordinate space [0, num_slots). A slot is one +// element in that space. The outer-most domain is the page's top-level rows; each +// layer's child domain is the next layer's parent domain, and the terminal child +// domain contains num_visible_items leaf value slots. +message SparseStructuralLayer { + // Exactly one layer kind is required. + oneof layer { + SparseValidityLayer validity = 1; + SparseListLayer list = 2; + SparseFixedSizeListLayer fixed_size_list = 3; + } +} + +message SparseValidityLayer { + // Number of nullable item or struct slots in this layer's parent and child domain. + uint64 num_slots = 1; + // Validity for the slots in this layer. + SparseValiditySet validity = 2; +} + +message SparseListLayer { + // Number of list, large-list, or map slots in this layer's parent domain. + uint64 num_slots = 1; + // Number of slots in this layer's child domain. + uint64 num_child_slots = 2; + // Non-empty parent slots. Valid parent slots absent from this set are empty lists. + SparsePositionSet non_empty_positions = 3; + // Positive child counts corresponding one-for-one with non_empty_positions. + SparseCountSet counts = 4; + // Validity for the parent slots in this layer. + SparseValiditySet validity = 5; +} + +message SparseFixedSizeListLayer { + // Number of fixed-size-list slots in this layer's parent domain. + uint64 num_slots = 1; + // Number of children per parent slot. The child domain has num_slots * dimension slots. + uint64 dimension = 2; + // Validity for the parent slots in this layer. + SparseValiditySet validity = 3; +} + +message SparseValiditySet { + enum Meaning { + SPARSE_VALIDITY_UNSPECIFIED = 0; + // Stored positions are null; all other positions are valid. + SPARSE_VALIDITY_NULL_POSITIONS = 1; + // Stored positions are valid; all other positions are null. + SPARSE_VALIDITY_VALID_POSITIONS = 2; + } + + Meaning meaning = 1; + SparsePositionSet positions = 2; +} + +message SparsePositionEmpty {} + +message SparsePositionAll {} + +message SparsePositionRange { + uint64 start = 1; + uint64 length = 2; +} + +message SparsePositionSet { + oneof positions { + // Delta-compressed u64 positions. Cardinality is num_positions. + CompressiveEncoding explicit = 1; + // One contiguous, non-empty range. + SparsePositionRange range = 2; + // Every position in the domain. + SparsePositionAll all = 3; + // No positions in the domain. + SparsePositionEmpty empty = 4; + } + // Semantic cardinality of this set. + uint64 num_positions = 5; +} + +message SparseCountEmpty {} + +message SparseCountConstant { + // Child count shared by every non-empty list slot. + uint64 value = 1; +} + +message SparseCountSet { + oneof counts { + // Compressed u64 child counts. Cardinality comes from the containing position set. + CompressiveEncoding explicit = 1; + // One positive child count shared by every non-empty list slot. + SparseCountConstant constant = 2; + // No counts; valid only when there are no non-empty list slots. + SparseCountEmpty empty = 3; + } +} + // A layout used for pages where all (visible) values are the same scalar value. // // This generalizes the prior AllNullLayout semantics for file_version >= 2.2. @@ -206,6 +328,8 @@ message PageLayout { // A layout where large binary data is encoded externally // and only the descriptions are put in the page BlobLayout blob_layout = 4; + // A sparse structural layout. This variant requires file version 2.3 or later. + SparseLayout sparse_layout = 5; } } diff --git a/protos/file2.proto b/protos/file2.proto index da0b1d5e96c..650a1568da6 100644 --- a/protos/file2.proto +++ b/protos/file2.proto @@ -207,4 +207,4 @@ message ColumnMetadata { // // This file format is extremely minimal. It is a building block for // creating more useful readers and writers and not terribly useful by itself. -// Other protobuf files will describe how this can be extended. \ No newline at end of file +// Other protobuf files will describe how this can be extended. diff --git a/protos/filtered_read.proto b/protos/filtered_read.proto index d81f6b02cfb..4fb1c8a81e7 100644 --- a/protos/filtered_read.proto +++ b/protos/filtered_read.proto @@ -62,6 +62,16 @@ message FilteredReadOptionsProto { optional uint64 io_buffer_size_bytes = 11; // Arrow IPC schema for decoding Substrait filters (may be wider than projection). optional bytes filter_schema_ipc = 12; + // If present, a nonzero upper bound on bytes reserved by Blob v2 + // materialization awaiting ordered emission in one scanner execution. + // Admission follows output order; one oversized output batch may exceed the + // bound when no other batch is reserved. If absent, Blob v2 materialization + // has no independent memory bound. + optional uint64 materialization_readahead_bytes = 13; + // If present, the scanner-level byte budget for output batches. When set, + // the file reader uses it as an additional batch boundary alongside the + // row-based batch_size. This corresponds to FileReaderOptions.batch_size_bytes. + optional uint64 batch_size_bytes = 14; } // Serializable form of FilteredReadPlan (planned/distributed mode). diff --git a/protos/index_old.proto b/protos/index_old.proto index 601aa2681da..236d1f110c0 100644 --- a/protos/index_old.proto +++ b/protos/index_old.proto @@ -24,8 +24,55 @@ message BTreeIndexDetails {} message BitmapIndexDetails {} message LabelListIndexDetails {} message NGramIndexDetails {} -message ZoneMapIndexDetails {} +message ZoneMapIndexDetails { + // Number of rows per zone. Optional for backwards compatibility: absent on + // datasets written before this field was added. When absent, no seed writer + // is created for the index. + optional uint64 rows_per_zone = 1; + // Whether seed-based incremental updates are enabled for this index. + // On-disk semantics: absent means seeds are disabled (old datasets written + // before this field was added). Present false means explicitly disabled. + // Present true means seeds are enabled: the index will embed per-fragment + // seed buffers in data files and harvest them during incremental updates + // to skip full column scans. + // Creation-time default: index creation code sets this to true for + // variable-length types (strings, binary) and fixed-width types wider than + // 8 bytes, and to false for narrow fixed-width types (e.g. Int64, Float64). + optional bool use_seeds = 2; + // Whether this index tracks exact null row addresses in a separate bitmap. + // Absent or false means legacy format: null positions are not tracked and + // IS NULL searches fall back to approximate zone-level statistics. Present + // true means IS NULL is exact and IS NOT NULL can be answered without a + // full scan. + optional bool has_null_bitmap = 3; +} message InvertedIndexDetails { + enum DocumentGranularity { + ROW = 0; + LIST_ELEMENT = 1; + } + + message CodeTokenizerConfig { + // Split one lexical identifier into subwords, e.g. getUserName -> + // get/user/name. + bool split_identifiers = 1; + // Split identifier subwords across letter/number boundaries, e.g. + // HTML2JSON -> html/2/json. An absent value uses the code tokenizer default; + // a present value records the explicit index-time choice. + optional bool split_on_numerics = 2; + // Keep the complete lexical identifier in addition to subwords, e.g. + // user_name plus user/name. An absent value uses the code tokenizer default; + // a present value records the explicit index-time choice. + optional bool preserve_original = 3; + // Index operator tokens such as "::", "->", and "!=". Operators are not + // indexed by default because they are often high-frequency noise. + bool index_operators = 4; + } + + // Lexical tokenizer used after document-level text extraction. This is an + // implementation component such as "simple", "icu", "ngram", or "code". + // Input-time analyzer profiles are expanded into this field and the concrete + // options below before these details are persisted. // Marking this field as optional as old versions of the index store blank details and we // need to make sure we have a proper optional field to detect this. optional string base_tokenizer = 1; @@ -39,4 +86,19 @@ message InvertedIndexDetails { uint32 min_ngram_length = 9; uint32 max_ngram_length = 10; bool prefix_only = 11; + // Number of documents per compressed posting block. An absent value means + // the index predates this field and must use the legacy block size of 128. + // A present value records the block size used by the index; 256 is valid + // with format versions 3 and 4. + optional uint32 block_size = 12; + // Options for base_tokenizer = "code". Presence records the code tokenizer + // configuration used to build the index; absence means there is no + // code-specific configuration to apply. + CodeTokenizerConfig code_config = 13; + // The logical FTS document boundary. The protobuf default preserves the + // legacy row-document behavior when this field is absent. + DocumentGranularity document_granularity = 14; + // The posting-list payload format. This is separate from index_version, + // which identifies the overall inverted-index layout. + optional uint32 posting_format_version = 15; } diff --git a/protos/table.proto b/protos/table.proto index 8d0cb249fda..e9722a780e5 100644 --- a/protos/table.proto +++ b/protos/table.proto @@ -115,6 +115,19 @@ message Manifest { // * 1 << 3: table config is present // * 1 << 4: dataset uses multiple base paths // * 1 << 5: transaction file writes are disabled + // * 1 << 6: data overlay files are present (see DataOverlayFile). Readers that do + // not understand overlays must refuse the dataset, since ignoring an overlay + // would silently return stale base values. + // * 1 << 7: some index declares covering columns, so IndexMetadata.fields means + // the keyed columns followed by the carried ones named in covering_fields (see + // IndexMetadata). Readers that do not understand it must refuse the dataset, + // since selecting an index by membership of fields would answer a query on a + // merely-carried column with an index keyed on a different column. Writers must + // refuse it too: one that treats every entry of fields as keyed would maintain + // the index against the wrong dependency set. + // * 1 << 8: reserved for datasets that may reference recognized V2 data files + // with different exact versions. Implementations that do not support the + // per-file exact-version contract must treat this bit as unknown. uint64 reader_feature_flags = 9; // Feature flags for writers. @@ -236,6 +249,9 @@ message IndexMetadata { UUID uuid = 1; // The columns to build the index. These refer to file.Field.id. + // + // fields[0] is always a column the index is keyed on. Trailing entries may + // instead be merely carried, not keyed on -- see `covering_fields` below. repeated int32 fields = 2; // Index name. Must be unique within one dataset version. @@ -288,6 +304,29 @@ message IndexMetadata { // of index sizes without extra IO. // If this is empty, the index files sizes are unknown. repeated IndexFile files = 10; + + // The subset of `fields` whose values this index co-locates alongside its own + // data, so a query projecting only those columns can be answered from the + // index without a take against the base table. + // + // Must be a suffix of `fields`: the columns the index is keyed on come first, + // the columns it merely carries come last, and at least one keyed column + // always remains. Empty for an index that carries no extra columns, which is + // every index written before this field existed. + // + // Carried columns are listed in `fields` as well. That is deliberate: every + // consumer that reads `fields` as the index's dependency set -- staleness, + // commit conflict detection, schema evolution guards -- then covers them with + // no change and no way to forget one. + // + // This declaration is not authoritative for what the segment can actually + // serve. The segment's own storage schema is: a reader must confirm the + // storage carries a column before answering a query from it, and fall back to + // a take against the base table otherwise. A declaration naming columns the + // storage does not hold is a legal state, not corruption -- a maintenance + // operation that cannot carry the values through a rebuild is permitted to + // withdraw the payload while leaving this declaration in place. + repeated int32 covering_fields = 11; } // Metadata about a single file within an index segment. @@ -313,6 +352,15 @@ message DataFragment { repeated DataFile files = 2; + // Optional overlay files for this fragment, which supply new values for a + // subset of cells without rewriting the base data files. This MUST be empty + // if the data overlay files feature flag (64) is not set in the manifest. + // + // Order is significant: a later entry is newer than an earlier one. When two + // overlays cover the same (offset, field) and share a `committed_version`, the + // later entry wins. See DataOverlayFile for the full resolution rules. + repeated DataOverlayFile overlays = 11; + // File that indicates which rows, if any, should be considered deleted. DeletionFile deletion_file = 3; @@ -435,6 +483,66 @@ message DataFile { optional uint32 base_id = 7; } // DataFile +// An overlay file supplies new values for a subset of (row offset, field) cells +// within a fragment, without rewriting the fragment's base data files. It is +// used for efficient updates when only a small fraction of rows and/or columns +// change. +// +// On read, a cell is resolved by consulting the fragment's overlays from newest +// to oldest: the first overlay that covers that (offset, field) wins; if none +// cover it, the value falls through to the base data file. Because deletions +// take precedence over overlays, an overlay value for an offset that is also +// marked deleted is dead and is ignored. +// +// The overlay's data file does NOT store a row-offset key column. Within a value +// column, the position of a covered offset's value is the rank (0-based count of +// set bits below it) of that offset within the field's coverage bitmap. Because +// fields may cover different offset sets, the value columns of a single overlay +// data file may have different lengths (which the Lance file format permits). +message DataOverlayFile { + // The data file storing the overlay's new cell values, one value column per + // field in `data_file.fields`. No row-offset key column is stored. + DataFile data_file = 1; + + // Which (offset, field) cells this overlay provides values for. + oneof coverage { + // A single 32-bit Roaring bitmap of physical row offsets that applies to + // every field in `data_file.fields` (a "dense" / rectangular overlay). + // Every covered offset has a value for every field. This is the common case + // for a plain UPDATE, where one SET list is applied to one set of rows. + bytes shared_offset_bitmap = 2; + // Per-field coverage for a "sparse" overlay, used when different fields cover + // different offset sets (e.g. a MERGE with multiple WHEN MATCHED branches). + FieldCoverage field_coverage = 4; + } + + // The dataset version at which this overlay became effective: the version of + // the commit that introduced it, NOT the version it was read from. It is + // stamped at commit time and re-stamped if the commit is retried, in the same + // way as the created-at / last-updated-at version sequences. + // + // This drives two orderings: + // * Versus index builds: an index whose `dataset_version` >= this value + // already incorporates this overlay. Otherwise the overlay's covered cells + // are excluded from index results for the affected fields and re-evaluated + // against their current values (see the Data Overlay Files specification). + // * Versus other overlays: when two overlays cover the same (offset, field), + // the one with the higher `committed_version` wins. Overlays that share a + // `committed_version` are ordered by their position in + // `DataFragment.overlays`, where a later entry is newer and wins. + uint64 committed_version = 3; +} + +// Per-field coverage for a sparse overlay. +message FieldCoverage { + // One entry per field in the overlay's `data_file.fields`, in the same order. + // Each is a 32-bit Roaring bitmap of the physical row offsets covered for that + // field. An offset present in a field's bitmap but mapped to a NULL value + // means the cell is overridden to NULL (distinct from an offset that is absent, + // which falls through to the base data file). + repeated bytes offset_bitmaps = 1; +} + // Deletion File // // The path of the deletion file is constructed as: @@ -570,14 +678,14 @@ message ShardManifest { // files to find actual state. uint64 wal_entry_position_last_seen = 4; - // Next generation ID to create (incremented after each MemTable flush). + // Generation to assign to the next SSTable (incremented after each MemTable flush). uint64 current_generation = 6; - // Field 7 removed: merged_generation moved to MemWalIndexDetails.merged_generations - // which is the authoritative source for merge progress. + // Field 7 removed: compaction progress lives in + // MemWalIndexDetails.compacted_sstables. - // List of flushed MemTable generations and their directory paths. - repeated FlushedGeneration flushed_generations = 8; + // List of SSTables created by flushing MemTables and their directory paths. + repeated SsTable sstables = 8; // Lifecycle status. Default ACTIVE; SEALED marks an in-flight drop // (drop-table 2PC). A SEALED manifest refuses claims at claim_epoch. @@ -594,40 +702,41 @@ message ShardFieldEntry { bytes value = 2; } -// A flushed MemTable generation and its storage location. -message FlushedGeneration { - // Generation number. +// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset. +message SsTable { + // Generation number identifying this SSTable. uint64 generation = 1; // Directory name relative to the shard directory. string path = 2; } -// A shard's merged generation, used in MemWalIndexDetails. -message MergedGeneration { +// A pointer to the latest SSTable compacted for a shard. +message CompactedSsTable { // Shard identifier (UUID v4). UUID shard_id = 1; - // Last generation merged to base table for this shard. + // Generation of the latest SSTable compacted into the base table for this shard. uint64 generation = 2; } -// Tracks which merged generation a base table index has been rebuilt to cover. -// Used to determine whether to read from flushed MemTable indexes or base table. +// Tracks which compacted SSTable generation a base table index has been rebuilt to cover. +// Used to determine whether to read from SSTable indexes or base table. message IndexCatchupProgress { // Name of the base table index (must match an entry in maintained_indexes). string index_name = 1; // Per-shard progress: the generation up to which this index covers. - // If a shard is not present, the index is assumed to be fully caught up - // (i.e., caught_up_generation >= merged_generation for that shard). - repeated MergedGeneration caught_up_generations = 2; + // + // An absent shard means *unknown*: this index has recorded no catch-up for + // that shard, so its SSTables must be retained and a repair scheduled. + repeated CompactedSsTable caught_up_generations = 2; } // Index details for MemWAL Index, stored in IndexMetadata.index_details. // This is the centralized structure for all MemWAL metadata: // - Configuration (sharding specs, indexes to maintain) -// - Merge progress (merged generations per shard) +// - SSTable compaction progress // - Shard state snapshots // // Writers read this index to get configuration before writing. @@ -669,24 +778,28 @@ message MemWalIndexDetails { // SQ params) from the base table index to ensure distance comparability. repeated string maintained_indexes = 8; - // Last generation merged to base table for each shard. + // Latest SSTable compacted into the base table for each shard. // This is updated atomically with merge-insert data commits, enabling - // conflict resolution when multiple mergers operate concurrently. + // conflict resolution when multiple compactors operate concurrently. // // Note: This is separate from shard snapshots because: - // 1. merged_generations is updated by mergers (atomic with data commit) + // 1. compacted_sstables is updated by compactors (atomic with data commit) // 2. shard snapshots are updated by background index builder - repeated MergedGeneration merged_generations = 9; + repeated CompactedSsTable compacted_sstables = 9; // Per-index catchup progress tracking. - // When data is merged to the base table, base table indexes are rebuilt + // When data is compacted into the base table, base table indexes are rebuilt // asynchronously. This field tracks which generation each index covers. // - // For indexed queries, if an index's caught_up_generation < merged_generation, - // readers should use flushed MemTable indexes for the gap instead of + // For indexed queries, if an index's caught_up_generation < compacted_generation, + // readers should use SSTable indexes for the gap instead of // scanning unindexed data in the base table. // - // If an index is not present in this list, it is assumed to be fully caught up. + // An index absent from this list has recorded no catch-up, so the SSTables it + // would need stay live until a repair records it. Only the dedicated WAL + // index-repair path may add entries here; + // ordinary index operations have their entry removed automatically when they + // change an index, since they do not report what the new index covers. repeated IndexCatchupProgress index_catchup = 10; // Default ShardWriter configuration values for this MemWAL index. diff --git a/protos/transaction.proto b/protos/transaction.proto index e72e95025a4..ec03eb143bf 100644 --- a/protos/transaction.proto +++ b/protos/transaction.proto @@ -144,12 +144,24 @@ message Transaction { repeated lance.file.Field schema = 2; // Schema metadata. map schema_metadata = 3; + // Set when this merge makes no nullability-affecting schema change: it + // introduces no field that data staged against an earlier schema could + // not safely omit. Without the assertion (including transactions written + // before this field existed) the merge conservatively conflicts with + // concurrent value-writes, which can only cause a retry. + bool preserves_nullability = 4; } // An operation that projects a subset of columns, altering the schema. message Project { // The new schema repeated lance.file.Field schema = 1; + // Set when this projection makes no nullability-affecting schema change, + // as a rename or a drop does not. Without the assertion (including + // transactions written before this field existed) the projection + // conservatively conflicts with concurrent value-writes, which can only + // cause a retry. A nullability tightening must not set this. + bool preserves_nullability = 2; } // An operation that restores a dataset to a previous version. @@ -243,8 +255,8 @@ message Transaction { repeated DataFragment new_fragments = 3; // The ids of the fields that have been modified. repeated uint32 fields_modified = 4; - /// List of MemWAL shard generations to mark as merged after this transaction - repeated MergedGeneration merged_generations = 5; + /// SSTables to mark as compacted after this transaction. + repeated CompactedSsTable compacted_sstables = 5; /// The fields that used to judge whether to preserve the new frag's id into /// the frag bitmap of the specified indices. repeated uint32 fields_for_preserving_frag_bitmap = 6; @@ -254,7 +266,12 @@ message Transaction { // Only tracks keys from INSERT operations during merge insert, not updates. optional KeyExistenceFilter inserted_rows = 8; // Per-fragment physical row offsets that matched an update_columns hash join (RewriteColumns). + // Deprecated: use updated_fragment_offset_bitmaps (field 10) instead. map updated_fragment_offsets = 9; + // Per-fragment matched offsets as portable RoaringBitmap bytes (replaces field 9). + // Writers emit field 10 only. Readers prefer field 10; fall back to field 9 for + // manifests written before this change. + map updated_fragment_offset_bitmaps = 10; } // The mode of update operation @@ -315,12 +332,34 @@ message Transaction { repeated DataReplacementGroup replacements = 1; } - // Update the merged generations in MemWAL index. + // Overlay files to append to a single fragment, in order (the last entry is + // newest). The overlays are appended to the fragment's existing `overlays` + // list; they do not replace it, so overlays written by concurrent commits are + // preserved. + message DataOverlayGroup { + uint64 fragment_id = 1; + // Each DataOverlayFile.committed_version is left 0 by the writer and stamped + // to the new dataset version at commit time (re-stamped on retry), in the + // same way as the created-at / last-updated-at version sequences. The fields + // touched are read from each overlay's `data_file.fields`. + repeated DataOverlayFile overlays = 2; + } + + // Attach overlay files to fragments, supplying new values for a subset of + // (row offset, field) cells without rewriting the fragments' base data files. + // See the DataOverlayFile message in table.proto for resolution, coverage, and + // versioning rules, and the Data Overlay Files and Transactions specifications + // for the (intentionally permissive) conflict semantics. + message DataOverlay { + repeated DataOverlayGroup groups = 1; + } + + // Update SSTable compaction progress in the MemWAL index. // This operation is used during merge-insert to atomically record which - // generations have been merged to the base table. + // SSTables have been compacted into the base table. message UpdateMemWalState { - // Shards and generations being marked as merged. - repeated MergedGeneration merged_generations = 1; + // SSTables being marked as compacted. + repeated CompactedSsTable compacted_sstables = 1; } // An operation that updates base paths in the dataset. @@ -346,6 +385,7 @@ message Transaction { UpdateMemWalState update_mem_wal_state = 112; Clone clone = 113; UpdateBases update_bases = 114; + DataOverlay data_overlay = 115; } // Fields 200/202 (`blob_append` / `blob_overwrite`) previously represented blob dataset ops. diff --git a/python/.cargo/config.toml b/python/.cargo/config.toml index f9f9bc0544a..5e72e0592aa 100644 --- a/python/.cargo/config.toml +++ b/python/.cargo/config.toml @@ -22,6 +22,11 @@ rustflags = [ ] [target.x86_64-unknown-linux-gnu] +# See the note in ../../.cargo/config.toml for details on this choice. +# +# Note that pylance users cannot easily change this value (they would need to build +# pylance from source). This is an intentional choice to provide strong performance +# by default for pylance wheels. rustflags = ["-C", "target-cpu=haswell", "-C", "target-feature=+avx2,+fma,+f16c"] [target.aarch64-apple-darwin] diff --git a/python/AGENTS.md b/python/AGENTS.md index 2025f4ef9df..40301a4ddc7 100644 --- a/python/AGENTS.md +++ b/python/AGENTS.md @@ -7,12 +7,10 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. * Environment: use `uv` for all local Python environment setup in this repository. * First step in every new worktree or fresh checkout: run `make install` from `python/` before any Python command. This runs `uv sync` (dev and test dependencies are included by default via `[tool.uv] default-groups`) and sets up pre-commit hooks. Add `--group benchmarks`, `--extra torch`, or `--extra geo` only when needed. * `uv sync` builds the local `pylance` Rust extension as part of environment setup. This can take a long time. Start it early, let it finish, and do not interrupt it or switch to a different setup path just because the build is slow. -* After the initial `make install`, use `uv run make test` to run tests. * Only run `uv sync` again when dependencies change (e.g., after pulling new commits that update `pyproject.toml` or `uv.lock`). * Command execution: always use `uv run ...` for Python-related repository commands. Do not rely on a globally activated environment. * Never invoke bare `python`, `pytest`, `pip`, `maturin`, `make test`, `make doctest`, `make lint`, or `make format` for repository work. * If a Python command fails outside `uv run`, that does not count as a dependency or test failure. Fix the environment usage first and rerun correctly. -* Build time expectations: `make install` and `make build` build the local `pylance` Rust extension as part of the environment workflow. This can be slow, especially on the first run or after Rust dependency changes; treat that as expected and do not switch to a different environment manager or shortcut around the build just because it takes time. * Build: `make build` (required after Rust changes) * Test: `uv run make test` * Run single test: `uv run pytest python/tests/.py::` @@ -39,11 +37,4 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. ## Testing -- Use `@pytest.mark.parametrize` for tests that differ only in inputs — extract shared setup into helpers. - Add tests to existing `test_{module}.py` files rather than creating new test files for the same module. -- Replace `print()` in tests with `assert` statements. - -## Common Failure Mode - -- A missing module or missing command error from bare `python`, `pytest`, `pip`, `maturin`, or `make` is usually an environment usage mistake, not a repository issue. -- Before reporting a Python dependency as unavailable, verify that `uv sync` has been run in the current worktree and that the failing command was executed with `uv run ...`. diff --git a/python/Cargo.lock b/python/Cargo.lock index 440538da1ee..ef387e01568 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -2,54 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "abi_stable" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" -dependencies = [ - "abi_stable_derive", - "abi_stable_shared", - "const_panic", - "core_extensions", - "crossbeam-channel", - "generational-arena", - "libloading", - "lock_api", - "parking_lot", - "paste", - "repr_offset", - "rustc_version", - "serde", - "serde_derive", - "serde_json", -] - -[[package]] -name = "abi_stable_derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" -dependencies = [ - "abi_stable_shared", - "as_derive_utils", - "core_extensions", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", - "typed-arena", -] - -[[package]] -name = "abi_stable_shared" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" -dependencies = [ - "core_extensions", -] - [[package]] name = "adler2" version = "2.0.1" @@ -172,9 +124,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "approx" @@ -208,9 +160,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" dependencies = [ "arrow-arith", "arrow-array", @@ -230,9 +182,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -263,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -275,9 +227,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -286,7 +238,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -297,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ "arrow-array", "arrow-cast", @@ -312,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -325,9 +277,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,9 +318,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -379,9 +331,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" dependencies = [ "arrow-array", "arrow-data", @@ -391,9 +343,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -404,9 +356,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -415,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -429,9 +381,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ "arrow-array", "arrow-buffer", @@ -444,18 +396,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "as_derive_utils" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" -dependencies = [ - "core_extensions", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "async-channel" version = "2.5.0" @@ -485,9 +425,6 @@ name = "async-ffi" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" -dependencies = [ - "abi_stable", -] [[package]] name = "async-lock" @@ -513,13 +450,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.2", ] [[package]] @@ -1007,6 +944,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1188,9 +1131,23 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "byteorder" @@ -1200,9 +1157,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "bytes-utils" @@ -1225,9 +1182,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" dependencies = [ "find-msvc-tools", "jobserver", @@ -1309,9 +1266,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1319,9 +1276,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1331,14 +1288,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.2", ] [[package]] @@ -1502,21 +1459,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_extensions" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bb5e5d0269fd4f739ea6cedaf29c16d81c27a7ce7582008e90eb50dcd57003" -dependencies = [ - "core_extensions_proc_macros", -] - -[[package]] -name = "core_extensions_proc_macros" -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533d38ecd2709b7608fb8e18e4504deb99e9a72879e6aa66373a76d8dc4259ea" - [[package]] name = "countio" version = "0.3.0" @@ -1545,12 +1487,13 @@ dependencies = [ ] [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ - "rustc_version", + "digest 0.10.7", + "spin 0.10.1", ] [[package]] @@ -1583,18 +1526,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.12" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" dependencies = [ "crossbeam-utils", ] @@ -1787,14 +1730,13 @@ dependencies = [ [[package]] name = "datafusion" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b" +checksum = "754ef4e8f073922a26f5b23133b9db4829342362b09be0bc94309cf261c2f098" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1821,12 +1763,11 @@ dependencies = [ "datafusion-session", "datafusion-sql", "futures", + "indexmap 2.14.0", "itertools 0.14.0", "log", "object_store", "parking_lot", - "rand 0.9.4", - "regex", "sqlparser", "tempfile", "tokio", @@ -1836,9 +1777,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66" +checksum = "06afd1e38dd27bbb1258685a1fc6524df6aff4e07b25b393a47de59635178d99" dependencies = [ "arrow", "async-trait", @@ -1861,9 +1802,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb" +checksum = "f0668fb32c12065ec242be0e5b4bc62bd7a06a0be3ecd83791ef877e4be67e02" dependencies = [ "arrow", "async-trait", @@ -1884,33 +1825,34 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2" +checksum = "ca43b263cdff57042cfa8fb817fb3469f4878933380dccff25f5e793580abbf9" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", "object_store", "parquet", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def" +checksum = "05f0ba2b864792bdca4d76c59a1de0ab6e1b61946596b9936888dbd6360035f2" dependencies = [ "futures", "log", @@ -1919,9 +1861,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6" +checksum = "b840a8bce0bcbf5afad02946d438591e7c373f7afccaf3d874c04485772514dd" dependencies = [ "arrow", "async-trait", @@ -1941,16 +1883,17 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.4", + "parking_lot", + "rand 0.9.5", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096" +checksum = "a24cc0b9cf6e367f27f27406eff13abf48a11b72446aaa40b3105c0ded5c17d9" dependencies = [ "arrow", "arrow-ipc", @@ -1972,9 +1915,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca" +checksum = "e1abe56b2a7a2d1d6de5117dd1a203181e267f28529faa5da546947621b697d7" dependencies = [ "arrow", "async-trait", @@ -1995,9 +1938,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2" +checksum = "9c3e467f0611ad7bdd5aad17c63c9bb6182d04e5282e5496d897ea2b49c024ba" dependencies = [ "arrow", "async-trait", @@ -2012,16 +1955,15 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] [[package]] name = "datafusion-datasource-parquet" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997" +checksum = "4cc35b92cd560082155e80d9c826929c852d3c51543f4affd3a51c464a0aab3a" dependencies = [ "arrow", "async-trait", @@ -2031,6 +1973,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -2049,20 +1992,19 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee" +checksum = "d69bb69d8769e34f76839c960dbde24c1ac0c885a79b6c3c2287bdc56ec67891" [[package]] name = "datafusion-execution" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709" +checksum = "d8eac0a09bc8d263f52025cad9e001da4d8138d633fa288edda4d06b1772eae6" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -2071,18 +2013,19 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd" +checksum = "eeb14d374767ee0fc62dc79a5ba8bcf8a63c14e993c7d992d0e63adfa23d77d3" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2093,35 +2036,33 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] [[package]] name = "datafusion-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e" +checksum = "b7b19a8c95522bee8cbb313d74263b85e355d2b52f42e67ef5694bf5de9e9356" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-ffi" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95173344d04ba62755c949bf44f8d1a6e4414cf6392a635db96c07e711b9a3c" +checksum = "d9a4a09f24f1387408810acca070207a9cf6c7fc2368122982a8e7671f643d1e" dependencies = [ - "abi_stable", "arrow", "arrow-schema", "async-ffi", "async-trait", + "chrono", "datafusion-catalog", "datafusion-common", "datafusion-datasource", @@ -2130,26 +2071,29 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-proto", "datafusion-proto-common", "datafusion-session", "futures", + "libloading", "log", "prost", "semver", + "stabby", "tokio", ] [[package]] name = "datafusion-functions" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6" +checksum = "5f64c983bbbdcb729d921a2b2ac3375598719b5cc0c30345ad664936f3176fc7" dependencies = [ "arrow", "arrow-buffer", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -2160,26 +2104,25 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", + "datafusion-physical-expr-common", "hex", "itertools 0.14.0", "log", - "md-5 0.10.6", + "md-5 0.11.0", "memchr", "num-traits", - "rand 0.9.4", + "rand 0.9.5", "regex", - "sha2 0.10.9", - "unicode-segmentation", + "sha2 0.11.0", "uuid", ] [[package]] name = "datafusion-functions-aggregate" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad" +checksum = "89bc17041e424a47ed062f43df24d84aab8b57c4c3221e5c1a5eef46d6c5718b" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -2189,19 +2132,18 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47" +checksum = "97dd2a9e865c6108059f5b37b77934f84b50bfb108f837bd0e5c9536e03f0545" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2210,9 +2152,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a" +checksum = "75f0bdfeef16d96417b9632ef855645376b242e9006a126dfd0bedfc54a93f5f" dependencies = [ "arrow", "arrow-ord", @@ -2226,34 +2168,34 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e" +checksum = "f4e4941673c917819616877e9993da4503e4f4739812be0bc32c5356184c6383" dependencies = [ "arrow", "async-trait", "datafusion-catalog", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6" +checksum = "12dd2e16c12b84b6f6b41b19f55b366dd1c46876bb35b86896c6349067379e8d" dependencies = [ "arrow", "datafusion-common", @@ -2264,14 +2206,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c" +checksum = "4cdc5e4b6f8b6ef823cc1c761f85088ad4c884fe8df64df3cbcc6b2b84698441" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2279,9 +2220,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd" +checksum = "1a3614234dd93578c92428cb4f408e020874f0d2b7e6c90c928d9d28b5df2ceb" dependencies = [ "datafusion-doc", "quote", @@ -2290,9 +2231,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace" +checksum = "a0635620b050b81bb92764e99250868f654e2cd5ad1bece413283b3f73c83179" dependencies = [ "arrow", "chrono", @@ -2309,11 +2250,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59" +checksum = "8cabf7a86eb70b816729e33c81bf7767c936ee1226f607a114f5dac2decac8d0" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2321,20 +2261,19 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4" +checksum = "de222e04f7e6744501555a54ab0abe26bfdfebee380af79a9bdc175704246859" dependencies = [ "arrow", "datafusion-common", @@ -2347,26 +2286,26 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362" +checksum = "72d0d0057fc5a502d45c870cb6d47c66eb7bdd5edb1bd71ad6f3f724975ac2a8" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", + "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95" +checksum = "86046eed10950c5f9aaed9acfd148e9bd2e1dfdfe4f9aef607d1447b271e4183" dependencies = [ "arrow", "datafusion-common", @@ -2382,12 +2321,13 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79" +checksum = "9bc84da934c903407ba297971ebcc020c4c1a38aafd765d6c144c76eff3fa6a1" dependencies = [ - "ahash", "arrow", + "arrow-data", + "arrow-ipc", "arrow-ord", "arrow-schema", "async-trait", @@ -2402,7 +2342,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2414,9 +2354,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +checksum = "67791dcfacd142a9d95f8f73c2a161094cc936d231d80c235f5017dacf24e84e" dependencies = [ "arrow", "chrono", @@ -2437,14 +2377,13 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost", - "rand 0.9.4", ] [[package]] name = "datafusion-proto-common" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +checksum = "b8cd9e80d637891645d074db0f6c650b591117367247deb313fdfb78dff559cb" dependencies = [ "arrow", "datafusion-common", @@ -2453,9 +2392,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a" +checksum = "eb63eeac6de19be40f487b65dd84e546195f783c5a9928618e0c4f2a3569b0d7" dependencies = [ "arrow", "datafusion-common", @@ -2464,15 +2403,14 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e" +checksum = "5f961d209177f91bd014db5cbb2c33b7d28a2597b9003e77f17aeb712964315a" dependencies = [ "async-trait", "datafusion-common", @@ -2484,9 +2422,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1" +checksum = "b1d71cb454da682b2af7488e1fc1ddd72ee1b28f19297b8ccad73f0a21ee9a69" dependencies = [ "arrow", "bigdecimal", @@ -2502,9 +2440,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "53.1.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98494539a5468979cc42d86c7bc5f0f8cb71ee5c742694c26fc34efdd29dd2e5" +checksum = "f047a6fbf967b6b523758a48d2377bb5f9373a20e7ae4c4326d3c569f80ba3d7" dependencies = [ "async-recursion", "async-trait", @@ -2698,6 +2636,12 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "env_filter" version = "2.0.0" @@ -2745,11 +2689,10 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -2842,6 +2785,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frostem" +version = "1.20260804.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0ae10cfccbae085dd8612669ccc116fe88bd5dfe39d202a8fa68c24c1e546f" + [[package]] name = "fs_extra" version = "1.3.0" @@ -2850,10 +2799,10 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -2968,15 +2917,6 @@ dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "generational-arena" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" -dependencies = [ - "cfg-if 1.0.4", -] - [[package]] name = "generator" version = "0.8.9" @@ -3087,12 +3027,13 @@ dependencies = [ [[package]] name = "geodatafusion" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cd430f1a1f59bc97053d824ad410ea6fd123c8977b3c1a75335e289233b8b" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" dependencies = [ "arrow-arith", "arrow-array", + "arrow-buffer", "arrow-schema", "datafusion", "geo", @@ -3144,11 +3085,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if 1.0.4", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -3205,17 +3144,25 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae079b88ffe7772d12cfc5c40a5a324babb357893d95b5e3a22ae857f236c5f" +checksum = "4a9bc9414e3b2cb0bd08dfe0eb315b177e86b119c7fa5e16179c92fc7b184860" dependencies = [ + "arc-swap", "async-trait", "bytes", "dashmap", + "futures", "hostname", + "io-uring", + "itoa", + "libc", + "lru", + "memmap2", + "moka", "prost", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "serde", "thiserror 2.0.18", @@ -3225,13 +3172,14 @@ dependencies = [ "tonic-prost", "tracing", "uuid", + "xxhash-rust", ] [[package]] name = "h2" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -3252,6 +3200,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ + "bytemuck", "cfg-if 1.0.4", "crunchy", "num-traits", @@ -3285,6 +3234,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.1.5", ] @@ -3304,6 +3255,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heapify" @@ -3459,9 +3415,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" @@ -3530,7 +3486,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3867,18 +3823,18 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jieba-macros" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" +checksum = "38fc0f3831de71556de69643b80a08a5c8cd260a23c6b8dbeb7cd923c779cac5" dependencies = [ "phf_codegen", ] [[package]] name = "jieba-rs" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" +checksum = "bb5bdea4dc241d589e179f39d2a778f31490f3370aa2f626223dbd930ebc5c9d" dependencies = [ "bytecount", "cedarwood", @@ -4015,7 +3971,7 @@ dependencies = [ "nom", "num-traits", "ordered-float 5.3.0", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "zmij", @@ -4049,7 +4005,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arc-swap", "arrow", @@ -4065,7 +4021,6 @@ dependencies = [ "async-recursion", "async-trait", "async_cell", - "aws-credential-types", "aws-sdk-dynamodb", "byteorder", "bytes", @@ -4081,7 +4036,6 @@ dependencies = [ "either", "fst", "futures", - "half", "humantime", "itertools 0.14.0", "lance-arrow", @@ -4105,7 +4059,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rayon", "roaring", "rustc-hash", @@ -4123,7 +4077,7 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", @@ -4132,13 +4086,14 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-select", + "bytemuck", "bytes", "futures", "getrandom 0.2.17", "half", "jsonb", "num-traits", - "rand 0.9.4", + "rand 0.9.5", ] [[package]] @@ -4165,7 +4120,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrayref", "crunchy", @@ -4175,19 +4130,18 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "async-trait", - "byteorder", + "blake3", "bytes", "datafusion-common", "datafusion-sql", "futures", - "itertools 0.14.0", "lance-arrow", "lance-derive", "libc", @@ -4198,13 +4152,13 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.4", + "quick_cache", + "rand 0.9.5", "roaring", "serde_json", "snafu", "tempfile", "tokio", - "tokio-stream", "tokio-util", "tracing", "twox-hash", @@ -4213,7 +4167,7 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4230,10 +4184,10 @@ dependencies = [ "datafusion-physical-expr", "datafusion-substrait", "futures", + "half", "jsonb", "lance-arrow", "lance-core", - "lance-datagen", "lance-geo", "log", "pin-project", @@ -4245,7 +4199,7 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4255,14 +4209,14 @@ dependencies = [ "futures", "half", "hex", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rand_xoshiro", ] [[package]] name = "lance-derive" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "proc-macro2", "quote", @@ -4271,7 +4225,7 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-arith", "arrow-array", @@ -4296,8 +4250,6 @@ dependencies = [ "num-traits", "prost", "prost-build", - "rand 0.9.4", - "strum 0.26.3", "tokio", "tracing", "xxhash-rust", @@ -4306,11 +4258,12 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-arith", "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -4336,7 +4289,7 @@ dependencies = [ [[package]] name = "lance-geo" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "datafusion", "geo-traits", @@ -4350,12 +4303,13 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arc-swap", "arrow", "arrow-arith", "arrow-array", + "arrow-ipc", "arrow-ord", "arrow-schema", "arrow-select", @@ -4364,7 +4318,6 @@ dependencies = [ "async-trait", "bitvec", "bytes", - "chrono", "crossbeam-queue", "datafusion", "datafusion-common", @@ -4385,10 +4338,10 @@ dependencies = [ "lance-bitpacking", "lance-core", "lance-datafusion", - "lance-datagen", "lance-encoding", "lance-file", "lance-geo", + "lance-index-core", "lance-io", "lance-linalg", "lance-select", @@ -4402,7 +4355,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "rangemap", "rayon", @@ -4414,22 +4367,37 @@ dependencies = [ "tempfile", "tokio", "tracing", - "uuid", +] + +[[package]] +name = "lance-index-core" +version = "12.0.0-beta.11" +dependencies = [ + "arrow-array", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "datafusion", + "datafusion-common", + "datafusion-expr", + "futures", + "lance-core", + "lance-io", + "lance-select", + "prost-types", + "roaring", + "serde", + "serde_json", ] [[package]] name = "lance-io" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", - "arrow-arith", "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", "arrow-schema", - "arrow-select", - "async-recursion", "async-trait", "aws-config", "aws-credential-types", @@ -4439,10 +4407,10 @@ dependencies = [ "futures", "http 1.4.2", "io-uring", - "lance-arrow", "lance-core", "lance-namespace", "log", + "metrics", "moka", "object_store", "object_store_opendal", @@ -4450,45 +4418,50 @@ dependencies = [ "path_abs", "pin-project", "prost", - "rand 0.9.4", + "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", + "serde_json", "tempfile", "tokio", "tracing", "url", + "uuid", ] [[package]] name = "lance-linalg" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", - "arrow-buffer", "arrow-schema", "cc", "half", "lance-arrow", "lance-core", "num-traits", - "rand 0.9.4", "rayon", ] [[package]] name = "lance-namespace" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", + "serde_json", "snafu", ] [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-ipc", @@ -4508,12 +4481,11 @@ dependencies = [ "lance-table", "log", "object_store", - "rand 0.9.4", + "rand 0.9.5", "reqwest 0.12.28", "roaring", "serde", "serde_json", - "time", "tokio", "tower", "tower-http 0.5.2", @@ -4523,9 +4495,9 @@ dependencies = [ [[package]] name = "lance-namespace-reqwest-client" -version = "0.8.6" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +checksum = "d8d23e54b1634d5bbb434f8dd33dc3c05f6e58d876a9a27b3b4aef58ddbe11af" dependencies = [ "reqwest 0.12.28", "serde", @@ -4537,13 +4509,12 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", "byteorder", - "bytes", "itertools 0.14.0", "lance-core", "roaring", @@ -4552,7 +4523,7 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "arrow", "arrow-array", @@ -4562,6 +4533,7 @@ dependencies = [ "async-trait", "aws-credential-types", "aws-sdk-dynamodb", + "blake3", "byteorder", "bytes", "chrono", @@ -4576,7 +4548,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.4", + "rand 0.9.5", "rangemap", "roaring", "semver", @@ -4591,12 +4563,12 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ + "frostem", "icu_segmenter", "jieba-rs", "lindera", - "rust-stemmers", "serde", "stop-words", "unicode-normalization", @@ -4608,7 +4580,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.9", ] [[package]] @@ -4670,18 +4642,18 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libloading" -version = "0.7.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if 1.0.4", - "winapi", + "windows-link", ] [[package]] @@ -4727,8 +4699,8 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "unicode-blocks", "unicode-normalization", "unicode-segmentation", @@ -4757,8 +4729,8 @@ dependencies = [ "rkyv", "serde", "serde_json", - "strum 0.28.0", - "strum_macros 0.28.0", + "strum", + "strum_macros", "thiserror 2.0.18", ] @@ -4814,6 +4786,15 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4913,13 +4894,43 @@ checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" dependencies = [ "libc", ] +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "metrics", + "ordered-float 4.6.0", + "quanta", + "radix_trie", + "rand 0.9.5", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -5024,6 +5035,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nom" version = "8.0.0" @@ -5189,7 +5209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "form_urlencoded", @@ -5224,9 +5244,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.57.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" +checksum = "88f165780495c17aa3ce86846600504198c3fffd99073521552751c2430fa6ac" dependencies = [ "async-trait", "bytes", @@ -5259,12 +5279,13 @@ checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" [[package]] name = "opendal" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" +checksum = "4f20562cc7447fcc915fc5c23df305a412ea80a733c9f2fd9e2d267e2815be6d" dependencies = [ "ctor 1.0.7", "opendal-core", + "opendal-http-transport-reqwest", "opendal-layer-concurrent-limit", "opendal-layer-logging", "opendal-layer-retry", @@ -5282,24 +5303,22 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" +checksum = "ec75551ff4cf3e57da98979f6a937aaa9ddb3915bf68cc17d03df733be6646ed" dependencies = [ "anyhow", - "base64", + "base64 0.23.0", "bytes", "futures", "http 1.4.2", - "http-body 1.0.1", "jiff", "log", "md-5 0.11.0", "mea", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", - "reqwest 0.13.4", "serde", "serde_json", "tokio", @@ -5308,11 +5327,25 @@ dependencies = [ "web-time", ] +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4d4f19c3ce01126a30611f8e544eaa217104a278c889ac17c9374fe4f9e4ef" +dependencies = [ + "bytes", + "futures", + "http 1.4.2", + "http-body 1.0.1", + "opendal-core", + "reqwest 0.13.4", +] + [[package]] name = "opendal-layer-concurrent-limit" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" +checksum = "249ac5b0aa5a7a6c3737342d10456067937f9c9a6f3f02544271f7908ab91081" dependencies = [ "futures", "http 1.4.2", @@ -5322,9 +5355,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" +checksum = "5c75411ab00f77851ff086b686c1e9ca8175ac18c15afa2cb75b9036436cb06c" dependencies = [ "log", "opendal-core", @@ -5332,9 +5365,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" +checksum = "80b7738bd5f233ad8da39af9b9316b9b7a4eaddd91e8e32a1e19b7030688121d" dependencies = [ "backon", "log", @@ -5343,9 +5376,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" +checksum = "a704141924500f3803c05ed871b53305d2a2f11cb5ef20160c3ee688a1857f66" dependencies = [ "opendal-core", "tokio", @@ -5353,17 +5386,17 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +checksum = "b3310fbbb48f111c6f590473c2cd15e1b7f8e384444b0d4e328f0464c864d767" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5374,17 +5407,18 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dea4908d490143a9b0b7f7a790e139ff829b06a023f670455ed3d44f664b361" +checksum = "2e3c406729935fe214ce574d68681a1ff7e0b322548f14094912bdbfe50e5c53" dependencies = [ - "base64", + "base64 0.23.0", "bytes", "http 1.4.2", "log", + "mea", "opendal-core", "opendal-service-azure-common", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-azure-storage", "reqsign-core", "reqsign-file-read-tokio", @@ -5394,9 +5428,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +checksum = "7348c88edf15af435b7be930077746b569fac5e738c1bf6a363b675e7317c9df" dependencies = [ "http 1.4.2", "opendal-core", @@ -5404,15 +5438,15 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8cafe9729213375c7331019b0cb756ad3e1aff7f45cd32c45eae91ebde8901" +checksum = "d533d4582105d269c8aebeee5f0e8bcf960f41b8aab6197df7012254d9f39bf0" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-tencent-cos", @@ -5421,9 +5455,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" +checksum = "007f3fba63c21e516c956b891e96ff9892d8175662bfb781cdada9d3766a11e6" dependencies = [ "async-trait", "bytes", @@ -5431,7 +5465,7 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-google", @@ -5442,9 +5476,9 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e43048bde419947ba826fbdc2f134d6c03f44ebf48bd33a03b72f9fc45fcb4" +checksum = "60871e6386f04d831e6a5bdbc032af4a91aeba49963252d0ef456a2cf36a9b78" dependencies = [ "bytes", "goosefs-sdk", @@ -5456,9 +5490,9 @@ dependencies = [ [[package]] name = "opendal-service-hf" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4922661976a1d40794a2adfbdb888cc3c23097690f825a92f773af38908a848" +checksum = "b41fd41eb7ed03c5e66cefda61e8e117808ffd2908f2916737cb020a6beb02c7" dependencies = [ "bytes", "hf-xet", @@ -5466,22 +5500,21 @@ dependencies = [ "log", "opendal-core", "percent-encoding", - "reqwest 0.13.4", "serde", "serde_json", ] [[package]] name = "opendal-service-oss" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" +checksum = "cd528ec2d49c5ca69e674ffed7b3e0686fb9cfcfea0596870de381467fda4f1b" dependencies = [ "bytes", "http 1.4.2", "log", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aliyun-oss", "reqsign-core", "reqsign-file-read-tokio", @@ -5490,18 +5523,18 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" +checksum = "58e80cdf192d7eff05feed747894d64f81905ac4eaf132edf7ea270abdd2d663" dependencies = [ - "base64", + "base64 0.23.0", "bytes", - "crc32c", + "crc-fast", "http 1.4.2", "log", "md-5 0.11.0", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-aws-v4", "reqsign-core", "reqsign-file-read-tokio", @@ -5511,14 +5544,14 @@ dependencies = [ [[package]] name = "opendal-service-tos" -version = "0.57.0" +version = "0.58.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2f7a4c32e5202eb4ac72e76c4b5e30c86ab60762811172f4111103b9d673a1" +checksum = "7841a1a09485bd08eeac34d67804321b62456c855027c580bce12a75564f4609" dependencies = [ "bytes", "http 1.4.2", "opendal-core", - "quick-xml 0.39.4", + "quick-xml 0.41.0", "reqsign-core", "reqsign-file-read-tokio", "reqsign-volcengine-tos", @@ -5547,6 +5580,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.3.0" @@ -5612,9 +5654,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", "arrow-array", @@ -5623,7 +5665,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -5670,7 +5712,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "898bac3fa00d0ba57a4e8289837e965baa2dee8c3749f3b11d45a64b4223d9c3" dependencies = [ - "base64", + "base64 0.22.1", "serde", ] @@ -5717,7 +5759,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -6029,7 +6071,7 @@ dependencies = [ [[package]] name = "pylance" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" dependencies = [ "alloc-stdlib", "arrow", @@ -6061,6 +6103,8 @@ dependencies = [ "lance-table", "libc", "log", + "metrics", + "metrics-util", "object_store", "prost", "prost-types", @@ -6074,6 +6118,7 @@ dependencies = [ "tracing", "tracing-chrome", "tracing-subscriber", + "url", "uuid", ] @@ -6146,6 +6191,21 @@ dependencies = [ "serde", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.39.4" @@ -6158,14 +6218,26 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.40.1" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quick_cache" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c6658afe513a3b484e3abfdaa0d03ef3c0bbf017542c178dd55f94eb3051f9" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6188,15 +6260,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.1", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -6249,6 +6322,16 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rancor" version = "0.1.1" @@ -6270,9 +6353,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -6340,7 +6423,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -6358,6 +6450,24 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" +[[package]] +name = "rapidhash" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b266a82f4aa99bb5c25e28d11cc44ace63d91adbcbcee4d323e2ae3d49ef37" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -6487,20 +6597,11 @@ dependencies = [ "bytecheck", ] -[[package]] -name = "repr_offset" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" -dependencies = [ - "tstr", -] - [[package]] name = "reqsign-aliyun-oss" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372266b4733756738eeb199a98188037d27a0989980e2600ae7ce1faf00a867d" +checksum = "9c0f9f69a519dd6958c4b43606bb8e1278cdc76d611fc8fed4b796eee548dc0f" dependencies = [ "anyhow", "form_urlencoded", @@ -6515,9 +6616,9 @@ dependencies = [ [[package]] name = "reqsign-aws-v4" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75624bd8a466e37ddc0a7b6c33ac859a85347c153a916e1dd9d0b68338f74a" +checksum = "cc883bc56889f3e4a419265c87facea222a921debc5c6f15c7fd8b68ec4b36b2" dependencies = [ "anyhow", "bytes", @@ -6526,7 +6627,7 @@ dependencies = [ "http 1.4.2", "log", "percent-encoding", - "quick-xml 0.40.1", + "quick-xml 0.41.0", "reqsign-core", "rust-ini", "serde", @@ -6537,12 +6638,12 @@ dependencies = [ [[package]] name = "reqsign-azure-storage" -version = "3.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "http 1.4.2", @@ -6558,14 +6659,13 @@ dependencies = [ [[package]] name = "reqsign-core" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fa5cb48808693614d1701fcd3db0b30fa292e0f18e122ae068b6d32eaeed3f" +checksum = "7e38b44697c60a823705ccef85cb04d8e0527c9d16ed7c58bf1c6395bdd24ceb" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "bytes", - "form_urlencoded", "futures", "hex", "hmac 0.13.0", @@ -6583,9 +6683,9 @@ dependencies = [ [[package]] name = "reqsign-file-read-tokio" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4b6f3a3fd29ffcc99a90aec585a65217783badfd73acddf847b63ae683bda9" +checksum = "688ff0ae421b8d4b92b53fdafaf53df2de28f428a9962edcf21702990b26f74b" dependencies = [ "anyhow", "reqsign-core", @@ -6594,9 +6694,9 @@ dependencies = [ [[package]] name = "reqsign-google" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb215d0876a18b6bd9cdd380b589e5292aaa638ca15266de794b1122d898b6b2" +checksum = "a96da0b579b846d358090cb06b9e3c2ad1375529efbe3e0c45f96bd7bcf043ea" dependencies = [ "form_urlencoded", "http 1.4.2", @@ -6612,9 +6712,9 @@ dependencies = [ [[package]] name = "reqsign-tencent-cos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84110aabba799fbcd48b3abb51fbbff4749f879252e5806b6f5d0cbe0fef6abb" +checksum = "f6497dd9f6e3d1349b420521484099b284f95e8d3a65f088fccef42493a7b644" dependencies = [ "anyhow", "http 1.4.2", @@ -6627,9 +6727,9 @@ dependencies = [ [[package]] name = "reqsign-volcengine-tos" -version = "3.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d083a363b3577f519ce8425bb50f902622a28a83f7c4a26a5c990b66ec75b3" +checksum = "4335f949a3fd8b53867fd716dac97fbd545e45bcaed44343796517f2e19cd609" dependencies = [ "anyhow", "http 1.4.2", @@ -6644,7 +6744,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -6690,7 +6790,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -6754,9 +6854,9 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck", "bytes", @@ -6773,9 +6873,9 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.16" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", @@ -6840,21 +6940,11 @@ dependencies = [ "ordered-multimap", ] -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -7111,9 +7201,9 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -7121,22 +7211,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.2", ] [[package]] @@ -7152,10 +7242,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -7215,7 +7306,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", @@ -7310,6 +7401,12 @@ dependencies = [ "cc", ] +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + [[package]] name = "sharded-slab" version = "0.1.7" @@ -7384,6 +7481,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" @@ -7398,18 +7501,18 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "snafu" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" +checksum = "e45cb604038abb7b926b679887b3226d8d0f23874b66623625a0454be425a4b7" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" +checksum = "287f59010008f0d7cf5e3b03196d666c1acc46c8d3e9cf34c28a1a7157601e72" dependencies = [ "heck", "proc-macro2", @@ -7447,9 +7550,15 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -7463,9 +7572,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.61.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", "sqlparser_derive", @@ -7482,6 +7591,40 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "stabby" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7b834ec7ced12095fea1e4b07dcb7e8cf2b59b18afa3eac52494d835965a5ec" +dependencies = [ + "rustversion", + "stabby-abi", +] + +[[package]] +name = "stabby-abi" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1a4f477858a5bdf927c9fab7f579899de9b13e39f8b3b3b300c89fbab632f4" +dependencies = [ + "rustc_version", + "rustversion", + "sha2-const-stable", + "stabby-macros", +] + +[[package]] +name = "stabby-macros" +version = "72.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b31c4b2434980b67ad83f300a58088ba14d59454dcd79ba3d87419bbd924d31e" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -7531,35 +7674,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.118", + "strum_macros", ] [[package]] @@ -7576,11 +7697,12 @@ dependencies = [ [[package]] name = "substrait" -version = "0.62.2" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62fc4b483a129b9772ccb9c3f7945a472112fdd9140da87f8a4e7f1d44e045d0" +checksum = "e620ff4d5c02fd6f7752931aa74b16a26af66a63022cc1ad412c77edbe0bab47" dependencies = [ "heck", + "indexmap 2.14.0", "pbjson", "pbjson-build", "pbjson-types", @@ -7613,9 +7735,9 @@ checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -7624,9 +7746,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" dependencies = [ "proc-macro2", "quote", @@ -7856,9 +7978,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -7905,9 +8027,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -7917,13 +8039,14 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] @@ -7965,7 +8088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.4.2", @@ -8173,36 +8296,15 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "tstr" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" -dependencies = [ - "tstr_proc_macros", -] - -[[package]] -name = "tstr_proc_macros" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" - [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" dependencies = [ - "rand 0.9.4", + "rand 0.10.1", ] -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - [[package]] name = "typenum" version = "1.20.1" @@ -8351,9 +8453,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -8934,7 +9036,7 @@ checksum = "3e1e496dcbe6a09017acdfaf48e1a646735e7ff5b2a49e2c7e081cca77a59bc8" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.22.1", "bytes", "clap", "crc32fast", @@ -8971,7 +9073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb838aa8eb67d730af301584cf003caad407487606058292a6750711b603fbee" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "blake3", "bytemuck", "bytes", @@ -9081,9 +9183,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yoke" diff --git a/python/Cargo.toml b/python/Cargo.toml index f0c84c01a16..13c7fc93689 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pylance" -version = "9.0.0-beta.16" +version = "12.0.0-beta.11" edition = "2024" authors = ["Lance Devs "] license = "Apache-2.0" @@ -25,9 +25,10 @@ arrow-cast = "58.0.0" arrow-data = "58.0.0" arrow-schema = "58.0.0" object_store = "0.13.2" -datafusion = { version = "53.0.0", default-features = false } -datafusion-ffi = "53.0.0" -datafusion-common = "53.0.0" +url = "2.5.7" +datafusion = { version = "54.0.0", default-features = false } +datafusion-ffi = "54.0.0" +datafusion-common = "54.0.0" # Keep the Python FFI build on the working Brotli allocator resolution until # datafusion-ffi no longer enables datafusion-proto/default. # See https://github.com/lance-format/lance/issues/7271. @@ -44,6 +45,7 @@ lance = { path = "../rust/lance", features = [ "goosefs", "dynamodb", "substrait", + "metrics", ] } lance-arrow = { path = "../rust/lance-arrow" } lance-core = { path = "../rust/lance-core" } @@ -54,7 +56,7 @@ lance-index = { path = "../rust/lance-index", features = [ "tokenizer-lindera", "tokenizer-jieba", ] } -lance-io = { path = "../rust/lance-io" } +lance-io = { path = "../rust/lance-io", features = ["metrics"] } lance-linalg = { path = "../rust/lance-linalg" } lance-namespace = { path = "../rust/lance-namespace" } lance-namespace-impls = { path = "../rust/lance-namespace-impls", features = ["rest", "rest-adapter", "dir-goosefs"] } @@ -62,10 +64,11 @@ lance-table = { path = "../rust/lance-table" } lance-datafusion = { path = "../rust/lance-datafusion" } libc = "0.2.176" log = "0.4" +metrics = "0.24" +metrics-util = "0.19" prost = "0.14.1" prost-types = "0.14.1" pyo3 = { version = "0.28", features = [ - "extension-module", "abi3-py310", "py-clone", "chrono", @@ -80,11 +83,12 @@ serde_yaml = "0.9.34" tracing-chrome = "0.7.1" tracing-subscriber = "0.3.17" tracing = { version = "0.1" } -bytes = "1.4" +bytes = "1.11.1" [features] -default = [] +default = ["extension-module"] datagen = ["lance-datagen"] +extension-module = ["pyo3/extension-module"] fp16kernels = ["lance/fp16kernels"] [profile.ci] diff --git a/python/Makefile b/python/Makefile index d5077019f35..12dad5196c6 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,7 +1,7 @@ .DEFAULT_GOAL := help .PHONY: help install build test integtest doctest compattest format format-python lint lint-python lint-rust clean PYTHON ?= -PYTEST_ARGS ?= -vvv -s -m "not recurring" +PYTEST_ARGS ?= -vvv -s KEEP_COMPOSE ?= 0 COMPOSE_FILE ?= ../docker-compose.yml UV_SYNC = uv sync @@ -18,6 +18,17 @@ ifeq ($(CI), true) PYTEST_ARGS += --durations=30 endif +# Number of pytest-xdist workers, e.g. `auto` or an explicit count. Empty (the +# default) runs serially; CI sets it per runner size. Much of the suite is spent +# waiting on IO rather than saturating the Rust core's own thread pools, so +# sharding across processes wins back time a single pytest process leaves idle. +PYTEST_WORKERS ?= +ifneq ($(strip $(PYTEST_WORKERS)),) + # loadgroup rather than the default scheduler so that tests sharing state on + # disk can be pinned to a single worker with @pytest.mark.xdist_group. + PYTEST_ARGS += -n $(PYTEST_WORKERS) --dist loadgroup +endif + help: ## Show this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) @@ -32,7 +43,7 @@ install: ## Sync dependencies and set up local development tools build: ## Build the local Rust extension with maturin $(UV_RUN) maturin develop --uv -test: ## Run Python tests except recurring tests +test: ## Run Python tests pytest $(PYTEST_ARGS) python/tests integtest: ## Start LocalStack and run integration tests diff --git a/python/pyproject.toml b/python/pyproject.toml index dd11344324c..f6972e03af5 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pylance" dynamic = ["version"] -dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.8.5,<0.9"] +dependencies = ["pyarrow>=14", "numpy>=1.22", "lance-namespace>=0.11.1,<0.12"] description = "python wrapper for Lance columnar format" authors = [{ name = "Lance Devs", email = "dev@lance.org" }] license = { file = "LICENSE" } @@ -49,21 +49,21 @@ build-backend = "maturin" [project.optional-dependencies] tests = [ "boto3", - "datasets", - "duckdb", + "datasets==4.4.0", + "duckdb>=1.5.0,<1.6.0", "ml_dtypes", "pillow", "pandas", "polars[pyarrow,pandas]", "psutil", "pytest", - # Only test tensorflow on linux for now. We will deprecate tensorflow soon. - "tensorflow; sys_platform == 'linux'", + "pytest-xdist", "tqdm", - "datafusion>=53,<54", + "datafusion>=54,<55", ] dev = ["ruff==0.11.2", "pyright"] benchmarks = ["pytest-benchmark"] +otel = ["opentelemetry-api", "opentelemetry-sdk"] torch = ["torch>=2.0"] geo = [ "geoarrow-rust-core", @@ -73,17 +73,18 @@ geo = [ [dependency-groups] tests = [ "boto3==1.40.43", - "datasets==4.1.1", - "duckdb==1.4.0", + "datasets==4.4.0", + "duckdb>=1.5.0,<1.6.0", "ml_dtypes==0.5.3", "pillow==11.3.0", "pandas==2.3.3", "polars[pyarrow,pandas]==1.34.0", "psutil==7.1.0", "pytest==8.4.2", - "tensorflow==2.20.0; sys_platform == 'linux'", + "pytest-xdist==3.8.0", "tqdm==4.67.1", - "datafusion==53.0.0", + "datafusion==54.0.0", + "opentelemetry-sdk==1.30.0", ] dev = [ "maturin==1.13.3", @@ -104,7 +105,7 @@ lint.select = ["F", "E", "W", "I", "G", "TCH", "PERF", "B019"] "*.pyi" = ["E301", "E302"] [tool.pyright] -pythonVersion = "3.13" +pythonVersion = "3.14" # TODO: expand this list as we fix more files. include = [ "python/lance/util.py", @@ -112,10 +113,13 @@ include = [ "python/lance/tracing.py", "python/lance/dependencies.py", "python/lance/schema.py", + "python/tests/test_schema.py", "python/lance/file.py", "python/lance/util.py", "python/lance/arrow.py", "python/tests/test_arrow.py", + "python/tests/test_fragment_typing.py", + "python/tests/test_udf.py", ] # Dependencies like pyarrow make this difficult to enforce strictly. reportMissingTypeStubs = "warning" @@ -132,14 +136,10 @@ markers = [ "gpu: tests which rely on pytorch and some kind of gpu", "slow", "torch: tests which rely on pytorch being installed", - "recurring: marks tests as recurring tests", ] filterwarnings = [ 'error::FutureWarning', 'error::DeprecationWarning', - # TensorFlow import can emit NumPy deprecation FutureWarnings in some environments. - # We keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning. - 'ignore:.*`np\\.object` will be defined as the corresponding NumPy scalar\\..*:FutureWarning', # Boto3 'ignore:.*datetime\.datetime\.utcnow\(\) is deprecated.*:DeprecationWarning', # Hugging Face Hub calls this deprecated hf-xet API internally. @@ -153,9 +153,8 @@ filterwarnings = [ 'ignore:.*the load_module\(\) method is deprecated.*:DeprecationWarning', # Pytorch uses deprecated jit.script_method internally (torch/utils/mkldnn.py) 'ignore:.*torch\.jit\.script_method.*is deprecated.*:DeprecationWarning', + # Pytorch uses the same API internally on Python 3.14+ with a different warning message. + 'ignore:.*torch\.jit\.script_method.*is not supported in Python 3\.14\+.*:DeprecationWarning', # huggingface_hub still calls the deprecated hf_xet.download_files() during Xet downloads 'ignore:.*hf_xet\.download_files\(\) is deprecated.*:DeprecationWarning', - # TensorFlow/Keras import can emit NumPy deprecation FutureWarnings in some environments. - # Keep FutureWarnings as errors generally, but ignore this known-noisy import-time warning. - 'ignore:.*np\.object.*:FutureWarning', ] diff --git a/python/python/benchmarks/test_blob.py b/python/python/benchmarks/test_blob.py new file mode 100644 index 00000000000..c2465f36fad --- /dev/null +++ b/python/python/benchmarks/test_blob.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +from itertools import count +from time import process_time + +import pyarrow as pa +import pytest +from lance.file import LanceFileSession + +# Many small blobs isolate per-row Python overhead; fewer large blobs show how +# the bulk path behaves once payload copying accounts for more of the CPU time. +WORKLOADS = [ + pytest.param(50_000, 256, id="50000x256b"), + pytest.param(2_000, 64 * 1024, id="2000x64kib"), +] + + +def _packed_blob_benchmark(benchmark, tmpdir_factory, row_count, payload_size, mode): + payload = b"x" * payload_size + payloads = pa.repeat(payload, row_count) + files = LanceFileSession(str(tmpdir_factory.mktemp("packed_blob_writer"))) + file_number = count() + + if mode == "scalar_preconverted": + python_payloads = payloads.to_pylist() + + def write(): + writer = files.open_packed_blob_writer( + f"scalar-{next(file_number)}.lance", 1 + ) + for value in python_payloads: + writer.write_blob(value) + return writer.finish() + + elif mode == "scalar_from_arrow": + + def write(): + writer = files.open_packed_blob_writer( + f"scalar-arrow-{next(file_number)}.lance", 1 + ) + for value in payloads.to_pylist(): + writer.write_blob(value) + return writer.finish() + + elif mode == "bulk": + + def write(): + writer = files.open_packed_blob_writer(f"bulk-{next(file_number)}.lance", 1) + writer.write_blobs(payloads) + return writer.finish_array("blob") + + else: + raise ValueError(f"Unknown benchmark mode: {mode}") + + result = benchmark.pedantic(write, iterations=1, rounds=5) + assert len(result) == row_count + + +@pytest.mark.benchmark(group="packed_blob_writer_cpu", timer=process_time) +@pytest.mark.parametrize("row_count,payload_size", WORKLOADS) +@pytest.mark.parametrize( + "mode", + ["scalar_preconverted", "scalar_from_arrow", "bulk"], +) +def test_packed_blob_writer(benchmark, tmpdir_factory, row_count, payload_size, mode): + _packed_blob_benchmark( + benchmark, + tmpdir_factory, + row_count, + payload_size, + mode, + ) diff --git a/python/python/benchmarks/test_scan.py b/python/python/benchmarks/test_scan.py index c813ab5653d..c43e1703ac8 100644 --- a/python/python/benchmarks/test_scan.py +++ b/python/python/benchmarks/test_scan.py @@ -9,6 +9,7 @@ import pytest NUM_ROWS = 10_000 +READER_BRIDGE_ROWS = 100_000 @pytest.mark.parametrize( @@ -76,6 +77,28 @@ def sample_dataset(tmpdir_factory): return lance.write_dataset(table, tmp_path) +@pytest.fixture(scope="module") +def reader_bridge_dataset(tmpdir_factory): + tmp_path = Path(tmpdir_factory.mktemp("reader_bridge")) + table = pa.table({"value": pa.array(range(READER_BRIDGE_ROWS), type=pa.int32())}) + return lance.write_dataset(table, tmp_path) + + +@pytest.mark.parametrize("batch_size", [64, 1024, 8192, 65536]) +@pytest.mark.parametrize( + "columns", + [pytest.param([], id="zero_columns"), pytest.param(["value"], id="i32")], +) +@pytest.mark.benchmark(group="scan_reader_bridge") +def test_scan_reader_bridge(benchmark, reader_bridge_dataset, columns, batch_size): + scanner = reader_bridge_dataset.scanner(columns=columns, batch_size=batch_size) + + def consume_reader(): + return sum(batch.num_rows for batch in scanner.to_reader()) + + assert benchmark(consume_reader) == READER_BRIDGE_ROWS + + @pytest.mark.benchmark(group="scan_table") def test_scan_table_full(benchmark, sample_dataset): result = benchmark( diff --git a/python/python/benchmarks/test_search.py b/python/python/benchmarks/test_search.py index b4e33338cb1..b86a2818e97 100644 --- a/python/python/benchmarks/test_search.py +++ b/python/python/benchmarks/test_search.py @@ -210,6 +210,49 @@ def test_ann_with_refine(test_dataset, benchmark): assert result.num_rows > 0 +N_BATCH_QUERIES = 32 + + +@pytest.mark.benchmark(group="query_ann_batch") +def test_batch_ann_search(test_dataset, benchmark): + # One request carrying all query vectors: the index shares each partition's + # scan across the batch (issue #6822). + queries = np.random.randn(N_BATCH_QUERIES, N_DIMS).astype(np.float32) + result = benchmark( + test_dataset.to_table, + columns=[], + with_row_id=True, + nearest=dict( + column="vector", + q=queries, + k=100, + nprobes=10, + ), + ) + assert result.num_rows > 0 + + +@pytest.mark.benchmark(group="query_ann_batch") +def test_repeated_single_ann_search(test_dataset, benchmark): + # Baseline: the same query vectors issued one indexed search at a time. + queries = np.random.randn(N_BATCH_QUERIES, N_DIMS).astype(np.float32) + + def run(): + for q in queries: + test_dataset.to_table( + columns=[], + with_row_id=True, + nearest=dict( + column="vector", + q=q, + k=100, + nprobes=10, + ), + ) + + benchmark(run) + + @pytest.mark.benchmark(group="query_ann") @pytest.mark.parametrize("selectivity", (0.25, 0.75)) @pytest.mark.parametrize("prefilter", (False, True)) diff --git a/python/python/benchmarks/zone_map_seeds.py b/python/python/benchmarks/zone_map_seeds.py new file mode 100644 index 00000000000..023e14cfcf7 --- /dev/null +++ b/python/python/benchmarks/zone_map_seeds.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors +""" +Benchmark: Zone Map Index Seeds vs. No Seeds + +Measures the time to update a zone map index when new data is appended to an +already-indexed dataset, with and without use_seeds. + +Seeds allow Lance to embed per-zone min/max statistics in data files at write +time. When optimize_indices() runs, it harvests those pre-computed stats +instead of re-scanning the full column, which eliminates the read I/O for +wide data types. + +WHY the workflow matters +------------------------ +Seeds only engage when merging new unindexed fragments into an existing +non-empty index segment. A first optimize_indices() call on a freshly +created empty-index always does a full rebuild (no old fragments to merge +against), so seeds would never be used. + +Correct workflow: + 1. Write an initial batch of data and build the zone map index on it. + This creates a non-empty indexed segment; all subsequent writes will + embed seed buffers in their data files (when use_seeds=True). + 2. Append the bulk of the data in multiple batches. + 3. optimize_indices() — harvests seeds (phase 2 data) and merges them into + the segment from phase 1. Without seeds, phase 3 must re-read all of + the phase-2 data from disk. + +Phases timed: + initial_write write seed_fraction of total rows + create/build index + ingest append remaining rows (seeds collected here when enabled) + update_index optimize_indices() — seed harvest happens here + total sum of the three phases above + +Three scenarios: + int64 10 M rows ~80 MB on disk + FixedSizeBinary(4096) 10 M rows ~38 GB on disk + LargeBinary(20 KiB) 1 M rows ~19 GB on disk + +Usage: + uv run python python/benchmarks/zone_map_seeds.py + uv run python python/benchmarks/zone_map_seeds.py --tmpdir /fast/nvme/bench + uv run python python/benchmarks/zone_map_seeds.py --scale 0.01 +""" + +import argparse +import os +import shutil +import tempfile +import time +from pathlib import Path + +import lance +import numpy as np +import pyarrow as pa +from lance.indices import IndexConfig + +KiB = 1024 + +# Fraction of total rows written in the initial indexed batch. +SEED_FRACTION = 0.1 + + +# --------------------------------------------------------------------------- +# Data generators (yield pa.Table one batch at a time) +# --------------------------------------------------------------------------- + + +def gen_int_batches(num_rows: int, batch_size: int): + rng = np.random.default_rng() + for start in range(0, num_rows, batch_size): + n = min(batch_size, num_rows - start) + yield pa.table({"value": pa.array(rng.integers(0, 2**31, n, dtype=np.int64))}) + + +def gen_fsb_batches(num_rows: int, blob_bytes: int, batch_size: int): + for start in range(0, num_rows, batch_size): + n = min(batch_size, num_rows - start) + raw = os.urandom(n * blob_bytes) + arr = pa.FixedSizeBinaryArray.from_buffers( + pa.binary(blob_bytes), n, [None, pa.py_buffer(raw)] + ) + yield pa.table({"value": arr}) + + +def gen_large_binary_batches(num_rows: int, blob_bytes: int, batch_size: int): + for start in range(0, num_rows, batch_size): + n = min(batch_size, num_rows - start) + yield pa.table( + { + "value": pa.array( + [os.urandom(blob_bytes) for _ in range(n)], + type=pa.large_binary(), + ) + } + ) + + +# --------------------------------------------------------------------------- +# Core benchmark runner +# --------------------------------------------------------------------------- + + +def run_scenario( + dataset_path: Path, + schema: pa.Schema, + seed_rows: int, + seed_batch_size: int, + gen_seed_batches, # () -> Iterator[pa.Table] for the initial write + gen_bulk_batches, # () -> Iterator[pa.Table] for the appended data + use_seeds: bool, +) -> dict: + """ + Run one full scenario and return per-phase timings. + + Seeds are only engaged by optimize_indices() when there is at least one + previously indexed fragment. This scenario ensures that by writing + seed_rows of data and building the index before any bulk ingest. + """ + if dataset_path.exists(): + shutil.rmtree(dataset_path) + + index_config = IndexConfig( + index_type="ZONEMAP", parameters={"use_seeds": use_seeds} + ) + timings = {} + + # Phase 1: initial write + index build + t0 = time.perf_counter() + ds = None + for batch in gen_seed_batches(): + if ds is None: + ds = lance.write_dataset(batch, dataset_path, schema=schema) + else: + ds = lance.write_dataset(batch, dataset_path, mode="append") + ds.create_scalar_index("value", index_type=index_config, replace=True) + timings["initial_write"] = time.perf_counter() - t0 + + # Phase 2: bulk ingest — seeds are collected per-batch when enabled + t0 = time.perf_counter() + for batch in gen_bulk_batches(): + ds = lance.write_dataset(batch, dataset_path, mode="append") + timings["ingest"] = time.perf_counter() - t0 + + # Phase 3: index update — seed harvest happens here + t0 = time.perf_counter() + ds.optimize.optimize_indices() + timings["update_index"] = time.perf_counter() - t0 + + timings["total"] = sum(timings.values()) + shutil.rmtree(dataset_path) + return timings + + +# --------------------------------------------------------------------------- +# Output formatting +# --------------------------------------------------------------------------- + + +def print_comparison(title: str, no_seeds: dict, with_seeds: dict) -> None: + phases = ["initial_write", "ingest", "update_index", "total"] + col_w = 13 + + print(f"\n{'=' * 70}") + print(f" {title}") + print(f"{'=' * 70}") + print( + f" {'Phase':<20} {'No Seeds':>{col_w}} " + + f"{'With Seeds':>{col_w}} {'Speedup':>{col_w}}" + ) + print(f" {'-' * (20 + col_w * 3 + 6)}") + for phase in phases: + t_no = no_seeds[phase] + t_yes = with_seeds[phase] + speedup = t_no / t_yes if t_yes > 0 else float("inf") + marker = "* " if phase == "total" else " " + print( + f"{marker}{phase:<20} {t_no:>{col_w}.2f}s" + f" {t_yes:>{col_w}.2f}s" + f" {speedup:>{col_w - 1}.2f}x" + ) + + +# --------------------------------------------------------------------------- +# Per-type benchmark wrappers +# --------------------------------------------------------------------------- + + +def bench_integers(base: Path, num_rows: int) -> None: + seed_rows = max(1, int(num_rows * SEED_FRACTION)) + bulk_rows = num_rows - seed_rows + # ~400 KB per seed batch; ~4 MB per bulk batch → ~20 bulk fragments + seed_bs = max(1, seed_rows // 5) + bulk_bs = max(1, bulk_rows // 20) + print( + f"\nInteger (int64): {num_rows:,} rows " + f"(initial={seed_rows:,}, bulk={bulk_rows:,}) ..." + ) + schema = pa.schema([pa.field("value", pa.int64())]) + + no_seeds = run_scenario( + base / "int_no_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_int_batches(seed_rows, seed_bs), + lambda: gen_int_batches(bulk_rows, bulk_bs), + use_seeds=False, + ) + with_seeds = run_scenario( + base / "int_with_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_int_batches(seed_rows, seed_bs), + lambda: gen_int_batches(bulk_rows, bulk_bs), + use_seeds=True, + ) + print_comparison(f"int64 — {num_rows:,} rows", no_seeds, with_seeds) + + +def bench_vectors(base: Path, num_rows: int) -> None: + blob_bytes = 4 * KiB + seed_rows = max(1, int(num_rows * SEED_FRACTION)) + bulk_rows = num_rows - seed_rows + total_gb = num_rows * blob_bytes / (1024**3) + # ~200 MB per batch to stay memory-friendly + seed_bs = max(1, (200 * KiB * KiB) // blob_bytes) + bulk_bs = seed_bs + print( + f"\nVector (FixedSizeBinary {blob_bytes // KiB} KiB): {num_rows:,} rows" + f" (~{total_gb:.1f} GB) (initial={seed_rows:,}, bulk={bulk_rows:,}) ..." + ) + schema = pa.schema([pa.field("value", pa.binary(blob_bytes))]) + + no_seeds = run_scenario( + base / "vec_no_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_fsb_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_fsb_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=False, + ) + with_seeds = run_scenario( + base / "vec_with_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_fsb_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_fsb_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=True, + ) + print_comparison( + f"FixedSizeBinary({blob_bytes // KiB} KiB) — {num_rows:,} rows", + no_seeds, + with_seeds, + ) + + +def bench_large_binary(base: Path, num_rows: int) -> None: + blob_bytes = 20 * KiB + seed_rows = max(1, int(num_rows * SEED_FRACTION)) + bulk_rows = num_rows - seed_rows + total_gb = num_rows * blob_bytes / (1024**3) + # ~200 MB per batch + seed_bs = max(1, (200 * KiB * KiB) // blob_bytes) + bulk_bs = seed_bs + print( + f"\nLargeBinary ({blob_bytes // KiB} KiB blobs): {num_rows:,} rows" + f" (~{total_gb:.1f} GB) (initial={seed_rows:,}, bulk={bulk_rows:,}) ..." + ) + schema = pa.schema([pa.field("value", pa.large_binary())]) + + no_seeds = run_scenario( + base / "bin_no_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_large_binary_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_large_binary_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=False, + ) + with_seeds = run_scenario( + base / "bin_with_seeds", + schema, + seed_rows, + seed_bs, + lambda: gen_large_binary_batches(seed_rows, blob_bytes, seed_bs), + lambda: gen_large_binary_batches(bulk_rows, blob_bytes, bulk_bs), + use_seeds=True, + ) + print_comparison( + f"LargeBinary({blob_bytes // KiB} KiB) — {num_rows:,} rows", + no_seeds, + with_seeds, + ) + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--tmpdir", + type=Path, + default=None, + help="Directory for benchmark datasets (default: system temp dir)", + ) + parser.add_argument( + "--scale", + type=float, + default=1.0, + help="Row-count scale factor (default 1.0; use 0.01 for a quick smoke test)", + ) + args = parser.parse_args() + + int_rows = max(2, int(10_000_000 * args.scale)) + vec_rows = max(2, int(10_000_000 * args.scale)) + bin_rows = max(2, int(1_000_000 * args.scale)) + + print( + f"Scale: {args.scale} → " + f"{int_rows:,} int / {vec_rows:,} vector / {bin_rows:,} binary rows" + ) + + def run(tmp_dir: Path) -> None: + bench_integers(tmp_dir, int_rows) + bench_vectors(tmp_dir, vec_rows) + bench_large_binary(tmp_dir, bin_rows) + + if args.tmpdir is not None: + args.tmpdir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix="lance_zonemap_bench_", dir=args.tmpdir + ) as tmp: + run(Path(tmp)) + else: + with tempfile.TemporaryDirectory(prefix="lance_zonemap_bench_") as tmp: + run(Path(tmp)) + + +if __name__ == "__main__": + main() diff --git a/python/python/ci_benchmarks/benchmark.py b/python/python/ci_benchmarks/benchmark.py index 7d80596e305..8d17bef5ae5 100644 --- a/python/python/ci_benchmarks/benchmark.py +++ b/python/python/ci_benchmarks/benchmark.py @@ -20,7 +20,7 @@ def workload(dataset): import json from dataclasses import dataclass -from typing import Any, Callable, List +from typing import Any, Callable, List, Optional import pytest @@ -90,6 +90,7 @@ def __call__( func: Callable, dataset: Any, warmup: bool = True, + setup: Optional[Callable[[], Any]] = None, ) -> Any: """ Run a benchmark function with IO and memory tracking. @@ -102,28 +103,47 @@ def __call__( The dataset to pass to the function. warmup : bool, default True Whether to run a warmup iteration before measuring. + setup : Callable, optional + Called before the warmup run and again before the measured run. + Required for workloads that mutate the dataset, so each run starts + from the same state. Setup runs before IO stats are reset, so its + own IO is never attributed to the benchmark. If it returns a value, + that value is passed to ``func`` instead of ``dataset`` -- use this + when the reset produces a new dataset handle. Returns ------- Any The return value of the benchmark function. """ + + def prepare() -> Any: + if setup is None: + return dataset + replacement = setup() + return dataset if replacement is None else replacement + # Warmup run (not measured) if warmup: - func(dataset) + func(prepare()) + + target = prepare() - # Reset IO stats before the measured run + # Reset IO stats before the measured run. Stats are tracked on the + # ObjectStore, which is shared by every handle from the same session, + # so resetting/reading through `dataset` also captures work that `func` + # performs through a handle returned by `setup`. dataset.io_stats_incremental() # Run with memory tracking if available if MEMTEST_AVAILABLE: memtest.reset_stats() - result = func(dataset) + result = func(target) mem_stats = memtest.get_stats() self._stats.peak_bytes = mem_stats["peak_bytes"] self._stats.total_allocations = mem_stats["total_allocations"] else: - result = func(dataset) + result = func(target) # Capture IO stats io_stats = dataset.io_stats_incremental() @@ -159,7 +179,11 @@ def workload(dataset): if marker is None: # Not an io_memory_benchmark test, return a simple passthrough class PassthroughBenchmark: - def __call__(self, func, dataset, warmup=True): + def __call__(self, func, dataset, warmup=True, setup=None): + if setup is not None: + replacement = setup() + if replacement is not None: + dataset = replacement return func(dataset) yield PassthroughBenchmark() diff --git a/python/python/ci_benchmarks/benchmarks/test_merge_insert.py b/python/python/ci_benchmarks/benchmarks/test_merge_insert.py new file mode 100644 index 00000000000..49fcd693eb6 --- /dev/null +++ b/python/python/ci_benchmarks/benchmarks/test_merge_insert.py @@ -0,0 +1,805 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Benchmarks for `merge_insert`. + +merge_insert has two execution paths and the routing between them is +structural, not a flag. `use_index` is the one knob that selects between them, +so every benchmark here is parametrized on it: + + ``v1_indexed`` — ``use_index(True)`` with an indexed key. Takes the legacy + indexed-scan path (``create_indexed_scan_joined_stream``). + ``v2_hash`` — ``use_index(False)``. Disables the index gate in + ``can_use_create_plan``, so the DataFusion path + (``LanceRead + HashJoin``) runs instead. + +For a partial-schema source the write sink is a second, independent choice, made +with ``write_mode``. Under the default ``"auto"`` v1 patches columns in place +(``UpdateMode::RewriteColumns``) while v2 rewrites whole rows +(``RewriteRows``); the ``v2_rewrite_columns`` variants of +``test_update_subset_*`` ask v2 for the patching sink instead. That makes +``write_bytes`` the interesting metric for the ``test_update_*`` benchmarks: +which sink writes less depends on how wide the omitted columns are and how many +rows are matched, so these sweeps are where the crossover shows up. + +Targets are mutated, so each measured run is preceded by an untimed restore to +the ``merge_insert_base`` tag written by ``datagen/merge_insert.py``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Callable, Iterable, Optional, Sequence + +import lance +import numpy as np +import pyarrow as pa +import pytest +from ci_benchmarks.datagen.merge_insert import ( + BASE_TAG, + DELETED_ROW_STRIDE, + FRAGS_NUM_ROWS, + FRAGS_SCHEMA, + NARROW_NUM_ROWS, + NARROW_SCHEMA, + UNINDEXED_TAIL_ROWS, + WIDE_NUM_ROWS, + WIDE_ROWS_PER_FRAGMENT, + WIDE_SCALAR_COLUMNS, + WIDE_SCHEMA, + WIDE_VECTOR_DIM, + narrow_batch, +) +from ci_benchmarks.datasets import get_dataset_uri + +if TYPE_CHECKING: + from lance.dataset import ExecuteResult + +PLANS = ["v1_indexed", "v2_hash"] + +# Partial-column updates have a third shape: v2 asked for the column-patching +# sink, which is the only v2 sink that does not rewrite whole rows. +WRITE_PLANS = ["v1_indexed", "v2_hash", "v2_rewrite_columns"] + +# Brackets the cold-random break-even, which the design analysis puts at +# roughly target_rows / 4096 -- about 2.4K rows for the 10M-row narrow target. +SOURCE_SIZES = [1_000, 10_000, 100_000] + +NARROW_KEYS = ["id_int", "id_uuid7", "id_uuid4"] + +# DataFusion accounts each 8,192-row slice of the 1.32 GiB full-schema source +# for its shared backing buffers. Budget the resulting ~159 GiB of reservations +# without changing the source layout measured by this benchmark. +_MEM_POOL_SIZE = 160 * 1024**3 + + +@pytest.fixture(scope="module", autouse=True) +def _merge_insert_mem_pool() -> Iterable[None]: + """Use a bounded pool large enough for every merge_insert benchmark.""" + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setenv("LANCE_MEM_POOL_SIZE", str(_MEM_POOL_SIZE)) + yield + + +# --------------------------------------------------------------------------- +# Target management +# --------------------------------------------------------------------------- + + +@dataclass +class Target: + """A merge_insert target that can be rewound to a pristine state. + + ``reset`` returns a handle sitting on a fresh version whose contents match + the ``merge_insert_base`` tag. It is called from an untimed benchmark + setup hook, so the restore never lands in the measurement. + """ + + uri: str + dataset: lance.LanceDataset + base_version: int + + def reset(self, cold: bool = False) -> lance.LanceDataset: + if cold: + # A new session means an empty index cache. `checkout_version` + # deliberately shares the cache, so a cold run cannot reuse + # `self.dataset`. + handle = lance.dataset(self.uri) + else: + handle = self.dataset + reverted = handle.checkout_version(self.base_version) + reverted.restore() + return reverted + + +def _open_target(name: str) -> Iterable[Target]: + uri = get_dataset_uri(name) + dataset = lance.dataset(uri) + tags = dataset.tags.list() + if BASE_TAG not in tags: + pytest.skip( + f"Dataset {name} has no {BASE_TAG} tag; " + "run python/ci_benchmarks/datagen/gen_all.py" + ) + base_version = tags[BASE_TAG]["version"] + + yield Target(uri=uri, dataset=dataset, base_version=base_version) + + # Leave the dataset pristine and drop the versions and data files the + # benchmarks produced. Cleanup never deletes a tagged version, so the tag + # has to move onto the restored version first or the old base would be + # retained forever. + final = dataset.checkout_version(base_version) + final.restore() + final.tags.update(BASE_TAG, final.version) + final.cleanup_old_versions(older_than=timedelta(0), delete_unverified=True) + + +@pytest.fixture(scope="module") +def narrow() -> Iterable[Target]: + yield from _open_target("merge_insert_narrow") + + +@pytest.fixture(scope="module") +def wide() -> Iterable[Target]: + yield from _open_target("merge_insert_wide") + + +@pytest.fixture(scope="module") +def frags() -> Iterable[Target]: + yield from _open_target("merge_insert_frags") + + +@pytest.fixture(scope="module") +def deleted() -> Iterable[Target]: + yield from _open_target("merge_insert_deleted") + + +@pytest.fixture(scope="module") +def unindexed_tail() -> Iterable[Target]: + yield from _open_target("merge_insert_unindexed_tail") + + +# --------------------------------------------------------------------------- +# Source construction +# --------------------------------------------------------------------------- +# +# Sources are built from a contiguous run of row indices. Which key column the +# merge joins on then decides whether those keys are clustered or scattered in +# the index's key order: `id_int` and `id_uuid7` are monotonic in the row index, +# `id_uuid4` scrambles it. This keeps one source builder for all key shapes. + + +def narrow_source(row_indices: np.ndarray) -> pa.Table: + """A full-schema narrow source. ``value`` is offset so updates are real.""" + return pa.Table.from_batches([narrow_batch(row_indices, value_offset=1)]) + + +def existing_rows(num_rows: int, offset: int = 0) -> np.ndarray: + return np.arange(offset, offset + num_rows, dtype=np.int64) + + +def new_rows(num_rows: int) -> np.ndarray: + """Row indices past the end of the target, so every key is a new key.""" + return np.arange(NARROW_NUM_ROWS, NARROW_NUM_ROWS + num_rows, dtype=np.int64) + + +def wide_row_indices(fraction: float) -> np.ndarray: + """Row indices covering ``fraction`` of every fragment's rows. + + Spread evenly within each fragment rather than contiguously: that is the + harder case for the in-place column updater, which has to interleave + updated and untouched values. + """ + per_fragment = max(1, round(fraction * WIDE_ROWS_PER_FRAGMENT)) + num_fragments = WIDE_NUM_ROWS // WIDE_ROWS_PER_FRAGMENT + return np.concatenate( + [ + fragment * WIDE_ROWS_PER_FRAGMENT + + np.linspace(0, WIDE_ROWS_PER_FRAGMENT - 1, per_fragment, dtype=np.int64) + for fragment in range(num_fragments) + ] + ) + + +def wide_source(row_indices: np.ndarray, columns: Sequence[str]) -> pa.Table: + """A wide source carrying ``id_int`` plus ``columns``.""" + arrays = [pa.array(row_indices)] + names = ["id_int"] + for column in columns: + names.append(column) + if column == "vec": + values = np.linspace( + 1.0, + 2.0, + num=len(row_indices) * WIDE_VECTOR_DIM, + dtype=np.float32, + ) + arrays.append( + pa.FixedSizeListArray.from_arrays(pa.array(values), WIDE_VECTOR_DIM) + ) + elif WIDE_SCHEMA.field(column).type == pa.string(): + arrays.append( + pa.array([f"updated_{column}_{v}" for v in row_indices], pa.string()) + ) + else: + arrays.append(pa.array(row_indices + 1)) + return pa.table(arrays, schema=pa.schema([WIDE_SCHEMA.field(n) for n in names])) + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def run( + benchmark, + target: Target, + job: Callable[[lance.LanceDataset], ExecuteResult], + *, + rounds: int = 3, + cold: bool = False, + warmup: bool = True, + expected_rows: Optional[int] = None, +) -> None: + """Benchmark ``job``, restoring the target before every round. + + ``cold`` opens a fresh session for each round so the index cache starts + empty. ``warmup`` runs one unmeasured round first; turn it off for + expensive shapes where a second pass is not worth the wall clock. + + ``expected_rows`` guards against a benchmark that silently stops doing + work: a semantics regression should fail the run, not post a fast time. + Row count is checked instead of the returned stats because it is + independent of which execution path ran. + """ + state: dict = {} + + def setup() -> None: + state["dataset"] = target.reset(cold=cold) + + def bench() -> None: + dataset = state["dataset"] + state["stats"] = job(dataset) + if expected_rows is not None: + state["row_count"] = dataset.count_rows() + + benchmark.pedantic( + bench, + setup=setup, + rounds=rounds, + iterations=1, + # `setup` runs before each warmup round too, so a warmup never leaves + # the target dirty for the measured rounds. + warmup_rounds=1 if warmup and not cold else 0, + ) + + if expected_rows is not None: + assert state["row_count"] == expected_rows, ( + f"expected {expected_rows} rows after merge_insert, " + f"got {state['row_count']} (stats: {state['stats']})" + ) + + +def upsert( + key: str, source: pa.Table, *, use_index: bool +) -> Callable[[lance.LanceDataset], ExecuteResult]: + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert(key) + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(use_index) + .execute(source) + ) + + return job + + +def uses_index(plan: str) -> bool: + return plan == "v1_indexed" + + +def write_mode_for(plan: str) -> str: + return "rewrite_columns" if plan == "v2_rewrite_columns" else "auto" + + +# --------------------------------------------------------------------------- +# A. Cost model core -- merge_insert_narrow, 10M rows +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_point(benchmark, narrow: Target, plan: str) -> None: + """Single-row upsert latency -- the extreme where a probe should win.""" + source = narrow_source(existing_rows(1)) + run( + benchmark, + narrow, + upsert("id_int", source, use_index=uses_index(plan)), + rounds=5, + expected_rows=NARROW_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("key", NARROW_KEYS) +@pytest.mark.parametrize("num_rows", SOURCE_SIZES) +def test_upsert_ratio_sweep( + benchmark, narrow: Target, num_rows: int, key: str, plan: str +) -> None: + """Source/target ratio sweep against a warm index.""" + source = narrow_source(existing_rows(num_rows)) + run( + benchmark, + narrow, + upsert(key, source, use_index=uses_index(plan)), + expected_rows=NARROW_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("key", ["id_int", "id_uuid4"]) +@pytest.mark.parametrize("num_rows", [1_000, 100_000]) +def test_upsert_cold_ratio_sweep( + benchmark, narrow: Target, num_rows: int, key: str, plan: str +) -> None: + """Same sweep with a cold index cache, where page reads are not free.""" + source = narrow_source(existing_rows(num_rows)) + run( + benchmark, + narrow, + upsert(key, source, use_index=uses_index(plan)), + cold=True, + expected_rows=NARROW_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("num_rows", [10_000, 100_000]) +def test_upsert_all_new_keys( + benchmark, narrow: Target, num_rows: int, plan: str +) -> None: + """Time-ordered ingest: every key is new, so no index page should be read.""" + source = narrow_source(new_rows(num_rows)) + run( + benchmark, + narrow, + upsert("id_uuid7", source, use_index=uses_index(plan)), + expected_rows=NARROW_NUM_ROWS + num_rows, + ) + + +@pytest.mark.parametrize("num_rows", [1_000, 100_000]) +def test_upsert_unindexed_baseline(benchmark, narrow: Target, num_rows: int) -> None: + """Joining on a column with no index at all -- the no-index reference.""" + source = narrow_source(existing_rows(num_rows)) + run( + benchmark, + narrow, + upsert("id_no_index", source, use_index=True), + expected_rows=NARROW_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# B. Write path -- merge_insert_wide, 1M rows +# --------------------------------------------------------------------------- + +# Fractions are of each fragment's rows, which is what decides whether the +# in-place updater interleaves or rewrites a whole column file. +ROW_FRACTIONS = [0.001, 0.01, 0.1, 1.0] +FRACTION_IDS = ["0.1pct", "1pct", "10pct", "100pct"] + +# Isolates the two pressures a partial-column update can exert: field count +# (scalar columns, cheap per field) and byte volume (the vector column). +PROJECTIONS = { + "one_scalar": WIDE_SCALAR_COLUMNS[:1], + "ten_scalars": WIDE_SCALAR_COLUMNS[:10], + "vector": ["vec"], + "vector_and_ten_scalars": ["vec"] + WIDE_SCALAR_COLUMNS[:10], +} + + +def _wide_rounds(fraction: float) -> int: + # A full-fragment update rewrites every column file it touches, which for + # the vector column is ~1 GB per round. + return 1 if fraction == 1.0 else 3 + + +def update_subset( + source: pa.Table, *, use_index: bool, write_mode: str = "auto" +) -> Callable[[lance.LanceDataset], ExecuteResult]: + """Partial-schema update. No insert clause: matched rows only.""" + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .use_index(use_index) + .write_mode(write_mode) + .execute(source) + ) + + return job + + +@pytest.mark.parametrize("plan", WRITE_PLANS) +@pytest.mark.parametrize("fraction", ROW_FRACTIONS, ids=FRACTION_IDS) +def test_update_subset_row_fraction( + benchmark, wide: Target, fraction: float, plan: str +) -> None: + """Row-fraction sweep at a fixed, minimal projection. + + Locates the crossover between patching a column in place and rewriting the + whole column file, without byte volume confounding it. + """ + source = wide_source(wide_row_indices(fraction), PROJECTIONS["one_scalar"]) + run( + benchmark, + wide, + update_subset( + source, + use_index=uses_index(plan), + write_mode=write_mode_for(plan), + ), + rounds=_wide_rounds(fraction), + warmup=fraction != 1.0, + expected_rows=WIDE_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", WRITE_PLANS) +@pytest.mark.parametrize("fraction", [0.01, 1.0], ids=["1pct", "100pct"]) +@pytest.mark.parametrize("projection", list(PROJECTIONS), ids=list(PROJECTIONS)) +def test_update_subset_projection( + benchmark, wide: Target, projection: str, fraction: float, plan: str +) -> None: + """Projection sweep: field count vs byte volume, plus the cross term.""" + source = wide_source(wide_row_indices(fraction), PROJECTIONS[projection]) + run( + benchmark, + wide, + update_subset( + source, + use_index=uses_index(plan), + write_mode=write_mode_for(plan), + ), + rounds=_wide_rounds(fraction), + warmup=fraction != 1.0, + expected_rows=WIDE_NUM_ROWS, + ) + + +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("fraction", [0.01, 1.0], ids=["1pct", "100pct"]) +def test_upsert_wide_full_schema( + benchmark, wide: Target, fraction: float, plan: str +) -> None: + """Full-schema baseline for the partial-column benchmarks above.""" + row_indices = wide_row_indices(fraction) + source = wide_source( + row_indices, [f.name for f in WIDE_SCHEMA if f.name != "id_int"] + ) + run( + benchmark, + wide, + upsert("id_int", source, use_index=uses_index(plan)), + rounds=_wide_rounds(fraction), + warmup=fraction != 1.0, + expected_rows=WIDE_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# C. Clause shapes -- merge_insert_narrow, 10K-row source +# --------------------------------------------------------------------------- + +CLAUSE_SOURCE_ROWS = 10_000 + + +@pytest.mark.parametrize("plan", PLANS) +def test_insert_if_not_exists(benchmark, narrow: Target, plan: str) -> None: + """Dedup ingest: half the keys already exist, matched rows are untouched. + + The probe only needs to know whether a key exists, so no target payload has + to be read. + """ + half = CLAUSE_SOURCE_ROWS // 2 + row_indices = np.concatenate([existing_rows(half), new_rows(half)]) + source = narrow_source(row_indices) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS + half) + + +@pytest.mark.parametrize("plan", PLANS) +def test_update_only(benchmark, narrow: Target, plan: str) -> None: + """No insert clause: unmatched source rows are dropped.""" + half = CLAUSE_SOURCE_ROWS // 2 + row_indices = np.concatenate([existing_rows(half), new_rows(half)]) + source = narrow_source(row_indices) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +def test_delete_by_source(benchmark, narrow: Target) -> None: + """Deleting rows absent from the source requires a full target scan. + + The index gate in `can_use_create_plan` rejects this shape outright, so + there is no v1 variant to compare against. + """ + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .when_not_matched_insert_all() + .when_not_matched_by_source_delete() + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=CLAUSE_SOURCE_ROWS) + + +@pytest.mark.parametrize("plan", PLANS) +def test_conditional_update(benchmark, narrow: Target, plan: str) -> None: + """A condition on target columns forces the target payload to be read.""" + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all(condition="source.value > target.value") + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +@pytest.mark.parametrize("plan", PLANS) +def test_composite_key_fully_indexed(benchmark, narrow: Target, plan: str) -> None: + """Both key columns indexed: v1 probes each index and AND-folds.""" + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert(["composite_a", "composite_b"]) + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +def test_composite_key_partially_indexed(benchmark, narrow: Target) -> None: + """One key column unindexed, which forces the hash join regardless of flag. + + A partial index probe would under-match, so `can_use_create_plan` keeps + this shape off the indexed path. + """ + source = narrow_source(existing_rows(CLAUSE_SOURCE_ROWS)) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + # `composite_a` is indexed, `id_no_index` is not. Both are equal to the + # row index in source and target, so every source row matches. + return ( + dataset.merge_insert(["composite_a", "id_no_index"]) + .when_matched_update_all() + .when_not_matched_insert_all() + .execute(source) + ) + + run(benchmark, narrow, job, expected_rows=NARROW_NUM_ROWS) + + +# --------------------------------------------------------------------------- +# D. Target shape +# --------------------------------------------------------------------------- + +TARGET_SHAPE_ROWS = 10_000 + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_unindexed_tail(benchmark, unindexed_tail: Target, plan: str) -> None: + """10% of the target was appended after indexing, so a scan is unioned in.""" + source = narrow_source(existing_rows(TARGET_SHAPE_ROWS)) + total_rows = NARROW_NUM_ROWS + UNINDEXED_TAIL_ROWS + run( + benchmark, + unindexed_tail, + upsert("id_int", source, use_index=uses_index(plan)), + expected_rows=total_rows, + ) + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_with_deletion_files(benchmark, deleted: Target, plan: str) -> None: + """Every fragment carries a deletion file, which the probe has to mask.""" + source = narrow_source(existing_rows(TARGET_SHAPE_ROWS)) + # Row 0 and every DELETED_ROW_STRIDE-th row after it were deleted at + # generation time, so those source rows insert rather than update. + reinserted = TARGET_SHAPE_ROWS // DELETED_ROW_STRIDE + total_rows = NARROW_NUM_ROWS - NARROW_NUM_ROWS // DELETED_ROW_STRIDE + reinserted + run( + benchmark, + deleted, + upsert("id_int", source, use_index=uses_index(plan)), + expected_rows=total_rows, + ) + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_many_small_fragments(benchmark, frags: Target, plan: str) -> None: + """10K fragments of 1K rows, to expose per-fragment overhead.""" + row_indices = existing_rows(TARGET_SHAPE_ROWS) + source = pa.table( + [pa.array(row_indices), pa.array(row_indices + 1)], schema=FRAGS_SCHEMA + ) + run( + benchmark, + frags, + upsert("id_int", source, use_index=uses_index(plan)), + expected_rows=FRAGS_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# E. Memory and regime edges +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("plan", PLANS) +def test_upsert_streaming_source(benchmark, narrow: Target, plan: str) -> None: + """A one-shot reader source, which v1 has to buffer in full to fork it. + + Peak memory is the number of interest here. + """ + num_rows = 1_000_000 + batch_size = 100_000 + + def make_reader() -> pa.RecordBatchReader: + def batches(): + for start in range(0, num_rows, batch_size): + yield narrow_batch( + existing_rows(batch_size, offset=start), value_offset=1 + ) + + return pa.RecordBatchReader.from_batches(NARROW_SCHEMA, batches()) + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(make_reader()) + ) + + run( + benchmark, + narrow, + job, + rounds=1, + warmup=False, + expected_rows=NARROW_NUM_ROWS, + ) + + +def test_upsert_source_equals_target(benchmark, narrow: Target) -> None: + """Source the same size as the target -- the probe must not be chosen here. + + v2 only: the v1 indexed path cannot run this shape at all. Its source-side + hash join asks for more than the whole memory pool and the operation fails + with "Resources exhausted". So there is no v1 baseline to compare against, + and this benchmark exists to keep the v2 path honest at this size. + """ + source = narrow_source(existing_rows(NARROW_NUM_ROWS)) + run( + benchmark, + narrow, + upsert("id_int", source, use_index=False), + rounds=1, + warmup=False, + expected_rows=NARROW_NUM_ROWS, + ) + + +# --------------------------------------------------------------------------- +# IO / memory variants +# --------------------------------------------------------------------------- +# +# A subset of the above, re-run under the io_memory_benchmark fixture. Write +# amplification (`write_bytes`) is the headline for the partial-column cases and +# peak memory is the headline for the streaming source. + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", PLANS) +@pytest.mark.parametrize("key", NARROW_KEYS) +@pytest.mark.parametrize("num_rows", [1_000, 100_000]) +def test_io_mem_upsert_ratio( + io_mem_benchmark, narrow: Target, num_rows: int, key: str, plan: str +) -> None: + source = narrow_source(existing_rows(num_rows)) + job = upsert(key, source, use_index=uses_index(plan)) + io_mem_benchmark(job, narrow.dataset, setup=lambda: narrow.reset()) + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", WRITE_PLANS) +@pytest.mark.parametrize("fraction", ROW_FRACTIONS, ids=FRACTION_IDS) +def test_io_mem_update_subset_row_fraction( + io_mem_benchmark, wide: Target, fraction: float, plan: str +) -> None: + source = wide_source(wide_row_indices(fraction), PROJECTIONS["one_scalar"]) + job = update_subset( + source, use_index=uses_index(plan), write_mode=write_mode_for(plan) + ) + io_mem_benchmark( + job, + wide.dataset, + warmup=fraction != 1.0, + setup=lambda: wide.reset(), + ) + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", WRITE_PLANS) +@pytest.mark.parametrize("projection", list(PROJECTIONS), ids=list(PROJECTIONS)) +def test_io_mem_update_subset_projection( + io_mem_benchmark, wide: Target, projection: str, plan: str +) -> None: + source = wide_source(wide_row_indices(0.01), PROJECTIONS[projection]) + job = update_subset( + source, use_index=uses_index(plan), write_mode=write_mode_for(plan) + ) + io_mem_benchmark(job, wide.dataset, setup=lambda: wide.reset()) + + +@pytest.mark.io_memory_benchmark() +@pytest.mark.parametrize("plan", PLANS) +def test_io_mem_upsert_streaming_source( + io_mem_benchmark, narrow: Target, plan: str +) -> None: + num_rows = 1_000_000 + batch_size = 100_000 + + def job(dataset: lance.LanceDataset) -> ExecuteResult: + def batches(): + for start in range(0, num_rows, batch_size): + yield narrow_batch( + existing_rows(batch_size, offset=start), value_offset=1 + ) + + reader = pa.RecordBatchReader.from_batches(NARROW_SCHEMA, batches()) + return ( + dataset.merge_insert("id_int") + .when_matched_update_all() + .when_not_matched_insert_all() + .use_index(uses_index(plan)) + .execute(reader) + ) + + io_mem_benchmark(job, narrow.dataset, warmup=False, setup=lambda: narrow.reset()) diff --git a/python/python/ci_benchmarks/benchmarks/test_overlay_manifest.py b/python/python/ci_benchmarks/benchmarks/test_overlay_manifest.py new file mode 100644 index 00000000000..4a9d8c75e83 --- /dev/null +++ b/python/python/ci_benchmarks/benchmarks/test_overlay_manifest.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Impact of overlay files on manifest size. + +Each committed overlay adds a ``DataOverlayFile`` entry (a value-file pointer +plus a serialized coverage bitmap) to every overlaid fragment's metadata in the +manifest. This measures how the manifest grows with the number of overlays and +the coverage size, since manifests are read on every dataset open. +""" + +import pytest +from ci_benchmarks.overlays import ( + commit_overlay_layers, + make_base_dataset, + manifest_size, +) + +NUM_ROWS = 1_000_000 +# Single fragment so every overlay lands on the same fragment's metadata. +ROWS_PER_FILE = NUM_ROWS + +# Strided 10% coverage is the worst case for bitmap size, so manifest growth is +# measured at its upper bound rather than swept across coverage shapes. +COVERAGE_FRACTION = 0.1 +COVERAGE_PATTERN = "stride" + + +@pytest.mark.parametrize("num_overlays", [0, 4, 64]) +def test_overlay_manifest_size(tmp_path, record_property, num_overlays): + base = str(tmp_path / "ds") + ds = make_base_dataset(base, NUM_ROWS, ROWS_PER_FILE, "int32") + base_bytes = manifest_size(ds) + + if num_overlays: + ds = commit_overlay_layers( + ds, num_overlays, COVERAGE_FRACTION, COVERAGE_PATTERN, "int32" + ) + + total = manifest_size(ds) + growth = total - base_bytes + # Guard the fixture: committed overlays must enlarge the manifest, else the + # benchmark would report growth=0 for overlays that were never recorded. + if num_overlays: + assert growth > 0, "overlays did not grow the manifest" + per_overlay = growth / num_overlays if num_overlays else 0 + + record_property("manifest_bytes", total) + record_property("manifest_growth_bytes", growth) + record_property("bytes_per_overlay", per_overlay) diff --git a/python/python/ci_benchmarks/benchmarks/test_overlay_read.py b/python/python/ci_benchmarks/benchmarks/test_overlay_read.py new file mode 100644 index 00000000000..a5774e5ea85 --- /dev/null +++ b/python/python/ci_benchmarks/benchmarks/test_overlay_read.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Impact of overlay files on take and scan workloads. + +On read, every overlay covering a requested cell must be consulted and its value +merged over the base. This measures how take and full-scan cost scale with: +- the number of overlay layers stacked on a fragment (compaction payoff), +- fragmentation (contiguous run vs. strided -> pages touched), and +- value width (a 4-byte int32 vs. a wide fixed-size-list embedding). + +Coverage is fixed at 1% and only the default storage version is measured — these +exist to catch regressions, not to explore the parameter space. + +Wall time is measured warm via pytest-benchmark; read IO (bytes + IOPS) is +measured once cold, after dropping the page cache, via ``io_stats_incremental``. +""" + +import random + +import lance +import pytest +from ci_benchmarks.overlays import commit_overlay_layers, make_base_dataset +from ci_benchmarks.utils import wipe_os_cache + +NUM_ROWS = 1_000_000 +ROWS_PER_FILE = NUM_ROWS # single fragment: isolates overlay-layer scaling +TAKE_ROWS = 100 + +# Wide value column: a 3072-d float32 embedding is 12 KiB/row, ~750x an int32 +# cell. Fewer rows keep the base file to ~1.2 GiB while each cell still dominates +# read cost, so the merge/interleave a scan pays per overlay layer moves real +# payload rather than 4-byte integers. +WIDE_EMBEDDING_DIM = 3072 +NUM_ROWS_WIDE = 100_000 + +# Coverage size is held constant so the sweeps isolate layer count and coverage +# shape; 1% models a small targeted update, the case overlays are built for. +COVERAGE_FRACTION = 0.01 + + +def _take_indices(num_rows: int) -> list[int]: + rng = random.Random(0) + return sorted(rng.sample(range(num_rows), TAKE_ROWS)) + + +def _covered_value(ds: lance.LanceDataset): + """``val`` at offset 0, which every coverage pattern includes. + + Used to guard the fixture: after committing overlays a covered cell must + read back a new value, otherwise a read-invisible overlay would let the + benchmark silently time plain base reads. + """ + return ds.take([0], columns=["val"]).column("val").to_pylist()[0] + + +def _measure_cold_io(ds: lance.LanceDataset, base: str, work): + """Drop the page cache, run ``work`` once, return its read IO stats.""" + wipe_os_cache(base) + ds.io_stats_incremental() # reset + work() + stats = ds.io_stats_incremental() + return stats.read_bytes, stats.read_iops + + +def _run_read(benchmark, record_property, base, ds, workload, num_rows): + if workload == "take": + indices = _take_indices(num_rows) + + def work(): + ds.take(indices, columns=["val"]) + else: + + def work(): + ds.to_table(columns=["val"]) + + read_bytes, read_iops = _measure_cold_io(ds, base, work) + record_property("cold_read_bytes", read_bytes) + record_property("cold_read_iops", read_iops) + + benchmark(work) + + +@pytest.mark.parametrize("workload", ["take", "scan"]) +@pytest.mark.parametrize("num_overlays", [0, 4]) +@pytest.mark.parametrize("pattern", ["contiguous", "stride"]) +def test_overlay_read_scaling( + benchmark, + tmp_path, + record_property, + workload, + num_overlays, + pattern, +): + base = str(tmp_path / "ds") + ds = make_base_dataset(base, NUM_ROWS, ROWS_PER_FILE, "int32") + if num_overlays: + base_val = _covered_value(ds) + ds = commit_overlay_layers( + ds, num_overlays, COVERAGE_FRACTION, pattern, "int32" + ) + assert _covered_value(ds) != base_val, "overlay not visible on read" + _run_read(benchmark, record_property, base, ds, workload, NUM_ROWS) + + +# Mirror test_overlay_read_scaling but on a wide 3072-d embedding column, so the +# take/scan-vs-layers story can be read for a fat value column rather than a +# 4-byte one. +@pytest.mark.parametrize("workload", ["take", "scan"]) +@pytest.mark.parametrize("num_overlays", [0, 4]) +@pytest.mark.parametrize("pattern", ["contiguous", "stride"]) +def test_overlay_read_wide( + benchmark, + tmp_path, + record_property, + workload, + num_overlays, + pattern, +): + base = str(tmp_path / "ds") + ds = make_base_dataset( + base, + NUM_ROWS_WIDE, + NUM_ROWS_WIDE, + "embedding", + embedding_dim=WIDE_EMBEDDING_DIM, + ) + if num_overlays: + base_val = _covered_value(ds) + ds = commit_overlay_layers( + ds, + num_overlays, + COVERAGE_FRACTION, + pattern, + "embedding", + embedding_dim=WIDE_EMBEDDING_DIM, + ) + assert _covered_value(ds) != base_val, "overlay not visible on read" + _run_read(benchmark, record_property, base, ds, workload, NUM_ROWS_WIDE) diff --git a/python/python/ci_benchmarks/benchmarks/test_scan.py b/python/python/ci_benchmarks/benchmarks/test_scan.py index 22186ea33c1..468fab7ebf3 100644 --- a/python/python/ci_benchmarks/benchmarks/test_scan.py +++ b/python/python/ci_benchmarks/benchmarks/test_scan.py @@ -30,4 +30,9 @@ def bench(): ds = lance.dataset(dataset_uri) ds.to_table(offset=num_rows - 100, limit=50) - benchmark.pedantic(bench, rounds=1, iterations=1) + # A single unwarmed round measures whatever page-cache state the preceding + # benchmarks left behind (test_full_scan alone cycles the dataset plus a + # full in-memory table through RAM), not the scan itself. Warm up once and + # take several rounds so the recorded value tracks the code path + # deterministically (see issue #8289). + benchmark.pedantic(bench, rounds=5, iterations=1, warmup_rounds=1) diff --git a/python/python/ci_benchmarks/datagen/gen_all.py b/python/python/ci_benchmarks/datagen/gen_all.py index d5120d20ff7..79725708bb8 100644 --- a/python/python/ci_benchmarks/datagen/gen_all.py +++ b/python/python/ci_benchmarks/datagen/gen_all.py @@ -8,6 +8,7 @@ from ci_benchmarks.datagen.basic import gen_basic from ci_benchmarks.datagen.count_rows import gen_count_rows from ci_benchmarks.datagen.lineitems import gen_tcph +from ci_benchmarks.datagen.merge_insert import gen_merge_insert from ci_benchmarks.datagen.wikipedia import gen_wikipedia @@ -44,6 +45,9 @@ def setup_logging(): LOGGER.info("Generating count_rows benchmark dataset...") gen_count_rows() + LOGGER.info("Generating merge_insert benchmark datasets...") + gen_merge_insert() + LOGGER.info("=" * 80) LOGGER.info("All datasets generated successfully!") LOGGER.info("=" * 80) diff --git a/python/python/ci_benchmarks/datagen/merge_insert.py b/python/python/ci_benchmarks/datagen/merge_insert.py new file mode 100644 index 00000000000..d69a7f4c85f --- /dev/null +++ b/python/python/ci_benchmarks/datagen/merge_insert.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Generate the merge_insert benchmark datasets. + +merge_insert benchmarks mutate their target, so each dataset carries a +``merge_insert_base`` tag pointing at a pristine version. Benchmarks restore +to that tag before every measured run (see ``benchmarks/test_merge_insert.py``), +which makes the suite tolerant of a crashed run leaving stale versions behind. + +Datasets +-------- +``merge_insert_narrow`` + 10M rows, 10 fragments. The key columns differ along two independent + axes so benchmarks can separate them: + + * ``id_int`` vs ``id_uuid7`` -- key *type* (int64 vs string), both + clustered. + * ``id_uuid7`` vs ``id_uuid4`` -- key *distribution* (clustered vs + random), both string. + + ``id_no_index`` holds the same values as ``id_int`` with no index, for the + "user never built an index" baseline. ``composite_a``/``composite_b`` are + both indexed, for composite-key probes. + +``merge_insert_wide`` + 1M rows, 10 fragments. 20 narrow scalar columns (~200 MB) plus one + 256-dim float32 vector column (~1 GB). These exert different pressures on + a partial-column update: the scalar columns stress field count and the + number of distinct buffers to schedule, the vector column stresses raw + byte volume. + +``merge_insert_frags`` + 10M rows in 10K fragments of 1K rows, for per-fragment overhead. + +``merge_insert_deleted`` + ``merge_insert_narrow`` layout with a deletion file on every fragment. + +``merge_insert_unindexed_tail`` + ``merge_insert_narrow`` layout where 10% of the rows were appended after + the index was built, so the probe must union an unindexed scan. +""" + +from __future__ import annotations + +import os +from contextlib import contextmanager + +import lance +import numpy as np +import pyarrow as pa +from lance.log import LOGGER + +from ci_benchmarks.datasets import get_dataset_uri + +# Tag marking the pristine version that benchmarks restore to. +BASE_TAG = "merge_insert_base" + +# BTREE training sorts the full column under DataFusion's FairSpillPool. The +# default 150 MiB pool is tight for 10M 32-byte string keys (uuid columns): +# ExternalSorterMerge has peaked near the pool edge and aborted datagen with +# ResourcesExhausted. Index build is one-shot setup, not a measured path, so +# temporarily enlarge the pool only around create_scalar_index. +_INDEX_BUILD_MEM_POOL_SIZE = str(1024 * 1024 * 1024) + +NARROW_NUM_ROWS = 10_000_000 +NARROW_ROWS_PER_FRAGMENT = 1_000_000 + +WIDE_NUM_ROWS = 1_000_000 +WIDE_ROWS_PER_FRAGMENT = 100_000 +WIDE_NUM_SCALAR_COLUMNS = 20 +WIDE_VECTOR_DIM = 256 + +FRAGS_NUM_ROWS = 10_000_000 +FRAGS_ROWS_PER_FRAGMENT = 1_000 + +# Rows appended to `merge_insert_unindexed_tail` after the index is built. +UNINDEXED_TAIL_ROWS = NARROW_NUM_ROWS // 10 + +# `merge_insert_deleted` deletes every Nth row, which puts a deletion file on +# every fragment. +DELETED_ROW_STRIDE = 1000 + +_BATCH_SIZE = 1_000_000 + +NARROW_INDEXED_COLUMNS = [ + "id_int", + "id_uuid7", + "id_uuid4", + "composite_a", + "composite_b", +] + +NARROW_SCHEMA = pa.schema( + [ + ("id_int", pa.int64()), + ("id_uuid7", pa.string()), + ("id_uuid4", pa.string()), + ("id_no_index", pa.int64()), + ("composite_a", pa.int64()), + ("composite_b", pa.int64()), + ("value", pa.int64()), + ] +) + +FRAGS_SCHEMA = pa.schema([("id_int", pa.int64()), ("value", pa.int64())]) + +WIDE_SCALAR_COLUMNS = [f"scalar_{i}" for i in range(WIDE_NUM_SCALAR_COLUMNS)] + +# Half int64, half string, so a partial-column update touches a mix of +# fixed-width and variable-width buffers. +WIDE_SCHEMA = pa.schema( + [("id_int", pa.int64())] + + [ + (name, pa.int64() if i % 2 == 0 else pa.string()) + for i, name in enumerate(WIDE_SCALAR_COLUMNS) + ] + + [("vec", pa.list_(pa.float32(), WIDE_VECTOR_DIM))] +) + + +_GOLDEN = np.uint64(0x9E3779B97F4A7C15) +_MIX1 = np.uint64(0xBF58476D1CE4E5B9) +_MIX2 = np.uint64(0x94D049BB133111EB) +_HEX_DIGITS = np.frombuffer(b"0123456789abcdef", dtype="S1") +_NIBBLE_SHIFTS = np.arange(60, -4, -4, dtype=np.uint64) +# Two uint64 halves rendered as hex. +_KEY_WIDTH = 32 + + +def _scramble(values: np.ndarray) -> np.ndarray: + """splitmix64 finalizer. Wrapping uint64 arithmetic, vectorized.""" + x = values.astype(np.uint64) * _GOLDEN + x = (x ^ (x >> np.uint64(30))) * _MIX1 + x = (x ^ (x >> np.uint64(27))) * _MIX2 + return x ^ (x >> np.uint64(31)) + + +def _hex_keys(high: np.ndarray, low: np.ndarray) -> pa.Array: + """Format two uint64 columns as 32-character lowercase hex strings. + + Vectorized because the narrow dataset needs 10M of these and the + benchmarks rebuild source keys from row indices on every run. + """ + # One nibble at a time into a preallocated byte matrix. Broadcasting all 16 + # shifts at once would materialize an (n, 16) uint64 array per half, which is + # 2.5 GB of scratch for a 10M-row source. + num_rows = len(high) + half = len(_NIBBLE_SHIFTS) + digits = np.empty((num_rows, _KEY_WIDTH), dtype="S1") + for position, shift in enumerate(_NIBBLE_SHIFTS): + digits[:, position] = _HEX_DIGITS[(high >> shift) & np.uint64(0xF)] + digits[:, position + half] = _HEX_DIGITS[(low >> shift) & np.uint64(0xF)] + + # Built from buffers rather than `pa.array`, which splits a numpy byte array + # into chunks above ~1M elements. RecordBatch needs a contiguous Array. + offsets = np.arange(0, _KEY_WIDTH * (num_rows + 1), _KEY_WIDTH, dtype=np.int32) + return pa.StringArray.from_buffers( + num_rows, pa.py_buffer(offsets), pa.py_buffer(digits) + ) + + +def uuid7_keys(row_indices: np.ndarray) -> pa.Array: + """Sortable, UUIDv7-shaped keys: monotonic prefix, scrambled suffix. + + Real UUIDv7 keys are time-ordered, so a stream of them lands in a narrow, + advancing slice of the index. Reproducing that ordering is what matters + for the benchmark; the exact bit layout is not. + + Deterministic in the row index so benchmarks can reconstruct a key without + reading the dataset. + """ + row_indices = row_indices.astype(np.uint64) + return _hex_keys(row_indices, _scramble(row_indices)) + + +def uuid4_keys(row_indices: np.ndarray) -> pa.Array: + """Same width as `uuid7_keys`, but scattered across the index key space. + + Deterministic for the same reason, but the leading bytes are scrambled, so + a contiguous run of row indices maps to keys spread over the whole index + rather than a narrow slice. + """ + row_indices = row_indices.astype(np.uint64) + return _hex_keys(_scramble(row_indices), row_indices) + + +def narrow_batch(row_indices: np.ndarray, value_offset: int = 0) -> pa.RecordBatch: + """Build a full-schema narrow batch from row indices. + + Shared with the benchmarks, which reconstruct source rows from row indices. + ``value_offset`` shifts the payload column so an update writes a value that + differs from what the target already holds. + """ + return pa.record_batch( + [ + pa.array(row_indices), + uuid7_keys(row_indices), + uuid4_keys(row_indices), + pa.array(row_indices), + pa.array(row_indices), + # `composite_b` is a deterministic function of `composite_a` so a + # source row can target an existing composite key without the + # benchmark tracking extra state. + pa.array(row_indices % 1024), + pa.array(row_indices + value_offset), + ], + schema=NARROW_SCHEMA, + ) + + +def _narrow_data(num_rows: int, offset: int = 0): + LOGGER.info("Generating %d narrow rows starting at %d", num_rows, offset) + for start in range(offset, offset + num_rows, _BATCH_SIZE): + count = min(_BATCH_SIZE, offset + num_rows - start) + yield narrow_batch(np.arange(start, start + count, dtype=np.int64)) + + +def _frags_data(num_rows: int): + LOGGER.info("Generating %d small-fragment rows", num_rows) + for start in range(0, num_rows, _BATCH_SIZE): + ids = np.arange( + start, start + min(_BATCH_SIZE, num_rows - start), dtype=np.int64 + ) + yield pa.record_batch([pa.array(ids), pa.array(ids)], schema=FRAGS_SCHEMA) + + +def _wide_batch(offset: int, num_rows: int) -> pa.RecordBatch: + ids = np.arange(offset, offset + num_rows, dtype=np.int64) + columns: list[pa.Array] = [pa.array(ids)] + for i in range(WIDE_NUM_SCALAR_COLUMNS): + if i % 2 == 0: + columns.append(pa.array(ids + i)) + else: + columns.append(pa.array([f"s{i}_{v}" for v in ids], type=pa.string())) + # Deterministic float payload; the values are irrelevant, the byte volume + # is the point. + vectors = np.linspace( + 0.0, 1.0, num=num_rows * WIDE_VECTOR_DIM, dtype=np.float32 + ).reshape(num_rows, WIDE_VECTOR_DIM) + columns.append( + pa.FixedSizeListArray.from_arrays( + pa.array(vectors.reshape(-1)), WIDE_VECTOR_DIM + ) + ) + return pa.record_batch(columns, schema=WIDE_SCHEMA) + + +def _wide_data(num_rows: int): + LOGGER.info("Generating %d wide rows", num_rows) + # The vector column makes full batches large, so use a smaller batch here. + batch_size = WIDE_ROWS_PER_FRAGMENT + for start in range(0, num_rows, batch_size): + yield _wide_batch(start, min(batch_size, num_rows - start)) + + +def _tag_base(ds: lance.LanceDataset) -> None: + """Point BASE_TAG at the current version, creating the tag if needed.""" + if BASE_TAG in ds.tags.list(): + ds.tags.update(BASE_TAG, ds.version) + else: + ds.tags.create(BASE_TAG, ds.version) + + +def _already_generated(uri: str, expected_rows: int) -> bool: + """True when a usable dataset with a pristine base tag already exists. + + A previous benchmark run may have left extra versions behind, so the row + count is checked at the tagged version rather than at the latest one. + + Incomplete generations (dataset written, tag never created) and missing + tags both return False so the caller can overwrite and retag. + """ + try: + ds = lance.dataset(uri) + except ValueError: + return False + tags = ds.tags.list() + if BASE_TAG not in tags: + return False + base_version = tags[BASE_TAG]["version"] + return ds.checkout_version(base_version).count_rows() == expected_rows + + +@contextmanager +def _enlarged_mem_pool_for_index_build(): + """Raise LANCE_MEM_POOL_SIZE only for the duration of index training.""" + previous = os.environ.get("LANCE_MEM_POOL_SIZE") + os.environ["LANCE_MEM_POOL_SIZE"] = _INDEX_BUILD_MEM_POOL_SIZE + try: + yield + finally: + if previous is None: + os.environ.pop("LANCE_MEM_POOL_SIZE", None) + else: + os.environ["LANCE_MEM_POOL_SIZE"] = previous + + +def _gen( + name: str, + data, + schema: pa.Schema, + expected_rows: int, + rows_per_fragment: int, + indexed_columns: list[str], +) -> lance.LanceDataset: + dataset_uri = get_dataset_uri(name) + if _already_generated(dataset_uri, expected_rows): + LOGGER.info("Dataset %s already exists, skipping", name) + return lance.dataset(dataset_uri) + + LOGGER.info("Creating dataset %s", name) + ds = lance.write_dataset( + data, + dataset_uri, + schema=schema, + mode="overwrite", + max_rows_per_file=rows_per_fragment, + max_rows_per_group=min(rows_per_fragment, 100_000), + ) + with _enlarged_mem_pool_for_index_build(): + for column in indexed_columns: + LOGGER.info("Building BTREE index on %s.%s", name, column) + ds.create_scalar_index(column, "BTREE") + _tag_base(ds) + return ds + + +def gen_merge_insert_narrow() -> lance.LanceDataset: + return _gen( + "merge_insert_narrow", + _narrow_data(NARROW_NUM_ROWS), + NARROW_SCHEMA, + NARROW_NUM_ROWS, + NARROW_ROWS_PER_FRAGMENT, + NARROW_INDEXED_COLUMNS, + ) + + +def gen_merge_insert_wide() -> lance.LanceDataset: + return _gen( + "merge_insert_wide", + _wide_data(WIDE_NUM_ROWS), + WIDE_SCHEMA, + WIDE_NUM_ROWS, + WIDE_ROWS_PER_FRAGMENT, + ["id_int"], + ) + + +def gen_merge_insert_frags() -> lance.LanceDataset: + return _gen( + "merge_insert_frags", + _frags_data(FRAGS_NUM_ROWS), + FRAGS_SCHEMA, + FRAGS_NUM_ROWS, + FRAGS_ROWS_PER_FRAGMENT, + ["id_int"], + ) + + +def gen_merge_insert_deleted() -> lance.LanceDataset: + """Narrow layout with a deletion file on every fragment. + + One row in every `DELETED_ROW_STRIDE` is deleted, which lands in every + fragment. + """ + name = "merge_insert_deleted" + dataset_uri = get_dataset_uri(name) + expected_rows = NARROW_NUM_ROWS - NARROW_NUM_ROWS // DELETED_ROW_STRIDE + if _already_generated(dataset_uri, expected_rows): + LOGGER.info("Dataset %s already exists, skipping", name) + return lance.dataset(dataset_uri) + + ds = _gen( + name, + _narrow_data(NARROW_NUM_ROWS), + NARROW_SCHEMA, + NARROW_NUM_ROWS, + NARROW_ROWS_PER_FRAGMENT, + NARROW_INDEXED_COLUMNS, + ) + LOGGER.info("Deleting rows from %s to create deletion files", name) + ds.delete(f"id_int % {DELETED_ROW_STRIDE} == 0") + _tag_base(ds) + return ds + + +def gen_merge_insert_unindexed_tail() -> lance.LanceDataset: + """Narrow layout where the last 10% of rows are not covered by the index.""" + name = "merge_insert_unindexed_tail" + dataset_uri = get_dataset_uri(name) + expected_rows = NARROW_NUM_ROWS + UNINDEXED_TAIL_ROWS + if _already_generated(dataset_uri, expected_rows): + LOGGER.info("Dataset %s already exists, skipping", name) + return lance.dataset(dataset_uri) + + _gen( + name, + _narrow_data(NARROW_NUM_ROWS), + NARROW_SCHEMA, + NARROW_NUM_ROWS, + NARROW_ROWS_PER_FRAGMENT, + NARROW_INDEXED_COLUMNS, + ) + LOGGER.info("Appending %d unindexed rows to %s", UNINDEXED_TAIL_ROWS, name) + ds = lance.write_dataset( + _narrow_data(UNINDEXED_TAIL_ROWS, offset=NARROW_NUM_ROWS), + dataset_uri, + schema=NARROW_SCHEMA, + mode="append", + max_rows_per_file=NARROW_ROWS_PER_FRAGMENT, + ) + _tag_base(ds) + return ds + + +def gen_merge_insert() -> None: + gen_merge_insert_narrow() + gen_merge_insert_wide() + gen_merge_insert_frags() + gen_merge_insert_deleted() + gen_merge_insert_unindexed_tail() diff --git a/python/python/ci_benchmarks/overlays.py b/python/python/ci_benchmarks/overlays.py new file mode 100644 index 00000000000..9d9d182de12 --- /dev/null +++ b/python/python/ci_benchmarks/overlays.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Shared helpers for the data-overlay benchmark suite. + +Data overlay files supply replacement values for a subset of (row offset, field) +cells in a fragment, merged on read, without rewriting the base data file. These +helpers build synthetic base datasets, commit overlay layers through the public +``lance.LanceOperation.DataOverlay`` operation, and measure their cost. +""" + +import os + +# Data overlay support is gated off in release builds unless this is set (it is +# always on in debug builds). Benchmarks usually run against a release build, so +# enable it here, before lance is imported, or reading overlay datasets fails. +os.environ.setdefault("LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES", "1") + +from typing import List # noqa: E402 +from urllib.parse import urlparse # noqa: E402 + +import lance # noqa: E402 +import numpy as np # noqa: E402 +import pyarrow as pa # noqa: E402 +from lance._datagen import rand_batches # noqa: E402 +from lance.file import LanceFileWriter # noqa: E402 + +# Default width for the ``embedding`` dtype. Individual benchmarks override it +# via ``embedding_dim`` to model narrow vs. wide (e.g. 3072-d) value columns. +EMBEDDING_DIM = 128 + + +def _value_type(dtype: str, embedding_dim: int = EMBEDDING_DIM) -> pa.DataType: + if dtype == "int32": + return pa.int32() + if dtype == "embedding": + return pa.list_(pa.float32(), embedding_dim) + raise ValueError(f"unknown overlay benchmark dtype {dtype!r}") + + +def _gen_values( + dtype: str, + n: int, + rng: np.random.Generator, + embedding_dim: int = EMBEDDING_DIM, +) -> pa.Array: + if dtype == "int32": + return pa.array(rng.integers(0, 1 << 30, size=n, dtype=np.int32)) + if dtype == "embedding": + flat = rng.random(n * embedding_dim, dtype=np.float32) + return pa.FixedSizeListArray.from_arrays(pa.array(flat), embedding_dim) + raise ValueError(f"unknown overlay benchmark dtype {dtype!r}") + + +def make_base_dataset( + base_path: str, + num_rows: int, + rows_per_file: int, + dtype: str, + embedding_dim: int = EMBEDDING_DIM, +) -> lance.LanceDataset: + """Create a base dataset with an ``id`` column and a ``val`` column. + + ``val`` is the column overlays target. ``rows_per_file`` controls the number + of fragments (``num_rows // rows_per_file``), which must divide ``num_rows``. + ``embedding_dim`` sets the width of the ``val`` column when ``dtype`` is + ``embedding`` (e.g. 3072 for a wide embedding). + """ + if num_rows % rows_per_file: + raise ValueError( + f"num_rows ({num_rows}) must be a multiple of rows_per_file " + f"({rows_per_file})" + ) + schema = pa.schema({"id": pa.int64(), "val": _value_type(dtype, embedding_dim)}) + # One fragment per batch; lance-datagen fills both columns with random data. + reader = rand_batches( + schema, num_batches=num_rows // rows_per_file, rows_per_batch=rows_per_file + ) + return lance.write_dataset(reader, base_path, max_rows_per_file=rows_per_file) + + +def coverage_offsets(num_rows: int, fraction: float, pattern: str) -> List[int]: + """Offsets within a fragment covered by an overlay. + + ``contiguous`` packs the covered cells into a single leading run (few pages + touched); ``stride`` spreads them evenly across the fragment (many pages + touched). Both cover ``round(num_rows * fraction)`` cells. + """ + count = max(1, int(round(num_rows * fraction))) + if pattern == "contiguous": + return list(range(count)) + if pattern == "stride": + step = max(1, num_rows // count) + return list(range(0, num_rows, step))[:count] + raise ValueError(f"unknown coverage pattern {pattern!r}") + + +def _val_field_id(ds: lance.LanceDataset) -> int: + base_df = ds.get_fragments()[0].metadata.files[0] + names = [f.name for f in ds.schema] + return base_df.fields[names.index("val")] + + +def commit_overlay_layers( + ds: lance.LanceDataset, + num_layers: int, + fraction: float, + pattern: str, + dtype: str, + *, + seed: int = 0, + embedding_dim: int = EMBEDDING_DIM, +) -> lance.LanceDataset: + """Commit ``num_layers`` overlays on ``val``, each covering the same offsets + in every fragment so that all layers must be consulted on read (the case + that motivates compaction). Returns the updated dataset. + """ + base_df = ds.get_fragments()[0].metadata.files[0] + field_id = _val_field_id(ds) + data_dir = os.path.join(_local_path(ds), "data") + for layer in range(num_layers): + rng = np.random.default_rng(seed + layer + 1) + groups = [] + for frag in ds.get_fragments(): + offsets = coverage_offsets(frag.count_rows(), fraction, pattern) + values = _gen_values(dtype, len(offsets), rng, embedding_dim) + batch = pa.record_batch([values], names=["val"]) + name = f"overlay_l{layer}_f{frag.fragment_id}.lance" + path = os.path.join(data_dir, name) + with LanceFileWriter(path) as writer: + writer.write_batch(batch) + df = lance.fragment.DataFile( + path=name, + fields=[field_id], + column_indices=[0], + file_major_version=base_df.file_major_version, + file_minor_version=base_df.file_minor_version, + file_size_bytes=os.path.getsize(path), + ) + groups.append( + lance.LanceOperation.DataOverlayGroup( + fragment_id=frag.fragment_id, + overlays=[ + lance.LanceOperation.DataOverlayFile( + data_file=df, offsets=offsets + ) + ], + ) + ) + op = lance.LanceOperation.DataOverlay(groups=groups) + ds = lance.LanceDataset.commit(ds, op, read_version=ds.version) + return ds + + +# --- Measurement helpers ---------------------------------------------------- + + +def _local_path(ds: lance.LanceDataset) -> str: + parsed = urlparse(ds.uri) + return parsed.path if parsed.scheme == "file" else ds.uri + + +def manifest_size(ds: lance.LanceDataset) -> int: + """Size in bytes of the manifest for the dataset's current version.""" + # Manifests are named `{u64::MAX - version}.manifest` so that a plain + # lexicographic directory listing yields newest-version-first. + name = f"{(1 << 64) - 1 - ds.version}.manifest" + return os.path.getsize(os.path.join(_local_path(ds), "_versions", name)) diff --git a/python/python/lance/__init__.py b/python/python/lance/__init__.py index 61d94b5550a..981140858c7 100644 --- a/python/python/lance/__init__.py +++ b/python/python/lance/__init__.py @@ -44,16 +44,19 @@ CleanupStats, DatasetBasePath, FFILanceTableProvider, + FtsToken, ScanStatistics, bytes_read_counter, iops_counter, + simd_info, + tokenize, ) from .mem_wal import ( + CompactedSsTable, ExecutionPlan, LsmPointLookupPlanner, LsmScanner, LsmVectorSearchPlanner, - MergedGeneration, ShardingField, ShardingSpec, ShardSnapshot, @@ -97,6 +100,7 @@ "DataStatistics", "FieldStatistics", "FragmentMetadata", + "FtsToken", "Index", "IndexFile", "LanceDataset", @@ -115,6 +119,8 @@ "json_to_schema", "schema_to_json", "set_logger", + "simd_info", + "tokenize", "write_dataset", "FFILanceTableProvider", "IndexProgress", @@ -122,7 +128,7 @@ "LsmPointLookupPlanner", "LsmScanner", "LsmVectorSearchPlanner", - "MergedGeneration", + "CompactedSsTable", "ShardSnapshot", "ShardWriter", "ShardingField", diff --git a/python/python/lance/_datagen.py b/python/python/lance/_datagen.py index b156066eca6..45826503c5b 100644 --- a/python/python/lance/_datagen.py +++ b/python/python/lance/_datagen.py @@ -21,10 +21,13 @@ def rand_batches( *, num_batches: Optional[int] = None, batch_size_bytes: Optional[int] = None, + rows_per_batch: Optional[int] = None, ): if not datagen.is_datagen_supported(): raise NotImplementedError( "This version of lance was not built with the datagen feature" ) - batch_iter = datagen.rand_batches(schema, num_batches, batch_size_bytes) + batch_iter = datagen.rand_batches( + schema, num_batches, batch_size_bytes, rows_per_batch + ) return pa.RecordBatchReader.from_batches(schema, batch_iter) diff --git a/python/python/lance/arrow.py b/python/python/lance/arrow.py index da022cdc8f6..5e9f671ab80 100644 --- a/python/python/lance/arrow.py +++ b/python/python/lance/arrow.py @@ -310,16 +310,7 @@ def pillow_metadata_decoder(images): img = Image.open(io.BytesIO(images[0].as_py())) return img - def tensorflow_metadata_decoder(images): - import tensorflow as tf - - img = tf.io.decode_image(images[0].as_py()) - return img - - decoders = ( - ("tensorflow", tensorflow_metadata_decoder), - ("PIL", pillow_metadata_decoder), - ) + decoders = (("PIL", pillow_metadata_decoder),) decoder = None for libname, metadata_decoder in decoders: @@ -351,7 +342,7 @@ def to_tensor( decoder : Callable[pa.binary()], optional A function that takes a binary array and returns a numpy.ndarray or pa.fixed_shape_tensor. If not provided, will attempt to use - tensorflow and then pillow decoder in that order. + pillow. Returns ------- @@ -385,20 +376,7 @@ def pillow_decoder(images) -> "np.ndarray": ] ) - def tensorflow_decoder(images) -> "np.ndarray": - import tensorflow as tf - - decoded_to_tensor = tuple( - tf.io.decode_image(img) for img in images.to_pylist() - ) - return tf.stack( # pyright: ignore[reportOptionalCall] - decoded_to_tensor, axis=0 - ).numpy() - - decoders = [ - ("tensorflow", tensorflow_decoder), - ("PIL", pillow_decoder), - ] + decoders = [("PIL", pillow_decoder)] for libname, decoder_function in decoders: try: __import__(libname) @@ -408,8 +386,8 @@ def tensorflow_decoder(images) -> "np.ndarray": pass else: raise ValueError( - "No image decoder available. Please either install one of " - "tensorflow, pillow, or pass a decoder argument." + "No image decoder available. Please install pillow or pass a " + "decoder argument." ) image_array = decoder(self.storage) @@ -499,19 +477,8 @@ def pillow_encoder(x): encoded_images.append(buf.getvalue()) return pa.array(encoded_images, type=storage_type) - def tensorflow_encoder(x): - import tensorflow as tf - - encoded_images = ( - tf.io.encode_png(y).numpy() for y in tf.convert_to_tensor(x) - ) - return pa.array(encoded_images, type=storage_type) - if not encoder: - encoders = ( - ("PIL", pillow_encoder), - ("tensorflow", tensorflow_encoder), - ) + encoders = (("PIL", pillow_encoder),) for libname, encoder_function in encoders: try: __import__(libname) @@ -521,8 +488,8 @@ def tensorflow_encoder(x): pass else: raise ValueError( - "No image encoder available. Please either install one of " - "tensorflow, pillow, or pass an encoder argument." + "No image encoder available. Please install pillow or pass an " + "encoder argument." ) return EncodedImageArray.from_storage( diff --git a/python/python/lance/blob.py b/python/python/lance/blob.py index 6d1af6797dc..d8415c6a7a0 100644 --- a/python/python/lance/blob.py +++ b/python/python/lance/blob.py @@ -41,8 +41,10 @@ class Blob: A blob can be represented as: - inline bytes - - an external URI with position and size, if position and size are not set, - use the full uri. + - an external URI, optionally with a non-empty range + + Every blob must use exactly one representation. Use ``None`` for a null + blob and :meth:`empty` for a valid empty blob. """ data: Optional[bytes] = None @@ -65,6 +67,10 @@ def __post_init__(self) -> None: raise ValueError( "Blob cannot have both inline data and external slice metadata" ) + if self.data is None and self.uri is None: + raise ValueError("Blob must set `data` or `uri`; use None for a null blob") + if self.size == 0: + raise ValueError("External blob range size must be greater than zero") @staticmethod def from_bytes(data: Union[bytes, bytearray, memoryview]) -> "Blob": @@ -90,7 +96,13 @@ class BlobType(pa.ExtensionType): A PyArrow extension type for Lance blob columns. This is the "logical" type users write. Lance will store it in a compact - descriptor format, and reads will return descriptors by default. + descriptor format, and reads will return descriptors by default. Its storage + type defaults to ``Struct``. Arrow deserialization also preserves the accepted minimal + ``Struct`` storage type. ``position`` and + ``size`` select a range within an external ``uri`` and must either both be set + or both be null. When set, ``size`` must be greater than zero. Every non-null + value must set exactly one of ``data`` and ``uri``. """ def __init__(self) -> None: @@ -107,11 +119,47 @@ def __init__(self) -> None: def __arrow_ext_serialize__(self) -> bytes: return b"" + @staticmethod + def _validate_storage_type(storage_type: pa.DataType) -> None: + if not pa.types.is_struct(storage_type): + raise TypeError("BlobType storage type must be a struct") + + fields = list(storage_type) + if len(fields) not in (2, 4): + raise TypeError( + "BlobType storage struct must contain either data/uri or " + "data/uri/position/size" + ) + + expected_fields = [ + ("data", pa.large_binary()), + ("uri", pa.utf8()), + ("position", pa.uint64()), + ("size", pa.uint64()), + ] + for index, field in enumerate(fields): + expected_name, expected_type = expected_fields[index] + if field.name != expected_name or field.type != expected_type: + raise TypeError( + "BlobType storage field " + f"{index} must be {expected_name}: {expected_type}, got " + f"{field.name}: {field.type}" + ) + if index < 2 and not field.nullable: + raise TypeError(f"BlobType storage field {field.name} must be nullable") + + @classmethod + def _from_storage_type(cls, storage_type: pa.DataType) -> "BlobType": + cls._validate_storage_type(storage_type) + instance = cls.__new__(cls) + pa.ExtensionType.__init__(instance, storage_type, "lance.blob.v2") + return instance + @classmethod def __arrow_ext_deserialize__( cls, storage_type: pa.DataType, serialized: bytes ) -> "BlobType": - return BlobType() + return cls._from_storage_type(storage_type) def __arrow_ext_class__(self): return BlobArray @@ -239,6 +287,13 @@ def blob_field( """ Construct an Arrow field for a Lance blob column. + The returned field uses the complete logical blob shape + ``Struct``. + Every non-null value must set exactly one of ``data`` and ``uri``. External + ranges must set both ``position`` and a positive ``size``. Lance preserves + this logical schema across create, append, and merge-insert writes while + storing compact descriptors internally. + Parameters ---------- name : str @@ -383,6 +438,28 @@ def read_range(self, offset: int, length: int) -> bytes: """Read a blob-local byte range without changing the current cursor.""" return self.inner.read_range(offset, length) + def read_ranges(self, ranges: list[tuple[int, int]]) -> list[bytes]: + """ + Read multiple blob-local byte ranges without changing the current cursor. + + Each range is an ``(offset, length)`` pair, matching + :py:meth:`read_range`. The underlying physical reads may be reordered, + coalesced, or split for efficiency. For every range, offset plus length + must fit in an unsigned 64-bit integer and must not extend beyond the + blob size. + + Parameters + ---------- + ranges : List[Tuple[int, int]] + The ``(offset, length)`` byte ranges to read. + + Returns + ------- + data : List[bytes] + One payload per requested range, in input order. + """ + return self.inner.read_ranges(ranges) + def readinto(self, b: bytearray) -> int: return self.inner.read_into(b) diff --git a/python/python/lance/commit.py b/python/python/lance/commit.py index 806067a38c5..9b5d59de7eb 100644 --- a/python/python/lance/commit.py +++ b/python/python/lance/commit.py @@ -7,5 +7,15 @@ CommitLock = Callable[[int], AbstractContextManager] -class CommitConflictError(Exception): - pass +class CommitConflictError(OSError): + """A commit conflicted with a concurrent transaction. + + Subclasses :class:`OSError` so existing ``except OSError`` handlers keep + working. ``retryable`` is ``True`` when the transaction was preempted and + can be retried against the newer version, ``False`` when the conflict is + incompatible and retrying will not help. + """ + + def __init__(self, message: str = "", retryable: bool = True): + super().__init__(message) + self.retryable = retryable diff --git a/python/python/lance/dataset.py b/python/python/lance/dataset.py index 6be0a78d2e8..616c5f31004 100644 --- a/python/python/lance/dataset.py +++ b/python/python/lance/dataset.py @@ -9,6 +9,7 @@ import operator import os import random +import re import time import uuid import warnings @@ -44,6 +45,9 @@ from .dependencies import ( _check_for_numpy, _check_for_torch, + _is_pydantic_base_model_class, + _validate_pydantic_list, + model_to_dict, torch, ) from .dependencies import numpy as np @@ -65,17 +69,22 @@ _MergeInsertBuilder, _parse_field_path, _Scanner, + _serialize_row_addrs, _write_dataset, indices, ) from .lance import __version__ as __version__ from .lance import _Session as Session -from .query import FullTextQuery -from .types import _coerce_reader +from .query import DocumentGranularity, FullTextQuery +from .types import _coerce_reader, _is_materialized from .udf import BatchUDF, normalize_transform from .udf import BatchUDFCheckpoint as BatchUDFCheckpoint from .udf import batch_udf as batch_udf -from .util import _target_partition_size_to_num_partitions, td_to_micros +from .util import ( + _normalize_index_segment_ids, + _target_partition_size_to_num_partitions, + td_to_micros, +) if TYPE_CHECKING: from pyarrow._compute import Expression @@ -235,14 +244,6 @@ def _is_null_blob_description(description: Any) -> bool: return False if description.keys() == {"position", "size"}: return description["position"] == 1 and description["size"] == 0 - if description.keys() == {"kind", "position", "size", "blob_id", "blob_uri"}: - return ( - description["kind"] == 0 - and description["position"] == 0 - and description["size"] == 0 - and description["blob_id"] == 0 - and description["blob_uri"] == "" - ) return False @@ -393,6 +394,12 @@ def execute(self, data_obj: ReaderLike, *, schema: Optional[pa.Schema] = None): """ reader = _coerce_reader(data_obj, schema) + # Materialized sources are wrapped in an in-memory table so retries never + # spill and the source's statistics can drive the join; everything else is + # treated as a one-shot stream. + if _is_materialized(data_obj): + return super(MergeInsertBuilder, self).execute_batches(reader) + return super(MergeInsertBuilder, self).execute(reader) def execute_uncommitted( @@ -417,6 +424,9 @@ def execute_uncommitted( """ reader = _coerce_reader(data_obj, schema) + if _is_materialized(data_obj): + return super(MergeInsertBuilder, self).execute_uncommitted_batches(reader) + return super(MergeInsertBuilder, self).execute_uncommitted(reader) # These next three overrides exist only to document the methods @@ -536,6 +546,49 @@ def use_index(self, use_index: bool) -> "MergeInsertBuilder": """ return super(MergeInsertBuilder, self).use_index(use_index) + def write_mode( + self, mode: Literal["auto", "rewrite_rows", "rewrite_columns"] + ) -> "MergeInsertBuilder": + """ + Selects how the merged rows are written to disk. + + For a partial-schema update (the source omits some dataset columns) the + two modes have different cost shapes. ``rewrite_columns`` never reads or + writes the columns the source omits, but its replacement column file + covers every row of each fragment it touches, so the bytes written barely + fall as fewer rows match. ``rewrite_rows`` instead scales with the number + of matched rows. Patching columns wins once the fraction of rows matched + exceeds roughly the fraction of each row's bytes the source columns + occupy: for a KB-scale update of a MB-per-row table that is nearly + always, and for a table whose columns are all narrow it may never be. + + The per-fragment matched row count that decides this is only known once + the join has run, so the caller picks rather than the planner guessing. + + Parameters + ---------- + mode : {'auto', 'rewrite_rows', 'rewrite_columns'} + ``auto`` (default) lets the engine choose. It rewrites whole rows, + except on the one path that predates this parameter: a + partial-schema update whose join key carries a scalar index patches + columns. ``rewrite_rows`` deletes the matched rows and writes whole + rows into new fragments; for a partial-schema update it gives up + that index probe, since the indexed path only ever patches columns. + ``rewrite_columns`` attaches new data files holding the source + columns to the fragments that already hold the matched rows; it + raises if the merge cannot be expressed that way, which requires + updating matched rows only (no inserts, no matched deletes, no + delete-by-source) with a source that omits at least one dataset + column, carries at least one column besides the join key, and + carries no blob column. + + Returns + ------- + MergeInsertBuilder + The builder instance for method chaining. + """ + return super(MergeInsertBuilder, self).write_mode(mode) + def target_bases(self, bases: List[str]) -> "MergeInsertBuilder": """ Write new fragments produced by this merge insert to these bases. @@ -590,9 +643,9 @@ def explain_plan( """ Generate the execution plan for the merge insert operation. - This method creates the execution plan that would be used for the given - source schema and returns it as a formatted string for debugging and - analysis purposes. + This reports the plan a *streaming* source of the given schema would run. + It takes a schema rather than data, so it cannot know how ``execute`` would + wrap the source; see the note under the example. Parameters ---------- @@ -634,6 +687,13 @@ def explain_plan( StreamingTableExec: partition_sizes=1, ... + This is always the streaming shape. `explain_plan` receives a schema rather + than data, so it cannot know how `execute` would wrap the source, and the + wrapping affects the plan. Use `analyze_plan`, which receives the real + source, when that matters. Note that `analyze_plan` runs the merge to + collect metrics and may write data files, whereas `explain_plan` writes + nothing. + >>> # Or with explicit schema >>> source_schema = pa.schema([ ... pa.field("id", pa.int64()), @@ -708,11 +768,19 @@ def analyze_plan( MergeInsert: elapsed=..., on=[id], ..., metrics=[..., bytes_written=..., ...] CoalescePartitionsExec, elapsed=..., metrics=[output_rows=..., elapsed_compute=...] ProjectionExec: elapsed=..., expr=[...], metrics=[...] - HashJoinExec: elapsed=..., mode=CollectLeft, join_type=Right, ... - LanceRead: elapsed=..., ..., metrics=[..., bytes_read=..., ...] - RepartitionExec: ... + RepartitionExec: ... + HashJoinExec: elapsed=..., mode=CollectLeft, join_type=Left, ... ProjectionExec: elapsed=..., expr=[..., true as __merge_source_sentinel], metrics=[...] - StreamingTableExec: ..., metrics=[] + DataSourceExec: ..., metrics=[] + LanceRead: elapsed=..., ..., metrics=[..., bytes_read=..., ...] + + The reported plan follows how the source was passed. `new_data` above is a + `pa.Table`, so it is wrapped in an in-memory table that reports exact + statistics, while a `pa.RecordBatchReader` reports none. DataFusion chooses + which side of the join to collect from those statistics and from the two + sides' sizes, so the same merge can plan differently depending on which one + you hand it. Use `explain_plan` only for the streaming shape: it takes a + schema rather than data, so it cannot know how the source would be wrapped. The two key parts of the plan analysis are LanceRead and MergeInsert. LanceRead scans join keys and columns in conditions. MergeInsert writes @@ -733,25 +801,39 @@ def analyze_plan( - requests: number of storage requests made """ # noqa: E501 reader = _coerce_reader(data_obj, schema) + + # Route exactly as execute() does, so the reported plan is the one that + # would run. A materialized source reports exact statistics where a stream + # reports none, which can change which side of the join is collected. + if _is_materialized(data_obj): + return super(MergeInsertBuilder, self).analyze_plan_batches(reader) + return super(MergeInsertBuilder, self).analyze_plan(reader) - def mark_generations_as_merged( - self, generations: "List[mem_wal.MergedGeneration]" + def mark_sstables_as_compacted( + self, sstables: "List[mem_wal.CompactedSsTable]" ) -> "MergeInsertBuilder": - """Mark MemWAL generations as merged into the base table. + """Mark MemWAL SSTables as compacted into the base table. - Call this before executing the merge_insert when the source data - includes rows from MemWAL flushed generations. + Call this before executing merge_insert when it compacts MemWAL SSTables. + The progress is recorded in the same commit as the data. + + For multi-pass compaction, call this only on the final successful + data-changing pass. Intermediate passes must not carry compaction + progress. Lance cannot tell whether a caller has another pass planned, + so it cannot enforce this: if a delete pass carried the marker and the + process then died before the matching upsert, the recorded progress + would claim rows were copied in that never were. Parameters ---------- - generations : list of MergedGeneration - Generations to mark as merged. + sstables : list of CompactedSsTable + SSTables to mark as compacted. """ - from .mem_wal import _to_raw_merged_generations + from .mem_wal import _to_raw_compacted_sstables - raw_gens = _to_raw_merged_generations(generations) - super(MergeInsertBuilder, self).mark_generations_as_merged(raw_gens) + raw_sstables = _to_raw_compacted_sstables(sstables) + super(MergeInsertBuilder, self).mark_sstables_as_compacted(raw_sstables) return self @@ -842,6 +924,55 @@ def __deserialize__( base_store_params=base_store_params, ) + @classmethod + def from_pydantic_model( + cls, + model_class, + data, + uri: Optional[Union[str, Path]] = None, + mode: str = "create", + **kwargs, + ) -> "LanceDataset": + """Create a LanceDataset from a Pydantic model class and a list of instances. + + The table name is inferred from the model class name converted to snake_case. + The schema is inferred from the model class's field annotations, not from + the data, so optional fields are typed correctly even if every value in a + given batch happens to be None. + + Parameters + ---------- + model_class : type + A Pydantic BaseModel subclass. + data : list + A list of Pydantic model instances. + uri : str or Path, optional + The URI to write the dataset to. If not provided, the model class name + converted to snake_case is used as the path. + mode : str, optional + The write mode. One of "create", "overwrite", or "append". + **kwargs + Additional arguments passed to write_dataset(). + """ + if not _is_pydantic_base_model_class(model_class): + raise TypeError( + f"`model_class` must be a Pydantic BaseModel subclass, " + f"got {model_class!r}" + ) + _validate_pydantic_list(data, model_class) + if not data: + raise ValueError( + "`data` must be a non-empty list of Pydantic model instances." + ) + if uri is None: + uri = re.sub(r"(? LanceScanner: """Return a Scanner that can support various pushdowns. @@ -1189,26 +1322,31 @@ def scanner( batch_size: int, default None The maximum number of rows per batch. In some cases batches can be - smaller than this size. Note: this can be overridden by - ``batch_size_bytes`` or by a dataset-level ``batch_size_bytes`` - configured via ``FileReaderOptions``. + smaller than this size. If a byte limit is also configured, both + limits apply and the one reached first determines the batch size. batch_size_bytes: int, default None If set, the scanner will produce batches whose total size in bytes - is approximately this value, overriding the row-based ``batch_size``. + is approximately this value. If ``batch_size`` is also set, both + limits apply and the one reached first determines the batch size. + This cannot be combined with ``strict_batch_size=True`` because + strict row batching can merge batches beyond the byte limit. This can also be configured at the dataset level via ``FileReaderOptions``. A scanner-level setting takes precedence over the dataset-level default. io_buffer_size: int, default None - The size of the IO buffer. See ``ScannerBuilder.io_buffer_size`` - for more information. + The maximum number of bytes to buffer from storage before applying + backpressure. See ``ScannerBuilder.io_buffer_size`` for more information. batch_readahead: int, optional - The number of batches to read ahead. + The number of batches to decode concurrently. fragment_readahead: int, optional - The number of fragments to read ahead. + The number of fragments whose reads may be scheduled concurrently. + This applies even when ``scan_in_order`` is true. Set this to ``1`` + to avoid overlapping I/O from multiple fragments. scan_in_order: bool, default True - Whether to read the fragments and batches in order. If false, - throughput may be higher, but batches will be returned out of order - and memory use might increase. + Whether to return fragments and batches in order. This does not make + storage reads sequential; use ``fragment_readahead=1`` for that. If + false, throughput may be higher, but batches will be returned out of + order and memory use might increase. fragments: iterable of LanceFragment, default None If specified, only scan these fragments. If scan_in_order is True, then the fragments will be scanned in the order given. @@ -1272,6 +1410,10 @@ def scanner( A callback function that will be called with the scan statistics after the scan is complete. Errors raised by the callback will be logged but not re-raised. + strict_batch_size: bool, default False + If True, all batches except the last batch will have exactly + ``batch_size`` rows. This cannot be combined with a byte limit, + including one configured through ``FileReaderOptions``. include_deleted_rows: bool, default False If True, then rows that have been deleted, but are still present in the fragment, will be returned. These rows will have the _rowid column set @@ -1294,6 +1436,14 @@ def scanner( This parameter allows you to opt-in to the new behavior early, to avoid being subject to breaking changes in the future. + row_addr_allowlist: bytes, default None + Restrict the scan to these row addresses. A serialized roaring treemap + over ``_rowid`` (``RowAddrTreeMap::serialize_into`` output). Applied + before KNN / BM25 ranking, so top-k is computed over the surviving rows + rather than filtered afterwards. + row_addr_blocklist: bytes, default None + Exclude these row addresses, same encoding as ``row_addr_allowlist``. + Combined with it when both are given. .. note:: @@ -1336,6 +1486,8 @@ def setopt(opt, val): setopt(builder.filter, filter) setopt(builder.prefilter, prefilter) + if row_addr_allowlist is not None or row_addr_blocklist is not None: + builder.row_addr_prefilter(row_addr_allowlist, row_addr_blocklist) setopt(builder.limit, limit) setopt(builder.offset, offset) setopt(builder.batch_size, batch_size) @@ -2162,7 +2314,7 @@ def take_blobs( ids: Optional[Union[List[int], pa.Array]] = None, addresses: Optional[Union[List[int], pa.Array]] = None, indices: Optional[Union[List[int], pa.Array]] = None, - ) -> List[BlobFile]: + ) -> List[Optional[BlobFile]]: """ Select blobs by row IDs. @@ -2189,7 +2341,9 @@ def take_blobs( Returns ------- - blob_files : List[BlobFile] + blob_files : List[Optional[BlobFile]] + One element per selected row. Null blob values return ``None``; + valid empty blobs return a ``BlobFile`` with size zero. """ selection_kind, selection_values = _resolve_blob_selection( ids, addresses, indices @@ -2205,7 +2359,10 @@ def take_blobs( lance_blob_files = self._ds.take_blobs_by_indices( selection_values, blob_column ) - return [BlobFile(lance_blob_file) for lance_blob_file in lance_blob_files] + return [ + BlobFile(lance_blob_file) if lance_blob_file is not None else None + for lance_blob_file in lance_blob_files + ] def read_blobs( self, @@ -2216,7 +2373,7 @@ def read_blobs( *, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: + ) -> List[Tuple[int, Optional[bytes]]]: """ Read blobs directly into memory using Lance's planned blob reader. @@ -2244,8 +2401,9 @@ def read_blobs( Returns ------- - blobs : List[Tuple[int, bytes]] - A list of ``(row_address, blob_bytes)`` pairs. + blobs : List[Tuple[int, Optional[bytes]]] + One ``(row_address, blob_bytes)`` pair per selected row. Null blob + values return ``None``; valid empty blobs return ``b""``. """ selection_kind, selection_values = _resolve_blob_selection( ids, addresses, indices @@ -2263,6 +2421,74 @@ def read_blobs( ) return self._ds.read_blobs_by_indices(selection_values, blob_column, **kwargs) + def read_blob_ranges( + self, + blob_column: str, + requests: Sequence[Tuple[int, int, int]], + *, + selector: Literal["ids", "addresses", "indices"], + io_buffer_size: Optional[int] = None, + preserve_order: Optional[bool] = None, + ) -> List[Tuple[int, int, Optional[bytes]]]: + """ + Read row-specific blob-local byte ranges with one planned API call. + + Each request is a ``(row, offset, length)`` tuple. ``selector`` defines + whether every ``row`` value is interpreted as a stable row ID, physical + row address, or dataset index. Repeat a row value to read multiple ranges + from the same blob. Planning, range validation, physical-source grouping, + request coalescing, and bounded I/O scheduling all happen in Rust; this + method does not use a Python thread pool. + + Every request produces one result. Requests on null blobs return ``None``, + including when the requested range is empty. Empty ranges on non-null + blobs return empty bytes without issuing payload I/O. For every request, + offset plus length must fit in an unsigned 64-bit integer. Ranges on + non-null blobs must not extend beyond the logical blob size. Blob-local + bounds are not evaluated for null values because they have no logical + payload length. By default results preserve the input request order. + + Parameters + ---------- + blob_column : str + The name of the blob column to read. + requests : Sequence[Tuple[int, int, int]] + Complete ``(row, offset, length)`` requests. + selector : {"ids", "addresses", "indices"} + Interpretation of every request's ``row`` value. + io_buffer_size : int, optional + Override the scheduler I/O buffer size used while materializing ranges. + preserve_order : bool, optional + If True, returned results follow request order. If False, + ``request_index`` still identifies every result. + + Examples + -------- + Read two disjoint ranges from the same row index: + + .. code-block:: python + + dataset.read_blob_ranges( + "images", + requests=[(7, 0, 1024), (7, 4096, 1024)], + selector="indices", + ) + + Returns + ------- + results : List[Tuple[int, int, Optional[bytes]]] + One ``(request_index, row_address, data)`` tuple per request. + The Python list retains all returned payload bytes in memory; peak + result memory is therefore proportional to their total logical size. + """ + return self._ds.read_blob_ranges( + list(requests), + blob_column, + selector, + io_buffer_size=io_buffer_size, + preserve_order=preserve_order, + ) + def head(self, num_rows, **kwargs): """ Load the first N rows of the dataset. @@ -2281,6 +2507,52 @@ def head(self, num_rows, **kwargs): kwargs["limit"] = num_rows return self.scanner(**kwargs).to_table() + def slice( + self, + start: int, + end: int, + columns: Optional[Union[List[str], Dict[str, str]]] = None, + ) -> pa.Table: + """Select a contiguous range of rows by position. + + Equivalent to ``dataset.take(list(range(start, end)))``, but pushed + down as an offset/limit scan instead of a materialized index list. + + Parameters + ---------- + start : int + The index of the first row to include (inclusive). Must be + non-negative. + end : int + The index to stop before (exclusive). Must be greater than or + equal to ``start``. + columns: list of str, or dict of str to str default None + List of column names to be fetched. + Or a dictionary of column names to SQL expressions. + All columns are fetched if None or unspecified. + + Returns + ------- + table : pyarrow.Table + + Examples + -------- + >>> import lance + >>> import pyarrow as pa + >>> tbl = pa.table({"id": range(100)}) + >>> dataset = lance.write_dataset(tbl, "memory://slice_dataset") + >>> dataset.slice(10, 20) + pyarrow.Table + id: int64 + ---- + id: [[10,11,12,13,14,15,16,17,18,19]] + """ + if start < 0: + raise ValueError(f"start must be non-negative, got {start}") + if end < start: + raise ValueError(f"end ({end}) must be >= start ({start})") + return self.scanner(offset=start, limit=end - start, columns=columns).to_table() + def count_rows( self, filter: Optional[Union[str, pa.compute.Expression]] = None, **kwargs ) -> int: @@ -2823,7 +3095,7 @@ def update( where = str(where) return self._ds.update(updates, where, conflict_retries, retry_timeout) - def versions(self): + def versions(self) -> List[Version]: """ Return all versions in this dataset. """ @@ -2838,6 +3110,15 @@ def versions(self): ) return versions + def version_refs(self) -> List[VersionRef]: + """ + Return lightweight references to all attached versions in the current branch. + + Unlike :meth:`versions`, this does not read or deserialize every manifest. + Use :attr:`latest_version` instead when only the latest version is needed. + """ + return self._ds.version_refs() + @property def version(self) -> int: """ @@ -2986,6 +3267,7 @@ def cleanup_old_versions( delete_unverified: bool = False, error_if_tagged_old_versions: bool = True, delete_rate_limit: Optional[int] = None, + versions: Optional[List[int]] = None, ) -> CleanupStats: """ Cleans up old versions of the dataset. @@ -3006,7 +3288,7 @@ def cleanup_old_versions( ``retain_versions`` are not specified, this will default to two weeks. retain_versions: int, optional - Retain the last N versions of the dataset. + Retain the last N versions of the dataset. Must be positive. delete_unverified: bool, default False Files leftover from a failed transaction may appear to be part of an @@ -3031,8 +3313,13 @@ def cleanup_old_versions( deletions run at full speed. Set this to a positive integer to avoid hitting object store request rate limits (e.g. S3 HTTP 503 SlowDown). For example, ``delete_rate_limit=100`` limits to 100 operations/second. + + versions: list[int], optional + Clean up only the specified dataset versions. The current version is + never removed, and tagged versions are still protected by + ``error_if_tagged_old_versions``. """ - if older_than is None and retain_versions is None: + if older_than is None and retain_versions is None and versions is None: older_than = timedelta(days=14) return self._ds.cleanup_old_versions( @@ -3041,6 +3328,7 @@ def cleanup_old_versions( delete_unverified, error_if_tagged_old_versions, delete_rate_limit, + versions, ) def explain_cleanup_old_versions( @@ -3051,6 +3339,7 @@ def explain_cleanup_old_versions( delete_unverified: bool = False, error_if_tagged_old_versions: bool = True, delete_rate_limit: Optional[int] = None, + versions: Optional[List[int]] = None, include_files: bool = False, max_files: int = 1000, ) -> CleanupExplanation: @@ -3065,7 +3354,7 @@ def explain_cleanup_old_versions( ``retain_versions`` are not specified, this will default to two weeks. retain_versions: int, optional - Retain the last N versions of the dataset. + Retain the last N versions of the dataset. Must be positive. delete_unverified: bool, default False Include unverified files that cleanup would remove when this is set. @@ -3078,6 +3367,9 @@ def explain_cleanup_old_versions( Accepted for parity with :meth:`cleanup_old_versions`; no deletes are issued by explain. + versions: list[int], optional + Explain cleanup only for the specified dataset versions. + include_files: bool, default False If `True`, include candidate files in the explanation up to ``max_files`` entries. Aggregate stats always include all candidates. @@ -3086,7 +3378,7 @@ def explain_cleanup_old_versions( Maximum number of candidate files to include when ``include_files`` is `True`. """ - if older_than is None and retain_versions is None: + if older_than is None and retain_versions is None and versions is None: older_than = timedelta(days=14) if max_files <= 0: raise ValueError("max_files must be positive") @@ -3097,6 +3389,7 @@ def explain_cleanup_old_versions( delete_unverified, error_if_tagged_old_versions, delete_rate_limit, + versions, include_files, max_files, ) @@ -3120,8 +3413,6 @@ def _prepare_scalar_index_request( column = column[0] lance_field = self._ds.lance_schema.field_case_insensitive(column) - if lance_field is None: - raise KeyError(f"{column} not found in schema") if isinstance(index_type, str): index_type = index_type.upper() @@ -3144,6 +3435,12 @@ def _prepare_scalar_index_request( ) ) + if lance_field is None: + if index_type in ["INVERTED", "FTS"]: + # Rust resolves public FTS paths through intervening list layers. + return column, index_type, index_type + raise KeyError(f"{column} not found in schema") + field = lance_field.to_arrow() field_type = field.type @@ -3151,23 +3448,32 @@ def _prepare_scalar_index_request( if hasattr(field_type, "storage_type"): field_type = field_type.storage_type - if index_type in ["BTREE", "BITMAP", "ZONEMAP"]: + if index_type in ["BTREE", "BITMAP"]: if ( not pa.types.is_integer(field_type) and not pa.types.is_floating(field_type) and not pa.types.is_boolean(field_type) and not pa.types.is_string(field_type) and not pa.types.is_large_string(field_type) + and not pa.types.is_binary(field_type) + and not pa.types.is_large_binary(field_type) and not pa.types.is_temporal(field_type) + and not pa.types.is_decimal128(field_type) + and not pa.types.is_decimal256(field_type) and not pa.types.is_fixed_size_binary(field_type) ): raise TypeError( - f"BTREE/BITMAP/ZONEMAP index column {column} must be int", - ", float, bool, str, large_str, fixed-size-binary, or temporal", + f"BTREE/BITMAP index column {column} must be int", + ", float, bool, str, large_str, binary, large_binary, " + "decimal, fixed-size-binary, or temporal", ) elif index_type == "LABEL_LIST": - if not pa.types.is_list(field_type): - raise TypeError(f"LABEL_LIST index column {column} must be a list") + if not ( + pa.types.is_list(field_type) or pa.types.is_large_list(field_type) + ): + raise TypeError( + f"LABEL_LIST index column {column} must be a list or large list" + ) elif index_type == "NGRAM": if not pa.types.is_string(field_type) and not pa.types.is_large_string( field_type @@ -3190,14 +3496,15 @@ def _prepare_scalar_index_request( f" or list of strings, or json, but got {value_type}" ) - if pa.types.is_duration(field_type): - raise TypeError( - f"Scalar index column {column} cannot currently be a duration" - ) return column, index_type, index_type elif isinstance(index_type, IndexConfig): logical_index_type = index_type.index_type.upper() - config = json.dumps(index_type.parameters) + if lance_field is None and logical_index_type not in ["INVERTED", "FTS"]: + raise KeyError(f"{column} not found in schema") + parameters = dict(index_type.parameters) + if logical_index_type in ["INVERTED", "FTS"]: + parameters["document_granularity"] = kwargs["document_granularity"] + config = json.dumps(parameters) kwargs["config"] = indices.IndexConfig(index_type.index_type, config) return column, "scalar", logical_index_type else: @@ -3221,7 +3528,11 @@ def _is_segment_native_scalar_index_type( "BITMAP", "INVERTED", "FTS", + "NGRAM", + "RTREE", "ZONEMAP", + "BLOOMFILTER", + "LABEL_LIST", } @classmethod @@ -3232,7 +3543,11 @@ def _requires_uncommitted_scalar_index( return cls._normalized_index_type(index_type) in { "BTREE", "BITMAP", + "NGRAM", + "RTREE", "ZONEMAP", + "BLOOMFILTER", + "LABEL_LIST", } def create_scalar_index( @@ -3258,6 +3573,7 @@ def create_scalar_index( index_uuid: Optional[str] = None, progress_callback: Optional[Callable[[IndexProgress], None]] = None, format_version: Optional[Union[int, str]] = None, + document_granularity: DocumentGranularity = DocumentGranularity.ROW, **kwargs, ): """Create a scalar index on a column. @@ -3334,7 +3650,8 @@ def create_scalar_index( ---------- column : str The column to be indexed. Must be a boolean, integer, float, - or string column. + string, binary, decimal, fixed-size-binary, or + supported temporal column. index_type : str The type of the index. One of ``"BTREE"``, ``"BITMAP"``, ``"LABEL_LIST"``, ``"NGRAM"``, ``"ZONEMAP"``, ``"INVERTED"``, @@ -3366,8 +3683,15 @@ def create_scalar_index( format_version: int or str, optional This is for the ``INVERTED`` / ``FTS`` index. Explicit on-disk FTS format version to write when creating a new index. Accepts ``1``, - ``2``, ``"v1"``, or ``"v2"``. If unset, Lance chooses the current - default format. + ``2``, ``3``, ``"v1"``, ``"v2"``, or ``"v3"``. + If unset, Lance uses ``LANCE_FTS_FORMAT_VERSION`` when present and + otherwise writes v2 for text analysis with ``block_size=128`` and + v3 for code analysis or ``block_size=256``. + + document_granularity: DocumentGranularity, default ROW + This is for the ``INVERTED`` / ``FTS`` index. ``ROW`` treats all + selected text in one dataset row as one document. ``LIST_ELEMENT`` + treats each element of the deepest list on ``column`` as one document. with_position: bool, default False This is for the ``INVERTED`` index. If True, the index will store the @@ -3375,6 +3699,12 @@ def create_scalar_index( query. This will significantly increase the index size. It won't impact the performance of non-phrase queries even if it is set to True. + block_size: int, default 128 + This is for the ``INVERTED`` index. Number of documents per compressed + posting block. Must be one of ``128`` or ``256``. + ``block_size=256`` is experimental and may introduce breaking changes. + Use ``128`` when stable compatibility with the legacy posting layout is + required. memory_limit: int, optional This is for the ``INVERTED`` index. Total build-time memory limit in MiB. If set, Lance divides this budget evenly across the workers. If unset, @@ -3396,6 +3726,7 @@ def create_scalar_index( * "simple": splits tokens on whitespace and punctuation. * "whitespace": splits tokens on whitespace. * "raw": no tokenization. + * "ngram": produces character N-grams for substring search. * "icu": ICU dictionary-based Unicode word segmentation. * "icu/split": ICU segmentation with simple-style delimiter splitting. language: str, default "English" @@ -3407,10 +3738,10 @@ def create_scalar_index( lower_case: bool, default True This is for the ``INVERTED`` index. If True, the index will convert all text to lowercase. - stem: bool, default True + stem: bool, default True (False for the "ngram" tokenizer) This is for the ``INVERTED`` index. If True, the index will stem the tokens. - remove_stop_words: bool, default True + remove_stop_words: bool, default True (False for the "ngram" tokenizer) This is for the ``INVERTED`` index. If True, the index will remove stop words. custom_stop_words: Optional[List[str]], default None @@ -3456,6 +3787,11 @@ def create_scalar_index( ``MaterializeIndex`` operator. """ + if not isinstance(document_granularity, DocumentGranularity): + raise TypeError( + "document_granularity must be a lance.query.DocumentGranularity" + ) + kwargs["document_granularity"] = document_granularity.value column, index_type, logical_index_type = self._prepare_scalar_index_request( column, index_type, kwargs ) @@ -3477,7 +3813,6 @@ def create_scalar_index( kwargs["progress_callback"] = progress_callback if format_version is not None: kwargs["format_version"] = format_version - self._ds.create_index([column], index_type, name, replace, train, None, kwargs) def _create_index_impl( @@ -4040,7 +4375,7 @@ def create_index( Optional parameters for `IVF_RQ`: - num_bits - The number of bits for RQ (Rabit Quantization). Default is 1. + The number of bits for RQ (Rabit Quantization). Default is 5. Optional parameters for `IVF_HNSW_*`: max_level @@ -4180,10 +4515,10 @@ def create_index_uncommitted( Create one segment without publishing it and return its metadata. This is the public distributed-build API for vector, BTREE scalar, - canonical bitmap scalar, INVERTED scalar, and ZONEMAP scalar index - construction. Unlike - :meth:`create_index`, this method does not publish the index into the - dataset manifest. Instead, it writes one segment under + canonical bitmap scalar, INVERTED scalar, NGRAM scalar, RTREE scalar, + ZONEMAP scalar, BLOOMFILTER scalar, and LABEL_LIST scalar index construction. + Unlike :meth:`create_index`, this method does not publish the index into + the dataset manifest. Instead, it writes one segment under ``_indices//`` and returns the resulting :class:`Index` metadata. @@ -4197,8 +4532,11 @@ def create_index_uncommitted( 4. commit the final segment list with :meth:`commit_existing_index_segments` - BTREE, BITMAP, INVERTED, and ZONEMAP segments may - be merged with :meth:`merge_existing_index_segments` before commit. + BTREE, BITMAP, INVERTED, NGRAM, RTREE, ZONEMAP, BLOOMFILTER, and + LABEL_LIST segments may be merged with + :meth:`merge_existing_index_segments` before commit. NGRAM segments + built before a deferred compaction must be merged before commit so + their postings can be rebuilt against current row addresses. Parameters are the same as :meth:`create_index`, with one additional requirement: @@ -4294,13 +4632,22 @@ def drop_index(self, name: str): """ return self._ds.drop_index(name) - def prewarm_index(self, name: str, *, with_position: bool = False): + def prewarm_index( + self, + name: str, + *, + with_position: bool = False, + index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None, + ): """ Prewarm an index - This will load the entire index into memory. This can help avoid cold start - issues with index queries. If the index does not fit in the index cache, then - this will result in wasted I/O. + By default, this will load the entire index into memory. This can help + avoid cold start issues with index queries. If the index does not fit in + the index cache, then this will result in wasted I/O. + + Use ``session().index_cache_size_bytes()`` before and after prewarm to + inspect how much the index cache grew. Parameters ---------- @@ -4310,8 +4657,16 @@ def prewarm_index(self, name: str, *, with_position: bool = False): This is only supported for ``INVERTED`` indices. If True, positions are also loaded into the cache during prewarm so phrase queries do not need a separate lazy positions read. - """ - return self._ds.prewarm_index(name, with_position=with_position) + index_segments: iterable of str or uuid.UUID, default None + If specified, prewarm only these physical index segment UUIDs from the + named logical index. Use :meth:`describe_indices` to inspect logical + indices and obtain segment UUIDs from ``IndexDescription.segments``. + """ + return self._ds.prewarm_index( + name, + with_position=with_position, + index_segments=_normalize_index_segment_ids(index_segments), + ) def merge_index_metadata( self, @@ -4986,6 +5341,31 @@ def drop( storage_options: Optional[Dict[str, str]] = None, ignore_not_found: Optional[bool] = None, ) -> None: + """Delete a dataset and everything under ``base_uri``. + + To limit the damage a mistyped or misconfigured path can do, ``base_uri`` + must be a dataset root, meaning it holds a manifest that can be read, or a + namespace declare/deregister marker. Anything else raises + :class:`ValueError`, including a path that holds only data files or only + unreadable manifests: such leftovers need an explicit storage-level delete. + + Note that a path which passes this check is deleted in full, including any + unmanaged files kept next to the dataset. + + Parameters + ---------- + base_uri : str or Path + Root of the dataset to delete. + storage_options : optional, dict + Extra options for the storage backend. + ignore_not_found : optional, bool + If True, return successfully when ``base_uri`` does not exist. + + Raises + ------ + ValueError + If ``base_uri`` is not a Lance dataset root. + """ _Dataset.drop(str(base_uri), storage_options, ignore_not_found=ignore_not_found) def get_ivf_model(self, index_name: str): @@ -5057,7 +5437,6 @@ def initialize_mem_wal( identity_column: Optional[str] = None, unsharded: bool = False, durable_write: Optional[bool] = None, - sync_indexed_write: Optional[bool] = None, max_wal_buffer_size: Optional[int] = None, max_wal_flush_interval_ms: Optional[int] = None, max_memtable_size: Optional[int] = None, @@ -5065,8 +5444,6 @@ def initialize_mem_wal( max_memtable_batches: Optional[int] = None, max_unflushed_memtable_bytes: Optional[int] = None, manifest_scan_batch_size: Optional[int] = None, - async_index_buffer_rows: Optional[int] = None, - async_index_interval_ms: Optional[int] = None, backpressure_log_interval_ms: Optional[int] = None, stats_log_interval_ms: Optional[int] = None, hnsw_params: Optional[Dict[str, Dict[str, int]]] = None, @@ -5127,7 +5504,6 @@ def initialize_mem_wal( identity_column=identity_column, unsharded=unsharded, durable_write=durable_write, - sync_indexed_write=sync_indexed_write, max_wal_buffer_size=max_wal_buffer_size, max_wal_flush_interval_ms=max_wal_flush_interval_ms, max_memtable_size=max_memtable_size, @@ -5135,8 +5511,6 @@ def initialize_mem_wal( max_memtable_batches=max_memtable_batches, max_unflushed_memtable_bytes=max_unflushed_memtable_bytes, manifest_scan_batch_size=manifest_scan_batch_size, - async_index_buffer_rows=async_index_buffer_rows, - async_index_interval_ms=async_index_interval_ms, backpressure_log_interval_ms=backpressure_log_interval_ms, stats_log_interval_ms=stats_log_interval_ms, hnsw_params=hnsw_params, @@ -5159,7 +5533,6 @@ def mem_wal_writer( shard_id: str, *, durable_write: Optional[bool] = None, - sync_indexed_write: Optional[bool] = None, max_wal_buffer_size: Optional[int] = None, max_wal_flush_interval_ms: Optional[int] = None, max_memtable_size: Optional[int] = None, @@ -5167,8 +5540,6 @@ def mem_wal_writer( max_memtable_batches: Optional[int] = None, max_unflushed_memtable_bytes: Optional[int] = None, manifest_scan_batch_size: Optional[int] = None, - async_index_buffer_rows: Optional[int] = None, - async_index_interval_ms: Optional[int] = None, backpressure_log_interval_ms: Optional[int] = None, stats_log_interval_ms: Optional[int] = None, hnsw_params: Optional[Dict[str, Dict[str, int]]] = None, @@ -5186,8 +5557,6 @@ def mem_wal_writer( ``str(uuid.uuid4())``). durable_write : bool, optional Whether to fsync WAL writes (default: ``True``). - sync_indexed_write : bool, optional - Whether index updates are synchronous (default: ``True``). max_wal_buffer_size : int, optional Maximum WAL buffer size in bytes (default: 10 MB). max_wal_flush_interval_ms : int, optional @@ -5202,10 +5571,6 @@ def mem_wal_writer( Maximum unflushed bytes before backpressure (default: 1 GB). manifest_scan_batch_size : int, optional Batch size for manifest scans (default: 2). - async_index_buffer_rows : int, optional - Buffer rows for async index updates (default: 10 000). - async_index_interval_ms : int, optional - Interval for async index updates in milliseconds (default: 1000). backpressure_log_interval_ms : int, optional Interval for backpressure log messages in milliseconds (default: 30 000). @@ -5253,7 +5618,6 @@ def mem_wal_writer( name: val for name, val in [ ("durable_write", durable_write), - ("sync_indexed_write", sync_indexed_write), ("max_wal_buffer_size", max_wal_buffer_size), ("max_wal_flush_interval_ms", max_wal_flush_interval_ms), ("max_memtable_size", max_memtable_size), @@ -5261,8 +5625,6 @@ def mem_wal_writer( ("max_memtable_batches", max_memtable_batches), ("max_unflushed_memtable_bytes", max_unflushed_memtable_bytes), ("manifest_scan_batch_size", manifest_scan_batch_size), - ("async_index_buffer_rows", async_index_buffer_rows), - ("async_index_interval_ms", async_index_interval_ms), ("backpressure_log_interval_ms", backpressure_log_interval_ms), ("stats_log_interval_ms", stats_log_interval_ms), ("hnsw_params", hnsw_params), @@ -5414,6 +5776,41 @@ def with_row_addr(self, with_row_addr: bool = True) -> "SqlQueryBuilder": self._builder = self._builder.with_row_addr(with_row_addr) return self + def blob_handling( + self, + blob_handling: Literal["all_binary", "blobs_descriptions", "all_descriptions"], + ) -> "SqlQueryBuilder": + """ + Control how blob columns are returned by this SQL query. + + - ``"all_binary"`` materializes blob columns as binary values. + - ``"blobs_descriptions"`` returns blob descriptors (the default). + - ``"all_descriptions"`` returns descriptions for all binary-like + columns. + """ + self._builder = self._builder.blob_handling(blob_handling) + return self + + def batch_size(self, batch_size: int) -> "SqlQueryBuilder": + """ + Set the maximum number of rows produced by each query batch. + + If :meth:`batch_size_bytes` is also set, both limits apply and the one + reached first determines the scan batch size. + """ + self._builder = self._builder.batch_size(batch_size) + return self + + def batch_size_bytes(self, batch_size_bytes: int) -> "SqlQueryBuilder": + """ + Set the approximate maximum bytes produced by each scan batch. + + If :meth:`batch_size` is also set, both limits apply and the one + reached first determines the scan batch size. + """ + self._builder = self._builder.batch_size_bytes(batch_size_bytes) + return self + def build(self) -> SqlQuery: """ Build the query. @@ -5455,6 +5852,14 @@ def get_updated_rows(self) -> pa.RecordBatchReader: """ return self._delta.get_updated_rows() + def get_deleted_row_ids(self) -> pa.RecordBatchReader: + """ + Return a streaming RecordBatchReader of the row ids deleted in the range. + + The batches carry a single ``_rowid`` column. Requires stable row ids. + """ + return self._delta.get_deleted_row_ids() + class _DatasetDeltaBuilder: """Internal builder for :class:`DatasetDelta`. @@ -5520,6 +5925,10 @@ class Version(TypedDict): metadata: Dict[str, str] +class VersionRef(TypedDict): + version: int + + class UpdateResult(TypedDict): num_rows_updated: int @@ -5563,6 +5972,7 @@ class Index: base_id: Optional[int] = None files: Optional[List["IndexFile"]] = None index_details: Optional[Tuple[str, bytes]] = None + covering_fields: List[int] = dataclasses.field(default_factory=list) class IndexInformation(TypedDict): @@ -5615,7 +6025,13 @@ class Overwrite(BaseOperation): new_schema: pyarrow.Schema The schema of the new dataset. fragments: list[FragmentMetadata] - The fragments that make up the new dataset. + The newly written fragments that make up the new dataset. They are + assigned fresh ids when the operation is committed, continuing from + the highest id the dataset has ever used, so any id they carry is + ignored. Since we reassign fragment ids, a fragment with a deletion + file is rejected: use :class:`LanceOperation.Delete` to commit + deletions, or :class:`LanceOperation.Merge` to change the schema of + existing fragments. initial_bases: list[DatasetBasePath], optional Base paths to register when creating a new dataset (CREATE mode only). **Only valid in CREATE mode**. Will raise an error if used with @@ -5786,8 +6202,15 @@ class Update(BaseOperation): The ids of the fragments that have been removed entirely. updated_fragments: list[FragmentMetadata] The fragments that have been updated with new deletion vectors. + These are used as given, so pass back the metadata read from the + dataset rather than a freshly constructed object, or the fragment + loses its row id and version metadata. new_fragments: list[FragmentMetadata] - The fragments that contain the new rows. + The fragments that contain the new rows. On a dataset that uses + stable row ids, set ``row_id_meta`` on these to carry the ids of + rewritten rows over; see :class:`lance.fragment.RowIdSequence`. The + created-at and last-updated-at version metadata are derived during + the commit and should be left as None. fields_modified: list[int] If any fields are modified in updated_fragments, then they must be listed here so those fragments can be removed from indices covering @@ -5795,6 +6218,11 @@ class Update(BaseOperation): fields_for_preserving_frag_bitmap: list[int] The fields that used to judge whether to preserve the new frag's id into the frag bitmap of the specified indices. + updated_fragment_offsets: dict[int, bytes], optional + Physical row offsets that matched the update, keyed by fragment id, + each serialized in the portable RoaringBitmap format. Set on + ``rewrite_columns`` updates over stable row ids so the commit + refreshes row-level version metadata for the matched rows only. """ removed_fragment_ids: List[int] = dataclasses.field(default_factory=list) @@ -5807,6 +6235,7 @@ class Update(BaseOperation): default_factory=list ) update_mode: str = "" + updated_fragment_offsets: Optional[Dict[int, bytes]] = None def __post_init__(self): LanceOperation._validate_fragments(self.updated_fragments) @@ -5825,6 +6254,14 @@ class Merge(BaseOperation): schema: LanceSchema or pyarrow.Schema The schema of the new dataset. Passing a LanceSchema is preferred, and passing a pyarrow.Schema is deprecated. + preserves_nullability: bool + True when this merge makes no nullability-affecting schema change: + it introduces no field that data staged against an earlier schema + could not safely omit. Without the assertion (the default) the + merge conservatively conflicts with concurrent appends, whose + fragments would omit new columns and read as null; that can only + cause a retry. Pass True when every column this merge introduces + is nullable to let concurrent appends commit without conflict. Warning ------- @@ -5870,6 +6307,7 @@ class Merge(BaseOperation): fragments: Iterable[FragmentMetadata] schema: LanceSchema | pa.Schema + preserves_nullability: bool = False def __post_init__(self): if isinstance(self.schema, pa.Schema): @@ -5962,6 +6400,76 @@ class DataReplacement(BaseOperation): replacements: List[LanceOperation.DataReplacementGroup] + @dataclass + class DataOverlayFile: + """ + An overlay file supplying new values for a subset of + ``(physical offset, field)`` cells of a fragment, resolved on read and + layered over the base data without rewriting the base files. + + The overlay is dense or sparse depending on the shape of ``offsets``: + pass a flat ``List[int]`` for a dense overlay (one offset list shared by + every field in ``data_file``) or a ``List[List[int]]`` for a sparse + overlay (one offset list per field, in the order of the file's fields). + Offsets are **physical** row offsets (positions in the base files, + counting deleted rows), like deletion vectors. + + Attributes + ---------- + data_file : DataFile + The Lance data file storing the overlay's new cell values — one + value column per covered field. The value at each covered offset is + stored at the rank (0-based count of covered offsets below it) of + that offset in the field's coverage. + offsets : Union[List[int], List[List[int]]] + The covered physical row offsets. A flat list is dense coverage + (shared by every field); a list of per-field lists is sparse + coverage (in field order). Each list must be strictly ascending + with no duplicates, since the Nth offset maps to the Nth value row + in ``data_file``; a non-ascending list raises ``ValueError``. + committed_version : Optional[int] + The dataset version at which this overlay became effective. Leave as + ``None`` when creating an overlay to commit — the commit stamps it. + It is populated when reading an existing fragment's overlays so they + round-trip through :class:`FragmentMetadata`. + """ + + data_file: DataFile + offsets: Union[List[int], List[List[int]]] + committed_version: Optional[int] = None + + @dataclass + class DataOverlayGroup: + """ + Overlay files to append to a single fragment. + + Attributes + ---------- + fragment_id : int + The id of the fragment the overlays apply to. + overlays : List[LanceOperation.DataOverlayFile] + The overlay files to append, ordered oldest-first (a later entry is + newer and wins where coverage overlaps). + """ + + fragment_id: int + overlays: List[LanceOperation.DataOverlayFile] + + @dataclass + class DataOverlay(BaseOperation): + """ + Operation that appends data overlay files to fragments. + + Overlays are appended to each fragment's existing overlays (overlays + written by concurrent commits are preserved) and resolved on read + over the base data without rewriting it. + + If multiple groups target the same data then the values in the + latest group take precedence. + """ + + groups: List[LanceOperation.DataOverlayGroup] + @dataclass class Project(BaseOperation): """ @@ -5972,6 +6480,11 @@ class Project(BaseOperation): ---------- schema: LanceSchema The lance schema of the new dataset. + preserves_nullability: bool + True when this projection makes no nullability-affecting schema + change, as a rename or a drop does not. Without the assertion + (the default) the projection conservatively conflicts with + concurrent writes, which can only cause a retry. Examples -------- @@ -6000,6 +6513,7 @@ class Project(BaseOperation): """ schema: LanceSchema + preserves_nullability: bool = False @dataclass class UpdateMap: @@ -6089,6 +6603,18 @@ def _needs_substrait_placeholder(t: pa.DataType) -> bool: return False +def serialize_row_addrs(addrs: Iterable[int]) -> bytes: + """Encode row addresses for ``row_addr_allowlist`` / ``row_addr_blocklist``. + + Those parameters take a serialized roaring treemap over ``_rowid``; this is + the way to produce one from Python. + + >>> blob = serialize_row_addrs([0, 2, 4]) # doctest: +SKIP + >>> ds.scanner(row_addr_allowlist=blob).to_table() # doctest: +SKIP + """ + return _serialize_row_addrs(list(addrs)) + + class ScannerBuilder: def __init__(self, ds: LanceDataset): self.ds = ds @@ -6097,6 +6623,8 @@ def __init__(self, ds: LanceDataset): self._search_filter = None self._substrait_filter = None self._prefilter = False + self._row_addr_allowlist: Optional[bytes] = None + self._row_addr_blocklist: Optional[bytes] = None self._late_materialization = None self._blob_handling = None self._offset = None @@ -6138,9 +6666,8 @@ def apply_defaults(self, default_opts: Dict[str, Any]) -> ScannerBuilder: def batch_size(self, batch_size: int) -> ScannerBuilder: """Set the maximum number of rows per batch. - Note: this can be overridden by ``batch_size_bytes`` or by a - dataset-level ``batch_size_bytes`` configured via - ``FileReaderOptions``. + If a byte limit is also configured, both limits apply and the one + reached first determines the batch size. """ self._batch_size = batch_size return self @@ -6149,7 +6676,8 @@ def batch_size_bytes(self, batch_size_bytes: int) -> ScannerBuilder: """Set the target batch size in bytes. When set, the scanner will produce batches whose total size in bytes - is approximately this value, overriding the row-based ``batch_size``. + is approximately this value. If ``batch_size`` is also set, both + limits apply and the one reached first determines the batch size. This can also be configured at the dataset level via ``FileReaderOptions``. A scanner-level setting takes precedence @@ -6182,10 +6710,12 @@ def io_buffer_size(self, io_buffer_size: int) -> ScannerBuilder: def batch_readahead(self, nbatches: Optional[int] = None) -> ScannerBuilder: """ - This parameter is ignored when reading v2 files + Set the maximum number of batches to decode concurrently. + + This parameter must be greater than zero. """ - if nbatches is not None and int(nbatches) < 0: - raise ValueError("batch_readahead must be non-negative") + if nbatches is not None and int(nbatches) <= 0: + raise ValueError("batch_readahead must be greater than 0") self._batch_readahead = nbatches return self @@ -6310,6 +6840,25 @@ def filter( return self + def row_addr_prefilter( + self, + allowlist: Optional[bytes] = None, + blocklist: Optional[bytes] = None, + ) -> ScannerBuilder: + """Restrict the scan to an externally supplied set of row addresses. + + allowlist / blocklist are serialized roaring treemaps over ``_rowid`` + (``RowAddrTreeMap::serialize_into`` output); passing neither clears the + mask. Applied before KNN / BM25 ranking, so top-k is computed over the + surviving rows rather than filtered afterwards. + + Bytes rather than an object so the mask can be produced by a different + extension module -- nothing Rust-typed crosses the boundary. + """ + self._row_addr_allowlist = allowlist + self._row_addr_blocklist = blocklist + return self + def prefilter(self, prefilter: bool) -> ScannerBuilder: self._prefilter = prefilter return self @@ -6394,19 +6943,7 @@ def with_fragments( def with_index_segments( self, index_segments: Optional[Iterable[Union[str, uuid.UUID]]] ) -> ScannerBuilder: - if index_segments is not None: - segment_ids = [] - for segment_id in index_segments: - if isinstance(segment_id, (str, uuid.UUID)): - segment_ids.append(str(segment_id)) - else: - raise TypeError( - "index_segments must be an iterable of str or uuid.UUID. " - f"Got {type(segment_id)} instead." - ) - index_segments = segment_ids - - self._index_segments = index_segments + self._index_segments = _normalize_index_segment_ids(index_segments) return self def nearest( @@ -6538,6 +7075,9 @@ def strict_batch_size(self, strict_batch_size: bool = False) -> ScannerBuilder: If this is true then small batches will need to be merged together which will require a data copy and incur a (typically very small) performance penalty. + + This cannot be combined with ``batch_size_bytes`` because merging + batches to the strict row count can exceed the byte limit. """ self._strict_batch_size = strict_batch_size return self @@ -6611,6 +7151,8 @@ def to_scanner(self) -> LanceScanner: self._orderings, self._disable_scoring_autoprojection, self._substrait_aggregate, + self._row_addr_allowlist, + self._row_addr_blocklist, ) return LanceScanner(scanner, self.ds, _snapshot_scanner_builder(self)) @@ -6767,7 +7309,7 @@ def head(self, num_rows): """ return self.to_table()[:num_rows] - def count_rows(self): + def count_rows(self) -> int: """Count rows matching the scanner filter. Returns @@ -6795,6 +7337,9 @@ def explain_plan(self, verbose=False) -> str: def analyze_plan(self, count_rows: bool = False) -> str: """Execute the plan for this scanner and display with runtime metrics. + Full-text-search nodes include the execution-time ``tokenized_query`` + text and positions. + Parameters ---------- count_rows : bool, default False @@ -6829,6 +7374,10 @@ def compact_files( Literal["reencode", "try_binary_copy", "force_binary_copy"] ] = None, binary_copy_read_batch_bytes: Optional[int] = None, + max_source_fragments: Optional[int] = None, + max_source_rows: Optional[int] = None, + max_source_bytes: Optional[int] = None, + excluded_fragment_ids: Optional[list[int]] = None, ) -> CompactionMetrics: """Compacts small files in the dataset, reducing total number of files. @@ -6859,7 +7408,10 @@ def compact_files( ``lance.compaction.defer_index_remap``, ``lance.compaction.batch_size``, ``lance.compaction.compaction_mode``, - ``lance.compaction.binary_copy_read_batch_bytes``. + ``lance.compaction.binary_copy_read_batch_bytes``, + ``lance.compaction.max_source_fragments``, + ``lance.compaction.max_source_rows``, + ``lance.compaction.max_source_bytes``. Parameters ---------- @@ -6912,6 +7464,29 @@ def compact_files( The batch size in bytes for reading during binary copy operations. Controls how much data is read at once when performing binary copy. Defaults to 16MB. + max_source_fragments: int, optional + Maximum number of source fragments to compact in a single run. + Compaction tasks are included until adding the next task would + exceed this limit, allowing compaction to proceed incrementally. + Fragments are processed oldest first. If not specified, uses the + manifest config value, or applies no limit. + max_source_rows: int, optional + Maximum number of source rows to compact in a single run. Rows are + counted as live rows (physical rows minus soft-deleted rows). + Tasks are included until adding the next task would exceed this + limit. + max_source_bytes: int, optional + Maximum number of source bytes to compact in a single run, + measured as the total size of the source fragments' data and + overlay files. Tasks are included until adding the next task + would exceed this limit. Blob v2 payloads live in separate + blob files and are not counted, so this is not a cap on total + compaction I/O for datasets with blob columns. + excluded_fragment_ids: list[int], optional + Fragment IDs to exclude from compaction planning. Excluded + fragments remain unchanged and act as boundaries, so fragments + on opposite sides are not combined into the same compaction task. + Duplicate and unknown IDs are ignored. Returns ------- @@ -6935,12 +7510,16 @@ def compact_files( batch_size=batch_size, compaction_mode=compaction_mode, binary_copy_read_batch_bytes=binary_copy_read_batch_bytes, + max_source_fragments=max_source_fragments, + max_source_rows=max_source_rows, + max_source_bytes=max_source_bytes, + excluded_fragment_ids=excluded_fragment_ids, ).items() if v is not None } return Compaction.execute(self._dataset, opts) - def optimize_indices(self, **kwargs): + def optimize_indices(self, **kwargs) -> None: """Optimizes index performance. As new data arrives it is not added to existing indexes automatically. @@ -6948,10 +7527,10 @@ def optimize_indices(self, **kwargs): an expensive unindexed search on the new data. As the amount of new unindexed data grows this can have an impact on search latency. This function will add the new data to existing indexes, restoring the - performance. This function does not retrain the index, it only assigns - the new data to existing partitions. This means an update is much quicker - than retraining the entire index but may have less accuracy (especially - if the new data exhibits new patterns, concepts, or trends) + performance. By default, this function does not retrain the index, it only + assigns the new data to existing partitions. This means an update is much + quicker than retraining the entire index but may have less accuracy + (especially if the new data exhibits new patterns, concepts, or trends) Parameters ---------- @@ -6961,7 +7540,7 @@ def optimize_indices(self, **kwargs): index_names: List[str], default None The names of the indices to optimize. If None, all indices will be optimized. - retrain: bool, default False, deprecated + retrain: bool, default False Whether to retrain the whole index. If true, the index will be retrained based on the current data, `num_indices_to_merge` will be ignored, @@ -6969,7 +7548,7 @@ def optimize_indices(self, **kwargs): This is useful when the data distribution has changed significantly, and we want to retrain the index to improve the search quality. - This would be faster than re-create the index from scratch. + This rebuilds the index from the source data and may be expensive. """ self._dataset._ds.optimize_indices(**kwargs) @@ -7018,7 +7597,7 @@ def list(self) -> dict[str, Tag]: """ return self._ds.tags() - def get_version(self, tag: str) -> Optional[int]: + def get_version(self, tag: str) -> int: """ Get the version of a specific tag by name. @@ -7029,8 +7608,13 @@ def get_version(self, tag: str) -> Optional[int]: Returns ------- - int or None - The version number of the tag if it exists, otherwise None. + int + The version number of the tag. + + Raises + ------ + ValueError + If the tag does not exist. Use :meth:`list` to check for presence. """ return self._ds.get_version(tag) @@ -7243,6 +7827,7 @@ def write_dataset( blob_pack_file_size_threshold: Optional[int] = None, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> LanceDataset: """Write a given data_obj to the given uri @@ -7502,6 +8087,7 @@ def write_dataset( "external_blob_mode": external_blob_mode, "allow_external_blob_outside_bases": allow_external_blob_outside_bases, "blob_pack_file_size_threshold": blob_pack_file_size_threshold, + "session": session, } # Add namespace_client and table_id for storage options provider and managed @@ -7564,8 +8150,7 @@ def _coerce_query_vector(query: QueryVectorLike) -> tuple[pa.Array, int]: if isinstance(query.type, pa.FixedSizeListType): query = query.values elif isinstance(query, (list, tuple)) or ( - _check_for_numpy(query), - isinstance(query, np.ndarray), + _check_for_numpy(query) and isinstance(query, np.ndarray) ): query = np.array(query).astype("float64") # workaround for GH-608 query = pa.FloatingPointArray.from_pandas(query, type=pa.float32()) diff --git a/python/python/lance/dependencies.py b/python/python/lance/dependencies.py index def9a052504..e1435e6fca1 100644 --- a/python/python/lance/dependencies.py +++ b/python/python/lance/dependencies.py @@ -27,7 +27,7 @@ _CAGRA_AVAILABLE = True _RAFT_COMMON_AVAILABLE = True _HUGGING_FACE_AVAILABLE = True -_TENSORFLOW_AVAILABLE = True +_PYDANTIC_AVAILABLE = True class _LazyModule(ModuleType): @@ -49,7 +49,6 @@ class _LazyModule(ModuleType): "pandas": "pd.", "polars": "pl.", "torch": "torch.", - "tensorflow": "tf.", } def __init__( @@ -163,7 +162,6 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]: import numpy import pandas import polars - import tensorflow # type: ignore[reportMissingImports] import torch # type: ignore[reportMissingImports] else: # heavy/optional third party libs @@ -172,7 +170,7 @@ def _lazy_import(module_name: str) -> tuple[ModuleType, bool]: polars, _POLARS_AVAILABLE = _lazy_import("polars") torch, _TORCH_AVAILABLE = _lazy_import("torch") datasets, _HUGGING_FACE_AVAILABLE = _lazy_import("datasets") - tensorflow, _TENSORFLOW_AVAILABLE = _lazy_import("tensorflow") + _, _PYDANTIC_AVAILABLE = _lazy_import("pydantic") @lru_cache(maxsize=None) @@ -215,33 +213,77 @@ def _check_for_hugging_face(obj: Any, *, check_type: bool = True) -> bool: ) -def _check_for_tensorflow(obj: Any, *, check_type: bool = True) -> bool: - return _TENSORFLOW_AVAILABLE and _might_be( - cast("Hashable", type(obj) if check_type else obj), "tensorflow" +def _check_for_pydantic(obj: Any, *, check_type: bool = True) -> bool: + return _PYDANTIC_AVAILABLE and _might_be( + cast("Hashable", type(obj) if check_type else obj), "pydantic" ) +def _is_pydantic_base_model(obj: Any) -> bool: + if not _PYDANTIC_AVAILABLE: + return False + from pydantic import BaseModel + + return isinstance(obj, BaseModel) + + +def _is_pydantic_base_model_class(obj: Any) -> bool: + if not _PYDANTIC_AVAILABLE: + return False + from pydantic import BaseModel + + return isinstance(obj, type) and issubclass(obj, BaseModel) + + +def model_to_dict(obj: Any) -> dict[str, Any]: + return obj.model_dump() if hasattr(obj, "model_dump") else obj.dict() + + +def _validate_pydantic_list(data: Any, model_class: type) -> None: + """Validate that `data` is a list of exact `model_class` instances. + + Rejects non-list iterables (e.g. generators), which would otherwise be + drained by this validation loop and leave nothing for the caller to + serialize, and rejects subclass instances, whose extra/overridden fields + would not match the schema derived from `model_class`. + """ + if not isinstance(data, list): + raise TypeError( + f"Pydantic model data must be provided as a list, got {type(data)!r}" + ) + for i, item in enumerate(data): + if type(item) is not model_class: + raise TypeError( + f"data[{i}] must be an instance of {model_class!r} exactly " + f"(subclasses are not accepted), got {type(item)!r} " + f"(data has {len(data)} items)" + ) + + __all__ = [ # lazy-load third party libs "datasets", "numpy", "pandas", "polars", - "tensorflow", "torch", # lazy utilities "_check_for_hugging_face", "_check_for_numpy", "_check_for_pandas", "_check_for_polars", - "_check_for_tensorflow", + "_check_for_pydantic", "_check_for_torch", + "_is_pydantic_base_model", + "_is_pydantic_base_model_class", "_LazyModule", + "model_to_dict", + "_validate_pydantic_list", # exported flags/guards "_NUMPY_AVAILABLE", "_PANDAS_AVAILABLE", "_POLARS_AVAILABLE", + "_PYDANTIC_AVAILABLE", "_TORCH_AVAILABLE", "_HUGGING_FACE_AVAILABLE", - "_TENSORFLOW_AVAILABLE", ] diff --git a/python/python/lance/file.py b/python/python/lance/file.py index 5926241977e..89210099ccf 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -482,6 +482,12 @@ class LanceFileWriter: This class is used to write Lance data files, a low level structure optimized for storing multi-modal tabular data. If you are working with Lance datasets then you should use the LanceDataset class instead. + + Attributes + ---------- + size_bytes: Optional[int] + The final size of the file in bytes. This is None until `close` is + called. """ def __init__( @@ -548,6 +554,7 @@ def __init__( **kwargs, ) self.closed = False + self.size_bytes: Optional[int] = None def write_batch(self, batch: Union[pa.RecordBatch, pa.Table]) -> None: """ @@ -569,11 +576,17 @@ def close(self) -> Optional[int]: Write the file metadata and close the file Returns the number of rows written to the file + + After this returns, ``size_bytes`` holds the final size of the file. This + is reported by the writer itself, so it is available for object stores + without issuing a separate metadata request. """ if self.closed: return self.closed = True - return self._writer.finish() + summary = self._writer.finish() + self.size_bytes = summary.size_bytes + return summary.num_rows def add_schema_metadata(self, key: str, value: str) -> None: """ diff --git a/python/python/lance/fragment.py b/python/python/lance/fragment.py index adf220e59f6..5d8c3364bdf 100644 --- a/python/python/lance/fragment.py +++ b/python/python/lance/fragment.py @@ -35,7 +35,11 @@ from .lance import ( RowIdMeta as RowIdMeta, ) +from .lance import ( + RowIdSequence as RowIdSequence, +) from .lance import _Fragment, _write_fragments, _write_fragments_transaction +from .lance import _Session as Session from .progress import FragmentWriteProgress, NoopFragmentWriteProgress from .types import _coerce_reader from .udf import BatchUDF, normalize_transform @@ -45,6 +49,7 @@ ColumnOrdering, DatasetBasePath, LanceDataset, + LanceOperation, LanceScanner, ReaderLike, Transaction, @@ -73,11 +78,26 @@ class FragmentMetadata: deletion_file : Optional[DeletionFile] The deletion file, if any. row_id_meta : Optional[RowIdMeta] - The row id metadata, if any. + The stable row ids of this fragment's rows, if any. When committing a + transaction by hand on a dataset that uses stable row ids, set this to + carry the ids of rewritten rows over to their new fragment; build it with + :class:`RowIdSequence`. Rows left without an id are treated as newly + inserted and are assigned ids during the commit. created_at_version_meta : Optional[RowDatasetVersionMeta] - The row created at version metadata, if any. + The dataset version each row was created in. Derived during the commit + from ``row_id_meta`` -- a rewritten row keeps the version it first + appeared in -- so leave this as None when building a transaction. Any + value set here is ignored for newly written fragments. last_updated_at_version_meta : Optional[RowDatasetVersionMeta] - The row last updated at version metadata, if any. + The dataset version each row was last modified in. Derived during the + commit, like ``created_at_version_meta``; leave this as None. It cannot + be computed ahead of time because a commit that loses a race is retried + against a later version than the one it was built for. + overlays : List[LanceOperation.DataOverlayFile] + The data overlay files layered over this fragment's base data, if any. + Overlays are created via :class:`LanceOperation.DataOverlay`; they are + carried here so they survive operations that round-trip fragment + metadata (e.g. a manual ``Delete``, ``Update``, or ``Merge`` commit). """ id: int @@ -87,6 +107,7 @@ class FragmentMetadata: row_id_meta: Optional[RowIdMeta] = None created_at_version_meta: Optional[RowDatasetVersionMeta] = None last_updated_at_version_meta: Optional[RowDatasetVersionMeta] = None + overlays: List["LanceOperation.DataOverlayFile"] = field(default_factory=list) @property def num_deletions(self) -> int: @@ -110,12 +131,25 @@ def data_files(self) -> List[DataFile]: def to_json(self) -> dict: """Get this as a simple JSON-serializable dictionary.""" - files = [asdict(f) for f in self.files] - for f in files: - f["path"] = f.pop("_path") + + def _data_file_to_json(f: DataFile) -> dict: + d = asdict(f) + d["path"] = d.pop("_path") + return d + + files = [_data_file_to_json(f) for f in self.files] + overlays = [ + dict( + data_file=_data_file_to_json(o.data_file), + offsets=o.offsets, + committed_version=o.committed_version, + ) + for o in self.overlays + ] return dict( id=self.id, files=files, + overlays=overlays, physical_rows=self.physical_rows, deletion_file=( self.deletion_file.asdict() if self.deletion_file is not None else None @@ -159,6 +193,20 @@ def from_json(json_data: str) -> FragmentMetadata: json.dumps(last_updated_at_version_meta) ) + overlays = [] + overlays_json = json_data.get("overlays") + if overlays_json: + from .dataset import LanceOperation + + overlays = [ + LanceOperation.DataOverlayFile( + data_file=DataFile(**o["data_file"]), + offsets=o["offsets"], + committed_version=o.get("committed_version"), + ) + for o in overlays_json + ] + return FragmentMetadata( id=json_data["id"], files=[DataFile(**f) for f in json_data["files"]], @@ -167,6 +215,7 @@ def from_json(json_data: str) -> FragmentMetadata: row_id_meta=row_id_meta, created_at_version_meta=created_at_version_meta, last_updated_at_version_meta=last_updated_at_version_meta, + overlays=overlays, ) @@ -304,10 +353,7 @@ def __repr__(self): return self._fragment.__repr__() def __reduce__(self): - from .dataset import LanceDataset - - ds = LanceDataset(self._ds.uri, self._ds.version) - return LanceFragment, (ds, self.fragment_id) + return LanceFragment, (self._ds, self.fragment_id) @staticmethod def create_from_file( @@ -340,7 +386,7 @@ def create( data: ReaderLike, fragment_id: Optional[int] = None, schema: Optional[pa.Schema] = None, - max_rows_per_group: int = 1024, + max_rows_per_group: Optional[int] = 1024, progress: Optional[FragmentWriteProgress] = None, mode: str = "append", *, @@ -349,6 +395,7 @@ def create( storage_options: Optional[Dict[str, str]] = None, namespace_client: Optional["LanceNamespace"] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> FragmentMetadata: """Create a :class:`FragmentMetadata` from the given data. @@ -369,8 +416,9 @@ def create( schema: pa.Schema, optional The schema of the data. If not specified, the schema will be inferred from the data. - max_rows_per_group: int, default 1024 - The maximum number of rows per group in the data file. + max_rows_per_group: int, optional, default 1024 + The maximum number of rows per group in the data file. ``None`` + leaves the writer default in place. progress: FragmentWriteProgress, optional *Experimental API*. Progress tracking for writing the fragment. Pass a custom class that defines hooks to be called when each fragment is @@ -397,6 +445,9 @@ def create( table_id : optional, List[str] The table identifier when using a namespace (e.g., ["my_table"]). Must be provided together with `namespace_client`. + session : optional, Session + A session to reuse across operations. The session holds shared + caches (metadata and index) and the object store registry. See Also -------- @@ -450,6 +501,7 @@ def create( storage_options=storage_options, namespace_client=namespace_client, table_id=table_id, + session=session, ) @property @@ -480,6 +532,16 @@ def physical_rows(self) -> int: """ return self._fragment.physical_rows + def validate(self) -> None: + """ + Validate the fragment. + + This checks the integrity of the fragment and will raise an exception if + the fragment is corrupted. Unlike :meth:`lance.LanceDataset.validate`, + which checks every fragment, this validates only this fragment. + """ + self._fragment.validate() + @property def physical_schema(self) -> pa.Schema: # override the pyarrow super class method otherwise causes segfault @@ -508,6 +570,12 @@ def scanner( Literal["all_binary", "blobs_descriptions", "all_descriptions"] ] = None, order_by: Optional[List[ColumnOrdering]] = None, + use_scalar_index: Optional[bool] = None, + io_buffer_size: Optional[int] = None, + late_materialization: Optional[bool | List[str]] = None, + include_deleted_rows: Optional[bool] = None, + batch_size_bytes: Optional[int] = None, + strict_batch_size: Optional[bool] = None, ) -> "LanceScanner": """See Dataset::scanner for details""" filter_str = str(filter) if filter is not None else None @@ -529,6 +597,12 @@ def scanner( batch_readahead=batch_readahead, blob_handling=blob_handling, order_by=order_by, + use_scalar_index=use_scalar_index, + io_buffer_size=io_buffer_size, + late_materialization=late_materialization, + include_deleted_rows=include_deleted_rows, + batch_size_bytes=batch_size_bytes, + strict_batch_size=strict_batch_size, **columns_arg, ) from .dataset import LanceScanner @@ -539,7 +613,7 @@ def scanner( "_search_filter": None, "_substrait_filter": None, "_prefilter": False, - "_late_materialization": None, + "_late_materialization": late_materialization, "_blob_handling": blob_handling, "_offset": offset, "_columns": tuple(columns) if isinstance(columns, list) else None, @@ -548,7 +622,8 @@ def scanner( ), "_nearest": None, "_batch_size": batch_size, - "_io_buffer_size": None, + "_batch_size_bytes": batch_size_bytes, + "_io_buffer_size": io_buffer_size, "_batch_readahead": batch_readahead, "_fragment_readahead": None, "_scan_in_order": True, @@ -558,10 +633,12 @@ def scanner( "_use_stats": True, "_fast_search": False, "_full_text_query": None, - "_use_scalar_index": None, - "_include_deleted_rows": None, + "_use_scalar_index": use_scalar_index, + "_include_deleted_rows": include_deleted_rows, "_scan_stats_callback": None, - "_strict_batch_size": False, + "_strict_batch_size": ( + strict_batch_size if strict_batch_size is not None else False + ), "_orderings": tuple(order_by) if order_by is not None else None, "_disable_scoring_autoprojection": False, "_substrait_aggregate": None, @@ -612,6 +689,12 @@ def to_batches( Literal["all_binary", "blobs_descriptions", "all_descriptions"] ] = None, order_by: Optional[List[ColumnOrdering]] = None, + use_scalar_index: Optional[bool] = None, + io_buffer_size: Optional[int] = None, + late_materialization: Optional[bool | List[str]] = None, + include_deleted_rows: Optional[bool] = None, + batch_size_bytes: Optional[int] = None, + strict_batch_size: Optional[bool] = None, ) -> Iterator[pa.RecordBatch]: return self.scanner( columns=columns, @@ -624,6 +707,12 @@ def to_batches( batch_readahead=batch_readahead, blob_handling=blob_handling, order_by=order_by, + use_scalar_index=use_scalar_index, + io_buffer_size=io_buffer_size, + late_materialization=late_materialization, + include_deleted_rows=include_deleted_rows, + batch_size_bytes=batch_size_bytes, + strict_batch_size=strict_batch_size, ).to_batches() def to_table( @@ -638,6 +727,12 @@ def to_table( Literal["all_binary", "blobs_descriptions", "all_descriptions"] ] = None, order_by: Optional[List[ColumnOrdering]] = None, + use_scalar_index: Optional[bool] = None, + io_buffer_size: Optional[int] = None, + late_materialization: Optional[bool | List[str]] = None, + include_deleted_rows: Optional[bool] = None, + batch_size_bytes: Optional[int] = None, + strict_batch_size: Optional[bool] = None, ) -> pa.Table: return self.scanner( columns=columns, @@ -648,6 +743,12 @@ def to_table( with_row_address=with_row_address, blob_handling=blob_handling, order_by=order_by, + use_scalar_index=use_scalar_index, + io_buffer_size=io_buffer_size, + late_materialization=late_materialization, + include_deleted_rows=include_deleted_rows, + batch_size_bytes=batch_size_bytes, + strict_batch_size=strict_batch_size, ).to_table() def to_pandas( @@ -990,12 +1091,12 @@ def schema(self) -> pa.Schema: return self._fragment.schema() - def data_files(self): + def data_files(self) -> List[DataFile]: """Return the data files of this fragment.""" return self._fragment.data_files() - def deletion_file(self): + def deletion_file(self) -> Optional[str]: """Return the deletion file, if any""" return self._fragment.deletion_file() @@ -1021,7 +1122,7 @@ def write_fragments( return_transaction: Literal[True], mode: str = "append", max_rows_per_file: int = 1024 * 1024, - max_rows_per_group: int = 1024, + max_rows_per_group: Optional[int] = 1024, max_bytes_per_file: int = DEFAULT_MAX_BYTES_PER_FILE, progress: Optional[FragmentWriteProgress] = None, data_storage_version: Optional[str] = None, @@ -1036,6 +1137,7 @@ def write_fragments( allow_external_blob_outside_bases: bool = False, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> Transaction: ... @overload @@ -1047,7 +1149,7 @@ def write_fragments( return_transaction: Literal[False] = False, mode: str = "append", max_rows_per_file: int = 1024 * 1024, - max_rows_per_group: int = 1024, + max_rows_per_group: Optional[int] = 1024, max_bytes_per_file: int = DEFAULT_MAX_BYTES_PER_FILE, progress: Optional[FragmentWriteProgress] = None, data_storage_version: Optional[str] = None, @@ -1062,6 +1164,7 @@ def write_fragments( allow_external_blob_outside_bases: bool = False, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> List[FragmentMetadata]: ... @@ -1073,7 +1176,7 @@ def write_fragments( return_transaction: bool = False, mode: str = "append", max_rows_per_file: int = 1024 * 1024, - max_rows_per_group: int = 1024, + max_rows_per_group: Optional[int] = 1024, max_bytes_per_file: int = DEFAULT_MAX_BYTES_PER_FILE, progress: Optional[FragmentWriteProgress] = None, data_storage_version: Optional[str] = None, @@ -1088,6 +1191,7 @@ def write_fragments( allow_external_blob_outside_bases: bool = False, namespace_client: Optional[LanceNamespace] = None, table_id: Optional[List[str]] = None, + session: Optional[Session] = None, ) -> List[FragmentMetadata] | Transaction: """ Write data into one or more fragments. @@ -1114,8 +1218,9 @@ def write_fragments( "overwrite" to assign new field ids to the schema. max_rows_per_file : int, default 1024 * 1024 The maximum number of rows per data file. - max_rows_per_group : int, default 1024 - The maximum number of rows per group in the data file. + max_rows_per_group : int, optional, default 1024 + The maximum number of rows per group in the data file. ``None`` leaves + the writer default in place. max_bytes_per_file : int, default 90 * 1024 * 1024 * 1024 The max number of bytes to write before starting a new file. This is a soft limit. This limit is checked after each group is written, which @@ -1196,6 +1301,9 @@ def write_fragments( table_id : optional, List[str] The table identifier when using a namespace (e.g., ["my_table"]). Must be provided together with `namespace_client`. + session : optional, Session + A session to reuse across operations. The session holds shared caches + (metadata and index) and the object store registry. Returns ------- @@ -1232,6 +1340,12 @@ def write_fragments( base_store_params = dataset_uri._base_store_params if storage_options is None: storage_options = dataset_uri._storage_options + if session is not None and not session.is_same_as(dataset_uri.session()): + raise ValueError( + "The provided session is not the destination dataset's own " + "session. Please pass the dataset's session or omit the " + "'session' parameter." + ) dataset_uri = dataset_uri._ds elif not isinstance(dataset_uri, str): raise TypeError(f"Unknown dataset_uri type {type(dataset_uri)}") @@ -1267,6 +1381,7 @@ def write_fragments( base_store_params=base_store_params, external_blob_mode=external_blob_mode, allow_external_blob_outside_bases=allow_external_blob_outside_bases, + session=session, ) diff --git a/python/python/lance/indices/builder.py b/python/python/lance/indices/builder.py index 6059166d6ba..e235f348979 100644 --- a/python/python/lance/indices/builder.py +++ b/python/python/lance/indices/builder.py @@ -164,6 +164,7 @@ def train_pq( *, sample_rate: int = 256, max_iters: int = 50, + num_bits: int = 8, fragment_ids: Optional[list[int]] = None, ) -> PqModel: """ @@ -195,6 +196,8 @@ def train_pq( This parameter is used in the same way as in the IVF model. max_iters: int This parameter is used in the same way as in the IVF model. + num_bits: int + The number of bits used to encode each PQ centroid. fragment_ids: list[int], optional If provided, train using only the specified fragments from the dataset. """ @@ -202,7 +205,7 @@ def train_pq( num_rows = self._count_rows(fragment_ids) num_subvectors = self._normalize_pq_params(num_subvectors, self.dimension) - self._verify_pq_sample_rate(num_rows, sample_rate) + self._verify_pq_sample_rate(num_rows, sample_rate, num_bits) distance_type = ivf_model.distance_type pq_codebook = indices.train_pq_model( self.dataset._ds, @@ -214,8 +217,9 @@ def train_pq( max_iters, ivf_model.centroids, fragment_ids, + num_bits, ) - return PqModel(num_subvectors, pq_codebook) + return PqModel(num_subvectors, pq_codebook, num_bits=num_bits) def prepare_global_ivf_pq( self, @@ -226,6 +230,7 @@ def prepare_global_ivf_pq( accelerator: Optional[Union[str, "torch.Device"]] = None, sample_rate: int = 256, max_iters: int = 50, + num_bits: int = 8, fragment_ids: Optional[list[int]] = None, ) -> dict: """ @@ -267,6 +272,7 @@ def prepare_global_ivf_pq( num_subvectors, sample_rate=sample_rate, max_iters=max_iters, + num_bits=num_bits, fragment_ids=fragment_ids, ) @@ -381,6 +387,7 @@ def transform_vectors( dest_uri, fragments, partition_ds_uri, + pq.num_bits, ) def shuffle_transformed_vectors( @@ -471,6 +478,7 @@ def load_shuffled_vectors( num_subvectors, distance_type, index_name, + pq.num_bits, ) else: raise ValueError("filenames must be a list of strings") @@ -526,13 +534,17 @@ def _verify_base_sample_rate(self, sample_rate: int): f"The sample_rate must be an int greater than 1, got {sample_rate}" ) - def _verify_pq_sample_rate(self, num_rows: int, sample_rate: int): + def _verify_pq_sample_rate( + self, num_rows: int, sample_rate: int, num_bits: int = 8 + ): self._verify_base_sample_rate(sample_rate) - if 256 * sample_rate > num_rows: + required_rows = (2**num_bits) * sample_rate + if required_rows > num_rows: raise ValueError( "There are not enough rows in the dataset to create PQ" - f" codebook with a sample rate of {sample_rate}. {sample_rate * 256}" - f" rows needed and there are {num_rows}" + f" codebook with a sample rate of {sample_rate} and num_bits" + f" of {num_bits}. {required_rows} rows needed and there are" + f" {num_rows}" ) def _verify_ivf_sample_rate( diff --git a/python/python/lance/indices/pq.py b/python/python/lance/indices/pq.py index b3aeb50bcbe..e3d334ccf48 100644 --- a/python/python/lance/indices/pq.py +++ b/python/python/lance/indices/pq.py @@ -14,9 +14,13 @@ class PqModel: Can be saved / loaded to checkpoint progress. """ - def __init__(self, num_subvectors: int, codebook: pa.FixedSizeListArray): + def __init__( + self, num_subvectors: int, codebook: pa.FixedSizeListArray, *, num_bits: int = 8 + ): self.num_subvectors = num_subvectors """The number of subvectors to divide source vectors into""" + self.num_bits = num_bits + """The number of bits used to encode each PQ centroid""" self.codebook = codebook """The centroids of the PQ clusters""" @@ -42,7 +46,10 @@ def save(self, uri: str, *, storage_options: Optional[Dict[str, str]] = None): uri, pa.schema( [pa.field("codebook", self.codebook.type)], - metadata={b"num_subvectors": str(self.num_subvectors).encode()}, + metadata={ + b"num_subvectors": str(self.num_subvectors).encode(), + b"num_bits": str(self.num_bits).encode(), + }, ), storage_options=storage_options, ) as writer: @@ -65,9 +72,10 @@ def load(cls, uri: str, *, storage_options: Optional[Dict[str, str]] = None): """ reader = LanceFileReader(uri, storage_options=storage_options) num_rows = reader.metadata().num_rows - metadata = reader.metadata().schema.metadata + metadata = reader.metadata().schema.metadata or {} num_subvectors = int(metadata[b"num_subvectors"].decode()) + num_bits = int(metadata.get(b"num_bits", b"8").decode()) codebook = ( reader.read_all(batch_size=num_rows).to_table().column("codebook").chunk(0) ) - return cls(num_subvectors, codebook) + return cls(num_subvectors, codebook, num_bits=num_bits) diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi index f050c4c8422..91621bd9b49 100644 --- a/python/python/lance/lance/__init__.pyi +++ b/python/python/lance/lance/__init__.pyi @@ -44,6 +44,7 @@ from ..dataset import ( Transaction, UpdateResult, Version, + VersionRef, ) from ..fragment import ( DataFile, @@ -63,6 +64,9 @@ from .fragment import ( from .fragment import ( RowIdMeta as RowIdMeta, ) +from .fragment import ( + RowIdSequence as RowIdSequence, +) from .indices import IndexDescription as IndexDescription from .indices import IndexSegment as IndexSegment from .lance import PySearchFilter @@ -87,6 +91,56 @@ from .trace import capture_trace_events as capture_trace_events from .trace import shutdown_tracing as shutdown_tracing from .trace import trace_to_chrome as trace_to_chrome +class MetricPoint: + name: str + kind: str + attributes: Dict[str, str] + value: Optional[float] + buckets: Optional[List[Tuple[str, int]]] + count: Optional[int] + sum: Optional[float] + +class MetricDescription: + name: str + kind: str + unit: Optional[str] + description: str + +def register_lance_metrics_recorder() -> bool: ... +def lance_metrics_catalog() -> List[MetricDescription]: ... +def snapshot_lance_metrics() -> List[MetricPoint]: ... + +class FtsToken: + text: str + position: int + def __repr__(self) -> str: ... + +def tokenize( + query: str, + *, + analyzer: Optional[Literal["text", "code"]] = None, + base_tokenizer: Optional[str] = None, + language: Optional[str] = None, + max_token_length: Optional[int] = 40, + lower_case: Optional[bool] = None, + stem: Optional[bool] = None, + remove_stop_words: Optional[bool] = None, + custom_stop_words: Optional[List[str]] = None, + ascii_folding: Optional[bool] = None, + min_ngram_length: Optional[int] = None, + max_ngram_length: Optional[int] = None, + prefix_only: Optional[bool] = None, + split_identifiers: Optional[bool] = None, + split_on_numerics: Optional[bool] = None, + preserve_original: Optional[bool] = None, + index_operators: Optional[bool] = None, +) -> List[FtsToken]: + """Tokenize an FTS query without an index. + + ``max_token_length`` defaults to 40; pass ``None`` to disable the limit. + """ + ... + class CleanupStats: bytes_removed: int old_versions: int @@ -115,6 +169,12 @@ class CleanupExplanation: referenced_branches: List[CleanupReferencedBranch] warnings: List[str] +class LanceFileWriteSummary: + num_rows: int + size_bytes: int + + def __repr__(self) -> str: ... + class LanceFileWriter: def __init__( self, @@ -129,7 +189,7 @@ class LanceFileWriter: max_page_bytes: Optional[int], ): ... def write_batch(self, batch: pa.RecordBatch) -> None: ... - def finish(self) -> int: ... + def finish(self) -> LanceFileWriteSummary: ... def add_schema_metadata(self, key: str, value: str) -> None: ... def add_global_buffer(self, data: bytes) -> int: ... @@ -141,8 +201,21 @@ class PackedBlobWriter: def blob_id(self) -> int: ... @property def path(self) -> str: ... + @property + def field(self) -> pa.Field: ... def write_blob(self, data: bytes) -> None: ... + def write_blobs( + self, + payloads: Union[ + pa.BinaryArray, + pa.LargeBinaryArray, + pa.BinaryViewArray, + pa.FixedSizeBinaryArray, + pa.ChunkedArray, + ], + ) -> None: ... def finish(self) -> List[BlobDescriptor]: ... + def finish_array(self, field_name: str) -> pa.StructArray: ... class DedicatedBlobWriter: @property @@ -252,7 +325,25 @@ class LanceColumnStatistics: size_bytes: int class _Session: + def __init__( + self, + index_cache_size_bytes: Optional[int] = None, + metadata_cache_size_bytes: Optional[int] = None, + index_cache_backend: Optional[str | Dict[str, Any]] = None, + metadata_cache_backend: Optional[str | Dict[str, Any]] = None, + ) -> None: + """Create a Lance session. + + Cache backends may be backend URI strings such as + ``"moka://?capacity=1048576"`` or dictionaries such as + ``{"kind": "moka", "options": {"capacity": "1048576"}}``. + ``index_cache_backend`` is mutually exclusive with + ``index_cache_size_bytes``. ``metadata_cache_backend`` is mutually + exclusive with ``metadata_cache_size_bytes``. + """ + ... def size_bytes(self) -> int: ... + def index_cache_size_bytes(self) -> int: ... class LanceBlobFile: def close(self): ... @@ -261,6 +352,8 @@ class LanceBlobFile: def tell(self) -> int: ... def size(self) -> int: ... def readall(self) -> bytes: ... + def read_range(self, offset: int, length: int) -> bytes: ... + def read_ranges(self, ranges: List[Tuple[int, int]]) -> List[bytes]: ... def read_into(self, b: bytearray) -> int: ... class _Dataset: @@ -344,38 +437,46 @@ class _Dataset: self, row_ids: List[int], blob_column: str, - ) -> List[LanceBlobFile]: ... + ) -> List[Optional[LanceBlobFile]]: ... def take_blobs_by_addresses( self, row_addresses: List[int], blob_column: str, - ) -> List[LanceBlobFile]: ... + ) -> List[Optional[LanceBlobFile]]: ... def take_blobs_by_indices( self, row_indices: List[int], blob_column: str, - ) -> List[LanceBlobFile]: ... + ) -> List[Optional[LanceBlobFile]]: ... def read_blobs( self, row_ids: List[int], blob_column: str, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: ... + ) -> List[Tuple[int, Optional[bytes]]]: ... def read_blobs_by_addresses( self, row_addresses: List[int], blob_column: str, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: ... + ) -> List[Tuple[int, Optional[bytes]]]: ... def read_blobs_by_indices( self, row_indices: List[int], blob_column: str, io_buffer_size: Optional[int] = None, preserve_order: Optional[bool] = None, - ) -> List[Tuple[int, bytes]]: ... + ) -> List[Tuple[int, Optional[bytes]]]: ... + def read_blob_ranges( + self, + requests: List[Tuple[int, int, int]], + blob_column: str, + selector: Literal["ids", "addresses", "indices"], + io_buffer_size: Optional[int] = None, + preserve_order: Optional[bool] = None, + ) -> List[Tuple[int, int, Optional[bytes]]]: ... def take_scan( self, row_slices: Iterable[Tuple[int, int]], @@ -392,6 +493,7 @@ class _Dataset: ) -> UpdateResult: ... def count_deleted_rows(self) -> int: ... def versions(self) -> List[Version]: ... + def version_refs(self) -> List[VersionRef]: ... def version(self) -> int: ... def latest_version(self) -> int: ... def checkout_version( @@ -470,7 +572,13 @@ class _Dataset: kwargs: Optional[Dict[str, Any]] = None, ): ... def drop_index(self, name: str): ... - def prewarm_index(self, name: str, *, with_position: bool = False): ... + def prewarm_index( + self, + name: str, + *, + with_position: bool = False, + index_segments: Optional[List[str]] = None, + ): ... def merge_index_metadata( self, index_uuid: str, @@ -542,8 +650,13 @@ class _Dataset: index_name: str, partition_id: int, hamming_threshold: int, + index_segments: Optional[List[str]] = None, ) -> pa.RecordBatchReader: ... - def get_ivf_partition_info(self, index_name: str) -> List[dict]: ... + def get_ivf_partition_info( + self, + index_name: str, + index_segments: Optional[List[str]] = None, + ) -> List[dict]: ... def hamming_clustering_for_sample( self, column: str, @@ -565,9 +678,19 @@ class _MergeInsertBuilder: def when_matched_fail(self) -> Self: ... def when_not_matched_insert_all(self) -> Self: ... def when_not_matched_by_source_delete(self, expr: Optional[str] = None) -> Self: ... + def write_mode( + self, mode: Literal["auto", "rewrite_rows", "rewrite_columns"] + ) -> Self: ... def target_bases(self, bases: list[str]) -> Self: ... def target_all_bases(self, include_primary: bool = True) -> Self: ... def execute(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ... + def execute_batches(self, new_data: pa.RecordBatchReader) -> ExecuteResult: ... + def execute_uncommitted( + self, new_data: pa.RecordBatchReader + ) -> tuple[Transaction, ExecuteResult]: ... + def execute_uncommitted_batches( + self, new_data: pa.RecordBatchReader + ) -> tuple[Transaction, ExecuteResult]: ... class _Scanner: @property @@ -612,6 +735,12 @@ class _Fragment: batch_readahead: Optional[int] = None, blob_handling: Optional[str] = None, order_by: Optional[List[Any]] = None, + use_scalar_index: Optional[bool] = None, + io_buffer_size: Optional[int] = None, + late_materialization: Optional[bool | List[str]] = None, + include_deleted_rows: Optional[bool] = None, + batch_size_bytes: Optional[int] = None, + strict_batch_size: Optional[bool] = None, ) -> _Scanner: ... def add_columns_from_reader( self, @@ -633,6 +762,7 @@ class _Fragment: def physical_rows(self) -> int: ... @property def num_deletions(self) -> int: ... + def validate(self) -> None: ... def iops_counter() -> int: ... def bytes_read_counter() -> int: ... @@ -659,6 +789,7 @@ def _write_fragments( base_store_params: Optional[Dict[str, Dict[str, str]]] = None, external_blob_mode: Literal["reference", "ingest"] = "reference", allow_external_blob_outside_bases: bool = False, + session: Optional[_Session] = None, ): ... def _write_fragments_transaction( dataset_uri: str | Path | _Dataset, @@ -679,6 +810,7 @@ def _write_fragments_transaction( base_store_params: Optional[Dict[str, Dict[str, str]]] = None, external_blob_mode: Literal["reference", "ingest"] = "reference", allow_external_blob_outside_bases: bool = False, + session: Optional[_Session] = None, ) -> Transaction: ... def _json_to_schema(schema_json: str) -> pa.Schema: ... def _schema_to_json(schema: pa.Schema) -> str: ... @@ -690,7 +822,7 @@ def _evaluate_sharding_spec( schema: LanceSchema, ) -> pa.RecordBatch: ... -class _MergedGeneration: +class _CompactedSsTable: shard_id: str generation: int def __init__(self, shard_id: str, generation: int) -> None: ... @@ -700,11 +832,12 @@ class _ShardSnapshot: def __init__(self, shard_id: str) -> None: ... def with_spec_id(self, spec_id: int) -> Self: ... def with_current_generation(self, generation: int) -> Self: ... - def with_flushed_generation(self, generation: int, path: str) -> Self: ... + def with_sstable(self, generation: int, path: str) -> Self: ... class _ShardWriter: shard_id: str def put(self, data: Any) -> None: ... + def delete(self, keys: Any) -> None: ... def close(self) -> None: ... def stats(self) -> Dict[str, Any]: ... def memtable_stats(self) -> Dict[str, Any]: ... @@ -838,6 +971,31 @@ class ScanStatistics: indices_loaded: int parts_loaded: int index_comparisons: int + index_cache_hits: int + """Number of index cache page lookups where the loader was not executed + in this scan. Counts both true cache hits on already-populated entries + and coalesced concurrent loads (a follower attached to another caller's + in-flight load). + + Instrumented boundaries in this release: BTree, IVF v2 (write-cache scan + path), inverted posting list (grouped and per-token) and its per-token + metadata, inverted phrase positions, bitmap (Equals / Range / IsIn), + ngram, rtree. + + Caveats: + + * IVF v2 streaming scans and legacy v1 IVF partitions bypass the cache + by design and are therefore reported as a miss on every call. + * A cold posting-list lookup on the grouped inverted layout can record + up to two misses (group + per-token metadata) for a single term. + + Uninstrumented paths (HNSW graph pages, quantizer codebooks) do not + contribute to either counter.""" + index_cache_misses: int + """Number of index cache page lookups where the loader ran (the page was + not resident and had to be materialised, typically from storage). See + the sibling ``index_cache_hits`` for the paired counter and the list of + instrumented boundaries.""" all_counts: Dict[ str, int ] # Additional metrics for debugging purposes. Subject to change. diff --git a/python/python/lance/lance/fragment.pyi b/python/python/lance/lance/fragment.pyi index 6e80f847620..c92d59b1657 100644 --- a/python/python/lance/lance/fragment.pyi +++ b/python/python/lance/lance/fragment.pyi @@ -1,7 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -from typing import Literal, Optional +from typing import Iterable, Iterator, Literal, Optional, Union + +import pyarrow as pa class DeletionFile: """ @@ -120,3 +122,86 @@ class RowIdMeta: ... def __reduce__(self) -> tuple: ... + +class RowIdSequence: + """ + The stable row ids of the rows in a single fragment, in fragment order. + + Use this to attach pre-existing row ids to a fragment when assembling a + transaction manually, so that rewritten rows keep their identity:: + + sequence = RowIdSequence([7, 12]) + fragment = FragmentMetadata(..., row_id_meta=sequence.to_inline_metadata()) + + The sequence may be shorter than the fragment's ``physical_rows``. The ids + bind to the leading rows and the commit generates new ids for the remaining + ones, which is how a fragment holding both rewritten and newly inserted rows + is expressed: write the rewritten rows first and supply only their ids. + Passing more ids than the fragment has rows is rejected. + + Warning + ------- + Only duplicates within this sequence are rejected. Row ids must also be + unique across the dataset, and Lance does not re-check that when + committing, so the caller owns it. Supply only row ids that already exist + and are being relocated by the same transaction, which must also remove + every earlier occurrence of them. Do not generate ids for new rows yourself + -- they come from a counter in the manifest that a concurrent commit can + advance, so only the commit knows which values are free. Do not supply + unused row ids either: a sequence covering all of a fragment's rows leaves + the dataset's row id allocator untouched, so a later append will hand the + same id out again. + + Parameters + ---------- + row_ids : range | pa.Array | pa.ChunkedArray | Iterable[int] + The row ids, in the order the corresponding rows appear in the + fragment. A ``range`` with a step of one is stored compactly without + materializing its values. + """ + + def __init__( + self, row_ids: Union[range, pa.Array, pa.ChunkedArray, Iterable[int]] + ) -> None: ... + @staticmethod + def from_inline_metadata(metadata: RowIdMeta) -> RowIdSequence: + """ + Read back a sequence stored inline in fragment row id metadata. + + Parameters + ---------- + metadata : RowIdMeta + Row id metadata holding an inline sequence. Metadata pointing at an + external file is not supported. + + Returns + ------- + RowIdSequence + """ + ... + + def to_inline_metadata(self) -> RowIdMeta: + """ + Encode the sequence as row id metadata to store inline in the manifest. + + Returns + ------- + RowIdMeta + Suitable for the ``row_id_meta`` argument of + :class:`lance.fragment.FragmentMetadata`. + """ + ... + + def to_pyarrow(self) -> pa.UInt64Array: + """ + Get the row ids as a ``uint64`` array, in sequence order. + + Returns + ------- + pa.UInt64Array + """ + ... + + def __len__(self) -> int: ... + def __iter__(self) -> Iterator[int]: ... + def __reduce__(self) -> tuple: ... diff --git a/python/python/lance/lance/indices/__init__.pyi b/python/python/lance/lance/indices/__init__.pyi index 0f5db7037df..f4b0bf69592 100644 --- a/python/python/lance/lance/indices/__init__.pyi +++ b/python/python/lance/lance/indices/__init__.pyi @@ -17,6 +17,8 @@ from typing import Optional import pyarrow as pa +from .. import _Fragment + class IndexConfig: index_type: str config: str @@ -37,7 +39,7 @@ def train_ivf_model( sample_rate: int, max_iters: int, fragment_ids: Optional[list[int]] = None, -) -> pa.Array: ... +) -> pa.FixedSizeListArray: ... def train_pq_model( dataset, column: str, @@ -46,9 +48,12 @@ def train_pq_model( distance_type: str, sample_rate: int, max_iters: int, + # Kept as the ``Array`` base type: callers pass ``IvfModel.centroids``, + # which the public ``IvfModel`` constructor accepts as a plain ``pa.Array``. ivf_model: pa.Array, fragment_ids: Optional[list[int]] = None, -) -> pa.Array: ... + num_bits: int = 8, +) -> pa.FixedSizeListArray: ... def transform_vectors( dataset, column: str, @@ -58,10 +63,13 @@ def transform_vectors( ivf_centroids: pa.Array, pq_codebook: pa.Array, dst_uri: str, + fragments: list[_Fragment], + partitions_ds_uri: Optional[str] = None, + num_bits: int = 8, ): ... def build_rq_model( dimension: int, - num_bits: int = 1, + num_bits: int = 5, dtype: str = "float32", ) -> str: ... @@ -73,6 +81,7 @@ class IndexSegmentDescription: created_at: Optional[datetime] size_bytes: Optional[int] base_id: Optional[int] + covering_fields: list[int] def __repr__(self) -> str: ... diff --git a/python/python/lance/lance/schema.pyi b/python/python/lance/lance/schema.pyi index 76d3ad972f4..49d5229a721 100644 --- a/python/python/lance/lance/schema.pyi +++ b/python/python/lance/lance/schema.pyi @@ -15,7 +15,11 @@ class LanceField: def unenforced_clustering_key_position(self) -> Optional[int]: ... class LanceSchema: + @staticmethod + def _from_protos(metadata_json: str, *field_protos: bytes) -> "LanceSchema": ... def fields(self) -> List[LanceField]: ... + def field(self, name: str) -> Optional[LanceField]: ... + def field_case_insensitive(self, name: str) -> Optional[LanceField]: ... def unenforced_primary_key(self) -> List[LanceField]: ... def unenforced_clustering_key(self) -> List[LanceField]: ... def to_pyarrow(self) -> pa.Schema: ... diff --git a/python/python/lance/mem_wal.py b/python/python/lance/mem_wal.py index f87e811f830..b9a8890853a 100644 --- a/python/python/lance/mem_wal.py +++ b/python/python/lance/mem_wal.py @@ -9,8 +9,8 @@ 1. **WAL** – append-only durable log (raw writes) 2. **Active MemTable** – in-memory write buffer -3. **Flushed MemTable** – Lance files written to object store -4. **Base table** – canonical Lance dataset files (after merge_insert) +3. **SSTable** – Lance files written to object store +4. **Base table** – canonical Lance dataset files (after SSTable compaction) """ from __future__ import annotations @@ -22,12 +22,12 @@ import pyarrow as pa from .lance import ( + _CompactedSsTable, _evaluate_sharding_spec, _ExecutionPlan, _LsmPointLookupPlanner, _LsmScanner, _LsmVectorSearchPlanner, - _MergedGeneration, _ShardSnapshot, _ShardWriter, ) @@ -40,7 +40,7 @@ "ShardingField", "ShardingSpec", "evaluate_sharding_spec", - "MergedGeneration", + "CompactedSsTable", "ShardSnapshot", "ShardWriter", "LsmScanner", @@ -124,19 +124,19 @@ def _sharding_spec_to_dict(spec: Union[ShardingSpec, Mapping[str, object]]) -> d @dataclass -class MergedGeneration: - """Identifies a flushed MemWAL generation that has been merged. +class CompactedSsTable: + """Points to an SSTable compacted into the base table. - Pass a list of these to mark_generations_as_merged - so Lance knows which generations are now in the base table. + Pass a list of these to mark_sstables_as_compacted + so Lance can record compaction progress. Parameters ---------- shard_id : str UUID string for the write shard. generation : int - Generation number (from - :attr:`ShardSnapshot.flushed_generations`). + Generation number of the compacted SSTable (as passed to + :meth:`ShardSnapshot.with_sstable`). """ shard_id: str @@ -170,9 +170,9 @@ def with_current_generation(self, generation: int) -> "ShardSnapshot": self._raw = self._raw.with_current_generation(generation) return self - def with_flushed_generation(self, generation: int, path: str) -> "ShardSnapshot": - """Add a flushed generation with its storage path.""" - self._raw = self._raw.with_flushed_generation(generation, path) + def with_sstable(self, generation: int, path: str) -> "ShardSnapshot": + """Add an SSTable with its storage path.""" + self._raw = self._raw.with_sstable(generation, path) return self def __repr__(self) -> str: @@ -187,6 +187,7 @@ class ShardWriter: with dataset.mem_wal_writer(shard_id) as writer: writer.put(batch) + writer.delete(pa.table({"id": [1]})) Parameters ---------- @@ -224,6 +225,36 @@ def put(self, data, *, schema: Optional[pa.Schema] = None) -> None: reader = _coerce_reader(data, schema) self._raw.put(reader) + def delete(self, keys, *, schema: Optional[pa.Schema] = None) -> None: + """Delete rows by primary key from the MemWAL. + + Parameters + ---------- + keys : ReaderLike + Any Arrow-compatible data containing this shard's primary key + column(s). Non-primary-key columns, if present, are ignored by + the Rust core delete path. + schema : pa.Schema, optional + Schema hint, needed when *keys* is a generator. + + Raises + ------ + IOError + If delete validation fails, WAL flush fails, or the writer has + already been closed. Delete validation is centralized in Rust and + includes checks for primary-key metadata and tombstone-compatible + nullable non-key columns. + + Examples + -------- + :: + + with dataset.mem_wal_writer(shard_id) as writer: + writer.delete(pa.table({"id": [42]})) + """ + reader = _coerce_reader(keys, schema) + self._raw.delete(reader) + def close(self) -> None: """Flush and close the writer. @@ -252,7 +283,7 @@ def memtable_stats(self) -> dict: ------- dict Keys: ``row_count``, ``batch_count``, ``estimated_size_bytes``, - ``generation``. + ``generation``, ``frozen_count``, ``frozen_bytes``. """ return self._raw.memtable_stats() @@ -261,7 +292,7 @@ def lsm_scanner( ) -> "LsmScanner": """Create an LSM scanner that includes the active MemTable. - This scanner covers the base table, the given flushed generations, + This scanner covers the base table, the given SSTables, and the current active MemTable — providing strong read-your-writes consistency. @@ -290,10 +321,10 @@ class LsmScanner: """LSM-aware scanner covering all data levels. Deduplicates by primary key, always returning the newest version of - each row across base table, flushed MemTables, and the active MemTable. + each row across base table, SSTables, and the active MemTable. Obtain an instance from `ShardWriter.lsm_scanner` (includes - active MemTable) or `LsmScanner.from_snapshots` (flushed only). + active MemTable) or `LsmScanner.from_snapshots` (SSTables only). The builder methods (`project`, `filter`, `limit`) return ``self`` for chaining. @@ -323,7 +354,7 @@ def from_snapshots( dataset : LanceDataset The base dataset to scan. shard_snapshots : list of ShardSnapshot - Shard snapshots specifying flushed generations to include. + Shard snapshots specifying SSTables to include. """ raw = _LsmScanner.from_snapshots(dataset._ds, [s._raw for s in shard_snapshots]) return LsmScanner(raw) @@ -425,7 +456,7 @@ class LsmPointLookupPlanner: dataset : LanceDataset The base dataset. shard_snapshots : list of ShardSnapshot - Shard snapshots specifying flushed generations to include. + Shard snapshots specifying SSTables to include. pk_columns : list of str, optional Primary key column names. Inferred from schema metadata if omitted. @@ -484,7 +515,7 @@ class LsmVectorSearchPlanner: dataset : LanceDataset The base dataset. shard_snapshots : list of ShardSnapshot - Shard snapshots specifying flushed generations to include. + Shard snapshots specifying SSTables to include. vector_column : str Name of the ``FixedSizeList`` vector column. pk_columns : list of str, optional @@ -596,8 +627,8 @@ def _unwrap_shard_id(shard_id: str) -> str: return shard_id -def _to_raw_merged_generations( - generations: Iterable[MergedGeneration], +def _to_raw_compacted_sstables( + sstables: Iterable[CompactedSsTable], ) -> list: - """Convert Python MergedGeneration list to PyO3 _MergedGeneration list.""" - return [_MergedGeneration(g.shard_id, g.generation) for g in generations] + """Convert Python CompactedSsTable list to PyO3 _CompactedSsTable list.""" + return [_CompactedSsTable(s.shard_id, s.generation) for s in sstables] diff --git a/python/python/lance/namespace.py b/python/python/lance/namespace.py index fec3a1cfb1e..23c6b88cb3c 100644 --- a/python/python/lance/namespace.py +++ b/python/python/lance/namespace.py @@ -28,6 +28,7 @@ AlterTransactionResponse, AnalyzeTableQueryPlanRequest, CountTableRowsRequest, + CountTableRowsResponse, CreateMaterializedViewRequest, CreateMaterializedViewResponse, CreateNamespaceRequest, @@ -87,6 +88,8 @@ MergeInsertIntoTableRequest, MergeInsertIntoTableResponse, NamespaceExistsRequest, + NamespaceExistsResponse, + QueryTableResponse, RefreshMaterializedViewRequest, RefreshMaterializedViewResponse, RegisterTableRequest, @@ -96,6 +99,7 @@ RestoreTableRequest, RestoreTableResponse, TableExistsRequest, + TableExistsResponse, UpdateTableRequest, UpdateTableResponse, UpdateTableSchemaMetadataRequest, @@ -342,12 +346,13 @@ class DirectoryNamespace(LanceNamespace): >>> >>> # With AWS credential vending (requires credential-vendor-aws feature) >>> # Use **dict to pass property names with dots - >>> ns = lance.namespace.DirectoryNamespace(**{ + >>> aws_properties = { ... "root": "s3://my-bucket/data", ... "credential_vendor.enabled": "true", ... "credential_vendor.aws_role_arn": "arn:aws:iam::123456789012:role/MyRole", ... "credential_vendor.aws_duration_millis": "3600000", - ... }) + ... } + >>> # ns = lance.namespace.DirectoryNamespace(**aws_properties) With dynamic context provider: @@ -412,8 +417,11 @@ def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse response_dict = self._inner.drop_namespace(request.model_dump()) return DropNamespaceResponse.from_dict(response_dict) - def namespace_exists(self, request: NamespaceExistsRequest) -> None: + def namespace_exists( + self, request: NamespaceExistsRequest + ) -> NamespaceExistsResponse: self._inner.namespace_exists(request.model_dump()) + return NamespaceExistsResponse() # Table operations @@ -429,8 +437,9 @@ def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse response_dict = self._inner.register_table(request.model_dump()) return RegisterTableResponse.from_dict(response_dict) - def table_exists(self, request: TableExistsRequest) -> None: + def table_exists(self, request: TableExistsRequest) -> TableExistsResponse: self._inner.table_exists(request.model_dump()) + return TableExistsResponse() def drop_table(self, request: DropTableRequest) -> DropTableResponse: response_dict = self._inner.drop_table(request.model_dump()) @@ -522,7 +531,9 @@ def batch_delete_table_versions(self, request: dict) -> dict: # Data manipulation operations - def count_table_rows(self, request: CountTableRowsRequest) -> int: + def count_table_rows( + self, request: CountTableRowsRequest + ) -> CountTableRowsResponse: """Count the number of rows in a table, optionally filtered by a predicate. Parameters @@ -532,10 +543,11 @@ def count_table_rows(self, request: CountTableRowsRequest) -> int: Returns ------- - int - The number of rows matching the criteria + CountTableRowsResponse + Response whose ``count`` is the number of rows matching the criteria """ - return self._inner.count_table_rows(request.model_dump()) + count = self._inner.count_table_rows(request.model_dump()) + return CountTableRowsResponse(count=count) def insert_into_table( self, request: InsertIntoTableRequest, request_data: bytes @@ -615,7 +627,7 @@ def delete_from_table( response_dict = self._inner.delete_from_table(request.model_dump()) return DeleteFromTableResponse.from_dict(response_dict) - def query_table(self, request) -> bytes: + def query_table(self, request) -> QueryTableResponse: """Query a table and return results as Arrow IPC. Parameters @@ -626,12 +638,13 @@ def query_table(self, request) -> bytes: Returns ------- - bytes - Arrow IPC file format containing the query results + QueryTableResponse + Response whose ``data`` is the query results in Arrow IPC file format """ if hasattr(request, "model_dump"): request = request.model_dump() - return self._inner.query_table(request) + data = self._inner.query_table(request) + return QueryTableResponse(data=data) # Index operations @@ -1003,8 +1016,11 @@ def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse response_dict = self._inner.drop_namespace(request.model_dump()) return DropNamespaceResponse.from_dict(response_dict) - def namespace_exists(self, request: NamespaceExistsRequest) -> None: + def namespace_exists( + self, request: NamespaceExistsRequest + ) -> NamespaceExistsResponse: self._inner.namespace_exists(request.model_dump()) + return NamespaceExistsResponse() # Table operations @@ -1020,8 +1036,9 @@ def register_table(self, request: RegisterTableRequest) -> RegisterTableResponse response_dict = self._inner.register_table(request.model_dump()) return RegisterTableResponse.from_dict(response_dict) - def table_exists(self, request: TableExistsRequest) -> None: + def table_exists(self, request: TableExistsRequest) -> TableExistsResponse: self._inner.table_exists(request.model_dump()) + return TableExistsResponse() def drop_table(self, request: DropTableRequest) -> DropTableResponse: response_dict = self._inner.drop_table(request.model_dump()) @@ -1113,7 +1130,9 @@ def batch_delete_table_versions(self, request: dict) -> dict: # Data manipulation operations - def count_table_rows(self, request: CountTableRowsRequest) -> int: + def count_table_rows( + self, request: CountTableRowsRequest + ) -> CountTableRowsResponse: """Count the number of rows in a table, optionally filtered by a predicate. Parameters @@ -1123,10 +1142,11 @@ def count_table_rows(self, request: CountTableRowsRequest) -> int: Returns ------- - int - The number of rows matching the criteria + CountTableRowsResponse + Response whose ``count`` is the number of rows matching the criteria """ - return self._inner.count_table_rows(request.model_dump()) + count = self._inner.count_table_rows(request.model_dump()) + return CountTableRowsResponse(count=count) def insert_into_table( self, request: InsertIntoTableRequest, request_data: bytes @@ -1206,7 +1226,7 @@ def delete_from_table( response_dict = self._inner.delete_from_table(request.model_dump()) return DeleteFromTableResponse.from_dict(response_dict) - def query_table(self, request) -> bytes: + def query_table(self, request) -> QueryTableResponse: """Query a table and return results as Arrow IPC. Parameters @@ -1217,12 +1237,13 @@ def query_table(self, request) -> bytes: Returns ------- - bytes - Arrow IPC file format containing the query results + QueryTableResponse + Response whose ``data`` is the query results in Arrow IPC file format """ if hasattr(request, "model_dump"): request = request.model_dump() - return self._inner.query_table(request) + data = self._inner.query_table(request) + return QueryTableResponse(data=data) # Index operations diff --git a/python/python/lance/optimize.py b/python/python/lance/optimize.py index 3ac7547960b..4d45171b71f 100644 --- a/python/python/lance/optimize.py +++ b/python/python/lance/optimize.py @@ -97,3 +97,27 @@ class CompactionOptions(TypedDict): time). Fragments are processed oldest first. (default: None, no limit) """ + max_source_rows: Optional[int] + """ + Maximum number of source rows to compact in a single run. Rows are + counted as live rows (physical rows minus soft-deleted rows). Tasks + are included until adding the next task would exceed this limit. + (default: None, no limit) + """ + max_source_bytes: Optional[int] + """ + Maximum number of source bytes to compact in a single run, measured as + the total size of the source fragments' data and overlay files. Tasks + are included until adding the next task would exceed this limit. + Blob v2 payloads live in separate blob files and are not counted, so + this is not a cap on total compaction I/O for datasets with blob + columns. + (default: None, no limit) + """ + excluded_fragment_ids: Optional[list[int]] + """ + Fragment IDs to exclude from compaction planning. Excluded fragments + remain unchanged and act as boundaries, so fragments on opposite sides + are not combined into the same task. Duplicate and unknown IDs are + ignored. (default: None) + """ diff --git a/python/python/lance/otel.py b/python/python/lance/otel.py new file mode 100644 index 00000000000..0c9ee1790fd --- /dev/null +++ b/python/python/lance/otel.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Bridge Lance's internal metrics into OpenTelemetry. + +Lance core publishes metrics (currently object store request counts, bytes, +latency, errors, and throttles) through the Rust ``metrics`` facade. This module +installs a process-global recorder that aggregates them and registers +OpenTelemetry observable instruments that report the aggregated values into the +user's ``MeterProvider``. + +The bridge is generic: every metric Lance describes is surfaced automatically, +with no per-metric Python code. Histograms have no asynchronous OpenTelemetry +instrument, so each is exported Prometheus-style as cumulative ``le`` buckets +plus ``_count`` and ``_sum`` observable counters. +""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING, Optional + +from .lance import ( + lance_metrics_catalog, + register_lance_metrics_recorder, + snapshot_lance_metrics, +) + +if TYPE_CHECKING: + from opentelemetry.metrics import MeterProvider + +_INSTRUMENTED = False + + +def instrument_lance_metrics(meter_provider: Optional["MeterProvider"] = None) -> bool: + """Register Lance metrics as OpenTelemetry observable instruments. + + Installs a process-global metrics recorder and creates one observable + instrument per Lance metric on the given (or global) ``MeterProvider``. The + user's configured ``MetricReader`` then collects them on its own schedule. + + Counters and gauges map directly to observable counters/gauges. Each + histogram is exported as cumulative ``le`` bucket counts (``_bucket``, + with an ``le`` attribute) plus ``_count`` and ``_sum``. + + Parameters + ---------- + meter_provider : opentelemetry.metrics.MeterProvider, optional + The provider to register instruments on. Defaults to the global provider + from ``opentelemetry.metrics.get_meter_provider()``. + + Returns + ------- + bool + ``True`` if the recorder is installed and instruments are registered. + ``False`` if a different ``metrics`` recorder is already installed in + this process (``metrics`` permits only one global recorder), in which + case a warning is emitted and no instruments are created. + + Notes + ----- + Requires the OpenTelemetry SDK (``pip install pylance[otel]``). Calling this + more than once is safe; instruments are created only on the first successful + call. + """ + global _INSTRUMENTED + + try: + from opentelemetry.metrics import Observation, get_meter_provider + except ImportError as exc: + raise ImportError( + "instrument_lance_metrics requires the OpenTelemetry API/SDK. " + "Install it with `pip install pylance[otel]` or " + "`pip install opentelemetry-sdk`." + ) from exc + + if not register_lance_metrics_recorder(): + warnings.warn( + "Could not install the Lance metrics recorder: another `metrics` " + "recorder is already installed in this process. Lance metrics will " + "not be exported via OpenTelemetry.", + stacklevel=2, + ) + return False + + if _INSTRUMENTED: + return True + + provider = meter_provider or get_meter_provider() + meter = provider.get_meter("lance") + + def scalar_callback(metric_name: str): + def callback(_options): + return [ + Observation(point.value, point.attributes) + for point in snapshot_lance_metrics() + if point.name == metric_name and point.value is not None + ] + + return callback + + def bucket_callback(metric_name: str): + def callback(_options): + observations = [] + for point in snapshot_lance_metrics(): + if point.name != metric_name or point.buckets is None: + continue + for le, cumulative in point.buckets: + attributes = dict(point.attributes) + attributes["le"] = le + observations.append(Observation(cumulative, attributes)) + return observations + + return callback + + def field_callback(metric_name: str, field: str): + def callback(_options): + observations = [] + for point in snapshot_lance_metrics(): + if point.name != metric_name: + continue + value = getattr(point, field) + if value is not None: + observations.append(Observation(value, point.attributes)) + return observations + + return callback + + for desc in lance_metrics_catalog(): + unit = desc.unit or "" + if desc.kind == "counter": + meter.create_observable_counter( + desc.name, + callbacks=[scalar_callback(desc.name)], + unit=unit, + description=desc.description, + ) + elif desc.kind == "gauge": + meter.create_observable_gauge( + desc.name, + callbacks=[scalar_callback(desc.name)], + unit=unit, + description=desc.description, + ) + elif desc.kind == "histogram": + # `_bucket` and `_count` observe cumulative counts, so they are + # unitless; only `_sum` carries the histogram's unit (e.g. seconds). + meter.create_observable_counter( + f"{desc.name}_bucket", + callbacks=[bucket_callback(desc.name)], + description=f"{desc.description} (cumulative buckets)", + ) + meter.create_observable_counter( + f"{desc.name}_count", + callbacks=[field_callback(desc.name, "count")], + description=f"{desc.description} (count)", + ) + meter.create_observable_counter( + f"{desc.name}_sum", + callbacks=[field_callback(desc.name, "sum")], + unit=unit, + description=f"{desc.description} (sum)", + ) + + _INSTRUMENTED = True + return True diff --git a/python/python/lance/pydantic.py b/python/python/lance/pydantic.py new file mode 100644 index 00000000000..99226fcf664 --- /dev/null +++ b/python/python/lance/pydantic.py @@ -0,0 +1,434 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors +# +# Pydantic-to-Arrow conversion, ported from +# https://github.com/lancedb/lancedb/blob/f8dc2f78ee3219084d457a647e230b23e2f391b0/python/python/lancedb/pydantic.py +# +# Embedding-function-specific code (LanceModel, parse_embedding_functions, +# EmbeddingFunctionRegistry) is intentionally excluded -- lance has no concept +# of embedding functions, that stays in lancedb. Version detection uses +# hasattr checks against the field/model objects instead of branching on +# pydantic's version number, since pydantic is an optional dependency here. + +"""Pydantic (v1 / v2) to Arrow schema conversion""" + +from __future__ import annotations + +import inspect +import sys +import types +from abc import ABC, abstractmethod +from datetime import date, datetime +from enum import Enum +from typing import Any, Callable, Dict, Generator, List, Type, Union, _GenericAlias + +import pyarrow as pa +import pydantic + +from .dependencies import numpy as np + +_PYDANTIC_V2 = hasattr(pydantic, "GetCoreSchemaHandler") + +try: + from pydantic_core import CoreSchema, core_schema +except ImportError: + if _PYDANTIC_V2: + raise + + +class FixedSizeListMixin(ABC): + @staticmethod + @abstractmethod + def dim() -> int: + raise NotImplementedError + + @staticmethod + @abstractmethod + def value_arrow_type() -> pa.DataType: + raise NotImplementedError + + +def Vector( + dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True +) -> Type[FixedSizeListMixin]: + """Pydantic type for a fixed-size-list embedding vector column. + + Examples + -------- + >>> import pydantic + >>> from lance.pydantic import Vector + ... + >>> class MyModel(pydantic.BaseModel): + ... id: int + ... embedding: Vector(768) + """ + + class FixedSizeList(list, FixedSizeListMixin): + def __repr__(self): + return f"FixedSizeList(dim={dim})" + + @staticmethod + def nullable() -> bool: + return nullable + + @staticmethod + def dim() -> int: + return dim + + @staticmethod + def value_arrow_type() -> pa.DataType: + return value_type + + @classmethod + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: pydantic.GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_after_validator_function( + cls, + core_schema.list_schema( + min_length=dim, + max_length=dim, + items_schema=core_schema.float_schema(), + ), + ) + + @classmethod + def __get_validators__(cls) -> Generator[Callable, None, None]: + yield cls.validate + + # For pydantic v1 + @classmethod + def validate(cls, v): + if not isinstance(v, (list, range, np.ndarray)) or len(v) != dim: + raise TypeError("A list of numbers or numpy.ndarray is needed") + return cls(v) + + if not _PYDANTIC_V2: + + @classmethod + def __modify_schema__(cls, field_schema: Dict[str, Any]): + field_schema["items"] = {"type": "number"} + field_schema["maxItems"] = dim + field_schema["minItems"] = dim + + return FixedSizeList + + +def MultiVector( + dim: int, value_type: pa.DataType = pa.float32(), nullable: bool = True +) -> Type[FixedSizeListMixin]: + """Pydantic type for a list of fixed-size-list embedding vectors. + + Useful for models that produce multiple embeddings per input (e.g. + ColPali-style multi-vector embeddings). + + Examples + -------- + >>> import pydantic + >>> from lance.pydantic import MultiVector + ... + >>> class MyModel(pydantic.BaseModel): + ... id: int + ... embeddings: MultiVector(128) + """ + + class MultiVectorList(list, FixedSizeListMixin): + def __repr__(self): + return f"MultiVector(dim={dim})" + + @staticmethod + def nullable() -> bool: + return nullable + + @staticmethod + def dim() -> int: + return dim + + @staticmethod + def value_arrow_type() -> pa.DataType: + return value_type + + @staticmethod + def is_multi_vector() -> bool: + return True + + @classmethod + def __get_pydantic_core_schema__( + cls, _source_type: Any, _handler: pydantic.GetCoreSchemaHandler + ) -> CoreSchema: + return core_schema.no_info_after_validator_function( + cls, + core_schema.list_schema( + items_schema=core_schema.list_schema( + min_length=dim, + max_length=dim, + items_schema=core_schema.float_schema(), + ), + ), + ) + + @classmethod + def __get_validators__(cls) -> Generator[Callable, None, None]: + yield cls.validate + + # For pydantic v1 + @classmethod + def validate(cls, v): + if not isinstance(v, (list, range)): + raise TypeError("A list of vectors is needed") + for vec in v: + if not isinstance(vec, (list, range, np.ndarray)) or len(vec) != dim: + raise TypeError(f"Each vector must be a list of {dim} numbers") + return cls(v) + + if not _PYDANTIC_V2: + + @classmethod + def __modify_schema__(cls, field_schema: Dict[str, Any]): + field_schema["items"] = { + "type": "array", + "items": {"type": "number"}, + "minItems": dim, + "maxItems": dim, + } + + return MultiVectorList + + +def _field_annotation(field: Any) -> Any: + """Get the type annotation off a pydantic v1 ModelField or v2 FieldInfo.""" + if hasattr(field, "annotation"): + # Pydantic v2 FieldInfo + return field.annotation + # Pydantic v1 ModelField -- Optional-ness is tracked via `allow_none`, + # not folded into `outer_type_`. + return field.outer_type_ + + +def get_extras(field: Any, key: str) -> Any: + """Get extra metadata (from `json_schema_extra`) off a pydantic field.""" + if hasattr(field, "json_schema_extra"): + # Pydantic v2 FieldInfo + return (field.json_schema_extra or {}).get(key) + # Pydantic v1 ModelField + return (field.field_info.extra or {}).get("json_schema_extra", {}).get(key) + + +def _py_type_to_arrow_type(py_type: Type[Any], field: Any) -> pa.DataType: + """Convert a field with native Python type to Arrow data type. + + Raises + ------ + TypeError + If the type is not supported. + """ + if py_type is int: + return pa.int64() + elif py_type is float: + return pa.float64() + elif py_type is str: + return pa.utf8() + elif py_type is bool: + return pa.bool_() + elif py_type is bytes: + return pa.binary() + elif py_type is date: + return pa.date32() + elif py_type is datetime: + tz = get_extras(field, "tz") + return pa.timestamp("us", tz=tz) + elif getattr(py_type, "__origin__", None) in (list, tuple): + # A bare, unparameterised ``typing.List`` / ``typing.Tuple`` matches + # this branch (its ``__origin__`` is ``list`` / ``tuple``) but has no + # ``__args__``, so we cannot infer the element type. Raise a clear + # ``TypeError`` instead of crashing with an opaque ``AttributeError``. + args = getattr(py_type, "__args__", None) + if not args: + raise TypeError( + "Converting Pydantic type to Arrow Type: unsupported type " + f"{py_type}. Specify the element type, e.g. List[int] instead " + "of a bare List." + ) + child = args[0] + return _pydantic_list_child_to_arrow(child, field) + raise TypeError( + f"Converting Pydantic type to Arrow Type: unsupported type {py_type}." + ) + + +def _pydantic_model_to_fields(model: Type[pydantic.BaseModel]) -> List[pa.Field]: + if hasattr(model, "model_fields"): + # Pydantic v2 + return [ + _pydantic_to_field(name, field) + for name, field in model.model_fields.items() + ] + # Pydantic v1 + return [_pydantic_to_field(name, field) for name, field in model.__fields__.items()] + + +def _pydantic_type_to_arrow_type(tp: Any, field: Any) -> pa.DataType: + def _safe_issubclass(candidate: Any, base: type) -> bool: + try: + return issubclass(candidate, base) + except TypeError: + return False + + if inspect.isclass(tp): + if _safe_issubclass(tp, pydantic.BaseModel): + # Struct + fields = _pydantic_model_to_fields(tp) + return pa.struct(fields) + if _safe_issubclass(tp, FixedSizeListMixin): + if getattr(tp, "is_multi_vector", lambda: False)(): + return pa.list_(pa.list_(tp.value_arrow_type(), tp.dim())) + # For regular Vector + return pa.list_(tp.value_arrow_type(), tp.dim()) + if _safe_issubclass(tp, Enum): + # Map Enum to the Arrow type of its value. + # For string-valued enums, use dictionary encoding for efficiency. + # For integer enums, use the native type. + # Fall back to utf8 for mixed-type or empty enums. + value_types = {type(m.value) for m in tp} + if len(value_types) == 1: + value_type = value_types.pop() + if value_type is str: + # Use dictionary encoding for string enums + return pa.dictionary(pa.int32(), pa.utf8()) + return _py_type_to_arrow_type(value_type, field) + return pa.utf8() + return _py_type_to_arrow_type(tp, field) + + +def _pydantic_list_child_to_arrow(child: Any, field: Any) -> pa.DataType: + unwrapped = _unwrap_optional_annotation(child) + if unwrapped is not None: + return pa.list_( + pa.field("item", _pydantic_type_to_arrow_type(unwrapped, field), True) + ) + return pa.list_(_pydantic_type_to_arrow_type(child, field)) + + +def _unwrap_optional_annotation(annotation: Any) -> Any | None: + if isinstance(annotation, (_GenericAlias, types.GenericAlias)): + origin = annotation.__origin__ + args = annotation.__args__ + if origin == Union: + non_none = [arg for arg in args if arg is not type(None)] + if len(non_none) == 1 and len(non_none) != len(args): + return non_none[0] + elif sys.version_info >= (3, 10) and isinstance(annotation, types.UnionType): + args = annotation.__args__ + non_none = [arg for arg in args if arg is not type(None)] + if len(non_none) == 1 and len(non_none) != len(args): + return non_none[0] + return None + + +def _pydantic_to_arrow_type(field: Any) -> pa.DataType: + """Convert a pydantic field (v1 ModelField or v2 FieldInfo) to Arrow DataType""" + annotation = _field_annotation(field) + unwrapped = _unwrap_optional_annotation(annotation) + if unwrapped is not None: + return _pydantic_type_to_arrow_type(unwrapped, field) + if isinstance(annotation, (_GenericAlias, types.GenericAlias)): + origin = annotation.__origin__ + args = annotation.__args__ + + if origin is list: + child = args[0] + return _pydantic_list_child_to_arrow(child, field) + return _pydantic_type_to_arrow_type(annotation, field) + + +def is_nullable(field: Any) -> bool: + """Check if a pydantic field (v1 ModelField or v2 FieldInfo) is nullable. + + Only a true ``Optional``/``Union[..., None]`` annotation (or a nullable + ``Vector``/``MultiVector``) makes a field nullable -- a field with a + plain default value but a non-Optional type is not. + """ + if not hasattr(field, "annotation"): + # Pydantic v1 ModelField: Optional-ness is tracked via `allow_none` + # directly, since `outer_type_` already has Optional stripped. A + # nullable Vector/MultiVector still overrides this, same as v2. + v1_type = _field_annotation(field) + if inspect.isclass(v1_type): + try: + if issubclass(v1_type, FixedSizeListMixin): + return v1_type.nullable() + except TypeError: + pass + return bool(field.allow_none) + + annotation = field.annotation + if _unwrap_optional_annotation(annotation) is not None: + return True + if isinstance(annotation, (_GenericAlias, types.GenericAlias)): + origin = annotation.__origin__ + args = annotation.__args__ + if origin == Union: + if any(typ is type(None) for typ in args): + return True + elif sys.version_info >= (3, 10) and isinstance(annotation, types.UnionType): + args = annotation.__args__ + for typ in args: + if typ is type(None): + return True + elif inspect.isclass(annotation): + try: + if issubclass(annotation, FixedSizeListMixin): + return annotation.nullable() + except TypeError: + return False + return False + + +def _pydantic_to_field(name: str, field: Any) -> pa.Field: + """Convert a pydantic field (v1 ModelField or v2 FieldInfo) to a PyArrow Field.""" + dt = _pydantic_to_arrow_type(field) + return pa.field(name, dt, is_nullable(field)) + + +def pydantic_to_schema(model: Type[pydantic.BaseModel]) -> pa.Schema: + """Convert a [Pydantic Model][pydantic.BaseModel] to a + [PyArrow Schema][pyarrow.Schema]. + + Supports nested ``BaseModel`` fields (-> struct), ``Enum`` fields + (string-valued enums are dictionary-encoded), timezone-aware + ``datetime`` fields (via ``Field(json_schema_extra={"tz": ...})``), and + the ``Vector``/``MultiVector`` fixed-size-list types, in addition to + plain scalar/``Optional``/``List`` types. + + Parameters + ---------- + model : Type[pydantic.BaseModel] + The Pydantic BaseModel to convert to Arrow Schema. + + Returns + ------- + pyarrow.Schema + The Arrow Schema + + Examples + -------- + + >>> from typing import List, Optional + >>> import pydantic + >>> from lance.pydantic import pydantic_to_schema, Vector + >>> class FooModel(pydantic.BaseModel): + ... id: int + ... s: str + ... vec: Vector(1536) # fixed_size_list[1536] + ... li: List[int] + ... + >>> schema = pydantic_to_schema(FooModel) + >>> assert schema == pa.schema([ + ... pa.field("id", pa.int64(), False), + ... pa.field("s", pa.utf8(), False), + ... pa.field("vec", pa.list_(pa.float32(), 1536)), + ... pa.field("li", pa.list_(pa.int64()), False), + ... ]) + """ + fields = _pydantic_model_to_fields(model) + return pa.schema(fields) diff --git a/python/python/lance/query.py b/python/python/lance/query.py index 85bbf121e6e..a462f9e3f64 100644 --- a/python/python/lance/query.py +++ b/python/python/lance/query.py @@ -22,6 +22,13 @@ class FullTextOperator(Enum): OR = "OR" +class DocumentGranularity(str, Enum): + """The unit treated as one full-text-search document.""" + + ROW = "row" + LIST_ELEMENT = "list_element" + + class Occur(Enum): SHOULD = "SHOULD" MUST = "MUST" @@ -98,6 +105,7 @@ def __init__( max_expansions: int = 50, operator: FullTextOperator = FullTextOperator.OR, prefix_length: int = 0, + document_granularity: Optional[DocumentGranularity] = None, ): """ Match query for full-text search. @@ -129,6 +137,10 @@ def __init__( prefix_length : int, default 0 The number of beginning characters being unchanged for fuzzy matching. This is useful to achieve prefix matching. + document_granularity : DocumentGranularity, optional + Explicitly select row or deepest-list-element documents. If omitted, + the indexed granularity is inferred. When both granularities are indexed + for the field, this must be specified. With no index, ``ROW`` is used. """ self._inner = PyFullTextQuery.match_query( query, @@ -138,6 +150,9 @@ def __init__( max_expansions=max_expansions, operator=operator.value, prefix_length=prefix_length, + document_granularity=( + document_granularity.value if document_granularity is not None else None + ), ) def query_type(self) -> FullTextQueryType: @@ -145,7 +160,14 @@ def query_type(self) -> FullTextQueryType: class PhraseQuery(FullTextQuery): - def __init__(self, query: str, column: str, *, slop: int = 0): + def __init__( + self, + query: str, + column: str, + *, + slop: int = 0, + document_granularity: Optional[DocumentGranularity] = None, + ): """ Phrase query for full-text search. @@ -155,8 +177,21 @@ def __init__(self, query: str, column: str, *, slop: int = 0): The query string to match against. column : str The name of the column to match against. + slop : int, default 0 + The maximum number of intervening positions permitted in the phrase. + document_granularity : DocumentGranularity, optional + Explicitly select row or deepest-list-element documents. If omitted, + the indexed granularity is inferred. When both granularities are indexed + for the field, this must be specified. With no index, ``ROW`` is used. """ - self._inner = PyFullTextQuery.phrase_query(query, column, slop) + self._inner = PyFullTextQuery.phrase_query( + query, + column, + slop, + document_granularity=( + document_granularity.value if document_granularity is not None else None + ), + ) def query_type(self) -> FullTextQueryType: return FullTextQueryType.MATCH_PHRASE @@ -235,7 +270,9 @@ def __init__(self, queries: list[tuple[Occur, FullTextQuery]]): Parameters ---------- queries : list[tuple(Occur, FullTextQuery)] - The list of queries with their occurrence requirements. + The list of queries with their occurrence requirements. Every MUST + clause must match and contributes its score; matching SHOULD scores + are also added, while MUST_NOT clauses only exclude documents. """ self._inner = PyFullTextQuery.boolean_query( [(occur.value, query.inner) for occur, query in queries] diff --git a/python/python/lance/sampler.py b/python/python/lance/sampler.py index b7e7230dfc6..4c17825ab04 100644 --- a/python/python/lance/sampler.py +++ b/python/python/lance/sampler.py @@ -193,7 +193,7 @@ def maybe_sample( This is employed to minimize the number of random reads necessary for sampling. A sufficiently large value can provide an effective random sample without the need for excessive random reads. - filter : str, optional + filt : str, optional The filter to apply to the dataset, by default None. If a filter is provided, then we will first load all row ids in memory and then batch through the ids in random order until enough matches have been found. diff --git a/python/python/lance/schema.py b/python/python/lance/schema.py index aacdcb73e8e..e97e97e3322 100644 --- a/python/python/lance/schema.py +++ b/python/python/lance/schema.py @@ -12,17 +12,19 @@ def schema_to_json(schema: pa.Schema) -> Dict[str, Any]: """ - Converts a pyarrow schema to a JSON string. + Converts a pyarrow schema to a JSON-compatible dict. Parameters ---------- + schema: pa.Schema + The PyArrow schema to convert. """ return json.loads(_schema_to_json(schema)) def json_to_schema(schema_json: Dict[str, Any]) -> pa.Schema: """ - Converts a JSON string to a PyArrow schema. + Converts a JSON-compatible dict to a PyArrow schema. Parameters ---------- diff --git a/python/python/lance/tf/__init__.py b/python/python/lance/tf/__init__.py deleted file mode 100644 index 1aa41beb99c..00000000000 --- a/python/python/lance/tf/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - -import importlib.util - -if importlib.util.find_spec("tensorflow") is None: - raise ImportError( - "Tensorflow is not installed. Please install tensorflow" - + " to use lance.tf module.", - ) diff --git a/python/python/lance/tf/data.py b/python/python/lance/tf/data.py deleted file mode 100644 index 68280f9211f..00000000000 --- a/python/python/lance/tf/data.py +++ /dev/null @@ -1,410 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - - -"""Tensorflow Dataset (`tf.data `_) -implementation for Lance. - -.. warning:: - - Experimental feature. API stability is not guaranteed. -""" - -from __future__ import annotations - -from functools import partial -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union - -import pyarrow as pa - -import lance -from lance import LanceDataset -from lance.arrow import EncodedImageType, FixedShapeImageTensorType, ImageURIType -from lance.dependencies import _check_for_numpy -from lance.dependencies import numpy as np -from lance.dependencies import tensorflow as tf -from lance.fragment import FragmentMetadata, LanceFragment -from lance.log import LOGGER - -if TYPE_CHECKING: - from pathlib import Path - - from lance import LanceNamespace - - -def arrow_data_type_to_tf(dt: pa.DataType) -> tf.DType: - """Convert Pyarrow DataType to Tensorflow.""" - if pa.types.is_boolean(dt): - return tf.bool - elif pa.types.is_int8(dt): - return tf.int8 - elif pa.types.is_int16(dt): - return tf.int16 - elif pa.types.is_int32(dt): - return tf.int32 - elif pa.types.is_int64(dt): - return tf.int64 - elif pa.types.is_uint8(dt): - return tf.uint8 - elif pa.types.is_uint16(dt): - return tf.uint16 - elif pa.types.is_uint32(dt): - return tf.uint32 - elif pa.types.is_uint64(dt): - return tf.uint64 - elif pa.types.is_float16(dt): - return tf.float16 - elif pa.types.is_float32(dt): - return tf.float32 - elif pa.types.is_float64(dt): - return tf.float64 - elif ( - pa.types.is_string(dt) - or pa.types.is_large_string(dt) - or pa.types.is_binary(dt) - or pa.types.is_large_binary(dt) - ): - return tf.string - - raise TypeError(f"Arrow/Tf conversion: Unsupported arrow data type: {dt}") - - -def data_type_to_tensor_spec(dt: pa.DataType) -> tf.TensorSpec: - """Convert PyArrow DataType to Tensorflow TensorSpec.""" - if ( - pa.types.is_boolean(dt) - or pa.types.is_integer(dt) - or pa.types.is_floating(dt) - or pa.types.is_string(dt) - or pa.types.is_binary(dt) - ): - return tf.TensorSpec(shape=(None,), dtype=arrow_data_type_to_tf(dt)) - elif isinstance(dt, pa.FixedShapeTensorType): - return tf.TensorSpec( - shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.value_type) - ) - elif pa.types.is_fixed_size_list(dt): - return tf.TensorSpec( - shape=(None, dt.list_size), dtype=arrow_data_type_to_tf(dt.value_type) - ) - elif pa.types.is_list(dt) or pa.types.is_large_list(dt): - return tf.TensorSpec( - shape=( - None, - None, - ), - dtype=arrow_data_type_to_tf(dt.value_type), - ) - elif pa.types.is_struct(dt): - return {field.name: data_type_to_tensor_spec(field.type) for field in dt} - elif isinstance(dt, (EncodedImageType, ImageURIType)): - return tf.TensorSpec(shape=(None,), dtype=tf.string) - elif isinstance(dt, FixedShapeImageTensorType): - return tf.TensorSpec( - shape=(None, *dt.shape), dtype=arrow_data_type_to_tf(dt.arrow_type) - ) - - raise TypeError("Unsupported data type: ", dt) - - -def schema_to_spec(schema: pa.Schema) -> tf.TypeSpec: - """Convert PyArrow Schema to Tensorflow output signature.""" - signature = {} - for name in schema.names: - field = schema.field(name) - signature[name] = data_type_to_tensor_spec(field.type) - return signature - - -def column_to_tensor(array: pa.Array, tensor_spec: tf.TensorSpec) -> tf.Tensor: - """Convert a PyArrow array into a TensorFlow tensor.""" - if isinstance(tensor_spec, tf.RaggedTensorSpec): - return tf.ragged.constant(array.to_pylist(), dtype=tensor_spec.dtype) - elif isinstance(array.type, pa.FixedShapeTensorType): - return tf.constant(array.to_numpy_ndarray(), dtype=tensor_spec.dtype) - elif isinstance(array.type, FixedShapeImageTensorType): - return tf.constant(array.to_numpy(), dtype=tensor_spec.dtype) - elif isinstance(array.type, pa.StructType): - return { - field.name: column_to_tensor(array.field(i), tensor_spec[field.name]) - for (i, field) in enumerate(array.type) - } - else: - return tf.constant(array.to_pylist(), dtype=tensor_spec.dtype) - - -def from_lance( - dataset: Optional[Union[str, Path, LanceDataset]] = None, - *, - columns: Optional[Union[List[str], Dict[str, str]]] = None, - batch_size: int = 256, - filter: Optional[str] = None, - fragments: Union[Iterable[int], Iterable[LanceFragment], tf.data.Dataset] = None, - output_signature: Optional[Dict[str, tf.TypeSpec]] = None, - namespace_client: Optional["LanceNamespace"] = None, - table_id: Optional[List[str]] = None, - ignore_namespace_table_storage_options: bool = False, -) -> tf.data.Dataset: - """Create a ``tf.data.Dataset`` from a Lance dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset], optional - Lance dataset or dataset URI/path. Either ``dataset`` or both - ``namespace_client`` and ``table_id`` must be provided. - columns : Optional[List[str]], optional - List of columns to include in the output dataset. - If not set, all columns will be read. - batch_size : int, optional - Batch size, by default 256 - filter : Optional[str], optional - SQL filter expression, by default None. - fragments : Union[List[LanceFragment], tf.data.Dataset], optional - If provided, only the fragments are read. It can be used to feed - for distributed training. - output_signature : Optional[tf.TypeSpec], optional - Override output signature of the returned tensors. If not provided, - the output signature is inferred from the projection Schema. - namespace_client : Optional[LanceNamespace], optional - Namespace client to resolve the table location when ``table_id`` is - provided. - table_id : Optional[List[str]], optional - Table identifier used together with ``namespace_client`` to locate - the table. - ignore_namespace_table_storage_options : bool, default False - When using ``namespace_client``/``table_id``, ignore storage options - returned by the namespace. - - Examples - -------- - - .. code-block:: python - - import tensorflow as tf - import lance.tf.data - - ds = lance.tf.data.from_lance( - "s3://bucket/path", - columns=["image", "id"], - filter="catalog = 'train' AND split = 'train'", - batch_size=100) - - for batch in ds.repeat(10).shuffle(128).map(io_decode): - print(batch["image"].shape) - - ``from_lance`` can take an iterator or ``tf.data.Dataset`` of - Fragments. So that it can be used to feed for distributed training. - - .. code-block:: python - - import tensorflow as tf - import lance.tf.data - - seed = 200 # seed to shuffle the fragments in distributed machines. - fragments = lance.tf.data.lance_fragments("s3://bucket/path") - repeat(10).shuffle(4, seed=seed) - ds = lance.tf.data.from_lance( - "s3://bucket/path", - columns=["image", "id"], - filter="catalog = 'train' AND split = 'train'", - fragments=fragments, - batch_size=100) - for batch in ds.shuffle(128).map(io_decode): - print(batch["image"].shape) - - """ - if isinstance(dataset, LanceDataset): - if namespace_client is not None or table_id is not None: - raise ValueError( - "Cannot specify 'namespace_client' or 'table_id' when passing " - "a LanceDataset instance" - ) - else: - dataset = lance.dataset( - dataset, - namespace_client=namespace_client, - table_id=table_id, - ignore_namespace_table_storage_options=ignore_namespace_table_storage_options, - ) - - if isinstance(fragments, tf.data.Dataset): - fragments = list(fragments.as_numpy_iterator()) - elif _check_for_numpy(fragments) and isinstance(fragments, np.ndarray): - fragments = list(fragments) - - if fragments is not None: - - def gen_fragments(fragments): - for f in fragments: - if isinstance(f, int) or ( - _check_for_numpy(f) and isinstance(f, np.integer) - ): - yield LanceFragment(dataset, int(f)) - elif isinstance(f, FragmentMetadata): - yield LanceFragment(dataset, f.id) - elif isinstance(f, LanceFragment): - yield f - else: - raise TypeError(f"Invalid type passed to `fragments`: {type(f)}") - - # A Generator of Fragments - fragments = gen_fragments(fragments) - - scanner = dataset.scanner( - filter=filter, columns=columns, batch_size=batch_size, fragments=fragments - ) - - if output_signature is None: - schema = scanner.projected_schema - output_signature = schema_to_spec(schema) - LOGGER.debug("Output signature: %s", output_signature) - - def generator(): - for batch in scanner.to_batches(): - yield { - name: column_to_tensor(batch[name], output_signature[name]) - for name in batch.schema.names - } - - return tf.data.Dataset.from_generator(generator, output_signature=output_signature) - - -def lance_fragments(dataset: Union[str, Path, LanceDataset]) -> tf.data.Dataset: - """Create a ``tf.data.Dataset`` of Lance Fragments in the dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset] - A Lance Dataset or dataset URI/path. - """ - if not isinstance(dataset, LanceDataset): - dataset = lance.dataset(dataset) - return tf.data.Dataset.from_tensor_slices( - [f.fragment_id for f in dataset.get_fragments()] - ) - - -def _ith_batch(i: int, batch_size: int, total_size: int) -> Tuple[int, int]: - """ - Get the start and end index of the ith batch. - - This takes into account the total_size, the total number of rows in the dataset. - """ - start = i * batch_size - end = tf.math.minimum(start + batch_size, total_size) - return (start, end) - - -def from_lance_batches( - dataset: Union[str, Path, LanceDataset], - *, - shuffle: bool = False, - seed: Optional[int] = None, - batch_size: int = 1024, - skip: int = 0, -) -> tf.data.Dataset: - """ - Create a ``tf.data.Dataset`` of batch indices for a Lance dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset] - A Lance Dataset or dataset URI/path. - shuffle : bool, optional - Shuffle the batches, by default False - seed : Optional[int], optional - Random seed for shuffling, by default None - batch_size : int, optional - Batch size, by default 1024 - skip : int, optional - Number of batches to skip. - - Returns - ------- - tf.data.Dataset - A tensorflow dataset of batch slice ranges. These can be passed to - :func:`lance_take_batches` to create a Tensorflow dataset of batches. - """ - if not isinstance(dataset, LanceDataset): - dataset = lance.dataset(dataset) - num_rows = dataset.count_rows() - num_batches = (num_rows + batch_size - 1) // batch_size - indices = tf.data.Dataset.range(num_batches, dtype=tf.int64) - if shuffle: - indices = indices.shuffle(num_batches, seed=seed) - if skip > 0: - indices = indices.skip(skip) - return indices.map(partial(_ith_batch, batch_size=batch_size, total_size=num_rows)) - - -def lance_take_batches( - dataset: Union[str, Path, LanceDataset], - batch_ranges: Iterable[Tuple[int, int]], - *, - columns: Optional[List[str]] = None, - output_signature: Optional[Dict[str, tf.TypeSpec]] = None, - batch_readahead: int = 10, -) -> tf.data.Dataset: - """ - Create a ``tf.data.Dataset`` of batches from a Lance dataset. - - Parameters - ---------- - dataset : Union[str, Path, LanceDataset] - A Lance Dataset or dataset URI/path. - batch_ranges : Iterable[Tuple[int, int]] - Iterable of batch indices. - columns : Optional[List[str]], optional - List of columns to include in the output dataset. - If not set, all columns will be read. - output_signature : Optional[tf.TypeSpec], optional - Override output signature of the returned tensors. If not provided, - the output signature is inferred from the projection Schema. - batch_readahead : int, default 10 - The number of batches to read ahead in parallel. - - Examples - -------- - You can compose this with ``from_lance_batches`` to create a randomized Tensorflow - dataset. With ``from_lance_batches``, you can deterministically randomized the - batches by setting ``seed``. - - .. code-block:: python - - batch_iter = from_lance_batches(dataset, batch_size=100, shuffle=True, seed=200) - batch_iter = batch_iter.as_numpy_iterator() - lance_ds = lance_take_batches(dataset, batch_iter) - lance_ds = lance_ds.unbatch().shuffle(500, seed=42).batch(100) - """ - if not isinstance(dataset, LanceDataset): - dataset = lance.dataset(dataset) - - if output_signature is None: - schema = dataset.scanner(columns=columns).projected_schema - output_signature = schema_to_spec(schema) - LOGGER.debug("Output signature: %s", output_signature) - - def gen_ranges(): - for start, end in batch_ranges: - yield (start, end) - - def gen_batches(): - batches = dataset._ds.take_scan( - gen_ranges(), - columns=columns, - batch_readahead=batch_readahead, - ) - for batch in batches: - yield { - name: column_to_tensor(batch[name], output_signature[name]) - for name in batch.schema.names - } - - return tf.data.Dataset.from_generator( - gen_batches, output_signature=output_signature - ) - - -# Register `from_lance` to ``tf.data.Dataset``. -tf.data.Dataset.from_lance = from_lance -tf.data.Dataset.from_lance_batches = from_lance_batches diff --git a/python/python/lance/torch/bench_utils.py b/python/python/lance/torch/bench_utils.py index b0b19fc22a9..41d8f8278f3 100644 --- a/python/python/lance/torch/bench_utils.py +++ b/python/python/lance/torch/bench_utils.py @@ -128,7 +128,7 @@ def recall(expected: np.ndarray, actual: np.ndarray) -> np.ndarray: ---------- expected: ndarray The ground truth - results: ndarray + actual: ndarray The ANN results """ assert expected.shape == actual.shape diff --git a/python/python/lance/torch/kmeans.py b/python/python/lance/torch/kmeans.py index 44fca9ae60a..84874c47f46 100644 --- a/python/python/lance/torch/kmeans.py +++ b/python/python/lance/torch/kmeans.py @@ -177,7 +177,7 @@ def _updated_centroids( for idx in zero_counts.nonzero(as_tuple=False): # split the largest cluster and remove empty cluster max_idx = torch.argmax(counts).item() - # add 1% gassuian noise to the largest centroid + # add 1% gaussian noise to the largest centroid # do this twice so we effectively split the largest cluster into 2 # rand_like returns on [0, 1) so we need to shift it to [-0.5, 0.5) noise = (torch.rand_like(centroids[idx]) - 0.5) * 0.01 + 1 diff --git a/python/python/lance/types.py b/python/python/lance/types.py index 41cc191e4d6..3b5cc5b0683 100644 --- a/python/python/lance/types.py +++ b/python/python/lance/types.py @@ -9,7 +9,13 @@ from pyarrow import RecordBatch from . import dataset -from .dependencies import _check_for_hugging_face, _check_for_pandas +from .dependencies import ( + _check_for_hugging_face, + _check_for_pandas, + _is_pydantic_base_model, + _validate_pydantic_list, + model_to_dict, +) from .dependencies import pandas as pd if TYPE_CHECKING: @@ -52,6 +58,34 @@ def _casting_recordbatch_iter( yield batch +def _is_materialized(data_obj: ReaderLike) -> bool: + """Whether ``data_obj`` is fully materialized in memory. + + Materialized sources (tables, in-memory frames) can be wrapped in an + in-memory table for replay without spilling and to expose exact statistics. + Streaming or re-readable sources (readers, scanners, datasets, generators) + are not considered materialized. + """ + if _check_for_pandas(data_obj) and isinstance(data_obj, pd.DataFrame): + return True + if isinstance(data_obj, (pa.Table, pa.RecordBatch)): + return True + if ( + type(data_obj).__module__.startswith("polars") + and data_obj.__class__.__name__ == "DataFrame" + ): + return True + if isinstance(data_obj, dict): + return True + if ( + isinstance(data_obj, list) + and len(data_obj) > 0 + and isinstance(data_obj[0], dict) + ): + return True + return False + + def _coerce_reader( data_obj: ReaderLike, schema: Optional[pa.Schema] = None ) -> pa.RecordBatchReader: @@ -116,6 +150,20 @@ def batch_iter(): # List of dictionaries batch = pa.RecordBatch.from_pylist(data_obj, schema=schema) return pa.RecordBatchReader.from_batches(batch.schema, [batch]) + elif ( + isinstance(data_obj, list) + and len(data_obj) > 0 + and _is_pydantic_base_model(data_obj[0]) + ): + model_class = type(data_obj[0]) + _validate_pydantic_list(data_obj, model_class) + if schema is None: + from .pydantic import pydantic_to_schema + + schema = pydantic_to_schema(model_class) + dicts = [model_to_dict(item) for item in data_obj] + batch = pa.RecordBatch.from_pylist(dicts, schema=schema) + return pa.RecordBatchReader.from_batches(batch.schema, [batch]) # for other iterables, assume they are of type Iterable[RecordBatch] elif isinstance(data_obj, Iterable): if schema is not None: diff --git a/python/python/lance/udf.py b/python/python/lance/udf.py index 3a80349479e..45bd008f564 100644 --- a/python/python/lance/udf.py +++ b/python/python/lance/udf.py @@ -7,7 +7,7 @@ import pickle import sqlite3 from contextlib import closing -from typing import TYPE_CHECKING, Dict, List, NamedTuple, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple, Optional, Union import pyarrow as pa @@ -18,6 +18,8 @@ from .types import _coerce_reader if TYPE_CHECKING: + from pathlib import Path + from .dataset import LanceDataset, LanceFragment from .types import ReaderLike @@ -28,20 +30,26 @@ class BatchUDF: Use :func:`lance.add_columns_udf` decorator to wrap a function with this class. """ - def __init__(self, func, output_schema=None, checkpoint_file=None): + def __init__( + self, + func: Callable[[pa.RecordBatch], Any], + output_schema: Optional[pa.Schema] = None, + checkpoint_file: Optional[Union[str, Path]] = None, + ) -> None: self.func = func self.output_schema = output_schema + self.cache: Optional[BatchUDFCheckpoint] if checkpoint_file is not None: self.cache = BatchUDFCheckpoint(checkpoint_file) else: self.cache = None - def __call__(self, batch: pa.RecordBatch): + def __call__(self, batch: pa.RecordBatch) -> Any: # Directly call inner function. This is to allow the user to test the # function and have it behave exactly as it was written. return self.func(batch) - def _call(self, batch: pa.RecordBatch): + def _call(self, batch: pa.RecordBatch) -> pa.RecordBatch: if self.output_schema is None: raise ValueError( "output_schema must be provided when using a function that " @@ -59,7 +67,10 @@ def _call(self, batch: pa.RecordBatch): return result -def batch_udf(output_schema=None, checkpoint_file=None): +def batch_udf( + output_schema: Optional[pa.Schema] = None, + checkpoint_file: Optional[Union[str, Path]] = None, +) -> Callable[[Callable[[pa.RecordBatch], Any]], BatchUDF]: """ Create a user defined function (UDF) that adds columns to a dataset. @@ -88,7 +99,7 @@ def batch_udf(output_schema=None, checkpoint_file=None): AddColumnsUDF """ - def inner(func): + def inner(func: Callable[[pa.RecordBatch], Any]) -> BatchUDF: return BatchUDF(func, output_schema, checkpoint_file) return inner diff --git a/python/python/lance/util.py b/python/python/lance/util.py index 2161c4e0d45..180e2441b43 100644 --- a/python/python/lance/util.py +++ b/python/python/lance/util.py @@ -3,8 +3,18 @@ from __future__ import annotations +import uuid from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Iterator, Literal, Optional, Union, cast +from typing import ( + TYPE_CHECKING, + Iterable, + Iterator, + List, + Literal, + Optional, + Union, + cast, +) import pyarrow as pa @@ -51,6 +61,29 @@ def td_to_micros(td: timedelta) -> int: return round(td / timedelta(microseconds=1)) +def _normalize_index_segment_ids( + index_segments: Optional[Iterable[Union[str, uuid.UUID]]], +) -> Optional[List[str]]: + """Normalize a physical index segment selection to a list of UUID strings.""" + if index_segments is None: + return None + if isinstance(index_segments, (str, uuid.UUID)): + raise TypeError( + "index_segments must be an iterable of str or uuid.UUID, " + f"not a single {type(index_segments)}." + ) + segment_ids = [] + for segment_id in index_segments: + if isinstance(segment_id, (str, uuid.UUID)): + segment_ids.append(str(segment_id)) + else: + raise TypeError( + "index_segments must be an iterable of str or uuid.UUID. " + f"Got {type(segment_id)} instead." + ) + return segment_ids + + class KMeans: """KMean model for clustering. diff --git a/python/python/lance/vector.py b/python/python/lance/vector.py index 5ce5e8b61e5..30e77c865c9 100644 --- a/python/python/lance/vector.py +++ b/python/python/lance/vector.py @@ -19,9 +19,10 @@ ) from .dependencies import numpy as np from .log import LOGGER -from .util import MetricType, _normalize_metric_type +from .util import MetricType, _normalize_index_segment_ids, _normalize_metric_type if TYPE_CHECKING: + import uuid from pathlib import Path from . import LanceDataset @@ -197,6 +198,33 @@ def train_pq_codebook_on_accelerator( return pq_codebook, kmeans_list +def _sample_init_centroids( + ds: Iterable["torch.Tensor"], k: int, filter_nan: bool +) -> "torch.Tensor": + """Take up to k vectors from ds to seed kmeans, skipping non-finite ones.""" + # `column is not null` does not exclude NaN vectors, so they can still be + # sampled here. Training drops them (distance returns id -1), but a NaN + # centroid never recovers and leaves every partition NaN. + sampled = [] + num_sampled = 0 + for batch in ds: + if filter_nan: + batch = batch[batch.isfinite().flatten(1).all(dim=1)] + if batch.shape[0] == 0: + continue + sampled.append(batch) + num_sampled += batch.shape[0] + if num_sampled >= k: + break + + if num_sampled == 0: + raise ValueError( + "Cannot initialize centroids: the sampled vectors are all null or " + "non-finite" + ) + return torch.cat(sampled)[:k] + + def train_ivf_centroids_on_accelerator( dataset: LanceDataset, column: str, @@ -244,7 +272,7 @@ def train_ivf_centroids_on_accelerator( filter=filt, ) - init_centroids = next(iter(ds)) + init_centroids = _sample_init_centroids(ds, k, filter_nan) LOGGER.info("Done sampling: centroids shape: %s", init_centroids.shape) ds = TorchDataset( @@ -288,12 +316,12 @@ def compute_pq_codes( Dataset to compute pq codes for. kmeans_list: List[lance.torch.kmeans.KMeans] KMeans models to use to compute pq (one per subspace) - batch_size: int, default 10240 + batch_size: int, default 40960 The batch size used to read the dataset. dst_dataset_uri: Union[str, Path], optional The path to store the partitions. If not specified a random directory is used instead - allow_tf32: bool, default True + allow_cuda_tf32: bool, default True Whether to allow tf32 for matmul on CUDA. Returns @@ -417,12 +445,12 @@ def compute_partitions( Column name of the vector column. kmeans: lance.torch.kmeans.KMeans KMeans model to use to compute partitions. - batch_size: int, default 10240 + batch_size: int, default 40960 The batch size used to read the dataset. dst_dataset_uri: Union[str, Path], optional The path to store the partitions. If not specified a random directory is used instead - allow_tf32: bool, default True + allow_cuda_tf32: bool, default True Whether to allow tf32 for matmul on CUDA. Returns @@ -761,13 +789,17 @@ def hamming_clustering_for_ivf_partition( index_name: str, partition_id: int, hamming_threshold: int, + *, + index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None, ) -> pa.RecordBatchReader: """ Perform hamming clustering on a partition of an IVF_FLAT index. - Loads a partition from an IVF_FLAT index on a hash column, computes - pairwise hamming distances between all hashes in the partition, - filters by threshold, and clusters the results using union-find. + Loads a partition from every segment of an IVF_FLAT index on a hash + column, computes pairwise hamming distances between all hashes in the + combined partition, filters by threshold, and clusters the results using + union-find. All segments of the logical index must share the same global + IVF centroids; an error is raised if they do not. Parameters ---------- @@ -779,6 +811,11 @@ def hamming_clustering_for_ivf_partition( The partition ID within the IVF_FLAT index hamming_threshold : int Maximum hamming distance to consider as similar + index_segments : iterable of str or uuid.UUID, optional + If specified, only these physical index segment UUIDs of the named + logical index contribute rows. Use + :meth:`LanceDataset.describe_indices` to obtain segment UUIDs from + ``IndexDescription.segments``. Defaults to all segments. Returns ------- @@ -789,30 +826,43 @@ def hamming_clustering_for_ivf_partition( - 'duplicates': list - List of duplicate row IDs in each cluster """ return dataset._ds.hamming_clustering_for_ivf_partition( - index_name, partition_id, hamming_threshold + index_name, + partition_id, + hamming_threshold, + _normalize_index_segment_ids(index_segments), ) def get_ivf_partition_info( dataset: "LanceDataset", index_name: str, + *, + index_segments: Optional[Iterable[Union[str, uuid.UUID]]] = None, ) -> List[dict]: """ Get partition information for an IVF_FLAT index. + Partition sizes are aggregated across all segments of the logical index + unless a subset is selected via ``index_segments``. + Parameters ---------- dataset : LanceDataset The Lance dataset containing the hash column with an IVF_FLAT index. index_name : str Name of the IVF_FLAT index + index_segments : iterable of str or uuid.UUID, optional + If specified, only these physical index segment UUIDs of the named + logical index contribute to the sizes. Defaults to all segments. Returns ------- list[dict] List of partition info dicts with 'partition_id' and 'size' """ - return dataset._ds.get_ivf_partition_info(index_name) + return dataset._ds.get_ivf_partition_info( + index_name, _normalize_index_segment_ids(index_segments) + ) def hamming_clustering_for_sample( @@ -833,7 +883,8 @@ def hamming_clustering_for_sample( dataset : LanceDataset The Lance dataset containing the hash column. column : str - Name of the hash column (must be FixedSizeList) + Name of the hash column (must be FixedSizeList where N is a + positive multiple of 8 bytes) sample_size : int, optional Number of rows to sample. If None, uses all rows. hamming_threshold : int, default 10 @@ -875,7 +926,8 @@ def hamming_clustering_for_range( dataset : LanceDataset The Lance dataset containing the hash column. column : str - Name of the hash column (must be FixedSizeList) + Name of the hash column (must be FixedSizeList where N is a + positive multiple of 8 bytes) fragment_id : int The fragment ID to read from start_row : int diff --git a/python/python/tests/compat/compat_decorator.py b/python/python/tests/compat/compat_decorator.py index fdfe09a6879..3b6c039cea3 100644 --- a/python/python/tests/compat/compat_decorator.py +++ b/python/python/tests/compat/compat_decorator.py @@ -15,15 +15,13 @@ import sys import urllib.request from contextlib import contextmanager -from functools import lru_cache -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional import pytest from packaging.version import Version -@lru_cache(maxsize=1) -def pylance_stable_versions() -> List[Version]: +def _fetch_stable_versions() -> List[Version]: """Fetches and returns a sorted list of stable pylance versions from PyPI.""" try: with urllib.request.urlopen( @@ -67,12 +65,10 @@ def key(v: Version): return major_versions -@lru_cache(maxsize=1) -def last_beta_release(): +def _fetch_last_beta_release(): """Returns the latest beta version available on fury.io. Uses pip to query the fury.io index for pre-release versions of pylance. - Results are cached to avoid repeated network calls. """ try: # Use pip index to get versions from fury.io @@ -125,10 +121,45 @@ def last_beta_release(): return None -VERSIONS = recent_major_versions(3) -LAST_BETA_RELEASE = last_beta_release() -if LAST_BETA_RELEASE is not None: - VERSIONS.append(LAST_BETA_RELEASE) +_SNAPSHOT: Optional[Dict[str, Any]] = None + + +def version_snapshot() -> Dict[str, Any]: + """The set of published pylance releases these tests are built from. + + Every process collecting these tests has to agree on this, because it + decides the `version` parameters and pytest-xdist aborts a run whose workers + collected different tests. Resolving it takes two network queries, so it is + resolved once per process -- and once per run on the xdist controller, then + handed down to the workers by `pytest_configure_node` in the root tests + conftest. + """ + global _SNAPSHOT + if _SNAPSHOT is None: + _SNAPSHOT = { + "stable": [str(v) for v in _fetch_stable_versions()], + "beta": _fetch_last_beta_release(), + } + return _SNAPSHOT + + +def use_version_snapshot(snapshot: Dict[str, Any]) -> None: + global _SNAPSHOT + _SNAPSHOT = snapshot + + +def pylance_stable_versions() -> List[Version]: + """Sorted stable pylance versions published to PyPI.""" + return [Version(v) for v in version_snapshot()["stable"]] + + +def compat_versions() -> List[str]: + """The pylance versions every compat test is parametrized over.""" + versions = recent_major_versions(3) + beta = version_snapshot()["beta"] + if beta is not None: + versions.append(beta) + return versions class UpgradeDowngradeTest: @@ -210,7 +241,7 @@ def compat_test(min_version: str = "0.16.0"): Parameters ---------- versions : list of str, optional - List of Lance versions to test against. Defaults to VERSIONS. + List of Lance versions to test against. Defaults to `compat_versions()`. Example ------- @@ -233,8 +264,18 @@ def check_write(self): # Write data pass """ - version = set([min_version, *VERSIONS]) - versions = [v for v in version if Version(v) >= Version(min_version)] + # Sorted rather than taken straight off the set: set iteration order for + # strings depends on PYTHONHASHSEED, which differs per process, so every + # pytest-xdist worker would otherwise collect these parameters in its own + # order and xdist rejects the run as an inconsistent collection. + versions = sorted( + ( + v + for v in {min_version, *compat_versions()} + if Version(v) >= Version(min_version) + ), + key=Version, + ) def decorator(cls): # Extract existing parametrize marks from the class diff --git a/python/python/tests/compat/compat_sequence.py b/python/python/tests/compat/compat_sequence.py index 48df46500ca..40330c6a1e0 100644 --- a/python/python/tests/compat/compat_sequence.py +++ b/python/python/tests/compat/compat_sequence.py @@ -11,8 +11,8 @@ without hand-coding the triggering sequence. The scenario is parameterized by index *kind* so every scalar index type gets the same -aged-lifecycle, cross-version treatment. The oracle runs the same predicate twice -- -normally and with use_scalar_index=False (lance ignores the index) -- and requires +aged-lifecycle, cross-version treatment. The scalar oracle runs the same predicate twice +-- normally and with use_scalar_index=False (lance ignores the index) -- and requires the results to match. If the two query plans are identical the index wasn't used, so the comparison is skipped rather than failed (uninformative, not a regression). FTS has no "ignore the index" mode to diff against, so its oracle reconstructs ground truth from a @@ -22,9 +22,15 @@ pin this with the create-index `format_version` parameter; old Lance versions still use `LANCE_FTS_FORMAT_VERSION`. -The op vocabulary and bounds are deliberately small so the search is runnable; this is -exhaustive over the maintenance-lifecycle grammar up to the configured lengths, not over -every op permutation. +IVF_PQ uses a separate bounded grammar because exhaustively applying the scalar grammar +to a trained vector index would be prohibitively expensive. It covers every ordered pair +of vector/scalar maintenance operations plus a few deeper lifecycle sequences. Its +oracle compares scalar-prefiltered ANN results with an exact, index-free KNN scan and +requires at least 0.5 recall, as well as checking the scalar filter exactly. + +The op vocabulary and bounds are deliberately small so the search is runnable. Scalar +and FTS cases are exhaustive over their maintenance grammar up to the configured length; +vector cases cover every ordered pair plus the curated deeper lifecycles above. """ import itertools @@ -33,9 +39,26 @@ from pathlib import Path ROWS_PER_WRITE = 200 +VECTOR_ROWS_PER_WRITE = 512 +VECTOR_DIM = 8 +VECTOR_K = 10 +VECTOR_KIND = "IVF_PQ" +VECTOR_INDEX_NAME = "vector_idx" +VECTOR_SCALAR_INDEX_NAME = "scalar_idx" SETUP_TAIL_OPS = ["D", "C", "W"] EXERCISE_OPS = ["W", "D", "C", "Oa", "Om", "Od"] +VECTOR_OPS = ("W", "D", "C", "Os", "Ov", "Om") + +# All ordered operation pairs are searched below. These longer cases preserve the +# state combinations that motivated the old recurring test without growing as 6**N. +VECTOR_CRITICAL_SEQUENCES = ( + ("W", "Ov", "W", "Ov", "W"), # two vector deltas, then unindexed rows + ("W", "Os", "W", "Ov", "Om"), + ("D", "C", "W", "Ov", "Om"), + ("W", "D", "Os", "C", "Ov"), + ("W", "Ov", "D", "C", "Om"), +) OP_NAMES = { "W": "write rows", @@ -45,6 +68,8 @@ "Oa": "optimize (append)", "Om": "optimize (merge)", "Od": "optimize", + "Os": "optimize scalar index (append)", + "Ov": "optimize vector index (append)", } @@ -58,7 +83,7 @@ def describe(kind, from_ref, to_ref, setup_ops, exercise_ops, fts_version=None): # Index kinds covered by the maintenance-sequence search. SCALAR_KINDS = ["BTREE", "BITMAP", "LABEL_LIST", "NGRAM", "ZONEMAP", "BLOOMFILTER"] -ALL_KINDS = ["INVERTED", *SCALAR_KINDS] +ALL_KINDS = ["INVERTED", *SCALAR_KINDS, VECTOR_KIND] class IndexScenario: @@ -83,6 +108,19 @@ def _batch(self, a, b): import pyarrow as pa idx = list(range(a, b)) + if self.kind == VECTOR_KIND: + # Deterministic pseudo-random vectors keep every appended batch in the + # training distribution without depending on numpy or process RNG state. + flat = [] + for i in idx: + state = ((i + 1) * 2654435761) & 0xFFFFFFFF + for _ in range(VECTOR_DIM): + state = (state * 1664525 + 1013904223) & 0xFFFFFFFF + flat.append(state / 4294967296.0) + vector = pa.FixedSizeListArray.from_arrays( + pa.array(flat, type=pa.float32()), VECTOR_DIM + ) + return pa.table({"idx": idx, "vector": vector}) if self.kind == "INVERTED": # Each row's text mixes tokens of different frequency: a unique term, a # mid-frequency bucket (~1/7 of rows), and one shared by every row. Sampling @@ -115,7 +153,8 @@ def _oracle_pred(self): def _op_W(self): import lance - a, b = self.next_idx, self.next_idx + ROWS_PER_WRITE + num_rows = VECTOR_ROWS_PER_WRITE if self.kind == VECTOR_KIND else ROWS_PER_WRITE + a, b = self.next_idx, self.next_idx + num_rows self.next_idx = b tbl = self._batch(a, b) if not os.path.exists(self.path): @@ -124,6 +163,18 @@ def _op_W(self): self._open().insert(tbl) def _op_I(self): + if self.kind == VECTOR_KIND: + self._open().create_scalar_index( + "idx", "BTREE", name=VECTOR_SCALAR_INDEX_NAME + ) + self._open().create_index( + "vector", + index_type="IVF_PQ", + name=VECTOR_INDEX_NAME, + num_partitions=2, + num_sub_vectors=2, + ) + return kwargs = {"with_position": True} if self.kind == "INVERTED" else {} if self.kind == "INVERTED" and self.fts_version is not None: kwargs["format_version"] = int(self.fts_version) @@ -145,11 +196,27 @@ def _op_Oa(self): self._open().optimize.optimize_indices(num_indices_to_merge=0) def _op_Om(self): - self._open().optimize.optimize_indices(num_indices_to_merge=10) + kwargs = {"num_indices_to_merge": 10} + if self.kind == VECTOR_KIND: + kwargs = { + "num_indices_to_merge": 1, + "index_names": [VECTOR_INDEX_NAME], + } + self._open().optimize.optimize_indices(**kwargs) def _op_Od(self): self._open().optimize.optimize_indices() + def _op_Os(self): + self._open().optimize.optimize_indices( + num_indices_to_merge=0, index_names=[VECTOR_SCALAR_INDEX_NAME] + ) + + def _op_Ov(self): + self._open().optimize.optimize_indices( + num_indices_to_merge=0, index_names=[VECTOR_INDEX_NAME] + ) + def _run(self, ops): for op in ops: getattr(self, f"_op_{op}")() @@ -164,6 +231,9 @@ def setup(self): def exercise_and_check(self): self._run(self.exercise_ops) ds = self._open() + if self.kind == VECTOR_KIND: + self._check_vector_prefilter(ds) + return if self.kind == "INVERTED": # Differential oracle: rebuild the token -> rows map from a full (unindexed) # scan, then require an FTS search for a spread of sampled terms to return @@ -208,6 +278,136 @@ def exercise_and_check(self): f"{self.kind}: index gave {got} rows, full scan {expected}, for '{pred}'" ) + def _check_vector_prefilter(self, ds): + """Check both BTREE filtering and IVF_PQ recall against index-free scans.""" + # Both ranges avoid the deterministic delete window. The first is always in + # the original index while the newest range may be an unindexed append, so + # the same query covers the indexed + unindexed prefilter path. + filter_rows = VECTOR_ROWS_PER_WRITE // 8 + lo = self.next_idx - filter_rows + pred = f"idx < {filter_rows} OR (idx >= {lo} AND idx < {self.next_idx})" + + filtered_scan = ds.to_table( + columns=["idx", "vector"], filter=pred, use_scalar_index=False + ) + filtered_rows = sorted( + zip( + filtered_scan.column("idx").to_pylist(), + filtered_scan.column("vector").to_pylist(), + ) + ) + filtered_ids = [idx for idx, _ in filtered_rows] + filtered_id_set = set(filtered_ids) + assert len(filtered_ids) == len(filtered_id_set), ( + f"BTREE prefilter returned duplicate row ids for '{pred}'" + ) + assert len(filtered_ids) >= VECTOR_K, ( + f"not enough live rows ({len(filtered_ids)}) for vector oracle '{pred}'" + ) + + scalar_plan = ds.scanner(filter=pred).explain_plan(True) + scan_plan = ds.scanner(filter=pred, use_scalar_index=False).explain_plan(True) + scalar_markers = ("ScalarIndexQuery", "MaterializeIndex") + assert any(marker in scalar_plan for marker in scalar_markers), ( + f"BTREE index was not used for '{pred}':\n{scalar_plan}" + ) + assert not any(marker in scan_plan for marker in scalar_markers), ( + f"BTREE index disabling was ignored for '{pred}':\n{scan_plan}" + ) + scalar_ids = ds.to_table(columns=["idx"], filter=pred).column("idx").to_pylist() + assert len(scalar_ids) == len(set(scalar_ids)), ( + f"BTREE returned duplicate row ids for '{pred}'" + ) + assert set(scalar_ids) == filtered_id_set, ( + f"BTREE returned {len(scalar_ids)} rows, full scan returned " + f"{len(filtered_ids)}, for '{pred}'" + ) + + query_positions = (0, len(filtered_ids) // 2, len(filtered_ids) - 1) + vectors = [vector for _, vector in filtered_rows] + for query_position in query_positions: + query = vectors[query_position] + indexed_nearest = { + "column": "vector", + "q": query, + "k": VECTOR_K, + "nprobes": 2, + "refine_factor": 10, + } + exact_nearest = { + "column": "vector", + "q": query, + "k": VECTOR_K, + "use_index": False, + } + + ann_plan = ds.scanner( + nearest=indexed_nearest, filter=pred, prefilter=True + ).explain_plan(True) + exact_plan = ds.scanner( + nearest=exact_nearest, + filter=pred, + prefilter=True, + use_scalar_index=False, + ).explain_plan(True) + ann_markers = ("ANNSubIndex", "ANNIvfPartition") + assert any(marker in ann_plan for marker in ann_markers), ( + f"IVF_PQ index was not used by vector search:\n{ann_plan}" + ) + assert not any(marker in exact_plan for marker in ann_markers), ( + f"IVF_PQ index disabling was ignored:\n{exact_plan}" + ) + assert any(marker in ann_plan for marker in scalar_markers), ( + f"BTREE index was not used by vector prefilter:\n{ann_plan}" + ) + assert not any(marker in exact_plan for marker in scalar_markers), ( + f"BTREE index disabling was ignored by exact prefilter:\n{exact_plan}" + ) + + got = ( + ds.to_table( + columns=["idx", "_distance"], + nearest=indexed_nearest, + filter=pred, + prefilter=True, + ) + .column("idx") + .to_pylist() + ) + expected = ( + ds.to_table( + columns=["idx", "_distance"], + nearest=exact_nearest, + filter=pred, + prefilter=True, + use_scalar_index=False, + ) + .column("idx") + .to_pylist() + ) + + assert len(got) == len(set(got)), "IVF_PQ search returned duplicate row ids" + assert len(expected) == len(set(expected)), ( + "exact vector search returned duplicate row ids" + ) + assert set(got) <= filtered_id_set, ( + f"IVF_PQ prefilter returned ids outside '{pred}': " + f"{sorted(set(got) - filtered_id_set)[:5]}" + ) + assert set(expected) <= filtered_id_set, ( + f"exact prefilter returned ids outside '{pred}': " + f"{sorted(set(expected) - filtered_id_set)[:5]}" + ) + assert len(got) == len(expected) == VECTOR_K, ( + f"IVF_PQ returned {len(got)} rows, exact search returned " + f"{len(expected)}" + ) + recall = len(set(got) & set(expected)) / VECTOR_K + assert recall >= 0.5, ( + f"IVF_PQ prefilter recall@{VECTOR_K}={recall:.3f}; " + f"expected at least 0.5 (got={got}, exact={expected})" + ) + def generate(max_length): """Yield every (setup_ops, exercise_ops) whose combined length is 1..max_length, @@ -223,6 +423,31 @@ def generate(max_length): yield list(s), list(e) +def generate_vector(max_length): + """Yield a bounded IVF_PQ + BTREE maintenance search space. + + Every ordered pair of operations is covered on both sides of the version split. + A small set of deeper cases captures multi-delta, unindexed-row, delete, compact, + and merge interactions without expanding the full operation grammar to 6**N. + """ + seen = set() + for total in range(1, min(max_length, 2) + 1): + for sequence in itertools.product(VECTOR_OPS, repeat=total): + for setup_len in range(total): + case = (sequence[:setup_len], sequence[setup_len:]) + if case not in seen: + seen.add(case) + yield list(case[0]), list(case[1]) + for sequence in VECTOR_CRITICAL_SEQUENCES: + if len(sequence) > max_length: + continue + for setup_len in range(len(sequence)): + case = (sequence[:setup_len], sequence[setup_len:]) + if case not in seen: + seen.add(case) + yield list(case[0]), list(case[1]) + + def search( venv_factory, from_ref, @@ -254,7 +479,10 @@ def search( # than rebuilding the index). Cached per shard, keyed by the setup ops. snapshots = {} # tuple(setup) -> (snapshot_path, next_idx), or None if setup failed try: - for i, (setup_tail, exercise) in enumerate(generate(max_length)): + cases = ( + generate_vector(max_length) if kind == VECTOR_KIND else generate(max_length) + ) + for i, (setup_tail, exercise) in enumerate(cases): if i % num_shards != shard: continue key = tuple(setup_tail) diff --git a/python/python/tests/compat/test_index_sequence.py b/python/python/tests/compat/test_index_sequence.py index 4d0db694064..0ab1fd65869 100644 --- a/python/python/tests/compat/test_index_sequence.py +++ b/python/python/tests/compat/test_index_sequence.py @@ -11,16 +11,51 @@ Refs and max length are environment-driven so the suite can run between two refs (versions, commits, or branches): COMPAT_FROM_REF / COMPAT_TO_REF / COMPAT_MAX_LENGTH / -COMPAT_KINDS (comma-separated subset of kinds) / COMPAT_SHARDS (split each kind's search -into this many cases so pytest-xdist (`-n auto`) parallelizes them across cores). +COMPAT_VECTOR_MAX_LENGTH / COMPAT_KINDS (comma-separated subset of kinds) / +COMPAT_SHARDS (split each scalar/FTS kind's search into this many cases so pytest-xdist +(`-n auto`) parallelizes them across cores) / COMPAT_VECTOR_SHARDS (the bounded IVF_PQ +search uses fewer shards to avoid repeatedly training the same small index). """ import os +from itertools import product import pytest from .compat_decorator import pylance_stable_versions -from .compat_sequence import ALL_KINDS, search +from .compat_sequence import ( + ALL_KINDS, + VECTOR_KIND, + VECTOR_OPS, + generate_vector, + search, +) + + +def test_vector_sequence_generation_is_bounded_and_covers_high_risk_orders(): + cases = list(generate_vector(max_length=5)) + combined = [tuple(setup + exercise) for setup, exercise in cases] + + assert cases + assert len(cases) < 128 + assert len(cases) == len(set((tuple(s), tuple(e)) for s, e in cases)) + assert all(exercise for _, exercise in cases) + assert all(1 <= len(sequence) <= 5 for sequence in combined) + + split_cases = {(tuple(setup), tuple(exercise)) for setup, exercise in cases} + for pair in product(VECTOR_OPS, repeat=2): + assert ((), pair) in split_cases + assert (pair[:1], pair[1:]) in split_cases + + two_delta_then_unindexed = ("W", "Ov", "W", "Ov", "W") + assert { + (tuple(setup), tuple(exercise)) + for setup, exercise in cases + if tuple(setup + exercise) == two_delta_then_unindexed + } == { + (two_delta_then_unindexed[:split], two_delta_then_unindexed[split:]) + for split in range(len(two_delta_then_unindexed)) + } def _default_refs(): @@ -35,10 +70,15 @@ def _default_refs(): FROM_REF = os.environ.get("COMPAT_FROM_REF") or _default_from TO_REF = os.environ.get("COMPAT_TO_REF") or _default_to MAX_LENGTH = int(os.environ.get("COMPAT_MAX_LENGTH", "4")) +VECTOR_MAX_LENGTH = int(os.environ.get("COMPAT_VECTOR_MAX_LENGTH", str(MAX_LENGTH))) KINDS = os.environ.get("COMPAT_KINDS", ",".join(ALL_KINDS)).split(",") # Many small shards (default 4x cores) so xdist's dynamic scheduler keeps every worker # busy and an oversubscribed `-n` has work to overlap. NUM_SHARDS = int(os.environ.get("COMPAT_SHARDS", str((os.cpu_count() or 1) * 4))) +# Training IVF_PQ once per generic shard would multiply total work without expanding +# coverage. Four cases still parallelize the bounded vector search while retaining +# snapshot reuse within each case. +VECTOR_NUM_SHARDS = int(os.environ.get("COMPAT_VECTOR_SHARDS", str(min(NUM_SHARDS, 4)))) def _cases(): @@ -53,25 +93,42 @@ def _cases(): return cases -CASES = _cases() -CASE_IDS = [k if v is None else f"{k}-fmtv{v}" for k, v in CASES] +def _search_cases(): + cases = [] + for kind, fts_version in _cases(): + num_shards = VECTOR_NUM_SHARDS if kind == VECTOR_KIND else NUM_SHARDS + kind_id = kind if fts_version is None else f"{kind}-fmtv{fts_version}" + cases.extend( + pytest.param( + kind, + fts_version, + shard, + num_shards, + id=f"{kind_id}-shard{shard}", + ) + for shard in range(num_shards) + ) + return cases + + +SEARCH_CASES = _search_cases() @pytest.mark.compat -@pytest.mark.parametrize("kind,fts_version", CASES, ids=CASE_IDS) -@pytest.mark.parametrize("shard", range(NUM_SHARDS)) +@pytest.mark.parametrize("kind,fts_version,shard,num_shards", SEARCH_CASES) def test_index_maintenance_sequence_search( - venv_factory, tmp_path, kind, fts_version, shard + venv_factory, tmp_path, kind, fts_version, shard, num_shards ): + max_length = VECTOR_MAX_LENGTH if kind == VECTOR_KIND else MAX_LENGTH failures = search( venv_factory, FROM_REF, TO_REF, tmp_path, kind, - max_length=MAX_LENGTH, + max_length=max_length, shard=shard, - num_shards=NUM_SHARDS, + num_shards=num_shards, fts_version=fts_version, ) # First line is the failure itself so it shows in pytest's bottom summary; the rest diff --git a/python/python/tests/compat/test_scalar_indices.py b/python/python/tests/compat/test_scalar_indices.py index c3bf301eee0..850c1212532 100644 --- a/python/python/tests/compat/test_scalar_indices.py +++ b/python/python/tests/compat/test_scalar_indices.py @@ -9,6 +9,7 @@ and written by other versions. """ +import os import shutil from pathlib import Path @@ -193,12 +194,17 @@ def __init__(self, path: Path): self.path = path def create(self): - """Create dataset with ZONEMAP and BLOOMFILTER indices.""" + """Create dataset with ZONEMAP and BLOOMFILTER indices. + + The zonemap column contains nulls at rows 0 and 500 so that IS NULL + queries can be verified across version boundaries. + """ shutil.rmtree(self.path, ignore_errors=True) + zonemap_values = [None if i in (0, 500) else i for i in range(1000)] data = pa.table( { "idx": pa.array(range(1000)), - "zonemap": pa.array(range(1000)), + "zonemap": pa.array(zonemap_values, type=pa.int64()), "bloomfilter": pa.array(range(1000)), } ) @@ -215,11 +221,22 @@ def check_read(self): """Verify ZONEMAP and BLOOMFILTER indices can be queried.""" ds = lance.dataset(self.path) - # Test ZONEMAP + # Test ZONEMAP equality table = ds.to_table(filter="zonemap == 7") assert table.num_rows == 1 assert table.column("idx").to_pylist() == [7] + # Test ZONEMAP IS NULL — two nulls were inserted at rows 0 and 500. + # Older versions without a null bitmap fall back to a zone scan, which + # is still correct; newer versions may return an exact result. + table = ds.to_table(filter="zonemap IS NULL") + if 1000 in table.column("idx").to_pylist(): + # After write, there are 3 NULLs + assert table.num_rows == 3 + else: + # Before write, there are 2 NULLs + assert table.num_rows == 2 + # Test BLOOMFILTER table = ds.to_table(filter="bloomfilter == 7") assert table.num_rows == 1 @@ -231,7 +248,7 @@ def check_write(self): data = pa.table( { "idx": pa.array([1000]), - "zonemap": pa.array([1000]), + "zonemap": pa.array([None], type=pa.int64()), "bloomfilter": pa.array([1000]), } ) @@ -239,6 +256,16 @@ def check_write(self): ds.optimize.optimize_indices() ds.optimize.compact_files() + # IS NULL must still return results after the index is updated and + # files are compacted. The newly inserted null must be found + # regardless of which version handles the seed-based index update. + table = ds.to_table(filter="zonemap IS NULL") + assert table.num_rows >= 1 + + def skip_downgrade(self, version: str) -> bool: + # In 0.X the zonemap index did not properly handle NULL in filters + return version.startswith("0.") + @compat_test(min_version="0.36.0") class JsonIndex(UpgradeDowngradeTest): @@ -320,9 +347,12 @@ def create(self): max_rows_per_file=100, data_storage_version=safe_data_storage_version(self.compat_version), ) - dataset.create_scalar_index( - "text", "INVERTED", with_position=True, format_version=1 - ) + kwargs = {"with_position": True} + # Downgrade reads use older wheels, so current-created FTS indexes must + # stay on the legacy posting block layout. + if os.environ.get("LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE") == "1": + kwargs["block_size"] = 128 + dataset.create_scalar_index("text", "INVERTED", format_version=1, **kwargs) def check_read(self): """Verify FTS index can be queried.""" @@ -351,6 +381,16 @@ def check_write(self): def skip_downgrade(self, version: str) -> bool: return version.startswith("0.") + def current_env(self, method_name: str) -> dict[str, str]: + if method_name == "create": + return { + "LANCE_COMPAT_FTS_LEGACY_BLOCK_SIZE": "1", + "LANCE_FTS_FORMAT_VERSION": "1", + } + if method_name == "check_write": + return {"LANCE_FTS_FORMAT_VERSION": "2"} + return {} + def compat_env(self, version: str, method_name: str) -> dict[str, str]: if method_name in {"create", "check_write"}: return {"LANCE_FTS_FORMAT_VERSION": "1"} diff --git a/python/python/tests/compat/test_venv_manager.py b/python/python/tests/compat/test_venv_manager.py index ebe3dfc766b..18a576bf088 100644 --- a/python/python/tests/compat/test_venv_manager.py +++ b/python/python/tests/compat/test_venv_manager.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +import subprocess + import pytest -from .venv_manager import _lance_namespace_dependency +from . import venv_manager +from .venv_manager import _lance_namespace_dependency, _pip_install @pytest.mark.parametrize( @@ -15,7 +18,64 @@ ("6.0.0", "lance-namespace>=0.7.2,<0.8"), ("7.2.0b5", "lance-namespace>=0.8.0,<0.9"), ("7.2.0", "lance-namespace>=0.8.0,<0.9"), + ("12.0.0b5", "lance-namespace>=0.8.0,<0.9"), + ("12.0.0b6", "lance-namespace>=0.11.1,<0.12"), + ("12.0.0", "lance-namespace>=0.11.1,<0.12"), ], ) def test_lance_namespace_dependency(version: str, expected: str): assert _lance_namespace_dependency(version) == expected + + +@pytest.fixture +def no_sleep(monkeypatch): + sleeps: list[float] = [] + monkeypatch.setattr(venv_manager.time, "sleep", sleeps.append) + return sleeps + + +def test_pip_install_failure_includes_pip_output(monkeypatch, no_sleep): + calls: list[list[str]] = [] + + def failing_run(cmd, **kwargs): + calls.append(cmd) + raise subprocess.CalledProcessError( + 2, cmd, output="resolving deps", stderr="No matching distribution found" + ) + + monkeypatch.setattr(venv_manager.subprocess, "run", failing_run) + with pytest.raises(RuntimeError) as exc_info: + _pip_install("python", ["pylance==0.38.0"]) + # The error must surface pip's captured output, which CalledProcessError hides. + message = str(exc_info.value) + assert "No matching distribution found" in message + assert "resolving deps" in message + assert "pylance==0.38.0" in message + assert len(calls) == venv_manager._PIP_INSTALL_ATTEMPTS + assert len(no_sleep) == venv_manager._PIP_INSTALL_ATTEMPTS - 1 + + +def test_pip_install_retries_transient_failure(monkeypatch, no_sleep): + calls: list[list[str]] = [] + + def flaky_run(cmd, **kwargs): + calls.append(cmd) + if len(calls) == 1: + raise subprocess.CalledProcessError( + 2, cmd, output="", stderr="ReadTimeoutError: pypi.fury.io" + ) + + monkeypatch.setattr(venv_manager.subprocess, "run", flaky_run) + _pip_install("python", ["pylance==0.38.0"]) + assert len(calls) == 2 + assert len(no_sleep) == 1 + + +def test_pip_install_succeeds_first_try_without_sleeping(monkeypatch, no_sleep): + calls: list[list[str]] = [] + monkeypatch.setattr( + venv_manager.subprocess, "run", lambda cmd, **kwargs: calls.append(cmd) + ) + _pip_install("python", ["pytest"]) + assert calls == [["python", "-m", "pip", "install", "--quiet", "pytest"]] + assert no_sleep == [] diff --git a/python/python/tests/compat/venv_manager.py b/python/python/tests/compat/venv_manager.py index c4b23486cd3..6b1e3b85e2d 100644 --- a/python/python/tests/compat/venv_manager.py +++ b/python/python/tests/compat/venv_manager.py @@ -17,9 +17,11 @@ import struct import subprocess import sys +import time from pathlib import Path -from typing import Any, Optional +from typing import Any, Optional, Union +import pytest from packaging.version import InvalidVersion, Version try: @@ -42,12 +44,46 @@ def _venv_lock(lock_path: Path): fcntl.flock(handle, fcntl.LOCK_UN) +_PIP_INSTALL_ATTEMPTS = 3 +_PIP_RETRY_BACKOFF_SECONDS = 2.0 + + +def _pip_install(python: Union[str, Path], args: list[str]) -> None: + """Run ``pip install`` with output captured, retrying transient failures. + + pip installs hit the network (PyPI / fury.io) and a single flaky download + should not fail a whole test run, so retry a bounded number of times with a + short backoff. On final failure raise an error that includes pip's + stdout/stderr, which a bare CalledProcessError from ``capture_output=True`` + would hide from the test log. + """ + cmd = [str(python), "-m", "pip", "install", "--quiet", *args] + for attempt in range(1, _PIP_INSTALL_ATTEMPTS + 1): + try: + subprocess.run(cmd, check=True, capture_output=True, text=True) + return + except subprocess.CalledProcessError as exc: # noqa: PERF203 + if attempt == _PIP_INSTALL_ATTEMPTS: + raise RuntimeError( + f"pip install failed after {attempt} attempts " + f"(exit status {exc.returncode}): {cmd}\n" + f"stdout:\n{exc.stdout}\n" + f"stderr:\n{exc.stderr}" + ) from exc + time.sleep(_PIP_RETRY_BACKOFF_SECONDS * attempt) + + NAMESPACE_0_6_DEPENDENCY = "lance-namespace<0.7" NAMESPACE_0_7_DEPENDENCY = "lance-namespace>=0.7.2,<0.8" NAMESPACE_0_8_DEPENDENCY = "lance-namespace>=0.8.0,<0.9" +NAMESPACE_0_11_DEPENDENCY = "lance-namespace>=0.11.1,<0.12" def _lance_namespace_dependency(pylance_version: str) -> str: + # 12.0.0b5 is the last release published while pylance still pinned + # lance-namespace <0.9; releases cut after that carry the 0.11 range. + if Version(pylance_version) > Version("12.0.0b5"): + return NAMESPACE_0_11_DEPENDENCY if Version(pylance_version) >= Version("7.2.0b5"): return NAMESPACE_0_8_DEPENDENCY if Version(pylance_version) >= Version("6.0.0b0"): @@ -96,6 +132,36 @@ def _safe(ref: str) -> str: return re.sub(r"[^A-Za-z0-9._-]", "_", ref) +# Explicit skip whitelist for known (lance_version, python_version) incompatibilities. +# Each entry is (max_lance_version_inclusive, min_python_version_inclusive). +# A test is skipped when the tested lance version <= max_lance AND the current +# Python >= min_python. Add entries here only after verifying the incompatibility +# is a runtime/ABI issue rather than a format regression — anything NOT listed +# will surface as a test failure so new problems are visible. +_COMPAT_SKIP: list[tuple[str, tuple[int, int]]] = [ + # lance 0.22.x abi3 wheel was built with old PyO3 (<0.23) that crashes on + # Python 3.14+ due to removed internal CPython APIs. + ("0.22.0", (3, 14)), +] + + +def _skip_reason(lance_version: str) -> Optional[str]: + """Return a skip reason if this lance/Python combo is whitelisted, else None.""" + try: + ver = Version(lance_version) + except InvalidVersion: + return None + py = sys.version_info[:2] + for max_lance, min_python in _COMPAT_SKIP: + if ver <= Version(max_lance) and py >= min_python: + py_str = f"{py[0]}.{py[1]}" + return ( + f"Lance {lance_version} + Python {py_str}: whitelisted skip " + f"(see _COMPAT_SKIP in venv_manager.py)" + ) + return None + + class VenvExecutor: """Manages a virtual environment with a specific Lance version.""" @@ -130,15 +196,29 @@ def python_path(self) -> Path: def _marker_path(self) -> Path: return self.venv_path / ".compat_ref" + @staticmethod + def _python_version_tag() -> str: + return f"{sys.version_info.major}.{sys.version_info.minor}" + def _validate_venv(self) -> bool: - """A cached venv is reusable if it exists and its recorded ref matches. A marker - file is used (not `pip show`) so source-built commit refs also validate.""" + """A cached venv is reusable if it exists, its recorded ref matches, and it was + built with the same Python major.minor as the current interpreter. + + The marker file format is two lines: `\\n`. + Old single-line markers (no Python version) are treated as stale so the venv + is rebuilt — this handles cached venvs from a different Python installation.""" if not self.python_path.exists(): return False try: - return self._marker_path.read_text().strip() == self.version + lines = self._marker_path.read_text().strip().splitlines() except OSError: return False + if not lines or lines[0] != self.version: + return False + # Require a Python version line; single-line markers are stale. + if len(lines) < 2 or lines[1] != self._python_version_tag(): + return False + return True def create(self): """Create the virtual environment and install the specified Lance version.""" @@ -170,24 +250,18 @@ def create(self): self._install_release_wheel() else: self._build_from_source() - self._marker_path.write_text(self.version) + self._marker_path.write_text( + f"{self.version}\n{self._python_version_tag()}" + ) self._created = True def _install_wheel(self, wheel: str): - subprocess.run( - [str(self.python_path), "-m", "pip", "install", "--quiet", wheel, "pytest"], - check=True, - capture_output=True, - ) + _pip_install(self.python_path, [wheel, "pytest"]) def _install_release_wheel(self): - subprocess.run( + _pip_install( + self.python_path, [ - str(self.python_path), - "-m", - "pip", - "install", - "--quiet", "--pre", "--extra-index-url", "https://pypi.fury.io/lance-format/", @@ -201,8 +275,6 @@ def _install_release_wheel(self): _lance_namespace_dependency(self.version), "pytest", ], - check=True, - capture_output=True, ) def _build_from_source(self): @@ -225,11 +297,7 @@ def _build_from_source(self): check=True, capture_output=True, ) - subprocess.run( - [py, "-m", "pip", "install", "--quiet", "maturin", "pytest", "pyarrow"], - check=True, - capture_output=True, - ) + _pip_install(py, ["maturin", "pytest", "pyarrow"]) wheels = src / "target" / "compat-wheels" subprocess.run( [ @@ -249,11 +317,7 @@ def _build_from_source(self): capture_output=True, ) wheel = next(wheels.glob("pylance-*.whl")) - subprocess.run( - [py, "-m", "pip", "install", "--quiet", str(wheel)], - check=True, - capture_output=True, - ) + _pip_install(py, [str(wheel)]) def _ensure_subprocess(self): """Ensure the persistent subprocess is running.""" @@ -264,10 +328,13 @@ def _ensure_subprocess(self): # Start persistent subprocess runner_script = Path(__file__).parent / "venv_runner.py" - # Set PYTHONPATH to include the tests directory + # Set PYTHONPATH so the subprocess can import compat test modules. + # pytest adds the `tests/` directory (the first ancestor without __init__.py) + # to sys.path, so test modules are imported as `compat.`. env = os.environ.copy() tests_dir = Path(__file__).parent.parent env["PYTHONPATH"] = str(tests_dir) + env.setdefault("RUST_BACKTRACE", "full") # Capture stderr to a file so a Rust panic (which crashes the runner) can be # surfaced in the error instead of an opaque "broken pipe". @@ -357,6 +424,10 @@ def execute_method( if not self._created: raise RuntimeError("Virtual environment not created. Call create() first.") + reason = _skip_reason(self.version) + if reason: + pytest.skip(reason) + # Ensure subprocess is running self._ensure_subprocess() try: @@ -379,14 +450,16 @@ def execute_method( except (BrokenPipeError, EOFError, struct.error) as e: # Subprocess died (usually a Rust panic); flush it, then surface that. + returncode = "unknown" if self._subprocess is not None: try: self._subprocess.wait(timeout=2) + returncode = str(self._subprocess.returncode) except Exception: pass panic = self._last_panic() detail = panic or f"subprocess communication failed: {e}" - raise RuntimeError(f"Lance {self.version}: {detail}") + raise RuntimeError(f"Lance {self.version} (exit={returncode}): {detail}") def cleanup(self): """Remove the virtual environment directory and terminate subprocess.""" @@ -404,8 +477,6 @@ def cleanup(self): # Remove venv directory if self.venv_path.exists(): - import shutil - shutil.rmtree(self.venv_path) self._created = False diff --git a/python/python/tests/conftest.py b/python/python/tests/conftest.py index 3790535efc7..c9d8911fd2b 100644 --- a/python/python/tests/conftest.py +++ b/python/python/tests/conftest.py @@ -102,8 +102,41 @@ def pytest_configure(config): "compat: mark tests that run upgrade/downgrade compatibility checks", ) + workerinput = getattr(config, "workerinput", None) + if workerinput is not None: + from compat.compat_decorator import use_version_snapshot + use_version_snapshot(workerinput["lance_compat_versions"]) + + +def pytest_configure_node(node): + """Resolve the compat version list once, on the xdist controller. + + The compat tests are parametrized over the pylance releases published to + PyPI and fury.io, and collecting `python/tests` imports them whether or not + --run-compat was passed. Left to itself every worker queries for that list + while collecting, so a request that times out or a release that lands + mid-run gives one worker a different parameter set, and xdist aborts the + whole run with "Different tests were collected". + """ + from compat.compat_decorator import version_snapshot + + node.workerinput["lance_compat_versions"] = version_snapshot() + + +# tryfirst because xdist reads xdist_group off each item to build its scheduling +# groups before ordinary pytest_collection_modifyitems hooks run; a mark added +# later is silently ignored rather than rejected. +@pytest.hookimpl(tryfirst=True) def pytest_collection_modifyitems(config, items): + # The lindera fixture unzips a dictionary into the checked-out models tree and + # removes it again on teardown, and the tokenizer configs name that path + # relative to the repo, so it cannot be relocated per worker. Pinning these + # tests to one xdist worker keeps a teardown in one worker from deleting the + # dictionary another is still reading. Without -n it changes nothing. + for item in items: + if "lindera_ipadic" in getattr(item, "fixturenames", ()): + item.add_marker(pytest.mark.xdist_group("lindera")) if not config.getoption("--run-integration"): disable_items_with_mark(items, "integration", "--run-integration not specified") if not config.getoption("--run-slow"): diff --git a/python/python/tests/recurring/test_recurring.py b/python/python/tests/recurring/test_recurring.py deleted file mode 100644 index ab39269528a..00000000000 --- a/python/python/tests/recurring/test_recurring.py +++ /dev/null @@ -1,231 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - -# Recurring tests that runs all operations on a large dataset, -# these operations are ran in random order repeated 10 times - -import abc -import itertools -from datetime import timedelta -from typing import Optional - -import lance -import numpy as np -import pyarrow as pa -import pytest - -# For testing, use smaller numbers to make tests run faster -# In production, you might want to use: NUM_ROWS = 1_000_000 -NUM_ROWS = 1_000_000 -BATCH_SIZE = 1_000 -DIM = 32 - -schema = pa.schema( - [ - pa.field("id", pa.int64()), - pa.field("vector", pa.list_(pa.float32(), DIM)), - pa.field("text", pa.string()), - ] -) -words = ["hello", "world", "this", "is", "a", "test", "sentence"] - - -def random_text(num_words: int) -> str: - return " ".join(np.random.choice(words, num_words)) - - -def random_batch(start_id: int, batch_size: int) -> pa.Table: - return pa.Table.from_arrays( - [ - pa.array(np.arange(start_id, start_id + batch_size)), - pa.array(np.random.rand(batch_size, DIM).tolist()), - pa.array( - [random_text(np.random.randint(1, 10)) for _ in range(batch_size)] - ), - ], - schema=schema, - ) - - -def create_or_load_dataset(dataset_name: str, kwargs: dict): - uri = f"tests/recurring/{dataset_name}" - - # Try to open existing dataset first - try: - ds = lance.dataset(uri) - if ds.count_rows() > 0: - return ds - except Exception: - pass - - # Create new dataset with initial data - initial_batch = random_batch(0, BATCH_SIZE) - ds = lance.write_dataset(initial_batch, uri, schema=schema, mode="overwrite") - - # Add remaining data - for i in range(BATCH_SIZE, NUM_ROWS, BATCH_SIZE): - batch = random_batch(i, BATCH_SIZE) - ds.insert(batch) - - # Create indices - ds.create_scalar_index("id", index_type="BTREE", replace=True) - ds.create_index( - "vector", - index_type="IVF_PQ", - metric="cosine", - num_partitions=128, - num_sub_vectors=DIM // 8, - replace=True, - ) - - # Note: FTS index creation is async, but we'll handle this differently for pytest - # For now, we'll skip the async part and create it synchronously if possible - try: - ds.create_scalar_index( - "text", - index_type="INVERTED", - with_position=kwargs.get("with_position", False), - replace=True, - ) - except Exception as e: - print(f"Warning: Could not create FTS index: {e}") - - return ds - - -class Operation(abc.ABC): - @abc.abstractmethod - def read_only(self) -> bool: ... - - @abc.abstractmethod - def run(self, ds: lance.LanceDataset): ... - - -class ReadOnlyOperation(Operation): - def read_only(self) -> bool: - return True - - -class WriteOperation(Operation): - def read_only(self) -> bool: - return False - - -class Append(WriteOperation): - def run(self, ds: lance.LanceDataset): - batch = random_batch(ds.count_rows(), BATCH_SIZE) - ds.insert(batch) - - -class Delete(WriteOperation): - def __init__(self, delete_num_rows: int = 100): - self.delete_num_rows = delete_num_rows - - def run(self, ds: lance.LanceDataset): - num_rows = ds.count_rows() - to_delete = np.random.randint(0, num_rows, self.delete_num_rows) - to_delete = ", ".join([str(v) for v in to_delete]) - ds.delete(f"id IN ({to_delete})") - - -class Optimize(WriteOperation): - def __init__(self, num_indices_to_merge: int, column: str): - self.num_indices_to_merge = num_indices_to_merge - self.column = column - - def run(self, ds: lance.LanceDataset): - ds.optimize.optimize_indices( - num_indices_to_merge=self.num_indices_to_merge, - index_names=[f"{self.column}_idx"], - ) - - -class Compact(WriteOperation): - def run(self, ds: lance.LanceDataset): - ds.optimize.compact_files() - - -class VectorSearch(ReadOnlyOperation): - def __init__(self, filter: Optional[str] = None): - self.filter = filter - - def run(self, ds: lance.LanceDataset): - stats = ds.stats.index_stats("vector_idx") - if stats is None: - print("No vector index found") - return - query_vector = np.random.rand(DIM).tolist() - query = ds.scanner( - nearest={ - "q": query_vector, - "k": 10, - "column": "vector", - }, - filter=self.filter, - ) - query.analyze_plan() - - -class FullTextSearch(ReadOnlyOperation): - def __init__(self, has_position: bool, filter: Optional[str] = None): - self.has_position = has_position - self.filter = filter - - def run(self, ds: lance.LanceDataset): - stats = ds.stats.index_stats("text_idx") - if stats is None: - print("No text index found") - return - query_text = random_text(np.random.randint(1, 10)) - self.do_query(ds, query_text) - - if self.has_position: - query_text = f'"{query_text}"' - self.do_query(ds, query_text) - - def do_query(self, ds: lance.LanceDataset, query_text: str): - query: lance.LanceScanner = ds.scanner( - full_text_query=query_text, - filter=self.filter, - limit=10, - ) - query.analyze_plan() - - -@pytest.mark.recurring -@pytest.mark.parametrize("with_position", [True]) -def test_all_permutations(with_position): - """Test all operations on dataset without FTS position tracking""" - dataset_name = f"test_table_with_position_{with_position}" - ds = create_or_load_dataset(dataset_name, {"with_position": with_position}) - - write_operations = [ - Append(), - Delete(delete_num_rows=1000), - Optimize(num_indices_to_merge=0, column="id"), - Optimize(num_indices_to_merge=0, column="vector"), # delta index - Optimize(num_indices_to_merge=1, column="vector"), # merge index - Optimize(num_indices_to_merge=0, column="text"), - Compact(), - ] - - read_only_operations = [ - # Read only operations - VectorSearch(), - VectorSearch(filter="id >= 1000 and id < 8000"), - FullTextSearch(has_position=False), - FullTextSearch(has_position=False, filter="id >= 1000 and id < 8000"), - ] - - for permutation in itertools.permutations(range(len(write_operations))): - for idx in permutation: - write_operation = write_operations[idx] - print(f"Running {write_operation.__class__.__name__}") - write_operation.run(ds) - ds.cleanup_old_versions(older_than=timedelta(seconds=0)) - - # write operation changed the status of the table, - # then we need to run all read only operations after it - for read_only_operation in read_only_operations: - print(f"Running {read_only_operation.__class__.__name__}") - read_only_operation.run(ds) diff --git a/python/python/tests/test_arrow.py b/python/python/tests/test_arrow.py index 92ab52021ff..a0be09846e1 100644 --- a/python/python/tests/test_arrow.py +++ b/python/python/tests/test_arrow.py @@ -273,8 +273,6 @@ def test_image_uri_arrays(tmp_path: Path, png_uris): def test_image_tensor_arrays(tmp_path: Path, png_uris): - tf = pytest.importorskip("tensorflow") - n = 10 encoded_image_array = ImageURIArray.from_uris(png_uris).read_uris() @@ -297,22 +295,22 @@ def test_image_tensor_arrays(tmp_path: Path, png_uris): assert tensor_image_array.storage.type == pa.list_(pa.uint8(), 4) assert tensor_image_array[2].as_py() == [42, 42, 42, 255] - test_tensor = tf.constant( - np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4)) - ) + test_tensor = np.array([42, 42, 42, 255] * n, dtype=np.uint8).reshape((n, 1, 1, 4)) assert test_tensor.shape == (n, 1, 1, 4) - assert tf.math.reduce_all( - tf.convert_to_tensor(tensor_image_array.to_numpy()) == test_tensor - ) + assert np.array_equal(tensor_image_array.to_numpy(), test_tensor) assert tensor_image_array.to_encoded().to_tensor() == tensor_image_array def png_encoder(images): - import tensorflow as tf + import io - encoded_images = ( - tf.io.encode_png(x).numpy() for x in tf.convert_to_tensor(images) - ) + from PIL import Image # pyright: ignore[reportMissingImports] + + encoded_images = [] + for image in images: + with io.BytesIO() as buf: + Image.fromarray(image).save(buf, format="PNG") + encoded_images.append(buf.getvalue()) return pa.array(encoded_images, type=pa.binary()) assert tensor_image_array.to_encoded(png_encoder).to_tensor() == tensor_image_array @@ -324,20 +322,18 @@ def png_encoder(images): uris = [str(Path(x)) for x in uris] encoded_image_array = ImageArray.from_array(uris).read_uris() - with pytest.raises( - tf.errors.InvalidArgumentError, match="Shapes of all inputs must match" - ): + with pytest.raises(ValueError, match="all input arrays must have the same shape"): encoded_image_array.to_tensor() pattern = r"(object at) 0x[\w\d]+(:?>)" repl = r"\1 0x..\2" - assert re.sub(pattern, repl, encoded_image_array.__repr__()) == ( - "\n" - "[, ..]\n" + repr_ = re.sub(pattern, repl, encoded_image_array.__repr__()) + assert repr_.startswith("\n[ IndexMetadata PyO3 conversion. + committed_txn = dataset_without_index.get_transactions(1)[0] + committed_index = committed_txn.operation.new_indices[0] + assert committed_index.covering_fields == [price_id] + + # Exercises indices.rs: the IndexMetadata -> PyIndexSegmentDescription + # conversion used by describe_indices(). + segment = dataset_without_index.describe_indices()[0].segments[0] + assert segment.covering_fields == [price_id] + + +def test_commit_index_rejects_invalid_covering_fields(dataset_with_index, tmp_path): + """The invariant is enforced at commit, not at construction. + + LanceOperation.CreateIndex is a plain dataclass; it only reaches Rust at + commit(), so a construction-time gate would miss this path entirely. + """ + from lance.dataset import Index + + index_id = dataset_with_index.describe_indices()[0].segments[0].uuid + field_id = _get_field_id_by_name(dataset_with_index.lance_schema, "meta") + + # Constructing it is fine -- this is a dataclass, nothing is validated. + index = Index( + uuid=index_id, + name="meta_idx", + fields=[field_id], + dataset_version=dataset_with_index.version, + fragment_ids=set([f.fragment_id for f in dataset_with_index.get_fragments()]), + index_version=0, + covering_fields=[field_id], # covers the only field -- degenerate + ) + + create_index_op = lance.LanceOperation.CreateIndex( + new_indices=[index], + removed_indices=[], + ) + + with pytest.raises(OSError, match="at least one field must remain indexed"): + lance.LanceDataset.commit( + dataset_with_index.uri, + create_index_op, + read_version=dataset_with_index.version, + ) diff --git a/python/python/tests/test_compat_version_snapshot.py b/python/python/tests/test_compat_version_snapshot.py new file mode 100644 index 00000000000..ce2ccc3d405 --- /dev/null +++ b/python/python/tests/test_compat_version_snapshot.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Coverage for the version snapshot the compat tests are parametrized over.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +TESTS_DIR = Path(__file__).resolve().parent + +# Returns an extra release on gw1 only, so a worker that resolves the list itself +# parametrizes differently from the controller and from its sibling. +STUB_PLUGIN = """ +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) + +from packaging.version import Version + +from compat import compat_decorator + + +def _fetch_stable_versions(): + extra = ["8.0.0"] if os.environ.get("PYTEST_XDIST_WORKER") == "gw1" else [] + return sorted(Version(v) for v in ["6.0.0", "7.0.0", *extra]) + + +def _fetch_last_beta_release(): + return "7.1.0b1" + + +compat_decorator._fetch_stable_versions = _fetch_stable_versions +compat_decorator._fetch_last_beta_release = _fetch_last_beta_release +""" + +# Not run with --run-compat: the generated cases stay collected but skipped, which +# is what makes their ids part of the collection xdist compares between workers. +INNER_TEST = """ +import os + +from compat import compat_decorator +from compat.compat_decorator import UpgradeDowngradeTest, compat_test + +SEEDED_AT_IMPORT = compat_decorator._SNAPSHOT is not None + + +@compat_test() +class Sample(UpgradeDowngradeTest): + def __init__(self, path): + self.path = path + + +def test_snapshot_arrived_before_collection(): + assert os.environ["PYTEST_XDIST_WORKER"] + assert SEEDED_AT_IMPORT, "worker resolved the version list itself" +""" + + +def test_workers_share_the_controller_version_snapshot(tmp_path): + """Every xdist worker parametrizes on the releases the controller resolved. + + The compat suite discovers pylance releases over the network, so a worker + left to query for them itself can collect a different parameter set than its + siblings and abort the run. This pins both halves of the fix: that the + snapshot reaches the worker at all, and that it arrives before collection + imports any test module. + """ + root = tmp_path / "inner" + (root / "compat").mkdir(parents=True) + shutil.copy(TESTS_DIR / "conftest.py", root / "conftest.py") + shutil.copy( + TESTS_DIR / "compat" / "compat_decorator.py", + root / "compat" / "compat_decorator.py", + ) + (root / "compat" / "__init__.py").touch() + (root / "stubnet.py").write_text(STUB_PLUGIN) + (root / "test_snapshot.py").write_text(INNER_TEST) + + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "-q", + "-p", + "no:cacheprovider", + "-p", + "stubnet", + "-n", + "2", + "--dist", + "loadgroup", + ], + cwd=root, + capture_output=True, + text=True, + ) + output = result.stdout + result.stderr + + assert "Different tests were collected" not in output, output + assert "8.0.0" not in output, output + assert result.returncode == 0, output diff --git a/python/python/tests/test_custom_registry.py b/python/python/tests/test_custom_registry.py new file mode 100644 index 00000000000..9e38f47f693 --- /dev/null +++ b/python/python/tests/test_custom_registry.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Smoke test for the runtime object-store scheme registration hook. + +Exercises the pyo3 additions in ``python/src/object_store.rs`` and the new +``store_registry=`` parameter on ``Session(...)``. Registers the built-in +``MemoryStoreProvider`` under a made-up scheme (``test-mem``) and proves that +``lance.write_dataset(...)`` and ``lance.dataset(...)`` route reads and writes +through the registry-selected store rather than the default built-in scheme +resolver. + +The full Python-to-Rust ``ObjectStoreProvider`` callable bridge is not exercised +here — that is a follow-up. See the module docstring in +``python/src/object_store.rs``. +""" + +import lance +import pyarrow as pa +import pytest +from lance.lance import ( + _ObjectStoreProvider, + _ObjectStoreRegistry, + _Session, +) + + +def _make_registry_and_session(scheme: str) -> tuple[_ObjectStoreRegistry, _Session]: + """Return a fresh registry with ``scheme`` bound to an in-memory provider, + plus a Session that consults it. + """ + registry = _ObjectStoreRegistry() + provider = _ObjectStoreProvider.memory() + registry.register_provider(scheme, provider) + session = _Session(store_registry=registry) + return registry, session + + +def test_custom_scheme_registration_roundtrip(): + """Register ``test-mem`` and round-trip a small table through it. + + Note on lifetimes: ``ObjectStoreRegistry`` caches active stores under + ``Weak``. Every call to ``MemoryStoreProvider::new_store`` + allocates a fresh ``InMemory`` backend, so the writer's dataset handle + must stay alive across the read to keep the same in-memory store visible + from the reader. + """ + _registry, session = _make_registry_and_session("test-mem") + + table = pa.table( + { + "i": pa.array([1, 2, 3, 4, 5], type=pa.int64()), + "s": pa.array(["a", "b", "c", "d", "e"], type=pa.string()), + } + ) + uri = "test-mem://cache/dataset.lance" + + # Keep the write-side dataset alive across the read so the shared + # ObjectStore held by the registry's Weak cache remains upgradeable. + written = lance.write_dataset(table, uri, session=session) + assert written.count_rows() == 5 + + read_ds = lance.dataset(uri, session=session) + round_trip = read_ds.to_table() + + assert round_trip.equals(table), ( + "round-trip through test-mem:// scheme did not match the written table" + ) + assert read_ds.count_rows() == 5 + + +def test_registry_repr_and_reuse(): + """Registry ``__repr__`` reports cache stats, and reusing a registered + scheme with the same params returns a cached store rather than a new one. + """ + registry, session = _make_registry_and_session("test-mem-reuse") + + table = pa.table({"i": pa.array([1, 2, 3], type=pa.int64())}) + uri = "test-mem-reuse://cache/reuse.lance" + + written = lance.write_dataset(table, uri, session=session) + + stats_after_write = repr(registry) + assert "active_stores=" in stats_after_write + assert "hits=" in stats_after_write + assert "misses=" in stats_after_write + + # Re-open the same URI with the same session. This should hit the + # registry's active-stores cache (weak ref still upgradeable because + # ``written`` holds a strong ref). + _reopened = lance.dataset(uri, session=session) + stats_after_reopen = repr(registry) + + # We do not assert exact hit counts — the registry accounts for both the + # write- and read-path resolutions — but the reuse call must not have + # produced additional active-store entries. + _ = stats_after_reopen # kept for post-mortem debugging when running verbose + _ = written # keep the write handle alive until the assertion above passes + + +def test_missing_scheme_raises_helpful_error(): + """A URI whose scheme is neither built-in nor registered must raise, and + the error message must name the missing scheme. + """ + # Fresh registry with nothing custom registered. + session = _Session(store_registry=_ObjectStoreRegistry()) + + with pytest.raises(Exception) as excinfo: # OSError or lance.LanceError + lance.dataset("no-such-scheme://foo/bar.lance", session=session) + + assert "no-such-scheme" in str(excinfo.value), ( + "expected the missing scheme name in the error, got: " + str(excinfo.value) + ) + + +def test_register_provider_rejects_empty_scheme(): + """Empty schemes are rejected at registration time.""" + registry = _ObjectStoreRegistry() + with pytest.raises(ValueError): + registry.register_provider("", _ObjectStoreProvider.memory()) + + +def test_from_capsule_roundtrip(): + """Exercise the ``PyCapsule`` handoff end to end with an in-process producer. + + ``_memory_capsule()`` mirrors what an external, ABI-compatible wheel emits: + a ``PyCapsule`` named ``lance_object_store_provider`` holding an + ``Arc``. ``from_capsule`` must accept it (name + check passes), adopt the provider, and the result must be registrable and + usable for a read/write round-trip — covering the clone-on-adopt and the + capsule's own drop when it is garbage-collected. + """ + registry = _ObjectStoreRegistry() + capsule = _ObjectStoreProvider._memory_capsule() + provider = _ObjectStoreProvider.from_capsule(capsule) + # Drop our reference to the capsule; the adopted Arc must keep the provider + # alive independently of the capsule. + del capsule + registry.register_provider("test-cap", provider) + session = _Session(store_registry=registry) + + table = pa.table({"i": pa.array([1, 2, 3, 4], type=pa.int64())}) + uri = "test-cap://cache/cap.lance" + written = lance.write_dataset(table, uri, session=session) + assert written.count_rows() == 4 + + read_ds = lance.dataset(uri, session=session) + assert read_ds.to_table().equals(table) + + +def test_from_capsule_rejects_wrong_name(): + """A capsule whose name does not match is rejected before its pointer is + ever dereferenced (regression for the ``pointer_checked(None)`` bug, which + rejected *every* correctly-named capsule). + """ + import ctypes + + make = ctypes.pythonapi.PyCapsule_New + make.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p] + make.restype = ctypes.py_object + # A capsule with the wrong name and a dummy pointer. The name check must + # fail first, so the bogus pointer is never read. + storage = (ctypes.c_void_p * 2)() + capsule = make(ctypes.addressof(storage), b"not-a-lance-provider", None) + + with pytest.raises(ValueError): + _ObjectStoreProvider.from_capsule(capsule) diff --git a/python/python/tests/test_datagen.py b/python/python/tests/test_datagen.py index bdc205a3187..219075eaae0 100644 --- a/python/python/tests/test_datagen.py +++ b/python/python/tests/test_datagen.py @@ -7,6 +7,14 @@ import pyarrow as pa import pytest +SCHEMA = pa.schema( + [ + pa.field("int", pa.int64()), + pa.field("vector", pa.list_(pa.float32(), 128)), + ] +) +BYTES_PER_ROW = 8 + 128 * 4 + @pytest.mark.skipif(datagen.is_datagen_supported(), reason="datagen is supported") def test_import_error(): @@ -17,19 +25,29 @@ def test_import_error(): @pytest.mark.skipif(not datagen.is_datagen_supported(), reason="datagen not supported") -def test_rand_batches(): - import lance._datagen as datagen +def test_rand_batches_by_bytes(): + reader = datagen.rand_batches(SCHEMA, batch_size_bytes=16 * 1024, num_batches=10) - schema = pa.schema( - [ - pa.field("int", pa.int64()), - pa.field("vector", pa.list_(pa.float32(), 128)), - ] - ) + batches = list(reader) + assert len(batches) == 10 + for batch in batches: + assert batch.num_rows == math.ceil(16 * 1024 / BYTES_PER_ROW) + assert batch.schema == SCHEMA - batches = datagen.rand_batches(schema, batch_size_bytes=16 * 1024, num_batches=10) - assert len(batches) == 10 +@pytest.mark.skipif(not datagen.is_datagen_supported(), reason="datagen not supported") +@pytest.mark.parametrize("rows_per_batch", [1, 100]) +def test_rand_batches_by_rows(rows_per_batch): + reader = datagen.rand_batches(SCHEMA, rows_per_batch=rows_per_batch, num_batches=3) + + batches = list(reader) + assert len(batches) == 3 for batch in batches: - assert batch.num_rows == math.ceil(16 * 1024 / (129 * 4)) - assert batch.schema == schema + assert batch.num_rows == rows_per_batch + assert batch.schema == SCHEMA + + +@pytest.mark.skipif(not datagen.is_datagen_supported(), reason="datagen not supported") +def test_rand_batches_rejects_both_sizes(): + with pytest.raises(ValueError, match="mutually exclusive"): + datagen.rand_batches(SCHEMA, batch_size_bytes=16 * 1024, rows_per_batch=100) diff --git a/python/python/tests/test_dataset.py b/python/python/tests/test_dataset.py index e449c6b9865..7891f7e4215 100644 --- a/python/python/tests/test_dataset.py +++ b/python/python/tests/test_dataset.py @@ -3,6 +3,7 @@ import base64 import contextlib +import importlib import os import pickle import platform @@ -12,7 +13,7 @@ import uuid from datetime import date, datetime, timedelta from pathlib import Path -from typing import List +from typing import List, Optional from unittest import mock import lance @@ -28,15 +29,23 @@ import pytest from helper import ProgressForTest from lance._dataset.sharded_batch_iterator import ShardedBatchIterator -from lance.commit import CommitConflictError from lance.dataset import LANCE_COMMIT_MESSAGE_KEY, AutoCleanupConfig from lance.debug import format_fragment -from lance.file import stable_version +from lance.file import LanceFileWriter, stable_version from lance.schema import LanceSchema from lance.util import validate_vector_index +BaseModel = pytest.importorskip("pydantic").BaseModel + # Various valid inputs for write_dataset input_schema = pa.schema([pa.field("a", pa.float64()), pa.field("b", pa.int64())]) + + +class _InputModel(BaseModel): + a: float + b: int + + input_data = [ # (schema, data) (None, pa.table({"a": [1.0, 2.0], "b": [20, 30]})), @@ -59,6 +68,8 @@ ).to_batches() ), ), + # Pydantic model instances are auto-converted + (None, [_InputModel(a=1.0, b=20), _InputModel(a=2.0, b=30)]), ] @@ -66,7 +77,138 @@ def test_input_data(tmp_path: Path, schema, data): base_dir = tmp_path / "test" dataset = lance.write_dataset(data, base_dir, schema=schema) - assert dataset.to_table() == input_data[0][1] + expected = input_data[0][1] + # Pydantic model instances derive their schema from the model class, so + # required (non-Optional) fields are correctly non-nullable -- stricter + # than, but castable to, the nullable reference schema below. + assert dataset.to_table().cast(expected.schema) == expected + + +def test_from_pydantic_model(tmp_path: Path): + class UserRecord(BaseModel): + name: str + score: float + + data = [UserRecord(name="alice", score=0.9), UserRecord(name="bob", score=0.8)] + uri = str(tmp_path / "user_record") + ds = lance.LanceDataset.from_pydantic_model(UserRecord, data, uri=uri) + + table = ds.to_table() + assert table.num_rows == 2 + assert table.schema.names == ["name", "score"] + assert table.column("name").to_pylist() == ["alice", "bob"] + assert table.column("score").to_pylist() == [0.9, 0.8] + + +def test_from_pydantic_model_rejects_non_model_class(tmp_path: Path): + uri = str(tmp_path / "not_a_model") + with pytest.raises(TypeError, match="BaseModel subclass"): + lance.LanceDataset.from_pydantic_model(dict, [{"a": 1}], uri=uri) + + +class _OtherRecord(BaseModel): + x: int + + +@pytest.mark.parametrize( + "bad_item", + [{"name": "eve", "score": 1.0}, _OtherRecord(x=1)], + ids=["dict", "different_model"], +) +def test_from_pydantic_model_rejects_invalid_item(tmp_path: Path, bad_item): + """A dict or an instance of a different model slipped into `data` must be + rejected up front with the offending index, rather than either reaching + model_to_dict() and raising an opaque AttributeError (dict case) or being + silently serialized against the first item's schema (different-model + case).""" + + class UserRecord(BaseModel): + name: str + score: float + + data = [UserRecord(name="alice", score=0.9), bad_item] + uri = str(tmp_path / "invalid_item") + with pytest.raises(TypeError, match=r"data\[1\]"): + lance.LanceDataset.from_pydantic_model(UserRecord, data, uri=uri) + + +@pytest.mark.parametrize( + "bad_item", + [{"name": "eve"}, _OtherRecord(x=1)], + ids=["dict", "different_model"], +) +def test_write_dataset_pydantic_list_rejects_invalid_item(tmp_path: Path, bad_item): + """Same guard as from_pydantic_model(), but via the plain write_dataset() + entry point (the _coerce_reader() Pydantic branch in types.py).""" + + class Record(BaseModel): + name: str + + data = [Record(name="alice"), bad_item] + uri = str(tmp_path / "invalid_item_write_dataset") + with pytest.raises(TypeError, match=r"data\[1\]"): + lance.write_dataset(data, uri) + + +def test_from_pydantic_model_rejects_non_list_data(tmp_path: Path): + """A generator (or other non-list iterable) must be rejected up front -- + otherwise it gets drained by item validation and the caller silently + writes a 0-row table instead of getting an error.""" + + class Record(BaseModel): + name: str + + data = (Record(name=n) for n in ["alice", "bob"]) + uri = str(tmp_path / "generator_input") + with pytest.raises(TypeError, match="must be provided as a list"): + lance.LanceDataset.from_pydantic_model(Record, data, uri=uri) + + +class _RecordSubclass(_OtherRecord): + y: int = 0 + + +@pytest.mark.parametrize( + "entry_point", + ["from_pydantic_model", "write_dataset"], +) +def test_pydantic_list_rejects_subclass_instance(tmp_path: Path, entry_point): + """A subclass instance must not be accepted in place of the exact model + class: the schema is derived from the declared model class, but a + subclass instance may carry extra/overridden fields that model_to_dict() + would then try to serialize against that schema.""" + + data = [_OtherRecord(x=1), _RecordSubclass(x=2, y=3)] + uri = str(tmp_path / f"subclass_rejected_{entry_point}") + with pytest.raises(TypeError, match=r"data\[1\].*exactly"): + if entry_point == "from_pydantic_model": + lance.LanceDataset.from_pydantic_model(_OtherRecord, data, uri=uri) + else: + lance.write_dataset(data, uri) + + +def test_write_dataset_pydantic_optional_field_typed_correctly_when_all_none( + tmp_path: Path, +): + """Regression test: passing a list of Pydantic instances directly to + write_dataset() (not via from_pydantic_model()) must still derive the + schema from the model class, not from the batch's row data -- otherwise + an Optional field that happens to be None for every row in the batch + gets typed as Arrow's literal null type instead of its real type.""" + + class Record(BaseModel): + name: str + tag: Optional[str] = None + + data = [Record(name="alice"), Record(name="bob")] + uri = str(tmp_path / "all_none_optional") + dataset = lance.write_dataset(data, uri) + + assert dataset.schema.field("tag").type == pa.string() + assert dataset.schema.field("tag").nullable is True + + dataset.insert([Record(name="carol", tag="vip")]) + assert dataset.to_table().column("tag").to_pylist() == [None, None, "vip"] def test_roundtrip_types(tmp_path: Path): @@ -312,13 +454,19 @@ def test_versions(tmp_path: Path): base_dir = tmp_path / "test" lance.write_dataset(table1, base_dir) - assert len(lance.dataset(base_dir).versions()) == 1 + dataset = lance.dataset(base_dir) + assert len(dataset.versions()) == 1 + assert dataset.version_refs() == [{"version": 1}] + assert dataset.latest_version == dataset.version_refs()[-1]["version"] table2 = pa.Table.from_pylist([{"s": "one"}, {"s": "two"}]) time.sleep(1) lance.write_dataset(table2, base_dir, mode="overwrite") - assert len(lance.dataset(base_dir).versions()) == 2 + dataset = lance.dataset(base_dir) + assert len(dataset.versions()) == 2 + assert dataset.version_refs() == [{"version": 1}, {"version": 2}] + assert dataset.latest_version == dataset.version_refs()[-1]["version"] v1, v2 = lance.dataset(base_dir).versions() assert v1["version"] == 1 @@ -700,6 +848,51 @@ def test_take(tmp_path: Path): assert table2 == table1 +@pytest.mark.parametrize("data_storage_version", ["legacy", "stable"]) +def test_slice(tmp_path: Path, data_storage_version: str): + table = pa.Table.from_pydict({"a": range(100), "b": range(100)}) + base_dir = tmp_path / "test" + lance.write_dataset( + table, base_dir, data_storage_version=data_storage_version, max_rows_per_file=10 + ) + dataset = lance.dataset(base_dir) + + # Equivalent to take(range(start, end)) + assert dataset.slice(10, 20) == dataset.take(list(range(10, 20))) + + # Basic range within a single fragment + assert dataset.slice(5, 8) == table.slice(5, 3) + + # Range spanning multiple fragments + assert dataset.slice(5, 25) == table.slice(5, 20) + + # Skipping entire fragments + assert dataset.slice(50, 75) == table.slice(50, 25) + + # Full dataset + assert dataset.slice(0, 100) == table.slice(0, 100) + + # Empty range (start == end) + assert dataset.slice(10, 10) == table.slice(10, 0) + + # Range extending past the end of the dataset + assert dataset.slice(90, 1000) == table.slice(90, 10) + + # Range entirely past the end of the dataset + assert dataset.slice(100, 110) == table.slice(100, 0) + + # With column projection + assert dataset.slice(10, 20, columns=["a"]) == table.select(["a"]).slice(10, 10) + + # Invalid start + with pytest.raises(ValueError, match="start must be non-negative"): + dataset.slice(-1, 10) + + # end < start + with pytest.raises(ValueError, match="must be >= start"): + dataset.slice(10, 5) + + def test_take_rowid_rowaddr(tmp_path: Path): sample_size = 10 table1 = pa.table({"a": range(1000), "b": range(1000)}) @@ -1409,13 +1602,20 @@ def test_get_fragments(tmp_path: Path): def test_pickle_fragment(tmp_path: Path): table = pa.Table.from_pydict({"a": range(100), "b": range(100)}) base_dir = tmp_path / "test" - lance.write_dataset(table, base_dir) + storage_options = {"allow_http": "true"} + lance.write_dataset(table, base_dir, storage_options=storage_options) - dataset = lance.dataset(base_dir) + dataset = lance.dataset(base_dir, storage_options=storage_options) fragment = dataset.get_fragments()[0] - pickled = pickle.dumps(fragment) + with mock.patch.object( + lance.LanceDataset, + "__init__", + side_effect=AssertionError("pickling reopened the dataset"), + ): + pickled = pickle.dumps(fragment) unpickled = pickle.loads(pickled) + assert unpickled._ds._storage_options == storage_options assert fragment.to_table() == unpickled.to_table() @@ -1565,6 +1765,10 @@ def test_cleanup_with_retain_versions(tmp_path: Path): ds = lance.write_dataset(table, base_dir, mode="append") assert len(ds.versions()) == 4 + with pytest.raises(OSError, match="retain_versions must be greater than 0, got 0"): + ds.cleanup_old_versions(retain_versions=0) + assert len(ds.versions()) == 4 + stats = ds.cleanup_old_versions(retain_versions=3) assert stats.old_versions == 1 assert stats.data_files_removed == 1 @@ -1575,6 +1779,24 @@ def test_cleanup_with_retain_versions(tmp_path: Path): assert ds.count_rows() == len(ds.to_table()) +def test_cleanup_specific_versions(tmp_path: Path): + base_dir = tmp_path / "cleanup_specific_versions" + table = pa.Table.from_pydict({"a": range(100), "b": range(100)}) + lance.write_dataset(table, base_dir, mode="create") + time.sleep(0.05) + lance.write_dataset(table, base_dir, mode="overwrite") + time.sleep(0.05) + lance.write_dataset(table, base_dir, mode="overwrite") + time.sleep(0.05) + ds = lance.write_dataset(table, base_dir, mode="append") + + assert [v["version"] for v in ds.versions()] == [1, 2, 3, 4] + + stats = ds.cleanup_old_versions(versions=[2]) + assert stats.old_versions == 1 + assert [v["version"] for v in ds.versions()] == [1, 3, 4] + + def test_cleanup_with_older_than_and_retain_versions(tmp_path: Path): base_dir = tmp_path / "cleanup_policy" table = pa.Table.from_pydict({"a": range(100), "b": range(100)}) @@ -1595,6 +1817,27 @@ def test_cleanup_with_older_than_and_retain_versions(tmp_path: Path): assert ds.count_rows() == len(ds.to_table()) +def _wait_until_latest_version_is_older_than(dataset, older_than_seconds): + latest_timestamp = dataset.versions()[-1]["timestamp"] + threshold = latest_timestamp + timedelta(seconds=older_than_seconds) + deadline = time.monotonic() + older_than_seconds + 1 + + while True: + now = ( + datetime.now(latest_timestamp.tzinfo) + if latest_timestamp.tzinfo is not None + else datetime.now() + ) + remaining = (threshold - now).total_seconds() + if remaining < 0: + return + + timeout_remaining = deadline - time.monotonic() + if timeout_remaining <= 0: + pytest.fail("latest dataset version did not pass the cleanup age threshold") + time.sleep(min(remaining + 0.05, timeout_remaining)) + + def test_auto_cleanup(tmp_path): table = pa.Table.from_pydict({"a": range(100), "b": range(100)}) base_dir = tmp_path / "test" @@ -1609,11 +1852,11 @@ def test_auto_cleanup(tmp_path): lance.write_dataset(table, base_dir, mode="append") lance.write_dataset(table, base_dir, mode="append") - time.sleep(5) + dataset = lance.dataset(base_dir) + _wait_until_latest_version_is_older_than(dataset, 1) # trigger cleanup lance.write_dataset(table, base_dir, mode="append") - dataset = lance.dataset(base_dir) assert len(dataset.versions()) == 2 @@ -1628,7 +1871,7 @@ def test_config_update_auto_cleanup(tmp_path): lance.write_dataset(table, base_dir, mode="append") lance.write_dataset(table, base_dir, mode="append") - time.sleep(5) + _wait_until_latest_version_is_older_than(ds, 0.001) # trigger cleanup lance.write_dataset(table, base_dir, mode="append") @@ -1664,12 +1907,13 @@ def test_auto_cleanup_invalid(tmp_path): table, base_dir, auto_cleanup_options=auto_cleanup_options, mode="append" ) - time.sleep(3) + dataset = lance.dataset(base_dir) + assert "lance.auto_cleanup.interval" not in dataset.config() + assert "lance.auto_cleanup.older_than" not in dataset.config() lance.write_dataset( table, base_dir, auto_cleanup_options=auto_cleanup_options, mode="append" ) - dataset = lance.dataset(base_dir) assert len(dataset.versions()) == 4 @@ -1687,7 +1931,7 @@ def test_enable_disable_auto_cleanup(tmp_path): lance.write_dataset(table, base_dir, mode="append") lance.write_dataset(table, base_dir, mode="append") - time.sleep(5) + _wait_until_latest_version_is_older_than(ds, 1) # trigger cleanup lance.write_dataset(table, base_dir, mode="append") @@ -1695,12 +1939,14 @@ def test_enable_disable_auto_cleanup(tmp_path): # this is a transactional commit, so will increase a version ds.optimize.disable_auto_cleanup() + assert "lance.auto_cleanup.interval" not in ds.config() + assert "lance.auto_cleanup.older_than" not in ds.config() lance.write_dataset(table, base_dir, mode="append") lance.write_dataset(table, base_dir, mode="append") lance.write_dataset(table, base_dir, mode="append") - time.sleep(5) + _wait_until_latest_version_is_older_than(ds, 1) # wait to see if cleanup would be trigger lance.write_dataset(table, base_dir, mode="append") @@ -1761,10 +2007,12 @@ def test_strict_overwrite(tmp_path: Path): ) with pytest.raises( OSError, match=f"Commit conflict for version {dataset_v1.version + 1}" - ): + ) as exc_info: lance.LanceDataset.commit( base_dir, operation, read_version=dataset_v1.version, max_retries=0 ) + # CommitConflict means commit-step retries were exhausted; it is safe to retry. + assert exc_info.value.retryable is True def test_commit_timeout(tmp_path: Path): @@ -1934,6 +2182,52 @@ def test_merge_insert_with_commit(): ) +def test_update_with_commit_updated_fragment_offsets(): + table = pa.table({"id": range(10), "updated": [False] * 10}) + dataset = lance.write_dataset(table, "memory://test") + + updates = pa.Table.from_pylist([{"id": 1, "updated": True}]) + transaction, _ = ( + dataset.merge_insert(on="id") + .when_matched_update_all() + .execute_uncommitted(updates) + ) + assert transaction.operation.updated_fragment_offsets is None + + # Portable RoaringBitmap serializations of {1, 3, 5, 7} and {0, 2, 100000}. + offsets = { + 0: ( + b"\x3a\x30\x00\x00\x01\x00\x00\x00\x00\x00\x03\x00" + b"\x10\x00\x00\x00\x01\x00\x03\x00\x05\x00\x07\x00" + ), + 2: ( + b"\x3a\x30\x00\x00\x02\x00\x00\x00\x00\x00\x01\x00\x01\x00\x00" + b"\x00\x18\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x02\x00\xa0\x86" + ), + } + transaction.operation.updated_fragment_offsets = offsets + + dataset = lance.LanceDataset.commit(dataset, transaction) + read_back = dataset.read_transaction(dataset.version) + assert read_back.operation.updated_fragment_offsets == offsets + + +def test_update_with_commit_rejects_invalid_offset_bytes(): + table = pa.table({"id": range(10), "updated": [False] * 10}) + dataset = lance.write_dataset(table, "memory://test") + + updates = pa.Table.from_pylist([{"id": 1, "updated": True}]) + transaction, _ = ( + dataset.merge_insert(on="id") + .when_matched_update_all() + .execute_uncommitted(updates) + ) + transaction.operation.updated_fragment_offsets = {0: b"not a bitmap"} + + with pytest.raises(ValueError, match="RoaringBitmap"): + lance.LanceDataset.commit(dataset, transaction) + + def test_merge_with_commit(tmp_path: Path): table = pa.Table.from_pydict({"a": range(100), "b": range(100)}) base_dir = tmp_path / "test" @@ -1957,6 +2251,68 @@ def test_merge_with_commit(tmp_path: Path): assert tbl == expected +@pytest.mark.parametrize( + ("delete_predicate", "expected_ids"), + [ + pytest.param("id < 50", list(range(50, 150)), id="leading"), + pytest.param( + "id >= 50 AND id < 100", + list(range(50)) + list(range(100, 150)), + id="middle", + ), + pytest.param("id >= 100", list(range(100)), id="trailing"), + ], +) +def test_merge_columns_with_deleted_batch_commit( + tmp_path: Path, delete_predicate: str, expected_ids: list +): + # A fully deleted read batch must still contribute its rows to the new data + # file, otherwise the fragment's data files disagree on the physical row + # count. The deleted run is placed at the start, middle, and end because the + # updater can only borrow a placeholder row from a batch that has live rows. + base_dir = tmp_path / "test" + table = pa.table({"id": range(150), "value": range(150)}) + dataset = lance.write_dataset(table, base_dir, max_rows_per_file=200) + + dataset.delete(delete_predicate) + assert dataset.count_rows() == 100 + + merged_frags = [] + schema = None + for frag in dataset.get_fragments(): + live_ids = frag.scanner(columns=["id"]).to_table()["id"].to_pylist() + right_table = pa.table( + {"merged": pa.array([row_id * 10 for row_id in live_ids], pa.int64())}, + schema=pa.schema([pa.field("merged", pa.int64(), nullable=False)]), + ) + merged, schema = frag.merge_columns(right_table, batch_size=50) + merged_frags.append(merged) + + dataset = lance.LanceDataset.commit( + dataset.uri, + lance.LanceOperation.Merge(merged_frags, schema), + read_version=dataset.version, + ) + dataset.validate() + + assert dataset.to_table() == pa.table( + { + "id": expected_ids, + "value": expected_ids, + "merged": [row_id * 10 for row_id in expected_ids], + }, + schema=pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("value", pa.int64()), + # The blanks written for the deleted rows are copies of a live row, + # so the merged column stays non-nullable end to end. + pa.field("merged", pa.int64(), nullable=False), + ] + ), + ) + + def test_merge_with_schema_holes(tmp_path: Path): # Create table with 3 cols table = pa.table({"a": range(10)}) @@ -2135,8 +2491,13 @@ def test_deletion_file(tmp_path: Path): assert re.match( "_deletions/0-1-[0-9]{1,32}.arrow", new_fragment.deletion_file.path(0) ) - operation = lance.LanceOperation.Overwrite(table.schema, [new_fragment]) - dataset = lance.LanceDataset.commit(base_dir, operation) + # Delete, not Overwrite: the deletion file belongs to fragment 0 of this + # dataset, and an overwrite's fragments are newly written ones that get fresh + # ids, which a deletion file cannot follow. + operation = lance.LanceOperation.Delete([new_fragment], [], "a < 10") + dataset = lance.LanceDataset.commit( + base_dir, operation, read_version=dataset.version + ) assert dataset.count_rows() == 90 @@ -2424,6 +2785,29 @@ def test_merge_insert(tmp_path: Path): check_merge_stats(merge_dict, (None, None, None)) +@pytest.mark.parametrize("materialized", [True, False]) +def test_merge_insert_input_kinds(tmp_path: Path, materialized: bool): + # A materialized pa.Table is routed through the in-memory (MemTable) path, + # while a RecordBatchReader is routed through the streaming path. Both must + # produce identical results. + schema = pa.schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())]) + base = pa.table({"id": range(5), "value": [0] * 5}, schema=schema) + new = pa.table({"id": [1, 2, 5, 6], "value": [10, 20, 50, 60]}, schema=schema) + + dataset = lance.write_dataset(base, tmp_path / "dataset", mode="create") + source = new if materialized else new.to_reader() + + dataset.merge_insert( + "id" + ).when_matched_update_all().when_not_matched_insert_all().execute(source) + + result = dataset.to_table().sort_by("id").to_pydict() + assert result == { + "id": [0, 1, 2, 3, 4, 5, 6], + "value": [0, 10, 20, 0, 0, 50, 60], + } + + def test_merge_insert_subcols(tmp_path: Path): initial_data = pa.table( { @@ -2486,6 +2870,129 @@ def test_merge_insert_subcols(tmp_path: Path): assert dataset.to_table().sort_by("a") == expected +@pytest.mark.parametrize("container", ["struct", "list"]) +def test_merge_insert_subcols_preserves_nested_blob(tmp_path: Path, container: str): + blob_field = lance.blob_field("blob") + blob_values = lance.blob_array([b"one", b"two"]) + if container == "struct": + nested_values = pa.StructArray.from_arrays( + [blob_values], + fields=[blob_field], + ) + expected_nested = [{"blob": b"one"}, {"blob": b"two"}] + else: + nested_values = pa.ListArray.from_arrays( + pa.array([0, 1, 2], type=pa.int32()), + blob_values, + type=pa.list_(blob_field), + ) + expected_nested = [[b"one"], [b"two"]] + + dataset_uri = tmp_path / f"partial_nested_blob_{container}" + dataset = lance.write_dataset( + pa.table( + { + "id": pa.array([1, 2]), + "nested": nested_values, + "other": pa.array([10, 20]), + } + ), + dataset_uri, + data_storage_version="2.2", + ) + source = pa.table({"id": pa.array([2]), "other": pa.array([200])}) + + dataset.merge_insert("id").when_matched_update_all().execute(source) + + result = ( + lance.dataset(dataset_uri).to_table(blob_handling="all_binary").sort_by("id") + ) + assert result["other"].to_pylist() == [10, 200] + assert result["nested"].to_pylist() == expected_nested + + +def test_merge_insert_subcols_in_place(tmp_path: Path): + """`write_mode("rewrite_columns")` patches the source columns into the + fragments that already hold the matched rows. + + Compare with `test_merge_insert_subcols`, which runs the same merge in the + default `"auto"` mode and gets whole rows rewritten into a new fragment + instead. + """ + initial_data = pa.table( + { + "a": range(10), + "b": range(10), + "c": range(10, 20), + } + ) + # Split across two fragments + dataset = lance.write_dataset( + initial_data, tmp_path / "dataset", max_rows_per_file=5 + ) + fragments_before = [f.fragment_id for f in dataset.get_fragments()] + + new_values = pa.table( + { + "a": range(3, 5), + "b": range(20, 22), + } + ) + ( + dataset.merge_insert("a") + .when_matched_update_all() + .write_mode("rewrite_columns") + .execute(new_values) + ) + + # No fragment is added, removed, or renumbered, and column `c` (absent from + # the source) is neither read nor written. + assert [f.fragment_id for f in dataset.get_fragments()] == fragments_before + expected = pa.table( + { + "a": range(10), + "b": [0, 1, 2, 20, 21, 5, 6, 7, 8, 9], + "c": range(10, 20), + } + ) + assert dataset.to_table().sort_by("a") == expected + + # Patching columns cannot add rows, so asking for it explicitly on a merge + # that also inserts is rejected rather than quietly rewriting whole rows. + new_values = pa.table( + { + "a": range(9, 12), + "b": range(30, 33), + } + ) + with pytest.raises(OSError, match="adds rows, which patching cannot do"): + ( + dataset.merge_insert("a") + .when_not_matched_insert_all() + .when_matched_update_all() + .write_mode("rewrite_columns") + .execute(new_values) + ) + + # The same merge under the default mode picks the row-rewrite sink. + ( + dataset.merge_insert("a") + .when_not_matched_insert_all() + .when_matched_update_all() + .execute(new_values) + ) + + assert dataset.count_rows() == 12 + expected = pa.table( + { + "a": range(0, 12), + "b": [0, 1, 2, 20, 21, 5, 6, 7, 8, 30, 31, 32], + "c": list(range(10, 20)) + [None] * 2, + } + ) + assert dataset.to_table().sort_by("a") == expected + + def test_merge_insert_full_fragment_rewrite_json_e2e(tmp_path: Path): """End-to-end test: merge_insert with JSON columns where ALL rows are updated. @@ -2566,6 +3073,66 @@ def test_merge_insert_full_fragment_rewrite_json_e2e(tmp_path: Path): assert sample_result.num_rows == 3 +def test_merge_insert_subcols_with_json_column(tmp_path: Path): + """Test merge_insert with subschema update on a JSON extension type column. + + Previously this would fail with: + 'Incorrect datatype for StructArray field, expected Utf8 got LargeBinary' + because the update_fragments path didn't handle the Arrow JSON ↔ Lance JSON + type mismatch during interleave. + """ + import json + + json_type = pa.json_() + initial_data = pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()), + "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()), + "score": pa.array([10, 20, 30, 40, 50], type=pa.int64()), + "meta": pa.array( + ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'], + type=json_type, + ), + } + ) + dataset = lance.write_dataset(initial_data, tmp_path / "merge_json_subcols") + + # Subschema update: only provide id (key) + meta (JSON column to update) + new_values = pa.table( + { + "id": pa.array([2, 4], type=pa.int64()), + "meta": pa.array( + ['{"updated":true,"id":2}', '{"updated":true,"id":4}'], + type=json_type, + ), + } + ) + + # This should NOT raise a type mismatch error + dataset.merge_insert("id").when_matched_update_all().execute(new_values) + + # Verify results + result = dataset.to_table().sort_by("id") + ids = result.column("id").to_pylist() + scores = result.column("score").to_pylist() + metas = result.column("meta").to_pylist() + + # Score column (not in update) should be preserved + assert scores == [10, 20, 30, 40, 50] + + # Meta column should be updated for id=2 and id=4 + for id_val, meta_val in zip(ids, metas): + parsed = json.loads(meta_val) if isinstance(meta_val, str) else meta_val + if id_val in (2, 4): + assert parsed.get("updated") is True, ( + f"id={id_val} should have updated meta, got {meta_val}" + ) + else: + assert "x" in str(parsed), ( + f"id={id_val} should have original meta, got {meta_val}" + ) + + def test_merge_insert_defaults_to_pk_when_on_omitted(tmp_path): base_dir = tmp_path / "merge_insert_pk_default" @@ -2784,6 +3351,42 @@ def test_merge_insert_multiple_keys(tmp_path: Path): check_merge_stats(merge_dict, (0, 350, 0)) +def test_indexed_merge_insert_deduplicates_cross_batch_candidates(tmp_path: Path): + target = pa.table( + { + "a": [1, 1, 2, 2], + "b": [10, 20, 10, 20], + "value": [110, 120, 210, 220], + } + ) + dataset = lance.write_dataset(target, tmp_path / "dataset") + dataset.create_scalar_index("a", "BTREE") + dataset.create_scalar_index("b", "BTREE") + + first = pa.RecordBatch.from_pydict( + {"a": [1], "b": [10], "value": [901]}, schema=target.schema + ) + # The per-column index probe for this batch also reaches (1, 10), which + # was already emitted for the first batch. The exact join filters that + # over-match, but the indexed scan must not read the target row twice. + second = pa.RecordBatch.from_pydict( + {"a": [1, 2], "b": [20, 10], "value": [902, 903]}, + schema=target.schema, + ) + source = pa.RecordBatchReader.from_batches(target.schema, [first, second]) + + stats = dataset.merge_insert(["a", "b"]).when_matched_update_all().execute(source) + + check_merge_stats(stats, (0, 3, 0)) + assert ( + dataset.to_table().sort_by([("a", "ascending"), ("b", "ascending")]).to_pydict() + ) == { + "a": [1, 1, 2, 2], + "b": [10, 20, 10, 20], + "value": [901, 902, 903, 220], + } + + def test_merge_insert_vector_column(tmp_path: Path): table = pa.Table.from_pydict( { @@ -3120,6 +3723,42 @@ def test_merge_insert_explain_analyze_plan(): assert "num_files_written" in analysis +def test_merge_insert_analyze_plan_matches_execute_routing(): + """analyze_plan must report the plan the given source would actually run. + + execute() wraps a materialized source in an in-memory table, which reports + exact statistics; a stream reports none. DataFusion picks the collected side of + the join from those statistics and from the two sides' sizes, so the same merge + plans differently depending on which one it is handed. analyze_plan used to + coerce every input to a stream, so it reported the stream's plan whatever it + was given. + """ + data = pa.table({"id": range(64), "value": [i * 10 for i in range(64)]}) + dataset = lance.write_dataset(data, "memory://test-merge-analyze-routing") + + def builder(): + return ( + dataset.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + ) + + # Two source rows against the target's 64 keeps the source the smaller side, + # which is what lets the join collect it. Raise it above 64 and the join + # collects the target instead and the join type stays Right. + source = pa.table({"id": [1, 100], "value": [999, 999]}) + + materialized = builder().analyze_plan(source) + assert "DataSourceExec" in materialized, materialized + assert "StreamingTableExec" not in materialized, materialized + assert "join_type=Left" in materialized, materialized + + streaming = builder().analyze_plan(source.to_reader()) + assert "StreamingTableExec" in streaming, streaming + assert "DataSourceExec" not in streaming, streaming + assert "join_type=Right" in streaming, streaming + + def test_merge_insert_use_index(): """Test that use_index parameter controls whether indices are used.""" data = pa.table({"id": range(100), "value": [i * 10 for i in range(100)]}) @@ -4143,10 +4782,14 @@ def commit_lock(_version: int): lance.write_dataset( pa.table({"a": range(100)}), tmp_path / "test2", commit_lock=commit_lock ) + assert lance.dataset(tmp_path / "test2").count_rows() == 100 + + # Import only after the generic error case to verify users need not import it first. + commit_module = importlib.import_module("lance.commit") @contextlib.contextmanager def commit_lock(_version: int): - raise CommitConflictError() + raise commit_module.CommitConflictError() with pytest.raises(Exception, match="CommitConflictError"): lance.write_dataset( @@ -4627,16 +5270,20 @@ def test_late_materialization_param(tmp_path: Path): ) filt = "filter % 2 == 0" - assert "(values)" in dataset.scanner( - filter=filt, late_materialization=None - ).explain_plan(True) + # A late-materialized column is fetched by a row-stream read + # (`projection=[values], source=stream`); an eager column appears in the + # scan projection + late = "projection=[values], source=stream" + assert late in dataset.scanner(filter=filt, late_materialization=None).explain_plan( + True + ) assert ", values" in dataset.scanner( filter=filt, late_materialization=False ).explain_plan(True) - assert "(values)" in dataset.scanner( - filter=filt, late_materialization=True - ).explain_plan(True) - assert "(values)" in dataset.scanner( + assert late in dataset.scanner(filter=filt, late_materialization=True).explain_plan( + True + ) + assert late in dataset.scanner( filter=filt, late_materialization=["values"] ).explain_plan(True) assert ", values" in dataset.scanner( @@ -4867,6 +5514,69 @@ def test_dataset_drop(tmp_path: Path): lance.LanceDataset.drop(tmp_path) +def test_dataset_drop_rejects_non_dataset_directory(tmp_path: Path): + warehouse = tmp_path / "warehouse" + lance.write_dataset(pa.table({"x": [0]}), warehouse / "t.lance") + + # Pointing at the parent of a dataset must not wipe out the whole warehouse. + with pytest.raises(ValueError, match="no readable Lance manifest"): + lance.LanceDataset.drop(warehouse) + assert (warehouse / "t.lance").exists() + + lance.LanceDataset.drop(warehouse / "t.lance") + assert not (warehouse / "t.lance").exists() + + +@pytest.mark.parametrize( + "entries", + [ + # A storage root holding a "data" prefix is indistinguishable from the + # leftovers of a write that died before committing. + ["data/0.lance"], + # A file merely sitting under _versions/, or merely named like a manifest, is + # not evidence that a dataset was ever committed here. + ["_versions/README", "reports/q1.csv"], + ["_versions/1.manifest", "reports/q1.csv"], + ], +) +def test_dataset_drop_rejects_paths_without_readable_manifest( + tmp_path: Path, entries: list +): + storage_root = tmp_path / "storage_root" + for entry in entries: + path = storage_root / entry + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"irreplaceable") + + with pytest.raises(ValueError, match="no readable Lance manifest"): + lance.LanceDataset.drop(storage_root) + for entry in entries: + assert (storage_root / entry).exists() + + +def test_dataset_drop_allows_create_over_uncommitted_leftovers(tmp_path: Path): + # Refusing to drop data files without a manifest costs nothing, because those + # leftovers do not stop the dataset from being created and then dropped. + dataset_dir = tmp_path / "t.lance" + (dataset_dir / "data").mkdir(parents=True) + (dataset_dir / "data" / "0.lance").write_bytes(b"partial") + + lance.write_dataset(pa.table({"x": [0]}), dataset_dir) + lance.LanceDataset.drop(dataset_dir) + assert not dataset_dir.exists() + + +def test_dataset_drop_allows_dataset_with_unmanaged_files(tmp_path: Path): + # Cleanup deliberately preserves unmanaged files under a dataset root, so their + # presence must not stop a drop either. + dataset_dir = tmp_path / "t.lance" + lance.write_dataset(pa.table({"x": [0]}), dataset_dir) + (dataset_dir / "notes.txt").write_text("kept next to the dataset") + + lance.LanceDataset.drop(dataset_dir) + assert not dataset_dir.exists() + + def test_dataset_schema(tmp_path: Path): table = pa.table({"x": [0]}) ds = lance.write_dataset(table, str(tmp_path)) # noqa: F841 @@ -4903,6 +5613,294 @@ def test_data_replacement(tmp_path: Path): assert tbl == expected +def _write_overlay_file( + dataset, base_dir: Path, name: str, batch: pa.Table, fields: List[int] +): + """Write an overlay value file (one value column per covered field, no key + column) and return a DataFile mapping its columns to the given dataset + `fields`. The file version is copied from a base data file.""" + path = base_dir / "data" / name + with LanceFileWriter(str(path)) as writer: + writer.write_batch(batch) + base_df = dataset.get_fragments()[0].metadata.files[0] + return lance.fragment.DataFile( + path=name, + fields=fields, + column_indices=list(range(len(fields))), + file_major_version=base_df.file_major_version, + file_minor_version=base_df.file_minor_version, + file_size_bytes=os.path.getsize(path), + ) + + +@pytest.fixture +def enable_unstable_data_overlay_files(monkeypatch): + monkeypatch.setenv("LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES", "1") + + +def test_data_overlay_dense(tmp_path: Path, enable_unstable_data_overlay_files): + base_dir = tmp_path / "test" + table = pa.table( + { + "id": pa.array(range(10), pa.int32()), + "val": pa.array([i * 10 for i in range(10)], pa.int32()), + } + ) + dataset = lance.write_dataset(table, base_dir) + + # Overlay `val` at physical offsets {1, 4} with new values. + data_file = _write_overlay_file( + dataset, + base_dir, + "ov.lance", + pa.table({"val": pa.array([111, 444], pa.int32())}), + fields=[1], + ) + assert data_file.fields == [1] # `val` is field id 1 + + overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[1, 4]) + op = lance.LanceOperation.DataOverlay( + [lance.LanceOperation.DataOverlayGroup(0, [overlay])] + ) + dataset = lance.LanceDataset.commit(dataset, op, read_version=dataset.version) + + result = dataset.to_table() + assert result.column("val").to_pylist() == [0, 111, 20, 30, 444, 50, 60, 70, 80, 90] + # The unrelated `id` column is untouched. + assert result.column("id").to_pylist() == list(range(10)) + + +def test_data_overlay_newest_wins(tmp_path: Path, enable_unstable_data_overlay_files): + base_dir = tmp_path / "test" + table = pa.table( + { + "id": pa.array(range(10), pa.int32()), + "val": pa.array([i * 10 for i in range(10)], pa.int32()), + } + ) + dataset = lance.write_dataset(table, base_dir) + + older = _write_overlay_file( + dataset, + base_dir, + "older.lance", + pa.table({"val": pa.array([111, 444], pa.int32())}), + fields=[1], + ) + dataset = lance.LanceDataset.commit( + dataset, + lance.LanceOperation.DataOverlay( + [ + lance.LanceOperation.DataOverlayGroup( + 0, + [lance.LanceOperation.DataOverlayFile(older, offsets=[1, 4])], + ) + ] + ), + read_version=dataset.version, + ) + # A newer overlay re-covers offset 1; it must win there. + newer = _write_overlay_file( + dataset, + base_dir, + "newer.lance", + pa.table({"val": pa.array([999], pa.int32())}), + fields=[1], + ) + dataset = lance.LanceDataset.commit( + dataset, + lance.LanceOperation.DataOverlay( + [ + lance.LanceOperation.DataOverlayGroup( + 0, [lance.LanceOperation.DataOverlayFile(newer, offsets=[1])] + ) + ] + ), + read_version=dataset.version, + ) + + val = dataset.to_table().column("val").to_pylist() + assert val[1] == 999 # newest overlay wins + assert val[4] == 444 # only the older overlay covers offset 4 + + +def test_data_overlay_sparse_per_field( + tmp_path: Path, enable_unstable_data_overlay_files +): + base_dir = tmp_path / "test" + table = pa.table( + { + "id": pa.array(range(10), pa.int32()), + "val": pa.array([i * 10 for i in range(10)], pa.int32()), + } + ) + dataset = lance.write_dataset(table, base_dir) + + # Sparse overlay: `id` covers offset {2}, `val` covers offset {3}. The value + # file carries one value per field (rank 0 of each field's coverage). + data_file = _write_overlay_file( + dataset, + base_dir, + "sparse.lance", + pa.table( + { + "id": pa.array([777], pa.int32()), + "val": pa.array([330], pa.int32()), + } + ), + fields=[0, 1], + ) + assert data_file.fields == [0, 1] + + overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[[2], [3]]) + op = lance.LanceOperation.DataOverlay( + [lance.LanceOperation.DataOverlayGroup(0, [overlay])] + ) + dataset = lance.LanceDataset.commit(dataset, op, read_version=dataset.version) + + result = dataset.to_table() + assert result.column("id").to_pylist()[2] == 777 + assert result.column("val").to_pylist()[3] == 330 + # Fields resolve independently: id at offset 3 and val at offset 2 fall through. + assert result.column("id").to_pylist()[3] == 3 + assert result.column("val").to_pylist()[2] == 20 + + +def test_data_overlay_round_trips_through_fragment_metadata( + tmp_path: Path, enable_unstable_data_overlay_files +): + import json + + base_dir = tmp_path / "test" + table = pa.table( + { + "id": pa.array(range(10), pa.int32()), + "val": pa.array([i * 10 for i in range(10)], pa.int32()), + } + ) + dataset = lance.write_dataset(table, base_dir) + + data_file = _write_overlay_file( + dataset, + base_dir, + "ov.lance", + pa.table({"val": pa.array([111, 444], pa.int32())}), + fields=[1], + ) + overlay = lance.LanceOperation.DataOverlayFile(data_file, offsets=[1, 4]) + dataset = lance.LanceDataset.commit( + dataset, + lance.LanceOperation.DataOverlay( + [lance.LanceOperation.DataOverlayGroup(0, [overlay])] + ), + read_version=dataset.version, + ) + overlay_version = dataset.version + + # Reading the fragment surfaces its overlays, stamped with the commit version. + metadata = dataset.get_fragments()[0].metadata + assert len(metadata.overlays) == 1 + assert metadata.overlays[0].offsets == [1, 4] + assert metadata.overlays[0].committed_version == overlay_version + + # The overlays survive a JSON round-trip of the metadata. + restored = lance.fragment.FragmentMetadata.from_json(json.dumps(metadata.to_json())) + assert len(restored.overlays) == 1 + assert restored.overlays[0].offsets == [1, 4] + assert restored.overlays[0].committed_version == overlay_version + + # A commit that round-trips the fragment (here an Overwrite) must keep the + # overlays, so the overlay still resolves on read instead of being dropped. + dataset = lance.LanceDataset.commit( + dataset, + lance.LanceOperation.Overwrite(dataset.schema, [restored]), + read_version=dataset.version, + ) + result = dataset.to_table() + assert result.column("val").to_pylist() == [0, 111, 20, 30, 444, 50, 60, 70, 80, 90] + assert result.column("id").to_pylist() == list(range(10)) + + +def test_data_overlay_rejects_invalid_offsets( + tmp_path: Path, enable_unstable_data_overlay_files +): + base_dir = tmp_path / "test" + table = pa.table({"val": pa.array([0, 1, 2], pa.int32())}) + dataset = lance.write_dataset(table, base_dir) + data_file = _write_overlay_file( + dataset, + base_dir, + "ov.lance", + pa.table({"val": pa.array([9], pa.int32())}), + fields=[0], + ) + + # offsets is neither a flat list of ints (dense) nor a list of per-field int + # lists (sparse), so the coverage shape can't be resolved. + with pytest.raises(ValueError, match="offsets must be a list"): + lance.LanceDataset.commit( + dataset, + lance.LanceOperation.DataOverlay( + [ + lance.LanceOperation.DataOverlayGroup( + 0, + [ + lance.LanceOperation.DataOverlayFile( + data_file, offsets=[0, [1]] + ) + ], + ) + ] + ), + read_version=dataset.version, + ) + + +@pytest.mark.parametrize( + "offsets", + [ + [2, 1], # dense, descending + [1, 1], # dense, duplicate + [[2, 1]], # sparse, descending + [[1, 1]], # sparse, duplicate + ], +) +def test_data_overlay_rejects_unsorted_offsets( + tmp_path: Path, offsets, enable_unstable_data_overlay_files +): + # Offsets map positionally to value rows in data_file. A RoaringBitmap would + # silently reorder/dedup them, so a non-ascending list must be rejected up + # front rather than corrupting the row mapping. + base_dir = tmp_path / "test" + table = pa.table({"val": pa.array([0, 1, 2], pa.int32())}) + dataset = lance.write_dataset(table, base_dir) + data_file = _write_overlay_file( + dataset, + base_dir, + "ov.lance", + pa.table({"val": pa.array([9, 9], pa.int32())}), + fields=[0], + ) + + with pytest.raises(ValueError, match="strictly ascending"): + lance.LanceDataset.commit( + dataset, + lance.LanceOperation.DataOverlay( + [ + lance.LanceOperation.DataOverlayGroup( + 0, + [ + lance.LanceOperation.DataOverlayFile( + data_file, offsets=offsets + ) + ], + ) + ] + ), + read_version=dataset.version, + ) + + def test_schema_project_drop_column(tmp_path: Path): table = pa.Table.from_pydict({"a": range(100, 200), "b": range(300, 400)}) base_dir = tmp_path / "test" @@ -5046,6 +6044,33 @@ def test_dataset_sql(tmp_path: Path): assert pa.Table.from_batches(complex_result) == expected_complex +def test_dataset_sql_batch_size_rows(tmp_path: Path): + table = pa.table({"id": range(50)}) + ds = lance.write_dataset(table, tmp_path / "test_sql_batch_size_rows") + + batches = list( + ds.sql("SELECT * FROM dataset").batch_size(7).build().to_stream_reader() + ) + + assert sum(batch.num_rows for batch in batches) == 50 + assert all(batch.num_rows <= 7 for batch in batches) + + +@pytest.mark.parametrize("batch_size", [0, 2**32]) +def test_dataset_sql_rejects_invalid_batch_size(tmp_path: Path, batch_size: int): + ds = lance.write_dataset( + pa.table({"id": range(3)}), tmp_path / "test_sql_invalid_batch_size" + ) + + with pytest.raises(ValueError, match="batch_size must be between 1 and 4294967295"): + ( + ds.sql("SELECT * FROM dataset") + .batch_size(batch_size) + .build() + .to_batch_records() + ) + + def test_file_reader_options(tmp_path: Path): """Test cache_repetition_index and validate_on_decode options""" # Create a dataset with large repetitive strings to test cache_repetition_index diff --git a/python/python/tests/test_delta.py b/python/python/tests/test_delta.py index 589dab8dc3f..7875df6dd00 100755 --- a/python/python/tests/test_delta.py +++ b/python/python/tests/test_delta.py @@ -134,3 +134,28 @@ def test_delta_validation_errors(): "and with_end_version", ): ds.delta(end_version=2) + + +def test_delta_get_deleted_row_ids(): + table = pa.table( + { + "id": pa.array([1, 2, 3, 4], type=pa.int32()), + "val": pa.array(["a", "b", "c", "d"], type=pa.string()), + } + ) + ds = write_dataset( + table, "memory://delta_api_test_delete", enable_stable_row_ids=True + ) + row_ids = ds.to_table(columns=[], with_row_id=True).column("_rowid").to_pylist() + + ds.delete("id in (2, 3)") + + delta = ds.delta(compared_against=1) + reader = delta.get_deleted_row_ids() + + deleted = [] + for batch in reader: + assert batch.schema.names == ["_rowid"] + deleted.extend(batch.column("_rowid").to_pylist()) + + assert sorted(deleted) == sorted([row_ids[1], row_ids[2]]) diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py index d0f0fb0b8b4..2662edf5524 100644 --- a/python/python/tests/test_file.py +++ b/python/python/tests/test_file.py @@ -87,7 +87,30 @@ def test_multiple_close(tmp_path): writer = LanceFileWriter(str(path), schema) writer.write_batch(pa.table({"a": [1, 2, 3]})) writer.close() + size_bytes = writer.size_bytes + # The second close is a no-op and must not clear the recorded size writer.close() + assert writer.size_bytes == size_bytes + + +@pytest.mark.parametrize("num_rows", [0, 3]) +def test_size_bytes(tmp_path, num_rows): + path = tmp_path / "foo.lance" + schema = pa.schema([pa.field("a", pa.int64())]) + writer = LanceFileWriter(str(path), schema) + assert writer.size_bytes is None + writer.write_batch(pa.table({"a": list(range(num_rows))})) + assert writer.close() == num_rows + # Even an empty file has a footer, so the size is always positive + assert writer.size_bytes > 0 + assert writer.size_bytes == os.path.getsize(path) + + +def test_size_bytes_with_session(tmp_path): + session = LanceFileSession(tmp_path) + with session.open_writer("foo.lance") as writer: + writer.write_batch(pa.table({"a": [1, 2, 3]})) + assert writer.size_bytes == os.path.getsize(tmp_path / "foo.lance") def test_version(tmp_path): @@ -459,18 +482,27 @@ def test_write_read_additional_schema_metadata(tmp_path): def test_writer_maintains_order(tmp_path): - # 100Ki strings, each string is a couple of KiBs - big_strings = [f"{i}" * 1024 for i in range(100 * 1024)] - table = pa.table({"big_strings": big_strings}) - - for i in range(4): - path = tmp_path / f"foo-{i}.lance" - with LanceFileWriter(str(path)) as writer: - writer.write_batch(table) + row_ids = list(range(8)) + payloads = ["0123456789abcdef" * (64 * 1024)] + [ + f"page-{row_id}" for row_id in row_ids[1:] + ] + table = pa.table({"payload": payloads, "row_id": row_ids}) + path = tmp_path / "ordered-pages.lance" + + # Before #2836, the seven cheap pages could finish encoding before the + # expensive first page and be written out of order. + with LanceFileWriter( + str(path), + table.schema, + version="2.0", + data_cache_bytes=1, + max_page_bytes=64 * 1024, + ) as writer: + writer.write_batch(table) - reader = LanceFileReader(str(path)) - result = reader.read_all().to_table() - assert result == table + reader = LanceFileReader(str(path)) + assert [len(column.pages) for column in reader.metadata().columns] == [8, 1] + assert reader.read_all().to_table() == table def test_compression(tmp_path): diff --git a/python/python/tests/test_filter.py b/python/python/tests/test_filter.py index 9416c191e36..e2d4cbe0121 100644 --- a/python/python/tests/test_filter.py +++ b/python/python/tests/test_filter.py @@ -8,6 +8,7 @@ from datetime import date, datetime, timedelta from decimal import Decimal from pathlib import Path +from zoneinfo import ZoneInfo import lance import numpy as np @@ -108,6 +109,28 @@ def test_sql_predicates(dataset): assert dataset.to_table(filter=expr).num_rows == expected_num_rows +@pytest.mark.parametrize("unit", ["s", "ms", "us"]) +@pytest.mark.parametrize("timezone", [None, "UTC", "America/New_York"]) +def test_timestamp_pyarrow_predicates(tmp_path: Path, unit: str, timezone: str | None): + # PyArrow filters reach Lance as Substrait, where the timestamp literal used to be + # decoded in the wrong unit. + tz = ZoneInfo(timezone) if timezone else None + start = datetime(2021, 1, 1, tzinfo=tz) + ts_type = pa.timestamp(unit, timezone) + table = pa.table( + {"ts": pa.array([start + timedelta(hours=i) for i in range(100)], ts_type)} + ) + dataset = lance.write_dataset(table, tmp_path / f"{unit}_{timezone}") + + cutoff = pa.scalar(start + timedelta(hours=50), ts_type) + for expr in [ + pc.field("ts") > cutoff, + pc.field("ts") < cutoff, + pc.field("ts") == cutoff, + ]: + assert dataset.to_table(filter=expr) == table.filter(expr) + + def test_sql_current_date(tmp_path: Path): table = pa.table( {"date": pa.array([date(2020, 1, 1), date(2020, 1, 2)], type=pa.date32())} @@ -371,8 +394,8 @@ def test_filter_on_column_beside_root_extension_type(tmp_path): @pytest.mark.skip( - reason="enable this in recurring test https://github.com/lance-format/lance/pull/4190" - " as it requires release mode" + reason="requires a release build; see " + "https://github.com/lance-format/lance/pull/4190" ) def test_filter_depth_limit(): column_name = "a_very_long_column_name" diff --git a/python/python/tests/test_fork.py b/python/python/tests/test_fork.py index f36e13debed..c032781623a 100644 --- a/python/python/tests/test_fork.py +++ b/python/python/tests/test_fork.py @@ -3,6 +3,7 @@ import os import sys +import traceback from pathlib import Path import lance @@ -24,14 +25,7 @@ def create_table(num_rows) -> pa.Table: ) -@pytest.mark.skipif(sys.platform == "win32", reason="Test not applicable on Windows") -def test_table_roundtrip(tmp_path: Path): - uri = tmp_path - - tbl = create_table(100) - lance.write_dataset(tbl, uri) - - os.fork() +def check_reads(uri: Path, tbl: pa.Table): dataset = lance.dataset(uri) assert dataset.uri == str(uri.absolute()) assert tbl.schema == dataset.schema @@ -42,3 +36,36 @@ def test_table_roundtrip(tmp_path: Path): table = dataset.to_table(columns=["a"], limit=20) assert len(table) == 20 + + +@pytest.mark.skipif(sys.platform == "win32", reason="Test not applicable on Windows") +def test_table_roundtrip(tmp_path: Path): + uri = tmp_path + + tbl = create_table(100) + lance.write_dataset(tbl, uri) + + child = os.fork() + if child == 0: + # The child has to leave through os._exit. Returning would run the rest + # of the pytest session a second time, and under pytest-xdist it would + # also report a second result for this test over the execnet connection + # inherited from the worker, which crashes the controller's scheduler. + status = 0 + try: + check_reads(uri, tbl) + except BaseException: + traceback.print_exc() + status = 1 + os._exit(status) + + check_reads(uri, tbl) + _, wait_status = os.waitpid(child, 0) + exitcode = os.waitstatus_to_exitcode(wait_status) + # Nothing the child raises can reach this process, so its exit status is the + # only evidence the post-fork read worked. On macOS the child dies of a + # signal before finishing that read -- long-standing behaviour that this + # test could not see while it never waited on the child at all. Checking it + # where it does hold at least keeps the Linux path honest. + if sys.platform != "darwin": + assert exitcode == 0, "reading the dataset failed in the forked child" diff --git a/python/python/tests/test_fragment.py b/python/python/tests/test_fragment.py index b05888df31f..220ab7cfc76 100644 --- a/python/python/tests/test_fragment.py +++ b/python/python/tests/test_fragment.py @@ -22,7 +22,7 @@ ) from lance.debug import format_fragment from lance.file import LanceFileWriter -from lance.fragment import write_fragments +from lance.fragment import RowIdMeta, RowIdSequence, write_fragments from lance.progress import FileSystemFragmentWriteProgress @@ -279,7 +279,7 @@ def test_fragment_meta(): "file_size_bytes=100), DataFile(path='1.lance', fields=[1], column_indices=[], " "file_major_version=0, file_minor_version=0, file_size_bytes=None)], " "physical_rows=100, deletion_file=None, row_id_meta=None, " - "created_at_version_meta=None, last_updated_at_version_meta=None)" + "created_at_version_meta=None, last_updated_at_version_meta=None, overlays=[])" ) @@ -617,6 +617,139 @@ def test_fragment_update_columns_with_custom_join_key(tmp_path): assert result["name"][2] == "Chase" # id=3 should have name Chase +def test_fragment_update_columns_with_blob_v2(tmp_path): + data = pa.table( + { + "id": pa.array([1, 2, 3, 4]), + "payload": lance.blob_array([b"one", b"two", b"", None]), + } + ) + dataset_uri = tmp_path / "test_dataset_update_columns_blob_v2" + dataset = lance.write_dataset( + data, + dataset_uri, + data_storage_version="2.2", + ) + + fragment = dataset.get_fragment(0) + updated_fragment, fields_modified = fragment.update_columns( + pa.table( + { + "id": pa.array([2]), + "payload": lance.blob_array([b"NEW"]), + } + ), + left_on="id", + ) + + operation = LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ) + updated_dataset = LanceDataset.commit( + dataset_uri, + operation, + read_version=dataset.version, + ) + + result = updated_dataset.to_table(blob_handling="all_binary") + assert result["id"].to_pylist() == [1, 2, 3, 4] + assert result["payload"].to_pylist() == [b"one", b"NEW", b"", None] + + +def test_fragment_update_columns_with_nested_blob_v2(tmp_path): + def info_array(names, payloads): + fields = [pa.field("name", pa.string()), lance.blob_field("blob")] + return pa.StructArray.from_arrays( + [pa.array(names), lance.blob_array(payloads)], fields=fields + ) + + dataset_uri = tmp_path / "test_dataset_update_columns_nested_blob_v2" + dataset = lance.write_dataset( + pa.table( + { + "id": pa.array([1, 2]), + "info": info_array(["a", "b"], [b"one", b"two"]), + } + ), + dataset_uri, + data_storage_version="2.2", + ) + + updated_fragment, fields_modified = dataset.get_fragment(0).update_columns( + pa.table( + { + "id": pa.array([2]), + "info": info_array(["B"], [b"NEW"]), + } + ), + left_on="id", + ) + updated_dataset = LanceDataset.commit( + dataset_uri, + LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ), + read_version=dataset.version, + ) + + info = updated_dataset.to_table(blob_handling="all_binary")["info"].combine_chunks() + assert info.field("name").to_pylist() == ["a", "B"] + assert info.field("blob").to_pylist() == [b"one", b"NEW"] + + +def test_fragment_update_columns_preserves_external_blob_v2(tmp_path): + dataset_uri = tmp_path / "test_dataset_update_columns_external_blob_v2" + external = tmp_path / "existing-payload.bin" + external.write_bytes(b"outside") + dataset = lance.write_dataset( + pa.table( + { + "id": pa.array([1, 2]), + "payload": lance.blob_array([external.as_uri(), b"two"]), + } + ), + dataset_uri, + data_storage_version="2.2", + allow_external_blob_outside_bases=True, + ) + + updated_fragment, fields_modified = dataset.get_fragment(0).update_columns( + pa.table( + { + "id": pa.array([2]), + "payload": lance.blob_array([b"NEW"]), + } + ), + left_on="id", + ) + updated_dataset = LanceDataset.commit( + dataset_uri, + LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ), + read_version=dataset.version, + ) + + result = updated_dataset.to_table(blob_handling="all_binary") + assert result["payload"].to_pylist() == [b"outside", b"NEW"] + + new_external = tmp_path / "new-payload.bin" + new_external.write_bytes(b"new outside") + with pytest.raises(ValueError, match="outside registered external bases"): + updated_dataset.get_fragment(0).update_columns( + pa.table( + { + "id": pa.array([2]), + "payload": lance.blob_array([new_external.as_uri()]), + } + ), + left_on="id", + ) + + def test_fragment_update_columns_with_nulls(tmp_path): """Test fragment update columns with null values.""" # Create initial dataset @@ -865,3 +998,501 @@ def test_fragment_take_with_json_column(tmp_path): assert metas[0] == '{"val":1}' assert metas[1] == '{"val":4}' assert metas[2] == '{"val":7}' + + +def test_fragment_create_with_json_column(tmp_path): + """Test that LanceFragment.create works with Arrow JSON extension type. + + Previously the single-fragment create path skipped the Arrow JSON (Utf8) -> + Lance JSON (JSONB LargeBinary) conversion that write_dataset/write_fragments + perform, so the raw UTF-8 string bytes were written into a column whose schema + declared JSONB. Reads then miss-decoded the bytes and returned garbage. + """ + json_type = pa.json_() + data = pa.table( + { + "uid": pa.array(["a", "b", "c", "d"], type=pa.utf8()), + "payload": pa.array( + ['{"x":1}', '{"x":2}', '{"y":3}', '{"y":4}'], + type=json_type, + ), + } + ) + + frag = LanceFragment.create(tmp_path, data) + operation = LanceOperation.Overwrite(data.schema, [frag]) + dataset = LanceDataset.commit(tmp_path, operation) + + result = dataset.to_table() + assert result.column("uid").to_pylist() == ["a", "b", "c", "d"] + payloads = result.column("payload").to_pylist() + assert [json.loads(p) for p in payloads] == [ + {"x": 1}, + {"x": 2}, + {"y": 3}, + {"y": 4}, + ] + + +def test_fragment_update_columns_with_json_column(tmp_path): + """Test that fragment update_columns works with Arrow JSON extension type. + + Previously this would fail with a type mismatch error because the + HashJoiner didn't convert Arrow JSON (Utf8) to Lance JSON (LargeBinary). + """ + # Create initial dataset with a JSON extension type column + json_type = pa.json_() + data = pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int64()), + "name": pa.array(["a", "b", "c", "d", "e"], type=pa.utf8()), + "meta": pa.array( + ['{"x":1}', '{"x":2}', '{"x":3}', '{"x":4}', '{"x":5}'], + type=json_type, + ), + } + ) + dataset_uri = tmp_path / "test_update_cols_json" + dataset = lance.write_dataset(data, dataset_uri) + + # Prepare update data: update the JSON column for some rows + update_data = pa.table( + { + "_rowid": pa.array([1, 3], type=pa.uint64()), + "meta": pa.array( + ['{"updated":true,"id":2}', '{"updated":true,"id":4}'], + type=json_type, + ), + } + ) + + # This should NOT raise a type mismatch error + fragment = dataset.get_fragment(0) + updated_fragment, fields_modified = fragment.update_columns(update_data) + + assert len(fields_modified) > 0 + + # Commit and verify + op = LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ) + updated_dataset = lance.LanceDataset.commit( + str(dataset_uri), op, read_version=dataset.version + ) + + result = updated_dataset.to_table() + ids = result.column("id").to_pylist() + metas = result.column("meta").to_pylist() + + for i, (id_val, meta_val) in enumerate(zip(ids, metas)): + meta = json.loads(meta_val) if isinstance(meta_val, str) else meta_val + if id_val == 2 or id_val == 4: + assert "updated" in meta_val or meta.get("updated") is True, ( + f"id={id_val} should be updated, got {meta_val}" + ) + else: + assert "x" in meta_val or "x" in str(meta), ( + f"id={id_val} should have original value, got {meta_val}" + ) + + +def test_row_id_sequence_from_range(): + # A step-of-one range is the compact case and must not materialize its ids. + sequence = RowIdSequence(range(10)) + + assert len(sequence) == 10 + assert sequence.to_pyarrow() == pa.array(range(10), type=pa.uint64()) + assert sequence.to_pyarrow().type == pa.uint64() + assert list(sequence) == list(range(10)) + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param(pa.array([1, 2, 3]), id="pyarrow_array"), + pytest.param(pa.array([1, 2, 3], type=pa.uint64()), id="pyarrow_uint64_array"), + pytest.param(pa.array([1, 2, 3], type=pa.int8()), id="pyarrow_int8_array"), + pytest.param(pa.array([1, 2, 3], type=pa.uint16()), id="pyarrow_uint16_array"), + # A slice carries an offset into a larger buffer; only the slice counts. + pytest.param(pa.array([9, 1, 2, 3, 9]).slice(1, 3), id="pyarrow_sliced_array"), + pytest.param(pa.chunked_array([[1], [2, 3]]), id="pyarrow_chunked_array"), + pytest.param((x for x in [1, 2, 3]), id="generator"), + pytest.param([1, 2, 3], id="list"), + pytest.param(range(1, 4), id="range"), + pytest.param(range(3, 0, -1), id="descending_range"), + ], +) +def test_row_id_sequence_accepts_input_types(row_ids): + sequence = RowIdSequence(row_ids) + + assert sorted(sequence) == [1, 2, 3] + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param([], id="empty_list"), + pytest.param(range(0), id="empty_range"), + pytest.param(pa.array([], type=pa.uint64()), id="empty_array"), + ], +) +def test_row_id_sequence_empty(row_ids): + sequence = RowIdSequence(row_ids) + + assert len(sequence) == 0 + assert list(sequence) == [] + assert sequence.to_pyarrow() == pa.array([], type=pa.uint64()) + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param(list(range(4100)), id="contiguous"), + pytest.param(list(range(0, 8200, 2)), id="gapped"), + pytest.param(list(range(4100))[::-1], id="unsorted"), + ], +) +def test_row_id_sequence_iterates_large_sequences(row_ids): + # Each shape picks a different segment encoding, so all of them have to + # round-trip through iteration. + sequence = RowIdSequence(row_ids) + + assert list(sequence) == row_ids + # Each call must hand back a fresh iterator rather than a spent one. + assert list(sequence) == row_ids + + +def test_row_id_sequence_from_range_above_isize(): + # Range bounds are read as isize; beyond that the values are read one at a + # time instead, which must still cover the whole uint64 row id domain. + start = 2**63 + sequence = RowIdSequence(range(start, start + 3)) + + assert list(sequence) == [start, start + 1, start + 2] + + +def test_row_id_sequence_unsorted_round_trips(): + sequence = RowIdSequence([12, 11, 10]) + + assert list(sequence) == [12, 11, 10] + assert sequence.to_pyarrow() == pa.array([12, 11, 10], type=pa.uint64()) + + +@pytest.mark.parametrize( + "row_ids", + [ + pytest.param([1, 1, 2], id="adjacent"), + pytest.param([1, 2, 3, 1], id="separated"), + pytest.param(pa.array([5, 3, 5]), id="unsorted_array"), + ], +) +def test_row_id_sequence_rejects_duplicates(row_ids): + with pytest.raises(ValueError, match="Row ids must be unique"): + RowIdSequence(row_ids) + + +@pytest.mark.parametrize( + ("row_ids", "message"), + [ + pytest.param(pa.array([1, None, 3]), "must not be null", id="null_in_array"), + pytest.param(pa.array([1.5, 2.5]), "array of integers", id="float_array"), + pytest.param(pa.array([-1, 2]), "uint64", id="negative_in_array"), + pytest.param(range(-5, 5), "non-negative", id="negative_range"), + pytest.param(5, "iterable of integers", id="not_iterable"), + ], +) +def test_row_id_sequence_rejects_invalid_input(row_ids, message): + with pytest.raises((ValueError, TypeError), match=message): + RowIdSequence(row_ids) + + +def test_row_id_sequence_metadata_round_trip(): + sequence = RowIdSequence([7, 12, 3]) + + metadata = sequence.to_inline_metadata() + assert isinstance(metadata, RowIdMeta) + assert RowIdSequence.from_inline_metadata(metadata) == sequence + + +def test_row_id_sequence_equality_and_repr(): + assert RowIdSequence(range(3)) == RowIdSequence([0, 1, 2]) + assert RowIdSequence(range(3)) != RowIdSequence([0, 1]) + # Comparing against an unrelated type is False rather than an error. + assert RowIdSequence(range(3)) != "not a sequence" + + assert repr(RowIdSequence([1, 2])) == "RowIdSequence([1, 2])" + assert repr(RowIdSequence(range(12))) == ( + "RowIdSequence([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...], len=12)" + ) + + +def test_row_id_sequence_pickle(): + sequence = RowIdSequence([7, 12, 3]) + + assert pickle.loads(pickle.dumps(sequence)) == sequence + + +def _row_ids_by_id(dataset: LanceDataset) -> dict: + table = dataset.to_table(columns=["id"], with_row_id=True) + return dict(zip(table["id"].to_pylist(), table["_rowid"].to_pylist())) + + +def test_row_id_sequence_preserves_ids_in_manual_update(tmp_path: Path): + # An update assembled externally (delete the old row, append a replacement) + # keeps the row's identity only if the new fragment carries its row id. + dataset = write_dataset( + pa.table({"id": [1, 2, 3, 4], "v": [10, 20, 30, 40]}), + tmp_path, + max_rows_per_file=2, + enable_stable_row_ids=True, + ) + row_ids_before = _row_ids_by_id(dataset) + + updated_fragment = dataset.get_fragments()[0].delete("id = 2") + (new_fragment,) = write_fragments(pa.table({"id": [2], "v": [99]}), tmp_path) + new_fragment.row_id_meta = RowIdSequence([row_ids_before[2]]).to_inline_metadata() + + dataset = LanceDataset.commit( + tmp_path, + LanceOperation.Update( + removed_fragment_ids=[], + updated_fragments=[updated_fragment], + new_fragments=[new_fragment], + fields_modified=[], + ), + read_version=dataset.version, + ) + + assert _row_ids_by_id(dataset) == row_ids_before + assert dataset.to_table().sort_by("id").to_pydict() == { + "id": [1, 2, 3, 4], + "v": [10, 99, 30, 40], + } + + +def test_row_id_sequence_reads_back_fragment_metadata(tmp_path: Path): + dataset = write_dataset( + pa.table({"a": range(10)}), + tmp_path, + max_rows_per_file=5, + enable_stable_row_ids=True, + ) + + sequences = [ + RowIdSequence.from_inline_metadata(fragment.metadata.row_id_meta) + for fragment in dataset.get_fragments() + ] + + assert [list(sequence) for sequence in sequences] == [ + [0, 1, 2, 3, 4], + [5, 6, 7, 8, 9], + ] + + +def test_fragment_validate(tmp_path: Path): + dataset = write_dataset( + pa.table({"a": range(100), "b": range(100)}), + tmp_path, + max_rows_per_file=50, + ) + # A valid fragment validates without raising. + for fragment in dataset.get_fragments(): + assert fragment.validate() is None + + +def test_fragment_validate_across_data_files(tmp_path: Path): + # add_columns writes a second data file per fragment; validate must still + # pass (field ids increasing and unique across a fragment's data files). + dataset = write_dataset(pa.table({"a": range(100)}), tmp_path, max_rows_per_file=50) + dataset.add_columns({"b": "a + 1"}) + for fragment in dataset.get_fragments(): + assert len(fragment.data_files()) > 1 + fragment.validate() + + +def test_fragment_validate_after_delete(tmp_path: Path): + dataset = write_dataset(pa.table({"a": range(100)}), tmp_path, max_rows_per_file=50) + dataset.delete("a < 10") + # A fragment carrying a deletion vector still validates. + for fragment in dataset.get_fragments(): + fragment.validate() + + +def _dataset_with_scalar_index(tmp_path: Path) -> LanceDataset: + dataset = write_dataset( + pa.table({"val": range(10000), "other": range(10000)}), + tmp_path, + max_rows_per_file=5000, + ) + dataset.create_scalar_index("val", index_type="BTREE") + return dataset + + +def test_fragment_scanner_use_scalar_index_disables_index_query(tmp_path: Path): + # A filtered fragment scan on an indexed column plans a dataset-wide + # ScalarIndexQuery (and caches its index pages) unless the scan opts out. + dataset = _dataset_with_scalar_index(tmp_path) + fragment = dataset.get_fragments()[0] + filt = "val >= 10 AND val <= 20" + + default_plan = fragment.scanner(filter=filt, with_row_id=True).explain_plan(True) + assert "ScalarIndexQuery" in default_plan + + opted_out_plan = fragment.scanner( + filter=filt, with_row_id=True, use_scalar_index=False + ).explain_plan(True) + assert "ScalarIndexQuery" not in opted_out_plan + + +@pytest.mark.parametrize("use_scalar_index", [None, True, False]) +def test_fragment_scanner_matches_dataset_scanner(tmp_path: Path, use_scalar_index): + # The fragment scanner must build the same plan as the dataset scanner + # restricted to that single fragment. + dataset = _dataset_with_scalar_index(tmp_path) + fragment = dataset.get_fragments()[0] + filt = "val >= 10 AND val <= 20" + + frag_plan = fragment.scanner( + filter=filt, with_row_id=True, use_scalar_index=use_scalar_index + ).explain_plan(True) + dataset_plan = dataset.scanner( + fragments=[fragment], + filter=filt, + with_row_id=True, + use_scalar_index=use_scalar_index, + ).explain_plan(True) + assert frag_plan == dataset_plan + + +def _fragment_with_deletions(tmp_path: Path) -> LanceFragment: + dataset = write_dataset(pa.table({"a": range(20)}), tmp_path, max_rows_per_file=10) + dataset.delete("a < 3") + return dataset.get_fragments()[0] + + +def test_fragment_scanner_include_deleted_rows(tmp_path: Path): + fragment = _fragment_with_deletions(tmp_path) + assert fragment.physical_rows == 10 + assert fragment.num_deletions == 3 + + # By default the deleted rows are omitted. + default = fragment.to_table(with_row_id=True) + assert default.num_rows == 7 + assert default["a"].to_pylist() == list(range(3, 10)) + + # With include_deleted_rows the deleted rows are surfaced with a null _rowid. + included = fragment.scanner(with_row_id=True, include_deleted_rows=True).to_table() + assert included.num_rows == fragment.physical_rows + assert included["a"].to_pylist() == list(range(10)) + assert included["_rowid"].null_count == fragment.num_deletions + + +def test_fragment_scanner_include_deleted_rows_requires_row_id(tmp_path: Path): + fragment = _fragment_with_deletions(tmp_path) + with pytest.raises(ValueError, match="with_row_id"): + fragment.scanner(include_deleted_rows=True).to_table() + + +def test_fragment_scanner_include_deleted_rows_matches_dataset_scanner(tmp_path: Path): + dataset = write_dataset(pa.table({"a": range(20)}), tmp_path, max_rows_per_file=10) + dataset.delete("a < 3") + fragment = dataset.get_fragments()[0] + + frag_plan = fragment.scanner( + with_row_id=True, include_deleted_rows=True + ).explain_plan(True) + dataset_plan = dataset.scanner( + fragments=[fragment], with_row_id=True, include_deleted_rows=True + ).explain_plan(True) + assert frag_plan == dataset_plan + + +@pytest.mark.parametrize( + ("late_materialization", "is_late"), + [ + pytest.param(None, False, id="default"), + pytest.param(True, True, id="all_late"), + pytest.param(False, False, id="all_early"), + pytest.param(["values"], True, id="late_column"), + pytest.param(["filter"], False, id="early_column"), + ], +) +def test_fragment_scanner_late_materialization( + tmp_path: Path, late_materialization, is_late +): + # With no index, the plan shows whether `values` is fetched late (a take over + # the row stream) or early (materialized in the scan projection). + dataset = write_dataset( + pa.table({"filter": range(2000), "values": range(2000)}), + tmp_path, + data_storage_version="stable", + ) + fragment = dataset.get_fragments()[0] + + plan = fragment.scanner( + filter="filter % 2 == 0", late_materialization=late_materialization + ).explain_plan(True) + + if is_late: + assert "projection=[values], source=stream" in plan + else: + assert "projection=[filter, values]" in plan + + +def test_fragment_scanner_rejects_invalid_late_materialization(tmp_path: Path): + dataset = write_dataset(pa.table({"a": range(10)}), tmp_path) + fragment = dataset.get_fragments()[0] + + with pytest.raises( + ValueError, match="late_materialization must be a bool or a list of strings" + ): + fragment.scanner(late_materialization=123) + + +def test_fragment_scanner_io_buffer_size_forwarded(tmp_path: Path): + # io_buffer_size has no plan-visible marker, so assert it is accepted through + # both scan entry points and leaves results unchanged. + dataset = write_dataset(pa.table({"val": range(1000)}), tmp_path) + fragment = dataset.get_fragments()[0] + filt = "val < 100" + expected = fragment.to_table(filter=filt) + + assert fragment.to_table(filter=filt, io_buffer_size=4 * 1024 * 1024) == expected + + batched = pa.Table.from_batches( + list(fragment.to_batches(filter=filt, io_buffer_size=4 * 1024 * 1024)) + ) + assert batched == expected + + +def test_fragment_scanner_strict_batch_size(tmp_path: Path): + dataset = write_dataset(pa.table({"a": range(1000)}), tmp_path) + fragment = dataset.get_fragments()[0] + filt = "a % 3 == 0" + + # A filtered scan emits uneven, sub-batch_size batches by default. + loose = [b.num_rows for b in fragment.to_batches(batch_size=100, filter=filt)] + assert any(n < 100 for n in loose[:-1]) + + # strict_batch_size coalesces to exactly batch_size (except the last batch). + strict = [ + b.num_rows + for b in fragment.to_batches( + batch_size=100, filter=filt, strict_batch_size=True + ) + ] + assert all(n == 100 for n in strict[:-1]) + assert sum(strict) == sum(loose) + + +def test_fragment_scanner_batch_size_bytes(tmp_path: Path): + # A small byte budget over wide rows forces many more batches than the + # default, without changing the results. + dataset = write_dataset(pa.table({"s": ["x" * 1024] * 2000}), tmp_path) + fragment = dataset.get_fragments()[0] + + default_batches = list(fragment.to_batches()) + small_budget = list(fragment.to_batches(batch_size_bytes=64 * 1024)) + assert len(small_budget) > len(default_batches) + assert pa.Table.from_batches(small_budget) == fragment.to_table() diff --git a/python/python/tests/test_fragment_typing.py b/python/python/tests/test_fragment_typing.py new file mode 100644 index 00000000000..1b33b96919e --- /dev/null +++ b/python/python/tests/test_fragment_typing.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Type-checking regression tests for the fragment write APIs. + +This module is part of the pyright target configured in ``pyproject.toml``, so +a regression in the annotations below fails the repository type check and not +only the runtime suite. + +It is separate from ``test_fragment.py`` because that file currently carries +pre-existing pyright diagnostics unrelated to these APIs, so it cannot join the +type-check target without a much larger cleanup. + +The results are bound to annotated locals on purpose: that pins the selected +overload, and pyright rejects the ``None`` argument if ``max_rows_per_group`` +regresses to a plain ``int``. +""" + +from pathlib import Path +from typing import TYPE_CHECKING, List + +import pyarrow as pa +from lance.fragment import FragmentMetadata, LanceFragment, write_fragments + +if TYPE_CHECKING: + from lance import Transaction + + +def test_write_fragments_accepts_none_max_rows_per_group(tmp_path: Path) -> None: + table = pa.table({"a": range(8)}) + + fragments: List[FragmentMetadata] = write_fragments( + table, str(tmp_path / "fragments"), max_rows_per_group=None + ) + assert len(fragments) == 1 + assert fragments[0].physical_rows == 8 + + +def test_write_fragments_transaction_accepts_none_max_rows_per_group( + tmp_path: Path, +) -> None: + table = pa.table({"a": range(8)}) + + transaction: "Transaction" = write_fragments( + table, + str(tmp_path / "transaction"), + max_rows_per_group=None, + return_transaction=True, + ) + assert transaction.operation is not None + + +def test_fragment_create_accepts_none_max_rows_per_group(tmp_path: Path) -> None: + table = pa.table({"a": range(8)}) + + fragment: FragmentMetadata = LanceFragment.create( + str(tmp_path / "create"), table, max_rows_per_group=None + ) + assert fragment.physical_rows == 8 diff --git a/python/python/tests/test_geo.py b/python/python/tests/test_geo.py index c011c2de3de..f7d9d7b3e1a 100644 --- a/python/python/tests/test_geo.py +++ b/python/python/tests/test_geo.py @@ -19,6 +19,19 @@ ) +def _query_point_ids(dataset: lance.LanceDataset, wkt: str) -> list[int]: + sql = f""" + SELECT id, point + FROM dataset + WHERE St_Intersects(point, ST_GeomFromText('{wkt}')) + """ + return [ + value + for batch in dataset.sql(sql).build().to_batch_records() + for value in batch.column("id").to_pylist() + ] + + def test_geo_types(tmp_path: Path): uri = str(tmp_path / "test_geo_types.lance") # Points @@ -153,3 +166,276 @@ def query(ds: lance.LanceDataset, has_index=False): table_with_index = query(ds, has_index=True) assert table_with_index == table_without_index + + +def test_rtree_segment_merge_and_commit(tmp_path: Path): + num_points = 120 + points_2d = points( + [ + np.arange(num_points, dtype=np.float64), + np.arange(num_points, dtype=np.float64), + ] + ) + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point("xy")).with_name("point"), + ] + ) + table = pa.Table.from_arrays( + [np.arange(num_points, dtype=np.int64), points_2d], schema=schema + ) + ds = lance.write_dataset( + table, + str(tmp_path / "segmented_rtree.lance"), + max_rows_per_file=40, + ) + fragments = ds.get_fragments() + assert len(fragments) == 3 + segments = [ + ds.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[fragment.fragment_id], + ) + for fragment in fragments + ] + + merged = ds.merge_existing_index_segments(segments) + assert set(merged.fragment_ids) == {fragment.fragment_id for fragment in fragments} + ds = ds.commit_existing_index_segments("point_rtree", "point", [merged]) + + sql = """ + SELECT id, point + FROM dataset + WHERE St_Intersects(point, ST_GeomFromText('LINESTRING (10 10, 110 110)')) + """ + indexed = pa.Table.from_batches(ds.sql(sql).build().to_batch_records()) + assert indexed["id"].to_pylist() == list(range(10, 111)) + explain = ( + pa.Table.from_batches( + ds.sql("EXPLAIN ANALYZE " + sql).build().to_batch_records() + ) + .to_pandas() + .to_string() + ) + assert "ScalarIndexQuery" in explain + + +def test_staged_rtree_after_rewrite_columns(tmp_path: Path): + uri = str(tmp_path / "stale_rtree.lance") + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + dataset = lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array([0, 1], type=pa.int64()), + points([np.array([0.0, 1.0]), np.array([0.0, 1.0])]), + ], + schema=schema, + ), + uri, + ) + segment = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[0], + ) + + update_schema = pa.schema( + [ + pa.field("_rowid", pa.uint64()), + pa.field(point_type).with_name("point"), + ] + ) + update = pa.Table.from_arrays( + [ + pa.array([0], type=pa.uint64()), + points([np.array([10.0]), np.array([10.0])]), + ], + schema=update_schema, + ) + fragment, fields = dataset.get_fragment(0).update_columns(update) + updated = lance.LanceDataset.commit( + uri, + lance.LanceOperation.Update( + updated_fragments=[fragment], + fields_modified=fields, + ), + read_version=dataset.version, + ) + + assert _query_point_ids(updated, "POINT (10 10)") == [0] + committed = dataset.commit_existing_index_segments( + "point_rtree", + "point", + [segment], + ) + assert committed.describe_indices()[0].segments[0].fragment_ids == set() + assert _query_point_ids(committed, "POINT (10 10)") == [0] + + +def test_rtree_rejects_distributed_uuid_reuse(tmp_path: Path): + uri = str(tmp_path / "uuid_reuse.lance") + num_points = 120 + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + dataset = lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array(range(num_points), type=pa.int64()), + points( + [ + np.arange(num_points, dtype=np.float64), + np.arange(num_points, dtype=np.float64), + ] + ), + ], + schema=schema, + ), + uri, + max_rows_per_file=40, + ) + dataset.create_scalar_index("point", "RTREE") + index_uuid = dataset.describe_indices()[0].segments[0].uuid + + with pytest.raises( + ValueError, + match="index_uuid is no longer accepted for RTree distributed index builds", + ): + dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree_reuse", + fragment_ids=[0], + index_uuid=index_uuid, + ) + + assert _query_point_ids( + lance.dataset(uri), + "LINESTRING (100 100, 110 110)", + ) == list(range(100, 111)) + + +def test_rtree_merge_all_deleted_stable_row_ids(tmp_path: Path): + uri = str(tmp_path / "all_deleted.lance") + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + dataset = lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array([0, 1], type=pa.int64()), + points([np.array([0.0, 1.0]), np.array([0.0, 1.0])]), + ], + schema=schema, + ), + uri, + enable_stable_row_ids=True, + ) + segment = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[0], + ) + + dataset.delete("true") + merged = dataset.merge_existing_index_segments([segment]) + assert merged.fragment_ids == set() + committed = dataset.commit_existing_index_segments( + "point_rtree", + "point", + [merged], + ) + assert _query_point_ids(committed, "POINT (0 0)") == [] + + +def test_rtree_merge_preserves_newer_fragment_coverage(tmp_path: Path): + uri = str(tmp_path / "mixed_versions.lance") + point_type = point("xy") + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field(point_type).with_name("point"), + ] + ) + + def batch(start: int, stop: int) -> pa.Table: + values = np.arange(start, stop, dtype=np.float64) + return pa.Table.from_arrays( + [ + pa.array(range(start, stop), type=pa.int64()), + points([values, values]), + ], + schema=schema, + ) + + dataset = lance.write_dataset(batch(0, 40), uri) + first = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[0], + ) + dataset = lance.write_dataset(batch(40, 80), uri, mode="append") + second = dataset.create_index_uncommitted( + column="point", + index_type="RTREE", + name="point_rtree", + fragment_ids=[1], + ) + + merged = dataset.merge_existing_index_segments([first, second]) + assert merged.fragment_ids == {0, 1} + assert merged.dataset_version == dataset.version + + update_schema = pa.schema( + [ + pa.field("_rowid", pa.uint64()), + pa.field(point_type).with_name("point"), + ] + ) + update = pa.Table.from_arrays( + [ + pa.array([1 << 32], type=pa.uint64()), + points([np.array([100.0]), np.array([100.0])]), + ], + schema=update_schema, + ) + fragment, fields = dataset.get_fragment(1).update_columns(update) + dataset = lance.LanceDataset.commit( + uri, + lance.LanceOperation.Update( + updated_fragments=[fragment], + fields_modified=fields, + ), + read_version=dataset.version, + ) + + committed = dataset.commit_existing_index_segments( + "point_rtree", + "point", + [merged], + ) + assert committed.describe_indices()[0].segments[0].fragment_ids == {0} + assert _query_point_ids( + committed, + "POINT (100 100)", + ) == [40] diff --git a/python/python/tests/test_indices.py b/python/python/tests/test_indices.py index 02cf64541d6..25b391b5ad1 100644 --- a/python/python/tests/test_indices.py +++ b/python/python/tests/test_indices.py @@ -8,27 +8,37 @@ import numpy as np import pyarrow as pa import pytest -from lance.file import LanceFileReader +from lance.file import LanceFileReader, LanceFileWriter from lance.indices import IndicesBuilder, IvfModel, PqModel -NUM_ROWS_PER_FRAGMENT = 10000 DIMENSION = 128 NUM_SUBVECTORS = 8 NUM_FRAGMENTS = 3 -NUM_ROWS = NUM_ROWS_PER_FRAGMENT * NUM_FRAGMENTS -NUM_PARTITIONS = round(np.sqrt(NUM_ROWS)) - - SMALL_ROWS_PER_FRAGMENT = 100 SMALL_NUM_ROWS = SMALL_ROWS_PER_FRAGMENT * NUM_FRAGMENTS - - -def make_ds(num_rows: int, rows_per_frag: int, tmpdir: pathlib.Path, dtype: str): - vectors = np.random.randn(num_rows, DIMENSION).astype(dtype) +SMALL_NUM_PARTITIONS = round(np.sqrt(SMALL_NUM_ROWS)) +PQ_ROWS_PER_FRAGMENT = 512 +PQ_NUM_ROWS = PQ_ROWS_PER_FRAGMENT * NUM_FRAGMENTS +MOSTLY_NULL_ROWS_PER_FRAGMENT = 2000 +MOSTLY_NULL_NUM_ROWS = MOSTLY_NULL_ROWS_PER_FRAGMENT * NUM_FRAGMENTS +MOSTLY_NULL_NUM_PARTITIONS = round(np.sqrt(MOSTLY_NULL_NUM_ROWS)) +TRAINING_SAMPLE_RATE = 2 +TRAINING_MAX_ITERS = 2 + + +def make_ds( + num_rows: int, + rows_per_frag: int, + tmpdir: pathlib.Path, + dtype: str, + name: str = "dataset", +): + vectors = np.random.default_rng(42).standard_normal((num_rows, DIMENSION)) + vectors = vectors.astype(dtype) vectors = vectors.reshape(-1) vectors = pa.FixedSizeListArray.from_arrays(vectors, DIMENSION) table = pa.Table.from_arrays([vectors], names=["vectors"]) - uri = str(tmpdir / "dataset") + uri = str(tmpdir / name) ds = lance.write_dataset(table, uri, max_rows_per_file=rows_per_frag) return ds @@ -38,30 +48,58 @@ def make_ds(num_rows: int, rows_per_frag: int, tmpdir: pathlib.Path, dtype: str) params=[np.float16, np.float32, np.float64], ids=["f16", "f32", "f64"], ) -def rand_dataset(tmpdir, request): - return make_ds(NUM_ROWS, NUM_ROWS_PER_FRAGMENT, tmpdir, request.param) +def small_rand_dataset(tmpdir, request): + return make_ds(SMALL_NUM_ROWS, SMALL_ROWS_PER_FRAGMENT, tmpdir, request.param) + + +@pytest.fixture +def small_float32_dataset(tmpdir): + return make_ds(SMALL_NUM_ROWS, SMALL_ROWS_PER_FRAGMENT, tmpdir, np.float32) @pytest.fixture( params=[np.float16, np.float32, np.float64], ids=["f16", "f32", "f64"], ) -def small_rand_dataset(tmpdir, request): - return make_ds(SMALL_NUM_ROWS, SMALL_ROWS_PER_FRAGMENT, tmpdir, request.param) +def pq_rand_dataset(tmpdir, request): + return make_ds( + PQ_NUM_ROWS, + PQ_ROWS_PER_FRAGMENT, + tmpdir, + request.param, + name="pq_dataset", + ) @pytest.fixture -def mostly_null_dataset(tmpdir, request): - vectors = np.random.randn(NUM_ROWS, DIMENSION).astype(np.float32) - vectors = vectors.reshape(-1) - vectors = pa.FixedSizeListArray.from_arrays(vectors, DIMENSION) - vectors = vectors.to_pylist() - vectors = [vec if i % 10 == 0 else None for i, vec in enumerate(vectors)] - vectors = pa.array(vectors, pa.list_(pa.float32(), DIMENSION)) +def pq_float32_dataset(tmpdir): + return make_ds( + PQ_NUM_ROWS, + PQ_ROWS_PER_FRAGMENT, + tmpdir, + np.float32, + name="pq_dataset", + ) + + +@pytest.fixture +def mostly_null_dataset(tmpdir): + values = np.random.default_rng(42).standard_normal(MOSTLY_NULL_NUM_ROWS * DIMENSION) + values = pa.array(values.astype(np.float32)) + null_mask = pa.array(np.arange(MOSTLY_NULL_NUM_ROWS) % 10 != 0) + vectors = pa.FixedSizeListArray.from_arrays( + values, + DIMENSION, + mask=null_mask, + ) table = pa.Table.from_arrays([vectors], names=["vectors"]) uri = str(tmpdir / "nulls_dataset") - ds = lance.write_dataset(table, uri, max_rows_per_file=NUM_ROWS_PER_FRAGMENT) + ds = lance.write_dataset( + table, + uri, + max_rows_per_file=MOSTLY_NULL_ROWS_PER_FRAGMENT, + ) return ds @@ -91,11 +129,14 @@ def make_multivector_dataset(tmpdir): return ds, dimension -def test_ivf_centroids(tmpdir, rand_dataset): - ivf = IndicesBuilder(rand_dataset, "vectors").train_ivf(sample_rate=16) +def test_ivf_centroids(tmpdir, small_rand_dataset): + ivf = IndicesBuilder(small_rand_dataset, "vectors").train_ivf( + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + ) assert ivf.distance_type == "l2" - assert len(ivf.centroids) == NUM_PARTITIONS + assert len(ivf.centroids) == SMALL_NUM_PARTITIONS ivf.save(str(tmpdir / "ivf")) reloaded = IvfModel.load(str(tmpdir / "ivf")) @@ -104,18 +145,25 @@ def test_ivf_centroids(tmpdir, rand_dataset): def test_ivf_centroids_hamming(tmpdir): - num_rows = NUM_ROWS - vectors = np.random.randint(0, 256, size=(num_rows, DIMENSION), dtype=np.uint8) + num_rows = SMALL_NUM_ROWS + vectors = np.random.default_rng(42).integers( + 0, + 256, + size=(num_rows, DIMENSION), + dtype=np.uint8, + ) vectors_flat = vectors.reshape(-1) vectors_arr = pa.FixedSizeListArray.from_arrays( pa.array(vectors_flat, type=pa.uint8()), DIMENSION ) table = pa.Table.from_arrays([vectors_arr], names=["vectors"]) uri = str(tmpdir / "hamming_dataset") - ds = lance.write_dataset(table, uri, max_rows_per_file=NUM_ROWS_PER_FRAGMENT) + ds = lance.write_dataset(table, uri, max_rows_per_file=SMALL_ROWS_PER_FRAGMENT) ivf = IndicesBuilder(ds, "vectors").train_ivf( - sample_rate=16, distance_type="hamming" + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + distance_type="hamming", ) assert ivf.distance_type == "hamming" @@ -131,40 +179,49 @@ def test_ivf_centroids_hamming(tmpdir): @pytest.mark.parametrize("distance_type", ["l2", "cosine", "dot"]) def test_ivf_centroids_mostly_null(mostly_null_dataset, distance_type): ivf = IndicesBuilder(mostly_null_dataset, "vectors").train_ivf( - sample_rate=16, distance_type=distance_type + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + distance_type=distance_type, ) assert ivf.distance_type == distance_type - assert len(ivf.centroids) == NUM_PARTITIONS + assert len(ivf.centroids) == MOSTLY_NULL_NUM_PARTITIONS @pytest.mark.cuda -def test_ivf_centroids_cuda(rand_dataset): - ivf = IndicesBuilder(rand_dataset, "vectors").train_ivf( - sample_rate=16, accelerator="cuda" +def test_ivf_centroids_cuda(small_rand_dataset): + ivf = IndicesBuilder(small_rand_dataset, "vectors").train_ivf( + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + accelerator="cuda", ) assert ivf.distance_type == "l2" - # Can't use NUM_PARTITIONS here because + # Can't use SMALL_NUM_PARTITIONS here because # CUDA uses math.ceil and CPU uses round to calc. num_partitions - assert len(ivf.centroids) == math.ceil(np.sqrt(NUM_ROWS)) + assert len(ivf.centroids) == math.ceil(np.sqrt(SMALL_NUM_ROWS)) @pytest.mark.cuda @pytest.mark.parametrize("distance_type", ["l2", "cosine", "dot"]) def test_ivf_centroids_mostly_null_cuda(mostly_null_dataset, distance_type): ivf = IndicesBuilder(mostly_null_dataset, "vectors").train_ivf( - sample_rate=16, accelerator="cuda", distance_type=distance_type + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + accelerator="cuda", + distance_type=distance_type, ) assert ivf.distance_type == distance_type - assert len(ivf.centroids) == NUM_PARTITIONS + assert len(ivf.centroids) == MOSTLY_NULL_NUM_PARTITIONS -def test_ivf_centroids_distance_type(tmpdir, rand_dataset): +def test_ivf_centroids_distance_type(tmpdir, small_float32_dataset): def check(distance_type): - ivf = IndicesBuilder(rand_dataset, "vectors").train_ivf( - sample_rate=16, distance_type=distance_type + ivf = IndicesBuilder(small_float32_dataset, "vectors").train_ivf( + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + distance_type=distance_type, ) assert ivf.distance_type == distance_type ivf.save(str(tmpdir / "ivf")) @@ -176,31 +233,44 @@ def check(distance_type): check("dot") -def test_num_partitions(rand_dataset): - ivf = IndicesBuilder(rand_dataset, "vectors").train_ivf( - sample_rate=16, num_partitions=10 +def test_num_partitions(small_float32_dataset): + ivf = IndicesBuilder(small_float32_dataset, "vectors").train_ivf( + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + num_partitions=10, ) assert ivf.num_partitions == 10 @pytest.fixture -def rand_ivf(rand_dataset): - dtype = rand_dataset.schema.field("vectors").type.value_type.to_pandas_dtype() - centroids = np.random.rand(DIMENSION * 100).astype(dtype) +def small_rand_ivf(small_rand_dataset): + dtype = small_rand_dataset.schema.field("vectors").type.value_type.to_pandas_dtype() + centroids = np.random.default_rng(42).random(DIMENSION * 100).astype(dtype) centroids = pa.FixedSizeListArray.from_arrays(centroids, DIMENSION) return IvfModel(centroids, "l2") @pytest.fixture -def small_rand_ivf(small_rand_dataset): - dtype = small_rand_dataset.schema.field("vectors").type.value_type.to_pandas_dtype() - centroids = np.random.rand(DIMENSION * 100).astype(dtype) +def pq_rand_ivf(pq_rand_dataset): + dtype = pq_rand_dataset.schema.field("vectors").type.value_type.to_pandas_dtype() + centroids = np.random.default_rng(42).random(DIMENSION * 100).astype(dtype) centroids = pa.FixedSizeListArray.from_arrays(centroids, DIMENSION) return IvfModel(centroids, "l2") -def test_gen_pq(tmpdir, rand_dataset, rand_ivf): - pq = IndicesBuilder(rand_dataset, "vectors").train_pq(rand_ivf, sample_rate=2) +@pytest.fixture +def small_float32_ivf(small_float32_dataset): + centroids = np.random.default_rng(42).random(DIMENSION * 100).astype(np.float32) + centroids = pa.FixedSizeListArray.from_arrays(centroids, DIMENSION) + return IvfModel(centroids, "l2") + + +def test_gen_pq(tmpdir, pq_rand_dataset, pq_rand_ivf): + pq = IndicesBuilder(pq_rand_dataset, "vectors").train_pq( + pq_rand_ivf, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + ) assert pq.dimension == DIMENSION assert pq.num_subvectors == NUM_SUBVECTORS @@ -209,6 +279,30 @@ def test_gen_pq(tmpdir, rand_dataset, rand_ivf): assert pq.dimension == reloaded.dimension assert pq.codebook == reloaded.codebook + pq_4bit = IndicesBuilder(pq_rand_dataset, "vectors").train_pq( + pq_rand_ivf, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + num_bits=4, + ) + assert pq_4bit.num_bits == 4 + assert len(pq_4bit.codebook) == 16 + + pq_4bit.save(str(tmpdir / "pq_4bit")) + reloaded = PqModel.load(str(tmpdir / "pq_4bit")) + assert reloaded.num_bits == 4 + + legacy_pq_uri = str(tmpdir / "legacy_pq") + with LanceFileWriter( + legacy_pq_uri, + pa.schema( + [pa.field("codebook", pq.codebook.type)], + metadata={b"num_subvectors": str(pq.num_subvectors).encode()}, + ), + ) as writer: + writer.write_batch(pa.table([pq.codebook], names=["codebook"])) + assert PqModel.load(legacy_pq_uri).num_bits == 8 + def test_ivf_centroids_fragment_ids(tmpdir): rows_per_fragment = 32 @@ -231,10 +325,16 @@ def test_ivf_centroids_fragment_ids(tmpdir): fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] first_ivf = IndicesBuilder(ds, "vectors").train_ivf( - num_partitions=1, sample_rate=2, fragment_ids=[fragment_ids[0]] + num_partitions=1, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + fragment_ids=[fragment_ids[0]], ) second_ivf = IndicesBuilder(ds, "vectors").train_ivf( - num_partitions=1, sample_rate=2, fragment_ids=[fragment_ids[1]] + num_partitions=1, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + fragment_ids=[fragment_ids[1]], ) first_centroid = first_ivf.centroids.values.to_numpy().reshape(-1, DIMENSION)[0] @@ -300,7 +400,7 @@ def test_indices_builder_multivector_distributed_dimensions(tmpdir, monkeypatch) captured_dimensions = {} - def train_pq_model(*args): + def train_pq_model(*args, **kwargs): captured_dimensions["train_pq"] = args[2] return codebook @@ -326,17 +426,19 @@ def load_shuffled_vectors(*args): } -def test_pq_fragment_ids(rand_dataset): - fragment_id = rand_dataset.get_fragments()[0].fragment_id - ivf = IndicesBuilder(rand_dataset, "vectors").train_ivf( +def test_pq_fragment_ids(pq_float32_dataset): + fragment_id = pq_float32_dataset.get_fragments()[0].fragment_id + ivf = IndicesBuilder(pq_float32_dataset, "vectors").train_ivf( num_partitions=4, - sample_rate=16, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, fragment_ids=[fragment_id], ) - pq = IndicesBuilder(rand_dataset, "vectors").train_pq( + pq = IndicesBuilder(pq_float32_dataset, "vectors").train_pq( ivf, - sample_rate=2, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, fragment_ids=[fragment_id], ) @@ -344,31 +446,41 @@ def test_pq_fragment_ids(rand_dataset): assert pq.num_subvectors == NUM_SUBVECTORS -def test_pq_invalid_sub_vectors(tmpdir, rand_dataset, rand_ivf): +def test_pq_invalid_sub_vectors( + small_float32_dataset, + small_float32_ivf, +): with pytest.raises( ValueError, match="must be divisible by num_subvectors .* without remainder", ): - IndicesBuilder(rand_dataset, "vectors").train_pq( - rand_ivf, sample_rate=2, num_subvectors=5 + IndicesBuilder(small_float32_dataset, "vectors").train_pq( + small_float32_ivf, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + num_subvectors=5, ) def test_gen_pq_mostly_null(mostly_null_dataset): - centroids = np.random.rand(DIMENSION * 100).astype(np.float32) + centroids = np.random.default_rng(42).random(DIMENSION * 100).astype(np.float32) centroids = pa.FixedSizeListArray.from_arrays(centroids, DIMENSION) ivf = IvfModel(centroids, "l2") - pq = IndicesBuilder(mostly_null_dataset, "vectors").train_pq(ivf, sample_rate=2) + pq = IndicesBuilder(mostly_null_dataset, "vectors").train_pq( + ivf, + sample_rate=TRAINING_SAMPLE_RATE, + max_iters=TRAINING_MAX_ITERS, + ) assert pq.dimension == DIMENSION assert pq.num_subvectors == NUM_SUBVECTORS @pytest.mark.cuda -def test_assign_partitions(rand_dataset, rand_ivf): - builder = IndicesBuilder(rand_dataset, "vectors") +def test_assign_partitions(small_rand_dataset, small_rand_ivf): + builder = IndicesBuilder(small_rand_dataset, "vectors") - partitions_uri = builder.assign_ivf_partitions(rand_ivf, accelerator="cuda") + partitions_uri = builder.assign_ivf_partitions(small_rand_ivf, accelerator="cuda") partitions = lance.dataset(partitions_uri) found_row_ids = set() @@ -379,13 +491,13 @@ def test_assign_partitions(rand_dataset, rand_ivf): part_ids = batch["partition"] for part_id in part_ids: assert part_id.as_py() < 100 - assert len(found_row_ids) == rand_dataset.count_rows() + assert len(found_row_ids) == small_rand_dataset.count_rows() @pytest.mark.cuda @pytest.mark.parametrize("distance_type", ["l2", "cosine", "dot"]) def test_assign_partitions_mostly_null(mostly_null_dataset, distance_type): - centroids = np.random.rand(DIMENSION * 100).astype(np.float32) + centroids = np.random.default_rng(42).random(DIMENSION * 100).astype(np.float32) centroids = pa.FixedSizeListArray.from_arrays(centroids, DIMENSION) ivf = IvfModel(centroids, distance_type) @@ -408,7 +520,7 @@ def test_assign_partitions_mostly_null(mostly_null_dataset, distance_type): @pytest.fixture def small_rand_pq(small_rand_dataset, small_rand_ivf): dtype = small_rand_dataset.schema.field("vectors").type.value_type.to_pandas_dtype() - codebook = np.random.rand(DIMENSION * 256).astype(dtype) + codebook = np.random.default_rng(42).random(DIMENSION * 256).astype(dtype) codebook = pa.FixedSizeListArray.from_arrays(codebook, DIMENSION) pq = PqModel(NUM_SUBVECTORS, codebook) return pq diff --git a/python/python/tests/test_lance.py b/python/python/tests/test_lance.py index 0162e370665..4ba9281823d 100644 --- a/python/python/tests/test_lance.py +++ b/python/python/tests/test_lance.py @@ -248,6 +248,74 @@ def test_io_counters(tmp_path): assert lance.bytes_read_counter() > starting_bytes +def test_simd_info(): + info = lance.simd_info() + assert info["tier"] in ( + "none", + "sse", + "avx", + "avx_fma", + "avx2", + "avx512", + "avx512_fp16", + "neon", + "lsx", + "lasx", + ) + assert isinstance(info["target_arch"], str) and info["target_arch"] + if info["target_arch"] == "x86_64": + # The x86_64 ABI mandates SSE2. + assert "sse2" in info["host_features"] + else: + assert info["host_features"] == [] + + +@pytest.mark.parametrize( + ("query", "options", "expected"), + [ + ("the cats and dogs", {}, [("cat", 0), ("dog", 2)]), + ( + "the skip alpha", + {"stem": False, "custom_stop_words": ["skip"]}, + [("the", 0), ("alpha", 2)], + ), + ( + "getUserName", + {"analyzer": "code", "split_identifiers": True}, + [ + ("getusername", 0), + ("get", 0), + ("user", 1), + ("name", 2), + ], + ), + ( + "a::b", + {"analyzer": "code", "index_operators": True}, + [("a", 0), ("::", 1), ("b", 2)], + ), + ], +) +def test_tokenize_fts_query(query, options, expected): + tokens = lance.tokenize(query, **options) + assert [(token.text, token.position) for token in tokens] == expected + + +def test_tokenize_fts_query_validates_analyzer_options(): + with pytest.raises(ValueError, match="code analyzer flags require analyzer='code'"): + lance.tokenize("getUserName", split_identifiers=True) + + with pytest.raises(ValueError, match="unknown base tokenizer"): + lance.tokenize("hello", base_tokenizer="unknown") + + +def test_tokenize_fts_query_can_disable_max_token_length(): + long_token = "x" * 41 + assert lance.tokenize(long_token, stem=False) == [] + unlimited_tokens = lance.tokenize(long_token, max_token_length=None, stem=False) + assert [token.text for token in unlimited_tokens] == [long_token] + + @pytest.mark.parametrize( "row_param, column_name", [("with_row_id", "_rowid"), ("with_row_address", "_rowaddr")], diff --git a/python/python/tests/test_mem_wal.py b/python/python/tests/test_mem_wal.py index 95596a7123e..1729297e5f5 100644 --- a/python/python/tests/test_mem_wal.py +++ b/python/python/tests/test_mem_wal.py @@ -9,6 +9,7 @@ import pyarrow as pa import pytest from lance.mem_wal import ( + CompactedSsTable, LsmPointLookupPlanner, LsmScanner, LsmVectorSearchPlanner, @@ -56,15 +57,15 @@ def _append_only_table(ids, prefix: str) -> pa.Table: ) -def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa.Table): - """Write a flushed-generation Lance dataset at the expected sub-path. +def _write_sstable(base_path: str, shard_id: str, gen_folder: str, data: pa.Table): + """Write an SSTable Lance dataset at the expected sub-path. - The collector resolves flushed generation paths as: + The collector resolves SSTable paths as: {base_dataset_path}/_mem_wal/{shard_id}/{gen_folder} Production flush also writes a primary-key dedup sidecar (`_pk_index/`) that the LSM scanner opens to dedup across generations; stage it here too so the - flushed generation faithfully matches what flush produces. + SSTable faithfully matches what flush produces. """ from lance.lance import _write_pk_sidecar @@ -73,17 +74,36 @@ def _write_flushed_gen(base_path: str, shard_id: str, gen_folder: str, data: pa. _write_pk_sidecar(gen_path, data, ["id"]) +def test_mark_sstables_as_compacted(tmp_path): + ds_path = str(tmp_path / "base") + shard_id = str(uuid.uuid4()) + dataset = lance.write_dataset( + _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA + ) + dataset.initialize_mem_wal() + + ( + dataset.merge_insert("id") + .when_matched_update_all() + .when_not_matched_insert_all() + .mark_sstables_as_compacted([CompactedSsTable(shard_id, 1)]) + .execute(_lookup_table([2, 4], "compacted")) + ) + + assert dataset.count_rows() == 4 + + def test_point_lookup_with_memtables(tmp_path): """ - Lookup against a base table that has one flushed generation containing an - update. The flushed version must win over the base table version. + Lookup against a base table that has one SSTable containing an + update. The SSTable version must win over the base table version. Setup ----- base : ids [1, 2, 3] names ["base_1", "base_2", "base_3"] gen_1 : ids [2] names ["gen1_2"] ← update to id=2 - ShardSnapshot: flushed_generation(gen=1, path="gen_1"), current_generation=2 + ShardSnapshot: sstable(gen=1, path="gen_1"), current_generation=2 """ ds_path = str(tmp_path / "base") shard_id = str(uuid.uuid4()) @@ -94,28 +114,25 @@ def test_point_lookup_with_memtables(tmp_path): ) base_ds.initialize_mem_wal() - # --- Flushed generation: overwrites id=2 --- - _write_flushed_gen(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) + # --- SSTable: overwrites id=2 --- + _write_sstable(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) - # --- ShardSnapshot describing the flushed state --- - snap = ( - ShardSnapshot(shard_id) - .with_flushed_generation(1, "gen_1") - .with_current_generation(2) - ) + # --- ShardSnapshot describing the SSTable state --- + snap = ShardSnapshot(shard_id).with_sstable(1, "gen_1").with_current_generation(2) planner = LsmPointLookupPlanner(base_ds, [snap]) assert not hasattr(planner, "lookup") - # id=2 must return the flushed version + # id=2 must return the SSTable version plan = planner.plan_lookup(pa.array([2], type=pa.int64())) assert plan.schema.names == ["id", "name"] assert plan.dataset_schema.names == ["id", "name"] - assert "Take" in plan.explain() or "Scan" in plan.explain() + explained = plan.explain() + assert "Take" in explained or "Scan" in explained or "LanceRead" in explained result = plan.to_table() assert len(result) == 1, "Expected exactly one row for id=2" assert result.column("name")[0].as_py() == "gen1_2", ( - "Flushed generation must win over base table" + "SSTable must win over base table" ) # id=1 is only in the base table @@ -146,13 +163,9 @@ def test_lsm_scanner_with_memtables(tmp_path): ) base_ds.initialize_mem_wal() - _write_flushed_gen(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) + _write_sstable(ds_path, shard_id, "gen_1", _lookup_table([2], "gen1")) - snap = ( - ShardSnapshot(shard_id) - .with_flushed_generation(1, "gen_1") - .with_current_generation(2) - ) + snap = ShardSnapshot(shard_id).with_sstable(1, "gen_1").with_current_generation(2) scanner = LsmScanner.from_snapshots(base_ds, [snap]) table = scanner.to_table() @@ -161,7 +174,7 @@ def test_lsm_scanner_with_memtables(tmp_path): name_by_id = {row["id"]: row["name"] for row in table.to_pylist()} assert name_by_id[1] == "base_1" - assert name_by_id[2] == "gen1_2", "Flushed gen must overwrite base for id=2" + assert name_by_id[2] == "gen1_2", "SSTable gen must overwrite base for id=2" assert name_by_id[3] == "base_3" offset_table = ( @@ -170,7 +183,7 @@ def test_lsm_scanner_with_memtables(tmp_path): assert len(offset_table) == 2, "Offset-only LSM scan should not require a limit" -def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path): +def test_shard_writer_lsm_scanner_includes_own_sstables(tmp_path): ds_path = str(tmp_path / "base") shard_id = str(uuid.uuid4()) ds = lance.write_dataset(_lookup_table([0], "base"), ds_path, schema=_LOOKUP_SCHEMA) @@ -192,10 +205,37 @@ def test_shard_writer_lsm_scanner_includes_own_flushed_generations(tmp_path): if name_by_id.get(1) == "writer_1" and name_by_id.get(2) == "writer_2": break if time.time() >= deadline: - assert False, "writer.lsm_scanner() did not include flushed writer rows" + assert False, "writer.lsm_scanner() did not include SSTable writer rows" time.sleep(0.05) +def test_shard_writer_delete_binding_masks_base_row(tmp_path): + ds_path = str(tmp_path / "base") + shard_id = str(uuid.uuid4()) + ds = lance.write_dataset( + _lookup_table([1, 2, 3], "base"), ds_path, schema=_LOOKUP_SCHEMA + ) + ds.initialize_mem_wal() + + delete_keys = pa.table({"id": pa.array([2], type=pa.int64())}) + + with ds.mem_wal_writer( + shard_id, + durable_write=True, + max_wal_buffer_size=1, + max_wal_flush_interval_ms=10, + ) as writer: + writer.put(_lookup_table([4], "writer")) + writer.delete(delete_keys) + table = writer.lsm_scanner().to_table() + + rows = {row["id"]: row["name"] for row in table.to_pylist()} + assert rows[1] == "base_1" + assert 2 not in rows, "deleted base row should be masked by the tombstone" + assert rows[3] == "base_3" + assert rows[4] == "writer_4" + + _VDIM = 4 # matches Rust test fixture dimension @@ -305,7 +345,7 @@ def test_shard_writer_e2e_correctness(tmp_path): End-to-end correctness test for ShardWriter covering: - Multi-round writes that trigger WAL and MemTable flushes - File-system layout verification (_mem_wal//wal/ and manifest/) - - Flushed generation data readable via LsmScanner + - SSTable data readable via LsmScanner - New writer created after close can write and scan correctly Mirrors Rust test: shard_writer_tests::test_shard_writer_e2e_correctness @@ -327,7 +367,6 @@ def test_shard_writer_e2e_correctness(tmp_path): writer = ds.mem_wal_writer( shard_id, durable_write=True, - sync_indexed_write=True, max_wal_buffer_size=10 * 1024, # 10 KB max_wal_flush_interval_ms=50, max_memtable_size=80, # flush after ~80 rows @@ -353,6 +392,9 @@ def test_shard_writer_e2e_correctness(tmp_path): assert closed_memtable_stats["row_count"] == 0 assert closed_memtable_stats["batch_count"] == 0 assert closed_memtable_stats["generation"] >= 1 + assert "frozen_count" in closed_memtable_stats + # close() flushes every frozen memtable, so nothing is still owed to flush. + assert closed_memtable_stats["frozen_bytes"] == 0 # === File-system layout === mem_wal_dir = os.path.join(ds_path, "_mem_wal", shard_id) @@ -377,9 +419,7 @@ def test_shard_writer_e2e_correctness(tmp_path): # === New writer: write and read back via active MemTable scanner === ds2 = lance.dataset(ds_path) shard_id2 = str(uuid.uuid4()) - with ds2.mem_wal_writer( - shard_id2, durable_write=False, sync_indexed_write=True - ) as writer2: + with ds2.mem_wal_writer(shard_id2, durable_write=False) as writer2: verify_batch = _e2e_batch(schema, start_id=10000, num_rows=10) writer2.put(pa.Table.from_batches([verify_batch])) result = writer2.lsm_scanner().to_table() @@ -494,7 +534,6 @@ def test_initialize_mem_wal_writer_config_defaults(tmp_path): # Duration knobs are recorded in milliseconds with a `_ms` suffix. assert defaults["max_wal_flush_interval_ms"] == "250" # Every ShardWriterConfig tunable is recorded once any default is set. - assert "sync_indexed_write" in defaults assert "enable_memtable" in defaults diff --git a/python/python/tests/test_namespace_dir.py b/python/python/tests/test_namespace_dir.py index fa1bc93b422..72b89ffdcf2 100644 --- a/python/python/tests/test_namespace_dir.py +++ b/python/python/tests/test_namespace_dir.py @@ -27,6 +27,7 @@ from lance.namespace import LanceNamespace from lance_namespace import ( CountTableRowsRequest, + CountTableRowsResponse, CreateNamespaceRequest, CreateNamespaceResponse, CreateTableBranchRequest, @@ -67,10 +68,13 @@ ListTableVersionsRequest, ListTableVersionsResponse, NamespaceExistsRequest, + NamespaceExistsResponse, QueryTableRequest, + QueryTableResponse, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, + TableExistsResponse, connect, ) from lance_namespace.errors import ( @@ -107,7 +111,9 @@ def describe_namespace( ) -> DescribeNamespaceResponse: return self._inner.describe_namespace(request) - def namespace_exists(self, request: NamespaceExistsRequest) -> None: + def namespace_exists( + self, request: NamespaceExistsRequest + ) -> NamespaceExistsResponse: return self._inner.namespace_exists(request) def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse: @@ -127,7 +133,7 @@ def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse: def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse: return self._inner.describe_table(request) - def table_exists(self, request: TableExistsRequest) -> None: + def table_exists(self, request: TableExistsRequest) -> TableExistsResponse: return self._inner.table_exists(request) def drop_table(self, request: DropTableRequest) -> DropTableResponse: @@ -184,7 +190,9 @@ def list_table_indices( ) -> ListTableIndicesResponse: return self._inner.list_table_indices(request) - def count_table_rows(self, request: CountTableRowsRequest) -> int: + def count_table_rows( + self, request: CountTableRowsRequest + ) -> CountTableRowsResponse: return self._inner.count_table_rows(request) def insert_into_table( @@ -192,7 +200,7 @@ def insert_into_table( ) -> InsertIntoTableResponse: return self._inner.insert_into_table(request, request_data) - def query_table(self, request) -> bytes: + def query_table(self, request) -> QueryTableResponse: # Accept both QueryTableRequest and dict, like DirectoryNamespace does if hasattr(request, "model_dump"): request = request.model_dump() @@ -1416,7 +1424,7 @@ def test_count_table_rows(self, temp_ns_client): # Count rows count_req = CountTableRowsRequest(id=["workspace", "test_table"]) - count = temp_ns_client.count_table_rows(count_req) + count = temp_ns_client.count_table_rows(count_req).count assert count == 3 def test_count_table_rows_with_filter(self, temp_ns_client): @@ -1434,7 +1442,7 @@ def test_count_table_rows_with_filter(self, temp_ns_client): count_req = CountTableRowsRequest( id=["workspace", "test_table"], predicate="age > 28" ) - count = temp_ns_client.count_table_rows(count_req) + count = temp_ns_client.count_table_rows(count_req).count assert count == 2 # Alice (30) and Charlie (35) def test_insert_into_table(self, temp_ns_client): @@ -1464,7 +1472,7 @@ def test_insert_into_table(self, temp_ns_client): # Verify row count increased count_req = CountTableRowsRequest(id=["workspace", "test_table"]) - count = temp_ns_client.count_table_rows(count_req) + count = temp_ns_client.count_table_rows(count_req).count assert count == 5 def test_query_table(self, temp_ns_client): @@ -1480,7 +1488,7 @@ def test_query_table(self, temp_ns_client): # Query table with empty vector (for non-vector queries) query_req = QueryTableRequest(id=["workspace", "test_table"], k=10, vector={}) - result_bytes = temp_ns_client.query_table(query_req) + result_bytes = temp_ns_client.query_table(query_req).data assert result_bytes is not None assert len(result_bytes) > 0 @@ -1506,7 +1514,7 @@ def test_query_table_with_filter(self, temp_ns_client): query_req = QueryTableRequest( id=["workspace", "test_table"], filter="age >= 30", k=10, vector={} ) - result_bytes = temp_ns_client.query_table(query_req) + result_bytes = temp_ns_client.query_table(query_req).data reader = pa.ipc.open_file(pa.BufferReader(result_bytes)) result_table = reader.read_all() assert result_table.num_rows == 2 # Alice and Charlie diff --git a/python/python/tests/test_namespace_integration.py b/python/python/tests/test_namespace_integration.py index fc08370d247..ea2c8f08c4d 100644 --- a/python/python/tests/test_namespace_integration.py +++ b/python/python/tests/test_namespace_integration.py @@ -56,9 +56,11 @@ ListTableVersionsRequest, ListTableVersionsResponse, NamespaceExistsRequest, + NamespaceExistsResponse, RegisterTableRequest, RegisterTableResponse, TableExistsRequest, + TableExistsResponse, ) @@ -86,7 +88,9 @@ def describe_namespace( ) -> DescribeNamespaceResponse: return self._inner.describe_namespace(request) - def namespace_exists(self, request: NamespaceExistsRequest) -> None: + def namespace_exists( + self, request: NamespaceExistsRequest + ) -> NamespaceExistsResponse: return self._inner.namespace_exists(request) def drop_namespace(self, request: DropNamespaceRequest) -> DropNamespaceResponse: @@ -106,7 +110,7 @@ def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse: def describe_table(self, request: DescribeTableRequest) -> DescribeTableResponse: return self._inner.describe_table(request) - def table_exists(self, request: TableExistsRequest) -> None: + def table_exists(self, request: TableExistsRequest) -> TableExistsResponse: return self._inner.table_exists(request) def drop_table(self, request: DropTableRequest) -> DropTableResponse: diff --git a/python/python/tests/test_optimize.py b/python/python/tests/test_optimize.py index e35093dc370..661945880bf 100644 --- a/python/python/tests/test_optimize.py +++ b/python/python/tests/test_optimize.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +import json import pickle import random import re @@ -27,6 +28,9 @@ def test_dataset_optimize(tmp_path: Path): target_rows_per_fragment=1000, materialize_deletions=False, num_threads=1, + # Loose source budgets: all fragments still compact in one run. + max_source_rows=100_000, + max_source_bytes=1024 * 1024 * 1024, ) assert metrics.fragments_removed == 10 @@ -37,6 +41,77 @@ def test_dataset_optimize(tmp_path: Path): assert dataset.version == 3 +def test_dataset_optimize_excluded_fragment_ids(tmp_path: Path): + dataset = lance.write_dataset( + pa.table({"a": range(800)}), + tmp_path / "dataset", + max_rows_per_file=200, + ) + fragments = dataset.get_fragments() + + metrics = dataset.optimize.compact_files( + target_rows_per_fragment=400, + excluded_fragment_ids=[1, 1, 999], + num_threads=1, + ) + + assert metrics.fragments_removed == 2 + remaining_fragment_ids = { + fragment.fragment_id for fragment in dataset.get_fragments() + } + assert fragments[1].fragment_id in remaining_fragment_ids + + +def test_compact_files_source_budgets(tmp_path: Path): + base_dir = tmp_path / "dataset" + data = pa.table({"a": range(1000), "b": range(1000)}) + dataset = lance.write_dataset(data, base_dir, max_rows_per_file=100) + assert len(dataset.get_fragments()) == 10 + + # A row budget of 250 admits the first two 100-row fragments only, so the + # run is incremental instead of compacting all 10 at once. + metrics = dataset.optimize.compact_files( + target_rows_per_fragment=200, + num_threads=1, + max_source_rows=250, + ) + assert metrics.fragments_removed == 2 + assert metrics.fragments_added == 1 + + # The budgets are hard upper bounds: a budget smaller than a single task + # produces an empty plan and compaction is a no-op. + version_before = dataset.version + metrics = dataset.optimize.compact_files( + target_rows_per_fragment=200, + num_threads=1, + max_source_bytes=1, + ) + assert metrics.fragments_removed == 0 + assert dataset.version == version_before + + with pytest.raises(OSError, match="must be greater than 0"): + dataset.optimize.compact_files(max_source_rows=0) + + +def test_compact_files_max_source_fragments(tmp_path: Path): + rows_per_fragment = 256 * 1024 + dataset = lance.write_dataset( + pa.table({"a": pa.nulls(10 * rows_per_fragment)}), + tmp_path / "dataset", + max_rows_per_file=rows_per_fragment, + ) + assert len(dataset.get_fragments()) == 10 + + metrics = dataset.optimize.compact_files( + max_source_fragments=4, + num_threads=1, + ) + + assert metrics.fragments_removed == 4 + assert metrics.fragments_added == 1 + assert len(dataset.get_fragments()) == 7 + + def test_blob_compaction(tmp_path: Path): base_dir = tmp_path / "blob_dataset" blob_field = pa.field( @@ -69,6 +144,121 @@ def test_blob_compaction(tmp_path: Path): assert contents == blobs +def test_blob_compaction_with_nested_json_sibling(tmp_path: Path): + dataset_uri = tmp_path / "nested_blob_json" + info_fields = [lance.blob_field("blob"), pa.field("meta", pa.json_())] + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("info", pa.struct(info_fields)), + ] + ) + for index, row_id in enumerate([1, 2]): + info = pa.StructArray.from_arrays( + [ + lance.blob_array([f"blob-{row_id}".encode()]), + pa.array([json.dumps({"row": row_id})], type=pa.json_()), + ], + fields=info_fields, + ) + lance.write_dataset( + pa.Table.from_arrays([pa.array([row_id]), info], schema=schema), + dataset_uri, + mode="create" if index == 0 else "append", + data_storage_version="2.2", + ) + + dataset = lance.dataset(dataset_uri) + dataset.optimize.compact_files(num_threads=1) + + assert len(dataset.get_fragments()) == 1 + assert [data for _, data in dataset.read_blobs("info.blob", indices=[0, 1])] == [ + b"blob-1", + b"blob-2", + ] + assert [ + json.loads(value) + for value in dataset.to_table(columns=["info.meta"])["info.meta"].to_pylist() + ] == [{"row": 1}, {"row": 2}] + + +@pytest.mark.parametrize("storage_version", ["2.0", "2.1", "2.2"]) +def test_blob_compaction_preserves_null_empty_and_read_parity( + tmp_path: Path, storage_version: str +): + base_dir = tmp_path / f"blob_dataset_{storage_version}" + if storage_version == "2.2": + blob_field = lance.blob_field("blob") + else: + blob_field = pa.field( + "blob", + pa.large_binary(), + metadata={"lance-encoding:blob": "true"}, + ) + schema = pa.schema([pa.field("id", pa.int64()), blob_field]) + + def make_blob_array(values): + if storage_version == "2.2": + return lance.blob_array(values) + return pa.array(values, type=pa.large_binary()) + + for index, (ids, values) in enumerate( + [ + ([1, 2], [b"P1", None]), + ([3, 4, 5], [b"P3", None, b""]), + ] + ): + lance.write_dataset( + pa.Table.from_arrays( + [ + pa.array(ids, type=pa.int64()), + make_blob_array(values), + ], + schema=schema, + ), + base_dir, + mode="create" if index == 0 else "append", + data_storage_version=storage_version, + ) + + dataset = lance.dataset(base_dir) + dataset.delete("id = 2") + dataset.optimize.compact_files(num_threads=1) + + expected = {1: b"P1", 3: b"P3", 4: None, 5: b""} + binary_table = dataset.to_table(columns=["id", "blob"], blob_handling="all_binary") + scan_values = dict( + zip( + binary_table.column("id").to_pylist(), + binary_table.column("blob").to_pylist(), + ) + ) + + ids = dataset.to_table(columns=["id"]).column("id").to_pylist() + blob_files = dataset.take_blobs("blob", indices=range(len(ids))) + take_values = { + row_id: None if blob_file is None else blob_file.readall() + for row_id, blob_file in zip(ids, blob_files) + } + + assert take_values == expected + assert scan_values == take_values + + descriptor_table = dataset.to_table(columns=["id", "blob"]) + descriptions = dict( + zip( + descriptor_table.column("id").to_pylist(), + descriptor_table.column("blob").to_pylist(), + ) + ) + if storage_version == "2.0": + assert descriptions[4] == {"position": 1, "size": 0} + else: + assert descriptions[4] is None + assert descriptions[5] is not None + assert descriptions[5]["size"] == 0 + + def test_optimize_max_bytes(tmp_path: Path): base_dir = tmp_path / "dataset" arr = pa.array(range(4 * 1024 * 1024)) @@ -414,6 +604,31 @@ def test_dataset_distributed_optimize(tmp_path: Path): assert plan.tasks[0].fragments == [frag.metadata for frag in fragments[0:2]] assert plan.tasks[1].fragments == [frag.metadata for frag in fragments[2:4]] assert repr(plan) == "CompactionPlan(read_version=1, tasks=<2 compaction tasks>)" + + excluded_plan = Compaction.plan( + dataset, + options=dict( + target_rows_per_fragment=400, + excluded_fragment_ids=[1, 1, 999], + num_threads=1, + ), + ) + assert excluded_plan.num_tasks() == 1 + assert excluded_plan.tasks[0].fragments == [ + frag.metadata for frag in fragments[2:4] + ] + assert pickle.loads(pickle.dumps(excluded_plan)) == excluded_plan + + none_plan = Compaction.plan( + dataset, + options=dict( + target_rows_per_fragment=400, + excluded_fragment_ids=None, + num_threads=1, + ), + ) + assert none_plan == plan + # Plan can be pickled assert pickle.loads(pickle.dumps(plan)) == plan @@ -492,6 +707,20 @@ def test_optimize_indices_second_call_is_noop(tmp_path: Path): must not write any new files to the dataset directory.""" base_dir = tmp_path / "dataset" + def assert_optimize_is_noop(dataset: lance.LanceDataset): + version_before = dataset.version + files_before = { + path.relative_to(base_dir) for path in base_dir.rglob("*") if path.is_file() + } + + dataset.optimize.optimize_indices() + + files_after = { + path.relative_to(base_dir) for path in base_dir.rglob("*") if path.is_file() + } + assert dataset.version == version_before + assert files_after == files_before + n = 1024 rng = np.random.default_rng(0) vectors = rng.standard_normal((n, 8)).astype(np.float32) @@ -546,16 +775,20 @@ def test_optimize_indices_second_call_is_noop(tmp_path: Path): # First optimize: should pull the new fragment into each index. dataset.optimize.optimize_indices() + assert_optimize_is_noop(dataset) - files_before = {p.relative_to(base_dir) for p in base_dir.rglob("*") if p.is_file()} + # Consolidate the vector delta so every fragment has the same physical index + # coverage; compaction intentionally avoids mixing fragments that do not. + dataset.optimize.optimize_indices(num_indices_to_merge=10) - # Second optimize: nothing has changed, so this must be a no-op on disk. + # Compaction invalidates the old fragment coverage, so one optimize is needed + # to rebuild each index. Further calls must return to the same steady state. + compaction = dataset.optimize.compact_files( + target_rows_per_fragment=2 * (n + extra_rows), num_threads=1 + ) + assert compaction.fragments_removed > 0 dataset.optimize.optimize_indices() - - files_after = {p.relative_to(base_dir) for p in base_dir.rglob("*") if p.is_file()} - - new_files = files_after - files_before - assert not new_files, f"second optimize_indices created new files: {new_files}" + assert_optimize_is_noop(dataset) def test_compaction_generates_rewrite_transaction(tmp_path: Path): @@ -591,6 +824,10 @@ def test_remap_row_addrs(tmp_path: Path): before = ds.scanner(columns=["id"], with_row_address=True).to_table() old = dict(zip(before["id"].to_pylist(), before["_rowaddr"].to_pylist())) + # A deferred-remap compaction records a fragment-reuse index only when it + # rewrites data an index covers, so index a column first. + ds.create_scalar_index("id", "BTREE") + ds.optimize.compact_files( target_rows_per_fragment=1_000, defer_index_remap=True, num_threads=1 ) diff --git a/python/python/tests/test_otel.py b/python/python/tests/test_otel.py new file mode 100644 index 00000000000..a96e0125f56 --- /dev/null +++ b/python/python/tests/test_otel.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import lance +import pyarrow as pa +import pytest + +# The metrics recorder is process-global and installed once, so the whole +# bridge is exercised in a single test to avoid cross-test global-state coupling. + + +def _metrics_by_name(reader): + data = reader.get_metrics_data() + result = {} + for resource_metrics in data.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + result[metric.name] = metric + return result + + +def test_instrument_lance_metrics_exports_object_store_metrics(tmp_path): + pytest.importorskip("opentelemetry.sdk.metrics") + from lance.otel import instrument_lance_metrics + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + assert instrument_lance_metrics(provider) + + # The catalog is populated once the recorder is installed. + from lance.lance import lance_metrics_catalog + + catalog = {desc.name: desc for desc in lance_metrics_catalog()} + assert "lance_object_store_requests_total" in catalog + assert catalog["lance_object_store_request_duration_seconds"].kind == "histogram" + # Gauges and the retryable counter are described too, so they surface in the + # export even when a plain write doesn't happen to emit them. + assert catalog["lance_object_store_retryable_responses_total"].kind == "counter" + assert catalog["lance_object_store_in_flight_requests"].kind == "gauge" + + # Generate object store activity on the local filesystem (scheme "file"). + table = pa.table({"id": pa.array(range(256))}) + dataset = lance.write_dataset(table, str(tmp_path / "ds.lance")) + assert dataset.to_table().num_rows == 256 + + metrics = _metrics_by_name(reader) + + requests = metrics["lance_object_store_requests_total"] + points = list(requests.data.data_points) + assert points, "expected at least one request data point" + # The `base` label carries the store scheme ("file") by default. + assert all("base" in p.attributes and "operation" in p.attributes for p in points) + assert sum(p.value for p in points) > 0 + + # Histograms are decomposed into bucket / count / sum observable counters. + bucket = metrics["lance_object_store_request_duration_seconds_bucket"] + bucket_points = list(bucket.data.data_points) + assert bucket_points + assert all("le" in p.attributes for p in bucket_points) + # The implicit +Inf bucket must be present and is the cumulative maximum. + assert any(p.attributes["le"] == "+Inf" for p in bucket_points) + + count = metrics["lance_object_store_request_duration_seconds_count"] + assert sum(p.value for p in count.data.data_points) > 0 + + # The `_sum` instrument must also be wired and report positive latency. + duration_sum = metrics["lance_object_store_request_duration_seconds_sum"] + assert sum(p.value for p in duration_sum.data.data_points) > 0 + + +def test_snapshot_empty_before_install_is_safe(): + # snapshot is callable regardless of installation state and never raises. + from lance.lance import snapshot_lance_metrics + + assert isinstance(snapshot_lance_metrics(), list) + + +def test_instrument_warns_when_recorder_unavailable(monkeypatch): + # A foreign `metrics` recorder already installed -> register returns False; + # instrument_lance_metrics must warn and return False without instrumenting. + pytest.importorskip("opentelemetry.sdk.metrics") + import lance.otel as otel + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import InMemoryMetricReader + + monkeypatch.setattr(otel, "register_lance_metrics_recorder", lambda: False) + + reader = InMemoryMetricReader() + provider = MeterProvider(metric_readers=[reader]) + with pytest.warns(UserWarning, match="recorder"): + assert otel.instrument_lance_metrics(provider) is False diff --git a/python/python/tests/test_pydantic.py b/python/python/tests/test_pydantic.py new file mode 100644 index 00000000000..3db60885f6b --- /dev/null +++ b/python/python/tests/test_pydantic.py @@ -0,0 +1,177 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +from enum import Enum +from typing import List, Optional + +import pyarrow as pa +import pytest + +BaseModel = pytest.importorskip("pydantic").BaseModel +Field = pytest.importorskip("pydantic").Field + +from lance.pydantic import ( # noqa: E402 + MultiVector, + Vector, + is_nullable, + pydantic_to_schema, +) + + +def test_scalar_and_optional_and_list_unchanged(): + class Simple(BaseModel): + name: str + score: float + tag: Optional[str] = None + values: List[int] + + schema = pydantic_to_schema(Simple) + assert schema == pa.schema( + [ + pa.field("name", pa.utf8(), False), + pa.field("score", pa.float64(), False), + pa.field("tag", pa.utf8(), True), + pa.field("values", pa.list_(pa.int64()), False), + ] + ) + + +def test_nested_model_becomes_struct(): + class Address(BaseModel): + city: str + zip_code: str + + class Person(BaseModel): + name: str + address: Address + + schema = pydantic_to_schema(Person) + address_type = pa.struct( + [ + pa.field("city", pa.utf8(), False), + pa.field("zip_code", pa.utf8(), False), + ] + ) + assert schema == pa.schema( + [ + pa.field("name", pa.utf8(), False), + pa.field("address", address_type, False), + ] + ) + + +def test_string_enum_is_dictionary_encoded(): + class Color(str, Enum): + RED = "red" + GREEN = "green" + + class Item(BaseModel): + color: Color + + schema = pydantic_to_schema(Item) + assert schema.field("color").type == pa.dictionary(pa.int32(), pa.utf8()) + + +def test_int_enum_uses_native_type(): + class Priority(int, Enum): + LOW = 0 + HIGH = 1 + + class Task(BaseModel): + priority: Priority + + schema = pydantic_to_schema(Task) + assert schema.field("priority").type == pa.int64() + + +def test_vector_field(): + class Doc(BaseModel): + id: int + embedding: Vector(8) + + schema = pydantic_to_schema(Doc) + assert schema.field("embedding").type == pa.list_(pa.float32(), 8) + assert schema.field("embedding").nullable is True + + +def test_vector_field_not_nullable(): + class Doc(BaseModel): + id: int + embedding: Vector(8, nullable=False) + + schema = pydantic_to_schema(Doc) + assert schema.field("embedding").nullable is False + + +def test_multi_vector_field(): + class Doc(BaseModel): + id: int + embeddings: MultiVector(4) + + schema = pydantic_to_schema(Doc) + assert schema.field("embeddings").type == pa.list_(pa.list_(pa.float32(), 4)) + + +def test_tz_aware_datetime_field(): + from datetime import datetime + + class Event(BaseModel): + occurred_at: datetime = Field(json_schema_extra={"tz": "UTC"}) + + schema = pydantic_to_schema(Event) + assert schema.field("occurred_at").type == pa.timestamp("us", tz="UTC") + + +def test_naive_datetime_field(): + from datetime import datetime + + class Event(BaseModel): + occurred_at: datetime + + schema = pydantic_to_schema(Event) + assert schema.field("occurred_at").type == pa.timestamp("us", tz=None) + + +def test_defaulted_non_optional_field_is_not_nullable(): + """A plain default value (without Optional) should not make a field + nullable -- this matches lancedb's behavior, tightened from lance's + previous default-implies-nullable inference.""" + + class Counter(BaseModel): + count: int = 0 + + schema = pydantic_to_schema(Counter) + assert schema.field("count").nullable is False + + field_info = Counter.model_fields["count"] + assert is_nullable(field_info) is False + + +def test_optional_field_is_nullable(): + class Counter(BaseModel): + count: Optional[int] = None + + schema = pydantic_to_schema(Counter) + assert schema.field("count").nullable is True + + +def test_v1_style_field_nullable_vector_overrides_allow_none(): + """Simulates a Pydantic v1 ModelField (no `.annotation` attribute; type + is read off `.outer_type_`, nullability off `.allow_none`) to verify a + nullable Vector/MultiVector annotation overrides `allow_none=False` + under v1, matching the v2 behavior tested in test_vector_field above.""" + + class FakeV1Field: + def __init__(self, outer_type_, allow_none): + self.outer_type_ = outer_type_ + self.allow_none = allow_none + + nullable_vector = Vector(8, nullable=True) + assert is_nullable(FakeV1Field(nullable_vector, allow_none=False)) is True + + non_nullable_vector = Vector(8, nullable=False) + assert is_nullable(FakeV1Field(non_nullable_vector, allow_none=False)) is False + + # Plain (non-Vector) types still fall back to `allow_none`. + assert is_nullable(FakeV1Field(str, allow_none=True)) is True + assert is_nullable(FakeV1Field(str, allow_none=False)) is False diff --git a/python/python/tests/test_row_addr_prefilter.py b/python/python/tests/test_row_addr_prefilter.py new file mode 100644 index 00000000000..7d3381ebdce --- /dev/null +++ b/python/python/tests/test_row_addr_prefilter.py @@ -0,0 +1,312 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""End-to-end tests for the external row-address prefilter. + +``row_addr_allowlist`` / ``row_addr_blocklist`` restrict a scan to a set of row +addresses supplied by the caller, rather than to rows a filter expression +selects. The mask is applied before ranking, so a KNN or full-text search +computes top-k over the surviving rows instead of trimming the result +afterwards -- the two differ whenever k is smaller than the candidate set. + +Each test asserts against ``_rowid`` ground truth so a mask that is silently +dropped (which would return every row) fails rather than passing by accident. +""" + +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import lance +import numpy as np +import pyarrow as pa +import pytest +from lance.dataset import ScannerBuilder, serialize_row_addrs +from lance.file import LanceFileWriter + +if TYPE_CHECKING: + from pathlib import Path + +N = 256 +DIM = 8 + + +def _write(tmp_path: Path, with_index: bool = False) -> lance.LanceDataset: + rng = np.random.default_rng(1234) + vectors = rng.standard_normal((N, DIM)).astype(np.float32) + tbl = pa.table( + { + "id": pa.array(range(N), pa.int64()), + "vector": pa.FixedSizeListArray.from_arrays( + pa.array(vectors.reshape(-1), pa.float32()), DIM + ), + "text": pa.array([f"row {i} lorem ipsum" for i in range(N)]), + } + ) + ds = lance.write_dataset(tbl, str(tmp_path / "t.lance"), mode="overwrite") + if with_index: + # IVF_FLAT with nprobes == num_partitions is exact, so the masked result + # can be compared against brute force without recall slack. + ds.create_index("vector", index_type="IVF_FLAT", num_partitions=4, metric="l2") + return ds + + +def _rowids(ds: lance.LanceDataset) -> list[int]: + return ds.to_table(with_row_id=True)["_rowid"].to_pylist() + + +def test_serialize_row_addrs_round_trips_through_a_scan(tmp_path: Path) -> None: + ds = _write(tmp_path) + addrs = _rowids(ds) + want = addrs[3:9] + + got = ds.scanner( + with_row_id=True, row_addr_allowlist=serialize_row_addrs(want) + ).to_table() + assert got["_rowid"].to_pylist() == want + + +def test_allowlist_and_blocklist_combine(tmp_path: Path) -> None: + ds = _write(tmp_path) + addrs = _rowids(ds) + + allow, block = addrs[:10], addrs[5:15] + got = ds.scanner( + with_row_id=True, + row_addr_allowlist=serialize_row_addrs(allow), + row_addr_blocklist=serialize_row_addrs(block), + ).to_table() + assert got["_rowid"].to_pylist() == addrs[:5] + + # Block alone excludes and leaves everything else. + got = ds.scanner( + with_row_id=True, row_addr_blocklist=serialize_row_addrs(addrs[:5]) + ).to_table() + assert got["_rowid"].to_pylist() == addrs[5:] + + +def test_no_mask_reads_everything(tmp_path: Path) -> None: + # Guards the "no mask" vs "empty mask" distinction: omitting both must not + # be read as an allowlist of nothing. + ds = _write(tmp_path) + assert ds.scanner().to_table().num_rows == N + + +def test_empty_allowlist_selects_nothing(tmp_path: Path) -> None: + ds = _write(tmp_path) + got = ds.scanner(row_addr_allowlist=serialize_row_addrs([])).to_table() + assert got.num_rows == 0 + + +def test_mask_composes_with_a_filter(tmp_path: Path) -> None: + ds = _write(tmp_path) + addrs = _rowids(ds) + got = ds.scanner( + columns=["id"], + filter="id % 2 == 0", + row_addr_allowlist=serialize_row_addrs(addrs[:20]), + ).to_table() + assert got["id"].to_pylist() == [i for i in range(20) if i % 2 == 0] + + +def test_builder_setter_matches_the_kwarg(tmp_path: Path) -> None: + ds = _write(tmp_path) + blob = serialize_row_addrs(_rowids(ds)[2:7]) + from_kwarg = ds.scanner(with_row_id=True, row_addr_allowlist=blob).to_table() + from_builder = ( + ScannerBuilder(ds) + .with_row_id(True) + .row_addr_prefilter(allowlist=blob) + .to_scanner() + .to_table() + ) + assert from_kwarg["_rowid"].to_pylist() == from_builder["_rowid"].to_pylist() + + +@pytest.mark.parametrize("with_index", [False, True]) +def test_knn_topk_is_computed_over_masked_rows( + tmp_path: Path, with_index: bool +) -> None: + # The point of a prefilter: with k=5 and a 10-row mask, post-filtering a + # global top-5 would usually return fewer than 5 (often 0) rows. + ds = _write(tmp_path, with_index=with_index) + addrs = _rowids(ds) + allowed = addrs[100:110] + query = np.zeros(DIM, dtype=np.float32) + + got = ds.scanner( + nearest={"column": "vector", "q": query, "k": 5, "nprobes": 4}, + with_row_id=True, + row_addr_allowlist=serialize_row_addrs(allowed), + ).to_table() + + assert got.num_rows == 5 + assert set(got["_rowid"].to_pylist()) <= set(allowed) + + # Exactly the 5 nearest *within* the mask, not the global 5 intersected. + vectors = np.stack( + [np.asarray(v) for v in ds.to_table(columns=["vector"])["vector"].to_pylist()] + ) + by_addr = dict(zip(addrs, vectors)) + expect = sorted(allowed, key=lambda a: np.linalg.norm(by_addr[a] - query))[:5] + assert sorted(got["_rowid"].to_pylist()) == sorted(expect) + + +def test_knn_blocklist_excludes_the_nearest(tmp_path: Path) -> None: + ds = _write(tmp_path) + query = np.zeros(DIM, dtype=np.float32) + unmasked = ( + ds.scanner(nearest={"column": "vector", "q": query, "k": 3}, with_row_id=True) + .to_table()["_rowid"] + .to_pylist() + ) + + got = ds.scanner( + nearest={"column": "vector", "q": query, "k": 3}, + with_row_id=True, + row_addr_blocklist=serialize_row_addrs(unmasked[:1]), + ).to_table() + + assert got.num_rows == 3 # refilled, not truncated + assert unmasked[0] not in got["_rowid"].to_pylist() + + +def test_full_text_search_honors_the_mask(tmp_path: Path) -> None: + ds = _write(tmp_path) + ds.create_scalar_index("text", index_type="INVERTED") + addrs = _rowids(ds) + allowed = addrs[50:60] + + got = ds.scanner( + full_text_query="lorem", + with_row_id=True, + row_addr_allowlist=serialize_row_addrs(allowed), + limit=5, + ).to_table() + + assert got.num_rows == 5 + assert set(got["_rowid"].to_pylist()) <= set(allowed) + + +def test_rejects_a_malformed_mask(tmp_path: Path) -> None: + ds = _write(tmp_path) + with pytest.raises(Exception, match="(?i)row address mask|invalid"): + ds.scanner(row_addr_allowlist=b"not a treemap").to_table() + + +def _overlay( + ds, base_dir: Path, name: str, batch: pa.Table, fields: list[int], offsets +): + """Commit a data overlay covering `offsets` of fragment 0. + + An overlay committed after an index makes the indexed values stale, so the + planner replays those rows through a separate take. That replay is a second + row source, and it has to honor the caller's mask like every other one. + """ + path = base_dir / "data" / name + with LanceFileWriter(str(path)) as writer: + writer.write_batch(batch) + base_df = ds.get_fragments()[0].metadata.files[0] + data_file = lance.fragment.DataFile( + path=name, + fields=fields, + column_indices=list(range(len(fields))), + file_major_version=base_df.file_major_version, + file_minor_version=base_df.file_minor_version, + file_size_bytes=os.path.getsize(path), + ) + op = lance.LanceOperation.DataOverlay( + [ + lance.LanceOperation.DataOverlayGroup( + 0, [lance.LanceOperation.DataOverlayFile(data_file, offsets=offsets)] + ) + ] + ) + return lance.LanceDataset.commit(ds, op, read_version=ds.version) + + +def test_overlay_stale_replay_scan_respects_mask(tmp_path: Path) -> None: + base_dir = tmp_path / "ov_scan" + ds = lance.write_dataset( + pa.table( + { + "id": pa.array(range(10), pa.int32()), + "val": pa.array([i * 10 for i in range(10)], pa.int32()), + } + ), + base_dir, + ) + # Index first, then overlay: offset 1 now reads 999 while the index still + # says 10, so `val = 999` can only be answered by the stale replay. + ds.create_scalar_index("val", index_type="BTREE") + ds = _overlay( + ds, + base_dir, + "ov.lance", + pa.table({"val": pa.array([999], pa.int32())}), + fields=[1], + offsets=[1], + ) + + base = ds.scanner(filter="val = 999", with_row_id=True).to_table() + assert base.num_rows == 1, "fixture did not produce a stale replay" + stale_addr = base["_rowid"].to_pylist()[0] + + got = ds.scanner( + filter="val = 999", row_addr_allowlist=serialize_row_addrs([]) + ).to_table() + assert got.num_rows == 0, "stale replay must not return rows the mask excludes" + + got = ds.scanner( + filter="val = 999", + with_row_id=True, + row_addr_allowlist=serialize_row_addrs([stale_addr]), + ).to_table() + assert got["_rowid"].to_pylist() == [stale_addr] + + +def test_overlay_stale_replay_ann_respects_mask(tmp_path: Path) -> None: + base_dir = tmp_path / "ov_ann" + rng = np.random.default_rng(7) + vectors = rng.standard_normal((N, DIM)).astype(np.float32) + ds = lance.write_dataset( + pa.table( + { + "id": pa.array(range(N), pa.int64()), + "vector": pa.FixedSizeListArray.from_arrays( + pa.array(vectors.reshape(-1), pa.float32()), DIM + ), + } + ), + base_dir, + ) + ds.create_index("vector", index_type="IVF_FLAT", num_partitions=4, metric="l2") + + # Move two rows onto the query point after indexing. The ANN index still has + # their old vectors, so they can only surface through the stale replay. + query = np.zeros(DIM, dtype=np.float32) + moved = pa.FixedSizeListArray.from_arrays( + pa.array(np.zeros(2 * DIM, dtype=np.float32), pa.float32()), DIM + ) + ds = _overlay( + ds, + base_dir, + "ov_vec.lance", + pa.table({"vector": moved}), + fields=[1], + offsets=[3, 7], + ) + + base = ds.scanner( + nearest={"column": "vector", "q": query, "k": 5}, with_row_id=True + ).to_table() + assert base.num_rows > 0, "fixture did not produce ANN results" + + got = ds.scanner( + nearest={"column": "vector", "q": query, "k": 5}, + row_addr_allowlist=serialize_row_addrs([]), + ).to_table() + assert got.num_rows == 0, ( + "the ANN stale replay must not return rows the mask excludes" + ) diff --git a/python/python/tests/test_s3_ddb.py b/python/python/tests/test_s3_ddb.py index dc9744115e2..5289dd176fb 100644 --- a/python/python/tests/test_s3_ddb.py +++ b/python/python/tests/test_s3_ddb.py @@ -315,6 +315,29 @@ def test_file_writer_reader(s3_bucket: str): == global_buffer_text ) + # The writer reports the size of the object it just wrote, so callers do not + # need a second client to stat it. + s3 = get_boto3_client("s3", endpoint_url=CONFIG["aws_endpoint"]) + head = s3.head_object(Bucket=s3_bucket, Key="foo.lance") + assert writer.size_bytes == head["ContentLength"] + + +@pytest.mark.integration +def test_file_writer_size_bytes_multipart(s3_bucket: str): + # Large enough to exceed the single-PUT threshold, so the write goes through + # the multipart path where the reported size is filled in on completion. + storage_options = copy.deepcopy(CONFIG) + del storage_options["dynamodb_endpoint"] + table = pa.table({"a": pa.array(range(4 * 1024 * 1024), type=pa.int64())}) + file_path = f"s3://{s3_bucket}/multipart.lance" + with LanceFileWriter(str(file_path), storage_options=storage_options) as writer: + writer.write_batch(table) + + s3 = get_boto3_client("s3", endpoint_url=CONFIG["aws_endpoint"]) + head = s3.head_object(Bucket=s3_bucket, Key="multipart.lance") + assert head["ContentLength"] > 5 * 1024 * 1024 + assert writer.size_bytes == head["ContentLength"] + @pytest.mark.integration def test_file_session_upload_download(s3_bucket: str, tmp_path): diff --git a/python/python/tests/test_scalar_index.py b/python/python/tests/test_scalar_index.py index ae92abdd427..79762a9539f 100644 --- a/python/python/tests/test_scalar_index.py +++ b/python/python/tests/test_scalar_index.py @@ -7,9 +7,12 @@ import re import shutil import string +import subprocess +import sys import uuid import zipfile from datetime import date, datetime, timedelta +from decimal import Decimal from pathlib import Path import lance @@ -21,6 +24,8 @@ from lance.query import ( BooleanQuery, BoostQuery, + DocumentGranularity, + FullTextOperator, MatchQuery, MultiMatchQuery, Occur, @@ -129,19 +134,26 @@ def test_create_scalar_index_rejects_invalid_uuid(tmp_path): def btree_comparison_datasets(tmp_path): """Setup datasets for B-tree comparison tests""" num_fragments = 3 - rows_per_fragment = 10000 + rows_per_fragment = 100 total_rows = num_fragments * rows_per_fragment + fragment_path = tmp_path / "fragment" fragment_ds = generate_multi_fragment_dataset( - tmp_path / "fragment", + fragment_path, num_fragments=num_fragments, rows_per_fragment=rows_per_fragment, ) - complete_ds = generate_multi_fragment_dataset( - tmp_path / "complete", - num_fragments=num_fragments, - rows_per_fragment=rows_per_fragment, + complete_path = tmp_path / "complete" + shutil.copytree(fragment_path, complete_path) + complete_ds = lance.dataset(complete_path) + fragment_count = len(fragment_ds.get_fragments()) + complete_count = len(complete_ds.get_fragments()) + assert fragment_count == num_fragments, ( + f"Expected {num_fragments} segmented fragments, got {fragment_count}" + ) + assert complete_count == num_fragments, ( + f"Expected {num_fragments} complete-index fragments, got {complete_count}" ) fragment_ds_committed = _commit_segmented_btree_index( @@ -624,13 +636,13 @@ def make_fts_search(ds): plan = make_vec_search(ds).explain_plan() assert "ScalarIndexQuery" in plan assert "KNNVectorDistance" not in plan - assert "LanceRead" not in plan + assert "num_fragments" not in plan # no scan; the take prints as LanceRead assert make_vec_search(ds).to_table().num_rows == 6 plan = make_fts_search(ds).explain_plan() assert "ScalarIndexQuery" in plan assert "KNNVectorDistance" not in plan - assert "LanceRead" not in plan + assert "num_fragments" not in plan # no scan; the take prints as LanceRead assert make_fts_search(ds).to_table().num_rows == 6 # Add new data (including 6 more results) @@ -692,24 +704,87 @@ def test_indexed_vector_scan_postfilter( assert scanner.to_table().num_rows == 0 -def test_fixed_size_binary(tmp_path): - arr = pa.array([b"0123012301230123", b"2345234523452345"], pa.uuid()) +@pytest.mark.parametrize( + "index_type, data_type, values, filter_expr", + [ + pytest.param( + "BTREE", + pa.uuid(), + [b"0123012301230123", b"2345234523452345"], + ( + "value = arrow_cast(0x32333435323334353233343532333435, " + "'FixedSizeBinary(16)')" + ), + id="btree-fixed-size-binary", + ), + *[ + pytest.param( + index_type, + data_type, + values, + filter_expr, + id=f"{index_type.lower()}-{type_name}", + ) + for type_name, data_type, values, filter_expr in [ + ( + "large-string", + pa.large_string(), + ["alpha", "beta", "gamma"], + "value = 'beta'", + ), + ( + "binary", + pa.binary(), + [b"alpha", b"beta", b"gamma"], + "value = arrow_cast(0x62657461, 'Binary')", + ), + ( + "large-binary", + pa.large_binary(), + [b"alpha", b"beta", b"gamma"], + "value = arrow_cast(0x62657461, 'LargeBinary')", + ), + ( + "decimal128", + pa.decimal128(10, 2), + [Decimal("1.00"), Decimal("2.00"), Decimal("3.00")], + "value = arrow_cast(2.00, 'Decimal128(10, 2)')", + ), + ( + "decimal256", + pa.decimal256(76, 2), + [Decimal("1.00"), Decimal("2.00"), Decimal("3.00")], + "value = arrow_cast(2.00, 'Decimal256(76, 2)')", + ), + ( + "duration", + pa.duration("ms"), + [1, 2, 3], + "value = arrow_cast(2, 'Duration(Millisecond)')", + ), + ] + for index_type in ["BTREE", "BITMAP", "ZONEMAP"] + ], + ], +) +def test_scalar_index_types(tmp_path, index_type, data_type, values, filter_expr): + values = pa.array(values, type=data_type) + ds = lance.write_dataset(pa.table({"value": values}), tmp_path) - ds = lance.write_dataset(pa.table({"uuid": arr}), tmp_path) + ds.create_scalar_index("value", index_type) - ds.create_scalar_index("uuid", "BTREE") + scanner = ds.scanner(filter=filter_expr) + assert "ScalarIndexQuery" in scanner.explain_plan() + assert scanner.to_table()["value"].to_pylist() == values.slice(1, 1).to_pylist() - query = ( - "uuid = arrow_cast(0x32333435323334353233343532333435, 'FixedSizeBinary(16)')" - ) - assert ( - "ScalarIndexQuery: query=[uuid = 32333435323334353233...]@uuid_idx" - in ds.scanner(filter=query).explain_plan() + fragment_id = ds.get_fragments()[0].fragment_id + segment = ds.create_index_uncommitted( + column="value", + index_type=index_type, + name=f"{index_type.lower()}_segment_idx", + fragment_ids=[fragment_id], ) - - table = ds.scanner(filter=query).to_table() - assert table.num_rows == 1 - assert table.column("uuid").to_pylist() == arr.slice(1, 1).to_pylist() + assert segment.fragment_ids == {fragment_id} def test_index_take_batch_size(tmp_path): @@ -806,6 +881,341 @@ def test_full_text_search(dataset, with_position, base_tokenizer): ) +@pytest.mark.parametrize("block_size", [128, 256]) +def test_code_analyzer_does_not_split_identifiers_by_default(tmp_path, block_size): + table = pa.table({"code": ["GetUserName", "GetUserEmail", "user"]}) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + block_size=block_size, + ) + assert ds.describe_indices()[0].segments[0].index_version == 3 + + results = ds.to_table( + columns=["code"], + full_text_query=MatchQuery("user", "code"), + ) + assert results["code"].to_pylist() == ["user"] + + stats = ds.stats.index_stats("code_idx")["indices"][0] + params = stats["params"] + assert "analyzer" not in params + assert params["base_tokenizer"] == "code" + assert params["split_identifiers"] is False + + +def test_code_analyzer_full_text_search_with_identifier_splitting(tmp_path): + table = pa.table( + { + "code": [ + "getUserName", + "set_user_name", + "user-name", + "username", + "other", + ] + } + ) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + split_identifiers=True, + ) + + results = ds.to_table( + columns=["code"], + full_text_query=MatchQuery("user", "code"), + ) + assert set(results["code"].to_pylist()) == { + "getUserName", + "set_user_name", + "user-name", + } + + stats = ds.stats.index_stats("code_idx")["indices"][0] + params = stats["params"] + assert "analyzer" not in params + assert params["base_tokenizer"] == "code" + assert params["split_identifiers"] is True + assert params["split_on_numerics"] is True + assert params["preserve_original"] is True + assert params["stem"] is False + assert params["remove_stop_words"] is False + + +def test_code_analyzer_operator_search_matches_rust_turbofish(tmp_path): + table = pa.table( + { + "path": ["turbofish.rs", "comparison.rs"], + "code": ["value.parse::()", "value.parse()"], + } + ) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + index_operators=True, + ) + + results = ds.to_table( + columns=["path"], + full_text_query=MatchQuery("::", "code", operator=FullTextOperator.OR), + ) + assert results["path"].to_pylist() == ["turbofish.rs"] + + +def test_code_analyzer_exact_identifier_survives_grouped_top_k(tmp_path): + table = pa.table( + { + "path": ["split_0.rs", "split_1.rs", "split_2.rs", "exact.rs"], + "code": ["get user name", "get user name", "get user name", "getUserName"], + } + ) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + split_identifiers=True, + ) + + results = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery( + "getUserName", "code", operator=FullTextOperator.AND + ), + limit=1, + ).to_table() + assert results["path"].to_pylist() == ["exact.rs"] + + +def test_code_analyzer_flags_require_code_analyzer(tmp_path): + table = pa.table({"text": ["getUserName"]}) + ds = lance.write_dataset(table, tmp_path) + + with pytest.raises(ValueError, match="code analyzer flags require analyzer='code'"): + ds.create_scalar_index( + "text", + index_type="INVERTED", + split_identifiers=True, + ) + + +def test_code_analyzer_requires_fts_v3(tmp_path): + table = pa.table({"code": ["getUserName"]}) + ds = lance.write_dataset(table, tmp_path) + + with pytest.raises(ValueError, match="requires FTS format_version=3"): + ds.create_scalar_index( + "code", + index_type="INVERTED", + analyzer="code", + format_version=2, + ) + + +def test_code_analyzer_complex_code_constructs(tmp_path): + table = pa.table( + { + "path": [ + "edge/trait.rs", + "edge/impl.rs", + "edge/fn_pointer.rs", + "edge/unit_result.rs", + "edge/hrtb.rs", + "edge/associated.rs", + "edge/operators.rs", + ], + "code": [ + """ +pub trait EdgeAsyncRepository<'a, T: Send + Sync> +where + T: TryFrom<&'a str, Error = EdgeParseError>, +{ + type Output<'b>: Iterator> + where + Self: 'b; + + async fn fetch_by_key( + &'a self, + key: [u8; N], + ) -> Result, EdgeRepoError>; +} +""", + """ +impl<'a, T, S> EdgeAsyncRepository<'a, T> for EdgeStore +where + T: TryFrom<&'a str, Error = EdgeParseError> + Clone + Send + Sync, + S: EdgeBackend + ?Sized, +{ + type Output<'b> = std::vec::IntoIter> where Self: 'b; + + async fn fetch_by_key( + &'a self, + key: [u8; N], + ) -> Result, EdgeRepoError> { + self.backend.fetch::(key).await + } +} +""", + """ +pub fn build_edge_handler( + factory: F, +) -> impl Fn() -> Result, EdgeError> +where + F: FnOnce() -> Result + Send + 'static, + T: Default + Send + Sync + 'static, +{ + move || factory().map(EdgeHandler::new) +} +""", + """ +pub fn edge_unit_result_callback() -> Result<()> { + Ok(()) +} +""", + """ +pub fn edge_higher_ranked<'a, T>( + visitor: impl for<'b> Fn(&'b T) -> Result<&'b str, EdgeVisitError>, + value: &'a T, +) -> Result<&'a str, EdgeVisitError> { + visitor(value) +} +""", + """ +pub fn edge_collect_stream(items: I) -> Result, E::Error> +where + I: IntoIterator, + E: EdgeExtract, +{ + items.into_iter().map(E::extract).collect() +} +""", + """ +pub fn edge_operator_arrow() -> Result { + let variant = EdgeModule::EdgeVariant; + if variant != EdgeModule::Default && EdgeMask::enabled() { + return Ok(EdgeArrow::new(variant)); + } + Err(EdgeError::empty()) +} +""", + ], + } + ) + table = table.append_column("code_ops", table["code"]) + ds = lance.write_dataset(table, tmp_path) + ds.create_scalar_index("code", index_type="INVERTED", analyzer="code") + ds.create_scalar_index( + "code_ops", + index_type="INVERTED", + analyzer="code", + index_operators=True, + ) + + ds.insert( + pa.table( + { + "path": ["edge/flat_unindexed.rs"], + "code": [ + """ +pub async fn edge_flat_generic_return() -> Result +where + T: TryFrom + Send, + E: Into, +{ + T::try_from(String::new()).map_err(Into::into) +} +""" + ], + "code_ops": [ + """ +pub async fn edge_flat_operator() -> Result { + EdgeFlat::try_new() -> Result +} +""" + ], + } + ) + ) + ds = lance.dataset(tmp_path) + + def assert_search(column, query, expected_path, operator=FullTextOperator.AND): + result = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery(query, column, operator=operator), + limit=50, + ).to_table() + assert expected_path in result["path"].to_pylist() + + assert_search( + "code", + "EdgeAsyncRepository fetch_by_key TryFrom EdgeRepoError", + "edge/trait.rs", + ) + assert_search( + "code", + "EdgeStore fetch_by_key const usize where Result", + "edge/impl.rs", + ) + assert_search( + "code", + "build_edge_handler FnOnce Result EdgeHandler", + "edge/fn_pointer.rs", + ) + assert_search( + "code", + "edge_unit_result_callback fn () -> Result", + "edge/unit_result.rs", + ) + assert_search( + "code", + "edge_higher_ranked for Fn EdgeVisitError Result", + "edge/hrtb.rs", + ) + assert_search( + "code", + "edge_collect_stream IntoIterator Item Error Result", + "edge/associated.rs", + ) + assert_search( + "code", + "edge_flat_generic_return TryFrom EdgeFlatError Result", + "edge/flat_unindexed.rs", + ) + assert_search( + "code_ops", + "edge_operator_arrow -> Result", + "edge/operators.rs", + ) + assert_search( + "code_ops", + "EdgeModule :: EdgeVariant !=", + "edge/operators.rs", + ) + assert_search( + "code_ops", + "edge_flat_operator -> Result EdgeFlatError", + "edge/flat_unindexed.rs", + ) + + default_operator_results = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery("->", "code", operator=FullTextOperator.OR), + ).to_table() + operator_results = ds.scanner( + columns=["path", "_score"], + full_text_query=MatchQuery("->", "code_ops", operator=FullTextOperator.OR), + ).to_table() + assert default_operator_results.num_rows == 0 + assert operator_results.num_rows > 0 + + def test_unindexed_full_text_search_on_empty_index(tmp_path): # Create fts index on empty table. schema = pa.schema({"text": pa.string()}) @@ -949,6 +1359,39 @@ def test_create_scalar_index_fts_alias(dataset): assert any(idx.index_type == "Inverted" for idx in dataset.describe_indices()) +def test_create_scalar_index_fts_block_size(dataset): + dataset.create_scalar_index( + "doc", index_type="INVERTED", with_position=False, block_size=256 + ) + indices = dataset.describe_indices() + doc_index = next(index for index in indices if index.name == "doc_idx") + assert doc_index.segments[0].index_version == 3 + + row = dataset.take(indices=[0], columns=["doc"]) + query = row.column(0)[0].as_py().split(" ")[0] + results = dataset.scanner(columns=["doc"], full_text_query=query).to_table() + assert results.num_rows > 0 + + with pytest.raises(ValueError, match="block_size"): + dataset.create_scalar_index( + "doc", index_type="INVERTED", name="doc_invalid_129", block_size=129 + ) + + with pytest.raises(ValueError, match="block_size"): + dataset.create_scalar_index( + "doc", index_type="INVERTED", name="doc_invalid_512", block_size=512 + ) + + with pytest.raises(ValueError, match="block_size=256"): + dataset.create_scalar_index( + "doc", + index_type="INVERTED", + name="doc_invalid_v2_256", + block_size=256, + format_version=2, + ) + + def test_multi_index_create(tmp_path): dataset = lance.write_dataset( pa.table({"ints": range(1024)}), tmp_path, max_rows_per_file=100 @@ -1100,14 +1543,23 @@ def test_indexed_filter_with_fts_index(tmp_path): def test_fts_ngram_tokenizer(tmp_path): - data = pa.table({"text": ["hello world", "lance database", "lance is cool"]}) - ds = lance.write_dataset(data, tmp_path) + data = pa.table( + {"text": ["hello world", "lance database", "lance is cool", "theatre", "other"]} + ) + ds = lance.write_dataset(data, tmp_path, max_rows_per_file=2) ds.create_scalar_index("text", index_type="INVERTED", base_tokenizer="ngram") results = ds.to_table(full_text_query="lan") assert results.num_rows == 2 assert set(results["text"].to_pylist()) == {"lance database", "lance is cool"} + results = ds.to_table(full_text_query="the") + assert set(results["text"].to_pylist()) == {"theatre", "other"} + + params = ds.stats.index_stats("text_idx")["indices"][0]["params"] + assert params["stem"] is False + assert params["remove_stop_words"] is False + results = ds.to_table(full_text_query="nce") # spellchecker:disable-line assert results.num_rows == 2 assert set(results["text"].to_pylist()) == {"lance database", "lance is cool"} @@ -1318,6 +1770,136 @@ def test_fts_on_list(tmp_path): assert results.num_rows == 6 +@pytest.mark.parametrize( + "list_type", + [ + pa.list_(pa.string()), + pa.list_(pa.large_string()), + pa.large_list(pa.string()), + pa.large_list(pa.large_string()), + ], +) +def test_fts_on_list_elements(tmp_path, list_type): + data = pa.table( + { + "id": pa.array([0, 1, 2]), + "tags": pa.array( + [ + ["alpha beta", "gamma alpha", None, "", "delta"], + ["beta", "gamma"], + None, + ], + type=list_type, + ), + } + ) + ds = lance.write_dataset(data, tmp_path) + + def hits(table): + return sorted(zip(table["id"].to_pylist(), table["_doc_index"].to_pylist())) + + list_element = DocumentGranularity.LIST_ELEMENT + query = MatchQuery("alpha", "tags", document_granularity=list_element) + flat = ds.to_table(full_text_query=query) + assert hits(flat) == [(0, [0]), (0, [1])] + assert pa.types.is_list(flat.schema.field("_doc_index").type) + assert flat.schema.field("_doc_index").type.value_type == pa.uint32() + assert hits( + ds.to_table( + full_text_query=MatchQuery( + "delta", "tags", document_granularity=list_element + ) + ) + ) == [(0, [4])] + + ds.create_scalar_index( + "tags", + "INVERTED", + with_position=True, + document_granularity=list_element, + ) + ds.create_scalar_index( + "tags", + IndexConfig(index_type="inverted", parameters={"with_position": True}), + document_granularity=list_element, + ) + inferred = ds.to_table(full_text_query=MatchQuery("alpha", "tags")) + assert hits(inferred) == [(0, [0]), (0, [1])] + with pytest.raises(ValueError, match=r"requested Row.*ListElement"): + ds.to_table( + full_text_query=MatchQuery( + "alpha", "tags", document_granularity=DocumentGranularity.ROW + ) + ) + + ds.create_scalar_index("tags", "INVERTED", with_position=True) + index_names = {index.name for index in ds.describe_indices()} + assert {"tags_idx", "tags_list_element_idx"}.issubset(index_names) + row_auto = ds.to_table(full_text_query="alpha") + assert row_auto["id"].to_pylist() == [0] + assert "_doc_index" not in row_auto.column_names + with pytest.raises(ValueError, match=r"ambiguous.*document_granularity"): + ds.to_table(full_text_query=MatchQuery("alpha", "tags")) + indexed = ds.to_table(full_text_query=query) + assert hits(indexed) == [(0, [0]), (0, [1])] + assert pa.types.is_list(indexed.schema.field("_doc_index").type) + assert indexed.schema.field("_doc_index").type.value_type == pa.uint32() + filtered = ds.to_table(full_text_query=query, filter="id = 0", prefilter=True) + assert hits(filtered) == [(0, [0]), (0, [1])] + assert hits( + ds.to_table( + full_text_query=MatchQuery( + "delta", "tags", document_granularity=list_element + ) + ) + ) == [(0, [4])] + + phrase = ds.to_table( + full_text_query=PhraseQuery( + "beta gamma", "tags", document_granularity=list_element + ) + ) + assert phrase.num_rows == 0 + assert hits( + ds.to_table( + full_text_query=PhraseQuery( + "alpha beta", "tags", document_granularity=list_element + ) + ) + ) == [(0, [0])] + row_phrase = ds.to_table( + full_text_query=PhraseQuery( + "beta gamma", + "tags", + document_granularity=DocumentGranularity.ROW, + ) + ) + assert sorted(row_phrase["id"].to_pylist()) == [0, 1] + assert "_doc_index" not in row_phrase.column_names + + ds.insert( + pa.table( + { + "id": pa.array([3]), + "tags": pa.array([["alpha", "alpha again"]], type=list_type), + } + ) + ) + assert hits(ds.to_table(full_text_query=query)) == [ + (0, [0]), + (0, [1]), + (3, [0]), + (3, [1]), + ] + + with pytest.raises(RuntimeError, match=r"tags\[\*\]"): + ds.create_scalar_index( + "tags[*]", + "INVERTED", + document_granularity=list_element, + ) + + def test_fts_fuzzy_query(tmp_path): data = pa.table( { @@ -2165,6 +2747,87 @@ def scan_stats_callback(stats: lance.ScanStatistics): assert small_bytes_read < large_bytes_read +@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"]) +def test_address_domain_index_with_stable_row_ids(tmp_path: Path, index_type): + """Regression test for issue #7434. + + Address-domain scalar indices (zonemap, bloom filter) report matches as + physical row addresses. On a stable-row-id dataset a row's stable id differs + from its physical address in every fragment except fragment 0, so the index + result must be translated back to the row-id domain before it prefilters the + scan. Without that translation the index silently drops matching rows in + fragments other than fragment 0 (often returning an empty result). + """ + + # A single value "a" is placed in non-adjacent fragments (0, 2, 4) so its + # matching rows span multiple fragments and diverge from fragment 0. + def block(v, n): + return [v] * n + + vals = ( + block("a", 5_000) + + block("b", 5_000) + + block("a", 5_000) + + block("c", 5_000) + + block("a", 5_000) + ) + tbl = pa.table({"category": pa.array(vals, pa.string()), "id": range(len(vals))}) + ds = lance.write_dataset( + tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True + ) + assert ds.has_stable_row_ids + assert len(ds.get_fragments()) == 5 + + true_count = ds.scanner( + filter="category = 'a'", use_scalar_index=False + ).count_rows() + assert true_count == 15_000 + + ds.create_scalar_index("category", index_type=index_type, replace=True) + + # The index must be consulted and must return the same rows as a full scan. + assert "ScalarIndexQuery" in ds.scanner(filter="category = 'a'").explain_plan() + indexed = ds.to_table(filter="category = 'a'", columns=["id"]) + assert indexed.num_rows == true_count + assert sorted(indexed["id"].to_pylist()) == sorted( + ds.to_table(filter="category = 'a'", columns=["id"], use_scalar_index=False)[ + "id" + ].to_pylist() + ) + + +def test_zonemap_with_stable_row_ids_after_compaction(tmp_path: Path): + """Zonemap results stay correct after a compaction relocates rows under + stable row ids (physical address != stable id for the surviving rows).""" + ds = lance.write_dataset( + pa.table({"x": range(0, 5_000)}), + tmp_path, + max_rows_per_file=5_000, + enable_stable_row_ids=True, + ) + ds = lance.write_dataset( + pa.table({"x": range(5_000, 10_000)}), + tmp_path, + mode="append", + max_rows_per_file=5_000, + enable_stable_row_ids=True, + ) + # Delete part of the first fragment, then compact so surviving rows keep + # their small stable ids but move to a freshly numbered fragment. + ds.delete("x >= 1000 AND x < 2000") + ds.optimize.compact_files(target_rows_per_fragment=100_000) + ds = lance.dataset(tmp_path) + assert len(ds.get_fragments()) == 1 + + ds.create_scalar_index("x", index_type="ZONEMAP") + + filter_expr = "x >= 6000 AND x <= 6500" + assert "ScalarIndexQuery" in ds.scanner(filter=filter_expr).explain_plan() + expected = ds.to_table(filter=filter_expr, use_scalar_index=False)["x"].to_pylist() + actual = ds.to_table(filter=filter_expr)["x"].to_pylist() + assert sorted(actual) == sorted(expected) == list(range(6000, 6501)) + + def test_zonemap_deletion_handling(tmp_path: Path): """Test zonemap deletion handling""" data = pa.table( @@ -2194,6 +2857,33 @@ def test_zonemap_deletion_handling(tmp_path: Path): assert ids == [0, 2, 4, 6, 8] +@pytest.mark.parametrize("index_type", ["ZONEMAP", "BLOOMFILTER"]) +def test_address_domain_index_not_query_with_stable_row_ids(tmp_path: Path, index_type): + """Regression test: != queries return correct results on stable-row-id datasets. + + Address-domain indices (zonemap, bloom filter) search returns physical row + addresses. Translation to row IDs happens at the Query leaf before the NOT + node is evaluated, so the NOT operates on a correctly translated AllowList. + Without the address-to-row-id translation the AllowList contains wrong IDs, + and the subsequent NOT excludes the wrong rows. + """ + vals = list(range(5_000)) + list(range(5_000, 10_000)) + tbl = pa.table({"x": vals}) + ds = lance.write_dataset( + tbl, tmp_path, max_rows_per_file=5_000, enable_stable_row_ids=True + ) + assert ds.has_stable_row_ids + + ds.create_scalar_index("x", index_type=index_type, replace=True) + + # Without address translation the NOT excludes the wrong rows, producing an + # incorrect result set rather than crashing. + expected = ds.to_table(filter="x != 42", use_scalar_index=False)["x"].to_pylist() + actual = ds.to_table(filter="x != 42")["x"].to_pylist() + assert sorted(actual) == sorted(expected) + assert len(actual) == 9_999 + + def test_zonemap_index_remapping(tmp_path: Path): """Test zonemap index remapping after compaction and optimization""" # Create a dataset with 5 fragments by writing data in chunks @@ -2222,7 +2912,7 @@ def test_zonemap_index_remapping(tmp_path: Path): # Run compaction to merge fragments compaction = dataset.optimize.compact_files(target_rows_per_fragment=2000) assert compaction.fragments_removed == 5 - assert len(dataset.get_fragments()) == 3 + assert len(dataset.get_fragments()) == 2 # Check if the zone map index is no longer being used scanner = dataset.scanner(filter="values > 2500", prefilter=True) @@ -2250,6 +2940,54 @@ def test_zonemap_index_remapping(tmp_path: Path): assert result.num_rows == 501 # 1000..1500 inclusive +def test_zonemap_fsl_column(tmp_path: Path): + """Zone map can be created on a FixedSizeList column and accelerates IS NULL.""" + dim = 8 + n = 1000 + rng = np.random.default_rng(42) + vectors = rng.standard_normal((n, dim)).astype(np.float32) + vec_type = pa.list_(pa.float32(), dim) + # Every 10th row is null + vec_list = [None if i % 10 == 0 else v.tolist() for i, v in enumerate(vectors)] + tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))}) + ds = lance.write_dataset(tbl, tmp_path) + ds.create_scalar_index("vec", index_type="ZONEMAP") + + scanner = ds.scanner(filter="vec IS NULL", prefilter=True) + plan = scanner.explain_plan() + assert "ScalarIndexQuery" in plan + result = scanner.to_table() + assert result.num_rows == 100 # every 10th row is null + + +def test_vector_and_zonemap_on_fsl_column(tmp_path: Path): + """Vector index and zone map can coexist on the same FSL column.""" + dim = 16 + n = 2000 + rng = np.random.default_rng(0) + vectors = rng.standard_normal((n, dim)).astype(np.float32) + vec_type = pa.list_(pa.float32(), dim) + # Every 20th row is null + vec_list = [None if i % 20 == 0 else v.tolist() for i, v in enumerate(vectors)] + tbl = pa.table({"vec": pa.array(vec_list, type=vec_type), "id": pa.array(range(n))}) + ds = lance.write_dataset(tbl, tmp_path) + + ds.create_index("vec", index_type="IVF_PQ", num_partitions=4, num_sub_vectors=2) + ds.create_scalar_index("vec", index_type="ZONEMAP") + + # Vector search still works + query = vectors[5] + result = ds.scanner(nearest={"column": "vec", "q": query, "k": 10}).to_table() + assert result.num_rows == 10 + + # IS NULL is zone-map-accelerated + scanner = ds.scanner(filter="vec IS NULL", prefilter=True) + plan = scanner.explain_plan() + assert "ScalarIndexQuery" in plan + null_result = scanner.to_table() + assert null_result.num_rows == 100 # every 20th row is null + + def test_bloomfilter_index(tmp_path: Path): """Test create bloomfilter index""" tbl = pa.Table.from_arrays([pa.array([i for i in range(10000)])], names=["values"]) @@ -2332,6 +3070,36 @@ def test_json_index(): ) +def test_json_index_non_exact_floats(): + # A JSON-path btree index is trained on the value extracted from the JSON + # column, not on the raw column itself, so the index build must sort by + # the extracted value rather than assuming the raw column's order matches + # it. Without that, range/equality queries silently miss rows whenever + # the extracted floats are not exactly representable in float64 (#7485). + vals = ['{"latitude": 10.5}', '{"latitude": 40.1}', '{"latitude": -3.2}'] + tbl = pa.table({"data": pa.array(vals, pa.json_())}) + ds = lance.write_dataset(tbl, "memory://test") + ds.create_scalar_index( + "data", + IndexConfig( + index_type="json", + parameters={"target_index_type": "btree", "path": "latitude"}, + ), + ) + + for filter in [ + "json_get_float(data, 'latitude') > 0", + "json_get_float(data, 'latitude') >= 10.5", + "json_get_float(data, 'latitude') = 40.1", + "json_get_float(data, 'latitude') = 10.5", + "json_get_float(data, 'latitude') < 100", + ]: + assert "ScalarIndexQuery" in ds.scanner(filter=filter).explain_plan() + assert ds.to_table(filter=filter) == ds.to_table( + filter=filter, use_scalar_index=False + ), filter + + def test_null_handling(): tbl = pa.table( { @@ -2793,6 +3561,15 @@ def scan_stats_callback(stats: lance.ScanStatistics): cache_entries_after_query = ds._ds.index_cache_entry_count() assert cache_entries_after_query == cache_entries_after_prewarm + segment_uuid = ds.describe_indices()[0].segments[0].uuid + ds = lance.dataset(phrase_path) + ds.prewarm_index("fts_idx", with_position=True, index_segments=[segment_uuid]) + cache_entries_after_prewarm = ds._ds.index_cache_entry_count() + results = ds.to_table(full_text_query=PhraseQuery("word word", "fts")) + assert results.num_rows == test_table_size + cache_entries_after_query = ds._ds.index_cache_entry_count() + assert cache_entries_after_query == cache_entries_after_prewarm + with pytest.raises( TypeError, match="takes 2 positional arguments", @@ -2828,19 +3605,179 @@ def scan_stats_callback(stats: lance.ScanStatistics): assert scan_stats.parts_loaded == 0 -def test_fts_backward_v0_27_0(tmp_path: Path): - path = ( - Path(__file__).parent.parent.parent.parent - / "test_data" - / "0.27.0" - / "legacy_fts_index" - ) - shutil.copytree(path, tmp_path, dirs_exist_ok=True) - ds = lance.dataset(tmp_path) +def test_btree_index_cache_hit_miss_stats(tmp_path: Path): + """Cold scan reports index cache misses; warm scan reports hits. - # we can read the old index - results = ds.to_table( - full_text_query=BoostQuery( + ScanStatistics.index_cache_{hits,misses} are populated at page-level cache + boundaries. On a freshly-loaded dataset the BTree page fetch must be a + miss; a second scan against the same in-memory Dataset re-uses the cached + page and therefore reports a hit with zero misses. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table({"val": list(range(1000))}) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("val", index_type="BTREE") + + # Reopen so the session cache starts cold. A single-key point lookup on + # a small dataset resolves to exactly one BTree page, so cold/warm counts + # are deterministic 1/0 and 0/1. + ds = lance.dataset(tmp_path) + ds.scanner(filter="val = 42", scan_stats_callback=scan_stats_callback).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses == 1 + assert scan_stats.index_cache_hits == 0 + + # Same Dataset, warm cache — no new page loads, only hits. + ds.scanner(filter="val = 42", scan_stats_callback=scan_stats_callback).to_table() + assert scan_stats.index_cache_hits == 1 + assert scan_stats.index_cache_misses == 0 + + +def test_bitmap_index_cache_hit_miss_stats(tmp_path: Path): + """Bitmap Range/IN queries report cold misses and warm hits; a value + that is not in the index never reaches the loader and must not count. + + Guards against the ``BitmapIndex::search`` regressions where the + ``Range`` / ``IsIn`` branches used to drop the ``MetricsCollector`` (so + every lookup was silently ``0/0``), and where an equality on a value + absent from ``index_map`` recorded a spurious miss before short-circuiting + to the empty result. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table({"color": ["red", "green", "blue", "yellow"] * 25}) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("color", index_type="BITMAP") + + # Reopen so the session cache starts cold. + ds = lance.dataset(tmp_path) + ds.scanner( + filter="color IN ('red', 'blue')", scan_stats_callback=scan_stats_callback + ).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses == 2 + assert scan_stats.index_cache_hits == 0 + + ds.scanner( + filter="color IN ('red', 'blue')", scan_stats_callback=scan_stats_callback + ).to_table() + assert scan_stats.index_cache_hits == 2 + assert scan_stats.index_cache_misses == 0 + + # A value that is not in the index short-circuits before the loader and + # must not touch either counter. + ds.scanner( + filter="color = 'purple'", scan_stats_callback=scan_stats_callback + ).to_table() + assert scan_stats.index_cache_hits == 0 + assert scan_stats.index_cache_misses == 0 + + +def test_phrase_query_cache_hit_miss_stats(tmp_path: Path): + """Phrase-query fallback populates ``PositionKey``; that boundary must + show up in per-query cache statistics. + + Guards against ``read_positions`` silently using the non-metric + ``get_or_insert_with_key`` API — before this fix, a warm phrase query + would report zero hits for the phrase-position cache slot even though + the loader was skipped. + + The cold path can already record a few hits (``bm25_stats_for_terms`` + populates ``PostingMetadataKey``, which is then re-read on the + posting-list path as a cross-boundary hit), so the cold assertion is + ``misses > hits`` rather than a strict zero. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table( + {"text": ["quick brown fox jumps over lazy dog" for _ in range(50)]} + ) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("text", index_type="INVERTED", with_position=True) + + ds = lance.dataset(tmp_path) + ds.scanner( + scan_stats_callback=scan_stats_callback, + full_text_query='"quick brown"', + ).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses > 0 + assert scan_stats.index_cache_misses > scan_stats.index_cache_hits + + ds.scanner( + scan_stats_callback=scan_stats_callback, + full_text_query='"quick brown"', + ).to_table() + assert scan_stats.index_cache_hits > 0 + assert scan_stats.index_cache_misses == 0 + + +def test_fts_index_cache_hit_miss_stats(tmp_path: Path): + """Cold FTS scan reports misses; warm FTS scan reports hits. + + Guards the wrapper-forwarding fix in ``FtsIndexMetrics``: previously the + two new cache-hit/miss trait methods had default no-op implementations + that swallowed FTS-side events, so cache activity was reported as ``0/0`` + even for hot inverted-index scans. + + The cold path can still record a few hits when the same cache key is + read across boundaries in one query (e.g. ``bm25_stats_for_terms`` + populates ``PostingMetadataKey`` before ``posting_list`` re-reads it), + so the cold assertion is ``misses > hits`` rather than a strict zero. + """ + scan_stats = None + + def scan_stats_callback(stats: lance.ScanStatistics): + nonlocal scan_stats + scan_stats = stats + + test_table = pa.table({"fts": ["word" for _ in range(100)]}) + ds = lance.write_dataset(test_table, tmp_path) + ds.create_scalar_index("fts", index_type="INVERTED") + + # Reopen so the session cache starts cold. + ds = lance.dataset(tmp_path) + ds.scanner( + scan_stats_callback=scan_stats_callback, full_text_query="word" + ).to_table() + assert scan_stats is not None + assert scan_stats.index_cache_misses > 0 + assert scan_stats.index_cache_misses > scan_stats.index_cache_hits + + # Same Dataset, warm cache — posting-list / metadata reads must now hit. + ds.scanner( + scan_stats_callback=scan_stats_callback, full_text_query="word" + ).to_table() + assert scan_stats.index_cache_hits > 0 + assert scan_stats.index_cache_misses == 0 + + +def test_fts_backward_v0_27_0(tmp_path: Path): + path = ( + Path(__file__).parent.parent.parent.parent + / "test_data" + / "0.27.0" + / "legacy_fts_index" + ) + shutil.copytree(path, tmp_path, dirs_exist_ok=True) + ds = lance.dataset(tmp_path) + + # we can read the old index + results = ds.to_table( + full_text_query=BoostQuery( MatchQuery("puppy", "text"), MatchQuery("happy", "text"), negative_boost=0.5, @@ -2853,6 +3790,14 @@ def test_fts_backward_v0_27_0(tmp_path: Path): "frodo was a happy puppy", } + # Requiring both disjoint terms advances "happy" past its final document while + # "tail" remains live. Legacy WAND must terminate without reading the exhausted + # posting. + results = ds.to_table( + full_text_query=MatchQuery("happy tail", "text", operator=FullTextOperator.AND) + ) + assert results.num_rows == 0 + data = pa.table( { "text": [ @@ -4151,6 +5096,126 @@ def test_bitmap_uncommitted_segments_can_be_committed_from_python(tmp_path): ) +def test_ngram_segment_merge_and_commit_from_python(tmp_path): + ds = lance.write_dataset( + pa.table( + { + "text": [ + "alpha needle", + None, + "beta needle", + "gamma stack", + "delta needle", + "", + ] + } + ), + tmp_path, + max_rows_per_file=2, + ) + index_name = "text_ngram_segments" + fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] + staged_segments = [ + ds.create_index_uncommitted( + column="text", + index_type="NGRAM", + name=index_name, + fragment_ids=[fragment_id], + ) + for fragment_id in fragment_ids + ] + source_version = staged_segments[0].dataset_version + + for segment, fragment_id in zip(staged_segments, fragment_ids): + assert segment.fragment_ids == {fragment_id} + assert any(file.path == "ngram_postings.lance" for file in segment.files) + + merged_segment = ds.merge_existing_index_segments(staged_segments) + assert merged_segment.dataset_version == source_version + assert merged_segment.fragment_ids == set(fragment_ids) + assert any(file.path == "ngram_postings.lance" for file in merged_segment.files) + + ds.insert(pa.table({"text": ["new stack"]})) + assert ds.version > source_version + ds = ds.commit_existing_index_segments(index_name, "text", [merged_segment]) + descriptions = {index.name: index for index in ds.describe_indices()} + assert descriptions[index_name].index_type == "NGram" + assert len(descriptions[index_name].segments) == 1 + assert ( + descriptions[index_name].segments[0].dataset_version_at_last_update + == source_version + ) + assert ds.count_rows("contains(text, 'needle')") == 3 + assert ds.count_rows("text IS NULL") == 1 + + +@pytest.mark.parametrize( + "label_type", + [pa.list_(pa.string()), pa.large_list(pa.string())], + ids=["list", "large_list"], +) +def test_label_list_segment_index(tmp_path, label_type): + rows_per_fragment = 8 + ds = lance.write_dataset( + pa.table( + { + "id": pa.array(range(rows_per_fragment * 4), type=pa.int32()), + "labels": pa.array( + [ + ["distributed"] if row_id % 2 == 0 else ["other"] + for row_id in range(rows_per_fragment * 4) + ], + type=label_type, + ), + } + ), + tmp_path, + max_rows_per_file=rows_per_fragment, + ) + + fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] + assert len(fragment_ids) == 4 + + with pytest.raises(ValueError, match="create_index_uncommitted"): + ds.create_scalar_index( + column="labels", + index_type="LABEL_LIST", + fragment_ids=[fragment_ids[0]], + ) + + index_name = "labels_segment_idx" + segments = [ + ds.create_index_uncommitted( + column="labels", + index_type="LABEL_LIST", + name=index_name, + fragment_ids=[fragment_id], + ) + for fragment_id in fragment_ids + ] + + merged_segment = ds.merge_existing_index_segments(segments) + ds = ds.commit_existing_index_segments(index_name, "labels", [merged_segment]) + + filter_expr = "array_has_any(labels, ['distributed'])" + without_index = ds.scanner( + filter=filter_expr, + columns=["id", "labels"], + use_scalar_index=False, + ).to_table() + with_index = ds.scanner( + filter=filter_expr, + columns=["id", "labels"], + use_scalar_index=True, + ).to_table() + + assert with_index.equals(without_index) + assert ( + "ScalarIndexQuery" + in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan() + ) + + def test_zonemap_fragment_ids_parameter_validation(tmp_path): ds = generate_multi_fragment_dataset( tmp_path, num_fragments=2, rows_per_fragment=100 @@ -4224,6 +5289,54 @@ def test_zonemap_segment_merge_and_commit_from_python(tmp_path): ) +def test_bloomfilter_segment_merge_and_commit_from_python(tmp_path): + ds = generate_multi_fragment_dataset( + tmp_path, num_fragments=3, rows_per_fragment=100 + ) + + index_name = "id_bloomfilter_segments" + fragment_ids = [fragment.fragment_id for fragment in ds.get_fragments()] + staged_segments = [ + ds.create_index_uncommitted( + column="id", + index_type="BLOOMFILTER", + name=index_name, + fragment_ids=[fragment_id], + ) + for fragment_id in fragment_ids + ] + + for segment, fragment_id in zip(staged_segments, fragment_ids): + assert segment.fragment_ids == {fragment_id} + assert any(file.path == "bloomfilter.lance" for file in segment.files) + + merged_segment = ds.merge_existing_index_segments(staged_segments) + assert merged_segment.fragment_ids == set(fragment_ids) + assert any(file.path == "bloomfilter.lance" for file in merged_segment.files) + + ds = ds.commit_existing_index_segments(index_name, "id", [merged_segment]) + descriptions = {index.name: index for index in ds.describe_indices()} + assert descriptions[index_name].index_type == "BloomFilter" + assert len(descriptions[index_name].segments) == 1 + + filter_expr = "id = 117" + without_index = ds.scanner( + filter=filter_expr, + columns=["id", "text"], + use_scalar_index=False, + ).to_table() + with_index = ds.scanner( + filter=filter_expr, + columns=["id", "text"], + use_scalar_index=True, + ).to_table() + assert with_index.to_pydict() == without_index.to_pydict() + assert ( + "ScalarIndexQuery" + in ds.scanner(filter=filter_expr, use_scalar_index=True).explain_plan() + ) + + def test_merge_index_metadata_btree_soft_break(tmp_path): ds = generate_multi_fragment_dataset( tmp_path, num_fragments=2, rows_per_fragment=100 @@ -4271,74 +5384,98 @@ def test_btree_fragment_ids_parameter_validation(tmp_path): assert segment.fragment_ids == {valid_fragment_id} -@pytest.mark.parametrize( - "test_name,filter_expr", - [ - # Test 1: Boundary values at fragment edges - ("First value", "id = 0"), - ("Fragment 0 last value", "id = 9999"), - ("Fragment 1 first value", "id = 10000"), - ("Fragment 1 last value", "id = 19999"), - ("Fragment 2 first value", "id = 20000"), - ("Last value", "id = 29999"), - # Test 2: Values in the middle of fragments - ("Fragment 0 middle", "id = 5000"), - ("Fragment 1 middle", "id = 15000"), - ("Fragment 2 middle", "id = 25000"), - # Test 3: Range queries within single fragments - ("Range within fragment 0", "id >= 10 AND id < 20"), - ("Range within fragment 1", "id >= 10010 AND id < 10020"), - ("Range within fragment 2", "id >= 20010 AND id < 20020"), - # Test 4: Range queries spanning multiple fragments - ("Cross fragment 0-1", "id >= 9995 AND id < 10005"), - ("Cross fragment 1-2", "id >= 19995 AND id < 20005"), - ("Cross all fragments", "id >= 5000 AND id < 25000"), - # Test 5: Edge cases - ("Non-existent small value", "id = -1"), - ("Non-existent large value", "id = 30100"), - ("Large range", "id >= 0 AND id < 30000"), - # Test 6: Comparison operators - ("Less than boundary", "id < 10000"), - ("Greater than boundary", "id > 19999"), - ("Less than or equal", "id <= 10050"), - ("Greater than or equal", "id >= 10050"), - ], -) -def test_btree_query_comparison_parametrized( - btree_comparison_datasets, test_name, filter_expr -): +def test_btree_query_comparison(btree_comparison_datasets): """ - Parametrized B-tree index query comparison test. + B-tree index query comparison test covering representative query shapes. Compares segmented fragment-built BTree results with a complete BTree index. """ fragment_ds = btree_comparison_datasets["fragment_ds"] complete_ds = btree_comparison_datasets["complete_ds"] + rows_per_fragment = btree_comparison_datasets["rows_per_fragment"] + total_rows = btree_comparison_datasets["total_rows"] + fragment_starts = [idx * rows_per_fragment for idx in range(3)] + fragment_ends = [start + rows_per_fragment - 1 for start in fragment_starts] + fragment_middles = [start + rows_per_fragment // 2 for start in fragment_starts] + range_start_offset = rows_per_fragment // 10 + range_end_offset = range_start_offset * 2 + cross_fragment_margin = rows_per_fragment // 20 + + cases = [ + # Boundary values at fragment edges + ("First value", f"id = {fragment_starts[0]}"), + ("Fragment 0 last value", f"id = {fragment_ends[0]}"), + ("Fragment 1 first value", f"id = {fragment_starts[1]}"), + ("Fragment 1 last value", f"id = {fragment_ends[1]}"), + ("Fragment 2 first value", f"id = {fragment_starts[2]}"), + ("Last value", f"id = {total_rows - 1}"), + # Values in the middle of fragments + ("Fragment 0 middle", f"id = {fragment_middles[0]}"), + ("Fragment 1 middle", f"id = {fragment_middles[1]}"), + ("Fragment 2 middle", f"id = {fragment_middles[2]}"), + # Range queries within single fragments + ( + "Range within fragment 0", + f"id >= {fragment_starts[0] + range_start_offset} " + f"AND id < {fragment_starts[0] + range_end_offset}", + ), + ( + "Range within fragment 1", + f"id >= {fragment_starts[1] + range_start_offset} " + f"AND id < {fragment_starts[1] + range_end_offset}", + ), + ( + "Range within fragment 2", + f"id >= {fragment_starts[2] + range_start_offset} " + f"AND id < {fragment_starts[2] + range_end_offset}", + ), + # Range queries spanning multiple fragments + ( + "Cross fragment 0-1", + f"id >= {fragment_ends[0] - cross_fragment_margin + 1} " + f"AND id < {fragment_starts[1] + cross_fragment_margin}", + ), + ( + "Cross fragment 1-2", + f"id >= {fragment_ends[1] - cross_fragment_margin + 1} " + f"AND id < {fragment_starts[2] + cross_fragment_margin}", + ), + ( + "Cross all fragments", + f"id >= {fragment_middles[0]} AND id < {fragment_middles[2]}", + ), + # Missing values and the full indexed range + ("Non-existent small value", f"id = {fragment_starts[0] - 1}"), + ( + "Non-existent large value", + f"id = {total_rows + rows_per_fragment}", + ), + ( + "Large range", + f"id >= {fragment_starts[0]} AND id < {total_rows}", + ), + # Comparison operators + ("Less than boundary", f"id < {fragment_starts[1]}"), + ("Greater than boundary", f"id > {fragment_ends[1]}"), + ("Less than or equal", f"id <= {fragment_middles[1]}"), + ("Greater than or equal", f"id >= {fragment_middles[1]}"), + ] - fragment_results = fragment_ds.scanner( - filter=filter_expr, - columns=["id", "text"], - ).to_table() - - complete_results = complete_ds.scanner( - filter=filter_expr, - columns=["id", "text"], - ).to_table() - - assert fragment_results.num_rows == complete_results.num_rows, ( - f"Test '{test_name}' failed: Fragment index " - f"returned {fragment_results.num_rows} rows, " - f"but complete index returned {complete_results.num_rows}" - f" rows for filter: {filter_expr}" - ) - - if fragment_results.num_rows > 0: - fragment_ids = sorted(fragment_results.column("id").to_pylist()) - complete_ids = sorted(complete_results.column("id").to_pylist()) - - assert fragment_ids == complete_ids, ( - f"Test '{test_name}' failed: Fragment index " - f"and complete index returned different results for filter: {filter_expr}" + for test_name, filter_expr in cases: + fragment_results = fragment_ds.scanner( + filter=filter_expr, + columns=["id", "text"], + ).to_table() + complete_results = complete_ds.scanner( + filter=filter_expr, + columns=["id", "text"], + ).to_table() + + fragment_results = fragment_results.sort_by([("id", "ascending")]) + complete_results = complete_results.sort_by([("id", "ascending")]) + assert fragment_results.equals(complete_results), ( + f"Test '{test_name}' failed: segmented and complete BTree indexes returned " + f"different results for filter: {filter_expr}" ) @@ -4719,6 +5856,77 @@ def test_nested_field_fts_index(tmp_path): assert results.num_rows == 50 +def test_multiple_nested_field_fts_indices_e2e(tmp_path): + """Test FTS queries against multiple indexed nested string fields.""" + + def make_table(ids, text_values, summary_values): + return pa.table( + { + "id": ids, + "data": pa.StructArray.from_arrays( + [ + pa.array(text_values, type=pa.string()), + pa.array(summary_values, type=pa.string()), + ], + names=["text", "summary"], + ), + } + ) + + def result_ids(query): + return sorted(ds.to_table(full_text_query=query)["id"].to_pylist()) + + ds = lance.write_dataset( + make_table( + [0, 1, 2, 3], + [ + "lance nested alpha", + "plain text", + None, + "phrase target here", + ], + [ + "metadata only", + "database nested beta", + "lance beta", + "other", + ], + ), + tmp_path, + ) + + ds.create_scalar_index("data.text", index_type="INVERTED", with_position=True) + ds.create_scalar_index("data.summary", index_type="INVERTED", with_position=False) + + indexed_fields = { + tuple(index.field_names) + for index in ds.describe_indices() + if index.index_type == "Inverted" + } + assert indexed_fields == {("data.text",), ("data.summary",)} + + assert result_ids(MatchQuery("alpha", "data.text")) == [0] + assert result_ids(MatchQuery("beta", "data.summary")) == [1, 2] + assert result_ids("lance") == [0, 2] + assert result_ids(MultiMatchQuery("nested", ["data.text", "data.summary"])) == [ + 0, + 1, + ] + assert result_ids(PhraseQuery("phrase target", "data.text")) == [3] + + ds = lance.write_dataset( + make_table( + [4, 5], + ["fresh lance append", "plain append"], + ["other", "fresh beta append"], + ), + tmp_path, + mode="append", + ) + + assert result_ids("fresh") == [4, 5] + + def test_nested_field_bitmap_index(tmp_path): """Test BITMAP index creation and querying on nested fields""" # Create dataset with nested categorical field @@ -4853,7 +6061,7 @@ def test_json_inverted_match_query(tmp_path): @pytest.mark.parametrize( ("format_version", "expected_format_version"), - [(1, 1), (2, 2), ("v1", 1), ("v2", 2)], + [(1, 1), (2, 2), (3, 3), ("v1", 1), ("v2", 2), ("v3", 3)], ) def test_describe_indices(tmp_path, format_version, expected_format_version): data = pa.table( @@ -4903,7 +6111,6 @@ def test_describe_indices(tmp_path, format_version, expected_format_version): assert details["lower_case"] assert details["stem"] assert details["remove_stop_words"] - assert details["custom_stop_words"] is None assert details["ascii_folding"] assert details["min_ngram_length"] == 3 assert details["max_ngram_length"] == 3 @@ -4984,15 +6191,92 @@ def test_describe_indices(tmp_path, format_version, expected_format_version): assert index.num_rows_indexed == 50 -def test_create_inverted_index_defaults_to_v2_and_ignores_env(tmp_path, monkeypatch): - monkeypatch.setenv("LANCE_FTS_FORMAT_VERSION", "1") - data = pa.table({"text": ["document about lance database"]}) - ds = lance.write_dataset(data, tmp_path) +def _run_fts_format_creation_probe( + tmp_path, env_value, creation_options=None, expected_format_version=None +): + script = """ +import json +import sys - ds.create_scalar_index("text", index_type="INVERTED") +import lance +import pyarrow as pa - indices = ds.describe_indices() - assert indices[0].segments[0].index_version == 2 +dataset = lance.write_dataset( + pa.table({"text": ["document about lance database"]}), sys.argv[1] +) +dataset.create_scalar_index( + "text", index_type="INVERTED", **json.loads(sys.argv[2]) +) +expected_format_version = json.loads(sys.argv[3]) +if expected_format_version is not None: + actual_format_version = dataset.describe_indices()[0].segments[0].index_version + assert actual_format_version == expected_format_version +""" + env = os.environ.copy() + if env_value is None: + env.pop("LANCE_FTS_FORMAT_VERSION", None) + else: + env["LANCE_FTS_FORMAT_VERSION"] = env_value + return subprocess.run( + [ + sys.executable, + "-c", + script, + str(tmp_path), + json.dumps(creation_options or {}), + json.dumps(expected_format_version), + ], + capture_output=True, + env=env, + text=True, + ) + + +@pytest.mark.parametrize( + ("env_value", "creation_options", "expected_format_version"), + [ + ("1", {}, 1), + ("2", {}, 2), + ("3", {}, 3), + ("3", {"block_size": 256}, 3), + ], +) +def test_create_inverted_index_uses_env_format_version( + tmp_path, env_value, creation_options, expected_format_version +): + result = _run_fts_format_creation_probe( + tmp_path, + env_value, + creation_options, + expected_format_version, + ) + + assert result.returncode == 0, result.stderr + + +def test_create_inverted_index_explicit_format_version_overrides_env(tmp_path): + result = _run_fts_format_creation_probe( + tmp_path, + "invalid", + {"format_version": 1}, + 1, + ) + + assert result.returncode == 0, result.stderr + + +def test_create_text_inverted_index_defaults_to_v2_without_env(tmp_path): + result = _run_fts_format_creation_probe(tmp_path, None, expected_format_version=2) + + assert result.returncode == 0, result.stderr + + +def test_create_inverted_index_rejects_invalid_env_format_version(tmp_path): + result = _run_fts_format_creation_probe(tmp_path, "invalid") + + assert result.returncode != 0 + assert "LANCE_FTS_FORMAT_VERSION" in result.stderr + assert "invalid" in result.stderr def test_create_inverted_index_rejects_invalid_format_version(tmp_path): @@ -5000,7 +6284,10 @@ def test_create_inverted_index_rejects_invalid_format_version(tmp_path): ds = lance.write_dataset(data, tmp_path) with pytest.raises(ValueError, match="unsupported FTS format version"): - ds.create_scalar_index("text", index_type="INVERTED", format_version="v3") + ds.create_scalar_index("text", index_type="INVERTED", format_version="v5") + + with pytest.raises(ValueError, match="unsupported FTS format version"): + ds.create_scalar_index("text", index_type="INVERTED", format_version="v4") def test_vector_filter_fts_search(tmp_path): diff --git a/python/python/tests/test_schema.py b/python/python/tests/test_schema.py index fcff283ebe2..67ba402a116 100644 --- a/python/python/tests/test_schema.py +++ b/python/python/tests/test_schema.py @@ -3,11 +3,28 @@ import pickle from pathlib import Path +from typing import TYPE_CHECKING, Optional import lance import pyarrow as pa +import pytest # pyright: ignore[reportMissingImports] from lance.schema import LanceSchema +if TYPE_CHECKING: + from typing import assert_type + + from lance.lance.schema import LanceField + + def _check_field_lookup_types(schema: LanceSchema) -> None: + """Static-only guard: both lookups return an optional field. + + ``LanceField`` exists only as a stub type -- ``lance.lance`` is a + compiled extension that re-exports ``LanceSchema`` alone -- so these + assertions cannot run, but pyright checks them. + """ + assert_type(schema.field("x"), Optional[LanceField]) + assert_type(schema.field_case_insensitive("x"), Optional[LanceField]) + def test_lance_schema(tmp_path: Path): # Include nested fields to test the reconstruction of the schema @@ -54,9 +71,61 @@ def test_lance_schema(tmp_path: Path): assert l_children[0].id() == 5 # Changing column name does not change the id - dataset.alter_columns({"path": "s.a", "name": "new_name"}) + # alter_columns is variadic, but its parameter is annotated + # Iterable[AlterColumn] rather than AlterColumn, so a single alteration + # does not type check. Unrelated to this file; suppressed rather than + # fixed here to keep the change focused. + dataset.alter_columns( + {"path": "s.a", "name": "new_name"} # pyright: ignore[reportArgumentType] + ) schema = dataset.lance_schema fields = schema.fields() s_fields = fields[1].children() assert s_fields[0].name() == "new_name" assert s_fields[0].id() == 2 + + +def test_lance_schema_from_protos_rejects_missing_parent(): + # name (field 2): child; id (field 3): 7; parent_id (field 4): 42; + # logical_type (field 5): int32. + field_proto = b"\x12\x05child\x18\x07\x20\x2a\x2a\x05int32" + + with pytest.raises( + ValueError, + match="Field 'child' \\(id=7\\) references parent id 42", + ): + LanceSchema._from_protos("{}", field_proto) + + +def test_lance_schema_field_lookup(tmp_path: Path): + dataset = lance.write_dataset( + pa.table({"x": range(2), "s": [{"a": 1}, {"a": 2}]}), tmp_path + ) + schema = dataset.lance_schema + + field = schema.field("x") + assert field is not None + assert field.name() == "x" + + # Dotted paths address nested fields; a miss returns None rather than + # raising, which is what the Optional return type encodes. + nested = schema.field("s.a") + assert nested is not None + assert nested.name() == "a" + assert schema.field("does_not_exist") is None + + +def test_lance_schema_field_case_insensitive(tmp_path: Path): + dataset = lance.write_dataset(pa.table({"MixedCase": range(2)}), tmp_path) + schema = dataset.lance_schema + + exact = schema.field_case_insensitive("MixedCase") + assert exact is not None + assert exact.name() == "MixedCase" + + # Falls back to a case-insensitive match, preserving the original casing. + relaxed = schema.field_case_insensitive("mixedcase") + assert relaxed is not None + assert relaxed.name() == "MixedCase" + + assert schema.field_case_insensitive("does_not_exist") is None diff --git a/python/python/tests/test_schema_evolution.py b/python/python/tests/test_schema_evolution.py index 7df6962789e..abd89e0cded 100644 --- a/python/python/tests/test_schema_evolution.py +++ b/python/python/tests/test_schema_evolution.py @@ -15,6 +15,7 @@ import pytest from lance import LanceDataset from lance.file import LanceFileReader, LanceFileWriter +from lance.fragment import write_fragments def test_drop_columns(tmp_path: Path): @@ -573,3 +574,56 @@ def test_add_cols_all_null_with_sql(tmp_path: Path): "b": pa.int32(), } ) + + +def test_merge_nullability_assertion(tmp_path: Path): + tbl = pa.table({"value": pa.array([1, 2], pa.int32())}) + lance.write_dataset(tbl, tmp_path) + written_at = lance.dataset(tmp_path).version + + # Stage an append against the original schema, then add a non-nullable + # column. The merge claims non-null, and reading it back must preserve + # the claim, or recommitting it would silently drop the barrier. + fragments = write_fragments( + pa.table({"value": pa.array([7], pa.int32())}), tmp_path, mode="append" + ) + lance.dataset(tmp_path).add_columns({"one": "1"}) + txn = lance.dataset(tmp_path).read_transaction(2) + assert txn is not None + assert txn.operation.preserves_nullability is False + + # The stale append omits the required column, so its rows would read as + # null under the merged schema; the claim refuses it. + op = lance.LanceOperation.Append(fragments) + with pytest.raises(Exception, match="preempted"): + lance.LanceDataset.commit(tmp_path, op, read_version=written_at) + + # A nullable add preserves nullability and skips the barrier. + lance.dataset(tmp_path).add_columns({"copied": "value"}) + txn = lance.dataset(tmp_path).read_transaction(3) + assert txn is not None + assert txn.operation.preserves_nullability is True + + +def test_project_nullability_assertion_round_trips(tmp_path: Path): + tbl = pa.table({"value": pa.array([1, 2], pa.int32())}) + lance.write_dataset(tbl, tmp_path) + lance.dataset(tmp_path).alter_columns({"path": "value", "nullable": False}) + + # Reading the tightening back must preserve the claim, or recommitting it + # would silently drop the concurrency barrier. + txn = lance.dataset(tmp_path).read_transaction(2) + assert txn is not None + assert txn.operation.preserves_nullability is False + + # A Python-built non-assertion must reach the barrier: race an append. + written_at = lance.dataset(tmp_path).version + appended = lance.write_dataset( + pa.table({"value": pa.array([7], pa.int32())}), tmp_path, mode="append" + ) + assert appended.version > written_at + relax = lance.LanceOperation.Project( + schema=txn.operation.schema, preserves_nullability=False + ) + with pytest.raises(Exception, match="preempted"): + lance.LanceDataset.commit(tmp_path, relax, read_version=written_at) diff --git a/python/python/tests/test_session.py b/python/python/tests/test_session.py index 18ffb6df7fa..05b8a637628 100644 --- a/python/python/tests/test_session.py +++ b/python/python/tests/test_session.py @@ -5,6 +5,7 @@ import lance import pyarrow as pa +import pytest def test_cache_size_bytes( @@ -37,3 +38,86 @@ def test_share_session(tmp_path: Path): assert ds1.session().size_bytes() == ds2.session().size_bytes() assert ds1.to_table() == ds2.to_table() + + +def test_fragment_write_with_session(tmp_path: Path): + from lance.fragment import LanceFragment, write_fragments + + data = pa.table({"a": range(10), "b": [str(i) for i in range(10)]}) + ds = lance.write_dataset(data, tmp_path) + # Drop a column so the surviving field id is non-trivial (!= 0). Appends + # that infer the schema must pick up this field id from the dataset. + ds.drop_columns(["a"]) + field_id = ds.lance_schema.field_case_insensitive("b").id() + assert field_id != 0 + + session = ds.session() + size_before = session.size_bytes() + + append_data = pa.table({"b": ["x", "y"]}) + fragments = write_fragments( + append_data, str(tmp_path), mode="append", session=session + ) + assert len(fragments) == 1 + assert fragments[0].files[0].fields == [field_id] + + fragment = LanceFragment.create( + str(tmp_path), append_data, mode="append", session=session + ) + assert fragment.files[0].fields == [field_id] + + # The manifest loads for schema inference went through the shared session. + assert session.size_bytes() > size_before + + # A LanceDataset destination always uses its own session; a different + # explicit session is rejected. + with pytest.raises(ValueError, match="not the destination dataset's own session"): + write_fragments(append_data, ds, mode="append", session=lance.Session()) + + +def test_cache_backend_uri_config(): + session = lance.Session(index_cache_backend="moka://?capacity=1048576") + + assert session.index_cache_size_bytes() == 0 + + +def test_cache_backend_dict_config(): + session = lance.Session( + index_cache_backend={ + "kind": "MOKA", + "options": {"capacity": "1048576"}, + }, + ) + + assert session.index_cache_size_bytes() == 0 + + +def test_cache_backend_rejects_size_and_backend(): + with pytest.raises( + ValueError, + match="index_cache_size_bytes and index_cache_backend are mutually exclusive", + ): + lance.Session( + index_cache_size_bytes=1024, + index_cache_backend="moka://?capacity=1048576", + ) + + +def test_cache_backend_rejects_unknown_dict_key(): + with pytest.raises(ValueError, match="unknown dict key"): + lance.Session( + index_cache_backend={ + "kind": "moka", + "capacity": "1048576", + }, + ) + + +def test_cache_backend_rejects_moka_without_capacity(): + with pytest.raises(ValueError, match="capacity is required"): + lance.Session(index_cache_backend="moka://") + + +def test_cache_backend_rejects_moka_empty_capacity(): + with pytest.raises(ValueError, match="capacity must not be empty"): + lance.Session(index_cache_backend="moka://?capacity=") diff --git a/python/python/tests/test_table_ops.py b/python/python/tests/test_table_ops.py index 777b89da05a..24cf3409f91 100644 --- a/python/python/tests/test_table_ops.py +++ b/python/python/tests/test_table_ops.py @@ -198,3 +198,57 @@ def test_data_file_create_unknown_column(tmp_path: str): with pytest.raises(Exception, match="z"): DataFile.create(ds, new_file_name) + + +def test_commit_conflict_raises_typed_error(tmp_path: str): + """A losing commit should raise CommitConflictError, not a bare OSError.""" + from lance.commit import CommitConflictError + + table = pa.table({"a": range(100)}) + ds = lance.write_dataset(table, tmp_path) + + # Two commits based on the same version; the second must conflict. + ds2 = lance.dataset(tmp_path) + ds3 = lance.dataset(tmp_path) + + new_data_file = make_data_file(ds, [0], pa.table({"a": range(100, 200)})) + ds2.commit( + ds2.uri, + lance.LanceOperation.DataReplacement( + [lance.LanceOperation.DataReplacementGroup(0, new_data_file)] + ), + read_version=ds2.version, + ) + + new_data_file = make_data_file(ds, [0], pa.table({"a": range(200, 300)})) + with pytest.raises(CommitConflictError) as exc_info: + ds3.commit( + ds3.uri, + lance.LanceOperation.DataReplacement( + [lance.LanceOperation.DataReplacementGroup(0, new_data_file)] + ), + read_version=ds3.version, + ) + + # It must be a CommitConflictError, still catchable as OSError, and retryable. + assert isinstance(exc_info.value, OSError) + assert exc_info.value.retryable is True + + +def test_incompatible_transaction_raises_non_retryable(tmp_path: str): + """An incompatible transaction should raise a non-retryable CommitConflictError.""" + from lance.commit import CommitConflictError + + table = pa.table({"a": [1]}) + uri = tmp_path + base = lance.write_dataset(table, uri) + fragment = lance.fragment.LanceFragment.create(uri, table) + stale_append = lance.LanceOperation.Append([fragment]) + + # Overwrite the table, then try to append based on the stale version. + lance.write_dataset(table, uri, mode="overwrite") + with pytest.raises(CommitConflictError) as exc_info: + lance.LanceDataset.commit(uri, stale_append, read_version=base.version) + + assert isinstance(exc_info.value, OSError) + assert exc_info.value.retryable is False diff --git a/python/python/tests/test_table_provider.py b/python/python/tests/test_table_provider.py index 1eddf220dd2..2252f12aa28 100644 --- a/python/python/tests/test_table_provider.py +++ b/python/python/tests/test_table_provider.py @@ -91,3 +91,31 @@ def make_ctx(): result = normalize(ctx.table("ffi_lance_table").limit(1, offset=1).collect()) assert len(result) == 1 assert result["col1"][0].as_py() == 1 + + +def test_custom_udf_filter(tmp_path): + pytest.importorskip("datafusion") + from datafusion import SessionContext, udf + + def is_even(values: pa.Array) -> pa.Array: + return pa.array([value.as_py() % 2 == 0 for value in values], type=pa.bool_()) + + is_even_udf = udf( + is_even, + input_fields=[pa.int64()], + return_field=pa.bool_(), + volatility="stable", + name="is_even", + ) + + dataset = lance.write_dataset(pa.table({"i": [1, 2, 3, 4]}), str(tmp_path)) + provider = FFILanceTableProvider(dataset, with_row_id=True, with_row_addr=True) + + ctx = SessionContext() + ctx.register_table("numbers", provider) + ctx.register_udf(is_even_udf) + + result = normalize( + ctx.sql("SELECT i FROM numbers WHERE i = 2 AND is_even(i)").collect() + ) + assert result["i"].to_pylist() == [2] diff --git a/python/python/tests/test_tf.py b/python/python/tests/test_tf.py deleted file mode 100644 index 3652df0e938..00000000000 --- a/python/python/tests/test_tf.py +++ /dev/null @@ -1,351 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright The Lance Authors - -import os -import warnings - -import lance -import numpy as np -import pandas as pd -import pyarrow as pa -import pytest -from lance.arrow import ImageArray -from lance.fragment import LanceFragment - -pytest.skip("Skip tensorflow tests", allow_module_level=True) - -try: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - import tensorflow as tf # noqa: F401 -except ImportError: - pytest.skip( - "Tensorflow is not installed. Please install tensorflow to " - + "test lance.tf module.", - allow_module_level=True, - ) - -from lance.tf.data import ( # noqa: E402 - from_lance, - from_lance_batches, - lance_fragments, - lance_take_batches, -) - - -@pytest.fixture -def tf_dataset(tmp_path): - df = pd.DataFrame( - { - "a": range(10000), - "s": [f"val-{i}" for i in range(10000)], - "vec": [[i * 0.2] * 128 for i in range(10000)], - } - ) - - schema = pa.schema( - [ - pa.field("a", pa.int64()), - pa.field("s", pa.string()), - pa.field("vec", pa.list_(pa.float32(), 128)), - ] - ) - tbl = pa.Table.from_pandas(df, schema=schema) - uri = tmp_path / "dataset.lance" - lance.write_dataset( - tbl, - uri, - schema=tbl.schema, - max_rows_per_group=100, - max_rows_per_file=1000, - ) - return uri - - -def test_fragment_dataset(tf_dataset): - ds = from_lance(tf_dataset, batch_size=100) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 - assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8") - assert batch["a"].shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) # Fixed size list - - -def test_projection(tf_dataset): - ds = from_lance(tf_dataset, batch_size=100, columns=["a"]) - - for idx, batch in enumerate(ds): - assert list(batch.keys()) == ["a"] - assert batch["a"].numpy()[0] == idx * 100 - assert batch["a"].shape == (100,) - - -def test_filter(tf_dataset): - ds = from_lance(tf_dataset, batch_size=100, filter="a >= 5000") - - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 + 5000 - assert batch["a"].shape == (100,) - - -def test_namespace_table_id(monkeypatch): - calls = {} - - class DummyScanner: - def __init__(self): - self._batch = pa.record_batch([pa.array([1, 2])], names=["a"]) - self.projected_schema = self._batch.schema - - def to_batches(self): - yield self._batch - - class DummyDataset: - def scanner(self, **kwargs): - return DummyScanner() - - def fake_dataset(uri=None, **kwargs): - calls["uri"] = uri - calls["kwargs"] = kwargs - return DummyDataset() - - monkeypatch.setattr(lance, "dataset", fake_dataset) - - ns = object() - ds = from_lance( - None, - namespace_client=ns, - table_id=["tbl"], - ignore_namespace_table_storage_options=True, - ) - - assert calls["kwargs"]["namespace_client"] is ns - assert calls["kwargs"]["table_id"] == ["tbl"] - assert calls["kwargs"]["ignore_namespace_table_storage_options"] is True - - batches = list(ds) - assert [b["a"].numpy().tolist() for b in batches] == [[1, 2]] - - -def test_scan_use_tf_data(tf_dataset): - ds = tf.data.Dataset.from_lance(tf_dataset) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 - assert batch["s"].numpy()[0] == f"val-{idx * 100}".encode("utf-8") - assert batch["a"].shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) # Fixed size list - - -def test_pass_fragments(tf_dataset): - # Can pass fragments directly to from_lance - dataset = lance.dataset(tf_dataset) - ds = from_lance(tf_dataset, fragments=dataset.get_fragments(), batch_size=100) - ds_default = from_lance(tf_dataset, batch_size=100) - for batch, batch_default in zip(ds, ds_default): - assert batch["a"].numpy()[0] == batch_default["a"].numpy()[0] - assert batch["a"].numpy().shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) - - # Can pass ids directly to from_lance - ds = from_lance(tf_dataset, fragments=[0, 1, 2], batch_size=100) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 100 - assert batch["a"].numpy().shape == (100,) - - -def test_shuffle(tf_dataset): - fragments = lance_fragments(tf_dataset).shuffle(4, seed=20).take(3) - - ds = from_lance(tf_dataset, fragments=fragments, batch_size=100) - raw_ds = lance.dataset(tf_dataset) - scanner = raw_ds.scanner( - fragments=[LanceFragment(raw_ds, fid) for fid in [0, 3, 1]], batch_size=100 - ) - - for batch, raw_batch in zip(ds, scanner.to_batches()): - assert batch["a"].numpy()[0] == raw_batch.to_pydict()["a"][0] - assert batch["a"].numpy().shape == (100,) - assert batch["vec"].shape == ( - 100, - 128, - ) # Fixed size list - - -def test_dataset_batches(tf_dataset): - tf_dataset = lance.dataset(tf_dataset) - batch_size = 300 - batches = list( - from_lance_batches(tf_dataset, batch_size=batch_size).as_numpy_iterator() - ) - assert tf_dataset.count_rows() // batch_size + 1 == len(batches) - assert all(end - start == batch_size for start, end in batches[:-2]) - assert batches[-1][1] - batches[-1][0] == tf_dataset.count_rows() % batch_size - - skip = 5 - batches_skipped = list( - from_lance_batches( - tf_dataset, batch_size=batch_size, skip=skip - ).as_numpy_iterator() - ) - assert batches_skipped == batches[skip:] - - batches_shuffled = list( - from_lance_batches( - tf_dataset, batch_size=batch_size, shuffle=True, seed=42 - ).as_numpy_iterator() - ) - # make sure it does a shuffle - assert batches_shuffled != batches - batches_shuffled2 = list( - from_lance_batches( - tf_dataset, batch_size=batch_size, shuffle=True, seed=42 - ).as_numpy_iterator() - ) - # make sure the shuffle can be deterministic - assert batches_shuffled == batches_shuffled2 - - -def test_take_dataset(tf_dataset): - tf_dataset = lance.dataset(tf_dataset) - batch_ds = from_lance_batches( - tf_dataset, batch_size=100, shuffle=True, seed=42 - ).as_numpy_iterator() - lance_ds = lance_take_batches(tf_dataset, batch_ds) - lance_ds = lance_ds.unbatch().shuffle(400, seed=42).batch(100) - - for batch in lance_ds: - assert batch["a"].numpy().shape == (100,) - - batches = [(0, 200), (100, 200)] - lance_ds = lance_take_batches(tf_dataset, batches, columns=["a"]) - for (start, end), batch in zip(batches, lance_ds): - assert batch["a"].numpy().tolist() == np.arange(start, end).tolist() - assert batch.keys() == {"a"} - - -def test_var_length_list(tmp_path): - """Treat var length list as RaggedTensor.""" - df = pd.DataFrame( - { - "a": range(200), - "l": [[i] * (i % 5 + 1) for i in range(200)], - } - ) - - schema = pa.schema( - [ - pa.field("a", pa.int64()), - pa.field("l", pa.list_(pa.int32())), - ] - ) - tbl = pa.Table.from_pandas(df, schema=schema) - - uri = tmp_path / "dataset.lance" - lance.write_dataset( - tbl, - uri, - schema=tbl.schema, - ) - - output_signature = { - "a": tf.TensorSpec(shape=(None,), dtype=tf.int64), - "l": tf.RaggedTensorSpec(dtype=tf.dtypes.int32, shape=(8, None), ragged_rank=1), - } - - ds = tf.data.Dataset.from_lance( - uri, - batch_size=8, - output_signature=output_signature, - ) - for idx, batch in enumerate(ds): - assert batch["a"].numpy()[0] == idx * 8 - assert batch["l"].shape == (8, None) - assert isinstance(batch["l"], tf.RaggedTensor) - - -def test_nested_struct(tmp_path): - table = pa.table( - { - "x": pa.array( - [ - { - "a": 1, - "json": {"b": "hello", "x": b"abc"}, - }, - { - "a": 24, - "json": {"b": "world", "x": b"def"}, - }, - ] - ) - } - ) - uri = tmp_path / "dataset.lance" - dataset = lance.write_dataset(table, uri) - - ds = tf.data.Dataset.from_lance( - dataset, - batch_size=8, - ) - - for batch in ds: - tf.debugging.assert_equal(batch["x"]["a"], tf.constant([1, 24], dtype=tf.int64)) - tf.debugging.assert_equal( - batch["x"]["json"]["b"], tf.constant(["hello", "world"]) - ) - tf.debugging.assert_equal( - batch["x"]["json"]["x"], tf.constant([b"abc", b"def"]) - ) - - -def test_tensor(tmp_path): - arr = np.array([[[1, 2, 3], [4, 5, 6]], [[1, 2, 3], [4, 5, 6]]], dtype=np.float32) - table = pa.table({"x": pa.FixedShapeTensorArray.from_numpy_ndarray(arr)}) - - uri = tmp_path / "dataset.lance" - dataset = lance.write_dataset(table, uri) - ds = tf.data.Dataset.from_lance(dataset) - - for batch in ds: - assert batch["x"].shape == (2, 2, 3) - assert batch["x"].dtype == tf.float32 - assert batch["x"].numpy().tolist() == arr.tolist() - - -def test_image_types(tmp_path): - path = [os.path.join(os.path.dirname(__file__), "images/1.png")] - uris = ImageArray.from_array(path * 3) - encoded_images = uris.read_uris() - tensors = encoded_images.to_tensor() - table = pa.table( - { - "uris": uris, - "encoded_images": encoded_images, - "tensor_images": tensors, - } - ) - - uri = tmp_path / "dataset.lance" - dataset = lance.write_dataset(table, uri) - ds = tf.data.Dataset.from_lance(dataset) - - for batch in ds: - assert batch["uris"].shape == (3,) - assert batch["uris"].dtype == tf.string - assert batch["uris"].numpy().astype("str").tolist() == uris.tolist() - - assert batch["encoded_images"].shape == (3,) - assert batch["encoded_images"].dtype == tf.string - assert batch["encoded_images"].numpy().tolist() == encoded_images.tolist() - - assert batch["tensor_images"].shape == (3, 1, 1, 4) - assert batch["tensor_images"].dtype == tf.uint8 - assert batch["tensor_images"].numpy().tolist() == tensors.to_numpy().tolist() diff --git a/python/python/tests/test_udf.py b/python/python/tests/test_udf.py new file mode 100644 index 00000000000..4ba53bc862e --- /dev/null +++ b/python/python/tests/test_udf.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +"""Tests for the ``BatchUDF`` public contract. + +This module is in the pyright target configured in ``pyproject.toml``, so it +doubles as a strict client for the signatures: the annotated locals below fail +the repository type check if a parameter or return type is narrowed or widened +incorrectly -- narrowing ``checkpoint_file`` back to ``str``, for example, +reports + + error: Argument of type "Path" cannot be assigned to parameter + "checkpoint_file" of type "str | None" + +It cannot catch the annotations being *deleted*, because pyright infers the +same types from the function bodies. That failure mode is mypy-specific +(``no-untyped-call``); policing it would take +``reportMissingParameterType`` on ``lance/udf.py``, which needs one more +parameter annotated and a pre-existing narrowing diagnostic resolved. +""" + +from pathlib import Path + +import pyarrow as pa +from lance.udf import BatchUDF, batch_udf + + +def _add_doubled(batch: pa.RecordBatch) -> pa.RecordBatch: + doubled = [value * 2 for value in batch.column("a").to_pylist()] + return pa.RecordBatch.from_pydict({"doubled": doubled}) + + +def test_batch_udf_constructor(tmp_path: Path) -> None: + output_schema = pa.schema([pa.field("doubled", pa.int64())]) + + udf: BatchUDF = BatchUDF(_add_doubled, output_schema=output_schema) + assert udf.output_schema == output_schema + assert udf.cache is None + + # checkpoint_file accepts a Path as well as a str. + checkpointed: BatchUDF = BatchUDF( + _add_doubled, + output_schema=output_schema, + checkpoint_file=tmp_path / "checkpoint.sqlite", + ) + assert checkpointed.cache is not None + + +def test_batch_udf_decorator() -> None: + output_schema = pa.schema([pa.field("doubled", pa.int64())]) + + @batch_udf(output_schema=output_schema) + def doubled(batch: pa.RecordBatch) -> pa.RecordBatch: + return _add_doubled(batch) + + # The decorator returns a BatchUDF, not the original function. + udf: BatchUDF = doubled + assert udf.output_schema == output_schema + + # Calling it delegates straight to the wrapped function, so a UDF stays + # testable on its own. + result = udf(pa.RecordBatch.from_pydict({"a": [1, 2, 3]})) + assert result.column("doubled").to_pylist() == [2, 4, 6] diff --git a/python/python/tests/test_vector.py b/python/python/tests/test_vector.py index 4ea4e7d425e..12caec5f0d5 100644 --- a/python/python/tests/test_vector.py +++ b/python/python/tests/test_vector.py @@ -5,7 +5,12 @@ import numpy as np import pyarrow as pa import pytest -from lance.vector import hamming_clustering_for_sample, vec_to_table +from lance.vector import ( + get_ivf_partition_info, + hamming_clustering_for_ivf_partition, + hamming_clustering_for_sample, + vec_to_table, +) def test_dict(): @@ -150,21 +155,26 @@ def test_binary_vectors_invalid_metric(tmp_path): def _hash_table(hashes): - """Build a table with a ``hash`` column of FixedSizeList. + """Build a table with a ``hash`` column of FixedSizeList. - ``hashes`` is a list of 8-byte sequences, one per row. + ``hashes`` is a list of byte sequences, one per row. The byte width must + be a positive multiple of 8. """ + byte_width = len(hashes[0]) + assert byte_width > 0 and byte_width % 8 == 0 + assert all(len(row) == byte_width for row in hashes) flat = [byte for row in hashes for byte in row] values = pa.FixedSizeListArray.from_arrays( - pa.array(flat, type=pa.uint8()), list_size=8 + pa.array(flat, type=pa.uint8()), list_size=byte_width ) return pa.Table.from_arrays([values], names=["hash"]) -def test_hamming_clustering_for_sample(tmp_path): - hash_a = [0, 0, 0, 0, 0, 0, 0, 0] - hash_b = [255, 0, 0, 0, 0, 0, 0, 0] # 8 bits from hash_a - hash_c = [1, 2, 3, 4, 5, 6, 7, 8] # far from both +@pytest.mark.parametrize("byte_width", [8, 16]) +def test_hamming_clustering_for_sample(tmp_path, byte_width): + hash_a = [0] * byte_width + hash_b = [0] * (byte_width - 8) + [255] + [0] * 7 # 8 bits from hash_a + hash_c = list(range(1, byte_width + 1)) # far from both # Rows 0,1,2 share hash_a; rows 3,4 share hash_b; row 5 is unique. table = _hash_table([hash_a, hash_a, hash_a, hash_b, hash_b, hash_c]) dataset = lance.write_dataset(table, tmp_path / "hashes") @@ -182,3 +192,89 @@ def test_hamming_clustering_for_sample(tmp_path): } # Singleton row 5 is not emitted as a cluster. assert clusters == {0: [1, 2], 3: [4]} + + +@pytest.mark.parametrize("byte_width", [8, 16]) +def test_hamming_clustering_multi_segment(tmp_path, byte_width): + mask = (1 << 64) - 1 + + def hash_bytes(value): + if byte_width == 8: + lanes = [(value * 0x9E3779B97F4A7C15) & mask] + else: + # Adjacent logical values share the first 64-bit lane and differ in + # later lanes, so threshold-0 clustering must compare every lane. + lanes = [ + ((value // 2) * 0x9E3779B97F4A7C15) & mask, + ((value * 0xD6E8FEB86659FD93) ^ 0xA5A5A5A5A5A5A5A5) & mask, + ] + return [ + byte for lane_value in lanes for byte in lane_value.to_bytes(8, "little") + ] + + # 25 distinct hash values, two copies each; the same table is written to + # fragment 0 and appended as fragment 1. + values = [i // 2 for i in range(50)] + table = _hash_table([hash_bytes(value) for value in values]) + dataset = lance.write_dataset(table, tmp_path / "hashes") + dataset.create_index( + "hash", index_type="IVF_FLAT", num_partitions=4, metric="hamming" + ) + dataset = lance.write_dataset(table, tmp_path / "hashes", mode="append") + # Optimizing with merge disabled creates a delta segment for fragment 1. + dataset.optimize.optimize_indices(num_indices_to_merge=0) + + index = dataset.describe_indices()[0] + assert len(index.segments) == 2 + + infos = get_ivf_partition_info(dataset, index.name) + assert sum(info["size"] for info in infos) == 100 + + # All four copies of each value cluster together across both fragments. + frag1_start = 1 << 32 + clusters = [] + for info in infos: + result = hamming_clustering_for_ivf_partition( + dataset, index.name, info["partition_id"], 0 + ).read_all() + clusters.extend( + zip( + result["representative"].to_pylist(), + result["duplicates"].to_pylist(), + ) + ) + assert len(clusters) == 25 + for representative, duplicates in clusters: + assert representative < frag1_start + assert len(duplicates) == 3 + assert any(dup >= frag1_start for dup in duplicates) + + # Selecting the fragment-0 segment reproduces the single-segment scope. + first_segment = next( + segment for segment in index.segments if segment.fragment_ids == {0} + ) + infos = get_ivf_partition_info( + dataset, index.name, index_segments=[first_segment.uuid] + ) + assert sum(info["size"] for info in infos) == 50 + num_selected_clusters = 0 + for info in infos: + result = hamming_clustering_for_ivf_partition( + dataset, + index.name, + info["partition_id"], + 0, + index_segments=[first_segment.uuid], + ).read_all() + for duplicates in result["duplicates"].to_pylist(): + num_selected_clusters += 1 + assert duplicates == [dup for dup in duplicates if dup < frag1_start] + assert len(duplicates) == 1 + assert num_selected_clusters == 25 + + with pytest.raises(ValueError, match="invalid index segment uuid"): + get_ivf_partition_info(dataset, index.name, index_segments=["not-a-uuid"]) + with pytest.raises(TypeError, match="str or uuid.UUID"): + get_ivf_partition_info(dataset, index.name, index_segments=[123]) + with pytest.raises(TypeError, match="not a single"): + get_ivf_partition_info(dataset, index.name, index_segments=first_segment.uuid) diff --git a/python/python/tests/test_vector_index.py b/python/python/tests/test_vector_index.py index fa2c4047cd0..e3a76a6ce92 100644 --- a/python/python/tests/test_vector_index.py +++ b/python/python/tests/test_vector_index.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +import json import logging import os import platform @@ -60,18 +61,26 @@ def gen_str(n): def create_multivec_table( - nvec=1000, nvec_per_row=5, ndim=128, nans=0, nullify=False, dtype=np.float32 + nvec=1000, + nvec_per_row=5, + ndim=128, + nans=0, + nullify=False, + dtype=np.float32, + seed=None, ): - mat = np.random.randn(nvec, nvec_per_row, ndim) + rng = np.random.default_rng(seed) + text_rng = random.Random(seed) + mat = rng.standard_normal((nvec, nvec_per_row, ndim)) if nans > 0: nans_mat = np.empty((nans, ndim)) nans_mat[:] = np.nan mat = np.concatenate((mat, nans_mat), axis=0) mat = mat.astype(dtype) - price = np.random.rand(nvec + nans) * 100 + price = rng.random(nvec + nans) * 100 def gen_str(n): - return "".join(random.choices(string.ascii_letters + string.digits, k=n)) + return "".join(text_rng.choices(string.ascii_letters + string.digits, k=n)) meta = np.array([gen_str(100) for _ in range(nvec + nans)]) @@ -112,13 +121,20 @@ def indexed_dataset(tmp_path): tbl = create_table() dataset = lance.write_dataset(tbl, tmp_path) yield dataset.create_index( - "vector", index_type="IVF_PQ", num_partitions=4, num_sub_vectors=16 + "vector", + index_type="IVF_PQ", + num_partitions=4, + num_sub_vectors=16, + max_iters=2, + sample_rate=2, ) @pytest.fixture() def multivec_dataset(): - tbl = create_multivec_table() + # Keep at least 100 logical rows for the top-k assertions below. Five + # vectors per row still exercises multivector deduplication and fanout. + tbl = create_multivec_table(nvec=128, seed=42) yield lance.write_dataset(tbl, "memory://") @@ -127,8 +143,11 @@ def indexed_multivec_dataset(multivec_dataset): yield multivec_dataset.create_index( "vector", index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, + num_partitions=1, + num_sub_vectors=4, + num_bits=4, + max_iters=2, + sample_rate=2, metric="cosine", ) @@ -222,6 +241,46 @@ def test_batch_flat_query_matches_repeated_single_queries(dataset, queries): ) +@pytest.mark.parametrize("metric", ["l2", "cosine"]) +@pytest.mark.parametrize("query_count", [3, 1], ids=["three_queries", "single_query"]) +def test_batch_indexed_query_matches_repeated_single_queries( + dataset, metric, query_count +): + indexed = dataset.create_index( + "vector", + index_type="IVF_PQ", + num_partitions=4, + num_sub_vectors=16, + metric=metric, + ) + # Give the query vectors deliberately different magnitudes: a cosine batch + # that normalized the whole concatenated key by one global norm would scale + # them unequally and diverge from per-query single search. + scales = np.linspace(0.1, 10.0, query_count).reshape(-1, 1) + queries = (np.random.randn(query_count, 128) * scales).astype(np.float32) + k = 5 + + # nprobes covers every partition so the shared-scan batch path and the + # repeated single-query path search the same partitions deterministically. + nearest_kwargs = {"use_index": True, "nprobes": 4} + batch = indexed.to_table( + columns=["id"], + nearest={"column": "vector", "q": queries, "k": k, **nearest_kwargs}, + ) + + assert batch.column_names == ["query_index", "id", "_distance"] + assert batch["query_index"].to_pylist() == sum( + [[i] * k for i in range(query_count)], [] + ) + + _assert_batch_matches_single_queries( + indexed, + queries, + k=k, + nearest_kwargs=nearest_kwargs, + ) + + def _assert_batch_matches_single_queries(ds, queries, k, nearest_kwargs): batch = ds.to_table( columns=["id"], @@ -371,13 +430,19 @@ def test_distributed_ivf_pq_partition_window_env_override(tmp_path, monkeypatch) monkeypatch.setenv("LANCE_IVF_PQ_MERGE_PARTITION_WINDOW_SIZE", "4") monkeypatch.setenv("LANCE_IVF_PQ_MERGE_PARTITION_PREFETCH_WINDOW_COUNT", "2") - data = create_table(nvec=3000, ndim=128) - q = np.random.randn(128).astype(np.float32) + rng = np.random.default_rng(42) + matrix = rng.standard_normal((640, 32), dtype=np.float32) + data = vec_to_table(data=matrix).append_column("id", pa.array(range(640))) + q = rng.standard_normal(32).astype(np.float32) assert_distributed_vector_consistency( data, "vector", index_type="IVF_PQ", - index_params={"num_partitions": 10, "num_sub_vectors": 16}, + index_params={ + "num_partitions": 10, + "num_sub_vectors": 4, + "max_iters": 2, + }, queries=[q], topk=10, world=2, @@ -404,7 +469,7 @@ def test_distributed_vector( request, fixture_name, index_type, index_params, similarity_threshold ): ds = request.getfixturevalue(fixture_name) - q = np.random.randn(128).astype(np.float32) + q = np.random.default_rng(42).standard_normal(128).astype(np.float32) assert_distributed_vector_consistency( ds.to_table(), "vector", @@ -502,20 +567,20 @@ def test_f16_cuda(tmp_path): "index_file_version", [IndexFileVersion.V3, IndexFileVersion.LEGACY] ) def test_index_with_nans(tmp_path, index_file_version): - # 1024 rows, the entire table should be sampled - tbl = create_table(nvec=1000, nans=24) + tbl = create_table(nvec=256, ndim=32, nans=8) dataset = lance.write_dataset(tbl, tmp_path) dataset = dataset.create_index( "vector", index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, + num_partitions=1, + num_sub_vectors=4, + max_iters=2, index_file_version=index_file_version, ) idx_stats = dataset.stats.index_stats("vector_idx") assert idx_stats["indices"][0]["index_file_version"] == index_file_version - validate_vector_index(dataset, "vector") + validate_vector_index(dataset, "vector", sample_size=16) @pytest.mark.parametrize( @@ -524,22 +589,60 @@ def test_index_with_nans(tmp_path, index_file_version): def test_torch_index_with_nans(tmp_path, index_file_version): torch = pytest.importorskip("torch") - # 1024 rows, the entire table should be sampled - tbl = create_table(nvec=1000, nans=24) + # Torch PQ initialization samples 256 valid residuals. Keep a small margin + # after NaN filtering so every platform can produce a complete sample batch. + tbl = create_table(nvec=320, ndim=32, nans=8) dataset = lance.write_dataset(tbl, tmp_path) dataset = dataset.create_index( "vector", index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, + num_partitions=1, + num_sub_vectors=4, + max_iters=2, accelerator=torch.device("cpu"), one_pass_ivfpq=True, index_file_version=index_file_version, ) idx_stats = dataset.stats.index_stats("vector_idx") assert idx_stats["indices"][0]["index_file_version"] == index_file_version - validate_vector_index(dataset, "vector") + validate_vector_index(dataset, "vector", sample_size=16) + + +def test_torch_index_nan_init_centroid(tmp_path): + """A NaN vector must never seed a centroid. + + `vector is not null` does not exclude NaN, so sampling could pick one; with + a single partition that left every residual NaN and the index build failed. + """ + torch = pytest.importorskip("torch") + from lance.torch.data import LanceDataset as TorchDataset + from lance.vector import _sample_init_centroids + + # Only the last 8 rows are finite, so any sample that keeps NaN rows seeds + # the centroid with one. + mat = np.full((32, 8), np.nan, dtype=np.float32) + mat[24:] = np.random.randn(8, 8).astype(np.float32) + dataset = lance.write_dataset(vec_to_table(data=mat), tmp_path) + assert dataset.to_table()["vector"].null_count == 0 + + ds = TorchDataset(dataset, batch_size=1, columns=["vector"], samples=32) + centroids = _sample_init_centroids(ds, 4, filter_nan=True) + assert centroids.shape[0] == 4 + assert torch.isfinite(centroids).all() + + +def test_torch_index_all_nan_rejected(tmp_path): + pytest.importorskip("torch") + from lance.torch.data import LanceDataset as TorchDataset + from lance.vector import _sample_init_centroids + + mat = np.full((16, 8), np.nan, dtype=np.float32) + dataset = lance.write_dataset(vec_to_table(data=mat), tmp_path) + + ds = TorchDataset(dataset, batch_size=1, columns=["vector"], samples=16) + with pytest.raises(ValueError, match="all null or non-finite"): + _sample_init_centroids(ds, 1, filter_nan=True) def test_index_with_no_centroid_movement(tmp_path): @@ -548,7 +651,8 @@ def test_index_with_no_centroid_movement(tmp_path): # this test makes the centroids essentially [1..] # this makes sure the early stop condition in the index building code # doesn't do divide by zero - mat = np.concatenate([np.ones((256, 32))]) + # Torch one-pass PQ emits an 8-bit codebook, which requires 256 rows. + mat = np.ones((256, 16), dtype=np.float32) tbl = vec_to_table(data=mat) @@ -558,27 +662,37 @@ def test_index_with_no_centroid_movement(tmp_path): index_type="IVF_PQ", num_partitions=1, num_sub_vectors=4, + max_iters=2, accelerator=torch.device("cpu"), ) - validate_vector_index(dataset, "vector") + validate_vector_index(dataset, "vector", sample_size=8) def test_index_with_pq_codebook(tmp_path): - tbl = create_table(nvec=1024, ndim=128) + dim = 16 + rng = np.random.default_rng(42) + # Eight-bit PQ still requires its 256 centroid training rows even when the + # initial codebook is supplied; reducing the dimension keeps this fixture small. + vectors = rng.standard_normal((256, dim), dtype=np.float32) + tbl = vec_to_table(data=vectors) dataset = lance.write_dataset(tbl, tmp_path) - pq_codebook = np.random.randn(4, 256, 128 // 4).astype(np.float32) + pq_codebook = rng.standard_normal((4, 256, dim // 4), dtype=np.float32) + ivf_centroids = rng.standard_normal((1, dim), dtype=np.float32) dataset = dataset.create_index( "vector", index_type="IVF_PQ", num_partitions=1, num_sub_vectors=4, - ivf_centroids=np.random.randn(1, 128).astype(np.float32), + max_iters=2, + ivf_centroids=ivf_centroids, pq_codebook=pq_codebook, ) index = dataset.stats.index_stats("vector_idx") assert index["indices"][0]["sub_index"]["nbits"] == 8 - validate_vector_index(dataset, "vector", refine_factor=10, pass_threshold=0.99) + validate_vector_index( + dataset, "vector", refine_factor=256, sample_size=8, pass_threshold=0.99 + ) pq_codebook = pa.FixedShapeTensorArray.from_numpy_ndarray(pq_codebook) @@ -587,17 +701,23 @@ def test_index_with_pq_codebook(tmp_path): index_type="IVF_PQ", num_partitions=1, num_sub_vectors=4, - ivf_centroids=np.random.randn(1, 128).astype(np.float32), + max_iters=2, + ivf_centroids=ivf_centroids, pq_codebook=pq_codebook, replace=True, ) - validate_vector_index(dataset, "vector", refine_factor=10, pass_threshold=0.99) + validate_vector_index( + dataset, "vector", refine_factor=256, sample_size=8, pass_threshold=0.99 + ) def test_index_with_4bit_numpy_pq_codebook(tmp_path): - tbl = create_table(nvec=1024, ndim=128) + dim = 32 + rng = np.random.default_rng(42) + vectors = rng.standard_normal((32, dim), dtype=np.float32) + tbl = vec_to_table(data=vectors) dataset = lance.write_dataset(tbl, tmp_path) - pq_codebook = np.random.randn(4, 16, 128 // 4).astype(np.float32) + pq_codebook = rng.standard_normal((4, 16, dim // 4), dtype=np.float32) dataset = dataset.create_index( "vector", @@ -605,7 +725,8 @@ def test_index_with_4bit_numpy_pq_codebook(tmp_path): num_partitions=1, num_sub_vectors=4, num_bits=4, - ivf_centroids=np.random.randn(1, 128).astype(np.float32), + max_iters=2, + ivf_centroids=rng.standard_normal((1, dim), dtype=np.float32), pq_codebook=pq_codebook, ) @@ -615,7 +736,7 @@ def test_index_with_4bit_numpy_pq_codebook(tmp_path): result = dataset.to_table( nearest={ "column": "vector", - "q": np.random.randn(128).astype(np.float32), + "q": vectors[0], "k": 10, } ) @@ -623,13 +744,15 @@ def test_index_with_4bit_numpy_pq_codebook(tmp_path): def test_index_with_pq_codebook_rejects_wrong_num_bits_shape(tmp_path): - tbl = create_table(nvec=8, ndim=128) + dim = 16 + rng = np.random.default_rng(42) + tbl = vec_to_table(data=rng.standard_normal((8, dim), dtype=np.float32)) dataset = lance.write_dataset(tbl, tmp_path) - pq_codebook = np.random.randn(4, 256, 128 // 4).astype(np.float32) + pq_codebook = rng.standard_normal((4, 256, dim // 4), dtype=np.float32) with pytest.raises( ValueError, - match=r"\(sub_vectors, 16, dim\) for num_bits=4, got \(4, 256, 32\)", + match=r"\(sub_vectors, 16, dim\) for num_bits=4, got \(4, 256, 4\)", ): dataset.create_index( "vector", @@ -637,7 +760,7 @@ def test_index_with_pq_codebook_rejects_wrong_num_bits_shape(tmp_path): num_partitions=1, num_sub_vectors=4, num_bits=4, - ivf_centroids=np.random.randn(1, 128).astype(np.float32), + ivf_centroids=rng.standard_normal((1, dim), dtype=np.float32), pq_codebook=pq_codebook, ) @@ -739,14 +862,18 @@ def test_create_index_unsupported_accelerator(tmp_path): def test_create_index_accelerator_fallback(tmp_path, caplog): - tbl = create_table() + tbl = create_table(nvec=64, ndim=32) dataset = lance.write_dataset(tbl, tmp_path) with caplog.at_level(logging.WARNING): dataset = dataset.create_index( "vector", index_type="IVF_HNSW_SQ", - num_partitions=4, + num_partitions=1, + max_iters=2, + max_level=2, + m=4, + ef_construction=16, accelerator="cuda", ) @@ -816,62 +943,106 @@ def test_has_index(dataset, tmp_path): assert ann_ds.describe_indices()[0].field_names == ["vector"] -def test_index_type(dataset, tmp_path): - ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance") - - ann_ds = ann_ds.create_index( - "vector", - index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, - replace=True, - ) - stats = ann_ds.stats.index_stats("vector_idx") - assert stats["index_type"] == "IVF_PQ" - - ann_ds = ann_ds.create_index( - "vector", - index_type="IVF_HNSW_SQ", - num_partitions=4, - num_sub_vectors=16, - replace=True, - ) - stats = ann_ds.stats.index_stats("vector_idx") - assert stats["index_type"] == "IVF_HNSW_SQ" +def test_index_type(tmp_path): + index_cases = [ + ("IVF_PQ", {"num_sub_vectors": 4, "num_bits": 4}), + ( + "IVF_HNSW_SQ", + {"max_level": 2, "m": 4, "ef_construction": 16}, + ), + ( + "IVF_HNSW_PQ", + { + "num_sub_vectors": 4, + "num_bits": 4, + "max_level": 2, + "m": 4, + "ef_construction": 16, + }, + ), + ( + "IVF_HNSW_FLAT", + {"max_level": 2, "m": 4, "ef_construction": 16}, + ), + ] + rng = np.random.default_rng(42) + vectors = rng.standard_normal((64, 32), dtype=np.float32) + table = vec_to_table(data=vectors).append_column("id", pa.array(range(64))) + ann_ds = lance.write_dataset(table, tmp_path / "replace_index_type") + assert not ann_ds.has_index - ann_ds = ann_ds.create_index( - "vector", - index_type="IVF_HNSW_PQ", - num_partitions=4, - num_sub_vectors=16, - replace=True, - ) - stats = ann_ds.stats.index_stats("vector_idx") - assert stats["index_type"] == "IVF_HNSW_PQ" + for case_index, (index_type, index_options) in enumerate(index_cases): + ann_ds = ann_ds.create_index( + "vector", + index_type=index_type, + num_partitions=1, + max_iters=2, + sample_rate=2, + replace=case_index > 0, + **index_options, + ) + stats = ann_ds.stats.index_stats("vector_idx") + assert stats["index_type"] == index_type + assert stats["num_indices"] == 1 + indices = ann_ds.describe_indices() + assert len(indices) == 1 + assert indices[0].field_names == ["vector"] + + nearest = { + "column": "vector", + "q": vectors[0], + "k": 10, + "nprobes": 1, + "refine_factor": 4, + } + if "HNSW" in index_type: + nearest["ef"] = 64 + actual = ann_ds.to_table(columns=["id"], nearest=nearest) + expected = ann_ds.to_table( + columns=["id"], + nearest={ + "column": "vector", + "q": vectors[0], + "k": 10, + "use_index": False, + }, + ) + actual_ids = set(actual["id"].to_pylist()) + expected_ids = set(expected["id"].to_pylist()) + assert actual.num_rows == 10 + assert len(actual_ids) == 10 + assert len(actual_ids & expected_ids) / len(expected_ids) >= 0.5 -def test_create_dot_index(dataset, tmp_path): - assert not dataset.has_index - ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance") +def test_create_dot_index(tmp_path): + rng = np.random.default_rng(42) + table = vec_to_table(data=rng.standard_normal((64, 32), dtype=np.float32)) + ann_ds = lance.write_dataset(table, tmp_path / "indexed.lance") + assert not ann_ds.has_index ann_ds = ann_ds.create_index( "vector", index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, + num_partitions=1, + num_sub_vectors=4, + num_bits=4, + max_iters=2, metric="dot", ) assert ann_ds.has_index -def test_create_4bit_ivf_pq_index(dataset, tmp_path): - assert not dataset.has_index - ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance") +def test_create_4bit_ivf_pq_index(tmp_path): + rng = np.random.default_rng(42) + table = vec_to_table(data=rng.standard_normal((32, 32), dtype=np.float32)) + ann_ds = lance.write_dataset(table, tmp_path / "indexed.lance") + assert not ann_ds.has_index ann_ds = ann_ds.create_index( "vector", index_type="IVF_PQ", num_partitions=1, - num_sub_vectors=16, + num_sub_vectors=4, num_bits=4, + max_iters=2, metric="l2", ) index = ann_ds.stats.index_stats("vector_idx") @@ -1061,10 +1232,10 @@ def test_create_ivf_rq_index(): "vector", index_type="IVF_RQ", num_partitions=4, - num_bits=1, ) assert ds.describe_indices()[0].field_names == ["vector"] stats = ds.stats.index_stats("vector_idx") + assert stats["indices"][0]["sub_index"]["num_bits"] == 5 assert stats["indices"][0]["sub_index"]["packed"] is True with pytest.raises( @@ -1104,6 +1275,13 @@ def test_create_ivf_rq_index(): assert res["_distance"].to_numpy().max() == 0.0 +def test_build_rq_model_default_num_bits(): + from lance.lance import indices + + model = json.loads(indices.build_rq_model(dimension=8)) + assert model["num_bits"] == 5 + + def test_create_ivf_rq_skip_transpose(): ds = lance.write_dataset(create_table(), "memory://") ds = ds.create_index( @@ -1215,53 +1393,31 @@ def test_create_ivf_rq_mostly_null(): assert result.num_rows == 10 -def test_create_ivf_hnsw_pq_index(dataset, tmp_path): - assert not dataset.has_index - ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance") - ann_ds = ann_ds.create_index( - "vector", - index_type="IVF_HNSW_PQ", - num_partitions=4, - num_sub_vectors=16, - ) - assert ann_ds.describe_indices()[0].field_names == ["vector"] - - -def test_create_ivf_hnsw_sq_index(dataset, tmp_path): - assert not dataset.has_index - ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance") - ann_ds = ann_ds.create_index( - "vector", - index_type="IVF_HNSW_SQ", - num_partitions=4, - num_sub_vectors=16, - ) - assert ann_ds.describe_indices()[0].field_names == ["vector"] - - -def test_create_ivf_hnsw_flat_index(dataset, tmp_path): - assert not dataset.has_index - ann_ds = lance.write_dataset(dataset.to_table(), tmp_path / "indexed.lance") - ann_ds = ann_ds.create_index( - "vector", - index_type="IVF_HNSW_FLAT", - num_partitions=4, - num_sub_vectors=16, - ) - assert ann_ds.describe_indices()[0].field_names == ["vector"] - - def test_multivec_ann(indexed_multivec_dataset: lance.LanceDataset): - query = np.random.rand(5, 128) + rng = np.random.default_rng(42) + query = rng.random((5, 128)) results = indexed_multivec_dataset.scanner( - nearest={"column": "vector", "q": query, "k": 100} + nearest={ + "column": "vector", + "q": query, + "k": 100, + "nprobes": 1, + "refine_factor": 2, + } ).to_table() assert results.num_rows == 100 assert results["vector"].type == pa.list_(pa.list_(pa.float32(), 128)) assert len(results["vector"][0]) == 5 + ground_truth = indexed_multivec_dataset.to_table( + columns=["id"], + nearest={"column": "vector", "q": query, "k": 100, "use_index": False}, + ) + actual_ids = set(results["id"].to_pylist()) + expected_ids = set(ground_truth["id"].to_pylist()) + assert len(actual_ids & expected_ids) / len(expected_ids) >= 0.5 # query with single vector also works - query = np.random.rand(128) + query = rng.random(128) results = indexed_multivec_dataset.to_table( nearest={"column": "vector", "q": query, "k": 100} ) @@ -1282,20 +1438,71 @@ def test_multivec_ann(indexed_multivec_dataset: lance.LanceDataset): ) # query with a vector that dim not match - query = np.random.rand(256) + query = rng.random(256) with pytest.raises(ValueError, match="does not match index column size"): indexed_multivec_dataset.to_table( nearest={"column": "vector", "q": query, "k": 100} ) # query with a list of vectors that some dim not match - query = [np.random.rand(128)] * 5 + [np.random.rand(256)] + query = [rng.random(128)] * 5 + [rng.random(256)] with pytest.raises(ValueError, match="All query vectors must have the same length"): indexed_multivec_dataset.to_table( nearest={"column": "vector", "q": query, "k": 100} ) +def test_multivec_search_paths(tmp_path: Path): + vector_type = pa.list_(pa.list_(pa.float32(), 2)) + query = np.array([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32) + uri = tmp_path / "multivec_distance.lance" + + indexed_rows = pa.table( + { + "id": pa.array([0, 1], type=pa.int32()), + "vector": pa.array( + [ + [[1.0, 0.0], [0.0, 1.0]], + [[1.0, 0.0], [1.0, 0.0]], + ], + type=vector_type, + ), + } + ) + dataset = lance.write_dataset(indexed_rows, uri) + dataset = dataset.create_index( + "vector", + index_type="IVF_FLAT", + metric="cosine", + num_partitions=1, + ) + + unindexed_rows = pa.table( + { + "id": pa.array([2, 3], type=pa.int32()), + "vector": pa.array( + [ + [[1.0, 0.0], [0.0, 1.0]], + [[-1.0, 0.0], [0.0, -1.0]], + ], + type=vector_type, + ), + } + ) + dataset = lance.write_dataset(unindexed_rows, uri, mode="append") + + nearest = {"column": "vector", "q": query, "k": 4, "metric": "cosine"} + flat = dataset.to_table(columns=["id"], nearest={**nearest, "use_index": False}) + mixed = dataset.to_table(columns=["id"], nearest=nearest) + + dataset.optimize.optimize_indices() + fully_indexed = dataset.to_table(columns=["id"], nearest=nearest, fast_search=True) + + for result in [flat, mixed, fully_indexed]: + assert result["id"].to_pylist() == [0, 2, 1, 3] + np.testing.assert_allclose(result["_distance"].to_numpy(), [0.0, 0.0, 1.0, 2.0]) + + def test_pre_populated_ivf_centroids(dataset, tmp_path: Path): centroids = np.random.randn(5, 128).astype(np.float32) # IVF5 dataset_with_index = dataset.create_index( @@ -1703,17 +1910,23 @@ def test_index_cache_size_deprecation(tmp_path): def test_f16_index(tmp_path: Path): - DIM = 64 + DIM = 32 + total = 256 uri = tmp_path / "f16data.lance" - f16_data = np.random.uniform(0, 1, 2048 * DIM).astype(np.float16) + rng = np.random.default_rng(42) + f16_data = rng.uniform(0, 1, total * DIM).astype(np.float16) fsl = pa.FixedSizeListArray.from_arrays(f16_data, DIM) tbl = pa.Table.from_pydict({"vector": fsl}) dataset = lance.write_dataset(tbl, uri) dataset.create_index( - "vector", index_type="IVF_PQ", num_partitions=4, num_sub_vectors=2 + "vector", + index_type="IVF_PQ", + num_partitions=1, + num_sub_vectors=4, + max_iters=2, ) - q = np.random.uniform(0, 1, DIM).astype(np.float16) + q = rng.uniform(0, 1, DIM).astype(np.float16) rst = dataset.to_table( nearest={ "column": "vector", @@ -1728,8 +1941,9 @@ def test_f16_index(tmp_path: Path): def test_vector_with_nans(tmp_path: Path): DIM = 32 - TOTAL = 2048 - data = np.random.uniform(0, 1, TOTAL * DIM).astype(np.float32) + TOTAL = 320 + rng = np.random.default_rng(42) + data = rng.uniform(0, 1, TOTAL * DIM).astype(np.float32) # Put the 1st vector as NaN. np.put(data, range(DIM, 2 * DIM), np.nan) @@ -1743,12 +1957,13 @@ def test_vector_with_nans(tmp_path: Path): ds = dataset.create_index( "vector", index_type="IVF_PQ", - num_partitions=2, - num_sub_vectors=2, + num_partitions=1, + num_sub_vectors=4, + max_iters=2, replace=True, ) tbl = ds.to_table( - nearest={"column": "vector", "q": data[0:DIM], "k": TOTAL, "nprobes": 2}, + nearest={"column": "vector", "q": data[0:DIM], "k": TOTAL, "nprobes": 1}, with_row_id=True, ) assert len(tbl) == TOTAL - 1 @@ -1804,14 +2019,18 @@ def test_dynamic_projection_with_vectors_index(tmp_path: Path): def test_index_cast_centroids(tmp_path): torch = pytest.importorskip("torch") - tbl = create_table(nvec=1000) + dim = 16 + rng = np.random.default_rng(42) + # Torch one-pass PQ emits an 8-bit codebook, which requires 256 rows. + tbl = vec_to_table(data=rng.standard_normal((256, dim), dtype=np.float32)) dataset = lance.write_dataset(tbl, tmp_path) dataset = dataset.create_index( "vector", index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, + num_partitions=2, + num_sub_vectors=4, + max_iters=2, accelerator=torch.device("cpu"), ) @@ -1820,18 +2039,19 @@ def test_index_cast_centroids(tmp_path): index_stats = dataset.stats.index_stats(index_name) centroids = index_stats["indices"][0]["centroids"] values = pa.array([x for arr in centroids for x in arr], pa.float32()) - centroids = pa.FixedSizeListArray.from_arrays(values, 128) + centroids = pa.FixedSizeListArray.from_arrays(values, dim) # Cast invalidates the attached index; drop it first per the new contract. dataset.drop_index(index_name) - dataset.alter_columns(dict(path="vector", data_type=pa.list_(pa.float16(), 128))) + dataset.alter_columns(dict(path="vector", data_type=pa.list_(pa.float16(), dim))) # centroids are f32, but the column is now f16 dataset = dataset.create_index( "vector", index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, + num_partitions=2, + num_sub_vectors=4, + max_iters=2, accelerator=torch.device("cpu"), ivf_centroids=centroids, ) @@ -1961,29 +2181,220 @@ def test_optimize_indices(indexed_dataset): assert stats["num_indices"] == 2 -@pytest.mark.skip(reason="retrain is deprecated") -def test_retrain_indices(indexed_dataset): - data = create_table() - indexed_dataset = lance.write_dataset(data, indexed_dataset.uri, mode="append") +@pytest.mark.parametrize("enable_stable_row_ids", [False, True]) +def test_segment_ownership_filter_precedes_partition_topk( + tmp_path, enable_stable_row_ids +): + ndim = 4 + + def table(ids, value): + vectors = np.full((len(ids), ndim), value, dtype=np.float32) + return pa.table( + { + "id": pa.array(ids, type=pa.int64()), + "vector": pa.FixedSizeListArray.from_arrays( + pa.array(vectors.reshape(-1), type=pa.float32()), ndim + ), + } + ) + + dataset = lance.write_dataset( + table(range(20), 1.0), + tmp_path, + mode="create", + enable_stable_row_ids=enable_stable_row_ids, + ) + dataset = lance.write_dataset( + table(range(100, 120), 0.0), dataset.uri, mode="append" + ) + dataset = dataset.create_index( + "vector", index_type="IVF_FLAT", metric="l2", num_partitions=1 + ) + + fragment = dataset.get_fragment(1) + row_ids = fragment.to_table(columns=["id"], with_row_id=True)["_rowid"].to_pylist() + update_data = pa.table( + { + "_rowid": pa.array(row_ids, type=pa.uint64()), + "vector": pa.array( + [[10.0] * ndim] * len(row_ids), type=pa.list_(pa.float32(), ndim) + ), + } + ) + updated_fragment, fields_modified = fragment.update_columns(update_data) + dataset = lance.LanceDataset.commit( + dataset.uri, + lance.LanceOperation.Update( + updated_fragments=[updated_fragment], fields_modified=fields_modified + ), + read_version=dataset.version, + ) + dataset.optimize.optimize_indices(num_indices_to_merge=0) + dataset = lance.dataset(dataset.uri) + + def assert_current_nearest_rows(): + result = dataset.to_table( + columns=["id"], + nearest={ + "column": "vector", + "q": np.zeros(ndim, dtype=np.float32), + "k": 5, + }, + ) + + assert all(row_id < 20 for row_id in result["id"].to_pylist()) + assert result["_distance"].to_pylist() == pytest.approx([4.0] * 5) + + assert_current_nearest_rows() + dataset.optimize.optimize_indices(num_indices_to_merge=2) + dataset = lance.dataset(dataset.uri) + assert_current_nearest_rows() + + +def test_no_stale_duplicate_after_partial_column_update(tmp_path): + # Regression test: updating an indexed vector column in place (via the + # low-level fragment.update_columns API + LanceOperation.Update) and then + # delta-optimizing the index must not leave a stale copy of the row in the + # original index segment. + # + # Mechanism: update_columns rewrites only the column data file, keeping the + # fragment id and row address. Committing the Update prunes the fragment + # from the old index segment's fragment_bitmap, but that segment's index + # file still physically holds the row's OLD vector. optimize_indices then + # builds a new delta segment with the NEW vector. Before the fix a KNN query + # searched both segments and returned the updated row TWICE - once with the + # stale vector (old segment) and once with the new value (delta segment). + np.random.seed(42) + ndim = 16 + + # Fragment 0: a "far" cluster bounded to [-1, 1]. No bulk vector is close to + # the query (all-10.8), so the bulk cannot crowd the stale copy out of top-k. + n_bulk = 1000 + bulk = np.random.uniform(-1, 1, (n_bulk, ndim)).astype(np.float32) + table0 = pa.table( + { + "id": pa.array(range(n_bulk), type=pa.int64()), + "vector": pa.FixedSizeListArray.from_arrays( + pa.array(bulk.reshape(-1), type=pa.float32()), list_size=ndim + ), + } + ) + ds = lance.write_dataset(table0, tmp_path, mode="create") + + # Fragment 1: a single row whose ORIGINAL vector (all 2.0) is closer to the + # query than any bulk vector, so its stale copy ranks well inside top-k. + orig = np.full((1, ndim), 2.0, dtype=np.float32) + table1 = pa.table( + { + "id": pa.array([10_000], type=pa.int64()), + "vector": pa.FixedSizeListArray.from_arrays( + pa.array(orig.reshape(-1), type=pa.float32()), list_size=ndim + ), + } + ) + ds = lance.write_dataset(table1, tmp_path, mode="append") + assert len(ds.get_fragments()) == 2 + + # One index segment covering BOTH fragments {0, 1}. + ds = ds.create_index( + "vector", + index_type="IVF_PQ", + metric="l2", + num_partitions=1, + num_sub_vectors=ndim, + ) + + # Overwrite fragment 1's vector in place and commit Update(fields_modified). + new_vec = [10.8] * ndim + frag = ds.get_fragment(1) + rowids = frag.to_table(columns=["id"], with_row_id=True)["_rowid"].to_pylist() + update_data = pa.table( + { + "_rowid": pa.array(rowids, type=pa.uint64()), + "vector": pa.array( + [new_vec] * len(rowids), type=pa.list_(pa.float32(), ndim) + ), + } + ) + updated_fragment, fields_modified = frag.update_columns(update_data) + op = lance.LanceOperation.Update( + updated_fragments=[updated_fragment], + fields_modified=fields_modified, + ) + ds = lance.LanceDataset.commit(ds.uri, op, read_version=ds.version) + + # Delta-optimize: appends a new segment for the updated fragment; the old + # segment is left intact, still physically holding the stale vector. + ds.optimize.optimize_indices(num_indices_to_merge=0) + ds = lance.dataset(ds.uri) + assert ds.stats.index_stats("vector_idx")["num_indices"] == 2 + + # KNN near the NEW value via the default vector search (searches all + # segments). The updated row must appear EXACTLY ONCE. + # + # This pins the filtering only. With a single partition the late search + # returns before the shared budget is consulted, so the accounting half of + # the fix is pinned by the Rust unit test + # `test_unowned_row_does_not_fill_the_shared_budget` instead. + q = np.array(new_vec, dtype=np.float32) + res = ds.to_table( + columns=["id"], + nearest={"column": "vector", "q": q, "k": 10}, + with_row_id=True, + ).to_pandas() + dupes = res[res["id"] == 10_000] + assert len(dupes) == 1, ( + f"updated row id=10000 returned {len(dupes)} times " + f"(stale index segment not masked); rowids={res['_rowid'].tolist()}" + ) + # A mask that over-restricts would drop the old segment wholesale and still + # satisfy the assertion above, so pin the full result set too. + assert len(res) == 10, f"expected a full top-10, got {len(res)} rows" + assert res["id"].is_unique, f"duplicate ids in result: {res['id'].tolist()}" + + +@pytest.mark.parametrize("retrain", [None, False, True]) +def test_retrain_indices(tmp_path, retrain): + rng = np.random.default_rng(42) + ndim = 16 + initial_vectors = rng.standard_normal((64, ndim), dtype=np.float32) + appended_vectors = rng.standard_normal((64, ndim), dtype=np.float32) + 100 + old_centroid = np.full((1, ndim), -1000, dtype=np.float32) + + indexed_dataset = lance.write_dataset(vec_to_table(initial_vectors), tmp_path) + indexed_dataset = indexed_dataset.create_index( + "vector", + index_type="IVF_FLAT", + num_partitions=1, + ivf_centroids=old_centroid, + index_file_version=IndexFileVersion.V3, + ) + indexed_dataset = lance.write_dataset( + vec_to_table(appended_vectors), indexed_dataset.uri, mode="append" + ) + stats = indexed_dataset.stats.index_stats("vector_idx") assert stats["num_indices"] == 1 indexed_dataset.optimize.optimize_indices(num_indices_to_merge=0) stats = indexed_dataset.stats.index_stats("vector_idx") assert stats["num_indices"] == 2 + assert all( + index["centroids"] == old_centroid.tolist() for index in stats["indices"] + ) + kwargs = {} if retrain is None else {"retrain": retrain} + indexed_dataset.optimize.optimize_indices(**kwargs) stats = indexed_dataset.stats.index_stats("vector_idx") - centroids = stats["indices"][0]["centroids"] - delta_centroids = stats["indices"][1]["centroids"] - assert centroids == delta_centroids - - indexed_dataset.optimize.optimize_indices(retrain=True) - new_centroids = indexed_dataset.stats.index_stats("vector_idx")["indices"][0][ - "centroids" - ] - stats = indexed_dataset.stats.index_stats("vector_idx") - assert stats["num_indices"] == 1 - assert centroids != new_centroids + centroids = [index["centroids"] for index in stats["indices"]] + if retrain: + expected_centroid = np.concatenate([initial_vectors, appended_vectors]).mean( + axis=0 + ) + assert stats["num_indices"] == 1 + assert np.allclose(centroids[0][0], expected_centroid) + else: + assert all(centroid == old_centroid.tolist() for centroid in centroids) def test_no_include_deleted_rows(indexed_dataset): @@ -2053,6 +2464,34 @@ def test_read_partition(indexed_dataset): VectorIndexReader(indexed_dataset, "id_idx") +def test_read_partition_nested_vector_quoted_field(tmp_path): + num_rows = 1024 + dimensions = 8 + rng = np.random.default_rng(42) + values = rng.integers(0, 256, size=num_rows * dimensions, dtype=np.uint8) + vectors = pa.FixedSizeListArray.from_arrays(pa.array(values), dimensions) + nested = pa.StructArray.from_arrays([vectors], names=["embedding.v1"]) + dataset = lance.write_dataset(pa.table({"data": nested}), tmp_path) + # Match nested uint8 pHash indexes without introducing PQ training setup. + dataset = dataset.create_index( + "data.`embedding.v1`", + index_type="IVF_FLAT", + name="vector_idx", + metric="hamming", + num_partitions=4, + ) + + reader = VectorIndexReader(dataset, "vector_idx") + for with_vector in (False, True): + partitions = [ + reader.read_partition(partition_id, with_vector=with_vector) + for partition_id in range(reader.num_partitions()) + ] + + assert all("_rowid" in partition.column_names for partition in partitions) + assert sum(partition.num_rows for partition in partitions) == num_rows + + def test_vector_index_with_prefilter_and_scalar_index(indexed_dataset): uri = indexed_dataset.uri new_table = create_table() @@ -2272,6 +2711,14 @@ def test_nested_field_vector_index(tmp_path): assert len(indices) == 1 assert indices[0].field_names == ["data.embedding"] + reader = VectorIndexReader(dataset, indices[0].name) + for with_vector in (False, True): + partition_rows = sum( + reader.read_partition(partition_id, with_vector=with_vector).num_rows + for partition_id in range(reader.num_partitions()) + ) + assert partition_rows == num_rows + # Test querying with the index query_vec = vectors[0] result = dataset.to_table( @@ -2468,7 +2915,7 @@ def test_vector_index_distance_range(tmp_path): assert np.all(index_distances >= distance_range[0]) and np.all( index_distances < distance_range[1] ) - assert np.allclose(brute_distances, index_distances, rtol=0.0, atol=0.0) + assert np.allclose(brute_distances, index_distances, rtol=1e-5, atol=0.0) # ============================================================================= @@ -2585,17 +3032,16 @@ def assert_distributed_vector_consistency( """Recall-only consistency check between single-machine and distributed indices. This helper keeps the original signature for compatibility but ignores - similarity_metric/similarity_threshold. It compares recall@K against a ground - truth computed via exact search (use_index=False) on the single dataset and - asserts that the recall difference between single-machine and distributed - indices is within 10%. + similarity_metric. It compares recall@K against a ground truth computed via + exact search (use_index=False), requires both indices to reach at least 0.5 + recall, and bounds their recall difference with similarity_threshold. Steps ----- 1) Write `data` to two URIs (single, distributed); ensure distributed has >=2 fragments (rewrite with max_rows_per_file if needed) 2) Build a single-machine index via `create_index` - 3) Global training (IVF/PQ) using `IndicesBuilder.prepare_global_ivfpq` when + 3) Global training (IVF/PQ) using `IndicesBuilder.prepare_global_ivf_pq` when appropriate; for IVF_FLAT/SQ variants, train IVF centroids via `IndicesBuilder.train_ivf` 4) Build the distributed index via @@ -2603,11 +3049,12 @@ def assert_distributed_vector_consistency( preprocessed artifacts 5) For each query, compute ground-truth TopK IDs using exact search (use_index=False), then compute TopK using single index and the distributed - index with consistent nearest settings (refine_factor=1; IVF uses nprobes) - 6) Compute recall for single and distributed using the provided formula and - assert the absolute difference is <= 0.10. Also print the recalls. + index with consistent nearest settings (refine_factor=100; IVF probes all + fixture partitions) + 6) Compute recall for single and distributed, require each to be >= 0.5, + and bound their absolute difference with similarity_threshold. """ - # Keep signature compatibility but ignore similarity_metric/threshold + # Keep signature compatibility but ignore the superseded metric selector. _ = similarity_metric index_params = index_params or {} @@ -2633,33 +3080,37 @@ def assert_distributed_vector_consistency( data, dist_uri, mode="overwrite", max_rows_per_file=500 ) + num_rows = single_ds.count_rows() + nparts = index_params.get("num_partitions", None) + is_pq = index_type in {"IVF_PQ", "IVF_HNSW_PQ"} + # Eight-bit PQ needs at least 256 centroids and sample_rate >= 2. + sample_rate = 2 if is_pq else min(8, num_rows // max(1, nparts or 1)) + max_iters = index_params.get("max_iters", 5) + build_params = dict(index_params) + build_params.setdefault("sample_rate", sample_rate) + build_params.setdefault("max_iters", max_iters) + # Build single-machine index single_ds = single_ds.create_index( column=column, index_type=index_type, - **index_params, + **build_params, ) # Global training / preparation for distributed build preprocessed = None builder = IndicesBuilder(single_ds, column) - nparts = index_params.get("num_partitions", None) nsub = index_params.get("num_sub_vectors", None) dist_type = index_params.get("metric", "l2") - num_rows = single_ds.count_rows() - - # Choose a safe sample_rate that satisfies IVF (nparts*sr <= rows) and PQ - # (256*sr <= rows). Minimum 2 as required by builder verification. - safe_sr_ivf = num_rows // max(1, nparts or 1) - safe_sr_pq = num_rows // 256 - safe_sr = max(2, min(safe_sr_ivf, safe_sr_pq)) - if index_type in {"IVF_PQ", "IVF_HNSW_PQ"}: + if is_pq: + assert num_rows >= 512, "8-bit PQ training requires at least 512 rows" preprocessed = builder.prepare_global_ivf_pq( nparts, nsub, distance_type=dist_type, - sample_rate=safe_sr, + sample_rate=sample_rate, + max_iters=max_iters, ) elif ( ("IVF_FLAT" in index_type) @@ -2669,7 +3120,8 @@ def assert_distributed_vector_consistency( ivf_model = builder.train_ivf( nparts, distance_type=dist_type, - sample_rate=safe_sr, + sample_rate=sample_rate, + max_iters=max_iters, ) preprocessed = {"ivf_centroids": ivf_model.centroids} @@ -2723,7 +3175,7 @@ def assert_distributed_vector_consistency( # Consistent nearest settings for index-based search nearest = {"column": column, "q": q, "k": topk, "refine_factor": 100} if "IVF" in index_type: - nearest["nprobes"] = max(16, int(index_params.get("num_partitions", 4)) * 4) + nearest["nprobes"] = int(index_params.get("num_partitions", 4)) if "HNSW" in index_type: # Ensure ef is large enough even when refine_factor multiplies k for HNSW effective_k = topk * int( @@ -2751,10 +3203,18 @@ def compute_recall(gt: np.ndarray, result: np.ndarray) -> float: rs = compute_recall(gt_ids, single_ids) rd = compute_recall(gt_ids, dist_ids) - # Assert recall difference within 10% - assert abs(rs - rd) <= 1 - similarity_threshold, ( + assert rs >= 0.5, ( + f"Single-machine {index_type} recall below 0.5: recall={rs:.3f}, " + f"num_partitions={nparts}, topk={topk}, queries={len(queries)}" + ) + assert rd >= 0.5, ( + f"Distributed {index_type} recall below 0.5: recall={rd:.3f}, " + f"num_partitions={nparts}, topk={topk}, queries={len(queries)}" + ) + max_recall_difference = 1 - similarity_threshold + assert abs(rs - rd) <= max_recall_difference, ( f"Recall difference too large: single={rs:.3f}, distributed={rd:.3f}, " - f"diff={abs(rs - rd):.3f} (> {similarity_threshold})" + f"diff={abs(rs - rd):.3f} (> {max_recall_difference:.3f})" ) # Cleanup temporary directory if used @@ -2773,324 +3233,88 @@ def _make_sample_dataset_base( max_rows_per_file: int = 500, ): """Common helper to construct sample datasets for distributed index tests.""" - mat = np.random.rand(n_rows, dim).astype(np.float32) + mat = np.random.default_rng(42).random((n_rows, dim), dtype=np.float32) ids = np.arange(n_rows) - arr = pa.array(mat.tolist(), type=pa.list_(pa.float32(), dim)) + arr = pa.FixedSizeListArray.from_arrays(pa.array(mat.reshape(-1)), dim) tbl = pa.table({"id": ids, "vector": arr}) return lance.write_dataset( tbl, tmp_path / name, max_rows_per_file=max_rows_per_file ) -def test_prepared_global_ivfpq_distributed_merge_and_search(tmp_path: Path): - ds = _make_sample_dataset_base(tmp_path, "preproc_ds", 2000, 128) - - # Global preparation - builder = IndicesBuilder(ds, "vector") - preprocessed = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=4, - distance_type="l2", - sample_rate=3, - max_iters=20, - ) - - # Distributed build using prepared centroids/codebook - ds = build_distributed_vector_index( - ds, - "vector", - index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=4, - world=2, - ivf_centroids=preprocessed["ivf_centroids"], - pq_codebook=preprocessed["pq_codebook"], - ) - - # Query sanity - q = np.random.rand(128).astype(np.float32) - results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10}) - assert 0 < len(results) <= 10 - - -def test_consistency_improves_with_preprocessed_centroids(tmp_path: Path): - ds = _make_sample_dataset_base(tmp_path, "preproc_ds", 2000, 128) - - builder = IndicesBuilder(ds, "vector") - pre = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=16, - distance_type="l2", - sample_rate=7, - max_iters=20, - ) - - # Build single-machine index as ground truth target index - single_ds = lance.write_dataset(ds.to_table(), tmp_path / "single_ivfpq") - single_ds = single_ds.create_index( - column="vector", - index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, - ) - - # Distributed with preprocessed IVF centroids - dist_pre = lance.write_dataset(ds.to_table(), tmp_path / "dist_pre") - dist_pre = build_distributed_vector_index( - dist_pre, - "vector", - index_type="IVF_PQ", - num_partitions=4, - num_sub_vectors=16, - world=2, - ivf_centroids=pre["ivf_centroids"], - pq_codebook=pre["pq_codebook"], - ) - - # Evaluate recall vs exact search - q = np.random.rand(128).astype(np.float32) - topk = 10 - gt = single_ds.to_table( - nearest={"column": "vector", "q": q, "k": topk, "use_index": False} +@pytest.mark.parametrize( + "index_type", + [ + "IVF_FLAT", + "IVF_PQ", + "IVF_SQ", + ], +) +def test_distributed_ivf_two_shard_build_merge_and_search(tmp_path, index_type): + dim = 32 + num_partitions = 2 + ds = _make_sample_dataset_base( + tmp_path, + f"dist_{index_type.lower()}", + n_rows=640, + dim=dim, + max_rows_per_file=320, ) - res_pre = dist_pre.to_table(nearest={"column": "vector", "q": q, "k": topk}) - - gt_ids = gt["id"].to_pylist() - pre_ids = res_pre["id"].to_pylist() - - def _recall(gt_ids, res_ids): - s = set(int(x) for x in gt_ids) - d = set(int(x) for x in res_ids) - return len(s & d) / max(1, len(s)) - - recall_pre = _recall(gt_ids, pre_ids) - - # Expect some non-zero recall with preprocessed IVF centroids - if recall_pre < 0.10: - pytest.skip( - "Distributed IVF_PQ recall below threshold in current " - "environment - known issue" - ) - assert recall_pre >= 0.10 - - -def test_metadata_merge_pq_success(tmp_path): - ds = _make_sample_dataset_base(tmp_path, "dist_ds", 2000, 128) frags = ds.get_fragments() - assert len(frags) >= 2, "Need at least 2 fragments for distributed testing" - mid = max(1, len(frags) // 2) - node1 = [f.fragment_id for f in frags[:mid]] - node2 = [f.fragment_id for f in frags[mid:]] + assert len(frags) == 2 + fragment_groups = [[fragment.fragment_id] for fragment in frags] builder = IndicesBuilder(ds, "vector") - pre = builder.prepare_global_ivf_pq( - num_partitions=8, - num_subvectors=16, - distance_type="l2", - sample_rate=7, - max_iters=20, - ) - try: - segments = _build_segments( - ds, - "vector", - "IVF_PQ", - [node1, node2], - index_name="vector_idx", - num_partitions=8, - num_sub_vectors=16, - ivf_centroids=pre["ivf_centroids"], - pq_codebook=pre["pq_codebook"], + build_kwargs = {"num_partitions": num_partitions} + if index_type == "IVF_PQ": + preprocessed = builder.prepare_global_ivf_pq( + num_partitions=num_partitions, + num_subvectors=4, + distance_type="l2", + sample_rate=2, + max_iters=2, ) - ds = _commit_segments_helper(ds, segments, "vector") - q = np.random.rand(128).astype(np.float32) - results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10}) - assert 0 < len(results) <= 10 - except ValueError as e: - raise e - - -def test_distributed_workflow_merge_and_search(tmp_path): - """End-to-end: build IVF_PQ on two groups, merge, and verify search returns - results.""" - ds = _make_sample_dataset_base(tmp_path, "dist_ds", 2000, 128) - frags = ds.get_fragments() - if len(frags) < 2: - pytest.skip("Need at least 2 fragments for distributed testing") - mid = len(frags) // 2 - node1 = [f.fragment_id for f in frags[:mid]] - node2 = [f.fragment_id for f in frags[mid:]] - builder = IndicesBuilder(ds, "vector") - pre = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=4, - distance_type="l2", - sample_rate=7, - max_iters=20, - ) - try: - segments = _build_segments( - ds, - "vector", - "IVF_PQ", - [node1, node2], - index_name="vector_idx", - num_partitions=4, + assert set(preprocessed) == {"ivf_centroids", "pq_codebook"} + assert len(preprocessed["ivf_centroids"]) == num_partitions + assert preprocessed["ivf_centroids"].type.list_size == dim + assert len(preprocessed["pq_codebook"]) > 0 + assert preprocessed["pq_codebook"].type.list_size == dim + build_kwargs.update( num_sub_vectors=4, - ivf_centroids=pre["ivf_centroids"], - pq_codebook=pre["pq_codebook"], + ivf_centroids=preprocessed["ivf_centroids"], + pq_codebook=preprocessed["pq_codebook"], ) - ds = _commit_segments_helper(ds, segments, "vector") - q = np.random.rand(128).astype(np.float32) - results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10}) - assert 0 < len(results) <= 10 - except ValueError as e: - raise e - - -def test_vector_merge_two_shards_success_flat(tmp_path): - ds = _make_sample_dataset_base(tmp_path, "dist_ds", 1000, 128) - frags = ds.get_fragments() - assert len(frags) >= 2 - shard1 = [frags[0].fragment_id] - shard2 = [frags[1].fragment_id] - # Global preparation - builder = IndicesBuilder(ds, "vector") - preprocessed = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=4, - distance_type="l2", - sample_rate=3, - max_iters=20, - ) + else: + ivf_model = builder.train_ivf( + num_partitions=num_partitions, + distance_type="l2", + sample_rate=8, + max_iters=2, + ) + build_kwargs["ivf_centroids"] = ivf_model.centroids segments = _build_segments( ds, "vector", - "IVF_FLAT", - [shard1, shard2], + index_type, + fragment_groups, index_name="vector_idx", - num_partitions=4, - num_sub_vectors=128, - ivf_centroids=preprocessed["ivf_centroids"], - pq_codebook=preprocessed["pq_codebook"], + **build_kwargs, ) + assert len(segments) == 2 ds = _commit_segments_helper(ds, segments, column="vector") - q = np.random.rand(128).astype(np.float32) - result = ds.to_table(nearest={"column": "vector", "q": q, "k": 5}) - assert 0 < len(result) <= 5 - - -@pytest.mark.parametrize( - "index_type,num_sub_vectors", - [ - ("IVF_PQ", 4), - ("IVF_FLAT", 128), - ], -) -def test_distributed_ivf_parameterized(tmp_path, index_type, num_sub_vectors): - ds = _make_sample_dataset_base(tmp_path, "dist_ds", 2000, 128) - frags = ds.get_fragments() - assert len(frags) >= 2 - mid = len(frags) // 2 - node1 = [f.fragment_id for f in frags[:mid]] - node2 = [f.fragment_id for f in frags[mid:]] - builder = IndicesBuilder(ds, "vector") - pre = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=num_sub_vectors, - distance_type="l2", - sample_rate=7, - max_iters=20, - ) - - try: - base_kwargs = dict( - column="vector", - index_type=index_type, - num_partitions=4, - num_sub_vectors=num_sub_vectors, - ) - - kwargs1 = dict(base_kwargs, fragment_ids=node1) - kwargs2 = dict(base_kwargs, fragment_ids=node2) - if pre is not None: - kwargs1.update( - ivf_centroids=pre["ivf_centroids"], pq_codebook=pre["pq_codebook"] - ) - kwargs2.update( - ivf_centroids=pre["ivf_centroids"], pq_codebook=pre["pq_codebook"] - ) - - segments = [ - ds.create_index_uncommitted(**kwargs1), - ds.create_index_uncommitted(**kwargs2), - ] - ds = _commit_segments_helper(ds, segments, "vector") - - q = np.random.rand(128).astype(np.float32) - results = ds.to_table(nearest={"column": "vector", "q": q, "k": 10}) - assert 0 < len(results) <= 10 - except ValueError as e: - raise e - - -@pytest.mark.parametrize( - "index_type,num_sub_vectors", - [ - ("IVF_PQ", 128), - ("IVF_SQ", None), - ], -) -def test_merge_two_shards_parameterized(tmp_path, index_type, num_sub_vectors): - ds = _make_sample_dataset_base(tmp_path, "dist_ds2", 2000, 128) - frags = ds.get_fragments() - assert len(frags) >= 2 - shard1 = [frags[0].fragment_id] - shard2 = [frags[1].fragment_id] - builder = IndicesBuilder(ds, "vector") - pre = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=num_sub_vectors, - distance_type="l2", - sample_rate=7, - max_iters=20, + stats = ds.stats.index_stats("vector_idx") + assert stats["index_type"] == index_type + q = np.random.default_rng(43).random(dim, dtype=np.float32) + results = ds.to_table( + nearest={ + "column": "vector", + "q": q, + "k": 5, + "nprobes": num_partitions, + "refine_factor": 10, + } ) - - base_kwargs = { - "column": "vector", - "index_type": index_type, - "num_partitions": 4, - } - - # first shard - kwargs1 = dict(base_kwargs) - kwargs1["fragment_ids"] = shard1 - if num_sub_vectors is not None: - kwargs1["num_sub_vectors"] = num_sub_vectors - if pre is not None: - kwargs1["ivf_centroids"] = pre["ivf_centroids"] - # only PQ has pq_codebook - if "pq_codebook" in pre: - kwargs1["pq_codebook"] = pre["pq_codebook"] - segment1 = ds.create_index_uncommitted(**kwargs1) - - # second shard - kwargs2 = dict(base_kwargs) - kwargs2["fragment_ids"] = shard2 - if num_sub_vectors is not None: - kwargs2["num_sub_vectors"] = num_sub_vectors - if pre is not None: - kwargs2["ivf_centroids"] = pre["ivf_centroids"] - if "pq_codebook" in pre: - kwargs2["pq_codebook"] = pre["pq_codebook"] - segment2 = ds.create_index_uncommitted(**kwargs2) - - segments = [segment1, segment2] - ds = _commit_segments_helper(ds, segments, column="vector") - - q = np.random.rand(128).astype(np.float32) - results = ds.to_table(nearest={"column": "vector", "q": q, "k": 5}) assert 0 < len(results) <= 5 @@ -3105,6 +3329,7 @@ def test_commit_existing_index_segments_accepts_index_metadata(tmp_path): num_partitions=2, distance_type="l2", sample_rate=8, + max_iters=2, ) base_kwargs = { "column": "vector", @@ -3147,6 +3372,7 @@ def test_distributed_ivf_rq_shared_rotation(tmp_path): num_partitions=2, distance_type="l2", sample_rate=8, + max_iters=2, ) rabitq_model = indices.build_rq_model(dimension=dim, num_bits=1) base_kwargs = { @@ -3175,16 +3401,21 @@ def test_distributed_ivf_rq_shared_rotation(tmp_path): def test_commit_existing_index_segments_accepts_uncommitted_vector_segments(tmp_path): - ds = _make_sample_dataset_base(tmp_path, "segment_commit_ds", 2000, 128) + dim = 32 + ds = _make_sample_dataset_base( + tmp_path, + "segment_commit_ds", + n_rows=512, + dim=dim, + max_rows_per_file=256, + ) frags = ds.get_fragments() - assert len(frags) >= 2 - builder = IndicesBuilder(ds, "vector") - preprocessed = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=4, + assert len(frags) == 2 + ivf_model = IndicesBuilder(ds, "vector").train_ivf( + num_partitions=2, distance_type="l2", - sample_rate=7, - max_iters=20, + sample_rate=8, + max_iters=2, ) segments = [ @@ -3194,62 +3425,60 @@ def test_commit_existing_index_segments_accepts_uncommitted_vector_segments(tmp_ name="vector_idx", train=True, fragment_ids=[fragment.fragment_id], - num_partitions=4, - num_sub_vectors=128, - ivf_centroids=preprocessed["ivf_centroids"], - pq_codebook=preprocessed["pq_codebook"], + num_partitions=2, + ivf_centroids=ivf_model.centroids, ) - for fragment in frags[:2] + for fragment in frags ] assert len(segments) == 2 ds = ds.commit_existing_index_segments("vector_idx", "vector", segments) - q = np.random.rand(128).astype(np.float32) + q = np.random.rand(dim).astype(np.float32) results = ds.to_table(nearest={"column": "vector", "q": q, "k": 5}) assert 0 < len(results) <= 5 def test_distributed_ivf_pq_order_invariance(tmp_path: Path): """Ensure distributed IVF_PQ build is invariant to shard build order.""" - ds = _make_sample_dataset_base(tmp_path, "dist_ds", 2000, 128) + dim = 32 + ds = _make_sample_dataset_base( + tmp_path, "dist_ds", n_rows=640, dim=dim, max_rows_per_file=320 + ) # Global IVF+PQ training once; artifacts are reused across shard orders. builder = IndicesBuilder(ds, "vector") pre = builder.prepare_global_ivf_pq( - num_partitions=4, - num_subvectors=16, + num_partitions=2, + num_subvectors=4, distance_type="l2", - sample_rate=7, + sample_rate=2, + max_iters=2, ) # Copy the dataset twice so index manifests do not clash and we can vary # the shard build order independently on identical data. ds_order_12 = lance.write_dataset( - ds.to_table(), tmp_path / "pq_order_node1_node2", max_rows_per_file=500 + ds.to_table(), tmp_path / "pq_order_node1_node2", max_rows_per_file=320 ) ds_order_21 = lance.write_dataset( - ds.to_table(), tmp_path / "pq_order_node2_node1", max_rows_per_file=500 + ds.to_table(), tmp_path / "pq_order_node2_node1", max_rows_per_file=320 ) # For each copy, derive two shard groups from its own fragments. frags_12 = ds_order_12.get_fragments() - if len(frags_12) < 2: - pytest.skip("Need at least 2 fragments for distributed indexing (order_12)") + assert len(frags_12) == 2 mid_12 = len(frags_12) // 2 node1_12 = [f.fragment_id for f in frags_12[:mid_12]] node2_12 = [f.fragment_id for f in frags_12[mid_12:]] - if not node1_12 or not node2_12: - pytest.skip("Failed to split fragments into two non-empty groups (order_12)") + assert node1_12 and node2_12 frags_21 = ds_order_21.get_fragments() - if len(frags_21) < 2: - pytest.skip("Need at least 2 fragments for distributed indexing (order_21)") + assert len(frags_21) == 2 mid_21 = len(frags_21) // 2 node1_21 = [f.fragment_id for f in frags_21[:mid_21]] node2_21 = [f.fragment_id for f in frags_21[mid_21:]] - if not node1_21 or not node2_21: - pytest.skip("Failed to split fragments into two non-empty groups (order_21)") + assert node1_21 and node2_21 def build_distributed_ivf_pq(ds_copy, shard_order): try: @@ -3259,8 +3488,8 @@ def build_distributed_ivf_pq(ds_copy, shard_order): "IVF_PQ", shard_order, index_name="vector_idx", - num_partitions=4, - num_sub_vectors=16, + num_partitions=2, + num_sub_vectors=4, ivf_centroids=pre["ivf_centroids"], pq_codebook=pre["pq_codebook"], ) @@ -3273,11 +3502,8 @@ def build_distributed_ivf_pq(ds_copy, shard_order): # Sample queries once from the original dataset and reuse for both index builds # to check order invariance under distributed PQ training and merging. - k = 10 - sample_tbl = ds.sample(10, columns=["vector"]) - queries = [ - np.asarray(v, dtype=np.float32) for v in sample_tbl["vector"].to_pylist() - ] + k = 5 + queries = np.random.default_rng(43).random((3, dim), dtype=np.float32) def collect_ids_and_distances(ds_with_index): ids_per_query = [] @@ -3289,8 +3515,8 @@ def collect_ids_and_distances(ds_with_index): "column": "vector", "q": q, "k": k, - "nprobes": 16, - "refine_factor": 100, + "nprobes": 2, + "refine_factor": 10, }, ) ids_per_query.append([int(x) for x in tbl["id"].to_pylist()]) diff --git a/python/python/tests/torch_tests/test_bench_utils.py b/python/python/tests/torch_tests/test_bench_utils.py index f479bb7f158..5c4943f75bb 100644 --- a/python/python/tests/torch_tests/test_bench_utils.py +++ b/python/python/tests/torch_tests/test_bench_utils.py @@ -10,7 +10,6 @@ torch = pytest.importorskip("torch") from lance.torch.bench_utils import ground_truth, sort_tensors # noqa: E402 -from lance.torch.distance import pairwise_l2 # noqa: E402 def test_sort_tensor(): @@ -31,23 +30,36 @@ def test_ground_truth(tmp_path: Path): N = 1000 NUM_QUERIES = 50 DIM = 128 + K = 20 device = "cpu" # Github action friendly. - data = np.random.rand(N * DIM).astype(np.float32) - fsl = pa.FixedSizeListArray.from_arrays(data, DIM) - data = torch.from_numpy(data.reshape((-1, DIM))).to(device) + # Keep the fixture independent of other tests that seed NumPy's global RNG. + # This seed also keeps every top-20 boundary more than 8e-3 apart. + rng = np.random.RandomState(4415) + data = rng.rand(N, DIM).astype(np.float32) + fsl = pa.FixedSizeListArray.from_arrays(data.reshape(-1), DIM) + torch_data = torch.from_numpy(data).to(device) tbl = pa.Table.from_arrays([fsl], ["vec"]) ds = lance.write_dataset(tbl, tmp_path) - idx = np.random.choice(range(N), NUM_QUERIES) - keys = data[idx, :] + idx = rng.choice(N, NUM_QUERIES) + keys = torch_data[idx, :] - gt = ground_truth(ds, "vec", keys, k=20, batch_size=128, device=device) + gt = ground_truth(ds, "vec", keys, k=K, batch_size=128, device=device) gt, _ = torch.sort(gt, dim=1) - actual_dists = pairwise_l2(keys, data) - expected, _ = torch.sort(torch.argsort(actual_dists, 1)[:, :20], dim=1) - - assert torch.allclose(expected, gt) + # Use direct float64 distances as an oracle, independent of pairwise_l2's + # float32 matrix-multiplication reduction order. + data64 = data.astype(np.float64) + expected = [] + boundary_gaps = [] + for query in data64[idx]: + distances = np.sum(np.square(data64 - query), axis=1) + row_ids = np.argsort(distances, kind="stable") + expected.append(np.sort(row_ids[:K])) + boundary_gaps.append(distances[row_ids[K]] - distances[row_ids[K - 1]]) + + assert min(boundary_gaps) > 8e-3, "fixture is too close to the top-k boundary" + np.testing.assert_array_equal(np.stack(expected), gt.cpu().numpy()) diff --git a/python/python/tests/torch_tests/test_torch_kmeans.py b/python/python/tests/torch_tests/test_torch_kmeans.py index edfb0a0329a..96bbf76c51f 100644 --- a/python/python/tests/torch_tests/test_torch_kmeans.py +++ b/python/python/tests/torch_tests/test_torch_kmeans.py @@ -10,15 +10,16 @@ torch = pytest.importorskip("torch") -from lance.torch import preferred_device # noqa: E402 from lance.torch.kmeans import KMeans # noqa: E402 from lance.vector import train_ivf_centroids_on_accelerator # noqa: E402 -@pytest.mark.skip(reason="flaky") def test_kmeans(): arr = np.array(range(128)).reshape(-1, 8).astype(np.float32) - kmeans = KMeans(4, device="cpu") + # These duplicate centroids reproduce the empty clusters that made this test + # depend on random initialization before empty-cluster recovery was fixed. + centroids = torch.from_numpy(arr[[5, 5, 13, 13]]) + kmeans = KMeans(4, centroids=centroids, device="cpu") kmeans.fit(arr) cluster_ids = kmeans.transform(arr) @@ -27,7 +28,6 @@ def test_kmeans(): assert len(cnts) == 4 # all cluster has data -@pytest.mark.skip(reason="TODO: async dataset hangs on github CI") def test_torch_kmeans_accept_torch_device(tmp_path: Path): values = pa.array(np.array(range(128)).astype(np.float32)) arr = pa.FixedSizeListArray.from_arrays(values, 8) @@ -39,7 +39,7 @@ def test_torch_kmeans_accept_torch_device(tmp_path: Path): "vector", 2, metric_type="L2", - accelerator=preferred_device(), + accelerator=torch.device("cpu"), ) diff --git a/python/src/blob.rs b/python/src/blob.rs index 82e8a01ae8a..cb5fd84323c 100644 --- a/python/src/blob.rs +++ b/python/src/blob.rs @@ -2,18 +2,162 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use crate::{error::PythonErrorExt, rt}; -use arrow::pyarrow::ToPyArrow; +use arrow::{ + array::{Array, ArrayRef, make_array}, + pyarrow::{FromPyArrow, ToPyArrow}, +}; +use arrow_data::ArrayData; +use arrow_schema::{DataType, Field}; use bytes::Bytes; use lance::{ BlobDescriptor, BlobDescriptorArrayBuilder, BlobRange, DedicatedBlobWriter, PackedBlobWriter, }; +use lance_arrow::iter_binary_array; use pyo3::{ - Bound, PyResult, - exceptions::PyValueError, + Bound, PyErr, PyResult, + exceptions::{PyRuntimeError, PyValueError}, pyclass, pymethods, - types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule}, + types::{PyAny, PyAnyMethods, PyDict, PyList, PyListMethods, PyModule, PyTypeMethods}, +}; +use std::{ + borrow::Cow, + sync::{Arc, Mutex}, }; -use std::sync::Arc; + +fn with_writer( + inner: &Mutex>, + writer_name: &str, + operation: impl FnOnce(&W) -> R, +) -> PyResult { + let guard = inner.lock().map_err(|_| poisoned_writer(writer_name))?; + let writer = guard.as_ref().ok_or_else(|| finished_writer(writer_name))?; + Ok(operation(writer)) +} + +fn writer_mut<'a, W>(inner: &'a mut Mutex>, writer_name: &str) -> PyResult<&'a mut W> { + inner + .get_mut() + .map_err(|_| poisoned_writer(writer_name))? + .as_mut() + .ok_or_else(|| finished_writer(writer_name)) +} + +fn take_writer(inner: &mut Mutex>, writer_name: &str) -> PyResult { + inner + .get_mut() + .map_err(|_| poisoned_writer(writer_name))? + .take() + .ok_or_else(|| finished_writer(writer_name)) +} + +fn finished_writer(writer_name: &str) -> PyErr { + PyValueError::new_err(format!("{writer_name} is already finished")) +} + +fn poisoned_writer(writer_name: &str) -> PyErr { + PyRuntimeError::new_err(format!("{writer_name} lock is poisoned")) +} + +/// Reconstruct the PyArrow equivalent of [`BlobDescriptorArrayBuilder::field`]. +/// +/// Arrow's array bridge does not carry the enclosing extension field, so this +/// rebuilds the canonical six nullable blob-v2 children and +/// `ARROW:extension:name = lance.blob.v2` metadata. +fn descriptor_field_to_pyarrow<'py>( + field: &Field, + py: pyo3::Python<'py>, +) -> PyResult> { + let pyarrow = PyModule::import(py, "pyarrow")?; + let child_fields = PyList::empty(py); + for (name, type_fn) in [ + ("kind", "uint8"), + ("data", "large_binary"), + ("uri", "utf8"), + ("blob_id", "uint32"), + ("blob_size", "uint64"), + ("position", "uint64"), + ] { + let data_type = pyarrow.getattr(type_fn)?.call0()?; + let child = pyarrow.call_method1("field", (name, data_type, true))?; + child_fields.append(child)?; + } + let data_type = pyarrow.call_method1("struct", (child_fields,))?; + let metadata = PyDict::new(py); + metadata.set_item("ARROW:extension:name", "lance.blob.v2")?; + let kwargs = PyDict::new(py); + kwargs.set_item("nullable", field.is_nullable())?; + kwargs.set_item("metadata", metadata)?; + pyarrow.call_method("field", (field.name().as_str(), data_type), Some(&kwargs)) +} + +/// Normalize inputs accepted by [`PyPackedBlobWriter::write_blobs`] into Arrow arrays. +/// +/// BinaryArray, LargeBinaryArray, BinaryViewArray, FixedSizeBinaryArray, and +/// ChunkedArray values of any binary type are accepted. Chunk boundaries, +/// nulls, and empty values remain in the arrays; each row is later passed to +/// the core writer as an optional byte slice. +fn extract_blob_payloads(payloads: &Bound<'_, PyAny>) -> PyResult> { + match ArrayData::from_pyarrow_bound(payloads) { + Ok(data) => Ok(vec![validated_blob_payload(data, None)?]), + Err(_) => { + let pyarrow = PyModule::import(payloads.py(), "pyarrow")?; + let chunked_array_type = pyarrow.getattr("ChunkedArray")?; + if !payloads.is_instance(&chunked_array_type)? { + return Err(PyValueError::new_err(format!( + "payloads must be a pyarrow BinaryArray, LargeBinaryArray, or ChunkedArray, got {}", + payloads.get_type().name()? + ))); + } + + let chunked_data_type = DataType::from_pyarrow_bound(&payloads.getattr("type")?)?; + if !chunked_data_type.is_binary() { + return Err(PyValueError::new_err(format!( + "Packed blob payloads must have a Binary Arrow type, got {chunked_data_type}" + ))); + } + + let chunks = payloads.getattr("chunks")?; + let mut arrays = Vec::with_capacity(chunks.len()?); + for (chunk_index, chunk) in chunks.try_iter()?.enumerate() { + let data = ArrayData::from_pyarrow_bound(&chunk?)?; + arrays.push(validated_blob_payload(data, Some(chunk_index))?); + } + Ok(arrays) + } + } +} + +fn validated_blob_payload(data: ArrayData, chunk_index: Option) -> PyResult { + let context = chunk_index + .map(|index| format!("Packed blob payload chunk {index}")) + .unwrap_or_else(|| "Packed blob payload array".to_string()); + if !data.data_type().is_binary() { + return Err(PyValueError::new_err(format!( + "{context} must have a Binary Arrow type, got {}", + data.data_type() + ))); + } + if data.is_empty() { + // PyArrow may export an empty slice without the values preceding its + // nonzero first offset. Normalize it because an empty array never + // observes those buffers, and Arrow validation would reject the slice. + return Ok(make_array(ArrayData::new_empty(data.data_type()))); + } + data.validate_full().map_err(|error| { + PyValueError::new_err(format!("{context} contains invalid Arrow data: {error}")) + })?; + Ok(make_array(data)) +} + +async fn write_binary_payloads(writer: &mut PackedBlobWriter, payloads: &ArrayRef) -> PyResult<()> { + let iter = iter_binary_array(payloads.as_ref()).map_err(|error| { + PyValueError::new_err(format!( + "Packed blob payloads must have a Binary Arrow type, got {}: {error}", + payloads.data_type() + )) + })?; + writer.write_packed_blobs(iter).await.infer_error() +} #[pyclass(name = "BlobDescriptor", skip_from_py_object)] #[derive(Clone)] @@ -61,31 +205,7 @@ impl PyBlobDescriptorArrayBuilder { #[getter] pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { - let pyarrow = PyModule::import(py, "pyarrow")?; - let child_fields = PyList::empty(py); - for (name, type_fn) in [ - ("kind", "uint8"), - ("data", "large_binary"), - ("uri", "utf8"), - ("blob_id", "uint32"), - ("blob_size", "uint64"), - ("position", "uint64"), - ] { - let data_type = pyarrow.getattr(type_fn)?.call0()?; - let child = pyarrow.call_method1("field", (name, data_type, true))?; - child_fields.append(child)?; - } - let data_type = pyarrow.call_method1("struct", (child_fields,))?; - let metadata = PyDict::new(py); - metadata.set_item("ARROW:extension:name", "lance.blob.v2")?; - let kwargs = PyDict::new(py); - kwargs.set_item("nullable", self.field.is_nullable())?; - kwargs.set_item("metadata", metadata)?; - pyarrow.call_method( - "field", - (self.field.name().as_str(), data_type), - Some(&kwargs), - ) + descriptor_field_to_pyarrow(&self.field, py) } pub fn extend_packed( @@ -150,9 +270,10 @@ impl PyBlobDescriptorArrayBuilder { } } -#[pyclass(name = "PackedBlobWriter", skip_from_py_object, unsendable)] +#[pyclass(name = "PackedBlobWriter", skip_from_py_object)] pub struct PyPackedBlobWriter { - inner: Option, + field: Option, + inner: Mutex>, } impl PyPackedBlobWriter { @@ -165,19 +286,22 @@ impl PyPackedBlobWriter { PackedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id) .await .infer_error()?; - Ok(Self { inner: Some(inner) }) + Ok(Self { + field: None, + inner: Mutex::new(Some(inner)), + }) } - fn inner(&self) -> PyResult<&PackedBlobWriter> { - self.inner - .as_ref() - .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished")) + fn with_inner(&self, operation: impl FnOnce(&PackedBlobWriter) -> R) -> PyResult { + with_writer(&self.inner, "PackedBlobWriter", operation) } fn inner_mut(&mut self) -> PyResult<&mut PackedBlobWriter> { - self.inner - .as_mut() - .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished")) + writer_mut(&mut self.inner, "PackedBlobWriter") + } + + fn take_inner(&mut self) -> PyResult { + take_writer(&mut self.inner, "PackedBlobWriter") } } @@ -185,32 +309,132 @@ impl PyPackedBlobWriter { impl PyPackedBlobWriter { #[getter] pub fn blob_id(&self) -> PyResult { - Ok(self.inner()?.blob_id()) + self.with_inner(PackedBlobWriter::blob_id) } #[getter] pub fn path(&self) -> PyResult { - Ok(self.inner()?.path().to_string()) + self.with_inner(|writer| writer.path().to_string()) + } + + /// The descriptor field associated with the array returned by + /// :meth:`finish_array`. + /// + /// The field uses the name passed to ``finish_array`` and carries the + /// ``lance.blob.v2`` extension metadata. It is available only after + /// ``finish_array`` succeeds; accessing it earlier raises ``ValueError``. + #[getter] + pub fn field<'py>(&self, py: pyo3::Python<'py>) -> PyResult> { + let field = self.field.as_ref().ok_or_else(|| { + PyValueError::new_err("PackedBlobWriter field is available after finish_array") + })?; + descriptor_field_to_pyarrow(field, py) } - pub fn write_blob(&mut self, data: Vec) -> PyResult<()> { - rt().block_on(None, self.inner_mut()?.write_blob(data))? + /// Append one packed blob. + /// + /// Python ``bytes`` are borrowed without copying. Other compatible byte + /// sequences use owned storage for the duration of the write. + pub fn write_blob(&mut self, data: Cow<'_, [u8]>) -> PyResult<()> { + rt().block_on(None, self.inner_mut()?.write_blob(data.as_ref()))? .infer_error() } + /// Append a batch of packed blob payloads. + /// + /// Parameters + /// ---------- + /// payloads : pyarrow.BinaryArray, pyarrow.LargeBinaryArray, + /// pyarrow.BinaryViewArray, pyarrow.FixedSizeBinaryArray, or + /// pyarrow.ChunkedArray + /// A binary Arrow array. Every chunk of a chunked array must be binary. + /// Each input row produces one descriptor row, in order, across chunks + /// and repeated calls. Null rows produce null descriptors; empty but + /// non-null byte strings produce valid zero-length blobs. + /// + /// Examples + /// -------- + /// >>> import pyarrow as pa + /// >>> payloads = pa.array([b"first", None, b""], type=pa.large_binary()) + /// >>> writer.write_blobs(payloads) + /// >>> descriptors = writer.finish_array("blob") + /// >>> len(descriptors) + /// 3 + pub fn write_blobs(&mut self, payloads: &Bound<'_, PyAny>) -> PyResult<()> { + let payloads = extract_blob_payloads(payloads)?; + let result = { + let writer = self.inner_mut()?; + rt().block_on(None, async { + for payloads in payloads { + write_binary_payloads(writer, &payloads).await?; + } + Ok(()) + }) + }; + match result { + Ok(result) => result, + Err(error) => { + // KeyboardInterrupt drops the async batch future. Remove the core + // writer as well so RAII cleanup runs and a completed prefix cannot + // be reused as a new batch. + self.take_inner()?; + Err(error) + } + } + } + pub fn finish(&mut self) -> PyResult> { - let inner = self - .inner - .take() - .ok_or_else(|| PyValueError::new_err("PackedBlobWriter is already finished"))?; + let inner = self.take_inner()?; let values = rt().block_on(None, inner.finish())?.infer_error()?; Ok(values.into_iter().map(Into::into).collect()) } + + /// Finish the upload and return its blob descriptors as a PyArrow array. + /// + /// The returned ``pyarrow.StructArray`` has one row per payload previously + /// passed to :meth:`write_blob` or :meth:`write_blobs`. The writer is consumed + /// by this call. After it succeeds, :attr:`field` returns the matching + /// extension field with ``field_name`` as its name. + /// + /// Parameters + /// ---------- + /// field_name : str + /// Name for the descriptor field exposed by :attr:`field`. + /// + /// Returns + /// ------- + /// pyarrow.StructArray + /// Row-aligned blob descriptors, including null rows from bulk input. + /// + /// Examples + /// -------- + /// >>> import pyarrow as pa + /// >>> writer.write_blobs(pa.array([b"value", None])) + /// >>> descriptors = writer.finish_array("payload") + /// >>> descriptors.is_null().to_pylist() + /// [False, True] + /// >>> writer.field.name + /// 'payload' + pub fn finish_array<'py>( + &mut self, + py: pyo3::Python<'py>, + field_name: String, + ) -> PyResult> { + let inner = self.take_inner()?; + let values = rt().block_on(None, inner.finish())?.infer_error()?; + let mut builder = BlobDescriptorArrayBuilder::new(field_name); + builder.extend(values).infer_error()?; + let column = builder.finish().infer_error()?; + let (field, array) = column.into_parts(); + let array = array.to_data().to_pyarrow(py)?; + self.field = Some(field); + Ok(array) + } } -#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object, unsendable)] +#[pyclass(name = "DedicatedBlobWriter", skip_from_py_object)] pub struct PyDedicatedBlobWriter { - inner: Option, + inner: Mutex>, } impl PyDedicatedBlobWriter { @@ -223,19 +447,21 @@ impl PyDedicatedBlobWriter { DedicatedBlobWriter::try_new(object_store.as_ref().clone(), data_file_path, blob_id) .await .infer_error()?; - Ok(Self { inner: Some(inner) }) + Ok(Self { + inner: Mutex::new(Some(inner)), + }) } - fn inner(&self) -> PyResult<&DedicatedBlobWriter> { - self.inner - .as_ref() - .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished")) + fn with_inner(&self, operation: impl FnOnce(&DedicatedBlobWriter) -> R) -> PyResult { + with_writer(&self.inner, "DedicatedBlobWriter", operation) } fn inner_mut(&mut self) -> PyResult<&mut DedicatedBlobWriter> { - self.inner - .as_mut() - .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished")) + writer_mut(&mut self.inner, "DedicatedBlobWriter") + } + + fn take_inner(&mut self) -> PyResult { + take_writer(&mut self.inner, "DedicatedBlobWriter") } } @@ -243,12 +469,12 @@ impl PyDedicatedBlobWriter { impl PyDedicatedBlobWriter { #[getter] pub fn blob_id(&self) -> PyResult { - Ok(self.inner()?.blob_id()) + self.with_inner(DedicatedBlobWriter::blob_id) } #[getter] pub fn path(&self) -> PyResult { - Ok(self.inner()?.path().to_string()) + self.with_inner(|writer| writer.path().to_string()) } pub fn write(&mut self, data: Vec) -> PyResult<()> { @@ -257,10 +483,7 @@ impl PyDedicatedBlobWriter { } pub fn finish(&mut self) -> PyResult { - let inner = self - .inner - .take() - .ok_or_else(|| PyValueError::new_err("DedicatedBlobWriter is already finished"))?; + let inner = self.take_inner()?; let value = rt().block_on(None, inner.finish())?.infer_error()?; Ok(value.into()) } diff --git a/python/src/datagen.rs b/python/src/datagen.rs index 8b046c37f24..194ba3d6a73 100644 --- a/python/src/datagen.rs +++ b/python/src/datagen.rs @@ -1,7 +1,7 @@ use arrow::pyarrow::PyArrowType; -use arrow_array::RecordBatch; +use arrow_array::{RecordBatch, RecordBatchReader}; use arrow_schema::Schema; -use lance_datagen::{BatchCount, ByteCount}; +use lance_datagen::{BatchCount, ByteCount, RowCount}; use pyo3::{ Bound, PyResult, Python, pyfunction, types::{PyModule, PyModuleMethods}, @@ -16,22 +16,44 @@ pub fn is_datagen_supported() -> bool { true } +/// Generate `batch_count` batches of random data for `schema`. +/// +/// Batch size is set either by `rows_in_batch` (exact rows per batch) or +/// `bytes_in_batch` (approximate bytes per batch); the two are mutually +/// exclusive. When neither is given the byte-based default is used. #[pyfunction] -#[pyo3(signature=(schema, batch_count=None, bytes_in_batch=None))] +#[pyo3(signature=(schema, batch_count=None, bytes_in_batch=None, rows_in_batch=None))] pub fn rand_batches( schema: PyArrowType, batch_count: Option, bytes_in_batch: Option, + rows_in_batch: Option, ) -> PyResult>> { - lance_datagen::rand(&schema.0) - .into_reader_bytes( - ByteCount::from(bytes_in_batch.unwrap_or(DEFAULT_BATCH_SIZE_BYTES)), - BatchCount::from(batch_count.unwrap_or(DEFAULT_BATCH_COUNT)), - lance_datagen::RoundingBehavior::RoundUp, - ) - .map_err(|e| { - pyo3::exceptions::PyValueError::new_err(format!("Failed to generate batches: {}", e)) - })? + if rows_in_batch.is_some() && bytes_in_batch.is_some() { + return Err(pyo3::exceptions::PyValueError::new_err( + "rows_in_batch and bytes_in_batch are mutually exclusive", + )); + } + let builder = lance_datagen::rand(&schema.0); + let batch_count = BatchCount::from(batch_count.unwrap_or(DEFAULT_BATCH_COUNT)); + let reader: Box = match rows_in_batch { + Some(rows) => Box::new(builder.into_reader_rows(RowCount::from(rows), batch_count)), + None => Box::new( + builder + .into_reader_bytes( + ByteCount::from(bytes_in_batch.unwrap_or(DEFAULT_BATCH_SIZE_BYTES)), + batch_count, + lance_datagen::RoundingBehavior::RoundUp, + ) + .map_err(|e| { + pyo3::exceptions::PyValueError::new_err(format!( + "Failed to generate batches: {}", + e + )) + })?, + ), + }; + reader .map(|item| { item.map(PyArrowType::from).map_err(|e| { pyo3::exceptions::PyValueError::new_err(format!("Failed to generate batch: {}", e)) diff --git a/python/src/dataset.rs b/python/src/dataset.rs index 350428c89aa..547ba91d16b 100644 --- a/python/src/dataset.rs +++ b/python/src/dataset.rs @@ -41,19 +41,19 @@ use lance::dataset::cleanup::{CleanupFileKind, CleanupPolicyBuilder}; use lance::dataset::refs::{Ref, TagContents}; use lance::dataset::scanner::{ AggregateExpr, ColumnOrdering, DatasetRecordBatchStream, ExecutionStatsCallback, - MaterializationStyle, QueryFilter, + MaterializationStyle, QueryFilter, RowAddrMask, RowAddrTreeMap, }; use lance::dataset::statistics::{DataStatistics, DatasetStatisticsExt}; use lance::dataset::{ BatchInfo, BatchUDF, CommitBuilder, MergeStats, NewColumnTransform, UDFCheckpointStore, WriteDestination, }; -use lance::dataset::{ColumnAlteration, ProjectionRequest}; +use lance::dataset::{ColumnAlteration, ProjectionRequest, validate_dataset_root_for_drop}; use lance::dataset::{ Dataset as LanceDataset, DeleteBuilder, ExternalBlobMode, - MergeInsertBuilder as LanceMergeInsertBuilder, ReadParams, UncommittedMergeInsert, - UpdateBuilder, Version, WhenMatched, WhenNotMatched, WhenNotMatchedBySource, WriteMode, - WriteParams, + MergeInsertBuilder as LanceMergeInsertBuilder, MergeInsertWriteMode, ReadParams, + UncommittedMergeInsert, UpdateBuilder, Version, VersionRef, WhenMatched, WhenNotMatched, + WhenNotMatchedBySource, WriteMode, WriteParams, fragment::FileFragment as LanceFileFragment, progress::WriteFragmentProgress, scanner::Scanner as LanceScanner, @@ -61,7 +61,8 @@ use lance::dataset::{ }; use lance::index::vector::utils::get_vector_type; use lance::index::{ - DatasetIndexExt, DatasetIndexInternalExt, IndexSegment, vector::VectorIndexParams, + DatasetIndexExt, DatasetIndexInternalExt, IndexSegment, IntoIndexSegment, + vector::VectorIndexParams, }; use lance::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion}; use lance_arrow::as_fixed_size_list_array; @@ -78,7 +79,7 @@ use lance_index::{ FtsPrewarmOptions, IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions, progress::{IndexBuildProgress, NoopIndexBuildProgress}, - scalar::inverted::InvertedListFormatVersion, + scalar::inverted::{DocumentGranularity, InvertedListFormatVersion}, scalar::{FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams}, vector::{ ApproxMode, DEFAULT_QUERY_PARALLELISM, Query as VectorQuery, @@ -97,6 +98,7 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; use crate::error::PythonErrorExt; use crate::file::object_store_from_uri_or_path; use crate::fragment::FileFragment; +use crate::fts::FtsTokenizerOptions; use crate::indices::{PyIndexConfig, PyIndexDescription, PyIndexSegment}; use crate::namespace::extract_namespace_arc; use crate::rt; @@ -125,13 +127,44 @@ const DEFAULT_NPROBES: usize = 1; const LANCE_COMMIT_MESSAGE_KEY: &str = "__lance_commit_message"; const INDEX_PROGRESS_QUEUE_SIZE: usize = 1024; -fn read_blobs_to_python( - py: Python<'_>, - blobs: Vec, -) -> Vec<(u64, Py)> { +type PyBlobBytes = Option>; +type PyReadBlob = (u64, PyBlobBytes); +type PyReadBlobRange = (usize, u64, PyBlobBytes); + +fn read_blobs_to_python(py: Python<'_>, blobs: Vec) -> Vec { blobs .into_iter() - .map(|blob| (blob.row_address, PyBytes::new(py, &blob.data).unbind())) + .map(|blob| { + ( + blob.row_address, + blob.data.map(|data| PyBytes::new(py, &data).unbind()), + ) + }) + .collect() +} + +fn read_blob_ranges_to_python( + py: Python<'_>, + ranges: Vec, +) -> Vec { + ranges + .into_iter() + .map(|range| { + ( + range.request_index, + range.row_address, + range.data.map(|data| PyBytes::new(py, &data).unbind()), + ) + }) + .collect() +} + +fn blob_range_requests_from_tuples( + requests: Vec<(u64, u64, u64)>, +) -> Vec { + requests + .into_iter() + .map(|(row, offset, length)| lance::dataset::BlobRangeRequest::new(row, offset, length)) .collect() } @@ -149,6 +182,20 @@ fn configure_read_blobs_builder( builder } +fn configure_read_blob_ranges_builder( + mut builder: lance::dataset::ReadBlobRangesBuilder, + io_buffer_size: Option, + preserve_order: Option, +) -> lance::dataset::ReadBlobRangesBuilder { + if let Some(bytes) = io_buffer_size { + builder = builder.with_io_buffer_size_bytes(bytes); + } + if let Some(preserve) = preserve_order { + builder = builder.preserve_order(preserve); + } + builder +} + fn stats_log_interval_from_millis(ms: u64) -> Option { if ms == 0 { None @@ -164,7 +211,6 @@ fn stats_log_interval_from_millis(ms: u64) -> Option { #[allow(clippy::too_many_arguments)] fn writer_config_from_kwargs( durable_write: Option, - sync_indexed_write: Option, max_wal_buffer_size: Option, max_wal_flush_interval_ms: Option, max_memtable_size: Option, @@ -172,8 +218,6 @@ fn writer_config_from_kwargs( max_memtable_batches: Option, max_unflushed_memtable_bytes: Option, manifest_scan_batch_size: Option, - async_index_buffer_rows: Option, - async_index_interval_ms: Option, backpressure_log_interval_ms: Option, stats_log_interval_ms: Option, hnsw_params: Option>>, @@ -186,10 +230,6 @@ fn writer_config_from_kwargs( config = config.with_durable_write(v); any = true; } - if let Some(v) = sync_indexed_write { - config = config.with_sync_indexed_write(v); - any = true; - } if let Some(v) = max_wal_buffer_size { config = config.with_max_wal_buffer_size(v); any = true; @@ -218,14 +258,6 @@ fn writer_config_from_kwargs( config = config.with_manifest_scan_batch_size(v); any = true; } - if let Some(v) = async_index_buffer_rows { - config = config.with_async_index_buffer_rows(v); - any = true; - } - if let Some(v) = async_index_interval_ms { - config = config.with_async_index_interval(Duration::from_millis(v)); - any = true; - } if let Some(v) = backpressure_log_interval_ms { config = config.with_backpressure_log_interval(Duration::from_millis(v)); any = true; @@ -396,6 +428,22 @@ impl MergeInsertBuilder { Ok(slf) } + pub fn write_mode<'a>(mut slf: PyRefMut<'a, Self>, mode: &str) -> PyResult> { + let mode = match mode { + "auto" => MergeInsertWriteMode::Auto, + "rewrite_rows" => MergeInsertWriteMode::RewriteRows, + "rewrite_columns" => MergeInsertWriteMode::RewriteColumns, + other => { + return Err(PyValueError::new_err(format!( + "Invalid write_mode: {other}. Expected one of \ + 'auto', 'rewrite_rows', 'rewrite_columns'" + ))); + } + }; + slf.builder.write_mode(mode); + Ok(slf) + } + pub fn target_bases( mut slf: PyRefMut<'_, Self>, bases: Vec, @@ -456,6 +504,60 @@ impl MergeInsertBuilder { Ok((PyLance(transaction), stats)) } + /// Execute the merge insert from fully-materialized data. + /// + /// The data is read into memory and wrapped in an in-memory table, so retries + /// never spill to disk and the source's statistics drive the join. Callers + /// should only route in-memory inputs (e.g. a `pa.Table`) here. + pub fn execute_batches(&mut self, new_data: &Bound) -> PyResult> { + let py = new_data.py(); + let reader = convert_reader(new_data)?; + let batches = reader + .collect::, _>>() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let job = self + .builder + .try_build() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let (new_dataset, stats) = rt() + .spawn(Some(py), job.execute_batches(batches))? + .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + + let dataset = self.dataset.bind(py); + dataset.borrow_mut().ds = new_dataset; + + Ok(Self::build_stats(&stats, py)?.into()) + } + + /// [`Self::execute_batches`] without committing; returns the transaction. + pub fn execute_uncommitted_batches<'a>( + &mut self, + new_data: &Bound<'a, PyAny>, + ) -> PyResult<(PyLance, Bound<'a, PyDict>)> { + let py = new_data.py(); + let reader = convert_reader(new_data)?; + let batches = reader + .collect::, _>>() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let job = self + .builder + .try_build() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + let UncommittedMergeInsert { + transaction, stats, .. + } = rt() + .spawn(Some(py), job.execute_uncommitted_batches(batches))? + .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + + let stats = Self::build_stats(&stats, py)?; + + Ok((PyLance(transaction), stats)) + } + #[pyo3(signature=(schema = None, verbose = false))] pub fn explain_plan( &mut self, @@ -487,46 +589,46 @@ impl MergeInsertBuilder { .map_err(|err| PyIOError::new_err(err.to_string())) } - /// Mark MemWAL generations as merged into the base table. + /// [`Self::analyze_plan`] for fully-materialized data. /// - /// Call this when executing a merge_insert that incorporates MemWAL - /// flushed generation data. This updates the MemWAL generation tracking - /// to prevent duplicate merges. - pub fn mark_generations_as_merged<'a>( + /// Routed to the same in-memory table `execute_batches` uses, so the reported + /// plan is the one such a source actually runs. + pub fn analyze_plan_batches(&mut self, new_data: &Bound) -> PyResult { + let reader = convert_reader(new_data)?; + let batches = reader + .collect::, _>>() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + let job = self + .builder + .clone() + .try_build() + .map_err(|err| PyValueError::new_err(err.to_string()))?; + + rt().block_on(None, job.analyze_plan_batches(batches))? + .map_err(|err| PyIOError::new_err(err.to_string())) + } + + /// Mark MemWAL SSTables as compacted into the base table. + /// + /// Call this when executing a merge_insert that compacts MemWAL SSTables. + /// This updates MemWAL compaction progress to prevent duplicate compactions. + pub fn mark_sstables_as_compacted<'a>( mut slf: PyRefMut<'a, Self>, - generations: Vec>, + sstables: Vec>, ) -> PyResult> { - use lance_index::mem_wal::MergedGeneration; + use lance_index::mem_wal::CompactedSsTable; - let gens: Vec = generations + let compacted_sstables: Vec = sstables .iter() - .map(|g| g.borrow().to_lance()) + .map(|sstable| sstable.borrow().to_lance()) .collect::>()?; - slf.builder.mark_generations_as_merged(gens); + slf.builder.mark_sstables_as_compacted(compacted_sstables); Ok(slf) } } fn index_metadata_to_segment(metadata: IndexMetadata) -> PyResult { - let fragment_bitmap = metadata.fragment_bitmap.ok_or_else(|| { - PyValueError::new_err(format!( - "Index metadata {} is missing fragment coverage", - metadata.uuid - )) - })?; - let index_details = metadata.index_details.ok_or_else(|| { - PyValueError::new_err(format!( - "Index metadata {} is missing index details", - metadata.uuid - )) - })?; - - Ok(IndexSegment::new( - metadata.uuid, - fragment_bitmap.iter(), - index_details, - metadata.index_version, - )) + metadata.into_index_segment().infer_error() } fn extract_index_segments(segments: &Bound<'_, PyAny>) -> PyResult> { @@ -549,6 +651,23 @@ fn extract_index_segments(segments: &Bound<'_, PyAny>) -> PyResult>) -> PyResult>> { + index_segments + .map(|segments| { + segments + .into_iter() + .map(|segment| { + Uuid::parse_str(&segment).map_err(|err| { + PyValueError::new_err(format!( + "invalid index segment uuid '{segment}': {err}" + )) + }) + }) + .collect::>>() + }) + .transpose() +} + impl MergeInsertBuilder { fn build_stats<'a>(stats: &MergeStats, py: Python<'a>) -> PyResult> { let dict = PyDict::new(py); @@ -712,6 +831,7 @@ impl Dataset { delete_unverified: Option, error_if_tagged_old_versions: Option, delete_rate_limit: Option, + versions: Option>, ) -> lance_core::Result { let mut builder = CleanupPolicyBuilder::default(); if let Some(v) = older_than_micros { @@ -730,6 +850,9 @@ impl Dataset { if let Some(v) = delete_rate_limit { builder = builder.delete_rate_limit(v)?; } + if let Some(v) = versions { + builder = builder.versions(v)?; + } Ok(builder.build()) } } @@ -1019,7 +1142,13 @@ impl Dataset { #[getter(data_storage_version)] fn data_storage_version(&self) -> PyResult { - Ok(self.ds.manifest().data_storage_format.version.clone()) + Ok(self + .ds + .manifest() + .data_storage_format + .version + .to_manifest_string() + .to_string()) } #[getter(has_stable_row_ids)] @@ -1088,7 +1217,7 @@ impl Dataset { } #[allow(clippy::too_many_arguments)] - #[pyo3(signature=(columns=None, columns_with_transform=None, filter=None, search_filter=None, prefilter=None, limit=None, offset=None, nearest=None, batch_size=None, batch_size_bytes=None, io_buffer_size=None, batch_readahead=None, fragment_readahead=None, scan_in_order=None, fragments=None, index_segments=None, with_row_id=None, with_row_address=None, use_stats=None, substrait_filter=None, fast_search=None, full_text_query=None, late_materialization=None, blob_handling=None, use_scalar_index=None, include_deleted_rows=None, scan_stats_callback=None, strict_batch_size=None, order_by=None, disable_scoring_autoprojection=None, substrait_aggregate=None))] + #[pyo3(signature=(columns=None, columns_with_transform=None, filter=None, search_filter=None, prefilter=None, limit=None, offset=None, nearest=None, batch_size=None, batch_size_bytes=None, io_buffer_size=None, batch_readahead=None, fragment_readahead=None, scan_in_order=None, fragments=None, index_segments=None, with_row_id=None, with_row_address=None, use_stats=None, substrait_filter=None, fast_search=None, full_text_query=None, late_materialization=None, blob_handling=None, use_scalar_index=None, include_deleted_rows=None, scan_stats_callback=None, strict_batch_size=None, order_by=None, disable_scoring_autoprojection=None, substrait_aggregate=None, row_addr_allowlist=None, row_addr_blocklist=None))] fn scanner( self_: PyRef<'_, Self>, columns: Option>, @@ -1122,6 +1251,8 @@ impl Dataset { order_by: Option>>, disable_scoring_autoprojection: Option, substrait_aggregate: Option>, + row_addr_allowlist: Option>, + row_addr_blocklist: Option>, ) -> PyResult { let mut scanner: LanceScanner = self_.ds.scan(); @@ -1194,6 +1325,13 @@ impl Dataset { let is_phrase = query.len() >= 2 && query.starts_with('"') && query.ends_with('"'); let is_multi_match = columns.as_ref().map(|cols| cols.len() > 1).unwrap_or(false); + let document_granularity = match full_text_query.get_item("document_granularity")? { + Some(value) if !value.is_none() => Some( + DocumentGranularity::try_from(value.extract::()?.as_str()) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ), + _ => None, + }; if is_phrase { // Remove the surrounding quotes for phrase queries @@ -1201,8 +1339,20 @@ impl Dataset { } let query: FtsQuery = match (is_phrase, is_multi_match) { - (false, _) => MatchQuery::new(query).into(), - (true, false) => PhraseQuery::new(query).into(), + (false, _) => { + let mut query = MatchQuery::new(query); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } + query.into() + } + (true, false) => { + let mut query = PhraseQuery::new(query); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity(document_granularity); + } + query.into() + } (true, true) => { return Err(PyValueError::new_err( "Phrase queries cannot be used with multiple columns.", @@ -1238,6 +1388,18 @@ impl Dataset { if let Some(prefilter) = prefilter { scanner.prefilter(prefilter); } + // Serialized RowAddrTreeMap payloads rather than an object: a mask built by + // another extension module cannot hand over a Rust value, but both sides + // agree on this encoding. RowAddrMask::from_serialized_parts is the shared + // entry point, so no binding has to reimplement the allow/block combination. + if let Some(mask) = RowAddrMask::from_serialized_parts( + row_addr_allowlist.as_deref(), + row_addr_blocklist.as_deref(), + ) + .infer_error()? + { + scanner.with_row_addr_prefilter(mask); + } scanner .limit(limit, offset) @@ -1523,18 +1685,21 @@ impl Dataset { self_: PyRef<'_, Self>, row_ids: Vec, blob_column: &str, - ) -> PyResult> { + ) -> PyResult>> { let blobs = rt() .block_on(Some(self_.py()), self_.ds.take_blobs(&row_ids, blob_column))? .infer_error()?; - Ok(blobs.into_iter().map(LanceBlobFile::from).collect()) + Ok(blobs + .into_iter() + .map(|blob| blob.map(LanceBlobFile::from)) + .collect()) } fn take_blobs_by_addresses( self_: PyRef<'_, Self>, row_addresses: Vec, blob_column: &str, - ) -> PyResult> { + ) -> PyResult>> { let blobs = rt() .block_on( Some(self_.py()), @@ -1543,21 +1708,27 @@ impl Dataset { .take_blobs_by_addresses(&row_addresses, blob_column), )? .infer_error()?; - Ok(blobs.into_iter().map(LanceBlobFile::from).collect()) + Ok(blobs + .into_iter() + .map(|blob| blob.map(LanceBlobFile::from)) + .collect()) } fn take_blobs_by_indices( self_: PyRef<'_, Self>, row_indices: Vec, blob_column: &str, - ) -> PyResult> { + ) -> PyResult>> { let blobs = rt() .block_on( Some(self_.py()), self_.ds.take_blobs_by_indices(&row_indices, blob_column), )? .infer_error()?; - Ok(blobs.into_iter().map(LanceBlobFile::from).collect()) + Ok(blobs + .into_iter() + .map(|blob| blob.map(LanceBlobFile::from)) + .collect()) } #[pyo3(signature=( @@ -1572,7 +1743,7 @@ impl Dataset { blob_column: &str, io_buffer_size: Option, preserve_order: Option, - ) -> PyResult)>> { + ) -> PyResult> { let builder = configure_read_blobs_builder( self_ .ds @@ -1600,7 +1771,7 @@ impl Dataset { blob_column: &str, io_buffer_size: Option, preserve_order: Option, - ) -> PyResult)>> { + ) -> PyResult> { let builder = configure_read_blobs_builder( self_ .ds @@ -1628,7 +1799,7 @@ impl Dataset { blob_column: &str, io_buffer_size: Option, preserve_order: Option, - ) -> PyResult)>> { + ) -> PyResult> { let builder = configure_read_blobs_builder( self_ .ds @@ -1644,6 +1815,40 @@ impl Dataset { Ok(read_blobs_to_python(self_.py(), blobs)) } + #[pyo3(signature=( + requests, + blob_column, + selector, + io_buffer_size=None, + preserve_order=None + ))] + fn read_blob_ranges( + self_: PyRef<'_, Self>, + requests: Vec<(u64, u64, u64)>, + blob_column: &str, + selector: &str, + io_buffer_size: Option, + preserve_order: Option, + ) -> PyResult> { + let requests = blob_range_requests_from_tuples(requests); + let builder = self_.ds.read_blob_ranges(blob_column).infer_error()?; + let builder = match selector { + "ids" => builder.with_row_ids(requests), + "addresses" => builder.with_row_addresses(requests), + "indices" => builder.with_row_indices(requests), + selector => { + return Err(PyValueError::new_err(format!( + "selector must be one of 'ids', 'addresses', or 'indices', got {selector:?}" + ))); + } + }; + let builder = configure_read_blob_ranges_builder(builder, io_buffer_size, preserve_order); + let ranges = rt() + .block_on(Some(self_.py()), builder.execute())? + .infer_error()?; + Ok(read_blob_ranges_to_python(self_.py(), ranges)) + } + #[pyo3(signature = (row_slices, columns = None, batch_readahead = 10))] fn take_scan( &self, @@ -1881,6 +2086,19 @@ impl Dataset { Ok(pyvers) } + fn version_refs(self_: PyRef<'_, Self>) -> PyResult>> { + let py = self_.py(); + self_ + .list_version_refs()? + .iter() + .map(|version| { + let dict = PyDict::new(py); + dict.set_item("version", version.version)?; + dict.into_py_any(py) + }) + .collect() + } + /// Fetches the currently checked out version of the dataset. fn version(&self) -> PyResult { Ok(self.ds.version().version) @@ -1982,7 +2200,7 @@ impl Dataset { } /// Cleanup old versions from the dataset - #[pyo3(signature = (older_than_micros = None, retain_versions = None, delete_unverified = None, error_if_tagged_old_versions = None, delete_rate_limit = None))] + #[pyo3(signature = (older_than_micros = None, retain_versions = None, delete_unverified = None, error_if_tagged_old_versions = None, delete_rate_limit = None, versions = None))] fn cleanup_old_versions( &self, older_than_micros: Option, @@ -1990,6 +2208,7 @@ impl Dataset { delete_unverified: Option, error_if_tagged_old_versions: Option, delete_rate_limit: Option, + versions: Option>, ) -> PyResult { let stats = rt() .block_on(None, async { @@ -2000,6 +2219,7 @@ impl Dataset { delete_unverified, error_if_tagged_old_versions, delete_rate_limit, + versions, ) .await?; self.ds.cleanup_with_policy(policy).await @@ -2010,7 +2230,7 @@ impl Dataset { /// Explain cleanup old versions from the dataset without deleting files #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (older_than_micros = None, retain_versions = None, delete_unverified = None, error_if_tagged_old_versions = None, delete_rate_limit = None, include_files = false, max_files = 1000))] + #[pyo3(signature = (older_than_micros = None, retain_versions = None, delete_unverified = None, error_if_tagged_old_versions = None, delete_rate_limit = None, versions = None, include_files = false, max_files = 1000))] fn explain_cleanup_old_versions( &self, older_than_micros: Option, @@ -2018,6 +2238,7 @@ impl Dataset { delete_unverified: Option, error_if_tagged_old_versions: Option, delete_rate_limit: Option, + versions: Option>, include_files: bool, max_files: usize, ) -> PyResult { @@ -2030,6 +2251,7 @@ impl Dataset { delete_unverified, error_if_tagged_old_versions, delete_rate_limit, + versions, ) .await?; self.ds @@ -2280,6 +2502,9 @@ impl Dataset { if let Some(num_indices_to_merge) = kwargs.get_item("num_indices_to_merge")? { options.num_indices_to_merge = num_indices_to_merge.extract()?; } + if let Some(retrain) = kwargs.get_item("retrain")? { + options.retrain = retrain.extract()?; + } if let Some(index_names) = kwargs.get_item("index_names")? { options.index_names = Some( index_names @@ -2404,48 +2629,58 @@ impl Dataset { "INVERTED" | "FTS" => { let mut params = InvertedIndexParams::default(); if let Some(kwargs) = kwargs { - if let Some(with_position) = kwargs.get_item("with_position")? { - params = params.with_position(with_position.extract()?); - } - if let Some(base_tokenizer) = kwargs.get_item("base_tokenizer")? { - params = params.base_tokenizer(base_tokenizer.extract()?); - } - if let Some(language) = kwargs.get_item("language")? { - let language: PyBackedStr = - language.cast::()?.clone().try_into()?; - params = params.language(&language).map_err(|e| { - PyValueError::new_err(format!( - "can't set tokenizer language to {}: {:?}", - language, e - )) - })?; - } - if let Some(max_token_length) = kwargs.get_item("max_token_length")? { - params = params.max_token_length(max_token_length.extract()?); - } - if let Some(lower_case) = kwargs.get_item("lower_case")? { - params = params.lower_case(lower_case.extract()?); - } - if let Some(stem) = kwargs.get_item("stem")? { - params = params.stem(stem.extract()?); - } - if let Some(remove_stop_words) = kwargs.get_item("remove_stop_words")? { - params = params.remove_stop_words(remove_stop_words.extract()?); - } - if let Some(stop_words_file) = kwargs.get_item("custom_stop_words")? { - params = params.custom_stop_words(stop_words_file.extract()?); - } - if let Some(ascii_folding) = kwargs.get_item("ascii_folding")? { - params = params.ascii_folding(ascii_folding.extract()?); + let allowed_kwargs = [ + "analyzer", + "document_granularity", + "with_position", + "base_tokenizer", + "language", + "max_token_length", + "lower_case", + "stem", + "remove_stop_words", + "custom_stop_words", + "ascii_folding", + "min_ngram_length", + "max_ngram_length", + "prefix_only", + "block_size", + "split_identifiers", + "split_on_numerics", + "preserve_original", + "index_operators", + "memory_limit", + "num_workers", + "format_version", + "fragment_ids", + "index_uuid", + "progress_callback", + ]; + for (key, _) in kwargs.iter() { + let key: String = key.extract()?; + if !allowed_kwargs.contains(&key.as_str()) { + return Err(PyValueError::new_err(format!( + "unknown FTS index parameter '{}'", + key + ))); + } } - if let Some(min_ngram_length) = kwargs.get_item("min_ngram_length")? { - params = params.ngram_min_length(min_ngram_length.extract()?); + + params = FtsTokenizerOptions::from_kwargs(kwargs)?.apply(params)?; + if let Some(document_granularity) = kwargs.get_item("document_granularity")? { + let document_granularity: String = document_granularity.extract()?; + params = params.document_granularity( + DocumentGranularity::try_from(document_granularity.as_str()) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ); } - if let Some(max_ngram_length) = kwargs.get_item("max_ngram_length")? { - params = params.ngram_max_length(max_ngram_length.extract()?); + if let Some(with_position) = kwargs.get_item("with_position")? { + params = params.with_position(with_position.extract()?); } - if let Some(prefix_only) = kwargs.get_item("prefix_only")? { - params = params.ngram_prefix_only(prefix_only.extract()?); + if let Some(block_size) = kwargs.get_item("block_size")? { + params = params + .block_size(block_size.extract()?) + .map_err(|e| PyValueError::new_err(e.to_string()))?; } if let Some(memory_limit) = kwargs.get_item("memory_limit")? { params = params.memory_limit_mb(memory_limit.extract()?); @@ -2462,7 +2697,7 @@ impl Dataset { value.to_string() } else { return Err(PyValueError::new_err( - "format_version must be 1, 2, 'v1', or 'v2'", + "format_version must be 1, 2, 3, 'v1', 'v2', or 'v3'", )); }; let format_version = value @@ -2600,18 +2835,31 @@ impl Dataset { Ok(()) } - #[pyo3(signature = (name, *, with_position = false))] - fn prewarm_index(&self, name: &str, with_position: bool) -> PyResult<()> { + #[pyo3(signature = (name, *, with_position = false, index_segments = None))] + fn prewarm_index( + &self, + name: &str, + with_position: bool, + index_segments: Option>, + ) -> PyResult<()> { + let index_segments = parse_index_segment_ids(index_segments)?; + rt().block_on(None, async { if with_position { - self.ds - .prewarm_index_with_options( - name, - &PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)), - ) - .await + let options = PrewarmOptions::Fts(FtsPrewarmOptions::new().with_position(true)); + if let Some(index_segments) = index_segments.as_deref() { + self.ds + .prewarm_index_segments_with_options(name, index_segments, &options) + .await + } else { + self.ds.prewarm_index_with_options(name, &options).await + } } else { - self.ds.prewarm_index(name).await + if let Some(index_segments) = index_segments.as_deref() { + self.ds.prewarm_index_segments(name, index_segments).await + } else { + self.ds.prewarm_index(name).await + } } })? .infer_error() @@ -2724,6 +2972,9 @@ impl Dataset { rt().spawn(None, async move { let (object_store, path) = object_store_from_uri_or_path(&dest, storage_options).await?; + validate_dataset_root_for_drop(&object_store, &path) + .await + .map_err(|e| PyValueError::new_err(e.to_string()))?; let result = object_store.remove_dir_all(path).await; match result { @@ -2990,7 +3241,7 @@ impl Dataset { new_self.add_columns(transforms, None, batch_size).await?; Ok(new_self) })? - .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + .io_or_commit_conflict_error()?; self.ds = Arc::new(new_self); Ok(()) @@ -3014,7 +3265,7 @@ impl Dataset { .await?; Ok(new_self) })? - .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + .io_or_commit_conflict_error()?; self.ds = Arc::new(new_self); Ok(()) @@ -3032,7 +3283,7 @@ impl Dataset { new_self.add_columns(transform, None, None).await?; Ok(new_self) })? - .map_err(|err: lance::Error| PyIOError::new_err(err.to_string()))?; + .io_or_commit_conflict_error()?; self.ds = Arc::new(new_self); Ok(()) } @@ -3379,7 +3630,6 @@ impl Dataset { identity_column=None, unsharded=false, durable_write=None, - sync_indexed_write=None, max_wal_buffer_size=None, max_wal_flush_interval_ms=None, max_memtable_size=None, @@ -3387,8 +3637,6 @@ impl Dataset { max_memtable_batches=None, max_unflushed_memtable_bytes=None, manifest_scan_batch_size=None, - async_index_buffer_rows=None, - async_index_interval_ms=None, backpressure_log_interval_ms=None, stats_log_interval_ms=None, hnsw_params=None, @@ -3402,7 +3650,6 @@ impl Dataset { identity_column: Option, unsharded: bool, durable_write: Option, - sync_indexed_write: Option, max_wal_buffer_size: Option, max_wal_flush_interval_ms: Option, max_memtable_size: Option, @@ -3410,8 +3657,6 @@ impl Dataset { max_memtable_batches: Option, max_unflushed_memtable_bytes: Option, manifest_scan_batch_size: Option, - async_index_buffer_rows: Option, - async_index_interval_ms: Option, backpressure_log_interval_ms: Option, stats_log_interval_ms: Option, hnsw_params: Option>>, @@ -3439,7 +3684,6 @@ impl Dataset { let writer_config = writer_config_from_kwargs( durable_write, - sync_indexed_write, max_wal_buffer_size, max_wal_flush_interval_ms, max_memtable_size, @@ -3447,8 +3691,6 @@ impl Dataset { max_memtable_batches, max_unflushed_memtable_bytes, manifest_scan_batch_size, - async_index_buffer_rows, - async_index_interval_ms, backpressure_log_interval_ms, stats_log_interval_ms, hnsw_params, @@ -3530,7 +3772,6 @@ impl Dataset { shard_id, *, durable_write=None, - sync_indexed_write=None, max_wal_buffer_size=None, max_wal_flush_interval_ms=None, max_memtable_size=None, @@ -3538,8 +3779,6 @@ impl Dataset { max_memtable_batches=None, max_unflushed_memtable_bytes=None, manifest_scan_batch_size=None, - async_index_buffer_rows=None, - async_index_interval_ms=None, backpressure_log_interval_ms=None, stats_log_interval_ms=None, hnsw_params=None, @@ -3549,7 +3788,6 @@ impl Dataset { py: Python<'_>, shard_id: String, durable_write: Option, - sync_indexed_write: Option, max_wal_buffer_size: Option, max_wal_flush_interval_ms: Option, max_memtable_size: Option, @@ -3557,8 +3795,6 @@ impl Dataset { max_memtable_batches: Option, max_unflushed_memtable_bytes: Option, manifest_scan_batch_size: Option, - async_index_buffer_rows: Option, - async_index_interval_ms: Option, backpressure_log_interval_ms: Option, stats_log_interval_ms: Option, hnsw_params: Option>>, @@ -3570,7 +3806,6 @@ impl Dataset { let config = writer_config_from_kwargs( durable_write, - sync_indexed_write, max_wal_buffer_size, max_wal_flush_interval_ms, max_memtable_size, @@ -3578,8 +3813,6 @@ impl Dataset { max_memtable_batches, max_unflushed_memtable_bytes, manifest_scan_batch_size, - async_index_buffer_rows, - async_index_interval_ms, backpressure_log_interval_ms, stats_log_interval_ms, hnsw_params, @@ -3603,9 +3836,10 @@ impl Dataset { /// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index. /// - /// This function loads a specific partition from an IVF_FLAT index on a hash column, - /// computes pairwise hamming distances between all hashes in the partition, - /// filters by threshold, and clusters the results using union-find. + /// This function loads a specific partition from every segment of an IVF_FLAT + /// index on a hash column, computes pairwise hamming distances between all + /// hashes in the combined partition, filters by threshold, and clusters the + /// results using union-find. /// /// Parameters /// ---------- @@ -3615,6 +3849,9 @@ impl Dataset { /// The partition ID within the IVF_FLAT index /// hamming_threshold : int /// Maximum hamming distance to consider as similar + /// index_segments : list of str, optional + /// If specified, only these physical index segment UUIDs of the named + /// logical index contribute rows. Defaults to all segments. /// /// Returns /// ------- @@ -3622,27 +3859,45 @@ impl Dataset { /// A reader yielding batches with columns: /// - 'representative': uint64 - The representative row ID for each cluster /// - 'duplicates': list - List of duplicate row IDs in each cluster - #[pyo3(signature = (index_name, partition_id, hamming_threshold))] + #[pyo3(signature = (index_name, partition_id, hamming_threshold, index_segments=None))] fn hamming_clustering_for_ivf_partition( &self, py: Python<'_>, index_name: &str, partition_id: usize, hamming_threshold: u32, + index_segments: Option>, ) -> PyResult>> { - use lance::index::vector::hamming::hamming_clustering_for_ivf_partition; + use lance::index::vector::hamming::{ + hamming_clustering_for_ivf_partition, hamming_clustering_for_ivf_partition_segments, + }; + let segment_ids = parse_index_segment_ids(index_segments)?; let ds = self.ds.as_ref(); let reader = rt() - .block_on( - Some(py), - hamming_clustering_for_ivf_partition( - ds, - index_name, - partition_id, - hamming_threshold, - ), - )? + .block_on(Some(py), async { + match segment_ids.as_deref() { + Some(segment_ids) => { + hamming_clustering_for_ivf_partition_segments( + ds, + index_name, + segment_ids, + partition_id, + hamming_threshold, + ) + .await + } + None => { + hamming_clustering_for_ivf_partition( + ds, + index_name, + partition_id, + hamming_threshold, + ) + .await + } + } + })? .map_err(|err| PyValueError::new_err(err.to_string()))?; Ok(PyArrowType(reader)) @@ -3650,26 +3905,43 @@ impl Dataset { /// Get partition information for an IVF_FLAT index. /// + /// Partition sizes are aggregated across all segments of the logical index + /// unless a subset is selected via ``index_segments``. + /// /// Parameters /// ---------- /// index_name : str /// Name of the IVF_FLAT index + /// index_segments : list of str, optional + /// If specified, only these physical index segment UUIDs of the named + /// logical index contribute to the sizes. Defaults to all segments. /// /// Returns /// ------- /// List[dict] /// List of partition info dicts with 'partition_id' and 'size' - #[pyo3(signature = (index_name))] + #[pyo3(signature = (index_name, index_segments=None))] fn get_ivf_partition_info( &self, py: Python<'_>, index_name: &str, + index_segments: Option>, ) -> PyResult>> { - use lance::index::vector::hamming::get_ivf_partition_info; + use lance::index::vector::hamming::{ + get_ivf_partition_info, get_ivf_partition_info_segments, + }; + let segment_ids = parse_index_segment_ids(index_segments)?; let ds = self.ds.as_ref(); let result = rt() - .block_on(Some(py), get_ivf_partition_info(ds, index_name))? + .block_on(Some(py), async { + match segment_ids.as_deref() { + Some(segment_ids) => { + get_ivf_partition_info_segments(ds, index_name, segment_ids).await + } + None => get_ivf_partition_info(ds, index_name).await, + } + })? .map_err(|err| PyValueError::new_err(err.to_string()))?; let partitions: PyResult> = result @@ -3694,7 +3966,8 @@ impl Dataset { /// Parameters /// ---------- /// column : str - /// Name of the hash column (must be FixedSizeList) + /// Name of the hash column (must be FixedSizeList where N is + /// a positive multiple of 8 bytes) /// sample_size : int, optional /// Number of rows to sample (if None or >= total rows, uses all rows) /// hamming_threshold : int @@ -3737,7 +4010,8 @@ impl Dataset { /// Parameters /// ---------- /// column : str - /// Name of the hash column (must be FixedSizeList) + /// Name of the hash column (must be FixedSizeList where N is + /// a positive multiple of 8 bytes) /// fragment_id : int /// The fragment ID to read from /// start_row : int @@ -3878,6 +4152,37 @@ impl SqlQueryBuilder { } } + #[pyo3(signature = (blob_handling))] + fn blob_handling(&self, blob_handling: &str) -> PyResult { + let blob_handling = match blob_handling { + "all_binary" => BlobHandling::AllBinary, + "blobs_descriptions" => BlobHandling::BlobsDescriptions, + "all_descriptions" => BlobHandling::AllDescriptions, + other => { + return Err(PyValueError::new_err(format!( + "Invalid blob_handling: {other}. Expected one of: all_binary, blobs_descriptions, all_descriptions" + ))); + } + }; + Ok(Self { + builder: self.builder.clone().blob_handling(blob_handling), + }) + } + + #[pyo3(signature = (batch_size))] + fn batch_size(&self, batch_size: usize) -> Self { + Self { + builder: self.builder.clone().batch_size(batch_size), + } + } + + #[pyo3(signature = (batch_size_bytes))] + fn batch_size_bytes(&self, batch_size_bytes: u64) -> Self { + Self { + builder: self.builder.clone().batch_size_bytes(batch_size_bytes), + } + } + /// Build the SQL query. fn build(&self) -> PyResult { Ok(SqlQuery { @@ -3926,6 +4231,18 @@ impl DatasetDelta { let reader: Box = Box::new(LanceReader::from_stream(stream)); reader.into_pyarrow(py) } + /// Get the row ids deleted between begin_version (exclusive) and end_version (inclusive) as a stream reader. + /// + /// Requires stable row ids on the dataset. + fn get_deleted_row_ids<'py>(&self, py: Python<'py>) -> PyResult> { + use arrow::pyarrow::IntoPyArrow; + use arrow_array::RecordBatchReader; + let stream = rt() + .block_on(None, self.inner.get_deleted_row_ids())? + .infer_error()?; + let reader: Box = Box::new(LanceReader::from_stream(stream)); + reader.into_pyarrow(py) + } } #[pyclass( @@ -4239,6 +4556,10 @@ impl Dataset { rt().block_on(None, self.ds.versions())?.infer_error() } + fn list_version_refs(&self) -> PyResult> { + rt().block_on(None, self.ds.version_refs())?.infer_error() + } + fn list_tags(&self) -> PyResult> { rt().block_on(None, self.ds.tags().list())?.infer_error() } @@ -4339,6 +4660,21 @@ impl Dataset { } } +/// Serialize row addresses into the payload the scanner's `row_addr_allowlist` / +/// `row_addr_blocklist` parameters accept. +/// +/// Without this those parameters are unusable from Python: they take the roaring +/// `RowAddrTreeMap` encoding, which nothing else exposed here can produce. The +/// result stays plain bytes, so a mask may equally be built by another extension +/// module and handed in. +#[pyfunction(name = "_serialize_row_addrs")] +pub fn serialize_row_addrs(py: Python<'_>, addrs: Vec) -> PyResult> { + let treemap = RowAddrTreeMap::from_iter(addrs); + let mut buf = Vec::with_capacity(treemap.serialized_size()); + treemap.serialize_into(&mut buf).infer_error()?; + Ok(PyBytes::new(py, &buf).unbind()) +} + #[pyfunction(name = "_write_dataset")] pub fn write_dataset( reader: &Bound<'_, PyAny>, @@ -4444,6 +4780,9 @@ pub fn get_write_params( if let Some(progress) = get_dict_opt::>(options, "progress")? { p.progress = Arc::new(PyWriteProgress::new(progress.into_py_any(options.py())?)); } + if let Some(session) = get_dict_opt::(options, "session")? { + p.session = Some(session.inner.clone()); + } let storage_options = get_dict_opt::>(options, "storage_options")?; @@ -5005,7 +5344,8 @@ pub struct PyFullTextQuery { #[pymethods] impl PyFullTextQuery { #[staticmethod] - #[pyo3(signature = (query, column, boost=1.0, fuzziness=Some(0), max_expansions=50, operator="OR", prefix_length=0))] + #[pyo3(signature = (query, column, boost=1.0, fuzziness=Some(0), max_expansions=50, operator="OR", prefix_length=0, document_granularity=None))] + #[allow(clippy::too_many_arguments)] fn match_query( query: String, column: String, @@ -5014,30 +5354,48 @@ impl PyFullTextQuery { max_expansions: usize, operator: &str, prefix_length: u32, + document_granularity: Option<&str>, ) -> PyResult { + let mut query = MatchQuery::new(query) + .with_column(Some(column)) + .with_boost(boost) + .with_fuzziness(fuzziness) + .with_max_expansions(max_expansions) + .with_operator( + Operator::try_from(operator) + .map_err(|e| PyValueError::new_err(format!("Invalid operator: {}", e)))?, + ) + .with_prefix_length(prefix_length); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity( + DocumentGranularity::try_from(document_granularity) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ); + } Ok(Self { - inner: MatchQuery::new(query) - .with_column(Some(column)) - .with_boost(boost) - .with_fuzziness(fuzziness) - .with_max_expansions(max_expansions) - .with_operator( - Operator::try_from(operator) - .map_err(|e| PyValueError::new_err(format!("Invalid operator: {}", e)))?, - ) - .with_prefix_length(prefix_length) - .into(), + inner: query.into(), }) } #[staticmethod] - #[pyo3(signature = (query, column, slop))] - fn phrase_query(query: String, column: String, slop: u32) -> PyResult { + #[pyo3(signature = (query, column, slop, document_granularity=None))] + fn phrase_query( + query: String, + column: String, + slop: u32, + document_granularity: Option<&str>, + ) -> PyResult { + let mut query = PhraseQuery::new(query) + .with_column(Some(column)) + .with_slop(slop); + if let Some(document_granularity) = document_granularity { + query = query.with_document_granularity( + DocumentGranularity::try_from(document_granularity) + .map_err(|err| PyValueError::new_err(err.to_string()))?, + ); + } Ok(Self { - inner: PhraseQuery::new(query) - .with_column(Some(column)) - .with_slop(slop) - .into(), + inner: query.into(), }) } diff --git a/python/src/dataset/blob.rs b/python/src/dataset/blob.rs index 1f6272075f2..7952ce76fb1 100644 --- a/python/src/dataset/blob.rs +++ b/python/src/dataset/blob.rs @@ -79,6 +79,32 @@ impl LanceBlobFile { Ok(PyBytes::new(py, &data)) } + /// Read multiple blob-local `(offset, length)` ranges without changing the current cursor. + pub fn read_ranges<'py>( + &self, + py: Python<'py>, + ranges: Vec<(u64, u64)>, + ) -> PyResult>> { + let ranges = ranges + .into_iter() + .enumerate() + .map(|(i, (offset, length))| { + let end = offset.checked_add(length).ok_or_else(|| { + PyValueError::new_err(format!( + "Blob range request {i} offset + length overflowed u64: \ + offset={offset}, length={length}" + )) + })?; + Ok(offset..end) + }) + .collect::>>()?; + let inner = self.inner.clone(); + let data = rt() + .block_on(Some(py), inner.read_ranges(&ranges))? + .infer_error()?; + Ok(data.iter().map(|bytes| PyBytes::new(py, bytes)).collect()) + } + pub fn read_into(&self, dst: Bound<'_, PyByteArray>) -> PyResult { let inner = self.inner.clone(); diff --git a/python/src/dataset/commit.rs b/python/src/dataset/commit.rs index 41a0ef2f69a..bb904f89bf7 100644 --- a/python/src/dataset/commit.rs +++ b/python/src/dataset/commit.rs @@ -23,8 +23,7 @@ use pyo3::{exceptions::PyIOError, prelude::*}; static PY_CONFLICT_ERROR: LazyLock>> = LazyLock::new(|| { Python::attach(|py| { - py.import("lance") - .and_then(|lance| lance.getattr("commit")) + py.import("lance.commit") .and_then(|commit| commit.getattr("CommitConflictError")) .map(|err| err.unbind()) }) diff --git a/python/src/dataset/optimize.rs b/python/src/dataset/optimize.rs index 4bb29246f45..a2c1f973727 100644 --- a/python/src/dataset/optimize.rs +++ b/python/src/dataset/optimize.rs @@ -76,6 +76,16 @@ fn parse_compaction_options( "max_source_fragments" => { opts.max_source_fragments = value.extract()?; } + "max_source_rows" => { + opts.max_source_rows = value.extract()?; + } + "max_source_bytes" => { + opts.max_source_bytes = value.extract()?; + } + "excluded_fragment_ids" => { + opts.excluded_fragment_ids = + value.extract::>>()?.unwrap_or_default(); + } _ => { return Err(PyValueError::new_err(format!( "Invalid compaction option: {}", diff --git a/python/src/error.rs b/python/src/error.rs index fa4264638a8..c1d2235f2f7 100644 --- a/python/src/error.rs +++ b/python/src/error.rs @@ -21,6 +21,37 @@ use pyo3::{ use lance::Error as LanceError; +/// Return the `retryable` flag for a commit-conflict error. +/// +/// Mirrors the Rust conflict contract: `RetryableCommitConflict` and +/// `CommitConflict` (commit-step retries exhausted, safe to retry) are +/// retryable; `IncompatibleTransaction` (a conflict retrying cannot fix) is not. +fn commit_conflict_retryable(err: &LanceError) -> bool { + match err { + LanceError::RetryableCommitConflict { .. } | LanceError::CommitConflict { .. } => true, + LanceError::IncompatibleTransaction { .. } => false, + _ => false, + } +} + +/// Convert a commit-conflict `LanceError` to the Python `CommitConflictError`. +/// +/// The `retryable` flag is carried as a typed attribute on the exception so +/// clients can drive retry loops without string-matching the message. +fn commit_conflict_error(py: Python<'_>, err: &LanceError, retryable: bool) -> PyErr { + let message = err.to_string(); + match PyModule::import(py, "lance.commit") { + Ok(module) => match module.getattr("CommitConflictError") { + Ok(conflict_type) => match conflict_type.call1((message.clone(), retryable)) { + Ok(instance) => PyErr::from_value(instance), + Err(_) => PyIOError::new_err(message), + }, + Err(_) => PyIOError::new_err(message), + }, + Err(_) => PyIOError::new_err(message), + } +} + /// Try to convert a NamespaceError to the corresponding Python exception. /// Returns the appropriate Python exception from lance_namespace.errors module. fn namespace_error_to_pyerr(py: Python<'_>, ns_err: &NamespaceError) -> PyErr { @@ -73,6 +104,11 @@ pub trait PythonErrorExt { /// Used by call sites that historically mapped every `lance::Error` to /// PyIoError but should surface timeouts distinctly. fn io_or_timeout_error(self) -> PyResult; + /// Convert commit conflicts to `CommitConflictError`, otherwise PyIoError. + /// + /// Used by call sites that historically mapped every `lance::Error` to + /// PyIoError but should surface commit conflicts distinctly. + fn io_or_commit_conflict_error(self) -> PyResult; } impl PythonErrorExt for std::result::Result { @@ -87,6 +123,12 @@ impl PythonErrorExt for std::result::Result { LanceError::NotFound { .. } => self.value_error(), LanceError::RefNotFound { .. } => self.value_error(), LanceError::VersionNotFound { .. } => self.value_error(), + LanceError::RetryableCommitConflict { .. } + | LanceError::CommitConflict { .. } + | LanceError::IncompatibleTransaction { .. } => { + let retryable = commit_conflict_retryable(err); + Python::attach(|py| Err(commit_conflict_error(py, err, retryable))) + } LanceError::Namespace { source, .. } => { // Try to downcast to NamespaceError and convert to proper Python exception if let Some(ns_err) = source.downcast_ref::() { @@ -128,6 +170,28 @@ impl PythonErrorExt for std::result::Result { fn io_or_timeout_error(self) -> PyResult { match &self { Err(LanceError::Timeout { .. }) => self.timeout_error(), + Err( + err @ (LanceError::RetryableCommitConflict { .. } + | LanceError::CommitConflict { .. } + | LanceError::IncompatibleTransaction { .. }), + ) => { + let retryable = commit_conflict_retryable(err); + Python::attach(|py| Err(commit_conflict_error(py, err, retryable))) + } + _ => self.io_error(), + } + } + + fn io_or_commit_conflict_error(self) -> PyResult { + match &self { + Err( + err @ (LanceError::RetryableCommitConflict { .. } + | LanceError::CommitConflict { .. } + | LanceError::IncompatibleTransaction { .. }), + ) => { + let retryable = commit_conflict_retryable(err); + Python::attach(|py| Err(commit_conflict_error(py, err, retryable))) + } _ => self.io_error(), } } diff --git a/python/src/executor.rs b/python/src/executor.rs index e2e573a8193..e2c1ca4b383 100644 --- a/python/src/executor.rs +++ b/python/src/executor.rs @@ -19,6 +19,10 @@ use pyo3::{PyResult, Python, exceptions::PyRuntimeError}; pub const SIGNAL_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_millis(100); +fn is_python314_or_later(py: Option>) -> bool { + py.is_some_and(|py| py.version_info() >= (3, 14)) +} + /// A wrapper around tokio runtime. /// /// This is used to spawn tasks in the background and wait synchronously for them @@ -195,6 +199,8 @@ impl BackgroundExecutor { F::Output: Send, P: FnMut() -> PyResult<()>, { + let should_propagate_on_completion = is_python314_or_later(py); + let mut future = std::pin::pin!(future); loop { @@ -236,11 +242,22 @@ impl BackgroundExecutor { }; if let Some(output) = maybe_output { - if let Err(err) = pump() { - log::warn!( - "Ignoring progress callback error after operation completed successfully: {}", - err - ); + // When the index build finishes so fast that no pump cycles + // occurred during execution, pending events sit in the channel + // buffer and get drained after completion. Python ≥ 3.14 changed + // GIL/async scheduling timing such that callback invocations may + // only be visible in this post-completion drain, so we propagate + // errors for 3.14+. For ≤ 3.13 we keep the old tolerant behavior + // to avoid spurious failures when scheduling shifts slightly. + if should_propagate_on_completion { + pump()?; + } else { + if let Err(err) = pump() { + log::warn!( + "Ignoring progress callback error after operation completed successfully: {}", + err + ); + } } return Ok(output); } diff --git a/python/src/file.rs b/python/src/file.rs index 2a3dd09e17f..8b3511496b0 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -26,10 +26,9 @@ use lance_core::utils::path::LancePathExt; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::reader::{ BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics, - ReaderProjection, }; use lance_file::writer::{FileWriter, FileWriterOptions}; -use lance_file::{LanceEncodingsIo, version::LanceFileVersion}; +use lance_file::{LanceEncodingsIo, version::LanceFileVersion, versions as file_versions}; use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, ObjectStoreParams}; use lance_io::{ ReadBatchParams, @@ -230,6 +229,26 @@ impl LanceFileMetadata { } } +/// Summary of a completed Lance file write +#[pyclass(get_all, skip_from_py_object)] +#[derive(Clone, Debug, Serialize)] +pub struct LanceFileWriteSummary { + /// The number of rows written to the file + pub num_rows: u64, + /// The final size of the file, in bytes + pub size_bytes: u64, +} + +#[pymethods] +impl LanceFileWriteSummary { + fn __repr__(&self) -> String { + format!( + "FileWriteSummary(num_rows={}, size_bytes={})", + self.num_rows, self.size_bytes + ) + } +} + #[pyclass] pub struct LanceFileWriter { inner: Arc>>, @@ -275,21 +294,23 @@ impl LanceFileWriter { max_page_bytes: Option, ) -> PyResult { let object_writer = object_store.create(&path).await.infer_error()?; + let version = version + .map(|value| value.parse::()) + .transpose() + .infer_error()? + .unwrap_or_default() + .resolve(); let options = FileWriterOptions { data_cache_bytes, keep_original_array, max_page_bytes, - format_version: version - .map(|v| v.parse::()) - .transpose() - .infer_error()?, - ..Default::default() }; let inner = if let Some(schema) = schema { let lance_schema = lance_core::datatypes::Schema::try_from(&schema.0).infer_error()?; - FileWriter::try_new(object_writer, lance_schema, options).infer_error() + file_versions::create_writer(version, object_writer, lance_schema, options) + .infer_error() } else { - Ok(FileWriter::new_lazy(object_writer, options)) + file_versions::create_lazy_writer(version, object_writer, options).infer_error() }?; Ok(Self { inner: Arc::new(Mutex::new(Box::new(inner))), @@ -347,11 +368,18 @@ impl LanceFileWriter { .infer_error() } - pub fn finish(&self) -> PyResult { - rt().block_on(None, async { - self.inner.lock().await.finish().await.map(|s| s.num_rows) - })? - .infer_error() + /// Finish the file and return the row count and the final file size + /// + /// The size is reported by the object writer once the file has been closed + /// and so it is accurate for object stores as well as local filesystems. + pub fn finish(&self) -> PyResult { + let summary = rt() + .block_on(None, async { self.inner.lock().await.finish().await })? + .infer_error()?; + Ok(LanceFileWriteSummary { + num_rows: summary.num_rows, + size_bytes: summary.size_bytes, + }) } pub fn add_global_buffer(&self, bytes: Vec) -> PyResult { @@ -817,7 +845,7 @@ impl LanceFileReader { let mut base_projection = None; if let Some(columns) = columns { base_projection = Some( - ReaderProjection::from_column_names( + file_versions::reader_projection_from_column_names( file_metadata.version(), &file_metadata.file_schema, &columns.iter().map(|s| s.as_str()).collect::>(), diff --git a/python/src/fragment.rs b/python/src/fragment.rs index dbe5c426903..8729eb1ce0a 100644 --- a/python/src/fragment.rs +++ b/python/src/fragment.rs @@ -21,11 +21,12 @@ use arrow_array::RecordBatchReader; use futures::TryFutureExt; use lance::Error; use lance::dataset::fragment::FileFragment as LanceFragment; -use lance::dataset::scanner::ColumnOrdering; +use lance::dataset::scanner::{ColumnOrdering, MaterializationStyle}; use lance::dataset::transaction::{Operation, Transaction}; use lance::dataset::{InsertBuilder, NewColumnTransform, WriteParams}; use lance_core::datatypes::BlobHandling; use lance_io::utils::CachedFileSize; +use lance_table::format::overlay::DataOverlayFile; use lance_table::format::{ DataFile, DeletionFile, DeletionFileType, Fragment, RowDatasetVersionMeta, RowIdMeta, }; @@ -210,7 +211,7 @@ impl FileFragment { } #[allow(clippy::too_many_arguments)] - #[pyo3(signature=(columns=None, columns_with_transform=None, batch_size=None, filter=None, limit=None, offset=None, with_row_id=None, with_row_address=None, batch_readahead=None, blob_handling=None, order_by=None))] + #[pyo3(signature=(columns=None, columns_with_transform=None, batch_size=None, filter=None, limit=None, offset=None, with_row_id=None, with_row_address=None, batch_readahead=None, blob_handling=None, order_by=None, use_scalar_index=None, io_buffer_size=None, late_materialization=None, include_deleted_rows=None, batch_size_bytes=None, strict_batch_size=None))] fn scanner( self_: PyRef<'_, Self>, columns: Option>, @@ -224,6 +225,12 @@ impl FileFragment { batch_readahead: Option, blob_handling: Option>, order_by: Option>>, + use_scalar_index: Option, + io_buffer_size: Option, + late_materialization: Option>, + include_deleted_rows: Option, + batch_size_bytes: Option, + strict_batch_size: Option, ) -> PyResult { let mut scanner = self_.fragment.scan(); @@ -292,6 +299,39 @@ impl FileFragment { .order_by(col_orderings) .map_err(|err| PyValueError::new_err(err.to_string()))?; } + if let Some(io_buffer_size) = io_buffer_size { + scanner.io_buffer_size(io_buffer_size); + } + if let Some(use_scalar_index) = use_scalar_index { + scanner.use_scalar_index(use_scalar_index); + } + if let Some(late_materialization) = late_materialization { + if let Ok(style_as_bool) = late_materialization.extract::() { + if style_as_bool { + scanner.materialization_style(MaterializationStyle::AllLate); + } else { + scanner.materialization_style(MaterializationStyle::AllEarly); + } + } else if let Ok(columns) = late_materialization.extract::>() { + scanner.materialization_style( + MaterializationStyle::all_early_except(&columns, self_.fragment.schema()) + .infer_error()?, + ); + } else { + return Err(PyValueError::new_err( + "late_materialization must be a bool or a list of strings", + )); + } + } + if let Some(batch_size_bytes) = batch_size_bytes { + scanner.batch_size_bytes(batch_size_bytes); + } + if let Some(true) = include_deleted_rows { + scanner.include_deleted_rows(); + } + if let Some(strict_batch_size) = strict_batch_size { + scanner.strict_batch_size(strict_batch_size); + } let scn = Arc::new(scanner); Ok(Scanner::new(scn)) } @@ -452,6 +492,11 @@ impl FileFragment { rt().block_on(None, self.fragment.physical_rows())? .map_err(|err| PyIOError::new_err(err.to_string())) } + + fn validate(&self) -> PyResult<()> { + rt().block_on(None, self.fragment.validate())? + .map_err(|err| PyIOError::new_err(err.to_string())) + } } impl From for LanceFragment { @@ -825,6 +870,10 @@ impl FromPyObject<'_, '_> for PyLance { row_id_meta, last_updated_at_version_meta, created_at_version_meta, + // Round-tripped so overlays survive operations that pass existing + // fragments back (a manual Delete/Update/Merge commit). Sorting + // newest-last is deferred to the manifest reload after commit. + overlays: extract_vec::(&ob.getattr("overlays")?)?, })) } } @@ -857,6 +906,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> { .created_at_version_meta .as_ref() .map(|r| PyRowDatasetVersionMeta(r.clone())); + let overlays = export_vec(py, &self.0.overlays)?; cls.call1(( self.0.id, @@ -866,6 +916,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Fragment> { row_id_meta, created_at_version_meta, last_updated_at_version_meta, + overlays, )) } } diff --git a/python/src/fts.rs b/python/src/fts.rs new file mode 100644 index 00000000000..c24cefd6300 --- /dev/null +++ b/python/src/fts.rs @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use lance_index::scalar::InvertedIndexParams; +use lance_index::scalar::inverted::query::collect_query_tokens; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDict; + +fn extract_kwarg<'py, T>(kwargs: &Bound<'py, PyDict>, key: &str) -> PyResult> +where + T: FromPyObjectOwned<'py>, +{ + kwargs + .get_item(key)? + .map(|value| value.extract::().map_err(Into::into)) + .transpose() +} + +pub(crate) struct FtsTokenizerOptions { + analyzer: Option, + base_tokenizer: Option, + language: Option, + // The outer option tracks omission; the inner None explicitly disables the limit. + max_token_length: Option>, + lower_case: Option, + stem: Option, + remove_stop_words: Option, + // Preserve omitted versus explicit None to match create-index keyword semantics. + custom_stop_words: Option>>, + ascii_folding: Option, + min_ngram_length: Option, + max_ngram_length: Option, + prefix_only: Option, + split_identifiers: Option, + split_on_numerics: Option, + preserve_original: Option, + index_operators: Option, +} + +impl FtsTokenizerOptions { + pub(crate) fn from_kwargs(kwargs: &Bound<'_, PyDict>) -> PyResult { + Ok(Self { + analyzer: extract_kwarg(kwargs, "analyzer")?, + base_tokenizer: extract_kwarg(kwargs, "base_tokenizer")?, + language: extract_kwarg(kwargs, "language")?, + max_token_length: extract_kwarg(kwargs, "max_token_length")?, + lower_case: extract_kwarg(kwargs, "lower_case")?, + stem: extract_kwarg(kwargs, "stem")?, + remove_stop_words: extract_kwarg(kwargs, "remove_stop_words")?, + custom_stop_words: extract_kwarg(kwargs, "custom_stop_words")?, + ascii_folding: extract_kwarg(kwargs, "ascii_folding")?, + min_ngram_length: extract_kwarg(kwargs, "min_ngram_length")?, + max_ngram_length: extract_kwarg(kwargs, "max_ngram_length")?, + prefix_only: extract_kwarg(kwargs, "prefix_only")?, + split_identifiers: extract_kwarg(kwargs, "split_identifiers")?, + split_on_numerics: extract_kwarg(kwargs, "split_on_numerics")?, + preserve_original: extract_kwarg(kwargs, "preserve_original")?, + index_operators: extract_kwarg(kwargs, "index_operators")?, + }) + } + + pub(crate) fn apply(self, mut params: InvertedIndexParams) -> PyResult { + match (self.analyzer.as_deref(), self.base_tokenizer.as_deref()) { + (Some("text"), Some("code")) => { + return Err(PyValueError::new_err( + "base_tokenizer='code' requires analyzer='code'", + )); + } + (Some("code"), Some(base_tokenizer)) if base_tokenizer != "code" => { + return Err(PyValueError::new_err(format!( + "analyzer='code' requires base_tokenizer='code', got '{base_tokenizer}'" + ))); + } + _ => {} + } + + let uses_code_analyzer = match self.analyzer.as_deref() { + Some("code") => true, + Some("text") | None => self.base_tokenizer.as_deref() == Some("code"), + Some(_) => true, + }; + if !uses_code_analyzer + && [ + self.split_identifiers, + self.split_on_numerics, + self.preserve_original, + self.index_operators, + ] + .into_iter() + .flatten() + .any(|value| value) + { + return Err(PyValueError::new_err( + "code analyzer flags require analyzer='code'", + )); + } + + if let Some(analyzer) = self.analyzer { + params = params + .analyzer(&analyzer) + .map_err(|err| PyValueError::new_err(err.to_string()))?; + } + if let Some(base_tokenizer) = self.base_tokenizer { + params = params.base_tokenizer(base_tokenizer); + } + if let Some(language) = self.language { + params = params.language(&language).map_err(|err| { + PyValueError::new_err(format!("can't set tokenizer language to {language}: {err}")) + })?; + } + if let Some(max_token_length) = self.max_token_length { + params = params.max_token_length(max_token_length); + } + if let Some(lower_case) = self.lower_case { + params = params.lower_case(lower_case); + } + if let Some(stem) = self.stem { + params = params.stem(stem); + } + if let Some(remove_stop_words) = self.remove_stop_words { + params = params.remove_stop_words(remove_stop_words); + } + if let Some(custom_stop_words) = self.custom_stop_words { + params = params.custom_stop_words(custom_stop_words); + } + if let Some(ascii_folding) = self.ascii_folding { + params = params.ascii_folding(ascii_folding); + } + if let Some(min_ngram_length) = self.min_ngram_length { + params = params.ngram_min_length(min_ngram_length); + } + if let Some(max_ngram_length) = self.max_ngram_length { + params = params.ngram_max_length(max_ngram_length); + } + if let Some(prefix_only) = self.prefix_only { + params = params.ngram_prefix_only(prefix_only); + } + if let Some(split_identifiers) = self.split_identifiers { + params = params.split_identifiers(split_identifiers); + } + if let Some(split_on_numerics) = self.split_on_numerics { + params = params.split_on_numerics(split_on_numerics); + } + if let Some(preserve_original) = self.preserve_original { + params = params.preserve_original(preserve_original); + } + if let Some(index_operators) = self.index_operators { + params = params.index_operators(index_operators); + } + Ok(params) + } +} + +/// A token produced by the full-text search query tokenizer. +#[pyclass(get_all, skip_from_py_object)] +#[derive(Clone, Debug)] +pub struct FtsToken { + /// The token text after all configured filters have been applied. + pub text: String, + /// The position used by full-text query matching. + pub position: u32, +} + +#[pymethods] +impl FtsToken { + fn __repr__(&self) -> String { + format!("FtsToken(text={:?}, position={})", self.text, self.position) + } +} + +/// Tokenize a full-text search query without creating a dataset or index. +/// +/// The tokenizer options are the same as the tokenizer-related options accepted by +/// ``LanceDataset.create_scalar_index(..., index_type="INVERTED")``. ``None`` uses +/// the selected analyzer profile's default, except ``max_token_length=None``, which +/// disables the length limit; omitting it keeps the default limit of 40. Returned +/// positions are normalized to the first retained token while preserving gaps left +/// by token filters. +/// +/// Examples +/// -------- +/// >>> import lance +/// >>> tokens = lance.tokenize("the Cats and Dogs") +/// >>> [(token.text, token.position) for token in tokens] +/// [('cat', 0), ('dog', 2)] +#[pyfunction] +#[pyo3(signature = ( + query, + *, + analyzer = None, + base_tokenizer = None, + language = None, + max_token_length = Some(40), + lower_case = None, + stem = None, + remove_stop_words = None, + custom_stop_words = None, + ascii_folding = None, + min_ngram_length = None, + max_ngram_length = None, + prefix_only = None, + split_identifiers = None, + split_on_numerics = None, + preserve_original = None, + index_operators = None, +))] +#[allow(clippy::too_many_arguments)] +pub fn tokenize( + query: &str, + analyzer: Option, + base_tokenizer: Option, + language: Option, + max_token_length: Option, + lower_case: Option, + stem: Option, + remove_stop_words: Option, + custom_stop_words: Option>, + ascii_folding: Option, + min_ngram_length: Option, + max_ngram_length: Option, + prefix_only: Option, + split_identifiers: Option, + split_on_numerics: Option, + preserve_original: Option, + index_operators: Option, +) -> PyResult> { + let params = FtsTokenizerOptions { + analyzer, + base_tokenizer, + language, + max_token_length: Some(max_token_length), + lower_case, + stem, + remove_stop_words, + custom_stop_words: custom_stop_words.map(Some), + ascii_folding, + min_ngram_length, + max_ngram_length, + prefix_only, + split_identifiers, + split_on_numerics, + preserve_original, + index_operators, + } + .apply(InvertedIndexParams::default())?; + + let mut tokenizer = params + .build() + .map_err(|err| PyValueError::new_err(format!("Failed to build tokenizer: {err}")))?; + let tokens = collect_query_tokens(query, &mut tokenizer); + Ok((0..tokens.len()) + .map(|index| FtsToken { + text: tokens.get_token(index).to_string(), + position: tokens.position(index), + }) + .collect()) +} diff --git a/python/src/indices.rs b/python/src/indices.rs index 7ce7a297924..acb15491fcb 100644 --- a/python/src/indices.rs +++ b/python/src/indices.rs @@ -232,6 +232,7 @@ async fn do_train_pq_model( distance_type: &str, sample_rate: u32, max_iters: u32, + num_bits: u32, ivf_model: IvfModel, fragment_ids: Option>, ) -> PyResult { @@ -239,7 +240,7 @@ async fn do_train_pq_model( let distance_type = DistanceType::try_from(distance_type).unwrap(); let params = PQBuildParams { num_sub_vectors: num_subvectors as usize, - num_bits: 8, + num_bits: num_bits as usize, max_iters: max_iters as usize, sample_rate: sample_rate as usize, ..Default::default() @@ -260,7 +261,7 @@ async fn do_train_pq_model( #[pyfunction] #[allow(clippy::too_many_arguments)] -#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None))] +#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, sample_rate, max_iters, ivf_centroids, fragment_ids=None, num_bits=8))] fn train_pq_model<'py>( py: Python<'py>, dataset: &Dataset, @@ -272,6 +273,7 @@ fn train_pq_model<'py>( max_iters: u32, ivf_centroids: PyArrowType, fragment_ids: Option>, + num_bits: u32, ) -> PyResult> { let ivf_centroids = ivf_centroids.0; let ivf_centroids = FixedSizeListArray::from(ivf_centroids); @@ -291,6 +293,7 @@ fn train_pq_model<'py>( distance_type, sample_rate, max_iters, + num_bits, ivf_model, fragment_ids, ), @@ -316,7 +319,7 @@ fn train_pq_model<'py>( /// from lance.lance import indices /// /// # Mint one model and broadcast `model` to every worker. -/// model = indices.build_rq_model(dimension=128, num_bits=1) +/// model = indices.build_rq_model(dimension=128, num_bits=5) /// seg = ds.create_index_uncommitted( /// column="vector", /// index_type="IVF_RQ", @@ -327,7 +330,7 @@ fn train_pq_model<'py>( /// ) /// ``` #[pyfunction] -#[pyo3(signature = (dimension, num_bits=1, dtype="float32"))] +#[pyo3(signature = (dimension, num_bits=5, dtype="float32"))] pub fn build_rq_model(dimension: usize, num_bits: u8, dtype: &str) -> PyResult { use arrow::datatypes::{Float16Type, Float32Type, Float64Type}; use lance_index::vector::bq::RQRotationType; @@ -398,7 +401,7 @@ async fn do_transform_vectors( #[pyfunction] #[allow(clippy::too_many_arguments)] -#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None))] +#[pyo3(signature=(dataset, column, dimension, num_subvectors, distance_type, ivf_centroids, pq_codebook, dst_uri, fragments, partitions_ds_uri=None, num_bits=8))] pub fn transform_vectors( py: Python<'_>, dataset: &Dataset, @@ -411,6 +414,7 @@ pub fn transform_vectors( dst_uri: &str, fragments: Vec, partitions_ds_uri: Option<&str>, + num_bits: u32, ) -> PyResult<()> { let ivf_centroids = ivf_centroids.0; let ivf_centroids = FixedSizeListArray::from(ivf_centroids); @@ -419,7 +423,7 @@ pub fn transform_vectors( let distance_type = DistanceType::try_from(distance_type).unwrap(); let pq = ProductQuantizer::new( num_subvectors as usize, - /*num_bits=*/ 8, + num_bits, dimension, codebook, distance_type, @@ -529,6 +533,7 @@ async fn do_load_shuffled_vectors( uuid: index_id, name: index_name.to_string(), fields: vec![ds.schema().field(column).unwrap().id], + covering_fields: vec![], dataset_version: ds.manifest.version, fragment_bitmap: Some(ds.fragments().iter().map(|f| f.id as u32).collect()), index_details: Some(Arc::new( @@ -539,21 +544,7 @@ async fn do_load_shuffled_vectors( base_id: None, files: Some(files), }; - let segment = IndexSegment::new( - metadata.uuid, - metadata - .fragment_bitmap - .as_ref() - .expect("vector metadata should include fragment coverage") - .iter(), - metadata - .index_details - .as_ref() - .expect("vector metadata should include index details") - .clone(), - metadata.index_version, - ); - ds.commit_existing_index_segments(index_name, column, vec![segment]) + ds.commit_existing_index_segments(index_name, column, vec![metadata]) .await .infer_error()?; @@ -561,7 +552,7 @@ async fn do_load_shuffled_vectors( } #[pyfunction] -#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None))] +#[pyo3(signature=(filenames, dir_path, dataset, column, ivf_centroids, pq_codebook, pq_dimension, num_subvectors, distance_type, index_name=None, num_bits=8))] #[allow(clippy::too_many_arguments)] pub fn load_shuffled_vectors( filenames: Vec, @@ -574,6 +565,7 @@ pub fn load_shuffled_vectors( num_subvectors: u32, distance_type: &str, index_name: Option<&str>, + num_bits: u32, ) -> PyResult<()> { let mut default_idx_name = column.to_string(); default_idx_name.push_str("_idx"); @@ -595,7 +587,7 @@ pub fn load_shuffled_vectors( let distance_type = DistanceType::try_from(distance_type).unwrap(); let pq_model = ProductQuantizer::new( num_subvectors as usize, - /*num_bits=*/ 8, + num_bits, pq_dimension, codebook, distance_type, @@ -633,6 +625,9 @@ pub struct PyIndexSegmentDescription { /// The id of the dataset base path that stores this segment /// (None when the segment is stored in the dataset's default base path) pub base_id: Option, + /// The ids of the fields whose values this segment carries but is not keyed on. + /// Always the trailing entries of the segment's fields. + pub covering_fields: Vec, } impl PyIndexSegmentDescription { @@ -652,19 +647,21 @@ impl PyIndexSegmentDescription { created_at: segment.created_at, size_bytes, base_id: segment.base_id.map(|id| id as i64), + covering_fields: segment.covering_fields.clone(), } } pub fn __repr__(&self) -> String { format!( - "IndexSegmentDescription(uuid={}, dataset_version_at_last_update={}, fragment_ids={:?}, index_version={}, created_at={:?}, size_bytes={:?}, base_id={:?})", + "IndexSegmentDescription(uuid={}, dataset_version_at_last_update={}, fragment_ids={:?}, index_version={}, created_at={:?}, size_bytes={:?}, base_id={:?}, covering_fields={:?})", self.uuid, self.dataset_version_at_last_update, self.fragment_ids, self.index_version, self.created_at, self.size_bytes, - self.base_id + self.base_id, + self.covering_fields ) } } @@ -701,7 +698,7 @@ impl PyIndexDescription { .map(|field| { dataset .schema() - .field_path(*field as i32) + .field_path_minimal(*field as i32) .unwrap_or_else(|_| "".to_string()) }) .collect(); diff --git a/python/src/lib.rs b/python/src/lib.rs index 466d4ea90f2..a0304ecff61 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -49,7 +49,7 @@ use dataset::{DatasetBasePath, MergeInsertBuilder, PyFullTextQuery, PySearchFilt use env_logger::{Builder, Env}; use file::{ LanceBufferDescriptor, LanceColumnMetadata, LanceFileMetadata, LanceFileReader, - LanceFileStatistics, LanceFileWriter, LancePageMetadata, stable_version, + LanceFileStatistics, LanceFileWriteSummary, LanceFileWriter, LancePageMetadata, stable_version, }; use log::Level; use pyo3::exceptions::PyIOError; @@ -71,10 +71,14 @@ pub(crate) mod error; pub(crate) mod executor; pub(crate) mod file; pub(crate) mod fragment; +pub(crate) mod fts; pub(crate) mod indices; pub(crate) mod mem_wal; pub(crate) mod namespace; +pub(crate) mod object_store; +pub(crate) mod otel; pub(crate) mod reader; +pub(crate) mod rowids; pub(crate) mod scanner; pub(crate) mod schema; pub(crate) mod session; @@ -91,10 +95,12 @@ pub use crate::tracing::{TraceGuard, trace_to_chrome}; use crate::utils::Hnsw; use crate::utils::KMeans; pub use dataset::Dataset; +pub use dataset::serialize_row_addrs; pub use dataset::write_dataset; use fragment::{FileFragment, PyDeletionFile, PyRowDatasetVersionMeta, PyRowIdMeta}; pub use indices::register_indices; pub use reader::LanceReader; +use rowids::{PyRowIdSequence, PyRowIdSequenceIterator}; pub use scanner::Scanner; use crate::blob::{ @@ -131,11 +137,13 @@ static EXECUTOR_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false); static ATFORK_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false); -pub fn rt() -> &'static mut BackgroundExecutor { +pub fn rt() -> &'static BackgroundExecutor { loop { let ptr = BACKGROUND_EXECUTOR.load(Ordering::SeqCst); if !ptr.is_null() { - return unsafe { &mut *ptr }; + // SAFETY: installed executors are leaked and remain valid for the + // process lifetime. BackgroundExecutor uses shared access only. + return unsafe { &*ptr }; } if !EXECUTOR_INSTALLED.fetch_or(true, Ordering::SeqCst) { break; @@ -147,7 +155,8 @@ pub fn rt() -> &'static mut BackgroundExecutor { } let new_ptr = Box::into_raw(Box::new(create_background_executor())); BACKGROUND_EXECUTOR.store(new_ptr, Ordering::SeqCst); - unsafe { &mut *new_ptr } + // SAFETY: the executor is leaked and all of its operations take `&self`. + unsafe { &*new_ptr } } /// After a fork() operation, force re-creation of the BackgroundExecutor. Note: this function @@ -258,6 +267,8 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -267,6 +278,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -288,8 +300,11 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -298,7 +313,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; // MemWAL classes - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -309,6 +324,7 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(mem_wal::py_write_pk_sidecar))?; m.add_wrapped(wrap_pyfunction!(bfloat16_array))?; m.add_wrapped(wrap_pyfunction!(write_dataset))?; + m.add_wrapped(wrap_pyfunction!(serialize_row_addrs))?; m.add_wrapped(wrap_pyfunction!(write_fragments))?; m.add_wrapped(wrap_pyfunction!(write_fragments_transaction))?; m.add_wrapped(wrap_pyfunction!(schema_to_json))?; @@ -318,10 +334,18 @@ fn lance(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(trace_to_chrome))?; m.add_wrapped(wrap_pyfunction!(capture_trace_events))?; m.add_wrapped(wrap_pyfunction!(shutdown_tracing))?; + m.add_wrapped(wrap_pyfunction!(fts::tokenize))?; + // OpenTelemetry metrics bridge + m.add_class::()?; + m.add_class::()?; + m.add_wrapped(wrap_pyfunction!(otel::register_lance_metrics_recorder))?; + m.add_wrapped(wrap_pyfunction!(otel::lance_metrics_catalog))?; + m.add_wrapped(wrap_pyfunction!(otel::snapshot_lance_metrics))?; m.add_wrapped(wrap_pyfunction!(manifest_needs_migration))?; m.add_wrapped(wrap_pyfunction!(language_model_home))?; m.add_wrapped(wrap_pyfunction!(bytes_read_counter))?; m.add_wrapped(wrap_pyfunction!(iops_counter))?; + m.add_wrapped(wrap_pyfunction!(simd_info))?; m.add_wrapped(wrap_pyfunction!(stable_version))?; // Debug functions m.add_wrapped(wrap_pyfunction!(debug::format_schema))?; @@ -340,6 +364,37 @@ fn iops_counter() -> PyResult { Ok(::lance::io::iops_counter()) } +/// Returns a dict describing which SIMD tier the lance runtime dispatches to +/// on this host, plus the raw CPU feature flags it detected. +/// +/// Mirrors `pyarrow.runtime_info()`: a cheap, transparent way to verify that +/// the host is hitting the expected SIMD tier (e.g., `"avx512_fp16"`, +/// `"avx2"`) when debugging vector-search performance. +/// +/// Returns: +/// { +/// "tier": str, # e.g. "avx2", "avx_fma", "neon", "none" +/// "target_arch": str, # e.g. "x86_64", "aarch64", "loongarch64" +/// "host_features": list[str], # raw CPU feature flags (x86_64 only) +/// } +/// +/// Examples: +/// >>> import lance +/// >>> info = lance.simd_info() +/// >>> sorted(info) +/// ['host_features', 'target_arch', 'tier'] +/// >>> isinstance(info["tier"], str) +/// True +#[pyfunction] +pub fn simd_info(py: Python<'_>) -> PyResult> { + let info = lance_core::utils::cpu::simd_info(); + let dict = pyo3::types::PyDict::new(py); + dict.set_item("tier", info.tier.to_string())?; + dict.set_item("target_arch", info.target_arch)?; + dict.set_item("host_features", info.host_features)?; + Ok(dict.into()) +} + #[pyfunction(name = "bytes_read_counter")] fn bytes_read_counter() -> PyResult { Ok(::lance::io::bytes_read_counter()) @@ -464,3 +519,15 @@ fn ffi_logical_codec_from_pycapsule(obj: Bound) -> PyResult( result.to_pyarrow(py) } -/// Write a primary-key dedup sidecar (`_pk_index/`) for a flushed-generation +/// Write a primary-key dedup sidecar (`_pk_index/`) for an SSTable /// dataset already written at `gen_path`, mirroring what production flush emits. /// -/// Test-support only: lets Python tests stage a *faithful* flushed generation +/// Test-support only: lets Python tests stage a *faithful* SSTable /// (dataset + sidecar). Production always writes the sidecar during flush, so a /// dataset-without-sidecar is not a state the system otherwise produces. #[pyfunction(name = "_write_pk_sidecar", signature = (gen_path, data, pk_columns))] @@ -115,16 +115,17 @@ fn optional_string(value: Bound<'_, PyAny>) -> PyResult> { } } -/// Represents a single generation of a MemWAL shard that has been merged -/// into the base table. Used with `MergeInsertBuilder.mark_generations_as_merged()`. -#[pyclass(name = "_MergedGeneration", module = "_lib")] -pub struct PyMergedGeneration { +/// Points to an SSTable compacted into the base table. +/// +/// Used with `MergeInsertBuilder.mark_sstables_as_compacted()`. +#[pyclass(name = "_CompactedSsTable", module = "_lib")] +pub struct PyCompactedSsTable { pub shard_id: String, pub generation: u64, } #[pymethods] -impl PyMergedGeneration { +impl PyCompactedSsTable { #[new] pub fn new(shard_id: String, generation: u64) -> Self { Self { @@ -145,23 +146,23 @@ impl PyMergedGeneration { pub fn __repr__(&self) -> String { format!( - "_MergedGeneration(shard_id='{}', generation={})", + "_CompactedSsTable(shard_id='{}', generation={})", self.shard_id, self.generation ) } } -impl PyMergedGeneration { - pub fn to_lance(&self) -> PyResult { +impl PyCompactedSsTable { + pub fn to_lance(&self) -> PyResult { let uuid = Uuid::parse_str(&self.shard_id) .map_err(|e| PyValueError::new_err(format!("Invalid shard_id UUID: {}", e)))?; - Ok(LanceMergedGeneration::new(uuid, self.generation)) + Ok(LanceCompactedSsTable::new(uuid, self.generation)) } } /// Snapshot of a MemWAL shard's state at a point in time. /// -/// Used to specify which flushed generations to include when creating an +/// Used to specify which SSTables to include when creating an /// `_LsmScanner`. Supports a builder pattern for adding generations. #[pyclass(name = "_ShardSnapshot", module = "_lib", skip_from_py_object)] #[derive(Clone)] @@ -195,13 +196,13 @@ impl PyShardSnapshot { slf } - /// Add a flushed generation by its generation number and storage path. - pub fn with_flushed_generation( + /// Add an SSTable by its generation number and storage path. + pub fn with_sstable( mut slf: PyRefMut<'_, Self>, generation: u64, path: String, ) -> PyRefMut<'_, Self> { - slf.inner = slf.inner.clone().with_flushed_generation(generation, path); + slf.inner = slf.inner.clone().with_sstable(generation, path); slf } @@ -212,10 +213,10 @@ impl PyShardSnapshot { pub fn __repr__(&self) -> String { format!( - "_ShardSnapshot(shard_id='{}', current_gen={}, flushed_gens={})", + "_ShardSnapshot(shard_id='{}', current_gen={}, sstables={})", self.inner.shard_id, self.inner.current_generation, - self.inner.flushed_generations.len() + self.inner.sstables.len() ) } } @@ -238,6 +239,14 @@ struct ClosedShardWriterState { memtable_stats: MemTableStats, } +fn collect_record_batches(data: &Bound<'_, PyAny>) -> PyResult> { + let reader = ArrowArrayStreamReader::from_pyarrow_bound(data) + .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?; + reader + .collect::>() + .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e))) +} + #[pymethods] impl PyShardWriter { /// Write data batches to the MemWAL. @@ -245,11 +254,7 @@ impl PyShardWriter { /// Accepts any PyArrow-compatible data source (RecordBatch, Table, /// or an Arrow stream reader). pub fn put(&self, py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult<()> { - let reader = ArrowArrayStreamReader::from_pyarrow_bound(data) - .map_err(|e| PyValueError::new_err(format!("Cannot read data as Arrow: {}", e)))?; - let batches: Vec = reader - .collect::>() - .map_err(|e| PyIOError::new_err(format!("Failed to read batches: {}", e)))?; + let batches = collect_record_batches(data)?; if batches.is_empty() { return Ok(()); @@ -268,6 +273,31 @@ impl PyShardWriter { .map_err(|e: lance::Error| PyIOError::new_err(e.to_string())) } + /// Delete rows from the MemWAL by primary key. + /// + /// Accepts any PyArrow-compatible data source carrying the shard's primary + /// key column(s). Rust core validates that primary keys exist and builds the + /// tombstone rows. + pub fn delete(&self, py: Python<'_>, keys: &Bound<'_, PyAny>) -> PyResult<()> { + let batches = collect_record_batches(keys)?; + + if batches.is_empty() { + return Ok(()); + } + + let inner = self.inner.clone(); + rt().block_on(Some(py), async move { + let guard = inner.lock().await; + match guard.as_ref() { + Some(writer) => writer.delete(batches).await.map(|_| ()), + None => Err(lance_core::Error::invalid_input( + "ShardWriter is already closed", + )), + } + })? + .map_err(|e: lance::Error| PyIOError::new_err(e.to_string())) + } + /// Flush pending data and close the writer. /// /// After close(), calling put() will raise an error. @@ -328,20 +358,27 @@ impl PyShardWriter { /// Return current MemTable statistics. /// /// Returns a dict with keys: row_count, batch_count, estimated_size_bytes, - /// generation. + /// index_bytes, frozen_bytes, generation. pub fn memtable_stats(&self, py: Python<'_>) -> PyResult> { let inner = self.inner.clone(); let closed_state = self.closed_state.clone(); - let stats = rt() + let (stats, bytes) = rt() .block_on(Some(py), async move { let guard = inner.lock().await; match guard.as_ref() { - Some(w) => w.memtable_stats().await, + // Byte totals come from `memory()`, the lock-free view; + // `memtable_stats` carries none. + Some(w) => w + .memtable_stats() + .await + .map(|stats| (stats, Some(w.memory()))), None => { let closed_guard = closed_state.lock().await; closed_guard .as_ref() - .map(|state| state.memtable_stats.clone()) + // A closed writer holds nothing, so the byte totals + // are zero by construction rather than stale. + .map(|state| (state.memtable_stats.clone(), None)) .ok_or_else(|| { lance_core::Error::invalid_input("ShardWriter is already closed") }) @@ -350,12 +387,12 @@ impl PyShardWriter { })? .map_err(|e: lance::Error| PyIOError::new_err(e.to_string()))?; - memtable_stats_to_pydict(py, &stats) + memtable_stats_to_pydict(py, &stats, bytes.as_ref()) } /// Create an LSM scanner that includes the active MemTable for strong consistency. /// - /// The scanner covers: base table + given flushed generations + current active MemTable. + /// The scanner covers: base table + given SSTables + current active MemTable. #[pyo3(signature = (shard_snapshots=vec![]))] pub fn lsm_scanner( &self, @@ -508,7 +545,7 @@ impl PyExecutionPlan { } } -/// LSM-aware scanner covering base table, flushed MemTables, and active MemTable. +/// LSM-aware scanner covering base table, SSTables, and active MemTable. /// /// Provides deduplication by primary key, always returning the newest version /// of each row across all LSM levels. @@ -910,20 +947,28 @@ fn write_stats_to_pydict(py: Python<'_>, stats: &WriteStatsSnapshot) -> PyResult Ok(dict.into_any().unbind()) } -fn memtable_stats_to_pydict(py: Python<'_>, stats: &MemTableStats) -> PyResult> { +fn memtable_stats_to_pydict( + py: Python<'_>, + stats: &MemTableStats, + bytes: Option<&ShardMemory>, +) -> PyResult> { let dict = PyDict::new(py); dict.set_item("row_count", stats.row_count)?; dict.set_item("batch_count", stats.batch_count)?; - dict.set_item("estimated_size_bytes", stats.estimated_size)?; + // Row data only, as this key has always meant; `index_bytes` below is the + // rest of the active memtable's footprint. + dict.set_item( + "estimated_size_bytes", + bytes.map_or(0, ShardMemory::row_bytes), + )?; + dict.set_item("index_bytes", bytes.map_or(0, ShardMemory::index_bytes))?; dict.set_item("generation", stats.generation)?; dict.set_item( "max_buffered_batch_position", stats.max_buffered_batch_position, )?; - dict.set_item( - "max_flushed_batch_position", - stats.max_flushed_batch_position, - )?; + dict.set_item("durable_batch_count", stats.durable_batch_count)?; + dict.set_item("global_offset", stats.global_offset)?; dict.set_item( "pending_wal_start_batch_position", stats.pending_wal_start_batch_position, @@ -938,6 +983,17 @@ fn memtable_stats_to_pydict(py: Python<'_>, stats: &MemTableStats) -> PyResult

    MemTableStats { + // Close awaits every frozen memtable's flush, so nothing is owed afterwards + // regardless of whether the active memtable had buffered batches. + let stats_before_close = MemTableStats { + frozen_count: 0, + ..stats_before_close + }; + if stats_before_close.batch_count == 0 { return stats_before_close; } + // After a successful close every buffered batch is flushed and WAL-durable, + // so the synthesized empty memtable starts at the writer's global end and the + // durable cursor has caught up to it. + let global_end = stats_before_close.global_offset + stats_before_close.batch_count; MemTableStats { row_count: 0, batch_count: 0, - estimated_size: 0, generation: stats_before_close.generation.saturating_add(1), max_buffered_batch_position: None, - max_flushed_batch_position: None, + durable_batch_count: global_end, + global_offset: global_end, pending_wal_start_batch_position: None, pending_wal_end_batch_position: None, pending_wal_batch_count: 0, pending_wal_row_count: 0, pending_wal_estimated_bytes: 0, + frozen_count: 0, } } diff --git a/python/src/namespace.rs b/python/src/namespace.rs index e88ff40de2c..996d3f92ca7 100644 --- a/python/src/namespace.rs +++ b/python/src/namespace.rs @@ -9,6 +9,7 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; use lance_namespace::LanceNamespace as LanceNamespaceTrait; +use lance_namespace::compat::merge_insert_request_from_json; use lance_namespace::models::{ AlterTableAddColumnsRequest, AlterTableAlterColumnsRequest, AlterTableBackfillColumnsRequest, AlterTableDropColumnsRequest, AlterTransactionRequest, AnalyzeTableQueryPlanRequest, @@ -19,10 +20,9 @@ use lance_namespace::models::{ DescribeTableVersionResponse, DescribeTransactionRequest, DropTableIndexRequest, ExplainTableQueryPlanRequest, GetTableStatsRequest, GetTableTagVersionRequest, InsertIntoTableRequest, ListTableIndicesRequest, ListTableTagsRequest, - ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, - MergeInsertIntoTableRequest, QueryTableRequest, RefreshMaterializedViewRequest, - RestoreTableRequest, UpdateTableRequest, UpdateTableSchemaMetadataRequest, - UpdateTableTagRequest, + ListTableVersionsRequest, ListTableVersionsResponse, ListTablesRequest, QueryTableRequest, + RefreshMaterializedViewRequest, RestoreTableRequest, UpdateTableRequest, + UpdateTableSchemaMetadataRequest, UpdateTableTagRequest, }; use lance_namespace_impls::RestNamespaceBuilder; use lance_namespace_impls::{ConnectBuilder, RestAdapter, RestAdapterConfig, RestAdapterHandle}; @@ -460,7 +460,7 @@ impl PyDirectoryNamespace { request: &Bound<'_, PyAny>, request_data: &Bound<'_, PyBytes>, ) -> PyResult> { - let request: MergeInsertIntoTableRequest = depythonize(request)?; + let request = merge_insert_request_from_json(depythonize(request)?).infer_error()?; let data = Bytes::copy_from_slice(request_data.as_bytes()); let response = crate::rt() .block_on(Some(py), self.inner.merge_insert_into_table(request, data))? @@ -1160,7 +1160,7 @@ impl PyRestNamespace { request: &Bound<'_, PyAny>, request_data: &Bound<'_, PyBytes>, ) -> PyResult> { - let request: MergeInsertIntoTableRequest = depythonize(request)?; + let request = merge_insert_request_from_json(depythonize(request)?).infer_error()?; let data = Bytes::copy_from_slice(request_data.as_bytes()); let response = crate::rt() .block_on(Some(py), self.inner.merge_insert_into_table(request, data))? diff --git a/python/src/object_store.rs b/python/src/object_store.rs new file mode 100644 index 00000000000..a7704002481 --- /dev/null +++ b/python/src/object_store.rs @@ -0,0 +1,238 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Runtime registration hook for external `ObjectStoreProvider` implementations. +//! +//! This module exposes two pyclasses: +//! +//! - `PyObjectStoreRegistry` wraps [`lance_io::object_store::ObjectStoreRegistry`] +//! and lets Python code register additional `ObjectStoreProvider`s under new +//! URL schemes. A registry constructed here can be passed to `Session` so +//! `lance.dataset("myscheme://...")` dispatches through the new provider. +//! - `PyObjectStoreProvider` is a bridge that adapts a built-in Rust provider +//! (currently just `MemoryStoreProvider`), a Python object that implements +//! the `new_store` protocol, or an `Arc` produced +//! by a *separate* wheel and handed across a `PyCapsule`, to +//! `Arc`. +//! +//! The Python-callable path is intentionally stubbed for this first cut: the +//! full Python-to-Rust `ObjectStore` bridge (i.e. wrapping a Python-returned +//! object as an `object_store::ObjectStore`) is a follow-up. The smoke test +//! against the built-in memory provider proves the registration + dispatch +//! plumbing works end to end. +//! +//! # Out-of-tree providers via `PyCapsule` +//! +//! [`PyObjectStoreProvider::from_capsule`] lets an external wheel register a +//! Rust `ObjectStoreProvider` it compiled itself. The external wheel builds an +//! `Arc`, wraps it in a `PyCapsule` named +//! [`PROVIDER_CAPSULE_NAME`], and passes that capsule here. Because Rust has +//! no stable ABI, this is sound **only when both wheels are built in lockstep**: +//! identical `rustc`, identical `lance-io` / `object_store` source, and +//! identical resolved dependency versions, so the trait object's vtable and the +//! types in `new_store`'s signature have the same layout on both sides. In +//! Phase I both wheels are built locally from the same branch and toolchain, so +//! the constraint holds; distributing pre-built wheels that must interoperate +//! is deferred (a packaging-phase concern). + +use std::ffi::CStr; +use std::sync::Arc; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::{PyCapsule, PyCapsuleMethods}; + +use lance_io::object_store::providers::memory::MemoryStoreProvider; +use lance_io::object_store::{ + ObjectStore, ObjectStoreParams, ObjectStoreProvider, ObjectStoreRegistry, +}; + +/// Name that every capsule passed to [`PyObjectStoreProvider::from_capsule`] +/// must carry. External wheels create their capsule with this exact name so a +/// capsule holding some unrelated pointer cannot be mistaken for a provider. +pub const PROVIDER_CAPSULE_NAME: &CStr = c"lance_object_store_provider"; + +/// Bridge between a Python object and the Rust `ObjectStoreProvider` trait. +/// +/// For the memory variant we short-circuit to a real Rust provider so the +/// smoke test can prove the scheme-dispatch plumbing works. For the Python +/// callable variant we hold a `Py` and (in a follow-up) will call +/// `new_store(base_path, storage_options)` on it via the GIL. +#[derive(Debug)] +enum PyProviderBridge { + /// Built-in `MemoryStoreProvider`, wrapped directly. + Memory(MemoryStoreProvider), + /// A Python object implementing the `new_store(base_path, storage_options)` + /// protocol. Not yet dispatchable end-to-end (see module docstring). + /// + /// The wrapped `Py` is intentionally held even though we do not + /// invoke it yet: keeping the Python object alive here means once the + /// bridge lands, we can dispatch without changing the enum shape. + #[allow(dead_code)] + PyCallable(Py), +} + +#[async_trait::async_trait] +impl ObjectStoreProvider for PyProviderBridge { + async fn new_store( + &self, + base_path: url::Url, + params: &ObjectStoreParams, + ) -> lance_core::Result { + match self { + Self::Memory(inner) => inner.new_store(base_path, params).await, + Self::PyCallable(_) => Err(lance_core::Error::not_supported( + "PyObjectStoreProvider: the Python-callable bridge is not yet \ + implemented. Use PyObjectStoreProvider.memory() for the current cut.", + )), + } + } +} + +/// Python-facing wrapper around `Arc`. +/// +/// There are three ways to construct one from Python, ordered here by how +/// complete they are today: +/// +/// 1. `_ObjectStoreProvider.from_capsule(capsule)` — **fully dispatches.** +/// Adopt an `Arc` built by a separate, +/// ABI-compatible wheel and handed over in a `PyCapsule`. `new_store` then +/// calls that provider's own Rust implementation directly — the +/// `PyProviderBridge` below is not involved. This is how an out-of-tree Rust +/// provider (e.g. an on-node NVMe cache) plugs in without living in the +/// Lance source tree; see the module docs for the `PyCapsule` handoff and +/// its ABI-lockstep build requirement. +/// 2. `_ObjectStoreProvider.memory()` — **fully dispatches.** Wrap the built-in +/// `MemoryStoreProvider`; registrable under any scheme and functional for +/// read/write. Primarily a test/reference vehicle. +/// 3. `_ObjectStoreProvider(py_obj)` — **stub; does not dispatch yet.** Hold a +/// Python object implementing `new_store(base_path, storage_options)`. +/// Registration succeeds, but dispatch raises `not_supported`: the full +/// Python-to-Rust `ObjectStore` bridge (calling back into Python from +/// `new_store` under the GIL) is a follow-up. +#[pyclass(name = "_ObjectStoreProvider", module = "_lib", from_py_object)] +#[derive(Clone)] +pub struct PyObjectStoreProvider { + pub(crate) inner: Arc, +} + +#[pymethods] +impl PyObjectStoreProvider { + /// Wrap a Python object implementing the `new_store(base_path, storage_options)` + /// protocol. Registration will succeed, but scheme-dispatch will raise until + /// the full Python-to-Rust `ObjectStore` bridge is implemented. + #[new] + fn new(py_object: Py) -> Self { + Self { + inner: Arc::new(PyProviderBridge::PyCallable(py_object)), + } + } + + /// Return a provider backed by the built-in `MemoryStoreProvider`. Every + /// call to `new_store` allocates a fresh in-memory `object_store::InMemory`; + /// the enclosing `ObjectStoreRegistry` caches the resulting `ObjectStore` + /// so writers and readers using the same scheme share storage as long as + /// something holds a strong reference. + #[staticmethod] + fn memory() -> Self { + Self { + inner: Arc::new(PyProviderBridge::Memory(MemoryStoreProvider)), + } + } + + /// Adopt an `Arc` carried in a `PyCapsule` created + /// by a separate wheel. The capsule must be named [`PROVIDER_CAPSULE_NAME`] + /// and hold exactly an `Arc`. + /// + /// See the module docstring for the ABI-lockstep requirement: the calling + /// wheel must be built against the identical `lance-io` / `object_store` + /// source and toolchain as this one. + #[staticmethod] + fn from_capsule(capsule: &Bound<'_, PyCapsule>) -> PyResult { + // `pointer_checked(Some(name))` asks CPython for the pointer *and* + // requires the capsule to carry exactly this name and a non-null + // pointer, so a foreign or misnamed capsule is rejected here rather + // than dereferenced. (Passing `None` asks for a *nameless* capsule and + // would reject every correctly-named one.) + let ptr = capsule + .pointer_checked(Some(PROVIDER_CAPSULE_NAME)) + .map_err(|e| { + PyValueError::new_err(format!( + "expected a PyCapsule named {:?}: {e}", + PROVIDER_CAPSULE_NAME.to_string_lossy(), + )) + })?; + + // SAFETY: by the capsule-name contract above, the capsule carries an + // `Arc` built against the identical lance-io / + // object_store types (same source, rustc, and resolved dependency + // versions). We dereference only long enough to clone the `Arc` + // (bumping the strong count); the capsule keeps its own reference and + // its destructor drops that on GC. + let provider = unsafe { ptr.cast::>().as_ref() }; + Ok(Self { + inner: provider.clone(), + }) + } + + /// Test/reference producer: wrap the built-in memory provider in a + /// `PyCapsule` named [`PROVIDER_CAPSULE_NAME`], mirroring what an external, + /// ABI-compatible wheel emits. Lets `from_capsule` be exercised end to end + /// from Python without a second wheel in the tree. + #[staticmethod] + fn _memory_capsule(py: Python<'_>) -> PyResult> { + let provider: Arc = + Arc::new(PyProviderBridge::Memory(MemoryStoreProvider)); + PyCapsule::new(py, provider, Some(PROVIDER_CAPSULE_NAME.to_owned())) + } + + fn __repr__(&self) -> String { + format!("_ObjectStoreProvider({:?})", self.inner) + } +} + +/// Python-facing wrapper around `Arc`. +/// +/// A new instance starts from `ObjectStoreRegistry::default()`, so all +/// built-in schemes (memory, file, and any of s3/az/gs/oss/... enabled at +/// build time) are already registered. Additional providers can be inserted +/// under new (or overridden) schemes via `register_provider`. +/// +/// Pass an instance as the `store_registry` argument of `Session(...)` to +/// make its schemes visible to `lance.dataset(uri, session=...)` and +/// `lance.write_dataset(..., uri, session=...)`. +#[pyclass(name = "_ObjectStoreRegistry", module = "_lib", from_py_object)] +#[derive(Clone)] +pub struct PyObjectStoreRegistry { + pub(crate) inner: Arc, +} + +#[pymethods] +impl PyObjectStoreRegistry { + /// Create a new registry pre-populated with the built-in schemes. + #[new] + fn new() -> Self { + Self { + inner: Arc::new(ObjectStoreRegistry::default()), + } + } + + /// Register a provider under a scheme. Idempotent: registering the same + /// scheme again replaces the previous provider. Registering under a + /// built-in scheme (e.g. `"memory"`) overrides that built-in. + fn register_provider(&self, scheme: &str, provider: &PyObjectStoreProvider) -> PyResult<()> { + if scheme.is_empty() { + return Err(PyValueError::new_err("scheme must be a non-empty string")); + } + self.inner.insert(scheme, provider.inner.clone()); + Ok(()) + } + + fn __repr__(&self) -> String { + let stats = self.inner.stats(); + format!( + "_ObjectStoreRegistry(active_stores={}, hits={}, misses={})", + stats.active_stores, stats.hits, stats.misses, + ) + } +} diff --git a/python/src/otel.rs b/python/src/otel.rs new file mode 100644 index 00000000000..71d191d7db9 --- /dev/null +++ b/python/src/otel.rs @@ -0,0 +1,640 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Bridge from the [`metrics`] crate facade to Python OpenTelemetry. +//! +//! Lance core publishes metrics through the global [`metrics`] facade without +//! choosing a backend. This module installs a process-global [`Recorder`] that +//! aggregates those metrics into lock-free cumulative storage, and exposes that +//! state to Python so the bindings can feed it into the user's OpenTelemetry +//! `MeterProvider`. +//! +//! While it targets OpenTelemetry on the Python side, the recorder is agnostic +//! to the metric *source*: it records any metric emitted through the facade, +//! keyed by name and labels. Object store metrics are the first producer, but +//! nothing here is specific to them. New metrics flow through automatically; +//! they only need to be described (see [`describe_all`]) so the Python layer can +//! discover their name, kind, and unit up front. +//! +//! ## Why pull, not push +//! +//! OpenTelemetry collects on its own schedule and invokes observable-instrument +//! callbacks at collection time. Cumulative counters map directly onto OTel's +//! `ObservableCounter` semantics. So the bridge aggregates in Rust and lets the +//! Python collection thread pull a [`snapshot`](snapshot_lance_metrics). The +//! snapshot is lock-free, but it still walks every registered series and +//! allocates owned copies of their names and labels, so it runs with the GIL +//! released to avoid stalling other Python threads during collection. +//! +//! ## Histograms +//! +//! OpenTelemetry has no asynchronous histogram instrument, so histograms cannot +//! be pulled as-is. Instead each histogram is aggregated into fixed buckets +//! (Prometheus style) and exposed as cumulative `le` bucket counts plus a count +//! and sum, which the Python layer surfaces as observable counters. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, OnceLock, RwLock}; + +use metrics::{Counter, Gauge, Histogram, Key, KeyName, Metadata, Recorder, SharedString, Unit}; +use metrics_util::registry::{Registry, Storage}; +use pyo3::prelude::*; + +/// Bucket boundaries used when a histogram has no registered bounds. Covers a +/// broad latency range so unknown histograms still produce useful buckets. +const DEFAULT_BOUNDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, +]; + +/// The kind of a metric, mirroring the three `metrics` instrument types. +#[derive(Clone, Copy)] +enum MetricKind { + Counter, + Gauge, + Histogram, +} + +impl MetricKind { + fn as_str(self) -> &'static str { + match self { + Self::Counter => "counter", + Self::Gauge => "gauge", + Self::Histogram => "histogram", + } + } +} + +/// Description of a metric, populated by the recorder's `describe_*` methods. +struct MetricDescription { + kind: MetricKind, + unit: Option, + description: String, +} + +/// Catalog of described metrics, keyed by metric name. The Python layer reads +/// this to create one OpenTelemetry instrument per metric up front. +static CATALOG: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +/// Per-metric histogram bucket boundaries, keyed by metric name. Producers +/// register their recommended bounds before any metric is recorded. +static HISTOGRAM_BOUNDS: LazyLock>>> = + LazyLock::new(|| RwLock::new(HashMap::new())); + +/// The installed recorder's registry, available once installation succeeds. +static REGISTRY: OnceLock>> = OnceLock::new(); + +fn bounds_for(name: &str) -> Arc<[f64]> { + HISTOGRAM_BOUNDS + .read() + .unwrap() + .get(name) + .cloned() + .unwrap_or_else(|| Arc::from(DEFAULT_BOUNDS)) +} + +/// A histogram that buckets samples at record time into fixed boundaries, +/// keeping a cumulative count and sum. Bucketing eagerly keeps memory bounded +/// (unlike retaining raw samples) and produces Prometheus-style `le` buckets. +struct BucketedHistogram { + /// Sorted, finite upper bounds. A sample `v` falls in the first bucket whose + /// bound is `>= v`; samples above all bounds fall in the implicit `+Inf` + /// bucket stored as the final entry of `counts`. + bounds: Arc<[f64]>, + /// Per-bucket (non-cumulative) counts; length is `bounds.len() + 1`. + counts: Box<[AtomicU64]>, + count: AtomicU64, + /// Running sum of recorded values, stored as `f64` bits (there is no atomic + /// f64, so the bit pattern is held in a `u64`; see [`Self::add_to_sum`]). + sum_bits: AtomicU64, +} + +// All atomics here use `Ordering::Relaxed`: each metric counter is independent, +// so no happens-before relationship is needed between them, and a snapshot +// reader tolerates slightly stale values. This matches `metrics_util`'s +// `AtomicStorage`. + +impl BucketedHistogram { + fn new(bounds: Arc<[f64]>) -> Self { + let counts = (0..bounds.len() + 1) + .map(|_| AtomicU64::new(0)) + .collect::>() + .into_boxed_slice(); + Self { + bounds, + counts, + count: AtomicU64::new(0), + sum_bits: AtomicU64::new(0), + } + } + + fn add_to_sum(&self, value: f64) { + // No atomic offers an f64 add, so read the current bit pattern, add in + // float space, and CAS it back, retrying if another thread won the race. + let mut current = self.sum_bits.load(Ordering::Relaxed); + loop { + let updated = (f64::from_bits(current) + value).to_bits(); + match self.sum_bits.compare_exchange_weak( + current, + updated, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } + + /// Cumulative `le` buckets, total count, and sum at this instant. + fn snapshot(&self) -> MetricValue { + let mut cumulative = 0u64; + let mut buckets = Vec::with_capacity(self.bounds.len() + 1); + for (i, bound) in self.bounds.iter().enumerate() { + cumulative += self.counts[i].load(Ordering::Relaxed); + buckets.push((format!("{}", bound), cumulative)); + } + cumulative += self.counts[self.bounds.len()].load(Ordering::Relaxed); + buckets.push(("+Inf".to_string(), cumulative)); + MetricValue::Histogram { + buckets, + count: self.count.load(Ordering::Relaxed), + sum: f64::from_bits(self.sum_bits.load(Ordering::Relaxed)), + } + } +} + +impl metrics::HistogramFn for BucketedHistogram { + fn record(&self, value: f64) { + let idx = self.bounds.partition_point(|&bound| bound < value); + self.counts[idx].fetch_add(1, Ordering::Relaxed); + self.count.fetch_add(1, Ordering::Relaxed); + self.add_to_sum(value); + } +} + +/// Storage backing the registry. Counters and gauges are plain atomics (as in +/// `metrics_util`'s `AtomicStorage`); histograms use [`BucketedHistogram`]. +struct LanceStorage; + +impl Storage for LanceStorage { + type Counter = Arc; + type Gauge = Arc; + type Histogram = Arc; + + fn counter(&self, _key: &Key) -> Self::Counter { + Arc::new(AtomicU64::new(0)) + } + + fn gauge(&self, _key: &Key) -> Self::Gauge { + // The `metrics` facade writes the f64 bit pattern into this `u64` (the + // snapshot decodes it with `f64::from_bits`), matching `AtomicStorage`. + // `0` decodes to `0.0`, the correct initial value. + Arc::new(AtomicU64::new(0)) + } + + fn histogram(&self, key: &Key) -> Self::Histogram { + Arc::new(BucketedHistogram::new(bounds_for(key.name()))) + } +} + +struct LanceRecorder { + registry: Arc>, +} + +impl LanceRecorder { + fn describe( + &self, + key: KeyName, + kind: MetricKind, + unit: Option, + description: SharedString, + ) { + CATALOG.lock().unwrap().insert( + key.as_str().to_string(), + MetricDescription { + kind, + unit: unit.map(|u| u.as_canonical_label().to_string()), + description: description.into_owned(), + }, + ); + } +} + +impl Recorder for LanceRecorder { + fn describe_counter(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Counter, unit, description); + } + + fn describe_gauge(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Gauge, unit, description); + } + + fn describe_histogram(&self, key: KeyName, unit: Option, description: SharedString) { + self.describe(key, MetricKind::Histogram, unit, description); + } + + fn register_counter(&self, key: &Key, _metadata: &Metadata<'_>) -> Counter { + self.registry + .get_or_create_counter(key, |c| Counter::from_arc(c.clone())) + } + + fn register_gauge(&self, key: &Key, _metadata: &Metadata<'_>) -> Gauge { + self.registry + .get_or_create_gauge(key, |g| Gauge::from_arc(g.clone())) + } + + fn register_histogram(&self, key: &Key, _metadata: &Metadata<'_>) -> Histogram { + self.registry + .get_or_create_histogram(key, |h| Histogram::from_arc(h.clone())) + } +} + +/// Register the recommended histogram bounds for every metric-emitting +/// subsystem. New subsystems add their `histogram_bounds()` here. +fn register_bounds() { + let mut bounds = HISTOGRAM_BOUNDS.write().unwrap(); + for (name, values) in lance_io::object_store::metrics::histogram_bounds() { + bounds.insert((*name).to_string(), Arc::from(*values)); + } +} + +/// Describe every metric-emitting subsystem so the catalog is populated. Must +/// run after the recorder is installed. New subsystems add their +/// `describe_metrics()` here. +fn describe_all() { + lance_io::object_store::metrics::describe_metrics(); +} + +enum MetricValue { + Scalar(f64), + Histogram { + buckets: Vec<(String, u64)>, + count: u64, + sum: f64, + }, +} + +struct MetricPoint { + name: String, + kind: &'static str, + attributes: HashMap, + value: MetricValue, +} + +fn labels(key: &Key) -> HashMap { + key.labels() + .map(|label| (label.key().to_string(), label.value().to_string())) + .collect() +} + +fn collect_points(registry: &Registry) -> Vec { + let mut points = Vec::new(); + for (key, handle) in registry.get_counter_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "counter", + attributes: labels(&key), + // OpenTelemetry observations are float; counts stay well within the + // f64-exact integer range (2^53), so this cast is lossless in practice. + value: MetricValue::Scalar(handle.load(Ordering::Relaxed) as f64), + }); + } + for (key, handle) in registry.get_gauge_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "gauge", + attributes: labels(&key), + value: MetricValue::Scalar(f64::from_bits(handle.load(Ordering::Relaxed))), + }); + } + for (key, handle) in registry.get_histogram_handles() { + points.push(MetricPoint { + name: key.name().to_string(), + kind: "histogram", + attributes: labels(&key), + value: handle.snapshot(), + }); + } + points +} + +/// One metric data point exposed to Python. For counters and gauges only +/// `value` is set; for histograms `buckets` (cumulative `le` counts), `count`, +/// and `sum` are set. +#[pyclass(name = "MetricPoint", get_all)] +pub struct PyMetricPoint { + name: String, + kind: String, + attributes: HashMap, + value: Option, + buckets: Option>, + count: Option, + sum: Option, +} + +impl From for PyMetricPoint { + fn from(point: MetricPoint) -> Self { + let (value, buckets, count, sum) = match point.value { + MetricValue::Scalar(v) => (Some(v), None, None, None), + MetricValue::Histogram { + buckets, + count, + sum, + } => (None, Some(buckets), Some(count), Some(sum)), + }; + Self { + name: point.name, + kind: point.kind.to_string(), + attributes: point.attributes, + value, + buckets, + count, + sum, + } + } +} + +/// A described metric, used by the Python layer to create instruments up front. +#[pyclass(name = "MetricDescription", get_all)] +pub struct PyMetricDescription { + name: String, + kind: String, + unit: Option, + description: String, +} + +/// Install the Lance metrics recorder as the process-global `metrics` recorder. +/// +/// Returns `True` if the recorder is installed (now or previously). Returns +/// `False` if a *different* recorder is already installed — `metrics` allows +/// only one global recorder per process, so Lance cannot coexist with another. +#[pyfunction] +pub fn register_lance_metrics_recorder() -> bool { + if REGISTRY.get().is_some() { + return true; + } + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + // Register histogram bounds *before* installing the recorder. Once the + // recorder is global, another thread can emit a metric and create the + // histogram handle concurrently; if the bounds aren't registered yet that + // handle would be built with the fallback bounds and keep them for the + // life of the process. `register_bounds()` doesn't need the recorder. + register_bounds(); + match metrics::set_global_recorder(recorder) { + Ok(()) => { + let _ = REGISTRY.set(registry); + describe_all(); + true + } + Err(_) => false, + } +} + +/// The catalog of described Lance metrics. Empty until the recorder is installed. +#[pyfunction] +pub fn lance_metrics_catalog() -> Vec { + CATALOG + .lock() + .unwrap() + .iter() + .map(|(name, desc)| PyMetricDescription { + name: name.clone(), + kind: desc.kind.as_str().to_string(), + unit: desc.unit.clone(), + description: desc.description.clone(), + }) + .collect() +} + +/// A point-in-time snapshot of every recorded metric. Empty until the recorder +/// is installed. The read is lock-free but walks every series and allocates, so +/// it runs with the GIL released. +#[pyfunction] +pub fn snapshot_lance_metrics(py: Python<'_>) -> Vec { + let Some(registry) = REGISTRY.get() else { + return Vec::new(); + }; + let points = py.detach(|| collect_points(registry)); + points.into_iter().map(PyMetricPoint::from).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use metrics::HistogramFn; + + fn bucket_count(buckets: &[(String, u64)], le: &str) -> u64 { + buckets + .iter() + .find(|(b, _)| b == le) + .map(|(_, c)| *c) + .unwrap_or_else(|| panic!("no bucket with le={le}")) + } + + #[test] + fn bucketed_histogram_records_cumulative_buckets() { + let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice())); + hist.record(0.05); // le=0.1 + hist.record(0.5); // le=1 + hist.record(0.5); // le=1 + hist.record(50.0); // +Inf + + let MetricValue::Histogram { + buckets, + count, + sum, + } = hist.snapshot() + else { + panic!("expected histogram"); + }; + + // Buckets are cumulative (Prometheus `le` semantics). + assert_eq!(bucket_count(&buckets, "0.1"), 1); + assert_eq!(bucket_count(&buckets, "1"), 3); + assert_eq!(bucket_count(&buckets, "10"), 3); + assert_eq!(bucket_count(&buckets, "+Inf"), 4); + assert_eq!(count, 4); + assert!((sum - 51.05).abs() < 1e-9); + } + + #[test] + fn bucketed_histogram_boundary_is_inclusive() { + let hist = BucketedHistogram::new(Arc::from([1.0f64].as_slice())); + hist.record(1.0); // exactly the bound -> le=1, not +Inf + let MetricValue::Histogram { buckets, .. } = hist.snapshot() else { + panic!("expected histogram"); + }; + assert_eq!(bucket_count(&buckets, "1"), 1); + assert_eq!(bucket_count(&buckets, "+Inf"), 1); + } + + #[test] + fn bucketed_histogram_boundary_is_inclusive_mid_range() { + // A value equal to a middle bound lands in that bucket, not the next. + let hist = BucketedHistogram::new(Arc::from([0.1f64, 1.0, 10.0].as_slice())); + hist.record(1.0); + let MetricValue::Histogram { buckets, .. } = hist.snapshot() else { + panic!("expected histogram"); + }; + assert_eq!(bucket_count(&buckets, "0.1"), 0); + assert_eq!(bucket_count(&buckets, "1"), 1); + assert_eq!(bucket_count(&buckets, "10"), 1); // cumulative, so still 1 + assert_eq!(bucket_count(&buckets, "+Inf"), 1); + } + + #[test] + fn recorder_aggregates_counters_with_labels() { + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3") + .increment(2); + metrics::counter!("test_requests_total", "operation" => "get", "scheme" => "s3") + .increment(3); + // A distinct label set must produce a separate point, not merge. + metrics::counter!("test_requests_total", "operation" => "put", "scheme" => "gs") + .increment(7); + }); + + let scalar = |attrs: &[(&str, &str)]| { + let points = collect_points(®istry); + let point = points + .into_iter() + .find(|p| { + p.name == "test_requests_total" + && attrs + .iter() + .all(|(k, v)| p.attributes.get(*k).map(String::as_str) == Some(*v)) + }) + .expect("counter recorded for label set"); + assert_eq!(point.kind, "counter"); + match point.value { + MetricValue::Scalar(v) => v, + _ => panic!("expected scalar"), + } + }; + + // Same labels aggregate; distinct labels stay separate. + assert!((scalar(&[("operation", "get"), ("scheme", "s3")]) - 5.0).abs() < 1e-9); + assert!((scalar(&[("operation", "put"), ("scheme", "gs")]) - 7.0).abs() < 1e-9); + } + + #[test] + fn recorder_records_gauges() { + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + // Gauges store the f64 bit pattern in a u64; the snapshot must decode it. + metrics::with_local_recorder(&recorder, || { + metrics::gauge!("test_gauge", "scheme" => "s3").set(3.5); + }); + + let points = collect_points(®istry); + let point = points + .iter() + .find(|p| p.name == "test_gauge") + .expect("gauge recorded"); + assert_eq!(point.kind, "gauge"); + assert!(matches!(point.value, MetricValue::Scalar(v) if (v - 3.5).abs() < 1e-9)); + } + + #[test] + fn recorder_falls_back_to_default_bounds() { + // A histogram with no registered bounds uses DEFAULT_BOUNDS. + let name = "test_unregistered_histogram"; + assert!(!HISTOGRAM_BOUNDS.read().unwrap().contains_key(name)); + + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::histogram!(name).record(0.02); + }); + + let points = collect_points(®istry); + let point = points.iter().find(|p| p.name == name).expect("recorded"); + let MetricValue::Histogram { buckets, count, .. } = &point.value else { + panic!("expected histogram"); + }; + assert_eq!(*count, 1); + // DEFAULT_BOUNDS yields one bucket per bound plus the implicit `+Inf`. + assert_eq!(buckets.len(), DEFAULT_BOUNDS.len() + 1); + // 0.02 falls in the le=0.025 bucket (the third DEFAULT_BOUNDS entry). + assert_eq!(bucket_count(buckets, "0.025"), 1); + assert_eq!(bucket_count(buckets, "0.01"), 0); + assert_eq!(bucket_count(buckets, "+Inf"), 1); + } + + #[test] + fn recorder_uses_registered_histogram_bounds() { + let name = "test_recorder_bounds_seconds"; + HISTOGRAM_BOUNDS + .write() + .unwrap() + .insert(name.to_string(), Arc::from([0.1f64, 1.0].as_slice())); + + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { + registry: registry.clone(), + }; + metrics::with_local_recorder(&recorder, || { + metrics::histogram!(name).record(0.05); + metrics::histogram!(name).record(5.0); + }); + + let points = collect_points(®istry); + let point = points.iter().find(|p| p.name == name).expect("recorded"); + let MetricValue::Histogram { buckets, count, .. } = &point.value else { + panic!("expected histogram"); + }; + assert_eq!(*count, 2); + assert_eq!(bucket_count(buckets, "0.1"), 1); + assert_eq!(bucket_count(buckets, "+Inf"), 2); + } + + #[test] + fn describe_populates_catalog() { + let name = "test_describe_catalog_total"; + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { registry }; + metrics::with_local_recorder(&recorder, || { + metrics::describe_counter!(name, Unit::Count, "a test counter"); + }); + + let catalog = CATALOG.lock().unwrap(); + let desc = catalog.get(name).expect("described"); + assert!(matches!(desc.kind, MetricKind::Counter)); + assert_eq!(desc.description, "a test counter"); + } + + #[test] + fn describe_all_covers_object_store_metrics() { + use lance_io::object_store::metrics as os; + + let registry = Arc::new(Registry::new(LanceStorage)); + let recorder = LanceRecorder { registry }; + metrics::with_local_recorder(&recorder, describe_all); + + let catalog = CATALOG.lock().unwrap(); + let kind = |name: &str| catalog.get(name).expect("described").kind; + // Every emitted object store metric must be described so the OTel + // bridge can create an instrument for it, including the gauge and the + // retryable counter that a plain request path might never emit. + assert!(matches!(kind(os::METRIC_REQUESTS), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_BYTES), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_ERRORS), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_THROTTLE), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_RETRYABLE), MetricKind::Counter)); + assert!(matches!(kind(os::METRIC_IN_FLIGHT), MetricKind::Gauge)); + assert!(matches!(kind(os::METRIC_DURATION), MetricKind::Histogram)); + } +} diff --git a/python/src/reader.rs b/python/src/reader.rs index f8917d4ff53..3c57c49da51 100644 --- a/python/src/reader.rs +++ b/python/src/reader.rs @@ -14,44 +14,125 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -use std::sync::Arc; - use arrow_array::{RecordBatch, RecordBatchReader}; use arrow_schema::{ArrowError, SchemaRef}; -use futures::lock::Mutex; -use futures::stream::StreamExt; +use futures::{lock::Mutex, stream::StreamExt}; +use tokio::sync::{mpsc, oneshot}; use lance::dataset::scanner::{DatasetRecordBatchStream, Scanner as LanceScanner}; use lance_io::stream::RecordBatchStream; use crate::rt; +const READER_CHANNEL_CAPACITY: usize = 4; + +enum ReaderMessage { + Batch(Result), + Finished, +} + /// Lance's RecordBatchReader -/// This implements Arrow's RecordBatchReader trait -/// which is then used for FFI to turn this into -/// an ArrowArrayStream in the Arrow C Data Interface +/// +/// The async scan is driven by one background producer for the lifetime of the +/// reader. The synchronous Arrow C stream consumer receives batches through a +/// channel with capacity four, avoiding a runtime task spawn and cross-thread +/// rendezvous for every batch while preserving backpressure. The channel can +/// queue four batches while the producer holds at most one more pending send. pub struct LanceReader { schema: SchemaRef, - /// We wrap stream in a mutex so we can call `next` in the background - /// executor while we still have a reference to the stream on the main thread. - stream: Arc>, + receiver: std::sync::Arc>>, + cancel_sender: Option>, + finished: bool, } impl LanceReader { - pub async fn try_new(mut scanner: Arc) -> ::lance::Result { - let stream = Arc::make_mut(&mut scanner).try_into_stream().await?; + pub async fn try_new(mut scanner: std::sync::Arc) -> ::lance::Result { + let stream = std::sync::Arc::make_mut(&mut scanner) + .try_into_stream() + .await?; + Ok(Self::from_stream(stream)) + } + + pub fn from_stream(mut stream: DatasetRecordBatchStream) -> Self { let schema = stream.schema(); - Ok(Self { + let (sender, receiver) = mpsc::channel(READER_CHANNEL_CAPACITY); + let (cancel_sender, mut cancel_receiver) = oneshot::channel(); + rt().spawn_background(None, async move { + loop { + let next = tokio::select! { + biased; + _ = &mut cancel_receiver => break, + _ = sender.closed() => break, + next = stream.next() => next, + }; + let (message, terminal) = match next { + Some(Ok(batch)) => (ReaderMessage::Batch(Ok(batch)), false), + Some(Err(error)) => (ReaderMessage::Batch(Err(ArrowError::from(error))), true), + None => (ReaderMessage::Finished, true), + }; + + let sent = tokio::select! { + biased; + _ = &mut cancel_receiver => false, + _ = sender.closed() => false, + result = sender.send(message) => result.is_ok(), + }; + if !sent || terminal { + break; + } + } + }); + Self { schema, - stream: Arc::new(Mutex::new(stream)), // needs tokio Runtime - }) + receiver: std::sync::Arc::new(Mutex::new(receiver)), + cancel_sender: Some(cancel_sender), + finished: false, + } } - pub fn from_stream(stream: DatasetRecordBatchStream) -> Self { - Self { - schema: stream.schema(), - stream: Arc::new(Mutex::new(stream)), + fn finish(&mut self) { + self.cancel_sender.take(); + self.finished = true; + } + + fn cancel_producer(&mut self) { + if let Some(cancel_sender) = self.cancel_sender.take() { + let _ = cancel_sender.send(()); } + self.finished = true; + } + + fn handle_receive_result( + &mut self, + result: pyo3::PyResult>, + ) -> Option> { + match result { + Ok(Some(ReaderMessage::Batch(Ok(batch)))) => Some(Ok(batch)), + Ok(Some(ReaderMessage::Batch(Err(error)))) => { + self.finish(); + Some(Err(error)) + } + Ok(Some(ReaderMessage::Finished)) => { + self.finish(); + None + } + Ok(None) => { + self.finish(); + Some(Err(ArrowError::ExternalError(Box::new( + std::io::Error::other("Lance reader producer terminated before end of stream"), + )))) + } + Err(error) => { + self.cancel_producer(); + Some(Err(ArrowError::ExternalError(Box::new(error)))) + } + } + } +} + +impl Drop for LanceReader { + fn drop(&mut self) { + self.cancel_producer(); } } @@ -59,17 +140,26 @@ impl Iterator for LanceReader { type Item = Result; fn next(&mut self) -> Option { - let stream = self.stream.clone(); - rt().spawn(None, async move { - let mut stream = stream.lock().await; - stream.next().await - }) - .transpose() - .map(|rs| match rs { - Ok(Ok(batch)) => Ok(batch), - Ok(Err(err)) => Err(ArrowError::from(err)), - Err(err) => Err(ArrowError::ExternalError(Box::new(err))), - }) + if self.finished { + return None; + } + let receiver = self.receiver.clone(); + let recv = async move { receiver.lock().await.recv().await }; + let result = match tokio::runtime::Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => { + // Tell Tokio that this worker will block before using the + // signal-aware cross-thread rendezvous. Without this, a task + // spawned onto the same runtime can remain in this worker's + // local queue and deadlock. + tokio::task::block_in_place(|| rt().spawn(None, recv)) + } + // A current-thread runtime cannot be the multi-threaded Lance + // runtime. Hand the receive to Lance's runtime instead of nesting + // block_on on the caller's runtime. + Ok(_) => rt().spawn(None, recv), + Err(_) => rt().block_on(None, recv), + }; + self.handle_receive_result(result) } } @@ -78,3 +168,253 @@ impl RecordBatchReader for LanceReader { self.schema.clone() } } + +#[cfg(test)] +mod tests { + use std::{ + sync::{Arc, mpsc::Sender}, + time::Duration, + }; + + use arrow_array::{Int32Array, RecordBatchReader, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use datafusion::{ + error::DataFusionError, + physical_plan::{SendableRecordBatchStream, stream::RecordBatchStreamAdapter}, + }; + use futures::stream; + + use super::*; + + fn make_reader( + schema: SchemaRef, + batches: impl futures::Stream> + Send + 'static, + ) -> LanceReader { + let stream: SendableRecordBatchStream = + Box::pin(RecordBatchStreamAdapter::new(schema, batches)); + LanceReader::from_stream(DatasetRecordBatchStream::new(stream)) + } + + #[test] + fn test_reader_preserves_schema_batches_and_end_of_stream() { + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let expected = (0..3) + .map(|batch_index| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![ + batch_index * 2, + batch_index * 2 + 1, + ]))], + ) + .unwrap() + }) + .collect::>(); + let batches = stream::iter(expected.clone().into_iter().map(Ok)); + let mut reader = make_reader(schema.clone(), batches); + + assert_eq!(reader.schema(), schema); + let actual = reader.by_ref().collect::, _>>().unwrap(); + assert_eq!(actual, expected); + assert!(reader.next().is_none()); + } + + #[test] + fn test_reader_propagates_stream_errors() { + let schema = Arc::new(Schema::empty()); + let batches = stream::once(async { + Err(DataFusionError::Execution( + "expected reader error".to_string(), + )) + }) + .chain(stream::poll_fn(|_| { + panic!("the stream must not be polled after its first error"); + })); + let mut reader = make_reader(schema, batches); + + let error = reader.next().unwrap().unwrap_err(); + assert!(error.to_string().contains("expected reader error")); + assert!(reader.next().is_none()); + } + + #[test] + fn test_reader_receive_error_cancels_producer_without_drop() { + pyo3::Python::initialize(); + let schema = Arc::new(Schema::empty()); + let (drop_sender, drop_receiver) = std::sync::mpsc::channel(); + let (started_sender, started_receiver) = std::sync::mpsc::channel(); + let batches = stream::once(async move { + let _drop_notify = DropNotify(drop_sender); + started_sender.send(()).ok(); + std::future::pending::>().await + }); + let mut reader = make_reader(schema, batches); + + started_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("the producer should poll the stream"); + let error = reader + .handle_receive_result(Err(pyo3::exceptions::PyKeyboardInterrupt::new_err( + "expected interrupt", + ))) + .unwrap() + .unwrap_err(); + assert!(error.to_string().contains("expected interrupt")); + drop_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("an interrupted receive should cancel the producer"); + assert!(reader.next().is_none()); + } + + #[test] + fn test_reader_reports_producer_panic_after_a_batch() { + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let expected = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + let batches = stream::iter([Ok(expected.clone())]).chain(stream::poll_fn(|_| { + panic!("expected producer panic"); + })); + let mut reader = make_reader(schema, batches); + + assert_eq!(reader.next().unwrap().unwrap(), expected); + let error = reader.next().unwrap().unwrap_err(); + assert!( + error + .to_string() + .contains("producer terminated before end of stream") + ); + assert!(reader.next().is_none()); + } + + struct DropNotify(Sender<()>); + + impl Drop for DropNotify { + fn drop(&mut self) { + self.0.send(()).ok(); + } + } + + #[test] + fn test_reader_drop_cancels_pending_stream() { + let schema = Arc::new(Schema::empty()); + let (drop_sender, drop_receiver) = std::sync::mpsc::channel(); + let (started_sender, started_receiver) = std::sync::mpsc::channel(); + let batches = stream::once(async move { + let _drop_notify = DropNotify(drop_sender); + started_sender.send(()).ok(); + std::future::pending::>().await + }); + let reader = make_reader(schema, batches); + + started_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("the producer should poll the stream"); + drop(reader); + drop_receiver + .recv_timeout(Duration::from_secs(1)) + .expect("dropping the reader should cancel and drop the producer stream"); + } + + #[test] + fn test_reader_bounds_wide_batch_read_ahead() { + let schema = Arc::new(Schema::new(vec![Field::new( + "payload", + DataType::Utf8, + false, + )])); + let (poll_sender, poll_receiver) = std::sync::mpsc::channel(); + let batch_schema = schema.clone(); + let batches = stream::unfold(0, move |batch_index| { + let poll_sender = poll_sender.clone(); + let batch_schema = batch_schema.clone(); + async move { + if batch_index == 10 { + return None; + } + poll_sender.send(batch_index).ok(); + let batch = RecordBatch::try_new( + batch_schema, + vec![Arc::new(StringArray::from_iter_values(std::iter::once( + "x".repeat(1024 * 1024), + )))], + ) + .unwrap(); + Some((Ok(batch), batch_index + 1)) + } + }); + let mut reader = make_reader(schema, batches); + + assert_eq!( + (0..5) + .map(|_| poll_receiver.recv_timeout(Duration::from_secs(1)).unwrap()) + .collect::>(), + [0, 1, 2, 3, 4] + ); + assert!( + poll_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err() + ); + + assert_eq!(reader.next().unwrap().unwrap().num_rows(), 1); + assert_eq!( + poll_receiver.recv_timeout(Duration::from_secs(1)).unwrap(), + 5 + ); + } + + #[test] + fn test_reader_can_be_consumed_from_background_runtime() { + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let expected = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + let batches = stream::iter([Ok(expected.clone())]); + let mut reader = make_reader(schema, batches); + + let actual = rt() + .spawn(None, async move { reader.next().unwrap().unwrap() }) + .unwrap(); + assert_eq!(actual, expected); + } + + #[test] + fn test_reader_can_be_consumed_from_current_thread_runtime() { + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Int32, + false, + )])); + let expected = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + let batches = stream::iter([Ok(expected.clone())]); + let mut reader = make_reader(schema, batches); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + let actual = runtime.block_on(async move { reader.next().unwrap().unwrap() }); + assert_eq!(actual, expected); + } +} diff --git a/python/src/rowids.rs b/python/src/rowids.rs new file mode 100644 index 00000000000..0ad83c90579 --- /dev/null +++ b/python/src/rowids.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::ops::Range; + +use arrow::array::{Array, UInt64Array, make_array}; +use arrow::compute::CastOptions; +use arrow::compute::kernels::cast::cast_with_options; +use arrow::datatypes::DataType; +use arrow::pyarrow::{FromPyArrow, ToPyArrow}; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_data::ArrayData; +use lance_table::format::RowIdMeta; +use lance_table::rowids::{RowIdSequence, read_row_ids, write_row_ids}; +use pyo3::basic::CompareOp; +use pyo3::exceptions::{PyNotImplementedError, PyTypeError, PyValueError}; +use pyo3::prelude::*; +use pyo3::types::{PyRange, PyRangeMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, intern}; + +use crate::error::PythonErrorExt; +use crate::fragment::PyRowIdMeta; + +/// The number of row ids shown in `RowIdSequence.__repr__` before eliding. +const REPR_PREVIEW_LEN: usize = 10; + +/// A sequence of stable row ids belonging to a single fragment. +#[pyclass(name = "RowIdSequence", module = "lance.fragment")] +pub struct PyRowIdSequence(pub RowIdSequence); + +#[pymethods] +impl PyRowIdSequence { + #[new] + fn new(row_ids: &Bound<'_, PyAny>) -> PyResult { + let sequence = match contiguous_range(row_ids)? { + // A `Range` segment is the most compact encoding, and taking it + // directly avoids materializing the ids of a large range. + Some(range) if range.is_empty() => RowIdSequence::new(), + Some(range) => RowIdSequence::from(range), + None => RowIdSequence::try_from_iter(extract_row_ids(row_ids)?).infer_error()?, + }; + Ok(Self(sequence)) + } + + /// Read back the sequence stored inline in fragment row id metadata. + #[staticmethod] + fn from_inline_metadata(metadata: PyRef<'_, PyRowIdMeta>) -> PyResult { + match &metadata.0 { + RowIdMeta::Inline(data) => read_row_ids(data).infer_error().map(Self), + RowIdMeta::External(_) => Err(PyNotImplementedError::new_err( + "Row ids stored in an external file cannot be read into a RowIdSequence", + )), + } + } + + /// Encode the sequence as row id metadata stored inline in the manifest. + fn to_inline_metadata(&self) -> PyRowIdMeta { + PyRowIdMeta(RowIdMeta::Inline(write_row_ids(&self.0).into())) + } + + fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { + let array = UInt64Array::from(self.0.iter().collect::>()); + array.into_data().to_pyarrow(py) + } + + fn __len__(&self) -> usize { + self.0.len() as usize + } + + fn __iter__(&self, py: Python<'_>) -> PyResult> { + let row_ids: Vec = self.0.iter().collect(); + Py::new(py, PyRowIdSequenceIterator(row_ids.into_iter())) + } + + fn __repr__(&self) -> String { + let len = self.0.len(); + let preview = self + .0 + .iter() + .take(REPR_PREVIEW_LEN) + .map(|row_id| row_id.to_string()) + .collect::>() + .join(", "); + if len > REPR_PREVIEW_LEN as u64 { + format!("RowIdSequence([{}, ...], len={})", preview, len) + } else { + format!("RowIdSequence([{}])", preview) + } + } + + fn __richcmp__( + &self, + other: &Bound<'_, PyAny>, + op: CompareOp, + py: Python<'_>, + ) -> PyResult> { + let Ok(other) = other.cast::() else { + return Ok(py.NotImplemented()); + }; + let equal = self.0 == other.borrow().0; + match op { + CompareOp::Eq => equal.into_py_any(py), + CompareOp::Ne => (!equal).into_py_any(py), + _ => Ok(py.NotImplemented()), + } + } + + fn __reduce__(&self, py: Python<'_>) -> PyResult<(Py, Py)> { + let from_inline_metadata = PyModule::import(py, "lance.fragment")? + .getattr("RowIdSequence")? + .getattr("from_inline_metadata")? + .extract()?; + let metadata = Py::new(py, self.to_inline_metadata())?; + let state = PyTuple::new(py, [metadata])?.extract()?; + Ok((from_inline_metadata, state)) + } +} + +/// The row ids are materialized up front because `RowIdSequence::iter` borrows +/// the sequence and a `#[pyclass]` cannot hold a borrowing iterator. +/// +/// Refilling a bounded buffer from `RowIdSequence::slice` looks like the way to +/// avoid that, but it is quadratic: slicing takes an absolute offset, and for +/// the gapped `RangeWithHoles` and `RangeWithBitmap` encodings the slice skips +/// its prefix one element at a time. Making iteration both lazy and linear +/// needs a forward cursor over the private segments, which belongs in +/// `lance-table` rather than here. Bulk callers should use `to_pyarrow`. +#[pyclass(name = "RowIdSequenceIterator", module = "lance.fragment")] +pub struct PyRowIdSequenceIterator(std::vec::IntoIter); + +#[pymethods] +impl PyRowIdSequenceIterator { + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } + + fn __next__(mut slf: PyRefMut<'_, Self>) -> Option { + slf.0.next() + } +} + +/// Recognize a `range` with a step of one, whose row ids need no materialization. +/// +/// Returns `None` for anything else, including strided and descending ranges, +/// which are handled by the general iterable path. +fn contiguous_range(ob: &Bound<'_, PyAny>) -> PyResult>> { + let Ok(range) = ob.cast::() else { + return Ok(None); + }; + // Bounds are read as `isize`. A range reaching past that is well beyond any + // dataset's row count, so it falls through to the element-wise path, which + // reads each value as a u64, rather than failing. + let (Ok(start), Ok(stop), Ok(step)) = (range.start(), range.stop(), range.step()) else { + return Ok(None); + }; + if step != 1 { + return Ok(None); + } + if start < 0 { + return Err(PyValueError::new_err(format!( + "Row ids must be non-negative, but the range starts at {}", + start + ))); + } + if stop <= start { + return Ok(Some(0..0)); + } + Ok(Some(start as u64..stop as u64)) +} + +fn extract_row_ids(ob: &Bound<'_, PyAny>) -> PyResult> { + let py = ob.py(); + if ob.hasattr(intern!(py, "__arrow_c_array__"))? { + return row_ids_from_arrow(ob); + } + // A `pyarrow.ChunkedArray`, such as a `_rowid` column taken from a table. + let chunks = intern!(py, "chunks"); + if ob.hasattr(chunks)? { + let mut row_ids = Vec::new(); + for chunk in ob.getattr(chunks)?.try_iter()? { + row_ids.extend(row_ids_from_arrow(&chunk?)?); + } + return Ok(row_ids); + } + + let iter = ob.try_iter().map_err(|_| { + PyTypeError::new_err(format!( + "Row ids must be an iterable of integers or an Arrow array, but got {}", + ob.get_type().name().map_or_else( + |_| "an object of unknown type".to_string(), + |name| name.to_string() + ) + )) + })?; + iter.map(|row_id| row_id?.extract::()).collect() +} + +fn row_ids_from_arrow(array: &Bound<'_, PyAny>) -> PyResult> { + let array = make_array(ArrayData::from_pyarrow_bound(array)?); + if !array.data_type().is_integer() { + return Err(PyTypeError::new_err(format!( + "Row ids must be an array of integers, but got an array of type {}", + array.data_type() + ))); + } + if array.null_count() > 0 { + return Err(PyValueError::new_err(format!( + "Row ids must not be null, but the array has {} null values", + array.null_count() + ))); + } + + // `safe: false` so that negative values raise instead of wrapping around + // into the top of the row id space. + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + let array = cast_with_options(&array, &DataType::UInt64, &cast_options) + .map_err(|err| PyValueError::new_err(format!("Row ids must fit in a uint64: {}", err)))?; + Ok(array.as_primitive::().values().to_vec()) +} diff --git a/python/src/scanner.rs b/python/src/scanner.rs index bbf1b3f35a3..8702537a340 100644 --- a/python/src/scanner.rs +++ b/python/src/scanner.rs @@ -66,6 +66,10 @@ pub struct ScanStatistics { pub parts_loaded: usize, /// Number of index comparisons performed pub index_comparisons: usize, + /// Number of index cache page lookups that were served from memory + pub index_cache_hits: usize, + /// Number of index cache page lookups that had to load from storage + pub index_cache_misses: usize, /// Additional metrics for more detailed statistics. These are subject to change in the future /// and should only be used for debugging purposes. pub all_counts: HashMap, @@ -80,6 +84,8 @@ impl ScanStatistics { indices_loaded: stats.indices_loaded, parts_loaded: stats.parts_loaded, index_comparisons: stats.index_comparisons, + index_cache_hits: stats.index_cache_hits(), + index_cache_misses: stats.index_cache_misses(), all_counts: stats.all_counts.clone(), } } @@ -89,13 +95,15 @@ impl ScanStatistics { impl ScanStatistics { fn __repr__(&self) -> String { format!( - "ScanStatistics(iops={}, requests={}, bytes_read={}, indices_loaded={}, parts_loaded={}, index_comparisons={}, all_counts={:?})", + "ScanStatistics(iops={}, requests={}, bytes_read={}, indices_loaded={}, parts_loaded={}, index_comparisons={}, index_cache_hits={}, index_cache_misses={}, all_counts={:?})", self.iops, self.requests, self.bytes_read, self.indices_loaded, self.parts_loaded, self.index_comparisons, + self.index_cache_hits, + self.index_cache_misses, self.all_counts ) } diff --git a/python/src/schema.rs b/python/src/schema.rs index db4f7369710..8cdc2115cd1 100644 --- a/python/src/schema.rs +++ b/python/src/schema.rs @@ -179,7 +179,8 @@ impl LanceSchema { fields: Fields(fields), metadata, }; - let schema = Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta) + .map_err(|err| PyValueError::new_err(format!("Failed to reconstruct schema: {err}")))?; Ok(Self(schema)) } diff --git a/python/src/session.rs b/python/src/session.rs index c91329ec1ee..99d3d904dcf 100644 --- a/python/src/session.rs +++ b/python/src/session.rs @@ -1,18 +1,41 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::collections::HashMap; use std::sync::Arc; -use pyo3::{pyclass, pymethods}; +use pyo3::exceptions::PyValueError; +use pyo3::types::{PyAnyMethods, PyDict, PyDictMethods, PyString}; +use pyo3::{Bound, PyAny, PyResult, pyclass, pymethods}; -use lance::dataset::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE}; -use lance::session::Session as LanceSession; +use lance::session::{CacheSpec, Session as LanceSession}; +use lance_core::cache::{BackendConfig, build_from_config, build_from_uri}; +use crate::object_store::PyObjectStoreRegistry; use crate::rt; /// The Session holds stateful information for a dataset. /// /// The session contains caches for opened indices and file metadata. +/// +/// Parameters +/// ---------- +/// index_cache_size_bytes : int, optional +/// Capacity of the default index cache in bytes. +/// metadata_cache_size_bytes : int, optional +/// Capacity of the default metadata cache in bytes. +/// index_cache_backend : str or dict, optional +/// Custom index cache backend. Strings are backend URIs such as +/// ``"moka://?capacity=1048576"``. Dicts must contain ``"kind"`` and may +/// contain ``"options"``, for example +/// ``{"kind": "moka", "options": {"capacity": "1048576"}}``. +/// metadata_cache_backend : str or dict, optional +/// Custom metadata cache backend with the same format as +/// ``index_cache_backend``. +/// +/// ``index_cache_backend`` is mutually exclusive with +/// ``index_cache_size_bytes``. ``metadata_cache_backend`` is mutually +/// exclusive with ``metadata_cache_size_bytes``. #[pyclass(name = "_Session", module = "_lib", from_py_object)] #[derive(Clone)] pub struct Session { @@ -25,22 +48,154 @@ impl Session { } } +/// Turn a Python-supplied backend descriptor into an `Arc`, +/// or return `Ok(None)` when the caller did not pass one. +/// +/// Accepts: +/// * `str` — treated as a URI (`moka://?capacity=...`) and passed to +/// [`build_from_uri`]. +/// * `dict` — must have string keys `kind` (required) and `options` +/// (optional `dict[str, str]`) matching [`BackendConfig`]; passed to +/// [`build_from_config`]. +/// +/// Any other Python type is rejected with a clear `TypeError`-style +/// `PyValueError`. +/// +/// If `size_field_set` is `true` and `backend` is `Some`, both a size and a +/// backend were provided for the same cache. Rather than silently letting +/// one override the other (Proposal §7), this is rejected up-front so the +/// operator gets an actionable error. +fn resolve_cache_spec( + backend_field: &str, + backend: Option<&Bound<'_, PyAny>>, + size_field: &str, + size: Option, +) -> PyResult { + if backend.is_some() && size.is_some() { + return Err(PyValueError::new_err(format!( + "{} and {} are mutually exclusive; set one or the other", + size_field, backend_field, + ))); + } + + let Some(value) = backend else { + return Ok(size.map(CacheSpec::Size).unwrap_or(CacheSpec::Default)); + }; + + if value.cast::().is_ok() { + let uri: String = value.extract()?; + return build_from_uri(&uri) + .map(CacheSpec::Backend) + .map_err(|e| PyValueError::new_err(format!("{}: {}", backend_field, e))); + } + + if let Ok(dict) = value.cast::() { + let cfg = backend_config_from_dict(backend_field, dict)?; + return build_from_config(&cfg) + .map(CacheSpec::Backend) + .map_err(|e| PyValueError::new_err(format!("{}: {}", backend_field, e))); + } + + let type_name: String = value.get_type().getattr("__name__")?.extract()?; + Err(PyValueError::new_err(format!( + "{}: expected str (URI) or dict with 'kind'/'options' keys, got {}", + backend_field, type_name, + ))) +} + +fn backend_config_from_dict(field: &str, dict: &Bound<'_, PyDict>) -> PyResult { + for (key, _) in dict.iter() { + if key.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: dict keys must be strings", + field + ))); + } + let key: String = key.extract()?; + if key != "kind" && key != "options" { + return Err(PyValueError::new_err(format!( + "{}: unknown dict key {:?}; expected 'kind' or 'options'", + field, key + ))); + } + } + + let kind_obj = dict.get_item("kind")?.ok_or_else(|| { + PyValueError::new_err(format!("{}: dict must contain a 'kind' key", field)) + })?; + if kind_obj.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: 'kind' must be a string", + field + ))); + } + let kind: String = kind_obj.extract()?; + + let mut options: HashMap = HashMap::new(); + if let Some(options_obj) = dict.get_item("options")? { + let options_dict = options_obj.cast::().map_err(|_| { + PyValueError::new_err(format!("{}: 'options' must be a dict[str, str]", field)) + })?; + for (k, v) in options_dict.iter() { + if k.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: 'options' keys must be strings", + field + ))); + } + if v.cast::().is_err() { + return Err(PyValueError::new_err(format!( + "{}: 'options' values must be strings", + field + ))); + } + let key: String = k.extract()?; + let value: String = v.extract()?; + options.insert(key, value); + } + } + + let mut config = BackendConfig::new(&kind) + .map_err(|e| PyValueError::new_err(format!("{}: {}", field, e)))?; + config.options = options; + Ok(config) +} + #[pymethods] impl Session { #[new] - #[pyo3(signature=(index_cache_size_bytes=None, metadata_cache_size_bytes=None))] + #[pyo3(signature=( + index_cache_size_bytes=None, + metadata_cache_size_bytes=None, + index_cache_backend=None, + metadata_cache_backend=None, + store_registry=None, + ))] fn create( index_cache_size_bytes: Option, metadata_cache_size_bytes: Option, - ) -> Self { - let session = LanceSession::new( - index_cache_size_bytes.unwrap_or(DEFAULT_INDEX_CACHE_SIZE), - metadata_cache_size_bytes.unwrap_or(DEFAULT_METADATA_CACHE_SIZE), - Default::default(), - ); - Self { + index_cache_backend: Option>, + metadata_cache_backend: Option>, + store_registry: Option, + ) -> PyResult { + let index_cache = resolve_cache_spec( + "index_cache_backend", + index_cache_backend.as_ref(), + "index_cache_size_bytes", + index_cache_size_bytes, + )?; + let metadata_cache = resolve_cache_spec( + "metadata_cache_backend", + metadata_cache_backend.as_ref(), + "metadata_cache_size_bytes", + metadata_cache_size_bytes, + )?; + let store_registry = store_registry.map(|r| r.inner).unwrap_or_default(); + let session = + LanceSession::with_cache_backends(index_cache, metadata_cache, store_registry); + Ok(Self { inner: Arc::new(session), - } + }) } fn __repr__(&self) -> String { @@ -63,6 +218,13 @@ impl Session { self.inner.size_bytes() } + /// Return the current size of the index cache in bytes. + pub fn index_cache_size_bytes(&self) -> PyResult { + rt().block_on(None, async move { + self.inner.index_cache_stats().await.size_bytes as u64 + }) + } + /// Return whether the other session is the same as this one. pub fn is_same_as(&self, other: &Self) -> bool { Arc::ptr_eq(&self.inner, &other.inner) diff --git a/python/src/transaction.rs b/python/src/transaction.rs index 1b659395099..b77d04e2513 100644 --- a/python/src/transaction.rs +++ b/python/src/transaction.rs @@ -7,10 +7,11 @@ use crate::utils::{PyLance, class_name, export_vec, extract_vec}; use arrow::pyarrow::PyArrowType; use arrow_schema::Schema as ArrowSchema; use lance::dataset::transaction::{ - DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, UpdateMap, - UpdateMapEntry, UpdateMode, + DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, + UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, }; use lance::datatypes::Schema; +use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; use lance_table::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata}; use pyo3::exceptions::PyValueError; use pyo3::types::PySet; @@ -92,11 +93,18 @@ impl FromPyObject<'_, '_> for PyLance { .map(|(type_url, value)| Arc::new(prost_types::Any { type_url, value })), Err(_) => None, }; + // Tolerate an object predating this attribute, as with `index_details` + // above: absent means the index carries no covered columns. + let covering_fields: Vec = match ob.getattr("covering_fields") { + Ok(value) => value.extract()?, + Err(_) => Vec::new(), + }; Ok(Self(IndexMetadata { uuid: Uuid::parse_str(&uuid).map_err(|e| PyValueError::new_err(e.to_string()))?, name, fields, + covering_fields, dataset_version, fragment_bitmap, index_details, @@ -121,6 +129,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&IndexMetadata> { let uuid = self.0.uuid.to_string(); let name = &self.0.name; let fields = &self.0.fields; + let covering_fields = &self.0.covering_fields; let dataset_version = self.0.dataset_version; let index_version = self.0.index_version; let fragment_ids = self.0.fragment_bitmap.as_ref().map_or_else( @@ -161,6 +170,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&IndexMetadata> { base_id, files, index_details, + covering_fields.clone(), )) } } @@ -206,6 +216,128 @@ impl<'py> IntoPyObject<'py> for PyLance<&DataReplacementGroup> { } } +// The Nth offset in an overlay list positionally maps to the Nth value row in +// `data_file`, but `RoaringBitmap` stores offsets in ascending order and drops +// duplicates. A caller-supplied list that isn't strictly ascending would be +// silently reordered, breaking that mapping, so reject it here instead. This can +// go away once we expose RoaringBitmap directly to Python (issue #7695). +fn bitmap_from_sorted_offsets(offsets: Vec) -> PyResult { + if offsets.windows(2).any(|w| w[0] >= w[1]) { + return Err(PyValueError::new_err( + "DataOverlayFile.offsets must be strictly ascending with no duplicates; \ + each offset positionally maps to a value row in data_file", + )); + } + Ok(RoaringBitmap::from_sorted_iter(offsets).expect("offsets verified strictly ascending")) +} + +impl FromPyObject<'_, '_> for PyLance { + type Error = PyErr; + fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { + let data_file = ob.getattr("data_file")?.extract::>()?.0; + let offsets = ob.getattr("offsets")?; + + // A flat list of offsets is a dense overlay (one coverage shared by every + // field); a list of per-field lists is a sparse overlay. Differentiate by + // shape, trying the dense form first. + let coverage = if let Ok(shared) = offsets.extract::>() { + OverlayCoverage::dense(bitmap_from_sorted_offsets(shared)?) + } else if let Ok(per_field) = offsets.extract::>>() { + OverlayCoverage::sparse( + per_field + .into_iter() + .map(bitmap_from_sorted_offsets) + .collect::>>()?, + ) + } else { + return Err(PyValueError::new_err( + "DataOverlayFile.offsets must be a list of ints (dense coverage shared by \ + every field) or a list of per-field int lists (sparse coverage)", + )); + }; + + // Present (and preserved) when round-tripping an existing fragment's + // overlays; None/0 when creating an overlay to commit, since the + // DataOverlay commit stamps the effective version. + let committed_version = ob + .getattr("committed_version")? + .extract::>()? + .unwrap_or(0); + + Ok(Self(DataOverlayFile { + data_file, + coverage, + committed_version, + })) + } +} + +impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayFile> { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let namespace = py + .import(intern!(py, "lance")) + .and_then(|module| module.getattr(intern!(py, "LanceOperation"))) + .expect("Failed to import LanceOperation namespace"); + + let data_file = PyLance(&self.0.data_file).into_pyobject(py)?; + let cls = namespace + .getattr("DataOverlayFile") + .expect("Failed to get DataOverlayFile class"); + + let committed_version = self.0.committed_version; + + // Mirror the read side: a dense overlay becomes a flat list of offsets, a + // sparse overlay a list of per-field lists. + match &self.0.coverage { + OverlayCoverage::Shared(bitmap) => { + let offsets: Vec = bitmap.iter().collect(); + cls.call1((data_file, offsets, committed_version)) + } + OverlayCoverage::PerField(bitmaps) => { + let offsets: Vec> = bitmaps.iter().map(|b| b.iter().collect()).collect(); + cls.call1((data_file, offsets, committed_version)) + } + } + } +} + +impl FromPyObject<'_, '_> for PyLance { + type Error = PyErr; + fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult { + let fragment_id = ob.getattr("fragment_id")?.extract::()?; + let overlays = extract_vec(&ob.getattr("overlays")?)?; + Ok(Self(DataOverlayGroup { + fragment_id, + overlays, + })) + } +} + +impl<'py> IntoPyObject<'py> for PyLance<&DataOverlayGroup> { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let namespace = py + .import(intern!(py, "lance")) + .and_then(|module| module.getattr(intern!(py, "LanceOperation"))) + .expect("Failed to import LanceOperation namespace"); + + let fragment_id = self.0.fragment_id; + let overlays = export_vec(py, self.0.overlays.as_slice())?; + + let cls = namespace + .getattr("DataOverlayGroup") + .expect("Failed to get DataOverlayGroup class"); + cls.call1((fragment_id, overlays)) + } +} + #[derive(Debug, Clone)] pub struct PyUpdateMode(pub UpdateMode); @@ -290,16 +422,41 @@ impl FromPyObject<'_, '_> for PyLance { .ok() .map(|py_mode| py_mode.0); + // Absent on objects predating the field. + let updated_fragment_offsets = ob + .getattr("updated_fragment_offsets") + .ok() + .map(|v| v.extract::>>>()) + .transpose()? + .flatten() + .map(|offsets| { + offsets + .into_iter() + .map(|(frag_id, bytes)| { + RoaringBitmap::deserialize_from(&bytes[..]) + .map(|bitmap| (frag_id, bitmap)) + .map_err(|e| { + PyValueError::new_err(format!( + "updated_fragment_offsets[{frag_id}]: invalid \ + portable RoaringBitmap bytes: {e}" + )) + }) + }) + .collect::>>() + }) + .transpose()? + .map(UpdatedFragmentOffsets); + let op = Operation::Update { removed_fragment_ids, updated_fragments, new_fragments, fields_modified, - merged_generations: vec![], + compacted_sstables: vec![], fields_for_preserving_frag_bitmap, update_mode, inserted_rows_filter: None, - updated_fragment_offsets: None, + updated_fragment_offsets, }; Ok(Self(op)) } @@ -311,7 +468,18 @@ impl FromPyObject<'_, '_> for PyLance { .extract::>>()?; let fragments = fragments.into_iter().map(|f| f.0).collect(); - let op = Operation::Merge { schema, fragments }; + // Absent on objects predating the field: no assertion, which + // conservatively conflicts. + let preserves_nullability = ob + .getattr("preserves_nullability") + .and_then(|v| v.extract()) + .unwrap_or(false); + + let op = Operation::Merge { + schema, + fragments, + preserves_nullability, + }; Ok(Self(op)) } "Restore" => { @@ -350,10 +518,26 @@ impl FromPyObject<'_, '_> for PyLance { Ok(Self(op)) } + "DataOverlay" => { + let groups = extract_vec(&ob.getattr("groups")?)?; + + let op = Operation::DataOverlay { groups }; + + Ok(Self(op)) + } "Project" => { let schema = extract_schema(&ob.getattr("schema")?)?; - - let op = Operation::Project { schema }; + // Absent on objects predating the field: no assertion, which + // conservatively conflicts. + let preserves_nullability = ob + .getattr("preserves_nullability") + .and_then(|v| v.extract()) + .unwrap_or(false); + + let op = Operation::Project { + schema, + preserves_nullability, + }; Ok(Self(op)) } "UpdateConfig" => { @@ -451,6 +635,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { fields_modified, fields_for_preserving_frag_bitmap, update_mode, + updated_fragment_offsets, .. } => { let removed_fragment_ids = removed_fragment_ids.into_pyobject(py)?; @@ -468,6 +653,21 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { }, None => "rewrite_rows", }; + let updated_fragment_offsets = + updated_fragment_offsets + .as_ref() + .map(|UpdatedFragmentOffsets(offsets)| { + offsets + .iter() + .map(|(frag_id, bitmap)| { + let mut buf = Vec::with_capacity(bitmap.serialized_size()); + bitmap + .serialize_into(&mut buf) + .expect("RoaringBitmap serialization cannot fail"); + (*frag_id, buf) + }) + .collect::>>() + }); let cls = namespace .getattr("Update") .expect("Failed to get Update class"); @@ -478,6 +678,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { fields_modified, fields_for_preserving_frag_bitmap, update_mode, + updated_fragment_offsets, )) } Operation::DataReplacement { replacements } => { @@ -487,6 +688,13 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { .expect("Failed to get DataReplacement class"); cls.call1((replacements,)) } + Operation::DataOverlay { groups } => { + let groups = export_vec(py, groups.as_slice())?; + let cls = namespace + .getattr("DataOverlay") + .expect("Failed to get DataOverlay class"); + cls.call1((groups,)) + } Operation::Delete { updated_fragments, deleted_fragment_ids, @@ -499,13 +707,17 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { .expect("Failed to get Delete class"); cls.call1((updated_fragments, deleted_fragment_ids, predicate)) } - Operation::Merge { fragments, schema } => { + Operation::Merge { + fragments, + schema, + preserves_nullability, + } => { let fragments_py = export_vec(py, fragments.as_slice())?; let schema_py = LanceSchema(schema.clone()); let cls = namespace .getattr("Merge") .expect("Failed to get Merge class"); - cls.call1((fragments_py, schema_py)) + cls.call1((fragments_py, schema_py, *preserves_nullability)) } Operation::Restore { version } => { let cls = namespace @@ -528,6 +740,7 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { Operation::CreateIndex { new_indices, removed_indices, + .. } => { let new_indices_py = export_vec(py, new_indices.as_slice())?; let removed_indices_py = export_vec(py, removed_indices.as_slice())?; @@ -537,12 +750,15 @@ impl<'py> IntoPyObject<'py> for PyLance<&Operation> { .expect("Failed to get CreateIndex class"); cls.call1((new_indices_py, removed_indices_py)) } - Operation::Project { schema } => { + Operation::Project { + schema, + preserves_nullability, + } => { let schema_py = LanceSchema(schema.clone()); let cls = namespace .getattr("Project") .expect("Failed to get Project class"); - cls.call1((schema_py,)) + cls.call1((schema_py, *preserves_nullability)) } Operation::ReserveFragments { num_fragments } => { if let Ok(cls) = namespace.getattr("ReserveFragments") { diff --git a/python/src/utils.rs b/python/src/utils.rs index 4f7d6d7dde2..5b3801e79aa 100644 --- a/python/src/utils.rs +++ b/python/src/utils.rs @@ -25,8 +25,7 @@ use arrow_schema::DataType; use lance::Result; use lance::datatypes::Schema; use lance_arrow::FixedSizeListArrayExt; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; -use lance_index::scalar::IndexWriter; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_index::vector::hnsw::{HNSW, builder::HnswBuildParams}; use lance_index::vector::kmeans::{ KMeans as LanceKMeans, KMeansAlgoFloat, KMeansParams, compute_partitions, @@ -243,7 +242,7 @@ impl Hnsw { let mut writer = rt() .block_on( Some(py), - PreviousFileWriter::::try_new( + V1FileWriter::::try_new( &object_store, &path, Schema::try_from(HNSW::schema().as_ref()) @@ -255,7 +254,7 @@ impl Hnsw { rt().block_on(Some(py), async { let batch = self.hnsw.to_batch()?; let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await?; + writer.write(&[batch]).await?; writer.finish_with_metadata(&metadata).await?; Result::Ok(()) })? diff --git a/python/uv.lock b/python/uv.lock index a8a7febeea4..e50986d9b2c 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -9,27 +9,18 @@ resolution-markers = [ "python_full_version < '3.11'", ] -[[package]] -name = "absl-py" -version = "2.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588, upload-time = "2025-07-03T09:31:44.05Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811, upload-time = "2025-07-03T09:31:42.253Z" }, -] - [[package]] name = "aiohappyeyeballs" -version = "2.6.1" +version = "2.7.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/30/f84a107a9c4331c14b2b586036f40965c128aa4fee4dda5d3d51cb14ad54/aiohappyeyeballs-2.6.1.tar.gz", hash = "sha256:c3f9d0113123803ccadfdf3f0faa505bc78e6a72d1cc4806cbd719826e943558", size = 22760, upload-time = "2025-03-12T01:42:48.764Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/15/5bf3b99495fb160b63f95972b81750f18f7f4e02ad051373b669d17d44f2/aiohappyeyeballs-2.6.1-py3-none-any.whl", hash = "sha256:f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8", size = 15265, upload-time = "2025-03-12T01:42:47.083Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, ] [[package]] name = "aiohttp" -version = "3.14.0" +version = "3.14.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohappyeyeballs" }, @@ -42,126 +33,126 @@ dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.13'" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/ab/93ce242f899b68c51b0578c027aafa791ab3614cb9345fa5d37b5f5c8e3e/aiohttp-3.14.0.tar.gz", hash = "sha256:2882de819734c715fd1b9c11c97e09fa020d14438203d1d354d8ed1702791c9b", size = 7940674, upload-time = "2026-06-01T19:41:02.763Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/f0/f81190ba488cd106c2fc6d92680e56bb223bbbbf1e6908c2617011290112/aiohttp-3.14.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:692e409052e7436029bbb32977cd7c5bf806ac5fa4085b973996785ffadad33c", size = 760606, upload-time = "2026-06-01T19:36:39.054Z" }, - { url = "https://files.pythonhosted.org/packages/f6/54/444d37eebf0f15db661ca44ec7caf93962f3c5ca92eb4c9a5d888b70aaa2/aiohttp-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40af7ebe53c7990e110dc4ad03566b12c3ac996254298a3d39046dd69cfcb2c2", size = 514677, upload-time = "2026-06-01T19:36:42.408Z" }, - { url = "https://files.pythonhosted.org/packages/d0/d1/da280e23321c132c0a3fa7c8cc2830621d79174edc64c829443346489a36/aiohttp-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02cb2ffbb7da32f82e21ad9952669c45bd88a80e0878264c2f59fe1c6fb2badd", size = 510155, upload-time = "2026-06-01T19:36:44.072Z" }, - { url = "https://files.pythonhosted.org/packages/09/b8/2e36d54d0991ec5bba451444004591ee0af58cb1662a3a81c562878b9c1f/aiohttp-3.14.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2514cb7195f6d7c219339635bea71ae47d1569b051300d32df9dcfabcdb869", size = 1699947, upload-time = "2026-06-01T19:36:45.762Z" }, - { url = "https://files.pythonhosted.org/packages/57/95/a31d8ea1a0b9ecc084f5a7dd0b431ce64ef585918bb7bdc82afe11843877/aiohttp-3.14.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:30e8b7eeb42d02c120ca90d6c6e076a221a16b70a6dac9ae44c7ab5104cc7fe4", size = 1664364, upload-time = "2026-06-01T19:36:47.653Z" }, - { url = "https://files.pythonhosted.org/packages/01/f6/5de3ddffc87a9e8d09b3be38fbd6dd1a736b2ad477a7e787dcb85f57f338/aiohttp-3.14.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63e38be0d75a654deaa06be32fb4cab883a4222940be1d05861b6717679cbadb", size = 1761186, upload-time = "2026-06-01T19:36:49.355Z" }, - { url = "https://files.pythonhosted.org/packages/33/8c/03c5438ec35d7e3a4f33fe895d6c3ec7540a7cec46065f21851211e1ee4d/aiohttp-3.14.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1210d4c87cc00128160c7384ab41877a701295b97cffa6362f908a49b6e8a7ca", size = 1849727, upload-time = "2026-06-01T19:36:51.478Z" }, - { url = "https://files.pythonhosted.org/packages/22/32/5a05303b0874458920b73f48b8779cc3a93d503f121b38dcc0456dbd698c/aiohttp-3.14.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a78a77366ed158a0a54b076990e575d7b7cdb728cbfd02711eadab150f2269f", size = 1708197, upload-time = "2026-06-01T19:36:53.241Z" }, - { url = "https://files.pythonhosted.org/packages/7d/62/478f169488d61414c0a05e7fe423b59ae3d9dcc933d1f0e4acc2c5d5bc3e/aiohttp-3.14.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f4d2038c64f36df96cfd3fa0937910e231eafbf897e70a06c155a817bb632fa6", size = 1578147, upload-time = "2026-06-01T19:36:55.154Z" }, - { url = "https://files.pythonhosted.org/packages/1d/af/b20af85765658972d3337834bd5eebba91b962794f2b4fc3e0ee8c85c0e1/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4714c70067a08b604d0bf3bc4dfdf82e52944afab41d0428d460862763d2f79b", size = 1665836, upload-time = "2026-06-01T19:36:56.94Z" }, - { url = "https://files.pythonhosted.org/packages/8d/a3/771879cfd59948f4544b172189048905feff802f20f1c6c5411e998a3e06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f79bfd2847513a7ac801bbafd1de02348a37926ac439eeb4bfe96fcff4eada15", size = 1680335, upload-time = "2026-06-01T19:36:58.642Z" }, - { url = "https://files.pythonhosted.org/packages/f4/16/582e36ad1d32133cd40659f3bc98e71c22179665a1cfbbb4713bce339c06/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:25e9f1d2465a210d60edb64d7b204a147e85d4c194eecef3d1604fb5ace678ce", size = 1731180, upload-time = "2026-06-01T19:37:00.583Z" }, - { url = "https://files.pythonhosted.org/packages/11/bc/80708fe3f64a07a2c306a42fc7b009118a952709761d215f6d1b4c57195b/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b5314743ebe926c2fda35d0a298c565c885505f6635c2a30936363404cf274a7", size = 1565805, upload-time = "2026-06-01T19:37:02.446Z" }, - { url = "https://files.pythonhosted.org/packages/57/8f/8d25897f8273a32fe4ad40a8885eec4f397377ed46e8e383078169f60316/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:28eee8de1d69711c53116df8202f1c2aa0e3f80ef912a88fc18d159d53e7110b", size = 1742496, upload-time = "2026-06-01T19:37:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/9f/7d/c341d32ab2dec56c8478740695743dc6c21b383cace9376a3eab16311a07/aiohttp-3.14.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:89ed35666c95d3efe1955056afcde09e62a57a34e2a4398b17f9f6c1564f0b25", size = 1691240, upload-time = "2026-06-01T19:37:06.277Z" }, - { url = "https://files.pythonhosted.org/packages/37/0f/a81207dd7a2d4a4f645b3a3f8b5a1da1159dc63117ffb137b698fd6df50f/aiohttp-3.14.0-cp310-cp310-win32.whl", hash = "sha256:5e4646e9a6af29af354204011bf5769cb0276ec5b64653e42f90b3e13845169f", size = 454686, upload-time = "2026-06-01T19:37:07.96Z" }, - { url = "https://files.pythonhosted.org/packages/7f/ae/842357f2afb9c915715c6f5775239d987f5d0f845abf7675fa794e0a9d40/aiohttp-3.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:22a8d06f204e0518a586d770032db3c7043c9ba3693081b3e3ad425e1458d594", size = 478677, upload-time = "2026-06-01T19:37:09.652Z" }, - { url = "https://files.pythonhosted.org/packages/6b/d1/330fb22c9535ec177b52396905131c6e39447244b6ca876262939af668ef/aiohttp-3.14.0-cp310-cp310-win_arm64.whl", hash = "sha256:4acfc34bd4d3c58754fc9f22ff1b5e92aabce68f3d4bf7b71a0b732d9bceb78a", size = 450364, upload-time = "2026-06-01T19:37:11.279Z" }, - { url = "https://files.pythonhosted.org/packages/67/47/7727bfe8db93f8835a001bd4359d8480cc68d1259b8bce334668f8be97bd/aiohttp-3.14.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:54bf3522d6f7351e55f89a62d5c2bf138ad557b031670266c5df604ae88e0b5a", size = 759147, upload-time = "2026-06-01T19:37:12.918Z" }, - { url = "https://files.pythonhosted.org/packages/eb/f2/cd3fedff6fade73d71df9ec908c210cec518ef90fd00289250684b90aecf/aiohttp-3.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0746d9fb0ac4fdef643a84494efe3f06d50335dd8c7a530228b86448aae0a803", size = 513705, upload-time = "2026-06-01T19:37:14.633Z" }, - { url = "https://files.pythonhosted.org/packages/5a/fe/49746b6b610144a06323bebd8e1211a390310d8c69b98dd6d52df341bc3e/aiohttp-3.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9f3a96b6d39a4872222beee72e1df41d2ff886ae96152cf3e757ef8c5673ef0e", size = 509627, upload-time = "2026-06-01T19:37:16.385Z" }, - { url = "https://files.pythonhosted.org/packages/4c/3f/28f2f6cf3d5c0e7b01b27140d0e7873fd11fb341169ad3ce78ad04aba628/aiohttp-3.14.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d336820adbb914debbc90a1d8c1bfc4bea55996aecf64866a989d35d1f9fd903", size = 1769293, upload-time = "2026-06-01T19:37:18.067Z" }, - { url = "https://files.pythonhosted.org/packages/97/6f/2e5f1b525d5474b12b3c60abf733a755845f3bceff21542081ada515f837/aiohttp-3.14.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:71b2604c9bfc1b115547d63a094d5244b3f02799833513a99a68aaa7b167c4cb", size = 1732363, upload-time = "2026-06-01T19:37:20.138Z" }, - { url = "https://files.pythonhosted.org/packages/a8/ce/596120faa85ca7b19cd061e3f2f3be23aa8f11a0aedf9191db9e0da1bd76/aiohttp-3.14.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:610d68800435903e303ca0542b9d3e4eb72a12ff33a6d471a070c1d81eebd3c2", size = 1840375, upload-time = "2026-06-01T19:37:22.104Z" }, - { url = "https://files.pythonhosted.org/packages/72/3c/a7ffe05a757a4a7867643da69357ec41f506879fbd1b231d2ed90af246b2/aiohttp-3.14.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:514db9a79337068981ee2137310283a07b4b885c584991097a91a4da419bcb81", size = 1921484, upload-time = "2026-06-01T19:37:24.068Z" }, - { url = "https://files.pythonhosted.org/packages/93/fa/2c861170bbd4a491de93a69e081db1d971092569e0d593a98ef62c384dc1/aiohttp-3.14.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c452d17eeb95d563fc8b936f3050301dbd1d268126c4632d8b70ede9696202ee", size = 1774153, upload-time = "2026-06-01T19:37:26.256Z" }, - { url = "https://files.pythonhosted.org/packages/9d/da/1d2f5a165f47ec9b1f69d37b8b977fdc4d501aa72ffb7930db27bb9e49ea/aiohttp-3.14.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ed94a81506e3d1bdbad5108f497a58f2a2354aedb4ca314d5326f07d1fd1ac2d", size = 1632569, upload-time = "2026-06-01T19:37:28.192Z" }, - { url = "https://files.pythonhosted.org/packages/46/1d/7a6e295c4257252f70f69e90864fdad74b6a1293054fb3f9e65a15de6d63/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1394dce36e0f0d260ac0b555a654de19cb989f3c1b8bdd24f505314dfea18a00", size = 1740325, upload-time = "2026-06-01T19:37:30.08Z" }, - { url = "https://files.pythonhosted.org/packages/f1/7e/e1899b1ca3ec62f1eab2a5cbde14039b97493f7f53eb88d9b668562ffa8d/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d1467d1e7b48a73ca7237e0ee4335f3d02b923dbc27b82fd254bc301c97d4026", size = 1748691, upload-time = "2026-06-01T19:37:32.211Z" }, - { url = "https://files.pythonhosted.org/packages/ec/54/4e6b61c1fe7d3433f82bcc6bd7e4d7c683a742a10c9b12a025fd3695c047/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6a5f3532125233c261cf61f32df4059cfcf482eb793c7d3db8452e3142028b86", size = 1814477, upload-time = "2026-06-01T19:37:34.173Z" }, - { url = "https://files.pythonhosted.org/packages/9c/38/86fd51be2e08d8e45c83d879d255f10391903cd9fe2a16512f7591a15873/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3ea81eb518a2ecb319d8ec6d1424a37c773f6634bd87d6985eb606b2faac419f", size = 1623393, upload-time = "2026-06-01T19:37:36.281Z" }, - { url = "https://files.pythonhosted.org/packages/78/49/466e947a42a88ee23c486d036e7e5d1b097f1bafd8084ad9c9a0a92f0f43/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:32e735c3182de7b64f6941a4ede48b38c7f47d9437bd615dd30b5bda8fa1bc93", size = 1824097, upload-time = "2026-06-01T19:37:38.421Z" }, - { url = "https://files.pythonhosted.org/packages/f3/89/35f3410bc284682338a1be6b6ea0c5abfa05f063942cfaa9256608440434/aiohttp-3.14.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c21ca9a1c63d4509158f478aeb9d02914dcc52adc68d1bc9dee2452284ee5996", size = 1764790, upload-time = "2026-06-01T19:37:40.755Z" }, - { url = "https://files.pythonhosted.org/packages/42/80/2d4291bd5724d3d17e5951aff5a3e02281483fb47295f0788276ee66cd73/aiohttp-3.14.0-cp311-cp311-win32.whl", hash = "sha256:19ca5fc84130675ba11c6ca5c7da5cb65f7bf8a32cdd2b616bf49cd334688aae", size = 454176, upload-time = "2026-06-01T19:37:42.837Z" }, - { url = "https://files.pythonhosted.org/packages/59/ed/41d0ad4f6ececffc32bdf1f7b494e5498f7ca5c849ea2e3cc9bbd1668251/aiohttp-3.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:d488e6e9d3bb8ba5ae7066d5be885ae9670eba021b8c6ccb9a3a568e6b19d6e5", size = 479334, upload-time = "2026-06-01T19:37:44.776Z" }, - { url = "https://files.pythonhosted.org/packages/d1/86/c0b5e305c770053f8c3d069bb52b8196917ba91949d1962d52eb307fb0d2/aiohttp-3.14.0-cp311-cp311-win_arm64.whl", hash = "sha256:8b93618102caf12801638a01a2b478a55410ddd71bd41cfaf6f707953a49ac43", size = 450262, upload-time = "2026-06-01T19:37:46.461Z" }, - { url = "https://files.pythonhosted.org/packages/89/97/2b6889bfb6b6847520d50d95eb8c4307a45e28aaca39faf4a9454b3d1b2f/aiohttp-3.14.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b29518c9c2ec7e373e68259206a137c7f4f5439c58baaec4b5ab3ab799850a4e", size = 750194, upload-time = "2026-06-01T19:37:48.164Z" }, - { url = "https://files.pythonhosted.org/packages/21/e2/62634b7fff918ed98c3c6b2f0e70d520f7f28846cb412d451b04354c6459/aiohttp-3.14.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:dbec68ce61b64cb73cab4d33df9433427b1713c8bcccb181dce695c1b6f8e87c", size = 506966, upload-time = "2026-06-01T19:37:50.014Z" }, - { url = "https://files.pythonhosted.org/packages/dd/fb/5ce075150828c797a5106f1c2fb26034e709d4289b9d2bf8b07f1e59fac6/aiohttp-3.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3cdf534aa455593e589302990c5097aa5c92c06c4262a20da22934f9186a5fff", size = 507527, upload-time = "2026-06-01T19:37:51.96Z" }, - { url = "https://files.pythonhosted.org/packages/01/d5/405a0ae4e6b081754a3609c1c97c63a950e000a2def16046f1e736933a0e/aiohttp-3.14.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cb6c657104393b5fbff01a5f59b2023db74058a8077d94475d6c25d03882a108", size = 1762420, upload-time = "2026-06-01T19:37:53.839Z" }, - { url = "https://files.pythonhosted.org/packages/ae/1d/e05a7c896b15a6bc6fb8fc5319eb437861c2c49c34559ef928add6590315/aiohttp-3.14.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:46fbbec4e4fab7428d4396a3823f9320e4560aa3113b89eeebce712c27c9ed5a", size = 1733672, upload-time = "2026-06-01T19:37:55.791Z" }, - { url = "https://files.pythonhosted.org/packages/cc/22/a72f7c459e195fa41bf4f7abd1f925b91fe91f8097e51c654229ba144a33/aiohttp-3.14.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c2c7e05dd5335b298085abf45ddf98673934c3ee1c083d0b9ea13d4186ad500", size = 1805064, upload-time = "2026-06-01T19:37:57.931Z" }, - { url = "https://files.pythonhosted.org/packages/80/50/e85bdaba0be59ca4838005ebfef4048fcdd5f35a02b07057a9a123394440/aiohttp-3.14.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3c7139100fbaae76515b73051d8f0aa3a3ff02e415eec8a8eee8e2223d9ba955", size = 1902125, upload-time = "2026-06-01T19:38:00.225Z" }, - { url = "https://files.pythonhosted.org/packages/19/d8/51de5c6b971c27bb1ef620293b8d1ca611ec78736b34b3f6ccf68e4c8785/aiohttp-3.14.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78d6f9286a629ce52728430afe18f8ed2b6c39a1fddb3802d7244b9983910ad2", size = 1783112, upload-time = "2026-06-01T19:38:02.641Z" }, - { url = "https://files.pythonhosted.org/packages/73/ae/b4402bfde77e43dfb1b6ccff83c7b7ab63ed06b50c4754f0c5423fb374fe/aiohttp-3.14.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc3c3e12cdaeb92d7dcf13db00e9f6b1956b910e47256e696df1cfa946d02159", size = 1586356, upload-time = "2026-06-01T19:38:04.637Z" }, - { url = "https://files.pythonhosted.org/packages/bc/05/750a3265ca4dc54a460bd0cb1121a8f2ce9171fce4a135fb47ea7fd594d2/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4d6a998191f5ebe3b8c28463ff72bc030250008b3193c402464efadd08b5ca02", size = 1723119, upload-time = "2026-06-01T19:38:06.713Z" }, - { url = "https://files.pythonhosted.org/packages/37/01/8c0812c50b3b1b1c37b323bf170d6be8847a8f234060485b7d1e71953f60/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0fc2b75ae8d169d853be2862d960be8550da6c5c65711d5476407eb3fdb006bd", size = 1757216, upload-time = "2026-06-01T19:38:08.736Z" }, - { url = "https://files.pythonhosted.org/packages/47/2a/50fb98028a26887cbe48dcc1df92a90825615bc73b5584301304090cded8/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:16eee56bcc72d04600bc56c1759982c2385ec0b41d3fd3521f836bf64a0957ef", size = 1770500, upload-time = "2026-06-01T19:38:11.111Z" }, - { url = "https://files.pythonhosted.org/packages/bd/32/0ffd598a2fa2b9a423daf242e700cfdabda35d6e602394ad9ae58972c1c7/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5a2e7ca615c3ddc15b82687e05a624e5f5cba3f1d6c20cb81172d70ea498451e", size = 1576224, upload-time = "2026-06-01T19:38:13.391Z" }, - { url = "https://files.pythonhosted.org/packages/0b/f9/b9fc381dd9b66afb33f2634c40e229d106467be0afcabe79648631ab6712/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:f0b7b8bbbec3ce9467ee0ebe334622fd90624f593edd3136c567811453fc4fae", size = 1794252, upload-time = "2026-06-01T19:38:15.498Z" }, - { url = "https://files.pythonhosted.org/packages/a8/fb/05d9214c975f23225a8cd5c439325e338c7c377b315480ef3871db51f54e/aiohttp-3.14.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5ba10966d4f03dd96a14365be4b8e37c327c76f11c3ca867116966cdd9f98066", size = 1760193, upload-time = "2026-06-01T19:38:17.624Z" }, - { url = "https://files.pythonhosted.org/packages/d9/4b/02992fc4fb9e1b6673ee3f888a8e587a6447afda1f6f4aca776c148c2876/aiohttp-3.14.0-cp312-cp312-win32.whl", hash = "sha256:101df7779c80c0636014a6b2c6642acd3efb5b355d48347c9d7dfb720aee9430", size = 448650, upload-time = "2026-06-01T19:38:19.545Z" }, - { url = "https://files.pythonhosted.org/packages/39/e9/246532214c3abda518477cbaaf16d420295ad8effa5233844cbb38f299ab/aiohttp-3.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:b0a5747586d4467efd1f932710b269131c9717a872dce082cd92a00c1c13123a", size = 476145, upload-time = "2026-06-01T19:38:21.505Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c3/63f8c20090048915711598b0adf475b149216d736157961de06480a45b15/aiohttp-3.14.0-cp312-cp312-win_arm64.whl", hash = "sha256:5f1c5be60add78fabb4aacd13c5a348ae79d2fcbfc7fa78da8f1eb192273b370", size = 444250, upload-time = "2026-06-01T19:38:24.027Z" }, - { url = "https://files.pythonhosted.org/packages/21/61/d11f7d9a3144bffe825247d6367cd93053666da50b94707c9129c78868d5/aiohttp-3.14.0-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:25400d710641a8040bf022a8a99f579e581ffa1c5bd42c33255d7d6f3957c127", size = 502399, upload-time = "2026-06-01T19:38:25.955Z" }, - { url = "https://files.pythonhosted.org/packages/4f/9b/a7e317625d36356844f8bb022cabd305b541f968856cc3c2e0b58e53ee6e/aiohttp-3.14.0-cp313-cp313-android_21_x86_64.whl", hash = "sha256:c5492b9929826e07cc3fcb9739ae87aab05dff6b5e67a9b73fd1700c6d008981", size = 510068, upload-time = "2026-06-01T19:38:27.828Z" }, - { url = "https://files.pythonhosted.org/packages/11/41/cc2d2cfbfbdc3126ba258f3cd27d1ac8a33492ae3c35a4583ee21f0ba7f1/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3366751d68d237c621264233a32f3078bbc21b7904ab90a77e03d21390c742c6", size = 481670, upload-time = "2026-06-01T19:38:29.836Z" }, - { url = "https://files.pythonhosted.org/packages/3c/07/381f4023c3b08cb616e520f566d8c58957abad54e56441d41fe67cfb0195/aiohttp-3.14.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:57ea07d28695a7a40304d42251892a8df765e5588c10ee32afeddcd5df33c0a2", size = 487591, upload-time = "2026-06-01T19:38:31.704Z" }, - { url = "https://files.pythonhosted.org/packages/fb/4d/4506fdb7a022bdf70011a3bbb4ca00c5c570026ef6a3c5bd7bc70c39089c/aiohttp-3.14.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:076cb014191ae2e65d949e1ad01f1dcfe33e32789b5172510f3e79c79fc04d50", size = 496503, upload-time = "2026-06-01T19:38:33.6Z" }, - { url = "https://files.pythonhosted.org/packages/ef/7d/c814111e04894a45d9e2defc94443879a6f118d9633d5fedfe6e2e8af5f0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2f3fc37054564dee64a855b5b092d87ec35dcddfaabf7dacb1c8a2b1f83dc0a9", size = 745870, upload-time = "2026-06-01T19:38:36.013Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ee/80eee0efddfe187e7cd05027086b7ce1c0e492e82a4eda58f5c5543a44a0/aiohttp-3.14.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8fcaef74d2ab0f607d7ff85a0d15e21bb5a258c4a58df1908396eb50d7f4ed3c", size = 505588, upload-time = "2026-06-01T19:38:38.282Z" }, - { url = "https://files.pythonhosted.org/packages/d6/f8/0f28f04eef75d52fc9c715dde7ce9c0abb810fd20cfeb0fea7afd2ab1e98/aiohttp-3.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e4c01b0bfc6209590960e68eac083cd22d5d87c21f974dd6208cafa5d3542bc8", size = 504492, upload-time = "2026-06-01T19:38:40.611Z" }, - { url = "https://files.pythonhosted.org/packages/ff/db/44c755232085545065c94378dfce38641b1aee647f4939fcd32f5b32e719/aiohttp-3.14.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f12eb7896e81caf403a2b18c9406426f1207361e7239c057ab29c076d4257e83", size = 1752111, upload-time = "2026-06-01T19:38:42.682Z" }, - { url = "https://files.pythonhosted.org/packages/5e/6a/42e030a46743841414402a3b00cd3d78419055e86c66fb5822c14b5abfc6/aiohttp-3.14.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6c79a044cacf360ec46738d863d2f41c9300d2a06ef4a7402ea0df306a350e61", size = 1729674, upload-time = "2026-06-01T19:38:44.79Z" }, - { url = "https://files.pythonhosted.org/packages/34/26/3199beb415202e3108e7b83ecebe10914d806d33fb9860c3e4aa60a19be3/aiohttp-3.14.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:85e0675f47be4eff0636bf88c02140ea89168ae0df3ff1f3f464e9de9610d277", size = 1798808, upload-time = "2026-06-01T19:38:47.01Z" }, - { url = "https://files.pythonhosted.org/packages/bd/94/b9b6fcf0ee17c21d0d19fb8c22bf83ad18f82e702a9c3bd901a868f5e446/aiohttp-3.14.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b33e751cab03fdc960095b1e326cb5a03f5ee577d6ded59f3d1c100f8668882", size = 1891921, upload-time = "2026-06-01T19:38:49.233Z" }, - { url = "https://files.pythonhosted.org/packages/c5/a3/3800dbd095cb2bb165a7ea5d94d790914677e27f45638c7d80e3f34c8945/aiohttp-3.14.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26d9224c6dd7f5c749aba4f61315a894601448b28d94d12f4dea0903e26d2096", size = 1777241, upload-time = "2026-06-01T19:38:52.04Z" }, - { url = "https://files.pythonhosted.org/packages/21/2a/45be91ad1b860508557448d4cc2e165a2ee68dd865657b73bf66cc5a00fb/aiohttp-3.14.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6281aecdf2732940f4fe06bd6adec5ae4d59b78b080b8e3a6b81467301010988", size = 1579554, upload-time = "2026-06-01T19:38:54.508Z" }, - { url = "https://files.pythonhosted.org/packages/b4/3d/dc94df99ed1511fdf28314f722643ed334112643cab00223577085e788c4/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:23e8314e7aed8576fbe33314d218bd81447a3adbc91dc36f1163bf583cd3084c", size = 1714864, upload-time = "2026-06-01T19:38:56.788Z" }, - { url = "https://files.pythonhosted.org/packages/ae/e4/1f1c8acbb3acd5c8f795473b92c9c3d44eb60a5692c6104256c8a1c83a0c/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3b54fbff46127aeafdd764cecd0d99fa2f24a0e37ea5c18a7c3a4ac450df1db3", size = 1749803, upload-time = "2026-06-01T19:38:59.367Z" }, - { url = "https://files.pythonhosted.org/packages/0b/c8/c45ea6e7ed84cebba939b9c334498a045ba19d79c61b0110df5f21580de3/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b27d89af91a555f58e08e4902dbcbc48862fd40095720ca705990476bd93b7ac", size = 1765023, upload-time = "2026-06-01T19:39:01.651Z" }, - { url = "https://files.pythonhosted.org/packages/a8/a1/a932941784432962fe390e1066823aaef64b4e5ac9fa595df57b5fe472a9/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:25d2326a4967bf705a9f9913a13005e93b6020ad8a9f6bd6bd78850d5171332e", size = 1571671, upload-time = "2026-06-01T19:39:04.044Z" }, - { url = "https://files.pythonhosted.org/packages/b0/01/e1280feac522597a4d46eb67a0cdfa053cfae263033030b761ab146f29fb/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1d209375c503472b3c0a340cdf3c55fcd82e84b46dda7caeaced59faba373ec", size = 1789904, upload-time = "2026-06-01T19:39:06.294Z" }, - { url = "https://files.pythonhosted.org/packages/fa/10/ab28818262f4d26bdb47ed5f1fc7999b69e2fc6e0370b02d0f49011f45ea/aiohttp-3.14.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:666c7c5036df57b693026398b69b41874a1931ac5b3485fd910e57bfac253869", size = 1754516, upload-time = "2026-06-01T19:39:08.788Z" }, - { url = "https://files.pythonhosted.org/packages/af/cc/c122eabd7a1b7e0c9bbdd6be60e4715905b858399145d9df872bb94f1427/aiohttp-3.14.0-cp313-cp313-win32.whl", hash = "sha256:23f094a1ef64823fd35854ddf5c7a80a078162f37f9d2f7c6142b51a6affa456", size = 448656, upload-time = "2026-06-01T19:39:11.171Z" }, - { url = "https://files.pythonhosted.org/packages/41/a5/bab07d79848a00eedd8ed979ccb302aaea3ac6eb9fa16bd0ed87135869b4/aiohttp-3.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:e03abdaa17d553f17e1d1d06bb266b3970106c78051d06795723e748d8e49d11", size = 475803, upload-time = "2026-06-01T19:39:13.439Z" }, - { url = "https://files.pythonhosted.org/packages/d1/a0/f03ade8566c153666a3871afccbedf6d99911da006325e1fc6cf72a2de99/aiohttp-3.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:acdb400538cf4769543548bb5d1eb23d39bed4f96554a6078cb728c7cb2c268b", size = 443889, upload-time = "2026-06-01T19:39:15.945Z" }, - { url = "https://files.pythonhosted.org/packages/28/03/5f36ab196a88ba5e9648ae5643e6531e67a3a8c0e96f9c6510ff41540fec/aiohttp-3.14.0-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:363ef9e91014e7891679bfb2ac0a7c6ea93435dbbfd10ecf41b9f06fcf506c5f", size = 503330, upload-time = "2026-06-01T19:39:18.195Z" }, - { url = "https://files.pythonhosted.org/packages/2c/ce/8b49ec2f30f68e02f314f4832186cd45e583360a5a386058be36855d23b6/aiohttp-3.14.0-cp314-cp314-android_24_x86_64.whl", hash = "sha256:884a4edbdad77be9d0ef36142c8b504351b170df0bf62b51e784fadabf311c42", size = 509822, upload-time = "2026-06-01T19:39:20.396Z" }, - { url = "https://files.pythonhosted.org/packages/1a/fe/6edbf5d39bf29322b6816365b17ed8ede4dace164a3aea1abcd30110eb78/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:70ea956f6cc4a37620966b56c2e205d88ca3e6d85ec063277e414b1035cddad3", size = 483329, upload-time = "2026-06-01T19:39:22.607Z" }, - { url = "https://files.pythonhosted.org/packages/1b/5a/fae531bdbc6456fb6241f46b7b81e4d8a0dd3fc09118a0055dc7141ac1ec/aiohttp-3.14.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:ea3b9806c89f61da22fddf1f12dd524fb368e5e28f1261fbdafe5c3cd8ce893b", size = 489502, upload-time = "2026-06-01T19:39:24.881Z" }, - { url = "https://files.pythonhosted.org/packages/36/f4/48a7b0414db7fed77a03d5dde34508c026afd83510ab6bca08c313855776/aiohttp-3.14.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a071be341c2bd9b0188e62d173509f024e0a35b1c342c53c50f8daaeda8c3bd8", size = 497357, upload-time = "2026-06-01T19:39:27.197Z" }, - { url = "https://files.pythonhosted.org/packages/75/75/e85a13a370acc007fca5feb1fd1b88ac2d8426e6dadd625479b7cadd55a3/aiohttp-3.14.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:198cfe61bf253b19da1fb3e0fa122249dc4f14c12709493fed8054aa0411cc76", size = 750898, upload-time = "2026-06-01T19:39:29.563Z" }, - { url = "https://files.pythonhosted.org/packages/9e/e4/3d637f800c724eff0e2bed64df72557444482366fd0a35b0cec0e6968f6c/aiohttp-3.14.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9dc203d6ce6b9106d54e2a93f41dfdfebfbca2d99962ba503bfd3e5921a6549e", size = 506986, upload-time = "2026-06-01T19:39:31.872Z" }, - { url = "https://files.pythonhosted.org/packages/1d/df/35161f3598bf7501d2b2a805b41ab4f45a2e34150c421bcb4ef8c0d281a7/aiohttp-3.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9e19d17ab02bf16832a2c8c0d55a486792c5b1645665652ee9531aebcc30cb72", size = 508033, upload-time = "2026-06-01T19:39:34.137Z" }, - { url = "https://files.pythonhosted.org/packages/e5/39/b36e5d3d31e850fb4691dd3e941684ac490a2559249f6fa634b6b0fdf020/aiohttp-3.14.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d925fba0c14d5b498a8028b0107beebdfd16c5d48d702ff54f879cb017aaaca3", size = 1746213, upload-time = "2026-06-01T19:39:36.654Z" }, - { url = "https://files.pythonhosted.org/packages/b1/28/24e1409e605a9aa5d84abe0e2acb365354b70ae56d40948101cabe3341ab/aiohttp-3.14.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d33e61021222ce7f9792bcac870d6f58d8adfceda33ab857b01264f4560f2c5f", size = 1705862, upload-time = "2026-06-01T19:39:38.968Z" }, - { url = "https://files.pythonhosted.org/packages/8c/d0/e5eb3ff1daeaf644c7e36a957517672494122628e067c38b263fa04eda77/aiohttp-3.14.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:44eca38755d0105bb32f47d085f5dd449846a449e1245fc105889e3279dcf8e3", size = 1798909, upload-time = "2026-06-01T19:39:41.334Z" }, - { url = "https://files.pythonhosted.org/packages/d3/ba/8943f906f0570342886ababb9a722a44e360f786a028c5e0b0e29e3f735b/aiohttp-3.14.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f13087e06f68fea4941c21a0c541c00553aa16e4f8fd7bbe2b198df761e964d6", size = 1868892, upload-time = "2026-06-01T19:39:43.807Z" }, - { url = "https://files.pythonhosted.org/packages/3a/05/27df32c844b2156e1675a8d8ec22d963e3c8ba469ed7ceb1863320c7b521/aiohttp-3.14.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff82be7f1ef73634cb77890a770743239bc3d487b848669be1c599889336dc0a", size = 1751659, upload-time = "2026-06-01T19:39:46.398Z" }, - { url = "https://files.pythonhosted.org/packages/7f/62/da182e5910ab912b2e88aa919b61a16046a37a95714a5795b02eb57b2d18/aiohttp-3.14.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a150c0875ac8fd87f1c398650841308a30d65facf7416b12dbdb9cfdcbe5a48c", size = 1578775, upload-time = "2026-06-01T19:39:48.902Z" }, - { url = "https://files.pythonhosted.org/packages/66/e3/53c67097e8a5ce98625e91e3fa7f43c9c6940de680345d03b3509a72a078/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:edc01ea4e1ec5a1649a28866262bf24195889ff7b27bdd947029a6086741de9b", size = 1710090, upload-time = "2026-06-01T19:39:51.392Z" }, - { url = "https://files.pythonhosted.org/packages/dd/55/0e2732ca598c7a4dfe8a775662376d0ca2977cb1030e48386d4da5d9a456/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:540632bf882ff8fc88f2e1697be0761578e89e0d79fb4a8a6d65dc5da7e729d4", size = 1715016, upload-time = "2026-06-01T19:39:53.807Z" }, - { url = "https://files.pythonhosted.org/packages/5a/96/f0b73730798c9ca525afc30b39f1f81bbe24e245d9654c54d3b39d63212d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:860a86bc2c80237f5dff52edcf427e10a8d8352271fd84845429a3e60199e02c", size = 1763810, upload-time = "2026-06-01T19:39:56.31Z" }, - { url = "https://files.pythonhosted.org/packages/71/cc/11acb6c4518f448323405a7312b6f255d0f974a34373ad1db7633c4aadc8/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cbd50e6a50d6b99283a826b18cbdebf65b0797689a7535cb0e9dd37be0f63c3", size = 1573064, upload-time = "2026-06-01T19:39:58.718Z" }, - { url = "https://files.pythonhosted.org/packages/de/2d/28c31dde0a7dc98c0ee7d0da2ddcec3f7688c4fc131e5989e278d0c03c0a/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:20144819e99db593e22bbd2f3f2691a5e149f879142d6b8670254708853ff4fb", size = 1775765, upload-time = "2026-06-01T19:40:01.195Z" }, - { url = "https://files.pythonhosted.org/packages/b8/69/155c4ef3aec96417d47024800472b33b16c5d8a665371dcd044c2afdf25d/aiohttp-3.14.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:26b6d79aa54cb4ed50cc7d41ed14e99e0f1fc8e7c2d42f2e05b37aea897b2b52", size = 1733716, upload-time = "2026-06-01T19:40:03.631Z" }, - { url = "https://files.pythonhosted.org/packages/5f/44/6126116fd8a316b712bb615660b855c78466bb67ba1bb1742427eafcf7ac/aiohttp-3.14.0-cp314-cp314-win32.whl", hash = "sha256:106ed074a856f3e21d186b8579e2c8afb6da598e267cdaab01059e13db2fc44d", size = 453684, upload-time = "2026-06-01T19:40:06.277Z" }, - { url = "https://files.pythonhosted.org/packages/a2/d7/eff4c58a88c5cac5e38b55f44fb8a6d3929c3cbd77356e383e094d3220bd/aiohttp-3.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f770846edae8f00ecc57af825bce811f787f87a7dcf0e90d191790efe5b31f7", size = 481758, upload-time = "2026-06-01T19:40:08.653Z" }, - { url = "https://files.pythonhosted.org/packages/d7/ed/17b5bd9fbcb46e688f02e572f517754a9a75831e7b54702f027761dc4fa5/aiohttp-3.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:acf1581c4f21ed4b80a2dded504d87b055a071a84d5737ea966435f768275ac6", size = 450557, upload-time = "2026-06-01T19:40:11.03Z" }, - { url = "https://files.pythonhosted.org/packages/12/34/6180103ce9aabc8ebff3f7bb55a1228ffe60f61042823031d9692cb7b101/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6aa1a40f9cbb3da9f80714c5966b8946c21e6a2530d809b9498b33161e3c8733", size = 787878, upload-time = "2026-06-01T19:40:13.401Z" }, - { url = "https://files.pythonhosted.org/packages/92/e9/08954a40e8b7baa3d8beadd2b074b186e9b1e9c8ddabc288678a6265de50/aiohttp-3.14.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b62af5a8cc96a194eaa01a9ed7b34a3ffa58d3d8daaa1a0d7a749353ad12d228", size = 524400, upload-time = "2026-06-01T19:40:15.972Z" }, - { url = "https://files.pythonhosted.org/packages/08/6a/b5965a634ac4d5ba99a463314cf4ab214ca073fcdc38a15e0294273701fc/aiohttp-3.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6eb63b1417efaf7d1002a6ad034a40d44376afcc16508a57f8e74b49ad26a095", size = 527904, upload-time = "2026-06-01T19:40:18.28Z" }, - { url = "https://files.pythonhosted.org/packages/06/b4/932bcdd850c354d9bcca30f360e475d7852e30413fbbd44b182782ed5432/aiohttp-3.14.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c20b9ad156a79eb97be5cf9e069eec01d2f0dc8472ffbd75299a8b2d4c2cbbde", size = 1912162, upload-time = "2026-06-01T19:40:20.825Z" }, - { url = "https://files.pythonhosted.org/packages/c6/85/ce79bab0310d2e3fd2d7bc7e44412abeff7c8338f8a21dd0f2f1714989e5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:40ae7b0642c25632c7eabc4a04754012691864d2a1b93becf7cddb76027b838a", size = 1778813, upload-time = "2026-06-01T19:40:23.726Z" }, - { url = "https://files.pythonhosted.org/packages/05/54/ba62ac2d1bc87e010aad23751e383b8794e45d931df67677313a2da78823/aiohttp-3.14.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:95f5217e76a046b9f228a101717ef8d42b1eb3d9d196d15202db5bf41df88936", size = 1899969, upload-time = "2026-06-01T19:40:26.406Z" }, - { url = "https://files.pythonhosted.org/packages/dc/82/7cc7907725d83a19f31551334061e1ab8e108b1d7ac52632a2a844a4acb5/aiohttp-3.14.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1a4a9f17e85b80878c176695c1998c790e83731d8271881e5d356488652a1f9e", size = 1991771, upload-time = "2026-06-01T19:40:29.061Z" }, - { url = "https://files.pythonhosted.org/packages/d0/1c/a57de71a4508c93a830b77c28af3d08cd97f606dedfc6b94275347744508/aiohttp-3.14.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:145262119b07d7f95abc1839add35ba2bfc84551d4b4660ca11542c0b215455b", size = 1868606, upload-time = "2026-06-01T19:40:31.843Z" }, - { url = "https://files.pythonhosted.org/packages/9c/ae/3839726cd49150a53ed340cc24ce5ba09d4c2117020ef9d45542bec5eb2f/aiohttp-3.14.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:49a33ded29b0b2fa7a367a02cf0fb89af602bb87542a16177ec8ce1c9c51d12a", size = 1665437, upload-time = "2026-06-01T19:40:35.01Z" }, - { url = "https://files.pythonhosted.org/packages/35/1e/c237923232c7da7f0392ea25d89fc5e60c0e93f685f4ebca8e7bcdd5271c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cc736a9c9fc2bc4dd71fd404815741b6573df27c3f985948ec4076989ac57de", size = 1834090, upload-time = "2026-06-01T19:40:37.733Z" }, - { url = "https://files.pythonhosted.org/packages/98/02/a5a7a2524f92d3911761b405a7c067c751891942144adc13e2ad79611e39/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:b4141a3e5342ee3053a9cab54d25b64ed28289c1041e4c54b3d99839314d90ce", size = 1816907, upload-time = "2026-06-01T19:40:40.46Z" }, - { url = "https://files.pythonhosted.org/packages/fa/76/a8b9f0d09234d516af9f2d7dd715557f33b5da3b0b56ead41d1170e86e3c/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:e30871b2d58996cb81aac52d2b1d15ac05257131ef0f90f18c2115a380fbfe7c", size = 1840382, upload-time = "2026-06-01T19:40:43.48Z" }, - { url = "https://files.pythonhosted.org/packages/c9/8e/140e715a0a4bbc211979ea30ec8396ad2ed5bf90ab87d8058fc4668b1923/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:667b881d083ccae3900ea5a241e17e5007ca78844c53ed389bb63d48f729d9c7", size = 1659497, upload-time = "2026-06-01T19:40:46.265Z" }, - { url = "https://files.pythonhosted.org/packages/10/c7/7ba5de8af9650b9767b063c675427b8685f43fa7ce563673a7bc3af60f08/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:b584dfe615d151e9b8f0a8ecb3aee6147f2927ec5b95ba25fe621f5377510928", size = 1870829, upload-time = "2026-06-01T19:40:49.583Z" }, - { url = "https://files.pythonhosted.org/packages/cc/bc/2aaab2f85cadb26ea59c091fa2b8e370d625154b5c14b478f1b489d07551/aiohttp-3.14.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6199707cc40e0e9cd39c36fbc97bec416c704e1d0ddce03412bb3b3e6a90ccd0", size = 1832281, upload-time = "2026-06-01T19:40:52.303Z" }, - { url = "https://files.pythonhosted.org/packages/39/98/31b9ad9fbc01f0075ee7221002df5fd2d10b647f451ca5f30edc802d9dd6/aiohttp-3.14.0-cp314-cp314t-win32.whl", hash = "sha256:a8d93334d4961c9d566b1f046c81dee475b7c21eb730728d38237bfa70d1c8e6", size = 490597, upload-time = "2026-06-01T19:40:54.937Z" }, - { url = "https://files.pythonhosted.org/packages/59/1f/299b21441c8de42ff70fddc7cfe65e92f810abcf740739a09b56f7835364/aiohttp-3.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2d2ffe9b614f50f069068b3b52e73414e4107fc10b7efc939a76acff9251fdd2", size = 525789, upload-time = "2026-06-01T19:40:57.306Z" }, - { url = "https://files.pythonhosted.org/packages/70/11/7f83fcba9ee05d4c54d61b3f8104da0d43a59adac44dd28effc0c9a10422/aiohttp-3.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:7a3fc4358e65826c515350f199c210de747cf669998211b1ee6c2e46de364b24", size = 467399, upload-time = "2026-06-01T19:40:59.993Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/82/78/8ea7308cac6934de8c74a14f3d5f65d1c89287426688be79538d0e5c013d/aiohttp-3.14.1.tar.gz", hash = "sha256:307f2cff90a764d329e77040603fa032db89c5c24fdad50c4c15334cba744035", size = 7955794, upload-time = "2026-06-07T21:09:35.529Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/67/58ded4b3f2e10f94972d8928050c85330e249a31dd45a0e5f3c0e9c3fa05/aiohttp-3.14.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8f6bb621e5863cfe8fe5ff5468002d200ec31f30f1280b259dc505b02595099e", size = 766140, upload-time = "2026-06-07T21:05:37.471Z" }, + { url = "https://files.pythonhosted.org/packages/18/68/4ae5b4e08943f316594bb68da89957d3baf5760588fa09509594bd777e4b/aiohttp-3.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f7215cb3933784f79ed20e5f050e15984f390424339b22375d5a53c933a0491", size = 519430, upload-time = "2026-06-07T21:05:40.751Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c1/316c8f3549dbe5245f92bfd523ec6f32dd4d98cafe21df3f6a19b1184c75/aiohttp-3.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d9d4e294455b23a68c9b8f042d0e8e377a265bcb15332753695f6e5b6819e0ce", size = 514406, upload-time = "2026-06-07T21:05:42.111Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ee/fb0ac28684e8d753b83c8a4eebc19a5846912aa0a4daaabb6a9936363840/aiohttp-3.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b238af795833d5731d049d82bc84b768ae6f8f97f0495963b3ed9935c5901cc3", size = 1703649, upload-time = "2026-06-07T21:05:43.427Z" }, + { url = "https://files.pythonhosted.org/packages/3b/57/aa2beab673331f111885db8a7b69dfe3ab0e53e446a0ace18ca694b4dc58/aiohttp-3.14.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4e5e0ae56914ecdbf446493addefc0159053dd53962cef37d7839f37f73d505", size = 1675126, upload-time = "2026-06-07T21:05:44.897Z" }, + { url = "https://files.pythonhosted.org/packages/47/ea/dad128abe365e79be03b16ed464198ac73e0d257e8260c6f7d6f31cbef26/aiohttp-3.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:092e4ce3619a7c6dee52a6bdabda973d9b34b66781f840ce93c7e0cec30cf521", size = 1771558, upload-time = "2026-06-07T21:05:46.405Z" }, + { url = "https://files.pythonhosted.org/packages/63/f3/b5b4e10327cb85d34d24232c6b71b64602f190b3ccb238a043ac6b187dac/aiohttp-3.14.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb33777ea21e8b7ecde0e6fc84f598be0a1192eab1a63bc746d75aa75d38e7bd", size = 1856631, upload-time = "2026-06-07T21:05:47.844Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9d/93294c3045775c708ac8310eb3d3622a11d2951345ad590d532d62a1faa4/aiohttp-3.14.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23119f8fd4f5d16902ed459b63b100bcd269628075162bddac56cc7b5273b3fb", size = 1714139, upload-time = "2026-06-07T21:05:49.982Z" }, + { url = "https://files.pythonhosted.org/packages/29/c4/93067c85a0373492ce8e577435203c5947c454af074ac48ed4f3a1b9dd4a/aiohttp-3.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:57fc6745a4b7d0f5a9eb4f40a69718be6c0bc1b8368cc9fe89e90118719f4f42", size = 1588321, upload-time = "2026-06-07T21:05:51.431Z" }, + { url = "https://files.pythonhosted.org/packages/c4/39/9ff91aaf02af8b7b8222a987466da539f154c3e01732c22b5f5a20a8ee66/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6fd35beba67c4183b09375c5fff9accb47524191a244a99f95fd4472f5402c2b", size = 1670375, upload-time = "2026-06-07T21:05:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e4/77452a3676b8d99ac1375f77691d6bf65ea6e9f4b201b82ef77c916dc767/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:672b9d65f42eb877f5c3f234a4547e4e1a226ca8c2eed879bb34670a0ce51192", size = 1690933, upload-time = "2026-06-07T21:05:54.902Z" }, + { url = "https://files.pythonhosted.org/packages/7d/84/b0059a7c7fc05ea23f3bc1596ba91c12f79588b9450564a24cac37536d0a/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:24ba13339fed9251d9b1a1bec8c7ab84c0d1675d79d33501e11f94f8b9a84e05", size = 1740798, upload-time = "2026-06-07T21:05:56.458Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3a/e2a513ecbfc362591caa51a7f7e011b3bfc8938b388ae44cd95560d36999/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:94da27378da0610e341c4d30de29a191672683cc82b8f9556e8f7c7212a020fe", size = 1576412, upload-time = "2026-06-07T21:05:57.953Z" }, + { url = "https://files.pythonhosted.org/packages/a1/10/08f1654f538f93d36dcac66310a06eefce4641cdafca83f9f0a5317be254/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:52cdac9432d8b4a719f35094a818d95adcae0f0b4fe9b9b921909e0c87de9e7d", size = 1750199, upload-time = "2026-06-07T21:05:59.488Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/d91b70c57d8b8e9611e4a2e52238ca3698d3dc1c2efe25b7a9bf594ac584/aiohttp-3.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:672ac254412a24d0d0cf00a9e6c238877e4be5e5fa2d188832c1244f45f31966", size = 1699356, upload-time = "2026-06-07T21:06:01.131Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f1/15340176f35ff61b95dbe34020bcf43f9e624a2d7bbac934715ff97d2033/aiohttp-3.14.1-cp310-cp310-win32.whl", hash = "sha256:2fe3607e71acc6ebb0ec8e492a247bf7a291226192dc0084236dfc12478916f6", size = 458939, upload-time = "2026-06-07T21:06:02.86Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c2/a2f1ec5b37f903109e43ae2862268cfe4a67a60c1b2cf43169fcdff5995f/aiohttp-3.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:30099eda75a53c32efb0920e9c33c195314d2cc1c680fbfd30894932ac5f27df", size = 482583, upload-time = "2026-06-07T21:06:04.666Z" }, + { url = "https://files.pythonhosted.org/packages/d0/7a/7b56f6732ef79530afaa72aa335d41b67c8d79b946995f0b11ad72985435/aiohttp-3.14.1-cp310-cp310-win_arm64.whl", hash = "sha256:5a837f49d901f9e368651b676912bff1104ed8c1a83b280bcd7b29adccef5c9c", size = 453470, upload-time = "2026-06-07T21:06:06.322Z" }, + { url = "https://files.pythonhosted.org/packages/26/dd/bf526e6f0a1120dd6f2df2e97bacfe4d358f13d17a0ff5847301a1375a51/aiohttp-3.14.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa00140699487bd435fde4342d85c94cb256b7cd3a5b9c3396c67f19922afda2", size = 765225, upload-time = "2026-06-07T21:06:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/8f/e1/a2872aa55495a70f61310d411541c6ee23812d9a884e000c716e1bc3edbf/aiohttp-3.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1c1af67559445498b502030c35c59db59966f47041ca9de5b4e707f86bd10b5f", size = 518743, upload-time = "2026-06-07T21:06:09.749Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e7/c60c7b209e509cc787de3cea0550a518538cfc08003e1c1e14c1c63fff71/aiohttp-3.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d44ec478e713ee7f29b439f7eb8dc2b9d4079e11ae114d2c2ac3d5daf30516c8", size = 514139, upload-time = "2026-06-07T21:06:11.26Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8d/614ace2f579702c9840ab1e1447fd8509e35b0b904f7196418fa2f57b25d/aiohttp-3.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3b1a184a9a8f548a6b73f1e26b96b052193e4b3175ed7342aaf1151a1f00a04", size = 1784088, upload-time = "2026-06-07T21:06:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/726e90f99542bf292f81a96a12cc4847deb86f3ccf62c6f4014a201f4d33/aiohttp-3.14.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5f2504bc0322437c9a1ff6d3333ca56c7477b727c995f036b976ae17b98372c8", size = 1737835, upload-time = "2026-06-07T21:06:14.564Z" }, + { url = "https://files.pythonhosted.org/packages/0b/4b/d176d5c4db9d33dacf0543102ea59503bc1d528af4cfd0b719949ca49389/aiohttp-3.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73f05ea02013e02512c3bf42714f1208c57168c779cc6fe23516e4543089d0a6", size = 1842801, upload-time = "2026-06-07T21:06:16.228Z" }, + { url = "https://files.pythonhosted.org/packages/dc/d6/5a99b563690ea0cbed912ae94a2ce33993a5709a651a3a4fe761e7dd973a/aiohttp-3.14.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:797457503c2d426bee06eef808d07b31ede30b65e054444e7de64cad0061b7af", size = 1929992, upload-time = "2026-06-07T21:06:17.947Z" }, + { url = "https://files.pythonhosted.org/packages/76/7f/a987b14a3859094b3cea3f4825219c3e5536242564af6e3f9c2f6c994eb2/aiohttp-3.14.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b821a1f7dedf7e37450654e620038ac3b2e81e8fa6ea269337e97101978ec730", size = 1786989, upload-time = "2026-06-07T21:06:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/f1/1a/420e5c85a3e73349372ed22ce0b6af86bfa6ce16a4b20a64a2e94608c781/aiohttp-3.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4cd96b5ba05d67ed0cf00b5b405c8cd99586d8e3481e8ee0a831057591af7621", size = 1640129, upload-time = "2026-06-07T21:06:22.558Z" }, + { url = "https://files.pythonhosted.org/packages/a7/80/18a592ed3be0a402cc03670bd72ee1f8563ddbe1d8d5542dbf868f274136/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d459b98a932296c6f0e94f87511a0b1b90a8a02c30a50e60a297619cd5a58ee", size = 1756576, upload-time = "2026-06-07T21:06:24.8Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0b/8b3d5713373858ff71a617daf6e3b0e81ad63e79d09a3cf2f6b6b983939c/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:764457a7be60825fb770a644852ff717bcbb5042f189f2bd16df61a81b3f6573", size = 1754668, upload-time = "2026-06-07T21:06:26.528Z" }, + { url = "https://files.pythonhosted.org/packages/9f/49/fd564575cf225821d7ba5a117cb8bc27213d8a7e1811162afb43ae077039/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f7a16ef45b081454ef844502d87a848876c490c4cb5c650c230f6ec79ed2c1e7", size = 1817019, upload-time = "2026-06-07T21:06:28.297Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1b/e850c9ae6fc91356552ae668bb6c51e93fa29c8aef13398a10b56678557f/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2fbc3ed048b3475b9f0cbcb9978e9d2d3511acd91ead203af26ed9f0056004cf", size = 1631638, upload-time = "2026-06-07T21:06:30.242Z" }, + { url = "https://files.pythonhosted.org/packages/eb/94/3c337ba72451a89806ace6f75bddc92bafc5b8d53d90115a512858024b63/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:bedb0cd073cc2dc035e30aeb99444389d3cd2113afe4ef9fcd23d439f5bade85", size = 1835660, upload-time = "2026-06-07T21:06:31.943Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9c/9c18cf367a0498212d9ba7daf990b504a5e8ae064cda4b504e2647c89c03/aiohttp-3.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b6feea921016eb3d4e04d65fc4e9ca402d1a3801f562aef94989f54694917af3", size = 1775698, upload-time = "2026-06-07T21:06:33.72Z" }, + { url = "https://files.pythonhosted.org/packages/b5/63/a251a9d2a6cb45065b2ddc0bde2b3dd10108740a9a42f632c66405a761a2/aiohttp-3.14.1-cp311-cp311-win32.whl", hash = "sha256:313701e488100074ce99850404ee36e741abf6330179fec908a1944ecf570126", size = 458386, upload-time = "2026-06-07T21:06:35.279Z" }, + { url = "https://files.pythonhosted.org/packages/17/ca/69274c51dcd6e8947d77b2806cf47a4a15f2c846e2cbeb1882547d3da283/aiohttp-3.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:03ab4530fdcb3a543a122ba4b65ac9919da9fe9f78a03d328a6e38ff962f7aa5", size = 483406, upload-time = "2026-06-07T21:06:36.824Z" }, + { url = "https://files.pythonhosted.org/packages/2c/8a/c25904f77690c3688ec140f87591ef11a0cfe36bf3d5c0f1f38056fb62b3/aiohttp-3.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:486f7d16ed54c39c2cbd7ca71fd8ba2b8bb7860df65bd7b6ed640bab96a38a8b", size = 452987, upload-time = "2026-06-07T21:06:38.371Z" }, + { url = "https://files.pythonhosted.org/packages/1d/21/151624b51cd92553d95424daf4bf19f19ce9be9002d19253e7e7ce67197b/aiohttp-3.14.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d35143e27778b4bb0fb189562d7f275bff79c62ab8e98459717c0ea617ff2480", size = 757402, upload-time = "2026-06-07T21:06:40.311Z" }, + { url = "https://files.pythonhosted.org/packages/c2/82/280619e0bd7bf2454987e19282616e84762255dd9c8468f62382e8c191f1/aiohttp-3.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bcfb80a2cc36fba2534e5e5b5264dc7ae6fcd9bf15256da3e53d2f499e6fa29d", size = 512310, upload-time = "2026-06-07T21:06:42.207Z" }, + { url = "https://files.pythonhosted.org/packages/55/b2/2aac325583aaa1353045f96dffa586d8a34e8322e14a7ba49cffeb103ab4/aiohttp-3.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27fd7c91e51729b4f7e1577865fa6d34c9adccbc39aabe9000285b48af9f0ec2", size = 512448, upload-time = "2026-06-07T21:06:43.813Z" }, + { url = "https://files.pythonhosted.org/packages/8a/72/a60607cb849faa8af8a356c9329ea2eb6f395d49e82cc82ccba1fd8deb8f/aiohttp-3.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:64c567bf9eaf664280116a8688f63016e6b32db2505908e2bdaca1b6438142f2", size = 1766854, upload-time = "2026-06-07T21:06:45.391Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d3/d9fe1c9ec7557ab4d0d82bebaa728c6418f0b93295ec2f4ab015f7710cc7/aiohttp-3.14.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f5e6ff2bdbb8f4cd3fbe41f99e25bbcd58e3bf9f13d3dd31a11e7917251cc77a", size = 1740884, upload-time = "2026-06-07T21:06:47.413Z" }, + { url = "https://files.pythonhosted.org/packages/c1/dc/f2cecfaf9337ba3e63f181500814ff502aa3d00d9c7ec93a9d23d10a27b2/aiohttp-3.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2f73e01dc37122325caf079982621262f96d74823c179038a82fddfc50359264", size = 1810034, upload-time = "2026-06-07T21:06:50.165Z" }, + { url = "https://files.pythonhosted.org/packages/66/d7/2ff65c5e65c0d7476daf7e15c032e0805e36811185b9623e3238ad6c763e/aiohttp-3.14.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bb2c0c80d431c0d03f2c7dbf125150fedd4f0de17366a7ca33f7ccb822391842", size = 1904054, upload-time = "2026-06-07T21:06:52.035Z" }, + { url = "https://files.pythonhosted.org/packages/20/9c/d445818389df371f56d141d881153ba23183c4735a03f7356ffb43f7757d/aiohttp-3.14.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3e6fc1a85fa7194a1a7d19f44e8609180f4a8eb5fa4c7ed8b4355f080fad235c", size = 1790278, upload-time = "2026-06-07T21:06:54.049Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/bf04cb4d865fc6101c2229a294ad744973b72e513fdc5a6b791e6983d72a/aiohttp-3.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:686b6c0d3911ec387b444ddf5dc62fb7f7c0a7d5186a7861626496a5ab4aff95", size = 1591795, upload-time = "2026-06-07T21:06:55.911Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b4/4dac0038960427ba832f6609dfb4ea5437d7fd80c72001b9e48f834f428b/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c6fa4dc7ad6f8109c70bb1499e589f76b0b792baf39f9b017eb92c8a81d0a199", size = 1728397, upload-time = "2026-06-07T21:06:57.777Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/7cd4e8ad7aa3b75f17d56bb5498dd604a93d4e6eece822ba0568c413fff0/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:87a5eea1b2a5e21e1ebdbb33ad4165359189327e63fc4e4894693e7f821ac817", size = 1766504, upload-time = "2026-06-07T21:07:00.009Z" }, + { url = "https://files.pythonhosted.org/packages/f9/df/fc01d9fcad0f73fed3f3d361f1f94f975947b50dff82919f6dc2bf4316cc/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1c1421eb01d4fd608d88cc8290211d177a58532b55ad94076fb349c5bf467f0a", size = 1777806, upload-time = "2026-06-07T21:07:02.064Z" }, + { url = "https://files.pythonhosted.org/packages/41/09/47e2d090bddcc8fb4ccb4c314aadc32d7c5d9bb55f50f6ad1c92fc15d501/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:34b257ec41345c1e8f2df68fa908a7952f5de932723871eb633ecbbff396c9a4", size = 1580707, upload-time = "2026-06-07T21:07:03.942Z" }, + { url = "https://files.pythonhosted.org/packages/3d/36/f1a4ce904ae0b6930cfe9afc96d0896f7ec1a620c400405d63783bb95a9c/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:de538791a80e5d862addbc183f70f0158ac9b9bb872bb147f1fd2a683691e087", size = 1798121, upload-time = "2026-06-07T21:07:05.987Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/e0075ce9ca0279ee1d4f0c0b85f54fea02ebc83c3007651a72bece658fec/aiohttp-3.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f71173be42d3241d428f760122febb748de0623f44308a6f120d0dd9ec572e3", size = 1767580, upload-time = "2026-06-07T21:07:07.873Z" }, + { url = "https://files.pythonhosted.org/packages/3e/61/a0c0a8f327a9c52095cdd8e312391b00d3ed64ab6c72bb5c33d8ec251cf7/aiohttp-3.14.1-cp312-cp312-win32.whl", hash = "sha256:ec8dc383ee57ea3e883477dcca3f11b65d58199f1080acaf4cd6ad9a99698be4", size = 452771, upload-time = "2026-06-07T21:07:09.669Z" }, + { url = "https://files.pythonhosted.org/packages/df/d9/ea367c75f16ac9c6cdc8febb25e8318fa21a2b1bc8d6514d4b2d890bface/aiohttp-3.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2aa92c87868cd13674989f9ee83e5f9f7ea4237589b728048e1f0c8f6caa3271", size = 479873, upload-time = "2026-06-07T21:07:11.538Z" }, + { url = "https://files.pythonhosted.org/packages/03/64/8d96784a7851156db8a4c6c3f6f91042fdf39fb15a4cc38c8b3c14833c45/aiohttp-3.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:2c840c90759922cb5e6dda94596e079a30fb5a5ba548e7e0dc00574703940847", size = 448073, upload-time = "2026-06-07T21:07:13.637Z" }, + { url = "https://files.pythonhosted.org/packages/bc/97/bd137012dd97e1649162b099135a80e1fd59aaa807b2430fc448d1029aff/aiohttp-3.14.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:b3a03285a7f9c7b016324574a6d92a1c895da6b978cb8f1deee3ac72bc6da178", size = 506882, upload-time = "2026-06-07T21:07:15.501Z" }, + { url = "https://files.pythonhosted.org/packages/ef/79/e5cc690e9d922a66887ceeaca53a8ffd5a7b0be3816142b7abc433742d89/aiohttp-3.14.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:2a73f487ab8ef5abbb24b7aa9b73e98eaba9e9e031804ff2416f02eca315ccaf", size = 515270, upload-time = "2026-06-07T21:07:17.53Z" }, + { url = "https://files.pythonhosted.org/packages/fe/22/a73ccbf9dbd6e26dda0b24d5fd5db7da92ee3383a79f47677ffb834c5c5b/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:915fbb7b41b115192259f8c9ae58f3ddc444d2b5579917270211858e606a4afd", size = 485841, upload-time = "2026-06-07T21:07:19.555Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/57ed8eaf596321c2ad747bd480fb1700dbd7177c60dfc9e4c187f629662e/aiohttp-3.14.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:7fb4bdf95b0561a79f259f9d28fbc109728c5ee7f27aff6391f0ca703a329abe", size = 492088, upload-time = "2026-06-07T21:07:21.581Z" }, + { url = "https://files.pythonhosted.org/packages/78/c0/5ebe5270a7c140d7c6f79dcb018640225f14d406c149e4eec04a7d82fe71/aiohttp-3.14.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:1b9748363260121d2927704f5d4fc498150669ca3ae93625986ee89c8f80dcd4", size = 501564, upload-time = "2026-06-07T21:07:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/75/7f/8cdaa24fc7983865e0915153b96a9ac5bcdd3548d64c5a27d17cecccad2d/aiohttp-3.14.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:86a6dab78b0e43e2897a3bbe15745aa60dc5423ca437b7b0b164c069bf91b876", size = 751998, upload-time = "2026-06-07T21:07:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/b2/f4/c4227aacfacc5cb0cc2d119b65301d177912a6842cd64e120c47af76064f/aiohttp-3.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4dfd6e47d3c44c2279907607f73a4240b88c69eb8b90da7e2441a8045dfd21da", size = 510918, upload-time = "2026-06-07T21:07:27.28Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a2d5f96cd4e74424864d30bc0a7e44d0a12dacdcfa91b5b2d1bd3dca6bf3/aiohttp-3.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:317acd9f8602858dc7d59679812c376c7f0b97bcbbf16e0d6237f54141d8a8a6", size = 508657, upload-time = "2026-06-07T21:07:29.252Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ed/3c0fb5c500fdd8e7ebc10d1889c04384fffa1a9163eac1356088ca9da1b1/aiohttp-3.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bd869c427324e5cb15195793de951295710db28be7d818247f3097b4ab5d4b96", size = 1757907, upload-time = "2026-06-07T21:07:31.03Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ab/d4c924d9bd5be3050c226612413ce68cb54c70d2c31b661bfc8d9a5b6a70/aiohttp-3.14.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:93b032b5ec3255473c143627d21a69ac74ae12f7f33974cb587c564d11b1066f", size = 1737565, upload-time = "2026-06-07T21:07:33.031Z" }, + { url = "https://files.pythonhosted.org/packages/19/2a/37326821ff779084020cdc33224d20b19f42f4183a500ff92022a739eda7/aiohttp-3.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f234b4deb12f3ad59127e037bc57c40c21e45b45282df7d3a55a0f409f595296", size = 1799018, upload-time = "2026-06-07T21:07:35.003Z" }, + { url = "https://files.pythonhosted.org/packages/b3/4f/6e947ba73e4ce09070761c05ed3a8ceb7c21f5e46798671d8b2aac0e4626/aiohttp-3.14.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9af6779bfb46abf124068327abcdf9ce95c9ef8287a3e8da76ccf2d0f16c28fa", size = 1894416, upload-time = "2026-06-07T21:07:36.956Z" }, + { url = "https://files.pythonhosted.org/packages/9d/6e/dbf1d0625dc711fb2851f4f3c3055c39ed58bae92082d8c627dbe6013736/aiohttp-3.14.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faccab372e66bc76d5731525e7f1143c922271725b9d38c9f97edcc66266b451", size = 1783881, upload-time = "2026-06-07T21:07:39.063Z" }, + { url = "https://files.pythonhosted.org/packages/44/c2/5e25098a67268ed369483ae7d1a58bd0a13d03aab860d2a0e4a6eb25b046/aiohttp-3.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f380468b09d2a81633ee863b0ec5648d364bd17bb8ecfb8c2f387f7ac1faf42c", size = 1587572, upload-time = "2026-06-07T21:07:41.058Z" }, + { url = "https://files.pythonhosted.org/packages/2a/bd/cf9cee17e140f942a3de73e658a543aa8fbf35a5fc67a9d2538d52d77f0b/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:97e704dcd26271f5bda3fa07c3ce0fb76d6d3f8659f4baa1a24442cc9ba177ca", size = 1722137, upload-time = "2026-06-07T21:07:43.014Z" }, + { url = "https://files.pythonhosted.org/packages/89/6d/5684f8c59045c96f81a18cefbc1fbbd79d25b88f1c622f2a5c5c08fcb632/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:269b76ac5394092b95bc4a098f4fc6c191c083c3bd12775d1e30e663132f6a09", size = 1755953, upload-time = "2026-06-07T21:07:45.933Z" }, + { url = "https://files.pythonhosted.org/packages/a8/40/35caf3170f8359760740a7d9aa0fff2e344bef98e1d1186f5a0f6dec17e6/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0b3e614340c889d575451696374c9d17affd54cd607ca0babed8f8c37b9397", size = 1766479, upload-time = "2026-06-07T21:07:48.047Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a1/b0c61e7a137f0d81de49a82023a6df73c3c16d6fefb0f8e4a93d21639002/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:5663ee9257cfa1add7253a7da3035a02f31b6600ec48261585e1800a81533080", size = 1580077, upload-time = "2026-06-07T21:07:50.069Z" }, + { url = "https://files.pythonhosted.org/packages/0b/41/194ea4623693009fcefebef7aef63c141754f153e9cd0d39d3b9e36c175c/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:603a2c834142172ffddc054067f5ec0ca65d57a0aa98a71bc81952573208e345", size = 1791688, upload-time = "2026-06-07T21:07:52.106Z" }, + { url = "https://files.pythonhosted.org/packages/ba/45/4de841f005cfe1fd63e2a2fe011262c515e2a62aa6994b15947e7d717ac9/aiohttp-3.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cb21957bb8aca671c1765e32f58164cf0c50e6bf41c0bbbd16da20732ecaf588", size = 1761094, upload-time = "2026-06-07T21:07:54.113Z" }, + { url = "https://files.pythonhosted.org/packages/e4/ae/dbce10533d3896d544d5053939ed75b7dc31a1b0973d959b1b5ae21028d6/aiohttp-3.14.1-cp313-cp313-win32.whl", hash = "sha256:e509a55f681e6158c20f70f102f9cf61fb20fbc382272bc6d94b7343f2582780", size = 452662, upload-time = "2026-06-07T21:07:56.06Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/0bf1a19362c32f06229da5e7ddfcec91f93474d6307f7a2d3135e9c674dc/aiohttp-3.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:1ac8531b638959718e18c2207fbfe297819875da46a740b29dfa29beba64355a", size = 479748, upload-time = "2026-06-07T21:07:58.319Z" }, + { url = "https://files.pythonhosted.org/packages/22/0a/62e7232dc9484fbec112ceb32efb6a624cc7994ec6e2b019286f17c4e8f2/aiohttp-3.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:250d14af67f6b6a1a4a811049b1afa69d61d617fca6bf33149b3ab1a6dbcf7b8", size = 447723, upload-time = "2026-06-07T21:08:00.154Z" }, + { url = "https://files.pythonhosted.org/packages/c4/a1/5fafa04e1ca91ddb47608699d60649c1c6db3cf41c99e78fc4056f9513db/aiohttp-3.14.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:7c106c26852ca1c2047c6b80384f17100b4e439af276f21ef3d4e2f450ae7e15", size = 508531, upload-time = "2026-06-07T21:08:02.093Z" }, + { url = "https://files.pythonhosted.org/packages/fa/2e/bfa02f699d87ffc86d5959270b28f1cb410add3ccaced8ed2e0b8a5238fc/aiohttp-3.14.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:20205f7f5ade7aaec9f4b500549bbc071b046453aed72f9c06dcab87896a83e8", size = 514718, upload-time = "2026-06-07T21:08:04.476Z" }, + { url = "https://files.pythonhosted.org/packages/85/a5/9594ad6289eebbc97d167c44213d557807f90e59115caad24de21ad2c3b1/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:62a759436b29e677181a9e76bab8b8f689a29cb9c535f45f7c48c9c830d3f8c3", size = 487918, upload-time = "2026-06-07T21:08:06.377Z" }, + { url = "https://files.pythonhosted.org/packages/b4/61/16a32c36c3c49edec122a3dc811f2057df2f94d3b14aa107c8017d981618/aiohttp-3.14.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:2964cbf553df4d7a57348da44d961d871895fc1ee4e8c322b2a95612c7b17fba", size = 494014, upload-time = "2026-06-07T21:08:08.263Z" }, + { url = "https://files.pythonhosted.org/packages/9b/89/3ebcf96ed99c05bec9c434aaac6963fd3cbab4a786ae739908a144d9ce44/aiohttp-3.14.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:237651caadc3a59badd39319c54642b5299e9cc98a3a194310e55d5bb9f5e397", size = 502398, upload-time = "2026-06-07T21:08:10.244Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3d/b74870a0c2d40c355928cd5b96c7a11fa821b8a40fc41365e64479b151fb/aiohttp-3.14.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:896e12dfdbbab9d8f7e16d2b28c6769a60126fa92095d1ebf9473d02593a2448", size = 758018, upload-time = "2026-06-07T21:08:12.447Z" }, + { url = "https://files.pythonhosted.org/packages/d3/66/f42f5c984d99e49c6cff5f26f590750f2e2f7ef1fcfb99966ab5be1b632e/aiohttp-3.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d03f281ed22579314ba00821ce20115a7c0ac430660b4cc05704a3f818b3e004", size = 512462, upload-time = "2026-06-07T21:08:14.624Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a7/248e1aebe0c7810b0271e021a0f2a5eb6e78a051885b3c9df49f42a5802d/aiohttp-3.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:07eabb979d236335fed927e137a928c9adfb7df3b9ec7aa31726f133a62be983", size = 512824, upload-time = "2026-06-07T21:08:16.572Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/2aa0e5ba0727dc3bd5aaebb7ccbc510f7dfb7fb961ec87497cd496635ab1/aiohttp-3.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4fe1f1087cbadb280b5e1bb054a4f00d1423c74d6626c5e48400d871d34ecefe", size = 1749898, upload-time = "2026-06-07T21:08:18.635Z" }, + { url = "https://files.pythonhosted.org/packages/00/8d/e97f6c96c891d457c8479d92a514ba194d0412f981d72c70341ee18488ed/aiohttp-3.14.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:367a9314fdc79dab0fac96e216cb41dd73c85bdca85306ce8999118ba7e0f333", size = 1710114, upload-time = "2026-06-07T21:08:20.892Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e6/aa8d7e863048c8fceb5cd6ce74017311cec3ead07847387e12265fb4444e/aiohttp-3.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a24f677ebe83749039e7bdf862ff0bbb16818ae4193d4ef96505e269375bcce0", size = 1802541, upload-time = "2026-06-07T21:08:23.044Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/72193137de57fda4ebfae4563182d082c8856e3b6e9871d0b46f028fb369/aiohttp-3.14.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c83afe0ba876be7e943d2e0ba645809ad441575d2840c895c21ee5de93b9377a", size = 1875776, upload-time = "2026-06-07T21:08:25.288Z" }, + { url = "https://files.pythonhosted.org/packages/a0/18/938441025db6769a3464596b2410af3afde0b21eb2f204c6f766f68af4bd/aiohttp-3.14.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:634e385930fb6d2d479cf3aa66515955863b77a5e3c2b5894ca259a25b308602", size = 1760329, upload-time = "2026-06-07T21:08:27.363Z" }, + { url = "https://files.pythonhosted.org/packages/60/29/bf2496b4065e76e09fe48015aaffe5ce161d8f089b06ac6982070f653076/aiohttp-3.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eeea07c4397bbc57719c4eed8f9c284874d4f175f9b6d57f7a1546b976d455ca", size = 1587293, upload-time = "2026-06-07T21:08:29.805Z" }, + { url = "https://files.pythonhosted.org/packages/49/a2/2136674d52123b1354bd05dd5753c318db47dc0c927cc70b27bab3755456/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:335c0cc3e3545ce98dcb9cfcb836f40c3411f43fa03dab757597d80c89af8a35", size = 1714756, upload-time = "2026-06-07T21:08:32.094Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b9/e5fd2e6f915503081c0f9b1e8540947037929c70c191da2e4d54b31a21a1/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:ae6be797afdef264e8a84864a85b196ca06045586481b3df8a967322fd2fa844", size = 1721052, upload-time = "2026-06-07T21:08:34.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/5a/2833e324a2263e104e31e2e91bc5bbee81bc499afd32203faee048a883f0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:8560b4d712474335d08907db7973f71912d3a9a8f1dee992ec06b5d2fe359496", size = 1766888, upload-time = "2026-06-07T21:08:36.95Z" }, + { url = "https://files.pythonhosted.org/packages/57/fa/dea6511870913162f3b2e8c42a7614eb203a4540b8c2da43e0bfb0548f3c/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b7edd08e0a5deb1e8564a2fcd8f4561014a3f05252334671bbf55ddd47db0e5", size = 1581679, upload-time = "2026-06-07T21:08:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/14/bd/3cf0d55e71784b33534e9710a67d382d900598b4787fbce6cc7317f8c42a/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:b6ff7fcee63287ae57b5df3e4f5957ce032122802509246dec1a5bcc55904c95", size = 1782021, upload-time = "2026-06-07T21:08:41.407Z" }, + { url = "https://files.pythonhosted.org/packages/c1/af/14bb5843eccbe234f4dfb78ab73e549d99727247e62ae5d62cbd22eaf5b0/aiohttp-3.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ffbb2f4ec1ceaff7e07d43922954da26b223d188bf30658e561b98e23089444", size = 1742574, upload-time = "2026-06-07T21:08:43.795Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1e/fbeb7af9210a67ac0f9c9bec0f8f4568497924e33137a3d5b48e1cf85f3f/aiohttp-3.14.1-cp314-cp314-win32.whl", hash = "sha256:a9875b46d910cff3ea2f5962f9d266b465459fe634e22556ab9bd6fc1192eea0", size = 457773, upload-time = "2026-06-07T21:08:46.168Z" }, + { url = "https://files.pythonhosted.org/packages/f0/2b/13e8d741a9ec5db7d900c060554cf8352ab85e44e2a4469ebb9d377bda17/aiohttp-3.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:af8b4b81a960eeaf1234971ac3cd0ba5901f3cd42eae42a46b4d089a8b492719", size = 485001, upload-time = "2026-06-07T21:08:48.401Z" }, + { url = "https://files.pythonhosted.org/packages/df/30/491acfa2c4d6c3ff59c49a14fc1b50be3241e25bbb0c84c09e2da4d11395/aiohttp-3.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:cf4491381b1b57425c315a56a439251b1bdac07b2275f19a8c44bc57744532ec", size = 453809, upload-time = "2026-06-07T21:08:50.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/e3/19dbe1a1f4cc6230eb9e314de7fe68053b0992f9302b27d12141a0b5db53/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:819c054312f1af92947e6a55883d1b66feefab11531a7fc45e0fb9b63880b5c2", size = 793320, upload-time = "2026-06-07T21:08:52.775Z" }, + { url = "https://files.pythonhosted.org/packages/7f/20/1b7182219ba1b108430d6e4dc53d25ae02dcfcf5a045b33af4e8c5167527/aiohttp-3.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:10ee9c1753a8f706345b22496c79fbddb5be0599e0823f3738b1534058e25340", size = 529077, upload-time = "2026-06-07T21:08:55Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c8/14ce60ec31a2e5f5274bb17d383a6f7a3aabca31ac04eee05585bbadab16/aiohttp-3.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1601cc37baf5750ccacae618ec2daf020769581695550e3b654a911f859c563d", size = 532476, upload-time = "2026-06-07T21:08:57.176Z" }, + { url = "https://files.pythonhosted.org/packages/7e/02/9ac85e081e53da2e061b02fa7758fe0a12d17b8ce2d1f5e6c7cb76730328/aiohttp-3.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4d6e0ac9da31c9c04c84e1c0182ad8d6df35965a85cae29cd71d089621b3ae94", size = 1922347, upload-time = "2026-06-07T21:08:59.563Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3e/d3ba07a0ab38b5389e10bec4362d21e10a4f667cba2d79ba30837b3a5059/aiohttp-3.14.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e8f2d660c350b3d0e259c7a7e3d9b7fc8b41210cbcc3d4a7076ff0a5e5c2fdc", size = 1786465, upload-time = "2026-06-07T21:09:01.909Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cb/e2ee978a00cfb2df829704a69528b18154eba5939f45bc1efa8f33aee4c5/aiohttp-3.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4691802dda97be727f79d86818acaad7eb8e9252626a1d6b519fedbb92d5e251", size = 1909423, upload-time = "2026-06-07T21:09:04.357Z" }, + { url = "https://files.pythonhosted.org/packages/73/5d/1430334858b1022b58ae50399a918f0bd6fe8fa7fa183598d657ff61e040/aiohttp-3.14.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c389c482a7e9b9dc3ee2701ac46c4125297a3818875b9c305ddb603c04828fd1", size = 2001906, upload-time = "2026-06-07T21:09:06.722Z" }, + { url = "https://files.pythonhosted.org/packages/66/4e/560c7472d3d198a23aa5c8b19a5115bf6a9b77b7d3e4bb363da320430ad2/aiohttp-3.14.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fc0cacab7ba4e56f0f81c82a98c09bed2f39c940107b03a34b168bdf7597edd3", size = 1877095, upload-time = "2026-06-07T21:09:09.011Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f1/4745806578d447db4a784a8591e2dae3afdfc2bcb96f8f81271b13df6543/aiohttp-3.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:979ed4717f59b8bb12e3963378fa285d93d367e15bcd66c721311826d3c44a6c", size = 1676222, upload-time = "2026-06-07T21:09:11.461Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c9/48255813cca749a229ef0ab476004ec623728ad79a9c0840616f6c076325/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:38e1e7daaea81df51c952e18483f323d878499a1e2bfe564790e0f9701d6f203", size = 1842922, upload-time = "2026-06-07T21:09:14.118Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c0/bbd054e2bee909f529523a5af3891052606af5143c09f5f183ec3b234676/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:4132e72c608fe9fecb8f409113567605915b83e9bdd3ea56538d2f9cd35002f1", size = 1825035, upload-time = "2026-06-07T21:09:16.447Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ae/90395d4376deceb74e09ec26b6adf7d2015a6f8802d6d84446af860fef04/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:eefd9cc9b6d4a2db5f00a26bc3e4f9acf71926a6ec557cd56c9c6f27c290b665", size = 1849512, upload-time = "2026-06-07T21:09:18.742Z" }, + { url = "https://files.pythonhosted.org/packages/93/bd/fb25f3049957553d4ce0ba6ae480aa2f592a6985497fca590837d16c1be0/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:b165790117eea512d7f3fb22f1f6dad3d55a7189571993eb015591c1401276d1", size = 1668571, upload-time = "2026-06-07T21:09:21.458Z" }, + { url = "https://files.pythonhosted.org/packages/3f/22/7f73303d64dd567ff3addca90b556690ed1233a47b8f55d242fb90af3681/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:ed09c7eb1c391271c2ed0314a51903e72a3acb653d5ccfc264cdf3ef11f8269d", size = 1881159, upload-time = "2026-06-07T21:09:23.813Z" }, + { url = "https://files.pythonhosted.org/packages/44/be/0474c5a8b5640e1e4aa1923430a91f4151be82e511373fe764189b89aef5/aiohttp-3.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:99abd37084b82f5830c635fddd0b4993b9742a66eb746dacf433c8590e8f9e3c", size = 1841409, upload-time = "2026-06-07T21:09:26.207Z" }, + { url = "https://files.pythonhosted.org/packages/7b/3c/bb4a7cba26956cb3da4553cc2056cf67be5b5ff6e6d8fa4fbdff73bfb7ae/aiohttp-3.14.1-cp314-cp314t-win32.whl", hash = "sha256:47ddf841cdecc810749921d25606dee45857d12d2ad5ddb7b5bd7eab12e4b365", size = 494166, upload-time = "2026-06-07T21:09:28.505Z" }, + { url = "https://files.pythonhosted.org/packages/8a/84/ec80c2c1f66a952555a9f86df6b33af65108a6febfa0471b69013a12f807/aiohttp-3.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5e78b522b7a6e27e0b25d19b247b75039ac4c94f99823e3c9e53ae1603a9f7e9", size = 530255, upload-time = "2026-06-07T21:09:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/2a/71/6e22be134a4061ada85a92951b842f2657f17d926b727f3f94c56ae963d6/aiohttp-3.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:90d53f1609c29ccc2193945ef732428382a28f78d0456ae4d3daf0d48b74f0f6", size = 469640, upload-time = "2026-06-07T21:09:33.028Z" }, ] [[package]] @@ -187,78 +178,94 @@ wheels = [ ] [[package]] -name = "arro3-core" -version = "0.6.5" +name = "anyio" +version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2a/01/f06342d2eb822153f63d188153e41fbeabb29b48247f7a11ce76c538f7d1/arro3_core-0.6.5.tar.gz", hash = "sha256:768078887cd7ac82de4736f94bbd91f6d660f10779848bd5b019f511badd9d75", size = 107522, upload-time = "2025-10-13T23:12:38.872Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/8a/24b35cf01a68621f5f07e3191ca96f70a145022ca367347266901eb504a7/arro3_core-0.6.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:da193dc2fb8c2005d0b3887b09d1a90d42cec1f59f17a8a1a5791f0de90946ae", size = 2678116, upload-time = "2025-10-13T23:09:04.198Z" }, - { url = "https://files.pythonhosted.org/packages/5a/7a/4398bb0582fb22d575f256f2b9ac7be735c765222cc61fb214d606bdb77c/arro3_core-0.6.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed1a760ec39fe19c65e98f45515582408002d0212df5db227a5959ffeb07ad4a", size = 2383214, upload-time = "2025-10-13T23:09:06.841Z" }, - { url = "https://files.pythonhosted.org/packages/82/3f/a321501c5da4bf3ff7438c3e5eb6e63bcecb5630c0f4a89a017cbfa8e4a0/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6584a3d28007740afcef1e301332876e2b785bd8edd59a458a6bc9b051bce052", size = 2883536, upload-time = "2025-10-13T23:09:08.877Z" }, - { url = "https://files.pythonhosted.org/packages/0d/50/1d1e55b9a8c4cf2fdeb954947aa135010554a3333b709e8cad3d5d084be2/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8e0af4789618f02bead4a0cd4d0a54abd9c8aa4fcedf9872b4891d2e3e984161", size = 2908828, upload-time = "2025-10-13T23:09:10.958Z" }, - { url = "https://files.pythonhosted.org/packages/12/75/b4b1de1ccb17890bada9a3f4131cf3137f145d5d10490db51de6b8799926/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c73f212e549e9b6d11cfe3f14bbf3fba9d0891426afb5916688d16d0df724085", size = 3145458, upload-time = "2025-10-13T23:09:13.275Z" }, - { url = "https://files.pythonhosted.org/packages/08/4f/f42ce1840490fd0863bfbc56f28eaaec3bcb4eb322079af9c070111657e5/arro3_core-0.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f88f62e4e276a9e84f250722d2e5ffc078af9a3f67ac691f572a0e05dd6095", size = 2775793, upload-time = "2025-10-13T23:09:15.342Z" }, - { url = "https://files.pythonhosted.org/packages/2b/aa/9637efc8d8733c34bedef44e5b2c170dea14d15ab56b3566d8d7963c2616/arro3_core-0.6.5-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:b2635e4c227f25ff8784dc8efb38cb7c1674646cfdc68ded53f2426289885f0e", size = 2516697, upload-time = "2025-10-13T23:09:17.584Z" }, - { url = "https://files.pythonhosted.org/packages/60/84/1fcfadf956bc25eb5251b1ea7a7099f05198a55764635d2fc9ceafdbdbd1/arro3_core-0.6.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a5f3e936686bcd8542fafc94c68fdb23ec42d1d51a4777967ae815c90aff7296", size = 3023625, upload-time = "2025-10-13T23:09:21.556Z" }, - { url = "https://files.pythonhosted.org/packages/58/d0/52d0cb3c0dfa8e94ba2118b7e91a70da76d6ede9de4e70374f831f38cfdf/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:705c32fec03dadc08f807d69ce557882005d43eb20ec62699f7036340f0d580f", size = 2701346, upload-time = "2025-10-13T23:09:25.031Z" }, - { url = "https://files.pythonhosted.org/packages/69/bf/42a6f6501805c31cb65d8a6e3379eeec4fa6c26dc07c9ce894f363ccad1c/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:56d8166235a4c54e4f7ba082ec76890c820fa8c1b6c995ec59cead62a9698e59", size = 3153207, upload-time = "2025-10-13T23:09:28.254Z" }, - { url = "https://files.pythonhosted.org/packages/4f/e5/41fdee468b33759b42958347c2d70b0461bf8f70ba1762a94cdf2e9b0142/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1ba43ba9081c00767083195222b6be74913de668296f55599658c4b0bb7cd327", size = 3105033, upload-time = "2025-10-13T23:09:31.545Z" }, - { url = "https://files.pythonhosted.org/packages/03/e0/b6d733b4540c05bac546162e045b547031f4d88c67b7c864929d9bce29ad/arro3_core-0.6.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4f5df13c6742e3f0b494cfe9025dccdc8426a74cc9e3e5a1239311e07a4b24e0", size = 2954793, upload-time = "2025-10-13T23:09:34.988Z" }, - { url = "https://files.pythonhosted.org/packages/c0/34/8353ba79c8d0498eaacc077d58b384ef785e0b69c9cbff7c2580136b8fe3/arro3_core-0.6.5-cp310-cp310-win_amd64.whl", hash = "sha256:34676b728178236df63c9ea10b21432392d4b5bb51e2030e77c68eed4dede2ad", size = 2837495, upload-time = "2025-10-13T23:09:38.539Z" }, - { url = "https://files.pythonhosted.org/packages/78/85/20e46d3ed59d2f93be4a4d1abea4f6bef3e96acd59bf5a50726f84303c51/arro3_core-0.6.5-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9d5999506daec1ab31096b3deb1e3573041d6ecadb4ca99c96f7ab26720c592c", size = 2685615, upload-time = "2025-10-13T23:09:41.793Z" }, - { url = "https://files.pythonhosted.org/packages/d0/9c/427d578f7d2bf3149515a8b75217e7189e7b1d74e5c5609e1a7e7f0f8d3c/arro3_core-0.6.5-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:bd3e251184c2dd6ade81c5613256b6d85ab3ddbd5af838b1de657e0ddec017f8", size = 2391944, upload-time = "2025-10-13T23:09:45.266Z" }, - { url = "https://files.pythonhosted.org/packages/90/24/7e4af478eb889bfa401e1c1b8868048ca692e6205affbf81cf3666347852/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7cadb29349960d3821b0515d9df80f2725cea155ad966c699f6084de32e313cb", size = 2888376, upload-time = "2025-10-13T23:09:48.737Z" }, - { url = "https://files.pythonhosted.org/packages/70/3b/01006a96bc980275aa4d2eb759c5f10afb7c85fcdce3c36ddb18635ad23b/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a922e560ed2ccee3293d51b39e013b51cc233895d25ddafcacfb83c540a19e6f", size = 2916568, upload-time = "2025-10-13T23:09:51.95Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2f/4e04c7f5687de6fb6f88aa7590b16bcf507ba17ddbd268525f27b70b7a68/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:68fe6672bf51f039b12046a209cba0a9405e10ae44e5a0d557f091b356a62051", size = 3144223, upload-time = "2025-10-13T23:09:55.387Z" }, - { url = "https://files.pythonhosted.org/packages/31/4a/72dc383d1a0d14f1d453e334e3461e229762edb1bf3f75b3ab977e9386ed/arro3_core-0.6.5-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c3ee95603e375401a58ff763ce2c8aa858e0c4f757c1fb719f48fb070f540b2", size = 2781862, upload-time = "2025-10-13T23:09:59.035Z" }, - { url = "https://files.pythonhosted.org/packages/14/dc/0df7684b683114eaf8e57989b4230edb359cbfb6e98b8770d69128b27572/arro3_core-0.6.5-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:fbaf6b65213630007b798b565e0701c2092a330deeba16bd3d896d401f7e9f28", size = 2522442, upload-time = "2025-10-13T23:10:02.134Z" }, - { url = "https://files.pythonhosted.org/packages/c9/04/75f8627cd7fe4d103eca51760d50269cfbc0bf6beaf83a3cdefb4ebd37c7/arro3_core-0.6.5-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:20679f874558bb2113e96325522625ec64a72687000b7a9578031a4d082c6ef5", size = 3033454, upload-time = "2025-10-13T23:10:05.192Z" }, - { url = "https://files.pythonhosted.org/packages/ea/19/f2d54985da65bf6d3da76218bee56383285035541c8d0cadb53095845b3e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d82d6ec32d5c7c73057fb9c528390289fd5bc94b8d8f28fca9c56fc8e41c412c", size = 2705984, upload-time = "2025-10-13T23:10:08.518Z" }, - { url = "https://files.pythonhosted.org/packages/6c/53/b1d7742d6db7b4aa44d3785956955d651b3ac36db321625fd15466be1aca/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4cba4db0a4203a3ccf131c3fb7804d77f0740d6165ec9efa3aa3acbca87c43a3", size = 3157472, upload-time = "2025-10-13T23:10:11.976Z" }, - { url = "https://files.pythonhosted.org/packages/05/31/68711327dbdd480aed54158fc1c46ab245e860ab0286e0916ce788f9889e/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:e358affc4a0fe5c1b5dccf4f92c43a836aaa4c4eab0906c83b00b60275de3b6d", size = 3117099, upload-time = "2025-10-13T23:10:15.374Z" }, - { url = "https://files.pythonhosted.org/packages/31/e3/15ffca0797d9500b23759ae4477cf052fde8dd47a3890f4e4e1d04639016/arro3_core-0.6.5-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:324e43f07b7681846d00a8995b78bdc4b4a719047aa0d34426b462b8f208ee98", size = 2963677, upload-time = "2025-10-13T23:10:18.828Z" }, - { url = "https://files.pythonhosted.org/packages/bc/02/69e60dbe3bbe2bfc8b6dfa4f4bfcb8d1dd240a137bf2a5f7bcc84703f05c/arro3_core-0.6.5-cp311-abi3-win_amd64.whl", hash = "sha256:285f802c8a42fe29ecb84584d1700bc4c4f974552b75f805e1f4362d28b97080", size = 2850445, upload-time = "2025-10-13T23:10:22.345Z" }, - { url = "https://files.pythonhosted.org/packages/b1/29/2e5b091f6b5cffb6489dbe7ed353841568dde8ac4d1232c77321da1d0925/arro3_core-0.6.5-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:8c20e69c3b3411fd6ed56091f388e699072651e880e682be5bd14f3a392ed3e8", size = 2671985, upload-time = "2025-10-13T23:10:25.515Z" }, - { url = "https://files.pythonhosted.org/packages/30/74/764ac4b58fef3fdfc655416c42349206156db5c687fa24a0674acaeaadbb/arro3_core-0.6.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:92211f1d03221ff74d0b535a576b39601083d8e98e9d47228314573f9d4f9ae2", size = 2382931, upload-time = "2025-10-13T23:10:29.893Z" }, - { url = "https://files.pythonhosted.org/packages/6a/07/bd8c92e218240ae8a30150a5d7a2dab359b452ab54a8bb7b90effe806e3d/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:280d933b75f2649779d76e32a07f91d2352a952f2c97ddf7b320e267f440cd42", size = 2879900, upload-time = "2025-10-13T23:10:33.238Z" }, - { url = "https://files.pythonhosted.org/packages/0f/d4/253725019fe2ae5f5fde87928118ffa568cc59f07b2d6a0e90620938c537/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfc3f6b93b924f43fb7985b06202343c30b43da6bd5055ba8b84eda431e494d4", size = 2904149, upload-time = "2025-10-13T23:10:36.547Z" }, - { url = "https://files.pythonhosted.org/packages/f0/b0/7a3dea641ac8de041c1a34859a2f2a82d3cdf3c3360872101c1d198a1e24/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a5963635eb698ebc7da689e641f68b3998864bab894cf0ca84bd058b8c60d97f", size = 3143477, upload-time = "2025-10-13T23:10:40.232Z" }, - { url = "https://files.pythonhosted.org/packages/a7/05/1a50575be33fe9240898a1b5a8574658a905b5675865285585e070dcf7e2/arro3_core-0.6.5-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac291b3e74b57e56e03373d57530540cbbbfd92e4219fe2778ea531006673fe9", size = 2776522, upload-time = "2025-10-13T23:10:43.413Z" }, - { url = "https://files.pythonhosted.org/packages/2e/bd/e7b03207e7906e94e327cd4190fdb2d26ae52bc4ee1edeb057fed760796b/arro3_core-0.6.5-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:5d3f4cc58a654037d61f61ba230419da2c8f88a0ac82b9d41fe307f7cf9fda97", size = 2515426, upload-time = "2025-10-13T23:10:46.926Z" }, - { url = "https://files.pythonhosted.org/packages/f9/ed/82d1febd5c104eccdfb82434e3619125c328c36da143e19dfa3c86de4a81/arro3_core-0.6.5-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:93cddac90238d64451f5e66c630ded89d0b5fd6d2c099bf3a5151dde2c1ddf1d", size = 3024759, upload-time = "2025-10-13T23:10:50.281Z" }, - { url = "https://files.pythonhosted.org/packages/da/cd/00e06907e42e404c21eb08282dee94ac7a1961facfa9a96d116829031721/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:1fa7ac10db5846c33f4e8b66a6eaa705d84998e38575a835acac9a6a6649933d", size = 2700191, upload-time = "2025-10-13T23:10:53.776Z" }, - { url = "https://files.pythonhosted.org/packages/a3/11/a4bb9a900f456a6905d481bd2289f7a2371dcde024de56779621fd6a92c3/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:ca69f698a065cdbf845d59d412bc204e8f8af12f93737d82e6a18f3cff812349", size = 3149963, upload-time = "2025-10-13T23:10:57.163Z" }, - { url = "https://files.pythonhosted.org/packages/28/8a/79c76ad88b16f2fac25684f7313593738f353355eb1af2307e43efd7b1ca/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:de74a2512e2e2366d4b064c498c38672bf6ddea38acec8b1999b4e66182dd001", size = 3104663, upload-time = "2025-10-13T23:11:00.582Z" }, - { url = "https://files.pythonhosted.org/packages/20/66/9152feaa87f851a37c1a2bd74fb89d7e82e4c76447ee590bf8e6fff5e9d8/arro3_core-0.6.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:806ca8e20507675b2de68b3d009f76e898cc3c3e441c834ea5220866f68aac50", size = 2956440, upload-time = "2025-10-13T23:11:03.769Z" }, - { url = "https://files.pythonhosted.org/packages/ad/66/f4179ef64d5c18fe76ec93cfbff42c0f401438ef771c6766b880044d7e13/arro3_core-0.6.5-cp313-cp313t-win_amd64.whl", hash = "sha256:8f6f0cc78877ade7ad6e678a4671b191406547e7b407bc9637436869c017ed47", size = 2845345, upload-time = "2025-10-13T23:11:07.447Z" }, - { url = "https://files.pythonhosted.org/packages/10/ca/b2139dbb25f9fefb9b1cdce8a73785615de6763af6a16bf6ff96a3b630f2/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:26d5b50139f1a96727fa1760b4d70393acf5ee0fba45346ad2d4f69824d3bdc2", size = 2676788, upload-time = "2025-10-13T23:11:56.965Z" }, - { url = "https://files.pythonhosted.org/packages/34/a1/c68dde2944f493c8ccfcb91bf6da6d27a27c3674316dd09c9560f9e6ab1a/arro3_core-0.6.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b65b3d8d7f65f2f3c36002dc467380d7a31ea771132986dddc6341c5a9dc726f", size = 2382809, upload-time = "2025-10-13T23:12:00.175Z" }, - { url = "https://files.pythonhosted.org/packages/c6/fc/2fb81d42a3cecd632deace97dc23ac74083d60d158106440c783bae4ff01/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6c3442a79a757ed3fbd7793de180019ae3201f04237537c2e2e3f1e3dd99b31c", size = 2882818, upload-time = "2025-10-13T23:12:03.721Z" }, - { url = "https://files.pythonhosted.org/packages/58/7f/16f741e1d49ba5c5a893ce6f8eb0283d64bc68d6cc9e07ac62f96eaadfae/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:def7b0065a684d6f903a658d2567da47e2fcecde716e0b34eff4d899c6468c8d", size = 2907503, upload-time = "2025-10-13T23:12:07.066Z" }, - { url = "https://files.pythonhosted.org/packages/eb/45/2eb7972e0bbec0ee0ab22b0f166ec1ea74b53bd76c93a18ced434713e495/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cbfe2f2d4d0d393833cd6a4bd9c15266a02307a3028f159155a1c536469c3ae7", size = 3143706, upload-time = "2025-10-13T23:12:10.492Z" }, - { url = "https://files.pythonhosted.org/packages/2d/af/b78e28842faa675e4e6c4d82e861accf21ac08bbab80a65fa80c578f80a1/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a191a3e4f72c34f7ace7724a94f2d90b06c804a6cbece4ae0f18d36325479cf3", size = 2775462, upload-time = "2025-10-13T23:12:14.026Z" }, - { url = "https://files.pythonhosted.org/packages/45/df/950e57e4915e0457acadaaca13c4423d5e2652e403135eb7606d5e6e5443/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:e3f6ab4c6ea96c451eff72aa6c5b9835a0ea8a9847cfe3995c88cce0c7701fb5", size = 2516212, upload-time = "2025-10-13T23:12:17.548Z" }, - { url = "https://files.pythonhosted.org/packages/07/73/821640d0827a829ed2565c2d4812080ab7fb86f0d271b462f9b37e6d946e/arro3_core-0.6.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:27df5239835330299636a02977f2cb34d5c460cc03b2ae1d6ab6a03d28051b08", size = 3023342, upload-time = "2025-10-13T23:12:21.308Z" }, - { url = "https://files.pythonhosted.org/packages/fd/30/51302d2f4d1b627dd11e2be979f2c48550b782d8d58d0378316342e284a8/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:71dce89c0e91be4cfb42591f03809235bbc374c396e08acdf93c4d85b09e40f5", size = 2700740, upload-time = "2025-10-13T23:12:24.968Z" }, - { url = "https://files.pythonhosted.org/packages/1d/e8/0c8a345a013bb64abea60b4864bacc01e43b8699b8874794baec9c8a7e76/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:d380c28f85568ed99c1686fb9d64b5a811d76d569f367cbec8ef7e58f6e2fdf9", size = 3152749, upload-time = "2025-10-13T23:12:28.393Z" }, - { url = "https://files.pythonhosted.org/packages/6a/42/003b30c4da394366d5967a5b993f7471a74182c983d8f757891b3dd5d594/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:8e359c0c4fe9992f5a863a4a31502ea58eb2f92988fc2e501850540b3eff0328", size = 3104676, upload-time = "2025-10-13T23:12:31.711Z" }, - { url = "https://files.pythonhosted.org/packages/0b/fd/4f8dac58ea17e05978bf35cb9a3e485b1ff3cdd6e2cc29deb08f54080de4/arro3_core-0.6.5-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:9a58acbc61480b533aa84d735db04b1e68fc7f6807ab694d606c03b5e694d83d", size = 2954405, upload-time = "2025-10-13T23:12:35.328Z" }, -] - -[[package]] -name = "astunparse" -version = "1.6.3" +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "arro3-core" +version = "0.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, - { name = "wheel" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/03/13dde6512ad7b4557eb792fbcf0c653af6076b81e5941d36ec61f7ce6028/astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8", size = 12732, upload-time = "2019-12-22T18:12:11.297Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/c0/c8/fc5bacb6fc264dc61e46d4832f690015b7f6c693ff5dea8a1e53b63cb772/arro3_core-0.8.1.tar.gz", hash = "sha256:1df54a8e2c14a877f291d90de65f00bafe9cc6d5958417ff749f178f059dcd39", size = 93684, upload-time = "2026-06-11T18:01:37.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/18/81ff17882bb43bcef22bfc67d5b9522ee52f39b0bc6516eda718f99fb1aa/arro3_core-0.8.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:88b15a937ebe7e64f63cbbd1134d18daa3cb3f769e936a1f0284136375707ebb", size = 3077128, upload-time = "2026-06-11T17:59:25.637Z" }, + { url = "https://files.pythonhosted.org/packages/a2/c1/f9b9675f12f0cc2da922f4e1d029600f76b834879e11eed22b1a2b5d9521/arro3_core-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0501662cb3c7c6a05dcd6b998db55f9a97743aa571da42203733366c42166a7a", size = 2798583, upload-time = "2026-06-11T17:59:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/39/80/62e19c6935d5633452a76354881be62f7a25f1bc7d8ad2e74faf3c68790a/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5674a052c68721963f462b10ce92596a7e85cfacd3fbdc3839633182844f6f3c", size = 3270157, upload-time = "2026-06-11T17:59:28.785Z" }, + { url = "https://files.pythonhosted.org/packages/9a/7d/0eadbcc314d61cad40679878bab76ae224a476059097af271051e4f36024/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ebc5e8917ccc638f276292532e7a59dad3d89ba8a26e874e596a6bc4ffb0ca4b", size = 3404256, upload-time = "2026-06-11T17:59:30.276Z" }, + { url = "https://files.pythonhosted.org/packages/76/6f/6b2ed8c8c0f4bff723e9485ac64daf3d34e424f3d266517e002644b9171d/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3f30c53a51934884416b752e95ef50b1958d6b55de6a386e919c202ceec7d4ff", size = 3467310, upload-time = "2026-06-11T17:59:31.889Z" }, + { url = "https://files.pythonhosted.org/packages/34/48/0fe2f381f29e16e639e691392e1bbc628d275f950f3a8dec3bab7d04742f/arro3_core-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31b966b0f9b2a6fb3f285339ed5ec4eac69591bbb92a9fdb2cf80ad995b759af", size = 3190458, upload-time = "2026-06-11T17:59:33.385Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/fd6af88fe75a1316f7742ac11937fb9ab94ffbc582e9e722b807cf2957fa/arro3_core-0.8.1-cp310-cp310-manylinux_2_24_aarch64.whl", hash = "sha256:f27615f05e55e7c277b4386fd18356d7b052e0c92be66cc13cf823286d94b141", size = 2950698, upload-time = "2026-06-11T17:59:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/40/a6/5c3f9c4a8f0f8c510e6b0fea41329f704d6be569cc52f7479c179a84d196/arro3_core-0.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f6c9e50b3bc594260cbcc94e4b9fff590c5fd7f35a6c5729ecc6a6ff0a117981", size = 3398996, upload-time = "2026-06-11T17:59:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0c/b52e17b284e9b0b3cb0c402071e1057bf422309407e7f9c3bc80818ee581/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97584eb695f6d9a2cb5e66aa4866fdd17316c313d1724257660bcb0f37a59fb1", size = 3128930, upload-time = "2026-06-11T17:59:37.914Z" }, + { url = "https://files.pythonhosted.org/packages/58/99/6e6b15a8e9b6ea8995f2b0ccd01a91fd3e72af74ef45b856b7675bb58b04/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:35dfbc1c19afb63371ae23bd653202b57fd99b57f17ed1561849c4b5994b63d9", size = 3547956, upload-time = "2026-06-11T17:59:39.327Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3e/82d1df9edcd26632c5b6271ebbc6c825307abb76dc9c8de04094510b3dc6/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:2dd781426d776fad17dba1c991df422d071d22e59cfd0db80b104648b6c96040", size = 3505514, upload-time = "2026-06-11T17:59:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/c1/c8/ffc2d069ba82444fbeb0a41b24a4c51ddfe99935b11c8572ce3c1911c19f/arro3_core-0.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:540f85f0b6a31820347d069c217b002488fe60f8e509d134bd1e0bc2eeffbfff", size = 3405220, upload-time = "2026-06-11T17:59:43.275Z" }, + { url = "https://files.pythonhosted.org/packages/df/12/d7de53951ed8b89bc3f390b416e25e92dd765bc2faa13eedcd467248bc24/arro3_core-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:507c4a58c993af697eb2e7b324c1f2c4ced22a72c85e9c2adbc23d28c6a6c20b", size = 3357259, upload-time = "2026-06-11T17:59:44.961Z" }, + { url = "https://files.pythonhosted.org/packages/ca/75/29517738623cccba1d60d0aa65b896bdd046ab4a82c5cd5a9fca115f6d7a/arro3_core-0.8.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:64d3ea60da4279e9c6a9b2e60abf7f4fefb6643544ecb2dfde899f72f1f91bad", size = 3085640, upload-time = "2026-06-11T17:59:46.597Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b5/bff1842ed8dbb2134b004f78c52d977b52a6df1d9389b1284aad02bf4d93/arro3_core-0.8.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:463db0b7f698abc19f4274d6df697560bc9c19463cfdbd01094bb0fa6ddd1206", size = 2800705, upload-time = "2026-06-11T17:59:48.242Z" }, + { url = "https://files.pythonhosted.org/packages/32/20/ef60404d5008f84bca50510f65d7acfe26812e61638ee19a416bf3f11530/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:881afcb9c83334ac26498b429265223c676e028cd5d88fb0c909578f8e0330de", size = 3272103, upload-time = "2026-06-11T17:59:49.641Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8e/6687efc8e0414cfde6281b1f4ea3b5b44aeb318584e39113d645d1f913a2/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1dc03e1e5965528bfc7e26a6ee857e3b6fc5f201ff530a03dc731b9f3fb14fd", size = 3405750, upload-time = "2026-06-11T17:59:51.037Z" }, + { url = "https://files.pythonhosted.org/packages/2e/38/2567f26cd041387a2c1f381f1757b5246184de22dd8bd9dbbcb91a1b3586/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:410cdca92392be39a5580059ebf9a6f89a0410af6dc10a8baa76e9d511fa6624", size = 3467290, upload-time = "2026-06-11T17:59:52.672Z" }, + { url = "https://files.pythonhosted.org/packages/85/38/2c5af3a3e8c806188f1b3a651cbaa6c22d1f094c41e82900d40219c2b337/arro3_core-0.8.1-cp311-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4082c6bbff7f164619d99dffcbc2313ed69024653a9e58d9f94cc65d9226f6c", size = 3195582, upload-time = "2026-06-11T17:59:54.536Z" }, + { url = "https://files.pythonhosted.org/packages/e4/8d/d7d8686901a273743c53aae98f898f00caaedea807e28854200f2a7365e4/arro3_core-0.8.1-cp311-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:40da2ee0701f217cd482e61228023ee9d8993984daccaf6ded089035b5fdc132", size = 2950909, upload-time = "2026-06-11T17:59:56.05Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8d/f44198a859b13f519b331906d08079c7c281f8267d718082a3ca858c93ee/arro3_core-0.8.1-cp311-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d79b12a83403b1a100731587d33f09a818a4ff93472f84fcdf3a38d3934dfc5e", size = 3403497, upload-time = "2026-06-11T17:59:57.651Z" }, + { url = "https://files.pythonhosted.org/packages/88/6d/431d2ef9942a30599db4a86cf13b7f1de92d340717438074a25659d382f4/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:3ff2da3c888ec504291deafd6965f2364f14f1fb63977f202f2f7680f10909f6", size = 3130901, upload-time = "2026-06-11T17:59:59.262Z" }, + { url = "https://files.pythonhosted.org/packages/ce/9e/458e8596c675fb88075137ab21d4c4a2f9e585101a97d77e910839862168/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0bbcfb60ddf9b571387b4c0aaf78171d2ee7e0d335c08514bc247685f01bc086", size = 3550071, upload-time = "2026-06-11T18:00:00.808Z" }, + { url = "https://files.pythonhosted.org/packages/93/cd/8fa1f4a49e707ea364022620ca5033ba69b6396215e0bae1fb9fa9efcfe0/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_i686.whl", hash = "sha256:44f6256cae47d369fe9076e9fa61f5c2899447b76ca5bda641668ad07cf26bf6", size = 3508957, upload-time = "2026-06-11T18:00:02.341Z" }, + { url = "https://files.pythonhosted.org/packages/7f/02/0126ff3b2ff48187315a9782280c0f6412c066559e22bf206ceec2791299/arro3_core-0.8.1-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b2517ea3fb7cb15a41e2d0e3496d91afe4703eca86a88072f917bef044c2b8ee", size = 3408923, upload-time = "2026-06-11T18:00:03.97Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fd/a19cb50480a769b1cc3f347efa1a808e13b853bb3b1d94b289677115fe92/arro3_core-0.8.1-cp311-abi3-win_amd64.whl", hash = "sha256:6a96df94a4538ab9acf01873eecf35161ad0c54ab79134247e69a42cd66515dc", size = 3368045, upload-time = "2026-06-11T18:00:05.578Z" }, + { url = "https://files.pythonhosted.org/packages/5c/86/1ac6eed229482b1ef6ac6f3211c6760012aaaa026b0a385a18e42ec0ff74/arro3_core-0.8.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:71165a75a7d22c16226085ed2255afe84741369cd35eb01cc153c4687e85b49f", size = 1724964, upload-time = "2026-06-11T18:00:07.653Z" }, + { url = "https://files.pythonhosted.org/packages/4a/32/c2e4a65b3b4f7e00552d280e9c2eaaca59c19466b0f95e4e08d6a020ab2d/arro3_core-0.8.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dc81e079d182dfc2ba6bace815608dbab0158809b9268c4eb59816685f7871fb", size = 3071070, upload-time = "2026-06-11T18:00:09.12Z" }, + { url = "https://files.pythonhosted.org/packages/f0/32/56016de27757bcc0d66999ae2fec624eaa27bd097ee69ac7a5a0d52d407b/arro3_core-0.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7695e9fca1e2c0064571bea65d9b7909a957ed9be4c81718d19356e578818649", size = 2794497, upload-time = "2026-06-11T18:00:10.872Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6e/a7aac0e87d6d63672d4ebb7e8466b276fde2a6413579008fadd1e8e919c5/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ad0f507dfafb1e8a8c15e8a902bccdd9a9c2cf2102fc39ad35c22e23ca76a65e", size = 3272090, upload-time = "2026-06-11T18:00:12.561Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fd/8b96e57eea8ebf5fa4e2556b06c84a29ea25ded924f56d300807ff377a94/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fc91a59cb27c7660134b9ed0c067d53c0f5bb1722f16f2cf7d4c446ce1086c4", size = 3399753, upload-time = "2026-06-11T18:00:13.992Z" }, + { url = "https://files.pythonhosted.org/packages/80/8b/76f1385ecd0448fcc60902a4474f3f00bd2197fe6d7fa4434bb6a0b3dee9/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71bc86880256cca5f22802ca444a92b3e9edbb5c9e5b74df1cbfcfe2ca788097", size = 3468912, upload-time = "2026-06-11T18:00:15.502Z" }, + { url = "https://files.pythonhosted.org/packages/0f/15/e1c3aefee23d926be9f75f125d6d94f189f28826aa01713498f1de60b080/arro3_core-0.8.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94528515f59146b5b4421e468bc269f682754ff082434cccf0ecbfb0d74d0b38", size = 3193338, upload-time = "2026-06-11T18:00:17.064Z" }, + { url = "https://files.pythonhosted.org/packages/de/3d/bfa3963bd09941f69d810a387bbea61deb8a091101c1765216266ec61394/arro3_core-0.8.1-cp313-cp313t-manylinux_2_24_aarch64.whl", hash = "sha256:e348fa5349e6655df4b1ea281ab71e0794c749ba842da54248eb942201f93fb8", size = 2950166, upload-time = "2026-06-11T18:00:18.607Z" }, + { url = "https://files.pythonhosted.org/packages/21/07/67bdc66582294d5914b8f7462907b1e44377ccb77a0f7e7b7760c9f81fd1/arro3_core-0.8.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d97170d56690655699977f4535120f170c5047813341c2b0264abc3af00625dc", size = 3401097, upload-time = "2026-06-11T18:00:20.135Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a3/ab0d46e8fe2337f0ce1789949c5078e63d532fd628af4f254702e1359989/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6f71fdd46e1a86bc1db0733361130e51447d2d3c7d5fca13937381e2ba085dca", size = 3128454, upload-time = "2026-06-11T18:00:21.598Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6e/e90cea2555aaa429c16173cbc121b41fb6a656e70ae9f21edbe3f9e73eb0/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3e2c2e4c6b36a0f493f07222eadf7c628155f4a306c650a14c4d6d343870043c", size = 3550294, upload-time = "2026-06-11T18:00:24.344Z" }, + { url = "https://files.pythonhosted.org/packages/15/38/f3db72e411fbaea83844d19b1965abbdf3dd387083f99a715d3b3752b461/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:90d31db38bc7cbbe6061d23ae11b3e830b55ce60ba6f0550c70915bab4cafe41", size = 3507806, upload-time = "2026-06-11T18:00:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/47/73/5d011dad78ac58aaa61e9cc65d349b879afec6781c5aa78f80c839fe359f/arro3_core-0.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9d28d87625d6c743fee1d2e20252531db6cff6c5218b89055a9f7492af99c14c", size = 3408910, upload-time = "2026-06-11T18:00:27.843Z" }, + { url = "https://files.pythonhosted.org/packages/6a/43/be86ea01c2d53d32376602cb2efa9a9b7a25d98e14ad4b5bf0d7da8b0564/arro3_core-0.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:b394218ad71ce266088e0e65bfad0542556adf0eb93197b66afaa2009358662f", size = 3356174, upload-time = "2026-06-11T18:00:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0f/2f7b5b458fd38ecf9277fd59919e4c90047795d1abdfd47f0081c48c28fa/arro3_core-0.8.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:a7c227982c0fd762271d550a6242e54ff7d2aa7aff73c918e127042cf9601ec0", size = 1710676, upload-time = "2026-06-11T18:00:30.947Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9e/b64c9325173b7d87030955007af55348b866a1c02545382261d8966d8bb1/arro3_core-0.8.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:5861781e6363db0707e1a4ba5166025a1113d921ccff870a0a097f27ab2dece1", size = 3072327, upload-time = "2026-06-11T18:00:32.313Z" }, + { url = "https://files.pythonhosted.org/packages/3c/bf/2c58a549b2409439fcac4ce2abe31ef060d05114a1e5412ce3088e2a2cde/arro3_core-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8c1d9ff8b2848bf48bbcad962271b2adeda922457cf23591464c12af9b0ddbf6", size = 2794872, upload-time = "2026-06-11T18:00:33.92Z" }, + { url = "https://files.pythonhosted.org/packages/1f/21/dee8d1c9309820783fe30ad29149508638743cdc61d2851f0132b370ac49/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:246057dc9d283a283cb05b228f710e1ada3b4a8c483e519b0ec8b0d8499204db", size = 3272432, upload-time = "2026-06-11T18:00:35.563Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b2/dbc6a3d5ece2cb11c3041821c8322df500b8c8030534d9f791b66b29c090/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d55178ad56c637f278286370532e58256eff48f71d3c36a4e5f8652eb7300b3", size = 3400822, upload-time = "2026-06-11T18:00:37.129Z" }, + { url = "https://files.pythonhosted.org/packages/a4/44/0ca0e96eebc7f96af4ad8b78c545a9477e8db271aa090043bb8f797e7f58/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a917195d7553a188bbb9fa815d430ff70ee53357fcc6fdb395d0e4436a66243", size = 3469220, upload-time = "2026-06-11T18:00:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/8e/70/3e9c63e9e4499d373304ceb1315afc8dd326395dd0692ccdebcfec703dab/arro3_core-0.8.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2435dc4057d604650a34d195e6977ef40b26af0b9dea8f1b842a924a270de2fc", size = 3193813, upload-time = "2026-06-11T18:00:41.281Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a8/48842a836d7fc2bdac16ecaf4e60c9fd98f46d1e9c80f45b85b071ccaffb/arro3_core-0.8.1-cp314-cp314t-manylinux_2_24_aarch64.whl", hash = "sha256:c8b90ad6fbdd3507c20bdbc2b23c88929a3f7c49a8a43697f316e084b5664289", size = 2950818, upload-time = "2026-06-11T18:00:42.987Z" }, + { url = "https://files.pythonhosted.org/packages/bb/46/635ca59c20b972f0575b9fd1b7debebd94bcba1d6fd68271bfff65d3ae91/arro3_core-0.8.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8918dd0fcf5c3650332678ddbfb7c13ce1f0534f9669ab6839c38069885318cf", size = 3401202, upload-time = "2026-06-11T18:00:44.654Z" }, + { url = "https://files.pythonhosted.org/packages/4a/03/020d032cf4937cca0652f434613d327f8f11e10b4a3fbd4ccf48846c3158/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cc8c7c9b81cad2b4eaaeb29da25854bddb5bb373a278f1f1046b277fa7f2ee6", size = 3129352, upload-time = "2026-06-11T18:00:46.322Z" }, + { url = "https://files.pythonhosted.org/packages/b2/55/ce1c840af64e8c5d4f8b684a4c00c5f6d260659c3d2f732c6074eb5597fe/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2aa0bda5251e93c67318aa8315e563b02fe39d5577b72654c335d69fce802d03", size = 3550475, upload-time = "2026-06-11T18:00:48.021Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/fa8f1a53cf10fe33029619d14f06525e08ea35c1bcdce42157b230098e1b/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:cd462e7ef7735c341c27f754756b244d90a9485818c5ea090fa46009bd8ae252", size = 3507877, upload-time = "2026-06-11T18:00:49.798Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e5/bf8349b2e5613e6c0caf2f4f0758d61dc273649c2bdcfc96978ab9bc1686/arro3_core-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:480c33d62d66adf92fd731e77728f81b7bd3fcde4e2eb313f5616b5097b1875f", size = 3409360, upload-time = "2026-06-11T18:00:51.393Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0d/e13efbbc448bc817f8343bf12606d22f83011682c821ccd32dda304816be/arro3_core-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:51bea8dc0bc0230b6af8d1b258cf2a7e4f69770ed99062592ce1b43b17732727", size = 3355557, upload-time = "2026-06-11T18:00:52.922Z" }, + { url = "https://files.pythonhosted.org/packages/de/bc/ab3081cc65cd38a5ceda6dd112ed9d70a481e2810c8f92d37441edab6a8a/arro3_core-0.8.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d70c68665525744154dc1f65b5038fe97600a5204e8a875b3fe7675455363024", size = 3076025, upload-time = "2026-06-11T18:01:16.781Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5b/b278236c7425f36fe1d25087964ca143d2a11f99b69b19f3c050cf3a0f4b/arro3_core-0.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b89c867f65852de7fa0586a4883195a2cca1ee3a7bca8c95c309a02ce6be15b5", size = 2797246, upload-time = "2026-06-11T18:01:18.648Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/a3a96bef25a7d0d7fd49ea3762d901a17d908fda774c868abe9aa065e0f2/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66d685f7c53137798e05322a17dabc09445a818c9c8959d21e9fb3e084371f51", size = 3269959, upload-time = "2026-06-11T18:01:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/f7/2f/4da148dea5890ae24eade70d3f9dfc93d191f501bf425d906c21dee89510/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a7563856e609dd24be03efde72edd5b1c16dd1fb3d67fd84970df6d9ca36b96", size = 3400877, upload-time = "2026-06-11T18:01:22.112Z" }, + { url = "https://files.pythonhosted.org/packages/7b/9a/2ed015d4a03c131b9e3da322ee8681c5921320a90218865a711d7fe39768/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7379e14f8f49e99d53e364c26533293d2136920a3cf818138718a9a9bc114d0", size = 3466119, upload-time = "2026-06-11T18:01:23.909Z" }, + { url = "https://files.pythonhosted.org/packages/25/a8/154478c9b3093602eaec8a5d71b657205177f0ac9fd9da2eef9ddfb42fe0/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af88a17346e9e7897b6d1bc165f686b87ea928b4c7e89666156b2281f0236cee", size = 3190211, upload-time = "2026-06-11T18:01:25.786Z" }, + { url = "https://files.pythonhosted.org/packages/5b/65/60c6323bbaffd2399f03336a90d7c02450a5b9120348808eb0260651de56/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_24_aarch64.whl", hash = "sha256:1e9e04cbbba877f31073146798b328537b63675f65d0b7e2154754d5203260ed", size = 2950345, upload-time = "2026-06-11T18:01:27.423Z" }, + { url = "https://files.pythonhosted.org/packages/21/81/3a0c786ec86a8c3aee946e4588b188f08a3ccbba71c4e0db144a0151b56c/arro3_core-0.8.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:35c46c49442853f8a1b09d37ceabe4ce31e2a276d4a73c407ba3a25fcf1b9221", size = 3398153, upload-time = "2026-06-11T18:01:29.036Z" }, + { url = "https://files.pythonhosted.org/packages/f6/52/e230d0af9881056e527f91e58c518b85000e9ae49f8706c3d0e027cbadd4/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:4e4af152b4ccf7694d634573db906e1a9b28f5beda84bac5adeb60f7e98262a8", size = 3128123, upload-time = "2026-06-11T18:01:30.733Z" }, + { url = "https://files.pythonhosted.org/packages/62/79/9bc36279f1ef2754fde18e1c3c282b2cabe6093b7ca8bd8c0c6c735685ed/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:7bf8cb1053cb51529437544eedbc6643819912eec15c0491d7304d6bad0fb151", size = 3547345, upload-time = "2026-06-11T18:01:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/f9/33/cee8670131009d3a23e140dd0a6cd454111dac54a9a2e83567a6f972ad46/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:212dfde17bd26193754aaee6024f7e99624163e0d44966745e8752af79c4c454", size = 3504672, upload-time = "2026-06-11T18:01:34.207Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5f/daa7a8af9ad53eebbdb818078d5034525663f298ec9bfdb1933cfd0cc694/arro3_core-0.8.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:81426cb122d6bfccacd2e0ae5df8b847dc3c3e07b5f336547cb18f873d186f54", size = 3404906, upload-time = "2026-06-11T18:01:36.097Z" }, ] [[package]] @@ -272,11 +279,11 @@ wheels = [ [[package]] name = "attrs" -version = "25.3.0" +version = "26.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] [[package]] @@ -295,89 +302,133 @@ wheels = [ [[package]] name = "botocore" -version = "1.40.43" +version = "1.40.76" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/d0/3888673417202262ddd7e6361cab8e01ee2705e39643af8445e2eb276eab/botocore-1.40.43.tar.gz", hash = "sha256:d87412dc1ea785df156f412627d3417c9f9eb45601fd0846d8fe96fe3c78b630", size = 14389164, upload-time = "2025-10-01T19:38:16.06Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/eb/50e2d280589a3c20c3b649bb66262d2b53a25c03262e4cc492048ac7540a/botocore-1.40.76.tar.gz", hash = "sha256:2b16024d68b29b973005adfb5039adfe9099ebe772d40a90ca89f2e165c495dc", size = 14494001, upload-time = "2025-11-18T20:22:59.131Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/46/2eb4802e15e38befbea6cab7dafa1ab796722ab6f0833991c2a05e9f8ef0/botocore-1.40.43-py3-none-any.whl", hash = "sha256:1639f38999fc0cf42c92c5c83c5fbe189a4857a86f55b842be868e3283c6d3bb", size = 14057986, upload-time = "2025-10-01T19:38:13.714Z" }, + { url = "https://files.pythonhosted.org/packages/7f/6c/522e05388aa6fc66cf8ea46c6b29809a1a6f527ea864998b01ffb368ca36/botocore-1.40.76-py3-none-any.whl", hash = "sha256:fe425d386e48ac64c81cbb4a7181688d813df2e2b4c78b95ebe833c9e868c6f4", size = 14161738, upload-time = "2025-11-18T20:22:55.332Z" }, ] [[package]] name = "certifi" -version = "2025.8.3" +version = "2026.6.17" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, + { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" }, ] [[package]] name = "charset-normalizer" -version = "3.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/83/2d/5fd176ceb9b2fc619e63405525573493ca23441330fcdaee6bef9460e924/charset_normalizer-3.4.3.tar.gz", hash = "sha256:6fce4b8500244f6fcb71465d4a4930d132ba9ab8e71a7859e6a5d59851068d14", size = 122371, upload-time = "2025-08-09T07:57:28.46Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/98/f3b8013223728a99b908c9344da3aa04ee6e3fa235f19409033eda92fb78/charset_normalizer-3.4.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fb7f67a1bfa6e40b438170ebdc8158b78dc465a5a67b6dde178a46987b244a72", size = 207695, upload-time = "2025-08-09T07:55:36.452Z" }, - { url = "https://files.pythonhosted.org/packages/21/40/5188be1e3118c82dcb7c2a5ba101b783822cfb413a0268ed3be0468532de/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc9370a2da1ac13f0153780040f465839e6cccb4a1e44810124b4e22483c93fe", size = 147153, upload-time = "2025-08-09T07:55:38.467Z" }, - { url = "https://files.pythonhosted.org/packages/37/60/5d0d74bc1e1380f0b72c327948d9c2aca14b46a9efd87604e724260f384c/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07a0eae9e2787b586e129fdcbe1af6997f8d0e5abaa0bc98c0e20e124d67e601", size = 160428, upload-time = "2025-08-09T07:55:40.072Z" }, - { url = "https://files.pythonhosted.org/packages/85/9a/d891f63722d9158688de58d050c59dc3da560ea7f04f4c53e769de5140f5/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:74d77e25adda8581ffc1c720f1c81ca082921329452eba58b16233ab1842141c", size = 157627, upload-time = "2025-08-09T07:55:41.706Z" }, - { url = "https://files.pythonhosted.org/packages/65/1a/7425c952944a6521a9cfa7e675343f83fd82085b8af2b1373a2409c683dc/charset_normalizer-3.4.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0e909868420b7049dafd3a31d45125b31143eec59235311fc4c57ea26a4acd2", size = 152388, upload-time = "2025-08-09T07:55:43.262Z" }, - { url = "https://files.pythonhosted.org/packages/f0/c9/a2c9c2a355a8594ce2446085e2ec97fd44d323c684ff32042e2a6b718e1d/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c6f162aabe9a91a309510d74eeb6507fab5fff92337a15acbe77753d88d9dcf0", size = 150077, upload-time = "2025-08-09T07:55:44.903Z" }, - { url = "https://files.pythonhosted.org/packages/3b/38/20a1f44e4851aa1c9105d6e7110c9d020e093dfa5836d712a5f074a12bf7/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:4ca4c094de7771a98d7fbd67d9e5dbf1eb73efa4f744a730437d8a3a5cf994f0", size = 161631, upload-time = "2025-08-09T07:55:46.346Z" }, - { url = "https://files.pythonhosted.org/packages/a4/fa/384d2c0f57edad03d7bec3ebefb462090d8905b4ff5a2d2525f3bb711fac/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:02425242e96bcf29a49711b0ca9f37e451da7c70562bc10e8ed992a5a7a25cc0", size = 159210, upload-time = "2025-08-09T07:55:47.539Z" }, - { url = "https://files.pythonhosted.org/packages/33/9e/eca49d35867ca2db336b6ca27617deed4653b97ebf45dfc21311ce473c37/charset_normalizer-3.4.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:78deba4d8f9590fe4dae384aeff04082510a709957e968753ff3c48399f6f92a", size = 153739, upload-time = "2025-08-09T07:55:48.744Z" }, - { url = "https://files.pythonhosted.org/packages/2a/91/26c3036e62dfe8de8061182d33be5025e2424002125c9500faff74a6735e/charset_normalizer-3.4.3-cp310-cp310-win32.whl", hash = "sha256:d79c198e27580c8e958906f803e63cddb77653731be08851c7df0b1a14a8fc0f", size = 99825, upload-time = "2025-08-09T07:55:50.305Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/f05db471f81af1fa01839d44ae2a8bfeec8d2a8b4590f16c4e7393afd323/charset_normalizer-3.4.3-cp310-cp310-win_amd64.whl", hash = "sha256:c6e490913a46fa054e03699c70019ab869e990270597018cef1d8562132c2669", size = 107452, upload-time = "2025-08-09T07:55:51.461Z" }, - { url = "https://files.pythonhosted.org/packages/7f/b5/991245018615474a60965a7c9cd2b4efbaabd16d582a5547c47ee1c7730b/charset_normalizer-3.4.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b256ee2e749283ef3ddcff51a675ff43798d92d746d1a6e4631bf8c707d22d0b", size = 204483, upload-time = "2025-08-09T07:55:53.12Z" }, - { url = "https://files.pythonhosted.org/packages/c7/2a/ae245c41c06299ec18262825c1569c5d3298fc920e4ddf56ab011b417efd/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13faeacfe61784e2559e690fc53fa4c5ae97c6fcedb8eb6fb8d0a15b475d2c64", size = 145520, upload-time = "2025-08-09T07:55:54.712Z" }, - { url = "https://files.pythonhosted.org/packages/3a/a4/b3b6c76e7a635748c4421d2b92c7b8f90a432f98bda5082049af37ffc8e3/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:00237675befef519d9af72169d8604a067d92755e84fe76492fef5441db05b91", size = 158876, upload-time = "2025-08-09T07:55:56.024Z" }, - { url = "https://files.pythonhosted.org/packages/e2/e6/63bb0e10f90a8243c5def74b5b105b3bbbfb3e7bb753915fe333fb0c11ea/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:585f3b2a80fbd26b048a0be90c5aae8f06605d3c92615911c3a2b03a8a3b796f", size = 156083, upload-time = "2025-08-09T07:55:57.582Z" }, - { url = "https://files.pythonhosted.org/packages/87/df/b7737ff046c974b183ea9aa111b74185ac8c3a326c6262d413bd5a1b8c69/charset_normalizer-3.4.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0e78314bdc32fa80696f72fa16dc61168fda4d6a0c014e0380f9d02f0e5d8a07", size = 150295, upload-time = "2025-08-09T07:55:59.147Z" }, - { url = "https://files.pythonhosted.org/packages/61/f1/190d9977e0084d3f1dc169acd060d479bbbc71b90bf3e7bf7b9927dec3eb/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:96b2b3d1a83ad55310de8c7b4a2d04d9277d5591f40761274856635acc5fcb30", size = 148379, upload-time = "2025-08-09T07:56:00.364Z" }, - { url = "https://files.pythonhosted.org/packages/4c/92/27dbe365d34c68cfe0ca76f1edd70e8705d82b378cb54ebbaeabc2e3029d/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:939578d9d8fd4299220161fdd76e86c6a251987476f5243e8864a7844476ba14", size = 160018, upload-time = "2025-08-09T07:56:01.678Z" }, - { url = "https://files.pythonhosted.org/packages/99/04/baae2a1ea1893a01635d475b9261c889a18fd48393634b6270827869fa34/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:fd10de089bcdcd1be95a2f73dbe6254798ec1bda9f450d5828c96f93e2536b9c", size = 157430, upload-time = "2025-08-09T07:56:02.87Z" }, - { url = "https://files.pythonhosted.org/packages/2f/36/77da9c6a328c54d17b960c89eccacfab8271fdaaa228305330915b88afa9/charset_normalizer-3.4.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e8ac75d72fa3775e0b7cb7e4629cec13b7514d928d15ef8ea06bca03ef01cae", size = 151600, upload-time = "2025-08-09T07:56:04.089Z" }, - { url = "https://files.pythonhosted.org/packages/64/d4/9eb4ff2c167edbbf08cdd28e19078bf195762e9bd63371689cab5ecd3d0d/charset_normalizer-3.4.3-cp311-cp311-win32.whl", hash = "sha256:6cf8fd4c04756b6b60146d98cd8a77d0cdae0e1ca20329da2ac85eed779b6849", size = 99616, upload-time = "2025-08-09T07:56:05.658Z" }, - { url = "https://files.pythonhosted.org/packages/f4/9c/996a4a028222e7761a96634d1820de8a744ff4327a00ada9c8942033089b/charset_normalizer-3.4.3-cp311-cp311-win_amd64.whl", hash = "sha256:31a9a6f775f9bcd865d88ee350f0ffb0e25936a7f930ca98995c05abf1faf21c", size = 107108, upload-time = "2025-08-09T07:56:07.176Z" }, - { url = "https://files.pythonhosted.org/packages/e9/5e/14c94999e418d9b87682734589404a25854d5f5d0408df68bc15b6ff54bb/charset_normalizer-3.4.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:e28e334d3ff134e88989d90ba04b47d84382a828c061d0d1027b1b12a62b39b1", size = 205655, upload-time = "2025-08-09T07:56:08.475Z" }, - { url = "https://files.pythonhosted.org/packages/7d/a8/c6ec5d389672521f644505a257f50544c074cf5fc292d5390331cd6fc9c3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cacf8f7297b0c4fcb74227692ca46b4a5852f8f4f24b3c766dd94a1075c4884", size = 146223, upload-time = "2025-08-09T07:56:09.708Z" }, - { url = "https://files.pythonhosted.org/packages/fc/eb/a2ffb08547f4e1e5415fb69eb7db25932c52a52bed371429648db4d84fb1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c6fd51128a41297f5409deab284fecbe5305ebd7e5a1f959bee1c054622b7018", size = 159366, upload-time = "2025-08-09T07:56:11.326Z" }, - { url = "https://files.pythonhosted.org/packages/82/10/0fd19f20c624b278dddaf83b8464dcddc2456cb4b02bb902a6da126b87a1/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cfb2aad70f2c6debfbcb717f23b7eb55febc0bb23dcffc0f076009da10c6392", size = 157104, upload-time = "2025-08-09T07:56:13.014Z" }, - { url = "https://files.pythonhosted.org/packages/16/ab/0233c3231af734f5dfcf0844aa9582d5a1466c985bbed6cedab85af9bfe3/charset_normalizer-3.4.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1606f4a55c0fd363d754049cdf400175ee96c992b1f8018b993941f221221c5f", size = 151830, upload-time = "2025-08-09T07:56:14.428Z" }, - { url = "https://files.pythonhosted.org/packages/ae/02/e29e22b4e02839a0e4a06557b1999d0a47db3567e82989b5bb21f3fbbd9f/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:027b776c26d38b7f15b26a5da1044f376455fb3766df8fc38563b4efbc515154", size = 148854, upload-time = "2025-08-09T07:56:16.051Z" }, - { url = "https://files.pythonhosted.org/packages/05/6b/e2539a0a4be302b481e8cafb5af8792da8093b486885a1ae4d15d452bcec/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:42e5088973e56e31e4fa58eb6bd709e42fc03799c11c42929592889a2e54c491", size = 160670, upload-time = "2025-08-09T07:56:17.314Z" }, - { url = "https://files.pythonhosted.org/packages/31/e7/883ee5676a2ef217a40ce0bffcc3d0dfbf9e64cbcfbdf822c52981c3304b/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:cc34f233c9e71701040d772aa7490318673aa7164a0efe3172b2981218c26d93", size = 158501, upload-time = "2025-08-09T07:56:18.641Z" }, - { url = "https://files.pythonhosted.org/packages/c1/35/6525b21aa0db614cf8b5792d232021dca3df7f90a1944db934efa5d20bb1/charset_normalizer-3.4.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:320e8e66157cc4e247d9ddca8e21f427efc7a04bbd0ac8a9faf56583fa543f9f", size = 153173, upload-time = "2025-08-09T07:56:20.289Z" }, - { url = "https://files.pythonhosted.org/packages/50/ee/f4704bad8201de513fdc8aac1cabc87e38c5818c93857140e06e772b5892/charset_normalizer-3.4.3-cp312-cp312-win32.whl", hash = "sha256:fb6fecfd65564f208cbf0fba07f107fb661bcd1a7c389edbced3f7a493f70e37", size = 99822, upload-time = "2025-08-09T07:56:21.551Z" }, - { url = "https://files.pythonhosted.org/packages/39/f5/3b3836ca6064d0992c58c7561c6b6eee1b3892e9665d650c803bd5614522/charset_normalizer-3.4.3-cp312-cp312-win_amd64.whl", hash = "sha256:86df271bf921c2ee3818f0522e9a5b8092ca2ad8b065ece5d7d9d0e9f4849bcc", size = 107543, upload-time = "2025-08-09T07:56:23.115Z" }, - { url = "https://files.pythonhosted.org/packages/65/ca/2135ac97709b400c7654b4b764daf5c5567c2da45a30cdd20f9eefe2d658/charset_normalizer-3.4.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:14c2a87c65b351109f6abfc424cab3927b3bdece6f706e4d12faaf3d52ee5efe", size = 205326, upload-time = "2025-08-09T07:56:24.721Z" }, - { url = "https://files.pythonhosted.org/packages/71/11/98a04c3c97dd34e49c7d247083af03645ca3730809a5509443f3c37f7c99/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41d1fc408ff5fdfb910200ec0e74abc40387bccb3252f3f27c0676731df2b2c8", size = 146008, upload-time = "2025-08-09T07:56:26.004Z" }, - { url = "https://files.pythonhosted.org/packages/60/f5/4659a4cb3c4ec146bec80c32d8bb16033752574c20b1252ee842a95d1a1e/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1bb60174149316da1c35fa5233681f7c0f9f514509b8e399ab70fea5f17e45c9", size = 159196, upload-time = "2025-08-09T07:56:27.25Z" }, - { url = "https://files.pythonhosted.org/packages/86/9e/f552f7a00611f168b9a5865a1414179b2c6de8235a4fa40189f6f79a1753/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30d006f98569de3459c2fc1f2acde170b7b2bd265dc1943e87e1a4efe1b67c31", size = 156819, upload-time = "2025-08-09T07:56:28.515Z" }, - { url = "https://files.pythonhosted.org/packages/7e/95/42aa2156235cbc8fa61208aded06ef46111c4d3f0de233107b3f38631803/charset_normalizer-3.4.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:416175faf02e4b0810f1f38bcb54682878a4af94059a1cd63b8747244420801f", size = 151350, upload-time = "2025-08-09T07:56:29.716Z" }, - { url = "https://files.pythonhosted.org/packages/c2/a9/3865b02c56f300a6f94fc631ef54f0a8a29da74fb45a773dfd3dcd380af7/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6aab0f181c486f973bc7262a97f5aca3ee7e1437011ef0c2ec04b5a11d16c927", size = 148644, upload-time = "2025-08-09T07:56:30.984Z" }, - { url = "https://files.pythonhosted.org/packages/77/d9/cbcf1a2a5c7d7856f11e7ac2d782aec12bdfea60d104e60e0aa1c97849dc/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdabf8315679312cfa71302f9bd509ded4f2f263fb5b765cf1433b39106c3cc9", size = 160468, upload-time = "2025-08-09T07:56:32.252Z" }, - { url = "https://files.pythonhosted.org/packages/f6/42/6f45efee8697b89fda4d50580f292b8f7f9306cb2971d4b53f8914e4d890/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:bd28b817ea8c70215401f657edef3a8aa83c29d447fb0b622c35403780ba11d5", size = 158187, upload-time = "2025-08-09T07:56:33.481Z" }, - { url = "https://files.pythonhosted.org/packages/70/99/f1c3bdcfaa9c45b3ce96f70b14f070411366fa19549c1d4832c935d8e2c3/charset_normalizer-3.4.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:18343b2d246dc6761a249ba1fb13f9ee9a2bcd95decc767319506056ea4ad4dc", size = 152699, upload-time = "2025-08-09T07:56:34.739Z" }, - { url = "https://files.pythonhosted.org/packages/a3/ad/b0081f2f99a4b194bcbb1934ef3b12aa4d9702ced80a37026b7607c72e58/charset_normalizer-3.4.3-cp313-cp313-win32.whl", hash = "sha256:6fb70de56f1859a3f71261cbe41005f56a7842cc348d3aeb26237560bfa5e0ce", size = 99580, upload-time = "2025-08-09T07:56:35.981Z" }, - { url = "https://files.pythonhosted.org/packages/9a/8f/ae790790c7b64f925e5c953b924aaa42a243fb778fed9e41f147b2a5715a/charset_normalizer-3.4.3-cp313-cp313-win_amd64.whl", hash = "sha256:cf1ebb7d78e1ad8ec2a8c4732c7be2e736f6e5123a4146c5b89c9d1f585f8cef", size = 107366, upload-time = "2025-08-09T07:56:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/8e/91/b5a06ad970ddc7a0e513112d40113e834638f4ca1120eb727a249fb2715e/charset_normalizer-3.4.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3cd35b7e8aedeb9e34c41385fda4f73ba609e561faedfae0a9e75e44ac558a15", size = 204342, upload-time = "2025-08-09T07:56:38.687Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ec/1edc30a377f0a02689342f214455c3f6c2fbedd896a1d2f856c002fc3062/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89bc04de1d83006373429975f8ef9e7932534b8cc9ca582e4db7d20d91816db", size = 145995, upload-time = "2025-08-09T07:56:40.048Z" }, - { url = "https://files.pythonhosted.org/packages/17/e5/5e67ab85e6d22b04641acb5399c8684f4d37caf7558a53859f0283a650e9/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2001a39612b241dae17b4687898843f254f8748b796a2e16f1051a17078d991d", size = 158640, upload-time = "2025-08-09T07:56:41.311Z" }, - { url = "https://files.pythonhosted.org/packages/f1/e5/38421987f6c697ee3722981289d554957c4be652f963d71c5e46a262e135/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8dcfc373f888e4fb39a7bc57e93e3b845e7f462dacc008d9749568b1c4ece096", size = 156636, upload-time = "2025-08-09T07:56:43.195Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e4/5a075de8daa3ec0745a9a3b54467e0c2967daaaf2cec04c845f73493e9a1/charset_normalizer-3.4.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18b97b8404387b96cdbd30ad660f6407799126d26a39ca65729162fd810a99aa", size = 150939, upload-time = "2025-08-09T07:56:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/02/f7/3611b32318b30974131db62b4043f335861d4d9b49adc6d57c1149cc49d4/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccf600859c183d70eb47e05a44cd80a4ce77394d1ac0f79dbd2dd90a69a3a049", size = 148580, upload-time = "2025-08-09T07:56:46.684Z" }, - { url = "https://files.pythonhosted.org/packages/7e/61/19b36f4bd67f2793ab6a99b979b4e4f3d8fc754cbdffb805335df4337126/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:53cd68b185d98dde4ad8990e56a58dea83a4162161b1ea9272e5c9182ce415e0", size = 159870, upload-time = "2025-08-09T07:56:47.941Z" }, - { url = "https://files.pythonhosted.org/packages/06/57/84722eefdd338c04cf3030ada66889298eaedf3e7a30a624201e0cbe424a/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:30a96e1e1f865f78b030d65241c1ee850cdf422d869e9028e2fc1d5e4db73b92", size = 157797, upload-time = "2025-08-09T07:56:49.756Z" }, - { url = "https://files.pythonhosted.org/packages/72/2a/aff5dd112b2f14bcc3462c312dce5445806bfc8ab3a7328555da95330e4b/charset_normalizer-3.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d716a916938e03231e86e43782ca7878fb602a125a91e7acb8b5112e2e96ac16", size = 152224, upload-time = "2025-08-09T07:56:51.369Z" }, - { url = "https://files.pythonhosted.org/packages/b7/8c/9839225320046ed279c6e839d51f028342eb77c91c89b8ef2549f951f3ec/charset_normalizer-3.4.3-cp314-cp314-win32.whl", hash = "sha256:c6dbd0ccdda3a2ba7c2ecd9d77b37f3b5831687d8dc1b6ca5f56a4880cc7b7ce", size = 100086, upload-time = "2025-08-09T07:56:52.722Z" }, - { url = "https://files.pythonhosted.org/packages/ee/7a/36fbcf646e41f710ce0a563c1c9a343c6edf9be80786edeb15b6f62e17db/charset_normalizer-3.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:73dc19b562516fc9bcf6e5d6e596df0b4eb98d87e4f79f3ae71840e6ed21361c", size = 107400, upload-time = "2025-08-09T07:56:55.172Z" }, - { url = "https://files.pythonhosted.org/packages/8a/1f/f041989e93b001bc4e44bb1669ccdcf54d3f00e628229a85b08d330615c5/charset_normalizer-3.4.3-py3-none-any.whl", hash = "sha256:ce571ab16d890d23b5c278547ba694193a45011ff86a9162a71307ed9f86759a", size = 53175, upload-time = "2025-08-09T07:57:26.864Z" }, +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/81/8e983840c6e5b93b33c2ba81aa3d52c2e42f0e9a690ce7607a2e61da4a5c/charset_normalizer-3.4.9-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a", size = 322240, upload-time = "2026-07-07T14:32:36.236Z" }, + { url = "https://files.pythonhosted.org/packages/de/d1/b4319dc3229d8272fba305e206fc0a148e2de8d4087917ce62ae6382f359/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616", size = 216475, upload-time = "2026-07-07T14:32:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/33/6c99c1b3e6b8bf730e1bc809b9a2608f224145069114c479a2e9e1494346/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209", size = 238670, upload-time = "2026-07-07T14:32:39.658Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f4/ffbb83546e1f198ecc70ecd372b65cf2b50f9068b380abd67640f17a8e18/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99", size = 233476, upload-time = "2026-07-07T14:32:41.155Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5f/b98b8da398637b551e427e7be922bdec19177dc54d6811dcdaa503f23aac/charset_normalizer-3.4.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8", size = 223817, upload-time = "2026-07-07T14:32:42.592Z" }, + { url = "https://files.pythonhosted.org/packages/36/31/a276bb2e66243072a3fd06fdcab9cbb61a305b02143d70d2bda21d888fa8/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b", size = 207974, upload-time = "2026-07-07T14:32:44.258Z" }, + { url = "https://files.pythonhosted.org/packages/5e/be/7ee4453d7e88dfbc4104ccd34900b9f2c7c17dac22881865fe0e82424a25/charset_normalizer-3.4.9-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2", size = 221655, upload-time = "2026-07-07T14:32:45.64Z" }, + { url = "https://files.pythonhosted.org/packages/1d/85/181c652953eb5276d198f375b1dd641047392050098100a3a02d6534f657/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9", size = 219229, upload-time = "2026-07-07T14:32:47.376Z" }, + { url = "https://files.pythonhosted.org/packages/0c/e7/aaf6da33fc9f4691cda8f7efbc9f69179d3d39ec8a4799baf273ee1d8db0/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15", size = 209704, upload-time = "2026-07-07T14:32:48.855Z" }, + { url = "https://files.pythonhosted.org/packages/63/01/f2fb3bd3a73be48b173ee0c6aa8d2497af97d5663a8c4c4b491de4c62f7a/charset_normalizer-3.4.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d", size = 226243, upload-time = "2026-07-07T14:32:50.239Z" }, + { url = "https://files.pythonhosted.org/packages/c4/02/c57a22739fe05246b0b5783b3bfb6afaac4eebb46f3ececdfb2f048f780e/charset_normalizer-3.4.9-cp310-cp310-win32.whl", hash = "sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381", size = 150935, upload-time = "2026-07-07T14:32:51.676Z" }, + { url = "https://files.pythonhosted.org/packages/37/8d/ca39a7559a4797505530d084fd3a49a2c959efbbbff146302fb7be4e3b35/charset_normalizer-3.4.9-cp310-cp310-win_amd64.whl", hash = "sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee", size = 162314, upload-time = "2026-07-07T14:32:53.193Z" }, + { url = "https://files.pythonhosted.org/packages/01/da/a44bd7a13d426e69e4894557106cd58669097bfad4a8681123b618fbfc5d/charset_normalizer-3.4.9-cp310-cp310-win_arm64.whl", hash = "sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419", size = 153075, upload-time = "2026-07-07T14:32:54.554Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, ] [[package]] @@ -413,81 +464,98 @@ wheels = [ [[package]] name = "cuda-pathfinder" -version = "1.5.5" +version = "1.5.6" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/11/c8/26f2e4aae92f11522a96043892ba39a90eac610d5242523aa863212bc1c7/cuda_pathfinder-1.5.5-py3-none-any.whl", hash = "sha256:0228c023f95d1480f143ef5c8922d27a2ab052087a942e81dc289c9eb8f91689", size = 51671, upload-time = "2026-05-27T01:21:25.413Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8fc9b0cdc5b7f62746e6a01b85b6461e5ae27f871010a5fcf8fa6950766d/cuda_pathfinder-1.5.6-py3-none-any.whl", hash = "sha256:7e4c07c117b78ba1fb35dac4c444d21f3677b1b1ff56175c53a8e3025c5b43c0", size = 52972, upload-time = "2026-06-30T00:58:04.34Z" }, ] [[package]] name = "cuda-toolkit" -version = "13.0.2" +version = "13.0.3.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, ] [package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] [[package]] name = "datafusion" -version = "53.0.0" +version = "54.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "cloudpickle" }, { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/2b/0f96f12b70839c93930c4e17d767fc32b6c77d548c78784128049e944701/datafusion-53.0.0.tar.gz", hash = "sha256:ba9a5ec06b5453fbd8710d6aeeb515a8bcac4b6c140e254409bb53a5f322ef22", size = 224267, upload-time = "2026-04-13T00:45:02.686Z" } +sdist = { url = "https://files.pythonhosted.org/packages/60/90/886f7e9cf827f07ebd60bd293e54e0a028a50dd49bbaef0ee42aae1981ea/datafusion-54.0.0.tar.gz", hash = "sha256:cfe7e8dfc026efc05824f49b53ad6a72caf5c2d6820759b6212a09e245a427ed", size = 276448, upload-time = "2026-06-29T11:19:34.816Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/4c/60e052813d81f1ffe3123ead013dbdd2cf961daa576cb9056cbb80228e6b/datafusion-53.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a0bd1a98d736571321416dc4ed361a9d1225da1ec9f6c5fad818d75f547697a7", size = 35774913, upload-time = "2026-04-13T00:44:46.235Z" }, - { url = "https://files.pythonhosted.org/packages/6e/59/beabe5301df3338d8206446cd624079e43bdad46e20377a6336017fb6ccf/datafusion-53.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:ce186a8d2405afd67e11e2fb75715019f16b00d070b8d0da89d8aa61cc74c8b5", size = 32667118, upload-time = "2026-04-13T00:44:50.269Z" }, - { url = "https://files.pythonhosted.org/packages/ae/94/636ab61ade98395daea6e733e225e9c7beef111c7c5b575ac851513e203c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:288a00a7ef03e2807a4667683f7560efd80d60ed1d41696ac15ca9ded14c8251", size = 35585824, upload-time = "2026-04-13T00:44:53.683Z" }, - { url = "https://files.pythonhosted.org/packages/34/80/b9f4889209af02f8d14bccb0e6f0519c329b072bc4d2595025a1303f144c/datafusion-53.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:8fef0004f0161fcfc556c025a7201f9cc3169aa3adb97a86419ebb34182d9efb", size = 38083690, upload-time = "2026-04-13T00:44:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/4b/1a/ea4831fc6aeefedbcf186c9f6a273d507b1787c03cbb905bded7e1149a6a/datafusion-53.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:4c8410f5f659b926677be6c7d443bbc05d825c078c970b7d8cf977ebcf948314", size = 38120687, upload-time = "2026-04-13T00:45:00.633Z" }, + { url = "https://files.pythonhosted.org/packages/46/58/4c5b981e3d9ade32a906c15a4941eef50c9b862781cdc14bf4dff48d026a/datafusion-54.0.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:946f55e48b8d523d7b4ac106bdf588b4493c2c66f81877d6952aafeaf7c3ec73", size = 39810553, upload-time = "2026-06-29T11:19:02.1Z" }, + { url = "https://files.pythonhosted.org/packages/66/e5/5e4dbd42ce9a2affb3be90d9ab17cebde1a6f28b0d9fb4b83d612d5c8e42/datafusion-54.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:2a3bf43185c7e43e25242e5fb17b6a11b86bf976434c0bc493fdedbd9a080969", size = 37145255, upload-time = "2026-06-29T11:19:05.491Z" }, + { url = "https://files.pythonhosted.org/packages/c6/5e/dbb9e6e3e5006d34f295d7ac73f1302c8f2df140666402a06e6c55028edb/datafusion-54.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9432bf162381e9282cbc74915b8b773895de18be836f7e3f6d0de4d981f24630", size = 38853856, upload-time = "2026-06-29T11:19:08.732Z" }, + { url = "https://files.pythonhosted.org/packages/a8/81/e69008e3479f4d0134875bc4ae39503bedcd55ca2597e71392c963c651b4/datafusion-54.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3bcd4d213fa74710e75e6e182cc468c2bdbc5ffc74a08c8155d414fbbfa1b3f6", size = 41050149, upload-time = "2026-06-29T11:19:12.108Z" }, + { url = "https://files.pythonhosted.org/packages/61/d4/8ba6e3fe3291c9ccc94b5ca3ec3c1fbcbfbe5ece5ffb965e4550844e2c56/datafusion-54.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:b934e097e1bdca7d5768a81ac1bc4a1812cb459269f8b1a5d892a5d930f18376", size = 43444869, upload-time = "2026-06-29T11:19:15.963Z" }, + { url = "https://files.pythonhosted.org/packages/9d/41/5608323226f21a0fa180823c531dbc0ed270e9b694f299b7647505cb6a06/datafusion-54.0.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:c4e79048da82ad89b768bd0be7df39254cd2a0afe2b719d1f129e8a7229af683", size = 39796248, upload-time = "2026-06-29T11:19:19.208Z" }, + { url = "https://files.pythonhosted.org/packages/18/81/392ee323104ab14ca689384723b69e137064a828233c165574f97a74c0e9/datafusion-54.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fe57038003b18e28b90752c1e32b44af74ec4f552a1904aee725e1129a00c447", size = 37153577, upload-time = "2026-06-29T11:19:22.397Z" }, + { url = "https://files.pythonhosted.org/packages/40/c4/ebd5ef5349ecbea7f5f9da76c213581c13e7bbe1b5735c9925b279eeb4eb/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:574f642832a106456cfc4f32aa82484c504fc32f4be2b510202bcb579de8e6d1", size = 38849839, upload-time = "2026-06-29T11:19:25.783Z" }, + { url = "https://files.pythonhosted.org/packages/5a/b9/2383d30d317bb913cab97dbf2e6e1d5f37f594860d5c5bc176e025cf7d4a/datafusion-54.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:796fd5683927443c5bc61999d00b9007ef9b5ce107725ea8d241df718860985d", size = 41074623, upload-time = "2026-06-29T11:19:29.119Z" }, + { url = "https://files.pythonhosted.org/packages/35/5c/553fd1107dede0a56727fda7216a7198d41394f2d19697f4fb104cc695ea/datafusion-54.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:64973c63874ec31670dd97b32b18af7b07fad679cb20d58ed154038e3a5c204e", size = 43438801, upload-time = "2026-06-29T11:19:32.799Z" }, ] [[package]] name = "datasets" -version = "4.1.1" +version = "4.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, { name = "filelock" }, { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, { name = "huggingface-hub" }, { name = "multiprocess" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "packaging" }, { name = "pandas" }, { name = "pyarrow" }, @@ -496,9 +564,21 @@ dependencies = [ { name = "tqdm" }, { name = "xxhash" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/91/a4/73f8e6ef52c535e1d20d5b2ca83bfe6de399d8b8b8a61ccc8d63d60735aa/datasets-4.1.1.tar.gz", hash = "sha256:7d8d5ba8b12861d2c44bfff9c83484ebfafff1ff553371e5901a8d3aab5450e2", size = 579324, upload-time = "2025-09-18T13:14:27.108Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/13/f05a80bbbac5f62e492e5e463ec59a4479647ef9c376b1fdfaa4d3ed01cc/datasets-4.4.0.tar.gz", hash = "sha256:0430d39b9f13b53c37afb80c23c7e5d8c6ceccc014c14a14d15fa2b4e8688d2a", size = 585143, upload-time = "2025-11-04T10:36:22.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/31/d552336985f747b19f0a852d98ca7a2ef4727ba956b38041cfbda08dde0a/datasets-4.4.0-py3-none-any.whl", hash = "sha256:b7e6d1d48c2e1d3a95d6b378e8fc3d7ab29f24f14ddf505a8d417dd09c692f19", size = 511463, upload-time = "2025-11-04T10:36:20.062Z" }, +] + +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/c8/09012ac195a0aab58755800d2efdc0e7d5905053509f12cb5d136c911cda/datasets-4.1.1-py3-none-any.whl", hash = "sha256:62e4f6899a36be9ec74a7e759a6951253cc85b3fcfa0a759b0efa8353b149dac", size = 503623, upload-time = "2025-09-18T13:14:25.111Z" }, + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] [[package]] @@ -512,167 +592,204 @@ wheels = [ [[package]] name = "duckdb" -version = "1.4.0" +version = "1.5.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/93/adc0d183642fc9a602ca9b97cb16754c84b8c1d92e5b99aec412e0c419a8/duckdb-1.4.0.tar.gz", hash = "sha256:bd5edee8bd5a73b5822f2b390668597b5fcdc2d3292c244d8d933bb87ad6ac4c", size = 18453175, upload-time = "2025-09-16T10:22:41.509Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/4a/b2e17dbe2953481b084f355f162ed319a67ef760e28794c6870058583aec/duckdb-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e24e981a6c87e299201694b9bb24fff0beb04ccad399fca6f13072a59814488f", size = 31293005, upload-time = "2025-09-16T10:21:28.296Z" }, - { url = "https://files.pythonhosted.org/packages/a9/89/e34ed03cce7e35b83c1f056126aa4e8e8097eb93e7324463020f85d5cbfa/duckdb-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db500ef2c8cb7dc1ca078740ecf1dceaa20d3f5dc5bce269be45d5cff4170c0f", size = 17288207, upload-time = "2025-09-16T10:21:31.129Z" }, - { url = "https://files.pythonhosted.org/packages/f8/17/7ff24799ee98c4dbb177c3ec6c93e38e9513828785c31757c727b47ad71e/duckdb-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a65739b8a7106634e6e77d0e110fc5e057b88edc9df6cb1683d499a1e5aa3177", size = 14817523, upload-time = "2025-09-16T10:21:33.397Z" }, - { url = "https://files.pythonhosted.org/packages/fc/ab/7a482a76ff75212b5cf4f2172a802f2a59b4ab096416e5821aa62a305bc4/duckdb-1.4.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d59f7be24862adb803a1ddfc9c3b8cb09e6005bca0c9c6f7c631a1da1c3aa0c", size = 18410654, upload-time = "2025-09-16T10:21:35.864Z" }, - { url = "https://files.pythonhosted.org/packages/1e/f6/a235233b973652b31448b6d600604620d02fc552b90ab94ca7f645fd5ac0/duckdb-1.4.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d052a87e9edf4eb3bab0b7a6ac995676018c6083b8049421628dfa3b983a2d4", size = 20399121, upload-time = "2025-09-16T10:21:38.524Z" }, - { url = "https://files.pythonhosted.org/packages/b1/cf/63fedb74d00d7c4e19ffc73a1d8d98ee8d3d6498cf2865509c104aa8e799/duckdb-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:0329b81e587f745b2fc6f3a488ea3188b0f029c3b5feef43792a25eaac84ac01", size = 12283288, upload-time = "2025-09-16T10:21:40.732Z" }, - { url = "https://files.pythonhosted.org/packages/60/e9/b29cc5bceac52e049b20d613551a2171a092df07f26d4315f3f9651c80d4/duckdb-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6505fed1ccae8df9f574e744c48fa32ee2feaeebe5346c2daf4d4d10a8dac5aa", size = 31290878, upload-time = "2025-09-16T10:21:43.256Z" }, - { url = "https://files.pythonhosted.org/packages/1f/68/d88a15dba48bf6a4b33f1be5097ef45c83f7b9e97c854cc638a85bb07d70/duckdb-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:36974a04b29c74ac2143457e95420a7422016d050e28573060b89a90b9cf2b57", size = 17288823, upload-time = "2025-09-16T10:21:45.716Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7e/e3d2101dc6bbd60f2b3c1d748351ff541fc8c48790ac1218c0199cb930f6/duckdb-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90484b896e5059f145d1facfabea38e22c54a2dcc2bd62dd6c290423f0aee258", size = 14819684, upload-time = "2025-09-16T10:21:48.117Z" }, - { url = "https://files.pythonhosted.org/packages/c4/bb/4ec8e4d03cb5b77d75b9ee0057c2c714cffaa9bda1e55ffec833458af0a3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a969d624b385853b31a43b0a23089683297da2f14846243921c6dbec8382d659", size = 18410075, upload-time = "2025-09-16T10:21:50.517Z" }, - { url = "https://files.pythonhosted.org/packages/ec/21/e896616d892d50dc1e0c142428e9359b483d4dd6e339231d822e57834ad3/duckdb-1.4.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5935644f96a75e9f6f3c3eeb3da14cdcaf7bad14d1199c08439103decb29466a", size = 20402984, upload-time = "2025-09-16T10:21:52.808Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c0/b5eb9497e4a9167d23fbad745969eaa36e28d346648e17565471892d1b33/duckdb-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:300aa0e963af97969c38440877fffd576fc1f49c1f5914789a9d01f2fe7def91", size = 12282971, upload-time = "2025-09-16T10:21:55.314Z" }, - { url = "https://files.pythonhosted.org/packages/e8/6d/0c774d6af1aed82dbe855d266cb000a1c09ea31ed7d6c3a79e2167a38e7a/duckdb-1.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:18b3a048fca6cc7bafe08b10e1b0ab1509d7a0381ffb2c70359e7dc56d8a705d", size = 31307425, upload-time = "2025-09-16T10:21:57.83Z" }, - { url = "https://files.pythonhosted.org/packages/d3/c0/1fd7b7b2c0c53d8d748d2f28ea9096df5ee9dc39fa736cca68acabe69656/duckdb-1.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c1271cb85aeacccfd0b1284e816280a7450df1dd4dd85ccb2848563cfdf90e9", size = 17295727, upload-time = "2025-09-16T10:22:02.242Z" }, - { url = "https://files.pythonhosted.org/packages/98/d3/4d4c4bd667b7ada5f6c207c2f127591ebb8468333f207f8f10ff0532578e/duckdb-1.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55064dd2e25711eeaa6a72c25405bdd7994c81a3221657e94309a2faf65d25a6", size = 14826879, upload-time = "2025-09-16T10:22:05.162Z" }, - { url = "https://files.pythonhosted.org/packages/b0/48/e0c1b97d76fb7567c53db5739931323238fad54a642707008104f501db37/duckdb-1.4.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536d7c81bc506532daccf373ddbc8c6add46aeb70ef3cd5ee70ad5c2b3165ea", size = 18417856, upload-time = "2025-09-16T10:22:07.919Z" }, - { url = "https://files.pythonhosted.org/packages/12/78/297b838f3b9511589badc8f472f70b31cf3bbf9eb99fa0a4d6e911d3114a/duckdb-1.4.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:784554e3ddfcfc5c5c7b1aa1f9925fedb7938f6628729adba48f7ea37554598f", size = 20427154, upload-time = "2025-09-16T10:22:10.216Z" }, - { url = "https://files.pythonhosted.org/packages/ea/57/500d251b886494f6c52d56eeab8a1860572ee62aed05d7d50c71ba2320f3/duckdb-1.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:c5d2aa4d6981f525ada95e6db41bb929403632bb5ff24bd6d6dd551662b1b613", size = 12290108, upload-time = "2025-09-16T10:22:12.668Z" }, - { url = "https://files.pythonhosted.org/packages/2f/64/ee22b2b8572746e1523143b9f28d606575782e0204de5020656a1d15dd14/duckdb-1.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1d94d010a09b1a62d9021a2a71cf266188750f3c9b1912ccd6afe104a6ce8010", size = 31307662, upload-time = "2025-09-16T10:22:14.9Z" }, - { url = "https://files.pythonhosted.org/packages/76/2e/4241cd00046ca6b781bd1d9002e8223af061e85d1cc21830aa63e7a7db7c/duckdb-1.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c61756fa8b3374627e5fa964b8e0d5b58e364dce59b87dba7fb7bc6ede196b26", size = 17295617, upload-time = "2025-09-16T10:22:17.239Z" }, - { url = "https://files.pythonhosted.org/packages/f7/98/5ab136bc7b12ac18580350a220db7c00606be9eac2d89de259cce733f64c/duckdb-1.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e70d7d9881ea2c0836695de70ea68c970e18a2856ba3d6502e276c85bd414ae7", size = 14826727, upload-time = "2025-09-16T10:22:19.415Z" }, - { url = "https://files.pythonhosted.org/packages/23/32/57866cf8881288b3dfb9212720221fb890daaa534dbdc6fe3fff3979ecd1/duckdb-1.4.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2de258a93435c977a0ec3a74ec8f60c2f215ddc73d427ee49adc4119558facd3", size = 18421289, upload-time = "2025-09-16T10:22:21.564Z" }, - { url = "https://files.pythonhosted.org/packages/a0/83/7438fb43be451a7d4a04650aaaf662b2ff2d95895bbffe3e0e28cbe030c9/duckdb-1.4.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a6d3659641d517dd9ed1ab66f110cdbdaa6900106f116effaf2dbedd83c38de3", size = 20426547, upload-time = "2025-09-16T10:22:23.759Z" }, - { url = "https://files.pythonhosted.org/packages/21/b2/98fb89ae81611855f35984e96f648d871f3967bb3f524b51d1372d052f0c/duckdb-1.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:07fcc612ea5f0fe6032b92bcc93693034eb00e7a23eb9146576911d5326af4f7", size = 12290467, upload-time = "2025-09-16T10:22:25.923Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/d4/298acf9331a80b3ce6ac64dd940e7e13f4058fb69d18914445f02e3c7bfe/duckdb-1.5.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3b805507f88171b428b21c966c30e9a3d54e30b24528918a44ed0032542bc26f", size = 32702934, upload-time = "2026-07-22T10:53:19.069Z" }, + { url = "https://files.pythonhosted.org/packages/d5/90/c489fb63d64b2e7ee109ce8460bdede003a0f256e5b41a03a2a1c4764058/duckdb-1.5.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b08e19cc856220d8a26fa62abc2264b349aff67255e9373c6a3f607addd56dc6", size = 17343604, upload-time = "2026-07-22T10:53:22.767Z" }, + { url = "https://files.pythonhosted.org/packages/0b/27/effa80a15b1f0c61c235622f797868485359e8c9ad6a8e358e7a0c479151/duckdb-1.5.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a17e6a922e42a5c06ed2353fe78c5dff2610f6632d603836f9606ad0bf754079", size = 15488179, upload-time = "2026-07-22T10:53:25.945Z" }, + { url = "https://files.pythonhosted.org/packages/5d/07/21212345c8d24ba62dceaa20be3b21f5c46f1510b1b42ce93bb058afe0c4/duckdb-1.5.5-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1bdc38922c365c37720149f90d90b1e9823eb82dad6830855b5f87537fa6fc0c", size = 19367323, upload-time = "2026-07-22T10:53:30.23Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d0/10371ae875fb4b5ef61bb892743b4b2e90c512b371fdf29317deb744857d/duckdb-1.5.5-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e238060db5ca59879882a6e9b015e2c65d5c64ddf281ba1d7a9a2033764152cf", size = 21476568, upload-time = "2026-07-22T10:53:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/09568ce617dd7bc0757b3d7b6a981660b9e4f0b7594de8ed776755eae740/duckdb-1.5.5-cp310-cp310-win_amd64.whl", hash = "sha256:4acc72798ba1885a9c17d1242903d2cd502f13b1271c7677f7cab25d8578eceb", size = 13156129, upload-time = "2026-07-22T10:53:37.55Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c2/b62ec24d57bb8df4e24b0b58f7f8facb32f5fdb9f1895aed9e9fcdded168/duckdb-1.5.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b543841b0ae18a9c982345cfa3987e9c065d3a4b0f067daa473d92d1e65f528", size = 32708371, upload-time = "2026-07-22T10:53:41.642Z" }, + { url = "https://files.pythonhosted.org/packages/8a/ce/769171ba45f0b73632dc3bc3108d891e81dd6c6bbfba630a34a75b4dcc0f/duckdb-1.5.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a925d06c2a4c3b64553d6cc1aced5028d376d4479bed689a7d47e9b1dccd80a", size = 17343979, upload-time = "2026-07-22T10:53:44.951Z" }, + { url = "https://files.pythonhosted.org/packages/46/59/a8e3384ee916e00d5dcf985194c1511d61978540778a1e96fa47f9fb3e0d/duckdb-1.5.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0c42757cb34722144bd4dfb94b6f336339e7b2468f6813fa7fa9a319ba07bab4", size = 15493704, upload-time = "2026-07-22T10:53:47.912Z" }, + { url = "https://files.pythonhosted.org/packages/6f/1d/9840179c2607b90523a2884a129c4d4e6dbdc1178ba62a976c1043beba88/duckdb-1.5.5-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e72f9e1a4f90a5c8483ad4d540e495bf0834ba61c360b52499a573d7ed62a3f", size = 19366574, upload-time = "2026-07-22T10:53:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/b5/55/f9641a4eebcc2f4df631287d6c3b9ed2eea3b92644f93acbad825e3972b6/duckdb-1.5.5-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9b6f86ed85d4ef5e0211eaebf75d057bd8bb520bba438a95dd0f4e42234bbfe", size = 21477952, upload-time = "2026-07-22T10:53:55.575Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3a/07c3556e37a5c97b95917b029c8fdde4a25fbd76a660bacdac195cf20dcb/duckdb-1.5.5-cp311-cp311-win_amd64.whl", hash = "sha256:9f4287f97ccf0c1f3d471e7115be2b067cbf99627e2d34bffd462dd64703cddc", size = 13156986, upload-time = "2026-07-22T10:53:58.823Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ff/07b48eef2078ca033847e9caa46cc7633b714c5f91ad1ce091c8ca89d792/duckdb-1.5.5-cp311-cp311-win_arm64.whl", hash = "sha256:179633a3fc6296c75d57c69c1e239fa9e5cdcb670fd1dbff88a02663f932905c", size = 14001317, upload-time = "2026-07-22T10:54:01.724Z" }, + { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, + { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, + { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, + { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, + { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, + { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, + { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, + { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, + { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, + { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, + { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, ] [[package]] name = "exceptiongroup" -version = "1.3.0" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] [[package]] -name = "filelock" -version = "3.19.1" +name = "execnet" +version = "2.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, ] [[package]] -name = "flatbuffers" -version = "25.9.23" +name = "filelock" +version = "3.29.7" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9d/1f/3ee70b0a55137442038f2a33469cc5fddd7e0ad2abf83d7497c18a2b6923/flatbuffers-25.9.23.tar.gz", hash = "sha256:676f9fa62750bb50cf531b42a0a2a118ad8f7f797a511eda12881c016f093b12", size = 22067, upload-time = "2025-09-24T05:25:30.106Z" } +sdist = { url = "https://files.pythonhosted.org/packages/35/94/00f2059e4835eace3ae8fde680b932c496f8ec7bdc99168dfa53fb2e6b79/filelock-3.29.7.tar.gz", hash = "sha256:5b481979797ae69e72f0b389d89a80bdd585c260c5b3f1fb9c0a5ba9bb3f195d", size = 71521, upload-time = "2026-07-08T05:46:58.716Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/1b/00a78aa2e8fbd63f9af08c9c19e6deb3d5d66b4dda677a0f61654680ee89/flatbuffers-25.9.23-py2.py3-none-any.whl", hash = "sha256:255538574d6cb6d0a79a17ec8bc0d30985913b87513a01cce8bcdb6b4c44d0e2", size = 30869, upload-time = "2025-09-24T05:25:28.912Z" }, + { url = "https://files.pythonhosted.org/packages/60/02/be4a57b60c7149b55b9e3b3c13f609cd8eb5307c751f22bd8fb8d262e75b/filelock-3.29.7-py3-none-any.whl", hash = "sha256:987db6f789a3a2a59f55081801b2b3697cb97e2a736b5f1a9e99b559285fbc51", size = 46036, upload-time = "2026-07-08T05:46:57.53Z" }, ] [[package]] name = "frozenlist" -version = "1.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/79/b1/b64018016eeb087db503b038296fd782586432b9c077fc5c7839e9cb6ef6/frozenlist-1.7.0.tar.gz", hash = "sha256:2e310d81923c2437ea8670467121cc3e9b0f76d3043cc1d2331d56c7fb7a3a8f", size = 45078, upload-time = "2025-06-09T23:02:35.538Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/36/0da0a49409f6b47cc2d060dc8c9040b897b5902a8a4e37d9bc1deb11f680/frozenlist-1.7.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc4df77d638aa2ed703b878dd093725b72a824c3c546c076e8fdf276f78ee84a", size = 81304, upload-time = "2025-06-09T22:59:46.226Z" }, - { url = "https://files.pythonhosted.org/packages/77/f0/77c11d13d39513b298e267b22eb6cb559c103d56f155aa9a49097221f0b6/frozenlist-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:716a9973a2cc963160394f701964fe25012600f3d311f60c790400b00e568b61", size = 47735, upload-time = "2025-06-09T22:59:48.133Z" }, - { url = "https://files.pythonhosted.org/packages/37/12/9d07fa18971a44150593de56b2f2947c46604819976784bcf6ea0d5db43b/frozenlist-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a0fd1bad056a3600047fb9462cff4c5322cebc59ebf5d0a3725e0ee78955001d", size = 46775, upload-time = "2025-06-09T22:59:49.564Z" }, - { url = "https://files.pythonhosted.org/packages/70/34/f73539227e06288fcd1f8a76853e755b2b48bca6747e99e283111c18bcd4/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3789ebc19cb811163e70fe2bd354cea097254ce6e707ae42e56f45e31e96cb8e", size = 224644, upload-time = "2025-06-09T22:59:51.35Z" }, - { url = "https://files.pythonhosted.org/packages/fb/68/c1d9c2f4a6e438e14613bad0f2973567586610cc22dcb1e1241da71de9d3/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:af369aa35ee34f132fcfad5be45fbfcde0e3a5f6a1ec0712857f286b7d20cca9", size = 222125, upload-time = "2025-06-09T22:59:52.884Z" }, - { url = "https://files.pythonhosted.org/packages/b9/d0/98e8f9a515228d708344d7c6986752be3e3192d1795f748c24bcf154ad99/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac64b6478722eeb7a3313d494f8342ef3478dff539d17002f849101b212ef97c", size = 233455, upload-time = "2025-06-09T22:59:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/79/df/8a11bcec5600557f40338407d3e5bea80376ed1c01a6c0910fcfdc4b8993/frozenlist-1.7.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f89f65d85774f1797239693cef07ad4c97fdd0639544bad9ac4b869782eb1981", size = 227339, upload-time = "2025-06-09T22:59:56.187Z" }, - { url = "https://files.pythonhosted.org/packages/50/82/41cb97d9c9a5ff94438c63cc343eb7980dac4187eb625a51bdfdb7707314/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1073557c941395fdfcfac13eb2456cb8aad89f9de27bae29fabca8e563b12615", size = 212969, upload-time = "2025-06-09T22:59:57.604Z" }, - { url = "https://files.pythonhosted.org/packages/13/47/f9179ee5ee4f55629e4f28c660b3fdf2775c8bfde8f9c53f2de2d93f52a9/frozenlist-1.7.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ed8d2fa095aae4bdc7fdd80351009a48d286635edffee66bf865e37a9125c50", size = 222862, upload-time = "2025-06-09T22:59:59.498Z" }, - { url = "https://files.pythonhosted.org/packages/1a/52/df81e41ec6b953902c8b7e3a83bee48b195cb0e5ec2eabae5d8330c78038/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:24c34bea555fe42d9f928ba0a740c553088500377448febecaa82cc3e88aa1fa", size = 222492, upload-time = "2025-06-09T23:00:01.026Z" }, - { url = "https://files.pythonhosted.org/packages/84/17/30d6ea87fa95a9408245a948604b82c1a4b8b3e153cea596421a2aef2754/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:69cac419ac6a6baad202c85aaf467b65ac860ac2e7f2ac1686dc40dbb52f6577", size = 238250, upload-time = "2025-06-09T23:00:03.401Z" }, - { url = "https://files.pythonhosted.org/packages/8f/00/ecbeb51669e3c3df76cf2ddd66ae3e48345ec213a55e3887d216eb4fbab3/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:960d67d0611f4c87da7e2ae2eacf7ea81a5be967861e0c63cf205215afbfac59", size = 218720, upload-time = "2025-06-09T23:00:05.282Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c0/c224ce0e0eb31cc57f67742071bb470ba8246623c1823a7530be0e76164c/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:41be2964bd4b15bf575e5daee5a5ce7ed3115320fb3c2b71fca05582ffa4dc9e", size = 232585, upload-time = "2025-06-09T23:00:07.962Z" }, - { url = "https://files.pythonhosted.org/packages/55/3c/34cb694abf532f31f365106deebdeac9e45c19304d83cf7d51ebbb4ca4d1/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:46d84d49e00c9429238a7ce02dc0be8f6d7cd0cd405abd1bebdc991bf27c15bd", size = 234248, upload-time = "2025-06-09T23:00:09.428Z" }, - { url = "https://files.pythonhosted.org/packages/98/c0/2052d8b6cecda2e70bd81299e3512fa332abb6dcd2969b9c80dfcdddbf75/frozenlist-1.7.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:15900082e886edb37480335d9d518cec978afc69ccbc30bd18610b7c1b22a718", size = 221621, upload-time = "2025-06-09T23:00:11.32Z" }, - { url = "https://files.pythonhosted.org/packages/c5/bf/7dcebae315436903b1d98ffb791a09d674c88480c158aa171958a3ac07f0/frozenlist-1.7.0-cp310-cp310-win32.whl", hash = "sha256:400ddd24ab4e55014bba442d917203c73b2846391dd42ca5e38ff52bb18c3c5e", size = 39578, upload-time = "2025-06-09T23:00:13.526Z" }, - { url = "https://files.pythonhosted.org/packages/8f/5f/f69818f017fa9a3d24d1ae39763e29b7f60a59e46d5f91b9c6b21622f4cd/frozenlist-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:6eb93efb8101ef39d32d50bce242c84bcbddb4f7e9febfa7b524532a239b4464", size = 43830, upload-time = "2025-06-09T23:00:14.98Z" }, - { url = "https://files.pythonhosted.org/packages/34/7e/803dde33760128acd393a27eb002f2020ddb8d99d30a44bfbaab31c5f08a/frozenlist-1.7.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:aa51e147a66b2d74de1e6e2cf5921890de6b0f4820b257465101d7f37b49fb5a", size = 82251, upload-time = "2025-06-09T23:00:16.279Z" }, - { url = "https://files.pythonhosted.org/packages/75/a9/9c2c5760b6ba45eae11334db454c189d43d34a4c0b489feb2175e5e64277/frozenlist-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9b35db7ce1cd71d36ba24f80f0c9e7cff73a28d7a74e91fe83e23d27c7828750", size = 48183, upload-time = "2025-06-09T23:00:17.698Z" }, - { url = "https://files.pythonhosted.org/packages/47/be/4038e2d869f8a2da165f35a6befb9158c259819be22eeaf9c9a8f6a87771/frozenlist-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:34a69a85e34ff37791e94542065c8416c1afbf820b68f720452f636d5fb990cd", size = 47107, upload-time = "2025-06-09T23:00:18.952Z" }, - { url = "https://files.pythonhosted.org/packages/79/26/85314b8a83187c76a37183ceed886381a5f992975786f883472fcb6dc5f2/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a646531fa8d82c87fe4bb2e596f23173caec9185bfbca5d583b4ccfb95183e2", size = 237333, upload-time = "2025-06-09T23:00:20.275Z" }, - { url = "https://files.pythonhosted.org/packages/1f/fd/e5b64f7d2c92a41639ffb2ad44a6a82f347787abc0c7df5f49057cf11770/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:79b2ffbba483f4ed36a0f236ccb85fbb16e670c9238313709638167670ba235f", size = 231724, upload-time = "2025-06-09T23:00:21.705Z" }, - { url = "https://files.pythonhosted.org/packages/20/fb/03395c0a43a5976af4bf7534759d214405fbbb4c114683f434dfdd3128ef/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a26f205c9ca5829cbf82bb2a84b5c36f7184c4316617d7ef1b271a56720d6b30", size = 245842, upload-time = "2025-06-09T23:00:23.148Z" }, - { url = "https://files.pythonhosted.org/packages/d0/15/c01c8e1dffdac5d9803507d824f27aed2ba76b6ed0026fab4d9866e82f1f/frozenlist-1.7.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bcacfad3185a623fa11ea0e0634aac7b691aa925d50a440f39b458e41c561d98", size = 239767, upload-time = "2025-06-09T23:00:25.103Z" }, - { url = "https://files.pythonhosted.org/packages/14/99/3f4c6fe882c1f5514b6848aa0a69b20cb5e5d8e8f51a339d48c0e9305ed0/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c1b0fe8fe451b34f12dce46445ddf14bd2a5bcad7e324987194dc8e3a74c86", size = 224130, upload-time = "2025-06-09T23:00:27.061Z" }, - { url = "https://files.pythonhosted.org/packages/4d/83/220a374bd7b2aeba9d0725130665afe11de347d95c3620b9b82cc2fcab97/frozenlist-1.7.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61d1a5baeaac6c0798ff6edfaeaa00e0e412d49946c53fae8d4b8e8b3566c4ae", size = 235301, upload-time = "2025-06-09T23:00:29.02Z" }, - { url = "https://files.pythonhosted.org/packages/03/3c/3e3390d75334a063181625343e8daab61b77e1b8214802cc4e8a1bb678fc/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7edf5c043c062462f09b6820de9854bf28cc6cc5b6714b383149745e287181a8", size = 234606, upload-time = "2025-06-09T23:00:30.514Z" }, - { url = "https://files.pythonhosted.org/packages/23/1e/58232c19608b7a549d72d9903005e2d82488f12554a32de2d5fb59b9b1ba/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d50ac7627b3a1bd2dcef6f9da89a772694ec04d9a61b66cf87f7d9446b4a0c31", size = 248372, upload-time = "2025-06-09T23:00:31.966Z" }, - { url = "https://files.pythonhosted.org/packages/c0/a4/e4a567e01702a88a74ce8a324691e62a629bf47d4f8607f24bf1c7216e7f/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ce48b2fece5aeb45265bb7a58259f45027db0abff478e3077e12b05b17fb9da7", size = 229860, upload-time = "2025-06-09T23:00:33.375Z" }, - { url = "https://files.pythonhosted.org/packages/73/a6/63b3374f7d22268b41a9db73d68a8233afa30ed164c46107b33c4d18ecdd/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:fe2365ae915a1fafd982c146754e1de6ab3478def8a59c86e1f7242d794f97d5", size = 245893, upload-time = "2025-06-09T23:00:35.002Z" }, - { url = "https://files.pythonhosted.org/packages/6d/eb/d18b3f6e64799a79673c4ba0b45e4cfbe49c240edfd03a68be20002eaeaa/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:45a6f2fdbd10e074e8814eb98b05292f27bad7d1883afbe009d96abdcf3bc898", size = 246323, upload-time = "2025-06-09T23:00:36.468Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f5/720f3812e3d06cd89a1d5db9ff6450088b8f5c449dae8ffb2971a44da506/frozenlist-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:21884e23cffabb157a9dd7e353779077bf5b8f9a58e9b262c6caad2ef5f80a56", size = 233149, upload-time = "2025-06-09T23:00:37.963Z" }, - { url = "https://files.pythonhosted.org/packages/69/68/03efbf545e217d5db8446acfd4c447c15b7c8cf4dbd4a58403111df9322d/frozenlist-1.7.0-cp311-cp311-win32.whl", hash = "sha256:284d233a8953d7b24f9159b8a3496fc1ddc00f4db99c324bd5fb5f22d8698ea7", size = 39565, upload-time = "2025-06-09T23:00:39.753Z" }, - { url = "https://files.pythonhosted.org/packages/58/17/fe61124c5c333ae87f09bb67186d65038834a47d974fc10a5fadb4cc5ae1/frozenlist-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:387cbfdcde2f2353f19c2f66bbb52406d06ed77519ac7ee21be0232147c2592d", size = 44019, upload-time = "2025-06-09T23:00:40.988Z" }, - { url = "https://files.pythonhosted.org/packages/ef/a2/c8131383f1e66adad5f6ecfcce383d584ca94055a34d683bbb24ac5f2f1c/frozenlist-1.7.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3dbf9952c4bb0e90e98aec1bd992b3318685005702656bc6f67c1a32b76787f2", size = 81424, upload-time = "2025-06-09T23:00:42.24Z" }, - { url = "https://files.pythonhosted.org/packages/4c/9d/02754159955088cb52567337d1113f945b9e444c4960771ea90eb73de8db/frozenlist-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1f5906d3359300b8a9bb194239491122e6cf1444c2efb88865426f170c262cdb", size = 47952, upload-time = "2025-06-09T23:00:43.481Z" }, - { url = "https://files.pythonhosted.org/packages/01/7a/0046ef1bd6699b40acd2067ed6d6670b4db2f425c56980fa21c982c2a9db/frozenlist-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3dabd5a8f84573c8d10d8859a50ea2dec01eea372031929871368c09fa103478", size = 46688, upload-time = "2025-06-09T23:00:44.793Z" }, - { url = "https://files.pythonhosted.org/packages/d6/a2/a910bafe29c86997363fb4c02069df4ff0b5bc39d33c5198b4e9dd42d8f8/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa57daa5917f1738064f302bf2626281a1cb01920c32f711fbc7bc36111058a8", size = 243084, upload-time = "2025-06-09T23:00:46.125Z" }, - { url = "https://files.pythonhosted.org/packages/64/3e/5036af9d5031374c64c387469bfcc3af537fc0f5b1187d83a1cf6fab1639/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c193dda2b6d49f4c4398962810fa7d7c78f032bf45572b3e04dd5249dff27e08", size = 233524, upload-time = "2025-06-09T23:00:47.73Z" }, - { url = "https://files.pythonhosted.org/packages/06/39/6a17b7c107a2887e781a48ecf20ad20f1c39d94b2a548c83615b5b879f28/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe2b675cf0aaa6d61bf8fbffd3c274b3c9b7b1623beb3809df8a81399a4a9c4", size = 248493, upload-time = "2025-06-09T23:00:49.742Z" }, - { url = "https://files.pythonhosted.org/packages/be/00/711d1337c7327d88c44d91dd0f556a1c47fb99afc060ae0ef66b4d24793d/frozenlist-1.7.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8fc5d5cda37f62b262405cf9652cf0856839c4be8ee41be0afe8858f17f4c94b", size = 244116, upload-time = "2025-06-09T23:00:51.352Z" }, - { url = "https://files.pythonhosted.org/packages/24/fe/74e6ec0639c115df13d5850e75722750adabdc7de24e37e05a40527ca539/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0d5ce521d1dd7d620198829b87ea002956e4319002ef0bc8d3e6d045cb4646e", size = 224557, upload-time = "2025-06-09T23:00:52.855Z" }, - { url = "https://files.pythonhosted.org/packages/8d/db/48421f62a6f77c553575201e89048e97198046b793f4a089c79a6e3268bd/frozenlist-1.7.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:488d0a7d6a0008ca0db273c542098a0fa9e7dfaa7e57f70acef43f32b3f69dca", size = 241820, upload-time = "2025-06-09T23:00:54.43Z" }, - { url = "https://files.pythonhosted.org/packages/1d/fa/cb4a76bea23047c8462976ea7b7a2bf53997a0ca171302deae9d6dd12096/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:15a7eaba63983d22c54d255b854e8108e7e5f3e89f647fc854bd77a237e767df", size = 236542, upload-time = "2025-06-09T23:00:56.409Z" }, - { url = "https://files.pythonhosted.org/packages/5d/32/476a4b5cfaa0ec94d3f808f193301debff2ea42288a099afe60757ef6282/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1eaa7e9c6d15df825bf255649e05bd8a74b04a4d2baa1ae46d9c2d00b2ca2cb5", size = 249350, upload-time = "2025-06-09T23:00:58.468Z" }, - { url = "https://files.pythonhosted.org/packages/8d/ba/9a28042f84a6bf8ea5dbc81cfff8eaef18d78b2a1ad9d51c7bc5b029ad16/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4389e06714cfa9d47ab87f784a7c5be91d3934cd6e9a7b85beef808297cc025", size = 225093, upload-time = "2025-06-09T23:01:00.015Z" }, - { url = "https://files.pythonhosted.org/packages/bc/29/3a32959e68f9cf000b04e79ba574527c17e8842e38c91d68214a37455786/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:73bd45e1488c40b63fe5a7df892baf9e2a4d4bb6409a2b3b78ac1c6236178e01", size = 245482, upload-time = "2025-06-09T23:01:01.474Z" }, - { url = "https://files.pythonhosted.org/packages/80/e8/edf2f9e00da553f07f5fa165325cfc302dead715cab6ac8336a5f3d0adc2/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99886d98e1643269760e5fe0df31e5ae7050788dd288947f7f007209b8c33f08", size = 249590, upload-time = "2025-06-09T23:01:02.961Z" }, - { url = "https://files.pythonhosted.org/packages/1c/80/9a0eb48b944050f94cc51ee1c413eb14a39543cc4f760ed12657a5a3c45a/frozenlist-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:290a172aae5a4c278c6da8a96222e6337744cd9c77313efe33d5670b9f65fc43", size = 237785, upload-time = "2025-06-09T23:01:05.095Z" }, - { url = "https://files.pythonhosted.org/packages/f3/74/87601e0fb0369b7a2baf404ea921769c53b7ae00dee7dcfe5162c8c6dbf0/frozenlist-1.7.0-cp312-cp312-win32.whl", hash = "sha256:426c7bc70e07cfebc178bc4c2bf2d861d720c4fff172181eeb4a4c41d4ca2ad3", size = 39487, upload-time = "2025-06-09T23:01:06.54Z" }, - { url = "https://files.pythonhosted.org/packages/0b/15/c026e9a9fc17585a9d461f65d8593d281fedf55fbf7eb53f16c6df2392f9/frozenlist-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:563b72efe5da92e02eb68c59cb37205457c977aa7a449ed1b37e6939e5c47c6a", size = 43874, upload-time = "2025-06-09T23:01:07.752Z" }, - { url = "https://files.pythonhosted.org/packages/24/90/6b2cebdabdbd50367273c20ff6b57a3dfa89bd0762de02c3a1eb42cb6462/frozenlist-1.7.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee80eeda5e2a4e660651370ebffd1286542b67e268aa1ac8d6dbe973120ef7ee", size = 79791, upload-time = "2025-06-09T23:01:09.368Z" }, - { url = "https://files.pythonhosted.org/packages/83/2e/5b70b6a3325363293fe5fc3ae74cdcbc3e996c2a11dde2fd9f1fb0776d19/frozenlist-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d1a81c85417b914139e3a9b995d4a1c84559afc839a93cf2cb7f15e6e5f6ed2d", size = 47165, upload-time = "2025-06-09T23:01:10.653Z" }, - { url = "https://files.pythonhosted.org/packages/f4/25/a0895c99270ca6966110f4ad98e87e5662eab416a17e7fd53c364bf8b954/frozenlist-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cbb65198a9132ebc334f237d7b0df163e4de83fb4f2bdfe46c1e654bdb0c5d43", size = 45881, upload-time = "2025-06-09T23:01:12.296Z" }, - { url = "https://files.pythonhosted.org/packages/19/7c/71bb0bbe0832793c601fff68cd0cf6143753d0c667f9aec93d3c323f4b55/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dab46c723eeb2c255a64f9dc05b8dd601fde66d6b19cdb82b2e09cc6ff8d8b5d", size = 232409, upload-time = "2025-06-09T23:01:13.641Z" }, - { url = "https://files.pythonhosted.org/packages/c0/45/ed2798718910fe6eb3ba574082aaceff4528e6323f9a8570be0f7028d8e9/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6aeac207a759d0dedd2e40745575ae32ab30926ff4fa49b1635def65806fddee", size = 225132, upload-time = "2025-06-09T23:01:15.264Z" }, - { url = "https://files.pythonhosted.org/packages/ba/e2/8417ae0f8eacb1d071d4950f32f229aa6bf68ab69aab797b72a07ea68d4f/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bd8c4e58ad14b4fa7802b8be49d47993182fdd4023393899632c88fd8cd994eb", size = 237638, upload-time = "2025-06-09T23:01:16.752Z" }, - { url = "https://files.pythonhosted.org/packages/f8/b7/2ace5450ce85f2af05a871b8c8719b341294775a0a6c5585d5e6170f2ce7/frozenlist-1.7.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:04fb24d104f425da3540ed83cbfc31388a586a7696142004c577fa61c6298c3f", size = 233539, upload-time = "2025-06-09T23:01:18.202Z" }, - { url = "https://files.pythonhosted.org/packages/46/b9/6989292c5539553dba63f3c83dc4598186ab2888f67c0dc1d917e6887db6/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a5c505156368e4ea6b53b5ac23c92d7edc864537ff911d2fb24c140bb175e60", size = 215646, upload-time = "2025-06-09T23:01:19.649Z" }, - { url = "https://files.pythonhosted.org/packages/72/31/bc8c5c99c7818293458fe745dab4fd5730ff49697ccc82b554eb69f16a24/frozenlist-1.7.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bd7eb96a675f18aa5c553eb7ddc24a43c8c18f22e1f9925528128c052cdbe00", size = 232233, upload-time = "2025-06-09T23:01:21.175Z" }, - { url = "https://files.pythonhosted.org/packages/59/52/460db4d7ba0811b9ccb85af996019f5d70831f2f5f255f7cc61f86199795/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:05579bf020096fe05a764f1f84cd104a12f78eaab68842d036772dc6d4870b4b", size = 227996, upload-time = "2025-06-09T23:01:23.098Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c9/f4b39e904c03927b7ecf891804fd3b4df3db29b9e487c6418e37988d6e9d/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:376b6222d114e97eeec13d46c486facd41d4f43bab626b7c3f6a8b4e81a5192c", size = 242280, upload-time = "2025-06-09T23:01:24.808Z" }, - { url = "https://files.pythonhosted.org/packages/b8/33/3f8d6ced42f162d743e3517781566b8481322be321b486d9d262adf70bfb/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0aa7e176ebe115379b5b1c95b4096fb1c17cce0847402e227e712c27bdb5a949", size = 217717, upload-time = "2025-06-09T23:01:26.28Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e8/ad683e75da6ccef50d0ab0c2b2324b32f84fc88ceee778ed79b8e2d2fe2e/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3fbba20e662b9c2130dc771e332a99eff5da078b2b2648153a40669a6d0e36ca", size = 236644, upload-time = "2025-06-09T23:01:27.887Z" }, - { url = "https://files.pythonhosted.org/packages/b2/14/8d19ccdd3799310722195a72ac94ddc677541fb4bef4091d8e7775752360/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f3f4410a0a601d349dd406b5713fec59b4cee7e71678d5b17edda7f4655a940b", size = 238879, upload-time = "2025-06-09T23:01:29.524Z" }, - { url = "https://files.pythonhosted.org/packages/ce/13/c12bf657494c2fd1079a48b2db49fa4196325909249a52d8f09bc9123fd7/frozenlist-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e2cdfaaec6a2f9327bf43c933c0319a7c429058e8537c508964a133dffee412e", size = 232502, upload-time = "2025-06-09T23:01:31.287Z" }, - { url = "https://files.pythonhosted.org/packages/d7/8b/e7f9dfde869825489382bc0d512c15e96d3964180c9499efcec72e85db7e/frozenlist-1.7.0-cp313-cp313-win32.whl", hash = "sha256:5fc4df05a6591c7768459caba1b342d9ec23fa16195e744939ba5914596ae3e1", size = 39169, upload-time = "2025-06-09T23:01:35.503Z" }, - { url = "https://files.pythonhosted.org/packages/35/89/a487a98d94205d85745080a37860ff5744b9820a2c9acbcdd9440bfddf98/frozenlist-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:52109052b9791a3e6b5d1b65f4b909703984b770694d3eb64fad124c835d7cba", size = 43219, upload-time = "2025-06-09T23:01:36.784Z" }, - { url = "https://files.pythonhosted.org/packages/56/d5/5c4cf2319a49eddd9dd7145e66c4866bdc6f3dbc67ca3d59685149c11e0d/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a6f86e4193bb0e235ef6ce3dde5cbabed887e0b11f516ce8a0f4d3b33078ec2d", size = 84345, upload-time = "2025-06-09T23:01:38.295Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7d/ec2c1e1dc16b85bc9d526009961953df9cec8481b6886debb36ec9107799/frozenlist-1.7.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:82d664628865abeb32d90ae497fb93df398a69bb3434463d172b80fc25b0dd7d", size = 48880, upload-time = "2025-06-09T23:01:39.887Z" }, - { url = "https://files.pythonhosted.org/packages/69/86/f9596807b03de126e11e7d42ac91e3d0b19a6599c714a1989a4e85eeefc4/frozenlist-1.7.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:912a7e8375a1c9a68325a902f3953191b7b292aa3c3fb0d71a216221deca460b", size = 48498, upload-time = "2025-06-09T23:01:41.318Z" }, - { url = "https://files.pythonhosted.org/packages/5e/cb/df6de220f5036001005f2d726b789b2c0b65f2363b104bbc16f5be8084f8/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9537c2777167488d539bc5de2ad262efc44388230e5118868e172dd4a552b146", size = 292296, upload-time = "2025-06-09T23:01:42.685Z" }, - { url = "https://files.pythonhosted.org/packages/83/1f/de84c642f17c8f851a2905cee2dae401e5e0daca9b5ef121e120e19aa825/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:f34560fb1b4c3e30ba35fa9a13894ba39e5acfc5f60f57d8accde65f46cc5e74", size = 273103, upload-time = "2025-06-09T23:01:44.166Z" }, - { url = "https://files.pythonhosted.org/packages/88/3c/c840bfa474ba3fa13c772b93070893c6e9d5c0350885760376cbe3b6c1b3/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:acd03d224b0175f5a850edc104ac19040d35419eddad04e7cf2d5986d98427f1", size = 292869, upload-time = "2025-06-09T23:01:45.681Z" }, - { url = "https://files.pythonhosted.org/packages/a6/1c/3efa6e7d5a39a1d5ef0abeb51c48fb657765794a46cf124e5aca2c7a592c/frozenlist-1.7.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2038310bc582f3d6a09b3816ab01737d60bf7b1ec70f5356b09e84fb7408ab1", size = 291467, upload-time = "2025-06-09T23:01:47.234Z" }, - { url = "https://files.pythonhosted.org/packages/4f/00/d5c5e09d4922c395e2f2f6b79b9a20dab4b67daaf78ab92e7729341f61f6/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b8c05e4c8e5f36e5e088caa1bf78a687528f83c043706640a92cb76cd6999384", size = 266028, upload-time = "2025-06-09T23:01:48.819Z" }, - { url = "https://files.pythonhosted.org/packages/4e/27/72765be905619dfde25a7f33813ac0341eb6b076abede17a2e3fbfade0cb/frozenlist-1.7.0-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:765bb588c86e47d0b68f23c1bee323d4b703218037765dcf3f25c838c6fecceb", size = 284294, upload-time = "2025-06-09T23:01:50.394Z" }, - { url = "https://files.pythonhosted.org/packages/88/67/c94103a23001b17808eb7dd1200c156bb69fb68e63fcf0693dde4cd6228c/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:32dc2e08c67d86d0969714dd484fd60ff08ff81d1a1e40a77dd34a387e6ebc0c", size = 281898, upload-time = "2025-06-09T23:01:52.234Z" }, - { url = "https://files.pythonhosted.org/packages/42/34/a3e2c00c00f9e2a9db5653bca3fec306349e71aff14ae45ecc6d0951dd24/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:c0303e597eb5a5321b4de9c68e9845ac8f290d2ab3f3e2c864437d3c5a30cd65", size = 290465, upload-time = "2025-06-09T23:01:53.788Z" }, - { url = "https://files.pythonhosted.org/packages/bb/73/f89b7fbce8b0b0c095d82b008afd0590f71ccb3dee6eee41791cf8cd25fd/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:a47f2abb4e29b3a8d0b530f7c3598badc6b134562b1a5caee867f7c62fee51e3", size = 266385, upload-time = "2025-06-09T23:01:55.769Z" }, - { url = "https://files.pythonhosted.org/packages/cd/45/e365fdb554159462ca12df54bc59bfa7a9a273ecc21e99e72e597564d1ae/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:3d688126c242a6fabbd92e02633414d40f50bb6002fa4cf995a1d18051525657", size = 288771, upload-time = "2025-06-09T23:01:57.4Z" }, - { url = "https://files.pythonhosted.org/packages/00/11/47b6117002a0e904f004d70ec5194fe9144f117c33c851e3d51c765962d0/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4e7e9652b3d367c7bd449a727dc79d5043f48b88d0cbfd4f9f1060cf2b414104", size = 288206, upload-time = "2025-06-09T23:01:58.936Z" }, - { url = "https://files.pythonhosted.org/packages/40/37/5f9f3c3fd7f7746082ec67bcdc204db72dad081f4f83a503d33220a92973/frozenlist-1.7.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:1a85e345b4c43db8b842cab1feb41be5cc0b10a1830e6295b69d7310f99becaf", size = 282620, upload-time = "2025-06-09T23:02:00.493Z" }, - { url = "https://files.pythonhosted.org/packages/0b/31/8fbc5af2d183bff20f21aa743b4088eac4445d2bb1cdece449ae80e4e2d1/frozenlist-1.7.0-cp313-cp313t-win32.whl", hash = "sha256:3a14027124ddb70dfcee5148979998066897e79f89f64b13328595c4bdf77c81", size = 43059, upload-time = "2025-06-09T23:02:02.072Z" }, - { url = "https://files.pythonhosted.org/packages/bb/ed/41956f52105b8dbc26e457c5705340c67c8cc2b79f394b79bffc09d0e938/frozenlist-1.7.0-cp313-cp313t-win_amd64.whl", hash = "sha256:3bf8010d71d4507775f658e9823210b7427be36625b387221642725b515dcf3e", size = 47516, upload-time = "2025-06-09T23:02:03.779Z" }, - { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/4a/557715d5047da48d54e659203b9335be7bfaafda2c3f627b7c47e0b3aaf3/frozenlist-1.8.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011", size = 86230, upload-time = "2025-10-06T05:35:23.699Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fb/c85f9fed3ea8fe8740e5b46a59cc141c23b842eca617da8876cfce5f760e/frozenlist-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565", size = 49621, upload-time = "2025-10-06T05:35:25.341Z" }, + { url = "https://files.pythonhosted.org/packages/63/70/26ca3f06aace16f2352796b08704338d74b6d1a24ca38f2771afbb7ed915/frozenlist-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad", size = 49889, upload-time = "2025-10-06T05:35:26.797Z" }, + { url = "https://files.pythonhosted.org/packages/5d/ed/c7895fd2fde7f3ee70d248175f9b6cdf792fb741ab92dc59cd9ef3bd241b/frozenlist-1.8.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2", size = 219464, upload-time = "2025-10-06T05:35:28.254Z" }, + { url = "https://files.pythonhosted.org/packages/6b/83/4d587dccbfca74cb8b810472392ad62bfa100bf8108c7223eb4c4fa2f7b3/frozenlist-1.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186", size = 221649, upload-time = "2025-10-06T05:35:29.454Z" }, + { url = "https://files.pythonhosted.org/packages/6a/c6/fd3b9cd046ec5fff9dab66831083bc2077006a874a2d3d9247dea93ddf7e/frozenlist-1.8.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e", size = 219188, upload-time = "2025-10-06T05:35:30.951Z" }, + { url = "https://files.pythonhosted.org/packages/ce/80/6693f55eb2e085fc8afb28cf611448fb5b90e98e068fa1d1b8d8e66e5c7d/frozenlist-1.8.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450", size = 231748, upload-time = "2025-10-06T05:35:32.101Z" }, + { url = "https://files.pythonhosted.org/packages/97/d6/e9459f7c5183854abd989ba384fe0cc1a0fb795a83c033f0571ec5933ca4/frozenlist-1.8.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef", size = 236351, upload-time = "2025-10-06T05:35:33.834Z" }, + { url = "https://files.pythonhosted.org/packages/97/92/24e97474b65c0262e9ecd076e826bfd1d3074adcc165a256e42e7b8a7249/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4", size = 218767, upload-time = "2025-10-06T05:35:35.205Z" }, + { url = "https://files.pythonhosted.org/packages/ee/bf/dc394a097508f15abff383c5108cb8ad880d1f64a725ed3b90d5c2fbf0bb/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff", size = 235887, upload-time = "2025-10-06T05:35:36.354Z" }, + { url = "https://files.pythonhosted.org/packages/40/90/25b201b9c015dbc999a5baf475a257010471a1fa8c200c843fd4abbee725/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c", size = 228785, upload-time = "2025-10-06T05:35:37.949Z" }, + { url = "https://files.pythonhosted.org/packages/84/f4/b5bc148df03082f05d2dd30c089e269acdbe251ac9a9cf4e727b2dbb8a3d/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f", size = 230312, upload-time = "2025-10-06T05:35:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/db/4b/87e95b5d15097c302430e647136b7d7ab2398a702390cf4c8601975709e7/frozenlist-1.8.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7", size = 217650, upload-time = "2025-10-06T05:35:40.377Z" }, + { url = "https://files.pythonhosted.org/packages/e5/70/78a0315d1fea97120591a83e0acd644da638c872f142fd72a6cebee825f3/frozenlist-1.8.0-cp310-cp310-win32.whl", hash = "sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a", size = 39659, upload-time = "2025-10-06T05:35:41.863Z" }, + { url = "https://files.pythonhosted.org/packages/66/aa/3f04523fb189a00e147e60c5b2205126118f216b0aa908035c45336e27e4/frozenlist-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6", size = 43837, upload-time = "2025-10-06T05:35:43.205Z" }, + { url = "https://files.pythonhosted.org/packages/39/75/1135feecdd7c336938bd55b4dc3b0dfc46d85b9be12ef2628574b28de776/frozenlist-1.8.0-cp310-cp310-win_arm64.whl", hash = "sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e", size = 39989, upload-time = "2025-10-06T05:35:44.596Z" }, + { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912, upload-time = "2025-10-06T05:35:45.98Z" }, + { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046, upload-time = "2025-10-06T05:35:47.009Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119, upload-time = "2025-10-06T05:35:48.38Z" }, + { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067, upload-time = "2025-10-06T05:35:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160, upload-time = "2025-10-06T05:35:51.729Z" }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544, upload-time = "2025-10-06T05:35:53.246Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797, upload-time = "2025-10-06T05:35:54.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923, upload-time = "2025-10-06T05:35:55.861Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886, upload-time = "2025-10-06T05:35:57.399Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731, upload-time = "2025-10-06T05:35:58.563Z" }, + { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544, upload-time = "2025-10-06T05:35:59.719Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806, upload-time = "2025-10-06T05:36:00.959Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382, upload-time = "2025-10-06T05:36:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647, upload-time = "2025-10-06T05:36:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064, upload-time = "2025-10-06T05:36:04.368Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937, upload-time = "2025-10-06T05:36:05.669Z" }, + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] [[package]] name = "fsspec" -version = "2025.9.0" +version = "2025.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/e0/bab50af11c2d75c9c4a2a26a5254573c0bd97cea152254401510950486fa/fsspec-2025.9.0.tar.gz", hash = "sha256:19fd429483d25d28b65ec68f9f4adc16c17ea2c7c7bf54ec61360d478fb19c19", size = 304847, upload-time = "2025-09-02T19:10:49.215Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/47/71/70db47e4f6ce3e5c37a607355f80da8860a33226be640226ac52cb05ef2e/fsspec-2025.9.0-py3-none-any.whl", hash = "sha256:530dc2a2af60a414a832059574df4a6e10cce927f6f4a78209390fe38955cfb7", size = 199289, upload-time = "2025-09-02T19:10:47.708Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" }, ] [package.optional-dependencies] @@ -680,15 +797,6 @@ http = [ { name = "aiohttp" }, ] -[[package]] -name = "gast" -version = "0.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3c/14/c566f5ca00c115db7725263408ff952b8ae6d6a4e792ef9c84e77d9af7a1/gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb", size = 27708, upload-time = "2024-06-27T20:31:49.527Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/61/8001b38461d751cd1a0c3a6ae84346796a5758123f3ed97a1b121dfbf4f3/gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54", size = 21173, upload-time = "2024-07-09T13:15:15.615Z" }, -] - [[package]] name = "geoarrow-rust-core" version = "0.6.3" @@ -806,133 +914,122 @@ wheels = [ ] [[package]] -name = "google-pasta" -version = "0.2.0" +name = "h11" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/35/4a/0bd53b36ff0323d10d5f24ebd67af2de10a1117f5cf4d7add90df92756f1/google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e", size = 40430, upload-time = "2020-03-13T18:57:50.34Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/de/c648ef6835192e6e2cc03f40b19eeda4382c49b5bafb43d88b931c4c74ac/google_pasta-0.2.0-py3-none-any.whl", hash = "sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed", size = 57471, upload-time = "2020-03-13T18:57:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] [[package]] -name = "grpcio" -version = "1.75.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9d/f7/8963848164c7604efb3a3e6ee457fdb3a469653e19002bd24742473254f8/grpcio-1.75.1.tar.gz", hash = "sha256:3e81d89ece99b9ace23a6916880baca613c03a799925afb2857887efa8b1b3d2", size = 12731327, upload-time = "2025-09-26T09:03:36.887Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/51/57/89fd829fb00a6d0bee3fbcb2c8a7aa0252d908949b6ab58bfae99d39d77e/grpcio-1.75.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:1712b5890b22547dd29f3215c5788d8fc759ce6dd0b85a6ba6e2731f2d04c088", size = 5705534, upload-time = "2025-09-26T09:00:52.225Z" }, - { url = "https://files.pythonhosted.org/packages/9a/3d/affe2fb897804c98d56361138e73786af8f4dd876b9d9851cfe6342b53c8/grpcio-1.75.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:683cfc70be0c1383449097cba637317e4737a357cfc185d887fd984206380403", size = 6289953, upload-time = "2025-09-26T09:01:03.699Z" }, - { url = "https://files.pythonhosted.org/packages/87/aa/0f40b7f47a0ff10d7e482bc3af22dac767c7ff27205915f08962d5ca87a2/grpcio-1.75.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:491444c081a54dcd5e6ada57314321ae526377f498d4aa09d975c3241c5b9e1c", size = 6949785, upload-time = "2025-09-26T09:01:07.504Z" }, - { url = "https://files.pythonhosted.org/packages/a5/45/b04407e44050781821c84f26df71b3f7bc469923f92f9f8bc27f1406dbcc/grpcio-1.75.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ce08d4e112d0d38487c2b631ec8723deac9bc404e9c7b1011426af50a79999e4", size = 6465708, upload-time = "2025-09-26T09:01:11.028Z" }, - { url = "https://files.pythonhosted.org/packages/09/3e/4ae3ec0a4d20dcaafbb6e597defcde06399ccdc5b342f607323f3b47f0a3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5a2acda37fc926ccc4547977ac3e56b1df48fe200de968e8c8421f6e3093df6c", size = 7100912, upload-time = "2025-09-26T09:01:14.393Z" }, - { url = "https://files.pythonhosted.org/packages/34/3f/a9085dab5c313bb0cb853f222d095e2477b9b8490a03634cdd8d19daa5c3/grpcio-1.75.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:745c5fe6bf05df6a04bf2d11552c7d867a2690759e7ab6b05c318a772739bd75", size = 8042497, upload-time = "2025-09-26T09:01:17.759Z" }, - { url = "https://files.pythonhosted.org/packages/c3/87/ea54eba931ab9ed3f999ba95f5d8d01a20221b664725bab2fe93e3dee848/grpcio-1.75.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:259526a7159d39e2db40d566fe3e8f8e034d0fb2db5bf9c00e09aace655a4c2b", size = 7493284, upload-time = "2025-09-26T09:01:20.896Z" }, - { url = "https://files.pythonhosted.org/packages/0c/3c/35ca9747473a306bfad0cee04504953f7098527cd112a4ab55c55af9e7bd/grpcio-1.75.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:573855ca2e58e35032aff30bfbd1ee103fbcf4472e4b28d4010757700918e326", size = 5709761, upload-time = "2025-09-26T09:01:28.528Z" }, - { url = "https://files.pythonhosted.org/packages/81/40/bc07aee2911f0d426fa53fe636216100c31a8ea65a400894f280274cb023/grpcio-1.75.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b1ea1bbe77ecbc1be00af2769f4ae4a88ce93be57a4f3eebd91087898ed749f9", size = 6296084, upload-time = "2025-09-26T09:01:34.596Z" }, - { url = "https://files.pythonhosted.org/packages/b8/d1/10c067f6c67396cbf46448b80f27583b5e8c4b46cdfbe18a2a02c2c2f290/grpcio-1.75.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:e5b425aee54cc5e3e3c58f00731e8a33f5567965d478d516d35ef99fd648ab68", size = 6950403, upload-time = "2025-09-26T09:01:36.736Z" }, - { url = "https://files.pythonhosted.org/packages/3f/42/5f628abe360b84dfe8dd8f32be6b0606dc31dc04d3358eef27db791ea4d5/grpcio-1.75.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0049a7bf547dafaeeb1db17079ce79596c298bfe308fc084d023c8907a845b9a", size = 6470166, upload-time = "2025-09-26T09:01:39.474Z" }, - { url = "https://files.pythonhosted.org/packages/c3/93/a24035080251324019882ee2265cfde642d6476c0cf8eb207fc693fcebdc/grpcio-1.75.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b8ea230c7f77c0a1a3208a04a1eda164633fb0767b4cefd65a01079b65e5b1f", size = 7107828, upload-time = "2025-09-26T09:01:41.782Z" }, - { url = "https://files.pythonhosted.org/packages/e4/f8/d18b984c1c9ba0318e3628dbbeb6af77a5007f02abc378c845070f2d3edd/grpcio-1.75.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:36990d629c3c9fb41e546414e5af52d0a7af37ce7113d9682c46d7e2919e4cca", size = 8045421, upload-time = "2025-09-26T09:01:45.835Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b6/4bf9aacff45deca5eac5562547ed212556b831064da77971a4e632917da3/grpcio-1.75.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b10ad908118d38c2453ade7ff790e5bce36580c3742919007a2a78e3a1e521ca", size = 7503290, upload-time = "2025-09-26T09:01:49.28Z" }, - { url = "https://files.pythonhosted.org/packages/3a/81/42be79e73a50aaa20af66731c2defeb0e8c9008d9935a64dd8ea8e8c44eb/grpcio-1.75.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:7b888b33cd14085d86176b1628ad2fcbff94cfbbe7809465097aa0132e58b018", size = 5668314, upload-time = "2025-09-26T09:01:55.424Z" }, - { url = "https://files.pythonhosted.org/packages/14/85/21c71d674f03345ab183c634ecd889d3330177e27baea8d5d247a89b6442/grpcio-1.75.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bb658f703468d7fbb5dcc4037c65391b7dc34f808ac46ed9136c24fc5eeb041d", size = 6246335, upload-time = "2025-09-26T09:02:00.76Z" }, - { url = "https://files.pythonhosted.org/packages/fd/db/3beb661bc56a385ae4fa6b0e70f6b91ac99d47afb726fe76aaff87ebb116/grpcio-1.75.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4b7177a1cdb3c51b02b0c0a256b0a72fdab719600a693e0e9037949efffb200b", size = 6916309, upload-time = "2025-09-26T09:02:02.894Z" }, - { url = "https://files.pythonhosted.org/packages/1e/9c/eda9fe57f2b84343d44c1b66cf3831c973ba29b078b16a27d4587a1fdd47/grpcio-1.75.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d4fa6ccc3ec2e68a04f7b883d354d7fea22a34c44ce535a2f0c0049cf626ddf", size = 6435419, upload-time = "2025-09-26T09:02:05.055Z" }, - { url = "https://files.pythonhosted.org/packages/c3/b8/090c98983e0a9d602e3f919a6e2d4e470a8b489452905f9a0fa472cac059/grpcio-1.75.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d86880ecaeb5b2f0a8afa63824de93adb8ebe4e49d0e51442532f4e08add7d6", size = 7064893, upload-time = "2025-09-26T09:02:07.275Z" }, - { url = "https://files.pythonhosted.org/packages/ec/c0/6d53d4dbbd00f8bd81571f5478d8a95528b716e0eddb4217cc7cb45aae5f/grpcio-1.75.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a8041d2f9e8a742aeae96f4b047ee44e73619f4f9d24565e84d5446c623673b6", size = 8011922, upload-time = "2025-09-26T09:02:09.527Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7c/48455b2d0c5949678d6982c3e31ea4d89df4e16131b03f7d5c590811cbe9/grpcio-1.75.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3652516048bf4c314ce12be37423c79829f46efffb390ad64149a10c6071e8de", size = 7466181, upload-time = "2025-09-26T09:02:12.279Z" }, - { url = "https://files.pythonhosted.org/packages/46/74/bac4ab9f7722164afdf263ae31ba97b8174c667153510322a5eba4194c32/grpcio-1.75.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:3bed22e750d91d53d9e31e0af35a7b0b51367e974e14a4ff229db5b207647884", size = 5672779, upload-time = "2025-09-26T09:02:19.11Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e4/d1954dce2972e32384db6a30273275e8c8ea5a44b80347f9055589333b3f/grpcio-1.75.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5bf4001d3293e3414d0cf99ff9b1139106e57c3a66dfff0c5f60b2a6286ec133", size = 6248838, upload-time = "2025-09-26T09:02:26.426Z" }, - { url = "https://files.pythonhosted.org/packages/06/43/073363bf63826ba8077c335d797a8d026f129dc0912b69c42feaf8f0cd26/grpcio-1.75.1-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:9f82ff474103e26351dacfe8d50214e7c9322960d8d07ba7fa1d05ff981c8b2d", size = 6922663, upload-time = "2025-09-26T09:02:28.724Z" }, - { url = "https://files.pythonhosted.org/packages/c2/6f/076ac0df6c359117676cacfa8a377e2abcecec6a6599a15a672d331f6680/grpcio-1.75.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0ee119f4f88d9f75414217823d21d75bfe0e6ed40135b0cbbfc6376bc9f7757d", size = 6436149, upload-time = "2025-09-26T09:02:30.971Z" }, - { url = "https://files.pythonhosted.org/packages/6b/27/1d08824f1d573fcb1fa35ede40d6020e68a04391709939e1c6f4193b445f/grpcio-1.75.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:664eecc3abe6d916fa6cf8dd6b778e62fb264a70f3430a3180995bf2da935446", size = 7067989, upload-time = "2025-09-26T09:02:33.233Z" }, - { url = "https://files.pythonhosted.org/packages/c6/98/98594cf97b8713feb06a8cb04eeef60b4757e3e2fb91aa0d9161da769843/grpcio-1.75.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c32193fa08b2fbebf08fe08e84f8a0aad32d87c3ad42999c65e9449871b1c66e", size = 8010717, upload-time = "2025-09-26T09:02:36.011Z" }, - { url = "https://files.pythonhosted.org/packages/8c/7e/bb80b1bba03c12158f9254762cdf5cced4a9bc2e8ed51ed335915a5a06ef/grpcio-1.75.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5cebe13088b9254f6e615bcf1da9131d46cfa4e88039454aca9cb65f639bd3bc", size = 7463822, upload-time = "2025-09-26T09:02:38.26Z" }, - { url = "https://files.pythonhosted.org/packages/f2/1b/9a0a5cecd24302b9fdbcd55d15ed6267e5f3d5b898ff9ac8cbe17ee76129/grpcio-1.75.1-cp314-cp314-linux_armv7l.whl", hash = "sha256:c05da79068dd96723793bffc8d0e64c45f316248417515f28d22204d9dae51c7", size = 5673319, upload-time = "2025-09-26T09:02:44.742Z" }, - { url = "https://files.pythonhosted.org/packages/09/7a/26da709e42c4565c3d7bf999a9569da96243ce34a8271a968dee810a7cf1/grpcio-1.75.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4484f4b7287bdaa7a5b3980f3c7224c3c622669405d20f69549f5fb956ad0421", size = 6254706, upload-time = "2025-09-26T09:02:50.4Z" }, - { url = "https://files.pythonhosted.org/packages/f1/08/dcb26a319d3725f199c97e671d904d84ee5680de57d74c566a991cfab632/grpcio-1.75.1-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2720c239c1180eee69f7883c1d4c83fc1a495a2535b5fa322887c70bf02b16e8", size = 6922501, upload-time = "2025-09-26T09:02:52.711Z" }, - { url = "https://files.pythonhosted.org/packages/78/66/044d412c98408a5e23cb348845979a2d17a2e2b6c3c34c1ec91b920f49d0/grpcio-1.75.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07a554fa31c668cf0e7a188678ceeca3cb8fead29bbe455352e712ec33ca701c", size = 6437492, upload-time = "2025-09-26T09:02:55.542Z" }, - { url = "https://files.pythonhosted.org/packages/4e/9d/5e3e362815152aa1afd8b26ea613effa005962f9da0eec6e0e4527e7a7d1/grpcio-1.75.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3e71a2105210366bfc398eef7f57a664df99194f3520edb88b9c3a7e46ee0d64", size = 7081061, upload-time = "2025-09-26T09:02:58.261Z" }, - { url = "https://files.pythonhosted.org/packages/1e/1a/46615682a19e100f46e31ddba9ebc297c5a5ab9ddb47b35443ffadb8776c/grpcio-1.75.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:8679aa8a5b67976776d3c6b0521e99d1c34db8a312a12bcfd78a7085cb9b604e", size = 8010849, upload-time = "2025-09-26T09:03:00.548Z" }, - { url = "https://files.pythonhosted.org/packages/67/8e/3204b94ac30b0f675ab1c06540ab5578660dc8b690db71854d3116f20d00/grpcio-1.75.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aad1c774f4ebf0696a7f148a56d39a3432550612597331792528895258966dc0", size = 7464478, upload-time = "2025-09-26T09:03:03.096Z" }, -] - -[[package]] -name = "h5py" -version = "3.14.0" +name = "hf-xet" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/2d/57fd21d84d93efb4bd0b962383790e19dd1bc053501b4264c97903b4e83e/hf_xet-1.5.1.tar.gz", hash = "sha256:51ef4500dab3764b41135ee1381a4b62ce56fc54d4c92b719b59e597d6df5bf6", size = 876636, upload-time = "2026-06-08T23:02:53.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/ee/dd9ba7beae1005e54131b7d45263cc74c8a066d47d354e6d58ae9445a388/hf_xet-1.5.1-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:dbf48c0d02cf0b2e568944330c60d9120c272dabe013bd892d48e25bc6797577", size = 4069485, upload-time = "2026-06-08T23:02:13.193Z" }, + { url = "https://files.pythonhosted.org/packages/b6/bc/9cae6cfeb4e03070874e73e5c97c66eb90369d3206b6a2b1ef5f96520888/hf_xet-1.5.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78e4e5192ad2b674c2e1160b651cb9134db974f8ae1835bdfbfb0166b894a43", size = 3838493, upload-time = "2026-06-08T23:02:15.282Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b4/d5c01e0eb6d9f2ca2dacd84d0d1b71e6cfbb2ef3208c968528e010e9b3d7/hf_xet-1.5.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f7a04a8ad962422e225bc49fbbac99dc1806764b1f3e54dbd154bffa7593947", size = 4505658, upload-time = "2026-06-08T23:02:17.196Z" }, + { url = "https://files.pythonhosted.org/packages/76/c5/29a7598c0c6383c523dc22186d577f4e04267a626cd95ae60f67c00bfe66/hf_xet-1.5.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d48199c2bf4f8df0adc55d31d1368b6ec0e4d4f45bc86b08038089c23db0bed8", size = 4292822, upload-time = "2026-06-08T23:02:18.608Z" }, + { url = "https://files.pythonhosted.org/packages/04/9a/dceaf6ca69390126b86ea825fb354b93d01163199070b7bd849225de9468/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:97f212a88d14bbf573619a74b7fecb238de77d08fc702e54dec6f78276ca3283", size = 4491255, upload-time = "2026-06-08T23:02:20.124Z" }, + { url = "https://files.pythonhosted.org/packages/48/a7/e5a7afaacf6c1791fdbeeac42951fb81c3d2bc482992b115dedcc86d963e/hf_xet-1.5.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f61e3665892a6c8c5e765395838b8ddf36185da835253d4bc4509a81e49fb342", size = 4711062, upload-time = "2026-06-08T23:02:21.863Z" }, + { url = "https://files.pythonhosted.org/packages/53/49/2802f8433c9742ce281bddc1e65c02c32268ca3098d66828b05e12e45ee2/hf_xet-1.5.1-cp313-cp313t-win_amd64.whl", hash = "sha256:f4ad3ebd4c32dd2b27099d69dc7b2df821e30767e46fb6ee6a0713778243b8ff", size = 4017205, upload-time = "2026-06-08T23:02:23.495Z" }, + { url = "https://files.pythonhosted.org/packages/9e/5a/50c71195b9fb883659f596e7252faf4c18c58e753a9013bdbf9bac5d2250/hf_xet-1.5.1-cp313-cp313t-win_arm64.whl", hash = "sha256:8298485c1e36e7e67cbd01eeb1376619b7af43d4f1ec245caae306f890a8a32d", size = 3845426, upload-time = "2026-06-08T23:02:25.124Z" }, + { url = "https://files.pythonhosted.org/packages/05/24/5e0c28f80371c17d49fed004597d9d132cb75c1f6f53db2cb95f459d2312/hf_xet-1.5.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3474760d10e3bb6f92ff3f024fcb00c0b3e4001e9b035c7483e49a5dd17aa70f", size = 4069676, upload-time = "2026-06-08T23:02:26.759Z" }, + { url = "https://files.pythonhosted.org/packages/d2/17/261ba565b6a4d960fb478f61fdf919c0be5824645aaf1c319eca660c1611/hf_xet-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6762d89b9e3267dfd502b29b2a327b4525f33b17e7b509a78d94e2151a30ce30", size = 3838509, upload-time = "2026-06-08T23:02:28.573Z" }, + { url = "https://files.pythonhosted.org/packages/4e/44/7ffdc2e184b0d41fc0f683ba3936ef669ab63cf242cf36ef50e57d683668/hf_xet-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf67e6ed10260cef62e852789dc91ebb03f382d5bdc4b1dbeb64763ea275e7d6", size = 4505881, upload-time = "2026-06-08T23:02:30.257Z" }, + { url = "https://files.pythonhosted.org/packages/63/b6/788060d5aa4d5e671f1a31bf69624c314eb2d8babab3aa562f9e5d53444e/hf_xet-1.5.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:c6b6cd08ca095058780b50b8ce4d6cbf6787bcf27841705d58a9d32246e3e47a", size = 4292995, upload-time = "2026-06-08T23:02:31.993Z" }, + { url = "https://files.pythonhosted.org/packages/22/93/c5540cbd6b55529b7dc42f6734e88cebee21aefbea34128b66229df56c57/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e1af0de8ca6f190d4294a28b88023db64a1e2d1d719cab044baf75bec569e7a9", size = 4491570, upload-time = "2026-06-08T23:02:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/03/f3/9d8ceab30f44f36c1679b1b8683054c71a0dadc787dbf07421891742d3ca/hf_xet-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4f561cbbb92f80960772059864b7fb07eae879adde1b2e781ec6f86f6ac26c59", size = 4711565, upload-time = "2026-06-08T23:02:35.454Z" }, + { url = "https://files.pythonhosted.org/packages/cd/54/27ed9a5e2cc583b4df82f75a03a4df8dbf55f5a9fa1f47f1fadfb20dbeac/hf_xet-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:e7dbb40617410f432182d918e37c12303fe6700fd6aa6c5964e30a535a4461d6", size = 4017343, upload-time = "2026-06-08T23:02:37.14Z" }, + { url = "https://files.pythonhosted.org/packages/ae/12/ecb2fc8d45e767580e3a37faa97cb895608b614965567efb4f18cff67e27/hf_xet-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:6071d5ccb4d8d2cbd5fea5cc798da4f0ba3f44e25369591c4e89a4987050e61d", size = 3845716, upload-time = "2026-06-08T23:02:39.073Z" }, + { url = "https://files.pythonhosted.org/packages/7a/d8/5e54cf37434759d1f4f2ba9b66077ff9d4c4e1f37b6bd7975da5c40d94ab/hf_xet-1.5.1-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:6abd35c3221eff63836618ddfb954dcf84798603f71d8e33e3ed7b04acfdbe6e", size = 4077794, upload-time = "2026-06-08T23:02:40.656Z" }, + { url = "https://files.pythonhosted.org/packages/35/94/4b2ecfbad8f8b04701a23aefb62f540b9137d058b7e1dbef16a32676f0e9/hf_xet-1.5.1-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:94e761bbd266bf4c03cee73753916062665ce8365aa40ed321f45afcb934b41e", size = 3845354, upload-time = "2026-06-08T23:02:42.702Z" }, + { url = "https://files.pythonhosted.org/packages/de/cc/f99f4bc7295023d7bd9ebbfd51f75cc530ca262c1227666268b8208f4b77/hf_xet-1.5.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:892e3a3a3aecc12aded8b93cf4f9cd059282c7de0732f7d55026f3abdf474350", size = 4514864, upload-time = "2026-06-08T23:02:44.497Z" }, + { url = "https://files.pythonhosted.org/packages/cd/6e/21f7e5a2381278bd3b7b7a5a4d90038518bb6308a0c1daf5d9f8268bb178/hf_xet-1.5.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:a93df2039190502835b1db8cd7e178b0b7b889fe9ab51299d5ced26e0dd879a4", size = 4303784, upload-time = "2026-06-08T23:02:46.203Z" }, + { url = "https://files.pythonhosted.org/packages/35/0e/f992bb6927ac1cb30ef74e62268f551f338bc32b2191f7c96a44c6f7283e/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0c97106032ef70467b4f6bc2d0ccc266d7613ee076afc56516c502f87ce1c4a6", size = 4500703, upload-time = "2026-06-08T23:02:47.628Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d1/90a498d05447980b977b1669246eeeeae4cfb0ea3e7a286eaba627f91bf9/hf_xet-1.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6208adb15d192b90e4c2ad2a27ed864359b2cb0f2494eb6d7c7f3699ac02e2bf", size = 4719498, upload-time = "2026-06-08T23:02:49.268Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b6/20f99cfe97cc663a711f7b33cc21d4793e51968e9a26125b4afcd77315ba/hf_xet-1.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:f7b3002f95d1c13e24bcb4537baa8f0eb3838957067c91bb4959bc004a6435f5", size = 4026419, upload-time = "2026-06-08T23:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/f9/fa/77453694888f03e5a8c8852d1514a0894d8e81c622d39edbaf308ea0dcf4/hf_xet-1.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:93d090b57b211133f6c0dab0205ef5cb6d89162979ba75a74845045cc3063b8e", size = 3855178, upload-time = "2026-06-08T23:02:52.452Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "certifi" }, + { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" } +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/cd/3dd38cdb7cc9266dc4d85f27f0261680cb62f553f1523167ad7454e32b11/h5py-3.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:016e89d3be4c44f8d5e115fab60548e518ecd9efe9fa5c5324505a90773e6f03", size = 4324677, upload-time = "2025-06-06T14:04:23.438Z" }, - { url = "https://files.pythonhosted.org/packages/b1/45/e1a754dc7cd465ba35e438e28557119221ac89b20aaebef48282654e3dc7/h5py-3.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1223b902ef0b5d90bcc8a4778218d6d6cd0f5561861611eda59fa6c52b922f4d", size = 4557272, upload-time = "2025-06-06T14:04:28.863Z" }, - { url = "https://files.pythonhosted.org/packages/08/0c/5e6aaf221557314bc15ba0e0da92e40b24af97ab162076c8ae009320a42b/h5py-3.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c497600c0496548810047257e36360ff551df8b59156d3a4181072eed47d8ad", size = 4298002, upload-time = "2025-06-06T14:04:47.106Z" }, - { url = "https://files.pythonhosted.org/packages/21/d4/d461649cafd5137088fb7f8e78fdc6621bb0c4ff2c090a389f68e8edc136/h5py-3.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:723a40ee6505bd354bfd26385f2dae7bbfa87655f4e61bab175a49d72ebfc06b", size = 4516618, upload-time = "2025-06-06T14:04:52.467Z" }, - { url = "https://files.pythonhosted.org/packages/3f/19/c8bfe8543bfdd7ccfafd46d8cfd96fce53d6c33e9c7921f375530ee1d39a/h5py-3.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:554ef0ced3571366d4d383427c00c966c360e178b5fb5ee5bb31a435c424db0c", size = 4708455, upload-time = "2025-06-06T14:05:11.528Z" }, - { url = "https://files.pythonhosted.org/packages/86/f9/f00de11c82c88bfc1ef22633557bfba9e271e0cb3189ad704183fc4a2644/h5py-3.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cbd41f4e3761f150aa5b662df991868ca533872c95467216f2bec5fcad84882", size = 4929422, upload-time = "2025-06-06T14:05:18.399Z" }, - { url = "https://files.pythonhosted.org/packages/0d/ce/3a21d87896bc7e3e9255e0ad5583ae31ae9e6b4b00e0bcb2a67e2b6acdbc/h5py-3.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8cbaf6910fa3983c46172666b0b8da7b7bd90d764399ca983236f2400436eeb", size = 4700675, upload-time = "2025-06-06T14:05:37.38Z" }, - { url = "https://files.pythonhosted.org/packages/e7/ec/86f59025306dcc6deee5fda54d980d077075b8d9889aac80f158bd585f1b/h5py-3.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d90e6445ab7c146d7f7981b11895d70bc1dd91278a4f9f9028bc0c95e4a53f13", size = 4921632, upload-time = "2025-06-06T14:05:43.464Z" }, + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] [[package]] -name = "hf-xet" -version = "1.1.10" +name = "httpx" +version = "0.28.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/31/feeddfce1748c4a233ec1aa5b7396161c07ae1aa9b7bdbc9a72c3c7dd768/hf_xet-1.1.10.tar.gz", hash = "sha256:408aef343800a2102374a883f283ff29068055c111f003ff840733d3b715bb97", size = 487910, upload-time = "2025-09-12T20:10:27.12Z" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/a2/343e6d05de96908366bdc0081f2d8607d61200be2ac802769c4284cc65bd/hf_xet-1.1.10-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:686083aca1a6669bc85c21c0563551cbcdaa5cf7876a91f3d074a030b577231d", size = 2761466, upload-time = "2025-09-12T20:10:22.836Z" }, - { url = "https://files.pythonhosted.org/packages/31/f9/6215f948ac8f17566ee27af6430ea72045e0418ce757260248b483f4183b/hf_xet-1.1.10-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:71081925383b66b24eedff3013f8e6bbd41215c3338be4b94ba75fd75b21513b", size = 2623807, upload-time = "2025-09-12T20:10:21.118Z" }, - { url = "https://files.pythonhosted.org/packages/15/07/86397573efefff941e100367bbda0b21496ffcdb34db7ab51912994c32a2/hf_xet-1.1.10-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b6bceb6361c80c1cc42b5a7b4e3efd90e64630bcf11224dcac50ef30a47e435", size = 3186960, upload-time = "2025-09-12T20:10:19.336Z" }, - { url = "https://files.pythonhosted.org/packages/01/a7/0b2e242b918cc30e1f91980f3c4b026ff2eedaf1e2ad96933bca164b2869/hf_xet-1.1.10-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eae7c1fc8a664e54753ffc235e11427ca61f4b0477d757cc4eb9ae374b69f09c", size = 3087167, upload-time = "2025-09-12T20:10:17.255Z" }, - { url = "https://files.pythonhosted.org/packages/4a/25/3e32ab61cc7145b11eee9d745988e2f0f4fafda81b25980eebf97d8cff15/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0a0005fd08f002180f7a12d4e13b22be277725bc23ed0529f8add5c7a6309c06", size = 3248612, upload-time = "2025-09-12T20:10:24.093Z" }, - { url = "https://files.pythonhosted.org/packages/2c/3d/ab7109e607ed321afaa690f557a9ada6d6d164ec852fd6bf9979665dc3d6/hf_xet-1.1.10-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f900481cf6e362a6c549c61ff77468bd59d6dd082f3170a36acfef2eb6a6793f", size = 3353360, upload-time = "2025-09-12T20:10:25.563Z" }, - { url = "https://files.pythonhosted.org/packages/ee/0e/471f0a21db36e71a2f1752767ad77e92d8cde24e974e03d662931b1305ec/hf_xet-1.1.10-cp37-abi3-win_amd64.whl", hash = "sha256:5f54b19cc347c13235ae7ee98b330c26dd65ef1df47e5316ffb1e87713ca7045", size = 2804691, upload-time = "2025-09-12T20:10:28.433Z" }, + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] name = "huggingface-hub" -version = "0.35.3" +version = "1.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, - { name = "hf-xet", marker = "platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "requests" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/10/7e/a0a97de7c73671863ca6b3f61fa12518caf35db37825e43d63a70956738c/huggingface_hub-0.35.3.tar.gz", hash = "sha256:350932eaa5cc6a4747efae85126ee220e4ef1b54e29d31c3b45c5612ddf0b32a", size = 461798, upload-time = "2025-09-29T14:29:58.625Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/8f/999e4dda11c6187c78f090eac00895a47e11a0049308f07579bcb7aa3aa2/huggingface_hub-1.23.0.tar.gz", hash = "sha256:c04997fb8bbdace1e57b7703d30ed7678af51f70d00d241819ff411b92ae9a88", size = 919163, upload-time = "2026-07-09T14:49:32.315Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/a0/651f93d154cb72323358bf2bbae3e642bdb5d2f1bfc874d096f7cb159fa0/huggingface_hub-0.35.3-py3-none-any.whl", hash = "sha256:0e3a01829c19d86d03793e4577816fe3bdfc1602ac62c7fb220d593d351224ba", size = 564262, upload-time = "2025-09-29T14:29:55.813Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ce/13b2ba57838b8db1e6bd033c1b21ce0b9f6153b87d4e4939f77074e41eb0/huggingface_hub-1.23.0-py3-none-any.whl", hash = "sha256:b1d604788f5adc7f0eb246e03e0ec19011ca06e38400218c347dccc3dffa64a2", size = 770336, upload-time = "2026-07-09T14:49:30.597Z" }, ] [[package]] name = "idna" -version = "3.15" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514, upload-time = "2024-09-11T14:56:07.019Z" }, ] [[package]] name = "iniconfig" -version = "2.1.0" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] [[package]] @@ -949,48 +1046,28 @@ wheels = [ [[package]] name = "jmespath" -version = "1.0.1" +version = "1.1.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843, upload-time = "2022-06-17T18:00:12.224Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256, upload-time = "2022-06-17T18:00:10.251Z" }, -] - -[[package]] -name = "keras" -version = "3.11.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "h5py" }, - { name = "ml-dtypes" }, - { name = "namex" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "optree" }, - { name = "packaging" }, - { name = "rich" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6a/89/646425fe9a46f9053430e1271f817c36041c6f33469950a3caafc3d2591e/keras-3.11.3.tar.gz", hash = "sha256:efda616835c31b7d916d72303ef9adec1257320bc9fd4b2b0138840fc65fb5b7", size = 1065906, upload-time = "2025-08-21T22:08:57.643Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/94/5b/4c778cc921ce4b864b238f63f8e3ff6e954ab19b80c9fa680593ad8093d4/keras-3.11.3-py3-none-any.whl", hash = "sha256:f484f050e05ee400455b05ec8c36ed35edc34de94256b6073f56cfe68f65491f", size = 1408438, upload-time = "2025-08-21T22:08:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] [[package]] name = "lance-namespace" -version = "0.8.6" +version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace-urllib3-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/12/f7ab93b29be3edbf5fc3610714bf2d06088e7f4524bfb38dfd6852458b08/lance_namespace-0.8.6.tar.gz", hash = "sha256:18232e721c8188145f4ec9389cc2dfbeeabf54a619d94885ea1b3375bee9f4af", size = 11529, upload-time = "2026-06-12T17:36:41.651Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/1b/5b1668ee2dc8910965f390640359112a31157092fcf8e000b89c79b58708/lance_namespace-0.8.6-py3-none-any.whl", hash = "sha256:571eae34f9aad70e5b05020416c2860889b9ec82993ccd0eb015e7b39c3ea309", size = 13383, upload-time = "2026-06-12T17:36:43.456Z" }, + { url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" }, ] [[package]] name = "lance-namespace-urllib3-client" -version = "0.8.6" +version = "0.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, @@ -998,42 +1075,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c7/80/fb224b4a89c1c1638cde949cb6cce6c3aca7759effbfea46a3d9c3960b21/lance_namespace_urllib3_client-0.8.6.tar.gz", hash = "sha256:b6fb1d306e74a7576e5309919020be744527de484a63dbf5eed10f8b368548df", size = 228772, upload-time = "2026-06-12T17:36:42.609Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/90/1e27de15cd1b16785a1c7312beb0a59e75c8344a815f600f58173a565bd1/lance_namespace_urllib3_client-0.8.6-py3-none-any.whl", hash = "sha256:9d78249c3fb15aa3d15d668f78f04a275af3d08d800a7027492f37996ac4968b", size = 369950, upload-time = "2026-06-12T17:36:40.438Z" }, -] - -[[package]] -name = "libclang" -version = "18.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/5c/ca35e19a4f142adffa27e3d652196b7362fa612243e2b916845d801454fc/libclang-18.1.1.tar.gz", hash = "sha256:a1214966d08d73d971287fc3ead8dfaf82eb07fb197680d8b3859dbbbbf78250", size = 39612, upload-time = "2024-03-17T16:04:37.434Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/fc/716c1e62e512ef1c160e7984a73a5fc7df45166f2ff3f254e71c58076f7c/libclang-18.1.1-py2.py3-none-manylinux2010_x86_64.whl", hash = "sha256:c533091d8a3bbf7460a00cb6c1a71da93bffe148f172c7d03b1c31fbf8aa2a0b", size = 24515943, upload-time = "2024-03-17T16:03:45.942Z" }, - { url = "https://files.pythonhosted.org/packages/3c/3d/f0ac1150280d8d20d059608cf2d5ff61b7c3b7f7bcf9c0f425ab92df769a/libclang-18.1.1-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:54dda940a4a0491a9d1532bf071ea3ef26e6dbaf03b5000ed94dd7174e8f9592", size = 23784972, upload-time = "2024-03-17T16:12:47.677Z" }, - { url = "https://files.pythonhosted.org/packages/fe/2f/d920822c2b1ce9326a4c78c0c2b4aa3fde610c7ee9f631b600acb5376c26/libclang-18.1.1-py2.py3-none-manylinux2014_armv7l.whl", hash = "sha256:cf4a99b05376513717ab5d82a0db832c56ccea4fd61a69dbb7bccf2dfb207dbe", size = 20259606, upload-time = "2024-03-17T16:17:42.437Z" }, - { url = "https://files.pythonhosted.org/packages/2d/c2/de1db8c6d413597076a4259cea409b83459b2db997c003578affdd32bf66/libclang-18.1.1-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:69f8eb8f65c279e765ffd28aaa7e9e364c776c17618af8bff22a8df58677ff4f", size = 24921494, upload-time = "2024-03-17T16:14:20.132Z" }, -] - -[[package]] -name = "markdown" -version = "3.9" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" }, -] - -[[package]] -name = "markdown-it-py" -version = "4.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mdurl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, + { url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" }, ] [[package]] @@ -1145,22 +1189,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/93/e32e79333f0902ba292b996f504f5f06be59587f7d02ab8d5ed1e3066445/maturin-1.13.3-py3-none-win_arm64.whl", hash = "sha256:2389fe92d017cea9d94e521fa0175314a4c52f79a1057b901fbc9f8686ef7d0b", size = 9706562, upload-time = "2026-05-11T07:43:31.743Z" }, ] -[[package]] -name = "mdurl" -version = "0.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, -] - [[package]] name = "ml-dtypes" version = "0.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/78/a7/aad060393123cfb383956dca68402aff3db1e1caffd5764887ed5153f41b/ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9", size = 692316, upload-time = "2025-07-29T18:39:19.454Z" } wheels = [ @@ -1207,131 +1243,163 @@ wheels = [ [[package]] name = "multidict" -version = "6.6.4" +version = "6.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/69/7f/0652e6ed47ab288e3756ea9c0df8b14950781184d4bd7883f4d87dd41245/multidict-6.6.4.tar.gz", hash = "sha256:d2d4e4787672911b48350df02ed3fa3fffdc2f2e8ca06dd6afdf34189b76a9dd", size = 101843, upload-time = "2025-08-11T12:08:48.217Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/6b/86f353088c1358e76fd30b0146947fddecee812703b604ee901e85cd2a80/multidict-6.6.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b8aa6f0bd8125ddd04a6593437bad6a7e70f300ff4180a531654aa2ab3f6d58f", size = 77054, upload-time = "2025-08-11T12:06:02.99Z" }, - { url = "https://files.pythonhosted.org/packages/19/5d/c01dc3d3788bb877bd7f5753ea6eb23c1beeca8044902a8f5bfb54430f63/multidict-6.6.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b9e5853bbd7264baca42ffc53391b490d65fe62849bf2c690fa3f6273dbcd0cb", size = 44914, upload-time = "2025-08-11T12:06:05.264Z" }, - { url = "https://files.pythonhosted.org/packages/46/44/964dae19ea42f7d3e166474d8205f14bb811020e28bc423d46123ddda763/multidict-6.6.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0af5f9dee472371e36d6ae38bde009bd8ce65ac7335f55dcc240379d7bed1495", size = 44601, upload-time = "2025-08-11T12:06:06.627Z" }, - { url = "https://files.pythonhosted.org/packages/31/20/0616348a1dfb36cb2ab33fc9521de1f27235a397bf3f59338e583afadd17/multidict-6.6.4-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:d24f351e4d759f5054b641c81e8291e5d122af0fca5c72454ff77f7cbe492de8", size = 224821, upload-time = "2025-08-11T12:06:08.06Z" }, - { url = "https://files.pythonhosted.org/packages/14/26/5d8923c69c110ff51861af05bd27ca6783011b96725d59ccae6d9daeb627/multidict-6.6.4-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:db6a3810eec08280a172a6cd541ff4a5f6a97b161d93ec94e6c4018917deb6b7", size = 242608, upload-time = "2025-08-11T12:06:09.697Z" }, - { url = "https://files.pythonhosted.org/packages/5c/cc/e2ad3ba9459aa34fa65cf1f82a5c4a820a2ce615aacfb5143b8817f76504/multidict-6.6.4-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a1b20a9d56b2d81e2ff52ecc0670d583eaabaa55f402e8d16dd062373dbbe796", size = 222324, upload-time = "2025-08-11T12:06:10.905Z" }, - { url = "https://files.pythonhosted.org/packages/19/db/4ed0f65701afbc2cb0c140d2d02928bb0fe38dd044af76e58ad7c54fd21f/multidict-6.6.4-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c9854df0eaa610a23494c32a6f44a3a550fb398b6b51a56e8c6b9b3689578db", size = 253234, upload-time = "2025-08-11T12:06:12.658Z" }, - { url = "https://files.pythonhosted.org/packages/94/c1/5160c9813269e39ae14b73debb907bfaaa1beee1762da8c4fb95df4764ed/multidict-6.6.4-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4bb7627fd7a968f41905a4d6343b0d63244a0623f006e9ed989fa2b78f4438a0", size = 251613, upload-time = "2025-08-11T12:06:13.97Z" }, - { url = "https://files.pythonhosted.org/packages/05/a9/48d1bd111fc2f8fb98b2ed7f9a115c55a9355358432a19f53c0b74d8425d/multidict-6.6.4-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:caebafea30ed049c57c673d0b36238b1748683be2593965614d7b0e99125c877", size = 241649, upload-time = "2025-08-11T12:06:15.204Z" }, - { url = "https://files.pythonhosted.org/packages/85/2a/f7d743df0019408768af8a70d2037546a2be7b81fbb65f040d76caafd4c5/multidict-6.6.4-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ad887a8250eb47d3ab083d2f98db7f48098d13d42eb7a3b67d8a5c795f224ace", size = 239238, upload-time = "2025-08-11T12:06:16.467Z" }, - { url = "https://files.pythonhosted.org/packages/cb/b8/4f4bb13323c2d647323f7919201493cf48ebe7ded971717bfb0f1a79b6bf/multidict-6.6.4-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:ed8358ae7d94ffb7c397cecb62cbac9578a83ecefc1eba27b9090ee910e2efb6", size = 233517, upload-time = "2025-08-11T12:06:18.107Z" }, - { url = "https://files.pythonhosted.org/packages/33/29/4293c26029ebfbba4f574febd2ed01b6f619cfa0d2e344217d53eef34192/multidict-6.6.4-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:ecab51ad2462197a4c000b6d5701fc8585b80eecb90583635d7e327b7b6923eb", size = 243122, upload-time = "2025-08-11T12:06:19.361Z" }, - { url = "https://files.pythonhosted.org/packages/20/60/a1c53628168aa22447bfde3a8730096ac28086704a0d8c590f3b63388d0c/multidict-6.6.4-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:c5c97aa666cf70e667dfa5af945424ba1329af5dd988a437efeb3a09430389fb", size = 248992, upload-time = "2025-08-11T12:06:20.661Z" }, - { url = "https://files.pythonhosted.org/packages/a3/3b/55443a0c372f33cae5d9ec37a6a973802884fa0ab3586659b197cf8cc5e9/multidict-6.6.4-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:9a950b7cf54099c1209f455ac5970b1ea81410f2af60ed9eb3c3f14f0bfcf987", size = 243708, upload-time = "2025-08-11T12:06:21.891Z" }, - { url = "https://files.pythonhosted.org/packages/7c/60/a18c6900086769312560b2626b18e8cca22d9e85b1186ba77f4755b11266/multidict-6.6.4-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:163c7ea522ea9365a8a57832dea7618e6cbdc3cd75f8c627663587459a4e328f", size = 237498, upload-time = "2025-08-11T12:06:23.206Z" }, - { url = "https://files.pythonhosted.org/packages/11/3d/8bdd8bcaff2951ce2affccca107a404925a2beafedd5aef0b5e4a71120a6/multidict-6.6.4-cp310-cp310-win32.whl", hash = "sha256:17d2cbbfa6ff20821396b25890f155f40c986f9cfbce5667759696d83504954f", size = 41415, upload-time = "2025-08-11T12:06:24.77Z" }, - { url = "https://files.pythonhosted.org/packages/c0/53/cab1ad80356a4cd1b685a254b680167059b433b573e53872fab245e9fc95/multidict-6.6.4-cp310-cp310-win_amd64.whl", hash = "sha256:ce9a40fbe52e57e7edf20113a4eaddfacac0561a0879734e636aa6d4bb5e3fb0", size = 46046, upload-time = "2025-08-11T12:06:25.893Z" }, - { url = "https://files.pythonhosted.org/packages/cf/9a/874212b6f5c1c2d870d0a7adc5bb4cfe9b0624fa15cdf5cf757c0f5087ae/multidict-6.6.4-cp310-cp310-win_arm64.whl", hash = "sha256:01d0959807a451fe9fdd4da3e139cb5b77f7328baf2140feeaf233e1d777b729", size = 43147, upload-time = "2025-08-11T12:06:27.534Z" }, - { url = "https://files.pythonhosted.org/packages/6b/7f/90a7f01e2d005d6653c689039977f6856718c75c5579445effb7e60923d1/multidict-6.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c7a0e9b561e6460484318a7612e725df1145d46b0ef57c6b9866441bf6e27e0c", size = 76472, upload-time = "2025-08-11T12:06:29.006Z" }, - { url = "https://files.pythonhosted.org/packages/54/a3/bed07bc9e2bb302ce752f1dabc69e884cd6a676da44fb0e501b246031fdd/multidict-6.6.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6bf2f10f70acc7a2446965ffbc726e5fc0b272c97a90b485857e5c70022213eb", size = 44634, upload-time = "2025-08-11T12:06:30.374Z" }, - { url = "https://files.pythonhosted.org/packages/a7/4b/ceeb4f8f33cf81277da464307afeaf164fb0297947642585884f5cad4f28/multidict-6.6.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66247d72ed62d5dd29752ffc1d3b88f135c6a8de8b5f63b7c14e973ef5bda19e", size = 44282, upload-time = "2025-08-11T12:06:31.958Z" }, - { url = "https://files.pythonhosted.org/packages/03/35/436a5da8702b06866189b69f655ffdb8f70796252a8772a77815f1812679/multidict-6.6.4-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:105245cc6b76f51e408451a844a54e6823bbd5a490ebfe5bdfc79798511ceded", size = 229696, upload-time = "2025-08-11T12:06:33.087Z" }, - { url = "https://files.pythonhosted.org/packages/b6/0e/915160be8fecf1fca35f790c08fb74ca684d752fcba62c11daaf3d92c216/multidict-6.6.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbbc54e58b34c3bae389ef00046be0961f30fef7cb0dd9c7756aee376a4f7683", size = 246665, upload-time = "2025-08-11T12:06:34.448Z" }, - { url = "https://files.pythonhosted.org/packages/08/ee/2f464330acd83f77dcc346f0b1a0eaae10230291450887f96b204b8ac4d3/multidict-6.6.4-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:56c6b3652f945c9bc3ac6c8178cd93132b8d82dd581fcbc3a00676c51302bc1a", size = 225485, upload-time = "2025-08-11T12:06:35.672Z" }, - { url = "https://files.pythonhosted.org/packages/71/cc/9a117f828b4d7fbaec6adeed2204f211e9caf0a012692a1ee32169f846ae/multidict-6.6.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b95494daf857602eccf4c18ca33337dd2be705bccdb6dddbfc9d513e6addb9d9", size = 257318, upload-time = "2025-08-11T12:06:36.98Z" }, - { url = "https://files.pythonhosted.org/packages/25/77/62752d3dbd70e27fdd68e86626c1ae6bccfebe2bb1f84ae226363e112f5a/multidict-6.6.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e5b1413361cef15340ab9dc61523e653d25723e82d488ef7d60a12878227ed50", size = 254689, upload-time = "2025-08-11T12:06:38.233Z" }, - { url = "https://files.pythonhosted.org/packages/00/6e/fac58b1072a6fc59af5e7acb245e8754d3e1f97f4f808a6559951f72a0d4/multidict-6.6.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e167bf899c3d724f9662ef00b4f7fef87a19c22b2fead198a6f68b263618df52", size = 246709, upload-time = "2025-08-11T12:06:39.517Z" }, - { url = "https://files.pythonhosted.org/packages/01/ef/4698d6842ef5e797c6db7744b0081e36fb5de3d00002cc4c58071097fac3/multidict-6.6.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:aaea28ba20a9026dfa77f4b80369e51cb767c61e33a2d4043399c67bd95fb7c6", size = 243185, upload-time = "2025-08-11T12:06:40.796Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c9/d82e95ae1d6e4ef396934e9b0e942dfc428775f9554acf04393cce66b157/multidict-6.6.4-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8c91cdb30809a96d9ecf442ec9bc45e8cfaa0f7f8bdf534e082c2443a196727e", size = 237838, upload-time = "2025-08-11T12:06:42.595Z" }, - { url = "https://files.pythonhosted.org/packages/57/cf/f94af5c36baaa75d44fab9f02e2a6bcfa0cd90acb44d4976a80960759dbc/multidict-6.6.4-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a0ccbfe93ca114c5d65a2471d52d8829e56d467c97b0e341cf5ee45410033b3", size = 246368, upload-time = "2025-08-11T12:06:44.304Z" }, - { url = "https://files.pythonhosted.org/packages/4a/fe/29f23460c3d995f6a4b678cb2e9730e7277231b981f0b234702f0177818a/multidict-6.6.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:55624b3f321d84c403cb7d8e6e982f41ae233d85f85db54ba6286f7295dc8a9c", size = 253339, upload-time = "2025-08-11T12:06:45.597Z" }, - { url = "https://files.pythonhosted.org/packages/29/b6/fd59449204426187b82bf8a75f629310f68c6adc9559dc922d5abe34797b/multidict-6.6.4-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:4a1fb393a2c9d202cb766c76208bd7945bc194eba8ac920ce98c6e458f0b524b", size = 246933, upload-time = "2025-08-11T12:06:46.841Z" }, - { url = "https://files.pythonhosted.org/packages/19/52/d5d6b344f176a5ac3606f7a61fb44dc746e04550e1a13834dff722b8d7d6/multidict-6.6.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:43868297a5759a845fa3a483fb4392973a95fb1de891605a3728130c52b8f40f", size = 242225, upload-time = "2025-08-11T12:06:48.588Z" }, - { url = "https://files.pythonhosted.org/packages/ec/d3/5b2281ed89ff4d5318d82478a2a2450fcdfc3300da48ff15c1778280ad26/multidict-6.6.4-cp311-cp311-win32.whl", hash = "sha256:ed3b94c5e362a8a84d69642dbeac615452e8af9b8eb825b7bc9f31a53a1051e2", size = 41306, upload-time = "2025-08-11T12:06:49.95Z" }, - { url = "https://files.pythonhosted.org/packages/74/7d/36b045c23a1ab98507aefd44fd8b264ee1dd5e5010543c6fccf82141ccef/multidict-6.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:d8c112f7a90d8ca5d20213aa41eac690bb50a76da153e3afb3886418e61cb22e", size = 46029, upload-time = "2025-08-11T12:06:51.082Z" }, - { url = "https://files.pythonhosted.org/packages/0f/5e/553d67d24432c5cd52b49047f2d248821843743ee6d29a704594f656d182/multidict-6.6.4-cp311-cp311-win_arm64.whl", hash = "sha256:3bb0eae408fa1996d87247ca0d6a57b7fc1dcf83e8a5c47ab82c558c250d4adf", size = 43017, upload-time = "2025-08-11T12:06:52.243Z" }, - { url = "https://files.pythonhosted.org/packages/05/f6/512ffd8fd8b37fb2680e5ac35d788f1d71bbaf37789d21a820bdc441e565/multidict-6.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0ffb87be160942d56d7b87b0fdf098e81ed565add09eaa1294268c7f3caac4c8", size = 76516, upload-time = "2025-08-11T12:06:53.393Z" }, - { url = "https://files.pythonhosted.org/packages/99/58/45c3e75deb8855c36bd66cc1658007589662ba584dbf423d01df478dd1c5/multidict-6.6.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d191de6cbab2aff5de6c5723101705fd044b3e4c7cfd587a1929b5028b9714b3", size = 45394, upload-time = "2025-08-11T12:06:54.555Z" }, - { url = "https://files.pythonhosted.org/packages/fd/ca/e8c4472a93a26e4507c0b8e1f0762c0d8a32de1328ef72fd704ef9cc5447/multidict-6.6.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38a0956dd92d918ad5feff3db8fcb4a5eb7dba114da917e1a88475619781b57b", size = 43591, upload-time = "2025-08-11T12:06:55.672Z" }, - { url = "https://files.pythonhosted.org/packages/05/51/edf414f4df058574a7265034d04c935aa84a89e79ce90fcf4df211f47b16/multidict-6.6.4-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6865f6d3b7900ae020b495d599fcf3765653bc927951c1abb959017f81ae8287", size = 237215, upload-time = "2025-08-11T12:06:57.213Z" }, - { url = "https://files.pythonhosted.org/packages/c8/45/8b3d6dbad8cf3252553cc41abea09ad527b33ce47a5e199072620b296902/multidict-6.6.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a2088c126b6f72db6c9212ad827d0ba088c01d951cee25e758c450da732c138", size = 258299, upload-time = "2025-08-11T12:06:58.946Z" }, - { url = "https://files.pythonhosted.org/packages/3c/e8/8ca2e9a9f5a435fc6db40438a55730a4bf4956b554e487fa1b9ae920f825/multidict-6.6.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0f37bed7319b848097085d7d48116f545985db988e2256b2e6f00563a3416ee6", size = 242357, upload-time = "2025-08-11T12:07:00.301Z" }, - { url = "https://files.pythonhosted.org/packages/0f/84/80c77c99df05a75c28490b2af8f7cba2a12621186e0a8b0865d8e745c104/multidict-6.6.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:01368e3c94032ba6ca0b78e7ccb099643466cf24f8dc8eefcfdc0571d56e58f9", size = 268369, upload-time = "2025-08-11T12:07:01.638Z" }, - { url = "https://files.pythonhosted.org/packages/0d/e9/920bfa46c27b05fb3e1ad85121fd49f441492dca2449c5bcfe42e4565d8a/multidict-6.6.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe323540c255db0bffee79ad7f048c909f2ab0edb87a597e1c17da6a54e493c", size = 269341, upload-time = "2025-08-11T12:07:02.943Z" }, - { url = "https://files.pythonhosted.org/packages/af/65/753a2d8b05daf496f4a9c367fe844e90a1b2cac78e2be2c844200d10cc4c/multidict-6.6.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8eb3025f17b0a4c3cd08cda49acf312a19ad6e8a4edd9dbd591e6506d999402", size = 256100, upload-time = "2025-08-11T12:07:04.564Z" }, - { url = "https://files.pythonhosted.org/packages/09/54/655be13ae324212bf0bc15d665a4e34844f34c206f78801be42f7a0a8aaa/multidict-6.6.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bbc14f0365534d35a06970d6a83478b249752e922d662dc24d489af1aa0d1be7", size = 253584, upload-time = "2025-08-11T12:07:05.914Z" }, - { url = "https://files.pythonhosted.org/packages/5c/74/ab2039ecc05264b5cec73eb018ce417af3ebb384ae9c0e9ed42cb33f8151/multidict-6.6.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:75aa52fba2d96bf972e85451b99d8e19cc37ce26fd016f6d4aa60da9ab2b005f", size = 251018, upload-time = "2025-08-11T12:07:08.301Z" }, - { url = "https://files.pythonhosted.org/packages/af/0a/ccbb244ac848e56c6427f2392741c06302bbfba49c0042f1eb3c5b606497/multidict-6.6.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fefd4a815e362d4f011919d97d7b4a1e566f1dde83dc4ad8cfb5b41de1df68d", size = 251477, upload-time = "2025-08-11T12:07:10.248Z" }, - { url = "https://files.pythonhosted.org/packages/0e/b0/0ed49bba775b135937f52fe13922bc64a7eaf0a3ead84a36e8e4e446e096/multidict-6.6.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:db9801fe021f59a5b375ab778973127ca0ac52429a26e2fd86aa9508f4d26eb7", size = 263575, upload-time = "2025-08-11T12:07:11.928Z" }, - { url = "https://files.pythonhosted.org/packages/3e/d9/7fb85a85e14de2e44dfb6a24f03c41e2af8697a6df83daddb0e9b7569f73/multidict-6.6.4-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a650629970fa21ac1fb06ba25dabfc5b8a2054fcbf6ae97c758aa956b8dba802", size = 259649, upload-time = "2025-08-11T12:07:13.244Z" }, - { url = "https://files.pythonhosted.org/packages/03/9e/b3a459bcf9b6e74fa461a5222a10ff9b544cb1cd52fd482fb1b75ecda2a2/multidict-6.6.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:452ff5da78d4720d7516a3a2abd804957532dd69296cb77319c193e3ffb87e24", size = 251505, upload-time = "2025-08-11T12:07:14.57Z" }, - { url = "https://files.pythonhosted.org/packages/86/a2/8022f78f041dfe6d71e364001a5cf987c30edfc83c8a5fb7a3f0974cff39/multidict-6.6.4-cp312-cp312-win32.whl", hash = "sha256:8c2fcb12136530ed19572bbba61b407f655e3953ba669b96a35036a11a485793", size = 41888, upload-time = "2025-08-11T12:07:15.904Z" }, - { url = "https://files.pythonhosted.org/packages/c7/eb/d88b1780d43a56db2cba24289fa744a9d216c1a8546a0dc3956563fd53ea/multidict-6.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:047d9425860a8c9544fed1b9584f0c8bcd31bcde9568b047c5e567a1025ecd6e", size = 46072, upload-time = "2025-08-11T12:07:17.045Z" }, - { url = "https://files.pythonhosted.org/packages/9f/16/b929320bf5750e2d9d4931835a4c638a19d2494a5b519caaaa7492ebe105/multidict-6.6.4-cp312-cp312-win_arm64.whl", hash = "sha256:14754eb72feaa1e8ae528468f24250dd997b8e2188c3d2f593f9eba259e4b364", size = 43222, upload-time = "2025-08-11T12:07:18.328Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5d/e1db626f64f60008320aab00fbe4f23fc3300d75892a3381275b3d284580/multidict-6.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f46a6e8597f9bd71b31cc708195d42b634c8527fecbcf93febf1052cacc1f16e", size = 75848, upload-time = "2025-08-11T12:07:19.912Z" }, - { url = "https://files.pythonhosted.org/packages/4c/aa/8b6f548d839b6c13887253af4e29c939af22a18591bfb5d0ee6f1931dae8/multidict-6.6.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:22e38b2bc176c5eb9c0a0e379f9d188ae4cd8b28c0f53b52bce7ab0a9e534657", size = 45060, upload-time = "2025-08-11T12:07:21.163Z" }, - { url = "https://files.pythonhosted.org/packages/eb/c6/f5e97e5d99a729bc2aa58eb3ebfa9f1e56a9b517cc38c60537c81834a73f/multidict-6.6.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5df8afd26f162da59e218ac0eefaa01b01b2e6cd606cffa46608f699539246da", size = 43269, upload-time = "2025-08-11T12:07:22.392Z" }, - { url = "https://files.pythonhosted.org/packages/dc/31/d54eb0c62516776f36fe67f84a732f97e0b0e12f98d5685bebcc6d396910/multidict-6.6.4-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:49517449b58d043023720aa58e62b2f74ce9b28f740a0b5d33971149553d72aa", size = 237158, upload-time = "2025-08-11T12:07:23.636Z" }, - { url = "https://files.pythonhosted.org/packages/c4/1c/8a10c1c25b23156e63b12165a929d8eb49a6ed769fdbefb06e6f07c1e50d/multidict-6.6.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae9408439537c5afdca05edd128a63f56a62680f4b3c234301055d7a2000220f", size = 257076, upload-time = "2025-08-11T12:07:25.049Z" }, - { url = "https://files.pythonhosted.org/packages/ad/86/90e20b5771d6805a119e483fd3d1e8393e745a11511aebca41f0da38c3e2/multidict-6.6.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:87a32d20759dc52a9e850fe1061b6e41ab28e2998d44168a8a341b99ded1dba0", size = 240694, upload-time = "2025-08-11T12:07:26.458Z" }, - { url = "https://files.pythonhosted.org/packages/e7/49/484d3e6b535bc0555b52a0a26ba86e4d8d03fd5587d4936dc59ba7583221/multidict-6.6.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:52e3c8d43cdfff587ceedce9deb25e6ae77daba560b626e97a56ddcad3756879", size = 266350, upload-time = "2025-08-11T12:07:27.94Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b4/aa4c5c379b11895083d50021e229e90c408d7d875471cb3abf721e4670d6/multidict-6.6.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ad8850921d3a8d8ff6fbef790e773cecfc260bbfa0566998980d3fa8f520bc4a", size = 267250, upload-time = "2025-08-11T12:07:29.303Z" }, - { url = "https://files.pythonhosted.org/packages/80/e5/5e22c5bf96a64bdd43518b1834c6d95a4922cc2066b7d8e467dae9b6cee6/multidict-6.6.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:497a2954adc25c08daff36f795077f63ad33e13f19bfff7736e72c785391534f", size = 254900, upload-time = "2025-08-11T12:07:30.764Z" }, - { url = "https://files.pythonhosted.org/packages/17/38/58b27fed927c07035abc02befacab42491e7388ca105e087e6e0215ead64/multidict-6.6.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:024ce601f92d780ca1617ad4be5ac15b501cc2414970ffa2bb2bbc2bd5a68fa5", size = 252355, upload-time = "2025-08-11T12:07:32.205Z" }, - { url = "https://files.pythonhosted.org/packages/d0/a1/dad75d23a90c29c02b5d6f3d7c10ab36c3197613be5d07ec49c7791e186c/multidict-6.6.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a693fc5ed9bdd1c9e898013e0da4dcc640de7963a371c0bd458e50e046bf6438", size = 250061, upload-time = "2025-08-11T12:07:33.623Z" }, - { url = "https://files.pythonhosted.org/packages/b8/1a/ac2216b61c7f116edab6dc3378cca6c70dc019c9a457ff0d754067c58b20/multidict-6.6.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:190766dac95aab54cae5b152a56520fd99298f32a1266d66d27fdd1b5ac00f4e", size = 249675, upload-time = "2025-08-11T12:07:34.958Z" }, - { url = "https://files.pythonhosted.org/packages/d4/79/1916af833b800d13883e452e8e0977c065c4ee3ab7a26941fbfdebc11895/multidict-6.6.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:34d8f2a5ffdceab9dcd97c7a016deb2308531d5f0fced2bb0c9e1df45b3363d7", size = 261247, upload-time = "2025-08-11T12:07:36.588Z" }, - { url = "https://files.pythonhosted.org/packages/c5/65/d1f84fe08ac44a5fc7391cbc20a7cedc433ea616b266284413fd86062f8c/multidict-6.6.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:59e8d40ab1f5a8597abcef00d04845155a5693b5da00d2c93dbe88f2050f2812", size = 257960, upload-time = "2025-08-11T12:07:39.735Z" }, - { url = "https://files.pythonhosted.org/packages/13/b5/29ec78057d377b195ac2c5248c773703a6b602e132a763e20ec0457e7440/multidict-6.6.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:467fe64138cfac771f0e949b938c2e1ada2b5af22f39692aa9258715e9ea613a", size = 250078, upload-time = "2025-08-11T12:07:41.525Z" }, - { url = "https://files.pythonhosted.org/packages/c4/0e/7e79d38f70a872cae32e29b0d77024bef7834b0afb406ddae6558d9e2414/multidict-6.6.4-cp313-cp313-win32.whl", hash = "sha256:14616a30fe6d0a48d0a48d1a633ab3b8bec4cf293aac65f32ed116f620adfd69", size = 41708, upload-time = "2025-08-11T12:07:43.405Z" }, - { url = "https://files.pythonhosted.org/packages/9d/34/746696dffff742e97cd6a23da953e55d0ea51fa601fa2ff387b3edcfaa2c/multidict-6.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:40cd05eaeb39e2bc8939451f033e57feaa2ac99e07dbca8afe2be450a4a3b6cf", size = 45912, upload-time = "2025-08-11T12:07:45.082Z" }, - { url = "https://files.pythonhosted.org/packages/c7/87/3bac136181e271e29170d8d71929cdeddeb77f3e8b6a0c08da3a8e9da114/multidict-6.6.4-cp313-cp313-win_arm64.whl", hash = "sha256:f6eb37d511bfae9e13e82cb4d1af36b91150466f24d9b2b8a9785816deb16605", size = 43076, upload-time = "2025-08-11T12:07:46.746Z" }, - { url = "https://files.pythonhosted.org/packages/64/94/0a8e63e36c049b571c9ae41ee301ada29c3fee9643d9c2548d7d558a1d99/multidict-6.6.4-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:6c84378acd4f37d1b507dfa0d459b449e2321b3ba5f2338f9b085cf7a7ba95eb", size = 82812, upload-time = "2025-08-11T12:07:48.402Z" }, - { url = "https://files.pythonhosted.org/packages/25/1a/be8e369dfcd260d2070a67e65dd3990dd635cbd735b98da31e00ea84cd4e/multidict-6.6.4-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0e0558693063c75f3d952abf645c78f3c5dfdd825a41d8c4d8156fc0b0da6e7e", size = 48313, upload-time = "2025-08-11T12:07:49.679Z" }, - { url = "https://files.pythonhosted.org/packages/26/5a/dd4ade298674b2f9a7b06a32c94ffbc0497354df8285f27317c66433ce3b/multidict-6.6.4-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3f8e2384cb83ebd23fd07e9eada8ba64afc4c759cd94817433ab8c81ee4b403f", size = 46777, upload-time = "2025-08-11T12:07:51.318Z" }, - { url = "https://files.pythonhosted.org/packages/89/db/98aa28bc7e071bfba611ac2ae803c24e96dd3a452b4118c587d3d872c64c/multidict-6.6.4-cp313-cp313t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f996b87b420995a9174b2a7c1a8daf7db4750be6848b03eb5e639674f7963773", size = 229321, upload-time = "2025-08-11T12:07:52.965Z" }, - { url = "https://files.pythonhosted.org/packages/c7/bc/01ddda2a73dd9d167bd85d0e8ef4293836a8f82b786c63fb1a429bc3e678/multidict-6.6.4-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc356250cffd6e78416cf5b40dc6a74f1edf3be8e834cf8862d9ed5265cf9b0e", size = 249954, upload-time = "2025-08-11T12:07:54.423Z" }, - { url = "https://files.pythonhosted.org/packages/06/78/6b7c0f020f9aa0acf66d0ab4eb9f08375bac9a50ff5e3edb1c4ccd59eafc/multidict-6.6.4-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dadf95aa862714ea468a49ad1e09fe00fcc9ec67d122f6596a8d40caf6cec7d0", size = 228612, upload-time = "2025-08-11T12:07:55.914Z" }, - { url = "https://files.pythonhosted.org/packages/00/44/3faa416f89b2d5d76e9d447296a81521e1c832ad6e40b92f990697b43192/multidict-6.6.4-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dd57515bebffd8ebd714d101d4c434063322e4fe24042e90ced41f18b6d3395", size = 257528, upload-time = "2025-08-11T12:07:57.371Z" }, - { url = "https://files.pythonhosted.org/packages/05/5f/77c03b89af0fcb16f018f668207768191fb9dcfb5e3361a5e706a11db2c9/multidict-6.6.4-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:967af5f238ebc2eb1da4e77af5492219fbd9b4b812347da39a7b5f5c72c0fa45", size = 256329, upload-time = "2025-08-11T12:07:58.844Z" }, - { url = "https://files.pythonhosted.org/packages/cf/e9/ed750a2a9afb4f8dc6f13dc5b67b514832101b95714f1211cd42e0aafc26/multidict-6.6.4-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2a4c6875c37aae9794308ec43e3530e4aa0d36579ce38d89979bbf89582002bb", size = 247928, upload-time = "2025-08-11T12:08:01.037Z" }, - { url = "https://files.pythonhosted.org/packages/1f/b5/e0571bc13cda277db7e6e8a532791d4403dacc9850006cb66d2556e649c0/multidict-6.6.4-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:7f683a551e92bdb7fac545b9c6f9fa2aebdeefa61d607510b3533286fcab67f5", size = 245228, upload-time = "2025-08-11T12:08:02.96Z" }, - { url = "https://files.pythonhosted.org/packages/f3/a3/69a84b0eccb9824491f06368f5b86e72e4af54c3067c37c39099b6687109/multidict-6.6.4-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:3ba5aaf600edaf2a868a391779f7a85d93bed147854925f34edd24cc70a3e141", size = 235869, upload-time = "2025-08-11T12:08:04.746Z" }, - { url = "https://files.pythonhosted.org/packages/a9/9d/28802e8f9121a6a0804fa009debf4e753d0a59969ea9f70be5f5fdfcb18f/multidict-6.6.4-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:580b643b7fd2c295d83cad90d78419081f53fd532d1f1eb67ceb7060f61cff0d", size = 243446, upload-time = "2025-08-11T12:08:06.332Z" }, - { url = "https://files.pythonhosted.org/packages/38/ea/6c98add069b4878c1d66428a5f5149ddb6d32b1f9836a826ac764b9940be/multidict-6.6.4-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:37b7187197da6af3ee0b044dbc9625afd0c885f2800815b228a0e70f9a7f473d", size = 252299, upload-time = "2025-08-11T12:08:07.931Z" }, - { url = "https://files.pythonhosted.org/packages/3a/09/8fe02d204473e14c0af3affd50af9078839dfca1742f025cca765435d6b4/multidict-6.6.4-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:e1b93790ed0bc26feb72e2f08299691ceb6da5e9e14a0d13cc74f1869af327a0", size = 246926, upload-time = "2025-08-11T12:08:09.467Z" }, - { url = "https://files.pythonhosted.org/packages/37/3d/7b1e10d774a6df5175ecd3c92bff069e77bed9ec2a927fdd4ff5fe182f67/multidict-6.6.4-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a506a77ddee1efcca81ecbeae27ade3e09cdf21a8ae854d766c2bb4f14053f92", size = 243383, upload-time = "2025-08-11T12:08:10.981Z" }, - { url = "https://files.pythonhosted.org/packages/50/b0/a6fae46071b645ae98786ab738447de1ef53742eaad949f27e960864bb49/multidict-6.6.4-cp313-cp313t-win32.whl", hash = "sha256:f93b2b2279883d1d0a9e1bd01f312d6fc315c5e4c1f09e112e4736e2f650bc4e", size = 47775, upload-time = "2025-08-11T12:08:12.439Z" }, - { url = "https://files.pythonhosted.org/packages/b2/0a/2436550b1520091af0600dff547913cb2d66fbac27a8c33bc1b1bccd8d98/multidict-6.6.4-cp313-cp313t-win_amd64.whl", hash = "sha256:6d46a180acdf6e87cc41dc15d8f5c2986e1e8739dc25dbb7dac826731ef381a4", size = 53100, upload-time = "2025-08-11T12:08:13.823Z" }, - { url = "https://files.pythonhosted.org/packages/97/ea/43ac51faff934086db9c072a94d327d71b7d8b40cd5dcb47311330929ef0/multidict-6.6.4-cp313-cp313t-win_arm64.whl", hash = "sha256:756989334015e3335d087a27331659820d53ba432befdef6a718398b0a8493ad", size = 45501, upload-time = "2025-08-11T12:08:15.173Z" }, - { url = "https://files.pythonhosted.org/packages/fd/69/b547032297c7e63ba2af494edba695d781af8a0c6e89e4d06cf848b21d80/multidict-6.6.4-py3-none-any.whl", hash = "sha256:27d8f8e125c07cb954e54d75d04905a9bba8a439c1d84aca94949d4d03d8601c", size = 12313, upload-time = "2025-08-11T12:08:46.891Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/0b/19348d4c98980c4851d2f943f8ebafdece2ae7ef737adcfa5994ce8e5f10/multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5", size = 77176, upload-time = "2026-01-26T02:42:59.784Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/9de3f8077852e3d438215c81e9b691244532d2e05b4270e89ce67b7d103c/multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8", size = 44996, upload-time = "2026-01-26T02:43:01.674Z" }, + { url = "https://files.pythonhosted.org/packages/31/5c/08c7f7fe311f32e83f7621cd3f99d805f45519cd06fafb247628b861da7d/multidict-6.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdea2e7b2456cfb6694fb113066fd0ec7ea4d67e3a35e1f4cbeea0b448bf5872", size = 44631, upload-time = "2026-01-26T02:43:03.169Z" }, + { url = "https://files.pythonhosted.org/packages/b7/7f/0e3b1390ae772f27501199996b94b52ceeb64fe6f9120a32c6c3f6b781be/multidict-6.7.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17207077e29342fdc2c9a82e4b306f1127bf1ea91f8b71e02d4798a70bb99991", size = 242561, upload-time = "2026-01-26T02:43:04.733Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/8719f4f167586af317b69dd3e90f913416c91ca610cac79a45c53f590312/multidict-6.7.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4f49cb5661344764e4c7c7973e92a47a59b8fc19b6523649ec9dc4960e58a03", size = 242223, upload-time = "2026-01-26T02:43:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/47/ab/7c36164cce64a6ad19c6d9a85377b7178ecf3b89f8fd589c73381a5eedfd/multidict-6.7.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a9fc4caa29e2e6ae408d1c450ac8bf19892c5fca83ee634ecd88a53332c59981", size = 222322, upload-time = "2026-01-26T02:43:08.472Z" }, + { url = "https://files.pythonhosted.org/packages/f5/79/a25add6fb38035b5337bc5734f296d9afc99163403bbcf56d4170f97eb62/multidict-6.7.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c5f0c21549ab432b57dcc82130f388d84ad8179824cc3f223d5e7cfbfd4143f6", size = 254005, upload-time = "2026-01-26T02:43:10.127Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7b/64a87cf98e12f756fc8bd444b001232ffff2be37288f018ad0d3f0aae931/multidict-6.7.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7dfb78d966b2c906ae1d28ccf6e6712a3cd04407ee5088cd276fe8cb42186190", size = 251173, upload-time = "2026-01-26T02:43:11.731Z" }, + { url = "https://files.pythonhosted.org/packages/4b/ac/b605473de2bb404e742f2cc3583d12aedb2352a70e49ae8fce455b50c5aa/multidict-6.7.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9b0d9b91d1aa44db9c1f1ecd0d9d2ae610b2f4f856448664e01a3b35899f3f92", size = 243273, upload-time = "2026-01-26T02:43:13.063Z" }, + { url = "https://files.pythonhosted.org/packages/03/65/11492d6a0e259783720f3bc1d9ea55579a76f1407e31ed44045c99542004/multidict-6.7.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:dd96c01a9dcd4889dcfcf9eb5544ca0c77603f239e3ffab0524ec17aea9a93ee", size = 238956, upload-time = "2026-01-26T02:43:14.843Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a7/7ee591302af64e7c196fb63fe856c788993c1372df765102bd0448e7e165/multidict-6.7.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:067343c68cd6612d375710f895337b3a98a033c94f14b9a99eff902f205424e2", size = 233477, upload-time = "2026-01-26T02:43:16.025Z" }, + { url = "https://files.pythonhosted.org/packages/9c/99/c109962d58756c35fd9992fed7f2355303846ea2ff054bb5f5e9d6b888de/multidict-6.7.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5884a04f4ff56c6120f6ccf703bdeb8b5079d808ba604d4d53aec0d55dc33568", size = 243615, upload-time = "2026-01-26T02:43:17.84Z" }, + { url = "https://files.pythonhosted.org/packages/d5/5f/1973e7c771c86e93dcfe1c9cc55a5481b610f6614acfc28c0d326fe6bfad/multidict-6.7.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8affcf1c98b82bc901702eb73b6947a1bfa170823c153fe8a47b5f5f02e48e40", size = 249930, upload-time = "2026-01-26T02:43:19.06Z" }, + { url = "https://files.pythonhosted.org/packages/5d/a5/f170fc2268c3243853580203378cd522446b2df632061e0a5409817854c7/multidict-6.7.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:0d17522c37d03e85c8098ec8431636309b2682cf12e58f4dbc76121fb50e4962", size = 243807, upload-time = "2026-01-26T02:43:20.286Z" }, + { url = "https://files.pythonhosted.org/packages/de/01/73856fab6d125e5bc652c3986b90e8699a95e84b48d72f39ade6c0e74a8c/multidict-6.7.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24c0cf81544ca5e17cfcb6e482e7a82cd475925242b308b890c9452a074d4505", size = 239103, upload-time = "2026-01-26T02:43:21.508Z" }, + { url = "https://files.pythonhosted.org/packages/e7/46/f1220bd9944d8aa40d8ccff100eeeee19b505b857b6f603d6078cb5315b0/multidict-6.7.1-cp310-cp310-win32.whl", hash = "sha256:d82dd730a95e6643802f4454b8fdecdf08667881a9c5670db85bc5a56693f122", size = 41416, upload-time = "2026-01-26T02:43:22.703Z" }, + { url = "https://files.pythonhosted.org/packages/68/00/9b38e272a770303692fc406c36e1a4c740f401522d5787691eb38a8925a8/multidict-6.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:cf37cbe5ced48d417ba045aca1b21bafca67489452debcde94778a576666a1df", size = 46022, upload-time = "2026-01-26T02:43:23.77Z" }, + { url = "https://files.pythonhosted.org/packages/64/65/d8d42490c02ee07b6bbe00f7190d70bb4738b3cce7629aaf9f213ef730dd/multidict-6.7.1-cp310-cp310-win_arm64.whl", hash = "sha256:59bc83d3f66b41dac1e7460aac1d196edc70c9ba3094965c467715a70ecb46db", size = 43238, upload-time = "2026-01-26T02:43:24.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626, upload-time = "2026-01-26T02:43:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706, upload-time = "2026-01-26T02:43:27.607Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356, upload-time = "2026-01-26T02:43:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355, upload-time = "2026-01-26T02:43:31.165Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433, upload-time = "2026-01-26T02:43:32.581Z" }, + { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376, upload-time = "2026-01-26T02:43:34.417Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365, upload-time = "2026-01-26T02:43:35.741Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747, upload-time = "2026-01-26T02:43:36.976Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293, upload-time = "2026-01-26T02:43:38.258Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962, upload-time = "2026-01-26T02:43:40.034Z" }, + { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360, upload-time = "2026-01-26T02:43:41.752Z" }, + { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940, upload-time = "2026-01-26T02:43:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502, upload-time = "2026-01-26T02:43:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065, upload-time = "2026-01-26T02:43:45.745Z" }, + { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870, upload-time = "2026-01-26T02:43:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302, upload-time = "2026-01-26T02:43:48.753Z" }, + { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981, upload-time = "2026-01-26T02:43:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159, upload-time = "2026-01-26T02:43:51.635Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] [[package]] name = "multiprocess" -version = "0.70.16" +version = "0.70.18" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dill" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/ae/04f39c5d0d0def03247c2893d6f2b83c136bf3320a2154d7b8858f2ba72d/multiprocess-0.70.16.tar.gz", hash = "sha256:161af703d4652a0e1410be6abccecde4a7ddffd19341be0a7011b94aeb171ac1", size = 1772603, upload-time = "2024-01-28T18:52:34.85Z" } +sdist = { url = "https://files.pythonhosted.org/packages/72/fd/2ae3826f5be24c6ed87266bc4e59c46ea5b059a103f3d7e7eb76a52aeecb/multiprocess-0.70.18.tar.gz", hash = "sha256:f9597128e6b3e67b23956da07cf3d2e5cba79e2f4e0fba8d7903636663ec6d0d", size = 1798503, upload-time = "2025-04-17T03:11:27.742Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ef/76/6e712a2623d146d314f17598df5de7224c85c0060ef63fd95cc15a25b3fa/multiprocess-0.70.16-pp310-pypy310_pp73-macosx_10_13_x86_64.whl", hash = "sha256:476887be10e2f59ff183c006af746cb6f1fd0eadcfd4ef49e605cbe2659920ee", size = 134980, upload-time = "2024-01-28T18:52:15.731Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ab/1e6e8009e380e22254ff539ebe117861e5bdb3bff1fc977920972237c6c7/multiprocess-0.70.16-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:d951bed82c8f73929ac82c61f01a7b5ce8f3e5ef40f5b52553b4f547ce2b08ec", size = 134982, upload-time = "2024-01-28T18:52:17.783Z" }, - { url = "https://files.pythonhosted.org/packages/bc/f7/7ec7fddc92e50714ea3745631f79bd9c96424cb2702632521028e57d3a36/multiprocess-0.70.16-py310-none-any.whl", hash = "sha256:c4a9944c67bd49f823687463660a2d6daae94c289adff97e0f9d696ba6371d02", size = 134824, upload-time = "2024-01-28T18:52:26.062Z" }, - { url = "https://files.pythonhosted.org/packages/50/15/b56e50e8debaf439f44befec5b2af11db85f6e0f344c3113ae0be0593a91/multiprocess-0.70.16-py311-none-any.whl", hash = "sha256:af4cabb0dac72abfb1e794fa7855c325fd2b55a10a44628a3c1ad3311c04127a", size = 143519, upload-time = "2024-01-28T18:52:28.115Z" }, - { url = "https://files.pythonhosted.org/packages/0a/7d/a988f258104dcd2ccf1ed40fdc97e26c4ac351eeaf81d76e266c52d84e2f/multiprocess-0.70.16-py312-none-any.whl", hash = "sha256:fc0544c531920dde3b00c29863377f87e1632601092ea2daca74e4beb40faa2e", size = 146741, upload-time = "2024-01-28T18:52:29.395Z" }, - { url = "https://files.pythonhosted.org/packages/ea/89/38df130f2c799090c978b366cfdf5b96d08de5b29a4a293df7f7429fa50b/multiprocess-0.70.16-py38-none-any.whl", hash = "sha256:a71d82033454891091a226dfc319d0cfa8019a4e888ef9ca910372a446de4435", size = 132628, upload-time = "2024-01-28T18:52:30.853Z" }, - { url = "https://files.pythonhosted.org/packages/da/d9/f7f9379981e39b8c2511c9e0326d212accacb82f12fbfdc1aa2ce2a7b2b6/multiprocess-0.70.16-py39-none-any.whl", hash = "sha256:a0bafd3ae1b732eac64be2e72038231c1ba97724b60b09400d68f229fcc2fbf3", size = 133351, upload-time = "2024-01-28T18:52:31.981Z" }, -] - -[[package]] -name = "namex" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0c/c0/ee95b28f029c73f8d49d8f52edaed02a1d4a9acb8b69355737fdb1faa191/namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306", size = 6649, upload-time = "2025-05-26T23:17:38.918Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/bc/465daf1de06409cdd4532082806770ee0d8d7df434da79c76564d0f69741/namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c", size = 5905, upload-time = "2025-05-26T23:17:37.695Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f8/7f9a8f08bf98cea1dfaa181e05cc8bbcb59cecf044b5a9ac3cce39f9c449/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:25d4012dcaaf66b9e8e955f58482b42910c2ee526d532844d8bcf661bbc604df", size = 135083, upload-time = "2025-04-17T03:11:04.223Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/b7b10dbfc17b2b3ce07d4d30b3ba8367d0ed32d6d46cd166e298f161dd46/multiprocess-0.70.18-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:06b19433de0d02afe5869aec8931dd5c01d99074664f806c73896b0d9e527213", size = 135128, upload-time = "2025-04-17T03:11:06.045Z" }, + { url = "https://files.pythonhosted.org/packages/c1/a3/5f8d3b9690ea5580bee5868ab7d7e2cfca74b7e826b28192b40aa3881cdc/multiprocess-0.70.18-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:6fa1366f994373aaf2d4738b0f56e707caeaa05486e97a7f71ee0853823180c2", size = 135132, upload-time = "2025-04-17T03:11:07.533Z" }, + { url = "https://files.pythonhosted.org/packages/55/4d/9af0d1279c84618bcd35bf5fd7e371657358c7b0a523e54a9cffb87461f8/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8b8940ae30139e04b076da6c5b83e9398585ebdf0f2ad3250673fef5b2ff06d6", size = 144695, upload-time = "2025-04-17T03:11:09.161Z" }, + { url = "https://files.pythonhosted.org/packages/17/bf/87323e79dd0562474fad3373c21c66bc6c3c9963b68eb2a209deb4c8575e/multiprocess-0.70.18-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0929ba95831adb938edbd5fb801ac45e705ecad9d100b3e653946b7716cb6bd3", size = 144742, upload-time = "2025-04-17T03:11:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/dd/74/cb8c831e58dc6d5cf450b17c7db87f14294a1df52eb391da948b5e0a0b94/multiprocess-0.70.18-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d77f8e4bfe6c6e2e661925bbf9aed4d5ade9a1c6502d5dfc10129b9d1141797", size = 144745, upload-time = "2025-04-17T03:11:11.453Z" }, + { url = "https://files.pythonhosted.org/packages/ba/d8/0cba6cf51a1a31f20471fbc823a716170c73012ddc4fb85d706630ed6e8f/multiprocess-0.70.18-py310-none-any.whl", hash = "sha256:60c194974c31784019c1f459d984e8f33ee48f10fcf42c309ba97b30d9bd53ea", size = 134948, upload-time = "2025-04-17T03:11:20.223Z" }, + { url = "https://files.pythonhosted.org/packages/4b/88/9039f2fed1012ef584751d4ceff9ab4a51e5ae264898f0b7cbf44340a859/multiprocess-0.70.18-py311-none-any.whl", hash = "sha256:5aa6eef98e691281b3ad923be2832bf1c55dd2c859acd73e5ec53a66aae06a1d", size = 144462, upload-time = "2025-04-17T03:11:21.657Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b6/5f922792be93b82ec6b5f270bbb1ef031fd0622847070bbcf9da816502cc/multiprocess-0.70.18-py312-none-any.whl", hash = "sha256:9b78f8e5024b573730bfb654783a13800c2c0f2dfc0c25e70b40d184d64adaa2", size = 150287, upload-time = "2025-04-17T03:11:22.69Z" }, + { url = "https://files.pythonhosted.org/packages/ee/25/7d7e78e750bc1aecfaf0efbf826c69a791d2eeaf29cf20cba93ff4cced78/multiprocess-0.70.18-py313-none-any.whl", hash = "sha256:871743755f43ef57d7910a38433cfe41319e72be1bbd90b79c7a5ac523eb9334", size = 151917, upload-time = "2025-04-17T03:11:24.044Z" }, + { url = "https://files.pythonhosted.org/packages/3b/c3/ca84c19bd14cdfc21c388fdcebf08b86a7a470ebc9f5c3c084fc2dbc50f7/multiprocess-0.70.18-py38-none-any.whl", hash = "sha256:dbf705e52a154fe5e90fb17b38f02556169557c2dd8bb084f2e06c2784d8279b", size = 132636, upload-time = "2025-04-17T03:11:24.936Z" }, + { url = "https://files.pythonhosted.org/packages/6c/28/dd72947e59a6a8c856448a5e74da6201cb5502ddff644fbc790e4bd40b9a/multiprocess-0.70.18-py39-none-any.whl", hash = "sha256:e78ca805a72b1b810c690b6b4cc32579eba34f403094bbbae962b7b5bf9dfcb8", size = 133478, upload-time = "2025-04-17T03:11:26.253Z" }, ] [[package]] @@ -1348,7 +1416,7 @@ wheels = [ [[package]] name = "networkx" -version = "3.5" +version = "3.6.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", @@ -1356,18 +1424,18 @@ resolution-markers = [ "python_full_version == '3.12.*'", "python_full_version == '3.11.*'", ] -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] [[package]] name = "nodeenv" -version = "1.9.1" +version = "1.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/16/fc88b08840de0e0a72a2f9d8c6bae36be573e475a6326ae854bcc549fc45/nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f", size = 47437, upload-time = "2024-06-04T18:44:11.171Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] [[package]] @@ -1437,89 +1505,140 @@ wheels = [ [[package]] name = "numpy" -version = "2.3.3" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, + { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", "python_full_version == '3.13.*'", "python_full_version == '3.12.*'", - "python_full_version == '3.11.*'", ] -sdist = { url = "https://files.pythonhosted.org/packages/d0/19/95b3d357407220ed24c139018d2518fab0a61a948e68286a25f1a4d049ff/numpy-2.3.3.tar.gz", hash = "sha256:ddc7c39727ba62b80dfdbedf400d1c10ddfa8eefbd7ec8dcb118be8b56d31029", size = 20576648, upload-time = "2025-09-09T16:54:12.543Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/45/e80d203ef6b267aa29b22714fb558930b27960a0c5ce3c19c999232bb3eb/numpy-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ffc4f5caba7dfcbe944ed674b7eef683c7e94874046454bb79ed7ee0236f59d", size = 21259253, upload-time = "2025-09-09T15:56:02.094Z" }, - { url = "https://files.pythonhosted.org/packages/52/18/cf2c648fccf339e59302e00e5f2bc87725a3ce1992f30f3f78c9044d7c43/numpy-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7e946c7170858a0295f79a60214424caac2ffdb0063d4d79cb681f9aa0aa569", size = 14450980, upload-time = "2025-09-09T15:56:05.926Z" }, - { url = "https://files.pythonhosted.org/packages/93/fb/9af1082bec870188c42a1c239839915b74a5099c392389ff04215dcee812/numpy-2.3.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cd4260f64bc794c3390a63bf0728220dd1a68170c169088a1e0dfa2fde1be12f", size = 5379709, upload-time = "2025-09-09T15:56:07.95Z" }, - { url = "https://files.pythonhosted.org/packages/75/0f/bfd7abca52bcbf9a4a65abc83fe18ef01ccdeb37bfb28bbd6ad613447c79/numpy-2.3.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:f0ddb4b96a87b6728df9362135e764eac3cfa674499943ebc44ce96c478ab125", size = 6913923, upload-time = "2025-09-09T15:56:09.443Z" }, - { url = "https://files.pythonhosted.org/packages/79/55/d69adad255e87ab7afda1caf93ca997859092afeb697703e2f010f7c2e55/numpy-2.3.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afd07d377f478344ec6ca2b8d4ca08ae8bd44706763d1efb56397de606393f48", size = 14589591, upload-time = "2025-09-09T15:56:11.234Z" }, - { url = "https://files.pythonhosted.org/packages/10/a2/010b0e27ddeacab7839957d7a8f00e91206e0c2c47abbb5f35a2630e5387/numpy-2.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bc92a5dedcc53857249ca51ef29f5e5f2f8c513e22cfb90faeb20343b8c6f7a6", size = 16938714, upload-time = "2025-09-09T15:56:14.637Z" }, - { url = "https://files.pythonhosted.org/packages/1c/6b/12ce8ede632c7126eb2762b9e15e18e204b81725b81f35176eac14dc5b82/numpy-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7af05ed4dc19f308e1d9fc759f36f21921eb7bbfc82843eeec6b2a2863a0aefa", size = 16370592, upload-time = "2025-09-09T15:56:17.285Z" }, - { url = "https://files.pythonhosted.org/packages/b4/35/aba8568b2593067bb6a8fe4c52babb23b4c3b9c80e1b49dff03a09925e4a/numpy-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:433bf137e338677cebdd5beac0199ac84712ad9d630b74eceeb759eaa45ddf30", size = 18884474, upload-time = "2025-09-09T15:56:20.943Z" }, - { url = "https://files.pythonhosted.org/packages/45/fa/7f43ba10c77575e8be7b0138d107e4f44ca4a1ef322cd16980ea3e8b8222/numpy-2.3.3-cp311-cp311-win32.whl", hash = "sha256:eb63d443d7b4ffd1e873f8155260d7f58e7e4b095961b01c91062935c2491e57", size = 6599794, upload-time = "2025-09-09T15:56:23.258Z" }, - { url = "https://files.pythonhosted.org/packages/0a/a2/a4f78cb2241fe5664a22a10332f2be886dcdea8784c9f6a01c272da9b426/numpy-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:ec9d249840f6a565f58d8f913bccac2444235025bbb13e9a4681783572ee3caa", size = 13088104, upload-time = "2025-09-09T15:56:25.476Z" }, - { url = "https://files.pythonhosted.org/packages/79/64/e424e975adbd38282ebcd4891661965b78783de893b381cbc4832fb9beb2/numpy-2.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:74c2a948d02f88c11a3c075d9733f1ae67d97c6bdb97f2bb542f980458b257e7", size = 10460772, upload-time = "2025-09-09T15:56:27.679Z" }, - { url = "https://files.pythonhosted.org/packages/51/5d/bb7fc075b762c96329147799e1bcc9176ab07ca6375ea976c475482ad5b3/numpy-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cfdd09f9c84a1a934cde1eec2267f0a43a7cd44b2cca4ff95b7c0d14d144b0bf", size = 20957014, upload-time = "2025-09-09T15:56:29.966Z" }, - { url = "https://files.pythonhosted.org/packages/6b/0e/c6211bb92af26517acd52125a237a92afe9c3124c6a68d3b9f81b62a0568/numpy-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cb32e3cf0f762aee47ad1ddc6672988f7f27045b0783c887190545baba73aa25", size = 14185220, upload-time = "2025-09-09T15:56:32.175Z" }, - { url = "https://files.pythonhosted.org/packages/22/f2/07bb754eb2ede9073f4054f7c0286b0d9d2e23982e090a80d478b26d35ca/numpy-2.3.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:396b254daeb0a57b1fe0ecb5e3cff6fa79a380fa97c8f7781a6d08cd429418fe", size = 5113918, upload-time = "2025-09-09T15:56:34.175Z" }, - { url = "https://files.pythonhosted.org/packages/81/0a/afa51697e9fb74642f231ea36aca80fa17c8fb89f7a82abd5174023c3960/numpy-2.3.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:067e3d7159a5d8f8a0b46ee11148fc35ca9b21f61e3c49fbd0a027450e65a33b", size = 6647922, upload-time = "2025-09-09T15:56:36.149Z" }, - { url = "https://files.pythonhosted.org/packages/5d/f5/122d9cdb3f51c520d150fef6e87df9279e33d19a9611a87c0d2cf78a89f4/numpy-2.3.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c02d0629d25d426585fb2e45a66154081b9fa677bc92a881ff1d216bc9919a8", size = 14281991, upload-time = "2025-09-09T15:56:40.548Z" }, - { url = "https://files.pythonhosted.org/packages/51/64/7de3c91e821a2debf77c92962ea3fe6ac2bc45d0778c1cbe15d4fce2fd94/numpy-2.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9192da52b9745f7f0766531dcfa978b7763916f158bb63bdb8a1eca0068ab20", size = 16641643, upload-time = "2025-09-09T15:56:43.343Z" }, - { url = "https://files.pythonhosted.org/packages/30/e4/961a5fa681502cd0d68907818b69f67542695b74e3ceaa513918103b7e80/numpy-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cd7de500a5b66319db419dc3c345244404a164beae0d0937283b907d8152e6ea", size = 16056787, upload-time = "2025-09-09T15:56:46.141Z" }, - { url = "https://files.pythonhosted.org/packages/99/26/92c912b966e47fbbdf2ad556cb17e3a3088e2e1292b9833be1dfa5361a1a/numpy-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:93d4962d8f82af58f0b2eb85daaf1b3ca23fe0a85d0be8f1f2b7bb46034e56d7", size = 18579598, upload-time = "2025-09-09T15:56:49.844Z" }, - { url = "https://files.pythonhosted.org/packages/17/b6/fc8f82cb3520768718834f310c37d96380d9dc61bfdaf05fe5c0b7653e01/numpy-2.3.3-cp312-cp312-win32.whl", hash = "sha256:5534ed6b92f9b7dca6c0a19d6df12d41c68b991cef051d108f6dbff3babc4ebf", size = 6320800, upload-time = "2025-09-09T15:56:52.499Z" }, - { url = "https://files.pythonhosted.org/packages/32/ee/de999f2625b80d043d6d2d628c07d0d5555a677a3cf78fdf868d409b8766/numpy-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:497d7cad08e7092dba36e3d296fe4c97708c93daf26643a1ae4b03f6294d30eb", size = 12786615, upload-time = "2025-09-09T15:56:54.422Z" }, - { url = "https://files.pythonhosted.org/packages/49/6e/b479032f8a43559c383acb20816644f5f91c88f633d9271ee84f3b3a996c/numpy-2.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:ca0309a18d4dfea6fc6262a66d06c26cfe4640c3926ceec90e57791a82b6eee5", size = 10195936, upload-time = "2025-09-09T15:56:56.541Z" }, - { url = "https://files.pythonhosted.org/packages/7d/b9/984c2b1ee61a8b803bf63582b4ac4242cf76e2dbd663efeafcb620cc0ccb/numpy-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f5415fb78995644253370985342cd03572ef8620b934da27d77377a2285955bf", size = 20949588, upload-time = "2025-09-09T15:56:59.087Z" }, - { url = "https://files.pythonhosted.org/packages/a6/e4/07970e3bed0b1384d22af1e9912527ecbeb47d3b26e9b6a3bced068b3bea/numpy-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d00de139a3324e26ed5b95870ce63be7ec7352171bc69a4cf1f157a48e3eb6b7", size = 14177802, upload-time = "2025-09-09T15:57:01.73Z" }, - { url = "https://files.pythonhosted.org/packages/35/c7/477a83887f9de61f1203bad89cf208b7c19cc9fef0cebef65d5a1a0619f2/numpy-2.3.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:9dc13c6a5829610cc07422bc74d3ac083bd8323f14e2827d992f9e52e22cd6a6", size = 5106537, upload-time = "2025-09-09T15:57:03.765Z" }, - { url = "https://files.pythonhosted.org/packages/52/47/93b953bd5866a6f6986344d045a207d3f1cfbad99db29f534ea9cee5108c/numpy-2.3.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:d79715d95f1894771eb4e60fb23f065663b2298f7d22945d66877aadf33d00c7", size = 6640743, upload-time = "2025-09-09T15:57:07.921Z" }, - { url = "https://files.pythonhosted.org/packages/23/83/377f84aaeb800b64c0ef4de58b08769e782edcefa4fea712910b6f0afd3c/numpy-2.3.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:952cfd0748514ea7c3afc729a0fc639e61655ce4c55ab9acfab14bda4f402b4c", size = 14278881, upload-time = "2025-09-09T15:57:11.349Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a5/bf3db6e66c4b160d6ea10b534c381a1955dfab34cb1017ea93aa33c70ed3/numpy-2.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b83648633d46f77039c29078751f80da65aa64d5622a3cd62aaef9d835b6c93", size = 16636301, upload-time = "2025-09-09T15:57:14.245Z" }, - { url = "https://files.pythonhosted.org/packages/a2/59/1287924242eb4fa3f9b3a2c30400f2e17eb2707020d1c5e3086fe7330717/numpy-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b001bae8cea1c7dfdb2ae2b017ed0a6f2102d7a70059df1e338e307a4c78a8ae", size = 16053645, upload-time = "2025-09-09T15:57:16.534Z" }, - { url = "https://files.pythonhosted.org/packages/e6/93/b3d47ed882027c35e94ac2320c37e452a549f582a5e801f2d34b56973c97/numpy-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8e9aced64054739037d42fb84c54dd38b81ee238816c948c8f3ed134665dcd86", size = 18578179, upload-time = "2025-09-09T15:57:18.883Z" }, - { url = "https://files.pythonhosted.org/packages/20/d9/487a2bccbf7cc9d4bfc5f0f197761a5ef27ba870f1e3bbb9afc4bbe3fcc2/numpy-2.3.3-cp313-cp313-win32.whl", hash = "sha256:9591e1221db3f37751e6442850429b3aabf7026d3b05542d102944ca7f00c8a8", size = 6312250, upload-time = "2025-09-09T15:57:21.296Z" }, - { url = "https://files.pythonhosted.org/packages/1b/b5/263ebbbbcede85028f30047eab3d58028d7ebe389d6493fc95ae66c636ab/numpy-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f0dadeb302887f07431910f67a14d57209ed91130be0adea2f9793f1a4f817cf", size = 12783269, upload-time = "2025-09-09T15:57:23.034Z" }, - { url = "https://files.pythonhosted.org/packages/fa/75/67b8ca554bbeaaeb3fac2e8bce46967a5a06544c9108ec0cf5cece559b6c/numpy-2.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:3c7cf302ac6e0b76a64c4aecf1a09e51abd9b01fc7feee80f6c43e3ab1b1dbc5", size = 10195314, upload-time = "2025-09-09T15:57:25.045Z" }, - { url = "https://files.pythonhosted.org/packages/11/d0/0d1ddec56b162042ddfafeeb293bac672de9b0cfd688383590090963720a/numpy-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:eda59e44957d272846bb407aad19f89dc6f58fecf3504bd144f4c5cf81a7eacc", size = 21048025, upload-time = "2025-09-09T15:57:27.257Z" }, - { url = "https://files.pythonhosted.org/packages/36/9e/1996ca6b6d00415b6acbdd3c42f7f03ea256e2c3f158f80bd7436a8a19f3/numpy-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:823d04112bc85ef5c4fda73ba24e6096c8f869931405a80aa8b0e604510a26bc", size = 14301053, upload-time = "2025-09-09T15:57:30.077Z" }, - { url = "https://files.pythonhosted.org/packages/05/24/43da09aa764c68694b76e84b3d3f0c44cb7c18cdc1ba80e48b0ac1d2cd39/numpy-2.3.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:40051003e03db4041aa325da2a0971ba41cf65714e65d296397cc0e32de6018b", size = 5229444, upload-time = "2025-09-09T15:57:32.733Z" }, - { url = "https://files.pythonhosted.org/packages/bc/14/50ffb0f22f7218ef8af28dd089f79f68289a7a05a208db9a2c5dcbe123c1/numpy-2.3.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:6ee9086235dd6ab7ae75aba5662f582a81ced49f0f1c6de4260a78d8f2d91a19", size = 6738039, upload-time = "2025-09-09T15:57:34.328Z" }, - { url = "https://files.pythonhosted.org/packages/55/52/af46ac0795e09657d45a7f4db961917314377edecf66db0e39fa7ab5c3d3/numpy-2.3.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94fcaa68757c3e2e668ddadeaa86ab05499a70725811e582b6a9858dd472fb30", size = 14352314, upload-time = "2025-09-09T15:57:36.255Z" }, - { url = "https://files.pythonhosted.org/packages/a7/b1/dc226b4c90eb9f07a3fff95c2f0db3268e2e54e5cce97c4ac91518aee71b/numpy-2.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da1a74b90e7483d6ce5244053399a614b1d6b7bc30a60d2f570e5071f8959d3e", size = 16701722, upload-time = "2025-09-09T15:57:38.622Z" }, - { url = "https://files.pythonhosted.org/packages/9d/9d/9d8d358f2eb5eced14dba99f110d83b5cd9a4460895230f3b396ad19a323/numpy-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2990adf06d1ecee3b3dcbb4977dfab6e9f09807598d647f04d385d29e7a3c3d3", size = 16132755, upload-time = "2025-09-09T15:57:41.16Z" }, - { url = "https://files.pythonhosted.org/packages/b6/27/b3922660c45513f9377b3fb42240bec63f203c71416093476ec9aa0719dc/numpy-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ed635ff692483b8e3f0fcaa8e7eb8a75ee71aa6d975388224f70821421800cea", size = 18651560, upload-time = "2025-09-09T15:57:43.459Z" }, - { url = "https://files.pythonhosted.org/packages/5b/8e/3ab61a730bdbbc201bb245a71102aa609f0008b9ed15255500a99cd7f780/numpy-2.3.3-cp313-cp313t-win32.whl", hash = "sha256:a333b4ed33d8dc2b373cc955ca57babc00cd6f9009991d9edc5ddbc1bac36bcd", size = 6442776, upload-time = "2025-09-09T15:57:45.793Z" }, - { url = "https://files.pythonhosted.org/packages/1c/3a/e22b766b11f6030dc2decdeff5c2fb1610768055603f9f3be88b6d192fb2/numpy-2.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:4384a169c4d8f97195980815d6fcad04933a7e1ab3b530921c3fef7a1c63426d", size = 12927281, upload-time = "2025-09-09T15:57:47.492Z" }, - { url = "https://files.pythonhosted.org/packages/7b/42/c2e2bc48c5e9b2a83423f99733950fbefd86f165b468a3d85d52b30bf782/numpy-2.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:75370986cc0bc66f4ce5110ad35aae6d182cc4ce6433c40ad151f53690130bf1", size = 10265275, upload-time = "2025-09-09T15:57:49.647Z" }, - { url = "https://files.pythonhosted.org/packages/6b/01/342ad585ad82419b99bcf7cebe99e61da6bedb89e213c5fd71acc467faee/numpy-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cd052f1fa6a78dee696b58a914b7229ecfa41f0a6d96dc663c1220a55e137593", size = 20951527, upload-time = "2025-09-09T15:57:52.006Z" }, - { url = "https://files.pythonhosted.org/packages/ef/d8/204e0d73fc1b7a9ee80ab1fe1983dd33a4d64a4e30a05364b0208e9a241a/numpy-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:414a97499480067d305fcac9716c29cf4d0d76db6ebf0bf3cbce666677f12652", size = 14186159, upload-time = "2025-09-09T15:57:54.407Z" }, - { url = "https://files.pythonhosted.org/packages/22/af/f11c916d08f3a18fb8ba81ab72b5b74a6e42ead4c2846d270eb19845bf74/numpy-2.3.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:50a5fe69f135f88a2be9b6ca0481a68a136f6febe1916e4920e12f1a34e708a7", size = 5114624, upload-time = "2025-09-09T15:57:56.5Z" }, - { url = "https://files.pythonhosted.org/packages/fb/11/0ed919c8381ac9d2ffacd63fd1f0c34d27e99cab650f0eb6f110e6ae4858/numpy-2.3.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:b912f2ed2b67a129e6a601e9d93d4fa37bef67e54cac442a2f588a54afe5c67a", size = 6642627, upload-time = "2025-09-09T15:57:58.206Z" }, - { url = "https://files.pythonhosted.org/packages/ee/83/deb5f77cb0f7ba6cb52b91ed388b47f8f3c2e9930d4665c600408d9b90b9/numpy-2.3.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e318ee0596d76d4cb3d78535dc005fa60e5ea348cd131a51e99d0bdbe0b54fe", size = 14296926, upload-time = "2025-09-09T15:58:00.035Z" }, - { url = "https://files.pythonhosted.org/packages/77/cc/70e59dcb84f2b005d4f306310ff0a892518cc0c8000a33d0e6faf7ca8d80/numpy-2.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce020080e4a52426202bdb6f7691c65bb55e49f261f31a8f506c9f6bc7450421", size = 16638958, upload-time = "2025-09-09T15:58:02.738Z" }, - { url = "https://files.pythonhosted.org/packages/b6/5a/b2ab6c18b4257e099587d5b7f903317bd7115333ad8d4ec4874278eafa61/numpy-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6687dc183aa55dae4a705b35f9c0f8cb178bcaa2f029b241ac5356221d5c021", size = 16071920, upload-time = "2025-09-09T15:58:05.029Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f1/8b3fdc44324a259298520dd82147ff648979bed085feeacc1250ef1656c0/numpy-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d8f3b1080782469fdc1718c4ed1d22549b5fb12af0d57d35e992158a772a37cf", size = 18577076, upload-time = "2025-09-09T15:58:07.745Z" }, - { url = "https://files.pythonhosted.org/packages/f0/a1/b87a284fb15a42e9274e7fcea0dad259d12ddbf07c1595b26883151ca3b4/numpy-2.3.3-cp314-cp314-win32.whl", hash = "sha256:cb248499b0bc3be66ebd6578b83e5acacf1d6cb2a77f2248ce0e40fbec5a76d0", size = 6366952, upload-time = "2025-09-09T15:58:10.096Z" }, - { url = "https://files.pythonhosted.org/packages/70/5f/1816f4d08f3b8f66576d8433a66f8fa35a5acfb3bbd0bf6c31183b003f3d/numpy-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:691808c2b26b0f002a032c73255d0bd89751425f379f7bcd22d140db593a96e8", size = 12919322, upload-time = "2025-09-09T15:58:12.138Z" }, - { url = "https://files.pythonhosted.org/packages/8c/de/072420342e46a8ea41c324a555fa90fcc11637583fb8df722936aed1736d/numpy-2.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:9ad12e976ca7b10f1774b03615a2a4bab8addce37ecc77394d8e986927dc0dfe", size = 10478630, upload-time = "2025-09-09T15:58:14.64Z" }, - { url = "https://files.pythonhosted.org/packages/d5/df/ee2f1c0a9de7347f14da5dd3cd3c3b034d1b8607ccb6883d7dd5c035d631/numpy-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9cc48e09feb11e1db00b320e9d30a4151f7369afb96bd0e48d942d09da3a0d00", size = 21047987, upload-time = "2025-09-09T15:58:16.889Z" }, - { url = "https://files.pythonhosted.org/packages/d6/92/9453bdc5a4e9e69cf4358463f25e8260e2ffc126d52e10038b9077815989/numpy-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:901bf6123879b7f251d3631967fd574690734236075082078e0571977c6a8e6a", size = 14301076, upload-time = "2025-09-09T15:58:20.343Z" }, - { url = "https://files.pythonhosted.org/packages/13/77/1447b9eb500f028bb44253105bd67534af60499588a5149a94f18f2ca917/numpy-2.3.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:7f025652034199c301049296b59fa7d52c7e625017cae4c75d8662e377bf487d", size = 5229491, upload-time = "2025-09-09T15:58:22.481Z" }, - { url = "https://files.pythonhosted.org/packages/3d/f9/d72221b6ca205f9736cb4b2ce3b002f6e45cd67cd6a6d1c8af11a2f0b649/numpy-2.3.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:533ca5f6d325c80b6007d4d7fb1984c303553534191024ec6a524a4c92a5935a", size = 6737913, upload-time = "2025-09-09T15:58:24.569Z" }, - { url = "https://files.pythonhosted.org/packages/3c/5f/d12834711962ad9c46af72f79bb31e73e416ee49d17f4c797f72c96b6ca5/numpy-2.3.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0edd58682a399824633b66885d699d7de982800053acf20be1eaa46d92009c54", size = 14352811, upload-time = "2025-09-09T15:58:26.416Z" }, - { url = "https://files.pythonhosted.org/packages/a1/0d/fdbec6629d97fd1bebed56cd742884e4eead593611bbe1abc3eb40d304b2/numpy-2.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:367ad5d8fbec5d9296d18478804a530f1191e24ab4d75ab408346ae88045d25e", size = 16702689, upload-time = "2025-09-09T15:58:28.831Z" }, - { url = "https://files.pythonhosted.org/packages/9b/09/0a35196dc5575adde1eb97ddfbc3e1687a814f905377621d18ca9bc2b7dd/numpy-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8f6ac61a217437946a1fa48d24c47c91a0c4f725237871117dea264982128097", size = 16133855, upload-time = "2025-09-09T15:58:31.349Z" }, - { url = "https://files.pythonhosted.org/packages/7a/ca/c9de3ea397d576f1b6753eaa906d4cdef1bf97589a6d9825a349b4729cc2/numpy-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:179a42101b845a816d464b6fe9a845dfaf308fdfc7925387195570789bb2c970", size = 18652520, upload-time = "2025-09-09T15:58:33.762Z" }, - { url = "https://files.pythonhosted.org/packages/fd/c2/e5ed830e08cd0196351db55db82f65bc0ab05da6ef2b72a836dcf1936d2f/numpy-2.3.3-cp314-cp314t-win32.whl", hash = "sha256:1250c5d3d2562ec4174bce2e3a1523041595f9b651065e4a4473f5f48a6bc8a5", size = 6515371, upload-time = "2025-09-09T15:58:36.04Z" }, - { url = "https://files.pythonhosted.org/packages/47/c7/b0f6b5b67f6788a0725f744496badbb604d226bf233ba716683ebb47b570/numpy-2.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b37a0b2e5935409daebe82c1e42274d30d9dd355852529eab91dab8dcca7419f", size = 13112576, upload-time = "2025-09-09T15:58:37.927Z" }, - { url = "https://files.pythonhosted.org/packages/06/b9/33bba5ff6fb679aa0b1f8a07e853f002a6b04b9394db3069a1270a7784ca/numpy-2.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:78c9f6560dc7e6b3990e32df7ea1a50bbd0e2a111e05209963f5ddcab7073b0b", size = 10545953, upload-time = "2025-09-09T15:58:40.576Z" }, - { url = "https://files.pythonhosted.org/packages/b8/f2/7e0a37cfced2644c9563c529f29fa28acbd0960dde32ece683aafa6f4949/numpy-2.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1e02c7159791cd481e1e6d5ddd766b62a4d5acf8df4d4d1afe35ee9c5c33a41e", size = 21131019, upload-time = "2025-09-09T15:58:42.838Z" }, - { url = "https://files.pythonhosted.org/packages/1a/7e/3291f505297ed63831135a6cc0f474da0c868a1f31b0dd9a9f03a7a0d2ed/numpy-2.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:dca2d0fc80b3893ae72197b39f69d55a3cd8b17ea1b50aa4c62de82419936150", size = 14376288, upload-time = "2025-09-09T15:58:45.425Z" }, - { url = "https://files.pythonhosted.org/packages/bf/4b/ae02e985bdeee73d7b5abdefeb98aef1207e96d4c0621ee0cf228ddfac3c/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:99683cbe0658f8271b333a1b1b4bb3173750ad59c0c61f5bbdc5b318918fffe3", size = 5305425, upload-time = "2025-09-09T15:58:48.6Z" }, - { url = "https://files.pythonhosted.org/packages/8b/eb/9df215d6d7250db32007941500dc51c48190be25f2401d5b2b564e467247/numpy-2.3.3-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:d9d537a39cc9de668e5cd0e25affb17aec17b577c6b3ae8a3d866b479fbe88d0", size = 6819053, upload-time = "2025-09-09T15:58:50.401Z" }, - { url = "https://files.pythonhosted.org/packages/57/62/208293d7d6b2a8998a4a1f23ac758648c3c32182d4ce4346062018362e29/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8596ba2f8af5f93b01d97563832686d20206d303024777f6dfc2e7c7c3f1850e", size = 14420354, upload-time = "2025-09-09T15:58:52.704Z" }, - { url = "https://files.pythonhosted.org/packages/ed/0c/8e86e0ff7072e14a71b4c6af63175e40d1e7e933ce9b9e9f765a95b4e0c3/numpy-2.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1ec5615b05369925bd1125f27df33f3b6c8bc10d788d5999ecd8769a1fa04db", size = 16760413, upload-time = "2025-09-09T15:58:55.027Z" }, - { url = "https://files.pythonhosted.org/packages/af/11/0cc63f9f321ccf63886ac203336777140011fb669e739da36d8db3c53b98/numpy-2.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2e267c7da5bf7309670523896df97f93f6e469fb931161f483cd6882b3b1a5dc", size = 12971844, upload-time = "2025-09-09T15:58:57.359Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, + { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, + { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, + { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, ] [[package]] @@ -1649,11 +1768,11 @@ wheels = [ [[package]] name = "nvidia-nvjitlink" -version = "13.0.88" +version = "13.3.33" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5", size = 40742423, upload-time = "2026-05-26T16:54:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/69/30/45414e35ff2eee7db3da037e5707037ccf9d2b5218ffbdb055ea4d5aa98a/nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e", size = 39168635, upload-time = "2026-05-26T16:54:13.906Z" }, ] [[package]] @@ -1675,73 +1794,52 @@ wheels = [ ] [[package]] -name = "opt-einsum" -version = "3.4.0" +name = "opentelemetry-api" +version = "1.30.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/b9/2ac072041e899a52f20cf9510850ff58295003aa75525e58343591b0cbfb/opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac", size = 63004, upload-time = "2024-09-26T14:33:24.483Z" } +dependencies = [ + { name = "deprecated" }, + { name = "importlib-metadata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/6d/bbbf879826b7f3c89a45252010b5796fb1f1a0d45d9dc4709db0ef9a06c8/opentelemetry_api-1.30.0.tar.gz", hash = "sha256:375893400c1435bf623f7dfb3bcd44825fe6b56c34d0667c542ea8257b1a1240", size = 63703, upload-time = "2025-02-04T18:17:13.789Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/cd/066e86230ae37ed0be70aae89aabf03ca8d9f39c8aea0dec8029455b5540/opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd", size = 71932, upload-time = "2024-09-26T14:33:23.039Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/eea862fae6413d8181b23acf8e13489c90a45f17986ee9cf4eab8a0b9ad9/opentelemetry_api-1.30.0-py3-none-any.whl", hash = "sha256:d5f5284890d73fdf47f843dda3210edf37a38d66f44f2b5aedc1e89ed455dc09", size = 64955, upload-time = "2025-02-04T18:16:46.167Z" }, ] [[package]] -name = "optree" -version = "0.17.0" +name = "opentelemetry-sdk" +version = "1.30.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/c7/0853e0c59b135dff770615d2713b547b6b3b5cde7c10995b4a5825244612/optree-0.17.0.tar.gz", hash = "sha256:5335a5ec44479920620d72324c66563bd705ab2a698605dd4b6ee67dbcad7ecd", size = 163111, upload-time = "2025-07-25T11:26:11.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/f9/6ca076fd4c6f16be031afdc711a2676c1ff15bd1717ee2e699179b1a29bc/optree-0.17.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98990201f352dba253af1a995c1453818db5f08de4cae7355d85aa6023676a52", size = 350398, upload-time = "2025-07-25T11:24:26.672Z" }, - { url = "https://files.pythonhosted.org/packages/95/4c/81344cbdcf8ea8525a21c9d65892d7529010ee2146c53423b2e9a84441ba/optree-0.17.0-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:e1a40adf6bb78a6a4b4f480879de2cb6b57d46d680a4d9834aa824f41e69c0d9", size = 404834, upload-time = "2025-07-25T11:24:28.988Z" }, - { url = "https://files.pythonhosted.org/packages/e5/c4/ac1880372a89f5c21514a7965dfa23b1afb2ad683fb9804d366727de9ecf/optree-0.17.0-cp310-cp310-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:78a113436a0a440f900b2799584f3cc2b2eea1b245d81c3583af42ac003e333c", size = 402116, upload-time = "2025-07-25T11:24:30.396Z" }, - { url = "https://files.pythonhosted.org/packages/ff/72/ad6be4d6a03805cf3921b492494cb3371ca28060d5ad19d5a36e10c4d67d/optree-0.17.0-cp310-cp310-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e45c16018f4283f028cf839b707b7ac734e8056a31b7198a1577161fcbe146d", size = 398491, upload-time = "2025-07-25T11:24:31.725Z" }, - { url = "https://files.pythonhosted.org/packages/d9/c1/6827fb504351f9a3935699b0eb31c8a6af59d775ee78289a25e0ba54f732/optree-0.17.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b698613d821d80cc216a2444ebc3145c8bf671b55a2223058a6574c1483a65f6", size = 387957, upload-time = "2025-07-25T11:24:32.759Z" }, - { url = "https://files.pythonhosted.org/packages/73/5c/13a2a864b0c0b39c3c193be534a195a3ab2463c7d0443d4a76e749e3ff83/optree-0.17.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3080c564c9760711aa72d1b4d700ce1417f99ad087136f415c4eb8221169e2a3", size = 362797, upload-time = "2025-07-25T11:24:39.509Z" }, - { url = "https://files.pythonhosted.org/packages/da/f5/ff7dcb5a0108ee89c2be09aed2ebd26a7e1333d8122031aa9d9322b24ee6/optree-0.17.0-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:834a8fb358b608240b3a38706a09b43974675624485fad64c8ee641dae2eb57d", size = 419450, upload-time = "2025-07-25T11:24:40.555Z" }, - { url = "https://files.pythonhosted.org/packages/1b/e6/48a97aefd18770b55e5ed456d8183891f325cdb6d90592e5f072ed6951f8/optree-0.17.0-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1a2bd263e6b5621d000d0f94de1f245414fd5dbce365a24b7b89b1ed0ef56cf9", size = 417557, upload-time = "2025-07-25T11:24:42.396Z" }, - { url = "https://files.pythonhosted.org/packages/c4/b1/4e280edab8a86be47ec1f9bd9ed4b685d2e15f0950ae62b613b26d12a1da/optree-0.17.0-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9b37daca4ad89339b1f5320cc61ac600dcf976adbb060769d36d5542d6ebfedf", size = 414174, upload-time = "2025-07-25T11:24:43.51Z" }, - { url = "https://files.pythonhosted.org/packages/db/3b/49a9a1986215dd342525974deeb17c260a83fee8fad147276fd710ac8718/optree-0.17.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a146a6917f3e28cfdc268ff1770aa696c346482dd3da681c3ff92153d94450ea", size = 402000, upload-time = "2025-07-25T11:24:44.819Z" }, - { url = "https://files.pythonhosted.org/packages/41/90/e12dea2cb5d8a5e17bbe3011ed4e972b89c027272a816db4897589751cad/optree-0.17.0-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e13ae51a63d69db445f269a3a4fd1d6edb064a705188d007ea47c9f034788fc5", size = 365869, upload-time = "2025-07-25T11:24:51.807Z" }, - { url = "https://files.pythonhosted.org/packages/76/ee/21af214663960a479863cd6c03d7a0abc8123ea22a6ea34689c2eed88ccd/optree-0.17.0-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:5958f58423cc7870cb011c8c8f92687397380886e8c9d33adac752147e7bbc3f", size = 424465, upload-time = "2025-07-25T11:24:53.124Z" }, - { url = "https://files.pythonhosted.org/packages/54/a3/64b184a79373753f4f46a5cd301ea581f71d6dc1a5c103bd2394f0925d40/optree-0.17.0-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:970ae4e47727b4c5526fc583b87d29190e576f6a2b6c19e8671589b73d256250", size = 420686, upload-time = "2025-07-25T11:24:54.212Z" }, - { url = "https://files.pythonhosted.org/packages/6c/6d/b6051b0b1ef9a49df96a66e9e62fc02620d2115d1ba659888c94e67fcfc9/optree-0.17.0-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54177fd3e6e05c08b66329e26d7d44b85f24125f25c6b74c921499a1b31b8f70", size = 421225, upload-time = "2025-07-25T11:24:55.213Z" }, - { url = "https://files.pythonhosted.org/packages/f6/f1/940bc959aaef9eede8bb1b1127833b0929c6ffa9268ec0f6cb19877e2027/optree-0.17.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1959cfbc38c228c8195354967cda64887b96219924b7b3759e5ee355582c1ec", size = 408819, upload-time = "2025-07-25T11:24:56.315Z" }, - { url = "https://files.pythonhosted.org/packages/21/04/9706d11b880186e9e9d66d7c21ce249b2ce0212645137cc13fdd18247c26/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5995a3efce4b00a14049268a81ab0379656a41ddf3c3761e3b88937fca44d48", size = 348177, upload-time = "2025-07-25T11:25:00.999Z" }, - { url = "https://files.pythonhosted.org/packages/ae/4b/0415c18816818ac871c9f3d5c7c5f4ceb83baff03ed511c9c94591ace4bc/optree-0.17.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:d06e8143d16fe6c0708f3cc2807b5b65f815d60ee2b52f3d79e4022c95563482", size = 354389, upload-time = "2025-07-25T11:25:02.337Z" }, - { url = "https://files.pythonhosted.org/packages/dd/12/24d4a417fd325ec06cfbce52716ac4f816ef696653b868960ac2ccb28436/optree-0.17.0-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfeea4aa0fd354d27922aba63ff9d86e4e126c6bf89cfb02849e68515519f1a5", size = 368513, upload-time = "2025-07-25T11:25:05.548Z" }, - { url = "https://files.pythonhosted.org/packages/30/e2/34e392209933e2c582c67594a7a6b4851bca4015c83b51c7508384b616b4/optree-0.17.0-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6b2ff8999a9b84d00f23a032b6b3f13678894432a335d024e0670b9880f238ca", size = 430378, upload-time = "2025-07-25T11:25:06.918Z" }, - { url = "https://files.pythonhosted.org/packages/5f/16/0a0d6139022e9a53ecb1212fb6fbc5b60eff824371071ef5f5fa481d8167/optree-0.17.0-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ea8bef525432b38a84e7448348da1a2dc308375bce79c77675cc50a501305851", size = 423294, upload-time = "2025-07-25T11:25:08.043Z" }, - { url = "https://files.pythonhosted.org/packages/ef/60/2e083dabb6aff6d939d8aab16ba3dbe6eee9429597a13f3fca57b33cdcde/optree-0.17.0-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f95b81aa67538d38316b184a6ff39a3725ee5c8555fba21dcb692f8d7c39302e", size = 424633, upload-time = "2025-07-25T11:25:09.141Z" }, - { url = "https://files.pythonhosted.org/packages/af/fd/0e4229b5fa3fd9d3c779a606c0f358ffbdfee717f49b3477facd04de2cec/optree-0.17.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e808a1125169ae90de623456ef2423eb84a8578a74f03fe48b06b8561c2cc31d", size = 414866, upload-time = "2025-07-25T11:25:10.214Z" }, - { url = "https://files.pythonhosted.org/packages/39/df/b8882f5519c85af146de3a79a08066a56fe634b23052c593fcedc70bfcd7/optree-0.17.0-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8e45a13b35873712e095fe0f7fd6e9c4f98f3bd5af6f5dc33c17b80357bc97fc", size = 386945, upload-time = "2025-07-25T11:25:17.728Z" }, - { url = "https://files.pythonhosted.org/packages/ca/d7/91f4efb509bda601a1591465c4a5bd55320e4bafe06b294bf80754127b0e/optree-0.17.0-cp313-cp313t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:bfaf04d833dc53e5cfccff3b564e934a49086158472e31d84df31fce6d4f7b1c", size = 444177, upload-time = "2025-07-25T11:25:18.749Z" }, - { url = "https://files.pythonhosted.org/packages/84/17/a4833006e925c6ed5c45ceb02e65c9e9a260e70da6523858fcf628481847/optree-0.17.0-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b4c1d030ac1c881803f5c8e23d241159ae403fd00cdf57625328f282fc671ebd", size = 439198, upload-time = "2025-07-25T11:25:19.865Z" }, - { url = "https://files.pythonhosted.org/packages/ef/d1/c08fc60f6dfcb1b86ca1fdc0add08a98412a1596cd45830acbdc309f2cdb/optree-0.17.0-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd7738709970acab5d963896192b63b2718be93bb6c0bcea91895ea157fa2b13", size = 439391, upload-time = "2025-07-25T11:25:20.942Z" }, - { url = "https://files.pythonhosted.org/packages/05/8f/461e10201003e6ad6bff3c594a29a7e044454aba68c5f795f4c8386ce47c/optree-0.17.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1644bc24b6e93cafccfdeee44157c3d4ae9bb0af3e861300602d716699865b1a", size = 426555, upload-time = "2025-07-25T11:25:21.968Z" }, - { url = "https://files.pythonhosted.org/packages/3c/21/6480d23b52b2e23b976fe254b9fbdc4b514e90a349b1ee73565b185c69f1/optree-0.17.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd21e0a89806cc3b86aaa578a73897d56085038fe432043534a23b2e559d7691", size = 369929, upload-time = "2025-07-25T11:25:28.897Z" }, - { url = "https://files.pythonhosted.org/packages/b3/29/69bb26473ff862a1792f5568c977e7a2580e08afe0fdcd7a7b3e1e4d6933/optree-0.17.0-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:9211c61285b8b3e42fd0e803cebd6e2b0987d8b2edffe45b42923debca09a9df", size = 430381, upload-time = "2025-07-25T11:25:29.984Z" }, - { url = "https://files.pythonhosted.org/packages/c8/8b/2c0a38c0d0c2396d698b97216cd6814d6754d11997b6ac66c57d87d71bae/optree-0.17.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:87938255749a45979c4e331627cb33d81aa08b0a09d024368b3e25ff67f0e9f2", size = 424461, upload-time = "2025-07-25T11:25:31.116Z" }, - { url = "https://files.pythonhosted.org/packages/a7/77/08fda3f97621190d50762225ee8bad87463a8b3a55fba451a999971ff130/optree-0.17.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3432858145fd1955a3be12207507466ac40a6911f428bf5d2d6c7f67486530a2", size = 427234, upload-time = "2025-07-25T11:25:32.289Z" }, - { url = "https://files.pythonhosted.org/packages/ea/b5/b4f19952c36d6448c85a6ef6be5f916dd13548de2b684ab123f04b450850/optree-0.17.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5afe3e9e2f6da0a0a5c0892f32f675eb88965036b061aa555b74e6c412a05e17", size = 413863, upload-time = "2025-07-25T11:25:33.379Z" }, - { url = "https://files.pythonhosted.org/packages/88/42/6003f13e66cfbe7f0011bf8509da2479aba93068cdb9d79bf46010255089/optree-0.17.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5739c03a3362be42cb7649e82457c90aa818aa3e82af9681d3100c3346f4a90f", size = 386975, upload-time = "2025-07-25T11:25:40.376Z" }, - { url = "https://files.pythonhosted.org/packages/d0/53/621642abd76eda5a941b47adc98be81f0052683160be776499d11b4af83d/optree-0.17.0-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:ee07b59a08bd45aedd5252241a98841f1a5082a7b9b73df2dae6a433aa2a91d8", size = 444173, upload-time = "2025-07-25T11:25:41.474Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d3/8819a2d5105a240d6793d11a61d597db91756ce84da5cee08808c6b8f61f/optree-0.17.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:875c017890a4b5d566af5593cab67fe3c4845544942af57e6bb9dea17e060297", size = 439080, upload-time = "2025-07-25T11:25:42.605Z" }, - { url = "https://files.pythonhosted.org/packages/c6/ef/9dbd34dfd1ad89feb239ca9925897a14ac94f190379a3bd991afdfd94186/optree-0.17.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ffa5686191139f763e13445a169765c83517164bc28e60dbedb19bed2b2655f1", size = 439422, upload-time = "2025-07-25T11:25:43.672Z" }, - { url = "https://files.pythonhosted.org/packages/86/ca/a7a7549af2951925a692df508902ed2a6a94a51bc846806d2281b1029ef9/optree-0.17.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:575cf48cc2190acb565bd2b26b6f9b15c4e3b60183e86031215badc9d5441345", size = 426579, upload-time = "2025-07-25T11:25:44.765Z" }, - { url = "https://files.pythonhosted.org/packages/ed/d7/3036d15c028c447b1bd65dcf8f66cfd775bfa4e52daa74b82fb1d3c88faf/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:adde1427e0982cfc5f56939c26b4ebbd833091a176734c79fb95c78bdf833dff", size = 350952, upload-time = "2025-07-25T11:26:02.692Z" }, - { url = "https://files.pythonhosted.org/packages/71/45/e710024ef77324e745de48efd64f6270d8c209f14107a48ffef4049ac57a/optree-0.17.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a80b7e5de5dd09b9c8b62d501e29a3850b047565c336c9d004b07ee1c01f4ae1", size = 389568, upload-time = "2025-07-25T11:26:04.094Z" }, - { url = "https://files.pythonhosted.org/packages/69/c4/94a187ed3ca71194b9da6a276790e1703c7544c8f695ac915214ae8ce934/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f87f6f39015fc82d7adeee19900d246b89911319726e93cb2dbd4d1a809899bd", size = 363728, upload-time = "2025-07-25T11:26:07.959Z" }, - { url = "https://files.pythonhosted.org/packages/cd/99/23b7a484da8dfb814107b20ef2c93ef27c04f36aeb83bd976964a5b69e06/optree-0.17.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58b0a83a967d2ef0f343db7182f0ad074eb1166bcaea909ae33909462013f151", size = 404649, upload-time = "2025-07-25T11:26:09.463Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/93/ee/d710062e8a862433d1be0b85920d0c653abe318878fef2d14dfe2c62ff7b/opentelemetry_sdk-1.30.0.tar.gz", hash = "sha256:c9287a9e4a7614b9946e933a67168450b9ab35f08797eb9bc77d998fa480fa18", size = 158633, upload-time = "2025-02-04T18:17:28.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/28/64d781d6adc6bda2260067ce2902bd030cf45aec657e02e28c5b4480b976/opentelemetry_sdk-1.30.0-py3-none-any.whl", hash = "sha256:14fe7afc090caad881addb6926cec967129bd9260c4d33ae6a217359f6b61091", size = 118717, upload-time = "2025-02-04T18:17:09.353Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.51b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/c0/0f9ef4605fea7f2b83d55dd0b0d7aebe8feead247cd6facd232b30907b4f/opentelemetry_semantic_conventions-0.51b0.tar.gz", hash = "sha256:3fabf47f35d1fd9aebcdca7e6802d86bd5ebc3bc3408b7e3248dde6e87a18c47", size = 107191, upload-time = "2025-02-04T18:17:29.903Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/75/d7bdbb6fd8630b4cafb883482b75c4fc276b6426619539d266e32ac53266/opentelemetry_semantic_conventions-0.51b0-py3-none-any.whl", hash = "sha256:fdc777359418e8d06c86012c3dc92c88a6453ba662e941593adb062e48c2eeae", size = 177416, upload-time = "2025-02-04T18:17:11.305Z" }, ] [[package]] name = "packaging" -version = "25.0" +version = "26.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, ] [[package]] @@ -1750,7 +1848,8 @@ version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -1954,102 +2053,130 @@ wheels = [ [[package]] name = "propcache" -version = "0.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/16/43264e4a779dd8588c21a70f0709665ee8f611211bdd2c87d952cfa7c776/propcache-0.3.2.tar.gz", hash = "sha256:20d7d62e4e7ef05f221e0db2856b979540686342e7dd9973b815599c7057e168", size = 44139, upload-time = "2025-06-09T22:56:06.081Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ab/14/510deed325e262afeb8b360043c5d7c960da7d3ecd6d6f9496c9c56dc7f4/propcache-0.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:22d9962a358aedbb7a2e36187ff273adeaab9743373a272976d2e348d08c7770", size = 73178, upload-time = "2025-06-09T22:53:40.126Z" }, - { url = "https://files.pythonhosted.org/packages/cd/4e/ad52a7925ff01c1325653a730c7ec3175a23f948f08626a534133427dcff/propcache-0.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d0fda578d1dc3f77b6b5a5dce3b9ad69a8250a891760a548df850a5e8da87f3", size = 43133, upload-time = "2025-06-09T22:53:41.965Z" }, - { url = "https://files.pythonhosted.org/packages/63/7c/e9399ba5da7780871db4eac178e9c2e204c23dd3e7d32df202092a1ed400/propcache-0.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3def3da3ac3ce41562d85db655d18ebac740cb3fa4367f11a52b3da9d03a5cc3", size = 43039, upload-time = "2025-06-09T22:53:43.268Z" }, - { url = "https://files.pythonhosted.org/packages/22/e1/58da211eb8fdc6fc854002387d38f415a6ca5f5c67c1315b204a5d3e9d7a/propcache-0.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9bec58347a5a6cebf239daba9bda37dffec5b8d2ce004d9fe4edef3d2815137e", size = 201903, upload-time = "2025-06-09T22:53:44.872Z" }, - { url = "https://files.pythonhosted.org/packages/c4/0a/550ea0f52aac455cb90111c8bab995208443e46d925e51e2f6ebdf869525/propcache-0.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55ffda449a507e9fbd4aca1a7d9aa6753b07d6166140e5a18d2ac9bc49eac220", size = 213362, upload-time = "2025-06-09T22:53:46.707Z" }, - { url = "https://files.pythonhosted.org/packages/5a/af/9893b7d878deda9bb69fcf54600b247fba7317761b7db11fede6e0f28bd0/propcache-0.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:64a67fb39229a8a8491dd42f864e5e263155e729c2e7ff723d6e25f596b1e8cb", size = 210525, upload-time = "2025-06-09T22:53:48.547Z" }, - { url = "https://files.pythonhosted.org/packages/7c/bb/38fd08b278ca85cde36d848091ad2b45954bc5f15cce494bb300b9285831/propcache-0.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da1cf97b92b51253d5b68cf5a2b9e0dafca095e36b7f2da335e27dc6172a614", size = 198283, upload-time = "2025-06-09T22:53:50.067Z" }, - { url = "https://files.pythonhosted.org/packages/78/8c/9fe55bd01d362bafb413dfe508c48753111a1e269737fa143ba85693592c/propcache-0.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f559e127134b07425134b4065be45b166183fdcb433cb6c24c8e4149056ad50", size = 191872, upload-time = "2025-06-09T22:53:51.438Z" }, - { url = "https://files.pythonhosted.org/packages/54/14/4701c33852937a22584e08abb531d654c8bcf7948a8f87ad0a4822394147/propcache-0.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aff2e4e06435d61f11a428360a932138d0ec288b0a31dd9bd78d200bd4a2b339", size = 199452, upload-time = "2025-06-09T22:53:53.229Z" }, - { url = "https://files.pythonhosted.org/packages/16/44/447f2253d859602095356007657ee535e0093215ea0b3d1d6a41d16e5201/propcache-0.3.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:4927842833830942a5d0a56e6f4839bc484785b8e1ce8d287359794818633ba0", size = 191567, upload-time = "2025-06-09T22:53:54.541Z" }, - { url = "https://files.pythonhosted.org/packages/f2/b3/e4756258749bb2d3b46defcff606a2f47410bab82be5824a67e84015b267/propcache-0.3.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:6107ddd08b02654a30fb8ad7a132021759d750a82578b94cd55ee2772b6ebea2", size = 193015, upload-time = "2025-06-09T22:53:56.44Z" }, - { url = "https://files.pythonhosted.org/packages/1e/df/e6d3c7574233164b6330b9fd697beeac402afd367280e6dc377bb99b43d9/propcache-0.3.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:70bd8b9cd6b519e12859c99f3fc9a93f375ebd22a50296c3a295028bea73b9e7", size = 204660, upload-time = "2025-06-09T22:53:57.839Z" }, - { url = "https://files.pythonhosted.org/packages/b2/53/e4d31dd5170b4a0e2e6b730f2385a96410633b4833dc25fe5dffd1f73294/propcache-0.3.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2183111651d710d3097338dd1893fcf09c9f54e27ff1a8795495a16a469cc90b", size = 206105, upload-time = "2025-06-09T22:53:59.638Z" }, - { url = "https://files.pythonhosted.org/packages/7f/fe/74d54cf9fbe2a20ff786e5f7afcfde446588f0cf15fb2daacfbc267b866c/propcache-0.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fb075ad271405dcad8e2a7ffc9a750a3bf70e533bd86e89f0603e607b93aa64c", size = 196980, upload-time = "2025-06-09T22:54:01.071Z" }, - { url = "https://files.pythonhosted.org/packages/22/ec/c469c9d59dada8a7679625e0440b544fe72e99311a4679c279562051f6fc/propcache-0.3.2-cp310-cp310-win32.whl", hash = "sha256:404d70768080d3d3bdb41d0771037da19d8340d50b08e104ca0e7f9ce55fce70", size = 37679, upload-time = "2025-06-09T22:54:03.003Z" }, - { url = "https://files.pythonhosted.org/packages/38/35/07a471371ac89d418f8d0b699c75ea6dca2041fbda360823de21f6a9ce0a/propcache-0.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:7435d766f978b4ede777002e6b3b6641dd229cd1da8d3d3106a45770365f9ad9", size = 41459, upload-time = "2025-06-09T22:54:04.134Z" }, - { url = "https://files.pythonhosted.org/packages/80/8d/e8b436717ab9c2cfc23b116d2c297305aa4cd8339172a456d61ebf5669b8/propcache-0.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0b8d2f607bd8f80ddc04088bc2a037fdd17884a6fcadc47a96e334d72f3717be", size = 74207, upload-time = "2025-06-09T22:54:05.399Z" }, - { url = "https://files.pythonhosted.org/packages/d6/29/1e34000e9766d112171764b9fa3226fa0153ab565d0c242c70e9945318a7/propcache-0.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06766d8f34733416e2e34f46fea488ad5d60726bb9481d3cddf89a6fa2d9603f", size = 43648, upload-time = "2025-06-09T22:54:08.023Z" }, - { url = "https://files.pythonhosted.org/packages/46/92/1ad5af0df781e76988897da39b5f086c2bf0f028b7f9bd1f409bb05b6874/propcache-0.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2dc1f4a1df4fecf4e6f68013575ff4af84ef6f478fe5344317a65d38a8e6dc9", size = 43496, upload-time = "2025-06-09T22:54:09.228Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ce/e96392460f9fb68461fabab3e095cb00c8ddf901205be4eae5ce246e5b7e/propcache-0.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be29c4f4810c5789cf10ddf6af80b041c724e629fa51e308a7a0fb19ed1ef7bf", size = 217288, upload-time = "2025-06-09T22:54:10.466Z" }, - { url = "https://files.pythonhosted.org/packages/c5/2a/866726ea345299f7ceefc861a5e782b045545ae6940851930a6adaf1fca6/propcache-0.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59d61f6970ecbd8ff2e9360304d5c8876a6abd4530cb752c06586849ac8a9dc9", size = 227456, upload-time = "2025-06-09T22:54:11.828Z" }, - { url = "https://files.pythonhosted.org/packages/de/03/07d992ccb6d930398689187e1b3c718339a1c06b8b145a8d9650e4726166/propcache-0.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62180e0b8dbb6b004baec00a7983e4cc52f5ada9cd11f48c3528d8cfa7b96a66", size = 225429, upload-time = "2025-06-09T22:54:13.823Z" }, - { url = "https://files.pythonhosted.org/packages/5d/e6/116ba39448753b1330f48ab8ba927dcd6cf0baea8a0ccbc512dfb49ba670/propcache-0.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c144ca294a204c470f18cf4c9d78887810d04a3e2fbb30eea903575a779159df", size = 213472, upload-time = "2025-06-09T22:54:15.232Z" }, - { url = "https://files.pythonhosted.org/packages/a6/85/f01f5d97e54e428885a5497ccf7f54404cbb4f906688a1690cd51bf597dc/propcache-0.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5c2a784234c28854878d68978265617aa6dc0780e53d44b4d67f3651a17a9a2", size = 204480, upload-time = "2025-06-09T22:54:17.104Z" }, - { url = "https://files.pythonhosted.org/packages/e3/79/7bf5ab9033b8b8194cc3f7cf1aaa0e9c3256320726f64a3e1f113a812dce/propcache-0.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5745bc7acdafa978ca1642891b82c19238eadc78ba2aaa293c6863b304e552d7", size = 214530, upload-time = "2025-06-09T22:54:18.512Z" }, - { url = "https://files.pythonhosted.org/packages/31/0b/bd3e0c00509b609317df4a18e6b05a450ef2d9a963e1d8bc9c9415d86f30/propcache-0.3.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c0075bf773d66fa8c9d41f66cc132ecc75e5bb9dd7cce3cfd14adc5ca184cb95", size = 205230, upload-time = "2025-06-09T22:54:19.947Z" }, - { url = "https://files.pythonhosted.org/packages/7a/23/fae0ff9b54b0de4e819bbe559508da132d5683c32d84d0dc2ccce3563ed4/propcache-0.3.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5f57aa0847730daceff0497f417c9de353c575d8da3579162cc74ac294c5369e", size = 206754, upload-time = "2025-06-09T22:54:21.716Z" }, - { url = "https://files.pythonhosted.org/packages/b7/7f/ad6a3c22630aaa5f618b4dc3c3598974a72abb4c18e45a50b3cdd091eb2f/propcache-0.3.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:eef914c014bf72d18efb55619447e0aecd5fb7c2e3fa7441e2e5d6099bddff7e", size = 218430, upload-time = "2025-06-09T22:54:23.17Z" }, - { url = "https://files.pythonhosted.org/packages/5b/2c/ba4f1c0e8a4b4c75910742f0d333759d441f65a1c7f34683b4a74c0ee015/propcache-0.3.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:2a4092e8549031e82facf3decdbc0883755d5bbcc62d3aea9d9e185549936dcf", size = 223884, upload-time = "2025-06-09T22:54:25.539Z" }, - { url = "https://files.pythonhosted.org/packages/88/e4/ebe30fc399e98572019eee82ad0caf512401661985cbd3da5e3140ffa1b0/propcache-0.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:85871b050f174bc0bfb437efbdb68aaf860611953ed12418e4361bc9c392749e", size = 211480, upload-time = "2025-06-09T22:54:26.892Z" }, - { url = "https://files.pythonhosted.org/packages/96/0a/7d5260b914e01d1d0906f7f38af101f8d8ed0dc47426219eeaf05e8ea7c2/propcache-0.3.2-cp311-cp311-win32.whl", hash = "sha256:36c8d9b673ec57900c3554264e630d45980fd302458e4ac801802a7fd2ef7897", size = 37757, upload-time = "2025-06-09T22:54:28.241Z" }, - { url = "https://files.pythonhosted.org/packages/e1/2d/89fe4489a884bc0da0c3278c552bd4ffe06a1ace559db5ef02ef24ab446b/propcache-0.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53af8cb6a781b02d2ea079b5b853ba9430fcbe18a8e3ce647d5982a3ff69f39", size = 41500, upload-time = "2025-06-09T22:54:29.4Z" }, - { url = "https://files.pythonhosted.org/packages/a8/42/9ca01b0a6f48e81615dca4765a8f1dd2c057e0540f6116a27dc5ee01dfb6/propcache-0.3.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8de106b6c84506b31c27168582cd3cb3000a6412c16df14a8628e5871ff83c10", size = 73674, upload-time = "2025-06-09T22:54:30.551Z" }, - { url = "https://files.pythonhosted.org/packages/af/6e/21293133beb550f9c901bbece755d582bfaf2176bee4774000bd4dd41884/propcache-0.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:28710b0d3975117239c76600ea351934ac7b5ff56e60953474342608dbbb6154", size = 43570, upload-time = "2025-06-09T22:54:32.296Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c8/0393a0a3a2b8760eb3bde3c147f62b20044f0ddac81e9d6ed7318ec0d852/propcache-0.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce26862344bdf836650ed2487c3d724b00fbfec4233a1013f597b78c1cb73615", size = 43094, upload-time = "2025-06-09T22:54:33.929Z" }, - { url = "https://files.pythonhosted.org/packages/37/2c/489afe311a690399d04a3e03b069225670c1d489eb7b044a566511c1c498/propcache-0.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bca54bd347a253af2cf4544bbec232ab982f4868de0dd684246b67a51bc6b1db", size = 226958, upload-time = "2025-06-09T22:54:35.186Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ca/63b520d2f3d418c968bf596839ae26cf7f87bead026b6192d4da6a08c467/propcache-0.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55780d5e9a2ddc59711d727226bb1ba83a22dd32f64ee15594b9392b1f544eb1", size = 234894, upload-time = "2025-06-09T22:54:36.708Z" }, - { url = "https://files.pythonhosted.org/packages/11/60/1d0ed6fff455a028d678df30cc28dcee7af77fa2b0e6962ce1df95c9a2a9/propcache-0.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:035e631be25d6975ed87ab23153db6a73426a48db688070d925aa27e996fe93c", size = 233672, upload-time = "2025-06-09T22:54:38.062Z" }, - { url = "https://files.pythonhosted.org/packages/37/7c/54fd5301ef38505ab235d98827207176a5c9b2aa61939b10a460ca53e123/propcache-0.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee6f22b6eaa39297c751d0e80c0d3a454f112f5c6481214fcf4c092074cecd67", size = 224395, upload-time = "2025-06-09T22:54:39.634Z" }, - { url = "https://files.pythonhosted.org/packages/ee/1a/89a40e0846f5de05fdc6779883bf46ba980e6df4d2ff8fb02643de126592/propcache-0.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ca3aee1aa955438c4dba34fc20a9f390e4c79967257d830f137bd5a8a32ed3b", size = 212510, upload-time = "2025-06-09T22:54:41.565Z" }, - { url = "https://files.pythonhosted.org/packages/5e/33/ca98368586c9566a6b8d5ef66e30484f8da84c0aac3f2d9aec6d31a11bd5/propcache-0.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7a4f30862869fa2b68380d677cc1c5fcf1e0f2b9ea0cf665812895c75d0ca3b8", size = 222949, upload-time = "2025-06-09T22:54:43.038Z" }, - { url = "https://files.pythonhosted.org/packages/ba/11/ace870d0aafe443b33b2f0b7efdb872b7c3abd505bfb4890716ad7865e9d/propcache-0.3.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:b77ec3c257d7816d9f3700013639db7491a434644c906a2578a11daf13176251", size = 217258, upload-time = "2025-06-09T22:54:44.376Z" }, - { url = "https://files.pythonhosted.org/packages/5b/d2/86fd6f7adffcfc74b42c10a6b7db721d1d9ca1055c45d39a1a8f2a740a21/propcache-0.3.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cab90ac9d3f14b2d5050928483d3d3b8fb6b4018893fc75710e6aa361ecb2474", size = 213036, upload-time = "2025-06-09T22:54:46.243Z" }, - { url = "https://files.pythonhosted.org/packages/07/94/2d7d1e328f45ff34a0a284cf5a2847013701e24c2a53117e7c280a4316b3/propcache-0.3.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0b504d29f3c47cf6b9e936c1852246c83d450e8e063d50562115a6be6d3a2535", size = 227684, upload-time = "2025-06-09T22:54:47.63Z" }, - { url = "https://files.pythonhosted.org/packages/b7/05/37ae63a0087677e90b1d14710e532ff104d44bc1efa3b3970fff99b891dc/propcache-0.3.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:ce2ac2675a6aa41ddb2a0c9cbff53780a617ac3d43e620f8fd77ba1c84dcfc06", size = 234562, upload-time = "2025-06-09T22:54:48.982Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7c/3f539fcae630408d0bd8bf3208b9a647ccad10976eda62402a80adf8fc34/propcache-0.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:62b4239611205294cc433845b914131b2a1f03500ff3c1ed093ed216b82621e1", size = 222142, upload-time = "2025-06-09T22:54:50.424Z" }, - { url = "https://files.pythonhosted.org/packages/7c/d2/34b9eac8c35f79f8a962546b3e97e9d4b990c420ee66ac8255d5d9611648/propcache-0.3.2-cp312-cp312-win32.whl", hash = "sha256:df4a81b9b53449ebc90cc4deefb052c1dd934ba85012aa912c7ea7b7e38b60c1", size = 37711, upload-time = "2025-06-09T22:54:52.072Z" }, - { url = "https://files.pythonhosted.org/packages/19/61/d582be5d226cf79071681d1b46b848d6cb03d7b70af7063e33a2787eaa03/propcache-0.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:7046e79b989d7fe457bb755844019e10f693752d169076138abf17f31380800c", size = 41479, upload-time = "2025-06-09T22:54:53.234Z" }, - { url = "https://files.pythonhosted.org/packages/dc/d1/8c747fafa558c603c4ca19d8e20b288aa0c7cda74e9402f50f31eb65267e/propcache-0.3.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ca592ed634a73ca002967458187109265e980422116c0a107cf93d81f95af945", size = 71286, upload-time = "2025-06-09T22:54:54.369Z" }, - { url = "https://files.pythonhosted.org/packages/61/99/d606cb7986b60d89c36de8a85d58764323b3a5ff07770a99d8e993b3fa73/propcache-0.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9ecb0aad4020e275652ba3975740f241bd12a61f1a784df044cf7477a02bc252", size = 42425, upload-time = "2025-06-09T22:54:55.642Z" }, - { url = "https://files.pythonhosted.org/packages/8c/96/ef98f91bbb42b79e9bb82bdd348b255eb9d65f14dbbe3b1594644c4073f7/propcache-0.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7f08f1cc28bd2eade7a8a3d2954ccc673bb02062e3e7da09bc75d843386b342f", size = 41846, upload-time = "2025-06-09T22:54:57.246Z" }, - { url = "https://files.pythonhosted.org/packages/5b/ad/3f0f9a705fb630d175146cd7b1d2bf5555c9beaed54e94132b21aac098a6/propcache-0.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1a342c834734edb4be5ecb1e9fb48cb64b1e2320fccbd8c54bf8da8f2a84c33", size = 208871, upload-time = "2025-06-09T22:54:58.975Z" }, - { url = "https://files.pythonhosted.org/packages/3a/38/2085cda93d2c8b6ec3e92af2c89489a36a5886b712a34ab25de9fbca7992/propcache-0.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a544caaae1ac73f1fecfae70ded3e93728831affebd017d53449e3ac052ac1e", size = 215720, upload-time = "2025-06-09T22:55:00.471Z" }, - { url = "https://files.pythonhosted.org/packages/61/c1/d72ea2dc83ac7f2c8e182786ab0fc2c7bd123a1ff9b7975bee671866fe5f/propcache-0.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:310d11aa44635298397db47a3ebce7db99a4cc4b9bbdfcf6c98a60c8d5261cf1", size = 215203, upload-time = "2025-06-09T22:55:01.834Z" }, - { url = "https://files.pythonhosted.org/packages/af/81/b324c44ae60c56ef12007105f1460d5c304b0626ab0cc6b07c8f2a9aa0b8/propcache-0.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c1396592321ac83157ac03a2023aa6cc4a3cc3cfdecb71090054c09e5a7cce3", size = 206365, upload-time = "2025-06-09T22:55:03.199Z" }, - { url = "https://files.pythonhosted.org/packages/09/73/88549128bb89e66d2aff242488f62869014ae092db63ccea53c1cc75a81d/propcache-0.3.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cabf5b5902272565e78197edb682017d21cf3b550ba0460ee473753f28d23c1", size = 196016, upload-time = "2025-06-09T22:55:04.518Z" }, - { url = "https://files.pythonhosted.org/packages/b9/3f/3bdd14e737d145114a5eb83cb172903afba7242f67c5877f9909a20d948d/propcache-0.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0a2f2235ac46a7aa25bdeb03a9e7060f6ecbd213b1f9101c43b3090ffb971ef6", size = 205596, upload-time = "2025-06-09T22:55:05.942Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ca/2f4aa819c357d3107c3763d7ef42c03980f9ed5c48c82e01e25945d437c1/propcache-0.3.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:92b69e12e34869a6970fd2f3da91669899994b47c98f5d430b781c26f1d9f387", size = 200977, upload-time = "2025-06-09T22:55:07.792Z" }, - { url = "https://files.pythonhosted.org/packages/cd/4a/e65276c7477533c59085251ae88505caf6831c0e85ff8b2e31ebcbb949b1/propcache-0.3.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:54e02207c79968ebbdffc169591009f4474dde3b4679e16634d34c9363ff56b4", size = 197220, upload-time = "2025-06-09T22:55:09.173Z" }, - { url = "https://files.pythonhosted.org/packages/7c/54/fc7152e517cf5578278b242396ce4d4b36795423988ef39bb8cd5bf274c8/propcache-0.3.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4adfb44cb588001f68c5466579d3f1157ca07f7504fc91ec87862e2b8e556b88", size = 210642, upload-time = "2025-06-09T22:55:10.62Z" }, - { url = "https://files.pythonhosted.org/packages/b9/80/abeb4a896d2767bf5f1ea7b92eb7be6a5330645bd7fb844049c0e4045d9d/propcache-0.3.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fd3e6019dc1261cd0291ee8919dd91fbab7b169bb76aeef6c716833a3f65d206", size = 212789, upload-time = "2025-06-09T22:55:12.029Z" }, - { url = "https://files.pythonhosted.org/packages/b3/db/ea12a49aa7b2b6d68a5da8293dcf50068d48d088100ac016ad92a6a780e6/propcache-0.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4c181cad81158d71c41a2bce88edce078458e2dd5ffee7eddd6b05da85079f43", size = 205880, upload-time = "2025-06-09T22:55:13.45Z" }, - { url = "https://files.pythonhosted.org/packages/d1/e5/9076a0bbbfb65d1198007059c65639dfd56266cf8e477a9707e4b1999ff4/propcache-0.3.2-cp313-cp313-win32.whl", hash = "sha256:8a08154613f2249519e549de2330cf8e2071c2887309a7b07fb56098f5170a02", size = 37220, upload-time = "2025-06-09T22:55:15.284Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f5/b369e026b09a26cd77aa88d8fffd69141d2ae00a2abaaf5380d2603f4b7f/propcache-0.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e41671f1594fc4ab0a6dec1351864713cb3a279910ae8b58f884a88a0a632c05", size = 40678, upload-time = "2025-06-09T22:55:16.445Z" }, - { url = "https://files.pythonhosted.org/packages/a4/3a/6ece377b55544941a08d03581c7bc400a3c8cd3c2865900a68d5de79e21f/propcache-0.3.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:9a3cf035bbaf035f109987d9d55dc90e4b0e36e04bbbb95af3055ef17194057b", size = 76560, upload-time = "2025-06-09T22:55:17.598Z" }, - { url = "https://files.pythonhosted.org/packages/0c/da/64a2bb16418740fa634b0e9c3d29edff1db07f56d3546ca2d86ddf0305e1/propcache-0.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:156c03d07dc1323d8dacaa221fbe028c5c70d16709cdd63502778e6c3ccca1b0", size = 44676, upload-time = "2025-06-09T22:55:18.922Z" }, - { url = "https://files.pythonhosted.org/packages/36/7b/f025e06ea51cb72c52fb87e9b395cced02786610b60a3ed51da8af017170/propcache-0.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74413c0ba02ba86f55cf60d18daab219f7e531620c15f1e23d95563f505efe7e", size = 44701, upload-time = "2025-06-09T22:55:20.106Z" }, - { url = "https://files.pythonhosted.org/packages/a4/00/faa1b1b7c3b74fc277f8642f32a4c72ba1d7b2de36d7cdfb676db7f4303e/propcache-0.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f066b437bb3fa39c58ff97ab2ca351db465157d68ed0440abecb21715eb24b28", size = 276934, upload-time = "2025-06-09T22:55:21.5Z" }, - { url = "https://files.pythonhosted.org/packages/74/ab/935beb6f1756e0476a4d5938ff44bf0d13a055fed880caf93859b4f1baf4/propcache-0.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1304b085c83067914721e7e9d9917d41ad87696bf70f0bc7dee450e9c71ad0a", size = 278316, upload-time = "2025-06-09T22:55:22.918Z" }, - { url = "https://files.pythonhosted.org/packages/f8/9d/994a5c1ce4389610838d1caec74bdf0e98b306c70314d46dbe4fcf21a3e2/propcache-0.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab50cef01b372763a13333b4e54021bdcb291fc9a8e2ccb9c2df98be51bcde6c", size = 282619, upload-time = "2025-06-09T22:55:24.651Z" }, - { url = "https://files.pythonhosted.org/packages/2b/00/a10afce3d1ed0287cef2e09506d3be9822513f2c1e96457ee369adb9a6cd/propcache-0.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad3b2a085ec259ad2c2842666b2a0a49dea8463579c606426128925af1ed725", size = 265896, upload-time = "2025-06-09T22:55:26.049Z" }, - { url = "https://files.pythonhosted.org/packages/2e/a8/2aa6716ffa566ca57c749edb909ad27884680887d68517e4be41b02299f3/propcache-0.3.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:261fa020c1c14deafd54c76b014956e2f86991af198c51139faf41c4d5e83892", size = 252111, upload-time = "2025-06-09T22:55:27.381Z" }, - { url = "https://files.pythonhosted.org/packages/36/4f/345ca9183b85ac29c8694b0941f7484bf419c7f0fea2d1e386b4f7893eed/propcache-0.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:46d7f8aa79c927e5f987ee3a80205c987717d3659f035c85cf0c3680526bdb44", size = 268334, upload-time = "2025-06-09T22:55:28.747Z" }, - { url = "https://files.pythonhosted.org/packages/3e/ca/fcd54f78b59e3f97b3b9715501e3147f5340167733d27db423aa321e7148/propcache-0.3.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:6d8f3f0eebf73e3c0ff0e7853f68be638b4043c65a70517bb575eff54edd8dbe", size = 255026, upload-time = "2025-06-09T22:55:30.184Z" }, - { url = "https://files.pythonhosted.org/packages/8b/95/8e6a6bbbd78ac89c30c225210a5c687790e532ba4088afb8c0445b77ef37/propcache-0.3.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:03c89c1b14a5452cf15403e291c0ccd7751d5b9736ecb2c5bab977ad6c5bcd81", size = 250724, upload-time = "2025-06-09T22:55:31.646Z" }, - { url = "https://files.pythonhosted.org/packages/ee/b0/0dd03616142baba28e8b2d14ce5df6631b4673850a3d4f9c0f9dd714a404/propcache-0.3.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:0cc17efde71e12bbaad086d679ce575268d70bc123a5a71ea7ad76f70ba30bba", size = 268868, upload-time = "2025-06-09T22:55:33.209Z" }, - { url = "https://files.pythonhosted.org/packages/c5/98/2c12407a7e4fbacd94ddd32f3b1e3d5231e77c30ef7162b12a60e2dd5ce3/propcache-0.3.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:acdf05d00696bc0447e278bb53cb04ca72354e562cf88ea6f9107df8e7fd9770", size = 271322, upload-time = "2025-06-09T22:55:35.065Z" }, - { url = "https://files.pythonhosted.org/packages/35/91/9cb56efbb428b006bb85db28591e40b7736847b8331d43fe335acf95f6c8/propcache-0.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4445542398bd0b5d32df908031cb1b30d43ac848e20470a878b770ec2dcc6330", size = 265778, upload-time = "2025-06-09T22:55:36.45Z" }, - { url = "https://files.pythonhosted.org/packages/9a/4c/b0fe775a2bdd01e176b14b574be679d84fc83958335790f7c9a686c1f468/propcache-0.3.2-cp313-cp313t-win32.whl", hash = "sha256:f86e5d7cd03afb3a1db8e9f9f6eff15794e79e791350ac48a8c924e6f439f394", size = 41175, upload-time = "2025-06-09T22:55:38.436Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ff/47f08595e3d9b5e149c150f88d9714574f1a7cbd89fe2817158a952674bf/propcache-0.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:9704bedf6e7cbe3c65eca4379a9b53ee6a83749f047808cbb5044d40d7d72198", size = 44857, upload-time = "2025-06-09T22:55:39.687Z" }, - { url = "https://files.pythonhosted.org/packages/cc/35/cc0aaecf278bb4575b8555f2b137de5ab821595ddae9da9d3cd1da4072c7/propcache-0.3.2-py3-none-any.whl", hash = "sha256:98f1ec44fb675f5052cccc8e609c46ed23a35a1cfd18545ad4e29002d858a43f", size = 12663, upload-time = "2025-06-09T22:56:04.484Z" }, -] - -[[package]] -name = "protobuf" -version = "6.32.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fa/a4/cc17347aa2897568beece2e674674359f911d6fe21b0b8d6268cd42727ac/protobuf-6.32.1.tar.gz", hash = "sha256:ee2469e4a021474ab9baafea6cd070e5bf27c7d29433504ddea1a4ee5850f68d", size = 440635, upload-time = "2025-09-11T21:38:42.935Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/be/8dd0a927c559b37d7a6c8ab79034fd167dcc1f851595f2e641ad62be8643/protobuf-6.32.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:2f5b80a49e1eb7b86d85fcd23fe92df154b9730a725c3b38c4e43b9d77018bf4", size = 322874, upload-time = "2025-09-11T21:38:35.509Z" }, - { url = "https://files.pythonhosted.org/packages/5c/f6/88d77011b605ef979aace37b7703e4eefad066f7e84d935e5a696515c2dd/protobuf-6.32.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:b1864818300c297265c83a4982fd3169f97122c299f56a56e2445c3698d34710", size = 322013, upload-time = "2025-09-11T21:38:37.017Z" }, - { url = "https://files.pythonhosted.org/packages/97/b7/15cc7d93443d6c6a84626ae3258a91f4c6ac8c0edd5df35ea7658f71b79c/protobuf-6.32.1-py3-none-any.whl", hash = "sha256:2601b779fc7d32a866c6b4404f9d42a3f67c5b9f3f15b4db3cccabe06b95c346", size = 169289, upload-time = "2025-09-11T21:38:41.234Z" }, +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/56/030b7b4719d53085722893e0009dffb9236aa10bca1b12121bdc5626ef16/propcache-0.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d5a81be28596d6559f6131ef33e10200de6e17643b3c74ce03f9eb103be6ae8b", size = 93417, upload-time = "2026-05-08T20:59:15.597Z" }, + { url = "https://files.pythonhosted.org/packages/1a/55/1140a8e067b8ec093a18a4ae7bb0045d9db65da38a08618ddc5e2f1994aa/propcache-0.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29cbaac5ea0212663e6845e04b5e188d5a6ae6dd919810ac835bf1d3b42c3f4c", size = 53847, upload-time = "2026-05-08T20:59:17.096Z" }, + { url = "https://files.pythonhosted.org/packages/20/42/0e7443c90310498561addf346e7d57fe3c6ba1914e1ba938b5464c7bbfd2/propcache-0.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6bf3be92233808fcd338eba0fb4d0b59ec5772af4f4ecfcec450d1bfc0f8b5eb", size = 53512, upload-time = "2026-05-08T20:59:18.64Z" }, + { url = "https://files.pythonhosted.org/packages/b7/db/cf51a71bab2009517d1a7f0ee07657e3bd446c4d69f67e6966cf17bcf956/propcache-0.5.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f8ea531c794b9d6274acd4e8d2c2ebcac590a4361d27482edd3010b79f1325e", size = 58068, upload-time = "2026-05-08T20:59:20.683Z" }, + { url = "https://files.pythonhosted.org/packages/b7/43/39b6bdee9699fa1e1641c519feeb64a67e2a9f93bb465c70776b37a7333f/propcache-0.5.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:decfca4c79dd53ebab484b00cc4b6717d8c369f86e74aa4ca395a64ac651495e", size = 61020, upload-time = "2026-05-08T20:59:22.112Z" }, + { url = "https://files.pythonhosted.org/packages/26/0b/843726fbb0a29a8c5684fdb25971823638399f31e52e9d1f06a02dc9aa6b/propcache-0.5.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4621064bbf28fa77ff64dd5d94367c04684c67d3a5bf1dff25f0cd0d98a38f3b", size = 62732, upload-time = "2026-05-08T20:59:23.805Z" }, + { url = "https://files.pythonhosted.org/packages/39/6e/899fed76dc1942b8a64193a4f059d7f1a2c7ef65085e8a9366ed8ec0d199/propcache-0.5.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b96db7141a592cbc968daf1feea83a118e6ab378af4abbc72b248c895414c22d", size = 60140, upload-time = "2026-05-08T20:59:25.389Z" }, + { url = "https://files.pythonhosted.org/packages/ab/09/3da4be9b5b879219ad234aa535b3dd4a080ed1ad48d3a73ca07a9e798f22/propcache-0.5.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1ca071adabaab6e9219924bbe00af821f1ee7de113a9eca1cdc292de3d120f4d", size = 60400, upload-time = "2026-05-08T20:59:27.238Z" }, + { url = "https://files.pythonhosted.org/packages/60/2f/09b72b874a9aa0044faf52a69807a6ed618e267ceaa9ec4a63195fa5b504/propcache-0.5.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e4294d04a94dcab1b3bccd8b66d962dcad411a1d19414b2a41d1445f1de32ad0", size = 58155, upload-time = "2026-05-08T20:59:28.48Z" }, + { url = "https://files.pythonhosted.org/packages/8a/37/97489848c54c95578045473954f10956d619ce6a09e7ac137b71cdcb698b/propcache-0.5.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:a0e399a2eccb91ed18721f86aa85757727400b6865c89e88934781deb9c8498b", size = 57037, upload-time = "2026-05-08T20:59:30.146Z" }, + { url = "https://files.pythonhosted.org/packages/22/db/6c695285ccfc49012743ee9c98212b8c5dd0aed7b63cfd816d4a0f7a1601/propcache-0.5.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:823581fd5cb08b12a48bfa11fe962a7916766b6170c17b028fbdf762b85eb9bf", size = 61103, upload-time = "2026-05-08T20:59:31.626Z" }, + { url = "https://files.pythonhosted.org/packages/98/a9/1e500401ca593b0bdb6bf75a70bc2d723835fd53360edff6af70692c7546/propcache-0.5.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:949c91d1a990cf3b2e8188dfcfb25005e0b834a06c63fa4ef9f360878ce21ecf", size = 60394, upload-time = "2026-05-08T20:59:32.829Z" }, + { url = "https://files.pythonhosted.org/packages/1f/87/f638b6e375eae0f30a1a2325d8b34fd85fdc785bb9960cf805f3bf1ec69a/propcache-0.5.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:cc1177027eda740fdb152706bd215a3f124e3eea15afc39f2cb9fe351b50619e", size = 63084, upload-time = "2026-05-08T20:59:35.964Z" }, + { url = "https://files.pythonhosted.org/packages/f6/18/884573f5d97b6d9eba68de759a82c901b7e39d7904d30f7b8d58d42d2a12/propcache-0.5.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b05d643f944a8c3c4bd86d65ffd87bf3264b617f87791940302bc474d2ff5274", size = 60999, upload-time = "2026-05-08T20:59:38.481Z" }, + { url = "https://files.pythonhosted.org/packages/8f/1a/c3915eb059ceec9e758a56e4cfd955292bc0f201be2176a46b76d94b303a/propcache-0.5.2-cp310-cp310-win32.whl", hash = "sha256:8114f28879e0904748e831c3a7774261bd9e75f49be089f389a76f959dcd13fe", size = 39036, upload-time = "2026-05-08T20:59:40.323Z" }, + { url = "https://files.pythonhosted.org/packages/5b/02/1dfd5607501a602d19c1c449d2d193b7d1c611f9246b4059026a1189a80e/propcache-0.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:5fcb98e7598b1ee0addab320d90f65b530297a867dbfe9de52ea838077e16e3d", size = 42190, upload-time = "2026-05-08T20:59:42.232Z" }, + { url = "https://files.pythonhosted.org/packages/57/93/f71588ad08b3e6f4b555b5ef215808a3c02b042d0151ad82fa6f15be677a/propcache-0.5.2-cp310-cp310-win_arm64.whl", hash = "sha256:04dc2390d9edbbaef7461f33322555976ffddf0b650a038649d026358714e6c5", size = 38545, upload-time = "2026-05-08T20:59:44.087Z" }, + { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744, upload-time = "2026-05-08T20:59:45.799Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033, upload-time = "2026-05-08T20:59:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754, upload-time = "2026-05-08T20:59:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573, upload-time = "2026-05-08T20:59:50.778Z" }, + { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645, upload-time = "2026-05-08T20:59:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563, upload-time = "2026-05-08T20:59:53.866Z" }, + { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888, upload-time = "2026-05-08T20:59:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253, upload-time = "2026-05-08T20:59:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558, upload-time = "2026-05-08T20:59:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007, upload-time = "2026-05-08T20:59:59.837Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355, upload-time = "2026-05-08T21:00:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057, upload-time = "2026-05-08T21:00:02.401Z" }, + { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938, upload-time = "2026-05-08T21:00:03.638Z" }, + { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731, upload-time = "2026-05-08T21:00:04.881Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966, upload-time = "2026-05-08T21:00:06.511Z" }, + { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135, upload-time = "2026-05-08T21:00:08.088Z" }, + { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381, upload-time = "2026-05-08T21:00:09.692Z" }, + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] [[package]] @@ -2079,64 +2206,57 @@ wheels = [ [[package]] name = "pyarrow" -version = "24.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" }, - { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" }, - { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" }, - { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" }, - { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" }, - { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" }, - { url = "https://files.pythonhosted.org/packages/3a/ba/464cc70761c2a525d97ebd84e21c31ebd47f3ef4bdcee117009f51c46f24/pyarrow-24.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:c42ab9439498270139cc63e18847a02afe5c8b3ed9c931266533cfe378bd3591", size = 27251730, upload-time = "2026-04-21T10:46:30.913Z" }, - { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, - { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" }, - { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, - { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, - { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, - { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, - { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, - { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, - { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, - { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, - { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, - { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, - { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, - { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, - { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, - { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, - { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, - { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, - { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, - { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, - { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, - { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, - { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, - { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, - { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, - { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, - { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, - { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, - { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, - { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, - { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, - { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, - { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, - { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, - { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, - { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, - { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +version = "25.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/3e/5cd70becb51e1d044c54ba5e627424a6e87df5b98008cbd22cc6abd409ca/pyarrow-25.0.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:0b1edbb2f385a6a65e9711b62ba86ac54a7816a3f8d17bb3e8a5929d65fb2485", size = 35954271, upload-time = "2026-08-10T12:36:33.857Z" }, + { url = "https://files.pythonhosted.org/packages/64/be/17599e086df264ea7dc221d1101e3131e181e00da428a2f9bd0358f0d06b/pyarrow-25.0.1-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:a4dd8bf99a8fac133efc0ed6a92f5fddbe2adba0d0f6dd720e39ba9855cea85c", size = 37647543, upload-time = "2026-08-10T12:36:39.486Z" }, + { url = "https://files.pythonhosted.org/packages/42/34/e138b451fd3970a6eda4599f68ae3b2b32b661bc958de3239d54a0bf6575/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:bddd0c4f7630c2a3ddf6347c1bdaa79d97bcf6bd445f9e60c816b7d77c85a5ae", size = 46837120, upload-time = "2026-08-10T12:36:46.58Z" }, + { url = "https://files.pythonhosted.org/packages/57/5c/f8fc0eb2de03464a557d5a4d0c15e972d73362414696618833b771f7eddd/pyarrow-25.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a4d6d5e9a3d1879a97c08ded0c797579b7965eafd0f0c26c30b45ccc06db939b", size = 50066460, upload-time = "2026-08-10T12:36:53.702Z" }, + { url = "https://files.pythonhosted.org/packages/3f/d1/0dd64fd06de0333b808a02f60981635f067b71aad3a30698a9a104fae778/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:514ddb60285631af068875550c90eddc181db3e8e63a032b1559be189e82f056", size = 49937892, upload-time = "2026-08-10T12:37:00.349Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3c/f89d1bd76d5f3284c2a44d7d7ebbd8204535e5ae2b41f4077069b4ff2ec6/pyarrow-25.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:cab40b1edfef0262e0e5251aa2c58d75630f24d06dd7794480243acc001a1d7d", size = 53107240, upload-time = "2026-08-10T12:37:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/b554a8e09f3f3decccf405eb8fbe86696321cbcb5b62d18b4a5057a4c113/pyarrow-25.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:60e89d8f13861a1f7f8d950fa54aebb8023b30734d0ac51ffa80beabe2df4bba", size = 27848683, upload-time = "2026-08-10T12:37:12.058Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8b/0d23b47702fcfe8b3618d5292035099675c5a1c48258932350c08020f7b5/pyarrow-25.0.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:51093dd9e10325fbdb3c10a2ae7c4806e5c822d94e74ae4938b26524a3323fee", size = 35946180, upload-time = "2026-08-10T12:37:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/d8/17/707d17a5476c55a9541fde0db8213ac30979a792864d72415f176ba50c45/pyarrow-25.0.1-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:eb6203482ff3746a5632303a7279ae0b5a304c46985b49ed1378cb350ea6728d", size = 37644787, upload-time = "2026-08-10T12:37:25.795Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b2/cdc98ecf1a6408280bc3a6a07054cdd99a3f4670acc0545d383ce113e87d/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:880523be3d29efcf83d3998835d206118ccf35e3871dbd2fb60408cf6b007a80", size = 46834633, upload-time = "2026-08-10T12:37:33.604Z" }, + { url = "https://files.pythonhosted.org/packages/c8/6e/d3fafc41f378b2c65be43b827798c0fae42049a641c8526633ed3eb573e2/pyarrow-25.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:25f8720bf6387d5dc2ebd2622112de630760419e4b66134405dd24110d15f37e", size = 50065507, upload-time = "2026-08-10T12:37:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/d5/12/8d0698954b8c3001844a898e0a6900bebe83d7ee40c11195174c5122f324/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4facd65742a024a4a366328a1d2292062d72d6e023c1b7dda8d4c37544933a25", size = 49955690, upload-time = "2026-08-10T12:37:46.644Z" }, + { url = "https://files.pythonhosted.org/packages/d3/0b/1ecb936ac6409e90a34d58eea1c7cec09a9ae6d2141b9e49ad01a2b1ea47/pyarrow-25.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa0559502e1cd6254d6814614085dd9c5a3dd0419362978a936a3f68a9e5c3df", size = 53128198, upload-time = "2026-08-10T12:37:52.531Z" }, + { url = "https://files.pythonhosted.org/packages/8e/1c/5236033550633c9b7377b2a53660b2bbb06cb06dc09c4356332d67643ca1/pyarrow-25.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:62cd0d785b8aa6675ee355f9fc02252a340f4441257c42674937826fd7594325", size = 27857263, upload-time = "2026-08-10T12:37:56.943Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e2/9ab15b88cbfac28e16419ce5439ec29234c5172cb8259301b4ba639bdec0/pyarrow-25.0.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:df961f2e7ae9cf496459259d798652c70625f6c080650d6952f8c04053c58ee9", size = 35861559, upload-time = "2026-08-10T12:38:02.567Z" }, + { url = "https://files.pythonhosted.org/packages/58/79/a0036dbe1eabe1f73127427342f1d99982584c4a2cde2651d6c93499c6f6/pyarrow-25.0.1-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:cc4aa407fde9fc660be3939e49ea31f50f3e9fec17c0ec63159f7711edd3efc9", size = 37628383, upload-time = "2026-08-10T12:38:09.083Z" }, + { url = "https://files.pythonhosted.org/packages/13/49/d93a57d375f4bf0cf82913dd6bb54acafde83dd993be2282c81ac5616cad/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:4340f0ba6c1d2e13f21658de1d7c662ca2545018568d0030a1e9afca159d87e3", size = 46820190, upload-time = "2026-08-10T12:38:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/60/c9/711ca85d79f1ec98f29a5eae2b051e25b4ecec5de3e3c0e2d5c5dcb15664/pyarrow-25.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5389cdf79447ed1515c9e31620e6e1e2302249564d603f2ad727d4f6d313e4c3", size = 50102437, upload-time = "2026-08-10T12:38:22.487Z" }, + { url = "https://files.pythonhosted.org/packages/80/53/8fb8359ff17cfb6263a1cf3ebf7caec9fe197de118719e84fcb1d0618026/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d51592cb7561e87877c506113e7adbf1342ab579e6c21f0ef44b8ba41cb74c80", size = 49942424, upload-time = "2026-08-10T12:38:28.755Z" }, + { url = "https://files.pythonhosted.org/packages/e8/83/4e5ae02a9341571b18a6fca380ac7a58ce6ddae7ab3c060208c0a1e79f02/pyarrow-25.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6109c94d8b9f3b17a041daca16cacb2f651ad8f1ef70a4232c2c0f37a23da2a8", size = 53144206, upload-time = "2026-08-10T12:38:34.862Z" }, + { url = "https://files.pythonhosted.org/packages/65/ee/197cbf47e49f83e6ebeb946a5259a48a638dea27ac774db42fe78022179d/pyarrow-25.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:8858d7bfc22e3f51529aeaa4077225029724623e4595dc9eff8c793935c34140", size = 27953934, upload-time = "2026-08-10T12:38:39.808Z" }, + { url = "https://files.pythonhosted.org/packages/cc/8d/8f271a7a034c834910ec925d56fa4b29733b1380f5289419f5aaa3b02777/pyarrow-25.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c7c534ec03c358a76ea3e505e74c1b6aef290af90c444dfd092dbfe23e755b85", size = 35855328, upload-time = "2026-08-10T12:38:45.489Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/5bac242f4e841b9971d5eb94fdfe2577e2b70be983e27401e72055786037/pyarrow-25.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dda9470024204d7bbf2042b47c6e8a0e47a3eeb8e34405882dfaea6577e0c153", size = 37622415, upload-time = "2026-08-10T12:38:51.107Z" }, + { url = "https://files.pythonhosted.org/packages/63/1f/96d03b4e1506524f7087adb0fd6b2f69f0c9c7aaff1ec36d8030082e15a5/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:44a9120ce5bd81936b8ab9a88076e3fd47c2c6838e0e43630fed83626aca81d9", size = 46813813, upload-time = "2026-08-10T12:38:57.773Z" }, + { url = "https://files.pythonhosted.org/packages/98/d6/33a411115b61dbfc16ad6ad73e71730f6fea654ee3667673bc53ab0e2fe7/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f", size = 50104452, upload-time = "2026-08-10T12:39:04.579Z" }, + { url = "https://files.pythonhosted.org/packages/33/ae/b1b97c9ca87f9f9ddbb5230c798df94eccce61bd79b9b45458c69a478588/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f89685964f46e4216103c75483aac0c0692a5f72212d7ca835adba5ede56ce3", size = 49951343, upload-time = "2026-08-10T12:39:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/98/9e/a112df5cfd5a68cb1d9fc31cfe38c28d5aec9f10865ce37ecef2e4450873/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6943e2fe7954d29d84de45d29d34c8dc36ce96570e67d89aa9976e650a4a9138", size = 53144784, upload-time = "2026-08-10T12:39:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/31/24/97e8bd98f1e3b07e2ba08bcdff690674fbe16d69a7d2712cc3884665e615/pyarrow-25.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:31e49a7888fcdf3a835da33ae777f6bb9a866334e5a789282fc26dcf426f7f15", size = 27870159, upload-time = "2026-08-10T12:39:26.161Z" }, + { url = "https://files.pythonhosted.org/packages/36/4c/b525824ad3094076919273cd97db61fb3d78252dee76fa3b8dc8f76774aa/pyarrow-25.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:bf0b672390cdcb640d7288f96b826d71ff4e9abb254a86c89890baf51a29cee6", size = 35885255, upload-time = "2026-08-10T12:39:32.366Z" }, + { url = "https://files.pythonhosted.org/packages/08/62/448bb0e940de41aec31d1a956e63ad9c54afdf122a103cc3ab20c2a3ce33/pyarrow-25.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:38a9a4b4b9613380e200641891495a56c3d5a98a092db4a870af9975e220471d", size = 37644461, upload-time = "2026-08-10T12:39:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9a/13587e38bd4806fd218f50fd13b8903fab60588a699ff0c406372e5b4043/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b", size = 46877146, upload-time = "2026-08-10T12:39:43.722Z" }, + { url = "https://files.pythonhosted.org/packages/8d/61/1c5d1229fa21da4cff5365e41e57177aaac57c563c727f35419b8513d1c1/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:9171748cdf796972d85a4b60157c279913e242992e350c90c7450182a9838b2a", size = 50131616, upload-time = "2026-08-10T12:39:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/43/20/291e1d65cc0b09aa19f03cf25cf51a2f5fa94b5db315178f2d254ed5cad4/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b7a296aac7a71fa0886c08e155ddb6c636a50013f801f6178daafa0f9e726188", size = 50008879, upload-time = "2026-08-10T12:39:56.891Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7c/1b7c9ec28e76576337e4f97b31141c9a181b89b6d1d6221e9d8205621a58/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0fe7c8b6c03969b49c8c66182e4a18e3819ab92d07cfab5d8370c531b9369ef0", size = 53170864, upload-time = "2026-08-10T12:40:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/75/f3d789dc06011a765d14d86bda799cf72ac1d715b6a6edecaa0d73d95062/pyarrow-25.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f", size = 28620729, upload-time = "2026-08-10T12:40:51.41Z" }, + { url = "https://files.pythonhosted.org/packages/fc/05/647a8ee6f7c2662feb6921315617bc04dcd6034763fb61b1199720bf6162/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:59a2de54c0cbd954da861eee4d1d330f8e909c45b53455baef696380f2c55033", size = 36130288, upload-time = "2026-08-10T12:40:11.014Z" }, + { url = "https://files.pythonhosted.org/packages/93/f8/c9ee997554d7bea94520667dd1933f109ac1da3ee3556d2b49381e023484/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:35935cd5de130aa5cf4dea052a63e6bf2e17006c35c3a468194242b9b2bf5956", size = 37762187, upload-time = "2026-08-10T12:40:16.592Z" }, + { url = "https://files.pythonhosted.org/packages/a2/08/a28c01c7fe9e96e8233ce2d13df1d402f4f999f848f51d2daacd6bb4c036/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44", size = 46888003, upload-time = "2026-08-10T12:40:23.242Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b9/58612e977d28dc58c878448866838369ee8da2f1e7cc8ed2c84b952aafee/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a1fdfc6659b6b19022f2e50627fb5cf7156a66c46bf4299379955cbe742382a", size = 50079036, upload-time = "2026-08-10T12:40:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/72/13/66e1402dcc860e1dc2760b1e0292c9a569b62b3bccab69def1b3e907d006/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:169d3429d5be7c752125890620f75a60776d38b0035eddae939651640822332e", size = 50040226, upload-time = "2026-08-10T12:40:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/78/10/3f1a5497a7ef732ab0f03ecca3e66d89d9c0f57fdc61b4794c456b781f01/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:119297a6dc197e45d9c6d4415f7814a67ffa36c180d26f68c154c58067ae782d", size = 53149035, upload-time = "2026-08-10T12:40:41.454Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, ] [[package]] name = "pydantic" -version = "2.12.4" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -2144,136 +2264,134 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/ad/a17bc283d7d81837c061c49e3eaa27a45991759a1b7eae1031921c6bd924/pydantic-2.12.4.tar.gz", hash = "sha256:0f8cb9555000a4b5b617f66bfd2566264c4984b27589d3b845685983e8ea85ac", size = 821038, upload-time = "2025-11-05T10:50:08.59Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/2f/e68750da9b04856e2a7ec56fc6f034a5a79775e9b9a81882252789873798/pydantic-2.12.4-py3-none-any.whl", hash = "sha256:92d3d202a745d46f9be6df459ac5a064fdaa3c1c4cd8adcfa332ccf3c05f871e", size = 463400, upload-time = "2025-11-05T10:50:06.732Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [[package]] name = "pydantic-core" -version = "2.41.5" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/90/32c9941e728d564b411d574d8ee0cf09b12ec978cb22b294995bae5549a5/pydantic_core-2.41.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77b63866ca88d804225eaa4af3e664c5faf3568cea95360d21f4725ab6e07146", size = 2107298, upload-time = "2025-11-04T13:39:04.116Z" }, - { url = "https://files.pythonhosted.org/packages/fb/a8/61c96a77fe28993d9a6fb0f4127e05430a267b235a124545d79fea46dd65/pydantic_core-2.41.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dfa8a0c812ac681395907e71e1274819dec685fec28273a28905df579ef137e2", size = 1901475, upload-time = "2025-11-04T13:39:06.055Z" }, - { url = "https://files.pythonhosted.org/packages/5d/b6/338abf60225acc18cdc08b4faef592d0310923d19a87fba1faf05af5346e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5921a4d3ca3aee735d9fd163808f5e8dd6c6972101e4adbda9a4667908849b97", size = 1918815, upload-time = "2025-11-04T13:39:10.41Z" }, - { url = "https://files.pythonhosted.org/packages/d1/1c/2ed0433e682983d8e8cba9c8d8ef274d4791ec6a6f24c58935b90e780e0a/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25c479382d26a2a41b7ebea1043564a937db462816ea07afa8a44c0866d52f9", size = 2065567, upload-time = "2025-11-04T13:39:12.244Z" }, - { url = "https://files.pythonhosted.org/packages/b3/24/cf84974ee7d6eae06b9e63289b7b8f6549d416b5c199ca2d7ce13bbcf619/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f547144f2966e1e16ae626d8ce72b4cfa0caedc7fa28052001c94fb2fcaa1c52", size = 2230442, upload-time = "2025-11-04T13:39:13.962Z" }, - { url = "https://files.pythonhosted.org/packages/fd/21/4e287865504b3edc0136c89c9c09431be326168b1eb7841911cbc877a995/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f52298fbd394f9ed112d56f3d11aabd0d5bd27beb3084cc3d8ad069483b8941", size = 2350956, upload-time = "2025-11-04T13:39:15.889Z" }, - { url = "https://files.pythonhosted.org/packages/a8/76/7727ef2ffa4b62fcab916686a68a0426b9b790139720e1934e8ba797e238/pydantic_core-2.41.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:100baa204bb412b74fe285fb0f3a385256dad1d1879f0a5cb1499ed2e83d132a", size = 2068253, upload-time = "2025-11-04T13:39:17.403Z" }, - { url = "https://files.pythonhosted.org/packages/d5/8c/a4abfc79604bcb4c748e18975c44f94f756f08fb04218d5cb87eb0d3a63e/pydantic_core-2.41.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:05a2c8852530ad2812cb7914dc61a1125dc4e06252ee98e5638a12da6cc6fb6c", size = 2177050, upload-time = "2025-11-04T13:39:19.351Z" }, - { url = "https://files.pythonhosted.org/packages/67/b1/de2e9a9a79b480f9cb0b6e8b6ba4c50b18d4e89852426364c66aa82bb7b3/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29452c56df2ed968d18d7e21f4ab0ac55e71dc59524872f6fc57dcf4a3249ed2", size = 2147178, upload-time = "2025-11-04T13:39:21Z" }, - { url = "https://files.pythonhosted.org/packages/16/c1/dfb33f837a47b20417500efaa0378adc6635b3c79e8369ff7a03c494b4ac/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:d5160812ea7a8a2ffbe233d8da666880cad0cbaf5d4de74ae15c313213d62556", size = 2341833, upload-time = "2025-11-04T13:39:22.606Z" }, - { url = "https://files.pythonhosted.org/packages/47/36/00f398642a0f4b815a9a558c4f1dca1b4020a7d49562807d7bc9ff279a6c/pydantic_core-2.41.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:df3959765b553b9440adfd3c795617c352154e497a4eaf3752555cfb5da8fc49", size = 2321156, upload-time = "2025-11-04T13:39:25.843Z" }, - { url = "https://files.pythonhosted.org/packages/7e/70/cad3acd89fde2010807354d978725ae111ddf6d0ea46d1ea1775b5c1bd0c/pydantic_core-2.41.5-cp310-cp310-win32.whl", hash = "sha256:1f8d33a7f4d5a7889e60dc39856d76d09333d8a6ed0f5f1190635cbec70ec4ba", size = 1989378, upload-time = "2025-11-04T13:39:27.92Z" }, - { url = "https://files.pythonhosted.org/packages/76/92/d338652464c6c367e5608e4488201702cd1cbb0f33f7b6a85a60fe5f3720/pydantic_core-2.41.5-cp310-cp310-win_amd64.whl", hash = "sha256:62de39db01b8d593e45871af2af9e497295db8d73b085f6bfd0b18c83c70a8f9", size = 2013622, upload-time = "2025-11-04T13:39:29.848Z" }, - { url = "https://files.pythonhosted.org/packages/e8/72/74a989dd9f2084b3d9530b0915fdda64ac48831c30dbf7c72a41a5232db8/pydantic_core-2.41.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a3a52f6156e73e7ccb0f8cced536adccb7042be67cb45f9562e12b319c119da6", size = 2105873, upload-time = "2025-11-04T13:39:31.373Z" }, - { url = "https://files.pythonhosted.org/packages/12/44/37e403fd9455708b3b942949e1d7febc02167662bf1a7da5b78ee1ea2842/pydantic_core-2.41.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7f3bf998340c6d4b0c9a2f02d6a400e51f123b59565d74dc60d252ce888c260b", size = 1899826, upload-time = "2025-11-04T13:39:32.897Z" }, - { url = "https://files.pythonhosted.org/packages/33/7f/1d5cab3ccf44c1935a359d51a8a2a9e1a654b744b5e7f80d41b88d501eec/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:378bec5c66998815d224c9ca994f1e14c0c21cb95d2f52b6021cc0b2a58f2a5a", size = 1917869, upload-time = "2025-11-04T13:39:34.469Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6a/30d94a9674a7fe4f4744052ed6c5e083424510be1e93da5bc47569d11810/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b576130c69225432866fe2f4a469a85a54ade141d96fd396dffcf607b558f8", size = 2063890, upload-time = "2025-11-04T13:39:36.053Z" }, - { url = "https://files.pythonhosted.org/packages/50/be/76e5d46203fcb2750e542f32e6c371ffa9b8ad17364cf94bb0818dbfb50c/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6cb58b9c66f7e4179a2d5e0f849c48eff5c1fca560994d6eb6543abf955a149e", size = 2229740, upload-time = "2025-11-04T13:39:37.753Z" }, - { url = "https://files.pythonhosted.org/packages/d3/ee/fed784df0144793489f87db310a6bbf8118d7b630ed07aa180d6067e653a/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88942d3a3dff3afc8288c21e565e476fc278902ae4d6d134f1eeda118cc830b1", size = 2350021, upload-time = "2025-11-04T13:39:40.94Z" }, - { url = "https://files.pythonhosted.org/packages/c8/be/8fed28dd0a180dca19e72c233cbf58efa36df055e5b9d90d64fd1740b828/pydantic_core-2.41.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f31d95a179f8d64d90f6831d71fa93290893a33148d890ba15de25642c5d075b", size = 2066378, upload-time = "2025-11-04T13:39:42.523Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3b/698cf8ae1d536a010e05121b4958b1257f0b5522085e335360e53a6b1c8b/pydantic_core-2.41.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c1df3d34aced70add6f867a8cf413e299177e0c22660cc767218373d0779487b", size = 2175761, upload-time = "2025-11-04T13:39:44.553Z" }, - { url = "https://files.pythonhosted.org/packages/b8/ba/15d537423939553116dea94ce02f9c31be0fa9d0b806d427e0308ec17145/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4009935984bd36bd2c774e13f9a09563ce8de4abaa7226f5108262fa3e637284", size = 2146303, upload-time = "2025-11-04T13:39:46.238Z" }, - { url = "https://files.pythonhosted.org/packages/58/7f/0de669bf37d206723795f9c90c82966726a2ab06c336deba4735b55af431/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:34a64bc3441dc1213096a20fe27e8e128bd3ff89921706e83c0b1ac971276594", size = 2340355, upload-time = "2025-11-04T13:39:48.002Z" }, - { url = "https://files.pythonhosted.org/packages/e5/de/e7482c435b83d7e3c3ee5ee4451f6e8973cff0eb6007d2872ce6383f6398/pydantic_core-2.41.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c9e19dd6e28fdcaa5a1de679aec4141f691023916427ef9bae8584f9c2fb3b0e", size = 2319875, upload-time = "2025-11-04T13:39:49.705Z" }, - { url = "https://files.pythonhosted.org/packages/fe/e6/8c9e81bb6dd7560e33b9053351c29f30c8194b72f2d6932888581f503482/pydantic_core-2.41.5-cp311-cp311-win32.whl", hash = "sha256:2c010c6ded393148374c0f6f0bf89d206bf3217f201faa0635dcd56bd1520f6b", size = 1987549, upload-time = "2025-11-04T13:39:51.842Z" }, - { url = "https://files.pythonhosted.org/packages/11/66/f14d1d978ea94d1bc21fc98fcf570f9542fe55bfcc40269d4e1a21c19bf7/pydantic_core-2.41.5-cp311-cp311-win_amd64.whl", hash = "sha256:76ee27c6e9c7f16f47db7a94157112a2f3a00e958bc626e2f4ee8bec5c328fbe", size = 2011305, upload-time = "2025-11-04T13:39:53.485Z" }, - { url = "https://files.pythonhosted.org/packages/56/d8/0e271434e8efd03186c5386671328154ee349ff0354d83c74f5caaf096ed/pydantic_core-2.41.5-cp311-cp311-win_arm64.whl", hash = "sha256:4bc36bbc0b7584de96561184ad7f012478987882ebf9f9c389b23f432ea3d90f", size = 1972902, upload-time = "2025-11-04T13:39:56.488Z" }, - { url = "https://files.pythonhosted.org/packages/5f/5d/5f6c63eebb5afee93bcaae4ce9a898f3373ca23df3ccaef086d0233a35a7/pydantic_core-2.41.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f41a7489d32336dbf2199c8c0a215390a751c5b014c2c1c5366e817202e9cdf7", size = 2110990, upload-time = "2025-11-04T13:39:58.079Z" }, - { url = "https://files.pythonhosted.org/packages/aa/32/9c2e8ccb57c01111e0fd091f236c7b371c1bccea0fa85247ac55b1e2b6b6/pydantic_core-2.41.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:070259a8818988b9a84a449a2a7337c7f430a22acc0859c6b110aa7212a6d9c0", size = 1896003, upload-time = "2025-11-04T13:39:59.956Z" }, - { url = "https://files.pythonhosted.org/packages/68/b8/a01b53cb0e59139fbc9e4fda3e9724ede8de279097179be4ff31f1abb65a/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e96cea19e34778f8d59fe40775a7a574d95816eb150850a85a7a4c8f4b94ac69", size = 1919200, upload-time = "2025-11-04T13:40:02.241Z" }, - { url = "https://files.pythonhosted.org/packages/38/de/8c36b5198a29bdaade07b5985e80a233a5ac27137846f3bc2d3b40a47360/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed2e99c456e3fadd05c991f8f437ef902e00eedf34320ba2b0842bd1c3ca3a75", size = 2052578, upload-time = "2025-11-04T13:40:04.401Z" }, - { url = "https://files.pythonhosted.org/packages/00/b5/0e8e4b5b081eac6cb3dbb7e60a65907549a1ce035a724368c330112adfdd/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:65840751b72fbfd82c3c640cff9284545342a4f1eb1586ad0636955b261b0b05", size = 2208504, upload-time = "2025-11-04T13:40:06.072Z" }, - { url = "https://files.pythonhosted.org/packages/77/56/87a61aad59c7c5b9dc8caad5a41a5545cba3810c3e828708b3d7404f6cef/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e536c98a7626a98feb2d3eaf75944ef6f3dbee447e1f841eae16f2f0a72d8ddc", size = 2335816, upload-time = "2025-11-04T13:40:07.835Z" }, - { url = "https://files.pythonhosted.org/packages/0d/76/941cc9f73529988688a665a5c0ecff1112b3d95ab48f81db5f7606f522d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eceb81a8d74f9267ef4081e246ffd6d129da5d87e37a77c9bde550cb04870c1c", size = 2075366, upload-time = "2025-11-04T13:40:09.804Z" }, - { url = "https://files.pythonhosted.org/packages/d3/43/ebef01f69baa07a482844faaa0a591bad1ef129253ffd0cdaa9d8a7f72d3/pydantic_core-2.41.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d38548150c39b74aeeb0ce8ee1d8e82696f4a4e16ddc6de7b1d8823f7de4b9b5", size = 2171698, upload-time = "2025-11-04T13:40:12.004Z" }, - { url = "https://files.pythonhosted.org/packages/b1/87/41f3202e4193e3bacfc2c065fab7706ebe81af46a83d3e27605029c1f5a6/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c23e27686783f60290e36827f9c626e63154b82b116d7fe9adba1fda36da706c", size = 2132603, upload-time = "2025-11-04T13:40:13.868Z" }, - { url = "https://files.pythonhosted.org/packages/49/7d/4c00df99cb12070b6bccdef4a195255e6020a550d572768d92cc54dba91a/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:482c982f814460eabe1d3bb0adfdc583387bd4691ef00b90575ca0d2b6fe2294", size = 2329591, upload-time = "2025-11-04T13:40:15.672Z" }, - { url = "https://files.pythonhosted.org/packages/cc/6a/ebf4b1d65d458f3cda6a7335d141305dfa19bdc61140a884d165a8a1bbc7/pydantic_core-2.41.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:bfea2a5f0b4d8d43adf9d7b8bf019fb46fdd10a2e5cde477fbcb9d1fa08c68e1", size = 2319068, upload-time = "2025-11-04T13:40:17.532Z" }, - { url = "https://files.pythonhosted.org/packages/49/3b/774f2b5cd4192d5ab75870ce4381fd89cf218af999515baf07e7206753f0/pydantic_core-2.41.5-cp312-cp312-win32.whl", hash = "sha256:b74557b16e390ec12dca509bce9264c3bbd128f8a2c376eaa68003d7f327276d", size = 1985908, upload-time = "2025-11-04T13:40:19.309Z" }, - { url = "https://files.pythonhosted.org/packages/86/45/00173a033c801cacf67c190fef088789394feaf88a98a7035b0e40d53dc9/pydantic_core-2.41.5-cp312-cp312-win_amd64.whl", hash = "sha256:1962293292865bca8e54702b08a4f26da73adc83dd1fcf26fbc875b35d81c815", size = 2020145, upload-time = "2025-11-04T13:40:21.548Z" }, - { url = "https://files.pythonhosted.org/packages/f9/22/91fbc821fa6d261b376a3f73809f907cec5ca6025642c463d3488aad22fb/pydantic_core-2.41.5-cp312-cp312-win_arm64.whl", hash = "sha256:1746d4a3d9a794cacae06a5eaaccb4b8643a131d45fbc9af23e353dc0a5ba5c3", size = 1976179, upload-time = "2025-11-04T13:40:23.393Z" }, - { url = "https://files.pythonhosted.org/packages/87/06/8806241ff1f70d9939f9af039c6c35f2360cf16e93c2ca76f184e76b1564/pydantic_core-2.41.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:941103c9be18ac8daf7b7adca8228f8ed6bb7a1849020f643b3a14d15b1924d9", size = 2120403, upload-time = "2025-11-04T13:40:25.248Z" }, - { url = "https://files.pythonhosted.org/packages/94/02/abfa0e0bda67faa65fef1c84971c7e45928e108fe24333c81f3bfe35d5f5/pydantic_core-2.41.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:112e305c3314f40c93998e567879e887a3160bb8689ef3d2c04b6cc62c33ac34", size = 1896206, upload-time = "2025-11-04T13:40:27.099Z" }, - { url = "https://files.pythonhosted.org/packages/15/df/a4c740c0943e93e6500f9eb23f4ca7ec9bf71b19e608ae5b579678c8d02f/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbaad15cb0c90aa221d43c00e77bb33c93e8d36e0bf74760cd00e732d10a6a0", size = 1919307, upload-time = "2025-11-04T13:40:29.806Z" }, - { url = "https://files.pythonhosted.org/packages/9a/e3/6324802931ae1d123528988e0e86587c2072ac2e5394b4bc2bc34b61ff6e/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:03ca43e12fab6023fc79d28ca6b39b05f794ad08ec2feccc59a339b02f2b3d33", size = 2063258, upload-time = "2025-11-04T13:40:33.544Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d4/2230d7151d4957dd79c3044ea26346c148c98fbf0ee6ebd41056f2d62ab5/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc799088c08fa04e43144b164feb0c13f9a0bc40503f8df3e9fde58a3c0c101e", size = 2214917, upload-time = "2025-11-04T13:40:35.479Z" }, - { url = "https://files.pythonhosted.org/packages/e6/9f/eaac5df17a3672fef0081b6c1bb0b82b33ee89aa5cec0d7b05f52fd4a1fa/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:97aeba56665b4c3235a0e52b2c2f5ae9cd071b8a8310ad27bddb3f7fb30e9aa2", size = 2332186, upload-time = "2025-11-04T13:40:37.436Z" }, - { url = "https://files.pythonhosted.org/packages/cf/4e/35a80cae583a37cf15604b44240e45c05e04e86f9cfd766623149297e971/pydantic_core-2.41.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:406bf18d345822d6c21366031003612b9c77b3e29ffdb0f612367352aab7d586", size = 2073164, upload-time = "2025-11-04T13:40:40.289Z" }, - { url = "https://files.pythonhosted.org/packages/bf/e3/f6e262673c6140dd3305d144d032f7bd5f7497d3871c1428521f19f9efa2/pydantic_core-2.41.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b93590ae81f7010dbe380cdeab6f515902ebcbefe0b9327cc4804d74e93ae69d", size = 2179146, upload-time = "2025-11-04T13:40:42.809Z" }, - { url = "https://files.pythonhosted.org/packages/75/c7/20bd7fc05f0c6ea2056a4565c6f36f8968c0924f19b7d97bbfea55780e73/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:01a3d0ab748ee531f4ea6c3e48ad9dac84ddba4b0d82291f87248f2f9de8d740", size = 2137788, upload-time = "2025-11-04T13:40:44.752Z" }, - { url = "https://files.pythonhosted.org/packages/3a/8d/34318ef985c45196e004bc46c6eab2eda437e744c124ef0dbe1ff2c9d06b/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:6561e94ba9dacc9c61bce40e2d6bdc3bfaa0259d3ff36ace3b1e6901936d2e3e", size = 2340133, upload-time = "2025-11-04T13:40:46.66Z" }, - { url = "https://files.pythonhosted.org/packages/9c/59/013626bf8c78a5a5d9350d12e7697d3d4de951a75565496abd40ccd46bee/pydantic_core-2.41.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:915c3d10f81bec3a74fbd4faebe8391013ba61e5a1a8d48c4455b923bdda7858", size = 2324852, upload-time = "2025-11-04T13:40:48.575Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d9/c248c103856f807ef70c18a4f986693a46a8ffe1602e5d361485da502d20/pydantic_core-2.41.5-cp313-cp313-win32.whl", hash = "sha256:650ae77860b45cfa6e2cdafc42618ceafab3a2d9a3811fcfbd3bbf8ac3c40d36", size = 1994679, upload-time = "2025-11-04T13:40:50.619Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8b/341991b158ddab181cff136acd2552c9f35bd30380422a639c0671e99a91/pydantic_core-2.41.5-cp313-cp313-win_amd64.whl", hash = "sha256:79ec52ec461e99e13791ec6508c722742ad745571f234ea6255bed38c6480f11", size = 2019766, upload-time = "2025-11-04T13:40:52.631Z" }, - { url = "https://files.pythonhosted.org/packages/73/7d/f2f9db34af103bea3e09735bb40b021788a5e834c81eedb541991badf8f5/pydantic_core-2.41.5-cp313-cp313-win_arm64.whl", hash = "sha256:3f84d5c1b4ab906093bdc1ff10484838aca54ef08de4afa9de0f5f14d69639cd", size = 1981005, upload-time = "2025-11-04T13:40:54.734Z" }, - { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, - { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, - { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, - { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, - { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, - { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, - { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, - { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, - { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, - { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, - { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, - { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, - { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, - { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, - { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, - { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, - { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, - { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, - { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, - { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, - { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, - { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, - { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, - { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, - { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, - { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, - { url = "https://files.pythonhosted.org/packages/11/72/90fda5ee3b97e51c494938a4a44c3a35a9c96c19bba12372fb9c634d6f57/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:b96d5f26b05d03cc60f11a7761a5ded1741da411e7fe0909e27a5e6a0cb7b034", size = 2115441, upload-time = "2025-11-04T13:42:39.557Z" }, - { url = "https://files.pythonhosted.org/packages/1f/53/8942f884fa33f50794f119012dc6a1a02ac43a56407adaac20463df8e98f/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:634e8609e89ceecea15e2d61bc9ac3718caaaa71963717bf3c8f38bfde64242c", size = 1930291, upload-time = "2025-11-04T13:42:42.169Z" }, - { url = "https://files.pythonhosted.org/packages/79/c8/ecb9ed9cd942bce09fc888ee960b52654fbdbede4ba6c2d6e0d3b1d8b49c/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:93e8740d7503eb008aa2df04d3b9735f845d43ae845e6dcd2be0b55a2da43cd2", size = 1948632, upload-time = "2025-11-04T13:42:44.564Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1b/687711069de7efa6af934e74f601e2a4307365e8fdc404703afc453eab26/pydantic_core-2.41.5-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f15489ba13d61f670dcc96772e733aad1a6f9c429cc27574c6cdaed82d0146ad", size = 2138905, upload-time = "2025-11-04T13:42:47.156Z" }, - { url = "https://files.pythonhosted.org/packages/09/32/59b0c7e63e277fa7911c2fc70ccfb45ce4b98991e7ef37110663437005af/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:7da7087d756b19037bc2c06edc6c170eeef3c3bafcb8f532ff17d64dc427adfd", size = 2110495, upload-time = "2025-11-04T13:42:49.689Z" }, - { url = "https://files.pythonhosted.org/packages/aa/81/05e400037eaf55ad400bcd318c05bb345b57e708887f07ddb2d20e3f0e98/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:aabf5777b5c8ca26f7824cb4a120a740c9588ed58df9b2d196ce92fba42ff8dc", size = 1915388, upload-time = "2025-11-04T13:42:52.215Z" }, - { url = "https://files.pythonhosted.org/packages/6e/0d/e3549b2399f71d56476b77dbf3cf8937cec5cd70536bdc0e374a421d0599/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c007fe8a43d43b3969e8469004e9845944f1a80e6acd47c150856bb87f230c56", size = 1942879, upload-time = "2025-11-04T13:42:56.483Z" }, - { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017, upload-time = "2025-11-04T13:42:59.471Z" }, - { url = "https://files.pythonhosted.org/packages/e6/b0/1a2aa41e3b5a4ba11420aba2d091b2d17959c8d1519ece3627c371951e73/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b5819cd790dbf0c5eb9f82c73c16b39a65dd6dd4d1439dcdea7816ec9adddab8", size = 2103351, upload-time = "2025-11-04T13:43:02.058Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ee/31b1f0020baaf6d091c87900ae05c6aeae101fa4e188e1613c80e4f1ea31/pydantic_core-2.41.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:5a4e67afbc95fa5c34cf27d9089bca7fcab4e51e57278d710320a70b956d1b9a", size = 1925363, upload-time = "2025-11-04T13:43:05.159Z" }, - { url = "https://files.pythonhosted.org/packages/e1/89/ab8e86208467e467a80deaca4e434adac37b10a9d134cd2f99b28a01e483/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ece5c59f0ce7d001e017643d8d24da587ea1f74f6993467d85ae8a5ef9d4f42b", size = 2135615, upload-time = "2025-11-04T13:43:08.116Z" }, - { url = "https://files.pythonhosted.org/packages/99/0a/99a53d06dd0348b2008f2f30884b34719c323f16c3be4e6cc1203b74a91d/pydantic_core-2.41.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:16f80f7abe3351f8ea6858914ddc8c77e02578544a0ebc15b4c2e1a0e813b0b2", size = 2175369, upload-time = "2025-11-04T13:43:12.49Z" }, - { url = "https://files.pythonhosted.org/packages/6d/94/30ca3b73c6d485b9bb0bc66e611cff4a7138ff9736b7e66bcf0852151636/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:33cb885e759a705b426baada1fe68cbb0a2e68e34c5d0d0289a364cf01709093", size = 2144218, upload-time = "2025-11-04T13:43:15.431Z" }, - { url = "https://files.pythonhosted.org/packages/87/57/31b4f8e12680b739a91f472b5671294236b82586889ef764b5fbc6669238/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:c8d8b4eb992936023be7dee581270af5c6e0697a8559895f527f5b7105ecd36a", size = 2329951, upload-time = "2025-11-04T13:43:18.062Z" }, - { url = "https://files.pythonhosted.org/packages/7d/73/3c2c8edef77b8f7310e6fb012dbc4b8551386ed575b9eb6fb2506e28a7eb/pydantic_core-2.41.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:242a206cd0318f95cd21bdacff3fcc3aab23e79bba5cac3db5a841c9ef9c6963", size = 2318428, upload-time = "2025-11-04T13:43:20.679Z" }, - { url = "https://files.pythonhosted.org/packages/2f/02/8559b1f26ee0d502c74f9cca5c0d2fd97e967e083e006bbbb4e97f3a043a/pydantic_core-2.41.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d3a978c4f57a597908b7e697229d996d77a6d3c94901e9edee593adada95ce1a", size = 2147009, upload-time = "2025-11-04T13:43:23.286Z" }, - { url = "https://files.pythonhosted.org/packages/5f/9b/1b3f0e9f9305839d7e84912f9e8bfbd191ed1b1ef48083609f0dabde978c/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:b2379fa7ed44ddecb5bfe4e48577d752db9fc10be00a6b7446e9663ba143de26", size = 2101980, upload-time = "2025-11-04T13:43:25.97Z" }, - { url = "https://files.pythonhosted.org/packages/a4/ed/d71fefcb4263df0da6a85b5d8a7508360f2f2e9b3bf5814be9c8bccdccc1/pydantic_core-2.41.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:266fb4cbf5e3cbd0b53669a6d1b039c45e3ce651fd5442eff4d07c2cc8d66808", size = 1923865, upload-time = "2025-11-04T13:43:28.763Z" }, - { url = "https://files.pythonhosted.org/packages/ce/3a/626b38db460d675f873e4444b4bb030453bbe7b4ba55df821d026a0493c4/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58133647260ea01e4d0500089a8c4f07bd7aa6ce109682b1426394988d8aaacc", size = 2134256, upload-time = "2025-11-04T13:43:31.71Z" }, - { url = "https://files.pythonhosted.org/packages/83/d9/8412d7f06f616bbc053d30cb4e5f76786af3221462ad5eee1f202021eb4e/pydantic_core-2.41.5-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:287dad91cfb551c363dc62899a80e9e14da1f0e2b6ebde82c806612ca2a13ef1", size = 2174762, upload-time = "2025-11-04T13:43:34.744Z" }, - { url = "https://files.pythonhosted.org/packages/55/4c/162d906b8e3ba3a99354e20faa1b49a85206c47de97a639510a0e673f5da/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:03b77d184b9eb40240ae9fd676ca364ce1085f203e1b1256f8ab9984dca80a84", size = 2143141, upload-time = "2025-11-04T13:43:37.701Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f2/f11dd73284122713f5f89fc940f370d035fa8e1e078d446b3313955157fe/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:a668ce24de96165bb239160b3d854943128f4334822900534f2fe947930e5770", size = 2330317, upload-time = "2025-11-04T13:43:40.406Z" }, - { url = "https://files.pythonhosted.org/packages/88/9d/b06ca6acfe4abb296110fb1273a4d848a0bfb2ff65f3ee92127b3244e16b/pydantic_core-2.41.5-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f14f8f046c14563f8eb3f45f499cc658ab8d10072961e07225e507adb700e93f", size = 2316992, upload-time = "2025-11-04T13:43:43.602Z" }, - { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/08/f1ba952f1c8ae5581c70fa9c6da89f247b83e3dd8c09c035d5d7931fc23d/pydantic_core-2.46.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4", size = 2113146, upload-time = "2026-05-06T13:37:36.537Z" }, + { url = "https://files.pythonhosted.org/packages/56/c6/65f646c7ff09bd257f660434adb45c4dfcbbcebcc030562fecf6f5bf887d/pydantic_core-2.46.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5", size = 1949769, upload-time = "2026-05-06T13:37:46.365Z" }, + { url = "https://files.pythonhosted.org/packages/64/ba/bfb1d928fd5b49e1258935ff104ae356e9fd89384a55bf9f847e9193ad40/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba", size = 1974958, upload-time = "2026-05-06T13:37:28.611Z" }, + { url = "https://files.pythonhosted.org/packages/4e/74/76223bfb117b64af743c9b6670d1364516f5c0604f96b48f3272f6af6cc6/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b", size = 2042118, upload-time = "2026-05-06T13:36:55.216Z" }, + { url = "https://files.pythonhosted.org/packages/cb/7b/848732968bc8f48f3187542f08358b9d842db564147b256669426ebb1652/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c", size = 2222876, upload-time = "2026-05-06T13:38:25.455Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2f/e90b63ee2e14bd8d3db8f705a6d75d64e6ee1b7c2c8833747ce706e1e0ce/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50", size = 2286703, upload-time = "2026-05-06T13:37:53.304Z" }, + { url = "https://files.pythonhosted.org/packages/ba/1e/acc4d70f88a0a277e4a1fa77ebb985ceabaf900430f875bf9338e11c9420/pydantic_core-2.46.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd", size = 2092042, upload-time = "2026-05-06T13:38:46.981Z" }, + { url = "https://files.pythonhosted.org/packages/a9/da/0a422b57bf8504102bf3c4ccea9c41bab5a5cee6a54650acf8faf67f5a24/pydantic_core-2.46.4-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01", size = 2117231, upload-time = "2026-05-06T13:39:23.146Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2a/2ac13c3af305843e23c5078c53d135656b3f05a2fd78cb7bbbb12e97b473/pydantic_core-2.46.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d", size = 2168388, upload-time = "2026-05-06T13:40:08.06Z" }, + { url = "https://files.pythonhosted.org/packages/72/04/2beacf7e1607e93eefe4aed1b4709f079b905fb77530179d4f7c71745f22/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4", size = 2184769, upload-time = "2026-05-06T13:38:13.901Z" }, + { url = "https://files.pythonhosted.org/packages/9e/29/d2b9fd9f539133548eaf622c06a4ce176cb46ac59f32d0359c4abc0de047/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f", size = 2319312, upload-time = "2026-05-06T13:39:08.24Z" }, + { url = "https://files.pythonhosted.org/packages/7c/af/0f7a5b85fec6075bea96e3ef9187de38fccced0de92c1e7feda8d5cc7bb9/pydantic_core-2.46.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39", size = 2361817, upload-time = "2026-05-06T13:38:43.2Z" }, + { url = "https://files.pythonhosted.org/packages/25/a4/73363fec545fd3ec025490bdda2743c56d0dd5b6266b1a53bbe9e4265375/pydantic_core-2.46.4-cp310-cp310-win32.whl", hash = "sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d", size = 1987085, upload-time = "2026-05-06T13:39:25.497Z" }, + { url = "https://files.pythonhosted.org/packages/01/aa/62f082da2c91fac1c234bc9ee0066257ce83f0604abd72e4c9d5991f2d84/pydantic_core-2.46.4-cp310-cp310-win_amd64.whl", hash = "sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf", size = 2074311, upload-time = "2026-05-06T13:39:59.922Z" }, + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872, upload-time = "2026-05-06T13:40:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255, upload-time = "2026-05-06T13:39:12.574Z" }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827, upload-time = "2026-05-06T13:38:19.841Z" }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051, upload-time = "2026-05-06T13:38:10.447Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314, upload-time = "2026-05-06T13:40:13.089Z" }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146, upload-time = "2026-05-06T13:38:59.224Z" }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685, upload-time = "2026-05-06T13:38:17.762Z" }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420, upload-time = "2026-05-06T13:37:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122, upload-time = "2026-05-06T13:37:01.167Z" }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573, upload-time = "2026-05-06T13:38:45.04Z" }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139, upload-time = "2026-05-06T13:37:15.539Z" }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433, upload-time = "2026-05-06T13:37:30.099Z" }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513, upload-time = "2026-05-06T13:38:15.669Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114, upload-time = "2026-05-06T13:40:35.416Z" }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298, upload-time = "2026-05-06T13:38:29.754Z" }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158, upload-time = "2026-05-06T13:38:57.215Z" }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724, upload-time = "2026-05-06T13:37:02.697Z" }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742, upload-time = "2026-05-06T13:37:09.448Z" }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418, upload-time = "2026-05-06T13:37:38.234Z" }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274, upload-time = "2026-05-06T13:38:27.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940, upload-time = "2026-05-06T13:38:05.353Z" }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516, upload-time = "2026-05-06T13:39:10.577Z" }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854, upload-time = "2026-05-06T13:40:22.59Z" }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306, upload-time = "2026-05-06T13:40:10.666Z" }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044, upload-time = "2026-05-06T13:40:43.231Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133, upload-time = "2026-05-06T13:39:57.365Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464, upload-time = "2026-05-06T13:38:06.976Z" }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823, upload-time = "2026-05-06T13:40:47.985Z" }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919, upload-time = "2026-05-06T13:39:21.153Z" }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604, upload-time = "2026-05-06T13:39:03.753Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a4/73995fd4ebbb46ba0ee51e6fa049b8f02c40daebb762208feda8a6b7894d/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c", size = 2111589, upload-time = "2026-05-06T13:37:10.817Z" }, + { url = "https://files.pythonhosted.org/packages/fb/7f/f37d3a5e8bfcc2e403f5c57a730f2d815693fb42119e8ea48b3789335af1/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b", size = 1944552, upload-time = "2026-05-06T13:36:56.717Z" }, + { url = "https://files.pythonhosted.org/packages/15/3c/d7eb777b3ff43e8433a4efb39a17aa8fd98a4ee8561a24a67ef5db07b2d6/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b", size = 1982984, upload-time = "2026-05-06T13:39:06.207Z" }, + { url = "https://files.pythonhosted.org/packages/63/87/70b9f40170a81afd55ca26c9b2acb25c20d64bcfbf888fafecb3ba077d4c/pydantic_core-2.46.4-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea", size = 2138417, upload-time = "2026-05-06T13:39:45.476Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1d/8987ad40f65ae1432753072f214fb5c74fe47ffbd0698bb9cbbb585664f8/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7", size = 2095527, upload-time = "2026-05-06T13:39:52.283Z" }, + { url = "https://files.pythonhosted.org/packages/64/d3/84c282a7eee1d3ac4c0377546ef5a1ea436ce26840d9ac3b7ed54a377507/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df", size = 1936024, upload-time = "2026-05-06T13:40:15.671Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ca/eac61596cdeb4d7e174d3dc0bd8a6238f14f75f97a24e7b7db4c7e7340a0/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526", size = 1990696, upload-time = "2026-05-06T13:38:34.717Z" }, + { url = "https://files.pythonhosted.org/packages/fa/c3/7c8b240552251faf6b3a957db200fcfbbcec36763c050428b601e0c9b83b/pydantic_core-2.46.4-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0", size = 2147590, upload-time = "2026-05-06T13:39:29.883Z" }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782, upload-time = "2026-05-06T13:37:04.016Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146, upload-time = "2026-05-06T13:39:43.092Z" }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492, upload-time = "2026-05-06T13:36:58.124Z" }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604, upload-time = "2026-05-06T13:37:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828, upload-time = "2026-05-06T13:37:43.053Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000, upload-time = "2026-05-06T13:37:56.694Z" }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286, upload-time = "2026-05-06T13:40:05.667Z" }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] [[package]] name = "pygments" -version = "2.19.2" +version = "2.20.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] [[package]] @@ -2282,7 +2400,8 @@ source = { editable = "." } dependencies = [ { name = "lance-namespace" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "pyarrow" }, ] @@ -2298,6 +2417,10 @@ geo = [ { name = "geoarrow-rust-core" }, { name = "geoarrow-rust-io" }, ] +otel = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, +] tests = [ { name = "boto3" }, { name = "datafusion" }, @@ -2309,7 +2432,7 @@ tests = [ { name = "polars", extra = ["pandas", "pyarrow"] }, { name = "psutil" }, { name = "pytest" }, - { name = "tensorflow", marker = "sys_platform == 'linux'" }, + { name = "pytest-xdist" }, { name = "tqdm" }, ] torch = [ @@ -2331,26 +2454,29 @@ tests = [ { name = "datasets" }, { name = "duckdb" }, { name = "ml-dtypes" }, + { name = "opentelemetry-sdk" }, { name = "pandas" }, { name = "pillow" }, { name = "polars", extra = ["pandas", "pyarrow"] }, { name = "psutil" }, { name = "pytest" }, - { name = "tensorflow", marker = "sys_platform == 'linux'" }, + { name = "pytest-xdist" }, { name = "tqdm" }, ] [package.metadata] requires-dist = [ { name = "boto3", marker = "extra == 'tests'" }, - { name = "datafusion", marker = "extra == 'tests'", specifier = ">=53,<54" }, - { name = "datasets", marker = "extra == 'tests'" }, - { name = "duckdb", marker = "extra == 'tests'" }, + { name = "datafusion", marker = "extra == 'tests'", specifier = ">=54,<55" }, + { name = "datasets", marker = "extra == 'tests'", specifier = "==4.4.0" }, + { name = "duckdb", marker = "extra == 'tests'", specifier = ">=1.5.0,<1.6.0" }, { name = "geoarrow-rust-core", marker = "extra == 'geo'" }, { name = "geoarrow-rust-io", marker = "extra == 'geo'" }, - { name = "lance-namespace", specifier = ">=0.8.5,<0.9" }, + { name = "lance-namespace", specifier = ">=0.11.1,<0.12" }, { name = "ml-dtypes", marker = "extra == 'tests'" }, { name = "numpy", specifier = ">=1.22" }, + { name = "opentelemetry-api", marker = "extra == 'otel'" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'" }, { name = "pandas", marker = "extra == 'tests'" }, { name = "pillow", marker = "extra == 'tests'" }, { name = "polars", extras = ["pyarrow", "pandas"], marker = "extra == 'tests'" }, @@ -2359,12 +2485,12 @@ requires-dist = [ { name = "pyright", marker = "extra == 'dev'" }, { name = "pytest", marker = "extra == 'tests'" }, { name = "pytest-benchmark", marker = "extra == 'benchmarks'" }, + { name = "pytest-xdist", marker = "extra == 'tests'" }, { name = "ruff", marker = "extra == 'dev'", specifier = "==0.11.2" }, - { name = "tensorflow", marker = "sys_platform == 'linux' and extra == 'tests'" }, { name = "torch", marker = "extra == 'torch'", specifier = ">=2.0" }, { name = "tqdm", marker = "extra == 'tests'" }, ] -provides-extras = ["benchmarks", "dev", "geo", "tests", "torch"] +provides-extras = ["benchmarks", "dev", "geo", "otel", "tests", "torch"] [package.metadata.requires-dev] benchmarks = [{ name = "pytest-benchmark", specifier = "==5.1.0" }] @@ -2375,16 +2501,17 @@ dev = [ ] tests = [ { name = "boto3", specifier = "==1.40.43" }, - { name = "datafusion", specifier = "==53.0.0" }, - { name = "datasets", specifier = "==4.1.1" }, - { name = "duckdb", specifier = "==1.4.0" }, + { name = "datafusion", specifier = "==54.0.0" }, + { name = "datasets", specifier = "==4.4.0" }, + { name = "duckdb", specifier = ">=1.5.0,<1.6.0" }, { name = "ml-dtypes", specifier = "==0.5.3" }, + { name = "opentelemetry-sdk", specifier = "==1.30.0" }, { name = "pandas", specifier = "==2.3.3" }, { name = "pillow", specifier = "==11.3.0" }, { name = "polars", extras = ["pyarrow", "pandas"], specifier = "==1.34.0" }, { name = "psutil", specifier = "==7.1.0" }, { name = "pytest", specifier = "==8.4.2" }, - { name = "tensorflow", marker = "sys_platform == 'linux'", specifier = "==2.20.0" }, + { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "tqdm", specifier = "==4.67.1" }, ] @@ -2396,7 +2523,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "certifi", marker = "python_full_version < '3.11'" }, + { name = "certifi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/10/a8480ea27ea4bbe896c168808854d00f2a9b49f95c0319ddcbba693c8a90/pyproj-3.7.1.tar.gz", hash = "sha256:60d72facd7b6b79853f19744779abcd3f804c4e0d4fa8815469db20c9f640a47", size = 226339, upload-time = "2025-02-16T04:28:46.621Z" } wheels = [ @@ -2445,7 +2572,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.11'" }, + { name = "certifi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/90/67bd7260b4ea9b8b20b4f58afef6c223ecb3abf368eb4ec5bc2cdef81b49/pyproj-3.7.2.tar.gz", hash = "sha256:39a0cf1ecc7e282d1d30f36594ebd55c9fae1fda8a2622cee5d100430628f88c", size = 226279, upload-time = "2025-08-14T12:05:42.18Z" } wheels = [ @@ -2549,6 +2676,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -2563,11 +2703,11 @@ wheels = [ [[package]] name = "pytz" -version = "2025.2" +version = "2026.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/46/dd499ec9038423421951e4fad73051febaa13d2df82b4064f87af8b8c0c3/pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a", size = 320861, upload-time = "2026-05-04T01:35:29.667Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, + { url = "https://files.pythonhosted.org/packages/ec/dd/96da98f892250475bdf2328112d7468abdd4acc7b902b6af23f4ed958ea0/pytz-2026.2-py2.py3-none-any.whl", hash = "sha256:04156e608bee23d3792fd45c94ae47fae1036688e75032eea2e3bf0323d1f126", size = 510141, upload-time = "2026-05-04T01:35:27.408Z" }, ] [[package]] @@ -2636,7 +2776,7 @@ wheels = [ [[package]] name = "requests" -version = "2.33.0" +version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "certifi" }, @@ -2644,22 +2784,9 @@ dependencies = [ { name = "idna" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/64/8860370b167a9721e8956ae116825caff829224fbca0ca6e7bf8ddef8430/requests-2.33.0.tar.gz", hash = "sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652", size = 134232, upload-time = "2026-03-25T15:10:41.586Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, -] - -[[package]] -name = "rich" -version = "14.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fe/75/af448d8e52bf1d8fa6a9d089ca6c07ff4453d86c65c145d0a300bb073b9b/rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8", size = 224441, upload-time = "2025-07-25T07:32:58.125Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/30/3c4d035596d3cf444529e0b2953ad0466f6049528a879d27534700580395/rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f", size = 243368, upload-time = "2025-07-25T07:32:56.73Z" }, + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] [[package]] @@ -2701,11 +2828,11 @@ wheels = [ [[package]] name = "setuptools" -version = "80.9.0" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]] @@ -2729,170 +2856,106 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, ] -[[package]] -name = "tensorboard" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "grpcio" }, - { name = "markdown" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "protobuf" }, - { name = "setuptools" }, - { name = "tensorboard-data-server" }, - { name = "werkzeug" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680, upload-time = "2025-07-17T19:20:49.638Z" }, -] - -[[package]] -name = "tensorboard-data-server" -version = "0.7.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356, upload-time = "2023-10-23T21:23:32.16Z" }, - { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363, upload-time = "2023-10-23T21:23:35.583Z" }, -] - -[[package]] -name = "tensorflow" -version = "2.20.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "absl-py" }, - { name = "astunparse" }, - { name = "flatbuffers" }, - { name = "gast" }, - { name = "google-pasta" }, - { name = "grpcio" }, - { name = "h5py" }, - { name = "keras" }, - { name = "libclang" }, - { name = "ml-dtypes" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "opt-einsum" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "requests" }, - { name = "setuptools" }, - { name = "six" }, - { name = "tensorboard" }, - { name = "termcolor" }, - { name = "typing-extensions" }, - { name = "wrapt" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/07/ea91ac67a9fd36d3372099f5a3e69860ded544f877f5f2117802388f4212/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02a0293d94f5c8b7125b66abf622cc4854a33ae9d618a0d41309f95e091bbaea", size = 259307122, upload-time = "2025-08-13T16:50:47.909Z" }, - { url = "https://files.pythonhosted.org/packages/e5/9e/0d57922cf46b9e91de636cd5b5e0d7a424ebe98f3245380a713f1f6c2a0b/tensorflow-2.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7abd7f3a010e0d354dc804182372779a722d474c4d8a3db8f4a3f5baef2a591e", size = 620425510, upload-time = "2025-08-13T16:51:02.608Z" }, - { url = "https://files.pythonhosted.org/packages/f1/b7/a3d455db88ab5b35ce53ab885ec0dd9f28d905a86a2250423048bc8cafa0/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e9568c8efcb05c0266be223e3269c62ebf7ad3498f156438311735f6fa5ced5", size = 259465882, upload-time = "2025-08-13T16:51:39.546Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0c/7df285ee8a88139fab0b237003634d90690759fae9c18f55ddb7c04656ec/tensorflow-2.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:481499fd0f824583de8945be61d5e827898cdaa4f5ea1bc2cc28ca2ccff8229e", size = 620570129, upload-time = "2025-08-13T16:51:55.104Z" }, - { url = "https://files.pythonhosted.org/packages/ec/b4/f028a5de27d0fda10ba6145bc76e40c37ff6d2d1e95b601adb5ae17d635e/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bfbfb3dd0e22bffc45fe1e922390d27753e99261fab8a882e802cf98a0e078f", size = 259533109, upload-time = "2025-08-13T16:52:31.513Z" }, - { url = "https://files.pythonhosted.org/packages/9c/d1/6aa15085d672056d5f08b5f28b1c7ce01c4e12149a23b0c98e3c79d04441/tensorflow-2.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:25265b0bc527e0d54b1e9cc60c44a24f44a809fe27666b905f0466471f9c52ec", size = 620682547, upload-time = "2025-08-13T16:52:46.396Z" }, - { url = "https://files.pythonhosted.org/packages/ea/4c/c1aa90c5cc92e9f7f9c78421e121ef25bae7d378f8d1d4cbad46c6308836/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47c88e05a07f1ead4977b4894b3ecd4d8075c40191065afc4fd9355c9db3d926", size = 259663776, upload-time = "2025-08-13T16:53:24.507Z" }, - { url = "https://files.pythonhosted.org/packages/43/fb/8be8547c128613d82a2b006004026d86ed0bd672e913029a98153af4ffab/tensorflow-2.20.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fa3729b0126f75a99882b89fb7d536515721eda8014a63e259e780ba0a37372", size = 620815537, upload-time = "2025-08-13T16:53:42.577Z" }, -] - -[[package]] -name = "termcolor" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/6c/3d75c196ac07ac8749600b60b03f4f6094d54e132c4d94ebac6ee0e0add0/termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970", size = 14324, upload-time = "2025-04-30T11:37:53.791Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/bd/de8d508070629b6d84a30d01d57e4a65c69aa7f5abe7560b8fad3b50ea59/termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa", size = 7684, upload-time = "2025-04-30T11:37:52.382Z" }, -] - [[package]] name = "tomli" -version = "2.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, - { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, - { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, - { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, - { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, - { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, - { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, - { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, - { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, - { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, - { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, - { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, - { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, - { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, - { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, - { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, - { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, - { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, - { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, - { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, - { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, - { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, - { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, - { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, - { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, - { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, - { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, - { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, - { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, - { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] [[package]] name = "torch" -version = "2.12.1" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, { name = "filelock" }, { name = "fsspec" }, { name = "jinja2" }, { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, { name = "setuptools" }, { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/db/ed/ff0c4f8cef63977a646dc80e40c05cae873f4097b12dc87e1cd7e1cecf42/torch-2.12.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:ec56e82be6a8b0c036771a77f7d32ad3c299770571af9815b3dafe61434389d5", size = 87967927, upload-time = "2026-06-17T21:08:43.16Z" }, - { url = "https://files.pythonhosted.org/packages/85/1b/c8ecf60c9dba535f9ea341c359c600c0bd877a7ca14b3296f13316321847/torch-2.12.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:42cd7339bf266f14944710e8274be63e7e012bb937834a8d85a8327a9860eba6", size = 426366829, upload-time = "2026-06-17T21:07:18.574Z" }, - { url = "https://files.pythonhosted.org/packages/ab/d6/73d4a3f27e00526e98086f3a64ab609af1345cca62367749fbc3c8e4b83c/torch-2.12.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a7817f0f89a796d9de239d06f69faf5d7e19a6a5db6710a5ead777c912f9f50a", size = 532144834, upload-time = "2026-06-17T21:08:00.633Z" }, - { url = "https://files.pythonhosted.org/packages/e3/51/4010c8fa6f9d1f42c054a321970ca95ec58e4e4494f5b53a34c3f3c9e310/torch-2.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:2af3d9cc866e0a15ae7635ff0a9c61d6624a353ad657f5bcd8d86c26cdc64693", size = 122949863, upload-time = "2026-06-17T21:08:39.016Z" }, - { url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" }, - { url = "https://files.pythonhosted.org/packages/5a/e3/750b3e3548635ceac03ba255daa26dbc7ed66ca3484dc4b4d955ab7f4501/torch-2.12.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:107df6888624bdea41508f9aeb6149d9333c737a5530ceecb56c904e811369ae", size = 426379894, upload-time = "2026-06-17T21:06:55.077Z" }, - { url = "https://files.pythonhosted.org/packages/dc/ca/ed24783da629ff3e640ba3f70a7639e9045d3d88b93ee6bc47b8a28a1f2c/torch-2.12.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:6e29e7e74d05bda7d955c75e99459f878ebd970ef851b4057edbd3b34a5eb4a3", size = 532169264, upload-time = "2026-06-17T21:08:17.65Z" }, - { url = "https://files.pythonhosted.org/packages/46/61/c63f0158446f3a98ea672b004d761b848911eba567ea4a624c7db5aadc04/torch-2.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:a513506cfda3c1c78dabeb6574c1597538c0254b3d39af174dde35d8177f4ce3", size = 122953086, upload-time = "2026-06-17T21:08:27.69Z" }, - { url = "https://files.pythonhosted.org/packages/f0/54/efb7ebca77970012b0cc21687a55d70eb2ba514b2c2b8e18d9fb1222f3be/torch-2.12.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:d2dd0f2c5f7ccbddaf34cade0deaf476808368f902b9cdb7f36a2ab42301bc0e", size = 87991951, upload-time = "2026-06-17T21:07:49.309Z" }, - { url = "https://files.pythonhosted.org/packages/1e/00/4210d76ca7424981f04033ebe7e48816ab83287a62538747a58825db770c/torch-2.12.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2de4e19b88a481482c6c75291f2d6a52eda3ce51f311b29aa9b68499c830c07c", size = 426382721, upload-time = "2026-06-17T21:06:41.842Z" }, - { url = "https://files.pythonhosted.org/packages/76/1f/bc9f5a5aa569307076365f25afcebacb22e9c754b1bcfbaaa146627c7fda/torch-2.12.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:649e4ced014ba646f76f8cb9c9726735a6323eb321b7919f942790a923f90921", size = 532261322, upload-time = "2026-06-17T21:06:06.673Z" }, - { url = "https://files.pythonhosted.org/packages/9e/49/c549461daa008159d006a76a991fbc2f26fa8bac27a4030c858463dcb20f/torch-2.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:e86550597877fb272ddc52db2f85b82cb601ea7bd932576a0340152cae2200b3", size = 122988095, upload-time = "2026-06-17T21:07:44.9Z" }, - { url = "https://files.pythonhosted.org/packages/ff/4a/0300261818e1560d72cc160ac826005507e8b7ca0a35788b591436d05b4a/torch-2.12.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:c75e93173c700bccd6bfcc4a9d19ce242ab6dacd1f1781483027a16239b9e650", size = 87992358, upload-time = "2026-06-17T21:07:40.299Z" }, - { url = "https://files.pythonhosted.org/packages/30/a7/874a5ca05e8f159211dca7921060f7057acc1adb26431e119fd150623efc/torch-2.12.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:fcb61ccd20784b62bdd78ec84238a5cfb383b4994902e03bac95505ab360884c", size = 426386134, upload-time = "2026-06-17T21:07:31.481Z" }, - { url = "https://files.pythonhosted.org/packages/e1/75/20bb8fe9c1ad6538cce8cd0391b51927ae5af0b17ed1eab44b8824465dc1/torch-2.12.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:f4afc8083dff08719edbea346644476e3cec0cf40ebe256be0ee5d5b7c7e8c0d", size = 532268019, upload-time = "2026-06-17T21:05:37.925Z" }, - { url = "https://files.pythonhosted.org/packages/d1/fa/824ddb662af55b2eabc0dbb7b57c7c0b1bcd93693754a2b8509ec4d16490/torch-2.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:f92609e3b3ce72f25e2eb780d043ced2480c1a86c47c852604fc7a9108648386", size = 122987777, upload-time = "2026-06-17T21:07:09.49Z" }, - { url = "https://files.pythonhosted.org/packages/63/b7/1b49fe7086ea36839cc80abc43174c43d0ab6f676c0891c871c162f44fe3/torch-2.12.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:e9b6f7d2dd66ea87a3ae620069d31335d594c06effb1a383bdd21cfe61e44ece", size = 88010025, upload-time = "2026-06-17T21:07:03.934Z" }, - { url = "https://files.pythonhosted.org/packages/d7/06/5b44063a6545036dcc680d2d303b137d9176cfb2cc1e1863e3ef94abeb52/torch-2.12.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:7973ccd3d2cd35c74449213f7bded199bec6c6247e705cbeda7407af79703d91", size = 426392891, upload-time = "2026-06-17T21:05:52.261Z" }, - { url = "https://files.pythonhosted.org/packages/f8/dd/c9ce9a4b0eb3c5bb92d9ea56766e2c22559f0b45171149188494edcce80f/torch-2.12.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c64ac4aac16be5e296dcd912305605804b203333c690bf98c55bc09494ee92ad", size = 532272494, upload-time = "2026-06-17T21:06:22.72Z" }, - { url = "https://files.pythonhosted.org/packages/21/7c/f3a601fc1b1f663ff269bfe553654e638651939aa6563e8daa7167c33098/torch-2.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:f6dc4caf7eb4adb38a2d9f536b51db56310fdd1254e69a2d96767e1367c892b3", size = 122987254, upload-time = "2026-06-17T21:06:33.199Z" }, - { url = "https://files.pythonhosted.org/packages/e6/8c/b8087556cf81ddd808dbeb34afb8396d7ae7a1694ab489f08b1a0004e7d0/torch-2.12.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:2afbb2bdaa8a95040e733f05492ddf133c3967c9b7ce0abd218d704b6cab437d", size = 88303173, upload-time = "2026-06-17T21:05:06.603Z" }, - { url = "https://files.pythonhosted.org/packages/4a/07/fe09d1699fbed2afa10ebc692ff2b99d113f2605b6748cea633989e2789a/torch-2.12.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:97eba061fcb042fed191400b15568990073d67eaacaa6ee9b7ca01dd8b790fe9", size = 426404009, upload-time = "2026-06-17T21:04:57.557Z" }, - { url = "https://files.pythonhosted.org/packages/2e/f7/0ce4f6c1962c60ded7270e0a9eb560fb615c92b89d332cf9e3dff36d5ecc/torch-2.12.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:3867b861391701012adb2df93360efb88494dca245a185e3bb7624495cfe3f33", size = 532184292, upload-time = "2026-06-17T21:05:17.526Z" }, - { url = "https://files.pythonhosted.org/packages/70/db/e384c12aba30320ca92aaaf557456cbcb26f04b4df307728bb8f019f5000/torch-2.12.1-cp314-cp314t-win_amd64.whl", hash = "sha256:dd15595f8fc764cffde8c6361a3beb6ef69a028c851b1b3e70e077f615980d4e", size = 123231142, upload-time = "2026-06-17T21:05:27.061Z" }, + { url = "https://files.pythonhosted.org/packages/7f/e7/19894fdb51c7dbaf94f5a79bb0871da0992e8e4241e579cb006da46d2e58/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", size = 111178962, upload-time = "2026-07-08T16:05:49.855Z" }, + { url = "https://files.pythonhosted.org/packages/d1/5c/b1d5de470c54e339b30a92d96683a71bcebd78f5f2a7fc714cd6dc6bbd68/torch-2.13.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0ab4b69f3ee03a62a002cfbf77b1ca5e88aceb4ea64cb4388bb28f638ddbb045", size = 427198333, upload-time = "2026-07-08T16:05:36.847Z" }, + { url = "https://files.pythonhosted.org/packages/50/c0/68a84105e1fcb8970144b388ff3d3e5dc15a3be28c1e247841f7d7247e41/torch-2.13.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:c78b7b4d04461855a764cf01bae9a462bb88bc93defcfa11235cbc8fdf3e12c4", size = 526555154, upload-time = "2026-07-08T16:05:06.507Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c9/0bb9d097b03cbaf96bb75b15e867347b8e41bfcdfe0539452d17d9e63993/torch-2.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:2bd30b6b730d987fa386ce3898933762c5cb8cc82eb0535211d787cc3ce2dfeb", size = 122015602, upload-time = "2026-07-08T16:05:45.25Z" }, + { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" }, + { url = "https://files.pythonhosted.org/packages/c2/59/1e3160e18e12aa3038390efab3ce02b36a9d4d6a527ecdd8520dca2e68d8/torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c", size = 427199369, upload-time = "2026-07-08T16:04:51.054Z" }, + { url = "https://files.pythonhosted.org/packages/01/79/1f2d34ad7034ee1c7ffc1cf8bf0f8213af2a81df6ecdb3997ecec107c09d/torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7", size = 526574961, upload-time = "2026-07-08T16:04:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/0f2ce40f58aefbdb3392f9acce3c8171940943ae2d661f70558bfa73befb/torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330", size = 122015870, upload-time = "2026-07-08T16:05:27.59Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" }, + { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, + { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fa/c1c10b7aff4a9a3e8956d4f0a5f468fa6db7abc3208805719076772b4833/torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09", size = 111213743, upload-time = "2026-07-08T16:03:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" }, + { url = "https://files.pythonhosted.org/packages/91/3d/e7adcc6aaf36961cd18f56cf8ad0f3058c3a5c84ccf391762176c94581b8/torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6", size = 122057920, upload-time = "2026-07-08T16:03:01.808Z" }, + { url = "https://files.pythonhosted.org/packages/36/76/6dcc7f0c07052102dd36f83cbc5800842a909c8c3fbf1a7f8a5844954de9/torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c", size = 111227066, upload-time = "2026-07-08T16:03:33.6Z" }, + { url = "https://files.pythonhosted.org/packages/e9/09/2c10e8cd0e00fa5d23c052df6ce467eaa7182399f5e0f824f1e4ff42ccae/torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c", size = 427226309, upload-time = "2026-07-08T16:02:53.127Z" }, + { url = "https://files.pythonhosted.org/packages/76/c6/22c2102bbef14ca6a6cb4c20e42f088e49c5f812be4e160ae57502e325f9/torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2", size = 526614507, upload-time = "2026-07-08T16:02:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0c/7d1deb6bce5bc3e6042caf39100ac768eba3b9a098e1dddd16f75bd6489b/torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd", size = 122051871, upload-time = "2026-07-08T16:03:23.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ce/aa8b7f9949d32e0f2f624f342bc3b48112c1b8a130288465938bc83bcbf9/torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1", size = 111537025, upload-time = "2026-07-08T16:02:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/69/d1/491e3a0389430946145888b0203f2b6a759ce2a61481b96a85c2da4f2ced/torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc", size = 427219769, upload-time = "2026-07-08T16:02:31.18Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1d/38006e045bf0a1fc28ef01e757c554e59e59a8770c284bc4f47b14e60441/torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92", size = 526571320, upload-time = "2026-07-08T16:01:59.348Z" }, + { url = "https://files.pythonhosted.org/packages/56/94/655c91992a882bd5071aa0b5d22a07dbb130d801e872be97c0b627a7c693/torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8", size = 122306773, upload-time = "2026-07-08T16:02:39.832Z" }, ] [[package]] @@ -2928,11 +2991,11 @@ wheels = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, ] [[package]] @@ -2949,289 +3012,386 @@ wheels = [ [[package]] name = "tzdata" -version = "2025.2" +version = "2026.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, ] [[package]] name = "urllib3" -version = "2.5.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/15/22/9ee70a2574a4f4599c47dd506532914ce044817c7752a79b6a51286319bc/urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", size = 393185, upload-time = "2025-06-18T14:07:41.644Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/c2/fe1e52489ae3122415c51f387e221dd0773709bad6c6cdaa599e8a2c5185/urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc", size = 129795, upload-time = "2025-06-18T14:07:40.39Z" }, -] - -[[package]] -name = "werkzeug" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" }, -] - -[[package]] -name = "wheel" -version = "0.45.1" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8a/98/2d9906746cdc6a6ef809ae6338005b3f21bb568bea3165cfc6a243fdc25c/wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729", size = 107545, upload-time = "2024-11-23T00:18:23.513Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/2c/87f3254fd8ffd29e4c02732eee68a83a1d3c346ae39bc6822dcbcb697f2b/wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248", size = 72494, upload-time = "2024-11-23T00:18:21.207Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]] name = "wrapt" -version = "1.17.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/8f/aeb76c5b46e273670962298c23e7ddde79916cb74db802131d49a85e4b7d/wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0", size = 55547, upload-time = "2025-08-12T05:53:21.714Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/de/17/9f8f86755c191d6779d7ddead1a53c7a8aa18bccb7cea8e7e72dfa6a8a09/wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775", size = 81975, upload-time = "2025-08-12T05:52:30.109Z" }, - { url = "https://files.pythonhosted.org/packages/f2/15/dd576273491f9f43dd09fce517f6c2ce6eb4fe21681726068db0d0467096/wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd", size = 83149, upload-time = "2025-08-12T05:52:09.316Z" }, - { url = "https://files.pythonhosted.org/packages/0c/c4/5eb4ce0d4814521fee7aa806264bf7a114e748ad05110441cd5b8a5c744b/wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05", size = 82209, upload-time = "2025-08-12T05:52:10.331Z" }, - { url = "https://files.pythonhosted.org/packages/31/4b/819e9e0eb5c8dc86f60dfc42aa4e2c0d6c3db8732bce93cc752e604bb5f5/wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418", size = 81551, upload-time = "2025-08-12T05:52:31.137Z" }, - { url = "https://files.pythonhosted.org/packages/5d/8f/a32a99fc03e4b37e31b57cb9cefc65050ea08147a8ce12f288616b05ef54/wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311", size = 82376, upload-time = "2025-08-12T05:52:32.134Z" }, - { url = "https://files.pythonhosted.org/packages/31/57/4930cb8d9d70d59c27ee1332a318c20291749b4fba31f113c2f8ac49a72e/wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1", size = 83604, upload-time = "2025-08-12T05:52:11.663Z" }, - { url = "https://files.pythonhosted.org/packages/a8/f3/1afd48de81d63dd66e01b263a6fbb86e1b5053b419b9b33d13e1f6d0f7d0/wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5", size = 82782, upload-time = "2025-08-12T05:52:12.626Z" }, - { url = "https://files.pythonhosted.org/packages/1e/d7/4ad5327612173b144998232f98a85bb24b60c352afb73bc48e3e0d2bdc4e/wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2", size = 82076, upload-time = "2025-08-12T05:52:33.168Z" }, - { url = "https://files.pythonhosted.org/packages/9f/81/5d931d78d0eb732b95dc3ddaeeb71c8bb572fb01356e9133916cd729ecdd/wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828", size = 88036, upload-time = "2025-08-12T05:52:34.784Z" }, - { url = "https://files.pythonhosted.org/packages/ca/38/2e1785df03b3d72d34fc6252d91d9d12dc27a5c89caef3335a1bbb8908ca/wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9", size = 88156, upload-time = "2025-08-12T05:52:13.599Z" }, - { url = "https://files.pythonhosted.org/packages/b3/8b/48cdb60fe0603e34e05cffda0b2a4adab81fd43718e11111a4b0100fd7c1/wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396", size = 87102, upload-time = "2025-08-12T05:52:14.56Z" }, - { url = "https://files.pythonhosted.org/packages/3c/51/d81abca783b58f40a154f1b2c56db1d2d9e0d04fa2d4224e357529f57a57/wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc", size = 87732, upload-time = "2025-08-12T05:52:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/0c/37/6faf15cfa41bf1f3dba80cd3f5ccc6622dfccb660ab26ed79f0178c7497f/wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277", size = 88072, upload-time = "2025-08-12T05:52:37.53Z" }, - { url = "https://files.pythonhosted.org/packages/78/f2/efe19ada4a38e4e15b6dff39c3e3f3f73f5decf901f66e6f72fe79623a06/wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d", size = 88214, upload-time = "2025-08-12T05:52:15.886Z" }, - { url = "https://files.pythonhosted.org/packages/40/90/ca86701e9de1622b16e09689fc24b76f69b06bb0150990f6f4e8b0eeb576/wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa", size = 87105, upload-time = "2025-08-12T05:52:17.914Z" }, - { url = "https://files.pythonhosted.org/packages/fd/e0/d10bd257c9a3e15cbf5523025252cc14d77468e8ed644aafb2d6f54cb95d/wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050", size = 87766, upload-time = "2025-08-12T05:52:39.243Z" }, - { url = "https://files.pythonhosted.org/packages/c3/f7/c983d2762bcce2326c317c26a6a1e7016f7eb039c27cdf5c4e30f4160f31/wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b", size = 87163, upload-time = "2025-08-12T05:52:40.965Z" }, - { url = "https://files.pythonhosted.org/packages/e4/0f/f673f75d489c7f22d17fe0193e84b41540d962f75fce579cf6873167c29b/wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa", size = 87963, upload-time = "2025-08-12T05:52:20.326Z" }, - { url = "https://files.pythonhosted.org/packages/df/61/515ad6caca68995da2fac7a6af97faab8f78ebe3bf4f761e1b77efbc47b5/wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7", size = 86945, upload-time = "2025-08-12T05:52:21.581Z" }, - { url = "https://files.pythonhosted.org/packages/d3/bd/4e70162ce398462a467bc09e768bee112f1412e563620adc353de9055d33/wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4", size = 86857, upload-time = "2025-08-12T05:52:43.043Z" }, - { url = "https://files.pythonhosted.org/packages/64/0e/f4472f2fdde2d4617975144311f8800ef73677a159be7fe61fa50997d6c0/wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e", size = 108571, upload-time = "2025-08-12T05:52:44.521Z" }, - { url = "https://files.pythonhosted.org/packages/cc/01/9b85a99996b0a97c8a17484684f206cbb6ba73c1ce6890ac668bcf3838fb/wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f", size = 113094, upload-time = "2025-08-12T05:52:22.618Z" }, - { url = "https://files.pythonhosted.org/packages/25/02/78926c1efddcc7b3aa0bc3d6b33a822f7d898059f7cd9ace8c8318e559ef/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056", size = 110659, upload-time = "2025-08-12T05:52:24.057Z" }, - { url = "https://files.pythonhosted.org/packages/dc/ee/c414501ad518ac3e6fe184753632fe5e5ecacdcf0effc23f31c1e4f7bfcf/wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804", size = 106946, upload-time = "2025-08-12T05:52:45.976Z" }, - { url = "https://files.pythonhosted.org/packages/1f/f6/a933bd70f98e9cf3e08167fc5cd7aaaca49147e48411c0bd5ae701bb2194/wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22", size = 23591, upload-time = "2025-08-12T05:53:20.674Z" }, +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/a4/282c8e64300a59fc834518a54bf0afabb4ff9218b5fa76958b450459a844/wrapt-2.2.2.tar.gz", hash = "sha256:0788e321027c999bf221b667bd4a54aaefd1a36283749a860ac3eb77daed0302", size = 129068, upload-time = "2026-06-20T23:49:44.49Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/8b/59781d0fe7b0adfbea37f600857de4be68921e454aeecf1a11bda35cdccc/wrapt-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:055e6fcfaa28e58c6a8c247d48b92be9d56f818b7068aa4f22b15b3343a09931", size = 80556, upload-time = "2026-06-20T23:47:28.473Z" }, + { url = "https://files.pythonhosted.org/packages/94/dc/66c61aca927230c9cf97a3cb005c803971a1076ff9f7d61085d035c20085/wrapt-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8374eb6b1a58809211e84ff835a182bb17ab2807a5bfef23204c8cff38178a00", size = 81648, upload-time = "2026-06-20T23:47:30.504Z" }, + { url = "https://files.pythonhosted.org/packages/23/1b/545eee1c18f3af4cf140bb5822b6ef81ebe569df0a63ac109973103a30a5/wrapt-2.2.2-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:656593bb3f5529f03d27af4136c4d7b11990e470bcbc6fefa5ef218695bece55", size = 152956, upload-time = "2026-06-20T23:47:31.867Z" }, + { url = "https://files.pythonhosted.org/packages/44/a7/6f42a3d03e44dc612a5dcff324e7366075a7857f0be2d49a8cb8a68279b8/wrapt-2.2.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfb00cb7bb22099e2f64b7340fb96113639aa7260c0972af3797ace2297b936c", size = 154771, upload-time = "2026-06-20T23:47:33.352Z" }, + { url = "https://files.pythonhosted.org/packages/bf/55/4d76175aaa97523c38f1d28f79d18ab41a1b116814158a818bc0eba00571/wrapt-2.2.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e7f10ee0bd53673bfd52b67cbce83336fe6cad90d2377b03baf66491d2bbfb91", size = 149460, upload-time = "2026-06-20T23:47:34.712Z" }, + { url = "https://files.pythonhosted.org/packages/84/9b/12e23264d8f4735e8483262f95c5a6b03c3665fd2a84bdf99a45b6a2f4ec/wrapt-2.2.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4402f57c5f0d0579599858ffbdd9bf4e3f0972f51096f2bd6cc7dab6b76ee49e", size = 153648, upload-time = "2026-06-20T23:47:36.092Z" }, + { url = "https://files.pythonhosted.org/packages/d6/a3/bcd5ec37289dcd85ecd4d15395a6a6063d60bc45ff94a9d77814e1e54d64/wrapt-2.2.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:3a4eb7964ff4643d333c84f880bcf554652b2a1050aebc54ae696327f61acfaf", size = 148502, upload-time = "2026-06-20T23:47:37.623Z" }, + { url = "https://files.pythonhosted.org/packages/f2/be/716d708f607fa70f8a6eb47dff8ee945d5278dfc89ffeeff33039d052e63/wrapt-2.2.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e542b7c5af91e2123a8aabf19894319d5ec4268d2a9ffd2f239386133fc47746", size = 152238, upload-time = "2026-06-20T23:47:39.118Z" }, + { url = "https://files.pythonhosted.org/packages/b5/c0/1a48e7e54501274f5d906f18372221b13183b0afbb5b8bb4c7ca0392c0b4/wrapt-2.2.2-cp310-cp310-win32.whl", hash = "sha256:6e7e45b43d3c774d244fe7264378f5a3f0f383bc55a54a9866434e524540110f", size = 77278, upload-time = "2026-06-20T23:47:40.476Z" }, + { url = "https://files.pythonhosted.org/packages/b0/82/9cd69a1af288fbdedf01a10e3c8a0b6890b08c7f3f96d36a213699dbcd94/wrapt-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:955f1d6e72a352e478de8d8b503abe301c5e139a141b62eb0923bd694995025f", size = 80131, upload-time = "2026-06-20T23:47:41.785Z" }, + { url = "https://files.pythonhosted.org/packages/7f/73/8db7e27daef37ae70a53ea62bef7fe80cc51a8b5e9e9181a8be6eb9a999c/wrapt-2.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:b89d8d73c82db2bb7e6090b3afd7973f980d24e905cc34394eab60b884b3bf67", size = 79615, upload-time = "2026-06-20T23:47:43.109Z" }, + { url = "https://files.pythonhosted.org/packages/27/15/0c2d55168707465abfc41f33c0b23d792a5fa9b65c26983606940900a120/wrapt-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f1a2ff355ece6a111ca7a20dc86df6659c9205d3fcee674ca34f2a2854fd4e73", size = 80782, upload-time = "2026-06-20T23:47:44.367Z" }, + { url = "https://files.pythonhosted.org/packages/7d/b5/5c0b093eb48f8a062ef6267d3cb36e9bb1b88440181f6545a383c60efdf8/wrapt-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:55b9a899e6fff5444f229d30aa6e9ac92d2216d9d60f33c771b5d76a760d5f8e", size = 81678, upload-time = "2026-06-20T23:47:45.857Z" }, + { url = "https://files.pythonhosted.org/packages/34/f3/de70937472dd3e8a4e6811192f9c6075efdffd4a2cd9b4596bf160f89668/wrapt-2.2.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a2d78c363f97d8bd718ee40432c66395685e9e98528ccaa423c3355d1715a26d", size = 159671, upload-time = "2026-06-20T23:47:47.345Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/40aed2330e7f02ecf74386ffcfef9ccb7108c6a430f15b6a252b663b1bed/wrapt-2.2.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d619e1eed9bd4f6ed9f24cd61971aa086fa86505289628d464bcf8a2c2e3f328", size = 160785, upload-time = "2026-06-20T23:47:48.759Z" }, + { url = "https://files.pythonhosted.org/packages/45/04/aa5309beed5344b00220ae6b3b24055852192656194c27947bee1736306a/wrapt-2.2.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:518b0c5e323511ec56a38894802ddd5e1222626484e68efe63f201854ad788e5", size = 153699, upload-time = "2026-06-20T23:47:50.177Z" }, + { url = "https://files.pythonhosted.org/packages/01/df/2def7e99d1fe87eea413f95f671924cdddcb08823b1ffd212748dfa6d062/wrapt-2.2.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4bccea5cdecffa9dd70e343741f0e41e0a16619313d04b72f78bb525162ebcd0", size = 159695, upload-time = "2026-06-20T23:47:51.602Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f6/a906d01a2ce12157bad2404957b3e2140da354b8a70b2fa48bbf282871c0/wrapt-2.2.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:209112cafd963710a05d199aae431d79a28bc76eb8e6d1bbbb8ad24340722cae", size = 152813, upload-time = "2026-06-20T23:47:53.03Z" }, + { url = "https://files.pythonhosted.org/packages/02/49/bc0086292d239575b4c08f4cf8a4079fa58abbad58ec23abf84833a283ed/wrapt-2.2.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e5a5290e4bf2f332fc29ce72ffb9a2fff678aaac047e2e9f5f7165cd7792e099", size = 158809, upload-time = "2026-06-20T23:47:54.391Z" }, + { url = "https://files.pythonhosted.org/packages/55/83/8fbd034de1f3e907edaa18786d5dd8f6932874edee0826c7cecb5cab03a1/wrapt-2.2.2-cp311-cp311-win32.whl", hash = "sha256:5499236ad1dc116012e2a5dd943f3f31af12fce452128e2bbcbd55a7d3d4d14c", size = 77414, upload-time = "2026-06-20T23:47:55.882Z" }, + { url = "https://files.pythonhosted.org/packages/7e/9c/23695baa331c6de4e874c3d78b8e0bed92e1d2a274e665b29858f6841672/wrapt-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:8636809939152be6ae20a6cef0fed9fe60f411b47847d0426a826884b469e971", size = 80368, upload-time = "2026-06-20T23:47:57.237Z" }, + { url = "https://files.pythonhosted.org/packages/08/49/40cefc342bf89b234a4490d741290fce781774b831aefb39c25471da96c9/wrapt-2.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:5d0a142f7af07caeb5e5da87493162a7b8efa19ba919e550a746f7446e13fb30", size = 79489, upload-time = "2026-06-20T23:47:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/85/180b40628b23772692a0c76e8030114e1c0ae068470ed531919f0a5f2a4a/wrapt-2.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8417fd3c674d3c8023d080292d29301531a12daf8bd938dd419710dd2f464f2b", size = 81484, upload-time = "2026-06-20T23:47:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/94/f2/21c90f2a16689702e2aaff45795b11018dff2c9b1242bac10d225483f676/wrapt-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e7070c7472582e31af3dfc2622b2381a0df7435110a9388ed8db5ffbce67efb", size = 82151, upload-time = "2026-06-20T23:48:01.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b3/7e6e9fcf4fe7e1b69a49fe6cc5a44e8224bab6283c5233c97e132f14908e/wrapt-2.2.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2e096c9d39a59b35b63c9aacfbbbec2088ff51ff1fc31051acc60a07f42f273a", size = 169828, upload-time = "2026-06-20T23:48:02.719Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/894f132d857ed5a9904d937baf368badcbe5ea9e436e2f1930fe21c9f1f0/wrapt-2.2.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d1a6050405bf334be33bf66296f113563622972a34900ae6fa60fd283a1a900", size = 171544, upload-time = "2026-06-20T23:48:04.266Z" }, + { url = "https://files.pythonhosted.org/packages/29/de/3c833e03725b477e9ea34028224dd21a48781830101e4e036f77e8b6b102/wrapt-2.2.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10adb01371408c6de504a6658b9886480f1a4919a83752748a387a504a21df79", size = 160663, upload-time = "2026-06-20T23:48:05.708Z" }, + { url = "https://files.pythonhosted.org/packages/33/be/27edce350b24e3054d9d047f65f16d4c4d4c1f3f31c4278a1f8a95c723c8/wrapt-2.2.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3442eee2a5798f9b451f1b2cd7518ce8b7e28a2a364696c414460a0e295c012a", size = 169387, upload-time = "2026-06-20T23:48:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/e2/c4/9fd9679af8bf38e146652c7f47b6b352c3e5795b4ad1c0b7f94e15ac2aa7/wrapt-2.2.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6c99012a22f735a85eed7c4b86a3e99c30fdd57d9e115b2b45f796264b58d0bf", size = 158849, upload-time = "2026-06-20T23:48:08.91Z" }, + { url = "https://files.pythonhosted.org/packages/bc/c2/aa6c0c2206803068c6859dabe01f8c84c43744da93d4c67b8946d21655ee/wrapt-2.2.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b686cfc008776a3952d6213cb296ed7f45d782a8453936406faa89eac0835ab", size = 168147, upload-time = "2026-06-20T23:48:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/42/63/3eb25da41049d20ae18fcab2dd8b056e02387c4bfa626cbdfb7c3b872e4f/wrapt-2.2.2-cp312-cp312-win32.whl", hash = "sha256:ef2cce266b5b0b07e19fa82e59673b81142b7a3607c8ed1254113d048ed668da", size = 77734, upload-time = "2026-06-20T23:48:11.769Z" }, + { url = "https://files.pythonhosted.org/packages/da/09/0390e008a305360948fa9ce69507d041ac12cb2ee5d28e34467e2ee79391/wrapt-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:abf8c20a2d72ee69e16328b3c91342c446e723bfe48bfcc4dded3b9722ac027f", size = 80585, upload-time = "2026-06-20T23:48:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b3/84c445c66969f2d3457276b183a48c91097d59bbef9af6c075366b0f8c36/wrapt-2.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:c6c64c5d02578bc4c4bca4f0aef1504de933c1d5b4ac2710b9131111459506c8", size = 79553, upload-time = "2026-06-20T23:48:14.5Z" }, + { url = "https://files.pythonhosted.org/packages/43/fc/f32f4b22c6511173c11d9e541ab4e7d8467a0f1b3455acaf784115d31ff8/wrapt-2.2.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e8b648270c613720a202d9a45ebabc33261b22c3a839b115ac5bce8c0bb0d69", size = 81296, upload-time = "2026-06-20T23:48:15.881Z" }, + { url = "https://files.pythonhosted.org/packages/72/06/4d117d5d77a9344776c0248b24dae3d3dd2f58e5f765fa08cf887072e719/wrapt-2.2.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e6fb7e94e8fe3e4c3067bb1653a91cce7c5e83acc119fdd41501b1bf74654617", size = 81841, upload-time = "2026-06-20T23:48:17.262Z" }, + { url = "https://files.pythonhosted.org/packages/15/ff/63ad96f98eb58a742b1a20d80f21da88924405910149950b912368150468/wrapt-2.2.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb18fc51e813df0d9c98049e3bf2298a5495a648602040e21fa3c7329371159e", size = 167882, upload-time = "2026-06-20T23:48:18.764Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/8bb62d8933df7acf3247194e6e9fc68edf9d2fa203252c89c94b319dd472/wrapt-2.2.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94b00b00f806eb3ef2abe9049ed45994a81ee9284884d96e6b8314927c6cea3d", size = 167411, upload-time = "2026-06-20T23:48:20.315Z" }, + { url = "https://files.pythonhosted.org/packages/17/09/8789dcb09ee1de715727db7521aabbb68ffa68dfade3a49468440cfced49/wrapt-2.2.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:62415fd095bc590b842b6d092f2b5d9ccbaeb7e0b28535c03dcea2718b48636b", size = 158607, upload-time = "2026-06-20T23:48:21.728Z" }, + { url = "https://files.pythonhosted.org/packages/9c/20/66e02562d53ee67d841f175e38e3c993c2d78a3e104c576cad61c028b43c/wrapt-2.2.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a41e758d80dc0ab8c210f641ac892009d356cf1f955d97db544c8dd317b4d14c", size = 166367, upload-time = "2026-06-20T23:48:23.177Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a3/832ac4e41222fb263b3042d42c2f08d305db7d0f0c9b1d3a271a9eede8f6/wrapt-2.2.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b84cd4058001c9727b0e9980b7a9e66325b5ca748b1b578e822cade1bc6b304f", size = 157176, upload-time = "2026-06-20T23:48:24.711Z" }, + { url = "https://files.pythonhosted.org/packages/b7/01/1bd5e4d2df9c0178989ac8da9186543465388588ee2ef153e2591accebef/wrapt-2.2.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26fc73a1b15e0946d2942b9a4426d162b51676338327dc067ccd8d2d76385f94", size = 167025, upload-time = "2026-06-20T23:48:26.118Z" }, + { url = "https://files.pythonhosted.org/packages/1c/69/583ed25291ab53e1ec117135fb1c33425e2f46d2bc8f29c17f7a94cf4274/wrapt-2.2.2-cp313-cp313-win32.whl", hash = "sha256:3c4095803491f6ef72128914c28ec05bbad9758433bb35f6715a3e9c8e46fb2d", size = 77605, upload-time = "2026-06-20T23:48:27.643Z" }, + { url = "https://files.pythonhosted.org/packages/29/68/e69fc6d06e1523c68e0d00f95c9aed1158ce9908ee41603f7f2eae3d5db6/wrapt-2.2.2-cp313-cp313-win_amd64.whl", hash = "sha256:2cb07f414fab25dbe6b5c7398e1491423a5c81a6209533639969a6c928d474a4", size = 80508, upload-time = "2026-06-20T23:48:29.013Z" }, + { url = "https://files.pythonhosted.org/packages/55/21/fe7a393d9e5dc0923bed8f5d857e9dcff210f1fa0888c02cc8f3ffaa55aa/wrapt-2.2.2-cp313-cp313-win_arm64.whl", hash = "sha256:1fc7691f070220215cccb2a20836b9adbaecb8ff22ad47abe63de5f110994fac", size = 79565, upload-time = "2026-06-20T23:48:30.429Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e5/c120d13bf5091164f68c3c1657e84f16f57e71d978421b626393ac5bd7eb/wrapt-2.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:ec8f83949028366531383603139403cac7a826e4011955813cdd640017845ce5", size = 83264, upload-time = "2026-06-20T23:48:31.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/d4a1eb97e0e286625bdf21bc7f702637f9607787ffbbdb5ec14d50c79dbf/wrapt-2.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b481fb0c40d9fd90a5809911208da700987d373a20a4709dc9e3944af7a6bec", size = 83791, upload-time = "2026-06-20T23:48:33.482Z" }, + { url = "https://files.pythonhosted.org/packages/18/1e/f060df47755e87b57684cee7bfc1362b204df55fac96ffebc0631b697b79/wrapt-2.2.2-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0065a3b657cec06813b4241d2462ccec287f6863103d7445b725fb3a889736f9", size = 203399, upload-time = "2026-06-20T23:48:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/2316a757a1abb6453700b79d83e532146dcef2611348282d4d8889792161/wrapt-2.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:30f7424af5c5c345b7f26490e097f74a2ef45b3d08b664dc33571aee3bd3b56c", size = 210461, upload-time = "2026-06-20T23:48:36.569Z" }, + { url = "https://files.pythonhosted.org/packages/ed/29/d1160785ae18ca2495a6d82a21154103d74f656c9fd457fb35f6b11b965a/wrapt-2.2.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07fdcb012821859168641acf68afad61ef9783cf37100af85f152550e9677194", size = 195313, upload-time = "2026-06-20T23:48:38.175Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7caa9598ae61a9cf0989cc501739cbeeb7d650ab3193cca1407b9af0c6ab/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:f90038ab58fafb584801ca62d72384d7d5225d93c76f7b773c22fae545bd8066", size = 206116, upload-time = "2026-06-20T23:48:39.804Z" }, + { url = "https://files.pythonhosted.org/packages/ac/02/281ea1088b8650d865f311b35cf86fd21df89128e2909714f1161e01c9d0/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:c5d7825491bfa2d08b97e9557768987952c7b9ae687d06c3320b40a37ccb7f20", size = 192668, upload-time = "2026-06-20T23:48:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/be/7d/976e2d5b4b5c5babda40974edd54d0a5585cb60132ed86b46f4b80239b16/wrapt-2.2.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ad520e6daa9bbf136f14de735474dbec7dcc0891f718e1d274ce8dc92e645af", size = 198891, upload-time = "2026-06-20T23:48:43.056Z" }, + { url = "https://files.pythonhosted.org/packages/59/b7/e47651797c097f75a37e2ce86dcf04048ff576f3a674f7c558df7b5e9622/wrapt-2.2.2-cp313-cp313t-win32.whl", hash = "sha256:25904acb9475f46c24fe0423dbc8fda8cc5fbc282ab3dc6e72e919748c53f4e9", size = 78537, upload-time = "2026-06-20T23:48:44.509Z" }, + { url = "https://files.pythonhosted.org/packages/d1/6f/9fa5d59fb06d890defb5a8f727ce6a14d2932c8760153f96956628559fee/wrapt-2.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:305d4c247d61c4115794a169141823c62f719525ddb90b23aa332741c77d2c28", size = 82005, upload-time = "2026-06-20T23:48:46.391Z" }, + { url = "https://files.pythonhosted.org/packages/15/80/4c7bd9873d1f9f7d138d93556b500469dbe24f42710b877519c2b9eb380d/wrapt-2.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:c20279cd1a29800815d7b2d6338b60a6c6e78263f9d6e62e0eda251ba9cae2d0", size = 80762, upload-time = "2026-06-20T23:48:47.964Z" }, + { url = "https://files.pythonhosted.org/packages/24/05/7fd9c3f83b2c74cbfc572a0b88aa37431e04bd8aed70d2c0efd3464206de/wrapt-2.2.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:0e64826f920c42d9d9f87e8cc09ffae66c51ede12d59061a5a426deb9aa71745", size = 81341, upload-time = "2026-06-20T23:48:49.39Z" }, + { url = "https://files.pythonhosted.org/packages/4b/68/1bfa43100dd90d4ef74a05897b86275cf57e1313ca14aae2545bc9f872c9/wrapt-2.2.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dcaa5e1451bd8751d7bd1568dfa3321c78092a52a7ecb5d1a0f18a5791e1fd00", size = 81921, upload-time = "2026-06-20T23:48:50.986Z" }, + { url = "https://files.pythonhosted.org/packages/74/eb/df7b7f0b631dbbc750f39be27d8b55f65777d8ac86da80e12be41a644c4b/wrapt-2.2.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0abfd648dac9ac9c5b3aa9b523d27f1789046640b58dcd5652a720ddb325e1fc", size = 167713, upload-time = "2026-06-20T23:48:52.598Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9a/d1bd36f6d088c8e652a9383cabbd49af30b8c576302a7eccddbab6963e3f/wrapt-2.2.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f4bfd8d1eb438153eff8b8cfe87f032ba65731e1ce06138b5090f745a33f6f95", size = 166779, upload-time = "2026-06-20T23:48:54.33Z" }, + { url = "https://files.pythonhosted.org/packages/4c/ae/24ffacd4187fac2740a1972093929e836dea092d42c87d728cd98fee11a6/wrapt-2.2.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c427c9d06d859848a69f0d928fe28b5c33a941b2265d10a0e1f15cd244f1ee33", size = 158407, upload-time = "2026-06-20T23:48:55.944Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ed/974427668249a356051e8d67d47fa54ef6c777f0fcf3bae9d292c047d4b6/wrapt-2.2.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4250b43d1a129d947e083c4dc6baf333c9bb34edd26f912d5b0457841fc858ab", size = 166594, upload-time = "2026-06-20T23:48:57.617Z" }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e1d7c6e4523f78db2fbd7826babd0348da1d5e0834c4f918b9ab5757dfae/wrapt-2.2.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:173e5bb5ca350a6e0abab60b7ec7cdd7992a814cb14b4de670a28f067f105663", size = 157068, upload-time = "2026-06-20T23:48:59.171Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c1/7ebd1027f00700c0b0233b20aceef2b4784294ed64971424c4a78e069e34/wrapt-2.2.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:aa14b01804bce36c6d63d7b6a4f55df390f29f8648cc13a1f40b166f4d54680d", size = 166470, upload-time = "2026-06-20T23:49:00.737Z" }, + { url = "https://files.pythonhosted.org/packages/99/eb/974e471a6a978b8180186b8a9dc5ae3361ce269a967190b709b8ce17abfb/wrapt-2.2.2-cp314-cp314-win32.whl", hash = "sha256:58f9f8d637c9a6e245c6ef5b109b67ec187d2faed23d1405656b51d96e0a5b56", size = 78062, upload-time = "2026-06-20T23:49:02.327Z" }, + { url = "https://files.pythonhosted.org/packages/49/ec/e1281156cdc7a66693838ad7a0865ad641c74abd337a957d668b575aaffb/wrapt-2.2.2-cp314-cp314-win_amd64.whl", hash = "sha256:385cb1866f20479e83299af585375bfa0a4b0c6c9907a981483ea782ea8ae406", size = 80832, upload-time = "2026-06-20T23:49:03.837Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/1b6b5ddd94005a2dac97a4490c9838f3154977850d633abcb65b30089437/wrapt-2.2.2-cp314-cp314-win_arm64.whl", hash = "sha256:8ffbeaea6771a6eba6e6eeb09767864995726bc8240bb54baf88a9bb1db34d5c", size = 80029, upload-time = "2026-06-20T23:49:05.237Z" }, + { url = "https://files.pythonhosted.org/packages/b0/33/9ebcf8aafe91c601127cbd93708c16aa8f688f34a10bf004046803ecdc4f/wrapt-2.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09f811d43f6f33ec7515f0be76b159569f4057ab54d3e079c3204dddb90afa2a", size = 83357, upload-time = "2026-06-20T23:49:06.632Z" }, + { url = "https://files.pythonhosted.org/packages/39/38/ec45b635153327b52e52732a0ea980e5f00b7efba65f9e018828f1e69daa/wrapt-2.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a795d3c06e5fbf9ea2f13196180b77aeab1b4685917256ee0d014cc163d90063", size = 83794, upload-time = "2026-06-20T23:49:08.098Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ea/1a89e6d3b7a83c3affe5c09cde77792c947e63e4bc85ad84cd5bb9abb0d8/wrapt-2.2.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:45c2f2768e790c9f8db90f239ef23a2af8e7570f25a35619ef902df4a738447f", size = 203362, upload-time = "2026-06-20T23:49:09.811Z" }, + { url = "https://files.pythonhosted.org/packages/19/d8/3b58763d9863b5a73771c0d97110f9595d248db454009e07e1535ee905a4/wrapt-2.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbf00ee0cb55ec24e2b0995a71942b85b21a066db8f3f46e1dbfdb9433ffba81", size = 210449, upload-time = "2026-06-20T23:49:11.521Z" }, + { url = "https://files.pythonhosted.org/packages/2d/6f/17fd9e053103d8be148d20d5d7505facc72d5fe1f9127973904ceaed79cf/wrapt-2.2.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2252f77663651b89255895f58cc6ac08fcb206d4371813e5af61bb62d4f7689c", size = 195349, upload-time = "2026-06-20T23:49:13.346Z" }, + { url = "https://files.pythonhosted.org/packages/ef/04/d0d1ccaaa12cb7dccf28a23f0279a608ba498f71e81d949d5ed54bcfd5c1/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2cd7181ab1c31192ff5219269830744b5a62020b3a6d433588c4f1c95b8f8bff", size = 206099, upload-time = "2026-06-20T23:49:15.051Z" }, + { url = "https://files.pythonhosted.org/packages/44/b3/e8aa07b619890a2aa6cde1931b1887abb08820721b564a5f80b7ca3f3aa0/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:6fe35fd51b74867d8b80174c277bd6bbf6a73e443f908129dc531c4b688a20d5", size = 192728, upload-time = "2026-06-20T23:49:16.854Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f0/1819fb50f0d3c9bd758d8a83b56f1b470dee8b5b8eac8702b7c137cea9d4/wrapt-2.2.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:11d95fc2fbad3163596c39d440e6f21ca9fccece74b56e30a37ac2fca786a07c", size = 198842, upload-time = "2026-06-20T23:49:18.504Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/e88313f16a99930b899ef970d91c281544a470749a359decad994483bbda/wrapt-2.2.2-cp314-cp314t-win32.whl", hash = "sha256:d8a15813215f33fa83667bfc978b300e35669ea8bb424e970a1426bcb7bc6cca", size = 79059, upload-time = "2026-06-20T23:49:20.107Z" }, + { url = "https://files.pythonhosted.org/packages/a0/4f/ac12fda57a55068a094ec42851fb0a40e8489d8941863d517452de62e507/wrapt-2.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d09db0f7e8357060d3c38fc22a018aba683a796bf184360fd1a58f6fc180dc77", size = 82462, upload-time = "2026-06-20T23:49:21.631Z" }, + { url = "https://files.pythonhosted.org/packages/48/a7/df732dac86d9b2027c56bd163dbc883e037b16c3469614752e148d219c61/wrapt-2.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:f32fe639c39561ccc187bcae17e9271be0eb45f1c2952510d2f29b33ab577347", size = 81182, upload-time = "2026-06-20T23:49:23.199Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d2/6317eb6d4554855bbf12d61857774af34747bf88a42c19bf306de67e2fa3/wrapt-2.2.2-py3-none-any.whl", hash = "sha256:5bad217350f19ce99ca5b5e71d406765ea86fe541628426772b657375ee1c048", size = 61460, upload-time = "2026-06-20T23:49:42.966Z" }, ] [[package]] name = "xxhash" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/84/30869e01909fb37a6cc7e18688ee8bf1e42d57e7e0777636bd47524c43c7/xxhash-3.6.0.tar.gz", hash = "sha256:f0162a78b13a0d7617b2845b90c763339d1f1d82bb04a4b07f4ab535cc5e05d6", size = 85160, upload-time = "2025-10-02T14:37:08.097Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/34/ee/f9f1d656ad168681bb0f6b092372c1e533c4416b8069b1896a175c46e484/xxhash-3.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:87ff03d7e35c61435976554477a7f4cd1704c3596a89a8300d5ce7fc83874a71", size = 32845, upload-time = "2025-10-02T14:33:51.573Z" }, - { url = "https://files.pythonhosted.org/packages/a3/b1/93508d9460b292c74a09b83d16750c52a0ead89c51eea9951cb97a60d959/xxhash-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f572dfd3d0e2eb1a57511831cf6341242f5a9f8298a45862d085f5b93394a27d", size = 30807, upload-time = "2025-10-02T14:33:52.964Z" }, - { url = "https://files.pythonhosted.org/packages/07/55/28c93a3662f2d200c70704efe74aab9640e824f8ce330d8d3943bf7c9b3c/xxhash-3.6.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:89952ea539566b9fed2bbd94e589672794b4286f342254fad28b149f9615fef8", size = 193786, upload-time = "2025-10-02T14:33:54.272Z" }, - { url = "https://files.pythonhosted.org/packages/c1/96/fec0be9bb4b8f5d9c57d76380a366f31a1781fb802f76fc7cda6c84893c7/xxhash-3.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48e6f2ffb07a50b52465a1032c3cf1f4a5683f944acaca8a134a2f23674c2058", size = 212830, upload-time = "2025-10-02T14:33:55.706Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a0/c706845ba77b9611f81fd2e93fad9859346b026e8445e76f8c6fd057cc6d/xxhash-3.6.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b5b848ad6c16d308c3ac7ad4ba6bede80ed5df2ba8ed382f8932df63158dd4b2", size = 211606, upload-time = "2025-10-02T14:33:57.133Z" }, - { url = "https://files.pythonhosted.org/packages/67/1e/164126a2999e5045f04a69257eea946c0dc3e86541b400d4385d646b53d7/xxhash-3.6.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a034590a727b44dd8ac5914236a7b8504144447a9682586c3327e935f33ec8cc", size = 444872, upload-time = "2025-10-02T14:33:58.446Z" }, - { url = "https://files.pythonhosted.org/packages/2d/4b/55ab404c56cd70a2cf5ecfe484838865d0fea5627365c6c8ca156bd09c8f/xxhash-3.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8a8f1972e75ebdd161d7896743122834fe87378160c20e97f8b09166213bf8cc", size = 193217, upload-time = "2025-10-02T14:33:59.724Z" }, - { url = "https://files.pythonhosted.org/packages/45/e6/52abf06bac316db33aa269091ae7311bd53cfc6f4b120ae77bac1b348091/xxhash-3.6.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ee34327b187f002a596d7b167ebc59a1b729e963ce645964bbc050d2f1b73d07", size = 210139, upload-time = "2025-10-02T14:34:02.041Z" }, - { url = "https://files.pythonhosted.org/packages/34/37/db94d490b8691236d356bc249c08819cbcef9273a1a30acf1254ff9ce157/xxhash-3.6.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:339f518c3c7a850dd033ab416ea25a692759dc7478a71131fe8869010d2b75e4", size = 197669, upload-time = "2025-10-02T14:34:03.664Z" }, - { url = "https://files.pythonhosted.org/packages/b7/36/c4f219ef4a17a4f7a64ed3569bc2b5a9c8311abdb22249ac96093625b1a4/xxhash-3.6.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:bf48889c9630542d4709192578aebbd836177c9f7a4a2778a7d6340107c65f06", size = 210018, upload-time = "2025-10-02T14:34:05.325Z" }, - { url = "https://files.pythonhosted.org/packages/fd/06/bfac889a374fc2fc439a69223d1750eed2e18a7db8514737ab630534fa08/xxhash-3.6.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:5576b002a56207f640636056b4160a378fe36a58db73ae5c27a7ec8db35f71d4", size = 413058, upload-time = "2025-10-02T14:34:06.925Z" }, - { url = "https://files.pythonhosted.org/packages/c9/d1/555d8447e0dd32ad0930a249a522bb2e289f0d08b6b16204cfa42c1f5a0c/xxhash-3.6.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:af1f3278bd02814d6dedc5dec397993b549d6f16c19379721e5a1d31e132c49b", size = 190628, upload-time = "2025-10-02T14:34:08.669Z" }, - { url = "https://files.pythonhosted.org/packages/d1/15/8751330b5186cedc4ed4b597989882ea05e0408b53fa47bcb46a6125bfc6/xxhash-3.6.0-cp310-cp310-win32.whl", hash = "sha256:aed058764db109dc9052720da65fafe84873b05eb8b07e5e653597951af57c3b", size = 30577, upload-time = "2025-10-02T14:34:10.234Z" }, - { url = "https://files.pythonhosted.org/packages/bb/cc/53f87e8b5871a6eb2ff7e89c48c66093bda2be52315a8161ddc54ea550c4/xxhash-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:e82da5670f2d0d98950317f82a0e4a0197150ff19a6df2ba40399c2a3b9ae5fb", size = 31487, upload-time = "2025-10-02T14:34:11.618Z" }, - { url = "https://files.pythonhosted.org/packages/9f/00/60f9ea3bb697667a14314d7269956f58bf56bb73864f8f8d52a3c2535e9a/xxhash-3.6.0-cp310-cp310-win_arm64.whl", hash = "sha256:4a082ffff8c6ac07707fb6b671caf7c6e020c75226c561830b73d862060f281d", size = 27863, upload-time = "2025-10-02T14:34:12.619Z" }, - { url = "https://files.pythonhosted.org/packages/17/d4/cc2f0400e9154df4b9964249da78ebd72f318e35ccc425e9f403c392f22a/xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b47bbd8cf2d72797f3c2772eaaac0ded3d3af26481a26d7d7d41dc2d3c46b04a", size = 32844, upload-time = "2025-10-02T14:34:14.037Z" }, - { url = "https://files.pythonhosted.org/packages/5e/ec/1cc11cd13e26ea8bc3cb4af4eaadd8d46d5014aebb67be3f71fb0b68802a/xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2b6821e94346f96db75abaa6e255706fb06ebd530899ed76d32cd99f20dc52fa", size = 30809, upload-time = "2025-10-02T14:34:15.484Z" }, - { url = "https://files.pythonhosted.org/packages/04/5f/19fe357ea348d98ca22f456f75a30ac0916b51c753e1f8b2e0e6fb884cce/xxhash-3.6.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d0a9751f71a1a65ce3584e9cae4467651c7e70c9d31017fa57574583a4540248", size = 194665, upload-time = "2025-10-02T14:34:16.541Z" }, - { url = "https://files.pythonhosted.org/packages/90/3b/d1f1a8f5442a5fd8beedae110c5af7604dc37349a8e16519c13c19a9a2de/xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b29ee68625ab37b04c0b40c3fafdf24d2f75ccd778333cfb698f65f6c463f62", size = 213550, upload-time = "2025-10-02T14:34:17.878Z" }, - { url = "https://files.pythonhosted.org/packages/c4/ef/3a9b05eb527457d5db13a135a2ae1a26c80fecd624d20f3e8dcc4cb170f3/xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6812c25fe0d6c36a46ccb002f40f27ac903bf18af9f6dd8f9669cb4d176ab18f", size = 212384, upload-time = "2025-10-02T14:34:19.182Z" }, - { url = "https://files.pythonhosted.org/packages/0f/18/ccc194ee698c6c623acbf0f8c2969811a8a4b6185af5e824cd27b9e4fd3e/xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4ccbff013972390b51a18ef1255ef5ac125c92dc9143b2d1909f59abc765540e", size = 445749, upload-time = "2025-10-02T14:34:20.659Z" }, - { url = "https://files.pythonhosted.org/packages/a5/86/cf2c0321dc3940a7aa73076f4fd677a0fb3e405cb297ead7d864fd90847e/xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:297b7fbf86c82c550e12e8fb71968b3f033d27b874276ba3624ea868c11165a8", size = 193880, upload-time = "2025-10-02T14:34:22.431Z" }, - { url = "https://files.pythonhosted.org/packages/82/fb/96213c8560e6f948a1ecc9a7613f8032b19ee45f747f4fca4eb31bb6d6ed/xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dea26ae1eb293db089798d3973a5fc928a18fdd97cc8801226fae705b02b14b0", size = 210912, upload-time = "2025-10-02T14:34:23.937Z" }, - { url = "https://files.pythonhosted.org/packages/40/aa/4395e669b0606a096d6788f40dbdf2b819d6773aa290c19e6e83cbfc312f/xxhash-3.6.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7a0b169aafb98f4284f73635a8e93f0735f9cbde17bd5ec332480484241aaa77", size = 198654, upload-time = "2025-10-02T14:34:25.644Z" }, - { url = "https://files.pythonhosted.org/packages/67/74/b044fcd6b3d89e9b1b665924d85d3f400636c23590226feb1eb09e1176ce/xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:08d45aef063a4531b785cd72de4887766d01dc8f362a515693df349fdb825e0c", size = 210867, upload-time = "2025-10-02T14:34:27.203Z" }, - { url = "https://files.pythonhosted.org/packages/bc/fd/3ce73bf753b08cb19daee1eb14aa0d7fe331f8da9c02dd95316ddfe5275e/xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:929142361a48ee07f09121fe9e96a84950e8d4df3bb298ca5d88061969f34d7b", size = 414012, upload-time = "2025-10-02T14:34:28.409Z" }, - { url = "https://files.pythonhosted.org/packages/ba/b3/5a4241309217c5c876f156b10778f3ab3af7ba7e3259e6d5f5c7d0129eb2/xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:51312c768403d8540487dbbfb557454cfc55589bbde6424456951f7fcd4facb3", size = 191409, upload-time = "2025-10-02T14:34:29.696Z" }, - { url = "https://files.pythonhosted.org/packages/c0/01/99bfbc15fb9abb9a72b088c1d95219fc4782b7d01fc835bd5744d66dd0b8/xxhash-3.6.0-cp311-cp311-win32.whl", hash = "sha256:d1927a69feddc24c987b337ce81ac15c4720955b667fe9b588e02254b80446fd", size = 30574, upload-time = "2025-10-02T14:34:31.028Z" }, - { url = "https://files.pythonhosted.org/packages/65/79/9d24d7f53819fe301b231044ea362ce64e86c74f6e8c8e51320de248b3e5/xxhash-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:26734cdc2d4ffe449b41d186bbeac416f704a482ed835d375a5c0cb02bc63fef", size = 31481, upload-time = "2025-10-02T14:34:32.062Z" }, - { url = "https://files.pythonhosted.org/packages/30/4e/15cd0e3e8772071344eab2961ce83f6e485111fed8beb491a3f1ce100270/xxhash-3.6.0-cp311-cp311-win_arm64.whl", hash = "sha256:d72f67ef8bf36e05f5b6c65e8524f265bd61071471cd4cf1d36743ebeeeb06b7", size = 27861, upload-time = "2025-10-02T14:34:33.555Z" }, - { url = "https://files.pythonhosted.org/packages/9a/07/d9412f3d7d462347e4511181dea65e47e0d0e16e26fbee2ea86a2aefb657/xxhash-3.6.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:01362c4331775398e7bb34e3ab403bc9ee9f7c497bc7dee6272114055277dd3c", size = 32744, upload-time = "2025-10-02T14:34:34.622Z" }, - { url = "https://files.pythonhosted.org/packages/79/35/0429ee11d035fc33abe32dca1b2b69e8c18d236547b9a9b72c1929189b9a/xxhash-3.6.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b7b2df81a23f8cb99656378e72501b2cb41b1827c0f5a86f87d6b06b69f9f204", size = 30816, upload-time = "2025-10-02T14:34:36.043Z" }, - { url = "https://files.pythonhosted.org/packages/b7/f2/57eb99aa0f7d98624c0932c5b9a170e1806406cdbcdb510546634a1359e0/xxhash-3.6.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:dc94790144e66b14f67b10ac8ed75b39ca47536bf8800eb7c24b50271ea0c490", size = 194035, upload-time = "2025-10-02T14:34:37.354Z" }, - { url = "https://files.pythonhosted.org/packages/4c/ed/6224ba353690d73af7a3f1c7cdb1fc1b002e38f783cb991ae338e1eb3d79/xxhash-3.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93f107c673bccf0d592cdba077dedaf52fe7f42dcd7676eba1f6d6f0c3efffd2", size = 212914, upload-time = "2025-10-02T14:34:38.6Z" }, - { url = "https://files.pythonhosted.org/packages/38/86/fb6b6130d8dd6b8942cc17ab4d90e223653a89aa32ad2776f8af7064ed13/xxhash-3.6.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2aa5ee3444c25b69813663c9f8067dcfaa2e126dc55e8dddf40f4d1c25d7effa", size = 212163, upload-time = "2025-10-02T14:34:39.872Z" }, - { url = "https://files.pythonhosted.org/packages/ee/dc/e84875682b0593e884ad73b2d40767b5790d417bde603cceb6878901d647/xxhash-3.6.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7f99123f0e1194fa59cc69ad46dbae2e07becec5df50a0509a808f90a0f03f0", size = 445411, upload-time = "2025-10-02T14:34:41.569Z" }, - { url = "https://files.pythonhosted.org/packages/11/4f/426f91b96701ec2f37bb2b8cec664eff4f658a11f3fa9d94f0a887ea6d2b/xxhash-3.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49e03e6fe2cac4a1bc64952dd250cf0dbc5ef4ebb7b8d96bce82e2de163c82a2", size = 193883, upload-time = "2025-10-02T14:34:43.249Z" }, - { url = "https://files.pythonhosted.org/packages/53/5a/ddbb83eee8e28b778eacfc5a85c969673e4023cdeedcfcef61f36731610b/xxhash-3.6.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd17fede52a17a4f9a7bc4472a5867cb0b160deeb431795c0e4abe158bc784e9", size = 210392, upload-time = "2025-10-02T14:34:45.042Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c2/ff69efd07c8c074ccdf0a4f36fcdd3d27363665bcdf4ba399abebe643465/xxhash-3.6.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6fb5f5476bef678f69db04f2bd1efbed3030d2aba305b0fc1773645f187d6a4e", size = 197898, upload-time = "2025-10-02T14:34:46.302Z" }, - { url = "https://files.pythonhosted.org/packages/58/ca/faa05ac19b3b622c7c9317ac3e23954187516298a091eb02c976d0d3dd45/xxhash-3.6.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:843b52f6d88071f87eba1631b684fcb4b2068cd2180a0224122fe4ef011a9374", size = 210655, upload-time = "2025-10-02T14:34:47.571Z" }, - { url = "https://files.pythonhosted.org/packages/d4/7a/06aa7482345480cc0cb597f5c875b11a82c3953f534394f620b0be2f700c/xxhash-3.6.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:7d14a6cfaf03b1b6f5f9790f76880601ccc7896aff7ab9cd8978a939c1eb7e0d", size = 414001, upload-time = "2025-10-02T14:34:49.273Z" }, - { url = "https://files.pythonhosted.org/packages/23/07/63ffb386cd47029aa2916b3d2f454e6cc5b9f5c5ada3790377d5430084e7/xxhash-3.6.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:418daf3db71e1413cfe211c2f9a528456936645c17f46b5204705581a45390ae", size = 191431, upload-time = "2025-10-02T14:34:50.798Z" }, - { url = "https://files.pythonhosted.org/packages/0f/93/14fde614cadb4ddf5e7cebf8918b7e8fac5ae7861c1875964f17e678205c/xxhash-3.6.0-cp312-cp312-win32.whl", hash = "sha256:50fc255f39428a27299c20e280d6193d8b63b8ef8028995323bf834a026b4fbb", size = 30617, upload-time = "2025-10-02T14:34:51.954Z" }, - { url = "https://files.pythonhosted.org/packages/13/5d/0d125536cbe7565a83d06e43783389ecae0c0f2ed037b48ede185de477c0/xxhash-3.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:c0f2ab8c715630565ab8991b536ecded9416d615538be8ecddce43ccf26cbc7c", size = 31534, upload-time = "2025-10-02T14:34:53.276Z" }, - { url = "https://files.pythonhosted.org/packages/54/85/6ec269b0952ec7e36ba019125982cf11d91256a778c7c3f98a4c5043d283/xxhash-3.6.0-cp312-cp312-win_arm64.whl", hash = "sha256:eae5c13f3bc455a3bbb68bdc513912dc7356de7e2280363ea235f71f54064829", size = 27876, upload-time = "2025-10-02T14:34:54.371Z" }, - { url = "https://files.pythonhosted.org/packages/33/76/35d05267ac82f53ae9b0e554da7c5e281ee61f3cad44c743f0fcd354f211/xxhash-3.6.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:599e64ba7f67472481ceb6ee80fa3bd828fd61ba59fb11475572cc5ee52b89ec", size = 32738, upload-time = "2025-10-02T14:34:55.839Z" }, - { url = "https://files.pythonhosted.org/packages/31/a8/3fbce1cd96534a95e35d5120637bf29b0d7f5d8fa2f6374e31b4156dd419/xxhash-3.6.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7d8b8aaa30fca4f16f0c84a5c8d7ddee0e25250ec2796c973775373257dde8f1", size = 30821, upload-time = "2025-10-02T14:34:57.219Z" }, - { url = "https://files.pythonhosted.org/packages/0c/ea/d387530ca7ecfa183cb358027f1833297c6ac6098223fd14f9782cd0015c/xxhash-3.6.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d597acf8506d6e7101a4a44a5e428977a51c0fadbbfd3c39650cca9253f6e5a6", size = 194127, upload-time = "2025-10-02T14:34:59.21Z" }, - { url = "https://files.pythonhosted.org/packages/ba/0c/71435dcb99874b09a43b8d7c54071e600a7481e42b3e3ce1eb5226a5711a/xxhash-3.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:858dc935963a33bc33490128edc1c12b0c14d9c7ebaa4e387a7869ecc4f3e263", size = 212975, upload-time = "2025-10-02T14:35:00.816Z" }, - { url = "https://files.pythonhosted.org/packages/84/7a/c2b3d071e4bb4a90b7057228a99b10d51744878f4a8a6dd643c8bd897620/xxhash-3.6.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba284920194615cb8edf73bf52236ce2e1664ccd4a38fdb543506413529cc546", size = 212241, upload-time = "2025-10-02T14:35:02.207Z" }, - { url = "https://files.pythonhosted.org/packages/81/5f/640b6eac0128e215f177df99eadcd0f1b7c42c274ab6a394a05059694c5a/xxhash-3.6.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b54219177f6c6674d5378bd862c6aedf64725f70dd29c472eaae154df1a2e89", size = 445471, upload-time = "2025-10-02T14:35:03.61Z" }, - { url = "https://files.pythonhosted.org/packages/5e/1e/3c3d3ef071b051cc3abbe3721ffb8365033a172613c04af2da89d5548a87/xxhash-3.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42c36dd7dbad2f5238950c377fcbf6811b1cdb1c444fab447960030cea60504d", size = 193936, upload-time = "2025-10-02T14:35:05.013Z" }, - { url = "https://files.pythonhosted.org/packages/2c/bd/4a5f68381939219abfe1c22a9e3a5854a4f6f6f3c4983a87d255f21f2e5d/xxhash-3.6.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f22927652cba98c44639ffdc7aaf35828dccf679b10b31c4ad72a5b530a18eb7", size = 210440, upload-time = "2025-10-02T14:35:06.239Z" }, - { url = "https://files.pythonhosted.org/packages/eb/37/b80fe3d5cfb9faff01a02121a0f4d565eb7237e9e5fc66e73017e74dcd36/xxhash-3.6.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b45fad44d9c5c119e9c6fbf2e1c656a46dc68e280275007bbfd3d572b21426db", size = 197990, upload-time = "2025-10-02T14:35:07.735Z" }, - { url = "https://files.pythonhosted.org/packages/d7/fd/2c0a00c97b9e18f72e1f240ad4e8f8a90fd9d408289ba9c7c495ed7dc05c/xxhash-3.6.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:6f2580ffab1a8b68ef2b901cde7e55fa8da5e4be0977c68f78fc80f3c143de42", size = 210689, upload-time = "2025-10-02T14:35:09.438Z" }, - { url = "https://files.pythonhosted.org/packages/93/86/5dd8076a926b9a95db3206aba20d89a7fc14dd5aac16e5c4de4b56033140/xxhash-3.6.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40c391dd3cd041ebc3ffe6f2c862f402e306eb571422e0aa918d8070ba31da11", size = 414068, upload-time = "2025-10-02T14:35:11.162Z" }, - { url = "https://files.pythonhosted.org/packages/af/3c/0bb129170ee8f3650f08e993baee550a09593462a5cddd8e44d0011102b1/xxhash-3.6.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f205badabde7aafd1a31e8ca2a3e5a763107a71c397c4481d6a804eb5063d8bd", size = 191495, upload-time = "2025-10-02T14:35:12.971Z" }, - { url = "https://files.pythonhosted.org/packages/e9/3a/6797e0114c21d1725e2577508e24006fd7ff1d8c0c502d3b52e45c1771d8/xxhash-3.6.0-cp313-cp313-win32.whl", hash = "sha256:2577b276e060b73b73a53042ea5bd5203d3e6347ce0d09f98500f418a9fcf799", size = 30620, upload-time = "2025-10-02T14:35:14.129Z" }, - { url = "https://files.pythonhosted.org/packages/86/15/9bc32671e9a38b413a76d24722a2bf8784a132c043063a8f5152d390b0f9/xxhash-3.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:757320d45d2fbcce8f30c42a6b2f47862967aea7bf458b9625b4bbe7ee390392", size = 31542, upload-time = "2025-10-02T14:35:15.21Z" }, - { url = "https://files.pythonhosted.org/packages/39/c5/cc01e4f6188656e56112d6a8e0dfe298a16934b8c47a247236549a3f7695/xxhash-3.6.0-cp313-cp313-win_arm64.whl", hash = "sha256:457b8f85dec5825eed7b69c11ae86834a018b8e3df5e77783c999663da2f96d6", size = 27880, upload-time = "2025-10-02T14:35:16.315Z" }, - { url = "https://files.pythonhosted.org/packages/f3/30/25e5321c8732759e930c555176d37e24ab84365482d257c3b16362235212/xxhash-3.6.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a42e633d75cdad6d625434e3468126c73f13f7584545a9cf34e883aa1710e702", size = 32956, upload-time = "2025-10-02T14:35:17.413Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3c/0573299560d7d9f8ab1838f1efc021a280b5ae5ae2e849034ef3dee18810/xxhash-3.6.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:568a6d743219e717b07b4e03b0a828ce593833e498c3b64752e0f5df6bfe84db", size = 31072, upload-time = "2025-10-02T14:35:18.844Z" }, - { url = "https://files.pythonhosted.org/packages/7a/1c/52d83a06e417cd9d4137722693424885cc9878249beb3a7c829e74bf7ce9/xxhash-3.6.0-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bec91b562d8012dae276af8025a55811b875baace6af510412a5e58e3121bc54", size = 196409, upload-time = "2025-10-02T14:35:20.31Z" }, - { url = "https://files.pythonhosted.org/packages/e3/8e/c6d158d12a79bbd0b878f8355432075fc82759e356ab5a111463422a239b/xxhash-3.6.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78e7f2f4c521c30ad5e786fdd6bae89d47a32672a80195467b5de0480aa97b1f", size = 215736, upload-time = "2025-10-02T14:35:21.616Z" }, - { url = "https://files.pythonhosted.org/packages/bc/68/c4c80614716345d55071a396cf03d06e34b5f4917a467faf43083c995155/xxhash-3.6.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ed0df1b11a79856df5ffcab572cbd6b9627034c1c748c5566fa79df9048a7c5", size = 214833, upload-time = "2025-10-02T14:35:23.32Z" }, - { url = "https://files.pythonhosted.org/packages/7e/e9/ae27c8ffec8b953efa84c7c4a6c6802c263d587b9fc0d6e7cea64e08c3af/xxhash-3.6.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0e4edbfc7d420925b0dd5e792478ed393d6e75ff8fc219a6546fb446b6a417b1", size = 448348, upload-time = "2025-10-02T14:35:25.111Z" }, - { url = "https://files.pythonhosted.org/packages/d7/6b/33e21afb1b5b3f46b74b6bd1913639066af218d704cc0941404ca717fc57/xxhash-3.6.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fba27a198363a7ef87f8c0f6b171ec36b674fe9053742c58dd7e3201c1ab30ee", size = 196070, upload-time = "2025-10-02T14:35:26.586Z" }, - { url = "https://files.pythonhosted.org/packages/96/b6/fcabd337bc5fa624e7203aa0fa7d0c49eed22f72e93229431752bddc83d9/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:794fe9145fe60191c6532fa95063765529770edcdd67b3d537793e8004cabbfd", size = 212907, upload-time = "2025-10-02T14:35:28.087Z" }, - { url = "https://files.pythonhosted.org/packages/4b/d3/9ee6160e644d660fcf176c5825e61411c7f62648728f69c79ba237250143/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:6105ef7e62b5ac73a837778efc331a591d8442f8ef5c7e102376506cb4ae2729", size = 200839, upload-time = "2025-10-02T14:35:29.857Z" }, - { url = "https://files.pythonhosted.org/packages/0d/98/e8de5baa5109394baf5118f5e72ab21a86387c4f89b0e77ef3e2f6b0327b/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:f01375c0e55395b814a679b3eea205db7919ac2af213f4a6682e01220e5fe292", size = 213304, upload-time = "2025-10-02T14:35:31.222Z" }, - { url = "https://files.pythonhosted.org/packages/7b/1d/71056535dec5c3177eeb53e38e3d367dd1d16e024e63b1cee208d572a033/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:d706dca2d24d834a4661619dcacf51a75c16d65985718d6a7d73c1eeeb903ddf", size = 416930, upload-time = "2025-10-02T14:35:32.517Z" }, - { url = "https://files.pythonhosted.org/packages/dc/6c/5cbde9de2cd967c322e651c65c543700b19e7ae3e0aae8ece3469bf9683d/xxhash-3.6.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5f059d9faeacd49c0215d66f4056e1326c80503f51a1532ca336a385edadd033", size = 193787, upload-time = "2025-10-02T14:35:33.827Z" }, - { url = "https://files.pythonhosted.org/packages/19/fa/0172e350361d61febcea941b0cc541d6e6c8d65d153e85f850a7b256ff8a/xxhash-3.6.0-cp313-cp313t-win32.whl", hash = "sha256:1244460adc3a9be84731d72b8e80625788e5815b68da3da8b83f78115a40a7ec", size = 30916, upload-time = "2025-10-02T14:35:35.107Z" }, - { url = "https://files.pythonhosted.org/packages/ad/e6/e8cf858a2b19d6d45820f072eff1bea413910592ff17157cabc5f1227a16/xxhash-3.6.0-cp313-cp313t-win_amd64.whl", hash = "sha256:b1e420ef35c503869c4064f4a2f2b08ad6431ab7b229a05cce39d74268bca6b8", size = 31799, upload-time = "2025-10-02T14:35:36.165Z" }, - { url = "https://files.pythonhosted.org/packages/56/15/064b197e855bfb7b343210e82490ae672f8bc7cdf3ddb02e92f64304ee8a/xxhash-3.6.0-cp313-cp313t-win_arm64.whl", hash = "sha256:ec44b73a4220623235f67a996c862049f375df3b1052d9899f40a6382c32d746", size = 28044, upload-time = "2025-10-02T14:35:37.195Z" }, - { url = "https://files.pythonhosted.org/packages/7e/5e/0138bc4484ea9b897864d59fce9be9086030825bc778b76cb5a33a906d37/xxhash-3.6.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a40a3d35b204b7cc7643cbcf8c9976d818cb47befcfac8bbefec8038ac363f3e", size = 32754, upload-time = "2025-10-02T14:35:38.245Z" }, - { url = "https://files.pythonhosted.org/packages/18/d7/5dac2eb2ec75fd771957a13e5dda560efb2176d5203f39502a5fc571f899/xxhash-3.6.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a54844be970d3fc22630b32d515e79a90d0a3ddb2644d8d7402e3c4c8da61405", size = 30846, upload-time = "2025-10-02T14:35:39.6Z" }, - { url = "https://files.pythonhosted.org/packages/fe/71/8bc5be2bb00deb5682e92e8da955ebe5fa982da13a69da5a40a4c8db12fb/xxhash-3.6.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:016e9190af8f0a4e3741343777710e3d5717427f175adfdc3e72508f59e2a7f3", size = 194343, upload-time = "2025-10-02T14:35:40.69Z" }, - { url = "https://files.pythonhosted.org/packages/e7/3b/52badfb2aecec2c377ddf1ae75f55db3ba2d321c5e164f14461c90837ef3/xxhash-3.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f6f72232f849eb9d0141e2ebe2677ece15adfd0fa599bc058aad83c714bb2c6", size = 213074, upload-time = "2025-10-02T14:35:42.29Z" }, - { url = "https://files.pythonhosted.org/packages/a2/2b/ae46b4e9b92e537fa30d03dbc19cdae57ed407e9c26d163895e968e3de85/xxhash-3.6.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63275a8aba7865e44b1813d2177e0f5ea7eadad3dd063a21f7cf9afdc7054063", size = 212388, upload-time = "2025-10-02T14:35:43.929Z" }, - { url = "https://files.pythonhosted.org/packages/f5/80/49f88d3afc724b4ac7fbd664c8452d6db51b49915be48c6982659e0e7942/xxhash-3.6.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cd01fa2aa00d8b017c97eb46b9a794fbdca53fc14f845f5a328c71254b0abb7", size = 445614, upload-time = "2025-10-02T14:35:45.216Z" }, - { url = "https://files.pythonhosted.org/packages/ed/ba/603ce3961e339413543d8cd44f21f2c80e2a7c5cfe692a7b1f2cccf58f3c/xxhash-3.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0226aa89035b62b6a86d3c68df4d7c1f47a342b8683da2b60cedcddb46c4d95b", size = 194024, upload-time = "2025-10-02T14:35:46.959Z" }, - { url = "https://files.pythonhosted.org/packages/78/d1/8e225ff7113bf81545cfdcd79eef124a7b7064a0bba53605ff39590b95c2/xxhash-3.6.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c6e193e9f56e4ca4923c61238cdaced324f0feac782544eb4c6d55ad5cc99ddd", size = 210541, upload-time = "2025-10-02T14:35:48.301Z" }, - { url = "https://files.pythonhosted.org/packages/6f/58/0f89d149f0bad89def1a8dd38feb50ccdeb643d9797ec84707091d4cb494/xxhash-3.6.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9176dcaddf4ca963d4deb93866d739a343c01c969231dbe21680e13a5d1a5bf0", size = 198305, upload-time = "2025-10-02T14:35:49.584Z" }, - { url = "https://files.pythonhosted.org/packages/11/38/5eab81580703c4df93feb5f32ff8fa7fe1e2c51c1f183ee4e48d4bb9d3d7/xxhash-3.6.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c1ce4009c97a752e682b897aa99aef84191077a9433eb237774689f14f8ec152", size = 210848, upload-time = "2025-10-02T14:35:50.877Z" }, - { url = "https://files.pythonhosted.org/packages/5e/6b/953dc4b05c3ce678abca756416e4c130d2382f877a9c30a20d08ee6a77c0/xxhash-3.6.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8cb2f4f679b01513b7adbb9b1b2f0f9cdc31b70007eaf9d59d0878809f385b11", size = 414142, upload-time = "2025-10-02T14:35:52.15Z" }, - { url = "https://files.pythonhosted.org/packages/08/a9/238ec0d4e81a10eb5026d4a6972677cbc898ba6c8b9dbaec12ae001b1b35/xxhash-3.6.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:653a91d7c2ab54a92c19ccf43508b6a555440b9be1bc8be553376778be7f20b5", size = 191547, upload-time = "2025-10-02T14:35:53.547Z" }, - { url = "https://files.pythonhosted.org/packages/f1/ee/3cf8589e06c2164ac77c3bf0aa127012801128f1feebf2a079272da5737c/xxhash-3.6.0-cp314-cp314-win32.whl", hash = "sha256:a756fe893389483ee8c394d06b5ab765d96e68fbbfe6fde7aa17e11f5720559f", size = 31214, upload-time = "2025-10-02T14:35:54.746Z" }, - { url = "https://files.pythonhosted.org/packages/02/5d/a19552fbc6ad4cb54ff953c3908bbc095f4a921bc569433d791f755186f1/xxhash-3.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:39be8e4e142550ef69629c9cd71b88c90e9a5db703fecbcf265546d9536ca4ad", size = 32290, upload-time = "2025-10-02T14:35:55.791Z" }, - { url = "https://files.pythonhosted.org/packages/b1/11/dafa0643bc30442c887b55baf8e73353a344ee89c1901b5a5c54a6c17d39/xxhash-3.6.0-cp314-cp314-win_arm64.whl", hash = "sha256:25915e6000338999236f1eb68a02a32c3275ac338628a7eaa5a269c401995679", size = 28795, upload-time = "2025-10-02T14:35:57.162Z" }, - { url = "https://files.pythonhosted.org/packages/2c/db/0e99732ed7f64182aef4a6fb145e1a295558deec2a746265dcdec12d191e/xxhash-3.6.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c5294f596a9017ca5a3e3f8884c00b91ab2ad2933cf288f4923c3fd4346cf3d4", size = 32955, upload-time = "2025-10-02T14:35:58.267Z" }, - { url = "https://files.pythonhosted.org/packages/55/f4/2a7c3c68e564a099becfa44bb3d398810cc0ff6749b0d3cb8ccb93f23c14/xxhash-3.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1cf9dcc4ab9cff01dfbba78544297a3a01dafd60f3bde4e2bfd016cf7e4ddc67", size = 31072, upload-time = "2025-10-02T14:35:59.382Z" }, - { url = "https://files.pythonhosted.org/packages/c6/d9/72a29cddc7250e8a5819dad5d466facb5dc4c802ce120645630149127e73/xxhash-3.6.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:01262da8798422d0685f7cef03b2bd3f4f46511b02830861df548d7def4402ad", size = 196579, upload-time = "2025-10-02T14:36:00.838Z" }, - { url = "https://files.pythonhosted.org/packages/63/93/b21590e1e381040e2ca305a884d89e1c345b347404f7780f07f2cdd47ef4/xxhash-3.6.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51a73fb7cb3a3ead9f7a8b583ffd9b8038e277cdb8cb87cf890e88b3456afa0b", size = 215854, upload-time = "2025-10-02T14:36:02.207Z" }, - { url = "https://files.pythonhosted.org/packages/ce/b8/edab8a7d4fa14e924b29be877d54155dcbd8b80be85ea00d2be3413a9ed4/xxhash-3.6.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b9c6df83594f7df8f7f708ce5ebeacfc69f72c9fbaaababf6cf4758eaada0c9b", size = 214965, upload-time = "2025-10-02T14:36:03.507Z" }, - { url = "https://files.pythonhosted.org/packages/27/67/dfa980ac7f0d509d54ea0d5a486d2bb4b80c3f1bb22b66e6a05d3efaf6c0/xxhash-3.6.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:627f0af069b0ea56f312fd5189001c24578868643203bca1abbc2c52d3a6f3ca", size = 448484, upload-time = "2025-10-02T14:36:04.828Z" }, - { url = "https://files.pythonhosted.org/packages/8c/63/8ffc2cc97e811c0ca5d00ab36604b3ea6f4254f20b7bc658ca825ce6c954/xxhash-3.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa912c62f842dfd013c5f21a642c9c10cd9f4c4e943e0af83618b4a404d9091a", size = 196162, upload-time = "2025-10-02T14:36:06.182Z" }, - { url = "https://files.pythonhosted.org/packages/4b/77/07f0e7a3edd11a6097e990f6e5b815b6592459cb16dae990d967693e6ea9/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b465afd7909db30168ab62afe40b2fcf79eedc0b89a6c0ab3123515dc0df8b99", size = 213007, upload-time = "2025-10-02T14:36:07.733Z" }, - { url = "https://files.pythonhosted.org/packages/ae/d8/bc5fa0d152837117eb0bef6f83f956c509332ce133c91c63ce07ee7c4873/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:a881851cf38b0a70e7c4d3ce81fc7afd86fbc2a024f4cfb2a97cf49ce04b75d3", size = 200956, upload-time = "2025-10-02T14:36:09.106Z" }, - { url = "https://files.pythonhosted.org/packages/26/a5/d749334130de9411783873e9b98ecc46688dad5db64ca6e04b02acc8b473/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:9b3222c686a919a0f3253cfc12bb118b8b103506612253b5baeaac10d8027cf6", size = 213401, upload-time = "2025-10-02T14:36:10.585Z" }, - { url = "https://files.pythonhosted.org/packages/89/72/abed959c956a4bfc72b58c0384bb7940663c678127538634d896b1195c10/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:c5aa639bc113e9286137cec8fadc20e9cd732b2cc385c0b7fa673b84fc1f2a93", size = 417083, upload-time = "2025-10-02T14:36:12.276Z" }, - { url = "https://files.pythonhosted.org/packages/0c/b3/62fd2b586283b7d7d665fb98e266decadf31f058f1cf6c478741f68af0cb/xxhash-3.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5c1343d49ac102799905e115aee590183c3921d475356cb24b4de29a4bc56518", size = 193913, upload-time = "2025-10-02T14:36:14.025Z" }, - { url = "https://files.pythonhosted.org/packages/9a/9a/c19c42c5b3f5a4aad748a6d5b4f23df3bed7ee5445accc65a0fb3ff03953/xxhash-3.6.0-cp314-cp314t-win32.whl", hash = "sha256:5851f033c3030dd95c086b4a36a2683c2ff4a799b23af60977188b057e467119", size = 31586, upload-time = "2025-10-02T14:36:15.603Z" }, - { url = "https://files.pythonhosted.org/packages/03/d6/4cc450345be9924fd5dc8c590ceda1db5b43a0a889587b0ae81a95511360/xxhash-3.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0444e7967dac37569052d2409b00a8860c2135cff05502df4da80267d384849f", size = 32526, upload-time = "2025-10-02T14:36:16.708Z" }, - { url = "https://files.pythonhosted.org/packages/0f/c9/7243eb3f9eaabd1a88a5a5acadf06df2d83b100c62684b7425c6a11bcaa8/xxhash-3.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:bb79b1e63f6fd84ec778a4b1916dfe0a7c3fdb986c06addd5db3a0d413819d95", size = 28898, upload-time = "2025-10-02T14:36:17.843Z" }, - { url = "https://files.pythonhosted.org/packages/93/1e/8aec23647a34a249f62e2398c42955acd9b4c6ed5cf08cbea94dc46f78d2/xxhash-3.6.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0f7b7e2ec26c1666ad5fc9dbfa426a6a3367ceaf79db5dd76264659d509d73b0", size = 30662, upload-time = "2025-10-02T14:37:01.743Z" }, - { url = "https://files.pythonhosted.org/packages/b8/0b/b14510b38ba91caf43006209db846a696ceea6a847a0c9ba0a5b1adc53d6/xxhash-3.6.0-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5dc1e14d14fa0f5789ec29a7062004b5933964bb9b02aae6622b8f530dc40296", size = 41056, upload-time = "2025-10-02T14:37:02.879Z" }, - { url = "https://files.pythonhosted.org/packages/50/55/15a7b8a56590e66ccd374bbfa3f9ffc45b810886c8c3b614e3f90bd2367c/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:881b47fc47e051b37d94d13e7455131054b56749b91b508b0907eb07900d1c13", size = 36251, upload-time = "2025-10-02T14:37:04.44Z" }, - { url = "https://files.pythonhosted.org/packages/62/b2/5ac99a041a29e58e95f907876b04f7067a0242cb85b5f39e726153981503/xxhash-3.6.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c6dc31591899f5e5666f04cc2e529e69b4072827085c1ef15294d91a004bc1bd", size = 32481, upload-time = "2025-10-02T14:37:05.869Z" }, - { url = "https://files.pythonhosted.org/packages/7b/d9/8d95e906764a386a3d3b596f3c68bb63687dfca806373509f51ce8eea81f/xxhash-3.6.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:15e0dac10eb9309508bfc41f7f9deaa7755c69e35af835db9cb10751adebc35d", size = 31565, upload-time = "2025-10-02T14:37:06.966Z" }, +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/97/1a8cebf0a6650417f08a18231590e2515aacd5ce39c3ad8b9e013ebd437d/xxhash-3.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:27a9e475157f7315826118e3f3127909a0fe25f1b43d3d3be9c584f9d265f937", size = 34695, upload-time = "2026-07-06T10:43:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/2f/cf/745b9bc0dd9c341bc074b5fc700db7bbef0f3b69ab21446492296ab37e50/xxhash-3.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9b2ce44bf8f4a1d01f418b3110ff8dff32fd3f3e836c0e06333c3725f243fa6c", size = 32376, upload-time = "2026-07-06T10:43:41.97Z" }, + { url = "https://files.pythonhosted.org/packages/65/a4/8512a901b1d6ad4a9838d1b40385907a879d7e005a5afbec5d39526b69f6/xxhash-3.8.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:942bc86e9be6fdd6e1175048f5fe8f8fdaaf2309dd1323ef1e155a69cd346780", size = 217470, upload-time = "2026-07-06T10:43:43.572Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ad/0ffd8094ea29579bb2dc42fa74d08570e9ea3d95db561e6b1105e69b9ca6/xxhash-3.8.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0204701e6d01f64254e0e5ff4255812b1febe027ddd7dda63372e27f98b5e91f", size = 237799, upload-time = "2026-07-06T10:43:45.248Z" }, + { url = "https://files.pythonhosted.org/packages/b3/90/783c6b3f9336bd07449fe672be32cef6833633936bbfda8d3b23ee18d202/xxhash-3.8.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7dc4bdf008f77c88d544849c48c1a40faf25a5eff6cc466de2e8edc37c191fce", size = 262587, upload-time = "2026-07-06T10:43:46.733Z" }, + { url = "https://files.pythonhosted.org/packages/c4/77/ba0316a7c3e661b86830a47ae4987798616ce1b15af8d2a6358e2d89ef60/xxhash-3.8.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5c566b123dce7e4867ca518434cdfb9f84e5023771235b2e3107a26c9a41cbd8", size = 238484, upload-time = "2026-07-06T10:43:48.453Z" }, + { url = "https://files.pythonhosted.org/packages/09/79/33001037c1cba90f4ced38b257161c13452024c0db44208f883e2e47f3fc/xxhash-3.8.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f23083e1bd9d901f844af7a126727c486e7eada9a1a6791c8f7e73f94fac656", size = 469909, upload-time = "2026-07-06T10:43:50.188Z" }, + { url = "https://files.pythonhosted.org/packages/45/90/237eded9dd6ae638083294e5a9f77b317aaebd480a330806b39c192a0de1/xxhash-3.8.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64af54dd1c3a45a27c04942f9a1a4683322bdd127f4745cca4e02549c1d2d2bb", size = 217166, upload-time = "2026-07-06T10:43:51.816Z" }, + { url = "https://files.pythonhosted.org/packages/0b/6a/8cb439dc9920e1468e1c2d69ef77cbeb4be3b1ae9f4b5344c07a2b59af18/xxhash-3.8.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8ea8a141eeced4f6262ab6dd71c681ac546a558c30bb586abe087d814b5f85ea", size = 307593, upload-time = "2026-07-06T10:43:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c6/c0607d373c8affea92101a3926c4fc8b026bcf8983e05fd58f3a0380ebf8/xxhash-3.8.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a98b2f95cab589e0f5e92c48431afb4d56238b8bf6668edcc66166180e9b509b", size = 234702, upload-time = "2026-07-06T10:43:55.042Z" }, + { url = "https://files.pythonhosted.org/packages/5b/cb/f4cfd456624c1f017858168b7ba9443dad810da8aac779a612658450e827/xxhash-3.8.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:1b86ae798a976ccbc1d02af6ccb98f5b4d24756b1f65e995f11d10fe071f486f", size = 265749, upload-time = "2026-07-06T10:43:56.749Z" }, + { url = "https://files.pythonhosted.org/packages/33/f3/9006669c04b01206e21b2177425c649461ba188930a052c2f1728d6ec6a8/xxhash-3.8.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81f4ed9ca9644bc95cd976bfe10f7a4cafab8ffdc3aed52877d4600e445be7ef", size = 221992, upload-time = "2026-07-06T10:43:58.12Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/7e6f3eaa05df5e0b6c94aa452b0672801f7031e602081f07fd441aaaaed5/xxhash-3.8.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:cb3fe820c27593f170770d6c8d791936cf6275d9269405fbb7b30a55363c10c8", size = 236899, upload-time = "2026-07-06T10:43:59.562Z" }, + { url = "https://files.pythonhosted.org/packages/da/cc/bbaee4987f3aab1d7b33bb430bb49e940646160af448b9167431c931126d/xxhash-3.8.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:7345007c12780985de4fd740148776d1eee18c0d41407c6fa1e48c5450304fe5", size = 297934, upload-time = "2026-07-06T10:44:01.132Z" }, + { url = "https://files.pythonhosted.org/packages/a7/97/6bee358660eb8b4f73c00b00b00bc616ebde00e1ab4b67c63486ce360648/xxhash-3.8.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:12eaeaa9ab8b9e6033a1fa5f6b338aaf55ff4df4bee11b59fd6ee03b19186ee4", size = 439315, upload-time = "2026-07-06T10:44:02.878Z" }, + { url = "https://files.pythonhosted.org/packages/c6/50/7e35275f39256bedace0c3cd5be3c72d4ac9d5aecf5e5fdc3530337cd263/xxhash-3.8.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2a845687219ba3214126f14a8a5861f97c9e065a7d0b8252adb6df13eea86fb", size = 214038, upload-time = "2026-07-06T10:44:04.504Z" }, + { url = "https://files.pythonhosted.org/packages/59/2d/69d02d096ee50bdf3ef0d208d874f52c71b1aa6906066bce3c52fedb8bc6/xxhash-3.8.1-cp310-cp310-win32.whl", hash = "sha256:656256c9f9303e47f07d5cb8ae4468285370adfafd7ba48aea33a458e7697626", size = 31939, upload-time = "2026-07-06T10:44:06.213Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1d/e06fca9844919ca91c6587d530cfa1e745830ec73ad38f44f04b25d1bfb7/xxhash-3.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:27cfc2f1ed76f956f36dfe0c56e5f5a3e94cd91eb78b893f63e2ef2ae404fcdf", size = 32729, upload-time = "2026-07-06T10:44:07.621Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/800648d99039927b5a86d8ae02cd86a556a5ee1678d388216f6b44c8966c/xxhash-3.8.1-cp310-cp310-win_arm64.whl", hash = "sha256:c85949d02c85adf6d786eb94858e124989a632a4e65739835b2fc5761827fac3", size = 29215, upload-time = "2026-07-06T10:44:08.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/5a/05eaa129555f85476a3e16ff869e95f81a78bbe4647eef9d0229f515a317/xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147", size = 34699, upload-time = "2026-07-06T10:44:10.14Z" }, + { url = "https://files.pythonhosted.org/packages/80/59/0df1133958b2228929355e022aab1e958c7b2c43e27bf7f59bc9edfa8a54/xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10", size = 32373, upload-time = "2026-07-06T10:44:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bf/1cfda5b5e6bf26617812b4a31662ef2220d2ad04e0a55b8ff9eb36e56a5c/xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670", size = 220284, upload-time = "2026-07-06T10:44:12.683Z" }, + { url = "https://files.pythonhosted.org/packages/70/93/45dc0ad7913b69e5b08bd039236cf628380e4c9cc76a8a4c6625a328e058/xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05", size = 240980, upload-time = "2026-07-06T10:44:14.297Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/f28ba7d17f2c1410ee397982c817ab1bd5b2701070c2d2c373539aad000a/xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae", size = 264526, upload-time = "2026-07-06T10:44:16.017Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d0/f10651cec2c7981b20d693deae6bdfc438427d92be2db4ccabb6181f0021/xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a", size = 241369, upload-time = "2026-07-06T10:44:17.698Z" }, + { url = "https://files.pythonhosted.org/packages/ff/40/136e0cbaf5db51e191423b1c98643593189f02b6cd90837bf64b19113d70/xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c", size = 473186, upload-time = "2026-07-06T10:44:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3f/6aa808a96bdc43dba9a740dec56c744526ee3c0019e32c75e810fa90ae4d/xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60", size = 220092, upload-time = "2026-07-06T10:44:20.956Z" }, + { url = "https://files.pythonhosted.org/packages/47/28/a8675e78a9ced96dab853416162268e10e05b452e95db7888cf69f58ac5f/xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342", size = 309846, upload-time = "2026-07-06T10:44:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/0f/7fe4d4ef4e69f0033e012396ee2a115886bca7b10b7e45ce398626436bfc/xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723", size = 237659, upload-time = "2026-07-06T10:44:24.135Z" }, + { url = "https://files.pythonhosted.org/packages/38/8f/83e9e31d4ed57fe963b99cb5b13a23e3e0f0dad1885aa0ebd2a7819dd423/xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a", size = 268737, upload-time = "2026-07-06T10:44:25.875Z" }, + { url = "https://files.pythonhosted.org/packages/57/79/7e7de46dbe5d1f49afc96a0bc42e6b8df24eae3d6bad6007b99e42f48430/xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937", size = 224955, upload-time = "2026-07-06T10:44:27.777Z" }, + { url = "https://files.pythonhosted.org/packages/ec/34/b8540839e958d5ef5c6101af6f16032109e7099698ae8edbc8dcefe4d8f4/xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661", size = 239653, upload-time = "2026-07-06T10:44:29.422Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/a735d05f7f859354acadabe470ff40e2c46672275f96dcf096a761904def/xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673", size = 300213, upload-time = "2026-07-06T10:44:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/3e1cb020237b68117fc212dc5f9753b87f865b4dfee7c1ce62d0836955b5/xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872", size = 442508, upload-time = "2026-07-06T10:44:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/bf/f80090622141cc734b039ce1d15ce3ff6dced375e9680249bf5b9b8c6bf9/xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef", size = 216853, upload-time = "2026-07-06T10:44:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a3/60157acecc307b238d3651c2483168e224b48b23a36ae6d6903588341d80/xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792", size = 31936, upload-time = "2026-07-06T10:44:36.542Z" }, + { url = "https://files.pythonhosted.org/packages/59/5c/ef70c418d878d187b8da56d4cdc06aea6cf5e456b301e96e51e1d2cc8625/xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f", size = 32724, upload-time = "2026-07-06T10:44:38.177Z" }, + { url = "https://files.pythonhosted.org/packages/2c/25/f008db952cec6b2a26445b456eeed2ebebd65e08e848ebe09ed6ac0634e6/xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d", size = 29212, upload-time = "2026-07-06T10:44:39.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/91/f65c34a7aa7b4e7cf4854f8e6ef3f7ee32ceac41d4f008da0780db0612f6/xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc", size = 34680, upload-time = "2026-07-06T10:44:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/57/04/b10a245a4c09a9cfa88f8e9ae755029413ad1ac17047f9a61906e5ae0799/xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215", size = 32397, upload-time = "2026-07-06T10:44:42.196Z" }, + { url = "https://files.pythonhosted.org/packages/3a/75/45ab795b5945b6388583bd75202106af505537935566c15a1577797a0e08/xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478", size = 220549, upload-time = "2026-07-06T10:44:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/13/44/5ba2bd0a14ddf4193fc7d8ec29625f659f22c06d60b28f04bf46305d8330/xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc", size = 241186, upload-time = "2026-07-06T10:44:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/23/32/c4147def4d1e4538b906f82731e0ba23424377fc50a7cddd03cd284c8f63/xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8", size = 264852, upload-time = "2026-07-06T10:44:47.199Z" }, + { url = "https://files.pythonhosted.org/packages/6c/bd/71ed14f4f0318bb7fd7b2ec51999413487fa8da8d41208e84d50d1ef0f98/xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56", size = 242663, upload-time = "2026-07-06T10:44:48.846Z" }, + { url = "https://files.pythonhosted.org/packages/91/09/70af22c565a8473b3f2ae73f88e7721af281bc4a575236dbd1970c9f76f6/xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170", size = 473510, upload-time = "2026-07-06T10:44:50.695Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/34db781c8f0cf99c544ca1f2bc2e5bf55426e1eb4ca6de8ea5da56a9f352/xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc", size = 220469, upload-time = "2026-07-06T10:44:52.422Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/9a184f615fa5a4dce30c01534f62946ce5a11ce40f73785cbd356ccabaa9/xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c", size = 310290, upload-time = "2026-07-06T10:44:54.142Z" }, + { url = "https://files.pythonhosted.org/packages/a9/dc/9b9a9789011ee153723a5eb9e7dd7fcbae2ba9b3fe7a729249ca7c252056/xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d", size = 238173, upload-time = "2026-07-06T10:44:55.693Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4d/71c6005ada9dcb608a4e1902e8475ecadb5f3fbfa04e1e244d276a2d0c43/xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb", size = 269026, upload-time = "2026-07-06T10:44:57.424Z" }, + { url = "https://files.pythonhosted.org/packages/2f/87/d6c036ba25dfbd9c8633be5aa86fc9474bbb9e2c68212a841d090abe7344/xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061", size = 224970, upload-time = "2026-07-06T10:44:59.085Z" }, + { url = "https://files.pythonhosted.org/packages/48/62/4c1f035a41c5752aa05e195b6c904c07b94fe9061a16de61e72a6e6b135f/xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887", size = 240820, upload-time = "2026-07-06T10:45:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/da/14/d39d565069b87e86d21a2af2a31d04db79249d25aa8d5b62959056a89857/xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e", size = 300619, upload-time = "2026-07-06T10:45:02.716Z" }, + { url = "https://files.pythonhosted.org/packages/13/22/75467acc887edc8cf71c97ab1708feb3df7a88bda589b9f399765c6387d2/xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975", size = 443267, upload-time = "2026-07-06T10:45:04.653Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b6/1da3baa5fa6ef705e3425fddd382be7dfc4dfba2686df90a20f16e9c7b1b/xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a", size = 217338, upload-time = "2026-07-06T10:45:06.304Z" }, + { url = "https://files.pythonhosted.org/packages/78/dd/b5295a9f97484e7a1c2b283a742ca45e3104991c55a1ef670dde161829ba/xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e", size = 31970, upload-time = "2026-07-06T10:45:07.823Z" }, + { url = "https://files.pythonhosted.org/packages/ec/31/3fa0b807d7e21515cd975e7fe5c039d52ac3e9401a96d6ad68dae6305215/xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724", size = 32741, upload-time = "2026-07-06T10:45:09.42Z" }, + { url = "https://files.pythonhosted.org/packages/b8/05/86feada74e239600e6875aa507afb40482a89b92700aa74a92da83bdcb77/xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396", size = 29234, upload-time = "2026-07-06T10:45:10.809Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/446bb782cd0d27007a917b5569a08dd73219c3e8d6e459014db104b27bdb/xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913", size = 38562, upload-time = "2026-07-06T10:45:12.425Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ec/c0c45627eaa6be7a5d6117423adf8f7a15b17ee74b4b17072cca5959a225/xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec", size = 36656, upload-time = "2026-07-06T10:45:13.932Z" }, + { url = "https://files.pythonhosted.org/packages/f6/94/8324c04cc7597154caaeba6c094e01fbd2e7601d01e7a13eea9f5420e77b/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297", size = 31169, upload-time = "2026-07-06T10:45:15.687Z" }, + { url = "https://files.pythonhosted.org/packages/40/a4/beb6bb26e1184e126dbe7a5682330214ef54dcfbf882078aa9f4b5428d42/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed", size = 32177, upload-time = "2026-07-06T10:45:17.035Z" }, + { url = "https://files.pythonhosted.org/packages/56/0f/fc4c92a5a528f839b34b6419b2e53c8597f2a629d5a1f5d721f65bfa1fd6/xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276", size = 34642, upload-time = "2026-07-06T10:45:18.39Z" }, + { url = "https://files.pythonhosted.org/packages/d4/58/edbfb141d4000767ac6a9694f8ac0763e2c2e983e65c9e31620ba56e2667/xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0", size = 34684, upload-time = "2026-07-06T10:45:20.033Z" }, + { url = "https://files.pythonhosted.org/packages/07/3f/5072f1f0f5714186f0ac2a0b5a4929ce30d4b845e94886b6c01b6ebda0be/xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315", size = 32401, upload-time = "2026-07-06T10:45:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/49/c7/802ea2f9c2ed59219934d6d65c470d502b1788043eae277a52af8658bda6/xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907", size = 220617, upload-time = "2026-07-06T10:45:23.234Z" }, + { url = "https://files.pythonhosted.org/packages/99/a8/e10488efd31fcb13fcd6acbc6e788f10c6f8e3a0cc4ae3eb89dc19c55a12/xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0", size = 241295, upload-time = "2026-07-06T10:45:25.364Z" }, + { url = "https://files.pythonhosted.org/packages/18/cc/14180b17d44892a631f8ae7323c30bfbb1328efc8209e528a480293528ac/xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2", size = 264688, upload-time = "2026-07-06T10:45:27.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/72/a14019d0c5f6c41ee407a503036ae32787c91325ca218a96a9b5627be651/xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329", size = 242740, upload-time = "2026-07-06T10:45:28.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/92550e556c6fcfcb96c6a336945eb53a431ed43120ed749636debb16c5cf/xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0", size = 473599, upload-time = "2026-07-06T10:45:30.524Z" }, + { url = "https://files.pythonhosted.org/packages/29/83/e361d3c1acd1b21e1d489616de6fa4aaf843365d8179f612e3743eac20a9/xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437", size = 220559, upload-time = "2026-07-06T10:45:32.979Z" }, + { url = "https://files.pythonhosted.org/packages/05/01/006a4243c2c2a6831827f9999f6d1c23feeef100eb023c1f886022a00bf3/xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62", size = 310383, upload-time = "2026-07-06T10:45:35.875Z" }, + { url = "https://files.pythonhosted.org/packages/d8/20/af388e8bf9f9a0f89eeef7d2a1935d176ee1c20bc6adeda05035879379cf/xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf", size = 238228, upload-time = "2026-07-06T10:45:38.02Z" }, + { url = "https://files.pythonhosted.org/packages/63/6b/4666579a87eebd1744663c404297355fa0658617b015cedfa58810ee7036/xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf", size = 269137, upload-time = "2026-07-06T10:45:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/de/d3/e963a8a46f900a137d91b02144d8ea07a8f812971b138204a3b2f8b8e55c/xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3", size = 225068, upload-time = "2026-07-06T10:45:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/aa/80/9d181dbcde4b0fe48375f48833a5832d4b8cd2b349b15110c92ee472d874/xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104", size = 240874, upload-time = "2026-07-06T10:45:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/39/15/ce3ab5a1cd27ead25a5196e55a7284220f6ad6e316da494ffd900b2b600f/xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e", size = 300702, upload-time = "2026-07-06T10:45:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/96/c0/2281a8ab5f2a62dbf57a23c58a01ccc1d98abf40f71193c8a81f59e759b5/xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893", size = 443351, upload-time = "2026-07-06T10:45:47.188Z" }, + { url = "https://files.pythonhosted.org/packages/81/2e/071a58c1a53a52d4f7a3aa0987be0c396dffd40da8204805fe1b130a81f4/xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3", size = 217396, upload-time = "2026-07-06T10:45:48.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/44/36ab58134badd9d3433fc7b53c4ca8d113d8e807782885628640f8297a4d/xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a", size = 31974, upload-time = "2026-07-06T10:45:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/96/2a/2a0b84798448e766f7b89ceed073cb0cb5a43fc9ebbacbdea74a38de18e3/xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55", size = 32739, upload-time = "2026-07-06T10:45:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/d4/60/bb51dbf7c363ff88a7cbd50b7959718219577ef44d7cf255929ffc4a2194/xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a", size = 29239, upload-time = "2026-07-06T10:45:53.714Z" }, + { url = "https://files.pythonhosted.org/packages/56/d3/827ca123c2ee5443a6aaed3c5dd199237dc2f010e2bebd7ec09ef36f3a5f/xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc", size = 34964, upload-time = "2026-07-06T10:45:55.535Z" }, + { url = "https://files.pythonhosted.org/packages/05/67/67ae2a3ccdeb8b8ef025d35aee9edd1d26c3abe5051d47da9286232afbf8/xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92", size = 32697, upload-time = "2026-07-06T10:45:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/38/5a/3d3994346e1f45493679cb5c1ffc2bf454e410e9d1e8a662d253becee91e/xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75", size = 225954, upload-time = "2026-07-06T10:45:58.897Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2c/53169270309b7cd8e05504e07fe123bac053b89d00ac63617faacf0a2ec0/xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda", size = 249776, upload-time = "2026-07-06T10:46:00.977Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/5c551d8d592f944506f7c5185e210255c15e672a3c6008c156a1bd9b775e/xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629", size = 274776, upload-time = "2026-07-06T10:46:02.869Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/d3a762270cee2d7bcd0e25e28c623e5f3f5c0dc637b66e3e47dd5b0bb3f0/xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068", size = 252056, upload-time = "2026-07-06T10:46:04.688Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/b78e4373b2cb6d1c42af60ea2d7e9146ad0710b239ac7f706d5d31d5bb98/xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1", size = 482108, upload-time = "2026-07-06T10:46:06.498Z" }, + { url = "https://files.pythonhosted.org/packages/e6/0d/642d923336ea61a15f8ce64fc7e078729e6e06c3a026e517fa79b2c23b7a/xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647", size = 226739, upload-time = "2026-07-06T10:46:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0a/a37d6da6427d45a8d23e3ee3a0ca9c9d4a90364849c6637fe2963a755f9b/xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1", size = 319658, upload-time = "2026-07-06T10:46:10.504Z" }, + { url = "https://files.pythonhosted.org/packages/4a/51/ebbd40da8a3f1bc53b4b7a9a87f8e28bd95c5f21bc14b8a57860cf367d1b/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113", size = 246059, upload-time = "2026-07-06T10:46:12.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/4c/d9014030147e1f0bb26e7da47aa240dd9ec61c763c573e558111d869f8e1/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9", size = 275535, upload-time = "2026-07-06T10:46:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/caee2db41fadcd5a25aa4323213f9afec5a8586d4e419241e3d659362bd7/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa", size = 231292, upload-time = "2026-07-06T10:46:16.452Z" }, + { url = "https://files.pythonhosted.org/packages/0b/60/f52f08bcdc904c4514ea5c25caa19e9f3214144434a6ff96dc82dc1cbddd/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31", size = 250490, upload-time = "2026-07-06T10:46:18.318Z" }, + { url = "https://files.pythonhosted.org/packages/24/a0/94dc7ae310838f250669c6ad7168e6d6fca17d49dac1053f06dc232c4a56/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6", size = 309861, upload-time = "2026-07-06T10:46:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/adeead7d0eb28cdfc2832544ea639ffbc6749ccde47a8e228d667459182e/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398", size = 448739, upload-time = "2026-07-06T10:46:22.79Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/22ec0e07db57d901c9298ae98aa3cf2be45bafded6f07c13131e85b89032/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838", size = 223657, upload-time = "2026-07-06T10:46:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/94/32/8a9531f37b59e5a013003db7cb7414baf4ce7e0e1268e0d5947cd3d6a2df/xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22", size = 32377, upload-time = "2026-07-06T10:46:26.86Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ab/2ca45fd7f671de5f81fc297ef1c95080b40c86ec6be0cc6034b8f7707ac8/xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629", size = 33274, upload-time = "2026-07-06T10:46:28.39Z" }, + { url = "https://files.pythonhosted.org/packages/5a/54/20d7163463ddb6438b73a427d1655a77a502cf9b9b0c3ada3599629d9c0a/xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65", size = 29375, upload-time = "2026-07-06T10:46:29.962Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" }, + { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" }, + { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" }, + { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" }, + { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" }, + { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" }, + { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/4d8040435aeac814fc69ba63621565fbeb19229a138e2568324a26b2a45c/xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12", size = 32687, upload-time = "2026-07-06T10:49:42.803Z" }, + { url = "https://files.pythonhosted.org/packages/da/6a/975f1f2318c760e5bcec109ed379713ae645d8d856c2a3b9ec5d26857087/xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9", size = 29879, upload-time = "2026-07-06T10:49:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/08/0b/40a2a55ff52cf635bfdc5eae67a772bec85b4f44c6c737f73f6f528d51d1/xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd", size = 43246, upload-time = "2026-07-06T10:49:47.905Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/56ed2b6b200f26fb474f3fd387d95d0601efcd5bb33430c90c68924bdd77/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02", size = 38202, upload-time = "2026-07-06T10:49:50.59Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a3/56864d895d1161a9f17502088e9c1fb7c06bde2c2efdde620d22bb7a9c43/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20", size = 34448, upload-time = "2026-07-06T10:49:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/6b/57/5c6e0908a47f61dca96d01c8ee6fce01ed1050611eb779083ba8758fed81/xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42", size = 32869, upload-time = "2026-07-06T10:49:55.972Z" }, ] [[package]] name = "yarl" -version = "1.20.1" +version = "1.24.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, { name = "multidict" }, { name = "propcache" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/fb/efaa23fa4e45537b827620f04cf8f3cd658b76642205162e072703a5b963/yarl-1.20.1.tar.gz", hash = "sha256:d017a4997ee50c91fd5466cef416231bb82177b93b029906cefc542ce14c35ac", size = 186428, upload-time = "2025-06-10T00:46:09.923Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/65/7fed0d774abf47487c64be14e9223749468922817b5e8792b8a64792a1bb/yarl-1.20.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6032e6da6abd41e4acda34d75a816012717000fa6839f37124a47fcefc49bec4", size = 132910, upload-time = "2025-06-10T00:42:31.108Z" }, - { url = "https://files.pythonhosted.org/packages/8a/7b/988f55a52da99df9e56dc733b8e4e5a6ae2090081dc2754fc8fd34e60aa0/yarl-1.20.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2c7b34d804b8cf9b214f05015c4fee2ebe7ed05cf581e7192c06555c71f4446a", size = 90644, upload-time = "2025-06-10T00:42:33.851Z" }, - { url = "https://files.pythonhosted.org/packages/f7/de/30d98f03e95d30c7e3cc093759982d038c8833ec2451001d45ef4854edc1/yarl-1.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0c869f2651cc77465f6cd01d938d91a11d9ea5d798738c1dc077f3de0b5e5fed", size = 89322, upload-time = "2025-06-10T00:42:35.688Z" }, - { url = "https://files.pythonhosted.org/packages/e0/7a/f2f314f5ebfe9200724b0b748de2186b927acb334cf964fd312eb86fc286/yarl-1.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62915e6688eb4d180d93840cda4110995ad50c459bf931b8b3775b37c264af1e", size = 323786, upload-time = "2025-06-10T00:42:37.817Z" }, - { url = "https://files.pythonhosted.org/packages/15/3f/718d26f189db96d993d14b984ce91de52e76309d0fd1d4296f34039856aa/yarl-1.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:41ebd28167bc6af8abb97fec1a399f412eec5fd61a3ccbe2305a18b84fb4ca73", size = 319627, upload-time = "2025-06-10T00:42:39.937Z" }, - { url = "https://files.pythonhosted.org/packages/a5/76/8fcfbf5fa2369157b9898962a4a7d96764b287b085b5b3d9ffae69cdefd1/yarl-1.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:21242b4288a6d56f04ea193adde174b7e347ac46ce6bc84989ff7c1b1ecea84e", size = 339149, upload-time = "2025-06-10T00:42:42.627Z" }, - { url = "https://files.pythonhosted.org/packages/3c/95/d7fc301cc4661785967acc04f54a4a42d5124905e27db27bb578aac49b5c/yarl-1.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bea21cdae6c7eb02ba02a475f37463abfe0a01f5d7200121b03e605d6a0439f8", size = 333327, upload-time = "2025-06-10T00:42:44.842Z" }, - { url = "https://files.pythonhosted.org/packages/65/94/e21269718349582eee81efc5c1c08ee71c816bfc1585b77d0ec3f58089eb/yarl-1.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f8a891e4a22a89f5dde7862994485e19db246b70bb288d3ce73a34422e55b23", size = 326054, upload-time = "2025-06-10T00:42:47.149Z" }, - { url = "https://files.pythonhosted.org/packages/32/ae/8616d1f07853704523519f6131d21f092e567c5af93de7e3e94b38d7f065/yarl-1.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd803820d44c8853a109a34e3660e5a61beae12970da479cf44aa2954019bf70", size = 315035, upload-time = "2025-06-10T00:42:48.852Z" }, - { url = "https://files.pythonhosted.org/packages/48/aa/0ace06280861ef055855333707db5e49c6e3a08840a7ce62682259d0a6c0/yarl-1.20.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b982fa7f74c80d5c0c7b5b38f908971e513380a10fecea528091405f519b9ebb", size = 338962, upload-time = "2025-06-10T00:42:51.024Z" }, - { url = "https://files.pythonhosted.org/packages/20/52/1e9d0e6916f45a8fb50e6844f01cb34692455f1acd548606cbda8134cd1e/yarl-1.20.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:33f29ecfe0330c570d997bcf1afd304377f2e48f61447f37e846a6058a4d33b2", size = 335399, upload-time = "2025-06-10T00:42:53.007Z" }, - { url = "https://files.pythonhosted.org/packages/f2/65/60452df742952c630e82f394cd409de10610481d9043aa14c61bf846b7b1/yarl-1.20.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:835ab2cfc74d5eb4a6a528c57f05688099da41cf4957cf08cad38647e4a83b30", size = 338649, upload-time = "2025-06-10T00:42:54.964Z" }, - { url = "https://files.pythonhosted.org/packages/7b/f5/6cd4ff38dcde57a70f23719a838665ee17079640c77087404c3d34da6727/yarl-1.20.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:46b5e0ccf1943a9a6e766b2c2b8c732c55b34e28be57d8daa2b3c1d1d4009309", size = 358563, upload-time = "2025-06-10T00:42:57.28Z" }, - { url = "https://files.pythonhosted.org/packages/d1/90/c42eefd79d0d8222cb3227bdd51b640c0c1d0aa33fe4cc86c36eccba77d3/yarl-1.20.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:df47c55f7d74127d1b11251fe6397d84afdde0d53b90bedb46a23c0e534f9d24", size = 357609, upload-time = "2025-06-10T00:42:59.055Z" }, - { url = "https://files.pythonhosted.org/packages/03/c8/cea6b232cb4617514232e0f8a718153a95b5d82b5290711b201545825532/yarl-1.20.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:76d12524d05841276b0e22573f28d5fbcb67589836772ae9244d90dd7d66aa13", size = 350224, upload-time = "2025-06-10T00:43:01.248Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a3/eaa0ab9712f1f3d01faf43cf6f1f7210ce4ea4a7e9b28b489a2261ca8db9/yarl-1.20.1-cp310-cp310-win32.whl", hash = "sha256:6c4fbf6b02d70e512d7ade4b1f998f237137f1417ab07ec06358ea04f69134f8", size = 81753, upload-time = "2025-06-10T00:43:03.486Z" }, - { url = "https://files.pythonhosted.org/packages/8f/34/e4abde70a9256465fe31c88ed02c3f8502b7b5dead693a4f350a06413f28/yarl-1.20.1-cp310-cp310-win_amd64.whl", hash = "sha256:aef6c4d69554d44b7f9d923245f8ad9a707d971e6209d51279196d8e8fe1ae16", size = 86817, upload-time = "2025-06-10T00:43:05.231Z" }, - { url = "https://files.pythonhosted.org/packages/b1/18/893b50efc2350e47a874c5c2d67e55a0ea5df91186b2a6f5ac52eff887cd/yarl-1.20.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:47ee6188fea634bdfaeb2cc420f5b3b17332e6225ce88149a17c413c77ff269e", size = 133833, upload-time = "2025-06-10T00:43:07.393Z" }, - { url = "https://files.pythonhosted.org/packages/89/ed/b8773448030e6fc47fa797f099ab9eab151a43a25717f9ac043844ad5ea3/yarl-1.20.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d0f6500f69e8402d513e5eedb77a4e1818691e8f45e6b687147963514d84b44b", size = 91070, upload-time = "2025-06-10T00:43:09.538Z" }, - { url = "https://files.pythonhosted.org/packages/e3/e3/409bd17b1e42619bf69f60e4f031ce1ccb29bd7380117a55529e76933464/yarl-1.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a8900a42fcdaad568de58887c7b2f602962356908eedb7628eaf6021a6e435b", size = 89818, upload-time = "2025-06-10T00:43:11.575Z" }, - { url = "https://files.pythonhosted.org/packages/f8/77/64d8431a4d77c856eb2d82aa3de2ad6741365245a29b3a9543cd598ed8c5/yarl-1.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bad6d131fda8ef508b36be3ece16d0902e80b88ea7200f030a0f6c11d9e508d4", size = 347003, upload-time = "2025-06-10T00:43:14.088Z" }, - { url = "https://files.pythonhosted.org/packages/8d/d2/0c7e4def093dcef0bd9fa22d4d24b023788b0a33b8d0088b51aa51e21e99/yarl-1.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:df018d92fe22aaebb679a7f89fe0c0f368ec497e3dda6cb81a567610f04501f1", size = 336537, upload-time = "2025-06-10T00:43:16.431Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f3/fc514f4b2cf02cb59d10cbfe228691d25929ce8f72a38db07d3febc3f706/yarl-1.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f969afbb0a9b63c18d0feecf0db09d164b7a44a053e78a7d05f5df163e43833", size = 362358, upload-time = "2025-06-10T00:43:18.704Z" }, - { url = "https://files.pythonhosted.org/packages/ea/6d/a313ac8d8391381ff9006ac05f1d4331cee3b1efaa833a53d12253733255/yarl-1.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:812303eb4aa98e302886ccda58d6b099e3576b1b9276161469c25803a8db277d", size = 357362, upload-time = "2025-06-10T00:43:20.888Z" }, - { url = "https://files.pythonhosted.org/packages/00/70/8f78a95d6935a70263d46caa3dd18e1f223cf2f2ff2037baa01a22bc5b22/yarl-1.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98c4a7d166635147924aa0bf9bfe8d8abad6fffa6102de9c99ea04a1376f91e8", size = 348979, upload-time = "2025-06-10T00:43:23.169Z" }, - { url = "https://files.pythonhosted.org/packages/cb/05/42773027968968f4f15143553970ee36ead27038d627f457cc44bbbeecf3/yarl-1.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12e768f966538e81e6e7550f9086a6236b16e26cd964cf4df35349970f3551cf", size = 337274, upload-time = "2025-06-10T00:43:27.111Z" }, - { url = "https://files.pythonhosted.org/packages/05/be/665634aa196954156741ea591d2f946f1b78ceee8bb8f28488bf28c0dd62/yarl-1.20.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fe41919b9d899661c5c28a8b4b0acf704510b88f27f0934ac7a7bebdd8938d5e", size = 363294, upload-time = "2025-06-10T00:43:28.96Z" }, - { url = "https://files.pythonhosted.org/packages/eb/90/73448401d36fa4e210ece5579895731f190d5119c4b66b43b52182e88cd5/yarl-1.20.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:8601bc010d1d7780592f3fc1bdc6c72e2b6466ea34569778422943e1a1f3c389", size = 358169, upload-time = "2025-06-10T00:43:30.701Z" }, - { url = "https://files.pythonhosted.org/packages/c3/b0/fce922d46dc1eb43c811f1889f7daa6001b27a4005587e94878570300881/yarl-1.20.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:daadbdc1f2a9033a2399c42646fbd46da7992e868a5fe9513860122d7fe7a73f", size = 362776, upload-time = "2025-06-10T00:43:32.51Z" }, - { url = "https://files.pythonhosted.org/packages/f1/0d/b172628fce039dae8977fd22caeff3eeebffd52e86060413f5673767c427/yarl-1.20.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:03aa1e041727cb438ca762628109ef1333498b122e4c76dd858d186a37cec845", size = 381341, upload-time = "2025-06-10T00:43:34.543Z" }, - { url = "https://files.pythonhosted.org/packages/6b/9b/5b886d7671f4580209e855974fe1cecec409aa4a89ea58b8f0560dc529b1/yarl-1.20.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:642980ef5e0fa1de5fa96d905c7e00cb2c47cb468bfcac5a18c58e27dbf8d8d1", size = 379988, upload-time = "2025-06-10T00:43:36.489Z" }, - { url = "https://files.pythonhosted.org/packages/73/be/75ef5fd0fcd8f083a5d13f78fd3f009528132a1f2a1d7c925c39fa20aa79/yarl-1.20.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:86971e2795584fe8c002356d3b97ef6c61862720eeff03db2a7c86b678d85b3e", size = 371113, upload-time = "2025-06-10T00:43:38.592Z" }, - { url = "https://files.pythonhosted.org/packages/50/4f/62faab3b479dfdcb741fe9e3f0323e2a7d5cd1ab2edc73221d57ad4834b2/yarl-1.20.1-cp311-cp311-win32.whl", hash = "sha256:597f40615b8d25812f14562699e287f0dcc035d25eb74da72cae043bb884d773", size = 81485, upload-time = "2025-06-10T00:43:41.038Z" }, - { url = "https://files.pythonhosted.org/packages/f0/09/d9c7942f8f05c32ec72cd5c8e041c8b29b5807328b68b4801ff2511d4d5e/yarl-1.20.1-cp311-cp311-win_amd64.whl", hash = "sha256:26ef53a9e726e61e9cd1cda6b478f17e350fb5800b4bd1cd9fe81c4d91cfeb2e", size = 86686, upload-time = "2025-06-10T00:43:42.692Z" }, - { url = "https://files.pythonhosted.org/packages/5f/9a/cb7fad7d73c69f296eda6815e4a2c7ed53fc70c2f136479a91c8e5fbdb6d/yarl-1.20.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdcc4cd244e58593a4379fe60fdee5ac0331f8eb70320a24d591a3be197b94a9", size = 133667, upload-time = "2025-06-10T00:43:44.369Z" }, - { url = "https://files.pythonhosted.org/packages/67/38/688577a1cb1e656e3971fb66a3492501c5a5df56d99722e57c98249e5b8a/yarl-1.20.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b29a2c385a5f5b9c7d9347e5812b6f7ab267193c62d282a540b4fc528c8a9d2a", size = 91025, upload-time = "2025-06-10T00:43:46.295Z" }, - { url = "https://files.pythonhosted.org/packages/50/ec/72991ae51febeb11a42813fc259f0d4c8e0507f2b74b5514618d8b640365/yarl-1.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1112ae8154186dfe2de4732197f59c05a83dc814849a5ced892b708033f40dc2", size = 89709, upload-time = "2025-06-10T00:43:48.22Z" }, - { url = "https://files.pythonhosted.org/packages/99/da/4d798025490e89426e9f976702e5f9482005c548c579bdae792a4c37769e/yarl-1.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90bbd29c4fe234233f7fa2b9b121fb63c321830e5d05b45153a2ca68f7d310ee", size = 352287, upload-time = "2025-06-10T00:43:49.924Z" }, - { url = "https://files.pythonhosted.org/packages/1a/26/54a15c6a567aac1c61b18aa0f4b8aa2e285a52d547d1be8bf48abe2b3991/yarl-1.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:680e19c7ce3710ac4cd964e90dad99bf9b5029372ba0c7cbfcd55e54d90ea819", size = 345429, upload-time = "2025-06-10T00:43:51.7Z" }, - { url = "https://files.pythonhosted.org/packages/d6/95/9dcf2386cb875b234353b93ec43e40219e14900e046bf6ac118f94b1e353/yarl-1.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a979218c1fdb4246a05efc2cc23859d47c89af463a90b99b7c56094daf25a16", size = 365429, upload-time = "2025-06-10T00:43:53.494Z" }, - { url = "https://files.pythonhosted.org/packages/91/b2/33a8750f6a4bc224242a635f5f2cff6d6ad5ba651f6edcccf721992c21a0/yarl-1.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255b468adf57b4a7b65d8aad5b5138dce6a0752c139965711bdcb81bc370e1b6", size = 363862, upload-time = "2025-06-10T00:43:55.766Z" }, - { url = "https://files.pythonhosted.org/packages/98/28/3ab7acc5b51f4434b181b0cee8f1f4b77a65919700a355fb3617f9488874/yarl-1.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a97d67108e79cfe22e2b430d80d7571ae57d19f17cda8bb967057ca8a7bf5bfd", size = 355616, upload-time = "2025-06-10T00:43:58.056Z" }, - { url = "https://files.pythonhosted.org/packages/36/a3/f666894aa947a371724ec7cd2e5daa78ee8a777b21509b4252dd7bd15e29/yarl-1.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8570d998db4ddbfb9a590b185a0a33dbf8aafb831d07a5257b4ec9948df9cb0a", size = 339954, upload-time = "2025-06-10T00:43:59.773Z" }, - { url = "https://files.pythonhosted.org/packages/f1/81/5f466427e09773c04219d3450d7a1256138a010b6c9f0af2d48565e9ad13/yarl-1.20.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97c75596019baae7c71ccf1d8cc4738bc08134060d0adfcbe5642f778d1dca38", size = 365575, upload-time = "2025-06-10T00:44:02.051Z" }, - { url = "https://files.pythonhosted.org/packages/2e/e3/e4b0ad8403e97e6c9972dd587388940a032f030ebec196ab81a3b8e94d31/yarl-1.20.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1c48912653e63aef91ff988c5432832692ac5a1d8f0fb8a33091520b5bbe19ef", size = 365061, upload-time = "2025-06-10T00:44:04.196Z" }, - { url = "https://files.pythonhosted.org/packages/ac/99/b8a142e79eb86c926f9f06452eb13ecb1bb5713bd01dc0038faf5452e544/yarl-1.20.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4c3ae28f3ae1563c50f3d37f064ddb1511ecc1d5584e88c6b7c63cf7702a6d5f", size = 364142, upload-time = "2025-06-10T00:44:06.527Z" }, - { url = "https://files.pythonhosted.org/packages/34/f2/08ed34a4a506d82a1a3e5bab99ccd930a040f9b6449e9fd050320e45845c/yarl-1.20.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e9642f27036283550f5f57dc6156c51084b458570b9d0d96100c8bebb186a8", size = 381894, upload-time = "2025-06-10T00:44:08.379Z" }, - { url = "https://files.pythonhosted.org/packages/92/f8/9a3fbf0968eac704f681726eff595dce9b49c8a25cd92bf83df209668285/yarl-1.20.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:2c26b0c49220d5799f7b22c6838409ee9bc58ee5c95361a4d7831f03cc225b5a", size = 383378, upload-time = "2025-06-10T00:44:10.51Z" }, - { url = "https://files.pythonhosted.org/packages/af/85/9363f77bdfa1e4d690957cd39d192c4cacd1c58965df0470a4905253b54f/yarl-1.20.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:564ab3d517e3d01c408c67f2e5247aad4019dcf1969982aba3974b4093279004", size = 374069, upload-time = "2025-06-10T00:44:12.834Z" }, - { url = "https://files.pythonhosted.org/packages/35/99/9918c8739ba271dcd935400cff8b32e3cd319eaf02fcd023d5dcd487a7c8/yarl-1.20.1-cp312-cp312-win32.whl", hash = "sha256:daea0d313868da1cf2fac6b2d3a25c6e3a9e879483244be38c8e6a41f1d876a5", size = 81249, upload-time = "2025-06-10T00:44:14.731Z" }, - { url = "https://files.pythonhosted.org/packages/eb/83/5d9092950565481b413b31a23e75dd3418ff0a277d6e0abf3729d4d1ce25/yarl-1.20.1-cp312-cp312-win_amd64.whl", hash = "sha256:48ea7d7f9be0487339828a4de0360d7ce0efc06524a48e1810f945c45b813698", size = 86710, upload-time = "2025-06-10T00:44:16.716Z" }, - { url = "https://files.pythonhosted.org/packages/8a/e1/2411b6d7f769a07687acee88a062af5833cf1966b7266f3d8dfb3d3dc7d3/yarl-1.20.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:0b5ff0fbb7c9f1b1b5ab53330acbfc5247893069e7716840c8e7d5bb7355038a", size = 131811, upload-time = "2025-06-10T00:44:18.933Z" }, - { url = "https://files.pythonhosted.org/packages/b2/27/584394e1cb76fb771371770eccad35de400e7b434ce3142c2dd27392c968/yarl-1.20.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:14f326acd845c2b2e2eb38fb1346c94f7f3b01a4f5c788f8144f9b630bfff9a3", size = 90078, upload-time = "2025-06-10T00:44:20.635Z" }, - { url = "https://files.pythonhosted.org/packages/bf/9a/3246ae92d4049099f52d9b0fe3486e3b500e29b7ea872d0f152966fc209d/yarl-1.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f60e4ad5db23f0b96e49c018596707c3ae89f5d0bd97f0ad3684bcbad899f1e7", size = 88748, upload-time = "2025-06-10T00:44:22.34Z" }, - { url = "https://files.pythonhosted.org/packages/a3/25/35afe384e31115a1a801fbcf84012d7a066d89035befae7c5d4284df1e03/yarl-1.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49bdd1b8e00ce57e68ba51916e4bb04461746e794e7c4d4bbc42ba2f18297691", size = 349595, upload-time = "2025-06-10T00:44:24.314Z" }, - { url = "https://files.pythonhosted.org/packages/28/2d/8aca6cb2cabc8f12efcb82749b9cefecbccfc7b0384e56cd71058ccee433/yarl-1.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:66252d780b45189975abfed839616e8fd2dbacbdc262105ad7742c6ae58f3e31", size = 342616, upload-time = "2025-06-10T00:44:26.167Z" }, - { url = "https://files.pythonhosted.org/packages/0b/e9/1312633d16b31acf0098d30440ca855e3492d66623dafb8e25b03d00c3da/yarl-1.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:59174e7332f5d153d8f7452a102b103e2e74035ad085f404df2e40e663a22b28", size = 361324, upload-time = "2025-06-10T00:44:27.915Z" }, - { url = "https://files.pythonhosted.org/packages/bc/a0/688cc99463f12f7669eec7c8acc71ef56a1521b99eab7cd3abb75af887b0/yarl-1.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e3968ec7d92a0c0f9ac34d5ecfd03869ec0cab0697c91a45db3fbbd95fe1b653", size = 359676, upload-time = "2025-06-10T00:44:30.041Z" }, - { url = "https://files.pythonhosted.org/packages/af/44/46407d7f7a56e9a85a4c207724c9f2c545c060380718eea9088f222ba697/yarl-1.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1a4fbb50e14396ba3d375f68bfe02215d8e7bc3ec49da8341fe3157f59d2ff5", size = 352614, upload-time = "2025-06-10T00:44:32.171Z" }, - { url = "https://files.pythonhosted.org/packages/b1/91/31163295e82b8d5485d31d9cf7754d973d41915cadce070491778d9c9825/yarl-1.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11a62c839c3a8eac2410e951301309426f368388ff2f33799052787035793b02", size = 336766, upload-time = "2025-06-10T00:44:34.494Z" }, - { url = "https://files.pythonhosted.org/packages/b4/8e/c41a5bc482121f51c083c4c2bcd16b9e01e1cf8729e380273a952513a21f/yarl-1.20.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:041eaa14f73ff5a8986b4388ac6bb43a77f2ea09bf1913df7a35d4646db69e53", size = 364615, upload-time = "2025-06-10T00:44:36.856Z" }, - { url = "https://files.pythonhosted.org/packages/e3/5b/61a3b054238d33d70ea06ebba7e58597891b71c699e247df35cc984ab393/yarl-1.20.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:377fae2fef158e8fd9d60b4c8751387b8d1fb121d3d0b8e9b0be07d1b41e83dc", size = 360982, upload-time = "2025-06-10T00:44:39.141Z" }, - { url = "https://files.pythonhosted.org/packages/df/a3/6a72fb83f8d478cb201d14927bc8040af901811a88e0ff2da7842dd0ed19/yarl-1.20.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1c92f4390e407513f619d49319023664643d3339bd5e5a56a3bebe01bc67ec04", size = 369792, upload-time = "2025-06-10T00:44:40.934Z" }, - { url = "https://files.pythonhosted.org/packages/7c/af/4cc3c36dfc7c077f8dedb561eb21f69e1e9f2456b91b593882b0b18c19dc/yarl-1.20.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d25ddcf954df1754ab0f86bb696af765c5bfaba39b74095f27eececa049ef9a4", size = 382049, upload-time = "2025-06-10T00:44:42.854Z" }, - { url = "https://files.pythonhosted.org/packages/19/3a/e54e2c4752160115183a66dc9ee75a153f81f3ab2ba4bf79c3c53b33de34/yarl-1.20.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:909313577e9619dcff8c31a0ea2aa0a2a828341d92673015456b3ae492e7317b", size = 384774, upload-time = "2025-06-10T00:44:45.275Z" }, - { url = "https://files.pythonhosted.org/packages/9c/20/200ae86dabfca89060ec6447649f219b4cbd94531e425e50d57e5f5ac330/yarl-1.20.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:793fd0580cb9664548c6b83c63b43c477212c0260891ddf86809e1c06c8b08f1", size = 374252, upload-time = "2025-06-10T00:44:47.31Z" }, - { url = "https://files.pythonhosted.org/packages/83/75/11ee332f2f516b3d094e89448da73d557687f7d137d5a0f48c40ff211487/yarl-1.20.1-cp313-cp313-win32.whl", hash = "sha256:468f6e40285de5a5b3c44981ca3a319a4b208ccc07d526b20b12aeedcfa654b7", size = 81198, upload-time = "2025-06-10T00:44:49.164Z" }, - { url = "https://files.pythonhosted.org/packages/ba/ba/39b1ecbf51620b40ab402b0fc817f0ff750f6d92712b44689c2c215be89d/yarl-1.20.1-cp313-cp313-win_amd64.whl", hash = "sha256:495b4ef2fea40596bfc0affe3837411d6aa3371abcf31aac0ccc4bdd64d4ef5c", size = 86346, upload-time = "2025-06-10T00:44:51.182Z" }, - { url = "https://files.pythonhosted.org/packages/43/c7/669c52519dca4c95153c8ad96dd123c79f354a376346b198f438e56ffeb4/yarl-1.20.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:f60233b98423aab21d249a30eb27c389c14929f47be8430efa7dbd91493a729d", size = 138826, upload-time = "2025-06-10T00:44:52.883Z" }, - { url = "https://files.pythonhosted.org/packages/6a/42/fc0053719b44f6ad04a75d7f05e0e9674d45ef62f2d9ad2c1163e5c05827/yarl-1.20.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6f3eff4cc3f03d650d8755c6eefc844edde99d641d0dcf4da3ab27141a5f8ddf", size = 93217, upload-time = "2025-06-10T00:44:54.658Z" }, - { url = "https://files.pythonhosted.org/packages/4f/7f/fa59c4c27e2a076bba0d959386e26eba77eb52ea4a0aac48e3515c186b4c/yarl-1.20.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:69ff8439d8ba832d6bed88af2c2b3445977eba9a4588b787b32945871c2444e3", size = 92700, upload-time = "2025-06-10T00:44:56.784Z" }, - { url = "https://files.pythonhosted.org/packages/2f/d4/062b2f48e7c93481e88eff97a6312dca15ea200e959f23e96d8ab898c5b8/yarl-1.20.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cf34efa60eb81dd2645a2e13e00bb98b76c35ab5061a3989c7a70f78c85006d", size = 347644, upload-time = "2025-06-10T00:44:59.071Z" }, - { url = "https://files.pythonhosted.org/packages/89/47/78b7f40d13c8f62b499cc702fdf69e090455518ae544c00a3bf4afc9fc77/yarl-1.20.1-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8e0fe9364ad0fddab2688ce72cb7a8e61ea42eff3c7caeeb83874a5d479c896c", size = 323452, upload-time = "2025-06-10T00:45:01.605Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2b/490d3b2dc66f52987d4ee0d3090a147ea67732ce6b4d61e362c1846d0d32/yarl-1.20.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f64fbf81878ba914562c672024089e3401974a39767747691c65080a67b18c1", size = 346378, upload-time = "2025-06-10T00:45:03.946Z" }, - { url = "https://files.pythonhosted.org/packages/66/ad/775da9c8a94ce925d1537f939a4f17d782efef1f973039d821cbe4bcc211/yarl-1.20.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f6342d643bf9a1de97e512e45e4b9560a043347e779a173250824f8b254bd5ce", size = 353261, upload-time = "2025-06-10T00:45:05.992Z" }, - { url = "https://files.pythonhosted.org/packages/4b/23/0ed0922b47a4f5c6eb9065d5ff1e459747226ddce5c6a4c111e728c9f701/yarl-1.20.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56dac5f452ed25eef0f6e3c6a066c6ab68971d96a9fb441791cad0efba6140d3", size = 335987, upload-time = "2025-06-10T00:45:08.227Z" }, - { url = "https://files.pythonhosted.org/packages/3e/49/bc728a7fe7d0e9336e2b78f0958a2d6b288ba89f25a1762407a222bf53c3/yarl-1.20.1-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7d7f497126d65e2cad8dc5f97d34c27b19199b6414a40cb36b52f41b79014be", size = 329361, upload-time = "2025-06-10T00:45:10.11Z" }, - { url = "https://files.pythonhosted.org/packages/93/8f/b811b9d1f617c83c907e7082a76e2b92b655400e61730cd61a1f67178393/yarl-1.20.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:67e708dfb8e78d8a19169818eeb5c7a80717562de9051bf2413aca8e3696bf16", size = 346460, upload-time = "2025-06-10T00:45:12.055Z" }, - { url = "https://files.pythonhosted.org/packages/70/fd/af94f04f275f95da2c3b8b5e1d49e3e79f1ed8b6ceb0f1664cbd902773ff/yarl-1.20.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:595c07bc79af2494365cc96ddeb772f76272364ef7c80fb892ef9d0649586513", size = 334486, upload-time = "2025-06-10T00:45:13.995Z" }, - { url = "https://files.pythonhosted.org/packages/84/65/04c62e82704e7dd0a9b3f61dbaa8447f8507655fd16c51da0637b39b2910/yarl-1.20.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:7bdd2f80f4a7df852ab9ab49484a4dee8030023aa536df41f2d922fd57bf023f", size = 342219, upload-time = "2025-06-10T00:45:16.479Z" }, - { url = "https://files.pythonhosted.org/packages/91/95/459ca62eb958381b342d94ab9a4b6aec1ddec1f7057c487e926f03c06d30/yarl-1.20.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c03bfebc4ae8d862f853a9757199677ab74ec25424d0ebd68a0027e9c639a390", size = 350693, upload-time = "2025-06-10T00:45:18.399Z" }, - { url = "https://files.pythonhosted.org/packages/a6/00/d393e82dd955ad20617abc546a8f1aee40534d599ff555ea053d0ec9bf03/yarl-1.20.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:344d1103e9c1523f32a5ed704d576172d2cabed3122ea90b1d4e11fe17c66458", size = 355803, upload-time = "2025-06-10T00:45:20.677Z" }, - { url = "https://files.pythonhosted.org/packages/9e/ed/c5fb04869b99b717985e244fd93029c7a8e8febdfcffa06093e32d7d44e7/yarl-1.20.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:88cab98aa4e13e1ade8c141daeedd300a4603b7132819c484841bb7af3edce9e", size = 341709, upload-time = "2025-06-10T00:45:23.221Z" }, - { url = "https://files.pythonhosted.org/packages/24/fd/725b8e73ac2a50e78a4534ac43c6addf5c1c2d65380dd48a9169cc6739a9/yarl-1.20.1-cp313-cp313t-win32.whl", hash = "sha256:b121ff6a7cbd4abc28985b6028235491941b9fe8fe226e6fdc539c977ea1739d", size = 86591, upload-time = "2025-06-10T00:45:25.793Z" }, - { url = "https://files.pythonhosted.org/packages/94/c3/b2e9f38bc3e11191981d57ea08cab2166e74ea770024a646617c9cddd9f6/yarl-1.20.1-cp313-cp313t-win_amd64.whl", hash = "sha256:541d050a355bbbc27e55d906bc91cb6fe42f96c01413dd0f4ed5a5240513874f", size = 93003, upload-time = "2025-06-10T00:45:27.752Z" }, - { url = "https://files.pythonhosted.org/packages/b4/2d/2345fce04cfd4bee161bf1e7d9cdc702e3e16109021035dbb24db654a622/yarl-1.20.1-py3-none-any.whl", hash = "sha256:83b8eb083fe4683c6115795d9fc1cfaf2cbbefb19b3a1cb68f6527460f483a77", size = 46542, upload-time = "2025-06-10T00:46:07.521Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798, upload-time = "2026-05-19T21:31:05.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/df/f1c7a3de0831cd83194f1a85c5bb431b13f81e6b45079314c86d1c4ef3f2/yarl-1.24.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5249a113065c2b7a958bc699759e359cd61cfc81e3069662208f48f191b7ed12", size = 129057, upload-time = "2026-05-19T21:27:47.564Z" }, + { url = "https://files.pythonhosted.org/packages/48/41/7daafb32dd7562bf45b1ce56562e7e1a9146f6479b6456873eb8a3413c40/yarl-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7f4425fa244fbf530b006d0c5f79ce920114cfff5b4f5f6056e669f8e160fdc0", size = 91545, upload-time = "2026-05-19T21:27:50.089Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8f/7b3ec212f1ea0683f55f978e3246bc313c38818664edfc97a9f349a4901e/yarl-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:15c0b5e49d3c44e2a0b93e6a49476c5edad0a7686b92c395765a7ea775572a75", size = 91380, upload-time = "2026-05-19T21:27:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1b/8bafab7db23b0567ae9db749099b329d91e3b82bc6028b2050ba583e116c/yarl-1.24.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:246d32a53a947c8f0189f5d699cbd4c7036de45d9359e13ba238d1239678c727", size = 105957, upload-time = "2026-05-19T21:27:53.98Z" }, + { url = "https://files.pythonhosted.org/packages/7f/77/21030c2f8d21d21559719beafc772ada2014be933418ed1eaed9cc800e42/yarl-1.24.2-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:64480fb3e4d4ed9ed71c48a91a477384fc342a50ca30071d2f8a88d51d9c9413", size = 97242, upload-time = "2026-05-19T21:27:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/50/d8/f9ea63d1b6aa910a866e089d871fff6cbd49caab29b86b35221a62dfa0d5/yarl-1.24.2-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:349de4701dc3760b6e876628423a8f147ef4f5599d10aba1e10702075d424ed9", size = 114719, upload-time = "2026-05-19T21:27:58.037Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/04e0ee98ac58a249ea7ed75223f5f901ba81a834f0b4921b58e5cec11757/yarl-1.24.2-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d162677af8d5d3d6ebab8394b021f4d041ac107a4b705873148a77a49dc9e1b2", size = 112140, upload-time = "2026-05-19T21:27:59.618Z" }, + { url = "https://files.pythonhosted.org/packages/02/ad/0b9cc9f38a7324a7eb1d80f834eaa5283d17e9271bbda3186e598dddaeac/yarl-1.24.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f5f5c6ec23a9043f2d139cc072f53dd23168d202a334b9b2fda8de4c3e890d90", size = 106721, upload-time = "2026-05-19T21:28:02.586Z" }, + { url = "https://files.pythonhosted.org/packages/65/e7/a52478ebfc66ec989e085c6ae038b9f1bfa4190baa193b133b669c709e2f/yarl-1.24.2-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:60de6742447fbbf697f16f070b8a443f1b5fe6ca3826fbef9fe70ecd5328e643", size = 106478, upload-time = "2026-05-19T21:28:04.523Z" }, + { url = "https://files.pythonhosted.org/packages/04/d8/5508530fea8472542de00013ae280765fc938ee196fc4030c43a498afb36/yarl-1.24.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:acf93187c3710e422368eb768aee98db551ec7c85adc250207a95c16548ab7ac", size = 105423, upload-time = "2026-05-19T21:28:06.515Z" }, + { url = "https://files.pythonhosted.org/packages/84/f1/ece28505e9628e8b756e11bb4f28864a17cc33b6b44db4d2aaf0622bf630/yarl-1.24.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:f4b0352fd41fd34b6651934606268816afd6914d09626f9bcbbf018edb0afb3f", size = 99878, upload-time = "2026-05-19T21:28:08.637Z" }, + { url = "https://files.pythonhosted.org/packages/3f/52/fb5d34529b46dd84013afcfb30b8d2bc2832ed03d412736f577d604fa393/yarl-1.24.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:6b208bb939099b4b297438da4e9b25357f0b1c791888669b963e45b203ea9f36", size = 114025, upload-time = "2026-05-19T21:28:10.64Z" }, + { url = "https://files.pythonhosted.org/packages/43/f0/ff9d31aaab024f7a251c0ed308a98ae29bf9f7dc344e78f28b1322431ca2/yarl-1.24.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4b85b8825e631295ff4bc8943f7471d54c533a9360bbe15ebb38e018b555bb8a", size = 105613, upload-time = "2026-05-19T21:28:12.784Z" }, + { url = "https://files.pythonhosted.org/packages/31/7d/3296fb3f3ecd52bf9ae6c16b0895c1cda7e9170a2083861552b683f70264/yarl-1.24.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e26acf20c26cb4fefc631fdb75aca2a6b8fa8b7b5d7f204fb6a8f1e63c706f53", size = 111665, upload-time = "2026-05-19T21:28:14.393Z" }, + { url = "https://files.pythonhosted.org/packages/1a/74/77aa6ddaca4fbf42e45e675a465c43956dd40702281049975a2aa04eae59/yarl-1.24.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:819ca24f8eafcfb683c1bd5f44f2f488cea1274eb8944731ffd2e1f10f619342", size = 106914, upload-time = "2026-05-19T21:28:15.893Z" }, + { url = "https://files.pythonhosted.org/packages/d8/02/7611f22cd1d4ed7373eb7f9ee21fde1046edba2e7c0e514880d760352f48/yarl-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:5cb0f995a901c36be096ccbf4c673591c2faabbe96279598ffaec8c030f85bf4", size = 92658, upload-time = "2026-05-19T21:28:17.471Z" }, + { url = "https://files.pythonhosted.org/packages/91/00/671d0add79938127292839ae44506ce2f7fe8909c72d5a931864f128fd0b/yarl-1.24.2-cp310-cp310-win_arm64.whl", hash = "sha256:f408eace7e22a68b467a0562e0d27d322f91fe3eaaa6f466b962c6cfaea9fa39", size = 87887, upload-time = "2026-05-19T21:28:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c5/1ce244152ff2839645e7cae92f90e7bafcb2c52bea7ff586ac714f14f5df/yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1", size = 128971, upload-time = "2026-05-19T21:28:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/87/5a/00f36967203ed89cb3acd2c8ed526cc3fed9418eb70ce128160a911c8499/yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c", size = 91507, upload-time = "2026-05-19T21:28:22.556Z" }, + { url = "https://files.pythonhosted.org/packages/31/d0/1fb0c1cd27288f39f6974da4318c32768d72c9890984541fdf1e2e32a51d/yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d", size = 91343, upload-time = "2026-05-19T21:28:24.092Z" }, + { url = "https://files.pythonhosted.org/packages/03/ce/d4a646508bed2f8dec6435b40166fe9308dd191262033d3f307b2bbcaecd/yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae", size = 105704, upload-time = "2026-05-19T21:28:25.872Z" }, + { url = "https://files.pythonhosted.org/packages/4b/07/b3278e82d8bc41485bcf6d856cd0433262593de615b1d3dc43bd3f5bead4/yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a", size = 97281, upload-time = "2026-05-19T21:28:27.352Z" }, + { url = "https://files.pythonhosted.org/packages/17/5b/4cee6e7c92e487bebe7afc797da0aa54a248ab4e776a68fe369ec29665a5/yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e", size = 114020, upload-time = "2026-05-19T21:28:29.458Z" }, + { url = "https://files.pythonhosted.org/packages/5c/82/111076571545a7d4f9cca3fbd5c6f40615af58642be09f12328f48022468/yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50", size = 111450, upload-time = "2026-05-19T21:28:31.262Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ec/08f671f69a444d704aeecebf92af659b67b97a869942411d0a578b08c334/yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003", size = 106384, upload-time = "2026-05-19T21:28:32.856Z" }, + { url = "https://files.pythonhosted.org/packages/e5/86/ce41e7a7a199340b2330d52b60f25c4074b6636dd0e60b1a80d31a9db042/yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f", size = 106153, upload-time = "2026-05-19T21:28:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5d/31be8a729531ab3e55ac3e7e5c800be8c89ea98947f418b2f6ea259fb6ee/yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f", size = 105322, upload-time = "2026-05-19T21:28:36.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/9b/b57afb22b386ae87ac9940f09878b98d8c333f89113e6fc96fcf4ca9eb64/yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294", size = 99057, upload-time = "2026-05-19T21:28:38.386Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4f/06348c27c8389256c313e8a57d796808fc0264c915dd5e7cfd3c0e314dc7/yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2", size = 113502, upload-time = "2026-05-19T21:28:40.091Z" }, + { url = "https://files.pythonhosted.org/packages/5f/1c/284f307b298e4a17b7943b07d9d7ecc4151537f8d137ba51f3bb6c31ca20/yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c", size = 105253, upload-time = "2026-05-19T21:28:41.987Z" }, + { url = "https://files.pythonhosted.org/packages/c8/bf/0de123bec8619e45c80cbded9085f61b5b4a9eddb8abe6d25d28ee1ec866/yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b", size = 111345, upload-time = "2026-05-19T21:28:43.93Z" }, + { url = "https://files.pythonhosted.org/packages/90/af/0248eb065e51129d2a9b2436cd1b5c772c19a6b04e5b6a186955671e3319/yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5", size = 106558, upload-time = "2026-05-19T21:28:45.806Z" }, + { url = "https://files.pythonhosted.org/packages/21/3c/f960d7a65ef97d8ba9b424fb5128796a4bc710fc6df2ddbbd7dfdc3bbd20/yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45", size = 92808, upload-time = "2026-05-19T21:28:48.465Z" }, + { url = "https://files.pythonhosted.org/packages/03/1a/49fb03750e4de4d2284cd5b885a383133c34eef45bd59631b2bb8b7e81e8/yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122", size = 87610, upload-time = "2026-05-19T21:28:50.07Z" }, + { url = "https://files.pythonhosted.org/packages/f0/da/866bcb01076ba49d2b42b309867bed3826421f1c479655eb7a607b44f20b/yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8", size = 129957, upload-time = "2026-05-19T21:28:51.695Z" }, + { url = "https://files.pythonhosted.org/packages/bf/1d/fcefb70922ea2268a8971d8e5874d9a8218644200fb8465f1dcad55e6851/yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2", size = 92164, upload-time = "2026-05-19T21:28:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/29/b6/170e2b8d4e3bc30e6bfdcca53556537f5bf595e938632dfcb059311f3ff6/yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d", size = 91688, upload-time = "2026-05-19T21:28:54.865Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c9f655d5553ea0b99fdac9d6a99ad3f9b3e73b8e5758bb46f58c9831f74c/yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035", size = 102902, upload-time = "2026-05-19T21:28:56.963Z" }, + { url = "https://files.pythonhosted.org/packages/5d/bc/6b9664d815d79af4ee553337f9d606c56bbf269186ada9172de45f1b5f60/yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576", size = 97931, upload-time = "2026-05-19T21:28:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/98/ec/32ba48acae30fecd60928f5791188b80a9d6ee3840507ffda29fecd37b71/yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8", size = 111030, upload-time = "2026-05-19T21:29:00.148Z" }, + { url = "https://files.pythonhosted.org/packages/82/5a/6f4cd081e5f4934d2ae3a8ef4abe3afacc010d26f0035ee91b35cd7d7c37/yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7", size = 110392, upload-time = "2026-05-19T21:29:02.155Z" }, + { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612, upload-time = "2026-05-19T21:29:04.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/80/264ab684f181e1a876389374519ff05d10248725535ae2ac4e8ac4e563d6/yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d", size = 104487, upload-time = "2026-05-19T21:29:06.491Z" }, + { url = "https://files.pythonhosted.org/packages/41/07/efabe5df87e96d7ad5959760b888344be48cd6884db127b407c6b5503adc/yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db", size = 102333, upload-time = "2026-05-19T21:29:08.267Z" }, + { url = "https://files.pythonhosted.org/packages/44/0c/bcf7c42603e1009295f586d8890f2ba032c8b53310e815adf0a202c73d9f/yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712", size = 99025, upload-time = "2026-05-19T21:29:10.682Z" }, + { url = "https://files.pythonhosted.org/packages/4f/82/84482ab1a57a0f21a08afe6a7004c61d741f8f2ecc3b05c321577c612164/yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996", size = 110507, upload-time = "2026-05-19T21:29:12.954Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8d/a546ba1dfe1b0f290e05fef145cd07614c0f15df1a707195e512d1e39d1d/yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b", size = 103719, upload-time = "2026-05-19T21:29:14.893Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b6/267f2a09213138473adfce6b8a6e17791d7fee70bd4d9003218e4dec58b0/yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c", size = 110438, upload-time = "2026-05-19T21:29:16.485Z" }, + { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719, upload-time = "2026-05-19T21:29:18.367Z" }, + { url = "https://files.pythonhosted.org/packages/a7/25/722e3b93bd687009afb2d59a35e13d30ddd8f80571445bb0c4e4ce26ec66/yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad", size = 92901, upload-time = "2026-05-19T21:29:20.014Z" }, + { url = "https://files.pythonhosted.org/packages/39/47/4486ccfb674c04854a1ef8aa77868b6a6f765feaf69633409d7ca4f02cb8/yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30", size = 87229, upload-time = "2026-05-19T21:29:22.1Z" }, + { url = "https://files.pythonhosted.org/packages/82/62/fcf0ce677f17e5c471c06311dd25964be38a4c586993632910d2e75278bc/yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536", size = 128978, upload-time = "2026-05-19T21:29:23.83Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/8e63299bb71ed61a834121d9d3fe6c9fcf2a6a5d09754ff4f20f2d20baf5/yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607", size = 91733, upload-time = "2026-05-19T21:29:25.375Z" }, + { url = "https://files.pythonhosted.org/packages/c1/24/16748d5dab6daec8b0ed81ccec639a1cded0f18dcc62a4f696b4fe366c37/yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1", size = 91113, upload-time = "2026-05-19T21:29:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/1b/66/b63fff7b71211e866624b21432d5943cbb633eb0c2872d9ee3070648f22c/yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986", size = 103899, upload-time = "2026-05-19T21:29:28.842Z" }, + { url = "https://files.pythonhosted.org/packages/9d/ac/ba1974b8533909636f7733fe86cf677e3619527c3c2fa913e0ea89c48757/yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488", size = 97862, upload-time = "2026-05-19T21:29:31.086Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/123ac993b5c2ba6f554a140305620cb8f150fa543711bbc49be3ec0a65a4/yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b", size = 111060, upload-time = "2026-05-19T21:29:32.657Z" }, + { url = "https://files.pythonhosted.org/packages/23/37/c472d3af3509688392134a88a825276770a187f1daa4de3f6dc0a327a751/yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592", size = 110613, upload-time = "2026-05-19T21:29:34.379Z" }, + { url = "https://files.pythonhosted.org/packages/df/88/09c28dad91e662ccfaa1b78f1c57badde74fc9d0b23e74aef644750ecd73/yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617", size = 107012, upload-time = "2026-05-19T21:29:36.216Z" }, + { url = "https://files.pythonhosted.org/packages/07/ab/9d4f69d571a94f4d112fa7e2e007200f5a54d319f58c82ac7b7baa61f5c6/yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92", size = 105887, upload-time = "2026-05-19T21:29:38.746Z" }, + { url = "https://files.pythonhosted.org/packages/8e/9a/000b2b66c0d772a499fc531d21dab92dfeb73b640a12eed6ba89f49bb2d0/yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a", size = 103620, upload-time = "2026-05-19T21:29:40.368Z" }, + { url = "https://files.pythonhosted.org/packages/41/7c/7c1050f73450fbdaa3f0c72017059f00ce5e13366692f3dba25275a1083d/yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44", size = 100599, upload-time = "2026-05-19T21:29:42.66Z" }, + { url = "https://files.pythonhosted.org/packages/ec/b1/29e5756b3926705f5f6089bd5b9f50a56eaac550da6e260bf713ead44d04/yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a", size = 110604, upload-time = "2026-05-19T21:29:44.632Z" }, + { url = "https://files.pythonhosted.org/packages/a3/4b/8415bc96e9b150cde942fbac9a8182985e58f40ce5c54c34ed015407d3ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf", size = 105161, upload-time = "2026-05-19T21:29:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d4/cde059abfa229553b7298a2eadde2752e723d50aeedaef86ce59da2718ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056", size = 110619, upload-time = "2026-05-19T21:29:48.972Z" }, + { url = "https://files.pythonhosted.org/packages/e7/2c/d6a6c9a61549f7b6c7e6dc6937d195bcf069582b47b7200dcd0e7b256acf/yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992", size = 107362, upload-time = "2026-05-19T21:29:51Z" }, + { url = "https://files.pythonhosted.org/packages/92/dd/3ae5fe417e9d1c353a548553326eb9935e76b6b727161563b424cc296df3/yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656", size = 92667, upload-time = "2026-05-19T21:29:52.743Z" }, + { url = "https://files.pythonhosted.org/packages/10/cc/a7beb239f78f27fca1b053c8e8595e4179c02e62249b4687ec218c370c50/yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461", size = 87069, upload-time = "2026-05-19T21:29:54.442Z" }, + { url = "https://files.pythonhosted.org/packages/40/0e/e08087695fc12789263821c5dc0f8dc52b5b17efd0887cacf419f8a43ba3/yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2", size = 129670, upload-time = "2026-05-19T21:29:56.631Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/ab4b5ed1b1b5cd973c8a3eb994c3a6aefb6ce6d399e21bb5f0316c33815c/yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630", size = 91916, upload-time = "2026-05-19T21:29:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b1/5297bb6a7df4782f7605bffc43b31f5044070935fbbcaa6c705a07e6ac65/yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8", size = 91625, upload-time = "2026-05-19T21:30:00.412Z" }, + { url = "https://files.pythonhosted.org/packages/02/a7/45baabfff76829264e623b185cff0c340d7e11bf3e1cd9ea37e7d17934bd/yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14", size = 104574, upload-time = "2026-05-19T21:30:02.544Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/3a5ab144d3d650ca37d4f4b57e56169be8af3ca34c448793e064b30baaed/yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535", size = 97534, upload-time = "2026-05-19T21:30:04.319Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b5/5658fef3681fb5776b4513b052bec750009f47b3a592251c705d75375798/yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14", size = 111481, upload-time = "2026-05-19T21:30:05.988Z" }, + { url = "https://files.pythonhosted.org/packages/4c/06/fdcd7dde037f00866dce123ed4ba23dba94beb56fc4cf561668d27be37f2/yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3", size = 111529, upload-time = "2026-05-19T21:30:07.738Z" }, + { url = "https://files.pythonhosted.org/packages/c2/53/d81269aaafccea0d33396c03035de997b743f11e648e6e27a0df99c72980/yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208", size = 107338, upload-time = "2026-05-19T21:30:09.713Z" }, + { url = "https://files.pythonhosted.org/packages/ae/04/23049463f729bd899df203a7960505a75333edd499cda8aa1d5a82b64df5/yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50", size = 106147, upload-time = "2026-05-19T21:30:11.365Z" }, + { url = "https://files.pythonhosted.org/packages/14/18/04a4b5830b43ed5e4c5015b40e9f6241ad91487d71611061b4e111d6ac80/yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd", size = 104272, upload-time = "2026-05-19T21:30:12.978Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/8cffdf319aee7a7c1dbd07b61d91c3e3fda460c7a93b5f93e445f3806c4c/yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67", size = 99962, upload-time = "2026-05-19T21:30:15.001Z" }, + { url = "https://files.pythonhosted.org/packages/d7/39/b3cce3b7dbef64ac700ad4cea156a207d01bede0f507587616c364b5468e/yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1", size = 111063, upload-time = "2026-05-19T21:30:16.683Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ea/100818505e7ebf165c7242ff17fdf7d9fee79e27234aeca871c1082920d7/yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1", size = 105438, upload-time = "2026-05-19T21:30:18.769Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d2/e075a0b32aa6625087de9e653087df0759fed5de4a435fef594181102a77/yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b", size = 111458, upload-time = "2026-05-19T21:30:21.024Z" }, + { url = "https://files.pythonhosted.org/packages/e6/5c/ceea7ba98b65c8eb8d947fdc52f9bedfcd43c6a57c9e3c90c17be8f324a3/yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8", size = 107589, upload-time = "2026-05-19T21:30:23.412Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d9/5582d57e2b2db9b85eb6663a22efdd78e08805f3f5389566e9fcad254d1b/yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0", size = 94424, upload-time = "2026-05-19T21:30:25.425Z" }, + { url = "https://files.pythonhosted.org/packages/92/10/7dc07a0e22806a9280f42a57361395506e800c64e22737cd7b0886feab42/yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57", size = 88690, upload-time = "2026-05-19T21:30:27.623Z" }, + { url = "https://files.pythonhosted.org/packages/9e/13/d5b8e2c8667db955bcb3de233f18798fefe7edf1d7429c2c9d4f9c401114/yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b", size = 136248, upload-time = "2026-05-19T21:30:29.297Z" }, + { url = "https://files.pythonhosted.org/packages/de/46/a4a97c05c9c9b8fd266bb2a0df12992c7fbd02391eb9640583411b6dab32/yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761", size = 95084, upload-time = "2026-05-19T21:30:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/95/b2/845cf2074a015e6fe0d0808cf1a2d9e868386c4220d657ebd8302b199043/yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8", size = 95272, upload-time = "2026-05-19T21:30:33.062Z" }, + { url = "https://files.pythonhosted.org/packages/fe/16/e69d4aa244aef45235ddfebc0e04036a6829842bc5a6a795aedc6c998d23/yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed", size = 101497, upload-time = "2026-05-19T21:30:34.842Z" }, + { url = "https://files.pythonhosted.org/packages/15/94/c07107715d621076863ee88b3ddf183fa5e9d4aba5769623c9979828410a/yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543", size = 94002, upload-time = "2026-05-19T21:30:37.724Z" }, + { url = "https://files.pythonhosted.org/packages/a9/35/fc1bbdd895b5e4010b8fdd037f7ed3aa289d3863e08231b30231ca9a0815/yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0", size = 106524, upload-time = "2026-05-19T21:30:40.196Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f2/32b66d0a4ba47c296cf86d03e2c67bff58399fe6d6d84d5205c04c66cc6d/yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024", size = 106165, upload-time = "2026-05-19T21:30:41.888Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/37cb5ff50c5e825d4d38e81bb04d1b7e96bf960f7ab89f9850b162f3f114/yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf", size = 103010, upload-time = "2026-05-19T21:30:43.985Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d2/4597912315096f7bb359e46e13bf8b60994fcbb2db29b804c0902ef4eff5/yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc", size = 101128, upload-time = "2026-05-19T21:30:46.291Z" }, + { url = "https://files.pythonhosted.org/packages/b9/d5/c8e86e120521e646013d02a8e3b8884392e28494be8f392366e50d208efc/yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb", size = 101382, upload-time = "2026-05-19T21:30:48.085Z" }, + { url = "https://files.pythonhosted.org/packages/fa/98/70b229236118f89dbeb739b76f10225bbf53b5497725502594c9a01d699a/yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420", size = 95964, upload-time = "2026-05-19T21:30:49.785Z" }, + { url = "https://files.pythonhosted.org/packages/87/f8/56c386981e3c8648d279fdef2397ffec577e8320fd5649745e34d54faeb7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f", size = 106204, upload-time = "2026-05-19T21:30:51.862Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1e/765afe97811ca35933e2a7de70ac57b1997ea2e4ee895719ee7a231fb7e5/yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa", size = 101510, upload-time = "2026-05-19T21:30:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/ee/78/393913f4b9039e1edd09ae8a9bbb9d539be909a8abf6d8a2084585bed4b7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe", size = 105584, upload-time = "2026-05-19T21:30:55.962Z" }, + { url = "https://files.pythonhosted.org/packages/78/87/deb17b7049bbe74ea11a713b86f8f27800cc1c8648b0b797243ebb4830ba/yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd", size = 103410, upload-time = "2026-05-19T21:30:57.962Z" }, + { url = "https://files.pythonhosted.org/packages/8f/be/f9f7594e23b5b93affff0318e4593c1920331bcaefda326cabcad94296a1/yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215", size = 102980, upload-time = "2026-05-19T21:30:59.735Z" }, + { url = "https://files.pythonhosted.org/packages/65/a4/ba80dccd3593ff1f01051a818694d07b58cb8232677ee9a22a5a1f93a9fc/yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d", size = 91219, upload-time = "2026-05-19T21:31:01.934Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576, upload-time = "2026-05-19T21:31:03.909Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/d8/eab98a517c14134c0b2eb4e2387bc5f457334293ec5d2dd3857ec2966802/zipp-4.1.0.tar.gz", hash = "sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602", size = 26214, upload-time = "2026-05-18T20:08:57.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/13/547360d81e6d88d58492968ffda9f9542854f11310ee556fef14260cc886/zipp-4.1.0-py3-none-any.whl", hash = "sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f", size = 10238, upload-time = "2026-05-18T20:08:57.045Z" }, ] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5699bd4d536..c4058187023 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ # We keep this pinned to keep clippy and rustfmt in sync between local and CI. # Feel free to upgrade to bring in new lints. [toolchain] -channel = "1.94.0" +channel = "1.97.0" components = ["rustfmt", "clippy", "rust-analyzer"] diff --git a/rust/AGENTS.md b/rust/AGENTS.md index 6b2729c6692..70a803c6c76 100644 --- a/rust/AGENTS.md +++ b/rust/AGENTS.md @@ -17,6 +17,10 @@ Also see [root AGENTS.md](../AGENTS.md) for cross-language standards. - Delete obsolete internal (`pub(crate)` / private) methods in the same PR that introduces their replacements. For public API methods, follow the deprecation path in root AGENTS.md instead. - Choose log levels by audience: `debug!` for routine/high-frequency ops, `info!` for infrequent operator-visible state changes, `warn!` for unexpected conditions. +## Concurrency + +- The closure passed to `spawn_cpu()` must only consume CPU and return — it must **never** wait on anything: **no channels** (blocking send/recv), **no I/O**, **no locks**, and no `block_on`/`.blocking_*`. The CPU pool can collapse to a single worker in resource-constrained environments (`<= 3` CPUs), so a parked closure can deadlock the whole pool with a silent 0% hang. Keep the waiting in surrounding async code and hand only the pure-CPU work to `spawn_cpu()`. Only dispatch substantial work (rule of thumb: ~100µs+ of CPU); below that the pool overhead outweighs the benefit and the work is better left inline. See the doc comment on `spawn_cpu` for the rationale. + ## API Design - Use `with_`-prefixed builder methods for optional config (e.g., `MyStruct::new(required).with_option(v)`) — don't create separate constructor variants. diff --git a/rust/CLAUDE.md b/rust/CLAUDE.md new file mode 120000 index 00000000000..47dc3e3d863 --- /dev/null +++ b/rust/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/rust/arrow-stats/src/lib.rs b/rust/arrow-stats/src/lib.rs index 5c00a015749..3c4cec35bbe 100644 --- a/rust/arrow-stats/src/lib.rs +++ b/rust/arrow-stats/src/lib.rs @@ -445,6 +445,10 @@ fn find_min_max_indices(array: &ArrayRef) -> Result<(Option, Option find_extrema_float!(array, Float32Type), Float64 => find_extrema_float!(array, Float64Type), + // Decimal types + Decimal128(_, _) => find_extrema_primitive!(array, Decimal128Type), + Decimal256(_, _) => find_extrema_primitive!(array, Decimal256Type), + // Temporal types Date32 => find_extrema_primitive!(array, Date32Type), Date64 => find_extrema_primitive!(array, Date64Type), @@ -734,6 +738,26 @@ mod tests { Arc::new(Float64Array::from(vec![3.0f64, 1.0, 2.0])) as ArrayRef, "1.0", "3.0" )] + #[case::decimal128( + DataType::Decimal128(10, 2), + Arc::new( + Decimal128Array::from(vec![300_i128, 100, 200]) + .with_precision_and_scale(10, 2) + .unwrap(), + ) as ArrayRef, + "1.00", "3.00" + )] + #[case::decimal256( + DataType::Decimal256(76, 2), + Arc::new( + Decimal256Array::from_iter_values( + [300_i64, 100, 200].into_iter().map(Into::into), + ) + .with_precision_and_scale(76, 2) + .unwrap(), + ) as ArrayRef, + "1.00", "3.00" + )] fn test_rstest_primitives( #[case] dt: DataType, #[case] array: ArrayRef, diff --git a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs index 188a1f4ce2a..b17edacabb2 100644 --- a/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs +++ b/rust/compression/bitpacking/src/bitpacker_internal/bitpacker8x.rs @@ -420,12 +420,11 @@ enum InstructionSet { Scalar, } -/// Internal 8-wide bitpacker implementation. +/// 8-wide bitpacker implementation. /// -/// One block contains 256 integers. This stays private to avoid exposing a new -/// block-size choice through the public Lance bitpacking API. +/// One block contains 256 integers. #[derive(Clone, Copy)] -pub(crate) struct BitPacker8x(InstructionSet); +pub struct BitPacker8x(InstructionSet); impl BitPacker8x { #[cfg(target_arch = "x86_64")] diff --git a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs index c287a29da0b..80803e50ec8 100644 --- a/rust/compression/bitpacking/src/bitpacker_internal/mod.rs +++ b/rust/compression/bitpacking/src/bitpacker_internal/mod.rs @@ -20,6 +20,7 @@ mod bitpacker4x; mod bitpacker8x; pub use bitpacker4x::BitPacker4x; +pub use bitpacker8x::BitPacker8x; pub(crate) trait Available { fn available() -> bool; diff --git a/rust/compression/bitpacking/src/lib.rs b/rust/compression/bitpacking/src/lib.rs index f0e25e37e8c..4c2458fb153 100644 --- a/rust/compression/bitpacking/src/lib.rs +++ b/rust/compression/bitpacking/src/lib.rs @@ -14,11 +14,11 @@ // https://github.com/spiraldb/fastlanes/blob/8e0ff374f815d919d0c0ebdccf5ffd9e6dc7d663/LICENSE use arrayref::{array_mut_ref, array_ref}; -use core::mem::size_of; +use core::mem::{MaybeUninit, size_of}; mod bitpacker_internal; -pub use bitpacker_internal::{BitPacker, BitPacker4x}; +pub use bitpacker_internal::{BitPacker, BitPacker4x, BitPacker8x}; pub const FL_ORDER: [usize; 8] = [0, 4, 2, 6, 1, 5, 3, 7]; @@ -55,7 +55,7 @@ macro_rules! pack { // Special case for W=T, we can just copy the input value directly to the packed value. paste!(seq_t!(row in $T { let idx = index(row, $lane); - $packed[<$T>::LANES * row + $lane] = __kernel__!(idx); + $packed[<$T>::LANES * row + $lane].write(__kernel__!(idx)); })); } else { // A mask of W bits. @@ -86,7 +86,7 @@ macro_rules! pack { #[allow(unused_assignments)] if next_word > curr_word { - $packed[<$T>::LANES * curr_word + $lane] = tmp; + $packed[<$T>::LANES * curr_word + $lane].write(tmp); let remaining_bits: usize = ((row + 1) * $W) % T; // Keep the remaining bits for the next packed value. tmp = src >> $W - remaining_bits; @@ -200,8 +200,66 @@ pub trait BitPacking: FastLanes { unsafe fn unchecked_unpack(width: usize, input: &[Self], output: &mut [Self]); } -impl BitPacking for u8 { - unsafe fn unchecked_pack(width: usize, input: &[Self], output: &mut [Self]) { +/// Bitpacking kernels that can initialize previously uninitialized output storage. +pub trait BitPackingUninit: BitPacking { + /// Packs into potentially uninitialized output storage. + /// + /// # Safety + /// The input and output lengths have the same requirements as + /// [`BitPacking::unchecked_pack`]. Every output element is initialized on return. + unsafe fn unchecked_pack_uninit(width: usize, input: &[Self], output: &mut [MaybeUninit]); + + /// Unpacks into potentially uninitialized output storage. + /// + /// # Safety + /// The input and output lengths have the same requirements as + /// [`BitPacking::unchecked_unpack`]. Every output element is initialized on return. + unsafe fn unchecked_unpack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ); +} + +macro_rules! impl_bitpacking_compat { + ($ty:ty) => { + impl BitPacking for $ty { + unsafe fn unchecked_pack(width: usize, input: &[Self], output: &mut [Self]) { + let output = unsafe { + core::slice::from_raw_parts_mut( + output.as_mut_ptr().cast::>(), + output.len(), + ) + }; + unsafe { ::unchecked_pack_uninit(width, input, output) }; + } + + unsafe fn unchecked_unpack(width: usize, input: &[Self], output: &mut [Self]) { + let output = unsafe { + core::slice::from_raw_parts_mut( + output.as_mut_ptr().cast::>(), + output.len(), + ) + }; + unsafe { + ::unchecked_unpack_uninit(width, input, output) + }; + } + } + }; +} + +impl_bitpacking_compat!(u8); +impl_bitpacking_compat!(u16); +impl_bitpacking_compat!(u32); +impl_bitpacking_compat!(u64); + +impl BitPackingUninit for u8 { + unsafe fn unchecked_pack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( output.len(), @@ -256,7 +314,11 @@ impl BitPacking for u8 { } } - unsafe fn unchecked_unpack(width: usize, input: &[Self], output: &mut [Self]) { + unsafe fn unchecked_unpack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( input.len(), @@ -273,7 +335,7 @@ impl BitPacking for u8 { match width { 0 => { // A zero-width packed chunk implies all zeros. - output.fill(0); + output.fill(MaybeUninit::new(0)); } 1 => unpack_8_1( array_ref![input, 0, 1024 / 8], @@ -313,8 +375,12 @@ impl BitPacking for u8 { } } -impl BitPacking for u16 { - unsafe fn unchecked_pack(width: usize, input: &[Self], output: &mut [Self]) { +impl BitPackingUninit for u16 { + unsafe fn unchecked_pack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( output.len(), @@ -402,7 +468,11 @@ impl BitPacking for u16 { } } - unsafe fn unchecked_unpack(width: usize, input: &[Self], output: &mut [Self]) { + unsafe fn unchecked_unpack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( input.len(), @@ -418,7 +488,7 @@ impl BitPacking for u16 { match width { 0 => { - output.fill(0); + output.fill(MaybeUninit::new(0)); } 1 => unpack_16_1( array_ref![input, 0, 1024 / 16], @@ -491,8 +561,12 @@ impl BitPacking for u16 { } } -impl BitPacking for u32 { - unsafe fn unchecked_pack(width: usize, input: &[Self], output: &mut [Self]) { +impl BitPackingUninit for u32 { + unsafe fn unchecked_pack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( output.len(), @@ -646,7 +720,11 @@ impl BitPacking for u32 { } } - unsafe fn unchecked_unpack(width: usize, input: &[Self], output: &mut [Self]) { + unsafe fn unchecked_unpack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( input.len(), @@ -662,7 +740,7 @@ impl BitPacking for u32 { match width { 0 => { - output.fill(0); + output.fill(MaybeUninit::new(0)); } 1 => unpack_32_1( array_ref![input, 0, 1024 / 32], @@ -801,8 +879,12 @@ impl BitPacking for u32 { } } -impl BitPacking for u64 { - unsafe fn unchecked_pack(width: usize, input: &[Self], output: &mut [Self]) { +impl BitPackingUninit for u64 { + unsafe fn unchecked_pack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( output.len(), @@ -1087,7 +1169,11 @@ impl BitPacking for u64 { } } - unsafe fn unchecked_unpack(width: usize, input: &[Self], output: &mut [Self]) { + unsafe fn unchecked_unpack_uninit( + width: usize, + input: &[Self], + output: &mut [MaybeUninit], + ) { let packed_len = 128 * width / size_of::(); debug_assert_eq!( input.len(), @@ -1103,7 +1189,7 @@ impl BitPacking for u64 { match width { 0 => { - output.fill(0); + output.fill(MaybeUninit::new(0)); } 1 => unpack_64_1( array_ref![input, 0, 1024 / 64], @@ -1375,10 +1461,10 @@ impl BitPacking for u64 { macro_rules! unpack_8 { ($name:ident, $bits:expr) => { - fn $name(input: &[u8; 1024 * $bits / u8::T], output: &mut [u8; 1024]) { + fn $name(input: &[u8; 1024 * $bits / u8::T], output: &mut [MaybeUninit; 1024]) { for lane in 0..u8::LANES { unpack!(u8, $bits, input, lane, |$idx, $elem| { - output[$idx] = $elem; + output[$idx].write($elem); }); } } @@ -1396,7 +1482,7 @@ unpack_8!(unpack_8_8, 8); macro_rules! pack_8 { ($name:ident, $bits:expr) => { - fn $name(input: &[u8; 1024], output: &mut [u8; 1024 * $bits / u8::T]) { + fn $name(input: &[u8; 1024], output: &mut [MaybeUninit; 1024 * $bits / u8::T]) { for lane in 0..u8::LANES { pack!(u8, $bits, output, lane, |$idx| { input[$idx] }); } @@ -1414,10 +1500,10 @@ pack_8!(pack_8_8, 8); macro_rules! unpack_16 { ($name:ident, $bits:expr) => { - fn $name(input: &[u16; 1024 * $bits / u16::T], output: &mut [u16; 1024]) { + fn $name(input: &[u16; 1024 * $bits / u16::T], output: &mut [MaybeUninit; 1024]) { for lane in 0..u16::LANES { unpack!(u16, $bits, input, lane, |$idx, $elem| { - output[$idx] = $elem; + output[$idx].write($elem); }); } } @@ -1443,7 +1529,7 @@ unpack_16!(unpack_16_16, 16); macro_rules! pack_16 { ($name:ident, $bits:expr) => { - fn $name(input: &[u16; 1024], output: &mut [u16; 1024 * $bits / u16::T]) { + fn $name(input: &[u16; 1024], output: &mut [MaybeUninit; 1024 * $bits / u16::T]) { for lane in 0..u16::LANES { pack!(u16, $bits, output, lane, |$idx| { input[$idx] }); } @@ -1470,10 +1556,10 @@ pack_16!(pack_16_16, 16); macro_rules! unpack_32 { ($name:ident, $bit_width:expr) => { - fn $name(input: &[u32; 1024 * $bit_width / u32::T], output: &mut [u32; 1024]) { + fn $name(input: &[u32; 1024 * $bit_width / u32::T], output: &mut [MaybeUninit; 1024]) { for lane in 0..u32::LANES { unpack!(u32, $bit_width, input, lane, |$idx, $elem| { - output[$idx] = $elem + output[$idx].write($elem); }); } } @@ -1515,7 +1601,10 @@ unpack_32!(unpack_32_32, 32); macro_rules! pack_32 { ($name:ident, $bits:expr) => { - fn $name(input: &[u32; 1024], output: &mut [u32; 1024 * $bits / u32::BITS as usize]) { + fn $name( + input: &[u32; 1024], + output: &mut [MaybeUninit; 1024 * $bits / u32::BITS as usize], + ) { for lane in 0..u32::LANES { pack!(u32, $bits, output, lane, |$idx| { input[$idx] }); } @@ -1558,10 +1647,10 @@ pack_32!(pack_32_32, 32); macro_rules! unpack_64 { ($name:ident, $bit_width:expr) => { - fn $name(input: &[u64; 1024 * $bit_width / u64::T], output: &mut [u64; 1024]) { + fn $name(input: &[u64; 1024 * $bit_width / u64::T], output: &mut [MaybeUninit; 1024]) { for lane in 0..u64::LANES { unpack!(u64, $bit_width, input, lane, |$idx, $elem| { - output[$idx] = $elem + output[$idx].write($elem); }); } } @@ -1636,7 +1725,10 @@ unpack_64!(unpack_64_64, 64); macro_rules! pack_64 { ($name:ident, $bits:expr) => { - fn $name(input: &[u64; 1024], output: &mut [u64; 1024 * $bits / u64::BITS as usize]) { + fn $name( + input: &[u64; 1024], + output: &mut [MaybeUninit; 1024 * $bits / u64::BITS as usize], + ) { for lane in 0..u64::LANES { pack!(u64, $bits, output, lane, |$idx| { input[$idx] }); } @@ -1842,13 +1934,16 @@ mod test { *value = (rng.next() % (1 << bit_width)) as u8; } - let mut packed = vec![0; 1024 * bit_width / 8]; + let mut packed = vec![MaybeUninit::uninit(); 1024 * bit_width / 8]; for lane in 0..u8::LANES { // Always loop over lanes first. This is what the compiler vectorizes. pack!(u8, bit_width, packed, lane, |$pos| { values[$pos] }); } + // The pack kernel writes every element of the packed output. + let packed = + unsafe { core::slice::from_raw_parts(packed.as_ptr().cast::(), packed.len()) }; let mut unpacked: [u8; 1024] = [0; 1024]; for lane in 0..u8::LANES { @@ -1868,13 +1963,16 @@ mod test { *value = (rng.next() % (1 << bit_width)) as u16; } - let mut packed = vec![0; 1024 * bit_width / 16]; + let mut packed = vec![MaybeUninit::uninit(); 1024 * bit_width / 16]; for lane in 0..u16::LANES { // Always loop over lanes first. This is what the compiler vectorizes. pack!(u16, bit_width, packed, lane, |$pos| { values[$pos] }); } + // The pack kernel writes every element of the packed output. + let packed = + unsafe { core::slice::from_raw_parts(packed.as_ptr().cast::(), packed.len()) }; let mut unpacked: [u16; 1024] = [0; 1024]; for lane in 0..u16::LANES { @@ -1894,13 +1992,16 @@ mod test { *value = (rng.next() % (1 << bit_width)) as u32; } - let mut packed = vec![0; 1024 * bit_width / 32]; + let mut packed = vec![MaybeUninit::uninit(); 1024 * bit_width / 32]; for lane in 0..u32::LANES { // Always loop over lanes first. This is what the compiler vectorizes. pack!(u32, bit_width, packed, lane, |$pos| { values[$pos] }); } + // The pack kernel writes every element of the packed output. + let packed = + unsafe { core::slice::from_raw_parts(packed.as_ptr().cast::(), packed.len()) }; let mut unpacked: [u32; 1024] = [0; 1024]; for lane in 0..u32::LANES { @@ -1926,13 +2027,16 @@ mod test { } } - let mut packed = vec![0; 1024 * bit_width / 64]; + let mut packed = vec![MaybeUninit::uninit(); 1024 * bit_width / 64]; for lane in 0..u64::LANES { // Always loop over lanes first. This is what the compiler vectorizes. pack!(u64, bit_width, packed, lane, |$pos| { values[$pos] }); } + // The pack kernel writes every element of the packed output. + let packed = + unsafe { core::slice::from_raw_parts(packed.as_ptr().cast::(), packed.len()) }; let mut unpacked: [u64; 1024] = [0; 1024]; for lane in 0..u64::LANES { diff --git a/rust/compression/fsst/Cargo.toml b/rust/compression/fsst/Cargo.toml index da5d8f01d04..7056896e3b9 100644 --- a/rust/compression/fsst/Cargo.toml +++ b/rust/compression/fsst/Cargo.toml @@ -16,7 +16,6 @@ arrow-array.workspace = true rand.workspace = true [dev-dependencies] -arrow-array.workspace = true test-log.workspace = true tokio.workspace = true diff --git a/rust/compression/fsst/examples/benchmark.rs b/rust/compression/fsst/examples/benchmark.rs index c442243e112..f71abeefedd 100644 --- a/rust/compression/fsst/examples/benchmark.rs +++ b/rust/compression/fsst/examples/benchmark.rs @@ -56,7 +56,10 @@ fn benchmark(file_path: &str) { let mut decompression_out_bufs = vec![]; let mut decompression_out_offsets_bufs = vec![]; for _ in 0..TEST_NUM { - let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 3]; + // `decompress` requires the output buffer to be at least 8x the compressed input (a 1-byte + // code can expand to an 8-byte symbol). The compressed buffer is at most `BUFFER_SIZE`, so + // `BUFFER_SIZE * 8` is a safe upper bound. + let this_decom_out_buf = vec![0u8; BUFFER_SIZE * 8]; let this_decom_out_offsets_buf = vec![0i32; BUFFER_SIZE * 3]; decompression_out_bufs.push(this_decom_out_buf); decompression_out_offsets_bufs.push(this_decom_out_offsets_buf); diff --git a/rust/compression/fsst/src/fsst.rs b/rust/compression/fsst/src/fsst.rs index 0a2bf1d03d7..c5b619ba2d2 100644 --- a/rust/compression/fsst/src/fsst.rs +++ b/rust/compression/fsst/src/fsst.rs @@ -48,6 +48,7 @@ pub const FSST_SYMBOL_TABLE_SIZE: usize = 8 + 256 * 8 + 256; // 8 bytes for the use arrow_array::OffsetSizeTrait; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; +use std::cell::Cell; use std::cmp::Ordering; use std::collections::BinaryHeap; use std::collections::HashSet; @@ -57,6 +58,12 @@ use std::ptr; #[inline] fn fsst_unaligned_load_unchecked(v: *const u8) -> u64 { + // SAFETY: the caller must guarantee that `v` points to at least 8 readable bytes. All callers + // uphold this: `compress_bulk` loads from a 520-byte stack buffer at an offset < 511 (leaving + // >= 8 bytes), `build_symbol_table` guards the load with `word.len() > 7 && curr < word.len() - 7`, + // `find_longest_symbol_from_char_slice` copies into a stack `[u8; 8]` before loading, and + // `FsstDecoder::init` reads symbols from a `symbol_table` buffer already validated to be + // exactly `FSST_SYMBOL_TABLE_SIZE` bytes. unsafe { ptr::read_unaligned(v as *const u64) } } @@ -801,6 +808,121 @@ fn compress_bulk( Ok(()) } +fn offset_to_usize(offset: T) -> io::Result { + offset.to_usize().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!( + "FSST offset (as usize {}) is negative or exceeds {}", + offset.as_usize(), + T::MAX_OFFSET + ), + ) + }) +} + +fn validate_offsets(offsets: &[T], compressed_len: usize) -> io::Result<()> { + let Some((first, rest)) = offsets.split_first() else { + return Ok(()); + }; + let mut previous = offset_to_usize(*first)?; + if previous > compressed_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "FSST offset[0] = {previous} is out of bounds for compressed buffer of length {compressed_len}" + ), + )); + } + for (index, offset) in rest.iter().enumerate() { + let current = offset_to_usize(*offset)?; + let position = index + 1; + if current < previous { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("FSST offset at position {position} decreases: {current} < {previous}"), + )); + } + if current > compressed_len { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "FSST offset at position {position} = {current} is out of bounds for compressed buffer of length {compressed_len}" + ), + )); + } + previous = current; + } + Ok(()) +} + +fn encode_offset(value: usize) -> io::Result { + T::from_usize(value).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("FSST decompressed size {value} does not fit in the offset type"), + ) + }) +} + +#[inline(always)] +fn write_symbol(out: &mut [u8], out_curr: usize, symbol: u64) { + debug_assert!( + out_curr.checked_add(8).is_some_and(|end| end <= out.len()), + "FSST symbol write at {out_curr} overflows buffer of length {}", + out.len() + ); + // SAFETY: `FsstDecoder::init` rejected any declared `lens[i]` outside 1..=8 and left + // undeclared slots at 0, so every code advances `out_curr` by at most 8. Combined with + // the 8x output-buffer check, `out_curr + 8 <= out.len()` holds for every write. + unsafe { + ptr::write_unaligned(out.as_mut_ptr().add(out_curr) as *mut u64, symbol); + } +} + +fn store_out_byte(out: &mut [u8], out_curr: usize, byte: u8) { + debug_assert!( + out_curr < out.len(), + "FSST literal write at {out_curr} overflows buffer of length {}", + out.len() + ); + // SAFETY: same 8x + `lens[code] <= 8` proof as `write_symbol`. A literal + // writes one byte at `out_curr` after a run that advanced by at most 8 + // bytes per consumed input byte. + unsafe { + *out.get_unchecked_mut(out_curr) = byte; + } +} + +/// Consume `FSST_ESC` at `in_curr` and emit its payload byte. +/// +/// Returns `false` when the payload would leave the current value interval +/// `[in_curr, in_end)`, including a dangling escape at the last byte. +#[inline(always)] +fn emit_escape( + compressed_strs: &[u8], + in_curr: &mut usize, + in_end: usize, + out: &mut [u8], + out_curr: &mut usize, +) -> bool { + let payload_index = *in_curr + 1; + if payload_index >= in_end { + return false; + } + store_out_byte(out, *out_curr, compressed_strs[payload_index]); + *out_curr += 1; + *in_curr += 2; + true +} + +fn missing_escape_payload_error() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "FSST escape is missing a payload byte inside the current value", + ) +} + fn decompress_bulk( decoder: &FsstDecoder, compressed_strs: &[u8], @@ -810,14 +932,28 @@ fn decompress_bulk( out_pos: &mut usize, out_offsets_len: &mut usize, ) -> io::Result<()> { + validate_offsets(offsets, compressed_strs.len())?; + let symbols = decoder.symbols; let lens = decoder.lens; + // SAFETY invariant shared by every `unsafe` block in this closure: + // - `out` is sized to at least 8x `compressed_strs` (checked in `FsstDecoder::init`). + // `init` also rejects any declared symbol length outside 1..=8, so each consumed + // input byte yields at most 8 output bytes and `out_curr + 8 <= out.len()` at every + // 8-byte write, including the final one. + // - Offsets have been normalized with `to_usize` and checked to be non-decreasing + // and within `compressed_strs.len()`, so `in_curr + 4 <= in_end` implies the + // `read_unaligned::` is in bounds. + let corrupt_escape = Cell::new(false); let mut decompress = |mut in_curr: usize, in_end: usize, out_curr: &mut usize| { // Do SIMD operation here by 4 bytes while in_curr + 4 <= in_end { let next_block; let mut code; let mut len; + // SAFETY: the loop guard proves `in_curr + 4 <= in_end`. Per the closure-level + // invariant, `in_end <= compressed_strs.len()` is a trusted precondition (not checked + // here), so the 4-byte read is in bounds for well-formed input. unsafe { next_block = ptr::read_unaligned(compressed_strs.as_ptr().add(in_curr) as *const u32); @@ -828,40 +964,28 @@ fn decompress_bulk( // 0th byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; // 1st byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; // 2nd byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; // 3rd byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; } else { @@ -870,137 +994,104 @@ fn decompress_bulk( // 0th byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; // 1st byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; // 2nd byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; - // escape byte - in_curr += 2; - out[*out_curr] = compressed_strs[in_curr - 1]; - *out_curr += 1; + // ESC is the last byte of this 4-byte window; its payload is the next + // byte and may lie outside the current value. + if !emit_escape(compressed_strs, &mut in_curr, in_end, out, out_curr) { + corrupt_escape.set(true); + return; + } } else if first_escape_pos == 2 { // 0th byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; // 1st byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; - // escape byte + // payload is inside the 4-byte window (`in_curr + 4 <= in_end`) in_curr += 2; - out[*out_curr] = compressed_strs[in_curr - 1]; + store_out_byte(out, *out_curr, compressed_strs[in_curr - 1]); *out_curr += 1; } else if first_escape_pos == 1 { // 0th byte code = compressed_strs[in_curr] as usize; len = lens[code] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += len; - // escape byte in_curr += 2; - out[*out_curr] = compressed_strs[in_curr - 1]; + store_out_byte(out, *out_curr, compressed_strs[in_curr - 1]); *out_curr += 1; } else { - // escape byte in_curr += 2; - out[*out_curr] = compressed_strs[in_curr - 1]; + store_out_byte(out, *out_curr, compressed_strs[in_curr - 1]); *out_curr += 1; } } } - // handle the remaining bytes - if in_curr + 2 <= in_end { - out[*out_curr] = compressed_strs[in_curr + 1]; - if compressed_strs[in_curr] != FSST_ESC { - let code = compressed_strs[in_curr] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); + while in_curr < in_end { + if compressed_strs[in_curr] == FSST_ESC { + if !emit_escape(compressed_strs, &mut in_curr, in_end, out, out_curr) { + corrupt_escape.set(true); + return; } + } else { + let code = compressed_strs[in_curr] as usize; + write_symbol(out, *out_curr, symbols[code]); in_curr += 1; *out_curr += lens[code] as usize; - if compressed_strs[in_curr] != FSST_ESC { - let code = compressed_strs[in_curr] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); - } - in_curr += 1; - *out_curr += lens[code] as usize; - } else { - in_curr += 2; - out[*out_curr] = compressed_strs[in_curr - 1]; - *out_curr += 1; - } - } else { - in_curr += 2; - *out_curr += 1; - } - } - - if in_curr < in_end { - // last code cannot be an escape code - let code = compressed_strs[in_curr] as usize; - unsafe { - let src = symbols[code]; - ptr::write_unaligned(out.as_mut_ptr().add(*out_curr) as *mut u64, src); } - *out_curr += lens[code] as usize; } }; let mut out_curr = *out_pos; - out_offsets[0] = T::from_usize(*out_pos).unwrap(); + if offsets.is_empty() { + out.resize(out_curr, 0); + out_offsets.clear(); + *out_offsets_len = 0; + return Ok(()); + } + + out_offsets[0] = encode_offset(*out_pos)?; for i in 1..offsets.len() { + // `validate_offsets` already proved these convert and stay in range. let in_curr = offsets[i - 1].as_usize(); let in_end = offsets[i].as_usize(); decompress(in_curr, in_end, &mut out_curr); - out_offsets[i] = T::from_usize(out_curr).unwrap(); + if corrupt_escape.get() { + return Err(missing_escape_payload_error()); + } + out_offsets[i] = encode_offset(out_curr)?; } out.resize(out_curr, 0); - out_offsets.resize(offsets.len(), T::from_usize(0).unwrap()); + out_offsets.resize(offsets.len(), encode_offset(0)?); *out_pos = out_curr; *out_offsets_len = offsets.len(); Ok(()) @@ -1169,14 +1260,6 @@ impl FsstDecoder { out_buf: &[u8], out_offsets_buf: &[T], ) -> io::Result<()> { - let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().unwrap()); - if st_info & FSST_MAGIC != FSST_MAGIC { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "the input buffer is not a valid FSST compressed data", - )); - } - if symbol_table.len() != FSST_SYMBOL_TABLE_SIZE { return Err(io::Error::new( io::ErrorKind::InvalidInput, @@ -1187,9 +1270,33 @@ impl FsstDecoder { )); } + let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::InvalidData, + "FSST symbol table is too short to contain a header", + ) + })?); + if st_info & FSST_MAGIC != FSST_MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the input buffer is not a valid FSST compressed data", + )); + } + self.decoder_switch_on = (st_info & (1 << 24)) != 0; - // when decoder_switch_on is true, we make sure the out_buf is at least 3 times the size of the in_buf, - if self.decoder_switch_on && in_buf.len() * 3 > out_buf.len() { + // A single 1-byte code can decode to a symbol of up to MAX_SYMBOL_LENGTH (8) bytes, so the + // decoded output can be up to 8x the input. `decompress_bulk` also relies on this bound: it + // writes a full 8-byte word per code (advancing only by the symbol length), so the output + // buffer must be large enough that even the final write stays in bounds. Require out_buf to + // be at least 8x in_buf. `checked_mul` guards against `in_buf.len() * 8` wrapping on 32-bit + // targets (an input >= 512 MiB would otherwise bypass the check); treat overflow as too + // small. + if self.decoder_switch_on + && in_buf + .len() + .checked_mul(8) + .is_none_or(|needed| needed > out_buf.len()) + { return Err(io::Error::new( io::ErrorKind::InvalidInput, "output buffer too small for FSST decoder", @@ -1221,7 +1328,16 @@ impl FsstDecoder { pos += 8; } for i in 0..symbol_num as usize { - self.lens[i] = symbol_table[pos]; + let len = symbol_table[pos]; + if !(1..=MAX_SYMBOL_LENGTH as u8).contains(&len) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "FSST symbol length at index {i} is {len}, expected 1..={MAX_SYMBOL_LENGTH}" + ), + )); + } + self.lens[i] = len; pos += 1; } Ok(()) @@ -1235,6 +1351,7 @@ impl FsstDecoder { out_offsets_buf: &mut Vec, ) -> io::Result<()> { if !self.decoder_switch_on { + validate_offsets(in_offsets_buf, in_buf.len())?; out_buf.resize(in_buf.len(), 0); out_buf.copy_from_slice(in_buf); out_offsets_buf.resize(in_offsets_buf.len(), T::from_usize(0).unwrap()); @@ -1288,9 +1405,14 @@ pub fn compress( // the following 32 bits after FSST_MAGIC contains information about FSST encoding, such as decoder_switch_on, suffix_lim, terminator, n_symbols // when the decoder_switch_on is off in the in_buf header, `decompress` first make sure the out_buf is at least the same size as the in_buf, then simply copy the // input data to the output -// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 3 times the size of the in_buf, then start decoding the -// data using the symbol table +// when the decoder_switch_on is on, `decompress` first make sure the out_buf is at least 8 times the size of the in_buf, then start decoding the +// data using the symbol table. The 8x bound is required for correctness: a 1-byte code can expand to +// an 8-byte symbol, and the decode loop writes a full 8-byte word per code, so a smaller buffer can +// be written out of bounds. // the out_offsets_buf should be at least the same size as the in_offsets_buf, otherwise an error is returned +// the symbol_table, compressed bytes, and offsets are untrusted: declared symbol lengths must be +// 1..=8 and offsets must be a non-decreasing sequence of values that fit in usize and lie within +// the compressed buffer. Corrupt input returns InvalidData instead of writing out of bounds. // the symbol_table is the same symbol table created by `compression` pub fn decompress( symbol_table: &[u8], @@ -1644,4 +1766,338 @@ But exactly how the acquaintance and friendship came about, we cannot say."; ); } } + + // Build a genuinely FSST-compressed (decoder_switch_on) buffer to exercise the decode-side + // output-buffer size contract. Returns (symbol_table, compressed_bytes, compressed_offsets). + fn compress_paragraph() -> ([u8; FSST_SYMBOL_TABLE_SIZE], Vec, Vec) { + let test_input = TEST_PARAGRAPH.repeat((1024 * 1024) / TEST_PARAGRAPH.len()); + let lines_vec = test_input.lines().collect::>(); + let string_array = StringArray::from(lines_vec); + let mut compress_output_buf: Vec = vec![0; string_array.value_data().len()]; + let mut compress_offset_buf: Vec = vec![0; string_array.value_offsets().len()]; + let mut symbol_table = [0; FSST_SYMBOL_TABLE_SIZE]; + compress( + symbol_table.as_mut(), + string_array.value_data(), + string_array.value_offsets(), + &mut compress_output_buf, + &mut compress_offset_buf, + ) + .unwrap(); + (symbol_table, compress_output_buf, compress_offset_buf) + } + + // The decoder writes a full 8-byte word per code, so the output buffer must be at least 8x the + // compressed input. A buffer sized below 8x must be rejected rather than written out of bounds. + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_undersized_output_buffer() { + let (symbol_table, compressed, compressed_offsets) = compress_paragraph(); + // Sanity check: this input actually engaged FSST compression (decoder_switch_on). + let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().unwrap()); + assert!(st_info & (1 << 24) != 0, "expected decoder_switch_on input"); + + // One byte short of the 8x requirement must be rejected. + let mut too_small = vec![0u8; compressed.len() * 8 - 1]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut too_small, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + + // Exactly 8x is the tight bound and must succeed. + let mut exact = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut exact, + &mut out_offsets, + ) + .unwrap(); + } + + fn declared_lens_range(symbol_table: &[u8]) -> std::ops::Range { + let n_symbols = (u64::from_ne_bytes(symbol_table[..8].try_into().unwrap()) & 255) as usize; + let start = 8 + n_symbols * 8; + start..start + n_symbols + } + + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_corrupt_symbol_length() { + let (mut symbol_table, compressed, compressed_offsets) = compress_paragraph(); + let lens = declared_lens_range(&symbol_table); + assert!(!lens.is_empty(), "expected at least one declared symbol"); + symbol_table[lens.start] = 9; + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + err.to_string().contains("symbol length"), + "unexpected error: {err}" + ); + } + + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_zero_declared_symbol_length() { + let (mut symbol_table, compressed, compressed_offsets) = compress_paragraph(); + let lens = declared_lens_range(&symbol_table); + symbol_table[lens.start] = 0; + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + err.to_string().contains("symbol length"), + "unexpected error: {err}" + ); + } + + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_out_of_range_offset() { + let (symbol_table, compressed, mut compressed_offsets) = compress_paragraph(); + let last = compressed_offsets.len() - 1; + compressed_offsets[last] = i32::try_from(compressed.len()).unwrap() + 1; + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + err.to_string().contains("out of bounds"), + "unexpected error: {err}" + ); + } + + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_decreasing_offset() { + let (symbol_table, compressed, mut compressed_offsets) = compress_paragraph(); + assert!(compressed_offsets.len() >= 2); + if compressed_offsets[0] == 0 { + compressed_offsets[0] = 1; + } + compressed_offsets[1] = compressed_offsets[0] - 1; + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + err.to_string().contains("decreases"), + "unexpected error: {err}" + ); + } + + #[test_log::test(tokio::test)] + async fn test_decompress_rejects_negative_offset() { + let (symbol_table, compressed, mut compressed_offsets) = compress_paragraph(); + compressed_offsets[0] = -1; + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + err.to_string().contains("negative") || err.to_string().contains("out of bounds"), + "unexpected error: {err}" + ); + } + + #[test_log::test(tokio::test)] + async fn test_decompress_accepts_max_symbol_length() { + let (symbol_table, compressed, compressed_offsets) = compress_paragraph(); + let lens = declared_lens_range(&symbol_table); + assert!( + symbol_table[lens].contains(&(MAX_SYMBOL_LENGTH as u8)), + "expected encoder to emit at least one 8-byte symbol" + ); + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap(); + } + + #[test_log::test(tokio::test)] + async fn test_undeclared_code_does_not_overflow() { + let (symbol_table, mut compressed, compressed_offsets) = compress_paragraph(); + let n_symbols = (u64::from_ne_bytes(symbol_table[..8].try_into().unwrap()) & 255) as usize; + if n_symbols >= 255 { + return; + } + let undeclared = n_symbols as u8; + if let Some(byte) = compressed.iter_mut().find(|byte| **byte != FSST_ESC) { + *byte = undeclared; + } + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap(); + } + + fn switch_off_roundtrip() -> ([u8; FSST_SYMBOL_TABLE_SIZE], Vec, Vec) { + let input = b"raw"; + let offsets = [0_i32, 3]; + let mut table = [0_u8; FSST_SYMBOL_TABLE_SIZE]; + let mut compressed = vec![0; input.len()]; + let mut compressed_offsets = vec![0_i32; offsets.len()]; + compress( + &mut table, + input, + &offsets, + &mut compressed, + &mut compressed_offsets, + ) + .unwrap(); + let st_info = u64::from_ne_bytes(table[..8].try_into().unwrap()); + assert!(st_info & (1 << 24) == 0, "expected decoder_switch_on off"); + (table, compressed, compressed_offsets) + } + + #[test] + fn test_decompress_rejects_corrupt_offsets_when_switch_off() { + let (table, compressed, _) = switch_off_roundtrip(); + let corrupt_offsets = [-1_i32, 99]; + let mut out = vec![0; compressed.len()]; + let mut out_offsets = vec![0_i32; corrupt_offsets.len()]; + let err = decompress( + &table, + &compressed, + &corrupt_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + fn test_decompress_switch_off_accepts_valid_offsets() { + let (table, compressed, compressed_offsets) = switch_off_roundtrip(); + let mut out = vec![0; compressed.len()]; + let mut out_offsets = vec![0_i32; compressed_offsets.len()]; + decompress( + &table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .unwrap(); + assert_eq!(out, compressed); + assert_eq!(out_offsets, compressed_offsets); + } + + fn assert_missing_escape_payload(table: &[u8], bytes: &[u8], offsets: &[i32]) { + let mut out = vec![0_u8; bytes.len() * 8]; + let mut out_offsets = vec![0_i32; offsets.len()]; + let err = decompress(table, bytes, offsets, &mut out, &mut out_offsets).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!( + err.to_string().contains("escape"), + "unexpected error: {err}" + ); + } + + #[test] + fn test_decompress_rejects_dangling_escape_in_scalar_tail() { + let (table, _, _) = compress_paragraph(); + assert_missing_escape_payload(&table, &[0, FSST_ESC], &[0, 2]); + assert_missing_escape_payload(&table, &[FSST_ESC], &[0, 1]); + } + + #[test] + fn test_decompress_rejects_dangling_escape_in_fast_path() { + let (table, _, _) = compress_paragraph(); + // 4-byte window ending in ESC: payload sits at index 4, outside in_end. + assert_missing_escape_payload(&table, &[0, 0, 0, FSST_ESC], &[0, 4]); + } + + #[test] + fn test_decompress_rejects_escape_payload_from_next_value() { + let (table, _, _) = compress_paragraph(); + // First value ends on ESC; the next value's first byte must not be stolen as payload. + assert_missing_escape_payload(&table, &[0, 0, 0, FSST_ESC, b'X'], &[0, 4, 5]); + } + + #[test] + fn test_decompress_accepts_escape_with_payload() { + let (table, _, _) = compress_paragraph(); + let bytes = [FSST_ESC, b'A']; + let offsets = [0_i32, 2]; + let mut out = vec![0_u8; bytes.len() * 8]; + let mut out_offsets = vec![0_i32; offsets.len()]; + decompress(&table, &bytes, &offsets, &mut out, &mut out_offsets).unwrap(); + assert_eq!(&out[..], b"A"); + assert_eq!(out_offsets, [0, 1]); + } + + #[test] + fn test_decompress_accepts_fast_path_escape_with_payload() { + let (table, _, _) = compress_paragraph(); + let bytes = [0, 0, 0, FSST_ESC, b'Z']; + let offsets = [0_i32, 5]; + let mut out = vec![0_u8; bytes.len() * 8]; + let mut out_offsets = vec![0_i32; offsets.len()]; + decompress(&table, &bytes, &offsets, &mut out, &mut out_offsets).unwrap(); + assert_eq!(out_offsets[0], 0); + assert_eq!(*out.last().unwrap(), b'Z'); + assert_eq!(out_offsets[1], i32::try_from(out.len()).unwrap()); + } } diff --git a/rust/examples/Cargo.toml b/rust/examples/Cargo.toml index 80eff457140..fa6d0676655 100644 --- a/rust/examples/Cargo.toml +++ b/rust/examples/Cargo.toml @@ -46,9 +46,9 @@ lance-datagen = { workspace = true } object_store = {workspace = true} tempfile = { workspace = true } tokio = { workspace = true } -all_asserts = "2.3.1" -env_logger = "0.11.7" +all_asserts.workspace = true +env_logger.workspace = true hf-hub = "0.4.2" -parquet = { version = "58.0.0", default-features = false, features = ["arrow", "async"] } +parquet = { workspace = true } tokenizers = "0.15.2" rand.workspace = true diff --git a/rust/examples/src/hnsw.rs b/rust/examples/src/hnsw.rs index 5be0debf6e1..52c15bd0851 100644 --- a/rust/examples/src/hnsw.rs +++ b/rust/examples/src/hnsw.rs @@ -129,6 +129,7 @@ async fn main() { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let results: HashSet = hnsw .search_basic(q.clone(), k, ¶ms, None, vector_store.as_ref()) diff --git a/rust/lance-arrow/Cargo.toml b/rust/lance-arrow/Cargo.toml index afbc9c26ed3..21513e638d8 100644 --- a/rust/lance-arrow/Cargo.toml +++ b/rust/lance-arrow/Cargo.toml @@ -21,6 +21,7 @@ arrow-ipc = { workspace = true } arrow-ord = { workspace = true } arrow-schema = { workspace = true } arrow-select = { workspace = true } +bytemuck = { workspace = true } bytes = { workspace = true } futures = { workspace = true } half = { workspace = true } diff --git a/rust/lance-arrow/README.md b/rust/lance-arrow/README.md index 8c3bedac3f3..f5feccaa191 100644 --- a/rust/lance-arrow/README.md +++ b/rust/lance-arrow/README.md @@ -1,7 +1,7 @@ # lance-arrow -`lance-arrow` is a internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) -extensions used by [Lance](https://github.com/lancedb/lance). +`lance-arrow` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) +extensions used by [Lance](https://github.com/lance-format/lance). **Important Note**: This crate is **not intended for external usage**. diff --git a/rust/lance-arrow/src/bfloat16.rs b/rust/lance-arrow/src/bfloat16.rs index 2f59de51317..74c7f259a40 100644 --- a/rust/lance-arrow/src/bfloat16.rs +++ b/rust/lance-arrow/src/bfloat16.rs @@ -7,7 +7,7 @@ use std::fmt::Formatter; use std::slice; use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder}; -use arrow_buffer::MutableBuffer; +use arrow_buffer::{Buffer, MutableBuffer}; use arrow_data::ArrayData; use arrow_schema::{ArrowError, DataType, Field as ArrowField}; use half::bf16; @@ -164,19 +164,18 @@ impl FromIterator for BFloat16Array { impl From> for BFloat16Array { fn from(data: Vec) -> Self { - let mut buffer = MutableBuffer::with_capacity(data.len() * 2); - - // Write each value's little-endian bytes straight into the buffer. Going - // through an intermediate `Vec` per element would allocate once per value. - for val in &data { - buffer.extend_from_slice(&val.to_bits().to_le_bytes()); - } - + let len = data.len(); + // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives + // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place — + // no per-element copy or heap alloc. The crate-root `compile_error!` + // pins `target_endian = "little"`, so the resulting bytes match the + // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere. + let raw: Vec = bytemuck::cast_vec(data); let array_data = ArrayData::builder(DataType::FixedSizeBinary(2)) - .len(data.len()) - .add_buffer(buffer.into()); - // SAFETY: the buffer contains exactly `2 * data.len()` bytes — each - // `bf16` writes its two little-endian bytes once — matching the + .len(len) + .add_buffer(Buffer::from_vec(raw)); + // SAFETY: the value buffer contains exactly `2 * len` bytes — one + // `u16` per element after the layout-compatible cast — matching the // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so // every element is logically valid. let array_data = unsafe { array_data.build_unchecked() }; @@ -284,9 +283,10 @@ impl FloatArray for FixedSizeBinaryArray { /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape /// used by [`BFloat16Array`]). Asserted at entry. /// - The value buffer must be at least 2-byte aligned. Lance's in-tree - /// constructors always satisfy this (every value buffer goes through - /// `MutableBuffer`, which is aligned to arrow-buffer's `ALIGNMENT` - /// constant — ≥32 bytes on every supported target). Externally-built + /// constructors always satisfy this: value buffers are built either via + /// `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32 + /// bytes) or via `Buffer::from_vec::` (aligned to `align_of::()` + /// == 2); both meet `bf16`'s 2-byte requirement. Externally-built /// `FixedSizeBinaryArray`s arriving via FFI, IPC, or /// `Buffer::from_custom_allocation` are not required by arrow-rs to be /// aligned beyond a single byte; passing one to this method violates the @@ -329,8 +329,8 @@ impl FloatArray for FixedSizeBinaryArray { // (arrow-data `data.rs`), so arrow-rs alone does not guarantee // 2-byte alignment. Lance's in-tree construction paths build value // buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant, - // ≥32 bytes on every supported target), which trivially satisfies - // `bf16`'s 2-byte requirement. + // ≥32 bytes) or `Buffer::from_vec::` (2-byte aligned), both of + // which satisfy `bf16`'s 2-byte requirement. // - The returned slice borrows from `self`; the underlying ref-counted, // immutable Arrow buffer cannot be mutated or freed for the slice's // lifetime. @@ -361,6 +361,16 @@ mod tests { assert_eq!(array, array2); assert_eq!(array.len(), 3); + // Pin the raw little-endian bytes emitted by `From>` (rewritten to + // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order + // regression is caught directly rather than only through Debug formatting. + // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040. + let inner = array2.clone().into_inner(); + let raw_bytes: Vec = (0..inner.len()) + .flat_map(|i| inner.value(i).to_vec()) + .collect(); + assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]); + let expected_fmt = "BFloat16Array\n[\n 1.0,\n 2.0,\n 3.0,\n]"; assert_eq!(expected_fmt, format!("{:?}", array)); diff --git a/rust/lance-arrow/src/lib.rs b/rust/lance-arrow/src/lib.rs index a55b42cb6c0..21e9ac39dee 100644 --- a/rust/lance-arrow/src/lib.rs +++ b/rust/lance-arrow/src/lib.rs @@ -20,7 +20,7 @@ use std::{collections::HashMap, ptr::NonNull}; use arrow_array::{ Array, ArrayRef, ArrowNumericType, FixedSizeBinaryArray, FixedSizeListArray, GenericListArray, - LargeListArray, ListArray, OffsetSizeTrait, PrimitiveArray, RecordBatch, StructArray, + LargeListArray, ListArray, MapArray, OffsetSizeTrait, PrimitiveArray, RecordBatch, StructArray, UInt8Array, UInt32Array, cast::AsArray, }; use arrow_array::{ @@ -475,6 +475,20 @@ pub fn iter_str_array(arr: &dyn Array) -> Box> } } +pub fn iter_binary_array( + arr: &dyn Array, +) -> Result> + Send + '_>> { + match arr.data_type() { + DataType::Binary => Ok(Box::new(arr.as_binary::().iter())), + DataType::LargeBinary => Ok(Box::new(arr.as_binary::().iter())), + DataType::BinaryView => Ok(Box::new(arr.as_binary_view().iter())), + DataType::FixedSizeBinary(_) => Ok(Box::new(arr.as_fixed_size_binary().iter())), + data_type => Err(ArrowError::InvalidArgumentError(format!( + "Expecting a binary type, found {data_type}" + ))), + } +} + /// Extends Arrow's [RecordBatch]. pub trait RecordBatchExt { /// Append a new column to this [`RecordBatch`] and returns a new RecordBatch. @@ -846,6 +860,30 @@ fn project_array(array: &ArrayRef, target_field: &Field) -> Result { list_arr.nulls().cloned(), ))) } + // A nullable entries field fails MapArray::try_new unconditionally, + // so a (schema-invalid) map declared that way keeps the clone + // fallthrough it always had rather than gaining a new error. + DataType::Map(entries_field, sorted) if !entries_field.is_nullable() => { + let map_arr = array.as_map(); + let DataType::Struct(entry_fields) = entries_field.data_type() else { + return Err(ArrowError::SchemaError(format!( + "Map entries field must be a struct, got {}", + entries_field.data_type() + ))); + }; + let projected_entries = project(map_arr.entries(), entry_fields)?; + // try_new re-checks the entries invariants (a non-null entries + // struct, two entry columns, offset bounds); null keys are ruled + // out one level down, by the struct rebuild against the + // non-nullable key field. + Ok(Arc::new(MapArray::try_new( + entries_field.clone(), + map_arr.offsets().clone(), + projected_entries, + map_arr.nulls().cloned(), + *sorted, + )?)) + } _ => Ok(array.clone()), } } @@ -1017,40 +1055,25 @@ fn merge_list_struct(left: &dyn Array, right: &dyn Array) -> Arc { } } -/// Helper function to normalize validity buffers -/// Returns None for all-null validity (placeholder structs) -fn normalize_validity( - validity: Option<&arrow_buffer::NullBuffer>, -) -> Option<&arrow_buffer::NullBuffer> { - validity.and_then(|v| { - if v.null_count() == v.len() { - None - } else { - Some(v) - } - }) -} - -/// Helper function to merge validity buffers from two struct arrays -/// Returns None only if both arrays are null at the same position +/// Helper function to merge validity buffers from two struct arrays. /// -/// Special handling for placeholder structs (all-null validity) +/// A row is valid if it is valid in either input. +/// An absent validity buffer means all rows are valid, an all-null buffer acts as the identity for this merge. fn merge_struct_validity( left_validity: Option<&arrow_buffer::NullBuffer>, right_validity: Option<&arrow_buffer::NullBuffer>, ) -> Option { - // Normalize both validity buffers (convert all-null to None) - let left_normalized = normalize_validity(left_validity); - let right_normalized = normalize_validity(right_validity); - - match (left_normalized, right_normalized) { + match (left_validity, right_validity) { // Fast paths: no computation needed - (None, None) => None, - (Some(left), None) => Some(left.clone()), - (None, Some(right)) => Some(right.clone()), + (None, _) | (_, None) => None, (Some(left), Some(right)) => { - // Fast path: if both have no nulls, can return either one - if left.null_count() == 0 && right.null_count() == 0 { + if left.null_count() == 0 || right.null_count() == 0 { + return None; + } + if left.null_count() == left.len() { + return Some(right.clone()); + } + if right.null_count() == right.len() { return Some(left.clone()); } @@ -1245,22 +1268,23 @@ fn merge(left_struct_array: &StructArray, right_struct_array: &StructArray) -> S if left_list.data_type().is_struct() && right_list.data_type().is_struct() => { - // If there is nothing to merge just use the left field + // Identical inner types: nothing to merge, keep the left column. if left_list.data_type() == right_list.data_type() { fields.push(left_field.as_ref().clone()); columns.push(left_column.clone()); + } else { + // The struct fields differ, so merge them structurally. merge_list_struct + // only succeeds when both lists share offsets or one side is all-null; + // it panics otherwise. + let merged_sub_array = merge_list_struct(&left_column, &right_column); + + fields.push(Field::new( + left_field.name(), + merged_sub_array.data_type().clone(), + left_field.is_nullable(), + )); + columns.push(merged_sub_array); } - // If we have two List and they have different sets of fields then - // we can merge them if the offsets arrays are the same. Otherwise, we - // have to consider it an error. - let merged_sub_array = merge_list_struct(&left_column, &right_column); - - fields.push(Field::new( - left_field.name(), - merged_sub_array.data_type().clone(), - left_field.is_nullable(), - )); - columns.push(merged_sub_array); } // otherwise, just use the field on the left hand side _ => { @@ -1385,9 +1409,12 @@ fn merge_with_schema( ); let merged_validity = merge_struct_validity(left_list.nulls(), right_list.nulls()); + // `trimmed_values` starts at the first used value, so offsets + // must be shifted to match or `ListArray::new` panics when the + // input list was sliced (e.g. from a filtered batch). let merged_list = ListArray::new( child_field.clone(), - left_list.offsets().clone(), + left_list.trimmed_offsets(), merged_values, merged_validity, ); @@ -1412,7 +1439,7 @@ fn merge_with_schema( merge_struct_validity(left_list.nulls(), right_list.nulls()); let merged_list = LargeListArray::new( child_field.clone(), - left_list.offsets().clone(), + left_list.trimmed_offsets(), merged_values, merged_validity, ); @@ -1572,8 +1599,11 @@ impl BufferExt for arrow_buffer::Buffer { #[cfg(test)] mod tests { use super::*; - use arrow_array::{Float32Array, Int32Array, NullArray, StructArray}; - use arrow_array::{ListArray, StringArray, new_empty_array, new_null_array}; + use arrow_array::{ + BinaryArray, BinaryViewArray, FixedSizeBinaryArray, Float32Array, Int32Array, + LargeBinaryArray, ListArray, NullArray, StringArray, StructArray, new_empty_array, + new_null_array, + }; use arrow_buffer::OffsetBuffer; #[test] @@ -1850,6 +1880,105 @@ mod tests { assert_eq!(merged, expected); } + #[test] + fn test_merge_list_struct_identical_schema() { + // Merging two batches whose `List` columns have identical types + // should yield a single column equal to the input (there is nothing to + // merge), not a struct with the field pushed twice. + let x_field = Arc::new(Field::new("x", DataType::Int32, true)); + let item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![x_field.clone()])), + true, + )); + let schema = Arc::new(Schema::new(vec![Field::new( + "list_struct", + DataType::List(item_field.clone()), + true, + )])); + + let build_list = |values: Vec| { + let len = values.len(); + let item_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field.clone()]), + vec![Arc::new(Int32Array::from(values))], + None, + )); + ListArray::new( + item_field.clone(), + OffsetBuffer::from_lengths([len]), + item_struct, + None, + ) + }; + + // Distinct values so the equality assertion proves the left column is kept, + // not that some column with the same values happens to be present. + let left = + RecordBatch::try_new(schema.clone(), vec![Arc::new(build_list(vec![1, 2]))]).unwrap(); + let right = RecordBatch::try_new(schema, vec![Arc::new(build_list(vec![3, 4]))]).unwrap(); + + let merged = left.merge(&right).unwrap(); + + // Exactly one column, equal to the left input: the field must not be duplicated. + assert_eq!(merged.num_columns(), 1); + assert_eq!(merged, left); + } + + #[test] + fn test_merge_nested_list_struct_identical_schema() { + // A `List` nested inside a struct reaches the identical-type + // short-circuit through the recursive `merge` call. The recursion must + // keep the field exactly once, equal to the left input. + let x_field = Arc::new(Field::new("x", DataType::Int32, true)); + let item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![x_field.clone()])), + true, + )); + let companies_field = Arc::new(Field::new( + "companies", + DataType::List(item_field.clone()), + true, + )); + let schema = Arc::new(Schema::new(vec![Field::new( + "outer", + DataType::Struct(Fields::from(vec![companies_field.clone()])), + true, + )])); + + let build_outer = |x: i32| { + // One list row: [{x}]. + let item_struct = Arc::new(StructArray::new( + Fields::from(vec![x_field.clone()]), + vec![Arc::new(Int32Array::from(vec![x]))], + None, + )); + let companies = Arc::new(ListArray::new( + item_field.clone(), + OffsetBuffer::from_lengths([1]), + item_struct, + None, + )); + StructArray::new( + Fields::from(vec![companies_field.clone()]), + vec![companies], + None, + ) + }; + + // Distinct values so equality proves the left column is kept. + let left = RecordBatch::try_new(schema.clone(), vec![Arc::new(build_outer(10))]).unwrap(); + let right = RecordBatch::try_new(schema, vec![Arc::new(build_outer(20))]).unwrap(); + + let merged = left.merge(&right).unwrap(); + + // The recursive identical-type merge keeps exactly one `companies` field, + // with no double-push inside the nested struct. + assert_eq!(merged.column(0).as_struct().num_columns(), 1); + assert_eq!(merged, left); + } + #[test] fn test_byte_width_opt() { assert_eq!(DataType::Int32.byte_width_opt(), Some(4)); @@ -1976,6 +2105,60 @@ mod tests { ); } + #[test] + fn test_project_rebuilds_sliced_map() { + // A sliced MapArray keeps its full entries array behind sliced + // offsets and validity; the Map projection arm must rebuild it + // without renormalizing either. + let entry_fields = Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ]); + let entries_field = Arc::new(Field::new( + "entries", + DataType::Struct(entry_fields.clone()), + false, + )); + let entries = StructArray::new( + entry_fields, + vec![ + Arc::new(StringArray::from(vec!["k0", "k1", "k2"])) as ArrayRef, + Arc::new(Int32Array::from(vec![1, 2, 3])) as ArrayRef, + ], + None, + ); + let map = MapArray::new( + entries_field.clone(), + OffsetBuffer::new(vec![0, 1, 1, 3].into()), + entries, + Some(arrow_buffer::NullBuffer::from(vec![true, false, true])), + false, + ); + let schema = Arc::new(Schema::new(vec![Field::new( + "m", + DataType::Map(entries_field, false), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(map) as ArrayRef]).unwrap(); + + // Rows 1..3: a null slot and a two-entry slot, offsets not zero-based. + let projected = batch + .slice(1, 2) + .project_by_schema(schema.as_ref()) + .unwrap(); + let map = projected.column(0).as_map(); + assert!(map.is_null(0)); + assert!(map.is_valid(1)); + assert_eq!(map.value_length(1), 2); + assert_eq!( + map.value(1) + .column(1) + .as_primitive::() + .values(), + &[2, 3] + ); + } + #[test] fn test_project_preserves_struct_validity() { // Test that projecting a struct array preserves its validity (fix for issue #4385) @@ -2054,6 +2237,47 @@ mod tests { assert_eq!(width_values.value(0), 300); assert_eq!(width_values.value(1), 200); assert!(width_values.is_null(2)); // width is null when right struct was null + + // An all-null validity buffer is data, not a placeholder meaning "this side has no + // validity": merging two of them keeps the rows null. + let all_null_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + Some(vec![false, false].into()), + ); + let all_null_right = StructArray::new( + Fields::from(vec![Field::new("width", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![None, None])) as ArrayRef], + Some(vec![false, false].into()), + ); + + let merged = merge(&all_null_left, &all_null_right); + assert_eq!(merged.null_count(), 2); + + // An all-null side is the identity of the merge, so the other side decides each row. + let partial_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef], + Some(vec![true, false].into()), + ); + let merged = merge(&partial_left, &all_null_right); + assert!(!merged.is_null(0)); + assert!(merged.is_null(1)); + + // A missing validity buffer means all rows are valid, which absorbs an all-null side. + let all_valid_left = StructArray::new( + Fields::from(vec![Field::new("height", DataType::Int32, true)]), + vec![Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef], + None, + ); + let merged = merge(&all_valid_left, &all_null_right); + assert_eq!(merged.null_count(), 0); + + // An explicit all-valid buffer has the same semantics as a missing buffer. + let all_valid: arrow_buffer::NullBuffer = vec![true, true].into(); + let partial: arrow_buffer::NullBuffer = vec![true, false].into(); + assert!(merge_struct_validity(Some(&all_valid), Some(&partial)).is_none()); + assert!(merge_struct_validity(Some(&partial), Some(&all_valid)).is_none()); } #[test] @@ -2380,6 +2604,118 @@ mod tests { assert_eq!(merged_array.len(), 2); } + #[test] + fn test_merge_with_schema_sliced_list_struct() { + test_merge_with_schema_sliced_list_struct_generic::(); + } + + #[test] + fn test_merge_with_schema_sliced_large_list_struct() { + test_merge_with_schema_sliced_list_struct_generic::(); + } + + // Regression for #6580: merge_with_schema panicked when the left list was a + // sliced view whose offsets did not start at zero (common after a filtered + // scan). Cloning those offsets alongside `trimmed_values` produced offsets + // larger than the trimmed child, panicking in `(Large)ListArray::new`. + fn test_merge_with_schema_sliced_list_struct_generic() { + let make_list_dtype = |item_field: Arc| { + if O::IS_LARGE { + DataType::LargeList(item_field) + } else { + DataType::List(item_field) + } + }; + + // Build a List with two rows of 5 items each, then slice away + // the first row so the remaining list's offsets start at 5, not 0. + let struct_fields_a = Fields::from(vec![Field::new("a", DataType::Int32, true)]); + let left_values = Arc::new(StructArray::new( + struct_fields_a.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10)) as ArrayRef], + None, + )); + let full_list = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(struct_fields_a), true)), + OffsetBuffer::::from_lengths([5, 5]), + left_values, + None, + ); + let sliced_left = full_list.slice(1, 1); + assert_eq!(sliced_left.offsets()[0].as_usize(), 5); + assert_eq!(sliced_left.offsets()[1].as_usize(), 10); + + let struct_fields_b = Fields::from(vec![Field::new("b", DataType::Int32, true)]); + let right_values = Arc::new(StructArray::new( + struct_fields_b.clone(), + vec![Arc::new(Int32Array::from_iter_values(100..105)) as ArrayRef], + None, + )); + let right_list = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(struct_fields_b), true)), + OffsetBuffer::::from_lengths([5]), + right_values, + None, + ); + + let target_item_field = Arc::new(Field::new( + "item", + DataType::Struct(Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ])), + true, + )); + let target_fields = Fields::from(vec![Field::new( + "items", + make_list_dtype(target_item_field), + true, + )]); + + let left_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "items", + sliced_left.data_type().clone(), + true, + )])), + vec![Arc::new(sliced_left) as ArrayRef], + ) + .unwrap(); + let right_batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "items", + right_list.data_type().clone(), + true, + )])), + vec![Arc::new(right_list) as ArrayRef], + ) + .unwrap(); + + let merged = left_batch + .merge_with_schema(&right_batch, &Schema::new(target_fields.to_vec())) + .unwrap(); + + let merged_list = merged + .column_by_name("items") + .unwrap() + .as_any() + .downcast_ref::>() + .unwrap(); + assert_eq!(merged_list.len(), 1); + assert_eq!(merged_list.value_length(0).as_usize(), 5); + let merged_struct = merged_list.values().as_struct(); + assert_eq!(merged_struct.num_columns(), 2); + let a = merged_struct + .column_by_name("a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + // After shifting offsets to zero, values 5..10 should be first. + let a_vals: Vec = a.iter().map(|v| v.unwrap()).collect(); + assert_eq!(a_vals, vec![5, 6, 7, 8, 9]); + } + #[test] fn test_project_by_schema_list_struct_reorder() { // Test that project_by_schema correctly reorders fields inside List @@ -2620,4 +2956,42 @@ mod tests { &Int32Array::from(vec![1, 2]) as &dyn Array ); } + + #[test] + fn test_iter_binary_array_accepts_binary_variants() { + let binary = BinaryArray::from(vec![b"a".as_slice(), b"bc"]); + assert_eq!( + iter_binary_array(&binary).unwrap().collect::>(), + vec![Some(b"a".as_slice()), Some(b"bc".as_slice())] + ); + + let large_binary = LargeBinaryArray::from(vec![b"x".as_slice(), b"yz"]); + assert_eq!( + iter_binary_array(&large_binary) + .unwrap() + .collect::>(), + vec![Some(b"x".as_slice()), Some(b"yz".as_slice())] + ); + + let binary_view = BinaryViewArray::from(vec![b"1".as_slice(), b"23"]); + assert_eq!( + iter_binary_array(&binary_view).unwrap().collect::>(), + vec![Some(b"1".as_slice()), Some(b"23".as_slice())] + ); + + let fixed_size = FixedSizeBinaryArray::from(vec![b"abcd", b"efgh"]); + assert_eq!( + iter_binary_array(&fixed_size).unwrap().collect::>(), + vec![Some(b"abcd".as_slice()), Some(b"efgh".as_slice())] + ); + } + + #[test] + fn test_iter_binary_array_rejects_non_binary() { + let int_array = Int32Array::from(vec![1, 2, 3]); + let Err(error) = iter_binary_array(&int_array) else { + panic!("expected an error for non-binary array"); + }; + assert!(error.to_string().contains("Expecting a binary type")); + } } diff --git a/rust/lance-arrow/src/list.rs b/rust/lance-arrow/src/list.rs index 0c24fc579da..06b0fc592cf 100644 --- a/rust/lance-arrow/src/list.rs +++ b/rust/lance-arrow/src/list.rs @@ -23,6 +23,16 @@ pub trait ListArrayExt { /// behaves similarly to `values()` except it slices the array so that it starts at /// the first list offset and ends at the last list offset. fn trimmed_values(&self) -> Arc; + /// The offset type of the underlying list array. + type Offset: OffsetSizeTrait; + /// Returns offsets shifted so the first offset is zero, matching + /// [`Self::trimmed_values`]. + /// + /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the + /// original values buffer, so combining them with trimmed values produces + /// offsets that exceed the values length. Use this together with + /// `trimmed_values` when constructing a new list array. + fn trimmed_offsets(&self) -> OffsetBuffer; } impl ListArrayExt for GenericListArray { @@ -90,6 +100,20 @@ impl ListArrayExt for GenericListArray .unwrap_or(0); self.values().slice(first_value, last_value - first_value) } + + type Offset = OffsetSize; + + fn trimmed_offsets(&self) -> OffsetBuffer { + let offsets = self.offsets(); + let Some(&first) = offsets.first() else { + return offsets.clone(); + }; + if first == OffsetSize::zero() { + return offsets.clone(); + } + let shifted: Vec = offsets.iter().map(|&o| o - first).collect(); + OffsetBuffer::new(ScalarBuffer::from(shifted)) + } } #[cfg(test)] diff --git a/rust/lance-arrow/src/scalar.rs b/rust/lance-arrow/src/scalar.rs index e9fd2516f17..6003eb13497 100644 --- a/rust/lance-arrow/src/scalar.rs +++ b/rust/lance-arrow/src/scalar.rs @@ -1,10 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use arrow_array::{ArrayRef, make_array}; +use arrow_array::{ArrayRef, UInt64Array, make_array}; use arrow_buffer::Buffer; -use arrow_data::{ArrayDataBuilder, transform::MutableArrayData}; +use arrow_data::ArrayDataBuilder; use arrow_schema::{ArrowError, DataType}; +use arrow_select::take::take; use crate::DataTypeExt; @@ -19,10 +20,7 @@ pub fn extract_scalar_value(array: &ArrayRef, idx: usize) -> Result { )); } - let data = array.to_data(); - let mut mutable = MutableArrayData::new(vec![&data], /*use_nulls=*/ true, 1); - mutable.extend(0, idx, idx + 1); - Ok(make_array(mutable.freeze())) + take(array.as_ref(), &UInt64Array::from(vec![idx as u64]), None) } fn read_u32(buf: &[u8], offset: &mut usize) -> Result { @@ -195,7 +193,10 @@ pub fn try_inline_value(scalar: &ArrayRef) -> Option> { mod tests { use std::sync::Arc; - use arrow_array::{BooleanArray, FixedSizeBinaryArray, Int32Array, StringArray, cast::AsArray}; + use arrow_array::{ + BooleanArray, DictionaryArray, FixedSizeBinaryArray, Int8Array, Int32Array, StringArray, + cast::AsArray, types::Int8Type, + }; use super::*; @@ -212,6 +213,24 @@ mod tests { ); } + #[test] + fn test_extract_scalar_value_from_full_dictionary() { + let values = Arc::new(StringArray::from( + (0..=i8::MAX) + .map(|value| format!("value-{value}")) + .collect::>(), + )); + let keys = Int8Array::from((0..=i8::MAX).collect::>()); + let array: ArrayRef = Arc::new(DictionaryArray::::new(keys, values)); + + let scalar = extract_scalar_value(&array, i8::MAX as usize).unwrap(); + + let scalar = scalar.as_dictionary::(); + assert_eq!(scalar.len(), 1); + assert_eq!(scalar.key(0), Some(i8::MAX as usize)); + assert_eq!(scalar.values().len(), i8::MAX as usize + 1); + } + #[test] fn test_scalar_value_buffer_utf8_round_trip() { let scalar: ArrayRef = Arc::new(StringArray::from(vec!["hello"])); diff --git a/rust/lance-core/Cargo.toml b/rust/lance-core/Cargo.toml index 7f956c70430..4f4e32168ff 100644 --- a/rust/lance-core/Cargo.toml +++ b/rust/lance-core/Cargo.toml @@ -18,16 +18,16 @@ arrow-data.workspace = true arrow-schema.workspace = true async-trait.workspace = true lance-arrow.workspace = true -byteorder.workspace = true +blake3.workspace = true bytes.workspace = true datafusion-common = { workspace = true, optional = true } datafusion-sql = { workspace = true, optional = true } lance-derive.workspace = true futures.workspace = true -itertools.workspace = true libc.workspace = true libm.workspace = true moka.workspace = true +quick_cache = "0.6" num_cpus = "1.0" object_store = { workspace = true } pin-project.workspace = true @@ -38,24 +38,27 @@ serde_json.workspace = true snafu.workspace = true tempfile.workspace = true tokio.workspace = true -tokio-stream.workspace = true tokio-util.workspace = true tracing.workspace = true twox-hash.workspace = true url.workspace = true log.workspace = true -# This is used to detect CPU features at runtime. -# See src/utils/cpu.rs -[target.'cfg(all(any(target_arch = "aarch64", target_arch = "loongarch64"), target_os = "linux"))'.dependencies] -libc = { version = "0.2" } - [dev-dependencies] -proptest.workspace = true +criterion.workspace = true rstest.workspace = true +tokio-stream.workspace = true [features] +# Capture Rust backtraces in error types. When disabled (the default), +# the backtrace field is zero-sized with no overhead. At runtime, capture +# is still gated by RUST_BACKTRACE=1. +backtrace = [] datafusion = ["dep:datafusion-common", "dep:datafusion-sql"] +[[bench]] +name = "cache_keys" +harness = false + [lints] workspace = true diff --git a/rust/lance-core/benches/cache_keys.rs b/rust/lance-core/benches/cache_keys.rs new file mode 100644 index 00000000000..10f3917f10e --- /dev/null +++ b/rust/lance-core/benches/cache_keys.rs @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::any::Any; +use std::borrow::Cow; +use std::hash::{BuildHasher, RandomState}; +use std::hint::black_box; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + +use async_trait::async_trait; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use futures::FutureExt; +use lance_core::cache::{ + CacheKey, CacheKeySchema, CacheNamespace, KeyBuilder, LanceCache, WeakLanceCache, +}; + +struct PageKey { + column_index: u32, + page_index: u64, +} + +impl CacheKey for PageKey { + type ValueType = Vec; + + fn key(&self) -> Cow<'_, str> { + format!("{}-{}", self.column_index, self.page_index).into() + } + + fn type_name() -> &'static str { + "bench.Page" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("bench.page-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.column_index); + builder.write_u64(self.page_index); + } +} + +#[derive(Clone, Eq, Hash, PartialEq)] +struct LegacyPhysicalKey { + namespace: Arc, + logical_key: Arc, + type_name: &'static str, +} + +type LegacyEntryValue = Arc; + +#[derive(Clone)] +struct LegacyEntry { + value: LegacyEntryValue, + size_bytes: usize, +} + +#[async_trait] +trait LegacyBackend: Send + Sync { + async fn get(&self, key: &LegacyPhysicalKey) -> Option; + async fn insert(&self, key: &LegacyPhysicalKey, value: LegacyEntryValue, size_bytes: usize); +} + +struct LegacyMokaBackend { + cache: moka::future::Cache, +} + +impl LegacyMokaBackend { + fn with_capacity(capacity: usize) -> Self { + let cache = moka::future::Cache::builder() + .max_capacity(capacity as u64) + .weigher(|key: &LegacyPhysicalKey, entry: &LegacyEntry| { + std::mem::size_of::() + .saturating_add(key.logical_key.len()) + .saturating_add(entry.size_bytes) + .try_into() + .unwrap_or(u32::MAX) + }) + .support_invalidation_closures() + .build(); + Self { cache } + } +} + +#[async_trait] +impl LegacyBackend for LegacyMokaBackend { + async fn get(&self, key: &LegacyPhysicalKey) -> Option { + self.cache.get(key).await.map(|entry| entry.value) + } + + async fn insert(&self, key: &LegacyPhysicalKey, value: LegacyEntryValue, size_bytes: usize) { + self.cache + .insert(key.clone(), LegacyEntry { value, size_bytes }) + .await; + } +} + +#[derive(Clone)] +struct LegacyCache { + backend: Arc, + namespace: Arc, + hits: Arc, + misses: Arc, +} + +impl LegacyCache { + fn with_capacity(capacity: usize) -> Self { + Self { + backend: Arc::new(LegacyMokaBackend::with_capacity(capacity)), + namespace: Arc::from(""), + hits: Arc::new(AtomicU64::new(0)), + misses: Arc::new(AtomicU64::new(0)), + } + } + + fn with_key_prefix(&self, segment: &str) -> Self { + Self { + backend: self.backend.clone(), + namespace: Arc::from(format!("{}{segment}/", self.namespace)), + hits: self.hits.clone(), + misses: self.misses.clone(), + } + } + + fn physical_key(&self, key: &PageKey) -> LegacyPhysicalKey { + let logical_key = key.key(); + LegacyPhysicalKey { + namespace: self.namespace.clone(), + logical_key: Arc::from(logical_key.as_ref()), + type_name: PageKey::type_name(), + } + } + + async fn insert(&self, key: &PageKey, value: Arc>) { + let size_bytes = std::mem::size_of::>() + + value.capacity() + + std::mem::size_of::() * 2; + self.backend + .insert(&self.physical_key(key), value, size_bytes) + .boxed() + .await; + } + + async fn get(&self, key: &PageKey) -> Option>> { + async { + let Some(value) = self.backend.get(&self.physical_key(key)).await else { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match value.downcast::>() { + Ok(value) => { + self.hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } + Err(_) => { + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } + .boxed() + .await + } +} + +struct LegacyWeakCache { + backend: Weak, + namespace: Arc, + hits: Arc, + misses: Arc, +} + +impl LegacyWeakCache { + fn from(cache: &LegacyCache) -> Self { + Self { + backend: Arc::downgrade(&cache.backend), + namespace: cache.namespace.clone(), + hits: cache.hits.clone(), + misses: cache.misses.clone(), + } + } + + async fn get(&self, key: &PageKey) -> Option>> { + let backend = self.backend.upgrade()?; + let logical_key = key.key(); + let physical_key = LegacyPhysicalKey { + namespace: self.namespace.clone(), + logical_key: Arc::from(logical_key.as_ref()), + type_name: PageKey::type_name(), + }; + let Some(value) = backend.get(&physical_key).await else { + self.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match value.downcast::>() { + Ok(value) => { + self.hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } + Err(_) => { + self.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } +} + +fn benchmark_key_preparation(c: &mut Criterion) { + let key = PageKey { + column_index: 17, + page_index: 42, + }; + let outer_hashes = RandomState::new(); + let prefixes: [(&str, Arc); 2] = [ + ("short", Arc::from("dataset")), + ("long", Arc::from("p".repeat(1024))), + ]; + let mut group = c.benchmark_group("cache_key_preparation"); + + for (case, prefix) in prefixes { + let namespace = CacheNamespace::root().child(&prefix); + + group.bench_with_input(BenchmarkId::new("legacy", case), &prefix, |b, prefix| { + b.iter(|| { + let logical_key = key.key(); + let physical_key = LegacyPhysicalKey { + namespace: Arc::clone(prefix), + logical_key: Arc::from(logical_key.as_ref()), + type_name: PageKey::type_name(), + }; + black_box(outer_hashes.hash_one(physical_key)) + }) + }); + + group.bench_function(BenchmarkId::new("blake3_typed", case), |b| { + b.iter(|| { + let mut builder = + KeyBuilder::new(namespace, PageKey::stable_type_id(), PageKey::schema()); + key.write_key(&mut builder); + black_box(outer_hashes.hash_one(builder.finish())) + }) + }); + } + + group.finish(); +} + +fn benchmark_namespace_derivation(c: &mut Criterion) { + let root = CacheNamespace::root(); + let long_segment = "p".repeat(160); + let mut group = c.benchmark_group("cache_namespace_derivation"); + + group.bench_function("root", |b| { + b.iter(|| black_box(CacheNamespace::root())); + }); + group.bench_with_input( + BenchmarkId::new("child", "short"), + &"dataset", + |b, segment| { + b.iter(|| black_box(root.child(black_box(segment)))); + }, + ); + group.bench_with_input( + BenchmarkId::new("child", "long"), + &long_segment, + |b, segment| { + b.iter(|| black_box(root.child(black_box(segment)))); + }, + ); + + group.finish(); +} + +fn benchmark_cache_operations(c: &mut Criterion) { + const CAPACITY: usize = 64 * 1024; + const ROTATING_KEYS: u64 = 512; + + let runtime = tokio::runtime::Runtime::new().unwrap(); + let prefix = "p".repeat(1024); + let legacy = LegacyCache::with_capacity(CAPACITY).with_key_prefix(&prefix); + let legacy_weak = LegacyWeakCache::from(&legacy); + let fixed = LanceCache::with_capacity(CAPACITY).with_key_prefix(&prefix); + let fixed_weak = WeakLanceCache::from(&fixed); + let hit_key = PageKey { + column_index: 17, + page_index: 42, + }; + runtime.block_on(async { + legacy.insert(&hit_key, Arc::new(vec![1_u8; 32])).await; + fixed + .insert_with_key(&hit_key, Arc::new(vec![1_u8; 32])) + .await; + }); + + let mut group = c.benchmark_group("cache_operations"); + group.bench_function(BenchmarkId::new("strong_warmed_hit", "legacy"), |b| { + b.to_async(&runtime) + .iter(|| legacy.get(black_box(&hit_key))); + }); + group.bench_function(BenchmarkId::new("strong_warmed_hit", "fixed"), |b| { + b.to_async(&runtime) + .iter(|| fixed.get_with_key(black_box(&hit_key))); + }); + group.bench_function(BenchmarkId::new("weak_warmed_hit", "legacy"), |b| { + b.to_async(&runtime) + .iter(|| legacy_weak.get(black_box(&hit_key))); + }); + group.bench_function(BenchmarkId::new("weak_warmed_hit", "fixed"), |b| { + b.to_async(&runtime) + .iter(|| fixed_weak.get_with_key(black_box(&hit_key))); + }); + + let values: Vec<_> = (0..16).map(|value| Arc::new(vec![value; 32])).collect(); + let next_legacy_insert = AtomicU64::new(0); + group.bench_function(BenchmarkId::new("bounded_rotating_insert", "legacy"), |b| { + b.to_async(&runtime).iter(|| { + let sequence = next_legacy_insert.fetch_add(1, Ordering::Relaxed); + let page_index = sequence % ROTATING_KEYS; + let value = Arc::clone(&values[sequence as usize % values.len()]); + let legacy = &legacy; + async move { + legacy + .insert( + &PageKey { + column_index: 17, + page_index, + }, + value, + ) + .await; + } + }); + }); + let next_fixed_insert = AtomicU64::new(0); + group.bench_function(BenchmarkId::new("bounded_rotating_insert", "fixed"), |b| { + b.to_async(&runtime).iter(|| { + let sequence = next_fixed_insert.fetch_add(1, Ordering::Relaxed); + let page_index = sequence % ROTATING_KEYS; + let value = Arc::clone(&values[sequence as usize % values.len()]); + let fixed = &fixed; + async move { + fixed + .insert_with_key( + &PageKey { + column_index: 17, + page_index, + }, + value, + ) + .await; + } + }); + }); + + group.finish(); +} + +criterion_group!( + benches, + benchmark_key_preparation, + benchmark_namespace_derivation, + benchmark_cache_operations +); +criterion_main!(benches); diff --git a/rust/lance-core/src/cache/backend.rs b/rust/lance-core/src/cache/backend.rs index 9307868f399..fdd612f1046 100644 --- a/rust/lance-core/src/cache/backend.rs +++ b/rust/lance-core/src/cache/backend.rs @@ -4,9 +4,31 @@ //! Backend interface for cache implementors. //! //! This module defines the trait that custom cache backends must implement, -//! along with the key and entry types they operate on. Most callers should +//! along with the entry type they operate on. Most callers should //! use [`LanceCache`](super::LanceCache) instead of interacting with //! backends directly. +//! +//! # Migrating custom backends +//! +//! Cache keys are opaque 16-byte values. Store +//! [`InternalCacheKey::as_bytes`] directly instead of decomposing a logical +//! prefix, key string, and Rust type name. The physical namespace must also +//! include [`CACHE_KEY_FORMAT`](super::CACHE_KEY_FORMAT), so a future key +//! protocol produces cold misses instead of aliases. Persistent or tiered +//! backends can route serializable values with [`CacheCodec::type_id`]. +//! +//! Prefix invalidation and key inventory are intentionally not part of this +//! interface: one-way digests cannot support either operation without +//! retaining the logical strings that fixed-size keys are designed to remove. +//! Existing callers should migrate removed symbols as follows: +//! - replace `with_backend_and_prefix(backend, prefix)` with +//! [`LanceCache::with_backend`](super::LanceCache::with_backend) followed by +//! [`LanceCache::with_key_prefix`](super::LanceCache::with_key_prefix); +//! - replace `invalidate_prefix` with [`LanceCache::clear`](super::LanceCache::clear) +//! when clearing the shared backend is acceptable, or rotate a versioned +//! namespace to leave older entries to age out; +//! - remove uses of `prefix`, `keys`, and session key-inventory methods; opaque +//! keys have no readable or enumerable equivalent. use std::any::Any; use std::pin::Pin; @@ -16,57 +38,13 @@ use async_trait::async_trait; use futures::Future; use crate::Result; +use crate::deepsize::Context; -use super::CacheCodec; +use super::{CacheCodec, InternalCacheKey}; /// A type-erased cache entry. pub type CacheEntry = Arc; -/// Iterator over cache keys currently known to a backend. -pub type CacheKeyIterator<'a> = Box + Send + 'a>; - -/// Structured cache key passed to [`CacheBackend`] methods. -/// -/// CacheBackend impls receive these ready-made from [`LanceCache`](super::LanceCache) -/// — you do not construct them yourself. Composed of three parts: -/// - **prefix**: scopes the key to a dataset or index (e.g. `"s3://bucket/dataset/"`) -/// - **key**: identifies the specific entry (e.g. `"42"` for a version number) -/// - **type_name**: distinguishes different value types stored under the same -/// user key (e.g. `"Vec"`) -#[derive(Clone, Debug, Hash, PartialEq, Eq)] -pub struct InternalCacheKey { - prefix: Arc, - key: Arc, - type_name: &'static str, -} - -impl InternalCacheKey { - pub fn new(prefix: Arc, key: Arc, type_name: &'static str) -> Self { - Self { - prefix, - key, - type_name, - } - } - - pub fn prefix(&self) -> &str { - &self.prefix - } - - pub fn key(&self) -> &str { - &self.key - } - - pub fn type_name(&self) -> &'static str { - self.type_name - } - - /// Returns true if this key's prefix starts with the given string. - pub fn starts_with(&self, prefix: &str) -> bool { - self.prefix.starts_with(prefix) - } -} - /// Low-level pluggable cache backend. /// /// Implementations store entries keyed by [`InternalCacheKey`] and return @@ -113,21 +91,9 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug { codec: Option, ) -> Result<(CacheEntry, bool)>; - /// Remove all entries whose prefix starts with the given string. - async fn invalidate_prefix(&self, prefix: &str); - /// Remove all entries. async fn clear(&self); - /// Return an iterator over cache keys currently known to this backend. - /// - /// Backends that cannot enumerate keys cheaply or accurately should return - /// `None`. An empty iterator means key inventory is supported and the - /// cache currently has no entries. - async fn keys(&self) -> Option> { - None - } - /// Number of entries currently stored (may flush pending operations). async fn num_entries(&self) -> usize; @@ -141,7 +107,7 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug { } /// Approximate weighted size in bytes, callable from synchronous contexts. - /// Used by `DeepSizeOf` to report cache memory usage. + /// Used as a `DeepSizeOf` fallback when exact entry traversal is unavailable. /// Backends that cannot provide this cheaply should return 0. /// /// Assumes entries do not share underlying buffers; if they do, the @@ -149,4 +115,23 @@ pub trait CacheBackend: Send + Sync + std::fmt::Debug { fn approx_size_bytes(&self) -> usize { 0 } + + /// Computes the size of the entries currently held in memory. + /// + /// `size_of_entry` threads a shared [`Context`] through each value so + /// allocations shared by multiple entries are counted once. It returns + /// `None` when the value's concrete type was not registered by + /// [`LanceCache`](super::LanceCache); implementations should use the + /// entry's declared eviction size as a fallback in that case. + /// + /// Backends that can enumerate their in-memory entries should include the + /// physical key footprint in the returned total. The default returns + /// `None`, causing `LanceCache` to use [`approx_size_bytes`](Self::approx_size_bytes). + fn deep_size_of_entries( + &self, + _context: &mut Context, + _size_of_entry: &dyn Fn(&CacheEntry, &mut Context) -> Option, + ) -> Option { + None + } } diff --git a/rust/lance-core/src/cache/backend_uri.rs b/rust/lance-core/src/cache/backend_uri.rs new file mode 100644 index 00000000000..33daa6adbe3 --- /dev/null +++ b/rust/lance-core/src/cache/backend_uri.rs @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! URI-based configuration for cache backends. +//! +//! [`build_from_uri`] parses a compact string form such as +//! `moka://?capacity=1073741824` into a [`BackendConfig`] and hands it to +//! the registry. This gives Python/Java bindings and configuration files a +//! single-string representation of a backend without having to expose a +//! typed builder for every backend. +//! +//! Grammar (intentionally a subset of RFC 3986 — Lance only needs a +//! predictable, unambiguous form): +//! +//! ```text +//! uri ::= scheme ":" hier ( "?" query )? +//! scheme ::= ALPHA ( ALPHA | DIGIT | "+" | "-" | "." )* +//! hier ::= "//" authority path? -- e.g. moka://?..., other:///path?... +//! | path -- e.g. moka:capacity=... (rare) +//! authority ::= *( any char except "/" | "?" ) +//! path ::= *( any char except "?" ) +//! query ::= pair ( "&" pair )* +//! pair ::= key "=" value -- both percent-decoded +//! ``` +//! +//! Mapping to [`BackendConfig`]: +//! +//! * `scheme` becomes `kind`. +//! * The joined `authority + path` (with any leading `//` stripped) is stored +//! under the option key `path` when non-empty. Empty-authority absolute +//! paths such as `backend:///tmp/cache` keep their leading `/`; host-style +//! paths such as `backend://localhost:6379/0` are stored as +//! `localhost:6379/0`. If the query already contains a `path` key, the +//! URI-supplied path wins and the parser errors on the conflict. +//! * Each `key=value` pair from the query becomes an entry in `options`. +//! Duplicate keys are rejected. + +use std::sync::Arc; + +use super::backend::CacheBackend; +use super::registry::{BackendConfig, build_from_config, normalize_backend_kind}; +use crate::{Error, Result}; + +/// Parse `uri` into a [`BackendConfig`] and build the backend registered +/// under its `scheme`. +/// +/// Returns an error if: +/// * the URI cannot be parsed, +/// * no backend is registered for that scheme, or +/// * the constructor itself fails. +pub fn build_from_uri(uri: &str) -> Result> { + let config = parse_backend_uri(uri)?; + build_from_config(&config) +} + +/// Parse `uri` into a [`BackendConfig`] without touching the registry. +/// +/// See the module docs for the accepted grammar. +pub fn parse_backend_uri(uri: &str) -> Result { + let (scheme, rest) = split_scheme(uri)?; + let (path, query) = split_path_query(rest); + + let mut config = BackendConfig::new(&scheme)?; + + let normalized_path = normalize_path(path); + if !normalized_path.is_empty() { + config.options.insert("path".to_string(), normalized_path); + } + + if let Some(query) = query { + for raw_pair in query.split('&') { + if raw_pair.is_empty() { + continue; + } + let (raw_key, raw_value) = raw_pair.split_once('=').ok_or_else(|| { + Error::invalid_input(format!( + "cache backend uri {:?}: query pair {:?} is missing '='", + uri, raw_pair + )) + })?; + let key = percent_decode(raw_key).map_err(|err| { + Error::invalid_input(format!( + "cache backend uri {:?}: cannot decode query key {:?}: {}", + uri, raw_key, err + )) + })?; + let value = percent_decode(raw_value).map_err(|err| { + Error::invalid_input(format!( + "cache backend uri {:?}: cannot decode query value {:?}: {}", + uri, raw_value, err + )) + })?; + if config.options.contains_key(&key) { + return Err(Error::invalid_input(format!( + "cache backend uri {:?}: option {:?} is set more than once", + uri, key + ))); + } + config.options.insert(key, value); + } + } + + Ok(config) +} + +fn split_scheme(uri: &str) -> Result<(String, &str)> { + let colon = uri.find(':').ok_or_else(|| { + Error::invalid_input(format!("cache backend uri {:?} is missing ':'", uri)) + })?; + let scheme = &uri[..colon]; + let scheme = normalize_backend_kind(scheme) + .map_err(|err| Error::invalid_input(format!("cache backend uri {:?}: {}", uri, err)))?; + Ok((scheme, &uri[colon + 1..])) +} + +fn split_path_query(rest: &str) -> (&str, Option<&str>) { + match rest.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (rest, None), + } +} + +/// Strip leading `//authority/` boilerplate and return the useful path +/// component. Empty authorities (e.g. `moka://`) yield an empty path, while +/// empty-authority absolute paths (e.g. `disk:///tmp/cache`) retain their +/// leading slash. +fn normalize_path(raw: &str) -> String { + let Some(without_marker) = raw.strip_prefix("//") else { + return raw.to_string(); + }; + if without_marker.is_empty() { + return String::new(); + } + without_marker.to_string() +} + +fn percent_decode(input: &str) -> std::result::Result { + let bytes = input.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'%' => { + if i + 2 >= bytes.len() { + return Err(format!("truncated percent-escape at offset {}", i)); + } + let hi = decode_hex_digit(bytes[i + 1])?; + let lo = decode_hex_digit(bytes[i + 2])?; + out.push((hi << 4) | lo); + i += 3; + } + b => { + out.push(b); + i += 1; + } + } + } + String::from_utf8(out).map_err(|err| err.to_string()) +} + +fn decode_hex_digit(b: u8) -> std::result::Result { + match b { + b'0'..=b'9' => Ok(b - b'0'), + b'a'..=b'f' => Ok(10 + b - b'a'), + b'A'..=b'F' => Ok(10 + b - b'A'), + _ => Err(format!( + "invalid hex digit {:?} in percent-escape", + b as char + )), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_authority_only() { + let cfg = parse_backend_uri("moka://?capacity=1073741824").unwrap(); + assert_eq!(cfg.kind, "moka"); + assert_eq!( + cfg.options.get("capacity").map(String::as_str), + Some("1073741824") + ); + assert!(!cfg.options.contains_key("path")); + } + + #[test] + fn test_parse_path_and_query() { + let cfg = parse_backend_uri("example:///var/lance/cache?capacity=10G").unwrap(); + assert_eq!(cfg.kind, "example"); + assert_eq!( + cfg.options.get("path").map(String::as_str), + Some("/var/lance/cache") + ); + assert_eq!(cfg.options.get("capacity").map(String::as_str), Some("10G")); + } + + #[test] + fn test_parse_host_style() { + // Redis-style URI with host:port + path segment. All of it lives + // under the "path" option; the backend is responsible for + // interpreting it. + let cfg = parse_backend_uri("redis://localhost:6379/0?prefix=lance").unwrap(); + assert_eq!(cfg.kind, "redis"); + assert_eq!( + cfg.options.get("path").map(String::as_str), + Some("localhost:6379/0"), + ); + assert_eq!(cfg.options.get("prefix").map(String::as_str), Some("lance")); + } + + #[test] + fn test_scheme_is_lowercased() { + // Different upper/lower cases must resolve to the same registry + // key, otherwise `Moka://` and `moka://` would look up different + // backends. + let cfg = parse_backend_uri("MOKA://?capacity=1").unwrap(); + assert_eq!(cfg.kind, "moka"); + } + + #[test] + fn test_percent_decoding() { + let cfg = parse_backend_uri("kv://?prefix=a%2Fb&name=hello%20world&token=a+b%2Bc").unwrap(); + assert_eq!(cfg.options.get("prefix").map(String::as_str), Some("a/b")); + assert_eq!( + cfg.options.get("name").map(String::as_str), + Some("hello world") + ); + assert_eq!(cfg.options.get("token").map(String::as_str), Some("a+b+c")); + } + + #[test] + fn test_empty_query_pair_is_skipped() { + // A trailing "&" should not cause a spurious "" pair to appear. + let cfg = parse_backend_uri("moka://?capacity=1&").unwrap(); + assert_eq!(cfg.options.len(), 1); + } + + #[test] + fn test_missing_scheme_errors() { + let err = parse_backend_uri("no-scheme-here").unwrap_err(); + assert!(err.to_string().contains("missing ':'")); + } + + #[test] + fn test_invalid_scheme_errors() { + // Digit-leading schemes are invalid per RFC 3986 and would clash + // with URI-like values elsewhere in the config. + let err = parse_backend_uri("1moka://").unwrap_err(); + assert!(err.to_string().contains("must start with an ASCII letter")); + } + + #[test] + fn test_duplicate_option_errors() { + let err = parse_backend_uri("moka://?capacity=1&capacity=2").unwrap_err(); + assert!(err.to_string().contains("more than once")); + } + + #[test] + fn test_query_pair_without_equals_errors() { + let err = parse_backend_uri("moka://?capacity").unwrap_err(); + assert!(err.to_string().contains("missing '='")); + } +} diff --git a/rust/lance-core/src/cache/codec.rs b/rust/lance-core/src/cache/codec.rs index bba54840829..eff1b5e2b31 100644 --- a/rust/lance-core/src/cache/codec.rs +++ b/rust/lance-core/src/cache/codec.rs @@ -306,6 +306,14 @@ impl CacheCodec { } } + /// Return the stable entry type identity. + /// + /// Persistent and tiered backends can use this metadata to route encoded + /// values without retaining a readable logical cache key. + pub const fn type_id(&self) -> &'static str { + self.type_id + } + /// Serialize `value` into `writer`: envelope first, then the body. pub fn serialize(&self, value: &ArcAny, writer: &mut dyn Write) -> Result<()> { let body_offset = write_envelope(writer, self.type_id, self.version)?; diff --git a/rust/lance-core/src/cache/entry_io.rs b/rust/lance-core/src/cache/entry_io.rs index fe91b11ca7d..47a6b6a9190 100644 --- a/rust/lance-core/src/cache/entry_io.rs +++ b/rust/lance-core/src/cache/entry_io.rs @@ -200,3 +200,193 @@ impl<'a> CacheEntryReader<'a> { self.data.slice(self.offset..) } } + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use arrow_array::{Int32Array, UInt64Array}; + use arrow_schema::{DataType, Field, Schema}; + use lance_arrow::ipc::IPC_SECTION_ALIGNMENT; + + /// Write a body starting at entry offset `pos` and return the bytes. + /// + /// `pos` models the envelope the [`CacheCodec`](super::CacheCodec) wrapper + /// writes ahead of the body; it only affects section alignment. + fn write_body(pos: usize, f: impl FnOnce(&mut CacheEntryWriter<'_>)) -> Bytes { + let mut buf = Vec::new(); + let mut writer = CacheEntryWriter::with_pos(&mut buf, pos); + f(&mut writer); + Bytes::from(buf) + } + + fn int_batch(values: Vec) -> RecordBatch { + let schema = Schema::new(vec![Field::new("i", DataType::Int32, false)]); + RecordBatch::try_new(schema.into(), vec![Arc::new(Int32Array::from(values))]).unwrap() + } + + #[test] + fn test_u8_roundtrip_and_truncation() { + let data = write_body(0, |w| { + w.write_u8(7).unwrap(); + w.write_u8(255).unwrap(); + }); + assert_eq!(data.as_ref(), &[7, 255]); + + let mut reader = CacheEntryReader::new(&data, 0, 1); + assert_eq!(reader.read_u8().unwrap(), 7); + assert_eq!(reader.read_u8().unwrap(), 255); + + // A third read has nothing left and must say so rather than wrap around. + let message = reader.read_u8().unwrap_err().to_string(); + assert!(message.contains("missing tag byte"), "{message}"); + } + + /// Headers are framed as `[len: u32 LE][bytes]`; `u64` stands in for a real + /// header proto here (prost encodes it as `google.protobuf.UInt64Value`). + #[test] + fn test_header_roundtrip_is_length_prefixed() { + let data = write_body(0, |w| w.write_header(&1234u64).unwrap()); + + let encoded_len = 1234u64.encoded_len(); + assert_eq!( + u32::from_le_bytes(data[..4].try_into().unwrap()) as usize, + encoded_len + ); + assert_eq!(data.len(), 4 + encoded_len); + + let mut reader = CacheEntryReader::new(&data, 0, 1); + assert_eq!(reader.read_header::().unwrap(), 1234); + // The reader consumed exactly the prefix plus the payload. + assert!(reader.body().is_empty()); + } + + #[test] + fn test_read_header_rejects_truncated_length_prefix() { + let data = Bytes::from_static(&[0, 0]); + let message = CacheEntryReader::new(&data, 0, 1) + .read_header::() + .unwrap_err() + .to_string(); + assert!(message.contains("truncated length prefix"), "{message}"); + } + + #[test] + fn test_read_header_rejects_truncated_body() { + // Prefix claims 16 payload bytes; only 3 follow. + let mut data = 16u32.to_le_bytes().to_vec(); + data.extend_from_slice(&[1, 2, 3]); + let data = Bytes::from(data); + + let message = CacheEntryReader::new(&data, 0, 1) + .read_header::() + .unwrap_err() + .to_string(); + assert!(message.contains("truncated body"), "{message}"); + } + + /// A length prefix that is in range but whose payload is not valid protobuf + /// must surface as a decode error, not a panic inside prost. + #[test] + fn test_read_header_rejects_undecodable_payload() { + // Field 1 tagged as a varint, then a varint that never terminates. + let payload = [0x08u8, 0xFF, 0xFF, 0xFF]; + let mut data = (payload.len() as u32).to_le_bytes().to_vec(); + data.extend_from_slice(&payload); + let data = Bytes::from(data); + + let message = CacheEntryReader::new(&data, 0, 1) + .read_header::() + .unwrap_err() + .to_string(); + assert!(message.contains("decode failed"), "{message}"); + } + + #[test] + fn test_raw_roundtrip_leaves_the_rest_as_body() { + let data = write_body(0, |w| { + w.write_raw(&[1, 2, 3]).unwrap(); + w.raw_writer().write_all(&[9, 9]).unwrap(); + }); + // 8-byte length prefix + 3 payload + 2 trailing. + assert_eq!(data.len(), 13); + + let mut reader = CacheEntryReader::new(&data, 0, 1); + assert_eq!(reader.read_raw().unwrap().as_ref(), &[1, 2, 3]); + assert_eq!(reader.body().as_ref(), &[9, 9]); + } + + #[test] + fn test_reader_exposes_the_entry_version() { + let data = Bytes::from_static(&[0]); + assert_eq!(CacheEntryReader::new(&data, 0, 7).version(), 7); + } + + /// The reason `pos` is tracked at all: an IPC section must begin on a + /// 64-byte boundary *of the whole entry*, so the envelope bytes ahead of the + /// body count toward the padding. Writer and reader have to agree on that, + /// and only a non-multiple-of-64 prefix makes a disagreement visible. + #[test] + fn test_ipc_section_is_aligned_against_the_envelope() { + const ENVELOPE: usize = 13; + let batch = int_batch(vec![1, 2, 3]); + + let mut buf = vec![0xAAu8; ENVELOPE]; + let mut writer = CacheEntryWriter::with_pos(&mut buf, ENVELOPE); + writer.write_header(&1234u64).unwrap(); + writer.write_ipc(&batch).unwrap(); + let data = Bytes::from(buf); + + let header_end = ENVELOPE + 4 + 1234u64.encoded_len(); + let stream_start = header_end.next_multiple_of(IPC_SECTION_ALIGNMENT); + assert!(stream_start > header_end, "padding should be non-empty"); + assert!( + data[header_end..stream_start].iter().all(|b| *b == 0), + "the gap must be zero padding" + ); + + let mut reader = CacheEntryReader::new(&data, ENVELOPE, 1); + assert_eq!(reader.read_header::().unwrap(), 1234); + assert_eq!(reader.read_ipc().unwrap(), batch); + assert!(reader.body().is_empty()); + } + + #[test] + fn test_ipc_batches_roundtrip() { + let batches = vec![int_batch(vec![1, 2]), int_batch(vec![3])]; + let data = write_body(0, |w| w.write_ipc_batches(batches.clone()).unwrap()); + + let mut reader = CacheEntryReader::new(&data, 0, 1); + assert_eq!(reader.read_ipc_batches().unwrap(), batches); + } + + /// The shape a real codec writes: a discriminant, a header, an arrow section + /// and a blob. Each reader step has to pick up exactly where the previous one + /// stopped, and the arrow section still has to land on its boundary even + /// though a `write_u8` moved the position by one. + #[test] + fn test_mixed_sections_stay_in_sync() { + let schema = Schema::new(vec![Field::new("u", DataType::UInt64, false)]); + let batch = RecordBatch::try_new( + schema.into(), + vec![Arc::new(UInt64Array::from(vec![u64::MAX, 0]))], + ) + .unwrap(); + + let data = write_body(0, |w| { + w.write_u8(2).unwrap(); + w.write_header(&99u64).unwrap(); + w.write_ipc(&batch).unwrap(); + w.write_raw(&[7, 7, 7]).unwrap(); + }); + + let mut reader = CacheEntryReader::new(&data, 0, 1); + assert_eq!(reader.read_u8().unwrap(), 2); + assert_eq!(reader.read_header::().unwrap(), 99); + assert_eq!(reader.read_ipc().unwrap(), batch); + assert_eq!(reader.read_raw().unwrap().as_ref(), &[7, 7, 7]); + assert!(reader.body().is_empty()); + } +} diff --git a/rust/lance-core/src/cache/key.rs b/rust/lance-core/src/cache/key.rs new file mode 100644 index 00000000000..b7687d26459 --- /dev/null +++ b/rust/lance-core/src/cache/key.rs @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Canonical fixed-size cache key construction. +//! +//! Cache keys are BLAKE3 digests truncated to 128 bits. Logical key fields are +//! encoded with explicit type tags, fixed-width little-endian integers, and +//! length framing for variable-width values. This makes the pre-hash encoding +//! unambiguous and stable across processes, platforms, and builds. +//! +//! The digest is a cache identity, not an authentication or access-control +//! primitive: namespace derivation keys are deterministic and not secret. +//! Truncating to 128 bits gives generic birthday resistance of approximately +//! 64 bits. This protocol does not introduce a FIPS mode; BLAKE3 is the +//! repository's selected cache-key algorithm. + +use std::fmt; + +/// Storage namespace identifier for canonical cache keys. +/// +/// Persistent backends should include this identifier in their physical +/// namespace so future algorithm or framing changes produce cold misses. +pub const CACHE_KEY_FORMAT: &str = "blake3-128-v1"; + +const KEY_FORMAT_VERSION: u32 = 1; +const NAMESPACE_CONTEXT: &str = "lance-format/lance 2026-07-17 cache namespace v1"; +const NAMESPACE_DOMAIN: &[u8] = b"lance-cache-namespace\0"; +const ENTRY_DOMAIN: &[u8] = b"lance-cache-entry\0"; + +/// One-byte type discriminants in the stable key encoding. +#[derive(Clone, Copy)] +#[repr(u8)] +enum FieldTag { + U8 = 1, + U16 = 2, + U32 = 3, + U64 = 4, + I32 = 5, + I64 = 6, + Bool = 7, + Str = 8, + Bytes = 9, + FixedBytes = 10, + None = 11, + Some = 12, + Variant = 13, + Sequence = 14, +} + +impl FieldTag { + const fn as_u8(self) -> u8 { + self as u8 + } +} + +/// Versioned schema identity for fields emitted by a cache key. +/// +/// Change the version whenever the encoded fields or their meaning changes. +/// The identifier must be stable and globally unique to the logical layout. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CacheKeySchema { + id: &'static str, + version: u32, +} + +impl CacheKeySchema { + /// Compatibility schema used by the default string-key bridge. + pub const LEGACY_TEXT: Self = Self::new("lance.cache.legacy-text", 1); + + /// Create a stable schema identifier and encoding version. + pub const fn new(id: &'static str, version: u32) -> Self { + Self { id, version } + } + + /// Return the author-assigned schema identifier. + pub const fn id(self) -> &'static str { + self.id + } + + /// Return the schema encoding version. + pub const fn version(self) -> u32 { + self.version + } +} + +/// Opaque 128-bit key passed to cache backends. +/// +/// The byte representation is canonical. It can be persisted directly and is +/// independent of the host's native integer endianness. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct InternalCacheKey([u8; 16]); + +impl InternalCacheKey { + /// Reconstruct a key from its canonical bytes. + pub const fn from_bytes(bytes: [u8; 16]) -> Self { + Self(bytes) + } + + /// Borrow the canonical byte representation. + pub const fn as_bytes(&self) -> &[u8; 16] { + &self.0 + } + + /// Consume the key and return its canonical bytes. + pub const fn into_bytes(self) -> [u8; 16] { + self.0 + } +} + +impl fmt::Debug for InternalCacheKey { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("InternalCacheKey(")?; + for byte in self.0 { + write!(f, "{byte:02x}")?; + } + f.write_str(")") + } +} + +/// Pre-derived namespace key shared by entries in one logical cache scope. +#[derive(Clone, Copy, Debug)] +pub struct CacheNamespace([u8; 32]); + +impl CacheNamespace { + /// Construct the stable root namespace. + pub fn root() -> Self { + Self(blake3::derive_key(NAMESPACE_CONTEXT, b"")) + } + + /// Derive a child namespace from one framed hierarchy segment. + pub fn child(self, segment: &str) -> Self { + let mut hasher = blake3::Hasher::new_keyed(&self.0); + write_framed(&mut hasher, NAMESPACE_DOMAIN); + hasher.update(&KEY_FORMAT_VERSION.to_le_bytes()); + write_framed(&mut hasher, segment.as_bytes()); + Self(hasher.finalize().into()) + } +} + +/// Streams typed logical fields into a canonical cache key. +/// +/// Integer methods use little-endian fixed-width encoding. Variable-width +/// strings and bytes are type-tagged and length-prefixed. There is deliberately +/// no `usize` method because cache identities must not depend on target width. +/// +/// # Examples +/// +/// ``` +/// use lance_core::cache::{CacheKeySchema, CacheNamespace, KeyBuilder}; +/// +/// let namespace = CacheNamespace::root().child("dataset"); +/// let mut builder = KeyBuilder::new( +/// namespace, +/// "example.Page", +/// CacheKeySchema::new("example.page-key", 1), +/// ); +/// builder.write_u32(7); +/// builder.write_str("values"); +/// let key = builder.finish(); +/// assert_eq!(key.as_bytes().len(), 16); +/// ``` +pub struct KeyBuilder { + hasher: blake3::Hasher, +} + +impl KeyBuilder { + /// Start a key in a namespace with a stable value type and key schema. + pub fn new( + namespace: CacheNamespace, + stable_type_id: &'static str, + schema: CacheKeySchema, + ) -> Self { + let mut hasher = blake3::Hasher::new_keyed(&namespace.0); + write_framed(&mut hasher, ENTRY_DOMAIN); + hasher.update(&KEY_FORMAT_VERSION.to_le_bytes()); + write_framed(&mut hasher, stable_type_id.as_bytes()); + write_framed(&mut hasher, schema.id().as_bytes()); + hasher.update(&schema.version().to_le_bytes()); + Self { hasher } + } + + /// Append a tagged, fixed-width `u8`. + #[inline] + pub fn write_u8(&mut self, value: u8) { + self.hasher.update(&[FieldTag::U8.as_u8(), value]); + } + + /// Append a tagged, little-endian `u16`. + #[inline] + pub fn write_u16(&mut self, value: u16) { + let mut encoded = [0; 3]; + encoded[0] = FieldTag::U16.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `u32`. + #[inline] + pub fn write_u32(&mut self, value: u32) { + let mut encoded = [0; 5]; + encoded[0] = FieldTag::U32.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `u64`. + #[inline] + pub fn write_u64(&mut self, value: u64) { + let mut encoded = [0; 9]; + encoded[0] = FieldTag::U64.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `i32`. + #[inline] + pub fn write_i32(&mut self, value: i32) { + let mut encoded = [0; 5]; + encoded[0] = FieldTag::I32.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged, little-endian `i64`. + #[inline] + pub fn write_i64(&mut self, value: i64) { + let mut encoded = [0; 9]; + encoded[0] = FieldTag::I64.as_u8(); + encoded[1..].copy_from_slice(&value.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append a tagged boolean. + #[inline] + pub fn write_bool(&mut self, value: bool) { + self.hasher + .update(&[FieldTag::Bool.as_u8(), u8::from(value)]); + } + + /// Append a tagged, length-prefixed UTF-8 string. + #[inline] + pub fn write_str(&mut self, value: &str) { + self.write_variable(FieldTag::Str, value.as_bytes()); + } + + /// Append tagged, length-prefixed bytes. + #[inline] + pub fn write_bytes(&mut self, value: &[u8]) { + self.write_variable(FieldTag::Bytes, value); + } + + /// Append a tagged fixed-size byte array, including its length. + #[inline] + pub fn write_fixed_bytes(&mut self, value: &[u8; N]) { + self.write_variable(FieldTag::FixedBytes, value); + } + + /// Append the canonical marker for an absent optional value. + #[inline] + pub fn write_none(&mut self) { + self.hasher.update(&[FieldTag::None.as_u8()]); + } + + /// Append the canonical marker for a present optional value. + #[inline] + pub fn write_some(&mut self) { + self.hasher.update(&[FieldTag::Some.as_u8()]); + } + + /// Append a tagged enum variant ordinal. + #[inline] + pub fn write_variant(&mut self, variant: u32) { + let mut encoded = [0; 5]; + encoded[0] = FieldTag::Variant.as_u8(); + encoded[1..].copy_from_slice(&variant.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Append the length of a following sequence. + #[inline] + pub fn write_sequence_len(&mut self, len: u64) { + let mut encoded = [0; 9]; + encoded[0] = FieldTag::Sequence.as_u8(); + encoded[1..].copy_from_slice(&len.to_le_bytes()); + self.hasher.update(&encoded); + } + + /// Finalize and return the canonical 128-bit key. + #[inline] + pub fn finish(self) -> InternalCacheKey { + let hash = self.hasher.finalize(); + let mut bytes = [0; 16]; + bytes.copy_from_slice(&hash.as_bytes()[..16]); + InternalCacheKey(bytes) + } + + #[inline] + fn write_variable(&mut self, tag: FieldTag, value: &[u8]) { + self.hasher.update(&[tag.as_u8()]); + self.hasher.update(&encoded_len(value)); + self.hasher.update(value); + } +} + +#[inline] +fn write_framed(hasher: &mut blake3::Hasher, value: &[u8]) { + hasher.update(&encoded_len(value)); + hasher.update(value); +} + +#[inline] +fn encoded_len(value: &[u8]) -> [u8; 8] { + (value.len() as u64).to_le_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + const SCHEMA: CacheKeySchema = CacheKeySchema::new("test.key", 1); + + fn builder() -> KeyBuilder { + KeyBuilder::new( + CacheNamespace::root().child("s3://bucket/dataset"), + "test.Value", + SCHEMA, + ) + } + + fn key_with(write: impl FnOnce(&mut KeyBuilder)) -> InternalCacheKey { + let mut key = builder(); + write(&mut key); + key.finish() + } + + #[test] + fn key_and_namespace_have_fixed_sizes() { + assert_eq!(std::mem::size_of::(), 16); + assert_eq!(std::mem::size_of::(), 32); + assert_eq!(std::mem::size_of::(), 1); + } + + #[test] + fn blake3_matches_official_empty_keyed_hash_vector() { + let key = *b"whats the Elvish word for friend"; + assert_eq!( + blake3::keyed_hash(&key, b"").as_bytes(), + &[ + 0x92, 0xb2, 0xb7, 0x56, 0x04, 0xed, 0x3c, 0x76, 0x1f, 0x9d, 0x6f, 0x62, 0x39, 0x2c, + 0x8a, 0x92, 0x27, 0xad, 0x0e, 0xa3, 0xf0, 0x95, 0x73, 0xe7, 0x83, 0xf1, 0x49, 0x8a, + 0x4e, 0xd6, 0x0d, 0x26, + ] + ); + } + + #[test] + fn typed_fields_and_boundaries_are_unambiguous() { + let cases = [ + key_with(|key| { + key.write_str("ab"); + key.write_str("c"); + }), + key_with(|key| { + key.write_str("a"); + key.write_str("bc"); + }), + key_with(|key| key.write_str("")), + key_with(|key| key.write_bytes(b"")), + key_with(|key| key.write_fixed_bytes(b"")), + key_with(|key| key.write_u8(1)), + key_with(|key| key.write_u16(1)), + key_with(|key| key.write_u32(1)), + key_with(|key| key.write_u64(1)), + key_with(|key| key.write_i32(1)), + key_with(|key| key.write_i64(1)), + key_with(|key| key.write_bool(false)), + key_with(|key| key.write_bool(true)), + key_with(KeyBuilder::write_none), + key_with(KeyBuilder::write_some), + key_with(|key| key.write_variant(0)), + key_with(|key| key.write_variant(1)), + ]; + assert_eq!(std::collections::BTreeSet::from(cases).len(), cases.len()); + + assert_ne!( + key_with(|key| { + key.write_sequence_len(2); + key.write_u32(1); + key.write_u32(2); + }), + key_with(|key| { + key.write_u32(1); + key.write_u32(2); + }) + ); + } + + #[test] + fn namespace_type_schema_and_version_are_domain_separated() { + let root = CacheNamespace::root(); + let namespace = root.child("dataset").child("index"); + let nested = KeyBuilder::new(namespace, "test.Value", SCHEMA).finish(); + let combined = KeyBuilder::new(root.child("dataset/index"), "test.Value", SCHEMA).finish(); + assert_ne!(nested, combined); + + assert_ne!( + nested, + KeyBuilder::new(namespace, "test.OtherValue", SCHEMA).finish() + ); + assert_ne!( + nested, + KeyBuilder::new( + namespace, + "test.Value", + CacheKeySchema::new("test.other-key", 1), + ) + .finish() + ); + assert_ne!( + nested, + KeyBuilder::new(namespace, "test.Value", CacheKeySchema::new("test.key", 2),).finish() + ); + + let tenant_a_memory = + KeyBuilder::new(root.child("tenant-a").child("memory"), "test.Value", SCHEMA).finish(); + assert_ne!( + tenant_a_memory, + KeyBuilder::new(root.child("tenant-b").child("memory"), "test.Value", SCHEMA,).finish() + ); + assert_ne!( + tenant_a_memory, + KeyBuilder::new( + root.child("tenant-a").child("persistent"), + "test.Value", + SCHEMA, + ) + .finish() + ); + } + + #[test] + fn integers_use_fixed_width_little_endian_encoding() { + let namespace = CacheNamespace::root().child("endianness"); + let mut key = KeyBuilder::new(namespace, "test.Value", SCHEMA); + key.write_u32(0x0102_0304); + let actual = key.finish(); + + let mut reference = blake3::Hasher::new_keyed(&namespace.0); + write_framed(&mut reference, ENTRY_DOMAIN); + reference.update(&KEY_FORMAT_VERSION.to_le_bytes()); + write_framed(&mut reference, b"test.Value"); + write_framed(&mut reference, SCHEMA.id().as_bytes()); + reference.update(&SCHEMA.version().to_le_bytes()); + reference.update(&[FieldTag::U32.as_u8(), 0x04, 0x03, 0x02, 0x01]); + let mut expected = [0; 16]; + expected.copy_from_slice(&reference.finalize().as_bytes()[..16]); + + assert_eq!(actual, InternalCacheKey::from_bytes(expected)); + } + + #[test] + fn key_has_stable_golden_vector() { + let mut key = builder(); + key.write_u32(7); + key.write_str("page"); + key.write_some(); + key.write_fixed_bytes(&[0xAB; 16]); + assert_eq!( + key.finish().into_bytes(), + [ + 0xc4, 0x38, 0xff, 0x22, 0x30, 0x55, 0x30, 0xfc, 0x74, 0x16, 0x38, 0xe9, 0x7d, 0x45, + 0xa5, 0x68, + ] + ); + } +} diff --git a/rust/lance-core/src/cache/mod.rs b/rust/lance-core/src/cache/mod.rs index 4f93f261bea..44b5604bd1a 100644 --- a/rust/lance-core/src/cache/mod.rs +++ b/rust/lance-core/src/cache/mod.rs @@ -31,9 +31,9 @@ //! ## For backend implementors //! //! Implement [`CacheBackend`] to provide a custom storage layer (disk, Redis, -//! etc.). Backends receive [`InternalCacheKey`] keys and type-erased -//! [`CacheEntry`] values — the typed wrapping is handled by [`LanceCache`]. -//! See the [`backend`] module for details. +//! etc.). Backends receive opaque, fixed-size [`InternalCacheKey`] values and +//! type-erased [`CacheEntry`] values. The typed wrapping is handled by +//! [`LanceCache`]. See the [`backend`] module for migration details. //! //! ## Serialization flow //! @@ -46,26 +46,36 @@ //! `codec.deserialize(reader)` on get to persist entries across restarts. pub mod backend; +mod backend_uri; pub mod codec; mod entry_io; +mod key; mod moka; +mod quick; +mod registry; -pub use backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey}; +pub use backend::{CacheBackend, CacheEntry}; +pub use backend_uri::{build_from_uri, parse_backend_uri}; pub use codec::{ CacheCodec, CacheCodecImpl, CacheDecode, CacheMissReason, MAGIC, has_cache_envelope, }; pub use entry_io::{CacheEntryReader, CacheEntryWriter}; +pub use key::{CACHE_KEY_FORMAT, CacheKeySchema, CacheNamespace, InternalCacheKey, KeyBuilder}; pub use moka::MokaCacheBackend; +pub use quick::{QuickCacheBackend, recommended_cache_shards}; +pub use registry::{BackendBuildFn, BackendConfig, build_from_config, register_backend}; +use std::any::TypeId; use std::borrow::Cow; +use std::collections::HashMap; use std::sync::{ - Arc, + Arc, RwLock, Weak, atomic::{AtomicU64, Ordering}, }; -use futures::{Future, FutureExt}; +use futures::Future; -use crate::Result; +use crate::{Error, Result}; pub use crate::deepsize::{Context, DeepSizeOf}; @@ -75,9 +85,10 @@ pub use crate::deepsize::{Context, DeepSizeOf}; /// Typed cache key for sized value types. /// -/// Implement this trait to define a new type of cached entry. [`LanceCache`] -/// uses the key string and type name to construct an [`InternalCacheKey`] -/// for the backend. +/// Existing implementations can continue returning a logical string from +/// [`key`](Self::key). Performance-sensitive implementations should also +/// provide a stable schema and stream typed fields through +/// [`write_key`](Self::write_key), avoiding construction of that string. /// /// # Example /// @@ -98,14 +109,34 @@ pub trait CacheKey { /// Short, stable string identifying this value type. /// /// Two `CacheKey` impls that store different `ValueType`s **must** return - /// different type names; if they collide, gets will silently return `None` - /// due to failed downcasts. + /// different type names. /// /// Use a short literal (e.g. `"Vec"`), not /// `std::any::type_name` — the latter is not guaranteed stable across /// compiler versions or build configurations. fn type_name() -> &'static str; + /// Stable identity included in the physical key. + /// + /// The compatibility default preserves existing implementations by using + /// their author-assigned [`type_name`](Self::type_name). + fn stable_type_id() -> &'static str { + Self::type_name() + } + + /// Versioned schema for the logical key fields. + fn schema() -> CacheKeySchema { + CacheKeySchema::LEGACY_TEXT + } + + /// Stream the logical key fields into the canonical key builder. + /// + /// The compatibility default hashes the existing string key. In-tree hot + /// paths override this with typed, allocation-free field encoding. + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.key().as_ref()); + } + /// Optional codec for serializing/deserializing this key's value type. /// /// Returns `None` by default. Cache backends that support persistence @@ -134,6 +165,21 @@ pub trait UnsizedCacheKey { /// Short, stable string identifying this value type. /// See [`CacheKey::type_name`] for requirements. fn type_name() -> &'static str; + + /// Stable identity included in the physical key. + fn stable_type_id() -> &'static str { + Self::type_name() + } + + /// Versioned schema for the logical key fields. + fn schema() -> CacheKeySchema { + CacheKeySchema::LEGACY_TEXT + } + + /// Stream the logical key fields into the canonical key builder. + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.key().as_ref()); + } } // --------------------------------------------------------------------------- @@ -145,10 +191,61 @@ fn cache_entry_size(value: &T) -> usize { value.deep_size_of() + std::mem::size_of::() * 2 } -/// Build an [`InternalCacheKey`] from a cache's prefix, a user key string, -/// and a type name. -fn build_key(prefix: &Arc, key: &str, type_name: &'static str) -> InternalCacheKey { - InternalCacheKey::new(prefix.clone(), Arc::from(key), type_name) +type CacheEntrySizeAccessor = fn(&CacheEntry, &mut Context) -> Option; + +fn cache_entry_size_with_context(entry: &CacheEntry, context: &mut Context) -> Option +where + T: DeepSizeOf + Send + Sync + 'static, +{ + let value = entry.downcast_ref::()?; + let entry_ptr = Arc::as_ptr(entry) as *const () as usize; + if !context.mark_seen(entry_ptr) { + return Some(0); + } + Some( + std::mem::size_of_val(value) + + value.deep_size_of_children(context) + + std::mem::size_of::() * 2, + ) +} + +#[derive(Debug)] +struct CacheState { + backend: Arc, + hits: AtomicU64, + misses: AtomicU64, + entry_size_accessors: RwLock>, +} + +impl CacheState { + fn new(backend: Arc) -> Self { + Self { + backend, + hits: AtomicU64::new(0), + misses: AtomicU64::new(0), + entry_size_accessors: RwLock::new(HashMap::new()), + } + } + + fn entry_size(&self, value: &T) -> usize + where + T: DeepSizeOf + Send + Sync + 'static, + { + let type_id = TypeId::of::(); + let is_registered = self + .entry_size_accessors + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains_key(&type_id); + if !is_registered { + self.entry_size_accessors + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .entry(type_id) + .or_insert(cache_entry_size_with_context::); + } + cache_entry_size(value) + } } // --------------------------------------------------------------------------- @@ -161,184 +258,98 @@ fn build_key(prefix: &Arc, key: &str, type_name: &'static str) -> InternalC /// [`MokaCacheBackend`]; pass a custom backend via [`LanceCache::with_backend`]. #[derive(Clone)] pub struct LanceCache { - cache: Arc, - prefix: Arc, - hits: Arc, - misses: Arc, + state: Arc, + namespace: key::CacheNamespace, } impl std::fmt::Debug for LanceCache { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LanceCache") - .field("cache", &self.cache) - .finish() + .field("backend", &self.state.backend) + .finish_non_exhaustive() } } impl DeepSizeOf for LanceCache { - fn deep_size_of_children(&self, _: &mut Context) -> usize { - self.cache.approx_size_bytes() + fn deep_size_of_children(&self, context: &mut Context) -> usize { + let state_ptr = Arc::as_ptr(&self.state) as usize; + if !context.mark_seen(state_ptr) { + return 0; + } + + let accessors = self + .state + .entry_size_accessors + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + self.state + .backend + .deep_size_of_entries(context, &|entry, context| { + accessors + .get(&entry.as_ref().type_id()) + .and_then(|size_of_entry| size_of_entry(entry, context)) + }) + .unwrap_or_else(|| self.state.backend.approx_size_bytes()) } } impl LanceCache { pub fn with_capacity(capacity: usize) -> Self { - Self { - cache: Arc::new(MokaCacheBackend::with_capacity(capacity)), - prefix: Arc::from(""), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), - } + Self::with_backend(Arc::new(MokaCacheBackend::with_capacity(capacity))) } /// Create a cache backed by a custom [`CacheBackend`]. pub fn with_backend(backend: Arc) -> Self { Self { - cache: backend, - prefix: Arc::from(""), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), + state: Arc::new(CacheState::new(backend)), + namespace: key::CacheNamespace::root(), } } pub fn no_cache() -> Self { - Self { - cache: Arc::new(MokaCacheBackend::no_cache()), - prefix: Arc::from(""), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), - } + Self::with_backend(Arc::new(MokaCacheBackend::no_cache())) } - /// Create a cache with the given backend and an exact prefix string. - /// Unlike `with_key_prefix`, this sets the prefix verbatim (no trailing slash added). - pub fn with_backend_and_prefix(backend: Arc, prefix: String) -> Self { - Self { - cache: backend, - prefix: Arc::from(prefix), - hits: Arc::new(AtomicU64::new(0)), - misses: Arc::new(AtomicU64::new(0)), - } - } - - /// Appends a prefix to the cache key. + /// Derive a child namespace for all keys in the returned cache handle. + /// + /// Each call adds one framed hierarchy segment. Consequently, + /// `cache.with_key_prefix("a").with_key_prefix("b")` is deliberately + /// distinct from `cache.with_key_prefix("a/b")`. pub fn with_key_prefix(&self, prefix: &str) -> Self { Self { - cache: self.cache.clone(), - prefix: Arc::from(format!("{}{}/", self.prefix, prefix)), - hits: self.hits.clone(), - misses: self.misses.clone(), + state: self.state.clone(), + namespace: self.namespace.child(prefix), } } - /// Invalidate all entries whose prefix starts with the given string. - pub async fn invalidate_prefix(&self, prefix: &str) { - let full_prefix = format!("{}{}", self.prefix, prefix); - self.cache.invalidate_prefix(&full_prefix).await; - } - pub async fn size(&self) -> usize { - self.cache.num_entries().await + self.state.backend.num_entries().await } pub fn approx_size(&self) -> usize { - self.cache.approx_num_entries() + self.state.backend.approx_num_entries() } pub async fn size_bytes(&self) -> usize { - self.cache.size_bytes().await - } - - /// Return an iterator over keys currently stored under this cache's prefix. - /// - /// Returns `None` when the backend does not support key inventory. The - /// iterator is intended for diagnostics and may be weakly consistent with - /// concurrent cache mutations. - /// - /// # Examples - /// - /// ``` - /// # use std::{borrow::Cow, sync::Arc}; - /// # use lance_core::cache::{CacheKey, LanceCache}; - /// # struct MyKey; - /// # impl CacheKey for MyKey { - /// # type ValueType = Vec; - /// # fn key(&self) -> Cow<'_, str> { Cow::Borrowed("my-key") } - /// # fn type_name() -> &'static str { "VecI32" } - /// # } - /// # async fn example() { - /// let cache = LanceCache::with_capacity(1024); - /// cache.insert_with_key(&MyKey, Arc::new(vec![1, 2, 3])).await; - /// - /// let mut keys = cache.keys().await.expect("Moka supports key inventory"); - /// assert_eq!(keys.next().unwrap().key(), "my-key"); - /// # } - /// ``` - pub async fn keys(&self) -> Option> { - Some(Box::new( - self.cache - .keys() - .await? - .filter(|key| key.starts_with(&self.prefix)), - )) - } - - // -- Sized insert/get (internal, shared by sized and unsized paths) -------- - - async fn insert_with_id( - &self, - key: &str, - type_name: &'static str, - codec: Option, - metadata: Arc, - ) { - let size = cache_entry_size(&*metadata); - let cache_key = build_key(&self.prefix, key, type_name); - self.cache.insert(&cache_key, metadata, size, codec).await; - } - - async fn get_with_id( - &self, - key: &str, - type_name: &'static str, - codec: Option, - ) -> Option> { - let cache_key = build_key(&self.prefix, key, type_name); - if let Some(entry) = self.cache.get(&cache_key, codec).await { - match entry.downcast::() { - Ok(val) => { - self.hits.fetch_add(1, Ordering::Relaxed); - Some(val) - } - Err(_) => { - // Type mismatch: the backend returned a different concrete - // type than expected (e.g. a disk cache may store - // intermediate state). Treat as a miss. - self.misses.fetch_add(1, Ordering::Relaxed); - None - } - } - } else { - self.misses.fetch_add(1, Ordering::Relaxed); - None - } + self.state.backend.size_bytes().await } // -- Stats / clear -------------------------------------------------------- pub async fn stats(&self) -> CacheStats { CacheStats { - hits: self.hits.load(Ordering::Relaxed), - misses: self.misses.load(Ordering::Relaxed), - num_entries: self.cache.num_entries().await, - size_bytes: self.cache.size_bytes().await, + hits: self.state.hits.load(Ordering::Relaxed), + misses: self.state.misses.load(Ordering::Relaxed), + num_entries: self.state.backend.num_entries().await, + size_bytes: self.state.backend.size_bytes().await, } } pub async fn clear(&self) { - self.cache.clear().await; - self.hits.store(0, Ordering::Relaxed); - self.misses.store(0, Ordering::Relaxed); + self.state.backend.clear().await; + self.state.hits.store(0, Ordering::Relaxed); + self.state.misses.store(0, Ordering::Relaxed); } // -- CacheKey-based methods ----------------------------------------------- @@ -348,9 +359,12 @@ impl LanceCache { K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - self.insert_with_id(&cache_key.key(), K::type_name(), K::codec(), metadata) - .boxed() - .await + let size = self.state.entry_size(metadata.as_ref()); + let key = self.sized_key(cache_key); + self.state + .backend + .insert(&key, metadata, size, K::codec()) + .await; } pub async fn get_with_key(&self, cache_key: &K) -> Option> @@ -358,9 +372,28 @@ impl LanceCache { K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - self.get_with_id::(&cache_key.key(), K::type_name(), K::codec()) - .boxed() - .await + let key = self.sized_key(cache_key); + let Some(entry) = self.state.backend.get(&key, K::codec()).await else { + self.state.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match entry.downcast::() { + Ok(value) => { + self.state.hits.fetch_add(1, Ordering::Relaxed); + Some(value) + } + Err(_) => { + // Type mismatch: the backend returned a different concrete + // type than expected (e.g. a disk cache may store + // intermediate state). Treat as a miss. + log::warn!( + "cache backend returned a value with the wrong concrete type for key type {:?}", + K::stable_type_id() + ); + self.state.misses.fetch_add(1, Ordering::Relaxed); + None + } + } } pub async fn get_or_insert_with_key( @@ -374,27 +407,63 @@ impl LanceCache { F: FnOnce() -> Fut + Send, Fut: Future> + Send, { - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); + self.get_or_insert_with_key_hit(cache_key, loader) + .await + .map(|(value, _)| value) + } + /// Same as [`get_or_insert_with_key`](Self::get_or_insert_with_key), but + /// also returns a boolean indicating whether the loader was skipped for + /// this call. + /// + /// - `true` means this call did **not** execute the loader. That covers + /// both a true cache hit on an already-populated entry and a coalesced + /// concurrent load where an in-flight loader started by a different + /// caller produced the value. + /// - `false` means the loader ran on this call (a real cache miss). + /// + /// Callers that want strict "served from cache" semantics should treat + /// coalesced loads as misses; the current backend does not distinguish the + /// two cases. Prefer this over rolling a caller-side `Arc` + /// when the caller needs per-query hit/miss counters — the backend already + /// tracks this bit internally and this method just exposes it. + pub async fn get_or_insert_with_key_hit( + &self, + cache_key: K, + loader: F, + ) -> Result<(Arc, bool)> + where + K: CacheKey, + K::ValueType: DeepSizeOf + Send + Sync + 'static, + F: FnOnce() -> Fut + Send, + Fut: Future> + Send, + { + let key = self.sized_key(&cache_key); + let state = self.state.clone(); let typed_loader = Box::pin(async move { - let value = loader().await?; - let arc = Arc::new(value); - let size = cache_entry_size(&*arc); - Ok((arc as CacheEntry, size)) + let value = Arc::new(loader().await?); + let size = state.entry_size(value.as_ref()); + Ok((value as CacheEntry, size)) }); let (entry, was_cached) = self - .cache + .state + .backend .get_or_insert(&key, typed_loader, K::codec()) .await?; - + let entry = entry.downcast::().map_err(|_| { + self.state.misses.fetch_add(1, Ordering::Relaxed); + Error::io(format!( + "cache backend returned a value with the wrong concrete type for key type {:?}", + K::stable_type_id() + )) + })?; if was_cached { - self.hits.fetch_add(1, Ordering::Relaxed); + self.state.hits.fetch_add(1, Ordering::Relaxed); } else { - self.misses.fetch_add(1, Ordering::Relaxed); + self.state.misses.fetch_add(1, Ordering::Relaxed); } - - Ok(entry.downcast::().unwrap()) + Ok((entry, was_cached)) } pub async fn insert_unsized_with_key(&self, cache_key: &K, metadata: Arc) @@ -402,21 +471,90 @@ impl LanceCache { K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - self.insert_with_id(&cache_key.key(), K::type_name(), None, Arc::new(metadata)) - .boxed() - .await + let metadata = Arc::new(metadata); + let size = self.state.entry_size(metadata.as_ref()); + let key = self.unsized_key(cache_key); + self.state.backend.insert(&key, metadata, size, None).await; } - pub async fn get_unsized_with_key(&self, cache_key: &K) -> Option> + pub async fn get_or_insert_unsized_with_key( + &self, + cache_key: K, + loader: F, + ) -> Result> where K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, + F: FnOnce() -> Fut + Send, + Fut: Future>> + Send, { - let outer = self - .get_with_id::>(&cache_key.key(), K::type_name(), None) - .boxed() + let key = self.unsized_key(&cache_key); + let state = self.state.clone(); + let typed_loader = Box::pin(async move { + let value = loader().await?; + let size = state.entry_size(&value); + Ok((Arc::new(value) as CacheEntry, size)) + }); + + let (entry, was_cached) = self + .state + .backend + .get_or_insert(&key, typed_loader, None) .await?; - Some(outer.as_ref().clone()) + let entry = entry.downcast::>().map_err(|_| { + self.state.misses.fetch_add(1, Ordering::Relaxed); + Error::io(format!( + "cache backend returned a value with the wrong concrete type for unsized key type {:?}", + K::stable_type_id() + )) + })?; + if was_cached { + self.state.hits.fetch_add(1, Ordering::Relaxed); + } else { + self.state.misses.fetch_add(1, Ordering::Relaxed); + } + Ok(entry.as_ref().clone()) + } + + pub async fn get_unsized_with_key(&self, cache_key: &K) -> Option> + where + K: UnsizedCacheKey, + K::ValueType: DeepSizeOf + Send + Sync + 'static, + { + let key = self.unsized_key(cache_key); + let Some(entry) = self.state.backend.get(&key, None).await else { + self.state.misses.fetch_add(1, Ordering::Relaxed); + return None; + }; + match entry.downcast::>() { + Ok(value) => { + self.state.hits.fetch_add(1, Ordering::Relaxed); + Some(value.as_ref().clone()) + } + Err(_) => { + // Type mismatch: the backend returned a different concrete + // type than expected (e.g. a disk cache may store + // intermediate state). Treat as a miss. + log::warn!( + "cache backend returned a value with the wrong concrete type for unsized key type {:?}", + K::stable_type_id() + ); + self.state.misses.fetch_add(1, Ordering::Relaxed); + None + } + } + } + + fn sized_key(&self, cache_key: &K) -> InternalCacheKey { + let mut builder = KeyBuilder::new(self.namespace, K::stable_type_id(), K::schema()); + cache_key.write_key(&mut builder); + builder.finish() + } + + fn unsized_key(&self, cache_key: &K) -> InternalCacheKey { + let mut builder = KeyBuilder::new(self.namespace, K::stable_type_id(), K::schema()); + cache_key.write_key(&mut builder); + builder.finish() } } @@ -428,50 +566,31 @@ impl LanceCache { /// When the original cache is dropped, operations on this will gracefully no-op. #[derive(Clone, Debug)] pub struct WeakLanceCache { - inner: std::sync::Weak, - prefix: Arc, - hits: Arc, - misses: Arc, + state: Weak, + namespace: key::CacheNamespace, } impl WeakLanceCache { pub fn from(cache: &LanceCache) -> Self { Self { - inner: Arc::downgrade(&cache.cache), - prefix: cache.prefix.clone(), - hits: cache.hits.clone(), - misses: cache.misses.clone(), + state: Arc::downgrade(&cache.state), + namespace: cache.namespace, } } pub fn with_key_prefix(&self, prefix: &str) -> Self { Self { - inner: self.inner.clone(), - prefix: Arc::from(format!("{}{}/", self.prefix, prefix)), - hits: self.hits.clone(), - misses: self.misses.clone(), + state: self.state.clone(), + namespace: self.namespace.child(prefix), } } - /// The key prefix used for all entries in this cache. - pub fn prefix(&self) -> &str { - &self.prefix - } - pub async fn get_with_key(&self, cache_key: &K) -> Option> where K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - let cache = self.inner.upgrade()?; - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - if let Some(entry) = cache.get(&key, K::codec()).await { - self.hits.fetch_add(1, Ordering::Relaxed); - Some(entry.downcast::().unwrap()) - } else { - self.misses.fetch_add(1, Ordering::Relaxed); - None - } + self.upgrade()?.get_with_key(cache_key).await } pub async fn insert_with_key(&self, cache_key: &K, value: Arc) -> bool @@ -479,15 +598,12 @@ impl WeakLanceCache { K: CacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - if let Some(cache) = self.inner.upgrade() { - let size = cache_entry_size(&*value); - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - cache.insert(&key, value, size, K::codec()).await; - true - } else { + let Some(cache) = self.upgrade() else { log::warn!("WeakLanceCache: cache no longer available, unable to insert item"); - false - } + return false; + }; + cache.insert_with_key(cache_key, value).await; + true } /// Get or insert an item, computing it if necessary. @@ -504,25 +620,31 @@ impl WeakLanceCache { F: FnOnce() -> Fut + Send, Fut: Future> + Send, { - if let Some(cache) = self.inner.upgrade() { - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - let typed_loader = Box::pin(async move { - let value = loader().await?; - let arc = Arc::new(value); - let size = cache_entry_size(&*arc); - Ok((arc as CacheEntry, size)) - }); - let (entry, was_cached) = cache.get_or_insert(&key, typed_loader, K::codec()).await?; - if was_cached { - self.hits.fetch_add(1, Ordering::Relaxed); - } else { - self.misses.fetch_add(1, Ordering::Relaxed); - } - Ok(entry.downcast::().unwrap()) - } else { + self.get_or_insert_with_key_hit(cache_key, loader) + .await + .map(|(value, _)| value) + } + + /// Same as [`get_or_insert_with_key`](Self::get_or_insert_with_key), but + /// also returns a boolean indicating whether the loader was skipped for + /// this call. See [`LanceCache::get_or_insert_with_key_hit`] for the + /// coalesced-load caveat. + pub async fn get_or_insert_with_key_hit( + &self, + cache_key: K, + loader: F, + ) -> Result<(Arc, bool)> + where + K: CacheKey, + K::ValueType: DeepSizeOf + Send + Sync + 'static, + F: FnOnce() -> Fut + Send, + Fut: Future> + Send, + { + let Some(cache) = self.upgrade() else { log::warn!("WeakLanceCache: cache no longer available, computing without caching"); - loader().await.map(Arc::new) - } + return loader().await.map(|value| (Arc::new(value), false)); + }; + cache.get_or_insert_with_key_hit(cache_key, loader).await } pub async fn get_unsized_with_key(&self, cache_key: &K) -> Option> @@ -530,16 +652,7 @@ impl WeakLanceCache { K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - let cache = self.inner.upgrade()?; - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - if let Some(entry) = cache.get(&key, None).await { - entry - .downcast::>() - .ok() - .map(|arc| arc.as_ref().clone()) - } else { - None - } + self.upgrade()?.get_unsized_with_key(cache_key).await } pub async fn insert_unsized_with_key(&self, cache_key: &K, value: Arc) @@ -547,14 +660,18 @@ impl WeakLanceCache { K: UnsizedCacheKey, K::ValueType: DeepSizeOf + Send + Sync + 'static, { - if let Some(cache) = self.inner.upgrade() { - let wrapper = Arc::new(value); - let size = cache_entry_size(&*wrapper); - let key = build_key(&self.prefix, &cache_key.key(), K::type_name()); - cache.insert(&key, wrapper, size, None).await; - } else { + let Some(cache) = self.upgrade() else { log::warn!("WeakLanceCache: cache no longer available, unable to insert unsized item"); - } + return; + }; + cache.insert_unsized_with_key(cache_key, value).await; + } + + fn upgrade(&self) -> Option { + Some(LanceCache { + state: self.state.upgrade()?, + namespace: self.namespace, + }) } } @@ -594,457 +711,764 @@ impl CacheStats { #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::pin::Pin; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, + }; + use std::task::Poll; + use std::thread; + use std::time::Duration; + use super::*; - use std::collections::{BTreeSet, HashMap}; - use std::marker::PhantomData; - struct TestKey { - key: String, - _phantom: PhantomData, + async fn report_first_pending( + future: F, + parked: tokio::sync::oneshot::Sender<()>, + ) -> F::Output + where + F: Future, + { + tokio::pin!(future); + let mut parked = Some(parked); + futures::future::poll_fn(|cx| match future.as_mut().poll(cx) { + Poll::Pending => { + if let Some(parked) = parked.take() { + let _ = parked.send(()); + } + Poll::Pending + } + Poll::Ready(output) => Poll::Ready(output), + }) + .await } - impl TestKey { - fn new(key: &str) -> Self { - Self { - key: key.to_string(), - _phantom: PhantomData, - } + #[derive(Clone)] + struct VersionedTestKey { + id: u64, + } + + type TestKey = VersionedTestKey<1>; + type TestKeyV2 = VersionedTestKey<2>; + + impl VersionedTestKey { + fn new(id: u64) -> Self { + Self { id } } } - impl CacheKey for TestKey { - type ValueType = T; - fn key(&self) -> std::borrow::Cow<'_, str> { - std::borrow::Cow::Borrowed(&self.key) + impl CacheKey for VersionedTestKey { + type ValueType = Vec; + + fn key(&self) -> Cow<'_, str> { + self.id.to_string().into() } + fn type_name() -> &'static str { - std::any::type_name::() + "test.VecU32" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("test.vec-u32-key", SCHEMA_VERSION) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.id); } } - /// Test helper: an UnsizedCacheKey for trait object values. - struct TestUnsizedKey { - key: String, - _phantom: PhantomData, + struct SharedTestValue { + data: Arc>, } - impl TestUnsizedKey { - fn new(key: &str) -> Self { - Self { - key: key.to_string(), - _phantom: PhantomData, - } + impl DeepSizeOf for SharedTestValue { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.data.deep_size_of_children(context) } } - impl UnsizedCacheKey for TestUnsizedKey { - type ValueType = T; - fn key(&self) -> std::borrow::Cow<'_, str> { - std::borrow::Cow::Borrowed(&self.key) + struct SharedTestKey(u64); + + impl CacheKey for SharedTestKey { + type ValueType = SharedTestValue; + + fn key(&self) -> Cow<'_, str> { + self.0.to_string().into() } + fn type_name() -> &'static str { - std::any::type_name::() + "test.SharedValue" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("test.shared-value-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.0); } } - fn key_fields(keys: &[InternalCacheKey]) -> BTreeSet<(String, String, &'static str)> { - keys.iter() - .map(|key| { - ( - key.prefix().to_string(), - key.key().to_string(), - key.type_name(), - ) - }) - .collect() + struct ReentrantValue(LanceCache); + + impl DeepSizeOf for ReentrantValue { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.0.deep_size_of_children(context) + } } - #[tokio::test] - async fn test_cache_bytes() { - let item = Arc::new(vec![1, 2, 3]); - let item_size = item.deep_size_of(); - let capacity = 10 * item_size; - let cache = LanceCache::with_capacity(capacity); + struct ReentrantKey; - cache - .insert_with_key(&TestKey::>::new("key"), item.clone()) - .await; - assert_eq!(cache.size().await, 1); + impl CacheKey for ReentrantKey { + type ValueType = ReentrantValue; - let retrieved = cache - .get_with_key(&TestKey::>::new("key")) - .await - .unwrap(); - assert_eq!(*retrieved, *item); + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed("reentrant") + } - for i in 0..20 { - cache - .insert_with_key( - &TestKey::>::new(&format!("key_{}", i)), - Arc::new(vec![i, i, i]), - ) - .await; + fn type_name() -> &'static str { + "test.ReentrantValue" } - assert!(cache.size_bytes().await <= capacity); } - #[tokio::test] - async fn test_cache_weighs_key_footprint() { - // Weighted size charges the key's unique bytes, not just the value. - let cache = LanceCache::with_capacity(usize::MAX); - let key = "k".repeat(10_000); - let value = Arc::new(vec![1_i32]); - let expected = - std::mem::size_of::() + key.len() + cache_entry_size(&*value); - cache - .insert_with_key(&TestKey::>::new(&key), value) - .await; - assert_eq!(cache.size_bytes().await, expected); + #[derive(Clone, Copy, Debug)] + enum TestBackendKind { + Moka, + Quick, } - #[tokio::test] - async fn test_cache_shared_prefix_not_charged_per_entry() { - // The shared prefix contributes nothing per entry (it isn't freed on a - // single eviction); only struct + unique key + value are charged. - let cache = LanceCache::with_capacity(usize::MAX).with_key_prefix(&"p".repeat(10_000)); - for i in 0..100 { - cache - .insert_with_key( - &TestKey::>::new(&i.to_string()), - Arc::new(vec![1_i32]), - ) - .await; + impl TestBackendKind { + fn cache(self, capacity: usize) -> LanceCache { + match self { + Self::Moka => LanceCache::with_capacity(capacity), + Self::Quick => { + LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(capacity))) + } + } } - let value_cost = cache_entry_size(&vec![1_i32]); - let key_bytes: usize = (0..100).map(|i| i.to_string().len()).sum(); - let expected = 100 * (std::mem::size_of::() + value_cost) + key_bytes; - assert_eq!(cache.size_bytes().await, expected); } - #[tokio::test] - async fn test_cache_trait_objects() { - #[derive(Debug, DeepSizeOf)] - struct MyType(i32); + struct LegacyBridgeKey(&'static str); + + impl CacheKey for LegacyBridgeKey { + type ValueType = Vec; - trait MyTrait: DeepSizeOf + Send + Sync + std::any::Any { - fn as_any(&self) -> &dyn std::any::Any; + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) } - impl MyTrait for MyType { - fn as_any(&self) -> &dyn std::any::Any { - self - } + fn type_name() -> &'static str { + "test.LegacyBridge" } + } - let item: Arc = Arc::new(MyType(42)); - let cache = LanceCache::with_capacity(1000); - cache - .insert_unsized_with_key(&TestUnsizedKey::::new("test"), item) - .await; + struct ExplicitBridgeKey(&'static str); - let retrieved = cache - .get_unsized_with_key(&TestUnsizedKey::::new("test")) - .await - .unwrap(); - assert_eq!(retrieved.as_any().downcast_ref::().unwrap().0, 42); + impl CacheKey for ExplicitBridgeKey { + type ValueType = Vec; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) + } + + fn type_name() -> &'static str { + "test.LegacyBridge" + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.0); + } } - #[tokio::test] - async fn test_cache_stats_basic() { - let cache = LanceCache::with_capacity(1000); - assert_eq!(cache.stats().await.hits, 0); + trait TestDynValue: DeepSizeOf + Send + Sync { + fn values(&self) -> &[u32]; + } - // Miss - assert!( - cache - .get_with_key(&TestKey::>::new("x")) + impl TestDynValue for Vec { + fn values(&self) -> &[u32] { + self + } + } + + struct LegacyUnsizedBridgeKey(&'static str); + + impl UnsizedCacheKey for LegacyUnsizedBridgeKey { + type ValueType = dyn TestDynValue; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) + } + + fn type_name() -> &'static str { + "test.LegacyUnsizedBridge" + } + } + + struct ExplicitUnsizedBridgeKey(&'static str); + + impl UnsizedCacheKey for ExplicitUnsizedBridgeKey { + type ValueType = dyn TestDynValue; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed(self.0) + } + + fn type_name() -> &'static str { + "test.LegacyUnsizedBridge" + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.0); + } + } + + #[derive(Debug, Default)] + struct HashMapBackend { + entries: tokio::sync::Mutex>, + } + + #[async_trait::async_trait] + impl CacheBackend for HashMapBackend { + async fn get( + &self, + key: &InternalCacheKey, + _codec: Option, + ) -> Option { + self.entries + .lock() .await - .is_none() - ); - assert_eq!(cache.stats().await.misses, 1); + .get(key) + .map(|(entry, _)| entry.clone()) + } - // Insert then hit - cache - .insert_with_key(&TestKey::new("k"), Arc::new(vec![1, 2, 3])) - .await; - assert!( - cache - .get_with_key(&TestKey::>::new("k")) + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + _codec: Option, + ) { + self.entries.lock().await.insert(*key, (entry, size_bytes)); + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + codec: Option, + ) -> Result<(CacheEntry, bool)> { + if let Some(entry) = self.get(key, codec).await { + return Ok((entry, true)); + } + let (entry, size_bytes) = loader.await?; + self.insert(key, entry.clone(), size_bytes, codec).await; + Ok((entry, false)) + } + + async fn clear(&self) { + self.entries.lock().await.clear(); + } + + async fn num_entries(&self) -> usize { + self.entries.lock().await.len() + } + + async fn size_bytes(&self) -> usize { + self.entries + .lock() .await - .is_some() + .values() + .map(|(_, size_bytes)| size_bytes) + .sum() + } + } + + #[derive(Debug)] + struct WrongTypeBackend; + + #[async_trait::async_trait] + impl CacheBackend for WrongTypeBackend { + async fn get( + &self, + _key: &InternalCacheKey, + _codec: Option, + ) -> Option { + Some(Arc::new(String::from("wrong type"))) + } + + async fn insert( + &self, + _key: &InternalCacheKey, + _entry: CacheEntry, + _size_bytes: usize, + _codec: Option, + ) { + } + + async fn get_or_insert<'a>( + &self, + _key: &InternalCacheKey, + _loader: Pin> + Send + 'a>>, + _codec: Option, + ) -> Result<(CacheEntry, bool)> { + Ok((Arc::new(String::from("wrong type")), true)) + } + + async fn clear(&self) {} + + async fn num_entries(&self) -> usize { + 0 + } + + async fn size_bytes(&self) -> usize { + 0 + } + } + + #[tokio::test] + async fn typed_roundtrip_stats_clear_and_namespace_isolation() { + let cache = LanceCache::with_capacity(4096); + let left = cache.with_key_prefix("left"); + let right = cache.with_key_prefix("right"); + left.insert_with_key(&TestKey::new(7), Arc::new(vec![1, 2, 3])) + .await; + + assert_eq!( + left.get_with_key(&TestKey::new(7)).await.as_deref(), + Some(&vec![1, 2, 3]) ); - assert_eq!(cache.stats().await.hits, 1); + assert!(right.get_with_key(&TestKey::new(7)).await.is_none()); + let stats = cache.stats().await; + assert_eq!((stats.hits, stats.misses, stats.num_entries), (1, 1, 1)); + + cache.clear().await; + let stats = left.stats().await; + assert_eq!((stats.hits, stats.misses, stats.num_entries), (0, 0, 0)); } #[tokio::test] - async fn test_cache_stats_with_prefixes() { - let base = LanceCache::with_capacity(1000); - let prefixed = base.with_key_prefix("ns"); + async fn strong_and_weak_handles_share_state_and_namespace() { + let cache = LanceCache::with_capacity(4096); + let child = cache.with_key_prefix("child"); + let weak = WeakLanceCache::from(&child); assert!( - prefixed - .get_with_key(&TestKey::>::new("k")) + weak.insert_with_key(&TestKey::new(1), Arc::new(vec![1])) .await - .is_none() ); - assert_eq!(base.stats().await.misses, 1); - - prefixed - .insert_with_key(&TestKey::new("k"), Arc::new(vec![1])) + assert_eq!( + child.get_with_key(&TestKey::new(1)).await.as_deref(), + Some(&vec![1]) + ); + child + .insert_with_key(&TestKey::new(2), Arc::new(vec![2])) .await; - assert!( - prefixed - .get_with_key(&TestKey::>::new("k")) - .await - .is_some() + assert_eq!( + weak.get_with_key(&TestKey::new(2)).await.as_deref(), + Some(&vec![2]) ); - assert_eq!(base.stats().await.hits, 1); + assert_eq!((cache.stats().await.hits, cache.size().await), (2, 2)); } #[tokio::test] - async fn test_cache_keys_with_prefixes() { - let base = LanceCache::with_capacity(1000); - let prefixed = base.with_key_prefix("ns"); - let nested = prefixed.with_key_prefix("index"); - let other = base.with_key_prefix("ns-other"); - - base.insert_with_key(&TestKey::new("root"), Arc::new(vec![0])) - .await; - prefixed - .insert_with_key(&TestKey::new("child"), Arc::new(vec![1])) - .await; + async fn nested_namespace_segments_do_not_alias_combined_segments() { + let cache = LanceCache::with_capacity(4096); + let nested = cache.with_key_prefix("a").with_key_prefix("b"); + let combined = cache.with_key_prefix("a/b"); nested - .insert_with_key(&TestKey::new("nested"), Arc::new(vec![2])) + .insert_with_key(&TestKey::new(1), Arc::new(vec![10])) .await; - other - .insert_with_key(&TestKey::new("other"), Arc::new(vec![3])) + assert!(combined.get_with_key(&TestKey::new(1)).await.is_none()); + } + + #[tokio::test] + async fn schema_change_produces_a_cold_miss() { + let cache = LanceCache::with_capacity(4096); + cache + .insert_with_key(&TestKey::new(1), Arc::new(vec![10])) .await; + assert!(cache.get_with_key(&TestKeyV2::new(1)).await.is_none()); + } - let base_keys = base.keys().await.unwrap().collect::>(); - assert_eq!( - key_fields(&base_keys), - BTreeSet::from([ - ( - "".to_string(), - "root".to_string(), - TestKey::>::type_name() - ), - ( - "ns/".to_string(), - "child".to_string(), - TestKey::>::type_name() - ), - ( - "ns/index/".to_string(), - "nested".to_string(), - TestKey::>::type_name() - ), - ( - "ns-other/".to_string(), - "other".to_string(), - TestKey::>::type_name() - ), - ]) - ); + #[tokio::test] + async fn get_or_insert_with_key_hit_reports_loader_execution() { + let cache = LanceCache::with_capacity(4096); + + // Cold: loader runs, was_cached = false. + let (value, was_cached) = cache + .get_or_insert_with_key_hit(TestKey::new(1), || async { Ok(vec![1, 2, 3]) }) + .await + .unwrap(); + assert_eq!(*value, vec![1, 2, 3]); + assert!(!was_cached); + + // Warm: loader must not run and was_cached = true. + let (value, was_cached) = cache + .get_or_insert_with_key_hit(TestKey::new(1), || async { + panic!("should not be called") + }) + .await + .unwrap(); + assert_eq!(*value, vec![1, 2, 3]); + assert!(was_cached); + } - let prefixed_keys = prefixed.keys().await.unwrap().collect::>(); + #[tokio::test] + async fn default_string_bridge_matches_explicit_legacy_encoding() { + let cache = LanceCache::with_capacity(4096); + cache + .insert_with_key(&LegacyBridgeKey("same"), Arc::new(vec![10])) + .await; assert_eq!( - key_fields(&prefixed_keys), - BTreeSet::from([ - ( - "ns/".to_string(), - "child".to_string(), - TestKey::>::type_name() - ), - ( - "ns/index/".to_string(), - "nested".to_string(), - TestKey::>::type_name() - ), - ]) + cache + .get_with_key(&ExplicitBridgeKey("same")) + .await + .as_deref(), + Some(&vec![10]) ); } #[tokio::test] - async fn test_cache_keys_reflect_invalidation_and_clear() { - let base = LanceCache::with_capacity(1000); - let prefixed = base.with_key_prefix("ns"); - let other = base.with_key_prefix("other"); - - prefixed - .insert_with_key(&TestKey::new("child"), Arc::new(vec![1])) - .await; - other - .insert_with_key(&TestKey::new("other"), Arc::new(vec![2])) + async fn unsized_default_string_bridge_matches_explicit_legacy_encoding() { + let cache = LanceCache::with_capacity(4096); + let value: Arc = Arc::new(vec![10, 20]); + cache + .insert_unsized_with_key(&LegacyUnsizedBridgeKey("same"), value) .await; - assert_eq!(base.keys().await.unwrap().count(), 2); - prefixed.invalidate_prefix("").await; - let keys = base.keys().await.unwrap().collect::>(); + let cached = cache + .get_unsized_with_key(&ExplicitUnsizedBridgeKey("same")) + .await + .unwrap(); + assert_eq!(cached.values(), &[10, 20]); + } + + #[tokio::test] + async fn custom_backend_receives_opaque_keys_and_shared_clear() { + let backend = Arc::new(HashMapBackend::default()); + let cache = LanceCache::with_backend(backend.clone()); + let child = cache.with_key_prefix("child"); + let value = Arc::new(vec![1, 2, 3]); + let value_size = cache_entry_size(value.as_ref()); + + child.insert_with_key(&TestKey::new(7), value).await; assert_eq!( - key_fields(&keys), - BTreeSet::from([( - "other/".to_string(), - "other".to_string(), - TestKey::>::type_name() - )]) + child.get_with_key(&TestKey::new(7)).await.as_deref(), + Some(&vec![1, 2, 3]) ); + assert_eq!(backend.entries.lock().await.len(), 1); + assert_eq!(cache.size_bytes().await, value_size); - base.clear().await; - assert_eq!(base.keys().await.unwrap().count(), 0); + cache.clear().await; + assert!(backend.entries.lock().await.is_empty()); + assert_eq!(child.stats().await.hits, 0); } #[tokio::test] - async fn test_cache_get_or_insert() { - let cache = LanceCache::with_capacity(1000); + async fn backend_type_collisions_are_contextual_misses_or_errors() { + let cache = LanceCache::with_backend(Arc::new(WrongTypeBackend)); - let v: Arc> = cache - .get_or_insert_with_key(TestKey::>::new("k"), || async { - Ok(vec![1, 2, 3]) - }) + assert!(cache.get_with_key(&TestKey::new(1)).await.is_none()); + let error = cache + .get_or_insert_with_key(TestKey::new(2), || async { Ok(vec![2]) }) .await - .unwrap(); - assert_eq!(*v, vec![1, 2, 3]); - assert_eq!(cache.stats().await.misses, 1); - assert_eq!(cache.stats().await.hits, 0); + .unwrap_err(); + assert!(error.to_string().contains("test.VecU32")); + let stats = cache.stats().await; + assert_eq!((stats.hits, stats.misses), (0, 2)); + } - // Second call should not invoke loader and should be a hit - let v: Arc> = cache - .get_or_insert_with_key(TestKey::>::new("k"), || async { - panic!("should not be called") - }) - .await + #[tokio::test] + async fn moka_weight_includes_the_fixed_physical_key() { + let value = Arc::new(vec![0_u32; 3]); + let expected = cache_entry_size(value.as_ref()) + .checked_add(std::mem::size_of::()) .unwrap(); - assert_eq!(*v, vec![1, 2, 3]); - assert_eq!(cache.stats().await.hits, 1); + let cache = LanceCache::with_capacity(expected * 2); + cache.insert_with_key(&TestKey::new(1), value).await; + assert_eq!(cache.size_bytes().await, expected); } + #[rstest::rstest] + #[case::moka(TestBackendKind::Moka)] + #[case::quick(TestBackendKind::Quick)] #[tokio::test] - async fn test_custom_backend() { - use async_trait::async_trait; - use tokio::sync::Mutex; + async fn deep_size_deduplicates_shared_entry_allocations( + #[case] backend_kind: TestBackendKind, + ) { + let cache = backend_kind.cache(1 << 20); + let shared_data = Arc::new(vec![0_u8; 1024]); - #[derive(Debug)] - struct HashMapBackend { - map: Mutex>, + for id in 0..2 { + let data = shared_data.clone(); + cache + .get_or_insert_with_key(SharedTestKey(id), || async move { + Ok(SharedTestValue { data }) + }) + .await + .unwrap(); } - impl HashMapBackend { - fn new() -> Self { - Self { - map: Mutex::new(HashMap::new()), - } - } - } + let arc_overhead = std::mem::size_of::() * 2; + let shared_allocation = std::mem::size_of::>() + shared_data.capacity(); + let expected_entries = 2 * std::mem::size_of::() + + 2 * (std::mem::size_of::() + arc_overhead) + + shared_allocation; - #[async_trait] - impl CacheBackend for HashMapBackend { - async fn get( - &self, - key: &InternalCacheKey, - _codec: Option, - ) -> Option { - self.map.lock().await.get(key).map(|(e, _)| e.clone()) - } - async fn insert( - &self, - key: &InternalCacheKey, - entry: CacheEntry, - size_bytes: usize, - _codec: Option, - ) { - self.map - .lock() - .await - .insert(key.clone(), (entry, size_bytes)); - } - async fn get_or_insert<'a>( - &self, - key: &InternalCacheKey, - loader: std::pin::Pin< - Box> + Send + 'a>, - >, - _codec: Option, - ) -> Result<(CacheEntry, bool)> { - if let Some((entry, _)) = self.map.lock().await.get(key) { - Ok((entry.clone(), true)) - } else { - let (entry, size) = loader.await?; - self.map - .lock() - .await - .insert(key.clone(), (entry.clone(), size)); - Ok((entry, false)) - } - } - async fn invalidate_prefix(&self, prefix: &str) { - self.map.lock().await.retain(|k, _| !k.starts_with(prefix)); - } - async fn clear(&self) { - self.map.lock().await.clear(); - } - async fn num_entries(&self) -> usize { - self.map.lock().await.len() - } - async fn size_bytes(&self) -> usize { - self.map.lock().await.values().map(|(_, s)| *s).sum() - } + let weighted_size = cache.size_bytes().await; + assert_eq!(weighted_size, expected_entries + shared_allocation); + assert_eq!( + cache.deep_size_of(), + std::mem::size_of::() + expected_entries + ); + + let mut context = Context::new(); + assert_eq!(cache.deep_size_of_children(&mut context), expected_entries); + assert_eq!( + cache + .with_key_prefix("another-handle") + .deep_size_of_children(&mut context), + 0 + ); + } + + #[test] + fn sizing_can_reenter_the_same_cache() { + let (done_tx, done_rx) = mpsc::channel(); + let worker = thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let cache = LanceCache::with_capacity(4096); + cache + .insert_with_key(&ReentrantKey, Arc::new(ReentrantValue(cache.clone()))) + .await; + done_tx.send(()).unwrap(); + }); + }); + + done_rx + .recv_timeout(Duration::from_secs(1)) + .expect("cache insertion deadlocked during sizing"); + worker.join().unwrap(); + } + + #[tokio::test] + async fn no_cache_computes_each_time() { + let cache = LanceCache::no_cache(); + let loads = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let loads = loads.clone(); + let value = cache + .get_or_insert_with_key(TestKey::new(1), move || async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(vec![42]) + }) + .await + .unwrap(); + assert_eq!(value.as_slice(), &[42]); } + assert_eq!(loads.load(Ordering::SeqCst), 2); + assert_eq!(cache.size().await, 0); + } - let cache = LanceCache::with_backend(Arc::new(HashMapBackend::new())); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_flight_coalesces_success_after_contenders_are_parked() { + const CONTENDERS: usize = 4; - cache - .insert_with_key(&TestKey::new("k"), Arc::new(vec![1, 2, 3])) - .await; - assert!( - cache - .get_with_key(&TestKey::>::new("k")) + let cache = Arc::new(LanceCache::with_capacity(4096)); + let loader_calls = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + + let owner = { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + let release = release.clone(); + tokio::spawn(async move { + cache + .get_or_insert_with_key(TestKey::new(10), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + let _ = started_tx.send(()); + release.notified().await; + Ok(vec![10]) + }) + .await + }) + }; + started_rx.await.unwrap(); + + let mut contenders = Vec::new(); + let mut parked = Vec::new(); + for _ in 0..CONTENDERS { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + let (parked_tx, parked_rx) = tokio::sync::oneshot::channel(); + parked.push(parked_rx); + contenders.push(tokio::spawn(async move { + report_first_pending( + cache.get_or_insert_with_key(TestKey::new(10), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![99]) + }), + parked_tx, + ) .await - .is_some() - ); - // Different type at same key = miss - assert!( - cache - .get_with_key(&TestKey::>::new("k")) + })); + } + for parked in parked { + parked .await - .is_none() - ); - assert!(cache.keys().await.is_none()); + .expect("contender completed instead of parking behind owner"); + } + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + assert!(contenders.iter().all(|handle| !handle.is_finished())); + + release.notify_one(); + assert_eq!(owner.await.unwrap().unwrap().as_slice(), &[10]); + for contender in contenders { + assert_eq!(contender.await.unwrap().unwrap().as_slice(), &[10]); + } + let stats = cache.stats().await; + assert_eq!((stats.hits, stats.misses), (CONTENDERS as u64, 1),); } - #[tokio::test] - async fn test_get_or_insert_dedup() { - use std::sync::atomic::AtomicUsize; + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_flight_coalesces_errors_after_contenders_are_parked() { + const CONTENDERS: usize = 4; - let load_count = Arc::new(AtomicUsize::new(0)); - let cache = LanceCache::with_capacity(10000); + let cache = Arc::new(LanceCache::with_capacity(4096)); + let loader_calls = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); - let (barrier_tx, _) = tokio::sync::broadcast::channel::<()>(1); - let mut handles = Vec::new(); - for _ in 0..5 { + let owner = { let cache = cache.clone(); - let load_count = load_count.clone(); - let mut barrier_rx = barrier_tx.subscribe(); - handles.push(tokio::spawn(async move { - barrier_rx.recv().await.ok(); + let loader_calls = loader_calls.clone(); + let release = release.clone(); + tokio::spawn(async move { cache - .get_or_insert_with_key(TestKey::>::new("key"), || { - let load_count = load_count.clone(); - async move { - load_count.fetch_add(1, Ordering::SeqCst); - tokio::task::yield_now().await; - Ok(vec![1, 2, 3]) - } + .get_or_insert_with_key(TestKey::new(20), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + let _ = started_tx.send(()); + release.notified().await; + Err(Error::timeout("owner loader timed out")) }) .await + }) + }; + started_rx.await.unwrap(); + + let mut contenders = Vec::new(); + let mut parked = Vec::new(); + for _ in 0..CONTENDERS { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + let (parked_tx, parked_rx) = tokio::sync::oneshot::channel(); + parked.push(parked_rx); + contenders.push(tokio::spawn(async move { + report_first_pending( + cache.get_or_insert_with_key(TestKey::new(20), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::timeout("contender loader timed out")) + }), + parked_tx, + ) + .await })); } - barrier_tx.send(()).unwrap(); - for h in handles { - let result: Arc> = h.await.unwrap().unwrap(); - assert_eq!(*result, vec![1, 2, 3]); + for parked in parked { + parked + .await + .expect("contender completed instead of parking behind owner"); + } + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + assert!(contenders.iter().all(|handle| !handle.is_finished())); + + release.notify_one(); + assert!(matches!(owner.await.unwrap(), Err(Error::Timeout { .. }))); + for contender in contenders { + assert!(matches!( + contender.await.unwrap(), + Err(Error::Timeout { .. }) + )); } + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + } - assert_eq!(load_count.load(Ordering::SeqCst), 1); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_flight_retries_after_the_owner_is_cancelled() { + let cache = Arc::new(LanceCache::with_capacity(4096)); + let loader_calls = Arc::new(AtomicUsize::new(0)); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + + let owner = { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + tokio::spawn(async move { + cache + .get_or_insert_with_key(TestKey::new(30), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + let _ = started_tx.send(()); + std::future::pending::<()>().await; + Ok(vec![30]) + }) + .await + }) + }; + started_rx.await.unwrap(); + + let (parked_tx, parked_rx) = tokio::sync::oneshot::channel(); + let contender = { + let cache = cache.clone(); + let loader_calls = loader_calls.clone(); + tokio::spawn(async move { + report_first_pending( + cache.get_or_insert_with_key(TestKey::new(30), move || async move { + loader_calls.fetch_add(1, Ordering::SeqCst); + Ok(vec![31]) + }), + parked_tx, + ) + .await + }) + }; + parked_rx + .await + .expect("contender completed instead of parking behind owner"); + assert_eq!(loader_calls.load(Ordering::SeqCst), 1); + assert!(!contender.is_finished()); + + owner.abort(); + assert!(owner.await.unwrap_err().is_cancelled()); + let value = tokio::time::timeout(std::time::Duration::from_secs(5), contender) + .await + .expect("contender remained parked after owner cancellation") + .unwrap() + .unwrap(); + assert_eq!(value.as_slice(), &[31]); + assert_eq!(loader_calls.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn expired_weak_cache_degrades_without_retaining_state() { + let cache = LanceCache::with_capacity(4096); + let weak = WeakLanceCache::from(&cache); + drop(cache); + + assert!(weak.get_with_key(&TestKey::new(1)).await.is_none()); + assert!( + !weak + .insert_with_key(&TestKey::new(1), Arc::new(vec![1])) + .await + ); + let value = weak + .get_or_insert_with_key(TestKey::new(1), || async { Ok(vec![7]) }) + .await + .unwrap(); + assert_eq!(value.as_slice(), &[7]); } } diff --git a/rust/lance-core/src/cache/moka.rs b/rust/lance-core/src/cache/moka.rs index fd86b064f6b..2a93ba6c2ca 100644 --- a/rust/lance-core/src/cache/moka.rs +++ b/rust/lance-core/src/cache/moka.rs @@ -9,9 +9,11 @@ use async_trait::async_trait; use futures::Future; use crate::Result; +use crate::deepsize::Context; +use crate::error::CloneableError; -use super::CacheCodec; -use super::backend::{CacheBackend, CacheEntry, CacheKeyIterator, InternalCacheKey}; +use super::backend::{CacheBackend, CacheEntry}; +use super::{CacheCodec, InternalCacheKey}; /// Internal record stored in the moka cache. #[derive(Clone, Debug)] @@ -20,10 +22,28 @@ struct MokaCacheEntry { size_bytes: usize, } -/// Per-entry key cost for eviction: the struct plus the unique `key` bytes. -/// Excludes the shared `prefix` `Arc`, which isn't freed per eviction. -fn key_footprint(key: &InternalCacheKey) -> usize { - std::mem::size_of::() + key.key().len() +/// Per-entry key cost for eviction. +pub(super) fn key_footprint(_key: &InternalCacheKey) -> usize { + std::mem::size_of::() +} + +fn physical_size(key: &InternalCacheKey, size_bytes: usize) -> usize { + key_footprint(key).saturating_add(size_bytes) +} + +/// Number of physical bytes represented by one Moka weight unit. +/// +/// Moka limits each entry's weight to `u32`, so capacities above 4 GiB need +/// coarser units to account for a single large entry without undercharging it. +fn weight_unit(capacity: usize) -> usize { + capacity.div_ceil(u32::MAX as usize).max(1) +} + +fn entry_weight(key: &InternalCacheKey, size_bytes: usize, weight_unit: usize) -> u32 { + physical_size(key, size_bytes) + .div_ceil(weight_unit) + .try_into() + .unwrap_or(u32::MAX) } /// Default [`CacheBackend`] backed by a [moka](https://crates.io/crates/moka) cache. @@ -32,6 +52,8 @@ fn key_footprint(key: &InternalCacheKey) -> usize { /// via moka's built-in `optionally_get_with`. pub struct MokaCacheBackend { cache: moka::future::Cache, + capacity: usize, + weight_unit: usize, } impl std::fmt::Debug for MokaCacheBackend { @@ -44,24 +66,41 @@ impl std::fmt::Debug for MokaCacheBackend { impl MokaCacheBackend { pub fn with_capacity(capacity: usize) -> Self { + let weight_unit = weight_unit(capacity); + let capacity_weight = capacity.div_ceil(weight_unit) as u64; let cache = moka::future::Cache::builder() - .max_capacity(capacity as u64) - .weigher(|key: &InternalCacheKey, entry: &MokaCacheEntry| { - key_footprint(key) - .saturating_add(entry.size_bytes) - .try_into() - .unwrap_or(u32::MAX) + .max_capacity(capacity_weight) + .weigher(move |key: &InternalCacheKey, entry: &MokaCacheEntry| { + entry_weight(key, entry.size_bytes, weight_unit) }) - .support_invalidation_closures() .build(); - Self { cache } + Self { + cache, + capacity, + weight_unit, + } } pub fn no_cache() -> Self { Self { cache: moka::future::Cache::new(0), + capacity: 0, + weight_unit: 1, } } + + /// Configured weighted capacity in bytes. + pub fn capacity(&self) -> usize { + self.capacity + } + + fn weighted_size_bytes(&self) -> usize { + self.cache + .weighted_size() + .saturating_mul(self.weight_unit as u64) + .try_into() + .unwrap_or(usize::MAX) + } } #[async_trait] @@ -78,7 +117,7 @@ impl CacheBackend for MokaCacheBackend { _codec: Option, ) { self.cache - .insert(key.clone(), MokaCacheEntry { entry, size_bytes }) + .insert(*key, MokaCacheEntry { entry, size_bytes }) .await; } @@ -88,59 +127,33 @@ impl CacheBackend for MokaCacheBackend { loader: Pin> + Send + 'a>>, _codec: Option, ) -> Result<(CacheEntry, bool)> { - // Use moka's built-in dedup: optionally_get_with runs the init future - // at most once per key, even under concurrent access. - let (error_tx, error_rx) = tokio::sync::oneshot::channel(); - // Track whether the loader actually ran (= cache miss). let was_miss = Arc::new(AtomicBool::new(false)); let was_miss_clone = was_miss.clone(); let init = async move { was_miss_clone.store(true, Ordering::Relaxed); - match loader.await { - Ok((entry, size_bytes)) => Some(MokaCacheEntry { entry, size_bytes }), - Err(e) => { - let _ = error_tx.send(e); - None - } - } + loader + .await + .map(|(entry, size_bytes)| MokaCacheEntry { entry, size_bytes }) + .map_err(CloneableError) }; - let owned_key = key.clone(); - match self.cache.optionally_get_with(owned_key, init).await { - Some(record) => { + let owned_key = *key; + match self.cache.try_get_with(owned_key, init).await { + Ok(record) => { let was_cached = !was_miss.load(Ordering::Relaxed); Ok((record.entry, was_cached)) } - None => match error_rx.await { - Ok(err) => Err(err), - Err(_) => Err(crate::Error::internal( - "Failed to retrieve error from cache loader", - )), - }, + Err(error) => Err(Arc::unwrap_or_clone(error).0), } } - async fn invalidate_prefix(&self, prefix: &str) { - let prefix = prefix.to_owned(); - self.cache - .invalidate_entries_if(move |key, _value| key.starts_with(&prefix)) - .expect("Cache configured correctly"); - } - async fn clear(&self) { self.cache.invalidate_all(); self.cache.run_pending_tasks().await; } - async fn keys(&self) -> Option> { - self.cache.run_pending_tasks().await; - Some(Box::new( - self.cache.iter().map(|(key, _)| key.as_ref().clone()), - )) - } - async fn num_entries(&self) -> usize { self.cache.run_pending_tasks().await; self.cache.entry_count() as usize @@ -148,7 +161,7 @@ impl CacheBackend for MokaCacheBackend { async fn size_bytes(&self) -> usize { self.cache.run_pending_tasks().await; - self.cache.weighted_size() as usize + self.weighted_size_bytes() } fn approx_num_entries(&self) -> usize { @@ -156,12 +169,181 @@ impl CacheBackend for MokaCacheBackend { } fn approx_size_bytes(&self) -> usize { - // Iterate rather than using `weighted_size()` because moka's - // weighted_size can be stale without `run_pending_tasks()`, which + // `weighted_size()` can be stale without `run_pending_tasks()`, which // is async and can't be called from this synchronous context. - self.cache - .iter() - .map(|(key, entry)| key_footprint(key.as_ref()) + entry.size_bytes) - .sum() + self.weighted_size_bytes() + } + + fn deep_size_of_entries( + &self, + context: &mut Context, + size_of_entry: &dyn Fn(&CacheEntry, &mut Context) -> Option, + ) -> Option { + Some( + self.cache + .iter() + .map(|(key, record)| { + key_footprint(key.as_ref()) + + size_of_entry(&record.entry, context).unwrap_or(record.size_bytes) + }) + .sum(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn entry_weights_are_exact_at_byte_granularity() { + let key = InternalCacheKey::from_bytes([0; 16]); + assert_eq!(weight_unit(4096), 1); + assert_eq!(entry_weight(&key, 7, 1), 23); + } + + #[tokio::test] + async fn size_methods_use_constant_time_weighted_accounting() { + let backend = MokaCacheBackend::with_capacity(4096); + let key = InternalCacheKey::from_bytes([0; 16]); + let entry: CacheEntry = Arc::new(()); + let value_size = 7; + let expected = physical_size(&key, value_size); + + backend.insert(&key, entry, value_size, None).await; + + assert_eq!(backend.size_bytes().await, expected); + assert_eq!(backend.approx_size_bytes(), expected); + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn entry_weights_scale_for_capacities_above_four_gibibytes() { + let key = InternalCacheKey::from_bytes([0; 16]); + let capacity = 6 * 1024 * 1024 * 1024; + let weight_unit = weight_unit(capacity); + assert_eq!(weight_unit, 2); + + let size_bytes = u32::MAX as usize + 1024; + let expected = physical_size(&key, size_bytes).div_ceil(weight_unit); + let weight = entry_weight(&key, size_bytes, weight_unit); + assert_eq!(weight as usize, expected); + assert_ne!(weight, u32::MAX); + } +} + +/// Registry identifier for the built-in Moka backend. +pub const MOKA_BACKEND_KIND: &str = "moka"; + +/// [`BackendBuildFn`](super::registry::BackendBuildFn) for [`MokaCacheBackend`]. +/// +/// Recognized options: +/// * `capacity` — total weighted capacity in bytes (`usize`). +/// This must be present and non-empty. +/// +/// Unknown options are rejected so typos surface immediately instead of +/// silently falling through to the default capacity. +pub(super) fn build_moka_backend( + config: &super::registry::BackendConfig, +) -> Result { + let mut capacity: Option = None; + for (key, value) in &config.options { + match key.as_str() { + "capacity" => { + if value.is_empty() { + return Err(crate::Error::invalid_input( + "moka cache backend: capacity must not be empty", + )); + } else { + capacity = Some(value.parse::().map_err(|err| { + crate::Error::invalid_input(format!( + "moka cache backend: cannot parse capacity {:?}: {}", + value, err + )) + })?); + } + } + other => { + return Err(crate::Error::invalid_input(format!( + "moka cache backend: unknown option {:?}", + other + ))); + } + } + } + let capacity = capacity.ok_or_else(|| { + crate::Error::invalid_input( + "moka cache backend: capacity is required; use moka://?capacity=", + ) + })?; + Ok(MokaCacheBackend::with_capacity(capacity)) +} + +pub(super) fn build_moka(config: &super::registry::BackendConfig) -> Result> { + Ok(Arc::new(build_moka_backend(config)?)) +} + +#[cfg(test)] +mod moka_registry_tests { + use super::super::backend_uri::{build_from_uri, parse_backend_uri}; + use super::super::registry::{BackendConfig, build_from_config, registry_test_lock}; + use super::*; + + #[test] + fn test_moka_builds_from_config() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("capacity", "1048576"); + let backend = build_moka_backend(&cfg).unwrap(); + assert_eq!(backend.capacity(), 1048576); + let _backend = build_from_config(&cfg).unwrap(); + } + + #[test] + fn test_moka_builds_from_uri() { + let _lock = registry_test_lock(); + let cfg = parse_backend_uri("moka://?capacity=1048576").unwrap(); + let backend = build_moka_backend(&cfg).unwrap(); + assert_eq!(backend.capacity(), 1048576); + let _backend = build_from_uri("moka://?capacity=1048576").unwrap(); + } + + #[test] + fn test_moka_rejects_unknown_option() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("mystery", "1"); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("unknown option")); + } + + #[test] + fn test_moka_rejects_bad_capacity() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("capacity", "not-a-number"); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("cannot parse capacity")); + } + + #[test] + fn test_moka_rejects_missing_capacity() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka").unwrap(); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("capacity is required")); + } + + #[test] + fn test_moka_rejects_empty_capacity() { + let _lock = registry_test_lock(); + let cfg = BackendConfig::new("moka") + .unwrap() + .with_option("capacity", ""); + let err = build_from_config(&cfg).unwrap_err(); + assert!(err.to_string().contains("capacity must not be empty")); } } diff --git a/rust/lance-core/src/cache/quick.rs b/rust/lance-core/src/cache/quick.rs new file mode 100644 index 00000000000..53e0d0e8870 --- /dev/null +++ b/rust/lance-core/src/cache/quick.rs @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! [`CacheBackend`] backed by [quick_cache](https://crates.io/crates/quick_cache), +//! whose hit path is one atomic bit — no read-op channel or inline +//! housekeeping. Used for the session index and metadata caches; the index +//! cache sees thousands of cache reads per query. + +use std::pin::Pin; + +use async_trait::async_trait; +use futures::Future; + +use super::backend::{CacheBackend, CacheEntry}; +use super::moka::key_footprint; +use super::{CacheCodec, InternalCacheKey}; +use crate::Result; +use crate::deepsize::Context; + +#[derive(Clone)] +struct QuickEntry { + entry: CacheEntry, + size_bytes: usize, +} + +#[derive(Clone)] +struct EntryWeighter; + +impl quick_cache::Weighter for EntryWeighter { + fn weight(&self, key: &InternalCacheKey, value: &QuickEntry) -> u64 { + // Same accounting as the moka backend. + key_footprint(key).saturating_add(value.size_bytes).max(1) as u64 + } +} + +pub struct QuickCacheBackend { + cache: quick_cache::sync::Cache, +} + +impl std::fmt::Debug for QuickCacheBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("QuickCacheBackend") + .field("entry_count", &self.cache.len()) + .finish() + } +} + +/// Minimum weight budget (4 GiB) per shard: shards don't borrow capacity, and +/// an entry heavier than ~its shard's budget is silently refused admission. +const MIN_SHARD_SHARE: usize = 4 << 30; + +/// Recommended shard count: `min(cpus / 2, capacity / 4 GiB)`, power of two +/// in `[1, 1024]`. The cpu term bounds lock contention; the capacity term +/// keeps each shard's budget >= 4 GiB so large entries stay admissible. +/// Rounded down because quick_cache rounds requests up. +pub fn recommended_cache_shards(capacity: usize) -> usize { + let by_cpu = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(2) + / 2; + let shards = (capacity / MIN_SHARD_SHARE).min(by_cpu).max(1); + let shards = if shards.is_power_of_two() { + shards + } else { + shards.next_power_of_two() / 2 + }; + shards.clamp(1, 1024) +} + +/// Assumed average entry size for pre-allocation sizing. +const ESTIMATED_AVG_ENTRY_BYTES: usize = 64 << 10; + +impl QuickCacheBackend { + /// Create a backend holding up to `capacity` bytes of weighted entries + /// (weight = key footprint + declared size), sharded per + /// [`recommended_cache_shards`]. + pub fn with_capacity(capacity: usize) -> Self { + let shards = recommended_cache_shards(capacity); + // Floor protects the shard count from quick_cache's items-per-shard + // heuristic; ceiling bounds pre-allocation. + let estimated_items = (capacity / ESTIMATED_AVG_ENTRY_BYTES).clamp(shards * 32, 1_000_000); + let options = quick_cache::OptionsBuilder::new() + .estimated_items_capacity(estimated_items) + .weight_capacity(capacity as u64) + .shards(shards) + .build() + // Only errors when weight/item capacity is missing; both are set. + .expect("quick_cache options"); + let cache = quick_cache::sync::Cache::with_options( + options, + EntryWeighter, + Default::default(), + Default::default(), + ); + Self { cache } + } +} + +#[async_trait] +impl CacheBackend for QuickCacheBackend { + async fn get(&self, key: &InternalCacheKey, _codec: Option) -> Option { + self.cache.get(key).map(|v| v.entry) + } + + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + _codec: Option, + ) { + self.cache.insert(*key, QuickEntry { entry, size_bytes }); + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + _codec: Option, + ) -> Result<(CacheEntry, bool)> { + match self.cache.get_value_or_guard_async(key).await { + Ok(value) => Ok((value.entry, true)), + Err(guard) => { + let (entry, size_bytes) = loader.await?; + let _ = guard.insert(QuickEntry { + entry: entry.clone(), + size_bytes, + }); + Ok((entry, false)) + } + } + } + + async fn clear(&self) { + self.cache.clear(); + } + + async fn num_entries(&self) -> usize { + self.cache.len() + } + + async fn size_bytes(&self) -> usize { + self.cache.weight() as usize + } + + fn approx_num_entries(&self) -> usize { + self.cache.len() + } + + fn approx_size_bytes(&self) -> usize { + self.cache.weight() as usize + } + + fn deep_size_of_entries( + &self, + context: &mut Context, + size_of_entry: &dyn Fn(&CacheEntry, &mut Context) -> Option, + ) -> Option { + Some( + self.cache + .iter() + .map(|(key, record)| { + key_footprint(&key) + + size_of_entry(&record.entry, context).unwrap_or(record.size_bytes) + }) + .sum(), + ) + } +} + +#[cfg(test)] +mod tests { + use std::marker::PhantomData; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use crate::cache::{CacheKey, LanceCache}; + + struct TestKey { + key: String, + _phantom: PhantomData, + } + + impl TestKey { + fn new(key: &str) -> Self { + Self { + key: key.to_string(), + _phantom: PhantomData, + } + } + } + + impl CacheKey for TestKey { + type ValueType = T; + fn key(&self) -> std::borrow::Cow<'_, str> { + std::borrow::Cow::Borrowed(&self.key) + } + fn type_name() -> &'static str { + std::any::type_name::() + } + } + + #[test] + fn entry_weight_includes_fixed_key() { + let key = InternalCacheKey::from_bytes([0; 16]); + let entry = QuickEntry { + entry: Arc::new(()), + size_bytes: 7, + }; + assert_eq!( + quick_cache::Weighter::weight(&EntryWeighter, &key, &entry), + 23 + ); + } + + #[tokio::test] + async fn test_quick_backend_roundtrip_singleflight_and_eviction() { + // Capacity must be large relative to one entry: quick_cache shards + // its weight budget, and an entry heavier than its shard's share is + // not admitted at all. + const CAPACITY: usize = 1 << 20; + let item = Arc::new(vec![1u8, 2, 3]); + let cache = LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(CAPACITY))); + + // insert + get roundtrip and weighted accounting + cache + .insert_with_key(&TestKey::>::new("a"), item.clone()) + .await; + assert_eq!( + cache + .get_with_key(&TestKey::>::new("a")) + .await + .as_deref(), + Some(&vec![1u8, 2, 3]) + ); + assert_eq!(cache.approx_size(), 1); + assert!(cache.size_bytes().await > 0); + + // get_or_insert runs the loader only on a miss + let loads = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let loads = loads.clone(); + let value = cache + .get_or_insert_with_key(TestKey::>::new("b"), || async move { + loads.fetch_add(1, Ordering::SeqCst); + Ok(vec![7u8]) + }) + .await + .unwrap(); + assert_eq!(value.as_ref(), &vec![7u8]); + } + assert_eq!(loads.load(Ordering::SeqCst), 1); + + // capacity is enforced: overfill with 4x capacity of 16KiB entries + // and confirm eviction kept the weighted size within budget + for i in 0..256 { + cache + .insert_with_key( + &TestKey::>::new(&format!("fill-{i}")), + Arc::new(vec![0u8; 16 << 10]), + ) + .await; + } + assert!(cache.size_bytes().await <= CAPACITY); + assert!(cache.size().await < 258); + + cache.clear().await; + assert_eq!(cache.size().await, 0); + } + + #[tokio::test] + async fn test_quick_backend_tiny_capacity() { + // A tiny cache must not over-provision item metadata and must still + // admit and evict correctly within its weight budget. + const CAPACITY: usize = 64 << 10; + let cache = LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(CAPACITY))); + for i in 0..64 { + cache + .insert_with_key( + &TestKey::>::new(&format!("k-{i}")), + Arc::new(vec![0u8; 4 << 10]), + ) + .await; + } + assert!(cache.size_bytes().await <= CAPACITY); + assert!(cache.size().await >= 1); + let hit = cache + .get_with_key(&TestKey::>::new("k-63")) + .await + .is_some() + || cache + .get_with_key(&TestKey::>::new("k-62")) + .await + .is_some(); + assert!(hit, "recently inserted entries should be resident"); + } +} diff --git a/rust/lance-core/src/cache/registry.rs b/rust/lance-core/src/cache/registry.rs new file mode 100644 index 00000000000..f370bdd3f8f --- /dev/null +++ b/rust/lance-core/src/cache/registry.rs @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Pluggable cache-backend registry. +//! +//! A [`BackendConfig`] identifies which backend to build (`kind`) and carries +//! backend-specific string options. Backends are constructed through a +//! [`BackendBuildFn`] registered under a unique `kind`. Third-party crates +//! integrate by calling [`register_backend`] once at application startup; +//! [`build_from_config`] then locates the constructor and hands it the +//! config. +//! +//! The registry uses `HashMap` for options so it can be +//! represented naturally across FFI (Python `dict[str, str]`, Java +//! `Map`, etc.). + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; + +use super::backend::CacheBackend; +use super::moka::{MOKA_BACKEND_KIND, build_moka}; +use crate::{Error, Result}; + +/// Backend-independent configuration passed to a [`BackendBuildFn`]. +/// +/// `kind` selects which registered backend to construct; `options` carries +/// backend-specific key/value settings (e.g. `capacity`, `path`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackendConfig { + /// Registered backend identifier, e.g. `"moka"`. + pub kind: String, + /// Backend-specific string options. + pub options: HashMap, +} + +impl BackendConfig { + /// Build a config with no options. + pub fn new(kind: impl AsRef) -> Result { + Ok(Self { + kind: normalize_backend_kind(kind.as_ref())?, + options: HashMap::new(), + }) + } + + /// Insert a single option and return `self`, enabling chaining. + pub fn with_option(mut self, key: impl Into, value: impl Into) -> Self { + self.options.insert(key.into(), value.into()); + self + } +} + +/// Normalize and validate a cache backend kind. +/// +/// Backend kinds share the same syntax as URI schemes. They are matched +/// case-insensitively and stored as lowercase ASCII so registry lookups, +/// config dictionaries, and URI parsing all address the same key. +pub fn normalize_backend_kind(kind: &str) -> Result { + let mut chars = kind.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() => {} + _ => { + return Err(Error::invalid_input(format!( + "cache backend kind {:?}: must start with an ASCII letter", + kind + ))); + } + } + for c in chars { + let ok = c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'); + if !ok { + return Err(Error::invalid_input(format!( + "cache backend kind {:?}: invalid character {:?}", + kind, c + ))); + } + } + Ok(kind.to_ascii_lowercase()) +} + +/// Constructor signature for a cache backend. +/// +/// Constructors are synchronous. Backends that need async initialization +/// should surface a `try_new_blocking` shim (or equivalent) and call it here. +pub type BackendBuildFn = fn(&BackendConfig) -> Result>; + +fn registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn registry_lock() -> Result>> { + registry() + .lock() + .map_err(|_| Error::internal("cache backend registry mutex is poisoned")) +} + +#[cfg(test)] +fn registry_lock_for_test() -> MutexGuard<'static, HashMap> { + registry() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Register a constructor for a cache backend under `kind`. +/// +/// Returns `Err` if a non-built-in `kind` is already registered. Built-in +/// backends may be replaced so callers can mask a built-in implementation +/// (for example, a patched `"moka"` backend) without changing URI/config +/// strings elsewhere. +/// +/// Typical usage from a backend crate: +/// +/// ``` +/// # use std::sync::Arc; +/// # use lance_core::Result; +/// # use lance_core::cache::{BackendConfig, CacheBackend, MokaCacheBackend, register_backend}; +/// fn build_my_backend(_config: &BackendConfig) -> Result> { +/// Ok(Arc::new(MokaCacheBackend::with_capacity(1024))) +/// } +/// +/// # fn main() -> Result<()> { +/// register_backend("my-backend", build_my_backend)?; +/// # Ok(()) +/// # } +/// ``` +pub fn register_backend(kind: &str, build: BackendBuildFn) -> Result<()> { + let kind = normalize_backend_kind(kind)?; + insert_backend(&kind, build, builtin_backend(&kind).is_some()) +} + +fn insert_backend(kind: &str, build: BackendBuildFn, allow_replace: bool) -> Result<()> { + let mut map = registry_lock()?; + if map.contains_key(kind) && !allow_replace { + return Err(Error::invalid_input(format!( + "cache backend {:?} is already registered", + kind + ))); + } + map.insert(kind.to_string(), build); + Ok(()) +} + +fn builtin_backend(kind: &str) -> Option { + match kind { + MOKA_BACKEND_KIND => Some(build_moka), + _ => None, + } +} + +/// Look up the constructor for `config.kind` and build a backend. +/// +/// Returns `Err` if no backend has been registered under that identifier. +pub fn build_from_config(config: &BackendConfig) -> Result> { + ensure_builtin_backends()?; + let kind = normalize_backend_kind(&config.kind)?; + let config = BackendConfig { + kind: kind.clone(), + options: config.options.clone(), + }; + let build = { + let map = registry_lock()?; + map.get(&kind).copied() + }; + match build { + Some(build) => build(&config), + None => Err(Error::invalid_input(format!( + "unknown cache backend kind: {:?}", + kind + ))), + } +} + +/// Idempotently register the backends that ship with `lance-core`. +/// +/// Called by [`build_from_config`] (and, transitively, by +/// [`build_from_uri`](super::backend_uri::build_from_uri)) so a bare Lance +/// installation can build a Moka backend without the caller having to +/// register it. Third-party backends still have to opt in with their own +/// `register()` call. +/// +/// The check is against the current registry contents rather than a +/// process-once flag so that `#[cfg(test)]` helpers which snapshot and +/// restore the registry still see the built-in backend after they take +/// ownership. +fn ensure_builtin_backends() -> Result<()> { + let mut map = registry_lock()?; + if !map.contains_key(MOKA_BACKEND_KIND) + && let Some(build) = builtin_backend(MOKA_BACKEND_KIND) + { + map.insert(MOKA_BACKEND_KIND.to_string(), build); + } + Ok(()) +} + +/// Test-only helper: replace the registry with an empty map so tests can +/// exercise duplicate-registration logic without polluting the global one. +#[cfg(test)] +pub(super) fn take_registry_for_test() -> HashMap { + let mut map = registry_lock_for_test(); + std::mem::take(&mut *map) +} + +/// Test-only helper: restore a previously captured registry state. +#[cfg(test)] +pub(super) fn restore_registry_for_test(saved: HashMap) { + let mut map = registry_lock_for_test(); + *map = saved; +} + +#[cfg(test)] +pub(super) fn registry_test_lock() -> std::sync::MutexGuard<'static, ()> { + static M: OnceLock> = OnceLock::new(); + M.get_or_init(|| std::sync::Mutex::new(())) + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + use std::pin::Pin; + + use crate::cache::InternalCacheKey; + use crate::cache::backend::CacheEntry; + use crate::cache::codec::CacheCodec; + use futures::Future; + + // A trivial no-op backend so tests do not depend on Moka or any other + // real backend. Every method returns "empty" / does nothing. + #[derive(Debug, Default)] + struct NullBackend; + + #[async_trait] + impl CacheBackend for NullBackend { + async fn get( + &self, + _key: &InternalCacheKey, + _codec: Option, + ) -> Option { + None + } + + async fn insert( + &self, + _key: &InternalCacheKey, + _entry: CacheEntry, + _size_bytes: usize, + _codec: Option, + ) { + } + + async fn get_or_insert<'a>( + &self, + _key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + _codec: Option, + ) -> crate::Result<(CacheEntry, bool)> { + let (entry, _size) = loader.await?; + Ok((entry, false)) + } + + async fn clear(&self) {} + async fn num_entries(&self) -> usize { + 0 + } + async fn size_bytes(&self) -> usize { + 0 + } + } + + fn build_null(_cfg: &BackendConfig) -> Result> { + Ok(Arc::new(NullBackend)) + } + + struct RegistryGuard { + // Hold the serialization lock for the full test. + _lock: std::sync::MutexGuard<'static, ()>, + saved: HashMap, + } + impl RegistryGuard { + fn new() -> Self { + Self { + _lock: registry_test_lock(), + saved: take_registry_for_test(), + } + } + } + impl Drop for RegistryGuard { + fn drop(&mut self) { + restore_registry_for_test(std::mem::take(&mut self.saved)); + } + } + + #[test] + fn test_register_and_build() { + let _guard = RegistryGuard::new(); + register_backend("null", build_null).unwrap(); + let backend = build_from_config(&BackendConfig::new("null").unwrap()).unwrap(); + // Backend is opaque; we just check that the constructor ran and + // gave us an Arc. + assert_eq!(Arc::strong_count(&backend), 1); + } + + #[test] + fn test_duplicate_registration_errors() { + let _guard = RegistryGuard::new(); + register_backend("dup", build_null).unwrap(); + let err = register_backend("dup", build_null).unwrap_err(); + assert!(err.to_string().contains("already registered")); + } + + #[test] + fn test_builtin_kind_can_be_overridden() { + let _guard = RegistryGuard::new(); + register_backend("moka", build_null).unwrap(); + let backend = build_from_config(&BackendConfig::new("moka").unwrap()).unwrap(); + assert_eq!(Arc::strong_count(&backend), 1); + } + + #[test] + fn test_unknown_kind_errors() { + let _guard = RegistryGuard::new(); + let err = build_from_config(&BackendConfig::new("missing").unwrap()).unwrap_err(); + assert!(err.to_string().contains("unknown cache backend kind")); + } + + #[test] + fn test_backend_kind_is_normalized() { + let _guard = RegistryGuard::new(); + register_backend("Echo.Backend", build_null).unwrap(); + let backend = build_from_config(&BackendConfig::new("echo.backend").unwrap()).unwrap(); + assert_eq!(Arc::strong_count(&backend), 1); + } + + #[test] + fn test_config_lookup_normalizes_direct_config() { + let _guard = RegistryGuard::new(); + fn build_echo(cfg: &BackendConfig) -> Result> { + assert_eq!(cfg.kind, "echo.backend"); + Ok(Arc::new(NullBackend)) + } + register_backend("echo.backend", build_echo).unwrap(); + let cfg = BackendConfig { + kind: "ECHO.Backend".to_string(), + options: HashMap::new(), + }; + build_from_config(&cfg).unwrap(); + } + + #[test] + fn test_invalid_backend_kind_errors() { + let err = register_backend("not a scheme", build_null).unwrap_err(); + assert!(err.to_string().contains("invalid character")); + let err = BackendConfig::new("1moka").unwrap_err(); + assert!(err.to_string().contains("must start with an ASCII letter")); + } + + #[test] + fn test_options_are_passed_through() { + let _guard = RegistryGuard::new(); + fn build_echo(cfg: &BackendConfig) -> Result> { + assert_eq!(cfg.options.get("capacity").map(String::as_str), Some("42")); + Ok(Arc::new(NullBackend)) + } + register_backend("echo", build_echo).unwrap(); + let cfg = BackendConfig::new("echo") + .unwrap() + .with_option("capacity", "42"); + build_from_config(&cfg).unwrap(); + } +} diff --git a/rust/lance-core/src/datatypes.rs b/rust/lance-core/src/datatypes.rs index 2f5c7b0680e..c345322abeb 100644 --- a/rust/lance-core/src/datatypes.rs +++ b/rust/lance-core/src/datatypes.rs @@ -24,7 +24,7 @@ pub use field::{ }; pub use schema::{ BlobHandling, FieldRef, OnMissing, Projectable, Projection, Schema, - escape_field_path_for_project, format_field_path, parse_field_path, + escape_field_path_for_project, format_field_path, format_field_path_minimal, parse_field_path, validate_fixed_size_list_dimensions, }; @@ -48,6 +48,84 @@ pub static BLOB_DESC_FIELD: LazyLock = LazyLock::new(|| { pub static BLOB_DESC_LANCE_FIELD: LazyLock = LazyLock::new(|| Field::try_from(&*BLOB_DESC_FIELD).unwrap()); +/// The minimal logical blob v2 fields accepted from writers. +/// +/// Logical values may also use [`BLOB_V2_LOGICAL_FIELDS`] when an external +/// object range is present. +pub static BLOB_V2_LOGICAL_MINIMAL_FIELDS: LazyLock = LazyLock::new(|| { + Fields::from(vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ]) +}); + +/// The complete logical blob v2 fields used for writer input and rewrite output. +/// +/// `position` and `size` are an optional range within the external object named +/// by `uri`; when present, `size` must be greater than zero. Every non-null row +/// must set exactly one of `data` and `uri`. These fields do not describe +/// Lance-managed data, packed, or dedicated storage. +pub static BLOB_V2_LOGICAL_FIELDS: LazyLock = LazyLock::new(|| { + let mut fields = BLOB_V2_LOGICAL_MINIMAL_FIELDS + .iter() + .cloned() + .collect::>(); + fields.extend([ + Arc::new(ArrowField::new("position", DataType::UInt64, true)), + Arc::new(ArrowField::new("size", DataType::UInt64, true)), + ]); + Fields::from(fields) +}); + +/// The complete logical blob v2 struct type. +pub static BLOB_V2_LOGICAL_TYPE: LazyLock = + LazyLock::new(|| DataType::Struct(BLOB_V2_LOGICAL_FIELDS.clone())); + +/// Writer-prepared blob v2 fields consumed by the structural encoder. +/// +/// The populated fields depend on [`BlobKind`]: +/// +/// - [`BlobKind::Inline`] carries `data`; the encoder derives the stored +/// `position` and `size` from the out-of-line buffer it creates. +/// - [`BlobKind::Packed`] carries `blob_id`, `position`, and `blob_size`. +/// - [`BlobKind::Dedicated`] carries `blob_id` and `blob_size`; its stored +/// `position` is zero. +/// - [`BlobKind::External`] carries `uri`, optional `blob_id`, `position`, and +/// `blob_size`. A zero `blob_size` is resolved to the complete external object +/// length when read. +/// +/// `blob_size` is distinct from the logical `size`, which is only an optional +/// external-object range before preparation. For external blobs, `uri` is +/// normalized into the stable stored `blob_uri` field. +pub static BLOB_V2_PREPARED_FIELDS: LazyLock = LazyLock::new(|| { + Fields::from(vec![ + ArrowField::new("kind", DataType::UInt8, true), + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ArrowField::new("blob_id", DataType::UInt32, true), + ArrowField::new("blob_size", DataType::UInt64, true), + ArrowField::new("position", DataType::UInt64, true), + ]) +}); + +/// The writer-prepared blob v2 struct type. +pub static BLOB_V2_PREPARED_TYPE: LazyLock = + LazyLock::new(|| DataType::Struct(BLOB_V2_PREPARED_FIELDS.clone())); + +/// Stored blob v2 descriptor fields. +/// +/// These field names are part of the stable file format. Their meaning depends +/// on `kind`: +/// +/// - [`BlobKind::Inline`]: `position` and `size` locate an out-of-line buffer in +/// the Lance data file. +/// - [`BlobKind::Packed`]: `blob_id` identifies a shared packed blob file, and +/// `position` and `size` locate a range within it. +/// - [`BlobKind::Dedicated`]: `blob_id` identifies a dedicated raw blob file, +/// `position` is zero, and `size` is the complete file length. +/// - [`BlobKind::External`]: `blob_uri` and `blob_id` identify the object, while +/// `position` and `size` select a range. A zero `size` is resolved to the +/// object's complete length when read. pub static BLOB_V2_DESC_FIELDS: LazyLock = LazyLock::new(|| { Fields::from(vec![ ArrowField::new("kind", DataType::UInt8, false), @@ -71,25 +149,95 @@ pub static BLOB_V2_DESC_FIELD: LazyLock = LazyLock::new(|| { pub static BLOB_V2_DESC_LANCE_FIELD: LazyLock = LazyLock::new(|| Field::try_from(&*BLOB_V2_DESC_FIELD).unwrap()); -/// Blob v2 user-view struct fields used by internal rewrite paths. -/// -/// This schema converts the descriptor view back into the write-side view used -/// by blob compaction. -pub static BLOB_V2_USER_FIELDS: LazyLock = LazyLock::new(|| { - Fields::from(vec![ - ArrowField::new("data", DataType::LargeBinary, true), - ArrowField::new("uri", DataType::Utf8, true), - ArrowField::new("position", DataType::UInt64, true), - ArrowField::new("size", DataType::UInt64, true), - ]) -}); +/// The in-memory representation of a blob v2 struct. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BlobV2Layout { + /// Writer input or rewrite output. + /// + /// Both the minimal `data, uri` fields and the complete + /// `data, uri, position, size` fields have this layout. + Logical, + /// Kind-aware writer intermediate consumed by the structural encoder. + Prepared, + /// Stable descriptor stored in Lance files and returned by descriptor scans. + Descriptor, +} -/// Blob v2 user-view struct type used by internal rewrite paths. -/// -/// This schema converts the descriptor view back into the write-side view used -/// by blob compaction. -pub static BLOB_V2_USER_TYPE: LazyLock = - LazyLock::new(|| DataType::Struct(BLOB_V2_USER_FIELDS.clone())); +impl BlobV2Layout { + /// Classify blob v2 child fields by name, type, order, and layout-specific + /// nullability requirements. + /// + /// Child metadata is not part of the representation. The complete logical + /// layout also accepts non-nullable `position` and `size` fields, matching + /// the existing writer-input contract. Descriptor child nullability is + /// ignored because it changed across released schemas; row nullness has + /// been represented by either parent struct validity or a nullable `kind` + /// child. + pub fn classify(fields: &Fields) -> Option { + if logical_blob_v2_fields_match(fields) { + Some(Self::Logical) + } else if blob_v2_fields_match(fields, &BLOB_V2_PREPARED_FIELDS, true) { + Some(Self::Prepared) + } else if blob_v2_fields_match(fields, &BLOB_V2_DESC_FIELDS, false) { + Some(Self::Descriptor) + } else { + None + } + } +} + +impl fmt::Display for BlobV2Layout { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Logical => write!(f, "logical"), + Self::Prepared => write!(f, "prepared"), + Self::Descriptor => write!(f, "descriptor"), + } + } +} + +fn blob_v2_field_matches( + actual: &ArrowField, + expected: &ArrowField, + compare_nullability: bool, +) -> bool { + actual.name() == expected.name() + && actual.data_type() == expected.data_type() + && (!compare_nullability || actual.is_nullable() == expected.is_nullable()) +} + +fn blob_v2_fields_match(actual: &Fields, expected: &Fields, compare_nullability: bool) -> bool { + actual.len() == expected.len() + && actual + .iter() + .zip(expected.iter()) + .all(|(actual, expected)| { + blob_v2_field_matches(actual.as_ref(), expected.as_ref(), compare_nullability) + }) +} + +fn logical_blob_v2_fields_match(fields: &Fields) -> bool { + if blob_v2_fields_match(fields, &BLOB_V2_LOGICAL_MINIMAL_FIELDS, true) { + return true; + } + fields.len() == BLOB_V2_LOGICAL_FIELDS.len() + && fields + .iter() + .zip(BLOB_V2_LOGICAL_FIELDS.iter()) + .enumerate() + .all(|(index, (actual, expected))| { + blob_v2_field_matches(actual.as_ref(), expected.as_ref(), index < 2) + }) +} + +/// Deprecated name for [`BLOB_V2_LOGICAL_FIELDS`]. +#[deprecated(note = "use BLOB_V2_LOGICAL_FIELDS")] +pub use self::BLOB_V2_LOGICAL_FIELDS as BLOB_V2_USER_FIELDS; + +/// Deprecated name for [`BLOB_V2_LOGICAL_TYPE`]. +#[deprecated(note = "use BLOB_V2_LOGICAL_TYPE")] +pub use self::BLOB_V2_LOGICAL_TYPE as BLOB_V2_USER_TYPE; pub const BLOB_LOGICAL_TYPE: &str = "blob"; @@ -462,3 +610,79 @@ impl TryFrom for BlobKind { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_classify_blob_v2_layouts() { + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_LOGICAL_MINIMAL_FIELDS), + Some(BlobV2Layout::Logical) + ); + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_LOGICAL_FIELDS), + Some(BlobV2Layout::Logical) + ); + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_PREPARED_FIELDS), + Some(BlobV2Layout::Prepared) + ); + assert_eq!( + BlobV2Layout::classify(&BLOB_V2_DESC_FIELDS), + Some(BlobV2Layout::Descriptor) + ); + } + + #[test] + fn test_classify_blob_v2_layout_uses_structural_contract() { + let logical_with_required_range = Fields::from(vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ArrowField::new("position", DataType::UInt64, false), + ArrowField::new("size", DataType::UInt64, false), + ]); + assert_eq!( + BlobV2Layout::classify(&logical_with_required_range), + Some(BlobV2Layout::Logical) + ); + + let prepared_with_child_metadata = + Fields::from( + BLOB_V2_PREPARED_FIELDS + .iter() + .map(|field| { + Arc::new(field.as_ref().clone().with_metadata(HashMap::from([( + "source".to_string(), + "test".to_string(), + )]))) + }) + .collect::>(), + ); + assert_eq!( + BlobV2Layout::classify(&prepared_with_child_metadata), + Some(BlobV2Layout::Prepared) + ); + + let nullable_descriptor = Fields::from( + BLOB_V2_DESC_FIELDS + .iter() + .map(|field| Arc::new(field.as_ref().clone().with_nullable(true))) + .collect::>(), + ); + assert_eq!( + BlobV2Layout::classify(&nullable_descriptor), + Some(BlobV2Layout::Descriptor) + ); + + let malformed_descriptor = Fields::from(vec![ + ArrowField::new("kind", DataType::UInt8, false), + ArrowField::new("position", DataType::UInt64, false), + ArrowField::new("size", DataType::UInt32, false), + ArrowField::new("blob_id", DataType::UInt32, false), + ArrowField::new("blob_uri", DataType::Utf8, false), + ]); + assert_eq!(BlobV2Layout::classify(&malformed_descriptor), None); + } +} diff --git a/rust/lance-core/src/datatypes/field.rs b/rust/lance-core/src/datatypes/field.rs index 9f06d421949..e734e69889f 100644 --- a/rust/lance-core/src/datatypes/field.rs +++ b/rust/lance-core/src/datatypes/field.rs @@ -30,7 +30,8 @@ use super::{ }; use crate::{ Error, Result, - datatypes::{BLOB_DESC_LANCE_FIELD, BLOB_V2_DESC_LANCE_FIELD}, + datatypes::{BLOB_DESC_LANCE_FIELD, BLOB_V2_DESC_LANCE_FIELD, BlobV2Layout}, + utils::parse::str_is_truthy, }; /// Use this config key in Arrow field metadata to indicate a column is a part of the primary key. @@ -58,6 +59,8 @@ pub const LANCE_UNENFORCED_CLUSTERING_KEY_POSITION: &str = /// The value should be non-negative i32 value. Any negative value will be seen as -1. pub const LANCE_FIELD_ID_KEY: &str = "lance:field_id"; +const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"]; + fn has_blob_v2_extension(field: &ArrowField) -> bool { field .metadata() @@ -269,24 +272,15 @@ impl Field { } pub fn apply_projection(&self, projection: &Projection) -> Option { - // Map fields encode their physical layout as a single child entries - // struct (`Struct`) whose presence is required for the - // parent to be readable — we never want to filter into that subtree. - // But the parent field itself is still subject to selection: if the - // caller didn't ask for this Map column, drop it like any other - // non-selected leaf. Without this early return the unconditional - // children clone would keep `children.is_empty() == false` forever - // and every Map column in the schema would survive every projection, - // pulling tens-of-bytes-per-row of unrelated data through downstream - // operators (notably `SortExec` in scalar-index training, where it - // was responsible for >100 GiB external-sort spills on real-world - // tables). - if self.logical_type.is_map() && !projection.contains_field_id(self.id) { + // Maps and blob descriptors are atomic physical layouts. Map children + // must remain together, while projected blob descriptor children may + // have synthetic IDs that cannot be selected independently. + let is_atomic_layout = self.logical_type.is_map() || self.is_blob(); + if is_atomic_layout && !projection.contains_field_id(self.id) { return None; } - let children = if self.logical_type.is_map() { - // Map field is selected: keep all children intact. + let children = if is_atomic_layout { self.children.clone() } else { self.children @@ -374,12 +368,22 @@ impl Field { self_name )); } - let children_differences = explain_fields_difference( - &self.children, - &expected.children, - options, - Some(&self_name), - ); + let children_differences = + if let Some(shared_child_count) = self.blob_v2_logical_shared_child_count(expected) { + explain_fields_difference( + &self.children[..shared_child_count], + &expected.children[..shared_child_count], + options, + Some(&self_name), + ) + } else { + explain_fields_difference( + &self.children, + &expected.children, + options, + Some(&self_name), + ) + }; if !children_differences.is_empty() { let children_differences = format!( "`{}` had mismatched children: {}", @@ -417,10 +421,20 @@ impl Field { } pub fn compare_with_options(&self, expected: &Self, options: &SchemaCompareOptions) -> bool { + let children_match = self + .blob_v2_logical_shared_child_count(expected) + .map(|shared_child_count| { + compare_fields( + &self.children[..shared_child_count], + &expected.children[..shared_child_count], + options, + ) + }) + .unwrap_or_else(|| compare_fields(&self.children, &expected.children, options)); self.name == expected.name && self.logical_type == expected.logical_type && Self::compare_nullability(expected.nullable, self.nullable, options) - && compare_fields(&self.children, &expected.children, options) + && children_match && (!options.compare_field_ids || self.id == expected.id) && (!options.compare_dictionary || self.dictionary == expected.dictionary) && (!options.compare_metadata || self.metadata == expected.metadata) @@ -549,6 +563,40 @@ impl Field { .get(ARROW_EXT_NAME_KEY) .map(|name| name == BLOB_V2_EXT_NAME) .unwrap_or(false) + || self.is_blob_v2_descriptor() + } + + fn blob_v2_layout(&self) -> Option { + if self.extension_name() != Some(BLOB_V2_EXT_NAME) { + return None; + } + let DataType::Struct(fields) = self.data_type() else { + return None; + }; + BlobV2Layout::classify(&fields) + } + + fn blob_v2_logical_shared_child_count(&self, other: &Self) -> Option { + if self.blob_v2_layout() == Some(BlobV2Layout::Logical) + && other.blob_v2_layout() == Some(BlobV2Layout::Logical) + { + Some(self.children.len().min(other.children.len())) + } else { + None + } + } + + fn is_blob_v2_descriptor(&self) -> bool { + self.metadata.contains_key(BLOB_META_KEY) + && self.logical_type == BLOB_V2_DESC_LANCE_FIELD.logical_type + && self.children.len() == BLOB_V2_DESC_LANCE_FIELD.children.len() + && self + .children + .iter() + .zip(BLOB_V2_DESC_LANCE_FIELD.children.iter()) + .all(|(child, expected)| { + child.name == expected.name && child.data_type() == expected.data_type() + }) } // Blob columns intentionally have two schema representations: @@ -575,6 +623,31 @@ impl Field { } } + /// Convert a blob field to the materialized binary payload view. + /// + /// The field keeps its name and id but uses `LargeBinary` with no children. + /// Blob v2 fields retain their extension marker internally so scan planning + /// can recognize the binary view before exposing a plain Arrow binary field. + pub fn binary_blob_mut(&mut self) { + if !self.is_blob() { + return; + } + let is_blob_v2 = self.is_blob_v2(); + + self.logical_type = LogicalType::try_from(&DataType::LargeBinary) + .expect("LargeBinary is always a valid logical type"); + self.children.clear(); + self.encoding = Some(Encoding::VarBinary); + if is_blob_v2 { + self.metadata.remove(BLOB_META_KEY); + for key in PACKED_KEYS { + self.metadata.remove(key); + } + self.metadata + .insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string()); + } + } + /// Convert blob v2 fields in this field tree to their descriptor view. pub fn unload_blobs_recursive(&mut self) { if self.is_blob_v2() { @@ -812,6 +885,13 @@ impl Field { } if self.is_blob() != other.is_blob() { + if ignore_types { + return Ok(if self.id >= 0 { + self.clone() + } else { + other.clone() + }); + } return Err(Error::arrow(format!( "Attempt to intersect blob and non-blob field: {}", self.name @@ -847,7 +927,7 @@ impl Field { .iter() .filter_map(|c| { if let Some(other_child) = other.child(&c.name) { - let intersection = c.intersection(other_child).ok()?; + let intersection = c.do_intersection(other_child, ignore_types).ok()?; Some(intersection) } else { None @@ -1038,11 +1118,10 @@ impl Field { // Check if field has metadata `packed` set to true, this check is case insensitive. pub fn is_packed_struct(&self) -> bool { - const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"]; PACKED_KEYS.iter().any(|key| { self.metadata .get(*key) - .map(|value| value.eq_ignore_ascii_case("true")) + .map(|value| str_is_truthy(value)) .unwrap_or(false) }) } @@ -1149,7 +1228,7 @@ impl TryFrom<&ArrowField> for Field { // Backward compatibility: use 0 for legacy boolean flag metadata .get(LANCE_UNENFORCED_PRIMARY_KEY) - .filter(|s| matches!(s.to_lowercase().as_str(), "true" | "1" | "yes")) + .filter(|s| str_is_truthy(s)) .map(|_| 0) }); let unenforced_clustering_key_position = metadata @@ -1237,6 +1316,101 @@ mod tests { use lance_arrow::BLOB_META_KEY; use std::collections::HashMap; + use crate::datatypes::{BLOB_V2_LOGICAL_FIELDS, BLOB_V2_LOGICAL_MINIMAL_FIELDS}; + + fn blob_v2_logical_field(children: Fields) -> Field { + ArrowField::new("blob", DataType::Struct(children), true) + .with_metadata(HashMap::from([( + ARROW_EXT_NAME_KEY.to_string(), + BLOB_V2_EXT_NAME.to_string(), + )])) + .try_into() + .unwrap() + } + + #[test] + fn blob_v2_logical_shapes_are_compatible() { + let minimal = blob_v2_logical_field(BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone()); + let complete = blob_v2_logical_field(BLOB_V2_LOGICAL_FIELDS.clone()); + let complete_required_range = blob_v2_logical_field(Fields::from( + BLOB_V2_LOGICAL_FIELDS + .iter() + .enumerate() + .map(|(index, field)| Arc::new(field.as_ref().clone().with_nullable(index < 2))) + .collect::>(), + )); + let options = SchemaCompareOptions::default(); + + for complete_shape in [&complete, &complete_required_range] { + assert!(minimal.compare_with_options(complete_shape, &options)); + assert!(complete_shape.compare_with_options(&minimal, &options)); + assert_eq!(minimal.explain_difference(complete_shape, &options), None); + assert_eq!(complete_shape.explain_difference(&minimal, &options), None); + } + + assert!(!complete.compare_with_options(&complete_required_range, &options)); + let ignore_nullability = SchemaCompareOptions { + compare_nullability: NullabilityComparison::Ignore, + ..Default::default() + }; + assert!(complete.compare_with_options(&complete_required_range, &ignore_nullability)); + + let complete_with_child_metadata = blob_v2_logical_field(Fields::from( + BLOB_V2_LOGICAL_FIELDS + .iter() + .enumerate() + .map(|(index, field)| { + let field = if index == 0 { + field.as_ref().clone().with_metadata(HashMap::from([( + "source".to_string(), + "test".to_string(), + )])) + } else { + field.as_ref().clone() + }; + Arc::new(field) + }) + .collect::>(), + )); + let compare_metadata = SchemaCompareOptions { + compare_metadata: true, + ..Default::default() + }; + assert!(!complete.compare_with_options(&complete_with_child_metadata, &compare_metadata)); + + let nested_minimal: Field = ArrowField::new( + "outer", + DataType::Struct(Fields::from(vec![ArrowField::from(&minimal)])), + true, + ) + .try_into() + .unwrap(); + let nested_complete: Field = ArrowField::new( + "outer", + DataType::Struct(Fields::from(vec![ArrowField::from(&complete)])), + true, + ) + .try_into() + .unwrap(); + assert!(nested_minimal.compare_with_options(&nested_complete, &options)); + assert!(nested_complete.compare_with_options(&nested_minimal, &options)); + } + + #[test] + fn malformed_blob_v2_logical_shapes_remain_incompatible() { + let minimal = blob_v2_logical_field(BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone()); + let malformed = blob_v2_logical_field(Fields::from(vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::LargeUtf8, true), + ])); + let options = SchemaCompareOptions::default(); + + assert!(!minimal.compare_with_options(&malformed, &options)); + assert!(!malformed.compare_with_options(&minimal, &options)); + assert!(minimal.explain_difference(&malformed, &options).is_some()); + assert!(malformed.explain_difference(&minimal, &options).is_some()); + } + #[test] fn arrow_field_to_field_metadata() { let mut metadata = HashMap::new(); @@ -1847,6 +2021,14 @@ mod tests { #[test] fn blob_unloaded_mut_selects_layout_from_metadata() { let metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); + let mut binary_field: Field = ArrowField::new("blob", DataType::LargeBinary, true) + .with_metadata(metadata.clone()) + .try_into() + .unwrap(); + binary_field.binary_blob_mut(); + assert!(binary_field.metadata.contains_key(BLOB_META_KEY)); + assert!(!binary_field.is_blob_v2()); + let mut field: Field = ArrowField::new("blob", DataType::LargeBinary, true) .with_metadata(metadata) .try_into() @@ -1854,6 +2036,12 @@ mod tests { field.unloaded_mut(); assert_eq!(field.children.len(), 2); assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type); + assert!(field.is_blob()); + assert!(!field.is_blob_v2()); + field.unloaded_mut(); + assert_eq!(field.children.len(), 2); + assert_eq!(field.logical_type, BLOB_DESC_LANCE_FIELD.logical_type); + assert!(!field.is_blob_v2()); let metadata = HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]); @@ -1874,6 +2062,12 @@ mod tests { field.unloaded_mut(); assert_eq!(field.children.len(), 5); assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); + assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY)); + assert!(field.is_blob_v2()); + field.unloaded_mut(); + assert_eq!(field.children.len(), 5); + assert_eq!(field.logical_type, BLOB_V2_DESC_LANCE_FIELD.logical_type); + assert!(!field.metadata.contains_key(ARROW_EXT_NAME_KEY)); } #[test] @@ -1944,4 +2138,43 @@ mod tests { .unwrap(); assert_eq!(unloaded_projected, unloaded); } + + #[test] + fn blob_descriptor_projection_preserves_synthetic_children() { + let metadata = + HashMap::from([(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string())]); + let mut blob: Field = ArrowField::new( + "blob", + DataType::Struct( + vec![ + ArrowField::new("data", DataType::LargeBinary, true), + ArrowField::new("uri", DataType::Utf8, true), + ] + .into(), + ), + true, + ) + .with_metadata(metadata) + .try_into() + .unwrap(); + let mut next_id = 0; + blob.set_id(-1, &mut next_id); + + let schema = Arc::new(crate::datatypes::Schema { + fields: vec![blob], + metadata: HashMap::new(), + }); + let descriptor_schema = Projection::full(schema) + .with_blob_handling(crate::datatypes::BlobHandling::BlobsDescriptions) + .to_bare_schema(); + assert!( + descriptor_schema.fields[0] + .children + .iter() + .all(|child| child.id == -1) + ); + + let projected = Projection::full(Arc::new(descriptor_schema)).to_bare_schema(); + assert_eq!(projected.fields[0].children.len(), 5); + } } diff --git a/rust/lance-core/src/datatypes/schema.rs b/rust/lance-core/src/datatypes/schema.rs index 7f2cbc02f07..2e328c88d24 100644 --- a/rust/lance-core/src/datatypes/schema.rs +++ b/rust/lance-core/src/datatypes/schema.rs @@ -339,17 +339,10 @@ impl Schema { ))); } - let column_path = self - .field_ancestry_by_id(field.id) - .unwrap() - .iter() - .map(|f| f.name.as_str()) - .collect::>() - .join("."); - if !seen_names.insert(column_path.clone()) { + if !seen_names.insert(field.name.as_str()) { return Err(Error::schema(format!( "Duplicate field name \"{}\" in schema:\n {:#?}", - column_path, self + field.name, self ))); } } @@ -726,7 +719,8 @@ impl Schema { /// Merge this schema from the other schema. /// /// After merging, the field IDs from `other` schema will be reassigned, - /// following the fields in `self`. + /// following the fields in `self`. Schema metadata is combined, with values + /// from `self` taking precedence when both schemas contain the same key. pub fn merge>(&self, other: S) -> Result { let mut other: Self = other.try_into()?; other.reset_id(); @@ -747,12 +741,12 @@ impl Schema { merged_fields.push(field.clone()); } } - let metadata = self - .metadata - .iter() - .chain(other.metadata.iter()) - .map(|(k, v)| (k.clone(), v.clone())) - .collect(); + let mut metadata = other.metadata; + metadata.extend( + self.metadata + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); let schema = Self { fields: merged_fields, metadata, @@ -762,6 +756,9 @@ impl Schema { /// Returns the properly formatted path from root to the field. /// Field names containing dots are quoted (e.g., struct.`field.with.dot`) + /// + /// The result is suitable for SQL parsing. For a human-readable path + /// (e.g. for display in index metadata), use [`Self::field_path_minimal`]. pub fn field_path(&self, field_id: i32) -> Result { self.field_ancestry_by_id(field_id) .map(|ancestry| { @@ -773,6 +770,57 @@ impl Schema { }) } + /// Returns the path from root to the field using *minimal* quoting. + /// + /// A segment is wrapped in backticks only when it contains a character that + /// [`parse_field_path`] treats specially (a `.` separator or a `` ` `` quote); + /// any other character — including hyphens — is left bare. Unlike + /// [`Self::field_path`] (which quotes for SQL-expression safety and so wraps + /// e.g. `my-col` in backticks), the result here is both human-readable and + /// round-trips back through `parse_field_path`, so it is safe to feed into + /// field-path APIs such as `drop_columns` / `update_field_metadata`. + /// + /// This is what should be exposed as the column name in index metadata. + /// + /// ``` + /// use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + /// use lance_core::datatypes::{parse_field_path, Schema}; + /// + /// let arrow = ArrowSchema::new(vec![ + /// Field::new("my-col", DataType::Int32, false), + /// Field::new( + /// "parent", + /// DataType::Struct(Fields::from(vec![Field::new("child.x", DataType::Int32, true)])), + /// true, + /// ), + /// ]); + /// let schema = Schema::try_from(&arrow).unwrap(); + /// + /// // A hyphen is not special to `parse_field_path`, so it is left bare + /// // (unlike `field_path`, which would quote it as `` `my-col` ``). + /// let hyphen_id = schema.field("my-col").unwrap().id; + /// assert_eq!(schema.field_path_minimal(hyphen_id).unwrap(), "my-col"); + /// + /// // A `.` in a segment forces quoting so the path still round-trips. + /// let dotted_id = schema.field("parent").unwrap().children[0].id; + /// let path = schema.field_path_minimal(dotted_id).unwrap(); + /// assert_eq!(path, "parent.`child.x`"); + /// assert_eq!( + /// parse_field_path(&path).unwrap(), + /// vec!["parent".to_string(), "child.x".to_string()], + /// ); + /// ``` + pub fn field_path_minimal(&self, field_id: i32) -> Result { + self.field_ancestry_by_id(field_id) + .map(|ancestry| { + let field_refs: Vec<&str> = ancestry.iter().map(|f| f.name.as_str()).collect(); + format_field_path_minimal(&field_refs) + }) + .ok_or_else(|| { + Error::index(format!("Could not find field ancestry for id {}", field_id)) + }) + } + pub fn verify_primary_key(&self) -> Result<()> { let pk = self.unenforced_primary_key(); for pk_col in pk.into_iter() { @@ -1071,6 +1119,17 @@ pub enum BlobHandling { } impl BlobHandling { + fn should_load_binary(&self, field: &Field) -> bool { + if !field.is_blob() { + return false; + } + match self { + Self::AllBinary => true, + Self::SomeBlobsBinary(set) | Self::SomeBinary(set) => set.contains(&(field.id as u32)), + Self::BlobsDescriptions | Self::AllDescriptions => false, + } + } + fn should_unload(&self, field: &Field) -> bool { // Blob v2 columns are Structs, so we need to treat any blob-marked field as unloadable // even if the physical data type is not binary-like. @@ -1086,10 +1145,34 @@ impl BlobHandling { } } + /// Whether `field` will be projected as a lightweight blob *description* + /// (offset + size) rather than its full binary value under this handling. + /// + /// A description is tiny and cheap to read eagerly; the full binary value is + /// not. Materialization heuristics use this to decide early vs late loading. + pub fn returns_description(&self, field: &Field) -> bool { + self.should_unload(field) + } + + /// Apply this blob handling policy to a projected field tree. + /// + /// Blob descriptor modes convert blob leaves to descriptor views. Binary + /// modes convert selected blob leaves to `LargeBinary`. Non-blob nested + /// fields are preserved while their children are handled recursively. pub fn unload_if_needed(&self, mut field: Field) -> Field { + if self.should_load_binary(&field) { + field.binary_blob_mut(); + return field; + } if self.should_unload(&field) { field.unloaded_mut(); + return field; } + field.children = field + .children + .into_iter() + .map(|child| self.unload_if_needed(child)) + .collect(); field } } @@ -1596,6 +1679,50 @@ pub fn format_field_path(fields: &[&str]) -> String { .join(".") } +/// Like [`format_field_path`], but quotes a segment only when strictly required +/// for the result to round-trip back through [`parse_field_path`]. +/// +/// `parse_field_path` only treats `.` (segment separator) and `` ` `` (quote) +/// specially, so those are the only characters that force quoting here. Notably +/// a hyphen does NOT force quoting (`my-col` stays `my-col`), unlike +/// `format_field_path` which quotes any non-identifier character for +/// SQL-expression safety. Use this for human-readable, round-trippable paths +/// (e.g. column names in index metadata); use `format_field_path` when the +/// result will be embedded in a SQL expression. +/// +/// ``` +/// use lance_core::datatypes::{format_field_path_minimal, parse_field_path}; +/// +/// // Plain identifiers and hyphenated names are left bare. +/// assert_eq!(format_field_path_minimal(&["parent", "my-col"]), "parent.my-col"); +/// // A `.` in a segment forces quoting. +/// assert_eq!(format_field_path_minimal(&["parent", "child.x"]), "parent.`child.x`"); +/// // Embedded backticks are escaped by doubling them. +/// assert_eq!(format_field_path_minimal(&["child`x"]), "`child``x`"); +/// +/// // Whatever it produces round-trips back through `parse_field_path`. +/// for segments in [vec!["parent", "my-col"], vec!["parent", "child.x"], vec!["child`x"]] { +/// let path = format_field_path_minimal(&segments); +/// assert_eq!(parse_field_path(&path).unwrap(), segments); +/// } +/// ``` +pub fn format_field_path_minimal(fields: &[&str]) -> String { + fields + .iter() + .map(|field| { + let needs_quoting = field.contains('.') || field.contains('`'); + if needs_quoting { + // Escape embedded backticks by doubling them, matching parse_field_path. + let escaped = field.replace('`', "``"); + format!("`{}`", escaped) + } else { + field.to_string() + } + }) + .collect::>() + .join(".") +} + /// Escape a field path for project /// /// Parses the field path and formats it for SQL usage. @@ -1736,6 +1863,11 @@ mod tests { vec!["simple".to_string()] ); + assert_eq!( + parse_field_path("tags[*]").unwrap(), + vec!["tags[*]".to_string()] + ); + // Quoted field at the end assert_eq!( parse_field_path("parent.`field.with.dot`").unwrap(), @@ -1772,6 +1904,22 @@ mod tests { ); } + #[test] + fn test_validate_top_level_names_without_field_id_lookup() { + let mut first = Field::new_arrow("first", DataType::Int32, false).unwrap(); + first.id = 0; + let mut second = Field::new_arrow("second", DataType::Int32, false).unwrap(); + second.id = 0; + let schema = Schema { + fields: vec![first, second], + metadata: HashMap::new(), + }; + + let error = schema.validate().unwrap_err(); + assert!(matches!(&error, Error::Schema { .. })); + assert!(error.to_string().contains("Duplicate field id 0")); + } + #[test] fn test_resolve_quoted_fields() { // Test that top-level fields with dots are rejected during validation @@ -2202,6 +2350,35 @@ mod tests { assert_eq!(merged.max_field_id(), Some(9)); } + #[test] + fn test_merge_schema_metadata_preserves_self_values() { + let schema = Schema { + metadata: HashMap::from([ + ("shared".to_string(), "left".to_string()), + ("left_only".to_string(), "left".to_string()), + ]), + ..Default::default() + }; + let other = Schema { + metadata: HashMap::from([ + ("shared".to_string(), "right".to_string()), + ("right_only".to_string(), "right".to_string()), + ]), + ..Default::default() + }; + + let merged = schema.merge(&other).unwrap(); + + assert_eq!( + merged.metadata, + HashMap::from([ + ("shared".to_string(), "left".to_string()), + ("left_only".to_string(), "left".to_string()), + ("right_only".to_string(), "right".to_string()), + ]) + ); + } + #[test] fn test_merge_arrow_schema() { let arrow_schema = ArrowSchema::new(vec![ @@ -2768,6 +2945,77 @@ mod tests { assert!(paths.contains(&"name".to_string())); } + #[test] + fn test_field_path_minimal() { + // A struct child whose own NAME contains a dot is the case that makes + // "just strip all backticks" wrong: it must stay quoted to round-trip. + let arrow_schema = ArrowSchema::new(vec![ + ArrowField::new("mycol", ArrowDataType::Int32, false), + ArrowField::new("my_col", ArrowDataType::Int32, false), + ArrowField::new("my-col", ArrowDataType::Int32, false), + ArrowField::new( + "parent", + ArrowDataType::Struct(ArrowFields::from(vec![ + ArrowField::new("child-field", ArrowDataType::Int32, true), + ArrowField::new("child.x", ArrowDataType::Int32, true), + ArrowField::new("child`x", ArrowDataType::Int32, true), + ])), + true, + ), + ]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + let id_of = |path: &str| schema.field(path).unwrap().id; + let child_id = |name: &str| { + schema + .field("parent") + .unwrap() + .children + .iter() + .find(|c| c.name == name) + .unwrap() + .id + }; + + // Plain identifiers: unchanged by either method. + assert_eq!(schema.field_path_minimal(id_of("mycol")).unwrap(), "mycol"); + assert_eq!( + schema.field_path_minimal(id_of("my_col")).unwrap(), + "my_col" + ); + + // Hyphen is NOT special to parse_field_path, so minimal quoting leaves it + // bare (field_path would quote it for SQL safety). + assert_eq!(schema.field_path(id_of("my-col")).unwrap(), "`my-col`"); + assert_eq!( + schema.field_path_minimal(id_of("my-col")).unwrap(), + "my-col" + ); + + // Nested hyphenated leaf: bare under minimal quoting. + assert_eq!( + schema.field_path_minimal(child_id("child-field")).unwrap(), + "parent.child-field" + ); + + // Nested leaf whose NAME contains a dot: MUST stay quoted so it + // round-trips through parse_field_path (this is the regression guard). + let dotted = schema.field_path_minimal(child_id("child.x")).unwrap(); + assert_eq!(dotted, "parent.`child.x`"); + assert_eq!( + parse_field_path(&dotted).unwrap(), + vec!["parent".to_string(), "child.x".to_string()] + ); + + // Nested leaf whose NAME contains a backtick: it must be quoted AND the + // backtick doubled so it round-trips through parse_field_path. + let backticked = schema.field_path_minimal(child_id("child`x")).unwrap(); + assert_eq!(backticked, "parent.`child``x`"); + assert_eq!( + parse_field_path(&backticked).unwrap(), + vec!["parent".to_string(), "child`x".to_string()] + ); + } + #[test] fn test_validate_rejects_zero_dimension_fixed_size_list() { // A zero dimension divides-by-zero further down the write path (#5102) diff --git a/rust/lance-core/src/error.rs b/rust/lance-core/src/error.rs index 2a6340da492..158af263933 100644 --- a/rust/lance-core/src/error.rs +++ b/rust/lance-core/src/error.rs @@ -8,6 +8,52 @@ use snafu::{IntoError as _, Location, Snafu}; type BoxedError = Box; +#[cfg(feature = "backtrace")] +mod backtrace_support { + use std::backtrace::Backtrace; + + use snafu::{AsBacktrace, GenerateImplicitData}; + + #[derive(Debug)] + pub struct MaybeBacktrace(pub Option); + + impl GenerateImplicitData for MaybeBacktrace { + fn generate() -> Self { + Self(>::generate()) + } + } + + impl AsBacktrace for MaybeBacktrace { + fn as_backtrace(&self) -> Option<&Backtrace> { + self.0.as_ref() + } + } +} + +#[cfg(not(feature = "backtrace"))] +mod backtrace_support { + use std::backtrace::Backtrace; + + use snafu::{AsBacktrace, GenerateImplicitData}; + + #[derive(Debug)] + pub struct MaybeBacktrace; + + impl GenerateImplicitData for MaybeBacktrace { + fn generate() -> Self { + Self + } + } + + impl AsBacktrace for MaybeBacktrace { + fn as_backtrace(&self) -> Option<&Backtrace> { + None + } + } +} + +use backtrace_support::MaybeBacktrace; + /// Error for when a requested field is not found in a schema. /// /// This error computes suggestions lazily (only when displayed) to avoid @@ -45,6 +91,41 @@ impl fmt::Display for FieldNotFoundError { impl std::error::Error for FieldNotFoundError {} +/// A manifest commit returned an error and its final outcome could not be +/// determined safely. +/// +/// This is wrapped in [`Error::Wrapped`] so Lance can expose a structured +/// source without adding a variant to the exhaustive public [`Error`] enum. +#[derive(Debug)] +pub struct CommitStatusUnknownError { + version: u64, + source: BoxedError, +} + +impl CommitStatusUnknownError { + /// Return the manifest version whose commit outcome is unknown. + pub fn version(&self) -> u64 { + self.version + } +} + +impl std::fmt::Display for CommitStatusUnknownError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Commit result for version {} is unknown: the commit may or may not have been \ + applied; check the table state before retrying: {}", + self.version, self.source + ) + } +} + +impl std::error::Error for CommitStatusUnknownError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + /// Allocates error on the heap and then places `e` into it. #[inline] pub fn box_error(e: impl std::error::Error + Send + Sync + 'static) -> BoxedError { @@ -81,18 +162,24 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Dataset already exists: {uri}, {location}"))] DatasetAlreadyExists { uri: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Append with different schema: {difference}, location: {location}"))] SchemaMismatch { difference: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Dataset at path {path} was not found: {source}, {location}"))] DatasetNotFound { @@ -100,6 +187,8 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Encountered corrupt file {path}: {source}, {location}"))] CorruptFile { @@ -107,13 +196,16 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, - // TODO: add backtrace? + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Not supported: {source}, {location}"))] NotSupported { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Commit conflict for version {version}: {source}, {location}"))] CommitConflict { @@ -121,12 +213,16 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Incompatible transaction: {source}, {location}"))] IncompatibleTransaction { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Retryable commit conflict for version {version}: {source}, {location}"))] RetryableCommitConflict { @@ -134,12 +230,16 @@ pub enum Error { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Too many concurrent writers. {message}, {location}"))] TooMuchWriteContention { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Operation timed out: {message}, {location}"))] Timeout { @@ -154,54 +254,72 @@ pub enum Error { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("A prerequisite task failed: {message}, {location}"))] PrerequisiteFailed { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Unprocessable: {message}, {location}"))] Unprocessable { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(Arrow): {message}, {location}"))] Arrow { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(Schema): {message}, {location}"))] Schema { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Not found: {uri}, {location}"))] NotFound { uri: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(IO): {source}, {location}"))] IO { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("LanceError(Index): {message}, {location}"))] Index { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Lance index not found: {identity}, {location}"))] IndexNotFound { identity: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Cannot infer storage location from: {message}"))] InvalidTableLocation { message: String }, @@ -209,21 +327,28 @@ pub enum Error { Stop, #[snafu(display("Wrapped error: {error}, {location}"))] Wrapped { + #[snafu(source)] error: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Cloned error: {message}, {location}"))] Cloned { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Query Execution error: {message}, {location}"))] Execution { message: String, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Ref is invalid: {message}"))] InvalidRef { message: String }, @@ -242,12 +367,16 @@ pub enum Error { minor_version: u16, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, #[snafu(display("Namespace error: {source}, {location}"))] Namespace { source: BoxedError, #[snafu(implicit)] location: Location, + #[snafu(implicit)] + backtrace: MaybeBacktrace, }, /// External error passed through from user code. /// @@ -280,14 +409,99 @@ pub enum Error { #[snafu(implicit)] location: Location, }, + /// A write was refused to keep the writer inside its memory budget. + /// + /// Unlike every other write error this one is *expected* under load and + /// carries no data loss: the write was never accepted, so a caller that + /// retries once the flush pipeline drains loses nothing. Callers should + /// surface it as a retryable "busy" signal (HTTP 503), not a failure. + /// Match via [`Error::is_backpressure`] rather than on the message. + #[snafu(display("Write rejected by backpressure: {message}, {location}"))] + Backpressure { + message: String, + #[snafu(implicit)] + location: Location, + }, } impl Error { + /// Returns the captured Rust backtrace, if available. + /// + /// Requires the `backtrace` feature to be enabled at compile time + /// and `RUST_BACKTRACE=1` at runtime. + #[cfg(feature = "backtrace")] + pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> { + match self { + Self::InvalidInput { backtrace, .. } + | Self::DatasetAlreadyExists { backtrace, .. } + | Self::SchemaMismatch { backtrace, .. } + | Self::DatasetNotFound { backtrace, .. } + | Self::CorruptFile { backtrace, .. } + | Self::NotSupported { backtrace, .. } + | Self::CommitConflict { backtrace, .. } + | Self::IncompatibleTransaction { backtrace, .. } + | Self::RetryableCommitConflict { backtrace, .. } + | Self::TooMuchWriteContention { backtrace, .. } + | Self::Internal { backtrace, .. } + | Self::PrerequisiteFailed { backtrace, .. } + | Self::Unprocessable { backtrace, .. } + | Self::Arrow { backtrace, .. } + | Self::Schema { backtrace, .. } + | Self::NotFound { backtrace, .. } + | Self::IO { backtrace, .. } + | Self::Index { backtrace, .. } + | Self::IndexNotFound { backtrace, .. } + | Self::Wrapped { backtrace, .. } + | Self::Cloned { backtrace, .. } + | Self::Execution { backtrace, .. } + | Self::VersionConflict { backtrace, .. } + | Self::Namespace { backtrace, .. } => { + use snafu::AsBacktrace; + backtrace.as_backtrace() + } + // Variants without a backtrace field — listed explicitly so that + // adding a new variant with a backtrace field triggers a compiler error. + Self::InvalidTableLocation { .. } + | Self::Stop + | Self::InvalidRef { .. } + | Self::RefConflict { .. } + | Self::RefNotFound { .. } + | Self::Cleanup { .. } + | Self::VersionNotFound { .. } + | Self::External { .. } + | Self::FieldNotFound { .. } + | Self::Timeout { .. } + | Self::DiskCapExceeded { .. } + | Self::Fenced { .. } + | Self::Backpressure { .. } => None, + } + } + + /// Returns the captured Rust backtrace, if available. + /// + /// Always returns `None` when the `backtrace` feature is not enabled. + #[cfg(not(feature = "backtrace"))] + pub fn backtrace(&self) -> Option<&std::backtrace::Backtrace> { + None + } + #[track_caller] pub fn corrupt_file(path: object_store::path::Path, message: impl Into) -> Self { CorruptFileSnafu { path }.into_error(message.into().into()) } + /// Reports a corrupt file when the caller only has a logical/section name + /// rather than the real file path (for example, a decoder that validates an + /// in-memory buffer and does not know where it came from). + /// + /// `name` is carried in the `path` field of the resulting [`Error::CorruptFile`] + /// variant and is NOT a filesystem path; callers that have the real path should + /// use [`Self::corrupt_file`] instead. + #[track_caller] + pub fn corrupt_file_named(name: &str, message: impl Into) -> Self { + Self::corrupt_file(object_store::path::Path::from(name), message) + } + #[track_caller] pub fn invalid_input(message: impl Into) -> Self { InvalidInputSnafu.into_error(message.into().into()) @@ -333,6 +547,23 @@ impl Error { } } + /// A write was refused because the writer is at its memory ceiling; the + /// data was never accepted. See [`Error::Backpressure`]. + #[track_caller] + pub fn backpressure(message: impl Into) -> Self { + BackpressureSnafu { + message: message.into(), + } + .build() + } + + /// Whether this is [`Error::Backpressure`] — i.e. a retryable "writer is + /// full" signal rather than a real failure. Prefer this over matching the + /// error message. + pub fn is_backpressure(&self) -> bool { + matches!(self, Self::Backpressure { .. }) + } + #[track_caller] pub fn io_source(source: BoxedError) -> Self { IOSnafu.into_error(source) @@ -367,9 +598,25 @@ impl Error { NotFoundSnafu { uri: uri.into() }.build() } + /// Return whether this error or one of its typed sources is a missing object. + pub fn is_not_found(&self) -> bool { + match self { + Self::NotFound { .. } => true, + Self::Wrapped { error, .. } + if error.downcast_ref::().is_some() => + { + false + } + Self::IO { source, .. } | Self::Wrapped { error: source, .. } => { + error_source_is_not_found(source.as_ref()) + } + _ => false, + } + } + #[track_caller] pub fn wrapped(error: BoxedError) -> Self { - WrappedSnafu { error }.build() + WrappedSnafu.into_error(error) } #[track_caller] @@ -498,6 +745,21 @@ impl Error { RetryableCommitConflictSnafu { version }.into_error(source) } + #[track_caller] + pub fn commit_status_unknown_source(version: u64, source: BoxedError) -> Self { + Self::wrapped(box_error(CommitStatusUnknownError { version, source })) + } + + /// Return whether this error represents a commit whose final outcome could + /// not be determined safely. + pub fn is_commit_status_unknown(&self) -> bool { + matches!( + self, + Self::Wrapped { error, .. } + if error.downcast_ref::().is_some() + ) + } + #[track_caller] pub fn incompatible_transaction_source(source: BoxedError) -> Self { IncompatibleTransactionSnafu.into_error(source) @@ -548,6 +810,17 @@ impl Error { } } +fn error_source_is_not_found(source: &(dyn std::error::Error + 'static)) -> bool { + if let Some(error) = source.downcast_ref::() { + return error.is_not_found(); + } + if let Some(error) = source.downcast_ref::() { + return matches!(error, object_store::Error::NotFound { .. }) + || std::error::Error::source(error).is_some_and(error_source_is_not_found); + } + source.source().is_some_and(error_source_is_not_found) +} + pub trait LanceOptionExt { /// Unwraps an option, returning an internal error if the option is None. /// @@ -611,7 +884,11 @@ impl From for Error { impl From for Error { #[track_caller] fn from(e: object_store::Error) -> Self { - Self::io_source(box_error(e)) + match e { + // source intentionally dropped; Error::NotFound carries only the path + object_store::Error::NotFound { path, .. } => Self::not_found(path), + other => Self::io_source(box_error(other)), + } } } @@ -717,6 +994,17 @@ impl From for Error { Self::not_supported_source(box_error(e)) } datafusion_common::DataFusionError::Execution(..) => Self::execution(e.to_string()), + datafusion_common::DataFusionError::Shared(shared) => { + // DataFusion shares an error across consumers (e.g. a join's + // build-side error fanned out to every probe partition) behind an + // `Arc`. If we are the sole owner we can recurse for full fidelity; + // otherwise the inner error can't be moved out, so we preserve its + // message under the execution category (its concrete type is lost). + match std::sync::Arc::try_unwrap(shared) { + Ok(inner) => Self::from(inner), + Err(shared) => Self::execution(shared.to_string()), + } + } datafusion_common::DataFusionError::External(source) => { // Try to downcast to lance_core::Error first match source.downcast::() { @@ -749,14 +1037,46 @@ pub fn get_caller_location() -> &'static std::panic::Location<'static> { /// Wrap an error in a new error type that implements Clone /// /// This is useful when two threads/streams share a common fallible source -/// The base error will always have the full error. Any cloned results will -/// only have Error::Cloned with the to_string of the base error. +/// Definite not-found errors preserve typed source-chain detection and their +/// human-readable representation. Timeout and I/O errors preserve their error +/// categories. Other cloned results use Error::Cloned with the string +/// representation of the base error. pub struct CloneableError(pub Error); +struct DisplayError(Error); + +impl fmt::Debug for DisplayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +impl fmt::Display for DisplayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl std::error::Error for DisplayError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} + impl Clone for CloneableError { #[track_caller] fn clone(&self) -> Self { - Self(Error::cloned(self.0.to_string())) + match &self.0 { + Error::NotFound { uri, .. } => Self(Error::wrapped(Box::new(DisplayError( + Error::not_found(uri.clone()), + )))), + error if error.is_not_found() => Self(Error::wrapped(Box::new(DisplayError( + Error::not_found(error.to_string()), + )))), + Error::Timeout { message, .. } => Self(Error::timeout(message.clone())), + Error::IO { source, .. } => Self(Error::io(source.to_string())), + error => Self(Error::cloned(error.to_string())), + } } } @@ -772,8 +1092,56 @@ impl From> for CloneableResult { #[cfg(test)] mod test { use super::*; + use std::error::Error as _; use std::fmt; + #[test] + fn cloneable_error_preserves_not_found_contract() { + let original = CloneableError(Error::not_found("metadata.lance")); + let cloned = original.clone(); + let cloned_again = cloned.clone(); + assert!(matches!(original.0, Error::NotFound { .. })); + assert!(cloned.0.is_not_found()); + assert!(cloned_again.0.is_not_found()); + assert!(cloned.0.to_string().to_lowercase().contains("not found")); + assert!( + cloned_again + .0 + .to_string() + .to_lowercase() + .contains("not found") + ); + assert!( + format!("{:?}", cloned.0) + .to_lowercase() + .contains("not found") + ); + assert!(cloned.0.source().is_some_and(|source| source.is::() + || source.source().is_some_and(|source| source.is::()))); + let downstream_error = Error::wrapped(Box::new(Error::io_source(Box::new( + object_store::Error::Generic { + store: "N/A", + source: Box::new(cloned.0), + }, + )))); + assert!(downstream_error.is_not_found()); + assert!( + format!("{downstream_error:?}") + .to_lowercase() + .contains("not found") + ); + + let original = CloneableError(Error::timeout("metadata read timed out")); + let cloned = original.clone(); + assert!(matches!(original.0, Error::Timeout { .. })); + assert!(matches!(cloned.0, Error::Timeout { .. })); + + let original = CloneableError(Error::io("metadata read was denied")); + let cloned = original.clone(); + assert!(matches!(original.0, Error::IO { .. })); + assert!(matches!(cloned.0, Error::IO { .. })); + } + #[test] fn test_caller_location_capture() { let current_fn = get_caller_location(); @@ -796,6 +1164,41 @@ mod test { } } + #[test] + fn test_caller_location_capture_not_found() { + let current_fn = get_caller_location(); + let f: Box Result<()>> = Box::new(|| { + Err(object_store::Error::NotFound { + path: "some/path".to_string(), + source: "not found".into(), + })?; + Ok(()) + }); + match f().unwrap_err() { + Error::NotFound { location, .. } => { + // +2 is the beginning of object_store::Error::NotFound... + assert_eq!(location.line(), current_fn.line() + 2, "{}", location) + } + #[allow(unreachable_patterns)] + other => panic!("expected NotFound, got {:?}", other), + } + } + + #[test] + fn test_object_store_not_found_converts_to_not_found() { + let os_err = object_store::Error::NotFound { + path: "test/path".to_string(), + source: "no such file".into(), + }; + let lance_err: Error = os_err.into(); + match lance_err { + Error::NotFound { uri, .. } => { + assert_eq!(uri, "test/path"); + } + other => panic!("Expected NotFound, got {:?}", other), + } + } + #[derive(Debug)] struct MyCustomError { code: i32, @@ -837,6 +1240,25 @@ mod test { assert!(matches!(converted, Error::IO { .. })); } + #[test] + fn test_commit_status_unknown_is_structured_without_masking_as_not_found() { + let error = Error::commit_status_unknown_source( + 42, + box_error(Error::not_found("temporarily invisible manifest")), + ); + + assert!(error.is_commit_status_unknown()); + assert!(!error.is_not_found()); + assert!(error.to_string().contains("version 42 is unknown")); + let Error::Wrapped { error, .. } = error else { + panic!("commit-status-unknown must use the semver-compatible wrapper") + }; + let status = error + .downcast_ref::() + .expect("wrapper must retain the typed commit status"); + assert_eq!(status.version(), 42); + } + #[test] fn test_external_error_creation() { let custom_err = MyCustomError { @@ -1048,4 +1470,68 @@ mod test { _ => panic!("Expected InvalidInput variant, got {:?}", recovered), } } + + #[test] + fn test_backtrace_accessor() { + // Verify that backtrace() returns the expected result based on feature state + let err = Error::io("test backtrace"); + let bt = err.backtrace(); + #[cfg(feature = "backtrace")] + { + // With the backtrace feature enabled, whether a backtrace is captured + // depends on the RUST_BACKTRACE env var at runtime. We just verify + // the accessor doesn't panic and returns a valid Option. + let _ = bt; + } + #[cfg(not(feature = "backtrace"))] + { + // Without the backtrace feature, this must always be None. + assert!(bt.is_none()); + } + } + + #[test] + fn test_backtrace_captured_when_feature_enabled() { + // Test that backtrace is actually captured when the feature is on and + // RUST_BACKTRACE=1 is set in the environment before the process starts. + // + // NOTE: std::backtrace::Backtrace caches the RUST_BACKTRACE env check, + // so set_var at runtime does not reliably enable capture. This test + // verifies the accessor works correctly in both cases: + // - If RUST_BACKTRACE=1 was set before the test binary started, we get Some. + // - If not, we get None (even with the feature on), which is expected. + #[cfg(feature = "backtrace")] + { + let err = Error::io("backtrace capture test"); + if std::env::var("RUST_BACKTRACE").is_ok() { + assert!( + err.backtrace().is_some(), + "Expected a backtrace when RUST_BACKTRACE=1 and backtrace feature is enabled" + ); + } + // When RUST_BACKTRACE is not set, backtrace() may return None even + // with the feature enabled — this is correct runtime gating behavior. + } + #[cfg(not(feature = "backtrace"))] + { + let err = Error::io("backtrace capture test"); + assert!(err.backtrace().is_none()); + } + } + + #[test] + fn test_backtrace_returns_none_for_variants_without_location() { + let err = Error::InvalidTableLocation { + message: "test".to_string(), + }; + assert!(err.backtrace().is_none()); + + let err = Error::InvalidRef { + message: "test".to_string(), + }; + assert!(err.backtrace().is_none()); + + let err = Error::Stop; + assert!(err.backtrace().is_none()); + } } diff --git a/rust/lance-core/src/lib.rs b/rust/lance-core/src/lib.rs index 32fb34ad5fe..0872dc97371 100644 --- a/rust/lance-core/src/lib.rs +++ b/rust/lance-core/src/lib.rs @@ -60,6 +60,9 @@ pub static ROW_CREATED_AT_VERSION_FIELD: LazyLock = /// - `_rowoffset`: The row offset /// - `_row_last_updated_at_version`: The version when the row was last updated /// - `_row_created_at_version`: The version when the row was created +/// +/// Write paths must reject a stored column named for one: the scanner injects +/// these itself, so a stored copy collides with the injected one on read. pub fn is_system_column(column_name: &str) -> bool { matches!( column_name, diff --git a/rust/lance-core/src/utils/aimd.rs b/rust/lance-core/src/utils/aimd.rs index 0cbae68ca71..13bca4aff91 100644 --- a/rust/lance-core/src/utils/aimd.rs +++ b/rust/lance-core/src/utils/aimd.rs @@ -25,7 +25,7 @@ use crate::Result; /// /// - initial_rate: 2000 req/s /// - min_rate: 1 req/s -/// - max_rate: 5000 req/s (0.0 disables ceiling) +/// - max_rate: 5000 req/s (0.0 disables the ceiling; must be finite otherwise) /// - decrease_factor: 0.5 (halve on throttle) /// - additive_increment: 300 req/s per success window /// - window_duration: 1 second @@ -101,6 +101,26 @@ impl AimdConfig { /// Validate that the configuration values are sensible. pub fn validate(&self) -> Result<()> { + // Reject NaN and infinity first. The sign and ordering checks below + // compare with `<`/`>`, which are `false` for NaN and for a `+inf` on + // a field with no opposing bound, so a non-finite rate would otherwise + // slip through and silently disable throttling (a NaN rate makes the + // token bucket refill to full on every acquire) or, with a zero burst + // capacity, panic in `Duration::from_secs_f64`. + for (name, value) in [ + ("initial_rate", self.initial_rate), + ("min_rate", self.min_rate), + ("max_rate", self.max_rate), + ("decrease_factor", self.decrease_factor), + ("additive_increment", self.additive_increment), + ("throttle_threshold", self.throttle_threshold), + ] { + if !value.is_finite() { + return Err(crate::Error::invalid_input(format!( + "{name} must be finite, got {value}" + ))); + } + } if self.initial_rate <= 0.0 { return Err(crate::Error::invalid_input(format!( "initial_rate must be positive, got {}", @@ -327,11 +347,47 @@ mod tests { AimdConfig::default().with_initial_rate(0.5).with_min_rate(1.0), "initial_rate (0.5) must not be below min_rate (1)" )] + #[case::nan_initial_rate( + AimdConfig::default().with_initial_rate(f64::NAN), + "initial_rate must be finite" + )] + #[case::inf_initial_rate( + AimdConfig::default().with_initial_rate(f64::INFINITY), + "initial_rate must be finite" + )] + #[case::nan_min_rate( + AimdConfig::default().with_min_rate(f64::NAN), + "min_rate must be finite" + )] + #[case::nan_max_rate( + AimdConfig::default().with_max_rate(f64::NAN), + "max_rate must be finite" + )] + #[case::inf_max_rate( + AimdConfig::default().with_max_rate(f64::INFINITY), + "max_rate must be finite" + )] + #[case::nan_decrease_factor( + AimdConfig::default().with_decrease_factor(f64::NAN), + "decrease_factor must be finite" + )] + #[case::nan_additive_increment( + AimdConfig::default().with_additive_increment(f64::NAN), + "additive_increment must be finite" + )] + #[case::nan_throttle_threshold( + AimdConfig::default().with_throttle_threshold(f64::NAN), + "throttle_threshold must be finite" + )] fn test_config_validation_rejects_invalid( #[case] config: AimdConfig, #[case] expected_msg: &str, ) { let err = config.validate().unwrap_err(); + assert!( + matches!(&err, crate::Error::InvalidInput { .. }), + "expected InvalidInput, got: {err:?}" + ); let msg = err.to_string(); assert!( msg.contains(expected_msg), diff --git a/rust/lance-core/src/utils/assume.rs b/rust/lance-core/src/utils/assume.rs index 2560e9bf35e..05f5abb6dc9 100644 --- a/rust/lance-core/src/utils/assume.rs +++ b/rust/lance-core/src/utils/assume.rs @@ -1,29 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -/// A macro that combines debug_assert and std::hint::assert_unchecked for optimized assertions +/// Assert an invariant that should also be visible to the optimizer. /// -/// In debug builds, this will perform a normal assertion check. -/// In release builds, this will use hint::assert_unchecked which tells the compiler to assume -/// the condition is true without actually checking it. -/// -/// # Safety -/// -/// This macro is unsafe in release builds since it uses hint::assert_unchecked. -/// The caller must ensure the condition will always be true. +/// Unlike [`debug_assert!`], this remains checked in release builds. This is +/// required because the macro can be invoked from safe Rust and an invalid +/// assumption must not become undefined behavior. #[macro_export] macro_rules! assume { ($cond:expr) => { - debug_assert!($cond); - // SAFETY: The debug_assert ensures this is true in debug builds. - // In release builds, caller must ensure the condition holds. - unsafe { std::hint::assert_unchecked($cond); } + assert!($cond) }; ($cond:expr, $($arg:tt)+) => { - debug_assert!($cond, $($arg)+); - // SAFETY: The debug_assert ensures this is true in debug builds. - // In release builds, caller must ensure the condition holds. - unsafe { std::hint::assert_unchecked($cond); } + assert!($cond, $($arg)+) }; } @@ -31,11 +20,24 @@ macro_rules! assume { #[macro_export] macro_rules! assume_eq { ($left:expr, $right:expr) => { - debug_assert_eq!($left, $right); - unsafe { std::hint::assert_unchecked($left == $right); } + assert_eq!($left, $right) }; ($left:expr, $right:expr, $($arg:tt)+) => { - debug_assert_eq!($left, $right, $($arg)+); - unsafe { std::hint::assert_unchecked($left == $right); } + assert_eq!($left, $right, $($arg)+) }; } + +#[cfg(test)] +mod tests { + #[test] + fn assume_rejects_false_conditions() { + assert!(std::panic::catch_unwind(|| assume!(false)).is_err()); + assert!(std::panic::catch_unwind(|| assume!(false, "invalid condition")).is_err()); + } + + #[test] + fn assume_eq_rejects_unequal_values() { + assert!(std::panic::catch_unwind(|| assume_eq!(1, 2)).is_err()); + assert!(std::panic::catch_unwind(|| assume_eq!(1, 2, "invalid equality")).is_err()); + } +} diff --git a/rust/lance-core/src/utils/backoff.rs b/rust/lance-core/src/utils/backoff.rs index b30c757bb23..2b1f81e16be 100644 --- a/rust/lance-core/src/utils/backoff.rs +++ b/rust/lance-core/src/utils/backoff.rs @@ -77,6 +77,15 @@ impl Backoff { } } +/// Upper bound on the number of retry slots. +/// +/// Slots double each attempt to spread contending writers apart, but a hundred +/// or so already exceeds any realistic number of concurrent committers, so +/// further doubling only inflates the wait without reducing collisions. Capping +/// the count also bounds a single backoff to `(MAX_SLOTS - 1) * unit` instead of +/// letting it grow without limit as `attempt` climbs. +const MAX_SLOTS: u32 = 128; + /// SlotBackoff is a backoff strategy that randomly chooses a time slot to retry. /// /// This is useful when you have multiple tasks that can't overlap, and each @@ -85,7 +94,7 @@ impl Backoff { /// The `unit` represents the time it takes to complete one attempt. Future attempts /// are divided into time slots, and a random slot is chosen for the retry. The number /// of slots increases exponentially with each attempt. Initially, there are 4 slots, -/// then 8, then 16, and so on. +/// then 8, then 16, and so on, up to a fixed cap. /// /// Example: /// Suppose you have 10 tasks that can't overlap, each taking 1 second. The tasks @@ -138,10 +147,15 @@ impl SlotBackoff { } pub fn next_backoff(&mut self) -> Duration { - let num_slots = self.base.saturating_pow(self.attempt + self.starting_i); + let num_slots = self + .base + .saturating_pow(self.attempt.saturating_add(self.starting_i)) + .min(MAX_SLOTS); let slot_i = self.rng.random_range(0..num_slots); - self.attempt += 1; - Duration::from_millis((slot_i * self.unit) as u64) + self.attempt = self.attempt.saturating_add(1); + // Widen before multiplying: `unit` is the first-attempt latency, which + // can be large enough that a `u32` slot * unit product would overflow. + Duration::from_millis(slot_i as u64 * self.unit as u64) } } @@ -228,4 +242,50 @@ mod tests { assert_eq!(backoff.attempt(), 3); } } + + #[test] + fn test_slot_backoff_high_attempt_is_bounded() { + // Without the slot cap the wait grows unbounded with `attempt`. The cap + // holds every backoff to `(MAX_SLOTS - 1) * unit`. + let unit = 100_000; // 100s first attempt + let mut backoff = SlotBackoff::default().with_unit(unit); + let max_backoff = Duration::from_millis((MAX_SLOTS - 1) as u64 * unit as u64); + for _ in 0..40 { + assert!(backoff.next_backoff() <= max_backoff); + } + assert_eq!(backoff.attempt(), 40); + } + + #[test] + fn test_slot_backoff_large_unit_does_not_overflow() { + // With unit = u32::MAX, any slot >= 2 makes the old u32 `slot_i * unit` + // product overflow: a debug panic, or in release a wrap to a value that + // is no longer a multiple of unit. The u64 widening keeps every backoff + // an exact multiple of unit. Seed the RNG so the drawn slots — and thus + // this check — are deterministic rather than dependent on random draws. + let unit = u32::MAX; + let mut backoff = SlotBackoff::default().with_unit(unit); + backoff.rng = rand::rngs::SmallRng::seed_from_u64(0); + let mut saw_high_slot = false; + for _ in 0..64 { + let backoff_ms = backoff.next_backoff().as_millis(); + // `slot_i * unit` is always a multiple of unit; a wrapped u32 + // product is not. + assert_eq!(backoff_ms % unit as u128, 0, "{backoff_ms} wrapped"); + saw_high_slot |= backoff_ms >= 2 * unit as u128; + } + assert!(saw_high_slot, "expected a slot >= 2 in 64 seeded draws"); + } + + #[test] + fn test_slot_backoff_attempt_saturates() { + // At u32::MAX the counter must stay put rather than panic (debug) or + // wrap to 0 (release), which would restart the low-slot distribution. + let mut backoff = SlotBackoff { + attempt: u32::MAX, + ..Default::default() + }; + let _ = backoff.next_backoff(); + assert_eq!(backoff.attempt(), u32::MAX); + } } diff --git a/rust/lance-core/src/utils/cpu.rs b/rust/lance-core/src/utils/cpu.rs index 4e7ab01871d..d1e8a498043 100644 --- a/rust/lance-core/src/utils/cpu.rs +++ b/rust/lance-core/src/utils/cpu.rs @@ -1,14 +1,32 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::fmt; use std::sync::LazyLock; -/// A level of SIMD support for some feature +/// A level of SIMD support for some feature. +/// +/// `#[non_exhaustive]` so future tiers (e.g. AVX-512 BF16, AMX) can be added +/// without breaking external `match` consumers. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum SimdSupport { None, Neon, Sse, + /// AVX (256-bit float ops) without FMA. + /// + /// This tier does not imply that AVX2 is absent: selecting [`Self::Avx2`] + /// requires both AVX2 and FMA, so a host with AVX2 but no FMA selects this + /// tier. Intel Sandy Bridge / Ivy Bridge are the typical hosts. + Avx, + /// AVX + FMA but no AVX2. + /// AMD Piledriver / Steamroller / FX-7500. + AvxFma, + /// AVX2 + FMA. Intel Haswell / AMD Excavator and later. + /// + /// Selecting this tier asserts FMA is present: the kernels it dispatches to + /// are `#[target_feature(enable = "avx,fma")]`. Avx2, Avx512, Avx512FP16, @@ -16,6 +34,132 @@ pub enum SimdSupport { Lasx, } +impl fmt::Display for SimdSupport { + /// Formats the tier name in lowercase, matching pyarrow's + /// `runtime_info().simd_level` convention. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + Self::None => "none", + Self::Neon => "neon", + Self::Sse => "sse", + Self::Avx => "avx", + Self::AvxFma => "avx_fma", + Self::Avx2 => "avx2", + Self::Avx512 => "avx512", + Self::Avx512FP16 => "avx512_fp16", + Self::Lsx => "lsx", + Self::Lasx => "lasx", + }; + f.write_str(name) + } +} + +/// Snapshot of the SIMD tier lance dispatches to on the current host, plus the +/// raw CPU features detected for diagnostic purposes. +/// +/// Mirrors the role of `pyarrow.runtime_info()`: a single, cheap call users can +/// make to verify which SIMD tier the runtime selected and what underlying +/// features the host advertises. Obtain one with [`simd_info()`]. +#[derive(Debug, Clone)] +pub struct SimdInfo { + /// The SIMD tier lance dispatches to at runtime on this host. + pub tier: SimdSupport, + /// The architecture name (e.g. "x86_64", "aarch64", "loongarch64"). + pub target_arch: &'static str, + /// Raw CPU feature flags detected on this host (x86_64 only; empty on + /// other architectures). Each entry is a feature name like "avx2", + /// "fma", "avx512f", "popcnt", etc. + pub host_features: Vec<&'static str>, +} + +/// Returns a snapshot of the SIMD tier lance is using on this host along with +/// the raw CPU feature flags that drove the decision. +/// +/// Useful for performance debugging and giving users a way to verify which +/// dispatch tier they are hitting without rebuilding lance. See [`SimdInfo`] +/// for the meaning of each field and [`SimdSupport`] for the tier values. +/// +/// # Examples +/// +/// ``` +/// use lance_core::utils::cpu::simd_info; +/// +/// let info = simd_info(); +/// println!("dispatching to {} on {}", info.tier, info.target_arch); +/// ``` +pub fn simd_info() -> SimdInfo { + SimdInfo { + tier: *SIMD_SUPPORT, + target_arch: std::env::consts::ARCH, + host_features: detect_host_features(), + } +} + +#[cfg(target_arch = "x86_64")] +fn detect_host_features() -> Vec<&'static str> { + // Each call must be inline: `is_x86_feature_detected!` does its own custom + // input parsing and rejects feature names received via a `macro_rules!` + // `:literal` metavariable on some toolchains. + let mut features = Vec::with_capacity(17); + if is_x86_feature_detected!("sse2") { + features.push("sse2"); + } + if is_x86_feature_detected!("sse3") { + features.push("sse3"); + } + if is_x86_feature_detected!("ssse3") { + features.push("ssse3"); + } + if is_x86_feature_detected!("sse4.1") { + features.push("sse4.1"); + } + if is_x86_feature_detected!("sse4.2") { + features.push("sse4.2"); + } + if is_x86_feature_detected!("popcnt") { + features.push("popcnt"); + } + if is_x86_feature_detected!("avx") { + features.push("avx"); + } + if is_x86_feature_detected!("avx2") { + features.push("avx2"); + } + if is_x86_feature_detected!("fma") { + features.push("fma"); + } + if is_x86_feature_detected!("f16c") { + features.push("f16c"); + } + if is_x86_feature_detected!("bmi1") { + features.push("bmi1"); + } + if is_x86_feature_detected!("bmi2") { + features.push("bmi2"); + } + if is_x86_feature_detected!("avx512f") { + features.push("avx512f"); + } + if is_x86_feature_detected!("avx512bw") { + features.push("avx512bw"); + } + if is_x86_feature_detected!("avx512cd") { + features.push("avx512cd"); + } + if is_x86_feature_detected!("avx512dq") { + features.push("avx512dq"); + } + if is_x86_feature_detected!("avx512vl") { + features.push("avx512vl"); + } + features +} + +#[cfg(not(target_arch = "x86_64"))] +fn detect_host_features() -> Vec<&'static str> { + Vec::new() +} + /// Support for SIMD operations pub static SIMD_SUPPORT: LazyLock = LazyLock::new(|| { #[cfg(all(target_arch = "aarch64", any(target_os = "ios", target_os = "tvos")))] @@ -42,8 +186,20 @@ pub static SIMD_SUPPORT: LazyLock = LazyLock::new(|| { } else { SimdSupport::Avx512 } - } else if is_x86_feature_detected!("avx2") { + } else if is_x86_feature_detected!("avx2") && is_x86_feature_detected!("fma") { + // FMA is checked explicitly: every kernel selected for this tier is + // `#[target_feature(enable = "avx,fma")]`, and AVX2 does not imply + // FMA in the ISA. Every shipping AVX2 part has FMA, so this only + // guards against a host that would otherwise take an FMA kernel + // without FMA. SimdSupport::Avx2 + } else if is_x86_feature_detected!("avx") && is_x86_feature_detected!("fma") { + // AMD Piledriver / Steamroller / FX-7500: 256-bit float ops + FMA but no AVX2. + SimdSupport::AvxFma + } else if is_x86_feature_detected!("avx") { + // This includes a possible AVX2 host without FMA because the Avx2 + // tier above requires both features. + SimdSupport::Avx } else { SimdSupport::None } @@ -58,6 +214,14 @@ pub static SIMD_SUPPORT: LazyLock = LazyLock::new(|| { SimdSupport::None } } + #[cfg(not(any( + target_arch = "aarch64", + target_arch = "x86_64", + target_arch = "loongarch64" + )))] + { + SimdSupport::None + } }); #[cfg(target_arch = "x86_64")] @@ -138,3 +302,68 @@ mod aarch64 { false } } + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[test] + fn simd_info_exposes_tier() { + let info = simd_info(); + assert_eq!(info.target_arch, std::env::consts::ARCH); + // Tier should match the detected SIMD support. + assert_eq!(info.tier, *SIMD_SUPPORT); + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn simd_info_features_include_baseline() { + let info = simd_info(); + // The x86_64 ABI mandates SSE2, so it must always be present on this + // architecture. + assert!(info.host_features.contains(&"sse2")); + } + + #[cfg(not(target_arch = "x86_64"))] + #[test] + fn simd_info_features_empty_off_x86_64() { + let info = simd_info(); + assert!(info.host_features.is_empty()); + } + + /// The `Avx2` and `AvxFma` tiers both dispatch to kernels declared + /// `#[target_feature(enable = "avx,fma")]`, so neither may be selected on a + /// host without FMA. AVX2 does not imply FMA in the ISA, so the detection + /// checks it explicitly. (`Avx512*` is excluded: its kernels declare + /// `avx512f`, which is what `has_avx512` verifies.) + #[cfg(target_arch = "x86_64")] + #[test] + fn avx_fma_tiers_are_only_selected_when_fma_is_detected() { + if matches!(*SIMD_SUPPORT, SimdSupport::Avx2 | SimdSupport::AvxFma) { + assert!( + is_x86_feature_detected!("fma"), + "tier {} dispatches to avx,fma kernels but the host has no FMA", + *SIMD_SUPPORT + ); + } + } + + #[rstest] + #[case::none(SimdSupport::None, "none")] + #[case::neon(SimdSupport::Neon, "neon")] + #[case::sse(SimdSupport::Sse, "sse")] + #[case::avx(SimdSupport::Avx, "avx")] + #[case::avx_fma(SimdSupport::AvxFma, "avx_fma")] + #[case::avx2(SimdSupport::Avx2, "avx2")] + #[case::avx512(SimdSupport::Avx512, "avx512")] + #[case::avx512_fp16(SimdSupport::Avx512FP16, "avx512_fp16")] + #[case::lsx(SimdSupport::Lsx, "lsx")] + #[case::lasx(SimdSupport::Lasx, "lasx")] + fn simd_support_display_matches_lowercase_convention( + #[case] tier: SimdSupport, + #[case] expected: &str, + ) { + assert_eq!(tier.to_string(), expected); + } +} diff --git a/rust/lance-core/src/utils/futures.rs b/rust/lance-core/src/utils/futures.rs index 95a1c39aaae..a7e174eb32e 100644 --- a/rust/lance-core/src/utils/futures.rs +++ b/rust/lance-core/src/utils/futures.rs @@ -382,6 +382,19 @@ mod tests { assert_eq!(right.next().await, None); } + #[test] + fn test_shared_stream_replaces_waiting_waker() { + let inner_stream = futures::stream::pending::(); + let (mut left, mut right) = inner_stream.boxed().share(Capacity::Unbounded); + + let mut left_fut = left.next(); + assert!(is_pending(&mut left_fut)); + + let mut right_fut = right.next(); + assert!(is_pending(&mut right_fut)); + assert!(is_pending(&mut right_fut)); + } + #[tokio::test] async fn test_unbounded_shared_stream() { let (tx, rx) = tokio::sync::mpsc::channel::(10); diff --git a/rust/lance-core/src/utils/parse.rs b/rust/lance-core/src/utils/parse.rs index e9e43e393cf..bba8fe7716e 100644 --- a/rust/lance-core/src/utils/parse.rs +++ b/rust/lance-core/src/utils/parse.rs @@ -10,6 +10,26 @@ pub fn str_is_truthy(val: &str) -> bool { | val.eq_ignore_ascii_case("y") } +/// Parse a string into an optional boolean value. +/// +/// Returns `Some(true)` for truthy values (1/true/on/yes/y, case-insensitive). +/// Returns `Some(false)` for falsy values (0/false/off/no/n, case-insensitive). +/// Returns `None` for unrecognized values. +pub fn str_to_bool(val: &str) -> Option { + if str_is_truthy(val) { + Some(true) + } else if val.eq_ignore_ascii_case("0") + || val.eq_ignore_ascii_case("false") + || val.eq_ignore_ascii_case("off") + || val.eq_ignore_ascii_case("no") + || val.eq_ignore_ascii_case("n") + { + Some(false) + } else { + None + } +} + /// Parse an environment variable as a truthy-only boolean. /// /// Returns `default_value` if the env var is not set. @@ -21,3 +41,43 @@ pub fn parse_env_as_bool(env_var_name: &str, default_value: bool) -> bool { .map(|value| str_is_truthy(value.trim())) .unwrap_or(default_value) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_str_to_bool_truthy() { + for val in [ + "1", "true", "True", "TRUE", "on", "ON", "yes", "YES", "y", "Y", + ] { + assert_eq!( + str_to_bool(val), + Some(true), + "expected Some(true) for {:?}", + val + ); + } + } + + #[test] + fn test_str_to_bool_falsy() { + for val in [ + "0", "false", "False", "FALSE", "off", "OFF", "no", "NO", "n", "N", + ] { + assert_eq!( + str_to_bool(val), + Some(false), + "expected Some(false) for {:?}", + val + ); + } + } + + #[test] + fn test_str_to_bool_unknown() { + for val in ["", "2", "maybe", "truthy", "nonsense"] { + assert_eq!(str_to_bool(val), None, "expected None for {:?}", val); + } + } +} diff --git a/rust/lance-core/src/utils/row_addr_remap.rs b/rust/lance-core/src/utils/row_addr_remap.rs index 6f5a6f2aae5..ddb3935062f 100644 --- a/rust/lance-core/src/utils/row_addr_remap.rs +++ b/rust/lance-core/src/utils/row_addr_remap.rs @@ -21,8 +21,10 @@ //! * An address whose fragment was not rewritten returns `None`. //! * For an address whose fragment was rewritten: //! * Read `(old_offsets, old_rows_before)` from the old-row layout. -//! * If `offset` is not in `old_offsets`, return `Some(None)` because the -//! row was deleted. +//! * If `offset` is outside the old fragment's physical row range, return +//! `None`; the direct-map representation would not contain that address. +//! * If a valid `offset` is not in `old_offsets`, return `Some(None)` +//! because the row was deleted. //! * Otherwise, `old_offsets.rank(offset) - 1` is this row's 0-based //! position among rewritten old rows in this old fragment. Add //! `old_rows_before` to get `k`, the row's 0-based position among all @@ -44,10 +46,12 @@ //! * Current compaction satisfies this because it scans selected fragments in //! order and writes the resulting stream without reordering rows. +use crate::deepsize::{Context, DeepSizeOf}; use crate::utils::address::RowAddress; use crate::{Error, Result}; use roaring::{RoaringBitmap, RoaringTreemap}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::mem::size_of; /// A queryable row-address remapping with the exact semantics of /// `HashMap>::get(&addr).copied()`: @@ -55,7 +59,7 @@ use std::collections::HashMap; /// * `None` — the address is not affected by this remap (keep it unchanged) /// * `Some(None)` — the row was deleted /// * `Some(Some(addr))` — the row moved to `addr` -#[derive(Clone)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum RowAddrRemap { /// Compact, `O(#fragments)` remap built from per-group rewritten-row /// bitmaps and new-fragment layouts. @@ -69,11 +73,34 @@ impl RowAddrRemap { Ok(Self::Compact(CompactRowAddrRemap::new(groups)?)) } + /// Build a compact remap with physical row counts for exact validation of + /// addresses loaded from persisted fragment layouts. + #[doc(hidden)] + pub fn compact_with_layout( + groups: impl IntoIterator, + ) -> Result { + Ok(Self::Compact(CompactRowAddrRemap::new_with_layout(groups)?)) + } + /// Build a remap from a fully materialized old-to-new address map. pub fn direct(map: HashMap>) -> Self { Self::Direct(map) } + /// Build an ordered remap chain, flattening nested chains and omitting + /// empty remaps. + pub fn chained(remaps: impl IntoIterator) -> Self { + let mut remaps = remaps + .into_iter() + .filter(|remap| !remap.is_empty()) + .collect::>(); + match remaps.len() { + 0 => Self::empty(), + 1 => remaps.pop().unwrap(), + _ => Self::Compact(CompactRowAddrRemap::chained(remaps)), + } + } + /// An empty remap that leaves every address unchanged. pub fn empty() -> Self { Self::Direct(HashMap::new()) @@ -88,6 +115,27 @@ impl RowAddrRemap { } } + /// Apply this remap to a batch in place. + /// + /// A `None` input remains deleted. An address missing from a remap remains + /// unchanged. Chained remaps are applied version-by-version so this path is + /// suitable for bulk index and transaction remapping without materializing + /// a composed per-row map. + pub fn remap_in_place(&self, row_addrs: &mut [Option]) { + match self { + Self::Compact(compact) => compact.remap_in_place(row_addrs), + Self::Direct(_) => { + for row_addr in row_addrs { + if let Some(addr) = *row_addr + && let Some(mapped) = self.get(addr) + { + *row_addr = mapped; + } + } + } + } + } + pub fn is_empty(&self) -> bool { match self { Self::Compact(c) => c.is_empty(), @@ -97,7 +145,7 @@ impl RowAddrRemap { pub fn affected_fragments(&self) -> RoaringBitmap { match self { - Self::Compact(c) => RoaringBitmap::from_iter(c.frag_to_group.keys().copied()), + Self::Compact(c) => c.affected_fragments(), Self::Direct(m) => RoaringBitmap::from_iter(m.keys().map(|addr| (addr >> 32) as u32)), } } @@ -118,6 +166,15 @@ impl RowAddrRemap { } } +impl DeepSizeOf for RowAddrRemap { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + match self { + Self::Compact(compact) => compact.deep_size_of_children(context), + Self::Direct(map) => map.deep_size_of_children(context), + } + } +} + /// Input describing one rewrite group: the old row addresses that were /// rewritten plus the fragment layout before/after the rewrite. pub struct GroupInput { @@ -129,83 +186,289 @@ pub struct GroupInput { pub new_frags: Vec<(u32, u32)>, } -#[derive(Clone)] +/// Internal compact-remap input that includes old-fragment physical row counts. +#[doc(hidden)] +pub struct GroupInputWithLayout { + pub rewritten_old_row_addrs: RoaringTreemap, + pub old_frags: Vec<(u32, u32)>, + pub new_frags: Vec<(u32, u32)>, +} + +/// Keep Roaring only when its serialized representation is substantially +/// smaller than either rank-friendly representation. This preserves compact +/// run containers while avoiding Roaring's linear word scan for dense rank. +/// Binary-copy compaction creates these runs with `RoaringTreemap::insert_range`, +/// and serialization preserves them without an explicit `optimize()` call. +const ROARING_SIZE_ADVANTAGE_FOR_RANK: usize = 4; + +#[derive(Clone, Debug, PartialEq, Eq)] +enum RankedOffsets { + /// Retained for highly compressible run layouts. + Roaring(RoaringBitmap), + /// Sorted rewritten offsets. Binary search returns membership and rank in + /// one operation. + Sparse(Vec), + /// Dense bits with the number of rewritten rows before every word. + Dense(DenseRankedOffsets), +} + +impl RankedOffsets { + fn try_new(offsets: RoaringBitmap, physical_rows: Option) -> Result { + let universe_rows = physical_rows.map(u64::from).unwrap_or_else(|| { + offsets + .max() + .map(|offset| u64::from(offset) + 1) + .unwrap_or(0) + }); + let word_count = usize::try_from(universe_rows.div_ceil(64)).map_err(|_| { + Error::invalid_input(format!( + "fragment row range {universe_rows} is too large for compact rank lookup" + )) + })?; + let sparse_bytes = usize::try_from(offsets.len()) + .ok() + .and_then(|len| len.checked_mul(size_of::())) + .ok_or_else(|| { + Error::invalid_input(format!( + "rewritten row count {} is too large for sparse rank lookup", + offsets.len() + )) + })?; + let dense_bytes = word_count + .checked_mul(size_of::() + size_of::()) + .ok_or_else(|| { + Error::invalid_input(format!( + "fragment row range {universe_rows} is too large for dense rank lookup" + )) + })?; + let rank_friendly_bytes = sparse_bytes.min(dense_bytes); + if offsets + .serialized_size() + .checked_mul(ROARING_SIZE_ADVANTAGE_FOR_RANK) + .is_some_and(|roaring_bytes| roaring_bytes < rank_friendly_bytes) + { + return Ok(Self::Roaring(offsets)); + } + if sparse_bytes <= dense_bytes { + return Ok(Self::Sparse(offsets.into_iter().collect())); + } + Ok(Self::Dense(DenseRankedOffsets::try_new( + offsets, word_count, + )?)) + } + + /// Return the zero-based rank when `offset` was rewritten. + #[inline] + fn rank_if_present(&self, offset: u32) -> Option { + match self { + Self::Roaring(offsets) => offsets.contains(offset).then(|| offsets.rank(offset) - 1), + Self::Sparse(offsets) => offsets.binary_search(&offset).ok().map(|rank| rank as u64), + Self::Dense(offsets) => offsets.rank_if_present(offset), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Roaring(offsets) => offsets.is_empty(), + Self::Sparse(offsets) => offsets.is_empty(), + Self::Dense(offsets) => offsets.words.is_empty(), + } + } +} + +impl DeepSizeOf for RankedOffsets { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + match self { + // Roaring does not expose its allocation capacity. Its serialized + // size is a stable proxy for the retained containers. + Self::Roaring(offsets) => offsets.serialized_size(), + Self::Sparse(offsets) => offsets.deep_size_of_children(context), + Self::Dense(offsets) => offsets.deep_size_of_children(context), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct DenseRankedOffsets { + words: Vec, + rank_before_word: Vec, +} + +impl DenseRankedOffsets { + fn try_new(offsets: RoaringBitmap, word_count: usize) -> Result { + let mut words = vec![0u64; word_count]; + for offset in offsets { + let word_idx = (offset / 64) as usize; + let Some(word) = words.get_mut(word_idx) else { + return Err(Error::invalid_input(format!( + "rewritten row offset {offset} is outside dense rank word_count={word_count}" + ))); + }; + *word |= 1u64 << (offset % 64); + } + + let mut rank_before_word = Vec::with_capacity(word_count); + let mut rewritten_rows_before = 0u64; + for word in &words { + rank_before_word.push(u32::try_from(rewritten_rows_before).map_err(|_| { + Error::invalid_input(format!( + "rewritten row count {rewritten_rows_before} exceeds the row-address offset range" + )) + })?); + rewritten_rows_before += u64::from(word.count_ones()); + } + Ok(Self { + words, + rank_before_word, + }) + } + + #[inline] + fn rank_if_present(&self, offset: u32) -> Option { + let word_idx = (offset / 64) as usize; + let word = *self.words.get(word_idx)?; + let bit = 1u64 << (offset % 64); + if word & bit == 0 { + return None; + } + Some( + u64::from(self.rank_before_word[word_idx]) + u64::from((word & (bit - 1)).count_ones()), + ) + } +} + +impl DeepSizeOf for DenseRankedOffsets { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.words.deep_size_of_children(context) + + self.rank_before_word.deep_size_of_children(context) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct OldFragmentRemap { + group_idx: usize, + rewritten_offsets: RankedOffsets, + rewritten_rows_before: u64, + physical_rows: Option, +} + +impl DeepSizeOf for OldFragmentRemap { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.rewritten_offsets.deep_size_of_children(context) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] struct GroupRemap { - /// Old fragment id -> (rewritten old row offsets in that fragment, - /// rewritten row count before this fragment in the group). - frags: HashMap, /// New fragment ranges as `(fragment_id, rewritten_rows_before, physical_rows)`, /// used to map a rewritten row's group-local index to its new address via binary search. new_frag_row_ranges: Vec<(u32, u64, u32)>, } impl GroupRemap { - fn new(input: GroupInput) -> Result { - // `compute_new_addr` maps a rewritten row's group-local index to a new - // address by accumulating `physical_rows` in `new_frags` order, so that - // order must be the order rows were written. New fragment ids are - // reserved monotonically in write order (see `reserve_fragment_ids` in - // compaction), so ascending id is a proxy for write order; reject any - // input that violates it before it can silently misplace addresses. - let mut new_frag_row_ranges = Vec::with_capacity(input.new_frags.len()); + fn new(input: GroupInput, group_idx: usize) -> Result<(Self, Vec<(u32, OldFragmentRemap)>)> { + Self::new_with_old_frags( + input.rewritten_old_row_addrs, + input.old_frag_ids.into_iter().map(|id| (id, None)), + input.new_frags, + group_idx, + ) + } + + fn new_with_layout( + input: GroupInputWithLayout, + group_idx: usize, + ) -> Result<(Self, Vec<(u32, OldFragmentRemap)>)> { + Self::new_with_old_frags( + input.rewritten_old_row_addrs, + input + .old_frags + .into_iter() + .map(|(id, rows)| (id, Some(rows))), + input.new_frags, + group_idx, + ) + } + + fn new_with_old_frags( + rewritten_old_row_addrs: RoaringTreemap, + old_frags: impl IntoIterator)>, + new_frags: Vec<(u32, u32)>, + group_idx: usize, + ) -> Result<(Self, Vec<(u32, OldFragmentRemap)>)> { + // `compute_new_addr` maps a rewritten row's group-local index by + // accumulating `physical_rows` in the caller-provided write order. + let mut new_frag_row_ranges = Vec::with_capacity(new_frags.len()); let mut rewritten_rows_before = 0u64; - let mut prev_frag_id: Option = None; - for (frag_id, physical_rows) in input.new_frags { + for (frag_id, physical_rows) in new_frags { if physical_rows == 0 { continue; } - if let Some(prev) = prev_frag_id - && frag_id <= prev - { - return Err(Error::invalid_input(format!( - "compaction new fragments must be in ascending id (write) order, but fragment {frag_id} follows {prev}", - ))); - } - prev_frag_id = Some(frag_id); new_frag_row_ranges.push((frag_id, rewritten_rows_before, physical_rows)); rewritten_rows_before += physical_rows as u64; } let total_new_rows = rewritten_rows_before; - let mut per_frag: HashMap = input - .rewritten_old_row_addrs + let mut per_frag: HashMap = rewritten_old_row_addrs .bitmaps() .map(|(frag_id, bitmap)| (frag_id, bitmap.clone())) .collect(); - let mut frags = HashMap::new(); + let old_frags = old_frags.into_iter().collect::>(); + let mut frags = Vec::with_capacity(old_frags.len()); + let mut seen_frag_ids = HashSet::with_capacity(old_frags.len()); let mut rewritten_rows_before = 0u64; - for &frag_id in &input.old_frag_ids { - // A fragment with no rewritten rows (fully deleted) contributes - // nothing to the rewritten row sequence. - if let Some(bitmap) = per_frag.remove(&frag_id) { - let num_rewritten_rows = bitmap.len(); - frags.insert(frag_id, (bitmap, rewritten_rows_before)); - rewritten_rows_before += num_rewritten_rows; + for &(frag_id, physical_rows) in &old_frags { + if !seen_frag_ids.insert(frag_id) { + return Err(Error::invalid_input(format!( + "rewrite group {group_idx} contains old fragment {frag_id} more than once" + ))); + } + let bitmap = per_frag.remove(&frag_id).unwrap_or_default(); + if let Some(physical_rows) = physical_rows + && bitmap.max().is_some_and(|offset| offset >= physical_rows) + { + return Err(Error::invalid_input(format!( + "rewrite group {group_idx} contains a row offset outside old fragment {frag_id} with physical_rows={physical_rows}" + ))); } + let num_rewritten_rows = bitmap.len(); + let rewritten_offsets = RankedOffsets::try_new(bitmap, physical_rows)?; + frags.push(( + frag_id, + OldFragmentRemap { + group_idx, + rewritten_offsets, + rewritten_rows_before, + physical_rows, + }, + )); + rewritten_rows_before += num_rewritten_rows; } - // Rewritten old row addresses must reference only fragments listed in `old_frag_ids`. + // Rewritten old row addresses must reference only listed old fragments. if !per_frag.is_empty() { return Err(Error::invalid_input(format!( - "compaction rewritten old row addresses reference fragments {:?} not in the rewrite group's old fragments {:?}", + "compaction rewrite group {group_idx} references rewritten old row addresses from fragments {:?} not in its old fragments {:?}", per_frag.keys().collect::>(), - input.old_frag_ids, + old_frags, ))); } // Rewritten old rows are mapped positionally onto the new rows, so the // two counts must match exactly - let total_rewritten_old_rows = input.rewritten_old_row_addrs.len(); + let total_rewritten_old_rows = rewritten_old_row_addrs.len(); if total_new_rows != total_rewritten_old_rows { return Err(Error::invalid_input(format!( - "compaction rewrote {total_rewritten_old_rows} old rows from fragments {:?} but the new fragments hold {total_new_rows} rows", - input.old_frag_ids, + "compaction rewrite group {group_idx} rewrote {total_rewritten_old_rows} old rows from fragments {:?} but the new fragments hold {total_new_rows} rows", + old_frags, ))); } - Ok(Self { + Ok(( + Self { + new_frag_row_ranges, + }, frags, - new_frag_row_ranges, - }) + )) } fn compute_new_addr(&self, rewritten_row_index: u64) -> u64 { @@ -222,43 +485,62 @@ impl GroupRemap { let offset = (rewritten_row_index - rewritten_rows_before) as u32; u64::from(RowAddress::new_from_parts(frag_id, offset)) } +} - /// Compute the new address for an old row in this group. - /// Returns `None` if the old row was not rewritten. - #[inline] - fn get(&self, frag: u32, offset: u32) -> Option { - match self.frags.get(&frag) { - Some((bitmap, rewritten_rows_before)) if bitmap.contains(offset) => { - let rewritten_row_index = rewritten_rows_before + bitmap.rank(offset) - 1; - Some(self.compute_new_addr(rewritten_row_index)) - } - _ => None, - } +impl DeepSizeOf for GroupRemap { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.new_frag_row_ranges.deep_size_of_children(context) } } -/// Compact remap backed by per-group rewritten row bitmaps + new-fragment layouts. -#[derive(Clone)] -pub struct CompactRowAddrRemap { +#[derive(Clone, Debug, PartialEq, Eq)] +struct CompactRemapStep { groups: Vec, - /// Old fragment id -> index into `groups`. Size is O(#fragments), not rows. - frag_to_group: HashMap, + /// Old fragment id -> its bitmap/rank layout and rewrite group. Size is + /// O(#fragments), not rows. + frags: HashMap, } -impl CompactRowAddrRemap { +impl CompactRemapStep { fn new(groups: impl IntoIterator) -> Result { - let mut frag_to_group = HashMap::new(); + let mut frags = HashMap::new(); + let mut group_remaps = Vec::new(); + for input in groups { + let gi = group_remaps.len(); + let (group_remap, group_frags) = GroupRemap::new(input, gi)?; + for (frag_id, frag) in group_frags { + if frags.insert(frag_id, frag).is_some() { + return Err(Error::invalid_input(format!( + "old fragment {frag_id} appears in more than one rewrite group, including group {gi}" + ))); + } + } + group_remaps.push(group_remap); + } + Ok(Self { + groups: group_remaps, + frags, + }) + } + + fn new_with_layout(groups: impl IntoIterator) -> Result { + let mut frags = HashMap::new(); let mut group_remaps = Vec::new(); for input in groups { let gi = group_remaps.len(); - for &frag_id in &input.old_frag_ids { - frag_to_group.insert(frag_id, gi); + let (group_remap, group_frags) = GroupRemap::new_with_layout(input, gi)?; + for (frag_id, frag) in group_frags { + if frags.insert(frag_id, frag).is_some() { + return Err(Error::invalid_input(format!( + "old fragment {frag_id} appears in more than one rewrite group, including group {gi}" + ))); + } } - group_remaps.push(GroupRemap::new(input)?); + group_remaps.push(group_remap); } Ok(Self { groups: group_remaps, - frag_to_group, + frags, }) } @@ -266,8 +548,21 @@ impl CompactRowAddrRemap { pub fn get(&self, addr: u64) -> Option> { let frag = (addr >> 32) as u32; // Not in any rewrite group -> unaffected by this remap. - let gi = *self.frag_to_group.get(&frag)?; - Some(self.groups[gi].get(frag, addr as u32)) + let old_frag = self.frags.get(&frag)?; + let offset = addr as u32; + if old_frag + .physical_rows + .is_some_and(|physical_rows| offset >= physical_rows) + { + return None; + } + let Some(rewritten_rank) = old_frag.rewritten_offsets.rank_if_present(offset) else { + return Some(None); + }; + let rewritten_row_index = old_frag.rewritten_rows_before + rewritten_rank; + Some(Some( + self.groups[old_frag.group_idx].compute_new_addr(rewritten_row_index), + )) } pub fn is_empty(&self) -> bool { @@ -276,10 +571,167 @@ impl CompactRowAddrRemap { fn fully_deleted_fragments(&self) -> Option { // A group with any rewritten row moved at least one row. - if self.groups.iter().any(|g| !g.frags.is_empty()) { + if self + .frags + .values() + .any(|frag| !frag.rewritten_offsets.is_empty()) + { return None; } - Some(RoaringBitmap::from_iter(self.frag_to_group.keys().copied())) + Some(RoaringBitmap::from_iter(self.frags.keys().copied())) + } + + fn affected_fragments(&self) -> RoaringBitmap { + RoaringBitmap::from_iter(self.frags.keys().copied()) + } +} + +impl DeepSizeOf for CompactRemapStep { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.groups.deep_size_of_children(context) + self.frags.deep_size_of_children(context) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum RemapStep { + Compact(CompactRemapStep), + Direct(HashMap>), +} + +impl RemapStep { + fn get(&self, addr: u64) -> Option> { + match self { + Self::Compact(compact) => compact.get(addr), + Self::Direct(direct) => direct.get(&addr).copied(), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Compact(compact) => compact.is_empty(), + Self::Direct(direct) => direct.is_empty(), + } + } + + fn affected_fragments(&self) -> RoaringBitmap { + match self { + Self::Compact(compact) => compact.affected_fragments(), + Self::Direct(direct) => { + RoaringBitmap::from_iter(direct.keys().map(|addr| (addr >> 32) as u32)) + } + } + } + + fn fully_deleted_fragments(&self) -> Option { + match self { + Self::Compact(compact) => compact.fully_deleted_fragments(), + Self::Direct(direct) if direct.values().all(Option::is_none) => Some( + RoaringBitmap::from_iter(direct.keys().map(|addr| (addr >> 32) as u32)), + ), + Self::Direct(_) => None, + } + } +} + +impl DeepSizeOf for RemapStep { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + match self { + Self::Compact(compact) => compact.deep_size_of_children(context), + Self::Direct(direct) => direct.deep_size_of_children(context), + } + } +} + +/// Compact remap backed by per-group rewritten row bitmaps + new-fragment layouts. +/// +/// Multiple remaps are retained as ordered private steps so a version chain +/// does not require another public [`RowAddrRemap`] variant. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CompactRowAddrRemap { + steps: Vec, +} + +impl CompactRowAddrRemap { + fn new(groups: impl IntoIterator) -> Result { + Ok(Self { + steps: vec![RemapStep::Compact(CompactRemapStep::new(groups)?)], + }) + } + + fn new_with_layout(groups: impl IntoIterator) -> Result { + Ok(Self { + steps: vec![RemapStep::Compact(CompactRemapStep::new_with_layout( + groups, + )?)], + }) + } + + fn chained(remaps: Vec) -> Self { + let mut steps = Vec::with_capacity(remaps.len()); + for remap in remaps { + match remap { + RowAddrRemap::Compact(compact) => steps.extend(compact.steps), + RowAddrRemap::Direct(direct) => steps.push(RemapStep::Direct(direct)), + } + } + Self { steps } + } + + #[inline] + pub fn get(&self, addr: u64) -> Option> { + let mut current = addr; + let mut was_affected = false; + for step in &self.steps { + match step.get(current) { + None => {} + Some(None) => return Some(None), + Some(Some(mapped)) => { + current = mapped; + was_affected = true; + } + } + } + was_affected.then_some(Some(current)) + } + + fn remap_in_place(&self, row_addrs: &mut [Option]) { + for step in &self.steps { + for row_addr in row_addrs.iter_mut() { + if let Some(addr) = *row_addr + && let Some(mapped) = step.get(addr) + { + *row_addr = mapped; + } + } + } + } + + pub fn is_empty(&self) -> bool { + self.steps.iter().all(RemapStep::is_empty) + } + + fn affected_fragments(&self) -> RoaringBitmap { + self.steps + .iter() + .fold(RoaringBitmap::new(), |mut affected, step| { + affected |= step.affected_fragments(); + affected + }) + } + + fn fully_deleted_fragments(&self) -> Option { + self.steps + .iter() + .try_fold(RoaringBitmap::new(), |mut deleted, step| { + deleted |= step.fully_deleted_fragments()?; + Some(deleted) + }) + } +} + +impl DeepSizeOf for CompactRowAddrRemap { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.steps.deep_size_of_children(context) } } @@ -291,6 +743,155 @@ mod tests { u64::from(RowAddress::new_from_parts(frag, offset)) } + #[derive(Clone, Copy)] + enum ExpectedRankedOffsets { + Sparse, + Dense, + Roaring, + } + + fn assert_layout_matches_legacy( + frag_id: u32, + physical_rows: u32, + rewritten_old_row_addrs: RoaringTreemap, + new_frags: Vec<(u32, u32)>, + expected_representation: ExpectedRankedOffsets, + ) { + let rewritten_addrs = rewritten_old_row_addrs.iter().collect::>(); + let new_addrs = new_frags + .iter() + .flat_map(|(new_frag_id, rows)| (0..*rows).map(|offset| addr(*new_frag_id, offset))) + .collect::>(); + assert_eq!(rewritten_addrs.len(), new_addrs.len()); + let expected_moved = rewritten_addrs + .iter() + .copied() + .zip(new_addrs) + .collect::>(); + + let remap = RowAddrRemap::compact_with_layout([GroupInputWithLayout { + rewritten_old_row_addrs, + old_frags: vec![(frag_id, physical_rows)], + new_frags, + }]) + .unwrap(); + + let RowAddrRemap::Compact(compact) = &remap else { + panic!("compact_with_layout must produce a compact remap"); + }; + let RemapStep::Compact(step) = &compact.steps[0] else { + panic!("compact_with_layout must produce a compact step"); + }; + let offsets = &step.frags[&frag_id].rewritten_offsets; + assert!(match expected_representation { + ExpectedRankedOffsets::Sparse => matches!(offsets, RankedOffsets::Sparse(_)), + ExpectedRankedOffsets::Dense => matches!(offsets, RankedOffsets::Dense(_)), + ExpectedRankedOffsets::Roaring => matches!(offsets, RankedOffsets::Roaring(_)), + }); + + for offset in 0..physical_rows { + let old_addr = addr(frag_id, offset); + assert_eq!( + remap.get(old_addr), + Some(expected_moved.get(&old_addr).copied()), + "mismatch at ({frag_id}, {offset})" + ); + } + assert_eq!(remap.get(addr(frag_id, physical_rows)), None); + assert_eq!(remap.get(addr(frag_id + 1, 0)), None); + } + + #[test] + fn test_sparse_ranked_offsets() { + let offsets = RankedOffsets::try_new( + RoaringBitmap::from_iter([1u32, 63, 511, 9_999]), + Some(10_000), + ) + .unwrap(); + assert!(matches!(offsets, RankedOffsets::Sparse(_))); + assert_eq!(offsets.rank_if_present(0), None); + assert_eq!(offsets.rank_if_present(1), Some(0)); + assert_eq!(offsets.rank_if_present(63), Some(1)); + assert_eq!(offsets.rank_if_present(511), Some(2)); + assert_eq!(offsets.rank_if_present(9_999), Some(3)); + } + + #[test] + fn test_dense_ranked_offsets_across_words() { + let rewritten = (0..1_024u32) + .filter(|offset| offset % 10 != 0) + .collect::(); + let offsets = RankedOffsets::try_new(rewritten.clone(), Some(1_024)).unwrap(); + assert!(matches!(offsets, RankedOffsets::Dense(_))); + + let mut expected_rank = 0u64; + for offset in 0..1_024 { + if rewritten.contains(offset) { + assert_eq!(offsets.rank_if_present(offset), Some(expected_rank)); + expected_rank += 1; + } else { + assert_eq!(offsets.rank_if_present(offset), None); + } + } + assert_eq!(expected_rank, rewritten.len()); + } + + #[test] + fn test_run_compressed_ranked_offsets() { + let mut rewritten = RoaringBitmap::new(); + rewritten.insert_range(100..9_900); + let offsets = RankedOffsets::try_new(rewritten, Some(10_000)).unwrap(); + assert!(matches!(offsets, RankedOffsets::Roaring(_))); + assert_eq!(offsets.rank_if_present(99), None); + assert_eq!(offsets.rank_if_present(100), Some(0)); + assert_eq!(offsets.rank_if_present(9_899), Some(9_799)); + assert_eq!(offsets.rank_if_present(9_900), None); + } + + #[test] + fn test_compact_with_layout_matches_legacy_across_rank_representations() { + assert_layout_matches_legacy( + 1, + 10_000, + RoaringTreemap::from_iter( + [1u32, 63, 511, 9_999] + .into_iter() + .map(|offset| addr(1, offset)), + ), + vec![(10, 2), (11, 2)], + ExpectedRankedOffsets::Sparse, + ); + + let dense = (0..1_024u32) + .filter(|offset| offset % 10 != 0) + .map(|offset| addr(2, offset)) + .collect::(); + let dense_rows = u32::try_from(dense.len()).unwrap(); + assert_layout_matches_legacy( + 2, + 1_024, + dense, + vec![(20, 400), (21, dense_rows - 400)], + ExpectedRankedOffsets::Dense, + ); + + // Binary-copy compaction captures complete fragment ranges with + // `RoaringTreemap::insert_range`, then persists that bitmap. The + // serialized round trip retains run containers without `optimize()`. + let mut captured = RoaringTreemap::new(); + captured.insert_range(addr(3, 100)..addr(3, 9_900)); + let mut serialized = Vec::with_capacity(captured.serialized_size()); + captured.serialize_into(&mut serialized).unwrap(); + let persisted = RoaringTreemap::deserialize_from(std::io::Cursor::new(serialized)).unwrap(); + assert_layout_matches_legacy( + 3, + 10_000, + persisted, + vec![(31, 5_000), (30, 4_800)], + ExpectedRankedOffsets::Roaring, + ); + } + #[test] fn test_compact_lookup() { // Group A: out-of-order old frags [4, 3], split new frags (11 empty), @@ -329,18 +930,27 @@ mod tests { assert_eq!(remap.get(addr(7, 0)), Some(None)); // Fragment in no group -> unaffected. assert_eq!(remap.get(addr(9, 0)), None); + assert_eq!(remap.get(addr(4, 5)), Some(None)); assert!(!remap.is_empty()); } #[test] fn test_fragment_sets() { - // No rewritten rows at all: every covered fragment is fully deleted. - let dead = RowAddrRemap::compact([GroupInput { + // Each deferred version deletes a different covered fragment. The + // chain must retain the flat direct map's union semantics. + let first_dead = RowAddrRemap::compact([GroupInput { rewritten_old_row_addrs: RoaringTreemap::new(), - old_frag_ids: vec![3, 7], + old_frag_ids: vec![3], new_frags: vec![], }]) .unwrap(); + let second_dead = RowAddrRemap::compact([GroupInput { + rewritten_old_row_addrs: RoaringTreemap::new(), + old_frag_ids: vec![7], + new_frags: vec![], + }]) + .unwrap(); + let dead = RowAddrRemap::chained([first_dead.clone(), second_dead]); assert_eq!( dead.fully_deleted_fragments(), Some(RoaringBitmap::from_iter([3u32, 7u32])) @@ -363,11 +973,16 @@ mod tests { alive.affected_fragments(), RoaringBitmap::from_iter([0u32, 1u32]) ); + assert!( + RowAddrRemap::chained([first_dead, alive]) + .fully_deleted_fragments() + .is_none() + ); } #[test] fn test_compact_rejects_rewritten_addrs_outside_old_frags() { - // Rewritten addresses reference frag 5, not in old_frag_ids. The count + // Rewritten addresses reference frag 5, not in old_frags. The count // still matches (2 == 2), so only the per-fragment split catches it. let input = GroupInput { rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(0, 0), addr(5, 0)]), @@ -378,16 +993,15 @@ mod tests { } #[test] - fn test_compact_rejects_new_frags_out_of_write_order() { - // New fragments out of ascending id (write) order would make - // `compute_new_addr` accumulate rows in the wrong order, silently - // misplacing addresses. A zero-row fragment between them is ignored. - let input = GroupInput { + fn test_compact_preserves_explicit_fragment_order() { + let remap = RowAddrRemap::compact([GroupInput { rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(0, 0), addr(0, 1)]), old_frag_ids: vec![0], new_frags: vec![(12, 1), (11, 1)], - }; - assert!(RowAddrRemap::compact([input]).is_err()); + }]) + .unwrap(); + assert_eq!(remap.get(addr(0, 0)), Some(Some(addr(12, 0)))); + assert_eq!(remap.get(addr(0, 1)), Some(Some(addr(11, 0)))); } #[test] @@ -410,4 +1024,39 @@ mod tests { assert!(empty.is_empty()); assert_eq!(empty.get(addr(0, 0)), None); } + + #[test] + fn test_chained_lookup_and_batch() { + let first = RowAddrRemap::compact([GroupInput { + rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(0, 0), addr(0, 2)]), + old_frag_ids: vec![0], + new_frags: vec![(10, 2)], + }]) + .unwrap(); + let second = RowAddrRemap::compact([GroupInput { + rewritten_old_row_addrs: RoaringTreemap::from_iter([addr(10, 1)]), + old_frag_ids: vec![10], + new_frags: vec![(20, 1)], + }]) + .unwrap(); + let chain = RowAddrRemap::chained([first, second]); + + assert_eq!(chain.get(addr(0, 0)), Some(None)); + assert_eq!(chain.get(addr(0, 1)), Some(None)); + assert_eq!(chain.get(addr(0, 2)), Some(Some(addr(20, 0)))); + assert_eq!(chain.get(addr(1, 0)), None); + + let mut batch = vec![ + Some(addr(0, 0)), + Some(addr(0, 1)), + Some(addr(0, 2)), + Some(addr(1, 0)), + None, + ]; + chain.remap_in_place(&mut batch); + assert_eq!( + batch, + vec![None, None, Some(addr(20, 0)), Some(addr(1, 0)), None] + ); + } } diff --git a/rust/lance-core/src/utils/tokio.rs b/rust/lance-core/src/utils/tokio.rs index 46c9475665b..9137c3631c9 100644 --- a/rust/lance-core/src/utils/tokio.rs +++ b/rust/lance-core/src/utils/tokio.rs @@ -21,8 +21,8 @@ pub fn get_num_compute_intensive_cpus() -> usize { } fn calculate_num_compute_intensive_cpus() -> usize { - if let Ok(user_specified) = std::env::var("LANCE_CPU_THREADS") { - return user_specified.parse().unwrap(); + if let Ok(raw) = std::env::var("LANCE_CPU_THREADS") { + return parse_env_usize("LANCE_CPU_THREADS", &raw, 1).unwrap_or_else(|e| panic!("{e}")); } let cpus = num_cpus::get(); @@ -42,12 +42,38 @@ fn calculate_num_compute_intensive_cpus() -> usize { num_cpus::get() - *IO_CORE_RESERVATION } -pub static IO_CORE_RESERVATION: LazyLock = LazyLock::new(|| { - std::env::var("LANCE_IO_CORE_RESERVATION") - .unwrap_or("2".to_string()) +/// Parse an integer environment variable, rejecting values below `min`. +/// +/// The error names the variable, so a bad value is diagnosable instead of +/// surfacing as a bare `ParseIntError` or, for `LANCE_CPU_THREADS=0`, a panic +/// deep inside tokio's `max_blocking_threads`. +fn parse_env_usize(name: &str, raw: &str, min: usize) -> Result { + let value: usize = raw + .trim() .parse() - .unwrap() -}); + .map_err(|e| format!("environment variable {name} must be an integer, got {raw:?}: {e}"))?; + if value < min { + return Err(format!( + "environment variable {name} must be at least {min}, got {value}" + )); + } + Ok(value) +} + +/// Number of CPU cores held back for I/O and control tasks. +/// +/// Overridable via the `LANCE_IO_CORE_RESERVATION` environment variable; +/// defaults to `2` when unset. `0` is allowed (reserve nothing); +/// [`get_num_compute_intensive_cpus`] subtracts this from the core count to +/// size the compute pool. A non-integer value panics on first access with an +/// error naming the variable. +pub static IO_CORE_RESERVATION: LazyLock = + LazyLock::new(|| match std::env::var("LANCE_IO_CORE_RESERVATION") { + Ok(raw) => { + parse_env_usize("LANCE_IO_CORE_RESERVATION", &raw, 0).unwrap_or_else(|e| panic!("{e}")) + } + Err(_) => 2, + }); fn create_runtime() -> Runtime { Builder::new_multi_thread() @@ -66,11 +92,15 @@ static RUNTIME_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false); static ATFORK_INSTALLED: atomic::AtomicBool = atomic::AtomicBool::new(false); -fn global_cpu_runtime() -> &'static mut Runtime { +fn global_cpu_runtime() -> &'static Runtime { loop { let ptr = CPU_RUNTIME.load(Ordering::SeqCst); if !ptr.is_null() { - return unsafe { &mut *ptr }; + // SAFETY: `ptr` was produced by `Box::into_raw` below and is only ever + // reset to null by `atfork_tokio_child` in the forked child (single- + // threaded, async-signal context). The `Box` is never reclaimed, so the + // `Runtime` lives for the rest of the process. + return unsafe { &*ptr }; } if !RUNTIME_INSTALLED.fetch_or(true, Ordering::SeqCst) { break; @@ -82,7 +112,9 @@ fn global_cpu_runtime() -> &'static mut Runtime { } let new_ptr = Box::into_raw(Box::new(create_runtime())); CPU_RUNTIME.store(new_ptr, Ordering::SeqCst); - unsafe { &mut *new_ptr } + // SAFETY: `new_ptr` was just obtained from `Box::into_raw`, so it is non-null, + // aligned, and points to a live `Runtime` that is never reclaimed. + unsafe { &*new_ptr } } /// After a fork() operation, force re-creation of the BackgroundExecutor. Note: this function @@ -108,6 +140,43 @@ fn install_atfork() {} /// /// This can also be used to convert a big chunk of synchronous work into a future /// so that it can be run in parallel with something like StreamExt::buffered() +/// +/// # Only hand over substantial CPU work +/// +/// Dispatching to the pool has real overhead (a `spawn_blocking` hop plus a oneshot +/// channel round trip). As a rule of thumb the closure should be expected to do at +/// least ~100µs of CPU work; below that the thread-pool overhead is likely to +/// outweigh any parallelism benefit, and the work is better left inline. +/// +/// # The task must never wait on anything +/// +/// The CPU pool is sized to [`get_num_compute_intensive_cpus`], which is +/// `max(1, num_cpus - LANCE_IO_CORE_RESERVATION)`. On a big host that is plenty of +/// workers (e.g. 62 on a 64-core box), but in resource-constrained environments it can +/// collapse to a **single blocking thread** — on machines with `<= 3` visible CPUs +/// (1-vCPU VMs, CI runners, CPU-limited Kubernetes pods) the pool has exactly one +/// worker. A closure passed to `spawn_cpu` occupies one of these threads for its entire +/// lifetime, including any time it spends *parked*. So the closure must only consume +/// CPU and return; it must +/// **never** block, wait, or park. Concretely, the closure must not, directly or +/// transitively: +/// +/// * **No channels** — no blocking send/recv (`send_blocking`, blocking `recv`, etc.). +/// A full/empty channel parks the thread, and whatever would drain/fill the channel +/// may need the same pool to run. +/// * **No I/O** — no file, network, or object-store reads/writes, and no disk spills. +/// I/O parks the thread while making no progress on CPU work. +/// * **No locks** — no acquiring a contended lock (or any lock that is held across an +/// `.await` elsewhere). Waiting for the lock parks the thread. +/// * **No `block_on` / `.blocking_*`** — never drive or wait on another async task +/// from inside the closure. +/// +/// If any of these hold, the parked thread can starve the exact work that would +/// unblock it, deadlocking the whole pool with no timeout and no error — a silent +/// hang at 0% CPU. (See .) When work +/// needs to wait on a channel/lock/I/O, keep the waiting in an async task and only +/// hand the pure-CPU portion to `spawn_cpu`, e.g. build each batch with `spawn_cpu` +/// and dispatch it with `tx.send(batch).await` in the surrounding async code. pub fn spawn_cpu< E: std::error::Error + Send + 'static, F: FnOnce() -> std::result::Result + Send + 'static, @@ -115,13 +184,89 @@ pub fn spawn_cpu< >( func: F, ) -> impl Future> { - let (send, recv) = tokio::sync::oneshot::channel(); // Propagate the current span into the task let span = Span::current(); - global_cpu_runtime().spawn_blocking(move || { + let handle = global_cpu_runtime().spawn_blocking(move || { let _span_guard = span.enter(); - let result = func(); - let _ = send.send(result); + func() }); - recv.map(|res| res.unwrap()) + // Awaited through the join handle, not a result channel: a panic in `func` + // arrives as a `JoinError` still carrying its payload, so resuming it + // re-raises the original panic in the caller. Reporting the closure's + // outcome over a channel instead loses that -- the sender drops unsent and + // every panic in any `spawn_cpu` closure surfaces identically as an opaque + // `RecvError`, pointing here rather than at the fault. + handle.map(|res| match res { + Ok(result) => result, + Err(join_error) => match join_error.try_into_panic() { + Ok(panic) => std::panic::resume_unwind(panic), + // The CPU runtime outlives every caller, so its tasks are not + // cancelled out from under one. + Err(join_error) => panic!("spawn_cpu task failed: {join_error}"), + }, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A panic in the closure must reach the caller intact. + /// + /// Reporting the closure's outcome over a channel loses it: the sender + /// drops unsent and the caller can only see an opaque receive error, so + /// every panic in every `spawn_cpu` closure looks the same. + #[tokio::test] + async fn spawn_cpu_reraises_the_closure_panic() { + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let joined = tokio::spawn(async { + spawn_cpu(|| -> std::result::Result<(), std::io::Error> { + panic!("the original message") + }) + .await + }) + .await; + std::panic::set_hook(hook); + + let payload = joined + .expect_err("the closure's panic propagates to the caller") + .into_panic(); + let message = payload + .downcast_ref::<&str>() + .copied() + .expect("the original payload survives"); + assert_eq!(message, "the original message"); + } + + // The env vars feed process-global `LazyLock`s that read once and are read + // in parallel by other tests, so the pure parser is tested directly rather + // than by mutating the environment. + + #[test] + fn parses_valid_value_and_trims_surrounding_whitespace() { + assert_eq!(parse_env_usize("VAR", "8", 1).unwrap(), 8); + assert_eq!(parse_env_usize("VAR", " 8 ", 1).unwrap(), 8); + } + + #[test] + fn rejects_non_integer_naming_the_variable() { + let err = parse_env_usize("LANCE_CPU_THREADS", "abc", 1).unwrap_err(); + assert!(err.contains("LANCE_CPU_THREADS"), "{err}"); + assert!(err.contains("must be an integer"), "{err}"); + } + + #[test] + fn rejects_value_below_minimum() { + // LANCE_CPU_THREADS=0 parses fine but would panic in tokio's + // max_blocking_threads(0); the minimum stops it at the boundary. + let err = parse_env_usize("LANCE_CPU_THREADS", "0", 1).unwrap_err(); + assert!(err.contains("at least 1"), "{err}"); + } + + #[test] + fn allows_zero_when_minimum_is_zero() { + // LANCE_IO_CORE_RESERVATION=0 is valid: no cores reserved for IO. + assert_eq!(parse_env_usize("VAR", "0", 0).unwrap(), 0); + } } diff --git a/rust/lance-core/src/utils/tracing.rs b/rust/lance-core/src/utils/tracing.rs index 603a666e313..e1f19e3c6f4 100644 --- a/rust/lance-core/src/utils/tracing.rs +++ b/rust/lance-core/src/utils/tracing.rs @@ -66,6 +66,7 @@ pub const AUDIT_TYPE_DELETION: &str = "deletion"; pub const AUDIT_TYPE_MANIFEST: &str = "manifest"; pub const AUDIT_TYPE_INDEX: &str = "index"; pub const AUDIT_TYPE_DATA: &str = "data"; +pub const AUDIT_TYPE_TRANSACTION: &str = "transaction"; pub const TRACE_FILE_CREATE: &str = "create"; pub const TRACE_IO_EVENTS: &str = "lance::io_events"; pub const IO_TYPE_OPEN_SCALAR: &str = "open_scalar_index"; diff --git a/rust/lance-core/tests/cache_key_allocations.rs b/rust/lance-core/tests/cache_key_allocations.rs new file mode 100644 index 00000000000..ef73d485ef5 --- /dev/null +++ b/rust/lance-core/tests/cache_key_allocations.rs @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::borrow::Cow; +use std::cell::Cell; +use std::hint::black_box; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use lance_core::cache::{CacheKey, CacheKeySchema, CacheNamespace, InternalCacheKey, KeyBuilder}; + +struct TrackingAllocator; + +thread_local! { + static TRACK_ALLOCATIONS: Cell = const { Cell::new(false) }; +} + +static ALLOCATION_COUNT: AtomicUsize = AtomicUsize::new(0); + +fn record_allocation() { + if TRACK_ALLOCATIONS.try_with(Cell::get).unwrap_or(false) { + ALLOCATION_COUNT.fetch_add(1, Ordering::Relaxed); + } +} + +unsafe impl GlobalAlloc for TrackingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + record_allocation(); + unsafe { System.alloc(layout) } + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + record_allocation(); + unsafe { System.alloc_zeroed(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + record_allocation(); + unsafe { System.realloc(ptr, layout, new_size) } + } +} + +#[global_allocator] +static ALLOCATOR: TrackingAllocator = TrackingAllocator; + +fn measured_allocations(operation: impl FnOnce()) -> usize { + TRACK_ALLOCATIONS.with(|tracking| tracking.set(false)); + ALLOCATION_COUNT.store(0, Ordering::Relaxed); + TRACK_ALLOCATIONS.with(|tracking| tracking.set(true)); + operation(); + TRACK_ALLOCATIONS.with(|tracking| tracking.set(false)); + ALLOCATION_COUNT.load(Ordering::Relaxed) +} + +fn prepare_key(namespace: CacheNamespace, key: &K) -> InternalCacheKey { + let mut builder = KeyBuilder::new(namespace, K::stable_type_id(), K::schema()); + key.write_key(&mut builder); + builder.finish() +} + +struct PageKey { + path: &'static str, + column_index: u32, + page_index: u64, +} + +impl CacheKey for PageKey { + type ValueType = u64; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed("unused") + } + + fn type_name() -> &'static str { + "allocation-test-page" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("allocation-test-page", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.path); + builder.write_u32(self.column_index); + builder.write_u64(self.page_index); + } +} + +struct OptionalUuidKey { + generation: u64, + uuid: Option<[u8; 16]>, +} + +impl CacheKey for OptionalUuidKey { + type ValueType = u64; + + fn key(&self) -> Cow<'_, str> { + Cow::Borrowed("unused") + } + + fn type_name() -> &'static str { + "allocation-test-optional-uuid" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("allocation-test-optional-uuid", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.generation); + match self.uuid { + Some(uuid) => { + builder.write_some(); + builder.write_fixed_bytes(&uuid); + } + None => builder.write_none(), + } + } +} + +#[test] +fn production_shaped_typed_keys_allocate_nothing_after_warmup() { + let namespace = CacheNamespace::root() + .child("tenant-with-a-long-stable-identifier") + .child("index-with-a-long-stable-identifier"); + let page = PageKey { + path: "indices/01999f62-c3c2-7d6f-820d-22e7db948f31/pages/000000000042.lance", + column_index: 17, + page_index: 42, + }; + let uuid = OptionalUuidKey { + generation: 9, + uuid: Some(*b"0123456789abcdef"), + }; + let no_uuid = OptionalUuidKey { + generation: 10, + uuid: None, + }; + + black_box(prepare_key(namespace, &page)); + black_box(prepare_key(namespace, &uuid)); + black_box(prepare_key(namespace, &no_uuid)); + + assert_eq!( + measured_allocations(|| { + black_box(prepare_key(namespace, &page)); + }), + 0 + ); + assert_eq!( + measured_allocations(|| { + black_box(prepare_key(namespace, &uuid)); + }), + 0 + ); + assert_eq!( + measured_allocations(|| { + black_box(prepare_key(namespace, &no_uuid)); + }), + 0 + ); +} diff --git a/rust/lance-datafusion/Cargo.toml b/rust/lance-datafusion/Cargo.toml index 7f93ab619cd..8b9c702d2fb 100644 --- a/rust/lance-datafusion/Cargo.toml +++ b/rust/lance-datafusion/Cargo.toml @@ -24,10 +24,11 @@ datafusion-physical-expr.workspace = true datafusion-substrait = {workspace = true, optional = true} datafusion.workspace = true futures.workspace = true +half.workspace = true jsonb = {workspace = true} lance-arrow.workspace = true lance-core = {workspace = true, features = ["datafusion"]} -lance-datagen.workspace = true +lance-datagen = {workspace = true, optional = true} lance-geo = {workspace = true, optional = true} chrono.workspace = true log.workspace = true @@ -38,12 +39,14 @@ tracing.workspace = true [build-dependencies] prost-build.workspace = true -protobuf-src = {version = "2.1", optional = true} +protobuf-src = { workspace = true, optional = true } [dev-dependencies] lance-datagen.workspace = true +rstest.workspace = true [features] +datagen = ["dep:lance-datagen"] geo = ["dep:lance-geo"] substrait = ["dep:datafusion-substrait"] protoc = ["dep:protobuf-src"] diff --git a/rust/lance-datafusion/src/chunker.rs b/rust/lance-datafusion/src/chunker.rs index f30e215e712..63523460899 100644 --- a/rust/lance-datafusion/src/chunker.rs +++ b/rust/lance-datafusion/src/chunker.rs @@ -42,8 +42,8 @@ impl BatchReaderChunker { buffer_total - self.i } - async fn fill_buffer(&mut self) -> Result<()> { - while self.buffered_len() < self.output_size { + async fn fill_buffer(&mut self, output_size: usize) -> Result<()> { + while self.buffered_len() < output_size { match self.inner.next().await { Some(Ok(batch)) => self.buffered.push_back(batch), Some(Err(e)) => return Err(e.into()), @@ -54,7 +54,11 @@ impl BatchReaderChunker { } async fn next(&mut self) -> Option>> { - match self.fill_buffer().await { + self.next_sized(self.output_size).await + } + + async fn next_sized(&mut self, output_size: usize) -> Option>> { + match self.fill_buffer(output_size).await { Ok(_) => {} Err(e) => return Some(Err(e)), }; @@ -63,7 +67,7 @@ impl BatchReaderChunker { let mut rows_collected = 0; - while rows_collected < self.output_size { + while rows_collected < output_size { if let Some(batch) = self.buffered.pop_front() { // Skip empty batch if batch.num_rows() == 0 { @@ -72,7 +76,7 @@ impl BatchReaderChunker { let rows_remaining_in_batch = batch.num_rows() - self.i; let rows_to_take = - std::cmp::min(rows_remaining_in_batch, self.output_size - rows_collected); + std::cmp::min(rows_remaining_in_batch, output_size - rows_collected); if rows_to_take == rows_remaining_in_batch { // We're taking the whole batch, so we can just move it @@ -104,6 +108,53 @@ impl BatchReaderChunker { Some(Ok(batches)) } } + + async fn next_at_most(&mut self, output_size: usize) -> Option>> { + loop { + let batch = match self.buffered.pop_front() { + Some(batch) => batch, + None => match self.inner.next().await { + Some(Ok(batch)) => batch, + Some(Err(error)) => return Some(Err(error.into())), + None => return None, + }, + }; + + if batch.num_rows() == 0 { + continue; + } + + let rows_remaining_in_batch = batch.num_rows() - self.i; + let rows_to_take = rows_remaining_in_batch.min(output_size); + if rows_to_take == rows_remaining_in_batch { + let batch = if self.i == 0 { + batch + } else { + batch.slice(self.i, rows_to_take) + }; + self.i = 0; + return Some(Ok(vec![batch])); + } + + let output = batch.slice(self.i, rows_to_take); + self.i += rows_to_take; + self.buffered.push_front(batch); + return Some(Ok(vec![output])); + } + } +} + +struct VariableBatchReaderChunker { + chunker: BatchReaderChunker, + output_sizes: I, + is_done: bool, +} + +struct VariableBreakStreamState { + chunker: BatchReaderChunker, + output_sizes: I, + rows_remaining: Option, + is_done: bool, } struct BreakStreamState { @@ -186,6 +237,210 @@ pub fn chunk_stream( .boxed() } +/// Preserve input batch boundaries while inserting the requested row boundaries. +/// +/// The requested sizes must describe the complete input. Unlike +/// [`chunk_stream_with_sizes`], this does not combine adjacent input batches. It +/// only slices a batch when it crosses a requested boundary. +/// +/// # Example +/// +/// ``` +/// # use datafusion::physical_plan::SendableRecordBatchStream; +/// # use lance_datafusion::chunker::break_stream_with_sizes; +/// # fn split_stream(stream: SendableRecordBatchStream) { +/// let batches = break_stream_with_sizes(stream, vec![512, 512, 256]); +/// # drop(batches); +/// # } +/// ``` +pub fn break_stream_with_sizes( + stream: SendableRecordBatchStream, + output_sizes: I, +) -> Pin>> + Send>> +where + I: IntoIterator, + I::IntoIter: Send + 'static, +{ + let state = VariableBreakStreamState { + chunker: BatchReaderChunker::new(stream, 1), + output_sizes: output_sizes.into_iter(), + rows_remaining: None, + is_done: false, + }; + futures::stream::unfold(state, |mut state| async move { + if state.is_done { + return None; + } + + if state.rows_remaining.is_none() { + let Some(output_size) = state.output_sizes.next() else { + return match state.chunker.next_at_most(1).await { + None => None, + Some(Ok(_)) => { + state.is_done = true; + Some(( + Err(lance_core::Error::invalid_input( + "Input contained more rows than the requested chunk sizes", + )), + state, + )) + } + Some(Err(error)) => { + state.is_done = true; + Some((Err(error), state)) + } + }; + }; + if output_size == 0 { + state.is_done = true; + return Some(( + Err(lance_core::Error::invalid_input( + "Requested chunk sizes must be greater than zero", + )), + state, + )); + } + state.rows_remaining = Some(output_size); + } + + let Some(rows_remaining) = state.rows_remaining else { + state.is_done = true; + return Some(( + Err(lance_core::Error::internal( + "Requested chunk boundary was not initialized", + )), + state, + )); + }; + match state.chunker.next_at_most(rows_remaining).await { + Some(Ok(batches)) => { + let actual_size = batches.iter().map(RecordBatch::num_rows).sum::(); + let Some(rows_remaining) = rows_remaining.checked_sub(actual_size) else { + state.is_done = true; + return Some(( + Err(lance_core::Error::internal( + "A boundary-preserving chunk exceeded its requested row count", + )), + state, + )); + }; + state.rows_remaining = (rows_remaining > 0).then_some(rows_remaining); + Some((Ok(batches), state)) + } + Some(Err(error)) => { + state.is_done = true; + Some((Err(error), state)) + } + None => { + state.is_done = true; + Some(( + Err(lance_core::Error::invalid_input(format!( + "Input ended with {rows_remaining} rows remaining in a requested chunk" + ))), + state, + )) + } + } + }) + .boxed() +} + +/// Given a stream of record batches, yield chunks with the requested row counts. +/// +/// The requested sizes must describe the complete input. An error is returned if +/// the input ends early, contains additional rows, or a requested size is zero. +/// Sizes are consumed lazily as chunks are requested. +/// +/// # Example +/// +/// ``` +/// # use datafusion::physical_plan::SendableRecordBatchStream; +/// # use lance_datafusion::chunker::chunk_stream_with_sizes; +/// # fn split_stream(stream: SendableRecordBatchStream) { +/// let chunks = chunk_stream_with_sizes(stream, vec![512, 512, 256]); +/// # drop(chunks); +/// # } +/// ``` +pub fn chunk_stream_with_sizes( + stream: SendableRecordBatchStream, + output_sizes: I, +) -> Pin>> + Send>> +where + I: IntoIterator, + I::IntoIter: Send + 'static, +{ + let state = VariableBatchReaderChunker { + chunker: BatchReaderChunker::new(stream, 1), + output_sizes: output_sizes.into_iter(), + is_done: false, + }; + futures::stream::unfold(state, |mut state| async move { + if state.is_done { + return None; + } + + let Some(output_size) = state.output_sizes.next() else { + return match state.chunker.next_sized(1).await { + None => None, + Some(Ok(_)) => { + state.is_done = true; + Some(( + Err(lance_core::Error::invalid_input( + "Input contained more rows than the requested chunk sizes", + )), + state, + )) + } + Some(Err(error)) => { + state.is_done = true; + Some((Err(error), state)) + } + }; + }; + + if output_size == 0 { + state.is_done = true; + return Some(( + Err(lance_core::Error::invalid_input( + "Requested chunk sizes must be greater than zero", + )), + state, + )); + } + + match state.chunker.next_sized(output_size).await { + Some(Ok(batches)) => { + let actual_size = batches.iter().map(RecordBatch::num_rows).sum::(); + if actual_size == output_size { + Some((Ok(batches), state)) + } else { + state.is_done = true; + Some(( + Err(lance_core::Error::invalid_input(format!( + "Input ended after {actual_size} rows while filling a requested {output_size}-row chunk" + ))), + state, + )) + } + } + Some(Err(error)) => { + state.is_done = true; + Some((Err(error), state)) + } + None => { + state.is_done = true; + Some(( + Err(lance_core::Error::invalid_input(format!( + "Input ended before a requested {output_size}-row chunk could be filled" + ))), + state, + )) + } + } + }) + .boxed() +} + /// Given a stream of record batches, this will yield batches of a fixed size. /// /// This stream _will_ combine record batches and so it can be fairly expensive as it will @@ -311,7 +566,10 @@ where #[cfg(test)] mod tests { - use std::sync::Arc; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use arrow::datatypes::{Int32Type, Int64Type}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; @@ -360,6 +618,82 @@ mod tests { assert_eq!(chunked[2].len(), 1); assert_eq!(chunked[2][0].num_rows(), 8); + let sizes_consumed = Arc::new(AtomicUsize::new(0)); + let requested_sizes = [9, 10, 9].into_iter().inspect({ + let sizes_consumed = sizes_consumed.clone(); + move |_| { + sizes_consumed.fetch_add(1, Ordering::SeqCst); + } + }); + let mut chunked = super::chunk_stream_with_sizes(make_stream(), requested_sizes); + assert_eq!(sizes_consumed.load(Ordering::SeqCst), 0); + let first_chunk = chunked.next().await.unwrap().unwrap(); + assert_eq!(sizes_consumed.load(Ordering::SeqCst), 1); + let mut chunked = chunked.try_collect::>().await.unwrap(); + chunked.insert(0, first_chunk); + assert_eq!(sizes_consumed.load(Ordering::SeqCst), 3); + assert_eq!( + chunked + .iter() + .map(|batches| batches.iter().map(|batch| batch.num_rows()).sum::()) + .collect::>(), + vec![9, 10, 9] + ); + + let error = super::chunk_stream_with_sizes(make_stream(), vec![10, 17]) + .try_collect::>() + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("more rows than the requested chunk sizes") + ); + + let error = super::chunk_stream_with_sizes(make_stream(), vec![10, 19]) + .try_collect::>() + .await + .unwrap_err(); + assert!(error.to_string().contains("ended after 18 rows")); + + let sizes_consumed = Arc::new(AtomicUsize::new(0)); + let requested_sizes = [9, 10, 9].into_iter().inspect({ + let sizes_consumed = sizes_consumed.clone(); + move |_| { + sizes_consumed.fetch_add(1, Ordering::SeqCst); + } + }); + let mut broken = super::break_stream_with_sizes(make_stream(), requested_sizes); + assert_eq!(sizes_consumed.load(Ordering::SeqCst), 0); + let first_batch = broken.next().await.unwrap().unwrap(); + assert_eq!(sizes_consumed.load(Ordering::SeqCst), 1); + let mut broken = broken.try_collect::>().await.unwrap(); + broken.insert(0, first_batch); + assert_eq!(sizes_consumed.load(Ordering::SeqCst), 3); + assert_eq!( + broken + .iter() + .map(|batches| batches.iter().map(|batch| batch.num_rows()).sum::()) + .collect::>(), + vec![9, 1, 5, 4, 9] + ); + + let error = super::break_stream_with_sizes(make_stream(), vec![27]) + .try_collect::>() + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("more rows than the requested chunk sizes") + ); + + let error = super::break_stream_with_sizes(make_stream(), vec![29]) + .try_collect::>() + .await + .unwrap_err(); + assert!(error.to_string().contains("1 rows remaining")); + let chunked = super::chunk_concat_stream(make_stream(), 10) .try_collect::>() .await diff --git a/rust/lance-datafusion/src/exec.rs b/rust/lance-datafusion/src/exec.rs index 8f346f45612..994e16b783d 100644 --- a/rust/lance-datafusion/src/exec.rs +++ b/rust/lance-datafusion/src/exec.rs @@ -6,6 +6,7 @@ use std::{ collections::HashMap, fmt::{self, Formatter}, + num::NonZero, sync::{Arc, Mutex, OnceLock}, time::Duration, }; @@ -14,9 +15,8 @@ use chrono::{DateTime, Utc}; use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; -use datafusion::physical_plan::metrics::MetricType; use datafusion::{ - catalog::streaming::StreamingTable, + catalog::{TableProvider, streaming::StreamingTable}, dataframe::DataFrame, execution::{ TaskContext, @@ -26,16 +26,19 @@ use datafusion::{ runtime_env::RuntimeEnvBuilder, }, physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + SendableRecordBatchStream, analyze::AnalyzeExec, coalesce_partitions::CoalescePartitionsExec, display::DisplayableExecutionPlan, execution_plan::{Boundedness, CardinalityEffect, EmissionType}, metrics::MetricValue, + sorts::sort_preserving_merge::SortPreservingMergeExec, stream::RecordBatchStreamAdapter, streaming::PartitionStream, }, }; +use datafusion::{execution::memory_pool::TrackConsumersPool, physical_plan::metrics::MetricType}; use datafusion_common::{DataFusionError, Statistics}; use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; @@ -55,8 +58,9 @@ use crate::udf::register_functions; use crate::{ chunker::StrictBatchSizeStream, utils::{ - BYTES_READ_METRIC, INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, IOPS_METRIC, - MetricsExt, PARTS_LOADED_METRIC, REQUESTS_METRIC, + BYTES_READ_METRIC, INDEX_CACHE_HITS_METRIC, INDEX_CACHE_MISSES_METRIC, + INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, IOPS_METRIC, MetricsExt, + PARTS_LOADED_METRIC, REQUESTS_METRIC, }, }; @@ -152,10 +156,6 @@ impl ExecutionPlan for OneShotExec { "OneShotExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.schema.clone() } @@ -243,10 +243,6 @@ impl ExecutionPlan for TracedExec { "TracedExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -310,7 +306,7 @@ impl std::fmt::Debug for LanceExecutionOptions { } } -const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 100 * 1024 * 1024; +const DEFAULT_LANCE_MEM_POOL_SIZE_PER_PARTITION: u64 = 150 * 1024 * 1024; const DEFAULT_LANCE_MAX_TEMP_DIRECTORY_SIZE: u64 = 100 * 1024 * 1024 * 1024; // 100GB impl LanceExecutionOptions { @@ -366,12 +362,21 @@ pub fn new_session_context(options: &LanceExecutionOptions) -> SessionContext { session_config = session_config.with_target_partitions(target_partition); } if options.use_spilling() { + // The default 10MB sort spill reservation seems to be too small for many common cases. + // + // There currently is no reasonable guidance provided by DataFusion for setting this value. + // We bump this to 40MB but try a smaller value if the mem pool is small. + let sort_spill_reservation_bytes = + (options.mem_pool_size() / 3).min(40 * 1024 * 1024) as usize; + session_config = + session_config.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes); let disk_manager_builder = DiskManagerBuilder::default() .with_max_temp_directory_size(options.max_temp_directory_size()); runtime_env_builder = runtime_env_builder .with_disk_manager_builder(disk_manager_builder) - .with_memory_pool(Arc::new(FairSpillPool::new( - options.mem_pool_size() as usize + .with_memory_pool(Arc::new(TrackConsumersPool::new( + FairSpillPool::new(options.mem_pool_size() as usize), + NonZero::try_from(16).unwrap(), ))); } let runtime_env = runtime_env_builder.build_arc().unwrap(); @@ -486,12 +491,88 @@ pub struct ExecutionSummaryCounts { pub index_comparisons: usize, /// Additional metrics for more detailed statistics. These are subject to change in the future /// and should only be used for debugging purposes. + /// + /// Newer metrics (e.g. [`INDEX_CACHE_HITS_METRIC`], [`INDEX_CACHE_MISSES_METRIC`]) are added + /// here rather than as `pub` fields, so this struct stays backwards compatible for callers + /// that construct or destructure it. Prefer the typed accessors below. pub all_counts: HashMap, /// Additional time metrics for more detailed statistics, stored in nanoseconds. /// These are subject to change in the future and should only be used for debugging purposes. pub all_times: HashMap, } +impl ExecutionSummaryCounts { + /// Number of index cache page lookups where the loader was not executed + /// (per-page granularity). + /// + /// A "hit" is any page-level lookup at an instrumented cache boundary that + /// did not run the loader on this call. That covers both a true cache hit + /// on an already-populated entry and a coalesced concurrent load where an + /// in-flight loader started by a different caller produced the value. + /// + /// Instrumented boundaries in this release: + /// BTree page, IVF partition (v2, `write_cache=true` scan path), inverted + /// posting list (grouped and per-token), inverted per-token metadata + /// (`PostingMetadataKey`), inverted phrase positions (`PositionKey`), + /// bitmap posting (Equals / Range / IsIn), ngram posting, and rtree page + /// / null slot. + /// + /// Caveats: + /// * IVF v2 streaming scans and legacy v1 IVF partitions run + /// `load_partition` with `write_cache=false`. Those loads always execute + /// the loader and never write the result back, so they are reported as a + /// miss on every call. See [`Self::index_cache_hit_ratio`]. + /// * A cold posting-list lookup on the grouped inverted layout can record + /// up to two misses (posting-list group + per-token metadata) for a + /// single term. + /// + /// Other index cache boundaries such as HNSW graph pages and quantizer + /// codebooks are not yet instrumented; a scan that only touches those + /// paths returns `0` here. + pub fn index_cache_hits(&self) -> usize { + self.all_counts + .get(INDEX_CACHE_HITS_METRIC) + .copied() + .unwrap_or(0) + } + + /// Number of index cache page lookups that had to execute the loader + /// (per-page granularity). + /// + /// A "miss" is any page-level lookup at an instrumented cache boundary + /// where the loader ran, i.e. the page was not resident and had to be + /// materialised (typically from storage). See + /// [`Self::index_cache_hits`] for the paired counter and the list of + /// instrumented boundaries. + pub fn index_cache_misses(&self) -> usize { + self.all_counts + .get(INDEX_CACHE_MISSES_METRIC) + .copied() + .unwrap_or(0) + } + + /// Ratio of index cache hits to total lookups. Returns `0.0` when no lookups + /// were recorded in this scan. + /// + /// This ratio only reflects paths that write their result back to the + /// index cache. Streaming scans (IVF v2 `write_cache=false` and legacy v1 + /// IVF `load_partition_stream`) intentionally bypass the cache and are + /// counted as misses on every call, so a workload dominated by streaming + /// vector scans will report a hit ratio near `0.0` regardless of cache + /// size. + pub fn index_cache_hit_ratio(&self) -> f32 { + // Widen to u128 before summing so a pathological (hits + misses) + // overflow can't panic in debug builds nor wrap in release builds. + let hits = self.index_cache_hits() as u128; + let total = hits + self.index_cache_misses() as u128; + if total == 0 { + 0.0 + } else { + hits as f32 / total as f32 + } + } +} + pub fn collect_execution_metrics(node: &dyn ExecutionPlan, counts: &mut ExecutionSummaryCounts) { if let Some(metrics) = node.metrics() { for (metric_name, count) in metrics.iter_counts() { @@ -552,6 +633,8 @@ fn report_plan_summary_metrics(plan: &dyn ExecutionPlan, options: &LanceExecutio indices_loaded = counts.indices_loaded, parts_loaded = counts.parts_loaded, index_comparisons = counts.index_comparisons, + index_cache_hits = counts.index_cache_hits(), + index_cache_misses = counts.index_cache_misses(), ); } if let Some(callback) = options.execution_stats_callback.as_ref() { @@ -610,8 +693,17 @@ pub fn execute_plan( // Coalesce to a single partition if the optimizer left more than one. // EnforceDistribution may remove RepartitionExec(1) nodes when the parent // declares UnspecifiedDistribution, leaving multi-partition plans here. + // + // If the plan carries an output ordering (e.g. a top-k `SortExec` whose + // result was later repartitioned to parallelize downstream operators), + // a plain `CoalescePartitionsExec` would scramble that order because it + // merges partitions in scheduling-dependent order. Use an order-preserving + // merge in that case instead, mirroring what `EnforceDistribution` itself + // does when it needs to merge an ordered, multi-partition plan. let plan: Arc = if plan.properties().partitioning.partition_count() == 1 { plan + } else if let Some(ordering) = plan.output_ordering() { + Arc::new(SortPreservingMergeExec::new(ordering.clone(), plan)) } else { Arc::new(CoalescePartitionsExec::new(plan)) }; @@ -640,7 +732,8 @@ pub async fn analyze_plan( let analyze = Arc::new(AnalyzeExec::new( true, true, - vec![MetricType::SUMMARY], + vec![MetricType::Summary], + None, plan, schema, )); @@ -867,6 +960,49 @@ impl SessionContextExt for SessionContext { } } +/// Scan a [`TableProvider`] into a single-partition [`SendableRecordBatchStream`]. +/// +/// Multi-partition providers are coalesced into a single partition. This adapts a +/// re-scannable provider back into the one stream the writer pipeline consumes; +/// re-scanning the same provider (e.g. on a write retry) yields a fresh stream. +/// +/// # Examples +/// +/// ``` +/// # use std::sync::Arc; +/// # use arrow_array::{Int32Array, RecordBatch}; +/// # use arrow_schema::{DataType, Field, Schema}; +/// # use datafusion::catalog::TableProvider; +/// # use datafusion::datasource::MemTable; +/// # use futures::TryStreamExt; +/// # use lance_datafusion::exec::provider_to_stream; +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); +/// let batch = +/// RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])?; +/// let provider: Arc = Arc::new(MemTable::try_new(schema, vec![vec![batch]])?); +/// +/// // A re-scannable provider yields a fresh stream on each call. +/// let batches: Vec = provider_to_stream(provider).await?.try_collect().await?; +/// assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 3); +/// # Ok(()) +/// # } +/// ``` +pub async fn provider_to_stream( + provider: Arc, +) -> Result { + let ctx = SessionContext::new(); + let plan = provider.scan(&ctx.state(), None, &[], None).await?; + let plan: Arc = + if plan.properties().output_partitioning().partition_count() > 1 { + Arc::new(CoalescePartitionsExec::new(plan)) + } else { + plan + }; + Ok(plan.execute(0, ctx.task_ctx())?) +} + #[derive(Clone, Debug)] pub struct StrictBatchSizeExec { input: Arc, @@ -894,10 +1030,6 @@ impl ExecutionPlan for StrictBatchSizeExec { "StrictBatchSizeExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { self.input.properties() } @@ -938,7 +1070,7 @@ impl ExecutionPlan for StrictBatchSizeExec { fn partition_statistics( &self, partition: Option, - ) -> datafusion_common::Result { + ) -> datafusion_common::Result> { self.input.partition_statistics(partition) } @@ -1000,10 +1132,6 @@ impl ExecutionPlan for HardCapBatchSizeExec { "HardCapBatchSizeExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { self.input.properties() } @@ -1065,7 +1193,7 @@ impl ExecutionPlan for HardCapBatchSizeExec { fn partition_statistics( &self, partition: Option, - ) -> datafusion_common::Result { + ) -> datafusion_common::Result> { self.input.partition_statistics(partition) } diff --git a/rust/lance-datafusion/src/expr.rs b/rust/lance-datafusion/src/expr.rs index a0da34ba2bb..059a991bd56 100644 --- a/rust/lance-datafusion/src/expr.rs +++ b/rust/lance-datafusion/src/expr.rs @@ -129,6 +129,16 @@ pub fn safe_coerce_scalar(value: &ScalarValue, ty: &DataType) -> Option val.map(|v| ScalarValue::Float32(Some(v as f32))), DataType::Float64 => val.map(|v| ScalarValue::Float64(Some(v as f64))), DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => value.cast_to(ty).ok(), + DataType::Time32(TimeUnit::Second) => val.and_then(|v| { + i32::try_from(v) + .ok() + .map(|v| ScalarValue::Time32Second(Some(v))) + }), + DataType::Time32(TimeUnit::Millisecond) => val.and_then(|v| { + i32::try_from(v) + .ok() + .map(|v| ScalarValue::Time32Millisecond(Some(v))) + }), _ => None, }, ScalarValue::UInt8(val) => match ty { @@ -448,6 +458,25 @@ pub fn safe_coerce_scalar(value: &ScalarValue, ty: &DataType) -> Option Some(value.clone()), _ => None, }, + ScalarValue::LargeBinary(_) => match ty { + DataType::LargeBinary => Some(value.clone()), + _ => None, + }, + ScalarValue::Decimal128(_, _, _) => match ty { + DataType::Decimal128(_, _) => value.cast_to(ty).ok(), + _ => None, + }, + ScalarValue::Decimal256(_, _, _) => match ty { + DataType::Decimal256(_, _) => value.cast_to(ty).ok(), + _ => None, + }, + ScalarValue::DurationSecond(_) + | ScalarValue::DurationMillisecond(_) + | ScalarValue::DurationMicrosecond(_) + | ScalarValue::DurationNanosecond(_) => match ty { + DataType::Duration(_) => value.cast_to(ty).ok(), + _ => None, + }, // A dictionary-encoded literal (e.g. produced by DataFusion's dictionary // cast in the scalar-index path) coerces by unwrapping its underlying value. ScalarValue::Dictionary(_, inner) => safe_coerce_scalar(inner, ty), @@ -457,10 +486,34 @@ pub fn safe_coerce_scalar(value: &ScalarValue, ty: &DataType) -> Options assert_eq!( @@ -732,6 +785,13 @@ mod tests { ), Some(ScalarValue::Time64Nanosecond(Some(5000000000))) ); + assert_eq!( + safe_coerce_scalar( + &ScalarValue::DurationNanosecond(Some(2_000_000)), + &DataType::Duration(TimeUnit::Millisecond), + ), + Some(ScalarValue::DurationMillisecond(Some(2))) + ); } #[test] @@ -789,6 +849,31 @@ mod tests { ), Some(ScalarValue::BinaryView(Some(vec![1, 2, 3]))) ); + assert_eq!( + safe_coerce_scalar( + &ScalarValue::LargeBinary(Some(vec![1, 2, 3])), + &DataType::LargeBinary + ), + Some(ScalarValue::LargeBinary(Some(vec![1, 2, 3]))) + ); + } + + #[test] + fn test_decimal_coerce() { + assert_eq!( + safe_coerce_scalar( + &ScalarValue::Decimal128(Some(2), 10, 0), + &DataType::Decimal128(12, 2), + ), + Some(ScalarValue::Decimal128(Some(200), 12, 2)) + ); + assert_eq!( + safe_coerce_scalar( + &ScalarValue::Decimal256(Some(i256::from_i128(2)), 76, 0), + &DataType::Decimal256(76, 2), + ), + Some(ScalarValue::Decimal256(Some(i256::from_i128(200)), 76, 2)) + ); } #[test] diff --git a/rust/lance-datafusion/src/lib.rs b/rust/lance-datafusion/src/lib.rs index ecc78672924..ce67c5fdd1c 100644 --- a/rust/lance-datafusion/src/lib.rs +++ b/rust/lance-datafusion/src/lib.rs @@ -4,6 +4,7 @@ pub mod aggregate; pub mod chunker; pub mod dataframe; +#[cfg(any(test, feature = "datagen"))] pub mod datagen; pub mod exec; pub mod expr; @@ -21,6 +22,7 @@ pub mod pb { #![allow(clippy::use_self)] include!(concat!(env!("OUT_DIR"), "/lance.datafusion.rs")); } +mod signed_zero; pub mod spill; pub mod sql; #[cfg(feature = "substrait")] diff --git a/rust/lance-datafusion/src/logical_expr.rs b/rust/lance-datafusion/src/logical_expr.rs index 0eed438dae7..db9abd7e204 100644 --- a/rust/lance-datafusion/src/logical_expr.rs +++ b/rust/lance-datafusion/src/logical_expr.rs @@ -51,14 +51,10 @@ pub fn resolve_column_type(expr: &Expr, schema: &Schema) -> Option { field_path.push(c.name.as_str()); break; } - Expr::ScalarFunction(udf) => { - if udf.name() == GetFieldFunc::default().name() { - let name = get_as_string_scalar_opt(&udf.args[1])?; - field_path.push(name); - current_expr = &udf.args[0]; - } else { - return None; - } + Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => { + let name = get_as_string_scalar_opt(&udf.args[1])?; + field_path.push(name); + current_expr = &udf.args[0]; } _ => return None, } @@ -285,8 +281,10 @@ pub fn field_path_to_expr(field_path: &str) -> Result { ))); } - // Build the column expression, handling nested fields - let mut expr = col(&parts[0]); + // Build the column expression, handling nested fields. + let mut expr = Expr::Column(datafusion::common::Column::new_unqualified( + parts[0].clone(), + )); for part in &parts[1..] { expr = expr.field_newstyle(part); } @@ -301,8 +299,26 @@ mod tests { use super::*; use arrow_schema::{Field, Schema as ArrowSchema}; + use datafusion::common::Column; use datafusion_functions::core::expr_ext::FieldAccessor; + #[test] + fn test_field_path_to_expr_preserves_case_sensitive_root_column() { + let expr = field_path_to_expr("VECTOR").unwrap(); + + assert_eq!(expr, Expr::Column(Column::new_unqualified("VECTOR"))); + } + + #[test] + fn test_field_path_to_expr_preserves_case_sensitive_escaped_nested_path() { + let expr = field_path_to_expr("Parent.`Child.With.Dot`").unwrap(); + + assert_eq!( + expr, + Expr::Column(Column::new_unqualified("Parent")).field_newstyle("Child.With.Dot") + ); + } + #[test] fn test_resolve_large_utf8() { let arrow_schema = ArrowSchema::new(vec![Field::new("a", DataType::LargeUtf8, false)]); diff --git a/rust/lance-datafusion/src/planner.rs b/rust/lance-datafusion/src/planner.rs index 1e62cba42d8..bc061491859 100644 --- a/rust/lance-datafusion/src/planner.rs +++ b/rust/lance-datafusion/src/planner.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use crate::exec::{LanceExecutionOptions, get_session_context}; use crate::expr::safe_coerce_scalar; use crate::logical_expr::{coerce_filter_type_to_boolean, get_as_string_scalar_opt, resolve_expr}; +use crate::signed_zero::{normalize_zero_comparisons, rewrite_signed_zero_comparisons}; use crate::sql::{parse_sql_expr, parse_sql_filter}; use arrow::compute::CastOptions; use arrow_array::ListArray; @@ -17,8 +18,9 @@ use arrow_buffer::OffsetBuffer; use arrow_cast::cast_with_options; use arrow_schema::{DataType as ArrowDataType, Field, SchemaRef, TimeUnit}; use arrow_select::concat::concat; +use datafusion::catalog::Session; use datafusion::common::DFSchema; -use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion, TreeNodeVisitor}; +use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor}; use datafusion::config::ConfigOptions; use datafusion::error::Result as DFResult; use datafusion::execution::context::SessionState; @@ -58,6 +60,31 @@ fn encode_jsonb(json_str: &str) -> Result { Ok(Expr::Literal(ScalarValue::LargeBinary(Some(bytes)), None)) } +// The escape in `LIKE/ILIKE ... ESCAPE ''` must be exactly one character. +// Reject empty or multi-character escape strings rather than silently treating +// them as "no escape" or truncating to the first character. +fn parse_like_escape_char(escape_char: &Option) -> Result> { + let Some(value) = escape_char else { + return Ok(None); + }; + let ValueWithSpan { + value: Value::SingleQuotedString(escape), + .. + } = value + else { + return Err(Error::invalid_input(format!( + "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {value}" + ))); + }; + let mut chars = escape.chars(); + match (chars.next(), chars.next()) { + (Some(c), None) => Ok(Some(c)), + _ => Err(Error::invalid_input(format!( + "Invalid escape character in LIKE expression. Expected a single character, got '{escape}'" + ))), + } +} + #[derive(Debug, Clone, Eq, PartialEq, Hash)] struct CastListF16Udf { signature: Signature, @@ -72,10 +99,6 @@ impl CastListF16Udf { } impl ScalarUDFImpl for CastListF16Udf { - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn name(&self) -> &str { "_cast_list_f16" } @@ -197,6 +220,13 @@ impl ContextProvider for LanceContextProvider { self.state.window_functions().get(name).cloned() } + fn get_higher_order_meta( + &self, + name: &str, + ) -> Option> { + self.state.higher_order_functions().get(name).cloned() + } + fn get_function_meta(&self, f: &str) -> Option> { match f { // TODO: cast should go thru CAST syntax instead of UDF @@ -227,6 +257,14 @@ impl ContextProvider for LanceContextProvider { self.state.window_functions().keys().cloned().collect() } + fn higher_order_function_names(&self) -> Vec { + self.state + .higher_order_functions() + .keys() + .cloned() + .collect() + } + fn get_expr_planners(&self) -> &[Arc] { &self.expr_planners } @@ -321,6 +359,8 @@ impl Planner { BinaryOperator::NotEq => Operator::NotEq, BinaryOperator::And => Operator::And, BinaryOperator::Or => Operator::Or, + BinaryOperator::PGBitwiseShiftLeft => Operator::BitwiseShiftLeft, + BinaryOperator::PGBitwiseShiftRight => Operator::BitwiseShiftRight, _ => { return Err(Error::invalid_input(format!( "Operator {op} is not supported" @@ -451,6 +491,8 @@ impl Planner { }; if let Ok(n) = value.parse::() { Ok(lit(n)) + } else if let Ok(n) = value.parse::() { + Ok(lit(n)) } else { value.parse::().map(lit).map_err(|_| { Error::invalid_input(format!("'{value}' is not supported number value.")) @@ -503,7 +545,7 @@ impl Planner { } _ => Err(Error::invalid_input(format!( "Unsupported function args: {:?}", - &func.args + func.args ))), } } @@ -747,10 +789,10 @@ impl Planner { data_type, value, .. }) => { let value = value.clone().into_string().expect_ok()?; - Ok(Expr::Cast(datafusion::logical_expr::Cast { - expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)), - data_type: self.parse_type(data_type)?, - })) + Ok(Expr::Cast(datafusion::logical_expr::Cast::new( + Box::new(Expr::Literal(ScalarValue::Utf8(Some(value)), None)), + self.parse_type(data_type)?, + ))) } SQLExpr::IsFalse(expr) => Ok(Expr::IsFalse(Box::new(self.parse_sql_expr(expr)?))), SQLExpr::IsNotFalse(expr) => Ok(Expr::IsNotFalse(Box::new(self.parse_sql_expr(expr)?))), @@ -782,16 +824,7 @@ impl Planner { *negated, Box::new(self.parse_sql_expr(expr)?), Box::new(self.parse_sql_expr(pattern)?), - match escape_char { - Some(Value::SingleQuotedString(char)) => char.chars().next(), - Some(value) => { - return Err(Error::invalid_input(format!( - "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}", - value - ))); - } - None => None, - }, + parse_like_escape_char(escape_char)?, true, ))), SQLExpr::Like { @@ -804,16 +837,7 @@ impl Planner { *negated, Box::new(self.parse_sql_expr(expr)?), Box::new(self.parse_sql_expr(pattern)?), - match escape_char { - Some(Value::SingleQuotedString(char)) => char.chars().next(), - Some(value) => { - return Err(Error::invalid_input(format!( - "Invalid escape character in LIKE expression. Expected a single character wrapped with single quotes, got {}", - value - ))); - } - None => None, - }, + parse_like_escape_char(escape_char)?, false, ))), // JSONB cast: CAST('...' AS JSONB) or '...'::jsonb @@ -838,15 +862,15 @@ impl Planner { } => match kind { datafusion::sql::sqlparser::ast::CastKind::TryCast | datafusion::sql::sqlparser::ast::CastKind::SafeCast => { - Ok(Expr::TryCast(datafusion::logical_expr::TryCast { - expr: Box::new(self.parse_sql_expr(expr)?), - data_type: self.parse_type(data_type)?, - })) + Ok(Expr::TryCast(datafusion::logical_expr::TryCast::new( + Box::new(self.parse_sql_expr(expr)?), + self.parse_type(data_type)?, + ))) } - _ => Ok(Expr::Cast(datafusion::logical_expr::Cast { - expr: Box::new(self.parse_sql_expr(expr)?), - data_type: self.parse_type(data_type)?, - })), + _ => Ok(Expr::Cast(datafusion::logical_expr::Cast::new( + Box::new(self.parse_sql_expr(expr)?), + self.parse_type(data_type)?, + ))), }, SQLExpr::JsonAccess { .. } => Err(Error::invalid_input("JSON access is not supported")), SQLExpr::CompoundFieldAccess { root, access_chain } => { @@ -989,17 +1013,72 @@ impl Planner { pub fn optimize_expr(&self, expr: Expr) -> Result { let df_schema = Arc::new(DFSchema::try_from(self.schema.as_ref().clone())?); + // DataFusion rewrites arrow_cast to Expr::Cast, whose Arrow kernel does not support + // integer-to-Time32 casts. Convert literal values with Lance's scalar coercion first. + let expr = expr + .transform_up(|expr| { + let coerced = match &expr { + Expr::ScalarFunction(ScalarFunction { func, args }) + if func.name() == "arrow_cast" => + { + match args.as_slice() { + [ + Expr::Literal(value, metadata), + Expr::Literal(ScalarValue::Utf8(Some(data_type)), _), + ] => data_type + .parse::() + .ok() + .filter(|data_type| matches!(data_type, ArrowDataType::Time32(_))) + .and_then(|data_type| { + if matches!(value, ScalarValue::Null) { + ScalarValue::try_new_null(&data_type).ok() + } else { + safe_coerce_scalar(value, &data_type) + } + }) + .map(|value| Expr::Literal(value, metadata.clone())), + _ => None, + } + } + _ => None, + }; + + Ok(match coerced { + Some(coerced) => Transformed::yes(coerced), + None => Transformed::no(expr), + }) + })? + .data; + // DataFusion needs the coerce and simplify passes to be applied before // expressions can be handled by the physical planner. - let simplify_context = SimplifyContext::default() + let simplify_context = SimplifyContext::builder() .with_schema(df_schema.clone()) - .with_query_execution_start_time(Some(Utc::now())); + .with_query_execution_start_time(Some(Utc::now())) + .build(); let simplifier = datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context); // Coerce before simplify to match DataFusion's analyzer-before-optimizer pipeline. let expr = simplifier.coerce(expr, &df_schema)?; + + // Fold each comparison's own operands and rewrite it before anything above + // it folds. `simplify` folds an operand and everything above it in one + // pass, so a fully constant predicate whose zero appears only as a result + // of folding never presents a zero literal to the rewrite: + // `-1.0 * 0.0 < (1.0 - 1.0)` answered `true` where IEEE says false, and a + // wrapper such as `IS TRUE` or a `CAST` did the same to the comparison's + // own result. + let expr = normalize_zero_comparisons(expr, &|operand| simplifier.simplify(operand))?; + + // Again after simplify, which is what expands `BETWEEN` into two + // comparisons and folds the casts `coerce` inserts, so those forms only + // become visible on this pass. + // + // Running the rewrite more than once is safe because its output is a fixed + // point of `optimize_expr`; `optimizing_twice_changes_nothing` pins that. let expr = simplifier.simplify(expr)?; + let expr = rewrite_signed_zero_comparisons(expr)?; Ok(expr) } @@ -1014,6 +1093,16 @@ impl Planner { )?) } + /// Create a [`PhysicalExpr`] using the caller's DataFusion session. + pub fn create_physical_expr_with_session( + &self, + expr: &Expr, + session: &dyn Session, + ) -> Result> { + let df_schema = DFSchema::try_from(self.schema.as_ref().clone())?; + Ok(session.create_physical_expr(expr.clone(), &df_schema)?) + } + /// Collect the columns in the expression. /// /// The columns are returned in sorted order. @@ -1062,13 +1151,9 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor { self.columns.insert(path); self.current_path.clear(); } - Expr::ScalarFunction(udf) => { - if udf.name() == GetFieldFunc::default().name() { - if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) { - self.current_path.push_front(name.to_string()) - } else { - self.current_path.clear(); - } + Expr::ScalarFunction(udf) if udf.name() == GetFieldFunc::default().name() => { + if let Some(name) = get_as_string_scalar_opt(&udf.args[1]) { + self.current_path.push_front(name.to_string()) } else { self.current_path.clear(); } @@ -1084,7 +1169,6 @@ impl TreeNodeVisitor<'_> for ColumnCapturingVisitor { #[cfg(test)] mod tests { - use std::any::Any; use crate::logical_expr::ExprExt; @@ -1093,8 +1177,8 @@ mod tests { use arrow::datatypes::Float64Type; use arrow_array::{ ArrayRef, BooleanArray, Float32Array, Int32Array, Int64Array, RecordBatch, StringArray, - StructArray, TimestampMicrosecondArray, TimestampMillisecondArray, - TimestampNanosecondArray, TimestampSecondArray, + StructArray, Time32SecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, + TimestampNanosecondArray, TimestampSecondArray, UInt64Array, }; use arrow_schema::{DataType, Fields, Schema}; use datafusion::{ @@ -1102,6 +1186,7 @@ mod tests { prelude::{array_element, get_field}, }; use datafusion_functions::core::expr_ext::FieldAccessor; + use rstest::rstest; #[test] fn test_parse_filter_simple() { @@ -1174,6 +1259,29 @@ mod tests { ); } + #[test] + fn test_parse_filter_uint64_literal_above_i64_max() { + let value = u64::MAX - 1; + let batch = arrow_array::record_batch!(("id", UInt64, [1, value])).unwrap(); + let planner = Planner::new(batch.schema()); + + let expr = planner.parse_filter(&format!("id = {value}")).unwrap(); + assert_eq!(expr, col("id").eq(lit(value))); + + let physical_expr = planner.create_physical_expr(&expr).unwrap(); + let predicates = physical_expr.evaluate(&batch).unwrap(); + assert_eq!( + predicates.into_array(0).unwrap().as_ref(), + &BooleanArray::from(vec![false, true]) + ); + + let expr = planner + .parse_expr("arrow_cast(NULL, 'Time32(Second)')") + .unwrap(); + let expr = planner.optimize_expr(expr).unwrap(); + assert_eq!(expr, Expr::Literal(ScalarValue::Time32Second(None), None)); + } + #[test] fn test_parse_deep_logical_filter() { let planner = Planner::new(Arc::new(Schema::empty())); @@ -1204,10 +1312,6 @@ mod tests { } impl ScalarUDFImpl for StrictFloat64Udf { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "strict_float64" } @@ -1375,6 +1479,40 @@ mod tests { ); } + #[rstest] + #[case::right("value >> 32", Operator::BitwiseShiftRight, vec![0, 1, 3])] + #[case::left( + "value << 1", + Operator::BitwiseShiftLeft, + vec![0, 2_u64 << 32, ((3_u64 << 32) + 7) << 1] + )] + fn test_bitwise_shift_expressions( + #[case] sql: &str, + #[case] expected_op: Operator, + #[case] expected: Vec, + ) { + let input = vec![0, 1_u64 << 32, (3_u64 << 32) + 7]; + let batch = + RecordBatch::try_from_iter([("value", Arc::new(UInt64Array::from(input)) as ArrayRef)]) + .unwrap(); + let planner = Planner::new(batch.schema()); + + let expr = planner.parse_expr(sql).unwrap(); + let Expr::BinaryExpr(binary_expr) = &expr else { + panic!("expected binary expression for {sql}, got {expr}"); + }; + assert_eq!(binary_expr.op, expected_op); + + let expr = planner.optimize_expr(expr).unwrap(); + let physical_expr = planner.create_physical_expr(&expr).unwrap(); + let values = physical_expr + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(values.as_ref(), &UInt64Array::from(expected)); + } + #[test] fn test_negative_array_expressions() { let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int64, false)])); @@ -1453,6 +1591,36 @@ mod tests { ); } + #[test] + fn test_like_escape_char() { + let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)])); + let planner = Planner::new(schema); + + // A valid single-character escape is captured for both LIKE and ILIKE. + for filter in ["s LIKE 'a!%' ESCAPE '!'", "s ILIKE 'a!%' ESCAPE '!'"] { + match planner.parse_filter(filter).unwrap() { + Expr::Like(like) => assert_eq!(like.escape_char, Some('!'), "{filter}"), + other => panic!("expected a LIKE expression for `{filter}`, got {other:?}"), + } + } + + // Empty and multi-character escapes are rejected rather than silently + // dropped or truncated to the first character. + for filter in [ + "s LIKE 'x' ESCAPE ''", + "s LIKE 'x' ESCAPE 'ab'", + "s ILIKE 'x' ESCAPE ''", + "s ILIKE 'x' ESCAPE 'ab'", + ] { + let err = planner.parse_filter(filter).unwrap_err(); + assert!( + err.to_string() + .contains("Invalid escape character in LIKE expression"), + "unexpected error for `{filter}`: {err}" + ); + } + } + #[test] fn test_sql_is_in() { let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, true)])); @@ -1609,7 +1777,7 @@ mod tests { match expr { Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() { - Expr::Cast(Cast { expr, data_type }) => { + Expr::Cast(Cast { expr, field }) => { match expr.as_ref() { Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => { assert_eq!(value_str, expected_value_str); @@ -1619,7 +1787,7 @@ mod tests { } _ => panic!("Expected cast to be applied to literal"), } - assert_eq!(data_type, expected_data_type); + assert_eq!(field.data_type(), expected_data_type); } _ => panic!("Expected right to be a cast"), }, @@ -1628,6 +1796,28 @@ mod tests { } } + #[test] + fn test_arrow_cast_int_literal_to_time32() { + let batch = RecordBatch::try_from_iter([( + "v", + Arc::new(Time32SecondArray::from(vec![3725, 3726])) as ArrayRef, + )]) + .unwrap(); + let planner = Planner::new(batch.schema()); + + let expr = planner + .parse_filter("v = arrow_cast(3726, 'Time32(Second)')") + .unwrap(); + let expr = planner.optimize_expr(expr).unwrap(); + let physical_expr = planner.create_physical_expr(&expr).unwrap(); + let predicates = physical_expr.evaluate(&batch).unwrap(); + + assert_eq!( + predicates.into_array(0).unwrap().as_ref(), + &BooleanArray::from(vec![false, true]) + ); + } + #[test] fn test_sql_literals() { let cases = &[ @@ -1660,14 +1850,14 @@ mod tests { match expr { Expr::BinaryExpr(BinaryExpr { right, .. }) => match right.as_ref() { - Expr::Cast(Cast { expr, data_type }) => { + Expr::Cast(Cast { expr, field }) => { match expr.as_ref() { Expr::Literal(ScalarValue::Utf8(Some(value_str)), _) => { assert_eq!(value_str, expected_value_str); } _ => panic!("Expected cast to be applied to literal"), } - assert_eq!(data_type, expected_data_type); + assert_eq!(field.data_type(), expected_data_type); } _ => panic!("Expected right to be a cast"), }, diff --git a/rust/lance-datafusion/src/projection.rs b/rust/lance-datafusion/src/projection.rs index 8a15b321e49..463e83a3011 100644 --- a/rust/lance-datafusion/src/projection.rs +++ b/rust/lance-datafusion/src/projection.rs @@ -24,11 +24,20 @@ use crate::{ planner::Planner, }; +const SCORING_COLUMNS: [&str; 2] = ["_distance", "_score"]; + +fn canonical_scoring_column(name: &str) -> Option<&'static str> { + SCORING_COLUMNS + .into_iter() + .find(|scoring_column| name.eq_ignore_ascii_case(scoring_column)) +} + struct ProjectionBuilder { base: Arc, planner: Planner, output: HashMap, output_cols: Vec, + scoring_exprs: HashMap, physical_cols_set: HashSet, physical_cols: Vec, needs_row_id: bool, @@ -50,6 +59,7 @@ impl ProjectionBuilder { planner, output: HashMap::default(), output_cols: Vec::default(), + scoring_exprs: HashMap::default(), physical_cols_set: HashSet::default(), physical_cols: Vec::default(), needs_row_id: false, @@ -75,10 +85,18 @@ impl ProjectionBuilder { self.check_duplicate_column(output_name)?; let expr = self.planner.parse_expr(raw_expr)?; - // Run simplification + coercion so that expressions like `coalesce(...)` - // (which DataFusion's physical evaluator expects to have been rewritten - // into a `CASE` expression by the simplifier) work correctly. - let expr = self.planner.optimize_expr(expr)?; + let expr = if Self::references_scoring_column(&expr) { + // A scoring name can refer to either a stored column or a search-generated + // Float32 column. Reparse and coerce once the physical input schema disambiguates it. + self.scoring_exprs + .insert(output_name.to_string(), raw_expr.to_string()); + expr + } else { + // Run simplification + coercion so that expressions like `coalesce(...)` + // (which DataFusion's physical evaluator expects to have been rewritten + // into a `CASE` expression by the simplifier) work correctly. + self.planner.optimize_expr(expr)? + }; // If the expression is a bare column reference to a system column, mark that we need it if let Expr::Column(Column { @@ -101,11 +119,23 @@ impl ProjectionBuilder { } for col in Planner::column_names_in_expr(&expr) { - if self.physical_cols_set.contains(&col) { + // Discovery can bind an exact provisional scoring field beside a mixed-case stored + // field. Load the stored field too so final-schema replanning can select the stored + // or search-generated field from the physical input. + let physical_col = if canonical_scoring_column(&col).is_some() { + self.base + .schema() + .field_case_insensitive(&col) + .map(|field| field.name.clone()) + .unwrap_or(col) + } else { + col + }; + if self.physical_cols_set.contains(&physical_col) { continue; } - self.physical_cols.push(col.clone()); - self.physical_cols_set.insert(col); + self.physical_cols.push(physical_col.clone()); + self.physical_cols_set.insert(physical_col); } self.output.insert(output_name.to_string(), expr.clone()); @@ -117,6 +147,12 @@ impl ProjectionBuilder { Ok(()) } + fn references_scoring_column(expr: &Expr) -> bool { + Planner::column_names_in_expr(expr) + .iter() + .any(|name| canonical_scoring_column(name).is_some()) + } + fn add_columns(&mut self, columns: &[(impl AsRef, impl AsRef)]) -> Result<()> { for (output_name, raw_expr) in columns { if raw_expr.as_ref() == WILDCARD { @@ -159,6 +195,7 @@ impl ProjectionBuilder { physical_projection, must_add_row_offset: self.must_add_row_offset, requested_output_expr: self.output_cols, + scoring_exprs: self.scoring_exprs, }) } } @@ -181,6 +218,9 @@ pub struct ProjectionPlan { /// The desired output columns pub requested_output_expr: Vec, + + /// Original SQL for scoring expressions that must be replanned against the physical schema. + scoring_exprs: HashMap, } impl ProjectionPlan { @@ -199,6 +239,14 @@ impl ProjectionPlan { fields.push(Arc::new( (*lance_core::ROW_CREATED_AT_VERSION_FIELD).clone(), )); + // Exact scoring fields are needed for initial parsing of schema-dependent functions, even + // beside a mixed-case stored field. The stored field is carried into the physical + // projection separately, and scoring expressions are replanned against the final schema. + for name in SCORING_COLUMNS { + if schema.field_with_name(name).is_err() { + fields.push(Arc::new(ArrowField::new(name, DataType::Float32, true))); + } + } ArrowSchema::new(fields) } @@ -312,6 +360,7 @@ impl ProjectionPlan { physical_projection, requested_output_expr: exprs, must_add_row_offset, + scoring_exprs: HashMap::default(), }) } @@ -338,6 +387,7 @@ impl ProjectionPlan { physical_projection, must_add_row_offset: false, requested_output_expr, + scoring_exprs: HashMap::default(), }) } @@ -352,9 +402,16 @@ impl ProjectionPlan { self.requested_output_expr .iter() .map(|output_column| { + let expr = if let Some(raw_expr) = self.scoring_exprs.get(&output_column.name) { + let planner = Planner::new(Arc::new(current_schema.clone())); + let expr = planner.parse_expr(raw_expr)?; + planner.optimize_expr(expr)? + } else { + output_column.expr.clone() + }; Ok(( datafusion::physical_expr::create_physical_expr( - &output_column.expr, + &expr, physical_df_schema.as_ref(), &Default::default(), )?, @@ -459,9 +516,166 @@ impl ProjectionPlan { mod tests { use super::*; - use arrow_array::Int64Array; + use arrow_array::{ArrayRef, Float32Array, Int64Array}; use lance_arrow::json::{is_json_field, json_field}; + #[test] + fn test_scoring_column_expression() { + for scoring_column in ["_distance", "_score"] { + for has_stored_column in [false, true] { + let base = if has_stored_column { + Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + scoring_column, + DataType::Float64, + true, + )])) + .unwrap(), + ) + } else { + Arc::new(Schema::default()) + }; + let expression = format!("1 - {scoring_column}"); + let plan = + ProjectionPlan::from_expressions(base, &[("inverted", expression.as_str())]) + .unwrap(); + + if has_stored_column { + let stored_output = plan.output_schema().unwrap(); + assert_eq!(stored_output.field(0).data_type(), &DataType::Float64); + } + + let batch = RecordBatch::try_from_iter([( + scoring_column, + Arc::new(Float32Array::from(vec![0.25, 0.75])) as ArrayRef, + )]) + .unwrap(); + + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + let values = physical_exprs[0] + .0 + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + + assert_eq!( + values.as_ref(), + &Float32Array::from(vec![0.75, 0.25]), + "unexpected result for {scoring_column}", + ); + } + } + } + + #[test] + fn test_stored_scoring_column_does_not_break_other_expressions() { + for scoring_column in ["_distance", "_score"] { + let base = Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int64, false), + ArrowField::new(scoring_column, DataType::Float64, true), + ])) + .unwrap(), + ); + + ProjectionPlan::from_expressions(base, &[("incremented", "id + 1")]).unwrap(); + } + } + + #[test] + fn test_stored_scoring_column_is_case_insensitive() { + for (stored_name, requested_name) in [("_Distance", "_distance"), ("_Score", "_score")] { + let base = Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + stored_name, + DataType::Float64, + true, + )])) + .unwrap(), + ); + let plan = + ProjectionPlan::from_expressions(base, &[("stored", requested_name)]).unwrap(); + + assert_eq!( + plan.output_schema().unwrap().field(0).data_type(), + &DataType::Float64, + ); + + let batch = RecordBatch::try_from_iter([( + requested_name, + Arc::new(Float32Array::from(vec![0.25, 0.75])) as ArrayRef, + )]) + .unwrap(); + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + assert_eq!( + physical_exprs[0] + .0 + .data_type(batch.schema().as_ref()) + .unwrap(), + DataType::Float32, + ); + } + } + + #[test] + fn test_generated_scoring_function_with_mixed_case_stored_column() { + for (stored_name, generated_name) in [("_Distance", "_distance"), ("_Score", "_score")] { + let base = Arc::new( + Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + stored_name, + DataType::Float64, + true, + )])) + .unwrap(), + ); + let expression = format!("coalesce(1 - {generated_name}, 0)"); + let plan = + ProjectionPlan::from_expressions(base, &[("normalized", expression.as_str())]) + .unwrap(); + let batch = RecordBatch::try_from_iter([( + generated_name, + Arc::new(Float32Array::from(vec![Some(0.25), None])) as ArrayRef, + )]) + .unwrap(); + + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + let values = physical_exprs[0] + .0 + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(values.as_ref(), &Float32Array::from(vec![0.75, 0.0])); + } + } + + #[test] + fn test_scoring_column_function_expression() { + for scoring_column in ["_distance", "_score"] { + let expression = format!("coalesce(1 - {scoring_column}, 0)"); + let plan = ProjectionPlan::from_expressions( + Arc::new(Schema::default()), + &[("normalized", expression.as_str())], + ) + .unwrap(); + let batch = RecordBatch::try_from_iter([( + scoring_column, + Arc::new(Float32Array::from(vec![Some(0.25), None])) as ArrayRef, + )]) + .unwrap(); + + let physical_exprs = plan.to_physical_exprs(batch.schema().as_ref()).unwrap(); + let values = physical_exprs[0] + .0 + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!(values.as_ref(), &Float32Array::from(vec![0.75, 0.0])); + } + } + #[tokio::test] async fn test_coalesce_in_column_map() { // Regression test: `coalesce` in a column-map expression used to fail with diff --git a/rust/lance-datafusion/src/signed_zero.rs b/rust/lance-datafusion/src/signed_zero.rs new file mode 100644 index 00000000000..1b642bc2a28 --- /dev/null +++ b/rust/lance-datafusion/src/signed_zero.rs @@ -0,0 +1,711 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Rewrites of comparisons against a floating point zero literal. + +use datafusion::error::Result as DFResult; +use datafusion::logical_expr::{BinaryExpr, Operator, expr::Between, expr::InList}; +use datafusion::prelude::Expr; +use datafusion::scalar::ScalarValue::{self, Float16, Float32, Float64}; +use datafusion_common::tree_node::{Transformed, TreeNode}; +use half::f16; +use lance_core::Result; + +/// Rewrite every comparison against a floating point zero literal into the form +/// that Arrow's total-order kernels answer the way IEEE 754 and SQL define it. +/// +/// Arrow sorts `-0.0` strictly below `+0.0` and compares the two encodings for +/// equality by bit pattern, while IEEE 754 and SQL treat them as one number. +/// Each comparison against a zero literal has an equivalent total-order form: +/// +/// | written | evaluated | +/// |------------------------------|------------------------------| +/// | `x < 0`, `x >= 0` | literal becomes `-0.0` | +/// | `x <= 0`, `x > 0` | literal becomes `+0.0` | +/// | `x = 0` | `x IN (-0.0, 0.0)` | +/// | `x != 0` | `x NOT IN (-0.0, 0.0)` | +/// | `x IN (0, ..)` | the missing encoding is added | +/// | `0 IN (a, b)` | `a IN (-0.0, 0.0) OR b IN (-0.0, 0.0)` | +/// | `x IS NOT DISTINCT FROM 0` | `x IS NOT NULL AND x IN (-0.0, 0.0)` | +/// | `x IS DISTINCT FROM 0` | `x IS NULL OR x NOT IN (-0.0, 0.0)` | +/// +/// Equality has to name both encodings because a scalar index keys on the bit +/// pattern: the btree and bitmap indices order candidates by `total_cmp`, and the +/// bloom filter hashes the value. +/// +/// Runs as the last step of [`crate::planner::Planner::optimize_expr`], after +/// coercion has given the literal the column's type and the simplifier has +/// expanded `BETWEEN` into two comparisons. Filters, computed output columns and +/// update expressions all compile through there, which is what keeps a filter and +/// a projected copy of the same predicate in agreement. +/// +/// NaN is out of scope. Arrow sorts it above every other value, so `x >= -0.0` +/// admits NaN where IEEE would not, and that holds for every comparison rather +/// than only the ones against zero. +pub fn rewrite_signed_zero_comparisons(expr: Expr) -> Result { + Ok(expr + .transform_up(|node| { + Ok(match rewrite_node(&node) { + Some(rewritten) => Transformed::yes(rewritten), + None => Transformed::no(node), + }) + })? + .data) +} + +/// Whether the rewrite acts on comparisons under `op`. +fn is_zero_sensitive(op: Operator) -> bool { + matches!( + op, + Operator::Lt + | Operator::LtEq + | Operator::Gt + | Operator::GtEq + | Operator::Eq + | Operator::NotEq + | Operator::IsDistinctFrom + | Operator::IsNotDistinctFrom + ) +} + +/// Fold each zero-sensitive comparison's own operands and rewrite it, bottom-up, +/// before anything above it has a chance to fold. +/// +/// [`rewrite_signed_zero_comparisons`] alone cannot reach a comparison whose zero +/// does not exist yet. `ExprSimplifier::simplify` folds an operand and everything +/// above it in one pass, so `-1.0 * 0.0 < (1.0 - 1.0)` goes straight to a boolean +/// decided by Arrow's total order, and a wrapper like `IS TRUE` or a `CAST` around +/// it does the same to the comparison's own result. +/// +/// Visiting bottom-up and folding only the operands of the node in hand is what +/// closes that: by the time any container is folded, every comparison inside it +/// already carries the corrected literal. This deliberately does not enumerate +/// which containers are allowed above a comparison. Enumerating them is what left +/// `IS TRUE`, `IS FALSE`, `= TRUE`, `CAST(.. AS BOOLEAN)` and `IN (TRUE)` exposed, +/// and any list would keep missing the next spelling. +pub fn normalize_zero_comparisons( + expr: Expr, + simplify: &dyn Fn(Expr) -> DFResult, +) -> Result { + // A literal is already folded, and an `IN` list can hold hundreds of them. + // Handing each one to the simplifier anyway roughly doubled planning time on + // large lists, for an operand that cannot change. + let fold = |operand: Expr| -> DFResult { + if matches!(operand, Expr::Literal(..)) { + return Ok(operand); + } + simplify(operand) + }; + Ok(expr + .transform_up(|node| { + let folded = match node { + Expr::BinaryExpr(BinaryExpr { left, op, right }) if is_zero_sensitive(op) => { + Expr::BinaryExpr(BinaryExpr { + left: Box::new(fold(*left)?), + op, + right: Box::new(fold(*right)?), + }) + } + Expr::Between(between) => Expr::Between(Between { + expr: Box::new(fold(*between.expr)?), + negated: between.negated, + low: Box::new(fold(*between.low)?), + high: Box::new(fold(*between.high)?), + }), + Expr::InList(in_list) => Expr::InList(InList { + expr: Box::new(fold(*in_list.expr)?), + list: in_list + .list + .into_iter() + .map(fold) + .collect::>>()?, + negated: in_list.negated, + }), + other => return Ok(Transformed::no(other)), + }; + Ok(match rewrite_node(&folded) { + Some(rewritten) => Transformed::yes(rewritten), + // The operands were still folded, so this is a change either way. + None => Transformed::yes(folded), + }) + })? + .data) +} + +/// Both encodings of a floating point zero, negative first. +/// +/// Returns `None` for anything else, including NULL, NaN, and integer zero. +fn zero_encodings(value: &ScalarValue) -> Option<(ScalarValue, ScalarValue)> { + match value { + Float16(Some(v)) if *v == f16::ZERO => { + Some((Float16(Some(f16::NEG_ZERO)), Float16(Some(f16::ZERO)))) + } + Float32(Some(v)) if *v == 0.0 => Some((Float32(Some(-0.0)), Float32(Some(0.0)))), + Float64(Some(v)) if *v == 0.0 => Some((Float64(Some(-0.0)), Float64(Some(0.0)))), + _ => None, + } +} + +/// Collect the terms of an `AND`/`OR` chain, in order, ignoring nesting. +fn flatten_chain<'a>(expr: &'a Expr, op: Operator, terms: &mut Vec<&'a Expr>) { + if let Expr::BinaryExpr(BinaryExpr { + left, + op: inner, + right, + }) = expr + && *inner == op + { + flatten_chain(left, op, terms); + flatten_chain(right, op, terms); + return; + } + terms.push(expr); +} + +/// True for the shape this rewrite emits for `=` and `!=`: a column tested +/// against both encodings of a floating point zero, negated or not. Only these +/// terms are deduplicated, so an expression the caller wrote twice is left alone. +fn is_zero_pair_over_column(expr: &Expr) -> bool { + let Expr::InList(InList { expr, list, .. }) = expr else { + return false; + }; + if !matches!(expr.as_ref(), Expr::Column(_)) { + return false; + } + let [Expr::Literal(first, _), ..] = list.as_slice() else { + return false; + }; + zero_encodings(first) + .is_some_and(|(negative, positive)| list_is_pair(list, &negative, &positive)) +} + +/// True when `list` is exactly the two encodings of a zero, negative first. +fn list_is_pair(list: &[Expr], negative: &ScalarValue, positive: &ScalarValue) -> bool { + let [Expr::Literal(first, _), Expr::Literal(second, _)] = list else { + return false; + }; + first == negative && second == positive +} + +/// The rewritten expression, or `None` when `expr` is not a comparison against a +/// floating point zero. +/// The encoding a zero bound needs to answer `op` correctly, or `None` when the +/// expression is not a zero literal. +fn rewrite_bound(bound: &Expr, op: Operator) -> Option { + let Expr::Literal(value, metadata) = bound else { + return None; + }; + let (negative, positive) = zero_encodings(value)?; + let encoding = match op { + Operator::GtEq => negative, + Operator::LtEq => positive, + _ => return None, + }; + Some(Expr::Literal(encoding, metadata.clone())) +} + +fn rewrite_node(expr: &Expr) -> Option { + match expr { + // DataFusion's simplifier expands an `IN` list of three or fewer values + // over a bare column back into an OR chain of equalities, so a second + // `optimize_expr` splits this rewrite's own output and re-runs it on each + // half. Both halves then produce the same list, and dropping the repeat is + // what makes the rewrite survive that round trip. + Expr::BinaryExpr(BinaryExpr { op, .. }) if matches!(op, Operator::Or | Operator::And) => { + let mut kept: Vec<&Expr> = Vec::new(); + flatten_chain(expr, *op, &mut kept); + let mut deduped: Vec<&Expr> = Vec::with_capacity(kept.len()); + for term in kept.iter() { + if is_zero_pair_over_column(term) && deduped.contains(term) { + continue; + } + deduped.push(term); + } + if deduped.len() == kept.len() { + return None; + } + deduped.into_iter().cloned().reduce(|left, right| match op { + Operator::Or => left.or(right), + _ => left.and(right), + }) + } + Expr::BinaryExpr(BinaryExpr { left, op, right }) => { + // `resolve_expr` accepts the literal on either side, and the + // operator mirrors when it sits on the left. + let (literal, other, op) = match (left.as_ref(), right.as_ref()) { + (_, Expr::Literal(..)) => (right.as_ref(), left.as_ref(), *op), + (Expr::Literal(..), _) => (left.as_ref(), right.as_ref(), op.swap()?), + _ => return None, + }; + let Expr::Literal(value, metadata) = literal else { + return None; + }; + let (negative, positive) = zero_encodings(value)?; + let zero = match op { + Operator::Lt | Operator::GtEq => negative, + Operator::LtEq | Operator::Gt => positive, + Operator::Eq | Operator::NotEq => { + return Some(Expr::InList(InList { + expr: Box::new(other.clone()), + list: vec![ + Expr::Literal(negative, metadata.clone()), + Expr::Literal(positive, metadata.clone()), + ], + negated: op == Operator::NotEq, + })); + } + Operator::IsNotDistinctFrom | Operator::IsDistinctFrom => { + // Both encodings have to be listed, and the null case has to + // stay decided rather than becoming NULL, so this pairs the + // list with `IS [NOT] TRUE`. `NULL IN (..)` is NULL, and + // `NULL IS TRUE` is false, which is what distinctness means + // for a null against a non-null literal. + // + // The list carries `other` once. An earlier version guarded + // this arm to a bare column so it could name `other` twice as + // `IS NOT NULL AND IN (..)`, but bailing out left the + // `filter_expr` path answering computed operands on Arrow's + // sign-sensitive order, which is wrong rows rather than an + // unsupported spelling. + // + // Always the non-negated list, so a second pass sees the same + // complete pair it would leave alone anywhere else. + let covered = Expr::InList(InList { + expr: Box::new(other.clone()), + list: vec![ + Expr::Literal(negative, metadata.clone()), + Expr::Literal(positive, metadata.clone()), + ], + negated: false, + }); + return Some(if op == Operator::IsDistinctFrom { + covered.is_not_true() + } else { + covered.is_true() + }); + } + _ => return None, + }; + Some(Expr::BinaryExpr(BinaryExpr { + left: Box::new(other.clone()), + op, + right: Box::new(Expr::Literal(zero, metadata.clone())), + })) + } + // `BETWEEN` normally reaches this rewrite already expanded into `>=` and + // `<=` by the simplifier. It survives unexpanded when every operand is + // constant, because then the simplifier expands and folds it in one pass + // and the comparison is gone before the post-pass looks. The bounds take + // the encodings their expanded operators would: `low` is a `>=` bound and + // `high` is a `<=` bound. + Expr::Between(between) => { + let low = rewrite_bound(&between.low, Operator::GtEq); + let high = rewrite_bound(&between.high, Operator::LtEq); + if low.is_none() && high.is_none() { + return None; + } + Some(Expr::Between(Between { + expr: between.expr.clone(), + negated: between.negated, + low: Box::new(low.unwrap_or_else(|| (*between.low).clone())), + high: Box::new(high.unwrap_or_else(|| (*between.high).clone())), + })) + } + Expr::InList(InList { + expr, + list, + negated, + }) => { + // A zero literal on the probe side needs the same treatment. The list + // elements are arbitrary expressions there, so expand into the + // equality form the binary arm already covers. A literal probe that is + // not a zero compares the same way against either encoding, so it + // needs no widening either. + if let Expr::Literal(value, metadata) = expr.as_ref() { + let (negative, positive) = zero_encodings(value)?; + // The expansion below puts a zero literal in front of exactly this + // list, so stop rather than expanding that term again. + if list_is_pair(list, &negative, &positive) { + return None; + } + let matches_any = list + .iter() + .map(|item| { + Expr::InList(InList { + expr: Box::new(item.clone()), + list: vec![ + Expr::Literal(negative.clone(), metadata.clone()), + Expr::Literal(positive.clone(), metadata.clone()), + ], + negated: false, + }) + }) + .reduce(Expr::or)?; + return Some(if *negated { + Expr::Not(Box::new(matches_any)) + } else { + matches_any + }); + } + Some(Expr::InList(InList { + expr: expr.clone(), + list: widen_zero_list(list)?, + negated: *negated, + })) + } + _ => None, + } +} + +/// Add the missing encoding next to every floating point zero in an `IN` list. +/// +/// Returns `None` when the list holds no zero, or already spells out both +/// encodings of each zero it holds. +fn widen_zero_list(list: &[Expr]) -> Option> { + // Most lists hold no zero, so collect what is missing before copying anything. + let mut missing: Vec = Vec::new(); + for item in list { + let Expr::Literal(value, metadata) = item else { + continue; + }; + let Some((negative, positive)) = zero_encodings(value) else { + continue; + }; + let counterpart = if *value == negative { + positive + } else { + negative + }; + // `ScalarValue` compares floats by bit pattern, so this distinguishes + // the two encodings rather than collapsing them. + let is_counterpart = + |other: &Expr| matches!(other, Expr::Literal(v, _) if *v == counterpart); + if list.iter().any(is_counterpart) || missing.iter().any(is_counterpart) { + continue; + } + missing.push(Expr::Literal(counterpart, metadata.clone())); + } + if missing.is_empty() { + return None; + } + let mut widened = Vec::with_capacity(list.len() + missing.len()); + widened.extend(list.iter().cloned()); + widened.append(&mut missing); + Some(widened) +} + +#[cfg(test)] +mod tests { + use datafusion::prelude::{col, lit}; + use rstest::rstest; + + use super::*; + + fn rewrite(expr: Expr) -> Expr { + rewrite_signed_zero_comparisons(expr).unwrap() + } + + fn compare(left: Expr, op: Operator, right: Expr) -> Expr { + Expr::BinaryExpr(BinaryExpr { + left: Box::new(left), + op, + right: Box::new(right), + }) + } + + #[rstest] + #[case::lt_from_positive(Operator::Lt, 0.0, -0.0)] + #[case::lt_from_negative(Operator::Lt, -0.0, -0.0)] + #[case::lt_eq_from_positive(Operator::LtEq, 0.0, 0.0)] + #[case::lt_eq_from_negative(Operator::LtEq, -0.0, 0.0)] + #[case::gt_from_positive(Operator::Gt, 0.0, 0.0)] + #[case::gt_from_negative(Operator::Gt, -0.0, 0.0)] + #[case::gt_eq_from_positive(Operator::GtEq, 0.0, -0.0)] + #[case::gt_eq_from_negative(Operator::GtEq, -0.0, -0.0)] + fn range_comparison_uses_the_encoding_for_the_operator( + #[case] op: Operator, + #[case] written: f64, + #[case] evaluated: f64, + ) { + assert_eq!( + rewrite(compare(col("x"), op, lit(written))), + compare(col("x"), op, lit(evaluated)) + ); + } + + #[rstest] + #[case::eq(Operator::Eq, false)] + #[case::not_eq(Operator::NotEq, true)] + fn equality_covers_both_encodings(#[case] op: Operator, #[case] negated: bool) { + assert_eq!( + rewrite(compare(col("x"), op, lit(0.0))), + Expr::InList(InList { + expr: Box::new(col("x")), + list: vec![lit(-0.0), lit(0.0)], + negated, + }) + ); + } + + #[test] + fn a_literal_on_the_left_mirrors_the_operator() { + // `0.0 > x` is `x < 0.0`, which evaluates against the negative encoding. + assert_eq!( + rewrite(compare(lit(0.0), Operator::Gt, col("x"))), + compare(col("x"), Operator::Lt, lit(-0.0)) + ); + } + + #[rstest] + #[case::float32(Float32(Some(-0.0)), Float32(Some(0.0)))] + #[case::float16(Float16(Some(f16::NEG_ZERO)), Float16(Some(f16::ZERO)))] + fn narrow_floats_are_rewritten_too( + #[case] written: ScalarValue, + #[case] evaluated: ScalarValue, + ) { + assert_eq!( + rewrite(compare( + col("x"), + Operator::LtEq, + Expr::Literal(written, None) + )), + compare(col("x"), Operator::LtEq, Expr::Literal(evaluated, None)) + ); + } + + #[test] + fn an_in_list_gains_the_missing_encoding() { + assert_eq!( + rewrite(Expr::InList(InList { + expr: Box::new(col("x")), + list: vec![lit(0.0), lit(5.0)], + negated: true, + })), + Expr::InList(InList { + expr: Box::new(col("x")), + list: vec![lit(0.0), lit(5.0), lit(-0.0)], + negated: true, + }) + ); + } + + #[test] + fn only_the_zero_comparison_in_a_conjunction_changes() { + assert_eq!( + rewrite(col("x").lt(lit(0.0)).and(col("y").eq(lit(1.0)))), + col("x").lt(lit(-0.0)).and(col("y").eq(lit(1.0))) + ); + } + + #[rstest] + #[case::non_zero(col("x").lt(lit(1.0)))] + #[case::integer_zero(col("x").eq(lit(0_i64)))] + #[case::null(compare(col("x"), Operator::Eq, Expr::Literal(Float64(None), None)))] + #[case::nan(col("x").lt(lit(f64::NAN)))] + #[case::column_on_both_sides(col("x").lt(col("y")))] + #[case::both_encodings_listed(Expr::InList(InList { + expr: Box::new(col("x")), + list: vec![lit(-0.0), lit(0.0)], + negated: false, + }))] + fn unrelated_comparisons_are_left_alone(#[case] expr: Expr) { + assert_eq!(rewrite(expr.clone()), expr); + } + + /// Distinctness has to stay decided for a null operand, and it has to name + /// the operand once so a computed one is not evaluated twice. + #[rstest] + #[case::is_not_distinct_from(Operator::IsNotDistinctFrom)] + #[case::is_distinct_from(Operator::IsDistinctFrom)] + fn distinct_from_lowers_through_a_null_defaulted_list(#[case] op: Operator) { + let covered = Expr::InList(InList { + expr: Box::new(col("x")), + list: vec![lit(-0.0), lit(0.0)], + negated: false, + }); + let expected = if op == Operator::IsDistinctFrom { + covered.is_not_true() + } else { + covered.is_true() + }; + assert_eq!(rewrite(compare(col("x"), op, lit(0.0))), expected); + } + + /// The operand does not have to be a column. Bailing out on anything else + /// used to leave `filter_expr` answering computed operands on Arrow's + /// sign-sensitive order, which returns wrong rows. + #[rstest] + #[case::is_not_distinct_from(Operator::IsNotDistinctFrom)] + #[case::is_distinct_from(Operator::IsDistinctFrom)] + fn distinct_from_rewrites_a_computed_operand(#[case] op: Operator) { + let computed = col("x") * lit(2.0); + let covered = Expr::InList(InList { + expr: Box::new(computed.clone()), + list: vec![lit(-0.0), lit(0.0)], + negated: false, + }); + let expected = if op == Operator::IsDistinctFrom { + covered.is_not_true() + } else { + covered.is_true() + }; + assert_eq!(rewrite(compare(computed, op, lit(0.0))), expected); + } + + /// Several paths optimize the same expression more than once, so every shape + /// the rewrite emits has to be a fixed point. + #[rstest] + #[case::lt(col("x").lt(lit(0.0)))] + #[case::gt_eq(col("x").gt_eq(lit(0.0)))] + #[case::eq(col("x").eq(lit(0.0)))] + #[case::not_eq(col("x").not_eq(lit(0.0)))] + #[case::in_list(Expr::InList(InList { + expr: Box::new(col("x")), + list: vec![lit(0.0), lit(5.0)], + negated: false, + }))] + #[case::zero_probe(Expr::InList(InList { + expr: Box::new(lit(0.0)), + list: vec![col("a"), col("b")], + negated: false, + }))] + #[case::zero_probe_over_literals(Expr::InList(InList { + expr: Box::new(lit(0.0)), + list: vec![col("a"), lit(0.0)], + negated: false, + }))] + #[case::is_not_distinct_from(compare(col("x"), Operator::IsNotDistinctFrom, lit(0.0)))] + #[case::is_distinct_from(compare(col("x"), Operator::IsDistinctFrom, lit(0.0)))] + fn rewriting_twice_changes_nothing(#[case] expr: Expr) { + let once = rewrite(expr); + assert_eq!(rewrite(once.clone()), once); + } + + #[rstest] + #[case::probe(false)] + #[case::negated_probe(true)] + fn a_zero_probe_expands_into_equalities(#[case] negated: bool) { + let covers = |column| { + Expr::InList(InList { + expr: Box::new(col(column)), + list: vec![lit(-0.0), lit(0.0)], + negated: false, + }) + }; + let matches_any = covers("a").or(covers("b")); + assert_eq!( + rewrite(Expr::InList(InList { + expr: Box::new(lit(0.0)), + list: vec![col("a"), col("b")], + negated, + })), + if negated { + Expr::Not(Box::new(matches_any)) + } else { + matches_any + } + ); + } + + #[test] + fn scalar_value_keeps_the_two_zero_encodings_apart() { + // The `IN` list widening decides "already listed" with this comparison. A + // DataFusion release that made the two encodings equal would silently stop + // it. + assert_ne!(Float64(Some(-0.0)), Float64(Some(0.0))); + assert_ne!(Float32(Some(-0.0)), Float32(Some(0.0))); + assert_ne!(Float16(Some(f16::NEG_ZERO)), Float16(Some(f16::ZERO))); + } + + /// The scan path optimizes the same expression twice, and the simplifier + /// expands a short `IN` list over a column back into an OR chain in between, + /// so a fixed point of the rewrite alone would not be enough. + #[rstest] + #[case::eq("value = 0.0")] + #[case::not_eq("value != 0.0")] + #[case::in_list("value IN (0.0, 1.0)")] + #[case::lt("value < 0.0")] + #[case::gt_eq("value >= 0.0")] + #[case::between("value BETWEEN -0.0 AND 0.0")] + // The dedup that makes the first three cases hold keys on the probe being a + // bare column, which is also what DataFusion requires before it shortens a + // list. This case fails if a release ever relaxes that. + #[case::non_column_probe("abs(value) = 0.0")] + // `IS [NOT] DISTINCT FROM` is missing because `Planner::parse_filter` rejects + // it as unsupported SQL; that arm is reachable only from a programmatically + // built expression, and `rewriting_twice_changes_nothing` covers it there. + fn optimizing_twice_changes_nothing(#[case] filter: &str) { + let schema = + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "value", + arrow_schema::DataType::Float64, + true, + )])); + let planner = crate::planner::Planner::new(schema); + let once = planner + .optimize_expr(planner.parse_filter(filter).unwrap()) + .unwrap(); + assert_eq!(planner.optimize_expr(once.clone()).unwrap(), once); + } + + /// A comparison whose operands are all constant never reaches the rewrite if + /// the rewrite only runs after `simplify`: the simplifier folds it to a bare + /// boolean under Arrow's total order first, and there is nothing left to + /// repair. These fold to the IEEE answer only because the rewrite also runs + /// before `simplify`. + #[rstest] + #[case::lt("-1.0 * 0.0 < 0.0", false)] + #[case::eq("(-1.0 * 0.0) = 0.0", true)] + #[case::gt_eq("(-1.0 * 0.0) >= 0.0", true)] + #[case::not_eq("(-1.0 * 0.0) != 0.0", false)] + #[case::gt("(-1.0 * 0.0) > 0.0", false)] + #[case::lt_eq("(-1.0 * 0.0) <= 0.0", true)] + // The zero on the right is produced by folding rather than written, so these + // reach the rewrite only because the operands are folded before the + // comparison is. + #[case::folded_rhs_lt("-1.0 * 0.0 < (1.0 - 1.0)", false)] + #[case::folded_rhs_eq("(-1.0 * 0.0) = (1.0 - 1.0)", true)] + #[case::folded_rhs_gt_eq("(-1.0 * 0.0) >= (1.0 - 1.0)", true)] + #[case::folded_rhs_not_eq("(-1.0 * 0.0) != (1.0 - 1.0)", false)] + #[case::both_sides_folded("(0.0 * -1.0) < (1.0 - 1.0)", false)] + // `BETWEEN` and `IN` fold the same way, and a fully constant `BETWEEN` never + // reaches the rewrite already expanded, which is why the rewrite has its own + // arm for it. + #[case::folded_between("(-1.0 * 0.0) BETWEEN (1.0 - 1.0) AND 1.0", true)] + #[case::folded_in_list("(-1.0 * 0.0) IN ((1.0 - 1.0), 1.0)", true)] + #[case::folded_not_in_list("(-1.0 * 0.0) NOT IN ((1.0 - 1.0), 1.0)", false)] + // Nested under a connective, so the operand pass has to descend. + #[case::under_or("(-1.0 * 0.0) < (1.0 - 1.0) OR 1.0 > 2.0", false)] + #[case::under_not("NOT ((-1.0 * 0.0) < (1.0 - 1.0))", true)] + // Wrapped in something that folds the comparison's own result. These are why + // the operand folding walks every container instead of a list of allowed + // parents: each of these is a different spelling of the same exposure. + #[case::under_is_true("((-1.0 * 0.0) < (1.0 - 1.0)) IS TRUE", false)] + #[case::under_is_false("((-1.0 * 0.0) < (1.0 - 1.0)) IS FALSE", true)] + #[case::under_is_not_true("((-1.0 * 0.0) < (1.0 - 1.0)) IS NOT TRUE", true)] + #[case::under_eq_true("((-1.0 * 0.0) < (1.0 - 1.0)) = TRUE", false)] + #[case::under_cast("CAST(((-1.0 * 0.0) < (1.0 - 1.0)) AS BOOLEAN)", false)] + #[case::under_in_true("((-1.0 * 0.0) < (1.0 - 1.0)) IN (TRUE)", false)] + #[case::under_is_true_eq("((-1.0 * 0.0) = (1.0 - 1.0)) IS TRUE", true)] + #[case::under_nested_wrappers("NOT (((-1.0 * 0.0) < (1.0 - 1.0)) IS TRUE)", true)] + fn folded_constant_comparisons_use_ieee_semantics( + #[case] filter: &str, + #[case] expected: bool, + ) { + let schema = + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "value", + arrow_schema::DataType::Float64, + true, + )])); + let planner = crate::planner::Planner::new(schema); + let optimized = planner + .optimize_expr(planner.parse_filter(filter).unwrap()) + .unwrap(); + assert_eq!( + optimized, + Expr::Literal(ScalarValue::Boolean(Some(expected)), None), + "filter: {filter}" + ); + } +} diff --git a/rust/lance-datafusion/src/spill.rs b/rust/lance-datafusion/src/spill.rs index 8fa60c93ab6..749a9637fc5 100644 --- a/rust/lance-datafusion/src/spill.rs +++ b/rust/lance-datafusion/src/spill.rs @@ -9,13 +9,17 @@ use std::{ use arrow::ipc::{reader::StreamReader, writer::StreamWriter}; use arrow_array::RecordBatch; -use arrow_schema::{ArrowError, Schema}; +use arrow_schema::{ArrowError, Schema, SchemaRef}; use datafusion::{ - execution::SendableRecordBatchStream, physical_plan::stream::RecordBatchStreamAdapter, + catalog::{TableProvider, streaming::StreamingTable}, + execution::{SendableRecordBatchStream, TaskContext}, + physical_plan::{stream::RecordBatchStreamAdapter, streaming::PartitionStream}, }; use datafusion_common::DataFusionError; +use futures::StreamExt; use lance_arrow::memory::MemoryAccumulator; use lance_core::error::LanceOptionExt; +use lance_core::utils::tempfile::TempDir; /// Start a spill of Arrow data to a file that can be read later multiple times. /// @@ -60,6 +64,141 @@ pub fn create_replay_spill( (sender, receiver) } +/// Wrap a one-shot [`SendableRecordBatchStream`] in a re-scannable [`TableProvider`]. +/// +/// The source is drained in the background into a replayable spill. Two properties +/// keep this cheap for the common case: +/// +/// - **Memory-first.** Up to `memory_limit` bytes are buffered in memory; the spill +/// only touches disk once that budget is exceeded. A source that fits under the +/// limit never hits the filesystem. +/// - **Streaming replay.** A scan can start consuming batches as soon as they land, +/// before the source has finished draining — the first reader is not blocked +/// waiting for the whole source to buffer. +/// +/// Each scan of the returned provider replays the full source, which is what makes a +/// one-shot stream usable in the write retry loop. +/// +/// The provider reports no statistics — the source size is not known until it has +/// been fully drained — so callers that need source statistics (e.g. to drive join +/// ordering) should prefer a materialized or file-backed provider instead. +/// +/// # Examples +/// +/// ``` +/// # use std::sync::Arc; +/// # use arrow_array::{Int32Array, RecordBatch}; +/// # use arrow_schema::{DataType, Field, Schema}; +/// # use datafusion::execution::SendableRecordBatchStream; +/// # use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +/// # use futures::TryStreamExt; +/// # use lance_datafusion::exec::provider_to_stream; +/// # use lance_datafusion::spill::spilling_table_provider; +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); +/// let batch = +/// RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2, 3]))])?; +/// // A one-shot stream can only be consumed once. +/// let source: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( +/// schema.clone(), +/// futures::stream::iter(vec![Ok(batch)]), +/// )); +/// +/// // Wrapping it makes it re-scannable: each scan replays the full source. +/// let provider = spilling_table_provider(source, 100 * 1024 * 1024).await?; +/// let first: Vec = provider_to_stream(provider.clone()).await?.try_collect().await?; +/// let second: Vec = provider_to_stream(provider).await?.try_collect().await?; +/// assert_eq!(first.iter().map(|b| b.num_rows()).sum::(), 3); +/// assert_eq!(second.iter().map(|b| b.num_rows()).sum::(), 3); +/// # Ok(()) +/// # } +/// ``` +pub async fn spilling_table_provider( + mut source: SendableRecordBatchStream, + memory_limit: usize, +) -> Result, DataFusionError> { + let schema = source.schema(); + let tmp_dir = tokio::task::spawn_blocking(TempDir::try_new) + .await + .map_err(|e| DataFusionError::Execution(format!("Failed to spawn temp dir task: {e}")))? + .map_err(|e| DataFusionError::Execution(format!("Failed to create temp dir: {e}")))?; + let tmp_path = tmp_dir.std_path().join("spill.arrows"); + let (mut sender, receiver) = create_replay_spill(tmp_path, schema.clone(), memory_limit); + + // Drain the one-shot source into the spill once, in the background. The spill + // tees to memory/disk so the first reader can consume batches as they arrive + // while later readers replay the complete source. + let drain_handle = tokio::task::spawn(async move { + let mut errored = false; + while let Some(res) = source.next().await { + match res { + Ok(batch) => { + if let Err(e) = sender.write(batch).await { + sender.send_error(e); + errored = true; + break; + } + } + Err(e) => { + sender.send_error(e); + errored = true; + break; + } + } + } + // Only finish on a clean drain. Calling finish() after an error would + // overwrite the original (replayable) error with a generic one, losing + // the source error's type (e.g. an external error from user code). + if !errored && let Err(err) = sender.finish().await { + sender.send_error(err); + } + sender + }); + + let partition = Arc::new(SpillPartition { + schema: schema.clone(), + receiver, + _tmp_dir: Arc::new(tmp_dir), + _drain_handle: Arc::new(drain_handle), + }); + Ok(Arc::new(StreamingTable::try_new(schema, vec![partition])?)) +} + +/// A [`PartitionStream`] backed by a replayable spill. +/// +/// Each call to [`PartitionStream::execute`] opens a fresh stream over the spill, +/// so the partition can be scanned repeatedly. The spill file and the background +/// task draining the source are kept alive for as long as this partition exists. +struct SpillPartition { + schema: SchemaRef, + receiver: SpillReceiver, + // The spilled data lives in this temp dir; dropping it deletes the spill file. + _tmp_dir: Arc, + // Keeps the background drain task (which owns the `SpillSender`) alive. The + // `SpillSender` must outlive the readers or they error out, so we hold the + // handle rather than detaching it. + _drain_handle: Arc>, +} + +impl std::fmt::Debug for SpillPartition { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SpillPartition") + .field("schema", &self.schema) + .finish() + } +} + +impl PartitionStream for SpillPartition { + fn schema(&self) -> &SchemaRef { + &self.schema + } + + fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { + self.receiver.read() + } +} + #[derive(Clone)] pub struct SpillReceiver { status_receiver: tokio::sync::watch::Receiver, @@ -250,7 +389,7 @@ impl Default for DataLocation { } } -/// A DataFusion error that be be emitted multiple times. We provide the +/// A DataFusion error that can be emitted multiple times. We provide the /// Original error first, and subsequent conversions provide a copy with a /// string representation of the original error. #[derive(Debug)] diff --git a/rust/lance-datafusion/src/sql.rs b/rust/lance-datafusion/src/sql.rs index 67ce2ea24a2..3ed420a0794 100644 --- a/rust/lance-datafusion/src/sql.rs +++ b/rust/lance-datafusion/src/sql.rs @@ -38,6 +38,10 @@ impl Dialect for LanceDialect { fn is_delimited_identifier_start(&self, ch: char) -> bool { ch == '`' } + + fn supports_bitwise_shift_operators(&self) -> bool { + self.0.supports_bitwise_shift_operators() + } } /// Parse sql filter to Expression. diff --git a/rust/lance-datafusion/src/substrait.rs b/rust/lance-datafusion/src/substrait.rs index 1c465fcae4a..f3dc9b4d3df 100644 --- a/rust/lance-datafusion/src/substrait.rs +++ b/rust/lance-datafusion/src/substrait.rs @@ -13,8 +13,9 @@ use datafusion_substrait::logical_plan::consumer::{ use datafusion_substrait::substrait::proto::{ AggregateRel, Expression, ExpressionReference, ExtendedExpression, NamedStruct, Plan, Type, expression::{ - RexType, + Literal, RexType, field_reference::{ReferenceType, RootType}, + literal::{LiteralType, PrecisionTimestamp}, reference_segment, }, expression_reference::ExprType, @@ -97,6 +98,114 @@ fn count_fields(dtype: &Type) -> usize { } } +fn count_fields_without_list_children(dtype: &Type) -> usize { + match dtype.kind.as_ref().unwrap() { + Kind::Struct(struct_type) => { + struct_type + .types + .iter() + .map(count_fields_without_list_children) + .sum::() + + 1 + } + Kind::List(_) => 1, + _ => 1, + } +} + +fn append_nested_field_names( + substrait_type: &Type, + arrow_type: &DataType, + names: &mut Vec, +) -> Result<()> { + match substrait_type.kind.as_ref().unwrap() { + Kind::Struct(substrait_struct) => { + let DataType::Struct(arrow_fields) = arrow_type else { + return Err(Error::invalid_input_source( + format!( + "the provided substrait schema contained a struct where the input schema contained {arrow_type}" + ) + .into(), + )); + }; + if substrait_struct.types.len() != arrow_fields.len() { + return Err(Error::invalid_input_source( + format!( + "the provided substrait struct had {} fields but the corresponding input struct had {} fields", + substrait_struct.types.len(), + arrow_fields.len() + ) + .into(), + )); + } + for (substrait_field, arrow_field) in + substrait_struct.types.iter().zip(arrow_fields.iter()) + { + names.push(arrow_field.name().to_string()); + append_nested_field_names(substrait_field, arrow_field.data_type(), names)?; + } + } + Kind::List(substrait_list) => { + let (DataType::List(arrow_field) | DataType::LargeList(arrow_field)) = arrow_type + else { + return Err(Error::invalid_input_source( + format!( + "the provided substrait schema contained a list where the input schema contained {arrow_type}" + ) + .into(), + )); + }; + let substrait_element = substrait_list.r#type.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "the provided substrait schema contained a list without an element type".into(), + ) + })?; + append_nested_field_names(substrait_element, arrow_field.data_type(), names)?; + } + _ => {} + } + Ok(()) +} + +fn normalize_substrait_names( + substrait_schema: &NamedStruct, + arrow_schema: &ArrowSchema, +) -> Result> { + let fields = substrait_schema.r#struct.as_ref().unwrap(); + let expected_names = fields.types.iter().map(count_fields).sum::(); + if substrait_schema.names.len() == expected_names { + return Ok(substrait_schema.names.clone()); + } + + // PyArrow stops emitting names below list element types, while DataFusion's + // Substrait consumer requires the complete depth-first list of struct names. + let expected_pyarrow_names = fields + .types + .iter() + .map(count_fields_without_list_children) + .sum::(); + if substrait_schema.names.len() != expected_pyarrow_names { + return Err(Error::invalid_input_source( + format!( + "the provided substrait schema had {} names but its types require either {} names or {} names when list children are omitted", + substrait_schema.names.len(), + expected_names, + expected_pyarrow_names + ) + .into(), + )); + } + + let mut names = Vec::with_capacity(expected_names); + let mut name_index = 0; + for (substrait_field, arrow_field) in fields.types.iter().zip(arrow_schema.fields().iter()) { + names.push(substrait_schema.names[name_index].clone()); + append_nested_field_names(substrait_field, arrow_field.data_type(), &mut names)?; + name_index += count_fields_without_list_children(substrait_field); + } + Ok(names) +} + fn remove_extension_types( substrait_schema: &NamedStruct, arrow_schema: Arc, @@ -105,13 +214,21 @@ fn remove_extension_types( if fields.types.len() != arrow_schema.fields.len() { return Err(Error::invalid_input_source("the number of fields in the provided substrait schema did not match the number of fields in the input schema.".into())); } + let substrait_names = normalize_substrait_names(substrait_schema, arrow_schema.as_ref())?; let mut kept_substrait_fields = Vec::with_capacity(fields.types.len()); let mut kept_arrow_fields = Vec::with_capacity(arrow_schema.fields.len()); - let mut index_mapping = HashMap::with_capacity(arrow_schema.fields.len()); - let mut field_counter = 0; - let mut field_index = 0; + let mut name_index_mapping = HashMap::with_capacity(substrait_names.len()); + let mut field_index_mapping = HashMap::with_capacity(arrow_schema.fields.len()); + let mut kept_name_count = 0; + let mut name_index = 0; + let mut kept_field_count = 0; // TODO: this logic doesn't catch user defined fields inside of struct fields - for (substrait_field, arrow_field) in fields.types.iter().zip(arrow_schema.fields.iter()) { + for (field_index, (substrait_field, arrow_field)) in fields + .types + .iter() + .zip(arrow_schema.fields.iter()) + .enumerate() + { let num_fields = count_fields(substrait_field); let kind = substrait_field.kind.as_ref().unwrap(); @@ -123,21 +240,23 @@ fn remove_extension_types( _ => false, }; - if !substrait_schema.names[field_index].starts_with("__unlikely_name_placeholder") + if !substrait_names[name_index].starts_with("__unlikely_name_placeholder") && !is_user_defined { kept_substrait_fields.push(substrait_field.clone()); kept_arrow_fields.push(arrow_field.clone()); for i in 0..num_fields { - index_mapping.insert(field_index + i, field_counter + i); + name_index_mapping.insert(name_index + i, kept_name_count + i); } - field_counter += num_fields; + field_index_mapping.insert(field_index, kept_field_count); + kept_name_count += num_fields; + kept_field_count += 1; } - field_index += num_fields; + name_index += num_fields; } - let mut names = vec![String::new(); index_mapping.len()]; - for (old_idx, old_name) in substrait_schema.names.iter().enumerate() { - if let Some(new_idx) = index_mapping.get(&old_idx) { + let mut names = vec![String::new(); name_index_mapping.len()]; + for (old_idx, old_name) in substrait_names.iter().enumerate() { + if let Some(new_idx) = name_index_mapping.get(&old_idx) { names[*new_idx] = old_name.clone(); } } @@ -150,13 +269,59 @@ fn remove_extension_types( types: kept_substrait_fields, }), }; - Ok((new_substrait_schema, new_arrow_schema, index_mapping)) + Ok((new_substrait_schema, new_arrow_schema, field_index_mapping)) +} + +/// Substrait's optional message fields are `None` when a producer omits them, so unwrapping one +/// turns a malformed (or merely terse) filter into a panic. This walker runs on every filter, so +/// report the missing field instead. +fn missing_field(what: &str) -> Error { + Error::invalid_input(format!( + "filter expression was missing a required {what} field" + )) } -fn remap_expr_references(expr: &mut Expression, mapping: &HashMap) -> Result<()> { - match expr.rex_type.as_mut().unwrap() { +/// Substrait's deprecated `timestamp`/`timestamp_tz` literals are always microseconds, but +/// DataFusion takes their unit from `type_variation_reference` and reads the default 0 as +/// seconds. PyArrow emits exactly that, so filters silently matched the wrong rows. +/// +/// Rewrite them into the `precision_timestamp` forms, which state the unit. +#[allow(deprecated)] +fn normalize_deprecated_timestamp_literal(lit: &mut Literal) { + let precision = match lit.type_variation_reference { + // 0 is Substrait's default reference (microseconds); 1..=3 are DataFusion's ms/us/ns. + 0 | 2 => 6, + 1 => 3, + 3 => 9, + _ => return, + }; + let replacement = match lit.literal_type { + Some(LiteralType::Timestamp(value)) => { + LiteralType::PrecisionTimestamp(PrecisionTimestamp { precision, value }) + } + Some(LiteralType::TimestampTz(value)) => { + LiteralType::PrecisionTimestampTz(PrecisionTimestamp { precision, value }) + } + _ => return, + }; + lit.literal_type = Some(replacement); + lit.type_variation_reference = 0; +} + +/// Reject operators we cannot push down, normalize ambiguous literals, and remap field +/// references onto the schema `remove_extension_types` left behind. +fn normalize_expr(expr: &mut Expression, mapping: &HashMap) -> Result<()> { + match expr + .rex_type + .as_mut() + .ok_or_else(|| missing_field("expression"))? + { + RexType::Literal(lit) => { + normalize_deprecated_timestamp_literal(lit); + Ok(()) + } // Simple, no field references possible - RexType::Literal(_) | RexType::Nested(_) | RexType::DynamicParameter(_) => Ok(()), + RexType::Nested(_) | RexType::DynamicParameter(_) => Ok(()), // Enum literals are deprecated in Substrait and should only appear in older plans. #[allow(deprecated)] RexType::Enum(_) => Ok(()), @@ -164,70 +329,114 @@ fn remap_expr_references(expr: &mut Expression, mapping: &HashMap) RexType::WindowFunction(_) | RexType::Subquery(_) => Err(Error::invalid_input( "Window functions or subqueries not allowed in filter expression", )), + RexType::Lambda(_) | RexType::LambdaInvocation(_) => Err(Error::invalid_input( + "Lambda expressions not allowed in filter expression", + )), // Pass through operators, nested children may have field references RexType::ScalarFunction(func) => { #[allow(deprecated)] for arg in &mut func.args { - remap_expr_references(arg, mapping)?; + normalize_expr(arg, mapping)?; } for arg in &mut func.arguments { - match arg.arg_type.as_mut().unwrap() { - ArgType::Value(expr) => remap_expr_references(expr, mapping)?, + match arg + .arg_type + .as_mut() + .ok_or_else(|| missing_field("function argument"))? + { + ArgType::Value(expr) => normalize_expr(expr, mapping)?, ArgType::Enum(_) | ArgType::Type(_) => {} } } Ok(()) } RexType::IfThen(ifthen) => { - for clause in ifthen.ifs.iter_mut() { - remap_expr_references(clause.r#if.as_mut().unwrap(), mapping)?; - remap_expr_references(clause.then.as_mut().unwrap(), mapping)?; + for (i, clause) in ifthen.ifs.iter_mut().enumerate() { + normalize_expr( + clause + .r#if + .as_mut() + .ok_or_else(|| missing_field("if clause condition"))?, + mapping, + )?; + match clause.then.as_mut() { + Some(then) => normalize_expr(then, mapping)?, + // Only the leading clause may omit `then`, in which case its condition is + // the case expression being matched against. + None if i == 0 => {} + None => return Err(missing_field("if clause result")), + } + } + if let Some(otherwise) = ifthen.r#else.as_mut() { + normalize_expr(otherwise, mapping)?; } - remap_expr_references(ifthen.r#else.as_mut().unwrap(), mapping)?; Ok(()) } RexType::SwitchExpression(switch) => { for clause in switch.ifs.iter_mut() { - remap_expr_references(clause.then.as_mut().unwrap(), mapping)?; + if let Some(then) = clause.then.as_mut() { + normalize_expr(then, mapping)?; + } + } + if let Some(otherwise) = switch.r#else.as_mut() { + normalize_expr(otherwise, mapping)?; } - remap_expr_references(switch.r#else.as_mut().unwrap(), mapping)?; Ok(()) } RexType::SingularOrList(orlist) => { for opt in orlist.options.iter_mut() { - remap_expr_references(opt, mapping)?; + normalize_expr(opt, mapping)?; } - remap_expr_references(orlist.value.as_mut().unwrap(), mapping)?; + normalize_expr( + orlist + .value + .as_mut() + .ok_or_else(|| missing_field("IN list value"))?, + mapping, + )?; Ok(()) } RexType::MultiOrList(orlist) => { for opt in orlist.options.iter_mut() { for field in opt.fields.iter_mut() { - remap_expr_references(field, mapping)?; + normalize_expr(field, mapping)?; } } for val in orlist.value.iter_mut() { - remap_expr_references(val, mapping)?; + normalize_expr(val, mapping)?; } Ok(()) } RexType::Cast(cast) => { - remap_expr_references(cast.input.as_mut().unwrap(), mapping)?; + normalize_expr( + cast.input + .as_mut() + .ok_or_else(|| missing_field("cast input"))?, + mapping, + )?; Ok(()) } RexType::Selection(sel) => { - // Finally, the selection, which might actually have field references - let root_type = sel.root_type.as_mut().unwrap(); - // These types of references do not reference input fields so no remap needed + // Finally, the selection, which might actually have field references. + // An omitted root is a reference into the input, same as RootReference. if matches!( - root_type, - RootType::Expression(_) | RootType::OuterReference(_) + sel.root_type.as_mut(), + Some(RootType::Expression(_) | RootType::OuterReference(_)) ) { + // These types of references do not reference input fields so no remap needed return Ok(()); } - match sel.reference_type.as_mut().unwrap() { + match sel + .reference_type + .as_mut() + .ok_or_else(|| missing_field("field reference"))? + { ReferenceType::DirectReference(direct) => { - match direct.reference_type.as_mut().unwrap() { + match direct + .reference_type + .as_mut() + .ok_or_else(|| missing_field("reference segment"))? + { reference_segment::ReferenceType::ListElement(_) | reference_segment::ReferenceType::MapKey(_) => Err(Error::invalid_input( "map/list nested references not supported in pushdown filters", @@ -298,19 +507,10 @@ pub async fn parse_substrait( let (substrait_schema, _, index_mapping) = remove_extension_types(envelope.base_schema.as_ref().unwrap(), input_schema.clone())?; - if substrait_schema.r#struct.as_ref().unwrap().types.len() - != envelope - .base_schema - .as_ref() - .unwrap() - .r#struct - .as_ref() - .unwrap() - .types - .len() - { - remap_expr_references(&mut expr, &index_mapping)?; - } + // Always walk the expression: this also rejects operators we cannot push down and + // normalizes literals. When no fields were removed the mapping is the identity, so the + // remap itself is a no-op. + normalize_expr(&mut expr, &index_mapping)?; substrait_schema } else { @@ -533,10 +733,10 @@ async fn parse_measures( mod tests { use std::sync::Arc; - use arrow_schema::{DataType, Field, Schema}; + use arrow_schema::{DataType, Field, Schema, TimeUnit}; use datafusion::{ execution::SessionState, - logical_expr::{BinaryExpr, Operator}, + logical_expr::{BinaryExpr, Case, Operator}, prelude::{Expr, SessionContext}, }; use datafusion_common::{Column, ScalarValue}; @@ -544,20 +744,22 @@ mod tests { Expression, ExpressionReference, ExtendedExpression, FunctionArgument, NamedStruct, Type, Version, expression::{ - FieldReference, Literal, ReferenceSegment, RexType, ScalarFunction, + FieldReference, IfThen, Literal, ReferenceSegment, RexType, ScalarFunction, field_reference::{ReferenceType, RootReference, RootType}, + if_then::IfClause, literal::LiteralType, reference_segment::{self, StructField}, }, expression_reference::ExprType, extensions::{ - SimpleExtensionDeclaration, SimpleExtensionUri, SimpleExtensionUrn, + SimpleExtensionDeclaration, SimpleExtensionUrn, simple_extension_declaration::{ExtensionFunction, MappingType}, }, function_argument::ArgType, r#type::{Boolean, I32, Kind, Nullability, Struct}, }; use prost::Message; + use rstest::rstest; use crate::substrait::{encode_substrait, parse_substrait}; @@ -576,13 +778,6 @@ mod tests { git_hash: "".to_string(), producer: "unit-test".to_string(), }), - #[expect(deprecated)] - extension_uris: vec![ - SimpleExtensionUri { - extension_uri_anchor: 1, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_comparison.yaml".to_string(), - } - ], extension_urns: vec![ SimpleExtensionUrn { extension_urn_anchor: 1, @@ -592,8 +787,6 @@ mod tests { extensions: vec![ SimpleExtensionDeclaration { mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction { - #[expect(deprecated)] - extension_uri_reference: 1, extension_urn_reference: 1, function_anchor: 1, name: "lt".to_string(), @@ -670,6 +863,211 @@ mod tests { assert_eq!(df_expr, expected); } + /// A base schema with no extension types needs no field pruning, which is the case that + /// used to skip validation entirely. + async fn parse_unpruned_expr(rex_type: RexType) -> lance_core::Result { + let expr = ExtendedExpression { + version: Some(Version { + major_number: 0, + minor_number: 63, + patch_number: 1, + git_hash: "".to_string(), + producer: "unit-test".to_string(), + }), + extension_urns: vec![], + extensions: vec![], + referred_expr: vec![ExpressionReference { + output_names: vec!["filter_mask".to_string()], + expr_type: Some(ExprType::Expression(Expression { + rex_type: Some(rex_type), + })), + }], + base_schema: Some(NamedStruct { + names: vec!["x".to_string()], + r#struct: Some(Struct { + types: vec![Type { + kind: Some(Kind::I32(I32 { + type_variation_reference: 0, + nullability: Nullability::Nullable as i32, + })), + }], + type_variation_reference: 0, + nullability: Nullability::Required as i32, + }), + }), + advanced_extensions: None, + expected_type_urls: vec![], + }; + + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, true)])); + parse_substrait(expr.encode_to_vec().as_slice(), schema, &session_state()).await + } + + #[tokio::test] + async fn test_unsupported_operator_rejected_without_pruning() { + let err = parse_unpruned_expr(RexType::Subquery(Box::default())) + .await + .expect_err("subqueries should be rejected in filter expressions"); + assert!( + err.to_string() + .contains("Window functions or subqueries not allowed in filter expression"), + "unexpected error: {err}" + ); + + let err = parse_unpruned_expr(RexType::Lambda(Box::default())) + .await + .expect_err("lambdas should be rejected in filter expressions"); + assert!( + err.to_string() + .contains("Lambda expressions not allowed in filter expression"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_unpruned_selection_without_explicit_root() { + let expr = parse_unpruned_expr(RexType::Selection(Box::new(FieldReference { + reference_type: Some(ReferenceType::DirectReference(ReferenceSegment { + reference_type: Some(reference_segment::ReferenceType::StructField(Box::new( + StructField { + field: 0, + child: None, + }, + ))), + })), + root_type: None, + }))) + .await + .unwrap(); + + assert_eq!(expr, Expr::Column(Column::new_unqualified("x"))); + } + + /// The deprecated literal is always microseconds, but DataFusion reads the default + /// variation reference as seconds. + #[rstest] + #[case::default_reference(0, TimeUnit::Microsecond)] + #[case::milli_reference(1, TimeUnit::Millisecond)] + #[case::micro_reference(2, TimeUnit::Microsecond)] + #[case::nano_reference(3, TimeUnit::Nanosecond)] + #[tokio::test] + async fn test_deprecated_timestamp_literal_units( + #[case] type_variation_reference: u32, + #[case] expected_unit: TimeUnit, + ) { + const MICROS: i64 = 1_704_247_200_000_000; + + #[allow(deprecated)] + let expr = parse_unpruned_expr(RexType::Literal(Literal { + nullable: false, + type_variation_reference, + literal_type: Some(LiteralType::Timestamp(MICROS)), + })) + .await + .unwrap(); + + let expected = match expected_unit { + TimeUnit::Millisecond => ScalarValue::TimestampMillisecond(Some(MICROS), None), + TimeUnit::Microsecond => ScalarValue::TimestampMicrosecond(Some(MICROS), None), + TimeUnit::Nanosecond => ScalarValue::TimestampNanosecond(Some(MICROS), None), + other => panic!("unexpected time unit {other:?}"), + }; + assert_eq!(expr, Expr::Literal(expected, None)); + } + + /// DataFusion has no consumer branch for the deprecated `timestamp_tz`, so it failed the + /// filter outright rather than answering wrongly. + #[tokio::test] + async fn test_deprecated_timestamp_tz_literal() { + const MICROS: i64 = 1_704_247_200_000_000; + + #[allow(deprecated)] + let expr = parse_unpruned_expr(RexType::Literal(Literal { + nullable: false, + type_variation_reference: 0, + literal_type: Some(LiteralType::TimestampTz(MICROS)), + })) + .await + .unwrap(); + + assert_eq!( + expr, + Expr::Literal( + ScalarValue::TimestampMicrosecond(Some(MICROS), Some("UTC".into())), + None + ) + ); + } + + /// Optional message fields that a producer may legitimately omit must not panic the walker. + #[tokio::test] + async fn test_unpruned_if_then_with_omitted_optional_fields() { + let condition = Expression { + rex_type: Some(RexType::Literal(Literal { + nullable: false, + type_variation_reference: 0, + literal_type: Some(LiteralType::Boolean(true)), + })), + }; + + // A leading clause without `then` supplies the case expression, and `else` is optional. + let expr = parse_unpruned_expr(RexType::IfThen(Box::new(IfThen { + ifs: vec![IfClause { + r#if: Some(condition), + then: None, + }], + r#else: None, + }))) + .await + .unwrap(); + + assert_eq!( + expr, + Expr::Case(Case { + expr: Some(Box::new(Expr::Literal( + ScalarValue::Boolean(Some(true)), + None + ))), + when_then_expr: vec![], + else_expr: None, + }) + ); + } + + /// DataFusion only tolerates an omitted `then` on the leading clause, so a later clause + /// missing one has to be rejected here rather than reaching the consumer. + #[tokio::test] + async fn test_unpruned_if_then_missing_nonleading_then() { + let condition = || Expression { + rex_type: Some(RexType::Literal(Literal { + nullable: false, + type_variation_reference: 0, + literal_type: Some(LiteralType::Boolean(true)), + })), + }; + + let err = parse_unpruned_expr(RexType::IfThen(Box::new(IfThen { + ifs: vec![ + IfClause { + r#if: Some(condition()), + then: Some(condition()), + }, + IfClause { + r#if: Some(condition()), + then: None, + }, + ], + r#else: None, + }))) + .await + .expect_err("a non-leading clause without `then` is not valid"); + + assert!( + err.to_string().contains("if clause result"), + "unexpected error: {err}" + ); + } + #[tokio::test] async fn test_expr_substrait_roundtrip() { let schema = arrow_schema::Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -740,6 +1138,93 @@ mod tests { assert_substrait_roundtrip(schema, id_filter("test-id")).await; } + #[tokio::test] + async fn test_parse_substrait_with_pyarrow_list_struct_names() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + list_of_struct( + "items", + vec![ + Field::new("value", DataType::Float32, true), + Field::new("label", DataType::Utf8, true), + ], + ), + Field::new("checkpoint", DataType::Int64, true), + ])); + let expr = Expr::BinaryExpr(BinaryExpr { + left: Box::new(Expr::Column(Column::new_unqualified("checkpoint"))), + op: Operator::Eq, + right: Box::new(Expr::Literal(ScalarValue::Int64(Some(0)), None)), + }); + + let bytes = encode_substrait(expr.clone(), schema.clone(), &session_state()).unwrap(); + let mut envelope = ExtendedExpression::decode(bytes.as_slice()).unwrap(); + let base_schema = envelope.base_schema.as_mut().unwrap(); + assert_eq!( + base_schema.names, + ["id", "items", "value", "label", "checkpoint"] + ); + + // PyArrow omits names nested beneath list element types. + base_schema.names = ["id", "items", "checkpoint"] + .map(ToString::to_string) + .to_vec(); + let bytes = envelope.encode_to_vec(); + + let decoded = parse_substrait(bytes.as_slice(), schema, &session_state()) + .await + .unwrap(); + assert_eq!(decoded, expr); + } + + #[tokio::test] + async fn test_pyarrow_shallow_names_with_placeholder_before_filter() { + let list_field = list_of_struct( + "items", + vec![ + Field::new("value", DataType::Float32, true), + Field::new("label", DataType::Utf8, true), + ], + ); + let placeholder = "__unlikely_name_placeholder_0"; + let serialized_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + list_field.clone(), + Field::new(placeholder, DataType::Int8, true), + Field::new("checkpoint", DataType::Int64, true), + ])); + let input_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + list_field, + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + Field::new("checkpoint", DataType::Int64, true), + ])); + let expr = Expr::BinaryExpr(BinaryExpr { + left: Box::new(Expr::Column(Column::new_unqualified("checkpoint"))), + op: Operator::Eq, + right: Box::new(Expr::Literal(ScalarValue::Int64(Some(0)), None)), + }); + + let bytes = encode_substrait(expr.clone(), serialized_schema, &session_state()).unwrap(); + let mut envelope = ExtendedExpression::decode(bytes.as_slice()).unwrap(); + envelope.base_schema.as_mut().unwrap().names = ["id", "items", placeholder, "checkpoint"] + .map(ToString::to_string) + .to_vec(); + + let decoded = parse_substrait( + envelope.encode_to_vec().as_slice(), + input_schema, + &session_state(), + ) + .await + .unwrap(); + assert_eq!(decoded, expr); + } + #[tokio::test] async fn test_substrait_roundtrip_with_list_struct_struct() { let schema = Schema::new(vec![ @@ -881,9 +1366,7 @@ mod tests { fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration { SimpleExtensionDeclaration { mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction { - #[allow(deprecated)] - extension_uri_reference: 1, - extension_urn_reference: 0, + extension_urn_reference: 1, function_anchor: anchor, name: name.to_string(), })), @@ -919,10 +1402,9 @@ mod tests { git_hash: String::new(), producer: "lance-test".to_string(), }), - #[allow(deprecated)] - extension_uris: vec![SimpleExtensionUri { - extension_uri_anchor: 1, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), + extension_urns: vec![SimpleExtensionUrn { + extension_urn_anchor: 1, + urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), }], extensions, relations: vec![PlanRel { @@ -935,7 +1417,6 @@ mod tests { }], advanced_extensions: None, expected_type_urls: vec![], - extension_urns: vec![], parameter_bindings: vec![], type_aliases: vec![], }; diff --git a/rust/lance-datafusion/src/utils.rs b/rust/lance-datafusion/src/utils.rs index 470831bf0bc..e660c8ee44d 100644 --- a/rust/lance-datafusion/src/utils.rs +++ b/rust/lance-datafusion/src/utils.rs @@ -244,6 +244,8 @@ pub const INDICES_LOADED_METRIC: &str = "indices_loaded"; pub const PARTS_LOADED_METRIC: &str = "parts_loaded"; pub const PARTITIONS_RANKED_METRIC: &str = "partitions_ranked"; pub const INDEX_COMPARISONS_METRIC: &str = "index_comparisons"; +pub const INDEX_CACHE_HITS_METRIC: &str = "index_cache_hits"; +pub const INDEX_CACHE_MISSES_METRIC: &str = "index_cache_misses"; pub const FRAGMENTS_SCANNED_METRIC: &str = "fragments_scanned"; pub const RANGES_SCANNED_METRIC: &str = "ranges_scanned"; pub const ROWS_SCANNED_METRIC: &str = "rows_scanned"; diff --git a/rust/lance-datagen/Cargo.toml b/rust/lance-datagen/Cargo.toml index 83b5aba3689..4a57f1f0804 100644 --- a/rust/lance-datagen/Cargo.toml +++ b/rust/lance-datagen/Cargo.toml @@ -17,7 +17,7 @@ arrow-schema = { workspace = true } chrono = { workspace = true } futures = { workspace = true } half = { workspace = true } -hex = "0.4.3" +hex.workspace = true rand = { workspace = true } rand_distr = { workspace = true } rand_xoshiro = { workspace = true } @@ -25,6 +25,7 @@ rand_xoshiro = { workspace = true } [dev-dependencies] criterion = { workspace = true } lance-testing.workspace = true +rstest.workspace = true [lib] bench = false diff --git a/rust/lance-datagen/src/generator.rs b/rust/lance-datagen/src/generator.rs index 39da4734619..333592747e8 100644 --- a/rust/lance-datagen/src/generator.rs +++ b/rust/lance-datagen/src/generator.rs @@ -5,7 +5,7 @@ use std::{collections::HashMap, iter, marker::PhantomData, sync::Arc, sync::Lazy use arrow::{ array::{ArrayData, AsArray, Float32Builder, GenericBinaryBuilder, GenericStringBuilder}, - buffer::{BooleanBuffer, Buffer, OffsetBuffer, ScalarBuffer}, + buffer::{BooleanBuffer, Buffer, MutableBuffer, OffsetBuffer, ScalarBuffer}, datatypes::{ ArrowPrimitiveType, Float32Type, Int32Type, Int64Type, IntervalDayTime, IntervalMonthDayNano, UInt32Type, @@ -569,7 +569,19 @@ where }; self.leftover_count = ((self.leftover_count as u64 + length.0) % self.repeat as u64) as u32; self.leftover = values.last().copied().unwrap_or(T::default()); - Ok(Arc::new(ArrayType::from(values))) + let array = ArrayType::from(values); + // `ArrayType::from` uses the primitive type's default metadata. For + // timezone-aware timestamps this drops the timezone, so restore the + // generator's declared type when it differs. + if array.data_type() == &self.data_type { + return Ok(Arc::new(array)); + } + let data = array + .into_data() + .into_builder() + .data_type(self.data_type.clone()) + .build()?; + Ok(make_array(data)) } fn data_type(&self) -> &DataType { @@ -814,9 +826,9 @@ impl ArrayGenerator for RandomBytesGenerato rng: &mut rand_xoshiro::Xoshiro256PlusPlus, ) -> Result, ArrowError> { let num_bytes = length.0 * Self::byte_width()?; - let mut bytes = vec![0; num_bytes as usize]; - rng.fill_bytes(&mut bytes); - let bytes = ScalarBuffer::new(Buffer::from(bytes), 0, length.0 as usize); + let mut bytes = MutableBuffer::from_len_zeroed(num_bytes as usize); + rng.fill_bytes(bytes.as_slice_mut()); + let bytes = ScalarBuffer::new(bytes.into(), 0, length.0 as usize); Ok(Arc::new( PrimitiveArray::::new(bytes, None).with_data_type(self.data_type.clone()), )) @@ -2818,14 +2830,29 @@ pub mod array { Box::new(RandomIntervalGenerator::new(unit)) } + /// The default sampling range for temporal generators: the 365 days ending at + /// 2024-01-01T00:00:00Z (exclusive) + /// + /// The range must be a fixed anchor and not derived from the wall clock + /// (e.g. `Utc::now()`), otherwise the same RNG seed would generate different + /// values depending on when the generator was created, breaking + /// reproducibility (e.g. of saved fuzz inputs). Callers that need a + /// time-relative range can use the `*_in_range` variants. + fn default_temporal_range() -> (chrono::DateTime, chrono::DateTime) { + let end = chrono::DateTime::::from_timestamp(1_704_067_200, 0) + .expect("2024-01-01T00:00:00Z is a valid timestamp"); + let start = end - chrono::TimeDelta::try_days(365).expect("TimeDelta try_days"); + (start, end) + } + /// Create a generator of randomly sampled date32 values /// - /// Instead of sampling the entire range, all values will be drawn from the last year as this - /// is a more common use pattern + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_date32_in_range`] to control the range. pub fn rand_date32() -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::TimeDelta::try_days(365).expect("TimeDelta try days"); - rand_date32_in_range(one_year_ago, now) + let (start, end) = default_temporal_range(); + rand_date32_in_range(start, end) } /// Create a generator of randomly sampled date32 values in the given range @@ -2853,12 +2880,12 @@ pub mod array { /// Create a generator of randomly sampled date64 values /// - /// Instead of sampling the entire range, all values will be drawn from the last year as this - /// is a more common use pattern + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_date64_in_range`] to control the range. pub fn rand_date64() -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::TimeDelta::try_days(365).expect("TimeDelta try_days"); - rand_date64_in_range(one_year_ago, now) + let (start, end) = default_temporal_range(); + rand_date64_in_range(start, end) } /// Create a generator of randomly sampled timestamp values in the given range @@ -2914,10 +2941,14 @@ pub mod array { } } + /// Create a generator of randomly sampled timestamp values + /// + /// Instead of sampling the entire range, all values will be drawn from a fixed + /// one-year range (the 365 days ending at 2024-01-01 UTC) as this is a more + /// common use pattern. Use [`rand_timestamp_in_range`] to control the range. pub fn rand_timestamp(data_type: &DataType) -> Box { - let now = chrono::Utc::now(); - let one_year_ago = now - chrono::Duration::try_days(365).unwrap(); - rand_timestamp_in_range(one_year_ago, now, data_type) + let (start, end) = default_temporal_range(); + rand_timestamp_in_range(start, end, data_type) } /// Create a generator of randomly sampled date64 values @@ -3214,11 +3245,55 @@ pub fn rand(schema: &Schema) -> BatchGeneratorBuilder { #[cfg(test)] mod tests { - use arrow::datatypes::{Float32Type, Int8Type, Int16Type, UInt32Type}; - use arrow_array::{BooleanArray, Float32Array, Int8Array, Int16Array, Int32Array, UInt32Array}; + use arrow::datatypes::{Float32Type, Int8Type, Int16Type, TimeUnit, UInt32Type}; + use arrow_array::{ + BooleanArray, Date32Array, Date64Array, Float32Array, Int8Array, Int16Array, Int32Array, + TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, + TimestampSecondArray, UInt32Array, + }; + use rstest::rstest; use super::*; + #[rstest] + #[case::float16(DataType::Float16)] + #[case::decimal128(DataType::Decimal128(38, 10))] + #[case::decimal256(DataType::Decimal256(76, 10))] + fn test_random_bytes_generator_alignment(#[case] data_type: DataType) { + for length in [0, 3] { + let generated = array::rand_type(&data_type) + .generate_default(RowCount::from(length)) + .unwrap(); + assert_eq!(generated.data_type(), &data_type); + assert_eq!(generated.len(), length as usize); + } + } + + #[test] + fn test_timestamp_timezone_is_preserved() { + let data_type = DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".into())); + let mut generator = array::rand_type(&data_type); + let generated = generator.generate_default(RowCount::from(2)).unwrap(); + assert_eq!(generated.data_type(), &data_type); + + let fields = Fields::from(vec![Field::new("timestamp", data_type, true)]); + let mut generator = array::rand_struct(fields.clone()); + let generated = generator.generate_default(RowCount::from(2)).unwrap(); + assert_eq!(generated.data_type(), &DataType::Struct(fields)); + } + + #[test] + fn test_fn_gen_propagates_array_data_build_error() { + // FnGen constructors are internal. Use an incompatible declared type to + // verify that ArrayDataBuilder validation failures are propagated. + let mut generator = FnGen::::new_unknown_size(DataType::Utf8, |_| 0, 1); + + assert!(matches!( + generator.generate_default(RowCount::from(1)), + Err(ArrowError::InvalidArgumentError(_)) + )); + } + #[test] fn test_step() { let mut rng = rand_xoshiro::Xoshiro256PlusPlus::seed_from_u64(DEFAULT_SEED.0); @@ -3395,6 +3470,72 @@ mod tests { ); } + #[test] + fn test_rng_temporal_deterministic() { + // The default temporal generators must not depend on the wall clock: the + // same seed must produce the same values no matter when the generator is + // created (https://github.com/lance-format/lance/issues/7913). These + // exact values pin both the RNG stream and the fixed default sampling + // range (the 365 days ending at 2024-01-01 UTC). + fn gen_values(mut genn: Box) -> Arc { + let mut rng = rand_xoshiro::Xoshiro256PlusPlus::seed_from_u64(DEFAULT_SEED.0); + genn.generate(RowCount::from(3), &mut rng).unwrap() + } + + assert_eq!( + *gen_values(array::rand_date32()), + Date32Array::from(vec![19655, 19474, 19717]) + ); + assert_eq!( + *gen_values(array::rand_date64()), + Date64Array::from(vec![ + 1_698_192_000_000, + 1_682_553_600_000, + 1_703_548_800_000 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Second, + None + ))), + TimestampSecondArray::from(vec![1_698_211_127, 1_682_585_540, 1_703_559_286]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Millisecond, + None + ))), + TimestampMillisecondArray::from(vec![ + 1_698_211_127_056, + 1_682_585_540_319, + 1_703_559_286_487 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Microsecond, + None + ))), + TimestampMicrosecondArray::from(vec![ + 1_698_211_127_056_596, + 1_682_585_540_319_384, + 1_703_559_286_487_645 + ]) + ); + assert_eq!( + *gen_values(array::rand_timestamp(&DataType::Timestamp( + TimeUnit::Nanosecond, + None + ))), + TimestampNanosecondArray::from(vec![ + 1_698_211_127_056_596_085, + 1_682_585_540_319_384_548, + 1_703_559_286_487_645_287 + ]) + ); + } + #[test] fn test_rng_list() { // Note: these tests are heavily dependent on the default seed. diff --git a/rust/lance-derive/Cargo.toml b/rust/lance-derive/Cargo.toml index 4bb99d3ac93..a51660c83a7 100644 --- a/rust/lance-derive/Cargo.toml +++ b/rust/lance-derive/Cargo.toml @@ -14,9 +14,9 @@ categories.workspace = true proc-macro = true [dependencies] -proc-macro2 = "1.0.67" -quote = "1.0.33" -syn = { version = "2.0.37", features = ["full"] } +proc-macro2.workspace = true +quote.workspace = true +syn.workspace = true [lints] workspace = true diff --git a/rust/lance-encoding/Cargo.toml b/rust/lance-encoding/Cargo.toml index df5352262fe..2f2490cfb84 100644 --- a/rust/lance-encoding/Cargo.toml +++ b/rust/lance-encoding/Cargo.toml @@ -25,18 +25,16 @@ lance-bitpacking = { workspace = true, optional = true } bytes.workspace = true futures.workspace = true fsst.workspace = true -hex = "0.4.3" +hex.workspace = true itertools.workspace = true log.workspace = true num-traits.workspace = true prost.workspace = true hyperloglogplus.workspace = true -rand.workspace = true -strum = { workspace =true, features = ["derive"] } tokio.workspace = true tracing.workspace = true xxhash-rust = { version = "0.8.15", features = ["xxh3"] } -bytemuck = { version = "1.14", features = ["extern_crate_alloc"] } +bytemuck.workspace = true byteorder.workspace = true lz4 = { version = "1", optional = true } zstd = { version = "0.13", optional = true } @@ -55,7 +53,7 @@ serial_test.workspace = true [build-dependencies] prost-build.workspace = true -protobuf-src = { version = "2.1", optional = true } +protobuf-src = { workspace = true, optional = true } [features] default = ["lz4", "zstd", "bitpacking"] diff --git a/rust/lance-encoding/benches/common/mod.rs b/rust/lance-encoding/benches/common/mod.rs new file mode 100644 index 00000000000..3fc77f4f03b --- /dev/null +++ b/rust/lance-encoding/benches/common/mod.rs @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_schema::DataType; +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + reject_packed_struct_per_value, try_bitpacking_block, try_bitpacking_miniblock, + try_byte_stream_split_miniblock, try_fixed_packed_struct_miniblock, try_fixed_u8_rle_block, + try_fixed_u8_rle_miniblock, try_general_block, try_raw_block, + try_raw_fixed_size_list_miniblock, try_raw_fixed_width_miniblock, try_raw_per_value, + try_uncompressed_fixed_width_miniblock, try_variable_packed_struct_per_value, + try_variable_width_miniblock, try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encoder::{ + ColumnIndexSequence, FieldEncoder, FieldEncodingContext, FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, + }, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BenchEncoding { + Array, + StructuralU16, + StructuralU32, +} + +impl std::fmt::Display for BenchEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Array => "array", + Self::StructuralU16 => "structural-u16", + Self::StructuralU32 => "structural-u32", + }) + } +} + +#[derive(Debug, Clone)] +struct BenchCompressionStrategy { + encoding: BenchEncoding, + params: CompressionParams, +} + +impl BenchCompressionStrategy { + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + let mut metadata = field_metadata_params(field); + if self.encoding == BenchEncoding::StructuralU16 + && metadata + .minichunk_size + .is_some_and(|size| size >= 32 * 1024) + { + metadata.minichunk_size = None; + } + params.merge(&metadata); + params + } +} + +impl CompressionStrategy for BenchCompressionStrategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_fixed_u8_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + let packed = match self.encoding { + BenchEncoding::StructuralU16 => reject_packed_struct_per_value(field, data)?, + BenchEncoding::StructuralU32 => { + try_variable_packed_struct_per_value(Arc::new(self.clone()), field, data)? + } + BenchEncoding::Array => unreachable!(), + }; + if let Some(compressor) = packed { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if self.encoding == BenchEncoding::StructuralU32 + && let Some(compressor) = try_fixed_u8_rle_block(data, ¶ms)? + { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if self.encoding == BenchEncoding::StructuralU32 + && let Some(compressor) = try_general_block(data, ¶ms)? + { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} + +#[derive(Debug)] +struct BenchFieldEncodingStrategy { + encoding: BenchEncoding, + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for BenchFieldEncodingStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if self.encoding == BenchEncoding::StructuralU32 + && let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if self.encoding == BenchEncoding::StructuralU32 { + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if self.encoding == BenchEncoding::StructuralU16 { + if matches!( + field.data_type(), + DataType::FixedSizeList(item, _) + if matches!(item.data_type(), DataType::Struct(_)) + ) { + return Err(Error::not_supported_source( + "FixedSizeList is not enabled by the selected file format".into(), + )); + } + if matches!(field.data_type(), DataType::Map(_, _)) { + return Err(Error::not_supported_source( + "Map data type is not enabled by the selected file format".into(), + )); + } + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "{} has no field encoding for '{}' with data type {}", + self.encoding, + field.name, + field.data_type() + ) + .into(), + )) + } +} + +pub fn encoding_strategy(encoding: BenchEncoding) -> Box { + if encoding == BenchEncoding::Array { + return Box::new(lance_encoding::encoder::ArrayFieldEncodingStrategy::new()); + } + + let compression = Arc::new(BenchCompressionStrategy { + encoding, + params: CompressionParams::default(), + }); + let page_encodings = match encoding { + BenchEncoding::StructuralU16 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::dense_u16(compression), + ], + BenchEncoding::StructuralU32 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ], + BenchEncoding::Array => unreachable!(), + }; + Box::new(BenchFieldEncodingStrategy { + encoding, + primitive: PrimitiveFieldEncoding::new(page_encodings), + }) +} diff --git a/rust/lance-encoding/benches/decoder.rs b/rust/lance-encoding/benches/decoder.rs index cc0404e1bb3..fa831b69b25 100644 --- a/rust/lance-encoding/benches/decoder.rs +++ b/rust/lance-encoding/benches/decoder.rs @@ -1,25 +1,42 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, hint::black_box, sync::Arc}; use arrow_array::{RecordBatch, UInt32Array}; +#[cfg(feature = "bitpacking")] +use arrow_buffer::ArrowNativeType; use arrow_schema::{DataType, Field, Schema, TimeUnit}; use arrow_select::take::take; +#[cfg(feature = "bitpacking")] +use bytemuck::Pod; use criterion::{Criterion, criterion_group, criterion_main}; use futures::StreamExt; +#[cfg(feature = "bitpacking")] +use lance_bitpacking::BitPacking; use lance_core::cache::LanceCache; use lance_datagen::ArrayGeneratorExt; +#[cfg(feature = "bitpacking")] +use lance_encoding::buffer::LanceBuffer; +#[cfg(feature = "bitpacking")] +use lance_encoding::compression::BlockDecompressor; +#[cfg(feature = "bitpacking")] +use lance_encoding::data::{BlockInfo, DataBlock, FixedWidthDataBlock}; +#[cfg(feature = "bitpacking")] +use lance_encoding::encodings::physical::bitpacking::{ELEMS_PER_CHUNK, InlineBitpacking}; use lance_encoding::{ decoder::{ - DecodeBatchScheduler, DecoderConfig, DecoderPlugins, FilterExpression, create_decode_stream, + DecodeBatchScheduler, DecoderConfig, DecoderPlugins, EncodedBatchLayout, FilterExpression, + create_decode_stream, }, - encoder::{EncodingOptions, default_encoding_strategy, encode_batch}, - version::LanceFileVersion, + encoder::{EncodingOptions, encode_batch}, }; use tokio::sync::mpsc::unbounded_channel; use rand::Rng; +pub mod common; +use common::{BenchEncoding, encoding_strategy}; + const PRIMITIVE_TYPES: &[DataType] = &[ DataType::Date32, DataType::Date64, @@ -49,6 +66,15 @@ const PRIMITIVE_TYPES: &[DataType] = &[ // schema doesn't yet parse them in the context of a fixed size list. const PRIMITIVE_TYPES_FOR_FSL: &[DataType] = &[DataType::Int8, DataType::Float32]; +fn encoded_batch_layout(encoding: BenchEncoding) -> EncodedBatchLayout { + match encoding { + BenchEncoding::Array => EncodedBatchLayout::Array, + BenchEncoding::StructuralU16 | BenchEncoding::StructuralU32 => { + EncodedBatchLayout::Structural + } + } +} + fn bench_decode(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().unwrap(); let mut group = c.benchmark_group("decode_primitive"); @@ -64,7 +90,7 @@ fn bench_decode(c: &mut Criterion) { .unwrap(); let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::default()); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU16); let encoded = rt .block_on(encode_batch( &data, @@ -81,7 +107,7 @@ fn bench_decode(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::default(), + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -95,14 +121,14 @@ fn bench_decode_fsl(c: &mut Criterion) { let rt = tokio::runtime::Runtime::new().unwrap(); let mut group = c.benchmark_group("decode_fsl"); const NUM_BYTES: u64 = 1024 * 1024 * 128; - for version in [ - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, + for encoding in [ + BenchEncoding::Array, + BenchEncoding::StructuralU16, + BenchEncoding::StructuralU32, ] { for data_type in PRIMITIVE_TYPES_FOR_FSL { for dimension in [4, 16, 32, 64, 128] { - let nullable_choices: &[bool] = if version == LanceFileVersion::V2_0 { + let nullable_choices: &[bool] = if encoding == BenchEncoding::Array { &[false] } else { &[false, true] @@ -110,7 +136,7 @@ fn bench_decode_fsl(c: &mut Criterion) { for nullable in nullable_choices { let func_name = format!( "{:?}_{}_v{}_null{}", - data_type, dimension, version, nullable + data_type, dimension, encoding, nullable ) .to_lowercase(); group.throughput(criterion::Throughput::Bytes(NUM_BYTES)); @@ -133,7 +159,7 @@ fn bench_decode_fsl(c: &mut Criterion) { lance_core::datatypes::Schema::try_from(data.schema().as_ref()) .unwrap(), ); - let encoding_strategy = default_encoding_strategy(version); + let encoding_strategy = encoding_strategy(encoding); let encoded = rt .block_on(encode_batch( &data, @@ -149,7 +175,7 @@ fn bench_decode_fsl(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - version, + encoded_batch_layout(encoding), Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -199,7 +225,7 @@ fn bench_decode_str_with_dict_encoding(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::default()); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU16); let encoded = rt .block_on(encode_batch( &data, @@ -215,7 +241,7 @@ fn bench_decode_str_with_dict_encoding(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::default(), + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -274,7 +300,7 @@ fn bench_decode_packed_struct(c: &mut Criterion) { RecordBatch::try_new(Arc::new(new_schema.clone()), data.columns().to_vec()).unwrap(); let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(&new_schema).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); let encoded = rt .block_on(encode_batch( &data, @@ -291,7 +317,7 @@ fn bench_decode_packed_struct(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::V2_2, + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -331,7 +357,7 @@ fn bench_decode_str_with_fixed_size_binary_encoding(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::default()); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU16); let encoded = rt .block_on(encode_batch( &data, @@ -347,7 +373,7 @@ fn bench_decode_str_with_fixed_size_binary_encoding(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::default(), + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -398,7 +424,7 @@ fn bench_decode_compressed(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(schema.as_ref()).unwrap()); // V2_2+ required for general compression - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); // Encode once during setup let encoded = rt @@ -423,7 +449,7 @@ fn bench_decode_compressed(c: &mut Criterion) { &FilterExpression::no_filter(), Arc::::default(), false, - LanceFileVersion::V2_2, + EncodedBatchLayout::Structural, Some(Arc::new(LanceCache::no_cache())), )) .unwrap(); @@ -476,7 +502,7 @@ fn bench_decode_compressed_parallel(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(schema.as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); let encoded = rt .block_on(encode_batch( @@ -563,6 +589,136 @@ fn bench_decode_compressed_parallel(c: &mut Criterion) { } } +#[cfg(feature = "bitpacking")] +fn make_inline_bitpacking_chunk(bit_width: usize) -> LanceBuffer +where + T: ArrowNativeType + BitPacking + Pod, +{ + let value_range = 1_usize << bit_width; + let values: Vec = (0..ELEMS_PER_CHUNK as usize) + .map(|i| T::from_usize((i * 31 + 7) % value_range).unwrap()) + .collect(); + let packed_words = ELEMS_PER_CHUNK as usize * bit_width / (std::mem::size_of::() * 8); + + let mut chunk = Vec::with_capacity(1 + packed_words); + chunk.push(T::from_usize(bit_width).unwrap()); + let payload_start = chunk.len(); + chunk.resize(payload_start + packed_words, T::from_usize(0).unwrap()); + unsafe { + BitPacking::unchecked_pack(bit_width, &values, &mut chunk[payload_start..]); + } + + LanceBuffer::reinterpret_vec(chunk) +} + +#[cfg(feature = "bitpacking")] +fn read_little_endian_header(bytes: &[u8]) -> usize { + bytes[..std::mem::size_of::()] + .iter() + .enumerate() + .fold(0_u64, |value, (idx, byte)| { + value | ((*byte as u64) << (idx * 8)) + }) as usize +} + +#[cfg(feature = "bitpacking")] +fn legacy_copy_unchunk(data: LanceBuffer, num_values: u64) -> DataBlock +where + T: ArrowNativeType + BitPacking + Pod, +{ + assert!(data.len() >= std::mem::size_of::()); + assert!(num_values <= ELEMS_PER_CHUNK); + + let chunk_in_u8 = data.to_vec(); + let bit_width_value = read_little_endian_header::(&chunk_in_u8); + let chunk = bytemuck::cast_slice(&chunk_in_u8[std::mem::size_of::()..]); + assert!(std::mem::size_of_val(chunk) == bit_width_value * ELEMS_PER_CHUNK as usize / 8); + + let mut decompressed = vec![T::from_usize(0).unwrap(); ELEMS_PER_CHUNK as usize]; + unsafe { + BitPacking::unchecked_unpack(bit_width_value, chunk, &mut decompressed); + } + + decompressed.truncate(num_values as usize); + DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(decompressed), + bits_per_value: (std::mem::size_of::() * 8) as u64, + num_values, + block_info: BlockInfo::new(), + }) +} + +#[cfg(feature = "bitpacking")] +fn typed_view_unchunk(buffer: LanceBuffer, uncompressed_bits: u64, num_values: u64) -> DataBlock { + InlineBitpacking::new(uncompressed_bits) + .decompress(Some(buffer), num_values) + .unwrap() +} + +#[cfg(feature = "bitpacking")] +fn assert_same_fixed_width_payloads(legacy: &DataBlock, typed_view: &DataBlock) { + let legacy = legacy.as_fixed_width_ref().unwrap(); + let typed_view = typed_view.as_fixed_width_ref().unwrap(); + + assert_eq!(legacy.num_values, typed_view.num_values); + assert_eq!(legacy.bits_per_value, typed_view.bits_per_value); + assert_eq!(legacy.data.as_ref(), typed_view.data.as_ref()); +} + +#[cfg(feature = "bitpacking")] +fn bench_inline_bitpacking_case( + group: &mut criterion::BenchmarkGroup<'_, criterion::measurement::WallTime>, + name: &str, + bit_width: usize, +) where + T: ArrowNativeType + BitPacking + Pod, +{ + let buffer = make_inline_bitpacking_chunk::(bit_width); + let compressed_bytes = buffer.len() as u64; + let uncompressed_bits = (std::mem::size_of::() * 8) as u64; + group.throughput(criterion::Throughput::Bytes(compressed_bytes)); + + let legacy = legacy_copy_unchunk::(buffer.clone(), ELEMS_PER_CHUNK); + let typed_view = typed_view_unchunk(buffer.clone(), uncompressed_bits, ELEMS_PER_CHUNK); + assert_same_fixed_width_payloads(&legacy, &typed_view); + + group.bench_function(format!("{name}/legacy_copy/compressed_bytes"), |b| { + b.iter(|| { + let decoded = + legacy_copy_unchunk::(black_box(buffer.clone()), black_box(ELEMS_PER_CHUNK)); + let fixed = decoded.as_fixed_width().unwrap(); + black_box(fixed.data.as_ref()); + }) + }); + + group.bench_function(format!("{name}/typed_view/compressed_bytes"), |b| { + b.iter(|| { + let decoded = typed_view_unchunk( + black_box(buffer.clone()), + black_box(uncompressed_bits), + black_box(ELEMS_PER_CHUNK), + ); + let fixed = decoded.as_fixed_width().unwrap(); + black_box(fixed.data.as_ref()); + }) + }); +} + +#[cfg(feature = "bitpacking")] +fn bench_decode_inline_bitpacking_unchunk(c: &mut Criterion) { + let mut group = c.benchmark_group("decode_inline_bitpacking_unchunk"); + bench_inline_bitpacking_case::(&mut group, "u32_bw12_1024", 12); + bench_inline_bitpacking_case::(&mut group, "u64_bw23_1024", 23); + group.finish(); +} + +#[cfg(not(feature = "bitpacking"))] +fn bench_decode_inline_bitpacking_unchunk(c: &mut Criterion) { + let mut group = c.benchmark_group("decode_inline_bitpacking_unchunk"); + group.bench_function("bitpacking_feature_disabled", |b| b.iter(|| black_box(()))); + group.finish(); +} + #[cfg(target_os = "linux")] criterion_group!( name=benches; @@ -570,7 +726,7 @@ criterion_group!( .with_profiler(lance_testing::pprof::PProfProfiler::new(100, lance_testing::pprof::Output::Flamegraph(None))); targets = bench_decode, bench_decode_fsl, bench_decode_str_with_dict_encoding, bench_decode_packed_struct, bench_decode_str_with_fixed_size_binary_encoding, bench_decode_compressed, - bench_decode_compressed_parallel); + bench_decode_compressed_parallel, bench_decode_inline_bitpacking_unchunk); // Non-linux version does not support pprof. #[cfg(not(target_os = "linux"))] @@ -578,5 +734,5 @@ criterion_group!( name=benches; config = Criterion::default().significance_level(0.1).sample_size(10); targets = bench_decode, bench_decode_fsl, bench_decode_str_with_dict_encoding, bench_decode_packed_struct, - bench_decode_compressed, bench_decode_compressed_parallel); + bench_decode_compressed, bench_decode_compressed_parallel, bench_decode_inline_bitpacking_unchunk); criterion_main!(benches); diff --git a/rust/lance-encoding/benches/encoder.rs b/rust/lance-encoding/benches/encoder.rs index 08eb89d32fe..02ffd92083c 100644 --- a/rust/lance-encoding/benches/encoder.rs +++ b/rust/lance-encoding/benches/encoder.rs @@ -7,15 +7,15 @@ use arrow_array::{ArrayRef, BooleanArray, ListArray, RecordBatch}; use arrow_buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Schema}; use criterion::{Criterion, criterion_group, criterion_main}; -use lance_encoding::{ - encoder::{EncodingOptions, default_encoding_strategy, encode_batch}, - version::LanceFileVersion, -}; +use lance_encoding::encoder::{EncodingOptions, encode_batch}; + +pub mod common; +use common::{BenchEncoding, encoding_strategy}; fn encode_batch_sync(rt: &tokio::runtime::Runtime, data: &RecordBatch) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap()); - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); rt.block_on(encode_batch( data, @@ -68,7 +68,7 @@ fn bench_encode_compressed(c: &mut Criterion) { let lance_schema = Arc::new(lance_core::datatypes::Schema::try_from(schema.as_ref()).unwrap()); // V2_2+ required for general compression - let encoding_strategy = default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = encoding_strategy(BenchEncoding::StructuralU32); group.throughput(criterion::Throughput::Elements( (NUM_ROWS * NUM_COLUMNS) as u64, diff --git a/rust/lance-encoding/src/array_encoding.rs b/rust/lance-encoding/src/array_encoding.rs new file mode 100644 index 00000000000..a76c6a186ec --- /dev/null +++ b/rust/lance-encoding/src/array_encoding.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Encoding and decoding mechanisms described by [`crate::format::pb::ArrayEncoding`]. +//! +//! File versions decide which mechanisms to compose and accept. This module +//! contains only the reusable implementation of that persisted grammar. + +pub mod logical; +pub mod physical; +mod strategy; + +pub use strategy::ArrayFieldEncodingStrategy; diff --git a/rust/lance-encoding/src/previous/encodings/logical.rs b/rust/lance-encoding/src/array_encoding/logical.rs similarity index 100% rename from rust/lance-encoding/src/previous/encodings/logical.rs rename to rust/lance-encoding/src/array_encoding/logical.rs diff --git a/rust/lance-encoding/src/previous/encodings/logical/binary.rs b/rust/lance-encoding/src/array_encoding/logical/binary.rs similarity index 97% rename from rust/lance-encoding/src/previous/encodings/logical/binary.rs rename to rust/lance-encoding/src/array_encoding/logical/binary.rs index 00715f64511..697c1503e0a 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/logical/binary.rs @@ -19,7 +19,7 @@ use crate::{ DecodeArrayTask, FilterExpression, MessageType, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, }, - previous::decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, + decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, }; /// Wraps a varbin scheduler and uses a BinaryPageDecoder to cast @@ -41,7 +41,7 @@ impl SchedulingJob for BinarySchedulingJob<'_> { .decoders .into_iter() .map(|message| { - let decoder = message.into_legacy(); + let decoder = message.into_array(); MessageType::DecoderReady(DecoderReady { path: decoder.path, decoder: Box::new(BinaryPageDecoder { @@ -179,7 +179,7 @@ impl DecodeArrayTask for BinaryArrayDecoder { DataType::LargeUtf8 => Self::from_list_array::(arr.as_list::()), _ => panic!("Binary decoder does not support this data type"), }; - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok((result, 0)) } diff --git a/rust/lance-encoding/src/previous/encodings/logical/blob.rs b/rust/lance-encoding/src/array_encoding/logical/blob.rs similarity index 94% rename from rust/lance-encoding/src/previous/encodings/logical/blob.rs rename to rust/lance-encoding/src/array_encoding/logical/blob.rs index 13fa3b346cb..6789125ac0d 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/blob.rs +++ b/rust/lance-encoding/src/array_encoding/logical/blob.rs @@ -23,9 +23,9 @@ use crate::{ DecodeArrayTask, FilterExpression, MessageType, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, }, + decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, encoder::{EncodeTask, FieldEncoder, OutOfLineBuffers}, format::pb::{Blob, ColumnEncoding, column_encoding}, - previous::decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, repdef::RepDefBuilder, }; @@ -67,7 +67,7 @@ impl SchedulingJob for BlobFieldSchedulingJob<'_> { let next_descriptions = self.descriptions_job.schedule_next(context, priority)?; let mut priority = priority.current_priority(); let decoders = next_descriptions.decoders.into_iter().map(|decoder| { - let decoder = decoder.into_legacy(); + let decoder = decoder.into_array(); let path = decoder.path; let mut decoder = decoder.decoder; let num_rows = decoder.num_rows(); @@ -285,7 +285,7 @@ impl DecodeArrayTask for BlobArrayDecodeTask { buffer.extend_from_slice(&bytes); } let data_buf = Buffer::from_vec(buffer); - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok(( Arc::new(LargeBinaryArray::new(offsets, data_buf, self.validity)), @@ -318,7 +318,7 @@ impl BlobFieldEncoder { .nulls() .cloned() .unwrap_or(NullBuffer::new_valid(binarray.len())); - for (w, is_valid) in binarray.value_offsets().windows(2).zip(nulls.into_iter()) { + for (w, is_valid) in binarray.value_offsets().windows(2).zip(&nulls) { if is_valid { let start = w[0] as u64; let end = w[1] as u64; @@ -417,10 +417,10 @@ mod tests { use super::BlobFieldDecoder; use crate::{ EncodingsIo, + decoder::LogicalPageDecoder, format::pb::column_encoding, - previous::decoder::LogicalPageDecoder, + testing::TestEncoding, testing::{TestCases, check_round_trip_encoding_of_data, check_specific_random}, - version::LanceFileVersion, }; static BLOB_META: LazyLock> = LazyLock::new(|| { @@ -430,12 +430,20 @@ mod tests { .collect::>() }); + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_basic_blob() { + async fn test_basic_blob( + #[values(TestEncoding::Array, TestEncoding::StructuralU16)] encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let field = Field::new("", DataType::LargeBinary, false).with_metadata(BLOB_META.clone()); check_specific_random( field, - TestCases::basic().with_max_file_version(LanceFileVersion::V2_1), + TestCases::basic() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]), ) .await; } @@ -446,10 +454,10 @@ mod tests { let val2: &[u8] = &[7, 8, 9]; let array = Arc::new(LargeBinaryArray::from(vec![Some(val1), None, Some(val2)])); let test_cases = TestCases::default() - .with_max_file_version(LanceFileVersion::V2_1) + .with_array_and_u16_encodings() .with_expected_encoding("packed_struct") - .with_verify_encoding(Arc::new(|cols, version| { - if version < &LanceFileVersion::V2_1 { + .with_verify_encoding(Arc::new(|cols, encoding| { + if *encoding == TestEncoding::Array { // In 2.0 we used a special "column encoding" to mark blob fields. In 2.1 we // don't do this and just rely on the regular page encoding. assert_eq!(cols.len(), 1); @@ -465,9 +473,9 @@ mod tests { .await; let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) - .with_verify_encoding(Arc::new(|cols, version| { - if version < &LanceFileVersion::V2_1 { + .with_structural_encodings() + .with_verify_encoding(Arc::new(|cols, encoding| { + if *encoding == TestEncoding::Array { assert_eq!(cols.len(), 1); let col = &cols[0]; assert!(!matches!( diff --git a/rust/lance-encoding/src/previous/encodings/logical/list.rs b/rust/lance-encoding/src/array_encoding/logical/list.rs similarity index 98% rename from rust/lance-encoding/src/previous/encodings/logical/list.rs rename to rust/lance-encoding/src/array_encoding/logical/list.rs index 3de886a21db..d76532ab3bb 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/list.rs +++ b/rust/lance-encoding/src/array_encoding/logical/list.rs @@ -22,19 +22,19 @@ use tokio::task::JoinHandle; use crate::{ EncodingsIo, + array_encoding::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}, buffer::LanceBuffer, data::{BlockInfo, DataBlock, FixedWidthDataBlock}, decoder::{ - DecodeArrayTask, DecodeBatchScheduler, FilterExpression, ListPriorityRange, MessageType, - NextDecodeTask, PageEncoding, PriorityRange, ScheduledScanLine, SchedulerContext, + DecodeArrayTask, DecodeBatchScheduler, FieldScheduler, FilterExpression, ListPriorityRange, + LogicalPageDecoder, MessageType, NextDecodeTask, PageEncoding, PriorityRange, + ScheduledScanLine, SchedulerContext, SchedulingJob, }, - encoder::{EncodeTask, EncodedColumn, EncodedPage, FieldEncoder, OutOfLineBuffers}, - format::pb, - previous::{ - decoder::{FieldScheduler, LogicalPageDecoder, SchedulingJob}, - encoder::{ArrayEncoder, EncodedArray}, - encodings::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}, + encoder::{ + ArrayEncoder, EncodeTask, EncodedArray, EncodedColumn, EncodedPage, FieldEncoder, + OutOfLineBuffers, }, + format::pb, repdef::RepDefBuilder, utils::accumulation::AccumulationQueue, }; @@ -397,7 +397,7 @@ async fn indirect_schedule_task( for message in indirect_messages { for decoder in message.decoders { - let decoder = decoder.into_legacy(); + let decoder = decoder.into_array(); if !decoder.path.is_empty() { root_decoder.accept_child(decoder)?; } @@ -467,7 +467,7 @@ impl SchedulingJob for ListFieldSchedulingJob<'_> { .into_iter() .next() .unwrap() - .into_legacy() + .into_array() .decoder; let items_scheduler = self.scheduler.items_scheduler.clone(); @@ -690,7 +690,7 @@ impl DecodeArrayTask for ListDecodeTask { } _ => panic!("ListDecodeTask with data type that is not i32 or i64"), }; - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok((array, 0)) } @@ -961,7 +961,7 @@ impl ListOffsetsEncoder { description: PageEncoding::Legacy(description), num_rows, column_idx, - row_number: 0, // Legacy encoders do not use + row_number: 0, // V2.0 encoders do not use }) }) .map(|res_res| res_res.unwrap()) diff --git a/rust/lance-encoding/src/previous/encodings/logical/primitive.rs b/rust/lance-encoding/src/array_encoding/logical/primitive.rs similarity index 98% rename from rust/lance-encoding/src/previous/encodings/logical/primitive.rs rename to rust/lance-encoding/src/array_encoding/logical/primitive.rs index d1debf3ef33..d528ff95d19 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/primitive.rs +++ b/rust/lance-encoding/src/array_encoding/logical/primitive.rs @@ -10,10 +10,10 @@ use futures::{FutureExt, future::BoxFuture}; use log::trace; use crate::decoder::{ColumnBuffers, PageBuffers}; -use crate::previous::decoder::{FieldScheduler, LogicalPageDecoder, SchedulingJob}; -use crate::previous::encoder::ArrayEncodingStrategy; +use crate::decoder::{FieldScheduler, LogicalPageDecoder, SchedulingJob}; +use crate::encoder::ArrayEncodingStrategy; use crate::utils::accumulation::AccumulationQueue; -use crate::{data::DataBlock, previous::encodings::physical::decoder_from_array_encoding}; +use crate::{array_encoding::physical::decoder_from_array_encoding, data::DataBlock}; use lance_core::{Error, Result, datatypes::Field}; use crate::{ @@ -316,7 +316,7 @@ impl DecodeArrayTask for PrimitiveFieldDecodeTask { return Ok((new_array, 0)); } } - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok((array, 0)) } @@ -439,7 +439,7 @@ impl PrimitiveFieldEncoder { description: PageEncoding::Legacy(description), num_rows: num_values, column_idx, - row_number: 0, // legacy encoders do not use + row_number: 0, // v2.0 encoders do not use }) }) .map(|res_res| { @@ -455,6 +455,7 @@ impl PrimitiveFieldEncoder { // Creates an encode task, consuming all buffered data fn do_flush(&mut self, arrays: Vec) -> Result> { + DataBlock::validate_arrays(&arrays, &self.field.name)?; if arrays.len() == 1 { let array = arrays.into_iter().next().unwrap(); let size_bytes = array.get_buffer_memory_size(); diff --git a/rust/lance-encoding/src/previous/encodings/logical/struct.rs b/rust/lance-encoding/src/array_encoding/logical/struct.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/logical/struct.rs rename to rust/lance-encoding/src/array_encoding/logical/struct.rs index b117f74ce2b..045b3bca71d 100644 --- a/rust/lance-encoding/src/previous/encodings/logical/struct.rs +++ b/rust/lance-encoding/src/array_encoding/logical/struct.rs @@ -12,7 +12,7 @@ use crate::{ DecodeArrayTask, FilterExpression, MessageType, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, }, - previous::decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, + decoder::{DecoderReady, FieldScheduler, LogicalPageDecoder, SchedulingJob}, }; use arrow_array::{ArrayRef, StructArray}; use arrow_schema::{DataType, Field, Fields}; @@ -57,7 +57,7 @@ struct EmptyStructDecodeTask { impl DecodeArrayTask for EmptyStructDecodeTask { fn decode(self: Box) -> Result<(ArrayRef, u64)> { - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok(( Arc::new(StructArray::new_empty_fields(self.num_rows as usize, None)), @@ -607,7 +607,7 @@ impl DecodeArrayTask for SimpleStructDecodeTask { .into_iter() .map(|child| child.decode()) .collect::>>()?; - // data_size is only tracked in the v2.1 structural decode path; the legacy + // data_size is only tracked in the v2.1 structural decode path; the v2.0 array // v2.0 path does not need it so we return 0. Ok(( Arc::new(StructArray::try_new(self.child_fields, child_arrays, None)?), diff --git a/rust/lance-encoding/src/previous/encodings/physical.rs b/rust/lance-encoding/src/array_encoding/physical.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical.rs rename to rust/lance-encoding/src/array_encoding/physical.rs index a3cb0adb4a7..66af9e806df 100644 --- a/rust/lance-encoding/src/previous/encodings/physical.rs +++ b/rust/lance-encoding/src/array_encoding/physical.rs @@ -5,16 +5,16 @@ use arrow_schema::DataType; use lance_arrow::DataTypeExt; use crate::{ - buffer::LanceBuffer, - decoder::{PageBuffers, PageScheduler}, - encodings::physical::block::{CompressionConfig, CompressionScheme}, - format::pb::{self, PackedStruct}, - previous::encodings::physical::{ + array_encoding::physical::{ basic::BasicPageScheduler, binary::BinaryPageScheduler, bitmap::DenseBitmapScheduler, dictionary::DictionaryPageScheduler, fixed_size_list::FixedListScheduler, fsst::FsstPageScheduler, packed_struct::PackedStructPageScheduler, value::ValuePageScheduler, }, + buffer::LanceBuffer, + decoder::{PageBuffers, PageScheduler}, + encodings::physical::block::{CompressionConfig, CompressionScheme}, + format::pb::{self, PackedStruct}, }; pub mod basic; @@ -293,9 +293,9 @@ pub fn decoder_from_array_encoding( #[cfg(test)] mod tests { + use crate::array_encoding::physical::get_buffer_decoder; use crate::decoder::{ColumnBuffers, FileBuffers, PageBuffers}; use crate::format::pb; - use crate::previous::encodings::physical::get_buffer_decoder; #[test] fn test_get_buffer_decoder_for_compressed_buffer() { diff --git a/rust/lance-encoding/src/previous/encodings/physical/basic.rs b/rust/lance-encoding/src/array_encoding/physical/basic.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/basic.rs rename to rust/lance-encoding/src/array_encoding/physical/basic.rs index ec098c3fff4..6dd7326a97f 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/basic.rs +++ b/rust/lance-encoding/src/array_encoding/physical/basic.rs @@ -11,8 +11,8 @@ use crate::{ EncodingsIo, data::{AllNullDataBlock, BlockInfo, DataBlock, NullableDataBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, + encoder::{ArrayEncoder, EncodedArray}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; use lance_core::Result; diff --git a/rust/lance-encoding/src/previous/encodings/physical/binary.rs b/rust/lance-encoding/src/array_encoding/physical/binary.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/binary.rs rename to rust/lance-encoding/src/array_encoding/physical/binary.rs index 0bd4d96b45e..294b295def3 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/binary.rs @@ -12,17 +12,17 @@ use futures::TryFutureExt; use futures::{FutureExt, future::BoxFuture}; +use crate::array_encoding::logical::primitive::PrimitiveFieldDecoder; use crate::buffer::LanceBuffer; use crate::data::{ BlockInfo, DataBlock, FixedWidthDataBlock, NullableDataBlock, VariableWidthBlock, }; +use crate::decoder::LogicalPageDecoder; +use crate::encoder::{ArrayEncoder, EncodedArray}; use crate::encodings::physical::block::{ BufferCompressor, CompressionConfig, GeneralBufferCompressor, }; use crate::format::ProtobufUtils; -use crate::previous::decoder::LogicalPageDecoder; -use crate::previous::encoder::{ArrayEncoder, EncodedArray}; -use crate::previous::encodings::logical::primitive::PrimitiveFieldDecoder; use crate::{ EncodingsIo, decoder::{PageScheduler, PrimitivePageDecoder}, diff --git a/rust/lance-encoding/src/previous/encodings/physical/bitmap.rs b/rust/lance-encoding/src/array_encoding/physical/bitmap.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/bitmap.rs rename to rust/lance-encoding/src/array_encoding/physical/bitmap.rs index fc6d295b0f9..2168aff24e2 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/bitmap.rs +++ b/rust/lance-encoding/src/array_encoding/physical/bitmap.rs @@ -131,9 +131,9 @@ mod tests { use bytes::Bytes; use std::{collections::HashMap, sync::Arc}; + use crate::array_encoding::physical::bitmap::BitmapData; use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::decoder::PrimitivePageDecoder; - use crate::previous::encodings::physical::bitmap::BitmapData; use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; use super::BitmapDecoder; diff --git a/rust/lance-encoding/src/array_encoding/physical/bitpack.rs b/rust/lance-encoding/src/array_encoding/physical/bitpack.rs new file mode 100644 index 00000000000..a76add68a29 --- /dev/null +++ b/rust/lance-encoding/src/array_encoding/physical/bitpack.rs @@ -0,0 +1,728 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_buffer::bit_util::ceil; +use bytes::Bytes; +use futures::future::{BoxFuture, FutureExt}; +use log::trace; + +use lance_bitpacking::BitPacking; +use lance_core::{Error, Result}; + +use crate::buffer::LanceBuffer; +use crate::data::BlockInfo; +use crate::data::{DataBlock, FixedWidthDataBlock}; +use crate::decoder::{PageScheduler, PrimitivePageDecoder}; +use bytemuck::cast_slice; + +const LOG_ELEMS_PER_CHUNK: u8 = 10; +const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; + +#[derive(Debug)] +pub struct BitpackedForNonNegScheduler { + compressed_bit_width: u64, + uncompressed_bits_per_value: u64, + buffer_offset: u64, +} + +impl BitpackedForNonNegScheduler { + pub fn new( + compressed_bit_width: u64, + uncompressed_bits_per_value: u64, + buffer_offset: u64, + ) -> Self { + Self { + compressed_bit_width, + uncompressed_bits_per_value, + buffer_offset, + } + } + + fn locate_chunk_start(&self, relative_row_num: u64) -> u64 { + let chunk_size = ELEMS_PER_CHUNK * self.compressed_bit_width / 8; + self.buffer_offset + (relative_row_num / ELEMS_PER_CHUNK * chunk_size) + } + + fn locate_chunk_end(&self, relative_row_num: u64) -> u64 { + let chunk_size = ELEMS_PER_CHUNK * self.compressed_bit_width / 8; + self.buffer_offset + (relative_row_num / ELEMS_PER_CHUNK * chunk_size) + chunk_size + } +} + +impl PageScheduler for BitpackedForNonNegScheduler { + fn schedule_ranges( + &self, + ranges: &[std::ops::Range], + scheduler: &Arc, + top_level_row: u64, + ) -> BoxFuture<'static, Result>> { + assert!(!ranges.is_empty()); + + let mut byte_ranges = vec![]; + + // map one bytes to multiple ranges, one bytes has at least one range corresponding to it + let mut bytes_idx_to_range_indices = vec![]; + let first_byte_range = std::ops::Range { + start: self.locate_chunk_start(ranges[0].start), + end: self.locate_chunk_end(ranges[0].end - 1), + }; // the ranges are half-open + byte_ranges.push(first_byte_range); + bytes_idx_to_range_indices.push(vec![ranges[0].clone()]); + + for (i, range) in ranges.iter().enumerate().skip(1) { + let this_start = self.locate_chunk_start(range.start); + let this_end = self.locate_chunk_end(range.end - 1); + + // when the current range start is in the same chunk as the previous range's end, we colaesce this two bytes ranges + // when the current range start is not in the same chunk as the previous range's end, we create a new bytes range + if this_start == self.locate_chunk_start(ranges[i - 1].end - 1) { + byte_ranges.last_mut().unwrap().end = this_end; + bytes_idx_to_range_indices + .last_mut() + .unwrap() + .push(range.clone()); + } else { + byte_ranges.push(this_start..this_end); + bytes_idx_to_range_indices.push(vec![range.clone()]); + } + } + + trace!( + "Scheduling I/O for {} ranges spread across byte range {}..{}", + byte_ranges.len(), + byte_ranges[0].start, + byte_ranges.last().unwrap().end + ); + + let bytes = scheduler.submit_request(byte_ranges.clone(), top_level_row); + + // copy the necessary data from `self` to move into the async block + let compressed_bit_width = self.compressed_bit_width; + let uncompressed_bits_per_value = self.uncompressed_bits_per_value; + let num_rows = ranges.iter().map(|range| range.end - range.start).sum(); + + async move { + let bytes = bytes.await?; + let decompressed_output = bitpacked_for_non_neg_decode( + compressed_bit_width, + uncompressed_bits_per_value, + &bytes, + &bytes_idx_to_range_indices, + num_rows, + ); + Ok(Box::new(BitpackedForNonNegPageDecoder { + uncompressed_bits_per_value, + decompressed_buf: decompressed_output, + }) as Box) + } + .boxed() + } +} + +#[derive(Debug)] +struct BitpackedForNonNegPageDecoder { + // number of bits in the uncompressed value. E.g. this will be 32 for DataType::UInt32 + uncompressed_bits_per_value: u64, + + decompressed_buf: LanceBuffer, +} + +impl PrimitivePageDecoder for BitpackedForNonNegPageDecoder { + fn decode(&self, rows_to_skip: u64, num_rows: u64) -> Result { + if ![8, 16, 32, 64].contains(&self.uncompressed_bits_per_value) { + return Err(Error::invalid_input_source("BitpackedForNonNegPageDecoder should only has uncompressed_bits_per_value of 8, 16, 32, or 64".into())); + } + + let elem_size_in_bytes = self.uncompressed_bits_per_value / 8; + + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + data: self.decompressed_buf.slice_with_length( + (rows_to_skip * elem_size_in_bytes) as usize, + (num_rows * elem_size_in_bytes) as usize, + ), + bits_per_value: self.uncompressed_bits_per_value, + num_values: num_rows, + block_info: BlockInfo::new(), + })) + } +} + +macro_rules! bitpacked_decode { + ($uncompressed_type:ty, $compressed_bit_width:expr, $data:expr, $bytes_idx_to_range_indices:expr, $num_rows:expr) => {{ + let mut decompressed: Vec<$uncompressed_type> = Vec::with_capacity($num_rows as usize); + let packed_chunk_size_in_byte: usize = (ELEMS_PER_CHUNK * $compressed_bit_width) as usize / 8; + let mut decompress_chunk_buf = vec![0 as $uncompressed_type; ELEMS_PER_CHUNK as usize]; + + for (i, bytes) in $data.iter().enumerate() { + let mut ranges_idx = 0; + let mut curr_range_start = $bytes_idx_to_range_indices[i][0].start; + let mut chunk_num = 0; + + while chunk_num * packed_chunk_size_in_byte < bytes.len() { + // Copy for memory alignment + // TODO: This copy should not be needed + let chunk_in_u8: Vec = bytes[chunk_num * packed_chunk_size_in_byte..] + [..packed_chunk_size_in_byte] + .to_vec(); + chunk_num += 1; + let chunk = cast_slice(&chunk_in_u8); + unsafe { + BitPacking::unchecked_unpack( + $compressed_bit_width as usize, + chunk, + &mut decompress_chunk_buf, + ); + } + + loop { + // Case 1: All the elements after (curr_range_start % ELEMS_PER_CHUNK) inside this chunk are needed. + let elems_after_curr_range_start_in_this_chunk = + ELEMS_PER_CHUNK - curr_range_start % ELEMS_PER_CHUNK; + if curr_range_start + elems_after_curr_range_start_in_this_chunk + <= $bytes_idx_to_range_indices[i][ranges_idx].end + { + decompressed.extend_from_slice( + &decompress_chunk_buf[(curr_range_start % ELEMS_PER_CHUNK) as usize..], + ); + curr_range_start += elems_after_curr_range_start_in_this_chunk; + break; + } else { + // Case 2: Only part of the elements after (curr_range_start % ELEMS_PER_CHUNK) inside this chunk are needed. + let elems_this_range_needed_in_this_chunk = + ($bytes_idx_to_range_indices[i][ranges_idx].end - curr_range_start) + .min(ELEMS_PER_CHUNK - curr_range_start % ELEMS_PER_CHUNK); + decompressed.extend_from_slice( + &decompress_chunk_buf[(curr_range_start % ELEMS_PER_CHUNK) as usize..] + [..elems_this_range_needed_in_this_chunk as usize], + ); + if curr_range_start + elems_this_range_needed_in_this_chunk + == $bytes_idx_to_range_indices[i][ranges_idx].end + { + ranges_idx += 1; + if ranges_idx == $bytes_idx_to_range_indices[i].len() { + break; + } + curr_range_start = $bytes_idx_to_range_indices[i][ranges_idx].start; + } else { + curr_range_start += elems_this_range_needed_in_this_chunk; + } + } + } + } + } + + LanceBuffer::reinterpret_vec(decompressed) + }}; +} + +fn bitpacked_for_non_neg_decode( + compressed_bit_width: u64, + uncompressed_bits_per_value: u64, + data: &[Bytes], + bytes_idx_to_range_indices: &[Vec>], + num_rows: u64, +) -> LanceBuffer { + match uncompressed_bits_per_value { + 8 => bitpacked_decode!( + u8, + compressed_bit_width, + data, + bytes_idx_to_range_indices, + num_rows + ), + 16 => bitpacked_decode!( + u16, + compressed_bit_width, + data, + bytes_idx_to_range_indices, + num_rows + ), + 32 => bitpacked_decode!( + u32, + compressed_bit_width, + data, + bytes_idx_to_range_indices, + num_rows + ), + 64 => bitpacked_decode!( + u64, + compressed_bit_width, + data, + bytes_idx_to_range_indices, + num_rows + ), + _ => unreachable!( + "bitpacked_for_non_neg_decode only supports 8, 16, 32, 64 uncompressed_bits_per_value" + ), + } +} + +// A physical scheduler for bitpacked buffers +#[derive(Debug, Clone, Copy)] +pub struct BitpackedScheduler { + bits_per_value: u64, + uncompressed_bits_per_value: u64, + buffer_offset: u64, + signed: bool, +} + +impl BitpackedScheduler { + pub fn new( + bits_per_value: u64, + uncompressed_bits_per_value: u64, + buffer_offset: u64, + signed: bool, + ) -> Self { + Self { + bits_per_value, + uncompressed_bits_per_value, + buffer_offset, + signed, + } + } +} + +impl PageScheduler for BitpackedScheduler { + fn schedule_ranges( + &self, + ranges: &[std::ops::Range], + scheduler: &Arc, + top_level_row: u64, + ) -> BoxFuture<'static, Result>> { + let mut min = u64::MAX; + let mut max = 0; + + let mut buffer_bit_start_offsets: Vec = vec![]; + let mut buffer_bit_end_offsets: Vec> = vec![]; + let byte_ranges = ranges + .iter() + .map(|range| { + let start_byte_offset = range.start * self.bits_per_value / 8; + let mut end_byte_offset = range.end * self.bits_per_value / 8; + if !(range.end * self.bits_per_value).is_multiple_of(8) { + // If the end of the range is not byte-aligned, we need to read one more byte + end_byte_offset += 1; + + let end_bit_offset = range.end * self.bits_per_value % 8; + buffer_bit_end_offsets.push(Some(end_bit_offset as u8)); + } else { + buffer_bit_end_offsets.push(None); + } + + let start_bit_offset = range.start * self.bits_per_value % 8; + buffer_bit_start_offsets.push(start_bit_offset as u8); + + let start = self.buffer_offset + start_byte_offset; + let end = self.buffer_offset + end_byte_offset; + min = min.min(start); + max = max.max(end); + + start..end + }) + .collect::>(); + + trace!( + "Scheduling I/O for {} ranges spread across byte range {}..{}", + byte_ranges.len(), + min, + max + ); + + let bytes = scheduler.submit_request(byte_ranges, top_level_row); + + let bits_per_value = self.bits_per_value; + let uncompressed_bits_per_value = self.uncompressed_bits_per_value; + let signed = self.signed; + async move { + let bytes = bytes.await?; + Ok(Box::new(BitpackedPageDecoder { + buffer_bit_start_offsets, + buffer_bit_end_offsets, + bits_per_value, + uncompressed_bits_per_value, + signed, + data: bytes, + }) as Box) + } + .boxed() + } +} + +#[derive(Debug)] +struct BitpackedPageDecoder { + // bit offsets of the first value within each buffer + buffer_bit_start_offsets: Vec, + + // bit offsets of the last value within each buffer. e.g. if there was a buffer + // with 2 values, packed into 5 bits, this would be [Some(3)], indicating that + // the bits from the 3rd->8th bit in the last byte shouldn't be decoded. + buffer_bit_end_offsets: Vec>, + + // the number of bits used to represent a compressed value. E.g. if the max value + // in the page was 7 (0b111), then this will be 3 + bits_per_value: u64, + + // number of bits in the uncompressed value. E.g. this will be 32 for u32 + uncompressed_bits_per_value: u64, + + // whether or not to use the msb as a sign bit during decoding + signed: bool, + + data: Vec, +} + +impl PrimitivePageDecoder for BitpackedPageDecoder { + fn decode(&self, rows_to_skip: u64, num_rows: u64) -> Result { + let num_bytes = self.uncompressed_bits_per_value / 8 * num_rows; + let mut dest = vec![0; num_bytes as usize]; + + // current maximum supported bits per value = 64 + debug_assert!(self.bits_per_value <= 64); + + let mut rows_to_skip = rows_to_skip; + let mut rows_taken = 0; + let byte_len = self.uncompressed_bits_per_value / 8; + let mut dst_idx = 0; // index for current byte being written to destination buffer + + // create bit mask for source bits + let mask = u64::MAX >> (64 - self.bits_per_value); + + for i in 0..self.data.len() { + let src = &self.data[i]; + let (mut src_idx, mut src_offset) = match compute_start_offset( + rows_to_skip, + src.len(), + self.bits_per_value, + self.buffer_bit_start_offsets[i], + self.buffer_bit_end_offsets[i], + ) { + StartOffset::SkipFull(rows_to_skip_here) => { + rows_to_skip -= rows_to_skip_here; + continue; + } + StartOffset::SkipSome(buffer_start_offset) => ( + buffer_start_offset.index, + buffer_start_offset.bit_offset as u64, + ), + }; + + while src_idx < src.len() && rows_taken < num_rows { + rows_taken += 1; + let mut curr_mask = mask; // copy mask + + // current source byte being written to destination + let mut curr_src = src[src_idx] & (curr_mask << src_offset) as u8; + + // how many bits from the current source value have been written to destination + let mut src_bits_written = 0; + + // the offset within the current destination byte to write to + let mut dst_offset = 0; + + let is_negative = is_encoded_item_negative( + src, + src_idx, + src_offset, + self.bits_per_value as usize, + ); + + while src_bits_written < self.bits_per_value { + // write bits from current source byte into destination + dest[dst_idx] += (curr_src >> src_offset) << dst_offset; + let bits_written = (self.bits_per_value - src_bits_written) + .min(8 - src_offset) + .min(8 - dst_offset); + src_bits_written += bits_written; + dst_offset += bits_written; + src_offset += bits_written; + curr_mask >>= bits_written; + + if dst_offset == 8 { + dst_idx += 1; + dst_offset = 0; + } + + if src_offset == 8 { + src_idx += 1; + src_offset = 0; + if src_idx == src.len() { + break; + } + curr_src = src[src_idx] & curr_mask as u8; + } + } + + // if the type is signed, need to pad out the rest of the byte with 1s + let mut negative_padded_current_byte = false; + if self.signed && is_negative && dst_offset > 0 { + negative_padded_current_byte = true; + while dst_offset < 8 { + dest[dst_idx] |= 1 << dst_offset; + dst_offset += 1; + } + } + + // advance destination offset to the next location + // note that we don't need to do this if we wrote the full number of bits + // because source index would have been advanced by the inner loop above + if self.uncompressed_bits_per_value != self.bits_per_value { + let partial_bytes_written = ceil(self.bits_per_value as usize, 8); + + // we also want to move one location to the next location in destination, + // unless we wrote something byte-aligned in which case the logic above + // would have already advanced dst_idx + let mut to_next_byte = 1; + if self.bits_per_value.is_multiple_of(8) { + to_next_byte = 0; + } + let next_dst_idx = + dst_idx + byte_len as usize - partial_bytes_written + to_next_byte; + + // pad remaining bytes with 1 for negative signed numbers + if self.signed && is_negative { + if !negative_padded_current_byte { + dest[dst_idx] = 0xFF; + } + for i in dest.iter_mut().take(next_dst_idx).skip(dst_idx + 1) { + *i = 0xFF; + } + } + + dst_idx = next_dst_idx; + } + + // If we've reached the last byte, there may be some extra bits from the + // next value outside the range. We don't want to be taking those. + if let Some(buffer_bit_end_offset) = self.buffer_bit_end_offsets[i] + && src_idx == src.len() - 1 + && src_offset >= buffer_bit_end_offset as u64 + { + break; + } + } + } + + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::from(dest), + bits_per_value: self.uncompressed_bits_per_value, + num_values: num_rows, + block_info: BlockInfo::new(), + })) + } +} + +fn is_encoded_item_negative(src: &Bytes, src_idx: usize, src_offset: u64, num_bits: usize) -> bool { + let mut last_byte_idx = src_idx + ((src_offset as usize + num_bits) / 8); + let shift_amount = (src_offset as usize + num_bits) % 8; + let shift_amount = if shift_amount == 0 { + last_byte_idx -= 1; + 7 + } else { + shift_amount - 1 + }; + let last_byte = src[last_byte_idx]; + let sign_bit_mask = 1 << shift_amount; + let sign_bit = last_byte & sign_bit_mask; + + sign_bit > 0 +} + +#[derive(Debug, PartialEq)] +struct BufferStartOffset { + index: usize, + bit_offset: u8, +} + +#[derive(Debug, PartialEq)] +enum StartOffset { + // skip the full buffer. The value is how many rows are skipped + // by skipping the full buffer (e.g., # rows in buffer) + SkipFull(u64), + + // skip to some start offset in the buffer + SkipSome(BufferStartOffset), +} + +/// compute how far ahead in this buffer should we skip ahead and start reading +/// +/// * `rows_to_skip` - how many rows to skip +/// * `buffer_len` - length buf buffer (in bytes) +/// * `bits_per_value` - number of bits used to represent a single bitpacked value +/// * `buffer_start_bit_offset` - offset of the start of the first value within the +/// buffer's first byte +/// * `buffer_end_bit_offset` - end bit of the last value within the buffer. Can be +/// `None` if the end of the last value is byte aligned with end of buffer. +fn compute_start_offset( + rows_to_skip: u64, + buffer_len: usize, + bits_per_value: u64, + buffer_start_bit_offset: u8, + buffer_end_bit_offset: Option, +) -> StartOffset { + let rows_in_buffer = rows_in_buffer( + buffer_len, + bits_per_value, + buffer_start_bit_offset, + buffer_end_bit_offset, + ); + if rows_to_skip >= rows_in_buffer { + return StartOffset::SkipFull(rows_in_buffer); + } + + let start_bit = rows_to_skip * bits_per_value + buffer_start_bit_offset as u64; + let start_byte = start_bit / 8; + + StartOffset::SkipSome(BufferStartOffset { + index: start_byte as usize, + bit_offset: (start_bit % 8) as u8, + }) +} + +/// calculates the number of rows in a buffer +fn rows_in_buffer( + buffer_len: usize, + bits_per_value: u64, + buffer_start_bit_offset: u8, + buffer_end_bit_offset: Option, +) -> u64 { + let mut bits_in_buffer = (buffer_len * 8) as u64 - buffer_start_bit_offset as u64; + + // if the end of the last value of the buffer isn't byte aligned, subtract the + // end offset from the total number of bits in buffer + if let Some(buffer_end_bit_offset) = buffer_end_bit_offset { + bits_in_buffer -= (8 - buffer_end_bit_offset) as u64; + } + + bits_in_buffer / bits_per_value +} + +#[cfg(test)] +pub mod test { + use super::*; + use crate::BufferScheduler; + + use arrow_buffer::ArrowNativeType; + + fn fixed_width_values(block: DataBlock) -> Vec { + let DataBlock::FixedWidth(FixedWidthDataBlock { data, .. }) = block else { + panic!("expected fixed-width data"); + }; + data.borrow_to_typed_slice::().to_vec() + } + + #[test] + fn test_rows_in_buffer() { + let test_cases = vec![ + (5usize, 5u64, 0u8, None, 8u64), + (2, 3, 0, Some(5), 4), + (2, 3, 7, Some(6), 2), + ]; + + for ( + buffer_len, + bits_per_value, + buffer_start_bit_offset, + buffer_end_bit_offset, + expected, + ) in test_cases + { + let result = rows_in_buffer( + buffer_len, + bits_per_value, + buffer_start_bit_offset, + buffer_end_bit_offset, + ); + assert_eq!(expected, result); + } + } + + #[test] + fn test_compute_start_offset() { + let result = compute_start_offset(0, 5, 5, 0, None); + assert_eq!( + StartOffset::SkipSome(BufferStartOffset { + index: 0, + bit_offset: 0 + }), + result + ); + + let result = compute_start_offset(10, 5, 5, 0, None); + assert_eq!(StartOffset::SkipFull(8), result); + } + + #[test_log::test(tokio::test)] + async fn test_bitpacked_scheduler_non_byte_aligned_ranges() { + // Legacy 5-bit LSB-first encoding of values 1..=10, with a two-byte prefix. + let data = Bytes::from_static(&[0xFA, 0xCE, 0x41, 0x0C, 0x52, 0xCC, 0x41, 0x49, 0x01]); + let io: Arc = Arc::new(BufferScheduler::new(data)); + let scheduler = BitpackedScheduler::new(5, 16, 2, false); + + let decoder = scheduler + .schedule_ranges(&[1..3, 5..9], &io, 0) + .await + .unwrap(); + let decoded = decoder.decode(3, 3).unwrap(); + + // The scheduled rows are [2, 3, 6, 7, 8, 9]. Skipping across the first + // Bytes response must begin in the middle of the second response. + assert_eq!(fixed_width_values::(decoded), vec![7, 8, 9]); + } + + #[test_log::test(tokio::test)] + async fn test_bitpacked_scheduler_signed_byte_aligned() { + // Legacy 16-bit little-endian encoding of [513, -1, 4660, -32768]. + let data = Bytes::from_static(&[0x01, 0x02, 0xFF, 0xFF, 0x34, 0x12, 0x00, 0x80]); + let io: Arc = Arc::new(BufferScheduler::new(data)); + let scheduler = BitpackedScheduler::new(16, 32, 0, true); + + let decoder = scheduler.schedule_ranges(&[1..4], &io, 0).await.unwrap(); + let decoded = decoder.decode(0, 3).unwrap(); + + assert_eq!(fixed_width_values::(decoded), vec![-1, 4660, -32768]); + } + + #[test_log::test(tokio::test)] + async fn test_bitpacked_for_non_negative_scheduler_chunk_boundary() { + const BIT_WIDTH: usize = 7; + const NUM_CHUNKS: usize = 2; + const WORDS_PER_CHUNK: usize = ELEMS_PER_CHUNK as usize * BIT_WIDTH / u32::BITS as usize; + + let values = (0..ELEMS_PER_CHUNK as usize * NUM_CHUNKS) + .map(|index| ((index * 13 + 7) % 127) as u32) + .collect::>(); + let mut packed = vec![0_u32; WORDS_PER_CHUNK * NUM_CHUNKS]; + for chunk_index in 0..NUM_CHUNKS { + let value_start = chunk_index * ELEMS_PER_CHUNK as usize; + let word_start = chunk_index * WORDS_PER_CHUNK; + // SAFETY: Both slices have the exact input and output lengths required + // for one 1,024-value chunk at this bit width. + unsafe { + BitPacking::unchecked_pack( + BIT_WIDTH, + &values[value_start..value_start + ELEMS_PER_CHUNK as usize], + &mut packed[word_start..word_start + WORDS_PER_CHUNK], + ); + } + } + + let mut data = vec![0xFA, 0xCE, 0x01]; + data.extend_from_slice(cast_slice(&packed)); + let io: Arc = Arc::new(BufferScheduler::new(Bytes::from(data))); + let scheduler = BitpackedForNonNegScheduler::new(BIT_WIDTH as u64, 32, 3); + + let decoder = scheduler + .schedule_ranges(&[1020..1028, 1530..1533], &io, 0) + .await + .unwrap(); + let decoded = decoder.decode(2, 7).unwrap(); + let expected = (1022..1028) + .chain(1530..1531) + .map(|index| values[index]) + .collect::>(); + + assert_eq!(fixed_width_values::(decoded), expected); + } +} diff --git a/rust/lance-encoding/src/previous/encodings/physical/block.rs b/rust/lance-encoding/src/array_encoding/physical/block.rs similarity index 97% rename from rust/lance-encoding/src/previous/encodings/physical/block.rs rename to rust/lance-encoding/src/array_encoding/physical/block.rs index 517bce4e3a7..3bbd966e24a 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/block.rs +++ b/rust/lance-encoding/src/array_encoding/physical/block.rs @@ -5,9 +5,9 @@ use arrow_schema::DataType; use crate::{ data::{BlockInfo, DataBlock, OpaqueBlock}, + encoder::{ArrayEncoder, EncodedArray}, encodings::physical::block::{CompressedBufferEncoder, CompressionConfig, CompressionScheme}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; use lance_core::Result; diff --git a/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs b/rust/lance-encoding/src/array_encoding/physical/dictionary.rs similarity index 90% rename from rust/lance-encoding/src/previous/encodings/physical/dictionary.rs rename to rust/lance-encoding/src/array_encoding/physical/dictionary.rs index a4c01ff9ef5..611ef22fe3e 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/dictionary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/dictionary.rs @@ -16,18 +16,18 @@ use lance_arrow::DataTypeExt; use lance_core::{Error, Result}; use std::collections::HashMap; +use crate::array_encoding::logical::primitive::PrimitiveFieldDecoder; use crate::buffer::LanceBuffer; use crate::data::{ BlockInfo, DataBlock, DictionaryDataBlock, FixedWidthDataBlock, NullableDataBlock, VariableWidthBlock, }; +use crate::decoder::LogicalPageDecoder; use crate::format::ProtobufUtils; -use crate::previous::decoder::LogicalPageDecoder; -use crate::previous::encodings::logical::primitive::PrimitiveFieldDecoder; use crate::{ EncodingsIo, decoder::{PageScheduler, PrimitivePageDecoder}, - previous::encoder::{ArrayEncoder, EncodedArray}, + encoder::{ArrayEncoder, EncodedArray}, }; #[derive(Debug)] @@ -259,6 +259,16 @@ impl ArrayEncoder for AlreadyDictionaryEncoder { } _ => panic!("Expected dictionary data"), }; + let declared_key_bits = key_type.byte_width() as u64 * 8; + if dict_data.indices.bits_per_value != declared_key_bits { + return Err(Error::invalid_input(format!( + "dictionary indices use {} bits but the declared {} key type uses {} bits; the normalized dictionary has {} values", + dict_data.indices.bits_per_value, + key_type, + declared_key_bits, + dict_data.dictionary.num_values() + ))); + } let num_dictionary_items = dict_data.dictionary.num_values() as u32; let encoded_indices = self.indices_encoder.encode( @@ -412,7 +422,10 @@ mod tests { use arrow_schema::{DataType, Field}; use std::{collections::HashMap, sync::Arc, vec}; - use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; + use crate::testing::{ + TestCases, TestEncoding, check_basic_random_case, check_round_trip_encoding_of_data, + }; + use rstest::rstest; use super::encode_dict_indices_and_items; @@ -440,28 +453,29 @@ mod tests { assert_eq!(&dict_items, &expected_items); } + #[rstest] #[test_log::test(tokio::test)] - async fn test_utf8() { - let field = Field::new("", DataType::Utf8, false); - check_basic_random(field).await; - } - - #[test_log::test(tokio::test)] - async fn test_binary() { - let field = Field::new("", DataType::Binary, false); - check_basic_random(field).await; - } - - #[test_log::test(tokio::test)] - async fn test_large_binary() { - let field = Field::new("", DataType::LargeBinary, true); - check_basic_random(field).await; - } - - #[test_log::test(tokio::test)] - async fn test_large_utf8() { - let field = Field::new("", DataType::LargeUtf8, true); - check_basic_random(field).await; + async fn test_random_dictionary( + #[values( + DataType::Utf8, + DataType::Binary, + DataType::LargeBinary, + DataType::LargeUtf8 + )] + data_type: DataType, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { + let nullable = matches!(data_type, DataType::LargeBinary | DataType::LargeUtf8); + let field = Field::new("", data_type, nullable); + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] @@ -560,14 +574,25 @@ mod tests { // These tests cover the case where the input is already dictionary encoded + #[rstest] #[test_log::test(tokio::test)] - async fn test_random_dictionary_input() { + async fn test_random_dictionary_input( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let dict_field = Field::new( "", DataType::Dictionary(Box::new(DataType::UInt16), Box::new(DataType::Utf8)), false, ); - check_basic_random(dict_field).await; + check_basic_random_case(dict_field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] diff --git a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_binary.rs b/rust/lance-encoding/src/array_encoding/physical/fixed_size_binary.rs similarity index 79% rename from rust/lance-encoding/src/previous/encodings/physical/fixed_size_binary.rs rename to rust/lance-encoding/src/array_encoding/physical/fixed_size_binary.rs index 696edde8c9b..688e812fa0f 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_binary.rs +++ b/rust/lance-encoding/src/array_encoding/physical/fixed_size_binary.rs @@ -4,17 +4,14 @@ use std::sync::Arc; use arrow_buffer::ScalarBuffer; -use arrow_schema::DataType; use futures::{FutureExt, future::BoxFuture}; use lance_core::Result; use crate::{ EncodingsIo, buffer::LanceBuffer, - data::{BlockInfo, DataBlock, FixedWidthDataBlock, VariableWidthBlock}, + data::{BlockInfo, DataBlock, VariableWidthBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, - format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; /// A scheduler for fixed size binary data @@ -116,51 +113,6 @@ impl PrimitivePageDecoder for FixedSizeBinaryDecoder { } } -#[derive(Debug)] -pub struct FixedSizeBinaryEncoder { - bytes_encoder: Box, - byte_width: usize, -} - -impl FixedSizeBinaryEncoder { - pub fn new(bytes_encoder: Box, byte_width: usize) -> Self { - Self { - bytes_encoder, - byte_width, - } - } -} - -impl ArrayEncoder for FixedSizeBinaryEncoder { - fn encode( - &self, - data: DataBlock, - _data_type: &DataType, - buffer_index: &mut u32, - ) -> Result { - let bytes_data = data.as_variable_width().unwrap(); - let fixed_data = DataBlock::FixedWidth(FixedWidthDataBlock { - bits_per_value: 8 * self.byte_width as u64, - data: bytes_data.data, - num_values: bytes_data.num_values, - block_info: BlockInfo::new(), - }); - - let encoded_data = self.bytes_encoder.encode( - fixed_data, - &DataType::FixedSizeBinary(self.byte_width as i32), - buffer_index, - )?; - let encoding = - ProtobufUtils::fixed_size_binary(encoded_data.encoding, self.byte_width as u32); - - Ok(EncodedArray { - data: encoded_data.data, - encoding, - }) - } -} - #[cfg(test)] mod tests { use std::{collections::HashMap, sync::Arc}; @@ -173,34 +125,38 @@ mod tests { use arrow_data::ArrayData; use arrow_schema::{DataType, Field}; + use crate::array_encoding::physical::fixed_size_binary::FixedSizeBinaryDecoder; use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::decoder::PrimitivePageDecoder; - use crate::previous::encodings::physical::fixed_size_binary::FixedSizeBinaryDecoder; - use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; - - #[test_log::test(tokio::test)] - async fn test_fixed_size_utf8_binary() { - let field = Field::new("", DataType::Utf8, false); - // This test only generates fixed size binary arrays anyway - check_basic_random(field).await; - } - - #[test_log::test(tokio::test)] - async fn test_fixed_size_binary() { - let field = Field::new("", DataType::Binary, false); - check_basic_random(field).await; - } - - #[test_log::test(tokio::test)] - async fn test_fixed_size_large_binary() { - let field = Field::new("", DataType::LargeBinary, true); - check_basic_random(field).await; - } + use crate::testing::{ + TestCases, TestEncoding, check_basic_random_case, check_round_trip_encoding_of_data, + }; + use rstest::rstest; + #[rstest] #[test_log::test(tokio::test)] - async fn test_fixed_size_large_utf8() { - let field = Field::new("", DataType::LargeUtf8, true); - check_basic_random(field).await; + async fn test_fixed_size_random( + #[values( + DataType::Utf8, + DataType::Binary, + DataType::LargeBinary, + DataType::LargeUtf8 + )] + data_type: DataType, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { + let nullable = matches!(data_type, DataType::LargeBinary | DataType::LargeUtf8); + let field = Field::new("", data_type, nullable); + // This test only generates fixed-size binary arrays for Utf8 and Binary. + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] diff --git a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_list.rs b/rust/lance-encoding/src/array_encoding/physical/fixed_size_list.rs similarity index 87% rename from rust/lance-encoding/src/previous/encodings/physical/fixed_size_list.rs rename to rust/lance-encoding/src/array_encoding/physical/fixed_size_list.rs index e980301d117..49d2dbff19b 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/fixed_size_list.rs +++ b/rust/lance-encoding/src/array_encoding/physical/fixed_size_list.rs @@ -12,8 +12,8 @@ use crate::{ EncodingsIo, data::{DataBlock, FixedSizeListBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, + encoder::{ArrayEncoder, EncodedArray}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; /// A scheduler for fixed size lists of primitive values @@ -138,21 +138,29 @@ mod tests { use arrow_schema::{DataType, Field}; use lance_datagen::{ArrayGeneratorExt, RowCount, array, gen_array}; - use crate::{ - testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + use crate::testing::{ + TestCases, TestEncoding, check_basic_random_case, check_round_trip_encoding_of_data, }; + use rstest::rstest; - const PRIMITIVE_TYPES: &[DataType] = &[DataType::Int8, DataType::Float32, DataType::Float64]; - + #[rstest] #[test_log::test(tokio::test)] - async fn test_value_fsl_primitive() { - for data_type in PRIMITIVE_TYPES { - let inner_field = Field::new("item", data_type.clone(), true); - let data_type = DataType::FixedSizeList(Arc::new(inner_field), 16); - let field = Field::new("", data_type, false); - check_basic_random(field).await; - } + async fn test_value_fsl_primitive( + #[values(DataType::Int8, DataType::Float32, DataType::Float64)] data_type: DataType, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { + let inner_field = Field::new("item", data_type, true); + let data_type = DataType::FixedSizeList(Arc::new(inner_field), 16); + let field = Field::new("", data_type, false); + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] @@ -182,7 +190,7 @@ mod tests { .with_indices(vec![0, 1, 2]) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![list], &test_cases, HashMap::default()).await; } @@ -209,7 +217,7 @@ mod tests { .with_indices(vec![0, 1, 2]) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![list], &test_cases, HashMap::default()).await; } @@ -260,7 +268,7 @@ mod tests { .with_range(1..3) .with_indices(vec![0, 1, 2]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![outer_list], &test_cases, HashMap::default()).await; } diff --git a/rust/lance-encoding/src/previous/encodings/physical/fsst.rs b/rust/lance-encoding/src/array_encoding/physical/fsst.rs similarity index 98% rename from rust/lance-encoding/src/previous/encodings/physical/fsst.rs rename to rust/lance-encoding/src/array_encoding/physical/fsst.rs index e9bb585ed73..49440f29b3f 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/fsst.rs +++ b/rust/lance-encoding/src/array_encoding/physical/fsst.rs @@ -14,8 +14,8 @@ use crate::{ buffer::LanceBuffer, data::{BlockInfo, DataBlock, NullableDataBlock, VariableWidthBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, + encoder::{ArrayEncoder, EncodedArray}, format::ProtobufUtils, - previous::encoder::{ArrayEncoder, EncodedArray}, }; #[derive(Debug)] @@ -88,7 +88,8 @@ impl PrimitivePageDecoder for FsstPageDecoder { &offsets, &mut decompressed_bytes, &mut decompressed_offsets, - )?; + ) + .map_err(crate::encodings::physical::fsst::map_fsst_error)?; // TODO: Change PrimitivePageDecoder to use Vec instead of BytesMut // since there is no way to get BytesMut from Vec but these copies should be avoidable diff --git a/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs b/rust/lance-encoding/src/array_encoding/physical/packed_struct.rs similarity index 94% rename from rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs rename to rust/lance-encoding/src/array_encoding/physical/packed_struct.rs index 0f3a5fc3832..aadd440c937 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/packed_struct.rs +++ b/rust/lance-encoding/src/array_encoding/physical/packed_struct.rs @@ -18,7 +18,7 @@ use crate::{ buffer::LanceBuffer, data::{DataBlock, FixedWidthDataBlock, StructDataBlock}, decoder::{PageScheduler, PrimitivePageDecoder}, - previous::encoder::{ArrayEncoder, EncodedArray}, + encoder::{ArrayEncoder, EncodedArray}, }; #[derive(Debug)] @@ -262,10 +262,23 @@ mod tests { use arrow_schema::{DataType, Field, Fields}; use std::{collections::HashMap, sync::Arc, vec}; - use crate::testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}; + use crate::testing::{ + TestCases, TestEncoding, check_basic_random_case, check_round_trip_encoding_of_data, + }; + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_random_packed_struct() { + async fn test_random_packed_struct( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let data_type = DataType::Struct(Fields::from(vec![ Field::new("a", DataType::UInt64, false), Field::new("b", DataType::UInt32, false), @@ -275,7 +288,7 @@ mod tests { let field = Field::new("", data_type, false).with_metadata(metadata); - check_basic_random(field).await; + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] diff --git a/rust/lance-encoding/src/previous/encodings/physical/value.rs b/rust/lance-encoding/src/array_encoding/physical/value.rs similarity index 99% rename from rust/lance-encoding/src/previous/encodings/physical/value.rs rename to rust/lance-encoding/src/array_encoding/physical/value.rs index 92ec3240a14..9f594a00252 100644 --- a/rust/lance-encoding/src/previous/encodings/physical/value.rs +++ b/rust/lance-encoding/src/array_encoding/physical/value.rs @@ -18,7 +18,7 @@ use crate::format::ProtobufUtils; use crate::{ EncodingsIo, decoder::{PageScheduler, PrimitivePageDecoder}, - previous::encoder::{ArrayEncoder, EncodedArray}, + encoder::{ArrayEncoder, EncodedArray}, }; use lance_core::{Error, Result}; diff --git a/rust/lance-encoding/src/previous/encoder.rs b/rust/lance-encoding/src/array_encoding/strategy.rs similarity index 64% rename from rust/lance-encoding/src/previous/encoder.rs rename to rust/lance-encoding/src/array_encoding/strategy.rs index 9c314ae97e9..c2e47d0df4b 100644 --- a/rust/lance-encoding/src/previous/encoder.rs +++ b/rust/lance-encoding/src/array_encoding/strategy.rs @@ -3,23 +3,14 @@ use std::{collections::HashMap, env, hash::RandomState, sync::Arc}; -use arrow_array::{ArrayRef, UInt8Array, cast::AsArray}; +#[cfg(test)] +use arrow_array::cast::AsArray; +use arrow_array::{ArrayRef, UInt8Array}; use arrow_schema::DataType; use hyperloglogplus::{HyperLogLog, HyperLogLogPlus}; use crate::{ - buffer::LanceBuffer, - data::DataBlock, - encoder::{ColumnIndexSequence, EncodingOptions, FieldEncoder, FieldEncodingStrategy}, - encodings::{ - logical::r#struct::StructFieldEncoder, - physical::{ - block::{CompressionConfig, CompressionScheme}, - value::ValueEncoder, - }, - }, - format::pb, - previous::encodings::{ + array_encoding::{ logical::{ blob::BlobFieldEncoder, list::ListFieldEncoder, primitive::PrimitiveFieldEncoder, }, @@ -27,98 +18,50 @@ use crate::{ basic::BasicEncoder, binary::BinaryEncoder, dictionary::{AlreadyDictionaryEncoder, DictionaryEncoder}, - fixed_size_binary::FixedSizeBinaryEncoder, fixed_size_list::FslEncoder, fsst::FsstArrayEncoder, packed_struct::PackedStructEncoder, }, }, - version::LanceFileVersion, -}; - -#[cfg(feature = "bitpacking")] -use crate::previous::encodings::physical::bitpack::{ - BitpackedForNonNegArrayEncoder, compute_compressed_bit_width_for_non_neg, -}; - -use crate::constants::{ - COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, PACKED_STRUCT_LEGACY_META_KEY, - PACKED_STRUCT_META_KEY, + constants::{ + COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, PACKED_STRUCT_LEGACY_META_KEY, + PACKED_STRUCT_META_KEY, + }, + encoder::{ + ArrayEncoder, ArrayEncodingStrategy, ColumnIndexSequence, FieldEncoder, + FieldEncodingContext, FieldEncodingStrategy, + }, + encodings::{ + logical::r#struct::StructFieldEncoder, + physical::{ + block::{CompressionConfig, CompressionScheme}, + value::ValueEncoder, + }, + }, }; use lance_arrow::BLOB_META_KEY; use lance_core::datatypes::{BLOB_DESC_FIELD, Field}; +use lance_core::utils::parse::str_is_truthy; use lance_core::{Error, Result}; -/// An encoded array -/// -/// Maps to a single Arrow array -/// -/// This contains the encoded data as well as a description of the encoding that was applied which -/// can be used to decode the data later. +/// Field-to-column composition for the `pb::ArrayEncoding` grammar. #[derive(Debug)] -pub struct EncodedArray { - /// The encoded buffers - pub data: DataBlock, - /// A description of the encoding used to encode the array - pub encoding: pb::ArrayEncoding, +pub struct ArrayFieldEncodingStrategy { + array_encoding_strategy: Arc, } -impl EncodedArray { - pub fn new(data: DataBlock, encoding: pb::ArrayEncoding) -> Self { - Self { data, encoding } - } - - pub fn into_buffers(self) -> (Vec, pb::ArrayEncoding) { - let buffers = self.data.into_buffers(); - (buffers, self.encoding) - } -} - -/// Encodes data from one format to another (hopefully more compact or useful) format -/// -/// The array encoder must be Send + Sync. Encoding is always done on its own -/// thread task in the background and there could potentially be multiple encode -/// tasks running for a column at once. -pub trait ArrayEncoder: std::fmt::Debug + Send + Sync { - /// Encode data +impl ArrayFieldEncodingStrategy { + /// Create the field strategy for the `pb::ArrayEncoding` grammar. /// - /// The result should contain a description of the encoding that was chosen. - /// This can be used to decode the data later. - fn encode( - &self, - data: DataBlock, - data_type: &DataType, - buffer_index: &mut u32, - ) -> Result; -} - -/// A trait to pick which encoding strategy to use for a single page -/// of data -/// -/// Presumably, implementations will make encoding decisions based on -/// array statistics. -pub trait ArrayEncodingStrategy: Send + Sync + std::fmt::Debug { - fn create_array_encoder( - &self, - arrays: &[ArrayRef], - field: &Field, - ) -> Result>; -} - -/// The core field encoding strategy is a set of basic encodings that -/// are generally applicable in most scenarios. -#[derive(Debug)] -pub struct CoreFieldEncodingStrategy { - pub array_encoding_strategy: Arc, - pub version: LanceFileVersion, -} - -impl CoreFieldEncodingStrategy { - pub fn new(version: LanceFileVersion) -> Self { + /// ``` + /// use lance_encoding::encoder::ArrayFieldEncodingStrategy; + /// + /// let strategy = ArrayFieldEncodingStrategy::new(); + /// ``` + pub fn new() -> Self { Self { - array_encoding_strategy: Arc::new(CoreArrayEncodingStrategy::new(version)), - version, + array_encoding_strategy: Arc::new(ArrayStrategy), } } @@ -157,14 +100,20 @@ impl CoreFieldEncodingStrategy { } } -impl FieldEncodingStrategy for CoreFieldEncodingStrategy { +impl Default for ArrayFieldEncodingStrategy { + fn default() -> Self { + Self::new() + } +} + +impl FieldEncodingStrategy for ArrayFieldEncodingStrategy { fn create_field_encoder( &self, - encoding_strategy_root: &dyn FieldEncodingStrategy, field: &Field, column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, + context: &FieldEncodingContext<'_>, ) -> Result> { + let options = context.options; let data_type = field.data_type(); if Self::is_primitive_type(&data_type) { let column_index = column_index.next_column_index(field.id as u32); @@ -192,11 +141,10 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { match data_type { DataType::List(_child) | DataType::LargeList(_child) => { let list_idx = column_index.next_column_index(field.id as u32); - let inner_encoding = encoding_strategy_root.create_field_encoder( - encoding_strategy_root, + let inner_encoding = context.strategy.create_field_encoder( &field.children[0], column_index, - options, + context, )?; let offsets_encoder = Arc::new(BasicEncoder::new(Box::new(ValueEncoder::default()))); @@ -212,7 +160,7 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { let field_metadata = &field.metadata; if field_metadata .get(PACKED_STRUCT_LEGACY_META_KEY) - .map(|v| v == "true") + .map(|v| str_is_truthy(v)) .unwrap_or(field_metadata.contains_key(PACKED_STRUCT_META_KEY)) { Ok(Box::new(PrimitiveFieldEncoder::try_new( @@ -227,12 +175,9 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { .children .iter() .map(|field| { - self.create_field_encoder( - encoding_strategy_root, - field, - column_index, - options, - ) + context + .strategy + .create_field_encoder(field, column_index, context) }) .collect::>>()?; Ok(Box::new(StructFieldEncoder::new( @@ -259,39 +204,24 @@ impl FieldEncodingStrategy for CoreFieldEncodingStrategy { Err(Error::not_supported_source(format!("cannot encode a dictionary column whose value type is a logical type ({})", value_type).into())) } } - _ => todo!("Implement encoding for field {}", field), + _ => Err(Error::not_supported_source( + format!( + "Lance v2.0 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )), } } } } -/// The core array encoding strategy is a set of basic encodings that -/// are generally applicable in most scenarios. +/// Page-encoding selection for the `pb::ArrayEncoding` grammar. #[derive(Debug)] -pub struct CoreArrayEncodingStrategy { - pub version: LanceFileVersion, -} - -const BINARY_DATATYPES: [DataType; 4] = [ - DataType::Binary, - DataType::LargeBinary, - DataType::Utf8, - DataType::LargeUtf8, -]; - -impl CoreArrayEncodingStrategy { - fn new(version: LanceFileVersion) -> Self { - Self { version } - } -} - -impl CoreArrayEncodingStrategy { - fn can_use_fsst(data_type: &DataType, data_size: u64, version: LanceFileVersion) -> bool { - version >= LanceFileVersion::V2_1 - && matches!(data_type, DataType::Utf8 | DataType::Binary) - && data_size > 4 * 1024 * 1024 - } +struct ArrayStrategy; +impl ArrayStrategy { fn get_field_compression(field_meta: &HashMap) -> Option { let compression = field_meta.get(COMPRESSION_META_KEY)?; let compression_scheme = compression.parse::(); @@ -308,16 +238,14 @@ impl CoreArrayEncodingStrategy { fn default_binary_encoder( arrays: &[ArrayRef], - data_type: &DataType, field_meta: Option<&HashMap>, data_size: u64, - version: LanceFileVersion, ) -> Result> { let bin_indices_encoder = - Self::choose_array_encoder(arrays, &DataType::UInt64, data_size, false, version, None)?; + Self::choose_array_encoder(arrays, &DataType::UInt64, data_size, false, None)?; if let Some(compression) = field_meta.and_then(Self::get_field_compression) { - if compression.scheme == CompressionScheme::Fsst { + if compression.scheme() == CompressionScheme::Fsst { // User requested FSST let raw_encoder = Box::new(BinaryEncoder::try_new(bin_indices_encoder, None)?); Ok(Box::new(FsstArrayEncoder::new(raw_encoder))) @@ -329,13 +257,7 @@ impl CoreArrayEncodingStrategy { )?)) } } else { - // No user-specified compression, use FSST if we can - let bin_encoder = Box::new(BinaryEncoder::try_new(bin_indices_encoder, None)?); - if Self::can_use_fsst(data_type, data_size, version) { - Ok(Box::new(FsstArrayEncoder::new(bin_encoder))) - } else { - Ok(bin_encoder) - } + Ok(Box::new(BinaryEncoder::try_new(bin_indices_encoder, None)?)) } } @@ -344,7 +266,6 @@ impl CoreArrayEncodingStrategy { data_type: &DataType, data_size: u64, use_dict_encoding: bool, - version: LanceFileVersion, field_meta: Option<&HashMap>, ) -> Result> { match data_type { @@ -355,7 +276,6 @@ impl CoreArrayEncodingStrategy { inner.data_type(), data_size, use_dict_encoding, - version, None, )?, *dimension as u32, @@ -363,10 +283,9 @@ impl CoreArrayEncodingStrategy { } DataType::Dictionary(key_type, value_type) => { let key_encoder = - Self::choose_array_encoder(arrays, key_type, data_size, false, version, None)?; - let value_encoder = Self::choose_array_encoder( - arrays, value_type, data_size, false, version, None, - )?; + Self::choose_array_encoder(arrays, key_type, data_size, false, None)?; + let value_encoder = + Self::choose_array_encoder(arrays, value_type, data_size, false, None)?; Ok(Box::new(AlreadyDictionaryEncoder::new( key_encoder, @@ -384,7 +303,6 @@ impl CoreArrayEncodingStrategy { &DataType::UInt8, data_size, false, - version, None, )?; let dict_items_encoder = Self::choose_array_encoder( @@ -392,7 +310,6 @@ impl CoreArrayEncodingStrategy { &DataType::Utf8, data_size, false, - version, None, )?; @@ -400,31 +317,8 @@ impl CoreArrayEncodingStrategy { dict_indices_encoder, dict_items_encoder, ))) - } - // The parent datatype should be binary or utf8 to use the fixed size encoding - // The variable 'data_type' is passed through recursion so comparing with it would be incorrect - else if BINARY_DATATYPES.contains(arrays[0].data_type()) { - if let Some(byte_width) = check_fixed_size_encoding(arrays, version) { - // use FixedSizeBinaryEncoder - let bytes_encoder = Self::choose_array_encoder( - arrays, - &DataType::UInt8, - data_size, - false, - version, - None, - )?; - - Ok(Box::new(BasicEncoder::new(Box::new( - FixedSizeBinaryEncoder::new(bytes_encoder, byte_width as usize), - )))) - } else { - Self::default_binary_encoder( - arrays, data_type, field_meta, data_size, version, - ) - } } else { - Self::default_binary_encoder(arrays, data_type, field_meta, data_size, version) + Self::default_binary_encoder(arrays, field_meta, data_size) } } DataType::Struct(fields) => { @@ -438,7 +332,6 @@ impl CoreArrayEncodingStrategy { inner_datatype, data_size, use_dict_encoding, - version, None, )?; inner_encoders.push(inner_encoder); @@ -446,54 +339,16 @@ impl CoreArrayEncodingStrategy { Ok(Box::new(PackedStructEncoder::new(inner_encoders))) } - DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => { - if version >= LanceFileVersion::V2_1 && arrays[0].data_type() == data_type { - #[cfg(feature = "bitpacking")] - { - let compressed_bit_width = compute_compressed_bit_width_for_non_neg(arrays); - Ok(Box::new(BitpackedForNonNegArrayEncoder::new( - compressed_bit_width as usize, - data_type.clone(), - ))) - } - #[cfg(not(feature = "bitpacking"))] - { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } else { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } + DataType::UInt8 | DataType::UInt16 | DataType::UInt32 | DataType::UInt64 => Ok( + Box::new(BasicEncoder::new(Box::new(ValueEncoder::default()))), + ), // TODO: for signed integers, I intend to make it a cascaded encoding, a sparse array for the negative values and very wide(bit-width) values, // then a bitpacked array for the narrow(bit-width) values, I need `BitpackedForNeg` to be merged first, I am // thinking about putting this sparse array in the metadata so bitpacking remain using one page buffer only. - DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => { - if version >= LanceFileVersion::V2_1 && arrays[0].data_type() == data_type { - #[cfg(feature = "bitpacking")] - { - let compressed_bit_width = compute_compressed_bit_width_for_non_neg(arrays); - Ok(Box::new(BitpackedForNonNegArrayEncoder::new( - compressed_bit_width as usize, - data_type.clone(), - ))) - } - #[cfg(not(feature = "bitpacking"))] - { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } else { - Ok(Box::new(BasicEncoder::new(Box::new( - ValueEncoder::default(), - )))) - } - } + DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 => Ok(Box::new( + BasicEncoder::new(Box::new(ValueEncoder::default())), + )), _ => Ok(Box::new(BasicEncoder::new(Box::new( ValueEncoder::default(), )))), @@ -539,8 +394,9 @@ fn check_dict_encoding(arrays: &[ArrayRef], threshold: u64) -> bool { true } -fn check_fixed_size_encoding(arrays: &[ArrayRef], version: LanceFileVersion) -> Option { - if version < LanceFileVersion::V2_1 || arrays.is_empty() { +#[cfg(test)] +fn check_fixed_size_encoding(arrays: &[ArrayRef]) -> Option { + if arrays.is_empty() { return None; } @@ -612,7 +468,7 @@ fn check_fixed_size_encoding(arrays: &[ArrayRef], version: LanceFileVersion) -> } } -impl ArrayEncodingStrategy for CoreArrayEncodingStrategy { +impl ArrayEncodingStrategy for ArrayStrategy { fn create_array_encoder( &self, arrays: &[ArrayRef], @@ -632,7 +488,6 @@ impl ArrayEncodingStrategy for CoreArrayEncodingStrategy { data_type, data_size, use_dict_encoding, - self.version, Some(&field.metadata), ) } @@ -640,17 +495,48 @@ impl ArrayEncodingStrategy for CoreArrayEncodingStrategy { #[cfg(test)] mod tests { - use crate::constants::{COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY}; - use crate::previous::encoder::{ - ArrayEncodingStrategy, CoreArrayEncodingStrategy, check_dict_encoding, + use super::{ + ArrayEncodingStrategy, ArrayFieldEncodingStrategy, ArrayStrategy, check_dict_encoding, check_fixed_size_encoding, }; - use crate::version::LanceFileVersion; + use crate::constants::{COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY}; + use crate::encoder::{BatchEncoder, EncodingOptions}; use arrow_array::{ArrayRef, StringArray}; - use arrow_schema::Field; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + use lance_core::{Error, datatypes::Schema}; use std::collections::HashMap; use std::sync::Arc; + #[test] + fn test_unsupported_field_type_returns_error() { + let entries = Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ])), + false, + ); + let arrow_schema = ArrowSchema::new(vec![Field::new( + "attributes", + DataType::Map(Arc::new(entries), false), + true, + )]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + + let error = BatchEncoder::try_new( + &schema, + &ArrayFieldEncodingStrategy::new(), + &EncodingOptions::default(), + ) + .err() + .unwrap(); + + assert!(matches!(error, Error::NotSupported { .. })); + assert!(error.to_string().contains("attributes")); + assert!(error.to_string().contains("Map")); + } + fn is_dict_encoding_applicable(arr: Vec>, threshold: u64) -> bool { let arr = StringArray::from(arr); let arr = Arc::new(arr) as ArrayRef; @@ -691,10 +577,7 @@ mod tests { assert!(!is_dict_encoding_applicable(vec![Some("a"), Some("a")], 3)); } - fn is_fixed_size_encoding_applicable( - arrays: Vec>>, - version: LanceFileVersion, - ) -> bool { + fn is_fixed_size_encoding_applicable(arrays: Vec>>) -> bool { let mut final_arrays = Vec::new(); for arr in arrays { let arr = StringArray::from(arr); @@ -702,82 +585,78 @@ mod tests { final_arrays.push(arr); } - check_fixed_size_encoding(&final_arrays.clone(), version).is_some() + check_fixed_size_encoding(&final_arrays).is_some() } #[test] fn test_fixed_size_binary_encoding_applicable() { - assert!(!is_fixed_size_encoding_applicable( - vec![vec![]], - LanceFileVersion::V2_1 - )); - - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("a"), Some("b")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("abc"), Some("de")]], - LanceFileVersion::V2_1 - )); - - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("pqr"), None]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("pqr"), Some("")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some(""), Some("")]], - LanceFileVersion::V2_1 - )); + assert!(!is_fixed_size_encoding_applicable(vec![vec![]])); + + assert!(is_fixed_size_encoding_applicable(vec![vec![ + Some("a"), + Some("b") + ]])); + + assert!(!is_fixed_size_encoding_applicable(vec![vec![ + Some("abc"), + Some("de") + ]])); + + assert!(is_fixed_size_encoding_applicable(vec![vec![ + Some("pqr"), + None + ]])); + + assert!(!is_fixed_size_encoding_applicable(vec![vec![ + Some("pqr"), + Some("") + ]])); + + assert!(!is_fixed_size_encoding_applicable(vec![vec![ + Some(""), + Some("") + ]])); } #[test] fn test_fixed_size_binary_encoding_applicable_multiple_arrays() { - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("a"), Some("b")], vec![Some("c"), Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("ab"), Some("bc")], vec![Some("c"), Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some("ab"), None], vec![None, Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(is_fixed_size_encoding_applicable( - vec![vec![Some("a"), None], vec![None, Some("d")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![Some(""), None], vec![None, Some("")]], - LanceFileVersion::V2_1 - )); - - assert!(!is_fixed_size_encoding_applicable( - vec![vec![None, None], vec![None, None]], - LanceFileVersion::V2_1 - )); + assert!(is_fixed_size_encoding_applicable(vec![ + vec![Some("a"), Some("b")], + vec![Some("c"), Some("d")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![Some("ab"), Some("bc")], + vec![Some("c"), Some("d")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![Some("ab"), None], + vec![None, Some("d")] + ])); + + assert!(is_fixed_size_encoding_applicable(vec![ + vec![Some("a"), None], + vec![None, Some("d")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![Some(""), None], + vec![None, Some("")] + ])); + + assert!(!is_fixed_size_encoding_applicable(vec![ + vec![None, None], + vec![None, None] + ])); } fn verify_array_encoder( array: ArrayRef, field_meta: Option>, - version: LanceFileVersion, expected_encoder: &str, ) { - let encoding_strategy = CoreArrayEncodingStrategy { version }; + let encoding_strategy = ArrayStrategy; let mut field = Field::new("test_field", array.data_type().clone(), true); if let Some(field_meta) = field_meta { field.set_metadata(field_meta); @@ -797,7 +676,6 @@ mod tests { COMPRESSION_META_KEY.to_string(), "zstd".to_string(), )])), - LanceFileVersion::V2_1, "BinaryEncoder { indices_encoder: BasicEncoder { values_encoder: ValueEncoder }, compression_config: Some(CompressionConfig { scheme: Zstd, level: None }), buffer_compressor: Some(ZstdBufferCompressor { compression_level: 0 }) }", ); } @@ -810,7 +688,6 @@ mod tests { (COMPRESSION_META_KEY.to_string(), "zstd".to_string()), (COMPRESSION_LEVEL_META_KEY.to_string(), "22".to_string()), ])), - LanceFileVersion::V2_1, "BinaryEncoder { indices_encoder: BasicEncoder { values_encoder: ValueEncoder }, compression_config: Some(CompressionConfig { scheme: Zstd, level: Some(22) }), buffer_compressor: Some(ZstdBufferCompressor { compression_level: 22 }) }", ); } diff --git a/rust/lance-encoding/src/compression.rs b/rust/lance-encoding/src/compression.rs index 9051e18e8b6..8b33de6c021 100644 --- a/rust/lance-encoding/src/compression.rs +++ b/rust/lance-encoding/src/compression.rs @@ -20,7 +20,7 @@ use crate::encodings::physical::bitpacking::{InlineBitpacking, OutOfLineBitpacking}; use crate::{ buffer::LanceBuffer, - compression_config::{BssMode, CompressionFieldParams, CompressionParams}, + compression_config::{BssMode, CompressionFieldParams}, constants::{ BSS_META_KEY, COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, RLE_THRESHOLD_META_KEY, }, @@ -49,24 +49,21 @@ use crate::{ }, general::{GeneralMiniBlockCompressor, GeneralMiniBlockDecompressor}, packed::{ + PackedStructFixedPerValueDecompressor, PackedStructFixedPerValueEncoder, PackedStructFixedWidthMiniBlockDecompressor, PackedStructFixedWidthMiniBlockEncoder, PackedStructVariablePerValueDecompressor, PackedStructVariablePerValueEncoder, VariablePackedStructFieldDecoder, VariablePackedStructFieldKind, }, rle::{ - RleDecompressor, RleEncoder, RunLengthWidth, rle_encoded_size, - select_run_length_width, + RleChildDecompressor, RleDecompressor, RleEncoder, RunLengthWidth, + rle_encoded_size, select_run_length_width, }, value::{ValueDecompressor, ValueEncoder}, }, }, - format::{ - ProtobufUtils21, - pb21::{CompressiveEncoding, compressive_encoding::Compression}, - }, + format::pb21::{CompressiveEncoding, compressive_encoding::Compression}, statistics::{GetStat, Stat}, - version::LanceFileVersion, }; use arrow_array::{cast::AsArray, types::UInt64Type}; @@ -99,11 +96,11 @@ const RLE_BLOCK_HEADER_BYTES: u128 = std::mem::size_of::() as u128; /// required (e.g. when encoding metadata buffers like a dictionary or for encoding rep/def /// mini-block chunks) pub trait BlockCompressor: std::fmt::Debug + Send + Sync { - /// Compress the data into a single buffer + /// Compress the data into zero or one buffers and describe the codec used. /// - /// Also returns a description of the compression that can be used to decompress - /// when reading the data back - fn compress(&self, data: DataBlock) -> Result; + /// `None` represents a metadata-only codec. `Some` represents a physical + /// payload, including a zero-byte payload. + fn compress(&self, data: DataBlock) -> Result<(Option, CompressiveEncoding)>; } /// A trait to pick which compression to use for given data @@ -119,12 +116,12 @@ pub trait BlockCompressor: std::fmt::Debug + Send + Sync { /// used for narrow data types (both fixed and variable length) where we can /// fit many values into an 16KiB block. pub trait CompressionStrategy: Send + Sync + std::fmt::Debug { - /// Create a block compressor for the given data + /// Create a block compressor for the given data. fn create_block_compressor( &self, field: &Field, data: &DataBlock, - ) -> Result<(Box, CompressiveEncoding)>; + ) -> Result>; /// Create a per-value compressor for the given data fn create_per_value( @@ -141,12 +138,17 @@ pub trait CompressionStrategy: Send + Sync + std::fmt::Debug { ) -> Result>; } -#[derive(Debug, Default, Clone)] -pub struct DefaultCompressionStrategy { - /// User-configured compression parameters - params: CompressionParams, - /// The lance file version for compatibilities. - version: LanceFileVersion, +pub(crate) fn compress_required_block( + strategy: &dyn CompressionStrategy, + field: &Field, + data: DataBlock, +) -> Result<(LanceBuffer, CompressiveEncoding)> { + let compressor = strategy.create_block_compressor(field, &data)?; + let (payload, encoding) = compressor.compress(data)?; + let payload = payload.ok_or_else(|| { + Error::internal("Required block compressor selected a metadata-only codec".to_string()) + })?; + Ok((payload, encoding)) } fn try_bss_for_mini_block( @@ -169,11 +171,7 @@ fn try_bss_for_mini_block( None } -fn try_rle_for_mini_block( - data: &FixedWidthDataBlock, - params: &CompressionFieldParams, - use_rle_v2: bool, -) -> Option> { +fn rle_is_applicable(data: &FixedWidthDataBlock, params: &CompressionFieldParams) -> Option { let bits = data.bits_per_value; if !matches!(bits, 8 | 16 | 32 | 64) { return None; @@ -196,48 +194,93 @@ fn try_rle_for_mini_block( return None; } - let num_values = data.num_values; - let raw_bytes = (num_values as u128) * (type_size as u128); - let (run_length_width, rle_bytes) = if use_rle_v2 { - estimate_rle_width_and_size_from_data(data, Some(*MAX_MINIBLOCK_VALUES)).ok()? - } else { - ( - RunLengthWidth::U8, - estimate_rle_size_for_width_from_data( - data, - Some(*MAX_MINIBLOCK_VALUES), - RunLengthWidth::U8, - ) - .ok()?, - ) - }; + Some((data.num_values as u128) * (type_size as u128)) +} + +fn rle_beats_raw_and_bitpacking( + data: &FixedWidthDataBlock, + encoded_bytes: u128, + raw_bytes: u128, +) -> bool { + if encoded_bytes >= raw_bytes { + return false; + } - if rle_bytes < raw_bytes { - #[cfg(feature = "bitpacking")] + #[cfg(feature = "bitpacking")] + { + if let Some(bitpack_bytes) = estimate_inline_bitpacking_bytes(data).map(u128::from) + && bitpack_bytes < encoded_bytes { - if let Some(bitpack_bytes) = estimate_inline_bitpacking_bytes(data) - && (bitpack_bytes as u128) < rle_bytes - { - return None; - } + return false; } - return Some(Box::new(RleEncoder::with_run_length_width( - run_length_width, - ))); } - None + true } -fn try_rle_for_block( +fn try_fixed_u8_rle_for_mini_block( data: &FixedWidthDataBlock, - version: LanceFileVersion, params: &CompressionFieldParams, - use_rle_v2: bool, -) -> Result, CompressiveEncoding)>> { - if version < LanceFileVersion::V2_2 { - return Ok(None); +) -> Option> { + let raw_bytes = rle_is_applicable(data, params)?; + let rle_bytes = estimate_rle_size_for_width_from_data( + data, + Some(*MAX_MINIBLOCK_VALUES), + RunLengthWidth::U8, + ) + .ok()?; + rle_beats_raw_and_bitpacking(data, rle_bytes, raw_bytes) + .then(|| Box::new(RleEncoder::with_run_length_width(RunLengthWidth::U8)) as _) +} + +fn try_child_rle_for_mini_block( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, +) -> Option> { + let raw_bytes = rle_is_applicable(data, params)?; + let (run_length_width, estimated_bytes) = + estimate_rle_width_and_size_from_data(data, Some(*MAX_MINIBLOCK_VALUES)).ok()?; + let child_compression = rle_child_compression_config(params); + let encoder = || { + RleEncoder::with_child_encoding( + run_length_width, + child_compression, + child_compression, + true, + ) + }; + + #[cfg(feature = "bitpacking")] + let bitpack_bytes = estimate_inline_bitpacking_bytes(data).map(u128::from); + #[cfg(not(feature = "bitpacking"))] + let bitpack_bytes = None::; + + let should_measure_children = (child_compression.is_some() || cfg!(feature = "bitpacking")) + && (estimated_bytes >= raw_bytes + || bitpack_bytes.is_some_and(|bytes| bytes < estimated_bytes)); + let selected_bytes = if should_measure_children { + encoder().selected_payload_size(data).ok()? + } else { + estimated_bytes + }; + + rle_beats_raw_and_bitpacking(data, selected_bytes, raw_bytes).then(|| Box::new(encoder()) as _) +} + +fn rle_child_compression_config(params: &CompressionFieldParams) -> Option { + let raw = params.compression.as_deref()?; + if matches!(raw, "none" | "fsst") { + return None; } + let scheme = CompressionScheme::from_str(raw).ok()?; + Some(CompressionConfig::new(scheme, params.compression_level)) +} +fn try_rle_for_block_with_width( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, + run_length_width: RunLengthWidth, + rle_payload_bytes: u128, +) -> Result>> { let bits = data.bits_per_value; if !matches!(bits, 8 | 16 | 32 | 64) { return Ok(None); @@ -258,14 +301,6 @@ fn try_rle_for_block( } let raw_bytes = (data.num_values as u128) * ((bits / 8) as u128); - let (run_length_width, rle_payload_bytes) = if use_rle_v2 { - estimate_rle_width_and_size_from_data(data, None)? - } else { - ( - RunLengthWidth::U8, - estimate_rle_size_for_width_from_data(data, None, RunLengthWidth::U8)?, - ) - }; let rle_bytes = rle_payload_bytes.saturating_add(RLE_BLOCK_HEADER_BYTES); if rle_bytes >= raw_bytes { @@ -281,12 +316,31 @@ fn try_rle_for_block( } } - let compressor = Box::new(RleEncoder::with_run_length_width(run_length_width)); - let encoding = ProtobufUtils21::rle( - ProtobufUtils21::flat(bits, None), - ProtobufUtils21::flat(run_length_width.bits_per_value(), None), - ); - Ok(Some((compressor, encoding))) + Ok(Some(Box::new(RleEncoder::with_run_length_width( + run_length_width, + )))) +} + +fn try_fixed_u8_rle_for_block( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, +) -> Result>> { + if !matches!(data.bits_per_value, 8 | 16 | 32 | 64) { + return Ok(None); + } + let encoded_bytes = estimate_rle_size_for_width_from_data(data, None, RunLengthWidth::U8)?; + try_rle_for_block_with_width(data, params, RunLengthWidth::U8, encoded_bytes) +} + +fn try_variable_rle_for_block( + data: &FixedWidthDataBlock, + params: &CompressionFieldParams, +) -> Result>> { + if !matches!(data.bits_per_value, 8 | 16 | 32 | 64) { + return Ok(None); + } + let (width, encoded_bytes) = estimate_rle_width_and_size_from_data(data, None)?; + try_rle_for_block_with_width(data, params, width, encoded_bytes) } fn estimate_rle_width_and_size_from_data( @@ -364,9 +418,7 @@ fn estimate_inline_bitpacking_bytes(data: &FixedWidthDataBlock) -> Option { u64::try_from(estimated_bytes).ok() } -fn try_bitpack_for_block( - data: &FixedWidthDataBlock, -) -> Option<(Box, CompressiveEncoding)> { +fn try_bitpack_for_block(data: &FixedWidthDataBlock) -> Option> { let bits = data.bits_per_value; if !matches!(bits, 8 | 16 | 32 | 64) { return None; @@ -384,16 +436,9 @@ fn try_bitpack_for_block( } if data.num_values <= 1024 { - let compressor = Box::new(InlineBitpacking::new(bits)); - let encoding = ProtobufUtils21::inline_bitpacking(bits, None); - Some((compressor, encoding)) + Some(Box::new(InlineBitpacking::new(bits))) } else { - let compressor = Box::new(OutOfLineBitpacking::new(max_bit_width, bits)); - let encoding = ProtobufUtils21::out_of_line_bitpacking( - bits, - ProtobufUtils21::flat(max_bit_width, None), - ); - Some((compressor, encoding)) + Some(Box::new(OutOfLineBitpacking::new(max_bit_width, bits))) } } @@ -474,7 +519,6 @@ fn maybe_wrap_general_for_mini_block( } fn try_general_compression( - version: LanceFileVersion, field_params: &CompressionFieldParams, data: &DataBlock, ) -> Result, CompressionConfig)>> { @@ -485,9 +529,7 @@ fn try_general_compression( // User-requested compression (unused today but perhaps still used // in the future someday) - if let Some(compression_scheme) = &field_params.compression - && version >= LanceFileVersion::V2_2 - { + if let Some(compression_scheme) = &field_params.compression { let scheme: CompressionScheme = compression_scheme.parse()?; let config = CompressionConfig::new(scheme, field_params.compression_level); let compressor = Box::new(CompressedBufferEncoder::try_new(config)?); @@ -495,9 +537,7 @@ fn try_general_compression( } // Automatic compression for large blocks - if data.data_size() > MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION - && version >= LanceFileVersion::V2_2 - { + if data.data_size() > MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION { let compressor = Box::new(CompressedBufferEncoder::default()); let config = compressor.compressor.config(); return Ok(Some((compressor, config))); @@ -506,353 +546,360 @@ fn try_general_compression( Ok(None) } -impl DefaultCompressionStrategy { - /// Create a new compression strategy with default behavior - pub fn new() -> Self { - Self::default() - } +/// Parse field-level compression metadata without applying format-specific constraints. +pub fn field_metadata_params(field: &Field) -> CompressionFieldParams { + let mut params = CompressionFieldParams::default(); - /// Create a new compression strategy with user-configured parameters - pub fn with_params(params: CompressionParams) -> Self { - Self { - params, - version: LanceFileVersion::default(), - } + if let Some(compression) = field.metadata.get(COMPRESSION_META_KEY) { + params.compression = Some(compression.clone()); } - - /// Override the file version used to make compression decisions - pub fn with_version(mut self, version: LanceFileVersion) -> Self { - self.version = version; - self + if let Some(level) = field.metadata.get(COMPRESSION_LEVEL_META_KEY) { + params.compression_level = level.parse().ok(); } - - fn use_rle_v2(&self) -> bool { - self.version.resolve() >= LanceFileVersion::V2_3 + if let Some(threshold) = field.metadata.get(RLE_THRESHOLD_META_KEY) { + params.rle_threshold = threshold.parse().ok(); } - - /// Parse compression parameters from field metadata - fn parse_field_metadata(field: &Field, version: &LanceFileVersion) -> CompressionFieldParams { - let mut params = CompressionFieldParams::default(); - - // Parse compression method - if let Some(compression) = field.metadata.get(COMPRESSION_META_KEY) { - params.compression = Some(compression.clone()); - } - - // Parse compression level - if let Some(level) = field.metadata.get(COMPRESSION_LEVEL_META_KEY) { - params.compression_level = level.parse().ok(); + if let Some(bss_str) = field.metadata.get(BSS_META_KEY) { + match BssMode::parse(bss_str) { + Some(mode) => params.bss = Some(mode), + None => log::warn!("Invalid BSS mode '{}', using default", bss_str), } - - // Parse RLE threshold - if let Some(threshold) = field.metadata.get(RLE_THRESHOLD_META_KEY) { - params.rle_threshold = threshold.parse().ok(); - } - - // Parse BSS mode - if let Some(bss_str) = field.metadata.get(BSS_META_KEY) { - match BssMode::parse(bss_str) { - Some(mode) => params.bss = Some(mode), - None => { - log::warn!("Invalid BSS mode '{}', using default", bss_str); - } - } + } + if let Some(minichunk_size_str) = field + .metadata + .get(super::constants::MINICHUNK_SIZE_META_KEY) + { + if let Ok(minichunk_size) = minichunk_size_str.parse::() { + params.minichunk_size = Some(minichunk_size); + } else { + log::warn!("Invalid minichunk_size '{}', skipping", minichunk_size_str); } + } - // Parse minichunk size - if let Some(minichunk_size_str) = field - .metadata - .get(super::constants::MINICHUNK_SIZE_META_KEY) - { - if let Ok(minichunk_size) = minichunk_size_str.parse::() { - // for lance v2.1, only 32kb or smaller is supported - if minichunk_size >= 32 * 1024 && *version <= LanceFileVersion::V2_1 { - log::warn!( - "minichunk_size '{}' too large for version '{}', using default", - minichunk_size, - version - ); - } else { - params.minichunk_size = Some(minichunk_size); - } - } else { - log::warn!("Invalid minichunk_size '{}', skipping", minichunk_size_str); - } - } + params +} - params +/// Apply general-purpose compression requested for a fixed-width miniblock. +pub fn finalize_miniblock_compressor( + data: &DataBlock, + compressor: Box, + params: &CompressionFieldParams, +) -> Result> { + if matches!(data, DataBlock::FixedWidth(_)) { + maybe_wrap_general_for_mini_block(compressor, params) + } else { + Ok(compressor) } +} - fn build_fixed_width_compressor( - &self, - params: &CompressionFieldParams, - data: &FixedWidthDataBlock, - ) -> Result> { - if params.compression.as_deref() == Some("none") { - return Ok(Box::new(ValueEncoder::default())); - } +/// Honor an explicit `compression = none` request for fixed-width miniblocks. +pub fn try_uncompressed_fixed_width_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + (matches!(data, DataBlock::FixedWidth(_)) && params.compression.as_deref() == Some("none")) + .then(|| Box::new(ValueEncoder::default()) as _) +} - let base = try_bss_for_mini_block(data, params) - .or_else(|| try_rle_for_mini_block(data, params, self.use_rle_v2())) - .or_else(|| try_bitpack_for_mini_block(data)) - .unwrap_or_else(|| Box::new(ValueEncoder::default())); +/// Select byte-stream-split compression for an applicable fixed-width miniblock. +pub fn try_byte_stream_split_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_bss_for_mini_block(data, params) +} - maybe_wrap_general_for_mini_block(base, params) - } +/// Select the original fixed-u8 RLE miniblock grammar. +pub fn try_fixed_u8_rle_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_fixed_u8_rle_for_mini_block(data, params) +} - /// Build compressor based on parameters for variable-width data - fn build_variable_width_compressor( - &self, - field: &Field, - data: &VariableWidthBlock, - ) -> Result> { - let params = self.get_merged_field_params(field); - let compression = params.compression.as_deref(); - if data.bits_per_offset != 32 && data.bits_per_offset != 64 { - return Err(Error::invalid_input(format!( - "Variable width compression not supported for {} bit offsets", - data.bits_per_offset - ))); - } +/// Select variable-width RLE with independently encoded children. +pub fn try_child_rle_miniblock( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_child_rle_for_mini_block(data, params) +} - // Get statistics - let data_size = data.expect_single_stat::(Stat::DataSize); - let max_len = data.expect_single_stat::(Stat::MaxLength); +/// Select inline bitpacking for applicable fixed-width miniblocks. +pub fn try_bitpacking_miniblock(data: &DataBlock) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_bitpack_for_mini_block(data) +} - // Explicitly disable all compression. - if compression == Some("none") { - return Ok(Box::new(BinaryMiniBlockEncoder::new(params.minichunk_size))); - } +/// Store fixed-width miniblock values without a value codec. +pub fn try_raw_fixed_width_miniblock(data: &DataBlock) -> Option> { + matches!(data, DataBlock::FixedWidth(_)).then(|| Box::new(ValueEncoder::default()) as _) +} - let use_fsst = compression == Some("fsst") - || (compression.is_none() - && !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) - && max_len >= FSST_LEAST_INPUT_MAX_LENGTH - && data_size >= FSST_LEAST_INPUT_SIZE as u64); +/// Encode variable-width miniblocks with binary or FSST encoding. +pub fn try_variable_width_miniblock( + field: &Field, + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let DataBlock::VariableWidth(data) = data else { + return Ok(None); + }; + if data.bits_per_offset != 32 && data.bits_per_offset != 64 { + return Err(Error::invalid_input(format!( + "Variable width compression not supported for {} bit offsets", + data.bits_per_offset + ))); + } - // Choose base encoder (FSST or Binary) once. - let mut base_encoder: Box = if use_fsst { - Box::new(FsstMiniBlockEncoder::new(params.minichunk_size)) - } else { - Box::new(BinaryMiniBlockEncoder::new(params.minichunk_size)) - }; + let compression = params.compression.as_deref(); + let data_size = data.expect_single_stat::(Stat::DataSize); + let max_len = data.expect_single_stat::(Stat::MaxLength); + if compression == Some("none") { + return Ok(Some(Box::new(BinaryMiniBlockEncoder::new( + params.minichunk_size, + )))); + } - // Wrap with general compression when configured (except FSST / none). - if let Some(compression_scheme) = compression.filter(|scheme| *scheme != "fsst") { - let scheme: CompressionScheme = compression_scheme.parse()?; - let config = CompressionConfig::new(scheme, params.compression_level); - base_encoder = Box::new(GeneralMiniBlockCompressor::new(base_encoder, config)); - } + let use_fsst = compression == Some("fsst") + || (compression.is_none() + && !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) + && max_len >= FSST_LEAST_INPUT_MAX_LENGTH + && data_size >= FSST_LEAST_INPUT_SIZE as u64); + let mut encoder: Box = if use_fsst { + Box::new(FsstMiniBlockEncoder::new(params.minichunk_size)) + } else { + Box::new(BinaryMiniBlockEncoder::new(params.minichunk_size)) + }; + if let Some(compression_scheme) = compression.filter(|scheme| *scheme != "fsst") { + let scheme: CompressionScheme = compression_scheme.parse()?; + let config = CompressionConfig::new(scheme, params.compression_level); + encoder = Box::new(GeneralMiniBlockCompressor::new(encoder, config)); + } + Ok(Some(encoder)) +} - Ok(base_encoder) +/// Encode fixed-width packed structs as miniblocks. +pub fn try_fixed_packed_struct_miniblock( + data: &DataBlock, +) -> Result>> { + let DataBlock::Struct(data) = data else { + return Ok(None); + }; + if data.has_variable_width_child() { + return Err(Error::invalid_input( + "Packed struct mini-block encoding supports only fixed-width children", + )); } + Ok(Some(Box::new( + PackedStructFixedWidthMiniBlockEncoder::default(), + ))) +} - /// Merge user-configured parameters with field metadata - /// Field metadata has highest priority - fn get_merged_field_params(&self, field: &Field) -> CompressionFieldParams { - let mut field_params = self - .params - .get_field_params(&field.name, &field.data_type()); +/// Store fixed-size-list miniblocks without a value codec. +pub fn try_raw_fixed_size_list_miniblock(data: &DataBlock) -> Option> { + matches!(data, DataBlock::FixedSizeList(_)).then(|| Box::new(ValueEncoder::default()) as _) +} - // Override with field metadata if present (highest priority) - let metadata_params = Self::parse_field_metadata(field, &self.version); - field_params.merge(&metadata_params); +/// Store fixed-width and fixed-size-list values directly in full-zip pages. +pub fn try_raw_per_value(data: &DataBlock) -> Option> { + matches!(data, DataBlock::FixedWidth(_) | DataBlock::FixedSizeList(_)) + .then(|| Box::new(ValueEncoder::default()) as _) +} - field_params +fn validate_packed_struct(field: &Field, data: &DataBlock) -> Result> { + let DataBlock::Struct(data) = data else { + return Ok(None); + }; + if field.children.len() != data.children.len() { + return Err(Error::invalid_input( + "Struct field metadata does not match data block children", + )); } + Ok(Some(data.has_variable_width_child())) } -impl CompressionStrategy for DefaultCompressionStrategy { - fn create_miniblock_compressor( - &self, - field: &Field, - data: &DataBlock, - ) -> Result> { - match data { - DataBlock::FixedWidth(fixed_width_data) => { - let field_params = self.get_merged_field_params(field); - self.build_fixed_width_compressor(&field_params, fixed_width_data) - } - DataBlock::VariableWidth(variable_width_data) => { - self.build_variable_width_compressor(field, variable_width_data) - } - DataBlock::Struct(struct_data_block) => { - // this condition is actually checked at `PrimitiveStructuralEncoder::do_flush`, - // just being cautious here. - if struct_data_block.has_variable_width_child() { - return Err(Error::invalid_input( - "Packed struct mini-block encoding supports only fixed-width children", - )); - } - Ok(Box::new(PackedStructFixedWidthMiniBlockEncoder::default())) - } - DataBlock::FixedSizeList(_) => { - // Ideally we would compress the list items but this creates something of a challenge. - // We don't want to break lists across chunks and we need to worry about inner validity - // layers. If we try and use a compression scheme then it is unlikely to respect these - // constraints. - // - // For now, we just don't compress. In the future, we might want to consider a more - // sophisticated approach. - Ok(Box::new(ValueEncoder::default())) - } - _ => Err(Error::not_supported_source( - format!( - "Mini-block compression not yet supported for block type {}", - data.name() - ) - .into(), - )), - } +/// Reject variable-width packed structs while preserving the fixed-width error. +pub fn reject_packed_struct_per_value( + field: &Field, + data: &DataBlock, +) -> Result>> { + let Some(has_variable_child) = validate_packed_struct(field, data)? else { + return Ok(None); + }; + if has_variable_child { + return Err(Error::not_supported_source( + "Variable packed struct encoding is not enabled by the selected file format".into(), + )); } + Err(Error::invalid_input( + "Packed struct per-value compression should not be used for fixed-width-only structs", + )) +} - fn create_per_value( - &self, - field: &Field, - data: &DataBlock, - ) -> Result> { - let field_params = self.get_merged_field_params(field); - - match data { - DataBlock::FixedWidth(_) => Ok(Box::new(ValueEncoder::default())), - DataBlock::FixedSizeList(_) => Ok(Box::new(ValueEncoder::default())), - DataBlock::Struct(struct_block) => { - if field.children.len() != struct_block.children.len() { - return Err(Error::invalid_input( - "Struct field metadata does not match data block children", - )); - } - let has_variable_child = struct_block.has_variable_width_child(); - if has_variable_child { - if self.version < LanceFileVersion::V2_2 { - return Err(Error::not_supported_source("Variable packed struct encoding requires Lance file version 2.2 or later".into())); - } - Ok(Box::new(PackedStructVariablePerValueEncoder::new( - self.clone(), - field.children.clone(), - ))) - } else { - Err(Error::invalid_input( - "Packed struct per-value compression should not be used for fixed-width-only structs", - )) - } - } - DataBlock::VariableWidth(variable_width) => { - let compression = field_params.compression.as_deref(); - // Check for explicit "none" compression - if compression == Some("none") { - return Ok(Box::new(VariableEncoder::default())); - } - - let max_len = variable_width.expect_single_stat::(Stat::MaxLength); - let data_size = variable_width.expect_single_stat::(Stat::DataSize); - - // If values are very large then use block compression on a per-value basis - // - // TODO: Could maybe use median here +/// Encode variable-width packed structs with the exact strategy recursively. +pub fn try_variable_packed_struct_per_value( + strategy: Arc, + field: &Field, + data: &DataBlock, +) -> Result>> { + let Some(has_variable_child) = validate_packed_struct(field, data)? else { + return Ok(None); + }; + if !has_variable_child { + return Err(Error::invalid_input( + "Packed struct per-value compression should not be used for fixed-width-only structs", + )); + } + Ok(Some(Box::new(PackedStructVariablePerValueEncoder::new( + strategy, + field.children.clone(), + )))) +} - let per_value_requested = - compression.is_some_and(|compression| compression != "fsst"); +/// Encode all packed structs with the exact strategy recursively. +pub fn try_packed_struct_per_value( + strategy: Arc, + field: &Field, + data: &DataBlock, +) -> Result>> { + let Some(has_variable_child) = validate_packed_struct(field, data)? else { + return Ok(None); + }; + if has_variable_child { + return Ok(Some(Box::new(PackedStructVariablePerValueEncoder::new( + strategy, + field.children.clone(), + )))); + } - if (max_len > 32 * 1024 || per_value_requested) - && data_size >= FSST_LEAST_INPUT_SIZE as u64 - { - return Ok(Box::new(CompressedBufferEncoder::default())); - } + Ok(Some(Box::new(PackedStructFixedPerValueEncoder::new( + field.children.clone(), + )))) +} - if variable_width.bits_per_offset == 32 || variable_width.bits_per_offset == 64 { - let variable_compression = Box::new(VariableEncoder::default()); - let use_fsst = compression == Some("fsst") - || (compression.is_none() - && !matches!( - field.data_type(), - DataType::Binary | DataType::LargeBinary - ) - && max_len >= FSST_LEAST_INPUT_MAX_LENGTH - && data_size >= FSST_LEAST_INPUT_SIZE as u64); - - // Use FSST if explicitly requested or if data characteristics warrant it. - if use_fsst { - Ok(Box::new(FsstPerValueEncoder::new(variable_compression))) - } else { - Ok(variable_compression) - } - } else { - panic!( - "Does not support MiniBlockCompression for VariableWidth DataBlock with {} bits offsets.", - variable_width.bits_per_offset - ); - } - } - _ => unreachable!( - "Per-value compression not yet supported for block type: {}", - data.name() - ), +/// Encode variable-width values directly, with FSST or per-value compression +/// when applicable. +pub fn try_variable_width_per_value( + field: &Field, + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let DataBlock::VariableWidth(data) = data else { + return Ok(None); + }; + let compression = params.compression.as_deref(); + if compression == Some("none") { + return Ok(Some(Box::new(VariableEncoder::default()))); + } + + let max_len = data.expect_single_stat::(Stat::MaxLength); + let data_size = data.expect_single_stat::(Stat::DataSize); + let per_value_requested = compression.is_some_and(|compression| compression != "fsst"); + if (max_len > 32 * 1024 || per_value_requested) && data_size >= FSST_LEAST_INPUT_SIZE as u64 { + if compression == Some("zstd") { + let config = CompressionConfig::new(CompressionScheme::Zstd, params.compression_level); + return Ok(Some(Box::new(CompressedBufferEncoder::try_new(config)?))); } + return Ok(Some(Box::new(CompressedBufferEncoder::default()))); } - fn create_block_compressor( - &self, - field: &Field, - data: &DataBlock, - ) -> Result<(Box, CompressiveEncoding)> { - let field_params = self.get_merged_field_params(field); + if data.bits_per_offset != 32 && data.bits_per_offset != 64 { + return Err(Error::invalid_input(format!( + "Per-value compression does not support variable-width data with {}-bit offsets", + data.bits_per_offset + ))); + } + let encoder = Box::new(VariableEncoder::default()); + let use_fsst = compression == Some("fsst") + || (compression.is_none() + && !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) + && max_len >= FSST_LEAST_INPUT_MAX_LENGTH + && data_size >= FSST_LEAST_INPUT_SIZE as u64); + Ok(Some(if use_fsst { + Box::new(FsstPerValueEncoder::new(encoder)) + } else { + encoder + })) +} - match data { - DataBlock::FixedWidth(fixed_width) => { - if let Some((compressor, encoding)) = - try_rle_for_block(fixed_width, self.version, &field_params, self.use_rle_v2())? - { - return Ok((compressor, encoding)); - } - if let Some((compressor, encoding)) = try_bitpack_for_block(fixed_width) { - return Ok((compressor, encoding)); - } +/// Select fixed-u8 RLE for block compression. +pub fn try_fixed_u8_rle_block( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let DataBlock::FixedWidth(data) = data else { + return Ok(None); + }; + try_fixed_u8_rle_for_block(data, params) +} - // Try general compression (user-requested or automatic over MIN_BLOCK_SIZE_FOR_GENERAL_COMPRESSION) - if let Some((compressor, config)) = - try_general_compression(self.version, &field_params, data)? - { - let encoding = ProtobufUtils21::wrapped( - config, - ProtobufUtils21::flat(fixed_width.bits_per_value, None), - )?; - return Ok((compressor, encoding)); - } +/// Select variable-width RLE for block compression. +pub fn try_variable_rle_block( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let DataBlock::FixedWidth(data) = data else { + return Ok(None); + }; + try_variable_rle_for_block(data, params) +} - let encoder = Box::new(ValueEncoder::default()); - let encoding = ProtobufUtils21::flat(fixed_width.bits_per_value, None); - Ok((encoder, encoding)) - } - DataBlock::VariableWidth(variable_width) => { - // Try general compression - if let Some((compressor, config)) = - try_general_compression(self.version, &field_params, data)? - { - let encoding = ProtobufUtils21::wrapped( - config, - ProtobufUtils21::variable( - ProtobufUtils21::flat(variable_width.bits_per_offset as u64, None), - None, - ), - )?; - return Ok((compressor, encoding)); - } +/// Select block bitpacking for applicable fixed-width values. +pub fn try_bitpacking_block(data: &DataBlock) -> Option> { + let DataBlock::FixedWidth(data) = data else { + return None; + }; + try_bitpack_for_block(data) +} - let encoder = Box::new(VariableEncoder::default()); - let encoding = ProtobufUtils21::variable( - ProtobufUtils21::flat(variable_width.bits_per_offset as u64, None), - None, - ); - Ok((encoder, encoding)) - } - _ => unreachable!(), +/// Select explicitly requested or automatic general-purpose block compression. +pub fn try_general_block( + data: &DataBlock, + params: &CompressionFieldParams, +) -> Result>> { + let Some((compressor, _config)) = try_general_compression(params, data)? else { + return Ok(None); + }; + Ok(Some(compressor)) +} + +/// Store fixed- and variable-width block values without block compression. +pub fn try_raw_block(data: &DataBlock) -> Option> { + match data { + DataBlock::FixedWidth(_) => { + Some(Box::new(ValueEncoder::default()) as Box) } + DataBlock::VariableWidth(_) => { + Some(Box::new(VariableEncoder::default()) as Box) + } + _ => None, } } pub trait MiniBlockDecompressor: std::fmt::Debug + Send + Sync { fn decompress(&self, data: Vec, num_values: u64) -> Result; + + /// Returns the exact aggregate decoded size when it is determined solely by the value count. + /// + /// Implementations should only return `Some` when this aggregate estimate can be used by + /// [`DataBlockBuilder`](crate::data::DataBlockBuilder) to preallocate the decoded output + /// exactly. Outputs with multiple buffers or whose layout-dependent allocation cannot be + /// represented by one aggregate estimate should return `None`. + fn decoded_size_bytes(&self, _num_values: u64) -> Option { + None + } } pub trait FixedPerValueDecompressor: std::fmt::Debug + Send + Sync { @@ -862,6 +909,16 @@ pub trait FixedPerValueDecompressor: std::fmt::Debug + Send + Sync { /// /// Currently (and probably long term) this must be a multiple of 8 fn bits_per_value(&self) -> u64; + + /// Returns the exact aggregate decoded size when it is determined solely by the value count. + /// + /// Implementations should only return `Some` when this aggregate estimate can be used by + /// [`DataBlockBuilder`](crate::data::DataBlockBuilder) to preallocate the decoded output + /// exactly. Outputs with multiple buffers or whose layout-dependent allocation cannot be + /// represented by one aggregate estimate should return `None`. + fn decoded_size_bytes(&self, _num_values: u64) -> Option { + None + } } pub trait VariablePerValueDecompressor: std::fmt::Debug + Send + Sync { @@ -870,7 +927,31 @@ pub trait VariablePerValueDecompressor: std::fmt::Debug + Send + Sync { } pub trait BlockDecompressor: std::fmt::Debug + Send + Sync { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result; + fn decompress(&self, data: Option, num_values: u64) -> Result; + + /// Whether this codec consumes one payload buffer. + fn requires_payload(&self) -> bool { + true + } + + /// Inspect a block for an exact payload-derived value count when supported. + /// + /// This must not materialize the decoded values. `None` means the encoding requires an + /// external count or cannot safely prove one from this payload. + fn infer_num_values(&self, _data: &LanceBuffer) -> Result> { + Ok(None) + } +} + +pub(crate) fn require_block_payload(data: Option, codec: &str) -> Result { + data.ok_or_else(|| Error::invalid_input(format!("{codec} requires one payload"))) +} + +pub(crate) fn require_no_block_payload(data: Option, codec: &str) -> Result<()> { + if data.is_some() { + return Err(Error::invalid_input(format!("{codec} expects no payload"))); + } + Ok(()) } pub trait DecompressionStrategy: std::fmt::Debug + Send + Sync { @@ -951,13 +1032,10 @@ impl DecompressionStrategy for DefaultDecompressionStrategy { // compression. Ok(Box::new(ValueDecompressor::from_fsl(fsl))) } - Compression::Rle(rle) => { - let (bits_per_value, run_length_width) = validate_rle_compression(rle)?; - Ok(Box::new(RleDecompressor::with_run_length_width( - bits_per_value, - run_length_width, - ))) - } + Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor( + rle, + decompression_strategy, + )?)), Compression::ByteStreamSplit(bss) => { let Compression::Flat(values) = bss.values.as_ref().unwrap().compression.as_ref().unwrap() @@ -1008,6 +1086,9 @@ impl DecompressionStrategy for DefaultDecompressionStrategy { ))), Compression::Flat(flat) => Ok(Box::new(ValueDecompressor::from_flat(flat))), Compression::FixedSizeList(fsl) => Ok(Box::new(ValueDecompressor::from_fsl(fsl))), + Compression::PackedStruct(description) => Ok(Box::new( + PackedStructFixedPerValueDecompressor::new(description)?, + )), _ => todo!("fixed-per-value decompressor for {:?}", description), } } @@ -1144,19 +1225,15 @@ impl DecompressionStrategy for DefaultDecompressionStrategy { Ok(Box::new(general_decompressor)) } - Compression::Rle(rle) => { - let (bits_per_value, run_length_width) = validate_rle_compression(rle)?; - Ok(Box::new(RleDecompressor::with_run_length_width( - bits_per_value, - run_length_width, - ))) - } + Compression::Rle(rle) => Ok(Box::new(create_rle_decompressor(rle, self)?)), _ => todo!(), } } } -/// Validates RLE compression format and extracts value and run length widths. -fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<(u64, RunLengthWidth)> { +pub(crate) fn create_rle_decompressor( + rle: &crate::format::pb21::Rle, + decompression_strategy: &dyn DecompressionStrategy, +) -> Result { let values = rle .values .as_ref() @@ -1166,54 +1243,198 @@ fn validate_rle_compression(rle: &crate::format::pb21::Rle) -> Result<(u64, RunL .as_ref() .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths encoding"))?; - let values = values - .compression - .as_ref() - .ok_or_else(|| Error::invalid_input("RLE compression missing values compression"))?; - let Compression::Flat(values) = values else { - return Err(Error::invalid_input( - "RLE compression only supports flat values", - )); - }; - - let run_lengths = run_lengths - .compression - .as_ref() - .ok_or_else(|| Error::invalid_input("RLE compression missing run lengths compression"))?; - let Compression::Flat(run_lengths) = run_lengths else { - return Err(Error::invalid_input( - "RLE compression only supports flat run lengths", - )); - }; + let values = create_rle_child_decompressor(values, "values", decompression_strategy)?; + let run_lengths = + create_rle_child_decompressor(run_lengths, "run lengths", decompression_strategy)?; - if !matches!(values.bits_per_value, 8 | 16 | 32 | 64) { + if !matches!(values.bits_per_value(), 8 | 16 | 32 | 64) { return Err(Error::invalid_input(format!( "RLE compression only supports 8, 16, 32, or 64-bit values, got {}", - values.bits_per_value + values.bits_per_value() ))); } let run_length_width = - RunLengthWidth::from_bits(run_lengths.bits_per_value).ok_or_else(|| { + RunLengthWidth::from_bits(run_lengths.bits_per_value()).ok_or_else(|| { Error::invalid_input(format!( "RLE compression only supports 8, 16, or 32-bit run lengths, got {}", - run_lengths.bits_per_value + run_lengths.bits_per_value() )) })?; - Ok((values.bits_per_value, run_length_width)) + if values.requires_num_values() && run_lengths.requires_num_values() { + return Err(Error::invalid_input( + "RLE values and run lengths child encodings cannot both require the run count", + )); + } + + if values.is_identity() && run_lengths.is_identity() { + return Ok(RleDecompressor::with_run_length_width( + values.bits_per_value(), + run_length_width, + )); + } + + Ok(RleDecompressor::with_child_decompressors( + values.bits_per_value(), + run_length_width, + values, + run_lengths, + )) +} + +fn create_rle_child_decompressor( + encoding: &CompressiveEncoding, + role: &str, + decompression_strategy: &dyn DecompressionStrategy, +) -> Result { + let compression = encoding + .compression + .as_ref() + .ok_or_else(|| Error::invalid_input(format!("RLE {role} missing child compression")))?; + let (bits_per_value, requires_num_values, needs_decompressor) = + validate_rle_child_compression(compression, role)?; + + if needs_decompressor { + Ok(RleChildDecompressor::block( + bits_per_value, + decompression_strategy.create_block_decompressor(encoding)?, + requires_num_values, + )) + } else { + Ok(RleChildDecompressor::flat(bits_per_value)) + } +} + +fn validate_rle_child_compression( + compression: &Compression, + role: &str, +) -> Result<(u64, bool, bool)> { + match compression { + Compression::Flat(flat) => Ok((flat.bits_per_value, false, false)), + Compression::General(general) => { + general.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} general child missing compression config" + )) + })?; + let values = general.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!("RLE {role} general child missing inner encoding")) + })?; + let inner = values.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} general child missing inner compression" + )) + })?; + let (bits_per_value, requires_num_values) = + validate_rle_block_child_inner(inner, role)?; + Ok((bits_per_value, requires_num_values, true)) + } + Compression::OutOfLineBitpacking(out_of_line) => { + let values = out_of_line.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values encoding" + )) + })?; + let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values compression" + )) + })? + else { + return Err(Error::invalid_input(format!( + "RLE {role} bitpacking child only supports flat values" + ))); + }; + Ok((out_of_line.uncompressed_bits_per_value, true, true)) + } + other => Err(Error::invalid_input(format!( + "RLE {role} only supports flat, general, or out-of-line bitpacking child encodings, got {}", + compression_name(other) + ))), + } +} + +fn validate_rle_block_child_inner(compression: &Compression, role: &str) -> Result<(u64, bool)> { + match compression { + Compression::Flat(flat) => Ok((flat.bits_per_value, false)), + Compression::OutOfLineBitpacking(out_of_line) => { + let values = out_of_line.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values encoding" + )) + })?; + let Compression::Flat(_) = values.compression.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "RLE {role} bitpacking child missing values compression" + )) + })? + else { + return Err(Error::invalid_input(format!( + "RLE {role} bitpacking child only supports flat values" + ))); + }; + Ok((out_of_line.uncompressed_bits_per_value, true)) + } + other => Err(Error::invalid_input(format!( + "RLE {role} general child only supports flat or out-of-line bitpacking inner encodings, got {}", + compression_name(other) + ))), + } +} + +fn compression_name(compression: &Compression) -> &'static str { + match compression { + Compression::Flat(_) => "flat", + Compression::Variable(_) => "variable", + Compression::Fsst(_) => "fsst", + Compression::OutOfLineBitpacking(_) => "out-of-line bitpacking", + Compression::InlineBitpacking(_) => "inline bitpacking", + Compression::General(_) => "general", + Compression::Constant(_) => "constant", + Compression::Dictionary(_) => "dictionary", + Compression::ByteStreamSplit(_) => "byte stream split", + Compression::PackedStruct(_) => "packed struct", + Compression::FixedSizeList(_) => "fixed-size list", + Compression::VariablePackedStruct(_) => "variable packed struct", + Compression::Rle(_) => "rle", + } } #[cfg(test)] mod tests { use super::*; use crate::buffer::LanceBuffer; + use crate::compression_config::CompressionParams; use crate::data::{BlockInfo, DataBlock, FixedWidthDataBlock}; + use crate::encodings::logical::primitive::miniblock::MiniBlockCompressionContext; use crate::statistics::ComputeStat; - use crate::testing::extract_array_encoding_chain; + use crate::testing::{TestEncoding, extract_array_encoding_chain, test_compression_strategy}; use arrow_schema::{DataType, Field as ArrowField}; use std::collections::HashMap; + fn strategy(encoding: TestEncoding, params: CompressionParams) -> Arc { + test_compression_strategy(encoding, params) + } + + fn baseline_strategy(params: CompressionParams) -> Arc { + strategy(TestEncoding::StructuralU16, params) + } + + fn selected_block_codec( + strategy: &Arc, + field: &Field, + data: &DataBlock, + ) -> (Box, CompressiveEncoding) { + let compressor = strategy.create_block_compressor(field, data).unwrap(); + let (_, encoding) = compressor.compress(data.clone()).unwrap(); + (compressor, encoding) + } + + fn miniblock_context() -> MiniBlockCompressionContext { + MiniBlockCompressionContext::new(0, true, true) + } + fn create_test_field(name: &str, data_type: DataType) -> Field { let arrow_field = ArrowField::new(name, data_type, true); let mut field = Field::try_from(&arrow_field).unwrap(); @@ -1307,6 +1528,20 @@ mod tests { run_lengths.bits_per_value } + fn expect_rle_encoding(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle { + match encoding.compression.as_ref().unwrap() { + Compression::Rle(rle) => rle, + Compression::General(general) => { + let inner = general.values.as_ref().unwrap(); + let Compression::Rle(rle) = inner.compression.as_ref().unwrap() else { + panic!("expected wrapped RLE encoding"); + }; + rle + } + other => panic!("expected RLE encoding, got {}", compression_name(other)), + } + } + fn create_variable_width_block( bits_per_offset: u8, num_values: u64, @@ -1388,7 +1623,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("user_id", DataType::Int32); // Create data with low run count for RLE @@ -1420,7 +1655,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("some_column", DataType::Int32); // Create data with very low run count (50 runs for 1000 values = 0.05 ratio) let data = create_fixed_width_block_with_stats(32, 1000, 50); @@ -1435,7 +1670,7 @@ mod tests { #[test] #[cfg(feature = "bitpacking")] fn test_block_bitpacks_with_zero_segment() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("levels", DataType::UInt16); // First 1024 zeros, then 1024 ones; max bit width is 1. @@ -1450,7 +1685,7 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let (compressor, _encoding) = strategy.create_block_compressor(&field, &data).unwrap(); + let compressor = strategy.create_block_compressor(&field, &data).unwrap(); let debug_str = format!("{:?}", compressor); assert!( debug_str.contains("OutOfLineBitpacking"), @@ -1460,7 +1695,7 @@ mod tests { #[test] fn test_rle_block_accounts_for_header_before_selecting() { - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let field = create_test_field("small_constant", DataType::Int32); let values = vec![42i32; 2]; let mut block = FixedWidthDataBlock { @@ -1472,7 +1707,7 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap(); + let (compressor, encoding) = selected_block_codec(&strategy, &field, &data); assert!(format!("{compressor:?}").contains("ValueEncoder")); assert!(matches!( @@ -1484,7 +1719,7 @@ mod tests { #[test] #[cfg(feature = "bitpacking")] fn test_rle_block_prefers_bitpacking_when_smaller() { - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let field = create_test_field("levels", DataType::UInt16); let mut values = Vec::with_capacity(2048); @@ -1500,7 +1735,7 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap(); + let (compressor, encoding) = selected_block_codec(&strategy, &field, &data); let debug_str = format!("{compressor:?}"); assert!( debug_str.contains("OutOfLineBitpacking"), @@ -1515,7 +1750,7 @@ mod tests { #[test] #[cfg(feature = "bitpacking")] fn test_low_cardinality_prefers_bitpacking_over_rle() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("int_score", DataType::Int64); // Low cardinality values (3/4/5) but with moderate run count: @@ -1578,7 +1813,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("embeddings", DataType::Float32); let fixed_data = create_fixed_width_block(32, 1000); let variable_data = create_variable_width_block(32, 10, 32 * 1024); @@ -1587,12 +1822,16 @@ mod tests { let compressor = strategy .create_miniblock_compressor(&field, &fixed_data) .unwrap(); - let (_block, encoding) = compressor.compress(fixed_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), fixed_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, false); let compressor = strategy .create_miniblock_compressor(&field, &variable_data) .unwrap(); - let (_block, encoding) = compressor.compress(variable_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), variable_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, true); // Test pervalue @@ -1613,7 +1852,7 @@ mod tests { arrow_field = arrow_field.with_metadata(metadata); let field = Field::try_from(&arrow_field).unwrap(); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()); + let strategy = baseline_strategy(CompressionParams::new()); // Test miniblock let fixed_data = create_fixed_width_block(32, 1000); @@ -1622,13 +1861,17 @@ mod tests { let compressor = strategy .create_miniblock_compressor(&field, &fixed_data) .unwrap(); - let (_block, encoding) = compressor.compress(fixed_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), fixed_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, false); let compressor = strategy .create_miniblock_compressor(&field, &variable_data) .unwrap(); - let (_block, encoding) = compressor.compress(variable_data.clone()).unwrap(); + let (_block, encoding) = compressor + .compress(miniblock_context(), variable_data.clone()) + .unwrap(); check_uncompressed_encoding(&encoding, true); // Test pervalue @@ -1643,7 +1886,7 @@ mod tests { #[test] fn test_auto_fsst_disabled_for_binary_fields() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("bytes", DataType::Binary); let variable_data = create_fsst_candidate_variable_width_block(); @@ -1674,7 +1917,7 @@ mod tests { #[test] fn test_auto_fsst_still_enabled_for_utf8_fields() { - let strategy = DefaultCompressionStrategy::new(); + let strategy = baseline_strategy(CompressionParams::default()); let field = create_test_field("text", DataType::Utf8); let variable_data = create_fsst_candidate_variable_width_block(); @@ -1706,7 +1949,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("bytes", DataType::Binary); let variable_data = create_fsst_candidate_variable_width_block(); @@ -1727,6 +1970,30 @@ mod tests { ); } + #[test] + #[cfg(feature = "zstd")] + fn test_compression_level_honored_for_large_per_value() { + let mut params = CompressionParams::new(); + params.columns.insert( + "html".to_string(), + CompressionFieldParams { + compression: Some("zstd".to_string()), + compression_level: Some(19), + ..Default::default() + }, + ); + let strategy = baseline_strategy(params); + let field = create_test_field("html", DataType::Utf8); + let large = create_variable_width_block(32, 64, 40 * 1024); + + let per_value = strategy.create_per_value(&field, &large).unwrap(); + let debug = format!("{per_value:?}"); + assert!( + debug.contains("ZstdBufferCompressor") && debug.contains("compression_level: 19"), + "expected zstd level 19 to reach the per-value compressor, got: {debug}" + ); + } + #[test] fn test_parameter_merge_priority() { let mut params = CompressionParams::new(); @@ -1753,12 +2020,8 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); - // Get merged params - let merged = strategy - .params - .get_field_params("user_id", &DataType::Int32); + let merged = params.get_field_params("user_id", &DataType::Int32); // Column params should override type params assert_eq!(merged.rle_threshold, Some(0.2)); @@ -1766,9 +2029,7 @@ mod tests { assert_eq!(merged.compression_level, Some(6)); // Test field with only type params - let merged = strategy - .params - .get_field_params("other_field", &DataType::Int32); + let merged = params.get_field_params("other_field", &DataType::Int32); assert_eq!(merged.rle_threshold, Some(0.5)); assert_eq!(merged.compression, Some("lz4".to_string())); assert_eq!(merged.compression_level, None); @@ -1788,26 +2049,20 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); - // Should match pattern - let merged = strategy - .params - .get_field_params("log_messages", &DataType::Utf8); + let merged = params.get_field_params("log_messages", &DataType::Utf8); assert_eq!(merged.compression, Some("zstd".to_string())); assert_eq!(merged.compression_level, Some(6)); // Should not match - let merged = strategy - .params - .get_field_params("messages_log", &DataType::Utf8); + let merged = params.get_field_params("messages_log", &DataType::Utf8); assert_eq!(merged.compression, None); } #[test] fn test_legacy_metadata_support() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test field with "none" compression metadata let mut metadata = HashMap::new(); @@ -1826,7 +2081,7 @@ mod tests { fn test_default_behavior() { // Empty params should fall back to default behavior let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); let field = create_test_field("random_column", DataType::Int32); // Create data with high run count that won't trigger RLE (600 runs for 1000 values = 0.6 ratio) @@ -1841,7 +2096,7 @@ mod tests { #[test] fn test_field_metadata_compression() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test field with compression metadata let mut metadata = HashMap::new(); @@ -1861,7 +2116,7 @@ mod tests { #[test] fn test_field_metadata_rle_threshold() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test field with RLE threshold metadata let mut metadata = HashMap::new(); @@ -1899,15 +2154,15 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 16); } #[test] fn test_rle_v2_miniblock_keeps_u8_run_lengths_before_v2_3() { - for version in [LanceFileVersion::V2_1, LanceFileVersion::V2_2] { + for version in [TestEncoding::StructuralU16, TestEncoding::StructuralU32] { let mut metadata = HashMap::new(); metadata.insert(RLE_THRESHOLD_META_KEY.to_string(), "1.0".to_string()); metadata.insert(BSS_META_KEY.to_string(), "off".to_string()); @@ -1924,9 +2179,9 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(version); + let strategy = strategy(version, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 8, "version={version}"); } } @@ -1949,12 +2204,12 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); let debug_str = format!("{compressor:?}"); assert!(debug_str.contains("RleEncoder")); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 16); } @@ -1975,9 +2230,9 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 16); } @@ -1998,12 +2253,160 @@ mod tests { data.compute_stat(); let data = DataBlock::FixedWidth(data); - let strategy = DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_3); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); - let (_compressed, encoding) = compressor.compress(data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 8); } + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_released_versions_keep_flat_children_when_compression_requested() { + for version in [TestEncoding::StructuralU16, TestEncoding::StructuralU32] { + let mut params = CompressionParams::new(); + params.columns.insert( + "dict_indices".to_string(), + CompressionFieldParams { + compression: Some( + if cfg!(feature = "lz4") { "lz4" } else { "zstd" }.to_string(), + ), + rle_threshold: Some(1.0), + bss: Some(BssMode::Off), + ..Default::default() + }, + ); + let strategy = strategy(version, params); + let field = create_test_field("dict_indices", DataType::UInt32); + + let mut values = Vec::with_capacity(8192 * 4); + for value in 0..8192u32 { + values.extend(std::iter::repeat_n(value, 4)); + } + let mut data = FixedWidthDataBlock { + bits_per_value: 32, + data: LanceBuffer::reinterpret_vec(values), + num_values: 8192 * 4, + block_info: BlockInfo::default(), + }; + data.compute_stat(); + let data = DataBlock::FixedWidth(data); + + let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); + let rle = expect_rle_encoding(&encoding); + + assert!( + matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + Compression::Flat(_) + ), + "version={version}" + ); + assert!( + matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + Compression::Flat(_) + ), + "version={version}" + ); + } + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_strategy_bitpacks_child_values_when_smaller() { + let field = create_test_field("dict_indices", DataType::Int32); + + let mut values = Vec::with_capacity(8192 * 4); + for value in 0..8192 { + values.extend(std::iter::repeat_n(value, 4)); + } + let mut data = FixedWidthDataBlock { + bits_per_value: 32, + data: LanceBuffer::reinterpret_vec(values), + num_values: 8192 * 4, + block_info: BlockInfo::default(), + }; + data.compute_stat(); + let data = DataBlock::FixedWidth(data); + + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); + let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); + let debug_str = format!("{compressor:?}"); + assert!(debug_str.contains("RleEncoder")); + + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); + let Compression::Rle(rle) = encoding.compression.as_ref().unwrap() else { + panic!("expected RLE encoding"); + }; + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + Compression::OutOfLineBitpacking(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + Compression::Flat(_) + )); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_keeps_child_bitpacked_rle_when_smaller_than_inline_bitpacking() { + let field = create_test_field("int_score", DataType::UInt64); + + let mut values = Vec::with_capacity(8192 * 8); + for run_idx in 0..8192 { + let value = match run_idx % 3 { + 0 => 3u64, + 1 => 4u64, + _ => 5u64, + }; + values.extend(std::iter::repeat_n(value, 8)); + } + let mut data = FixedWidthDataBlock { + bits_per_value: 64, + data: LanceBuffer::reinterpret_vec(values), + num_values: 8192 * 8, + block_info: BlockInfo::default(), + }; + data.compute_stat(); + let data = DataBlock::FixedWidth(data); + + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::default()); + let compressor = strategy.create_miniblock_compressor(&field, &data).unwrap(); + let debug_str = format!("{compressor:?}"); + assert!( + debug_str.contains("RleEncoder"), + "expected RLE to beat inline bitpacking after child selection, got: {debug_str}" + ); + + let (_compressed, encoding) = compressor.compress(miniblock_context(), data).unwrap(); + let rle = expect_rle_encoding(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + Compression::OutOfLineBitpacking(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + Compression::Flat(_) + )); + } + #[test] fn test_field_metadata_override_params() { // Set up params with one configuration @@ -2019,7 +2422,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Field metadata should override params let mut metadata = HashMap::new(); @@ -2047,7 +2450,7 @@ mod tests { }, ); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Field metadata provides partial override let mut metadata = HashMap::new(); @@ -2066,7 +2469,7 @@ mod tests { #[test] fn test_bss_field_metadata() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test BSS "on" mode with compression enabled (BSS requires compression to be effective) let mut metadata = HashMap::new(); @@ -2087,7 +2490,7 @@ mod tests { #[test] fn test_bss_with_compression() { let params = CompressionParams::new(); - let strategy = DefaultCompressionStrategy::with_params(params); + let strategy = baseline_strategy(params); // Test BSS with LZ4 compression let mut metadata = HashMap::new(); @@ -2120,8 +2523,7 @@ mod tests { }, ); - let mut strategy = DefaultCompressionStrategy::with_params(params); - strategy.version = LanceFileVersion::V2_2; + let strategy = strategy(TestEncoding::StructuralU32, params); let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); let data = create_fixed_width_block(24, 1024); @@ -2132,18 +2534,17 @@ mod tests { let expected_num_values = expected_block.num_values; let num_values = expected_num_values; - let (compressor, encoding) = strategy + let compressor = strategy .create_block_compressor(&field, &data) .expect("general compression should be selected"); + let (compressed_buffer, encoding) = compressor + .compress(data.clone()) + .expect("write path general compression should succeed"); match encoding.compression.as_ref() { Some(Compression::General(_)) => {} other => panic!("expected general compression, got {:?}", other), } - let compressed_buffer = compressor - .compress(data.clone()) - .expect("write path general compression should succeed"); - let decompressor = DefaultDecompressionStrategy::default() .create_block_decompressor(&encoding) .expect("general block decompressor should be created"); @@ -2162,6 +2563,60 @@ mod tests { } } + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn assert_general_block_preserves_compression_level( + compression: &str, + expected_scheme: crate::format::pb21::CompressionScheme, + compression_level: Option, + ) { + let mut params = CompressionParams::new(); + params.columns.insert( + "dict_values".to_string(), + CompressionFieldParams { + compression: Some(compression.to_string()), + compression_level, + ..Default::default() + }, + ); + let strategy = strategy(TestEncoding::StructuralU32, params); + let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); + let data = create_fixed_width_block(24, 1024); + + let compressor = strategy.create_block_compressor(&field, &data).unwrap(); + let (_, encoding) = compressor.compress(data).unwrap(); + let Some(Compression::General(general)) = encoding.compression.as_ref() else { + panic!("expected general compression"); + }; + + assert_eq!( + general.compression.as_ref(), + Some(&crate::format::pb21::BufferCompression { + scheme: expected_scheme as i32, + level: compression_level, + }) + ); + } + + #[test] + #[cfg(feature = "zstd")] + fn test_general_block_preserves_absent_zstd_level() { + assert_general_block_preserves_compression_level( + "zstd", + crate::format::pb21::CompressionScheme::CompressionAlgorithmZstd, + None, + ); + } + + #[test] + #[cfg(feature = "lz4")] + fn test_general_block_preserves_explicit_lz4_level() { + assert_general_block_preserves_compression_level( + "lz4", + crate::format::pb21::CompressionScheme::CompressionAlgorithmLz4, + Some(7), + ); + } + #[test] #[cfg(any(feature = "lz4", feature = "zstd"))] fn test_general_compression_not_selected_for_v2_1_even_if_requested() { @@ -2174,14 +2629,14 @@ mod tests { }, ); - let strategy = - DefaultCompressionStrategy::with_params(params).with_version(LanceFileVersion::V2_1); + let strategy = strategy(TestEncoding::StructuralU16, params); let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); let data = create_fixed_width_block(24, 1024); - let (_compressor, encoding) = strategy + let compressor = strategy .create_block_compressor(&field, &data) .expect("block compressor selection should succeed"); + let (_, encoding) = compressor.compress(data).unwrap(); assert!( !matches!(encoding.compression.as_ref(), Some(Compression::General(_))), @@ -2200,8 +2655,7 @@ mod tests { }, ); - let strategy = - DefaultCompressionStrategy::with_params(params).with_version(LanceFileVersion::V2_2); + let strategy = strategy(TestEncoding::StructuralU32, params); let field = create_test_field("dict_values", DataType::FixedSizeBinary(3)); let data = create_fixed_width_block(24, 20_000); @@ -2210,9 +2664,10 @@ mod tests { "test requires block size above automatic general compression threshold" ); - let (_compressor, encoding) = strategy + let compressor = strategy .create_block_compressor(&field, &data) .expect("block compressor selection should succeed"); + let (_, encoding) = compressor.compress(data).unwrap(); assert!( !matches!(encoding.compression.as_ref(), Some(Compression::General(_))), @@ -2233,12 +2688,10 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_3); - let (compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap(); + let strategy = strategy(TestEncoding::StructuralSparse, CompressionParams::new()); + let compressor = strategy.create_block_compressor(&field, &data).unwrap(); + let (compressed, encoding) = compressor.compress(data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 32); - - let compressed = compressor.compress(data).unwrap(); let decompressor = DefaultDecompressionStrategy::default() .create_block_decompressor(&encoding) .unwrap(); @@ -2268,9 +2721,9 @@ mod tests { block.compute_stat(); let data = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_2); - let (_compressor, encoding) = strategy.create_block_compressor(&field, &data).unwrap(); + let strategy = strategy(TestEncoding::StructuralU32, CompressionParams::new()); + let compressor = strategy.create_block_compressor(&field, &data).unwrap(); + let (_, encoding) = compressor.compress(data).unwrap(); assert_eq!(rle_run_length_bits(&encoding), 8); } @@ -2298,10 +2751,9 @@ mod tests { let data_block = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_2); + let strategy = strategy(TestEncoding::StructuralU32, CompressionParams::new()); - let (compressor, _) = strategy + let compressor = strategy .create_block_compressor(&field, &data_block) .unwrap(); @@ -2333,10 +2785,9 @@ mod tests { let data_block = DataBlock::FixedWidth(block); - let strategy = DefaultCompressionStrategy::with_params(CompressionParams::new()) - .with_version(LanceFileVersion::V2_1); + let strategy = strategy(TestEncoding::StructuralU16, CompressionParams::new()); - let (compressor, _) = strategy + let compressor = strategy .create_block_compressor(&field, &data_block) .unwrap(); diff --git a/rust/lance-encoding/src/constants.rs b/rust/lance-encoding/src/constants.rs index c95b587a532..0bd31d676cc 100644 --- a/rust/lance-encoding/src/constants.rs +++ b/rust/lance-encoding/src/constants.rs @@ -55,6 +55,8 @@ pub const STRUCTURAL_ENCODING_META_KEY: &str = "lance-encoding:structural-encodi pub const STRUCTURAL_ENCODING_MINIBLOCK: &str = "miniblock"; /// Value for fullzip structural encoding pub const STRUCTURAL_ENCODING_FULLZIP: &str = "fullzip"; +/// Value for sparse structural encoding +pub const STRUCTURAL_ENCODING_SPARSE: &str = "sparse"; // Byte stream split metadata keys /// Metadata key for byte stream split encoding configuration diff --git a/rust/lance-encoding/src/data.rs b/rust/lance-encoding/src/data.rs index c4539214706..84ea0300f17 100644 --- a/rust/lance-encoding/src/data.rs +++ b/rust/lance-encoding/src/data.rs @@ -101,15 +101,11 @@ pub struct NullableDataBlock { } impl NullableDataBlock { - fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { let nulls = self.nulls.into_buffer(); - let data = self.data.into_arrow(data_type, validate)?.into_builder(); + let data = self.data.into_arrow_impl(data_type, true)?.into_builder(); let data = data.null_bit_buffer(Some(nulls)); - if validate { - Ok(data.build()?) - } else { - Ok(unsafe { data.build_unchecked() }) - } + Ok(data.build()?) } fn into_buffers(self) -> Vec { @@ -173,7 +169,7 @@ impl FixedWidthDataBlock { self, data_type: DataType, num_values: u64, - validate: bool, + _validate: bool, ) -> Result { // Booleans expanded for full-zip (bits_per_value==8, one byte each) need re-packing to // Arrow's bit-packed format. @@ -190,16 +186,16 @@ impl FixedWidthDataBlock { .add_buffer(data_buffer) .len(num_values as usize) .null_count(0); - if validate { - Ok(builder.build()?) - } else { - Ok(unsafe { builder.build_unchecked() }) - } + Ok(builder.build()?) } - pub fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + /// Convert this block into Arrow data with full layout validation. + /// + /// The `validate` argument is retained for API compatibility. Conversion is + /// always validated because callers can construct this public type directly. + pub fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { let root_num_values = self.num_values; - self.do_into_arrow(data_type, root_num_values, validate) + self.do_into_arrow(data_type, root_num_values, true) } pub fn into_buffers(self) -> Vec { @@ -221,7 +217,7 @@ impl FixedWidthDataBlock { } #[derive(Debug)] -pub struct VariableWidthDataBlockBuilder { +struct VariableWidthDataBlockBuilder { offsets: Vec, bytes: Vec, } @@ -236,28 +232,54 @@ impl VariableWidthDataBlockBuilder { } impl DataBlockBuilderImpl for VariableWidthDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { + let block = data_block.as_variable_width_ref().unwrap(); + block.validate_offsets_for_append::(selection) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let block = data_block.as_variable_width_ref().unwrap(); - assert!(block.bits_per_offset == T::get_byte_width() as u8 * 8); + debug_assert_eq!(block.bits_per_offset, T::get_byte_width() as u8 * 8); let offsets = block.offsets.borrow_to_typed_view::(); let start_offset = offsets[selection.start as usize]; let end_offset = offsets[selection.end as usize]; - let mut previous_len = self.bytes.len(); + let selected_data_len = end_offset.as_usize() - start_offset.as_usize(); + let new_data_len = self + .bytes + .len() + .checked_add(selected_data_len) + .ok_or_else(|| { + Error::not_supported_source( + "appending variable-width data would overflow usize".into(), + ) + })?; + if T::from_usize(new_data_len).is_none() { + return Err(Error::not_supported_source( + format!( + "appending variable-width data would require {} bytes, which exceeds the \ + capacity of {}-bit offsets", + new_data_len, + T::get_byte_width() * 8 + ) + .into(), + )); + } + let previous_len = self.bytes.len(); self.bytes .extend_from_slice(&block.data[start_offset.as_usize()..end_offset.as_usize()]); self.offsets.extend( - offsets[selection.start as usize..selection.end as usize] + offsets[selection.start as usize + 1..=selection.end as usize] .iter() - .zip(&offsets[selection.start as usize + 1..=selection.end as usize]) - .map(|(¤t, &next)| { - let this_value_len = next - current; - previous_len += this_value_len.as_usize(); - T::from_usize(previous_len).unwrap() + .map(|&offset| { + let rebased_offset = + previous_len + (offset.as_usize() - start_offset.as_usize()); + T::from_usize(rebased_offset).unwrap() }), ); + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -286,12 +308,17 @@ impl BitmapDataBlockBuilder { } impl DataBlockBuilderImpl for BitmapDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, _data_block: &DataBlock, _selection: &Range) -> Result<()> { + Ok(()) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let bitmap_blk = data_block.as_fixed_width_ref().unwrap(); self.values.append_packed_range( selection.start as usize..selection.end as usize, &bitmap_blk.data, ); + Ok(()) } fn finish(mut self: Box) -> DataBlock { @@ -326,12 +353,17 @@ impl FixedWidthDataBlockBuilder { } impl DataBlockBuilderImpl for FixedWidthDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, _data_block: &DataBlock, _selection: &Range) -> Result<()> { + Ok(()) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let block = data_block.as_fixed_width_ref().unwrap(); assert_eq!(self.bits_per_value, block.bits_per_value); let start = selection.start as usize * self.bytes_per_value as usize; let end = selection.end as usize * self.bytes_per_value as usize; self.values.extend_from_slice(&block.data[start..end]); + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -357,11 +389,20 @@ impl StructDataBlockBuilder { } impl DataBlockBuilderImpl for StructDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { + let data_block = data_block.as_struct_ref().unwrap(); + for i in 0..self.children.len() { + self.children[i].validate_append(&data_block.children[i], selection)?; + } + Ok(()) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let data_block = data_block.as_struct_ref().unwrap(); for i in 0..self.children.len() { - self.children[i].append(&data_block.children[i], selection.clone()); + self.children[i].append_validated(&data_block.children[i], selection.clone())?; } + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -384,8 +425,13 @@ struct AllNullDataBlockBuilder { } impl DataBlockBuilderImpl for AllNullDataBlockBuilder { - fn append(&mut self, _data_block: &DataBlock, selection: Range) { + fn validate_append(&self, _data_block: &DataBlock, _selection: &Range) -> Result<()> { + Ok(()) + } + + fn append_validated(&mut self, _data_block: &DataBlock, selection: Range) -> Result<()> { self.num_values += selection.end - selection.start; + Ok(()) } fn finish(self: Box) -> DataBlock { @@ -460,13 +506,13 @@ impl FixedSizeListBlock { } } - fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { let num_values = self.num_values(); let builder = match &data_type { DataType::FixedSizeList(child_field, _) => { let child_data = self .child - .into_arrow(child_field.data_type().clone(), validate)?; + .into_arrow_impl(child_field.data_type().clone(), true)?; ArrayDataBuilder::new(data_type) .add_child_data(child_data) .len(num_values as usize) @@ -474,11 +520,7 @@ impl FixedSizeListBlock { } _ => panic!("Expected FixedSizeList data type and got {:?}", data_type), }; - if validate { - Ok(builder.build()?) - } else { - Ok(unsafe { builder.build_unchecked() }) - } + Ok(builder.build()?) } fn into_buffers(self) -> Vec { @@ -503,10 +545,16 @@ impl FixedSizeListBlockBuilder { } impl DataBlockBuilderImpl for FixedSizeListBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { let selection = selection.start * self.dimension..selection.end * self.dimension; let fsl = data_block.as_fixed_size_list_ref().unwrap(); - self.inner.append(fsl.child.as_ref(), selection); + self.inner.validate_append(fsl.child.as_ref(), &selection) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + let selection = selection.start * self.dimension..selection.end * self.dimension; + let fsl = data_block.as_fixed_size_list_ref().unwrap(); + self.inner.append_validated(fsl.child.as_ref(), selection) } fn finish(self: Box) -> DataBlock { @@ -534,15 +582,23 @@ impl NullableDataBlockBuilder { } impl DataBlockBuilderImpl for NullableDataBlockBuilder { - fn append(&mut self, data_block: &DataBlock, selection: Range) { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()> { + let nullable = data_block.as_nullable_ref().unwrap(); + self.inner + .validate_append(nullable.data.as_ref(), selection) + } + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { let nullable = data_block.as_nullable_ref().unwrap(); + self.inner + .append_validated(nullable.data.as_ref(), selection.clone())?; let bool_buf = BooleanBuffer::new( nullable.nulls.clone().into_buffer(), selection.start as usize, (selection.end - selection.start) as usize, ); self.validity.append_buffer(&bool_buf); - self.inner.append(nullable.data.as_ref(), selection); + Ok(()) } fn finish(mut self: Box) -> DataBlock { @@ -588,20 +644,315 @@ pub struct VariableWidthBlock { pub block_info: BlockInfo, } +/// Proof that a [`VariableWidthBlock`] satisfies the Arrow layout contract for +/// its target data type (offsets buffer long enough, offsets monotonic and +/// within the data buffer, values valid UTF-8 where required). +/// +/// Only [`VariableWidthBlock::validate_layout`] can construct it, which ties the +/// unchecked Arrow build below to an actual validation pass instead of a +/// caller-controlled flag. +struct ValidVariableWidthLayout; + impl VariableWidthBlock { - fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + fn append_error(&self, selection: &Range, detail: impl std::fmt::Display) -> Error { + Error::corrupt_file_named( + "variable width data block", + format!( + "cannot append offsets for selection {}..{}: {} (num_values: {}, \ + bits_per_offset: {}, offsets buffer size: {} bytes, data buffer size: {} bytes)", + selection.start, + selection.end, + detail, + self.num_values, + self.bits_per_offset, + self.offsets.len(), + self.data.len(), + ), + ) + } + + fn validate_offsets_for_append(&self, selection: &Range) -> Result<()> + where + T: OffsetSizeTrait + bytemuck::Pod, + { + let expected_bits_per_offset = T::get_byte_width() as u8 * 8; + if self.bits_per_offset != expected_bits_per_offset { + return Err(self.append_error( + selection, + format!( + "expected {}-bit offsets but found {}-bit offsets", + expected_bits_per_offset, self.bits_per_offset + ), + )); + } + let offset_size = std::mem::size_of::(); + if !self.offsets.len().is_multiple_of(offset_size) { + return Err(self.append_error( + selection, + format!( + "offsets buffer length {} is not a multiple of the {}-byte offset width", + self.offsets.len(), + offset_size + ), + )); + } + if selection.start > selection.end || selection.end > self.num_values { + return Err( + self.append_error(selection, "selection is outside the block's value range") + ); + } + let selection_start = usize::try_from(selection.start) + .map_err(|_| self.append_error(selection, "selection start does not fit in usize"))?; + let selection_end = usize::try_from(selection.end) + .map_err(|_| self.append_error(selection, "selection end does not fit in usize"))?; + let offsets = self.offsets.borrow_to_typed_view::(); + if selection_end >= offsets.len() { + return Err(self.append_error( + selection, + format!( + "selection requires offset {} but the buffer holds {} offsets", + selection_end, + offsets.len() + ), + )); + } + let selected_offsets = &offsets[selection_start..=selection_end]; + if let Some(detail) = + Self::offset_violation_detail(selected_offsets, self.data.len(), selection_start) + { + return Err(self.append_error(selection, detail)); + } + Ok(()) + } + + // The offsets buffer comes straight from file bytes, so an unchecked build would + // let a corrupt file smuggle out-of-bounds offsets into an Arrow array whose + // consumers then read (or crash on) memory outside the data buffer. This + // boundary therefore always validates the layout, ignoring the optional + // `validate` flag. Lance validates the common layouts itself (a branchless + // scan, measurably cheaper than Arrow's element-wise checked build) and only + // falls back to Arrow's checked build for the cold cases. + fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { + let Some(expected_bits_per_offset) = Self::expected_bits_per_offset(&data_type) else { + // Not an [offsets, bytes] layout we know how to prove; let Arrow + // check it. + return self.into_arrow_checked(data_type); + }; + if self.bits_per_offset != expected_bits_per_offset { + return Err(self.layout_error( + &data_type, + format!( + "expected {}-bit offsets but got {}-bit offsets", + expected_bits_per_offset, self.bits_per_offset + ), + )); + } + if self.num_values == 0 { + // Cold path; Arrow handles the empty-offsets special cases. + return self.into_arrow_checked(data_type); + } + let proof = self.validate_layout(&data_type)?; + Ok(self.into_arrow_unchecked(data_type, proof)) + } + + /// The offset width Arrow mandates for `data_type`, or `None` if the type + /// does not use the `[offsets, bytes]` layout this block represents. + fn expected_bits_per_offset(data_type: &DataType) -> Option { + match data_type { + DataType::Binary | DataType::Utf8 => Some(32), + DataType::LargeBinary | DataType::LargeUtf8 => Some(64), + _ => None, + } + } + + fn layout_error(&self, data_type: &DataType, detail: impl std::fmt::Display) -> Error { + Self::format_layout_error( + data_type, + detail, + self.num_values, + self.bits_per_offset, + self.offsets.len(), + self.data.len(), + ) + } + + fn format_layout_error( + data_type: &DataType, + detail: impl std::fmt::Display, + num_values: u64, + bits_per_offset: u8, + offsets_size: usize, + data_size: usize, + ) -> Error { + Error::corrupt_file_named( + "variable width data block", + format!( + "invalid variable-width layout for {}: {} (num_values: {}, bits_per_offset: {}, \ + offsets buffer size: {} bytes, data buffer size: {} bytes)", + data_type, detail, num_values, bits_per_offset, offsets_size, data_size, + ), + ) + } + + fn validate_layout(&self, data_type: &DataType) -> Result { + let bytes_per_offset = (self.bits_per_offset / 8) as u64; + let required_bytes = self + .num_values + .checked_add(1) + .and_then(|num_offsets| num_offsets.checked_mul(bytes_per_offset)) + .ok_or_else(|| self.layout_error(data_type, "offsets buffer size overflows"))?; + if (self.offsets.len() as u64) < required_bytes { + return Err(self.layout_error( + data_type, + format!( + "offsets buffer must hold at least {} offsets ({} bytes)", + self.num_values + 1, + required_bytes + ), + )); + } + let validate_utf8 = matches!(data_type, DataType::Utf8 | DataType::LargeUtf8); + match self.bits_per_offset { + 32 => self.validate_offsets_and_values::(data_type, validate_utf8), + 64 => self.validate_offsets_and_values::(data_type, validate_utf8), + other => Err(self.layout_error( + data_type, + format!("unsupported offset width: {} bits", other), + )), + } + } + + fn validate_offsets_and_values( + &self, + data_type: &DataType, + validate_utf8: bool, + ) -> Result { + let num_offsets = self.num_values as usize + 1; + // Slice before borrowing: the buffer may carry padding that is not a + // multiple of the offset width. + let offsets = self + .offsets + .slice_with_length(0, num_offsets * std::mem::size_of::()); + let offsets = offsets.borrow_to_typed_slice::(); + let offsets: &[T] = offsets.as_ref(); + let data = self.data.as_ref(); + + if let Some(detail) = Self::offset_violation_detail(offsets, data.len(), 0) { + return Err(self.layout_error(data_type, detail)); + } + + if validate_utf8 { + let (first, last) = (offsets[0].as_usize(), offsets[num_offsets - 1].as_usize()); + let values = std::str::from_utf8(&data[first..last]) + .map_err(|utf8_err| self.layout_error(data_type, utf8_err))?; + let mut on_char_boundaries = true; + for &offset in offsets { + on_char_boundaries &= values.is_char_boundary(offset.as_usize() - first); + } + if !on_char_boundaries { + // Cold path: rescan to pinpoint the offending offset. + let position = offsets + .iter() + .position(|offset| !values.is_char_boundary(offset.as_usize() - first)) + .expect("the fast scan found a non-boundary offset"); + return Err(self.layout_error( + data_type, + format!("offset at position {position} splits a UTF-8 character"), + )); + } + } + + Ok(ValidVariableWidthLayout) + } + + fn offset_violation_detail( + offsets: &[T], + data_size: usize, + position_base: usize, + ) -> Option { + // A monotonic sequence with a non-negative first offset and an + // in-bounds last offset is entirely within [0, data_size]. Keep this + // valid path branchless so it vectorizes, and only rescan on failure. + let mut is_monotonic = true; + for window in offsets.windows(2) { + is_monotonic &= window[0] <= window[1]; + } + let first = offsets[0]; + let last = offsets[offsets.len() - 1]; + let bounds_ok = + first >= T::usize_as(0) && last.to_usize().is_some_and(|last| last <= data_size); + if is_monotonic && bounds_ok { + return None; + } + + for (relative_position, window) in offsets.windows(2).enumerate() { + if window[0] > window[1] { + let position = position_base + relative_position + 1; + return Some(format!( + "non-monotonic offset at position {}: {:?} decreases from {:?}", + position, window[1], window[0] + )); + } + } + for (relative_position, offset) in offsets.iter().enumerate() { + let position = position_base + relative_position; + match offset.to_usize() { + None => { + return Some(format!( + "offset at position {} is negative: {:?}", + position, offset + )); + } + Some(offset) if offset > data_size => { + return Some(format!( + "offset at position {} is out of bounds: {} > {}", + position, offset, data_size + )); + } + Some(_) => {} + } + } + Some("offsets failed validation".to_string()) + } + + fn into_arrow_checked(self, data_type: DataType) -> Result { + let num_values = self.num_values; + let bits_per_offset = self.bits_per_offset; + let offsets_size = self.offsets.len(); + let data_size = self.data.len(); + let builder = self.into_arrow_builder(data_type.clone()); + builder.build().map_err(|arrow_err| { + Self::format_layout_error( + &data_type, + arrow_err, + num_values, + bits_per_offset, + offsets_size, + data_size, + ) + }) + } + + fn into_arrow_unchecked( + self, + data_type: DataType, + _proof: ValidVariableWidthLayout, + ) -> ArrayData { + let builder = self.into_arrow_builder(data_type); + // SAFETY: `_proof` witnesses that `validate_layout` proved this block + // satisfies the Arrow layout contract for `data_type`. + unsafe { builder.build_unchecked() } + } + + fn into_arrow_builder(self, data_type: DataType) -> ArrayDataBuilder { + let num_values = self.num_values; let data_buffer = self.data.into_buffer(); let offsets_buffer = self.offsets.into_buffer(); - let builder = ArrayDataBuilder::new(data_type) + ArrayDataBuilder::new(data_type) .add_buffer(offsets_buffer) .add_buffer(data_buffer) - .len(self.num_values as usize) - .null_count(0); - if validate { - Ok(builder.build()?) - } else { - Ok(unsafe { builder.build_unchecked() }) - } + .len(num_values as usize) + .null_count(0) } fn into_buffers(self) -> Vec { @@ -634,12 +985,12 @@ pub struct StructDataBlock { } impl StructDataBlock { - fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { if let DataType::Struct(fields) = &data_type { let mut builder = ArrayDataBuilder::new(DataType::Struct(fields.clone())); let mut num_rows = 0; for (field, child) in fields.iter().zip(self.children) { - let child_data = child.into_arrow(field.data_type().clone(), validate)?; + let child_data = child.into_arrow_impl(field.data_type().clone(), true)?; num_rows = child_data.len(); builder = builder.add_child_data(child_data); } @@ -655,11 +1006,7 @@ impl StructDataBlock { }; let builder = builder.len(num_rows); - if validate { - Ok(builder.build()?) - } else { - Ok(unsafe { builder.build_unchecked() }) - } + Ok(builder.build()?) } else { Err(Error::internal(format!( "Expected Struct, got {:?}", @@ -727,12 +1074,11 @@ impl DictionaryDataBlock { let indices = self.indices.data.borrow_to_typed_slice::(); let indices = indices.as_ref(); - indices - .iter() - .map(|idx| idx.to_usize().unwrap() as u64) - .for_each(|idx| { - data_builder.append(&self.dictionary, idx..idx + 1); - }); + let selections = indices.iter().map(|idx| { + let idx = idx.to_usize().unwrap() as u64; + idx..idx + 1 + }); + data_builder.append_ranges(&self.dictionary, selections)?; Ok(data_builder.finish()) } @@ -754,30 +1100,39 @@ impl DictionaryDataBlock { self, key_type: Box, value_type: Box, - validate: bool, + _validate: bool, ) -> Result { - let indices = self.indices.into_arrow((*key_type).clone(), validate)?; + let declared_key_bits = key_type.byte_width() as u64 * 8; + if self.indices.bits_per_value != declared_key_bits { + return Err(lance_core::Error::corrupt_file_named( + "dictionary", + format!( + "dictionary indices use {} bits but the declared {} key type uses {} bits", + self.indices.bits_per_value, key_type, declared_key_bits + ), + )); + } + let indices_num_values = self.indices.num_values; + let indices = self + .indices + .do_into_arrow((*key_type).clone(), indices_num_values, true)?; let dictionary = self .dictionary - .into_arrow((*value_type).clone(), validate)?; + .into_arrow_impl((*value_type).clone(), true)?; let builder = indices .into_builder() .add_child_data(dictionary) .data_type(DataType::Dictionary(key_type, value_type)); - if validate { - Ok(builder.build()?) - } else { - Ok(unsafe { builder.build_unchecked() }) - } + Ok(builder.build()?) } fn into_arrow(self, data_type: DataType, validate: bool) -> Result { if let DataType::Dictionary(key_type, value_type) = data_type { self.into_arrow_dict(key_type, value_type, validate) } else { - self.decode()?.into_arrow(data_type, validate) + self.decode()?.into_arrow_impl(data_type, validate) } } @@ -828,8 +1183,15 @@ pub enum DataBlock { } impl DataBlock { - /// Convert self into an Arrow ArrayData - pub fn into_arrow(self, data_type: DataType, validate: bool) -> Result { + /// Convert self into an Arrow ArrayData with full layout validation. + /// + /// The `validate` argument is retained for API compatibility. Conversion is + /// always validated because callers can construct data blocks directly. + pub fn into_arrow(self, data_type: DataType, _validate: bool) -> Result { + self.into_arrow_impl(data_type, true) + } + + fn into_arrow_impl(self, data_type: DataType, validate: bool) -> Result { match self { Self::Empty() => Ok(new_empty_array(&data_type).to_data()), Self::Constant(inner) => inner.into_arrow(data_type, validate), @@ -1002,7 +1364,7 @@ impl DataBlock { } } - pub fn make_builder(&self, estimated_size_bytes: u64) -> Box { + fn make_builder(&self, estimated_size_bytes: u64) -> Box { match self { Self::FixedWidth(inner) => { if inner.bits_per_value == 1 { @@ -1191,13 +1553,22 @@ fn arrow_binary_to_data_block( bits_per_offset: u8, ) -> DataBlock { let data_vec = arrays.iter().map(|arr| arr.to_data()).collect::>(); + arrow_binary_array_data_to_data_block(&data_vec, num_values, bits_per_offset) +} + +fn arrow_binary_array_data_to_data_block( + data_vec: &[ArrayData], + num_values: u64, + bits_per_offset: u8, +) -> DataBlock { let bytes_per_offset = bits_per_offset as usize / 8; let offsets = data_vec .iter() .map(|d| { - LanceBuffer::from( - d.buffers()[0].slice_with_length(d.offset(), (d.len() + 1) * bytes_per_offset), - ) + LanceBuffer::from(d.buffers()[0].slice_with_length( + d.offset() * bytes_per_offset, + (d.len() + 1) * bytes_per_offset, + )) }) .collect::>(); let (offsets, data_ranges) = if bits_per_offset == 32 { @@ -1335,8 +1706,6 @@ fn arrow_dictionary_to_data_block(arrays: &[ArrayRef], validity: Option max_index_val { - // Widen the index type - if max_index_val >= u32::MAX as u64 { - unimplemented!("Dictionary arrays with 2^32 unique value (or more) and a null") - } - upcast = Some(arrow_cast::cast(indices, &DataType::UInt32).unwrap()); - indices = upcast.as_ref().unwrap(); - } - null_index + values.len() - 1 }); + let max_index_val = max_index_val(indices.data_type()); + let upcast = if first_invalid_index as u64 > max_index_val { + // Widen the index type when the null dictionary value cannot be addressed by the + // declared key type, whether the value already existed or was appended above. + if max_index_val >= u32::MAX as u64 { + unimplemented!("Dictionary arrays with 2^32 unique value (or more) and a null") + } + Some(arrow_cast::cast(indices, &DataType::UInt32).unwrap()) + } else { + None + }; + if let Some(upcast) = upcast.as_ref() { + indices = upcast; + } // This can't fail since we already checked for fit let null_index_arr = arrow_cast::cast( &UInt64Array::from(vec![first_invalid_index as u64]), @@ -1450,6 +1823,71 @@ fn extract_nulls(arrays: &[ArrayRef], num_values: u64) -> Nullability { } impl DataBlock { + fn validate_variable_width_offsets( + array_data: &ArrayData, + ) -> std::result::Result<(), String> { + if array_data.is_empty() && array_data.buffers()[0].is_empty() { + return Ok(()); + } + let offset_size = std::mem::size_of::(); + let offset_start = array_data.offset() * offset_size; + let offset_len = (array_data.len() + 1) * offset_size; + let offset_buffer = + LanceBuffer::from(array_data.buffers()[0].slice_with_length(offset_start, offset_len)); + let offsets = offset_buffer.borrow_to_typed_slice::(); + VariableWidthBlock::offset_violation_detail( + offsets.as_ref(), + array_data.buffers()[1].len(), + 0, + ) + .map_or(Ok(()), Err) + } + + // `validate_full` also rescans UTF-8 contents and character boundaries on every flush. + // Encoding only needs a complete monotonicity and bounds proof before slicing offsets. + fn validate_variable_width_layouts(array_data: &ArrayData) -> std::result::Result<(), String> { + match array_data.data_type() { + DataType::Binary | DataType::Utf8 => { + Self::validate_variable_width_offsets::(array_data)?; + } + DataType::LargeBinary | DataType::LargeUtf8 => { + Self::validate_variable_width_offsets::(array_data)?; + } + _ => {} + } + for child_data in array_data.child_data() { + Self::validate_variable_width_layouts(child_data)?; + } + Ok(()) + } + + fn validate_array_data( + array_data: &ArrayData, + field_name: &str, + array_index: usize, + ) -> Result<()> { + let validation = array_data + .validate() + .map_err(|error| error.to_string()) + .and_then(|_| Self::validate_variable_width_layouts(array_data)); + validation.map_err(|error| { + Error::invalid_input_source( + format!( + "Invalid Arrow array for field '{}' at buffered array {}: {}", + field_name, array_index, error + ) + .into(), + ) + }) + } + + pub(crate) fn validate_arrays(arrays: &[ArrayRef], field_name: &str) -> Result<()> { + for (array_index, array) in arrays.iter().enumerate() { + Self::validate_array_data(&array.to_data(), field_name, array_index)?; + } + Ok(()) + } + pub fn from_arrays(arrays: &[ArrayRef], num_values: u64) -> Self { if arrays.is_empty() || num_values == 0 { return Self::AllNull(AllNullDataBlock { num_values: 0 }); @@ -1611,8 +2049,16 @@ impl From for DataBlock { } } -pub trait DataBlockBuilderImpl: std::fmt::Debug { - fn append(&mut self, data_block: &DataBlock, selection: Range); +trait DataBlockBuilderImpl: std::fmt::Debug { + fn validate_append(&self, data_block: &DataBlock, selection: &Range) -> Result<()>; + + fn append_validated(&mut self, data_block: &DataBlock, selection: Range) -> Result<()>; + + fn append(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + self.validate_append(data_block, &selection)?; + self.append_validated(data_block, selection) + } + fn finish(self: Box) -> DataBlock; } @@ -1637,8 +2083,31 @@ impl DataBlockBuilder { self.builder.as_mut().unwrap().as_mut() } - pub fn append(&mut self, data_block: &DataBlock, selection: Range) { - self.get_builder(data_block).append(data_block, selection); + pub fn append(&mut self, data_block: &DataBlock, selection: Range) -> Result<()> { + self.get_builder(data_block).append(data_block, selection) + } + + fn append_ranges( + &mut self, + data_block: &DataBlock, + selections: impl IntoIterator>, + ) -> Result<()> { + let full_selection = 0..data_block.num_values(); + let builder = self.get_builder(data_block); + builder.validate_append(data_block, &full_selection)?; + for selection in selections { + if selection.start > selection.end || selection.end > full_selection.end { + return Err(Error::corrupt_file_named( + "data block", + format!( + "cannot append selection {}..{} from a block with {} values", + selection.start, selection.end, full_selection.end + ), + )); + } + builder.append_validated(data_block, selection)?; + } + Ok(()) } pub fn finish(self) -> DataBlock { @@ -1652,19 +2121,26 @@ mod tests { use std::sync::Arc; use arrow_array::{ - ArrayRef, BinaryViewArray, DictionaryArray, Int8Array, LargeBinaryArray, StringArray, - StringViewArray, UInt8Array, UInt16Array, make_array, new_null_array, + ArrayRef, BinaryArray, BinaryViewArray, DictionaryArray, Int8Array, LargeBinaryArray, + LargeStringArray, StringArray, StringViewArray, UInt8Array, UInt16Array, make_array, + new_null_array, types::{Int8Type, Int32Type}, }; - use arrow_buffer::{BooleanBuffer, NullBuffer}; + use arrow_buffer::{BooleanBuffer, Buffer, NullBuffer}; + use arrow_data::ArrayData; use arrow_schema::{DataType, Field, Fields}; + use lance_core::Error; use lance_datagen::{ArrayGeneratorExt, DEFAULT_SEED, RowCount, array}; use rand::SeedableRng; + use rstest::rstest; use crate::buffer::LanceBuffer; - use super::{AllNullDataBlock, DataBlock}; + use super::{ + AllNullDataBlock, BlockInfo, DataBlock, DataBlockBuilder, DictionaryDataBlock, + FixedWidthDataBlock, VariableWidthBlock, + }; use arrow_array::Array; @@ -1818,6 +2294,70 @@ mod tests { ); } + #[rstest] + #[case::utf8( + DataType::Utf8, + Buffer::from_slice_ref([0_i32, 5, 10]), + 32, + LanceBuffer::reinterpret_vec(vec![0_i32, 5]) + )] + #[case::large_utf8( + DataType::LargeUtf8, + Buffer::from_slice_ref([0_i64, 5, 10]), + 64, + LanceBuffer::reinterpret_vec(vec![0_i64, 5]) + )] + fn test_variable_width_array_data_offset( + #[case] data_type: DataType, + #[case] offsets: Buffer, + #[case] bits_per_offset: u8, + #[case] expected_offsets: LanceBuffer, + ) { + let array_data = ArrayData::builder(data_type) + .len(1) + .offset(1) + .add_buffer(offsets) + .add_buffer(Buffer::from(b"helloworld")) + .build() + .unwrap(); + + DataBlock::validate_array_data(&array_data, "text", 0).unwrap(); + let data = super::arrow_binary_array_data_to_data_block(&[array_data], 1, bits_per_offset); + + let data = data.as_variable_width().unwrap(); + assert_eq!(data.offsets, expected_offsets); + assert_eq!(data.data, LanceBuffer::copy_slice(b"world")); + } + + #[rstest] + #[case::utf8(DataType::Utf8, Buffer::from_slice_ref([0_i32, -1]))] + #[case::large_utf8(DataType::LargeUtf8, Buffer::from_slice_ref([0_i64, -1]))] + fn test_invalid_string_offsets_rejected_before_encoding( + #[case] data_type: DataType, + #[case] offsets: Buffer, + ) { + let array_data = unsafe { + ArrayData::builder(data_type) + .len(1) + .add_buffer(offsets) + .add_buffer(Buffer::from(b"")) + .build_unchecked() + }; + + let error = DataBlock::validate_array_data(&array_data, "text", 0).unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. })); + let message = error.to_string(); + assert!( + message.contains("field 'text'"), + "unexpected message: {message}" + ); + assert!( + message.contains("offset[1] (-1)"), + "unexpected message: {message}" + ); + } + #[test] fn test_large() { let arr = LargeBinaryArray::from_vec(vec![b"hello", b"world"]); @@ -2079,4 +2619,280 @@ mod tests { let total_nulls_size_in_bytes = concatenated_array.nulls().unwrap().len().div_ceil(8); assert!(block.data_size() == (total_buffer_size + total_nulls_size_in_bytes) as u64); } + + #[test] + fn variable_width_rejects_out_of_bounds_offsets_without_optional_validation() { + let block = VariableWidthBlock { + data: LanceBuffer::copy_slice(b"alphabetagamma"), + offsets: LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + bits_per_offset: 32, + num_values: 3, + block_info: BlockInfo::new(), + }; + + let error = block + .into_arrow(DataType::Binary, false) + .expect_err("out-of-bounds offsets must be rejected"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + let message = error.to_string(); + assert!( + message.contains("100000") && message.contains("data buffer size: 14 bytes"), + "error must report the offending offset and the data buffer size: {message}" + ); + } + + #[test] + fn public_fixed_width_conversion_always_validates_layout() { + let block = FixedWidthDataBlock { + data: LanceBuffer::from(vec![0_u8; 4]), + bits_per_value: 32, + num_values: 2, + block_info: BlockInfo::new(), + }; + + for validate in [false, true] { + block + .clone() + .into_arrow(DataType::Int32, validate) + .expect_err("a short values buffer must be rejected"); + DataBlock::FixedWidth(block.clone()) + .into_arrow(DataType::Int32, validate) + .expect_err("a short values buffer must be rejected"); + } + } + + #[rstest] + #[case::i32_decreasing( + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 2]), + 32, + 2, + "decreases" + )] + #[case::i64_decreasing( + LanceBuffer::reinterpret_vec(vec![0_i64, 5, 2]), + 64, + 2, + "decreases" + )] + #[case::i32_out_of_bounds( + LanceBuffer::reinterpret_vec(vec![0_i32, 6]), + 32, + 1, + "out of bounds" + )] + fn variable_width_builder_rejects_malformed_offsets( + #[case] offsets: LanceBuffer, + #[case] bits_per_offset: u8, + #[case] num_values: u64, + #[case] expected_message: &str, + ) { + let block = DataBlock::VariableWidth(VariableWidthBlock { + data: LanceBuffer::copy_slice(b"abcde"), + offsets, + bits_per_offset, + num_values, + block_info: BlockInfo::new(), + }); + let mut builder = DataBlockBuilder::with_capacity_estimate(5); + + let error = builder + .append(&block, 0..num_values) + .expect_err("malformed offsets must fail concatenation"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains(expected_message), + "unexpected message: {error}" + ); + } + + #[rstest] + #[case::binary_i32_tail_out_of_bounds( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::utf8_i32_tail_out_of_bounds( + DataType::Utf8, + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::large_binary_i64_tail_out_of_bounds( + DataType::LargeBinary, + LanceBuffer::reinterpret_vec(vec![0_i64, 5, 9, 100_000]), + 64, + 3, + b"alphabetagamma".as_slice() + )] + #[case::large_utf8_i64_tail_out_of_bounds( + DataType::LargeUtf8, + LanceBuffer::reinterpret_vec(vec![0_i64, 5, 9, 100_000]), + 64, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_negative_offset( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, -1, 9, 14]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_non_monotonic_offsets( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 9, 5, 14]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_interior_offset_out_of_bounds( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 100_000, 100_000, 14]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::binary_offsets_buffer_too_short( + DataType::Binary, + LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9]), + 32, + 3, + b"alphabetagamma".as_slice() + )] + #[case::utf8_invalid_byte_sequence( + DataType::Utf8, + LanceBuffer::reinterpret_vec(vec![0_i32, 1, 2, 3]), + 32, + 3, + &[b'a', 0xFF, b'b'] + )] + #[case::utf8_offset_splits_multibyte_char( + DataType::Utf8, + LanceBuffer::reinterpret_vec(vec![0_i32, 1, 2]), + 32, + 2, + "é".as_bytes() + )] + #[case::large_utf8_invalid_byte_sequence( + DataType::LargeUtf8, + LanceBuffer::reinterpret_vec(vec![0_i64, 1, 2, 3]), + 64, + 3, + &[b'a', 0xFF, b'b'] + )] + fn variable_width_rejects_malformed_layout( + #[case] data_type: DataType, + #[case] offsets: LanceBuffer, + #[case] bits_per_offset: u8, + #[case] num_values: u64, + #[case] data: &[u8], + ) { + let block = VariableWidthBlock { + data: LanceBuffer::copy_slice(data), + offsets, + bits_per_offset, + num_values, + block_info: BlockInfo::new(), + }; + + // The malformed layout must be rejected regardless of the optional + // `validate` flag: the flag selects extra validation, not the memory + // safety proof required to construct an Arrow array. + for validate in [false, true] { + let error = DataBlock::VariableWidth(block.clone()) + .into_arrow(data_type.clone(), validate) + .expect_err("malformed variable-width layout must be rejected"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile with validate={validate}, got: {error}" + ); + } + } + + #[test] + fn dictionary_rejects_malformed_variable_width_values_without_optional_validation() { + let values = VariableWidthBlock { + data: LanceBuffer::copy_slice(b"alphabetagamma"), + offsets: LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 100_000]), + bits_per_offset: 32, + num_values: 3, + block_info: BlockInfo::new(), + }; + let dictionary = DataBlock::Dictionary(DictionaryDataBlock { + indices: FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(vec![0_i32, 1, 2]), + bits_per_value: 32, + num_values: 3, + block_info: BlockInfo::new(), + }, + dictionary: Box::new(DataBlock::VariableWidth(values)), + }); + + let data_type = DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Binary)); + let error = dictionary + .into_arrow(data_type, false) + .expect_err("dictionary with out-of-bounds value offsets must be rejected"); + assert!( + matches!(error, Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + } + + #[test] + fn dictionary_rejects_indices_wider_than_declared_key_type() { + let dictionary = DataBlock::Dictionary(DictionaryDataBlock { + indices: FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(vec![0_u32, 1, 128]), + bits_per_value: 32, + num_values: 3, + block_info: BlockInfo::new(), + }, + dictionary: Box::new(DataBlock::from_array(StringArray::from(vec![ + Some("zero"), + Some("one"), + None, + ]))), + }); + + let data_type = DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)); + let error = dictionary + .into_arrow(data_type, false) + .expect_err("mismatched dictionary index widths must be rejected"); + + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error.to_string().contains( + "dictionary indices use 32 bits but the declared Int8 key type uses 8 bits" + ) + ); + } + + #[rstest] + #[case::binary(Arc::new(BinaryArray::from_vec(vec![b"alpha", b"", b"gamma"])) as ArrayRef)] + #[case::large_binary( + Arc::new(LargeBinaryArray::from_vec(vec![b"alpha", b"", b"gamma"])) as ArrayRef + )] + #[case::utf8(Arc::new(StringArray::from(vec!["héllo", "", "world"])) as ArrayRef)] + #[case::large_utf8(Arc::new(LargeStringArray::from(vec!["héllo", "", "world"])) as ArrayRef)] + fn variable_width_valid_data_survives_mandatory_validation(#[case] array: ArrayRef) { + let block = DataBlock::from_array(array.clone()); + for validate in [false, true] { + let round_tripped = make_array( + block + .clone() + .into_arrow(array.data_type().clone(), validate) + .unwrap(), + ); + assert_eq!(&round_tripped, &array); + } + } } diff --git a/rust/lance-encoding/src/decoder.rs b/rust/lance-encoding/src/decoder.rs index 55340d09c94..ad049eed3a9 100644 --- a/rust/lance-encoding/src/decoder.rs +++ b/rust/lance-encoding/src/decoder.rs @@ -240,6 +240,12 @@ use lance_core::error::LanceOptionExt; use lance_core::{ArrowResult, Error, Result}; use tracing::instrument; +use crate::array_encoding::logical::list::OffsetPageInfo; +use crate::array_encoding::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}; +use crate::array_encoding::logical::{ + binary::BinaryFieldScheduler, blob::BlobFieldScheduler, list::ListFieldScheduler, + primitive::PrimitiveFieldScheduler, +}; use crate::compression::{DecompressionStrategy, DefaultDecompressionStrategy}; use crate::data::DataBlock; use crate::encoder::EncodedBatch; @@ -250,17 +256,76 @@ use crate::encodings::logical::primitive::StructuralPrimitiveFieldScheduler; use crate::encodings::logical::r#struct::{StructuralStructDecoder, StructuralStructScheduler}; use crate::format::pb::{self, column_encoding}; use crate::format::pb21; -use crate::previous::decoder::LogicalPageDecoder; -use crate::previous::encodings::logical::list::OffsetPageInfo; -use crate::previous::encodings::logical::r#struct::{SimpleStructDecoder, SimpleStructScheduler}; -use crate::previous::encodings::logical::{ - binary::BinaryFieldScheduler, blob::BlobFieldScheduler, list::ListFieldScheduler, - primitive::PrimitiveFieldScheduler, -}; use crate::repdef::{CompositeRepDefUnraveler, RepDefUnraveler}; -use crate::version::LanceFileVersion; use crate::{BufferScheduler, EncodingsIo}; +/// Candidate batch sizes evaluated during byte-budget planning. +/// Powers of 4, covering 1–16Ki rows in 8 probes. +pub const CANDIDATE_BATCH_SIZES: [u32; 8] = [1, 4, 16, 64, 256, 1024, 4096, 16384]; + +pub trait SchedulingJob: std::fmt::Debug { + fn schedule_next( + &mut self, + context: &mut SchedulerContext, + priority: &dyn PriorityRange, + ) -> Result; + + fn num_rows(&self) -> u64; +} + +/// Schedules the I/O needed to decode one field. +pub trait FieldScheduler: Send + Sync + std::fmt::Debug { + fn initialize<'a>( + &'a self, + filter: &'a FilterExpression, + context: &'a SchedulerContext, + ) -> BoxFuture<'a, Result<()>>; + + fn schedule_ranges<'a>( + &'a self, + ranges: &[Range], + filter: &FilterExpression, + ) -> Result>; + + fn num_rows(&self) -> u64; +} + +#[derive(Debug)] +pub struct DecoderReady { + pub decoder: Box, + pub path: VecDeque, +} + +/// Stateful decoder for one logical page. +pub trait LogicalPageDecoder: std::fmt::Debug + Send { + fn accept_child(&mut self, _child: DecoderReady) -> Result<()> { + Err(Error::internal(format!( + "The decoder {:?} does not expect children but received a child", + self + ))) + } + + fn wait_for_loaded(&'_ mut self, loaded_need: u64) -> BoxFuture<'_, Result<()>>; + + fn rows_loaded(&self) -> u64; + + fn rows_unloaded(&self) -> u64 { + self.num_rows() - self.rows_loaded() + } + + fn num_rows(&self) -> u64; + + fn rows_drained(&self) -> u64; + + fn rows_left(&self) -> u64 { + self.num_rows() - self.rows_drained() + } + + fn drain(&mut self, num_rows: u64) -> Result; + + fn data_type(&self) -> &DataType; +} + // If users are getting batches over 10MiB large then it's time to reduce the batch size const BATCH_SIZE_BYTES_WARNING: u64 = 10 * 1024 * 1024; const ENV_LANCE_STRUCTURAL_BATCH_DECODE_SPAWN_MODE: &str = @@ -288,13 +353,13 @@ fn inline_scheduling_threshold() -> u64 { }) } -/// Top-level encoding message for a page. Wraps both the -/// legacy pb::ArrayEncoding and the newer pb::PageLayout +/// Top-level encoding message for a page. Wraps both the v2.0 +/// [`pb::ArrayEncoding`] grammar and the structural [`pb21::PageLayout`] grammar. /// /// A file should only use one or the other and never both. /// 2.0 decoders can always assume this is pb::ArrayEncoding /// and 2.1+ decoders can always assume this is pb::PageLayout -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum PageEncoding { Legacy(pb::ArrayEncoding), Structural(pb21::PageLayout), @@ -402,21 +467,21 @@ impl ColumnInfo { enum RootScheduler { Structural(Box), - Legacy(Arc), + Array(Arc), } impl RootScheduler { - fn as_legacy(&self) -> &Arc { + fn as_array(&self) -> &Arc { match self { - Self::Structural(_) => panic!("Expected a legacy scheduler"), - Self::Legacy(s) => s, + Self::Structural(_) => panic!("Expected an array scheduler"), + Self::Array(s) => s, } } fn as_structural(&self) -> &dyn StructuralFieldScheduler { match self { Self::Structural(s) => s.as_ref(), - Self::Legacy(_) => panic!("Expected a structural scheduler"), + Self::Array(_) => panic!("Expected a structural scheduler"), } } } @@ -606,14 +671,14 @@ impl CoreFieldDecoderStrategy { } } - fn is_primitive_legacy(data_type: &DataType) -> bool { + fn is_array_primitive(data_type: &DataType) -> bool { if data_type.is_primitive() { true } else { match data_type { // DataType::is_primitive doesn't consider these primitive but we do DataType::Boolean | DataType::Null | DataType::FixedSizeBinary(_) => true, - DataType::FixedSizeList(inner, _) => Self::is_primitive_legacy(inner.data_type()), + DataType::FixedSizeList(inner, _) => Self::is_array_primitive(inner.data_type()), _ => false, } } @@ -624,7 +689,7 @@ impl CoreFieldDecoderStrategy { field: &Field, column: &ColumnInfo, buffers: FileBuffers, - ) -> Result> { + ) -> Result> { Self::ensure_values_encoded(column, &field.name)?; // Primitive fields map to a single column let column_buffers = ColumnBuffers { @@ -667,52 +732,57 @@ impl CoreFieldDecoderStrategy { column_infos: &mut ColumnInfoIter, buffers: FileBuffers, offsets_column: &ColumnInfo, - ) -> Result> { + ) -> Result> { Self::ensure_values_encoded(offsets_column, &list_field.name)?; let offsets_column_buffers = ColumnBuffers { file_buffers: buffers, positions_and_sizes: &offsets_column.buffer_offsets_and_sizes, }; let items_scheduler = - self.create_legacy_field_scheduler(&list_field.children[0], column_infos, buffers)?; + self.create_array_field_scheduler(&list_field.children[0], column_infos, buffers)?; - let (inner_infos, null_offset_adjustments): (Vec<_>, Vec<_>) = offsets_column + let mut inner_infos = Vec::with_capacity(offsets_column.page_infos.len()); + let mut null_offset_adjustments = Vec::with_capacity(offsets_column.page_infos.len()); + for (page_index, offsets_page) in offsets_column .page_infos .iter() - .filter(|offsets_page| offsets_page.num_rows > 0) - .map(|offsets_page| { - if let Some(pb::array_encoding::ArrayEncoding::List(list_encoding)) = - &offsets_page.encoding.as_legacy().array_encoding - { - let inner = PageInfo { - buffer_offsets_and_sizes: offsets_page.buffer_offsets_and_sizes.clone(), - encoding: PageEncoding::Legacy( - list_encoding.offsets.as_ref().unwrap().as_ref().clone(), - ), - num_rows: offsets_page.num_rows, - priority: 0, - }; - ( - inner, - OffsetPageInfo { - offsets_in_page: offsets_page.num_rows, - null_offset_adjustment: list_encoding.null_offset_adjustment, - num_items_referenced_by_page: list_encoding.num_items, - }, - ) - } else { - // TODO: Should probably return Err here - panic!("Expected a list column"); - } - }) - .unzip(); + .enumerate() + .filter(|(_, offsets_page)| offsets_page.num_rows > 0) + { + let PageEncoding::Legacy(pb::ArrayEncoding { + array_encoding: Some(pb::array_encoding::ArrayEncoding::List(list_encoding)), + }) = &offsets_page.encoding + else { + return Err(Error::invalid_input(format!( + "expected list encoding for field '{}' in column {}, page {} but got {:?}", + list_field.name, offsets_column.index, page_index, offsets_page.encoding + ))); + }; + let offsets_encoding = list_encoding.offsets.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "list encoding for field '{}' in column {}, page {} is missing its offsets encoding", + list_field.name, offsets_column.index, page_index + )) + })?; + inner_infos.push(PageInfo { + buffer_offsets_and_sizes: offsets_page.buffer_offsets_and_sizes.clone(), + encoding: PageEncoding::Legacy(offsets_encoding.as_ref().clone()), + num_rows: offsets_page.num_rows, + priority: 0, + }); + null_offset_adjustments.push(OffsetPageInfo { + offsets_in_page: offsets_page.num_rows, + null_offset_adjustment: list_encoding.null_offset_adjustment, + num_items_referenced_by_page: list_encoding.num_items, + }); + } let inner = Arc::new(PrimitiveFieldScheduler::new( offsets_column.index, DataType::UInt64, Arc::from(inner_infos.into_boxed_slice()), offsets_column_buffers, self.validate_data, - )) as Arc; + )) as Arc; let items_field = match list_field.data_type() { DataType::List(inner) => inner, DataType::LargeList(inner) => inner, @@ -853,15 +923,15 @@ impl CoreFieldDecoderStrategy { } } - fn create_legacy_field_scheduler( + fn create_array_field_scheduler( &self, field: &Field, column_infos: &mut ColumnInfoIter, buffers: FileBuffers, - ) -> Result> { + ) -> Result> { let data_type = field.data_type(); validate_fixed_size_list_dimensions(&field.name, &data_type)?; - if Self::is_primitive_legacy(&data_type) { + if Self::is_array_primitive(&data_type) { let column_info = column_infos.expect_next()?; let scheduler = self.create_primitive_scheduler(field, column_info, buffers)?; return Ok(scheduler); @@ -920,7 +990,7 @@ impl CoreFieldDecoderStrategy { DataType::FixedSizeList(inner, _dimension) => { // A fixed size list column could either be a physical or a logical decoder // depending on the child data type. - if Self::is_primitive_legacy(inner.data_type()) { + if Self::is_array_primitive(inner.data_type()) { let primitive_col = column_infos.expect_next()?; let scheduler = self.create_primitive_scheduler(field, primitive_col, buffers)?; @@ -930,7 +1000,7 @@ impl CoreFieldDecoderStrategy { } } DataType::Dictionary(_key_type, value_type) => { - if Self::is_primitive_legacy(value_type) || value_type.is_binary_like() { + if Self::is_array_primitive(value_type) || value_type.is_binary_like() { let primitive_col = column_infos.expect_next()?; let scheduler = self.create_primitive_scheduler(field, primitive_col, buffers)?; @@ -974,7 +1044,7 @@ impl CoreFieldDecoderStrategy { for field in &field.children { column_infos.next_top_level(); let field_scheduler = - self.create_legacy_field_scheduler(field, column_infos, buffers)?; + self.create_array_field_scheduler(field, column_infos, buffers)?; child_schedulers.push(Arc::from(field_scheduler)); } @@ -1008,7 +1078,7 @@ fn root_column(num_rows: u64) -> ColumnInfo { pb::SimpleStruct {}, )), }), - priority: 0, // not used in legacy scheduler + priority: 0, // not used by the array scheduler buffer_offsets_and_sizes: Arc::new([]), }) .collect::>(); @@ -1024,21 +1094,21 @@ fn root_column(num_rows: u64) -> ColumnInfo { pub enum RootDecoder { Structural(StructuralStructDecoder), - Legacy(SimpleStructDecoder), + Array(SimpleStructDecoder), } impl RootDecoder { pub fn into_structural(self) -> StructuralStructDecoder { match self { Self::Structural(decoder) => decoder, - Self::Legacy(_) => panic!("Expected a structural decoder"), + Self::Array(_) => panic!("Expected a structural decoder"), } } - pub fn into_legacy(self) -> SimpleStructDecoder { + pub fn into_array(self) -> SimpleStructDecoder { match self { - Self::Legacy(decoder) => decoder, - Self::Structural(_) => panic!("Expected a legacy decoder"), + Self::Array(decoder) => decoder, + Self::Structural(_) => panic!("Expected an array decoder"), } } } @@ -1104,27 +1174,27 @@ impl DecodeBatchScheduler { let mut column_iter = ColumnInfoIter::new(columns, &adjusted_column_indices); let strategy = CoreFieldDecoderStrategy::from_decoder_config(decoder_config); let root_scheduler = - strategy.create_legacy_field_scheduler(&root_field, &mut column_iter, buffers)?; + strategy.create_array_field_scheduler(&root_field, &mut column_iter, buffers)?; let context = SchedulerContext::new(io, cache.clone()); root_scheduler.initialize(filter, &context).await?; Ok(Self { - root_scheduler: RootScheduler::Legacy(root_scheduler.into()), + root_scheduler: RootScheduler::Array(root_scheduler.into()), root_fields, cache, }) } } - #[deprecated(since = "0.29.1", note = "This is for legacy 2.0 paths")] + #[deprecated(since = "0.29.1", note = "This is for v2.0 array-encoding paths")] pub fn from_scheduler( - root_scheduler: Arc, + root_scheduler: Arc, root_fields: Fields, cache: Arc, ) -> Self { Self { - root_scheduler: RootScheduler::Legacy(root_scheduler), + root_scheduler: RootScheduler::Array(root_scheduler), root_fields, cache, } @@ -1174,7 +1244,7 @@ impl DecodeBatchScheduler { } } - fn do_schedule_ranges_legacy( + fn do_schedule_ranges_array( &mut self, ranges: &[Range], filter: &FilterExpression, @@ -1185,7 +1255,7 @@ impl DecodeBatchScheduler { // tasks are scheduled at the same top level row. priority: Option>, ) { - let root_scheduler = self.root_scheduler.as_legacy(); + let root_scheduler = self.root_scheduler.as_array(); let rows_requested = ranges.iter().map(|r| r.end - r.start).sum::(); trace!( "Scheduling {} ranges across {}..{} ({} rows){}", @@ -1249,8 +1319,8 @@ impl DecodeBatchScheduler { priority: Option>, ) { match &self.root_scheduler { - RootScheduler::Legacy(_) => { - self.do_schedule_ranges_legacy(ranges, filter, io, schedule_action, priority) + RootScheduler::Array(_) => { + self.do_schedule_ranges_array(ranges, filter, io, schedule_action, priority) } RootScheduler::Structural(_) => { self.do_schedule_ranges_structural(ranges, filter, io, schedule_action) @@ -1422,7 +1492,7 @@ impl BatchDecodeStream { } } - fn accept_decoder(&mut self, decoder: crate::previous::decoder::DecoderReady) -> Result<()> { + fn accept_decoder(&mut self, decoder: DecoderReady) -> Result<()> { if decoder.path.is_empty() { // The root decoder we can ignore Ok(()) @@ -1443,7 +1513,7 @@ impl BatchDecodeStream { let scan_line = scan_line?; self.rows_scheduled = scan_line.scheduled_so_far; for message in scan_line.decoders { - self.accept_decoder(message.into_legacy())?; + self.accept_decoder(message.into_array())?; } } None => { @@ -1552,7 +1622,7 @@ impl BatchDecodeStream { // we can have a single implementation of the batch decode iterator enum RootDecoderMessage { LoadedPage(LoadedPageShard), - LegacyPage(crate::previous::decoder::DecoderReady), + ArrayPage(DecoderReady), } trait RootDecoderType { fn accept_message(&mut self, message: RootDecoderMessage) -> Result<()>; @@ -1576,10 +1646,10 @@ impl RootDecoderType for StructuralStructDecoder { } impl RootDecoderType for SimpleStructDecoder { fn accept_message(&mut self, message: RootDecoderMessage) -> Result<()> { - let RootDecoderMessage::LegacyPage(legacy_page) = message else { + let RootDecoderMessage::ArrayPage(array_page) = message else { unreachable!() }; - self.accept_child(legacy_page) + self.accept_child(array_page) } fn drain_batch(&mut self, num_rows: u64) -> Result { self.drain(num_rows) @@ -1663,7 +1733,7 @@ impl BatchDecodeIterator { // The root decoder we can ignore if !decoder_ready.path.is_empty() { self.root_decoder - .accept_message(RootDecoderMessage::LegacyPage(decoder_ready))?; + .accept_message(RootDecoderMessage::ArrayPage(decoder_ready))?; } } } @@ -1748,7 +1818,12 @@ impl RecordBatchReader for BatchDecodeIterator { /// This estimate ignores validity bitmaps at the moment. We can't infer /// their presence simply from the data_type and their impact is probably /// fairly negligible. -fn estimate_bytes_per_row(data_type: &DataType) -> f64 { +/// Returns a schema-based estimate of the decoded bytes per row for `data_type`. +/// +/// Fixed-width types are exact. Variable-width types (strings, lists, etc.) use +/// heuristic constants. This estimate is used both in batch-size planning and as +/// a fallback for V1 files that lack structural decoders. +pub fn estimate_bytes_per_row(data_type: &DataType) -> f64 { if let Some(w) = data_type.byte_width_opt() { return w as f64; } @@ -1789,7 +1864,8 @@ pub struct StructuralBatchDecodeStream { // - false: run `into_batch` inline, which avoids Tokio scheduling overhead and is // typically better for point lookups / small takes. spawn_batch_decode_tasks: bool, - /// If set, target this many bytes per batch instead of `rows_per_batch` rows. + /// If set, target this many bytes per batch while retaining `rows_per_batch` + /// as an independent upper bound. batch_size_bytes: Option, /// Schema-based estimate of bytes per row, computed once at construction. /// Only meaningful when `batch_size_bytes` is `Some`. @@ -1877,6 +1953,7 @@ impl StructuralBatchDecodeStream { return Ok(None); } + let row_limit = self.rows_remaining.min(self.rows_per_batch as u64); let mut to_take = if let Some(batch_size_bytes) = self.batch_size_bytes { let feedback = self.bytes_per_row_feedback.load(Ordering::Relaxed); let bpr = if feedback > 0 { @@ -1885,9 +1962,9 @@ impl StructuralBatchDecodeStream { self.schema_bytes_per_row }; let rows = (batch_size_bytes as f64 / bpr) as u64; - self.rows_remaining.min(rows.max(1)) + row_limit.min(rows.max(1)) } else { - self.rows_remaining.min(self.rows_per_batch as u64) + row_limit }; self.rows_remaining -= to_take; @@ -1949,8 +2026,7 @@ impl StructuralBatchDecodeStream { next_task.into_batch(emitted_batch_size_warning)? }; let num_rows = batch.num_rows() as u64; - if num_rows > 0 { - let bpr = data_size / num_rows; + if let Some(bpr) = data_size.checked_div(num_rows) { let prev = bytes_per_row_feedback.load(Ordering::Relaxed); let next = if prev == 0 || bpr >= prev { // First batch or actual size is larger than estimate: @@ -2052,7 +2128,8 @@ pub struct SchedulerDecoderConfig { pub cache: Arc, /// Decoder configuration pub decoder_config: DecoderConfig, - /// If set, target this many bytes per batch instead of using `batch_size` rows. + /// If set, target this many bytes per batch while retaining `batch_size` as + /// an independent row-count upper bound. /// /// Only supported for v2.1+ (structural) files. For v2.0 files this /// option is ignored and a warning is logged. @@ -2632,17 +2709,14 @@ impl SchedulerContext { VecDeque::from_iter(self.path.iter().copied()) } - #[deprecated(since = "0.29.1", note = "This is for legacy 2.0 paths")] - pub fn locate_decoder( - &mut self, - decoder: Box, - ) -> crate::previous::decoder::DecoderReady { + #[deprecated(since = "0.29.1", note = "This is for v2.0 array-encoding paths")] + pub fn locate_decoder(&mut self, decoder: Box) -> DecoderReady { trace!( "Scheduling decoder of type {:?} for {:?}", decoder.data_type(), self.path, ); - crate::previous::decoder::DecoderReady { + DecoderReady { decoder, path: self.current_path(), } @@ -2718,8 +2792,9 @@ pub trait DecodeArrayTask: Send { impl DecodeArrayTask for Box { fn decode(self: Box) -> Result<(ArrayRef, u64)> { - StructuralDecodeArrayTask::decode(*self) - .map(|decoded_array| (decoded_array.array, decoded_array.data_size)) + let decoded_array = StructuralDecodeArrayTask::decode(*self)?; + decoded_array.repdef.ensure_exhausted()?; + Ok((decoded_array.array, decoded_array.data_size)) } } @@ -2741,10 +2816,7 @@ impl NextDecodeTask { // suggesting the user try a smaller batch size. #[instrument(name = "task_to_batch", level = "debug", skip_all)] fn into_batch(self, emitted_batch_size_warning: Arc) -> Result<(RecordBatch, u64)> { - let (struct_arr, data_size) = self - .task - .decode() - .map_err(|e| Error::internal(format!("Error decoding batch: {}", e)))?; + let (struct_arr, data_size) = self.task.decode()?; let batch = RecordBatch::from(struct_arr.as_struct()); if data_size > BATCH_SIZE_BYTES_WARNING { emitted_batch_size_warning.call_once(|| { @@ -2765,7 +2837,7 @@ pub enum MessageType { // decoder itself. The messages were not sent in priority order and the decoder // had to wait for I/O, figuring out the correct priority. This was a lot of // complexity. - DecoderReady(crate::previous::decoder::DecoderReady), + DecoderReady(DecoderReady), // Starting in 2.1 we use a simpler scheme where the scheduling happens in priority // order and the message is an unloaded decoder. These can be awaited, in order, and // the decoder does not have to worry about waiting for I/O. @@ -2773,7 +2845,7 @@ pub enum MessageType { } impl MessageType { - pub fn into_legacy(self) -> crate::previous::decoder::DecoderReady { + pub fn into_array(self) -> DecoderReady { match self { Self::DecoderReady(decoder) => decoder, Self::UnloadedPage(_) => { @@ -2820,6 +2892,13 @@ pub trait DecodePageTask: Send + std::fmt::Debug { pub trait StructuralPageDecoder: std::fmt::Debug + Send { fn drain(&mut self, num_rows: u64) -> Result>; fn num_rows(&self) -> u64; + /// Returns the exact decoded byte count for the next `num_rows` rows + /// from this decoder's current position, without consuming any rows. + fn decoded_bytes(&self, _num_rows: u64) -> Result { + Err(Error::not_supported( + "decoded_bytes is not implemented for this page decoder".to_string(), + )) + } } #[derive(Debug)] @@ -2868,18 +2947,41 @@ pub trait StructuralFieldDecoder: std::fmt::Debug + Send { fn drain(&mut self, num_rows: u64) -> Result>; /// The data type of the decoded data fn data_type(&self) -> &DataType; + /// Returns the exact decoded byte count for each of [`CANDIDATE_BATCH_SIZES`] + /// row counts, clamped to `rows_remaining`. + /// + /// Implementations should do their best to estimate the exact size required for + /// the uncompressed data. In cases where this is not possible they should return + /// a worst-case estimate. + /// + /// The default implementation simply returns a "not supported" error though this + /// will hopefully be removed once implementation is complete. + fn plan_decoded_bytes(&self, _rows_remaining: u64) -> Result<[u64; 8]> { + Err(Error::not_supported( + "decoded_bytes is not implemented for this field decoder".to_string(), + )) + } } #[derive(Debug, Default)] pub struct DecoderPlugins {} +/// The top-level column layout used by an in-memory encoded batch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncodedBatchLayout { + /// Array pages include structural columns. + Array, + /// Structural pages include only leaf columns. + Structural, +} + /// Decodes a batch of data from an in-memory structure created by [`crate::encoder::encode_batch`] pub async fn decode_batch( batch: &EncodedBatch, filter: &FilterExpression, decoder_plugins: Arc, should_validate: bool, - version: LanceFileVersion, + layout: EncodedBatchLayout, cache: Option>, ) -> Result { // The io is synchronous so it shouldn't be possible for any async stuff to still be in progress @@ -2909,7 +3011,7 @@ pub async fn decode_batch( .await?; let (tx, rx) = unbounded_channel(); decode_scheduler.schedule_range(0..batch.num_rows, filter, tx, io_scheduler); - let is_structural = version >= LanceFileVersion::V2_1; + let is_structural = layout == EncodedBatchLayout::Structural; let mode = std::env::var(ENV_LANCE_STRUCTURAL_BATCH_DECODE_SPAWN_MODE); let spawn_structural_batch_decode_tasks = !matches!(mode.ok().as_deref(), Some("never")); let mut decode_stream = create_decode_stream( @@ -2929,7 +3031,6 @@ pub async fn decode_batch( // test coalesce indices to ranges mod tests { use super::*; - use crate::previous::decoder::{DecoderReady, LogicalPageDecoder}; use std::collections::VecDeque; #[derive(Debug)] @@ -2984,6 +3085,25 @@ mod tests { } } + struct InvalidInputDecodeTask; + + impl DecodeArrayTask for InvalidInputDecodeTask { + fn decode(self: Box) -> Result<(ArrayRef, u64)> { + Err(Error::invalid_input_source("malformed sparse page".into())) + } + } + + #[test] + fn next_decode_task_preserves_invalid_input_errors() { + let err = NextDecodeTask { + task: Box::new(InvalidInputDecodeTask), + num_rows: 0, + } + .into_batch(Arc::new(Once::new())) + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + } + #[test] fn test_read_zero_dimension_fsl_errors_instead_of_panicking() { // Simulates reading a column whose stored schema declares a @@ -3012,11 +3132,11 @@ mod tests { err ); - let mut legacy_columns = ColumnInfoIter::new(vec![], &[]); + let mut array_columns = ColumnInfoIter::new(vec![], &[]); let err = strategy - .create_legacy_field_scheduler( + .create_array_field_scheduler( &field, - &mut legacy_columns, + &mut array_columns, FileBuffers { positions_and_sizes: &[], }, @@ -3030,8 +3150,53 @@ mod tests { ); } + #[test] + fn test_list_page_with_non_list_encoding_returns_error() { + let item = Arc::new(ArrowField::new("item", DataType::Int32, true)); + let list = DataType::List(item); + let field = Field::try_from(&ArrowField::new("values", list, true)).unwrap(); + let values_encoding = pb::ColumnEncoding { + column_encoding: Some(pb::column_encoding::ColumnEncoding::Values(())), + }; + let offsets_column = Arc::new(ColumnInfo::new( + 0, + Arc::new([PageInfo { + num_rows: 1, + priority: 0, + encoding: PageEncoding::Legacy(pb::ArrayEncoding { + array_encoding: Some(pb::array_encoding::ArrayEncoding::Flat( + pb::Flat::default(), + )), + }), + buffer_offsets_and_sizes: Arc::new([]), + }]), + vec![], + values_encoding.clone(), + )); + let items_column = Arc::new(ColumnInfo::new(1, Arc::new([]), vec![], values_encoding)); + let column_indices = [0, 1]; + let mut columns = ColumnInfoIter::new(vec![offsets_column, items_column], &column_indices); + + let err = CoreFieldDecoderStrategy::default() + .create_array_field_scheduler( + &field, + &mut columns, + FileBuffers { + positions_and_sizes: &[], + }, + ) + .unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("expected list encoding for field 'values' in column 0, page 0 but got"), + "unexpected error: {err}" + ); + } + #[tokio::test] - async fn test_legacy_stream_stops_on_load_error() { + async fn test_array_stream_stops_on_load_error() { use arrow_schema::Field as ArrowField; let rows_per_batch = 1; @@ -3065,7 +3230,7 @@ mod tests { let err = batches .next() .await - .expect("stream should emit the legacy page-load error") + .expect("stream should emit the array page-load error") .unwrap_err(); assert!( err.to_string().contains(load_error_message), @@ -3074,7 +3239,7 @@ mod tests { ); assert!( batches.next().await.is_none(), - "stream should stop after the legacy page-load error" + "stream should stop after the array page-load error" ); } @@ -3170,15 +3335,14 @@ mod tests { batch_size: u32, batch_size_bytes: Option, ) -> Vec { - use crate::encoder::{EncodingOptions, default_encoding_strategy, encode_batch}; - use crate::version::LanceFileVersion; - - let version = LanceFileVersion::V2_1; - let options = EncodingOptions { - version, - ..Default::default() + use crate::{ + encoder::{EncodingOptions, encode_batch}, + testing::{TestEncoding, test_encoding_strategy}, }; - let strategy = default_encoding_strategy(version); + + let version = TestEncoding::StructuralU16; + let options = EncodingOptions::default(); + let strategy = test_encoding_strategy(version); let schema = Schema::try_from(batch.schema().as_ref()).unwrap(); let encoded = encode_batch(batch, Arc::new(schema.clone()), strategy.as_ref(), &options) .await @@ -3318,6 +3482,29 @@ mod tests { } } + #[tokio::test] + async fn test_byte_sized_batches_respect_row_limit() { + use arrow_array::Int32Array; + + let num_rows: i32 = 1000; + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "x", + DataType::Int32, + false, + )])); + let input_batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from_iter_values(0..num_rows))], + ) + .unwrap(); + + // The byte limit can hold every row, so the 100-row limit must win. + let batches = + decode_batches_with_byte_limit(&input_batch, /*batch_size=*/ 100, Some(10_000)).await; + assert_eq!(batches.len(), 10); + assert!(batches.iter().all(|batch| batch.num_rows() == 100)); + } + #[tokio::test] async fn test_byte_sized_batches_feedback_convergence() { use arrow_array::StringArray; diff --git a/rust/lance-encoding/src/encoder.rs b/rust/lance-encoding/src/encoder.rs index cd0731fd06f..91efb683e8f 100644 --- a/rust/lance-encoding/src/encoder.rs +++ b/rust/lance-encoding/src/encoder.rs @@ -6,44 +6,72 @@ //! Lance files are encoded using a [`FieldEncodingStrategy`] which choose //! what encoder to use for each field. //! -//! The current strategy is the [`StructuralEncodingStrategy`] which uses "structural" -//! encoding. A tree of encoders is built up for each field. The struct & list encoders -//! simply pull off the validity and offsets and collect them. Then, in the primitive leaf -//! encoder the validity, offsets, and values are accumulated in an accumulation buffer. Once -//! enough data has been collected the primitive encoder will either use a miniblock encoding -//! or a full zip encoding to create a page of data from the accumulation buffer. +//! Structural strategies build a tree of encoders for each field from the +//! version-free builders in [`structural`]. Struct and list encoders collect +//! validity and offsets; primitive leaf encoders accumulate values and emit +//! miniblock or full-zip pages. use std::{collections::HashMap, sync::Arc}; use arrow_array::{Array, ArrayRef, RecordBatch}; -use arrow_schema::DataType; use bytes::{Bytes, BytesMut}; use futures::future::BoxFuture; use lance_core::datatypes::{Field, Schema}; -use lance_core::error::LanceOptionExt; use lance_core::utils::bit::{is_pwr_two, pad_bytes_to}; use lance_core::{Error, Result}; use crate::buffer::LanceBuffer; -use crate::compression::{CompressionStrategy, DefaultCompressionStrategy}; -use crate::compression_config::CompressionParams; +use crate::data::DataBlock; use crate::decoder::PageEncoding; -use crate::encodings::logical::blob::{BlobStructuralEncoder, BlobV2StructuralEncoder}; -use crate::encodings::logical::fixed_size_list::FixedSizeListStructuralEncoder; -use crate::encodings::logical::list::ListStructuralEncoder; -use crate::encodings::logical::map::MapStructuralEncoder; -use crate::encodings::logical::primitive::PrimitiveStructuralEncoder; -use crate::encodings::logical::r#struct::StructStructuralEncoder; use crate::repdef::RepDefBuilder; -use crate::version::LanceFileVersion; use crate::{ decoder::{ColumnInfo, PageInfo}, format::pb, }; +pub use crate::array_encoding::ArrayFieldEncodingStrategy; + +pub mod structural; + /// The minimum alignment for a page buffer. Writers must respect this. pub const MIN_PAGE_BUFFER_ALIGNMENT: u64 = 8; +/// An array encoded with the `pb::ArrayEncoding` grammar. +#[derive(Debug)] +pub struct EncodedArray { + pub data: DataBlock, + pub encoding: pb::ArrayEncoding, +} + +impl EncodedArray { + pub fn new(data: DataBlock, encoding: pb::ArrayEncoding) -> Self { + Self { data, encoding } + } + + pub fn into_buffers(self) -> (Vec, pb::ArrayEncoding) { + (self.data.into_buffers(), self.encoding) + } +} + +/// Encodes one data block and describes it with `pb::ArrayEncoding`. +pub trait ArrayEncoder: std::fmt::Debug + Send + Sync { + fn encode( + &self, + data: DataBlock, + data_type: &arrow_schema::DataType, + buffer_index: &mut u32, + ) -> Result; +} + +/// Selects an `ArrayEncoder` for one page. +pub trait ArrayEncodingStrategy: Send + Sync + std::fmt::Debug { + fn create_array_encoder( + &self, + arrays: &[ArrayRef], + field: &Field, + ) -> Result>; +} + /// An encoded page of data /// /// Maps to a top-level array @@ -235,9 +263,6 @@ pub struct EncodingOptions { /// The encoder needs to know this so it figures the position of out-of-line /// buffers correctly pub buffer_alignment: u64, - - /// The Lance file version being written - pub version: LanceFileVersion, } impl Default for EncodingOptions { @@ -247,20 +272,10 @@ impl Default for EncodingOptions { max_page_bytes: 32 * 1024 * 1024, keep_original_array: true, buffer_alignment: 64, - version: LanceFileVersion::default(), } } } -impl EncodingOptions { - /// If true (for Lance file version 2.2+), miniblock chunk sizes are u32, - /// to allow storing larger chunks and their sizes for better compression. - /// For Lance file version 2.1, miniblock chunk sizes are u16. - pub fn support_large_chunk(&self) -> bool { - self.version >= LanceFileVersion::V2_2 - } -} - /// A trait to pick which kind of field encoding to use for a field /// /// Unlike the ArrayEncodingStrategy, the field encoding strategy is @@ -273,324 +288,23 @@ pub trait FieldEncodingStrategy: Send + Sync + std::fmt::Debug { /// The field encoder can be chosen on the data type as well as /// any metadata that is attached to the field. /// - /// The `encoding_strategy_root` is the encoder that should be - /// used to encode any inner data in struct / list / etc. fields. - /// - /// Initially it is the same as `self` and generally should be - /// forwarded to any inner encoding strategy. fn create_field_encoder( &self, - encoding_strategy_root: &dyn FieldEncodingStrategy, field: &Field, column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, + context: &FieldEncodingContext<'_>, ) -> Result>; } -pub fn default_encoding_strategy(version: LanceFileVersion) -> Box { - match version.resolve() { - LanceFileVersion::Legacy => panic!(), - LanceFileVersion::V2_0 => Box::new( - crate::previous::encoder::CoreFieldEncodingStrategy::new(version), - ), - _ => Box::new(StructuralEncodingStrategy::with_version(version)), - } -} - -/// Create an encoding strategy with user-configured compression parameters -pub fn default_encoding_strategy_with_params( - version: LanceFileVersion, - params: CompressionParams, -) -> Result> { - match version.resolve() { - LanceFileVersion::Legacy | LanceFileVersion::V2_0 => Err(Error::invalid_input( - "Compression parameters are only supported in Lance file version 2.1 and later", - )), - _ => { - let compression_strategy = - Arc::new(DefaultCompressionStrategy::with_params(params).with_version(version)); - Ok(Box::new(StructuralEncodingStrategy { - compression_strategy, - version, - })) - } - } -} - -/// An encoding strategy used for 2.1+ files -#[derive(Debug)] -pub struct StructuralEncodingStrategy { - pub compression_strategy: Arc, - pub version: LanceFileVersion, -} - -// For some reason, clippy thinks we can add Default to the above derive but -// rustc doesn't agree (no default for Arc) -#[allow(clippy::derivable_impls)] -impl Default for StructuralEncodingStrategy { - fn default() -> Self { - Self { - compression_strategy: Arc::new(DefaultCompressionStrategy::new()), - version: LanceFileVersion::default(), - } - } -} - -impl StructuralEncodingStrategy { - pub fn with_version(version: LanceFileVersion) -> Self { - Self { - compression_strategy: Arc::new(DefaultCompressionStrategy::new().with_version(version)), - version, - } - } - - fn is_primitive_type(data_type: &DataType) -> bool { - match data_type { - DataType::FixedSizeList(inner, _) => Self::is_primitive_type(inner.data_type()), - _ => matches!( - data_type, - DataType::Boolean - | DataType::Date32 - | DataType::Date64 - | DataType::Decimal128(_, _) - | DataType::Decimal256(_, _) - | DataType::Duration(_) - | DataType::Float16 - | DataType::Float32 - | DataType::Float64 - | DataType::Int16 - | DataType::Int32 - | DataType::Int64 - | DataType::Int8 - | DataType::Interval(_) - | DataType::Null - | DataType::Time32(_) - | DataType::Time64(_) - | DataType::Timestamp(_, _) - | DataType::UInt16 - | DataType::UInt32 - | DataType::UInt64 - | DataType::UInt8 - | DataType::FixedSizeBinary(_) - | DataType::Binary - | DataType::LargeBinary - | DataType::Utf8 - | DataType::LargeUtf8, - ), - } - } - - fn do_create_field_encoder( - &self, - _encoding_strategy_root: &dyn FieldEncodingStrategy, - field: &Field, - column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, - root_field_metadata: &HashMap, - ) -> Result> { - let data_type = field.data_type(); - - // Check if field is marked as blob - if field.is_blob() { - match data_type { - DataType::Binary | DataType::LargeBinary => { - return Ok(Box::new(BlobStructuralEncoder::new( - field, - column_index.next_column_index(field.id as u32), - options, - self.compression_strategy.clone(), - )?)); - } - DataType::Struct(_) if self.version >= LanceFileVersion::V2_2 => { - return Ok(Box::new(BlobV2StructuralEncoder::new( - field, - column_index.next_column_index(field.id as u32), - options, - self.compression_strategy.clone(), - )?)); - } - DataType::Struct(_) => { - return Err(Error::invalid_input_source( - "Blob v2 struct input requires file version >= 2.2".into(), - )); - } - _ => { - return Err(Error::invalid_input_source( - format!( - "Blob encoding only supports Binary/LargeBinary or v2 Struct, got {}", - data_type - ) - .into(), - )); - } - } - } - - if Self::is_primitive_type(&data_type) { - Ok(Box::new(PrimitiveStructuralEncoder::try_new( - options, - self.compression_strategy.clone(), - column_index.next_column_index(field.id as u32), - field.clone(), - Arc::new(root_field_metadata.clone()), - )?)) - } else { - match data_type { - DataType::List(_) | DataType::LargeList(_) => { - let child = field.children.first().expect_ok()?; - let child_encoder = self.do_create_field_encoder( - _encoding_strategy_root, - child, - column_index, - options, - root_field_metadata, - )?; - Ok(Box::new(ListStructuralEncoder::new( - options.keep_original_array, - child_encoder, - ))) - } - DataType::FixedSizeList(inner, _) - if matches!(inner.data_type(), DataType::Struct(_)) => - { - if self.version < LanceFileVersion::V2_2 { - return Err(Error::not_supported_source(format!( - "FixedSizeList is only supported in Lance file format 2.2+, current version: {}", - self.version - ) - .into())); - } - // Complex FixedSizeList needs structural encoding - let child = field.children.first().expect_ok()?; - let child_encoder = self.do_create_field_encoder( - _encoding_strategy_root, - child, - column_index, - options, - root_field_metadata, - )?; - Ok(Box::new(FixedSizeListStructuralEncoder::new( - options.keep_original_array, - child_encoder, - ))) - } - DataType::Map(_, keys_sorted) => { - // TODO: We only support keys_sorted=false for now, - // because converting a rust arrow map field to the python arrow field will - // lose the keys_sorted property. - if keys_sorted { - return Err(Error::not_supported_source(format!("Map data type is not supported with keys_sorted=true now, current value is {}", keys_sorted).into())); - } - if self.version < LanceFileVersion::V2_2 { - return Err(Error::not_supported_source(format!( - "Map data type is only supported in Lance file format 2.2+, current version: {}", - self.version - ) - .into())); - } - let entries_child = field.children.first().ok_or_else(|| { - Error::schema("Map should have an entries child".to_string()) - })?; - let DataType::Struct(struct_fields) = entries_child.data_type() else { - return Err(Error::schema( - "Map entries field must be a Struct".to_string(), - )); - }; - if struct_fields.len() < 2 { - return Err(Error::schema( - "Map entries struct must contain both key and value fields".to_string(), - )); - } - let key_field = &struct_fields[0]; - if key_field.is_nullable() { - return Err(Error::schema(format!( - "Map key field '{}' must be non-nullable according to Arrow Map specification", - key_field.name() - ))); - } - let child_encoder = self.do_create_field_encoder( - _encoding_strategy_root, - entries_child, - column_index, - options, - root_field_metadata, - )?; - Ok(Box::new(MapStructuralEncoder::new( - options.keep_original_array, - child_encoder, - ))) - } - DataType::Struct(fields) => { - if field.is_packed_struct() || fields.is_empty() { - // Both packed structs and empty structs are encoded as primitive - Ok(Box::new(PrimitiveStructuralEncoder::try_new( - options, - self.compression_strategy.clone(), - column_index.next_column_index(field.id as u32), - field.clone(), - Arc::new(root_field_metadata.clone()), - )?)) - } else { - let children_encoders = field - .children - .iter() - .map(|field| { - self.do_create_field_encoder( - _encoding_strategy_root, - field, - column_index, - options, - root_field_metadata, - ) - }) - .collect::>>()?; - Ok(Box::new(StructStructuralEncoder::new( - options.keep_original_array, - children_encoders, - ))) - } - } - DataType::Dictionary(_, value_type) => { - // A dictionary of primitive is, itself, primitive - if Self::is_primitive_type(&value_type) { - Ok(Box::new(PrimitiveStructuralEncoder::try_new( - options, - self.compression_strategy.clone(), - column_index.next_column_index(field.id as u32), - field.clone(), - Arc::new(root_field_metadata.clone()), - )?)) - } else { - // A dictionary of logical is, itself, logical and we don't support that today - // It could be possible (e.g. store indices in one column and values in remaining columns) - // but would be a significant amount of work - // - // An easier fallback implementation would be to decode-on-write and encode-on-read - Err(Error::not_supported_source(format!("cannot encode a dictionary column whose value type is a logical type ({})", value_type).into())) - } - } - _ => todo!("Implement encoding for field {}", field), - } - } - } -} - -impl FieldEncodingStrategy for StructuralEncodingStrategy { - fn create_field_encoder( - &self, - encoding_strategy_root: &dyn FieldEncodingStrategy, - field: &Field, - column_index: &mut ColumnIndexSequence, - options: &EncodingOptions, - ) -> Result> { - self.do_create_field_encoder( - encoding_strategy_root, - field, - column_index, - options, - &field.metadata, - ) - } +/// Context shared while one top-level field and all of its children are mapped +/// to concrete field encoders. +pub struct FieldEncodingContext<'a> { + /// The complete strategy composition used for recursive child fields. + pub strategy: &'a dyn FieldEncodingStrategy, + /// Runtime-only writer options. + pub options: &'a EncodingOptions, + /// Metadata inherited from the top-level field. + pub root_field_metadata: &'a HashMap, } /// A batch encoder that encodes RecordBatch objects by delegating @@ -612,12 +326,13 @@ impl BatchEncoder { .fields .iter() .map(|field| { - let encoder = strategy.create_field_encoder( + let context = FieldEncodingContext { strategy, - field, - &mut col_idx_sequence, options, - )?; + root_field_metadata: &field.metadata, + }; + let encoder = + strategy.create_field_encoder(field, &mut col_idx_sequence, &context)?; col_idx += encoder.as_ref().num_columns(); Ok(encoder) }) @@ -768,47 +483,65 @@ pub async fn encode_batch( #[cfg(test)] mod tests { use super::*; - use crate::compression_config::{CompressionFieldParams, CompressionParams}; + use crate::testing::{TestEncoding, create_test_field_encoder, test_encoding_strategy}; + use arrow_array::make_array; + use arrow_buffer::Buffer; + use arrow_data::ArrayData; use arrow_schema::{DataType as ArrowDataType, Field as ArrowField, Fields as ArrowFields}; - - #[test] - fn test_configured_encoding_strategy() { - // Create test parameters - let mut params = CompressionParams::new(); - params.columns.insert( - "*_id".to_string(), - CompressionFieldParams { - rle_threshold: Some(0.5), - compression: Some("lz4".to_string()), - compression_level: None, - bss: None, - minichunk_size: None, - }, - ); - - // Test with V2.1 - should succeed - let strategy = - default_encoding_strategy_with_params(LanceFileVersion::V2_1, params.clone()) - .expect("Should succeed for V2.1"); - - // Verify it's a StructuralEncodingStrategy - assert!(format!("{:?}", strategy).contains("StructuralEncodingStrategy")); - assert!(format!("{:?}", strategy).contains("DefaultCompressionStrategy")); - - // Test with V2.0 - should fail - let err = default_encoding_strategy_with_params(LanceFileVersion::V2_0, params.clone()) - .expect_err("Should fail for V2.0"); + use rstest::rstest; + + #[rstest] + fn test_nested_variable_width_offsets_are_validated_before_dispatch( + #[values(TestEncoding::Array, TestEncoding::StructuralU32)] encoding: TestEncoding, + #[values(ArrowDataType::Utf8, ArrowDataType::LargeUtf8)] item_type: ArrowDataType, + ) { + let offsets = match &item_type { + ArrowDataType::Utf8 => Buffer::from_slice_ref([0_i32, 2, 1, 3]), + ArrowDataType::LargeUtf8 => Buffer::from_slice_ref([0_i64, 2, 1, 3]), + _ => unreachable!(), + }; + let child_data = unsafe { + ArrayData::builder(item_type.clone()) + .len(3) + .add_buffer(offsets) + .add_buffer(Buffer::from(b"abc")) + .build_unchecked() + }; + let item_field = Arc::new(ArrowField::new("item", item_type, false)); + let data_type = ArrowDataType::FixedSizeList(item_field, 1); + let array_data = unsafe { + ArrayData::builder(data_type.clone()) + .len(3) + .add_child_data(child_data) + .build_unchecked() + }; + let array = make_array(array_data); + let field = Field::try_from(&ArrowField::new("payload", data_type, false)).unwrap(); + let strategy = test_encoding_strategy(encoding); + let mut column_index = ColumnIndexSequence::default(); + let options = EncodingOptions { + cache_bytes_per_column: 0, + ..Default::default() + }; + let mut encoder = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options) + .unwrap(); + let mut external_buffers = OutOfLineBuffers::new(0, MIN_PAGE_BUFFER_ALIGNMENT); + + let error = encoder + .maybe_encode(array, &mut external_buffers, RepDefBuilder::default(), 0, 3) + .err() + .expect("malformed nested offsets should fail before task dispatch"); + + assert!(matches!(error, Error::InvalidInput { .. })); + let message = error.to_string(); assert!( - err.to_string() - .contains("only supported in Lance file version 2.1") + message.contains("field 'payload'"), + "unexpected message: {message}" ); - - // Test with Legacy - should fail - let err = default_encoding_strategy_with_params(LanceFileVersion::Legacy, params) - .expect_err("Should fail for Legacy"); assert!( - err.to_string() - .contains("only supported in Lance file version 2.1") + message.contains("non-monotonic offset at position 2"), + "unexpected message: {message}" ); } @@ -830,11 +563,12 @@ mod tests { ); let field = Field::try_from(&arrow_field).unwrap(); - let strategy = StructuralEncodingStrategy::with_version(LanceFileVersion::V2_1); + let strategy = test_encoding_strategy(TestEncoding::StructuralU16); let mut column_index = ColumnIndexSequence::default(); let options = EncodingOptions::default(); - let result = strategy.create_field_encoder(&strategy, &field, &mut column_index, &options); + let result = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options); assert!( result.is_err(), "FixedSizeList should be rejected for file version 2.1" @@ -843,7 +577,7 @@ mod tests { assert!( err.to_string() - .contains("FixedSizeList is only supported in Lance file format 2.2+") + .contains("FixedSizeList is not enabled by the selected file format") ); } } diff --git a/rust/lance-encoding/src/encoder/structural.rs b/rust/lance-encoding/src/encoder/structural.rs new file mode 100644 index 00000000000..3d4e49f42c8 --- /dev/null +++ b/rust/lance-encoding/src/encoder/structural.rs @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Version-free structural field encoder builders. + +use std::sync::Arc; + +use arrow_schema::DataType; +use lance_core::{Error, Result, datatypes::Field, error::LanceOptionExt}; + +pub use crate::encodings::logical::primitive::PrimitivePageEncoding; + +use crate::encodings::logical::{ + blob::{BlobStructuralEncoder, BlobV2StructuralEncoder}, + fixed_size_list::FixedSizeListStructuralEncoder, + list::ListStructuralEncoder, + map::MapStructuralEncoder, + primitive::PrimitiveStructuralEncoder, + r#struct::StructStructuralEncoder, +}; + +use super::{ColumnIndexSequence, FieldEncoder, FieldEncodingContext}; + +/// Encode primitive leaves, primitive fixed-size lists, dictionaries, and +/// packed or empty structs using one concrete primitive page grammar. +#[derive(Debug, Clone)] +pub struct PrimitiveFieldEncoding { + page_encodings: Arc<[PrimitivePageEncoding]>, +} + +impl PrimitiveFieldEncoding { + /// Create a primitive field mechanism from ordered executable page behaviors. + pub fn new(page_encodings: impl IntoIterator) -> Self { + Self { + page_encodings: page_encodings.into_iter().collect(), + } + } + + fn is_primitive_type(data_type: &DataType) -> bool { + match data_type { + DataType::FixedSizeList(inner, _) => Self::is_primitive_type(inner.data_type()), + _ => matches!( + data_type, + DataType::Boolean + | DataType::Date32 + | DataType::Date64 + | DataType::Decimal128(_, _) + | DataType::Decimal256(_, _) + | DataType::Duration(_) + | DataType::Float16 + | DataType::Float32 + | DataType::Float64 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Int8 + | DataType::Interval(_) + | DataType::Null + | DataType::Time32(_) + | DataType::Time64(_) + | DataType::Timestamp(_, _) + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::UInt8 + | DataType::FixedSizeBinary(_) + | DataType::Binary + | DataType::LargeBinary + | DataType::Utf8 + | DataType::LargeUtf8, + ), + } + } + + fn create_at( + &self, + field: Field, + column_index: u32, + context: &FieldEncodingContext<'_>, + ) -> Result> { + Ok(Box::new(PrimitiveStructuralEncoder::try_new( + context.options, + self.page_encodings.clone(), + column_index, + field, + Arc::new(context.root_field_metadata.clone()), + )?)) + } + + /// Create a primitive field encoder when this mechanism recognizes `field`. + pub fn try_create( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result>> { + if field.is_blob() { + return Ok(None); + } + + let data_type = field.data_type(); + let is_primitive = Self::is_primitive_type(&data_type); + let is_packed_or_empty_struct = matches!( + &data_type, + DataType::Struct(fields) if field.is_packed_struct() || fields.is_empty() + ); + let is_primitive_dictionary = matches!( + &data_type, + DataType::Dictionary(_, value_type) if Self::is_primitive_type(value_type) + ); + + if !is_primitive && !is_packed_or_empty_struct && !is_primitive_dictionary { + if let DataType::Dictionary(_, value_type) = data_type { + return Err(Error::not_supported_source( + format!( + "cannot encode a dictionary column whose value type is a logical type ({})", + value_type + ) + .into(), + )); + } + return Ok(None); + } + + Ok(Some(self.create_at( + field.clone(), + column_index.next_column_index(field.id as u32), + context, + )?)) + } +} + +/// Create the original binary blob descriptor when `field` matches. +pub fn try_create_binary_blob( + primitive: &PrimitiveFieldEncoding, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !field.is_blob() || !matches!(field.data_type(), DataType::Binary | DataType::LargeBinary) { + return Ok(None); + } + let descriptor_column_index = column_index.next_column_index(field.id as u32); + Ok(Some(Box::new(BlobStructuralEncoder::new( + field, + |descriptor_field| primitive.create_at(descriptor_field, descriptor_column_index, context), + )?))) +} + +/// Create the structural blob descriptor when `field` matches. +pub fn try_create_structural_blob( + primitive: &PrimitiveFieldEncoding, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !field.is_blob() || !matches!(field.data_type(), DataType::Struct(_)) { + return Ok(None); + } + let descriptor_column_index = column_index.next_column_index(field.id as u32); + Ok(Some(Box::new(BlobV2StructuralEncoder::new( + field, + |descriptor_field| primitive.create_at(descriptor_field, descriptor_column_index, context), + )?))) +} + +/// Create a variable-size list encoder when `field` matches. +pub fn try_create_list( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !matches!( + field.data_type(), + DataType::List(_) | DataType::LargeList(_) + ) { + return Ok(None); + } + let child = field.children.first().expect_ok()?; + let child_encoder = context + .strategy + .create_field_encoder(child, column_index, context)?; + Ok(Some(Box::new(ListStructuralEncoder::new( + context.options.keep_original_array, + child_encoder, + )))) +} + +/// Create a fixed-size-list encoder whose child is a struct when applicable. +pub fn try_create_structural_fixed_size_list( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + if !matches!( + field.data_type(), + DataType::FixedSizeList(inner, _) if matches!(inner.data_type(), DataType::Struct(_)) + ) { + return Ok(None); + } + let child = field.children.first().expect_ok()?; + let child_encoder = context + .strategy + .create_field_encoder(child, column_index, context)?; + Ok(Some(Box::new(FixedSizeListStructuralEncoder::new( + context.options.keep_original_array, + child_encoder, + )))) +} + +/// Create an Arrow map encoder when `field` matches. +pub fn try_create_map( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + let DataType::Map(_, keys_sorted) = field.data_type() else { + return Ok(None); + }; + if keys_sorted { + return Err(Error::not_supported_source( + format!( + "Map data type is not supported with keys_sorted=true now, current value is {}", + keys_sorted + ) + .into(), + )); + } + let entries_child = field + .children + .first() + .ok_or_else(|| Error::schema("Map should have an entries child".to_string()))?; + let DataType::Struct(struct_fields) = entries_child.data_type() else { + return Err(Error::schema( + "Map entries field must be a Struct".to_string(), + )); + }; + if struct_fields.len() < 2 { + return Err(Error::schema( + "Map entries struct must contain both key and value fields".to_string(), + )); + } + let key_field = &struct_fields[0]; + if key_field.is_nullable() { + return Err(Error::schema(format!( + "Map key field '{}' must be non-nullable according to Arrow Map specification", + key_field.name() + ))); + } + let child_encoder = + context + .strategy + .create_field_encoder(entries_child, column_index, context)?; + Ok(Some(Box::new(MapStructuralEncoder::new( + context.options.keep_original_array, + child_encoder, + )))) +} + +/// Create a non-packed, non-empty struct encoder when `field` matches. +pub fn try_create_struct( + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, +) -> Result>> { + let DataType::Struct(fields) = field.data_type() else { + return Ok(None); + }; + if field.is_blob() || field.is_packed_struct() || fields.is_empty() { + return Ok(None); + } + let children_encoders = field + .children + .iter() + .map(|child| { + context + .strategy + .create_field_encoder(child, column_index, context) + }) + .collect::>>()?; + Ok(Some(Box::new(StructStructuralEncoder::new( + context.options.keep_original_array, + children_encoders, + )))) +} diff --git a/rust/lance-encoding/src/encodings/fuzz_tests.rs b/rust/lance-encoding/src/encodings/fuzz_tests.rs index b92bac09cca..7fb2cb123b8 100644 --- a/rust/lance-encoding/src/encodings/fuzz_tests.rs +++ b/rust/lance-encoding/src/encodings/fuzz_tests.rs @@ -14,9 +14,9 @@ use arrow_array::builder::{Int32Builder, ListBuilder}; use arrow_array::*; use arrow_schema::{DataType, Field}; use proptest::prelude::*; +use proptest::test_runner::{Config, TestRunner}; -use crate::testing::{TestCases, check_round_trip_encoding_of_data}; -use crate::version::LanceFileVersion; +use crate::testing::{TestCases, TestEncoding, check_round_trip_encoding_of_data}; use lance_core::Result; use lance_datagen::{ArrayGenerator, ByteCount, Dimension, RowCount, Seed, array, gen_batch}; @@ -253,46 +253,54 @@ fn generate_test_data_for_config( Ok(batch.column(0).clone()) } -// Main property test for encoding round-trip -proptest! { - #![proptest_config(ProptestConfig::with_cases(50))] - - #[test] - fn test_encoding_round_trip( - config in encoding_config_strategy(), - num_rows in 100..=5000usize, - seed in any::() - ) { - let rt = tokio::runtime::Runtime::new().unwrap(); - - rt.block_on(async { - // Generate test data - let test_data = generate_test_data_for_config(&config, num_rows, seed) - .expect("Failed to generate test data"); - - // Set up test cases - let _field = config.to_field("test"); - - let mut metadata = HashMap::new(); - // Force specific encoding through metadata hints if needed - if config.encoding_type == EncodingType::Miniblock { - metadata.insert("encoding_hint".to_string(), "miniblock".to_string()); - } - - let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) - .with_batch_size(100) - .with_range(0..num_rows.min(500) as u64) - .with_indices(vec![0, num_rows as u64 / 2, (num_rows - 1) as u64]); +#[rstest::rstest] +fn test_encoding_round_trip( + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49 + )] + _shard: usize, +) { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let strategy = (encoding_config_strategy(), 100..=1000usize, any::()); + let mut runner = TestRunner::new(Config { + cases: 1, + ..Config::default() + }); + runner + .run(&strategy, |(config, num_rows, seed)| { + rt.block_on(async { + let test_data = generate_test_data_for_config(&config, num_rows, seed) + .expect("Failed to generate test data"); + + let _field = config.to_field("test"); + + let mut metadata = HashMap::new(); + if config.encoding_type == EncodingType::Miniblock { + metadata.insert("encoding_hint".to_string(), "miniblock".to_string()); + } - // Execute round-trip test - check_round_trip_encoding_of_data( - vec![test_data], - &test_cases, - metadata - ).await; - }); - } + let test_cases = TestCases::default() + .with_encoding(encoding) + .with_batch_size(100) + .with_range(0..num_rows.min(500) as u64) + .with_indices(vec![0, num_rows as u64 / 2, (num_rows - 1) as u64]); + + check_round_trip_encoding_of_data(vec![test_data], &test_cases, metadata).await; + }); + Ok(()) + }) + .unwrap(); } #[tokio::test] @@ -301,7 +309,7 @@ async fn test_edge_cases_single_value() { let single_int32 = Arc::new(Int32Array::from(vec![42])) as Arc; let single_string = Arc::new(StringArray::from(vec!["test"])) as Arc; - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![single_int32], &test_cases, HashMap::new()).await; @@ -317,7 +325,7 @@ async fn test_edge_cases_all_nulls() { vec![None, None, None] as Vec> )) as Arc; - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![all_nulls_int32], &test_cases, HashMap::new()).await; @@ -347,7 +355,7 @@ proptest! { let list_array = Arc::new(list_builder.finish()) as Arc; let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_range(0..list_sizes.len().min(50) as u64); check_round_trip_encoding_of_data( @@ -359,37 +367,42 @@ proptest! { } } -// Test fixed size list encoding -proptest! { - #[test] - fn test_fixed_size_list_encoding( - list_size in 1..=100i32, - num_rows in 10..=1000usize, - seed in any::() - ) { - let rt = tokio::runtime::Runtime::new().unwrap(); - - rt.block_on(async { - let config = EncodingTestConfig { - encoding_type: EncodingType::Miniblock, - data_structure: DataStructure::FixedSizeList(list_size), - data_width: DataWidth::Fixed(FixedWidthType::Int32), - nullable: false, - }; - - let test_data = generate_test_data_for_config(&config, num_rows, seed) - .expect("Failed to generate test data"); - - let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1); - - check_round_trip_encoding_of_data( - vec![test_data], - &test_cases, - HashMap::new() - ).await; - }); - } +#[rstest::rstest] +fn test_fixed_size_list_encoding( + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, +) { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let strategy = (1..=100i32, 10..=1000usize, any::()); + let mut runner = TestRunner::new(Config::default()); + runner + .run(&strategy, |(list_size, num_rows, seed)| { + rt.block_on(async { + let config = EncodingTestConfig { + encoding_type: EncodingType::Miniblock, + data_structure: DataStructure::FixedSizeList(list_size), + data_width: DataWidth::Fixed(FixedWidthType::Int32), + nullable: false, + }; + + let test_data = generate_test_data_for_config(&config, num_rows, seed) + .expect("Failed to generate test data"); + + let test_cases = TestCases::default().with_encoding(encoding); + + check_round_trip_encoding_of_data(vec![test_data], &test_cases, HashMap::new()) + .await; + }); + Ok(()) + }) + .unwrap(); } #[tokio::test] @@ -424,7 +437,7 @@ async fn test_list_dict_empty_batch() { let list_array = Arc::new(list_builder.finish()); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() // Read only the empty/null lists (rows 50-99) // This batch will have 0 underlying values .with_range(50..100); @@ -539,7 +552,7 @@ async fn test_all_valid_combinations() { let test_data = generate_test_data_for_config(&config, 100, 42).expect("Failed to generate test data"); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![test_data], &test_cases, HashMap::new()).await; } diff --git a/rust/lance-encoding/src/encodings/logical/blob.rs b/rust/lance-encoding/src/encodings/logical/blob.rs index cad2112bafe..a5432fdc54c 100644 --- a/rust/lance-encoding/src/encodings/logical/blob.rs +++ b/rust/lance-encoding/src/encodings/logical/blob.rs @@ -13,7 +13,9 @@ use arrow_buffer::Buffer; use arrow_schema::{DataType, Field as ArrowField, Fields}; use futures::{FutureExt, future::BoxFuture}; use lance_core::{ - Error, Result, datatypes::BLOB_V2_DESC_FIELDS, datatypes::Field, error::LanceOptionExt, + Error, Result, + datatypes::{BLOB_V2_DESC_FIELDS, BlobV2Layout, Field}, + error::LanceOptionExt, }; use crate::{ @@ -21,7 +23,6 @@ use crate::{ constants::PACKED_STRUCT_META_KEY, decoder::PageEncoding, encoder::{EncodeTask, EncodedColumn, EncodedPage, FieldEncoder, OutOfLineBuffers}, - encodings::logical::primitive::PrimitiveStructuralEncoder, format::ProtobufUtils21, repdef::{DefinitionInterpretation, RepDefBuilder}, }; @@ -42,9 +43,7 @@ pub struct BlobStructuralEncoder { impl BlobStructuralEncoder { pub fn new( field: &Field, - column_index: u32, - options: &crate::encoder::EncodingOptions, - compression_strategy: Arc, + make_descriptor_encoder: impl FnOnce(Field) -> Result>, ) -> Result { // Create descriptor field: struct // Preserve the original field's metadata for packed struct @@ -63,13 +62,7 @@ impl BlobStructuralEncoder { )?; // Use PrimitiveStructuralEncoder to handle the descriptor - let descriptor_encoder = Box::new(PrimitiveStructuralEncoder::try_new( - options, - compression_strategy, - column_index, - descriptor_field, - Arc::new(HashMap::new()), - )?); + let descriptor_encoder = make_descriptor_encoder(descriptor_field)?; Ok(Self { descriptor_encoder, @@ -137,14 +130,18 @@ impl FieldEncoder for BlobStructuralEncoder { let def = repdef.definition_levels.as_ref(); let def_meaning: Arc<[DefinitionInterpretation]> = repdef.def_meaning.into(); - match self.def_meaning.as_ref() { - None => { - self.def_meaning = Some(def_meaning.clone()); + // A blob page stores one definition interpretation for all of its rows. + // The descriptor encoder can buffer multiple input arrays, so finish the + // pending page before a later array changes from all-valid to nullable (or + // vice versa). + let mut encode_tasks = match self.def_meaning.as_ref() { + Some(existing) if existing != &def_meaning => { + let existing = existing.clone(); + Self::wrap_tasks(self.descriptor_encoder.flush(external_buffers)?, existing) } - Some(existing) => { - debug_assert_eq!(existing, &def_meaning); - } - } + _ => Vec::new(), + }; + self.def_meaning = Some(def_meaning.clone()); // Collect positions and sizes let mut positions = Vec::with_capacity(binary_array.len()); @@ -192,15 +189,16 @@ impl FieldEncoder for BlobStructuralEncoder { )); // Delegate to descriptor encoder - let encode_tasks = self.descriptor_encoder.maybe_encode( + let descriptor_tasks = self.descriptor_encoder.maybe_encode( descriptor_array, external_buffers, RepDefBuilder::default(), row_number, num_rows, )?; + encode_tasks.extend(Self::wrap_tasks(descriptor_tasks, def_meaning)); - Ok(Self::wrap_tasks(encode_tasks, def_meaning)) + Ok(encode_tasks) } fn flush(&mut self, external_buffers: &mut OutOfLineBuffers) -> Result> { @@ -236,9 +234,7 @@ pub struct BlobV2StructuralEncoder { impl BlobV2StructuralEncoder { pub fn new( field: &Field, - column_index: u32, - options: &crate::encoder::EncodingOptions, - compression_strategy: Arc, + make_descriptor_encoder: impl FnOnce(Field) -> Result>, ) -> Result { let mut descriptor_metadata = HashMap::with_capacity(1); descriptor_metadata.insert(PACKED_STRUCT_META_KEY.to_string(), "true".to_string()); @@ -250,13 +246,7 @@ impl BlobV2StructuralEncoder { .with_metadata(descriptor_metadata), )?; - let descriptor_encoder = Box::new(PrimitiveStructuralEncoder::try_new( - options, - compression_strategy, - column_index, - descriptor_field, - Arc::new(HashMap::new()), - )?); + let descriptor_encoder = make_descriptor_encoder(descriptor_field)?; Ok(Self { descriptor_encoder }) } @@ -267,15 +257,30 @@ impl FieldEncoder for BlobV2StructuralEncoder { &mut self, array: ArrayRef, external_buffers: &mut OutOfLineBuffers, - mut repdef: RepDefBuilder, + repdef: RepDefBuilder, row_number: u64, num_rows: u64, ) -> Result> { - let struct_arr = array.as_struct(); - if let Some(validity) = struct_arr.nulls() { - repdef.add_validity_bitmap(validity.clone()); - } else { - repdef.add_no_null(struct_arr.len()); + let struct_arr = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Blob v2 encoder expected StructArray, got {}", + array.data_type() + ) + .into(), + ) + })?; + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Prepared) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input_source( + format!("Blob v2 encoder expected prepared array layout, got {actual} layout") + .into(), + )); } let kind_col = struct_arr @@ -403,7 +408,7 @@ impl FieldEncoder for BlobV2StructuralEncoder { let descriptor_array = Arc::new(StructArray::try_new( BLOB_V2_DESC_FIELDS.clone(), children, - None, + struct_arr.nulls().cloned(), )?) as ArrayRef; self.descriptor_encoder.maybe_encode( @@ -435,33 +440,82 @@ impl FieldEncoder for BlobV2StructuralEncoder { mod tests { use super::*; use crate::{ - compression::DefaultCompressionStrategy, encoder::{ColumnIndexSequence, EncodingOptions}, testing::{ - TestCases, check_round_trip_encoding_of_data, - check_round_trip_encoding_of_data_with_expected, + TestCases, TestEncoding, check_round_trip_encoding_of_data, + check_round_trip_encoding_of_data_with_expected, create_test_field_encoder, + test_encoding_strategy, }, - version::LanceFileVersion, }; use arrow_array::{ ArrayRef, LargeBinaryArray, StringArray, StructArray, UInt8Array, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field as ArrowField}; + use lance_core::datatypes::BLOB_V2_LOGICAL_MINIMAL_FIELDS; #[test] fn test_blob_encoder_creation() { - let field = - Field::try_from(ArrowField::new("blob_field", DataType::LargeBinary, true)).unwrap(); + let field = Field::try_from( + ArrowField::new("blob_field", DataType::LargeBinary, true).with_metadata( + HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]), + ), + ) + .unwrap(); let mut column_index = ColumnIndexSequence::default(); - let column_idx = column_index.next_column_index(0); let options = EncodingOptions::default(); - let compression = Arc::new(DefaultCompressionStrategy::new()); + let strategy = test_encoding_strategy(TestEncoding::StructuralU16); - let encoder = BlobStructuralEncoder::new(&field, column_idx, &options, compression); + let encoder = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options); assert!(encoder.is_ok()); } + #[test] + fn test_blob_v2_encoder_rejects_logical_array_layout() { + let field = Field::try_from( + ArrowField::new( + "blob_field", + DataType::Struct(BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone()), + true, + ) + .with_metadata(HashMap::from([( + lance_arrow::ARROW_EXT_NAME_KEY.to_string(), + lance_arrow::BLOB_V2_EXT_NAME.to_string(), + )])), + ) + .unwrap(); + let mut column_index = ColumnIndexSequence::default(); + let options = EncodingOptions::default(); + let strategy = test_encoding_strategy(TestEncoding::StructuralU32); + let mut encoder = + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options) + .unwrap(); + let array = Arc::new( + StructArray::try_new( + BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone(), + vec![ + Arc::new(LargeBinaryArray::from(vec![Some(b"payload".as_ref())])) as ArrayRef, + Arc::new(StringArray::from(vec![None::<&str>])) as ArrayRef, + ], + None, + ) + .unwrap(), + ) as ArrayRef; + let mut external_buffers = OutOfLineBuffers::new(0, 8); + let Err(error) = + encoder.maybe_encode(array, &mut external_buffers, RepDefBuilder::default(), 0, 1) + else { + panic!("logical array layout unexpectedly reached the descriptor encoder"); + }; + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("expected prepared array layout, got logical layout") + ); + } + #[tokio::test] async fn test_blob_encoding_simple() { let field = Field::try_from( @@ -471,12 +525,12 @@ mod tests { ) .unwrap(); let mut column_index = ColumnIndexSequence::default(); - let column_idx = column_index.next_column_index(0); let options = EncodingOptions::default(); - let compression = Arc::new(DefaultCompressionStrategy::new()); + let strategy = test_encoding_strategy(TestEncoding::StructuralU16); let mut encoder = - BlobStructuralEncoder::new(&field, column_idx, &options, compression).unwrap(); + create_test_field_encoder(strategy.as_ref(), &field, &mut column_index, &options) + .unwrap(); // Create test data with larger blobs let large_data = vec![0u8; 1024 * 100]; // 100KB blob @@ -529,7 +583,53 @@ mod tests { // Use the standard test harness check_round_trip_encoding_of_data( vec![array], - &TestCases::default().with_max_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_array_and_u16_encodings(), + blob_metadata, + ) + .await; + } + + #[tokio::test] + async fn test_blob_round_trip_empty_values() { + // Empty values share size == 0 with nulls in the descriptor layout + // and schedule no read; each must decode to zero-length bytes without + // consuming the read result of a following non-empty blob. Empties + // are placed before payloads so a misassignment corrupts the output + // instead of only exhausting the read iterator. + let blob_metadata = + HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]); + + let val1: &[u8] = &vec![1u8; 1024]; + let val2: &[u8] = &vec![2u8; 10240]; + let empty: &[u8] = &[]; + let array = Arc::new(LargeBinaryArray::from(vec![ + Some(empty), + Some(val1), + None, + Some(empty), + Some(val2), + None, + Some(empty), + ])); + + check_round_trip_encoding_of_data(vec![array], &TestCases::default(), blob_metadata).await; + } + + #[tokio::test] + async fn test_blob_round_trip_varying_chunk_nullability() { + let blob_metadata = + HashMap::from([(lance_arrow::BLOB_META_KEY.to_string(), "true".to_string())]); + let all_valid = Arc::new(LargeBinaryArray::from(vec![Some(b"first".as_ref())])); + let with_null = Arc::new(LargeBinaryArray::from(vec![ + Some(b"second".as_ref()), + None, + Some(b"".as_ref()), + ])); + let all_valid_again = Arc::new(LargeBinaryArray::from(vec![Some(b"last".as_ref())])); + + check_round_trip_encoding_of_data( + vec![all_valid, with_null, all_valid_again], + &TestCases::default().with_encoding(TestEncoding::StructuralU16), blob_metadata, ) .await; @@ -607,7 +707,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; @@ -672,7 +772,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; @@ -734,7 +834,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; @@ -796,7 +896,7 @@ mod tests { check_round_trip_encoding_of_data_with_expected( vec![Arc::new(struct_array)], Some(Arc::new(expected_descriptor)), - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), blob_metadata, ) .await; diff --git a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs index 9e8e3e109ea..f9a71714d21 100644 --- a/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs +++ b/rust/lance-encoding/src/encodings/logical/fixed_size_list.rs @@ -65,7 +65,7 @@ impl FieldEncoder for FixedSizeListStructuralEncoder { } else { deep_copy_nulls(array.nulls()) }; - repdef.add_fsl(validity.clone(), dimension, num_rows as usize); + repdef.add_fsl(validity.clone(), dimension, fsl_arr.len()); // FSL forces child elements to exist even under null rows. Normalize any // nested lists under null FSL rows to null empty lists. @@ -255,7 +255,7 @@ impl StructuralDecodeArrayTask for StructuralFixedSizeListDecodeTask { match &self.data_type { DataType::FixedSizeList(child_field, dimension) => { let num_rows = self.num_rows as usize; - let validity = repdef.unravel_fsl_validity(num_rows, *dimension as usize); + let validity = repdef.unravel_fsl_validity(num_rows, *dimension as usize)?; let fsl_array = arrow_array::FixedSizeListArray::try_new( child_field.clone(), *dimension, @@ -533,8 +533,7 @@ mod tests { STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK, }, - testing::{TestCases, check_specific_random}, - version::LanceFileVersion, + testing::{TestCases, TestEncoding, check_specific_random}, }; fn make_fsl_struct_type(struct_fields: Fields, dimension: i32) -> DataType { @@ -688,20 +687,24 @@ mod tests { } #[rstest] - #[case::simple(simple_struct_fields(), 2, LanceFileVersion::V2_2)] - #[case::nested_struct(nested_struct_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_list(struct_with_list_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_large_list(struct_with_large_list_fields(), 2, LanceFileVersion::V2_2)] - #[case::nested_struct_with_list(nested_struct_with_list_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_nested_fsl(struct_with_nested_fsl_fields(), 2, LanceFileVersion::V2_2)] - #[case::struct_with_map(struct_with_map_fields(), 2, LanceFileVersion::V2_2)] + #[case::simple(simple_struct_fields(), 2)] + #[case::nested_struct(nested_struct_fields(), 2)] + #[case::struct_with_list(struct_with_list_fields(), 2)] + #[case::struct_with_large_list(struct_with_large_list_fields(), 2)] + #[case::nested_struct_with_list(nested_struct_with_list_fields(), 2)] + #[case::struct_with_nested_fsl(struct_with_nested_fsl_fields(), 2)] + #[case::struct_with_map(struct_with_map_fields(), 2)] #[test_log::test(tokio::test)] async fn test_fsl_struct_random( #[case] struct_fields: Fields, #[case] dimension: i32, - #[case] min_version: LanceFileVersion, #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, + #[values(TestEncoding::StructuralU32, TestEncoding::StructuralSparse)] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + #[values(1, 5, 10)] ingest_batch_count: u32, ) { let data_type = make_fsl_struct_type(struct_fields, dimension); let mut field_metadata = HashMap::new(); @@ -710,7 +713,11 @@ mod tests { structural_encoding.into(), ); let field = Field::new("", data_type, true).with_metadata(field_metadata); - let test_cases = TestCases::basic().with_min_file_version(min_version); + let test_cases = TestCases::basic() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]) + .with_ingest_batch_counts([ingest_batch_count]); check_specific_random(field, test_cases).await; } diff --git a/rust/lance-encoding/src/encodings/logical/list.rs b/rust/lance-encoding/src/encodings/logical/list.rs index 250eb476671..df57d8c20ef 100644 --- a/rust/lance-encoding/src/encodings/logical/list.rs +++ b/rust/lance-encoding/src/encodings/logical/list.rs @@ -240,8 +240,8 @@ mod tests { STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK, }; use arrow_array::{ - Array, ArrayRef, BooleanArray, DictionaryArray, LargeStringArray, ListArray, StructArray, - UInt8Array, UInt64Array, + Array, ArrayRef, BooleanArray, DictionaryArray, LargeStringArray, ListArray, StringArray, + StructArray, UInt8Array, UInt64Array, builder::{ Int32Builder, Int64Builder, LargeListBuilder, ListBuilder, StringBuilder, UInt32Builder, }, @@ -249,11 +249,12 @@ mod tests { use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields}; + use lance_datagen::{RowCount, Seed, array, gen_batch}; use rstest::rstest; - use crate::{ - testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + use crate::testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, create_test_field_encoder, + test_encoding_strategy, }; fn make_list_type(inner_type: DataType) -> DataType { @@ -264,6 +265,61 @@ mod tests { DataType::LargeList(Arc::new(Field::new("item", inner_type, true))) } + #[derive(Clone, Copy)] + enum NullPattern { + None, + Mixed, + All, + } + + async fn check_nested_type( + data_type: DataType, + null_pattern: NullPattern, + encoding: TestEncoding, + ) { + check_nested_type_with_metadata(data_type, null_pattern, encoding, HashMap::new()).await; + } + + async fn check_nested_type_with_metadata( + data_type: DataType, + null_pattern: NullPattern, + encoding: TestEncoding, + field_metadata: HashMap, + ) { + let null_rate = match null_pattern { + NullPattern::None => None, + NullPattern::Mixed => Some(0.5), + NullPattern::All => Some(1.0), + }; + let make_batch = |seed, rows| { + let mut generator = gen_batch() + .with_seed(Seed::from(seed)) + .anon_col(array::rand_type(&data_type)); + if let Some(null_rate) = null_rate { + generator.with_random_nulls(null_rate); + } + generator + .into_batch_rows(RowCount::from(rows)) + .unwrap() + .column(0) + .clone() + }; + + // Combine a non-zero-offset slice with an independently generated batch. + // This covers both offset rebasing and rep/def accumulation at the ingest + // boundary without repeating the full generic random-test matrix. + let first = make_batch(0, 513).slice(1, 512); + let second = make_batch(1, 513); + let test_cases = TestCases::default() + .with_page_sizes(vec![4096]) + .with_encoding(encoding) + .with_batch_size(257) + .with_range(510..515) + .with_indices(vec![0, 511, 512, 1024]); + + check_round_trip_encoding_of_data(vec![first, second], &test_cases, field_metadata).await; + } + async fn try_encode_v22_pages( array: ArrayRef, ) -> lance_core::Result> { @@ -277,20 +333,16 @@ mod tests { let arrow_field = Field::new("", array.data_type().clone(), true).with_metadata(field_metadata); let lance_field = lance_core::datatypes::Field::try_from(&arrow_field).unwrap(); - let encoding_strategy = crate::encoder::default_encoding_strategy(LanceFileVersion::V2_2); + let encoding_strategy = test_encoding_strategy(TestEncoding::StructuralU32); let mut column_index_seq = crate::encoder::ColumnIndexSequence::default(); - let encoding_options = crate::encoder::EncodingOptions { - version: LanceFileVersion::V2_2, - ..Default::default() - }; - let mut encoder = encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap(); + let encoding_options = crate::encoder::EncodingOptions::default(); + let mut encoder = create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap(); let mut external_buffers = crate::encoder::OutOfLineBuffers::new(0, crate::encoder::MIN_PAGE_BUFFER_ALIGNMENT); let num_rows = array.len() as u64; @@ -319,6 +371,7 @@ mod tests { fn assert_split_miniblock_layout( pages: &[crate::encoder::EncodedPage], + min_miniblock_pages: usize, expect_structural_only_page: bool, ) { let mut miniblock_pages = 0; @@ -344,23 +397,23 @@ mod tests { } } crate::format::pb21::page_layout::Layout::BlobLayout(_) => {} + crate::format::pb21::page_layout::Layout::SparseLayout(_) => {} } } assert!( - miniblock_pages > 0, - "expected leaf values to remain on mini-block pages" + miniblock_pages >= min_miniblock_pages, + "expected at least {min_miniblock_pages} mini-block pages, got {miniblock_pages}" ); assert_eq!( fullzip_pages, 0, "split list pages should not fall back to full-zip" ); - if expect_structural_only_page { - assert!( - structural_only_pages > 0, - "expected at least one structural-only page" - ); - } + assert_eq!( + structural_only_pages > 0, + expect_structural_only_page, + "structural-only page presence did not match expectation; got {structural_only_pages}" + ); } fn assert_has_fullzip_layout(pages: &[crate::encoder::EncodedPage]) { @@ -381,15 +434,28 @@ mod tests { async fn test_list( #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, + #[values(NullPattern::None, NullPattern::Mixed, NullPattern::All)] + null_pattern: NullPattern, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, ) { let mut field_metadata = HashMap::new(); field_metadata.insert( STRUCTURAL_ENCODING_META_KEY.to_string(), structural_encoding.into(), ); - let field = - Field::new("", make_list_type(DataType::Int32), true).with_metadata(field_metadata); - check_basic_random(field).await; + check_nested_type_with_metadata( + make_list_type(DataType::Int32), + null_pattern, + encoding, + field_metadata, + ) + .await; } #[rstest] @@ -397,35 +463,103 @@ mod tests { async fn test_deeply_nested_lists( #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, + #[values(1, 2, 3, 4, 5)] depth: usize, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + test_encoding: TestEncoding, ) { - let mut field_metadata = HashMap::new(); - field_metadata.insert( + let mut data_type = DataType::Int32; + for _ in 0..depth { + data_type = make_list_type(data_type); + } + + let mut generator = gen_batch() + .with_seed(Seed::from(depth as u64)) + .anon_col(array::rand_type(&data_type)); + generator.with_random_nulls(0.2); + let source = generator + .into_batch_rows(RowCount::from(1026)) + .unwrap() + .column(0) + .clone(); + + // Two non-zero-offset slices cover nested offset rebasing across ingest + // batches. The selected range and indices straddle that batch boundary. + let data = vec![source.slice(1, 512), source.slice(513, 513)]; + let test_cases = TestCases::default() + .with_page_sizes(vec![4096]) + .with_encoding(test_encoding) + .with_batch_size(257) + .with_range(510..515) + .with_indices(vec![0, 511, 512, 1024]); + let field_metadata = HashMap::from([( STRUCTURAL_ENCODING_META_KEY.to_string(), structural_encoding.into(), - ); - let field = Field::new("item", DataType::Int32, true).with_metadata(field_metadata); - for _ in 0..5 { - let field = Field::new("", make_list_type(field.data_type().clone()), true); - check_basic_random(field).await; - } + )]); + + check_round_trip_encoding_of_data(data, &test_cases, field_metadata).await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_large_list() { - let field = Field::new("", make_large_list_type(DataType::Int32), true); - check_basic_random(field).await; + async fn test_large_list( + #[values(NullPattern::None, NullPattern::Mixed, NullPattern::All)] + null_pattern: NullPattern, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + check_nested_type( + make_large_list_type(DataType::Int32), + null_pattern, + encoding, + ) + .await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_nested_strings() { - let field = Field::new("", make_list_type(DataType::Utf8), true); - check_basic_random(field).await; + async fn test_nested_strings( + #[values(NullPattern::None, NullPattern::Mixed, NullPattern::All)] + null_pattern: NullPattern, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + check_nested_type(make_list_type(DataType::Utf8), null_pattern, encoding).await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_nested_list() { - let field = Field::new("", make_list_type(make_list_type(DataType::Int32)), true); - check_basic_random(field).await; + async fn test_nested_list( + #[values(NullPattern::None, NullPattern::Mixed, NullPattern::All)] + null_pattern: NullPattern, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + check_nested_type( + make_list_type(make_list_type(DataType::Int32)), + null_pattern, + encoding, + ) + .await; } /// Regression test: a `List>` column written as MULTIPLE @@ -451,16 +585,22 @@ mod tests { async fn test_multipage_nested_float_list( #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + test_encoding: TestEncoding, ) { use arrow_array::Float32Array; // Production shape: 3 inner lists per row, 768 floats each. let inner_per_row: usize = 3; let inner_len: usize = 768; - // Two chunks (batches) -> two pages; a read batch that spans the page - // boundary is where the multi-page outer-offset bug triggered. A single - // [2731] chunk (one page) decodes fine, which is why this needs >= 2. - let chunk_rows: &[usize] = &[1366, 1365]; + // Each chunk contains ~1.38 MiB of leaf values, so both cross the 1 MiB + // value-page limit. A read batch that spans the ingest boundary is where + // the multi-page outer-offset bug triggered. + let chunk_rows: &[usize] = &[150, 149]; let make_chunk = |start_row: usize, num_rows: usize| -> Arc { let total_inner = num_rows * inner_per_row; @@ -514,28 +654,55 @@ mod tests { structural_encoding.into(), ); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default() + .with_page_sizes(vec![1024 * 1024]) + .with_encoding(test_encoding) + .with_batch_size(151) + .with_range(148..152) + .with_indices(vec![0, 149, 150, 298]); check_round_trip_encoding_of_data(chunks, &test_cases, field_metadata).await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_list_struct_list() { + async fn test_list_struct_list( + #[values(NullPattern::None, NullPattern::Mixed, NullPattern::All)] + null_pattern: NullPattern, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { let struct_type = DataType::Struct(Fields::from(vec![Field::new( "inner_str", DataType::Utf8, false, )])); - let field = Field::new("", make_list_type(struct_type), true); - check_basic_random(field).await; + check_nested_type(make_list_type(struct_type), null_pattern, encoding).await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_list_struct_empty() { + async fn test_list_struct_empty( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { let fields = Fields::from(vec![Field::new("inner", DataType::UInt64, true)]); let items = UInt64Array::from(Vec::::new()); let structs = StructArray::new(fields, vec![Arc::new(items)], None); - let offsets = OffsetBuffer::new(ScalarBuffer::::from(vec![0; 2 * 1024 * 1024 + 1])); + // Exceed two 1 MiB offset pages so flushing the empty struct child is + // still exercised multiple times (the original #2762 regression). + let num_rows = 2 * 1024 * 1024 / size_of::() + 1; + let offsets = OffsetBuffer::new(ScalarBuffer::::from(vec![0; num_rows + 1])); let lists = ListArray::new( Arc::new(Field::new("item", structs.data_type().clone(), true)), offsets, @@ -545,7 +712,9 @@ mod tests { check_round_trip_encoding_of_data( vec![Arc::new(lists)], - &TestCases::default(), + &TestCases::default() + .with_page_sizes(vec![1024 * 1024]) + .with_encoding(encoding), HashMap::new(), ) .await; @@ -624,7 +793,7 @@ mod tests { .with_range(5..7) .with_indices(vec![1, 6]) .with_indices(vec![6]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(outer_list)], &test_cases, field_metadata) .await; } @@ -655,7 +824,7 @@ mod tests { .with_range(1..3) .with_indices(vec![1, 3]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -686,7 +855,7 @@ mod tests { .with_range(1..3) .with_indices(vec![1, 3]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -718,7 +887,7 @@ mod tests { .with_range(1..2) .with_indices(vec![0]) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) .await; } @@ -770,7 +939,7 @@ mod tests { .with_range(1..2) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data( vec![Arc::new(list_arr)], &test_cases, @@ -809,7 +978,7 @@ mod tests { .with_range(1..2) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_arr)], &test_cases, field_metadata) .await; } @@ -845,7 +1014,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_page_sizes(vec![100]) .with_range(800..900); check_round_trip_encoding_of_data( @@ -1005,7 +1174,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data( vec![struct_array.clone()], &test_cases, @@ -1027,7 +1196,7 @@ mod tests { outer_list_builder.append_null(); let list_array = Arc::new(outer_list_builder.finish()); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } @@ -1092,7 +1261,7 @@ mod tests { // This should trigger the assertion failure at primitive.rs:1362 // debug_assert!(rows_avail > 0) - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); // The bug manifests when encoding this specific pattern // Expected: successful round-trip encoding @@ -1106,8 +1275,11 @@ mod tests { #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, ) { - // 2.5 million rows, mostly empty lists. ~100 lists have 10 short strings each. - let num_rows = 2_500_000u32; + // Three chunks' worth of rep/def levels (1 rep bit + 1 def bit each), so the + // planner must split the page. See #6184. + let levels_per_chunk = + crate::encodings::logical::primitive::miniblock::max_repdef_levels_per_chunk(2); + let num_rows = (levels_per_chunk * 3) as u32; let num_non_empty = 100u32; let strings_per_list = 10; @@ -1142,13 +1314,28 @@ mod tests { structural_encoding.into(), ); + let list_array = Arc::new(list_array) as ArrayRef; + let pages = try_encode_v22_pages_with_metadata(list_array.clone(), field_metadata.clone()) + .await + .unwrap(); + if structural_encoding == STRUCTURAL_ENCODING_MINIBLOCK { + assert_split_miniblock_layout(&pages, 2, true); + } + + let chunk_boundary = levels_per_chunk; let test_cases = TestCases::default() - .with_range(0..1000) - .with_range(0..num_rows as u64) - .with_indices(vec![0, (step / 2) as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); - check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, field_metadata) - .await; + .with_range(chunk_boundary - 2..chunk_boundary + 2) + .with_indices(vec![ + 0, + (step / 2) as u64, + chunk_boundary - 1, + chunk_boundary, + num_rows as u64 - 1, + ]) + .with_batch_size(64 * 1024) + .with_page_sizes(vec![1024 * 1024]) + .with_encoding(TestEncoding::StructuralU32); + check_round_trip_encoding_of_data(vec![list_array], &test_cases, field_metadata).await; } #[test_log::test(tokio::test)] @@ -1156,20 +1343,20 @@ mod tests { // Redacted reproduction from a production schema shape containing ARRAY(BOOLEAN). // The field names are not relevant; the failure requires sparse list structure // with a 1-bit Boolean leaf value. - let num_rows = 200_000usize; - let num_non_empty = 10usize; + let levels_per_chunk = + crate::encodings::logical::primitive::miniblock::max_repdef_levels_per_chunk(2); + // One row past the chunk limit forces a split. Keeping values at both ends ensures + // both sides of that split remain mini-block pages instead of structural-only pages. + let num_rows = (levels_per_chunk + 1) as usize; let booleans_per_list = 8usize; - let step = num_rows / num_non_empty; let mut offsets = Vec::with_capacity(num_rows + 1); - let mut values = Vec::with_capacity(num_non_empty * booleans_per_list); + let mut values = Vec::with_capacity(2 * booleans_per_list); offsets.push(0i32); - let mut next_non_empty = step / 2; for row in 0..num_rows { - if row == next_non_empty { + if row == 0 || row == num_rows - 1 { values.extend((0..booleans_per_list).map(|idx| idx % 2 == 0)); - next_non_empty += step; } offsets.push(values.len() as i32); } @@ -1184,12 +1371,13 @@ mod tests { let test_cases = TestCases::default() .with_range(0..1000) - .with_range(0..num_rows as u64) - .with_indices(vec![0, (step / 2) as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_indices(vec![0, levels_per_chunk / 2, num_rows as u64 - 1]) + .with_batch_size(64 * 1024) + .with_page_sizes(vec![1024 * 1024]) + .with_encoding(TestEncoding::StructuralU32); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; - assert_split_miniblock_layout(&pages, false); + assert_split_miniblock_layout(&pages, 2, false); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } @@ -1222,15 +1410,23 @@ mod tests { let test_cases = TestCases::default() .with_range(0..num_rows as u64) .with_indices(vec![0, empty_prefix_rows as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; - assert_split_miniblock_layout(&pages, true); + assert_split_miniblock_layout(&pages, 1, true); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_sparse_boolean_list_with_long_null_prefix() { + async fn test_sparse_boolean_list_with_long_null_prefix( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32 + )] + encoding: TestEncoding, + ) { let null_prefix_rows = 70_000usize; let trailing_empty_rows = 9usize; let booleans_per_list = 8usize; @@ -1259,10 +1455,10 @@ mod tests { let test_cases = TestCases::default() .with_range(0..num_rows as u64) .with_indices(vec![0, null_prefix_rows as u64, num_rows as u64 - 1]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_encoding(encoding); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; - assert_split_miniblock_layout(&pages, true); + assert_split_miniblock_layout(&pages, 1, true); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } @@ -1290,81 +1486,44 @@ mod tests { let test_cases = TestCases::default() .with_range(0..num_rows as u64) .with_indices(vec![0, empty_prefix_rows as u64]) - .with_max_file_version(LanceFileVersion::V2_2); + .with_dense_encodings(); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; - assert_split_miniblock_layout(&pages, true); + assert_split_miniblock_layout(&pages, 1, true); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } - #[test_log::test(tokio::test)] - async fn test_nested_sparse_boolean_list_fails_without_panic() { - let empty_inner_lists = 70_000usize; - let booleans_per_list = 8usize; - + fn unsplittable_nested_list(items: ArrayRef, empty_inner_lists: usize) -> ArrayRef { let mut inner_offsets = vec![0i32; empty_inner_lists + 1]; - let values = (0..booleans_per_list) - .map(|idx| idx % 2 == 0) - .collect::>(); - inner_offsets.push(values.len() as i32); - - let inner_items = BooleanArray::from(values); + inner_offsets.push(items.len() as i32); let inner_list = ListArray::new( - Arc::new(Field::new("item", DataType::Boolean, true)), + Arc::new(Field::new("item", items.data_type().clone(), true)), OffsetBuffer::new(ScalarBuffer::from(inner_offsets)), - Arc::new(inner_items), + items, None, ); - let outer_list = ListArray::new( + Arc::new(ListArray::new( Arc::new(Field::new("item", inner_list.data_type().clone(), true)), OffsetBuffer::new(ScalarBuffer::from(vec![0i32, empty_inner_lists as i32 + 1])), Arc::new(inner_list), None, - ); - - let err = try_encode_v22_pages(Arc::new(outer_list)) - .await - .unwrap_err(); - assert!( - err.to_string().contains("Mini-block cannot encode"), - "unexpected error: {err}" - ); + )) } + #[rstest] + #[case::boolean(Arc::new(BooleanArray::from(vec![true, false, true, false, true, false, true, false])))] + #[case::string(Arc::new(StringArray::from(vec!["value", "other"])))] #[test_log::test(tokio::test)] - async fn test_nested_sparse_string_single_row_falls_back_to_fullzip() { - let empty_inner_lists = 70_000usize; - - let mut inner_offsets = vec![0i32; empty_inner_lists + 1]; - inner_offsets.push(1); - inner_offsets.push(2); - - let mut strings = StringBuilder::new(); - strings.append_value("value"); - strings.append_value("other"); - let inner_items = strings.finish(); - let inner_list = ListArray::new( - Arc::new(Field::new("item", DataType::Utf8, true)), - OffsetBuffer::new(ScalarBuffer::from(inner_offsets)), - Arc::new(inner_items), - None, - ); - let outer_list = ListArray::new( - Arc::new(Field::new("item", inner_list.data_type().clone(), true)), - OffsetBuffer::new(ScalarBuffer::from(vec![0i32, empty_inner_lists as i32 + 2])), - Arc::new(inner_list), - None, - ); - - let outer_list = Arc::new(outer_list) as ArrayRef; - let pages = encode_v22_pages(outer_list.clone()).await; + async fn test_nested_sparse_single_row_falls_back_to_fullzip(#[case] items: ArrayRef) { + let list = unsplittable_nested_list(items, 70_000); + let pages = encode_v22_pages(list.clone()).await; assert_has_fullzip_layout(&pages); let test_cases = TestCases::default() .with_range(0..1) .with_indices(vec![0]) - .with_max_file_version(LanceFileVersion::V2_2); - check_round_trip_encoding_of_data(vec![outer_list], &test_cases, HashMap::new()).await; + .with_dense_encodings(); + check_round_trip_encoding_of_data(vec![list], &test_cases, HashMap::new()).await; } /// Builds the HNSW-flush repro shape: a dense prefix where every row has @@ -1372,9 +1531,9 @@ mod tests { /// lists. Mirrors `HNSW::schema()` `__neighbors` / `__dists` columns: /// dense level-0 lists, then ~6x as many mostly-empty higher-level rows. fn make_hnsw_shaped_list_u32() -> ListArray { - const DENSE_ROWS: u32 = 40_000; + const DENSE_ROWS: u32 = 5_000; const NEIGHBORS_PER_ROW: u32 = 32; - const EMPTY_TAIL_ROWS: u32 = 240_000; + const EMPTY_TAIL_ROWS: u32 = 70_000; let mut list_builder = ListBuilder::new(UInt32Builder::new()); let mut next_val: u32 = 0; @@ -1403,7 +1562,7 @@ mod tests { #[test_log::test(tokio::test)] async fn test_list_hnsw_shape_splits_to_miniblock_v2_2() { let list_array = make_hnsw_shaped_list_u32(); - let dense_rows: u64 = 40_000; + let dense_rows: u64 = 5_000; let total_rows = list_array.len() as u64; let test_cases = TestCases::default() @@ -1411,11 +1570,10 @@ mod tests { .with_range(dense_rows.saturating_sub(8)..(dense_rows + 8)) .with_range(0..total_rows) .with_indices(vec![0, dense_rows - 1, dense_rows, total_rows - 1]) - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2); + .with_encoding(TestEncoding::StructuralU32); let list_array = Arc::new(list_array) as ArrayRef; let pages = encode_v22_pages(list_array.clone()).await; - assert_split_miniblock_layout(&pages, true); + assert_split_miniblock_layout(&pages, 1, true); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } @@ -1437,13 +1595,12 @@ mod tests { let test_cases = TestCases::default() .with_range(0..total_rows) - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2); + .with_encoding(TestEncoding::StructuralU32); let list_array = Arc::new(list_array) as ArrayRef; let pages = try_encode_v22_pages_with_metadata(list_array.clone(), field_metadata.clone()) .await .unwrap(); - assert_split_miniblock_layout(&pages, true); + assert_split_miniblock_layout(&pages, 1, true); check_round_trip_encoding_of_data(vec![list_array], &test_cases, field_metadata).await; } } diff --git a/rust/lance-encoding/src/encodings/logical/map.rs b/rust/lance-encoding/src/encodings/logical/map.rs index b5172d8c189..d07c267bd2a 100644 --- a/rust/lance-encoding/src/encodings/logical/map.rs +++ b/rust/lance-encoding/src/encodings/logical/map.rs @@ -223,7 +223,8 @@ impl StructuralDecodeArrayTask for StructuralMapDecodeTask { .clone(); // Build the MapArray from offsets, entries, validity, and keys_sorted - let map_array = MapArray::new(entries_field, offsets, entries, validity, keys_sorted); + let map_array = MapArray::try_new(entries_field, offsets, entries, validity, keys_sorted) + .map_err(|error| Error::invalid_input_source(error.to_string().into()))?; Ok(DecodedArray { array: Arc::new(map_array), @@ -244,14 +245,21 @@ mod tests { use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields}; - use crate::encoder::{ColumnIndexSequence, EncodingOptions, default_encoding_strategy}; - use crate::{ - testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + use crate::decoder::{DecodedArray, StructuralDecodeArrayTask}; + use crate::encoder::{ColumnIndexSequence, EncodingOptions}; + use crate::encodings::logical::primitive::sparse::{ + SparseCountSet, SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, + SparseValidityMeaning, SparseValiditySet, + }; + use crate::repdef::{CompositeRepDefUnraveler, RepDefUnraveler}; + use crate::testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, test_encoding_strategy, }; use arrow_schema::Field as ArrowField; use lance_core::datatypes::Field as LanceField; + use super::StructuralMapDecodeTask; + fn make_map_type(key_type: DataType, value_type: DataType) -> DataType { // Note: Arrow MapBuilder uses "keys" and "values" as field names (plural) let entries = Field::new( @@ -265,6 +273,70 @@ mod tests { DataType::Map(Arc::new(entries), false) } + #[derive(Debug)] + struct StaticMapEntriesTask { + entries: StructArray, + repdef: CompositeRepDefUnraveler, + } + + impl StructuralDecodeArrayTask for StaticMapEntriesTask { + fn decode(self: Box) -> lance_core::Result { + let Self { entries, repdef } = *self; + Ok(DecodedArray { + array: Arc::new(entries), + repdef, + data_size: 0, + }) + } + } + + #[test] + fn malformed_sparse_map_entries_return_invalid_input() { + let entry_fields = Fields::from(vec![ + Field::new("keys", DataType::Int32, false), + Field::new("values", DataType::Int32, true), + ]); + let entries = StructArray::try_new( + entry_fields.clone(), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(Int32Array::from(vec![2])), + ], + Some(NullBuffer::from(vec![false])), + ) + .unwrap(); + let validity = SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: SparsePositionSet::Empty, + }; + let plan = SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::List { + num_slots: 1, + num_child_slots: 1, + non_empty_positions: SparsePositionSet::All { len: 1 }, + counts: SparseCountSet::Constant { value: 1, len: 1 }, + validity, + }], + num_items: 1, + num_visible_items: 1, + }; + let child_task = StaticMapEntriesTask { + entries, + repdef: CompositeRepDefUnraveler::new(vec![RepDefUnraveler::new_sparse(plan)]), + }; + let map_type = DataType::Map( + Arc::new(Field::new("entries", DataType::Struct(entry_fields), false)), + false, + ); + + let Err(err) = + Box::new(StructuralMapDecodeTask::new(Box::new(child_task), map_type)).decode() + else { + panic!("expected malformed map entries to be rejected"); + }; + assert!(matches!(err, lance_core::Error::InvalidInput { .. })); + } + #[test_log::test(tokio::test)] async fn test_simple_map() { // Create a simple Map @@ -288,7 +360,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -321,7 +393,7 @@ mod tests { .with_range(0..4) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -352,7 +424,7 @@ mod tests { .with_range(0..2) .with_indices(vec![0]) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -401,7 +473,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) .with_indices(vec![0, 2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], @@ -455,7 +527,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], @@ -501,7 +573,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) .with_indices(vec![0, 2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) .await; @@ -561,7 +633,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..1) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(outer_map)], &test_cases, HashMap::new()) .await; @@ -591,7 +663,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) .with_indices(vec![0, 1]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -618,7 +690,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -639,7 +711,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..2) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(map_array)], &test_cases, HashMap::new()) .await; @@ -674,7 +746,7 @@ mod tests { let test_cases = TestCases::default() .with_range(0..3) .with_indices(vec![0, 1, 2]) - .with_min_file_version(LanceFileVersion::V2_2); + .with_u32_structural_encodings(); // This test ensures that regardless of the internal keep_original_array setting, // the end-to-end behavior produces equivalent results @@ -693,11 +765,11 @@ mod tests { let map_field = LanceField::try_from(&map_arrow_field).unwrap(); // Test encoder: Try to create encoder with V2_1 version - should fail - let encoder_strategy = default_encoding_strategy(LanceFileVersion::V2_1); + let encoder_strategy = test_encoding_strategy(TestEncoding::StructuralU16); let mut column_index = ColumnIndexSequence::default(); let options = EncodingOptions::default(); - let encoder_result = encoder_strategy.create_field_encoder( + let encoder_result = crate::testing::create_test_field_encoder( encoder_strategy.as_ref(), &map_field, &mut column_index, @@ -714,9 +786,8 @@ mod tests { let encoder_err_msg = format!("{}", encoder_err); assert!( - encoder_err_msg.contains("2.2"), - "Encoder error message should mention version 2.2, got: {}", - encoder_err_msg + encoder_err_msg.contains("not enabled by the selected file format"), + "unexpected encoder error: {encoder_err_msg}" ); assert!( encoder_err_msg.contains("Map data type"), diff --git a/rust/lance-encoding/src/encodings/logical/primitive.rs b/rust/lance-encoding/src/encodings/logical/primitive.rs index 8b6eed5d757..7e4e57a2b9e 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive.rs @@ -15,6 +15,7 @@ use std::{ use crate::{ constants::{ STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK, + STRUCTURAL_ENCODING_SPARSE, }, data::DictionaryDataBlock, encodings::logical::primitive::blob::{BlobDescriptionPageScheduler, BlobPageScheduler}, @@ -32,17 +33,19 @@ use itertools::Itertools; use lance_arrow::DataTypeExt; use lance_arrow::deepcopy::deep_copy_nulls; use lance_core::{ - cache::{CacheKey, Context, DeepSizeOf}, + cache::{CacheKey, CacheKeySchema, Context, DeepSizeOf, KeyBuilder}, error::{Error, LanceOptionExt}, utils::bit::pad_bytes, }; -use log::trace; +use log::{debug, trace}; use crate::encodings::logical::primitive::miniblock::MiniBlockChunk; +use crate::encodings::physical::rle::{RleDecompressor, RleRuns}; use crate::utils::bytepack::ByteUnpacker; use crate::{ compression::{ BlockDecompressor, CompressionStrategy, DecompressionStrategy, MiniBlockDecompressor, + compress_required_block, create_rle_decompressor, }, data::{AllNullDataBlock, DataBlock, VariableWidthBlock}, utils::bytepack::BytepackedIntegerEncoder, @@ -52,13 +55,14 @@ use crate::{ encodings::logical::primitive::fullzip::PerValueDataBlock, }; use crate::{ - encodings::logical::primitive::miniblock::MiniBlockCompressed, + encodings::logical::primitive::miniblock::{MiniBlockCompressed, MiniBlockCompressionContext}, statistics::{ComputeStat, GetStat, Stat}, }; use crate::{ repdef::{ CompositeRepDefUnraveler, ControlWordIterator, ControlWordParser, DefinitionInterpretation, - RepDefSlicer, SerializedRepDefs, StructuralPagePlan, build_control_word_iterator, + MiniBlockRepDefBudget, NormalizedStructuralPlan, RepDefSlicer, SerializedRepDefs, + build_control_word_iterator, }, utils::accumulation::AccumulationQueue, }; @@ -70,7 +74,6 @@ use crate::constants::{ DICT_VALUES_COMPRESSION_LEVEL_ENV_VAR, DICT_VALUES_COMPRESSION_LEVEL_META_KEY, DICT_VALUES_COMPRESSION_META_KEY, }; -use crate::version::LanceFileVersion; use crate::{ EncodingsIo, buffer::LanceBuffer, @@ -88,10 +91,15 @@ use crate::{ }; pub mod blob; +mod chunk_index; pub mod constant; pub mod dict; pub mod fullzip; +mod layout; pub mod miniblock; +pub(crate) mod sparse; + +use chunk_index::{ItemCounts, MiniBlockChunkIndex, PrefixSums, RowMapping, parse_nested_rep}; const FILL_BYTE: u8 = 0xFE; const DEFAULT_DICT_DIVISOR: u64 = 2; @@ -99,6 +107,14 @@ const DEFAULT_DICT_MAX_CARDINALITY: u64 = 100_000; const DEFAULT_DICT_SIZE_RATIO: f64 = 0.8; const DEFAULT_DICT_VALUES_COMPRESSION: &str = "lz4"; +/// Largest level count a direct legacy u16-value/U8-run RLE frame can prove. +/// +/// Mini-block level payload sizes are u16. After the eight-byte frame header, each run needs a +/// two-byte value and a one-byte length, and each U8 run can represent at most 255 levels. +const MAX_LEGACY_RLE_LEVELS: u64 = ((u16::MAX as u64 - std::mem::size_of::() as u64) + / (std::mem::size_of::() as u64 + std::mem::size_of::() as u64)) + * u8::MAX as u64; + struct PageLoadTask { decoder_fut: BoxFuture<'static, Result>>, num_rows: u64, @@ -166,16 +182,129 @@ struct DecodeMiniBlockTask { } impl DecodeMiniBlockTask { + fn decoded_size_bytes(&self) -> Option { + if self.rep_decompressor.is_some() || self.def_decompressor.is_some() { + return None; + } + let num_values = self + .instructions + .iter() + .try_fold(0_u64, |total, (instruction, _)| { + total.checked_add(instruction.rows_to_take) + })?; + self.value_decompressor.decoded_size_bytes(num_values) + } + + fn resolve_num_levels( + rep_decompressor: Option<&dyn BlockDecompressor>, + rep_levels: Option<&LanceBuffer>, + def_decompressor: Option<&dyn BlockDecompressor>, + def_levels: Option<&LanceBuffer>, + declared_num_levels: u16, + ) -> Result { + let rep_num_levels = match (rep_decompressor, rep_levels) { + (Some(decompressor), Some(levels)) => decompressor.infer_num_values(levels)?, + (None, None) => None, + _ => { + return Err(Error::invalid_input_source( + "miniblock repetition codec and payload presence disagree".into(), + )); + } + }; + let def_num_levels = match (def_decompressor, def_levels) { + (Some(decompressor), Some(levels)) => decompressor.infer_num_values(levels)?, + (None, None) => None, + _ => { + return Err(Error::invalid_input_source( + "miniblock definition codec and payload presence disagree".into(), + )); + } + }; + + let inferred_num_levels = match (rep_num_levels, def_num_levels) { + (Some(rep_num_levels), Some(def_num_levels)) if rep_num_levels != def_num_levels => { + return Err(Error::invalid_input_source( + format!( + "miniblock structural streams disagree on the level count: repetition inferred {rep_num_levels}, definition inferred {def_num_levels}" + ) + .into(), + )); + } + (Some(num_levels), _) | (_, Some(num_levels)) => num_levels, + (None, None) => return Ok(u64::from(declared_num_levels)), + }; + + let declared_num_levels = u64::from(declared_num_levels); + if inferred_num_levels == declared_num_levels { + return Ok(inferred_num_levels); + } + let u16_modulus = u64::from(u16::MAX) + 1; + if inferred_num_levels <= declared_num_levels + || inferred_num_levels % u16_modulus != declared_num_levels + { + return Err(Error::invalid_input_source( + format!( + "miniblock payload proves {inferred_num_levels} levels but the header declared {declared_num_levels}; the counts are not congruent modulo 65536" + ) + .into(), + )); + } + if inferred_num_levels > MAX_LEGACY_RLE_LEVELS { + return Err(Error::invalid_input_source( + format!( + "miniblock payload proves {inferred_num_levels} levels, exceeding the legacy RLE payload bound of {MAX_LEGACY_RLE_LEVELS}" + ) + .into(), + )); + } + Ok(inferred_num_levels) + } + fn decode_levels( - rep_decompressor: &dyn BlockDecompressor, + decompressor: &dyn BlockDecompressor, levels: LanceBuffer, - num_levels: u16, + expected_num_levels: u64, ) -> Result> { - let rep = rep_decompressor.decompress(levels, num_levels as u64)?; - let rep = rep.as_fixed_width().unwrap(); - debug_assert_eq!(rep.num_values, num_levels as u64); - debug_assert_eq!(rep.bits_per_value, 16); - Ok(rep.data.borrow_to_typed_slice::()) + let levels = decompressor.decompress(Some(levels), expected_num_levels)?; + let levels = levels.as_fixed_width().ok_or_else(|| { + Error::invalid_input_source( + "miniblock levels did not decode to fixed-width data".into(), + ) + })?; + if levels.num_values != expected_num_levels { + return Err(Error::invalid_input_source( + format!( + "miniblock levels decoded {} values, expected {expected_num_levels}", + levels.num_values + ) + .into(), + )); + } + if levels.bits_per_value != 16 { + return Err(Error::invalid_input_source( + format!( + "miniblock levels decoded {} bits per value, expected 16", + levels.bits_per_value + ) + .into(), + )); + } + let expected_bytes = expected_num_levels.checked_mul(2).ok_or_else(|| { + Error::invalid_input_source( + format!("miniblock level byte count overflowed for {expected_num_levels} levels") + .into(), + ) + })?; + if levels.data.len() as u64 != expected_bytes { + return Err(Error::invalid_input_source( + format!( + "miniblock levels decoded {} bytes, expected {expected_bytes}", + levels.data.len() + ) + .into(), + )); + } + Ok(levels.data.borrow_to_typed_slice::()) } // We are building a LevelBuffer (levels) and want to copy into it `total_len` @@ -503,6 +632,14 @@ impl DecodeMiniBlockTask { def }); + let num_levels = Self::resolve_num_levels( + self.rep_decompressor.as_deref(), + rep.as_ref(), + self.def_decompressor.as_deref(), + def.as_ref(), + num_levels, + )?; + let buffers = buffer_sizes .into_iter() .map(|buf_size| { @@ -536,6 +673,19 @@ impl DecodeMiniBlockTask { }) .transpose()?; + if let (Some(rep), Some(def)) = (&rep, &def) + && rep.len() != def.len() + { + return Err(Error::invalid_input_source( + format!( + "miniblock structural streams decoded different level counts: repetition {}, definition {}", + rep.len(), + def.len() + ) + .into(), + )); + } + Ok(DecodedMiniBlockChunk { rep, def, values }) } } @@ -548,15 +698,15 @@ impl DecodePageTask for DecodeMiniBlockTask { let max_rep = self.def_meaning.iter().filter(|l| l.is_list()).count() as u16; - // This is probably an over-estimate but it's quick and easy to calculate - let estimated_size_bytes = self - .instructions - .iter() - .map(|(_, chunk)| chunk.data.len()) - .sum::() - * 2; - let mut data_builder = - DataBlockBuilder::with_capacity_estimate(estimated_size_bytes as u64); + let estimated_size_bytes = self.decoded_size_bytes().unwrap_or_else(|| { + // Variable-width and rep/def encoded output sizes are not known before decoding. + self.instructions + .iter() + .map(|(_, chunk)| chunk.data.len() as u64) + .sum::() + * 2 + }); + let mut data_builder = DataBlockBuilder::with_capacity_estimate(estimated_size_bytes); // We need to keep track of the offset into repbuf/defbuf that we are building up let mut level_offset = 0; @@ -621,7 +771,7 @@ impl DecodePageTask for DecodeMiniBlockTask { Self::extend_levels(level_range.clone(), &mut repbuf, &rep, level_offset); Self::extend_levels(level_range.clone(), &mut defbuf, &def, level_offset); level_offset += (level_range.end - level_range.start) as usize; - data_builder.append(&values, item_range); + data_builder.append(&values, item_range)?; } let mut data = data_builder.finish(); @@ -743,181 +893,839 @@ impl StructuralPageDecoder for MiniBlockDecoder { } } -#[derive(Debug)] -struct CachedComplexAllNullState { - rep: Option>, - def: Option>, +/// How a complex-all-null page's rep/def level buffer is compressed on disk. +/// Captured at scheduler construction so `initialize` can keep RLE levels in run +/// form instead of expanding them. +#[derive(Debug, Clone)] +pub(crate) enum LevelCodec { + /// Raw little-endian u16 levels (no block compression). + Uncompressed, + /// RLE-compressed levels; the validated physical runs select their cached representation. + Rle(Arc), + /// Any other block compression; decoded eagerly into [`LazyLevels::Dense`] + /// (these encodings don't expand, so laziness buys nothing). + Block(Arc), } -impl DeepSizeOf for CachedComplexAllNullState { - fn deep_size_of_children(&self, _ctx: &mut Context) -> usize { - self.rep.as_ref().map(|buf| buf.len() * 2).unwrap_or(0) - + self.def.as_ref().map(|buf| buf.len() * 2).unwrap_or(0) +impl LevelCodec { + fn try_new( + encoding: Option<&CompressiveEncoding>, + decompression_strategy: &dyn DecompressionStrategy, + ) -> Result { + match encoding { + None => Ok(Self::Uncompressed), + Some(encoding) => match encoding.compression.as_ref() { + Some(Compression::Rle(rle)) => Ok(Self::Rle(Arc::new(create_rle_decompressor( + rle, + decompression_strategy, + )?))), + _ => Ok(Self::Block(Arc::from( + decompression_strategy.create_block_decompressor(encoding)?, + ))), + }, + } } } -impl CachedPageData for CachedComplexAllNullState { - fn as_arc_any(self: Arc) -> Arc { - self +#[derive(Debug)] +enum RunEnds { + U16(Box<[u16]>), + U32(Box<[u32]>), + U64(Box<[u64]>), +} + +impl RunEnds { + fn width_for(num_values: usize) -> usize { + if u16::try_from(num_values).is_ok() { + std::mem::size_of::() + } else if u32::try_from(num_values).is_ok() { + std::mem::size_of::() + } else { + std::mem::size_of::() + } + } + + fn len(&self) -> usize { + match self { + Self::U16(ends) => ends.len(), + Self::U32(ends) => ends.len(), + Self::U64(ends) => ends.len(), + } + } + + fn get(&self, run: usize) -> usize { + match self { + Self::U16(ends) => ends[run] as usize, + Self::U32(ends) => ends[run] as usize, + Self::U64(ends) => ends[run] as usize, + } + } + + fn partition_point(&self, logical_index: usize) -> usize { + match self { + Self::U16(ends) => ends.partition_point(|&end| end as usize <= logical_index), + Self::U32(ends) => ends.partition_point(|&end| end as usize <= logical_index), + Self::U64(ends) => ends.partition_point(|&end| end as usize <= logical_index), + } + } + + fn deep_size(&self) -> usize { + match self { + Self::U16(ends) => std::mem::size_of_val(ends.as_ref()), + Self::U32(ends) => std::mem::size_of_val(ends.as_ref()), + Self::U64(ends) => std::mem::size_of_val(ends.as_ref()), + } } } -/// A scheduler for all-null data that has repetition and definition levels -/// -/// We still need to do some I/O in this case because we need to figure out what kind of null we -/// are dealing with (null list, null struct, what level null struct, etc.) -/// -/// TODO: Right now we just load the entire rep/def at initialization time and cache it. This is a touch -/// RAM aggressive and maybe we want something more lazy in the future. On the other hand, it's simple -/// and fast so...maybe not :) -#[derive(Debug)] -pub struct ComplexAllNullScheduler { - // Set from protobuf - buffer_offsets_and_sizes: Arc<[(u64, u64)]>, - def_meaning: Arc<[DefinitionInterpretation]>, - repdef: Option>, - max_rep: u16, - max_visible_level: u16, - rep_decompressor: Option>, - def_decompressor: Option>, - num_rep_values: u64, - num_def_values: u64, +enum RunEndsBuilder { + U16(Vec), + U32(Vec), + U64(Vec), } -impl ComplexAllNullScheduler { - pub fn new( - buffer_offsets_and_sizes: Arc<[(u64, u64)]>, - def_meaning: Arc<[DefinitionInterpretation]>, - rep_decompressor: Option>, - def_decompressor: Option>, - num_rep_values: u64, - num_def_values: u64, - ) -> Self { - let max_rep = def_meaning.iter().filter(|l| l.is_list()).count() as u16; - let max_visible_level = def_meaning - .iter() - .take_while(|l| !l.is_list()) - .map(|l| l.num_def_levels()) - .sum::(); - Self { - buffer_offsets_and_sizes, - def_meaning, - repdef: None, - max_rep, - max_visible_level, - rep_decompressor, - def_decompressor, - num_rep_values, - num_def_values, +impl RunEndsBuilder { + fn with_capacity(num_values: usize, capacity: usize) -> Self { + if u16::try_from(num_values).is_ok() { + Self::U16(Vec::with_capacity(capacity)) + } else if u32::try_from(num_values).is_ok() { + Self::U32(Vec::with_capacity(capacity)) + } else { + Self::U64(Vec::with_capacity(capacity)) + } + } + + fn push(&mut self, end: usize) -> Result<()> { + match self { + Self::U16(ends) => ends.push( + u16::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u16")))?, + ), + Self::U32(ends) => ends.push( + u32::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u32")))?, + ), + Self::U64(ends) => ends.push(end as u64), + } + Ok(()) + } + + fn set_last(&mut self, end: usize) -> Result<()> { + match self { + Self::U16(ends) => { + let last = ends.last_mut().ok_or_else(|| { + Error::internal("Cannot extend an empty coalesced run buffer") + })?; + *last = u16::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u16")))?; + } + Self::U32(ends) => { + let last = ends.last_mut().ok_or_else(|| { + Error::internal("Cannot extend an empty coalesced run buffer") + })?; + *last = u32::try_from(end) + .map_err(|_| Error::internal(format!("Run end {end} does not fit in u32")))?; + } + Self::U64(ends) => { + let last = ends.last_mut().ok_or_else(|| { + Error::internal("Cannot extend an empty coalesced run buffer") + })?; + *last = end as u64; + } + } + Ok(()) + } + + fn finish(self) -> RunEnds { + match self { + Self::U16(ends) => RunEnds::U16(ends.into_boxed_slice()), + Self::U32(ends) => RunEnds::U32(ends.into_boxed_slice()), + Self::U64(ends) => RunEnds::U64(ends.into_boxed_slice()), } } } -impl StructuralPageScheduler for ComplexAllNullScheduler { - fn initialize<'a>( - &'a mut self, - io: &Arc, - ) -> BoxFuture<'a, Result>> { - // Fully load the rep & def buffers, as needed - let (rep_pos, rep_size) = self.buffer_offsets_and_sizes[0]; - let (def_pos, def_size) = self.buffer_offsets_and_sizes[1]; - let has_rep = rep_size > 0; - let has_def = def_size > 0; +#[derive(Debug)] +enum RunStorage { + Physical(RleRuns), + Coalesced { values: Box<[u16]>, ends: RunEnds }, +} - let mut reads = Vec::with_capacity(2); - if has_rep { - reads.push(rep_pos..rep_pos + rep_size); +impl RunStorage { + fn len(&self) -> usize { + match self { + Self::Physical(runs) => runs.num_values(), + Self::Coalesced { ends, .. } => ends.get(ends.len() - 1), } - if has_def { - reads.push(def_pos..def_pos + def_size); + } + + fn num_runs(&self) -> usize { + match self { + Self::Physical(runs) => runs.num_runs(), + Self::Coalesced { values, .. } => values.len(), } + } - let data = io.submit_request(reads, 0); - let rep_decompressor = self.rep_decompressor.clone(); - let def_decompressor = self.def_decompressor.clone(); - let num_rep_values = self.num_rep_values; - let num_def_values = self.num_def_values; + fn value(&self, run: usize) -> u16 { + match self { + Self::Physical(runs) => runs.value(run), + Self::Coalesced { values, .. } => values[run], + } + } - async move { - let data = data.await?; - let mut data_iter = data.into_iter(); + fn first_value_above(&self, max: u16) -> Option<(usize, u16)> { + (0..self.num_runs()).find_map(|run| { + let value = self.value(run); + (value > max).then_some((run, value)) + }) + } - let decompress_levels = |compressed_bytes: Bytes, - decompressor: &Arc, - num_values: u64, - level_type: &str| - -> Result> { - let compressed_buffer = LanceBuffer::from_bytes(compressed_bytes, 1); - let decompressed = decompressor.decompress(compressed_buffer, num_values)?; - match decompressed { - DataBlock::FixedWidth(block) => { - if block.num_values != num_values { - return Err(Error::invalid_input_source(format!( - "Unexpected {} level count after decompression: expected {}, got {}", - level_type, num_values, block.num_values - ) - .into())); - } - if block.bits_per_value != 16 { - return Err(Error::invalid_input_source(format!( - "Unexpected {} level bit width after decompression: expected 16, got {}", - level_type, block.bits_per_value - ) - .into())); - } - Ok(block.data.borrow_to_typed_slice::()) - } - _ => Err(Error::invalid_input_source(format!( - "Expected fixed-width data block for {} levels", - level_type - ) - .into())), - } + fn seek(&self, position: &mut RunPosition, logical_index: usize) { + if logical_index >= self.len() { + *position = RunPosition { + run: self.num_runs(), + start: self.len(), + end: self.len(), }; + return; + } - let rep = if has_rep { - let rep = data_iter.next().unwrap(); - if let Some(rep_decompressor) = rep_decompressor.as_ref() { - Some(decompress_levels( - rep, - rep_decompressor, - num_rep_values, - "repetition", - )?) - } else { - let rep = LanceBuffer::from_bytes(rep, 2); - let rep = rep.borrow_to_typed_slice::(); - Some(rep) + match self { + Self::Physical(runs) => { + if position.run >= runs.num_runs() + || position.end == 0 + || logical_index < position.start + { + *position = RunPosition { + run: 0, + start: 0, + end: runs.length(0), + }; } - } else { - None - }; - - let def = if has_def { - let def = data_iter.next().unwrap(); - if let Some(def_decompressor) = def_decompressor.as_ref() { - Some(decompress_levels( - def, - def_decompressor, - num_def_values, - "definition", - )?) - } else { - let def = LanceBuffer::from_bytes(def, 2); - let def = def.borrow_to_typed_slice::(); - Some(def) + while position.end <= logical_index { + self.advance(position); } - } else { - None - }; + } + Self::Coalesced { ends, .. } => { + if logical_index < position.start || logical_index >= position.end { + let run = ends.partition_point(logical_index); + *position = RunPosition { + run, + start: if run == 0 { 0 } else { ends.get(run - 1) }, + end: ends.get(run), + }; + } + } + } + } - let repdef = Arc::new(CachedComplexAllNullState { rep, def }); + fn advance(&self, position: &mut RunPosition) { + let next_run = position.run + 1; + if next_run >= self.num_runs() { + *position = RunPosition { + run: self.num_runs(), + start: self.len(), + end: self.len(), + }; + return; + } - self.repdef = Some(repdef.clone()); + let start = position.end; + position.run = next_run; + position.start = start; + position.end = match self { + Self::Physical(runs) => start + runs.length(next_run), + Self::Coalesced { ends, .. } => ends.get(next_run), + }; + } - Ok(repdef as Arc) + fn deep_size(&self) -> usize { + match self { + Self::Physical(runs) => runs.deep_size(), + Self::Coalesced { values, ends } => { + std::mem::size_of_val(values.as_ref()) + ends.deep_size() + } } - .boxed() + } +} + +/// Rep/def levels for a complex-all-null page. +/// +/// RLE pages retain the smallest of their validated physical runs, coalesced +/// runs, and dense values. The decoder materializes only the per-drain slices +/// it touches. +#[derive(Debug, Clone)] +enum LazyLevels { + Dense(ScalarBuffer), + Runs(Arc), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LevelPlan { + Physical, + Coalesced, + Dense, +} + +#[derive(Debug, Default, Clone, Copy)] +struct RunPosition { + run: usize, + start: usize, + end: usize, +} + +/// Monotonic forward cursor into a [`LazyLevels`] sequence. +/// +/// Drains seek to strictly increasing rows, so each [`LazyLevels::seek_row_start`] +/// resumes from the last position instead of rescanning — every run is visited at +/// most once per page while locating and counting level ranges. +#[derive(Debug, Default, Clone, Copy)] +struct LevelCursor { + /// Logical level index where the current row begins. + level: usize, + /// Row index at `level` (the number of `max_rep` occurrences before it). + row: u64, + /// Run containing `level`. Unused for [`LazyLevels::Dense`]. + run: RunPosition, +} + +impl LazyLevels { + fn from_rle_runs(runs: RleRuns) -> Result { + let plan = Self::select_plan(&runs); + match plan { + LevelPlan::Physical => Ok(Self::Runs(Arc::new(RunStorage::Physical( + runs.into_owned(), + )))), + LevelPlan::Coalesced => Self::build_coalesced(runs), + LevelPlan::Dense => Self::build_dense(runs), + } + } + + /// Minimize retained payload bytes first, then expected traversal work. + /// If both are equal, keep the physical runs and avoid another allocation. + fn select_plan(runs: &RleRuns) -> LevelPlan { + if runs.num_values() == 0 { + return LevelPlan::Dense; + } + + let run_storage_size = std::mem::size_of::() as u128; + let physical_size = run_storage_size + runs.owned_size() as u128; + let coalesced_size = run_storage_size + + (runs.coalesced_runs() as u128) + * (std::mem::size_of::() + RunEnds::width_for(runs.num_values())) as u128; + let dense_size = (runs.num_values() as u128) * std::mem::size_of::() as u128; + [ + (physical_size, runs.num_runs(), 0usize, LevelPlan::Physical), + ( + coalesced_size, + runs.coalesced_runs(), + 1usize, + LevelPlan::Coalesced, + ), + (dense_size, runs.num_values(), 2usize, LevelPlan::Dense), + ] + .into_iter() + .min_by_key(|(size, traversal, priority, _)| (*size, *traversal, *priority)) + .map(|(_, _, _, plan)| plan) + .unwrap_or(LevelPlan::Dense) + } + + fn build_coalesced(runs: RleRuns) -> Result { + let mut values = Vec::with_capacity(runs.coalesced_runs()); + let mut ends = RunEndsBuilder::with_capacity(runs.num_values(), runs.coalesced_runs()); + let mut logical_end = 0usize; + for (value, length) in runs.iter() { + logical_end = logical_end + .checked_add(length) + .ok_or_else(|| Error::internal("Validated RLE run length sum overflowed usize"))?; + if values.last().copied() == Some(value) { + ends.set_last(logical_end)?; + } else { + values.push(value); + ends.push(logical_end)?; + } + } + Ok(Self::Runs(Arc::new(RunStorage::Coalesced { + values: values.into_boxed_slice(), + ends: ends.finish(), + }))) + } + + fn build_dense(runs: RleRuns) -> Result { + let mut values = Vec::new(); + values.try_reserve_exact(runs.num_values()).map_err(|_| { + Error::internal(format!( + "Cannot allocate {} dense repetition/definition levels", + runs.num_values() + )) + })?; + for (value, length) in runs.iter() { + values.resize(values.len() + length, value); + } + Ok(Self::Dense(ScalarBuffer::from(values))) + } + + fn len(&self) -> usize { + match self { + Self::Dense(buf) => buf.len(), + Self::Runs(runs) => runs.len(), + } + } + + fn validate_max(&self, level_type: &str, max: u16) -> Result<()> { + let invalid = match self { + Self::Dense(levels) => levels + .iter() + .enumerate() + .find_map(|(index, &value)| (value > max).then_some(("index", index, value))), + Self::Runs(runs) => runs + .first_value_above(max) + .map(|(run, value)| ("run", run, value)), + }; + if let Some((position_type, position, value)) = invalid { + return Err(Error::invalid_input_source( + format!( + "Invalid {level_type} level {value} at {position_type} {position}: maximum is {max}" + ) + .into(), + )); + } + Ok(()) + } + + /// Advance `cursor` to the start of row `target_row`, returning that row's + /// starting level index. + /// + /// Rows begin at `max_rep` positions, so this finds the `target_row`-th one. + /// `target_row` must be `>= cursor.row`: the cursor only moves forward, which + /// is what keeps a full page decode O(runs) rather than O(rows). + fn seek_row_start( + &self, + cursor: &mut LevelCursor, + target_row: u64, + max_rep: u16, + ) -> Result { + let mut need = target_row.checked_sub(cursor.row).ok_or_else(|| { + Error::internal(format!( + "Complex all-null row ranges are not sorted: target row {target_row} follows {}", + cursor.row + )) + })?; + if need == 0 { + return Ok(cursor.level); + } + match self { + Self::Dense(buf) => { + let mut level = cursor.level; + while need > 0 { + if level >= buf.len() { + return Err(Error::internal( + "Invalid complex all-null layout: repetition buffer too short", + )); + } + if buf[level] != max_rep { + return Err(Error::internal( + "Invalid complex all-null layout: row did not start at max repetition level", + )); + } + level += 1; + while level < buf.len() && buf[level] != max_rep { + level += 1; + } + need -= 1; + } + cursor.level = level; + cursor.row = target_row; + Ok(level) + } + Self::Runs(runs) => { + let mut level = cursor.level; + let mut run = cursor.run; + runs.seek(&mut run, level); + while need > 0 { + if run.run >= runs.num_runs() { + return Err(Error::internal( + "Invalid complex all-null layout: repetition buffer too short", + )); + } + if runs.value(run.run) != max_rep { + return Err(Error::internal( + "Invalid complex all-null layout: row did not start at max repetition level", + )); + } + let avail = (run.end - level) as u64; + if need < avail { + // Target lands inside this max-rep run. + level += need as usize; + need = 0; + } else { + // Consume every row start in this run, then skip the + // trailing non-max-rep runs to reach the next row start. + need -= avail; + runs.advance(&mut run); + while run.run < runs.num_runs() && runs.value(run.run) != max_rep { + runs.advance(&mut run); + } + level = if run.run < runs.num_runs() { + run.start + } else { + self.len() + }; + } + } + cursor.level = level; + cursor.row = target_row; + cursor.run = run; + Ok(level) + } + } + } + + /// Count of levels in `range` that are `<= max`, resuming from `*run_cursor` + /// and leaving it on the last run that overlaps `range`. + /// + /// Successive calls must pass ascending, non-overlapping ranges (`range.start + /// >=` the previous `range.end`) so runs are swept at most once per page. + fn count_le_cursor( + &self, + run_cursor: &mut RunPosition, + range: Range, + max: u16, + ) -> (u64, RunPosition) { + if range.is_empty() { + return (0, *run_cursor); + } + match self { + Self::Dense(buf) => ( + buf[range].iter().filter(|&&d| d <= max).count() as u64, + RunPosition::default(), + ), + Self::Runs(runs) => { + // Advance to the first run overlapping the range. + runs.seek(run_cursor, range.start); + let start = *run_cursor; + let mut count = 0u64; + let mut current = *run_cursor; + while current.run < runs.num_runs() && current.start < range.end { + if runs.value(current.run) <= max { + let lo = current.start.max(range.start); + let hi = current.end.min(range.end); + count += (hi - lo) as u64; + } + if current.end >= range.end { + break; + } + runs.advance(&mut current); + } + // Resume the next (ascending) range from the last overlapping run; + // `current` remains valid because `range` is non-empty. + *run_cursor = current; + (count, start) + } + } + } + + fn extend_into(&self, range: Range, run: RunPosition, out: &mut Vec) { + if range.is_empty() { + return; + } + match self { + Self::Dense(buf) => out.extend_from_slice(&buf[range]), + Self::Runs(runs) => { + let mut current = run; + runs.seek(&mut current, range.start); + while current.run < runs.num_runs() && current.start < range.end { + let lo = current.start.max(range.start); + let hi = current.end.min(range.end); + if hi > lo { + out.resize(out.len() + (hi - lo), runs.value(current.run)); + } + runs.advance(&mut current); + } + } + } + } + + #[cfg(test)] + fn deep_size(&self) -> usize { + self.deep_size_of_children(&mut Context::new()) + } +} + +impl DeepSizeOf for LazyLevels { + fn deep_size_of_children(&self, ctx: &mut Context) -> usize { + match self { + Self::Dense(buf) => buf.deep_size_of_children(ctx), + Self::Runs(runs) => { + let pointer = Arc::as_ptr(runs) as *const () as usize; + if ctx.mark_seen(pointer) { + std::mem::size_of_val(runs.as_ref()) + runs.deep_size() + } else { + 0 + } + } + } + } +} + +fn validate_complex_all_null_levels( + rep: &Option, + def: &Option, + max_rep: u16, + max_def: u16, +) -> Result<()> { + if let Some(rep) = rep { + rep.validate_max("repetition", max_rep)?; + } + if let Some(def) = def { + def.validate_max("definition", max_def)?; + } + if let (Some(rep), Some(def)) = (rep, def) + && rep.len() != def.len() + { + return Err(Error::invalid_input_source( + format!( + "Mismatched complex all-null level counts: repetition has {}, definition has {}", + rep.len(), + def.len() + ) + .into(), + )); + } + Ok(()) +} + +fn expected_level_bytes(num_values: u64, level_type: &str) -> Result { + usize::try_from(num_values) + .ok() + .and_then(|num_values| num_values.checked_mul(std::mem::size_of::())) + .ok_or_else(|| { + Error::invalid_input_source( + format!("{level_type} level count {num_values} does not fit in memory").into(), + ) + }) +} + +fn dense_levels_from_block( + decompressed: DataBlock, + num_values: u64, + level_type: &str, +) -> Result { + let DataBlock::FixedWidth(block) = decompressed else { + return Err(Error::invalid_input_source( + format!("Expected fixed-width data block for {level_type} levels").into(), + )); + }; + if block.num_values != num_values { + return Err(Error::invalid_input_source( + format!( + "Unexpected {level_type} level count after decompression: expected {num_values}, got {}", + block.num_values + ) + .into(), + )); + } + if block.bits_per_value != 16 { + return Err(Error::invalid_input_source( + format!( + "Unexpected {level_type} level bit width after decompression: expected 16, got {}", + block.bits_per_value + ) + .into(), + )); + } + let expected_bytes = expected_level_bytes(num_values, level_type)?; + if block.data.len() != expected_bytes { + return Err(Error::invalid_input_source( + format!( + "Unexpected decompressed {level_type} level size: expected {expected_bytes} bytes for {num_values} values, got {}", + block.data.len() + ) + .into(), + )); + } + Ok(LazyLevels::Dense(block.data.borrow_to_typed_slice::())) +} + +#[derive(Debug)] +struct CachedComplexAllNullState { + rep: Option, + def: Option, +} + +impl DeepSizeOf for CachedComplexAllNullState { + fn deep_size_of_children(&self, ctx: &mut Context) -> usize { + self.rep.deep_size_of_children(ctx) + self.def.deep_size_of_children(ctx) + } +} + +impl CachedPageData for CachedComplexAllNullState { + fn as_arc_any(self: Arc) -> Arc { + self + } +} + +/// A scheduler for all-null data that has repetition and definition levels +/// +/// We still need to do some I/O in this case because we need to figure out what kind of null we +/// are dealing with (null list, null struct, what level null struct, etc.) +/// +/// TODO: Right now we just load the entire rep/def at initialization time and cache it. This is a touch +/// RAM aggressive and maybe we want something more lazy in the future. On the other hand, it's simple +/// and fast so...maybe not :) +#[derive(Debug)] +pub struct ComplexAllNullScheduler { + // Set from protobuf + buffer_offsets_and_sizes: Arc<[(u64, u64)]>, + def_meaning: Arc<[DefinitionInterpretation]>, + repdef: Option>, + max_rep: u16, + max_def: u16, + max_visible_level: u16, + rep_codec: LevelCodec, + def_codec: LevelCodec, + num_rep_values: u64, + num_def_values: u64, +} + +impl ComplexAllNullScheduler { + pub(crate) fn new( + buffer_offsets_and_sizes: Arc<[(u64, u64)]>, + def_meaning: Arc<[DefinitionInterpretation]>, + rep_codec: LevelCodec, + def_codec: LevelCodec, + num_rep_values: u64, + num_def_values: u64, + ) -> Self { + let max_rep = def_meaning.iter().filter(|l| l.is_list()).count() as u16; + let max_def = def_meaning + .iter() + .map(|meaning| meaning.num_def_levels()) + .sum::(); + let max_visible_level = def_meaning + .iter() + .take_while(|l| !l.is_list()) + .map(|l| l.num_def_levels()) + .sum::(); + Self { + buffer_offsets_and_sizes, + def_meaning, + repdef: None, + max_rep, + max_def, + max_visible_level, + rep_codec, + def_codec, + num_rep_values, + num_def_values, + } + } +} + +impl StructuralPageScheduler for ComplexAllNullScheduler { + fn initialize<'a>( + &'a mut self, + io: &Arc, + ) -> BoxFuture<'a, Result>> { + // Fully load the rep & def buffers, as needed + let (rep_pos, rep_size) = self.buffer_offsets_and_sizes[0]; + let (def_pos, def_size) = self.buffer_offsets_and_sizes[1]; + let has_rep = rep_size > 0; + let has_def = def_size > 0; + + let mut reads = Vec::with_capacity(2); + if has_rep { + reads.push(rep_pos..rep_pos + rep_size); + } + if has_def { + reads.push(def_pos..def_pos + def_size); + } + + let data = io.submit_request(reads, 0); + let rep_codec = self.rep_codec.clone(); + let def_codec = self.def_codec.clone(); + let num_rep_values = self.num_rep_values; + let num_def_values = self.num_def_values; + let max_rep = self.max_rep; + let max_def = self.max_def; + + async move { + let data = data.await?; + let mut data_iter = data.into_iter(); + + // RLE levels select the smallest validated cache representation; + // everything else expands eagerly to `LazyLevels::Dense`. + let build_levels = |compressed_bytes: Bytes, + codec: &LevelCodec, + num_values: u64, + level_type: &str| + -> Result { + match codec { + LevelCodec::Uncompressed => { + if num_values == 0 { + if !compressed_bytes + .len() + .is_multiple_of(std::mem::size_of::()) + { + return Err(Error::invalid_input_source( + format!( + "Unexpected uncompressed {level_type} level size: {} bytes is not divisible by {}", + compressed_bytes.len(), + std::mem::size_of::() + ) + .into(), + )); + } + } else { + let expected_bytes = expected_level_bytes(num_values, level_type)?; + if compressed_bytes.len() != expected_bytes { + return Err(Error::invalid_input_source( + format!( + "Unexpected uncompressed {level_type} level size: expected {expected_bytes} bytes for {num_values} values, got {}", + compressed_bytes.len() + ) + .into(), + )); + } + } + let buffer = LanceBuffer::from_bytes(compressed_bytes, 2); + Ok(LazyLevels::Dense(buffer.borrow_to_typed_slice::())) + } + LevelCodec::Rle(decompressor) => { + let frame = LanceBuffer::from_bytes(compressed_bytes, 1); + let runs = decompressor.decode_u16_runs(frame, num_values)?; + LazyLevels::from_rle_runs(runs) + } + LevelCodec::Block(decompressor) => { + let frame = LanceBuffer::from_bytes(compressed_bytes, 1); + let decompressed = decompressor.decompress(Some(frame), num_values)?; + dense_levels_from_block(decompressed, num_values, level_type) + } + } + }; + + let rep = if has_rep { + let rep = data_iter.next().unwrap(); + Some(build_levels(rep, &rep_codec, num_rep_values, "repetition")?) + } else { + None + }; + + let def = if has_def { + let def = data_iter.next().unwrap(); + Some(build_levels(def, &def_codec, num_def_values, "definition")?) + } else { + None + }; + + validate_complex_all_null_levels(&rep, &def, max_rep, max_def)?; + let repdef = Arc::new(CachedComplexAllNullState { rep, def }); + + self.repdef = Some(repdef.clone()); + + Ok(repdef as Arc) + } + .boxed() } fn load(&mut self, data: &Arc) { @@ -944,8 +1752,8 @@ impl StructuralPageScheduler for ComplexAllNullScheduler { def_meaning: self.def_meaning.clone(), max_rep: self.max_rep, max_visible_level: self.max_visible_level, - cursor_row: 0, - cursor_level: 0, + rep_cursor: LevelCursor::default(), + def_run_cursor: RunPosition::default(), }) as Box; let page_load_task = PageLoadTask { decoder_fut: std::future::ready(Ok(decoder)).boxed(), @@ -958,14 +1766,16 @@ impl StructuralPageScheduler for ComplexAllNullScheduler { #[derive(Debug)] pub struct ComplexAllNullPageDecoder { ranges: VecDeque>, - rep: Option>, - def: Option>, + rep: Option, + def: Option, num_rows: u64, def_meaning: Arc<[DefinitionInterpretation]>, max_rep: u16, max_visible_level: u16, - cursor_row: u64, - cursor_level: usize, + /// Monotonic cursor into `rep` tracking the current row's level start. + rep_cursor: LevelCursor, + /// Monotonic run cursor into `def` for `count_le_cursor`. + def_run_cursor: RunPosition, } impl ComplexAllNullPageDecoder { @@ -987,73 +1797,63 @@ impl ComplexAllNullPageDecoder { ranges } - fn take_row(&mut self) -> Result<(Range, u64)> { - let start = self.cursor_level; - let end = if let Some(rep) = &self.rep { - if start >= rep.len() { - return Err(Error::internal( - "Invalid complex all-null layout: repetition buffer too short", - )); - } - if rep[start] != self.max_rep { - return Err(Error::internal( - "Invalid complex all-null layout: row did not start at max repetition level", - )); - } - let mut end = start + 1; - while end < rep.len() && rep[end] != self.max_rep { - end += 1; - } - end - } else { - start + 1 - }; - - let visible = if let Some(def) = &self.def { - if end > def.len() { - return Err(Error::internal( - "Invalid complex all-null layout: definition buffer too short", - )); + /// Level index at which row `target_row` starts, advancing the monotonic + /// repetition cursor. Callers must request non-decreasing `target_row`. + fn seek_row_start(&mut self, target_row: u64) -> Result { + match &self.rep { + Some(rep) => rep.seek_row_start(&mut self.rep_cursor, target_row, self.max_rep), + None => { + // Without repetition every level is its own row. + self.rep_cursor.row = target_row; + self.rep_cursor.level = target_row as usize; + Ok(target_row as usize) } - def[start..end] - .iter() - .filter(|d| **d <= self.max_visible_level) - .count() as u64 - } else { - (end - start) as u64 - }; - - self.cursor_level = end; - self.cursor_row += 1; - Ok((start..end, visible)) + } } - fn skip_to_row(&mut self, target_row: u64) -> Result<()> { - while self.cursor_row < target_row { - self.take_row()?; + /// Number of visible items in the level range `levels` (definition levels + /// `<= max_visible_level`), advancing the monotonic definition cursor. + fn count_visible(&mut self, levels: Range) -> Result<(u64, RunPosition)> { + match &self.def { + Some(def) => { + if levels.end > def.len() { + return Err(Error::internal( + "Invalid complex all-null layout: definition buffer too short", + )); + } + Ok(def.count_le_cursor(&mut self.def_run_cursor, levels, self.max_visible_level)) + } + None => Ok(((levels.end - levels.start) as u64, RunPosition::default())), } - Ok(()) } } impl StructuralPageDecoder for ComplexAllNullPageDecoder { fn drain(&mut self, num_rows: u64) -> Result> { let drained_ranges = self.drain_ranges(num_rows); - let mut level_slices: Vec> = Vec::new(); + let mut level_slices: Vec = Vec::with_capacity(drained_ranges.len()); let mut visible_items_total = 0; + // Each row range is one contiguous level slice `[start_row_level, + // end_row_level)`, so we seek both boundaries and count its visibility at + // once rather than per row. The cursors only move forward, so locating and + // counting all requested ranges visits each intervening run at most once. for range in drained_ranges { - self.skip_to_row(range.start)?; - for _ in range.start..range.end { - let (level_range, visible) = self.take_row()?; - visible_items_total += visible; - if let Some(last) = level_slices.last_mut() - && last.end == level_range.start - { - last.end = level_range.end; - continue; - } - level_slices.push(level_range); + let level_start = self.seek_row_start(range.start)?; + let rep_run = self.rep_cursor.run; + let level_end = self.seek_row_start(range.end)?; + let (visible_items, def_run) = self.count_visible(level_start..level_end)?; + visible_items_total += visible_items; + if let Some(last) = level_slices.last_mut() + && last.range.end == level_start + { + last.range.end = level_end; + } else { + level_slices.push(LevelSlice { + range: level_start..level_end, + rep_run, + def_run, + }); } } @@ -1074,27 +1874,49 @@ impl StructuralPageDecoder for ComplexAllNullPageDecoder { /// We use `level_slices` to slice into `rep` and `def` and create rep/def buffers /// for the null data. +#[derive(Debug, Clone)] +struct LevelSlice { + range: Range, + rep_run: RunPosition, + def_run: RunPosition, +} + +#[derive(Clone, Copy)] +enum LevelKind { + Repetition, + Definition, +} + +impl LevelSlice { + fn run(&self, kind: LevelKind) -> RunPosition { + match kind { + LevelKind::Repetition => self.rep_run, + LevelKind::Definition => self.def_run, + } + } +} + #[derive(Debug)] pub struct DecodeComplexAllNullTask { - level_slices: Vec>, + level_slices: Vec, visible_items_total: u64, - rep: Option>, - def: Option>, + rep: Option, + def: Option, def_meaning: Arc<[DefinitionInterpretation]>, max_visible_level: u16, } impl DecodeComplexAllNullTask { - fn decode_level(&self, levels: &Option>) -> Option> { + fn decode_level(&self, levels: &Option, kind: LevelKind) -> Option> { levels.as_ref().map(|levels| { let num_levels = self .level_slices .iter() - .map(|range| range.end - range.start) + .map(|slice| slice.range.end - slice.range.start) .sum(); let mut referenced_levels = Vec::with_capacity(num_levels); - for range in &self.level_slices { - referenced_levels.extend(levels[range.start..range.end].iter().copied()); + for slice in &self.level_slices { + levels.extend_into(slice.range.clone(), slice.run(kind), &mut referenced_levels); } referenced_levels }) @@ -1103,8 +1925,8 @@ impl DecodeComplexAllNullTask { impl DecodePageTask for DecodeComplexAllNullTask { fn decode(self: Box) -> Result { - let rep = self.decode_level(&self.rep); - let def = self.decode_level(&self.def); + let rep = self.decode_level(&self.rep, LevelKind::Repetition); + let def = self.decode_level(&self.def, LevelKind::Definition); // If there are definition levels there may be empty / null lists which are not visible // in the items array. We need to account for that here to figure out how many values @@ -1202,125 +2024,22 @@ struct MiniBlockSchedulerDictionary { // These come from the protobuf dictionary_decompressor: Arc, dictionary_buf_position_and_size: (u64, u64), - dictionary_data_alignment: u64, - num_dictionary_items: u64, -} - -/// Individual block metadata within a MiniBlock repetition index. -#[derive(Debug)] -struct MiniBlockRepIndexBlock { - // The index of the first row that starts after the beginning of this block. If the block - // has a preamble this will be the row after the preamble. If the block is entirely preamble - // then this will be a row that starts in some future block. - first_row: u64, - // The number of rows in the block, including the trailer but not the preamble. - // Can be 0 if the block is entirely preamble - starts_including_trailer: u64, - // Whether the block has a preamble - has_preamble: bool, - // Whether the block has a trailer - has_trailer: bool, -} - -impl DeepSizeOf for MiniBlockRepIndexBlock { - fn deep_size_of_children(&self, _context: &mut Context) -> usize { - 0 - } -} - -/// Repetition index for MiniBlock encoding. -/// -/// Stores block-level offset information to enable efficient random -/// access to nested data structures within mini-blocks. -#[derive(Debug)] -struct MiniBlockRepIndex { - blocks: Vec, -} - -impl DeepSizeOf for MiniBlockRepIndex { - fn deep_size_of_children(&self, context: &mut Context) -> usize { - self.blocks.deep_size_of_children(context) - } -} - -impl MiniBlockRepIndex { - /// Decode repetition index from chunk metadata using default values. - /// - /// This creates a repetition index where each chunk has no partial values - /// and no trailers, suitable for simple sequential data layouts. - pub fn default_from_chunks(chunks: &[ChunkMeta]) -> Self { - let mut blocks = Vec::with_capacity(chunks.len()); - let mut offset: u64 = 0; - - for c in chunks { - blocks.push(MiniBlockRepIndexBlock { - first_row: offset, - starts_including_trailer: c.num_values, - has_preamble: false, - has_trailer: false, - }); - - offset += c.num_values; - } - - Self { blocks } - } - - /// Decode repetition index from raw bytes in little-endian format. - /// - /// The bytes should contain u64 values arranged in groups of `stride` elements, - /// where the first two values of each group represent ends_count and partial_count. - /// Returns an empty index if no bytes are provided. - pub fn decode_from_bytes(rep_bytes: &[u8], stride: usize) -> Self { - // Convert bytes to u64 slice, handling alignment automatically - let buffer = crate::buffer::LanceBuffer::from(rep_bytes.to_vec()); - let u64_slice = buffer.borrow_to_typed_slice::(); - let n = u64_slice.len() / stride; - - let mut blocks = Vec::with_capacity(n); - let mut chunk_has_preamble = false; - let mut offset: u64 = 0; - - // Extract first two values from each block: ends_count and partial_count - for i in 0..n { - let base_idx = i * stride; - let ends = u64_slice[base_idx]; - let partial = u64_slice[base_idx + 1]; - - let has_trailer = partial > 0; - // Convert branches to arithmetic for better compiler optimization - let starts_including_trailer = - ends + (has_trailer as u64) - (chunk_has_preamble as u64); - - blocks.push(MiniBlockRepIndexBlock { - first_row: offset, - starts_including_trailer, - has_preamble: chunk_has_preamble, - has_trailer, - }); - - chunk_has_preamble = has_trailer; - offset += starts_including_trailer; - } - - Self { blocks } - } + dictionary_data_alignment: u64, + num_dictionary_items: u64, } /// State that is loaded once and cached for future lookups #[derive(Debug)] struct MiniBlockCacheableState { - /// Metadata that describes each chunk in the page - chunk_meta: Vec, - /// The decoded repetition index - rep_index: MiniBlockRepIndex, + /// Compact per-chunk index (byte ranges + row/item mapping) for the page + chunk_index: MiniBlockChunkIndex, /// The dictionary for the page, if any dictionary: Option>, } impl DeepSizeOf for MiniBlockCacheableState { fn deep_size_of_children(&self, context: &mut Context) -> usize { - self.rep_index.deep_size_of_children(context) + self.chunk_index.deep_size_of_children(context) + self .dictionary .as_ref() @@ -1465,19 +2184,14 @@ impl MiniBlockScheduler { } fn lookup_chunks(&self, chunk_indices: &[usize]) -> Vec { - let page_meta = self.page_meta.as_ref().unwrap(); + let chunk_index = &self.page_meta.as_ref().unwrap().chunk_index; chunk_indices .iter() - .map(|&chunk_idx| { - let chunk_meta = &page_meta.chunk_meta[chunk_idx]; - let bytes_start = chunk_meta.offset_bytes; - let bytes_end = bytes_start + chunk_meta.chunk_size_bytes; - LoadedChunk { - byte_range: bytes_start..bytes_end, - items_in_chunk: chunk_meta.num_values, - chunk_idx, - data: LanceBuffer::empty(), - } + .map(|&chunk_idx| LoadedChunk { + byte_range: chunk_index.byte_range(chunk_idx), + items_in_chunk: chunk_index.items_in_chunk(chunk_idx), + chunk_idx, + data: LanceBuffer::empty(), }) .collect() } @@ -1582,9 +2296,12 @@ impl ChunkInstructions { // // The output will be a set of `ChunkInstructions` which tell us how to read from the chunks fn schedule_instructions( - rep_index: &MiniBlockRepIndex, + chunk_index: &MiniBlockChunkIndex, user_ranges: &[Range], ) -> Vec { + // Bind the per-page chunk count once; re-deriving it each iteration + // costs a width match plus a length read. + let num_chunks = chunk_index.num_chunks(); // This is an in-exact capacity guess but pretty good. The actual capacity can be // smaller if instructions are merged. It can be larger if there are multiple instructions // per row which can happen with lists. @@ -1596,43 +2313,30 @@ impl ChunkInstructions { // Need to find the first chunk with a first row >= user_range.start. If there are // multiple chunks with the same first row we need to take the first one. - let mut block_index = match rep_index - .blocks - .binary_search_by_key(&user_range.start, |block| block.first_row) - { - Ok(idx) => { - // Slightly tricky case, we may need to walk backwards a bit to make sure we - // are grabbing first eligible chunk - let mut idx = idx; - while idx > 0 && rep_index.blocks[idx - 1].first_row == user_range.start { - idx -= 1; - } - idx - } - // Easy case. idx is greater, and idx - 1 is smaller, so idx - 1 contains the start - Err(idx) => idx - 1, - }; + let mut block_index = chunk_index.find_chunk(user_range.start); - let mut to_skip = user_range.start - rep_index.blocks[block_index].first_row; + let mut to_skip = user_range.start - chunk_index.first_row(block_index); while rows_needed > 0 || need_preamble { // Check if we've gone past the last block (should not happen) - if block_index >= rep_index.blocks.len() { + if block_index >= num_chunks { log::warn!( - "schedule_instructions inconsistency: block_index >= rep_index.blocks.len(), exiting early" + "schedule_instructions inconsistency: block_index >= num_chunks, exiting early" ); break; } - let chunk = &rep_index.blocks[block_index]; - let rows_avail = chunk.starts_including_trailer.saturating_sub(to_skip); + let starts_including_trailer = chunk_index.rows_in_chunk(block_index); + let has_preamble = chunk_index.has_preamble(block_index); + let has_trailer = chunk_index.has_trailer(block_index); + let rows_avail = starts_including_trailer.saturating_sub(to_skip); // Handle blocks that are entirely preamble (rows_avail = 0) // These blocks have no rows to take but may have a preamble we need // We only look for preamble if to_skip == 0 (we're not skipping rows) if rows_avail == 0 && to_skip == 0 { // Only process if this chunk has a preamble we need - if chunk.has_preamble && need_preamble { + if has_preamble && need_preamble { chunk_instructions.push(Self { chunk_idx: block_index, preamble: PreambleAction::Take, @@ -1641,14 +2345,12 @@ impl ChunkInstructions { // We still need to look at has_trailer to distinguish between "all preamble // and row ends at end of chunk" and "all preamble and row bleeds into next // chunk". Both cases will have 0 rows available. - take_trailer: chunk.has_trailer, + take_trailer: has_trailer, }); // Only set need_preamble = false if the chunk has at least one row, // Or we are reaching the last block, // Otherwise, the chunk is entirely preamble and we need the next chunk's preamble too - if chunk.starts_including_trailer > 0 - || block_index == rep_index.blocks.len() - 1 - { + if starts_including_trailer > 0 || block_index == num_chunks - 1 { need_preamble = false; } } @@ -1663,7 +2365,7 @@ impl ChunkInstructions { if rows_avail == 0 && to_skip > 0 { // This block doesn't have enough rows to skip, move to next block // Adjust to_skip by the number of rows in this block - to_skip -= chunk.starts_including_trailer; + to_skip -= starts_including_trailer; block_index += 1; continue; } @@ -1672,7 +2374,7 @@ impl ChunkInstructions { rows_needed -= rows_to_take; let mut take_trailer = false; - let preamble = if chunk.has_preamble { + let preamble = if has_preamble { if need_preamble { PreambleAction::Take } else { @@ -1683,7 +2385,7 @@ impl ChunkInstructions { }; // Are we taking the trailer? If so, make sure we mark that we need the preamble - if rows_to_take == rows_avail && chunk.has_trailer { + if rows_to_take == rows_avail && has_trailer { take_trailer = true; need_preamble = true; } else { @@ -1707,25 +2409,33 @@ impl ChunkInstructions { // are _adjacent_ (i.e. don't merge "take first row of chunk 0" and "take third row of chunk 0" into "take 2 // rows of chunk 0 starting at 0") if user_ranges.len() > 1 { - // TODO: Could probably optimize this allocation away - let mut merged_instructions = Vec::with_capacity(chunk_instructions.len()); - let mut instructions_iter = chunk_instructions.into_iter(); - merged_instructions.push(instructions_iter.next().unwrap()); - for instruction in instructions_iter { - let last = merged_instructions.last_mut().unwrap(); - if last.chunk_idx == instruction.chunk_idx - && last.rows_to_take + last.rows_to_skip == instruction.rows_to_skip - { - last.rows_to_take += instruction.rows_to_take; - last.take_trailer |= instruction.take_trailer; + // Merge adjacent instructions in place. `write` indexes the last + // retained instruction; each following instruction is either folded + // into it (contiguous within the same chunk) or compacted forward. + let mut write = 0; + for read in 1..chunk_instructions.len() { + let merges = { + let last = &chunk_instructions[write]; + let candidate = &chunk_instructions[read]; + last.chunk_idx == candidate.chunk_idx + && last.rows_to_take + last.rows_to_skip == candidate.rows_to_skip + }; + if merges { + let rows_to_take = chunk_instructions[read].rows_to_take; + let take_trailer = chunk_instructions[read].take_trailer; + let last = &mut chunk_instructions[write]; + last.rows_to_take += rows_to_take; + last.take_trailer |= take_trailer; } else { - merged_instructions.push(instruction); + write += 1; + if write != read { + chunk_instructions.swap(write, read); + } } } - merged_instructions - } else { - chunk_instructions + chunk_instructions.truncate(write + 1); } + chunk_instructions } fn drain_from_instruction( @@ -1836,6 +2546,160 @@ impl<'a> Iterator for WordsIter<'a> { } } +/// Per-chunk leaf value-count analysis derived from the metadata words. +/// +/// `values_per_chunk` is the count shared by every non-last chunk (meaningful +/// when `uniform`), and `last_chunk_values` is the final chunk's count. +struct FlatValueCounts { + logs: Vec, + uniform: bool, + values_per_chunk: u64, + last_chunk_values: u64, +} + +fn analyze_value_counts(words: &Words, items_in_page: u64) -> Result { + let num_chunks = words.len(); + let logs = words.iter().map(|w| (w & 0x0F) as u8).collect::>(); + let mut counted = 0u64; + for (chunk_index, &log) in logs.iter().take(num_chunks.saturating_sub(1)).enumerate() { + if log == 0 { + return Err(Error::corrupt_file_named( + "miniblock_metadata", + format!( + "non-final chunk {chunk_index} of {num_chunks} has invalid log_num_values=0" + ), + )); + } + counted = counted.checked_add(1u64 << log).ok_or_else(|| { + Error::corrupt_file_named( + "miniblock_metadata", + format!( + "value count overflow at chunk {chunk_index}: counted_values={counted}, \ + log_num_values={log}, items_in_page={items_in_page}" + ), + ) + })?; + } + let last_chunk_values = items_in_page.checked_sub(counted).ok_or_else(|| { + Error::corrupt_file_named( + "miniblock_metadata", + format!( + "non-final chunks account for counted_values={counted}, exceeding \ + items_in_page={items_in_page}" + ), + ) + })?; + if let Some(&last_log) = logs.last() + && last_log != 0 + && (1u64 << last_log) != last_chunk_values + { + return Err(Error::corrupt_file_named( + "miniblock_metadata", + format!( + "final chunk log_num_values={last_log} does not match \ + last_chunk_values={last_chunk_values}: counted_values={counted}, \ + items_in_page={items_in_page}" + ), + )); + } + let uniform = num_chunks <= 1 || logs[..num_chunks - 1].iter().all(|&log| log == logs[0]); + // A single-chunk page has no "non-last" chunk to derive a stride from; use the + // page item count (min 1 so it stays a valid divisor in `find_chunk`). + let values_per_chunk = if num_chunks <= 1 { + items_in_page.max(1) + } else { + 1u64 << logs[0] + }; + Ok(FlatValueCounts { + logs, + uniform, + values_per_chunk, + last_chunk_values, + }) +} + +/// Iterator over per-chunk value counts for a non-uniform flat page. Non-last +/// chunks yield `1 << log`; the last yields the validated remaining item count. +fn flat_value_counts_iter(logs: &[u8], last_chunk_values: u64) -> impl Iterator + '_ { + let num_chunks = logs.len(); + (0..num_chunks).map(move |i| { + if i + 1 < num_chunks { + 1u64 << logs[i] + } else { + last_chunk_values + } + }) +} + +/// Builds the compact per-chunk index from the metadata words and, for nested +/// pages, the raw repetition-index bytes. The row axis is picked by page shape: +/// `UniformFlat` when all non-last chunks share a value count (fixed-width / +/// bitpacking), `Flat` for non-uniform flat pages (RLE / FSST), else `Nested`. +fn build_chunk_index( + words: &Words, + items_in_page: u64, + base: u64, + data_buf_size: u64, + rep_index_bytes: Option<&[u8]>, + repetition_index_depth: u16, +) -> Result { + let num_chunks = words.len(); + // Validate item counts before byte sizes because both share a metadata word, + // and an invalid count must not reach the final-chunk subtraction. + let value_counts = analyze_value_counts(words, items_in_page)?; + + // Each chunk stores `(divided_bytes + 1) * MINIBLOCK_ALIGNMENT` bytes, so the + // deltas are the chunk sizes and their grand total is the data buffer size. + let byte_starts = PrefixSums::from_deltas( + words + .iter() + .map(|word| ((word >> 4) as u64 + 1) * MINIBLOCK_ALIGNMENT as u64), + num_chunks, + data_buf_size, + ); + + // Nested pages track rows via the repetition index and keep leaf item counts + // separately; flat pages have row == value index, so value counts are rows. + let rows = if let Some(rep_index_data) = rep_index_bytes { + assert!(rep_index_data.len() % 8 == 0); + let stride = repetition_index_depth as usize + 1; + let (row_starts, has_trailer) = parse_nested_rep(rep_index_data, stride); + let item_counts = if value_counts.uniform { + ItemCounts::Uniform { + values_per_chunk: value_counts.values_per_chunk, + last_chunk_values: value_counts.last_chunk_values, + } + } else { + ItemCounts::PerChunkLog { + logs: value_counts.logs, + last_chunk_values: value_counts.last_chunk_values, + } + }; + RowMapping::Nested { + row_starts, + has_trailer, + item_counts, + } + } else { + if value_counts.uniform { + RowMapping::UniformFlat { + values_per_chunk: value_counts.values_per_chunk, + last_chunk_values: value_counts.last_chunk_values, + num_chunks, + } + } else { + let value_starts = PrefixSums::from_deltas( + flat_value_counts_iter(&value_counts.logs, value_counts.last_chunk_values), + num_chunks, + items_in_page, + ); + RowMapping::Flat { value_starts } + } + }; + + Ok(MiniBlockChunkIndex::new(base, byte_starts, rows)) +} + impl StructuralPageScheduler for MiniBlockScheduler { fn initialize<'a>( &'a mut self, @@ -1845,7 +2709,8 @@ impl StructuralPageScheduler for MiniBlockScheduler { // we may also need to fetch the repetition index. Here, we gather what buffers we // need. let (meta_buf_position, meta_buf_size) = self.buffer_offsets_and_sizes[0]; - let value_buf_position = self.buffer_offsets_and_sizes[1].0; + let base = self.buffer_offsets_and_sizes[1].0; + let data_buf_size = self.buffer_offsets_and_sizes[1].1; let mut bufs_needed = 1; if self.dictionary.is_some() { bufs_needed += 1; @@ -1874,65 +2739,34 @@ impl StructuralPageScheduler for MiniBlockScheduler { let dictionary_bytes = self.dictionary.as_ref().and_then(|_| buffers.next()); let rep_index_bytes = buffers.next(); - // Parse the metadata and build the chunk meta let words = Words::from_bytes(meta_bytes, self.has_large_chunk)?; - let mut chunk_meta = Vec::with_capacity(words.len()); - - let mut rows_counter = 0; - let mut offset_bytes = value_buf_position; - for (word_idx, word) in words.iter().enumerate() { - let log_num_values = word & 0x0F; - let divided_bytes = word >> 4; - let num_bytes = (divided_bytes as usize + 1) * MINIBLOCK_ALIGNMENT; - debug_assert!(num_bytes > 0); - let num_values = if word_idx < words.len() - 1 { - debug_assert!(log_num_values > 0); - 1 << log_num_values - } else { - debug_assert!( - log_num_values == 0 - || (1 << log_num_values) == (self.items_in_page - rows_counter) - ); - self.items_in_page - rows_counter - }; - rows_counter += num_values; - - chunk_meta.push(ChunkMeta { - num_values, - chunk_size_bytes: num_bytes as u64, - offset_bytes, - }); - offset_bytes += num_bytes as u64; - } - - // Build the repetition index - let rep_index = if let Some(rep_index_data) = rep_index_bytes { - assert!(rep_index_data.len() % 8 == 0); - let stride = self.repetition_index_depth as usize + 1; - MiniBlockRepIndex::decode_from_bytes(&rep_index_data, stride) - } else { - MiniBlockRepIndex::default_from_chunks(&chunk_meta) - }; - - let mut page_meta = MiniBlockCacheableState { - chunk_meta, - rep_index, - dictionary: None, - }; + let chunk_index = build_chunk_index( + &words, + self.items_in_page, + base, + data_buf_size, + rep_index_bytes.as_deref(), + self.repetition_index_depth, + )?; // decode dictionary - if let Some(ref mut dictionary) = self.dictionary { + let dictionary = if let Some(ref mut dictionary) = self.dictionary { let dictionary_data = dictionary_bytes.unwrap(); - page_meta.dictionary = - Some(Arc::new(dictionary.dictionary_decompressor.decompress( - LanceBuffer::from_bytes( - dictionary_data, - dictionary.dictionary_data_alignment, - ), - dictionary.num_dictionary_items, - )?)); + Some(Arc::new(dictionary.dictionary_decompressor.decompress( + Some(LanceBuffer::from_bytes( + dictionary_data, + dictionary.dictionary_data_alignment, + )), + dictionary.num_dictionary_items, + )?)) + } else { + None }; - let page_meta = Arc::new(page_meta); + + let page_meta = Arc::new(MiniBlockCacheableState { + chunk_index, + dictionary, + }); self.page_meta = Some(page_meta.clone()); Ok(page_meta as Arc) } @@ -1958,7 +2792,7 @@ impl StructuralPageScheduler for MiniBlockScheduler { let page_meta = self.page_meta.as_ref().unwrap(); let chunk_instructions = - ChunkInstructions::schedule_instructions(&page_meta.rep_index, ranges); + ChunkInstructions::schedule_instructions(&page_meta.chunk_index, ranges); debug_assert_eq!( num_rows, @@ -2294,7 +3128,7 @@ impl FullZipScheduler { num_rows, bits_per_offset, bits_per_offset, - ))) + )?)) } } } @@ -2709,7 +3543,7 @@ impl VariableFullZipDecoder { num_rows: u64, in_bits_per_length: u8, out_bits_per_offset: u8, - ) -> Self { + ) -> Result { let decompressor = match details.value_decompressor { PerValueDecompressor::Variable(ref d) => d.clone(), _ => unreachable!(), @@ -2754,9 +3588,9 @@ impl VariableFullZipDecoder { // - We could force each decode task to do a full unzip of all the data. Each decode task now // has to do more work but the work is all fused. // - We could just try doing this work on the decode thread and see if it is a problem. - decoder.unzip(data, in_bits_per_length, out_bits_per_offset, num_rows); + decoder.unzip(data, in_bits_per_length, out_bits_per_offset, num_rows)?; - decoder + Ok(decoder) } fn slice_batch_data_and_rebase_offsets_typed( @@ -2831,28 +3665,33 @@ impl VariableFullZipDecoder { } } - unsafe fn parse_length(data: &[u8], bits_per_offset: u8) -> u64 { - match bits_per_offset { - 8 => *data.get_unchecked(0) as u64, - 16 => u16::from_le_bytes([*data.get_unchecked(0), *data.get_unchecked(1)]) as u64, - 32 => u32::from_le_bytes([ - *data.get_unchecked(0), - *data.get_unchecked(1), - *data.get_unchecked(2), - *data.get_unchecked(3), - ]) as u64, - 64 => u64::from_le_bytes([ - *data.get_unchecked(0), - *data.get_unchecked(1), - *data.get_unchecked(2), - *data.get_unchecked(3), - *data.get_unchecked(4), - *data.get_unchecked(5), - *data.get_unchecked(6), - *data.get_unchecked(7), - ]), - _ => unreachable!(), + /// Reads a single length prefix from the front of `data`. + /// + /// The bytes come from the file. A page whose item walk ends with a partial + /// trailing item leaves fewer than `bits_per_offset / 8` bytes here, so this + /// is bounds checked and reports a corrupt file rather than reading past the + /// end of the buffer. + fn parse_length(data: &[u8], bits_per_offset: u8) -> Result { + let width = bits_per_offset as usize / 8; + if data.len() < width { + return Err(Error::corrupt_file_named( + "variable_full_zip", + format!( + "truncated length prefix: {} byte(s) remain in the page buffer but a \ + {}-bit length prefix requires {}", + data.len(), + bits_per_offset, + width + ), + )); } + Ok(match bits_per_offset { + 8 => data[0] as u64, + 16 => u16::from_le_bytes(data[..2].try_into().unwrap()) as u64, + 32 => u32::from_le_bytes(data[..4].try_into().unwrap()) as u64, + 64 => u64::from_le_bytes(data[..8].try_into().unwrap()), + _ => unreachable!(), + }) } fn unzip( @@ -2861,7 +3700,7 @@ impl VariableFullZipDecoder { in_bits_per_length: u8, out_bits_per_offset: u8, num_rows: u64, - ) { + ) -> Result<()> { // This undercounts if there are lists but, at this point, we don't really know how many items we have let mut rep = Vec::with_capacity(num_rows as usize); let mut def = Vec::with_capacity(num_rows as usize); @@ -2912,9 +3751,7 @@ impl VariableFullZipDecoder { if ctrl_desc.is_visible { visible_item_count += 1; if ctrl_desc.is_valid_item { - // Safety: Data should have at least bytes_per_length bytes remaining - debug_assert!(databuf.len() >= bytes_per_length); - let length = unsafe { Self::parse_length(databuf, in_bits_per_length) }; + let length = Self::parse_length(databuf, in_bits_per_length)?; match out_bits_per_offset { 32 => offsets_data .extend_from_slice(&(current_offset as u32).to_le_bytes()), @@ -2950,6 +3787,7 @@ impl VariableFullZipDecoder { self.def = ScalarBuffer::from(def); self.data = LanceBuffer::from(unzipped_data); self.offsets = LanceBuffer::from(offsets_data); + Ok(()) } } @@ -3066,15 +3904,31 @@ struct FixedFullZipDecodeTask { impl DecodePageTask for FixedFullZipDecodeTask { fn decode(self: Box) -> Result { - // Multiply by 2 to make a stab at the size of the output buffer (which will be decompressed and thus bigger) - let estimated_size_bytes = self - .data - .iter() - .map(|task_item| task_item.data.data_size() as usize) - .sum::() - * 2; - let mut data_builder = - DataBlockBuilder::with_capacity_estimate(estimated_size_bytes as u64); + let estimated_size_bytes = if self.details.ctrl_word_parser.bytes_per_word() == 0 { + let PerValueDecompressor::Fixed(decompressor) = &self.details.value_decompressor else { + return Err(Error::internal( + "FixedFullZipDecodeTask requires a fixed-width decompressor", + )); + }; + decompressor + .decoded_size_bytes(self.num_rows as u64) + .unwrap_or_else(|| { + self.data + .iter() + .map(|task_item| task_item.data.data_size()) + .sum::() + * 2 + }) + } else { + // Rep/def levels can suppress values, so the exact output size is not known + // until they are decoded. Keep the existing conservative estimate. + self.data + .iter() + .map(|task_item| task_item.data.data_size()) + .sum::() + * 2 + }; + let mut data_builder = DataBlockBuilder::with_capacity_estimate(estimated_size_bytes); if self.details.ctrl_word_parser.bytes_per_word() == 0 { // Fast path, no need to unzip because there is no rep/def @@ -3090,7 +3944,7 @@ impl DecodePageTask for FixedFullZipDecodeTask { }; debug_assert_eq!(fixed_data.num_values, task_item.rows_in_buf); let decompressed = decompressor.decompress(fixed_data, task_item.rows_in_buf)?; - data_builder.append(&decompressed, 0..task_item.rows_in_buf); + data_builder.append(&decompressed, 0..task_item.rows_in_buf)?; } let unraveler = RepDefUnraveler::new( @@ -3154,7 +4008,7 @@ impl DecodePageTask for FixedFullZipDecodeTask { unreachable!() }; let decompressed = decompressor.decompress(fixed_data, visible_items)?; - data_builder.append(&decompressed, 0..visible_items); + data_builder.append(&decompressed, 0..visible_items)?; } let repetition = if rep.is_empty() { None } else { Some(rep) }; @@ -3349,6 +4203,16 @@ impl StructuralPrimitiveFieldScheduler { mini_block, decompressors, )?), + Layout::SparseLayout(sparse_layout) => { + Box::new(sparse::SparseStructuralScheduler::try_new( + &page_info.buffer_offsets_and_sizes, + page_info.priority, + page_info.num_rows, + target_field.data_type(), + sparse_layout, + decompressors, + )?) + } Layout::FullZipLayout(full_zip) => { let mut scheduler = FullZipScheduler::try_new( &page_info.buffer_offsets_and_sizes, @@ -3381,25 +4245,22 @@ impl StructuralPrimitiveFieldScheduler { { Box::new(SimpleAllNullScheduler::default()) as Box } else { - let rep_decompressor = constant_layout - .rep_compression - .as_ref() - .map(|encoding| decompressors.create_block_decompressor(encoding)) - .transpose()? - .map(Arc::from); - - let def_decompressor = constant_layout - .def_compression - .as_ref() - .map(|encoding| decompressors.create_block_decompressor(encoding)) - .transpose()? - .map(Arc::from); + // RLE levels select a validated cache representation; other + // block compressions keep flowing through the eager decompressor. + let rep_codec = LevelCodec::try_new( + constant_layout.rep_compression.as_ref(), + decompressors, + )?; + let def_codec = LevelCodec::try_new( + constant_layout.def_compression.as_ref(), + decompressors, + )?; Box::new(ComplexAllNullScheduler::new( page_info.buffer_offsets_and_sizes.clone(), def_meaning.into(), - rep_decompressor, - def_decompressor, + rep_codec, + def_codec, constant_layout.num_rep_values, constant_layout.num_def_values, )) as Box @@ -3512,6 +4373,15 @@ impl CacheKey for FieldDataCacheKey { fn type_name() -> &'static str { "FieldData" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.encoding.logical.primitive.field-data-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.column_index); + builder.write_str(&self.view_tag); + } } impl StructuralFieldScheduler for StructuralPrimitiveFieldScheduler { @@ -3576,25 +4446,34 @@ impl StructuralCompositeDecodeArrayTask { fn restore_validity( array: Arc, unraveler: &mut CompositeRepDefUnraveler, - ) -> Arc { - let validity = unraveler.unravel_validity(array.len()); + ) -> Result> { + let validity = unraveler.unravel_validity(array.len())?; let Some(validity) = validity else { - return array; + return Ok(array); }; if array.data_type() == &DataType::Null { // We unravel from a null array but we don't add the null buffer because arrow-rs doesn't like it - return array; + return Ok(array); + } + if validity.len() != array.len() { + return Err(Error::invalid_input_source( + format!( + "Structural validity has {} entries for an array with {} values", + validity.len(), + array.len() + ) + .into(), + )); } - assert_eq!(validity.len(), array.len()); - // SAFETY: We've should have already asserted the buffers are all valid, we are just - // adding null buffers to the array here - make_array(unsafe { + // SAFETY: The array buffers have already been validated and the null buffer length + // matches the array. We are only attaching the null buffer here. + Ok(make_array(unsafe { array .to_data() .into_builder() .nulls(Some(validity)) .build_unchecked() - }) + })) } } @@ -3620,7 +4499,7 @@ impl StructuralDecodeArrayTask for StructuralCompositeDecodeArrayTask { let array = arrow_select::concat::concat(&array_refs)?; let mut repdef = CompositeRepDefUnraveler::new(unravelers); - let array = Self::restore_validity(array, &mut repdef); + let array = Self::restore_validity(array, &mut repdef)?; Ok(DecodedArray { array, @@ -3749,19 +4628,125 @@ const MINIBLOCK_ALIGNMENT: usize = 8; /// TODO: We should concatenate metadata buffers from all pages into a single buffer /// at (roughly) the end of the file so there is, at most, one read per column of /// metadata per file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MiniblockChunkSize { + U16, + U32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ComplexNullEncoding { + RawLevels, + CompressedLevels, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FixedWidthDictionaryEncoding { + Exclude64Bit, + Include64Bit, +} + +trait PrimitivePageEncodingBehavior: Send + Sync + Debug { + fn validate_field(&self, _field: &Field, _metadata: &HashMap) -> Result<()> { + Ok(()) + } + + fn try_plan_pages( + &self, + _ctx: &PrimitivePlanContext<'_>, + _arrays: &[ArrayRef], + _normalized: &NormalizedStructuralPlan, + _row_number: u64, + _num_rows: u64, + _num_values: u64, + ) -> Result>> { + Ok(None) + } + + fn try_encode_page( + &self, + _ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + Ok(PrimitiveEncodeAttempt::Unhandled(page)) + } +} + +/// One executable primitive-page behavior selected by an exact file +/// composition. +#[derive(Debug, Clone)] +pub struct PrimitivePageEncoding { + behavior: Arc, +} + +impl PrimitivePageEncoding { + /// Reject an explicit request for sparse structural encoding. + pub fn reject_sparse() -> Self { + Self { + behavior: Arc::new(RejectSparsePrimitiveEncoding), + } + } + + /// Encode constant non-null values as a constant page when applicable. + pub fn constant() -> Self { + Self { + behavior: Arc::new(ConstantPrimitiveEncoding), + } + } + + /// Plan and encode sparse structural pages when applicable. + pub fn sparse(compression: Arc) -> Self { + Self { + behavior: Arc::new(SparsePrimitiveEncoding { compression }), + } + } + + /// Encode dense pages with the original u16 miniblock grammar. + pub fn dense_u16(compression: Arc) -> Self { + Self { + behavior: Arc::new(DenseU16PrimitiveEncoding { compression }), + } + } + + /// Encode dense pages with the u32 miniblock grammar. + pub fn dense_u32(compression: Arc) -> Self { + Self { + behavior: Arc::new(DenseU32PrimitiveEncoding { compression }), + } + } +} + +#[derive(Debug)] +struct RejectSparsePrimitiveEncoding; + +#[derive(Debug)] +struct ConstantPrimitiveEncoding; + +#[derive(Debug)] +struct SparsePrimitiveEncoding { + compression: Arc, +} + +#[derive(Debug)] +struct DenseU16PrimitiveEncoding { + compression: Arc, +} + +#[derive(Debug)] +struct DenseU32PrimitiveEncoding { + compression: Arc, +} + pub struct PrimitiveStructuralEncoder { // Accumulates arrays until we have enough data to justify a disk page accumulation_queue: AccumulationQueue, keep_original_array: bool, - support_large_chunk: bool, accumulated_repdefs: Vec, - // The compression strategy we will use to compress the data - compression_strategy: Arc, + page_encodings: Arc<[PrimitivePageEncoding]>, column_index: u32, field: Field, encoding_metadata: Arc>, - version: LanceFileVersion, } struct CompressedLevelsChunk { @@ -3787,18 +4772,38 @@ struct DictEncodingBudget { max_encoded_size: usize, } -// A primitive page after optional structural splitting. +enum PrimitivePageStructure { + Dense { + repdef: SerializedRepDefs, + single_row_miniblock_repdef_levels: Option, + }, + Sparse { + plan: sparse::SparseStructuralPlan, + prepared_values: Option, + }, +} + +// A primitive page after structural encoding selection and optional dense splitting. struct PrimitivePageData { // Arrow leaf arrays that contain this page's visible values. arrays: Vec, - // Repetition / definition levels aligned to this page. - repdef: SerializedRepDefs, + // Structural representation aligned to this page. + structure: PrimitivePageStructure, // Top-level row number of the first row in this page. row_number: u64, // Number of top-level rows in this page. num_rows: u64, - // Present when one top-level row is too large for one miniblock rep/def chunk. - unsplittable_miniblock_levels: Option, +} + +struct PrimitivePlanContext<'a> { + column_idx: u32, + field: &'a Field, + encoding_metadata: &'a HashMap, +} + +enum PrimitiveEncodeAttempt { + Encoded(EncodedPage), + Unhandled(PrimitivePageData), } // Immutable encoder state shared by per-page encode tasks. @@ -3809,47 +4814,60 @@ struct PrimitivePageData { struct PrimitiveEncodeContext { // Column being encoded. column_idx: u32, - // Logical field metadata for compression/layout selection. field: Field, - // Compression strategy shared across pages. - compression_strategy: Arc, - // Field-level encoding metadata such as structural encoding overrides. encoding_metadata: Arc>, - // Whether miniblock chunks may use the v2.2 large-chunk metadata. - support_large_chunk: bool, - // Lance file version selected by the writer. - version: LanceFileVersion, - // True when the only rep/def information is simple nullable validity. is_simple_validity: bool, - // True when the field has any non-empty rep/def information. has_repdef_info: bool, } impl PrimitiveStructuralEncoder { pub fn try_new( options: &EncodingOptions, - compression_strategy: Arc, + page_encodings: Arc<[PrimitivePageEncoding]>, column_index: u32, field: Field, encoding_metadata: Arc>, ) -> Result { + for page_encoding in page_encodings.iter() { + page_encoding + .behavior + .validate_field(&field, &encoding_metadata)?; + } Ok(Self { accumulation_queue: AccumulationQueue::new( options.cache_bytes_per_column, column_index, options.keep_original_array, ), - support_large_chunk: options.support_large_chunk(), keep_original_array: options.keep_original_array, accumulated_repdefs: Vec::new(), column_index, - compression_strategy, + page_encodings, field, encoding_metadata, - version: options.version, }) } + fn encode_page( + page_encodings: &[PrimitivePageEncoding], + ctx: &PrimitiveEncodeContext, + mut page: PrimitivePageData, + ) -> Result { + for page_encoding in page_encodings { + match page_encoding.behavior.try_encode_page(ctx, page)? { + PrimitiveEncodeAttempt::Encoded(page) => return Ok(page), + PrimitiveEncodeAttempt::Unhandled(unhandled) => page = unhandled, + } + } + Err(Error::invalid_input_source( + format!( + "No primitive page encoding atom supports field '{}'", + ctx.field.name + ) + .into(), + )) + } + // TODO: This is a heuristic we may need to tune at some point // // As data gets narrow then the "zipping" process gets too expensive @@ -3944,7 +4962,7 @@ impl PrimitiveStructuralEncoder { miniblocks: MiniBlockCompressed, rep: Option>, def: Option>, - support_large_chunk: bool, + miniblock_chunk_size: MiniblockChunkSize, ) -> Result { let bytes_rep = rep .as_ref() @@ -3965,7 +4983,10 @@ impl PrimitiveStructuralEncoder { // 2 bytes for the length of each buffer and up to 7 bytes of padding per buffer let max_extra = 9 * num_buffers; let mut data_buffer = Vec::with_capacity(bytes_rep + bytes_def + bytes_data + max_extra); - let chunk_size_bytes = if support_large_chunk { 4 } else { 2 }; + let chunk_size_bytes = match miniblock_chunk_size { + MiniblockChunkSize::U16 => 2, + MiniblockChunkSize::U32 => 4, + }; let mut meta_buffer = Vec::with_capacity(miniblocks.chunks.len() * chunk_size_bytes); let mut rep_iter = rep.map(|r| r.into_iter()); @@ -4007,7 +5028,7 @@ impl PrimitiveStructuralEncoder { data_buffer.extend_from_slice(&bytes_def.to_le_bytes()); } - if support_large_chunk { + if miniblock_chunk_size == MiniblockChunkSize::U32 { for &buffer_size in &chunk.buffer_sizes { data_buffer.extend_from_slice(&buffer_size.to_le_bytes()); } @@ -4052,10 +5073,9 @@ impl PrimitiveStructuralEncoder { } let chunk_bytes = data_buffer.len() - start_pos; - let max_chunk_size = if support_large_chunk { - 1_u64 << 31 // 28 bits of 8-byte words in u32 metadata - } else { - 32 * 1024 // 32KiB limit with u16 metadata + let max_chunk_size = match miniblock_chunk_size { + MiniblockChunkSize::U16 => 32 * 1024, + MiniblockChunkSize::U32 => 1_u64 << 31, }; if chunk_bytes == 0 || chunk_bytes as u64 > max_chunk_size { return Err(Error::internal(format!( @@ -4082,7 +5102,7 @@ impl PrimitiveStructuralEncoder { let divided_bytes_minus_one = (divided_bytes - 1) as u64; let metadata = (divided_bytes_minus_one << 4) | chunk.log_num_values as u64; - if support_large_chunk { + if miniblock_chunk_size == MiniblockChunkSize::U32 { meta_buffer.extend_from_slice(&(metadata as u32).to_le_bytes()); } else { meta_buffer.extend_from_slice(&(metadata as u16).to_le_bytes()); @@ -4132,8 +5152,9 @@ impl PrimitiveStructuralEncoder { let levels_block = DataBlock::FixedWidth(fixed_width_block); let levels_field = Field::new_arrow("", DataType::UInt16, false)?; // Pick a block compressor - let (compressor, compressor_desc) = + let compressor = compression_strategy.create_block_compressor(&levels_field, &levels_block)?; + let mut compressor_desc = None; // Compress blocks of levels (sized according to the chunks) let mut level_chunks = Vec::with_capacity(chunks.len()); let mut values_counter = 0; @@ -4203,7 +5224,22 @@ impl PrimitiveStructuralEncoder { }; chunk_fixed_width.compute_stat(); let chunk_levels_block = DataBlock::FixedWidth(chunk_fixed_width); - let compressed_levels = compressor.compress(chunk_levels_block)?; + let (compressed_levels, chunk_compressor_desc) = + compressor.compress(chunk_levels_block)?; + if let Some(compressor_desc) = compressor_desc.as_ref() { + if compressor_desc != &chunk_compressor_desc { + return Err(Error::internal( + "Rep/def block compressor changed encoding between chunks".to_string(), + )); + } + } else { + compressor_desc = Some(chunk_compressor_desc); + } + let compressed_levels = compressed_levels.ok_or_else(|| { + Error::internal( + "Rep/def block compressor selected a metadata-only codec".to_string(), + ) + })?; let num_levels = u16::try_from(num_chunk_levels).map_err(|_| { Error::invalid_input_source( format!( @@ -4228,7 +5264,9 @@ impl PrimitiveStructuralEncoder { }; Ok(CompressedLevels { data: level_chunks, - compression: compressor_desc, + compression: compressor_desc.ok_or_else(|| { + Error::internal("Rep/def compression produced no chunks".to_string()) + })?, rep_index, }) } @@ -4264,9 +5302,8 @@ impl PrimitiveStructuralEncoder { let levels_block = DataBlock::FixedWidth(fixed_width_block); let levels_field = Field::new_arrow("", DataType::UInt16, false)?; - let (compressor, encoding) = - compression_strategy.create_block_compressor(&levels_field, &levels_block)?; - let compressed_buffer = compressor.compress(levels_block)?; + let (compressed_buffer, encoding) = + compress_required_block(compression_strategy, &levels_field, levels_block)?; Ok((compressed_buffer, encoding)) } @@ -4278,10 +5315,10 @@ impl PrimitiveStructuralEncoder { repdef: crate::repdef::SerializedRepDefs, row_number: u64, num_rows: u64, - version: LanceFileVersion, + complex_null_encoding: ComplexNullEncoding, compression_strategy: &dyn CompressionStrategy, ) -> Result { - if version.resolve() < LanceFileVersion::V2_2 { + if complex_null_encoding == ComplexNullEncoding::RawLevels { let rep_bytes = if let Some(rep) = repdef.repetition_levels.as_ref() { LanceBuffer::reinterpret_slice(rep.clone()) } else { @@ -4362,7 +5399,7 @@ impl PrimitiveStructuralEncoder { return Ok(None); } let mut validity = BooleanBufferBuilder::new(num_values); - unraveler.unravel_validity(&mut validity); + unraveler.unravel_validity(&mut validity)?; Ok(Some(validity.finish())) } @@ -4575,7 +5612,7 @@ impl PrimitiveStructuralEncoder { row_number: u64, dictionary_data: Option, num_rows: u64, - support_large_chunk: bool, + miniblock_chunk_size: MiniblockChunkSize, ) -> Result { if let DataBlock::AllNull(_null_block) = data { // We should not be using mini-block for all-null. There are other structural @@ -4586,7 +5623,12 @@ impl PrimitiveStructuralEncoder { let num_items = data.num_values(); let compressor = compression_strategy.create_miniblock_compressor(field, &data)?; - let (compressed_data, value_encoding) = compressor.compress(data)?; + let common_chunk_buffers = + u64::from(repdef.rep_slicer().is_some()) + u64::from(repdef.def_slicer().is_some()); + let support_large_chunk = miniblock_chunk_size == MiniblockChunkSize::U32; + let compression_context = + MiniBlockCompressionContext::new(common_chunk_buffers, support_large_chunk, true); + let (compressed_data, value_encoding) = compressor.compress(compression_context, data)?; let max_rep = repdef.def_meaning.iter().filter(|l| l.is_list()).count() as u16; @@ -4635,7 +5677,8 @@ impl PrimitiveStructuralEncoder { .map(|cd| std::mem::take(&mut cd.data)); let serialized = - Self::serialize_miniblocks(compressed_data, rep_data, def_data, support_large_chunk)?; + Self::serialize_miniblocks(compressed_data, rep_data, def_data, miniblock_chunk_size)?; + let has_large_chunk = miniblock_chunk_size == MiniblockChunkSize::U32; // Metadata, Data, Dictionary, (maybe) Repetition Index let mut data = Vec::with_capacity(4); @@ -4646,9 +5689,8 @@ impl PrimitiveStructuralEncoder { let num_dictionary_items = dictionary_data.num_values(); let dict_values_field = Self::build_dict_values_compressor_field(field)?; - let (compressor, dictionary_encoding) = compression_strategy - .create_block_compressor(&dict_values_field, &dictionary_data)?; - let dictionary_buffer = compressor.compress(dictionary_data)?; + let (dictionary_buffer, dictionary_encoding) = + compress_required_block(compression_strategy, &dict_values_field, dictionary_data)?; data.push(dictionary_buffer); if let Some(rep_index) = rep_index { @@ -4664,7 +5706,7 @@ impl PrimitiveStructuralEncoder { Some((dictionary_encoding, num_dictionary_items)), &repdef.def_meaning, num_items, - support_large_chunk, + has_large_chunk, ); Ok(EncodedPage { num_rows, @@ -4683,7 +5725,7 @@ impl PrimitiveStructuralEncoder { None, &repdef.def_meaning, num_items, - support_large_chunk, + has_large_chunk, ); if let Some(rep_index) = rep_index { @@ -4741,8 +5783,7 @@ impl PrimitiveStructuralEncoder { if control.is_new_row { // We have finished a row debug_assert!(offset <= len); - // SAFETY: We know that `start <= len` - unsafe { rep_index_builder.append(offset as u64) }; + rep_index_builder.append_trusted(offset as u64); } offset = zipped_data.len(); } @@ -4753,8 +5794,7 @@ impl PrimitiveStructuralEncoder { if control.is_new_row { // We have finished a row debug_assert!(offset <= len); - // SAFETY: We know that `start <= len` - unsafe { rep_index_builder.append(offset as u64) }; + rep_index_builder.append_trusted(offset as u64); } if control.is_visible { let value = data_iter.next().unwrap(); @@ -4766,10 +5806,7 @@ impl PrimitiveStructuralEncoder { debug_assert_eq!(zipped_data.len(), len); // Put the final value in the rep index - // SAFETY: `zipped_data.len() == len` - unsafe { - rep_index_builder.append(zipped_data.len() as u64); - } + rep_index_builder.append_trusted(zipped_data.len() as u64); let zipped_data = LanceBuffer::from(zipped_data); let rep_index = rep_index_builder.into_data(); @@ -4821,8 +5858,7 @@ impl PrimitiveStructuralEncoder { if control.is_new_row { // We have finished a row debug_assert!(rep_offset <= len); - // SAFETY: We know that `buf.len() <= len` - unsafe { rep_index_builder.append(rep_offset as u64) }; + rep_index_builder.append_trusted(rep_offset as u64); } if control.is_visible { let window = windows_iter.next().unwrap(); @@ -4844,8 +5880,7 @@ impl PrimitiveStructuralEncoder { if control.is_new_row { // We have finished a row debug_assert!(rep_offset <= len); - // SAFETY: We know that `buf.len() <= len` - unsafe { rep_index_builder.append(rep_offset as u64) }; + rep_index_builder.append_trusted(rep_offset as u64); } if control.is_visible { let window = windows_iter.next().unwrap(); @@ -4874,10 +5909,7 @@ impl PrimitiveStructuralEncoder { // if we are over `len` then we have a bug. debug_assert!(buf.len() <= len); // Put the final value in the rep index - // SAFETY: `zipped_data.len() == len` - unsafe { - rep_index_builder.append(buf.len() as u64); - } + rep_index_builder.append_trusted(buf.len() as u64); let zipped_data = LanceBuffer::from(buf); let rep_index = rep_index_builder.into_data(); @@ -5015,7 +6047,7 @@ impl PrimitiveStructuralEncoder { fn should_dictionary_encode( data_block: &DataBlock, field: &Field, - version: LanceFileVersion, + fixed_width_dictionary_encoding: FixedWidthDictionaryEncoding, ) -> Option { const DEFAULT_SAMPLE_SIZE: usize = 4096; const DEFAULT_SAMPLE_UNIQUE_RATIO: f64 = 0.98; @@ -5024,7 +6056,9 @@ impl PrimitiveStructuralEncoder { // estimating the size for other types. match data_block { DataBlock::FixedWidth(fixed) => { - if fixed.bits_per_value == 64 && version < LanceFileVersion::V2_2 { + if fixed.bits_per_value == 64 + && fixed_width_dictionary_encoding == FixedWidthDictionaryEncoding::Exclude64Bit + { return None; } if fixed.bits_per_value != 64 && fixed.bits_per_value != 128 { @@ -5292,33 +6326,37 @@ impl PrimitiveStructuralEncoder { Ok(sliced) } - fn split_structural_pages_for_miniblock_budget( + fn split_pages_for_miniblock_repdef_budget( arrays: Vec, repdef: SerializedRepDefs, - plan: StructuralPagePlan, + budget: MiniBlockRepDefBudget, row_number: u64, num_rows: u64, ) -> Result> { - if plan == StructuralPagePlan::Fits { + if budget == MiniBlockRepDefBudget::WithinBudget { return Ok(vec![PrimitivePageData { arrays, - repdef, + structure: PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels: None, + }, row_number, num_rows, - unsplittable_miniblock_levels: None, }]); } - if let StructuralPagePlan::UnsplittableOverBudget(num_levels) = plan { + if let MiniBlockRepDefBudget::SingleRowOverBudget(num_levels) = budget { return Ok(vec![PrimitivePageData { arrays, - repdef, + structure: PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels: Some(num_levels), + }, row_number, num_rows, - unsplittable_miniblock_levels: Some(num_levels), }]); } - let StructuralPagePlan::Split(splits) = plan else { + let MiniBlockRepDefBudget::RequiresPageSplit(splits) = budget else { unreachable!(); }; @@ -5328,35 +6366,50 @@ impl PrimitiveStructuralEncoder { let repdef = Self::slice_repdef(&repdef, split.level_range); pages.push(PrimitivePageData { arrays, - repdef, + structure: PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels: None, + }, row_number: row_number + split.row_start, num_rows: split.num_rows, - unsplittable_miniblock_levels: None, }); } Ok(pages) } - fn encode_page(ctx: PrimitiveEncodeContext, page: PrimitivePageData) -> Result { + fn encode_dense_page( + ctx: PrimitiveEncodeContext, + page: PrimitivePageData, + compression_strategy: Arc, + miniblock_chunk_size: MiniblockChunkSize, + complex_null_encoding: ComplexNullEncoding, + fixed_width_dictionary_encoding: FixedWidthDictionaryEncoding, + ) -> Result { let PrimitiveEncodeContext { column_idx, field, - compression_strategy, encoding_metadata, - support_large_chunk, - version, is_simple_validity, has_repdef_info, } = ctx; let PrimitivePageData { arrays, - repdef, + structure, row_number, num_rows, - unsplittable_miniblock_levels, } = page; let num_values = arrays.iter().map(|arr| arr.len() as u64).sum(); + let (repdef, single_row_miniblock_repdef_levels) = match structure { + PrimitivePageStructure::Dense { + repdef, + single_row_miniblock_repdef_levels, + } => (repdef, single_row_miniblock_repdef_levels), + PrimitivePageStructure::Sparse { .. } => { + unreachable!("dense atom received sparse page") + } + }; + if num_values == 0 { // This page contains only structural events, such as empty/null list rows. // The existing complex-null layout stores the rep/def stream without value buffers. @@ -5371,7 +6424,7 @@ impl PrimitiveStructuralEncoder { repdef, row_number, num_rows, - version, + complex_null_encoding, compression_strategy.as_ref(), ); } @@ -5403,7 +6456,7 @@ impl PrimitiveStructuralEncoder { repdef, row_number, num_rows, - version, + complex_null_encoding, compression_strategy.as_ref(), ) }; @@ -5422,26 +6475,15 @@ impl PrimitiveStructuralEncoder { let data_block = DataBlock::from_arrays(&arrays, num_values); - if version.resolve() >= LanceFileVersion::V2_2 - && let Some(scalar) = Self::find_constant_scalar(&arrays, leaf_validity.as_ref())? - { - log::debug!( - "Encoding column {} with {} items ({} rows) using constant layout", - column_idx, - num_values, - num_rows - ); - return constant::encode_constant_page( - column_idx, scalar, repdef, row_number, num_rows, - ); - } - - if let Some(num_levels) = unsplittable_miniblock_levels { + if let Some(num_levels) = single_row_miniblock_repdef_levels { let requested_encoding = encoding_metadata .get(STRUCTURAL_ENCODING_META_KEY) .map(|requested| requested.to_lowercase()); let fullzip_error = match &data_block { - DataBlock::FixedWidth(fixed) if !fixed.bits_per_value.is_multiple_of(8) => { + // 1-bit booleans are widened to bytes inside `encode_full_zip`. + DataBlock::FixedWidth(fixed) + if fixed.bits_per_value != 1 && !fixed.bits_per_value.is_multiple_of(8) => + { Some(format!( "Full-zip fixed-width values must be byte aligned, got {} bits per value", fixed.bits_per_value @@ -5463,14 +6505,6 @@ impl PrimitiveStructuralEncoder { variable.bits_per_offset )) } - DataBlock::Struct(struct_data_block) - if !struct_data_block.has_variable_width_child() => - { - Some( - "Full-zip packed struct requires at least one variable-width child" - .to_string(), - ) - } DataBlock::Dictionary(_) => { Some("Full-zip does not encode dictionary data blocks directly".to_string()) } @@ -5584,24 +6618,29 @@ impl PrimitiveStructuralEncoder { row_number, Some(dictionary_data_block), num_rows, - support_large_chunk, + miniblock_chunk_size, ); } // Try dictionary encoding first if applicable. If encoding aborts, fall back to the // preferred structural encoding. - let dict_result = Self::should_dictionary_encode(&data_block, &field, version).and_then(|budget| { - log::debug!( - "Encoding column {} with {} items using dictionary encoding (mini-block layout)", - column_idx, - num_values - ); - dict::dictionary_encode( - &data_block, - budget.max_dict_entries, - budget.max_encoded_size, - ) - }); + let dict_result = Self::should_dictionary_encode( + &data_block, + &field, + fixed_width_dictionary_encoding, + ) + .and_then(|budget| { + log::debug!( + "Encoding column {} with {} items using dictionary encoding (mini-block layout)", + column_idx, + num_values + ); + dict::dictionary_encode( + &data_block, + budget.max_dict_entries, + budget.max_encoded_size, + ) + }); if let Some((indices_data_block, dictionary_data_block)) = dict_result { Self::encode_miniblock( @@ -5613,7 +6652,7 @@ impl PrimitiveStructuralEncoder { row_number, Some(dictionary_data_block), num_rows, - support_large_chunk, + miniblock_chunk_size, ) } else if Self::prefers_miniblock(&data_block, encoding_metadata.as_ref()) { log::debug!( @@ -5630,7 +6669,7 @@ impl PrimitiveStructuralEncoder { row_number, None, num_rows, - support_large_chunk, + miniblock_chunk_size, ) } else if Self::prefers_fullzip(encoding_metadata.as_ref()) { log::debug!( @@ -5656,41 +6695,57 @@ impl PrimitiveStructuralEncoder { fn do_flush( &mut self, arrays: Vec, - repdefs: Vec, - row_number: u64, - num_rows: u64, - ) -> Result> { - let num_values = arrays.iter().map(|arr| arr.len() as u64).sum(); - let is_simple_validity = repdefs.iter().all(|rd| rd.is_simple_validity()); - let has_repdef_info = repdefs.iter().any(|rd| !rd.is_empty()); - let (repdef, structural_plan) = RepDefBuilder::serialize_with_structural_plan( - repdefs, - miniblock::max_repdef_levels_per_chunk, - num_rows, - num_values, - )?; - let pages = Self::split_structural_pages_for_miniblock_budget( - arrays, - repdef, - structural_plan, - row_number, - num_rows, - )?; + repdefs: Vec, + row_number: u64, + num_rows: u64, + ) -> Result> { + DataBlock::validate_arrays(&arrays, &self.field.name)?; + let num_values = arrays.iter().map(|arr| arr.len() as u64).sum(); + let is_simple_validity = repdefs.iter().all(|rd| rd.is_simple_validity()); + let has_repdef_info = repdefs.iter().any(|rd| !rd.is_empty()); + let normalized = RepDefBuilder::normalize(repdefs); + let plan_ctx = PrimitivePlanContext { + column_idx: self.column_index, + field: &self.field, + encoding_metadata: &self.encoding_metadata, + }; + let mut pages = None; + for page_encoding in self.page_encodings.iter() { + if let Some(planned) = page_encoding.behavior.try_plan_pages( + &plan_ctx, + &arrays, + &normalized, + row_number, + num_rows, + num_values, + )? { + pages = Some(planned); + break; + } + } + let pages = pages.ok_or_else(|| { + Error::invalid_input_source( + format!( + "No primitive page planner supports field '{}'", + self.field.name + ) + .into(), + ) + })?; let mut tasks = Vec::with_capacity(pages.len()); let ctx = PrimitiveEncodeContext { column_idx: self.column_index, field: self.field.clone(), - compression_strategy: self.compression_strategy.clone(), encoding_metadata: self.encoding_metadata.clone(), - support_large_chunk: self.support_large_chunk, - version: self.version, is_simple_validity, has_repdef_info, }; for page in pages { let ctx = ctx.clone(); - let task = spawn_cpu(move || Self::encode_page(ctx, page)).boxed(); + let page_encodings = self.page_encodings.clone(); + let task = + spawn_cpu(move || Self::encode_page(page_encodings.as_ref(), &ctx, page)).boxed(); tasks.push(task); } Ok(tasks) @@ -5727,6 +6782,7 @@ impl PrimitiveStructuralEncoder { } DataType::Dictionary(_, _) => { array = dict::normalize_dict_nulls(array)?; + array = dict::clear_out_of_range_null_keys(array)?; Self::extract_validity_buf(array, repdef, keep_original_array) } // Extract our validity buf but NOT any child validity bufs. (they will be encoded in @@ -5742,6 +6798,291 @@ impl PrimitiveStructuralEncoder { } } +impl PrimitivePageEncodingBehavior for RejectSparsePrimitiveEncoding { + fn validate_field(&self, field: &Field, metadata: &HashMap) -> Result<()> { + if metadata + .get(STRUCTURAL_ENCODING_META_KEY) + .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE)) + { + return Err(Error::invalid_input_source( + format!( + "Field '{}' requests sparse structural encoding, which is not enabled by the selected file format", + field.name + ) + .into(), + )); + } + Ok(()) + } +} + +fn plan_dense_primitive_pages( + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, +) -> Result> { + let (repdef, miniblock_repdef_budget) = normalized.serialize_with_miniblock_repdef_budget( + miniblock::max_repdef_levels_per_chunk, + num_rows, + num_values, + )?; + PrimitiveStructuralEncoder::split_pages_for_miniblock_repdef_budget( + arrays.to_vec(), + repdef, + miniblock_repdef_budget, + row_number, + num_rows, + ) +} + +impl PrimitivePageEncodingBehavior for DenseU16PrimitiveEncoding { + fn try_plan_pages( + &self, + _ctx: &PrimitivePlanContext<'_>, + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, + ) -> Result>> { + Ok(Some(plan_dense_primitive_pages( + arrays, normalized, row_number, num_rows, num_values, + )?)) + } + + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + if !matches!(&page.structure, PrimitivePageStructure::Dense { .. }) { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + Ok(PrimitiveEncodeAttempt::Encoded( + PrimitiveStructuralEncoder::encode_dense_page( + ctx.clone(), + page, + self.compression.clone(), + MiniblockChunkSize::U16, + ComplexNullEncoding::RawLevels, + FixedWidthDictionaryEncoding::Exclude64Bit, + )?, + )) + } +} + +impl PrimitivePageEncodingBehavior for DenseU32PrimitiveEncoding { + fn try_plan_pages( + &self, + _ctx: &PrimitivePlanContext<'_>, + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, + ) -> Result>> { + Ok(Some(plan_dense_primitive_pages( + arrays, normalized, row_number, num_rows, num_values, + )?)) + } + + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + if !matches!(&page.structure, PrimitivePageStructure::Dense { .. }) { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + Ok(PrimitiveEncodeAttempt::Encoded( + PrimitiveStructuralEncoder::encode_dense_page( + ctx.clone(), + page, + self.compression.clone(), + MiniblockChunkSize::U32, + ComplexNullEncoding::CompressedLevels, + FixedWidthDictionaryEncoding::Include64Bit, + )?, + )) + } +} + +impl PrimitivePageEncodingBehavior for SparsePrimitiveEncoding { + fn try_plan_pages( + &self, + ctx: &PrimitivePlanContext<'_>, + arrays: &[ArrayRef], + normalized: &NormalizedStructuralPlan, + row_number: u64, + num_rows: u64, + num_values: u64, + ) -> Result>> { + let requested_encoding = ctx.encoding_metadata.get(STRUCTURAL_ENCODING_META_KEY); + let requests_sparse = requested_encoding + .is_some_and(|requested| requested.eq_ignore_ascii_case(STRUCTURAL_ENCODING_SPARSE)); + if requests_sparse { + let plan = sparse::writer::plan(normalized, num_values)?; + if sparse::writer::uses_constant_layout(&plan, ctx.field) { + return Ok(None); + } + return Ok(Some(vec![PrimitivePageData { + arrays: arrays.to_vec(), + structure: PrimitivePageStructure::Sparse { + plan, + prepared_values: None, + }, + row_number, + num_rows, + }])); + } + + let (_, miniblock_repdef_budget) = normalized.serialize_with_miniblock_repdef_budget( + miniblock::max_repdef_levels_per_chunk, + num_rows, + num_values, + )?; + let automatic_sparse = layout::select_automatic_sparse( + requested_encoding.map(String::as_str), + &miniblock_repdef_budget, + || { + let data = DataBlock::from_arrays(arrays, num_values); + if !sparse::writer::supports_value_block(&data) { + return Ok(None); + } + let prepared_values = match sparse::writer::prepare_values( + ctx.field, + self.compression.as_ref(), + data, + MiniblockChunkSize::U32, + ) { + Ok(prepared_values) => prepared_values, + Err(error) => { + debug!( + "Keeping column {} on its dense structural path because sparse value preparation is unavailable: {}", + ctx.column_idx, error + ); + return Ok(None); + } + }; + let plan = sparse::writer::plan(normalized, num_values)?; + if sparse::writer::uses_constant_layout(&plan, ctx.field) { + return Ok(None); + } + Ok(Some((plan, prepared_values))) + }, + )?; + Ok(automatic_sparse.map(|(plan, prepared_values)| { + vec![PrimitivePageData { + arrays: arrays.to_vec(), + structure: PrimitivePageStructure::Sparse { + plan, + prepared_values: Some(prepared_values), + }, + row_number, + num_rows, + }] + })) + } + + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + if !matches!(&page.structure, PrimitivePageStructure::Sparse { .. }) { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + let PrimitivePageData { + arrays, + structure: + PrimitivePageStructure::Sparse { + plan, + prepared_values, + }, + row_number, + num_rows, + } = page + else { + unreachable!() + }; + let num_values = arrays.iter().map(|array| array.len() as u64).sum(); + log::debug!( + "Encoding column {} with {} visible items ({} rows) using sparse layout", + ctx.column_idx, + num_values, + num_rows + ); + Ok(PrimitiveEncodeAttempt::Encoded( + sparse::writer::encode_page( + ctx.column_idx, + &ctx.field, + self.compression.as_ref(), + prepared_values.map_or_else( + || { + sparse::writer::SparseValueInput::Unprepared(DataBlock::from_arrays( + &arrays, num_values, + )) + }, + sparse::writer::SparseValueInput::Prepared, + ), + plan, + row_number, + num_rows, + MiniblockChunkSize::U32, + )?, + )) + } +} + +impl PrimitivePageEncodingBehavior for ConstantPrimitiveEncoding { + fn try_encode_page( + &self, + ctx: &PrimitiveEncodeContext, + page: PrimitivePageData, + ) -> Result { + let PrimitivePageStructure::Dense { repdef, .. } = &page.structure else { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + }; + let num_values: u64 = page.arrays.iter().map(|array| array.len() as u64).sum(); + if num_values == 0 { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + let leaf_validity = PrimitiveStructuralEncoder::leaf_validity(repdef, num_values as usize)?; + if leaf_validity + .as_ref() + .is_some_and(|validity| validity.count_set_bits() == 0) + || matches!(ctx.field.data_type(), DataType::Struct(fields) if fields.is_empty()) + { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + } + let Some(scalar) = + PrimitiveStructuralEncoder::find_constant_scalar(&page.arrays, leaf_validity.as_ref())? + else { + return Ok(PrimitiveEncodeAttempt::Unhandled(page)); + }; + let PrimitivePageData { + structure: PrimitivePageStructure::Dense { repdef, .. }, + row_number, + num_rows, + .. + } = page + else { + unreachable!() + }; + log::debug!( + "Encoding column {} with {} items ({} rows) using constant layout", + ctx.column_idx, + num_values, + num_rows + ); + Ok(PrimitiveEncodeAttempt::Encoded( + constant::encode_constant_page(ctx.column_idx, scalar, repdef, row_number, num_rows)?, + )) + } +} + impl FieldEncoder for PrimitiveStructuralEncoder { // Buffers data, if there is enough to write a page then we create an encode task fn maybe_encode( @@ -5791,14 +7132,19 @@ impl FieldEncoder for PrimitiveStructuralEncoder { #[allow(clippy::single_range_in_vec_init)] mod tests { use super::{ - ChunkInstructions, DataBlock, DecodeMiniBlockTask, FixedPerValueDecompressor, - FixedWidthDataBlock, FullZipCacheableState, FullZipDecodeDetails, FullZipReadSource, - FullZipRepIndexDetails, FullZipScheduler, MiniBlockChunk, MiniBlockCompressed, - MiniBlockRepIndex, PerValueDecompressor, PreambleAction, StructuralPageScheduler, - VariableFullZipDecoder, + ChunkInstructions, DataBlock, DecodeMiniBlockTask, DecodePageTask, FixedFullZipDecodeTask, + FixedPerValueDecompressor, FixedWidthDataBlock, FixedWidthDictionaryEncoding, + FullZipCacheableState, FullZipDecodeDetails, FullZipDecodeTaskItem, FullZipReadSource, + FullZipRepIndexDetails, FullZipScheduler, LazyLevels, LevelCodec, LevelCursor, LevelPlan, + MiniBlockChunk, MiniBlockChunkIndex, MiniBlockCompressed, MiniblockChunkSize, + PerValueDataBlock, PerValueDecompressor, PreambleAction, RunEndsBuilder, RunPosition, + RunStorage, StructuralPageScheduler, VariableFullZipDecoder, dense_levels_from_block, + validate_complex_all_null_levels, }; use crate::buffer::LanceBuffer; - use crate::compression::DefaultDecompressionStrategy; + use crate::compression::{ + BlockCompressor, DefaultDecompressionStrategy, MiniBlockDecompressor, + }; use crate::constants::{ COMPRESSION_LEVEL_META_KEY, COMPRESSION_META_KEY, DICT_VALUES_COMPRESSION_LEVEL_META_KEY, DICT_VALUES_COMPRESSION_META_KEY, STRUCTURAL_ENCODING_META_KEY, @@ -5806,16 +7152,24 @@ mod tests { }; use crate::data::BlockInfo; use crate::decoder::{PageEncoding, StructuralFieldDecoder}; + use crate::encodings::logical::primitive::fullzip::PerValueCompressor; use crate::encodings::logical::primitive::{ - ChunkDrainInstructions, PrimitiveStructuralEncoder, StructuralPrimitiveFieldDecoder, + ChunkDrainInstructions, LoadedChunk, PrimitiveStructuralEncoder, + StructuralPrimitiveFieldDecoder, }; + use crate::encodings::physical::rle::{RleDecompressor, RleEncoder, RleRuns, RunLengthWidth}; + use crate::encodings::physical::value::{ValueDecompressor, ValueEncoder}; use crate::format::ProtobufUtils21; use crate::format::pb21; use crate::format::pb21::compressive_encoding::Compression; use crate::repdef::build_control_word_iterator; + use crate::testing::TestEncoding; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; - use arrow_array::{ArrayRef, Int8Array, StringArray}; + use arrow_array::{ + Array, ArrayRef, FixedSizeListArray, Float32Array, Int8Array, StringArray, UInt8Array, + make_array, + }; + use arrow_buffer::ScalarBuffer; use arrow_schema::{DataType, Field as ArrowField}; use std::collections::HashMap; use std::{collections::VecDeque, sync::Arc}; @@ -5881,9 +7235,315 @@ mod tests { else { panic!("expected full-zip to reject 1-bit fixed-width values"); }; - assert!( - err.to_string().contains("byte aligned"), - "unexpected error: {err}" + assert!( + err.to_string().contains("byte aligned"), + "unexpected error: {err}" + ); + } + + fn decode_fixed_fullzip_no_levels( + decompressor: Arc, + data: Vec, + num_rows: usize, + bytes_per_value: usize, + ) -> DataBlock { + Box::new(FixedFullZipDecodeTask { + details: Arc::new(FullZipDecodeDetails { + value_decompressor: PerValueDecompressor::Fixed(decompressor), + def_meaning: Arc::from([]), + ctrl_word_parser: crate::repdef::ControlWordParser::new(0, 0), + max_rep: 0, + max_visible_def: u16::MAX, + }), + data, + num_rows, + bytes_per_value, + }) + .decode() + .unwrap() + .data + } + + #[test] + fn test_fixed_fullzip_decode_preallocates_exact_output_size() { + #[derive(Debug)] + struct IdentityFixedDecompressor; + + impl FixedPerValueDecompressor for IdentityFixedDecompressor { + fn decompress( + &self, + data: FixedWidthDataBlock, + num_rows: u64, + ) -> crate::Result { + assert_eq!(data.num_values, num_rows); + Ok(DataBlock::FixedWidth(data)) + } + + fn bits_per_value(&self) -> u64 { + 32 + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values.checked_mul(4) + } + } + + let make_item = |num_rows: u64| FullZipDecodeTaskItem { + data: PerValueDataBlock::Fixed(FixedWidthDataBlock { + data: LanceBuffer::from(vec![7_u8; num_rows as usize * 4]), + bits_per_value: 32, + num_values: num_rows, + block_info: BlockInfo::new(), + }), + rows_in_buf: num_rows, + }; + + let num_rows = 512; + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(IdentityFixedDecompressor), + vec![make_item(128), make_item(384)], + num_rows, + 4, + ); + let values = decoded.as_fixed_width_ref().unwrap(); + let expected_size = num_rows * 4; + assert_eq!(values.data.len(), expected_size); + assert_eq!(values.data.clone().into_buffer().capacity(), expected_size); + } + + #[test] + fn test_fixed_fullzip_decode_falls_back_when_output_size_is_not_exact() { + #[derive(Debug)] + struct FallbackFixedDecompressor; + + impl FixedPerValueDecompressor for FallbackFixedDecompressor { + fn decompress( + &self, + data: FixedWidthDataBlock, + num_rows: u64, + ) -> crate::Result { + assert_eq!(data.num_values, num_rows); + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::from(vec![7_u8; num_rows as usize * 4]), + bits_per_value: 32, + num_values: num_rows, + block_info: BlockInfo::new(), + })) + } + + fn bits_per_value(&self) -> u64 { + // This deliberately cannot be multiplied by num_rows. If FullZip treats + // bits_per_value as an exact decoded-size estimate, decoding will fail. + u64::MAX - 7 + } + } + + let num_rows = 2; + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(FallbackFixedDecompressor), + vec![FullZipDecodeTaskItem { + data: PerValueDataBlock::Fixed(FixedWidthDataBlock { + data: LanceBuffer::from(vec![0_u8; num_rows * 4]), + bits_per_value: 32, + num_values: num_rows as u64, + block_info: BlockInfo::new(), + }), + rows_in_buf: num_rows as u64, + }], + num_rows, + 4, + ); + let values = decoded.as_fixed_width_ref().unwrap(); + assert_eq!(values.num_values, num_rows as u64); + assert_eq!(values.data.len(), num_rows * 4); + } + + #[test] + fn test_fixed_fullzip_real_fsl_preallocates_exact_output_size() { + let num_rows = 64; + let dimension = 32; + let items = Arc::new(Float32Array::from_iter_values( + (0..num_rows * dimension).map(|value| value as f32), + )); + let item_field = Arc::new(ArrowField::new("item", DataType::Float32, false)); + let sample = FixedSizeListArray::new(item_field, dimension as i32, items, None); + + let (data, compression) = PerValueCompressor::compress( + &ValueEncoder::default(), + DataBlock::from_array(sample.clone()), + ) + .unwrap(); + let Compression::FixedSizeList(fsl) = compression.compression.unwrap() else { + panic!("expected fixed-size-list compression"); + }; + let decompressor = ValueDecompressor::from_fsl(fsl.as_ref()); + let expected_size = num_rows * dimension * size_of::(); + assert_eq!( + FixedPerValueDecompressor::decoded_size_bytes(&decompressor, num_rows as u64), + Some(expected_size as u64) + ); + + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(decompressor), + vec![FullZipDecodeTaskItem { + data, + rows_in_buf: num_rows as u64, + }], + num_rows, + dimension * size_of::(), + ); + let fsl = decoded.as_fixed_size_list_ref().unwrap(); + let values = fsl.child.as_fixed_width_ref().unwrap(); + assert_eq!(values.data.len(), expected_size); + assert_eq!(values.data.clone().into_buffer().capacity(), expected_size); + + let decoded_array = make_array( + decoded + .into_arrow(sample.data_type().clone(), true) + .unwrap(), + ); + assert_eq!(decoded_array.as_ref(), &sample); + } + + #[test] + fn test_fixed_fullzip_nullable_fsl_uses_fallback_end_to_end() { + #[derive(Debug)] + struct NullableFslDecompressor { + inner: ValueDecompressor, + } + + impl FixedPerValueDecompressor for NullableFslDecompressor { + fn decompress( + &self, + data: FixedWidthDataBlock, + num_rows: u64, + ) -> crate::Result { + FixedPerValueDecompressor::decompress(&self.inner, data, num_rows) + } + + fn bits_per_value(&self) -> u64 { + // FullZip must not use this physical row width as an exact decoded-size + // estimate for the nullable, multi-buffer Arrow output. + u64::MAX - 7 + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + FixedPerValueDecompressor::decoded_size_bytes(&self.inner, num_values) + } + } + + let num_rows = 64; + let items = Arc::new(UInt8Array::from_iter( + (0..num_rows).map(|value| (value % 3 != 0).then_some(value as u8)), + )); + let item_field = Arc::new(ArrowField::new("item", DataType::UInt8, true)); + let sample = FixedSizeListArray::new(item_field, 1, items, None); + + let (data, compression) = PerValueCompressor::compress( + &ValueEncoder::default(), + DataBlock::from_array(sample.clone()), + ) + .unwrap(); + let Compression::FixedSizeList(fsl) = compression.compression.unwrap() else { + panic!("expected fixed-size-list compression"); + }; + let decompressor = NullableFslDecompressor { + inner: ValueDecompressor::from_fsl(fsl.as_ref()), + }; + assert_eq!( + FixedPerValueDecompressor::decoded_size_bytes(&decompressor, num_rows as u64), + None + ); + + let decoded = decode_fixed_fullzip_no_levels( + Arc::new(decompressor), + vec![FullZipDecodeTaskItem { + data, + rows_in_buf: num_rows as u64, + }], + num_rows, + 2, + ); + let decoded_array = make_array( + decoded + .into_arrow(sample.data_type().clone(), true) + .unwrap(), + ); + assert_eq!(decoded_array.as_ref(), &sample); + } + + #[test] + fn test_miniblock_decode_uses_exact_fixed_width_output_size() { + #[derive(Debug)] + struct FixedWidthMiniBlockDecompressor; + + impl MiniBlockDecompressor for FixedWidthMiniBlockDecompressor { + fn decompress( + &self, + data: Vec, + num_values: u64, + ) -> crate::Result { + assert_eq!(data.len(), 1); + Ok(DataBlock::FixedWidth(FixedWidthDataBlock { + data: data.into_iter().next().unwrap(), + bits_per_value: 32, + num_values, + block_info: BlockInfo::new(), + })) + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values.checked_mul(4) + } + } + + let num_rows = 512; + let expected_size = num_rows * 4; + let mut chunk_data = Vec::new(); + chunk_data.extend_from_slice(&0_u16.to_le_bytes()); + chunk_data.extend_from_slice(&(expected_size as u16).to_le_bytes()); + let header_padding = + lance_core::utils::bit::pad_bytes::<{ super::MINIBLOCK_ALIGNMENT }>(chunk_data.len()); + chunk_data.resize(chunk_data.len() + header_padding, 0); + chunk_data.resize(chunk_data.len() + expected_size as usize, 7); + + let task = DecodeMiniBlockTask { + rep_decompressor: None, + def_decompressor: None, + value_decompressor: Arc::new(FixedWidthMiniBlockDecompressor), + dictionary_data: None, + def_meaning: Arc::from([]), + num_buffers: 1, + max_visible_level: 0, + instructions: vec![( + ChunkDrainInstructions { + chunk_instructions: ChunkInstructions { + chunk_idx: 0, + preamble: PreambleAction::Absent, + rows_to_skip: 0, + rows_to_take: num_rows, + take_trailer: false, + }, + rows_to_skip: 0, + rows_to_take: num_rows, + preamble_action: PreambleAction::Absent, + }, + LoadedChunk { + byte_range: 0..chunk_data.len() as u64, + data: LanceBuffer::from(chunk_data), + items_in_chunk: num_rows, + chunk_idx: 0, + }, + )], + has_large_chunk: false, + }; + + let decoded = Box::new(task).decode().unwrap(); + let values = decoded.data.as_fixed_width_ref().unwrap(); + assert_eq!(values.data.len(), expected_size as usize); + assert_eq!( + values.data.clone().into_buffer().capacity(), + expected_size as usize ); } @@ -6289,11 +7949,10 @@ mod tests { // Convert repetition index to bytes for testing let rep_data: Vec = vec![5, 2, 3, 0, 4, 7, 2, 0]; let rep_bytes: Vec = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect(); - let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2); + let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2); let check = |user_ranges, expected_instructions| { - let instructions = - ChunkInstructions::schedule_instructions(&repetition_index, user_ranges); + let instructions = ChunkInstructions::schedule_instructions(&chunk_index, user_ranges); assert_eq!(instructions, expected_instructions); }; @@ -6445,11 +8104,11 @@ mod tests { // Convert repetition index to bytes for testing let rep_data: Vec = vec![5, 2, 3, 0, 4, 7, 2, 0]; let rep_bytes: Vec = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect(); - let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2); + let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2); let user_ranges = vec![1..7, 10..14]; // First, schedule the ranges - let scheduled = ChunkInstructions::schedule_instructions(&repetition_index, &user_ranges); + let scheduled = ChunkInstructions::schedule_instructions(&chunk_index, &user_ranges); let mut to_drain = VecDeque::from(scheduled.clone()); @@ -6530,11 +8189,11 @@ mod tests { // Regression case. Need a chunk with preamble, rows, and trailer (the middle chunk here) let rep_data: Vec = vec![5, 2, 3, 3, 20, 0]; let rep_bytes: Vec = rep_data.iter().flat_map(|v| v.to_le_bytes()).collect(); - let repetition_index = MiniBlockRepIndex::decode_from_bytes(&rep_bytes, 2); + let chunk_index = MiniBlockChunkIndex::new_nested_for_test(&rep_bytes, 2); let user_ranges = vec![0..28]; // First, schedule the ranges - let scheduled = ChunkInstructions::schedule_instructions(&repetition_index, &user_ranges); + let scheduled = ChunkInstructions::schedule_instructions(&chunk_index, &user_ranges); let mut to_drain = VecDeque::from(scheduled.clone()); @@ -6594,6 +8253,181 @@ mod tests { assert_eq!(skip_in_chunk, 0); } + use super::chunk_index::{PrefixSums, RowMapping}; + use super::{MINIBLOCK_ALIGNMENT, Words, build_chunk_index}; + use bytes::Bytes; + use lance_core::cache::{Context, DeepSizeOf}; + use rstest::rstest; + + /// Builds a `Words` metadata buffer (u16 words) from `(log_num_values, num_bytes)` + /// pairs, returning the words and the total data-buffer size. + fn words_from(entries: &[(u32, u32)]) -> (Words, u64) { + let mut raw = Vec::with_capacity(entries.len() * 2); + let mut total = 0u64; + for &(log, num_bytes) in entries { + assert!(num_bytes > 0 && num_bytes % MINIBLOCK_ALIGNMENT as u32 == 0); + let divided = num_bytes / MINIBLOCK_ALIGNMENT as u32 - 1; + let word = (divided << 4) | log; + assert!(word <= u16::MAX as u32, "test word {word} exceeds u16"); + raw.extend_from_slice(&(word as u16).to_le_bytes()); + total += num_bytes as u64; + } + (Words::from_bytes(Bytes::from(raw), false).unwrap(), total) + } + + fn rep_bytes_from(values: &[u64]) -> Vec { + values.iter().flat_map(|v| v.to_le_bytes()).collect() + } + + #[rstest] + // Two full chunks of 8 values (log 3) plus a partial last chunk; byte sizes vary + // independently of value counts. + #[case::uniform_partial_last(&[(3, 16), (3, 24), (0, 8)], 19, "uniform_flat", 8, 3)] + // Single chunk covers the whole page. + #[case::single_chunk(&[(0, 24)], 5, "uniform_flat", 5, 5)] + // Last chunk is also full (exact multiple). + #[case::exact_multiple(&[(3, 16), (3, 16)], 16, "uniform_flat", 8, 8)] + // Non-last chunks differ in size, so this is a non-uniform flat page. + #[case::non_uniform(&[(4, 16), (2, 16), (0, 8)], 21, "flat", 16, 1)] + fn test_flat_detection( + #[case] entries: &[(u32, u32)], + #[case] items_in_page: u64, + #[case] expected_kind: &str, + #[case] expected_first_items: u64, + #[case] expected_last_items: u64, + ) { + let base = 100u64; + let (words, data_buf_size) = words_from(entries); + let index = build_chunk_index(&words, items_in_page, base, data_buf_size, None, 0).unwrap(); + + assert_eq!(index.row_mapping_debug(), expected_kind); + assert_eq!(index.num_chunks(), entries.len()); + assert_eq!(index.items_in_chunk(0), expected_first_items); + assert_eq!(index.items_in_chunk(entries.len() - 1), expected_last_items); + + // Byte ranges are absolute, contiguous, and exactly cover the data buffer. + let mut expected_start = base; + for (i, &(_, num_bytes)) in entries.iter().enumerate() { + let range = index.byte_range(i); + assert_eq!(range.start, expected_start); + assert_eq!(range.end - range.start, num_bytes as u64); + expected_start = range.end; + } + assert_eq!(expected_start, base + data_buf_size); + + // For flat pages rows == items, so the per-chunk items sum to the page total. + let total_items: u64 = (0..index.num_chunks()) + .map(|i| index.items_in_chunk(i)) + .sum(); + assert_eq!(total_items, items_in_page); + } + + #[test] + fn test_nested_detection_and_axes() { + // Repetition index (stride 2): three chunks holding 5, 4, 3 rows, no trailers. + let rep = rep_bytes_from(&[5, 0, 4, 0, 3, 0]); + + // Uniform leaf chunking: value counts 4, 4, 2. + let (words, data_buf_size) = words_from(&[(2, 8), (2, 8), (0, 8)]); + let index = build_chunk_index(&words, 10, 0, data_buf_size, Some(&rep), 1).unwrap(); + assert_eq!(index.row_mapping_debug(), "nested"); + assert_eq!(index.num_chunks(), 3); + // Rows come from the repetition index, not the value counts. + assert_eq!(index.first_row(0), 0); + assert_eq!(index.rows_in_chunk(0), 5); + assert_eq!(index.first_row(1), 5); + assert_eq!(index.rows_in_chunk(1), 4); + assert_eq!(index.first_row(2), 9); + assert_eq!(index.rows_in_chunk(2), 3); + // Items come from the value words. + assert_eq!(index.items_in_chunk(0), 4); + assert_eq!(index.items_in_chunk(1), 4); + assert_eq!(index.items_in_chunk(2), 2); + + // Non-uniform leaf chunking: value counts 8, 2, 5. + let (words_nu, dbs_nu) = words_from(&[(3, 8), (1, 8), (0, 8)]); + let index_nu = build_chunk_index(&words_nu, 15, 0, dbs_nu, Some(&rep), 1).unwrap(); + assert_eq!(index_nu.row_mapping_debug(), "nested"); + assert_eq!(index_nu.items_in_chunk(0), 8); + assert_eq!(index_nu.items_in_chunk(1), 2); + assert_eq!(index_nu.items_in_chunk(2), 5); + // The row axis is unchanged by the leaf chunking. + assert_eq!(index_nu.rows_in_chunk(0), 5); + } + + #[test] + fn test_uniform_flat_matches_prefix_sum_flat() { + // Distribution: 4 chunks of 4 values, last chunk 3 (15 items total). + let (words, data_buf_size) = words_from(&[(2, 8), (2, 8), (2, 8), (0, 8)]); + let uniform = build_chunk_index(&words, 15, 0, data_buf_size, None, 0).unwrap(); + assert_eq!(uniform.row_mapping_debug(), "uniform_flat"); + + // The same distribution expressed as a non-uniform Flat prefix-sum index. + let byte_starts = PrefixSums::from_deltas([8u64, 8, 8, 8].into_iter(), 4, 32); + let value_starts = PrefixSums::from_deltas([4u64, 4, 4, 3].into_iter(), 4, 15); + let flat = MiniBlockChunkIndex::new(0, byte_starts, RowMapping::Flat { value_starts }); + assert_eq!(flat.row_mapping_debug(), "flat"); + + // Lookup parity: identical byte ranges and item counts. + for i in 0..4 { + assert_eq!(uniform.byte_range(i), flat.byte_range(i)); + assert_eq!(uniform.items_in_chunk(i), flat.items_in_chunk(i)); + } + + // Scheduler parity across scan / single-row / partial / scattered multi-range. + let range_sets: Vec>> = vec![ + vec![0..15], + vec![0..1], + vec![7..8], + vec![14..15], + vec![3..10], + vec![0..2, 5..6, 12..15], + ]; + for ranges in &range_sets { + let from_uniform = ChunkInstructions::schedule_instructions(&uniform, ranges); + let from_flat = ChunkInstructions::schedule_instructions(&flat, ranges); + assert_eq!(from_uniform, from_flat, "mismatch for ranges {ranges:?}"); + } + + // A full scan yields one Absent, no-trailer instruction per chunk. + let full = ChunkInstructions::schedule_instructions(&uniform, &[0..15]); + assert_eq!(full.len(), 4); + for (i, inst) in full.iter().enumerate() { + assert_eq!(inst.chunk_idx, i); + assert_eq!(inst.preamble, PreambleAction::Absent); + assert_eq!(inst.rows_to_skip, 0); + assert!(!inst.take_trailer); + } + assert_eq!(full.iter().map(|i| i.rows_to_take).sum::(), 15); + } + + #[test] + fn test_deep_size_per_variant_below_legacy() { + // The previous representation cached 48 bytes per chunk (24 for ChunkMeta plus + // 24 for a rep-index block); every variant's heap must be well below that. + const LEGACY_PER_CHUNK: usize = 48; + let num_chunks = 3; + let heap = |index: &MiniBlockChunkIndex| index.deep_size_of_children(&mut Context::new()); + + let (uniform_words, uniform_dbs) = words_from(&[(2, 8), (2, 8), (0, 8)]); + let uniform = build_chunk_index(&uniform_words, 10, 0, uniform_dbs, None, 0).unwrap(); + assert_eq!(uniform.row_mapping_debug(), "uniform_flat"); + assert!(heap(&uniform) < LEGACY_PER_CHUNK * num_chunks); + + let (flat_words, flat_dbs) = words_from(&[(3, 8), (1, 8), (0, 8)]); + let flat = build_chunk_index(&flat_words, 11, 0, flat_dbs, None, 0).unwrap(); + assert_eq!(flat.row_mapping_debug(), "flat"); + assert!(heap(&flat) < LEGACY_PER_CHUNK * num_chunks); + // Flat carries a value-starts array that UniformFlat derives arithmetically. + assert!(heap(&flat) > heap(&uniform)); + + let rep = rep_bytes_from(&[4, 0, 3, 0, 3, 0]); + let (nested_words, nested_dbs) = words_from(&[(2, 8), (2, 8), (0, 8)]); + let nested = build_chunk_index(&nested_words, 10, 0, nested_dbs, Some(&rep), 1).unwrap(); + assert_eq!(nested.row_mapping_debug(), "nested"); + assert!(heap(&nested) < LEGACY_PER_CHUNK * num_chunks); + } + #[tokio::test] async fn test_fullzip_initialize_is_lazy() { use futures::{FutureExt, future::BoxFuture}; @@ -6958,7 +8792,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_batch_size(100) .with_range(0..num_rows.min(500) as u64) .with_indices(vec![0, num_rows as u64 / 2, (num_rows - 1) as u64]); @@ -6969,7 +8803,7 @@ mod tests { async fn test_minichunk_size_helper( string_data: Vec>, minichunk_size: u64, - file_version: LanceFileVersion, + encodings: &[TestEncoding], ) { use crate::constants::MINICHUNK_SIZE_META_KEY; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; @@ -6989,7 +8823,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(file_version) + .with_encodings(encodings.iter().copied()) .with_batch_size(1000); check_round_trip_encoding_of_data(vec![string_array], &test_cases, metadata).await; @@ -7003,7 +8837,16 @@ mod tests { string_data.push(Some(format!("test_string_{}", i).repeat(50))); } // configure minichunk size to 64 bytes (smaller than the default 4kb) for Lance 2.1 - test_minichunk_size_helper(string_data, 64, LanceFileVersion::V2_1).await; + test_minichunk_size_helper( + string_data, + 64, + &[ + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse, + ], + ) + .await; } #[tokio::test] @@ -7014,7 +8857,12 @@ mod tests { for i in 0..10000 { string_data.push(Some(format!("test_string_{}", i).repeat(50))); } - test_minichunk_size_helper(string_data, 128 * 1024, LanceFileVersion::V2_2).await; + test_minichunk_size_helper( + string_data, + 128 * 1024, + &[TestEncoding::StructuralU32, TestEncoding::StructuralSparse], + ) + .await; } #[tokio::test] @@ -7024,7 +8872,12 @@ mod tests { for i in 0..10000 { string_data.push(Some(format!("t_{}", i))); } - test_minichunk_size_helper(string_data, 128 * 1024, LanceFileVersion::V2_2).await; + test_minichunk_size_helper( + string_data, + 128 * 1024, + &[TestEncoding::StructuralU32, TestEncoding::StructuralSparse], + ) + .await; } #[tokio::test] @@ -7043,7 +8896,7 @@ mod tests { let repeated_strings: Vec<_> = unique_values .iter() .cycle() - .take(100_000) + .take(10_000) .map(|s| Some(s.as_str())) .collect(); @@ -7051,7 +8904,7 @@ mod tests { // Configure test to use V2_2 and verify encoding let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_verify_encoding(Arc::new(|cols: &[crate::encoder::EncodedColumn], _| { assert_eq!(cols.len(), 1); let col = &cols[0]; @@ -7121,7 +8974,7 @@ mod tests { ) .with_metadata(metadata); - encode_first_page(field, dict_array, LanceFileVersion::V2_2).await + encode_first_page(field, dict_array, TestEncoding::StructuralU32).await } async fn encode_auto_fixed_dict_page( @@ -7151,7 +9004,7 @@ mod tests { let field = arrow_schema::Field::new("fixed_col", DataType::Decimal128(38, 0), false) .with_metadata(field_metadata); - encode_first_page(field, decimal, LanceFileVersion::V2_2).await + encode_first_page(field, decimal, TestEncoding::StructuralU32).await } #[tokio::test] @@ -7278,7 +9131,6 @@ mod tests { async fn test_dictionary_encode_int64() { use crate::constants::{DICT_SIZE_RATIO_META_KEY, STRUCTURAL_ENCODING_META_KEY}; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; use arrow_array::{ArrayRef, Int64Array}; use std::collections::HashMap; use std::sync::Arc; @@ -7301,7 +9153,7 @@ mod tests { metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.99".to_string()); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_batch_size(1000) .with_range(0..1000) .with_indices(vec![0, 1, 10, 999]) @@ -7314,7 +9166,6 @@ mod tests { async fn test_dictionary_encode_float64() { use crate::constants::{DICT_SIZE_RATIO_META_KEY, STRUCTURAL_ENCODING_META_KEY}; use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; use arrow_array::{ArrayRef, Float64Array}; use std::collections::HashMap; use std::sync::Arc; @@ -7337,7 +9188,7 @@ mod tests { metadata.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.99".to_string()); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_batch_size(1000) .with_range(0..1000) .with_indices(vec![0, 1, 10, 999]) @@ -7472,7 +9323,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_1, + FixedWidthDictionaryEncoding::Exclude64Bit, ); assert!( @@ -7519,7 +9370,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_2, + FixedWidthDictionaryEncoding::Include64Bit, ); assert!( @@ -7546,7 +9397,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_2, + FixedWidthDictionaryEncoding::Include64Bit, ); assert!( @@ -7564,7 +9415,7 @@ mod tests { let field = arrow_schema::Field::new("test", DataType::Utf8, false).with_metadata(metadata); let array = create_sorted_string_array(200_000, 8_000); - let page = encode_first_page(field, array, LanceFileVersion::V2_2).await; + let page = encode_first_page(field, array, TestEncoding::StructuralU32).await; let _ = dictionary_encoding_from_page(&page); } @@ -7584,7 +9435,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_1, + FixedWidthDictionaryEncoding::Exclude64Bit, ); assert!( @@ -7610,7 +9461,7 @@ mod tests { let result = PrimitiveStructuralEncoder::should_dictionary_encode( &block, &field, - LanceFileVersion::V2_1, + FixedWidthDictionaryEncoding::Exclude64Bit, ); assert!( @@ -7636,9 +9487,13 @@ mod tests { num_values: 32_769, }; - let serialized = - PrimitiveStructuralEncoder::serialize_miniblocks(miniblocks, None, None, false) - .unwrap(); + let serialized = PrimitiveStructuralEncoder::serialize_miniblocks( + miniblocks, + None, + None, + MiniblockChunkSize::U16, + ) + .unwrap(); let chunk_metadata = serialized.metadata.borrow_to_typed_slice::(); assert_eq!(chunk_metadata.len(), 2); @@ -7652,33 +9507,33 @@ mod tests { async fn encode_first_page( field: arrow_schema::Field, array: ArrayRef, - version: LanceFileVersion, + version: TestEncoding, ) -> crate::encoder::EncodedPage { - use crate::encoder::{ - ColumnIndexSequence, EncodingOptions, MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, - default_encoding_strategy, - }; use crate::repdef::RepDefBuilder; + use crate::{ + encoder::{ + ColumnIndexSequence, EncodingOptions, MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, + }, + testing::{create_test_field_encoder, test_encoding_strategy}, + }; let lance_field = lance_core::datatypes::Field::try_from(&field).unwrap(); - let encoding_strategy = default_encoding_strategy(version); + let encoding_strategy = test_encoding_strategy(version); let mut column_index_seq = ColumnIndexSequence::default(); let encoding_options = EncodingOptions { cache_bytes_per_column: 1, max_page_bytes: 32 * 1024 * 1024, keep_original_array: true, buffer_alignment: MIN_PAGE_BUFFER_ALIGNMENT, - version, }; - let mut encoder = encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap(); + let mut encoder = create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap(); let mut external_buffers = OutOfLineBuffers::new(0, MIN_PAGE_BUFFER_ALIGNMENT); let repdef = RepDefBuilder::default(); @@ -7709,7 +9564,7 @@ mod tests { .unwrap(), ); let field = arrow_schema::Field::new("c", DataType::FixedSizeBinary(33), true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7721,8 +9576,7 @@ mod tests { assert_eq!(page.data.len(), 1); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7735,7 +9589,7 @@ mod tests { std::iter::repeat_n("hello", 512), )); let field = arrow_schema::Field::new("c", DataType::Utf8, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7747,8 +9601,7 @@ mod tests { assert_eq!(page.data.len(), 1); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7765,7 +9618,7 @@ mod tests { Some(7), ])); let field = arrow_schema::Field::new("c", DataType::Int32, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7777,8 +9630,7 @@ mod tests { assert_eq!(page.data.len(), 2); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7811,7 +9663,7 @@ mod tests { ))), true, ); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7823,8 +9675,7 @@ mod tests { assert_eq!(page.data.len(), 2); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7850,7 +9701,7 @@ mod tests { ), true, ); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; if let PageEncoding::Structural(layout) = &page.description { assert!( @@ -7860,8 +9711,7 @@ mod tests { } let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7872,7 +9722,7 @@ mod tests { let arr: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![7; 1024])); let field = arrow_schema::Field::new("c", DataType::Int32, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_1).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU16).await; let PageEncoding::Structural(layout) = &page.description else { return; @@ -7883,8 +9733,7 @@ mod tests { ); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) - .with_max_file_version(LanceFileVersion::V2_1) + .with_encoding(TestEncoding::StructuralU16) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } @@ -7895,7 +9744,7 @@ mod tests { let arr: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![None, None, None])); let field = arrow_schema::Field::new("c", DataType::Int32, true); - let page = encode_first_page(field, arr.clone(), LanceFileVersion::V2_2).await; + let page = encode_first_page(field, arr.clone(), TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout) = &page.description else { panic!("Expected structural encoding"); @@ -7907,26 +9756,63 @@ mod tests { assert_eq!(page.data.len(), 0); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) - .with_max_file_version(LanceFileVersion::V2_2) + .with_encoding(TestEncoding::StructuralU32) .with_page_sizes(vec![4096]); check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; } + fn hand_built_dictionary_with_out_of_range_null_keys() -> ArrayRef { + use arrow_array::{DictionaryArray, Int32Array, types::Int32Type}; + use arrow_buffer::NullBuffer; + + let keys = Int32Array::new( + vec![0, 7, 7].into(), + Some(NullBuffer::from(vec![true, false, false])), + ); + let values = Arc::new(StringArray::from(vec!["a"])); + Arc::new(DictionaryArray::::try_new(keys, values).unwrap()) as ArrayRef + } + + fn concatenated_dictionary_with_out_of_range_null_keys() -> ArrayRef { + use arrow_array::{builder::StringDictionaryBuilder, new_null_array, types::Int32Type}; + + let mut builder = StringDictionaryBuilder::::new(); + builder.append_value("a"); + for _ in 0..7 { + builder.append_null(); + } + let valued = Arc::new(builder.finish()) as ArrayRef; + let all_null = new_null_array(valued.data_type(), 8); + arrow_select::concat::concat(&[valued.as_ref(), all_null.as_ref()]).unwrap() + } + + #[rstest::rstest] + #[case::hand_built(hand_built_dictionary_with_out_of_range_null_keys())] + #[case::concatenated(concatenated_dictionary_with_out_of_range_null_keys())] + #[tokio::test] + async fn test_dictionary_out_of_range_null_keys_round_trip(#[case] dictionary: ArrayRef) { + let test_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralU32) + .with_page_sizes(vec![4096]); + + check_round_trip_encoding_of_data(vec![dictionary], &test_cases, HashMap::new()).await; + } + #[test] fn test_encode_decode_complex_all_null_vals_roundtrip() { - use crate::compression::{ - DecompressionStrategy, DefaultCompressionStrategy, DefaultDecompressionStrategy, - }; + use crate::compression::{DecompressionStrategy, DefaultDecompressionStrategy}; let values: Arc<[u16]> = Arc::from((0..2048).map(|i| (i % 5) as u16).collect::>()); - let compression_strategy = DefaultCompressionStrategy::default(); + let compression_strategy = crate::testing::test_compression_strategy( + TestEncoding::StructuralU16, + crate::compression_config::CompressionParams::default(), + ); let decompression_strategy = DefaultDecompressionStrategy::default(); let (compressed_buf, encoding) = PrimitiveStructuralEncoder::encode_complex_all_null_vals( &values, - &compression_strategy, + compression_strategy.as_ref(), ) .unwrap(); @@ -7934,7 +9820,7 @@ mod tests { .create_block_decompressor(&encoding) .unwrap(); let decompressed = decompressor - .decompress(compressed_buf, values.len() as u64) + .decompress(Some(compressed_buf), values.len() as u64) .unwrap(); let decompressed_fixed_width = decompressed.as_fixed_width().unwrap(); assert_eq!(decompressed_fixed_width.num_values, values.len() as u64); @@ -7962,7 +9848,8 @@ mod tests { true, ); - let page_v21 = encode_first_page(field.clone(), arr.clone(), LanceFileVersion::V2_1).await; + let page_v21 = + encode_first_page(field.clone(), arr.clone(), TestEncoding::StructuralU16).await; let PageEncoding::Structural(layout_v21) = &page_v21.description else { panic!("Expected structural encoding"); }; @@ -7974,7 +9861,7 @@ mod tests { assert_eq!(layout_v21.num_rep_values, 0); assert_eq!(layout_v21.num_def_values, 0); - let page_v22 = encode_first_page(field, arr, LanceFileVersion::V2_2).await; + let page_v22 = encode_first_page(field, arr, TestEncoding::StructuralU32).await; let PageEncoding::Structural(layout_v22) = &page_v22.description else { panic!("Expected structural encoding"); }; @@ -7993,11 +9880,585 @@ mod tests { (0..1000).map(|i| if i % 2 == 0 { None } else { Some(vec![]) }), ); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_2); + let test_cases = TestCases::default().with_u32_structural_encodings(); + check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) + .await; + } + + #[tokio::test] + async fn test_complex_all_null_constant_def_round_trip() { + use arrow_array::ListArray; + + // Every row is a null list => constant def levels => a single RLE run, + // exercising the lazy run-form decode end to end. + let list_array = ListArray::from_iter_primitive::( + (0..5000).map(|_| None::>>), + ); + + let test_cases = TestCases::default().with_u32_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) .await; } + fn encoded_u16_frame(levels: &[u16], run_length_width: RunLengthWidth) -> LanceBuffer { + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(levels)), + bits_per_value: 16, + num_values: levels.len() as u64, + block_info: BlockInfo::new(), + }); + BlockCompressor::compress(&RleEncoder::with_run_length_width(run_length_width), block) + .unwrap() + .0 + .unwrap() + } + + fn encoded_u16_runs(levels: &[u16], run_length_width: RunLengthWidth) -> RleRuns { + let frame = encoded_u16_frame(levels, run_length_width); + RleDecompressor::with_run_length_width(16, run_length_width) + .decode_u16_runs(frame, levels.len() as u64) + .unwrap() + } + + #[test] + fn miniblock_levels_use_one_count_for_mixed_codecs() { + let actual_num_levels = usize::from(u16::MAX) + 8; + let levels = vec![1_u16; actual_num_levels]; + let rep_frame = encoded_u16_frame(&levels, RunLengthWidth::U8); + let rep_decompressor = RleDecompressor::new(16); + let def_frame = LanceBuffer::reinterpret_slice(Arc::from(levels.clone())); + let def_decompressor = ValueDecompressor::from_flat(&pb21::Flat { + bits_per_value: 16, + data: None, + }); + + let num_levels = DecodeMiniBlockTask::resolve_num_levels( + Some(&rep_decompressor), + Some(&rep_frame), + Some(&def_decompressor), + Some(&def_frame), + 7, + ) + .unwrap(); + assert_eq!(num_levels, actual_num_levels as u64); + + let rep = + DecodeMiniBlockTask::decode_levels(&rep_decompressor, rep_frame, num_levels).unwrap(); + let def = + DecodeMiniBlockTask::decode_levels(&def_decompressor, def_frame, num_levels).unwrap(); + assert_eq!(rep.as_ref(), levels); + assert_eq!(def, rep); + } + + #[test] + fn miniblock_levels_reject_non_wrapped_count_mismatch() { + let frame = encoded_u16_frame(&[1_u16; 8], RunLengthWidth::U8); + let decompressor = RleDecompressor::new(16); + + let error = DecodeMiniBlockTask::resolve_num_levels( + Some(&decompressor), + Some(&frame), + None, + None, + 7, + ) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!(error.to_string().contains("not congruent modulo 65536")); + } + + #[test] + fn miniblock_levels_reject_cross_stream_count_disagreement() { + let rep_levels = vec![1_u16; usize::from(u16::MAX) + 8]; + let def_levels = vec![1_u16; rep_levels.len() + usize::from(u16::MAX) + 1]; + let rep_frame = encoded_u16_frame(&rep_levels, RunLengthWidth::U8); + let def_frame = encoded_u16_frame(&def_levels, RunLengthWidth::U8); + let decompressor = RleDecompressor::new(16); + + let error = DecodeMiniBlockTask::resolve_num_levels( + Some(&decompressor), + Some(&rep_frame), + Some(&decompressor), + Some(&def_frame), + 7, + ) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("structural streams disagree on the level count") + ); + } + + fn physical_levels(levels: &[u16]) -> LazyLevels { + LazyLevels::Runs(Arc::new(RunStorage::Physical( + encoded_u16_runs(levels, RunLengthWidth::U8).into_owned(), + ))) + } + + fn coalesced_levels(levels: &[u16]) -> LazyLevels { + let mut values = Vec::new(); + let mut ends = RunEndsBuilder::with_capacity(levels.len(), levels.len()); + for (index, &value) in levels.iter().enumerate() { + if values.last() == Some(&value) { + ends.set_last(index + 1).unwrap(); + } else { + values.push(value); + ends.push(index + 1).unwrap(); + } + } + LazyLevels::Runs(Arc::new(RunStorage::Coalesced { + values: values.into_boxed_slice(), + ends: ends.finish(), + })) + } + + #[test] + fn lazy_levels_runs_match_dense() { + // Runs: 3x2, 1x1, 3x3, 0x2 => [3,3,1,3,3,3,0,0] + let expanded: Vec = vec![3, 3, 1, 3, 3, 3, 0, 0]; + let coalesced = coalesced_levels(&expanded); + let physical = physical_levels(&expanded); + let dense = LazyLevels::Dense(ScalarBuffer::::from(expanded.clone())); + let n = expanded.len(); + + assert_eq!(coalesced.len(), n); + assert_eq!(physical.len(), n); + assert_eq!(dense.len(), n); + + // Rows begin at each `max_rep` (3) position; row `num_rows` maps to `len`. + let max_rep = 3u16; + let row_starts: Vec = (0..n).filter(|&i| expanded[i] == max_rep).collect(); + for target in 0..=row_starts.len() as u64 { + let want = row_starts.get(target as usize).copied().unwrap_or(n); + for runs in [&coalesced, &physical] { + let mut cursor = LevelCursor::default(); + assert_eq!( + runs.seek_row_start(&mut cursor, target, max_rep).unwrap(), + want, + "seek_row_start({target})" + ); + } + let mut c_dense = LevelCursor::default(); + assert_eq!( + dense.seek_row_start(&mut c_dense, target, max_rep).unwrap(), + want + ); + } + + // `count_le_cursor` (fresh cursor per range) and `extend_into` agree with + // the dense reference on every sub-range. + for start in 0..=n { + for end in start..=n { + for max in [0u16, 1, 2, 3] { + let want = expanded[start..end].iter().filter(|&&d| d <= max).count() as u64; + for runs in [&coalesced, &physical] { + let mut cursor = RunPosition::default(); + assert_eq!( + runs.count_le_cursor(&mut cursor, start..end, max).0, + want, + "count_le_cursor({start}..{end}, {max})" + ); + } + let mut d_cur = RunPosition::default(); + assert_eq!(dense.count_le_cursor(&mut d_cur, start..end, max).0, want); + } + for runs in [&coalesced, &physical] { + let mut got = Vec::new(); + runs.extend_into(start..end, RunPosition::default(), &mut got); + assert_eq!( + got, + expanded[start..end].to_vec(), + "extend_into({start}..{end})" + ); + } + let mut got_dense = Vec::new(); + dense.extend_into(start..end, RunPosition::default(), &mut got_dense); + assert_eq!(got_dense, expanded[start..end].to_vec()); + } + } + } + + #[test] + fn physical_run_hints_support_deferred_materialization() { + let expanded: Vec = vec![3, 3, 1, 1, 2, 2, 0, 0]; + let physical = physical_levels(&expanded); + let LazyLevels::Runs(runs) = &physical else { + panic!("expected physical runs"); + }; + let mut first_hint = RunPosition::default(); + runs.seek(&mut first_hint, 2); + let mut second_hint = RunPosition::default(); + runs.seek(&mut second_hint, 6); + + let mut second = Vec::new(); + physical.extend_into(6..8, second_hint, &mut second); + let mut first = Vec::new(); + physical.extend_into(2..4, first_hint, &mut first); + assert_eq!(second, expanded[6..8]); + assert_eq!(first, expanded[2..4]); + } + + /// Fuzz parity for the run-oriented complex-all-null drain: the cursor walk + /// over `LazyLevels` must yield the exact level slices and visible + /// count that a brute-force reference over the fully expanded levels does, for + /// dense, physical-run, and coalesced-run forms and arbitrarily shaped range requests. + mod complex_all_null_drain_parity { + use std::ops::Range; + + use arrow_buffer::ScalarBuffer; + use proptest::prelude::*; + + use super::super::{LazyLevels, LevelCursor, RunPosition}; + use super::{coalesced_levels, physical_levels}; + use crate::Result; + + #[derive(Debug, Clone)] + struct DrainInput { + max_rep: u16, + max_visible: u16, + rep: Option>, + def: Option>, + ranges: Vec>, + } + + fn dense_levels(levels: &[u16]) -> LazyLevels { + LazyLevels::Dense(ScalarBuffer::from(levels.to_vec())) + } + + fn rle_levels(levels: &[u16]) -> LazyLevels { + coalesced_levels(levels) + } + + fn seek( + rep: Option<&LazyLevels>, + cursor: &mut LevelCursor, + row: u64, + max_rep: u16, + ) -> Result { + match rep { + Some(rep) => rep.seek_row_start(cursor, row, max_rep), + None => { + cursor.row = row; + cursor.level = row as usize; + Ok(row as usize) + } + } + } + + /// Mirror of `ComplexAllNullPageDecoder::drain`, driving the real + /// `seek_row_start` / `count_le_cursor` with monotonic cursors. + fn simulate_drain( + rep: Option<&LazyLevels>, + def: Option<&LazyLevels>, + max_rep: u16, + max_visible: u16, + ranges: &[Range], + ) -> Result<(Vec>, u64)> { + let mut rep_cursor = LevelCursor::default(); + let mut def_run_cursor = RunPosition::default(); + let mut slices: Vec> = Vec::new(); + let mut visible = 0u64; + for range in ranges { + let level_start = seek(rep, &mut rep_cursor, range.start, max_rep)?; + let level_end = seek(rep, &mut rep_cursor, range.end, max_rep)?; + visible += match def { + Some(def) => { + def.count_le_cursor( + &mut def_run_cursor, + level_start..level_end, + max_visible, + ) + .0 + } + None => (level_end - level_start) as u64, + }; + match slices.last_mut() { + Some(last) if last.end == level_start => last.end = level_end, + _ => slices.push(level_start..level_end), + } + } + Ok((slices, visible)) + } + + /// Independent brute-force reference over fully expanded levels. + fn reference_drain( + rep: Option<&[u16]>, + def: Option<&[u16]>, + max_rep: u16, + max_visible: u16, + ranges: &[Range], + ) -> (Vec>, u64) { + let total_levels = rep + .map(|r| r.len()) + .or_else(|| def.map(|d| d.len())) + .unwrap_or(0); + // Level index where each row starts (or `total_levels` for the end row). + let row_starts: Vec = match rep { + Some(rep) => (0..rep.len()).filter(|&i| rep[i] == max_rep).collect(), + None => (0..total_levels).collect(), + }; + let level_of_row = |row: u64| { + row_starts + .get(row as usize) + .copied() + .unwrap_or(total_levels) + }; + + let mut slices: Vec> = Vec::new(); + let mut visible = 0u64; + for range in ranges { + let ls = level_of_row(range.start); + let le = level_of_row(range.end); + visible += match def { + Some(def) => def[ls..le].iter().filter(|&&d| d <= max_visible).count() as u64, + None => (le - ls) as u64, + }; + match slices.last_mut() { + Some(last) if last.end == ls => last.end = le, + _ => slices.push(ls..le), + } + } + (slices, visible) + } + + fn ranges_strategy(num_rows: u64) -> BoxedStrategy>> { + if num_rows == 0 { + return Just(Vec::new()).boxed(); + } + // (gap, len) pairs; a zero gap yields ranges adjacent in row space, + // which exercises the level-slice coalescing path. + proptest::collection::vec((0u64..=3, 1u64..=4), 0..=8) + .prop_map(move |pairs| { + let mut ranges = Vec::new(); + let mut pos = 0u64; + for (gap, len) in pairs { + pos = pos.saturating_add(gap); + if pos >= num_rows { + break; + } + let end = (pos + len).min(num_rows); + ranges.push(pos..end); + pos = end; + } + ranges + }) + .boxed() + } + + fn drain_input() -> impl Strategy { + ( + 1u16..=3, + 0u16..=3, + any::(), + any::(), + 1usize..=48, + ) + .prop_flat_map(|(max_rep, max_visible, has_rep, has_def, len)| { + // Complex-all-null always has definition levels when there is + // no repetition, so force `def` present in that case. + let has_def = has_def || !has_rep; + let rep = if has_rep { + proptest::collection::vec(0u16..=max_rep, len) + .prop_map(move |mut v| { + // Row 0 must start at a max-rep boundary. + v[0] = max_rep; + Some(v) + }) + .boxed() + } else { + Just(None).boxed() + }; + let def = if has_def { + proptest::collection::vec(0u16..=(max_visible + 2), len) + .prop_map(Some) + .boxed() + } else { + Just(None).boxed() + }; + (Just(max_rep), Just(max_visible), rep, def) + }) + .prop_flat_map(|(max_rep, max_visible, rep, def)| { + let num_rows = match &rep { + Some(rep) => rep.iter().filter(|&&v| v == max_rep).count() as u64, + None => def.as_ref().map(|d| d.len() as u64).unwrap_or(0), + }; + ranges_strategy(num_rows).prop_map(move |ranges| DrainInput { + max_rep, + max_visible, + rep: rep.clone(), + def: def.clone(), + ranges, + }) + }) + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn drain_matches_reference(input in drain_input()) { + let DrainInput { max_rep, max_visible, rep, def, ranges } = input; + + let reference = + reference_drain(rep.as_deref(), def.as_deref(), max_rep, max_visible, &ranges); + + let rep_dense = rep.as_deref().map(dense_levels); + let def_dense = def.as_deref().map(dense_levels); + let got_dense = + simulate_drain(rep_dense.as_ref(), def_dense.as_ref(), max_rep, max_visible, &ranges) + .unwrap(); + prop_assert_eq!(&got_dense, &reference, "dense form diverged from reference"); + + let rep_rle = rep.as_deref().map(rle_levels); + let def_rle = def.as_deref().map(rle_levels); + let got_rle = + simulate_drain(rep_rle.as_ref(), def_rle.as_ref(), max_rep, max_visible, &ranges) + .unwrap(); + prop_assert_eq!(&got_rle, &reference, "rle form diverged from reference"); + + let rep_physical = rep.as_deref().map(physical_levels); + let def_physical = def.as_deref().map(physical_levels); + let got_physical = + simulate_drain(rep_physical.as_ref(), def_physical.as_ref(), max_rep, max_visible, &ranges) + .unwrap(); + prop_assert_eq!(&got_physical, &reference, "physical form diverged from reference"); + } + } + } + + #[test] + fn lazy_levels_runs_are_compact() { + let single_run = |n: usize| { + let mut ends = RunEndsBuilder::with_capacity(n, 1); + ends.push(n).unwrap(); + LazyLevels::Runs(Arc::new(RunStorage::Coalesced { + values: vec![1u16].into_boxed_slice(), + ends: ends.finish(), + })) + }; + // Run-form footprint is independent of the logical length within an end width... + assert_eq!(single_run(100).deep_size(), single_run(10_000).deep_size()); + assert!(single_run(10_000_000).deep_size() < 100); + assert_eq!(single_run(10_000_000).len(), 10_000_000); + // ...while Dense pays 2 bytes per value. + assert_eq!( + LazyLevels::Dense(ScalarBuffer::::from(vec![1u16; 1000])).deep_size(), + 2000 + ); + } + + #[test] + fn lazy_levels_selects_smallest_representation() { + let runs = encoded_u16_runs(&[7u16; 10], RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Dense); + + let equal_size: Vec = std::iter::repeat_n(0, 256) + .chain(std::iter::repeat_n(1, 100)) + .chain(std::iter::repeat_n(2, 100)) + .collect(); + let runs = encoded_u16_runs(&equal_size, RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Coalesced); + + let moderate_runs: Vec = (0..250) + .flat_map(|run| std::iter::repeat_n((run % 2) as u16, 4)) + .collect(); + let runs = encoded_u16_runs(&moderate_runs, RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Physical); + + let split_constant = vec![7u16; 5000]; + let runs = encoded_u16_runs(&split_constant, RunLengthWidth::U8); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Coalesced); + + let high_density: Vec = (0..70_000).map(|index| (index % 2) as u16).collect(); + let runs = encoded_u16_runs(&high_density, RunLengthWidth::U32); + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Dense); + } + + #[test] + fn physical_runs_detach_from_large_encoded_frame() { + let levels: Vec = (0..250) + .flat_map(|run| std::iter::repeat_n((run % 2) as u16, 4)) + .collect(); + let frame = encoded_u16_frame(&levels, RunLengthWidth::U8); + let frame_offset = 4096; + let mut allocation = vec![0; frame_offset + frame.len() + 1_000_000]; + allocation[frame_offset..frame_offset + frame.len()].copy_from_slice(frame.as_ref()); + let frame = LanceBuffer::from(allocation).slice_with_length(frame_offset, frame.len()); + let runs = RleDecompressor::with_run_length_width(16, RunLengthWidth::U8) + .decode_u16_runs(frame, levels.len() as u64) + .unwrap(); + + assert_eq!(LazyLevels::select_plan(&runs), LevelPlan::Physical); + let cached = LazyLevels::from_rle_runs(runs).unwrap(); + assert!( + matches!(cached, LazyLevels::Runs(ref runs) if matches!(runs.as_ref(), RunStorage::Physical(_))) + ); + assert_eq!(cached.len(), levels.len()); + assert!(cached.deep_size() < 4096); + } + + #[test] + fn complex_all_null_levels_reject_invalid_values_and_lengths() { + let invalid_levels = vec![0u16, 3]; + for levels in [ + LazyLevels::Dense(ScalarBuffer::from(invalid_levels.clone())), + physical_levels(&invalid_levels), + coalesced_levels(&invalid_levels), + ] { + let error = validate_complex_all_null_levels(&None, &Some(levels), 0, 2).unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!(error.to_string().contains("Invalid definition level 3")); + } + + let rep = Some(LazyLevels::Dense(ScalarBuffer::from(vec![0u16; 2]))); + let def = Some(LazyLevels::Dense(ScalarBuffer::from(vec![0u16]))); + let error = validate_complex_all_null_levels(&rep, &def, 0, 0).unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("repetition has 2, definition has 1") + ); + } + + #[test] + fn block_levels_reject_malformed_payload_size() { + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::from(vec![0]), + bits_per_value: 16, + num_values: 1, + block_info: BlockInfo::new(), + }); + let error = dense_levels_from_block(block, 1, "definition").unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("expected 2 bytes for 1 values, got 1") + ); + } + + #[test] + fn complex_all_null_level_codec_validates_rle_metadata() { + let encoding = pb21::CompressiveEncoding { + compression: Some(Compression::Rle(Box::new(pb21::Rle { + values: None, + run_lengths: Some(Box::new(ProtobufUtils21::flat(8, None))), + }))), + }; + + let error = LevelCodec::try_new(Some(&encoding), &DefaultDecompressionStrategy::default()) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("RLE compression missing values encoding") + ); + } + // https://github.com/lance-format/lance/issues/6681 #[tokio::test] async fn test_sparse_boolean_list_roundtrip() { @@ -8015,7 +10476,77 @@ mod tests { } let list_array = Arc::new(list_builder.finish()); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![list_array], &test_cases, HashMap::new()).await; } + + fn truncated_tail_details() -> std::sync::Arc { + use crate::compression::VariablePerValueDecompressor; + use crate::encodings::physical::binary::VariableDecoder; + use crate::repdef::{ControlWordParser, DefinitionInterpretation}; + use std::sync::Arc; + Arc::new(super::FullZipDecodeDetails { + value_decompressor: super::PerValueDecompressor::Variable(Arc::new( + VariableDecoder::default(), + ) + as Arc), + def_meaning: vec![DefinitionInterpretation::NullableItem].into(), + ctrl_word_parser: ControlWordParser::new(0, 0), + max_rep: 0, + max_visible_def: 0, + }) + } + + fn decode_variable_full_zip( + buf: Vec, + bits_per_offset: u8, + ) -> lance_core::Result { + use std::collections::VecDeque; + let mut data = VecDeque::new(); + data.push_back(crate::buffer::LanceBuffer::from(buf)); + super::VariableFullZipDecoder::new( + truncated_tail_details(), + data, + 1, + bits_per_offset, + bits_per_offset, + ) + } + + /// A well-formed length prefix decodes without incident, for both widths. + #[test] + fn variable_full_zip_wellformed_length_prefix() { + assert!(decode_variable_full_zip(0u32.to_le_bytes().to_vec(), 32).is_ok()); + assert!(decode_variable_full_zip(0u64.to_le_bytes().to_vec(), 64).is_ok()); + } + + /// A page whose item walk ends with a partial length prefix must surface a + /// corrupt-file error rather than read past the end of the buffer. + /// + /// This asserts the error variant and message rather than merely expecting a + /// panic: before the length prefix was bounds checked, the read was + /// `get_unchecked` behind a `debug_assert!`, so a debug build panicked here + /// (which a `#[should_panic]` test would have accepted as a pass) while a + /// release build read up to 8 bytes out of a 4 byte allocation. + #[test] + fn variable_full_zip_truncated_length_prefix_is_corrupt_file() { + use lance_core::Error; + + for (bits, buf_len) in [(32u8, 3usize), (64u8, 4usize)] { + let err = decode_variable_full_zip(vec![0xAA; buf_len], bits) + .expect_err("a truncated length prefix must not decode"); + assert!( + matches!(err, Error::CorruptFile { .. }), + "expected CorruptFile for a {}-bit prefix with {} byte(s), got: {:?}", + bits, + buf_len, + err + ); + let msg = err.to_string(); + assert!( + msg.contains("truncated length prefix"), + "error should say what is wrong, got: {msg}" + ); + } + } } diff --git a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs index eed3e584b7e..52a7039b2b6 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive/blob.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive/blob.rs @@ -258,7 +258,9 @@ impl BlobPageScheduler { let bytes = read_fut.await?; let mut bytes_iter = bytes.into_iter(); for blob in loaded_blobs.iter_mut() { - if blob.def == 0 { + // Empty values have def == 0 too but scheduled no read; their + // bytes were set at scheduling time. + if blob.def == 0 && blob.bytes.is_none() { blob.set_bytes(bytes_iter.next().expect_ok()?); } } @@ -364,7 +366,17 @@ impl StructuralPageScheduler for BlobPageScheduler { if size == 0 { let rep = (position & 0xFFFF) as u16; let def = ((position >> 16) & 0xFFFF) as u16; - loaded_blobs.push(LoadedBlob::new(rep, def)); + let mut blob = LoadedBlob::new(rep, def); + if def == 0 { + // A size-0 descriptor with definition level 0 is a + // valid, empty value (nulls carry their non-zero + // packed rep/def levels in `position`). No read is + // scheduled for it, so it gets its zero-length bytes + // here rather than consuming another blob's read + // result in the load task. + blob.set_bytes(Bytes::new()); + } + loaded_blobs.push(blob); } else { loaded_blobs.push(LoadedBlob::new(0, 0)); ranges_to_read.push(position..(position + size)); diff --git a/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs b/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs new file mode 100644 index 00000000000..19fb0fdb041 --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/chunk_index.rs @@ -0,0 +1,536 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Compact per-page chunk index for the mini-block structural encoding. +//! +//! The chunk index is stored on disk in an extremely compressed form that +//! requires a lot of CPU to work with. However, extracting it out to its full +//! width can be RAM-intensive. As a compromise we extract into a prefix-sum +//! array that we fit into `u32` if possible and we avoid storing per-block row +//! counts when those are redundant. The scheduler looks chunks up by index +//! (byte range, leaf value count) and by row (which chunk holds a row). +//! +//! ```text +//! MiniBlockChunkIndex +//! |- base: u64 absolute file position of the value buffer +//! |- byte_starts: PrefixSums cumulative chunk byte sizes (all pages) +//! `- rows: RowMapping +//! |- UniformFlat { .. } flat page, uniform leaf chunking (arithmetic) +//! |- Flat { value_starts } flat page, non-uniform leaf chunking +//! `- Nested { row_starts, .. } repetition present; rows tracked as prefix sums +//! ``` + +use std::ops::Range; + +use arrow_buffer::{BooleanBuffer, BooleanBufferBuilder}; +use lance_core::cache::{Context, DeepSizeOf}; + +/// Cumulative (prefix-sum) array of length `num_chunks + 1` (entry `0` is `0`, +/// the last entry is the grand total). Stored as `u32` when the total fits, +/// else `u64`. +#[derive(Debug, DeepSizeOf)] +pub enum PrefixSums { + U32(Vec), + U64(Vec), +} + +impl PrefixSums { + /// Builds the cumulative array from per-chunk `deltas`. `total` selects the + /// storage width (callers must pass the true sum); `num_chunks` only pre-sizes. + pub fn from_deltas(deltas: impl Iterator, num_chunks: usize, total: u64) -> Self { + if total <= u32::MAX as u64 { + let mut values = Vec::with_capacity(num_chunks + 1); + let mut acc = 0u32; + values.push(0); + for delta in deltas { + acc += delta as u32; + values.push(acc); + } + debug_assert_eq!(values.len(), num_chunks + 1); + debug_assert_eq!(acc as u64, total); + Self::U32(values) + } else { + let mut values = Vec::with_capacity(num_chunks + 1); + let mut acc = 0u64; + values.push(0); + for delta in deltas { + acc += delta; + values.push(acc); + } + debug_assert_eq!(values.len(), num_chunks + 1); + debug_assert_eq!(acc, total); + Self::U64(values) + } + } + + /// Builds a `PrefixSums` from an already-cumulative array (`[0, .., total]`), + /// narrowing to `u32` when the total fits. Avoids the deltas buffer + /// [`Self::from_deltas`] would need. + fn from_prefix(prefix: Vec) -> Self { + debug_assert!(!prefix.is_empty()); + debug_assert_eq!(prefix[0], 0); + let total = prefix.last().copied().unwrap_or(0); + if total <= u32::MAX as u64 { + Self::U32(prefix.into_iter().map(|v| v as u32).collect()) + } else { + Self::U64(prefix) + } + } + + /// Cumulative value at position `i` (i.e. the start of chunk `i`). + pub fn get(&self, i: usize) -> u64 { + match self { + Self::U32(values) => values[i] as u64, + Self::U64(values) => values[i], + } + } + + /// Start and end of chunk `i` (positions `i`, `i + 1`) behind one width + /// match -- halves the branching of two `get` calls on the hot per-chunk path. + pub fn get_pair(&self, i: usize) -> (u64, u64) { + match self { + Self::U32(values) => (values[i] as u64, values[i + 1] as u64), + Self::U64(values) => (values[i], values[i + 1]), + } + } + + /// Number of chunks (array length minus the trailing total). + pub fn num_chunks(&self) -> usize { + match self { + Self::U32(values) => values.len() - 1, + Self::U64(values) => values.len() - 1, + } + } + + /// Size of chunk `i` (the delta between consecutive cumulative values). + pub fn delta(&self, i: usize) -> u64 { + let (start, end) = self.get_pair(i); + end - start + } + + /// Index of the chunk whose half-open span `[get(i), get(i+1))` contains + /// `value`. On an exact hit against a chunk start, returns the *first* chunk + /// with that start (chunks can share a start row). + pub fn find(&self, value: u64) -> usize { + // Match the width once, then binary-search only the starts (not the + // trailing total). `partition_point` already yields the first of any + // duplicated starts; the `idx - 1` fallback is safe since `get(0) == 0`. + match self { + Self::U32(values) => { + let starts = &values[..values.len() - 1]; + let idx = starts.partition_point(|&start| (start as u64) < value); + if idx < starts.len() && starts[idx] as u64 == value { + idx + } else { + idx - 1 + } + } + Self::U64(values) => { + let starts = &values[..values.len() - 1]; + let idx = starts.partition_point(|&start| start < value); + if idx < starts.len() && starts[idx] == value { + idx + } else { + idx - 1 + } + } + } + } +} + +/// Leaf value counts per chunk, needed to decode. Tracked only for nested +/// pages; flat pages read items off the row mapping (rows == items). +#[derive(Debug, DeepSizeOf)] +pub enum ItemCounts { + /// Every non-last chunk holds the same number of values. + Uniform { + values_per_chunk: u64, + last_chunk_values: u64, + }, + /// `log2` of each chunk's value count, stored as one byte per chunk rather + /// than the full count because this index stays cached in RAM; the last + /// chunk is handled via `last_chunk_values`. + PerChunkLog { + logs: Vec, + last_chunk_values: u64, + }, +} + +impl ItemCounts { + fn get(&self, i: usize, num_chunks: usize) -> u64 { + match self { + Self::Uniform { + values_per_chunk, + last_chunk_values, + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + *values_per_chunk + } + } + Self::PerChunkLog { + logs, + last_chunk_values, + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + 1u64 << logs[i] + } + } + } + } +} + +/// How row ranges map onto chunks. +/// +/// Flat pages have row == value index and no preamble/trailer. Nested pages +/// track rows separately from leaf items and store a trailer bit per chunk; a +/// chunk's preamble is the previous chunk's trailer. +#[derive(Debug)] +pub enum RowMapping { + /// Flat page whose non-last chunks all hold `values_per_chunk` values, so + /// row->chunk is pure arithmetic. + UniformFlat { + values_per_chunk: u64, + last_chunk_values: u64, + num_chunks: usize, + }, + /// Flat page with non-uniform chunk sizes; `value_starts` are the cumulative + /// value counts (final entry == number of items in the page). + Flat { value_starts: PrefixSums }, + /// Nested page. `row_starts` are cumulative row counts (final entry == + /// number of rows in the page); `has_trailer[i]` is set when chunk `i` ends + /// with a partial list. + Nested { + row_starts: PrefixSums, + has_trailer: BooleanBuffer, + item_counts: ItemCounts, + }, +} + +impl DeepSizeOf for RowMapping { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + match self { + Self::UniformFlat { .. } => 0, + Self::Flat { value_starts } => value_starts.deep_size_of_children(context), + Self::Nested { + row_starts, + has_trailer, + item_counts, + } => { + row_starts.deep_size_of_children(context) + + has_trailer.len().div_ceil(8) + + item_counts.deep_size_of_children(context) + } + } + } +} + +/// Compact per-page chunk index that avoids fully materializing the repetition +/// index into u64's to save RAM. See the module docs for the layout. +#[derive(Debug)] +pub struct MiniBlockChunkIndex { + base: u64, + byte_starts: PrefixSums, + rows: RowMapping, +} + +impl MiniBlockChunkIndex { + pub fn new(base: u64, byte_starts: PrefixSums, rows: RowMapping) -> Self { + Self { + base, + byte_starts, + rows, + } + } + + /// Number of chunks in the page. + pub fn num_chunks(&self) -> usize { + self.byte_starts.num_chunks() + } + + /// Absolute byte range of chunk `i` within the file. + pub fn byte_range(&self, i: usize) -> Range { + let (start, end) = self.byte_starts.get_pair(i); + (self.base + start)..(self.base + end) + } + + /// Number of leaf values in chunk `i` (passed to the value decompressor). + pub fn items_in_chunk(&self, i: usize) -> u64 { + let num_chunks = self.num_chunks(); + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, + last_chunk_values, + .. + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + *values_per_chunk + } + } + RowMapping::Flat { value_starts } => value_starts.delta(i), + RowMapping::Nested { item_counts, .. } => item_counts.get(i, num_chunks), + } + } + + /// Index of the chunk that contains `row`. + pub fn find_chunk(&self, row: u64) -> usize { + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, + num_chunks, + .. + } => ((row / values_per_chunk) as usize).min(num_chunks - 1), + RowMapping::Flat { value_starts } => value_starts.find(row), + RowMapping::Nested { row_starts, .. } => row_starts.find(row), + } + } + + /// First row (relative to the page) that begins in chunk `i`. + pub fn first_row(&self, i: usize) -> u64 { + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, .. + } => i as u64 * values_per_chunk, + RowMapping::Flat { value_starts } => value_starts.get(i), + RowMapping::Nested { row_starts, .. } => row_starts.get(i), + } + } + + /// Number of rows that start in chunk `i`, including a trailer but not a + /// preamble (the previous `starts_including_trailer`). + pub fn rows_in_chunk(&self, i: usize) -> u64 { + let num_chunks = self.num_chunks(); + match &self.rows { + RowMapping::UniformFlat { + values_per_chunk, + last_chunk_values, + .. + } => { + if i == num_chunks - 1 { + *last_chunk_values + } else { + *values_per_chunk + } + } + RowMapping::Flat { value_starts } => value_starts.delta(i), + RowMapping::Nested { row_starts, .. } => row_starts.delta(i), + } + } + + /// Whether chunk `i` begins with a preamble (a continuation of the previous + /// chunk's list). Always false for flat pages; for nested pages this is the + /// previous chunk's trailer. + pub fn has_preamble(&self, i: usize) -> bool { + match &self.rows { + RowMapping::Nested { has_trailer, .. } => i > 0 && has_trailer.value(i - 1), + _ => false, + } + } + + /// Whether chunk `i` ends with a trailer (a partial list continued in the + /// next chunk). Always false for flat pages. + pub fn has_trailer(&self, i: usize) -> bool { + match &self.rows { + RowMapping::Nested { has_trailer, .. } => has_trailer.value(i), + _ => false, + } + } + + /// Name of the active row-mapping variant, used to assert detection in tests. + #[cfg(test)] + pub fn row_mapping_debug(&self) -> &'static str { + match &self.rows { + RowMapping::UniformFlat { .. } => "uniform_flat", + RowMapping::Flat { .. } => "flat", + RowMapping::Nested { .. } => "nested", + } + } + + /// Builds a nested index from raw repetition-index bytes, using placeholder + /// byte offsets and item counts. Only the row axis is populated, which is + /// all the scheduler exercises. + #[cfg(test)] + pub fn new_nested_for_test(rep_bytes: &[u8], stride: usize) -> Self { + let (row_starts, has_trailer) = parse_nested_rep(rep_bytes, stride); + let num_chunks = row_starts.num_chunks(); + let byte_starts = PrefixSums::from_deltas( + std::iter::repeat_n(8u64, num_chunks), + num_chunks, + 8 * num_chunks as u64, + ); + Self { + base: 0, + byte_starts, + rows: RowMapping::Nested { + row_starts, + has_trailer, + item_counts: ItemCounts::Uniform { + values_per_chunk: 1, + last_chunk_values: 1, + }, + }, + } + } +} + +impl DeepSizeOf for MiniBlockChunkIndex { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.byte_starts.deep_size_of_children(context) + self.rows.deep_size_of_children(context) + } +} + +/// Parses a mini-block repetition index into the compact nested row mapping. +/// +/// Bytes are `u64`s in groups of `stride`; the first two are `ends` (lists +/// finishing in the chunk) and `partial` (leftover items). Only cumulative row +/// starts and a trailer bit are kept: `has_preamble[i] = has_trailer[i-1]` and +/// `starts_including_trailer = ends + has_trailer - has_preamble`. +pub fn parse_nested_rep(rep_bytes: &[u8], stride: usize) -> (PrefixSums, BooleanBuffer) { + // Read the two `u64`s per group straight from the little-endian bytes rather + // than copying the buffer to reinterpret it. The caller guarantees + // `rep_bytes.len() % 8 == 0`, so the 8-byte windows stay in bounds. + const WORD: usize = std::mem::size_of::(); + let read_word = |word_idx: usize| -> u64 { + let byte = word_idx * WORD; + u64::from_le_bytes(rep_bytes[byte..byte + WORD].try_into().unwrap()) + }; + let num_chunks = (rep_bytes.len() / WORD) / stride; + + let mut has_trailer_builder = BooleanBufferBuilder::new(num_chunks); + // Accumulate the cumulative row starts in a single pass (entry 0 is 0, the + // trailing entry is the total) so there is no separate deltas buffer. + let mut row_starts = Vec::with_capacity(num_chunks + 1); + row_starts.push(0u64); + let mut acc = 0u64; + let mut chunk_has_preamble = false; + + for i in 0..num_chunks { + let base_idx = i * stride; + let ends = read_word(base_idx); + let partial = read_word(base_idx + 1); + + let has_trailer = partial > 0; + let starts_including_trailer = ends + (has_trailer as u64) - (chunk_has_preamble as u64); + + has_trailer_builder.append(has_trailer); + acc += starts_including_trailer; + row_starts.push(acc); + + chunk_has_preamble = has_trailer; + } + + ( + PrefixSums::from_prefix(row_starts), + has_trailer_builder.finish(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::buffer::LanceBuffer; + + /// Reference decode: the previous per-block repetition index, used as an + /// oracle for the compact `parse_nested_rep`. + struct RefBlock { + first_row: u64, + starts_including_trailer: u64, + has_preamble: bool, + has_trailer: bool, + } + + fn reference_decode(rep_bytes: &[u8], stride: usize) -> Vec { + let buffer = LanceBuffer::from(rep_bytes.to_vec()); + let u64_slice = buffer.borrow_to_typed_slice::(); + let n = u64_slice.len() / stride; + let mut blocks = Vec::with_capacity(n); + let mut chunk_has_preamble = false; + let mut offset = 0u64; + for i in 0..n { + let base_idx = i * stride; + let ends = u64_slice[base_idx]; + let partial = u64_slice[base_idx + 1]; + let has_trailer = partial > 0; + let starts_including_trailer = + ends + (has_trailer as u64) - (chunk_has_preamble as u64); + blocks.push(RefBlock { + first_row: offset, + starts_including_trailer, + has_preamble: chunk_has_preamble, + has_trailer, + }); + chunk_has_preamble = has_trailer; + offset += starts_including_trailer; + } + blocks + } + + fn rep_bytes(values: &[u64]) -> Vec { + values.iter().flat_map(|v| v.to_le_bytes()).collect() + } + + #[test] + fn test_prefix_sums_u32() { + let sums = PrefixSums::from_deltas([2u64, 3, 5].into_iter(), 3, 10); + assert!(matches!(sums, PrefixSums::U32(_))); + assert_eq!(sums.num_chunks(), 3); + assert_eq!(sums.get(0), 0); + assert_eq!(sums.get(1), 2); + assert_eq!(sums.get(3), 10); + assert_eq!(sums.delta(1), 3); + } + + #[test] + fn test_prefix_sums_u64_selected_by_total() { + let big = u32::MAX as u64 + 1; + let sums = PrefixSums::from_deltas([big].into_iter(), 1, big); + assert!(matches!(sums, PrefixSums::U64(_))); + assert_eq!(sums.get(1), big); + } + + #[test] + fn test_prefix_sums_find() { + // Chunk starts: 0, 5, 5, 12 (a zero-width chunk creates a duplicate start) + let sums = PrefixSums::from_deltas([5u64, 0, 7].into_iter(), 3, 12); + // Inside the first chunk + assert_eq!(sums.find(3), 0); + // Exact match on a duplicated start returns the first such chunk + assert_eq!(sums.find(5), 1); + // Inside the last chunk + assert_eq!(sums.find(11), 2); + // Start of the first chunk + assert_eq!(sums.find(0), 0); + } + + #[test] + fn test_parse_nested_rep_matches_reference() { + let cases: Vec> = vec![ + vec![5, 2, 3, 0, 4, 7, 2, 0], + vec![5, 2, 3, 3, 20, 0], + vec![0, 5, 0, 3, 10, 0], + vec![1, 0], + ]; + for values in cases { + let bytes = rep_bytes(&values); + let reference = reference_decode(&bytes, 2); + let (row_starts, has_trailer) = parse_nested_rep(&bytes, 2); + assert_eq!(row_starts.num_chunks(), reference.len()); + for (i, block) in reference.iter().enumerate() { + assert_eq!(row_starts.get(i), block.first_row, "first_row[{i}]"); + assert_eq!( + row_starts.delta(i), + block.starts_including_trailer, + "starts_including_trailer[{i}]" + ); + assert_eq!(has_trailer.value(i), block.has_trailer, "has_trailer[{i}]"); + let derived_preamble = i > 0 && has_trailer.value(i - 1); + assert_eq!(derived_preamble, block.has_preamble, "has_preamble[{i}]"); + } + } + } +} diff --git a/rust/lance-encoding/src/encodings/logical/primitive/dict.rs b/rust/lance-encoding/src/encodings/logical/primitive/dict.rs index 30d79ec7255..19582c167fb 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive/dict.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive/dict.rs @@ -110,6 +110,65 @@ pub fn normalize_dict_nulls(array: Arc) -> Result> { } } +fn clear_out_of_range_null_keys_impl( + array: Arc, +) -> Result> { + let dict_array = array.as_dictionary_opt::().expect_ok()?; + let num_values = dict_array.values().len(); + let Some(nulls) = dict_array.keys().nulls() else { + return Ok(array); + }; + + // There is no valid replacement key for an empty dictionary, so that case + // requires separate handling and must remain unchanged here. + if num_values == 0 { + return Ok(array); + } + + let has_out_of_range_null_key = dict_array + .keys() + .values() + .iter() + .zip(nulls.iter()) + .any(|(key, is_valid)| !is_valid && key.to_usize().is_none_or(|key| key >= num_values)); + if !has_out_of_range_null_key { + return Ok(array); + } + + // Building from the logical iterator writes the default physical key into + // every null slot while preserving the original validity bitmap. + let keys = PrimitiveArray::::from_iter(dict_array.keys().iter()); + let values = dict_array.values().clone(); + Ok(Arc::new(DictionaryArray::::try_new(keys, values)?) as Arc) +} + +/// Replaces out-of-range physical keys in null dictionary slots with a valid key. +/// +/// Arrow permits arbitrary keys in null slots, but the structural encoder removes +/// key validity after recording it as rep-def. The replacement keeps the array +/// valid when that null buffer is removed without changing its logical values. +pub(super) fn clear_out_of_range_null_keys(array: Arc) -> Result> { + match array.data_type() { + DataType::Dictionary(key_type, _) => match key_type.as_ref() { + DataType::UInt8 => clear_out_of_range_null_keys_impl::(array), + DataType::UInt16 => clear_out_of_range_null_keys_impl::(array), + DataType::UInt32 => clear_out_of_range_null_keys_impl::(array), + DataType::UInt64 => clear_out_of_range_null_keys_impl::(array), + DataType::Int8 => clear_out_of_range_null_keys_impl::(array), + DataType::Int16 => clear_out_of_range_null_keys_impl::(array), + DataType::Int32 => clear_out_of_range_null_keys_impl::(array), + DataType::Int64 => clear_out_of_range_null_keys_impl::(array), + _ => Err(Error::not_supported_source( + format!("Unsupported dictionary key type: {}", key_type).into(), + )), + }, + _ => Err(Error::internal(format!( + "Data type is not a dictionary: {}", + array.data_type() + ))), + } +} + fn dict_encode_variable_width( variable_width_data_block: &VariableWidthBlock, bits_per_offset: u8, diff --git a/rust/lance-encoding/src/encodings/logical/primitive/layout.rs b/rust/lance-encoding/src/encodings/logical/primitive/layout.rs new file mode 100644 index 00000000000..95df12ede6b --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/layout.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use lance_core::Result; + +use crate::repdef::MiniBlockRepDefBudget; + +/// Runs automatic sparse planning only after the dense mini-block budget makes it useful. +pub(super) fn select_automatic_sparse( + requested_encoding: Option<&str>, + dense_budget: &MiniBlockRepDefBudget, + candidate: impl FnOnce() -> Result>, +) -> Result> { + if requested_encoding.is_some() + || !matches!( + dense_budget, + MiniBlockRepDefBudget::RequiresPageSplit(_) + | MiniBlockRepDefBudget::SingleRowOverBudget(_) + ) + { + return Ok(None); + } + candidate() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::constants::{ + STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE, + }; + + #[test] + fn within_budget_does_not_construct_sparse_candidate() { + let selected = + select_automatic_sparse::<()>(None, &MiniBlockRepDefBudget::WithinBudget, || { + panic!("within-budget pages must not construct sparse candidates") + }) + .unwrap(); + assert!(selected.is_none()); + } + + #[test] + fn over_budget_selects_only_eligible_candidates() { + let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new()); + let selected = select_automatic_sparse(None, &split, || Ok(Some(42))).unwrap(); + assert_eq!(selected, Some(42)); + + let ineligible = select_automatic_sparse::<()>(None, &split, || Ok(None)).unwrap(); + assert!(ineligible.is_none()); + + let unsplittable = MiniBlockRepDefBudget::SingleRowOverBudget(70_000); + let selected = select_automatic_sparse(None, &unsplittable, || Ok(Some(7))).unwrap(); + assert_eq!(selected, Some(7)); + } + + #[test] + fn explicit_modes_and_lance_2_2_do_not_auto_select() { + let split = MiniBlockRepDefBudget::RequiresPageSplit(Vec::new()); + for requested in [ + STRUCTURAL_ENCODING_MINIBLOCK, + STRUCTURAL_ENCODING_FULLZIP, + STRUCTURAL_ENCODING_SPARSE, + ] { + let selected = select_automatic_sparse::<()>(Some(requested), &split, || { + panic!("explicit modes must not invoke automatic sparse planning") + }) + .unwrap(); + assert!(selected.is_none()); + } + } +} diff --git a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs index 1cf3b9bf581..0ee408a8510 100644 --- a/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs +++ b/rust/lance-encoding/src/encodings/logical/primitive/miniblock.rs @@ -19,7 +19,8 @@ use lance_core::Result; pub const MAX_MINIBLOCK_BYTES: u64 = 8 * 1024 - 6; const DEFAULT_MAX_MINIBLOCK_VALUES: u64 = 4096; -const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768; +/// Maximum number of values that any mini-block decoder accepts from page metadata. +pub(crate) const MAX_CONFIGURABLE_MINIBLOCK_VALUES: u64 = 32768; fn parse_max_miniblock_values() -> u64 { let val = std::env::var("LANCE_MINIBLOCK_MAX_VALUES") @@ -53,6 +54,29 @@ pub struct MiniBlockCompressed { pub num_values: u64, } +/// Per-page framing details that can affect a mini-block compressor's choice. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MiniBlockCompressionContext { + common_chunk_buffers: u64, + support_large_chunk: bool, + allow_generic_offsets: bool, +} + +impl MiniBlockCompressionContext { + /// Creates the framing context supplied by the owning mini-block page. + pub fn new( + common_chunk_buffers: u64, + support_large_chunk: bool, + allow_generic_offsets: bool, + ) -> Self { + Self { + common_chunk_buffers, + support_large_chunk, + allow_generic_offsets, + } + } +} + /// Describes the size of a mini-block chunk of data /// /// Mini-block chunks are designed to be small (just a few disk sectors) @@ -112,7 +136,11 @@ pub trait MiniBlockCompressor: std::fmt::Debug + Send + Sync { /// /// This method also returns a description of the encoding applied that will be /// used at decode time to read the data. - fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)>; + fn compress( + &self, + context: MiniBlockCompressionContext, + page: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)>; } #[cfg(test)] diff --git a/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs b/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs new file mode 100644 index 00000000000..46bb36c808c --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/sparse.rs @@ -0,0 +1,5614 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use super::*; +use arrow_array::new_empty_array; +use arrow_buffer::ArrowNativeType; + +fn invalid_enum( + value: std::result::Result, + label: &str, +) -> Result { + value.map_err(|error| { + Error::invalid_input_source( + format!("Sparse structural {label} has an invalid enum value: {error}").into(), + ) + }) +} + +pub(super) mod writer; + +fn usize_from_u64(value: u64, label: &str) -> Result { + usize::try_from(value).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} {value} exceeds usize::MAX").into(), + ) + }) +} + +/// Native sparse structural representation used by the 2.3 sparse layout. +/// +/// Layers are stored from outer-most to inner-most, matching the order Arrow structural +/// encoders record offsets and validity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SparseStructuralPlan { + pub(crate) layers: Vec, + pub(crate) num_items: u64, + pub(crate) num_visible_items: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SparsePositionSet { + Empty, + All { len: u64 }, + Range { start: u64, len: u64 }, + Explicit(Vec), +} + +impl SparsePositionSet { + pub(crate) fn from_positions( + positions: Vec, + domain_len: u64, + label: &str, + ) -> Result { + if positions.is_empty() { + return Ok(Self::Empty); + } + for window in positions.windows(2) { + let [previous, current] = window else { + continue; + }; + if previous >= current { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} positions must be strictly increasing") + .into(), + )); + } + } + if let Some(position) = positions.iter().find(|position| **position >= domain_len) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} position {} is outside layer with {} slots", + position, domain_len + ) + .into(), + )); + } + + let len = u64::try_from(positions.len()).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} position count exceeds u64::MAX").into(), + ) + })?; + let first = positions.first().copied().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are unexpectedly empty").into(), + ) + })?; + let last = positions.last().copied().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are unexpectedly empty").into(), + ) + })?; + if first == 0 && len == domain_len && domain_len > 0 && last == domain_len - 1 { + return Ok(Self::All { len: domain_len }); + } + if last - first + 1 == len { + return Ok(Self::Range { start: first, len }); + } + Ok(Self::Explicit(positions)) + } + + pub(crate) fn empty() -> Self { + Self::Empty + } + + pub(crate) fn all(len: u64) -> Self { + if len == 0 { + Self::Empty + } else { + Self::All { len } + } + } + + pub(crate) fn range(start: u64, len: u64) -> Self { + if len == 0 { + Self::Empty + } else { + Self::Range { start, len } + } + } + + pub(crate) fn len(&self) -> u64 { + match self { + Self::Empty => 0, + Self::All { len } | Self::Range { len, .. } => *len, + Self::Explicit(positions) => positions.len() as u64, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub(crate) fn deep_size(&self) -> usize { + match self { + Self::Explicit(positions) => positions.len() * std::mem::size_of::(), + Self::Empty | Self::All { .. } | Self::Range { .. } => 0, + } + } + + pub(crate) fn materialize(&self) -> Result> { + match self { + Self::Empty => Ok(Vec::new()), + Self::All { len } => Self::materialize_range(0, *len), + Self::Range { start, len } => Self::materialize_range(*start, *len), + Self::Explicit(positions) => Ok(positions.clone()), + } + } + + fn materialize_range(start: u64, len: u64) -> Result> { + let len_usize = usize::try_from(len).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural position range length {} exceeds usize::MAX", + len + ) + .into(), + ) + })?; + let end = start.checked_add(len).ok_or_else(|| { + Error::invalid_input_source("Sparse structural position range overflows".into()) + })?; + let mut positions = Vec::with_capacity(len_usize); + positions.extend(start..end); + Ok(positions) + } + + fn contains(&self, position: u64) -> bool { + match self { + Self::Empty => false, + Self::All { len } => position < *len, + Self::Range { start, len } => { + position >= *start && position < start.saturating_add(*len) + } + Self::Explicit(positions) => positions.binary_search(&position).is_ok(), + } + } + + fn is_subset_of(&self, other: &Self, domain_len: u64) -> Result { + self.validate_domain(domain_len, "subset")?; + other.validate_domain(domain_len, "superset")?; + Ok(match self { + Self::Empty => true, + Self::All { .. } => other.len() == domain_len, + Self::Range { start, len } => { + let end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source("Sparse structural subset range overflows".into()) + })?; + match other { + Self::All { .. } => true, + Self::Range { + start: other_start, + len: other_len, + } => { + let other_end = other_start.saturating_add(*other_len); + *start >= *other_start && end <= other_end + } + Self::Explicit(positions) => { + let first = positions.partition_point(|position| *position < *start); + let last = positions.partition_point(|position| *position < end); + u64::try_from(last.saturating_sub(first)).ok() == Some(*len) + } + Self::Empty => false, + } + } + Self::Explicit(positions) => positions.iter().all(|position| other.contains(*position)), + }) + } + + fn is_disjoint(&self, other: &Self, domain_len: u64) -> Result { + self.validate_domain(domain_len, "first disjoint set")?; + other.validate_domain(domain_len, "second disjoint set")?; + let (smaller, larger) = if self.len() <= other.len() { + (self, other) + } else { + (other, self) + }; + Ok(match smaller { + Self::Empty => true, + Self::All { .. } => larger.is_empty(), + Self::Range { start, len } => { + let end = start.saturating_add(*len); + match larger { + Self::Empty => true, + Self::All { .. } => false, + Self::Range { + start: other_start, + len: other_len, + } => end <= *other_start || other_start.saturating_add(*other_len) <= *start, + Self::Explicit(positions) => { + let index = positions.partition_point(|position| *position < *start); + positions.get(index).is_none_or(|position| *position >= end) + } + } + } + Self::Explicit(positions) => { + positions.iter().all(|position| !larger.contains(*position)) + } + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SparseValidityMeaning { + NullPositions, + ValidPositions, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SparseValiditySet { + pub(crate) meaning: SparseValidityMeaning, + pub(crate) positions: SparsePositionSet, +} + +impl SparseValiditySet { + pub(crate) fn deep_size(&self) -> usize { + self.positions.deep_size() + } + + fn contains_only_valid_positions( + &self, + positions: &SparsePositionSet, + num_slots: u64, + ) -> Result { + match self.meaning { + SparseValidityMeaning::NullPositions => { + positions.is_disjoint(&self.positions, num_slots) + } + SparseValidityMeaning::ValidPositions => { + positions.is_subset_of(&self.positions, num_slots) + } + } + } + + fn append_to(&self, validity: &mut BooleanBufferBuilder, num_slots: u64) -> Result<()> { + self.positions.validate_domain(num_slots, "validity")?; + let num_slots_usize = usize_from_u64(num_slots, "validity slot count")?; + match (self.meaning, &self.positions) { + (SparseValidityMeaning::NullPositions, SparsePositionSet::Empty) => { + validity.append_n(num_slots_usize, true); + } + (SparseValidityMeaning::ValidPositions, SparsePositionSet::Empty) => { + validity.append_n(num_slots_usize, false); + } + (SparseValidityMeaning::NullPositions, SparsePositionSet::All { .. }) => { + validity.append_n(num_slots_usize, false); + } + (SparseValidityMeaning::ValidPositions, SparsePositionSet::All { .. }) => { + validity.append_n(num_slots_usize, true); + } + (meaning, SparsePositionSet::Range { start, len }) => { + let range_end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source("Sparse structural validity range overflows".into()) + })?; + let default_valid = matches!(meaning, SparseValidityMeaning::NullPositions); + let range_valid = !default_valid; + validity.append_n( + usize_from_u64(*start, "validity range start")?, + default_valid, + ); + validity.append_n(usize_from_u64(*len, "validity range length")?, range_valid); + validity.append_n( + usize_from_u64(num_slots - range_end, "validity range tail")?, + default_valid, + ); + } + (_, SparsePositionSet::Explicit(_)) => { + let mut cursor = SparseValidityCursor::new(self, num_slots, "validity")?; + for slot in 0..num_slots { + validity.append(cursor.is_valid(slot)?); + } + cursor.finish()?; + } + } + Ok(()) + } +} + +impl SparsePositionSet { + fn validate_domain(&self, domain_len: u64, label: &str) -> Result<()> { + match self { + Self::Empty => {} + Self::All { len } => { + if *len != domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} all set length {} does not match domain {}", + len, domain_len + ) + .into(), + )); + } + } + Self::Range { start, len } => { + let end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} range overflows").into(), + ) + })?; + if end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} range {}..{} is outside domain {}", + start, end, domain_len + ) + .into(), + )); + } + } + Self::Explicit(positions) => { + for window in positions.windows(2) { + let [previous, current] = window else { + continue; + }; + if previous >= current { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} positions must be strictly increasing" + ) + .into(), + )); + } + } + if let Some(position) = positions.last() + && *position >= domain_len + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} position {} is outside layer with {} slots", + position, domain_len + ) + .into(), + )); + } + } + } + Ok(()) + } +} + +struct SparsePositionSetCursor<'a> { + set: &'a SparsePositionSet, + explicit: Option>>, +} + +impl<'a> SparsePositionSetCursor<'a> { + fn new(set: &'a SparsePositionSet, domain_len: u64, label: &str) -> Result { + set.validate_domain(domain_len, label)?; + let explicit = match set { + SparsePositionSet::Explicit(positions) => Some(positions.iter().peekable()), + _ => None, + }; + Ok(Self { set, explicit }) + } + + fn contains(&mut self, slot: u64) -> Result { + Ok(match self.set { + SparsePositionSet::Empty => false, + SparsePositionSet::All { .. } => true, + SparsePositionSet::Range { start, len } => { + slot >= *start && slot < start.saturating_add(*len) + } + SparsePositionSet::Explicit(_) => { + let iter = self.explicit.as_mut().ok_or_else(|| { + Error::internal("Sparse structural explicit cursor is missing".to_string()) + })?; + if let Some(position) = iter.peek() + && **position < slot + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural explicit position {} was skipped before slot {}", + **position, slot + ) + .into(), + )); + } + if iter.peek().is_some_and(|position| **position == slot) { + iter.next(); + true + } else { + false + } + } + }) + } + + fn finish(&mut self) -> Result<()> { + if let Some(iter) = self.explicit.as_mut() + && let Some(position) = iter.next() + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural explicit position {} was not consumed", + position + ) + .into(), + )); + } + Ok(()) + } +} + +struct SparseValidityCursor<'a> { + meaning: SparseValidityMeaning, + positions: SparsePositionSetCursor<'a>, +} + +impl<'a> SparseValidityCursor<'a> { + fn new(validity: &'a SparseValiditySet, domain_len: u64, label: &str) -> Result { + Ok(Self { + meaning: validity.meaning, + positions: SparsePositionSetCursor::new(&validity.positions, domain_len, label)?, + }) + } + + fn is_valid(&mut self, slot: u64) -> Result { + let is_stored = self.positions.contains(slot)?; + Ok(match self.meaning { + SparseValidityMeaning::NullPositions => !is_stored, + SparseValidityMeaning::ValidPositions => is_stored, + }) + } + + fn finish(&mut self) -> Result<()> { + self.positions.finish() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SparseCountSet { + Empty, + Constant { + value: u64, + len: u64, + }, + Explicit { + counts: Arc<[u64]>, + offsets: Arc<[u64]>, + }, +} + +impl SparseCountSet { + pub(crate) fn from_counts(counts: Vec) -> Result { + if counts.is_empty() { + return Ok(Self::Empty); + } + if let Some(first) = counts.first().copied() + && counts.iter().all(|count| *count == first) + { + return Ok(Self::Constant { + value: first, + len: counts.len() as u64, + }); + } + let offsets = offsets_from_counts(&counts)?; + Ok(Self::Explicit { + counts: counts.into(), + offsets: offsets.into(), + }) + } + + pub(crate) fn constant(value: u64, len: u64) -> Self { + if len == 0 { + Self::Empty + } else { + Self::Constant { value, len } + } + } + + pub(crate) fn len(&self) -> u64 { + match self { + Self::Empty => 0, + Self::Constant { len, .. } => *len, + Self::Explicit { counts, .. } => counts.len() as u64, + } + } + + pub(crate) fn deep_size(&self) -> usize { + match self { + Self::Explicit { counts, offsets } => { + counts.len() * std::mem::size_of::() + + offsets.len() * std::mem::size_of::() + } + Self::Empty | Self::Constant { .. } => 0, + } + } + + pub(crate) fn materialize(&self) -> Result> { + match self { + Self::Empty => Ok(Vec::new()), + Self::Constant { value, len } => { + let len = usize::try_from(*len).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count set length exceeds usize::MAX".into(), + ) + })?; + Ok(vec![*value; len]) + } + Self::Explicit { counts, .. } => Ok(counts.to_vec()), + } + } + + pub(crate) fn sum(&self) -> Result { + match self { + Self::Empty => Ok(0), + Self::Constant { value, len } => value.checked_mul(*len).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural constant count sum overflows: value={}, len={}", + value, len + ) + .into(), + ) + }), + Self::Explicit { offsets, .. } => offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural explicit count offsets are empty".into(), + ) + }), + } + } + + fn validate_positive(&self) -> Result<()> { + let has_zero = match self { + Self::Empty => false, + Self::Constant { value, .. } => *value == 0, + Self::Explicit { counts, .. } => counts.contains(&0), + }; + if has_zero { + return Err(Error::invalid_input_source( + "Sparse structural non-empty list count is zero".into(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SparseStructuralLayerPlan { + Validity { + num_slots: u64, + validity: SparseValiditySet, + }, + List { + num_slots: u64, + num_child_slots: u64, + non_empty_positions: SparsePositionSet, + counts: SparseCountSet, + validity: SparseValiditySet, + }, + FixedSizeList { + num_slots: u64, + dimension: u64, + validity: SparseValiditySet, + }, +} + +impl SparseStructuralPlan { + fn expected_num_items( + layers: &[SparseStructuralLayerPlan], + num_visible_items: u64, + ) -> Result { + layers.iter().try_fold(num_visible_items, |items, layer| { + let additional = match layer { + SparseStructuralLayerPlan::List { + num_slots, + non_empty_positions, + .. + } => num_slots + .checked_sub(non_empty_positions.len()) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list has more non-empty positions than slots".into(), + ) + })?, + SparseStructuralLayerPlan::Validity { .. } + | SparseStructuralLayerPlan::FixedSizeList { .. } => 0, + }; + items.checked_add(additional).ok_or_else(|| { + Error::invalid_input_source("Sparse structural item count overflows".into()) + }) + }) + } + + fn validate(&self, row_domain: u64) -> Result<()> { + usize_from_u64(self.num_visible_items, "visible item count")?; + let expected_num_items = Self::expected_num_items(&self.layers, self.num_visible_items)?; + if self.num_items != expected_num_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural item count {} does not match the {} items implied by its layers", + self.num_items, expected_num_items + ) + .into(), + )); + } + let mut expected_slots = row_domain; + for (layer_index, layer) in self.layers.iter().enumerate() { + let (num_slots, num_child_slots, validity) = match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } => (*num_slots, *num_slots, validity), + SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } => { + counts.validate_positive()?; + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list layer {} has {} non-empty positions but {} counts", + layer_index, + non_empty_positions.len(), + counts.len() + ) + .into(), + )); + } + if counts.sum()? != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list layer {} count sum does not match {} child slots", + layer_index, num_child_slots + ) + .into(), + )); + } + if !validity.contains_only_valid_positions(non_empty_positions, *num_slots)? { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list layer {} contains a non-empty null slot", + layer_index + ) + .into(), + )); + } + (*num_slots, *num_child_slots, validity) + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots, + dimension, + validity, + } => { + if *dimension == 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list layer {} has dimension zero", + layer_index + ) + .into(), + )); + } + let num_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list child domain overflows".into(), + ) + })?; + (*num_slots, num_child_slots, validity) + } + }; + usize_from_u64(num_slots, "layer slot count")?; + usize_from_u64(num_child_slots, "layer child slot count")?; + if num_slots != expected_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural layer {} has {} slots, expected {}", + layer_index, num_slots, expected_slots + ) + .into(), + )); + } + validity.positions.validate_domain(num_slots, "validity")?; + expected_slots = num_child_slots; + } + if expected_slots != self.num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural terminal domain has {} slots, expected {} visible items", + expected_slots, self.num_visible_items + ) + .into(), + )); + } + Ok(()) + } +} + +#[derive(Debug)] +pub struct SparseStructuralUnraveler { + layers: Vec, + next_layer: usize, + pending_fixed_size_list: bool, +} + +impl SparseStructuralUnraveler { + pub(crate) fn new(plan: SparseStructuralPlan) -> Self { + let next_layer = plan.layers.len(); + Self { + layers: plan.layers, + next_layer, + pending_fixed_size_list: false, + } + } + + fn current_layer(&self) -> Option<&SparseStructuralLayerPlan> { + self.next_layer + .checked_sub(1) + .and_then(|idx| self.layers.get(idx)) + } + + fn consume_current_layer(&mut self) -> Result<()> { + self.next_layer = self.next_layer.checked_sub(1).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural metadata has fewer layers than the Arrow schema".into(), + ) + })?; + Ok(()) + } + + pub(crate) fn ensure_exhausted(&self) -> Result<()> { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse structural metadata has an unconsumed fixed-size-list layer".into(), + )); + } + if self.next_layer != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural metadata has {} unconsumed layer(s)", + self.next_layer + ) + .into(), + )); + } + Ok(()) + } + + pub(crate) fn is_all_valid(&self) -> bool { + match self.current_layer() { + Some(SparseStructuralLayerPlan::Validity { + num_slots, + validity, + }) + | Some(SparseStructuralLayerPlan::FixedSizeList { + num_slots, + validity, + .. + }) + | Some(SparseStructuralLayerPlan::List { + num_slots, + validity, + .. + }) => match validity.meaning { + SparseValidityMeaning::NullPositions => validity.positions.is_empty(), + SparseValidityMeaning::ValidPositions => validity.positions.len() == *num_slots, + }, + None => true, + } + } + + pub(crate) fn max_lists(&self) -> Result { + match self.current_layer() { + Some(SparseStructuralLayerPlan::List { num_slots, .. }) => { + usize_from_u64(*num_slots, "list slot count") + } + _ => Ok(0), + } + } + + pub(crate) fn skip_validity(&mut self) -> Result<()> { + match self.current_layer() { + Some(SparseStructuralLayerPlan::Validity { .. }) => { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list schema does not match a validity layer".into(), + )); + } + self.consume_current_layer()?; + } + Some(SparseStructuralLayerPlan::FixedSizeList { .. }) => { + if !self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list layer does not match the Arrow schema".into(), + )); + } + self.pending_fixed_size_list = false; + self.consume_current_layer()?; + } + None => { + return Err(Error::invalid_input_source( + "Sparse structural metadata has fewer layers than the Arrow schema".into(), + )); + } + Some(SparseStructuralLayerPlan::List { .. }) => { + return Err(Error::invalid_input_source( + "Sparse structural list layer does not match an Arrow validity layer".into(), + )); + } + } + Ok(()) + } + + pub(crate) fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) -> Result<()> { + match self.current_layer() { + Some(SparseStructuralLayerPlan::Validity { + num_slots, + validity: layer_validity, + }) => { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list schema does not match a validity layer".into(), + )); + } + layer_validity.append_to(validity, *num_slots)?; + self.consume_current_layer()?; + } + Some(SparseStructuralLayerPlan::FixedSizeList { + num_slots, + validity: layer_validity, + .. + }) => { + if !self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list layer does not match the Arrow schema".into(), + )); + } + layer_validity.append_to(validity, *num_slots)?; + self.pending_fixed_size_list = false; + self.consume_current_layer()?; + } + None => { + return Err(Error::invalid_input_source( + "Sparse structural metadata has fewer layers than the Arrow schema".into(), + )); + } + Some(SparseStructuralLayerPlan::List { .. }) => { + return Err(Error::invalid_input_source( + "Sparse structural list layer does not match an Arrow validity layer".into(), + )); + } + } + Ok(()) + } + + pub(crate) fn decimate(&mut self, dimension: usize) -> Result<()> { + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list layer was decimated more than once".into(), + )); + } + let Some(SparseStructuralLayerPlan::FixedSizeList { + dimension: actual_dimension, + .. + }) = self.current_layer() + else { + return Err(Error::invalid_input_source( + "Sparse structural layer does not match an Arrow fixed-size-list layer".into(), + )); + }; + if usize_from_u64(*actual_dimension, "fixed-size-list dimension")? != dimension { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list dimension {} does not match Arrow dimension {}", + actual_dimension, dimension + ) + .into(), + )); + } + self.pending_fixed_size_list = true; + Ok(()) + } + + fn to_offset(value: u64) -> Result { + let value = usize::try_from(value).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural offset {} exceeds usize::MAX", value).into(), + ) + })?; + T::from_usize(value).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural offset does not fit the Arrow offset type".into(), + ) + }) + } + + pub(crate) fn unravel_offsets( + &mut self, + offsets: &mut Vec, + validity: Option<&mut BooleanBufferBuilder>, + ) -> Result<()> { + let Some(SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity: layer_validity, + .. + }) = self.current_layer() + else { + return Err(Error::invalid_input_source( + "Sparse structural layer does not match an Arrow list layer".into(), + )); + }; + if self.pending_fixed_size_list { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list schema does not match an Arrow list layer".into(), + )); + } + + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} non-empty positions but {} counts", + non_empty_positions.len(), + counts.len() + ) + .into(), + )); + } + let actual_child_slots = counts.sum()?; + if actual_child_slots != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + actual_child_slots, num_child_slots + ) + .into(), + )); + } + + let mut current_offset = offsets + .last() + .map(|offset| offset.as_usize() as u64) + .unwrap_or(0); + if offsets.is_empty() { + offsets.push(Self::to_offset(current_offset)?); + } + + if non_empty_positions.is_empty() { + if let Some(validity) = validity { + layer_validity.append_to(validity, *num_slots)?; + } + let offset = Self::to_offset(current_offset)?; + let new_len = offsets + .len() + .checked_add(usize_from_u64(*num_slots, "list slot count")?) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list offset length overflows usize".into(), + ) + })?; + offsets.resize(new_len, offset); + self.consume_current_layer()?; + return Ok(()); + } + + let non_empty_positions = non_empty_positions.materialize()?; + let counts = counts.materialize()?; + let mut non_empty_iter = non_empty_positions + .iter() + .copied() + .zip(counts.iter().copied()) + .peekable(); + let mut validity_cursor = + SparseValidityCursor::new(layer_validity, *num_slots, "list validity")?; + let mut validity = validity; + for slot in 0..*num_slots { + let is_valid = validity_cursor.is_valid(slot)?; + if let Some(validity) = validity.as_mut() { + validity.append(is_valid); + } + + if non_empty_iter + .peek() + .is_some_and(|(non_empty_pos, _)| *non_empty_pos == slot) + { + let (_, count) = non_empty_iter.next().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list position is missing its child count".into(), + ) + })?; + if !is_valid { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list slot {} is both invalid and non-empty", + slot + ) + .into(), + )); + } + current_offset = current_offset.checked_add(count).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list offsets overflow u64".into(), + ) + })?; + } + offsets.push(Self::to_offset(current_offset)?); + } + validity_cursor.finish()?; + if let Some((extra_pos, _)) = non_empty_iter.next() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural non-empty position {} is outside layer with {} slots", + extra_pos, num_slots + ) + .into(), + )); + } + + self.consume_current_layer()?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +enum SparsePositionSetDecoder { + Empty, + All { + len: u64, + }, + Range { + start: u64, + len: u64, + }, + Explicit { + decompressor: Arc, + encoding: CompressiveEncoding, + count: u64, + domain_len: u64, + }, +} + +#[derive(Debug, Clone)] +enum SparseCountSetDecoder { + Empty, + Constant { + value: u64, + len: u64, + }, + Explicit { + decompressor: Arc, + encoding: CompressiveEncoding, + count: u64, + }, +} + +#[derive(Debug, Clone)] +enum SparseLayerDecompressors { + Validity { + num_slots: u64, + validity: SparseValiditySetDecoder, + }, + List { + num_slots: u64, + num_child_slots: u64, + non_empty_positions: SparsePositionSetDecoder, + counts: SparseCountSetDecoder, + validity: SparseValiditySetDecoder, + }, + FixedSizeList { + num_slots: u64, + num_child_slots: u64, + dimension: u64, + validity: SparseValiditySetDecoder, + }, +} + +#[derive(Debug, Clone)] +struct SparseValiditySetDecoder { + meaning: SparseValidityMeaning, + positions: SparsePositionSetDecoder, +} + +#[derive(Debug)] +struct SparseStructuralCacheableState { + chunk_meta: Vec, + chunk_value_offsets: Arc<[u64]>, + plan: SparseStructuralPlan, + row_domain: u64, +} + +impl DeepSizeOf for SparseStructuralCacheableState { + fn deep_size_of_children(&self, _context: &mut Context) -> usize { + let structural_size = self + .plan + .layers + .iter() + .map(|layer| match layer { + SparseStructuralLayerPlan::Validity { validity, .. } => validity.deep_size(), + SparseStructuralLayerPlan::List { + non_empty_positions, + counts, + validity, + .. + } => non_empty_positions.deep_size() + counts.deep_size() + validity.deep_size(), + SparseStructuralLayerPlan::FixedSizeList { validity, .. } => validity.deep_size(), + }) + .sum::(); + self.chunk_meta.len() * std::mem::size_of::() + + self.chunk_value_offsets.len() * std::mem::size_of::() + + structural_size + } +} + +impl CachedPageData for SparseStructuralCacheableState { + fn as_arc_any(self: Arc) -> Arc { + self + } +} + +#[derive(Debug)] +pub(super) struct SparseStructuralScheduler { + buffer_offsets_and_sizes: Vec<(u64, u64)>, + priority: u64, + row_domain: u64, + row_scale: u64, + num_items: u64, + num_visible_items: u64, + num_buffers: u64, + value_encoding: CompressiveEncoding, + value_decompressor: Arc, + layer_decompressors: Vec, + data_type: DataType, + page_meta: Option>, + has_large_chunk: bool, +} + +impl SparseStructuralScheduler { + fn require_layer( + layer: &pb21::SparseStructuralLayer, + ) -> Result<&pb21::sparse_structural_layer::Layer> { + layer.layer.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural layer is missing its layer variant".into(), + ) + }) + } + + fn layer_num_slots(layer: &pb21::SparseStructuralLayer) -> Result { + Ok(match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::List(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer.num_slots, + }) + } + + fn layer_num_child_slots(layer: &pb21::SparseStructuralLayer) -> Result { + Ok(match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::List(layer) => layer.num_child_slots, + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer + .num_slots + .checked_mul(layer.dimension) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + layer.num_slots, layer.dimension + ) + .into(), + ) + })?, + }) + } + + pub(super) fn try_new( + buffer_offsets_and_sizes: &[(u64, u64)], + priority: u64, + encoded_row_domain: u64, + data_type: DataType, + layout: &pb21::SparseLayout, + decompressors: &dyn DecompressionStrategy, + ) -> Result { + let value_compression = layout.value_compression.as_ref().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing value compression".into()) + })?; + let value_buffer_count = Self::validate_value_encoding(value_compression)?; + if layout.num_buffers != value_buffer_count { + return Err(Error::invalid_input_source( + format!( + "Sparse layout declares {} value buffers, but its compression descriptor requires {}", + layout.num_buffers, value_buffer_count + ) + .into(), + )); + } + let row_domain = match layout.structural_layers.first() { + Some(layer) => Self::layer_num_slots(layer)?, + None => encoded_row_domain, + }; + Self::validate_domain_chain( + &layout.structural_layers, + row_domain, + layout.num_items, + layout.num_visible_items, + )?; + let expected_buffers = 2 + Self::structural_buffer_count(&layout.structural_layers)?; + if buffer_offsets_and_sizes.len() != expected_buffers { + return Err(Error::invalid_input_source( + format!( + "Sparse layout has {} buffers, expected {}", + buffer_offsets_and_sizes.len(), + expected_buffers + ) + .into(), + )); + } + Self::validate_page_buffers(buffer_offsets_and_sizes, layout.num_visible_items)?; + let row_scale = + layout + .structural_layers + .iter() + .try_fold(1_u64, |scale, layer| -> Result { + match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + scale.checked_mul(layer.dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list row scale overflows".into(), + ) + }) + } + pb21::sparse_structural_layer::Layer::Validity(_) + | pb21::sparse_structural_layer::Layer::List(_) => Ok(scale), + } + })?; + let expected_encoded_row_domain = row_domain.checked_mul(row_scale).ok_or_else(|| { + Error::invalid_input_source("Sparse structural encoded row domain overflows".into()) + })?; + if encoded_row_domain != expected_encoded_row_domain { + return Err(Error::invalid_input_source( + format!( + "Sparse structural encoded row domain {} does not match outer domain {} * fixed-size-list scale {}", + encoded_row_domain, row_domain, row_scale + ) + .into(), + )); + } + let value_decompressor = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + decompressors.create_miniblock_decompressor(value_compression, decompressors) + })) + .map_err(|_| { + Error::invalid_input_source( + "Sparse value compression descriptor caused decompressor construction to panic" + .into(), + ) + })? + .map_err(|error| { + Error::invalid_input_source( + format!("Sparse value decompressor construction failed: {error}").into(), + ) + })?; + let layer_decompressors = layout + .structural_layers + .iter() + .map(|layer| Self::layer_decompressors(layer, decompressors)) + .collect::>>()?; + + Ok(Self { + buffer_offsets_and_sizes: buffer_offsets_and_sizes.to_vec(), + priority, + row_domain, + row_scale, + num_items: layout.num_items, + num_visible_items: layout.num_visible_items, + num_buffers: layout.num_buffers, + value_encoding: value_compression.clone(), + value_decompressor: value_decompressor.into(), + layer_decompressors, + data_type, + page_meta: None, + has_large_chunk: layout.has_large_chunk, + }) + } + + fn validate_compression<'a>( + compression: &'a CompressiveEncoding, + label: &str, + ) -> Result<&'a CompressiveEncoding> { + compression.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} is missing compression details").into(), + ) + })?; + Ok(compression) + } + + fn validate_buffer_compression( + compression: Option<&pb21::BufferCompression>, + label: &str, + ) -> Result<()> { + let Some(compression) = compression else { + return Ok(()); + }; + match invalid_enum( + pb21::CompressionScheme::try_from(compression.scheme), + "compression scheme", + )? { + pb21::CompressionScheme::CompressionAlgorithmUnspecified => { + Err(Error::invalid_input_source( + format!("Sparse structural {label} buffer compression is unspecified").into(), + )) + } + pb21::CompressionScheme::CompressionAlgorithmLz4 + | pb21::CompressionScheme::CompressionAlgorithmZstd => Ok(()), + } + } + + fn encoding_contains_general(root: &CompressiveEncoding) -> bool { + use pb21::compressive_encoding::Compression; + + let mut stack = vec![root]; + while let Some(encoding) = stack.pop() { + let Some(compression) = encoding.compression.as_ref() else { + continue; + }; + match compression { + Compression::General(_) => return true, + Compression::Variable(variable) => { + stack.extend(variable.offsets.as_deref()); + } + Compression::OutOfLineBitpacking(bitpacking) => { + stack.extend(bitpacking.values.as_deref()); + } + Compression::Fsst(fsst) => stack.extend(fsst.values.as_deref()), + Compression::Dictionary(dictionary) => { + stack.extend(dictionary.indices.as_deref()); + stack.extend(dictionary.items.as_deref()); + } + Compression::Rle(rle) => { + stack.extend(rle.values.as_deref()); + stack.extend(rle.run_lengths.as_deref()); + } + Compression::ByteStreamSplit(split) => { + stack.extend(split.values.as_deref()); + } + Compression::FixedSizeList(fsl) => stack.extend(fsl.values.as_deref()), + Compression::PackedStruct(packed) => stack.extend(packed.values.as_deref()), + Compression::VariablePackedStruct(packed) => { + stack.extend( + packed + .fields + .iter() + .filter_map(|field| field.value.as_ref()), + ); + } + Compression::Flat(_) + | Compression::Constant(_) + | Compression::InlineBitpacking(_) => {} + } + } + false + } + + fn require_encoding<'a>( + encoding: &'a Option>, + label: &str, + ) -> Result<&'a CompressiveEncoding> { + encoding.as_deref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} encoding is required").into(), + ) + }) + } + + fn validate_flat(flat: &pb21::Flat, label: &str) -> Result<()> { + if flat.bits_per_value == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} flat bit width is zero").into(), + )); + } + if flat.data.is_some() { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} uses unsupported leaf buffer compression") + .into(), + )); + } + Ok(()) + } + + fn validate_value_encoding(encoding: &CompressiveEncoding) -> Result { + use pb21::compressive_encoding::Compression; + + let compression = encoding.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse value compression is missing compression details".into(), + ) + })?; + match compression { + Compression::Flat(flat) => { + Self::validate_flat(flat, "value")?; + Ok(1) + } + Compression::InlineBitpacking(bitpacking) => { + if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse inline bitpacking width {} is not supported", + bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + if bitpacking.values.is_some() { + return Err(Error::invalid_input_source( + "Sparse inline bitpacking uses unsupported leaf buffer compression".into(), + )); + } + Ok(1) + } + Compression::Variable(variable) => { + let offsets = variable.offsets.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse variable compression is missing offsets".into(), + ) + })?; + let Some(Compression::Flat(offsets)) = offsets.compression.as_ref() else { + return Err(Error::invalid_input_source( + "Sparse variable offsets must use flat compression".into(), + )); + }; + Self::validate_flat(offsets, "variable offsets")?; + if !matches!(offsets.bits_per_value, 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse variable offset width {} is not supported", + offsets.bits_per_value + ) + .into(), + )); + } + if variable.values.is_some() { + return Err(Error::invalid_input_source( + "Sparse variable values use unsupported leaf buffer compression".into(), + )); + } + Ok(1) + } + Compression::Fsst(fsst) => { + if fsst.symbol_table.is_empty() { + return Err(Error::invalid_input_source( + "Sparse FSST compression has an empty symbol table".into(), + )); + } + let values = Self::require_encoding(&fsst.values, "FSST values")?; + if !matches!(values.compression.as_ref(), Some(Compression::Variable(_))) { + return Err(Error::invalid_input_source( + "Sparse FSST values must use variable compression".into(), + )); + } + Self::validate_value_encoding(values) + } + Compression::ByteStreamSplit(split) => { + let values = Self::require_encoding(&split.values, "byte-stream-split values")?; + let Some(Compression::Flat(flat)) = values.compression.as_ref() else { + return Err(Error::invalid_input_source( + "Sparse byte-stream-split values must use flat compression".into(), + )); + }; + Self::validate_flat(flat, "byte-stream-split values")?; + if !matches!(flat.bits_per_value, 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse byte-stream-split width {} is not supported", + flat.bits_per_value + ) + .into(), + )); + } + Ok(1) + } + Compression::FixedSizeList(fsl) => Self::validate_fsl_value_encoding(fsl), + Compression::PackedStruct(packed) => Self::validate_packed_value_encoding(packed), + Compression::Rle(rle) => { + let values = Self::require_encoding(&rle.values, "RLE values")?; + let lengths = Self::require_encoding(&rle.run_lengths, "RLE run lengths")?; + Self::validate_block_encoding(values, "RLE values")?; + Self::validate_block_encoding(lengths, "RLE run lengths")?; + Ok(2) + } + Compression::General(general) => { + let compression = general.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "Sparse general compression is missing its buffer compression".into(), + ) + })?; + Self::validate_buffer_compression(Some(compression), "general")?; + Self::validate_value_encoding(Self::require_encoding( + &general.values, + "general values", + )?) + } + Compression::Constant(_) + | Compression::OutOfLineBitpacking(_) + | Compression::Dictionary(_) + | Compression::VariablePackedStruct(_) => Err(Error::invalid_input_source( + "Sparse value compression uses an unsupported mini-block encoding".into(), + )), + } + } + + fn validate_fsl_value_encoding(fsl: &pb21::FixedSizeList) -> Result { + use pb21::compressive_encoding::Compression; + + if fsl.items_per_value == 0 { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list value compression has dimension zero".into(), + )); + } + let values = Self::require_encoding(&fsl.values, "fixed-size-list values")?; + let child_buffers = match values.compression.as_ref() { + Some(Compression::Flat(flat)) => { + Self::validate_flat(flat, "fixed-size-list values")?; + 1_u64 + } + Some(Compression::FixedSizeList(inner)) => Self::validate_fsl_value_encoding(inner)?, + _ => { + return Err(Error::invalid_input_source( + "Sparse fixed-size-list values must use fixed-size-list or flat compression" + .into(), + )); + } + }; + child_buffers + .checked_add(u64::from(fsl.has_validity)) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list value buffer count overflows".into(), + ) + }) + } + + fn validate_packed_value_encoding(packed: &pb21::PackedStruct) -> Result { + use pb21::compressive_encoding::Compression; + + if packed.bits_per_value.is_empty() + || packed + .bits_per_value + .iter() + .any(|bits| *bits == 0 || !bits.is_multiple_of(8)) + { + return Err(Error::invalid_input_source( + "Sparse packed-struct widths must be non-empty positive byte widths".into(), + )); + } + let values = Self::require_encoding(&packed.values, "packed-struct values")?; + let Some(Compression::Flat(flat)) = values.compression.as_ref() else { + return Err(Error::invalid_input_source( + "Sparse packed-struct values must use flat compression".into(), + )); + }; + Self::validate_flat(flat, "packed-struct values")?; + let total_bits = packed.bits_per_value.iter().try_fold(0_u64, |sum, bits| { + sum.checked_add(*bits).ok_or_else(|| { + Error::invalid_input_source("Sparse packed-struct bit width sum overflows".into()) + }) + })?; + if total_bits != flat.bits_per_value { + return Err(Error::invalid_input_source( + format!( + "Sparse packed-struct child widths sum to {}, but values use {} bits", + total_bits, flat.bits_per_value + ) + .into(), + )); + } + Ok(1) + } + + fn validate_block_encoding(encoding: &CompressiveEncoding, label: &str) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} is missing compression details").into(), + ) + })? { + Compression::Flat(flat) => Self::validate_flat(flat, label), + Compression::InlineBitpacking(bitpacking) => { + if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} inline bitpacking width {} is unsupported", + bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + if bitpacking.values.is_some() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} uses unsupported leaf buffer compression" + ) + .into(), + )); + } + Ok(()) + } + Compression::OutOfLineBitpacking(bitpacking) => { + if !matches!(bitpacking.uncompressed_bits_per_value, 8 | 16 | 32 | 64) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} out-of-line bitpacking width {} is unsupported", + bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + let values = Self::require_encoding(&bitpacking.values, label)?; + let Some(Compression::Flat(flat)) = values.compression.as_ref() else { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} bitpacked values must be flat").into(), + )); + }; + Self::validate_flat(flat, label) + } + Compression::Constant(constant) => { + if constant + .value + .as_ref() + .is_some_and(|value| value.len() != 8) + { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} constant must be 64 bits").into(), + )); + } + Ok(()) + } + Compression::General(general) => { + let compression = general.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} general compression is missing config") + .into(), + ) + })?; + Self::validate_buffer_compression(Some(compression), label)?; + Self::validate_block_encoding( + Self::require_encoding(&general.values, label)?, + label, + ) + } + Compression::Rle(rle) => { + Self::validate_block_encoding( + Self::require_encoding(&rle.values, "RLE values")?, + "RLE values", + )?; + Self::validate_block_encoding( + Self::require_encoding(&rle.run_lengths, "RLE run lengths")?, + "RLE run lengths", + ) + } + _ => Err(Error::invalid_input_source( + format!("Sparse structural {label} uses an unsupported block encoding").into(), + )), + } + } + + fn validate_domain_chain( + layers: &[pb21::SparseStructuralLayer], + row_domain: u64, + num_items: u64, + num_visible_items: u64, + ) -> Result<()> { + let expected_num_items = + layers + .iter() + .try_fold(num_visible_items, |items, layer| -> Result { + let additional = match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::List(layer) => { + let positions = Self::require_position_set( + &layer.non_empty_positions, + "list non-empty", + )?; + let num_non_empty = Self::position_cardinality( + positions, + layer.num_slots, + "list non-empty positions", + )?; + layer.num_slots.checked_sub(num_non_empty).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list has more non-empty positions than slots" + .into(), + ) + })? + } + pb21::sparse_structural_layer::Layer::Validity(_) + | pb21::sparse_structural_layer::Layer::FixedSizeList(_) => 0, + }; + items.checked_add(additional).ok_or_else(|| { + Error::invalid_input_source("Sparse structural item count overflows".into()) + }) + })?; + if num_items != expected_num_items { + return Err(Error::invalid_input_source( + format!( + "Sparse layout has {} structural items, but its layers imply {}", + num_items, expected_num_items + ) + .into(), + )); + } + let mut expected_slots = row_domain; + for (layer_index, layer) in layers.iter().enumerate() { + let num_slots = Self::layer_num_slots(layer)?; + let num_child_slots = Self::layer_num_child_slots(layer)?; + usize_from_u64(num_slots, "layer slot count")?; + usize_from_u64(num_child_slots, "layer child slot count")?; + if num_slots != expected_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural layer {} has {} slots, expected {} from the outer domain", + layer_index, num_slots, expected_slots + ) + .into(), + )); + } + expected_slots = num_child_slots; + } + if expected_slots != num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural terminal domain has {} slots, expected {} visible items", + expected_slots, num_visible_items + ) + .into(), + )); + } + Ok(()) + } + + fn metadata_buffer(&self) -> Result<(u64, u64)> { + self.buffer_offsets_and_sizes + .first() + .copied() + .ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing metadata buffer".into()) + }) + } + + fn value_buffer(&self) -> Result<(u64, u64)> { + self.buffer_offsets_and_sizes + .get(1) + .copied() + .ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing value buffer".into()) + }) + } + + fn checked_buffer_range(position: u64, size: u64, label: &str) -> Result> { + let end = position.checked_add(size).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} buffer range overflows").into(), + ) + })?; + Ok(position..end) + } + + fn validate_page_buffers( + buffer_offsets_and_sizes: &[(u64, u64)], + num_visible_items: u64, + ) -> Result<()> { + let (_, metadata_size) = buffer_offsets_and_sizes.first().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing metadata buffer".into()) + })?; + if !metadata_size.is_multiple_of(8) { + return Err(Error::invalid_input_source( + format!( + "Sparse layout metadata buffer has {metadata_size} bytes, which is not a multiple of 8" + ) + .into(), + )); + } + let chunk_count = metadata_size / 8; + if num_visible_items == 0 { + if chunk_count != 0 { + return Err(Error::invalid_input_source( + "Sparse layout with no visible items must have empty chunk metadata".into(), + )); + } + } else { + let min_chunks = + num_visible_items.div_ceil(miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES); + if chunk_count < min_chunks || chunk_count > num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse layout metadata declares {chunk_count} chunks for {num_visible_items} visible items, expected {min_chunks}..={num_visible_items}" + ) + .into(), + )); + } + } + + let (_, value_size) = buffer_offsets_and_sizes.get(1).copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing value buffer".into()) + })?; + if (num_visible_items == 0) != (value_size == 0) { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value buffer has {value_size} bytes for {num_visible_items} visible items" + ) + .into(), + )); + } + + for (position, size) in buffer_offsets_and_sizes.iter().skip(2) { + Self::checked_buffer_range(*position, *size, "structural")?; + } + + for (position, size) in buffer_offsets_and_sizes.iter().take(2) { + Self::checked_buffer_range(*position, *size, "page")?; + } + Ok(()) + } + + fn require_position_set<'a>( + set: &'a Option, + label: &str, + ) -> Result<&'a pb21::SparsePositionSet> { + set.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is required").into(), + ) + }) + } + + fn require_count_set<'a>( + set: &'a Option, + label: &str, + ) -> Result<&'a pb21::SparseCountSet> { + set.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is required").into(), + ) + }) + } + + fn require_validity_set<'a>( + set: &'a Option, + label: &str, + ) -> Result<&'a pb21::SparseValiditySet> { + set.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} validity set is required").into(), + ) + }) + } + + fn validity_meaning( + validity_set: &pb21::SparseValiditySet, + label: &str, + ) -> Result { + match invalid_enum( + pb21::sparse_validity_set::Meaning::try_from(validity_set.meaning), + "validity meaning", + )? { + pb21::sparse_validity_set::Meaning::SparseValidityUnspecified => { + Err(Error::invalid_input_source( + format!("Sparse structural {label} meaning is unspecified").into(), + )) + } + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions => { + Ok(SparseValidityMeaning::NullPositions) + } + pb21::sparse_validity_set::Meaning::SparseValidityValidPositions => { + Ok(SparseValidityMeaning::ValidPositions) + } + } + } + + fn validity_buffer_count( + validity_set: &pb21::SparseValiditySet, + domain_len: u64, + label: &str, + ) -> Result<(usize, SparseValidityMeaning, u64)> { + let meaning = Self::validity_meaning(validity_set, label)?; + let position_set = validity_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are required").into(), + ) + })?; + let cardinality = Self::position_cardinality(position_set, domain_len, label)?; + let buffer_count = Self::position_buffer_count(position_set, domain_len, label)?; + Ok((buffer_count, meaning, cardinality)) + } + + fn position_cardinality( + position_set: &pb21::SparsePositionSet, + domain_len: u64, + label: &str, + ) -> Result { + let positions = position_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is missing its variant").into(), + ) + })?; + let cardinality = position_set.num_positions; + if cardinality > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} cardinality {} exceeds domain {}", + cardinality, domain_len + ) + .into(), + )); + } + match positions { + pb21::sparse_position_set::Positions::Empty(_) => { + if cardinality != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} empty set has cardinality {}", + cardinality + ) + .into(), + )); + } + } + pb21::sparse_position_set::Positions::All(_) => { + if domain_len == 0 || cardinality != domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} all set has cardinality {}, expected {}", + cardinality, domain_len + ) + .into(), + )); + } + } + pb21::sparse_position_set::Positions::Range(range) => { + let end = range.start.checked_add(range.length).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} range overflows").into(), + ) + })?; + if range.length == 0 || range.length != cardinality || end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} range {}..{} does not match cardinality {} in domain {}", + range.start, end, cardinality, domain_len + ) + .into(), + )); + } + } + pb21::sparse_position_set::Positions::Explicit(compression) => { + if cardinality == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} has compression but no values").into(), + )); + } + Self::validate_compression(compression, label)?; + } + } + Ok(cardinality) + } + + fn position_buffer_count( + position_set: &pb21::SparsePositionSet, + domain_len: u64, + label: &str, + ) -> Result { + Self::position_cardinality(position_set, domain_len, label)?; + let positions = position_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is missing its variant").into(), + ) + })?; + Ok(usize::from(matches!( + positions, + pb21::sparse_position_set::Positions::Explicit(_) + ))) + } + + fn count_buffer_count( + count_set: &pb21::SparseCountSet, + cardinality: u64, + label: &str, + ) -> Result { + let counts = count_set.counts.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is missing its variant").into(), + ) + })?; + match counts { + pb21::sparse_count_set::Counts::Empty(_) => { + if cardinality != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} empty count set has cardinality {}", + cardinality + ) + .into(), + )); + } + Ok(0) + } + pb21::sparse_count_set::Counts::Constant(constant) => { + if cardinality == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} constant count has no values").into(), + )); + } + if constant.value == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} constant count is zero").into(), + )); + } + Ok(0) + } + pb21::sparse_count_set::Counts::Explicit(compression) => { + if cardinality == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} has compression but no values").into(), + )); + } + Self::validate_compression(compression, label)?; + Ok(1) + } + } + } + + fn count_set_child_slots( + count_set: &pb21::SparseCountSet, + cardinality: u64, + label: &str, + ) -> Result> { + let counts = count_set.counts.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is missing its variant").into(), + ) + })?; + match counts { + pb21::sparse_count_set::Counts::Empty(_) => Ok(Some(0)), + pb21::sparse_count_set::Counts::Constant(constant) => constant + .value + .checked_mul(cardinality) + .map(Some) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} constant count sum overflows: value={}, len={}", + constant.value, cardinality + ) + .into(), + ) + }), + pb21::sparse_count_set::Counts::Explicit(_) => Ok(None), + } + } + + fn create_position_decompressor( + compression: &CompressiveEncoding, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result> { + let compression = Self::validate_compression(compression, label)?; + Self::validate_block_encoding(compression, label)?; + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + decompressors.create_block_decompressor(compression) + })) + .map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural {label} descriptor caused decompressor construction to panic" + ) + .into(), + ) + })? + .map(Arc::from) + .map_err(|error| { + Error::invalid_input_source( + format!("Sparse structural {label} decompressor construction failed: {error}") + .into(), + ) + }) + } + + fn position_set_decoder( + position_set: &pb21::SparsePositionSet, + domain_len: u64, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result<(SparsePositionSetDecoder, u64)> { + let cardinality = Self::position_cardinality(position_set, domain_len, label)?; + let positions = position_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position set is missing its variant").into(), + ) + })?; + Ok(( + match positions { + pb21::sparse_position_set::Positions::Empty(_) => SparsePositionSetDecoder::Empty, + pb21::sparse_position_set::Positions::All(_) => { + SparsePositionSetDecoder::All { len: domain_len } + } + pb21::sparse_position_set::Positions::Range(range) => { + SparsePositionSetDecoder::Range { + start: range.start, + len: range.length, + } + } + pb21::sparse_position_set::Positions::Explicit(compression) => { + SparsePositionSetDecoder::Explicit { + decompressor: Self::create_position_decompressor( + compression, + label, + decompressors, + )?, + encoding: compression.clone(), + count: cardinality, + domain_len, + } + } + }, + cardinality, + )) + } + + fn validity_set_decoder( + validity_set: &pb21::SparseValiditySet, + domain_len: u64, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result<(SparseValiditySetDecoder, u64)> { + let meaning = Self::validity_meaning(validity_set, label)?; + let position_set = validity_set.positions.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} positions are required").into(), + ) + })?; + let (positions, cardinality) = + Self::position_set_decoder(position_set, domain_len, label, decompressors)?; + Ok((SparseValiditySetDecoder { meaning, positions }, cardinality)) + } + + fn num_valid_slots( + meaning: SparseValidityMeaning, + cardinality: u64, + num_slots: u64, + label: &str, + ) -> Result { + match meaning { + SparseValidityMeaning::NullPositions => { + num_slots.checked_sub(cardinality).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} null cardinality {} exceeds slots {}", + cardinality, num_slots + ) + .into(), + ) + }) + } + SparseValidityMeaning::ValidPositions => Ok(cardinality), + } + } + + fn count_set_decoder( + count_set: &pb21::SparseCountSet, + cardinality: u64, + label: &str, + decompressors: &dyn DecompressionStrategy, + ) -> Result { + Self::count_buffer_count(count_set, cardinality, label)?; + let counts = count_set.counts.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} count set is missing its variant").into(), + ) + })?; + Ok(match counts { + pb21::sparse_count_set::Counts::Empty(_) => SparseCountSetDecoder::Empty, + pb21::sparse_count_set::Counts::Constant(constant) => SparseCountSetDecoder::Constant { + value: constant.value, + len: cardinality, + }, + pb21::sparse_count_set::Counts::Explicit(compression) => { + SparseCountSetDecoder::Explicit { + decompressor: Self::create_position_decompressor( + compression, + label, + decompressors, + )?, + encoding: compression.clone(), + count: cardinality, + } + } + }) + } + + fn add_buffer_count(count: &mut usize, additional: usize) -> Result<()> { + *count = count.checked_add(additional).ok_or_else(|| { + Error::invalid_input_source("Sparse structural buffer count overflows".into()) + })?; + Ok(()) + } + + fn structural_buffer_count(layers: &[pb21::SparseStructuralLayer]) -> Result { + layers + .iter() + .try_fold(0_usize, |mut count, layer| -> Result { + match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => { + let (validity_buffers, _, _) = Self::validity_buffer_count( + Self::require_validity_set(&layer.validity, "validity")?, + layer.num_slots, + "validity positions", + )?; + Self::add_buffer_count(&mut count, validity_buffers)?; + } + pb21::sparse_structural_layer::Layer::List(layer) => { + let non_empty_positions = Self::require_position_set( + &layer.non_empty_positions, + "list non-empty", + )?; + let num_non_empty = Self::position_cardinality( + non_empty_positions, + layer.num_slots, + "list non-empty positions", + )?; + let non_empty_buffers = Self::position_buffer_count( + non_empty_positions, + layer.num_slots, + "list non-empty positions", + )?; + Self::add_buffer_count(&mut count, non_empty_buffers)?; + let (validity_buffers, validity_meaning, validity_cardinality) = + Self::validity_buffer_count( + Self::require_validity_set(&layer.validity, "list")?, + layer.num_slots, + "list validity positions", + )?; + Self::add_buffer_count(&mut count, validity_buffers)?; + let num_valid_slots = Self::num_valid_slots( + validity_meaning, + validity_cardinality, + layer.num_slots, + "list validity", + )?; + if num_non_empty > num_valid_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} non-empty slots but only {} valid slots", + num_non_empty, num_valid_slots + ) + .into(), + )); + } + let counts = Self::require_count_set(&layer.counts, "list counts")?; + let count_buffers = + Self::count_buffer_count(counts, num_non_empty, "list counts")?; + Self::add_buffer_count(&mut count, count_buffers)?; + if let Some(child_slots) = + Self::count_set_child_slots(counts, num_non_empty, "list counts")? + && child_slots != layer.num_child_slots + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + child_slots, layer.num_child_slots + ) + .into(), + )); + } + } + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + if layer.dimension == 0 { + return Err(Error::invalid_input_source( + "Sparse structural fixed-size-list dimension is zero".into(), + )); + } + layer.num_slots.checked_mul(layer.dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + layer.num_slots, layer.dimension + ) + .into(), + ) + })?; + let (validity_buffers, _, _) = Self::validity_buffer_count( + Self::require_validity_set(&layer.validity, "fixed-size-list")?, + layer.num_slots, + "fixed-size-list validity positions", + )?; + Self::add_buffer_count(&mut count, validity_buffers)?; + } + } + Ok(count) + }) + } + + fn layer_decompressors( + layer: &pb21::SparseStructuralLayer, + decompressors: &dyn DecompressionStrategy, + ) -> Result { + Ok(match Self::require_layer(layer)? { + pb21::sparse_structural_layer::Layer::Validity(layer) => { + let (validity, _) = Self::validity_set_decoder( + Self::require_validity_set(&layer.validity, "validity")?, + layer.num_slots, + "validity positions", + decompressors, + )?; + SparseLayerDecompressors::Validity { + num_slots: layer.num_slots, + validity, + } + } + pb21::sparse_structural_layer::Layer::List(layer) => { + let non_empty_positions = + Self::require_position_set(&layer.non_empty_positions, "list non-empty")?; + let (non_empty_positions, num_non_empty) = Self::position_set_decoder( + non_empty_positions, + layer.num_slots, + "list non-empty positions", + decompressors, + )?; + let counts = Self::count_set_decoder( + Self::require_count_set(&layer.counts, "list counts")?, + num_non_empty, + "list counts", + decompressors, + )?; + let (validity, _) = Self::validity_set_decoder( + Self::require_validity_set(&layer.validity, "list")?, + layer.num_slots, + "list validity positions", + decompressors, + )?; + SparseLayerDecompressors::List { + num_slots: layer.num_slots, + num_child_slots: layer.num_child_slots, + non_empty_positions, + counts, + validity, + } + } + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + let (validity, _) = Self::validity_set_decoder( + Self::require_validity_set(&layer.validity, "fixed-size-list")?, + layer.num_slots, + "fixed-size-list validity positions", + decompressors, + )?; + SparseLayerDecompressors::FixedSizeList { + num_slots: layer.num_slots, + num_child_slots: layer.num_slots.checked_mul(layer.dimension).ok_or_else( + || { + Error::invalid_input_source( + "Sparse structural fixed-size-list child slot count overflows" + .into(), + ) + }, + )?, + dimension: layer.dimension, + validity, + } + } + }) + } + + fn parse_chunk_meta(&self, meta_bytes: Bytes) -> Result> { + if !meta_bytes.len().is_multiple_of(8) { + return Err(Error::invalid_input_source( + format!( + "Sparse layout metadata length {} is not a multiple of 8", + meta_bytes.len() + ) + .into(), + )); + } + + let (value_buf_position, value_buf_size) = self.value_buffer()?; + let value_buf_end = value_buf_position + .checked_add(value_buf_size) + .ok_or_else(|| { + Error::invalid_input_source("Sparse layout value buffer range overflows".into()) + })?; + let mut rows_counter = 0_u64; + let mut offset_bytes = value_buf_position; + let mut chunk_meta = Vec::with_capacity(meta_bytes.len() / 8); + for chunk in meta_bytes.chunks_exact(8) { + let entry: [u8; 8] = chunk.try_into().map_err(|_| { + Error::invalid_input_source( + "Sparse layout chunk metadata entry is not 8 bytes".into(), + ) + })?; + let divided_bytes_minus_one = u32::from_le_bytes( + entry + .get(..4) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout chunk byte-size field is malformed".into(), + ) + })?, + ); + let num_values = u64::from(u32::from_le_bytes( + entry + .get(4..) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout chunk value-count field is malformed".into(), + ) + })?, + )); + if num_values == 0 { + return Err(Error::invalid_input_source( + "Sparse layout contains an empty value chunk".into(), + )); + } + if num_values > miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value chunk has {} values, exceeding the mini-block limit {}", + num_values, + miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES + ) + .into(), + )); + } + let num_bytes = u64::from(divided_bytes_minus_one) + .checked_add(1) + .and_then(|units| units.checked_mul(MINIBLOCK_ALIGNMENT as u64)) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout value chunk byte size overflows".into(), + ) + })?; + rows_counter = rows_counter.checked_add(num_values).ok_or_else(|| { + Error::invalid_input_source("Sparse layout visible item count overflows".into()) + })?; + chunk_meta.push(ChunkMeta { + num_values, + chunk_size_bytes: num_bytes, + offset_bytes, + }); + offset_bytes = offset_bytes.checked_add(num_bytes).ok_or_else(|| { + Error::invalid_input_source("Sparse layout value chunk byte range overflows".into()) + })?; + } + if rows_counter != self.num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse layout visible item count mismatch: metadata has {}, layout has {}", + rows_counter, self.num_visible_items + ) + .into(), + )); + } + if offset_bytes != value_buf_end { + return Err(Error::invalid_input_source( + format!( + "Sparse layout chunk metadata describes {} value bytes, but value buffer has {} bytes", + offset_bytes - value_buf_position, + value_buf_size + ) + .into(), + )); + } + Ok(chunk_meta) + } + + fn validate_general_buffer_header( + general: &pb21::General, + data: &[u8], + label: &str, + ) -> Result<()> { + let compression = general.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} general compression is missing config").into(), + ) + })?; + Self::validate_buffer_compression(Some(compression), label)?; + let values = Self::require_encoding(&general.values, label)?; + if Self::encoding_contains_general(values) { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} contains nested general compression, which is unsupported" + ) + .into(), + )); + } + + let scheme = invalid_enum( + pb21::CompressionScheme::try_from(compression.scheme), + "compression scheme", + )?; + match scheme { + pb21::CompressionScheme::CompressionAlgorithmLz4 => { + data.get(..4).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} LZ4 buffer is missing its length prefix" + ) + .into(), + ) + })?; + } + pb21::CompressionScheme::CompressionAlgorithmZstd => { + data.get(..8).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} Zstd buffer is missing its length prefix" + ) + .into(), + ) + })?; + } + pb21::CompressionScheme::CompressionAlgorithmUnspecified => { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} general compression scheme is unspecified") + .into(), + )); + } + } + Ok(()) + } + + fn validate_general_child_buffer( + encoding: &CompressiveEncoding, + data: &[u8], + label: &str, + ) -> Result<()> { + if let Some(pb21::compressive_encoding::Compression::General(general)) = + encoding.compression.as_ref() + { + Self::validate_general_buffer_header(general, data, label)?; + } + Ok(()) + } + + fn validate_structural_buffer_headers( + encoding: &CompressiveEncoding, + data: &[u8], + label: &str, + ) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::General(general)) => { + Self::validate_general_buffer_header(general, data, label) + } + Some(Compression::Rle(rle)) => { + let values_size = u64::from_le_bytes( + data.get(..8) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} RLE buffer is missing its header" + ) + .into(), + ) + })? + .try_into() + .map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE header is malformed").into(), + ) + })?, + ); + let values_size = usize_from_u64(values_size, "RLE values buffer size")?; + let values_end = 8_usize.checked_add(values_size).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE values range overflows").into(), + ) + })?; + let values_data = data.get(8..values_end).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE values buffer is truncated").into(), + ) + })?; + let lengths_data = data.get(values_end..).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} RLE run-length buffer is missing") + .into(), + ) + })?; + Self::validate_general_child_buffer( + Self::require_encoding(&rle.values, "RLE values")?, + values_data, + "RLE values", + )?; + Self::validate_general_child_buffer( + Self::require_encoding(&rle.run_lengths, "RLE run lengths")?, + lengths_data, + "RLE run lengths", + ) + } + _ => Ok(()), + } + } + + fn decode_u64_values( + decompressor: &dyn BlockDecompressor, + encoding: &CompressiveEncoding, + data: Bytes, + num_values: u64, + label: &str, + ) -> Result> { + Self::validate_structural_buffer_headers(encoding, &data, label)?; + let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + decompressor.decompress(Some(LanceBuffer::from_bytes(data, 1)), num_values) + })) + .map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} decompression panicked").into(), + ) + })? + .map_err(|error| { + Error::invalid_input_source( + format!("Sparse structural {label} decompression failed: {error}").into(), + ) + })?; + let fixed = decoded.as_fixed_width().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} did not decode to fixed width data").into(), + ) + })?; + if fixed.bits_per_value != 64 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded to {} bits per value, expected 64", + fixed.bits_per_value + ) + .into(), + )); + } + if fixed.num_values != num_values { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded {} values, expected {}", + fixed.num_values, num_values + ) + .into(), + )); + } + let num_values_usize = usize::try_from(num_values).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} value count exceeds usize::MAX").into(), + ) + })?; + let expected_len = num_values_usize + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} decoded byte length overflows").into(), + ) + })?; + if fixed.data.len() != expected_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded {} bytes, expected {}", + fixed.data.len(), + expected_len + ) + .into(), + )); + } + let values = fixed.data.borrow_to_typed_slice::(); + if values.len() != num_values_usize { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} decoded {} u64 values, expected {}", + values.len(), + num_values + ) + .into(), + )); + } + Ok(values.to_vec()) + } + + fn decode_explicit_positions( + decompressor: &Arc, + encoding: &CompressiveEncoding, + data: Bytes, + num_positions: u64, + num_slots: u64, + label: &str, + ) -> Result { + let deltas = + Self::decode_u64_values(decompressor.as_ref(), encoding, data, num_positions, label)?; + let mut positions = Vec::with_capacity(deltas.len()); + let mut current = 0_u64; + for (idx, delta) in deltas.into_iter().enumerate() { + if idx == 0 { + current = delta; + } else { + if delta == 0 { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} positions must be strictly increasing") + .into(), + )); + } + current = current.checked_add(delta).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} position overflow").into(), + ) + })?; + } + if current >= num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} position {} is outside layer with {} slots", + current, num_slots + ) + .into(), + )); + } + positions.push(current); + } + SparsePositionSet::from_positions(positions, num_slots, label) + } + + fn decode_position_set( + decoder: &SparsePositionSetDecoder, + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + match decoder { + SparsePositionSetDecoder::Empty => Ok(SparsePositionSet::empty()), + SparsePositionSetDecoder::All { len } => Ok(SparsePositionSet::all(*len)), + SparsePositionSetDecoder::Range { start, len } => { + Ok(SparsePositionSet::range(*start, *len)) + } + SparsePositionSetDecoder::Explicit { + decompressor, + encoding, + count, + domain_len, + } => Self::decode_explicit_positions( + decompressor, + encoding, + Self::next_structural_buffer(buffers, label)?, + *count, + *domain_len, + label, + ), + } + } + + fn decode_validity_set( + decoder: &SparseValiditySetDecoder, + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + Ok(SparseValiditySet { + meaning: decoder.meaning, + positions: Self::decode_position_set(&decoder.positions, buffers, label)?, + }) + } + + fn decode_count_set( + decoder: &SparseCountSetDecoder, + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + match decoder { + SparseCountSetDecoder::Empty => Ok(SparseCountSet::Empty), + SparseCountSetDecoder::Constant { value, len } => { + Ok(SparseCountSet::constant(*value, *len)) + } + SparseCountSetDecoder::Explicit { + decompressor, + encoding, + count, + } => { + let counts = Self::decode_u64_values( + decompressor.as_ref(), + encoding, + Self::next_structural_buffer(buffers, label)?, + *count, + label, + )?; + SparseCountSet::from_counts(counts) + } + } + } + + fn next_structural_buffer( + buffers: &mut impl Iterator, + label: &str, + ) -> Result { + buffers.next().ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} is missing its buffer").into(), + ) + }) + } + + fn decode_layer( + layer: &SparseLayerDecompressors, + buffers: &mut impl Iterator, + ) -> Result { + Ok(match layer { + SparseLayerDecompressors::Validity { + num_slots, + validity, + } => SparseStructuralLayerPlan::Validity { + num_slots: *num_slots, + validity: Self::decode_validity_set(validity, buffers, "validity positions")?, + }, + SparseLayerDecompressors::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } => { + let non_empty_positions = Self::decode_position_set( + non_empty_positions, + buffers, + "list non-empty positions", + )?; + let counts = Self::decode_count_set(counts, buffers, "list counts")?; + let validity = + Self::decode_validity_set(validity, buffers, "list validity positions")?; + let actual_child_slots = counts.sum()?; + if actual_child_slots != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match declared child slots {}", + actual_child_slots, num_child_slots + ) + .into(), + )); + } + SparseStructuralLayerPlan::List { + num_slots: *num_slots, + num_child_slots: *num_child_slots, + non_empty_positions, + counts, + validity, + } + } + SparseLayerDecompressors::FixedSizeList { + num_slots, + num_child_slots, + dimension, + validity, + } => { + let expected_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + num_slots, dimension + ) + .into(), + ) + })?; + if expected_child_slots != *num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count {} does not match slots {} * dimension {}", + num_child_slots, num_slots, dimension + ) + .into(), + )); + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots: *num_slots, + dimension: *dimension, + validity: Self::decode_validity_set( + validity, + buffers, + "fixed-size-list validity positions", + )?, + } + } + }) + } + + fn lookup_value_chunks(&self, chunk_indices: &[usize]) -> Result> { + let page_meta = self.page_meta.as_ref().ok_or_else(|| { + Error::internal("Sparse page scheduler has not been initialized".to_string()) + })?; + chunk_indices + .iter() + .map(|&chunk_idx| { + let chunk_meta = page_meta.chunk_meta.get(chunk_idx).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse layout missing value chunk metadata for chunk {chunk_idx}") + .into(), + ) + })?; + let bytes_start = chunk_meta.offset_bytes; + let bytes_end = bytes_start + .checked_add(chunk_meta.chunk_size_bytes) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse layout value chunk {} byte range overflows", + chunk_idx + ) + .into(), + ) + })?; + Ok(LoadedChunk { + byte_range: bytes_start..bytes_end, + items_in_chunk: chunk_meta.num_values, + chunk_idx, + data: LanceBuffer::empty(), + }) + }) + .collect() + } + + fn value_chunk_index(chunk_value_offsets: &[u64], value: u64) -> Result { + let total_values = chunk_value_offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout has no value chunk offsets".into()) + })?; + if chunk_value_offsets.len() < 2 || value >= total_values { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value index {} is outside {} visible items", + value, total_values + ) + .into(), + )); + } + chunk_value_offsets + .partition_point(|&offset| offset <= value) + .checked_sub(1) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse layout value index {value} is before the first chunk").into(), + ) + }) + } + + fn value_chunk_range( + chunk_value_offsets: &[u64], + value_range: Range, + ) -> Result> { + if value_range.is_empty() { + return Ok(0..0); + } + let total_values = chunk_value_offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse layout has no value chunk offsets".into()) + })?; + if value_range.start > value_range.end || value_range.end > total_values { + return Err(Error::invalid_input_source( + format!( + "Sparse layout value range {}..{} is outside {} visible items", + value_range.start, value_range.end, total_values + ) + .into(), + )); + } + let start = Self::value_chunk_index(chunk_value_offsets, value_range.start)?; + let end = chunk_value_offsets + .partition_point(|&offset| offset < value_range.end) + .max(start + 1); + Ok(start..end) + } +} + +impl StructuralPageScheduler for SparseStructuralScheduler { + fn initialize<'a>( + &'a mut self, + io: &Arc, + ) -> BoxFuture<'a, Result>> { + let (meta_buf_position, meta_buf_size) = match self.metadata_buffer() { + Ok(buffer) => buffer, + Err(err) => return std::future::ready(Err(err)).boxed(), + }; + let required_ranges = match (|| -> Result>> { + let mut required_ranges = Vec::new(); + required_ranges.push(Self::checked_buffer_range( + meta_buf_position, + meta_buf_size, + "metadata", + )?); + for (position, size) in self.buffer_offsets_and_sizes.iter().skip(2) { + required_ranges.push(Self::checked_buffer_range(*position, *size, "structural")?); + } + Ok(required_ranges) + })() { + Ok(ranges) => ranges, + Err(err) => return std::future::ready(Err(err)).boxed(), + }; + let io_req = io.submit_request(required_ranges, 0); + + async move { + let mut buffers = io_req.await?.into_iter(); + let meta_bytes = buffers.next().ok_or_else(|| { + Error::invalid_input_source("Sparse layout is missing chunk metadata buffer".into()) + })?; + + let chunk_meta = self.parse_chunk_meta(meta_bytes)?; + let mut chunk_value_offsets = Vec::with_capacity(chunk_meta.len() + 1); + let mut value_offset = 0_u64; + chunk_value_offsets.push(value_offset); + for chunk in &chunk_meta { + value_offset = value_offset.checked_add(chunk.num_values).ok_or_else(|| { + Error::invalid_input_source( + "Sparse layout visible item offset overflows".into(), + ) + })?; + chunk_value_offsets.push(value_offset); + } + + let layers = self + .layer_decompressors + .iter() + .map(|layer| Self::decode_layer(layer, &mut buffers)) + .collect::>>()?; + let plan = SparseStructuralPlan { + layers, + num_items: self.num_items, + num_visible_items: self.num_visible_items, + }; + plan.validate(self.row_domain)?; + if buffers.next().is_some() { + return Err(Error::invalid_input_source( + "Sparse layout has unused structural buffers".into(), + )); + } + + let page_meta = Arc::new(SparseStructuralCacheableState { + chunk_meta, + chunk_value_offsets: chunk_value_offsets.into(), + plan, + row_domain: self.row_domain, + }); + self.page_meta = Some(page_meta.clone()); + Ok(page_meta as Arc) + } + .boxed() + } + + fn load(&mut self, data: &Arc) { + self.page_meta = data + .clone() + .as_arc_any() + .downcast::() + .ok(); + } + + fn schedule_ranges( + &self, + ranges: &[Range], + io: &Arc, + ) -> Result> { + let page_meta = self.page_meta.as_ref().ok_or_else(|| { + Error::internal("Sparse page scheduler has not been initialized".to_string()) + })?; + let encoded_row_domain = page_meta + .row_domain + .checked_mul(self.row_scale) + .ok_or_else(|| { + Error::invalid_input_source("Sparse structural encoded row domain overflows".into()) + })?; + let num_rows = validate_slice_ranges(ranges, encoded_row_domain, "row")?; + let ranges = ranges + .iter() + .map(|range| { + if !range.start.is_multiple_of(self.row_scale) + || !range.end.is_multiple_of(self.row_scale) + { + return Err(Error::invalid_input_source( + format!( + "Sparse structural encoded row range {}..{} is not aligned to fixed-size-list scale {}", + range.start, range.end, self.row_scale + ) + .into(), + )); + } + Ok((range.start / self.row_scale)..(range.end / self.row_scale)) + }) + .collect::>>()?; + + let mut chunks_needed = Vec::new(); + let selection = slice_sparse_plan(&page_meta.plan, &ranges, page_meta.row_domain)?; + for value_range in &selection.leaf_ranges { + chunks_needed.extend(Self::value_chunk_range( + &page_meta.chunk_value_offsets, + value_range.clone(), + )?); + } + chunks_needed.sort_unstable(); + chunks_needed.dedup(); + + let mut loaded_chunks = self.lookup_value_chunks(&chunks_needed)?; + let chunk_ranges = loaded_chunks + .iter() + .map(|chunk| chunk.byte_range.clone()) + .collect::>(); + let loaded_chunk_data = io.submit_request(chunk_ranges, self.priority); + let ranges = VecDeque::from(ranges); + let value_decompressor = self.value_decompressor.clone(); + let value_encoding = self.value_encoding.clone(); + let data_type = self.data_type.clone(); + let page_meta = page_meta.clone(); + let num_buffers = self.num_buffers; + let has_large_chunk = self.has_large_chunk; + let row_scale = self.row_scale; + + let res = async move { + let loaded_chunk_data = loaded_chunk_data.await?; + for (loaded_chunk, chunk_data) in loaded_chunks.iter_mut().zip(loaded_chunk_data) { + loaded_chunk.data = LanceBuffer::from_bytes(chunk_data, 1); + } + + Ok(Box::new(SparseStructuralDecoder { + value_decompressor, + value_encoding, + data_type, + page_meta, + loaded_chunks: Arc::new(loaded_chunks), + ranges, + offset_in_current_range: 0, + num_rows, + row_scale, + num_buffers, + has_large_chunk, + }) as Box) + } + .boxed(); + Ok(vec![PageLoadTask { + decoder_fut: res, + num_rows, + }]) + } +} + +#[derive(Debug)] +struct SparseStructuralDecoder { + value_decompressor: Arc, + value_encoding: CompressiveEncoding, + data_type: DataType, + page_meta: Arc, + loaded_chunks: Arc>, + ranges: VecDeque>, + offset_in_current_range: u64, + num_rows: u64, + row_scale: u64, + num_buffers: u64, + has_large_chunk: bool, +} + +impl SparseStructuralDecoder { + fn drain_ranges(&mut self, mut rows_desired: u64) -> Result>> { + if !rows_desired.is_multiple_of(self.row_scale) { + return Err(Error::invalid_input_source( + format!( + "Sparse page decoder drain of {} encoded rows is not aligned to fixed-size-list scale {}", + rows_desired, self.row_scale + ) + .into(), + )); + } + rows_desired /= self.row_scale; + let mut ranges = Vec::new(); + while rows_desired > 0 { + let range = self.ranges.front().ok_or_else(|| { + Error::invalid_input_source( + "Sparse page decoder was asked to drain more rows than were scheduled".into(), + ) + })?; + let start = range + .start + .checked_add(self.offset_in_current_range) + .ok_or_else(|| { + Error::invalid_input_source("Sparse page decoder row offset overflows".into()) + })?; + let rows_available = range.end.checked_sub(start).ok_or_else(|| { + Error::invalid_input_source( + "Sparse page decoder range offset exceeds the scheduled range".into(), + ) + })?; + let rows_to_take = rows_available.min(rows_desired); + let end = start.checked_add(rows_to_take).ok_or_else(|| { + Error::invalid_input_source("Sparse page decoder row range overflows".into()) + })?; + ranges.push(start..end); + rows_desired -= rows_to_take; + self.offset_in_current_range += rows_to_take; + if self.offset_in_current_range == range.end - range.start { + self.offset_in_current_range = 0; + self.ranges.pop_front(); + } + } + Ok(ranges) + } +} + +impl StructuralPageDecoder for SparseStructuralDecoder { + fn drain(&mut self, num_rows: u64) -> Result> { + Ok(Box::new(DecodeSparseStructuralTask { + row_ranges: self.drain_ranges(num_rows)?, + value_decompressor: self.value_decompressor.clone(), + value_encoding: self.value_encoding.clone(), + data_type: self.data_type.clone(), + page_meta: self.page_meta.clone(), + loaded_chunks: self.loaded_chunks.clone(), + num_buffers: self.num_buffers, + has_large_chunk: self.has_large_chunk, + })) + } + + fn num_rows(&self) -> u64 { + self.num_rows + } +} + +#[derive(Debug)] +struct DecodeSparseStructuralTask { + row_ranges: Vec>, + value_decompressor: Arc, + value_encoding: CompressiveEncoding, + data_type: DataType, + page_meta: Arc, + loaded_chunks: Arc>, + num_buffers: u64, + has_large_chunk: bool, +} + +impl DecodeSparseStructuralTask { + fn read_chunk_size( + buf: &[u8], + offset: &mut usize, + width: usize, + chunk_idx: usize, + ) -> Result { + let end = offset.checked_add(width).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} size header overflows").into(), + ) + })?; + let bytes = buf.get(*offset..end).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} has a truncated size header") + .into(), + ) + })?; + let size = match width { + 2 => u32::from(u16::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} has a malformed u16 size") + .into(), + ) + })?)), + 4 => u32::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural value chunk {chunk_idx} has a malformed u32 size") + .into(), + ) + })?), + _ => { + return Err(Error::internal(format!( + "Unsupported sparse value chunk size width {width}" + ))); + } + }; + *offset = end; + Ok(size) + } + + fn expected_fixed_bytes(num_values: u64, bits_per_value: u64, label: &str) -> Result { + let bits = num_values.checked_mul(bits_per_value).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} decoded bit length overflows").into(), + ) + })?; + usize_from_u64(bits.div_ceil(8), label) + } + + fn validate_fixed_buffer( + buffer: &LanceBuffer, + num_values: u64, + bits_per_value: u64, + label: &str, + ) -> Result<()> { + let expected = Self::expected_fixed_bytes(num_values, bits_per_value, label)?; + if buffer.len() != expected { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} buffer has {} bytes, expected {}", + buffer.len(), + expected + ) + .into(), + )); + } + Ok(()) + } + + fn validate_variable_buffer( + buffer: &LanceBuffer, + num_values: u64, + bits_per_offset: u64, + ) -> Result<()> { + let width = usize_from_u64(bits_per_offset / 8, "variable offset width")?; + let offset_count = num_values.checked_add(1).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset count overflows".into()) + })?; + let table_len = usize_from_u64(offset_count, "variable offset count")? + .checked_mul(width) + .ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset table size overflows".into()) + })?; + if buffer.len() < table_len { + return Err(Error::invalid_input_source( + format!( + "Sparse variable buffer has {} bytes, smaller than its {}-byte offset table", + buffer.len(), + table_len + ) + .into(), + )); + } + let mut previous = None; + for index in 0..usize_from_u64(offset_count, "variable offset count")? { + let start = index.checked_mul(width).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset index overflows".into()) + })?; + let end = start.checked_add(width).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset range overflows".into()) + })?; + let bytes = buffer.as_ref().get(start..end).ok_or_else(|| { + Error::invalid_input_source("Sparse variable offset table is truncated".into()) + })?; + let offset = match width { + 4 => u64::from(u32::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source("Sparse variable u32 offset is malformed".into()) + })?)), + 8 => u64::from_le_bytes(bytes.try_into().map_err(|_| { + Error::invalid_input_source("Sparse variable u64 offset is malformed".into()) + })?), + _ => { + return Err(Error::invalid_input_source( + format!( + "Sparse variable offset width {} is unsupported", + bits_per_offset + ) + .into(), + )); + } + }; + if offset < table_len as u64 || offset > buffer.len() as u64 { + return Err(Error::invalid_input_source( + format!( + "Sparse variable offset {} is outside payload range {}..{}", + offset, + table_len, + buffer.len() + ) + .into(), + )); + } + if previous.is_some_and(|previous| offset < previous) { + return Err(Error::invalid_input_source( + "Sparse variable offsets are not monotonically increasing".into(), + )); + } + previous = Some(offset); + } + Ok(()) + } + + fn validate_fsl_buffers( + fsl: &pb21::FixedSizeList, + buffers: &[LanceBuffer], + num_values: u64, + buffer_index: &mut usize, + ) -> Result<()> { + use pb21::compressive_encoding::Compression; + + let child_values = num_values.checked_mul(fsl.items_per_value).ok_or_else(|| { + Error::invalid_input_source("Sparse fixed-size-list value count overflows".into()) + })?; + if fsl.has_validity { + let validity = buffers.get(*buffer_index).ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list value validity buffer is missing".into(), + ) + })?; + Self::validate_fixed_buffer(validity, child_values, 1, "fixed-size-list validity")?; + *buffer_index = buffer_index.checked_add(1).ok_or_else(|| { + Error::invalid_input_source("Sparse fixed-size-list buffer index overflows".into()) + })?; + } + let values = fsl.values.as_deref().ok_or_else(|| { + Error::invalid_input_source("Sparse fixed-size-list value encoding is missing".into()) + })?; + match values.compression.as_ref() { + Some(Compression::FixedSizeList(inner)) => { + Self::validate_fsl_buffers(inner, buffers, child_values, buffer_index) + } + Some(Compression::Flat(flat)) => { + let values = buffers.get(*buffer_index).ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list leaf value buffer is missing".into(), + ) + })?; + Self::validate_fixed_buffer( + values, + child_values, + flat.bits_per_value, + "fixed-size-list leaf values", + )?; + *buffer_index = buffer_index.checked_add(1).ok_or_else(|| { + Error::invalid_input_source( + "Sparse fixed-size-list buffer index overflows".into(), + ) + })?; + Ok(()) + } + _ => Err(Error::invalid_input_source( + "Sparse fixed-size-list value encoding is malformed".into(), + )), + } + } + + fn validate_value_buffers(&self, buffers: &[LanceBuffer], num_values: u64) -> Result<()> { + use pb21::compressive_encoding::Compression; + + let compression = self.value_encoding.compression.as_ref().ok_or_else(|| { + Error::invalid_input_source("Sparse value compression is missing".into()) + })?; + match compression { + Compression::Flat(flat) => Self::validate_fixed_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source("Sparse flat value buffer is missing".into()) + })?, + num_values, + flat.bits_per_value, + "flat values", + ), + Compression::InlineBitpacking(bitpacking) => { + let buffer = buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked value buffer is missing".into(), + ) + })?; + if num_values > 1024 { + return Err(Error::invalid_input_source( + format!( + "Sparse inline-bitpacked chunk has {} values, exceeding 1024", + num_values + ) + .into(), + )); + } + let word_bytes = usize_from_u64( + bitpacking.uncompressed_bits_per_value / 8, + "inline bitpacking word width", + )?; + let header = buffer.as_ref().get(..word_bytes).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked buffer is missing its bit-width header".into(), + ) + })?; + let bit_width = + header + .iter() + .enumerate() + .try_fold(0_u64, |value, (idx, byte)| { + let shift = u32::try_from(idx.checked_mul(8).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline bit-width shift overflows".into(), + ) + })?) + .map_err(|_| { + Error::invalid_input_source( + "Sparse inline bit-width shift exceeds u32".into(), + ) + })?; + Ok::<_, Error>(value | (u64::from(*byte) << shift)) + })?; + if bit_width > bitpacking.uncompressed_bits_per_value { + return Err(Error::invalid_input_source( + format!( + "Sparse inline bit width {} exceeds uncompressed width {}", + bit_width, bitpacking.uncompressed_bits_per_value + ) + .into(), + )); + } + let payload_bytes = usize_from_u64( + bit_width.checked_mul(1024).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked payload size overflows".into(), + ) + })? / 8, + "inline-bitpacked payload size", + )?; + let expected = word_bytes.checked_add(payload_bytes).ok_or_else(|| { + Error::invalid_input_source( + "Sparse inline-bitpacked buffer size overflows".into(), + ) + })?; + if buffer.len() != expected { + return Err(Error::invalid_input_source( + format!( + "Sparse inline-bitpacked buffer has {} bytes, expected {}", + buffer.len(), + expected + ) + .into(), + )); + } + Ok(()) + } + Compression::Variable(variable) => { + let offsets = variable + .offsets + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Flat(flat) => Some(flat), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse variable offset encoding is malformed".into(), + ) + })?; + Self::validate_variable_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse variable value buffer is missing".into(), + ) + })?, + num_values, + offsets.bits_per_value, + ) + } + Compression::Fsst(fsst) => { + let variable = fsst + .values + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Variable(variable) => Some(variable), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse FSST value encoding is malformed".into(), + ) + })?; + let offsets = variable + .offsets + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Flat(flat) => Some(flat), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse FSST offset encoding is malformed".into(), + ) + })?; + Self::validate_variable_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source("Sparse FSST value buffer is missing".into()) + })?, + num_values, + offsets.bits_per_value, + ) + } + Compression::ByteStreamSplit(split) => { + let bits = split + .values + .as_deref() + .and_then(|encoding| encoding.compression.as_ref()) + .and_then(|compression| match compression { + Compression::Flat(flat) => Some(flat.bits_per_value), + _ => None, + }) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse byte-stream-split encoding is malformed".into(), + ) + })?; + Self::validate_fixed_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse byte-stream-split buffer is missing".into(), + ) + })?, + num_values, + bits, + "byte-stream-split values", + ) + } + Compression::FixedSizeList(fsl) => { + let mut buffer_index = 0; + Self::validate_fsl_buffers(fsl, buffers, num_values, &mut buffer_index)?; + if buffer_index != buffers.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse fixed-size-list descriptor consumed {} of {} buffers", + buffer_index, + buffers.len() + ) + .into(), + )); + } + Ok(()) + } + Compression::PackedStruct(packed) => { + let bits = packed.bits_per_value.iter().try_fold(0_u64, |sum, bits| { + sum.checked_add(*bits).ok_or_else(|| { + Error::invalid_input_source( + "Sparse packed-struct bit width sum overflows".into(), + ) + }) + })?; + Self::validate_fixed_buffer( + buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse packed-struct value buffer is missing".into(), + ) + })?, + num_values, + bits, + "packed-struct values", + ) + } + Compression::Rle(rle) => { + if buffers.len() != 2 { + return Err(Error::invalid_input_source( + format!( + "Sparse RLE value chunk has {} buffers, expected 2", + buffers.len() + ) + .into(), + )); + } + SparseStructuralScheduler::validate_general_child_buffer( + SparseStructuralScheduler::require_encoding(&rle.values, "RLE values")?, + buffers + .first() + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse RLE value buffer is missing after count validation".into(), + ) + })? + .as_ref(), + "value chunk RLE values", + )?; + SparseStructuralScheduler::validate_general_child_buffer( + SparseStructuralScheduler::require_encoding( + &rle.run_lengths, + "RLE run lengths", + )?, + buffers + .get(1) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse RLE run-length buffer is missing after count validation" + .into(), + ) + })? + .as_ref(), + "value chunk RLE run lengths", + ) + } + Compression::General(general) => { + let buffer = buffers.first().ok_or_else(|| { + Error::invalid_input_source( + "Sparse general-compressed value chunk is missing its first buffer".into(), + ) + })?; + SparseStructuralScheduler::validate_general_buffer_header( + general, + buffer.as_ref(), + "value chunk", + ) + } + _ => Err(Error::invalid_input_source( + "Sparse value chunk uses an unsupported compression descriptor".into(), + )), + } + } + + fn loaded_chunk(&self, chunk_idx: usize) -> Result<&LoadedChunk> { + let index = self + .loaded_chunks + .binary_search_by_key(&chunk_idx, |chunk| chunk.chunk_idx) + .map_err(|_| { + Error::internal(format!( + "Sparse structural decode missing loaded value chunk {}", + chunk_idx + )) + })?; + self.loaded_chunks.get(index).ok_or_else(|| { + Error::internal(format!( + "Sparse structural loaded chunk index {} is missing", + index + )) + }) + } + + fn decode_value_chunk(&self, chunk: &LoadedChunk) -> Result { + let buf = &chunk.data; + if buf.len() < 2 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} is too small for its header: {} bytes", + chunk.chunk_idx, + buf.len() + ) + .into(), + )); + } + let num_levels = u16::from_le_bytes( + buf.as_ref() + .get(..2) + .and_then(|bytes| bytes.try_into().ok()) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} has a malformed level header", + chunk.chunk_idx + ) + .into(), + ) + })?, + ); + let mut offset: usize = 2; + if num_levels != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk unexpectedly contains {} rep/def levels", + num_levels + ) + .into(), + )); + } + + let size_width = if self.has_large_chunk { 4 } else { 2 }; + let num_buffers = usize::try_from(self.num_buffers).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk has too many buffers: {}", + self.num_buffers + ) + .into(), + ) + })?; + let sizes_len = num_buffers.checked_mul(size_width).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural value chunk buffer-size header overflows".into(), + ) + })?; + let header_len = offset.checked_add(sizes_len).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural value chunk header length overflows".into(), + ) + })?; + if buf.len() < header_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} is too small for {} buffer sizes: {} bytes", + chunk.chunk_idx, + self.num_buffers, + buf.len() + ) + .into(), + )); + } + let buffer_sizes = (0..num_buffers) + .map(|_| Self::read_chunk_size(buf, &mut offset, size_width, chunk.chunk_idx)) + .collect::>>()?; + + offset = offset + .checked_add(pad_bytes::(offset)) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} padded header overflows", + chunk.chunk_idx + ) + .into(), + ) + })?; + if offset > buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} is missing padding after its header", + chunk.chunk_idx + ) + .into(), + )); + } + let buffers = buffer_sizes + .into_iter() + .map(|buf_size| { + let buf_size = buf_size as usize; + let end = offset.checked_add(buf_size).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} buffer size overflows", + chunk.chunk_idx + ) + .into(), + ) + })?; + if end > buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} buffer extends past chunk end", + chunk.chunk_idx + ) + .into(), + )); + } + let buffer = buf.slice_with_length(offset, buf_size); + offset = end; + offset = offset + .checked_add(pad_bytes::(offset)) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} padded buffer range overflows", + chunk.chunk_idx + ) + .into(), + ) + })?; + if offset > buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} padding extends past chunk end", + chunk.chunk_idx + ) + .into(), + )); + } + Ok(buffer) + }) + .collect::>>()?; + + if offset != buf.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} consumed {} of {} bytes", + chunk.chunk_idx, + offset, + buf.len() + ) + .into(), + )); + } + + self.validate_value_buffers(&buffers, chunk.items_in_chunk)?; + + let decoded = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.value_decompressor + .decompress(buffers, chunk.items_in_chunk) + })) + .map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} decompression panicked", + chunk.chunk_idx + ) + .into(), + ) + })? + .map_err(|error| { + Error::invalid_input_source( + format!( + "Sparse structural value chunk {} decompression failed: {error}", + chunk.chunk_idx + ) + .into(), + ) + })?; + if decoded.num_values() != chunk.items_in_chunk { + return Err(Error::invalid_input_source( + format!( + "Sparse structural value chunk {} decoded {} values, expected {}", + chunk.chunk_idx, + decoded.num_values(), + chunk.items_in_chunk + ) + .into(), + )); + } + Ok(decoded) + } + + fn append_value_range( + &self, + value_range: Range, + data_builder: &mut DataBlockBuilder, + chunk_cache: &mut Option<(usize, DataBlock)>, + ) -> Result<()> { + let mut value_start = value_range.start; + while value_start < value_range.end { + let chunk_idx = SparseStructuralScheduler::value_chunk_index( + &self.page_meta.chunk_value_offsets, + value_start, + )?; + let chunk_value_start = *self + .page_meta + .chunk_value_offsets + .get(chunk_idx) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse value chunk {} has no start offset", chunk_idx).into(), + ) + })?; + let chunk_value_end = *self + .page_meta + .chunk_value_offsets + .get(chunk_idx.checked_add(1).ok_or_else(|| { + Error::invalid_input_source("Sparse value chunk index overflows".into()) + })?) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse value chunk {} has no end offset", chunk_idx).into(), + ) + })?; + let take_end = value_range.end.min(chunk_value_end); + if value_start < chunk_value_start || take_end <= value_start { + return Err(Error::invalid_input_source( + format!( + "Sparse value range {}..{} does not make progress in chunk {} covering {}..{}", + value_range.start, + value_range.end, + chunk_idx, + chunk_value_start, + chunk_value_end + ) + .into(), + )); + } + + if !matches!(chunk_cache, Some((cached_idx, _)) if *cached_idx == chunk_idx) { + let chunk = self.loaded_chunk(chunk_idx)?; + *chunk_cache = Some((chunk_idx, self.decode_value_chunk(chunk)?)); + } + let values = &chunk_cache + .as_ref() + .ok_or_else(|| Error::internal("Sparse structural chunk cache is empty"))? + .1; + data_builder.append( + values, + value_start - chunk_value_start..take_end - chunk_value_start, + )?; + value_start = take_end; + } + Ok(()) + } + + fn decode_checked(self) -> Result { + let selection = slice_sparse_plan( + &self.page_meta.plan, + &self.row_ranges, + self.page_meta.row_domain, + )?; + let estimated_size_bytes = self + .loaded_chunks + .iter() + .map(|chunk| chunk.data.len()) + .try_fold(0_usize, |total, len| total.checked_add(len)) + .and_then(|total| total.checked_mul(2)) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural decode size estimate overflows".into(), + ) + })?; + let mut data_builder = DataBlockBuilder::with_capacity_estimate( + u64::try_from(estimated_size_bytes).map_err(|_| { + Error::invalid_input_source("Sparse structural decode size exceeds u64::MAX".into()) + })?, + ); + let mut chunk_cache: Option<(usize, DataBlock)> = None; + let mut appended_values = false; + for value_range in &selection.leaf_ranges { + self.append_value_range(value_range.clone(), &mut data_builder, &mut chunk_cache)?; + appended_values = true; + } + + let data = if appended_values { + data_builder.finish() + } else { + DataBlock::from_array(new_empty_array(&self.data_type)) + }; + let unraveler = RepDefUnraveler::new_sparse(selection.plan); + Ok(DecodedPage { + data, + repdef: unraveler, + }) + } +} + +impl DecodePageTask for DecodeSparseStructuralTask { + fn decode(self: Box) -> Result { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| (*self).decode_checked())) + .map_err(|_| { + Error::invalid_input_source( + "Sparse structural page decoding panicked on malformed input".into(), + ) + })? + } +} + +struct SparseStructuralSelection { + plan: SparseStructuralPlan, + leaf_ranges: Vec>, +} + +struct SparsePositionSelection { + positions: SparsePositionSet, + ordinal_ranges: Vec>, +} + +fn validate_slice_ranges(ranges: &[Range], domain_len: u64, label: &str) -> Result { + let mut total = 0_u64; + for range in ranges { + if range.start > range.end || range.end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} slice {}..{} is outside domain {}", + range.start, range.end, domain_len + ) + .into(), + )); + } + total = total.checked_add(range.end - range.start).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} slice length overflows").into(), + ) + })?; + } + Ok(total) +} + +fn push_coalesced_range(ranges: &mut Vec>, range: Range) { + if range.is_empty() { + return; + } + if let Some(last) = ranges.last_mut() + && last.end == range.start + { + last.end = range.end; + return; + } + ranges.push(range); +} + +fn position_segments_to_set( + segments: Vec>, + output_domain: u64, + label: &str, +) -> Result { + let segments = coalesce_ranges(segments); + if segments.is_empty() { + return Ok(SparsePositionSet::empty()); + } + if segments.len() == 1 { + let segment = segments.first().ok_or_else(|| { + Error::internal("Sparse structural segment unexpectedly missing".to_string()) + })?; + if segment.start == 0 && segment.end == output_domain { + return Ok(SparsePositionSet::all(output_domain)); + } + return Ok(SparsePositionSet::range( + segment.start, + segment.end - segment.start, + )); + } + + let total_len = segments + .iter() + .map(|range| range.end - range.start) + .try_fold(0_u64, |sum, len| { + sum.checked_add(len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} segment length overflows").into(), + ) + }) + })?; + let total_len = usize::try_from(total_len).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} segment length exceeds usize::MAX").into(), + ) + })?; + let mut positions = Vec::with_capacity(total_len); + for segment in segments { + positions.extend(segment); + } + SparsePositionSet::from_positions(positions, output_domain, label) +} + +fn select_position_set( + positions: &SparsePositionSet, + ranges: &[Range], + domain_len: u64, + label: &str, +) -> Result { + let output_domain = validate_slice_ranges(ranges, domain_len, label)?; + let mut segments = Vec::new(); + let mut ordinal_ranges = Vec::new(); + let mut output_base = 0_u64; + + match positions { + SparsePositionSet::Empty => {} + SparsePositionSet::All { len } => { + if *len != domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} all set length {} does not match domain {}", + len, domain_len + ) + .into(), + )); + } + for range in ranges { + let range_len = range.end - range.start; + let output_end = output_base.checked_add(range_len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output domain overflows").into(), + ) + })?; + push_coalesced_range(&mut segments, output_base..output_end); + push_coalesced_range(&mut ordinal_ranges, range.clone()); + output_base = output_end; + } + } + SparsePositionSet::Range { start, len } => { + let source_end = start.checked_add(*len).ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} range overflows").into(), + ) + })?; + if source_end > domain_len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} range {}..{} is outside domain {}", + start, source_end, domain_len + ) + .into(), + )); + } + for range in ranges { + let intersect_start = range.start.max(*start); + let intersect_end = range.end.min(source_end); + if intersect_start < intersect_end { + let out_start = output_base + .checked_add(intersect_start - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output position overflows") + .into(), + ) + })?; + let out_end = output_base + .checked_add(intersect_end - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output position overflows") + .into(), + ) + })?; + push_coalesced_range(&mut segments, out_start..out_end); + push_coalesced_range( + &mut ordinal_ranges, + intersect_start - *start..intersect_end - *start, + ); + } + output_base = output_base + .checked_add(range.end - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output domain overflows").into(), + ) + })?; + } + } + SparsePositionSet::Explicit(source_positions) => { + let mut out_positions = Vec::new(); + for range in ranges { + let idx_start = + source_positions.partition_point(|position| *position < range.start); + let idx_end = source_positions.partition_point(|position| *position < range.end); + if idx_start < idx_end { + let selected_positions = + source_positions.get(idx_start..idx_end).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} explicit position slice is invalid" + ) + .into(), + ) + })?; + for position in selected_positions { + out_positions.push( + output_base + .checked_add(*position - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural {label} output position overflows" + ) + .into(), + ) + })?, + ); + } + push_coalesced_range(&mut ordinal_ranges, idx_start as u64..idx_end as u64); + } + output_base = output_base + .checked_add(range.end - range.start) + .ok_or_else(|| { + Error::invalid_input_source( + format!("Sparse structural {label} output domain overflows").into(), + ) + })?; + } + let positions = SparsePositionSet::from_positions(out_positions, output_domain, label)?; + return Ok(SparsePositionSelection { + positions, + ordinal_ranges, + }); + } + } + + Ok(SparsePositionSelection { + positions: position_segments_to_set(segments, output_domain, label)?, + ordinal_ranges, + }) +} + +fn select_validity_set( + validity: &SparseValiditySet, + ranges: &[Range], + domain_len: u64, + label: &str, +) -> Result { + Ok(SparseValiditySet { + meaning: validity.meaning, + positions: select_position_set(&validity.positions, ranges, domain_len, label)?.positions, + }) +} + +fn offsets_from_counts(counts: &[u64]) -> Result> { + let mut offsets = Vec::with_capacity(counts.len() + 1); + let mut offset = 0_u64; + offsets.push(offset); + for count in counts { + offset = offset.checked_add(*count).ok_or_else(|| { + Error::invalid_input_source("Sparse structural list count offsets overflow".into()) + })?; + offsets.push(offset); + } + Ok(offsets) +} + +fn coalesce_ranges(ranges: Vec>) -> Vec> { + let mut coalesced: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + if range.is_empty() { + continue; + } + if let Some(last) = coalesced.last_mut() + && last.end == range.start + { + last.end = range.end; + continue; + } + coalesced.push(range); + } + coalesced +} + +fn slice_list_layer( + num_slots: u64, + num_child_slots: u64, + non_empty_positions: &SparsePositionSet, + counts: &SparseCountSet, + validity: &SparseValiditySet, + ranges: &[Range], +) -> Result<(SparseStructuralLayerPlan, Vec>)> { + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + "Sparse structural list has mismatched non-empty positions and counts".into(), + )); + } + let count_sum = counts.sum()?; + if count_sum != num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + count_sum, num_child_slots + ) + .into(), + )); + } + + let non_empty_selection = + select_position_set(non_empty_positions, ranges, num_slots, "list non-empty")?; + let out_counts = select_count_set( + counts, + &non_empty_selection.ordinal_ranges, + non_empty_selection.positions.len(), + )?; + let child_ranges = + child_ranges_from_counts(counts, num_child_slots, &non_empty_selection.ordinal_ranges)?; + let out_validity = select_validity_set(validity, ranges, num_slots, "list validity")?; + let out_num_slots = validate_slice_ranges(ranges, num_slots, "list")?; + let out_num_child_slots = out_counts.sum()?; + Ok(( + SparseStructuralLayerPlan::List { + num_slots: out_num_slots, + num_child_slots: out_num_child_slots, + non_empty_positions: non_empty_selection.positions, + counts: out_counts, + validity: out_validity, + }, + child_ranges, + )) +} + +fn select_count_set( + counts: &SparseCountSet, + ordinal_ranges: &[Range], + selected_len: u64, +) -> Result { + match counts { + SparseCountSet::Empty => { + if selected_len != 0 { + return Err(Error::invalid_input_source( + "Sparse structural selected non-empty positions but counts are empty".into(), + )); + } + Ok(SparseCountSet::Empty) + } + SparseCountSet::Constant { value, len } => { + validate_ordinal_ranges(ordinal_ranges, *len, "constant list counts")?; + Ok(SparseCountSet::constant(*value, selected_len)) + } + SparseCountSet::Explicit { + counts: source_counts, + .. + } => { + validate_ordinal_ranges( + ordinal_ranges, + source_counts.len() as u64, + "explicit list counts", + )?; + let selected_len = usize::try_from(selected_len).map_err(|_| { + Error::invalid_input_source( + "Sparse structural selected count length exceeds usize::MAX".into(), + ) + })?; + let mut out_counts = Vec::with_capacity(selected_len); + for range in ordinal_ranges { + let start = usize::try_from(range.start).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + let end = usize::try_from(range.end).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + out_counts.extend_from_slice(source_counts.get(start..end).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural explicit count slice is invalid".into(), + ) + })?); + } + SparseCountSet::from_counts(out_counts) + } + } +} + +fn validate_ordinal_ranges(ranges: &[Range], len: u64, label: &str) -> Result<()> { + for range in ranges { + if range.start > range.end || range.end > len { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} ordinal range {}..{} is outside {} values", + range.start, range.end, len + ) + .into(), + )); + } + } + Ok(()) +} + +fn child_ranges_from_counts( + counts: &SparseCountSet, + num_child_slots: u64, + ordinal_ranges: &[Range], +) -> Result>> { + if ordinal_ranges.is_empty() { + return Ok(Vec::new()); + } + let child_ranges = match counts { + SparseCountSet::Empty => { + return Err(Error::invalid_input_source( + "Sparse structural selected non-empty positions but counts are empty".into(), + )); + } + SparseCountSet::Constant { value, len } => { + let expected_child_slots = value.checked_mul(*len).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list constant count sum overflows child slots".into(), + ) + })?; + if expected_child_slots != num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list constant count sum {} does not match child slots {}", + expected_child_slots, num_child_slots + ) + .into(), + )); + } + validate_ordinal_ranges(ordinal_ranges, *len, "constant list counts")?; + ordinal_ranges + .iter() + .map(|range| { + let start = range.start.checked_mul(*value).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list child range start overflows".into(), + ) + })?; + let end = range.end.checked_mul(*value).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list child range end overflows".into(), + ) + })?; + Ok(start..end) + }) + .collect::>>()? + } + SparseCountSet::Explicit { + counts, + offsets: value_offsets, + } => { + let last_offset = *value_offsets.last().ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list count offsets are unexpectedly empty".into(), + ) + })?; + if last_offset != num_child_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list count sum {} does not match child slots {}", + last_offset, num_child_slots + ) + .into(), + )); + } + validate_ordinal_ranges(ordinal_ranges, counts.len() as u64, "explicit list counts")?; + ordinal_ranges + .iter() + .map(|range| { + let start = usize::try_from(range.start).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + let end = usize::try_from(range.end).map_err(|_| { + Error::invalid_input_source( + "Sparse structural count ordinal exceeds usize::MAX".into(), + ) + })?; + let start_offset = *value_offsets.get(start).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list start offset is missing".into(), + ) + })?; + let end_offset = *value_offsets.get(end).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list end offset is missing".into(), + ) + })?; + Ok(start_offset..end_offset) + }) + .collect::>>()? + } + }; + Ok(coalesce_ranges(child_ranges)) +} + +fn slice_sparse_plan( + plan: &SparseStructuralPlan, + row_ranges: &[Range], + row_domain: u64, +) -> Result { + plan.validate(row_domain)?; + let mut selected_ranges = row_ranges.to_vec(); + let mut selected_domain = row_domain; + let mut sliced_layers = Vec::with_capacity(plan.layers.len()); + + for layer in &plan.layers { + match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } => { + if selected_domain != *num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural validity slice domain {} does not match {} slots", + selected_domain, num_slots + ) + .into(), + )); + } + let out_num_slots = + validate_slice_ranges(&selected_ranges, *num_slots, "validity")?; + sliced_layers.push(SparseStructuralLayerPlan::Validity { + num_slots: out_num_slots, + validity: select_validity_set( + validity, + &selected_ranges, + *num_slots, + "validity", + )?, + }); + } + SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + .. + } => { + if selected_domain != *num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list slice domain {} does not match {} slots", + selected_domain, num_slots + ) + .into(), + )); + } + let (sliced_layer, child_ranges) = slice_list_layer( + *num_slots, + *num_child_slots, + non_empty_positions, + counts, + validity, + &selected_ranges, + )?; + sliced_layers.push(sliced_layer); + selected_ranges = child_ranges; + selected_domain = *num_child_slots; + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots, + dimension, + validity, + } => { + if selected_domain != *num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list slice domain {} does not match {} slots", + selected_domain, num_slots + ) + .into(), + )); + } + let num_child_slots = num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={}, dimension={}", + num_slots, dimension + ) + .into(), + ) + })?; + let out_num_slots = + validate_slice_ranges(&selected_ranges, *num_slots, "fixed-size-list")?; + let child_ranges = selected_ranges + .iter() + .map(|range| { + let start = range.start.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list child range start overflows" + .into(), + ) + })?; + let end = range.end.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural fixed-size-list child range end overflows" + .into(), + ) + })?; + Ok(start..end) + }) + .collect::>>()?; + sliced_layers.push(SparseStructuralLayerPlan::FixedSizeList { + num_slots: out_num_slots, + dimension: *dimension, + validity: select_validity_set( + validity, + &selected_ranges, + *num_slots, + "fixed-size-list validity", + )?, + }); + selected_ranges = child_ranges; + selected_domain = num_child_slots; + } + } + } + + if selected_domain != plan.num_visible_items { + return Err(Error::invalid_input_source( + format!( + "Sparse structural selected terminal domain {} does not match {} visible items", + selected_domain, plan.num_visible_items + ) + .into(), + )); + } + let num_visible_items = + validate_slice_ranges(&selected_ranges, plan.num_visible_items, "visible value")?; + let num_items = SparseStructuralPlan::expected_num_items(&sliced_layers, num_visible_items)?; + Ok(SparseStructuralSelection { + plan: SparseStructuralPlan { + layers: sliced_layers, + num_items, + num_visible_items, + }, + leaf_ranges: coalesce_ranges(selected_ranges), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use crate::{ + compression::DefaultDecompressionStrategy, + encodings::physical::block::{CompressionConfig, CompressionScheme}, + testing::SimulatedScheduler, + }; + + use super::*; + + fn position_set( + positions: pb21::sparse_position_set::Positions, + num_positions: u64, + ) -> Option { + Some(pb21::SparsePositionSet { + positions: Some(positions), + num_positions, + }) + } + + fn position_empty() -> Option { + position_set( + pb21::sparse_position_set::Positions::Empty(pb21::SparsePositionEmpty {}), + 0, + ) + } + + fn position_all(num_positions: u64) -> Option { + position_set( + pb21::sparse_position_set::Positions::All(pb21::SparsePositionAll {}), + num_positions, + ) + } + + fn position_explicit(num_positions: u64) -> Option { + position_set( + pb21::sparse_position_set::Positions::Explicit(ProtobufUtils21::flat(64, None)), + num_positions, + ) + } + + fn general_lz4(values: CompressiveEncoding) -> CompressiveEncoding { + ProtobufUtils21::wrapped(CompressionConfig::new(CompressionScheme::Lz4, None), values) + .unwrap() + } + + fn validity( + meaning: pb21::sparse_validity_set::Meaning, + positions: Option, + ) -> Option { + Some(pb21::SparseValiditySet { + meaning: meaning as i32, + positions, + }) + } + + fn null_positions( + positions: Option, + ) -> Option { + validity( + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions, + positions, + ) + } + + fn count_empty() -> Option { + Some(pb21::SparseCountSet { + counts: Some(pb21::sparse_count_set::Counts::Empty( + pb21::SparseCountEmpty {}, + )), + }) + } + + fn count_constant(value: u64) -> Option { + Some(pb21::SparseCountSet { + counts: Some(pb21::sparse_count_set::Counts::Constant( + pb21::SparseCountConstant { value }, + )), + }) + } + + fn sparse_layout() -> pb21::SparseLayout { + pb21::SparseLayout { + value_compression: Some(ProtobufUtils21::flat(32, None)), + num_buffers: 1, + num_items: 1, + num_visible_items: 1, + has_large_chunk: false, + structural_layers: Vec::new(), + } + } + + fn validity_layer( + num_slots: u64, + validity: Option, + ) -> pb21::SparseStructuralLayer { + pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::Validity( + pb21::SparseValidityLayer { + num_slots, + validity, + }, + )), + } + } + + fn list_layer( + num_slots: u64, + num_child_slots: u64, + non_empty_positions: Option, + counts: Option, + validity: Option, + ) -> pb21::SparseStructuralLayer { + pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::List( + pb21::SparseListLayer { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + }, + )), + } + } + + fn fixed_size_list_layer( + num_slots: u64, + dimension: u64, + validity: Option, + ) -> pb21::SparseStructuralLayer { + pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::FixedSizeList( + pb21::SparseFixedSizeListLayer { + num_slots, + dimension, + validity, + }, + )), + } + } + + fn assert_invalid_input_contains(err: Error, expected: &str) { + let message = err.to_string(); + assert!( + matches!(&err, Error::InvalidInput { .. }), + "expected InvalidInput, got {err:?}" + ); + assert!( + message.contains(expected), + "expected error to contain {expected:?}, got {message}" + ); + } + + #[test] + fn rejects_missing_layer_variants_and_item_count_mismatches() { + let decompressors = DefaultDecompressionStrategy::default(); + + let mut layout = sparse_layout(); + layout + .structural_layers + .push(pb21::SparseStructuralLayer { layer: None }); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "missing its layer variant"); + + let mut layout = sparse_layout(); + layout.num_items = 2; + layout + .structural_layers + .push(validity_layer(1, null_positions(position_empty()))); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "layers imply 1"); + } + + #[test] + fn accepts_large_structural_descriptors() { + let explicit_values = 8_u64 * 1024 * 1024 + 1; + let metadata_size = explicit_values * 8; + let value_position = metadata_size; + let value_size = explicit_values * 16; + let structural_position = value_position + value_size; + let structural_size = explicit_values * std::mem::size_of::() as u64; + let mut layout = sparse_layout(); + layout.num_items = explicit_values; + layout.num_visible_items = explicit_values; + layout.structural_layers.push(validity_layer( + explicit_values, + null_positions(position_explicit(explicit_values)), + )); + + SparseStructuralScheduler::try_new( + &[ + (0, metadata_size), + (value_position, value_size), + (structural_position, structural_size), + ], + 0, + explicit_values, + DataType::Int32, + &layout, + &DefaultDecompressionStrategy::default(), + ) + .unwrap(); + } + + #[test] + fn accepts_deep_supported_value_encodings() { + let encoding = (0..300).fold(ProtobufUtils21::flat(32, None), |values, _| { + ProtobufUtils21::fsl(1, false, values) + }); + + assert_eq!( + SparseStructuralScheduler::validate_value_encoding(&encoding).unwrap(), + 1 + ); + } + + fn null_set(positions: SparsePositionSet) -> SparseValiditySet { + SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions, + } + } + + fn valid_set(positions: SparsePositionSet) -> SparseValiditySet { + SparseValiditySet { + meaning: SparseValidityMeaning::ValidPositions, + positions, + } + } + + #[test] + fn semantic_position_and_count_sets_project_without_materializing_ranges() { + let ranges = [1..4]; + assert_eq!( + select_position_set(&SparsePositionSet::Empty, &ranges, 5, "empty") + .unwrap() + .positions, + SparsePositionSet::Empty + ); + assert_eq!( + select_position_set(&SparsePositionSet::all(5), &ranges, 5, "all") + .unwrap() + .positions, + SparsePositionSet::all(3) + ); + assert_eq!( + select_position_set(&SparsePositionSet::range(1, 3), &ranges, 5, "range") + .unwrap() + .positions, + SparsePositionSet::all(3) + ); + assert_eq!( + select_position_set( + &SparsePositionSet::Explicit(vec![0, 2, 4]), + &[0..1, 4..5], + 5, + "explicit", + ) + .unwrap() + .positions, + SparsePositionSet::all(2) + ); + + assert_eq!( + select_count_set(&SparseCountSet::Empty, &[], 0).unwrap(), + SparseCountSet::Empty + ); + assert_eq!( + select_count_set(&SparseCountSet::constant(2, 3), &[0..1, 2..3], 2,).unwrap(), + SparseCountSet::constant(2, 2) + ); + assert_eq!( + select_count_set( + &SparseCountSet::from_counts(vec![1, 2, 3]).unwrap(), + &[0..1, 2..3], + 2, + ) + .unwrap(), + SparseCountSet::from_counts(vec![1, 3]).unwrap() + ); + } + + #[test] + fn validity_polarities_rebuild_the_same_arrow_domain() { + let mut null_builder = BooleanBufferBuilder::new(4); + null_set(SparsePositionSet::Explicit(vec![1, 3])) + .append_to(&mut null_builder, 4) + .unwrap(); + assert_eq!( + null_builder.finish().iter().collect::>(), + vec![true, false, true, false] + ); + + let mut valid_builder = BooleanBufferBuilder::new(4); + valid_set(SparsePositionSet::range(1, 2)) + .append_to(&mut valid_builder, 4) + .unwrap(); + assert_eq!( + valid_builder.finish().iter().collect::>(), + vec![false, true, true, false] + ); + } + + #[test] + fn schema_layer_mismatches_are_invalid_input() { + let mut missing = SparseStructuralUnraveler::new(SparseStructuralPlan { + layers: Vec::new(), + num_items: 1, + num_visible_items: 1, + }); + let mut validity = BooleanBufferBuilder::new(1); + let err = missing.unravel_validity(&mut validity).unwrap_err(); + assert_invalid_input_contains(err, "fewer layers than the Arrow schema"); + + let mut fixed_size_list = SparseStructuralUnraveler::new(SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::FixedSizeList { + num_slots: 2, + dimension: 2, + validity: null_set(SparsePositionSet::empty()), + }], + num_items: 4, + num_visible_items: 4, + }); + let mut validity = BooleanBufferBuilder::new(2); + let err = fixed_size_list.unravel_validity(&mut validity).unwrap_err(); + assert_invalid_input_contains(err, "does not match the Arrow schema"); + + let extra = SparseStructuralUnraveler::new(SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::Validity { + num_slots: 1, + validity: null_set(SparsePositionSet::empty()), + }], + num_items: 1, + num_visible_items: 1, + }); + let err = extra.ensure_exhausted().unwrap_err(); + assert_invalid_input_contains(err, "1 unconsumed layer"); + } + + fn nested_plan() -> SparseStructuralPlan { + SparseStructuralPlan { + layers: vec![ + SparseStructuralLayerPlan::Validity { + num_slots: 6, + validity: null_set(SparsePositionSet::Explicit(vec![1, 4])), + }, + SparseStructuralLayerPlan::List { + num_slots: 6, + num_child_slots: 5, + non_empty_positions: SparsePositionSet::Explicit(vec![0, 2, 5]), + counts: SparseCountSet::from_counts(vec![2, 1, 2]).unwrap(), + validity: null_set(SparsePositionSet::Explicit(vec![1, 4])), + }, + SparseStructuralLayerPlan::FixedSizeList { + num_slots: 5, + dimension: 2, + validity: valid_set(SparsePositionSet::range(1, 3)), + }, + ], + num_items: 13, + num_visible_items: 10, + } + } + + #[test] + fn discontiguous_projection_preserves_outer_to_inner_order() { + let selection = slice_sparse_plan(&nested_plan(), &[5..6, 0..1], 6).unwrap(); + assert_eq!(selection.leaf_ranges, vec![6..10, 0..4]); + assert_eq!(selection.plan.num_visible_items, 8); + selection.plan.validate(2).unwrap(); + + let SparseStructuralLayerPlan::List { + non_empty_positions, + counts, + .. + } = &selection.plan.layers[1] + else { + panic!("expected projected list layer"); + }; + assert_eq!(*non_empty_positions, SparsePositionSet::all(2)); + assert_eq!(*counts, SparseCountSet::constant(2, 2)); + } + + #[test] + fn no_value_projection_keeps_empty_and_null_list_structure() { + let selection = slice_sparse_plan(&nested_plan(), &[1..2, 3..4], 6).unwrap(); + assert!(selection.leaf_ranges.is_empty()); + assert_eq!(selection.plan.num_visible_items, 0); + selection.plan.validate(2).unwrap(); + + let SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } = &selection.plan.layers[1] + else { + panic!("expected projected list layer"); + }; + assert_eq!((*num_slots, *num_child_slots), (2, 0)); + assert_eq!(*non_empty_positions, SparsePositionSet::Empty); + assert_eq!(*counts, SparseCountSet::Empty); + assert_eq!(*validity, null_set(SparsePositionSet::range(0, 1))); + } + + #[test] + fn rejects_missing_value_compression_and_buffer_count_mismatch() { + let decompressors = DefaultDecompressionStrategy::default(); + let mut layout = sparse_layout(); + layout.value_compression = None; + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "missing value compression"); + + let mut layout = sparse_layout(); + layout.num_buffers = 2; + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "declares 2 value buffers"); + } + + #[test] + fn rejects_inconsistent_chunk_count_before_io() { + let decompressors = DefaultDecompressionStrategy::default(); + + let layout = sparse_layout(); + let err = SparseStructuralScheduler::try_new( + &[(0, 16), (16, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "declares 2 chunks for 1 visible items"); + } + + #[cfg(feature = "lz4")] + #[test] + fn accepts_large_general_decompression_headers() { + let encoding = general_lz4(ProtobufUtils21::flat(64, None)); + let Some(pb21::compressive_encoding::Compression::General(general)) = + encoding.compression.as_ref() + else { + panic!("expected General compression"); + }; + let declared_size = 65_u32 * 1024 * 1024; + + SparseStructuralScheduler::validate_general_buffer_header( + general, + &declared_size.to_le_bytes(), + "test", + ) + .unwrap(); + } + + #[cfg(feature = "lz4")] + #[tokio::test] + async fn rejects_malformed_general_structural_buffers_as_invalid_input() { + let mut layout = sparse_layout(); + layout.structural_layers.push(validity_layer( + 1, + null_positions(position_set( + pb21::sparse_position_set::Positions::Explicit(general_lz4(ProtobufUtils21::flat( + 64, None, + ))), + 1, + )), + )); + let decompressors = DefaultDecompressionStrategy::default(); + + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8), (16, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + data.extend_from_slice(&8_u32.to_le_bytes()); + data.extend_from_slice(&[0xff; 4]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected malformed General buffer to be rejected"); + }; + assert_invalid_input_contains(err, "decompression failed"); + } + + #[cfg(feature = "lz4")] + #[tokio::test] + async fn rejects_malformed_general_value_buffer() { + let mut layout = sparse_layout(); + layout.value_compression = Some(general_lz4(ProtobufUtils21::flat(32, None))); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&0_u16.to_le_bytes()); + data.extend_from_slice(&0_u16.to_le_bytes()); + data.extend_from_slice(&[0; 4]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + scheduler.initialize(&io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[0..1], &io).unwrap(); + let mut decoder = page_tasks.pop().unwrap().decoder_fut.await.unwrap(); + let Err(err) = decoder.drain(1).unwrap().decode() else { + panic!("expected malformed General value buffer to be rejected"); + }; + assert_invalid_input_contains(err, "missing its length prefix"); + } + + #[test] + fn rejects_layer_domain_and_fixed_size_list_mismatches() { + let decompressors = DefaultDecompressionStrategy::default(); + let mut layout = sparse_layout(); + layout.num_items = 2; + layout.num_visible_items = 2; + layout + .structural_layers + .push(validity_layer(1, null_positions(position_empty()))); + layout + .structural_layers + .push(validity_layer(2, null_positions(position_empty()))); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "layer 1 has 2 slots, expected 1"); + + let mut layout = sparse_layout(); + layout.num_items = 4; + layout.num_visible_items = 4; + layout.structural_layers.push(fixed_size_list_layer( + 2, + 3, + null_positions(position_empty()), + )); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 2, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "terminal domain has 6 slots"); + } + + #[test] + fn rejects_invalid_validity_and_list_count_semantics() { + let decompressors = DefaultDecompressionStrategy::default(); + let mut layout = sparse_layout(); + layout.structural_layers.push(validity_layer( + 1, + validity( + pb21::sparse_validity_set::Meaning::SparseValidityUnspecified, + position_empty(), + ), + )); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "meaning is unspecified"); + + let mut layout = sparse_layout(); + layout.num_items = 3; + layout.num_visible_items = 3; + layout.structural_layers.push(list_layer( + 2, + 3, + position_all(2), + count_constant(2), + null_positions(position_empty()), + )); + let err = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 2, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap_err(); + assert_invalid_input_contains(err, "count sum 4 does not match child slots 3"); + } + + #[tokio::test] + async fn rejects_unordered_explicit_positions_after_decompression() { + let mut layout = sparse_layout(); + layout.num_items = 4; + layout.num_visible_items = 4; + layout + .structural_layers + .push(validity_layer(4, null_positions(position_explicit(2)))); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8), (16, 16)], + 0, + 4, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&4_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + data.extend_from_slice(&3_u64.to_le_bytes()); + data.extend_from_slice(&0_u64.to_le_bytes()); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected unordered sparse positions to be rejected"); + }; + assert_invalid_input_contains(err, "positions must be strictly increasing"); + } + + #[tokio::test] + async fn rejects_chunk_metadata_value_and_byte_sum_mismatches() { + let layout = sparse_layout(); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected chunk byte sum mismatch"); + }; + assert_invalid_input_contains(err, "describes 16 value bytes"); + + let mut layout = sparse_layout(); + layout.num_items = 2; + layout.num_visible_items = 2; + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 2, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected chunk value sum mismatch"); + }; + assert_invalid_input_contains(err, "metadata has 1, layout has 2"); + } + + #[tokio::test] + async fn rejects_malformed_value_chunk_before_decompression() { + let layout = sparse_layout(); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, 8)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&[0; 8]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + scheduler.initialize(&io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[0..1], &io).unwrap(); + let page_task = page_tasks.pop().unwrap(); + let mut decoder = page_task.decoder_fut.await.unwrap(); + let decode_task = decoder.drain(1).unwrap(); + let Err(err) = decode_task.decode() else { + panic!("expected malformed value chunk to be rejected"); + }; + assert_invalid_input_contains(err, "flat values buffer has 0 bytes, expected 4"); + } + + #[tokio::test] + async fn accepts_value_chunks_larger_than_64_mib() { + let chunk_size = 65_u64 * 1024 * 1024; + let layout = sparse_layout(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 8), (8, chunk_size)], + 0, + 1, + DataType::Int32, + &layout, + &DefaultDecompressionStrategy::default(), + ) + .unwrap(); + let words_minus_one = u32::try_from(chunk_size / MINIBLOCK_ALIGNMENT as u64 - 1).unwrap(); + let mut metadata = Vec::new(); + metadata.extend_from_slice(&words_minus_one.to_le_bytes()); + metadata.extend_from_slice(&1_u32.to_le_bytes()); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(metadata))); + + scheduler.initialize(&io).await.unwrap(); + } + + #[tokio::test] + async fn rejects_value_chunks_above_the_miniblock_limit() { + let num_values = miniblock::MAX_CONFIGURABLE_MINIBLOCK_VALUES + 1; + let mut layout = sparse_layout(); + layout.num_items = num_values; + layout.num_visible_items = num_values; + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 16), (16, 16)], + 0, + num_values, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let mut data = Vec::new(); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&(num_values as u32).to_le_bytes()); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&0_u32.to_le_bytes()); + data.extend_from_slice(&[0; 16]); + let io: Arc = Arc::new(SimulatedScheduler::new(Bytes::from(data))); + + let Err(err) = scheduler.initialize(&io).await else { + panic!("expected oversized value chunk to be rejected"); + }; + assert_invalid_input_contains(err, "exceeding the mini-block limit"); + } + + #[derive(Debug, Clone)] + struct RecordingIo { + data: Bytes, + calls: Arc>>>>, + } + + impl RecordingIo { + fn new(data: Bytes) -> Self { + Self { + data, + calls: Arc::new(Mutex::new(Vec::new())), + } + } + } + + impl EncodingsIo for RecordingIo { + fn submit_request( + &self, + ranges: Vec>, + _priority: u64, + ) -> BoxFuture<'static, Result>> { + self.calls.lock().unwrap().push(ranges.clone()); + let data = self.data.clone(); + async move { + ranges + .into_iter() + .map(|range| { + let start = usize_from_u64(range.start, "test range start")?; + let end = usize_from_u64(range.end, "test range end")?; + if start > end || end > data.len() { + return Err(Error::invalid_input_source( + "Test I/O range is outside fixture data".into(), + )); + } + Ok(data.slice(start..end)) + }) + .collect() + } + .boxed() + } + } + + #[tokio::test] + async fn selective_read_requests_only_intersecting_value_chunk() { + let mut layout = sparse_layout(); + layout.num_items = 4; + layout.num_visible_items = 4; + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 16), (16, 32)], + 0, + 4, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + + let mut data = Vec::new(); + for _ in 0..2 { + data.extend_from_slice(&1_u32.to_le_bytes()); + data.extend_from_slice(&2_u32.to_le_bytes()); + } + for values in [[10_i32, 20], [30, 40]] { + data.extend_from_slice(&0_u16.to_le_bytes()); + data.extend_from_slice(&8_u16.to_le_bytes()); + data.extend_from_slice(&[0; 4]); + for value in values { + data.extend_from_slice(&value.to_le_bytes()); + } + } + + let io = Arc::new(RecordingIo::new(Bytes::from(data))); + let trait_io: Arc = io.clone(); + scheduler.initialize(&trait_io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[2..3], &trait_io).unwrap(); + let page_task = page_tasks.pop().unwrap(); + let mut decoder = page_task.decoder_fut.await.unwrap(); + let decoded = decoder.drain(1).unwrap().decode().unwrap(); + assert_eq!(decoded.data.num_values(), 1); + + let calls = io.calls.lock().unwrap(); + assert_eq!(calls.as_slice(), &[vec![0..16], vec![32..48]]); + } + + #[tokio::test] + async fn empty_leaf_selection_rebuilds_offsets_without_value_io() { + let mut layout = sparse_layout(); + layout.num_visible_items = 0; + layout.structural_layers.push(list_layer( + 1, + 0, + position_empty(), + count_empty(), + null_positions(position_empty()), + )); + let decompressors = DefaultDecompressionStrategy::default(); + let mut scheduler = SparseStructuralScheduler::try_new( + &[(0, 0), (0, 0)], + 0, + 1, + DataType::Int32, + &layout, + &decompressors, + ) + .unwrap(); + let io = Arc::new(RecordingIo::new(Bytes::new())); + let trait_io: Arc = io.clone(); + scheduler.initialize(&trait_io).await.unwrap(); + let mut page_tasks = scheduler.schedule_ranges(&[0..1], &trait_io).unwrap(); + let page_task = page_tasks.pop().unwrap(); + let mut decoder = page_task.decoder_fut.await.unwrap(); + let decoded = decoder.drain(1).unwrap().decode().unwrap(); + assert_eq!(decoded.data.num_values(), 0); + + let mut repdef = CompositeRepDefUnraveler::new(vec![decoded.repdef]); + let (offsets, validity) = repdef.unravel_offsets::().unwrap(); + assert_eq!(offsets.as_ref(), &[0, 0]); + assert!(validity.is_none()); + + let calls = io.calls.lock().unwrap(); + assert_eq!(calls.len(), 2); + assert!(calls[1].is_empty(), "value payload must not be requested"); + } +} diff --git a/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs b/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs new file mode 100644 index 00000000000..5ec1d32b5e1 --- /dev/null +++ b/rust/lance-encoding/src/encodings/logical/primitive/sparse/writer.rs @@ -0,0 +1,2407 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Sparse structural planning and serialization. + +use std::iter; + +use arrow_buffer::BooleanBuffer; +use lance_core::{Error, Result, datatypes::Field, utils::bit::pad_bytes}; + +use crate::{ + buffer::LanceBuffer, + compression::{CompressionStrategy, compress_required_block}, + data::{BlockInfo, DataBlock, FixedWidthDataBlock}, + decoder::PageEncoding, + encoder::EncodedPage, + format::pb21::{self, CompressiveEncoding}, + repdef::{NormalizedStructuralLayer, NormalizedStructuralPlan}, + statistics::ComputeStat, +}; + +use super::super::MiniblockChunkSize; + +use super::{ + SparseCountSet, SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, + SparseValidityMeaning, SparseValiditySet, +}; +use crate::encodings::logical::primitive::{ + FILL_BYTE, MINIBLOCK_ALIGNMENT, + miniblock::{MiniBlockCompressed, MiniBlockCompressionContext}, +}; + +#[derive(Clone, Copy, Default)] +struct PositionSetStats { + count: u64, + first: u64, + last: u64, + is_contiguous: bool, +} + +impl PositionSetStats { + fn observe(&mut self, position: u64) { + if self.count == 0 { + self.first = position; + self.is_contiguous = true; + } else if self.last.checked_add(1) != Some(position) { + self.is_contiguous = false; + } + self.last = position; + self.count += 1; + } + + fn encoded_cost(&self, domain_len: u64) -> u64 { + if self.count == 0 || self.count == domain_len || self.is_contiguous { + 0 + } else { + self.count + } + } + + fn to_set( + self, + validity: &BooleanBuffer, + want_valid: bool, + domain_len: u64, + label: &str, + ) -> Result { + if self.count == 0 { + return Ok(SparsePositionSet::empty()); + } + if self.count == domain_len { + return Ok(SparsePositionSet::all(domain_len)); + } + if self.is_contiguous { + return Ok(SparsePositionSet::range(self.first, self.count)); + } + + let capacity = usize::try_from(self.count).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} positions exceed usize::MAX").into(), + ) + })?; + let mut positions = Vec::with_capacity(capacity); + for (index, is_valid) in validity.iter().enumerate() { + if is_valid == want_valid { + positions.push(u64::try_from(index).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} position exceeds u64::MAX").into(), + ) + })?); + } + } + SparsePositionSet::from_positions(positions, domain_len, label) + } +} + +fn usize_to_u64(value: usize, label: &str) -> Result { + u64::try_from(value).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural {label} {value} exceeds u64::MAX").into(), + ) + }) +} + +fn validity_set( + validity: Option<&BooleanBuffer>, + num_slots: usize, + label: &str, +) -> Result { + let Some(validity) = validity else { + return Ok(SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: SparsePositionSet::empty(), + }); + }; + if validity.len() != num_slots { + return Err(Error::invalid_input_source( + format!( + "Sparse structural {label} validity length {} does not match {} slots", + validity.len(), + num_slots + ) + .into(), + )); + } + + let domain_len = usize_to_u64(num_slots, "validity domain")?; + let mut valid_stats = PositionSetStats::default(); + let mut null_stats = PositionSetStats::default(); + for (index, is_valid) in validity.iter().enumerate() { + let index = usize_to_u64(index, "validity position")?; + if is_valid { + valid_stats.observe(index); + } else { + null_stats.observe(index); + } + } + + if null_stats.count == 0 { + return Ok(SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: SparsePositionSet::empty(), + }); + } + if valid_stats.count == 0 { + return Ok(SparseValiditySet { + meaning: SparseValidityMeaning::ValidPositions, + positions: SparsePositionSet::empty(), + }); + } + + let valid_cost = valid_stats.encoded_cost(domain_len); + let null_cost = null_stats.encoded_cost(domain_len); + if valid_cost < null_cost { + Ok(SparseValiditySet { + meaning: SparseValidityMeaning::ValidPositions, + positions: valid_stats.to_set(validity, true, domain_len, label)?, + }) + } else { + Ok(SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions: null_stats.to_set(validity, false, domain_len, label)?, + }) + } +} + +/// Builds the semantic sparse plan directly from the once-normalized Arrow layers. +pub(in crate::encodings::logical::primitive) fn plan( + normalized: &NormalizedStructuralPlan, + num_visible_items: u64, +) -> Result { + let mut layers = Vec::with_capacity(normalized.layers().len()); + let mut num_items = num_visible_items; + + for layer in normalized.layers() { + match layer { + NormalizedStructuralLayer::Validity { + validity, + num_slots, + } => { + layers.push(SparseStructuralLayerPlan::Validity { + num_slots: usize_to_u64(num_slots, "validity slot count")?, + validity: validity_set(validity, num_slots, "validity")?, + }); + } + NormalizedStructuralLayer::FixedSizeList { + validity, + dimension, + num_slots, + } => { + if dimension == 0 { + return Err(Error::invalid_input_source( + "Sparse structural fixed-size-list dimension is zero".into(), + )); + } + layers.push(SparseStructuralLayerPlan::FixedSizeList { + num_slots: usize_to_u64(num_slots, "fixed-size-list slot count")?, + dimension: usize_to_u64(dimension, "fixed-size-list dimension")?, + validity: validity_set(validity, num_slots, "fixed-size-list validity")?, + }); + } + NormalizedStructuralLayer::List { + offsets, + validity, + num_slots, + } => { + let expected_offsets = num_slots.checked_add(1).ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural list offset count overflows".into(), + ) + })?; + if offsets.len() != expected_offsets { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} offsets for {} slots", + offsets.len(), + num_slots + ) + .into(), + )); + } + if offsets.first().copied() != Some(0) { + return Err(Error::invalid_input_source( + "Sparse structural list offsets must start at zero".into(), + )); + } + + let mut non_empty_positions = Vec::new(); + let mut counts = Vec::new(); + for slot in 0..num_slots { + let start = offsets[slot]; + let end = offsets[slot + 1]; + let count = end.checked_sub(start).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural list offsets decrease at slot {slot}: {start}..{end}" + ) + .into(), + ) + })?; + let is_valid = validity.is_none_or(|validity| validity.value(slot)); + if !is_valid && count != 0 { + return Err(Error::invalid_input_source( + format!( + "Sparse structural null list slot {slot} has {count} child slots" + ) + .into(), + )); + } + if is_valid && count > 0 { + non_empty_positions.push(usize_to_u64(slot, "list position")?); + counts.push(u64::try_from(count).map_err(|_| { + Error::invalid_input_source( + format!("Sparse structural list count {count} exceeds u64::MAX") + .into(), + ) + })?); + } + } + + let num_slots_u64 = usize_to_u64(num_slots, "list slot count")?; + let num_non_empty = usize_to_u64(non_empty_positions.len(), "list position count")?; + num_items = num_items + .checked_add(num_slots_u64 - num_non_empty) + .ok_or_else(|| { + Error::invalid_input_source( + "Sparse structural item count overflows u64".into(), + ) + })?; + let num_child_slots = offsets.last().copied().ok_or_else(|| { + Error::invalid_input_source("Sparse structural list has no offsets".into()) + })?; + let num_child_slots = u64::try_from(num_child_slots).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse structural list child slot count {num_child_slots} is negative" + ) + .into(), + ) + })?; + layers.push(SparseStructuralLayerPlan::List { + num_slots: num_slots_u64, + num_child_slots, + non_empty_positions: SparsePositionSet::from_positions( + non_empty_positions, + num_slots_u64, + "list non-empty", + )?, + counts: SparseCountSet::from_counts(counts)?, + validity: validity_set(validity, num_slots, "list validity")?, + }); + } + } + } + + let row_domain = match layers.first() { + Some(SparseStructuralLayerPlan::Validity { num_slots, .. }) + | Some(SparseStructuralLayerPlan::List { num_slots, .. }) + | Some(SparseStructuralLayerPlan::FixedSizeList { num_slots, .. }) => *num_slots, + None => { + return Err(Error::invalid_input_source( + "Sparse structural encoding requires at least one Arrow structural layer".into(), + )); + } + }; + let plan = SparseStructuralPlan { + layers, + num_items, + num_visible_items, + }; + plan.validate(row_domain)?; + Ok(plan) +} + +/// ConstantLayout remains the canonical representation when no value payload exists. +pub(in crate::encodings::logical::primitive) fn uses_constant_layout( + plan: &SparseStructuralPlan, + field: &Field, +) -> bool { + if plan.num_visible_items == 0 { + return true; + } + if matches!(field.data_type(), arrow_schema::DataType::Struct(fields) if fields.is_empty()) { + return true; + } + + let Some(layer) = plan.layers.last() else { + return false; + }; + let (num_slots, validity) = match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } + | SparseStructuralLayerPlan::List { + num_slots, + validity, + .. + } + | SparseStructuralLayerPlan::FixedSizeList { + num_slots, + validity, + .. + } => (*num_slots, validity), + }; + match validity.meaning { + SparseValidityMeaning::NullPositions => validity.positions.len() == num_slots, + SparseValidityMeaning::ValidPositions => validity.positions.is_empty(), + } +} + +fn supports_fixed_size_list_values(data: &DataBlock) -> bool { + match data { + DataBlock::FixedWidth(_) => true, + DataBlock::FixedSizeList(list) => supports_fixed_size_list_values(list.child.as_ref()), + DataBlock::Nullable(nullable) => supports_fixed_size_list_values(nullable.data.as_ref()), + _ => false, + } +} + +/// Whether the sparse writer can encode this value block without changing its value path. +pub fn supports_value_block(data: &DataBlock) -> bool { + match data { + DataBlock::FixedWidth(_) | DataBlock::VariableWidth(_) => true, + DataBlock::Struct(data) => !data.has_variable_width_child(), + DataBlock::FixedSizeList(data) => supports_fixed_size_list_values(data.child.as_ref()), + DataBlock::Empty() + | DataBlock::Constant(_) + | DataBlock::AllNull(_) + | DataBlock::Nullable(_) + | DataBlock::Opaque(_) + | DataBlock::Dictionary(_) => false, + } +} + +struct SparseMiniBlockChunk { + buffer_sizes: Vec, + num_values: u32, +} + +struct SparseMiniBlockCompressed { + data: Vec, + chunks: Vec, +} + +struct SerializedValuePage { + num_buffers: u64, + data: LanceBuffer, + metadata: LanceBuffer, +} + +pub struct PreparedSparseValues { + num_values: u64, + value_compression: CompressiveEncoding, + values: SerializedValuePage, +} + +pub enum SparseValueInput { + Unprepared(DataBlock), + Prepared(PreparedSparseValues), +} + +struct EncodedStructuralPlan { + layers: Vec, + buffers: Vec, +} + +fn with_explicit_value_counts( + compressed: MiniBlockCompressed, +) -> Result { + let mut values_in_previous_chunks = 0_u64; + let mut chunks = Vec::with_capacity(compressed.chunks.len()); + for chunk in compressed.chunks { + let num_values = chunk.num_values(values_in_previous_chunks, compressed.num_values); + values_in_previous_chunks = values_in_previous_chunks + .checked_add(num_values) + .ok_or_else(|| Error::internal("Sparse value count overflows u64".to_string()))?; + chunks.push(SparseMiniBlockChunk { + buffer_sizes: chunk.buffer_sizes, + num_values: u32::try_from(num_values).map_err(|_| { + Error::invalid_input_source( + format!( + "Sparse value chunk has {num_values} visible values, which exceeds the u32 metadata limit" + ) + .into(), + ) + })?, + }); + } + if values_in_previous_chunks != compressed.num_values { + return Err(Error::internal(format!( + "Sparse value chunks describe {values_in_previous_chunks} values, expected {}", + compressed.num_values + ))); + } + Ok(SparseMiniBlockCompressed { + data: compressed.data, + chunks, + }) +} + +fn serialize_value_chunks( + compressed: SparseMiniBlockCompressed, + miniblock_chunk_size: MiniblockChunkSize, +) -> Result { + let bytes_data = compressed.data.iter().map(LanceBuffer::len).sum::(); + let num_buffers = compressed.data.len(); + let mut data_buffer = Vec::with_capacity(bytes_data + 9 * num_buffers); + let mut metadata = Vec::with_capacity(compressed.chunks.len() * 8); + let mut buffer_offsets = vec![0_usize; num_buffers]; + + for chunk in compressed.chunks { + if chunk.buffer_sizes.len() != num_buffers { + return Err(Error::internal(format!( + "Sparse chunk has {} value buffer sizes, expected {num_buffers}", + chunk.buffer_sizes.len() + ))); + } + + let chunk_start = data_buffer.len(); + debug_assert_eq!(chunk_start % MINIBLOCK_ALIGNMENT, 0); + data_buffer.extend_from_slice(&0_u16.to_le_bytes()); + if miniblock_chunk_size == MiniblockChunkSize::U32 { + for buffer_size in &chunk.buffer_sizes { + data_buffer.extend_from_slice(&buffer_size.to_le_bytes()); + } + } else { + for buffer_size in &chunk.buffer_sizes { + let buffer_size = u16::try_from(*buffer_size).map_err(|_| { + Error::internal(format!( + "Sparse value buffer size ({buffer_size} bytes) exceeds 16-bit metadata" + )) + })?; + data_buffer.extend_from_slice(&buffer_size.to_le_bytes()); + } + } + let add_padding = |buffer: &mut Vec| { + let padding = pad_bytes::(buffer.len()); + buffer.extend(iter::repeat_n(FILL_BYTE, padding)); + }; + add_padding(&mut data_buffer); + + for (buffer_size, (buffer, buffer_offset)) in chunk + .buffer_sizes + .iter() + .zip(compressed.data.iter().zip(buffer_offsets.iter_mut())) + { + let start = *buffer_offset; + let end = start.checked_add(*buffer_size as usize).ok_or_else(|| { + Error::internal("Sparse value buffer range overflows".to_string()) + })?; + let bytes = buffer.as_ref().get(start..end).ok_or_else(|| { + Error::internal(format!( + "Sparse value chunk requests bytes {start}..{end} from a {}-byte buffer", + buffer.len() + )) + })?; + *buffer_offset = end; + data_buffer.extend_from_slice(bytes); + add_padding(&mut data_buffer); + } + + let chunk_bytes = data_buffer.len() - chunk_start; + if chunk_bytes == 0 || !chunk_bytes.is_multiple_of(MINIBLOCK_ALIGNMENT) { + return Err(Error::internal(format!( + "Sparse value chunk size {chunk_bytes} is not a positive multiple of {MINIBLOCK_ALIGNMENT}" + ))); + } + let words_minus_one = chunk_bytes / MINIBLOCK_ALIGNMENT - 1; + metadata.extend_from_slice( + &u32::try_from(words_minus_one) + .map_err(|_| { + Error::internal(format!( + "Sparse value chunk size {chunk_bytes} exceeds the metadata limit" + )) + })? + .to_le_bytes(), + ); + metadata.extend_from_slice(&chunk.num_values.to_le_bytes()); + } + + for (index, (consumed, buffer)) in buffer_offsets + .iter() + .zip(compressed.data.iter()) + .enumerate() + { + if *consumed != buffer.len() { + return Err(Error::internal(format!( + "Sparse value buffer {index} consumed {consumed} bytes, expected {}", + buffer.len() + ))); + } + } + + Ok(SerializedValuePage { + num_buffers: usize_to_u64(num_buffers, "value buffer count")?, + data: LanceBuffer::from(data_buffer), + metadata: LanceBuffer::from(metadata), + }) +} + +pub fn prepare_values( + field: &Field, + compression_strategy: &dyn CompressionStrategy, + data: DataBlock, + miniblock_chunk_size: MiniblockChunkSize, +) -> Result { + match &data { + DataBlock::AllNull(_) => { + return Err(Error::internal( + "All-null values must use ConstantLayout".to_string(), + )); + } + DataBlock::Dictionary(_) => { + return Err(Error::not_supported_source( + "Sparse layout does not support dictionary data blocks".into(), + )); + } + DataBlock::Struct(data) if data.has_variable_width_child() => { + return Err(Error::not_supported_source( + "Sparse layout does not support variable-width packed struct data blocks".into(), + )); + } + _ => {} + } + + let num_values = data.num_values(); + let compressor = compression_strategy.create_miniblock_compressor(field, &data)?; + let support_large_chunk = miniblock_chunk_size == MiniblockChunkSize::U32; + let compression_context = MiniBlockCompressionContext::new(0, support_large_chunk, false); + let (compressed, value_compression) = compressor.compress(compression_context, data)?; + let values = serialize_value_chunks( + with_explicit_value_counts(compressed)?, + miniblock_chunk_size, + )?; + Ok(PreparedSparseValues { + num_values, + value_compression, + values, + }) +} + +fn encode_u64_values( + values: Vec, + compression_strategy: &dyn CompressionStrategy, +) -> Result<(LanceBuffer, CompressiveEncoding)> { + let num_values = usize_to_u64(values.len(), "u64 value count")?; + let mut block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_vec(values), + bits_per_value: 64, + num_values, + block_info: BlockInfo::new(), + }); + block.compute_stat(); + let field = Field::new_arrow("", arrow_schema::DataType::UInt64, false)?; + compress_required_block(compression_strategy, &field, block) +} + +fn positions_to_deltas(positions: &[u64], label: &str) -> Result> { + let mut previous = 0_u64; + positions + .iter() + .copied() + .enumerate() + .map(|(index, position)| { + if index > 0 && position <= previous { + return Err(Error::invalid_input_source( + format!("Sparse structural {label} positions must be strictly increasing") + .into(), + )); + } + let delta = if index == 0 { + position + } else { + position - previous + }; + previous = position; + Ok(delta) + }) + .collect() +} + +fn encode_position_set( + positions: &SparsePositionSet, + compression_strategy: &dyn CompressionStrategy, + label: &str, +) -> Result<(Option, pb21::SparsePositionSet)> { + let (buffer, positions_pb) = match positions { + SparsePositionSet::Empty => ( + None, + pb21::sparse_position_set::Positions::Empty(pb21::SparsePositionEmpty {}), + ), + SparsePositionSet::All { .. } => ( + None, + pb21::sparse_position_set::Positions::All(pb21::SparsePositionAll {}), + ), + SparsePositionSet::Range { start, len } => ( + None, + pb21::sparse_position_set::Positions::Range(pb21::SparsePositionRange { + start: *start, + length: *len, + }), + ), + SparsePositionSet::Explicit(positions) => { + if positions.is_empty() { + return Err(Error::internal(format!( + "Sparse structural {label} explicit set is empty" + ))); + } + let (buffer, encoding) = + encode_u64_values(positions_to_deltas(positions, label)?, compression_strategy)?; + ( + Some(buffer), + pb21::sparse_position_set::Positions::Explicit(encoding), + ) + } + }; + Ok(( + buffer, + pb21::SparsePositionSet { + positions: Some(positions_pb), + num_positions: positions.len(), + }, + )) +} + +fn encode_count_set( + counts: &SparseCountSet, + compression_strategy: &dyn CompressionStrategy, +) -> Result<(Option, pb21::SparseCountSet)> { + let (buffer, counts_pb) = match counts { + SparseCountSet::Empty => ( + None, + pb21::sparse_count_set::Counts::Empty(pb21::SparseCountEmpty {}), + ), + SparseCountSet::Constant { value, .. } => ( + None, + pb21::sparse_count_set::Counts::Constant(pb21::SparseCountConstant { value: *value }), + ), + SparseCountSet::Explicit { counts, .. } => { + if counts.is_empty() { + return Err(Error::internal( + "Sparse structural explicit count set is empty".to_string(), + )); + } + let (buffer, encoding) = encode_u64_values(counts.to_vec(), compression_strategy)?; + ( + Some(buffer), + pb21::sparse_count_set::Counts::Explicit(encoding), + ) + } + }; + Ok(( + buffer, + pb21::SparseCountSet { + counts: Some(counts_pb), + }, + )) +} + +fn encode_validity_set( + validity: &SparseValiditySet, + compression_strategy: &dyn CompressionStrategy, + label: &str, +) -> Result<(Option, pb21::SparseValiditySet)> { + let (buffer, positions) = + encode_position_set(&validity.positions, compression_strategy, label)?; + let meaning = match validity.meaning { + SparseValidityMeaning::NullPositions => { + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions + } + SparseValidityMeaning::ValidPositions => { + pb21::sparse_validity_set::Meaning::SparseValidityValidPositions + } + }; + Ok(( + buffer, + pb21::SparseValiditySet { + meaning: meaning as i32, + positions: Some(positions), + }, + )) +} + +fn encode_structural_plan( + plan: &SparseStructuralPlan, + compression_strategy: &dyn CompressionStrategy, +) -> Result { + let mut layers = Vec::with_capacity(plan.layers.len()); + let mut buffers = Vec::new(); + + for layer in &plan.layers { + match layer { + SparseStructuralLayerPlan::Validity { + num_slots, + validity, + } => { + let (validity_buffer, validity) = + encode_validity_set(validity, compression_strategy, "validity")?; + buffers.extend(validity_buffer); + layers.push(pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::Validity( + pb21::SparseValidityLayer { + num_slots: *num_slots, + validity: Some(validity), + }, + )), + }); + } + SparseStructuralLayerPlan::List { + num_slots, + num_child_slots, + non_empty_positions, + counts, + validity, + } => { + if non_empty_positions.len() != counts.len() { + return Err(Error::invalid_input_source( + format!( + "Sparse structural list has {} non-empty positions but {} counts", + non_empty_positions.len(), + counts.len() + ) + .into(), + )); + } + let (position_buffer, non_empty_positions) = encode_position_set( + non_empty_positions, + compression_strategy, + "list non-empty", + )?; + buffers.extend(position_buffer); + let (count_buffer, counts) = encode_count_set(counts, compression_strategy)?; + buffers.extend(count_buffer); + let (validity_buffer, validity) = + encode_validity_set(validity, compression_strategy, "list validity")?; + buffers.extend(validity_buffer); + layers.push(pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::List( + pb21::SparseListLayer { + num_slots: *num_slots, + num_child_slots: *num_child_slots, + non_empty_positions: Some(non_empty_positions), + counts: Some(counts), + validity: Some(validity), + }, + )), + }); + } + SparseStructuralLayerPlan::FixedSizeList { + num_slots, + dimension, + validity, + } => { + let (validity_buffer, validity) = encode_validity_set( + validity, + compression_strategy, + "fixed-size-list validity", + )?; + buffers.extend(validity_buffer); + num_slots.checked_mul(*dimension).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Sparse structural fixed-size-list child slot count overflows: slots={num_slots}, dimension={dimension}" + ) + .into(), + ) + })?; + layers.push(pb21::SparseStructuralLayer { + layer: Some(pb21::sparse_structural_layer::Layer::FixedSizeList( + pb21::SparseFixedSizeListLayer { + num_slots: *num_slots, + dimension: *dimension, + validity: Some(validity), + }, + )), + }); + } + } + } + + Ok(EncodedStructuralPlan { layers, buffers }) +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::encodings::logical::primitive) fn encode_page( + column_idx: u32, + field: &Field, + compression_strategy: &dyn CompressionStrategy, + values: SparseValueInput, + plan: SparseStructuralPlan, + row_number: u64, + num_rows: u64, + miniblock_chunk_size: MiniblockChunkSize, +) -> Result { + let PreparedSparseValues { + num_values, + value_compression, + values, + } = match values { + SparseValueInput::Unprepared(data) => { + prepare_values(field, compression_strategy, data, miniblock_chunk_size)? + } + SparseValueInput::Prepared(prepared) => prepared, + }; + if plan.num_visible_items != num_values { + return Err(Error::internal(format!( + "Sparse structural plan has {} visible items but data has {} values", + plan.num_visible_items, num_values + ))); + } + let structural = encode_structural_plan(&plan, compression_strategy)?; + let description = pb21::PageLayout { + layout: Some(pb21::page_layout::Layout::SparseLayout( + pb21::SparseLayout { + value_compression: Some(value_compression), + num_buffers: values.num_buffers, + num_items: plan.num_items, + num_visible_items: plan.num_visible_items, + has_large_chunk: miniblock_chunk_size == MiniblockChunkSize::U32, + structural_layers: structural.layers, + }, + )), + }; + + let mut page_data = Vec::with_capacity(2 + structural.buffers.len()); + page_data.push(values.metadata); + page_data.push(values.data); + page_data.extend(structural.buffers); + Ok(EncodedPage { + data: page_data, + description: PageEncoding::Structural(description), + num_rows, + row_number, + column_idx, + }) +} + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, sync::Arc}; + + use arrow_array::{ + Array, ArrayRef, DictionaryArray, FixedSizeBinaryArray, FixedSizeListArray, Int8Array, + Int32Array, LargeListArray, ListArray, StringArray, StructArray, + builder::{Int32Builder, MapBuilder, StringBuilder}, + types::Int8Type, + }; + use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; + use arrow_schema::{DataType, Field as ArrowField, Fields}; + + use crate::{ + constants::{ + PACKED_STRUCT_META_KEY, STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, + STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_SPARSE, + }, + data::FixedSizeListBlock, + encoder::{ + ColumnIndexSequence, EncodingOptions, FieldEncoder, MIN_PAGE_BUFFER_ALIGNMENT, + OutOfLineBuffers, + }, + testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, test_encoding_strategy, + }, + }; + + use super::*; + + fn sparse_metadata() -> HashMap { + HashMap::from([( + STRUCTURAL_ENCODING_META_KEY.to_string(), + STRUCTURAL_ENCODING_SPARSE.to_string(), + )]) + } + + fn structural_metadata(value: &str) -> HashMap { + HashMap::from([(STRUCTURAL_ENCODING_META_KEY.to_string(), value.to_string())]) + } + + fn null_buffer(validity: impl IntoIterator) -> NullBuffer { + NullBuffer::new(BooleanBuffer::from_iter(validity)) + } + + fn list_i32(offsets: Vec, validity: Option>) -> ArrayRef { + let num_values = offsets.last().copied().unwrap_or_default(); + let values = Arc::new(Int32Array::from_iter_values(0..num_values)) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + values, + validity.map(null_buffer), + ) + .unwrap(), + ) + } + + fn sparse_list_values( + num_rows: usize, + stride: usize, + values: ArrayRef, + item_field: Arc, + ) -> ArrayRef { + let mut offsets = Vec::with_capacity(num_rows + 1); + let mut num_values = 0_i32; + offsets.push(num_values); + for row in 0..num_rows { + if (row + 1).is_multiple_of(stride) || row + 1 == num_rows { + num_values += 1; + } + offsets.push(num_values); + } + assert_eq!(values.len(), num_values as usize); + Arc::new( + ListArray::try_new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(offsets)), + values, + None, + ) + .unwrap(), + ) + } + + fn sparse_i32_list(num_rows: usize, stride: usize) -> ArrayRef { + let num_values = num_rows.div_ceil(stride); + sparse_list_values( + num_rows, + stride, + Arc::new(Int32Array::from_iter_values(0..num_values as i32)), + Arc::new(ArrowField::new("item", DataType::Int32, true)), + ) + } + + fn unsplittable_nested_list(values: ArrayRef, item_field: Arc) -> ArrayRef { + const NUM_INNER_LISTS: usize = 70_000; + + let mut inner_offsets = vec![0_i32; NUM_INNER_LISTS + 1]; + assert!(!values.is_empty()); + assert!(values.len() <= NUM_INNER_LISTS); + for value_index in 1..=values.len() { + inner_offsets[NUM_INNER_LISTS - values.len() + value_index] = value_index as i32; + } + let inner = Arc::new( + ListArray::try_new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(inner_offsets)), + values, + None, + ) + .unwrap(), + ) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new("item", inner.data_type().clone(), true)), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, NUM_INNER_LISTS as i32])), + inner, + None, + ) + .unwrap(), + ) + } + + fn variable_packed_struct_values(num_values: usize) -> (ArrayRef, Arc) { + let fields = Fields::from(vec![ArrowField::new("value", DataType::Utf8, false)]); + let values = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(StringArray::from_iter_values( + (0..num_values).map(|index| format!("value-{index}")), + ))], + None, + )) as ArrayRef; + let item_field = Arc::new( + ArrowField::new("item", DataType::Struct(fields), true).with_metadata(HashMap::from([ + (PACKED_STRUCT_META_KEY.to_string(), "true".to_string()), + ])), + ); + (values, item_field) + } + + fn dictionary_values(num_values: usize) -> (ArrayRef, Arc) { + let keys = Int8Array::from_iter_values((0..num_values).map(|index| (index % 2) as i8)); + let values = Arc::new(StringArray::from(vec!["value-0", "value-1"])); + let dictionary = Arc::new(DictionaryArray::::try_new(keys, values).unwrap()); + let item_field = Arc::new(ArrowField::new( + "item", + dictionary.data_type().clone(), + true, + )); + (dictionary, item_field) + } + + fn large_list_i32(offsets: Vec, validity: Option>) -> ArrayRef { + let num_values = offsets.last().copied().unwrap_or_default(); + let values = Arc::new(Int32Array::from_iter_values(0..num_values as i32)) as ArrayRef; + Arc::new( + LargeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + values, + validity.map(null_buffer), + ) + .unwrap(), + ) + } + + fn map_i32() -> ArrayRef { + let mut builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new()); + builder.keys().append_value("a"); + builder.values().append_value(1); + builder.append(true).unwrap(); + builder.append(false).unwrap(); + builder.append(true).unwrap(); + builder.keys().append_value("b"); + builder.values().append_null(); + builder.keys().append_value("c"); + builder.values().append_value(3); + builder.append(true).unwrap(); + Arc::new(builder.finish()) + } + + fn fixed_size_list_struct() -> ArrayRef { + let fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]); + let child = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::new(Int32Array::from(vec![ + Some(0), + Some(1), + None, + Some(3), + Some(4), + Some(5), + None, + Some(7), + Some(8), + Some(9), + Some(10), + None, + ]))], + Some(null_buffer([ + true, true, false, true, true, true, true, false, true, true, true, true, + ])), + )) as ArrayRef; + Arc::new( + FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Struct(fields), true)), + 2, + child, + Some(null_buffer([true, false, true, true, false, true])), + ) + .unwrap(), + ) + } + + fn list_fixed_size_list_struct() -> ArrayRef { + let struct_fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]); + let structs = Arc::new(StructArray::new( + struct_fields.clone(), + vec![Arc::new(Int32Array::from(vec![ + Some(0), + None, + Some(2), + Some(3), + None, + Some(5), + ]))], + Some(null_buffer([true, true, false, true, true, true])), + )) as ArrayRef; + let fixed_size_list = Arc::new( + FixedSizeListArray::try_new( + Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields), + true, + )), + 2, + structs, + Some(null_buffer([true, false, true])), + ) + .unwrap(), + ) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new( + "item", + fixed_size_list.data_type().clone(), + true, + )), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 1, 1, 3, 3])), + fixed_size_list, + Some(null_buffer([true, false, true, true])), + ) + .unwrap(), + ) + } + + fn nullable_struct() -> ArrayRef { + let fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, true)]); + Arc::new(StructArray::new( + fields, + vec![Arc::new(Int32Array::from(vec![ + Some(10), + Some(20), + None, + Some(40), + Some(50), + ]))], + Some(null_buffer([true, false, true, true, false])), + )) + } + + fn deeply_nested() -> ArrayRef { + let leaf = Arc::new(Int32Array::from(vec![ + Some(0), + None, + Some(2), + Some(3), + None, + Some(5), + Some(6), + Some(7), + ])) as ArrayRef; + let inner = Arc::new( + LargeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i64, 2, 2, 3, 5, 5, 8])), + leaf, + Some(null_buffer([true, false, true, true, true, true])), + ) + .unwrap(), + ) as ArrayRef; + let struct_fields = Fields::from(vec![ArrowField::new( + "inner", + inner.data_type().clone(), + true, + )]); + let structs = Arc::new(StructArray::new( + struct_fields.clone(), + vec![inner], + Some(null_buffer([true, true, false, true, true, true])), + )) as ArrayRef; + Arc::new( + ListArray::try_new( + Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields), + true, + )), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 2, 2, 4, 6])), + structs, + Some(null_buffer([true, false, true, true, true])), + ) + .unwrap(), + ) + } + + fn page_layout(page: &EncodedPage) -> &pb21::page_layout::Layout { + let PageEncoding::Structural(layout) = &page.description else { + panic!("expected structural page encoding"); + }; + layout.layout.as_ref().expect("page layout must be present") + } + + fn create_encoder( + array: &ArrayRef, + version: TestEncoding, + metadata: HashMap, + ) -> Result> { + let arrow_field = + ArrowField::new("values", array.data_type().clone(), true).with_metadata(metadata); + let field = Field::try_from(&arrow_field)?; + let strategy = test_encoding_strategy(version); + let options = EncodingOptions { + cache_bytes_per_column: 1, + ..Default::default() + }; + crate::testing::create_test_field_encoder( + strategy.as_ref(), + &field, + &mut ColumnIndexSequence::default(), + &options, + ) + } + + async fn encode_pages( + array: ArrayRef, + version: TestEncoding, + metadata: HashMap, + ) -> Result> { + encode_chunks(vec![array], version, metadata).await + } + + async fn encode_chunks( + arrays: Vec, + version: TestEncoding, + metadata: HashMap, + ) -> Result> { + let first = arrays + .first() + .ok_or_else(|| Error::internal("test input has no arrays".to_string()))?; + let mut encoder = create_encoder(first, version, metadata)?; + let mut external_buffers = OutOfLineBuffers::new(0, MIN_PAGE_BUFFER_ALIGNMENT); + let mut pages = Vec::new(); + let mut row_number = 0_u64; + for array in arrays { + let num_rows = array.len() as u64; + for task in encoder.maybe_encode( + array, + &mut external_buffers, + crate::repdef::RepDefBuilder::default(), + row_number, + num_rows, + )? { + pages.push(task.await?); + } + row_number += num_rows; + } + for task in encoder.flush(&mut external_buffers)? { + pages.push(task.await?); + } + for column in encoder.finish(&mut external_buffers).await? { + pages.extend(column.final_pages); + } + Ok(pages) + } + + fn sparse_layout(page: &EncodedPage) -> &pb21::SparseLayout { + let pb21::page_layout::Layout::SparseLayout(sparse) = page_layout(page) else { + panic!("expected SparseLayout, got {:?}", page_layout(page)); + }; + sparse + } + + fn list_layer(sparse: &pb21::SparseLayout) -> &pb21::SparseListLayer { + sparse + .structural_layers + .iter() + .find_map(|layer| match layer.layer.as_ref() { + Some(pb21::sparse_structural_layer::Layer::List(layer)) => Some(layer), + _ => None, + }) + .expect("expected sparse list layer") + } + + fn validity_layer(layer: &pb21::SparseStructuralLayer) -> Option<&pb21::SparseValidityLayer> { + match layer.layer.as_ref() { + Some(pb21::sparse_structural_layer::Layer::Validity(layer)) => Some(layer), + _ => None, + } + } + + fn layer_num_slots(layer: &pb21::SparseStructuralLayer) -> u64 { + match layer.layer.as_ref().expect("expected sparse layer variant") { + pb21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::List(layer) => layer.num_slots, + pb21::sparse_structural_layer::Layer::FixedSizeList(layer) => layer.num_slots, + } + } + + fn fixed_size_list_dimension(layer: &pb21::SparseStructuralLayer) -> Option { + match layer.layer.as_ref() { + Some(pb21::sparse_structural_layer::Layer::FixedSizeList(layer)) => { + Some(layer.dimension) + } + _ => None, + } + } + + #[test] + fn test_sparse_value_block_eligibility() { + let fixed = DataBlock::from_array(Int32Array::from_iter_values(0..4)); + assert!(supports_value_block(&fixed)); + + let variable = DataBlock::from_array(StringArray::from(vec!["a", "b"])); + assert!(supports_value_block(&variable)); + + let nullable = DataBlock::from_array(Int32Array::from(vec![Some(1), None])); + assert!(!supports_value_block(&nullable)); + + let all_null = DataBlock::from_array(Int32Array::from(vec![None, None])); + assert!(!supports_value_block(&all_null)); + + let (dictionary, _) = dictionary_values(4); + assert!(!supports_value_block(&DataBlock::from_arrays( + &[dictionary], + 4 + ))); + + let fixed_fields = Fields::from(vec![ArrowField::new("value", DataType::Int32, false)]); + let fixed_struct = StructArray::new( + fixed_fields, + vec![Arc::new(Int32Array::from_iter_values(0..4))], + None, + ); + assert!(supports_value_block(&DataBlock::from_array(fixed_struct))); + + let variable_fields = Fields::from(vec![ArrowField::new("value", DataType::Utf8, false)]); + let variable_struct = StructArray::new( + variable_fields, + vec![Arc::new(StringArray::from(vec!["a", "b"]))], + None, + ); + assert!(!supports_value_block(&DataBlock::from_array( + variable_struct + ))); + + let fixed_size_list = FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, false)), + 2, + Arc::new(Int32Array::from_iter_values(0..4)), + None, + ) + .unwrap(); + assert!(supports_value_block(&DataBlock::from_array( + fixed_size_list + ))); + + let unsupported_fixed_size_list = DataBlock::FixedSizeList(FixedSizeListBlock { + child: Box::new(DataBlock::from_array(StringArray::from(vec![ + "a", "b", "c", "d", + ]))), + dimension: 2, + }); + assert!(!supports_value_block(&unsupported_fixed_size_list)); + } + + fn planned_list( + offsets: Vec, + list_validity: Option>, + leaf_validity: Option>, + ) -> SparseStructuralPlan { + let num_values = u64::try_from(*offsets.last().unwrap()).unwrap(); + let mut builder = crate::repdef::RepDefBuilder::default(); + assert!(!builder.add_offsets( + OffsetBuffer::new(ScalarBuffer::from(offsets)), + list_validity.map(null_buffer), + )); + if let Some(leaf_validity) = leaf_validity { + builder.add_validity_bitmap(null_buffer(leaf_validity)); + } else { + builder.add_no_null(num_values as usize); + } + let normalized = crate::repdef::RepDefBuilder::normalize(vec![builder]); + plan(&normalized, num_values).unwrap() + } + + fn planned_list_layer(plan: &SparseStructuralPlan) -> &SparseStructuralLayerPlan { + plan.layers + .iter() + .find(|layer| matches!(layer, SparseStructuralLayerPlan::List { .. })) + .expect("expected planned list layer") + } + + #[test] + fn test_semantic_position_and_count_forms() { + let empty = planned_list(vec![0, 0, 0, 0], None, None); + assert_eq!(empty.num_items, 3); + assert_eq!(empty.num_visible_items, 0); + assert!(matches!( + planned_list_layer(&empty), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::Empty, + counts: SparseCountSet::Empty, + .. + } + )); + + let all = planned_list(vec![0, 2, 4, 6], None, None); + assert_eq!(all.num_items, 6); + assert!(matches!( + planned_list_layer(&all), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::All { len: 3 }, + counts: SparseCountSet::Constant { value: 2, len: 3 }, + .. + } + )); + + let range = planned_list(vec![0, 2, 4, 4, 4], None, None); + assert_eq!(range.num_items, 6); + assert!(matches!( + planned_list_layer(&range), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::Range { start: 0, len: 2 }, + counts: SparseCountSet::Constant { value: 2, len: 2 }, + .. + } + )); + + let explicit = planned_list(vec![0, 1, 1, 4, 4, 6], None, None); + assert_eq!(explicit.num_items, 8); + assert!(matches!( + planned_list_layer(&explicit), + SparseStructuralLayerPlan::List { + non_empty_positions: SparsePositionSet::Explicit(positions), + counts: SparseCountSet::Explicit { counts, .. }, + .. + } if positions == &vec![0, 2, 4] && counts.as_ref() == [1, 3, 2] + )); + } + + #[test] + fn test_validity_polarity_uses_semantic_encoded_cost() { + let mostly_valid = BooleanBuffer::from_iter([true, false, true, true, false, true]); + let validity = validity_set(Some(&mostly_valid), mostly_valid.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::NullPositions); + assert!(matches!(validity.positions, SparsePositionSet::Explicit(ref p) if p == &[1, 4])); + + let mostly_null = BooleanBuffer::from_iter([false, true, false, false, true, false]); + let validity = validity_set(Some(&mostly_null), mostly_null.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions); + assert!(matches!(validity.positions, SparsePositionSet::Explicit(ref p) if p == &[1, 4])); + + let valid_island = BooleanBuffer::from_iter([false, false, true, true, false]); + let validity = validity_set(Some(&valid_island), valid_island.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions); + assert!(matches!( + validity.positions, + SparsePositionSet::Range { start: 2, len: 2 } + )); + + let all_valid = BooleanBuffer::from_iter([true, true, true]); + let validity = validity_set(Some(&all_valid), all_valid.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::NullPositions); + assert!(matches!(validity.positions, SparsePositionSet::Empty)); + + let all_null = BooleanBuffer::from_iter([false, false, false]); + let validity = validity_set(Some(&all_null), all_null.len(), "test").unwrap(); + assert_eq!(validity.meaning, SparseValidityMeaning::ValidPositions); + assert!(matches!(validity.positions, SparsePositionSet::Empty)); + } + + #[tokio::test] + async fn test_explicit_sparse_nullable_primitive_roundtrip() { + let array = Arc::new(Int32Array::from(vec![ + Some(10), + None, + Some(20), + Some(30), + None, + Some(40), + ])) as ArrayRef; + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert_eq!(pages.len(), 1); + let sparse = sparse_layout(&pages[0]); + assert_eq!(sparse.num_items, 6); + assert_eq!(sparse.num_visible_items, 6); + assert_eq!(sparse.structural_layers.len(), 1); + let validity = validity_layer(&sparse.structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32 + ); + assert!(matches!( + validity.positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::Explicit(_)) + )); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 5]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_nullable_struct_roundtrip() { + let array = nullable_struct(); + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert_eq!(pages.len(), 1); + let sparse = sparse_layout(&pages[0]); + assert_eq!(sparse.structural_layers.len(), 2); + assert!( + sparse + .structural_layers + .iter() + .all(|layer| validity_layer(layer).is_some()) + ); + let struct_validity = validity_layer(&sparse.structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + struct_validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32 + ); + assert!(matches!( + struct_validity.positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::Explicit(_)) + )); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 4]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_struct_with_constant_and_sparse_children() { + let fields = Fields::from(vec![ + ArrowField::new("constant", DataType::Int32, true), + ArrowField::new("sparse", DataType::Int32, true), + ]); + let array = Arc::new(StructArray::new( + fields, + vec![ + Arc::new(Int32Array::from(vec![None::; 5])), + Arc::new(Int32Array::from(vec![ + Some(10), + Some(20), + None, + Some(40), + Some(50), + ])), + ], + Some(null_buffer([true, false, true, true, false])), + )) as ArrayRef; + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.iter().any(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::ConstantLayout(_) + ))); + assert!(pages.iter().any(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::SparseLayout(_) + ))); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 4]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_emits_both_validity_polarities() { + let mostly_valid = Arc::new(Int32Array::from(vec![ + Some(0), + None, + Some(2), + Some(3), + None, + Some(5), + ])) as ArrayRef; + let mostly_null = Arc::new(Int32Array::from(vec![ + None, + Some(1), + None, + None, + Some(4), + None, + ])) as ArrayRef; + + let null_positions = encode_pages( + mostly_valid.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let validity = validity_layer(&sparse_layout(&null_positions[0]).structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityNullPositions as i32 + ); + + let valid_positions = encode_pages( + mostly_null.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let validity = validity_layer(&sparse_layout(&valid_positions[0]).structural_layers[0]) + .unwrap() + .validity + .as_ref() + .unwrap(); + assert_eq!( + validity.meaning, + pb21::sparse_validity_set::Meaning::SparseValidityValidPositions as i32 + ); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..5) + .with_indices(vec![0, 2, 5]); + for array in [mostly_valid, mostly_null] { + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + } + + #[tokio::test] + async fn test_explicit_sparse_nested_page_boundaries_range_and_take() { + let nested = deeply_nested(); + let chunks = vec![nested.slice(0, 2), nested.slice(2, 3)]; + let pages = encode_chunks( + chunks.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.len() >= 2, "expected multiple sparse pages"); + let sparse = pages + .iter() + .map(sparse_layout) + .collect::>(); + assert!(sparse.iter().any(|layout| { + layout + .structural_layers + .iter() + .filter(|layer| { + matches!( + layer.layer.as_ref(), + Some(pb21::sparse_structural_layer::Layer::List(_)) + ) + }) + .count() + >= 2 + })); + assert!(sparse.iter().any(|layout| { + layout + .structural_layers + .iter() + .any(|layer| validity_layer(layer).is_some()) + })); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_batch_size(2) + .with_range(1..5) + .with_range(2..4) + .with_indices(vec![0, 2, 4]) + .with_indices(vec![1, 3]); + check_round_trip_encoding_of_data(chunks, &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_list_and_large_list_null_empty_roundtrip() { + let list = list_i32( + vec![0, 0, 0, 2, 3, 3], + Some(vec![false, true, true, true, true]), + ); + let large_list = large_list_i32( + vec![0, 0, 0, 2, 3, 3], + Some(vec![false, true, true, true, true]), + ); + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(0..5) + .with_range(1..4) + .with_indices(vec![0, 1, 4]) + .with_indices(vec![2, 3]); + + for array in [list, large_list] { + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.iter().map(sparse_layout).all(|layout| { + layout.structural_layers.iter().any(|layer| { + matches!( + layer.layer.as_ref(), + Some(pb21::sparse_structural_layer::Layer::List(_)) + ) + }) + })); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + } + + #[tokio::test] + async fn test_explicit_sparse_map_and_fixed_size_list_roundtrip() { + let map = map_i32(); + let map_pages = encode_pages( + map.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(map_pages.iter().map(sparse_layout).any(|layout| { + layout.structural_layers.iter().any(|layer| { + matches!( + layer.layer.as_ref(), + Some(pb21::sparse_structural_layer::Layer::List(_)) + ) + }) + })); + let map_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..4) + .with_indices(vec![0, 2, 3]); + check_round_trip_encoding_of_data(vec![map], &map_cases, sparse_metadata()).await; + + let fsl = fixed_size_list_struct(); + let fsl_pages = encode_pages( + fsl.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + for page in &fsl_pages { + let layout = sparse_layout(page); + let outer_slots = layer_num_slots(layout.structural_layers.first().unwrap()); + let fixed_size_scale = layout + .structural_layers + .iter() + .filter_map(fixed_size_list_dimension) + .product::(); + assert_eq!(page.num_rows, outer_slots * fixed_size_scale); + } + assert!(fsl_pages.iter().map(sparse_layout).any(|layout| { + layout + .structural_layers + .iter() + .any(|layer| fixed_size_list_dimension(layer) == Some(2)) + })); + let fsl_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..6) + .with_indices(vec![0, 3, 5]); + check_round_trip_encoding_of_data(vec![fsl], &fsl_cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_list_fixed_size_list_struct_roundtrip() { + let array = list_fixed_size_list_struct(); + let pages = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(pages.iter().map(sparse_layout).any(|layout| { + let kinds = layout + .structural_layers + .iter() + .map(|layer| match layer.layer.as_ref().unwrap() { + pb21::sparse_structural_layer::Layer::Validity(_) => "validity", + pb21::sparse_structural_layer::Layer::List(_) => "list", + pb21::sparse_structural_layer::Layer::FixedSizeList(_) => "fixed-size-list", + }) + .collect::>(); + kinds.starts_with(&["list", "fixed-size-list"]) + })); + for page in &pages { + let layout = sparse_layout(page); + let outer_slots = layer_num_slots(layout.structural_layers.first().unwrap()); + assert_eq!(page.num_rows, outer_slots * 2); + } + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..4) + .with_indices(vec![0, 2, 3]) + .with_indices(vec![1, 3]); + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + + #[tokio::test] + async fn test_explicit_sparse_serializes_semantic_list_forms() { + let all_array = list_i32(vec![0, 2, 4, 6], None); + let all = encode_pages( + all_array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let all_layer = list_layer(sparse_layout(&all[0])); + assert!(matches!( + all_layer.non_empty_positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::All(_)) + )); + assert!(matches!( + all_layer.counts.as_ref().unwrap().counts, + Some(pb21::sparse_count_set::Counts::Constant( + pb21::SparseCountConstant { value: 2 } + )) + )); + assert_eq!(all[0].data.len(), 2); + + let range_array = list_i32(vec![0, 2, 4, 4, 4], None); + let range = encode_pages( + range_array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let range_layer = list_layer(sparse_layout(&range[0])); + assert!(matches!( + range_layer.non_empty_positions.as_ref().unwrap().positions, + Some(pb21::sparse_position_set::Positions::Range( + pb21::SparsePositionRange { + start: 0, + length: 2 + } + )) + )); + assert!(matches!( + range_layer.counts.as_ref().unwrap().counts, + Some(pb21::sparse_count_set::Counts::Constant( + pb21::SparseCountConstant { value: 2 } + )) + )); + assert_eq!(range[0].data.len(), 2); + + let explicit_array = list_i32(vec![0, 1, 1, 4, 4, 6], None); + let explicit = encode_pages( + explicit_array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + let explicit_layer = list_layer(sparse_layout(&explicit[0])); + assert!(matches!( + explicit_layer + .non_empty_positions + .as_ref() + .unwrap() + .positions, + Some(pb21::sparse_position_set::Positions::Explicit(_)) + )); + assert!(matches!( + explicit_layer.counts.as_ref().unwrap().counts, + Some(pb21::sparse_count_set::Counts::Explicit(_)) + )); + assert_eq!(explicit[0].data.len(), 4); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1]) + .with_range(1..3) + .with_indices(vec![0, 2]); + for array in [all_array, range_array, explicit_array] { + check_round_trip_encoding_of_data(vec![array], &cases, sparse_metadata()).await; + } + } + + #[tokio::test] + async fn test_constant_layout_boundary_is_explicit() { + let structural_only = encode_pages( + list_i32(vec![0, 0, 0, 0], None), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&structural_only[0]), + pb21::page_layout::Layout::ConstantLayout(_) + )); + + let empty_struct = Arc::new(StructArray::new_empty_fields(3, None)) as ArrayRef; + let empty_struct_pages = encode_pages( + empty_struct, + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&empty_struct_pages[0]), + pb21::page_layout::Layout::ConstantLayout(_) + )); + + let all_null = Arc::new(Int32Array::from(vec![None, None, None])) as ArrayRef; + let all_null_pages = + encode_pages(all_null, TestEncoding::StructuralSparse, sparse_metadata()) + .await + .unwrap(); + assert!(matches!( + page_layout(&all_null_pages[0]), + pb21::page_layout::Layout::ConstantLayout(_) + )); + + let constant = Arc::new(Int32Array::from(vec![7, 7, 7])) as ArrayRef; + let constant_pages = + encode_pages(constant, TestEncoding::StructuralSparse, sparse_metadata()) + .await + .unwrap(); + assert!(matches!( + page_layout(&constant_pages[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + } + + #[tokio::test] + async fn test_default_and_explicit_dense_layouts_are_unchanged() { + let array = Arc::new(Int32Array::from_iter_values(0..16)) as ArrayRef; + let v2_2_default = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + let v2_2_miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralU32, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert_eq!(v2_2_default.len(), v2_2_miniblock.len()); + for (default, explicit) in v2_2_default.iter().zip(v2_2_miniblock.iter()) { + assert!(matches!( + page_layout(default), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + assert_eq!(page_layout(default), page_layout(explicit)); + assert_eq!(default.data, explicit.data); + } + + let v2_3_default = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_default[0]), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + + let v2_3_miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_miniblock[0]), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + + let v2_3_fullzip = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_FULLZIP), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_fullzip[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let v2_3_sparse = encode_pages(array, TestEncoding::StructuralSparse, sparse_metadata()) + .await + .unwrap(); + assert!(matches!( + page_layout(&v2_3_sparse[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + } + + #[tokio::test] + async fn test_auto_sparse_for_split_required_page() { + const NUM_ROWS: usize = 70_000; + let array = sparse_i32_list(NUM_ROWS, 2_000); + + let within_budget = encode_pages( + sparse_i32_list(4_096, 1_024), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(within_budget.len(), 1); + assert!(matches!( + page_layout(&within_budget[0]), + pb21::page_layout::Layout::MiniBlockLayout(_) + )); + + let automatic = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(automatic.len(), 1); + assert!(matches!( + page_layout(&automatic[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert!(miniblock.len() > 1); + assert!(miniblock.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::MiniBlockLayout(_) + ))); + + let fullzip = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_FULLZIP), + ) + .await + .unwrap(); + assert_eq!(fullzip.len(), miniblock.len()); + assert!(fullzip.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::FullZipLayout(_) + ))); + + let sparse = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert_eq!(sparse.len(), 1); + assert!(matches!( + page_layout(&sparse[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let v2_2_default = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + let v2_2_miniblock = encode_pages( + array.clone(), + TestEncoding::StructuralU32, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap(); + assert_eq!(v2_2_default.len(), v2_2_miniblock.len()); + for (default, explicit) in v2_2_default.iter().zip(v2_2_miniblock.iter()) { + assert_eq!(page_layout(default), page_layout(explicit)); + assert_eq!(default.data, explicit.data); + } + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_batch_size(NUM_ROWS as u32) + .with_range(1_999..2_002) + .with_indices(vec![0, 1_999, 2_000, NUM_ROWS as u64 - 1]); + check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await; + } + + #[tokio::test] + async fn test_auto_sparse_for_single_row_over_budget() { + let array = unsplittable_nested_list( + Arc::new(Int32Array::from(vec![42, 43])), + Arc::new(ArrowField::new("item", DataType::Int32, true)), + ); + + let automatic = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(automatic.len(), 1); + assert!(matches!( + page_layout(&automatic[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let v2_2 = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + assert_eq!(v2_2.len(), 1); + assert!(matches!( + page_layout(&v2_2[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let miniblock_error = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_MINIBLOCK), + ) + .await + .unwrap_err(); + assert!( + miniblock_error + .to_string() + .contains("Mini-block cannot encode 70000 rep/def levels") + ); + + let fullzip = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + structural_metadata(STRUCTURAL_ENCODING_FULLZIP), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&fullzip[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let explicit_sparse = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap(); + assert!(matches!( + page_layout(&explicit_sparse[0]), + pb21::page_layout::Layout::SparseLayout(_) + )); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_range(0..1) + .with_indices(vec![0]); + check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await; + } + + #[tokio::test] + async fn test_auto_sparse_keeps_unsupported_values_dense() { + const NUM_ROWS: usize = 70_000; + let num_values = NUM_ROWS.div_ceil(2_000); + + let (dictionary, dictionary_field) = dictionary_values(num_values); + let dictionary_array = sparse_list_values(NUM_ROWS, 2_000, dictionary, dictionary_field); + let dictionary_pages = encode_pages( + dictionary_array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert!(dictionary_pages.len() > 1); + assert!(dictionary_pages.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::MiniBlockLayout(_) + ))); + + let (packed_values, packed_field) = variable_packed_struct_values(num_values); + let packed_array = sparse_list_values(NUM_ROWS, 2_000, packed_values, packed_field); + let packed_pages = encode_pages( + packed_array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert!(packed_pages.len() > 1); + assert!(packed_pages.iter().all(|page| matches!( + page_layout(page), + pb21::page_layout::Layout::FullZipLayout(_) + ))); + + let (packed_values, packed_field) = variable_packed_struct_values(2); + let unsplittable_packed = unsplittable_nested_list(packed_values, packed_field); + let packed_fallback = encode_pages( + unsplittable_packed.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + assert_eq!(packed_fallback.len(), 1); + assert!(matches!( + page_layout(&packed_fallback[0]), + pb21::page_layout::Layout::FullZipLayout(_) + )); + + let (dictionary, dictionary_field) = dictionary_values(2); + let unsplittable_dictionary = unsplittable_nested_list(dictionary, dictionary_field); + let v2_2_error = encode_pages( + unsplittable_dictionary.clone(), + TestEncoding::StructuralU32, + HashMap::new(), + ) + .await + .unwrap_err(); + let v2_3_error = encode_pages( + unsplittable_dictionary, + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap_err(); + assert_eq!(v2_3_error.to_string(), v2_2_error.to_string()); + assert!( + v2_3_error + .to_string() + .contains("Mini-block cannot encode 70000 rep/def levels") + ); + + let cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_batch_size(NUM_ROWS as u32) + .with_range(1_999..2_002) + .with_indices(vec![0, 2_000, NUM_ROWS as u64 - 1]); + check_round_trip_encoding_of_data(vec![dictionary_array], &cases, HashMap::new()).await; + let packed_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_batch_size(NUM_ROWS as u32); + check_round_trip_encoding_of_data(vec![packed_array], &packed_cases, HashMap::new()).await; + + let single_row_cases = TestCases::default() + .with_encoding(TestEncoding::StructuralSparse) + .with_page_sizes(vec![1024 * 1024]) + .with_range(0..1) + .with_indices(vec![0]); + check_round_trip_encoding_of_data( + vec![unsplittable_packed], + &single_row_cases, + HashMap::new(), + ) + .await; + } + + #[tokio::test] + async fn test_auto_sparse_wide_values_keep_dense_fallback() { + let first = vec![0xAB_u8; 5_000]; + let second = vec![0xCD_u8; 5_000]; + let fixed_size_binary = Arc::new( + FixedSizeBinaryArray::try_from_sparse_iter_with_size( + [Some(first.as_slice()), Some(second.as_slice())].into_iter(), + 5_000, + ) + .unwrap(), + ) as ArrayRef; + + const FSL_DIMENSION: i32 = 2_048; + let fixed_size_list = Arc::new( + FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Int32, true)), + FSL_DIMENSION, + Arc::new(Int32Array::from_iter_values(0..(FSL_DIMENSION * 2))), + None, + ) + .unwrap(), + ) as ArrayRef; + + for (label, values) in [ + ("fixed-size binary", fixed_size_binary), + ("fixed-size list", fixed_size_list), + ] { + let item_field = Arc::new(ArrowField::new("item", values.data_type().clone(), true)); + let array = unsplittable_nested_list(values, item_field); + + let v2_2 = encode_pages(array.clone(), TestEncoding::StructuralU32, HashMap::new()) + .await + .unwrap(); + let v2_3 = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + HashMap::new(), + ) + .await + .unwrap(); + for pages in [&v2_2, &v2_3] { + assert_eq!(pages.len(), 1, "unexpected {label} page count"); + assert!( + matches!( + page_layout(&pages[0]), + pb21::page_layout::Layout::FullZipLayout(_) + ), + "{label} should retain the dense full-zip fallback" + ); + } + + let explicit_error = encode_pages( + array.clone(), + TestEncoding::StructuralSparse, + sparse_metadata(), + ) + .await + .unwrap_err(); + assert!( + explicit_error.to_string().contains("too wide"), + "explicit sparse should preserve the {label} value error: {explicit_error}" + ); + + let cases = TestCases::default() + .with_u32_structural_encodings() + .with_range(0..1) + .with_indices(vec![0]); + check_round_trip_encoding_of_data(vec![array], &cases, HashMap::new()).await; + } + } + + #[test] + fn test_explicit_sparse_rejects_lance_2_2() { + let array = Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef; + let Err(error) = create_encoder(&array, TestEncoding::StructuralU32, sparse_metadata()) + else { + panic!("expected Lance 2.2 to reject explicit sparse encoding"); + }; + assert!( + error + .to_string() + .contains("not enabled by the selected file format") + ); + + let structural_only = list_i32(vec![0, 0, 0], None); + let Err(error) = create_encoder( + &structural_only, + TestEncoding::StructuralU32, + sparse_metadata(), + ) else { + panic!("expected Lance 2.2 structural-only input to reject explicit sparse encoding"); + }; + assert!( + error + .to_string() + .contains("not enabled by the selected file format") + ); + } +} diff --git a/rust/lance-encoding/src/encodings/logical/struct.rs b/rust/lance-encoding/src/encodings/logical/struct.rs index e8c6289275a..c281c971c14 100644 --- a/rust/lance-encoding/src/encodings/logical/struct.rs +++ b/rust/lance-encoding/src/encodings/logical/struct.rs @@ -31,7 +31,7 @@ use futures::{ use itertools::Itertools; use lance_arrow::FieldExt; use lance_arrow::{deepcopy::deep_copy_nulls, r#struct::StructArrayExt}; -use lance_core::{Error, Result}; +use lance_core::{Error, Result, datatypes::validate_fixed_size_list_dimensions}; use log::trace; #[derive(Debug)] @@ -276,6 +276,12 @@ impl StructuralStructDecoder { DataType::FixedSizeList(child_field, _) if matches!(child_field.data_type(), DataType::Struct(_)) => { + // The scheduler factories run the same guard, but the decoder tree can be + // built independently (e.g. `create_decode_stream`) so a zero dimension from + // a malformed schema must be rejected here as well. Draining and unraveling + // validity both scale by the dimension and a zero would make that math + // degenerate. + validate_fixed_size_list_dimensions(field.name(), field.data_type())?; // FixedSizeList containing Struct needs structural decoding let child_decoder = Self::field_to_decoder(child_field, should_validate)?; Ok(Box::new(StructuralFixedSizeListDecoder::new( @@ -368,26 +374,57 @@ impl StructuralDecodeArrayTask for RepDefStructDecodeTask { .map(|task| task.decode()) .collect::>>()?; let mut children = Vec::with_capacity(arrays.len()); + let mut repdefs = Vec::with_capacity(arrays.len()); let mut data_size = 0u64; let mut arrays_iter = arrays.into_iter(); - let first_array = arrays_iter.next().unwrap(); + let first_array = arrays_iter.next().ok_or_else(|| { + Error::internal("Struct decoder unexpectedly has no child arrays".to_string()) + })?; let length = first_array.array.len(); // The repdef should be identical across all children at this point - let mut repdef = first_array.repdef; + repdefs.push(first_array.repdef); data_size += first_array.data_size; children.push(first_array.array); for array in arrays_iter { - debug_assert_eq!(length, array.array.len()); + if length != array.array.len() { + return Err(Error::invalid_input_source( + format!( + "Struct child array length {} does not match sibling length {}", + array.array.len(), + length + ) + .into(), + )); + } data_size += array.data_size; children.push(array.array); + repdefs.push(array.repdef); + } + + // Dense rep/def state can retain child-specific repetition information after a child + // decoder finishes, so comparing dense siblings is not meaningful. If any child carries + // sparse state, keep a sparse child as the canonical structural plan and compare it with + // every other sparse sibling so sparse metadata is never silently discarded. + let primary_repdef = repdefs + .iter() + .position(CompositeRepDefUnraveler::has_sparse) + .unwrap_or(0); + let mut repdef = repdefs.swap_remove(primary_repdef); + if repdef.has_sparse() { + for sibling in repdefs { + if sibling.has_sparse() { + repdef.add_compatibility_check(sibling); + } + } } let validity = if self.is_root { + repdef.ensure_exhausted()?; None } else { - repdef.unravel_validity(length) + repdef.unravel_validity(length)? }; let array = StructArray::try_new(self.child_fields, children, validity) @@ -601,19 +638,55 @@ mod tests { use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields}; - use crate::{ - testing::{TestCases, check_basic_random, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + use super::StructuralStructDecoder; + use crate::testing::{ + TestCases, TestEncoding, check_basic_random_case, check_round_trip_encoding_of_data, }; + #[test] + fn test_zero_dimension_fsl_decoder_errors() { + // Simulates a stored schema declaring a zero-dimension FixedSizeList (writers reject + // it but old files may contain one). Building the decoder must fail cleanly instead + // of letting the zero dimension reach the rep/def decimation. + let item_fields = Fields::from(vec![Field::new("x", DataType::Int32, true)]); + let fields = Fields::from(vec![Field::new( + "vecs", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Struct(item_fields), true)), + 0, + ), + true, + )]); + + let err = StructuralStructDecoder::new(fields, false, /*is_root=*/ true).unwrap_err(); + assert!(matches!(err, lance_core::Error::Schema { .. })); + assert!( + err.to_string() + .contains("dimension must be a positive integer"), + "unexpected error: {}", + err + ); + } + + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_simple_struct() { + async fn test_simple_struct( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let data_type = DataType::Struct(Fields::from(vec![ Field::new("a", DataType::Int32, false), Field::new("b", DataType::Int32, false), ])); let field = Field::new("", data_type, false); - check_basic_random(field).await; + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] @@ -663,7 +736,7 @@ mod tests { Some(rows_validity), ); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(rows)], &test_cases, HashMap::new()).await; } @@ -693,7 +766,7 @@ mod tests { ); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_structural_encodings(), HashMap::new(), ) .await; @@ -724,14 +797,25 @@ mod tests { ); check_round_trip_encoding_of_data( vec![Arc::new(struct_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_structural_encodings(), HashMap::new(), ) .await; } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_struct_list() { + async fn test_struct_list( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let data_type = DataType::Struct(Fields::from(vec![ Field::new( "inner_list", @@ -741,20 +825,42 @@ mod tests { Field::new("outer_int", DataType::Int32, true), ])); let field = Field::new("row", data_type, false); - check_basic_random(field).await; + check_basic_random_case(field, encoding, page_size, use_slicing).await; } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_empty_struct() { + async fn test_empty_struct( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { // It's technically legal for a struct to have 0 children, need to // make sure we support that let data_type = DataType::Struct(Fields::from(Vec::::default())); let field = Field::new("row", data_type, false); - check_basic_random(field).await; + check_basic_random_case(field, encoding, page_size, use_slicing).await; } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_complicated_struct() { + async fn test_complicated_struct( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let data_type = DataType::Struct(Fields::from(vec![ Field::new("int", DataType::Int32, true), Field::new( @@ -772,7 +878,7 @@ mod tests { Field::new("outer_binary", DataType::Binary, true), ])); let field = Field::new("row", data_type, false); - check_basic_random(field).await; + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[test_log::test(tokio::test)] @@ -803,7 +909,7 @@ mod tests { check_round_trip_encoding_of_data( vec![Arc::new(list_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_2), + &TestCases::default().with_u32_structural_encodings(), HashMap::new(), ) .await; @@ -837,7 +943,7 @@ mod tests { .with_range(1..2) .with_indices(vec![0]) .with_indices(vec![1]) - .with_min_file_version(LanceFileVersion::V2_1), + .with_structural_encodings(), HashMap::new(), ) .await; @@ -885,7 +991,7 @@ mod tests { check_round_trip_encoding_of_data( vec![Arc::new(row_array)], - &TestCases::default().with_min_file_version(LanceFileVersion::V2_1), + &TestCases::default().with_structural_encodings(), HashMap::new(), ) .await; diff --git a/rust/lance-encoding/src/encodings/physical/binary.rs b/rust/lance-encoding/src/encodings/physical/binary.rs index d02cf2da693..e07d586a4f6 100644 --- a/rust/lance-encoding/src/encodings/physical/binary.rs +++ b/rust/lance-encoding/src/encodings/physical/binary.rs @@ -15,13 +15,15 @@ use core::panic; use crate::compression::{ BlockCompressor, BlockDecompressor, MiniBlockDecompressor, VariablePerValueDecompressor, + require_block_payload, }; use crate::buffer::LanceBuffer; use crate::data::{BlockInfo, DataBlock, VariableWidthBlock}; use crate::encodings::logical::primitive::fullzip::{PerValueCompressor, PerValueDataBlock}; use crate::encodings::logical::primitive::miniblock::{ - MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressor, + MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressionContext, + MiniBlockCompressor, }; use crate::format::pb21::CompressiveEncoding; use crate::format::pb21::compressive_encoding::Compression; @@ -245,7 +247,11 @@ impl BinaryMiniBlockEncoder { } impl MiniBlockCompressor for BinaryMiniBlockEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match data { DataBlock::VariableWidth(variable_width) => Ok(self.chunk_data(variable_width)), _ => Err(Error::invalid_input_source( @@ -288,52 +294,144 @@ impl BinaryMiniBlockDecompressor { } } +/// Cold path: pinpoint why the chunk-relative offsets of a binary mini-block +/// chunk failed validation. +fn chunk_offset_violation_error>(offsets: &[T], chunk_len: usize) -> Error { + let mut previous: u64 = offsets[0].into(); + for (position, &offset) in offsets.iter().enumerate().skip(1) { + let offset: u64 = offset.into(); + if offset < previous { + return Error::corrupt_file_named( + "binary mini-block", + format!( + "value offset at position {position} decreases: {offset} < {previous} \ + (chunk is {chunk_len} bytes)" + ), + ); + } + previous = offset; + } + Error::corrupt_file_named( + "binary mini-block", + format!("value offset {previous} is out of bounds for a chunk of {chunk_len} bytes"), + ) +} + impl MiniBlockDecompressor for BinaryMiniBlockDecompressor { // decompress a MiniBlock of binary data, the num_values must be less than or equal // to the number of values this MiniBlock has, BinaryMiniBlock doesn't store `the number of values` // it has so assertion can not be done here and the caller of `decompress` must ensure // `num_values` <= number of values in the chunk. + // + // The chunk-relative value offsets at the front of the chunk come straight + // from the file and are used to slice the chunk buffer, so corrupt values + // must surface as a typed error instead of a panic or an out-of-bounds + // read. The monotonicity check rides along the existing rebase loop (the + // `&=` accumulation keeps it branchless) so validation adds no extra pass. fn decompress(&self, data: Vec, num_values: u64) -> Result { assert_eq!(data.len(), 1); let data = data.into_iter().next().unwrap(); - if self.bits_per_offset == 64 { - // offset and at least one value - assert!(data.len() >= 16); + let bytes_per_offset = self.bits_per_offset as usize / 8; + if !data.len().is_multiple_of(bytes_per_offset) { + return Err(Error::corrupt_file_named( + "binary mini-block", + format!( + "chunk size {} is not a multiple of the {}-byte offset width", + data.len(), + bytes_per_offset + ), + )); + } + let num_offsets = (num_values as usize).checked_add(1).ok_or_else(|| { + Error::corrupt_file_named( + "binary mini-block", + format!("cannot decode {num_values} values from a single chunk"), + ) + })?; + if data.len() / bytes_per_offset < num_offsets { + return Err(Error::corrupt_file_named( + "binary mini-block", + format!( + "chunk of {} bytes holds {} offsets but decoding {} values requires {}", + data.len(), + data.len() / bytes_per_offset, + num_values, + num_offsets + ), + )); + } + // The value region must start past the offsets being decoded, otherwise + // the offset table itself aliases into the value bytes. A lower bound + // (not equality) because a prefix read of the chunk legitimately leaves + // unrequested offsets between the requested prefix and the values. + let min_value_region_start = num_offsets * bytes_per_offset; + let value_region_overlap_error = |first: u64| { + Error::corrupt_file_named( + "binary mini-block", + format!( + "value region starts at offset {first} which overlaps the {num_offsets} \ + requested offsets ({min_value_region_start} bytes)" + ), + ) + }; + + if self.bits_per_offset == 64 { let offsets_buffer = data.borrow_to_typed_slice::(); - let offsets = offsets_buffer.as_ref(); + let offsets = &offsets_buffer.as_ref()[..num_offsets]; - let result_offsets = offsets[0..(num_values + 1) as usize] + let first = offsets[0]; + if first < min_value_region_start as u64 { + return Err(value_region_overlap_error(first)); + } + let mut previous = first; + let mut is_monotonic = true; + let result_offsets = offsets .iter() - .map(|offset| offset - offsets[0]) + .map(|&offset| { + is_monotonic &= previous <= offset; + previous = offset; + offset.wrapping_sub(first) + }) .collect::>(); + let last = offsets[num_offsets - 1]; + if !is_monotonic || last as usize > data.len() { + return Err(chunk_offset_violation_error(offsets, data.len())); + } Ok(DataBlock::VariableWidth(VariableWidthBlock { - data: LanceBuffer::from( - data[offsets[0] as usize..offsets[num_values as usize] as usize].to_vec(), - ), + data: LanceBuffer::from(data[first as usize..last as usize].to_vec()), offsets: LanceBuffer::reinterpret_vec(result_offsets), bits_per_offset: 64, num_values, block_info: BlockInfo::new(), })) } else { - // offset and at least one value - assert!(data.len() >= 8); - let offsets_buffer = data.borrow_to_typed_slice::(); - let offsets = offsets_buffer.as_ref(); + let offsets = &offsets_buffer.as_ref()[..num_offsets]; - let result_offsets = offsets[0..(num_values + 1) as usize] + let first = offsets[0]; + if (first as u64) < min_value_region_start as u64 { + return Err(value_region_overlap_error(first as u64)); + } + let mut previous = first; + let mut is_monotonic = true; + let result_offsets = offsets .iter() - .map(|offset| offset - offsets[0]) + .map(|&offset| { + is_monotonic &= previous <= offset; + previous = offset; + offset.wrapping_sub(first) + }) .collect::>(); + let last = offsets[num_offsets - 1]; + if !is_monotonic || last as usize > data.len() { + return Err(chunk_offset_violation_error(offsets, data.len())); + } Ok(DataBlock::VariableWidth(VariableWidthBlock { - data: LanceBuffer::from( - data[offsets[0] as usize..offsets[num_values as usize] as usize].to_vec(), - ), + data: LanceBuffer::from(data[first as usize..last as usize].to_vec()), offsets: LanceBuffer::reinterpret_vec(result_offsets), bits_per_offset: 32, num_values, @@ -356,7 +454,15 @@ impl MiniBlockDecompressor for BinaryMiniBlockDecompressor { pub struct VariableEncoder {} impl BlockCompressor for VariableEncoder { - fn compress(&self, mut data: DataBlock) -> Result { + fn compress(&self, mut data: DataBlock) -> Result<(Option, CompressiveEncoding)> { + let bits_per_offset = match &data { + DataBlock::VariableWidth(data) => data.bits_per_offset, + _ => { + return Err(Error::invalid_input( + "BinaryBlockEncoder requires a variable-width block", + )); + } + }; match data { DataBlock::VariableWidth(ref mut variable_width_data) => { match variable_width_data.bits_per_offset { @@ -408,18 +514,23 @@ impl BlockCompressor for VariableEncoder { output.extend_from_slice(&variable_width_data.data); Ok(LanceBuffer::from(output)) } - _ => { - panic!( - "BinaryBlockEncoder does not work with {} bits per offset VariableWidth DataBlock.", - variable_width_data.bits_per_offset - ); - } + _ => Err(Error::invalid_input(format!( + "BinaryBlockEncoder does not support {}-bit offsets", + variable_width_data.bits_per_offset + ))), } } - _ => { - panic!("BinaryBlockEncoder can only work with Variable Width DataBlock."); - } + _ => unreachable!("variable-width input was validated above"), } + .map(|payload| { + ( + Some(payload), + ProtobufUtils21::variable( + ProtobufUtils21::flat(bits_per_offset as u64, None), + None, + ), + ) + }) } } @@ -450,7 +561,8 @@ impl VariablePerValueDecompressor for VariableDecoder { pub struct BinaryBlockDecompressor {} impl BlockDecompressor for BinaryBlockDecompressor { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "Binary block")?; // In older (not quite stable) versions we stored the bits per offset as a single byte and then the num_values // as four bytes. However, this led to alignment problems and was wasteful since we already store the num_values // in higher layers. @@ -462,18 +574,48 @@ impl BlockDecompressor for BinaryBlockDecompressor { // never be more than 255 and it's little endian so the last 3 bytes will always be 0. These will be the least // significant 3 bytes of the number of values in the old scheme. It's pretty unlikely these are all 0 (that would // mean there are at least 16M values in a single page) so we'll use this to determine if the old scheme is used. + // + // The header fields and the offsets themselves come straight from the file. + // The structural checks below (all O(1)) reject blocks whose regions do not + // line up; the offset *values* are validated later, by the mandatory layout + // validation in `VariableWidthBlock::into_arrow`, so they are not rescanned + // here. + if data.len() < 4 { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "block of {} bytes is too small to hold a header", + data.len() + ), + )); + } let is_old_scheme = data[1] != 0 || data[2] != 0 || data[3] != 0; + let ensure_header = |header_len: usize| { + if data.len() < header_len { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "block of {} bytes is too small for a {} byte header", + data.len(), + header_len + ), + )); + } + Ok(()) + }; let (bits_per_offset, bytes_start_offset, offset_start) = if is_old_scheme { // Old scheme let bits_per_offset = data[0]; match bits_per_offset { 32 => { + ensure_header(9)?; debug_assert_eq!(LittleEndian::read_u32(&data[1..5]), num_values as u32); let bytes_start_offset = LittleEndian::read_u32(&data[5..9]); - (bits_per_offset, bytes_start_offset as u64, 9) + (bits_per_offset, bytes_start_offset as u64, 9_u64) } 64 => { + ensure_header(17)?; debug_assert_eq!(LittleEndian::read_u64(&data[1..9]), num_values); let bytes_start_offset = LittleEndian::read_u64(&data[9..17]); (bits_per_offset, bytes_start_offset, 17) @@ -489,10 +631,12 @@ impl BlockDecompressor for BinaryBlockDecompressor { let bits_per_offset = LittleEndian::read_u32(&data[0..4]) as u8; match bits_per_offset { 32 => { + ensure_header(8)?; let bytes_start_offset = LittleEndian::read_u32(&data[4..8]); (bits_per_offset, bytes_start_offset as u64, 8) } 64 => { + ensure_header(16)?; let bytes_start_offset = LittleEndian::read_u64(&data[8..16]); (bits_per_offset, bytes_start_offset, 16) } @@ -504,9 +648,55 @@ impl BlockDecompressor for BinaryBlockDecompressor { } }; + // The offsets region sits between the header and `bytes_start_offset` + // and must hold exactly `num_values + 1` offsets starting at zero. + let expected_offsets_bytes = num_values + .checked_add(1) + .and_then(|num_offsets| num_offsets.checked_mul(bits_per_offset as u64 / 8)) + .ok_or_else(|| { + Error::corrupt_file_named( + "variable-width block", + format!("offsets region size overflows for {num_values} values"), + ) + })?; + if bytes_start_offset < offset_start || bytes_start_offset > data.len() as u64 { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "bytes start offset {} is outside the block (header: {} bytes, block: {} bytes)", + bytes_start_offset, + offset_start, + data.len() + ), + )); + } + if bytes_start_offset - offset_start != expected_offsets_bytes { + return Err(Error::corrupt_file_named( + "variable-width block", + format!( + "expected {} offset bytes for {} values but found {}", + expected_offsets_bytes, + num_values, + bytes_start_offset - offset_start + ), + )); + } + // the next `bytes_start_offset - offset_start` stores the offsets. - let offsets = - data.slice_with_length(offset_start, bytes_start_offset as usize - offset_start); + let offsets = data.slice_with_length( + offset_start as usize, + (bytes_start_offset - offset_start) as usize, + ); + let first_offset = match bits_per_offset { + 32 => LittleEndian::read_u32(&offsets[0..4]) as u64, + _ => LittleEndian::read_u64(&offsets[0..8]), + }; + if first_offset != 0 { + return Err(Error::corrupt_file_named( + "variable-width block", + format!("first offset must be 0 but found {first_offset}"), + )); + } // the rest are the binary bytes. let data = data.slice_with_length( @@ -533,29 +723,41 @@ mod tests { use arrow_schema::{DataType, Field}; use crate::{ + buffer::LanceBuffer, constants::{ COMPRESSION_META_KEY, STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_MINIBLOCK, }, + data::{BlockInfo, DataBlock, VariableWidthBlock}, testing::check_specific_random, }; use rstest::rstest; use std::{collections::HashMap, sync::Arc, vec}; - use crate::{ - testing::{ - FnArrayGeneratorProvider, TestCases, check_basic_random, - check_round_trip_encoding_of_data, - }, - version::LanceFileVersion, + use crate::testing::{ + FnArrayGeneratorProvider, TestCases, TestEncoding, check_basic_random_case, + check_round_trip_encoding_generated, check_round_trip_encoding_of_data, }; + #[rstest] #[test_log::test(tokio::test)] - async fn test_utf8_binary() { + async fn test_utf8_binary( + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { let field = Field::new("", DataType::Utf8, false); check_specific_random( field, - TestCases::basic().with_min_file_version(LanceFileVersion::V2_1), + TestCases::basic() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]), ) .await; } @@ -566,6 +768,15 @@ mod tests { #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, #[values(DataType::Utf8, DataType::Binary)] data_type: DataType, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, ) { let mut field_metadata = HashMap::new(); field_metadata.insert( @@ -574,7 +785,7 @@ mod tests { ); let field = Field::new("", data_type, false).with_metadata(field_metadata); - check_basic_random(field).await; + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[rstest] @@ -583,6 +794,14 @@ mod tests { #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, #[values(DataType::Binary, DataType::Utf8)] data_type: DataType, + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, ) { let mut field_metadata = HashMap::new(); field_metadata.insert( @@ -592,7 +811,10 @@ mod tests { field_metadata.insert(COMPRESSION_META_KEY.to_string(), "fsst".into()); let field = Field::new("", data_type, true).with_metadata(field_metadata); // TODO (https://github.com/lance-format/lance/issues/4783) - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]); check_specific_random(field, test_cases).await; } @@ -602,6 +824,14 @@ mod tests { #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, #[values(DataType::LargeBinary, DataType::LargeUtf8)] data_type: DataType, + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, ) { let mut field_metadata = HashMap::new(); field_metadata.insert( @@ -612,21 +842,30 @@ mod tests { let field = Field::new("", data_type, true).with_metadata(field_metadata); check_specific_random( field, - TestCases::basic().with_min_file_version(LanceFileVersion::V2_1), + TestCases::basic() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]), ) .await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_large_binary() { - let field = Field::new("", DataType::LargeBinary, true); - check_basic_random(field).await; - } - - #[test_log::test(tokio::test)] - async fn test_large_utf8() { - let field = Field::new("", DataType::LargeUtf8, true); - check_basic_random(field).await; + async fn test_large_binary_types( + #[values(DataType::LargeBinary, DataType::LargeUtf8)] data_type: DataType, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { + let field = Field::new("", data_type, true); + check_basic_random_case(field, encoding, page_size, use_slicing).await; } #[rstest] @@ -634,20 +873,31 @@ mod tests { async fn test_small_strings( #[values(STRUCTURAL_ENCODING_MINIBLOCK, STRUCTURAL_ENCODING_FULLZIP)] structural_encoding: &str, + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, ) { - use crate::testing::check_basic_generated; - let mut field_metadata = HashMap::new(); field_metadata.insert( STRUCTURAL_ENCODING_META_KEY.to_string(), structural_encoding.into(), ); let field = Field::new("", DataType::Utf8, true).with_metadata(field_metadata); - check_basic_generated( + check_round_trip_encoding_generated( field, Box::new(FnArrayGeneratorProvider::new(move || { lance_datagen::array::utf8_prefix_plus_counter("user_", /*is_large=*/ false) })), + TestCases::basic() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]), ) .await; } @@ -698,10 +948,19 @@ mod tests { .await; } + #[rstest] #[test_log::test(tokio::test)] - async fn test_bigger_than_max_page_size() { - // Create an array with one single 32MiB string - let big_string = String::from_iter((0..(32 * 1024 * 1024)).map(|_| '0')); + async fn test_value_bigger_than_max_page_size( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + // Create one value larger than the configured 1MiB page budget. + let big_string = String::from_iter((0..(2 * 1024 * 1024)).map(|_| '0')); let string_array = StringArray::from(vec![ Some(big_string), Some("abc".to_string()), @@ -711,7 +970,9 @@ mod tests { ]); // Drop the max page size to 1MiB - let test_cases = TestCases::default().with_max_page_size(1024 * 1024); + let test_cases = TestCases::default() + .with_max_page_size(1024 * 1024) + .with_encoding(encoding); check_round_trip_encoding_of_data( vec![Arc::new(string_array)], @@ -719,16 +980,29 @@ mod tests { HashMap::new(), ) .await; + } - // This is a regression testing the case where a page with X rows is split into Y parts - // where the number of parts is not evenly divisible by the number of rows. In this - // case we are splitting 90 rows into 4 parts. - let big_string = String::from_iter((0..(1000 * 1000)).map(|_| '0')); + #[rstest] + #[test_log::test(tokio::test)] + async fn test_page_split_parts_do_not_evenly_divide_rows( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + // Regression: split 90 rows into four parts, where the part count does + // not evenly divide the row count. + let big_string = String::from_iter((0..45_000).map(|_| '0')); let string_array = StringArray::from_iter_values((0..90).map(|_| big_string.clone())); check_round_trip_encoding_of_data( vec![Arc::new(string_array)], - &TestCases::default(), + &TestCases::default() + .with_max_page_size(1024 * 1024) + .with_encoding(encoding), HashMap::new(), ) .await; @@ -801,7 +1075,7 @@ mod tests { #[values(true, false)] with_nulls: bool, #[values(100, 500, 35000)] dict_size: u32, ) { - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); let strings = (0..dict_size) .map(|i| i.to_string()) .collect::>(); @@ -829,7 +1103,7 @@ mod tests { let test_cases = TestCases::default() .with_expected_encoding("variable") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Test both automatic selection and explicit configuration // 1. Test automatic binary encoding selection (small strings that won't trigger FSST) @@ -968,4 +1242,195 @@ mod tests { } } } + + #[test] + fn test_binary_miniblock_rejects_corrupt_offsets() { + use super::BinaryMiniBlockDecompressor; + use crate::compression::MiniBlockDecompressor; + use lance_core::Error; + + // Chunk layout mirrors the on-disk format for ["alpha", "beta", "gamma"]: + // LE u32 offsets [16, 21, 25, 30] followed by the value bytes, padded to + // a multiple of 8 bytes. + fn chunk_u32(offsets: &[u32], values: &[u8]) -> LanceBuffer { + let mut chunk = offsets + .iter() + .flat_map(|offset| offset.to_le_bytes()) + .collect::>(); + chunk.extend_from_slice(values); + chunk.resize(chunk.len().next_multiple_of(8), 0); + LanceBuffer::from(chunk) + } + + let decompressor = BinaryMiniBlockDecompressor::new(32); + + // The tail offset points past the end of the 32-byte chunk. + let err = decompressor + .decompress( + vec![chunk_u32(&[16, 21, 25, 100_000], b"alphabetagamma")], + 3, + ) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("out of bounds"), "{err}"); + + // Offsets go backwards, which would underflow the rebase subtraction. + let err = decompressor + .decompress(vec![chunk_u32(&[16, 25, 21, 30], b"alphabetagamma")], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("decreases"), "{err}"); + + // The first offset points inside the offset table, which would alias + // the serialized offsets into the value bytes. + let err = decompressor + .decompress(vec![chunk_u32(&[0, 21, 25, 30], b"alphabetagamma")], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("overlaps"), "{err}"); + + // The chunk stores fewer offsets than the requested value count needs. + let err = decompressor + .decompress(vec![chunk_u32(&[8, 8], &[])], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("requires 4"), "{err}"); + + // The chunk size is not a multiple of the offset width. + let err = decompressor + .decompress(vec![LanceBuffer::from(vec![0u8; 10])], 1) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("multiple"), "{err}"); + + // 64-bit offsets take the same validation path. + fn chunk_u64(offsets: &[u64], values: &[u8]) -> LanceBuffer { + let mut chunk = offsets + .iter() + .flat_map(|offset| offset.to_le_bytes()) + .collect::>(); + chunk.extend_from_slice(values); + chunk.resize(chunk.len().next_multiple_of(8), 0); + LanceBuffer::from(chunk) + } + let decompressor = BinaryMiniBlockDecompressor::new(64); + let err = decompressor + .decompress( + vec![chunk_u64(&[32, 37, 41, 100_000], b"alphabetagamma")], + 3, + ) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("out of bounds"), "{err}"); + let err = decompressor + .decompress(vec![chunk_u64(&[0, 37, 41, 46], b"alphabetagamma")], 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("overlaps"), "{err}"); + + // A valid chunk still decodes: offsets rebase to [0, 5, 9, 14]. + let decompressor = BinaryMiniBlockDecompressor::new(32); + let block = decompressor + .decompress(vec![chunk_u32(&[16, 21, 25, 30], b"alphabetagamma")], 3) + .unwrap(); + let DataBlock::VariableWidth(block) = block else { + panic!("expected a variable-width block"); + }; + assert_eq!(block.data.as_ref(), b"alphabetagamma"); + assert_eq!( + block.offsets, + LanceBuffer::reinterpret_vec(vec![0_u32, 5, 9, 14]) + ); + } + + fn encoded_binary_block(bits_per_offset: u8) -> Vec { + use crate::compression::BlockCompressor; + + let offsets = match bits_per_offset { + 32 => LanceBuffer::reinterpret_vec(vec![0_i32, 5, 9, 14]), + 64 => LanceBuffer::reinterpret_vec(vec![0_i64, 5, 9, 14]), + _ => unreachable!(), + }; + let block = DataBlock::VariableWidth(VariableWidthBlock { + data: LanceBuffer::copy_slice(b"alphabetagamma"), + offsets, + bits_per_offset, + num_values: 3, + block_info: BlockInfo::new(), + }); + BlockCompressor::compress(&super::VariableEncoder::default(), block) + .unwrap() + .0 + .as_ref() + .unwrap() + .to_vec() + } + + /// The block decompressor only checks the block structure (all O(1)); bad + /// offset values inside a structurally-sound block are rejected by the + /// mandatory layout validation when the block is converted to Arrow. + #[rstest] + #[case::i32_tail_out_of_bounds(32, 3, 100_000, "out of bounds")] + #[case::i64_tail_out_of_bounds(64, 3, 15, "out of bounds")] + #[case::i32_non_monotonic(32, 2, 4, "non-monotonic")] + #[case::i64_non_monotonic(64, 2, 4, "non-monotonic")] + fn test_binary_block_bad_offsets_rejected_at_arrow_conversion( + #[case] bits_per_offset: u8, + #[case] mutated_offset_index: usize, + #[case] mutated_offset_value: u64, + #[case] expected_message: &str, + ) { + use crate::compression::BlockDecompressor; + use lance_core::Error; + + let mut encoded = encoded_binary_block(bits_per_offset); + let bytes_per_offset = (bits_per_offset / 8) as usize; + // The standard scheme header is two offset-width fields. + let mutated_offset_start = bytes_per_offset * (2 + mutated_offset_index); + encoded[mutated_offset_start..mutated_offset_start + bytes_per_offset] + .copy_from_slice(&mutated_offset_value.to_le_bytes()[..bytes_per_offset]); + + let block = super::BinaryBlockDecompressor::default() + .decompress(Some(LanceBuffer::from(encoded)), 3) + .unwrap(); + let data_type = match bits_per_offset { + 32 => DataType::Binary, + _ => DataType::LargeBinary, + }; + let err = block.into_arrow(data_type, false).unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains(expected_message), "{err}"); + } + + #[test] + fn test_binary_block_rejects_corrupt_structure() { + use crate::compression::BlockDecompressor; + use lance_core::Error; + + let decompressor = super::BinaryBlockDecompressor::default(); + + // The first offset must be zero. + let mut encoded = encoded_binary_block(32); + encoded[8..12].copy_from_slice(&5_u32.to_le_bytes()); + let err = decompressor + .decompress(Some(LanceBuffer::from(encoded)), 3) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("first offset"), "{err}"); + + // The offsets region must hold exactly num_values + 1 offsets. + let encoded = encoded_binary_block(32); + let err = decompressor + .decompress(Some(LanceBuffer::from(encoded)), 4) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("offset bytes"), "{err}"); + + // A block too small to hold its header is rejected, not a panic. + let err = decompressor + .decompress(Some(LanceBuffer::from(vec![0_u8; 2])), 1) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("too small"), "{err}"); + } } diff --git a/rust/lance-encoding/src/encodings/physical/bitpacking.rs b/rust/lance-encoding/src/encodings/physical/bitpacking.rs index 8ebdcc13c56..6c957e31e71 100644 --- a/rust/lance-encoding/src/encodings/physical/bitpacking.rs +++ b/rust/lance-encoding/src/encodings/physical/bitpacking.rs @@ -18,25 +18,27 @@ use arrow_array::types::UInt64Type; use arrow_array::{Array, PrimitiveArray}; use arrow_buffer::ArrowNativeType; -use byteorder::{ByteOrder, LittleEndian}; -use lance_bitpacking::BitPacking; +use lance_bitpacking::BitPackingUninit; use lance_core::{Error, Result}; use crate::buffer::LanceBuffer; -use crate::compression::{BlockCompressor, BlockDecompressor, MiniBlockDecompressor}; +use crate::compression::{ + BlockCompressor, BlockDecompressor, MiniBlockDecompressor, require_block_payload, +}; use crate::data::BlockInfo; use crate::data::{DataBlock, FixedWidthDataBlock}; use crate::encodings::logical::primitive::miniblock::{ - MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressor, + MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor, }; use crate::format::pb21::CompressiveEncoding; use crate::format::{ProtobufUtils21, pb21}; use crate::statistics::{GetStat, Stat}; -use bytemuck::{AnyBitPattern, cast_slice}; +use bytemuck::Pod; -const LOG_ELEMS_PER_CHUNK: u8 = 10; -const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; +pub(crate) const LOG_ELEMS_PER_CHUNK: u8 = 10; +/// Number of values encoded in each inline bitpacking chunk. +pub const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; #[derive(Debug, Default)] pub struct InlineBitpacking { @@ -70,7 +72,7 @@ impl InlineBitpacking { /// Each chunk can have a different bit width /// /// Each chunk has the compressed bit width stored inline in the chunk itself. - fn bitpack_chunked( + fn bitpack_chunked( data: FixedWidthDataBlock, ) -> MiniBlockCompressed { debug_assert!(data.num_values > 0); @@ -111,12 +113,13 @@ impl InlineBitpacking { output.push(T::from_usize(bit_width).unwrap()); let output_len = output.len(); unsafe { - output.set_len(output_len + *packed_chunk_size); - BitPacking::unchecked_pack( + BitPackingUninit::unchecked_pack_uninit( bit_width, &data_buffer[start_elem..][..ELEMS_PER_CHUNK as usize], - &mut output[output_len..][..*packed_chunk_size], + &mut output.spare_capacity_mut()[..*packed_chunk_size], ); + // The bitpacking kernel initialized every reserved output word. + output.set_len(output_len + *packed_chunk_size); } chunks.push(MiniBlockChunk { buffer_sizes: vec![((1 + *packed_chunk_size) * std::mem::size_of::()) as u32], @@ -137,13 +140,15 @@ impl InlineBitpacking { let bit_width = bit_widths_array.value(bit_widths_array.len() - 1) as usize; output.push(T::from_usize(bit_width).unwrap()); let output_len = output.len(); + let packed_chunk_size = packed_chunk_sizes[bit_widths_array.len() - 1]; unsafe { - output.set_len(output_len + packed_chunk_sizes[bit_widths_array.len() - 1]); - BitPacking::unchecked_pack( + BitPackingUninit::unchecked_pack_uninit( bit_width, &last_chunk, - &mut output[output_len..][..packed_chunk_sizes[bit_widths_array.len() - 1]], + &mut output.spare_capacity_mut()[..packed_chunk_size], ); + // The bitpacking kernel initialized every reserved output word. + output.set_len(output_len + packed_chunk_size); } chunks.push(MiniBlockChunk { buffer_sizes: vec![ @@ -181,27 +186,91 @@ impl InlineBitpacking { ) } - fn unchunk( + fn unchunk( data: LanceBuffer, num_values: u64, ) -> Result { - // Ensure at least the header is present - assert!(data.len() >= std::mem::size_of::()); - assert!(num_values <= ELEMS_PER_CHUNK); - // This macro decompresses a chunk(1024 values) of bitpacked values. let uncompressed_bit_width = std::mem::size_of::() * 8; - let mut decompressed = vec![T::from_usize(0).unwrap(); ELEMS_PER_CHUNK as usize]; - - // Copy for memory alignment - let chunk_in_u8: Vec = data.to_vec(); - let bit_width_bytes = &chunk_in_u8[..std::mem::size_of::()]; - let bit_width_value = LittleEndian::read_uint(bit_width_bytes, std::mem::size_of::()); - let chunk = cast_slice(&chunk_in_u8[std::mem::size_of::()..]); - // The bit-packed chunk should have number of bytes (bit_width_value * ELEMS_PER_CHUNK / 8) - assert!(std::mem::size_of_val(chunk) == (bit_width_value * ELEMS_PER_CHUNK) as usize / 8); + let word_size = std::mem::size_of::(); + + if data.len() < word_size { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking chunk is too small for {}-byte header: {} bytes", + word_size, + data.len() + ), + )); + } + if !data.len().is_multiple_of(word_size) { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking chunk size must be a multiple of {} bytes, got {} bytes", + word_size, + data.len() + ), + )); + } + if num_values > ELEMS_PER_CHUNK { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking chunk has {} values, expected at most {}", + num_values, ELEMS_PER_CHUNK + ), + )); + } + + let chunk_words = data.borrow_to_typed_view::(); + let bit_width_value = chunk_words[0].as_usize(); + if bit_width_value > uncompressed_bit_width { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking width {} exceeds {}-bit values", + bit_width_value, uncompressed_bit_width + ), + )); + } + let chunk = &chunk_words[1..]; + // bit_width_value has already been verified to be <= uncompressed_bit_width + // (8/16/32/64), so bit_width_value * ELEMS_PER_CHUNK (1024) can never + // overflow usize on supported targets. Keep checked_mul as defense in depth. + let expected_num_bits = bit_width_value + .checked_mul(ELEMS_PER_CHUNK as usize) + .ok_or_else(|| { + Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking width {} overflows chunk bit count", + bit_width_value + ), + ) + })?; + let expected_num_bytes = expected_num_bits / 8; + let actual_num_bytes = std::mem::size_of_val(chunk); + if actual_num_bytes != expected_num_bytes { + return Err(Error::corrupt_file_named( + "inline_bitpacking", + format!( + "Inline bitpacking payload has {} bytes, expected {} bytes for bit width {}", + actual_num_bytes, expected_num_bytes, bit_width_value + ), + )); + } + + let mut decompressed = Vec::with_capacity(ELEMS_PER_CHUNK as usize); unsafe { - BitPacking::unchecked_unpack(bit_width_value as usize, chunk, &mut decompressed); + BitPackingUninit::unchecked_unpack_uninit( + bit_width_value, + chunk, + &mut decompressed.spare_capacity_mut()[..ELEMS_PER_CHUNK as usize], + ); + // The bitpacking kernel initialized all 1024 decoded values. + decompressed.set_len(ELEMS_PER_CHUNK as usize); } decompressed.truncate(num_values as usize); @@ -212,10 +281,25 @@ impl InlineBitpacking { block_info: BlockInfo::new(), })) } + + /// An empty fixed-width block, used for the `num_values == 0` short-circuit in + /// both decompressor entry points so empty blocks skip chunk validation entirely. + fn empty_block(&self) -> DataBlock { + DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::empty(), + bits_per_value: self.uncompressed_bit_width, + num_values: 0, + block_info: BlockInfo::new(), + }) + } } impl MiniBlockCompressor for InlineBitpacking { - fn compress(&self, chunk: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + chunk: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match chunk { DataBlock::FixedWidth(fixed_width) => Ok(self.chunk_data(fixed_width)), _ => Err(Error::invalid_input_source( @@ -230,10 +314,27 @@ impl MiniBlockCompressor for InlineBitpacking { } impl BlockCompressor for InlineBitpacking { - fn compress(&self, data: DataBlock) -> Result { - let fixed_width = data.as_fixed_width().unwrap(); + fn compress(&self, data: DataBlock) -> Result<(Option, CompressiveEncoding)> { + let DataBlock::FixedWidth(fixed_width) = data else { + return Err(Error::invalid_input( + "Inline bitpacking requires fixed-width data", + )); + }; + if fixed_width.bits_per_value != self.uncompressed_bit_width { + return Err(Error::invalid_input(format!( + "Inline bitpacking expects {}-bit values, got {}", + self.uncompressed_bit_width, fixed_width.bits_per_value + ))); + } let (chunked, _) = self.chunk_data(fixed_width); - Ok(chunked.data.into_iter().next().unwrap()) + let payload = + chunked.data.into_iter().next().ok_or_else(|| { + Error::internal("Inline bitpacking produced no payload".to_string()) + })?; + Ok(( + Some(payload), + ProtobufUtils21::inline_bitpacking(self.uncompressed_bit_width, None), + )) } } @@ -241,6 +342,10 @@ impl MiniBlockDecompressor for InlineBitpacking { fn decompress(&self, data: Vec, num_values: u64) -> Result { assert_eq!(data.len(), 1); let data = data.into_iter().next().unwrap(); + if num_values == 0 { + // Empty mini-blocks have no inline bit-width header to decode. + return Ok(self.empty_block()); + } match self.uncompressed_bit_width { 8 => Self::unchunk::(data, num_values), 16 => Self::unchunk::(data, num_values), @@ -249,10 +354,23 @@ impl MiniBlockDecompressor for InlineBitpacking { _ => unimplemented!("Bitpacking word size must be 8, 16, 32, or 64"), } } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values + .checked_mul(self.uncompressed_bit_width) + .map(|bits| bits.div_ceil(8)) + } } impl BlockDecompressor for InlineBitpacking { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "Inline bitpacking")?; + if num_values == 0 { + // Empty blocks carry no inline bit-width header to decode; avoid + // spurious "too small for header" corrupt-file errors and mirror + // the MiniBlockDecompressor path. See #7794. + return Ok(self.empty_block()); + } match self.uncompressed_bit_width { 8 => Self::unchunk::(data, num_values), 16 => Self::unchunk::(data, num_values), @@ -268,7 +386,7 @@ impl BlockDecompressor for InlineBitpacking { /// Each chunk of 1024 values is packed with a constant bit width. For the tail we compare the /// cost of padding and packing against storing the raw values: if padding yields a smaller /// representation we pack; otherwise we append the raw tail. -fn bitpack_out_of_line( +fn bitpack_out_of_line( data: FixedWidthDataBlock, compressed_bits_per_value: usize, ) -> LanceBuffer { @@ -279,12 +397,7 @@ fn bitpack_out_of_line( let last_chunk_is_runt = data_buffer.len() % ELEMS_PER_CHUNK as usize != 0; let words_per_chunk = (ELEMS_PER_CHUNK as usize * compressed_bits_per_value) .div_ceil(data.bits_per_value as usize); - #[allow(clippy::uninit_vec)] let mut output: Vec = Vec::with_capacity(num_chunks * words_per_chunk); - #[allow(clippy::uninit_vec)] - unsafe { - output.set_len(num_chunks * words_per_chunk); - } let num_whole_chunks = if last_chunk_is_runt { num_chunks - 1 @@ -296,14 +409,15 @@ fn bitpack_out_of_line( for i in 0..num_whole_chunks { let input_start = i * ELEMS_PER_CHUNK as usize; let input_end = input_start + ELEMS_PER_CHUNK as usize; - let output_start = i * words_per_chunk; - let output_end = output_start + words_per_chunk; + let output_start = output.len(); unsafe { - BitPacking::unchecked_pack( + BitPackingUninit::unchecked_pack_uninit( compressed_bits_per_value, &data_buffer[input_start..input_end], - &mut output[output_start..output_end], + &mut output.spare_capacity_mut()[..words_per_chunk], ); + // The bitpacking kernel initialized this complete packed chunk. + output.set_len(output_start + words_per_chunk); } } @@ -312,10 +426,6 @@ fn bitpack_out_of_line( } let last_chunk_start = num_whole_chunks * ELEMS_PER_CHUNK as usize; - // Safety: output ensures to have those values. - unsafe { - output.set_len(num_whole_chunks * words_per_chunk); - } let remaining_items = data_buffer.len() - last_chunk_start; let uncompressed_bits = data.bits_per_value as usize; @@ -331,13 +441,13 @@ fn bitpack_out_of_line( last_chunk[..remaining_items].copy_from_slice(&data_buffer[last_chunk_start..]); let start = output.len(); unsafe { - // Capacity reserves a full chunk for each block; extend the visible length and fill it immediately. - output.set_len(start + words_per_chunk); - BitPacking::unchecked_pack( + BitPackingUninit::unchecked_pack_uninit( compressed_bits_per_value, &last_chunk, - &mut output[start..start + words_per_chunk], + &mut output.spare_capacity_mut()[..words_per_chunk], ); + // The bitpacking kernel initialized the padded tail chunk. + output.set_len(start + words_per_chunk); } } else { // Padding would waste space; append tail values as-is. @@ -352,7 +462,7 @@ fn bitpack_out_of_line( /// The compressed bit width is provided while the uncompressed width comes from `T`. /// Depending on the encoding decision the final chunk may be fully packed (with padding) /// or stored as raw tail values. We infer the layout from the buffer length. -fn unpack_out_of_line( +fn unpack_out_of_line( data: FixedWidthDataBlock, num_values: usize, compressed_bits_per_value: usize, @@ -368,25 +478,21 @@ fn unpack_out_of_line( let tail_is_raw = tail_values > 0 && compressed_words.len() == expected_new_len; let extra_tail_capacity = ELEMS_PER_CHUNK as usize; - #[allow(clippy::uninit_vec)] let mut decompressed: Vec = Vec::with_capacity(num_values.saturating_add(extra_tail_capacity)); - let chunk_value_len = num_whole_chunks * ELEMS_PER_CHUNK as usize; - unsafe { - decompressed.set_len(chunk_value_len); - } for chunk_idx in 0..num_whole_chunks { let input_start = chunk_idx * words_per_chunk; let input_end = input_start + words_per_chunk; - let output_start = chunk_idx * ELEMS_PER_CHUNK as usize; - let output_end = output_start + ELEMS_PER_CHUNK as usize; + let output_start = decompressed.len(); unsafe { - BitPacking::unchecked_unpack( + BitPackingUninit::unchecked_unpack_uninit( compressed_bits_per_value, &compressed_words[input_start..input_end], - &mut decompressed[output_start..output_end], + &mut decompressed.spare_capacity_mut()[..ELEMS_PER_CHUNK as usize], ); + // The bitpacking kernel initialized this complete decoded chunk. + decompressed.set_len(output_start + ELEMS_PER_CHUNK as usize); } } @@ -400,14 +506,13 @@ fn unpack_out_of_line( let tail_start = expected_full_words; let output_start = decompressed.len(); unsafe { - decompressed.set_len(output_start + ELEMS_PER_CHUNK as usize); - } - unsafe { - BitPacking::unchecked_unpack( + BitPackingUninit::unchecked_unpack_uninit( compressed_bits_per_value, &compressed_words[tail_start..tail_start + words_per_chunk], - &mut decompressed[output_start..output_start + ELEMS_PER_CHUNK as usize], + &mut decompressed.spare_capacity_mut()[..ELEMS_PER_CHUNK as usize], ); + // The kernel initialized a full chunk; only the requested tail stays visible. + decompressed.set_len(output_start + ELEMS_PER_CHUNK as usize); } decompressed.truncate(output_start + tail_values); } @@ -464,21 +569,43 @@ impl OutOfLineBitpacking { } impl BlockCompressor for OutOfLineBitpacking { - fn compress(&self, data: DataBlock) -> Result { - let fixed_width = data.as_fixed_width().unwrap(); + fn compress(&self, data: DataBlock) -> Result<(Option, CompressiveEncoding)> { + let DataBlock::FixedWidth(fixed_width) = data else { + return Err(Error::invalid_input( + "Out-of-line bitpacking requires fixed-width data", + )); + }; + if fixed_width.bits_per_value != self.uncompressed_bit_width { + return Err(Error::invalid_input(format!( + "Out-of-line bitpacking expects {}-bit values, got {}", + self.uncompressed_bit_width, fixed_width.bits_per_value + ))); + } let compressed = match fixed_width.bits_per_value { 8 => bitpack_out_of_line::(fixed_width, self.compressed_bit_width as usize), 16 => bitpack_out_of_line::(fixed_width, self.compressed_bit_width as usize), 32 => bitpack_out_of_line::(fixed_width, self.compressed_bit_width as usize), 64 => bitpack_out_of_line::(fixed_width, self.compressed_bit_width as usize), - _ => panic!("Bitpacking word size must be 8,16,32,64"), + _ => { + return Err(Error::invalid_input(format!( + "Bitpacking word size must be 8, 16, 32, or 64, got {}", + fixed_width.bits_per_value + ))); + } }; - Ok(compressed) + Ok(( + Some(compressed), + ProtobufUtils21::out_of_line_bitpacking( + self.uncompressed_bit_width, + ProtobufUtils21::flat(self.compressed_bit_width, None), + ), + )) } } impl BlockDecompressor for OutOfLineBitpacking { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "Out-of-line bitpacking")?; let word_size = match self.uncompressed_bit_width { 8 => std::mem::size_of::(), 16 => std::mem::size_of::(), @@ -527,19 +654,134 @@ mod test { use std::{collections::HashMap, sync::Arc}; use arrow_array::{Array, Int8Array, Int64Array}; + use arrow_buffer::ArrowNativeType; use arrow_schema::DataType; + use bytemuck::Pod; + use lance_bitpacking::{BitPacking, BitPackingUninit}; + use rstest::rstest; - use super::{ELEMS_PER_CHUNK, bitpack_out_of_line, unpack_out_of_line}; + use super::{ELEMS_PER_CHUNK, InlineBitpacking, bitpack_out_of_line, unpack_out_of_line}; use crate::{ buffer::LanceBuffer, - data::{BlockInfo, FixedWidthDataBlock}, + compression::{BlockDecompressor, MiniBlockDecompressor}, + data::{BlockInfo, DataBlock, FixedWidthDataBlock}, testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, }; + #[rstest] + #[case::u8(8)] + #[case::u16(16)] + #[case::u32(32)] + #[case::u64(64)] + fn test_inline_bitpacking_decompress_empty_miniblock(#[case] bit_width: u64) { + let decompressor = InlineBitpacking::new(bit_width); + let decompressed = + MiniBlockDecompressor::decompress(&decompressor, vec![LanceBuffer::empty()], 0) + .unwrap(); + + let DataBlock::FixedWidth(block) = decompressed else { + panic!("Expected FixedWidth block"); + }; + assert_eq!(block.bits_per_value, bit_width); + assert_eq!(block.num_values, 0); + assert_eq!(block.data.len(), 0); + } + + // Regression test for #7794: the block-level decompressor must short-circuit + // on num_values == 0 the same way the mini-block decompressor does, instead + // of reporting a spurious "too small for header" corrupt-file error. + #[rstest] + #[case::u8(8)] + #[case::u16(16)] + #[case::u32(32)] + #[case::u64(64)] + fn test_inline_bitpacking_decompress_empty_block(#[case] bit_width: u64) { + let decompressor = InlineBitpacking::new(bit_width); + let decompressed = + BlockDecompressor::decompress(&decompressor, Some(LanceBuffer::empty()), 0).unwrap(); + + let DataBlock::FixedWidth(block) = decompressed else { + panic!("Expected FixedWidth block"); + }; + assert_eq!(block.bits_per_value, bit_width); + assert_eq!(block.num_values, 0); + assert_eq!(block.data.len(), 0); + } + + fn roundtrip_unchunk(values: &[T], bit_width: usize) + where + T: ArrowNativeType + BitPackingUninit + Pod, + { + assert!(values.len() <= ELEMS_PER_CHUNK as usize); + let num_values = values.len() as u64; + + let mut padded = vec![T::from_usize(0).unwrap(); ELEMS_PER_CHUNK as usize]; + padded[..values.len()].copy_from_slice(values); + + let packed_words = ELEMS_PER_CHUNK as usize * bit_width / (std::mem::size_of::() * 8); + let mut chunk: Vec = Vec::with_capacity(1 + packed_words); + chunk.push(T::from_usize(bit_width).unwrap()); + let out_len = chunk.len(); + chunk.resize(out_len + packed_words, T::from_usize(0).unwrap()); + unsafe { + BitPacking::unchecked_pack(bit_width, &padded, &mut chunk[out_len..]); + } + + let data = LanceBuffer::reinterpret_vec(chunk); + let decoded = InlineBitpacking::unchunk::(data, num_values).unwrap(); + let DataBlock::FixedWidth(fixed) = decoded else { + panic!("expected FixedWidth DataBlock"); + }; + let decoded_values = fixed.data.borrow_to_typed_view::(); + assert_eq!(decoded_values.as_ref(), values); + } + + fn assert_corrupt_unchunk(data: LanceBuffer, num_values: u64, expected_message: &str) + where + T: ArrowNativeType + BitPackingUninit + Pod, + { + let err = InlineBitpacking::unchunk::(data, num_values).unwrap_err(); + assert!(matches!(err, lance_core::Error::CorruptFile { .. })); + let err = err.to_string(); + assert!( + err.contains(expected_message), + "expected error containing {expected_message:?}, got {err:?}" + ); + } + + #[test] + fn unchunk_u32_bw12_tail() { + let values: Vec = (0..500).map(|i| ((i * 7) % (1 << 12)) as u32).collect(); + roundtrip_unchunk(&values, 12); + } + + #[test] + fn unchunk_u64_bw23_full() { + let values: Vec = (0..1024).map(|i| ((i * 3) % (1 << 23)) as u64).collect(); + roundtrip_unchunk(&values, 23); + } + + #[rstest] + #[case::too_small_header(LanceBuffer::from(vec![1, 2, 3]), 1, "too small")] + #[case::misaligned_chunk_size(LanceBuffer::from(vec![0, 0, 0, 0, 0]), 1, "multiple")] + #[case::too_many_values( + LanceBuffer::reinterpret_vec(vec![0_u32]), + ELEMS_PER_CHUNK + 1, + "expected at most" + )] + #[case::payload_size_mismatch(LanceBuffer::reinterpret_vec(vec![12_u32]), 1, "payload")] + #[case::invalid_bit_width(LanceBuffer::reinterpret_vec(vec![33_u32]), 1, "exceeds")] + fn unchunk_rejects( + #[case] data: LanceBuffer, + #[case] num_values: u64, + #[case] expected_message: &str, + ) { + assert_corrupt_unchunk::(data, num_values, expected_message); + } + #[test_log::test(tokio::test)] async fn test_miniblock_bitpack() { - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); let arrays = vec![ Arc::new(Int8Array::from(vec![100; 1024])) as Arc, @@ -578,7 +820,7 @@ mod test { // Test bitpacking encoding verification with varied small values that should trigger bitpacking let test_cases = TestCases::default() .with_expected_encoding("inline_bitpacking") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Generate data with varied small values to avoid RLE // Mix different values but keep them small to trigger bitpacking @@ -602,7 +844,7 @@ mod test { let test_cases = TestCases::default() .with_expected_encoding("inline_bitpacking") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Build 2048 values: first 1024 all zeros (bit_width=0), // next 1024 small varied values to avoid RLE and trigger bitpacking. diff --git a/rust/lance-encoding/src/encodings/physical/block.rs b/rust/lance-encoding/src/encodings/physical/block.rs index a1f5bdb3fdd..9d97c9e72b9 100644 --- a/rust/lance-encoding/src/encodings/physical/block.rs +++ b/rust/lance-encoding/src/encodings/physical/block.rs @@ -26,7 +26,7 @@ use lance_core::{Error, Result}; use std::str::FromStr; -use crate::compression::{BlockCompressor, BlockDecompressor}; +use crate::compression::{BlockCompressor, BlockDecompressor, require_block_payload}; use crate::encodings::physical::binary::{BinaryBlockDecompressor, VariableEncoder}; use crate::format::{ ProtobufUtils21, @@ -46,9 +46,20 @@ pub struct CompressionConfig { } impl CompressionConfig { - pub(crate) fn new(scheme: CompressionScheme, level: Option) -> Self { + /// Create a compression configuration for an encoding mechanism. + pub fn new(scheme: CompressionScheme, level: Option) -> Self { Self { scheme, level } } + + /// Return the selected compression scheme. + pub fn scheme(&self) -> CompressionScheme { + self.scheme + } + + /// Return the optional compression level. + pub fn level(&self) -> Option { + self.level + } } impl Default for CompressionConfig { @@ -439,11 +450,12 @@ impl GeneralBlockDecompressor { } impl BlockDecompressor for GeneralBlockDecompressor { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "General block compression")?; let mut decompressed = Vec::new(); self.compressor.decompress(&data, &mut decompressed)?; self.inner - .decompress(LanceBuffer::from(decompressed), num_values) + .decompress(Some(LanceBuffer::from(decompressed)), num_values) } } @@ -451,6 +463,9 @@ impl BlockDecompressor for GeneralBlockDecompressor { #[derive(Debug)] pub struct CompressedBufferEncoder { pub(crate) compressor: Box, + // Runtime compressors normalize levels that they default or ignore. Block descriptors must + // retain the selected configuration so stable writers preserve those present/absent values. + block_compression: CompressionConfig, } impl Default for CompressedBufferEncoder { @@ -463,25 +478,33 @@ impl Default for CompressedBufferEncoder { #[cfg(not(any(feature = "zstd", feature = "lz4")))] let (scheme, level) = (CompressionScheme::None, None); - let compressor = - GeneralBufferCompressor::get_compressor(CompressionConfig { scheme, level }).unwrap(); - Self { compressor } + let block_compression = CompressionConfig { scheme, level }; + let compressor = GeneralBufferCompressor::get_compressor(block_compression).unwrap(); + Self { + compressor, + block_compression, + } } } impl CompressedBufferEncoder { pub fn try_new(compression_config: CompressionConfig) -> Result { let compressor = GeneralBufferCompressor::get_compressor(compression_config)?; - Ok(Self { compressor }) + Ok(Self { + compressor, + block_compression: compression_config, + }) } pub fn from_scheme(scheme: pb21::CompressionScheme) -> Result { let scheme = CompressionScheme::try_from(scheme)?; + let block_compression = CompressionConfig { + scheme, + level: Some(0), + }; Ok(Self { - compressor: GeneralBufferCompressor::get_compressor(CompressionConfig { - scheme, - level: Some(0), - })?, + compressor: GeneralBufferCompressor::get_compressor(block_compression)?, + block_compression, }) } } @@ -603,13 +626,26 @@ impl VariablePerValueDecompressor for CompressedBufferEncoder { } impl BlockCompressor for CompressedBufferEncoder { - fn compress(&self, data: DataBlock) -> Result { - let encoded = match data { - DataBlock::FixedWidth(fixed_width) => fixed_width.data, + fn compress(&self, data: DataBlock) -> Result<(Option, CompressiveEncoding)> { + let (encoded, inner_encoding) = match data { + DataBlock::FixedWidth(fixed_width) => ( + fixed_width.data, + ProtobufUtils21::flat(fixed_width.bits_per_value, None), + ), DataBlock::VariableWidth(variable_width) => { // Wrap VariableEncoder to handle the encoding let encoder = VariableEncoder::default(); - BlockCompressor::compress(&encoder, DataBlock::VariableWidth(variable_width))? + let (payload, encoding) = + BlockCompressor::compress(&encoder, DataBlock::VariableWidth(variable_width))?; + ( + payload.ok_or_else(|| { + Error::internal( + "VariableEncoder returned no payload for general compression" + .to_string(), + ) + })?, + encoding, + ) } _ => { return Err(Error::invalid_input_source( @@ -620,18 +656,22 @@ impl BlockCompressor for CompressedBufferEncoder { let mut compressed = Vec::new(); self.compressor.compress(&encoded, &mut compressed)?; - Ok(LanceBuffer::from(compressed)) + Ok(( + Some(LanceBuffer::from(compressed)), + ProtobufUtils21::wrapped(self.block_compression, inner_encoding)?, + )) } } impl BlockDecompressor for CompressedBufferEncoder { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "Compressed variable block")?; let mut decompressed = Vec::new(); self.compressor.decompress(&data, &mut decompressed)?; // Delegate to BinaryBlockDecompressor which handles the inline metadata let inner_decoder = BinaryBlockDecompressor::default(); - inner_decoder.decompress(LanceBuffer::from(decompressed), num_values) + inner_decoder.decompress(Some(LanceBuffer::from(decompressed)), num_values) } } @@ -764,8 +804,10 @@ mod tests { STRUCTURAL_ENCODING_META_KEY, }, encodings::physical::block::lz4::Lz4BufferCompressor, - testing::{FnArrayGeneratorProvider, TestCases, check_round_trip_encoding_generated}, - version::LanceFileVersion, + testing::{ + FnArrayGeneratorProvider, TestCases, TestEncoding, + check_round_trip_encoding_generated, + }, }; #[test] @@ -784,49 +826,59 @@ mod tests { assert_eq!(input_data, decompressed_data.as_slice()); } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_lz4_compress_round_trip() { - for data_type in &[ + async fn test_lz4_compress_round_trip( + #[values( DataType::Utf8, DataType::LargeUtf8, DataType::Binary, - DataType::LargeBinary, - ] { - let field = Field::new("", data_type.clone(), false); - let mut field_meta = HashMap::new(); - field_meta.insert(COMPRESSION_META_KEY.to_string(), "lz4".to_string()); - // Some bad cardinality estimatation causes us to use dictionary encoding currently - // which causes the expected encoding check to fail. - field_meta.insert(DICT_DIVISOR_META_KEY.to_string(), "100000".to_string()); - field_meta.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.0001".to_string()); - // Also disable size-based dictionary encoding - field_meta.insert( - STRUCTURAL_ENCODING_META_KEY.to_string(), - STRUCTURAL_ENCODING_FULLZIP.to_string(), - ); - let field = field.with_metadata(field_meta); - let test_cases = TestCases::basic() - // Need to use large pages as small pages might be too small to compress - .with_page_sizes(vec![1024 * 1024]) - .with_expected_encoding("zstd") - .with_min_file_version(LanceFileVersion::V2_1); - - // Can't use the default random provider because random data isn't compressible - // and we will fallback to uncompressed encoding - let datagen = Box::new(FnArrayGeneratorProvider::new(move || match data_type { - DataType::Utf8 => utf8_prefix_plus_counter("compressme", false), - DataType::Binary => { - binary_prefix_plus_counter(Arc::from(b"compressme".to_owned()), false) - } - DataType::LargeUtf8 => utf8_prefix_plus_counter("compressme", true), - DataType::LargeBinary => { - binary_prefix_plus_counter(Arc::from(b"compressme".to_owned()), true) - } - _ => panic!("Unsupported data type: {:?}", data_type), - })); - - check_round_trip_encoding_generated(field, datagen, test_cases).await; - } + DataType::LargeBinary + )] + data_type: DataType, + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(false, true)] use_slicing: bool, + ) { + let field = Field::new("", data_type.clone(), false); + let mut field_meta = HashMap::new(); + field_meta.insert(COMPRESSION_META_KEY.to_string(), "lz4".to_string()); + // Some bad cardinality estimatation causes us to use dictionary encoding currently + // which causes the expected encoding check to fail. + field_meta.insert(DICT_DIVISOR_META_KEY.to_string(), "100000".to_string()); + field_meta.insert(DICT_SIZE_RATIO_META_KEY.to_string(), "0.0001".to_string()); + // Also disable size-based dictionary encoding + field_meta.insert( + STRUCTURAL_ENCODING_META_KEY.to_string(), + STRUCTURAL_ENCODING_FULLZIP.to_string(), + ); + let field = field.with_metadata(field_meta); + let test_cases = TestCases::basic() + // Need to use large pages as small pages might be too small to compress + .with_page_sizes(vec![1024 * 1024]) + .with_expected_encoding("zstd") + .with_encoding(encoding) + .with_slicing_modes([use_slicing]); + + // Can't use the default random provider because random data isn't compressible + // and we will fallback to uncompressed encoding + let datagen = Box::new(FnArrayGeneratorProvider::new(move || match data_type { + DataType::Utf8 => utf8_prefix_plus_counter("compressme", false), + DataType::Binary => { + binary_prefix_plus_counter(Arc::from(b"compressme".to_owned()), false) + } + DataType::LargeUtf8 => utf8_prefix_plus_counter("compressme", true), + DataType::LargeBinary => { + binary_prefix_plus_counter(Arc::from(b"compressme".to_owned()), true) + } + _ => panic!("Unsupported data type: {:?}", data_type), + })); + + check_round_trip_encoding_generated(field, datagen, test_cases).await; } } } diff --git a/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs b/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs index c2b7aac9b9c..d4dbd1045b3 100644 --- a/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs +++ b/rust/lance-encoding/src/encodings/physical/byte_stream_split.rs @@ -62,7 +62,7 @@ use crate::compression::MiniBlockDecompressor; use crate::compression_config::BssMode; use crate::data::{BlockInfo, DataBlock, FixedWidthDataBlock}; use crate::encodings::logical::primitive::miniblock::{ - MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressor, + MiniBlockChunk, MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor, }; use crate::format::ProtobufUtils21; use crate::format::pb21::CompressiveEncoding; @@ -107,7 +107,11 @@ impl ByteStreamSplitEncoder { } impl MiniBlockCompressor for ByteStreamSplitEncoder { - fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + page: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match page { DataBlock::FixedWidth(data) => { let num_values = data.num_values; @@ -263,6 +267,10 @@ impl MiniBlockDecompressor for ByteStreamSplitDecompressor { block_info: BlockInfo::new(), })) } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values.checked_mul(self.bytes_per_value() as u64) + } } /// Determine if BSS should be used based on mode and data characteristics @@ -345,7 +353,9 @@ mod tests { }); // Compress - let (compressed, _encoding) = encoder.compress(data_block).unwrap(); + let (compressed, _encoding) = encoder + .compress(MiniBlockCompressionContext::new(0, true, true), data_block) + .unwrap(); // Decompress let decompressed = decompressor @@ -391,7 +401,9 @@ mod tests { }); // Compress - let (compressed, _encoding) = encoder.compress(data_block).unwrap(); + let (compressed, _encoding) = encoder + .compress(MiniBlockCompressionContext::new(0, true, true), data_block) + .unwrap(); // Decompress let decompressed = decompressor @@ -424,7 +436,9 @@ mod tests { }); // Compress empty data - let (compressed, _encoding) = encoder.compress(data_block).unwrap(); + let (compressed, _encoding) = encoder + .compress(MiniBlockCompressionContext::new(0, true, true), data_block) + .unwrap(); // Decompress empty data let decompressed = decompressor.decompress(compressed.data, 0).unwrap(); diff --git a/rust/lance-encoding/src/encodings/physical/constant.rs b/rust/lance-encoding/src/encodings/physical/constant.rs index c3fa16863f4..dd153f789d5 100644 --- a/rust/lance-encoding/src/encodings/physical/constant.rs +++ b/rust/lance-encoding/src/encodings/physical/constant.rs @@ -5,7 +5,7 @@ use crate::{ buffer::LanceBuffer, - compression::{BlockDecompressor, FixedPerValueDecompressor}, + compression::{BlockDecompressor, FixedPerValueDecompressor, require_no_block_payload}, data::{AllNullDataBlock, ConstantDataBlock, DataBlock, FixedWidthDataBlock}, }; @@ -24,7 +24,8 @@ impl ConstantDecompressor { } impl BlockDecompressor for ConstantDecompressor { - fn decompress(&self, _data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + require_no_block_payload(data, "Constant")?; if let Some(scalar) = self.scalar.clone() { Ok(DataBlock::Constant(ConstantDataBlock { data: scalar, @@ -34,6 +35,10 @@ impl BlockDecompressor for ConstantDecompressor { Ok(DataBlock::AllNull(AllNullDataBlock { num_values })) } } + + fn requires_payload(&self) -> bool { + false + } } impl FixedPerValueDecompressor for ConstantDecompressor { @@ -55,3 +60,22 @@ impl FixedPerValueDecompressor for ConstantDecompressor { .unwrap_or(0) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_constant_requires_no_payload() { + let decompressor = ConstantDecompressor::new(None); + + assert!(!decompressor.requires_payload()); + assert!(matches!( + BlockDecompressor::decompress(&decompressor, None, 3).unwrap(), + DataBlock::AllNull(AllNullDataBlock { num_values: 3 }) + )); + assert!( + BlockDecompressor::decompress(&decompressor, Some(LanceBuffer::empty()), 3).is_err() + ); + } +} diff --git a/rust/lance-encoding/src/encodings/physical/fsst.rs b/rust/lance-encoding/src/encodings/physical/fsst.rs index 8c1fe4141df..38acd24af0b 100644 --- a/rust/lance-encoding/src/encodings/physical/fsst.rs +++ b/rust/lance-encoding/src/encodings/physical/fsst.rs @@ -23,7 +23,7 @@ use crate::{ data::{BlockInfo, DataBlock, VariableWidthBlock}, encodings::logical::primitive::{ fullzip::{PerValueCompressor, PerValueDataBlock}, - miniblock::{MiniBlockCompressed, MiniBlockCompressor}, + miniblock::{MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor}, }, format::{ ProtobufUtils21, @@ -33,6 +33,13 @@ use crate::{ use super::binary::BinaryMiniBlockEncoder; +pub(crate) fn map_fsst_error(err: std::io::Error) -> Error { + match err.kind() { + std::io::ErrorKind::InvalidData => Error::corrupt_file_named("fsst", err.to_string()), + _ => err.into(), + } +} + struct FsstCompressed { data: VariableWidthBlock, symbol_table: Vec, @@ -138,7 +145,11 @@ impl FsstMiniBlockEncoder { } impl MiniBlockCompressor for FsstMiniBlockEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { let compressed = FsstCompressed::fsst_compress(data)?; let data_block = DataBlock::VariableWidth(compressed.data); @@ -148,7 +159,7 @@ impl MiniBlockCompressor for FsstMiniBlockEncoder { as Box; let (binary_miniblock_compressed, binary_array_encoding) = - binary_compressor.compress(data_block)?; + binary_compressor.compress(context, data_block)?; Ok(( binary_miniblock_compressed, @@ -232,7 +243,8 @@ impl VariablePerValueDecompressor for FsstPerValueDecompressor { offsets, &mut decompress_bytes_buf, &mut decompress_offset_buf, - )?; + ) + .map_err(map_fsst_error)?; // Ensure the offsets array is trimmed to exactly num_values + 1 elements decompress_offset_buf.truncate((num_values + 1) as usize); @@ -262,7 +274,8 @@ impl VariablePerValueDecompressor for FsstPerValueDecompressor { offsets, &mut decompress_bytes_buf, &mut decompress_offset_buf, - )?; + ) + .map_err(map_fsst_error)?; // Ensure the offsets array is trimmed to exactly num_values + 1 elements decompress_offset_buf.truncate((num_values + 1) as usize); @@ -327,7 +340,8 @@ impl MiniBlockDecompressor for FsstMiniBlockDecompressor { offsets, &mut decompress_bytes_buf, &mut decompress_offset_buf, - )?; + ) + .map_err(map_fsst_error)?; // Ensure the offsets array is trimmed to exactly num_values + 1 elements decompress_offset_buf.truncate((num_values + 1) as usize); @@ -350,7 +364,8 @@ impl MiniBlockDecompressor for FsstMiniBlockDecompressor { offsets, &mut decompress_bytes_buf, &mut decompress_offset_buf, - )?; + ) + .map_err(map_fsst_error)?; // Ensure the offsets array is trimmed to exactly num_values + 1 elements decompress_offset_buf.truncate((num_values + 1) as usize); @@ -375,18 +390,28 @@ impl MiniBlockDecompressor for FsstMiniBlockDecompressor { mod tests { use std::collections::HashMap; + use arrow_array::StringArray; + use fsst::fsst::{FSST_SYMBOL_TABLE_SIZE, compress, decompress}; + use lance_core::Error; use lance_datagen::{ByteCount, RowCount}; - use crate::{ - testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, - }; + use super::map_fsst_error; + use crate::testing::{TestCases, TestEncoding, check_round_trip_encoding_of_data}; + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_fsst() { + async fn test_fsst( + #[values(false, true)] explicit: bool, + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { let test_cases = TestCases::default() .with_expected_encoding("fsst") - .with_min_file_version(LanceFileVersion::V2_1); + .with_encoding(encoding); // Generate data suitable for FSST (large strings, total size > 32KB) let arr = lance_datagen::gen_batch() @@ -396,14 +421,49 @@ mod tests { .column(0) .clone(); - // Test both explicit metadata and automatic selection - // 1. Test with explicit FSST metadata - let metadata_explicit = - HashMap::from([("lance-encoding:compression".to_string(), "fsst".to_string())]); - check_round_trip_encoding_of_data(vec![arr.clone()], &test_cases, metadata_explicit).await; + let metadata = if explicit { + HashMap::from([("lance-encoding:compression".to_string(), "fsst".to_string())]) + } else { + // Automatic selection requires max_len >= 5 and total_size >= 32KB. + HashMap::new() + }; + check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await; + } - // 2. Test automatic FSST selection based on data characteristics - // FSST should be chosen automatically: max_len >= 5 and total_size >= 32KB - check_round_trip_encoding_of_data(vec![arr], &test_cases, HashMap::new()).await; + #[test] + fn test_corrupt_fsst_symbol_table_is_corrupt_file() { + let input = "the rain in spain stays mainly in the plain ".repeat(2048); + let array = StringArray::from(vec![input.as_str()]); + let mut symbol_table = [0u8; FSST_SYMBOL_TABLE_SIZE]; + let mut compressed = vec![0u8; array.value_data().len().max(1)]; + let mut compressed_offsets = vec![0i32; array.value_offsets().len()]; + compress( + symbol_table.as_mut(), + array.value_data(), + array.value_offsets(), + &mut compressed, + &mut compressed_offsets, + ) + .unwrap(); + + let st_info = u64::from_ne_bytes(symbol_table[..8].try_into().unwrap()); + assert!(st_info & (1 << 24) != 0, "expected decoder_switch_on input"); + let n_symbols = (st_info & 255) as usize; + assert!(n_symbols > 0); + symbol_table[8 + n_symbols * 8] = 9; + + let mut out = vec![0u8; compressed.len() * 8]; + let mut out_offsets = vec![0i32; compressed_offsets.len()]; + let err = decompress( + &symbol_table, + &compressed, + &compressed_offsets, + &mut out, + &mut out_offsets, + ) + .map_err(map_fsst_error) + .unwrap_err(); + assert!(matches!(err, Error::CorruptFile { .. }), "{err}"); + assert!(err.to_string().contains("symbol length"), "{err}"); } } diff --git a/rust/lance-encoding/src/encodings/physical/general.rs b/rust/lance-encoding/src/encodings/physical/general.rs index 53c61928870..bea6a85cb01 100644 --- a/rust/lance-encoding/src/encodings/physical/general.rs +++ b/rust/lance-encoding/src/encodings/physical/general.rs @@ -9,7 +9,9 @@ use crate::{ compression::MiniBlockDecompressor, data::DataBlock, encodings::{ - logical::primitive::miniblock::{MiniBlockCompressed, MiniBlockCompressor}, + logical::primitive::miniblock::{ + MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor, + }, physical::block::{CompressionConfig, GeneralBufferCompressor}, }, format::{ProtobufUtils21, pb21::CompressiveEncoding}, @@ -35,9 +37,13 @@ const MIN_BUFFER_SIZE_FOR_COMPRESSION: usize = 4 * 1024; use super::super::logical::primitive::miniblock::MiniBlockChunk; impl MiniBlockCompressor for GeneralMiniBlockCompressor { - fn compress(&self, page: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + context: MiniBlockCompressionContext, + page: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { // First, compress with the inner compressor - let (inner_compressed, inner_encoding) = self.inner.compress(page)?; + let (inner_compressed, inner_encoding) = self.inner.compress(context, page)?; // Return the original encoding without compression if there's no data or // the first buffer is not large enough @@ -132,6 +138,10 @@ impl MiniBlockDecompressor for GeneralMiniBlockDecompressor { self.inner.decompress(data, num_values) } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + self.inner.decoded_size_bytes(num_values) + } } #[cfg(test)] @@ -146,6 +156,10 @@ mod tests { use crate::format::pb21::compressive_encoding::Compression; use arrow_array::{Float64Array, Int32Array}; + fn miniblock_context() -> MiniBlockCompressionContext { + MiniBlockCompressionContext::new(0, true, true) + } + #[derive(Debug)] struct TestCase { name: &'static str, @@ -249,7 +263,9 @@ mod tests { GeneralMiniBlockCompressor::new(test_case.inner_encoder, test_case.compression); // Compress the data - let (compressed, encoding) = compressor.compress(test_case.data).unwrap(); + let (compressed, encoding) = compressor + .compress(miniblock_context(), test_case.data) + .unwrap(); // Check if compression was applied as expected match &encoding.compression { @@ -461,7 +477,7 @@ mod tests { let compressor = GeneralMiniBlockCompressor::new(inner, compression); // Compress the data - let (compressed, encoding) = compressor.compress(block).unwrap(); + let (compressed, encoding) = compressor.compress(miniblock_context(), block).unwrap(); // Should get GeneralMiniBlock encoding since buffer is 4KB match &encoding.compression { @@ -503,7 +519,7 @@ mod tests { }, ); - let (compressed, _) = compressor.compress(data).unwrap(); + let (compressed, _) = compressor.compress(miniblock_context(), data).unwrap(); // RLE produces 2 buffers, but only the first one is compressed assert_eq!(compressed.data.len(), 2); } @@ -539,7 +555,9 @@ mod tests { }, ); - let (_compressed, encoding) = compressor.compress(test_32.data).unwrap(); + let (_compressed, encoding) = compressor + .compress(miniblock_context(), test_32.data) + .unwrap(); // Verify the encoding structure match &encoding.compression { @@ -596,7 +614,9 @@ mod tests { }, ); - let (_compressed_64, encoding_64) = compressor_64.compress(block_64).unwrap(); + let (_compressed_64, encoding_64) = compressor_64 + .compress(miniblock_context(), block_64) + .unwrap(); // Verify the encoding structure for 64-bit match &encoding_64.compression { @@ -650,7 +670,7 @@ mod tests { }, ); - let result = compressor.compress(empty_block); + let result = compressor.compress(miniblock_context(), empty_block); match result { Ok((compressed, _)) => { assert_eq!(compressed.num_values, 0); diff --git a/rust/lance-encoding/src/encodings/physical/packed.rs b/rust/lance-encoding/src/encodings/physical/packed.rs index 3ade6a70818..3ae3dc4d6f8 100644 --- a/rust/lance-encoding/src/encodings/physical/packed.rs +++ b/rust/lance-encoding/src/encodings/physical/packed.rs @@ -12,13 +12,12 @@ use std::{convert::TryInto, sync::Arc}; use arrow_array::types::UInt64Type; - use lance_core::{Error, Result, datatypes::Field}; use crate::{ buffer::LanceBuffer, compression::{ - DefaultCompressionStrategy, FixedPerValueDecompressor, MiniBlockDecompressor, + CompressionStrategy, FixedPerValueDecompressor, MiniBlockDecompressor, VariablePerValueDecompressor, }, data::{ @@ -27,7 +26,7 @@ use crate::{ }, encodings::logical::primitive::{ fullzip::{PerValueCompressor, PerValueDataBlock}, - miniblock::{MiniBlockCompressed, MiniBlockCompressor}, + miniblock::{MiniBlockCompressed, MiniBlockCompressionContext, MiniBlockCompressor}, }, format::{ ProtobufUtils21, @@ -73,7 +72,11 @@ fn struct_data_block_to_fixed_width_data_block( pub struct PackedStructFixedWidthMiniBlockEncoder {} impl MiniBlockCompressor for PackedStructFixedWidthMiniBlockEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match data { DataBlock::Struct(struct_data_block) => { let bits_per_values = struct_data_block.children.iter().map(|data_block| data_block.as_fixed_width_ref().unwrap().bits_per_value).collect::>(); @@ -84,7 +87,7 @@ impl MiniBlockCompressor for PackedStructFixedWidthMiniBlockEncoder { // store and transformed fixed-width data block. let value_miniblock_compressor = Box::new(ValueEncoder::default()) as Box; let (value_miniblock_compressed, value_array_encoding) = - value_miniblock_compressor.compress(data_block)?; + value_miniblock_compressor.compress(context, data_block)?; Ok(( value_miniblock_compressed, @@ -182,11 +185,43 @@ impl MiniBlockDecompressor for PackedStructFixedWidthMiniBlockDecompressor { } } +#[derive(Debug)] +struct FixedPackedFieldData { + block: FixedWidthDataBlock, +} + +impl FixedPackedFieldData { + fn append_row_bytes(&self, row_idx: usize, output: &mut Vec) -> Result<()> { + let bits_per_value = self.block.bits_per_value; + if !bits_per_value.is_multiple_of(8) { + return Err(Error::invalid_input( + "Packed struct encoding requires byte-aligned fixed-width children", + )); + } + let bytes_per_value = (bits_per_value / 8) as usize; + let start = row_idx + .checked_mul(bytes_per_value) + .ok_or_else(|| Error::invalid_input("Packed struct row size overflow"))?; + let end = start.checked_add(bytes_per_value).ok_or_else(|| { + Error::invalid_input(format!( + "Packed struct fixed child range overflow: row_idx={row_idx}, \ + bytes_per_value={bytes_per_value}" + )) + })?; + let data = self.block.data.as_ref(); + if end > data.len() { + return Err(Error::invalid_input( + "Packed struct fixed child out of bounds", + )); + } + output.extend_from_slice(&data[start..end]); + Ok(()) + } +} + #[derive(Debug)] enum VariablePackedFieldData { - Fixed { - block: FixedWidthDataBlock, - }, + Fixed(FixedPackedFieldData), Variable { block: VariableWidthBlock, bits_per_length: u64, @@ -196,32 +231,12 @@ enum VariablePackedFieldData { impl VariablePackedFieldData { fn append_row_bytes(&self, row_idx: usize, output: &mut Vec) -> Result<()> { match self { - Self::Fixed { block } => { - let bits_per_value = block.bits_per_value; - if bits_per_value % 8 != 0 { - return Err(Error::invalid_input( - "Packed struct variable encoding requires byte-aligned fixed-width children", - )); - } - let bytes_per_value = (bits_per_value / 8) as usize; - let start = row_idx - .checked_mul(bytes_per_value) - .ok_or_else(|| Error::invalid_input("Packed struct row size overflow"))?; - let end = start + bytes_per_value; - let data = block.data.as_ref(); - if end > data.len() { - return Err(Error::invalid_input( - "Packed struct fixed child out of bounds", - )); - } - output.extend_from_slice(&data[start..end]); - Ok(()) - } + Self::Fixed(fixed_data) => fixed_data.append_row_bytes(row_idx, output), Self::Variable { block, bits_per_length, } => { - if bits_per_length % 8 != 0 { + if !bits_per_length.is_multiple_of(8) { return Err(Error::invalid_input( "Packed struct variable children must have byte-aligned length prefixes", )); @@ -280,55 +295,56 @@ impl VariablePackedFieldData { } } +fn check_struct_validity(data: DataBlock, field_length: usize) -> Result { + let DataBlock::Struct(struct_block) = data else { + return Err(Error::invalid_input( + "Packed struct encoder requires Struct data block", + )); + }; + + if struct_block.children.is_empty() { + return Err(Error::invalid_input( + "Packed struct encoder requires at least one child field", + )); + } + if struct_block.children.len() != field_length { + return Err(Error::invalid_input( + "Struct field metadata does not match number of children", + )); + } + + let num_values = struct_block.children[0].num_values(); + for child in struct_block.children.iter() { + if child.num_values() != num_values { + return Err(Error::invalid_input( + "Packed struct children must have matching value counts", + )); + } + } + Ok(struct_block) +} + #[derive(Debug)] pub struct PackedStructVariablePerValueEncoder { - strategy: DefaultCompressionStrategy, + strategy: Arc, fields: Vec, } impl PackedStructVariablePerValueEncoder { - pub fn new(strategy: DefaultCompressionStrategy, fields: Vec) -> Self { + pub fn new(strategy: Arc, fields: Vec) -> Self { Self { strategy, fields } } } impl PerValueCompressor for PackedStructVariablePerValueEncoder { fn compress(&self, data: DataBlock) -> Result<(PerValueDataBlock, CompressiveEncoding)> { - let DataBlock::Struct(struct_block) = data else { - return Err(Error::invalid_input( - "Packed struct encoder requires Struct data block", - )); - }; - - if struct_block.children.is_empty() { - return Err(Error::invalid_input( - "Packed struct encoder requires at least one child field", - )); - } - if struct_block.children.len() != self.fields.len() { - return Err(Error::invalid_input( - "Struct field metadata does not match number of children", - )); - } - + let struct_block = check_struct_validity(data, self.fields.len())?; let num_values = struct_block.children[0].num_values(); - for child in struct_block.children.iter() { - if child.num_values() != num_values { - return Err(Error::invalid_input( - "Packed struct children must have matching value counts", - )); - } - } - let mut field_data = Vec::with_capacity(self.fields.len()); let mut field_metadata = Vec::with_capacity(self.fields.len()); - for (field, child_block) in self.fields.iter().zip(struct_block.children.into_iter()) { - let compressor = crate::compression::CompressionStrategy::create_per_value( - &self.strategy, - field, - &child_block, - )?; + for (field, child_block) in self.fields.iter().zip(struct_block.children) { + let compressor = self.strategy.create_per_value(field, &child_block)?; let (compressed, encoding) = compressor.compress(child_block)?; match compressed { PerValueDataBlock::Fixed(block) => { @@ -336,7 +352,8 @@ impl PerValueCompressor for PackedStructVariablePerValueEncoder { encoding, block.bits_per_value, )); - field_data.push(VariablePackedFieldData::Fixed { block }); + let block = FixedPackedFieldData { block }; + field_data.push(VariablePackedFieldData::Fixed(block)); } PerValueDataBlock::Variable(block) => { let bits_per_length = block.bits_per_offset as u64; @@ -399,6 +416,83 @@ impl PerValueCompressor for PackedStructVariablePerValueEncoder { } } +#[derive(Debug)] +pub(crate) struct PackedStructFixedPerValueEncoder { + field_len: usize, +} + +impl PackedStructFixedPerValueEncoder { + pub(crate) fn new(fields: Vec) -> Self { + Self { + field_len: fields.len(), + } + } +} + +impl PerValueCompressor for PackedStructFixedPerValueEncoder { + fn compress(&self, data: DataBlock) -> Result<(PerValueDataBlock, CompressiveEncoding)> { + let struct_block = check_struct_validity(data, self.field_len)?; + + if struct_block.has_variable_width_child() { + return Err(Error::invalid_input( + "Packed struct fixed encoding requires all children to be fixed-width", + )); + } + + let num_values = struct_block.children[0].num_values(); + // Fixed length - supporting only flat encoding + let compressor = Box::new(ValueEncoder::default()) as Box; + let mut field_data = Vec::with_capacity(self.field_len); + let mut field_bits_per_value = Vec::with_capacity(self.field_len); + let mut bits_per_row: u64 = 0; + + for child_block in struct_block.children.into_iter() { + let (compressed, ..) = compressor.compress(child_block)?; + match compressed { + PerValueDataBlock::Fixed(block) => { + bits_per_row = bits_per_row + .checked_add(block.bits_per_value) + .ok_or_else(|| Error::invalid_input("Packed struct row width overflow"))?; + field_bits_per_value.push(block.bits_per_value); + field_data.push(FixedPackedFieldData { block }); + } + _ => { + return Err(Error::invalid_input( + "Packed struct fixed encoding requires all children to be fixed-width", + )); + } + } + } + + // Children are validated byte-aligned in `append_row_bytes`, so the row width + // is an exact number of bytes. + let bytes_per_row = (bits_per_row / 8) as usize; + let mut row_data: Vec = + Vec::with_capacity(bytes_per_row.saturating_mul(num_values as usize)); + for row in 0..num_values as usize { + for field in &field_data { + field.append_row_bytes(row, &mut row_data)?; + } + debug_assert_eq!(row_data.len(), bytes_per_row * (row + 1)); + } + + let data_block = FixedWidthDataBlock { + data: LanceBuffer::from(row_data), + bits_per_value: bits_per_row, + num_values, + block_info: BlockInfo::new(), + }; + + Ok(( + PerValueDataBlock::Fixed(data_block), + ProtobufUtils21::packed_struct( + ProtobufUtils21::flat(bits_per_row, None), + field_bits_per_value, + ), + )) + } +} + #[derive(Debug)] pub(crate) enum VariablePackedStructFieldKind { Fixed { @@ -427,12 +521,59 @@ impl PackedStructVariablePerValueDecompressor { } } +#[derive(Debug)] +struct FixedFieldAccumulator { + builder: DataBlockBuilder, + bits_per_value: u64, + empty_value: DataBlock, +} + +impl FixedFieldAccumulator { + fn append_empty(&mut self) -> Result<()> { + self.builder.append(&self.empty_value, 0..1) + } + + fn new(bits_per_value: u64, num_values: u64) -> Result { + if !bits_per_value.is_multiple_of(8) { + return Err(Error::invalid_input( + "Packed struct fixed child must be byte-aligned", + )); + } + + let bytes_per_value = bits_per_value.checked_div(8).ok_or_else(|| { + Error::invalid_input("Invalid bits per value for packed struct field") + })?; + + let estimate = bytes_per_value + .checked_mul(num_values) + .ok_or_else(|| Error::invalid_input("Packed struct fixed child allocation overflow"))?; + + let empty_value = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::from(vec![0_u8; bytes_per_value as usize]), + bits_per_value, + num_values: 1, + block_info: BlockInfo::new(), + }); + + Ok(Self { + builder: DataBlockBuilder::with_capacity_estimate(estimate), + bits_per_value, + empty_value, + }) + } + + fn finish(self) -> Result { + let DataBlock::FixedWidth(block) = self.builder.finish() else { + return Err(Error::invalid_input( + "Expected fixed-width datablock from builder", + )); + }; + Ok(block) + } +} + enum FieldAccumulator { - Fixed { - builder: DataBlockBuilder, - bits_per_value: u64, - empty_value: DataBlock, - }, + Fixed(FixedFieldAccumulator), Variable32 { builder: DataBlockBuilder, empty_value: DataBlock, @@ -447,13 +588,9 @@ impl FieldAccumulator { // In full-zip variable packed decoding, rep/def may produce a visible row // with an empty payload (e.g. null/invalid item). We still need to append // one placeholder per child so child row counts remain aligned. - fn append_empty(&mut self) { + fn append_empty(&mut self) -> Result<()> { match self { - Self::Fixed { - builder, - empty_value, - .. - } => builder.append(empty_value, 0..1), + Self::Fixed(fixed_field_accumulator) => fixed_field_accumulator.append_empty(), Self::Variable32 { builder, empty_value, @@ -498,28 +635,8 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { for field in &self.fields { match &field.kind { VariablePackedStructFieldKind::Fixed { bits_per_value, .. } => { - if bits_per_value % 8 != 0 { - return Err(Error::invalid_input( - "Packed struct fixed child must be byte-aligned", - )); - } - let bytes_per_value = bits_per_value.checked_div(8).ok_or_else(|| { - Error::invalid_input("Invalid bits per value for packed struct field") - })?; - let estimate = bytes_per_value.checked_mul(num_values).ok_or_else(|| { - Error::invalid_input("Packed struct fixed child allocation overflow") - })?; - let empty_value = DataBlock::FixedWidth(FixedWidthDataBlock { - data: LanceBuffer::from(vec![0_u8; bytes_per_value as usize]), - bits_per_value: *bits_per_value, - num_values: 1, - block_info: BlockInfo::new(), - }); - accumulators.push(FieldAccumulator::Fixed { - builder: DataBlockBuilder::with_capacity_estimate(estimate), - bits_per_value: *bits_per_value, - empty_value, - }); + let accumulator = FixedFieldAccumulator::new(*bits_per_value, num_values)?; + accumulators.push(FieldAccumulator::Fixed(accumulator)); } VariablePackedStructFieldKind::Variable { bits_per_length, .. @@ -563,7 +680,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { } if row_start == row_end { for accumulator in accumulators.iter_mut() { - accumulator.append_empty(); + accumulator.append_empty()?; } continue; } @@ -572,14 +689,10 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { match (&field.kind, accumulator) { ( VariablePackedStructFieldKind::Fixed { bits_per_value, .. }, - FieldAccumulator::Fixed { - builder, - bits_per_value: acc_bits, - .. - }, + FieldAccumulator::Fixed(fixed_accumulator), ) => { - debug_assert_eq!(bits_per_value, acc_bits); - let bytes_per_value = (bits_per_value / 8) as usize; + debug_assert_eq!(*bits_per_value, fixed_accumulator.bits_per_value); + let bytes_per_value = (*bits_per_value / 8) as usize; let end = cursor + bytes_per_value; if end > row_end { return Err(Error::invalid_input( @@ -592,7 +705,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { num_values: 1, block_info: BlockInfo::new(), }); - builder.append(&value_block, 0..1); + fixed_accumulator.builder.append(&value_block, 0..1)?; cursor = end; } ( @@ -631,7 +744,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { num_values: 1, block_info: BlockInfo::new(), }); - builder.append(&value_block, 0..1); + builder.append(&value_block, 0..1)?; cursor = value_end; } ( @@ -670,7 +783,7 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { num_values: 1, block_info: BlockInfo::new(), }); - builder.append(&value_block, 0..1); + builder.append(&value_block, 0..1)?; cursor = value_end; } _ => { @@ -688,18 +801,16 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { } let mut children = Vec::with_capacity(self.fields.len()); - for (field, accumulator) in self.fields.iter().zip(accumulators.into_iter()) { + for (field, accumulator) in self.fields.iter().zip(accumulators) { match (field, accumulator) { ( VariablePackedStructFieldDecoder { kind: VariablePackedStructFieldKind::Fixed { decompressor, .. }, }, - FieldAccumulator::Fixed { builder, .. }, + FieldAccumulator::Fixed(fixed_accumulator), ) => { - let DataBlock::FixedWidth(block) = builder.finish() else { - panic!("Expected fixed-width datablock from builder"); - }; - let decoded = decompressor.decompress(block, num_values)?; + let finished_accumulator = fixed_accumulator.finish()?; + let decoded = decompressor.decompress(finished_accumulator, num_values)?; children.push(decoded); } ( @@ -754,16 +865,135 @@ impl VariablePerValueDecompressor for PackedStructVariablePerValueDecompressor { } } +#[derive(Debug)] +struct PackedStructFixedFieldDecoder { + bits_per_value: u64, + decompressor: Box, +} + +#[derive(Debug)] +pub(crate) struct PackedStructFixedPerValueDecompressor { + decoders: Vec, +} + +impl PackedStructFixedPerValueDecompressor { + pub(crate) fn new(description: &PackedStruct) -> Result { + let compression = description + .values + .as_ref() + .ok_or_else(|| Error::invalid_input("PackedStruct missing values encoding"))? + .compression + .as_ref() + .ok_or_else(|| { + Error::invalid_input("PackedStruct values missing compression encoding") + })?; + + // The encoder always flat-encodes each child, so that is the only layout we can decode. + if !matches!(compression, Compression::Flat(..)) { + return Err(Error::invalid_input( + "PackedStruct fixed encoding currently requires flat compression", + )); + } + + let decoders = description + .bits_per_value + .iter() + .map(|&bits_per_value| { + let flat = crate::format::pb21::Flat { + bits_per_value, + data: None, + }; + PackedStructFixedFieldDecoder { + bits_per_value, + decompressor: Box::new(ValueDecompressor::from_flat(&flat)), + } + }) + .collect(); + Ok(Self { decoders }) + } +} + +impl FixedPerValueDecompressor for PackedStructFixedPerValueDecompressor { + fn decompress(&self, data: FixedWidthDataBlock, num_values: u64) -> Result { + if !data.bits_per_value.is_multiple_of(8) { + return Err(Error::invalid_input( + "Packed struct fixed encoding requires byte-aligned children", + )); + } + let bytes_per_row = (data.bits_per_value / 8) as usize; + + // Byte offset of each child within a packed row (a running prefix sum of the + // child widths). The final offset must equal the packed row width. + let mut child_bytes = Vec::with_capacity(self.decoders.len()); + for decoder in &self.decoders { + if !decoder.bits_per_value.is_multiple_of(8) { + return Err(Error::invalid_input( + "Packed struct fixed child must be byte-aligned", + )); + } + child_bytes.push((decoder.bits_per_value / 8) as usize); + } + if child_bytes.iter().sum::() != bytes_per_row { + return Err(Error::invalid_input( + "Packed struct child widths do not sum to the packed row width", + )); + } + if bytes_per_row.saturating_mul(num_values as usize) > data.data.len() { + return Err(Error::invalid_input( + "Packed struct row bounds exceed buffer", + )); + } + + // Un-zip the row-major buffer one child at a time by gathering that child's + // slice out of every row, then hand the column to the child decompressor. + let bytes = data.data.as_ref(); + let mut children = Vec::with_capacity(self.decoders.len()); + let mut field_offset = 0; + for (decoder, &field_bytes) in self.decoders.iter().zip(child_bytes.iter()) { + let mut child_buf = Vec::with_capacity(field_bytes * num_values as usize); + for row_idx in 0..num_values as usize { + let start = row_idx * bytes_per_row + field_offset; + child_buf.extend_from_slice(&bytes[start..start + field_bytes]); + } + let child_block = FixedWidthDataBlock { + data: LanceBuffer::from(child_buf), + bits_per_value: decoder.bits_per_value, + num_values, + block_info: BlockInfo::new(), + }; + children.push(decoder.decompressor.decompress(child_block, num_values)?); + field_offset += field_bytes; + } + + Ok(DataBlock::Struct(StructDataBlock { + children, + block_info: BlockInfo::new(), + validity: None, + })) + } + + fn bits_per_value(&self) -> u64 { + self.decoders + .iter() + .map(|decoder| decoder.bits_per_value) + .sum() + } +} + #[cfg(test)] mod tests { use super::*; use crate::{ compression::CompressionStrategy, - compression::{DefaultCompressionStrategy, DefaultDecompressionStrategy}, - constants::PACKED_STRUCT_META_KEY, + compression::DefaultDecompressionStrategy, + compression_config::CompressionParams, + constants::{ + PACKED_STRUCT_META_KEY, STRUCTURAL_ENCODING_FULLZIP, STRUCTURAL_ENCODING_META_KEY, + }, statistics::ComputeStat, - testing::{TestCases, check_round_trip_encoding_of_data}, - version::LanceFileVersion, + testing::{ + TestCases, TestEncoding, check_round_trip_encoding_of_data, test_compression_strategy, + }, }; use arrow_array::{ Array, ArrayRef, BinaryArray, Int32Array, Int64Array, LargeStringArray, StringArray, @@ -863,9 +1093,9 @@ mod tests { let data_block = DataBlock::Struct(struct_block); let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_2); + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); let compressor = crate::compression::CompressionStrategy::create_per_value( - &compression_strategy, + compression_strategy.as_ref(), &struct_field, &data_block, )?; @@ -931,9 +1161,9 @@ mod tests { let data_block = DataBlock::Struct(struct_block); let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_2); + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); let compressor = crate::compression::CompressionStrategy::create_per_value( - &compression_strategy, + compression_strategy.as_ref(), &struct_field, &data_block, )?; @@ -1013,7 +1243,7 @@ mod tests { ])); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_2) + .with_u32_structural_encodings() .with_expected_encoding("variable_packed_struct"); check_round_trip_encoding_of_data(vec![array], &test_cases, meta).await; @@ -1052,9 +1282,9 @@ mod tests { let data_block = DataBlock::Struct(struct_block); let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_2); + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); let compressor = crate::compression::CompressionStrategy::create_per_value( - &compression_strategy, + compression_strategy.as_ref(), &struct_field, &data_block, )?; @@ -1135,7 +1365,7 @@ mod tests { }; let compression_strategy = - DefaultCompressionStrategy::new().with_version(LanceFileVersion::V2_1); + test_compression_strategy(TestEncoding::StructuralU16, CompressionParams::default()); let result = compression_strategy.create_per_value(&struct_field, &DataBlock::Struct(struct_block)); @@ -1210,4 +1440,132 @@ mod tests { Ok(()) } + + #[test] + fn fixed_packed_struct_round_trip() -> Result<()> { + let arrow_fields: Fields = vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("value", DataType::Int64, false), + ] + .into(); + let arrow_struct = ArrowField::new("item", DataType::Struct(arrow_fields), false); + let struct_field = Field::try_from(&arrow_struct)?; + + let id_block = fixed_i32_block_from_array(Int32Array::from(vec![1, 2, 3, 4])); + let value_block = fixed_block_from_array(Int64Array::from(vec![10, 20, 30, 40])); + + let struct_block = StructDataBlock { + children: vec![ + DataBlock::FixedWidth(id_block.clone()), + DataBlock::FixedWidth(value_block.clone()), + ], + block_info: BlockInfo::new(), + validity: None, + }; + + let data_block = DataBlock::Struct(struct_block); + + let compression_strategy = + test_compression_strategy(TestEncoding::StructuralU32, CompressionParams::default()); + let compressor = CompressionStrategy::create_per_value( + compression_strategy.as_ref(), + &struct_field, + &data_block, + )?; + let (compressed, encoding) = compressor.compress(data_block)?; + + let PerValueDataBlock::Fixed(zipped) = compressed else { + panic!("expected fixed-width packed struct output"); + }; + + let decompression_strategy = DefaultDecompressionStrategy::default(); + let decompressor = + crate::compression::DecompressionStrategy::create_fixed_per_value_decompressor( + &decompression_strategy, + &encoding, + )?; + let decoded = decompressor.decompress(zipped, 4)?; + + let DataBlock::Struct(decoded_struct) = decoded else { + panic!("expected struct datablock after decode"); + }; + + let decoded_id = decoded_struct.children[0].as_fixed_width_ref().unwrap(); + assert_eq!(decoded_id.bits_per_value, 32); + assert_eq!(decoded_id.data.as_ref(), id_block.data.as_ref()); + + let decoded_value = decoded_struct.children[1].as_fixed_width_ref().unwrap(); + assert_eq!(decoded_value.bits_per_value, 64); + assert_eq!(decoded_value.data.as_ref(), value_block.data.as_ref()); + + Ok(()) + } + + // End-to-end round trip through the file writer. Requesting full-zip on an + // all-fixed-width struct routes to `PackedStructFixedPerValueEncoder` (mini-block + // is only chosen for narrow structs), exercising the writer/reader wiring rather + // than just the block-level compress/decompress above. + #[tokio::test] + async fn fixed_packed_struct_full_zip_round_trip() { + let fields = Fields::from(vec![ + Arc::new(ArrowField::new("id", DataType::Int32, false)), + Arc::new(ArrowField::new("value", DataType::Int64, false)), + ]); + + let mut meta = HashMap::new(); + meta.insert(PACKED_STRUCT_META_KEY.to_string(), "true".to_string()); + meta.insert( + STRUCTURAL_ENCODING_META_KEY.to_string(), + STRUCTURAL_ENCODING_FULLZIP.to_string(), + ); + + let array = Arc::new(StructArray::from(vec![ + ( + fields[0].clone(), + Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as ArrayRef, + ), + ( + fields[1].clone(), + Arc::new(Int64Array::from(vec![10, 20, 30, 40])) as ArrayRef, + ), + ])); + + let test_cases = TestCases::default() + .with_u32_structural_encodings() + .with_expected_encoding("packed_struct"); + + check_round_trip_encoding_of_data(vec![array], &test_cases, meta).await; + } + + #[test] + fn fixed_packed_struct_rejects_variable_child() -> Result<()> { + let arrow_fields: Fields = vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("name", DataType::Utf8, false), + ] + .into(); + let arrow_struct = ArrowField::new("item", DataType::Struct(arrow_fields), false); + let struct_field = Field::try_from(&arrow_struct)?; + + let struct_block = DataBlock::Struct(StructDataBlock { + children: vec![ + DataBlock::FixedWidth(fixed_i32_block_from_array(Int32Array::from(vec![1, 2]))), + DataBlock::VariableWidth(variable_block_from_string_array(StringArray::from( + vec!["a", "bb"], + ))), + ], + block_info: BlockInfo::new(), + validity: None, + }); + + let encoder = PackedStructFixedPerValueEncoder::new(struct_field.children); + let err = encoder.compress(struct_block).unwrap_err(); + assert!(matches!(&err, Error::InvalidInput { .. })); + assert!( + err.to_string().contains("fixed-width"), + "unexpected error: {err}" + ); + + Ok(()) + } } diff --git a/rust/lance-encoding/src/encodings/physical/rle.rs b/rust/lance-encoding/src/encodings/physical/rle.rs index da758b05bfe..8d4bb200349 100644 --- a/rust/lance-encoding/src/encodings/physical/rle.rs +++ b/rust/lance-encoding/src/encodings/physical/rle.rs @@ -54,17 +54,20 @@ //! When used in the block compression path, the encoded output is a single buffer: //! `[8-byte header: values buffer size][values buffer][run_lengths buffer]`. -use arrow_buffer::ArrowNativeType; +use arrow_buffer::{ArrowNativeType, ScalarBuffer}; use log::trace; use crate::buffer::LanceBuffer; -use crate::compression::{BlockCompressor, BlockDecompressor, MiniBlockDecompressor}; +use crate::compression::{ + BlockCompressor, BlockDecompressor, MiniBlockDecompressor, require_block_payload, +}; use crate::data::DataBlock; use crate::data::{BlockInfo, FixedWidthDataBlock}; use crate::encodings::logical::primitive::miniblock::{ MAX_MINIBLOCK_BYTES, MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, - MiniBlockCompressor, + MiniBlockCompressionContext, MiniBlockCompressor, }; +use crate::encodings::physical::block::{CompressionConfig, GeneralBufferCompressor}; use crate::format::ProtobufUtils21; use crate::format::pb21::CompressiveEncoding; @@ -326,6 +329,18 @@ pub(crate) fn accumulate_run_length_entries( #[derive(Debug)] pub struct RleEncoder { run_length_width: RunLengthWidth, + values_compression: Option, + run_lengths_compression: Option, + use_child_bitpacking: bool, +} + +#[derive(Clone)] +struct RleChildCandidate { + encoding: CompressiveEncoding, + data: LanceBuffer, + chunk_sizes: Vec, + size: usize, + requires_num_values: bool, } impl Default for RleEncoder { @@ -338,11 +353,33 @@ impl RleEncoder { pub fn new() -> Self { Self { run_length_width: RunLengthWidth::U8, + values_compression: None, + run_lengths_compression: None, + use_child_bitpacking: false, } } pub(crate) fn with_run_length_width(run_length_width: RunLengthWidth) -> Self { - Self { run_length_width } + Self { + run_length_width, + values_compression: None, + run_lengths_compression: None, + use_child_bitpacking: false, + } + } + + pub(crate) fn with_child_encoding( + run_length_width: RunLengthWidth, + values_compression: Option, + run_lengths_compression: Option, + use_child_bitpacking: bool, + ) -> Self { + Self { + run_length_width, + values_compression, + run_lengths_compression, + use_child_bitpacking, + } } fn encode_data( @@ -417,7 +454,15 @@ impl RleEncoder { }; if values_processed == 0 { - break; + // A non-final chunk needs at least two values because log_num_values == 0 + // identifies the final chunk. Report an error instead of returning partial data. + return Err(Error::internal(format!( + "RLE encoder made no progress: values_remaining={values_remaining}, \ + offset={offset}, data_len={}, bits_per_value={bits_per_value}, \ + max_miniblock_values={}", + data.len(), + *MAX_MINIBLOCK_VALUES + ))); } let log_num_values = if is_last_chunk { @@ -678,10 +723,329 @@ impl RleEncoder { total_chunks * (type_size + self.run_length_width.bytes_per_value()) } + + fn flat_child_candidate( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + ) -> RleChildCandidate { + RleChildCandidate { + encoding: ProtobufUtils21::flat(bits_per_value, None), + data: buffers[buffer_index].clone(), + chunk_sizes: chunks + .iter() + .map(|chunk| chunk.buffer_sizes[buffer_index]) + .collect(), + size: buffers[buffer_index].len(), + requires_num_values: false, + } + } + + fn general_child_candidate( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + compression: CompressionConfig, + ) -> Result> { + if buffers.is_empty() || buffers[buffer_index].is_empty() { + return Ok(None); + }; + + let compressor = GeneralBufferCompressor::get_compressor(compression)?; + let original = &buffers[buffer_index]; + let mut compressed = Vec::new(); + let mut offset = 0usize; + let mut total_original_size = 0usize; + let mut compressed_sizes = Vec::with_capacity(chunks.len()); + + for chunk in chunks.iter() { + let chunk_size = chunk.buffer_sizes[buffer_index] as usize; + let end = offset.checked_add(chunk_size).ok_or_else(|| { + Error::invalid_input_source("RLE child buffer offset overflow".into()) + })?; + if end > original.len() { + return Err(Error::invalid_input_source( + format!( + "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}", + buffer_index, + end, + original.len() + ) + .into(), + )); + } + + let start = compressed.len(); + compressor.compress(&original.as_ref()[offset..end], &mut compressed)?; + let compressed_size = compressed.len() - start; + let compressed_size = u32::try_from(compressed_size).map_err(|_| { + Error::invalid_input_source( + format!( + "RLE child buffer {} compressed chunk is too large: {} bytes", + buffer_index, compressed_size + ) + .into(), + ) + })?; + compressed_sizes.push(compressed_size); + total_original_size += chunk_size; + offset = end; + } + + if compressed.len() >= total_original_size { + return Ok(None); + } + + let encoding = + ProtobufUtils21::wrapped(compression, ProtobufUtils21::flat(bits_per_value, None))?; + Ok(Some( + RleChildCandidate { + encoding, + data: LanceBuffer::from(compressed), + chunk_sizes: compressed_sizes, + size: 0, + requires_num_values: false, + } + .with_size_from_data(), + )) + } + + #[cfg(feature = "bitpacking")] + fn bitpacked_child_candidate( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + ) -> Result> { + let original = &buffers[buffer_index]; + if original.is_empty() { + return Ok(None); + } + let packed_bits = Self::required_bits(original, bits_per_value)?; + if packed_bits >= bits_per_value { + return Ok(None); + } + + let compressor = crate::encodings::physical::bitpacking::OutOfLineBitpacking::new( + packed_bits, + bits_per_value, + ); + let mut packed = Vec::new(); + let mut offset = 0usize; + let mut packed_sizes = Vec::with_capacity(chunks.len()); + let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| { + Error::invalid_input_source( + format!("RLE child bit width is too large: {bits_per_value}").into(), + ) + })?; + + for chunk in chunks { + let chunk_size = chunk.buffer_sizes[buffer_index] as usize; + let end = offset.checked_add(chunk_size).ok_or_else(|| { + Error::invalid_input_source("RLE child buffer offset overflow".into()) + })?; + if end > original.len() { + return Err(Error::invalid_input_source( + format!( + "RLE child buffer {} chunk size exceeds buffer length: end {}, len {}", + buffer_index, + end, + original.len() + ) + .into(), + )); + } + if bytes_per_value == 0 || !chunk_size.is_multiple_of(bytes_per_value) { + return Err(Error::invalid_input_source( + format!( + "RLE child buffer {} chunk has invalid size {} for {} bits per value", + buffer_index, chunk_size, bits_per_value + ) + .into(), + )); + } + + let child_values = (chunk_size / bytes_per_value) as u64; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + bits_per_value, + data: original.slice_with_length(offset, chunk_size), + num_values: child_values, + block_info: BlockInfo::default(), + }); + let (chunk_packed, _) = BlockCompressor::compress(&compressor, block)?; + let chunk_packed = chunk_packed.ok_or_else(|| { + Error::internal("RLE bitpacking child returned no payload".to_string()) + })?; + let packed_size = u32::try_from(chunk_packed.len()).map_err(|_| { + Error::invalid_input_source( + format!( + "RLE child buffer {} bitpacked chunk is too large: {} bytes", + buffer_index, + chunk_packed.len() + ) + .into(), + ) + })?; + packed_sizes.push(packed_size); + packed.extend_from_slice(chunk_packed.as_ref()); + offset = end; + } + + if packed.len() >= original.len() { + return Ok(None); + } + + Ok(Some( + RleChildCandidate { + encoding: ProtobufUtils21::out_of_line_bitpacking( + bits_per_value, + ProtobufUtils21::flat(packed_bits, None), + ), + data: LanceBuffer::from(packed), + chunk_sizes: packed_sizes, + size: 0, + requires_num_values: true, + } + .with_size_from_data(), + )) + } + + #[cfg(feature = "bitpacking")] + fn required_bits(buffer: &LanceBuffer, bits_per_value: u64) -> Result { + let max_value = match bits_per_value { + 8 => buffer.as_ref().iter().map(|value| *value as u64).max(), + 16 => buffer + .as_ref() + .chunks_exact(2) + .map(|value| u16::from_le_bytes(value.try_into().unwrap()) as u64) + .max(), + 32 => buffer + .as_ref() + .chunks_exact(4) + .map(|value| u32::from_le_bytes(value.try_into().unwrap()) as u64) + .max(), + 64 => buffer + .as_ref() + .chunks_exact(8) + .map(|value| u64::from_le_bytes(value.try_into().unwrap())) + .max(), + _ => { + return Err(Error::invalid_input_source( + format!( + "RLE child bitpacking only supports 8, 16, 32, or 64-bit values, got {bits_per_value}" + ) + .into(), + )); + } + } + .unwrap_or(0); + Ok((u64::BITS - max_value.leading_zeros()).max(1) as u64) + } + + fn child_candidates( + buffers: &[LanceBuffer], + chunks: &[MiniBlockChunk], + buffer_index: usize, + bits_per_value: u64, + compression: Option, + use_child_bitpacking: bool, + ) -> Result> { + #[cfg(not(feature = "bitpacking"))] + let _ = use_child_bitpacking; + let mut candidates = vec![Self::flat_child_candidate( + buffers, + chunks, + buffer_index, + bits_per_value, + )]; + if let Some(compression) = compression + && let Some(candidate) = Self::general_child_candidate( + buffers, + chunks, + buffer_index, + bits_per_value, + compression, + )? + { + candidates.push(candidate); + } + #[cfg(feature = "bitpacking")] + { + if use_child_bitpacking + && let Some(candidate) = + Self::bitpacked_child_candidate(buffers, chunks, buffer_index, bits_per_value)? + { + candidates.push(candidate); + } + } + Ok(candidates) + } + + fn select_child_candidates( + values: Vec, + run_lengths: Vec, + ) -> (RleChildCandidate, RleChildCandidate) { + let mut best: Option<(usize, usize, usize)> = None; + for (value_idx, value) in values.iter().enumerate() { + for (length_idx, length) in run_lengths.iter().enumerate() { + if value.requires_num_values && length.requires_num_values { + continue; + } + let size = value.size + length.size; + if best.is_none_or(|(_, _, best_size)| size < best_size) { + best = Some((value_idx, length_idx, size)); + } + } + } + let (value_idx, length_idx, _) = + best.expect("flat RLE child candidates should always be selectable"); + (values[value_idx].clone(), run_lengths[length_idx].clone()) + } + + pub(crate) fn selected_payload_size(&self, data: &FixedWidthDataBlock) -> Result { + let (all_buffers, chunks) = + self.encode_data(&data.data, data.num_values, data.bits_per_value)?; + if all_buffers.is_empty() { + return Ok(0); + } + + let values_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 0, + data.bits_per_value, + self.values_compression, + self.use_child_bitpacking, + )?; + let run_lengths_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 1, + self.run_length_width.bits_per_value(), + self.run_lengths_compression, + self.use_child_bitpacking, + )?; + let (values, run_lengths) = + Self::select_child_candidates(values_candidates, run_lengths_candidates); + Ok((values.size as u128).saturating_add(run_lengths.size as u128)) + } +} + +impl RleChildCandidate { + fn with_size_from_data(mut self) -> Self { + self.size = self.data.len(); + self + } } impl MiniBlockCompressor for RleEncoder { - fn compress(&self, data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match data { DataBlock::FixedWidth(fixed_width) => { let num_values = fixed_width.num_values; @@ -689,17 +1053,53 @@ impl MiniBlockCompressor for RleEncoder { let (all_buffers, chunks) = self.encode_data(&fixed_width.data, num_values, bits_per_value)?; + if all_buffers.is_empty() { + let compressed = MiniBlockCompressed { + data: all_buffers, + chunks, + num_values, + }; + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::flat(bits_per_value, None), + ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None), + ); + return Ok((compressed, encoding)); + } + + let values_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 0, + bits_per_value, + self.values_compression, + self.use_child_bitpacking, + )?; + let run_lengths_candidates = Self::child_candidates( + &all_buffers, + &chunks, + 1, + self.run_length_width.bits_per_value(), + self.run_lengths_compression, + self.use_child_bitpacking, + )?; + let (values, run_lengths) = + Self::select_child_candidates(values_candidates, run_lengths_candidates); + let chunks = chunks + .into_iter() + .enumerate() + .map(|(idx, chunk)| MiniBlockChunk { + buffer_sizes: vec![values.chunk_sizes[idx], run_lengths.chunk_sizes[idx]], + log_num_values: chunk.log_num_values, + }) + .collect(); let compressed = MiniBlockCompressed { - data: all_buffers, + data: vec![values.data, run_lengths.data], chunks, num_values, }; - let encoding = ProtobufUtils21::rle( - ProtobufUtils21::flat(bits_per_value, None), - ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None), - ); + let encoding = ProtobufUtils21::rle(values.encoding, run_lengths.encoding); Ok((compressed, encoding)) } @@ -712,7 +1112,7 @@ impl MiniBlockCompressor for RleEncoder { impl BlockCompressor for RleEncoder { // Block format: [8-byte header: values buffer size][values buffer][run_lengths buffer] - fn compress(&self, data: DataBlock) -> Result { + fn compress(&self, data: DataBlock) -> Result<(Option, CompressiveEncoding)> { match data { DataBlock::FixedWidth(fixed_width) => { let num_values = fixed_width.num_values; @@ -727,7 +1127,13 @@ impl BlockCompressor for RleEncoder { combined.extend_from_slice(&values_size.to_le_bytes()); combined.extend_from_slice(&all_buffers[0]); combined.extend_from_slice(&all_buffers[1]); - Ok(LanceBuffer::from(combined)) + Ok(( + Some(LanceBuffer::from(combined)), + ProtobufUtils21::rle( + ProtobufUtils21::flat(bits_per_value, None), + ProtobufUtils21::flat(self.run_length_width.bits_per_value(), None), + ), + )) } _ => Err(Error::invalid_input_source( "RLE encoding only supports FixedWidth data blocks".into(), @@ -741,6 +1147,125 @@ impl BlockCompressor for RleEncoder { pub struct RleDecompressor { bits_per_value: u64, run_length_width: RunLengthWidth, + values: RleChildDecompressor, + run_lengths: RleChildDecompressor, +} + +#[derive(Debug)] +pub(crate) struct RleChildDecompressor { + bits_per_value: u64, + inner: RleChildDecompressorInner, +} + +#[derive(Debug)] +enum RleChildDecompressorInner { + Flat, + Block { + decompressor: Box, + requires_num_values: bool, + }, +} + +impl RleChildDecompressor { + pub(crate) fn flat(bits_per_value: u64) -> Self { + Self { + bits_per_value, + inner: RleChildDecompressorInner::Flat, + } + } + + pub(crate) fn block( + bits_per_value: u64, + decompressor: Box, + requires_num_values: bool, + ) -> Self { + Self { + bits_per_value, + inner: RleChildDecompressorInner::Block { + decompressor, + requires_num_values, + }, + } + } + + pub(crate) fn bits_per_value(&self) -> u64 { + self.bits_per_value + } + + pub(crate) fn requires_num_values(&self) -> bool { + match &self.inner { + RleChildDecompressorInner::Flat => false, + RleChildDecompressorInner::Block { + requires_num_values, + .. + } => *requires_num_values, + } + } + + pub(crate) fn is_identity(&self) -> bool { + matches!(self.inner, RleChildDecompressorInner::Flat) + } + + fn decode( + &self, + data: LanceBuffer, + num_values: Option, + label: &str, + ) -> Result { + match &self.inner { + RleChildDecompressorInner::Flat => Ok(data), + RleChildDecompressorInner::Block { + decompressor, + requires_num_values, + } => { + let num_values = if *requires_num_values { + num_values.ok_or_else(|| { + Error::invalid_input_source( + format!("RLE {label} child compression requires the run count").into(), + ) + })? + } else { + num_values.unwrap_or(0) + }; + let decoded = decompressor.decompress(Some(data), num_values)?; + self.extract_fixed_width(decoded, num_values, label) + } + } + } + + fn extract_fixed_width( + &self, + data: DataBlock, + expected_num_values: u64, + label: &str, + ) -> Result { + match data { + DataBlock::FixedWidth(block) => { + if block.bits_per_value != self.bits_per_value { + return Err(Error::invalid_input_source( + format!( + "RLE {label} child decoded {}-bit values, expected {}", + block.bits_per_value, self.bits_per_value + ) + .into(), + )); + } + if expected_num_values != 0 && block.num_values != expected_num_values { + return Err(Error::invalid_input_source( + format!( + "RLE {label} child decoded {} values, expected {}", + block.num_values, expected_num_values + ) + .into(), + )); + } + Ok(block.data) + } + _ => Err(Error::invalid_input_source( + format!("RLE {label} child decoded to a non fixed-width block").into(), + )), + } + } } impl RleDecompressor { @@ -748,6 +1273,8 @@ impl RleDecompressor { Self { bits_per_value, run_length_width: RunLengthWidth::U8, + values: RleChildDecompressor::flat(bits_per_value), + run_lengths: RleChildDecompressor::flat(RunLengthWidth::U8.bits_per_value()), } } @@ -758,10 +1285,31 @@ impl RleDecompressor { Self { bits_per_value, run_length_width, + values: RleChildDecompressor::flat(bits_per_value), + run_lengths: RleChildDecompressor::flat(run_length_width.bits_per_value()), + } + } + + pub(crate) fn with_child_decompressors( + bits_per_value: u64, + run_length_width: RunLengthWidth, + values: RleChildDecompressor, + run_lengths: RleChildDecompressor, + ) -> Self { + Self { + bits_per_value, + run_length_width, + values, + run_lengths, } } - fn decode_data(&self, data: Vec, num_values: u64) -> Result { + fn decode_data( + &self, + data: Vec, + num_values: u64, + clamp_overflow: bool, + ) -> Result { if num_values == 0 { return Ok(DataBlock::FixedWidth(FixedWidthDataBlock { bits_per_value: self.bits_per_value, @@ -781,14 +1329,47 @@ impl RleDecompressor { )); } - let values_buffer = &data[0]; - let lengths_buffer = &data[1]; + let mut data_iter = data.into_iter(); + let values_buffer = data_iter.next().unwrap(); + let lengths_buffer = data_iter.next().unwrap(); + let (values_buffer, lengths_buffer) = + self.decode_child_buffers(values_buffer, lengths_buffer)?; + + self.decode_child_data(&values_buffer, &lengths_buffer, num_values, clamp_overflow) + } + fn decode_child_data( + &self, + values_buffer: &LanceBuffer, + lengths_buffer: &LanceBuffer, + num_values: u64, + clamp_overflow: bool, + ) -> Result { let decoded_data = match self.bits_per_value { - 8 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, - 16 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, - 32 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, - 64 => self.decode_generic::(values_buffer, lengths_buffer, num_values)?, + 8 => self.decode_generic::( + values_buffer, + lengths_buffer, + num_values, + clamp_overflow, + )?, + 16 => self.decode_generic::( + values_buffer, + lengths_buffer, + num_values, + clamp_overflow, + )?, + 32 => self.decode_generic::( + values_buffer, + lengths_buffer, + num_values, + clamp_overflow, + )?, + 64 => self.decode_generic::( + values_buffer, + lengths_buffer, + num_values, + clamp_overflow, + )?, _ => { return Err(Error::invalid_input_source( format!( @@ -808,43 +1389,66 @@ impl RleDecompressor { })) } - fn decode_generic( + fn sum_run_lengths( &self, values_buffer: &LanceBuffer, lengths_buffer: &LanceBuffer, - num_values: u64, - ) -> Result - where - T: bytemuck::Pod + Copy + std::fmt::Debug + ArrowNativeType, - { - let type_size = std::mem::size_of::(); - let length_size = self.run_length_width.bytes_per_value(); - - if values_buffer.is_empty() || lengths_buffer.is_empty() { - if num_values == 0 { - return Ok(LanceBuffer::empty()); - } else { + ) -> Result { + let (value_size, value_type) = match self.bits_per_value { + 8 => (1, "u8"), + 16 => (2, "u16"), + 32 => (4, "u32"), + 64 => (8, "u64"), + _ => { return Err(Error::invalid_input_source( - format!("Empty buffers but expected {} values", num_values).into(), + format!( + "RLE decoding bits_per_value must be 8, 16, 32, or 64, got {}", + self.bits_per_value + ) + .into(), )); } - } + }; + let length_size = + self.validate_buffer_sizes(values_buffer, lengths_buffer, value_size, value_type)?; + + lengths_buffer + .chunks_exact(length_size) + .try_fold(0_u64, |num_values, length_bytes| { + let length = self.run_length_width.read_length(length_bytes); + if length == 0 { + return Err(Error::invalid_input_source( + "RLE decoding encountered a zero run length".into(), + )); + } + num_values.checked_add(length).ok_or_else(|| { + Error::invalid_input_source("RLE run length sum overflowed u64".into()) + }) + }) + } - if !values_buffer.len().is_multiple_of(type_size) + fn validate_buffer_sizes( + &self, + values_buffer: &LanceBuffer, + lengths_buffer: &LanceBuffer, + value_size: usize, + value_type: &str, + ) -> Result { + let length_size = self.run_length_width.bytes_per_value(); + if !values_buffer.len().is_multiple_of(value_size) || !lengths_buffer.len().is_multiple_of(length_size) { return Err(Error::invalid_input_source(format!( - "Invalid buffer sizes for RLE {} decoding: values {} bytes (not divisible by {}), lengths {} bytes (not divisible by {})", - std::any::type_name::(), + "Invalid buffer sizes for RLE {value_type} decoding: values {} bytes (not divisible by {}), lengths {} bytes (not divisible by {})", values_buffer.len(), - type_size, + value_size, lengths_buffer.len(), length_size ) .into())); } - let num_runs = values_buffer.len() / type_size; + let num_runs = values_buffer.len() / value_size; let num_length_entries = lengths_buffer.len() / length_size; if num_runs != num_length_entries { return Err(Error::invalid_input_source( @@ -855,59 +1459,164 @@ impl RleDecompressor { .into(), )); } + Ok(length_size) + } - let values_ref = values_buffer.borrow_to_typed_slice::(); - let values: &[T] = values_ref.as_ref(); - let lengths = lengths_buffer.as_ref(); + fn decode_child_buffers( + &self, + values_buffer: LanceBuffer, + lengths_buffer: LanceBuffer, + ) -> Result<(LanceBuffer, LanceBuffer)> { + let values_requires_num_runs = self.values.requires_num_values(); + let lengths_requires_num_runs = self.run_lengths.requires_num_values(); + if values_requires_num_runs && lengths_requires_num_runs { + return Err(Error::invalid_input_source( + "RLE values and run lengths child compression both require the run count".into(), + )); + } - let expected_value_count = usize::try_from(num_values).map_err(|_| { + if values_requires_num_runs { + let lengths_buffer = self + .run_lengths + .decode(lengths_buffer, None, "run lengths")?; + let num_runs = Self::num_child_values( + &lengths_buffer, + self.run_lengths.bits_per_value(), + "run lengths", + )?; + let values_buffer = self + .values + .decode(values_buffer, Some(num_runs), "values")?; + Ok((values_buffer, lengths_buffer)) + } else if lengths_requires_num_runs { + let values_buffer = self.values.decode(values_buffer, None, "values")?; + let num_runs = + Self::num_child_values(&values_buffer, self.values.bits_per_value(), "values")?; + let lengths_buffer = + self.run_lengths + .decode(lengths_buffer, Some(num_runs), "run lengths")?; + Ok((values_buffer, lengths_buffer)) + } else { + let values_buffer = self.values.decode(values_buffer, None, "values")?; + let lengths_buffer = self + .run_lengths + .decode(lengths_buffer, None, "run lengths")?; + Ok((values_buffer, lengths_buffer)) + } + } + + fn num_child_values(buffer: &LanceBuffer, bits_per_value: u64, label: &str) -> Result { + let bytes_per_value = usize::try_from(bits_per_value / 8).map_err(|_| { Error::invalid_input_source( - format!("RLE num_values does not fit in usize: {num_values}").into(), + format!("RLE {label} child bit width is too large: {bits_per_value}").into(), ) })?; - let mut decoded_value_count = 0usize; - for length_bytes in lengths.chunks_exact(length_size) { - let length = self.run_length_width.read_length(length_bytes); - if length == 0 { - return Err(Error::invalid_input_source( - "RLE decoding encountered a zero run length".into(), - )); - } - let length = usize::try_from(length).map_err(|_| { - Error::invalid_input_source( - format!("RLE run length does not fit in usize: {length}").into(), + if bytes_per_value == 0 || !buffer.len().is_multiple_of(bytes_per_value) { + return Err(Error::invalid_input_source( + format!( + "RLE {label} child decoded to {} bytes, not divisible by {}", + buffer.len(), + bytes_per_value + ) + .into(), + )); + } + Ok((buffer.len() / bytes_per_value) as u64) + } + + fn decode_generic( + &self, + values_buffer: &LanceBuffer, + lengths_buffer: &LanceBuffer, + num_values: u64, + clamp_overflow: bool, + ) -> Result + where + T: bytemuck::Pod + Copy + std::fmt::Debug + ArrowNativeType, + { + let type_size = std::mem::size_of::(); + + if values_buffer.is_empty() || lengths_buffer.is_empty() { + if num_values == 0 { + return Ok(LanceBuffer::empty()); + } else { + return Err(Error::invalid_input_source( + format!("Empty buffers but expected {} values", num_values).into(), + )); + } + } + + let length_size = self.validate_buffer_sizes( + values_buffer, + lengths_buffer, + type_size, + std::any::type_name::(), + )?; + + let values_ref = values_buffer.borrow_to_typed_slice::(); + let values: &[T] = values_ref.as_ref(); + let lengths = lengths_buffer.as_ref(); + + let expected_value_count = usize::try_from(num_values).map_err(|_| { + Error::invalid_input_source( + format!("RLE num_values does not fit in usize: {num_values}").into(), + ) + })?; + // Legacy miniblock encoders rolled back to a power-of-2 checkpoint after a run + // had already crossed it, so a chunk's run lengths can sum past its declared + // value count (the excess values are re-encoded at the start of the next chunk). + // The pre-run-length-width decoder truncated the excess, so miniblock decoding + // clamps rather than rejects to keep those files readable. Block payloads never + // legitimately overflow, so they decode strictly. + let mut decoded: Vec = Vec::new(); + decoded + .try_reserve_exact(expected_value_count) + .map_err(|_| { + Error::invalid_input_source( + format!("RLE decoding cannot allocate {expected_value_count} values").into(), ) })?; - decoded_value_count = decoded_value_count.checked_add(length).ok_or_else(|| { - Error::invalid_input_source("RLE run length sum overflowed usize".into()) - })?; - if decoded_value_count > expected_value_count { + for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) { + let length = self.run_length_width.read_length(length_bytes); + if length == 0 { return Err(Error::invalid_input_source( - format!( - "RLE decoding overflowed expected value count: produced at least {}, expected {}", - decoded_value_count, expected_value_count - ) - .into(), + "RLE decoding encountered a zero run length".into(), )); } + let length = usize::try_from(length).map_err(|_| { + Error::invalid_input_source( + format!("RLE run length does not fit in usize: {length}").into(), + ) + })?; + let remaining = expected_value_count - decoded.len(); + if length > remaining { + if !clamp_overflow { + return Err(Error::invalid_input_source( + format!( + "RLE decoding overflowed expected value count: produced at least {}, expected {}", + decoded.len() + length, + expected_value_count + ) + .into(), + )); + } + decoded.resize(expected_value_count, *value); + break; + } + decoded.resize(decoded.len() + length, *value); } - if decoded_value_count != expected_value_count { + if decoded.len() != expected_value_count { return Err(Error::invalid_input_source( format!( "RLE decoding produced {} values, expected {}", - decoded_value_count, expected_value_count + decoded.len(), + expected_value_count ) .into(), )); } - let mut decoded: Vec = Vec::with_capacity(expected_value_count); - for (value, length_bytes) in values.iter().zip(lengths.chunks_exact(length_size)) { - let length = self.run_length_width.read_length(length_bytes) as usize; - decoded.resize(decoded.len() + length, *value); - } - trace!( "RLE decoded {} {} values", num_values, @@ -919,54 +1628,566 @@ impl RleDecompressor { impl MiniBlockDecompressor for RleDecompressor { fn decompress(&self, data: Vec, num_values: u64) -> Result { - self.decode_data(data, num_values) + self.decode_data(data, num_values, true) + } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + num_values + .checked_mul(self.bits_per_value) + .map(|bits| bits.div_ceil(8)) } } impl BlockDecompressor for RleDecompressor { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { - // fetch the values_size - if data.len() < 8 { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "RLE")?; + let (values_buffer, lengths_buffer) = parse_rle_block_frame(&data)?; + self.decode_data(vec![values_buffer, lengths_buffer], num_values, false) + } + + fn infer_num_values(&self, data: &LanceBuffer) -> Result> { + // Pylance 6.0.1 used this exact RLE signature for structural levels. Newer RLE + // variants are not part of that compatibility case and may contain much wider, + // untrusted run lengths. + if self.bits_per_value != 16 + || self.run_length_width != RunLengthWidth::U8 + || !self.values.is_identity() + || !self.run_lengths.is_identity() + { + return Ok(None); + } + let (values_buffer, lengths_buffer) = parse_rle_block_frame(data)?; + self.sum_run_lengths(&values_buffer, &lengths_buffer) + .map(Some) + } +} + +/// Split an RLE block-format buffer into its `(values, lengths)` sub-buffers. +/// Frame: `[values_size: u64-le][values bytes][run-length bytes]`. +fn parse_rle_block_frame(data: &LanceBuffer) -> Result<(LanceBuffer, LanceBuffer)> { + // fetch the values_size + if data.len() < 8 { + return Err(Error::invalid_input_source( + format!("Insufficient data size: {}", data.len()).into(), + )); + } + + let values_size_bytes: [u8; 8] = data[..8].try_into().expect("slice length already checked"); + let values_size: usize = u64::from_le_bytes(values_size_bytes) + .try_into() + .map_err(|_| { + Error::invalid_input_source( + format!( + "Invalid values buffer size: {}", + u64::from_le_bytes(values_size_bytes) + ) + .into(), + ) + })?; + + // parse values + let values_start: usize = 8; + let lengths_start = values_start + .checked_add(values_size) + .ok_or_else(|| Error::invalid_input_source("Invalid RLE values buffer size".into()))?; + + if data.len() < lengths_start { + return Err(Error::invalid_input_source( + format!("Insufficient data size: {}", data.len()).into(), + )); + } + + let values_buffer = data.slice_with_length(values_start, values_size); + let lengths_buffer = data.slice_with_length(lengths_start, data.len() - lengths_start); + Ok((values_buffer, lengths_buffer)) +} + +#[derive(Clone, Debug)] +enum RleRunLengths { + U8(ScalarBuffer), + U16(ScalarBuffer), + U32(ScalarBuffer), +} + +impl RleRunLengths { + fn try_new(buffer: LanceBuffer, width: RunLengthWidth) -> Result { + let width_bytes = width.bytes_per_value(); + if !buffer.len().is_multiple_of(width_bytes) { return Err(Error::invalid_input_source( - format!("Insufficient data size: {}", data.len()).into(), + format!( + "Invalid RLE run lengths buffer: {} bytes (not divisible by {})", + buffer.len(), + width_bytes + ) + .into(), )); } + Ok(match width { + RunLengthWidth::U8 => Self::U8(buffer.borrow_to_typed_slice()), + RunLengthWidth::U16 => Self::U16(buffer.borrow_to_typed_slice()), + RunLengthWidth::U32 => Self::U32(buffer.borrow_to_typed_slice()), + }) + } - let values_size_bytes: [u8; 8] = - data[..8].try_into().expect("slice length already checked"); - let values_size: u64 = u64::from_le_bytes(values_size_bytes); + fn len(&self) -> usize { + match self { + Self::U8(lengths) => lengths.len(), + Self::U16(lengths) => lengths.len(), + Self::U32(lengths) => lengths.len(), + } + } - // parse values - let values_start: usize = 8; - let values_size: usize = values_size.try_into().map_err(|_| { + fn get(&self, index: usize) -> usize { + match self { + Self::U8(lengths) => lengths[index] as usize, + Self::U16(lengths) => lengths[index] as usize, + Self::U32(lengths) => lengths[index] as usize, + } + } + + fn owned_size(&self) -> usize { + match self { + Self::U8(lengths) => std::mem::size_of_val(lengths.as_ref()), + Self::U16(lengths) => std::mem::size_of_val(lengths.as_ref()), + Self::U32(lengths) => std::mem::size_of_val(lengths.as_ref()), + } + } + + fn into_owned(self) -> Self { + match self { + Self::U8(lengths) => Self::U8(ScalarBuffer::from(lengths.as_ref().to_vec())), + Self::U16(lengths) => Self::U16(ScalarBuffer::from(lengths.as_ref().to_vec())), + Self::U32(lengths) => Self::U32(ScalarBuffer::from(lengths.as_ref().to_vec())), + } + } + + fn deep_size(&self) -> usize { + match self { + Self::U8(lengths) => lengths.inner().capacity(), + Self::U16(lengths) => lengths.inner().capacity(), + Self::U32(lengths) => lengths.inner().capacity(), + } + } +} + +/// Validated physical RLE runs for `u16` values. +/// +/// The values and original-width lengths remain unexpanded. The constructor +/// verifies that every length is non-zero and that the runs cover exactly +/// `num_values` logical values. +#[derive(Clone, Debug)] +pub(crate) struct RleRuns { + values: ScalarBuffer, + lengths: RleRunLengths, + num_values: usize, + coalesced_runs: usize, +} + +impl RleRuns { + fn try_new( + values_buffer: LanceBuffer, + lengths_buffer: LanceBuffer, + run_length_width: RunLengthWidth, + num_values: u64, + ) -> Result { + let num_values = usize::try_from(num_values).map_err(|_| { Error::invalid_input_source( - format!("Invalid values buffer size: {}", values_size).into(), + format!("RLE num_values does not fit in usize: {num_values}").into(), ) })?; - let lengths_start = values_start - .checked_add(values_size) - .ok_or_else(|| Error::invalid_input_source("Invalid RLE values buffer size".into()))?; + let type_size = std::mem::size_of::(); + if !values_buffer.len().is_multiple_of(type_size) { + return Err(Error::invalid_input_source( + format!( + "Invalid RLE u16 values buffer: {} bytes (not divisible by {})", + values_buffer.len(), + type_size + ) + .into(), + )); + } - if data.len() < lengths_start { + let values = values_buffer.borrow_to_typed_slice::(); + let lengths = RleRunLengths::try_new(lengths_buffer, run_length_width)?; + if values.len() != lengths.len() { + return Err(Error::invalid_input_source( + format!( + "Inconsistent RLE buffers: {} runs but {} length entries", + values.len(), + lengths.len() + ) + .into(), + )); + } + if values.is_empty() && num_values != 0 { return Err(Error::invalid_input_source( - format!("Insufficient data size: {}", data.len()).into(), + format!("Empty RLE buffers but expected {num_values} values").into(), )); } - let values_buffer = data.slice_with_length(values_start, values_size); - let lengths_buffer = data.slice_with_length(lengths_start, data.len() - lengths_start); + let mut decoded_values = 0usize; + let mut coalesced_runs = 0usize; + let mut previous_value = None; + for run in 0..values.len() { + let length = lengths.get(run); + if length == 0 { + return Err(Error::invalid_input_source( + "RLE decoding encountered a zero run length".into(), + )); + } + decoded_values = decoded_values.checked_add(length).ok_or_else(|| { + Error::invalid_input_source("RLE run length sum overflowed usize".into()) + })?; + if decoded_values > num_values { + return Err(Error::invalid_input_source( + format!( + "RLE decoding overflowed expected value count: produced at least {}, expected {}", + decoded_values, num_values + ) + .into(), + )); + } + if previous_value != Some(values[run]) { + coalesced_runs += 1; + previous_value = Some(values[run]); + } + } + if decoded_values != num_values { + return Err(Error::invalid_input_source( + format!( + "RLE decoding produced {} values, expected {}", + decoded_values, num_values + ) + .into(), + )); + } - self.decode_data(vec![values_buffer, lengths_buffer], num_values) + Ok(Self { + values, + lengths, + num_values, + coalesced_runs, + }) + } + + pub(crate) fn num_values(&self) -> usize { + self.num_values + } + + pub(crate) fn num_runs(&self) -> usize { + self.values.len() + } + + pub(crate) fn coalesced_runs(&self) -> usize { + self.coalesced_runs + } + + pub(crate) fn owned_size(&self) -> usize { + std::mem::size_of_val(self.values.as_ref()) + self.lengths.owned_size() + } + + pub(crate) fn into_owned(self) -> Self { + Self { + values: ScalarBuffer::from(self.values.as_ref().to_vec()), + lengths: self.lengths.into_owned(), + num_values: self.num_values, + coalesced_runs: self.coalesced_runs, + } + } + + pub(crate) fn deep_size(&self) -> usize { + self.values.inner().capacity() + self.lengths.deep_size() + } + + pub(crate) fn value(&self, run: usize) -> u16 { + self.values[run] + } + + pub(crate) fn length(&self, run: usize) -> usize { + self.lengths.get(run) + } + + pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { + (0..self.num_runs()).map(|run| (self.value(run), self.length(run))) + } +} + +impl RleDecompressor { + /// Decode and validate a block frame while preserving its physical runs. + pub(crate) fn decode_u16_runs(&self, data: LanceBuffer, num_values: u64) -> Result { + if self.bits_per_value != 16 { + return Err(Error::invalid_input_source( + format!( + "RLE level values must be 16 bits, got {}", + self.bits_per_value + ) + .into(), + )); + } + let (values_buffer, lengths_buffer) = parse_rle_block_frame(&data)?; + let (values_buffer, lengths_buffer) = + self.decode_child_buffers(values_buffer, lengths_buffer)?; + RleRuns::try_new( + values_buffer, + lengths_buffer, + self.run_length_width, + num_values, + ) } } #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; + use crate::compression::{ + DecompressionStrategy, DefaultDecompressionStrategy, create_rle_decompressor, + }; use crate::data::DataBlock; use crate::encodings::logical::primitive::miniblock::MAX_MINIBLOCK_VALUES; - use crate::{buffer::LanceBuffer, compression::BlockDecompressor}; + use crate::encodings::physical::block::{CompressionConfig, CompressionScheme}; + use crate::{ + buffer::LanceBuffer, + compression::{BlockCompressor, BlockDecompressor}, + }; use arrow_array::Int32Array; + use rstest::rstest; + + fn compress_miniblock( + compressor: &dyn MiniBlockCompressor, + data: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + compressor.compress(MiniBlockCompressionContext::new(0, true, true), data) + } + + fn expand_u16_runs(runs: &RleRuns) -> Vec { + let mut expanded = Vec::with_capacity(runs.num_values()); + for (value, length) in runs.iter() { + expanded.extend(std::iter::repeat_n(value, length)); + } + expanded + } + + #[test] + fn decode_u16_runs_matches_eager() { + // Near-constant u16 levels (the all-null shape): a few long runs. + let mut levels: Vec = vec![1u16; 1000]; + levels.extend(std::iter::repeat_n(0u16, 500)); + levels.extend(std::iter::repeat_n(2u16, 300)); + let num_values = levels.len() as u64; + + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(levels.clone())), + bits_per_value: 16, + num_values, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block) + .unwrap() + .0 + .unwrap(); + + let eager = BlockDecompressor::decompress( + &RleDecompressor::new(16), + Some(frame.clone()), + num_values, + ) + .unwrap(); + let DataBlock::FixedWidth(eager) = eager else { + panic!("expected fixed-width block"); + }; + assert_eq!( + eager.data.borrow_to_typed_slice::().as_ref(), + levels.as_slice() + ); + + // Lazy run form preserves boundaries and expands identically. + let runs = RleDecompressor::new(16) + .decode_u16_runs(frame, num_values) + .unwrap(); + assert_eq!(runs.num_values(), num_values as usize); + // The encoder splits each value into <=255-length runs (4 + 2 + 2 = 8 + // on-disk runs here); the scan identifies 3 coalesced logical runs. + assert_eq!(runs.num_runs(), 8); + assert_eq!(runs.coalesced_runs(), 3); + assert_eq!(expand_u16_runs(&runs), levels); + } + + #[test] + fn decode_u16_runs_empty() { + let mut empty_frame = Vec::new(); + empty_frame.extend_from_slice(&0u64.to_le_bytes()); + let runs = RleDecompressor::new(16) + .decode_u16_runs(LanceBuffer::from(empty_frame), 0) + .unwrap(); + assert_eq!(runs.num_values(), 0); + assert_eq!(runs.num_runs(), 0); + assert_eq!(runs.coalesced_runs(), 0); + + let error = RleDecompressor::new(16) + .decode_u16_runs(LanceBuffer::empty(), 0) + .unwrap_err(); + assert!(error.to_string().contains("Insufficient data size: 0")); + } + + #[test] + fn legacy_block_rle_infers_value_count_without_materializing() { + let num_values = u64::from(u16::MAX) + 8; + let full_runs = num_values / u64::from(u8::MAX); + let remainder = num_values % u64::from(u8::MAX); + let num_runs = full_runs + u64::from(remainder != 0); + let mut frame = Vec::new(); + frame.extend_from_slice(&(num_runs * 2).to_le_bytes()); + frame.extend(std::iter::repeat_n(7_u16, num_runs as usize).flat_map(u16::to_le_bytes)); + frame.extend(std::iter::repeat_n(u8::MAX, full_runs as usize)); + if remainder != 0 { + frame.push(remainder as u8); + } + + let inferred_num_values = RleDecompressor::new(16) + .infer_num_values(&LanceBuffer::from(frame)) + .unwrap(); + assert_eq!(inferred_num_values, Some(num_values)); + } + + #[test] + fn newer_block_rle_does_not_infer_untrusted_run_sum() { + let mut frame = Vec::new(); + frame.extend_from_slice(&2_u64.to_le_bytes()); + frame.extend_from_slice(&7_u16.to_le_bytes()); + frame.extend_from_slice(&u32::MAX.to_le_bytes()); + let frame = LanceBuffer::from(frame); + let decompressor = RleDecompressor::with_run_length_width(16, RunLengthWidth::U32); + + assert_eq!(decompressor.infer_num_values(&frame).unwrap(), None); + let error = BlockDecompressor::decompress(&decompressor, Some(frame), u64::from(u16::MAX)) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("RLE decoding overflowed expected value count") + ); + } + + #[rstest] + #[case::zero(0, 1, "zero run length")] + #[case::underflow(1, 2, "produced 1 values, expected 2")] + #[case::overflow(2, 1, "overflowed expected value count")] + #[case::nonempty_for_empty(1, 0, "overflowed expected value count")] + fn decode_u16_runs_rejects_invalid_coverage( + #[case] run_length: u8, + #[case] num_values: u64, + #[case] expected_message: &str, + ) { + let mut frame = Vec::new(); + frame.extend_from_slice(&2u64.to_le_bytes()); + frame.extend_from_slice(&7u16.to_le_bytes()); + frame.push(run_length); + + let error = RleDecompressor::new(16) + .decode_u16_runs(LanceBuffer::from(frame), num_values) + .unwrap_err(); + assert!(matches!(error, lance_core::Error::InvalidInput { .. })); + assert!(error.to_string().contains(expected_message)); + } + + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn decode_u16_runs_supports_compressed_values_child() { + let levels: Vec = (0..1024) + .flat_map(|run| std::iter::repeat_n((run % 8) as u16, 4)) + .collect(); + let num_values = levels.len() as u64; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(levels.clone())), + bits_per_value: 16, + num_values, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block) + .unwrap() + .0 + .unwrap(); + let (values, lengths) = parse_rle_block_frame(&frame).unwrap(); + + let compression = test_general_compression(); + let compressor = GeneralBufferCompressor::get_compressor(compression).unwrap(); + let mut compressed_values = Vec::new(); + compressor + .compress(values.as_ref(), &mut compressed_values) + .unwrap(); + let mut compressed_frame = Vec::new(); + compressed_frame.extend_from_slice(&(compressed_values.len() as u64).to_le_bytes()); + compressed_frame.extend_from_slice(&compressed_values); + compressed_frame.extend_from_slice(lengths.as_ref()); + + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::wrapped(compression, ProtobufUtils21::flat(16, None)).unwrap(), + ProtobufUtils21::flat(8, None), + ); + let decompressor = create_rle_decompressor( + expect_rle(&encoding), + &DefaultDecompressionStrategy::default(), + ) + .unwrap(); + let runs = decompressor + .decode_u16_runs(LanceBuffer::from(compressed_frame), num_values) + .unwrap(); + assert_eq!(expand_u16_runs(&runs), levels); + } + + #[test] + fn decode_u16_runs_counts_coalesced_runs() { + // A logically constant page is emitted as ceil(N / 255) equal-valued + // runs (the encoder caps run lengths at 255); the validated view records + // that they can collapse to a single logical run. + let num_values = 5000u64; + let constant: Vec = vec![7u16; num_values as usize]; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(constant)), + bits_per_value: 16, + num_values, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block) + .unwrap() + .0 + .unwrap(); + let runs = RleDecompressor::new(16) + .decode_u16_runs(frame, num_values) + .unwrap(); + assert_eq!( + runs.num_runs(), + num_values.div_ceil(u8::MAX as u64) as usize + ); + assert_eq!(runs.coalesced_runs(), 1); + assert_eq!(expand_u16_runs(&runs), vec![7u16; num_values as usize]); + + // Distinct adjacent values must not be merged: alternating single-value + // runs stay separate and still expand to the original. + let alternating: Vec = (0..200u16).map(|i| i % 2).collect(); + let n = alternating.len() as u64; + let block = DataBlock::FixedWidth(FixedWidthDataBlock { + data: LanceBuffer::reinterpret_slice(Arc::from(alternating.clone())), + bits_per_value: 16, + num_values: n, + block_info: BlockInfo::new(), + }); + let frame = BlockCompressor::compress(&RleEncoder::new(), block) + .unwrap() + .0 + .unwrap(); + let runs = RleDecompressor::new(16).decode_u16_runs(frame, n).unwrap(); + assert_eq!( + runs.coalesced_runs() as u64, + n, + "no two adjacent values are equal" + ); + assert_eq!(expand_u16_runs(&runs), alternating); + } + // ========== Core Functionality Tests ========== #[test] @@ -977,7 +2198,7 @@ mod tests { let array = Int32Array::from(vec![1, 1, 1, 2, 2, 3, 3, 3, 3]); let data_block = DataBlock::from_array(array); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, data_block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, data_block).unwrap(); assert_eq!(compressed.num_values, 9); assert_eq!(compressed.chunks.len(), 1); @@ -998,8 +2219,7 @@ mod tests { data.extend(&[100i32; 300]); // Will be split into 255+45 let array = Int32Array::from(data); - let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Should have 6 runs total (4 for first value, 2 for second) let lengths_buffer = &compressed.data[1]; @@ -1013,7 +2233,7 @@ mod tests { let data = vec![42i32; 1000]; let array = Int32Array::from(data); let (compressed, encoding) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); assert_eq!(compressed.data[0].len(), 4); assert_eq!(compressed.data[1].len(), 2); @@ -1046,6 +2266,326 @@ mod tests { } } + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_compressed_values_child() { + let compression = test_general_compression(); + let encoder = + RleEncoder::with_child_encoding(RunLengthWidth::U8, Some(compression), None, false); + let array = Int32Array::from(repeating_runs(1024, 4)); + let (compressed, encoding) = + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + + let decompressor = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap(); + let decoded = + MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4) + .unwrap(); + assert_decoded_i32_eq(decoded, &repeating_runs(1024, 4)); + } + + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_compressed_run_lengths_child() { + let compression = test_general_compression(); + let encoder = + RleEncoder::with_child_encoding(RunLengthWidth::U8, None, Some(compression), false); + let expected = repeating_runs(1024, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + + let decompressor = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap(); + let decoded = + MiniBlockDecompressor::decompress(decompressor.as_ref(), compressed.data, 1024 * 4) + .unwrap(); + assert_decoded_i32_eq(decoded, &expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_bitpacked_run_lengths_child() { + use crate::encodings::physical::bitpacking::OutOfLineBitpacking; + + let expected = repeating_runs(1024, 4); + let (compressed, _) = compress_miniblock( + &RleEncoder::new(), + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + let run_lengths = compressed.data[1].clone(); + let num_runs = run_lengths.len() as u64; + let run_lengths_block = DataBlock::FixedWidth(FixedWidthDataBlock { + bits_per_value: 8, + data: run_lengths, + num_values: num_runs, + block_info: BlockInfo::default(), + }); + let bitpacked_run_lengths = + BlockCompressor::compress(&OutOfLineBitpacking::new(3, 8), run_lengths_block) + .unwrap() + .0 + .unwrap(); + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::flat(32, None), + ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)), + ); + + let decompressor = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap(); + let decoded = MiniBlockDecompressor::decompress( + decompressor.as_ref(), + vec![compressed.data[0].clone(), bitpacked_run_lengths], + expected.len() as u64, + ) + .unwrap(); + assert_decoded_i32_eq(decoded, &expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_rejects_two_count_dependent_child_encodings() { + let encoding = ProtobufUtils21::rle( + ProtobufUtils21::out_of_line_bitpacking(32, ProtobufUtils21::flat(3, None)), + ProtobufUtils21::out_of_line_bitpacking(8, ProtobufUtils21::flat(3, None)), + ); + + let err = DefaultDecompressionStrategy::default() + .create_miniblock_decompressor(&encoding, &DefaultDecompressionStrategy::default()) + .unwrap_err(); + assert!( + err.to_string() + .contains("cannot both require the run count") + ); + } + + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_general_compression() -> CompressionConfig { + if cfg!(feature = "zstd") { + CompressionConfig::new(CompressionScheme::Zstd, Some(3)) + } else { + CompressionConfig::new(CompressionScheme::Lz4, None) + } + } + + fn repeating_runs(num_runs: usize, run_length: usize) -> Vec { + let mut values = Vec::with_capacity(num_runs * run_length); + for run in 0..num_runs { + values.extend(std::iter::repeat_n((run % 8) as i32, run_length)); + } + values + } + + fn expect_rle(encoding: &CompressiveEncoding) -> &crate::format::pb21::Rle { + match encoding.compression.as_ref().unwrap() { + crate::format::pb21::compressive_encoding::Compression::Rle(rle) => rle, + other => panic!("expected RLE encoding, got {other:?}"), + } + } + + fn assert_decoded_i32_eq(decoded: DataBlock, expected: &[i32]) { + match decoded { + DataBlock::FixedWidth(block) => { + let values = block.data.borrow_to_typed_slice::(); + assert_eq!(values.as_ref(), expected); + } + _ => panic!("Expected FixedWidth block"), + } + } + + #[test] + #[cfg(any(feature = "lz4", feature = "zstd"))] + fn test_rle_miniblock_compressed_children_multiple_chunks() { + let compression = test_general_compression(); + let encoder = RleEncoder::with_child_encoding( + RunLengthWidth::U8, + Some(compression), + Some(compression), + false, + ); + let expected = repeating_runs(8192, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + assert!(compressed.chunks.len() > 1); + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::General(_) + )); + + let decoded = decompress_i32_chunks(&compressed, &encoding); + assert_eq!(decoded, expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_bitpacks_values_child_when_smaller() { + let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true); + let expected = monotonic_runs(2048, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + + let decoded = decompress_i32_chunks(&compressed, &encoding); + assert_eq!(decoded, expected); + } + + #[test] + #[cfg(feature = "bitpacking")] + fn test_rle_miniblock_bitpacks_run_lengths_when_values_do_not_shrink() { + let encoder = RleEncoder::with_child_encoding(RunLengthWidth::U8, None, None, true); + let expected = high_entropy_runs(2048, 4); + let (compressed, encoding) = compress_miniblock( + &encoder, + DataBlock::from_array(Int32Array::from(expected.clone())), + ) + .unwrap(); + + let rle = expect_rle(&encoding); + assert!(matches!( + rle.values.as_ref().unwrap().compression.as_ref().unwrap(), + crate::format::pb21::compressive_encoding::Compression::Flat(_) + )); + assert!(matches!( + rle.run_lengths + .as_ref() + .unwrap() + .compression + .as_ref() + .unwrap(), + crate::format::pb21::compressive_encoding::Compression::OutOfLineBitpacking(_) + )); + + let decoded = decompress_i32_chunks(&compressed, &encoding); + assert_eq!(decoded, expected); + } + + fn decompress_i32_chunks( + compressed: &MiniBlockCompressed, + encoding: &CompressiveEncoding, + ) -> Vec { + let strategy = DefaultDecompressionStrategy::default(); + let decompressor = strategy + .create_miniblock_decompressor(encoding, &strategy) + .unwrap(); + let mut offsets = vec![0usize; compressed.data.len()]; + let mut values_processed = 0u64; + let mut decoded_values = Vec::new(); + + for chunk in &compressed.chunks { + let chunk_values = chunk.num_values(values_processed, compressed.num_values); + let mut chunk_buffers = Vec::with_capacity(chunk.buffer_sizes.len()); + for (idx, size) in chunk.buffer_sizes.iter().enumerate() { + let size = *size as usize; + chunk_buffers.push(compressed.data[idx].slice_with_length(offsets[idx], size)); + offsets[idx] += size; + } + + let decoded = decompressor + .decompress(chunk_buffers, chunk_values) + .unwrap(); + match decoded { + DataBlock::FixedWidth(block) => { + let values = block.data.borrow_to_typed_slice::(); + decoded_values.extend_from_slice(values.as_ref()); + } + _ => panic!("Expected FixedWidth block"), + } + values_processed += chunk_values; + } + + assert_eq!(values_processed, compressed.num_values); + decoded_values + } + + #[cfg(feature = "bitpacking")] + fn monotonic_runs(num_runs: usize, run_length: usize) -> Vec { + let mut values = Vec::with_capacity(num_runs * run_length); + for run in 0..num_runs { + values.extend(std::iter::repeat_n(run as i32, run_length)); + } + values + } + + #[cfg(feature = "bitpacking")] + fn high_entropy_runs(num_runs: usize, run_length: usize) -> Vec { + let mut values = Vec::with_capacity(num_runs * run_length); + let mut state = 7u64; + for _ in 0..num_runs { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + values.extend(std::iter::repeat_n((state >> 32) as i32, run_length)); + } + values + } + #[test] fn test_select_run_length_width_prefers_u16_for_long_runs() { let mut entries = [0u64; 3]; @@ -1089,7 +2629,7 @@ mod tests { block_info: BlockInfo::default(), }); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, block).unwrap(); let decompressor = RleDecompressor::new(bits_per_value); let decompressed = MiniBlockDecompressor::decompress( &decompressor, @@ -1123,7 +2663,7 @@ mod tests { let array = Int32Array::from(data); let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Verify all non-last chunks have power-of-2 values for (i, chunk) in compressed.chunks.iter().enumerate() { @@ -1139,8 +2679,73 @@ mod tests { } } + #[rstest] + #[case::u8_lengths(RunLengthWidth::U8)] + #[case::u16_lengths(RunLengthWidth::U16)] + #[case::u32_lengths(RunLengthWidth::U32)] + fn test_miniblock_chunk_counts_match_encoded_runs(#[case] run_length_width: RunLengthWidth) { + // This pattern crosses the 2,048-value boundary in the middle of a two-value run. + let levels = (0..4098) + .map(|index| if index % 3 == 0 { 1u16 } else { 0u16 }) + .collect::>(); + let num_values = levels.len() as u64; + let encoder = RleEncoder::with_run_length_width(run_length_width); + let (buffers, chunks) = encoder + .encode_data( + &LanceBuffer::reinterpret_vec(levels), + num_values, + u16::BITS as u64, + ) + .unwrap(); + + assert_eq!(buffers.len(), 2); + let bytes_per_length = run_length_width.bytes_per_value(); + let mut values_offset = 0usize; + let mut lengths_offset = 0usize; + let mut values_processed = 0u64; + + for chunk in &chunks { + let values_size = chunk.buffer_sizes[0] as usize; + let lengths_size = chunk.buffer_sizes[1] as usize; + let lengths_end = lengths_offset + lengths_size; + let chunk_lengths = &buffers[1].as_ref()[lengths_offset..lengths_end]; + let length_chunks = chunk_lengths.chunks_exact(bytes_per_length); + assert!(length_chunks.remainder().is_empty()); + let num_runs = length_chunks.len(); + let encoded_values = length_chunks + .map(|bytes| run_length_width.read_length(bytes)) + .sum::(); + let declared_values = chunk.num_values(values_processed, num_values); + + assert_eq!(values_size, num_runs * size_of::()); + assert_eq!(encoded_values, declared_values); + + values_offset += values_size; + lengths_offset = lengths_end; + values_processed += declared_values; + } + + assert_eq!(values_processed, num_values); + assert_eq!(values_offset, buffers[0].len()); + assert_eq!(lengths_offset, buffers[1].len()); + } + // ========== Error Handling Tests ========== + #[test] + fn test_encoder_rejects_zero_progress() { + let error = RleEncoder::new() + .encode_data(&LanceBuffer::empty(), 1, u16::BITS as u64) + .unwrap_err(); + + assert!( + matches!(&error, Error::Internal { .. }), + "expected internal error, got: {error:?}" + ); + assert!(error.to_string().contains("made no progress")); + assert!(error.to_string().contains("values_remaining=1")); + } + #[test] fn test_invalid_buffer_count() { let decompressor = RleDecompressor::new(32); @@ -1189,7 +2794,7 @@ mod tests { } #[test] - fn test_rle_rejects_underflow_overflow_and_zero_lengths() { + fn test_rle_rejects_underflow_and_zero_lengths_and_clamps_overflow() { let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16); let value = LanceBuffer::from(1i32.to_le_bytes().to_vec()); @@ -1212,12 +2817,15 @@ mod tests { ], 5, ) - .unwrap_err(); - assert!( - overflow - .to_string() - .contains("overflowed expected value count") - ); + .unwrap(); + match overflow { + DataBlock::FixedWidth(block) => { + assert_eq!(block.num_values, 5); + let decoded = block.data.borrow_to_typed_slice::(); + assert_eq!(decoded.as_ref(), &[1i32; 5]); + } + _ => panic!("Expected FixedWidth block"), + } let zero = MiniBlockDecompressor::decompress( &decompressor, @@ -1228,6 +2836,61 @@ mod tests { assert!(zero.to_string().contains("zero run length")); } + #[test] + fn test_block_rle_rejects_overflow() { + // Block payloads have no chunk boundaries, so run lengths summing past + // num_values can only be corruption and must stay a hard error. + let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16); + let values = 1i32.to_le_bytes(); + let lengths = 6u16.to_le_bytes(); + let mut payload = Vec::new(); + payload.extend_from_slice(&(values.len() as u64).to_le_bytes()); + payload.extend_from_slice(&values); + payload.extend_from_slice(&lengths); + + let error = + BlockDecompressor::decompress(&decompressor, Some(LanceBuffer::from(payload)), 5) + .unwrap_err(); + assert!(matches!(&error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("overflowed expected value count") + ); + } + + #[test] + fn test_rle_truncates_legacy_chunk_boundary_overflow() { + // Legacy encoders emitted chunks declaring 2048 values whose final run crossed + // the checkpoint boundary (e.g. run lengths summing to 2080); the excess values + // are duplicated at the start of the next chunk and must be ignored here. + let decompressor = RleDecompressor::with_run_length_width(32, RunLengthWidth::U16); + let mut values = Vec::new(); + values.extend_from_slice(&7i32.to_le_bytes()); + values.extend_from_slice(&8i32.to_le_bytes()); + let mut lengths = Vec::new(); + lengths.extend_from_slice(&2000u16.to_le_bytes()); + lengths.extend_from_slice(&80u16.to_le_bytes()); + + let decoded = MiniBlockDecompressor::decompress( + &decompressor, + vec![LanceBuffer::from(values), LanceBuffer::from(lengths)], + 2048, + ) + .unwrap(); + match decoded { + DataBlock::FixedWidth(block) => { + assert_eq!(block.num_values, 2048); + let decoded = block.data.borrow_to_typed_slice::(); + let decoded = decoded.as_ref(); + assert_eq!(decoded.len(), 2048); + assert!(decoded[..2000].iter().all(|&v| v == 7)); + assert!(decoded[2000..].iter().all(|&v| v == 8)); + } + _ => panic!("Expected FixedWidth block"), + } + } + #[test] fn test_empty_data_handling() { let encoder = RleEncoder::new(); @@ -1240,7 +2903,7 @@ mod tests { block_info: BlockInfo::default(), }); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, empty_block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, empty_block).unwrap(); assert_eq!(compressed.num_values, 0); assert!(compressed.data.is_empty()); @@ -1274,8 +2937,7 @@ mod tests { data.extend(vec![777i32; 2000]); let array = Int32Array::from(data.clone()); - let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Manually decompress all chunks let mut reconstructed = Vec::new(); @@ -1388,7 +3050,7 @@ mod tests { // Compress the data let array = Int32Array::from(data.clone()); let (compressed, _) = - MiniBlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + compress_miniblock(&encoder, DataBlock::from_array(array)).unwrap(); // Decompress and verify match MiniBlockDecompressor::decompress( @@ -1458,7 +3120,7 @@ mod tests { block_info: BlockInfo::default(), }); - let (compressed, _) = MiniBlockCompressor::compress(&encoder, block).unwrap(); + let (compressed, _) = compress_miniblock(&encoder, block).unwrap(); // Debug first few chunks for (i, chunk) in compressed.chunks.iter().take(5).enumerate() { @@ -1508,7 +3170,6 @@ mod tests { #[test_log::test(tokio::test)] async fn test_rle_encoding_verification() { use crate::testing::{TestCases, check_round_trip_encoding_of_data}; - use crate::version::LanceFileVersion; use arrow_array::{Array, Int32Array}; use lance_datagen::{ArrayGenerator, RowCount}; use std::collections::HashMap; @@ -1516,7 +3177,7 @@ mod tests { let test_cases = TestCases::default() .with_expected_encoding("rle") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Test both explicit metadata and automatic selection // 1. Test with explicit RLE threshold metadata (also disable BSS) @@ -1566,6 +3227,30 @@ mod tests { ); // 20% variety let arr = Arc::new(Int32Array::from(values)) as Arc; check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await; + + #[cfg(any(feature = "lz4", feature = "zstd"))] + { + let mut metadata = HashMap::new(); + metadata.insert( + "lance-encoding:rle-threshold".to_string(), + "0.8".to_string(), + ); + metadata.insert("lance-encoding:bss".to_string(), "off".to_string()); + metadata.insert( + "lance-encoding:compression".to_string(), + if cfg!(feature = "zstd") { + "zstd".to_string() + } else { + "lz4".to_string() + }, + ); + let mut values = Vec::with_capacity(2048 * 4); + for run in 0..2048 { + values.extend(std::iter::repeat_n(i32::MIN + (run % 8), 4)); + } + let arr = Arc::new(Int32Array::from(values)) as Arc; + check_round_trip_encoding_of_data(vec![arr], &test_cases, metadata).await; + } } /// Generator that produces repetitive patterns suitable for RLE @@ -1616,7 +3301,7 @@ mod tests { let mut data = Vec::new(); data.extend_from_slice(&u64::MAX.to_le_bytes()); - let result = BlockDecompressor::decompress(&decompressor, LanceBuffer::from(data), 1); + let result = BlockDecompressor::decompress(&decompressor, Some(LanceBuffer::from(data)), 1); assert!(result.is_err()); assert!( result @@ -1629,8 +3314,11 @@ mod tests { #[test] fn test_block_decompressor_too_small() { let decompressor = RleDecompressor::new(32); - let result = - BlockDecompressor::decompress(&decompressor, LanceBuffer::from(vec![1, 2, 3]), 10); + let result = BlockDecompressor::decompress( + &decompressor, + Some(LanceBuffer::from(vec![1, 2, 3])), + 10, + ); assert!(result.is_err()); assert!( result @@ -1646,7 +3334,10 @@ mod tests { let data = vec![1i32, 1, 1]; let array = Int32Array::from(data); - let compressed = BlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + let compressed = BlockCompressor::compress(&encoder, DataBlock::from_array(array)) + .unwrap() + .0 + .unwrap(); // Verify header format: first 8 bytes should be values_size as u64 assert!(compressed.len() >= 8); @@ -1670,7 +3361,7 @@ mod tests { let array = Int32Array::from(data.clone()); let data_block = DataBlock::from_array(array); - let compressed = BlockCompressor::compress(&encoder, data_block).unwrap(); + let compressed = BlockCompressor::compress(&encoder, data_block).unwrap().0; let decompressed = BlockDecompressor::decompress(&decompressor, compressed, data.len() as u64).unwrap(); @@ -1699,7 +3390,9 @@ mod tests { assert_eq!(total_values, 10000); let array = Int32Array::from(data.clone()); - let compressed = BlockCompressor::compress(&encoder, DataBlock::from_array(array)).unwrap(); + let compressed = BlockCompressor::compress(&encoder, DataBlock::from_array(array)) + .unwrap() + .0; let decompressed = BlockDecompressor::decompress(&decompressor, compressed, total_values as u64).unwrap(); diff --git a/rust/lance-encoding/src/encodings/physical/value.rs b/rust/lance-encoding/src/encodings/physical/value.rs index c49bbd3efbd..1757d15a9de 100644 --- a/rust/lance-encoding/src/encodings/physical/value.rs +++ b/rust/lance-encoding/src/encodings/physical/value.rs @@ -6,6 +6,7 @@ use arrow_buffer::{BooleanBufferBuilder, bit_util}; use crate::buffer::LanceBuffer; use crate::compression::{ BlockCompressor, BlockDecompressor, FixedPerValueDecompressor, MiniBlockDecompressor, + require_block_payload, }; use crate::data::{ BlockInfo, DataBlock, FixedSizeListBlock, FixedWidthDataBlock, NullableDataBlock, @@ -13,7 +14,7 @@ use crate::data::{ use crate::encodings::logical::primitive::fullzip::{PerValueCompressor, PerValueDataBlock}; use crate::encodings::logical::primitive::miniblock::{ MAX_MINIBLOCK_BYTES, MAX_MINIBLOCK_VALUES, MiniBlockChunk, MiniBlockCompressed, - MiniBlockCompressor, + MiniBlockCompressionContext, MiniBlockCompressor, }; use crate::format::ProtobufUtils21; use crate::format::pb21::compressive_encoding::Compression; @@ -27,7 +28,7 @@ pub struct ValueEncoder {} impl ValueEncoder { /// Use the largest chunk we can smaller than 4KiB - fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> (u64, u64) { + fn find_log_vals_per_chunk(bytes_per_word: u64, values_per_word: u64) -> Result<(u64, u64)> { let mut size_bytes = 2 * bytes_per_word; let (mut log_num_vals, mut num_vals) = match values_per_word { 1 => (1, 2), @@ -35,8 +36,14 @@ impl ValueEncoder { _ => unreachable!(), }; - // If the type is so wide that we can't even fit 2 values we shouldn't be here - assert!(size_bytes < MAX_MINIBLOCK_BYTES); + if size_bytes >= MAX_MINIBLOCK_BYTES { + let num_values = 2 * values_per_word; + return Err(Error::invalid_input(format!( + "Value is too wide for miniblock encoding: {} values require {} bytes but a \ + miniblock chunk is limited to {} bytes.", + num_values, size_bytes, MAX_MINIBLOCK_BYTES + ))); + } while 2 * size_bytes < MAX_MINIBLOCK_BYTES && 2 * num_vals <= *MAX_MINIBLOCK_VALUES { log_num_vals += 1; @@ -44,10 +51,10 @@ impl ValueEncoder { num_vals *= 2; } - (log_num_vals, num_vals) + Ok((log_num_vals, num_vals)) } - fn chunk_data(data: FixedWidthDataBlock) -> MiniBlockCompressed { + fn chunk_data(data: FixedWidthDataBlock) -> Result { // Usually there are X bytes per value. However, when working with boolean // or FSL we might have some number of bits per value that isn't // divisible by 8. In this case, to avoid chunking in the middle of a byte @@ -60,7 +67,7 @@ impl ValueEncoder { // Aim for 4KiB chunks let (log_vals_per_chunk, vals_per_chunk) = - Self::find_log_vals_per_chunk(bytes_per_word, values_per_word); + Self::find_log_vals_per_chunk(bytes_per_word, values_per_word)?; let num_chunks = bit_util::ceil(data.num_values as usize, vals_per_chunk as usize); debug_assert_eq!(vals_per_chunk % values_per_word, 0); let bytes_per_chunk = bytes_per_word * (vals_per_chunk / values_per_word); @@ -99,11 +106,11 @@ impl ValueEncoder { debug_assert_eq!(chunks.len(), num_chunks); - MiniBlockCompressed { + Ok(MiniBlockCompressed { chunks, data: vec![data_buffer], num_values: data.num_values, - } + }) } } @@ -177,7 +184,7 @@ impl ValueEncoder { data: FixedWidthDataBlock, layers: Vec, num_rows: u64, - ) -> (MiniBlockCompressed, CompressiveEncoding) { + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { // Count size to calculate rows per chunk let mut ceil_bytes_validity = 0; let mut cum_dim = 1; @@ -198,7 +205,7 @@ impl ValueEncoder { }; let est_bytes_per_word = (ceil_bytes_validity * vals_per_word) + cum_bytes_per_word; let (log_rows_per_chunk, rows_per_chunk) = - Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word); + Self::find_log_vals_per_chunk(est_bytes_per_word, vals_per_word)?; let num_chunks = num_rows.div_ceil(rows_per_chunk) as usize; @@ -258,17 +265,17 @@ impl ValueEncoder { .chain(std::iter::once(data.data)) .collect::>(); - ( + Ok(( MiniBlockCompressed { chunks, data: buffers, num_values: num_rows, }, encoding, - ) + )) } - fn miniblock_fsl(data: DataBlock) -> (MiniBlockCompressed, CompressiveEncoding) { + fn miniblock_fsl(data: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { let num_rows = data.num_values(); let fsl = data.as_fixed_size_list().unwrap(); let mut layers = Vec::new(); @@ -452,26 +459,32 @@ impl ValueEncoder { } impl BlockCompressor for ValueEncoder { - fn compress(&self, data: DataBlock) -> Result { - let data = match data { - DataBlock::FixedWidth(fixed_width) => fixed_width.data, - _ => unimplemented!( - "Cannot compress block of type {} with ValueEncoder", + fn compress(&self, data: DataBlock) -> Result<(Option, CompressiveEncoding)> { + let DataBlock::FixedWidth(fixed_width) = data else { + return Err(Error::invalid_input(format!( + "ValueEncoder cannot compress a {} block", data.name() - ), + ))); }; - Ok(data) + Ok(( + Some(fixed_width.data), + ProtobufUtils21::flat(fixed_width.bits_per_value, None), + )) } } impl MiniBlockCompressor for ValueEncoder { - fn compress(&self, chunk: DataBlock) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { + fn compress( + &self, + _context: MiniBlockCompressionContext, + chunk: DataBlock, + ) -> Result<(MiniBlockCompressed, CompressiveEncoding)> { match chunk { DataBlock::FixedWidth(fixed_width) => { let encoding = ProtobufUtils21::flat(fixed_width.bits_per_value, None); - Ok((Self::chunk_data(fixed_width), encoding)) + Ok((Self::chunk_data(fixed_width)?, encoding)) } - DataBlock::FixedSizeList(_) => Ok(Self::miniblock_fsl(chunk)), + DataBlock::FixedSizeList(_) => Self::miniblock_fsl(chunk), _ => Err(Error::invalid_input_source( format!( "Cannot compress a data block of type {} with ValueEncoder", @@ -565,7 +578,8 @@ impl ValueDecompressor { } impl BlockDecompressor for ValueDecompressor { - fn decompress(&self, data: LanceBuffer, num_values: u64) -> Result { + fn decompress(&self, data: Option, num_values: u64) -> Result { + let data = require_block_payload(data, "Flat block")?; let block = self.buffer_to_block(data, num_values); assert_eq!(block.num_values(), num_values); Ok(block) @@ -600,6 +614,15 @@ impl MiniBlockDecompressor for ValueDecompressor { assert_eq!(lists.num_values(), num_values); Ok(lists) } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + if self.has_validity() { + return None; + } + num_values + .checked_mul(self.bits_per_value) + .map(|bits| bits.div_ceil(8)) + } } struct FslDecompressorValidityBuilder { @@ -728,6 +751,15 @@ impl FixedPerValueDecompressor for ValueDecompressor { fn bits_per_value(&self) -> u64 { self.bits_per_value } + + fn decoded_size_bytes(&self, num_values: u64) -> Option { + if self.has_validity() { + return None; + } + num_values + .checked_mul(self.bits_per_value) + .map(|bits| bits.div_ceil(8)) + } } impl PerValueCompressor for ValueEncoder { @@ -750,10 +782,7 @@ impl PerValueCompressor for ValueEncoder { // public tests module because we share the PRIMITIVE_TYPES constant with fixed_size_list #[cfg(test)] mod tests { - use std::{ - collections::HashMap, - sync::{Arc, LazyLock}, - }; + use std::{collections::HashMap, sync::Arc}; use arrow_array::{ Array, ArrayRef, Decimal128Array, FixedSizeListArray, Int32Array, ListArray, UInt8Array, @@ -761,7 +790,7 @@ mod tests { }; use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, TimeUnit}; - use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, array, gen_batch}; + use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, Seed, array, gen_batch}; use crate::{ compression::{FixedPerValueDecompressor, MiniBlockDecompressor}, @@ -769,20 +798,22 @@ mod tests { encodings::{ logical::primitive::{ fullzip::{PerValueCompressor, PerValueDataBlock}, - miniblock::MiniBlockCompressor, + miniblock::{MiniBlockCompressionContext, MiniBlockCompressor}, }, physical::value::ValueDecompressor, }, format::pb21::compressive_encoding::Compression, testing::{ - FnArrayGeneratorProvider, TestCases, check_basic_random, - check_round_trip_encoding_generated, check_round_trip_encoding_of_data, + TestCases, TestEncoding, check_basic_random_case, check_round_trip_encoding_of_data, }, - version::LanceFileVersion, }; use super::ValueEncoder; + fn miniblock_context() -> MiniBlockCompressionContext { + MiniBlockCompressionContext::new(0, true, true) + } + const PRIMITIVE_TYPES: &[DataType] = &[ DataType::Null, DataType::FixedSizeBinary(2), @@ -828,7 +859,7 @@ mod tests { .with_indices(vec![0, 1, 2]) .with_indices(vec![1]) .with_indices(vec![2]) - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); check_round_trip_encoding_of_data(vec![items], &test_cases, HashMap::default()).await; } @@ -839,89 +870,114 @@ mod tests { (0..5000).map(|i| if i % 2 == 0 { Some(i) } else { None }), )); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![items], &test_cases, HashMap::default()).await; } #[test_log::test(tokio::test)] async fn test_value_primitive() { - for data_type in PRIMITIVE_TYPES { + const NUM_ROWS: u32 = 1025; + + let test_cases = TestCases::default() + .with_batch_size(NUM_ROWS) + .with_page_sizes(vec![4096]) + .with_expected_encoding("flat"); + let value_metadata = + HashMap::from([("lance-encoding:compression".to_string(), "none".to_string())]); + + for (seed, data_type) in PRIMITIVE_TYPES.iter().enumerate() { log::info!("Testing encoding for {:?}", data_type); - let field = Field::new("", data_type.clone(), false); - check_basic_random(field).await; + let data = gen_batch() + .with_seed(Seed::from(seed as u64)) + .anon_col(array::rand_type(data_type)) + .into_batch_rows(RowCount::from(NUM_ROWS as u64)) + .unwrap() + .column(0) + .clone(); + + check_round_trip_encoding_of_data(vec![data], &test_cases, value_metadata.clone()) + .await; } } - static LARGE_TYPES: LazyLock> = LazyLock::new(|| { - vec![DataType::FixedSizeList( - Arc::new(Field::new("", DataType::Int32, false)), - 128, - )] - }); - + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_large_primitive() { - for data_type in LARGE_TYPES.iter() { - log::info!("Testing encoding for {:?}", data_type); - let field = Field::new("", data_type.clone(), false); - check_basic_random(field).await; - } + async fn test_large_primitive( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + #[values(4096, 1024 * 1024)] page_size: u64, + #[values(false, true)] use_slicing: bool, + ) { + let data_type = + DataType::FixedSizeList(Arc::new(Field::new("", DataType::Int32, false)), 128); + let field = Field::new("", data_type, false); + check_basic_random_case(field, encoding, page_size, use_slicing).await; } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_decimal128_dictionary_encoding() { - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + async fn test_decimal128_dictionary_encoding( + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + let test_cases = TestCases::default() + .with_encoding(encoding) + .with_expected_encoding("dictionary"); let decimals: Vec = (0..100).collect(); let repeated_strings: Vec<_> = decimals .iter() .cycle() - .take(decimals.len() * 10000) + .take(decimals.len() * 1000) .map(|&v| Some(v as i128)) .collect(); let decimal_array = Arc::new(Decimal128Array::from(repeated_strings)) as ArrayRef; check_round_trip_encoding_of_data(vec![decimal_array], &test_cases, HashMap::new()).await; } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_miniblock_stress() { + async fn test_miniblock_stress( + #[values(false, true)] mixed_validity: bool, + #[values(10, 100, 1500, 15000)] batch_size: u32, + #[values(1000, 2000, 3000, 60000)] page_size: u64, + #[values( + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { // Tests for strange page sizes and batch sizes and validity scenarios for miniblock - // 10K integers, 100 per array, all valid - let data1 = (0..100) - .map(|_| Arc::new(Int32Array::from_iter_values(0..100)) as Arc) - .collect::>(); - - // Same as above but with mixed validity - let data2 = (0..100) + // 10K integers, 100 per array, either all valid or mixed validity. + let data = (0..100) .map(|_| { - Arc::new(Int32Array::from_iter( - (0..100).map(|i| if i % 2 == 0 { Some(i) } else { None }), - )) as Arc - }) - .collect::>(); - - // Same as above but with all null for first half then all valid - // TODO: Re-enable once the all-null path is complete - let _data3 = (0..100) - .map(|chunk_idx| { - Arc::new(Int32Array::from_iter( - (0..100).map(|i| if chunk_idx < 50 { None } else { Some(i) }), - )) as Arc + if mixed_validity { + Arc::new(Int32Array::from_iter( + (0..100).map(|i| if i % 2 == 0 { Some(i) } else { None }), + )) as Arc + } else { + Arc::new(Int32Array::from_iter_values(0..100)) as Arc + } }) .collect::>(); - for data in [data1, data2 /*data3*/] { - for batch_size in [10, 100, 1500, 15000] { - // 40000 bytes of data - let test_cases = TestCases::default() - .with_page_sizes(vec![1000, 2000, 3000, 60000]) - .with_batch_size(batch_size) - .with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default() + .with_page_sizes(vec![page_size]) + .with_batch_size(batch_size) + .with_encoding(encoding); - check_round_trip_encoding_of_data(data.clone(), &test_cases, HashMap::new()).await; - } - } + check_round_trip_encoding_of_data(data, &test_cases, HashMap::new()).await; } fn create_simple_fsl() -> FixedSizeListArray { @@ -963,7 +1019,8 @@ mod tests { let starting_data = DataBlock::from_array(sample_list.clone()); let encoder = ValueEncoder::default(); - let (data, compression) = MiniBlockCompressor::compress(&encoder, starting_data).unwrap(); + let (data, compression) = + MiniBlockCompressor::compress(&encoder, miniblock_context(), starting_data).unwrap(); assert_eq!(data.num_values, 3); assert_eq!(data.data.len(), 3); @@ -989,6 +1046,59 @@ mod tests { assert_eq!(decompressed.as_ref(), &sample_list); } + fn wide_fixed_size_binary() -> ArrayRef { + let wide_value = vec![0xABu8; 5000]; + Arc::new( + arrow_array::FixedSizeBinaryArray::try_from_sparse_iter_with_size( + std::iter::repeat_n(Some(wide_value.as_slice()), 4), + 5000, + ) + .unwrap(), + ) + } + + fn wide_fixed_size_list_bool() -> ArrayRef { + // A wide FSL is sub-byte, so it chunks eight values per word and the + // smallest unit is 16 values rather than 2. + let dimension = 4095; + let values = arrow_array::BooleanArray::from(vec![false; dimension * 2]); + let field = Arc::new(Field::new("item", DataType::Boolean, true)); + Arc::new(FixedSizeListArray::new( + field, + dimension as i32, + Arc::new(values), + None, + )) + } + + #[rstest::rstest] + #[case::fixed_size_binary(wide_fixed_size_binary(), 2)] + #[case::fixed_size_list_bool(wide_fixed_size_list_bool(), 16)] + fn test_wide_value_miniblock_returns_error( + #[case] array: ArrayRef, + #[case] expected_min_values: u64, + ) { + let starting_data = DataBlock::from_array(array); + + let encoder = ValueEncoder::default(); + let result = MiniBlockCompressor::compress(&encoder, miniblock_context(), starting_data); + + let err = result.expect_err("wide values should not be encodable as miniblock"); + assert!( + matches!(err, lance_core::Error::InvalidInput { .. }), + "expected InvalidInput, got {err:?}" + ); + let msg = err.to_string(); + assert!( + msg.contains("too wide for miniblock encoding"), + "unexpected error message: {msg}" + ); + assert!( + msg.contains(&format!("{expected_min_values} values require")), + "unexpected error message: {msg}" + ); + } + #[test] fn test_fsl_value_compression_per_value() { let sample_list = create_simple_fsl(); @@ -1013,6 +1123,11 @@ mod tests { let decompressor = ValueDecompressor::from_fsl(fsl.as_ref()); let num_values = data.num_values; + assert_eq!( + FixedPerValueDecompressor::decoded_size_bytes(&decompressor, num_values), + None, + "nullable FSL output uses multiple buffers and requires the fallback estimate" + ); let decompressed = FixedPerValueDecompressor::decompress(&decompressor, data, num_values).unwrap(); @@ -1033,7 +1148,7 @@ mod tests { let list_array = FixedSizeListArray::new(items_field, 2, items, Some(NullBuffer::new(list_nulls))); - let test_cases = TestCases::default().with_min_file_version(LanceFileVersion::V2_1); + let test_cases = TestCases::default().with_structural_encodings(); check_round_trip_encoding_of_data(vec![Arc::new(list_array)], &test_cases, HashMap::new()) .await; @@ -1051,7 +1166,7 @@ mod tests { let list_arr = ListArray::new(list_field, OffsetBuffer::new(offsets), Arc::new(fsl), None); let test_cases = TestCases::default() - .with_min_file_version(LanceFileVersion::V2_1) + .with_structural_encodings() .with_batch_size(1); check_round_trip_encoding_of_data(vec![Arc::new(list_arr)], &test_cases, HashMap::new()) @@ -1083,7 +1198,8 @@ mod tests { ); let encoder = ValueEncoder::default(); - let (data, compression) = MiniBlockCompressor::compress(&encoder, starting_data).unwrap(); + let (data, compression) = + MiniBlockCompressor::compress(&encoder, miniblock_context(), starting_data).unwrap(); let Compression::FixedSizeList(fsl) = compression.compression.unwrap() else { panic!() @@ -1138,18 +1254,43 @@ mod tests { assert_eq!(decompressed.as_ref(), sample_array.as_ref()); } + #[rstest::rstest] #[test_log::test(tokio::test)] - async fn test_fsl_nullable_items() { - let datagen = Box::new(FnArrayGeneratorProvider::new(move || { - lance_datagen::array::rand_vec_nullable::(Dimension::from(128), 0.5) - })); - - let field = Field::new( - "", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::UInt32, true)), 128), - false, - ); - check_round_trip_encoding_generated(field, datagen, TestCases::default()).await; + async fn test_fsl_nullable_items( + #[values( + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse + )] + encoding: TestEncoding, + ) { + let mut generator = + gen_batch() + .with_seed(Seed::from(0)) + .anon_col(array::rand_vec_nullable::( + Dimension::from(128), + 0.5, + )); + generator.with_random_nulls(0.2); + let source = generator + .into_batch_rows(RowCount::from(1026)) + .unwrap() + .column(0) + .clone(); + let test_cases = TestCases::default() + .with_page_sizes(vec![4096]) + .with_encoding(encoding) + .with_batch_size(257) + .with_range(510..515) + .with_indices(vec![0, 511, 512, 1024]); + + check_round_trip_encoding_of_data( + vec![source.slice(1, 512), source.slice(513, 513)], + &test_cases, + HashMap::new(), + ) + .await; } #[test_log::test(tokio::test)] @@ -1158,7 +1299,7 @@ mod tests { let test_cases = TestCases::default() .with_expected_encoding("flat") - .with_min_file_version(LanceFileVersion::V2_1); + .with_structural_encodings(); // Test both explicit configuration and automatic fallback scenarios // 1. Test explicit "none" compression to force flat encoding diff --git a/rust/lance-encoding/src/lib.rs b/rust/lance-encoding/src/lib.rs index a58e0a14c59..338028b79d0 100644 --- a/rust/lance-encoding/src/lib.rs +++ b/rust/lance-encoding/src/lib.rs @@ -8,6 +8,7 @@ use futures::{FutureExt, TryFutureExt, future::BoxFuture}; use lance_core::Result; +mod array_encoding; pub mod buffer; pub mod compression; pub mod compression_config; @@ -17,13 +18,11 @@ pub mod decoder; pub mod encoder; pub mod encodings; pub mod format; -pub mod previous; pub mod repdef; pub mod statistics; #[cfg(test)] pub mod testing; pub mod utils; -pub mod version; // We can definitely add support for big-endian machines someday. However, it's not a priority and // would involve extensive testing (probably through emulation) to ensure that the encodings are diff --git a/rust/lance-encoding/src/previous.rs b/rust/lance-encoding/src/previous.rs deleted file mode 100644 index eb4e1bcdb0d..00000000000 --- a/rust/lance-encoding/src/previous.rs +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Legacy code for the 2.0 format that is no longer used in 2.1+ - -pub mod decoder; -pub mod encoder; -pub mod encodings; diff --git a/rust/lance-encoding/src/previous/decoder.rs b/rust/lance-encoding/src/previous/decoder.rs deleted file mode 100644 index bf32bea3d7c..00000000000 --- a/rust/lance-encoding/src/previous/decoder.rs +++ /dev/null @@ -1,132 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::{collections::VecDeque, ops::Range}; - -use crate::decoder::{ - FilterExpression, NextDecodeTask, PriorityRange, ScheduledScanLine, SchedulerContext, -}; - -use arrow_schema::DataType; -use futures::future::BoxFuture; -use lance_core::{Error, Result}; - -pub trait SchedulingJob: std::fmt::Debug { - fn schedule_next( - &mut self, - context: &mut SchedulerContext, - priority: &dyn PriorityRange, - ) -> Result; - - fn num_rows(&self) -> u64; -} - -/// A scheduler for a field's worth of data -/// -/// Each field in a reader's output schema maps to one field scheduler. This scheduler may -/// map to more than one column. For example, one field of struct data may -/// cover many columns of child data. In fact, the entire file is treated as one -/// top-level struct field. -/// -/// The scheduler is responsible for calculating the necessary I/O. One schedule_range -/// request could trigger multiple batches of I/O across multiple columns. The scheduler -/// should emit decoders into the sink as quickly as possible. -/// -/// As soon as the scheduler encounters a batch of data that can decoded then the scheduler -/// should emit a decoder in the "unloaded" state. The decode stream will pull the decoder -/// and start decoding. -/// -/// The order in which decoders are emitted is important. Pages should be emitted in -/// row-major order allowing decode of complete rows as quickly as possible. -/// -/// The `FieldScheduler` should be stateless and `Send` and `Sync`. This is -/// because it might need to be shared. For example, a list page has a reference to -/// the field schedulers for its items column. This is shared with the follow-up I/O -/// task created when the offsets are loaded. -/// -/// See [`crate::decoder`] for more information -pub trait FieldScheduler: Send + Sync + std::fmt::Debug { - /// Called at the beginning of scheduling to initialize the scheduler - fn initialize<'a>( - &'a self, - filter: &'a FilterExpression, - context: &'a SchedulerContext, - ) -> BoxFuture<'a, Result<()>>; - /// Schedules I/O for the requested portions of the field. - /// - /// Note: `ranges` must be ordered and non-overlapping - /// TODO: Support unordered or overlapping ranges in file scheduler - fn schedule_ranges<'a>( - &'a self, - ranges: &[Range], - filter: &FilterExpression, - ) -> Result>; - /// The number of rows in this field - fn num_rows(&self) -> u64; -} - -#[derive(Debug)] -pub struct DecoderReady { - // The decoder that is ready to be decoded - pub decoder: Box, - // The path to the decoder, the first value is the column index - // following values, if present, are nested child indices - // - // For example, a path of [1, 1, 0] would mean to grab the second - // column, then the second child, and then the first child. - // - // It could represent x in the following schema: - // - // score: float64 - // points: struct - // color: string - // location: struct - // x: float64 - // - // Currently, only struct decoders have "children" although other - // decoders may at some point as well. List children are only - // handled through indirect I/O at the moment and so they don't - // need to be represented (yet) - pub path: VecDeque, -} - -/// A decoder for a field's worth of data -/// -/// The decoder is initially "unloaded" (doesn't have all its data). The [`Self::wait_for_loaded`] -/// method should be called to wait for the needed I/O data before attempting to decode -/// any further. -/// -/// Unlike the other decoder types it is assumed that `LogicalPageDecoder` is stateful -/// and only `Send`. This is why we don't need a `rows_to_skip` argument in [`Self::drain`] -pub trait LogicalPageDecoder: std::fmt::Debug + Send { - /// Add a newly scheduled child decoder - /// - /// The default implementation does not expect children and returns - /// an error. - fn accept_child(&mut self, _child: DecoderReady) -> Result<()> { - Err(Error::internal(format!( - "The decoder {:?} does not expect children but received a child", - self - ))) - } - /// Waits until at least `num_rows` have been loaded - fn wait_for_loaded(&'_ mut self, loaded_need: u64) -> BoxFuture<'_, Result<()>>; - /// The number of rows loaded so far - fn rows_loaded(&self) -> u64; - /// The number of rows that still need loading - fn rows_unloaded(&self) -> u64 { - self.num_rows() - self.rows_loaded() - } - /// The total number of rows in the field - fn num_rows(&self) -> u64; - /// The number of rows that have been drained so far - fn rows_drained(&self) -> u64; - /// The number of rows that are still available to drain - fn rows_left(&self) -> u64 { - self.num_rows() - self.rows_drained() - } - /// Creates a task to decode `num_rows` of data into an array - fn drain(&mut self, num_rows: u64) -> Result; - /// The data type of the decoded data - fn data_type(&self) -> &DataType; -} diff --git a/rust/lance-encoding/src/previous/encodings.rs b/rust/lance-encoding/src/previous/encodings.rs deleted file mode 100644 index 67a43993508..00000000000 --- a/rust/lance-encoding/src/previous/encodings.rs +++ /dev/null @@ -1,7 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Legacy code for the 2.0 format that is no longer used in 2.1+ - -pub mod logical; -pub mod physical; diff --git a/rust/lance-encoding/src/previous/encodings/physical/bitpack.rs b/rust/lance-encoding/src/previous/encodings/physical/bitpack.rs deleted file mode 100644 index d80dec351d1..00000000000 --- a/rust/lance-encoding/src/previous/encodings/physical/bitpack.rs +++ /dev/null @@ -1,1684 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::sync::Arc; - -use arrow_array::types::{ - Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, UInt64Type, -}; -use arrow_array::{Array, ArrayRef, ArrowPrimitiveType, PrimitiveArray, cast::AsArray}; -use arrow_buffer::ArrowNativeType; -use arrow_buffer::bit_util::ceil; -use arrow_schema::DataType; -use bytes::Bytes; -use futures::future::{BoxFuture, FutureExt}; -use log::trace; -use num_traits::{AsPrimitive, PrimInt}; - -use lance_arrow::DataTypeExt; -use lance_bitpacking::BitPacking; -use lance_core::{Error, Result}; - -use crate::buffer::LanceBuffer; -use crate::data::BlockInfo; -use crate::data::{DataBlock, FixedWidthDataBlock, NullableDataBlock}; -use crate::decoder::{PageScheduler, PrimitivePageDecoder}; -use crate::format::ProtobufUtils; -use crate::previous::encoder::{ArrayEncoder, EncodedArray}; -use bytemuck::cast_slice; - -const LOG_ELEMS_PER_CHUNK: u8 = 10; -const ELEMS_PER_CHUNK: u64 = 1 << LOG_ELEMS_PER_CHUNK; - -// Compute the compressed_bit_width for a given array of integers -// todo: compute all statistics before encoding -// todo: see how to use rust macro to rewrite this function -pub fn compute_compressed_bit_width_for_non_neg(arrays: &[ArrayRef]) -> u64 { - debug_assert!(!arrays.is_empty()); - - let res; - - match arrays[0].data_type() { - DataType::UInt8 => { - let mut global_max: u8 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array); - global_max = global_max.max(array_max.unwrap_or(0)); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - // we will have constant encoding later - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int8 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = global_max_width.max(8 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - - DataType::UInt16 => { - let mut global_max: u16 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max = global_max.max(array_max); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int16 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = - global_max_width.max(16 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - - DataType::UInt32 => { - let mut global_max: u32 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max = global_max.max(array_max); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int32 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = - global_max_width.max(32 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - - DataType::UInt64 => { - let mut global_max: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max = global_max.max(array_max); - } - let num_bits = - arrays[0].data_type().byte_width() as u64 * 8 - global_max.leading_zeros() as u64; - if num_bits == 0 { - res = 1; - } else { - res = num_bits; - } - } - - DataType::Int64 => { - let mut global_max_width: u64 = 0; - for array in arrays { - let primitive_array = array - .as_any() - .downcast_ref::>() - .unwrap(); - let array_max_width = arrow_arith::aggregate::bit_or(primitive_array).unwrap_or(0); - global_max_width = - global_max_width.max(64 - array_max_width.leading_zeros() as u64); - } - if global_max_width == 0 { - res = 1; - } else { - res = global_max_width; - } - } - _ => { - panic!( - "BitpackedForNonNegArrayEncoder only supports data types of UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64" - ); - } - }; - res -} - -// Bitpack integers using fastlanes algorithm, the input is sliced into chunks of 1024 integers, and bitpacked -// chunk by chunk. when the input is not a multiple of 1024, the last chunk is padded with zeros, this is fine because -// we also know the number of rows we have. -// Here self is a borrow of BitpackedForNonNegArrayEncoder, unpacked is a mutable borrow of FixedWidthDataBlock, -// data_type can be one of u8, u16, u32, or u64. -// buffer_index is a mutable borrow of u32, indicating the buffer index of the output EncodedArray. -// It outputs an fastlanes bitpacked EncodedArray -macro_rules! encode_fixed_width { - ($self:expr, $unpacked:expr, $data_type:ty, $buffer_index:expr) => {{ - let num_chunks = $unpacked.num_values.div_ceil(ELEMS_PER_CHUNK); - let num_full_chunks = $unpacked.num_values / ELEMS_PER_CHUNK; - let uncompressed_bit_width = std::mem::size_of::<$data_type>() as u64 * 8; - - // the output vector type is the same as the input type, for example, when input is u16, output is Vec - let packed_chunk_size = 1024 * $self.compressed_bit_width as usize / uncompressed_bit_width as usize; - - let input_slice = $unpacked.data.borrow_to_typed_slice::<$data_type>(); - let input = input_slice.as_ref(); - - let mut output = Vec::with_capacity(num_chunks as usize * packed_chunk_size); - - // Loop over all but the last chunk. - (0..num_full_chunks).for_each(|i| { - let start_elem = (i * ELEMS_PER_CHUNK) as usize; - - let output_len = output.len(); - unsafe { - output.set_len(output_len + packed_chunk_size); - BitPacking::unchecked_pack( - $self.compressed_bit_width, - &input[start_elem..][..ELEMS_PER_CHUNK as usize], - &mut output[output_len..][..packed_chunk_size], - ); - } - }); - - if num_chunks != num_full_chunks { - let last_chunk_elem_num = $unpacked.num_values % ELEMS_PER_CHUNK; - let mut last_chunk = vec![0 as $data_type; ELEMS_PER_CHUNK as usize]; - last_chunk[..last_chunk_elem_num as usize].clone_from_slice( - &input[$unpacked.num_values as usize - last_chunk_elem_num as usize..], - ); - - let output_len = output.len(); - unsafe { - output.set_len(output_len + packed_chunk_size); - BitPacking::unchecked_pack( - $self.compressed_bit_width, - &last_chunk, - &mut output[output_len..][..packed_chunk_size], - ); - } - } - - let bitpacked_for_non_neg_buffer_index = *$buffer_index; - *$buffer_index += 1; - - let encoding = ProtobufUtils::bitpacked_for_non_neg_encoding( - $self.compressed_bit_width as u64, - uncompressed_bit_width, - bitpacked_for_non_neg_buffer_index, - ); - let packed = DataBlock::FixedWidth(FixedWidthDataBlock { - bits_per_value: $self.compressed_bit_width as u64, - data: LanceBuffer::reinterpret_vec(output), - num_values: $unpacked.num_values, - block_info: BlockInfo::new(), - }); - - Result::Ok(EncodedArray { - data: packed, - encoding, - }) - }}; -} - -#[derive(Debug)] -pub struct BitpackedForNonNegArrayEncoder { - pub compressed_bit_width: usize, - pub original_data_type: DataType, -} - -impl BitpackedForNonNegArrayEncoder { - pub fn new(compressed_bit_width: usize, data_type: DataType) -> Self { - Self { - compressed_bit_width, - original_data_type: data_type, - } - } -} - -impl ArrayEncoder for BitpackedForNonNegArrayEncoder { - fn encode( - &self, - data: DataBlock, - data_type: &DataType, - buffer_index: &mut u32, - ) -> Result { - match data { - DataBlock::AllNull(_) => { - let encoding = ProtobufUtils::basic_all_null_encoding(); - Ok(EncodedArray { data, encoding }) - } - DataBlock::FixedWidth(unpacked) => { - match data_type { - DataType::UInt8 | DataType::Int8 => encode_fixed_width!(self, unpacked, u8, buffer_index), - DataType::UInt16 | DataType::Int16 => encode_fixed_width!(self, unpacked, u16, buffer_index), - DataType::UInt32 | DataType::Int32 => encode_fixed_width!(self, unpacked, u32, buffer_index), - DataType::UInt64 | DataType::Int64 => encode_fixed_width!(self, unpacked, u64, buffer_index), - _ => unreachable!("BitpackedForNonNegArrayEncoder only supports data types of UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64"), - } - } - DataBlock::Nullable(nullable) => { - let validity_buffer_index = *buffer_index; - *buffer_index += 1; - - let validity_desc = ProtobufUtils::flat_encoding( - 1, - validity_buffer_index, - /*compression=*/ None, - ); - let encoded_values: EncodedArray; - match *nullable.data { - DataBlock::FixedWidth(unpacked) => { - match data_type { - DataType::UInt8 | DataType::Int8 => encoded_values = encode_fixed_width!(self, unpacked, u8, buffer_index)?, - DataType::UInt16 | DataType::Int16 => encoded_values = encode_fixed_width!(self, unpacked, u16, buffer_index)?, - DataType::UInt32 | DataType::Int32 => encoded_values = encode_fixed_width!(self, unpacked, u32, buffer_index)?, - DataType::UInt64 | DataType::Int64 => encoded_values = encode_fixed_width!(self, unpacked, u64, buffer_index)?, - _ => unreachable!("BitpackedForNonNegArrayEncoder only supports data types of UInt8, Int8, UInt16, Int16, UInt32, Int32, UInt64, Int64"), - } - } - _ => { - return Err(Error::invalid_input_source("Bitpacking only supports fixed width data blocks or a nullable data block with fixed width data block inside or a all null data block".into())); - } - } - let encoding = - ProtobufUtils::basic_some_null_encoding(validity_desc, encoded_values.encoding); - let encoded = DataBlock::Nullable(NullableDataBlock { - data: Box::new(encoded_values.data), - nulls: nullable.nulls, - block_info: BlockInfo::new(), - }); - Ok(EncodedArray { - data: encoded, - encoding, - }) - } - _ => { - Err(Error::invalid_input_source("Bitpacking only supports fixed width data blocks or a nullable data block with fixed width data block inside or a all null data block".into())) - } - } - } -} - -#[derive(Debug)] -pub struct BitpackedForNonNegScheduler { - compressed_bit_width: u64, - uncompressed_bits_per_value: u64, - buffer_offset: u64, -} - -impl BitpackedForNonNegScheduler { - pub fn new( - compressed_bit_width: u64, - uncompressed_bits_per_value: u64, - buffer_offset: u64, - ) -> Self { - Self { - compressed_bit_width, - uncompressed_bits_per_value, - buffer_offset, - } - } - - fn locate_chunk_start(&self, relative_row_num: u64) -> u64 { - let chunk_size = ELEMS_PER_CHUNK * self.compressed_bit_width / 8; - self.buffer_offset + (relative_row_num / ELEMS_PER_CHUNK * chunk_size) - } - - fn locate_chunk_end(&self, relative_row_num: u64) -> u64 { - let chunk_size = ELEMS_PER_CHUNK * self.compressed_bit_width / 8; - self.buffer_offset + (relative_row_num / ELEMS_PER_CHUNK * chunk_size) + chunk_size - } -} - -impl PageScheduler for BitpackedForNonNegScheduler { - fn schedule_ranges( - &self, - ranges: &[std::ops::Range], - scheduler: &Arc, - top_level_row: u64, - ) -> BoxFuture<'static, Result>> { - assert!(!ranges.is_empty()); - - let mut byte_ranges = vec![]; - - // map one bytes to multiple ranges, one bytes has at least one range corresponding to it - let mut bytes_idx_to_range_indices = vec![]; - let first_byte_range = std::ops::Range { - start: self.locate_chunk_start(ranges[0].start), - end: self.locate_chunk_end(ranges[0].end - 1), - }; // the ranges are half-open - byte_ranges.push(first_byte_range); - bytes_idx_to_range_indices.push(vec![ranges[0].clone()]); - - for (i, range) in ranges.iter().enumerate().skip(1) { - let this_start = self.locate_chunk_start(range.start); - let this_end = self.locate_chunk_end(range.end - 1); - - // when the current range start is in the same chunk as the previous range's end, we colaesce this two bytes ranges - // when the current range start is not in the same chunk as the previous range's end, we create a new bytes range - if this_start == self.locate_chunk_start(ranges[i - 1].end - 1) { - byte_ranges.last_mut().unwrap().end = this_end; - bytes_idx_to_range_indices - .last_mut() - .unwrap() - .push(range.clone()); - } else { - byte_ranges.push(this_start..this_end); - bytes_idx_to_range_indices.push(vec![range.clone()]); - } - } - - trace!( - "Scheduling I/O for {} ranges spread across byte range {}..{}", - byte_ranges.len(), - byte_ranges[0].start, - byte_ranges.last().unwrap().end - ); - - let bytes = scheduler.submit_request(byte_ranges.clone(), top_level_row); - - // copy the necessary data from `self` to move into the async block - let compressed_bit_width = self.compressed_bit_width; - let uncompressed_bits_per_value = self.uncompressed_bits_per_value; - let num_rows = ranges.iter().map(|range| range.end - range.start).sum(); - - async move { - let bytes = bytes.await?; - let decompressed_output = bitpacked_for_non_neg_decode( - compressed_bit_width, - uncompressed_bits_per_value, - &bytes, - &bytes_idx_to_range_indices, - num_rows, - ); - Ok(Box::new(BitpackedForNonNegPageDecoder { - uncompressed_bits_per_value, - decompressed_buf: decompressed_output, - }) as Box) - } - .boxed() - } -} - -#[derive(Debug)] -struct BitpackedForNonNegPageDecoder { - // number of bits in the uncompressed value. E.g. this will be 32 for DataType::UInt32 - uncompressed_bits_per_value: u64, - - decompressed_buf: LanceBuffer, -} - -impl PrimitivePageDecoder for BitpackedForNonNegPageDecoder { - fn decode(&self, rows_to_skip: u64, num_rows: u64) -> Result { - if ![8, 16, 32, 64].contains(&self.uncompressed_bits_per_value) { - return Err(Error::invalid_input_source("BitpackedForNonNegPageDecoder should only has uncompressed_bits_per_value of 8, 16, 32, or 64".into())); - } - - let elem_size_in_bytes = self.uncompressed_bits_per_value / 8; - - Ok(DataBlock::FixedWidth(FixedWidthDataBlock { - data: self.decompressed_buf.slice_with_length( - (rows_to_skip * elem_size_in_bytes) as usize, - (num_rows * elem_size_in_bytes) as usize, - ), - bits_per_value: self.uncompressed_bits_per_value, - num_values: num_rows, - block_info: BlockInfo::new(), - })) - } -} - -macro_rules! bitpacked_decode { - ($uncompressed_type:ty, $compressed_bit_width:expr, $data:expr, $bytes_idx_to_range_indices:expr, $num_rows:expr) => {{ - let mut decompressed: Vec<$uncompressed_type> = Vec::with_capacity($num_rows as usize); - let packed_chunk_size_in_byte: usize = (ELEMS_PER_CHUNK * $compressed_bit_width) as usize / 8; - let mut decompress_chunk_buf = vec![0 as $uncompressed_type; ELEMS_PER_CHUNK as usize]; - - for (i, bytes) in $data.iter().enumerate() { - let mut ranges_idx = 0; - let mut curr_range_start = $bytes_idx_to_range_indices[i][0].start; - let mut chunk_num = 0; - - while chunk_num * packed_chunk_size_in_byte < bytes.len() { - // Copy for memory alignment - // TODO: This copy should not be needed - let chunk_in_u8: Vec = bytes[chunk_num * packed_chunk_size_in_byte..] - [..packed_chunk_size_in_byte] - .to_vec(); - chunk_num += 1; - let chunk = cast_slice(&chunk_in_u8); - unsafe { - BitPacking::unchecked_unpack( - $compressed_bit_width as usize, - chunk, - &mut decompress_chunk_buf, - ); - } - - loop { - // Case 1: All the elements after (curr_range_start % ELEMS_PER_CHUNK) inside this chunk are needed. - let elems_after_curr_range_start_in_this_chunk = - ELEMS_PER_CHUNK - curr_range_start % ELEMS_PER_CHUNK; - if curr_range_start + elems_after_curr_range_start_in_this_chunk - <= $bytes_idx_to_range_indices[i][ranges_idx].end - { - decompressed.extend_from_slice( - &decompress_chunk_buf[(curr_range_start % ELEMS_PER_CHUNK) as usize..], - ); - curr_range_start += elems_after_curr_range_start_in_this_chunk; - break; - } else { - // Case 2: Only part of the elements after (curr_range_start % ELEMS_PER_CHUNK) inside this chunk are needed. - let elems_this_range_needed_in_this_chunk = - ($bytes_idx_to_range_indices[i][ranges_idx].end - curr_range_start) - .min(ELEMS_PER_CHUNK - curr_range_start % ELEMS_PER_CHUNK); - decompressed.extend_from_slice( - &decompress_chunk_buf[(curr_range_start % ELEMS_PER_CHUNK) as usize..] - [..elems_this_range_needed_in_this_chunk as usize], - ); - if curr_range_start + elems_this_range_needed_in_this_chunk - == $bytes_idx_to_range_indices[i][ranges_idx].end - { - ranges_idx += 1; - if ranges_idx == $bytes_idx_to_range_indices[i].len() { - break; - } - curr_range_start = $bytes_idx_to_range_indices[i][ranges_idx].start; - } else { - curr_range_start += elems_this_range_needed_in_this_chunk; - } - } - } - } - } - - LanceBuffer::reinterpret_vec(decompressed) - }}; -} - -fn bitpacked_for_non_neg_decode( - compressed_bit_width: u64, - uncompressed_bits_per_value: u64, - data: &[Bytes], - bytes_idx_to_range_indices: &[Vec>], - num_rows: u64, -) -> LanceBuffer { - match uncompressed_bits_per_value { - 8 => bitpacked_decode!( - u8, - compressed_bit_width, - data, - bytes_idx_to_range_indices, - num_rows - ), - 16 => bitpacked_decode!( - u16, - compressed_bit_width, - data, - bytes_idx_to_range_indices, - num_rows - ), - 32 => bitpacked_decode!( - u32, - compressed_bit_width, - data, - bytes_idx_to_range_indices, - num_rows - ), - 64 => bitpacked_decode!( - u64, - compressed_bit_width, - data, - bytes_idx_to_range_indices, - num_rows - ), - _ => unreachable!( - "bitpacked_for_non_neg_decode only supports 8, 16, 32, 64 uncompressed_bits_per_value" - ), - } -} - -#[derive(Debug)] -pub struct BitpackParams { - pub num_bits: u64, - - pub signed: bool, -} - -// Compute the number of bits to use for each item, if this array can be encoded using -// bitpacking encoding. Returns `None` if the type or array data is not supported. -pub fn bitpack_params(arr: &dyn Array) -> Option { - match arr.data_type() { - DataType::UInt8 => bitpack_params_for_type::(arr.as_primitive()), - DataType::UInt16 => bitpack_params_for_type::(arr.as_primitive()), - DataType::UInt32 => bitpack_params_for_type::(arr.as_primitive()), - DataType::UInt64 => bitpack_params_for_type::(arr.as_primitive()), - DataType::Int8 => bitpack_params_for_signed_type::(arr.as_primitive()), - DataType::Int16 => bitpack_params_for_signed_type::(arr.as_primitive()), - DataType::Int32 => bitpack_params_for_signed_type::(arr.as_primitive()), - DataType::Int64 => bitpack_params_for_signed_type::(arr.as_primitive()), - // TODO -- eventually we could support temporal types as well - _ => None, - } -} - -// Compute the number bits to use for bitpacking generically. -// returns None if the array is empty or all nulls -fn bitpack_params_for_type(arr: &PrimitiveArray) -> Option -where - T: ArrowPrimitiveType, - T::Native: PrimInt + AsPrimitive, -{ - let max = arrow_arith::aggregate::bit_or(arr); - let num_bits = - max.map(|max| arr.data_type().byte_width() as u64 * 8 - max.leading_zeros() as u64); - - // we can't bitpack into 0 bits, so the minimum is 1 - num_bits - .map(|num_bits| num_bits.max(1)) - .map(|bits| BitpackParams { - num_bits: bits, - signed: false, - }) -} - -/// determine the minimum number of bits that can be used to represent -/// an array of signed values. It includes all the significant bits for -/// the value + plus 1 bit to represent the sign. If there are no negative values -/// then it will not add a signed bit -fn bitpack_params_for_signed_type(arr: &PrimitiveArray) -> Option -where - T: ArrowPrimitiveType, - T::Native: PrimInt + AsPrimitive, -{ - let mut add_signed_bit = false; - let mut min_leading_bits: Option = None; - for val in arr.iter() { - if val.is_none() { - continue; - } - let val = val.unwrap(); - if min_leading_bits.is_none() { - min_leading_bits = Some(u64::MAX); - } - - if val.to_i64().unwrap() < 0i64 { - min_leading_bits = min_leading_bits.map(|bits| bits.min(val.leading_ones() as u64)); - add_signed_bit = true; - } else { - min_leading_bits = min_leading_bits.map(|bits| bits.min(val.leading_zeros() as u64)); - } - } - - let mut min_leading_bits = arr.data_type().byte_width() as u64 * 8 - min_leading_bits?; - if add_signed_bit { - // Need extra sign bit - min_leading_bits += 1; - } - // cannot bitpack into <1 bit - let num_bits = min_leading_bits.max(1); - Some(BitpackParams { - num_bits, - signed: add_signed_bit, - }) -} -#[derive(Debug)] -pub struct BitpackedArrayEncoder { - num_bits: u64, - signed_type: bool, -} - -impl BitpackedArrayEncoder { - pub fn new(num_bits: u64, signed_type: bool) -> Self { - Self { - num_bits, - signed_type, - } - } -} - -impl ArrayEncoder for BitpackedArrayEncoder { - fn encode( - &self, - data: DataBlock, - _data_type: &DataType, - buffer_index: &mut u32, - ) -> Result { - // calculate the total number of bytes we need to allocate for the destination. - // this will be the number of items in the source array times the number of bits. - let dst_bytes_total = ceil(data.num_values() as usize * self.num_bits as usize, 8); - - let mut dst_buffer = vec![0u8; dst_bytes_total]; - let mut dst_idx = 0; - let mut dst_offset = 0; - - let DataBlock::FixedWidth(unpacked) = data else { - return Err(Error::invalid_input_source( - "Bitpacking only supports fixed width data blocks".into(), - )); - }; - - pack_bits( - &unpacked.data, - self.num_bits, - &mut dst_buffer, - &mut dst_idx, - &mut dst_offset, - ); - - let packed = DataBlock::FixedWidth(FixedWidthDataBlock { - bits_per_value: self.num_bits, - data: LanceBuffer::from(dst_buffer), - num_values: unpacked.num_values, - block_info: BlockInfo::new(), - }); - - let bitpacked_buffer_index = *buffer_index; - *buffer_index += 1; - - let encoding = ProtobufUtils::bitpacked_encoding( - self.num_bits, - unpacked.bits_per_value, - bitpacked_buffer_index, - self.signed_type, - ); - - Ok(EncodedArray { - data: packed, - encoding, - }) - } -} - -fn pack_bits( - src: &LanceBuffer, - num_bits: u64, - dst: &mut [u8], - dst_idx: &mut usize, - dst_offset: &mut u8, -) { - let bit_len = src.len() as u64 * 8; - - let mask = u64::MAX >> (64 - num_bits); - - let mut src_idx = 0; - while src_idx < src.len() { - let mut curr_mask = mask; - let mut curr_src = src[src_idx] & curr_mask as u8; - let mut src_offset = 0; - let mut src_bits_written = 0; - - while src_bits_written < num_bits { - dst[*dst_idx] += (curr_src >> src_offset) << *dst_offset as u64; - let bits_written = (num_bits - src_bits_written) - .min(8 - src_offset) - .min(8 - *dst_offset as u64); - src_bits_written += bits_written; - *dst_offset += bits_written as u8; - src_offset += bits_written; - - if *dst_offset == 8 { - *dst_idx += 1; - *dst_offset = 0; - } - - if src_offset == 8 { - src_idx += 1; - src_offset = 0; - curr_mask >>= 8; - if src_idx == src.len() { - break; - } - curr_src = src[src_idx] & curr_mask as u8; - } - } - - // advance source_offset to the next byte if we're not at the end.. - // note that we don't need to do this if we wrote the full number of bits - // because source index would have been advanced by the inner loop above - if bit_len != num_bits { - let partial_bytes_written = ceil(num_bits as usize, 8); - - // we also want to the next location in src, unless we wrote something - // byte-aligned in which case the logic above would have already advanced - let mut to_next_byte = 1; - if num_bits.is_multiple_of(8) { - to_next_byte = 0; - } - - src_idx += src.len() - partial_bytes_written + to_next_byte; - } - } -} - -// A physical scheduler for bitpacked buffers -#[derive(Debug, Clone, Copy)] -pub struct BitpackedScheduler { - bits_per_value: u64, - uncompressed_bits_per_value: u64, - buffer_offset: u64, - signed: bool, -} - -impl BitpackedScheduler { - pub fn new( - bits_per_value: u64, - uncompressed_bits_per_value: u64, - buffer_offset: u64, - signed: bool, - ) -> Self { - Self { - bits_per_value, - uncompressed_bits_per_value, - buffer_offset, - signed, - } - } -} - -impl PageScheduler for BitpackedScheduler { - fn schedule_ranges( - &self, - ranges: &[std::ops::Range], - scheduler: &Arc, - top_level_row: u64, - ) -> BoxFuture<'static, Result>> { - let mut min = u64::MAX; - let mut max = 0; - - let mut buffer_bit_start_offsets: Vec = vec![]; - let mut buffer_bit_end_offsets: Vec> = vec![]; - let byte_ranges = ranges - .iter() - .map(|range| { - let start_byte_offset = range.start * self.bits_per_value / 8; - let mut end_byte_offset = range.end * self.bits_per_value / 8; - if !(range.end * self.bits_per_value).is_multiple_of(8) { - // If the end of the range is not byte-aligned, we need to read one more byte - end_byte_offset += 1; - - let end_bit_offset = range.end * self.bits_per_value % 8; - buffer_bit_end_offsets.push(Some(end_bit_offset as u8)); - } else { - buffer_bit_end_offsets.push(None); - } - - let start_bit_offset = range.start * self.bits_per_value % 8; - buffer_bit_start_offsets.push(start_bit_offset as u8); - - let start = self.buffer_offset + start_byte_offset; - let end = self.buffer_offset + end_byte_offset; - min = min.min(start); - max = max.max(end); - - start..end - }) - .collect::>(); - - trace!( - "Scheduling I/O for {} ranges spread across byte range {}..{}", - byte_ranges.len(), - min, - max - ); - - let bytes = scheduler.submit_request(byte_ranges, top_level_row); - - let bits_per_value = self.bits_per_value; - let uncompressed_bits_per_value = self.uncompressed_bits_per_value; - let signed = self.signed; - async move { - let bytes = bytes.await?; - Ok(Box::new(BitpackedPageDecoder { - buffer_bit_start_offsets, - buffer_bit_end_offsets, - bits_per_value, - uncompressed_bits_per_value, - signed, - data: bytes, - }) as Box) - } - .boxed() - } -} - -#[derive(Debug)] -struct BitpackedPageDecoder { - // bit offsets of the first value within each buffer - buffer_bit_start_offsets: Vec, - - // bit offsets of the last value within each buffer. e.g. if there was a buffer - // with 2 values, packed into 5 bits, this would be [Some(3)], indicating that - // the bits from the 3rd->8th bit in the last byte shouldn't be decoded. - buffer_bit_end_offsets: Vec>, - - // the number of bits used to represent a compressed value. E.g. if the max value - // in the page was 7 (0b111), then this will be 3 - bits_per_value: u64, - - // number of bits in the uncompressed value. E.g. this will be 32 for u32 - uncompressed_bits_per_value: u64, - - // whether or not to use the msb as a sign bit during decoding - signed: bool, - - data: Vec, -} - -impl PrimitivePageDecoder for BitpackedPageDecoder { - fn decode(&self, rows_to_skip: u64, num_rows: u64) -> Result { - let num_bytes = self.uncompressed_bits_per_value / 8 * num_rows; - let mut dest = vec![0; num_bytes as usize]; - - // current maximum supported bits per value = 64 - debug_assert!(self.bits_per_value <= 64); - - let mut rows_to_skip = rows_to_skip; - let mut rows_taken = 0; - let byte_len = self.uncompressed_bits_per_value / 8; - let mut dst_idx = 0; // index for current byte being written to destination buffer - - // create bit mask for source bits - let mask = u64::MAX >> (64 - self.bits_per_value); - - for i in 0..self.data.len() { - let src = &self.data[i]; - let (mut src_idx, mut src_offset) = match compute_start_offset( - rows_to_skip, - src.len(), - self.bits_per_value, - self.buffer_bit_start_offsets[i], - self.buffer_bit_end_offsets[i], - ) { - StartOffset::SkipFull(rows_to_skip_here) => { - rows_to_skip -= rows_to_skip_here; - continue; - } - StartOffset::SkipSome(buffer_start_offset) => ( - buffer_start_offset.index, - buffer_start_offset.bit_offset as u64, - ), - }; - - while src_idx < src.len() && rows_taken < num_rows { - rows_taken += 1; - let mut curr_mask = mask; // copy mask - - // current source byte being written to destination - let mut curr_src = src[src_idx] & (curr_mask << src_offset) as u8; - - // how many bits from the current source value have been written to destination - let mut src_bits_written = 0; - - // the offset within the current destination byte to write to - let mut dst_offset = 0; - - let is_negative = is_encoded_item_negative( - src, - src_idx, - src_offset, - self.bits_per_value as usize, - ); - - while src_bits_written < self.bits_per_value { - // write bits from current source byte into destination - dest[dst_idx] += (curr_src >> src_offset) << dst_offset; - let bits_written = (self.bits_per_value - src_bits_written) - .min(8 - src_offset) - .min(8 - dst_offset); - src_bits_written += bits_written; - dst_offset += bits_written; - src_offset += bits_written; - curr_mask >>= bits_written; - - if dst_offset == 8 { - dst_idx += 1; - dst_offset = 0; - } - - if src_offset == 8 { - src_idx += 1; - src_offset = 0; - if src_idx == src.len() { - break; - } - curr_src = src[src_idx] & curr_mask as u8; - } - } - - // if the type is signed, need to pad out the rest of the byte with 1s - let mut negative_padded_current_byte = false; - if self.signed && is_negative && dst_offset > 0 { - negative_padded_current_byte = true; - while dst_offset < 8 { - dest[dst_idx] |= 1 << dst_offset; - dst_offset += 1; - } - } - - // advance destination offset to the next location - // note that we don't need to do this if we wrote the full number of bits - // because source index would have been advanced by the inner loop above - if self.uncompressed_bits_per_value != self.bits_per_value { - let partial_bytes_written = ceil(self.bits_per_value as usize, 8); - - // we also want to move one location to the next location in destination, - // unless we wrote something byte-aligned in which case the logic above - // would have already advanced dst_idx - let mut to_next_byte = 1; - if self.bits_per_value.is_multiple_of(8) { - to_next_byte = 0; - } - let next_dst_idx = - dst_idx + byte_len as usize - partial_bytes_written + to_next_byte; - - // pad remaining bytes with 1 for negative signed numbers - if self.signed && is_negative { - if !negative_padded_current_byte { - dest[dst_idx] = 0xFF; - } - for i in dest.iter_mut().take(next_dst_idx).skip(dst_idx + 1) { - *i = 0xFF; - } - } - - dst_idx = next_dst_idx; - } - - // If we've reached the last byte, there may be some extra bits from the - // next value outside the range. We don't want to be taking those. - if let Some(buffer_bit_end_offset) = self.buffer_bit_end_offsets[i] - && src_idx == src.len() - 1 - && src_offset >= buffer_bit_end_offset as u64 - { - break; - } - } - } - - Ok(DataBlock::FixedWidth(FixedWidthDataBlock { - data: LanceBuffer::from(dest), - bits_per_value: self.uncompressed_bits_per_value, - num_values: num_rows, - block_info: BlockInfo::new(), - })) - } -} - -fn is_encoded_item_negative(src: &Bytes, src_idx: usize, src_offset: u64, num_bits: usize) -> bool { - let mut last_byte_idx = src_idx + ((src_offset as usize + num_bits) / 8); - let shift_amount = (src_offset as usize + num_bits) % 8; - let shift_amount = if shift_amount == 0 { - last_byte_idx -= 1; - 7 - } else { - shift_amount - 1 - }; - let last_byte = src[last_byte_idx]; - let sign_bit_mask = 1 << shift_amount; - let sign_bit = last_byte & sign_bit_mask; - - sign_bit > 0 -} - -#[derive(Debug, PartialEq)] -struct BufferStartOffset { - index: usize, - bit_offset: u8, -} - -#[derive(Debug, PartialEq)] -enum StartOffset { - // skip the full buffer. The value is how many rows are skipped - // by skipping the full buffer (e.g., # rows in buffer) - SkipFull(u64), - - // skip to some start offset in the buffer - SkipSome(BufferStartOffset), -} - -/// compute how far ahead in this buffer should we skip ahead and start reading -/// -/// * `rows_to_skip` - how many rows to skip -/// * `buffer_len` - length buf buffer (in bytes) -/// * `bits_per_value` - number of bits used to represent a single bitpacked value -/// * `buffer_start_bit_offset` - offset of the start of the first value within the -/// buffer's first byte -/// * `buffer_end_bit_offset` - end bit of the last value within the buffer. Can be -/// `None` if the end of the last value is byte aligned with end of buffer. -fn compute_start_offset( - rows_to_skip: u64, - buffer_len: usize, - bits_per_value: u64, - buffer_start_bit_offset: u8, - buffer_end_bit_offset: Option, -) -> StartOffset { - let rows_in_buffer = rows_in_buffer( - buffer_len, - bits_per_value, - buffer_start_bit_offset, - buffer_end_bit_offset, - ); - if rows_to_skip >= rows_in_buffer { - return StartOffset::SkipFull(rows_in_buffer); - } - - let start_bit = rows_to_skip * bits_per_value + buffer_start_bit_offset as u64; - let start_byte = start_bit / 8; - - StartOffset::SkipSome(BufferStartOffset { - index: start_byte as usize, - bit_offset: (start_bit % 8) as u8, - }) -} - -/// calculates the number of rows in a buffer -fn rows_in_buffer( - buffer_len: usize, - bits_per_value: u64, - buffer_start_bit_offset: u8, - buffer_end_bit_offset: Option, -) -> u64 { - let mut bits_in_buffer = (buffer_len * 8) as u64 - buffer_start_bit_offset as u64; - - // if the end of the last value of the buffer isn't byte aligned, subtract the - // end offset from the total number of bits in buffer - if let Some(buffer_end_bit_offset) = buffer_end_bit_offset { - bits_in_buffer -= (8 - buffer_end_bit_offset) as u64; - } - - bits_in_buffer / bits_per_value -} - -#[cfg(test)] -pub mod test { - use crate::{ - format::pb, - testing::{ArrayGeneratorProvider, TestCases, check_round_trip_encoding_generated}, - version::LanceFileVersion, - }; - - use super::*; - use std::{marker::PhantomData, sync::Arc}; - - use arrow_array::{ - ArrayRef, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, Int64Array, - UInt8Array, UInt16Array, UInt32Array, UInt64Array, - types::{UInt8Type, UInt16Type}, - }; - - use arrow_schema::Field; - use lance_datagen::{ - ArrayGenerator, ArrayGeneratorExt, RowCount, - array::{fill, rand_with_distribution}, - gen_batch, - }; - use rand::distr::Uniform; - - #[test] - fn test_bitpack_params() { - fn gen_array(generator: Box) -> ArrayRef { - gen_batch() - .anon_col(generator) - .into_batch_rows(RowCount::from(10000)) - .unwrap() - .column(0) - .clone() - } - - macro_rules! do_test { - ($num_bits:expr, $data_type:ident, $null_probability:expr) => { - let max = 1 << $num_bits - 1; - let mut arr = - gen_array(fill::<$data_type>(max).with_random_nulls($null_probability)); - - // ensure we don't randomly generate all nulls, that won't work - while arr.null_count() == arr.len() { - arr = gen_array(fill::<$data_type>(max).with_random_nulls($null_probability)); - } - let result = bitpack_params(arr.as_ref()); - assert!(result.is_some()); - assert_eq!($num_bits, result.unwrap().num_bits); - }; - } - - let test_cases = vec![ - (5u64, 0.0f64), - (5u64, 0.9f64), - (1u64, 0.0f64), - (1u64, 0.5f64), - (8u64, 0.0f64), - (8u64, 0.5f64), - ]; - - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt8Type, *null_probability); - do_test!(*num_bits, UInt16Type, *null_probability); - do_test!(*num_bits, UInt32Type, *null_probability); - do_test!(*num_bits, UInt64Type, *null_probability); - } - - // do some test cases that that will only work on larger types - let test_cases = vec![ - (13u64, 0.0f64), - (13u64, 0.5f64), - (16u64, 0.0f64), - (16u64, 0.5f64), - ]; - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt16Type, *null_probability); - do_test!(*num_bits, UInt32Type, *null_probability); - do_test!(*num_bits, UInt64Type, *null_probability); - } - let test_cases = vec![ - (25u64, 0.0f64), - (25u64, 0.5f64), - (32u64, 0.0f64), - (32u64, 0.5f64), - ]; - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt32Type, *null_probability); - do_test!(*num_bits, UInt64Type, *null_probability); - } - let test_cases = vec![ - (48u64, 0.0f64), - (48u64, 0.5f64), - (64u64, 0.0f64), - (64u64, 0.5f64), - ]; - for (num_bits, null_probability) in &test_cases { - do_test!(*num_bits, UInt64Type, *null_probability); - } - - // test that it returns None for datatypes that don't support bitpacking - let arr = Float64Array::from_iter_values(vec![0.1, 0.2, 0.3]); - let result = bitpack_params(&arr); - assert!(result.is_none()); - } - - #[test] - fn test_num_compressed_bits_signed_types() { - let values = Int32Array::from(vec![1, 2, -7]); - let arr = values; - let result = bitpack_params(&arr); - assert!(result.is_some()); - let result = result.unwrap(); - assert_eq!(4, result.num_bits); - assert!(result.signed); - - // check that it doesn't add a sign bit if it doesn't need to - let values = Int32Array::from(vec![1, 2, 7]); - let arr = values; - let result = bitpack_params(&arr); - assert!(result.is_some()); - let result = result.unwrap(); - assert_eq!(3, result.num_bits); - assert!(!result.signed); - } - - #[test] - fn test_rows_in_buffer() { - let test_cases = vec![ - (5usize, 5u64, 0u8, None, 8u64), - (2, 3, 0, Some(5), 4), - (2, 3, 7, Some(6), 2), - ]; - - for ( - buffer_len, - bits_per_value, - buffer_start_bit_offset, - buffer_end_bit_offset, - expected, - ) in test_cases - { - let result = rows_in_buffer( - buffer_len, - bits_per_value, - buffer_start_bit_offset, - buffer_end_bit_offset, - ); - assert_eq!(expected, result); - } - } - - #[test] - fn test_compute_start_offset() { - let result = compute_start_offset(0, 5, 5, 0, None); - assert_eq!( - StartOffset::SkipSome(BufferStartOffset { - index: 0, - bit_offset: 0 - }), - result - ); - - let result = compute_start_offset(10, 5, 5, 0, None); - assert_eq!(StartOffset::SkipFull(8), result); - } - - #[test_log::test(test)] - fn test_will_bitpack_allowed_types_when_possible() { - let test_cases: Vec<(DataType, ArrayRef, u64)> = vec![ - ( - DataType::UInt8, - Arc::new(UInt8Array::from_iter_values(vec![0, 1, 2, 3, 4, 5])), - 3, // bits per value - ), - ( - DataType::UInt16, - Arc::new(UInt16Array::from_iter_values(vec![0, 1, 2, 3, 4, 5 << 8])), - 11, - ), - ( - DataType::UInt32, - Arc::new(UInt32Array::from_iter_values(vec![0, 1, 2, 3, 4, 5 << 16])), - 19, - ), - ( - DataType::UInt64, - Arc::new(UInt64Array::from_iter_values(vec![0, 1, 2, 3, 4, 5 << 32])), - 35, - ), - ( - DataType::Int8, - Arc::new(Int8Array::from_iter_values(vec![0, 2, 3, 4, -5])), - 4, - ), - ( - // check it will not pack with signed bit if all values of signed type are positive - DataType::Int8, - Arc::new(Int8Array::from_iter_values(vec![0, 2, 3, 4, 5])), - 3, - ), - ( - DataType::Int16, - Arc::new(Int16Array::from_iter_values(vec![0, 1, 2, 3, -4, 5 << 8])), - 12, - ), - ( - DataType::Int32, - Arc::new(Int32Array::from_iter_values(vec![0, 1, 2, 3, 4, -5 << 16])), - 20, - ), - ( - DataType::Int64, - Arc::new(Int64Array::from_iter_values(vec![ - 0, - 1, - 2, - -3, - -4, - -5 << 32, - ])), - 36, - ), - ]; - - for (data_type, arr, bits_per_value) in test_cases { - let mut buffed_index = 1; - let params = bitpack_params(arr.as_ref()).unwrap(); - let encoder = BitpackedArrayEncoder { - num_bits: params.num_bits, - signed_type: params.signed, - }; - let data = DataBlock::from_array(arr); - let result = encoder.encode(data, &data_type, &mut buffed_index).unwrap(); - - let data = result.data.as_fixed_width().unwrap(); - assert_eq!(bits_per_value, data.bits_per_value); - - let array_encoding = result.encoding.array_encoding.unwrap(); - - match array_encoding { - pb::array_encoding::ArrayEncoding::Bitpacked(bitpacked) => { - assert_eq!(bits_per_value, bitpacked.compressed_bits_per_value); - assert_eq!( - (data_type.byte_width() * 8) as u64, - bitpacked.uncompressed_bits_per_value - ); - } - _ => { - panic!("Array did not use bitpacking encoding") - } - } - } - - // check it will otherwise use flat encoding - let test_cases: Vec<(DataType, ArrayRef)> = vec![ - // it should use flat encoding for datatypes that don't support bitpacking - ( - DataType::Float32, - Arc::new(Float32Array::from_iter_values(vec![0.1, 0.2, 0.3])), - ), - // it should still use flat encoding if bitpacked encoding would be packed - // into the full byte range - ( - DataType::UInt8, - Arc::new(UInt8Array::from_iter_values(vec![0, 1, 2, 3, 4, 250])), - ), - ( - DataType::UInt16, - Arc::new(UInt16Array::from_iter_values(vec![0, 1, 2, 3, 4, 250 << 8])), - ), - ( - DataType::UInt32, - Arc::new(UInt32Array::from_iter_values(vec![ - 0, - 1, - 2, - 3, - 4, - 250 << 24, - ])), - ), - ( - DataType::UInt64, - Arc::new(UInt64Array::from_iter_values(vec![ - 0, - 1, - 2, - 3, - 4, - 250 << 56, - ])), - ), - ( - DataType::Int8, - Arc::new(Int8Array::from_iter_values(vec![-100])), - ), - ( - DataType::Int16, - Arc::new(Int16Array::from_iter_values(vec![-100 << 8])), - ), - ( - DataType::Int32, - Arc::new(Int32Array::from_iter_values(vec![-100 << 24])), - ), - ( - DataType::Int64, - Arc::new(Int64Array::from_iter_values(vec![-100 << 56])), - ), - ]; - - for (data_type, arr) in test_cases { - if let Some(params) = bitpack_params(arr.as_ref()) { - assert_eq!(params.num_bits, data_type.byte_width() as u64 * 8); - } - } - } - - struct DistributionArrayGeneratorProvider< - DataType, - Dist: rand::distr::Distribution + Clone + Send + Sync + 'static, - > - where - DataType::Native: Copy + 'static, - PrimitiveArray: From> + 'static, - DataType: ArrowPrimitiveType, - { - phantom: PhantomData, - distribution: Dist, - } - - impl DistributionArrayGeneratorProvider - where - Dist: rand::distr::Distribution + Clone + Send + Sync + 'static, - DataType::Native: Copy + 'static, - PrimitiveArray: From> + 'static, - DataType: ArrowPrimitiveType, - { - fn new(dist: Dist) -> Self { - Self { - distribution: dist, - phantom: Default::default(), - } - } - } - - impl ArrayGeneratorProvider for DistributionArrayGeneratorProvider - where - Dist: rand::distr::Distribution + Clone + Send + Sync + 'static, - DataType::Native: Copy + 'static, - PrimitiveArray: From> + 'static, - DataType: ArrowPrimitiveType, - { - fn provide(&self) -> Box { - rand_with_distribution::(self.distribution.clone()) - } - - fn copy(&self) -> Box { - Box::new(Self { - phantom: self.phantom, - distribution: self.distribution.clone(), - }) - } - } - - #[test_log::test(tokio::test)] - async fn test_bitpack_primitive() { - let bitpacked_test_cases: &Vec<(DataType, Box)> = &vec![ - // check less than one byte for multi-byte type - ( - DataType::UInt32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(0, 19).unwrap(), - ), - ), - ), - // // check that more than one byte for multi-byte type - ( - DataType::UInt32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(5 << 7, 6 << 7).unwrap(), - ), - ), - ), - ( - DataType::UInt64, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(5 << 42, 6 << 42).unwrap(), - ), - ), - ), - // check less than one byte for single-byte type - ( - DataType::UInt8, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(0, 19).unwrap(), - ), - ), - ), - // check less than one byte for single-byte type - ( - DataType::UInt64, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(129, 259).unwrap(), - ), - ), - ), - // check byte aligned for single byte - ( - DataType::UInt32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - // this range should always give 8 bits - Uniform::new(200, 250).unwrap(), - ), - ), - ), - // check where the num_bits divides evenly into the bit length of the type - ( - DataType::UInt64, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(1, 3).unwrap(), // 2 bits - ), - ), - ), - // check byte aligned for multiple bytes - ( - DataType::UInt32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - // this range should always always give 16 bits - Uniform::new(200 << 8, 250 << 8).unwrap(), - ), - ), - ), - // check byte aligned where the num bits doesn't divide evenly into the byte length - ( - DataType::UInt64, - Box::new( - DistributionArrayGeneratorProvider::>::new( - // this range should always give 24 hits - Uniform::new(200 << 16, 250 << 16).unwrap(), - ), - ), - ), - // check that we can still encode an all-0 array - ( - DataType::UInt32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(0, 1).unwrap(), - ), - ), - ), - // check for signed types - ( - DataType::Int16, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(-5, 5).unwrap(), - ), - ), - ), - ( - DataType::Int64, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(-(5 << 42), 6 << 42).unwrap(), - ), - ), - ), - ( - DataType::Int32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(-(5 << 7), 6 << 7).unwrap(), - ), - ), - ), - // check signed where packed to < 1 byte for multi-byte type - ( - DataType::Int32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(-19, 19).unwrap(), - ), - ), - ), - // check signed byte aligned to single byte - ( - DataType::Int32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - // this range should always give 8 bits - Uniform::new(-120, 120).unwrap(), - ), - ), - ), - // check signed byte aligned to multiple bytes - ( - DataType::Int32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - // this range should always give 16 bits - Uniform::new(-120 << 8, 120 << 8).unwrap(), - ), - ), - ), - // check that it works for all positive integers even if type is signed - ( - DataType::Int32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(10, 20).unwrap(), - ), - ), - ), - // check that all 0 works for signed type - ( - DataType::Int32, - Box::new( - DistributionArrayGeneratorProvider::>::new( - Uniform::new(0, 1).unwrap(), - ), - ), - ), - ]; - - for (data_type, array_gen_provider) in bitpacked_test_cases { - let field = Field::new("", data_type.clone(), false); - let test_cases = TestCases::basic().with_min_file_version(LanceFileVersion::V2_1); - check_round_trip_encoding_generated(field, array_gen_provider.copy(), test_cases).await; - } - } -} diff --git a/rust/lance-encoding/src/repdef.rs b/rust/lance-encoding/src/repdef.rs index b418b906de8..327afe6e3c8 100644 --- a/rust/lance-encoding/src/repdef.rs +++ b/rust/lance-encoding/src/repdef.rs @@ -118,13 +118,16 @@ use arrow_buffer::{ }; use lance_core::{Error, Result, utils::bit::log_2_ceil}; -use crate::buffer::LanceBuffer; +use crate::{ + buffer::LanceBuffer, + encodings::logical::primitive::sparse::{SparseStructuralPlan, SparseStructuralUnraveler}, +}; pub type LevelBuffer = Vec; -/// A contiguous top-level-row range that can be encoded as one structural page. +/// A top-level-row range whose dense rep/def stream fits one mini-block page. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct StructuralPageSplit { +pub(crate) struct MiniBlockRepDefSplit { /// Top-level row offset, relative to the original unsplit page. pub(crate) row_start: u64, /// Number of top-level rows in this split. @@ -137,15 +140,15 @@ pub(crate) struct StructuralPageSplit { pub(crate) num_values: u64, } -/// Planner result for structural page budget handling. +/// Dense mini-block rep/def budget result for one accumulated page. #[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) enum StructuralPagePlan { - /// The original page can be encoded as-is. - Fits, - /// The original page should be split on top-level row boundaries. - Split(Vec), - /// One top-level row is larger than the requested structural page budget. - UnsplittableOverBudget(u64), +pub(crate) enum MiniBlockRepDefBudget { + /// The dense rep/def stream fits one mini-block structural page. + WithinBudget, + /// The dense rep/def stream fits after splitting on top-level row boundaries. + RequiresPageSplit(Vec), + /// A single top-level row has this many rep/def levels and exceeds the budget. + SingleRowOverBudget(u64), } // As we build def levels we add this to special values to indicate that they @@ -199,6 +202,143 @@ enum RawRepDef { Fsl(FslDesc), } +/// A normalized Arrow structural layer shared by dense and sparse serializers. +#[derive(Clone, Copy, Debug)] +pub(crate) enum NormalizedStructuralLayer<'a> { + List { + offsets: &'a [i64], + validity: Option<&'a BooleanBuffer>, + num_slots: usize, + }, + Validity { + validity: Option<&'a BooleanBuffer>, + num_slots: usize, + }, + FixedSizeList { + validity: Option<&'a BooleanBuffer>, + dimension: usize, + num_slots: usize, + }, +} + +/// Structural layers concatenated across input batches exactly once. +/// +/// Dense rep/def serialization and sparse metadata planning both consume this +/// representation so the Arrow nesting is not independently reconstructed. +#[derive(Debug)] +pub(crate) struct NormalizedStructuralPlan { + layers: Vec, + dense_all_valid: bool, +} + +impl NormalizedStructuralPlan { + pub(crate) fn layers(&self) -> impl ExactSizeIterator> { + self.layers.iter().map(|layer| match layer { + RawRepDef::Offsets(OffsetDesc { + offsets, + validity, + num_values, + .. + }) => NormalizedStructuralLayer::List { + offsets, + validity: validity.as_ref(), + num_slots: *num_values, + }, + RawRepDef::Validity(ValidityDesc { + validity, + num_values, + }) => NormalizedStructuralLayer::Validity { + validity: validity.as_ref(), + num_slots: *num_values, + }, + RawRepDef::Fsl(FslDesc { + validity, + dimension, + num_values, + }) => NormalizedStructuralLayer::FixedSizeList { + validity: validity.as_ref(), + dimension: *dimension, + num_slots: *num_values, + }, + }) + } + + fn to_serializer(&self) -> (SerializerContext, Option) { + if self.dense_all_valid { + let def_meaning = self + .layers + .iter() + .map(|_| DefinitionInterpretation::AllValidItem) + .collect::>(); + return ( + SerializerContext { + def_meaning, + rep_levels: LevelBuffer::default(), + spare_rep: LevelBuffer::default(), + def_levels: LevelBuffer::default(), + spare_def: LevelBuffer::default(), + current_rep: 0, + current_def: 0, + current_len: 0, + current_num_specials: 0, + has_fsl: false, + }, + None, + ); + } + + let total_len = self.layers.last().map_or(0, RawRepDef::num_values) + + self + .layers + .iter() + .map(RawRepDef::num_specials) + .sum::(); + let max_rep = self.layers.iter().map(RawRepDef::max_rep).sum::(); + let max_def = self.layers.iter().map(RawRepDef::max_def).sum::(); + let bits_per_rep = if max_rep > 0 { + u64::from(u16::BITS - max_rep.leading_zeros()) + } else { + 0 + }; + let bits_per_def = if max_def > 0 { + u64::from(u16::BITS - max_def.leading_zeros()) + } else { + 0 + }; + let bits_per_level = + (bits_per_rep + bits_per_def > 0).then_some(bits_per_rep + bits_per_def); + + let num_layers = self.layers.len(); + let mut context = SerializerContext::new(total_len, num_layers, max_rep, max_def); + for layer in &self.layers { + match layer { + RawRepDef::Validity(def) => context.record_validity(def), + RawRepDef::Offsets(rep) => context.record_offsets(rep), + RawRepDef::Fsl(fsl) => context.record_fsl(fsl), + } + } + (context, bits_per_level) + } + + pub(crate) fn serialize(&self) -> SerializedRepDefs { + self.to_serializer().0.build() + } + + pub(crate) fn serialize_with_miniblock_repdef_budget( + &self, + max_levels_for_bits: impl FnOnce(u64) -> u64, + num_rows: u64, + num_values: u64, + ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> { + let (context, bits_per_level) = self.to_serializer(); + context.build_with_miniblock_repdef_budget( + bits_per_level.map(max_levels_for_bits), + num_rows, + num_values, + ) + } +} + impl RawRepDef { // Are there any nulls in this layer fn has_nulls(&self) -> bool { @@ -806,21 +946,20 @@ impl SerializerContext { max_levels_per_page: Option, num_rows: u64, num_values: u64, - ) -> Result { + ) -> Result { // Extremely sparse lists can have many rep/def levels for very few // visible leaf values. If this ratio becomes too skewed then a - // miniblock structural chunk can exceed its packed rep/def metadata - // budget even though the value buffers are small. We detect that case - // while normalizing special def levels and split the structural page on - // top-level row boundaries so each emitted page stays within the - // miniblock structural budget. + // mini-block rep/def chunk can exceed its packed metadata budget even + // though the value buffers are small. We detect that case while + // normalizing special def levels and split on top-level row boundaries + // so each emitted dense mini-block page stays within the budget. if self.def_levels.is_empty() { - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } if self.rep_levels.is_empty() { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } if self.rep_levels.len() != self.def_levels.len() { @@ -833,12 +972,12 @@ impl SerializerContext { let Some(max_levels_per_page) = max_levels_per_page else { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); }; if num_values == 0 { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } let max_schema_rep = def_meaning.iter().filter(|level| level.is_list()).count() as u16; @@ -847,7 +986,7 @@ impl SerializerContext { if !should_plan { self.normalize_specials(); - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } let max_visible_level = max_visible_level.unwrap(); @@ -855,7 +994,7 @@ impl SerializerContext { let mut counted_rows = 0u64; let mut counted_values = 0u64; let mut saw_structural_overhead = false; - let mut unsplittable_over_budget = None; + let mut single_row_over_budget_levels = None; let mut current_row_level_start = None; let mut current_row_num_values = 0u64; @@ -876,14 +1015,14 @@ impl SerializerContext { saw_structural_overhead |= row_has_structural_overhead; if row_has_structural_overhead && row_num_levels > max_levels_per_page { - unsplittable_over_budget = Some(row_num_levels); + single_row_over_budget_levels = Some(row_num_levels); } if current_page_num_rows > 0 && (current_page_has_structural_overhead || row_has_structural_overhead) && current_page_num_levels + row_num_levels > max_levels_per_page { - splits.push(StructuralPageSplit { + splits.push(MiniBlockRepDefSplit { row_start: current_page_row_start, num_rows: current_page_num_rows, level_range: current_page_level_start..current_page_level_end, @@ -966,14 +1105,14 @@ impl SerializerContext { ))); } if !saw_structural_overhead { - return Ok(StructuralPagePlan::Fits); + return Ok(MiniBlockRepDefBudget::WithinBudget); } - if let Some(row_num_levels) = unsplittable_over_budget { - return Ok(StructuralPagePlan::UnsplittableOverBudget(row_num_levels)); + if let Some(row_num_levels) = single_row_over_budget_levels { + return Ok(MiniBlockRepDefBudget::SingleRowOverBudget(row_num_levels)); } if current_page_num_rows > 0 { - splits.push(StructuralPageSplit { + splits.push(MiniBlockRepDefSplit { row_start: current_page_row_start, num_rows: current_page_num_rows, level_range: current_page_level_start..current_page_level_end, @@ -983,9 +1122,9 @@ impl SerializerContext { } if splits.len() > 1 { - Ok(StructuralPagePlan::Split(splits)) + Ok(MiniBlockRepDefBudget::RequiresPageSplit(splits)) } else { - Ok(StructuralPagePlan::Fits) + Ok(MiniBlockRepDefBudget::WithinBudget) } } @@ -1023,12 +1162,12 @@ impl SerializerContext { ) } - fn build_with_structural_plan( + fn build_with_miniblock_repdef_budget( mut self, max_levels_per_page: Option, num_rows: u64, num_values: u64, - ) -> Result<(SerializedRepDefs, StructuralPagePlan)> { + ) -> Result<(SerializedRepDefs, MiniBlockRepDefBudget)> { if self.current_len == 0 { return Ok(( SerializedRepDefs::new_with_fixed_size_list_levels( @@ -1037,7 +1176,7 @@ impl SerializerContext { self.def_meaning, self.has_fsl, ), - StructuralPagePlan::Fits, + MiniBlockRepDefBudget::WithinBudget, )); } @@ -1046,7 +1185,7 @@ impl SerializerContext { .into_iter() .rev() .collect::>(); - let plan = self.normalize_specials_and_plan_splits( + let budget = self.normalize_specials_and_plan_splits( &def_meaning, max_levels_per_page, num_rows, @@ -1071,7 +1210,7 @@ impl SerializerContext { def_meaning, self.has_fsl, ), - plan, + budget, )) } } @@ -1409,54 +1548,18 @@ impl RepDefBuilder { /// Converts the validity / offsets buffers that have been gathered so far /// into repetition and definition levels pub fn serialize(builders: Vec) -> SerializedRepDefs { - Self::serialize_builders(builders).0.build() - } - - /// Converts gathered structural buffers into rep/def levels and an encode-time plan. - pub(crate) fn serialize_with_structural_plan( - builders: Vec, - max_levels_for_bits: impl FnOnce(u64) -> u64, - num_rows: u64, - num_values: u64, - ) -> Result<(SerializedRepDefs, StructuralPagePlan)> { - let (context, bits_per_level) = Self::serialize_builders(builders); - context.build_with_structural_plan( - bits_per_level.map(max_levels_for_bits), - num_rows, - num_values, - ) + Self::normalize(builders).serialize() } - fn serialize_builders(builders: Vec) -> (SerializerContext, Option) { + pub(crate) fn normalize(builders: Vec) -> NormalizedStructuralPlan { assert!(!builders.is_empty()); - if builders.iter().all(|b| b.is_empty()) { - // No repetition, all-valid - let def_meaning = builders - .first() - .unwrap() - .repdefs - .iter() - .map(|_| DefinitionInterpretation::AllValidItem) - .collect::>(); - return ( - SerializerContext { - def_meaning, - rep_levels: LevelBuffer::default(), - spare_rep: LevelBuffer::default(), - def_levels: LevelBuffer::default(), - spare_def: LevelBuffer::default(), - current_rep: 0, - current_def: 0, - current_len: 0, - current_num_specials: 0, - has_fsl: false, - }, - None, - ); - } - let num_layers = builders[0].num_layers(); - let combined_layers = (0..num_layers) + debug_assert!( + builders + .iter() + .all(|builder| builder.num_layers() == num_layers) + ); + let layers = (0..num_layers) .map(|layer_index| { Self::concat_layers( builders.iter().map(|b| &b.repdefs[layer_index]), @@ -1464,47 +1567,10 @@ impl RepDefBuilder { ) }) .collect::>(); - debug_assert!( - builders - .iter() - .all(|b| b.num_layers() == builders[0].num_layers()) - ); - - let total_len = combined_layers.last().unwrap().num_values() - + combined_layers - .iter() - .map(|l| l.num_specials()) - .sum::(); - let max_rep = combined_layers.iter().map(|l| l.max_rep()).sum::(); - let max_def = combined_layers.iter().map(|l| l.max_def()).sum::(); - let bits_per_rep = if max_rep > 0 { - u64::from(u16::BITS - max_rep.leading_zeros()) - } else { - 0 - }; - let bits_per_def = if max_def > 0 { - u64::from(u16::BITS - max_def.leading_zeros()) - } else { - 0 - }; - let bits_per_level = - (bits_per_rep + bits_per_def > 0).then_some(bits_per_rep + bits_per_def); - - let mut context = SerializerContext::new(total_len, num_layers, max_rep, max_def); - for layer in combined_layers.into_iter() { - match layer { - RawRepDef::Validity(def) => { - context.record_validity(&def); - } - RawRepDef::Offsets(rep) => { - context.record_offsets(&rep); - } - RawRepDef::Fsl(fsl) => { - context.record_fsl(&fsl); - } - } + NormalizedStructuralPlan { + layers, + dense_all_valid: builders.iter().all(Self::is_empty), } - (context, bits_per_level) } } @@ -1514,6 +1580,7 @@ impl RepDefBuilder { /// This is used during decoding to create the necessary arrow structures #[derive(Debug)] pub struct RepDefUnraveler { + sparse: Option, rep_levels: Option, def_levels: Option, // Maps from definition level to the rep level at which that definition level is visible @@ -1567,6 +1634,7 @@ impl RepDefUnraveler { } } Self { + sparse: None, rep_levels, def_levels, current_def_cmp: 0, @@ -1578,7 +1646,35 @@ impl RepDefUnraveler { } } + pub(crate) fn new_sparse(plan: SparseStructuralPlan) -> Self { + Self { + sparse: Some(SparseStructuralUnraveler::new(plan)), + rep_levels: None, + def_levels: None, + levels_to_rep: Vec::new(), + def_meaning: Arc::new([]), + current_def_cmp: 0, + current_rep_cmp: 0, + current_layer: 0, + num_items: 0, + } + } + + fn ensure_exhausted(&self) -> Result<()> { + if let Some(sparse) = &self.sparse { + sparse.ensure_exhausted()?; + } + Ok(()) + } + + fn is_sparse(&self) -> bool { + self.sparse.is_some() + } + pub fn is_all_valid(&self) -> bool { + if let Some(sparse) = &self.sparse { + return sparse.is_all_valid(); + } self.def_levels.is_none() || self.def_meaning[self.current_layer].is_all_valid() } @@ -1587,15 +1683,19 @@ impl RepDefUnraveler { /// /// This is not valid to call when the current level is a struct/primitive layer because /// in some cases there may be no rep or def information to know this. - pub fn max_lists(&self) -> usize { + pub fn max_lists(&self) -> Result { + if let Some(sparse) = &self.sparse { + return sparse.max_lists(); + } debug_assert!( self.def_meaning[self.current_layer] != DefinitionInterpretation::NullableItem ); - self.rep_levels + Ok(self + .rep_levels .as_ref() // Worst case every rep item is max_rep and a new list .map(|levels| levels.len()) - .unwrap_or(0) + .unwrap_or(0)) } /// Unravels a layer of offsets from the unraveler into the given offset width @@ -1607,6 +1707,9 @@ impl RepDefUnraveler { offsets: &mut Vec, validity: Option<&mut BooleanBufferBuilder>, ) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.unravel_offsets(offsets, validity); + } let rep_levels = self .rep_levels .as_mut() @@ -1757,18 +1860,25 @@ impl RepDefUnraveler { } } - pub fn skip_validity(&mut self) { + pub fn skip_validity(&mut self) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.skip_validity(); + } debug_assert!(self.is_all_valid()); self.current_layer += 1; + Ok(()) } /// Unravels a layer of validity from the definition levels - pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) { + pub fn unravel_validity(&mut self, validity: &mut BooleanBufferBuilder) -> Result<()> { + if let Some(sparse) = self.sparse.as_mut() { + return sparse.unravel_validity(validity); + } let meaning = self.def_meaning[self.current_layer]; if meaning == DefinitionInterpretation::AllValidItem || self.def_levels.is_none() { self.current_layer += 1; validity.append_n(self.num_items as usize, true); - return; + return Ok(()); } self.current_layer += 1; @@ -1786,9 +1896,28 @@ impl RepDefUnraveler { }) { validity.append(is_valid); } + Ok(()) } - pub fn decimate(&mut self, dimension: usize) { + /// Removes all but the first definition level of each fixed-size-list slot + /// + /// The definition levels arrive with one entry per item. A fixed-size-list + /// layer has a single definition level per slot (all `dimension` items in a + /// slot share it) so we keep every `dimension`-th level and drop the rest. + /// + /// `dimension` must be non-zero. A zero dimension can only come from a + /// malformed schema (writers reject it, see + /// [`lance_core::datatypes::validate_fixed_size_list_dimensions`]) and is + /// rejected here rather than allowed to run off the end of the buffer. + pub fn decimate(&mut self, dimension: usize) -> Result<()> { + if dimension == 0 { + return Err(Error::invalid_input( + "Cannot decimate repetition/definition levels with a fixed-size-list dimension of 0; dimension must be a positive integer", + )); + } + if let Some(sparse) = self.sparse.as_mut() { + return sparse.decimate(dimension); + } if self.rep_levels.is_some() { // If we need to support this then I think we need to walk through the rep def levels to find // the spots at which we keep. E.g. if we have: @@ -1804,11 +1933,14 @@ impl RepDefUnraveler { todo!("Not yet supported FSL<...List<...>>"); } let Some(def_levels) = self.def_levels.as_mut() else { - return; + return Ok(()); }; let mut read_idx = 0; let mut write_idx = 0; while read_idx < def_levels.len() { + // SAFETY: `read_idx` is checked against the length by the loop condition and + // `dimension >= 1` (checked above) means `write_idx <= read_idx`, so both + // indices are in bounds. unsafe { *def_levels.get_unchecked_mut(write_idx) = *def_levels.get_unchecked(read_idx); } @@ -1816,6 +1948,7 @@ impl RepDefUnraveler { read_idx += dimension; } def_levels.truncate(write_idx); + Ok(()) } } @@ -1835,44 +1968,104 @@ impl RepDefUnraveler { #[derive(Debug)] pub struct CompositeRepDefUnraveler { unravelers: Vec, + comparisons: Vec, } impl CompositeRepDefUnraveler { pub fn new(unravelers: Vec) -> Self { - Self { unravelers } + Self { + unravelers, + comparisons: Vec::new(), + } + } + + pub(crate) fn add_compatibility_check(&mut self, other: Self) { + self.comparisons.push(other); + } + + pub(crate) fn has_sparse(&self) -> bool { + self.unravelers.iter().any(RepDefUnraveler::is_sparse) + || self.comparisons.iter().any(Self::has_sparse) + } + + pub(crate) fn ensure_exhausted(&self) -> Result<()> { + for unraveler in &self.unravelers { + unraveler.ensure_exhausted()?; + } + for comparison in &self.comparisons { + comparison.ensure_exhausted()?; + } + Ok(()) + } + + fn null_buffers_equal( + left: &Option, + right: &Option, + expected_len: usize, + ) -> bool { + match (left, right) { + (None, None) => true, + (Some(left), Some(right)) => { + left.len() == expected_len + && right.len() == expected_len + && left.iter().eq(right.iter()) + } + (None, Some(right)) => right.len() == expected_len && right.null_count() == 0, + (Some(left), None) => left.len() == expected_len && left.null_count() == 0, + } + } + + fn decimate(&mut self, dimension: usize) -> Result<()> { + for unraveler in &mut self.unravelers { + unraveler.decimate(dimension)?; + } + for comparison in &mut self.comparisons { + comparison.decimate(dimension)?; + } + Ok(()) } /// Unravels a layer of validity /// /// Returns None if there are no null items in this layer - pub fn unravel_validity(&mut self, num_values: usize) -> Option { + pub fn unravel_validity(&mut self, num_values: usize) -> Result> { let is_all_valid = self .unravelers .iter() .all(|unraveler| unraveler.is_all_valid()); - if is_all_valid { + let validity = if is_all_valid { for unraveler in self.unravelers.iter_mut() { - unraveler.skip_validity(); + unraveler.skip_validity()?; } None } else { let mut validity = BooleanBufferBuilder::new(num_values); for unraveler in self.unravelers.iter_mut() { - unraveler.unravel_validity(&mut validity); + unraveler.unravel_validity(&mut validity)?; } Some(NullBuffer::new(validity.finish())) + }; + for comparison in &mut self.comparisons { + let other = comparison.unravel_validity(num_values)?; + if !Self::null_buffers_equal(&validity, &other, num_values) { + return Err(Error::invalid_input_source( + format!( + "Structural sibling fields have incompatible validity metadata for {num_values} values" + ) + .into(), + )); + } } + Ok(validity) } pub fn unravel_fsl_validity( &mut self, num_values: usize, dimension: usize, - ) -> Option { - for unraveler in self.unravelers.iter_mut() { - unraveler.decimate(dimension); - } + ) -> Result> { + self.decimate(dimension)?; self.unravel_validity(num_values) } @@ -1881,10 +2074,16 @@ impl CompositeRepDefUnraveler { &mut self, ) -> Result<(OffsetBuffer, Option)> { let mut is_all_valid = true; - let mut max_num_lists = 0; + let mut max_num_lists: usize = 0; for unraveler in self.unravelers.iter() { is_all_valid &= unraveler.is_all_valid(); - max_num_lists += unraveler.max_lists(); + max_num_lists = max_num_lists + .checked_add(unraveler.max_lists()?) + .ok_or_else(|| { + Error::invalid_input_source( + "Combined repetition/definition list count exceeds usize::MAX".into(), + ) + })?; } let mut validity = if is_all_valid { @@ -1901,10 +2100,28 @@ impl CompositeRepDefUnraveler { unraveler.unravel_offsets(&mut offsets, validity.as_mut())?; } - Ok(( - OffsetBuffer::new(ScalarBuffer::from(offsets)), - validity.map(|mut v| NullBuffer::new(v.finish())), - )) + let offsets = OffsetBuffer::new(ScalarBuffer::from(offsets)); + let validity = validity.map(|mut v| NullBuffer::new(v.finish())); + for comparison in &mut self.comparisons { + let (other_offsets, other_validity) = comparison.unravel_offsets::()?; + if offsets.as_ref() != other_offsets.as_ref() + || !Self::null_buffers_equal( + &validity, + &other_validity, + offsets.len().saturating_sub(1), + ) + { + return Err(Error::invalid_input_source( + format!( + "Structural sibling fields have incompatible list metadata for {} slots", + offsets.len().saturating_sub(1) + ) + .into(), + )); + } + } + + Ok((offsets, validity)) } } @@ -2603,6 +2820,10 @@ impl ControlWordParser { mod tests { use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; + use crate::encodings::logical::primitive::sparse::{ + SparsePositionSet, SparseStructuralLayerPlan, SparseStructuralPlan, SparseValidityMeaning, + SparseValiditySet, + }; use crate::repdef::{ CompositeRepDefUnraveler, DefinitionInterpretation, RepDefUnraveler, SerializedRepDefs, }; @@ -2621,6 +2842,31 @@ mod tests { OffsetBuffer::::new(ScalarBuffer::from_iter(values.iter().copied())) } + #[test] + fn sparse_sibling_validity_mismatch_is_invalid_input() { + let sparse = |positions| { + RepDefUnraveler::new_sparse(SparseStructuralPlan { + layers: vec![SparseStructuralLayerPlan::Validity { + num_slots: 2, + validity: SparseValiditySet { + meaning: SparseValidityMeaning::NullPositions, + positions, + }, + }], + num_items: 2, + num_visible_items: 2, + }) + }; + let mut repdef = CompositeRepDefUnraveler::new(vec![sparse(SparsePositionSet::Empty)]); + repdef.add_compatibility_check(CompositeRepDefUnraveler::new(vec![sparse( + SparsePositionSet::Explicit(vec![0]), + )])); + + let err = repdef.unravel_validity(2).unwrap_err(); + assert!(matches!(err, lance_core::Error::InvalidInput { .. })); + assert!(err.to_string().contains("incompatible validity metadata")); + } + #[test] fn test_repdef_empty_offsets() { // Empty offsets should serialize without panicking. @@ -2666,7 +2912,7 @@ mod tests { // Note: validity doesn't exactly round-trip because repdef normalizes some of the // redundant validity values assert_eq!( - unraveler.unravel_validity(9), + unraveler.unravel_validity(9).unwrap(), Some(validity(&[ true, true, true, false, false, false, true, true, false ])) @@ -2804,18 +3050,48 @@ mod tests { )]); assert_eq!( - unraveler.unravel_validity(8), + unraveler.unravel_validity(8).unwrap(), Some(validity(&[ true, false, true, false, false, false, false, false ])) ); - assert_eq!(unraveler.unravel_fsl_validity(4, 2), None); + assert_eq!(unraveler.unravel_fsl_validity(4, 2).unwrap(), None); assert_eq!( - unraveler.unravel_fsl_validity(2, 2), + unraveler.unravel_fsl_validity(2, 2).unwrap(), Some(validity(&[true, false])) ); } + #[test] + fn test_repdef_fsl_zero_dimension_is_invalid_input() { + // A zero dimension can only reach us from a malformed schema. Decimating with it + // used to loop forever, writing past the end of the definition levels buffer. + let mut builder = RepDefBuilder::default(); + builder.add_fsl(Some(validity(&[true, false])), 2, 2); + builder.add_validity_bitmap(validity(&[true, false, true, false])); + + let repdefs = RepDefBuilder::serialize(vec![builder]); + let def = repdefs.definition_levels.unwrap(); + + let mut unraveler = CompositeRepDefUnraveler::new(vec![RepDefUnraveler::new( + None, + Some(def.as_ref().to_vec()), + repdefs.def_meaning.into(), + 4, + )]); + // Consume the item layer so the fixed-size-list layer is next + unraveler.unravel_validity(4).unwrap(); + + let err = unraveler.unravel_fsl_validity(2, 0).unwrap_err(); + assert!(matches!(err, lance_core::Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("dimension must be a positive integer"), + "unexpected error: {}", + err + ); + } + #[test] fn test_repdef_fsl_allvalid_item() { let mut builder = RepDefBuilder::default(); @@ -2847,10 +3123,10 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(8), None); - assert_eq!(unraveler.unravel_fsl_validity(4, 2), None); + assert_eq!(unraveler.unravel_validity(8).unwrap(), None); + assert_eq!(unraveler.unravel_fsl_validity(4, 2).unwrap(), None); assert_eq!( - unraveler.unravel_fsl_validity(2, 2), + unraveler.unravel_fsl_validity(2, 2).unwrap(), Some(validity(&[true, false])) ); } @@ -2928,7 +3204,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, None); @@ -2954,7 +3230,7 @@ mod tests { 9, )]); - assert_eq!(unraveler.unravel_validity(9), None); + assert_eq!(unraveler.unravel_validity(9).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 1, 3, 5, 7, 9]).inner()); assert_eq!(val, None); @@ -3018,7 +3294,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, None); @@ -3048,7 +3324,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, Some(validity(&[true, false, false, true]))); @@ -3078,7 +3354,7 @@ mod tests { 8, )]); - assert_eq!(unraveler.unravel_validity(6), None); + assert_eq!(unraveler.unravel_validity(6).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 4, 4, 4, 6]).inner()); assert_eq!(val, Some(validity(&[true, false, true, true]))); @@ -3106,11 +3382,11 @@ mod tests { )]); assert_eq!( - unraveler.unravel_validity(4), + unraveler.unravel_validity(4).unwrap(), Some(validity(&[false, true, false, false])) ); assert_eq!( - unraveler.unravel_validity(4), + unraveler.unravel_validity(4).unwrap(), Some(validity(&[false, true, false, false])) ); let (off, val) = unraveler.unravel_offsets::().unwrap(); @@ -3139,14 +3415,14 @@ mod tests { )]); assert_eq!( - unraveler.unravel_validity(5), + unraveler.unravel_validity(5).unwrap(), Some(validity(&[false, false, true, true, false])) ); assert_eq!( - unraveler.unravel_validity(5), + unraveler.unravel_validity(5).unwrap(), Some(validity(&[false, false, true, true, true])) ); - assert_eq!(unraveler.unravel_validity(5), None); + assert_eq!(unraveler.unravel_validity(5).unwrap(), None); } #[test] @@ -3188,7 +3464,7 @@ mod tests { let mut unraveler = CompositeRepDefUnraveler::new(vec![unravel1, unravel2]); - assert!(unraveler.unravel_validity(9).is_none()); + assert!(unraveler.unravel_validity(9).unwrap().is_none()); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!( off.inner(), @@ -3484,11 +3760,11 @@ mod tests { 0, )]); - assert_eq!(unraveler.unravel_validity(0), None); + assert_eq!(unraveler.unravel_validity(0).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 0, 0, 0]).inner()); assert_eq!(val, Some(validity(&[false, false, false]))); - let val = unraveler.unravel_validity(3).unwrap(); + let val = unraveler.unravel_validity(3).unwrap().unwrap(); assert_eq!(val.inner(), validity(&[true, false, true]).inner()); } @@ -3516,7 +3792,7 @@ mod tests { 1, )]); - assert_eq!(unraveler.unravel_validity(1), None); + assert_eq!(unraveler.unravel_validity(1).unwrap(), None); let (off, val) = unraveler.unravel_offsets::().unwrap(); assert_eq!(off.inner(), offsets_32(&[0, 1, 1]).inner()); assert_eq!(val, Some(validity(&[true, false]))); @@ -3547,7 +3823,7 @@ mod tests { ]); assert_eq!( - unraveler.unravel_validity(8), + unraveler.unravel_validity(8).unwrap(), Some(validity(&[ true, false, true, false, true, true, true, true ])) @@ -3584,7 +3860,7 @@ mod tests { ]); assert_eq!( - unraveler.unravel_validity(4), + unraveler.unravel_validity(4).unwrap(), Some(validity(&[true, false, true, true])) ); assert_eq!( @@ -3616,7 +3892,7 @@ mod tests { ]); assert_eq!( - unraveler.unravel_validity(8), + unraveler.unravel_validity(8).unwrap(), Some(validity(&[ true, false, true, false, true, true, true, true ])) diff --git a/rust/lance-encoding/src/testing.rs b/rust/lance-encoding/src/testing.rs index 176083d6d64..38559a4b65e 100644 --- a/rust/lance-encoding/src/testing.rs +++ b/rust/lance-encoding/src/testing.rs @@ -21,27 +21,344 @@ use futures::{FutureExt, StreamExt, future::BoxFuture}; use log::{debug, info, trace}; use tokio::sync::mpsc::{self, UnboundedSender}; -use lance_core::{Result, utils::bit::pad_bytes}; +use lance_core::{Error, Result, datatypes::Field as LanceField, utils::bit::pad_bytes}; use lance_datagen::{ArrayGenerator, RowCount, Seed, array, gen_batch}; +use crate::compression::try_packed_struct_per_value; use crate::{ EncodingsIo, buffer::LanceBuffer, + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + reject_packed_struct_per_value, try_bitpacking_block, try_bitpacking_miniblock, + try_byte_stream_split_miniblock, try_child_rle_miniblock, + try_fixed_packed_struct_miniblock, try_fixed_u8_rle_block, try_fixed_u8_rle_miniblock, + try_general_block, try_raw_block, try_raw_fixed_size_list_miniblock, + try_raw_fixed_width_miniblock, try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_rle_block, try_variable_width_miniblock, try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, decoder::{ ColumnInfo, DecodeBatchScheduler, DecoderMessage, DecoderPlugins, FilterExpression, PageInfo, create_decode_stream, }, encoder::{ ColumnIndexSequence, EncodedColumn, EncodedPage, EncodingOptions, FieldEncoder, - MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, default_encoding_strategy, + FieldEncodingContext, FieldEncodingStrategy, MIN_PAGE_BUFFER_ALIGNMENT, OutOfLineBuffers, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, }, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, repdef::RepDefBuilder, - version::LanceFileVersion, }; const MAX_PAGE_BYTES: u64 = 32 * 1024 * 1024; const TEST_ALIGNMENT: usize = MIN_PAGE_BUFFER_ALIGNMENT as usize; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TestEncoding { + Array, + StructuralU16, + StructuralU32, + StructuralSparse, +} + +impl TestEncoding { + fn all() -> impl Iterator { + [ + Self::Array, + Self::StructuralU16, + Self::StructuralU32, + Self::StructuralSparse, + ] + .into_iter() + } + + fn is_structural(self) -> bool { + self != Self::Array + } +} + +impl std::fmt::Display for TestEncoding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Array => write!(f, "array"), + Self::StructuralU16 => write!(f, "structural-u16"), + Self::StructuralU32 => write!(f, "structural-u32"), + Self::StructuralSparse => write!(f, "structural-sparse"), + } + } +} + +#[derive(Debug, Clone)] +struct TestCompressionStrategy { + encoding: TestEncoding, + params: CompressionParams, +} + +impl TestCompressionStrategy { + fn field_params(&self, field: &LanceField) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + let mut metadata = field_metadata_params(field); + if self.encoding == TestEncoding::StructuralU16 + && metadata + .minichunk_size + .is_some_and(|size| size >= 32 * 1024) + { + metadata.minichunk_size = None; + } + params.merge(&metadata); + params + } +} + +impl CompressionStrategy for TestCompressionStrategy { + fn create_miniblock_compressor( + &self, + field: &LanceField, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = match self.encoding { + TestEncoding::StructuralSparse => try_child_rle_miniblock(data, ¶ms), + TestEncoding::Array | TestEncoding::StructuralU16 | TestEncoding::StructuralU32 => { + try_fixed_u8_rle_miniblock(data, ¶ms) + } + } { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(lance_core::Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &LanceField, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + let packed = match self.encoding { + TestEncoding::Array | TestEncoding::StructuralU16 => { + reject_packed_struct_per_value(field, data)? + } + TestEncoding::StructuralU32 | TestEncoding::StructuralSparse => { + try_packed_struct_per_value(Arc::new(self.clone()), field, data)? + } + }; + if let Some(compressor) = packed { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(lance_core::Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &LanceField, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let rle = match self.encoding { + TestEncoding::Array | TestEncoding::StructuralU16 => None, + TestEncoding::StructuralU32 => try_fixed_u8_rle_block(data, ¶ms)?, + TestEncoding::StructuralSparse => try_variable_rle_block(data, ¶ms)?, + }; + if let Some(compressor) = rle { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if matches!( + self.encoding, + TestEncoding::StructuralU32 | TestEncoding::StructuralSparse + ) && let Some(compressor) = try_general_block(data, ¶ms)? + { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(lance_core::Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} + +pub fn test_compression_strategy( + encoding: TestEncoding, + params: CompressionParams, +) -> Arc { + Arc::new(TestCompressionStrategy { encoding, params }) +} + +#[derive(Debug)] +struct TestFieldEncodingStrategy { + encoding: TestEncoding, + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for TestFieldEncodingStrategy { + fn create_field_encoder( + &self, + field: &LanceField, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if self.encoding != TestEncoding::StructuralU16 + && let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if self.encoding != TestEncoding::StructuralU16 { + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if self.encoding == TestEncoding::StructuralU16 { + if matches!( + field.data_type(), + DataType::FixedSizeList(item, _) + if matches!(item.data_type(), DataType::Struct(_)) + ) { + return Err(Error::not_supported_source( + "FixedSizeList is not enabled by the selected file format".into(), + )); + } + if matches!(field.data_type(), DataType::Map(_, _)) { + return Err(Error::not_supported_source( + "Map data type is not enabled by the selected file format".into(), + )); + } + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "{} has no field encoding for '{}' with data type {}", + self.encoding, + field.name, + field.data_type() + ) + .into(), + )) + } +} + +pub fn test_encoding_strategy(encoding: TestEncoding) -> Box { + if encoding == TestEncoding::Array { + return Box::new(crate::array_encoding::ArrayFieldEncodingStrategy::new()); + } + + let compression = test_compression_strategy(encoding, CompressionParams::default()); + let page_encodings = match encoding { + TestEncoding::StructuralU16 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::dense_u16(compression), + ], + TestEncoding::StructuralU32 => vec![ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ], + TestEncoding::StructuralSparse => vec![ + PrimitivePageEncoding::sparse(compression.clone()), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ], + TestEncoding::Array => unreachable!(), + }; + Box::new(TestFieldEncodingStrategy { + encoding, + primitive: PrimitiveFieldEncoding::new(page_encodings), + }) +} + +pub fn create_test_field_encoder( + strategy: &dyn FieldEncodingStrategy, + field: &lance_core::datatypes::Field, + column_index: &mut ColumnIndexSequence, + options: &EncodingOptions, +) -> Result> { + let context = FieldEncodingContext { + strategy, + options, + root_field_metadata: &field.metadata, + }; + strategy.create_field_encoder(field, column_index, &context) +} + #[derive(Debug)] pub(crate) struct SimulatedScheduler { data: Bytes, @@ -295,6 +612,27 @@ pub async fn check_basic_random(field: Field) { check_specific_random(field, TestCases::basic()).await; } +/// Runs one independently schedulable slice of [`check_basic_random`]. +/// +/// The complete matrix is the Cartesian product of all encodings, page sizes, +/// and slicing modes. Keeping these axes outside the helper lets expensive +/// data types preserve the full matrix without concentrating it in one test. +pub async fn check_basic_random_case( + field: Field, + encoding: TestEncoding, + page_size: u64, + use_slicing: bool, +) { + check_specific_random( + field, + TestCases::basic() + .with_encoding(encoding) + .with_page_sizes(vec![page_size]) + .with_slicing_modes([use_slicing]), + ) + .await; +} + pub async fn check_specific_random(field: Field, test_cases: TestCases) { let array_generator_provider = RandomArrayGeneratorProvider { field: field.clone(), @@ -342,24 +680,22 @@ pub async fn check_round_trip_encoding_generated( let lance_field = lance_core::datatypes::Field::try_from(&field).unwrap(); for page_size in test_cases.page_sizes.iter().copied() { debug!("Testing random data with a page size of {}", page_size); - let encoder_factory = |version: LanceFileVersion| { - let encoding_strategy = default_encoding_strategy(version); + let encoder_factory = |encoding: TestEncoding| { + let encoding_strategy = test_encoding_strategy(encoding); let mut column_index_seq = ColumnIndexSequence::default(); let encoding_options = EncodingOptions { max_page_bytes: MAX_PAGE_BYTES, cache_bytes_per_column: page_size, keep_original_array: true, buffer_alignment: MIN_PAGE_BUFFER_ALIGNMENT, - version, }; - encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap() + create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap() }; check_round_trip_random( @@ -372,9 +708,9 @@ pub async fn check_round_trip_encoding_generated( } } -fn supports_nulls(data_type: &DataType, version: LanceFileVersion) -> bool { +fn supports_nulls(data_type: &DataType, encoding: TestEncoding) -> bool { if let DataType::Struct(fields) = data_type { - if version == LanceFileVersion::V2_0 { + if encoding == TestEncoding::Array { // 2.0 doesn't support nullability for structs false } else if fields.is_empty() { @@ -389,7 +725,7 @@ fn supports_nulls(data_type: &DataType, version: LanceFileVersion) -> bool { } } -type EncodingVerificationFn = dyn Fn(&[EncodedColumn], &LanceFileVersion); +type EncodingVerificationFn = dyn Fn(&[EncodedColumn], &TestEncoding); // The default will just test the full read #[derive(Clone)] @@ -400,8 +736,9 @@ pub struct TestCases { skip_validation: bool, max_page_size: Option, page_sizes: Vec, - min_file_version: Option, - max_file_version: Option, + slicing_modes: Vec, + ingest_batch_counts: Vec, + encodings: Vec, verify_encoding: Option>, expected_encoding: Option>, } @@ -415,8 +752,9 @@ impl Default for TestCases { skip_validation: false, max_page_size: None, page_sizes: vec![4096, 1024 * 1024], - min_file_version: None, - max_file_version: None, + slicing_modes: vec![false, true], + ingest_batch_counts: vec![1, 5, 10], + encodings: TestEncoding::all().collect(), verify_encoding: None, expected_encoding: None, } @@ -459,21 +797,58 @@ impl TestCases { self } - pub fn with_min_file_version(mut self, version: LanceFileVersion) -> Self { - self.min_file_version = Some(version); + pub fn with_encoding(mut self, encoding: TestEncoding) -> Self { + self.encodings = vec![encoding]; self } - pub fn with_max_file_version(mut self, version: LanceFileVersion) -> Self { - self.max_file_version = Some(version); + pub fn with_encodings(mut self, encodings: impl IntoIterator) -> Self { + self.encodings = encodings.into_iter().collect(); self } + pub fn with_structural_encodings(self) -> Self { + self.with_encodings([ + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + TestEncoding::StructuralSparse, + ]) + } + + pub fn with_u32_structural_encodings(self) -> Self { + self.with_encodings([TestEncoding::StructuralU32, TestEncoding::StructuralSparse]) + } + + pub fn with_dense_encodings(self) -> Self { + self.with_encodings([ + TestEncoding::Array, + TestEncoding::StructuralU16, + TestEncoding::StructuralU32, + ]) + } + + pub fn with_array_and_u16_encodings(self) -> Self { + self.with_encodings([TestEncoding::Array, TestEncoding::StructuralU16]) + } + pub fn with_page_sizes(mut self, page_sizes: Vec) -> Self { self.page_sizes = page_sizes; self } + pub fn with_slicing_modes(mut self, slicing_modes: impl IntoIterator) -> Self { + self.slicing_modes = slicing_modes.into_iter().collect(); + self + } + + pub fn with_ingest_batch_counts( + mut self, + ingest_batch_counts: impl IntoIterator, + ) -> Self { + self.ingest_batch_counts = ingest_batch_counts.into_iter().collect(); + self + } + pub fn with_max_page_size(mut self, max_page_size: u64) -> Self { self.max_page_size = Some(max_page_size); self @@ -483,22 +858,8 @@ impl TestCases { self.max_page_size.unwrap_or(MAX_PAGE_BYTES) } - fn get_versions(&self) -> Vec { - LanceFileVersion::iter_non_legacy() - .filter(|v| { - if let Some(min_file_version) = &self.min_file_version - && v < min_file_version - { - return false; - } - if let Some(max_file_version) = &self.max_file_version - && v > max_file_version - { - return false; - } - true - }) - .collect() + fn encodings(&self) -> impl Iterator + '_ { + self.encodings.iter().copied() } pub fn with_verify_encoding(mut self, verify_encoding: Arc) -> Self { @@ -506,9 +867,9 @@ impl TestCases { self } - fn verify_encoding(&self, encoding: &[EncodedColumn], version: &LanceFileVersion) { + fn verify_encoding(&self, columns: &[EncodedColumn], encoding: &TestEncoding) { if let Some(verify_encoding) = self.verify_encoding.as_ref() { - verify_encoding(encoding, version); + verify_encoding(columns, encoding); } } @@ -666,6 +1027,11 @@ fn collect_page_encoding(layout: &PageLayout, actual_chain: &mut Vec) -> collect_page_encoding(inner_layout.as_ref(), actual_chain)? } } + Layout::SparseLayout(sparse) => { + if let Some(value_compression) = &sparse.value_compression { + actual_chain.extend(extract_array_encoding_chain(value_compression)); + } + } } } @@ -700,7 +1066,7 @@ fn verify_page_encoding( } } PageEncoding::Legacy(_) => { - // We don't need to care about legacy. + // We don't need to care about the v2.0 array encoding. } } @@ -743,28 +1109,26 @@ pub async fn check_round_trip_encoding_of_data_with_expected( let mut field = Field::new("", example_data.data_type().clone(), true); field = field.with_metadata(metadata); let lance_field = lance_core::datatypes::Field::try_from(&field).unwrap(); - for file_version in test_cases.get_versions() { + for encoding in test_cases.encodings() { for page_size in test_cases.page_sizes.iter() { - let encoding_strategy = default_encoding_strategy(file_version); + let encoding_strategy = test_encoding_strategy(encoding); let mut column_index_seq = ColumnIndexSequence::default(); let encoding_options = EncodingOptions { cache_bytes_per_column: *page_size, max_page_bytes: test_cases.get_max_page_size(), keep_original_array: true, buffer_alignment: MIN_PAGE_BUFFER_ALIGNMENT, - version: file_version, }; - let encoder = encoding_strategy - .create_field_encoder( - encoding_strategy.as_ref(), - &lance_field, - &mut column_index_seq, - &encoding_options, - ) - .unwrap(); + let encoder = create_test_field_encoder( + encoding_strategy.as_ref(), + &lance_field, + &mut column_index_seq, + &encoding_options, + ) + .unwrap(); info!( - "Testing round trip encoding of data with file version {} and page size {}", - file_version, page_size + "Testing round trip encoding of data with test encoding {} and page size {}", + encoding, page_size ); check_round_trip_encoding_inner( encoder, @@ -772,7 +1136,7 @@ pub async fn check_round_trip_encoding_of_data_with_expected( data.clone(), expected_override.clone(), test_cases, - file_version, + encoding, ) .await } @@ -845,7 +1209,7 @@ async fn check_round_trip_encoding_inner( data: Vec>, expected_override: Option>, test_cases: &TestCases, - file_version: LanceFileVersion, + encoding: TestEncoding, ) { let mut writer = SimulatedWriter::new(encoder.num_columns()); @@ -886,7 +1250,7 @@ async fn check_round_trip_encoding_inner( log_page(&encoded_page); // For V2.1, verify encoding in the page if expected - if file_version >= LanceFileVersion::V2_1 + if encoding.is_structural() && let Some(ref expected) = test_cases.expected_encoding { verify_page_encoding(&encoded_page, expected, encoded_page.column_idx as usize) @@ -908,7 +1272,7 @@ async fn check_round_trip_encoding_inner( log_page(&encoded_page); // For V2.1, verify encoding in the page if expected - if file_version >= LanceFileVersion::V2_1 + if encoding.is_structural() && let Some(ref expected) = test_cases.expected_encoding { verify_page_encoding(&encoded_page, expected, encoded_page.column_idx as usize) @@ -920,7 +1284,7 @@ async fn check_round_trip_encoding_inner( let mut external_buffers = writer.new_external_buffers(); let encoded_columns = encoder.finish(&mut external_buffers).await.unwrap(); - test_cases.verify_encoding(&encoded_columns, &file_version); + test_cases.verify_encoding(&encoded_columns, &encoding); for buffer in external_buffers.take_buffers() { writer.write_lance_buffer(buffer); } @@ -973,7 +1337,7 @@ async fn check_round_trip_encoding_inner( let expected_data = expected_override.clone().or_else(|| concat_data.clone()); - let is_structural_encoding = file_version >= LanceFileVersion::V2_1; + let is_structural_encoding = encoding.is_structural(); let decode_field = if is_structural_encoding { let mut lance_field = lance_core::datatypes::Field::try_from(field).unwrap(); @@ -1113,20 +1477,20 @@ const NUM_RANDOM_ROWS: u32 = 10000; /// /// To test specific test cases use the async fn check_round_trip_random( - encoder_factory: impl Fn(LanceFileVersion) -> Box, + encoder_factory: impl Fn(TestEncoding) -> Box, field: Field, array_generator_provider: Box, test_cases: &TestCases, ) { for null_rate in [None, Some(0.5), Some(1.0)] { - for use_slicing in [false, true] { - for file_version in test_cases.get_versions() { + for use_slicing in test_cases.slicing_modes.iter().copied() { + for encoding in test_cases.encodings() { if null_rate != Some(1.0) && matches!(field.data_type(), DataType::Null) { continue; } let field = if null_rate.is_some() { - if !supports_nulls(field.data_type(), file_version) { + if !supports_nulls(field.data_type(), encoding) { continue; } field.clone().with_nullable(true) @@ -1134,7 +1498,7 @@ async fn check_round_trip_random( field.clone().with_nullable(false) }; - for num_ingest_batches in [1, 5, 10] { + for num_ingest_batches in test_cases.ingest_batch_counts.iter().copied() { let rows_per_batch = NUM_RANDOM_ROWS / num_ingest_batches; let mut data = Vec::new(); @@ -1184,8 +1548,8 @@ async fn check_round_trip_random( } info!( - "Testing version {} with {} rows divided across {} batches for {} rows per batch with null_rate={:?} and use_slicing={}", - file_version, + "Testing encoding {} with {} rows divided across {} batches for {} rows per batch with null_rate={:?} and use_slicing={}", + encoding, NUM_RANDOM_ROWS, num_ingest_batches, rows_per_batch, @@ -1193,12 +1557,12 @@ async fn check_round_trip_random( use_slicing ); check_round_trip_encoding_inner( - encoder_factory(file_version), + encoder_factory(encoding), &field, data, None, test_cases, - file_version, + encoding, ) .await } diff --git a/rust/lance-encoding/src/utils/bytepack.rs b/rust/lance-encoding/src/utils/bytepack.rs index 1b2c805b51c..7c92b8f86cf 100644 --- a/rust/lance-encoding/src/utils/bytepack.rs +++ b/rust/lance-encoding/src/utils/bytepack.rs @@ -4,6 +4,8 @@ //! Utilities for byte (not bit) packing for situations where saving a few //! bits is less important than simplicity and speed. +use lance_core::{Error, Result}; + pub struct U8BytePacker { data: Vec, } @@ -15,8 +17,8 @@ impl U8BytePacker { } } - fn append(&mut self, value: u64) { - self.data.push(value as u8); + fn append(&mut self, value: u8) { + self.data.push(value); } } @@ -31,8 +33,8 @@ impl U16BytePacker { } } - fn append(&mut self, value: u64) { - self.data.extend_from_slice(&(value as u16).to_le_bytes()); + fn append(&mut self, value: u16) { + self.data.extend_from_slice(&value.to_le_bytes()); } } @@ -47,8 +49,8 @@ impl U32BytePacker { } } - fn append(&mut self, value: u64) { - self.data.extend_from_slice(&(value as u32).to_le_bytes()); + fn append(&mut self, value: u32) { + self.data.extend_from_slice(&value.to_le_bytes()); } } @@ -105,16 +107,48 @@ impl BytepackedIntegerEncoder { /// Append a value to the encoder. /// - /// # Safety + /// # Errors /// - /// This function is unsafe because it doesn't check for overflow. If the - /// value is too large to fit in the chosen integer type, it will be silently - /// truncated. - pub unsafe fn append(&mut self, value: u64) { + /// Returns an error if `value` does not fit in the width selected at + /// construction time. + pub fn append(&mut self, value: u64) -> Result<()> { + match self { + Self::U8(_) if value > u8::MAX as u64 => { + return Err(Error::invalid_input(format!( + "value {value} does not fit in bytepacked u8" + ))); + } + Self::U16(_) if value > u16::MAX as u64 => { + return Err(Error::invalid_input(format!( + "value {value} does not fit in bytepacked u16" + ))); + } + Self::U32(_) if value > u32::MAX as u64 => { + return Err(Error::invalid_input(format!( + "value {value} does not fit in bytepacked u32" + ))); + } + _ => {} + } + self.append_trusted(value); + Ok(()) + } + + /// Append a value whose range is guaranteed by the caller's construction. + pub(crate) fn append_trusted(&mut self, value: u64) { match self { - Self::U8(packer) => packer.append(value), - Self::U16(packer) => packer.append(value), - Self::U32(packer) => packer.append(value), + Self::U8(packer) => { + debug_assert!(u8::try_from(value).is_ok()); + packer.append(value as u8); + } + Self::U16(packer) => { + debug_assert!(u16::try_from(value).is_ok()); + packer.append(value as u16); + } + Self::U32(packer) => { + debug_assert!(u32::try_from(value).is_ok()); + packer.append(value as u32); + } Self::U64(packer) => packer.append(value), Self::Zero => {} } @@ -197,11 +231,9 @@ mod tests { fn test_bytepacked_integer_encoder() { // Fits in u8 let mut encoder = BytepackedIntegerEncoder::with_capacity(10, 100); - unsafe { - encoder.append(50); - encoder.append(20); - encoder.append(30); - } + encoder.append(50).unwrap(); + encoder.append(20).unwrap(); + encoder.append(30).unwrap(); let data = encoder.into_data(); assert_eq!(data, vec![50, 20, 30]); @@ -212,11 +244,9 @@ mod tests { // Requires u16 let mut encoder = BytepackedIntegerEncoder::with_capacity(10, 1000); - unsafe { - encoder.append(500); - encoder.append(200); - encoder.append(300); - } + encoder.append(500).unwrap(); + encoder.append(200).unwrap(); + encoder.append(300).unwrap(); let data = encoder.into_data(); assert_eq!(data, vec![244, 1, 200, 0, 44, 1]); @@ -227,11 +257,9 @@ mod tests { // Requires u32 let mut encoder = BytepackedIntegerEncoder::with_capacity(10, 1000000); - unsafe { - encoder.append(500000); - encoder.append(200000); - encoder.append(300000); - } + encoder.append(500000).unwrap(); + encoder.append(200000).unwrap(); + encoder.append(300000).unwrap(); let data = encoder.into_data(); assert_eq!(data, vec![32, 161, 7, 0, 64, 13, 3, 0, 224, 147, 4, 0]); @@ -242,11 +270,9 @@ mod tests { // Requires u64 let mut encoder = BytepackedIntegerEncoder::with_capacity(10, 0x10000000000); - unsafe { - encoder.append(0x5000000000); - encoder.append(0x2000000000); - encoder.append(0x3000000000); - } + encoder.append(0x5000000000).unwrap(); + encoder.append(0x2000000000).unwrap(); + encoder.append(0x3000000000).unwrap(); let data = encoder.into_data(); assert_eq!( data, @@ -260,4 +286,21 @@ mod tests { vec![0x5000000000, 0x2000000000, 0x3000000000] ); } + + #[test] + fn test_bytepacked_integer_encoder_rejects_overflow() { + for (max_value, invalid_value, expected_width) in [ + (u8::MAX as u64, u8::MAX as u64 + 1, "u8"), + (u16::MAX as u64, u16::MAX as u64 + 1, "u16"), + (u32::MAX as u64, u32::MAX as u64 + 1, "u32"), + ] { + let mut encoder = BytepackedIntegerEncoder::with_capacity(1, max_value); + let error = encoder.append(invalid_value).unwrap_err(); + assert!(error.to_string().contains(expected_width), "{error}"); + } + + let mut disabled = BytepackedIntegerEncoder::with_capacity(1, 0); + disabled.append(u64::MAX).unwrap(); + assert!(disabled.into_data().is_empty()); + } } diff --git a/rust/lance-encoding/src/version.rs b/rust/lance-encoding/src/version.rs deleted file mode 100644 index cd8f09b011b..00000000000 --- a/rust/lance-encoding/src/version.rs +++ /dev/null @@ -1,147 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -use std::str::FromStr; - -use lance_arrow::DataTypeExt; -use lance_core::datatypes::Field; -use lance_core::deepsize::{Context, DeepSizeOf}; -use lance_core::{Error, Result}; - -pub const LEGACY_FORMAT_VERSION: &str = "0.1"; -pub const V2_FORMAT_2_0: &str = "2.0"; -pub const V2_FORMAT_2_1: &str = "2.1"; -pub const V2_FORMAT_2_2: &str = "2.2"; -pub const V2_FORMAT_2_3: &str = "2.3"; - -/// Lance file version -#[derive(Debug, Default, PartialEq, Eq, Clone, Copy, Ord, PartialOrd, strum::EnumIter)] -pub enum LanceFileVersion { - // This is a little confusing but we rely on the following facts: - // - // Any version <= Next is stable - // The latest version before Stable is the default version for new datasets - // Any version >= Next is unstable - // - // As a result, 'Stable' is not the divider between stable and unstable (Next does this) - // but only serves to mark the default version for new datasets. - // - /// The legacy (0.1) format - Legacy, - V2_0, - #[default] - V2_1, - /// The latest stable release (also the default version for new datasets) - Stable, - V2_2, - /// The latest unstable release - Next, - V2_3, -} - -impl DeepSizeOf for LanceFileVersion { - fn deep_size_of_children(&self, _context: &mut Context) -> usize { - 0 - } -} - -impl LanceFileVersion { - /// Convert Stable or Next to the actual version - pub fn resolve(&self) -> Self { - match self { - Self::Stable => Self::default(), - Self::Next => Self::V2_3, - _ => *self, - } - } - - pub fn is_unstable(&self) -> bool { - self >= &Self::Next - } - - pub fn try_from_major_minor(major: u32, minor: u32) -> Result { - match (major, minor) { - (0, 0) => Ok(Self::Legacy), - (0, 1) => Ok(Self::Legacy), - (0, 2) => Ok(Self::Legacy), - (0, 3) => Ok(Self::V2_0), - (2, 0) => Ok(Self::V2_0), - (2, 1) => Ok(Self::V2_1), - (2, 2) => Ok(Self::V2_2), - (2, 3) => Ok(Self::V2_3), - _ => Err(Error::invalid_input_source( - format!("Unknown Lance storage version: {}.{}", major, minor).into(), - )), - } - } - - pub fn to_numbers(&self) -> (u32, u32) { - match self { - Self::Legacy => (0, 2), - Self::V2_0 => (2, 0), - Self::V2_1 => (2, 1), - Self::V2_2 => (2, 2), - Self::V2_3 => (2, 3), - Self::Stable => self.resolve().to_numbers(), - Self::Next => self.resolve().to_numbers(), - } - } - - pub fn iter_non_legacy() -> impl Iterator { - use strum::IntoEnumIterator; - - Self::iter().filter(|&v| v != Self::Stable && v != Self::Next && v != Self::Legacy) - } - - pub fn support_add_sub_column(&self) -> bool { - self > &Self::V2_1 - } - - pub fn support_remove_sub_column(&self, field: &Field) -> bool { - if self <= &Self::V2_1 { - field.data_type().is_struct() - } else { - field.data_type().is_nested() - } - } -} - -impl std::fmt::Display for LanceFileVersion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}", - match self { - Self::Legacy => LEGACY_FORMAT_VERSION, - Self::V2_0 => V2_FORMAT_2_0, - Self::V2_1 => V2_FORMAT_2_1, - Self::V2_2 => V2_FORMAT_2_2, - Self::V2_3 => V2_FORMAT_2_3, - Self::Stable => "stable", - Self::Next => "next", - } - ) - } -} - -impl FromStr for LanceFileVersion { - type Err = Error; - - fn from_str(value: &str) -> Result { - match value.to_lowercase().as_str() { - LEGACY_FORMAT_VERSION => Ok(Self::Legacy), - V2_FORMAT_2_0 => Ok(Self::V2_0), - V2_FORMAT_2_1 => Ok(Self::V2_1), - V2_FORMAT_2_2 => Ok(Self::V2_2), - V2_FORMAT_2_3 => Ok(Self::V2_3), - "stable" => Ok(Self::Stable), - "legacy" => Ok(Self::Legacy), - "next" => Ok(Self::Next), - // Version 0.3 is an alias of 2.0 - "0.3" => Ok(Self::V2_0), - _ => Err(Error::invalid_input_source( - format!("Unknown Lance storage version: {}", value).into(), - )), - } - } -} diff --git a/rust/lance-file/Cargo.toml b/rust/lance-file/Cargo.toml index f08cd3457aa..c4af333c9ae 100644 --- a/rust/lance-file/Cargo.toml +++ b/rust/lance-file/Cargo.toml @@ -19,6 +19,7 @@ lance-io.workspace = true arrow-arith.workspace = true arrow-array.workspace = true arrow-buffer.workspace = true +arrow-cast.workspace = true arrow-data.workspace = true arrow-schema.workspace = true arrow-select.workspace = true @@ -43,12 +44,13 @@ criterion.workspace = true rstest.workspace = true proptest.workspace = true pretty_assertions.workspace = true +rand.workspace = true test-log.workspace = true libc.workspace = true [build-dependencies] prost-build.workspace = true -protobuf-src = { version = "2.1", optional = true } +protobuf-src = { workspace = true, optional = true } [features] protoc = ["dep:protobuf-src"] @@ -61,5 +63,9 @@ features = ["protoc"] name = "reader" harness = false +[[bench]] +name = "schema" +harness = false + [lints] workspace = true diff --git a/rust/lance-file/benches/reader.rs b/rust/lance-file/benches/reader.rs index 7ee36e0b6f4..5d280f00d26 100644 --- a/rust/lance-file/benches/reader.rs +++ b/rust/lance-file/benches/reader.rs @@ -14,8 +14,9 @@ use lance_encoding::decoder::{DecoderConfig, DecoderPlugins, FilterExpression}; use lance_file::{ reader::{FileReader, FileReaderOptions}, testing::test_cache, - version::LanceFileVersion, - writer::{FileWriter, FileWriterOptions}, + version::ConcreteFileVersion, + versions as file_versions, + writer::FileWriterOptions, }; use lance_io::{ object_store::ObjectStore, @@ -28,9 +29,9 @@ use tokio::runtime::Runtime; fn bench_reader(c: &mut Criterion) { for version in [ - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, ] { let mut group = c.benchmark_group(format!("reader_{}", version)); let data = lance_datagen::gen_batch() @@ -47,13 +48,11 @@ fn bench_reader(c: &mut Criterion) { let file_path = base_path.clone().join("foo.lance"); let object_writer = rt.block_on(object_store.create(&file_path)).unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = file_versions::create_writer( + version, object_writer, data.schema().as_ref().try_into().unwrap(), - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); rt.block_on(writer.write_batch(&data)).unwrap(); @@ -175,7 +174,7 @@ fn get_cached_readers( tmpdir: &TempDir, filesystem: &str, rt: &Runtime, - version: LanceFileVersion, + version: ConcreteFileVersion, ) -> Arc { use std::sync::{LazyLock, Mutex}; @@ -213,13 +212,11 @@ fn get_cached_readers( // Write file let object_writer = rt.block_on(object_store.create(&file_path)).unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = file_versions::create_writer( + version, object_writer, data.schema().as_ref().try_into().unwrap(), - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); rt.block_on(writer.write_batch(&data)).unwrap(); @@ -366,9 +363,9 @@ fn bench_random_access(c: &mut Criterion) { let mut group = c.benchmark_group("take"); let versions = [ - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, ]; for filesystem in filesystems { @@ -379,7 +376,8 @@ fn bench_random_access(c: &mut Criterion) { for multithreaded in [false, true] { for rows_at_a_time in [1, 100] { for cached in [true, false] { - if !cached && (filesystem == "mem" || version == LanceFileVersion::V2_0) { + if !cached && (filesystem == "mem" || version == ConcreteFileVersion::V2_0) + { continue; } diff --git a/rust/lance-file/benches/schema.rs b/rust/lance-file/benches/schema.rs new file mode 100644 index 00000000000..b8da23d9777 --- /dev/null +++ b/rust/lance-file/benches/schema.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::hint::black_box; + +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use lance_core::datatypes::Schema; +use lance_file::{datatypes::Fields, format::pb}; + +fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field { + pb::Field { + id, + parent_id, + name, + logical_type: logical_type.to_owned(), + ..Default::default() + } +} + +/// Builds a pre-order flat schema with `num_physical_columns` physical leaves. +/// +/// Each struct contributes one parent and two `int32` leaves. Root fields use +/// `-1` as `parent_id`, and each struct consumes a three-ID block. +fn wide_two_leaf_structs(num_physical_columns: usize) -> Fields { + assert_eq!(num_physical_columns % 2, 0); + let num_structs = num_physical_columns / 2; + let mut fields = Vec::with_capacity(num_structs + num_physical_columns); + + for struct_index in 0..num_structs { + let parent_id = (struct_index * 3) as i32; + fields.push(proto_field( + parent_id, + -1, + format!("struct_{struct_index}"), + "struct", + )); + fields.push(proto_field( + parent_id + 1, + parent_id, + format!("left_{struct_index}"), + "int32", + )); + fields.push(proto_field( + parent_id + 2, + parent_id, + format!("right_{struct_index}"), + "int32", + )); + } + + Fields(fields) +} + +fn bench_schema_reconstruction(c: &mut Criterion) { + let mut group = c.benchmark_group("schema_from_flat_fields"); + + for num_physical_columns in [1024, 4096, 16_384, 65_536] { + let fields = wide_two_leaf_structs(num_physical_columns); + group.throughput(Throughput::Elements(fields.0.len() as u64)); + group.bench_with_input( + BenchmarkId::new("physical_columns", num_physical_columns), + &fields, + |bencher, fields| { + bencher.iter(|| Schema::try_from(black_box(fields)).unwrap()); + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, bench_schema_reconstruction); +criterion_main!(benches); diff --git a/rust/lance-file/src/compatibility_tests.rs b/rust/lance-file/src/compatibility_tests.rs new file mode 100644 index 00000000000..c63137ec6e2 --- /dev/null +++ b/rust/lance-file/src/compatibility_tests.rs @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::builder::StringDictionaryBuilder; +use arrow_array::cast::AsArray; +use arrow_array::types::{Int8Type, Int32Type}; +use arrow_array::{ + Array, ArrayRef, Int32Array, LargeBinaryArray, ListArray, RecordBatch, StringArray, +}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use bytes::Bytes; +use futures::TryStreamExt; +use lance_core::cache::LanceCache; +use lance_core::datatypes::Schema as LanceSchema; +use lance_encoding::decoder::{DecoderPlugins, EncodedBatchLayout, FilterExpression, decode_batch}; +use lance_encoding::encoder::{EncodedBatch, EncodingOptions, encode_batch}; +use lance_io::ReadBatchParams; +use lance_io::traits::Writer; +use lance_io::utils::CachedFileSize; +use rstest::rstest; +use tokio::io::AsyncWriteExt; + +use crate::reader::{EncodedBatchReaderExt, FileReader, FileReaderOptions}; +use crate::testing::FsFixture; +use crate::version::ConcreteFileVersion; +use crate::versions; +use crate::versions::v1::reader::FileReader as V1Reader; +use crate::versions::v1::writer::{ + FileWriter as V1Writer, FileWriterOptions as V1WriterOptions, NotSelfDescribing, +}; +use crate::writer::FileWriterOptions; + +fn compatibility_fixture_batch() -> RecordBatch { + let row_count = 4097; + let ids = Arc::new(Int32Array::from_iter_values(0..row_count)) as ArrayRef; + let names = Arc::new(StringArray::from_iter((0..row_count).map(|index| { + (index % 7 != 0).then(|| format!("value-{index:04}-deterministic-fixture")) + }))) as ArrayRef; + let items = Arc::new(ListArray::from_iter_primitive::( + (0..row_count).map(|index| { + (index % 11 != 0).then(|| { + vec![ + Some(index), + (index % 5 != 0).then_some(index * 2), + Some(index * 3), + ] + }) + }), + )) as ArrayRef; + let mut categories = StringDictionaryBuilder::::new(); + for index in 0..row_count { + if index % 13 == 0 { + categories.append_null(); + } else { + categories + .append(match index % 3 { + 0 => "red", + 1 => "green", + _ => "blue", + }) + .unwrap(); + } + } + let categories = Arc::new(categories.finish()) as ArrayRef; + let blobs = Arc::new(LargeBinaryArray::from_iter_values( + (0..row_count).map(|index| format!("blob-{index:04}-deterministic-payload").into_bytes()), + )) as ArrayRef; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + "lance-encoding:compression".to_string(), + "none".to_string(), + )])), + Field::new( + "items", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + true, + ), + Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + ) + .with_metadata(HashMap::from([( + "lance-encoding:dict-values-compression".to_string(), + "none".to_string(), + )])), + Field::new("blob", DataType::LargeBinary, true).with_metadata(HashMap::from([( + "lance-encoding:blob".to_string(), + "true".to_string(), + )])), + ])); + RecordBatch::try_new(schema, vec![ids, names, items, categories, blobs]).unwrap() +} + +fn v1_reader_expected_batch(batch: &RecordBatch) -> RecordBatch { + // The V1 reader historically materializes null lists as empty, null child integers as zero, + // and null dictionary keys as the first dictionary value. + let items = Arc::new(ListArray::from_iter_primitive::( + (0..batch.num_rows() as i32).map(|index| { + Some(if index % 11 == 0 { + Vec::new() + } else { + vec![ + Some(index), + Some(if index % 5 == 0 { 0 } else { index * 2 }), + Some(index * 3), + ] + }) + }), + )) as ArrayRef; + let mut categories = StringDictionaryBuilder::::new(); + for index in 0..batch.num_rows() { + categories + .append(if index % 13 == 0 { + "green" + } else { + match index % 3 { + 0 => "red", + 1 => "green", + _ => "blue", + } + }) + .unwrap(); + } + let categories = Arc::new(categories.finish()) as ArrayRef; + let mut columns = batch.columns().to_vec(); + columns[2] = items; + columns[3] = categories; + RecordBatch::try_new(batch.schema(), columns).unwrap() +} + +fn stable_fixture(version: ConcreteFileVersion) -> &'static [u8] { + match version { + ConcreteFileVersion::V1 => include_bytes!("../test_data/exact_versions/v1.lance"), + ConcreteFileVersion::V2_0 => { + include_bytes!("../test_data/exact_versions/v2_0.lance") + } + ConcreteFileVersion::V2_1 => { + include_bytes!("../test_data/exact_versions/v2_1.lance") + } + ConcreteFileVersion::V2_2 => { + include_bytes!("../test_data/exact_versions/v2_2.lance") + } + ConcreteFileVersion::V2_3 => { + unreachable!("v2.3 is unstable and has no compatibility fixture") + } + } +} + +fn assert_blob_column_eq(actual: &dyn Array, expected: &dyn Array) { + let actual = actual.as_binary::(); + let expected = expected.as_binary::(); + assert_eq!(actual.len(), expected.len()); + for index in 0..actual.len() { + assert_eq!( + actual.is_null(index), + expected.is_null(index), + "blob validity differs at row {index}" + ); + if actual.is_valid(index) { + assert_eq!( + actual.value(index), + expected.value(index), + "blob payload differs at row {index}" + ); + } + } +} + +fn assert_record_batch_eq(actual: &RecordBatch, expected: &RecordBatch) { + assert_eq!(actual.schema_ref(), expected.schema_ref()); + assert_eq!(actual.num_rows(), expected.num_rows()); + assert_eq!(actual.num_columns(), expected.num_columns()); + + for column_index in 0..actual.num_columns() { + if expected.schema().field(column_index).name() == "blob" { + assert_blob_column_eq( + actual.column(column_index).as_ref(), + expected.column(column_index).as_ref(), + ); + } else if actual.column(column_index).to_data() != expected.column(column_index).to_data() { + let row_index = (0..actual.num_rows()) + .find(|row_index| { + actual.column(column_index).slice(*row_index, 1).to_data() + != expected.column(column_index).slice(*row_index, 1).to_data() + }) + .unwrap(); + panic!( + "column {} ({}) differs at row {}: actual={:?}, expected={:?}", + column_index, + expected.schema().field(column_index).name(), + row_index, + actual.column(column_index).slice(row_index, 1), + expected.column(column_index).slice(row_index, 1) + ); + } + } +} + +fn footer_version(bytes: &[u8]) -> (u16, u16) { + let version_start = bytes.len() - 8; + ( + u16::from_le_bytes([bytes[version_start], bytes[version_start + 1]]), + u16::from_le_bytes([bytes[version_start + 2], bytes[version_start + 3]]), + ) +} + +fn assert_wire_bytes_equal(actual: &[u8], expected: &[u8]) { + if let Some(offset) = actual + .iter() + .zip(expected) + .position(|(actual, expected)| actual != expected) + { + panic!( + "wire fixture first differs at byte {offset}: actual={}, expected={}", + actual[offset], expected[offset] + ); + } + assert_eq!( + actual.len(), + expected.len(), + "wire fixture length changed after a common {}-byte prefix", + actual.len().min(expected.len()) + ); +} + +async fn write_current_fixture( + version: ConcreteFileVersion, + batch: &RecordBatch, + schema: &LanceSchema, +) -> Vec { + let fs = FsFixture::default(); + let object_writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); + let options = FileWriterOptions { + data_cache_bytes: Some(1), + max_page_bytes: Some(1024), + ..Default::default() + }; + let summary = match version { + ConcreteFileVersion::V1 => { + unreachable!("legacy fixtures use the legacy writer") + } + ConcreteFileVersion::V2_0 => { + let mut writer = + versions::v2_0::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + ConcreteFileVersion::V2_1 => { + let mut writer = + versions::v2_1::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + ConcreteFileVersion::V2_2 => { + let mut writer = + versions::v2_2::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + ConcreteFileVersion::V2_3 => { + let mut writer = + versions::v2_3::create_writer(object_writer, schema.clone(), options).unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await.unwrap(); + } + writer.finish().await.unwrap() + } + }; + fs.object_store + .open(&fs.tmp_path) + .await + .unwrap() + .get_range(0..summary.size_bytes as usize) + .await + .unwrap() + .to_vec() +} + +async fn write_v2_0_embedded_fixtures(batch: &RecordBatch, schema: &LanceSchema) -> (Bytes, Bytes) { + let options = EncodingOptions { + cache_bytes_per_column: 1, + max_page_bytes: 1024, + keep_original_array: true, + buffer_alignment: 64, + }; + let encoding_strategy = crate::versions::v2_0::encoding_strategy(); + let encoded_batch = encode_batch( + batch, + Arc::new(schema.clone()), + encoding_strategy.as_ref(), + &options, + ) + .await + .unwrap(); + + ( + versions::v2_0::encode_self_described_batch(&encoded_batch).unwrap(), + versions::v2_0::encode_mini_batch(&encoded_batch).unwrap(), + ) +} + +async fn assert_current_reader_roundtrip( + fixture: &[u8], + version: ConcreteFileVersion, + expected: &RecordBatch, +) { + let fs = FsFixture::default(); + let mut fixture_writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); + fixture_writer.write_all(fixture).await.unwrap(); + Writer::shutdown(fixture_writer.as_mut()).await.unwrap(); + let scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::new(fixture.len() as u64)) + .await + .unwrap(); + let reader = FileReader::try_open( + scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(reader.metadata().version(), version); + assert!( + reader + .metadata() + .column_metadatas + .iter() + .any(|metadata| metadata.pages.len() > 1) + ); + let batches = reader + .read_stream( + ReadBatchParams::RangeFull, + 1024, + 16, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected.num_rows() + ); + assert!( + batches + .iter() + .all(|actual| actual.schema_ref() == expected.schema_ref()) + ); + let mut row_offset = 0; + for actual in &batches { + let expected = expected.slice(row_offset, actual.num_rows()); + assert_record_batch_eq(actual, &expected); + row_offset += actual.num_rows(); + } + assert_eq!(row_offset, expected.num_rows()); +} + +#[rstest] +#[case::v2_0(ConcreteFileVersion::V2_0)] +#[case::v2_1(ConcreteFileVersion::V2_1)] +#[case::v2_2(ConcreteFileVersion::V2_2)] +#[tokio::test] +async fn stable_current_writer_and_reader_are_wire_compatible( + #[case] version: ConcreteFileVersion, +) { + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + + let actual = write_current_fixture(version, &batch, &schema).await; + let expected = stable_fixture(version); + assert_wire_bytes_equal(&actual, expected); + assert_eq!( + footer_version(expected), + version.to_standard_footer_numbers() + ); + assert_current_reader_roundtrip(expected, version, &batch).await; +} + +#[tokio::test] +async fn v2_0_embedded_writer_and_reader_are_wire_compatible() { + let batch = compatibility_fixture_batch() + .project(&[0, 1]) + .unwrap() + .slice(0, 257); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + + let (actual_self_described, actual_mini) = write_v2_0_embedded_fixtures(&batch, &schema).await; + let expected_self_described = + include_bytes!("../test_data/exact_versions/v2_0_self_described.lance"); + let expected_mini = include_bytes!("../test_data/exact_versions/v2_0_mini.lance"); + assert_wire_bytes_equal(&actual_self_described, expected_self_described); + assert_wire_bytes_equal(&actual_mini, expected_mini); + + let expected_footer = ConcreteFileVersion::V2_0.to_embedded_footer_numbers(); + assert_eq!(footer_version(expected_self_described), expected_footer); + assert_eq!(footer_version(expected_mini), expected_footer); + + let self_described = + EncodedBatch::try_from_self_described_lance(Bytes::from_static(expected_self_described)) + .unwrap(); + let decoded = decode_batch( + &self_described, + &FilterExpression::no_filter(), + Arc::::default(), + false, + EncodedBatchLayout::Array, + None, + ) + .await + .unwrap(); + assert_record_batch_eq(&decoded, &batch); + + let mini = + EncodedBatch::try_from_mini_lance(Bytes::from_static(expected_mini), &schema).unwrap(); + let decoded = decode_batch( + &mini, + &FilterExpression::no_filter(), + Arc::::default(), + false, + EncodedBatchLayout::Array, + None, + ) + .await + .unwrap(); + assert_record_batch_eq(&decoded, &batch); +} + +#[tokio::test] +async fn v2_3_output_is_deterministic_within_the_current_revision() { + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + + let first = write_current_fixture(ConcreteFileVersion::V2_3, &batch, &schema).await; + let second = write_current_fixture(ConcreteFileVersion::V2_3, &batch, &schema).await; + assert_eq!(first, second); + assert_eq!( + footer_version(&first), + ConcreteFileVersion::V2_3.to_standard_footer_numbers() + ); + assert_current_reader_roundtrip(&first, ConcreteFileVersion::V2_3, &batch).await; +} + +#[tokio::test] +async fn v1_writer_and_reader_are_wire_compatible() { + let expected = stable_fixture(ConcreteFileVersion::V1); + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref()).unwrap(); + schema.set_dictionary(&batch).unwrap(); + let fs = FsFixture::default(); + let mut writer = V1Writer::::try_new( + fs.object_store.as_ref(), + &fs.tmp_path, + schema.clone(), + &V1WriterOptions { + collect_stats_for_fields: Some(Vec::new()), + }, + ) + .await + .unwrap(); + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write(std::slice::from_ref(&slice)).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + let actual = fs + .object_store + .open(&fs.tmp_path) + .await + .unwrap() + .get_range(0..summary.size_bytes as usize) + .await + .unwrap(); + assert_wire_bytes_equal(actual.as_ref(), expected); + assert_eq!( + footer_version(expected), + ConcreteFileVersion::V1.to_standard_footer_numbers() + ); + + let fixture_fs = FsFixture::default(); + let mut fixture_writer = fixture_fs + .object_store + .create(&fixture_fs.tmp_path) + .await + .unwrap(); + fixture_writer.write_all(expected).await.unwrap(); + Writer::shutdown(fixture_writer.as_mut()).await.unwrap(); + let reader = V1Reader::try_new( + fixture_fs.object_store.as_ref(), + &fixture_fs.tmp_path, + schema.clone(), + ) + .await + .unwrap(); + let actual_batch = reader + .read_range(0..batch.num_rows(), &schema) + .await + .unwrap(); + assert_eq!(reader.num_batches(), 5); + assert_record_batch_eq(&actual_batch, &v1_reader_expected_batch(&batch)); +} diff --git a/rust/lance-file/src/concat.rs b/rust/lance-file/src/concat.rs new file mode 100644 index 00000000000..c67892bf412 --- /dev/null +++ b/rust/lance-file/src/concat.rs @@ -0,0 +1,1587 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Concatenation of complete encoded Lance files. +//! +//! This module owns compatibility checks and metadata relocation for copying +//! already-encoded pages into a new ordinary Lance file. Callers retain +//! responsibility for dataset-level grouping, transactions, and fallbacks. + +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + fmt, + future::Future, + ops::Range, + sync::Arc, +}; + +use arrow_array::{Array, ArrayRef, cast::AsArray, types::UInt8Type}; +use arrow_schema::{DataType as ArrowDataType, Field as ArrowField}; +use futures::TryStreamExt; +use lance_arrow::FieldExt; +use lance_core::{ + Error, Result, + cache::LanceCache, + datatypes::{BLOB_V2_DESC_LANCE_FIELD, BlobHandling, BlobKind, Field, Schema}, +}; +use lance_encoding::decoder::{ColumnInfo, DecoderPlugins, FilterExpression, PageInfo}; +use lance_io::{ReadBatchParams, scheduler::FileScheduler, traits::Writer as ObjectWriter}; +use prost::Message; +use prost_types::Any; + +use crate::{ + reader::{CachedFileMetadata, FileReader, RawFileMetadataOpen}, + version::ConcreteFileVersion, + versions, + writer::{FileWriteSummary, FileWriterOptions}, +}; + +/// Caller-defined runtime identity of the final target for Blob-bearing parts. +/// +/// Lance only compares this value when assembling data-file parts. It does not +/// interpret it as a dataset, base, or object-store identity, persist it, or +/// define a recovery protocol for it. The caller must provide the same identity +/// for every part and target that belong to one assembly operation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BlobTargetId(Arc); + +impl BlobTargetId { + /// Create an opaque Blob target identity. + pub fn new(identity: impl Into>) -> Self { + Self(identity.into()) + } + + /// Return the caller-defined target identity. + pub fn as_str(&self) -> &str { + self.0.as_ref() + } +} + +/// One complete immutable Lance file supplied to [`concat_files`]. +#[derive(Clone)] +pub struct EncodedFileInput { + scheduler: FileScheduler, + expected_num_rows: Option, +} + +impl EncodedFileInput { + /// Create an input from an already-open file scheduler. + pub fn new(scheduler: FileScheduler) -> Self { + Self { + scheduler, + expected_num_rows: None, + } + } + + /// Require the file metadata to report this physical row count. + /// + /// A mismatch is an input error, not a compatibility result. + pub fn with_expected_num_rows(mut self, expected_num_rows: u64) -> Self { + self.expected_num_rows = Some(expected_num_rows); + self + } + + /// The path used to read this input. + pub fn path(&self) -> &object_store::path::Path { + self.scheduler.reader().path() + } + + fn scheduler(&self) -> FileScheduler { + self.scheduler.clone() + } +} + +/// A complete ordinary Lance file validated for data-file concatenation. +/// +/// A part is independently readable and is not an incomplete file-format +/// fragment or a persisted Manifest entity. Opening one reads its real footer, +/// verifies that its physical columns form a complete rectangular file, and +/// checks Blob v2 descriptors against the caller-provided ID lease. Parts with +/// Blob v2 columns are also bound to the caller-provided [`BlobTargetId`]. +/// +/// This runtime value has no fragment, source-row, range, dataset, or storage +/// identity. Lance defines no serialization or recovery contract for it. The +/// caller owns the input storage and must keep every Blob-bearing part associated +/// with the dataset and base where its managed payloads were written. The order +/// passed to [`concat_data_file_parts`] determines final physical row order. +/// +/// # Example +/// +/// ``` +/// use lance_file::concat::{DataFilePart, EncodedFileInput}; +/// use lance_io::scheduler::FileScheduler; +/// +/// # async fn open_part(file: FileScheduler) -> lance_core::Result<()> { +/// let part = DataFilePart::open(EncodedFileInput::new(file), None, None).await?; +/// println!("part rows: {}", part.num_rows()); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone)] +pub struct DataFilePart { + input: EncodedFileInput, + metadata: Arc, + schema: Arc, + blob_ids: Option>, + blob_target_id: Option, +} + +impl fmt::Debug for DataFilePart { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DataFilePart") + .field("path", &self.input.path()) + .field("version", &self.metadata.version) + .field("num_rows", &self.metadata.num_rows) + .field("blob_ids", &self.blob_ids) + .field("blob_target_id", &self.blob_target_id) + .finish() + } +} + +impl DataFilePart { + /// Open and validate one complete encoded file. + /// + /// `blob_ids` is the half-open ID range leased to this part. Managed + /// Packed and Dedicated descriptors must fall inside it. `blob_target_id` + /// associates the part with one final target for runtime equality checks and + /// is required whenever the part contains Blob v2 columns. It does not prove + /// that the target belongs to a particular dataset or storage namespace. + /// Non-empty Inline Blob v2 descriptors and legacy Blob v1 columns are + /// rejected because their payload locations cannot be reused in a different + /// data file. + pub async fn open( + input: EncodedFileInput, + blob_ids: Option>, + blob_target_id: Option, + ) -> Result { + validate_blob_id_range(blob_ids.as_ref())?; + let metadata = Arc::new(FileReader::read_all_metadata(&input.scheduler()).await?); + let schema = Arc::new(normalize_blob_footer_schema(metadata.file_schema.as_ref())); + if let Some(expected_num_rows) = input.expected_num_rows + && metadata.num_rows != expected_num_rows + { + return Err(Error::invalid_input(format!( + "part at '{}' has {} physical rows but {} were expected", + input.path(), + metadata.num_rows, + expected_num_rows + ))); + } + let has_blob_v1 = schema + .fields_pre_order() + .any(|field| field.is_blob() && !field.is_blob_v2()); + if has_blob_v1 { + return Err(Error::not_supported(format!( + "part at '{}' contains legacy Blob v1 columns", + input.path() + ))); + } + let validation_schema = descriptor_projection_schema(schema.as_ref()); + let normalized_rows = versions::validate_external_metadata( + metadata.version, + &validation_schema, + metadata.as_ref(), + ) + .map_err(|error| { + Error::corrupt_file( + input.path().clone(), + format!("part has incomplete file metadata: {error}"), + ) + })?; + if normalized_rows != metadata.num_rows { + return Err(Error::corrupt_file( + input.path().clone(), + format!( + "part descriptor reports {} physical rows but its columns normalize to {normalized_rows}", + metadata.num_rows + ), + )); + } + + let has_blob_v2 = schema.fields_pre_order().any(|field| field.is_blob_v2()); + if has_blob_v2 { + validate_blob_descriptors( + &input, + metadata.as_ref(), + schema.as_ref(), + blob_ids.as_ref(), + ) + .await?; + if blob_target_id.is_none() { + return Err(Error::invalid_input(format!( + "part at '{}' contains Blob v2 columns but no Blob target ID was provided", + input.path() + ))); + } + } + + Ok(Self { + input, + metadata, + schema, + blob_ids, + blob_target_id, + }) + } + + /// Number of physical rows described by the part footer. + pub fn num_rows(&self) -> u64 { + self.metadata.num_rows + } +} + +fn descriptor_projection_schema(schema: &Schema) -> Schema { + let mut projected = schema.clone(); + projected.fields = projected + .fields + .into_iter() + .map(|field| BlobHandling::BlobsDescriptions.unload_if_needed(field)) + .collect(); + projected +} + +fn descriptor_child_matches(field: &Field, expected: &Field) -> bool { + field.id == -1 + && field.parent_id == -1 + && field.name == expected.name + && field.logical_type == expected.logical_type + && field.children.is_empty() +} + +fn attach_blob_descriptor_children( + fields: &mut [Field], + descriptor_children: &mut VecDeque>, +) { + for field in fields { + if field.is_blob() && field.children.is_empty() { + if let Some(children) = descriptor_children.pop_front() { + field.children = children; + } + } else { + attach_blob_descriptor_children(&mut field.children, descriptor_children); + } + } +} + +/// Blob descriptor children historically use anonymous field IDs in the file +/// descriptor. Reconstruct their tree shape before applying ordinary schema +/// projection rules. This interprets the existing footer representation and +/// does not add persisted metadata or alter the file grammar. +fn normalize_blob_footer_schema(schema: &Schema) -> Schema { + let expected = &BLOB_V2_DESC_LANCE_FIELD.children; + let missing_descriptor_count = schema + .fields_pre_order() + .filter(|field| field.is_blob() && field.children.is_empty()) + .count(); + if missing_descriptor_count == 0 { + return schema.clone(); + } + let mut normalized = schema.clone(); + let mut descriptor_children = VecDeque::new(); + let mut field_index = 0; + while descriptor_children.len() < missing_descriptor_count + && field_index + expected.len() <= normalized.fields.len() + { + if normalized.fields[field_index..field_index + expected.len()] + .iter() + .zip(expected) + .all(|(field, expected)| descriptor_child_matches(field, expected)) + { + descriptor_children.push_back( + normalized + .fields + .drain(field_index..field_index + expected.len()) + .collect(), + ); + } else { + field_index += 1; + } + } + attach_blob_descriptor_children(&mut normalized.fields, &mut descriptor_children); + normalized +} + +fn validate_blob_id_range(blob_ids: Option<&Range>) -> Result<()> { + if let Some(blob_ids) = blob_ids + && (blob_ids.start == 0 || blob_ids.start >= blob_ids.end) + { + return Err(Error::invalid_input(format!( + "part Blob ID range must be non-empty and start at 1 or greater, got {}..{}", + blob_ids.start, blob_ids.end + ))); + } + Ok(()) +} + +async fn validate_blob_descriptors( + input: &EncodedFileInput, + metadata: &CachedFileMetadata, + schema: &Schema, + blob_ids: Option<&Range>, +) -> Result<()> { + let projected_schema = descriptor_projection_schema(schema); + let blob_field_ids = projected_schema + .fields_pre_order() + .filter(|field| field.is_blob_v2()) + .map(|field| field.id) + .collect::>(); + let unique_blob_field_ids = blob_field_ids.iter().copied().collect::>(); + if unique_blob_field_ids.len() != blob_field_ids.len() + || unique_blob_field_ids + .first() + .is_some_and(|field_id| *field_id < 0) + { + return Err(Error::corrupt_file( + input.path().clone(), + "Blob v2 fields in a data-file part must have unique non-negative field IDs", + )); + } + let blob_schema = projected_schema.project_by_ids(&blob_field_ids, true); + let (field_ids, column_indices) = + versions::data_file_columns(metadata.version, &projected_schema); + let field_id_to_column_index = field_ids + .into_iter() + .zip(column_indices) + .filter_map(|(field_id, column_index)| { + (field_id >= 0 && column_index >= 0).then_some((field_id as u32, column_index as u32)) + }) + .collect::>(); + let projection = versions::reader_projection_from_field_ids( + metadata.version, + &blob_schema, + &field_id_to_column_index, + )?; + let reader = FileReader::try_open( + input.scheduler(), + Some(projection), + Arc::::default(), + &LanceCache::no_cache(), + Default::default(), + ) + .await?; + let mut batches = reader + .read_stream( + ReadBatchParams::RangeFull, + 8192, + 4, + FilterExpression::no_filter(), + ) + .await?; + while let Some(batch) = batches.try_next().await? { + let selected = vec![true; batch.num_rows()]; + for (field, array) in batch.schema().fields().iter().zip(batch.columns()) { + validate_blob_field(field.as_ref(), array, &selected, blob_ids, input.path())?; + } + } + Ok(()) +} + +fn validate_blob_field( + field: &ArrowField, + array: &ArrayRef, + selected: &[bool], + blob_ids: Option<&Range>, + path: &object_store::path::Path, +) -> Result<()> { + if field.is_blob() { + let descriptors = array.as_struct(); + let kinds = descriptors + .column_by_name("kind") + .ok_or_else(|| Error::corrupt_file(path.clone(), "Blob v2 descriptor has no kind"))? + .as_primitive::(); + let positions = descriptors + .column_by_name("position") + .ok_or_else(|| Error::corrupt_file(path.clone(), "Blob v2 descriptor has no position"))? + .as_primitive::(); + let sizes = descriptors + .column_by_name("size") + .ok_or_else(|| Error::corrupt_file(path.clone(), "Blob v2 descriptor has no size"))? + .as_primitive::(); + let ids = descriptors + .column_by_name("blob_id") + .ok_or_else(|| Error::corrupt_file(path.clone(), "Blob v2 descriptor has no blob_id"))? + .as_primitive::(); + for (row, is_selected) in selected.iter().copied().enumerate() { + if !is_selected || descriptors.is_null(row) { + continue; + } + let kind = BlobKind::try_from(kinds.value(row))?; + match kind { + BlobKind::Inline if sizes.value(row) > 0 => { + return Err(Error::invalid_input(format!( + "part at '{}' contains a non-empty Inline Blob v2 descriptor at row {row}; data-file part concatenation requires Packed or Dedicated storage", + path + ))); + } + BlobKind::Packed | BlobKind::Dedicated => { + let blob_id = ids.value(row); + let Some(blob_ids) = blob_ids else { + return Err(Error::invalid_input(format!( + "part at '{}' contains managed Blob ID {blob_id} at row {row} but no Blob ID range was provided", + path + ))); + }; + if !blob_ids.contains(&blob_id) { + return Err(Error::invalid_input(format!( + "part at '{}' contains managed Blob ID {blob_id} at row {row}, outside declared range {}..{}", + path, blob_ids.start, blob_ids.end + ))); + } + if kind == BlobKind::Dedicated && positions.value(row) != 0 { + return Err(Error::corrupt_file( + path.clone(), + format!( + "Dedicated Blob descriptor at row {row} has non-zero position {}", + positions.value(row) + ), + )); + } + } + BlobKind::Inline | BlobKind::External => {} + } + } + return Ok(()); + } + + match field.data_type() { + ArrowDataType::Struct(children) => { + let struct_array = array.as_struct(); + let child_selected = selected + .iter() + .copied() + .enumerate() + .map(|(row, is_selected)| is_selected && struct_array.is_valid(row)) + .collect::>(); + for (child, child_array) in children.iter().zip(struct_array.columns()) { + validate_blob_field(child.as_ref(), child_array, &child_selected, blob_ids, path)?; + } + } + ArrowDataType::List(child) => { + let list = array.as_list::(); + let mut child_selected = vec![false; list.values().len()]; + for (row, is_selected) in selected.iter().copied().enumerate() { + if is_selected && list.is_valid(row) { + let start = list.value_offsets()[row] as usize; + let end = list.value_offsets()[row + 1] as usize; + child_selected[start..end].fill(true); + } + } + validate_blob_field( + child.as_ref(), + list.values(), + &child_selected, + blob_ids, + path, + )?; + } + ArrowDataType::LargeList(child) => { + let list = array.as_list::(); + let mut child_selected = vec![false; list.values().len()]; + for (row, is_selected) in selected.iter().copied().enumerate() { + if is_selected && list.is_valid(row) { + let start = list.value_offsets()[row] as usize; + let end = list.value_offsets()[row + 1] as usize; + child_selected[start..end].fill(true); + } + } + validate_blob_field( + child.as_ref(), + list.values(), + &child_selected, + blob_ids, + path, + )?; + } + _ => {} + } + Ok(()) +} + +/// The exact file grammar and schema required for concatenated output. +#[derive(Debug, Clone)] +pub struct FileConcatTarget { + /// Exact output grammar. Release aliases are resolved before this boundary. + pub version: ConcreteFileVersion, + /// Complete schema stored in every input and regenerated in the output. + pub schema: Arc, + blob_target_id: Option, +} + +impl FileConcatTarget { + /// Create a concatenation target. + pub fn new(version: ConcreteFileVersion, schema: Arc) -> Self { + Self { + version, + schema, + blob_target_id: None, + } + } + + /// Bind Blob-bearing parts to one caller-defined final target. + pub fn with_blob_target_id(mut self, blob_target_id: BlobTargetId) -> Self { + self.blob_target_id = Some(blob_target_id); + self + } +} + +/// Runtime controls for encoded-file concatenation. +#[derive(Debug, Clone)] +pub struct FileConcatOptions { + /// Maximum page-buffer bytes requested in one read batch. + pub read_batch_bytes: usize, + /// Options passed to the exact-version footer writer. + pub writer_options: FileWriterOptions, +} + +impl Default for FileConcatOptions { + fn default() -> Self { + Self { + read_batch_bytes: 16 * 1024 * 1024, + writer_options: FileWriterOptions::default(), + } + } +} + +/// Metadata describing the complete file represented by a concat result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileConcatOutput { + /// Exact grammar of the completed or reused file. + pub version: ConcreteFileVersion, + /// Total physical rows in input order. + pub num_rows: u64, + /// Size of the completed or reused object. + pub size_bytes: u64, +} + +/// A compatibility reason that requires a caller-controlled decode/re-encode fallback. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileConcatReason { + /// Lance v1 does not support encoded-file concatenation. + LegacyVersion, + /// An input uses a different exact grammar than the target. + VersionMismatch { + /// Zero-based input position. + input_index: usize, + /// Version found in the file footer. + actual: ConcreteFileVersion, + /// Version requested by the target. + expected: ConcreteFileVersion, + }, + /// An input's persisted schema differs from the target schema. + SchemaMismatch { + /// Zero-based input position. + input_index: usize, + }, + /// Inputs do not describe the same physical columns. + ColumnLayoutMismatch { + /// Zero-based input position. + input_index: usize, + /// Zero-based physical column when one could be identified. + column_index: Option, + }, + /// A column-level encoding cannot safely combine its buffers. + ColumnEncodingMismatch { + /// Zero-based input position. + input_index: usize, + /// Zero-based physical column. + column_index: usize, + }, + /// A column uses file-level buffers whose page references cannot be relocated. + ColumnBuffers { + /// Zero-based input position. + input_index: usize, + /// Zero-based physical column. + column_index: usize, + /// Number of column buffers referenced by the column metadata. + count: usize, + }, + /// A file contains global buffers whose relocation semantics are not defined. + ExtraGlobalBuffers { + /// Zero-based input position. + input_index: usize, + /// Number of global buffers, including the schema descriptor. + count: usize, + }, + /// The schema contains offsets into external blob storage. + BlobColumns, +} + +impl fmt::Display for FileConcatReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::LegacyVersion => f.write_str("Lance v1 files cannot be concatenated"), + Self::VersionMismatch { + input_index, + actual, + expected, + } => write!( + f, + "input {input_index} has file version {actual}, expected {expected}" + ), + Self::SchemaMismatch { input_index } => { + write!(f, "input {input_index} has a different file schema") + } + Self::ColumnLayoutMismatch { + input_index, + column_index, + } => match column_index { + Some(column_index) => write!( + f, + "input {input_index} has a different layout for physical column {column_index}" + ), + None => write!( + f, + "input {input_index} has a different physical column count" + ), + }, + Self::ColumnEncodingMismatch { + input_index, + column_index, + } => write!( + f, + "input {input_index} has an incompatible encoding for physical column {column_index}" + ), + Self::ColumnBuffers { + input_index, + column_index, + count, + } => write!( + f, + "input {input_index} physical column {column_index} has {count} column buffers whose references cannot be relocated" + ), + Self::ExtraGlobalBuffers { input_index, count } => write!( + f, + "input {input_index} has {count} global buffers; only the schema descriptor is supported" + ), + Self::BlobColumns => { + f.write_str("schemas containing blob columns cannot be concatenated") + } + } + } +} + +/// Result of one encoded-file concatenation attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileConcatResult { + /// A new ordinary Lance file was written. + Written(FileConcatOutput), + /// One compatible complete input already is the requested output. + Reused(usize, FileConcatOutput), + /// Compatibility was rejected before the output factory was called. + Unsupported(FileConcatReason), +} + +struct PreparedInput<'a> { + input: &'a EncodedFileInput, + metadata: &'a CachedFileMetadata, + schema: &'a Schema, +} + +fn encoded_column_encoding(column: &ColumnInfo) -> Result> { + Ok(Any::from_msg(&column.encoding)?.encode_to_vec()) +} + +fn check_compatibility( + target: &FileConcatTarget, + inputs: &[PreparedInput<'_>], + allow_blob_columns: bool, +) -> Result> { + if !allow_blob_columns + && target + .schema + .fields_pre_order() + .any(|field| field.is_blob()) + { + return Ok(Some(FileConcatReason::BlobColumns)); + } + + let Some(first) = inputs.first() else { + return Err(Error::invalid_input( + "concat_files requires at least one complete input file", + )); + }; + let baseline_columns = &first.metadata.column_infos; + let expected_schema = if allow_blob_columns { + descriptor_projection_schema(target.schema.as_ref()) + } else { + target.schema.as_ref().clone() + }; + let baseline_encodings = baseline_columns + .iter() + .map(|column| encoded_column_encoding(column)) + .collect::>>()?; + + for (input_index, prepared) in inputs.iter().enumerate() { + let metadata = &prepared.metadata; + if let Some(expected_num_rows) = prepared.input.expected_num_rows + && metadata.num_rows != expected_num_rows + { + return Err(Error::invalid_input(format!( + "input {input_index} at '{}' has {} physical rows but {} were expected", + prepared.input.path(), + metadata.num_rows, + expected_num_rows + ))); + } + if metadata.version != target.version { + return Ok(Some(FileConcatReason::VersionMismatch { + input_index, + actual: metadata.version, + expected: target.version, + })); + } + if prepared.schema != &expected_schema { + return Ok(Some(FileConcatReason::SchemaMismatch { input_index })); + } + let normalized_rows = + versions::validate_external_metadata(metadata.version, prepared.schema, metadata) + .map_err(|error| { + Error::corrupt_file( + prepared.input.path().clone(), + format!("input {input_index} has incomplete file metadata: {error}"), + ) + })?; + if normalized_rows != metadata.num_rows { + return Err(Error::corrupt_file( + prepared.input.path().clone(), + format!( + "input {input_index} descriptor reports {} physical rows but its columns normalize to {normalized_rows}", + metadata.num_rows + ), + )); + } + if metadata.file_buffers.len() > 1 { + return Ok(Some(FileConcatReason::ExtraGlobalBuffers { + input_index, + count: metadata.file_buffers.len(), + })); + } + if metadata.column_infos.len() != baseline_columns.len() { + return Ok(Some(FileConcatReason::ColumnLayoutMismatch { + input_index, + column_index: None, + })); + } + for (column_index, (column, baseline)) in metadata + .column_infos + .iter() + .zip(baseline_columns) + .enumerate() + { + if !column.buffer_offsets_and_sizes.is_empty() { + return Ok(Some(FileConcatReason::ColumnBuffers { + input_index, + column_index, + count: column.buffer_offsets_and_sizes.len(), + })); + } + if column.index != baseline.index { + return Ok(Some(FileConcatReason::ColumnLayoutMismatch { + input_index, + column_index: Some(column_index), + })); + } + if encoded_column_encoding(column)? != baseline_encodings[column_index] { + return Ok(Some(FileConcatReason::ColumnEncodingMismatch { + input_index, + column_index, + })); + } + } + } + Ok(None) +} + +async fn copy_page_buffers( + writer: &mut crate::writer::FileWriter, + scheduler: &FileScheduler, + pages: &[PageInfo], + read_batch_bytes: u64, + input_index: usize, + column_index: usize, + row_offset: u64, +) -> Result> { + let mut copied = Vec::with_capacity(pages.len()); + let mut page_index = 0; + while page_index < pages.len() { + let batch_start = page_index; + let mut batch_bytes = 0u64; + let mut batch_ranges = Vec::new(); + let mut batch_buffer_counts = Vec::new(); + while page_index < pages.len() { + let page = &pages[page_index]; + let page_bytes = page.buffer_offsets_and_sizes.iter().try_fold( + 0u64, + |total, (offset, size)| { + offset.checked_add(*size).ok_or_else(|| { + Error::corrupt_file( + scheduler.reader().path().clone(), + format!( + "input {input_index} column {column_index} page {page_index} buffer range overflows" + ), + ) + })?; + total.checked_add(*size).ok_or_else(|| { + Error::corrupt_file( + scheduler.reader().path().clone(), + format!( + "input {input_index} column {column_index} page {page_index} buffer sizes overflow" + ), + ) + }) + }, + )?; + if page_index > batch_start + && batch_bytes + .checked_add(page_bytes) + .is_none_or(|total| total > read_batch_bytes) + { + break; + } + batch_bytes = batch_bytes.checked_add(page_bytes).ok_or_else(|| { + Error::corrupt_file( + scheduler.reader().path().clone(), + format!("input {input_index} column {column_index} read batch size overflows"), + ) + })?; + batch_buffer_counts.push(page.buffer_offsets_and_sizes.len()); + batch_ranges.extend( + page.buffer_offsets_and_sizes + .iter() + .filter(|(_, size)| *size > 0) + .map(|(offset, size)| *offset..(*offset + *size)), + ); + page_index += 1; + } + + let batch_data = if batch_ranges.is_empty() { + Vec::new() + } else { + scheduler.submit_request(batch_ranges, 0).await? + }; + let mut batch_data = batch_data.into_iter(); + for (relative_page_index, (page, buffer_count)) in pages[batch_start..page_index] + .iter() + .zip(batch_buffer_counts) + .enumerate() + { + let source_page_index = batch_start + relative_page_index; + let mut relocated_buffers = Vec::with_capacity(buffer_count); + for (buffer_index, (_, size)) in page.buffer_offsets_and_sizes.iter().enumerate() { + let data = if *size == 0 { + None + } else { + let data = batch_data.next().ok_or_else(|| { + Error::io(format!( + "short read for input {input_index} column {column_index} page {source_page_index} buffer {buffer_index}: expected {size} bytes" + )) + })?; + if data.len() as u64 != *size { + return Err(Error::io(format!( + "short read for input {input_index} column {column_index} page {source_page_index} buffer {buffer_index}: expected {size} bytes, got {}", + data.len() + ))); + } + Some(data) + }; + relocated_buffers.push( + writer + .write_external_buffer(data.as_deref().unwrap_or_default()) + .await?, + ); + } + copied.push(PageInfo { + num_rows: page.num_rows, + priority: page.priority.checked_add(row_offset).ok_or_else(|| { + Error::invalid_input_source( + format!( + "input {input_index} column {column_index} page {source_page_index} priority overflows after row relocation" + ) + .into(), + ) + })?, + encoding: page.encoding.clone(), + buffer_offsets_and_sizes: Arc::from(relocated_buffers), + }); + } + if batch_data.next().is_some() { + return Err(Error::io(format!( + "read for input {input_index} column {column_index} returned more buffers than requested" + ))); + } + } + Ok(copied) +} + +async fn concat_prepared( + target: &FileConcatTarget, + prepared: &[PreparedInput<'_>], + allow_blob_columns: bool, + reuse_single_input: bool, + output_factory: Factory, + options: FileConcatOptions, +) -> Result +where + Factory: FnOnce() -> FactoryFuture, + FactoryFuture: Future>>, +{ + if options.read_batch_bytes == 0 { + return Err(Error::invalid_input( + "FileConcatOptions.read_batch_bytes must be greater than zero", + )); + } + if let Some(reason) = check_compatibility(target, prepared, allow_blob_columns)? { + return Ok(FileConcatResult::Unsupported(reason)); + } + + let total_rows = prepared.iter().try_fold(0u64, |total, input| { + total.checked_add(input.metadata.num_rows).ok_or_else(|| { + Error::invalid_input_source("concat_files total physical row count overflows".into()) + }) + })?; + if prepared.len() == 1 && reuse_single_input { + return Ok(FileConcatResult::Reused( + 0, + FileConcatOutput { + version: target.version, + num_rows: total_rows, + size_bytes: prepared[0].metadata.file_size_bytes, + }, + )); + } + + let object_writer = output_factory().await?; + let mut writer = + versions::create_lazy_writer(target.version, object_writer, options.writer_options)?; + let write_result: Result = async { + let column_count = prepared[0].metadata.column_infos.len(); + let mut output_pages = std::iter::repeat_with(Vec::new) + .take(column_count) + .collect::>>(); + let mut row_offset = 0u64; + + for (input_index, prepared_input) in prepared.iter().enumerate() { + for (column_index, column) in prepared_input.metadata.column_infos.iter().enumerate() { + let has_existing_pages = !output_pages[column_index].is_empty(); + versions::copy_external_metadata_column( + target.version, + target.schema.as_ref(), + column_index, + has_existing_pages, + || async { + let pages = copy_page_buffers( + &mut writer, + &prepared_input.input.scheduler, + &column.page_infos, + options.read_batch_bytes as u64, + input_index, + column_index, + row_offset, + ) + .await?; + output_pages[column_index].extend(pages); + + Ok(()) + }, + ) + .await?; + } + row_offset = row_offset + .checked_add(prepared_input.metadata.num_rows) + .ok_or_else(|| { + Error::invalid_input_source("concat_files physical row offset overflows".into()) + })?; + } + + let mut columns = Vec::with_capacity(column_count); + for (column_index, pages) in output_pages.iter_mut().enumerate() { + versions::finalize_external_metadata_column( + target.version, + target.schema.as_ref(), + column_index, + pages, + total_rows, + )?; + let baseline = &prepared[0].metadata.column_infos[column_index]; + columns.push(Arc::new(ColumnInfo::new( + baseline.index, + Arc::from(std::mem::take(pages)), + Vec::new(), + baseline.encoding.clone(), + ))); + } + // The schema descriptor is the first global buffer and must start at + // the page-buffer alignment required by the reader. + writer.write_external_buffer(&[]).await?; + writer.initialize_with_external_columns( + target.schema.as_ref().clone(), + &columns, + total_rows, + )?; + writer.finish().await + } + .await; + + match write_result { + Ok(summary) => Ok(FileConcatResult::Written(FileConcatOutput { + version: target.version, + num_rows: summary.num_rows, + size_bytes: summary.size_bytes, + })), + Err(error) => { + writer.abort().await; + Err(error) + } + } +} + +/// Concatenate complete compatible encoded files in the supplied order. +/// +/// Metadata is read exactly once per input. The factory is invoked only after +/// all compatibility checks succeed and is never invoked for [`FileConcatResult::Reused`] +/// or [`FileConcatResult::Unsupported`]. Page payloads are copied without Arrow +/// decoding; offsets, priorities, exact-version structural metadata, and the +/// footer are regenerated. +/// +/// ``` +/// # use std::sync::Arc; +/// # use lance_core::Result; +/// # use lance_file::concat::{concat_files, EncodedFileInput, FileConcatOptions, FileConcatResult, FileConcatTarget}; +/// # use lance_io::object_store::ObjectStore; +/// # use object_store::path::Path; +/// # async fn stitch( +/// # target: &FileConcatTarget, +/// # inputs: &[EncodedFileInput], +/// # output_store: Arc, +/// # output_path: Path, +/// # ) -> Result { +/// let store = output_store.clone(); +/// concat_files( +/// target, +/// inputs, +/// move || async move { store.create(&output_path).await }, +/// FileConcatOptions::default(), +/// ) +/// .await +/// # } +/// ``` +pub async fn concat_files( + target: &FileConcatTarget, + ordered_inputs: &[EncodedFileInput], + output_factory: Factory, + options: FileConcatOptions, +) -> Result +where + Factory: FnOnce() -> FactoryFuture, + FactoryFuture: Future>>, +{ + if ordered_inputs.is_empty() { + return Err(Error::invalid_input( + "concat_files requires at least one complete input file", + )); + } + let raw_metadata = futures::future::try_join_all( + ordered_inputs + .iter() + .map(|input| FileReader::read_raw_metadata_for_dispatch(&input.scheduler)), + ) + .await?; + if target.version == ConcreteFileVersion::V1 + || raw_metadata + .iter() + .any(|metadata| matches!(metadata, RawFileMetadataOpen::Legacy { .. })) + { + return Ok(FileConcatResult::Unsupported( + FileConcatReason::LegacyVersion, + )); + } + let metadata = raw_metadata + .into_iter() + .map(|metadata| match metadata { + RawFileMetadataOpen::Current { version, metadata } => { + versions::finish_metadata(version, metadata) + } + RawFileMetadataOpen::Legacy { .. } => Err(Error::internal( + "legacy concat input reached current metadata finalization".to_string(), + )), + }) + .collect::>>()?; + let prepared = ordered_inputs + .iter() + .zip(metadata.iter()) + .map(|(input, metadata)| PreparedInput { + input, + metadata, + schema: metadata.file_schema.as_ref(), + }) + .collect::>(); + concat_prepared(target, &prepared, false, true, output_factory, options).await +} + +/// Concatenate validated data-file parts in caller-supplied row order. +/// +/// Unlike [`concat_files`], this entry point accepts Blob v2 columns because +/// every [`DataFilePart`] has already rejected file-relative Inline payloads +/// and validated managed descriptors against an explicit ID lease. Leases from +/// different parts must not overlap. No logical values or Blob payloads are +/// decoded and re-encoded during concatenation. +pub async fn concat_data_file_parts( + target: &FileConcatTarget, + ordered_parts: &[DataFilePart], + output_factory: Factory, + options: FileConcatOptions, +) -> Result +where + Factory: FnOnce() -> FactoryFuture, + FactoryFuture: Future>>, +{ + if ordered_parts.is_empty() { + return Err(Error::invalid_input( + "concat_data_file_parts requires at least one data-file part", + )); + } + for (part_index, part) in ordered_parts.iter().enumerate() { + if part.blob_target_id != target.blob_target_id { + return Err(Error::invalid_input(format!( + "part {part_index} Blob target ID {:?} does not match target ID {:?}", + part.blob_target_id.as_ref().map(BlobTargetId::as_str), + target.blob_target_id.as_ref().map(BlobTargetId::as_str) + ))); + } + } + let mut ranges = ordered_parts + .iter() + .enumerate() + .filter_map(|(part_index, part)| { + part.blob_ids + .clone() + .map(|range| (range.start, range.end, part_index)) + }) + .collect::>(); + ranges.sort_unstable_by_key(|(start, _, _)| *start); + for pair in ranges.windows(2) { + let (left_start, left_end, left_index) = pair[0]; + let (right_start, right_end, right_index) = pair[1]; + if right_start < left_end { + return Err(Error::invalid_input(format!( + "part Blob ID ranges overlap: part {left_index} uses {left_start}..{left_end}, part {right_index} uses {right_start}..{right_end}" + ))); + } + } + + let prepared = ordered_parts + .iter() + .map(|part| PreparedInput { + input: &part.input, + metadata: part.metadata.as_ref(), + schema: part.schema.as_ref(), + }) + .collect::>(); + concat_prepared(target, &prepared, true, false, output_factory, options).await +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use lance_core::utils::tempfile::TempObjFile; + use lance_io::{ + object_store::ObjectStore, + scheduler::{ScanScheduler, SchedulerConfig}, + traits::Writer, + utils::CachedFileSize, + }; + use tokio::io::AsyncWriteExt; + + use super::*; + + async fn write_file( + store: &Arc, + path: &object_store::path::Path, + version: ConcreteFileVersion, + values: &[i32], + ) -> Arc { + let batch = arrow_array::record_batch!(("value", Int32, values.to_vec())).unwrap(); + let schema = Arc::new(Schema::try_from(batch.schema_ref().as_ref()).unwrap()); + let mut writer = versions::create_writer( + version, + store.create(path).await.unwrap(), + schema.as_ref().clone(), + FileWriterOptions::default(), + ) + .unwrap(); + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + schema + } + + async fn input( + store: Arc, + path: &object_store::path::Path, + expected_num_rows: u64, + ) -> EncodedFileInput { + let scheduler = ScanScheduler::new(store, SchedulerConfig::default_for_testing()); + let file = scheduler + .open_file(path, &CachedFileSize::unknown()) + .await + .unwrap(); + EncodedFileInput::new(file).with_expected_num_rows(expected_num_rows) + } + + #[tokio::test] + async fn concat_writes_relocated_metadata_and_reuses_single_input() { + let store = Arc::new(ObjectStore::local()); + let first_path = TempObjFile::default(); + let second_path = TempObjFile::default(); + let output_path = TempObjFile::default(); + let schema = write_file(&store, &first_path, ConcreteFileVersion::V2_1, &[1, 2, 3]).await; + write_file(&store, &second_path, ConcreteFileVersion::V2_1, &[4, 5]).await; + let inputs = vec![ + input(store.clone(), &first_path, 3).await, + input(store.clone(), &second_path, 2).await, + ]; + let target = FileConcatTarget::new(ConcreteFileVersion::V2_1, schema); + let factory_calls = Arc::new(AtomicUsize::new(0)); + let result = concat_files( + &target, + &inputs, + { + let store = store.clone(); + let output_path = output_path.clone(); + let factory_calls = factory_calls.clone(); + move || async move { + factory_calls.fetch_add(1, Ordering::SeqCst); + store.create(&output_path).await + } + }, + FileConcatOptions::default(), + ) + .await + .unwrap(); + assert!(matches!( + result, + FileConcatResult::Written(FileConcatOutput { num_rows: 5, .. }) + )); + assert_eq!(factory_calls.load(Ordering::SeqCst), 1); + let output = input(store.clone(), &output_path, 5).await; + let metadata = FileReader::read_all_metadata(&output.scheduler) + .await + .unwrap(); + assert_eq!(metadata.num_rows, 5); + assert_eq!(metadata.column_infos[0].page_infos.len(), 2); + assert!( + metadata.column_infos[0].page_infos[0].priority + < metadata.column_infos[0].page_infos[1].priority + ); + + let reuse_calls = Arc::new(AtomicUsize::new(0)); + let result = concat_files( + &target, + &inputs[..1], + { + let reuse_calls = reuse_calls.clone(); + move || async move { + reuse_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::internal("reuse factory must not be called")) + } + }, + FileConcatOptions::default(), + ) + .await + .unwrap(); + assert!(matches!(result, FileConcatResult::Reused(0, _))); + assert_eq!(reuse_calls.load(Ordering::SeqCst), 0); + } + + #[rstest::rstest] + #[case(ConcreteFileVersion::V2_0)] + #[case(ConcreteFileVersion::V2_1)] + #[case(ConcreteFileVersion::V2_2)] + #[case(ConcreteFileVersion::V2_3)] + #[tokio::test] + async fn concat_preserves_schema_metadata(#[case] version: ConcreteFileVersion) { + let store = Arc::new(ObjectStore::local()); + let first_path = TempObjFile::default(); + let second_path = TempObjFile::default(); + let output_path = TempObjFile::default(); + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let mut schema = Schema::try_from(batch.schema_ref().as_ref()).unwrap(); + schema + .metadata + .insert("review-key".into(), "review-value".into()); + let schema = Arc::new(schema); + + for path in [&first_path, &second_path] { + let mut writer = versions::create_writer( + version, + store.create(path).await.unwrap(), + schema.as_ref().clone(), + FileWriterOptions::default(), + ) + .unwrap(); + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + } + let inputs = vec![ + input(store.clone(), &first_path, 2).await, + input(store.clone(), &second_path, 2).await, + ]; + let result = concat_files( + &FileConcatTarget::new(version, schema), + &inputs, + { + let store = store.clone(); + let output_path = output_path.clone(); + move || async move { store.create(&output_path).await } + }, + FileConcatOptions::default(), + ) + .await + .unwrap(); + assert!(matches!(result, FileConcatResult::Written(_))); + + let output = input(store, &output_path, 4).await; + let metadata = FileReader::read_all_metadata(&output.scheduler) + .await + .unwrap(); + assert_eq!( + metadata.file_schema.metadata.get("review-key"), + Some(&"review-value".to_string()) + ); + } + + #[tokio::test] + async fn unsupported_does_not_create_output() { + let store = Arc::new(ObjectStore::local()); + let first_path = TempObjFile::default(); + let second_path = TempObjFile::default(); + let schema = write_file(&store, &first_path, ConcreteFileVersion::V2_1, &[1]).await; + write_file(&store, &second_path, ConcreteFileVersion::V2_2, &[2]).await; + let inputs = vec![ + input(store.clone(), &first_path, 1).await, + input(store, &second_path, 1).await, + ]; + let factory_calls = Arc::new(AtomicUsize::new(0)); + let result = concat_files( + &FileConcatTarget::new(ConcreteFileVersion::V2_1, schema), + &inputs, + { + let factory_calls = factory_calls.clone(); + move || async move { + factory_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::internal("unsupported factory must not be called")) + } + }, + FileConcatOptions::default(), + ) + .await + .unwrap(); + assert!(matches!( + result, + FileConcatResult::Unsupported(FileConcatReason::VersionMismatch { input_index: 1, .. }) + )); + assert_eq!(factory_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn legacy_input_is_unsupported_without_creating_output() { + let store = Arc::new(ObjectStore::local()); + let current_path = TempObjFile::default(); + let legacy_path = TempObjFile::default(); + let schema = write_file(&store, ¤t_path, ConcreteFileVersion::V2_1, &[1]).await; + let mut legacy_writer = store.create(&legacy_path).await.unwrap(); + legacy_writer + .write_all(include_bytes!("../test_data/exact_versions/v1.lance")) + .await + .unwrap(); + Writer::shutdown(&mut legacy_writer).await.unwrap(); + let factory_calls = Arc::new(AtomicUsize::new(0)); + + let result = concat_files( + &FileConcatTarget::new(ConcreteFileVersion::V2_1, schema), + &[input(store, &legacy_path, 0).await], + { + let factory_calls = factory_calls.clone(); + move || async move { + factory_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::internal("legacy factory must not be called")) + } + }, + FileConcatOptions::default(), + ) + .await + .unwrap(); + + assert!(matches!( + result, + FileConcatResult::Unsupported(FileConcatReason::LegacyVersion) + )); + assert_eq!(factory_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn incompatible_column_buffers_and_incomplete_metadata_are_rejected() { + let store = Arc::new(ObjectStore::local()); + let path = TempObjFile::default(); + let schema = write_file(&store, &path, ConcreteFileVersion::V2_1, &[1, 2]).await; + let encoded_input = input(store, &path, 2).await; + let target = FileConcatTarget::new(ConcreteFileVersion::V2_1, schema); + + let mut with_column_buffer = FileReader::read_all_metadata(&encoded_input.scheduler) + .await + .unwrap(); + let column = with_column_buffer.column_infos[0].as_ref(); + with_column_buffer.column_infos[0] = Arc::new(ColumnInfo::new( + column.index, + column.page_infos.clone(), + vec![(0, 1)], + column.encoding.clone(), + )); + let prepared = [PreparedInput { + input: &encoded_input, + metadata: &with_column_buffer, + schema: with_column_buffer.file_schema.as_ref(), + }]; + assert!(matches!( + check_compatibility(&target, &prepared, false).unwrap(), + Some(FileConcatReason::ColumnBuffers { + input_index: 0, + column_index: 0, + count: 1 + }) + )); + + let mut missing_column = FileReader::read_all_metadata(&encoded_input.scheduler) + .await + .unwrap(); + missing_column.column_infos.clear(); + let prepared = [PreparedInput { + input: &encoded_input, + metadata: &missing_column, + schema: missing_column.file_schema.as_ref(), + }]; + let error = check_compatibility(&target, &prepared, false).unwrap_err(); + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("schema requires 1 physical columns") + ); + + let mut wrong_rows = FileReader::read_all_metadata(&encoded_input.scheduler) + .await + .unwrap(); + let column = wrong_rows.column_infos[0].as_ref(); + let mut pages = column + .page_infos + .iter() + .map(|page| PageInfo { + num_rows: page.num_rows, + priority: page.priority, + encoding: page.encoding.clone(), + buffer_offsets_and_sizes: page.buffer_offsets_and_sizes.clone(), + }) + .collect::>(); + pages[0].num_rows -= 1; + wrong_rows.column_infos[0] = Arc::new(ColumnInfo::new( + column.index, + Arc::from(pages), + Vec::new(), + column.encoding.clone(), + )); + let prepared = [PreparedInput { + input: &encoded_input, + metadata: &wrong_rows, + schema: wrong_rows.file_schema.as_ref(), + }]; + let error = check_compatibility(&target, &prepared, false).unwrap_err(); + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("descriptor reports 2 physical rows") + ); + } + + #[tokio::test] + async fn data_file_parts_reject_overlapping_blob_leases_before_output() { + let store = Arc::new(ObjectStore::local()); + let first_path = TempObjFile::default(); + let second_path = TempObjFile::default(); + let schema = write_file(&store, &first_path, ConcreteFileVersion::V2_1, &[1]).await; + write_file(&store, &second_path, ConcreteFileVersion::V2_1, &[2]).await; + let first = DataFilePart::open( + input(store.clone(), &first_path, 1).await, + Some(1..10), + None, + ) + .await + .unwrap(); + let second = DataFilePart::open(input(store, &second_path, 1).await, Some(5..20), None) + .await + .unwrap(); + let factory_calls = Arc::new(AtomicUsize::new(0)); + + let error = concat_data_file_parts( + &FileConcatTarget::new(ConcreteFileVersion::V2_1, schema), + &[first, second], + { + let factory_calls = factory_calls.clone(); + move || async move { + factory_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::internal("overlap factory must not be called")) + } + }, + FileConcatOptions::default(), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("part 0 uses 1..10"), "{error}"); + assert!(error.to_string().contains("part 1 uses 5..20"), "{error}"); + assert_eq!(factory_calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn missing_and_corrupt_inputs_are_errors_without_output() { + let store = Arc::new(ObjectStore::local()); + let valid_path = TempObjFile::default(); + let missing_path = TempObjFile::default(); + let corrupt_path = TempObjFile::default(); + let schema = write_file(&store, &valid_path, ConcreteFileVersion::V2_1, &[1, 2]).await; + write_file(&store, &missing_path, ConcreteFileVersion::V2_1, &[3, 4]).await; + let missing_input = input(store.clone(), &missing_path, 2).await; + store.delete(&missing_path).await.unwrap(); + + let target = FileConcatTarget::new(ConcreteFileVersion::V2_1, schema.clone()); + let factory_calls = Arc::new(AtomicUsize::new(0)); + let result = concat_files( + &target, + &[input(store.clone(), &valid_path, 2).await, missing_input], + { + let factory_calls = factory_calls.clone(); + move || async move { + factory_calls.fetch_add(1, Ordering::SeqCst); + Err(Error::internal("error factory must not be called")) + } + }, + FileConcatOptions::default(), + ) + .await; + assert!(result.is_err()); + assert_eq!(factory_calls.load(Ordering::SeqCst), 0); + + let mut corrupt_writer = store.create(&corrupt_path).await.unwrap(); + corrupt_writer.write_all(b"not a Lance file").await.unwrap(); + Writer::shutdown(&mut corrupt_writer).await.unwrap(); + let corrupt_input = input(store.clone(), &corrupt_path, 2).await; + let result = concat_files( + &target, + &[input(store, &valid_path, 2).await, corrupt_input], + || async { Err(Error::internal("error factory must not be called")) }, + FileConcatOptions::default(), + ) + .await; + assert!(result.is_err()); + } +} diff --git a/rust/lance-file/src/datatypes.rs b/rust/lance-file/src/datatypes.rs index ac6a8d7b293..3d84e99267c 100644 --- a/rust/lance-file/src/datatypes.rs +++ b/rust/lance-file/src/datatypes.rs @@ -1,14 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use arrow_schema::DataType; -use async_recursion::async_recursion; use lance_arrow::ARROW_EXT_NAME_KEY; -use lance_arrow::DataTypeExt; use lance_core::datatypes::{Dictionary, Encoding, Field, LogicalType, Schema}; use lance_core::{Error, Result}; -use lance_io::traits::Reader; -use lance_io::utils::{read_binary_array, read_fixed_stride_array}; use std::collections::HashMap; use crate::format::pb; @@ -99,6 +94,32 @@ impl From<&Field> for pb::Field { pub struct Fields(pub Vec); +struct FieldNode { + field: Field, + child_indices: Vec, +} + +/// Searches in pre-order depth-first order and returns the first matching node, +/// preserving the legacy parent tie-break for duplicate field IDs. +fn first_field_index_by_id( + nodes: &[FieldNode], + root_indices: &[usize], + field_id: i32, +) -> Option { + let mut to_visit = Vec::with_capacity(nodes.len()); + to_visit.extend(root_indices.iter().rev().copied()); + + while let Some(node_index) = to_visit.pop() { + let node = &nodes[node_index]; + if node.field.id == field_id { + return Some(node_index); + } + to_visit.extend(node.child_indices.iter().rev().copied()); + } + + None +} + impl From<&Field> for Fields { fn from(field: &Field) -> Self { let mut protos = vec![pb::Field::from(field)]; @@ -107,24 +128,119 @@ impl From<&Field> for Fields { } } -/// Convert list of protobuf `Field` to a Schema. -impl From<&Fields> for Schema { - fn from(fields: &Fields) -> Self { - let mut schema = Self { - fields: vec![], - metadata: HashMap::default(), - }; - - fields.0.iter().for_each(|f| { - if f.parent_id == -1 { - schema.fields.push(Field::from(f)); +/// Reconstruct a schema from a flat, pre-order protobuf field list. +/// +/// Parent fields must appear before their children. Historical manifests may +/// contain duplicate field IDs, so an ID may not identify a unique parent. For +/// those references, reconstruction preserves the legacy +/// [`Schema::mut_field_by_id`] tie-break by selecting the first matching field +/// in pre-order depth-first traversal. +/// +/// # Examples +/// +/// ``` +/// use lance_core::datatypes::Schema; +/// use lance_file::{datatypes::Fields, format::pb}; +/// +/// let field = pb::Field { +/// id: 0, +/// parent_id: -1, +/// name: "value".to_owned(), +/// logical_type: "int32".to_owned(), +/// ..Default::default() +/// }; +/// let fields = Fields(vec![field]); +/// let schema = Schema::try_from(&fields)?; +/// assert_eq!(schema.fields[0].name, "value"); +/// # Ok::<(), lance_core::Error>(()) +/// ``` +impl TryFrom<&Fields> for Schema { + type Error = Error; + + fn try_from(fields: &Fields) -> Result { + let mut nodes: Vec = Vec::with_capacity(fields.0.len()); + let mut root_indices = Vec::with_capacity(fields.0.len()); + let mut field_indices: HashMap> = HashMap::with_capacity(fields.0.len()); + + for proto_field in &fields.0 { + let parent_index = if proto_field.parent_id == -1 { + None } else { - let parent = schema.mut_field_by_id(f.parent_id).unwrap(); - parent.children.push(Field::from(f)); + let parent_index = match field_indices.get(&proto_field.parent_id) { + Some(Some(parent_index)) => *parent_index, + Some(None) => { + // Duplicate IDs are invalid but occur in historical + // manifests. Match the legacy tree traversal only for + // these ambiguous parent references so valid schemas + // retain the linear fast path. + first_field_index_by_id(&nodes, &root_indices, proto_field.parent_id) + .ok_or_else(|| { + Error::internal(format!( + "Duplicate field id {} has no existing arena node", + proto_field.parent_id + )) + })? + } + None => { + return Err(Error::schema(format!( + "Field '{}' (id={}) references parent id {}, which must appear earlier in the protobuf field list", + proto_field.name, proto_field.id, proto_field.parent_id + ))); + } + }; + Some(parent_index) + }; + + let node_index = nodes.len(); + if let Some(parent_index) = parent_index { + nodes[parent_index].child_indices.push(node_index); + } else { + root_indices.push(node_index); } - }); + nodes.push(FieldNode { + field: Field::from(proto_field), + child_indices: Vec::new(), + }); + + field_indices + .entry(proto_field.id) + .and_modify(|field_index| *field_index = None) + .or_insert(Some(node_index)); + } + + let mut fields_by_node = Vec::with_capacity(nodes.len()); + fields_by_node.resize_with(nodes.len(), || None); + for (node_index, mut node) in nodes.into_iter().enumerate().rev() { + node.field.children.reserve(node.child_indices.len()); + for child_index in node.child_indices { + let child = fields_by_node + .get_mut(child_index) + .and_then(Option::take) + .ok_or_else(|| { + Error::internal(format!( + "Schema field arena node {child_index} was not materialized before its parent" + )) + })?; + node.field.children.push(child); + } + fields_by_node[node_index] = Some(node.field); + } + + let fields = root_indices + .into_iter() + .map(|root_index| { + fields_by_node[root_index].take().ok_or_else(|| { + Error::internal(format!( + "Schema field arena root node {root_index} was not materialized" + )) + }) + }) + .collect::>>()?; - schema + Ok(Self { + fields, + metadata: HashMap::default(), + }) } } @@ -133,9 +249,28 @@ pub struct FieldsWithMeta { pub metadata: HashMap>, } -/// Convert list of protobuf `Field` and Metadata to a Schema. -impl From for Schema { - fn from(fields_with_meta: FieldsWithMeta) -> Self { +/// Reconstruct a schema from flat protobuf fields and schema metadata. +/// +/// # Examples +/// +/// ``` +/// use std::collections::HashMap; +/// +/// use lance_core::datatypes::Schema; +/// use lance_file::datatypes::{Fields, FieldsWithMeta}; +/// +/// let fields = FieldsWithMeta { +/// fields: Fields(Vec::new()), +/// metadata: HashMap::from([("owner".to_owned(), b"lance".to_vec())]), +/// }; +/// let schema = Schema::try_from(fields)?; +/// assert_eq!(schema.metadata["owner"], "lance"); +/// # Ok::<(), lance_core::Error>(()) +/// ``` +impl TryFrom for Schema { + type Error = Error; + + fn try_from(fields_with_meta: FieldsWithMeta) -> Result { let lance_metadata = fields_with_meta .metadata .into_iter() @@ -145,11 +280,11 @@ impl From for Schema { }) .collect(); - let schema_with_fields = Self::from(&fields_with_meta.fields); - Self { + let schema_with_fields = Self::try_from(&fields_with_meta.fields)?; + Ok(Self { fields: schema_with_fields.fields, metadata: lance_metadata, - } + }) } } @@ -208,76 +343,29 @@ impl From for pb::Encoding { } } -#[async_recursion] -async fn load_field_dictionary<'a>(field: &mut Field, reader: &dyn Reader) -> Result<()> { - if let DataType::Dictionary(_, value_type) = field.data_type() { - assert!(field.dictionary.is_some()); - if let Some(dict_info) = field.dictionary.as_mut() { - use DataType::*; - match value_type.as_ref() { - _ if value_type.is_binary_like() => { - dict_info.values = Some( - read_binary_array( - reader, - value_type.as_ref(), - true, // Empty values are null - dict_info.offset, - dict_info.length, - .., - ) - .await?, - ); - } - Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 => { - dict_info.values = Some( - read_fixed_stride_array( - reader, - value_type.as_ref(), - dict_info.offset, - dict_info.length, - .., - ) - .await?, - ); - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_type - ))); - } - } - } else { - panic!("Should not reach here: dictionary field does not load dictionary info") - } - Ok(()) - } else { - for child in field.children.as_mut_slice() { - load_field_dictionary(child, reader).await?; - } - Ok(()) - } -} - -/// Load dictionary value array from manifest files. -// TODO: pub(crate) -pub async fn populate_schema_dictionary(schema: &mut Schema, reader: &dyn Reader) -> Result<()> { - for field in schema.fields.as_mut_slice() { - load_field_dictionary(field, reader).await?; - } - Ok(()) -} - #[cfg(test)] mod tests { + use std::collections::HashMap; + use arrow_schema::DataType; use arrow_schema::Field as ArrowField; use arrow_schema::Fields as ArrowFields; use arrow_schema::Schema as ArrowSchema; + use lance_core::Error; use lance_core::datatypes::Schema; - use std::collections::HashMap; use super::{Fields, FieldsWithMeta}; + use crate::format::pb; + + fn proto_field(id: i32, parent_id: i32, name: String, logical_type: &str) -> pb::Field { + pb::Field { + id, + parent_id, + name, + logical_type: logical_type.to_owned(), + ..Default::default() + } + } #[test] fn test_schema_set_ids() { @@ -317,10 +405,120 @@ mod tests { let expected_schema = Schema::try_from(&arrow_schema).unwrap(); let fields_with_meta: FieldsWithMeta = (&expected_schema).into(); - let schema = Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta).unwrap(); assert_eq!(expected_schema, schema); } + #[test] + fn test_reconstruct_wide_nested_schema() { + const NUM_STRUCTS: usize = 4096; + + let mut proto_fields = Vec::with_capacity(NUM_STRUCTS * 3); + for struct_index in 0..NUM_STRUCTS { + let parent_id = (struct_index * 3) as i32; + proto_fields.push(proto_field( + parent_id, + -1, + format!("struct_{struct_index}"), + "struct", + )); + proto_fields.push(proto_field( + parent_id + 1, + parent_id, + format!("left_{struct_index}"), + "int32", + )); + proto_fields.push(proto_field( + parent_id + 2, + parent_id, + format!("right_{struct_index}"), + "int32", + )); + } + + let fields = Fields(proto_fields); + let schema = Schema::try_from(&fields).unwrap(); + assert_eq!(schema.fields.len(), NUM_STRUCTS); + for (struct_index, field) in schema.fields.iter().enumerate() { + let parent_id = (struct_index * 3) as i32; + assert_eq!(field.id, parent_id); + assert_eq!(field.name, format!("struct_{struct_index}")); + assert_eq!(field.children.len(), 2); + assert_eq!(field.children[0].id, parent_id + 1); + assert_eq!(field.children[0].name, format!("left_{struct_index}")); + assert_eq!(field.children[1].id, parent_id + 2); + assert_eq!(field.children[1].name, format!("right_{struct_index}")); + } + } + + #[test] + fn test_reconstruct_deep_nested_schema() { + const DEPTH: usize = 1024; + + let proto_fields = (0..DEPTH) + .map(|depth| { + proto_field( + depth as i32, + if depth == 0 { -1 } else { depth as i32 - 1 }, + format!("level_{depth}"), + if depth + 1 == DEPTH { + "int32" + } else { + "struct" + }, + ) + }) + .collect(); + + let fields = Fields(proto_fields); + let schema = Schema::try_from(&fields).unwrap(); + assert_eq!(schema.fields.len(), 1); + let mut field = &schema.fields[0]; + for depth in 0..DEPTH { + assert_eq!(field.id, depth as i32); + assert_eq!(field.name, format!("level_{depth}")); + if depth + 1 == DEPTH { + assert!(field.children.is_empty()); + } else { + assert_eq!(field.children.len(), 1); + field = &field.children[0]; + } + } + } + + #[test] + fn test_reconstruct_schema_reports_missing_parent() { + let fields = Fields(vec![proto_field(7, 42, "child".to_owned(), "int32")]); + + let error = Schema::try_from(&fields).unwrap_err(); + assert!(matches!(&error, Error::Schema { .. })); + assert!( + error.to_string().contains( + "Field 'child' (id=7) references parent id 42, which must appear earlier" + ) + ); + } + + #[test] + fn test_reconstruct_schema_preserves_legacy_duplicate_id_match() { + let fields = Fields(vec![ + proto_field(1, -1, "root_a".to_owned(), "struct"), + proto_field(2, -1, "root_b".to_owned(), "struct"), + proto_field(2, 1, "nested_duplicate".to_owned(), "struct"), + proto_field(3, 2, "child".to_owned(), "int32"), + ]); + + let schema = Schema::try_from(&fields).unwrap(); + assert_eq!(schema.fields.len(), 2); + assert_eq!(schema.fields[0].name, "root_a"); + assert_eq!(schema.fields[0].children.len(), 1); + assert_eq!(schema.fields[0].children[0].name, "nested_duplicate"); + assert_eq!(schema.fields[0].children[0].children.len(), 1); + assert_eq!(schema.fields[0].children[0].children[0].name, "child"); + assert_eq!(schema.fields[1].name, "root_b"); + assert!(schema.fields[1].children.is_empty()); + } + #[test] fn test_clustering_key_roundtrip() { let arrow_schema = ArrowSchema::new(vec![ @@ -351,7 +549,7 @@ mod tests { // Round-trip through protobuf let fields_with_meta: FieldsWithMeta = (&schema).into(); - let restored = Schema::from(fields_with_meta); + let restored = Schema::try_from(fields_with_meta).unwrap(); let ck2 = restored.unenforced_clustering_key(); assert_eq!(ck2.len(), 2); diff --git a/rust/lance-file/src/io.rs b/rust/lance-file/src/io.rs index 1a8edf92b08..86e5189a81d 100644 --- a/rust/lance-file/src/io.rs +++ b/rust/lance-file/src/io.rs @@ -55,14 +55,27 @@ impl EncodingsIo for LanceEncodingsIo { ) -> BoxFuture<'static, lance_core::Result>> { let mut split_ranges = Vec::new(); let mut split_indices = Vec::new(); // Track which original range each split came from + // Large ranges (above read_chunk_size) will be split into + // multiple reads. Empty ranges will skip the I/O layer + // entirely. If we have either of these we will need to + // reassemble our results, inserting empties and merging parts + let mut needs_reassembly = false; // Split large ranges into smaller chunks // // TODO: consider read_chunk_size before submitting requests. for (idx, range) in ranges.iter().enumerate() { + if range.start == range.end { + // EncodingsIo requires one result per input range. Zero-length + // ranges schedule no I/O, so their empty results are restored + // after the non-empty requests complete. + needs_reassembly = true; + continue; + } let range_size = range.end - range.start; if range_size > self.read_chunk_size { + needs_reassembly = true; let num_chunks = range_size.div_ceil(self.read_chunk_size); let chunk_size = range_size / num_chunks; @@ -87,34 +100,48 @@ impl EncodingsIo for LanceEncodingsIo { async move { let split_results = fut.await?; - // Fast path: if no splitting occurred, return results directly - if split_results.len() == ranges.len() { + if split_results.len() != split_indices.len() { + return Err(lance_core::Error::internal(format!( + "Encoding I/O returned {} results for {} requested range chunks", + split_results.len(), + split_indices.len() + ))); + } + if !needs_reassembly { return Ok(split_results); } - // Slow path: reassemble split results let mut results = vec![Vec::new(); ranges.len()]; - for (split_result, &orig_idx) in split_results.iter().zip(split_indices.iter()) { - results[orig_idx].push(split_result.clone()); + for (split_result, orig_idx) in split_results.into_iter().zip(split_indices) { + results[orig_idx].push(split_result); } - Ok(results - .into_iter() - .map(|chunks| { - if chunks.len() == 1 { - chunks.into_iter().next().unwrap() - } else { - // Concatenate multiple chunks - let total_size: usize = chunks.iter().map(|c| c.len()).sum(); - let mut combined = Vec::with_capacity(total_size); - for chunk in chunks { - combined.extend_from_slice(&chunk); - } - bytes::Bytes::from(combined) + let mut reassembled = Vec::with_capacity(ranges.len()); + for (range, chunks) in ranges.iter().zip(results) { + if chunks.is_empty() { + if range.start == range.end { + reassembled.push(bytes::Bytes::new()); + continue; } - }) - .collect()) + return Err(lance_core::Error::internal(format!( + "Encoding I/O returned no data for non-empty range {}..{}", + range.start, range.end + ))); + } + if chunks.len() == 1 { + reassembled.push(chunks[0].clone()); + continue; + } + + let total_size: usize = chunks.iter().map(|c| c.len()).sum(); + let mut combined = Vec::with_capacity(total_size); + for chunk in chunks { + combined.extend_from_slice(&chunk); + } + reassembled.push(bytes::Bytes::from(combined)); + } + Ok(reassembled) } .boxed() } diff --git a/rust/lance-file/src/lib.rs b/rust/lance-file/src/lib.rs index c89f7c7b5bf..c1e6714076e 100644 --- a/rust/lance-file/src/lib.rs +++ b/rust/lance-file/src/lib.rs @@ -1,31 +1,36 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +pub mod concat; pub mod datatypes; pub mod format; pub(crate) mod io; -pub mod previous; pub mod reader; pub mod testing; +pub mod version; +pub mod versions; pub mod writer; +#[cfg(test)] +mod compatibility_tests; + pub use io::LanceEncodingsIo; use format::MAGIC; -pub use lance_encoding::version; - use lance_core::{Error, Result}; -use lance_encoding::version::LanceFileVersion; use lance_io::object_store::ObjectStore; use object_store::path::Path; +use version::ConcreteFileVersion; pub async fn determine_file_version( store: &ObjectStore, path: &Path, known_size: Option, -) -> Result { +) -> Result { let size = match known_size { - None => store.size(path).await.unwrap() as usize, + None => usize::try_from(store.size(path).await?).map_err(|_| { + Error::invalid_input(format!("file {} is too large for this platform", path)) + })?, Some(size) => size, }; if size < 8 { @@ -51,5 +56,5 @@ pub async fn determine_file_version( let major_version = u16::from_le_bytes([footer[0], footer[1]]); let minor_version = u16::from_le_bytes([footer[2], footer[3]]); - LanceFileVersion::try_from_major_minor(major_version as u32, minor_version as u32) + ConcreteFileVersion::from_footer_numbers(major_version, minor_version) } diff --git a/rust/lance-file/src/previous/mod.rs b/rust/lance-file/src/previous/mod.rs deleted file mode 100644 index 9031d2b4992..00000000000 --- a/rust/lance-file/src/previous/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Legacy Lance file v1 implementation kept for backwards compatibility. - -pub mod format; -pub mod page_table; -pub mod reader; -pub mod writer; diff --git a/rust/lance-file/src/reader.rs b/rust/lance-file/src/reader.rs index 048cf550d66..a8e4619e4d0 100644 --- a/rust/lance-file/src/reader.rs +++ b/rust/lance-file/src/reader.rs @@ -4,6 +4,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet}, + fmt::Debug, io::Cursor, ops::Range, pin::Pin, @@ -12,6 +13,7 @@ use std::{ use arrow_array::RecordBatchReader; use arrow_schema::Schema as ArrowSchema; +use async_trait::async_trait; use byteorder::{ByteOrder, LittleEndian, ReadBytesExt}; use bytes::{Bytes, BytesMut}; use futures::{Stream, StreamExt, stream::BoxStream}; @@ -19,20 +21,18 @@ use lance_core::deepsize::{Context, DeepSizeOf}; use lance_encoding::{ EncodingsIo, decoder::{ - ColumnInfo, DecoderConfig, DecoderPlugins, FilterExpression, PageEncoding, PageInfo, - ReadBatchTask, RequestedRows, SchedulerDecoderConfig, schedule_and_decode, - schedule_and_decode_blocking, + ColumnInfo, DecoderConfig, DecoderPlugins, FilterExpression, PageEncoding, ReadBatchTask, + RequestedRows, SchedulerDecoderConfig, schedule_and_decode, schedule_and_decode_blocking, }, encoder::EncodedBatch, - version::LanceFileVersion, }; use log::debug; use object_store::path::Path; -use prost::{Message, Name}; +use prost::Message; use lance_core::{ Error, Result, - cache::{CacheKey, LanceCache}, + cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}, datatypes::{Field, Schema}, }; use lance_encoding::format::pb as pbenc; @@ -45,11 +45,14 @@ use lance_io::{ use crate::{ datatypes::{Fields, FieldsWithMeta}, - format::{MAGIC, MAJOR_VERSION, MINOR_VERSION, pb, pbfile}, + format::{MAGIC, pb, pbfile}, io::LanceEncodingsIo, - writer::PAGE_BUFFER_ALIGNMENT, + version::ConcreteFileVersion, + versions, }; +pub(crate) mod structural; + /// Default chunk size for reading large pages (8MiB) /// Pages larger than this will be split into multiple chunks during read pub const DEFAULT_READ_CHUNK_SIZE: u64 = 8 * 1024 * 1024; @@ -64,6 +67,39 @@ pub struct BufferDescriptor { pub size: u64, } +impl BufferDescriptor { + fn checked_range(&self, buffer_index: usize, file_len: u64) -> Result> { + let end = self.position.checked_add(self.size).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Global buffer {} range overflows: position={}, size={}", + buffer_index, self.position, self.size + ) + .into(), + ) + })?; + if self.position > file_len { + return Err(Error::invalid_input_source( + format!( + "Global buffer {} position {} is outside file of size {}", + buffer_index, self.position, file_len + ) + .into(), + )); + } + if end > file_len { + return Err(Error::invalid_input_source( + format!( + "Global buffer {} range {}..{} is outside file of size {}", + buffer_index, self.position, end, file_len + ) + .into(), + )); + } + Ok(self.position..end) + } +} + /// Statistics summarize some of the file metadata for quick summary info #[derive(Debug)] pub struct FileStatistics { @@ -102,8 +138,11 @@ pub struct CachedFileMetadata { pub num_global_buffer_bytes: u64, /// The number of bytes contained in the CMO and GBO tables pub num_footer_bytes: u64, + /// The major version number stored in the file footer. pub major_version: u16, + /// The minor version number stored in the file footer. pub minor_version: u16, + pub version: ConcreteFileVersion, /// The actual total file size in bytes, as reported by the object store. pub file_size_bytes: u64, /// User global buffers (index >= 1) whose bytes were already captured by the @@ -176,14 +215,14 @@ impl DeepSizeOf for CachedFileMetadata { /// hold decoded metadata for every column. #[derive(Debug, DeepSizeOf)] pub struct FileMetadataIndex { - file_schema: Arc, - num_rows: u64, - file_buffers: Vec, - column_metadata_offsets: Arc<[(u64, u64)]>, - num_columns: u32, - version: LanceFileVersion, - file_size_bytes: u64, - retained_global_buffers: BTreeMap, + pub(crate) file_schema: Arc, + pub(crate) num_rows: u64, + pub(crate) file_buffers: Vec, + pub(crate) column_metadata_offsets: Arc<[(u64, u64)]>, + pub(crate) num_columns: u32, + pub(crate) version: ConcreteFileVersion, + pub(crate) file_size_bytes: u64, + pub(crate) retained_global_buffers: BTreeMap, } impl FileMetadataIndex { @@ -226,21 +265,19 @@ impl CacheKey for ColumnMetadataCacheKey { fn type_name() -> &'static str { "ColumnMetadata" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.file.column-metadata-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.column_index); + } } impl CachedFileMetadata { - pub fn version(&self) -> LanceFileVersion { - match (self.major_version, self.minor_version) { - (0, 3) => LanceFileVersion::V2_0, - (2, 0) => LanceFileVersion::V2_0, - (2, 1) => LanceFileVersion::V2_1, - (2, 2) => LanceFileVersion::V2_2, - (2, 3) => LanceFileVersion::V2_3, - _ => panic!( - "Unsupported version: {}.{}", - self.major_version, self.minor_version - ), - } + pub fn version(&self) -> ConcreteFileVersion { + self.version } } @@ -313,123 +350,14 @@ pub struct ReaderProjection { } impl ReaderProjection { - fn from_field_ids_helper<'a>( - file_version: LanceFileVersion, - fields: impl Iterator, - field_id_to_column_index: &BTreeMap, - column_indices: &mut Vec, - ) -> Result<()> { - for field in fields { - let is_structural = file_version >= LanceFileVersion::V2_1; - let (contributes, recurse) = field_column_shape(field, is_structural); - // In the 2.0 system we needed ids for intermediate fields. In 2.1+ - // we only need ids for leaf fields. - if contributes - && let Some(column_idx) = field_id_to_column_index.get(&(field.id as u32)).copied() - { - column_indices.push(column_idx); - } - if recurse { - Self::from_field_ids_helper( - file_version, - field.children.iter(), - field_id_to_column_index, - column_indices, - )?; - } - } - Ok(()) - } - - /// Creates a projection using a mapping from field IDs to column indices - /// - /// You can obtain such a mapping when the file is written using the - /// [`crate::writer::FileWriter::field_id_to_column_indices`] method. - pub fn from_field_ids( - file_version: LanceFileVersion, - schema: &Schema, - field_id_to_column_index: &BTreeMap, - ) -> Result { - let mut column_indices = Vec::new(); - Self::from_field_ids_helper( - file_version, - schema.fields.iter(), - field_id_to_column_index, - &mut column_indices, - )?; - let projection = Self { - schema: Arc::new(schema.clone()), - column_indices, - }; - Ok(projection) - } - - /// Creates a projection that reads the entire file - /// - /// If the schema provided is not the schema of the entire file then - /// the projection will be invalid and the read will fail. - /// If the field is a `struct datatype` with `packed` set to true in the field metadata, - /// the whole struct has one column index. - /// To support nested `packed-struct encoding`, this method need to be further adjusted. - pub fn from_whole_schema(schema: &Schema, version: LanceFileVersion) -> Self { - let schema = Arc::new(schema.clone()); - let is_structural = version >= LanceFileVersion::V2_1; - let mut column_indices = vec![]; - let mut curr_column_idx = 0; - let mut packed_struct_fields_num = 0; - for field in schema.fields_pre_order() { - if packed_struct_fields_num > 0 { - packed_struct_fields_num -= 1; - continue; - } - if field.is_packed_struct() { - column_indices.push(curr_column_idx); - curr_column_idx += 1; - packed_struct_fields_num = field.children.len(); - } else if field.children.is_empty() || !is_structural { - column_indices.push(curr_column_idx); - curr_column_idx += 1; - } - } - Self { - schema, - column_indices, - } - } - - /// Creates a projection that reads the specified columns provided by name + /// Returns whether this projection is selective enough to benefit from + /// loading column metadata through the file's metadata index. /// - /// The syntax for column names is the same as [`lance_core::datatypes::Schema::project`] - /// - /// If the schema provided is not the schema of the entire file then - /// the projection will be invalid and the read will fail. - pub fn from_column_names( - file_version: LanceFileVersion, - schema: &Schema, - column_names: &[&str], - ) -> Result { - let field_id_to_column_index = schema - .fields_pre_order() - // In the 2.0 system we needed ids for intermediate fields. In 2.1+ - // we only need ids for leaf fields. - .filter(|field| { - file_version < LanceFileVersion::V2_1 || field.is_leaf() || field.is_packed_struct() - }) - .enumerate() - .map(|(idx, field)| (field.id as u32, idx as u32)) - .collect::>(); - let projected = schema.project(column_names)?; - let mut column_indices = Vec::new(); - Self::from_field_ids_helper( - file_version, - projected.fields.iter(), - &field_id_to_column_index, - &mut column_indices, - )?; - Ok(Self { - schema: Arc::new(projected), - column_indices, - }) + /// The caller must already have selected a file format that supports indexed + /// metadata. This method only evaluates the projection shape and selectivity. + pub fn prefers_indexed_metadata(&self, total_columns: usize) -> bool { + FileMetadataProvider::projection_matches_indexed_metadata(self) + && self.column_indices.len().saturating_mul(4) < total_columns } } @@ -442,7 +370,8 @@ pub struct FileReaderOptions { /// Default: 8MB (DEFAULT_READ_CHUNK_SIZE) pub read_chunk_size: u64, /// If set, the reader will produce batches whose total size in bytes - /// is approximately this value, overriding the row-based `batch_size`. + /// is approximately this value. The row-based `batch_size` remains an + /// independent upper bound, and the limit reached first determines the batch size. /// /// This can be set at the dataset level (via `ReadParams::file_reader_options`) /// to provide a default for all scans, or at the scanner level (via @@ -461,28 +390,52 @@ impl Default for FileReaderOptions { } #[derive(Debug, Clone)] -struct PreparedProjection { - column_infos: Vec>, - decoder_projection: ReaderProjection, +pub(crate) struct PreparedProjection { + pub column_infos: Vec>, + pub decoder_projection: ReaderProjection, } #[derive(Debug, Clone)] -enum FileMetadataProvider { +pub(crate) enum FileMetadataProvider { Full(Arc), Indexed(Arc), } +/// Executable projection behavior selected by an exact file-version module. +/// +/// The shared reader invokes this behavior but never interprets a version or +/// accepted-grammar profile. +#[async_trait] +pub(crate) trait ReadProjection: Debug + Send + Sync { + fn validate_indexed( + &self, + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Result<()>; + + fn read_length(&self, prepared: &PreparedProjection) -> Result; + + async fn prepare( + &self, + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + io: &Arc, + cache: &Arc, + ) -> Result<(PreparedProjection, u64)>; +} + #[derive(Debug, Clone)] -struct FileReadCore { - scheduler: Arc, - base_projection: ReaderProjection, - metadata_provider: FileMetadataProvider, - decoder_plugins: Arc, - cache: Arc, - options: FileReaderOptions, +pub(crate) struct DecodeEngine { + pub scheduler: Arc, + pub base_projection: ReaderProjection, + pub metadata_provider: FileMetadataProvider, + pub read_projection: Arc, + pub decoder_plugins: Arc, + pub cache: Arc, + pub options: FileReaderOptions, } -/// A projection-scoped reader for Lance files. +/// A projection-scoped reader for a current-format Lance file. /// /// This reader fixes a base projection at construction time. All later reads /// must stay within that projection, which lets the reader load only the column @@ -491,125 +444,90 @@ struct FileReadCore { /// file metadata. #[derive(Debug, Clone)] pub struct ProjectedFileReader { - core: FileReadCore, + core: DecodeEngine, } -/// A Lance file reader backed by fully decoded file metadata. +/// A current-format Lance file reader backed by fully decoded metadata. #[derive(Debug, Clone)] pub struct FileReader { - core: FileReadCore, - metadata: Arc, + pub(crate) core: DecodeEngine, + pub(crate) metadata: Arc, +} + +pub(crate) fn tasks_to_record_batch_stream( + schema: Arc, + tasks: Pin + Send>>, + batch_readahead: u32, +) -> Pin> { + let arrow_schema = Arc::new(ArrowSchema::from(schema.as_ref())); + let batches = tasks + .map(|task| task.task) + .buffered(batch_readahead as usize) + .boxed(); + Box::pin(RecordBatchStreamAdapter::new(arrow_schema, batches)) +} + +pub(crate) enum RawFileMetadataOpen { + Legacy { + major_version: u16, + minor_version: u16, + }, + Current { + version: ConcreteFileVersion, + metadata: RawFileMetadata, + }, +} + +pub(crate) struct RawFileMetadata { + pub file_schema: Arc, + pub column_metadatas: Vec, + pub num_rows: u64, + pub file_buffers: Vec, + pub num_data_bytes: u64, + pub num_column_metadata_bytes: u64, + pub num_global_buffer_bytes: u64, + pub num_footer_bytes: u64, + pub footer: Footer, + pub file_size_bytes: u64, + pub retained_global_buffers: BTreeMap, } + #[derive(Debug)] -struct Footer { +pub(crate) struct Footer { #[allow(dead_code)] - column_meta_start: u64, + pub column_meta_start: u64, // We don't use this today because we always load metadata for every column // and don't yet support "metadata projection" #[allow(dead_code)] - column_meta_offsets_start: u64, - global_buff_offsets_start: u64, - num_global_buffers: u32, - num_columns: u32, - major_version: u16, - minor_version: u16, + pub column_meta_offsets_start: u64, + pub global_buff_offsets_start: u64, + pub num_global_buffers: u32, + pub num_columns: u32, + pub major_version: u16, + pub minor_version: u16, } const FOOTER_LEN: usize = 40; -// How a field maps onto physical columns, shared by the projection-building and -// projection-validation walks so they stay in lockstep. In the 2.0 layout every -// ordinary field (including structs and lists) has its own column; in 2.1 only -// leaves do. Blob/packed-struct fields are opaque in all versions: they are a -// single column with no descent, including unloaded blob descriptor schemas. -// Returns `(contributes, recurse)`: whether the field has its own column and -// whether to walk into its children. The DFS order is the field's own column (if -// any) followed by its children, so a field's root (first) column is always the -// first entry of its sub-slice. -fn field_column_shape(field: &Field, is_structural: bool) -> (bool, bool) { +// Count the V2.1 physical columns required to reconstruct a projected field. +// This is the same DFS shape consumed by `ColumnInfoIter`: ordinary structural +// nodes are transparent and leaves contribute columns. Indexed metadata loading +// can therefore compact any ordinary structural projection into 0..N while +// preserving this order. +// +// Blob and packed-struct fields remain unsupported by indexed projection. Their +// opaque decode semantics are handled by the existing full-metadata reader. +fn indexed_projection_column_count(field: &Field) -> Option { if field.is_blob() || field.is_packed_struct() { - return (true, false); + return None; } - let contributes = !is_structural || field.children.is_empty(); - let recurse = !field.children.is_empty(); - (contributes, recurse) -} - -// Whether a field's children each cover the same rows as the field itself. Struct -// children do (one value per parent row), so they must share its length. List, -// map, and fixed-size-list items have an independent cardinality (item count, not -// row count) and are validated only against themselves. -fn children_share_parent_length(field: &Field) -> bool { - field.logical_type.is_struct() -} - -// Validate one field's slice of a projection's flat `column_indices`, returning -// the field's top-level row count (the page-row sum of its root column). Walks the -// same DFS order as `from_field_ids_helper`, advancing `cursor` past every column -// the field contributes. -// -// `comparable` tracks whether the field's row count shares the read's top-level -// cardinality. A struct's children must all match that count -- the decoders -// combine them assuming equal lengths and would otherwise panic or read past a -// shorter child -- so the equality check runs only while `comparable` holds. Once -// the walk descends through a list/map/fixed-size-list its items have an -// independent cardinality (item count, not row count), so `comparable` turns off -// for that whole subtree and a nested struct's children are no longer compared. -fn validate_field_length Result>( - field: &Field, - is_structural: bool, - comparable: bool, - column_indices: &[u32], - cursor: &mut usize, - column_len: &F, -) -> Result { - let (contributes, recurse) = field_column_shape(field, is_structural); - let mut field_rows: Option = None; - if contributes { - let column = *column_indices.get(*cursor).ok_or_else(|| { - Error::invalid_input(format!( - "projection supplied fewer column indices than its fields require \ - (ran out at field '{}')", - field.name - )) - })?; - *cursor += 1; - field_rows = Some(column_len(column as usize)?); - } - if recurse { - // Only enforce equal-length children for a struct whose own count is still - // at the top-level cardinality; below a list/map/fixed-size-list the items - // have an independent cardinality, so neither this field nor its - // descendants are comparable. - let enforce_children = comparable && children_share_parent_length(field); - for child in &field.children { - let child_rows = validate_field_length( - child, - is_structural, - enforce_children, - column_indices, - cursor, - column_len, - )?; - // A struct that contributes no column of its own (the 2.1 layout) - // takes its row count from its first child. - let expected = *field_rows.get_or_insert(child_rows); - if enforce_children && child_rows != expected { - return Err(Error::invalid_input(format!( - "cannot read field '{}': its children have differing lengths \ - (child '{}' has {} rows, but the field has {}); a struct's \ - children must all have the same length", - field.name, child.name, child_rows, expected - ))); - } - } + if field.children.is_empty() { + return Some(1); } - field_rows.ok_or_else(|| { - Error::invalid_input(format!( - "projected field '{}' maps to no columns", - field.name - )) + + field.children.iter().try_fold(0usize, |count, child| { + count.checked_add(indexed_projection_column_count(child)?) }) } @@ -618,7 +536,31 @@ fn validate_field_length Result>( // error (naming each column's length) when they differ. Ordinary files always // pass; only files written with `FileWriter::write_column` whose columns ended up // unequal can fail, and those must be read separately. -fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result { +pub(crate) fn normalized_column_num_rows(info: &ColumnInfo) -> Result { + info.page_infos.iter().try_fold(0_u64, |rows, page| { + let page_rows = match &page.encoding { + PageEncoding::Structural(layout) => match &layout.layout { + Some(pbenc21::page_layout::Layout::SparseLayout(sparse)) => sparse + .structural_layers + .first() + .and_then(|layer| layer.layer.as_ref()) + .map_or(page.num_rows, |layer| match layer { + pbenc21::sparse_structural_layer::Layer::Validity(layer) => layer.num_slots, + pbenc21::sparse_structural_layer::Layer::List(layer) => layer.num_slots, + pbenc21::sparse_structural_layer::Layer::FixedSizeList(layer) => { + layer.num_slots + } + }), + _ => page.num_rows, + }, + _ => page.num_rows, + }; + rows.checked_add(page_rows) + .ok_or_else(|| Error::invalid_input_source("Column row count overflows u64".into())) + }) +} + +pub(crate) fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result { let first = field_lengths.first().map_or(0, |&(_, len)| len); if field_lengths.iter().all(|&(_, len)| len == first) { return Ok(first); @@ -635,6 +577,30 @@ fn verify_uniform_lengths(field_lengths: &[(&str, u64)]) -> Result { } impl FileReader { + pub(crate) fn base_projection(&self) -> &ReaderProjection { + &self.core.base_projection + } + + pub(crate) fn full_projection(&self, projection: ReaderProjection) -> PreparedProjection { + PreparedProjection { + column_infos: self.metadata.column_infos.clone(), + decoder_projection: projection, + } + } + + pub(crate) async fn read_prepared_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + prepared: PreparedProjection, + read_len: u64, + filter: FilterExpression, + ) -> Result + Send>>> { + self.core + .read_prepared_tasks(params, batch_size, prepared, read_len, filter) + .await + } + pub fn with_scheduler(&self, scheduler: Arc) -> Self { Self { core: self.core.with_scheduler(scheduler), @@ -753,25 +719,22 @@ impl FileReader { gbo_table: &[BufferDescriptor], tail_bytes: &Bytes, tail_offset: u64, - ) -> BTreeMap { - let tail_end = tail_offset + tail_bytes.len() as u64; - gbo_table - .iter() - .enumerate() - .skip(1) - .filter_map(|(index, buffer)| { - let start = buffer.position; - let end = buffer.position + buffer.size; - if start >= tail_offset && end <= tail_end { - let rel_start = (start - tail_offset) as usize; - let rel_end = (end - tail_offset) as usize; - let bytes = Bytes::copy_from_slice(&tail_bytes[rel_start..rel_end]); - Some((index as u32, bytes)) - } else { - None - } - }) - .collect() + file_len: u64, + ) -> Result> { + let tail_end = tail_offset + .checked_add(tail_bytes.len() as u64) + .ok_or_else(|| Error::invalid_input_source("Tail byte range overflows".into()))?; + let mut retained_buffers = BTreeMap::new(); + for (index, buffer) in gbo_table.iter().enumerate().skip(1) { + let range = buffer.checked_range(index, file_len)?; + if range.start >= tail_offset && range.end <= tail_end { + let rel_start = (range.start - tail_offset) as usize; + let rel_end = (range.end - tail_offset) as usize; + let bytes = Bytes::copy_from_slice(&tail_bytes[rel_start..rel_end]); + retained_buffers.insert(index as u32, bytes); + } + } + Ok(retained_buffers) } // Checks to make sure the footer is written correctly and returns the @@ -794,14 +757,6 @@ impl FileReader { let major_version = cursor.read_u16::()?; let minor_version = cursor.read_u16::()?; - if major_version == MAJOR_VERSION as u16 && minor_version == MINOR_VERSION as u16 { - return Err(Error::version_conflict( - "Attempt to use the lance v2 reader to read a legacy file".to_string(), - major_version, - minor_version, - )); - } - let magic_bytes = footer_bytes.slice(len - 4..); if magic_bytes.as_ref() != MAGIC { return Err(Error::invalid_input(format!( @@ -820,6 +775,22 @@ impl FileReader { }) } + fn current_file_version(footer: &Footer) -> Result { + let version = + ConcreteFileVersion::from_footer_numbers(footer.major_version, footer.minor_version)?; + match version { + ConcreteFileVersion::V1 => Err(Error::version_conflict( + "Attempt to use the lance v2 reader to read a legacy file".to_string(), + footer.major_version, + footer.minor_version, + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(version), + } + } + // TODO: Once we have coalesced I/O we should only read the column metadatas that we need fn read_all_column_metadata( column_metadata_bytes: Bytes, @@ -895,7 +866,15 @@ impl FileReader { scheduler: &FileScheduler, file_len: u64, ) -> Result { - let num_bytes_needed = (file_len - start_pos) as usize; + let num_bytes_needed = file_len.checked_sub(start_pos).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Tail read position {} is outside file of size {}", + start_pos, file_len + ) + .into(), + ) + })? as usize; if data.len() >= num_bytes_needed { Ok(data.slice((data.len() - num_bytes_needed)..)) } else { @@ -911,19 +890,12 @@ impl FileReader { } } - fn do_decode_gbo_table( - gbo_bytes: &Bytes, - footer: &Footer, - version: LanceFileVersion, - ) -> Result> { + fn do_decode_gbo_table(gbo_bytes: &Bytes, footer: &Footer) -> Result> { let mut global_bufs_cursor = Cursor::new(gbo_bytes); let mut global_buffers = Vec::with_capacity(footer.num_global_buffers as usize); for _ in 0..footer.num_global_buffers { let buf_pos = global_bufs_cursor.read_u64::()?; - assert!( - version < LanceFileVersion::V2_1 || buf_pos % PAGE_BUFFER_ALIGNMENT as u64 == 0 - ); let buf_size = global_bufs_cursor.read_u64::()?; global_buffers.push(BufferDescriptor { position: buf_pos, @@ -934,12 +906,24 @@ impl FileReader { Ok(global_buffers) } + fn validate_gbo_table( + gbo_table: &[BufferDescriptor], + file_len: u64, + version: ConcreteFileVersion, + ) -> Result<()> { + versions::validate_global_buffers(version, gbo_table)?; + for (buffer_index, buffer) in gbo_table.iter().enumerate() { + buffer.checked_range(buffer_index, file_len)?; + } + Ok(()) + } + async fn decode_gbo_table( tail_bytes: &Bytes, file_len: u64, scheduler: &FileScheduler, footer: &Footer, - version: LanceFileVersion, + version: ConcreteFileVersion, ) -> Result> { // This could, in theory, trigger another IOP but the GBO table should never be large // enough for that to happen @@ -950,7 +934,9 @@ impl FileReader { file_len, ) .await?; - Self::do_decode_gbo_table(&gbo_bytes, footer, version) + let gbo_table = Self::do_decode_gbo_table(&gbo_bytes, footer)?; + Self::validate_gbo_table(&gbo_table, file_len, version)?; + Ok(gbo_table) } fn decode_schema(schema_bytes: Bytes) -> Result<(u64, lance_core::datatypes::Schema)> { @@ -961,36 +947,27 @@ impl FileReader { fields: Fields(pb_schema.fields), metadata: pb_schema.metadata, }; - let schema = lance_core::datatypes::Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta)?; Ok((num_rows, schema)) } - // TODO: Support late projection. Currently, if we want to perform a - // projected read of a file, we load all of the column metadata, and then - // only read the column data that is requested. This is fine for most cases. - // - // However, if there are many columns then loading all of the column metadata - // may be expensive. We should support a mode where we only load the column - // metadata for the columns that are requested (the file format supports this). - // - // The main challenge is that we either need to ignore the column metadata cache - // or have a more sophisticated cache that can cache per-column metadata. - // - // Also, if the number of columns is fairly small, it's faster to read them as a - // single IOP, but we can fix this through coalescing. - pub async fn read_all_metadata(scheduler: &FileScheduler) -> Result { - // 1. read the footer + pub(crate) async fn read_raw_metadata_for_dispatch( + scheduler: &FileScheduler, + ) -> Result { let (tail_bytes, file_len) = Self::read_tail(scheduler).await?; let tail_offset = file_len - tail_bytes.len() as u64; let footer = Self::decode_footer(&tail_bytes)?; - - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; + let version = + ConcreteFileVersion::from_footer_numbers(footer.major_version, footer.minor_version)?; + if version == ConcreteFileVersion::V1 { + return Ok(RawFileMetadataOpen::Legacy { + major_version: footer.major_version, + minor_version: footer.minor_version, + }); + } let gbo_table = - Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, file_version).await?; + Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, version).await?; if gbo_table.is_empty() { return Err(Error::internal( "File did not contain any global buffers, schema expected".to_string(), @@ -998,19 +975,20 @@ impl FileReader { } let schema_start = gbo_table[0].position; let schema_size = gbo_table[0].size; - - let num_footer_bytes = file_len - schema_start; - - // By default we read all column metadatas. We do NOT read the column metadata buffers - // at this point. We only want to read the column metadata for columns we are actually loading. + let num_footer_bytes = file_len.checked_sub(schema_start).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Schema position {} is outside file of size {}", + schema_start, file_len + ) + .into(), + ) + })?; let all_metadata_bytes = Self::optimistic_tail_read(&tail_bytes, schema_start, scheduler, file_len).await?; - let schema_bytes = all_metadata_bytes.slice(0..schema_size as usize); let (num_rows, schema) = Self::decode_schema(schema_bytes)?; - // Next, read the metadata for the columns - // This is both the column metadata and the CMO table let column_metadata_start = (footer.column_meta_start - schema_start) as usize; let column_metadata_end = (footer.global_buff_offsets_start - schema_start) as usize; let column_metadata_bytes = @@ -1020,35 +998,37 @@ impl FileReader { let num_global_buffer_bytes = gbo_table.iter().map(|buf| buf.size).sum::(); let num_data_bytes = footer.column_meta_start - num_global_buffer_bytes; let num_column_metadata_bytes = footer.global_buff_offsets_start - footer.column_meta_start; - - let column_infos = Self::meta_to_col_infos(column_metadatas.as_slice(), file_version); - // The tail read above already pulled in any global buffer that lives within // the captured window. Copy those user buffers (index >= 1; the schema at 0 // is decoded above and never fetched via read_global_buffer) out of the tail // so read_global_buffer can serve them without I/O. We copy rather than slice // so the much larger tail allocation can be released once decoding is done. - let retained_global_buffers = - Self::retained_global_buffers_from_tail(&gbo_table, &tail_bytes, tail_offset); + let retained_global_buffers = Self::retained_global_buffers_from_tail( + &gbo_table, + &tail_bytes, + tail_offset, + file_len, + )?; - Ok(CachedFileMetadata { - file_schema: Arc::new(schema), - column_metadatas, - column_infos, - num_rows, - num_data_bytes, - num_column_metadata_bytes, - num_global_buffer_bytes, - num_footer_bytes, - file_buffers: gbo_table, - major_version: footer.major_version, - minor_version: footer.minor_version, - file_size_bytes: file_len, - retained_global_buffers, + Ok(RawFileMetadataOpen::Current { + version, + metadata: RawFileMetadata { + file_schema: Arc::new(schema), + column_metadatas, + num_rows, + file_buffers: gbo_table, + num_data_bytes, + num_column_metadata_bytes, + num_global_buffer_bytes, + num_footer_bytes, + footer, + file_size_bytes: file_len, + retained_global_buffers, + }, }) } - async fn read_metadata_index_with_known_schema( + async fn read_raw_metadata_index_with_known_schema( scheduler: &FileScheduler, known_schema: Option<(Arc, u64)>, ) -> Result { @@ -1056,10 +1036,7 @@ impl FileReader { let tail_offset = file_len - tail_bytes.len() as u64; let footer = Self::decode_footer(&tail_bytes)?; - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; + let file_version = Self::current_file_version(&footer)?; let gbo_table = Self::decode_gbo_table(&tail_bytes, file_len, scheduler, &footer, file_version).await?; @@ -1072,11 +1049,12 @@ impl FileReader { Some((file_schema, num_rows)) => (file_schema, num_rows), None => { let schema_buffer = &gbo_table[0]; + let schema_range = schema_buffer.checked_range(0, file_len)?; let schema_bytes = Self::read_range_from_tail_or_scheduler( &tail_bytes, tail_offset, scheduler, - schema_buffer.position..schema_buffer.position + schema_buffer.size, + schema_range, ) .await?; let (num_rows, schema) = Self::decode_schema(schema_bytes)?; @@ -1093,8 +1071,12 @@ impl FileReader { .await?; let column_metadata_offsets = Self::decode_cmo_table(cmo_table, &footer)?; - let retained_global_buffers = - Self::retained_global_buffers_from_tail(&gbo_table, &tail_bytes, tail_offset); + let retained_global_buffers = Self::retained_global_buffers_from_tail( + &gbo_table, + &tail_bytes, + tail_offset, + file_len, + )?; Ok(FileMetadataIndex { file_schema, @@ -1113,107 +1095,26 @@ impl FileReader { /// This reads the file schema from the schema global buffer. Use /// [`Self::read_metadata_index_with_schema`] when the caller already has /// the schema and row count from a higher-level metadata source. - pub async fn read_metadata_index(scheduler: &FileScheduler) -> Result { - Self::read_metadata_index_with_known_schema(scheduler, None).await + pub(crate) async fn read_raw_metadata_index( + scheduler: &FileScheduler, + ) -> Result { + Self::read_raw_metadata_index_with_known_schema(scheduler, None).await } /// Reads the metadata index without fetching the schema global buffer. /// /// Use this when the caller already has the file schema and physical row /// count from an enclosing metadata layer, such as a dataset manifest. - pub async fn read_metadata_index_with_schema( + pub(crate) async fn read_raw_metadata_index_with_schema( scheduler: &FileScheduler, file_schema: Arc, num_rows: u64, ) -> Result { - Self::read_metadata_index_with_known_schema(scheduler, Some((file_schema, num_rows))).await - } - - fn fetch_encoding(encoding: &pbfile::Encoding) -> M { - match &encoding.location { - Some(pbfile::encoding::Location::Indirect(_)) => todo!(), - Some(pbfile::encoding::Location::Direct(encoding)) => { - let encoding_buf = Bytes::from(encoding.encoding.clone()); - let encoding_any = prost_types::Any::decode(encoding_buf).unwrap(); - encoding_any.to_msg::().unwrap() - } - Some(pbfile::encoding::Location::None(_)) => panic!(), - None => panic!(), - } - } - - fn meta_to_col_infos( - column_metadatas: &[pbfile::ColumnMetadata], - file_version: LanceFileVersion, - ) -> Vec> { - column_metadatas - .iter() - .enumerate() - .map(|(col_idx, col_meta)| { - Self::meta_to_col_info(col_idx as u32, col_meta, file_version) - }) - .collect::>() - } - - fn meta_to_col_info( - col_idx: u32, - col_meta: &pbfile::ColumnMetadata, - file_version: LanceFileVersion, - ) -> Arc { - let page_infos = col_meta - .pages - .iter() - .map(|page| { - let num_rows = page.length; - let encoding = match file_version { - LanceFileVersion::V2_0 => { - PageEncoding::Legacy(Self::fetch_encoding::( - page.encoding.as_ref().unwrap(), - )) - } - _ => PageEncoding::Structural(Self::fetch_encoding::( - page.encoding.as_ref().unwrap(), - )), - }; - let buffer_offsets_and_sizes = Arc::from( - page.buffer_offsets - .iter() - .zip(page.buffer_sizes.iter()) - .map(|(offset, size)| { - // Starting with version 2.1 we can assert that page buffers are aligned - assert!( - file_version < LanceFileVersion::V2_1 - || offset % PAGE_BUFFER_ALIGNMENT as u64 == 0 - ); - (*offset, *size) - }) - .collect::>(), - ); - PageInfo { - buffer_offsets_and_sizes, - encoding, - num_rows, - priority: page.priority, - } - }) - .collect::>(); - let buffer_offsets_and_sizes = Arc::from( - col_meta - .buffer_offsets - .iter() - .zip(col_meta.buffer_sizes.iter()) - .map(|(offset, size)| (*offset, *size)) - .collect::>(), - ); - Arc::new(ColumnInfo { - index: col_idx, - page_infos: Arc::from(page_infos), - buffer_offsets_and_sizes, - encoding: Self::fetch_encoding(col_meta.encoding.as_ref().unwrap()), - }) + Self::read_raw_metadata_index_with_known_schema(scheduler, Some((file_schema, num_rows))) + .await } - fn validate_projection( + pub(crate) fn validate_projection( projection: &ReaderProjection, metadata: &CachedFileMetadata, ) -> Result<()> { @@ -1242,87 +1143,25 @@ impl FileReader { Ok(()) } - /// Opens a new file reader without any pre-existing knowledge - /// - /// This will read the file schema from the file itself and thus requires a bit more I/O - /// - /// A `base_projection` can also be provided. If provided, then the projection will apply - /// to all reads from the file that do not specify their own projection. - pub async fn try_open( - scheduler: FileScheduler, - base_projection: Option, - decoder_plugins: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let file_metadata = Arc::new(Self::read_all_metadata(&scheduler).await?); - let path = scheduler.reader().path().clone(); - - // Create LanceEncodingsIo with read chunk size from options - let encodings_io = - LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size); - - Self::try_open_with_file_metadata( - Arc::new(encodings_io), - path, - base_projection, - decoder_plugins, - file_metadata, - cache, - options, - ) - .await - } - - /// Same as `try_open` but with the file metadata already loaded. - /// - /// This method also can accept any kind of `EncodingsIo` implementation allowing - /// for custom strategies to be used for I/O scheduling (e.g. for takes on fast - /// disks it may be better to avoid asynchronous overhead). - /// Opens a data reader backed by fully decoded file metadata. - pub async fn try_open_with_file_metadata( - scheduler: Arc, - path: Path, - base_projection: Option, - decoder_plugins: Arc, - file_metadata: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let cache = Arc::new(cache.with_key_prefix(path.as_ref())); - let core = FileReadCore::try_new( - scheduler, - base_projection, - decoder_plugins, - FileMetadataProvider::Full(file_metadata.clone()), - cache, - options, - )?; - Ok(Self { - core, - metadata: file_metadata, - }) - } - - // The actual decoder needs all the column infos that make up a type. In other words, if - // the first type in the schema is Struct then the decoder will need 3 column infos. - // - // This is a file reader concern because the file reader needs to support late projection of columns - // and so it will need to figure this out anyways. - // - // It's a bit of a tricky process though because the number of column infos may depend on the - // encoding. Considering the above example, if we wrote it with a packed encoding, then there would - // only be a single column in the file (and not 3). - // - // At the moment this method words because our rules are simple and we just repeat them here. See - // Self::default_projection for a similar problem. In the future this is something the encodings - // registry will need to figure out. - fn collect_columns_from_projection( - &self, - _projection: &ReaderProjection, - ) -> Result>> { - Ok(self.metadata.column_infos.clone()) - } + // The actual decoder needs all the column infos that make up a type. In other words, if + // the first type in the schema is Struct then the decoder will need 3 column infos. + // + // This is a file reader concern because the file reader needs to support late projection of columns + // and so it will need to figure this out anyways. + // + // It's a bit of a tricky process though because the number of column infos may depend on the + // encoding. Considering the above example, if we wrote it with a packed encoding, then there would + // only be a single column in the file (and not 3). + // + // At the moment this method words because our rules are simple and we just repeat them here. See + // Self::default_projection for a similar problem. In the future this is something the encodings + // registry will need to figure out. + fn collect_columns_from_projection( + &self, + _projection: &ReaderProjection, + ) -> Result>> { + Ok(self.metadata.column_infos.clone()) + } #[allow(clippy::too_many_arguments)] async fn do_read_range( @@ -1459,92 +1298,6 @@ impl FileReader { .await } - /// Creates a stream of "read tasks" to read the data from the file - /// - /// The arguments are similar to [`Self::read_stream_projected`] but instead of returning a stream - /// of record batches it returns a stream of "read tasks". - /// - /// The tasks should be consumed with some kind of `buffered` argument if CPU parallelism is desired. - /// - /// Note that "read task" is probably a bit imprecise. The tasks are actually "decode tasks". The - /// reading happens asynchronously in the background. In other words, a single read task may map to - /// multiple I/O operations or a single I/O operation may map to multiple read tasks. - /// - /// # Why is this async? - /// - /// Constructing the read stream requires running the decode scheduler's - /// `initialize` step, which performs the metadata I/O (chunk metadata, - /// dictionaries, repetition index, ...) needed to plan the read. We - /// drive that I/O on the awaiting task rather than smuggling it into - /// the stream's first poll. This way callers control where the - /// scheduling I/O runs (typically inside a per-fragment - /// `tokio::spawn`), planning errors surface from the await instead of - /// from the first stream item, and small reads can also complete the - /// synchronous scheduling step before returning (see - /// [`DecoderConfig::inline_scheduling`]). - pub async fn read_tasks( - &self, - params: ReadBatchParams, - batch_size: u32, - projection: Option, - filter: FilterExpression, - ) -> Result + Send>>> { - self.core - .read_tasks(params, batch_size, projection, filter) - .await - } - - /// Reads data from the file as a stream of record batches - /// - /// * `params` - Specifies the range (or indices) of data to read - /// * `batch_size` - The maximum size of a single batch. A batch may be smaller - /// if it is the last batch or if it is not possible to create a batch of the - /// requested size. - /// - /// For example, if the batch size is 1024 and one of the columns is a string - /// column then there may be some ranges of 1024 rows that contain more than - /// 2^31 bytes of string data (which is the maximum size of a string column - /// in Arrow). In this case smaller batches may be emitted. - /// * `batch_readahead` - The number of batches to read ahead. This controls the - /// amount of CPU parallelism of the read. In other words it controls how many - /// batches will be decoded in parallel. It has no effect on the I/O parallelism - /// of the read (how many I/O requests are in flight at once). - /// - /// This parameter also is also related to backpressure. If the consumer of the - /// stream is slow then the reader will build up RAM. - /// * `projection` - A projection to apply to the read. This controls which columns - /// are read from the file. The projection is NOT applied on top of the base - /// projection. The projection is applied directly to the file schema. - /// - /// # Why is this async? - /// - /// This delegates to [`Self::read_tasks`], which awaits the decode - /// scheduler's `initialize` step (and, for small reads, the synchronous - /// scheduling that follows) before returning. See `read_tasks` for - /// details on why this work is performed up front rather than on the - /// stream's first poll. - pub async fn read_stream_projected( - &self, - params: ReadBatchParams, - batch_size: u32, - batch_readahead: u32, - projection: ReaderProjection, - filter: FilterExpression, - ) -> Result>> { - let arrow_schema = Arc::new(ArrowSchema::from(projection.schema.as_ref())); - let tasks_stream = self - .read_tasks(params, batch_size, Some(projection), filter) - .await?; - let batch_stream = tasks_stream - .map(|task| task.task) - .buffered(batch_readahead as usize) - .boxed(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - arrow_schema, - batch_stream, - ))) - } - fn take_rows_blocking( &self, indices: Vec, @@ -1663,36 +1416,15 @@ impl FileReader { ) } - /// Read data from the file as an iterator of record batches - /// - /// This is a blocking variant of [`Self::read_stream_projected`] that runs entirely in the - /// calling thread. It will block on I/O if the decode is faster than the I/O. It is useful - /// for benchmarking and potentially from "take"ing small batches from fast disks. - /// - /// Large scans of in-memory data will still benefit from threading (and should therefore not - /// use this method) because we can parallelize the decode. - /// - /// Note: calling this from within a tokio runtime will panic. It is acceptable to call this - /// from a spawn_blocking context. - pub fn read_stream_projected_blocking( + pub(crate) fn read_prepared_blocking( &self, params: ReadBatchParams, batch_size: u32, - projection: Option, + prepared: PreparedProjection, + read_len: u64, filter: FilterExpression, ) -> Result> { - let projection = projection.unwrap_or_else(|| self.core.base_projection.clone()); - Self::validate_projection(&projection, &self.metadata)?; - // Apply the same projection-length validation as the async path. This - // reader is always backed by full metadata, so we can build the prepared - // projection synchronously (no column-metadata I/O) and reuse the shared - // check. `read_len` is the projection's common column length, which - // `RangeFull`/`RangeFrom` resolve against rather than `num_rows`. - let prepared = PreparedProjection { - column_infos: self.metadata.column_infos.clone(), - decoder_projection: projection.clone(), - }; - let read_len = self.core.prepared_read_length(&prepared)?; + let projection = prepared.decoder_projection; let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| { if bound > read_len || (bound == read_len && inclusive) { Err(Error::invalid_input(format!( @@ -1704,17 +1436,13 @@ impl FileReader { }; match ¶ms { ReadBatchParams::Indices(indices) => { - for idx in indices { - match idx { - None => { - return Err(Error::invalid_input("Null value in indices array")); - } - Some(idx) => { - verify_bound(¶ms, idx as u64, true)?; - } + for index in indices { + match index { + None => return Err(Error::invalid_input("Null value in indices array")), + Some(index) => verify_bound(¶ms, index as u64, true)?, } } - let indices = indices.iter().map(|idx| idx.unwrap() as u64).collect(); + let indices = indices.iter().map(|index| index.unwrap() as u64).collect(); self.take_rows_blocking(indices, batch_size, projection, filter) } ReadBatchParams::Range(range) => { @@ -1753,76 +1481,55 @@ impl FileReader { } } - /// Reads data from the file as a stream of record batches - /// - /// This is similar to [`Self::read_stream_projected`] but uses the base projection - /// provided when the file was opened (or reads all columns if the file was - /// opened without a base projection) - /// - /// # Why is this async? - /// - /// This delegates to [`Self::read_stream_projected`], which awaits the - /// decode scheduler's `initialize` step before returning the stream. - /// See [`Self::read_tasks`] for the rationale. - pub async fn read_stream( - &self, - params: ReadBatchParams, - batch_size: u32, - batch_readahead: u32, - filter: FilterExpression, - ) -> Result>> { - self.read_stream_projected( - params, - batch_size, - batch_readahead, - self.core.base_projection.clone(), - filter, - ) - .await - } - pub fn schema(&self) -> &Arc { self.core.schema() } } impl FileMetadataProvider { - fn version(&self) -> LanceFileVersion { + pub(crate) fn version(&self) -> ConcreteFileVersion { match self { - Self::Full(metadata) => metadata.version(), + Self::Full(metadata) => metadata.version, Self::Indexed(metadata_index) => metadata_index.version, } } - fn num_rows(&self) -> u64 { + pub(crate) fn num_rows(&self) -> u64 { match self { Self::Full(metadata) => metadata.num_rows, Self::Indexed(metadata_index) => metadata_index.num_rows, } } - fn schema(&self) -> &Arc { + pub(crate) fn schema(&self) -> &Arc { match self { Self::Full(metadata) => &metadata.file_schema, Self::Indexed(metadata_index) => &metadata_index.file_schema, } } - fn file_buffers(&self) -> &Vec { + pub(crate) fn file_buffers(&self) -> &Vec { match self { Self::Full(metadata) => &metadata.file_buffers, Self::Indexed(metadata_index) => &metadata_index.file_buffers, } } - fn retained_global_buffers(&self) -> &BTreeMap { + pub(crate) fn retained_global_buffers(&self) -> &BTreeMap { match self { Self::Full(metadata) => &metadata.retained_global_buffers, Self::Indexed(metadata_index) => &metadata_index.retained_global_buffers, } } - fn file_statistics(&self) -> Option { + fn file_size(&self) -> u64 { + match self { + Self::Full(metadata) => metadata.file_size_bytes, + Self::Indexed(metadata_index) => metadata_index.file_size_bytes, + } + } + + pub(crate) fn file_statistics(&self) -> Option { let metadata = match self { Self::Full(metadata) => metadata, Self::Indexed(_) => return None, @@ -1832,19 +1539,22 @@ impl FileMetadataProvider { )) } - fn supports_indexed_projection( - projection: &ReaderProjection, - version: LanceFileVersion, - ) -> bool { - version >= LanceFileVersion::V2_1 - && !projection.schema.fields.is_empty() - && projection.schema.fields.len() == projection.column_indices.len() - && projection.schema.fields.iter().all(|field| { - field.children.is_empty() && !field.is_blob() && !field.is_packed_struct() + pub(crate) fn projection_matches_indexed_metadata(projection: &ReaderProjection) -> bool { + if projection.schema.fields.is_empty() { + return false; + } + + projection + .schema + .fields + .iter() + .try_fold(0usize, |count, field| { + count.checked_add(indexed_projection_column_count(field)?) }) + == Some(projection.column_indices.len()) } - fn validate_indexed_projection( + pub(crate) fn validate_indexed_projection_structure( projection: &ReaderProjection, metadata_index: &FileMetadataIndex, ) -> Result<()> { @@ -1869,24 +1579,19 @@ impl FileMetadataProvider { ))); } } - if !Self::supports_indexed_projection(projection, metadata_index.version) { - return Err(Error::not_supported(format!( - "lazy column metadata loading only supports direct V2.1+ top-level physical column projections; got file version {:?}, {} schema fields, and {} column indices", - metadata_index.version, - projection.schema.fields.len(), - projection.column_indices.len() - ))); - } Ok(()) } - fn validate_projection(&self, projection: &ReaderProjection) -> Result<()> { - match self { - Self::Full(metadata) => FileReader::validate_projection(projection, metadata), - Self::Indexed(metadata_index) => { - Self::validate_indexed_projection(projection, metadata_index) - } - } + pub(crate) fn indexed_projection_error( + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Error { + Error::not_supported(format!( + "lazy column metadata loading requires a V2.1+ ordinary structural projection without blob or packed-struct fields whose physical-column count matches the projection; got file version {:?}, {} schema fields, and {} column indices", + metadata_index.version, + projection.schema.fields.len(), + projection.column_indices.len() + )) } fn column_metadata_range( @@ -1912,12 +1617,16 @@ impl FileMetadataProvider { Ok(position..end) } - async fn load_indexed_column_infos( + pub(crate) async fn load_indexed_column_infos( metadata_index: &FileMetadataIndex, io: &Arc, cache: &Arc, column_indices: &[u32], - ) -> Result>> { + decode_column: F, + ) -> Result>> + where + F: Fn(u32, &pbfile::ColumnMetadata) -> Result>, + { let mut column_infos = vec![None; column_indices.len()]; let mut missing_columns = Vec::new(); @@ -1939,14 +1648,10 @@ impl FileMetadataProvider { .collect::>(); let metadata_bytes = io.submit_request(ranges, 0).await?; for ((result_index, column_index, _), bytes) in - missing_columns.into_iter().zip(metadata_bytes.into_iter()) + missing_columns.into_iter().zip(metadata_bytes) { let column_metadata = pbfile::ColumnMetadata::decode(bytes)?; - let column_info = FileReader::meta_to_col_info( - column_index, - &column_metadata, - metadata_index.version, - ); + let column_info = decode_column(column_index, &column_metadata)?; let cached = Arc::new(CachedColumnMetadata { column_metadata, column_info: column_info.clone(), @@ -1970,83 +1675,41 @@ impl FileMetadataProvider { }) .collect() } - - async fn prepare_projection( - &self, - projection: &ReaderProjection, - io: &Arc, - cache: &Arc, - ) -> Result { - self.validate_projection(projection)?; - match self { - Self::Full(metadata) => Ok(PreparedProjection { - column_infos: metadata.column_infos.clone(), - decoder_projection: projection.clone(), - }), - Self::Indexed(metadata_index) => { - let column_infos = Self::load_indexed_column_infos( - metadata_index, - io, - cache, - &projection.column_indices, - ) - .await?; - let decoder_projection = ReaderProjection { - schema: projection.schema.clone(), - column_indices: (0..projection.column_indices.len()) - .map(|idx| idx as u32) - .collect(), - }; - Ok(PreparedProjection { - column_infos, - decoder_projection, - }) - } - } - } } -impl FileReadCore { - fn try_new( +impl DecodeEngine { + pub(crate) fn try_new( scheduler: Arc, - base_projection: Option, + base_projection: ReaderProjection, decoder_plugins: Arc, metadata_provider: FileMetadataProvider, + read_projection: Arc, cache: Arc, options: FileReaderOptions, ) -> Result { - if let Some(base_projection) = base_projection.as_ref() { - metadata_provider.validate_projection(base_projection)?; - } - let base_projection = base_projection.unwrap_or(ReaderProjection::from_whole_schema( - metadata_provider.schema().as_ref(), - metadata_provider.version(), - )); Ok(Self { scheduler, base_projection, metadata_provider, + read_projection, decoder_plugins, cache, options, }) } - fn with_scheduler(&self, scheduler: Arc) -> Self { + pub(crate) fn with_scheduler(&self, scheduler: Arc) -> Self { Self { scheduler, base_projection: self.base_projection.clone(), metadata_provider: self.metadata_provider.clone(), + read_projection: self.read_projection.clone(), decoder_plugins: self.decoder_plugins.clone(), cache: self.cache.clone(), options: self.options.clone(), } } - fn version(&self) -> LanceFileVersion { - self.metadata_provider.version() - } - fn num_rows(&self) -> u64 { self.metadata_provider.num_rows() } @@ -2055,7 +1718,7 @@ impl FileReadCore { self.metadata_provider.schema() } - async fn read_global_buffer(&self, index: u32) -> Result { + pub(crate) async fn read_global_buffer(&self, index: u32) -> Result { let file_buffers = self.metadata_provider.file_buffers(); let buffer_desc = file_buffers.get(index as usize).ok_or_else(|| { Error::invalid_input(format!( @@ -2072,7 +1735,10 @@ impl FileReadCore { let bytes = self .scheduler .submit_request( - vec![buffer_desc.position..buffer_desc.position + buffer_desc.size], + vec![ + buffer_desc + .checked_range(index as usize, self.metadata_provider.file_size())?, + ], 0, ) .await?; @@ -2084,57 +1750,6 @@ impl FileReadCore { }) } - // The common length to read across a prepared projection, after validating - // its columns can be combined into rectangular batches. Each top-level field - // is checked for internal consistency (see `validate_field_length`); the - // top-level fields must then share a length, since one read combines them. - // Ordinary files always pass (every column has `num_rows` rows); files - // written with `FileWriter::write_column` whose columns ended up unequal are - // rejected here and must be read separately. - // - // `column_infos` and `decoder_projection.column_indices` line up for both - // metadata providers: the full provider keeps absolute indices into the whole - // file, while the indexed (lazy) provider loads only the projected columns and - // renumbers them 0..N -- in either case `column_infos[column_index]` is the - // requested column. - fn prepared_read_length(&self, prepared: &PreparedProjection) -> Result { - let is_structural = self.version() >= LanceFileVersion::V2_1; - let column_infos = &prepared.column_infos; - let column_len = |column: usize| -> Result { - let info = column_infos.get(column).ok_or_else(|| { - Error::invalid_input(format!( - "projection references column index {} but only {} columns are available", - column, - column_infos.len() - )) - })?; - Ok(info.page_infos.iter().map(|page| page.num_rows).sum()) - }; - let column_indices = &prepared.decoder_projection.column_indices; - let fields = &prepared.decoder_projection.schema.fields; - let mut cursor = 0usize; - let mut field_lengths = Vec::with_capacity(fields.len()); - for field in fields { - let rows = validate_field_length( - field, - is_structural, - true, - column_indices, - &mut cursor, - &column_len, - )?; - field_lengths.push((field.name.as_str(), rows)); - } - if cursor != column_indices.len() { - return Err(Error::invalid_input(format!( - "projection supplied {} column indices but its fields require {}", - column_indices.len(), - cursor - ))); - } - verify_uniform_lengths(&field_lengths) - } - async fn read_range( &self, range: Range, @@ -2201,25 +1816,14 @@ impl FileReadCore { .await } - async fn read_tasks( + pub(crate) async fn read_prepared_tasks( &self, params: ReadBatchParams, batch_size: u32, - projection: Option, + prepared: PreparedProjection, + read_len: u64, filter: FilterExpression, ) -> Result + Send>>> { - let projection = projection.unwrap_or_else(|| self.base_projection.clone()); - let prepared = self - .metadata_provider - .prepare_projection(&projection, &self.scheduler, &self.cache) - .await?; - // All projected columns must share a length: the reader combines them - // into rectangular batches. Ordinary files satisfy this (every column - // has `num_rows` rows); files written with `FileWriter::write_column` - // may not, and such columns must be read separately. `read_len` is that - // common length, which `RangeFull`/`RangeFrom` resolve against (rather - // than `num_rows`, the file's longest column). - let read_len = self.prepared_read_length(&prepared)?; let verify_bound = |params: &ReadBatchParams, bound: u64, inclusive: bool| { if bound > read_len || (bound == read_len && inclusive) { Err(Error::invalid_input(format!( @@ -2282,93 +1886,21 @@ impl FileReadCore { } impl ProjectedFileReader { - /// Opens a data reader backed by indexed column metadata. - /// - /// `base_projection` must be a supported indexed projection. Reads that do - /// not pass an explicit projection use this base projection. - pub async fn try_open( - scheduler: FileScheduler, - base_projection: Option, - decoder_plugins: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let base_projection = Self::require_indexed_base_projection(base_projection)?; - let metadata_index = Arc::new(FileReader::read_metadata_index(&scheduler).await?); - let path = scheduler.reader().path().clone(); - let encodings_io = - LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size); - Self::try_open_with_metadata_index( - Arc::new(encodings_io), - path, - Some(base_projection), - decoder_plugins, - metadata_index, - cache, - options, - ) - .await - } - - /// Opens a data reader from a previously loaded metadata index. - /// - /// `base_projection` must be a supported indexed projection. Use - /// [`Self::try_open_with_file_metadata`] when the default read should cover - /// the whole file schema. - pub async fn try_open_with_metadata_index( - scheduler: Arc, - path: Path, - base_projection: Option, - decoder_plugins: Arc, - metadata_index: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let base_projection = Self::require_indexed_base_projection(base_projection)?; - let cache = Arc::new(cache.with_key_prefix(path.as_ref())); - let core = FileReadCore::try_new( - scheduler, - Some(base_projection), - decoder_plugins, - FileMetadataProvider::Indexed(metadata_index), - cache, - options, - )?; - Ok(Self { core }) - } - - fn require_indexed_base_projection( - base_projection: Option, - ) -> Result { - base_projection.ok_or_else(|| { - Error::invalid_input("ProjectedFileReader requires an explicit base projection") - }) - } - - pub async fn try_open_with_file_metadata( - scheduler: Arc, - path: Path, - base_projection: Option, - decoder_plugins: Arc, - file_metadata: Arc, - cache: &LanceCache, - options: FileReaderOptions, - ) -> Result { - let cache = Arc::new(cache.with_key_prefix(path.as_ref())); - let core = FileReadCore::try_new( - scheduler, - base_projection, - decoder_plugins, - FileMetadataProvider::Full(file_metadata), - cache, - options, - )?; - Ok(Self { core }) + pub(crate) fn base_projection(&self) -> &ReaderProjection { + &self.core.base_projection } - /// Returns whether a projection can be served by indexed column metadata. - pub fn supports_projection(projection: &ReaderProjection, version: LanceFileVersion) -> bool { - FileMetadataProvider::supports_indexed_projection(projection, version) + pub(crate) async fn read_prepared_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + prepared: PreparedProjection, + read_len: u64, + filter: FilterExpression, + ) -> Result + Send>>> { + self.core + .read_prepared_tasks(params, batch_size, prepared, read_len, filter) + .await } /// Returns a clone of this reader using a different scheduler. @@ -2378,11 +1910,6 @@ impl ProjectedFileReader { } } - /// Returns the Lance file version. - pub fn version(&self) -> LanceFileVersion { - self.core.version() - } - /// Returns the number of rows in the file. pub fn num_rows(&self) -> u64 { self.core.num_rows() @@ -2399,7 +1926,7 @@ impl ProjectedFileReader { } #[cfg(test)] - fn metadata_index(&self) -> Option<&Arc> { + pub(crate) fn metadata_index(&self) -> Option<&Arc> { match &self.core.metadata_provider { FileMetadataProvider::Indexed(metadata_index) => Some(metadata_index), FileMetadataProvider::Full(_) => None, @@ -2410,17 +1937,348 @@ impl ProjectedFileReader { pub async fn read_global_buffer(&self, index: u32) -> Result { self.core.read_global_buffer(index).await } +} - /// Creates a stream of read tasks for the requested rows and projection. - pub async fn read_tasks( - &self, - params: ReadBatchParams, - batch_size: u32, - projection: Option, - filter: FilterExpression, - ) -> Result + Send>>> { - self.core - .read_tasks(params, batch_size, projection, filter) +impl FileReader { + #[cfg(test)] + fn scheduler(&self) -> Arc { + self.core.scheduler.clone() + } + + pub async fn try_open( + scheduler: FileScheduler, + base_projection: Option, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + match Self::try_open_for_dispatch( + scheduler, + base_projection, + decoder_plugins, + cache, + options, + ) + .await? + { + versions::OpenedFileReader::V1 { + major_version, + minor_version, + } => Err(Error::version_conflict( + "Attempt to use the Lance current-format reader to read a v1 file".to_string(), + major_version, + minor_version, + )), + versions::OpenedFileReader::Current(reader) => Ok(reader), + } + } + + pub(crate) async fn try_open_for_dispatch( + scheduler: FileScheduler, + base_projection: Option, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + let metadata = match Self::read_raw_metadata_for_dispatch(&scheduler).await? { + RawFileMetadataOpen::Legacy { + major_version, + minor_version, + } => { + return Ok(versions::OpenedFileReader::V1 { + major_version, + minor_version, + }); + } + RawFileMetadataOpen::Current { version, metadata } => { + Arc::new(versions::finish_metadata(version, metadata)?) + } + }; + let path = scheduler.reader().path().clone(); + let io = Arc::new( + LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size), + ); + Self::try_open_with_file_metadata( + io, + path, + base_projection, + decoder_plugins, + metadata, + cache, + options, + ) + .await + .map(versions::OpenedFileReader::Current) + } + + pub async fn try_open_with_file_metadata( + scheduler: Arc, + path: Path, + base_projection: Option, + decoder_plugins: Arc, + metadata: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + if metadata.version == ConcreteFileVersion::V1 { + return Err(Error::version_conflict( + "Attempt to use the Lance current-format reader with v1 metadata".to_string(), + metadata.major_version, + metadata.minor_version, + )); + } + let read_projection = versions::read_projection(metadata.version)?; + let has_explicit_projection = base_projection.is_some(); + let base_projection = base_projection.unwrap_or_else(|| { + versions::reader_projection_from_whole_schema(&metadata.file_schema, metadata.version) + }); + if has_explicit_projection { + Self::validate_projection(&base_projection, &metadata)?; + } + let cache = Arc::new(cache.with_key_prefix(path.as_ref())); + let core = DecodeEngine::try_new( + scheduler, + base_projection, + decoder_plugins, + FileMetadataProvider::Full(metadata.clone()), + read_projection, + cache, + options, + )?; + Ok(Self { core, metadata }) + } + + pub async fn read_all_metadata(scheduler: &FileScheduler) -> Result { + match Self::read_raw_metadata_for_dispatch(scheduler).await? { + RawFileMetadataOpen::Legacy { + major_version, + minor_version, + } => Err(Error::version_conflict( + "Attempt to use the Lance current-format reader to read v1 metadata".to_string(), + major_version, + minor_version, + )), + RawFileMetadataOpen::Current { version, metadata } => { + versions::finish_metadata(version, metadata) + } + } + } + + pub async fn read_metadata_index(scheduler: &FileScheduler) -> Result { + let index = Self::read_raw_metadata_index(scheduler).await?; + versions::finish_metadata_index(index) + } + + pub async fn read_metadata_index_with_schema( + scheduler: &FileScheduler, + file_schema: Arc, + num_rows: u64, + ) -> Result { + let index = + Self::read_raw_metadata_index_with_schema(scheduler, file_schema, num_rows).await?; + versions::finish_metadata_index(index) + } + + pub fn version(&self) -> ConcreteFileVersion { + self.metadata.version + } + + async fn prepare(&self, projection: ReaderProjection) -> Result<(PreparedProjection, u64)> { + self.core + .read_projection + .prepare( + &self.core.metadata_provider, + &projection, + &self.core.scheduler, + &self.core.cache, + ) + .await + } + + pub async fn read_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + projection: Option, + filter: FilterExpression, + ) -> Result + Send>>> { + let projection = projection.unwrap_or_else(|| self.base_projection().clone()); + let (prepared, read_len) = self.prepare(projection).await?; + self.read_prepared_tasks(params, batch_size, prepared, read_len, filter) + .await + } + + pub async fn read_stream_projected( + &self, + params: ReadBatchParams, + batch_size: u32, + batch_readahead: u32, + projection: ReaderProjection, + filter: FilterExpression, + ) -> Result>> { + let schema = projection.schema.clone(); + let tasks = self + .read_tasks(params, batch_size, Some(projection), filter) + .await?; + Ok(tasks_to_record_batch_stream(schema, tasks, batch_readahead)) + } + + pub fn read_stream_projected_blocking( + &self, + params: ReadBatchParams, + batch_size: u32, + projection: Option, + filter: FilterExpression, + ) -> Result> { + let projection = projection.unwrap_or_else(|| self.base_projection().clone()); + Self::validate_projection(&projection, self.metadata())?; + let prepared = self.full_projection(projection); + let read_len = self.core.read_projection.read_length(&prepared)?; + self.read_prepared_blocking(params, batch_size, prepared, read_len, filter) + } + + pub async fn read_stream( + &self, + params: ReadBatchParams, + batch_size: u32, + batch_readahead: u32, + filter: FilterExpression, + ) -> Result>> { + self.read_stream_projected( + params, + batch_size, + batch_readahead, + self.base_projection().clone(), + filter, + ) + .await + } +} + +impl ProjectedFileReader { + pub async fn try_open( + scheduler: FileScheduler, + base_projection: Option, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + let base_projection = base_projection.ok_or_else(|| { + Error::invalid_input("ProjectedReader requires an explicit base projection") + })?; + let metadata_index = Arc::new(FileReader::read_metadata_index(&scheduler).await?); + let path = scheduler.reader().path().clone(); + let io = Arc::new( + LanceEncodingsIo::new(scheduler).with_read_chunk_size(options.read_chunk_size), + ); + Self::try_open_with_metadata_index( + io, + path, + Some(base_projection), + decoder_plugins, + metadata_index, + cache, + options, + ) + .await + } + + pub async fn try_open_with_metadata_index( + scheduler: Arc, + path: Path, + base_projection: Option, + decoder_plugins: Arc, + metadata_index: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + if metadata_index.version == ConcreteFileVersion::V1 { + return Err(Error::version_conflict( + "Attempt to use the Lance projected current-format reader with v1 metadata" + .to_string(), + 0, + 2, + )); + } + let base_projection = base_projection.ok_or_else(|| { + Error::invalid_input("ProjectedReader requires an explicit base projection") + })?; + let read_projection = versions::read_projection(metadata_index.version)?; + read_projection.validate_indexed(&base_projection, &metadata_index)?; + let cache = Arc::new(cache.with_key_prefix(path.as_ref())); + let core = DecodeEngine::try_new( + scheduler, + base_projection, + decoder_plugins, + FileMetadataProvider::Indexed(metadata_index), + read_projection, + cache, + options, + )?; + Ok(Self { core }) + } + + pub async fn try_open_with_file_metadata( + scheduler: Arc, + path: Path, + base_projection: Option, + decoder_plugins: Arc, + metadata: Arc, + cache: &LanceCache, + options: FileReaderOptions, + ) -> Result { + if metadata.version == ConcreteFileVersion::V1 { + return Err(Error::version_conflict( + "Attempt to use the Lance projected current-format reader with v1 metadata" + .to_string(), + metadata.major_version, + metadata.minor_version, + )); + } + let read_projection = versions::read_projection(metadata.version)?; + let has_explicit_projection = base_projection.is_some(); + let base_projection = base_projection.unwrap_or_else(|| { + versions::reader_projection_from_whole_schema(&metadata.file_schema, metadata.version) + }); + if has_explicit_projection { + FileReader::validate_projection(&base_projection, &metadata)?; + } + let cache = Arc::new(cache.with_key_prefix(path.as_ref())); + let core = DecodeEngine::try_new( + scheduler, + base_projection, + decoder_plugins, + FileMetadataProvider::Full(metadata), + read_projection, + cache, + options, + )?; + Ok(Self { core }) + } + + pub fn version(&self) -> ConcreteFileVersion { + self.core.metadata_provider.version() + } + + pub async fn read_tasks( + &self, + params: ReadBatchParams, + batch_size: u32, + projection: Option, + filter: FilterExpression, + ) -> Result + Send>>> { + let projection = projection.unwrap_or_else(|| self.base_projection().clone()); + let (prepared, read_len) = self + .core + .read_projection + .prepare( + &self.core.metadata_provider, + &projection, + &self.core.scheduler, + &self.core.cache, + ) + .await?; + self.read_prepared_tasks(params, batch_size, prepared, read_len, filter) .await } } @@ -2475,11 +2333,7 @@ pub fn describe_encoding(page: &pbfile::column_metadata::Page) -> String { } pub trait EncodedBatchReaderExt { - fn try_from_mini_lance( - bytes: Bytes, - schema: &Schema, - version: LanceFileVersion, - ) -> Result + fn try_from_mini_lance(bytes: Bytes, schema: &Schema) -> Result where Self: Sized; fn try_from_self_described_lance(bytes: Bytes) -> Result @@ -2488,16 +2342,13 @@ pub trait EncodedBatchReaderExt { } impl EncodedBatchReaderExt for EncodedBatch { - fn try_from_mini_lance( - bytes: Bytes, - schema: &Schema, - file_version: LanceFileVersion, - ) -> Result + fn try_from_mini_lance(bytes: Bytes, schema: &Schema) -> Result where Self: Sized, { - let projection = ReaderProjection::from_whole_schema(schema, file_version); let footer = FileReader::decode_footer(&bytes)?; + let file_version = FileReader::current_file_version(&footer)?; + let projection = versions::reader_projection_from_whole_schema(schema, file_version); // Next, read the metadata for the columns // This is both the column metadata and the CMO table @@ -2507,12 +2358,7 @@ impl EncodedBatchReaderExt for EncodedBatch { let column_metadatas = FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?; - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; - - let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version); + let page_table = versions::decode_column_metadata(file_version, &column_metadatas)?; Ok(Self { data: bytes, @@ -2531,27 +2377,26 @@ impl EncodedBatchReaderExt for EncodedBatch { Self: Sized, { let footer = FileReader::decode_footer(&bytes)?; - let file_version = LanceFileVersion::try_from_major_minor( - footer.major_version as u32, - footer.minor_version as u32, - )?; + let file_version = FileReader::current_file_version(&footer)?; + let file_len = bytes.len() as u64; let gbo_table = FileReader::do_decode_gbo_table( &bytes.slice(footer.global_buff_offsets_start as usize..), &footer, - file_version, )?; + FileReader::validate_gbo_table(&gbo_table, file_len, file_version)?; if gbo_table.is_empty() { return Err(Error::internal( "File did not contain any global buffers, schema expected".to_string(), )); } - let schema_start = gbo_table[0].position as usize; - let schema_size = gbo_table[0].size as usize; + let schema_range = gbo_table[0].checked_range(0, file_len)?; + let schema_start = schema_range.start as usize; + let schema_end = schema_range.end as usize; - let schema_bytes = bytes.slice(schema_start..(schema_start + schema_size)); + let schema_bytes = bytes.slice(schema_start..schema_end); let (_, schema) = FileReader::decode_schema(schema_bytes)?; - let projection = ReaderProjection::from_whole_schema(&schema, file_version); + let projection = versions::reader_projection_from_whole_schema(&schema, file_version); // Next, read the metadata for the columns // This is both the column metadata and the CMO table @@ -2561,7 +2406,7 @@ impl EncodedBatchReaderExt for EncodedBatch { let column_metadatas = FileReader::read_all_column_metadata(column_metadata_bytes, &footer)?; - let page_table = FileReader::meta_to_col_infos(&column_metadatas, file_version); + let page_table = versions::decode_column_metadata(file_version, &column_metadatas)?; Ok(Self { data: bytes, @@ -2585,21 +2430,25 @@ mod tests { }; use arrow_array::{ - RecordBatch, UInt32Array, - types::{Float64Type, Int32Type}, + DictionaryArray, Int8Array, Int32Array, ListArray, RecordBatch, RecordBatchIterator, + StringArray, UInt32Array, + types::{Float64Type, Int8Type, Int32Type}, }; + use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; use bytes::Bytes; use futures::{StreamExt, prelude::stream::TryStreamExt}; use lance_arrow::{BLOB_META_KEY, RecordBatchExt}; use lance_core::{ArrowResult, datatypes::Schema}; - use lance_datagen::{BatchCount, ByteCount, RowCount, array, gen_batch}; + use lance_datagen::{ArrayGeneratorExt, BatchCount, ByteCount, RowCount, array, gen_batch}; use lance_encoding::{ + constants::{STRUCTURAL_ENCODING_META_KEY, STRUCTURAL_ENCODING_SPARSE}, decoder::{ - DecodeBatchScheduler, DecoderPlugins, FilterExpression, ReadBatchTask, decode_batch, + DecodeBatchScheduler, DecoderPlugins, EncodedBatchLayout, FilterExpression, + PageEncoding, ReadBatchTask, decode_batch, }, - encoder::{EncodedBatch, EncodingOptions, default_encoding_strategy, encode_batch}, - version::LanceFileVersion, + encoder::{EncodedBatch, EncodingOptions, encode_batch}, + format::pb21, }; use lance_io::{stream::RecordBatchStream, utils::CachedFileSize}; use log::debug; @@ -2607,14 +2456,205 @@ mod tests { use tokio::sync::mpsc; use crate::reader::{ - EncodedBatchReaderExt, FileReader, FileReaderOptions, ProjectedFileReader, - ReaderProjection, validate_field_length, verify_uniform_lengths, + EncodedBatchReaderExt, FileReader, FileReaderOptions, ProjectedFileReader, ReaderProjection, }; use crate::testing::{FsFixture, WrittenFile, test_cache, write_lance_file}; - use crate::writer::{EncodedBatchWriteExt, FileWriter, FileWriterOptions}; + use crate::version::{ConcreteFileVersion, LanceFileVersion}; + use crate::versions; + use crate::writer::{FileWriterOptions, PAGE_BUFFER_ALIGNMENT}; use lance_encoding::decoder::DecoderConfig; - async fn create_some_file(fs: &FsFixture, version: LanceFileVersion) -> WrittenFile { + fn footer_version(bytes: &[u8]) -> (u16, u16) { + let version_start = bytes.len() - 8; + ( + u16::from_le_bytes([bytes[version_start], bytes[version_start + 1]]), + u16::from_le_bytes([bytes[version_start + 2], bytes[version_start + 3]]), + ) + } + + #[tokio::test] + async fn sparse_file_writer_reader_scan_range_and_take_roundtrip() { + let fs = FsFixture::default(); + let sparse_metadata = HashMap::from([( + STRUCTURAL_ENCODING_META_KEY.to_string(), + STRUCTURAL_ENCODING_SPARSE.to_string(), + )]); + let value_field = + Field::new("values", DataType::Int32, true).with_metadata(sparse_metadata.clone()); + let item_field = Arc::new(Field::new("item", DataType::Int32, true)); + let list_field = Field::new("items", DataType::List(item_field.clone()), true) + .with_metadata(sparse_metadata); + let arrow_schema = Arc::new(ArrowSchema::new(vec![value_field, list_field])); + let list = ListArray::try_new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 2, 2, 3, 3, 5])), + Arc::new(Int32Array::from(vec![ + Some(1), + None, + Some(3), + Some(4), + Some(5), + ])), + Some(NullBuffer::from(vec![true, false, true, true, true, true])), + ) + .unwrap(); + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![ + Some(10), + None, + Some(30), + Some(40), + None, + Some(60), + ])), + Arc::new(list), + ], + ) + .unwrap(); + let input = RecordBatchIterator::new(vec![Ok(batch.clone())], arrow_schema); + write_lance_file( + input, + &fs, + ConcreteFileVersion::V2_3, + FileWriterOptions::default(), + ) + .await; + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(file_reader.metadata().column_infos.len(), 2); + assert!( + file_reader + .metadata() + .column_infos + .iter() + .flat_map(|column| column.page_infos.iter()) + .all(|page| { + matches!( + &page.encoding, + PageEncoding::Structural(layout) + if matches!( + layout.layout, + Some(pb21::page_layout::Layout::SparseLayout(_)) + ) + ) + }) + ); + + let scan = file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(scan, vec![batch.clone()]); + + let range = file_reader + .read_stream( + lance_io::ReadBatchParams::Range(1..5), + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(range, vec![batch.slice(1, 4)]); + + let indices = UInt32Array::from(vec![0, 3, 5]); + let take = file_reader + .read_stream( + lance_io::ReadBatchParams::Indices(indices.clone()), + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(take, vec![batch.take(&indices).unwrap()]); + } + + #[tokio::test] + async fn full_int8_dictionary_v2_2_roundtrip() { + let fs = FsFixture::default(); + let values = Arc::new(StringArray::from( + (0..=i8::MAX) + .map(|value| format!("value-{value}")) + .collect::>(), + )); + let keys = Int8Array::from((0..=i8::MAX).collect::>()); + let dictionary = Arc::new(DictionaryArray::::new(keys, values)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "dictionary", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + )])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![dictionary]).unwrap(); + + write_lance_file( + RecordBatchIterator::new([Ok(batch.clone())], arrow_schema), + &fs, + ConcreteFileVersion::V2_2, + FileWriterOptions::default(), + ) + .await; + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + let actual = file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 1, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + assert_eq!(actual, vec![batch]); + } + + async fn create_some_file(fs: &FsFixture, version: ConcreteFileVersion) -> WrittenFile { let location_type = DataType::Struct(Fields::from(vec![ Field::new("x", DataType::Float64, true), Field::new("y", DataType::Float64, true), @@ -2626,36 +2666,76 @@ mod tests { .col("location", array::rand_type(&location_type)) .col("categories", array::rand_type(&categories_type)) .col("binary", array::rand_type(&DataType::Binary)); - if version <= LanceFileVersion::V2_0 { + if version == ConcreteFileVersion::V2_0 { reader = reader.col("large_bin", array::rand_type(&DataType::LargeBinary)); } let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100)); + write_lance_file(reader, fs, version, FileWriterOptions::default()).await + } + + async fn create_wide_direct_file(fs: &FsFixture, num_columns: usize) -> WrittenFile { + let mut reader = gen_batch(); + for column_idx in 0..num_columns { + reader = reader.col(format!("c{column_idx}"), array::step::()); + } + let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100)); + write_lance_file( reader, fs, - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), ) .await } - async fn create_wide_direct_file(fs: &FsFixture, num_columns: usize) -> WrittenFile { + async fn create_wide_fixed_size_list_file(fs: &FsFixture, num_columns: usize) -> WrittenFile { + let data_type = + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4); let mut reader = gen_batch(); for column_idx in 0..num_columns { - reader = reader.col(format!("c{column_idx}"), array::step::()); + reader = reader.col( + format!("c{column_idx}"), + array::rand_type(&data_type).with_random_nulls(0.1), + ); } - let reader = reader.into_reader_rows(RowCount::from(1000), BatchCount::from(100)); + let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4)); write_lance_file( reader, fs, - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .await + } + + async fn create_wide_structural_file(fs: &FsFixture, num_groups: usize) -> WrittenFile { + let struct_type = DataType::Struct(Fields::from(vec![ + Field::new("x", DataType::Int32, true), + Field::new("y", DataType::Int32, true), + ])); + let list_type = DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + let mut reader = gen_batch(); + for group_idx in 0..num_groups { + reader = reader + .col( + format!("s{group_idx}"), + array::rand_type(&struct_type).with_random_nulls(0.5), + ) + .col( + format!("l{group_idx}"), + array::rand_type(&list_type).with_random_nulls(0.5), + ); + } + let reader = reader.into_reader_rows(RowCount::from(64), BatchCount::from(4)); + + write_lance_file( + reader, + fs, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), ) .await } @@ -2712,23 +2792,333 @@ mod tests { assert_eq!(remaining, 0); } - async fn collect_read_tasks( - tasks: Pin + Send>>, - readahead: usize, - ) -> Vec { - tasks - .map(|task| task.task) - .buffered(readahead) - .try_collect::>() - .await - .unwrap() + async fn collect_read_tasks( + tasks: Pin + Send>>, + readahead: usize, + ) -> Vec { + tasks + .map(|task| task.task) + .buffered(readahead) + .try_collect::>() + .await + .unwrap() + } + + /// Writes `batch` to a fresh file, overwrites `patch` bytes at `patch_offset` + /// into the single occurrence of `pattern`, and reads the file back with the + /// default reader configuration. + async fn read_file_with_mutated_bytes( + version: LanceFileVersion, + batch: RecordBatch, + pattern: &[u8], + patch_offset: usize, + patch: &[u8], + ) -> lance_core::Result> { + let fs = FsFixture::default(); + let schema = batch.schema(); + write_lance_file( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &fs, + version.resolve(), + FileWriterOptions::default(), + ) + .await; + + let mut bytes = fs + .object_store + .read_one_all(&fs.tmp_path) + .await + .unwrap() + .to_vec(); + let matches = bytes + .windows(pattern.len()) + .enumerate() + .filter_map(|(position, window)| (window == pattern).then_some(position)) + .collect::>(); + assert_eq!( + matches.len(), + 1, + "expected the byte pattern to appear exactly once in the file" + ); + let patch_start = matches[0] + patch_offset; + bytes[patch_start..patch_start + patch.len()].copy_from_slice(patch); + fs.object_store.put(&fs.tmp_path, &bytes).await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 16, + FilterExpression::no_filter(), + ) + .await? + .try_collect::>() + .await + } + + #[tokio::test] + async fn test_reader_rejects_excess_miniblock_row_counts() { + let batch = + arrow_array::record_batch!(("id", UInt64, (0..2048_u64).collect::>())).unwrap(); + let fs = FsFixture::default(); + write_lance_file( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + &fs, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .await; + + let mut bytes = fs + .object_store + .read_one_all(&fs.tmp_path) + .await + .unwrap() + .to_vec(); + // V2.1 places this column's first mini-block metadata word at byte zero. + // The issue's mutation makes its non-final item count exceed the page total. + bytes[0] ^= 0xf7; + fs.object_store.put(&fs.tmp_path, &bytes).await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + let result = file_reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + 1024, + 16, + FilterExpression::no_filter(), + ) + .await; + let error = match result { + Ok(stream) => stream + .try_collect::>() + .await + .expect_err("excess mini-block row counts must fail the read"), + Err(error) => error, + }; + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("exceeding items_in_page"), + "unexpected message: {error}" + ); + } + + /// A corrupt file whose variable-width offsets point outside the value bytes + /// must fail with a typed error under the default reader configuration + /// (`validate_on_decode` disabled) instead of materializing values outside + /// the data buffer. + /// + /// Uses a dictionary-encoded string column because its values page stores + /// the offsets verbatim, so flipping the tail offset in the file reaches the + /// Arrow conversion boundary without being rejected by an intermediate + /// decompressor. + #[rstest] + #[tokio::test] + async fn test_default_reader_rejects_out_of_bounds_variable_width_offsets( + #[values(LanceFileVersion::V2_1, LanceFileVersion::V2_2, LanceFileVersion::V2_3)] + version: LanceFileVersion, + ) { + use arrow_array::{Array, DictionaryArray, Int32Array, StringArray}; + + let values = StringArray::from(vec!["alpha", "beta", "gamma"]); + let indices = Int32Array::from((0..300).map(|i| i % 3).collect::>()); + let dictionary = DictionaryArray::new(indices, Arc::new(values)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "category", + dictionary.data_type().clone(), + false, + )])); + let batch = RecordBatch::try_new(arrow_schema, vec![Arc::new(dictionary)]).unwrap(); + + // The dictionary values page stores the value offsets as plain + // little-endian i32s ending with [5, 9, 14] (2.1 also stores the leading + // zero, 2.2+ omits it). If a future encoding change stops storing these + // offsets verbatim this lookup fails loudly and the test needs a new + // byte pattern. The patch rewrites the tail offset so it points far + // beyond the value bytes. + let offsets_tail_pattern = [5_i32, 9, 14] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let error = read_file_with_mutated_bytes( + version, + batch, + &offsets_tail_pattern, + 8, + &100_000_i32.to_le_bytes(), + ) + .await + .expect_err("out-of-bounds offsets must fail the read"); + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("out of bounds"), + "unexpected message: {error}" + ); + } + + /// Storage dictionaries expand their values through `DataBlockBuilder` + /// before the final Arrow layout validation. Corrupt dictionary offsets + /// must therefore fail at the append boundary instead of reaching a slice + /// operation with a decreasing range. + #[rstest] + #[tokio::test] + async fn test_default_reader_rejects_non_monotonic_storage_dictionary_offsets( + #[values(LanceFileVersion::V2_1, LanceFileVersion::V2_2, LanceFileVersion::V2_3)] + version: LanceFileVersion, + ) { + use arrow_array::StringArray; + + let metadata = HashMap::from([ + ( + "lance-encoding:dict-size-ratio".to_string(), + "0.99".to_string(), + ), + ( + "lance-encoding:dict-values-compression".to_string(), + "none".to_string(), + ), + ]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("category", DataType::Utf8, false).with_metadata(metadata), + ])); + let values = (0..300) + .map(|index| match index % 3 { + 0 => "alpha", + 1 => "beta", + _ => "gamma", + }) + .collect::>(); + let batch = + RecordBatch::try_new(arrow_schema, vec![Arc::new(StringArray::from(values))]).unwrap(); + + let offsets_tail_pattern = [5_i32, 9, 14] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let error = read_file_with_mutated_bytes( + version, + batch, + &offsets_tail_pattern, + 4, + &2_i32.to_le_bytes(), + ) + .await + .expect_err("non-monotonic dictionary offsets must fail the read"); + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("decreases"), + "unexpected message: {error}" + ); + } + + /// Same contract as the test above, but for a plain (non-dictionary) string + /// column: the mini-block chunk stores chunk-relative value offsets that are + /// used to slice the chunk, so a corrupt tail offset must surface as a typed + /// error from the chunk decompressor instead of a panic in the decode task. + #[rstest] + #[tokio::test] + async fn test_default_reader_rejects_out_of_bounds_miniblock_offsets( + #[values(LanceFileVersion::V2_1, LanceFileVersion::V2_2, LanceFileVersion::V2_3)] + version: LanceFileVersion, + ) { + use arrow_array::StringArray; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new( + "strings", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new( + arrow_schema, + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "gamma"]))], + ) + .unwrap(); + + // For ["alpha", "beta", "gamma"] the chunk stores LE i32 offsets + // [16, 21, 25, 30] (chunk-relative: a 16-byte offsets region precedes + // the value bytes). The patch rewrites the tail offset to point far + // past the chunk. + let chunk_offsets_pattern = [16_i32, 21, 25, 30] + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect::>(); + let error = read_file_with_mutated_bytes( + version, + batch, + &chunk_offsets_pattern, + 12, + &100_000_i32.to_le_bytes(), + ) + .await + .expect_err("an out-of-bounds chunk offset must fail the read"); + assert!( + matches!(error, lance_core::Error::CorruptFile { .. }), + "expected CorruptFile, got: {error}" + ); + assert!( + error.to_string().contains("out of bounds"), + "unexpected message: {error}" + ); } #[tokio::test] async fn test_round_trip() { let fs = FsFixture::default(); - let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await; + let WrittenFile { data, .. } = create_some_file(&fs, ConcreteFileVersion::V2_0).await; + + let file_size = fs.object_store.size(&fs.tmp_path).await.unwrap() as usize; + let footer = fs + .object_store + .open(&fs.tmp_path) + .await + .unwrap() + .get_range(file_size - 8..file_size) + .await + .unwrap(); + assert_eq!(footer_version(&footer), (0, 3)); + assert_eq!( + crate::determine_file_version(&fs.object_store, &fs.tmp_path, Some(file_size)) + .await + .unwrap(), + ConcreteFileVersion::V2_0 + ); for read_size in [32, 1024, 1024 * 1024] { let file_scheduler = fs @@ -2746,6 +3136,13 @@ mod tests { .await .unwrap(); + assert_eq!( + ( + file_reader.metadata().major_version, + file_reader.metadata().minor_version + ), + (0, 3) + ); let schema = file_reader.schema(); assert_eq!(schema.metadata.get("foo").unwrap(), "bar"); @@ -2767,7 +3164,7 @@ mod tests { #[test_log::test(tokio::test)] async fn test_encoded_batch_round_trip( // TODO: Add V2_1 (currently fails) - #[values(LanceFileVersion::V2_0)] version: LanceFileVersion, + #[values(ConcreteFileVersion::V2_0)] version: ConcreteFileVersion, ) { let data = gen_batch() .col("x", array::rand::()) @@ -2782,10 +3179,9 @@ mod tests { max_page_bytes: 32 * 1024 * 1024, keep_original_array: true, buffer_alignment: 64, - version, }; - let encoding_strategy = default_encoding_strategy(version); + let encoding_strategy = crate::versions::v2_0::encoding_strategy(); let encoded_batch = encode_batch( &data, @@ -2797,7 +3193,8 @@ mod tests { .unwrap(); // Test self described - let bytes = encoded_batch.try_to_self_described_lance(version).unwrap(); + let bytes = versions::encode_self_described_batch(version, &encoded_batch).unwrap(); + assert_eq!(footer_version(&bytes), (2, 0)); let decoded_batch = EncodedBatch::try_from_self_described_lance(bytes).unwrap(); @@ -2806,7 +3203,7 @@ mod tests { &FilterExpression::no_filter(), Arc::::default(), false, - version, + EncodedBatchLayout::Array, None, ) .await @@ -2815,16 +3212,16 @@ mod tests { assert_eq!(data, decoded); // Test mini - let bytes = encoded_batch.try_to_mini_lance(version).unwrap(); + let bytes = versions::encode_mini_batch(version, &encoded_batch).unwrap(); + assert_eq!(footer_version(&bytes), (2, 0)); let decoded_batch = - EncodedBatch::try_from_mini_lance(bytes, lance_schema.as_ref(), LanceFileVersion::V2_0) - .unwrap(); + EncodedBatch::try_from_mini_lance(bytes, lance_schema.as_ref()).unwrap(); let decoded = decode_batch( &decoded_batch, &FilterExpression::no_filter(), Arc::::default(), false, - version, + EncodedBatchLayout::Array, None, ) .await @@ -2836,8 +3233,12 @@ mod tests { #[rstest] #[test_log::test(tokio::test)] async fn test_projection( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)] - version: LanceFileVersion, + #[values( + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2 + )] + version: ConcreteFileVersion, ) { let fs = FsFixture::default(); @@ -2884,15 +3285,15 @@ mod tests { let projected_schema = written_file.schema.project(&columns).unwrap(); let projection = if use_field_ids { - ReaderProjection::from_field_ids( - file_reader.metadata.version(), + versions::reader_projection_from_field_ids( + file_reader.metadata().version(), &projected_schema, &field_id_mapping, ) .unwrap() } else { - ReaderProjection::from_column_names( - file_reader.metadata.version(), + versions::reader_projection_from_column_names( + file_reader.metadata().version(), &written_file.schema, &columns, ) @@ -3014,8 +3415,8 @@ mod tests { .open_file(&fs.tmp_path, &CachedFileSize::unknown()) .await .unwrap(); - let projection = ReaderProjection::from_column_names( - LanceFileVersion::V2_1, + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, &written_file.schema, &["c10"], ) @@ -3073,8 +3474,8 @@ mod tests { let fs = FsFixture::default(); let written_file = create_wide_direct_file(&fs, 512).await; - let projection = ReaderProjection::from_column_names( - LanceFileVersion::V2_1, + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, &written_file.schema, &["c0"], ) @@ -3157,6 +3558,192 @@ mod tests { ); } + async fn assert_lazy_projection_matches_eager_and_reads_metadata_subset( + fs: &FsFixture, + projection: ReaderProjection, + shape: &str, + ) -> Vec { + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let eager_reader = FileReader::try_open( + file_scheduler.clone(), + None, + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + let expected = eager_reader + .read_stream_projected( + lance_io::ReadBatchParams::RangeFull, + 127, + 16, + projection.clone(), + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + let cache = test_cache(); + let lazy_reader = ProjectedFileReader::try_open( + file_scheduler, + Some(projection.clone()), + Arc::::default(), + &cache, + FileReaderOptions::default(), + ) + .await + .unwrap(); + let metadata_index = lazy_reader.metadata_index().unwrap(); + let requested_metadata_bytes = projection + .column_indices + .iter() + .map(|column_index| metadata_index.column_metadata_offsets[*column_index as usize].1) + .sum::(); + let total_metadata_bytes = metadata_index + .column_metadata_offsets + .iter() + .map(|(_, length)| *length) + .sum::(); + assert!(total_metadata_bytes > requested_metadata_bytes * 8); + + fs.object_store.io_stats_incremental(); + let tasks = lazy_reader + .read_tasks( + lance_io::ReadBatchParams::Range(0..0), + 127, + None, + FilterExpression::no_filter(), + ) + .await + .unwrap(); + assert!(collect_read_tasks(tasks, 1).await.is_empty()); + let metadata_stats = fs.object_store.io_stats_incremental(); + assert!( + metadata_stats.read_bytes < total_metadata_bytes / 2, + "lazy {shape} read fetched too much metadata: read {} bytes, requested column metadata is {} bytes, total column metadata is {} bytes", + metadata_stats.read_bytes, + requested_metadata_bytes, + total_metadata_bytes + ); + + let tasks = lazy_reader + .read_tasks( + lance_io::ReadBatchParams::RangeFull, + 127, + None, + FilterExpression::no_filter(), + ) + .await + .unwrap(); + let actual = collect_read_tasks(tasks, 16).await; + assert_eq!(expected, actual); + actual + } + + #[tokio::test] + async fn test_lazy_reader_fixed_size_list_projection_matches_eager_reader() { + let fs = FsFixture::default(); + let written_file = create_wide_fixed_size_list_file(&fs, 512).await; + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &written_file.schema, + &["c17", "c509"], + ) + .unwrap(); + assert!(projection.prefers_indexed_metadata(512)); + assert_lazy_projection_matches_eager_and_reads_metadata_subset( + &fs, + projection, + "fixed-size-list", + ) + .await; + } + + #[tokio::test] + async fn test_v2_0_rejects_indexed_metadata_reader() { + let fs = FsFixture::default(); + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_0).await; + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_0, + &written_file.schema, + &["score"], + ) + .unwrap(); + assert!(projection.prefers_indexed_metadata(100)); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let err = ProjectedFileReader::try_open( + file_scheduler, + Some(projection), + Arc::::default(), + &test_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap_err(); + assert!( + matches!(err, lance_core::Error::NotSupported { .. }), + "expected V2.0 indexed metadata open to fail, got {err:?}" + ); + } + + #[tokio::test] + async fn test_lazy_reader_nested_projection_compacts_physical_columns() { + let fs = FsFixture::default(); + let written_file = create_wide_structural_file(&fs, 128).await; + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &written_file.schema, + &["s97.y", "l4", "s3"], + ) + .unwrap(); + + assert_eq!( + projection + .schema + .fields + .iter() + .map(|field| field.name.as_str()) + .collect::>(), + vec!["s97", "l4", "s3"] + ); + assert_eq!(projection.schema.fields[0].children.len(), 1); + assert_eq!(projection.schema.fields[0].children[0].name, "y"); + assert_eq!(projection.schema.fields[2].children.len(), 2); + assert_eq!(projection.column_indices.len(), 4); + assert!( + projection + .column_indices + .windows(2) + .any(|indices| indices[0] > indices[1]), + "the projection must reorder physical columns to exercise compact remapping" + ); + assert!(projection.prefers_indexed_metadata(128 * 4)); + let actual = assert_lazy_projection_matches_eager_and_reads_metadata_subset( + &fs, projection, "nested", + ) + .await; + assert!( + actual + .iter() + .flat_map(|batch| batch.columns()) + .any(|column| column.null_count() > 0), + "the structural projection must exercise nullable arrays" + ); + } + #[rstest] #[case::before_metadata_region(90, 5)] #[case::after_metadata_region(190, 20)] @@ -3185,21 +3772,22 @@ mod tests { ); } + #[rstest] + #[case::blob(BLOB_META_KEY)] + #[case::packed_struct("lance-encoding:packed")] #[tokio::test] - async fn test_lazy_reader_rejects_unsupported_projection() { + async fn test_lazy_reader_rejects_opaque_projection(#[case] metadata_key: &str) { let fs = FsFixture::default(); - let written_file = create_some_file(&fs, LanceFileVersion::V2_1).await; + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_1).await; - let projection = ReaderProjection::from_column_names( - LanceFileVersion::V2_1, + let ordinary_projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, &written_file.schema, - &["location"], + &["location.x"], ) .unwrap(); - assert!(!ProjectedFileReader::supports_projection( - &projection, - LanceFileVersion::V2_1 - )); + assert_eq!(ordinary_projection.schema.fields[0].children.len(), 1); + assert!(ordinary_projection.prefers_indexed_metadata(100)); let file_scheduler = fs .scheduler @@ -3220,6 +3808,12 @@ mod tests { "expected InvalidInput, got {err:?}" ); + let mut projection = ordinary_projection; + Arc::make_mut(&mut projection.schema).fields[0] + .metadata + .insert(metadata_key.to_string(), "true".to_string()); + assert!(!projection.prefers_indexed_metadata(100)); + let err = ProjectedFileReader::try_open( file_scheduler, Some(projection), @@ -3231,11 +3825,11 @@ mod tests { .unwrap_err(); assert!( matches!(err, lance_core::Error::NotSupported { .. }), - "expected NotSupported, got {err:?}" + "expected NotSupported for {metadata_key}, got {err:?}" ); } - // The projection-length validation lives in `FileReadCore`, shared by the + // The projection-length validation lives in `DecodeEngine`, shared by the // eager and the lazy (indexed) metadata providers. The indexed provider loads // only the projected columns and renumbers them 0..N, so this checks that the // renumbered `column_infos`/`column_indices` still line up for the length @@ -3253,14 +3847,10 @@ mod tests { let lance_schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - let mut writer = FileWriter::try_new( + let mut writer = versions::v2_1::create_writer( fs.object_store.create(&fs.tmp_path).await.unwrap(), lance_schema.clone(), - options, + FileWriterOptions::default(), ) .unwrap(); // "a" has 5 rows, "c" has 1 -- an unequal-length file. @@ -3281,9 +3871,12 @@ mod tests { .unwrap(); let cache = test_cache(); let open_indexed = |names: &[&str]| { - let projection = - ReaderProjection::from_column_names(LanceFileVersion::V2_1, &lance_schema, names) - .unwrap(); + let projection = versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &lance_schema, + names, + ) + .unwrap(); ProjectedFileReader::try_open( file_scheduler.clone(), Some(projection), @@ -3346,7 +3939,7 @@ mod tests { async fn test_compressing_buffer() { let fs = FsFixture::default(); - let written_file = create_some_file(&fs, LanceFileVersion::V2_0).await; + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let file_scheduler = fs .scheduler .open_file(&fs.tmp_path, &CachedFileSize::unknown()) @@ -3401,7 +3994,7 @@ mod tests { #[tokio::test] async fn test_read_all() { let fs = FsFixture::default(); - let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await; + let WrittenFile { data, .. } = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let total_rows = data.iter().map(|batch| batch.num_rows()).sum::(); let file_scheduler = fs @@ -3438,8 +4031,12 @@ mod tests { #[rstest] #[tokio::test] async fn test_blocking_take( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1, LanceFileVersion::V2_2)] - version: LanceFileVersion, + #[values( + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2 + )] + version: ConcreteFileVersion, ) { let fs = FsFixture::default(); let WrittenFile { data, schema, .. } = create_some_file(&fs, version).await; @@ -3452,7 +4049,10 @@ mod tests { .unwrap(); let file_reader = FileReader::try_open( file_scheduler.clone(), - Some(ReaderProjection::from_column_names(version, &schema, &["score"]).unwrap()), + Some( + versions::reader_projection_from_column_names(version, &schema, &["score"]) + .unwrap(), + ), Arc::::default(), &test_cache(), FileReaderOptions::default(), @@ -3483,7 +4083,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_drop_in_progress() { let fs = FsFixture::default(); - let WrittenFile { data, .. } = create_some_file(&fs, LanceFileVersion::V2_0).await; + let WrittenFile { data, .. } = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let total_rows = data.iter().map(|batch| batch.num_rows()).sum::(); let file_scheduler = fs @@ -3532,7 +4132,7 @@ mod tests { // if the stream was dropped before it finished. let fs = FsFixture::default(); - let written_file = create_some_file(&fs, LanceFileVersion::V2_0).await; + let written_file = create_some_file(&fs, ConcreteFileVersion::V2_0).await; let total_rows = written_file .data .iter() @@ -3554,11 +4154,11 @@ mod tests { .await .unwrap(); - let projection = - ReaderProjection::from_whole_schema(&written_file.schema, LanceFileVersion::V2_0); - let column_infos = file_reader - .collect_columns_from_projection(&projection) - .unwrap(); + let projection = versions::reader_projection_from_whole_schema( + &written_file.schema, + ConcreteFileVersion::V2_0, + ); + let column_infos = file_reader.metadata().column_infos.clone(); let mut decode_scheduler = DecodeBatchScheduler::try_new( &projection.schema, &projection.column_indices, @@ -3566,7 +4166,7 @@ mod tests { &vec![], total_rows as u64, Arc::::default(), - file_reader.core.scheduler.clone(), + file_reader.scheduler(), test_cache(), &FilterExpression::no_filter(), &DecoderConfig::default(), @@ -3586,14 +4186,14 @@ mod tests { range, &FilterExpression::no_filter(), tx, - file_reader.core.scheduler.clone(), + file_reader.scheduler(), ) } #[tokio::test] async fn test_read_empty_range() { let fs = FsFixture::default(); - create_some_file(&fs, LanceFileVersion::V2_0).await; + create_some_file(&fs, ConcreteFileVersion::V2_0).await; let file_scheduler = fs .scheduler @@ -3651,7 +4251,7 @@ mod tests { )])) .unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = versions::v2_1::create_writer( fs.object_store.create(&fs.tmp_path).await.unwrap(), lance_schema, FileWriterOptions::default(), @@ -3664,6 +4264,96 @@ mod tests { file_writer.finish().await.unwrap(); } + #[derive(Clone, Copy, Debug)] + enum MetadataReadPath { + Full, + Indexed, + } + + #[derive(Clone, Copy, Debug)] + enum InvalidGboDescriptor { + Unaligned, + PastEof, + Overflowing, + } + + #[rstest] + #[case::full_unaligned(MetadataReadPath::Full, InvalidGboDescriptor::Unaligned, "not aligned")] + #[case::full_past_eof(MetadataReadPath::Full, InvalidGboDescriptor::PastEof, "outside file")] + #[case::full_overflowing( + MetadataReadPath::Full, + InvalidGboDescriptor::Overflowing, + "overflows" + )] + #[case::indexed_unaligned( + MetadataReadPath::Indexed, + InvalidGboDescriptor::Unaligned, + "not aligned" + )] + #[case::indexed_past_eof( + MetadataReadPath::Indexed, + InvalidGboDescriptor::PastEof, + "outside file" + )] + #[case::indexed_overflowing( + MetadataReadPath::Indexed, + InvalidGboDescriptor::Overflowing, + "overflows" + )] + #[tokio::test] + async fn test_metadata_rejects_invalid_gbo_descriptor( + #[case] read_path: MetadataReadPath, + #[case] invalid_descriptor: InvalidGboDescriptor, + #[case] expected_message: &str, + ) { + let fs = FsFixture::default(); + write_file_with_global_buffer(&fs, Bytes::from_static(b"hello")).await; + + let mut file_bytes = fs + .object_store + .read_one_all(&fs.tmp_path) + .await + .unwrap() + .to_vec(); + let file_len = file_bytes.len() as u64; + let footer = FileReader::decode_footer(&Bytes::copy_from_slice(&file_bytes)).unwrap(); + let gbo_table_start = usize::try_from(footer.global_buff_offsets_start).unwrap(); + let alignment = PAGE_BUFFER_ALIGNMENT as u64; + let (position, size) = match invalid_descriptor { + InvalidGboDescriptor::Unaligned => (1, 0), + InvalidGboDescriptor::PastEof => (((file_len + alignment) / alignment) * alignment, 0), + InvalidGboDescriptor::Overflowing => (u64::MAX - (u64::MAX % alignment), alignment), + }; + file_bytes[gbo_table_start..gbo_table_start + 8].copy_from_slice(&position.to_le_bytes()); + file_bytes[gbo_table_start + 8..gbo_table_start + 16].copy_from_slice(&size.to_le_bytes()); + fs.object_store + .put(&fs.tmp_path, &file_bytes) + .await + .unwrap(); + + let scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let error = match read_path { + MetadataReadPath::Full => FileReader::read_all_metadata(&scheduler).await.map(|_| ()), + MetadataReadPath::Indexed => FileReader::read_metadata_index(&scheduler) + .await + .map(|_| ()), + } + .expect_err("invalid GBO descriptor must fail before metadata I/O"); + + assert!( + matches!(error, lance_core::Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + assert!( + error.to_string().contains(expected_message), + "unexpected error: {error}" + ); + } + /// A global buffer that fits inside the tail region captured at open is served /// from memory with no additional I/O. A buffer larger than that window cannot /// fit and falls back to a dedicated read. Both must round-trip correctly. @@ -3720,7 +4410,7 @@ mod tests { #[tokio::test] async fn test_read_global_buffer_no_user_buffers() { let fs = FsFixture::default(); - create_some_file(&fs, LanceFileVersion::V2_1).await; + create_some_file(&fs, ConcreteFileVersion::V2_1).await; let file_scheduler = fs .scheduler @@ -3749,12 +4439,12 @@ mod tests { #[tokio::test] async fn test_deep_size_of_includes_column_metadata( #[values( - LanceFileVersion::V2_0, - LanceFileVersion::V2_1, - LanceFileVersion::V2_2, - LanceFileVersion::V2_3 + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, + ConcreteFileVersion::V2_3 )] - version: LanceFileVersion, + version: ConcreteFileVersion, ) { // Regression test: CachedFileMetadata::deep_size_of must account for // column_metadatas and column_infos, otherwise the moka cache weigher @@ -3850,21 +4540,11 @@ mod tests { let run = |dt: DataType, indices: &[u32], lengths: Vec| -> lance_core::Result { let arrow = ArrowSchema::new(vec![Field::new("s", dt, true)]); let schema = Schema::try_from(&arrow).unwrap(); - let column_len = |c: usize| Ok(lengths[c]); - let mut cursor = 0usize; - let mut field_lengths = Vec::new(); - for field in &schema.fields { - let rows = validate_field_length( - field, - is_structural, - true, - indices, - &mut cursor, - &column_len, - )?; - field_lengths.push((field.name.as_str(), rows)); + if is_structural { + versions::v2_1::test_projection_length(&schema, indices, &lengths) + } else { + versions::v2_0::test_projection_length(&schema, indices, &lengths) } - verify_uniform_lengths(&field_lengths) }; let struct_ty = || { @@ -3904,24 +4584,14 @@ mod tests { schema: Arc::new(schema), column_indices: vec![0], }; - let column_len = |column: usize| { - assert_eq!(column, 0); - Ok(3) - }; - let mut cursor = 0usize; - - let rows = validate_field_length( - &projection.schema.fields[0], - false, - true, + let rows = versions::v2_0::test_projection_length( + &projection.schema, &projection.column_indices, - &mut cursor, - &column_len, + &[3], ) .unwrap(); assert_eq!(rows, 3); - assert_eq!(cursor, 1); } #[test] @@ -3933,16 +4603,11 @@ mod tests { -> lance_core::Result { let arrow = ArrowSchema::new(vec![Field::new("f", dt, true)]); let schema = Schema::try_from(&arrow).unwrap(); - let column_len = |c: usize| Ok(lengths[c]); - let mut cursor = 0usize; - validate_field_length( - &schema.fields[0], - is_structural, - true, - indices, - &mut cursor, - &column_len, - ) + if is_structural { + versions::v2_1::test_projection_length(&schema, indices, &lengths) + } else { + versions::v2_0::test_projection_length(&schema, indices, &lengths) + } }; // A list's items have a different cardinality than its rows; that gap diff --git a/rust/lance-file/src/reader/structural.rs b/rust/lance-file/src/reader/structural.rs new file mode 100644 index 00000000000..18bda6eb933 --- /dev/null +++ b/rust/lance-file/src/reader/structural.rs @@ -0,0 +1,445 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use lance_core::{ + Error, Result, + cache::LanceCache, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + EncodingsIo, + decoder::{ColumnInfo, PageEncoding, PageInfo}, + format::{pb, pb21}, +}; +use prost::{Message, Name}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, FileMetadataIndex, FileMetadataProvider, FileReader, PreparedProjection, + ReadProjection, ReaderProjection, normalized_column_num_rows, verify_uniform_lengths, + }, + writer::PAGE_BUFFER_ALIGNMENT, +}; + +fn fetch_encoding(encoding: &pbfile::Encoding) -> Result { + match &encoding.location { + Some(pbfile::encoding::Location::Indirect(_)) => Err(Error::invalid_input_source( + "Indirect file encodings are not supported".into(), + )), + Some(pbfile::encoding::Location::Direct(encoding)) => { + let envelope = prost_types::Any::decode(Bytes::from(encoding.encoding.clone())) + .map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding envelope: {error}", M::NAME).into(), + ) + })?; + envelope.to_msg::().map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding: {error}", M::NAME).into(), + ) + }) + } + Some(pbfile::encoding::Location::None(_)) => Err(Error::invalid_input_source( + format!("Missing {} encoding description", M::NAME).into(), + )), + None => Err(Error::invalid_input_source( + format!("Missing {} encoding location", M::NAME).into(), + )), + } +} + +/// Decode structural page syntax before an exact reader validates its grammar. +pub fn decode_page_layout( + column_index: u32, + page_index: usize, + page: &pbfile::column_metadata::Page, +) -> Result { + fetch_encoding(page.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!( + "Column {} page {} is missing its encoding", + column_index, page_index + ) + .into(), + ) + })?) +} + +/// Build normalized page metadata after exact grammar validation. +pub fn build_page_info( + column_index: u32, + page_index: usize, + page: &pbfile::column_metadata::Page, + page_layout: pb21::PageLayout, +) -> Result { + if page.buffer_offsets.len() != page.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} page {} has {} buffer offsets but {} buffer sizes", + column_index, + page_index, + page.buffer_offsets.len(), + page.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + page.buffer_offsets + .iter() + .zip(&page.buffer_sizes) + .map(|(offset, size)| { + if offset % PAGE_BUFFER_ALIGNMENT as u64 != 0 { + return Err(Error::invalid_input_source( + format!( + "Column {} page {} buffer offset {} is not aligned to {} bytes", + column_index, page_index, offset, PAGE_BUFFER_ALIGNMENT + ) + .into(), + )); + } + Ok((*offset, *size)) + }) + .collect::>>()?, + ); + Ok(PageInfo { + buffer_offsets_and_sizes, + encoding: PageEncoding::Structural(page_layout), + num_rows: page.length, + priority: page.priority, + }) +} + +/// Finish normalized column metadata after all pages have been validated. +pub fn build_column_info( + column_index: u32, + metadata: &pbfile::ColumnMetadata, + page_infos: Vec, +) -> Result> { + if metadata.buffer_offsets.len() != metadata.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} has {} buffer offsets but {} buffer sizes", + column_index, + metadata.buffer_offsets.len(), + metadata.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + metadata + .buffer_offsets + .iter() + .zip(&metadata.buffer_sizes) + .map(|(offset, size)| (*offset, *size)) + .collect::>(), + ); + let encoding: pb::ColumnEncoding = + fetch_encoding(metadata.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Column {} is missing its encoding", column_index).into(), + ) + })?)?; + Ok(Arc::new(ColumnInfo { + index: column_index, + page_infos: Arc::from(page_infos), + buffer_offsets_and_sizes, + encoding, + })) +} + +/// Validate the structural global-buffer alignment contract. +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + for (buffer_index, buffer) in buffers.iter().enumerate() { + if buffer.position % PAGE_BUFFER_ALIGNMENT as u64 != 0 { + return Err(Error::invalid_input_source( + format!( + "Global buffer {} position {} is not aligned to {} bytes", + buffer_index, buffer.position, PAGE_BUFFER_ALIGNMENT + ) + .into(), + )); + } + } + Ok(()) +} + +fn field_column_shape(field: &Field) -> (bool, bool) { + if field.is_blob() || field.is_packed_struct() { + return (true, false); + } + (field.children.is_empty(), !field.children.is_empty()) +} + +/// Count physical columns in the leaf-column layout used by v2.1+. +pub fn physical_column_count(field: &Field) -> usize { + if field.children.is_empty() || field.is_blob() || field.is_packed_struct() { + 1 + } else { + field.children.iter().map(physical_column_count).sum() + } +} + +fn append_physical_fields( + fields: &[Field], + field_ids: &mut Vec, + column_indices: &mut Vec, + next_column: &mut i32, +) { + for field in fields { + if field.children.is_empty() || field.is_blob() || field.is_packed_struct() { + field_ids.push(field.id); + column_indices.push(*next_column); + *next_column += 1; + } else { + append_physical_fields(&field.children, field_ids, column_indices, next_column); + } + } +} + +/// Build persisted field-to-column entries for the v2.1+ leaf layout. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + let mut field_ids = Vec::new(); + let mut column_indices = Vec::new(); + append_physical_fields(&schema.fields, &mut field_ids, &mut column_indices, &mut 0); + (field_ids, column_indices) +} + +/// Build the field-id lookup for the v2.1+ leaf layout. +pub fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + let (field_ids, column_indices) = data_file_columns(schema); + field_ids + .into_iter() + .zip(column_indices) + .filter_map(|(field_id, column_index)| { + (column_index >= 0).then_some((field_id as u32, column_index as u32)) + }) + .collect() +} + +fn append_field_ids( + fields: &[Field], + field_id_to_column_index: &BTreeMap, + column_indices: &mut Vec, +) { + for field in fields { + let (contributes, recurse) = field_column_shape(field); + if contributes + && let Some(column_index) = field_id_to_column_index.get(&(field.id as u32)).copied() + { + column_indices.push(column_index); + } + if recurse { + append_field_ids(&field.children, field_id_to_column_index, column_indices); + } + } +} + +/// Build the leaf-column projection selected by a v2.1+ exact reader. +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + let mut column_indices = Vec::new(); + append_field_ids( + &schema.fields, + field_id_to_column_index, + &mut column_indices, + ); + ReaderProjection { + schema: Arc::new(schema.clone()), + column_indices, + } +} + +/// Project names using a caller-selected leaf-column mapping. +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], + field_id_to_column_index: &BTreeMap, +) -> Result { + let projected = schema.project(column_names)?; + Ok(projection_from_field_ids( + &projected, + field_id_to_column_index, + )) +} + +fn children_share_parent_length(field: &Field) -> bool { + field.logical_type.is_struct() +} + +fn validate_field_length Result>( + field: &Field, + comparable: bool, + column_indices: &[u32], + cursor: &mut usize, + column_len: &F, +) -> Result { + let (contributes, recurse) = field_column_shape(field); + let mut field_rows = None; + if contributes { + let column = *column_indices.get(*cursor).ok_or_else(|| { + Error::invalid_input(format!( + "projection supplied fewer column indices than its fields require (ran out at field '{}')", + field.name + )) + })?; + *cursor += 1; + field_rows = Some(column_len(column as usize)?); + } + if recurse { + let enforce_children = comparable && children_share_parent_length(field); + for child in &field.children { + let child_rows = + validate_field_length(child, enforce_children, column_indices, cursor, column_len)?; + let expected = *field_rows.get_or_insert(child_rows); + if enforce_children && child_rows != expected { + return Err(Error::invalid_input(format!( + "cannot read field '{}': its children have differing lengths (child '{}' has {} rows, but the field has {}); a struct's children must all have the same length", + field.name, child.name, child_rows, expected + ))); + } + } + } + field_rows.ok_or_else(|| { + Error::invalid_input(format!( + "projected field '{}' maps to no columns", + field.name + )) + }) +} + +/// Determine the normalized logical read length for a structural projection. +pub fn prepared_read_length(prepared: &PreparedProjection) -> Result { + let column_len = |column: usize| { + let info = prepared.column_infos.get(column).ok_or_else(|| { + Error::invalid_input(format!( + "projection references column index {} but only {} columns are available", + column, + prepared.column_infos.len() + )) + })?; + normalized_column_num_rows(info) + }; + projection_length( + &prepared.decoder_projection.schema, + &prepared.decoder_projection.column_indices, + &column_len, + ) +} + +fn projection_length Result>( + schema: &Schema, + column_indices: &[u32], + column_len: &F, +) -> Result { + let mut cursor = 0; + let mut field_lengths = Vec::with_capacity(schema.fields.len()); + for field in &schema.fields { + let rows = validate_field_length(field, true, column_indices, &mut cursor, column_len)?; + field_lengths.push((field.name.as_str(), rows)); + } + if cursor != column_indices.len() { + return Err(Error::invalid_input(format!( + "projection supplied {} column indices but its fields require {}", + column_indices.len(), + cursor + ))); + } + verify_uniform_lengths(&field_lengths) +} + +pub type DecodeColumn = fn(u32, &pbfile::ColumnMetadata) -> Result>; + +#[derive(Debug)] +struct StructuralReadProjection { + decode_column: DecodeColumn, +} + +/// Compose structural projection execution with an exact column grammar. +pub fn read_projection(decode_column: DecodeColumn) -> Arc { + Arc::new(StructuralReadProjection { decode_column }) +} + +#[async_trait] +impl ReadProjection for StructuralReadProjection { + fn validate_indexed( + &self, + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Result<()> { + FileMetadataProvider::validate_indexed_projection_structure(projection, metadata_index)?; + if FileMetadataProvider::projection_matches_indexed_metadata(projection) { + Ok(()) + } else { + Err(FileMetadataProvider::indexed_projection_error( + projection, + metadata_index, + )) + } + } + + fn read_length(&self, prepared: &PreparedProjection) -> Result { + prepared_read_length(prepared) + } + + async fn prepare( + &self, + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + io: &Arc, + cache: &Arc, + ) -> Result<(PreparedProjection, u64)> { + let prepared = match metadata_provider { + FileMetadataProvider::Full(metadata) => { + FileReader::validate_projection(projection, metadata)?; + PreparedProjection { + column_infos: metadata.column_infos.clone(), + decoder_projection: projection.clone(), + } + } + FileMetadataProvider::Indexed(metadata_index) => { + self.validate_indexed(projection, metadata_index)?; + let column_infos = FileMetadataProvider::load_indexed_column_infos( + metadata_index, + io, + cache, + &projection.column_indices, + self.decode_column, + ) + .await?; + PreparedProjection { + column_infos, + decoder_projection: ReaderProjection { + schema: projection.schema.clone(), + column_indices: (0..projection.column_indices.len()) + .map(|index| index as u32) + .collect(), + }, + } + } + }; + let read_len = self.read_length(&prepared)?; + Ok((prepared, read_len)) + } +} + +#[cfg(test)] +pub fn test_projection_length( + schema: &Schema, + column_indices: &[u32], + column_lengths: &[u64], +) -> Result { + projection_length(schema, column_indices, &|column| { + column_lengths.get(column).copied().ok_or_else(|| { + Error::invalid_input(format!("missing synthetic length for column {column}")) + }) + }) +} diff --git a/rust/lance-file/src/testing.rs b/rust/lance-file/src/testing.rs index 7f554e39d31..65b340e7641 100644 --- a/rust/lance-file/src/testing.rs +++ b/rust/lance-file/src/testing.rs @@ -16,7 +16,8 @@ use lance_io::{ }; use crate::reader::{FileReader, FileReaderOptions}; -use crate::writer::{FileWriter, FileWriterOptions}; +use crate::version::ConcreteFileVersion; +use crate::{versions, writer::FileWriterOptions}; pub struct FsFixture { pub tmp_path: TempObjFile, @@ -47,13 +48,15 @@ pub struct WrittenFile { pub async fn write_lance_file( data: impl RecordBatchReader, fs: &FsFixture, + version: ConcreteFileVersion, options: FileWriterOptions, ) -> WrittenFile { let writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); let lance_schema = lance_core::datatypes::Schema::try_from(data.schema().as_ref()).unwrap(); - let mut file_writer = FileWriter::try_new(writer, lance_schema.clone(), options).unwrap(); + let mut file_writer = + versions::create_writer(version, writer, lance_schema.clone(), options).unwrap(); let data = data .collect::, ArrowError>>() diff --git a/rust/lance-file/src/version.rs b/rust/lance-file/src/version.rs new file mode 100644 index 00000000000..da29b52d576 --- /dev/null +++ b/rust/lance-file/src/version.rs @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{ + fmt::{Display, Formatter}, + str::FromStr, +}; + +use lance_core::deepsize::{Context, DeepSizeOf}; +use lance_core::{Error, Result}; + +pub const LEGACY_FORMAT_VERSION: &str = "0.1"; +pub const V2_FORMAT_2_0: &str = "2.0"; +pub const V2_FORMAT_2_1: &str = "2.1"; +pub const V2_FORMAT_2_2: &str = "2.2"; +pub const V2_FORMAT_2_3: &str = "2.3"; + +/// Resolve the current stable release policy to an exact file version. +pub const fn stable_file_version() -> ConcreteFileVersion { + ConcreteFileVersion::V2_1 +} + +/// Resolve the current next release policy to an exact file version. +pub const fn next_file_version() -> ConcreteFileVersion { + ConcreteFileVersion::V2_3 +} + +/// A caller-facing Lance file-version request. +/// +/// `Stable` and `Next` are release selectors. They resolve to an exact +/// [`ConcreteFileVersion`] before file or dataset dispatch and are never persisted. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum LanceFileVersion { + /// The legacy v1 format. + Legacy, + /// Exact v2.0. + V2_0, + /// Exact v2.1 and the current default. + #[default] + V2_1, + /// The latest stable release. + Stable, + /// Exact v2.2. + V2_2, + /// The latest unstable release. + Next, + /// Exact v2.3. + V2_3, +} + +impl DeepSizeOf for LanceFileVersion { + fn deep_size_of_children(&self, _context: &mut Context) -> usize { + 0 + } +} + +impl LanceFileVersion { + /// Resolve this request through the current release policy. + pub const fn resolve(self) -> ConcreteFileVersion { + match self { + Self::Legacy => ConcreteFileVersion::V1, + Self::V2_0 => ConcreteFileVersion::V2_0, + Self::V2_1 => ConcreteFileVersion::V2_1, + Self::Stable => stable_file_version(), + Self::V2_2 => ConcreteFileVersion::V2_2, + Self::Next => next_file_version(), + Self::V2_3 => ConcreteFileVersion::V2_3, + } + } + + /// Whether this request resolves to an unstable exact format. + pub const fn is_unstable(self) -> bool { + self.resolve().is_unstable() + } +} + +impl Display for LanceFileVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Legacy => LEGACY_FORMAT_VERSION, + Self::V2_0 => V2_FORMAT_2_0, + Self::V2_1 => V2_FORMAT_2_1, + Self::V2_2 => V2_FORMAT_2_2, + Self::V2_3 => V2_FORMAT_2_3, + Self::Stable => "stable", + Self::Next => "next", + }) + } +} + +impl FromStr for LanceFileVersion { + type Err = Error; + + fn from_str(value: &str) -> Result { + match value.to_lowercase().as_str() { + LEGACY_FORMAT_VERSION | "legacy" => Ok(Self::Legacy), + V2_FORMAT_2_0 | "0.3" => Ok(Self::V2_0), + V2_FORMAT_2_1 => Ok(Self::V2_1), + V2_FORMAT_2_2 => Ok(Self::V2_2), + V2_FORMAT_2_3 => Ok(Self::V2_3), + "stable" => Ok(Self::Stable), + "next" => Ok(Self::Next), + _ => Err(unknown_version(value)), + } + } +} + +/// The exact persisted identity of a Lance file format. +/// +/// Unlike [`LanceFileVersion`], this type cannot represent release selectors such as +/// `stable` or `next`. Exact versions deliberately have no ordering because format +/// capabilities are not implied by release order. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ConcreteFileVersion { + /// The legacy v1 file format. + V1, + /// The v2.0 file format. + V2_0, + /// The v2.1 file format. + V2_1, + /// The v2.2 file format. + V2_2, + /// The v2.3 file format. + V2_3, +} + +impl DeepSizeOf for ConcreteFileVersion { + fn deep_size_of_children(&self, _context: &mut Context) -> usize { + 0 + } +} + +impl ConcreteFileVersion { + /// Convert this exact identity to the corresponding exact public selector. + /// + /// This never produces the release selectors `stable` or `next`. + pub const fn to_selector(self) -> LanceFileVersion { + match self { + Self::V1 => LanceFileVersion::Legacy, + Self::V2_0 => LanceFileVersion::V2_0, + Self::V2_1 => LanceFileVersion::V2_1, + Self::V2_2 => LanceFileVersion::V2_2, + Self::V2_3 => LanceFileVersion::V2_3, + } + } + + /// Whether this exact format is covered only by the unstable release policy. + pub const fn is_unstable(self) -> bool { + matches!(self, Self::V2_3) + } + + /// Decode the exact version string stored in a dataset manifest. + /// + /// Public selector aliases such as `legacy`, `0.3`, `stable`, and `next` are + /// intentionally rejected because manifests only store canonical exact versions. + pub fn from_manifest_string(value: &str) -> Result { + match value { + LEGACY_FORMAT_VERSION => Ok(Self::V1), + V2_FORMAT_2_0 => Ok(Self::V2_0), + V2_FORMAT_2_1 => Ok(Self::V2_1), + V2_FORMAT_2_2 => Ok(Self::V2_2), + V2_FORMAT_2_3 => Ok(Self::V2_3), + _ => Err(unknown_version(value)), + } + } + + /// Encode this exact version as the canonical string stored in a dataset manifest. + pub const fn to_manifest_string(self) -> &'static str { + match self { + Self::V1 => LEGACY_FORMAT_VERSION, + Self::V2_0 => V2_FORMAT_2_0, + Self::V2_1 => V2_FORMAT_2_1, + Self::V2_2 => V2_FORMAT_2_2, + Self::V2_3 => V2_FORMAT_2_3, + } + } + + /// Decode the major/minor version stored in `DataFile` metadata. + /// + /// Legacy manifests may omit these fields and decode to `(0, 0)`, so all legacy + /// v1 number pairs accepted by the historical decoder remain valid inputs. The + /// historical generic decoder also accepted the standard v2.0 footer pair `(0, 3)`; + /// decoding retains that compatibility while encoding always emits `(2, 0)`. + pub fn from_data_file_numbers(major: u32, minor: u32) -> Result { + match (major, minor) { + (0, 0..=2) => Ok(Self::V1), + (0, 3) | (2, 0) => Ok(Self::V2_0), + (2, 1) => Ok(Self::V2_1), + (2, 2) => Ok(Self::V2_2), + (2, 3) => Ok(Self::V2_3), + _ => Err(unknown_version(format_args!("{}.{}", major, minor))), + } + } + + /// Encode the canonical major/minor pair stored in `DataFile` metadata. + pub const fn to_data_file_numbers(self) -> (u32, u32) { + match self { + Self::V1 => (0, 2), + Self::V2_0 => (2, 0), + Self::V2_1 => (2, 1), + Self::V2_2 => (2, 2), + Self::V2_3 => (2, 3), + } + } + + /// Decode the major/minor version stored in a Lance file footer. + /// + /// V2.0 has two accepted representations: `(0, 3)` from the standard file writer + /// and `(2, 0)` from self-described and mini-lance writers. + pub fn from_footer_numbers(major: u16, minor: u16) -> Result { + match (major, minor) { + (0, 0..=2) => Ok(Self::V1), + (0, 3) | (2, 0) => Ok(Self::V2_0), + (2, 1) => Ok(Self::V2_1), + (2, 2) => Ok(Self::V2_2), + (2, 3) => Ok(Self::V2_3), + _ => Err(unknown_version(format_args!("{}.{}", major, minor))), + } + } + + /// Encode the footer numbers emitted by the standard Lance file writer. + pub const fn to_standard_footer_numbers(self) -> (u16, u16) { + match self { + Self::V1 => (0, 2), + Self::V2_0 => (0, 3), + Self::V2_1 => (2, 1), + Self::V2_2 => (2, 2), + Self::V2_3 => (2, 3), + } + } + + /// Encode the footer numbers emitted by self-described and mini-lance writers. + pub const fn to_embedded_footer_numbers(self) -> (u16, u16) { + match self { + Self::V1 => (0, 2), + Self::V2_0 => (2, 0), + Self::V2_1 => (2, 1), + Self::V2_2 => (2, 2), + Self::V2_3 => (2, 3), + } + } +} + +impl Display for ConcreteFileVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(self.to_manifest_string()) + } +} + +fn unknown_version(value: impl Display) -> Error { + Error::invalid_input_source(format!("Unknown Lance storage version: {}", value).into()) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use lance_io::object_store::ObjectStore; + use object_store::path::Path; + + use super::*; + + const EXACT_VERSIONS: [ConcreteFileVersion; 5] = [ + ConcreteFileVersion::V1, + ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2, + ConcreteFileVersion::V2_3, + ]; + + #[test] + fn selector_resolution_is_exact() { + let cases = [ + (LanceFileVersion::Legacy, ConcreteFileVersion::V1), + (LanceFileVersion::V2_0, ConcreteFileVersion::V2_0), + (LanceFileVersion::V2_1, ConcreteFileVersion::V2_1), + (LanceFileVersion::Stable, ConcreteFileVersion::V2_1), + (LanceFileVersion::V2_2, ConcreteFileVersion::V2_2), + (LanceFileVersion::Next, ConcreteFileVersion::V2_3), + (LanceFileVersion::V2_3, ConcreteFileVersion::V2_3), + ]; + + for (selector, expected) in cases { + assert_eq!(selector.resolve(), expected); + } + } + + #[test] + fn public_selector_aliases_remain_unchanged() { + let cases = [ + ("0.1", LanceFileVersion::Legacy), + ("legacy", LanceFileVersion::Legacy), + ("2.0", LanceFileVersion::V2_0), + ("0.3", LanceFileVersion::V2_0), + ("2.1", LanceFileVersion::V2_1), + ("stable", LanceFileVersion::Stable), + ("2.2", LanceFileVersion::V2_2), + ("next", LanceFileVersion::Next), + ("2.3", LanceFileVersion::V2_3), + ]; + + for (value, expected) in cases { + assert_eq!(LanceFileVersion::from_str(value).unwrap(), expected); + } + } + + #[test] + fn manifest_codec_only_accepts_canonical_exact_versions() { + for version in EXACT_VERSIONS { + let encoded = version.to_manifest_string(); + assert_eq!( + ConcreteFileVersion::from_manifest_string(encoded).unwrap(), + version + ); + } + + for selector_or_alias in ["legacy", "0.3", "stable", "next"] { + assert!(ConcreteFileVersion::from_manifest_string(selector_or_alias).is_err()); + } + } + + #[test] + fn data_file_codec_preserves_wire_numbers() { + let cases = [ + (ConcreteFileVersion::V1, (0, 2)), + (ConcreteFileVersion::V2_0, (2, 0)), + (ConcreteFileVersion::V2_1, (2, 1)), + (ConcreteFileVersion::V2_2, (2, 2)), + (ConcreteFileVersion::V2_3, (2, 3)), + ]; + + for (version, encoded) in cases { + assert_eq!(version.to_data_file_numbers(), encoded); + assert_eq!( + ConcreteFileVersion::from_data_file_numbers(encoded.0, encoded.1).unwrap(), + version + ); + } + for minor in 0..=2 { + assert_eq!( + ConcreteFileVersion::from_data_file_numbers(0, minor).unwrap(), + ConcreteFileVersion::V1 + ); + } + assert_eq!( + ConcreteFileVersion::from_data_file_numbers(0, 3).unwrap(), + ConcreteFileVersion::V2_0 + ); + } + + #[test] + fn footer_codec_preserves_both_v2_0_writer_representations() { + let standard_cases = [ + (ConcreteFileVersion::V1, (0, 2)), + (ConcreteFileVersion::V2_0, (0, 3)), + (ConcreteFileVersion::V2_1, (2, 1)), + (ConcreteFileVersion::V2_2, (2, 2)), + (ConcreteFileVersion::V2_3, (2, 3)), + ]; + let embedded_cases = [ + (ConcreteFileVersion::V1, (0, 2)), + (ConcreteFileVersion::V2_0, (2, 0)), + (ConcreteFileVersion::V2_1, (2, 1)), + (ConcreteFileVersion::V2_2, (2, 2)), + (ConcreteFileVersion::V2_3, (2, 3)), + ]; + + for (version, encoded) in standard_cases { + assert_eq!(version.to_standard_footer_numbers(), encoded); + assert_eq!( + ConcreteFileVersion::from_footer_numbers(encoded.0, encoded.1).unwrap(), + version + ); + } + for (version, encoded) in embedded_cases { + assert_eq!(version.to_embedded_footer_numbers(), encoded); + assert_eq!( + ConcreteFileVersion::from_footer_numbers(encoded.0, encoded.1).unwrap(), + version + ); + } + for minor in 0..=2 { + assert_eq!( + ConcreteFileVersion::from_footer_numbers(0, minor).unwrap(), + ConcreteFileVersion::V1 + ); + } + } + + #[tokio::test] + async fn file_version_detection_accepts_all_legacy_footer_aliases() { + let object_store = ObjectStore::memory(); + for minor in 0u16..=2 { + let path = Path::from(format!("legacy-{minor}.lance")); + let mut footer = Vec::with_capacity(8); + footer.extend_from_slice(&0u16.to_le_bytes()); + footer.extend_from_slice(&minor.to_le_bytes()); + footer.extend_from_slice(crate::format::MAGIC); + object_store.put(&path, &footer).await.unwrap(); + + assert_eq!( + crate::determine_file_version(&object_store, &path, Some(footer.len())) + .await + .unwrap(), + ConcreteFileVersion::V1 + ); + } + } +} diff --git a/rust/lance-file/src/versions/mod.rs b/rust/lance-file/src/versions/mod.rs new file mode 100644 index 00000000000..580e29bf28b --- /dev/null +++ b/rust/lance-file/src/versions/mod.rs @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Exact file-format composition roots. +//! +//! Each version module lists the mechanisms used by that file format. Callers +//! resolve release selectors before entering this module. Prefer APIs under a +//! concrete module such as [`v2_1`] when the version is statically known. The +//! root functions perform the single exhaustive dispatch for runtime versions. + +use bytes::Bytes; +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{ + decoder::{ColumnInfo, DecoderPlugins, PageInfo}, + encoder::EncodedBatch, +}; +use lance_io::{scheduler::FileScheduler, traits::Writer}; +use std::{collections::BTreeMap, future::Future, sync::Arc}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, FileMetadataProvider, FileReader, + FileReaderOptions, PreparedProjection, ProjectedFileReader, RawFileMetadata, + ReadProjection, ReaderProjection, + }, + version::ConcreteFileVersion, + writer::{FileWriter, FileWriterOptions}, +}; +use lance_core::cache::LanceCache; + +pub mod v1; +pub mod v2_0; +pub mod v2_1; +pub mod v2_2; +pub mod v2_3; + +pub(crate) fn read_projection(version: ConcreteFileVersion) -> Result> { + match version { + ConcreteFileVersion::V1 => Err(Error::internal( + "current reader composition received Lance v1".to_string(), + )), + ConcreteFileVersion::V2_0 => Ok(v2_0::read_projection()), + ConcreteFileVersion::V2_1 => Ok(v2_1::read_projection()), + ConcreteFileVersion::V2_2 => Ok(v2_2::read_projection()), + ConcreteFileVersion::V2_3 => Ok(v2_3::read_projection()), + } +} + +/// A self-described file reader selected by the exact footer version. +pub enum OpenedFileReader { + /// A v1 file. The persisted footer numbers are retained for diagnostics. + V1 { + /// The major version stored in the footer. + major_version: u16, + /// The minor version stored in the footer. + minor_version: u16, + }, + /// A current-format reader selected from the exact footer identity. + Current(FileReader), +} + +pub(crate) fn finish_metadata( + version: ConcreteFileVersion, + metadata: RawFileMetadata, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::internal( + "current metadata dispatch received a Lance v1 file".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::finish_metadata(metadata), + ConcreteFileVersion::V2_1 => v2_1::finish_metadata(metadata), + ConcreteFileVersion::V2_2 => v2_2::finish_metadata(metadata), + ConcreteFileVersion::V2_3 => v2_3::finish_metadata(metadata), + } +} + +/// Validate that decoded metadata is a complete rectangular file for an exact +/// grammar and return its normalized physical row count. +pub(crate) fn validate_external_metadata( + version: ConcreteFileVersion, + schema: &Schema, + metadata: &CachedFileMetadata, +) -> Result { + let projection = reader_projection_from_whole_schema(schema, version); + if projection.column_indices.len() != metadata.column_infos.len() { + return Err(Error::invalid_input(format!( + "schema requires {} physical columns but file metadata contains {}", + projection.column_indices.len(), + metadata.column_infos.len() + ))); + } + FileReader::validate_projection(&projection, metadata)?; + for (expected_index, column) in metadata.column_infos.iter().enumerate() { + if column.index != expected_index as u32 { + return Err(Error::invalid_input(format!( + "physical column {} reports index {}", + expected_index, column.index + ))); + } + } + let prepared = PreparedProjection { + column_infos: metadata.column_infos.clone(), + decoder_projection: projection, + }; + read_projection(version)?.read_length(&prepared) +} + +pub(crate) fn finish_metadata_index(index: FileMetadataIndex) -> Result { + match index.version { + ConcreteFileVersion::V1 => Err(Error::version_conflict( + "Attempt to use the Lance current-format reader with a v1 metadata index".to_string(), + 0, + 2, + )), + ConcreteFileVersion::V2_0 => v2_0::finish_metadata_index(index), + ConcreteFileVersion::V2_1 => v2_1::finish_metadata_index(index), + ConcreteFileVersion::V2_2 => v2_2::finish_metadata_index(index), + ConcreteFileVersion::V2_3 => v2_3::finish_metadata_index(index), + } +} + +pub(crate) fn decode_column_metadata( + version: ConcreteFileVersion, + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "self-described batches are not part of the Lance v1 grammar".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::decode_column_metadata(column_metadatas), + ConcreteFileVersion::V2_1 => v2_1::decode_column_metadata(column_metadatas), + ConcreteFileVersion::V2_2 => v2_2::decode_column_metadata(column_metadatas), + ConcreteFileVersion::V2_3 => v2_3::decode_column_metadata(column_metadatas), + } +} + +pub(crate) fn validate_global_buffers( + version: ConcreteFileVersion, + buffers: &[BufferDescriptor], +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Ok(()), + ConcreteFileVersion::V2_0 => v2_0::validate_global_buffers(buffers), + ConcreteFileVersion::V2_1 => v2_1::validate_global_buffers(buffers), + ConcreteFileVersion::V2_2 => v2_2::validate_global_buffers(buffers), + ConcreteFileVersion::V2_3 => v2_3::validate_global_buffers(buffers), + } +} + +pub fn reader_projection_from_field_ids( + version: ConcreteFileVersion, + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> Result { + Ok(match version { + ConcreteFileVersion::V1 => v1::projection_from_field_ids(schema, field_id_to_column_index), + ConcreteFileVersion::V2_0 => { + v2_0::projection_from_field_ids(schema, field_id_to_column_index) + } + ConcreteFileVersion::V2_1 => { + v2_1::projection_from_field_ids(schema, field_id_to_column_index) + } + ConcreteFileVersion::V2_2 => { + v2_2::projection_from_field_ids(schema, field_id_to_column_index) + } + ConcreteFileVersion::V2_3 => { + v2_3::projection_from_field_ids(schema, field_id_to_column_index) + } + }) +} + +pub fn reader_projection_from_whole_schema( + schema: &Schema, + version: ConcreteFileVersion, +) -> ReaderProjection { + match version { + ConcreteFileVersion::V1 => v1::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_0 => v2_0::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_1 => v2_1::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_2 => v2_2::projection_from_whole_schema(schema), + ConcreteFileVersion::V2_3 => v2_3::projection_from_whole_schema(schema), + } +} + +pub fn reader_projection_from_column_names( + version: ConcreteFileVersion, + schema: &Schema, + column_names: &[&str], +) -> Result { + match version { + ConcreteFileVersion::V1 => v1::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_0 => v2_0::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_1 => v2_1::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_2 => v2_2::projection_from_column_names(schema, column_names), + ConcreteFileVersion::V2_3 => v2_3::projection_from_column_names(schema, column_names), + } +} + +/// Count the physical columns represented by one field in an exact grammar. +pub fn physical_column_count( + version: ConcreteFileVersion, + field: &lance_core::datatypes::Field, +) -> usize { + match version { + ConcreteFileVersion::V1 => v1::physical_column_count(field), + ConcreteFileVersion::V2_0 => v2_0::physical_column_count(field), + ConcreteFileVersion::V2_1 => v2_1::physical_column_count(field), + ConcreteFileVersion::V2_2 => v2_2::physical_column_count(field), + ConcreteFileVersion::V2_3 => v2_3::physical_column_count(field), + } +} + +/// Build persisted field-to-column entries for an exact grammar. +pub fn data_file_columns(version: ConcreteFileVersion, schema: &Schema) -> (Vec, Vec) { + match version { + ConcreteFileVersion::V1 => v1::data_file_columns(schema), + ConcreteFileVersion::V2_0 => v2_0::data_file_columns(schema), + ConcreteFileVersion::V2_1 => v2_1::data_file_columns(schema), + ConcreteFileVersion::V2_2 => v2_2::data_file_columns(schema), + ConcreteFileVersion::V2_3 => v2_3::data_file_columns(schema), + } +} + +/// Copy one column's external metadata and buffers according to the exact file +/// grammar. +/// +/// The caller supplies the version-free I/O operation. V2.0 may suppress that +/// operation when a structural header page has already been copied. +pub async fn copy_external_metadata_column( + version: ConcreteFileVersion, + schema: &Schema, + column_index: usize, + has_existing_pages: bool, + copy: Copy, +) -> Result<()> +where + Copy: FnOnce() -> CopyFuture + Send, + CopyFuture: Future> + Send, +{ + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "binary-copy metadata operations are not supported for Lance v1".to_string(), + )), + ConcreteFileVersion::V2_0 => { + if v2_0::should_copy_external_metadata_column(schema, column_index, has_existing_pages) + { + copy().await + } else { + Ok(()) + } + } + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + copy().await + } + } +} + +/// Normalize one copied column before an exact-version footer is written. +pub fn finalize_external_metadata_column( + version: ConcreteFileVersion, + schema: &Schema, + column_index: usize, + pages: &mut Vec, + num_rows: u64, +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "binary-copy metadata operations are not supported for Lance v1".to_string(), + )), + ConcreteFileVersion::V2_0 => { + v2_0::finalize_external_metadata_column(schema, column_index, pages, num_rows); + Ok(()) + } + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => Ok(()), + } +} + +/// Open a projected reader while keeping exact metadata-form selection in the +/// file layer. +/// +/// `open_indexed` returns `None` when the loaded index is not selective enough +/// to justify a projected reader. V2.0 never invokes it because indexed +/// metadata is not part of that reader's accepted grammar. +pub async fn open_projected_reader( + version: ConcreteFileVersion, + projection: &ReaderProjection, + prefer_indexed: bool, + open_indexed: OpenIndexed, + open_full: OpenFull, +) -> Result +where + OpenIndexed: FnOnce() -> IndexedFuture + Send, + IndexedFuture: Future>> + Send, + OpenFull: FnOnce() -> FullFuture + Send, + FullFuture: Future> + Send, +{ + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "projected current-format readers cannot open Lance v1 files".to_string(), + )), + ConcreteFileVersion::V2_0 => open_full().await, + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + if prefer_indexed + && FileMetadataProvider::projection_matches_indexed_metadata(projection) + && let Some(reader) = open_indexed().await? + { + return Ok(reader); + } + open_full().await + } + } +} + +/// Open a self-described file and dispatch to the matching reader. +/// +/// The current-format reader's optimistic tail read is also used for exact +/// version detection, so current files do not pay for a separate footer probe. +pub async fn open_self_described_reader( + scheduler: FileScheduler, + decoder_plugins: Arc, + cache: &LanceCache, + options: FileReaderOptions, +) -> Result { + FileReader::try_open_for_dispatch(scheduler, None, decoder_plugins, cache, options).await +} + +/// Create a current-format writer for an exact file version. +/// +/// V1 uses [`v1::writer::FileWriter`] directly because its manifest provider is +/// part of the writer type. +pub fn create_writer( + version: ConcreteFileVersion, + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "Lance v1 files must be created with versions::v1::writer::FileWriter".to_string(), + )), + ConcreteFileVersion::V2_0 => { + v2_0::create_writer(object_writer, schema, options).map(Into::into) + } + ConcreteFileVersion::V2_1 => { + v2_1::create_writer(object_writer, schema, options).map(Into::into) + } + ConcreteFileVersion::V2_2 => { + v2_2::create_writer(object_writer, schema, options).map(Into::into) + } + ConcreteFileVersion::V2_3 => { + v2_3::create_writer(object_writer, schema, options).map(Into::into) + } + } +} + +/// Create a lazy current-format writer for an exact file version. +pub fn create_lazy_writer( + version: ConcreteFileVersion, + object_writer: Box, + options: FileWriterOptions, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "legacy v1 files require an explicit schema and manifest provider".to_string(), + )), + ConcreteFileVersion::V2_0 => Ok(v2_0::create_lazy_writer(object_writer, options).into()), + ConcreteFileVersion::V2_1 => Ok(v2_1::create_lazy_writer(object_writer, options).into()), + ConcreteFileVersion::V2_2 => Ok(v2_2::create_lazy_writer(object_writer, options).into()), + ConcreteFileVersion::V2_3 => Ok(v2_3::create_lazy_writer(object_writer, options).into()), + } +} + +/// Encode a self-described batch for an exact file version. +pub fn encode_self_described_batch( + version: ConcreteFileVersion, + batch: &EncodedBatch, +) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "Lance v1 does not support self-described current-format batches".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::encode_self_described_batch(batch), + ConcreteFileVersion::V2_1 => v2_1::encode_self_described_batch(batch), + ConcreteFileVersion::V2_2 => v2_2::encode_self_described_batch(batch), + ConcreteFileVersion::V2_3 => v2_3::encode_self_described_batch(batch), + } +} + +/// Encode a mini-lance batch for an exact file version. +pub fn encode_mini_batch(version: ConcreteFileVersion, batch: &EncodedBatch) -> Result { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "Lance v1 does not support mini-lance current-format batches".to_string(), + )), + ConcreteFileVersion::V2_0 => v2_0::encode_mini_batch(batch), + ConcreteFileVersion::V2_1 => v2_1::encode_mini_batch(batch), + ConcreteFileVersion::V2_2 => v2_2::encode_mini_batch(batch), + ConcreteFileVersion::V2_3 => v2_3::encode_mini_batch(batch), + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + #[test] + fn v1_rejects_current_format_embedded_batches() { + let batch = EncodedBatch { + data: Bytes::new(), + page_table: Vec::new(), + schema: Arc::new(Schema::default()), + top_level_columns: Vec::new(), + num_rows: 0, + }; + + assert!(matches!( + encode_self_described_batch(ConcreteFileVersion::V1, &batch), + Err(Error::NotSupported { .. }) + )); + assert!(matches!( + encode_mini_batch(ConcreteFileVersion::V1, &batch), + Err(Error::NotSupported { .. }) + )); + } +} diff --git a/rust/lance-file/src/versions/v1/encoding.rs b/rust/lance-file/src/versions/v1/encoding.rs new file mode 100644 index 00000000000..674fd357fad --- /dev/null +++ b/rust/lance-file/src/versions/v1/encoding.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v1 column encoding grammar. +//! +//! These codecs own the v1 wire layout, including schema dictionary payloads. +//! They are intentionally not exposed through the version-free I/O layer. + +use arrow_array::{ + Array, ArrayRef, + types::{BinaryType, LargeBinaryType, LargeUtf8Type, Utf8Type}, +}; +use arrow_schema::DataType; +use async_recursion::async_recursion; + +pub mod binary; +pub mod dictionary; +pub mod plain; + +use lance_arrow::DataTypeExt; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_io::{ + ReadBatchParams, + traits::{Reader, Writer}, +}; + +use self::{ + binary::{BinaryDecoder, BinaryEncoder}, + plain::{PlainDecoder, PlainEncoder}, +}; + +/// Decode a binary-like array from a v1 values region. +pub async fn read_binary_array( + reader: &dyn Reader, + data_type: &DataType, + nullable: bool, + position: usize, + length: usize, + params: impl Into, +) -> Result { + use arrow_schema::DataType::*; + + let params = params.into(); + match data_type { + Utf8 => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + Binary => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + LargeUtf8 => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + LargeBinary => { + BinaryDecoder::::new(reader, position, length, nullable) + .get(params) + .await + } + _ => Err(lance_core::Error::invalid_input(format!( + "unsupported v1 binary data type: {data_type}" + ))), + } +} + +/// Decode a fixed-stride array from a v1 values region. +pub async fn read_fixed_stride_array( + reader: &dyn Reader, + data_type: &DataType, + position: usize, + length: usize, + params: impl Into, +) -> Result { + if !lance_arrow::DataTypeExt::is_fixed_stride(data_type) { + return Err(lance_core::Error::schema(format!( + "{data_type} is not a fixed stride type" + ))); + } + PlainDecoder::new(reader, data_type, position, length)? + .get(params.into()) + .await +} + +/// Persist every schema dictionary using the v1 value codecs. +pub async fn write_schema_dictionaries(writer: &mut dyn Writer, schema: &mut Schema) -> Result<()> { + let max_field_id = schema.max_field_id().unwrap_or(-1); + for field_id in 0..=max_field_id { + let Some(field) = schema.mut_field_by_id(field_id) else { + continue; + }; + if !field.data_type().is_dictionary() { + continue; + } + + let dict_info = field.dictionary.as_mut().ok_or_else(|| { + Error::io(format!( + "v1 dictionary field '{}' is missing dictionary metadata", + field.name + )) + })?; + let values = dict_info.values.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "v1 dictionary field '{}' is missing dictionary values", + field.name + )) + })?; + + let data_type = values.data_type(); + let position = if data_type.is_numeric() { + PlainEncoder::new(writer, data_type) + .encode(&[values]) + .await? + } else if data_type.is_binary_like() { + BinaryEncoder::new(writer).encode(&[values]).await? + } else { + return Err(Error::schema(format!( + "v1 dictionary values do not support data type {data_type}" + ))); + }; + dict_info.offset = position; + dict_info.length = values.len(); + } + Ok(()) +} + +#[async_recursion] +async fn populate_field_dictionary(field: &mut Field, reader: &dyn Reader) -> Result<()> { + if let DataType::Dictionary(_, value_type) = field.data_type() { + let dict_info = field.dictionary.as_mut().ok_or_else(|| { + Error::io(format!( + "v1 dictionary field '{}' is missing dictionary metadata", + field.name + )) + })?; + let values = if value_type.is_binary_like() { + read_binary_array( + reader, + value_type.as_ref(), + true, + dict_info.offset, + dict_info.length, + .., + ) + .await? + } else if matches!( + value_type.as_ref(), + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + ) { + read_fixed_stride_array( + reader, + value_type.as_ref(), + dict_info.offset, + dict_info.length, + .., + ) + .await? + } else { + return Err(Error::schema(format!( + "v1 dictionary values do not support data type {value_type}" + ))); + }; + dict_info.values = Some(values); + } else { + for child in &mut field.children { + populate_field_dictionary(child, reader).await?; + } + } + Ok(()) +} + +/// Load every persisted v1 schema dictionary into its in-memory field. +pub async fn populate_schema_dictionaries(schema: &mut Schema, reader: &dyn Reader) -> Result<()> { + for field in &mut schema.fields { + populate_field_dictionary(field, reader).await?; + } + Ok(()) +} diff --git a/rust/lance-io/src/encodings/binary.rs b/rust/lance-file/src/versions/v1/encoding/binary.rs similarity index 90% rename from rust/lance-io/src/encodings/binary.rs rename to rust/lance-file/src/versions/v1/encoding/binary.rs index ecd14f0d462..38aaa0de72d 100644 --- a/rust/lance-io/src/encodings/binary.rs +++ b/rust/lance-file/src/versions/v1/encoding/binary.rs @@ -5,7 +5,7 @@ //! use std::marker::PhantomData; -use std::ops::{Range, RangeFrom, RangeFull, RangeTo}; +use std::ops::Range; use std::sync::Arc; use arrow_arith::numeric::sub; @@ -23,16 +23,17 @@ use arrow_buffer::{ArrowNativeType, Buffer, MutableBuffer, ScalarBuffer, bit_uti use arrow_cast::cast::cast; use arrow_data::ArrayDataBuilder; use arrow_schema::DataType; -use async_trait::async_trait; use bytes::Bytes; use futures::{StreamExt, TryStreamExt}; use lance_arrow::BufferExt; use tokio::io::AsyncWriteExt; -use super::ReadBatchParams; -use super::{AsyncIndex, Decoder, Encoder, plain::PlainDecoder}; -use crate::traits::{Reader, Writer}; +use super::plain::PlainDecoder; use lance_core::Result; +use lance_io::{ + ReadBatchParams, + traits::{Reader, Writer}, +}; /// Encoder for Var-binary encoding. pub struct BinaryEncoder<'a> { @@ -87,9 +88,8 @@ impl<'a> BinaryEncoder<'a> { } } -#[async_trait] -impl Encoder for BinaryEncoder<'_> { - async fn encode(&mut self, arrs: &[&dyn Array]) -> Result { +impl BinaryEncoder<'_> { + pub async fn encode(&mut self, arrs: &[&dyn Array]) -> Result { assert!(!arrs.is_empty()); let data_type = arrs[0].data_type(); match data_type { @@ -97,12 +97,10 @@ impl Encoder for BinaryEncoder<'_> { DataType::Binary => self.encode_typed_arr::(arrs).await, DataType::LargeUtf8 => self.encode_typed_arr::(arrs).await, DataType::LargeBinary => self.encode_typed_arr::(arrs).await, - _ => { - return Err(lance_core::Error::invalid_input(format!( - "Unsupported data type for binary encoding: {}", - data_type - ))); - } + _ => Err(lance_core::Error::invalid_input(format!( + "Unsupported data type for binary encoding: {}", + data_type + ))), } } } @@ -134,7 +132,8 @@ impl<'a, T: ByteArrayType> BinaryDecoder<'a, T> { /// ```rust /// use arrow_array::types::Utf8Type; /// use object_store::path::Path; - /// use lance_io::{local::LocalObjectReader, encodings::binary::BinaryDecoder, traits::Reader}; + /// use lance_file::versions::v1::encoding::binary::BinaryDecoder; + /// use lance_io::{local::LocalObjectReader, traits::Reader}; /// /// async { /// let reader = LocalObjectReader::open_local_path("/tmp/foo.lance", 2048, None).await.unwrap(); @@ -285,16 +284,15 @@ fn plan_take_chunks( Ok(chunks) } -#[async_trait] -impl Decoder for BinaryDecoder<'_, T> { - async fn decode(&self) -> Result { +impl BinaryDecoder<'_, T> { + pub async fn decode(&self) -> Result { self.get(..).await } /// Take the values at the given indices. /// /// This function assumes indices are sorted. - async fn take(&self, indices: &UInt32Array) -> Result { + pub async fn take(&self, indices: &UInt32Array) -> Result { if indices.is_empty() { return Ok(new_empty_array(&T::DATA_TYPE)); } @@ -393,64 +391,17 @@ impl Decoder for BinaryDecoder<'_, T> { } } -#[async_trait] -impl AsyncIndex for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: usize) -> Self::Output { - self.get(index..index + 1).await - } -} - -#[async_trait] -impl AsyncIndex> for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: RangeFrom) -> Self::Output { - self.get(index.start..self.length).await - } -} - -#[async_trait] -impl AsyncIndex> for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: RangeTo) -> Self::Output { - self.get(0..index.end).await - } -} - -#[async_trait] -impl AsyncIndex for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, _: RangeFull) -> Self::Output { - self.get(0..self.length).await - } -} - -#[async_trait] -impl AsyncIndex for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, params: ReadBatchParams) -> Self::Output { - match params { - ReadBatchParams::Range(r) => self.get(r).await, - // Ranges not supported in v1 files - ReadBatchParams::Ranges(_) => unimplemented!(), - ReadBatchParams::RangeFull => self.get(..).await, - ReadBatchParams::RangeTo(r) => self.get(r).await, - ReadBatchParams::RangeFrom(r) => self.get(r).await, - ReadBatchParams::Indices(indices) => self.take(&indices).await, +impl BinaryDecoder<'_, T> { + async fn decode_range(&self, index: Range) -> Result { + if index.end > self.length { + return Err(lance_core::Error::invalid_input(format!( + "v1 binary row range {}..{} exceeds length {}", + index.start, index.end, self.length + ))); + } + if index.is_empty() { + return Ok(new_empty_array(&T::DATA_TYPE)); } - } -} - -#[async_trait] -impl AsyncIndex> for BinaryDecoder<'_, T> { - type Output = Result; - - async fn get(&self, index: Range) -> Self::Output { let position_decoder = PlainDecoder::new( self.reader, &DataType::Int64, @@ -462,6 +413,19 @@ impl AsyncIndex> for BinaryDecoder<'_, T> { self.get_range(int64_positions, 0..index.len()).await } + + pub async fn get(&self, params: impl Into) -> Result { + match params.into() { + ReadBatchParams::Range(range) => self.decode_range(range).await, + ReadBatchParams::Ranges(_) => Err(lance_core::Error::invalid_input( + "multiple ranges are not supported by v1 binary encoding", + )), + ReadBatchParams::RangeFull => self.decode_range(0..self.length).await, + ReadBatchParams::RangeTo(range) => self.decode_range(0..range.end).await, + ReadBatchParams::RangeFrom(range) => self.decode_range(range.start..self.length).await, + ReadBatchParams::Indices(indices) => self.take(&indices).await, + } + } } #[cfg(test)] @@ -474,7 +438,7 @@ mod tests { use arrow_select::concat::concat; use lance_core::utils::tempfile::TempStdFile; - use crate::local::LocalObjectReader; + use lance_io::local::LocalObjectReader; async fn write_test_data( path: impl AsRef, diff --git a/rust/lance-io/src/encodings/dictionary.rs b/rust/lance-file/src/versions/v1/encoding/dictionary.rs similarity index 88% rename from rust/lance-io/src/encodings/dictionary.rs rename to rust/lance-file/src/versions/v1/encoding/dictionary.rs index b51adf66a59..0056632f725 100644 --- a/rust/lance-io/src/encodings/dictionary.rs +++ b/rust/lance-file/src/versions/v1/encoding/dictionary.rs @@ -14,18 +14,14 @@ use arrow_array::types::{ }; use arrow_array::{Array, ArrayRef, DictionaryArray, PrimitiveArray, UInt32Array}; use arrow_schema::DataType; -use async_trait::async_trait; -use crate::{ +use lance_core::{Error, Result}; +use lance_io::{ ReadBatchParams, traits::{Reader, Writer}, }; -use lance_core::{Error, Result}; -use super::AsyncIndex; -use super::plain::PlainEncoder; -use crate::encodings::plain::PlainDecoder; -use crate::encodings::{Decoder, Encoder}; +use super::plain::{PlainDecoder, PlainEncoder}; /// Encoder for Dictionary encoding. pub struct DictionaryEncoder<'a> { @@ -60,9 +56,8 @@ impl<'a> DictionaryEncoder<'a> { } } -#[async_trait] -impl Encoder for DictionaryEncoder<'_> { - async fn encode(&mut self, array: &[&dyn Array]) -> Result { +impl DictionaryEncoder<'_> { + pub async fn encode(&mut self, array: &[&dyn Array]) -> Result { use DataType::*; match self.key_type { @@ -165,36 +160,17 @@ impl<'a> DictionaryDecoder<'a> { } } -#[async_trait] -impl Decoder for DictionaryDecoder<'_> { - async fn decode(&self) -> Result { +impl DictionaryDecoder<'_> { + pub async fn decode(&self) -> Result { self.decode_impl(..).await } - async fn take(&self, indices: &UInt32Array) -> Result { + pub async fn take(&self, indices: &UInt32Array) -> Result { self.decode_impl(indices.clone()).await } -} - -#[async_trait] -impl AsyncIndex for DictionaryDecoder<'_> { - type Output = Result; - - async fn get(&self, _index: usize) -> Self::Output { - Err(Error::not_supported_source( - "DictionaryDecoder does not support get()" - .to_string() - .into(), - )) - } -} - -#[async_trait] -impl AsyncIndex for DictionaryDecoder<'_> { - type Output = Result; - async fn get(&self, params: ReadBatchParams) -> Self::Output { - self.decode_impl(params.clone()).await + pub async fn get(&self, params: impl Into) -> Result { + self.decode_impl(params).await } } @@ -202,10 +178,10 @@ impl AsyncIndex for DictionaryDecoder<'_> { mod tests { use super::*; - use crate::local::LocalObjectReader; use arrow_array::StringArray; use arrow_buffer::ArrowNativeType; use lance_core::utils::tempfile::TempStdFile; + use lance_io::local::LocalObjectReader; use tokio::io::AsyncWriteExt; async fn test_dict_decoder_for_type() { diff --git a/rust/lance-io/src/encodings/plain.rs b/rust/lance-file/src/versions/v1/encoding/plain.rs similarity index 91% rename from rust/lance-io/src/encodings/plain.rs rename to rust/lance-file/src/versions/v1/encoding/plain.rs index a5ec97c7beb..18910e9251e 100644 --- a/rust/lance-io/src/encodings/plain.rs +++ b/rust/lance-file/src/versions/v1/encoding/plain.rs @@ -6,14 +6,10 @@ //! Plain encoding works with fixed stride types, i.e., `boolean`, `i8...i64`, `f16...f64`, //! it stores the array directly in the file. It offers O(1) read access. -use std::ops::{Range, RangeFrom, RangeFull, RangeTo}; +use std::ops::Range; use std::slice::from_raw_parts; use std::sync::Arc; -use crate::{ - ReadBatchParams, - traits::{Reader, Writer}, -}; use arrow_arith::numeric::sub; use arrow_array::{ Array, ArrayRef, BooleanArray, FixedSizeBinaryArray, FixedSizeListArray, UInt8Array, @@ -24,15 +20,15 @@ use arrow_data::{ArrayDataBuilder, BufferSpec, layout}; use arrow_schema::{DataType, Field}; use arrow_select::{concat::concat, take::take}; use async_recursion::async_recursion; -use async_trait::async_trait; -use bytes::Bytes; use futures::stream::{self, StreamExt, TryStreamExt}; use lance_arrow::*; use lance_core::{Error, Result}; +use lance_io::{ + ReadBatchParams, + traits::{Reader, Writer}, +}; use tokio::io::AsyncWriteExt; -use crate::encodings::{AsyncIndex, Decoder}; - /// Encoder for plain encoding. /// pub struct PlainEncoder<'a> { @@ -175,49 +171,41 @@ fn get_byte_range(data_type: &DataType, row_range: Range) -> Range } } -pub fn bytes_to_array( +fn bytes_to_array( data_type: &DataType, - bytes: Bytes, + bytes: bytes::Bytes, len: usize, offset: usize, ) -> Result { let layout = layout(data_type); - if layout.buffers.len() != 1 { return Err(Error::internal(format!( - "Can only convert datatypes that require one buffer, found {:?}", - data_type + "v1 plain encoding requires one value buffer, found {data_type}" ))); } - let buf: Buffer = if let BufferSpec::FixedWidth { + let buffer = if let BufferSpec::FixedWidth { byte_width, alignment, } = &layout.buffers[0] { - // this code is taken from - // https://github.com/apache/arrow-rs/blob/master/arrow-data/src/data.rs#L748-L768 - let len_plus_offset = len + offset; - let min_buffer_size = len_plus_offset.saturating_mul(*byte_width); - - // alignment or size isn't right -- just make a copy + let min_buffer_size = (len + offset).saturating_mul(*byte_width); if bytes.len() < min_buffer_size { Buffer::copy_bytes_bytes(bytes, min_buffer_size) } else { Buffer::from_bytes_bytes(bytes, *alignment as u64) } } else { - // cases we don't handle, just copy Buffer::from_slice_ref(bytes) }; - let array_data = ArrayDataBuilder::new(data_type.clone()) + let data = ArrayDataBuilder::new(data_type.clone()) .len(len) .offset(offset) .null_count(0) - .add_buffer(buf) + .add_buffer(buffer) .build()?; - Ok(make_array(array_data)) + Ok(make_array(data)) } impl<'a> PlainDecoder<'a> { @@ -383,13 +371,12 @@ fn make_chunked_requests( chunked_ranges } -#[async_trait] -impl Decoder for PlainDecoder<'_> { - async fn decode(&self) -> Result { +impl PlainDecoder<'_> { + pub async fn decode(&self) -> Result { self.get(0..self.length).await } - async fn take(&self, indices: &UInt32Array) -> Result { + pub async fn take(&self, indices: &UInt32Array) -> Result { if indices.is_empty() { return Ok(new_empty_array(self.data_type)); } @@ -419,23 +406,9 @@ impl Decoder for PlainDecoder<'_> { let references = arrays.iter().map(|a| a.as_ref()).collect::>(); Ok(concat(&references)?) } -} - -#[async_trait] -impl AsyncIndex for PlainDecoder<'_> { - // TODO: should this return a Scalar value? - type Output = Result; - - async fn get(&self, index: usize) -> Self::Output { - self.get(index..index + 1).await - } -} - -#[async_trait] -impl AsyncIndex> for PlainDecoder<'_> { - type Output = Result; - async fn get(&self, index: Range) -> Self::Output { + #[async_recursion] + async fn decode_range(&self, index: Range) -> Result { if index.is_empty() { return Ok(new_empty_array(self.data_type)); } @@ -451,47 +424,16 @@ impl AsyncIndex> for PlainDecoder<'_> { _ => self.decode_primitive(index.start, index.end).await, } } -} - -#[async_trait] -impl AsyncIndex> for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, index: RangeFrom) -> Self::Output { - self.get(index.start..self.length).await - } -} - -#[async_trait] -impl AsyncIndex> for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, index: RangeTo) -> Self::Output { - self.get(0..index.end).await - } -} - -#[async_trait] -impl AsyncIndex for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, _: RangeFull) -> Self::Output { - self.get(0..self.length).await - } -} -#[async_trait] -impl AsyncIndex for PlainDecoder<'_> { - type Output = Result; - - async fn get(&self, params: ReadBatchParams) -> Self::Output { - match params { - ReadBatchParams::Range(r) => self.get(r).await, - // Ranges not supported in v1 files - ReadBatchParams::Ranges(_) => unimplemented!(), - ReadBatchParams::RangeFull => self.get(..).await, - ReadBatchParams::RangeTo(r) => self.get(r).await, - ReadBatchParams::RangeFrom(r) => self.get(r).await, + pub async fn get(&self, params: impl Into) -> Result { + match params.into() { + ReadBatchParams::Range(range) => self.decode_range(range).await, + ReadBatchParams::Ranges(_) => Err(Error::invalid_input( + "multiple ranges are not supported by v1 plain encoding", + )), + ReadBatchParams::RangeFull => self.decode_range(0..self.length).await, + ReadBatchParams::RangeTo(range) => self.decode_range(0..range.end).await, + ReadBatchParams::RangeFrom(range) => self.decode_range(range.start..self.length).await, ReadBatchParams::Indices(indices) => self.take(&indices).await, } } @@ -502,11 +444,14 @@ mod tests { use std::ops::Deref; use arrow_array::*; + use arrow_buffer::Buffer; + use arrow_data::ArrayDataBuilder; + use bytes::Bytes; use lance_core::utils::tempfile::TempStdFile; + use lance_io::local::LocalObjectReader; use rand::prelude::*; use super::*; - use crate::local::LocalObjectReader; #[tokio::test] async fn test_encode_decode_primitive_array() { @@ -682,7 +627,7 @@ mod tests { } async fn make_array_(data_type: &DataType, buffer: &Buffer) -> ArrayRef { - make_array( + arrow_array::make_array( ArrayDataBuilder::new(data_type.clone()) .len(126) .add_buffer(buffer.clone()) diff --git a/rust/lance-file/src/previous/format/metadata.rs b/rust/lance-file/src/versions/v1/format/metadata.rs similarity index 98% rename from rust/lance-file/src/previous/format/metadata.rs rename to rust/lance-file/src/versions/v1/format/metadata.rs index 11ba00c3243..209d0733cec 100644 --- a/rust/lance-file/src/previous/format/metadata.rs +++ b/rust/lance-file/src/versions/v1/format/metadata.rs @@ -62,10 +62,10 @@ impl TryFrom for Metadata { manifest_position: Some(m.manifest_position as usize), stats_metadata: if let Some(stats_meta) = m.statistics { Some(StatisticsMetadata { - schema: Schema::from(FieldsWithMeta { + schema: Schema::try_from(FieldsWithMeta { fields: Fields(stats_meta.schema), metadata: Default::default(), - }), + })?, leaf_field_ids: stats_meta.fields, page_table_position: stats_meta.page_table_position as usize, }) diff --git a/rust/lance-file/src/previous/format/mod.rs b/rust/lance-file/src/versions/v1/format/mod.rs similarity index 100% rename from rust/lance-file/src/previous/format/mod.rs rename to rust/lance-file/src/versions/v1/format/mod.rs diff --git a/rust/lance-file/src/versions/v1/mod.rs b/rust/lance-file/src/versions/v1/mod.rs new file mode 100644 index 00000000000..a7da08de11e --- /dev/null +++ b/rust/lance-file/src/versions/v1/mod.rs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v1 file implementation. +//! +//! This module is the canonical home of the v1 reader, writer, metadata, and +//! page-table grammar. V1 accepts footer versions `(0, 0)` through `(0, 2)` and +//! writes the `(0, 2)` identity used by [`writer::FileWriter`]. + +pub mod encoding; +pub mod format; +pub mod page_table; +pub mod reader; +pub mod writer; + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{ + Result, + datatypes::{Field, Schema}, +}; + +use crate::reader::ReaderProjection; + +fn append_field_ids( + fields: &[Field], + field_id_to_column_index: &BTreeMap, + column_indices: &mut Vec, +) { + for field in fields { + if let Some(column_index) = field_id_to_column_index.get(&(field.id as u32)).copied() { + column_indices.push(column_index); + } + if !field.is_blob() && !field.is_packed_struct() { + append_field_ids(&field.children, field_id_to_column_index, column_indices); + } + } +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + let mut column_indices = Vec::new(); + append_field_ids( + &schema.fields, + field_id_to_column_index, + &mut column_indices, + ); + ReaderProjection { + schema: Arc::new(schema.clone()), + column_indices, + } +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + projection_from_field_ids(schema, &field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + let field_id_to_column_index = field_id_to_column_index(schema); + let projected = schema.project(column_names)?; + Ok(projection_from_field_ids( + &projected, + &field_id_to_column_index, + )) +} + +/// Count physical columns represented by a field in a v1 footer. +pub fn physical_column_count(field: &Field) -> usize { + if field.is_blob() || field.is_packed_struct() { + 1 + } else { + 1 + field + .children + .iter() + .map(physical_column_count) + .sum::() + } +} + +/// Build persisted field-to-column entries for a v1 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + let mut field_ids = Vec::new(); + let mut column_indices = Vec::new(); + append_physical_fields(&schema.fields, &mut field_ids, &mut column_indices, &mut 0); + (field_ids, column_indices) +} + +fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + let (field_ids, column_indices) = data_file_columns(schema); + field_ids + .into_iter() + .zip(column_indices) + .map(|(field_id, column_index)| (field_id as u32, column_index as u32)) + .collect() +} + +fn append_physical_fields( + fields: &[Field], + field_ids: &mut Vec, + column_indices: &mut Vec, + next_column: &mut i32, +) { + for field in fields { + field_ids.push(field.id); + column_indices.push(*next_column); + *next_column += 1; + if !field.is_blob() && !field.is_packed_struct() { + append_physical_fields(&field.children, field_ids, column_indices, next_column); + } + } +} diff --git a/rust/lance-file/src/previous/page_table.rs b/rust/lance-file/src/versions/v1/page_table.rs similarity index 99% rename from rust/lance-file/src/previous/page_table.rs rename to rust/lance-file/src/versions/v1/page_table.rs index cc246caa585..b2ff750c13a 100644 --- a/rust/lance-file/src/previous/page_table.rs +++ b/rust/lance-file/src/versions/v1/page_table.rs @@ -5,14 +5,14 @@ use arrow_array::builder::Int64Builder; use arrow_array::{Array, Int64Array}; use arrow_schema::DataType; use lance_core::deepsize::DeepSizeOf; -use lance_io::encodings::Decoder; -use lance_io::encodings::plain::PlainDecoder; use std::collections::BTreeMap; use tokio::io::AsyncWriteExt; use lance_core::{Error, Result}; use lance_io::traits::{Reader, Writer}; +use super::encoding::plain::PlainDecoder; + #[derive(Clone, Debug, PartialEq, DeepSizeOf)] pub struct PageInfo { pub position: usize, diff --git a/rust/lance-file/src/previous/reader.rs b/rust/lance-file/src/versions/v1/reader.rs similarity index 93% rename from rust/lance-file/src/previous/reader.rs rename to rust/lance-file/src/versions/v1/reader.rs index 1ab861985e1..0c1d490d5a6 100644 --- a/rust/lance-file/src/previous/reader.rs +++ b/rust/lance-file/src/versions/v1/reader.rs @@ -21,25 +21,22 @@ use arrow_select::concat::{self, concat_batches}; use async_recursion::async_recursion; use futures::{Future, FutureExt, StreamExt, TryStreamExt, stream}; use lance_arrow::*; -use lance_core::cache::{CacheKey, LanceCache}; +use lance_core::cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}; use lance_core::datatypes::{Field, Schema}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; -use lance_io::encodings::AsyncIndex; -use lance_io::encodings::dictionary::DictionaryDecoder; use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; use lance_io::traits::Reader; -use lance_io::utils::{ - read_fixed_stride_array, read_metadata_offset, read_struct, read_struct_from_buf, -}; +use lance_io::utils::{read_metadata_offset, read_struct, read_struct_from_buf}; use lance_io::{ReadBatchParams, object_store::ObjectStore}; use std::borrow::Cow; use object_store::path::Path; use tracing::instrument; -use crate::previous::format::metadata::Metadata; -use crate::previous::page_table::{PageInfo, PageTable}; +use crate::versions::v1::encoding::{dictionary::DictionaryDecoder, read_fixed_stride_array}; +use crate::versions::v1::format::metadata::Metadata; +use crate::versions::v1::page_table::{PageInfo, PageTable}; /// Lance File Reader. /// @@ -83,7 +80,23 @@ impl<'a, T> StringCacheKey<'a, T> { } } -impl CacheKey for StringCacheKey<'_, T> { +trait StableStringCacheValue: 'static { + const STABLE_TYPE_ID: &'static str; +} + +impl StableStringCacheValue for Metadata { + const STABLE_TYPE_ID: &'static str = "lance.file.previous.Metadata"; +} + +impl StableStringCacheValue for PageTable { + const STABLE_TYPE_ID: &'static str = "lance.file.previous.PageTable"; +} + +impl StableStringCacheValue for Option { + const STABLE_TYPE_ID: &'static str = "lance.file.previous.OptionalPageTable"; +} + +impl CacheKey for StringCacheKey<'_, T> { type ValueType = T; fn key(&self) -> Cow<'_, str> { @@ -91,11 +104,22 @@ impl CacheKey for StringCacheKey<'_, T> { } fn type_name() -> &'static str { - // This is a private, crate-internal key that is only instantiated with - // a single concrete T within one build, so std::any::type_name is fine - // here — there is no cross-crate collision risk. + // Keep the legacy diagnostic name for compatibility. `stable_type_id` + // provides the compiler-independent identity used by physical keys. std::any::type_name::() } + + fn stable_type_id() -> &'static str { + T::STABLE_TYPE_ID + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.file.previous.string-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.key); + } } impl FileReader { @@ -244,7 +268,7 @@ impl FileReader { } /// Load some metadata about the fragment from the cache, if there is one. - async fn load_from_cache( + async fn load_from_cache( cache: Option<&LanceCache>, key: String, loader: F, @@ -607,7 +631,7 @@ async fn read_binary_array( ) -> Result { let page_info = get_page_info(page_table, field, batch_id)?; - lance_io::utils::read_binary_array( + crate::versions::v1::encoding::read_binary_array( reader.object_reader.as_ref(), &field.data_type(), field.nullable, @@ -785,7 +809,7 @@ where #[cfg(test)] mod tests { - use crate::previous::writer::{FileWriter as PreviousFileWriter, NotSelfDescribing}; + use crate::versions::v1::writer::{FileWriter as V1FileWriter, NotSelfDescribing}; use super::*; @@ -800,6 +824,34 @@ mod tests { use arrow_schema::{Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema}; use lance_io::object_store::ObjectStoreParams; + #[test] + fn string_cache_key_discriminators_are_stable_and_type_scoped() { + assert_eq!( + [ + as CacheKey>::stable_type_id(), + as CacheKey>::stable_type_id(), + > as CacheKey>::stable_type_id(), + ], + [ + "lance.file.previous.Metadata", + "lance.file.previous.PageTable", + "lance.file.previous.OptionalPageTable", + ] + ); + assert_eq!( + [ + as CacheKey>::type_name(), + as CacheKey>::type_name(), + > as CacheKey>::type_name(), + ], + [ + std::any::type_name::(), + std::any::type_name::(), + std::any::type_name::>(), + ] + ); + } + #[tokio::test] async fn test_take() { let arrow_schema = ArrowSchema::new(vec![ @@ -840,7 +892,7 @@ mod tests { } schema.set_dictionary(&batches[0]).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -904,7 +956,7 @@ mod tests { )])); let batch = RecordBatch::try_new(arrow_schema.clone(), vec![struct_arr]).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -958,7 +1010,7 @@ mod tests { .collect::>(); let batches_ref = batches.iter().collect::>(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -985,7 +1037,7 @@ mod tests { let schema: Schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); let batch = RecordBatch::try_new(arrow_schema.clone(), vec![struct_array.clone()]).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1110,7 +1162,7 @@ mod tests { // write to a lance file let store = ObjectStore::memory(); let path = Path::from("/takes"); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1227,7 +1279,7 @@ mod tests { let store = ObjectStore::memory(); let path = Path::from("/take_list"); let schema: Schema = (&arrow_schema).try_into().unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1304,7 +1356,7 @@ mod tests { .unwrap(); let schema: Schema = (&arrow_schema).try_into().unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1338,7 +1390,7 @@ mod tests { // write to a lance file let store = ObjectStore::memory(); let path = Path::from("/read_range"); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1365,7 +1417,7 @@ mod tests { let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, true)]); let schema = Schema::try_from(&arrow_schema).unwrap(); - let mut writer = PreviousFileWriter::::try_new( + let mut writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1425,7 +1477,7 @@ mod tests { false, )])); let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -1470,7 +1522,7 @@ mod tests { let partial_schema = schema.project(&["f50"]).unwrap(); let partial_arrow: ArrowSchema = (&partial_schema).into(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, partial_schema.clone(), diff --git a/rust/lance-file/src/previous/writer/mod.rs b/rust/lance-file/src/versions/v1/writer/mod.rs similarity index 95% rename from rust/lance-file/src/previous/writer/mod.rs rename to rust/lance-file/src/versions/v1/writer/mod.rs index ab13e782367..e3188e0d6a7 100644 --- a/rust/lance-file/src/previous/writer/mod.rs +++ b/rust/lance-file/src/versions/v1/writer/mod.rs @@ -18,17 +18,18 @@ use async_trait::async_trait; use lance_arrow::*; use lance_core::datatypes::{Encoding, Field, NullabilityComparison, Schema, SchemaCompareOptions}; use lance_core::{Error, Result}; -use lance_io::encodings::{ - Encoder, binary::BinaryEncoder, dictionary::DictionaryEncoder, plain::PlainEncoder, -}; use lance_io::object_store::ObjectStore; use lance_io::traits::{WriteExt, Writer}; use object_store::path::Path; use tokio::io::AsyncWriteExt; use crate::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION}; -use crate::previous::format::metadata::{Metadata, StatisticsMetadata}; -use crate::previous::page_table::{PageInfo, PageTable}; +use crate::versions::v1::encoding::{ + binary::BinaryEncoder, dictionary::DictionaryEncoder, plain::PlainEncoder, + write_schema_dictionaries, +}; +use crate::versions::v1::format::metadata::{Metadata, StatisticsMetadata}; +use crate::versions::v1::page_table::{PageInfo, PageTable}; use crate::writer::FileWriteSummary; /// The file format currently includes a "manifest" where it stores the schema for @@ -623,53 +624,6 @@ impl FileWriter { } } - /// Writes the dictionaries (using plain/binary encoding) into the file - /// - /// The offsets and lengths of the written buffers are stored in the given - /// schema so that the dictionaries can be loaded in the future. - async fn write_dictionaries(writer: &mut dyn Writer, schema: &mut Schema) -> Result<()> { - // Write dictionary values. - let max_field_id = schema.max_field_id().unwrap_or(-1); - for field_id in 0..max_field_id + 1 { - if let Some(field) = schema.mut_field_by_id(field_id) - && field.data_type().is_dictionary() - { - let dict_info = field.dictionary.as_mut().ok_or_else(|| { - // and wrap it in here. - Error::io(format!("Lance field {} misses dictionary info", field.name)) - })?; - - let value_arr = dict_info.values.as_ref().ok_or_else(|| { - Error::invalid_input(format!( - "Lance field {} is dictionary type, but misses the dictionary value array", - field.name - )) - })?; - - let data_type = value_arr.data_type(); - let pos = match data_type { - dt if dt.is_numeric() => { - let mut encoder = PlainEncoder::new(writer, dt); - encoder.encode(&[value_arr]).await? - } - dt if dt.is_binary_like() => { - let mut encoder = BinaryEncoder::new(writer); - encoder.encode(&[value_arr]).await? - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_arr.data_type() - ))); - } - }; - dict_info.offset = pos; - dict_info.length = value_arr.len(); - } - } - Ok(()) - } - async fn write_footer(&mut self) -> Result<()> { // Step 1. Write page table. let field_id_offset = *self.schema.field_ids().iter().min().unwrap(); @@ -683,7 +637,7 @@ impl FileWriter { self.metadata.stats_metadata = self.write_statistics().await?; // Step 3. Write manifest and dictionary values. - Self::write_dictionaries(self.object_writer.as_mut(), &mut self.schema).await?; + write_schema_dictionaries(self.object_writer.as_mut(), &mut self.schema).await?; let pos = M::store_schema(self.object_writer.as_mut(), &self.schema).await?; // Step 4. Write metadata. @@ -756,7 +710,7 @@ mod tests { }; use arrow_select::concat::concat_batches; - use crate::previous::reader::FileReader; + use crate::versions::v1::reader::FileReader; #[tokio::test] async fn test_write_file() { @@ -1028,6 +982,37 @@ mod tests { assert_eq!(actual, batch); } + #[tokio::test] + async fn test_write_empty_non_nullable_string() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "value", + DataType::Utf8, + false, + )])); + let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); + let store = ObjectStore::memory(); + let path = Path::from("/empty"); + let mut file_writer = FileWriter::::try_new( + &store, + &path, + schema.clone(), + &Default::default(), + ) + .await + .unwrap(); + + file_writer + .write(&[RecordBatch::new_empty(arrow_schema)]) + .await + .unwrap(); + let summary = file_writer.finish().await.unwrap(); + assert_eq!(summary.num_rows, 0); + + let reader = FileReader::try_new(&store, &path, schema).await.unwrap(); + let actual = reader.read_batch(0, .., reader.schema()).await.unwrap(); + assert_eq!(actual.num_rows(), 0); + } + #[tokio::test] async fn test_collect_stats() { // Validate: diff --git a/rust/lance-file/src/previous/writer/statistics.rs b/rust/lance-file/src/versions/v1/writer/statistics.rs similarity index 99% rename from rust/lance-file/src/previous/writer/statistics.rs rename to rust/lance-file/src/versions/v1/writer/statistics.rs index 1ccc38ca43d..7eeb3db12bc 100644 --- a/rust/lance-file/src/previous/writer/statistics.rs +++ b/rust/lance-file/src/versions/v1/writer/statistics.rs @@ -642,8 +642,10 @@ impl StatisticsCollector { let max_value = Arc::new(builder.max_value.finish()); let struct_fields = vec![ ArrowField::new("null_count", DataType::Int64, false), - ArrowField::new("min_value", field.data_type(), field.nullable), - ArrowField::new("max_value", field.data_type(), field.nullable), + // Bounds can be absent for empty pages regardless of the data field's + // nullability. + ArrowField::new("min_value", field.data_type(), true), + ArrowField::new("max_value", field.data_type(), true), ]; let stats = StructArray::new( diff --git a/rust/lance-file/src/versions/v2_0/mod.rs b/rust/lance-file/src/versions/v2_0/mod.rs new file mode 100644 index 00000000000..83efef58289 --- /dev/null +++ b/rust/lance-file/src/versions/v2_0/mod.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.0 file composition. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use lance_core::{ + Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + decoder::PageInfo, + encoder::{ArrayFieldEncodingStrategy, EncodedBatch, FieldEncodingStrategy}, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{reader::ReadProjection, writer::FileWriterOptions}; + +mod reader; +mod writer; + +#[cfg(test)] +pub(crate) use reader::test_projection_length; +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; +pub use writer::Writer; + +pub(crate) fn read_projection() -> Arc { + reader::read_projection() +} + +/// Count physical columns represented by a field in a v2.0 footer. +pub fn physical_column_count(field: &Field) -> usize { + if field.is_blob() || field.is_packed_struct() { + 1 + } else { + 1 + field + .children + .iter() + .map(physical_column_count) + .sum::() + } +} + +/// Build persisted field-to-column entries for a v2.0 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + let mut field_ids = Vec::new(); + let mut column_indices = Vec::new(); + append_physical_fields(&schema.fields, &mut field_ids, &mut column_indices, &mut 0); + (field_ids, column_indices) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + let (field_ids, column_indices) = data_file_columns(schema); + field_ids + .into_iter() + .zip(column_indices) + .map(|(field_id, column_index)| (field_id as u32, column_index as u32)) + .collect() +} + +fn append_physical_fields( + fields: &[Field], + field_ids: &mut Vec, + column_indices: &mut Vec, + next_column: &mut i32, +) { + for field in fields { + field_ids.push(field.id); + column_indices.push(*next_column); + *next_column += 1; + if !field.is_blob() && !field.is_packed_struct() { + append_physical_fields(&field.children, field_ids, column_indices, next_column); + } + } +} + +fn is_external_metadata_structural_header( + fields: &[Field], + target_column: usize, + next_column: &mut usize, +) -> Option { + for field in fields { + if *next_column == target_column { + return Some(field.logical_type.is_struct() && !field.is_packed_struct()); + } + *next_column += 1; + if !field.is_blob() + && !field.is_packed_struct() + && let Some(is_header) = + is_external_metadata_structural_header(&field.children, target_column, next_column) + { + return Some(is_header); + } + } + None +} + +pub(super) fn should_copy_external_metadata_column( + schema: &Schema, + column_index: usize, + has_existing_pages: bool, +) -> bool { + let mut next_column = 0; + let is_header = + is_external_metadata_structural_header(&schema.fields, column_index, &mut next_column) + .unwrap_or(false); + !is_header || !has_existing_pages +} + +pub(super) fn finalize_external_metadata_column( + schema: &Schema, + column_index: usize, + pages: &mut Vec, + num_rows: u64, +) { + let mut next_column = 0; + let is_header = + is_external_metadata_structural_header(&schema.fields, column_index, &mut next_column) + .unwrap_or(false); + if is_header && !pages.is_empty() { + pages[0].num_rows = num_rows; + pages[0].priority = 0; + pages.truncate(1); + } +} + +/// Compose the v2.0 field encoding mechanisms. +pub fn encoding_strategy() -> Arc { + Arc::new(ArrayFieldEncodingStrategy::new()) +} + +/// Create a v2.0 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.0 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + Writer::new_lazy(object_writer, options) +} + +/// Encode a self-described v2.0 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.0 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_0/reader.rs b/rust/lance-file/src/versions/v2_0/reader.rs new file mode 100644 index 00000000000..f54c972f43b --- /dev/null +++ b/rust/lance-file/src/versions/v2_0/reader.rs @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use lance_core::{ + Error, Result, + cache::LanceCache, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + EncodingsIo, + decoder::{ColumnInfo, PageEncoding, PageInfo}, + format::pb, +}; +use prost::{Message, Name}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, FileMetadataProvider, FileReader, + PreparedProjection, RawFileMetadata, ReadProjection, ReaderProjection, + normalized_column_num_rows, verify_uniform_lengths, + }, + version::ConcreteFileVersion, +}; + +fn fetch_encoding(encoding: &pbfile::Encoding) -> Result { + match &encoding.location { + Some(pbfile::encoding::Location::Indirect(_)) => Err(Error::invalid_input_source( + "Indirect file encodings are not supported".into(), + )), + Some(pbfile::encoding::Location::Direct(encoding)) => { + let envelope = prost_types::Any::decode(Bytes::from(encoding.encoding.clone())) + .map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding envelope: {error}", M::NAME).into(), + ) + })?; + envelope.to_msg::().map_err(|error| { + Error::invalid_input_source( + format!("Invalid direct {} encoding: {error}", M::NAME).into(), + ) + }) + } + Some(pbfile::encoding::Location::None(_)) => Err(Error::invalid_input_source( + format!("Missing {} encoding description", M::NAME).into(), + )), + None => Err(Error::invalid_input_source( + format!("Missing {} encoding location", M::NAME).into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let array_encoding = + fetch_encoding::(page.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!( + "Column {} page {} is missing its encoding", + column_index, page_index + ) + .into(), + ) + })?)?; + if page.buffer_offsets.len() != page.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} page {} has {} buffer offsets but {} buffer sizes", + column_index, + page_index, + page.buffer_offsets.len(), + page.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + page.buffer_offsets + .iter() + .zip(&page.buffer_sizes) + .map(|(offset, size)| (*offset, *size)) + .collect::>(), + ); + Ok(PageInfo { + buffer_offsets_and_sizes, + encoding: PageEncoding::Legacy(array_encoding), + num_rows: page.length, + priority: page.priority, + }) + }) + .collect::>>()?; + + if metadata.buffer_offsets.len() != metadata.buffer_sizes.len() { + return Err(Error::invalid_input_source( + format!( + "Column {} has {} buffer offsets but {} buffer sizes", + column_index, + metadata.buffer_offsets.len(), + metadata.buffer_sizes.len() + ) + .into(), + )); + } + let buffer_offsets_and_sizes = Arc::from( + metadata + .buffer_offsets + .iter() + .zip(&metadata.buffer_sizes) + .map(|(offset, size)| (*offset, *size)) + .collect::>(), + ); + Ok(Arc::new(ColumnInfo { + index: column_index, + page_infos: Arc::from(page_infos), + buffer_offsets_and_sizes, + encoding: fetch_encoding(metadata.encoding.as_ref().ok_or_else(|| { + Error::invalid_input_source( + format!("Column {} is missing its encoding", column_index).into(), + ) + })?)?, + })) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub async fn prepare_projection( + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + _io: &Arc, + _cache: &Arc, +) -> Result { + match metadata_provider { + FileMetadataProvider::Full(metadata) => { + FileReader::validate_projection(projection, metadata)?; + Ok(PreparedProjection { + column_infos: metadata.column_infos.clone(), + decoder_projection: projection.clone(), + }) + } + FileMetadataProvider::Indexed(metadata_index) => { + FileMetadataProvider::validate_indexed_projection_structure( + projection, + metadata_index, + )?; + Err(FileMetadataProvider::indexed_projection_error( + projection, + metadata_index, + )) + } + } +} + +fn field_column_shape(field: &Field) -> (bool, bool) { + if field.is_blob() || field.is_packed_struct() { + return (true, false); + } + (true, !field.children.is_empty()) +} + +fn append_field_ids( + fields: &[Field], + field_id_to_column_index: &BTreeMap, + column_indices: &mut Vec, +) { + for field in fields { + let (contributes, recurse) = field_column_shape(field); + if contributes + && let Some(column_index) = field_id_to_column_index.get(&(field.id as u32)).copied() + { + column_indices.push(column_index); + } + if recurse { + append_field_ids(&field.children, field_id_to_column_index, column_indices); + } + } +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + let mut column_indices = Vec::new(); + append_field_ids( + &schema.fields, + field_id_to_column_index, + &mut column_indices, + ); + ReaderProjection { + schema: Arc::new(schema.clone()), + column_indices, + } +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + let field_id_to_column_index = super::field_id_to_column_index(schema); + let projected = schema.project(column_names)?; + Ok(projection_from_field_ids( + &projected, + &field_id_to_column_index, + )) +} + +fn children_share_parent_length(field: &Field) -> bool { + field.logical_type.is_struct() +} + +fn validate_field_length Result>( + field: &Field, + comparable: bool, + column_indices: &[u32], + cursor: &mut usize, + column_len: &F, +) -> Result { + let (contributes, recurse) = field_column_shape(field); + let mut field_rows = None; + if contributes { + let column = *column_indices.get(*cursor).ok_or_else(|| { + Error::invalid_input(format!( + "projection supplied fewer column indices than its fields require (ran out at field '{}')", + field.name + )) + })?; + *cursor += 1; + field_rows = Some(column_len(column as usize)?); + } + if recurse { + let enforce_children = comparable && children_share_parent_length(field); + for child in &field.children { + let child_rows = + validate_field_length(child, enforce_children, column_indices, cursor, column_len)?; + let expected = *field_rows.get_or_insert(child_rows); + if enforce_children && child_rows != expected { + return Err(Error::invalid_input(format!( + "cannot read field '{}': its children have differing lengths (child '{}' has {} rows, but the field has {}); a struct's children must all have the same length", + field.name, child.name, child_rows, expected + ))); + } + } + } + field_rows.ok_or_else(|| { + Error::invalid_input(format!( + "projected field '{}' maps to no columns", + field.name + )) + }) +} + +pub fn prepared_read_length(prepared: &PreparedProjection) -> Result { + let column_len = |column: usize| { + let info = prepared.column_infos.get(column).ok_or_else(|| { + Error::invalid_input(format!( + "projection references column index {} but only {} columns are available", + column, + prepared.column_infos.len() + )) + })?; + normalized_column_num_rows(info) + }; + let mut cursor = 0; + let mut field_lengths = Vec::with_capacity(prepared.decoder_projection.schema.fields.len()); + for field in &prepared.decoder_projection.schema.fields { + let rows = validate_field_length( + field, + true, + &prepared.decoder_projection.column_indices, + &mut cursor, + &column_len, + )?; + field_lengths.push((field.name.as_str(), rows)); + } + if cursor != prepared.decoder_projection.column_indices.len() { + return Err(Error::invalid_input(format!( + "projection supplied {} column indices but its fields require {}", + prepared.decoder_projection.column_indices.len(), + cursor + ))); + } + verify_uniform_lengths(&field_lengths) +} + +#[derive(Debug)] +struct V20ReadProjection; + +pub(super) fn read_projection() -> Arc { + Arc::new(V20ReadProjection) +} + +#[async_trait] +impl ReadProjection for V20ReadProjection { + fn validate_indexed( + &self, + projection: &ReaderProjection, + metadata_index: &FileMetadataIndex, + ) -> Result<()> { + FileMetadataProvider::validate_indexed_projection_structure(projection, metadata_index)?; + Err(FileMetadataProvider::indexed_projection_error( + projection, + metadata_index, + )) + } + + fn read_length(&self, prepared: &PreparedProjection) -> Result { + prepared_read_length(prepared) + } + + async fn prepare( + &self, + metadata_provider: &FileMetadataProvider, + projection: &ReaderProjection, + io: &Arc, + cache: &Arc, + ) -> Result<(PreparedProjection, u64)> { + let prepared = prepare_projection(metadata_provider, projection, io, cache).await?; + let read_len = self.read_length(&prepared)?; + Ok((prepared, read_len)) + } +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if !matches!( + (raw.footer.major_version, raw.footer.minor_version), + (0, 3) | (2, 0) + ) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.0 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_0, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(_buffers: &[BufferDescriptor]) -> Result<()> { + Ok(()) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version == ConcreteFileVersion::V2_0 { + Ok(index) + } else { + let (major, minor) = index.version.to_standard_footer_numbers(); + Err(Error::version_conflict( + "Attempt to use the Lance v2.0 reader for a different metadata index".to_string(), + major, + minor, + )) + } +} + +#[cfg(test)] +pub fn test_projection_length( + schema: &Schema, + column_indices: &[u32], + column_lengths: &[u64], +) -> Result { + let column_len = |column: usize| { + column_lengths.get(column).copied().ok_or_else(|| { + Error::invalid_input(format!("missing synthetic length for column {column}")) + }) + }; + let mut cursor = 0; + let mut field_lengths = Vec::with_capacity(schema.fields.len()); + for field in &schema.fields { + let rows = validate_field_length(field, true, column_indices, &mut cursor, &column_len)?; + field_lengths.push((field.name.as_str(), rows)); + } + verify_uniform_lengths(&field_lengths) +} diff --git a/rust/lance-file/src/versions/v2_0/writer.rs b/rust/lance-file/src/versions/v2_0/writer.rs new file mode 100644 index 00000000000..e56a8fa2210 --- /dev/null +++ b/rust/lance-file/src/versions/v2_0/writer.rs @@ -0,0 +1,962 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use core::panic; +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_data::ArrayData; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use futures::StreamExt; +use futures::stream::FuturesOrdered; +use lance_core::datatypes::{Field, Schema as LanceSchema}; +use lance_core::utils::bit::pad_bytes; +use lance_core::{Error, Result}; +use lance_encoding::decoder::PageEncoding; +use lance_encoding::encoder::{ + ArrayFieldEncodingStrategy, BatchEncoder, EncodeTask, EncodedBatch, EncodedPage, + EncodingOptions, FieldEncoder, FieldEncodingStrategy, OutOfLineBuffers, +}; +use lance_encoding::repdef::RepDefBuilder; +use lance_io::object_store::ObjectStore; +use lance_io::traits::Writer as ObjectWriter; +use log::{debug, warn}; +use object_store::path::Path; +use prost::Message; +use prost_types::Any; +use tokio::io::AsyncWrite; +use tokio::io::AsyncWriteExt; +use tracing::instrument; + +use crate::datatypes::FieldsWithMeta; +use crate::format::MAGIC; +use crate::format::pb; +use crate::format::pbfile; +use crate::format::pbfile::DirectEncoding; +use crate::writer::{ + ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriteSummary, FileWriterOptions, + PAGE_BUFFER_ALIGNMENT, +}; + +const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT]; +// In 2.1+, we split large pages on read instead of write to avoid empty pages +// and small pages issues. However, we keep the write-time limit at 32MB to avoid +// potential regressions in 2.0 format readers. +// +// This limit is not applied in the 2.1 writer +const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024; +// Total in-memory budget for buffering serialized page metadata before flushing +// to the spill file. Divided evenly across columns (with a floor of 64 bytes). +const DEFAULT_SPILL_BUFFER_LIMIT: usize = 256 * 1024; + +/// Spills serialized page metadata to a temporary file to bound memory usage. +/// +/// The spill file is an unstructured sequence of "chunks". Each chunk is a +/// contiguous run of length-delimited protobuf `Page` messages belonging to a +/// single column. Chunks from different columns are interleaved in the order +/// they are flushed (i.e. whenever a column's in-memory buffer exceeds +/// `per_column_limit`). The `column_chunks` index records the (offset, length) +/// of every chunk so each column's pages can be read back and reassembled in +/// order. +struct PageMetadataSpill { + writer: Box, + object_store: Arc, + path: Path, + /// Current write position in the spill file. + position: u64, + /// Per-column buffer of serialized (length-delimited protobuf) page metadata + /// that has not yet been flushed to the spill file. + column_buffers: Vec>, + /// Per-column list of chunks that have been flushed to the spill file. + /// Each entry is (offset, length) pointing into the spill file. + column_chunks: Vec>, + /// Maximum bytes to buffer per column before flushing to the spill file. + per_column_limit: usize, +} + +impl PageMetadataSpill { + async fn new(object_store: Arc, path: Path, num_columns: usize) -> Result { + let writer = object_store.create(&path).await?; + let per_column_limit = (DEFAULT_SPILL_BUFFER_LIMIT / num_columns.max(1)).max(64); + Ok(Self { + writer, + object_store, + path, + position: 0, + column_buffers: vec![Vec::new(); num_columns], + column_chunks: vec![Vec::new(); num_columns], + per_column_limit, + }) + } + + async fn append_page( + &mut self, + column_idx: usize, + page: &pbfile::column_metadata::Page, + ) -> Result<()> { + page.encode_length_delimited(&mut self.column_buffers[column_idx]) + .map_err(|e| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e, + ))) + })?; + if self.column_buffers[column_idx].len() >= self.per_column_limit { + self.flush_column(column_idx).await?; + } + Ok(()) + } + + async fn flush_column(&mut self, column_idx: usize) -> Result<()> { + let buf = &self.column_buffers[column_idx]; + if buf.is_empty() { + return Ok(()); + } + let len = buf.len(); + self.writer.write_all(buf).await?; + self.column_chunks[column_idx].push((self.position, len as u32)); + self.position += len as u64; + self.column_buffers[column_idx].clear(); + Ok(()) + } + + async fn shutdown_writer(&mut self) -> Result<()> { + for col_idx in 0..self.column_buffers.len() { + self.flush_column(col_idx).await?; + } + ObjectWriter::shutdown(self.writer.as_mut()).await?; + Ok(()) + } +} + +fn decode_spilled_chunk(data: &Bytes) -> Result> { + let mut pages = Vec::new(); + let mut cursor = data.clone(); + while cursor.has_remaining() { + let page = + pbfile::column_metadata::Page::decode_length_delimited(&mut cursor).map_err(|e| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + e, + ))) + })?; + pages.push(page); + } + Ok(pages) +} + +enum PageSpillState { + Pending(Arc, Path), + Active(PageMetadataSpill), +} + +/// A writer for the Lance v2.0 file grammar. +pub struct Writer { + writer: Box, + schema: Option, + column_writers: Vec>, + column_metadata: Vec, + field_id_to_column_indices: Vec<(u32, u32)>, + num_columns: u32, + rows_written: u64, + // The number of rows written for each top-level field (i.e. each entry in + // `column_writers`). With `write_batch` every field advances together and + // these are all equal, but `write_column` advances one field at a time, so + // a single file may end up with columns of differing item counts. + field_rows_written: Vec, + global_buffers: Vec<(u64, u64)>, + schema_metadata: HashMap, + encoding_strategy: Box, + options: FileWriterOptions, + page_spill: Option, +} + +fn initial_column_metadata() -> pbfile::ColumnMetadata { + pbfile::ColumnMetadata { + pages: Vec::new(), + buffer_offsets: Vec::new(), + buffer_sizes: Vec::new(), + encoding: None, + } +} + +impl Writer { + /// Create a new v2.0 writer with a desired output schema. + pub fn try_new( + object_writer: Box, + schema: LanceSchema, + options: FileWriterOptions, + ) -> Result { + let mut writer = Self::new_lazy(object_writer, options); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a new v2.0 writer without a desired output schema. + /// + /// The output schema will be set based on the first batch of data to arrive. + /// If no data arrives and the writer is finished then the write will fail. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self { + writer: object_writer, + schema: None, + column_writers: Vec::new(), + column_metadata: Vec::new(), + num_columns: 0, + rows_written: 0, + field_rows_written: Vec::new(), + field_id_to_column_indices: Vec::new(), + global_buffers: Vec::new(), + schema_metadata: HashMap::new(), + page_spill: None, + encoding_strategy: Box::new(ArrayFieldEncodingStrategy::new()), + options, + } + } + + /// Spill page metadata to a sidecar file instead of accumulating in memory. + /// + /// This can dramatically reduce memory usage when many writers are open + /// concurrently (e.g. IVF shuffle with thousands of partition writers). + /// The sidecar file is created lazily on the first page write. The caller + /// is responsible for cleaning up `path` (e.g. by placing it in a temp + /// directory that is removed via RAII). + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.page_spill = Some(PageSpillState::Pending(object_store, path)); + self + } + + async fn do_write_buffer(writer: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> Result<()> { + writer.write_all(buf).await?; + let pad_bytes = pad_bytes::(buf.len()); + writer.write_all(&PAD_BUFFER[..pad_bytes]).await?; + Ok(()) + } + + async fn write_page(&mut self, encoded_page: EncodedPage) -> Result<()> { + let buffers = encoded_page.data; + let mut buffer_offsets = Vec::with_capacity(buffers.len()); + let mut buffer_sizes = Vec::with_capacity(buffers.len()); + for buffer in buffers { + buffer_offsets.push(self.writer.tell().await? as u64); + buffer_sizes.push(buffer.len() as u64); + Self::do_write_buffer(&mut self.writer, &buffer).await?; + } + let encoded_encoding = match encoded_page.description { + PageEncoding::Legacy(array_encoding) => Any::from_msg(&array_encoding)?.encode_to_vec(), + PageEncoding::Structural(page_layout) => Any::from_msg(&page_layout)?.encode_to_vec(), + }; + let page = pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(DirectEncoding { + encoding: encoded_encoding, + })), + }), + length: encoded_page.num_rows, + priority: encoded_page.row_number, + }; + let col_idx = encoded_page.column_idx as usize; + if matches!(&self.page_spill, Some(PageSpillState::Pending(..))) { + let Some(PageSpillState::Pending(store, path)) = self.page_spill.take() else { + unreachable!() + }; + self.page_spill = Some(PageSpillState::Active( + PageMetadataSpill::new(store, path, self.num_columns as usize).await?, + )); + } + match &mut self.page_spill { + Some(PageSpillState::Active(spill)) => spill.append_page(col_idx, &page).await?, + None => self.column_metadata[col_idx].pages.push(page), + Some(PageSpillState::Pending(..)) => unreachable!(), + } + Ok(()) + } + + #[instrument(skip_all, level = "debug")] + async fn write_pages(&mut self, mut encoding_tasks: FuturesOrdered) -> Result<()> { + // As soon as an encoding task is done we write it. There is no parallelism + // needed here because "writing" is really just submitting the buffer to the + // underlying write scheduler (either the OS or object_store's scheduler for + // cloud writes). The only time we might truly await on write_page is if the + // scheduler's write queue is full. + // + // Also, there is no point in trying to make write_page parallel anyways + // because we wouldn't want buffers getting mixed up across pages. + while let Some(encoding_task) = encoding_tasks.next().await { + let encoded_page = encoding_task?; + self.write_page(encoded_page).await?; + } + // Flushing here reaps any upload that has already failed, so the error + // is attributed to this batch rather than to whichever later batch or + // the shutdown happens to poll the writer next. It does not wait for + // in-flight uploads: those are spawned tasks the runtime drives on its + // own, and blocking on them would stall the next batch behind them. + self.writer.flush().await?; + Ok(()) + } + + /// Schedule batches of data to be written to the file + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Reject a null in a non-nullable field whether or not a null ancestor + /// masks it: the 2.0 logical encoders cannot store such a slot. The 2.1+ + /// structural writer counts only visible nulls (`writer::nullability`). + fn verify_field_nullability(arr: &ArrayData, field: &Field) -> Result<()> { + if !field.nullable && arr.null_count() > 0 { + return Err(Error::invalid_input(format!( + "The field `{}` contained null values even though the field is marked non-null in the schema", + field.name + ))); + } + + for (child_field, child_arr) in field.children.iter().zip(arr.child_data()) { + Self::verify_field_nullability(child_arr, child_field)?; + } + + Ok(()) + } + + fn verify_nullability_constraints(&self, batch: &RecordBatch) -> Result<()> { + for (col, field) in batch + .columns() + .iter() + .zip(self.schema.as_ref().unwrap().fields.iter()) + { + Self::verify_field_nullability(&col.to_data(), field)?; + } + Ok(()) + } + + fn initialize(&mut self, mut schema: LanceSchema) -> Result<()> { + let cache_bytes_per_column = if let Some(data_cache_bytes) = self.options.data_cache_bytes { + data_cache_bytes / schema.fields.len() as u64 + } else { + 8 * 1024 * 1024 + }; + + let max_page_bytes = self.options.max_page_bytes.unwrap_or_else(|| { + std::env::var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES) + .map(|s| { + s.parse::().unwrap_or_else(|e| { + warn!( + "Failed to parse {}: {}, using default", + ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, e + ); + MAX_PAGE_BYTES as u64 + }) + }) + .unwrap_or(MAX_PAGE_BYTES as u64) + }); + + schema.validate()?; + + let keep_original_array = self.options.keep_original_array.unwrap_or(false); + let encoding_options = EncodingOptions { + cache_bytes_per_column, + max_page_bytes, + keep_original_array, + buffer_alignment: PAGE_BUFFER_ALIGNMENT as u64, + }; + let encoder = + BatchEncoder::try_new(&schema, self.encoding_strategy.as_ref(), &encoding_options)?; + self.num_columns = encoder.num_columns(); + + self.field_rows_written = vec![0; encoder.field_encoders.len()]; + self.column_writers = encoder.field_encoders; + self.column_metadata = vec![initial_column_metadata(); self.num_columns as usize]; + self.field_id_to_column_indices = encoder.field_id_to_column_index; + self.schema_metadata + .extend(std::mem::take(&mut schema.metadata)); + self.schema = Some(schema); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<&LanceSchema> { + if self.schema.is_none() { + let schema = LanceSchema::try_from(batch.schema().as_ref())?; + self.initialize(schema)?; + } + Ok(self.schema.as_ref().unwrap()) + } + + #[instrument(skip_all, level = "debug")] + fn encode_batch( + &mut self, + batch: &RecordBatch, + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + let field_arrays = self + .schema + .as_ref() + .unwrap() + .fields + .iter() + .enumerate() + .map(|(field_idx, field)| { + let array = + batch + .column_by_name(&field.name) + .ok_or(Error::invalid_input_source( + format!( + "Cannot write batch. The batch was missing the column `{}`", + field.name + ) + .into(), + ))?; + Ok((field_idx, array.clone())) + }) + .collect::>>()?; + self.encode_columns(&field_arrays, external_buffers) + } + + // Encode a set of `(field index, array)` pairs, each advancing only its own + // column. Each task captures its field's current row offset at encode time, + // so `advance_columns` must run after this call (never before); the order of + // the returned tasks relative to `write_pages` does not matter. + fn encode_columns( + &mut self, + field_arrays: &[(usize, ArrayRef)], + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + // Snapshot the starting row number of each field before borrowing the + // column writers mutably below. + let row_numbers = field_arrays + .iter() + .map(|(field_idx, _)| self.field_rows_written[*field_idx]) + .collect::>(); + field_arrays + .iter() + .zip(row_numbers) + .map(|((field_idx, array), row_number)| { + let repdef = RepDefBuilder::default(); + let num_rows = array.len() as u64; + self.column_writers[*field_idx].maybe_encode( + array.clone(), + external_buffers, + repdef, + row_number, + num_rows, + ) + }) + .collect::>>() + } + + // Advance the per-field row counters after a set of columns has been + // written, keeping `rows_written` (the file's logical length) in sync as the + // longest column. Only the written fields move, so their new totals fold into + // `rows_written` directly without rescanning every field. (`write_batch` + // advances every field uniformly and tracks this inline instead.) + fn advance_columns(&mut self, field_arrays: &[(usize, ArrayRef)]) { + for (field_idx, array) in field_arrays { + let new_total = self.field_rows_written[*field_idx] + array.len() as u64; + self.field_rows_written[*field_idx] = new_total; + self.rows_written = self.rows_written.max(new_total); + } + } + + /// Schedule a batch of data to be written to the file + /// + /// Note: the future returned by this method may complete before the data has been fully + /// flushed to the file (some data may be in the data cache or the I/O cache) + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + debug!( + "write_batch called with {} rows, {} columns, and {} bytes of data", + batch.num_rows(), + batch.num_columns(), + batch.get_array_memory_size() + ); + self.ensure_initialized(batch)?; + self.verify_nullability_constraints(batch)?; + let num_rows = batch.num_rows() as u64; + if num_rows == 0 { + return Ok(()); + } + if num_rows > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + // First we push each array into its column writer. This may or may not generate enough + // data to trigger an encoding task. We collect any encoding tasks into a queue. + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_batch(batch, &mut external_buffers)?; + // Next, write external buffers + for external_buffer in external_buffers.take_buffers() { + Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + } + + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + + // `write_batch` advances every field by the same amount, so the longest + // column simply grows by `num_rows`. Guard against overflowing the row + // counter. + if self.rows_written.checked_add(num_rows).is_none() { + return Err(Error::invalid_input_source(format!("cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", num_rows, self.rows_written).into())); + } + for field_rows in self.field_rows_written.iter_mut() { + *field_rows += num_rows; + } + self.rows_written += num_rows; + + self.write_pages(encoding_tasks).await?; + + Ok(()) + } + + /// Write a single column, advancing only that column's row counter. + /// + /// Unlike [`write_batch`](Self::write_batch), which advances every column + /// from a single shared row counter, this method advances one column + /// independently. Used across calls it produces a single file whose columns + /// may have different item counts. + /// + /// `column_index` refers to a top-level field in the writer's schema (the + /// same order as the schema's fields); a nested child cannot be targeted on + /// its own. Because each call writes the whole field from a single array, the + /// children of a struct field always advance together and stay equal-length; + /// only different top-level fields can diverge in length. A column may be + /// written across multiple calls; its values are appended. A field that is + /// never written ends up as a zero-length column. The writer must have been + /// created with an explicit schema (via [`try_new`](Self::try_new)); a lazy + /// schema cannot be inferred here because individual calls need not cover + /// every field. + /// + /// ``` + /// # use arrow_array::{ArrayRef, Int32Array}; + /// # use std::sync::Arc; + /// # use lance_file::writer::FileWriter; + /// # async fn example(writer: &mut FileWriter) -> lance_core::Result<()> { + /// // Field 0 gets three values, field 1 gets one — a non-rectangular file. + /// writer.write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))).await?; + /// writer.write_column(1, Arc::new(Int32Array::from(vec![10]))).await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + let schema = self.schema.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "write_column requires the writer to be created with an explicit schema".into(), + ) + })?; + let field = schema.fields.get(column_index).ok_or_else(|| { + Error::invalid_input_source( + format!( + "write_column: field index {} is out of bounds (schema has {} fields)", + column_index, + schema.fields.len() + ) + .into(), + ) + })?; + if array.len() as u64 > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + Self::verify_field_nullability(&array.to_data(), field)?; + + // A never-advanced field simply remains a zero-length column, which the + // encoders handle at `finish` time. + if array.is_empty() { + return Ok(()); + } + + let columns = [(column_index, array)]; + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_columns(&columns, &mut external_buffers)?; + for external_buffer in external_buffers.take_buffers() { + Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + + self.advance_columns(&columns); + self.write_pages(encoding_tasks).await?; + Ok(()) + } + + async fn write_column_metadata( + &mut self, + metadata: pbfile::ColumnMetadata, + ) -> Result<(u64, u64)> { + let metadata_bytes = metadata.encode_to_vec(); + let position = self.writer.tell().await? as u64; + let len = metadata_bytes.len() as u64; + self.writer.write_all(&metadata_bytes).await?; + Ok((position, len)) + } + + async fn write_column_metadatas(&mut self) -> Result> { + let metadatas = std::mem::take(&mut self.column_metadata); + + // If spilling, finalize the spill writer and reopen for reading. + // The spill file itself is cleaned up by the caller (it lives in a + // temp directory managed by the caller's RAII guard). + let spill_state = self.page_spill.take(); + let (spill_chunks, spill_reader) = + if let Some(PageSpillState::Active(mut spill)) = spill_state { + spill.shutdown_writer().await?; + let reader = spill.object_store.open(&spill.path).await?; + let chunks = std::mem::take(&mut spill.column_chunks); + (chunks, Some(reader)) + } else { + (Vec::new(), None) + }; + + let mut metadata_positions = Vec::with_capacity(metadatas.len()); + for (col_idx, mut metadata) in metadatas.into_iter().enumerate() { + if let Some(reader) = &spill_reader { + let mut pages = Vec::new(); + for &(offset, len) in &spill_chunks[col_idx] { + let data = reader + .get_range(offset as usize..(offset as usize + len as usize)) + .await + .map_err(|e| Error::io_source(Box::new(e)))?; + pages.extend(decode_spilled_chunk(&data)?); + } + metadata.pages = pages; + } + metadata_positions.push(self.write_column_metadata(metadata).await?); + } + + Ok(metadata_positions) + } + + fn make_file_descriptor( + schema: &lance_core::datatypes::Schema, + num_rows: u64, + ) -> Result { + let fields_with_meta = FieldsWithMeta::from(schema); + Ok(pb::FileDescriptor { + schema: Some(pb::Schema { + fields: fields_with_meta.fields.0, + metadata: fields_with_meta.metadata, + }), + length: num_rows, + }) + } + + async fn write_global_buffers(&mut self) -> Result> { + let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided. Schema is unknown and file cannot be created"))?; + schema.metadata = std::mem::take(&mut self.schema_metadata); + // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields. + schema + .fields + .iter_mut() + .for_each(|f| f.unload_blobs_recursive()); + + let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?; + let file_descriptor_bytes = file_descriptor.encode_to_vec(); + let file_descriptor_len = file_descriptor_bytes.len() as u64; + let file_descriptor_position = self.writer.tell().await? as u64; + self.writer.write_all(&file_descriptor_bytes).await?; + let mut gbo_table = Vec::with_capacity(1 + self.global_buffers.len()); + gbo_table.push((file_descriptor_position, file_descriptor_len)); + gbo_table.append(&mut self.global_buffers); + Ok(gbo_table) + } + + /// Add a metadata entry to the schema + /// + /// This method is useful because sometimes the metadata is not known until after the + /// data has been written. This method allows you to alter the schema metadata. It + /// must be called before `finish` is called. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.schema_metadata.insert(key.into(), value.into()); + } + + /// Prepare the writer when column data and metadata were produced externally. + /// + /// This is useful for flows that copy already-encoded pages (e.g., binary copy + /// during compaction) where the column buffers have been written directly and we + /// only need to write the footer and schema metadata. The provided + /// `column_metadata` must describe the buffers already persisted by the + /// underlying `ObjectWriter`, and `rows_written` should reflect the total number + /// of rows in those buffers. + pub fn initialize_with_external_metadata( + &mut self, + mut schema: lance_core::datatypes::Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.schema_metadata + .extend(std::mem::take(&mut schema.metadata)); + self.schema = Some(schema); + self.num_columns = column_metadata.len() as u32; + self.column_metadata = column_metadata; + self.rows_written = rows_written; + } + + /// Adds a global buffer to the file + /// + /// The global buffer can contain any arbitrary bytes. It will be written to the disk + /// immediately. This method returns the index of the global buffer (this will always + /// start at 1 and increment by 1 each time this method is called) + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + let position = self.writer.tell().await? as u64; + let len = buffer.len() as u64; + Self::do_write_buffer(&mut self.writer, &buffer).await?; + self.global_buffers.push((position, len)); + Ok(self.global_buffers.len() as u32) + } + + async fn finish_writers(&mut self) -> Result<()> { + let mut col_idx = 0; + for mut writer in std::mem::take(&mut self.column_writers) { + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let columns = writer.finish(&mut external_buffers).await?; + for buffer in external_buffers.take_buffers() { + self.writer.write_all(&buffer).await?; + } + debug_assert_eq!( + columns.len(), + writer.num_columns() as usize, + "Expected {} columns from column at index {} and got {}", + writer.num_columns(), + col_idx, + columns.len() + ); + for column in columns { + for page in column.final_pages { + self.write_page(page).await?; + } + let column_metadata = &mut self.column_metadata[col_idx]; + let mut buffer_pos = self.writer.tell().await? as u64; + for buffer in column.column_buffers { + column_metadata.buffer_offsets.push(buffer_pos); + let mut size = 0; + Self::do_write_buffer(&mut self.writer, &buffer).await?; + size += buffer.len() as u64; + buffer_pos += size; + column_metadata.buffer_sizes.push(size); + } + let encoded_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + column_metadata.encoding = Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_encoding, + })), + }); + col_idx += 1; + } + } + if col_idx != self.column_metadata.len() { + panic!( + "Column writers finished with {} columns but we expected {}", + col_idx, + self.column_metadata.len() + ); + } + Ok(()) + } + + /// Finishes writing the file + /// + /// This method will wait until all data has been flushed to the file. Then it + /// will write the file metadata and the footer. It will not return until all + /// data has been flushed and the file has been closed. + /// + /// Returns a summary of the completed file write. + pub async fn finish(&mut self) -> Result { + // 1. flush any remaining data and write out those pages + let mut external_buffers = + OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self + .column_writers + .iter_mut() + .map(|writer| writer.flush(&mut external_buffers)) + .collect::>>()?; + for external_buffer in external_buffers.take_buffers() { + Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + self.write_pages(encoding_tasks).await?; + + if !self.column_writers.is_empty() { + self.finish_writers().await?; + } + + // 3. write global buffers (we write the schema here) + let global_buffer_offsets = self.write_global_buffers().await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + // 4. write the column metadatas + let column_metadata_start = self.writer.tell().await? as u64; + let metadata_positions = self.write_column_metadatas().await?; + + // 5. write the column metadata offset table + let cmo_table_start = self.writer.tell().await? as u64; + for (meta_pos, meta_len) in metadata_positions { + self.writer.write_u64_le(meta_pos).await?; + self.writer.write_u64_le(meta_len).await?; + } + + // 6. write global buffers offset table + let gbo_table_start = self.writer.tell().await? as u64; + for (gbo_pos, gbo_len) in global_buffer_offsets { + self.writer.write_u64_le(gbo_pos).await?; + self.writer.write_u64_le(gbo_len).await?; + } + + // 7. write the footer + self.writer.write_u64_le(column_metadata_start).await?; + self.writer.write_u64_le(cmo_table_start).await?; + self.writer.write_u64_le(gbo_table_start).await?; + self.writer.write_u32_le(num_global_buffers).await?; + self.writer.write_u32_le(self.num_columns).await?; + self.writer.write_u16_le(0).await?; + self.writer.write_u16_le(3).await?; + self.writer.write_all(MAGIC).await?; + + // 7. close the writer + let write_result = ObjectWriter::shutdown(self.writer.as_mut()).await?; + + Ok(FileWriteSummary { + num_rows: self.rows_written, + size_bytes: write_result.size as u64, + }) + } + + pub async fn abort(&mut self) { + // For multipart uploads, ObjectWriter's Drop impl will abort + // the upload when the writer is dropped. + } + + pub async fn tell(&mut self) -> Result { + Ok(self.writer.tell().await? as u64) + } + + /// Append a buffer whose metadata is supplied by the caller. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + let start = self.tell().await?; + self.writer.write_all(bytes).await?; + Ok((start, bytes.len() as u64)) + } + + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + &self.field_id_to_column_indices + } +} + +// Creates a lance footer and appends it to the encoded data +// +// The logic here is very similar to logic in the FileWriter except we +// are using BufMut (put_xyz) instead of AsyncWrite (write_xyz). +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + // Estimating 1MiB for file footer + let mut data = BytesMut::with_capacity(batch.data.len() + 1024 * 1024); + data.put(batch.data.clone()); + // write global buffers (we write the schema here) + let global_buffers = if write_schema { + let schema_start = data.len() as u64; + let lance_schema = lance_core::datatypes::Schema::try_from(batch.schema.as_ref())?; + let descriptor = Writer::make_file_descriptor(&lance_schema, batch.num_rows)?; + let descriptor_bytes = descriptor.encode_to_vec(); + let descriptor_len = descriptor_bytes.len() as u64; + data.put(descriptor_bytes.as_slice()); + + vec![(schema_start, descriptor_len)] + } else { + vec![] + }; + let col_metadata_start = data.len() as u64; + + let mut col_metadata_positions = Vec::new(); + // Write column metadata + for col in &batch.page_table { + let position = data.len() as u64; + let pages = col + .page_infos + .iter() + .map(|page_info| { + let encoded_encoding = match &page_info.encoding { + PageEncoding::Legacy(array_encoding) => { + Any::from_msg(array_encoding)?.encode_to_vec() + } + PageEncoding::Structural(page_layout) => { + Any::from_msg(page_layout)?.encode_to_vec() + } + }; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = page_info + .buffer_offsets_and_sizes + .as_ref() + .iter() + .cloned() + .unzip(); + Ok(pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(DirectEncoding { + encoding: encoded_encoding, + })), + }), + length: page_info.num_rows, + priority: page_info.priority, + }) + }) + .collect::>>()?; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = + col.buffer_offsets_and_sizes.iter().cloned().unzip(); + let encoded_col_encoding = Any::from_msg(&col.encoding)?.encode_to_vec(); + let column = pbfile::ColumnMetadata { + pages, + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_col_encoding, + })), + }), + }; + let column_bytes = column.encode_to_vec(); + col_metadata_positions.push((position, column_bytes.len() as u64)); + data.put(column_bytes.as_slice()); + } + // Write column metadata offsets table + let cmo_table_start = data.len() as u64; + for (meta_pos, meta_len) in col_metadata_positions { + data.put_u64_le(meta_pos); + data.put_u64_le(meta_len); + } + // Write global buffers offsets table + let gbo_table_start = data.len() as u64; + let num_global_buffers = global_buffers.len() as u32; + for (gbo_pos, gbo_len) in global_buffers { + data.put_u64_le(gbo_pos); + data.put_u64_le(gbo_len); + } + + // write the footer + data.put_u64_le(col_metadata_start); + data.put_u64_le(cmo_table_start); + data.put_u64_le(gbo_table_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(batch.page_table.len() as u32); + data.put_u16_le(2); + data.put_u16_le(0); + data.put(MAGIC.as_slice()); + + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/versions/v2_1/compression.rs b/rust/lance-file/src/versions/v2_1/compression.rs new file mode 100644 index 00000000000..2ef40ed0241 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/compression.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + reject_packed_struct_per_value, try_bitpacking_block, try_bitpacking_miniblock, + try_byte_stream_split_miniblock, try_fixed_packed_struct_miniblock, + try_fixed_u8_rle_miniblock, try_raw_block, try_raw_fixed_size_list_miniblock, + try_raw_fixed_width_miniblock, try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_width_miniblock, try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, +}; + +#[derive(Debug, Clone)] +pub(super) struct Strategy { + params: CompressionParams, +} + +impl Strategy { + pub(super) fn new(params: CompressionParams) -> Self { + Self { params } + } + + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + let mut metadata = field_metadata_params(field); + if metadata + .minichunk_size + .is_some_and(|size| size >= 32 * 1024) + { + log::warn!( + "minichunk_size '{}' is too large for the selected u16 miniblock layout, using default", + metadata.minichunk_size.unwrap() + ); + metadata.minichunk_size = None; + } + params.merge(&metadata); + params + } +} + +impl CompressionStrategy for Strategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_fixed_u8_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + if let Some(compressor) = reject_packed_struct_per_value(field, data)? { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let _params = self.field_params(field); + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} diff --git a/rust/lance-file/src/versions/v2_1/mod.rs b/rust/lance-file/src/versions/v2_1/mod.rs new file mode 100644 index 00000000000..0555f3d0264 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/mod.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.1 file composition. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{ + ColumnIndexSequence, EncodedBatch, FieldEncoder, FieldEncodingContext, + FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_struct, + }, + }, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{ + reader::{ReadProjection, structural}, + writer::FileWriterOptions, +}; + +mod compression; +mod reader; +mod writer; + +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; +pub use writer::Writer; + +#[cfg(test)] +pub(crate) use reader::test_projection_length; +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; + +pub(crate) fn read_projection() -> Arc { + structural::read_projection(reader::decode_column) +} + +/// Count physical columns represented by a field in a v2.1 footer. +pub fn physical_column_count(field: &Field) -> usize { + structural::physical_column_count(field) +} + +/// Build persisted field-to-column entries for a v2.1 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + structural::data_file_columns(schema) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + structural::field_id_to_column_index(schema) +} + +#[derive(Debug)] +struct FieldStrategy { + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for FieldStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if matches!( + field.data_type(), + arrow_schema::DataType::FixedSizeList(item, _) + if matches!(item.data_type(), arrow_schema::DataType::Struct(_)) + ) { + return Err(Error::not_supported_source( + "FixedSizeList is not enabled by the selected file format".into(), + )); + } + if matches!(field.data_type(), arrow_schema::DataType::Map(_, _)) { + return Err(Error::not_supported_source( + "Map data type is not enabled by the selected file format".into(), + )); + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "Lance v2.1 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )) + } +} + +/// Compose the v2.1 field encoding mechanisms. +pub fn encoding_strategy(params: CompressionParams) -> Arc { + let compression = Arc::new(compression::Strategy::new(params)); + Arc::new(FieldStrategy { + primitive: PrimitiveFieldEncoding::new([ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::dense_u16(compression), + ]), + }) +} + +/// Create a v2.1 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.1 writer with explicit compression tuning. +pub fn create_writer_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, +) -> Result { + Writer::try_new_with_compression(object_writer, schema, options, compression) +} + +/// Create a v2.1 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + Writer::new_lazy(object_writer, options) +} + +/// Create a lazy v2.1 writer with explicit compression tuning. +pub fn create_lazy_writer_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, +) -> Writer { + Writer::new_lazy_with_compression(object_writer, options, compression) +} + +/// Encode a self-described v2.1 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.1 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_1/reader.rs b/rust/lance-file/src/versions/v2_1/reader.rs new file mode 100644 index 00000000000..0c11e5f8976 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/reader.rs @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{decoder::ColumnInfo, format::pb21}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, RawFileMetadata, ReaderProjection, + structural, + }, + version::ConcreteFileVersion, +}; + +fn required<'a, T>(value: Option<&'a T>, label: &str) -> Result<&'a T> { + value.ok_or_else(|| { + Error::invalid_input_source( + format!("Lance v2.1 {label} is missing its nested encoding").into(), + ) + }) +} + +fn validate_compressive_encoding(encoding: &pb21::CompressiveEncoding) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::Flat(_)) + | Some(Compression::InlineBitpacking(_)) + | Some(Compression::Constant(_)) => Ok(()), + Some(Compression::Variable(variable)) => validate_compressive_encoding(required( + variable.offsets.as_deref(), + "variable offsets", + )?), + Some(Compression::OutOfLineBitpacking(bitpacking)) => { + validate_compressive_encoding(required( + bitpacking.values.as_deref(), + "out-of-line bitpacking values", + )?) + } + Some(Compression::Fsst(fsst)) => { + validate_compressive_encoding(required(fsst.values.as_deref(), "FSST values")?) + } + Some(Compression::Dictionary(dictionary)) => { + validate_compressive_encoding(required( + dictionary.indices.as_deref(), + "dictionary indices", + )?)?; + validate_compressive_encoding(required( + dictionary.items.as_deref(), + "dictionary items", + )?) + } + Some(Compression::Rle(rle)) => { + let values = required(rle.values.as_deref(), "RLE values")?; + let run_lengths = required(rle.run_lengths.as_deref(), "RLE run lengths")?; + let fixed_values = matches!( + values.compression.as_ref(), + Some(Compression::Flat(flat)) + if matches!(flat.bits_per_value, 8 | 16 | 32 | 64) + && flat.data.is_none() + ); + let fixed_u8_lengths = matches!( + run_lengths.compression.as_ref(), + Some(Compression::Flat(flat)) + if flat.bits_per_value == 8 && flat.data.is_none() + ); + if !fixed_values || !fixed_u8_lengths { + return Err(Error::invalid_input_source( + "Lance v2.1 RLE requires flat values and flat u8 run lengths".into(), + )); + } + Ok(()) + } + Some(Compression::ByteStreamSplit(split)) => validate_compressive_encoding(required( + split.values.as_deref(), + "byte-stream-split values", + )?), + Some(Compression::General(general)) => validate_compressive_encoding(required( + general.values.as_deref(), + "general-compression values", + )?), + Some(Compression::FixedSizeList(list)) => validate_compressive_encoding(required( + list.values.as_deref(), + "fixed-size-list values", + )?), + Some(Compression::PackedStruct(packed)) => validate_compressive_encoding(required( + packed.values.as_deref(), + "packed-struct values", + )?), + Some(Compression::VariablePackedStruct(_)) => Err(Error::invalid_input_source( + "Variable packed struct compression is not part of the Lance v2.1 grammar".into(), + )), + None => Err(Error::invalid_input_source( + "Lance v2.1 compressive encoding is missing its compression variant".into(), + )), + } +} + +fn validate_page_layout(layout: &pb21::PageLayout) -> Result<()> { + use pb21::page_layout::Layout; + + match layout.layout.as_ref() { + Some(Layout::MiniBlockLayout(miniblock)) => { + if miniblock.has_large_chunk { + return Err(Error::invalid_input_source( + "Large miniblock chunks are not part of the Lance v2.1 grammar".into(), + )); + } + if let Some(rep) = miniblock.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = miniblock.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + validate_compressive_encoding(required( + miniblock.value_compression.as_ref(), + "miniblock values", + )?)?; + if let Some(dictionary) = miniblock.dictionary.as_ref() { + validate_compressive_encoding(dictionary)?; + } + Ok(()) + } + Some(Layout::FullZipLayout(fullzip)) => validate_compressive_encoding(required( + fullzip.value_compression.as_ref(), + "full-zip values", + )?), + Some(Layout::ConstantLayout(constant)) => { + if constant.inline_value.is_some() { + Err(Error::invalid_input_source( + "Lance v2.1 only accepts the all-null form of constant page layout".into(), + )) + } else if constant.rep_compression.is_some() || constant.def_compression.is_some() { + Err(Error::invalid_input_source( + "Compressed constant-page levels are not part of the Lance v2.1 grammar".into(), + )) + } else { + Ok(()) + } + } + Some(Layout::BlobLayout(blob)) => { + let inner = blob.inner_layout.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Lance v2.1 blob page layout is missing its inner layout".into(), + ) + })?; + validate_page_layout(inner) + } + Some(Layout::SparseLayout(_)) => Err(Error::invalid_input_source( + "Sparse page layout is not part of the Lance v2.1 grammar".into(), + )), + None => Err(Error::invalid_input_source( + "Lance v2.1 page is missing its page layout".into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let page_layout = structural::decode_page_layout(column_index, page_index, page)?; + validate_page_layout(&page_layout)?; + structural::build_page_info(column_index, page_index, page, page_layout) + }) + .collect::>>()?; + structural::build_column_info(column_index, metadata, page_infos) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + structural::projection_from_field_ids(schema, field_id_to_column_index) +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + structural::projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + structural::projection_from_column_names( + schema, + column_names, + &super::field_id_to_column_index(schema), + ) +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if (raw.footer.major_version, raw.footer.minor_version) != (2, 1) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.1 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + validate_global_buffers(&raw.file_buffers)?; + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_1, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + structural::validate_global_buffers(buffers) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version != ConcreteFileVersion::V2_1 { + let (major, minor) = index.version.to_standard_footer_numbers(); + return Err(Error::version_conflict( + "Attempt to use the Lance v2.1 reader for a different metadata index".to_string(), + major, + minor, + )); + } + validate_global_buffers(&index.file_buffers)?; + Ok(index) +} + +#[cfg(test)] +pub fn test_projection_length( + schema: &Schema, + column_indices: &[u32], + column_lengths: &[u64], +) -> Result { + structural::test_projection_length(schema, column_indices, column_lengths) +} + +#[cfg(test)] +mod grammar_tests { + use super::*; + use pb21::{ + CompressiveEncoding, FullZipLayout, General, PageLayout, VariablePackedStruct, + compressive_encoding::Compression, page_layout::Layout, + }; + + #[test] + fn rejects_nested_variable_packed_struct() { + let variable_packed = CompressiveEncoding { + compression: Some(Compression::VariablePackedStruct(VariablePackedStruct { + fields: Vec::new(), + })), + }; + let wrapped = CompressiveEncoding { + compression: Some(Compression::General(Box::new(General { + compression: None, + values: Some(Box::new(variable_packed)), + }))), + }; + let layout = PageLayout { + layout: Some(Layout::FullZipLayout(FullZipLayout { + value_compression: Some(wrapped), + ..Default::default() + })), + }; + + let error = validate_page_layout(&layout).unwrap_err(); + assert!( + error + .to_string() + .contains("Variable packed struct compression is not part") + ); + } +} diff --git a/rust/lance-file/src/versions/v2_1/writer.rs b/rust/lance-file/src/versions/v2_1/writer.rs new file mode 100644 index 00000000000..1599a9fd6e1 --- /dev/null +++ b/rust/lance-file/src/versions/v2_1/writer.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use bytes::{BufMut, Bytes}; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{BatchEncoder, EncodedBatch}, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use object_store::path::Path; +use tokio::io::AsyncWriteExt; + +use crate::{ + format::{MAGIC, pbfile}, + writer::{ + FileWriteSummary, FileWriterOptions, + structural::{EncodedBatchBody, EncodingPipeline, StructuralFileSink, encode_batch_body}, + }, +}; + +use super::encoding_strategy; + +/// A writer for the Lance v2.1 file grammar. +/// +/// The concrete writer owns the v2.1 encoding composition, finish ordering, +/// and exact footer identity. Shared components only execute the structural +/// encoding and I/O mechanisms selected here. +pub struct Writer { + sink: StructuralFileSink, + encoding: EncodingPipeline, + compression: CompressionParams, +} + +impl Writer { + /// Create a v2.1 writer with an explicit schema. + pub fn try_new( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + ) -> Result { + Self::try_new_with_compression(object_writer, schema, options, Default::default()) + } + + /// Create a v2.1 writer with explicit compression tuning. + pub fn try_new_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Result { + let mut writer = Self::new_lazy_with_compression(object_writer, options, compression); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a v2.1 writer whose schema is inferred from the first batch. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self::new_lazy_with_compression(object_writer, options, Default::default()) + } + + /// Create a lazy v2.1 writer with explicit compression tuning. + pub fn new_lazy_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Self { + Self { + sink: StructuralFileSink::new(object_writer), + encoding: EncodingPipeline::new(options), + compression, + } + } + + fn initialize(&mut self, schema: Schema) -> Result<()> { + let encoding_options = self.encoding.encoding_options(&schema); + schema.validate()?; + let strategy = encoding_strategy(self.compression.clone()); + let encoder = BatchEncoder::try_new(&schema, strategy.as_ref(), &encoding_options)?; + self.encoding.initialize(schema, encoder, &mut self.sink); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<()> { + if !self.encoding.is_initialized() { + self.initialize(Schema::try_from(batch.schema().as_ref())?)?; + } + Ok(()) + } + + /// Spill page metadata to a sidecar file instead of retaining it in memory. + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.sink.with_page_metadata_spill(object_store, path); + self + } + + /// Schedule batches to be written in iteration order. + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Schedule one record batch for writing. + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + self.ensure_initialized(batch)?; + self.encoding.write_batch(batch, &mut self.sink).await + } + + /// Write one top-level field, advancing only that field's row count. + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + self.encoding + .write_column(column_index, array, &mut self.sink) + .await + } + + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + self.sink.write_external_buffer(bytes).await + } + + /// Add an entry to the schema metadata written in the file descriptor. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.encoding.add_schema_metadata(key, value); + } + + /// Prepare the writer for encoded column data produced externally. + pub fn initialize_with_external_metadata( + &mut self, + schema: Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.encoding + .initialize_with_external_metadata(schema, rows_written); + self.sink.initialize_with_external_metadata(column_metadata); + } + + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.sink.add_global_buffer(buffer).await + } + + /// Finish the v2.1 file and close its object writer. + pub async fn finish(&mut self) -> Result { + // The order below is the v2.1 wire contract. + self.encoding.flush(&mut self.sink).await?; + self.encoding.finish_encoders(&mut self.sink).await?; + + let descriptor = self.encoding.make_file_descriptor()?; + let global_buffer_offsets = self.sink.write_global_buffers(descriptor).await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + let column_metadata_start = self.sink.tell().await?; + let column_metadata_offsets = self.sink.write_column_metadatas().await?; + let column_metadata_offsets_start = self + .sink + .write_offset_table(&column_metadata_offsets) + .await?; + let global_buffer_offsets_start = + self.sink.write_offset_table(&global_buffer_offsets).await?; + let num_columns = self.sink.num_columns(); + + let output = self.sink.output_mut(); + output.write_u64_le(column_metadata_start).await?; + output.write_u64_le(column_metadata_offsets_start).await?; + output.write_u64_le(global_buffer_offsets_start).await?; + output.write_u32_le(num_global_buffers).await?; + output.write_u32_le(num_columns).await?; + output.write_u16_le(2).await?; + output.write_u16_le(1).await?; + output.write_all(MAGIC).await?; + + Ok(FileWriteSummary { + num_rows: self.encoding.rows_written(), + size_bytes: self.sink.shutdown().await?, + }) + } + + /// Abandon this write. + pub async fn abort(&mut self) { + // Dropping a multipart ObjectWriter aborts the upload. + } + + /// Return the current object-writer position. + pub async fn tell(&mut self) -> Result { + self.sink.tell().await + } + + /// Return the field-id to physical-column mapping selected by v2.1. + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + self.encoding.field_id_to_column_indices() + } +} + +/// Append a self-described or mini-lance v2.1 footer to an encoded batch. +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + let EncodedBatchBody { + mut data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns, + } = encode_batch_body(batch, write_schema)?; + + data.put_u64_le(column_metadata_start); + data.put_u64_le(column_metadata_offsets_start); + data.put_u64_le(global_buffer_offsets_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(num_columns); + data.put_u16_le(2); + data.put_u16_le(1); + data.extend_from_slice(MAGIC); + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/versions/v2_2/compression.rs b/rust/lance-file/src/versions/v2_2/compression.rs new file mode 100644 index 00000000000..aa8e3b64958 --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/compression.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + try_bitpacking_block, try_bitpacking_miniblock, try_byte_stream_split_miniblock, + try_fixed_packed_struct_miniblock, try_fixed_u8_rle_block, try_fixed_u8_rle_miniblock, + try_general_block, try_raw_block, try_raw_fixed_size_list_miniblock, + try_raw_fixed_width_miniblock, try_raw_per_value, try_uncompressed_fixed_width_miniblock, + try_variable_packed_struct_per_value, try_variable_width_miniblock, + try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, +}; + +#[derive(Debug, Clone)] +pub(super) struct Strategy { + params: CompressionParams, +} + +impl Strategy { + pub(super) fn new(params: CompressionParams) -> Self { + Self { params } + } + + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + params.merge(&field_metadata_params(field)); + params + } +} + +impl CompressionStrategy for Strategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_fixed_u8_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + if let Some(compressor) = + try_variable_packed_struct_per_value(Arc::new(self.clone()), field, data)? + { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_fixed_u8_rle_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if let Some(compressor) = try_general_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} diff --git a/rust/lance-file/src/versions/v2_2/mod.rs b/rust/lance-file/src/versions/v2_2/mod.rs new file mode 100644 index 00000000000..b56f78a5c4d --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/mod.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.2 file composition. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::Bytes; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{ + ColumnIndexSequence, EncodedBatch, FieldEncoder, FieldEncodingContext, + FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, + }, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{ + reader::{ReadProjection, structural}, + writer::FileWriterOptions, +}; + +mod compression; +mod reader; +mod writer; + +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; + +pub(crate) fn read_projection() -> Arc { + structural::read_projection(reader::decode_column) +} +pub use writer::Writer; + +/// Count physical columns represented by a field in a v2.2 footer. +pub fn physical_column_count(field: &Field) -> usize { + structural::physical_column_count(field) +} + +/// Build persisted field-to-column entries for a v2.2 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + structural::data_file_columns(schema) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + structural::field_id_to_column_index(schema) +} + +#[derive(Debug)] +struct FieldStrategy { + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for FieldStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "Lance v2.2 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )) + } +} + +/// Compose the v2.2 field encoding mechanisms. +pub fn encoding_strategy(params: CompressionParams) -> Arc { + let compression = Arc::new(compression::Strategy::new(params)); + Arc::new(FieldStrategy { + primitive: PrimitiveFieldEncoding::new([ + PrimitivePageEncoding::reject_sparse(), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ]), + }) +} + +/// Create a v2.2 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.2 writer with explicit compression tuning. +pub fn create_writer_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, +) -> Result { + Writer::try_new_with_compression(object_writer, schema, options, compression) +} + +/// Create a v2.2 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + Writer::new_lazy(object_writer, options) +} + +/// Create a lazy v2.2 writer with explicit compression tuning. +pub fn create_lazy_writer_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, +) -> Writer { + Writer::new_lazy_with_compression(object_writer, options, compression) +} + +/// Encode a self-described v2.2 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.2 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_2/reader.rs b/rust/lance-file/src/versions/v2_2/reader.rs new file mode 100644 index 00000000000..81a88720c77 --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/reader.rs @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{decoder::ColumnInfo, format::pb21}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, RawFileMetadata, ReaderProjection, + structural, + }, + version::ConcreteFileVersion, +}; + +fn required<'a, T>(value: Option<&'a T>, label: &str) -> Result<&'a T> { + value.ok_or_else(|| { + Error::invalid_input_source( + format!("Lance v2.2 {label} is missing its nested encoding").into(), + ) + }) +} + +fn validate_compressive_encoding(encoding: &pb21::CompressiveEncoding) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::Flat(_)) + | Some(Compression::InlineBitpacking(_)) + | Some(Compression::Constant(_)) => Ok(()), + Some(Compression::Variable(variable)) => validate_compressive_encoding(required( + variable.offsets.as_deref(), + "variable offsets", + )?), + Some(Compression::OutOfLineBitpacking(bitpacking)) => { + validate_compressive_encoding(required( + bitpacking.values.as_deref(), + "out-of-line bitpacking values", + )?) + } + Some(Compression::Fsst(fsst)) => { + validate_compressive_encoding(required(fsst.values.as_deref(), "FSST values")?) + } + Some(Compression::Dictionary(dictionary)) => { + validate_compressive_encoding(required( + dictionary.indices.as_deref(), + "dictionary indices", + )?)?; + validate_compressive_encoding(required( + dictionary.items.as_deref(), + "dictionary items", + )?) + } + Some(Compression::Rle(rle)) => { + let values = required(rle.values.as_deref(), "RLE values")?; + let run_lengths = required(rle.run_lengths.as_deref(), "RLE run lengths")?; + let fixed_values = matches!( + values.compression.as_ref(), + Some(Compression::Flat(flat)) + if matches!(flat.bits_per_value, 8 | 16 | 32 | 64) + && flat.data.is_none() + ); + let fixed_u8_lengths = matches!( + run_lengths.compression.as_ref(), + Some(Compression::Flat(flat)) + if flat.bits_per_value == 8 && flat.data.is_none() + ); + if !fixed_values || !fixed_u8_lengths { + return Err(Error::invalid_input_source( + "Lance v2.2 RLE requires flat values and flat u8 run lengths".into(), + )); + } + Ok(()) + } + Some(Compression::ByteStreamSplit(split)) => validate_compressive_encoding(required( + split.values.as_deref(), + "byte-stream-split values", + )?), + Some(Compression::General(general)) => validate_compressive_encoding(required( + general.values.as_deref(), + "general-compression values", + )?), + Some(Compression::FixedSizeList(list)) => validate_compressive_encoding(required( + list.values.as_deref(), + "fixed-size-list values", + )?), + Some(Compression::PackedStruct(packed)) => validate_compressive_encoding(required( + packed.values.as_deref(), + "packed-struct values", + )?), + Some(Compression::VariablePackedStruct(packed)) => { + for field in &packed.fields { + validate_compressive_encoding(required( + field.value.as_ref(), + "variable packed-struct field", + )?)?; + } + Ok(()) + } + None => Err(Error::invalid_input_source( + "Lance v2.2 compressive encoding is missing its compression variant".into(), + )), + } +} + +fn validate_page_layout(layout: &pb21::PageLayout) -> Result<()> { + use pb21::page_layout::Layout; + + match layout.layout.as_ref() { + Some(Layout::MiniBlockLayout(miniblock)) => { + if !miniblock.has_large_chunk { + return Err(Error::invalid_input_source( + "Lance v2.2 miniblock pages require the u32 chunk grammar".into(), + )); + } + if let Some(rep) = miniblock.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = miniblock.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + validate_compressive_encoding(required( + miniblock.value_compression.as_ref(), + "miniblock values", + )?)?; + if let Some(dictionary) = miniblock.dictionary.as_ref() { + validate_compressive_encoding(dictionary)?; + } + Ok(()) + } + Some(Layout::FullZipLayout(fullzip)) => validate_compressive_encoding(required( + fullzip.value_compression.as_ref(), + "full-zip values", + )?), + Some(Layout::ConstantLayout(constant)) => { + if let Some(rep) = constant.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = constant.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + Ok(()) + } + Some(Layout::BlobLayout(blob)) => { + let inner = blob.inner_layout.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Lance v2.2 blob page layout is missing its inner layout".into(), + ) + })?; + validate_page_layout(inner) + } + Some(Layout::SparseLayout(_)) => Err(Error::invalid_input_source( + "Sparse page layout is not part of the Lance v2.2 grammar".into(), + )), + None => Err(Error::invalid_input_source( + "Lance v2.2 page is missing its page layout".into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let page_layout = structural::decode_page_layout(column_index, page_index, page)?; + validate_page_layout(&page_layout)?; + structural::build_page_info(column_index, page_index, page, page_layout) + }) + .collect::>>()?; + structural::build_column_info(column_index, metadata, page_infos) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + structural::projection_from_field_ids(schema, field_id_to_column_index) +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + structural::projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + structural::projection_from_column_names( + schema, + column_names, + &super::field_id_to_column_index(schema), + ) +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if (raw.footer.major_version, raw.footer.minor_version) != (2, 2) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.2 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + validate_global_buffers(&raw.file_buffers)?; + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_2, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + structural::validate_global_buffers(buffers) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version != ConcreteFileVersion::V2_2 { + let (major, minor) = index.version.to_standard_footer_numbers(); + return Err(Error::version_conflict( + "Attempt to use the Lance v2.2 reader for a different metadata index".to_string(), + major, + minor, + )); + } + validate_global_buffers(&index.file_buffers)?; + Ok(index) +} + +#[cfg(test)] +mod grammar_tests { + use super::*; + use pb21::{ + CompressiveEncoding, Dictionary, Flat, FullZipLayout, PageLayout, Rle, + compressive_encoding::Compression, page_layout::Layout, + }; + + fn flat(bits_per_value: u64) -> CompressiveEncoding { + CompressiveEncoding { + compression: Some(Compression::Flat(Flat { + bits_per_value, + data: None, + })), + } + } + + #[test] + fn rejects_nested_variable_width_rle() { + let rle = CompressiveEncoding { + compression: Some(Compression::Rle(Box::new(Rle { + values: Some(Box::new(flat(32))), + run_lengths: Some(Box::new(flat(16))), + }))), + }; + let dictionary = CompressiveEncoding { + compression: Some(Compression::Dictionary(Box::new(Dictionary { + indices: Some(Box::new(rle)), + items: Some(Box::new(flat(32))), + num_dictionary_items: 1, + }))), + }; + let layout = PageLayout { + layout: Some(Layout::FullZipLayout(FullZipLayout { + value_compression: Some(dictionary), + ..Default::default() + })), + }; + + let error = validate_page_layout(&layout).unwrap_err(); + assert!( + error + .to_string() + .contains("flat values and flat u8 run lengths") + ); + } +} diff --git a/rust/lance-file/src/versions/v2_2/writer.rs b/rust/lance-file/src/versions/v2_2/writer.rs new file mode 100644 index 00000000000..bfe5494cc1f --- /dev/null +++ b/rust/lance-file/src/versions/v2_2/writer.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use bytes::{BufMut, Bytes}; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{BatchEncoder, EncodedBatch}, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use object_store::path::Path; +use tokio::io::AsyncWriteExt; + +use crate::{ + format::{MAGIC, pbfile}, + writer::{ + FileWriteSummary, FileWriterOptions, + structural::{EncodedBatchBody, EncodingPipeline, StructuralFileSink, encode_batch_body}, + }, +}; + +use super::encoding_strategy; + +/// A writer for the Lance v2.2 file grammar. +/// +/// The concrete writer owns the v2.2 encoding composition, finish ordering, +/// and exact footer identity. Shared components only execute the structural +/// encoding and I/O mechanisms selected here. +pub struct Writer { + sink: StructuralFileSink, + encoding: EncodingPipeline, + compression: CompressionParams, +} + +impl Writer { + /// Create a v2.2 writer with an explicit schema. + pub fn try_new( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + ) -> Result { + Self::try_new_with_compression(object_writer, schema, options, Default::default()) + } + + /// Create a v2.2 writer with explicit compression tuning. + pub fn try_new_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Result { + let mut writer = Self::new_lazy_with_compression(object_writer, options, compression); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a v2.2 writer whose schema is inferred from the first batch. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self::new_lazy_with_compression(object_writer, options, Default::default()) + } + + /// Create a lazy v2.2 writer with explicit compression tuning. + pub fn new_lazy_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Self { + Self { + sink: StructuralFileSink::new(object_writer), + encoding: EncodingPipeline::new(options), + compression, + } + } + + fn initialize(&mut self, schema: Schema) -> Result<()> { + let encoding_options = self.encoding.encoding_options(&schema); + schema.validate()?; + let strategy = encoding_strategy(self.compression.clone()); + let encoder = BatchEncoder::try_new(&schema, strategy.as_ref(), &encoding_options)?; + self.encoding.initialize(schema, encoder, &mut self.sink); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<()> { + if !self.encoding.is_initialized() { + self.initialize(Schema::try_from(batch.schema().as_ref())?)?; + } + Ok(()) + } + + /// Spill page metadata to a sidecar file instead of retaining it in memory. + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.sink.with_page_metadata_spill(object_store, path); + self + } + + /// Schedule batches to be written in iteration order. + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Schedule one record batch for writing. + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + self.ensure_initialized(batch)?; + self.encoding.write_batch(batch, &mut self.sink).await + } + + /// Write one top-level field, advancing only that field's row count. + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + self.encoding + .write_column(column_index, array, &mut self.sink) + .await + } + + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + self.sink.write_external_buffer(bytes).await + } + + /// Add an entry to the schema metadata written in the file descriptor. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.encoding.add_schema_metadata(key, value); + } + + /// Prepare the writer for encoded column data produced externally. + pub fn initialize_with_external_metadata( + &mut self, + schema: Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.encoding + .initialize_with_external_metadata(schema, rows_written); + self.sink.initialize_with_external_metadata(column_metadata); + } + + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.sink.add_global_buffer(buffer).await + } + + /// Finish the v2.2 file and close its object writer. + pub async fn finish(&mut self) -> Result { + // The order below is the v2.2 wire contract. + self.encoding.flush(&mut self.sink).await?; + self.encoding.finish_encoders(&mut self.sink).await?; + + let descriptor = self.encoding.make_file_descriptor()?; + let global_buffer_offsets = self.sink.write_global_buffers(descriptor).await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + let column_metadata_start = self.sink.tell().await?; + let column_metadata_offsets = self.sink.write_column_metadatas().await?; + let column_metadata_offsets_start = self + .sink + .write_offset_table(&column_metadata_offsets) + .await?; + let global_buffer_offsets_start = + self.sink.write_offset_table(&global_buffer_offsets).await?; + let num_columns = self.sink.num_columns(); + + let output = self.sink.output_mut(); + output.write_u64_le(column_metadata_start).await?; + output.write_u64_le(column_metadata_offsets_start).await?; + output.write_u64_le(global_buffer_offsets_start).await?; + output.write_u32_le(num_global_buffers).await?; + output.write_u32_le(num_columns).await?; + output.write_u16_le(2).await?; + output.write_u16_le(2).await?; + output.write_all(MAGIC).await?; + + Ok(FileWriteSummary { + num_rows: self.encoding.rows_written(), + size_bytes: self.sink.shutdown().await?, + }) + } + + /// Abandon this write. + pub async fn abort(&mut self) { + // Dropping a multipart ObjectWriter aborts the upload. + } + + /// Return the current object-writer position. + pub async fn tell(&mut self) -> Result { + self.sink.tell().await + } + + /// Return the field-id to physical-column mapping selected by v2.2. + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + self.encoding.field_id_to_column_indices() + } +} + +/// Append a self-described or mini-lance v2.2 footer to an encoded batch. +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + let EncodedBatchBody { + mut data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns, + } = encode_batch_body(batch, write_schema)?; + + data.put_u64_le(column_metadata_start); + data.put_u64_le(column_metadata_offsets_start); + data.put_u64_le(global_buffer_offsets_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(num_columns); + data.put_u16_le(2); + data.put_u16_le(2); + data.extend_from_slice(MAGIC); + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/versions/v2_3/compression.rs b/rust/lance-file/src/versions/v2_3/compression.rs new file mode 100644 index 00000000000..021c69ad3c6 --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/compression.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_core::{Error, Result, datatypes::Field}; +use lance_encoding::compression::try_packed_struct_per_value; +use lance_encoding::{ + compression::{ + BlockCompressor, CompressionStrategy, field_metadata_params, finalize_miniblock_compressor, + try_bitpacking_block, try_bitpacking_miniblock, try_byte_stream_split_miniblock, + try_child_rle_miniblock, try_fixed_packed_struct_miniblock, try_general_block, + try_raw_block, try_raw_fixed_size_list_miniblock, try_raw_fixed_width_miniblock, + try_raw_per_value, try_uncompressed_fixed_width_miniblock, try_variable_rle_block, + try_variable_width_miniblock, try_variable_width_per_value, + }, + compression_config::{CompressionFieldParams, CompressionParams}, + data::DataBlock, + encodings::logical::primitive::{fullzip::PerValueCompressor, miniblock::MiniBlockCompressor}, +}; + +#[derive(Debug, Clone)] +pub(super) struct Strategy { + params: CompressionParams, +} + +impl Strategy { + pub(super) fn new(params: CompressionParams) -> Self { + Self { params } + } + + fn field_params(&self, field: &Field) -> CompressionFieldParams { + let mut params = self + .params + .get_field_params(&field.name, &field.data_type()); + params.merge(&field_metadata_params(field)); + params + } +} + +impl CompressionStrategy for Strategy { + fn create_miniblock_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + let compressor = + if let Some(compressor) = try_uncompressed_fixed_width_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_byte_stream_split_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_child_rle_miniblock(data, ¶ms) { + compressor + } else if let Some(compressor) = try_bitpacking_miniblock(data) { + compressor + } else if let Some(compressor) = try_raw_fixed_width_miniblock(data) { + compressor + } else if let Some(compressor) = try_variable_width_miniblock(field, data, ¶ms)? { + compressor + } else if let Some(compressor) = try_fixed_packed_struct_miniblock(data)? { + compressor + } else if let Some(compressor) = try_raw_fixed_size_list_miniblock(data) { + compressor + } else { + return Err(Error::not_supported_source( + format!( + "Mini-block compression not yet supported for block type {}", + data.name() + ) + .into(), + )); + }; + finalize_miniblock_compressor(data, compressor, ¶ms) + } + + fn create_per_value( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_raw_per_value(data) { + return Ok(compressor); + } + if let Some(compressor) = try_packed_struct_per_value(Arc::new(self.clone()), field, data)? + { + return Ok(compressor); + } + if let Some(compressor) = try_variable_width_per_value(field, data, ¶ms)? { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Per-value compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } + + fn create_block_compressor( + &self, + field: &Field, + data: &DataBlock, + ) -> Result> { + let params = self.field_params(field); + if let Some(compressor) = try_variable_rle_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_bitpacking_block(data) { + return Ok(compressor); + } + if let Some(compressor) = try_general_block(data, ¶ms)? { + return Ok(compressor); + } + if let Some(compressor) = try_raw_block(data) { + return Ok(compressor); + } + Err(Error::not_supported_source( + format!( + "Block compression not yet supported for block type {}", + data.name() + ) + .into(), + )) + } +} diff --git a/rust/lance-file/src/versions/v2_3/mod.rs b/rust/lance-file/src/versions/v2_3/mod.rs new file mode 100644 index 00000000000..efbd5008ef6 --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/mod.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Lance v2.3 file composition. + +use std::{ + collections::BTreeMap, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use bytes::Bytes; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, +}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{ + ColumnIndexSequence, EncodedBatch, FieldEncoder, FieldEncodingContext, + FieldEncodingStrategy, + structural::{ + PrimitiveFieldEncoding, PrimitivePageEncoding, try_create_binary_blob, try_create_list, + try_create_map, try_create_struct, try_create_structural_blob, + try_create_structural_fixed_size_list, + }, + }, +}; +use lance_io::traits::Writer as ObjectWriter; + +use crate::{ + reader::{ReadProjection, structural}, + writer::FileWriterOptions, +}; + +mod compression; +mod reader; +mod writer; + +pub(crate) use reader::{ + decode_column_metadata, finish_metadata, finish_metadata_index, validate_global_buffers, +}; +pub use reader::{ + projection_from_column_names, projection_from_field_ids, projection_from_whole_schema, +}; + +pub(crate) fn read_projection() -> Arc { + structural::read_projection(reader::decode_column) +} +pub use writer::Writer; + +static WARNED_ON_UNSTABLE_FORMAT: AtomicBool = AtomicBool::new(false); + +/// Count physical columns represented by a field in a v2.3 footer. +pub fn physical_column_count(field: &Field) -> usize { + structural::physical_column_count(field) +} + +/// Build persisted field-to-column entries for a v2.3 data file. +pub fn data_file_columns(schema: &Schema) -> (Vec, Vec) { + structural::data_file_columns(schema) +} + +pub(super) fn field_id_to_column_index(schema: &Schema) -> BTreeMap { + structural::field_id_to_column_index(schema) +} + +#[derive(Debug)] +struct FieldStrategy { + primitive: PrimitiveFieldEncoding, +} + +impl FieldEncodingStrategy for FieldStrategy { + fn create_field_encoder( + &self, + field: &Field, + column_index: &mut ColumnIndexSequence, + context: &FieldEncodingContext<'_>, + ) -> Result> { + if let Some(encoder) = + try_create_binary_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = + try_create_structural_blob(&self.primitive, field, column_index, context)? + { + return Ok(encoder); + } + if field.is_blob() { + return Err(Error::invalid_input_source( + format!( + "Blob encoding is not available for field '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )); + } + if let Some(encoder) = try_create_map(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_structural_fixed_size_list(field, column_index, context)? + { + return Ok(encoder); + } + if let Some(encoder) = self.primitive.try_create(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_list(field, column_index, context)? { + return Ok(encoder); + } + if let Some(encoder) = try_create_struct(field, column_index, context)? { + return Ok(encoder); + } + Err(Error::not_supported_source( + format!( + "Lance v2.3 has no field encoding for '{}' with data type {}", + field.name, + field.data_type() + ) + .into(), + )) + } +} + +/// Compose the v2.3 field encoding mechanisms. +pub fn encoding_strategy(params: CompressionParams) -> Arc { + let compression = Arc::new(compression::Strategy::new(params)); + Arc::new(FieldStrategy { + primitive: PrimitiveFieldEncoding::new([ + PrimitivePageEncoding::sparse(compression.clone()), + PrimitivePageEncoding::constant(), + PrimitivePageEncoding::dense_u32(compression), + ]), + }) +} + +fn warn_unstable_format() { + if WARNED_ON_UNSTABLE_FORMAT + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + log::warn!( + "You have requested an unstable format version. Files written with this format version may not be readable in the future! This is a development feature and should only be used for experimentation and never for production data." + ); + } +} + +/// Create a v2.3 writer with an explicit schema. +pub fn create_writer( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, +) -> Result { + warn_unstable_format(); + Writer::try_new(object_writer, schema, options) +} + +/// Create a v2.3 writer with explicit compression tuning. +pub fn create_writer_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, +) -> Result { + warn_unstable_format(); + Writer::try_new_with_compression(object_writer, schema, options, compression) +} + +/// Create a v2.3 writer whose schema is inferred from the first batch. +pub fn create_lazy_writer( + object_writer: Box, + options: FileWriterOptions, +) -> Writer { + warn_unstable_format(); + Writer::new_lazy(object_writer, options) +} + +/// Create a lazy v2.3 writer with explicit compression tuning. +pub fn create_lazy_writer_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, +) -> Writer { + warn_unstable_format(); + Writer::new_lazy_with_compression(object_writer, options, compression) +} + +/// Encode a self-described v2.3 batch. +pub fn encode_self_described_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, true) +} + +/// Encode a mini-lance v2.3 batch. +pub fn encode_mini_batch(batch: &EncodedBatch) -> Result { + writer::concat_lance_footer(batch, false) +} diff --git a/rust/lance-file/src/versions/v2_3/reader.rs b/rust/lance-file/src/versions/v2_3/reader.rs new file mode 100644 index 00000000000..cdafbd0b9db --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/reader.rs @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::BTreeMap, sync::Arc}; + +use lance_core::{Error, Result, datatypes::Schema}; +use lance_encoding::{decoder::ColumnInfo, format::pb21}; + +use crate::{ + format::pbfile, + reader::{ + BufferDescriptor, CachedFileMetadata, FileMetadataIndex, RawFileMetadata, ReaderProjection, + structural, + }, + version::ConcreteFileVersion, +}; + +fn required<'a, T>(value: Option<&'a T>, label: &str) -> Result<&'a T> { + value.ok_or_else(|| { + Error::invalid_input_source( + format!("Lance v2.3 {label} is missing its nested encoding").into(), + ) + }) +} + +fn validate_compressive_encoding(encoding: &pb21::CompressiveEncoding) -> Result<()> { + use pb21::compressive_encoding::Compression; + + match encoding.compression.as_ref() { + Some(Compression::Flat(_)) + | Some(Compression::InlineBitpacking(_)) + | Some(Compression::Constant(_)) => Ok(()), + Some(Compression::Variable(variable)) => validate_compressive_encoding(required( + variable.offsets.as_deref(), + "variable offsets", + )?), + Some(Compression::OutOfLineBitpacking(bitpacking)) => { + validate_compressive_encoding(required( + bitpacking.values.as_deref(), + "out-of-line bitpacking values", + )?) + } + Some(Compression::Fsst(fsst)) => { + validate_compressive_encoding(required(fsst.values.as_deref(), "FSST values")?) + } + Some(Compression::Dictionary(dictionary)) => { + validate_compressive_encoding(required( + dictionary.indices.as_deref(), + "dictionary indices", + )?)?; + validate_compressive_encoding(required( + dictionary.items.as_deref(), + "dictionary items", + )?) + } + Some(Compression::Rle(rle)) => { + validate_compressive_encoding(required(rle.values.as_deref(), "RLE values")?)?; + validate_compressive_encoding(required(rle.run_lengths.as_deref(), "RLE run lengths")?) + } + Some(Compression::ByteStreamSplit(split)) => validate_compressive_encoding(required( + split.values.as_deref(), + "byte-stream-split values", + )?), + Some(Compression::General(general)) => validate_compressive_encoding(required( + general.values.as_deref(), + "general-compression values", + )?), + Some(Compression::FixedSizeList(list)) => validate_compressive_encoding(required( + list.values.as_deref(), + "fixed-size-list values", + )?), + Some(Compression::PackedStruct(packed)) => validate_compressive_encoding(required( + packed.values.as_deref(), + "packed-struct values", + )?), + Some(Compression::VariablePackedStruct(packed)) => { + for field in &packed.fields { + validate_compressive_encoding(required( + field.value.as_ref(), + "variable packed-struct field", + )?)?; + } + Ok(()) + } + None => Err(Error::invalid_input_source( + "Lance v2.3 compressive encoding is missing its compression variant".into(), + )), + } +} + +fn validate_sparse_positions(positions: Option<&pb21::SparsePositionSet>) -> Result<()> { + use pb21::sparse_position_set::Positions; + + if let Some(Positions::Explicit(encoding)) = + positions.and_then(|positions| positions.positions.as_ref()) + { + validate_compressive_encoding(encoding)?; + } + Ok(()) +} + +fn validate_sparse_validity(validity: Option<&pb21::SparseValiditySet>) -> Result<()> { + if let Some(validity) = validity { + validate_sparse_positions(validity.positions.as_ref())?; + } + Ok(()) +} + +fn validate_sparse_layout(layout: &pb21::SparseLayout) -> Result<()> { + use pb21::{sparse_count_set::Counts, sparse_structural_layer::Layer}; + + if !layout.has_large_chunk { + return Err(Error::invalid_input_source( + "Lance v2.3 sparse pages require the u32 chunk grammar".into(), + )); + } + validate_compressive_encoding(required( + layout.value_compression.as_ref(), + "sparse values", + )?)?; + for layer in &layout.structural_layers { + match layer.layer.as_ref() { + Some(Layer::Validity(validity)) => { + validate_sparse_validity(validity.validity.as_ref())? + } + Some(Layer::List(list)) => { + validate_sparse_positions(list.non_empty_positions.as_ref())?; + if let Some(Counts::Explicit(encoding)) = list + .counts + .as_ref() + .and_then(|counts| counts.counts.as_ref()) + { + validate_compressive_encoding(encoding)?; + } + validate_sparse_validity(list.validity.as_ref())?; + } + Some(Layer::FixedSizeList(list)) => validate_sparse_validity(list.validity.as_ref())?, + None => { + return Err(Error::invalid_input_source( + "Lance v2.3 sparse structural layer is missing its layer variant".into(), + )); + } + } + } + Ok(()) +} + +fn validate_page_layout(layout: &pb21::PageLayout) -> Result<()> { + use pb21::page_layout::Layout; + + match layout.layout.as_ref() { + Some(Layout::MiniBlockLayout(miniblock)) => { + if !miniblock.has_large_chunk { + return Err(Error::invalid_input_source( + "Lance v2.3 miniblock pages require the u32 chunk grammar".into(), + )); + } + if let Some(rep) = miniblock.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = miniblock.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + validate_compressive_encoding(required( + miniblock.value_compression.as_ref(), + "miniblock values", + )?)?; + if let Some(dictionary) = miniblock.dictionary.as_ref() { + validate_compressive_encoding(dictionary)?; + } + Ok(()) + } + Some(Layout::FullZipLayout(fullzip)) => validate_compressive_encoding(required( + fullzip.value_compression.as_ref(), + "full-zip values", + )?), + Some(Layout::ConstantLayout(constant)) => { + if let Some(rep) = constant.rep_compression.as_ref() { + validate_compressive_encoding(rep)?; + } + if let Some(def) = constant.def_compression.as_ref() { + validate_compressive_encoding(def)?; + } + Ok(()) + } + Some(Layout::SparseLayout(sparse)) => validate_sparse_layout(sparse), + Some(Layout::BlobLayout(blob)) => { + let inner = blob.inner_layout.as_deref().ok_or_else(|| { + Error::invalid_input_source( + "Lance v2.3 blob page layout is missing its inner layout".into(), + ) + })?; + validate_page_layout(inner) + } + None => Err(Error::invalid_input_source( + "Lance v2.3 page is missing its page layout".into(), + )), + } +} + +pub fn decode_column( + column_index: u32, + metadata: &pbfile::ColumnMetadata, +) -> Result> { + let page_infos = metadata + .pages + .iter() + .enumerate() + .map(|(page_index, page)| { + let page_layout = structural::decode_page_layout(column_index, page_index, page)?; + validate_page_layout(&page_layout)?; + structural::build_page_info(column_index, page_index, page, page_layout) + }) + .collect::>>()?; + structural::build_column_info(column_index, metadata, page_infos) +} + +pub fn decode_column_metadata( + column_metadatas: &[pbfile::ColumnMetadata], +) -> Result>> { + column_metadatas + .iter() + .enumerate() + .map(|(column_index, metadata)| { + let column_index = u32::try_from(column_index).map_err(|_| { + Error::invalid_input_source("File has more than u32::MAX columns".into()) + })?; + decode_column(column_index, metadata) + }) + .collect() +} + +pub fn projection_from_field_ids( + schema: &Schema, + field_id_to_column_index: &BTreeMap, +) -> ReaderProjection { + structural::projection_from_field_ids(schema, field_id_to_column_index) +} + +pub fn projection_from_whole_schema(schema: &Schema) -> ReaderProjection { + structural::projection_from_field_ids(schema, &super::field_id_to_column_index(schema)) +} + +pub fn projection_from_column_names( + schema: &Schema, + column_names: &[&str], +) -> Result { + structural::projection_from_column_names( + schema, + column_names, + &super::field_id_to_column_index(schema), + ) +} + +pub fn finish_metadata(raw: RawFileMetadata) -> Result { + if (raw.footer.major_version, raw.footer.minor_version) != (2, 3) { + return Err(Error::version_conflict( + "Attempt to use the Lance v2.3 reader for a different file version".to_string(), + raw.footer.major_version, + raw.footer.minor_version, + )); + } + validate_global_buffers(&raw.file_buffers)?; + let column_infos = decode_column_metadata(&raw.column_metadatas)?; + Ok(CachedFileMetadata { + file_schema: raw.file_schema, + column_metadatas: raw.column_metadatas, + column_infos, + num_rows: raw.num_rows, + file_buffers: raw.file_buffers, + num_data_bytes: raw.num_data_bytes, + num_column_metadata_bytes: raw.num_column_metadata_bytes, + num_global_buffer_bytes: raw.num_global_buffer_bytes, + num_footer_bytes: raw.num_footer_bytes, + major_version: raw.footer.major_version, + minor_version: raw.footer.minor_version, + version: ConcreteFileVersion::V2_3, + file_size_bytes: raw.file_size_bytes, + retained_global_buffers: raw.retained_global_buffers, + }) +} + +pub fn validate_global_buffers(buffers: &[BufferDescriptor]) -> Result<()> { + structural::validate_global_buffers(buffers) +} + +pub fn finish_metadata_index(index: FileMetadataIndex) -> Result { + if index.version != ConcreteFileVersion::V2_3 { + let (major, minor) = index.version.to_standard_footer_numbers(); + return Err(Error::version_conflict( + "Attempt to use the Lance v2.3 reader for a different metadata index".to_string(), + major, + minor, + )); + } + validate_global_buffers(&index.file_buffers)?; + Ok(index) +} + +#[cfg(test)] +mod grammar_tests { + use super::*; + use pb21::{ + CompressiveEncoding, Dictionary, Flat, FullZipLayout, PageLayout, Rle, + compressive_encoding::Compression, page_layout::Layout, + }; + + fn flat(bits_per_value: u64) -> CompressiveEncoding { + CompressiveEncoding { + compression: Some(Compression::Flat(Flat { + bits_per_value, + data: None, + })), + } + } + + #[test] + fn accepts_nested_variable_width_rle() { + let rle = CompressiveEncoding { + compression: Some(Compression::Rle(Box::new(Rle { + values: Some(Box::new(flat(32))), + run_lengths: Some(Box::new(flat(16))), + }))), + }; + let dictionary = CompressiveEncoding { + compression: Some(Compression::Dictionary(Box::new(Dictionary { + indices: Some(Box::new(rle)), + items: Some(Box::new(flat(32))), + num_dictionary_items: 1, + }))), + }; + let layout = PageLayout { + layout: Some(Layout::FullZipLayout(FullZipLayout { + value_compression: Some(dictionary), + ..Default::default() + })), + }; + + validate_page_layout(&layout).unwrap(); + } +} diff --git a/rust/lance-file/src/versions/v2_3/writer.rs b/rust/lance-file/src/versions/v2_3/writer.rs new file mode 100644 index 00000000000..c18c7aa6e18 --- /dev/null +++ b/rust/lance-file/src/versions/v2_3/writer.rs @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{ArrayRef, RecordBatch}; +use bytes::{BufMut, Bytes}; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::{ + compression_config::CompressionParams, + encoder::{BatchEncoder, EncodedBatch}, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use object_store::path::Path; +use tokio::io::AsyncWriteExt; + +use crate::{ + format::{MAGIC, pbfile}, + writer::{ + FileWriteSummary, FileWriterOptions, + structural::{EncodedBatchBody, EncodingPipeline, StructuralFileSink, encode_batch_body}, + }, +}; + +use super::encoding_strategy; + +/// A writer for the Lance v2.3 file grammar. +/// +/// The concrete writer owns the v2.3 encoding composition, finish ordering, +/// and exact footer identity. Shared components only execute the structural +/// encoding and I/O mechanisms selected here. +pub struct Writer { + sink: StructuralFileSink, + encoding: EncodingPipeline, + compression: CompressionParams, +} + +impl Writer { + /// Create a v2.3 writer with an explicit schema. + pub fn try_new( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + ) -> Result { + Self::try_new_with_compression(object_writer, schema, options, Default::default()) + } + + /// Create a v2.3 writer with explicit compression tuning. + pub fn try_new_with_compression( + object_writer: Box, + schema: Schema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Result { + let mut writer = Self::new_lazy_with_compression(object_writer, options, compression); + writer.initialize(schema)?; + Ok(writer) + } + + /// Create a v2.3 writer whose schema is inferred from the first batch. + pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { + Self::new_lazy_with_compression(object_writer, options, Default::default()) + } + + /// Create a lazy v2.3 writer with explicit compression tuning. + pub fn new_lazy_with_compression( + object_writer: Box, + options: FileWriterOptions, + compression: CompressionParams, + ) -> Self { + Self { + sink: StructuralFileSink::new(object_writer), + encoding: EncodingPipeline::new(options), + compression, + } + } + + fn initialize(&mut self, schema: Schema) -> Result<()> { + let encoding_options = self.encoding.encoding_options(&schema); + schema.validate()?; + let strategy = encoding_strategy(self.compression.clone()); + let encoder = BatchEncoder::try_new(&schema, strategy.as_ref(), &encoding_options)?; + self.encoding.initialize(schema, encoder, &mut self.sink); + Ok(()) + } + + fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<()> { + if !self.encoding.is_initialized() { + self.initialize(Schema::try_from(batch.schema().as_ref())?)?; + } + Ok(()) + } + + /// Spill page metadata to a sidecar file instead of retaining it in memory. + pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { + self.sink.with_page_metadata_spill(object_store, path); + self + } + + /// Schedule batches to be written in iteration order. + pub async fn write_batches( + &mut self, + batches: impl Iterator, + ) -> Result<()> { + for batch in batches { + self.write_batch(batch).await?; + } + Ok(()) + } + + /// Schedule one record batch for writing. + pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { + self.ensure_initialized(batch)?; + self.encoding.write_batch(batch, &mut self.sink).await + } + + /// Write one top-level field, advancing only that field's row count. + pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { + self.encoding + .write_column(column_index, array, &mut self.sink) + .await + } + + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + self.sink.write_external_buffer(bytes).await + } + + /// Add an entry to the schema metadata written in the file descriptor. + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.encoding.add_schema_metadata(key, value); + } + + /// Prepare the writer for encoded column data produced externally. + pub fn initialize_with_external_metadata( + &mut self, + schema: Schema, + column_metadata: Vec, + rows_written: u64, + ) { + self.encoding + .initialize_with_external_metadata(schema, rows_written); + self.sink.initialize_with_external_metadata(column_metadata); + } + + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.sink.add_global_buffer(buffer).await + } + + /// Finish the v2.3 file and close its object writer. + pub async fn finish(&mut self) -> Result { + // The order below is the v2.3 wire contract. + self.encoding.flush(&mut self.sink).await?; + self.encoding.finish_encoders(&mut self.sink).await?; + + let descriptor = self.encoding.make_file_descriptor()?; + let global_buffer_offsets = self.sink.write_global_buffers(descriptor).await?; + let num_global_buffers = global_buffer_offsets.len() as u32; + + let column_metadata_start = self.sink.tell().await?; + let column_metadata_offsets = self.sink.write_column_metadatas().await?; + let column_metadata_offsets_start = self + .sink + .write_offset_table(&column_metadata_offsets) + .await?; + let global_buffer_offsets_start = + self.sink.write_offset_table(&global_buffer_offsets).await?; + let num_columns = self.sink.num_columns(); + + let output = self.sink.output_mut(); + output.write_u64_le(column_metadata_start).await?; + output.write_u64_le(column_metadata_offsets_start).await?; + output.write_u64_le(global_buffer_offsets_start).await?; + output.write_u32_le(num_global_buffers).await?; + output.write_u32_le(num_columns).await?; + output.write_u16_le(2).await?; + output.write_u16_le(3).await?; + output.write_all(MAGIC).await?; + + Ok(FileWriteSummary { + num_rows: self.encoding.rows_written(), + size_bytes: self.sink.shutdown().await?, + }) + } + + /// Abandon this write. + pub async fn abort(&mut self) { + // Dropping a multipart ObjectWriter aborts the upload. + } + + /// Return the current object-writer position. + pub async fn tell(&mut self) -> Result { + self.sink.tell().await + } + + /// Return the field-id to physical-column mapping selected by v2.3. + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + self.encoding.field_id_to_column_indices() + } +} + +/// Append a self-described or mini-lance v2.3 footer to an encoded batch. +pub fn concat_lance_footer(batch: &EncodedBatch, write_schema: bool) -> Result { + let EncodedBatchBody { + mut data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns, + } = encode_batch_body(batch, write_schema)?; + + data.put_u64_le(column_metadata_start); + data.put_u64_le(column_metadata_offsets_start); + data.put_u64_le(global_buffer_offsets_start); + data.put_u32_le(num_global_buffers); + data.put_u32_le(num_columns); + data.put_u16_le(2); + data.put_u16_le(3); + data.extend_from_slice(MAGIC); + Ok(data.freeze()) +} diff --git a/rust/lance-file/src/writer.rs b/rust/lance-file/src/writer.rs index f7042b6b99c..dcba2658225 100644 --- a/rust/lance-file/src/writer.rs +++ b/rust/lance-file/src/writer.rs @@ -1,53 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use core::panic; -use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::AtomicBool; use arrow_array::{ArrayRef, RecordBatch}; - -use arrow_data::ArrayData; -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use futures::StreamExt; -use futures::stream::FuturesOrdered; -use lance_core::datatypes::{Field, Schema as LanceSchema}; -use lance_core::utils::bit::pad_bytes; -use lance_core::{Error, Result}; -use lance_encoding::decoder::PageEncoding; -use lance_encoding::encoder::{ - BatchEncoder, EncodeTask, EncodedBatch, EncodedPage, EncodingOptions, FieldEncoder, - FieldEncodingStrategy, OutOfLineBuffers, default_encoding_strategy, -}; -use lance_encoding::repdef::RepDefBuilder; -use lance_encoding::version::LanceFileVersion; +use bytes::Bytes; +use lance_core::{Result, datatypes::Schema}; +use lance_encoding::decoder::{ColumnInfo, PageEncoding}; use lance_io::object_store::ObjectStore; -use lance_io::traits::Writer; -use log::{debug, warn}; use object_store::path::Path; use prost::Message; use prost_types::Any; -use tokio::io::AsyncWrite; -use tokio::io::AsyncWriteExt; -use tracing::instrument; -use crate::datatypes::FieldsWithMeta; -use crate::format::MAGIC; -use crate::format::pb; -use crate::format::pbfile; -use crate::format::pbfile::DirectEncoding; +use crate::{format::pbfile, versions}; + +pub(crate) mod structural; -/// Pages buffers are aligned to 64 bytes +/// Page buffers in current Lance files are aligned to 64 bytes. pub(crate) const PAGE_BUFFER_ALIGNMENT: usize = 64; -const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT]; -// In 2.1+, we split large pages on read instead of write to avoid empty pages -// and small pages issues. However, we keep the write-time limit at 32MB to avoid -// potential regressions in 2.0 format readers. -// -// This limit is not applied in the 2.1 writer -const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024; -const ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES: &str = "LANCE_FILE_WRITER_MAX_PAGE_BYTES"; +pub(crate) const ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES: &str = "LANCE_FILE_WRITER_MAX_PAGE_BYTES"; /// Summary of a completed Lance file write. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -58,345 +29,120 @@ pub struct FileWriteSummary { pub size_bytes: u64, } +/// Runtime options shared by all current-format writers. +/// +/// These options control buffering and execution only. Select the file grammar +/// by constructing a writer under [`crate::versions`]. #[derive(Debug, Clone, Default)] pub struct FileWriterOptions { - /// How many bytes to use for buffering column data + /// How many bytes to use for buffering column data. /// - /// When data comes in small batches the writer will buffer column data so that - /// larger pages can be created. This value will be divided evenly across all of the - /// columns. Generally you want this to be at least large enough to match your - /// filesystem's ideal read size per column. - /// - /// In some cases you might want this value to be even larger if you have highly - /// compressible data. However, if this is too large, then the writer could require - /// a lot of memory and write performance may suffer if the CPU-expensive encoding - /// falls behind and can't be interleaved with the I/O expensive flushing. - /// - /// The default will use 8MiB per column which should be reasonable for most cases. - // TODO: Do we need to be able to set this on a per-column basis? + /// The budget is divided evenly across columns. The default is 8 MiB per + /// column. pub data_cache_bytes: Option, - /// A hint to indicate the max size of a page - /// - /// This hint can't always be respected. A single value could be larger than this value - /// and we never slice single values. In addition, there are some cases where it can be - /// difficult to know size up-front and so we might not be able to respect this value. + /// A best-effort maximum encoded page size. pub max_page_bytes: Option, - /// The file writer buffers columns until enough data has arrived to flush a page - /// to disk. + /// Keep input arrays instead of copying buffered slices. /// - /// Some columns with small data types may not flush very often. These arrays can - /// stick around for a long time. These arrays might also be keeping larger data - /// structures alive. By default, the writer will make a deep copy of this array - /// to avoid any potential memory leaks. However, this can be disabled for a - /// (probably minor) performance boost if you are sure that arrays are not keeping - /// any sibling structures alive (this typically means the array was allocated in - /// the same language / runtime as the writer) - /// - /// Do not enable this if your data is arriving from the C data interface. - /// Data typically arrives one "batch" at a time (encoded in the C data interface - /// as a struct array). Each array in that batch keeps the entire batch alive. - /// This means a small boolean array (which we will buffer in memory for quite a - /// while) might keep a much larger record batch around in memory (even though most - /// of that batch's data has been written to disk) + /// Do not enable this for arrays arriving through the Arrow C data + /// interface because a small child array can keep an entire batch alive. pub keep_original_array: Option, - pub encoding_strategy: Option>, - /// The format version to use when writing the file - /// - /// This controls which encodings will be used when encoding the data. Newer - /// versions may have more efficient encodings. However, newer format versions will - /// require more up-to-date readers to read the data. - pub format_version: Option, } -// Total in-memory budget for buffering serialized page metadata before flushing -// to the spill file. Divided evenly across columns (with a floor of 64 bytes). -const DEFAULT_SPILL_BUFFER_LIMIT: usize = 256 * 1024; - -/// Spills serialized page metadata to a temporary file to bound memory usage. +/// A type-erased current-format file writer. /// -/// The spill file is an unstructured sequence of "chunks". Each chunk is a -/// contiguous run of length-delimited protobuf `Page` messages belonging to a -/// single column. Chunks from different columns are interleaved in the order -/// they are flushed (i.e. whenever a column's in-memory buffer exceeds -/// `per_column_limit`). The `column_chunks` index records the (offset, length) -/// of every chunk so each column's pages can be read back and reassembled in -/// order. -struct PageMetadataSpill { - writer: Box, - object_store: Arc, - path: Path, - /// Current write position in the spill file. - position: u64, - /// Per-column buffer of serialized (length-delimited protobuf) page metadata - /// that has not yet been flushed to the spill file. - column_buffers: Vec>, - /// Per-column list of chunks that have been flushed to the spill file. - /// Each entry is (offset, length) pointing into the spill file. - column_chunks: Vec>, - /// Maximum bytes to buffer per column before flushing to the spill file. - per_column_limit: usize, +/// This enum exists for callers that select a concrete file version at +/// runtime. Each variant owns the complete implementation for exactly one file +/// grammar; this type only forwards operations without adding format policy. +pub enum FileWriter { + V2_0(Box), + V2_1(Box), + V2_2(Box), + V2_3(Box), } -impl PageMetadataSpill { - async fn new(object_store: Arc, path: Path, num_columns: usize) -> Result { - let writer = object_store.create(&path).await?; - let per_column_limit = (DEFAULT_SPILL_BUFFER_LIMIT / num_columns.max(1)).max(64); - Ok(Self { - writer, - object_store, - path, - position: 0, - column_buffers: vec![Vec::new(); num_columns], - column_chunks: vec![Vec::new(); num_columns], - per_column_limit, +fn column_info_to_metadata(column: &ColumnInfo) -> Result { + let pages = column + .page_infos + .iter() + .map(|page| { + let encoding = match &page.encoding { + PageEncoding::Legacy(encoding) => Any::from_msg(encoding)?.encode_to_vec(), + PageEncoding::Structural(encoding) => Any::from_msg(encoding)?.encode_to_vec(), + }; + let (buffer_offsets, buffer_sizes) = + page.buffer_offsets_and_sizes.iter().copied().unzip(); + Ok(pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding, + })), + }), + length: page.num_rows, + priority: page.priority, + }) }) - } - - async fn append_page( - &mut self, - column_idx: usize, - page: &pbfile::column_metadata::Page, - ) -> Result<()> { - page.encode_length_delimited(&mut self.column_buffers[column_idx]) - .map_err(|e| { - Error::io_source(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - ))) - })?; - if self.column_buffers[column_idx].len() >= self.per_column_limit { - self.flush_column(column_idx).await?; - } - Ok(()) - } - - async fn flush_column(&mut self, column_idx: usize) -> Result<()> { - let buf = &self.column_buffers[column_idx]; - if buf.is_empty() { - return Ok(()); - } - let len = buf.len(); - self.writer.write_all(buf).await?; - self.column_chunks[column_idx].push((self.position, len as u32)); - self.position += len as u64; - self.column_buffers[column_idx].clear(); - Ok(()) - } - - async fn shutdown_writer(&mut self) -> Result<()> { - for col_idx in 0..self.column_buffers.len() { - self.flush_column(col_idx).await?; - } - Writer::shutdown(self.writer.as_mut()).await?; - Ok(()) - } + .collect::>>()?; + let (buffer_offsets, buffer_sizes) = column.buffer_offsets_and_sizes.iter().copied().unzip(); + let encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + Ok(pbfile::ColumnMetadata { + pages, + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding, + })), + }), + }) } -fn decode_spilled_chunk(data: &Bytes) -> Result> { - let mut pages = Vec::new(); - let mut cursor = data.clone(); - while cursor.has_remaining() { - let page = - pbfile::column_metadata::Page::decode_length_delimited(&mut cursor).map_err(|e| { - Error::io_source(Box::new(std::io::Error::new( - std::io::ErrorKind::InvalidData, - e, - ))) - })?; - pages.push(page); +impl From for FileWriter { + fn from(writer: versions::v2_0::Writer) -> Self { + Self::V2_0(Box::new(writer)) } - Ok(pages) } -enum PageSpillState { - Pending(Arc, Path), - Active(PageMetadataSpill), +impl From for FileWriter { + fn from(writer: versions::v2_1::Writer) -> Self { + Self::V2_1(Box::new(writer)) + } } -pub struct FileWriter { - writer: Box, - schema: Option, - column_writers: Vec>, - column_metadata: Vec, - field_id_to_column_indices: Vec<(u32, u32)>, - num_columns: u32, - rows_written: u64, - // The number of rows written for each top-level field (i.e. each entry in - // `column_writers`). With `write_batch` every field advances together and - // these are all equal, but `write_column` advances one field at a time, so - // a single file may end up with columns of differing item counts. - field_rows_written: Vec, - global_buffers: Vec<(u64, u64)>, - schema_metadata: HashMap, - options: FileWriterOptions, - page_spill: Option, +impl From for FileWriter { + fn from(writer: versions::v2_2::Writer) -> Self { + Self::V2_2(Box::new(writer)) + } } -fn initial_column_metadata() -> pbfile::ColumnMetadata { - pbfile::ColumnMetadata { - pages: Vec::new(), - buffer_offsets: Vec::new(), - buffer_sizes: Vec::new(), - encoding: None, +impl From for FileWriter { + fn from(writer: versions::v2_3::Writer) -> Self { + Self::V2_3(Box::new(writer)) } } -static WARNED_ON_UNSTABLE_API: AtomicBool = AtomicBool::new(false); - impl FileWriter { - /// Create a new FileWriter with a desired output schema - pub fn try_new( - object_writer: Box, - schema: LanceSchema, - options: FileWriterOptions, - ) -> Result { - let mut writer = Self::new_lazy(object_writer, options); - writer.initialize(schema)?; - Ok(writer) - } - - /// Create a new FileWriter without a desired output schema - /// - /// The output schema will be set based on the first batch of data to arrive. - /// If no data arrives and the writer is finished then the write will fail. - pub fn new_lazy(object_writer: Box, options: FileWriterOptions) -> Self { - if let Some(format_version) = options.format_version - && format_version.is_unstable() - && WARNED_ON_UNSTABLE_API - .compare_exchange( - false, - true, - std::sync::atomic::Ordering::Relaxed, - std::sync::atomic::Ordering::Relaxed, - ) - .is_ok() - { - warn!( - "You have requested an unstable format version. Files written with this format version may not be readable in the future! This is a development feature and should only be used for experimentation and never for production data." - ); - } - Self { - writer: object_writer, - schema: None, - column_writers: Vec::new(), - column_metadata: Vec::new(), - num_columns: 0, - rows_written: 0, - field_rows_written: Vec::new(), - field_id_to_column_indices: Vec::new(), - global_buffers: Vec::new(), - schema_metadata: HashMap::new(), - page_spill: None, - options, - } - } - - /// Spill page metadata to a sidecar file instead of accumulating in memory. - /// - /// This can dramatically reduce memory usage when many writers are open - /// concurrently (e.g. IVF shuffle with thousands of partition writers). - /// The sidecar file is created lazily on the first page write. The caller - /// is responsible for cleaning up `path` (e.g. by placing it in a temp - /// directory that is removed via RAII). - pub fn with_page_metadata_spill(mut self, object_store: Arc, path: Path) -> Self { - self.page_spill = Some(PageSpillState::Pending(object_store, path)); - self - } - - /// Write a series of record batches to a new file - /// - /// Returns the number of rows written - pub async fn create_file_with_batches( - store: &ObjectStore, - path: &Path, - schema: lance_core::datatypes::Schema, - batches: impl Iterator + Send, - options: FileWriterOptions, - ) -> Result { - let writer = store.create(path).await?; - let mut writer = Self::try_new(writer, schema, options)?; - for batch in batches { - writer.write_batch(&batch).await?; - } - Ok(writer.finish().await?.num_rows as usize) - } - - async fn do_write_buffer(writer: &mut (impl AsyncWrite + Unpin), buf: &[u8]) -> Result<()> { - writer.write_all(buf).await?; - let pad_bytes = pad_bytes::(buf.len()); - writer.write_all(&PAD_BUFFER[..pad_bytes]).await?; - Ok(()) - } - - /// Returns the format version that will be used when writing the file - pub fn version(&self) -> LanceFileVersion { - self.options.format_version.unwrap_or_default() - } - - async fn write_page(&mut self, encoded_page: EncodedPage) -> Result<()> { - let buffers = encoded_page.data; - let mut buffer_offsets = Vec::with_capacity(buffers.len()); - let mut buffer_sizes = Vec::with_capacity(buffers.len()); - for buffer in buffers { - buffer_offsets.push(self.writer.tell().await? as u64); - buffer_sizes.push(buffer.len() as u64); - Self::do_write_buffer(&mut self.writer, &buffer).await?; - } - let encoded_encoding = match encoded_page.description { - PageEncoding::Legacy(array_encoding) => Any::from_msg(&array_encoding)?.encode_to_vec(), - PageEncoding::Structural(page_layout) => Any::from_msg(&page_layout)?.encode_to_vec(), - }; - let page = pbfile::column_metadata::Page { - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(DirectEncoding { - encoding: encoded_encoding, - })), - }), - length: encoded_page.num_rows, - priority: encoded_page.row_number, - }; - let col_idx = encoded_page.column_idx as usize; - if matches!(&self.page_spill, Some(PageSpillState::Pending(..))) { - let Some(PageSpillState::Pending(store, path)) = self.page_spill.take() else { - unreachable!() - }; - self.page_spill = Some(PageSpillState::Active( - PageMetadataSpill::new(store, path, self.num_columns as usize).await?, - )); - } - match &mut self.page_spill { - Some(PageSpillState::Active(spill)) => spill.append_page(col_idx, &page).await?, - None => self.column_metadata[col_idx].pages.push(page), - Some(PageSpillState::Pending(..)) => unreachable!(), - } - Ok(()) - } - - #[instrument(skip_all, level = "debug")] - async fn write_pages(&mut self, mut encoding_tasks: FuturesOrdered) -> Result<()> { - // As soon as an encoding task is done we write it. There is no parallelism - // needed here because "writing" is really just submitting the buffer to the - // underlying write scheduler (either the OS or object_store's scheduler for - // cloud writes). The only time we might truly await on write_page is if the - // scheduler's write queue is full. - // - // Also, there is no point in trying to make write_page parallel anyways - // because we wouldn't want buffers getting mixed up across pages. - while let Some(encoding_task) = encoding_tasks.next().await { - let encoded_page = encoding_task?; - self.write_page(encoded_page).await?; - } - // It's important to flush here, we don't know when the next batch will arrive - // and the underlying cloud store could have writes in progress that won't advance - // until we interact with the writer again. These in-progress writes will time out - // if we don't flush. - self.writer.flush().await?; - Ok(()) - } - - /// Schedule batches of data to be written to the file + /// Spill page metadata to a sidecar file. + pub fn with_page_metadata_spill(self, object_store: Arc, path: Path) -> Self { + match self { + Self::V2_0(writer) => Self::V2_0(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + Self::V2_1(writer) => Self::V2_1(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + Self::V2_2(writer) => Self::V2_2(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + Self::V2_3(writer) => Self::V2_3(Box::new( + (*writer).with_page_metadata_spill(object_store, path), + )), + } + } + + /// Schedule batches of data to be written to the file. pub async fn write_batches( &mut self, batches: impl Iterator, @@ -407,2056 +153,127 @@ impl FileWriter { Ok(()) } - fn verify_field_nullability(arr: &ArrayData, field: &Field) -> Result<()> { - if !field.nullable && arr.null_count() > 0 { - return Err(Error::invalid_input(format!( - "The field `{}` contained null values even though the field is marked non-null in the schema", - field.name - ))); - } - - for (child_field, child_arr) in field.children.iter().zip(arr.child_data()) { - Self::verify_field_nullability(child_arr, child_field)?; - } - - Ok(()) - } - - fn verify_nullability_constraints(&self, batch: &RecordBatch) -> Result<()> { - for (col, field) in batch - .columns() - .iter() - .zip(self.schema.as_ref().unwrap().fields.iter()) - { - Self::verify_field_nullability(&col.to_data(), field)?; - } - Ok(()) - } - - fn initialize(&mut self, mut schema: LanceSchema) -> Result<()> { - let cache_bytes_per_column = if let Some(data_cache_bytes) = self.options.data_cache_bytes { - data_cache_bytes / schema.fields.len() as u64 - } else { - 8 * 1024 * 1024 - }; - - let max_page_bytes = self.options.max_page_bytes.unwrap_or_else(|| { - std::env::var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES) - .map(|s| { - s.parse::().unwrap_or_else(|e| { - warn!( - "Failed to parse {}: {}, using default", - ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, e - ); - MAX_PAGE_BYTES as u64 - }) - }) - .unwrap_or(MAX_PAGE_BYTES as u64) - }); - - schema.validate()?; - - let keep_original_array = self.options.keep_original_array.unwrap_or(false); - let encoding_strategy = self.options.encoding_strategy.clone().unwrap_or_else(|| { - let version = self.version(); - default_encoding_strategy(version).into() - }); - - let encoding_options = EncodingOptions { - cache_bytes_per_column, - max_page_bytes, - keep_original_array, - buffer_alignment: PAGE_BUFFER_ALIGNMENT as u64, - version: self.version(), - }; - let encoder = - BatchEncoder::try_new(&schema, encoding_strategy.as_ref(), &encoding_options)?; - self.num_columns = encoder.num_columns(); - - self.field_rows_written = vec![0; encoder.field_encoders.len()]; - self.column_writers = encoder.field_encoders; - self.column_metadata = vec![initial_column_metadata(); self.num_columns as usize]; - self.field_id_to_column_indices = encoder.field_id_to_column_index; - self.schema_metadata - .extend(std::mem::take(&mut schema.metadata)); - self.schema = Some(schema); - Ok(()) - } - - fn ensure_initialized(&mut self, batch: &RecordBatch) -> Result<&LanceSchema> { - if self.schema.is_none() { - let schema = LanceSchema::try_from(batch.schema().as_ref())?; - self.initialize(schema)?; - } - Ok(self.schema.as_ref().unwrap()) - } - - #[instrument(skip_all, level = "debug")] - fn encode_batch( - &mut self, - batch: &RecordBatch, - external_buffers: &mut OutOfLineBuffers, - ) -> Result>> { - let field_arrays = self - .schema - .as_ref() - .unwrap() - .fields - .iter() - .enumerate() - .map(|(field_idx, field)| { - let array = - batch - .column_by_name(&field.name) - .ok_or(Error::invalid_input_source( - format!( - "Cannot write batch. The batch was missing the column `{}`", - field.name - ) - .into(), - ))?; - Ok((field_idx, array.clone())) - }) - .collect::>>()?; - self.encode_columns(&field_arrays, external_buffers) - } - - // Encode a set of `(field index, array)` pairs, each advancing only its own - // column. Each task captures its field's current row offset at encode time, - // so `advance_columns` must run after this call (never before); the order of - // the returned tasks relative to `write_pages` does not matter. - fn encode_columns( - &mut self, - field_arrays: &[(usize, ArrayRef)], - external_buffers: &mut OutOfLineBuffers, - ) -> Result>> { - // Snapshot the starting row number of each field before borrowing the - // column writers mutably below. - let row_numbers = field_arrays - .iter() - .map(|(field_idx, _)| self.field_rows_written[*field_idx]) - .collect::>(); - field_arrays - .iter() - .zip(row_numbers) - .map(|((field_idx, array), row_number)| { - let repdef = RepDefBuilder::default(); - let num_rows = array.len() as u64; - self.column_writers[*field_idx].maybe_encode( - array.clone(), - external_buffers, - repdef, - row_number, - num_rows, - ) - }) - .collect::>>() - } - - // Advance the per-field row counters after a set of columns has been - // written, keeping `rows_written` (the file's logical length) in sync as the - // longest column. Only the written fields move, so their new totals fold into - // `rows_written` directly without rescanning every field. (`write_batch` - // advances every field uniformly and tracks this inline instead.) - fn advance_columns(&mut self, field_arrays: &[(usize, ArrayRef)]) { - for (field_idx, array) in field_arrays { - let new_total = self.field_rows_written[*field_idx] + array.len() as u64; - self.field_rows_written[*field_idx] = new_total; - self.rows_written = self.rows_written.max(new_total); - } - } - - /// Schedule a batch of data to be written to the file - /// - /// Note: the future returned by this method may complete before the data has been fully - /// flushed to the file (some data may be in the data cache or the I/O cache) + /// Schedule a batch of data to be written to the file. pub async fn write_batch(&mut self, batch: &RecordBatch) -> Result<()> { - debug!( - "write_batch called with {} rows, {} columns, and {} bytes of data", - batch.num_rows(), - batch.num_columns(), - batch.get_array_memory_size() - ); - self.ensure_initialized(batch)?; - self.verify_nullability_constraints(batch)?; - let num_rows = batch.num_rows() as u64; - if num_rows == 0 { - return Ok(()); - } - if num_rows > u32::MAX as u64 { - return Err(Error::invalid_input_source( - "cannot write Lance files with more than 2^32 rows".into(), - )); - } - // First we push each array into its column writer. This may or may not generate enough - // data to trigger an encoding task. We collect any encoding tasks into a queue. - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let encoding_tasks = self.encode_batch(batch, &mut external_buffers)?; - // Next, write external buffers - for external_buffer in external_buffers.take_buffers() { - Self::do_write_buffer(&mut self.writer, &external_buffer).await?; - } - - let encoding_tasks = encoding_tasks - .into_iter() - .flatten() - .collect::>(); - - // `write_batch` advances every field by the same amount, so the longest - // column simply grows by `num_rows`. Guard against overflowing the row - // counter. - if self.rows_written.checked_add(num_rows).is_none() { - return Err(Error::invalid_input_source(format!("cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", num_rows, self.rows_written).into())); - } - for field_rows in self.field_rows_written.iter_mut() { - *field_rows += num_rows; + match self { + Self::V2_0(writer) => writer.write_batch(batch).await, + Self::V2_1(writer) => writer.write_batch(batch).await, + Self::V2_2(writer) => writer.write_batch(batch).await, + Self::V2_3(writer) => writer.write_batch(batch).await, } - self.rows_written += num_rows; - - self.write_pages(encoding_tasks).await?; - - Ok(()) } - /// Write a single column, advancing only that column's row counter. - /// - /// Unlike [`write_batch`](Self::write_batch), which advances every column - /// from a single shared row counter, this method advances one column - /// independently. Used across calls it produces a single file whose columns - /// may have different item counts. - /// - /// `column_index` refers to a top-level field in the writer's schema (the - /// same order as the schema's fields); a nested child cannot be targeted on - /// its own. Because each call writes the whole field from a single array, the - /// children of a struct field always advance together and stay equal-length; - /// only different top-level fields can diverge in length. A column may be - /// written across multiple calls; its values are appended. A field that is - /// never written ends up as a zero-length column. The writer must have been - /// created with an explicit schema (via [`try_new`](Self::try_new)); a lazy - /// schema cannot be inferred here because individual calls need not cover - /// every field. - /// - /// ``` - /// # use arrow_array::{ArrayRef, Int32Array}; - /// # use std::sync::Arc; - /// # use lance_file::writer::FileWriter; - /// # async fn example(writer: &mut FileWriter) -> lance_core::Result<()> { - /// // Field 0 gets three values, field 1 gets one — a non-rectangular file. - /// writer.write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))).await?; - /// writer.write_column(1, Arc::new(Int32Array::from(vec![10]))).await?; - /// # Ok(()) - /// # } - /// ``` + /// Write one top-level column. pub async fn write_column(&mut self, column_index: usize, array: ArrayRef) -> Result<()> { - let schema = self.schema.as_ref().ok_or_else(|| { - Error::invalid_input_source( - "write_column requires the writer to be created with an explicit schema".into(), - ) - })?; - let field = schema.fields.get(column_index).ok_or_else(|| { - Error::invalid_input_source( - format!( - "write_column: field index {} is out of bounds (schema has {} fields)", - column_index, - schema.fields.len() - ) - .into(), - ) - })?; - if array.len() as u64 > u32::MAX as u64 { - return Err(Error::invalid_input_source( - "cannot write Lance files with more than 2^32 rows".into(), - )); - } - Self::verify_field_nullability(&array.to_data(), field)?; - - // A never-advanced field simply remains a zero-length column, which the - // encoders handle at `finish` time. - if array.is_empty() { - return Ok(()); - } - - let columns = [(column_index, array)]; - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let encoding_tasks = self.encode_columns(&columns, &mut external_buffers)?; - for external_buffer in external_buffers.take_buffers() { - Self::do_write_buffer(&mut self.writer, &external_buffer).await?; + match self { + Self::V2_0(writer) => writer.write_column(column_index, array).await, + Self::V2_1(writer) => writer.write_column(column_index, array).await, + Self::V2_2(writer) => writer.write_column(column_index, array).await, + Self::V2_3(writer) => writer.write_column(column_index, array).await, } - let encoding_tasks = encoding_tasks - .into_iter() - .flatten() - .collect::>(); - - self.advance_columns(&columns); - self.write_pages(encoding_tasks).await?; - Ok(()) - } - - async fn write_column_metadata( - &mut self, - metadata: pbfile::ColumnMetadata, - ) -> Result<(u64, u64)> { - let metadata_bytes = metadata.encode_to_vec(); - let position = self.writer.tell().await? as u64; - let len = metadata_bytes.len() as u64; - self.writer.write_all(&metadata_bytes).await?; - Ok((position, len)) } - async fn write_column_metadatas(&mut self) -> Result> { - let metadatas = std::mem::take(&mut self.column_metadata); - - // If spilling, finalize the spill writer and reopen for reading. - // The spill file itself is cleaned up by the caller (it lives in a - // temp directory managed by the caller's RAII guard). - let spill_state = self.page_spill.take(); - let (spill_chunks, spill_reader) = - if let Some(PageSpillState::Active(mut spill)) = spill_state { - spill.shutdown_writer().await?; - let reader = spill.object_store.open(&spill.path).await?; - let chunks = std::mem::take(&mut spill.column_chunks); - (chunks, Some(reader)) - } else { - (Vec::new(), None) - }; - - let mut metadata_positions = Vec::with_capacity(metadatas.len()); - for (col_idx, mut metadata) in metadatas.into_iter().enumerate() { - if let Some(reader) = &spill_reader { - let mut pages = Vec::new(); - for &(offset, len) in &spill_chunks[col_idx] { - let data = reader - .get_range(offset as usize..(offset as usize + len as usize)) - .await - .map_err(|e| Error::io_source(Box::new(e)))?; - pages.extend(decode_spilled_chunk(&data)?); - } - metadata.pages = pages; - } - metadata_positions.push(self.write_column_metadata(metadata).await?); + /// Append a buffer whose page or column metadata is supplied externally. + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + match self { + Self::V2_0(writer) => writer.write_external_buffer(bytes).await, + Self::V2_1(writer) => writer.write_external_buffer(bytes).await, + Self::V2_2(writer) => writer.write_external_buffer(bytes).await, + Self::V2_3(writer) => writer.write_external_buffer(bytes).await, } - - Ok(metadata_positions) } - fn make_file_descriptor( - schema: &lance_core::datatypes::Schema, - num_rows: u64, - ) -> Result { - let fields_with_meta = FieldsWithMeta::from(schema); - Ok(pb::FileDescriptor { - schema: Some(pb::Schema { - fields: fields_with_meta.fields.0, - metadata: fields_with_meta.metadata, - }), - length: num_rows, - }) - } - - async fn write_global_buffers(&mut self) -> Result> { - let schema = self.schema.as_mut().ok_or(Error::invalid_input("No schema provided on writer open and no data provided. Schema is unknown and file cannot be created"))?; - schema.metadata = std::mem::take(&mut self.schema_metadata); - // Use descriptor layout for blob v2 fields in the footer to avoid exposing logical child fields. - schema - .fields - .iter_mut() - .for_each(|f| f.unload_blobs_recursive()); - - let file_descriptor = Self::make_file_descriptor(schema, self.rows_written)?; - let file_descriptor_bytes = file_descriptor.encode_to_vec(); - let file_descriptor_len = file_descriptor_bytes.len() as u64; - let file_descriptor_position = self.writer.tell().await? as u64; - self.writer.write_all(&file_descriptor_bytes).await?; - let mut gbo_table = Vec::with_capacity(1 + self.global_buffers.len()); - gbo_table.push((file_descriptor_position, file_descriptor_len)); - gbo_table.append(&mut self.global_buffers); - Ok(gbo_table) - } - - /// Add a metadata entry to the schema - /// - /// This method is useful because sometimes the metadata is not known until after the - /// data has been written. This method allows you to alter the schema metadata. It - /// must be called before `finish` is called. + /// Add a metadata entry to the schema. pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { - self.schema_metadata.insert(key.into(), value.into()); + let key = key.into(); + let value = value.into(); + match self { + Self::V2_0(writer) => writer.add_schema_metadata(key, value), + Self::V2_1(writer) => writer.add_schema_metadata(key, value), + Self::V2_2(writer) => writer.add_schema_metadata(key, value), + Self::V2_3(writer) => writer.add_schema_metadata(key, value), + } } - /// Prepare the writer when column data and metadata were produced externally. - /// - /// This is useful for flows that copy already-encoded pages (e.g., binary copy - /// during compaction) where the column buffers have been written directly and we - /// only need to write the footer and schema metadata. The provided - /// `column_metadata` must describe the buffers already persisted by the - /// underlying `ObjectWriter`, and `rows_written` should reflect the total number - /// of rows in those buffers. - pub fn initialize_with_external_metadata( + /// Prepare a writer from encoded columns whose buffers were produced externally. + pub fn initialize_with_external_columns( &mut self, - schema: lance_core::datatypes::Schema, - column_metadata: Vec, + schema: Schema, + columns: &[Arc], rows_written: u64, - ) { - self.schema = Some(schema); - self.num_columns = column_metadata.len() as u32; - self.column_metadata = column_metadata; - self.rows_written = rows_written; - } - - /// Adds a global buffer to the file - /// - /// The global buffer can contain any arbitrary bytes. It will be written to the disk - /// immediately. This method returns the index of the global buffer (this will always - /// start at 1 and increment by 1 each time this method is called) - pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { - let position = self.writer.tell().await? as u64; - let len = buffer.len() as u64; - Self::do_write_buffer(&mut self.writer, &buffer).await?; - self.global_buffers.push((position, len)); - Ok(self.global_buffers.len() as u32) - } - - async fn finish_writers(&mut self) -> Result<()> { - let mut col_idx = 0; - for mut writer in std::mem::take(&mut self.column_writers) { - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let columns = writer.finish(&mut external_buffers).await?; - for buffer in external_buffers.take_buffers() { - self.writer.write_all(&buffer).await?; + ) -> Result<()> { + let column_metadata = columns + .iter() + .map(|column| column_info_to_metadata(column)) + .collect::>>()?; + match self { + Self::V2_0(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) } - debug_assert_eq!( - columns.len(), - writer.num_columns() as usize, - "Expected {} columns from column at index {} and got {}", - writer.num_columns(), - col_idx, - columns.len() - ); - for column in columns { - for page in column.final_pages { - self.write_page(page).await?; - } - let column_metadata = &mut self.column_metadata[col_idx]; - let mut buffer_pos = self.writer.tell().await? as u64; - for buffer in column.column_buffers { - column_metadata.buffer_offsets.push(buffer_pos); - let mut size = 0; - Self::do_write_buffer(&mut self.writer, &buffer).await?; - size += buffer.len() as u64; - buffer_pos += size; - column_metadata.buffer_sizes.push(size); - } - let encoded_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); - column_metadata.encoding = Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { - encoding: encoded_encoding, - })), - }); - col_idx += 1; + Self::V2_1(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) + } + Self::V2_2(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) + } + Self::V2_3(writer) => { + writer.initialize_with_external_metadata(schema, column_metadata, rows_written) } - } - if col_idx != self.column_metadata.len() { - panic!( - "Column writers finished with {} columns but we expected {}", - col_idx, - self.column_metadata.len() - ); } Ok(()) } - /// Converts self.version (which is a mix of "software version" and - /// "format version" into a format version) - fn version_to_numbers(&self) -> (u16, u16) { - let version = self.options.format_version.unwrap_or_default(); - match version.resolve() { - LanceFileVersion::V2_0 => (0, 3), - LanceFileVersion::V2_1 => (2, 1), - LanceFileVersion::V2_2 => (2, 2), - LanceFileVersion::V2_3 => (2, 3), - _ => panic!("Unsupported version: {}", version), + /// Add an arbitrary global buffer and return its one-based index. + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + match self { + Self::V2_0(writer) => writer.add_global_buffer(buffer).await, + Self::V2_1(writer) => writer.add_global_buffer(buffer).await, + Self::V2_2(writer) => writer.add_global_buffer(buffer).await, + Self::V2_3(writer) => writer.add_global_buffer(buffer).await, } } - /// Finishes writing the file - /// - /// This method will wait until all data has been flushed to the file. Then it - /// will write the file metadata and the footer. It will not return until all - /// data has been flushed and the file has been closed. - /// - /// Returns a summary of the completed file write. + /// Finish the file and close its object writer. pub async fn finish(&mut self) -> Result { - // 1. flush any remaining data and write out those pages - let mut external_buffers = - OutOfLineBuffers::new(self.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); - let encoding_tasks = self - .column_writers - .iter_mut() - .map(|writer| writer.flush(&mut external_buffers)) - .collect::>>()?; - for external_buffer in external_buffers.take_buffers() { - Self::do_write_buffer(&mut self.writer, &external_buffer).await?; - } - let encoding_tasks = encoding_tasks - .into_iter() - .flatten() - .collect::>(); - self.write_pages(encoding_tasks).await?; - - if !self.column_writers.is_empty() { - self.finish_writers().await?; + match self { + Self::V2_0(writer) => writer.finish().await, + Self::V2_1(writer) => writer.finish().await, + Self::V2_2(writer) => writer.finish().await, + Self::V2_3(writer) => writer.finish().await, } - - // 3. write global buffers (we write the schema here) - let global_buffer_offsets = self.write_global_buffers().await?; - let num_global_buffers = global_buffer_offsets.len() as u32; - - // 4. write the column metadatas - let column_metadata_start = self.writer.tell().await? as u64; - let metadata_positions = self.write_column_metadatas().await?; - - // 5. write the column metadata offset table - let cmo_table_start = self.writer.tell().await? as u64; - for (meta_pos, meta_len) in metadata_positions { - self.writer.write_u64_le(meta_pos).await?; - self.writer.write_u64_le(meta_len).await?; - } - - // 6. write global buffers offset table - let gbo_table_start = self.writer.tell().await? as u64; - for (gbo_pos, gbo_len) in global_buffer_offsets { - self.writer.write_u64_le(gbo_pos).await?; - self.writer.write_u64_le(gbo_len).await?; - } - - let (major, minor) = self.version_to_numbers(); - // 7. write the footer - self.writer.write_u64_le(column_metadata_start).await?; - self.writer.write_u64_le(cmo_table_start).await?; - self.writer.write_u64_le(gbo_table_start).await?; - self.writer.write_u32_le(num_global_buffers).await?; - self.writer.write_u32_le(self.num_columns).await?; - self.writer.write_u16_le(major).await?; - self.writer.write_u16_le(minor).await?; - self.writer.write_all(MAGIC).await?; - - // 7. close the writer - let write_result = Writer::shutdown(self.writer.as_mut()).await?; - - Ok(FileWriteSummary { - num_rows: self.rows_written, - size_bytes: write_result.size as u64, - }) } + /// Abandon the file write. pub async fn abort(&mut self) { - // For multipart uploads, ObjectWriter's Drop impl will abort - // the upload when the writer is dropped. + match self { + Self::V2_0(writer) => writer.abort().await, + Self::V2_1(writer) => writer.abort().await, + Self::V2_2(writer) => writer.abort().await, + Self::V2_3(writer) => writer.abort().await, + } } + /// Return the current object-writer position. pub async fn tell(&mut self) -> Result { - Ok(self.writer.tell().await? as u64) + match self { + Self::V2_0(writer) => writer.tell().await, + Self::V2_1(writer) => writer.tell().await, + Self::V2_2(writer) => writer.tell().await, + Self::V2_3(writer) => writer.tell().await, + } } + /// Return the field-id to physical-column mapping. pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { - &self.field_id_to_column_indices - } -} - -/// Utility trait for converting EncodedBatch to Bytes using the -/// lance file format -pub trait EncodedBatchWriteExt { - /// Serializes into a lance file, including the schema - fn try_to_self_described_lance(&self, version: LanceFileVersion) -> Result; - /// Serializes into a lance file, without the schema. - /// - /// The schema must be provided to deserialize the buffer - fn try_to_mini_lance(&self, version: LanceFileVersion) -> Result; -} - -// Creates a lance footer and appends it to the encoded data -// -// The logic here is very similar to logic in the FileWriter except we -// are using BufMut (put_xyz) instead of AsyncWrite (write_xyz). -fn concat_lance_footer( - batch: &EncodedBatch, - write_schema: bool, - version: LanceFileVersion, -) -> Result { - // Estimating 1MiB for file footer - let mut data = BytesMut::with_capacity(batch.data.len() + 1024 * 1024); - data.put(batch.data.clone()); - // write global buffers (we write the schema here) - let global_buffers = if write_schema { - let schema_start = data.len() as u64; - let lance_schema = lance_core::datatypes::Schema::try_from(batch.schema.as_ref())?; - let descriptor = FileWriter::make_file_descriptor(&lance_schema, batch.num_rows)?; - let descriptor_bytes = descriptor.encode_to_vec(); - let descriptor_len = descriptor_bytes.len() as u64; - data.put(descriptor_bytes.as_slice()); - - vec![(schema_start, descriptor_len)] - } else { - vec![] - }; - let col_metadata_start = data.len() as u64; - - let mut col_metadata_positions = Vec::new(); - // Write column metadata - for col in &batch.page_table { - let position = data.len() as u64; - let pages = col - .page_infos - .iter() - .map(|page_info| { - let encoded_encoding = match &page_info.encoding { - PageEncoding::Legacy(array_encoding) => { - Any::from_msg(array_encoding)?.encode_to_vec() - } - PageEncoding::Structural(page_layout) => { - Any::from_msg(page_layout)?.encode_to_vec() - } - }; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = page_info - .buffer_offsets_and_sizes - .as_ref() - .iter() - .cloned() - .unzip(); - Ok(pbfile::column_metadata::Page { - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(DirectEncoding { - encoding: encoded_encoding, - })), - }), - length: page_info.num_rows, - priority: page_info.priority, - }) - }) - .collect::>>()?; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = - col.buffer_offsets_and_sizes.iter().cloned().unzip(); - let encoded_col_encoding = Any::from_msg(&col.encoding)?.encode_to_vec(); - let column = pbfile::ColumnMetadata { - pages, - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { - encoding: encoded_col_encoding, - })), - }), - }; - let column_bytes = column.encode_to_vec(); - col_metadata_positions.push((position, column_bytes.len() as u64)); - data.put(column_bytes.as_slice()); - } - // Write column metadata offsets table - let cmo_table_start = data.len() as u64; - for (meta_pos, meta_len) in col_metadata_positions { - data.put_u64_le(meta_pos); - data.put_u64_le(meta_len); - } - // Write global buffers offsets table - let gbo_table_start = data.len() as u64; - let num_global_buffers = global_buffers.len() as u32; - for (gbo_pos, gbo_len) in global_buffers { - data.put_u64_le(gbo_pos); - data.put_u64_le(gbo_len); - } - - let (major, minor) = version.to_numbers(); - - // write the footer - data.put_u64_le(col_metadata_start); - data.put_u64_le(cmo_table_start); - data.put_u64_le(gbo_table_start); - data.put_u32_le(num_global_buffers); - data.put_u32_le(batch.page_table.len() as u32); - data.put_u16_le(major as u16); - data.put_u16_le(minor as u16); - data.put(MAGIC.as_slice()); - - Ok(data.freeze()) -} - -impl EncodedBatchWriteExt for EncodedBatch { - fn try_to_self_described_lance(&self, version: LanceFileVersion) -> Result { - concat_lance_footer(self, true, version) - } - - fn try_to_mini_lance(&self, version: LanceFileVersion) -> Result { - concat_lance_footer(self, false, version) + match self { + Self::V2_0(writer) => writer.field_id_to_column_indices(), + Self::V2_1(writer) => writer.field_id_to_column_indices(), + Self::V2_2(writer) => writer.field_id_to_column_indices(), + Self::V2_3(writer) => writer.field_id_to_column_indices(), + } } } #[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Arc; - - use crate::reader::{FileReader, FileReaderOptions, ReaderProjection, describe_encoding}; - use crate::testing::FsFixture; - use crate::writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriter, FileWriterOptions}; - use arrow_array::builder::{Float32Builder, Int32Builder}; - use arrow_array::{ArrayRef, Int32Array, RecordBatch, UInt64Array}; - use arrow_array::{RecordBatchReader, StringArray, types::Float64Type}; - use arrow_schema::{DataType, Field, Field as ArrowField, Schema, Schema as ArrowSchema}; - use lance_core::cache::LanceCache; - use lance_core::datatypes::Schema as LanceSchema; - use lance_core::utils::tempfile::TempObjFile; - use lance_datagen::{BatchCount, RowCount, array, gen_batch}; - use lance_encoding::compression_config::{CompressionFieldParams, CompressionParams}; - use lance_encoding::decoder::DecoderPlugins; - use lance_encoding::version::LanceFileVersion; - use lance_io::object_store::ObjectStore; - use lance_io::utils::CachedFileSize; - use rstest::rstest; - - #[tokio::test] - async fn test_basic_write() { - let tmp_path = TempObjFile::default(); - let obj_store = Arc::new(ObjectStore::local()); - - let reader = gen_batch() - .col("score", array::rand::()) - .into_reader_rows(RowCount::from(1000), BatchCount::from(10)); - - let writer = obj_store.create(&tmp_path).await.unwrap(); - - let lance_schema = - lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); - - let mut file_writer = - FileWriter::try_new(writer, lance_schema, FileWriterOptions::default()).unwrap(); - - for batch in reader { - file_writer.write_batch(&batch.unwrap()).await.unwrap(); - } - file_writer.add_schema_metadata("foo", "bar"); - file_writer.finish().await.unwrap(); - // Tests asserting the contents of the written file are in reader.rs - } - - #[tokio::test] - async fn test_write_empty() { - let tmp_path = TempObjFile::default(); - let obj_store = Arc::new(ObjectStore::local()); - - let reader = gen_batch() - .col("score", array::rand::()) - .into_reader_rows(RowCount::from(0), BatchCount::from(0)); - - let writer = obj_store.create(&tmp_path).await.unwrap(); - - let lance_schema = - lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); - - let mut file_writer = - FileWriter::try_new(writer, lance_schema, FileWriterOptions::default()).unwrap(); - - for batch in reader { - file_writer.write_batch(&batch.unwrap()).await.unwrap(); - } - file_writer.add_schema_metadata("foo", "bar"); - file_writer.finish().await.unwrap(); - } - - // Read a single column back at an explicit range/index set, returning its - // `Int32` values. Reading one column (or an equal-length group) at a time is - // how unequal-length files are consumed: a full scan across columns of - // differing lengths cannot form a single rectangular batch. - async fn read_int32_column( - reader: &FileReader, - schema: &LanceSchema, - version: LanceFileVersion, - name: &str, - params: lance_io::ReadBatchParams, - ) -> Vec> { - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - - let projection = ReaderProjection::from_column_names(version, schema, &[name]).unwrap(); - let batches: Vec = reader - .read_stream_projected(params, 1024, 16, projection, FilterExpression::no_filter()) - .await - .unwrap() - .try_collect() - .await - .unwrap(); - batches - .iter() - .flat_map(|b| { - b.column(0) - .as_any() - .downcast_ref::() - .unwrap() - .iter() - .collect::>() - }) - .collect() - } - - /// A single file may hold columns of differing item counts, written by - /// advancing each column's row counter independently (no shared global - /// counter). - #[rstest] - #[tokio::test] - async fn test_write_columns_unequal_lengths( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use lance_io::ReadBatchParams; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ArrowField::new("c", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - // Field "a" gets 5 values across two calls (appending), field "b" gets a - // single value, and field "c" is never written (a zero-length column). - let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); - let b: ArrayRef = Arc::new(Int32Array::from(vec![10])); - writer.write_column(0, a1).await.unwrap(); - writer.write_column(1, b).await.unwrap(); - let a2: ArrayRef = Arc::new(Int32Array::from(vec![4, 5])); - writer.write_column(0, a2).await.unwrap(); - // An empty array is a no-op whether or not the field already has rows: - // field "a" keeps its 5 rows, field "c" stays a zero-length column. - let empty: ArrayRef = Arc::new(Int32Array::from(Vec::::new())); - writer.write_column(0, empty.clone()).await.unwrap(); - writer.write_column(2, empty).await.unwrap(); - - let summary = writer.finish().await.unwrap(); - // The file's logical length is the longest column. - assert_eq!(summary.num_rows, 5); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - // Per-column row counts are recorded in / derivable from file metadata. - assert_eq!(reader.num_rows(), 5); - assert_eq!(reader.column_num_rows(0).unwrap(), 5); - assert_eq!(reader.column_num_rows(1).unwrap(), 1); - assert_eq!(reader.column_num_rows(2).unwrap(), 0); - assert!(reader.column_num_rows(3).is_err()); - - // Each column reads back independently at its own length. - assert_eq!( - read_int32_column( - &reader, - &lance_schema, - version, - "a", - ReadBatchParams::Range(0..5) - ) - .await, - vec![Some(1), Some(2), Some(3), Some(4), Some(5)], - ); - assert_eq!( - read_int32_column( - &reader, - &lance_schema, - version, - "b", - ReadBatchParams::Range(0..1) - ) - .await, - vec![Some(10)], - ); - - // Random access by position within the longer column returns the right - // value even though other columns are shorter. (The take path requires - // strictly increasing indices.) - assert_eq!( - read_int32_column( - &reader, - &lance_schema, - version, - "a", - ReadBatchParams::Indices(arrow_array::UInt32Array::from(vec![0, 2, 4])), - ) - .await, - vec![Some(1), Some(3), Some(5)], - ); - } - - /// Reading an unequal-length file: - /// - a projection whose columns are equal length full-scans normally; - /// - a full scan across columns of differing length is rejected up front, - /// before any batch is produced (even though a prefix would be rectangular); - /// - a bounded read is valid as long as every projected column covers it; - /// - a single-column `RangeFull` resolves to that column's own length, not - /// the file's (maximum) length. - #[rstest] - #[tokio::test] - async fn test_read_unequal_length_projection( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ArrowField::new("c", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - // "a" and "b" are equal length (5); "c" is shorter (1). - writer - .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) - .await - .unwrap(); - writer - .write_column(1, Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))) - .await - .unwrap(); - writer - .write_column(2, Arc::new(Int32Array::from(vec![100]))) - .await - .unwrap(); - writer.finish().await.unwrap(); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let read = |names: &'static [&'static str], params: ReadBatchParams| { - let projection = - ReaderProjection::from_column_names(version, &lance_schema, names).unwrap(); - async { - match reader - .read_stream_projected( - params, - 1024, - 16, - projection, - FilterExpression::no_filter(), - ) - .await - { - Ok(stream) => stream.try_collect::>().await, - Err(e) => Err(e), - } - } - }; - let col_values = |batches: &[RecordBatch], idx: usize| -> Vec> { - batches - .iter() - .flat_map(|b| { - b.column(idx) - .as_any() - .downcast_ref::() - .unwrap() - .iter() - .collect::>() - }) - .collect() - }; - - // Equal-length projection [a, b] full-scans into rectangular batches. - let batches = read(&["a", "b"], ReadBatchParams::RangeFull).await.unwrap(); - assert_eq!( - col_values(&batches, 0), - vec![Some(1), Some(2), Some(3), Some(4), Some(5)] - ); - assert_eq!( - col_values(&batches, 1), - vec![Some(6), Some(7), Some(8), Some(9), Some(10)] - ); - - // A mismatched-length projection [a, c] (5 vs 1) is rejected before any - // batch is yielded, regardless of the read params — its columns cannot - // be combined into rectangular batches. The error names each column's - // length so the caller can see which column is the odd one out. - let err = read(&["a", "c"], ReadBatchParams::RangeFull) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains("a=5") && err.contains("c=1"), - "error should name each column's length, got: {err}" - ); - assert!( - read(&["a", "c"], ReadBatchParams::Range(0..1)) - .await - .is_err(), - "even a common-prefix read of unequal-length columns must error" - ); - - // A single-column RangeFull resolves to that column's own length. - let batches = read(&["c"], ReadBatchParams::RangeFull).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(100)]); - let batches = read(&["a"], ReadBatchParams::RangeFull).await.unwrap(); - assert_eq!( - col_values(&batches, 0), - vec![Some(1), Some(2), Some(3), Some(4), Some(5)] - ); - - // RangeFrom/RangeTo likewise resolve against the projected column's own - // length rather than the file's longest column. - let batches = read(&["a"], ReadBatchParams::RangeFrom(2..)).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(3), Some(4), Some(5)]); - // RangeFrom on the short column "c" resolves to length 1, not 5. - let batches = read(&["c"], ReadBatchParams::RangeFrom(0..)).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(100)]); - let batches = read(&["a"], ReadBatchParams::RangeTo(..3)).await.unwrap(); - assert_eq!(col_values(&batches, 0), vec![Some(1), Some(2), Some(3)]); - // A bound past the projected column's length errors. - assert!( - read(&["a"], ReadBatchParams::RangeTo(..6)).await.is_err(), - "RangeTo past the column length must error" - ); - assert!( - read(&["c"], ReadBatchParams::RangeFrom(2..)).await.is_err(), - "RangeFrom past the column length must error" - ); - } - - /// A struct and a list column each map to multiple physical columns, and a - /// list's item column is longer than its top-level row count. The - /// projection-length check must partition `column_indices` by top-level - /// field and use each field's root column, so an ordinary (rectangular) file - /// with nested columns still reads under the new validation path. - #[rstest] - #[tokio::test] - async fn test_read_nested_columns_under_validation( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use arrow_array::types::Int32Type; - use arrow_array::{ListArray, StructArray}; - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - let struct_type = DataType::Struct( - vec![ - ArrowField::new("x", DataType::Int32, true), - ArrowField::new("y", DataType::Int32, true), - ] - .into(), - ); - let list_type = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))); - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("s", struct_type, true), - ArrowField::new("lst", list_type, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); - let s: ArrayRef = Arc::new(StructArray::from(vec![ - ( - Arc::new(ArrowField::new("x", DataType::Int32, true)), - Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, - ), - ( - Arc::new(ArrowField::new("y", DataType::Int32, true)), - Arc::new(Int32Array::from(vec![11, 21, 31])) as ArrayRef, - ), - ])); - // 3 lists, 6 items: the item column is longer than the top-level rows. - let lst: ArrayRef = Arc::new(ListArray::from_iter_primitive::(vec![ - Some(vec![Some(1), Some(2)]), - Some(vec![Some(3)]), - Some(vec![Some(4), Some(5), Some(6)]), - ])); - let batch = RecordBatch::try_new(arrow_schema.clone(), vec![a, s, lst]).unwrap(); - - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - // If `validate_field_length` mispartitioned the physical columns, the - // length check would read the wrong root column (e.g. the list's item - // column, length 6) and spuriously reject this rectangular file. - for names in [&["a", "s", "lst"][..], &["a", "lst"][..], &["a", "s"][..]] { - let projection = - ReaderProjection::from_column_names(version, &lance_schema, names).unwrap(); - let batches: Vec = reader - .read_stream_projected( - ReadBatchParams::RangeFull, - 1024, - 16, - projection, - FilterExpression::no_filter(), - ) - .await - .unwrap() - .try_collect() - .await - .unwrap(); - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!( - total_rows, 3, - "projection {names:?} should read 3 top-level rows" - ); - } - } - - /// `write_column` rejects invalid inputs at the API boundary with - /// descriptive errors: a writer without an explicit schema, an - /// out-of-bounds field index, and a null written into a non-nullable field. - #[tokio::test] - async fn test_write_column_validation_errors() { - // A lazy-schema writer cannot infer the schema from a single column. - let fs = FsFixture::default(); - let mut lazy_writer = FileWriter::new_lazy( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - FileWriterOptions::default(), - ); - let err = lazy_writer - .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains("explicit schema"), - "expected explicit-schema error, got: {err}" - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, false), - ArrowField::new("b", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // An out-of-bounds field index is rejected, naming the index and count. - let fs = FsFixture::default(); - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - FileWriterOptions::default(), - ) - .unwrap(); - let err = writer - .write_column(5, Arc::new(Int32Array::from(vec![1]))) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains('5') && err.contains('2'), - "expected out-of-bounds error naming index 5 and 2 fields, got: {err}" - ); - - // A null in a non-nullable field ("a") is rejected. - let err = writer - .write_column(0, Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]))) - .await - .unwrap_err() - .to_string(); - assert!( - err.contains("non-null"), - "expected nullability error, got: {err}" - ); - } - - /// The blocking read path applies the same projection-length validation as - /// the async path: a short single column resolves to its own length, and a - /// mismatched-length projection errors up front. - #[rstest] - #[tokio::test] - async fn test_blocking_read_unequal_length( - #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, - ) { - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("c", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let fs = FsFixture::default(); - let options = FileWriterOptions { - format_version: Some(version), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - writer - .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) - .await - .unwrap(); - writer - .write_column(1, Arc::new(Int32Array::from(vec![100]))) - .await - .unwrap(); - writer.finish().await.unwrap(); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = Arc::new( - FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(), - ); - - // Single short column: RangeFull resolves to its own length (1). - let proj_c = ReaderProjection::from_column_names(version, &lance_schema, &["c"]).unwrap(); - let reader_c = reader.clone(); - let batches = tokio::task::spawn_blocking(move || { - reader_c - .read_stream_projected_blocking( - ReadBatchParams::RangeFull, - 1024, - Some(proj_c), - FilterExpression::no_filter(), - ) - .unwrap() - .collect::, _>>() - .unwrap() - }) - .await - .unwrap(); - let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); - assert_eq!(total_rows, 1); - - // A mismatched projection [a, c] errors on the blocking path too. - let proj_ac = - ReaderProjection::from_column_names(version, &lance_schema, &["a", "c"]).unwrap(); - let reader_ac = reader.clone(); - let is_err = tokio::task::spawn_blocking(move || { - reader_ac - .read_stream_projected_blocking( - ReadBatchParams::RangeFull, - 1024, - Some(proj_ac), - FilterExpression::no_filter(), - ) - .is_err() - }) - .await - .unwrap(); - assert!( - is_err, - "blocking full scan across unequal-length columns must error" - ); - } - - /// Files written the ordinary (rectangular) way keep equal column lengths, - /// so the unequal-length support is backwards compatible. - #[tokio::test] - async fn test_write_batch_keeps_equal_lengths() { - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ])); - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - let fs = FsFixture::default(); - let mut writer = FileWriter::try_new( - fs.object_store.create(&fs.tmp_path).await.unwrap(), - lance_schema, - FileWriterOptions::default(), - ) - .unwrap(); - let batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(Int32Array::from(vec![4, 5, 6])), - ], - ) - .unwrap(); - writer.write_batch(&batch).await.unwrap(); - let summary = writer.finish().await.unwrap(); - assert_eq!(summary.num_rows, 3); - - let file_scheduler = fs - .scheduler - .open_file(&fs.tmp_path, &CachedFileSize::unknown()) - .await - .unwrap(); - let reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - assert_eq!(reader.column_num_rows(0).unwrap(), 3); - assert_eq!(reader.column_num_rows(1).unwrap(), 3); - } - - #[tokio::test] - async fn test_max_page_bytes_enforced() { - let arrow_field = Field::new("data", DataType::UInt64, false); - let arrow_schema = Schema::new(vec![arrow_field]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // 8MiB - let data: Vec = (0..1_000_000).collect(); - let array = UInt64Array::from(data); - let batch = - RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); - - let options = FileWriterOptions { - max_page_bytes: Some(1024 * 1024), // 1MB - // This is a 2.0 only test because 2.1+ splits large pages on read instead of write - format_version: Some(LanceFileVersion::V2_0), - ..Default::default() - }; - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema, - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let column_meta = file_reader.metadata(); - - let mut total_page_num: u32 = 0; - for (col_idx, col_metadata) in column_meta.column_metadatas.iter().enumerate() { - assert!( - !col_metadata.pages.is_empty(), - "Column {} has no pages", - col_idx - ); - - for (page_idx, page) in col_metadata.pages.iter().enumerate() { - total_page_num += 1; - let total_size: u64 = page.buffer_sizes.iter().sum(); - assert!( - total_size <= 1024 * 1024, - "Column {} Page {} size {} exceeds 1MB limit", - col_idx, - page_idx, - total_size - ); - } - } - - assert_eq!(total_page_num, 8) - } - - #[tokio::test(flavor = "current_thread")] - async fn test_max_page_bytes_env_var() { - let arrow_field = Field::new("data", DataType::UInt64, false); - let arrow_schema = Schema::new(vec![arrow_field]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - // 4MiB - let data: Vec = (0..500_000).collect(); - let array = UInt64Array::from(data); - let batch = - RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); - - // 2MiB - unsafe { - std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, "2097152"); - } - - let options = FileWriterOptions { - max_page_bytes: None, // enforce env - ..Default::default() - }; - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - for col_metadata in file_reader.metadata().column_metadatas.iter() { - for page in col_metadata.pages.iter() { - let total_size: u64 = page.buffer_sizes.iter().sum(); - assert!( - total_size <= 2 * 1024 * 1024, - "Page size {} exceeds 2MB limit", - total_size - ); - } - } - - unsafe { - std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, ""); - } - } - - #[tokio::test] - async fn test_compression_overrides_end_to_end() { - // Create test schema with different column types - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("customer_id", DataType::Int32, false), - ArrowField::new("product_id", DataType::Int32, false), - ArrowField::new("quantity", DataType::Int32, false), - ArrowField::new("price", DataType::Float32, false), - ArrowField::new("description", DataType::Utf8, false), - ])); - - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // Create test data with patterns suitable for different compression - let mut customer_ids = Int32Builder::new(); - let mut product_ids = Int32Builder::new(); - let mut quantities = Int32Builder::new(); - let mut prices = Float32Builder::new(); - let mut descriptions = Vec::new(); - - // Generate data with specific patterns: - // - customer_id: highly repetitive (good for RLE) - // - product_id: moderately repetitive (good for RLE) - // - quantity: random values (not good for RLE) - // - price: some repetition - // - description: long strings (good for Zstd) - for i in 0..10000 { - // Customer ID repeats every 100 rows (100 unique customers) - // This creates runs of 100 identical values - customer_ids.append_value(i / 100); - - // Product ID has only 5 unique values with long runs - product_ids.append_value(i / 2000); - - // Quantity is mostly 1 with occasional other values - quantities.append_value(if i % 10 == 0 { 5 } else { 1 }); - - // Price has only 3 unique values - prices.append_value(match i % 3 { - 0 => 9.99, - 1 => 19.99, - _ => 29.99, - }); - - // Descriptions are repetitive but we'll keep them simple - descriptions.push(format!("Product {}", i / 2000)); - } - - let batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new(customer_ids.finish()), - Arc::new(product_ids.finish()), - Arc::new(quantities.finish()), - Arc::new(prices.finish()), - Arc::new(StringArray::from(descriptions)), - ], - ) - .unwrap(); - - // Configure compression parameters - let mut params = CompressionParams::new(); - - // RLE for ID columns (ends with _id) - params.columns.insert( - "*_id".to_string(), - CompressionFieldParams { - rle_threshold: Some(0.5), // Lower threshold to trigger RLE more easily - compression: None, // Will use default compression if any - compression_level: None, - bss: Some(lance_encoding::compression_config::BssMode::Off), // Explicitly disable BSS to ensure RLE is used - minichunk_size: None, - }, - ); - - // For now, we'll skip Zstd compression since it's not imported - // In a real implementation, you could add other compression types here - - // Build encoding strategy with compression parameters - let encoding_strategy = lance_encoding::encoder::default_encoding_strategy_with_params( - LanceFileVersion::V2_1, - params, - ) - .unwrap(); - - // Configure file writer options - let options = FileWriterOptions { - encoding_strategy: Some(Arc::from(encoding_strategy)), - format_version: Some(LanceFileVersion::V2_1), - max_page_bytes: Some(64 * 1024), // 64KB pages - ..Default::default() - }; - - // Write the file - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.add_schema_metadata("compression_test", "configured_compression"); - writer.finish().await.unwrap(); - - // Now write the same data without compression overrides for comparison - let path_no_compression = TempObjFile::default(); - let default_options = FileWriterOptions { - format_version: Some(LanceFileVersion::V2_1), - max_page_bytes: Some(64 * 1024), - ..Default::default() - }; - - let mut writer_no_compression = FileWriter::try_new( - object_store.create(&path_no_compression).await.unwrap(), - lance_schema.clone(), - default_options, - ) - .unwrap(); - - writer_no_compression.write_batch(&batch).await.unwrap(); - writer_no_compression.finish().await.unwrap(); - - // Note: With our current data patterns and RLE compression, the compressed file - // might actually be slightly larger due to compression metadata overhead. - // This is expected and the test is mainly to verify the system works end-to-end. - - // Read back the compressed file and verify data integrity - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - // Verify metadata - let metadata = file_reader.metadata(); - assert_eq!(metadata.major_version, 2); - assert_eq!(metadata.minor_version, 1); - - let schema = file_reader.schema(); - assert_eq!( - schema.metadata.get("compression_test"), - Some(&"configured_compression".to_string()) - ); - - // Verify the actual encodings used - let column_metadatas = &metadata.column_metadatas; - - // Check customer_id column (index 0) - should use RLE due to our configuration - assert!(!column_metadatas[0].pages.is_empty()); - let customer_id_encoding = describe_encoding(&column_metadatas[0].pages[0]); - assert!( - customer_id_encoding.contains("RLE") || customer_id_encoding.contains("Rle"), - "customer_id column should use RLE encoding due to '*_id' pattern match, but got: {}", - customer_id_encoding - ); - - // Check product_id column (index 1) - should use RLE due to our configuration - assert!(!column_metadatas[1].pages.is_empty()); - let product_id_encoding = describe_encoding(&column_metadatas[1].pages[0]); - assert!( - product_id_encoding.contains("RLE") || product_id_encoding.contains("Rle"), - "product_id column should use RLE encoding due to '*_id' pattern match, but got: {}", - product_id_encoding - ); - } - - #[tokio::test] - async fn test_field_metadata_compression() { - // Test that field metadata compression settings are respected - let mut metadata = HashMap::new(); - metadata.insert( - lance_encoding::constants::COMPRESSION_META_KEY.to_string(), - "zstd".to_string(), - ); - metadata.insert( - lance_encoding::constants::COMPRESSION_LEVEL_META_KEY.to_string(), - "6".to_string(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", DataType::Int32, false), - ArrowField::new("text", DataType::Utf8, false).with_metadata(metadata.clone()), - ArrowField::new("data", DataType::Int32, false).with_metadata(HashMap::from([( - lance_encoding::constants::COMPRESSION_META_KEY.to_string(), - "none".to_string(), - )])), - ])); - - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // Create test data - let id_array = Int32Array::from_iter_values(0..1000); - let text_array = StringArray::from_iter_values( - (0..1000).map(|i| format!("test string {} repeated text", i)), - ); - let data_array = Int32Array::from_iter_values((0..1000).map(|i| i * 2)); - - let batch = RecordBatch::try_new( - arrow_schema.clone(), - vec![ - Arc::new(id_array), - Arc::new(text_array), - Arc::new(data_array), - ], - ) - .unwrap(); - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - - // Create encoding strategy that will read from field metadata - let params = CompressionParams::new(); - let encoding_strategy = lance_encoding::encoder::default_encoding_strategy_with_params( - LanceFileVersion::V2_1, - params, - ) - .unwrap(); - - let options = FileWriterOptions { - encoding_strategy: Some(Arc::from(encoding_strategy)), - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - // Read back metadata - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let column_metadatas = &file_reader.metadata().column_metadatas; - - // The text column (index 1) should use zstd compression based on metadata - let text_encoding = describe_encoding(&column_metadatas[1].pages[0]); - // For string columns, we expect Binary encoding with zstd compression - assert!( - text_encoding.contains("Zstd"), - "text column should use zstd compression from field metadata, but got: {}", - text_encoding - ); - - // The data column (index 2) should use no compression based on metadata - let data_encoding = describe_encoding(&column_metadatas[2].pages[0]); - // For Int32 columns with "none" compression, we expect Flat encoding without compression - assert!( - data_encoding.contains("Flat") && data_encoding.contains("compression: None"), - "data column should use no compression from field metadata, but got: {}", - data_encoding - ); - } - - #[tokio::test] - async fn test_field_metadata_rle_threshold() { - // Test that RLE threshold from field metadata is respected - let mut metadata = HashMap::new(); - metadata.insert( - lance_encoding::constants::RLE_THRESHOLD_META_KEY.to_string(), - "0.9".to_string(), - ); - // Also set compression to ensure RLE is used - metadata.insert( - lance_encoding::constants::COMPRESSION_META_KEY.to_string(), - "lz4".to_string(), - ); - // Explicitly disable BSS to ensure RLE is tested - metadata.insert( - lance_encoding::constants::BSS_META_KEY.to_string(), - "off".to_string(), - ); - - let arrow_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("status", DataType::Int32, false).with_metadata(metadata), - ])); - - let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - - // Create data with very high repetition (3 runs for 10000 values = 0.0003 ratio) - let status_array = Int32Array::from_iter_values( - std::iter::repeat_n(200, 8000) - .chain(std::iter::repeat_n(404, 1500)) - .chain(std::iter::repeat_n(500, 500)), - ); - - let batch = - RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(status_array)]).unwrap(); - - let path = TempObjFile::default(); - let object_store = ObjectStore::local(); - - // Create encoding strategy that will read from field metadata - let params = CompressionParams::new(); - let encoding_strategy = lance_encoding::encoder::default_encoding_strategy_with_params( - LanceFileVersion::V2_1, - params, - ) - .unwrap(); - - let options = FileWriterOptions { - encoding_strategy: Some(Arc::from(encoding_strategy)), - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - let mut writer = FileWriter::try_new( - object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - writer.finish().await.unwrap(); - - // Read back and check encoding - let fs = FsFixture::default(); - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - FileReaderOptions::default(), - ) - .await - .unwrap(); - - let column_metadatas = &file_reader.metadata().column_metadatas; - let status_encoding = describe_encoding(&column_metadatas[0].pages[0]); - assert!( - status_encoding.contains("RLE") || status_encoding.contains("Rle"), - "status column should use RLE encoding due to metadata threshold, but got: {}", - status_encoding - ); - } - - #[tokio::test] - async fn test_large_page_split_on_read() { - use arrow_array::Array; - use futures::TryStreamExt; - use lance_encoding::decoder::FilterExpression; - use lance_io::ReadBatchParams; - - // Test that large pages written with relaxed limits can be split during read - - let arrow_field = ArrowField::new("data", DataType::Binary, false); - let arrow_schema = ArrowSchema::new(vec![arrow_field]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // Create a large binary value (40MB) to trigger large page creation - let large_value = vec![42u8; 40 * 1024 * 1024]; - let array = arrow_array::BinaryArray::from(vec![ - Some(large_value.as_slice()), - Some(b"small value"), - ]); - let batch = RecordBatch::try_new(Arc::new(arrow_schema), vec![Arc::new(array)]).unwrap(); - - // Write with relaxed page size limit (128MB) - let options = FileWriterOptions { - max_page_bytes: Some(128 * 1024 * 1024), - format_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }; - - let fs = FsFixture::default(); - let path = fs.tmp_path; - - let mut writer = FileWriter::try_new( - fs.object_store.create(&path).await.unwrap(), - lance_schema.clone(), - options, - ) - .unwrap(); - - writer.write_batch(&batch).await.unwrap(); - let write_summary = writer.finish().await.unwrap(); - assert_eq!(write_summary.num_rows, 2); - assert_eq!( - write_summary.size_bytes, - fs.object_store.size(&path).await.unwrap() - ); - - // Read back with split configuration - let file_scheduler = fs - .scheduler - .open_file(&path, &CachedFileSize::unknown()) - .await - .unwrap(); - - // Configure reader to split pages larger than 10MB into chunks - let reader_options = FileReaderOptions { - read_chunk_size: 10 * 1024 * 1024, // 10MB chunks - ..Default::default() - }; - - let file_reader = FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &LanceCache::no_cache(), - reader_options, - ) - .await - .unwrap(); - - // Read the data back - let stream = file_reader - .read_stream( - ReadBatchParams::RangeFull, - 1024, - 10, // batch_readahead - FilterExpression::no_filter(), - ) - .await - .unwrap(); - - let batches: Vec = stream.try_collect().await.unwrap(); - assert_eq!(batches.len(), 1); - - // Verify the data is correctly read despite splitting - let read_array = batches[0].column(0); - let read_binary = read_array - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(read_binary.len(), 2); - assert_eq!(read_binary.value(0).len(), 40 * 1024 * 1024); - assert_eq!(read_binary.value(1), b"small value"); - - // Verify first value matches what we wrote - assert!(read_binary.value(0).iter().all(|&b| b == 42u8)); - } - - fn spill_config() -> (TempObjFile, Arc) { - let spill_path = TempObjFile::default(); - (spill_path, Arc::new(ObjectStore::local())) - } - - fn make_batches(num_batches: i32, num_cols: usize, rows_per_batch: i32) -> Vec { - let fields: Vec<_> = (0..num_cols) - .map(|c| ArrowField::new(format!("c{c}"), DataType::Int32, false)) - .collect(); - let schema = Arc::new(ArrowSchema::new(fields)); - (0..num_batches) - .map(|i| { - let cols: Vec> = (0..num_cols) - .map(|c| { - let start = (i * rows_per_batch + c as i32) * 100; - Arc::new(Int32Array::from_iter_values(start..start + rows_per_batch)) - as Arc - }) - .collect(); - RecordBatch::try_new(schema.clone(), cols).unwrap() - }) - .collect() - } - - async fn write_and_read_batches( - batches: &[RecordBatch], - spill: Option<(Arc, object_store::path::Path)>, - ) -> Vec { - let fs = FsFixture::default(); - let lance_schema = LanceSchema::try_from(batches[0].schema().as_ref()).unwrap(); - let writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); - let mut file_writer = - FileWriter::try_new(writer, lance_schema, FileWriterOptions::default()).unwrap(); - if let Some((store, path)) = spill { - file_writer = file_writer.with_page_metadata_spill(store, path); - } - for batch in batches { - file_writer.write_batch(batch).await.unwrap(); - } - file_writer.add_schema_metadata("foo", "bar"); - file_writer.finish().await.unwrap(); - - crate::testing::read_lance_file( - &fs, - Arc::::default(), - lance_encoding::decoder::FilterExpression::no_filter(), - ) - .await - } - - #[rstest::rstest] - #[case::multi_col(20, 2, 100)] - #[case::many_batches(50, 2, 100)] - #[tokio::test] - async fn test_page_metadata_spill_roundtrip( - #[case] num_batches: i32, - #[case] num_cols: usize, - #[case] rows_per_batch: i32, - ) { - let batches = make_batches(num_batches, num_cols, rows_per_batch); - let baseline = write_and_read_batches(&batches, None).await; - let (spill_path, spill_store) = spill_config(); - let spilled = - write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) - .await; - assert_eq!(baseline, spilled); - } - - #[tokio::test] - async fn test_page_metadata_spill_many_columns() { - // Many columns forces small per-column buffer limits, exercising mid-write flushing. - let batches = make_batches(10, 500, 100); - let baseline = write_and_read_batches(&batches, None).await; - let (spill_path, spill_store) = spill_config(); - let spilled = - write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) - .await; - assert_eq!(baseline, spilled); - } -} +#[path = "writer_tests.rs"] +mod writer_tests; diff --git a/rust/lance-file/src/writer/structural.rs b/rust/lance-file/src/writer/structural.rs new file mode 100644 index 00000000000..0a5986c5ef1 --- /dev/null +++ b/rust/lance-file/src/writer/structural.rs @@ -0,0 +1,842 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use core::panic; +use std::{collections::HashMap, sync::Arc}; + +use arrow_array::{ArrayRef, RecordBatch}; +use arrow_data::ArrayData; +use bytes::{Buf, Bytes, BytesMut}; +use futures::{StreamExt, stream::FuturesOrdered}; +use lance_core::{ + Error, Result, + datatypes::{Field, Schema}, + utils::bit::pad_bytes, +}; +use lance_encoding::{ + decoder::PageEncoding, + encoder::{ + BatchEncoder, EncodeTask, EncodedBatch, EncodedPage, EncodingOptions, FieldEncoder, + OutOfLineBuffers, + }, + repdef::RepDefBuilder, +}; +use lance_io::{object_store::ObjectStore, traits::Writer as ObjectWriter}; +use log::{debug, warn}; +use object_store::path::Path; +use prost::Message; +use prost_types::Any; +use tokio::io::{AsyncWrite, AsyncWriteExt}; +use tracing::instrument; + +use crate::{ + datatypes::FieldsWithMeta, + format::{pb, pbfile}, + writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriterOptions, PAGE_BUFFER_ALIGNMENT}, +}; + +const PAD_BUFFER: [u8; PAGE_BUFFER_ALIGNMENT] = [72; PAGE_BUFFER_ALIGNMENT]; + +// V2.1+ splits large pages on read instead of write. This write-time limit is +// retained as a best-effort memory bound and is not a grammar decision. +const MAX_PAGE_BYTES: usize = 32 * 1024 * 1024; + +// Total in-memory budget for serialized page metadata before spill. +const DEFAULT_SPILL_BUFFER_LIMIT: usize = 256 * 1024; + +struct PageMetadataSpill { + writer: Box, + object_store: Arc, + path: Path, + position: u64, + column_buffers: Vec>, + column_chunks: Vec>, + per_column_limit: usize, +} + +impl PageMetadataSpill { + async fn new(object_store: Arc, path: Path, num_columns: usize) -> Result { + let writer = object_store.create(&path).await?; + let per_column_limit = (DEFAULT_SPILL_BUFFER_LIMIT / num_columns.max(1)).max(64); + Ok(Self { + writer, + object_store, + path, + position: 0, + column_buffers: vec![Vec::new(); num_columns], + column_chunks: vec![Vec::new(); num_columns], + per_column_limit, + }) + } + + async fn append_page( + &mut self, + column_index: usize, + page: &pbfile::column_metadata::Page, + ) -> Result<()> { + page.encode_length_delimited(&mut self.column_buffers[column_index]) + .map_err(|error| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error, + ))) + })?; + if self.column_buffers[column_index].len() >= self.per_column_limit { + self.flush_column(column_index).await?; + } + Ok(()) + } + + async fn flush_column(&mut self, column_index: usize) -> Result<()> { + let buffer = &self.column_buffers[column_index]; + if buffer.is_empty() { + return Ok(()); + } + let len = buffer.len(); + self.writer.write_all(buffer).await?; + self.column_chunks[column_index].push((self.position, len as u32)); + self.position += len as u64; + self.column_buffers[column_index].clear(); + Ok(()) + } + + async fn shutdown_writer(&mut self) -> Result<()> { + for column_index in 0..self.column_buffers.len() { + self.flush_column(column_index).await?; + } + ObjectWriter::shutdown(self.writer.as_mut()).await?; + Ok(()) + } +} + +fn decode_spilled_chunk(data: &Bytes) -> Result> { + let mut pages = Vec::new(); + let mut cursor = data.clone(); + while cursor.has_remaining() { + let page = pbfile::column_metadata::Page::decode_length_delimited(&mut cursor).map_err( + |error| { + Error::io_source(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + error, + ))) + }, + )?; + pages.push(page); + } + Ok(pages) +} + +enum PageSpillState { + Pending(Arc, Path), + Active(PageMetadataSpill), +} + +fn initial_column_metadata() -> pbfile::ColumnMetadata { + pbfile::ColumnMetadata { + pages: Vec::new(), + buffer_offsets: Vec::new(), + buffer_sizes: Vec::new(), + encoding: None, + } +} + +/// Writes the structural page and metadata representation used by v2.1+. +/// +/// This component does not choose a file version, field encoding strategy, or +/// footer identity. Exact version writers opt into this representation and own +/// the order in which its operations are invoked. +pub struct StructuralFileSink { + writer: Box, + column_metadata: Vec, + num_columns: u32, + global_buffers: Vec<(u64, u64)>, + page_spill: Option, +} + +impl StructuralFileSink { + pub fn new(writer: Box) -> Self { + Self { + writer, + column_metadata: Vec::new(), + num_columns: 0, + global_buffers: Vec::new(), + page_spill: None, + } + } + + pub fn with_page_metadata_spill(&mut self, object_store: Arc, path: Path) { + self.page_spill = Some(PageSpillState::Pending(object_store, path)); + } + + pub fn initialize_columns(&mut self, num_columns: u32) { + self.num_columns = num_columns; + self.column_metadata = vec![initial_column_metadata(); num_columns as usize]; + } + + pub fn initialize_with_external_metadata( + &mut self, + column_metadata: Vec, + ) { + self.num_columns = column_metadata.len() as u32; + self.column_metadata = column_metadata; + } + + async fn write_aligned_buffer_to( + writer: &mut (impl AsyncWrite + Unpin), + buffer: &[u8], + ) -> Result<()> { + writer.write_all(buffer).await?; + let padding = pad_bytes::(buffer.len()); + writer.write_all(&PAD_BUFFER[..padding]).await?; + Ok(()) + } + + pub async fn write_aligned_buffer(&mut self, buffer: &[u8]) -> Result<()> { + Self::write_aligned_buffer_to(&mut self.writer, buffer).await + } + + pub async fn write_raw(&mut self, buffer: &[u8]) -> Result<()> { + self.writer.write_all(buffer).await?; + Ok(()) + } + + pub async fn write_page(&mut self, encoded_page: EncodedPage) -> Result<()> { + let buffers = encoded_page.data; + let mut buffer_offsets = Vec::with_capacity(buffers.len()); + let mut buffer_sizes = Vec::with_capacity(buffers.len()); + for buffer in buffers { + buffer_offsets.push(self.tell().await?); + buffer_sizes.push(buffer.len() as u64); + self.write_aligned_buffer(&buffer).await?; + } + let encoded_encoding = match encoded_page.description { + PageEncoding::Legacy(array_encoding) => Any::from_msg(&array_encoding)?.encode_to_vec(), + PageEncoding::Structural(page_layout) => Any::from_msg(&page_layout)?.encode_to_vec(), + }; + let page = pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_encoding, + })), + }), + length: encoded_page.num_rows, + priority: encoded_page.row_number, + }; + let column_index = encoded_page.column_idx as usize; + if matches!(&self.page_spill, Some(PageSpillState::Pending(..))) { + let Some(PageSpillState::Pending(store, path)) = self.page_spill.take() else { + unreachable!() + }; + self.page_spill = Some(PageSpillState::Active( + PageMetadataSpill::new(store, path, self.num_columns as usize).await?, + )); + } + match &mut self.page_spill { + Some(PageSpillState::Active(spill)) => spill.append_page(column_index, &page).await?, + None => self.column_metadata[column_index].pages.push(page), + Some(PageSpillState::Pending(..)) => unreachable!(), + } + Ok(()) + } + + #[instrument(skip_all, level = "debug")] + pub async fn write_pages( + &mut self, + mut encoding_tasks: FuturesOrdered, + ) -> Result<()> { + while let Some(encoding_task) = encoding_tasks.next().await { + self.write_page(encoding_task?).await?; + } + // Reaps any upload that has already failed so the error is attributed to + // this batch. This does not wait for in-flight uploads; see + // `ObjectWriter::poll_flush`. + self.writer.flush().await?; + Ok(()) + } + + pub async fn write_column_buffer_at( + &mut self, + column_index: usize, + position: u64, + buffer: &[u8], + ) -> Result<()> { + self.write_aligned_buffer(buffer).await?; + let metadata = &mut self.column_metadata[column_index]; + metadata.buffer_offsets.push(position); + metadata.buffer_sizes.push(buffer.len() as u64); + Ok(()) + } + + pub fn set_column_encoding(&mut self, column_index: usize, encoding: pbfile::Encoding) { + self.column_metadata[column_index].encoding = Some(encoding); + } + + pub async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + let position = self.tell().await?; + let len = buffer.len() as u64; + self.write_aligned_buffer(&buffer).await?; + self.global_buffers.push((position, len)); + Ok(self.global_buffers.len() as u32) + } + + pub async fn write_global_buffers( + &mut self, + descriptor: pb::FileDescriptor, + ) -> Result> { + let descriptor_bytes = descriptor.encode_to_vec(); + let descriptor_len = descriptor_bytes.len() as u64; + let descriptor_position = self.tell().await?; + self.writer.write_all(&descriptor_bytes).await?; + let mut offsets = Vec::with_capacity(1 + self.global_buffers.len()); + offsets.push((descriptor_position, descriptor_len)); + offsets.append(&mut self.global_buffers); + Ok(offsets) + } + + async fn write_column_metadata( + &mut self, + metadata: pbfile::ColumnMetadata, + ) -> Result<(u64, u64)> { + let metadata_bytes = metadata.encode_to_vec(); + let position = self.tell().await?; + let len = metadata_bytes.len() as u64; + self.writer.write_all(&metadata_bytes).await?; + Ok((position, len)) + } + + pub async fn write_column_metadatas(&mut self) -> Result> { + let metadatas = std::mem::take(&mut self.column_metadata); + let spill_state = self.page_spill.take(); + let (spill_chunks, spill_reader) = + if let Some(PageSpillState::Active(mut spill)) = spill_state { + spill.shutdown_writer().await?; + let reader = spill.object_store.open(&spill.path).await?; + let chunks = std::mem::take(&mut spill.column_chunks); + (chunks, Some(reader)) + } else { + (Vec::new(), None) + }; + + let mut metadata_positions = Vec::with_capacity(metadatas.len()); + for (column_index, mut metadata) in metadatas.into_iter().enumerate() { + if let Some(reader) = &spill_reader { + let mut pages = Vec::new(); + for &(offset, len) in &spill_chunks[column_index] { + let data = reader + .get_range(offset as usize..(offset as usize + len as usize)) + .await + .map_err(|error| Error::io_source(Box::new(error)))?; + pages.extend(decode_spilled_chunk(&data)?); + } + metadata.pages = pages; + } + metadata_positions.push(self.write_column_metadata(metadata).await?); + } + Ok(metadata_positions) + } + + pub async fn write_offset_table(&mut self, offsets: &[(u64, u64)]) -> Result { + let start = self.tell().await?; + for (position, len) in offsets { + self.writer.write_u64_le(*position).await?; + self.writer.write_u64_le(*len).await?; + } + Ok(start) + } + + pub async fn write_external_buffer(&mut self, bytes: &[u8]) -> Result<(u64, u64)> { + const ZERO_PADDING: [u8; PAGE_BUFFER_ALIGNMENT] = [0; PAGE_BUFFER_ALIGNMENT]; + let position = self.tell().await?; + let padding = (PAGE_BUFFER_ALIGNMENT - position as usize % PAGE_BUFFER_ALIGNMENT) + % PAGE_BUFFER_ALIGNMENT; + self.writer.write_all(&ZERO_PADDING[..padding]).await?; + let start = position + padding as u64; + self.writer.write_all(bytes).await?; + Ok((start, bytes.len() as u64)) + } + + pub fn output_mut(&mut self) -> &mut dyn ObjectWriter { + self.writer.as_mut() + } + + pub async fn tell(&mut self) -> Result { + Ok(self.writer.tell().await? as u64) + } + + pub fn num_columns(&self) -> u32 { + self.num_columns + } + + pub async fn shutdown(&mut self) -> Result { + let result = ObjectWriter::shutdown(self.writer.as_mut()).await?; + Ok(result.size as u64) + } +} + +/// Runs field encoders and row accounting without selecting a file grammar. +/// +/// The exact version writer constructs a [`BatchEncoder`] and supplies it when +/// a schema is initialized. This component only executes that decision. +pub struct EncodingPipeline { + schema: Option, + field_encoders: Vec>, + field_id_to_column_indices: Vec<(u32, u32)>, + rows_written: u64, + field_rows_written: Vec, + schema_metadata: HashMap, + options: FileWriterOptions, +} + +impl EncodingPipeline { + pub fn new(options: FileWriterOptions) -> Self { + Self { + schema: None, + field_encoders: Vec::new(), + field_id_to_column_indices: Vec::new(), + rows_written: 0, + field_rows_written: Vec::new(), + schema_metadata: HashMap::new(), + options, + } + } + + pub fn encoding_options(&self, schema: &Schema) -> EncodingOptions { + let cache_bytes_per_column = if let Some(data_cache_bytes) = self.options.data_cache_bytes { + data_cache_bytes / schema.fields.len() as u64 + } else { + 8 * 1024 * 1024 + }; + let max_page_bytes = self.options.max_page_bytes.unwrap_or_else(|| { + std::env::var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES) + .map(|value| { + value.parse::().unwrap_or_else(|error| { + warn!( + "Failed to parse {}: {}, using default", + ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, error + ); + MAX_PAGE_BYTES as u64 + }) + }) + .unwrap_or(MAX_PAGE_BYTES as u64) + }); + + EncodingOptions { + cache_bytes_per_column, + max_page_bytes, + keep_original_array: self.options.keep_original_array.unwrap_or(false), + buffer_alignment: PAGE_BUFFER_ALIGNMENT as u64, + } + } + + pub fn initialize( + &mut self, + mut schema: Schema, + encoder: BatchEncoder, + sink: &mut StructuralFileSink, + ) { + sink.initialize_columns(encoder.num_columns()); + self.field_rows_written = vec![0; encoder.field_encoders.len()]; + self.field_encoders = encoder.field_encoders; + self.field_id_to_column_indices = encoder.field_id_to_column_index; + self.schema_metadata + .extend(std::mem::take(&mut schema.metadata)); + self.schema = Some(schema); + } + + pub fn is_initialized(&self) -> bool { + self.schema.is_some() + } + + fn verify_field_nullability(array: &ArrayData, field: &Field) -> Result<()> { + if !field.nullable && array.null_count() > 0 { + return Err(Error::invalid_input(format!( + "The field `{}` contained null values even though the field is marked non-null in the schema", + field.name + ))); + } + for (child_field, child_array) in field.children.iter().zip(array.child_data()) { + Self::verify_field_nullability(child_array, child_field)?; + } + Ok(()) + } + + fn verify_nullability_constraints(&self, batch: &RecordBatch) -> Result<()> { + for (column, field) in batch + .columns() + .iter() + .zip(self.schema.as_ref().unwrap().fields.iter()) + { + Self::verify_field_nullability(&column.to_data(), field)?; + } + Ok(()) + } + + fn encode_columns( + &mut self, + fields: &[(usize, ArrayRef)], + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + let row_numbers = fields + .iter() + .map(|(field_index, _)| self.field_rows_written[*field_index]) + .collect::>(); + fields + .iter() + .zip(row_numbers) + .map(|((field_index, array), row_number)| { + self.field_encoders[*field_index].maybe_encode( + array.clone(), + external_buffers, + RepDefBuilder::default(), + row_number, + array.len() as u64, + ) + }) + .collect() + } + + fn encode_batch( + &mut self, + batch: &RecordBatch, + external_buffers: &mut OutOfLineBuffers, + ) -> Result>> { + let field_arrays = self + .schema + .as_ref() + .unwrap() + .fields + .iter() + .enumerate() + .map(|(field_index, field)| { + let array = batch.column_by_name(&field.name).ok_or_else(|| { + Error::invalid_input_source( + format!( + "Cannot write batch. The batch was missing the column `{}`", + field.name + ) + .into(), + ) + })?; + Ok((field_index, array.clone())) + }) + .collect::>>()?; + self.encode_columns(&field_arrays, external_buffers) + } + + #[instrument(skip_all, level = "debug")] + pub async fn write_batch( + &mut self, + batch: &RecordBatch, + sink: &mut StructuralFileSink, + ) -> Result<()> { + debug!( + "write_batch called with {} rows, {} columns, and {} bytes of data", + batch.num_rows(), + batch.num_columns(), + batch.get_array_memory_size() + ); + self.verify_nullability_constraints(batch)?; + let num_rows = batch.num_rows() as u64; + if num_rows == 0 { + return Ok(()); + } + if num_rows > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_batch(batch, &mut external_buffers)?; + for external_buffer in external_buffers.take_buffers() { + sink.write_aligned_buffer(&external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + + if self.rows_written.checked_add(num_rows).is_none() { + return Err(Error::invalid_input_source(format!( + "cannot write batch with {} rows because {} rows have already been written and Lance files cannot contain more than 2^64 rows", + num_rows, self.rows_written + ).into())); + } + for field_rows in &mut self.field_rows_written { + *field_rows += num_rows; + } + self.rows_written += num_rows; + sink.write_pages(encoding_tasks).await + } + + pub async fn write_column( + &mut self, + column_index: usize, + array: ArrayRef, + sink: &mut StructuralFileSink, + ) -> Result<()> { + let schema = self.schema.as_ref().ok_or_else(|| { + Error::invalid_input_source( + "write_column requires the writer to be created with an explicit schema".into(), + ) + })?; + let field = schema.fields.get(column_index).ok_or_else(|| { + Error::invalid_input_source( + format!( + "write_column: field index {} is out of bounds (schema has {} fields)", + column_index, + schema.fields.len() + ) + .into(), + ) + })?; + if array.len() as u64 > u32::MAX as u64 { + return Err(Error::invalid_input_source( + "cannot write Lance files with more than 2^32 rows".into(), + )); + } + Self::verify_field_nullability(&array.to_data(), field)?; + if array.is_empty() { + return Ok(()); + } + + let fields = [(column_index, array)]; + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self.encode_columns(&fields, &mut external_buffers)?; + for external_buffer in external_buffers.take_buffers() { + sink.write_aligned_buffer(&external_buffer).await?; + } + let encoding_tasks = encoding_tasks + .into_iter() + .flatten() + .collect::>(); + for (field_index, array) in &fields { + let new_total = self.field_rows_written[*field_index] + array.len() as u64; + self.field_rows_written[*field_index] = new_total; + self.rows_written = self.rows_written.max(new_total); + } + sink.write_pages(encoding_tasks).await + } + + pub async fn flush(&mut self, sink: &mut StructuralFileSink) -> Result<()> { + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let encoding_tasks = self + .field_encoders + .iter_mut() + .map(|writer| writer.flush(&mut external_buffers)) + .collect::>>()?; + for external_buffer in external_buffers.take_buffers() { + sink.write_aligned_buffer(&external_buffer).await?; + } + sink.write_pages( + encoding_tasks + .into_iter() + .flatten() + .collect::>(), + ) + .await + } + + pub async fn finish_encoders(&mut self, sink: &mut StructuralFileSink) -> Result<()> { + if self.field_encoders.is_empty() { + return Ok(()); + } + let mut column_index = 0; + for mut writer in std::mem::take(&mut self.field_encoders) { + let mut external_buffers = + OutOfLineBuffers::new(sink.tell().await?, PAGE_BUFFER_ALIGNMENT as u64); + let columns = writer.finish(&mut external_buffers).await?; + for buffer in external_buffers.take_buffers() { + sink.write_raw(&buffer).await?; + } + debug_assert_eq!( + columns.len(), + writer.num_columns() as usize, + "Expected {} columns from column at index {} and got {}", + writer.num_columns(), + column_index, + columns.len() + ); + for column in columns { + for page in column.final_pages { + sink.write_page(page).await?; + } + let mut buffer_position = sink.tell().await?; + for buffer in column.column_buffers { + sink.write_column_buffer_at(column_index, buffer_position, &buffer) + .await?; + buffer_position += buffer.len() as u64; + } + let encoded_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + sink.set_column_encoding( + column_index, + pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct( + pbfile::DirectEncoding { + encoding: encoded_encoding, + }, + )), + }, + ); + column_index += 1; + } + } + if column_index != sink.num_columns() as usize { + panic!( + "Column writers finished with {} columns but we expected {}", + column_index, + sink.num_columns() + ); + } + Ok(()) + } + + pub fn add_schema_metadata(&mut self, key: impl Into, value: impl Into) { + self.schema_metadata.insert(key.into(), value.into()); + } + + pub fn initialize_with_external_metadata(&mut self, mut schema: Schema, rows_written: u64) { + self.schema_metadata + .extend(std::mem::take(&mut schema.metadata)); + self.schema = Some(schema); + self.rows_written = rows_written; + } + + pub fn make_file_descriptor(&mut self) -> Result { + let schema = self.schema.as_mut().ok_or_else(|| { + Error::invalid_input( + "No schema provided on writer open and no data provided. Schema is unknown and file cannot be created", + ) + })?; + schema.metadata = std::mem::take(&mut self.schema_metadata); + schema + .fields + .iter_mut() + .for_each(|field| field.unload_blobs_recursive()); + make_file_descriptor(schema, self.rows_written) + } + + pub fn rows_written(&self) -> u64 { + self.rows_written + } + + pub fn field_id_to_column_indices(&self) -> &[(u32, u32)] { + &self.field_id_to_column_indices + } +} + +fn make_file_descriptor(schema: &Schema, num_rows: u64) -> Result { + let fields_with_meta = FieldsWithMeta::from(schema); + Ok(pb::FileDescriptor { + schema: Some(pb::Schema { + fields: fields_with_meta.fields.0, + metadata: fields_with_meta.metadata, + }), + length: num_rows, + }) +} + +/// Structural file body produced before an exact version appends its footer. +pub struct EncodedBatchBody { + pub data: BytesMut, + pub column_metadata_start: u64, + pub column_metadata_offsets_start: u64, + pub global_buffer_offsets_start: u64, + pub num_global_buffers: u32, + pub num_columns: u32, +} + +pub fn encode_batch_body(batch: &EncodedBatch, write_schema: bool) -> Result { + use bytes::BufMut; + + let mut data = BytesMut::with_capacity(batch.data.len() + 1024 * 1024); + data.extend_from_slice(&batch.data); + let global_buffers = if write_schema { + let schema_start = data.len() as u64; + let schema = Schema::try_from(batch.schema.as_ref())?; + let descriptor = make_file_descriptor(&schema, batch.num_rows)?; + let descriptor_bytes = descriptor.encode_to_vec(); + let descriptor_len = descriptor_bytes.len() as u64; + data.extend_from_slice(&descriptor_bytes); + vec![(schema_start, descriptor_len)] + } else { + Vec::new() + }; + let column_metadata_start = data.len() as u64; + + let mut column_metadata_positions = Vec::with_capacity(batch.page_table.len()); + for column in &batch.page_table { + let position = data.len() as u64; + let pages = column + .page_infos + .iter() + .map(|page_info| { + let encoded_encoding = match &page_info.encoding { + PageEncoding::Legacy(array_encoding) => { + Any::from_msg(array_encoding)?.encode_to_vec() + } + PageEncoding::Structural(page_layout) => { + Any::from_msg(page_layout)?.encode_to_vec() + } + }; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = + page_info.buffer_offsets_and_sizes.iter().copied().unzip(); + Ok(pbfile::column_metadata::Page { + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct( + pbfile::DirectEncoding { + encoding: encoded_encoding, + }, + )), + }), + length: page_info.num_rows, + priority: page_info.priority, + }) + }) + .collect::>>()?; + let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = + column.buffer_offsets_and_sizes.iter().copied().unzip(); + let encoded_column_encoding = Any::from_msg(&column.encoding)?.encode_to_vec(); + let metadata = pbfile::ColumnMetadata { + pages, + buffer_offsets, + buffer_sizes, + encoding: Some(pbfile::Encoding { + location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { + encoding: encoded_column_encoding, + })), + }), + }; + let metadata_bytes = metadata.encode_to_vec(); + column_metadata_positions.push((position, metadata_bytes.len() as u64)); + data.extend_from_slice(&metadata_bytes); + } + + let column_metadata_offsets_start = data.len() as u64; + for (position, len) in column_metadata_positions { + data.put_u64_le(position); + data.put_u64_le(len); + } + let global_buffer_offsets_start = data.len() as u64; + let num_global_buffers = global_buffers.len() as u32; + for (position, len) in global_buffers { + data.put_u64_le(position); + data.put_u64_le(len); + } + + Ok(EncodedBatchBody { + data, + column_metadata_start, + column_metadata_offsets_start, + global_buffer_offsets_start, + num_global_buffers, + num_columns: batch.page_table.len() as u32, + }) +} diff --git a/rust/lance-file/src/writer_tests.rs b/rust/lance-file/src/writer_tests.rs new file mode 100644 index 00000000000..49e40c01185 --- /dev/null +++ b/rust/lance-file/src/writer_tests.rs @@ -0,0 +1,1382 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use crate::reader::{FileReader, FileReaderOptions, ReaderProjection, describe_encoding}; + use crate::testing::FsFixture; + use crate::version::ConcreteFileVersion; + use crate::versions; + use crate::writer::{ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, FileWriter, FileWriterOptions}; + use arrow_array::builder::{Float32Builder, Int32Builder}; + use arrow_array::types::Float64Type; + use arrow_array::{ + Array, ArrayRef, Int32Array, RecordBatch, RecordBatchReader, StringArray, UInt64Array, + }; + use arrow_schema::{DataType, Field, Field as ArrowField, Schema, Schema as ArrowSchema}; + use lance_core::cache::LanceCache; + use lance_core::datatypes::Schema as LanceSchema; + use lance_core::utils::tempfile::TempObjFile; + use lance_datagen::{BatchCount, RowCount, array, gen_batch}; + use lance_encoding::compression_config::{CompressionFieldParams, CompressionParams}; + use lance_encoding::decoder::DecoderPlugins; + use lance_io::object_store::ObjectStore; + use lance_io::traits::Writer; + use lance_io::utils::CachedFileSize; + use rstest::rstest; + + fn create_writer( + object_writer: Box, + schema: LanceSchema, + version: ConcreteFileVersion, + options: FileWriterOptions, + ) -> lance_core::Result { + versions::create_writer(version, object_writer, schema, options) + } + + fn create_v2_1_writer_with_compression( + object_writer: Box, + schema: LanceSchema, + options: FileWriterOptions, + compression: CompressionParams, + ) -> lance_core::Result { + versions::v2_1::create_writer_with_compression(object_writer, schema, options, compression) + .map(Into::into) + } + + fn reader_projection_from_column_names( + version: ConcreteFileVersion, + schema: &LanceSchema, + column_names: &[&str], + ) -> lance_core::Result { + versions::reader_projection_from_column_names(version, schema, column_names) + } + + #[tokio::test] + async fn test_basic_write() { + let tmp_path = TempObjFile::default(); + let obj_store = Arc::new(ObjectStore::local()); + + let reader = gen_batch() + .col("score", array::rand::()) + .into_reader_rows(RowCount::from(1000), BatchCount::from(10)); + + let writer = obj_store.create(&tmp_path).await.unwrap(); + + let lance_schema = + lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); + + let mut file_writer = create_writer( + writer, + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + + for batch in reader { + file_writer.write_batch(&batch.unwrap()).await.unwrap(); + } + file_writer.add_schema_metadata("foo", "bar"); + file_writer.finish().await.unwrap(); + // Tests asserting the contents of the written file are in reader.rs + } + + #[tokio::test] + async fn test_write_empty() { + let tmp_path = TempObjFile::default(); + let obj_store = Arc::new(ObjectStore::local()); + + let reader = gen_batch() + .col("score", array::rand::()) + .into_reader_rows(RowCount::from(0), BatchCount::from(0)); + + let writer = obj_store.create(&tmp_path).await.unwrap(); + + let lance_schema = + lance_core::datatypes::Schema::try_from(reader.schema().as_ref()).unwrap(); + + let mut file_writer = create_writer( + writer, + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + + for batch in reader { + file_writer.write_batch(&batch.unwrap()).await.unwrap(); + } + file_writer.add_schema_metadata("foo", "bar"); + file_writer.finish().await.unwrap(); + } + + // Read a single column back at an explicit range/index set, returning its + // `Int32` values. Reading one column (or an equal-length group) at a time is + // how unequal-length files are consumed: a full scan across columns of + // differing lengths cannot form a single rectangular batch. + async fn read_int32_column( + reader: &FileReader, + schema: &LanceSchema, + version: ConcreteFileVersion, + name: &str, + params: lance_io::ReadBatchParams, + ) -> Vec> { + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + + let projection = reader_projection_from_column_names(version, schema, &[name]).unwrap(); + let batches: Vec = reader + .read_stream_projected(params, 1024, 16, projection, FilterExpression::no_filter()) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + batches + .iter() + .flat_map(|b| { + b.column(0) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .collect::>() + }) + .collect() + } + + /// A single file may hold columns of differing item counts, written by + /// advancing each column's row counter independently (no shared global + /// counter). + #[rstest] + #[tokio::test] + async fn test_write_columns_unequal_lengths( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use lance_io::ReadBatchParams; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + + // Field "a" gets 5 values across two calls (appending), field "b" gets a + // single value, and field "c" is never written (a zero-length column). + let a1: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let b: ArrayRef = Arc::new(Int32Array::from(vec![10])); + writer.write_column(0, a1).await.unwrap(); + writer.write_column(1, b).await.unwrap(); + let a2: ArrayRef = Arc::new(Int32Array::from(vec![4, 5])); + writer.write_column(0, a2).await.unwrap(); + // An empty array is a no-op whether or not the field already has rows: + // field "a" keeps its 5 rows, field "c" stays a zero-length column. + let empty: ArrayRef = Arc::new(Int32Array::from(Vec::::new())); + writer.write_column(0, empty.clone()).await.unwrap(); + writer.write_column(2, empty).await.unwrap(); + + let summary = writer.finish().await.unwrap(); + // The file's logical length is the longest column. + assert_eq!(summary.num_rows, 5); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + // Per-column row counts are recorded in / derivable from file metadata. + assert_eq!(reader.num_rows(), 5); + assert_eq!(reader.column_num_rows(0).unwrap(), 5); + assert_eq!(reader.column_num_rows(1).unwrap(), 1); + assert_eq!(reader.column_num_rows(2).unwrap(), 0); + assert!(reader.column_num_rows(3).is_err()); + + // Each column reads back independently at its own length. + assert_eq!( + read_int32_column( + &reader, + &lance_schema, + version, + "a", + ReadBatchParams::Range(0..5) + ) + .await, + vec![Some(1), Some(2), Some(3), Some(4), Some(5)], + ); + assert_eq!( + read_int32_column( + &reader, + &lance_schema, + version, + "b", + ReadBatchParams::Range(0..1) + ) + .await, + vec![Some(10)], + ); + + // Random access by position within the longer column returns the right + // value even though other columns are shorter. (The take path requires + // strictly increasing indices.) + assert_eq!( + read_int32_column( + &reader, + &lance_schema, + version, + "a", + ReadBatchParams::Indices(arrow_array::UInt32Array::from(vec![0, 2, 4])), + ) + .await, + vec![Some(1), Some(3), Some(5)], + ); + } + + /// Reading an unequal-length file: + /// - a projection whose columns are equal length full-scans normally; + /// - a full scan across columns of differing length is rejected up front, + /// before any batch is produced (even though a prefix would be rectangular); + /// - a bounded read is valid as long as every projected column covers it; + /// - a single-column `RangeFull` resolves to that column's own length, not + /// the file's (maximum) length. + #[rstest] + #[tokio::test] + async fn test_read_unequal_length_projection( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + // "a" and "b" are equal length (5); "c" is shorter (1). + writer + .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) + .await + .unwrap(); + writer + .write_column(1, Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))) + .await + .unwrap(); + writer + .write_column(2, Arc::new(Int32Array::from(vec![100]))) + .await + .unwrap(); + writer.finish().await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let read = |names: &'static [&'static str], params: ReadBatchParams| { + let projection = + reader_projection_from_column_names(version, &lance_schema, names).unwrap(); + async { + match reader + .read_stream_projected( + params, + 1024, + 16, + projection, + FilterExpression::no_filter(), + ) + .await + { + Ok(stream) => stream.try_collect::>().await, + Err(e) => Err(e), + } + } + }; + let col_values = |batches: &[RecordBatch], idx: usize| -> Vec> { + batches + .iter() + .flat_map(|b| { + b.column(idx) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .collect::>() + }) + .collect() + }; + + // Equal-length projection [a, b] full-scans into rectangular batches. + let batches = read(&["a", "b"], ReadBatchParams::RangeFull).await.unwrap(); + assert_eq!( + col_values(&batches, 0), + vec![Some(1), Some(2), Some(3), Some(4), Some(5)] + ); + assert_eq!( + col_values(&batches, 1), + vec![Some(6), Some(7), Some(8), Some(9), Some(10)] + ); + + // A mismatched-length projection [a, c] (5 vs 1) is rejected before any + // batch is yielded, regardless of the read params — its columns cannot + // be combined into rectangular batches. The error names each column's + // length so the caller can see which column is the odd one out. + let err = read(&["a", "c"], ReadBatchParams::RangeFull) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("a=5") && err.contains("c=1"), + "error should name each column's length, got: {err}" + ); + assert!( + read(&["a", "c"], ReadBatchParams::Range(0..1)) + .await + .is_err(), + "even a common-prefix read of unequal-length columns must error" + ); + + // A single-column RangeFull resolves to that column's own length. + let batches = read(&["c"], ReadBatchParams::RangeFull).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(100)]); + let batches = read(&["a"], ReadBatchParams::RangeFull).await.unwrap(); + assert_eq!( + col_values(&batches, 0), + vec![Some(1), Some(2), Some(3), Some(4), Some(5)] + ); + + // RangeFrom/RangeTo likewise resolve against the projected column's own + // length rather than the file's longest column. + let batches = read(&["a"], ReadBatchParams::RangeFrom(2..)).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(3), Some(4), Some(5)]); + // RangeFrom on the short column "c" resolves to length 1, not 5. + let batches = read(&["c"], ReadBatchParams::RangeFrom(0..)).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(100)]); + let batches = read(&["a"], ReadBatchParams::RangeTo(..3)).await.unwrap(); + assert_eq!(col_values(&batches, 0), vec![Some(1), Some(2), Some(3)]); + // A bound past the projected column's length errors. + assert!( + read(&["a"], ReadBatchParams::RangeTo(..6)).await.is_err(), + "RangeTo past the column length must error" + ); + assert!( + read(&["c"], ReadBatchParams::RangeFrom(2..)).await.is_err(), + "RangeFrom past the column length must error" + ); + } + + /// A struct and a list column each map to multiple physical columns, and a + /// list's item column is longer than its top-level row count. The + /// projection-length check must partition `column_indices` by top-level + /// field and use each field's root column, so an ordinary (rectangular) file + /// with nested columns still reads under the new validation path. + #[rstest] + #[tokio::test] + async fn test_read_nested_columns_under_validation( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use arrow_array::types::Int32Type; + use arrow_array::{ListArray, StructArray}; + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + let struct_type = DataType::Struct( + vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ] + .into(), + ); + let list_type = DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("s", struct_type, true), + ArrowField::new("lst", list_type, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + let a: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let s: ArrayRef = Arc::new(StructArray::from(vec![ + ( + Arc::new(ArrowField::new("x", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![10, 20, 30])) as ArrayRef, + ), + ( + Arc::new(ArrowField::new("y", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![11, 21, 31])) as ArrayRef, + ), + ])); + // 3 lists, 6 items: the item column is longer than the top-level rows. + let lst: ArrayRef = Arc::new(ListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2)]), + Some(vec![Some(3)]), + Some(vec![Some(4), Some(5), Some(6)]), + ])); + let batch = RecordBatch::try_new(arrow_schema.clone(), vec![a, s, lst]).unwrap(); + + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + // If `validate_field_length` mispartitioned the physical columns, the + // length check would read the wrong root column (e.g. the list's item + // column, length 6) and spuriously reject this rectangular file. + for names in [&["a", "s", "lst"][..], &["a", "lst"][..], &["a", "s"][..]] { + let projection = + reader_projection_from_column_names(version, &lance_schema, names).unwrap(); + let batches: Vec = reader + .read_stream_projected( + ReadBatchParams::RangeFull, + 1024, + 16, + projection, + FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + total_rows, 3, + "projection {names:?} should read 3 top-level rows" + ); + } + } + + /// `write_column` rejects invalid inputs at the API boundary with + /// descriptive errors: a writer without an explicit schema, an + /// out-of-bounds field index, and a null written into a non-nullable field. + #[tokio::test] + async fn test_write_column_validation_errors() { + // A lazy-schema writer cannot infer the schema from a single column. + let fs = FsFixture::default(); + let mut lazy_writer = versions::v2_1::create_lazy_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + FileWriterOptions::default(), + ); + let err = lazy_writer + .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3]))) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("explicit schema"), + "expected explicit-schema error, got: {err}" + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // An out-of-bounds field index is rejected, naming the index and count. + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + let err = writer + .write_column(5, Arc::new(Int32Array::from(vec![1]))) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains('5') && err.contains('2'), + "expected out-of-bounds error naming index 5 and 2 fields, got: {err}" + ); + + // A null in a non-nullable field ("a") is rejected. + let err = writer + .write_column(0, Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]))) + .await + .unwrap_err() + .to_string(); + assert!( + err.contains("non-null"), + "expected nullability error, got: {err}" + ); + } + + /// The blocking read path applies the same projection-length validation as + /// the async path: a short single column resolves to its own length, and a + /// mismatched-length projection errors up front. + #[rstest] + #[tokio::test] + async fn test_blocking_read_unequal_length( + #[values(ConcreteFileVersion::V2_0, ConcreteFileVersion::V2_1)] + version: ConcreteFileVersion, + ) { + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema.clone(), + version, + FileWriterOptions::default(), + ) + .unwrap(); + writer + .write_column(0, Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))) + .await + .unwrap(); + writer + .write_column(1, Arc::new(Int32Array::from(vec![100]))) + .await + .unwrap(); + writer.finish().await.unwrap(); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = Arc::new( + FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(), + ); + + // Single short column: RangeFull resolves to its own length (1). + let proj_c = reader_projection_from_column_names(version, &lance_schema, &["c"]).unwrap(); + let reader_c = reader.clone(); + let batches = tokio::task::spawn_blocking(move || { + reader_c + .read_stream_projected_blocking( + ReadBatchParams::RangeFull, + 1024, + Some(proj_c), + FilterExpression::no_filter(), + ) + .unwrap() + .collect::, _>>() + .unwrap() + }) + .await + .unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 1); + + // A mismatched projection [a, c] errors on the blocking path too. + let proj_ac = + reader_projection_from_column_names(version, &lance_schema, &["a", "c"]).unwrap(); + let reader_ac = reader.clone(); + let is_err = tokio::task::spawn_blocking(move || { + reader_ac + .read_stream_projected_blocking( + ReadBatchParams::RangeFull, + 1024, + Some(proj_ac), + FilterExpression::no_filter(), + ) + .is_err() + }) + .await + .unwrap(); + assert!( + is_err, + "blocking full scan across unequal-length columns must error" + ); + } + + /// Files written the ordinary (rectangular) way keep equal column lengths, + /// so the unequal-length support is backwards compatible. + #[tokio::test] + async fn test_write_batch_keeps_equal_lengths() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ])); + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + let fs = FsFixture::default(); + let mut writer = create_writer( + fs.object_store.create(&fs.tmp_path).await.unwrap(), + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int32Array::from(vec![4, 5, 6])), + ], + ) + .unwrap(); + writer.write_batch(&batch).await.unwrap(); + let summary = writer.finish().await.unwrap(); + assert_eq!(summary.num_rows, 3); + + let file_scheduler = fs + .scheduler + .open_file(&fs.tmp_path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(reader.column_num_rows(0).unwrap(), 3); + assert_eq!(reader.column_num_rows(1).unwrap(), 3); + } + + #[tokio::test] + async fn test_max_page_bytes_enforced() { + let arrow_field = Field::new("data", DataType::UInt64, false); + let arrow_schema = Schema::new(vec![arrow_field]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // 8MiB + let data: Vec = (0..1_000_000).collect(); + let array = UInt64Array::from(data); + let batch = + RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); + + let options = FileWriterOptions { + max_page_bytes: Some(1024 * 1024), // 1MB + ..Default::default() + }; + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = create_writer( + object_store.create(&path).await.unwrap(), + lance_schema, + ConcreteFileVersion::V2_0, + options, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let column_meta = file_reader.metadata(); + + let mut total_page_num: u32 = 0; + for (col_idx, col_metadata) in column_meta.column_metadatas.iter().enumerate() { + assert!( + !col_metadata.pages.is_empty(), + "Column {} has no pages", + col_idx + ); + + for (page_idx, page) in col_metadata.pages.iter().enumerate() { + total_page_num += 1; + let total_size: u64 = page.buffer_sizes.iter().sum(); + assert!( + total_size <= 1024 * 1024, + "Column {} Page {} size {} exceeds 1MB limit", + col_idx, + page_idx, + total_size + ); + } + } + + assert_eq!(total_page_num, 8) + } + + #[tokio::test(flavor = "current_thread")] + async fn test_max_page_bytes_env_var() { + let arrow_field = Field::new("data", DataType::UInt64, false); + let arrow_schema = Schema::new(vec![arrow_field]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + // 4MiB + let data: Vec = (0..500_000).collect(); + let array = UInt64Array::from(data); + let batch = + RecordBatch::try_new(arrow_schema.clone().into(), vec![Arc::new(array)]).unwrap(); + + // 2MiB + unsafe { + std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, "2097152"); + } + + let options = FileWriterOptions { + max_page_bytes: None, // enforce env + ..Default::default() + }; + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = create_writer( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + options, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + for col_metadata in file_reader.metadata().column_metadatas.iter() { + for page in col_metadata.pages.iter() { + let total_size: u64 = page.buffer_sizes.iter().sum(); + assert!( + total_size <= 2 * 1024 * 1024, + "Page size {} exceeds 2MB limit", + total_size + ); + } + } + + unsafe { + std::env::set_var(ENV_LANCE_FILE_WRITER_MAX_PAGE_BYTES, ""); + } + } + + #[tokio::test] + async fn test_compression_overrides_end_to_end() { + // Create test schema with different column types + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("customer_id", DataType::Int32, false), + ArrowField::new("product_id", DataType::Int32, false), + ArrowField::new("quantity", DataType::Int32, false), + ArrowField::new("price", DataType::Float32, false), + ArrowField::new("description", DataType::Utf8, false), + ])); + + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // Create test data with patterns suitable for different compression + let mut customer_ids = Int32Builder::new(); + let mut product_ids = Int32Builder::new(); + let mut quantities = Int32Builder::new(); + let mut prices = Float32Builder::new(); + let mut descriptions = Vec::new(); + + // Generate data with specific patterns: + // - customer_id: highly repetitive (good for RLE) + // - product_id: moderately repetitive (good for RLE) + // - quantity: random values (not good for RLE) + // - price: some repetition + // - description: long strings (good for Zstd) + for i in 0..10000 { + // Customer ID repeats every 100 rows (100 unique customers) + // This creates runs of 100 identical values + customer_ids.append_value(i / 100); + + // Product ID has only 5 unique values with long runs + product_ids.append_value(i / 2000); + + // Quantity is mostly 1 with occasional other values + quantities.append_value(if i % 10 == 0 { 5 } else { 1 }); + + // Price has only 3 unique values + prices.append_value(match i % 3 { + 0 => 9.99, + 1 => 19.99, + _ => 29.99, + }); + + // Descriptions are repetitive but we'll keep them simple + descriptions.push(format!("Product {}", i / 2000)); + } + + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(customer_ids.finish()), + Arc::new(product_ids.finish()), + Arc::new(quantities.finish()), + Arc::new(prices.finish()), + Arc::new(StringArray::from(descriptions)), + ], + ) + .unwrap(); + + // Configure compression parameters + let mut params = CompressionParams::new(); + + // RLE for ID columns (ends with _id) + params.columns.insert( + "*_id".to_string(), + CompressionFieldParams { + rle_threshold: Some(0.5), // Lower threshold to trigger RLE more easily + compression: None, // Will use default compression if any + compression_level: None, + bss: Some(lance_encoding::compression_config::BssMode::Off), // Explicitly disable BSS to ensure RLE is used + minichunk_size: None, + }, + ); + + // For now, we'll skip Zstd compression since it's not imported + // In a real implementation, you could add other compression types here + + // Configure file writer options + let options = FileWriterOptions { + max_page_bytes: Some(64 * 1024), // 64KB pages + ..Default::default() + }; + + // Write the file + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + + let mut writer = create_v2_1_writer_with_compression( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + options, + params, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.add_schema_metadata("compression_test", "configured_compression"); + writer.finish().await.unwrap(); + + // Now write the same data without compression overrides for comparison + let path_no_compression = TempObjFile::default(); + let default_options = FileWriterOptions { + max_page_bytes: Some(64 * 1024), + ..Default::default() + }; + + let mut writer_no_compression = create_writer( + object_store.create(&path_no_compression).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + default_options, + ) + .unwrap(); + + writer_no_compression.write_batch(&batch).await.unwrap(); + writer_no_compression.finish().await.unwrap(); + + // Note: With our current data patterns and RLE compression, the compressed file + // might actually be slightly larger due to compression metadata overhead. + // This is expected and the test is mainly to verify the system works end-to-end. + + // Read back the compressed file and verify data integrity + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + // Verify metadata + let metadata = file_reader.metadata(); + assert_eq!(metadata.version(), ConcreteFileVersion::V2_1); + + let schema = file_reader.schema(); + assert_eq!( + schema.metadata.get("compression_test"), + Some(&"configured_compression".to_string()) + ); + + // Verify the actual encodings used + let column_metadatas = &metadata.column_metadatas; + + // Check customer_id column (index 0) - should use RLE due to our configuration + assert!(!column_metadatas[0].pages.is_empty()); + let customer_id_encoding = describe_encoding(&column_metadatas[0].pages[0]); + assert!( + customer_id_encoding.contains("RLE") || customer_id_encoding.contains("Rle"), + "customer_id column should use RLE encoding due to '*_id' pattern match, but got: {}", + customer_id_encoding + ); + + // Check product_id column (index 1) - should use RLE due to our configuration + assert!(!column_metadatas[1].pages.is_empty()); + let product_id_encoding = describe_encoding(&column_metadatas[1].pages[0]); + assert!( + product_id_encoding.contains("RLE") || product_id_encoding.contains("Rle"), + "product_id column should use RLE encoding due to '*_id' pattern match, but got: {}", + product_id_encoding + ); + } + + #[tokio::test] + async fn test_field_metadata_compression() { + // Test that field metadata compression settings are respected + let mut metadata = HashMap::new(); + metadata.insert( + lance_encoding::constants::COMPRESSION_META_KEY.to_string(), + "zstd".to_string(), + ); + metadata.insert( + lance_encoding::constants::COMPRESSION_LEVEL_META_KEY.to_string(), + "6".to_string(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("text", DataType::Utf8, false).with_metadata(metadata.clone()), + ArrowField::new("data", DataType::Int32, false).with_metadata(HashMap::from([( + lance_encoding::constants::COMPRESSION_META_KEY.to_string(), + "none".to_string(), + )])), + ])); + + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // Create test data + let id_array = Int32Array::from_iter_values(0..1000); + let text_array = StringArray::from_iter_values( + (0..1000).map(|i| format!("test string {} repeated text", i)), + ); + let data_array = Int32Array::from_iter_values((0..1000).map(|i| i * 2)); + + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(id_array), + Arc::new(text_array), + Arc::new(data_array), + ], + ) + .unwrap(); + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + + // Create encoding strategy that will read from field metadata + let params = CompressionParams::new(); + let options = FileWriterOptions::default(); + let mut writer = create_v2_1_writer_with_compression( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + options, + params, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + // Read back metadata + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let column_metadatas = &file_reader.metadata().column_metadatas; + + // The text column (index 1) should use zstd compression based on metadata + let text_encoding = describe_encoding(&column_metadatas[1].pages[0]); + // For string columns, we expect Binary encoding with zstd compression + assert!( + text_encoding.contains("Zstd"), + "text column should use zstd compression from field metadata, but got: {}", + text_encoding + ); + + // The data column (index 2) should use no compression based on metadata + let data_encoding = describe_encoding(&column_metadatas[2].pages[0]); + // For Int32 columns with "none" compression, we expect Flat encoding without compression + assert!( + data_encoding.contains("Flat") && data_encoding.contains("compression: None"), + "data column should use no compression from field metadata, but got: {}", + data_encoding + ); + } + + #[tokio::test] + async fn test_field_metadata_rle_threshold() { + // Test that RLE threshold from field metadata is respected + let mut metadata = HashMap::new(); + metadata.insert( + lance_encoding::constants::RLE_THRESHOLD_META_KEY.to_string(), + "0.9".to_string(), + ); + // Also set compression to ensure RLE is used + metadata.insert( + lance_encoding::constants::COMPRESSION_META_KEY.to_string(), + "lz4".to_string(), + ); + // Explicitly disable BSS to ensure RLE is tested + metadata.insert( + lance_encoding::constants::BSS_META_KEY.to_string(), + "off".to_string(), + ); + + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("status", DataType::Int32, false).with_metadata(metadata), + ])); + + let lance_schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + + // Create data with very high repetition (3 runs for 10000 values = 0.0003 ratio) + let status_array = Int32Array::from_iter_values( + std::iter::repeat_n(200, 8000) + .chain(std::iter::repeat_n(404, 1500)) + .chain(std::iter::repeat_n(500, 500)), + ); + + let batch = + RecordBatch::try_new(arrow_schema.clone(), vec![Arc::new(status_array)]).unwrap(); + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + + // Create encoding strategy that will read from field metadata + let params = CompressionParams::new(); + let options = FileWriterOptions::default(); + let mut writer = create_v2_1_writer_with_compression( + object_store.create(&path).await.unwrap(), + lance_schema.clone(), + options, + params, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + writer.finish().await.unwrap(); + + // Read back and check encoding + let fs = FsFixture::default(); + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let column_metadatas = &file_reader.metadata().column_metadatas; + let status_encoding = describe_encoding(&column_metadatas[0].pages[0]); + assert!( + status_encoding.contains("RLE") || status_encoding.contains("Rle"), + "status column should use RLE encoding due to metadata threshold, but got: {}", + status_encoding + ); + } + + #[tokio::test] + async fn test_large_page_split_on_read() { + use arrow_array::Array; + use futures::TryStreamExt; + use lance_encoding::decoder::FilterExpression; + use lance_io::ReadBatchParams; + + // Test that large pages written with relaxed limits can be split during read + + let arrow_field = ArrowField::new("data", DataType::Binary, false); + let arrow_schema = ArrowSchema::new(vec![arrow_field]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // Create a large binary value (40MB) to trigger large page creation + let large_value = vec![42u8; 40 * 1024 * 1024]; + let array = arrow_array::BinaryArray::from(vec![ + Some(large_value.as_slice()), + Some(b"small value"), + ]); + let batch = RecordBatch::try_new(Arc::new(arrow_schema), vec![Arc::new(array)]).unwrap(); + + // Write with relaxed page size limit (128MB) + let options = FileWriterOptions { + max_page_bytes: Some(128 * 1024 * 1024), + ..Default::default() + }; + + let fs = FsFixture::default(); + let path = fs.tmp_path; + + let mut writer = create_writer( + fs.object_store.create(&path).await.unwrap(), + lance_schema.clone(), + ConcreteFileVersion::V2_1, + options, + ) + .unwrap(); + + writer.write_batch(&batch).await.unwrap(); + let write_summary = writer.finish().await.unwrap(); + assert_eq!(write_summary.num_rows, 2); + assert_eq!( + write_summary.size_bytes, + fs.object_store.size(&path).await.unwrap() + ); + + // Read back with split configuration + let file_scheduler = fs + .scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + + // Configure reader to split pages larger than 10MB into chunks + let reader_options = FileReaderOptions { + read_chunk_size: 10 * 1024 * 1024, // 10MB chunks + ..Default::default() + }; + + let file_reader = FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &LanceCache::no_cache(), + reader_options, + ) + .await + .unwrap(); + + // Read the data back + let stream = file_reader + .read_stream( + ReadBatchParams::RangeFull, + 1024, + 10, // batch_readahead + FilterExpression::no_filter(), + ) + .await + .unwrap(); + + let batches: Vec = stream.try_collect().await.unwrap(); + assert_eq!(batches.len(), 1); + + // Verify the data is correctly read despite splitting + let read_array = batches[0].column(0); + let read_binary = read_array + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(read_binary.len(), 2); + assert_eq!(read_binary.value(0).len(), 40 * 1024 * 1024); + assert_eq!(read_binary.value(1), b"small value"); + + // Verify first value matches what we wrote + assert!(read_binary.value(0).iter().all(|&b| b == 42u8)); + } + + fn spill_config() -> (TempObjFile, Arc) { + let spill_path = TempObjFile::default(); + (spill_path, Arc::new(ObjectStore::local())) + } + + fn make_batches(num_batches: i32, num_cols: usize, rows_per_batch: i32) -> Vec { + let fields: Vec<_> = (0..num_cols) + .map(|c| ArrowField::new(format!("c{c}"), DataType::Int32, false)) + .collect(); + let schema = Arc::new(ArrowSchema::new(fields)); + (0..num_batches) + .map(|i| { + let cols: Vec> = (0..num_cols) + .map(|c| { + let start = (i * rows_per_batch + c as i32) * 100; + Arc::new(Int32Array::from_iter_values(start..start + rows_per_batch)) + as Arc + }) + .collect(); + RecordBatch::try_new(schema.clone(), cols).unwrap() + }) + .collect() + } + + async fn write_and_read_batches( + batches: &[RecordBatch], + spill: Option<(Arc, object_store::path::Path)>, + ) -> Vec { + let fs = FsFixture::default(); + let lance_schema = LanceSchema::try_from(batches[0].schema().as_ref()).unwrap(); + let writer = fs.object_store.create(&fs.tmp_path).await.unwrap(); + let mut file_writer = create_writer( + writer, + lance_schema, + ConcreteFileVersion::V2_1, + FileWriterOptions::default(), + ) + .unwrap(); + if let Some((store, path)) = spill { + file_writer = file_writer.with_page_metadata_spill(store, path); + } + for batch in batches { + file_writer.write_batch(batch).await.unwrap(); + } + file_writer.add_schema_metadata("foo", "bar"); + file_writer.finish().await.unwrap(); + + crate::testing::read_lance_file( + &fs, + Arc::::default(), + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + } + + #[rstest::rstest] + #[case::multi_col(20, 2, 100)] + #[case::many_batches(50, 2, 100)] + #[tokio::test] + async fn test_page_metadata_spill_roundtrip( + #[case] num_batches: i32, + #[case] num_cols: usize, + #[case] rows_per_batch: i32, + ) { + let batches = make_batches(num_batches, num_cols, rows_per_batch); + let baseline = write_and_read_batches(&batches, None).await; + let (spill_path, spill_store) = spill_config(); + let spilled = + write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) + .await; + assert_eq!(baseline, spilled); + } + + #[tokio::test] + async fn test_page_metadata_spill_many_columns() { + // Many columns forces small per-column buffer limits, exercising mid-write flushing. + let batches = make_batches(10, 500, 100); + let baseline = write_and_read_batches(&batches, None).await; + let (spill_path, spill_store) = spill_config(); + let spilled = + write_and_read_batches(&batches, Some((spill_store, spill_path.as_ref().clone()))) + .await; + assert_eq!(baseline, spilled); + } +} diff --git a/rust/lance-file/test_data/exact_versions/README.md b/rust/lance-file/test_data/exact_versions/README.md new file mode 100644 index 00000000000..72b8edcd86d --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/README.md @@ -0,0 +1,41 @@ +# Exact File-Version Compatibility Fixtures + +These files were generated with the writers at baseline commit +`3a72f8a61e14613f517dded6816d4bfc77817c93`. The deterministic input batch is +defined by `compatibility_fixture_batch` in `src/compatibility_tests.rs` and +covers primitive, nullable UTF-8, nullable list, nullable dictionary, blob, +multiple input batches, and multiple pages. The V2.0 embedded fixtures use the +primitive and nullable UTF-8 columns from the first 257 rows of the same batch. + +`datagen.py` copies the baseline-compatible `datagen.rs` into a clean checkout +of that commit, runs it twice in separate processes with an isolated Cargo +target directory, and verifies that both runs produce identical bytes: + +```shell +git worktree add --detach /tmp/lance-exact-version-baseline \ + 3a72f8a61e14613f517dded6816d4bfc77817c93 +python3 rust/lance-file/test_data/exact_versions/datagen.py \ + --source /tmp/lance-exact-version-baseline +git worktree remove /tmp/lance-exact-version-baseline +``` + +Pass `--write` only when intentionally restoring these files from the locked +baseline. + +| File | SHA-256 | +| --- | --- | +| `v1.lance` | `fa8b3d81b9d4fd4ade5a7c3d077ebf2155664e12b9335e26fac1c0d0774e916c` | +| `v2_0.lance` | `073c8c24eb4433b83d0dda95bf7a731a9f5d8f32d78440f2f391474e99b9c49a` | +| `v2_1.lance` | `3af97ba176b72c7e00a248b4a270a53402a72e594631950f76eb3daab45c50ce` | +| `v2_2.lance` | `8298cd9301e657417b0725461345c27cf46515529d2a8b35824be139e3466a14` | +| `v2_0_self_described.lance` | `6a3a9ce8ef56f058d1d105e7f4494ce35a9026479e2f76fc6f26c04b3201a406` | +| `v2_0_mini.lance` | `5e3fc99b01a4d2f5d16a2fb051dacb49b4a736428b1494715cd83633ad142a63` | + +The compatibility tests require each stable writer to reproduce its fixture +byte-for-byte and each reader to open and read the baseline file. The V2.0 +standard fixture preserves footer `(0, 3)` while its self-described and +mini-Lance fixtures preserve `(2, 0)`. V2.3 is unstable, so it has deterministic +current-revision tests instead of a checked-in compatibility fixture. + +Regenerate these fixtures only from the baseline writer APIs. Files generated +with the implementation under test are not independent compatibility evidence. diff --git a/rust/lance-file/test_data/exact_versions/datagen.py b/rust/lance-file/test_data/exact_versions/datagen.py new file mode 100644 index 00000000000..5543d9fd190 --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/datagen.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import argparse +import hashlib +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +BASELINE_COMMIT = "3a72f8a61e14613f517dded6816d4bfc77817c93" +FIXTURE_NAMES = ( + "v1.lance", + "v2_0.lance", + "v2_1.lance", + "v2_2.lance", + "v2_0_self_described.lance", + "v2_0_mini.lance", +) + + +def git_output(source: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", source, *args], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def generate(source: Path, generator: Path, output: Path, target: Path) -> None: + examples = source / "rust/lance-file/examples" + target_generator = examples / "exact_version_fixture_generator.rs" + created_examples = not examples.exists() + examples.mkdir(parents=True, exist_ok=True) + if target_generator.exists(): + raise RuntimeError(f"refusing to replace existing {target_generator}") + + shutil.copyfile(generator, target_generator) + try: + env = os.environ.copy() + env["CARGO_TARGET_DIR"] = str(target) + subprocess.run( + [ + "cargo", + "run", + "-p", + "lance-file", + "--example", + "exact_version_fixture_generator", + "--", + str(output), + ], + cwd=source, + env=env, + check=True, + ) + finally: + target_generator.unlink(missing_ok=True) + if created_examples: + examples.rmdir() + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Reproduce exact-version fixtures with the locked baseline writers." + ) + parser.add_argument( + "--source", + type=Path, + required=True, + help=f"clean Lance checkout at {BASELINE_COMMIT}", + ) + parser.add_argument( + "--write", + action="store_true", + help="replace the checked-in fixtures after both baseline runs agree", + ) + args = parser.parse_args() + + source = args.source.resolve() + if git_output(source, "rev-parse", "HEAD") != BASELINE_COMMIT: + raise RuntimeError( + f"{source} must be checked out at baseline commit {BASELINE_COMMIT}" + ) + if git_output(source, "status", "--porcelain"): + raise RuntimeError(f"{source} must be clean before fixture generation") + + fixture_dir = Path(__file__).resolve().parent + generator = fixture_dir / "datagen.rs" + with ( + tempfile.TemporaryDirectory(prefix="lance-exact-fixtures-a-") as first_dir, + tempfile.TemporaryDirectory(prefix="lance-exact-fixtures-b-") as second_dir, + tempfile.TemporaryDirectory( + prefix="lance-exact-fixtures-target-" + ) as target_dir, + ): + first = Path(first_dir) + second = Path(second_dir) + target = Path(target_dir) + generate(source, generator, first, target) + generate(source, generator, second, target) + + for name in FIXTURE_NAMES: + first_path = first / name + second_path = second / name + if first_path.read_bytes() != second_path.read_bytes(): + raise RuntimeError(f"separate baseline runs disagree for {name}") + + checked_path = fixture_dir / name + if args.write: + shutil.copyfile(first_path, checked_path) + elif ( + not checked_path.exists() + or checked_path.read_bytes() != first_path.read_bytes() + ): + raise RuntimeError( + f"{name} differs from the reproducible baseline; rerun with --write" + ) + print(f"{name}: {digest(first_path)}") + + +if __name__ == "__main__": + main() diff --git a/rust/lance-file/test_data/exact_versions/datagen.rs b/rust/lance-file/test_data/exact_versions/datagen.rs new file mode 100644 index 00000000000..ebbefdc417b --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/datagen.rs @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use arrow_array::builder::StringDictionaryBuilder; +use arrow_array::types::{Int8Type, Int32Type}; +use arrow_array::{ArrayRef, Int32Array, LargeBinaryArray, ListArray, RecordBatch, StringArray}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use async_trait::async_trait; +use lance_core::datatypes::Schema as LanceSchema; +use lance_encoding::encoder::{EncodingOptions, default_encoding_strategy, encode_batch}; +use lance_file::previous::writer::{ + FileWriter as V1Writer, FileWriterOptions as V1WriterOptions, ManifestProvider, +}; +use lance_file::testing::FsFixture; +use lance_file::version::LanceFileVersion; +use lance_file::writer::{EncodedBatchWriteExt, FileWriter, FileWriterOptions}; +use lance_io::traits::Writer; + +type DynError = Box; + +struct NoManifest; + +#[async_trait] +impl ManifestProvider for NoManifest { + async fn store_schema( + _: &mut dyn Writer, + _: &LanceSchema, + ) -> lance_core::Result> { + Ok(None) + } +} + +fn compatibility_fixture_batch() -> RecordBatch { + let row_count = 4097; + let ids = Arc::new(Int32Array::from_iter_values(0..row_count)) as ArrayRef; + let names = Arc::new(StringArray::from_iter((0..row_count).map(|index| { + (index % 7 != 0).then(|| format!("value-{index:04}-deterministic-fixture")) + }))) as ArrayRef; + let items = Arc::new(ListArray::from_iter_primitive::( + (0..row_count).map(|index| { + (index % 11 != 0).then(|| { + vec![ + Some(index), + (index % 5 != 0).then_some(index * 2), + Some(index * 3), + ] + }) + }), + )) as ArrayRef; + let mut categories = StringDictionaryBuilder::::new(); + for index in 0..row_count { + if index % 13 == 0 { + categories.append_null(); + } else { + categories + .append(match index % 3 { + 0 => "red", + 1 => "green", + _ => "blue", + }) + .expect("fixture dictionary values must fit"); + } + } + let categories = Arc::new(categories.finish()) as ArrayRef; + let blobs = Arc::new(LargeBinaryArray::from_iter_values( + (0..row_count).map(|index| format!("blob-{index:04}-deterministic-payload").into_bytes()), + )) as ArrayRef; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true).with_metadata(HashMap::from([( + "lance-encoding:compression".to_string(), + "none".to_string(), + )])), + Field::new( + "items", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + true, + ), + Field::new( + "category", + DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8)), + true, + ) + .with_metadata(HashMap::from([( + "lance-encoding:dict-values-compression".to_string(), + "none".to_string(), + )])), + Field::new("blob", DataType::LargeBinary, true).with_metadata(HashMap::from([( + "lance-encoding:blob".to_string(), + "true".to_string(), + )])), + ])); + RecordBatch::try_new(schema, vec![ids, names, items, categories, blobs]) + .expect("fixture schema and arrays must agree") +} + +async fn write_current_fixture( + version: LanceFileVersion, + batch: &RecordBatch, + schema: &LanceSchema, +) -> Result, DynError> { + let fs = FsFixture::default(); + let object_writer = fs.object_store.create(&fs.tmp_path).await?; + let mut writer = FileWriter::try_new( + object_writer, + schema.clone(), + FileWriterOptions { + data_cache_bytes: Some(1), + max_page_bytes: Some(1024), + format_version: Some(version), + ..Default::default() + }, + )?; + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write_batch(&slice).await?; + } + let summary = writer.finish().await?; + Ok(fs + .object_store + .open(&fs.tmp_path) + .await? + .get_range(0..summary.size_bytes as usize) + .await? + .to_vec()) +} + +async fn write_v1_fixture(batch: &RecordBatch, schema: &LanceSchema) -> Result, DynError> { + let fs = FsFixture::default(); + let mut writer = V1Writer::::try_new( + fs.object_store.as_ref(), + &fs.tmp_path, + schema.clone(), + &V1WriterOptions { + collect_stats_for_fields: Some(Vec::new()), + }, + ) + .await?; + for offset in (0..batch.num_rows()).step_by(1024) { + let slice = batch.slice(offset, (batch.num_rows() - offset).min(1024)); + writer.write(std::slice::from_ref(&slice)).await?; + } + let summary = writer.finish().await?; + Ok(fs + .object_store + .open(&fs.tmp_path) + .await? + .get_range(0..summary.size_bytes as usize) + .await? + .to_vec()) +} + +async fn write_v2_0_embedded_fixtures( + batch: &RecordBatch, + schema: &LanceSchema, +) -> Result<(Vec, Vec), DynError> { + let version = LanceFileVersion::V2_0; + let options = EncodingOptions { + cache_bytes_per_column: 1, + max_page_bytes: 1024, + keep_original_array: true, + buffer_alignment: 64, + version, + }; + let encoding_strategy = default_encoding_strategy(version); + let encoded_batch = encode_batch( + batch, + Arc::new(schema.clone()), + encoding_strategy.as_ref(), + &options, + ) + .await?; + + Ok(( + encoded_batch.try_to_self_described_lance(version)?.to_vec(), + encoded_batch.try_to_mini_lance(version)?.to_vec(), + )) +} + +fn write_fixture(output_dir: &Path, name: &str, bytes: &[u8]) -> Result<(), DynError> { + std::fs::write(output_dir.join(name), bytes)?; + Ok(()) +} + +#[tokio::main] +async fn main() -> Result<(), DynError> { + let output_dir = std::env::args_os() + .nth(1) + .map(std::path::PathBuf::from) + .ok_or("usage: exact_version_fixture_generator ")?; + std::fs::create_dir_all(&output_dir)?; + + let batch = compatibility_fixture_batch(); + let mut schema = LanceSchema::try_from(batch.schema().as_ref())?; + schema.set_dictionary(&batch)?; + + write_fixture( + &output_dir, + "v1.lance", + &write_v1_fixture(&batch, &schema).await?, + )?; + for (version, name) in [ + (LanceFileVersion::V2_0, "v2_0.lance"), + (LanceFileVersion::V2_1, "v2_1.lance"), + (LanceFileVersion::V2_2, "v2_2.lance"), + ] { + write_fixture( + &output_dir, + name, + &write_current_fixture(version, &batch, &schema).await?, + )?; + } + + let embedded_batch = batch.project(&[0, 1])?.slice(0, 257); + let mut embedded_schema = LanceSchema::try_from(embedded_batch.schema().as_ref())?; + embedded_schema.set_dictionary(&embedded_batch)?; + let (self_described, mini) = + write_v2_0_embedded_fixtures(&embedded_batch, &embedded_schema).await?; + write_fixture(&output_dir, "v2_0_self_described.lance", &self_described)?; + write_fixture(&output_dir, "v2_0_mini.lance", &mini)?; + + Ok(()) +} diff --git a/rust/lance-file/test_data/exact_versions/v1.lance b/rust/lance-file/test_data/exact_versions/v1.lance new file mode 100644 index 00000000000..267835ffb2f Binary files /dev/null and b/rust/lance-file/test_data/exact_versions/v1.lance differ diff --git a/rust/lance-file/test_data/exact_versions/v2_0.lance b/rust/lance-file/test_data/exact_versions/v2_0.lance new file mode 100644 index 00000000000..5c713fc36b3 --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/v2_0.lance @@ -0,0 +1,10040 @@ +blob-0000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +  !"#$%&'()*+,-./0123456789:;<HHHHHHHHHHHH=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyHHHHHHHHHHHHz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0HHHHHHHHHHHH123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmHHHHHHHHHHHHnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$HHHHHHHHHHHH%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`aHHHHHHHHHHHHbcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +   HHHHHHHHHHHH !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUHHHHHHHHHHHHVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH @`value-0001-deterministic-fixturevalue-0002-deterministic-fixturevalue-0003-deterministic-fixturevalue-0004-deterministic-fixturevalue-0005-deterministic-fixturevalue-0006-deterministic-fixture @`value-0008-deterministic-fixturevalue-0009-deterministic-fixturevalue-0010-deterministic-fixturevalue-0011-deterministic-fixturevalue-0012-deterministic-fixturevalue-0013-deterministic-fixturevalue-0015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0016-deterministic-fixturevalue-0017-deterministic-fixturevalue-0018-deterministic-fixturevalue-0019-deterministic-fixturevalue-0020-deterministic-fixturevalue-0022-deterministic-fixturevalue-0023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0024-deterministic-fixturevalue-0025-deterministic-fixturevalue-0026-deterministic-fixturevalue-0027-deterministic-fixturevalue-0029-deterministic-fixturevalue-0030-deterministic-fixturevalue-0031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0032-deterministic-fixturevalue-0033-deterministic-fixturevalue-0034-deterministic-fixturevalue-0036-deterministic-fixturevalue-0037-deterministic-fixturevalue-0038-deterministic-fixturevalue-0039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0040-deterministic-fixturevalue-0041-deterministic-fixturevalue-0043-deterministic-fixturevalue-0044-deterministic-fixturevalue-0045-deterministic-fixturevalue-0046-deterministic-fixturevalue-0047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0048-deterministic-fixturevalue-0050-deterministic-fixturevalue-0051-deterministic-fixturevalue-0052-deterministic-fixturevalue-0053-deterministic-fixturevalue-0054-deterministic-fixturevalue-0055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0057-deterministic-fixturevalue-0058-deterministic-fixturevalue-0059-deterministic-fixturevalue-0060-deterministic-fixturevalue-0061-deterministic-fixturevalue-0062-deterministic-fixture @`value-0064-deterministic-fixturevalue-0065-deterministic-fixturevalue-0066-deterministic-fixturevalue-0067-deterministic-fixturevalue-0068-deterministic-fixturevalue-0069-deterministic-fixturevalue-0071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0072-deterministic-fixturevalue-0073-deterministic-fixturevalue-0074-deterministic-fixturevalue-0075-deterministic-fixturevalue-0076-deterministic-fixturevalue-0078-deterministic-fixturevalue-0079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0080-deterministic-fixturevalue-0081-deterministic-fixturevalue-0082-deterministic-fixturevalue-0083-deterministic-fixturevalue-0085-deterministic-fixturevalue-0086-deterministic-fixturevalue-0087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0088-deterministic-fixturevalue-0089-deterministic-fixturevalue-0090-deterministic-fixturevalue-0092-deterministic-fixturevalue-0093-deterministic-fixturevalue-0094-deterministic-fixturevalue-0095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0096-deterministic-fixturevalue-0097-deterministic-fixturevalue-0099-deterministic-fixturevalue-0100-deterministic-fixturevalue-0101-deterministic-fixturevalue-0102-deterministic-fixturevalue-0103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0104-deterministic-fixturevalue-0106-deterministic-fixturevalue-0107-deterministic-fixturevalue-0108-deterministic-fixturevalue-0109-deterministic-fixturevalue-0110-deterministic-fixturevalue-0111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0113-deterministic-fixturevalue-0114-deterministic-fixturevalue-0115-deterministic-fixturevalue-0116-deterministic-fixturevalue-0117-deterministic-fixturevalue-0118-deterministic-fixture @`value-0120-deterministic-fixturevalue-0121-deterministic-fixturevalue-0122-deterministic-fixturevalue-0123-deterministic-fixturevalue-0124-deterministic-fixturevalue-0125-deterministic-fixturevalue-0127-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0128-deterministic-fixturevalue-0129-deterministic-fixturevalue-0130-deterministic-fixturevalue-0131-deterministic-fixturevalue-0132-deterministic-fixturevalue-0134-deterministic-fixturevalue-0135-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0136-deterministic-fixturevalue-0137-deterministic-fixturevalue-0138-deterministic-fixturevalue-0139-deterministic-fixturevalue-0141-deterministic-fixturevalue-0142-deterministic-fixturevalue-0143-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0144-deterministic-fixturevalue-0145-deterministic-fixturevalue-0146-deterministic-fixturevalue-0148-deterministic-fixturevalue-0149-deterministic-fixturevalue-0150-deterministic-fixturevalue-0151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0152-deterministic-fixturevalue-0153-deterministic-fixturevalue-0155-deterministic-fixturevalue-0156-deterministic-fixturevalue-0157-deterministic-fixturevalue-0158-deterministic-fixturevalue-0159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0160-deterministic-fixturevalue-0162-deterministic-fixturevalue-0163-deterministic-fixturevalue-0164-deterministic-fixturevalue-0165-deterministic-fixturevalue-0166-deterministic-fixturevalue-0167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0169-deterministic-fixturevalue-0170-deterministic-fixturevalue-0171-deterministic-fixturevalue-0172-deterministic-fixturevalue-0173-deterministic-fixturevalue-0174-deterministic-fixture @`value-0176-deterministic-fixturevalue-0177-deterministic-fixturevalue-0178-deterministic-fixturevalue-0179-deterministic-fixturevalue-0180-deterministic-fixturevalue-0181-deterministic-fixturevalue-0183-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0184-deterministic-fixturevalue-0185-deterministic-fixturevalue-0186-deterministic-fixturevalue-0187-deterministic-fixturevalue-0188-deterministic-fixturevalue-0190-deterministic-fixturevalue-0191-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0192-deterministic-fixturevalue-0193-deterministic-fixturevalue-0194-deterministic-fixturevalue-0195-deterministic-fixturevalue-0197-deterministic-fixturevalue-0198-deterministic-fixturevalue-0199-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0200-deterministic-fixturevalue-0201-deterministic-fixturevalue-0202-deterministic-fixturevalue-0204-deterministic-fixturevalue-0205-deterministic-fixturevalue-0206-deterministic-fixturevalue-0207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0208-deterministic-fixturevalue-0209-deterministic-fixturevalue-0211-deterministic-fixturevalue-0212-deterministic-fixturevalue-0213-deterministic-fixturevalue-0214-deterministic-fixturevalue-0215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0216-deterministic-fixturevalue-0218-deterministic-fixturevalue-0219-deterministic-fixturevalue-0220-deterministic-fixturevalue-0221-deterministic-fixturevalue-0222-deterministic-fixturevalue-0223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0225-deterministic-fixturevalue-0226-deterministic-fixturevalue-0227-deterministic-fixturevalue-0228-deterministic-fixturevalue-0229-deterministic-fixturevalue-0230-deterministic-fixture @`value-0232-deterministic-fixturevalue-0233-deterministic-fixturevalue-0234-deterministic-fixturevalue-0235-deterministic-fixturevalue-0236-deterministic-fixturevalue-0237-deterministic-fixturevalue-0239-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0240-deterministic-fixturevalue-0241-deterministic-fixturevalue-0242-deterministic-fixturevalue-0243-deterministic-fixturevalue-0244-deterministic-fixturevalue-0246-deterministic-fixturevalue-0247-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0248-deterministic-fixturevalue-0249-deterministic-fixturevalue-0250-deterministic-fixturevalue-0251-deterministic-fixturevalue-0253-deterministic-fixturevalue-0254-deterministic-fixturevalue-0255-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0256-deterministic-fixturevalue-0257-deterministic-fixturevalue-0258-deterministic-fixturevalue-0260-deterministic-fixturevalue-0261-deterministic-fixturevalue-0262-deterministic-fixturevalue-0263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0264-deterministic-fixturevalue-0265-deterministic-fixturevalue-0267-deterministic-fixturevalue-0268-deterministic-fixturevalue-0269-deterministic-fixturevalue-0270-deterministic-fixturevalue-0271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0272-deterministic-fixturevalue-0274-deterministic-fixturevalue-0275-deterministic-fixturevalue-0276-deterministic-fixturevalue-0277-deterministic-fixturevalue-0278-deterministic-fixturevalue-0279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0281-deterministic-fixturevalue-0282-deterministic-fixturevalue-0283-deterministic-fixturevalue-0284-deterministic-fixturevalue-0285-deterministic-fixturevalue-0286-deterministic-fixture @`value-0288-deterministic-fixturevalue-0289-deterministic-fixturevalue-0290-deterministic-fixturevalue-0291-deterministic-fixturevalue-0292-deterministic-fixturevalue-0293-deterministic-fixturevalue-0295-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0296-deterministic-fixturevalue-0297-deterministic-fixturevalue-0298-deterministic-fixturevalue-0299-deterministic-fixturevalue-0300-deterministic-fixturevalue-0302-deterministic-fixturevalue-0303-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0304-deterministic-fixturevalue-0305-deterministic-fixturevalue-0306-deterministic-fixturevalue-0307-deterministic-fixturevalue-0309-deterministic-fixturevalue-0310-deterministic-fixturevalue-0311-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0312-deterministic-fixturevalue-0313-deterministic-fixturevalue-0314-deterministic-fixturevalue-0316-deterministic-fixturevalue-0317-deterministic-fixturevalue-0318-deterministic-fixturevalue-0319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0320-deterministic-fixturevalue-0321-deterministic-fixturevalue-0323-deterministic-fixturevalue-0324-deterministic-fixturevalue-0325-deterministic-fixturevalue-0326-deterministic-fixturevalue-0327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0328-deterministic-fixturevalue-0330-deterministic-fixturevalue-0331-deterministic-fixturevalue-0332-deterministic-fixturevalue-0333-deterministic-fixturevalue-0334-deterministic-fixturevalue-0335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0337-deterministic-fixturevalue-0338-deterministic-fixturevalue-0339-deterministic-fixturevalue-0340-deterministic-fixturevalue-0341-deterministic-fixturevalue-0342-deterministic-fixture @`value-0344-deterministic-fixturevalue-0345-deterministic-fixturevalue-0346-deterministic-fixturevalue-0347-deterministic-fixturevalue-0348-deterministic-fixturevalue-0349-deterministic-fixturevalue-0351-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0352-deterministic-fixturevalue-0353-deterministic-fixturevalue-0354-deterministic-fixturevalue-0355-deterministic-fixturevalue-0356-deterministic-fixturevalue-0358-deterministic-fixturevalue-0359-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0360-deterministic-fixturevalue-0361-deterministic-fixturevalue-0362-deterministic-fixturevalue-0363-deterministic-fixturevalue-0365-deterministic-fixturevalue-0366-deterministic-fixturevalue-0367-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0368-deterministic-fixturevalue-0369-deterministic-fixturevalue-0370-deterministic-fixturevalue-0372-deterministic-fixturevalue-0373-deterministic-fixturevalue-0374-deterministic-fixturevalue-0375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0376-deterministic-fixturevalue-0377-deterministic-fixturevalue-0379-deterministic-fixturevalue-0380-deterministic-fixturevalue-0381-deterministic-fixturevalue-0382-deterministic-fixturevalue-0383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0384-deterministic-fixturevalue-0386-deterministic-fixturevalue-0387-deterministic-fixturevalue-0388-deterministic-fixturevalue-0389-deterministic-fixturevalue-0390-deterministic-fixturevalue-0391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0393-deterministic-fixturevalue-0394-deterministic-fixturevalue-0395-deterministic-fixturevalue-0396-deterministic-fixturevalue-0397-deterministic-fixturevalue-0398-deterministic-fixture @`value-0400-deterministic-fixturevalue-0401-deterministic-fixturevalue-0402-deterministic-fixturevalue-0403-deterministic-fixturevalue-0404-deterministic-fixturevalue-0405-deterministic-fixturevalue-0407-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0408-deterministic-fixturevalue-0409-deterministic-fixturevalue-0410-deterministic-fixturevalue-0411-deterministic-fixturevalue-0412-deterministic-fixturevalue-0414-deterministic-fixturevalue-0415-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0416-deterministic-fixturevalue-0417-deterministic-fixturevalue-0418-deterministic-fixturevalue-0419-deterministic-fixturevalue-0421-deterministic-fixturevalue-0422-deterministic-fixturevalue-0423-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0424-deterministic-fixturevalue-0425-deterministic-fixturevalue-0426-deterministic-fixturevalue-0428-deterministic-fixturevalue-0429-deterministic-fixturevalue-0430-deterministic-fixturevalue-0431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0432-deterministic-fixturevalue-0433-deterministic-fixturevalue-0435-deterministic-fixturevalue-0436-deterministic-fixturevalue-0437-deterministic-fixturevalue-0438-deterministic-fixturevalue-0439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0440-deterministic-fixturevalue-0442-deterministic-fixturevalue-0443-deterministic-fixturevalue-0444-deterministic-fixturevalue-0445-deterministic-fixturevalue-0446-deterministic-fixturevalue-0447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0449-deterministic-fixturevalue-0450-deterministic-fixturevalue-0451-deterministic-fixturevalue-0452-deterministic-fixturevalue-0453-deterministic-fixturevalue-0454-deterministic-fixture @`value-0456-deterministic-fixturevalue-0457-deterministic-fixturevalue-0458-deterministic-fixturevalue-0459-deterministic-fixturevalue-0460-deterministic-fixturevalue-0461-deterministic-fixturevalue-0463-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0464-deterministic-fixturevalue-0465-deterministic-fixturevalue-0466-deterministic-fixturevalue-0467-deterministic-fixturevalue-0468-deterministic-fixturevalue-0470-deterministic-fixturevalue-0471-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0472-deterministic-fixturevalue-0473-deterministic-fixturevalue-0474-deterministic-fixturevalue-0475-deterministic-fixturevalue-0477-deterministic-fixturevalue-0478-deterministic-fixturevalue-0479-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0480-deterministic-fixturevalue-0481-deterministic-fixturevalue-0482-deterministic-fixturevalue-0484-deterministic-fixturevalue-0485-deterministic-fixturevalue-0486-deterministic-fixturevalue-0487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0488-deterministic-fixturevalue-0489-deterministic-fixturevalue-0491-deterministic-fixturevalue-0492-deterministic-fixturevalue-0493-deterministic-fixturevalue-0494-deterministic-fixturevalue-0495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0496-deterministic-fixturevalue-0498-deterministic-fixturevalue-0499-deterministic-fixturevalue-0500-deterministic-fixturevalue-0501-deterministic-fixturevalue-0502-deterministic-fixturevalue-0503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0505-deterministic-fixturevalue-0506-deterministic-fixturevalue-0507-deterministic-fixturevalue-0508-deterministic-fixturevalue-0509-deterministic-fixturevalue-0510-deterministic-fixture @`value-0512-deterministic-fixturevalue-0513-deterministic-fixturevalue-0514-deterministic-fixturevalue-0515-deterministic-fixturevalue-0516-deterministic-fixturevalue-0517-deterministic-fixturevalue-0519-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0520-deterministic-fixturevalue-0521-deterministic-fixturevalue-0522-deterministic-fixturevalue-0523-deterministic-fixturevalue-0524-deterministic-fixturevalue-0526-deterministic-fixturevalue-0527-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0528-deterministic-fixturevalue-0529-deterministic-fixturevalue-0530-deterministic-fixturevalue-0531-deterministic-fixturevalue-0533-deterministic-fixturevalue-0534-deterministic-fixturevalue-0535-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0536-deterministic-fixturevalue-0537-deterministic-fixturevalue-0538-deterministic-fixturevalue-0540-deterministic-fixturevalue-0541-deterministic-fixturevalue-0542-deterministic-fixturevalue-0543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0544-deterministic-fixturevalue-0545-deterministic-fixturevalue-0547-deterministic-fixturevalue-0548-deterministic-fixturevalue-0549-deterministic-fixturevalue-0550-deterministic-fixturevalue-0551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0552-deterministic-fixturevalue-0554-deterministic-fixturevalue-0555-deterministic-fixturevalue-0556-deterministic-fixturevalue-0557-deterministic-fixturevalue-0558-deterministic-fixturevalue-0559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0561-deterministic-fixturevalue-0562-deterministic-fixturevalue-0563-deterministic-fixturevalue-0564-deterministic-fixturevalue-0565-deterministic-fixturevalue-0566-deterministic-fixture @`value-0568-deterministic-fixturevalue-0569-deterministic-fixturevalue-0570-deterministic-fixturevalue-0571-deterministic-fixturevalue-0572-deterministic-fixturevalue-0573-deterministic-fixturevalue-0575-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0576-deterministic-fixturevalue-0577-deterministic-fixturevalue-0578-deterministic-fixturevalue-0579-deterministic-fixturevalue-0580-deterministic-fixturevalue-0582-deterministic-fixturevalue-0583-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0584-deterministic-fixturevalue-0585-deterministic-fixturevalue-0586-deterministic-fixturevalue-0587-deterministic-fixturevalue-0589-deterministic-fixturevalue-0590-deterministic-fixturevalue-0591-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0592-deterministic-fixturevalue-0593-deterministic-fixturevalue-0594-deterministic-fixturevalue-0596-deterministic-fixturevalue-0597-deterministic-fixturevalue-0598-deterministic-fixturevalue-0599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0600-deterministic-fixturevalue-0601-deterministic-fixturevalue-0603-deterministic-fixturevalue-0604-deterministic-fixturevalue-0605-deterministic-fixturevalue-0606-deterministic-fixturevalue-0607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0608-deterministic-fixturevalue-0610-deterministic-fixturevalue-0611-deterministic-fixturevalue-0612-deterministic-fixturevalue-0613-deterministic-fixturevalue-0614-deterministic-fixturevalue-0615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0617-deterministic-fixturevalue-0618-deterministic-fixturevalue-0619-deterministic-fixturevalue-0620-deterministic-fixturevalue-0621-deterministic-fixturevalue-0622-deterministic-fixture @`value-0624-deterministic-fixturevalue-0625-deterministic-fixturevalue-0626-deterministic-fixturevalue-0627-deterministic-fixturevalue-0628-deterministic-fixturevalue-0629-deterministic-fixturevalue-0631-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0632-deterministic-fixturevalue-0633-deterministic-fixturevalue-0634-deterministic-fixturevalue-0635-deterministic-fixturevalue-0636-deterministic-fixturevalue-0638-deterministic-fixturevalue-0639-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0640-deterministic-fixturevalue-0641-deterministic-fixturevalue-0642-deterministic-fixturevalue-0643-deterministic-fixturevalue-0645-deterministic-fixturevalue-0646-deterministic-fixturevalue-0647-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0648-deterministic-fixturevalue-0649-deterministic-fixturevalue-0650-deterministic-fixturevalue-0652-deterministic-fixturevalue-0653-deterministic-fixturevalue-0654-deterministic-fixturevalue-0655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0656-deterministic-fixturevalue-0657-deterministic-fixturevalue-0659-deterministic-fixturevalue-0660-deterministic-fixturevalue-0661-deterministic-fixturevalue-0662-deterministic-fixturevalue-0663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0664-deterministic-fixturevalue-0666-deterministic-fixturevalue-0667-deterministic-fixturevalue-0668-deterministic-fixturevalue-0669-deterministic-fixturevalue-0670-deterministic-fixturevalue-0671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0673-deterministic-fixturevalue-0674-deterministic-fixturevalue-0675-deterministic-fixturevalue-0676-deterministic-fixturevalue-0677-deterministic-fixturevalue-0678-deterministic-fixture @`value-0680-deterministic-fixturevalue-0681-deterministic-fixturevalue-0682-deterministic-fixturevalue-0683-deterministic-fixturevalue-0684-deterministic-fixturevalue-0685-deterministic-fixturevalue-0687-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0688-deterministic-fixturevalue-0689-deterministic-fixturevalue-0690-deterministic-fixturevalue-0691-deterministic-fixturevalue-0692-deterministic-fixturevalue-0694-deterministic-fixturevalue-0695-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0696-deterministic-fixturevalue-0697-deterministic-fixturevalue-0698-deterministic-fixturevalue-0699-deterministic-fixturevalue-0701-deterministic-fixturevalue-0702-deterministic-fixturevalue-0703-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0704-deterministic-fixturevalue-0705-deterministic-fixturevalue-0706-deterministic-fixturevalue-0708-deterministic-fixturevalue-0709-deterministic-fixturevalue-0710-deterministic-fixturevalue-0711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0712-deterministic-fixturevalue-0713-deterministic-fixturevalue-0715-deterministic-fixturevalue-0716-deterministic-fixturevalue-0717-deterministic-fixturevalue-0718-deterministic-fixturevalue-0719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0720-deterministic-fixturevalue-0722-deterministic-fixturevalue-0723-deterministic-fixturevalue-0724-deterministic-fixturevalue-0725-deterministic-fixturevalue-0726-deterministic-fixturevalue-0727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0729-deterministic-fixturevalue-0730-deterministic-fixturevalue-0731-deterministic-fixturevalue-0732-deterministic-fixturevalue-0733-deterministic-fixturevalue-0734-deterministic-fixture @`value-0736-deterministic-fixturevalue-0737-deterministic-fixturevalue-0738-deterministic-fixturevalue-0739-deterministic-fixturevalue-0740-deterministic-fixturevalue-0741-deterministic-fixturevalue-0743-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0744-deterministic-fixturevalue-0745-deterministic-fixturevalue-0746-deterministic-fixturevalue-0747-deterministic-fixturevalue-0748-deterministic-fixturevalue-0750-deterministic-fixturevalue-0751-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0752-deterministic-fixturevalue-0753-deterministic-fixturevalue-0754-deterministic-fixturevalue-0755-deterministic-fixturevalue-0757-deterministic-fixturevalue-0758-deterministic-fixturevalue-0759-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0760-deterministic-fixturevalue-0761-deterministic-fixturevalue-0762-deterministic-fixturevalue-0764-deterministic-fixturevalue-0765-deterministic-fixturevalue-0766-deterministic-fixturevalue-0767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0768-deterministic-fixturevalue-0769-deterministic-fixturevalue-0771-deterministic-fixturevalue-0772-deterministic-fixturevalue-0773-deterministic-fixturevalue-0774-deterministic-fixturevalue-0775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0776-deterministic-fixturevalue-0778-deterministic-fixturevalue-0779-deterministic-fixturevalue-0780-deterministic-fixturevalue-0781-deterministic-fixturevalue-0782-deterministic-fixturevalue-0783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0785-deterministic-fixturevalue-0786-deterministic-fixturevalue-0787-deterministic-fixturevalue-0788-deterministic-fixturevalue-0789-deterministic-fixturevalue-0790-deterministic-fixture @`value-0792-deterministic-fixturevalue-0793-deterministic-fixturevalue-0794-deterministic-fixturevalue-0795-deterministic-fixturevalue-0796-deterministic-fixturevalue-0797-deterministic-fixturevalue-0799-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0800-deterministic-fixturevalue-0801-deterministic-fixturevalue-0802-deterministic-fixturevalue-0803-deterministic-fixturevalue-0804-deterministic-fixturevalue-0806-deterministic-fixturevalue-0807-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0808-deterministic-fixturevalue-0809-deterministic-fixturevalue-0810-deterministic-fixturevalue-0811-deterministic-fixturevalue-0813-deterministic-fixturevalue-0814-deterministic-fixturevalue-0815-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0816-deterministic-fixturevalue-0817-deterministic-fixturevalue-0818-deterministic-fixturevalue-0820-deterministic-fixturevalue-0821-deterministic-fixturevalue-0822-deterministic-fixturevalue-0823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0824-deterministic-fixturevalue-0825-deterministic-fixturevalue-0827-deterministic-fixturevalue-0828-deterministic-fixturevalue-0829-deterministic-fixturevalue-0830-deterministic-fixturevalue-0831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0832-deterministic-fixturevalue-0834-deterministic-fixturevalue-0835-deterministic-fixturevalue-0836-deterministic-fixturevalue-0837-deterministic-fixturevalue-0838-deterministic-fixturevalue-0839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0841-deterministic-fixturevalue-0842-deterministic-fixturevalue-0843-deterministic-fixturevalue-0844-deterministic-fixturevalue-0845-deterministic-fixturevalue-0846-deterministic-fixture @`value-0848-deterministic-fixturevalue-0849-deterministic-fixturevalue-0850-deterministic-fixturevalue-0851-deterministic-fixturevalue-0852-deterministic-fixturevalue-0853-deterministic-fixturevalue-0855-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0856-deterministic-fixturevalue-0857-deterministic-fixturevalue-0858-deterministic-fixturevalue-0859-deterministic-fixturevalue-0860-deterministic-fixturevalue-0862-deterministic-fixturevalue-0863-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0864-deterministic-fixturevalue-0865-deterministic-fixturevalue-0866-deterministic-fixturevalue-0867-deterministic-fixturevalue-0869-deterministic-fixturevalue-0870-deterministic-fixturevalue-0871-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0872-deterministic-fixturevalue-0873-deterministic-fixturevalue-0874-deterministic-fixturevalue-0876-deterministic-fixturevalue-0877-deterministic-fixturevalue-0878-deterministic-fixturevalue-0879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0880-deterministic-fixturevalue-0881-deterministic-fixturevalue-0883-deterministic-fixturevalue-0884-deterministic-fixturevalue-0885-deterministic-fixturevalue-0886-deterministic-fixturevalue-0887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0888-deterministic-fixturevalue-0890-deterministic-fixturevalue-0891-deterministic-fixturevalue-0892-deterministic-fixturevalue-0893-deterministic-fixturevalue-0894-deterministic-fixturevalue-0895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0897-deterministic-fixturevalue-0898-deterministic-fixturevalue-0899-deterministic-fixturevalue-0900-deterministic-fixturevalue-0901-deterministic-fixturevalue-0902-deterministic-fixture @`value-0904-deterministic-fixturevalue-0905-deterministic-fixturevalue-0906-deterministic-fixturevalue-0907-deterministic-fixturevalue-0908-deterministic-fixturevalue-0909-deterministic-fixturevalue-0911-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0912-deterministic-fixturevalue-0913-deterministic-fixturevalue-0914-deterministic-fixturevalue-0915-deterministic-fixturevalue-0916-deterministic-fixturevalue-0918-deterministic-fixturevalue-0919-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0920-deterministic-fixturevalue-0921-deterministic-fixturevalue-0922-deterministic-fixturevalue-0923-deterministic-fixturevalue-0925-deterministic-fixturevalue-0926-deterministic-fixturevalue-0927-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0928-deterministic-fixturevalue-0929-deterministic-fixturevalue-0930-deterministic-fixturevalue-0932-deterministic-fixturevalue-0933-deterministic-fixturevalue-0934-deterministic-fixturevalue-0935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0936-deterministic-fixturevalue-0937-deterministic-fixturevalue-0939-deterministic-fixturevalue-0940-deterministic-fixturevalue-0941-deterministic-fixturevalue-0942-deterministic-fixturevalue-0943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0944-deterministic-fixturevalue-0946-deterministic-fixturevalue-0947-deterministic-fixturevalue-0948-deterministic-fixturevalue-0949-deterministic-fixturevalue-0950-deterministic-fixturevalue-0951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0953-deterministic-fixturevalue-0954-deterministic-fixturevalue-0955-deterministic-fixturevalue-0956-deterministic-fixturevalue-0957-deterministic-fixturevalue-0958-deterministic-fixture @`value-0960-deterministic-fixturevalue-0961-deterministic-fixturevalue-0962-deterministic-fixturevalue-0963-deterministic-fixturevalue-0964-deterministic-fixturevalue-0965-deterministic-fixturevalue-0967-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-0968-deterministic-fixturevalue-0969-deterministic-fixturevalue-0970-deterministic-fixturevalue-0971-deterministic-fixturevalue-0972-deterministic-fixturevalue-0974-deterministic-fixturevalue-0975-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-0976-deterministic-fixturevalue-0977-deterministic-fixturevalue-0978-deterministic-fixturevalue-0979-deterministic-fixturevalue-0981-deterministic-fixturevalue-0982-deterministic-fixturevalue-0983-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-0984-deterministic-fixturevalue-0985-deterministic-fixturevalue-0986-deterministic-fixturevalue-0988-deterministic-fixturevalue-0989-deterministic-fixturevalue-0990-deterministic-fixturevalue-0991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-0992-deterministic-fixturevalue-0993-deterministic-fixturevalue-0995-deterministic-fixturevalue-0996-deterministic-fixturevalue-0997-deterministic-fixturevalue-0998-deterministic-fixturevalue-0999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1000-deterministic-fixturevalue-1002-deterministic-fixturevalue-1003-deterministic-fixturevalue-1004-deterministic-fixturevalue-1005-deterministic-fixturevalue-1006-deterministic-fixturevalue-1007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1009-deterministic-fixturevalue-1010-deterministic-fixturevalue-1011-deterministic-fixturevalue-1012-deterministic-fixturevalue-1013-deterministic-fixturevalue-1014-deterministic-fixture @`value-1016-deterministic-fixturevalue-1017-deterministic-fixturevalue-1018-deterministic-fixturevalue-1019-deterministic-fixturevalue-1020-deterministic-fixturevalue-1021-deterministic-fixturevalue-1023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +  !$'*-0369<# ?BEHKNQTWZA ]`cfilorux_ {~}    #&), /258;>ADGJ1 MPSVY\_behO knqtwz}m      +  "%(+.147:! =@CFILORUX? [^adgjmpsv] y|{       !$'*-0369<?BEH/KNQTWZ]`cfMilorux{~k  #&),/258;>ADGJMPSV=Y\_behknqt[wz}y + "%(+.147:=@CF-ILORUX[^adKgjmpsvy|i  !$'*-0369<?BEHKNQT;WZ]`cfilorYux{~w  #& ),/258;>AD+GJMPSVY\_bIehknqtwz}g + "%(+.147:=@CFILOR9UX[^adgjmpWsvy|u          ! $ ' * - 0 3 6 9 < ? B )E H K N Q T W Z ] ` Gc f i l o r u x { ~ e      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +M +P +7S +V +Y +\ +_ +b +e +h +k +n +Uq +t +w +z +} + + + + + +s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH     + $ '*-HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 0"3$6&9<*?.E0HK4N6Q8T:WZ>HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH] @`"Df#i$Hl%Jo&Lr'Nu(x)R{*T~+V-.\/^HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH0`1b23f4h5j6l8p9r:t;v<=z>|?HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH~@ACDEFGHIJKLNOHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHPQRSTUVWY Z[\]^HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH_` a#b&d,e/f2g5h8i;j>kAlDmGoHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHMpPqSrVsYt\u_vbwexhzn{q|t}w~HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz} + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $&(*.028:HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH<>BDFHLNPRVXZHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH\ + bdfjl"n%p(+t.x4z7HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH:~=@CFILOUX[^adHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgjmpvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   +   + ! HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$ '*- 0"3$6(<*?,B.EH2K4N6QHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH8TW>] @`!Bc"f#Fi$Hl%Jo&Lr'u(Px*T~+V,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH-Z.\/^0`12d3f5j67n8p9r:t;HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH<x=z>|@ABCDEFGHIKHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHLMNOPQRSTVWXY Z[HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH\]^_a#b&c)d,e/f2g5h8i;jHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH>lDmGnJoMpPqSrVsYt\u_wexhykznHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH{q|t}w~z} +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"$&(,02HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH68:<@BFJLNPHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHTVX +^ `bdhjl"n%r+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHt.v1x47|:~=@CFLORUXHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[^adgmpsvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  + ! $ '*-"36&9(<*?,BEHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH0H2K4NT:W<Z>] @`!c"Df#Fi$Hl%Jo'Nu(PHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHx)R{*T~+,X-Z.\/^02d3f4h56l7nHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH8p9r:;v=z>|?@ABCDEFHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHIJKLMNOPQSTUVWHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHXY Z[\^_` a#b&c)d,e/f2HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg5i;j>kAlDmGnJoMpPqSrVt\u_vbwHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHexhykzn{q|t}w} HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH "$&*,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.0468:@BDHJHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHLNRVX\ +^ `bfhlHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"%p(r+t.v14z7|:~=CFILOHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHRUX[^dgjmpsvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +      + ! $ * -  0 3 $6 &9 (HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH< *? B .E 2K 4N Q 8T :W <Z >] ` !Bc "Df $Hl HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH%o &Lr 'Nu (Px )R{ *~ +V ,X -Z / 0` 1b 2d 3f 4HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 5j 6l 7n 8p :t ;v <x =z > ?~ @ A B CHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH E F G H I J K L M N P Q R S HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHT U V +W +X +Y +[ +\ +] +^ +_ +` +a# +b& +cHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH) +d, +f2 +g5 +h8 +i; +j> +kA +lD +mG +nJ +oM +qS +rV +sHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHY +t\ +u_ +vb +we +xh +yk +zn +|t +}w +~z +} + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + + + +  +$ +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +( +* +, +. + +2 +4 +6 + +< +> +@ +B +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +F +H +J +L +P +R +T +V  Z \ + ^ `  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf h j " n% p( r+ t. 1 x4 |: ~= @ C HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHF I L O R U [ ^ a d g j m p HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHs v |             HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH              HHHHHHHHHHHHHHHHHHHH?HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH             HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@blob-1024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0123456789:;<HHHHHHHHHHHH=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyHHHHHHHHHHHHz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0HHHHHHHHHHHH123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmHHHHHHHHHHHHnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$HHHHHHHHHHHH%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`aHHHHHHHHHHHHbcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +   HHHHHHHHHHHH !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUHHHHHHHHHHHHVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH @`value-1024-deterministic-fixturevalue-1025-deterministic-fixturevalue-1026-deterministic-fixturevalue-1027-deterministic-fixturevalue-1028-deterministic-fixturevalue-1030-deterministic-fixturevalue-1031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1032-deterministic-fixturevalue-1033-deterministic-fixturevalue-1034-deterministic-fixturevalue-1035-deterministic-fixturevalue-1037-deterministic-fixturevalue-1038-deterministic-fixturevalue-1039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1040-deterministic-fixturevalue-1041-deterministic-fixturevalue-1042-deterministic-fixturevalue-1044-deterministic-fixturevalue-1045-deterministic-fixturevalue-1046-deterministic-fixturevalue-1047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1048-deterministic-fixturevalue-1049-deterministic-fixturevalue-1051-deterministic-fixturevalue-1052-deterministic-fixturevalue-1053-deterministic-fixturevalue-1054-deterministic-fixturevalue-1055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1056-deterministic-fixturevalue-1058-deterministic-fixturevalue-1059-deterministic-fixturevalue-1060-deterministic-fixturevalue-1061-deterministic-fixturevalue-1062-deterministic-fixturevalue-1063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1065-deterministic-fixturevalue-1066-deterministic-fixturevalue-1067-deterministic-fixturevalue-1068-deterministic-fixturevalue-1069-deterministic-fixturevalue-1070-deterministic-fixture @`value-1072-deterministic-fixturevalue-1073-deterministic-fixturevalue-1074-deterministic-fixturevalue-1075-deterministic-fixturevalue-1076-deterministic-fixturevalue-1077-deterministic-fixturevalue-1079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1080-deterministic-fixturevalue-1081-deterministic-fixturevalue-1082-deterministic-fixturevalue-1083-deterministic-fixturevalue-1084-deterministic-fixturevalue-1086-deterministic-fixturevalue-1087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1088-deterministic-fixturevalue-1089-deterministic-fixturevalue-1090-deterministic-fixturevalue-1091-deterministic-fixturevalue-1093-deterministic-fixturevalue-1094-deterministic-fixturevalue-1095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1096-deterministic-fixturevalue-1097-deterministic-fixturevalue-1098-deterministic-fixturevalue-1100-deterministic-fixturevalue-1101-deterministic-fixturevalue-1102-deterministic-fixturevalue-1103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1104-deterministic-fixturevalue-1105-deterministic-fixturevalue-1107-deterministic-fixturevalue-1108-deterministic-fixturevalue-1109-deterministic-fixturevalue-1110-deterministic-fixturevalue-1111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1112-deterministic-fixturevalue-1114-deterministic-fixturevalue-1115-deterministic-fixturevalue-1116-deterministic-fixturevalue-1117-deterministic-fixturevalue-1118-deterministic-fixturevalue-1119-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1121-deterministic-fixturevalue-1122-deterministic-fixturevalue-1123-deterministic-fixturevalue-1124-deterministic-fixturevalue-1125-deterministic-fixturevalue-1126-deterministic-fixture @`value-1128-deterministic-fixturevalue-1129-deterministic-fixturevalue-1130-deterministic-fixturevalue-1131-deterministic-fixturevalue-1132-deterministic-fixturevalue-1133-deterministic-fixturevalue-1135-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1136-deterministic-fixturevalue-1137-deterministic-fixturevalue-1138-deterministic-fixturevalue-1139-deterministic-fixturevalue-1140-deterministic-fixturevalue-1142-deterministic-fixturevalue-1143-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1144-deterministic-fixturevalue-1145-deterministic-fixturevalue-1146-deterministic-fixturevalue-1147-deterministic-fixturevalue-1149-deterministic-fixturevalue-1150-deterministic-fixturevalue-1151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1152-deterministic-fixturevalue-1153-deterministic-fixturevalue-1154-deterministic-fixturevalue-1156-deterministic-fixturevalue-1157-deterministic-fixturevalue-1158-deterministic-fixturevalue-1159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1160-deterministic-fixturevalue-1161-deterministic-fixturevalue-1163-deterministic-fixturevalue-1164-deterministic-fixturevalue-1165-deterministic-fixturevalue-1166-deterministic-fixturevalue-1167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1168-deterministic-fixturevalue-1170-deterministic-fixturevalue-1171-deterministic-fixturevalue-1172-deterministic-fixturevalue-1173-deterministic-fixturevalue-1174-deterministic-fixturevalue-1175-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1177-deterministic-fixturevalue-1178-deterministic-fixturevalue-1179-deterministic-fixturevalue-1180-deterministic-fixturevalue-1181-deterministic-fixturevalue-1182-deterministic-fixture @`value-1184-deterministic-fixturevalue-1185-deterministic-fixturevalue-1186-deterministic-fixturevalue-1187-deterministic-fixturevalue-1188-deterministic-fixturevalue-1189-deterministic-fixturevalue-1191-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1192-deterministic-fixturevalue-1193-deterministic-fixturevalue-1194-deterministic-fixturevalue-1195-deterministic-fixturevalue-1196-deterministic-fixturevalue-1198-deterministic-fixturevalue-1199-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1200-deterministic-fixturevalue-1201-deterministic-fixturevalue-1202-deterministic-fixturevalue-1203-deterministic-fixturevalue-1205-deterministic-fixturevalue-1206-deterministic-fixturevalue-1207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1208-deterministic-fixturevalue-1209-deterministic-fixturevalue-1210-deterministic-fixturevalue-1212-deterministic-fixturevalue-1213-deterministic-fixturevalue-1214-deterministic-fixturevalue-1215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1216-deterministic-fixturevalue-1217-deterministic-fixturevalue-1219-deterministic-fixturevalue-1220-deterministic-fixturevalue-1221-deterministic-fixturevalue-1222-deterministic-fixturevalue-1223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1224-deterministic-fixturevalue-1226-deterministic-fixturevalue-1227-deterministic-fixturevalue-1228-deterministic-fixturevalue-1229-deterministic-fixturevalue-1230-deterministic-fixturevalue-1231-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1233-deterministic-fixturevalue-1234-deterministic-fixturevalue-1235-deterministic-fixturevalue-1236-deterministic-fixturevalue-1237-deterministic-fixturevalue-1238-deterministic-fixture @`value-1240-deterministic-fixturevalue-1241-deterministic-fixturevalue-1242-deterministic-fixturevalue-1243-deterministic-fixturevalue-1244-deterministic-fixturevalue-1245-deterministic-fixturevalue-1247-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1248-deterministic-fixturevalue-1249-deterministic-fixturevalue-1250-deterministic-fixturevalue-1251-deterministic-fixturevalue-1252-deterministic-fixturevalue-1254-deterministic-fixturevalue-1255-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1256-deterministic-fixturevalue-1257-deterministic-fixturevalue-1258-deterministic-fixturevalue-1259-deterministic-fixturevalue-1261-deterministic-fixturevalue-1262-deterministic-fixturevalue-1263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1264-deterministic-fixturevalue-1265-deterministic-fixturevalue-1266-deterministic-fixturevalue-1268-deterministic-fixturevalue-1269-deterministic-fixturevalue-1270-deterministic-fixturevalue-1271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1272-deterministic-fixturevalue-1273-deterministic-fixturevalue-1275-deterministic-fixturevalue-1276-deterministic-fixturevalue-1277-deterministic-fixturevalue-1278-deterministic-fixturevalue-1279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1280-deterministic-fixturevalue-1282-deterministic-fixturevalue-1283-deterministic-fixturevalue-1284-deterministic-fixturevalue-1285-deterministic-fixturevalue-1286-deterministic-fixturevalue-1287-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1289-deterministic-fixturevalue-1290-deterministic-fixturevalue-1291-deterministic-fixturevalue-1292-deterministic-fixturevalue-1293-deterministic-fixturevalue-1294-deterministic-fixture @`value-1296-deterministic-fixturevalue-1297-deterministic-fixturevalue-1298-deterministic-fixturevalue-1299-deterministic-fixturevalue-1300-deterministic-fixturevalue-1301-deterministic-fixturevalue-1303-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1304-deterministic-fixturevalue-1305-deterministic-fixturevalue-1306-deterministic-fixturevalue-1307-deterministic-fixturevalue-1308-deterministic-fixturevalue-1310-deterministic-fixturevalue-1311-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1312-deterministic-fixturevalue-1313-deterministic-fixturevalue-1314-deterministic-fixturevalue-1315-deterministic-fixturevalue-1317-deterministic-fixturevalue-1318-deterministic-fixturevalue-1319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1320-deterministic-fixturevalue-1321-deterministic-fixturevalue-1322-deterministic-fixturevalue-1324-deterministic-fixturevalue-1325-deterministic-fixturevalue-1326-deterministic-fixturevalue-1327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1328-deterministic-fixturevalue-1329-deterministic-fixturevalue-1331-deterministic-fixturevalue-1332-deterministic-fixturevalue-1333-deterministic-fixturevalue-1334-deterministic-fixturevalue-1335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1336-deterministic-fixturevalue-1338-deterministic-fixturevalue-1339-deterministic-fixturevalue-1340-deterministic-fixturevalue-1341-deterministic-fixturevalue-1342-deterministic-fixturevalue-1343-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1345-deterministic-fixturevalue-1346-deterministic-fixturevalue-1347-deterministic-fixturevalue-1348-deterministic-fixturevalue-1349-deterministic-fixturevalue-1350-deterministic-fixture @`value-1352-deterministic-fixturevalue-1353-deterministic-fixturevalue-1354-deterministic-fixturevalue-1355-deterministic-fixturevalue-1356-deterministic-fixturevalue-1357-deterministic-fixturevalue-1359-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1360-deterministic-fixturevalue-1361-deterministic-fixturevalue-1362-deterministic-fixturevalue-1363-deterministic-fixturevalue-1364-deterministic-fixturevalue-1366-deterministic-fixturevalue-1367-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1368-deterministic-fixturevalue-1369-deterministic-fixturevalue-1370-deterministic-fixturevalue-1371-deterministic-fixturevalue-1373-deterministic-fixturevalue-1374-deterministic-fixturevalue-1375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1376-deterministic-fixturevalue-1377-deterministic-fixturevalue-1378-deterministic-fixturevalue-1380-deterministic-fixturevalue-1381-deterministic-fixturevalue-1382-deterministic-fixturevalue-1383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1384-deterministic-fixturevalue-1385-deterministic-fixturevalue-1387-deterministic-fixturevalue-1388-deterministic-fixturevalue-1389-deterministic-fixturevalue-1390-deterministic-fixturevalue-1391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1392-deterministic-fixturevalue-1394-deterministic-fixturevalue-1395-deterministic-fixturevalue-1396-deterministic-fixturevalue-1397-deterministic-fixturevalue-1398-deterministic-fixturevalue-1399-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1401-deterministic-fixturevalue-1402-deterministic-fixturevalue-1403-deterministic-fixturevalue-1404-deterministic-fixturevalue-1405-deterministic-fixturevalue-1406-deterministic-fixture @`value-1408-deterministic-fixturevalue-1409-deterministic-fixturevalue-1410-deterministic-fixturevalue-1411-deterministic-fixturevalue-1412-deterministic-fixturevalue-1413-deterministic-fixturevalue-1415-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1416-deterministic-fixturevalue-1417-deterministic-fixturevalue-1418-deterministic-fixturevalue-1419-deterministic-fixturevalue-1420-deterministic-fixturevalue-1422-deterministic-fixturevalue-1423-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1424-deterministic-fixturevalue-1425-deterministic-fixturevalue-1426-deterministic-fixturevalue-1427-deterministic-fixturevalue-1429-deterministic-fixturevalue-1430-deterministic-fixturevalue-1431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1432-deterministic-fixturevalue-1433-deterministic-fixturevalue-1434-deterministic-fixturevalue-1436-deterministic-fixturevalue-1437-deterministic-fixturevalue-1438-deterministic-fixturevalue-1439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1440-deterministic-fixturevalue-1441-deterministic-fixturevalue-1443-deterministic-fixturevalue-1444-deterministic-fixturevalue-1445-deterministic-fixturevalue-1446-deterministic-fixturevalue-1447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1448-deterministic-fixturevalue-1450-deterministic-fixturevalue-1451-deterministic-fixturevalue-1452-deterministic-fixturevalue-1453-deterministic-fixturevalue-1454-deterministic-fixturevalue-1455-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1457-deterministic-fixturevalue-1458-deterministic-fixturevalue-1459-deterministic-fixturevalue-1460-deterministic-fixturevalue-1461-deterministic-fixturevalue-1462-deterministic-fixture @`value-1464-deterministic-fixturevalue-1465-deterministic-fixturevalue-1466-deterministic-fixturevalue-1467-deterministic-fixturevalue-1468-deterministic-fixturevalue-1469-deterministic-fixturevalue-1471-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1472-deterministic-fixturevalue-1473-deterministic-fixturevalue-1474-deterministic-fixturevalue-1475-deterministic-fixturevalue-1476-deterministic-fixturevalue-1478-deterministic-fixturevalue-1479-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1480-deterministic-fixturevalue-1481-deterministic-fixturevalue-1482-deterministic-fixturevalue-1483-deterministic-fixturevalue-1485-deterministic-fixturevalue-1486-deterministic-fixturevalue-1487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1488-deterministic-fixturevalue-1489-deterministic-fixturevalue-1490-deterministic-fixturevalue-1492-deterministic-fixturevalue-1493-deterministic-fixturevalue-1494-deterministic-fixturevalue-1495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1496-deterministic-fixturevalue-1497-deterministic-fixturevalue-1499-deterministic-fixturevalue-1500-deterministic-fixturevalue-1501-deterministic-fixturevalue-1502-deterministic-fixturevalue-1503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1504-deterministic-fixturevalue-1506-deterministic-fixturevalue-1507-deterministic-fixturevalue-1508-deterministic-fixturevalue-1509-deterministic-fixturevalue-1510-deterministic-fixturevalue-1511-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1513-deterministic-fixturevalue-1514-deterministic-fixturevalue-1515-deterministic-fixturevalue-1516-deterministic-fixturevalue-1517-deterministic-fixturevalue-1518-deterministic-fixture @`value-1520-deterministic-fixturevalue-1521-deterministic-fixturevalue-1522-deterministic-fixturevalue-1523-deterministic-fixturevalue-1524-deterministic-fixturevalue-1525-deterministic-fixturevalue-1527-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1528-deterministic-fixturevalue-1529-deterministic-fixturevalue-1530-deterministic-fixturevalue-1531-deterministic-fixturevalue-1532-deterministic-fixturevalue-1534-deterministic-fixturevalue-1535-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1536-deterministic-fixturevalue-1537-deterministic-fixturevalue-1538-deterministic-fixturevalue-1539-deterministic-fixturevalue-1541-deterministic-fixturevalue-1542-deterministic-fixturevalue-1543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1544-deterministic-fixturevalue-1545-deterministic-fixturevalue-1546-deterministic-fixturevalue-1548-deterministic-fixturevalue-1549-deterministic-fixturevalue-1550-deterministic-fixturevalue-1551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1552-deterministic-fixturevalue-1553-deterministic-fixturevalue-1555-deterministic-fixturevalue-1556-deterministic-fixturevalue-1557-deterministic-fixturevalue-1558-deterministic-fixturevalue-1559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1560-deterministic-fixturevalue-1562-deterministic-fixturevalue-1563-deterministic-fixturevalue-1564-deterministic-fixturevalue-1565-deterministic-fixturevalue-1566-deterministic-fixturevalue-1567-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1569-deterministic-fixturevalue-1570-deterministic-fixturevalue-1571-deterministic-fixturevalue-1572-deterministic-fixturevalue-1573-deterministic-fixturevalue-1574-deterministic-fixture @`value-1576-deterministic-fixturevalue-1577-deterministic-fixturevalue-1578-deterministic-fixturevalue-1579-deterministic-fixturevalue-1580-deterministic-fixturevalue-1581-deterministic-fixturevalue-1583-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1584-deterministic-fixturevalue-1585-deterministic-fixturevalue-1586-deterministic-fixturevalue-1587-deterministic-fixturevalue-1588-deterministic-fixturevalue-1590-deterministic-fixturevalue-1591-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1592-deterministic-fixturevalue-1593-deterministic-fixturevalue-1594-deterministic-fixturevalue-1595-deterministic-fixturevalue-1597-deterministic-fixturevalue-1598-deterministic-fixturevalue-1599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1600-deterministic-fixturevalue-1601-deterministic-fixturevalue-1602-deterministic-fixturevalue-1604-deterministic-fixturevalue-1605-deterministic-fixturevalue-1606-deterministic-fixturevalue-1607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1608-deterministic-fixturevalue-1609-deterministic-fixturevalue-1611-deterministic-fixturevalue-1612-deterministic-fixturevalue-1613-deterministic-fixturevalue-1614-deterministic-fixturevalue-1615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1616-deterministic-fixturevalue-1618-deterministic-fixturevalue-1619-deterministic-fixturevalue-1620-deterministic-fixturevalue-1621-deterministic-fixturevalue-1622-deterministic-fixturevalue-1623-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1625-deterministic-fixturevalue-1626-deterministic-fixturevalue-1627-deterministic-fixturevalue-1628-deterministic-fixturevalue-1629-deterministic-fixturevalue-1630-deterministic-fixture @`value-1632-deterministic-fixturevalue-1633-deterministic-fixturevalue-1634-deterministic-fixturevalue-1635-deterministic-fixturevalue-1636-deterministic-fixturevalue-1637-deterministic-fixturevalue-1639-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1640-deterministic-fixturevalue-1641-deterministic-fixturevalue-1642-deterministic-fixturevalue-1643-deterministic-fixturevalue-1644-deterministic-fixturevalue-1646-deterministic-fixturevalue-1647-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1648-deterministic-fixturevalue-1649-deterministic-fixturevalue-1650-deterministic-fixturevalue-1651-deterministic-fixturevalue-1653-deterministic-fixturevalue-1654-deterministic-fixturevalue-1655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1656-deterministic-fixturevalue-1657-deterministic-fixturevalue-1658-deterministic-fixturevalue-1660-deterministic-fixturevalue-1661-deterministic-fixturevalue-1662-deterministic-fixturevalue-1663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1664-deterministic-fixturevalue-1665-deterministic-fixturevalue-1667-deterministic-fixturevalue-1668-deterministic-fixturevalue-1669-deterministic-fixturevalue-1670-deterministic-fixturevalue-1671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1672-deterministic-fixturevalue-1674-deterministic-fixturevalue-1675-deterministic-fixturevalue-1676-deterministic-fixturevalue-1677-deterministic-fixturevalue-1678-deterministic-fixturevalue-1679-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1681-deterministic-fixturevalue-1682-deterministic-fixturevalue-1683-deterministic-fixturevalue-1684-deterministic-fixturevalue-1685-deterministic-fixturevalue-1686-deterministic-fixture @`value-1688-deterministic-fixturevalue-1689-deterministic-fixturevalue-1690-deterministic-fixturevalue-1691-deterministic-fixturevalue-1692-deterministic-fixturevalue-1693-deterministic-fixturevalue-1695-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1696-deterministic-fixturevalue-1697-deterministic-fixturevalue-1698-deterministic-fixturevalue-1699-deterministic-fixturevalue-1700-deterministic-fixturevalue-1702-deterministic-fixturevalue-1703-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1704-deterministic-fixturevalue-1705-deterministic-fixturevalue-1706-deterministic-fixturevalue-1707-deterministic-fixturevalue-1709-deterministic-fixturevalue-1710-deterministic-fixturevalue-1711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1712-deterministic-fixturevalue-1713-deterministic-fixturevalue-1714-deterministic-fixturevalue-1716-deterministic-fixturevalue-1717-deterministic-fixturevalue-1718-deterministic-fixturevalue-1719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1720-deterministic-fixturevalue-1721-deterministic-fixturevalue-1723-deterministic-fixturevalue-1724-deterministic-fixturevalue-1725-deterministic-fixturevalue-1726-deterministic-fixturevalue-1727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1728-deterministic-fixturevalue-1730-deterministic-fixturevalue-1731-deterministic-fixturevalue-1732-deterministic-fixturevalue-1733-deterministic-fixturevalue-1734-deterministic-fixturevalue-1735-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1737-deterministic-fixturevalue-1738-deterministic-fixturevalue-1739-deterministic-fixturevalue-1740-deterministic-fixturevalue-1741-deterministic-fixturevalue-1742-deterministic-fixture @`value-1744-deterministic-fixturevalue-1745-deterministic-fixturevalue-1746-deterministic-fixturevalue-1747-deterministic-fixturevalue-1748-deterministic-fixturevalue-1749-deterministic-fixturevalue-1751-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1752-deterministic-fixturevalue-1753-deterministic-fixturevalue-1754-deterministic-fixturevalue-1755-deterministic-fixturevalue-1756-deterministic-fixturevalue-1758-deterministic-fixturevalue-1759-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1760-deterministic-fixturevalue-1761-deterministic-fixturevalue-1762-deterministic-fixturevalue-1763-deterministic-fixturevalue-1765-deterministic-fixturevalue-1766-deterministic-fixturevalue-1767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1768-deterministic-fixturevalue-1769-deterministic-fixturevalue-1770-deterministic-fixturevalue-1772-deterministic-fixturevalue-1773-deterministic-fixturevalue-1774-deterministic-fixturevalue-1775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1776-deterministic-fixturevalue-1777-deterministic-fixturevalue-1779-deterministic-fixturevalue-1780-deterministic-fixturevalue-1781-deterministic-fixturevalue-1782-deterministic-fixturevalue-1783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1784-deterministic-fixturevalue-1786-deterministic-fixturevalue-1787-deterministic-fixturevalue-1788-deterministic-fixturevalue-1789-deterministic-fixturevalue-1790-deterministic-fixturevalue-1791-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1793-deterministic-fixturevalue-1794-deterministic-fixturevalue-1795-deterministic-fixturevalue-1796-deterministic-fixturevalue-1797-deterministic-fixturevalue-1798-deterministic-fixture @`value-1800-deterministic-fixturevalue-1801-deterministic-fixturevalue-1802-deterministic-fixturevalue-1803-deterministic-fixturevalue-1804-deterministic-fixturevalue-1805-deterministic-fixturevalue-1807-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1808-deterministic-fixturevalue-1809-deterministic-fixturevalue-1810-deterministic-fixturevalue-1811-deterministic-fixturevalue-1812-deterministic-fixturevalue-1814-deterministic-fixturevalue-1815-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1816-deterministic-fixturevalue-1817-deterministic-fixturevalue-1818-deterministic-fixturevalue-1819-deterministic-fixturevalue-1821-deterministic-fixturevalue-1822-deterministic-fixturevalue-1823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1824-deterministic-fixturevalue-1825-deterministic-fixturevalue-1826-deterministic-fixturevalue-1828-deterministic-fixturevalue-1829-deterministic-fixturevalue-1830-deterministic-fixturevalue-1831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1832-deterministic-fixturevalue-1833-deterministic-fixturevalue-1835-deterministic-fixturevalue-1836-deterministic-fixturevalue-1837-deterministic-fixturevalue-1838-deterministic-fixturevalue-1839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1840-deterministic-fixturevalue-1842-deterministic-fixturevalue-1843-deterministic-fixturevalue-1844-deterministic-fixturevalue-1845-deterministic-fixturevalue-1846-deterministic-fixturevalue-1847-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1849-deterministic-fixturevalue-1850-deterministic-fixturevalue-1851-deterministic-fixturevalue-1852-deterministic-fixturevalue-1853-deterministic-fixturevalue-1854-deterministic-fixture @`value-1856-deterministic-fixturevalue-1857-deterministic-fixturevalue-1858-deterministic-fixturevalue-1859-deterministic-fixturevalue-1860-deterministic-fixturevalue-1861-deterministic-fixturevalue-1863-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1864-deterministic-fixturevalue-1865-deterministic-fixturevalue-1866-deterministic-fixturevalue-1867-deterministic-fixturevalue-1868-deterministic-fixturevalue-1870-deterministic-fixturevalue-1871-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1872-deterministic-fixturevalue-1873-deterministic-fixturevalue-1874-deterministic-fixturevalue-1875-deterministic-fixturevalue-1877-deterministic-fixturevalue-1878-deterministic-fixturevalue-1879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1880-deterministic-fixturevalue-1881-deterministic-fixturevalue-1882-deterministic-fixturevalue-1884-deterministic-fixturevalue-1885-deterministic-fixturevalue-1886-deterministic-fixturevalue-1887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1888-deterministic-fixturevalue-1889-deterministic-fixturevalue-1891-deterministic-fixturevalue-1892-deterministic-fixturevalue-1893-deterministic-fixturevalue-1894-deterministic-fixturevalue-1895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1896-deterministic-fixturevalue-1898-deterministic-fixturevalue-1899-deterministic-fixturevalue-1900-deterministic-fixturevalue-1901-deterministic-fixturevalue-1902-deterministic-fixturevalue-1903-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1905-deterministic-fixturevalue-1906-deterministic-fixturevalue-1907-deterministic-fixturevalue-1908-deterministic-fixturevalue-1909-deterministic-fixturevalue-1910-deterministic-fixture @`value-1912-deterministic-fixturevalue-1913-deterministic-fixturevalue-1914-deterministic-fixturevalue-1915-deterministic-fixturevalue-1916-deterministic-fixturevalue-1917-deterministic-fixturevalue-1919-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1920-deterministic-fixturevalue-1921-deterministic-fixturevalue-1922-deterministic-fixturevalue-1923-deterministic-fixturevalue-1924-deterministic-fixturevalue-1926-deterministic-fixturevalue-1927-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1928-deterministic-fixturevalue-1929-deterministic-fixturevalue-1930-deterministic-fixturevalue-1931-deterministic-fixturevalue-1933-deterministic-fixturevalue-1934-deterministic-fixturevalue-1935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1936-deterministic-fixturevalue-1937-deterministic-fixturevalue-1938-deterministic-fixturevalue-1940-deterministic-fixturevalue-1941-deterministic-fixturevalue-1942-deterministic-fixturevalue-1943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-1944-deterministic-fixturevalue-1945-deterministic-fixturevalue-1947-deterministic-fixturevalue-1948-deterministic-fixturevalue-1949-deterministic-fixturevalue-1950-deterministic-fixturevalue-1951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1952-deterministic-fixturevalue-1954-deterministic-fixturevalue-1955-deterministic-fixturevalue-1956-deterministic-fixturevalue-1957-deterministic-fixturevalue-1958-deterministic-fixturevalue-1959-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1961-deterministic-fixturevalue-1962-deterministic-fixturevalue-1963-deterministic-fixturevalue-1964-deterministic-fixturevalue-1965-deterministic-fixturevalue-1966-deterministic-fixture @`value-1968-deterministic-fixturevalue-1969-deterministic-fixturevalue-1970-deterministic-fixturevalue-1971-deterministic-fixturevalue-1972-deterministic-fixturevalue-1973-deterministic-fixturevalue-1975-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-1976-deterministic-fixturevalue-1977-deterministic-fixturevalue-1978-deterministic-fixturevalue-1979-deterministic-fixturevalue-1980-deterministic-fixturevalue-1982-deterministic-fixturevalue-1983-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-1984-deterministic-fixturevalue-1985-deterministic-fixturevalue-1986-deterministic-fixturevalue-1987-deterministic-fixturevalue-1989-deterministic-fixturevalue-1990-deterministic-fixturevalue-1991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-1992-deterministic-fixturevalue-1993-deterministic-fixturevalue-1994-deterministic-fixturevalue-1996-deterministic-fixturevalue-1997-deterministic-fixturevalue-1998-deterministic-fixturevalue-1999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2000-deterministic-fixturevalue-2001-deterministic-fixturevalue-2003-deterministic-fixturevalue-2004-deterministic-fixturevalue-2005-deterministic-fixturevalue-2006-deterministic-fixturevalue-2007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2008-deterministic-fixturevalue-2010-deterministic-fixturevalue-2011-deterministic-fixturevalue-2012-deterministic-fixturevalue-2013-deterministic-fixturevalue-2014-deterministic-fixturevalue-2015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2017-deterministic-fixturevalue-2018-deterministic-fixturevalue-2019-deterministic-fixturevalue-2020-deterministic-fixturevalue-2021-deterministic-fixturevalue-2022-deterministic-fixture @`value-2024-deterministic-fixturevalue-2025-deterministic-fixturevalue-2026-deterministic-fixturevalue-2027-deterministic-fixturevalue-2028-deterministic-fixturevalue-2029-deterministic-fixturevalue-2031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2032-deterministic-fixturevalue-2033-deterministic-fixturevalue-2034-deterministic-fixturevalue-2035-deterministic-fixturevalue-2036-deterministic-fixturevalue-2038-deterministic-fixturevalue-2039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2040-deterministic-fixturevalue-2041-deterministic-fixturevalue-2042-deterministic-fixturevalue-2043-deterministic-fixturevalue-2045-deterministic-fixturevalue-2046-deterministic-fixturevalue-2047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  !$'*-0369<& ?BEHKNQTWZD ]`cfiloruxb {~    #&), /258;>ADGJ4 MPSVY\_behR knqtwz}p      +  "%(+.147:$ =@CFILORUXB [^adgjmpsv` y|~       !$'*-0369<?BEH2KNQTWZ]`cfPilorux{~n  #&),/258";>ADGJMPSV@Y\_behknqt^wz}| + "%(+.147:=@CF0ILORUX[^adNgjmpsvy|l  !$'*-036 9<?BEHKNQT>WZ]`cfilor\ux{~z  #&),/258;>AD.GJMPSVY\_bLehknqtwz}j + "%(+.147:=@CFILOR<UX[^adgjmpZsvy|x          ! $ ' * - 0 3 6 9 < ? B ,E H K N Q T W Z ] ` Jc f i l o r u x { ~ h      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +M +P +:S +V +Y +\ +_ +b +e +h +k +n +Xq +t +w +z +} + + + + + +v + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH      +     ! $ ' * HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH- 0 "3 $6 &9 (< ,B .E 0H 2K N 6Q 8T :W <HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ ] !Bc "Df #Fi $l %Jo &Lr 'Nu (Px ){ *T~ ,X -Z . HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH/^ 0` 1b 2d 3 4h 5j 7n 8 9r :t ;v <x = >HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH| ?~ @ B C D E F G H I J K M NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH O P Q R S T U V X Y Z [ \ ] HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH^ _ ` a# c) d, e/ f2 g5 h8 i; j> kA lD nHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ oM pP qS rV sY t\ u_ vb we yk zn {q |t }HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHw ~z }      +        HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   "  & ( * ,  0 4 6  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH: < > @  D F J  N P R T X HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ \ +b d f h l "n %p (r +v 1x 4HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz 7| := @ C F IL R U X[ ^ aHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH d gj m s vy |      HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH           HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH            HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH    + + +  +  + +  + +  +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH!  +$  +' +*- +0" +3& +9<* +?, +B. +E0 +HK4 +NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH6 +Q8 +TZ> +] @ +`!B +c"D +f#i$H +l%J +o&L +r'N +u)R +{*T +~+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV +,X +-.\ +/^ +0` +1b +24h +5j +6l +78p +9r +:t +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH;v +<=z +?~ +@ +AB +C +D +E +FG +H +J +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHKL +M +N +O +PQ +R +S +UV +W +X +Y + ZHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[ +\ +] +^ +` + a +#b +&c +)d,e +/f +2g +5h +8iHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH;k +Al +Dm +GnJo +Mp +Pq +Sr +VsYt +\v +bw +exhy +kHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz +n{ +q| +t}w~ +z +}    +   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH      $ & ( * . 0 2 HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH4 8 : < > D F H L N P HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHR V Z \ + ` b d f j l "p (HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH+t .v 1x 4z 7:~ = @ CI L O R UHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHX [ ^ a d j m p sv y |  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH           HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH            HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH          +   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   +   !  $  '* 0" 3$ 69( <* ?, BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH. EH2 K6 Q8 TW< Z> ] @ `!B c"f#F i$H l&L r'HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHu(P x)R {*T ~+V ,-Z .\ /^ 12d 3f 4h 5j 6HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH7n 8p 9r :t <x =z >| ?~ @A B C D EGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH H I JK L M N OP R S TU V HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHW X Y Z [ ] ^_ ` a #b &c)d ,e /HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf 2h8i ;j >k Al DmGn Jo Mp Pq Ss Yt \u _vHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH bwex hy kz n{ q|t~ z }     + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH       " $ ( HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH, . 0 2 6 8 : @ B D F HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ L N P T V X Z  +^ ` b d j HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHl "n %(r +t .v 1x 47| : @ CF I LHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH O RU X [ ad g j m ps v yHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH |            HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH           HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH         HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  +   + ! '*- 0"36&HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH9(<*?,B0H2K4N6QT:W<Z>] @`!c#FiHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Hl%Jo&r'Nu(Px)R{*T~+,X.\/^01b2d3HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf4h56l7n9r:;v<x=z>|?@ABHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHDEFGHIJKLMOPQRHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHSTUVWXZ[\]^_` a#bHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH&c)e/f2g5h8i;j>kAlDmGnJpPqSrHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHVsYt\u_vbwexhyk{q|t}w~z}HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH "HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$&*,.048:>@BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHDHJNRTVX\ +^ `HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHfhjl"%p(r+t.v1z7|:~=@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHCFILORX[^adgjmHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHpsy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[blob-2048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  +    !"#$%&'()*+,-./0123456789:;<HHHHHHHHHHHH=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyHHHHHHHHHHHHz{|}~HHHHHHHHHHHHHHHHHHHHHHHH         +                   ! " # $ % & ' ( ) * + , - . / 0 HHHHHHHHHHHH1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m HHHHHHHHHHHHn o p q r s t u v w x y z { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +! +" +# +$ +HHHHHHHHHHHH% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +A +B +C +D +E +F +G +H +I +J +K +L +M +N +O +P +Q +R +S +T +U +V +W +X +Y +Z +[ +\ +] +^ +_ +` +a +HHHHHHHHHHHHb +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHH + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         +            HHHHHHHHHHHH       ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U HHHHHHHHHHHHV W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH @`Avalue-2048-deterministic-fixturevalue-2049-deterministic-fixturevalue-2050-deterministic-fixturevalue-2052-deterministic-fixturevalue-2053-deterministic-fixturevalue-2054-deterministic-fixturevalue-2055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2056-deterministic-fixturevalue-2057-deterministic-fixturevalue-2059-deterministic-fixturevalue-2060-deterministic-fixturevalue-2061-deterministic-fixturevalue-2062-deterministic-fixturevalue-2063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2064-deterministic-fixturevalue-2066-deterministic-fixturevalue-2067-deterministic-fixturevalue-2068-deterministic-fixturevalue-2069-deterministic-fixturevalue-2070-deterministic-fixturevalue-2071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2073-deterministic-fixturevalue-2074-deterministic-fixturevalue-2075-deterministic-fixturevalue-2076-deterministic-fixturevalue-2077-deterministic-fixturevalue-2078-deterministic-fixture @`value-2080-deterministic-fixturevalue-2081-deterministic-fixturevalue-2082-deterministic-fixturevalue-2083-deterministic-fixturevalue-2084-deterministic-fixturevalue-2085-deterministic-fixturevalue-2087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2088-deterministic-fixturevalue-2089-deterministic-fixturevalue-2090-deterministic-fixturevalue-2091-deterministic-fixturevalue-2092-deterministic-fixturevalue-2094-deterministic-fixturevalue-2095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2096-deterministic-fixturevalue-2097-deterministic-fixturevalue-2098-deterministic-fixturevalue-2099-deterministic-fixturevalue-2101-deterministic-fixturevalue-2102-deterministic-fixturevalue-2103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2104-deterministic-fixturevalue-2105-deterministic-fixturevalue-2106-deterministic-fixturevalue-2108-deterministic-fixturevalue-2109-deterministic-fixturevalue-2110-deterministic-fixturevalue-2111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2112-deterministic-fixturevalue-2113-deterministic-fixturevalue-2115-deterministic-fixturevalue-2116-deterministic-fixturevalue-2117-deterministic-fixturevalue-2118-deterministic-fixturevalue-2119-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2120-deterministic-fixturevalue-2122-deterministic-fixturevalue-2123-deterministic-fixturevalue-2124-deterministic-fixturevalue-2125-deterministic-fixturevalue-2126-deterministic-fixturevalue-2127-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2129-deterministic-fixturevalue-2130-deterministic-fixturevalue-2131-deterministic-fixturevalue-2132-deterministic-fixturevalue-2133-deterministic-fixturevalue-2134-deterministic-fixture @`value-2136-deterministic-fixturevalue-2137-deterministic-fixturevalue-2138-deterministic-fixturevalue-2139-deterministic-fixturevalue-2140-deterministic-fixturevalue-2141-deterministic-fixturevalue-2143-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2144-deterministic-fixturevalue-2145-deterministic-fixturevalue-2146-deterministic-fixturevalue-2147-deterministic-fixturevalue-2148-deterministic-fixturevalue-2150-deterministic-fixturevalue-2151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2152-deterministic-fixturevalue-2153-deterministic-fixturevalue-2154-deterministic-fixturevalue-2155-deterministic-fixturevalue-2157-deterministic-fixturevalue-2158-deterministic-fixturevalue-2159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2160-deterministic-fixturevalue-2161-deterministic-fixturevalue-2162-deterministic-fixturevalue-2164-deterministic-fixturevalue-2165-deterministic-fixturevalue-2166-deterministic-fixturevalue-2167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2168-deterministic-fixturevalue-2169-deterministic-fixturevalue-2171-deterministic-fixturevalue-2172-deterministic-fixturevalue-2173-deterministic-fixturevalue-2174-deterministic-fixturevalue-2175-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2176-deterministic-fixturevalue-2178-deterministic-fixturevalue-2179-deterministic-fixturevalue-2180-deterministic-fixturevalue-2181-deterministic-fixturevalue-2182-deterministic-fixturevalue-2183-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2185-deterministic-fixturevalue-2186-deterministic-fixturevalue-2187-deterministic-fixturevalue-2188-deterministic-fixturevalue-2189-deterministic-fixturevalue-2190-deterministic-fixture @`value-2192-deterministic-fixturevalue-2193-deterministic-fixturevalue-2194-deterministic-fixturevalue-2195-deterministic-fixturevalue-2196-deterministic-fixturevalue-2197-deterministic-fixturevalue-2199-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2200-deterministic-fixturevalue-2201-deterministic-fixturevalue-2202-deterministic-fixturevalue-2203-deterministic-fixturevalue-2204-deterministic-fixturevalue-2206-deterministic-fixturevalue-2207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2208-deterministic-fixturevalue-2209-deterministic-fixturevalue-2210-deterministic-fixturevalue-2211-deterministic-fixturevalue-2213-deterministic-fixturevalue-2214-deterministic-fixturevalue-2215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2216-deterministic-fixturevalue-2217-deterministic-fixturevalue-2218-deterministic-fixturevalue-2220-deterministic-fixturevalue-2221-deterministic-fixturevalue-2222-deterministic-fixturevalue-2223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2224-deterministic-fixturevalue-2225-deterministic-fixturevalue-2227-deterministic-fixturevalue-2228-deterministic-fixturevalue-2229-deterministic-fixturevalue-2230-deterministic-fixturevalue-2231-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2232-deterministic-fixturevalue-2234-deterministic-fixturevalue-2235-deterministic-fixturevalue-2236-deterministic-fixturevalue-2237-deterministic-fixturevalue-2238-deterministic-fixturevalue-2239-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2241-deterministic-fixturevalue-2242-deterministic-fixturevalue-2243-deterministic-fixturevalue-2244-deterministic-fixturevalue-2245-deterministic-fixturevalue-2246-deterministic-fixture @`value-2248-deterministic-fixturevalue-2249-deterministic-fixturevalue-2250-deterministic-fixturevalue-2251-deterministic-fixturevalue-2252-deterministic-fixturevalue-2253-deterministic-fixturevalue-2255-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2256-deterministic-fixturevalue-2257-deterministic-fixturevalue-2258-deterministic-fixturevalue-2259-deterministic-fixturevalue-2260-deterministic-fixturevalue-2262-deterministic-fixturevalue-2263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2264-deterministic-fixturevalue-2265-deterministic-fixturevalue-2266-deterministic-fixturevalue-2267-deterministic-fixturevalue-2269-deterministic-fixturevalue-2270-deterministic-fixturevalue-2271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2272-deterministic-fixturevalue-2273-deterministic-fixturevalue-2274-deterministic-fixturevalue-2276-deterministic-fixturevalue-2277-deterministic-fixturevalue-2278-deterministic-fixturevalue-2279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2280-deterministic-fixturevalue-2281-deterministic-fixturevalue-2283-deterministic-fixturevalue-2284-deterministic-fixturevalue-2285-deterministic-fixturevalue-2286-deterministic-fixturevalue-2287-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2288-deterministic-fixturevalue-2290-deterministic-fixturevalue-2291-deterministic-fixturevalue-2292-deterministic-fixturevalue-2293-deterministic-fixturevalue-2294-deterministic-fixturevalue-2295-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2297-deterministic-fixturevalue-2298-deterministic-fixturevalue-2299-deterministic-fixturevalue-2300-deterministic-fixturevalue-2301-deterministic-fixturevalue-2302-deterministic-fixture @`value-2304-deterministic-fixturevalue-2305-deterministic-fixturevalue-2306-deterministic-fixturevalue-2307-deterministic-fixturevalue-2308-deterministic-fixturevalue-2309-deterministic-fixturevalue-2311-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2312-deterministic-fixturevalue-2313-deterministic-fixturevalue-2314-deterministic-fixturevalue-2315-deterministic-fixturevalue-2316-deterministic-fixturevalue-2318-deterministic-fixturevalue-2319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2320-deterministic-fixturevalue-2321-deterministic-fixturevalue-2322-deterministic-fixturevalue-2323-deterministic-fixturevalue-2325-deterministic-fixturevalue-2326-deterministic-fixturevalue-2327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2328-deterministic-fixturevalue-2329-deterministic-fixturevalue-2330-deterministic-fixturevalue-2332-deterministic-fixturevalue-2333-deterministic-fixturevalue-2334-deterministic-fixturevalue-2335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2336-deterministic-fixturevalue-2337-deterministic-fixturevalue-2339-deterministic-fixturevalue-2340-deterministic-fixturevalue-2341-deterministic-fixturevalue-2342-deterministic-fixturevalue-2343-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2344-deterministic-fixturevalue-2346-deterministic-fixturevalue-2347-deterministic-fixturevalue-2348-deterministic-fixturevalue-2349-deterministic-fixturevalue-2350-deterministic-fixturevalue-2351-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2353-deterministic-fixturevalue-2354-deterministic-fixturevalue-2355-deterministic-fixturevalue-2356-deterministic-fixturevalue-2357-deterministic-fixturevalue-2358-deterministic-fixture @`value-2360-deterministic-fixturevalue-2361-deterministic-fixturevalue-2362-deterministic-fixturevalue-2363-deterministic-fixturevalue-2364-deterministic-fixturevalue-2365-deterministic-fixturevalue-2367-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2368-deterministic-fixturevalue-2369-deterministic-fixturevalue-2370-deterministic-fixturevalue-2371-deterministic-fixturevalue-2372-deterministic-fixturevalue-2374-deterministic-fixturevalue-2375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2376-deterministic-fixturevalue-2377-deterministic-fixturevalue-2378-deterministic-fixturevalue-2379-deterministic-fixturevalue-2381-deterministic-fixturevalue-2382-deterministic-fixturevalue-2383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2384-deterministic-fixturevalue-2385-deterministic-fixturevalue-2386-deterministic-fixturevalue-2388-deterministic-fixturevalue-2389-deterministic-fixturevalue-2390-deterministic-fixturevalue-2391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2392-deterministic-fixturevalue-2393-deterministic-fixturevalue-2395-deterministic-fixturevalue-2396-deterministic-fixturevalue-2397-deterministic-fixturevalue-2398-deterministic-fixturevalue-2399-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2400-deterministic-fixturevalue-2402-deterministic-fixturevalue-2403-deterministic-fixturevalue-2404-deterministic-fixturevalue-2405-deterministic-fixturevalue-2406-deterministic-fixturevalue-2407-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2409-deterministic-fixturevalue-2410-deterministic-fixturevalue-2411-deterministic-fixturevalue-2412-deterministic-fixturevalue-2413-deterministic-fixturevalue-2414-deterministic-fixture @`value-2416-deterministic-fixturevalue-2417-deterministic-fixturevalue-2418-deterministic-fixturevalue-2419-deterministic-fixturevalue-2420-deterministic-fixturevalue-2421-deterministic-fixturevalue-2423-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2424-deterministic-fixturevalue-2425-deterministic-fixturevalue-2426-deterministic-fixturevalue-2427-deterministic-fixturevalue-2428-deterministic-fixturevalue-2430-deterministic-fixturevalue-2431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2432-deterministic-fixturevalue-2433-deterministic-fixturevalue-2434-deterministic-fixturevalue-2435-deterministic-fixturevalue-2437-deterministic-fixturevalue-2438-deterministic-fixturevalue-2439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2440-deterministic-fixturevalue-2441-deterministic-fixturevalue-2442-deterministic-fixturevalue-2444-deterministic-fixturevalue-2445-deterministic-fixturevalue-2446-deterministic-fixturevalue-2447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2448-deterministic-fixturevalue-2449-deterministic-fixturevalue-2451-deterministic-fixturevalue-2452-deterministic-fixturevalue-2453-deterministic-fixturevalue-2454-deterministic-fixturevalue-2455-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2456-deterministic-fixturevalue-2458-deterministic-fixturevalue-2459-deterministic-fixturevalue-2460-deterministic-fixturevalue-2461-deterministic-fixturevalue-2462-deterministic-fixturevalue-2463-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2465-deterministic-fixturevalue-2466-deterministic-fixturevalue-2467-deterministic-fixturevalue-2468-deterministic-fixturevalue-2469-deterministic-fixturevalue-2470-deterministic-fixture @`value-2472-deterministic-fixturevalue-2473-deterministic-fixturevalue-2474-deterministic-fixturevalue-2475-deterministic-fixturevalue-2476-deterministic-fixturevalue-2477-deterministic-fixturevalue-2479-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2480-deterministic-fixturevalue-2481-deterministic-fixturevalue-2482-deterministic-fixturevalue-2483-deterministic-fixturevalue-2484-deterministic-fixturevalue-2486-deterministic-fixturevalue-2487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2488-deterministic-fixturevalue-2489-deterministic-fixturevalue-2490-deterministic-fixturevalue-2491-deterministic-fixturevalue-2493-deterministic-fixturevalue-2494-deterministic-fixturevalue-2495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2496-deterministic-fixturevalue-2497-deterministic-fixturevalue-2498-deterministic-fixturevalue-2500-deterministic-fixturevalue-2501-deterministic-fixturevalue-2502-deterministic-fixturevalue-2503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2504-deterministic-fixturevalue-2505-deterministic-fixturevalue-2507-deterministic-fixturevalue-2508-deterministic-fixturevalue-2509-deterministic-fixturevalue-2510-deterministic-fixturevalue-2511-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2512-deterministic-fixturevalue-2514-deterministic-fixturevalue-2515-deterministic-fixturevalue-2516-deterministic-fixturevalue-2517-deterministic-fixturevalue-2518-deterministic-fixturevalue-2519-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2521-deterministic-fixturevalue-2522-deterministic-fixturevalue-2523-deterministic-fixturevalue-2524-deterministic-fixturevalue-2525-deterministic-fixturevalue-2526-deterministic-fixture @`value-2528-deterministic-fixturevalue-2529-deterministic-fixturevalue-2530-deterministic-fixturevalue-2531-deterministic-fixturevalue-2532-deterministic-fixturevalue-2533-deterministic-fixturevalue-2535-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2536-deterministic-fixturevalue-2537-deterministic-fixturevalue-2538-deterministic-fixturevalue-2539-deterministic-fixturevalue-2540-deterministic-fixturevalue-2542-deterministic-fixturevalue-2543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2544-deterministic-fixturevalue-2545-deterministic-fixturevalue-2546-deterministic-fixturevalue-2547-deterministic-fixturevalue-2549-deterministic-fixturevalue-2550-deterministic-fixturevalue-2551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2552-deterministic-fixturevalue-2553-deterministic-fixturevalue-2554-deterministic-fixturevalue-2556-deterministic-fixturevalue-2557-deterministic-fixturevalue-2558-deterministic-fixturevalue-2559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2560-deterministic-fixturevalue-2561-deterministic-fixturevalue-2563-deterministic-fixturevalue-2564-deterministic-fixturevalue-2565-deterministic-fixturevalue-2566-deterministic-fixturevalue-2567-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2568-deterministic-fixturevalue-2570-deterministic-fixturevalue-2571-deterministic-fixturevalue-2572-deterministic-fixturevalue-2573-deterministic-fixturevalue-2574-deterministic-fixturevalue-2575-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2577-deterministic-fixturevalue-2578-deterministic-fixturevalue-2579-deterministic-fixturevalue-2580-deterministic-fixturevalue-2581-deterministic-fixturevalue-2582-deterministic-fixture @`value-2584-deterministic-fixturevalue-2585-deterministic-fixturevalue-2586-deterministic-fixturevalue-2587-deterministic-fixturevalue-2588-deterministic-fixturevalue-2589-deterministic-fixturevalue-2591-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2592-deterministic-fixturevalue-2593-deterministic-fixturevalue-2594-deterministic-fixturevalue-2595-deterministic-fixturevalue-2596-deterministic-fixturevalue-2598-deterministic-fixturevalue-2599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2600-deterministic-fixturevalue-2601-deterministic-fixturevalue-2602-deterministic-fixturevalue-2603-deterministic-fixturevalue-2605-deterministic-fixturevalue-2606-deterministic-fixturevalue-2607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2608-deterministic-fixturevalue-2609-deterministic-fixturevalue-2610-deterministic-fixturevalue-2612-deterministic-fixturevalue-2613-deterministic-fixturevalue-2614-deterministic-fixturevalue-2615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2616-deterministic-fixturevalue-2617-deterministic-fixturevalue-2619-deterministic-fixturevalue-2620-deterministic-fixturevalue-2621-deterministic-fixturevalue-2622-deterministic-fixturevalue-2623-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2624-deterministic-fixturevalue-2626-deterministic-fixturevalue-2627-deterministic-fixturevalue-2628-deterministic-fixturevalue-2629-deterministic-fixturevalue-2630-deterministic-fixturevalue-2631-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2633-deterministic-fixturevalue-2634-deterministic-fixturevalue-2635-deterministic-fixturevalue-2636-deterministic-fixturevalue-2637-deterministic-fixturevalue-2638-deterministic-fixture @`value-2640-deterministic-fixturevalue-2641-deterministic-fixturevalue-2642-deterministic-fixturevalue-2643-deterministic-fixturevalue-2644-deterministic-fixturevalue-2645-deterministic-fixturevalue-2647-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2648-deterministic-fixturevalue-2649-deterministic-fixturevalue-2650-deterministic-fixturevalue-2651-deterministic-fixturevalue-2652-deterministic-fixturevalue-2654-deterministic-fixturevalue-2655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2656-deterministic-fixturevalue-2657-deterministic-fixturevalue-2658-deterministic-fixturevalue-2659-deterministic-fixturevalue-2661-deterministic-fixturevalue-2662-deterministic-fixturevalue-2663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2664-deterministic-fixturevalue-2665-deterministic-fixturevalue-2666-deterministic-fixturevalue-2668-deterministic-fixturevalue-2669-deterministic-fixturevalue-2670-deterministic-fixturevalue-2671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2672-deterministic-fixturevalue-2673-deterministic-fixturevalue-2675-deterministic-fixturevalue-2676-deterministic-fixturevalue-2677-deterministic-fixturevalue-2678-deterministic-fixturevalue-2679-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2680-deterministic-fixturevalue-2682-deterministic-fixturevalue-2683-deterministic-fixturevalue-2684-deterministic-fixturevalue-2685-deterministic-fixturevalue-2686-deterministic-fixturevalue-2687-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2689-deterministic-fixturevalue-2690-deterministic-fixturevalue-2691-deterministic-fixturevalue-2692-deterministic-fixturevalue-2693-deterministic-fixturevalue-2694-deterministic-fixture @`value-2696-deterministic-fixturevalue-2697-deterministic-fixturevalue-2698-deterministic-fixturevalue-2699-deterministic-fixturevalue-2700-deterministic-fixturevalue-2701-deterministic-fixturevalue-2703-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2704-deterministic-fixturevalue-2705-deterministic-fixturevalue-2706-deterministic-fixturevalue-2707-deterministic-fixturevalue-2708-deterministic-fixturevalue-2710-deterministic-fixturevalue-2711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2712-deterministic-fixturevalue-2713-deterministic-fixturevalue-2714-deterministic-fixturevalue-2715-deterministic-fixturevalue-2717-deterministic-fixturevalue-2718-deterministic-fixturevalue-2719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2720-deterministic-fixturevalue-2721-deterministic-fixturevalue-2722-deterministic-fixturevalue-2724-deterministic-fixturevalue-2725-deterministic-fixturevalue-2726-deterministic-fixturevalue-2727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2728-deterministic-fixturevalue-2729-deterministic-fixturevalue-2731-deterministic-fixturevalue-2732-deterministic-fixturevalue-2733-deterministic-fixturevalue-2734-deterministic-fixturevalue-2735-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2736-deterministic-fixturevalue-2738-deterministic-fixturevalue-2739-deterministic-fixturevalue-2740-deterministic-fixturevalue-2741-deterministic-fixturevalue-2742-deterministic-fixturevalue-2743-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2745-deterministic-fixturevalue-2746-deterministic-fixturevalue-2747-deterministic-fixturevalue-2748-deterministic-fixturevalue-2749-deterministic-fixturevalue-2750-deterministic-fixture @`value-2752-deterministic-fixturevalue-2753-deterministic-fixturevalue-2754-deterministic-fixturevalue-2755-deterministic-fixturevalue-2756-deterministic-fixturevalue-2757-deterministic-fixturevalue-2759-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2760-deterministic-fixturevalue-2761-deterministic-fixturevalue-2762-deterministic-fixturevalue-2763-deterministic-fixturevalue-2764-deterministic-fixturevalue-2766-deterministic-fixturevalue-2767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2768-deterministic-fixturevalue-2769-deterministic-fixturevalue-2770-deterministic-fixturevalue-2771-deterministic-fixturevalue-2773-deterministic-fixturevalue-2774-deterministic-fixturevalue-2775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2776-deterministic-fixturevalue-2777-deterministic-fixturevalue-2778-deterministic-fixturevalue-2780-deterministic-fixturevalue-2781-deterministic-fixturevalue-2782-deterministic-fixturevalue-2783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2784-deterministic-fixturevalue-2785-deterministic-fixturevalue-2787-deterministic-fixturevalue-2788-deterministic-fixturevalue-2789-deterministic-fixturevalue-2790-deterministic-fixturevalue-2791-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2792-deterministic-fixturevalue-2794-deterministic-fixturevalue-2795-deterministic-fixturevalue-2796-deterministic-fixturevalue-2797-deterministic-fixturevalue-2798-deterministic-fixturevalue-2799-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2801-deterministic-fixturevalue-2802-deterministic-fixturevalue-2803-deterministic-fixturevalue-2804-deterministic-fixturevalue-2805-deterministic-fixturevalue-2806-deterministic-fixture @`value-2808-deterministic-fixturevalue-2809-deterministic-fixturevalue-2810-deterministic-fixturevalue-2811-deterministic-fixturevalue-2812-deterministic-fixturevalue-2813-deterministic-fixturevalue-2815-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2816-deterministic-fixturevalue-2817-deterministic-fixturevalue-2818-deterministic-fixturevalue-2819-deterministic-fixturevalue-2820-deterministic-fixturevalue-2822-deterministic-fixturevalue-2823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2824-deterministic-fixturevalue-2825-deterministic-fixturevalue-2826-deterministic-fixturevalue-2827-deterministic-fixturevalue-2829-deterministic-fixturevalue-2830-deterministic-fixturevalue-2831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2832-deterministic-fixturevalue-2833-deterministic-fixturevalue-2834-deterministic-fixturevalue-2836-deterministic-fixturevalue-2837-deterministic-fixturevalue-2838-deterministic-fixturevalue-2839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2840-deterministic-fixturevalue-2841-deterministic-fixturevalue-2843-deterministic-fixturevalue-2844-deterministic-fixturevalue-2845-deterministic-fixturevalue-2846-deterministic-fixturevalue-2847-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2848-deterministic-fixturevalue-2850-deterministic-fixturevalue-2851-deterministic-fixturevalue-2852-deterministic-fixturevalue-2853-deterministic-fixturevalue-2854-deterministic-fixturevalue-2855-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2857-deterministic-fixturevalue-2858-deterministic-fixturevalue-2859-deterministic-fixturevalue-2860-deterministic-fixturevalue-2861-deterministic-fixturevalue-2862-deterministic-fixture @`value-2864-deterministic-fixturevalue-2865-deterministic-fixturevalue-2866-deterministic-fixturevalue-2867-deterministic-fixturevalue-2868-deterministic-fixturevalue-2869-deterministic-fixturevalue-2871-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2872-deterministic-fixturevalue-2873-deterministic-fixturevalue-2874-deterministic-fixturevalue-2875-deterministic-fixturevalue-2876-deterministic-fixturevalue-2878-deterministic-fixturevalue-2879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2880-deterministic-fixturevalue-2881-deterministic-fixturevalue-2882-deterministic-fixturevalue-2883-deterministic-fixturevalue-2885-deterministic-fixturevalue-2886-deterministic-fixturevalue-2887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2888-deterministic-fixturevalue-2889-deterministic-fixturevalue-2890-deterministic-fixturevalue-2892-deterministic-fixturevalue-2893-deterministic-fixturevalue-2894-deterministic-fixturevalue-2895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2896-deterministic-fixturevalue-2897-deterministic-fixturevalue-2899-deterministic-fixturevalue-2900-deterministic-fixturevalue-2901-deterministic-fixturevalue-2902-deterministic-fixturevalue-2903-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2904-deterministic-fixturevalue-2906-deterministic-fixturevalue-2907-deterministic-fixturevalue-2908-deterministic-fixturevalue-2909-deterministic-fixturevalue-2910-deterministic-fixturevalue-2911-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2913-deterministic-fixturevalue-2914-deterministic-fixturevalue-2915-deterministic-fixturevalue-2916-deterministic-fixturevalue-2917-deterministic-fixturevalue-2918-deterministic-fixture @`value-2920-deterministic-fixturevalue-2921-deterministic-fixturevalue-2922-deterministic-fixturevalue-2923-deterministic-fixturevalue-2924-deterministic-fixturevalue-2925-deterministic-fixturevalue-2927-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2928-deterministic-fixturevalue-2929-deterministic-fixturevalue-2930-deterministic-fixturevalue-2931-deterministic-fixturevalue-2932-deterministic-fixturevalue-2934-deterministic-fixturevalue-2935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2936-deterministic-fixturevalue-2937-deterministic-fixturevalue-2938-deterministic-fixturevalue-2939-deterministic-fixturevalue-2941-deterministic-fixturevalue-2942-deterministic-fixturevalue-2943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-2944-deterministic-fixturevalue-2945-deterministic-fixturevalue-2946-deterministic-fixturevalue-2948-deterministic-fixturevalue-2949-deterministic-fixturevalue-2950-deterministic-fixturevalue-2951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-2952-deterministic-fixturevalue-2953-deterministic-fixturevalue-2955-deterministic-fixturevalue-2956-deterministic-fixturevalue-2957-deterministic-fixturevalue-2958-deterministic-fixturevalue-2959-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2960-deterministic-fixturevalue-2962-deterministic-fixturevalue-2963-deterministic-fixturevalue-2964-deterministic-fixturevalue-2965-deterministic-fixturevalue-2966-deterministic-fixturevalue-2967-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2969-deterministic-fixturevalue-2970-deterministic-fixturevalue-2971-deterministic-fixturevalue-2972-deterministic-fixturevalue-2973-deterministic-fixturevalue-2974-deterministic-fixture @`value-2976-deterministic-fixturevalue-2977-deterministic-fixturevalue-2978-deterministic-fixturevalue-2979-deterministic-fixturevalue-2980-deterministic-fixturevalue-2981-deterministic-fixturevalue-2983-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-2984-deterministic-fixturevalue-2985-deterministic-fixturevalue-2986-deterministic-fixturevalue-2987-deterministic-fixturevalue-2988-deterministic-fixturevalue-2990-deterministic-fixturevalue-2991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-2992-deterministic-fixturevalue-2993-deterministic-fixturevalue-2994-deterministic-fixturevalue-2995-deterministic-fixturevalue-2997-deterministic-fixturevalue-2998-deterministic-fixturevalue-2999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3000-deterministic-fixturevalue-3001-deterministic-fixturevalue-3002-deterministic-fixturevalue-3004-deterministic-fixturevalue-3005-deterministic-fixturevalue-3006-deterministic-fixturevalue-3007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3008-deterministic-fixturevalue-3009-deterministic-fixturevalue-3011-deterministic-fixturevalue-3012-deterministic-fixturevalue-3013-deterministic-fixturevalue-3014-deterministic-fixturevalue-3015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3016-deterministic-fixturevalue-3018-deterministic-fixturevalue-3019-deterministic-fixturevalue-3020-deterministic-fixturevalue-3021-deterministic-fixturevalue-3022-deterministic-fixturevalue-3023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3025-deterministic-fixturevalue-3026-deterministic-fixturevalue-3027-deterministic-fixturevalue-3028-deterministic-fixturevalue-3029-deterministic-fixturevalue-3030-deterministic-fixture @`value-3032-deterministic-fixturevalue-3033-deterministic-fixturevalue-3034-deterministic-fixturevalue-3035-deterministic-fixturevalue-3036-deterministic-fixturevalue-3037-deterministic-fixturevalue-3039-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3040-deterministic-fixturevalue-3041-deterministic-fixturevalue-3042-deterministic-fixturevalue-3043-deterministic-fixturevalue-3044-deterministic-fixturevalue-3046-deterministic-fixturevalue-3047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3048-deterministic-fixturevalue-3049-deterministic-fixturevalue-3050-deterministic-fixturevalue-3051-deterministic-fixturevalue-3053-deterministic-fixturevalue-3054-deterministic-fixturevalue-3055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3056-deterministic-fixturevalue-3057-deterministic-fixturevalue-3058-deterministic-fixturevalue-3060-deterministic-fixturevalue-3061-deterministic-fixturevalue-3062-deterministic-fixturevalue-3063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3064-deterministic-fixturevalue-3065-deterministic-fixturevalue-3067-deterministic-fixturevalue-3068-deterministic-fixturevalue-3069-deterministic-fixturevalue-3070-deterministic-fixturevalue-3071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  !$'*-0369# <?BEHKNQTWA Z]`cfiloru_ x{~}    #&) ,/258;>ADG1 JMPSVY\_beO hknqtwz}m      +  "%(+.147! :=@CFILORU? X[^adgjmps] vy|{      !$'*-0369<?BE/HKNQTWZ]`cMfilorux{~k  #&),/258;>ADGJMPS=VY\_behknq[twz}y + "%(+.147:=@C-FILORUX[^aKdgjmpsvy|i  !$'*-0369<?BEHKNQ;TWZ]`cfiloYrux{~w  # &),/258;>A+DGJMPSVY\_Ibehknqtwz}g + "%(+.147:=@CFILO9RUX[^adgjmWpsvy|u         ! $ ' * - 0 3 6 9 < ? )B E H K N Q T W Z ] G` c f i l o r u x { e~      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +M +7P +S +V +Y +\ +_ +b +e +h +k +Un +q +t +w +z +} + + + + +s + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH   +  + ! $ '*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH- 03$6&9*?B.E0H2K4NQ8T:W<HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ `!Bc"Df#Fi$Hl%o&Lr'Nu(Px)R{+V,X-Z.\HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH/0`1b2d3f46l7n8p9:t;v<x=z>HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH?~ABCDEFGHIJLMNHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHOPQRSTUWXY Z[\]HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH^_` b&c)d,e/f2g5h8i;j>kAmGnHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJoMpPqSrVsYt\u_vbxhykzn{q|t}HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHw~z} + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH "$(*,.2468HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH<>@BHJLPRTVHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHZ^ `dfhj"n%p(t.1x4HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz7|:~=@CFIORUX[^aHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHdgjpsvy|HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH        +    +  HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH! $ ' * - 0 $6 &9 (< ? ,B .E 0H 2K NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 6Q :W <Z ] @`! Bc" Df# Fi$ l% Jo& Lr( Px) {* T~+ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV, X- Z. / ^0 `1 b3 4 h5 j6 l7 n8 9 r: tHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH; v< x> |? ~@ A B C D E F G I J HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHK L M N O P Q R T U V W X Y  Z HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH[ \ ] _ ` a #b &c )d ,e /f 2g 5h 8j HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH>k Al Dm Gn Jo Mp Pq Sr Vs Yu _v bw ex hy kHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz n{ q| t} w~ z      +    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH       "  & ( ,  0 2HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 4 6  : < >  D F H J  N PHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH R T X Z \ + ^   b d f h  n% p( HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHr+ . v1 x4 z7 |: = @ F I L O R U HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHX [ ^ a g j m p s v y |   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH               HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH              HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH        + + + +  + +  +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + +! +$ +' +- + 0 +"3 +$6 +&9 +< +*? +,B +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.E +0H +4N +6Q +8T +:W +Z +>] +@`! +Bc" +Df# +i% +Jo& +Lr' +NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHu( +x) +R{* +T~+ +V, +X- +. +\0 +`1 +b2 +3 +f4 +h5 +j6 +lHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH7 +8 +p9 +r; +v< += +z> +|? +~@ +A +B +C +D +F +G +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +I +J +K +L +M +N +O +Q +R +S +T +U +V +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHW +X +Y + Z +\ +] +^ +_ +` + a +#b +&c +)d +,e +/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg +5h +8i +;j +>k +Al +Dm +Gn +Jo +Mp +Pr +Vs +Yt +\u +_v +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHbw +ex +hy +kz +n{ +q} +w~ +z +} + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +  + + + + + + + + +  + +& +( +*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +. +0 +2 +4 + +8 +< +> + +B +D +F +H +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +L +N +R + +V +X +Z +\ + + +` +b +d + +jHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +l" +n% +p( ++ +t. +v1 +x4 +z7 +~= +@ +C +F +I +L HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +O +R +U +X +^ +a +d +g +j +m +p +s +v +y +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + + + + + HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + + + + + + + + + + +  ! !  ! HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ! +! ! ! ! ! + ! $! '! *! -! 0! "3! $6! HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH9! (]! @`!" f!# Fi!HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$ Hl!% Jo!& Lr!' u!( Px!) R{!* T~!+ V!- Z!. \!/ ^!0 `!1 !2 d!3 HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf!4 h!5 j!6 !8 p!9 r!: t!; !< x!= z!> |!? ~!@ !A !C HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH!D !E !F !G !H !I !J !K !L !N !O !P !Q !R !HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHS !T !U !V "W "Y "Z "[ "\ "] "^ "_ "`  "a #"b HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH&"d ,"e /"f 2"g 5"h 8"i ;"j >"k A"l D"m G"o M"p P"q S"r HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV"s Y"t \"u _"v b"w e"x h"z n"{ q"| t"} w"~ z" }" " "HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH " " +" " " " " " " " " " " "" HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$" &" (" " ," ." 0" 2" 6" 8" :" <" " @" BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH" D" F" " L" N" P" " T" V# X# Z# +# ^ # b#HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH d# # h# j# l"# n%# (# r+# t.# x4# 7# |:# ~=# @# HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHC# F# I# L# O# U# X# [# ^# a# d# g# j# m# HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHp# v# y# |# # # # # # # # # # # #HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH # # # # # # # # # # # # # # HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH# # # # # # # # # # # # # #HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@blob-3072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH         +                   ! " # $ % & ' ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < HHHHHHHHHHHH= > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y HHHHHHHHHHHHz { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH         +                   ! " # $ % & ' ( ) * + , - . / 0 HHHHHHHHHHHH1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m HHHHHHHHHHHHn o p q r s t u v w x y z { | } ~  HHHHHHHHHHHH HHHHHHHHHHHH   +    !"#$HHHHHHHHHHHH%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`aHHHHHHHHHHHHbcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH  +   HHHHHHHHHHHH !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUHHHHHHHHHHHHVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHH @`value-3072-deterministic-fixturevalue-3074-deterministic-fixturevalue-3075-deterministic-fixturevalue-3076-deterministic-fixturevalue-3077-deterministic-fixturevalue-3078-deterministic-fixturevalue-3079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3081-deterministic-fixturevalue-3082-deterministic-fixturevalue-3083-deterministic-fixturevalue-3084-deterministic-fixturevalue-3085-deterministic-fixturevalue-3086-deterministic-fixture @`value-3088-deterministic-fixturevalue-3089-deterministic-fixturevalue-3090-deterministic-fixturevalue-3091-deterministic-fixturevalue-3092-deterministic-fixturevalue-3093-deterministic-fixturevalue-3095-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3096-deterministic-fixturevalue-3097-deterministic-fixturevalue-3098-deterministic-fixturevalue-3099-deterministic-fixturevalue-3100-deterministic-fixturevalue-3102-deterministic-fixturevalue-3103-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3104-deterministic-fixturevalue-3105-deterministic-fixturevalue-3106-deterministic-fixturevalue-3107-deterministic-fixturevalue-3109-deterministic-fixturevalue-3110-deterministic-fixturevalue-3111-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3112-deterministic-fixturevalue-3113-deterministic-fixturevalue-3114-deterministic-fixturevalue-3116-deterministic-fixturevalue-3117-deterministic-fixturevalue-3118-deterministic-fixturevalue-3119-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3120-deterministic-fixturevalue-3121-deterministic-fixturevalue-3123-deterministic-fixturevalue-3124-deterministic-fixturevalue-3125-deterministic-fixturevalue-3126-deterministic-fixturevalue-3127-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3128-deterministic-fixturevalue-3130-deterministic-fixturevalue-3131-deterministic-fixturevalue-3132-deterministic-fixturevalue-3133-deterministic-fixturevalue-3134-deterministic-fixturevalue-3135-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3137-deterministic-fixturevalue-3138-deterministic-fixturevalue-3139-deterministic-fixturevalue-3140-deterministic-fixturevalue-3141-deterministic-fixturevalue-3142-deterministic-fixture @`value-3144-deterministic-fixturevalue-3145-deterministic-fixturevalue-3146-deterministic-fixturevalue-3147-deterministic-fixturevalue-3148-deterministic-fixturevalue-3149-deterministic-fixturevalue-3151-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3152-deterministic-fixturevalue-3153-deterministic-fixturevalue-3154-deterministic-fixturevalue-3155-deterministic-fixturevalue-3156-deterministic-fixturevalue-3158-deterministic-fixturevalue-3159-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3160-deterministic-fixturevalue-3161-deterministic-fixturevalue-3162-deterministic-fixturevalue-3163-deterministic-fixturevalue-3165-deterministic-fixturevalue-3166-deterministic-fixturevalue-3167-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3168-deterministic-fixturevalue-3169-deterministic-fixturevalue-3170-deterministic-fixturevalue-3172-deterministic-fixturevalue-3173-deterministic-fixturevalue-3174-deterministic-fixturevalue-3175-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3176-deterministic-fixturevalue-3177-deterministic-fixturevalue-3179-deterministic-fixturevalue-3180-deterministic-fixturevalue-3181-deterministic-fixturevalue-3182-deterministic-fixturevalue-3183-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3184-deterministic-fixturevalue-3186-deterministic-fixturevalue-3187-deterministic-fixturevalue-3188-deterministic-fixturevalue-3189-deterministic-fixturevalue-3190-deterministic-fixturevalue-3191-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3193-deterministic-fixturevalue-3194-deterministic-fixturevalue-3195-deterministic-fixturevalue-3196-deterministic-fixturevalue-3197-deterministic-fixturevalue-3198-deterministic-fixture @`value-3200-deterministic-fixturevalue-3201-deterministic-fixturevalue-3202-deterministic-fixturevalue-3203-deterministic-fixturevalue-3204-deterministic-fixturevalue-3205-deterministic-fixturevalue-3207-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3208-deterministic-fixturevalue-3209-deterministic-fixturevalue-3210-deterministic-fixturevalue-3211-deterministic-fixturevalue-3212-deterministic-fixturevalue-3214-deterministic-fixturevalue-3215-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3216-deterministic-fixturevalue-3217-deterministic-fixturevalue-3218-deterministic-fixturevalue-3219-deterministic-fixturevalue-3221-deterministic-fixturevalue-3222-deterministic-fixturevalue-3223-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3224-deterministic-fixturevalue-3225-deterministic-fixturevalue-3226-deterministic-fixturevalue-3228-deterministic-fixturevalue-3229-deterministic-fixturevalue-3230-deterministic-fixturevalue-3231-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3232-deterministic-fixturevalue-3233-deterministic-fixturevalue-3235-deterministic-fixturevalue-3236-deterministic-fixturevalue-3237-deterministic-fixturevalue-3238-deterministic-fixturevalue-3239-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3240-deterministic-fixturevalue-3242-deterministic-fixturevalue-3243-deterministic-fixturevalue-3244-deterministic-fixturevalue-3245-deterministic-fixturevalue-3246-deterministic-fixturevalue-3247-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3249-deterministic-fixturevalue-3250-deterministic-fixturevalue-3251-deterministic-fixturevalue-3252-deterministic-fixturevalue-3253-deterministic-fixturevalue-3254-deterministic-fixture @`value-3256-deterministic-fixturevalue-3257-deterministic-fixturevalue-3258-deterministic-fixturevalue-3259-deterministic-fixturevalue-3260-deterministic-fixturevalue-3261-deterministic-fixturevalue-3263-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3264-deterministic-fixturevalue-3265-deterministic-fixturevalue-3266-deterministic-fixturevalue-3267-deterministic-fixturevalue-3268-deterministic-fixturevalue-3270-deterministic-fixturevalue-3271-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3272-deterministic-fixturevalue-3273-deterministic-fixturevalue-3274-deterministic-fixturevalue-3275-deterministic-fixturevalue-3277-deterministic-fixturevalue-3278-deterministic-fixturevalue-3279-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3280-deterministic-fixturevalue-3281-deterministic-fixturevalue-3282-deterministic-fixturevalue-3284-deterministic-fixturevalue-3285-deterministic-fixturevalue-3286-deterministic-fixturevalue-3287-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3288-deterministic-fixturevalue-3289-deterministic-fixturevalue-3291-deterministic-fixturevalue-3292-deterministic-fixturevalue-3293-deterministic-fixturevalue-3294-deterministic-fixturevalue-3295-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3296-deterministic-fixturevalue-3298-deterministic-fixturevalue-3299-deterministic-fixturevalue-3300-deterministic-fixturevalue-3301-deterministic-fixturevalue-3302-deterministic-fixturevalue-3303-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3305-deterministic-fixturevalue-3306-deterministic-fixturevalue-3307-deterministic-fixturevalue-3308-deterministic-fixturevalue-3309-deterministic-fixturevalue-3310-deterministic-fixture @`value-3312-deterministic-fixturevalue-3313-deterministic-fixturevalue-3314-deterministic-fixturevalue-3315-deterministic-fixturevalue-3316-deterministic-fixturevalue-3317-deterministic-fixturevalue-3319-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3320-deterministic-fixturevalue-3321-deterministic-fixturevalue-3322-deterministic-fixturevalue-3323-deterministic-fixturevalue-3324-deterministic-fixturevalue-3326-deterministic-fixturevalue-3327-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3328-deterministic-fixturevalue-3329-deterministic-fixturevalue-3330-deterministic-fixturevalue-3331-deterministic-fixturevalue-3333-deterministic-fixturevalue-3334-deterministic-fixturevalue-3335-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3336-deterministic-fixturevalue-3337-deterministic-fixturevalue-3338-deterministic-fixturevalue-3340-deterministic-fixturevalue-3341-deterministic-fixturevalue-3342-deterministic-fixturevalue-3343-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3344-deterministic-fixturevalue-3345-deterministic-fixturevalue-3347-deterministic-fixturevalue-3348-deterministic-fixturevalue-3349-deterministic-fixturevalue-3350-deterministic-fixturevalue-3351-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3352-deterministic-fixturevalue-3354-deterministic-fixturevalue-3355-deterministic-fixturevalue-3356-deterministic-fixturevalue-3357-deterministic-fixturevalue-3358-deterministic-fixturevalue-3359-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3361-deterministic-fixturevalue-3362-deterministic-fixturevalue-3363-deterministic-fixturevalue-3364-deterministic-fixturevalue-3365-deterministic-fixturevalue-3366-deterministic-fixture @`value-3368-deterministic-fixturevalue-3369-deterministic-fixturevalue-3370-deterministic-fixturevalue-3371-deterministic-fixturevalue-3372-deterministic-fixturevalue-3373-deterministic-fixturevalue-3375-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3376-deterministic-fixturevalue-3377-deterministic-fixturevalue-3378-deterministic-fixturevalue-3379-deterministic-fixturevalue-3380-deterministic-fixturevalue-3382-deterministic-fixturevalue-3383-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3384-deterministic-fixturevalue-3385-deterministic-fixturevalue-3386-deterministic-fixturevalue-3387-deterministic-fixturevalue-3389-deterministic-fixturevalue-3390-deterministic-fixturevalue-3391-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3392-deterministic-fixturevalue-3393-deterministic-fixturevalue-3394-deterministic-fixturevalue-3396-deterministic-fixturevalue-3397-deterministic-fixturevalue-3398-deterministic-fixturevalue-3399-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3400-deterministic-fixturevalue-3401-deterministic-fixturevalue-3403-deterministic-fixturevalue-3404-deterministic-fixturevalue-3405-deterministic-fixturevalue-3406-deterministic-fixturevalue-3407-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3408-deterministic-fixturevalue-3410-deterministic-fixturevalue-3411-deterministic-fixturevalue-3412-deterministic-fixturevalue-3413-deterministic-fixturevalue-3414-deterministic-fixturevalue-3415-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3417-deterministic-fixturevalue-3418-deterministic-fixturevalue-3419-deterministic-fixturevalue-3420-deterministic-fixturevalue-3421-deterministic-fixturevalue-3422-deterministic-fixture @`value-3424-deterministic-fixturevalue-3425-deterministic-fixturevalue-3426-deterministic-fixturevalue-3427-deterministic-fixturevalue-3428-deterministic-fixturevalue-3429-deterministic-fixturevalue-3431-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3432-deterministic-fixturevalue-3433-deterministic-fixturevalue-3434-deterministic-fixturevalue-3435-deterministic-fixturevalue-3436-deterministic-fixturevalue-3438-deterministic-fixturevalue-3439-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3440-deterministic-fixturevalue-3441-deterministic-fixturevalue-3442-deterministic-fixturevalue-3443-deterministic-fixturevalue-3445-deterministic-fixturevalue-3446-deterministic-fixturevalue-3447-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3448-deterministic-fixturevalue-3449-deterministic-fixturevalue-3450-deterministic-fixturevalue-3452-deterministic-fixturevalue-3453-deterministic-fixturevalue-3454-deterministic-fixturevalue-3455-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3456-deterministic-fixturevalue-3457-deterministic-fixturevalue-3459-deterministic-fixturevalue-3460-deterministic-fixturevalue-3461-deterministic-fixturevalue-3462-deterministic-fixturevalue-3463-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3464-deterministic-fixturevalue-3466-deterministic-fixturevalue-3467-deterministic-fixturevalue-3468-deterministic-fixturevalue-3469-deterministic-fixturevalue-3470-deterministic-fixturevalue-3471-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3473-deterministic-fixturevalue-3474-deterministic-fixturevalue-3475-deterministic-fixturevalue-3476-deterministic-fixturevalue-3477-deterministic-fixturevalue-3478-deterministic-fixture @`value-3480-deterministic-fixturevalue-3481-deterministic-fixturevalue-3482-deterministic-fixturevalue-3483-deterministic-fixturevalue-3484-deterministic-fixturevalue-3485-deterministic-fixturevalue-3487-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3488-deterministic-fixturevalue-3489-deterministic-fixturevalue-3490-deterministic-fixturevalue-3491-deterministic-fixturevalue-3492-deterministic-fixturevalue-3494-deterministic-fixturevalue-3495-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3496-deterministic-fixturevalue-3497-deterministic-fixturevalue-3498-deterministic-fixturevalue-3499-deterministic-fixturevalue-3501-deterministic-fixturevalue-3502-deterministic-fixturevalue-3503-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3504-deterministic-fixturevalue-3505-deterministic-fixturevalue-3506-deterministic-fixturevalue-3508-deterministic-fixturevalue-3509-deterministic-fixturevalue-3510-deterministic-fixturevalue-3511-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3512-deterministic-fixturevalue-3513-deterministic-fixturevalue-3515-deterministic-fixturevalue-3516-deterministic-fixturevalue-3517-deterministic-fixturevalue-3518-deterministic-fixturevalue-3519-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3520-deterministic-fixturevalue-3522-deterministic-fixturevalue-3523-deterministic-fixturevalue-3524-deterministic-fixturevalue-3525-deterministic-fixturevalue-3526-deterministic-fixturevalue-3527-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3529-deterministic-fixturevalue-3530-deterministic-fixturevalue-3531-deterministic-fixturevalue-3532-deterministic-fixturevalue-3533-deterministic-fixturevalue-3534-deterministic-fixture @`value-3536-deterministic-fixturevalue-3537-deterministic-fixturevalue-3538-deterministic-fixturevalue-3539-deterministic-fixturevalue-3540-deterministic-fixturevalue-3541-deterministic-fixturevalue-3543-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3544-deterministic-fixturevalue-3545-deterministic-fixturevalue-3546-deterministic-fixturevalue-3547-deterministic-fixturevalue-3548-deterministic-fixturevalue-3550-deterministic-fixturevalue-3551-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3552-deterministic-fixturevalue-3553-deterministic-fixturevalue-3554-deterministic-fixturevalue-3555-deterministic-fixturevalue-3557-deterministic-fixturevalue-3558-deterministic-fixturevalue-3559-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3560-deterministic-fixturevalue-3561-deterministic-fixturevalue-3562-deterministic-fixturevalue-3564-deterministic-fixturevalue-3565-deterministic-fixturevalue-3566-deterministic-fixturevalue-3567-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3568-deterministic-fixturevalue-3569-deterministic-fixturevalue-3571-deterministic-fixturevalue-3572-deterministic-fixturevalue-3573-deterministic-fixturevalue-3574-deterministic-fixturevalue-3575-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3576-deterministic-fixturevalue-3578-deterministic-fixturevalue-3579-deterministic-fixturevalue-3580-deterministic-fixturevalue-3581-deterministic-fixturevalue-3582-deterministic-fixturevalue-3583-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3585-deterministic-fixturevalue-3586-deterministic-fixturevalue-3587-deterministic-fixturevalue-3588-deterministic-fixturevalue-3589-deterministic-fixturevalue-3590-deterministic-fixture @`value-3592-deterministic-fixturevalue-3593-deterministic-fixturevalue-3594-deterministic-fixturevalue-3595-deterministic-fixturevalue-3596-deterministic-fixturevalue-3597-deterministic-fixturevalue-3599-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3600-deterministic-fixturevalue-3601-deterministic-fixturevalue-3602-deterministic-fixturevalue-3603-deterministic-fixturevalue-3604-deterministic-fixturevalue-3606-deterministic-fixturevalue-3607-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3608-deterministic-fixturevalue-3609-deterministic-fixturevalue-3610-deterministic-fixturevalue-3611-deterministic-fixturevalue-3613-deterministic-fixturevalue-3614-deterministic-fixturevalue-3615-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3616-deterministic-fixturevalue-3617-deterministic-fixturevalue-3618-deterministic-fixturevalue-3620-deterministic-fixturevalue-3621-deterministic-fixturevalue-3622-deterministic-fixturevalue-3623-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3624-deterministic-fixturevalue-3625-deterministic-fixturevalue-3627-deterministic-fixturevalue-3628-deterministic-fixturevalue-3629-deterministic-fixturevalue-3630-deterministic-fixturevalue-3631-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3632-deterministic-fixturevalue-3634-deterministic-fixturevalue-3635-deterministic-fixturevalue-3636-deterministic-fixturevalue-3637-deterministic-fixturevalue-3638-deterministic-fixturevalue-3639-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3641-deterministic-fixturevalue-3642-deterministic-fixturevalue-3643-deterministic-fixturevalue-3644-deterministic-fixturevalue-3645-deterministic-fixturevalue-3646-deterministic-fixture @`value-3648-deterministic-fixturevalue-3649-deterministic-fixturevalue-3650-deterministic-fixturevalue-3651-deterministic-fixturevalue-3652-deterministic-fixturevalue-3653-deterministic-fixturevalue-3655-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3656-deterministic-fixturevalue-3657-deterministic-fixturevalue-3658-deterministic-fixturevalue-3659-deterministic-fixturevalue-3660-deterministic-fixturevalue-3662-deterministic-fixturevalue-3663-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3664-deterministic-fixturevalue-3665-deterministic-fixturevalue-3666-deterministic-fixturevalue-3667-deterministic-fixturevalue-3669-deterministic-fixturevalue-3670-deterministic-fixturevalue-3671-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3672-deterministic-fixturevalue-3673-deterministic-fixturevalue-3674-deterministic-fixturevalue-3676-deterministic-fixturevalue-3677-deterministic-fixturevalue-3678-deterministic-fixturevalue-3679-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3680-deterministic-fixturevalue-3681-deterministic-fixturevalue-3683-deterministic-fixturevalue-3684-deterministic-fixturevalue-3685-deterministic-fixturevalue-3686-deterministic-fixturevalue-3687-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3688-deterministic-fixturevalue-3690-deterministic-fixturevalue-3691-deterministic-fixturevalue-3692-deterministic-fixturevalue-3693-deterministic-fixturevalue-3694-deterministic-fixturevalue-3695-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3697-deterministic-fixturevalue-3698-deterministic-fixturevalue-3699-deterministic-fixturevalue-3700-deterministic-fixturevalue-3701-deterministic-fixturevalue-3702-deterministic-fixture @`value-3704-deterministic-fixturevalue-3705-deterministic-fixturevalue-3706-deterministic-fixturevalue-3707-deterministic-fixturevalue-3708-deterministic-fixturevalue-3709-deterministic-fixturevalue-3711-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3712-deterministic-fixturevalue-3713-deterministic-fixturevalue-3714-deterministic-fixturevalue-3715-deterministic-fixturevalue-3716-deterministic-fixturevalue-3718-deterministic-fixturevalue-3719-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3720-deterministic-fixturevalue-3721-deterministic-fixturevalue-3722-deterministic-fixturevalue-3723-deterministic-fixturevalue-3725-deterministic-fixturevalue-3726-deterministic-fixturevalue-3727-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3728-deterministic-fixturevalue-3729-deterministic-fixturevalue-3730-deterministic-fixturevalue-3732-deterministic-fixturevalue-3733-deterministic-fixturevalue-3734-deterministic-fixturevalue-3735-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3736-deterministic-fixturevalue-3737-deterministic-fixturevalue-3739-deterministic-fixturevalue-3740-deterministic-fixturevalue-3741-deterministic-fixturevalue-3742-deterministic-fixturevalue-3743-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3744-deterministic-fixturevalue-3746-deterministic-fixturevalue-3747-deterministic-fixturevalue-3748-deterministic-fixturevalue-3749-deterministic-fixturevalue-3750-deterministic-fixturevalue-3751-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3753-deterministic-fixturevalue-3754-deterministic-fixturevalue-3755-deterministic-fixturevalue-3756-deterministic-fixturevalue-3757-deterministic-fixturevalue-3758-deterministic-fixture @`value-3760-deterministic-fixturevalue-3761-deterministic-fixturevalue-3762-deterministic-fixturevalue-3763-deterministic-fixturevalue-3764-deterministic-fixturevalue-3765-deterministic-fixturevalue-3767-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3768-deterministic-fixturevalue-3769-deterministic-fixturevalue-3770-deterministic-fixturevalue-3771-deterministic-fixturevalue-3772-deterministic-fixturevalue-3774-deterministic-fixturevalue-3775-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3776-deterministic-fixturevalue-3777-deterministic-fixturevalue-3778-deterministic-fixturevalue-3779-deterministic-fixturevalue-3781-deterministic-fixturevalue-3782-deterministic-fixturevalue-3783-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3784-deterministic-fixturevalue-3785-deterministic-fixturevalue-3786-deterministic-fixturevalue-3788-deterministic-fixturevalue-3789-deterministic-fixturevalue-3790-deterministic-fixturevalue-3791-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3792-deterministic-fixturevalue-3793-deterministic-fixturevalue-3795-deterministic-fixturevalue-3796-deterministic-fixturevalue-3797-deterministic-fixturevalue-3798-deterministic-fixturevalue-3799-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3800-deterministic-fixturevalue-3802-deterministic-fixturevalue-3803-deterministic-fixturevalue-3804-deterministic-fixturevalue-3805-deterministic-fixturevalue-3806-deterministic-fixturevalue-3807-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3809-deterministic-fixturevalue-3810-deterministic-fixturevalue-3811-deterministic-fixturevalue-3812-deterministic-fixturevalue-3813-deterministic-fixturevalue-3814-deterministic-fixture @`value-3816-deterministic-fixturevalue-3817-deterministic-fixturevalue-3818-deterministic-fixturevalue-3819-deterministic-fixturevalue-3820-deterministic-fixturevalue-3821-deterministic-fixturevalue-3823-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3824-deterministic-fixturevalue-3825-deterministic-fixturevalue-3826-deterministic-fixturevalue-3827-deterministic-fixturevalue-3828-deterministic-fixturevalue-3830-deterministic-fixturevalue-3831-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3832-deterministic-fixturevalue-3833-deterministic-fixturevalue-3834-deterministic-fixturevalue-3835-deterministic-fixturevalue-3837-deterministic-fixturevalue-3838-deterministic-fixturevalue-3839-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3840-deterministic-fixturevalue-3841-deterministic-fixturevalue-3842-deterministic-fixturevalue-3844-deterministic-fixturevalue-3845-deterministic-fixturevalue-3846-deterministic-fixturevalue-3847-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3848-deterministic-fixturevalue-3849-deterministic-fixturevalue-3851-deterministic-fixturevalue-3852-deterministic-fixturevalue-3853-deterministic-fixturevalue-3854-deterministic-fixturevalue-3855-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3856-deterministic-fixturevalue-3858-deterministic-fixturevalue-3859-deterministic-fixturevalue-3860-deterministic-fixturevalue-3861-deterministic-fixturevalue-3862-deterministic-fixturevalue-3863-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3865-deterministic-fixturevalue-3866-deterministic-fixturevalue-3867-deterministic-fixturevalue-3868-deterministic-fixturevalue-3869-deterministic-fixturevalue-3870-deterministic-fixture @`value-3872-deterministic-fixturevalue-3873-deterministic-fixturevalue-3874-deterministic-fixturevalue-3875-deterministic-fixturevalue-3876-deterministic-fixturevalue-3877-deterministic-fixturevalue-3879-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3880-deterministic-fixturevalue-3881-deterministic-fixturevalue-3882-deterministic-fixturevalue-3883-deterministic-fixturevalue-3884-deterministic-fixturevalue-3886-deterministic-fixturevalue-3887-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3888-deterministic-fixturevalue-3889-deterministic-fixturevalue-3890-deterministic-fixturevalue-3891-deterministic-fixturevalue-3893-deterministic-fixturevalue-3894-deterministic-fixturevalue-3895-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3896-deterministic-fixturevalue-3897-deterministic-fixturevalue-3898-deterministic-fixturevalue-3900-deterministic-fixturevalue-3901-deterministic-fixturevalue-3902-deterministic-fixturevalue-3903-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3904-deterministic-fixturevalue-3905-deterministic-fixturevalue-3907-deterministic-fixturevalue-3908-deterministic-fixturevalue-3909-deterministic-fixturevalue-3910-deterministic-fixturevalue-3911-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3912-deterministic-fixturevalue-3914-deterministic-fixturevalue-3915-deterministic-fixturevalue-3916-deterministic-fixturevalue-3917-deterministic-fixturevalue-3918-deterministic-fixturevalue-3919-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3921-deterministic-fixturevalue-3922-deterministic-fixturevalue-3923-deterministic-fixturevalue-3924-deterministic-fixturevalue-3925-deterministic-fixturevalue-3926-deterministic-fixture @`value-3928-deterministic-fixturevalue-3929-deterministic-fixturevalue-3930-deterministic-fixturevalue-3931-deterministic-fixturevalue-3932-deterministic-fixturevalue-3933-deterministic-fixturevalue-3935-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3936-deterministic-fixturevalue-3937-deterministic-fixturevalue-3938-deterministic-fixturevalue-3939-deterministic-fixturevalue-3940-deterministic-fixturevalue-3942-deterministic-fixturevalue-3943-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-3944-deterministic-fixturevalue-3945-deterministic-fixturevalue-3946-deterministic-fixturevalue-3947-deterministic-fixturevalue-3949-deterministic-fixturevalue-3950-deterministic-fixturevalue-3951-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-3952-deterministic-fixturevalue-3953-deterministic-fixturevalue-3954-deterministic-fixturevalue-3956-deterministic-fixturevalue-3957-deterministic-fixturevalue-3958-deterministic-fixturevalue-3959-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-3960-deterministic-fixturevalue-3961-deterministic-fixturevalue-3963-deterministic-fixturevalue-3964-deterministic-fixturevalue-3965-deterministic-fixturevalue-3966-deterministic-fixturevalue-3967-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3968-deterministic-fixturevalue-3970-deterministic-fixturevalue-3971-deterministic-fixturevalue-3972-deterministic-fixturevalue-3973-deterministic-fixturevalue-3974-deterministic-fixturevalue-3975-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3977-deterministic-fixturevalue-3978-deterministic-fixturevalue-3979-deterministic-fixturevalue-3980-deterministic-fixturevalue-3981-deterministic-fixturevalue-3982-deterministic-fixture @`value-3984-deterministic-fixturevalue-3985-deterministic-fixturevalue-3986-deterministic-fixturevalue-3987-deterministic-fixturevalue-3988-deterministic-fixturevalue-3989-deterministic-fixturevalue-3991-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-3992-deterministic-fixturevalue-3993-deterministic-fixturevalue-3994-deterministic-fixturevalue-3995-deterministic-fixturevalue-3996-deterministic-fixturevalue-3998-deterministic-fixturevalue-3999-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-4000-deterministic-fixturevalue-4001-deterministic-fixturevalue-4002-deterministic-fixturevalue-4003-deterministic-fixturevalue-4005-deterministic-fixturevalue-4006-deterministic-fixturevalue-4007-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-4008-deterministic-fixturevalue-4009-deterministic-fixturevalue-4010-deterministic-fixturevalue-4012-deterministic-fixturevalue-4013-deterministic-fixturevalue-4014-deterministic-fixturevalue-4015-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-4016-deterministic-fixturevalue-4017-deterministic-fixturevalue-4019-deterministic-fixturevalue-4020-deterministic-fixturevalue-4021-deterministic-fixturevalue-4022-deterministic-fixturevalue-4023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4024-deterministic-fixturevalue-4026-deterministic-fixturevalue-4027-deterministic-fixturevalue-4028-deterministic-fixturevalue-4029-deterministic-fixturevalue-4030-deterministic-fixturevalue-4031-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4033-deterministic-fixturevalue-4034-deterministic-fixturevalue-4035-deterministic-fixturevalue-4036-deterministic-fixturevalue-4037-deterministic-fixturevalue-4038-deterministic-fixture @`value-4040-deterministic-fixturevalue-4041-deterministic-fixturevalue-4042-deterministic-fixturevalue-4043-deterministic-fixturevalue-4044-deterministic-fixturevalue-4045-deterministic-fixturevalue-4047-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4048-deterministic-fixturevalue-4049-deterministic-fixturevalue-4050-deterministic-fixturevalue-4051-deterministic-fixturevalue-4052-deterministic-fixturevalue-4054-deterministic-fixturevalue-4055-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`avalue-4056-deterministic-fixturevalue-4057-deterministic-fixturevalue-4058-deterministic-fixturevalue-4059-deterministic-fixturevalue-4061-deterministic-fixturevalue-4062-deterministic-fixturevalue-4063-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`Avalue-4064-deterministic-fixturevalue-4065-deterministic-fixturevalue-4066-deterministic-fixturevalue-4068-deterministic-fixturevalue-4069-deterministic-fixturevalue-4070-deterministic-fixturevalue-4071-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @!`value-4072-deterministic-fixturevalue-4073-deterministic-fixturevalue-4075-deterministic-fixturevalue-4076-deterministic-fixturevalue-4077-deterministic-fixturevalue-4078-deterministic-fixturevalue-4079-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4080-deterministic-fixturevalue-4082-deterministic-fixturevalue-4083-deterministic-fixturevalue-4084-deterministic-fixturevalue-4085-deterministic-fixturevalue-4086-deterministic-fixturevalue-4087-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @`value-4089-deterministic-fixturevalue-4090-deterministic-fixturevalue-4091-deterministic-fixturevalue-4092-deterministic-fixturevalue-4093-deterministic-fixturevalue-4094-deterministic-fixture  !$'*-036 9<?BEHKNQT> WZ]`cfilor\ ux{~z   #& ),/258;>AD. GJMPSVY\_bL ehknqtwz}j      +  "%(+.14 7:=@CFILOR< UX[^adgjmpZ svy|x      !$'*-0369<?B,EHKNQTWZ]`Jcfilorux{~h  #&),/258;>ADGJMP:SVY\_behknXqtwz}v + " %(+.147:=@*CFILORUX[^Hadgjmpsvy|f  !$'*-0369<?BEHKN8QTWZ]`cfilVorux{~t   +#&),/258;>(ADGJMPSVY\F_behknqtwzd} + "%(+.147:=@CFIL6ORUX[^adgjTmpsvy|r         ! $ ' * - 0 3 6 9 < &? B E H K N Q T W Z D] ` c f i l o r u x b{ ~      + + + + + + + + + + +# +& +) +, +/ +2 +5 +8 +; +> +A +D +G +J +4M +P +S +V +Y +\ +_ +b +e +h +Rk +n +q +t +w +z +} + + + +p + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $ $ $ $  $ +$ $ $ $ + $ !$ $$ '$ *$ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH-$ 0$ "3$ 6$ (<$ *?$ ,B$ E$ 0H$ 2K$ 4N$ 6Q$ T$ :W$ >HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH]$ @`$! c$" Df$# Fi$$ Hl$% Jo$& r$' Nu$( Px$* T~$+ $, X$- Z$. \$HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH/ ^$0 $1 b$2 d$3 f$5 $6 l$7 n$8 p$9 r$: $; v$< x$= z$> HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH|$@ $A $B $C $D $E $F $G $H $I $K $L $M $N HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$O $P $Q $R $S $T $V %W %X %Y  %Z %[ %\ %] %HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH^ %_ %a #%b &%c )%d ,%e /%f 2%g 5%h 8%i ;%j >%l D%m G%n HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ%o M%p P%q S%r V%s Y%t \%u _%w e%x h%y k%z n%{ q%| t%} HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHw%~ z% }% % % % % % % % % % % % %HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH % % % "% $% &% % *% ,% 0% % 4% 6% 8% HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH:% % >% @% B% % H% J% L% N% % R% T% V& XHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH& \ +& ^ & `& b& & f& h& j& l"& %& r+& t.& v1& 4&HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH z7& |:& ~=& @& C& F& L& O& R& U& X& [& ^& a& HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHd& g& m& p& s& v& y& |& & & & & & & HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH& & & & & & & & & & & & & & &HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH & & & & & & & & & & & & & & HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH& & & & ' ' '  '  ' ' ' ' ' + ' HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH!' $' '' *' -' 3' $6' &9' (<' *?' B' .E' 0H' 2K' 4N'HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH 8T' :W' <Z' >]' `'! Bc'" Df'# Fi'$ Hl'% o'' Nu'( Px') R{'* ~'+ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV', X'- Z'. \'/ '0 `'2 d'3 f'4 '5 j'6 l'7 n'8 p'9 ': tHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH'; v'= z'> '? ~'@ 'A 'B 'C 'D 'E 'F 'H 'I 'J 'HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHK 'L 'M 'N 'O 'P 'Q 'S 'T 'U 'V (W (X (Y  (Z HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH([ (\ (^ (_ (`  (a #(b &(c )(d ,(e /(f 2(g 5(i ;(j HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH>(k A(l D(m G(n J(o M(p P(q S(r V(t \(u _(v b(w e(x h(y k(HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHz n({ q(| t(} w( }( ( ( ( ( ( +( ( ( ( HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH( ( ( ( ( ( ( "( $( ( *( ,( .( ( 2HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH( 4( 6( 8( ( <( @( B( ( F( H( J( L( ( P(HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH R( V) ) Z) \ +) ^ ) `) ) d) f) h) ") n%) p() HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHr+) t.) 1) x4) z7) |:) ~=) C) F) I) L) O) R) U) HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHX) [) ^) d) g) j) m) p) s) v) y) |) ) ) )HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ) ) ) ) ) ) ) ) ) ) ) ) ) ) HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH) ) ) ) ) ) ) ) ) ) ) ) ) ) HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH) ) ) ) ) ) ) )** * * +***HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH* * +* !* $***-*0*"3*$6*&9*(<*?*,B*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.E*2K*N*6Q*8T*:W*<Z*]* @`*!Bc*"Df*$l*%Jo*&Lr*'NHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHu*(Px*){**T~*+V*,X*-Z*/^*0`*1b*2d*3*4h*5j*6l*HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH7n*8*:t*;v*<x*=*>|*?~*@*A*B*C*E*F*GHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH*H*I*J*K*L*M*N*P*Q*R*S*T*U*VHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH+W+X+Y +[+\+]+^+_+` +a#+b&+c)+d,+f2+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg5+h8+i;+j>+kA+lD+mG+nJ+oM+qS+rV+sY+t\+u_+vHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHb+we+xh+yk+zn+|t+}w+~z+}++++++ +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH++++++++++ ++&+(+*+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,++0+2+4+6+:+<+>+@++D+F+H+HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJ++P+R+T+,X,Z,\ +,^ ,,b,f,h,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,l",n%,p(,r+,.,v1,x4,|:,=,@,C,F,I,L,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHO,R,U,[,^,a,d,g,j,m,p,s,v,|,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,,,,,,,,,,,,,,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,,,,,,,,,,,,,,,HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH,,,,,,,,,,--- -HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH -- --- - !- $- '-*--- 0-"3-$6-&HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH9-<-,B-.E-0H-K-4N-6Q-8T-:W-Z->]-!Bc-"Df-#i-HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Hl-%Jo-&Lr-'Nu-(x-)R{-*T~-,X---.\-/^-0`-1b-2-3HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHf-4h-5j-7-8p-9r-:t-;v-<-=z->|-?~-@-B-CHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH-D-E-F-G-H-I-J-K-M-N-O-P-Q-R-HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHS-T-U-V.X.Y .Z.[.\.].^._.` .a#.cHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH).d,.e/.f2.g5.h8.i;.j>.kA.lD.nJ.oM.pP.qS.rHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHV.sY.t\.u_.vb.we.yk.zn.{q.|t.}w.~z.}...HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.. +. ......... ..HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$.&.(.*....0.4..8.:.<.>..BHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH.D.F..L.N.P.R..V/X/Z/\ +/`/b/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHd/f//j/l"/n%/p(/+/v1/x4/z7/:/~=/@/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHC/F/I/L/R/U/X/[/^/a/d/g/j/m/HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHs/v/y/|////////////HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH//////////////HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH//////////////HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@blob-4096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHvalue-4096-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH0HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHgreenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHo HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +id *int328 +Cname *string08R" +lance-encoding:compressionnone +items *list08 +item *int3208 +ecategory *dict:string:int8:false08BR. +&lance-encoding:dict-values-compressionnone +Bblob * large_binary08R +lance-encoding:blobtrue +)' +% +/lance.encodings.ColumnEncoding +? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +0"20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + 0"20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +="20 +. +/lance.encodings.ArrayEncoding  + + + + ? +0"20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + ="20 +. +/lance.encodings.ArrayEncoding  + + + + ? + 0"20 +. +/lance.encodings.ArrayEncoding  + + + + > +%"20 +. +/lance.encodings.ArrayEncoding  + + + +  +)' +% +/lance.encodings.ColumnEncoding +\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ + @"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +!!@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +none\ +""@"KI +G +/lance.encodings.ArrayEncoding%2# +  + + + +@ + +noneZ +%% "JH +F +/lance.encodings.ArrayEncoding$2" +  + + + +@ + +none! +)' +% +/lance.encodings.ColumnEncoding +J +@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@J +@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@J +@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@J +"@"<: +8 +/lance.encodings.ArrayEncoding" +  + + + +@F +%":8 +6 +/lance.encodings.ArrayEncoding" +  + + + +@ +)' +% +/lance.encodings.ColumnEncoding +M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M +&"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M +)"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M ++"<: +8 +/lance.encodings.ArrayEncoding + + + M +)"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +##+"<: +8 +/lance.encodings.ArrayEncoding + + + M +#$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$+"<: +8 +/lance.encodings.ArrayEncoding + + + M +$$)"<: +8 +/lance.encodings.ArrayEncoding + + + L +%%"<: +8 +/lance.encodings.ArrayEncoding + + + L +%%"<: +8 +/lance.encodings.ArrayEncoding + + + L +%%"<: +8 +/lance.encodings.ArrayEncoding + + +  +)' +% +/lance.encodings.ColumnEncoding +l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + : :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + : :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + : :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$E E"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + $$$: :"XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ + l + %%% "XV +T +/lance.encodings.ArrayEncoding2:0 +  + + + +2 + + + + +@ +  +-+ +) +/lance.encodings.ColumnEncoding + +S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S + @"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +$@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@S +%@"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@R +%"FD +B +/lance.encodings.ArrayEncoding J +  + + + +@ +  + + + +@t 6 B +`D +QQ +a +@s Gt +5 +LANC \ No newline at end of file diff --git a/rust/lance-file/test_data/exact_versions/v2_0_mini.lance b/rust/lance-file/test_data/exact_versions/v2_0_mini.lance new file mode 100644 index 00000000000..9de22610e08 Binary files /dev/null and b/rust/lance-file/test_data/exact_versions/v2_0_mini.lance differ diff --git a/rust/lance-file/test_data/exact_versions/v2_0_self_described.lance b/rust/lance-file/test_data/exact_versions/v2_0_self_described.lance new file mode 100644 index 00000000000..13b103c7761 Binary files /dev/null and b/rust/lance-file/test_data/exact_versions/v2_0_self_described.lance differ diff --git a/rust/lance-file/test_data/exact_versions/v2_1.lance b/rust/lance-file/test_data/exact_versions/v2_1.lance new file mode 100644 index 00000000000..7ac8ae0c845 --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/v2_1.lance @@ -0,0 +1,639 @@ +blob-0000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +P" 2BRPbr  $P +( , 0 4P8<@DQH"L2PBTRQXb\r`dQhlptQx|` +`` +*`:aJaZajazb"b &b +*b .c 2c 6c:c>dB +dFdJ*dN:eRJeVZeZje^zfbfffjfngrgvgzg~0@pABC 0DpEFG0H pI$J(K,0L0pM4N8O<1P@qQDRHSL1TPqUTVXW\1X`qYdZh[l1\pq]t^x_|p@TpA#pB3pCCqDSTqEcqFsqGrHTrIrJrKsLTsMsNsOtPUtQ#tR3tSCuTSUuUcuVsuWvXUvYvZv[w\Uw]w^w_ 4t ++;K4[tk{"4&t*.246t:>B 5FuJ+N;RK5V[uZk^{b5fujnr5vuz~ !R""#2$B%RR&b'r()R*+,-R./01S2"324B5RS6b7r89S:袓;<=S>?h +h!h"*h#:i$Ji%Zi&ji'zj(j)j*j+k,k-k.k/l0 +l1l2*l3:m4Jm5Zm6jm7zn8n9n:n;oo?2`rabc2drefg2hrijk2lrmno3psqijrs3tsuԳvw3xsyz{3|s}~x`Vxa#xb3xcCydSVyecyfsygzhVzizjzk{lV{m{n{o|pW|q#|r3|sC}tSW}uc}vs}w~xW~y~z~{|W}~ 6v+;K6[vk{6v6v 7w+;K7[wk{7w꫷7wHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0001-deterministic-fixturevalue-0002-deterministic-fixturevalue-0003-deterministic-fixturevalue-0004-deterministic-fixturevalue-0005-deterministic-fixturevalue-0006-deterministic-fixturevalue-0008-deterministic-fixturevalue-0009-deterministic-fixturevalue-0010-deterministic-fixturevalue-0011-deterministic-fixturevalue-0012-deterministic-fixturevalue-0013-deterministic-fixturevalue-0015-deterministic-fixturevalue-0016-deterministic-fixturevalue-0017-deterministic-fixturevalue-0018-deterministic-fixturevalue-0019-deterministic-fixturevalue-0020-deterministic-fixturevalue-0022-deterministic-fixturevalue-0023-deterministic-fixturevalue-0024-deterministic-fixturevalue-0025-deterministic-fixturevalue-0026-deterministic-fixturevalue-0027-deterministic-fixturevalue-0029-deterministic-fixturevalue-0030-deterministic-fixturevalue-0031-deterministic-fixturevalue-0032-deterministic-fixturevalue-0033-deterministic-fixturevalue-0034-deterministic-fixturevalue-0036-deterministic-fixturevalue-0037-deterministic-fixturevalue-0038-deterministic-fixturevalue-0039-deterministic-fixturevalue-0040-deterministic-fixturevalue-0041-deterministic-fixturevalue-0043-deterministic-fixturevalue-0044-deterministic-fixturevalue-0045-deterministic-fixturevalue-0046-deterministic-fixturevalue-0047-deterministic-fixturevalue-0048-deterministic-fixturevalue-0050-deterministic-fixturevalue-0051-deterministic-fixturevalue-0052-deterministic-fixturevalue-0053-deterministic-fixturevalue-0054-deterministic-fixturevalue-0055-deterministic-fixturevalue-0057-deterministic-fixturevalue-0058-deterministic-fixturevalue-0059-deterministic-fixturevalue-0060-deterministic-fixturevalue-0061-deterministic-fixturevalue-0062-deterministic-fixturevalue-0064-deterministic-fixturevalue-0065-deterministic-fixturevalue-0066-deterministic-fixturevalue-0067-deterministic-fixturevalue-0068-deterministic-fixturevalue-0069-deterministic-fixturevalue-0071-deterministic-fixturevalue-0072-deterministic-fixturevalue-0073-deterministic-fixturevalue-0074-deterministic-fixturevalue-0075-deterministic-fixturevalue-0076-deterministic-fixturevalue-0078-deterministic-fixturevalue-0079-deterministic-fixturevalue-0080-deterministic-fixturevalue-0081-deterministic-fixturevalue-0082-deterministic-fixturevalue-0083-deterministic-fixturevalue-0085-deterministic-fixturevalue-0086-deterministic-fixturevalue-0087-deterministic-fixturevalue-0088-deterministic-fixturevalue-0089-deterministic-fixturevalue-0090-deterministic-fixturevalue-0092-deterministic-fixturevalue-0093-deterministic-fixturevalue-0094-deterministic-fixturevalue-0095-deterministic-fixturevalue-0096-deterministic-fixturevalue-0097-deterministic-fixturevalue-0099-deterministic-fixturevalue-0100-deterministic-fixturevalue-0101-deterministic-fixturevalue-0102-deterministic-fixturevalue-0103-deterministic-fixturevalue-0104-deterministic-fixturevalue-0106-deterministic-fixturevalue-0107-deterministic-fixturevalue-0108-deterministic-fixturevalue-0109-deterministic-fixturevalue-0110-deterministic-fixturevalue-0111-deterministic-fixturevalue-0113-deterministic-fixturevalue-0114-deterministic-fixturevalue-0115-deterministic-fixturevalue-0116-deterministic-fixturevalue-0117-deterministic-fixturevalue-0118-deterministic-fixturevalue-0120-deterministic-fixturevalue-0121-deterministic-fixturevalue-0122-deterministic-fixturevalue-0123-deterministic-fixturevalue-0124-deterministic-fixturevalue-0125-deterministic-fixturevalue-0127-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-0128-deterministic-fixturevalue-0129-deterministic-fixturevalue-0130-deterministic-fixturevalue-0131-deterministic-fixturevalue-0132-deterministic-fixturevalue-0134-deterministic-fixturevalue-0135-deterministic-fixturevalue-0136-deterministic-fixturevalue-0137-deterministic-fixturevalue-0138-deterministic-fixturevalue-0139-deterministic-fixturevalue-0141-deterministic-fixturevalue-0142-deterministic-fixturevalue-0143-deterministic-fixturevalue-0144-deterministic-fixturevalue-0145-deterministic-fixturevalue-0146-deterministic-fixturevalue-0148-deterministic-fixturevalue-0149-deterministic-fixturevalue-0150-deterministic-fixturevalue-0151-deterministic-fixturevalue-0152-deterministic-fixturevalue-0153-deterministic-fixturevalue-0155-deterministic-fixturevalue-0156-deterministic-fixturevalue-0157-deterministic-fixturevalue-0158-deterministic-fixturevalue-0159-deterministic-fixturevalue-0160-deterministic-fixturevalue-0162-deterministic-fixturevalue-0163-deterministic-fixturevalue-0164-deterministic-fixturevalue-0165-deterministic-fixturevalue-0166-deterministic-fixturevalue-0167-deterministic-fixturevalue-0169-deterministic-fixturevalue-0170-deterministic-fixturevalue-0171-deterministic-fixturevalue-0172-deterministic-fixturevalue-0173-deterministic-fixturevalue-0174-deterministic-fixturevalue-0176-deterministic-fixturevalue-0177-deterministic-fixturevalue-0178-deterministic-fixturevalue-0179-deterministic-fixturevalue-0180-deterministic-fixturevalue-0181-deterministic-fixturevalue-0183-deterministic-fixturevalue-0184-deterministic-fixturevalue-0185-deterministic-fixturevalue-0186-deterministic-fixturevalue-0187-deterministic-fixturevalue-0188-deterministic-fixturevalue-0190-deterministic-fixturevalue-0191-deterministic-fixturevalue-0192-deterministic-fixturevalue-0193-deterministic-fixturevalue-0194-deterministic-fixturevalue-0195-deterministic-fixturevalue-0197-deterministic-fixturevalue-0198-deterministic-fixturevalue-0199-deterministic-fixturevalue-0200-deterministic-fixturevalue-0201-deterministic-fixturevalue-0202-deterministic-fixturevalue-0204-deterministic-fixturevalue-0205-deterministic-fixturevalue-0206-deterministic-fixturevalue-0207-deterministic-fixturevalue-0208-deterministic-fixturevalue-0209-deterministic-fixturevalue-0211-deterministic-fixturevalue-0212-deterministic-fixturevalue-0213-deterministic-fixturevalue-0214-deterministic-fixturevalue-0215-deterministic-fixturevalue-0216-deterministic-fixturevalue-0218-deterministic-fixturevalue-0219-deterministic-fixturevalue-0220-deterministic-fixturevalue-0221-deterministic-fixturevalue-0222-deterministic-fixturevalue-0223-deterministic-fixturevalue-0225-deterministic-fixturevalue-0226-deterministic-fixturevalue-0227-deterministic-fixturevalue-0228-deterministic-fixturevalue-0229-deterministic-fixturevalue-0230-deterministic-fixturevalue-0232-deterministic-fixturevalue-0233-deterministic-fixturevalue-0234-deterministic-fixturevalue-0235-deterministic-fixturevalue-0236-deterministic-fixturevalue-0237-deterministic-fixturevalue-0239-deterministic-fixturevalue-0240-deterministic-fixturevalue-0241-deterministic-fixturevalue-0242-deterministic-fixturevalue-0243-deterministic-fixturevalue-0244-deterministic-fixturevalue-0246-deterministic-fixturevalue-0247-deterministic-fixturevalue-0248-deterministic-fixturevalue-0249-deterministic-fixturevalue-0250-deterministic-fixturevalue-0251-deterministic-fixturevalue-0253-deterministic-fixturevalue-0254-deterministic-fixturevalue-0255-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-0256-deterministic-fixturevalue-0257-deterministic-fixturevalue-0258-deterministic-fixturevalue-0260-deterministic-fixturevalue-0261-deterministic-fixturevalue-0262-deterministic-fixturevalue-0263-deterministic-fixturevalue-0264-deterministic-fixturevalue-0265-deterministic-fixturevalue-0267-deterministic-fixturevalue-0268-deterministic-fixturevalue-0269-deterministic-fixturevalue-0270-deterministic-fixturevalue-0271-deterministic-fixturevalue-0272-deterministic-fixturevalue-0274-deterministic-fixturevalue-0275-deterministic-fixturevalue-0276-deterministic-fixturevalue-0277-deterministic-fixturevalue-0278-deterministic-fixturevalue-0279-deterministic-fixturevalue-0281-deterministic-fixturevalue-0282-deterministic-fixturevalue-0283-deterministic-fixturevalue-0284-deterministic-fixturevalue-0285-deterministic-fixturevalue-0286-deterministic-fixturevalue-0288-deterministic-fixturevalue-0289-deterministic-fixturevalue-0290-deterministic-fixturevalue-0291-deterministic-fixturevalue-0292-deterministic-fixturevalue-0293-deterministic-fixturevalue-0295-deterministic-fixturevalue-0296-deterministic-fixturevalue-0297-deterministic-fixturevalue-0298-deterministic-fixturevalue-0299-deterministic-fixturevalue-0300-deterministic-fixturevalue-0302-deterministic-fixturevalue-0303-deterministic-fixturevalue-0304-deterministic-fixturevalue-0305-deterministic-fixturevalue-0306-deterministic-fixturevalue-0307-deterministic-fixturevalue-0309-deterministic-fixturevalue-0310-deterministic-fixturevalue-0311-deterministic-fixturevalue-0312-deterministic-fixturevalue-0313-deterministic-fixturevalue-0314-deterministic-fixturevalue-0316-deterministic-fixturevalue-0317-deterministic-fixturevalue-0318-deterministic-fixturevalue-0319-deterministic-fixturevalue-0320-deterministic-fixturevalue-0321-deterministic-fixturevalue-0323-deterministic-fixturevalue-0324-deterministic-fixturevalue-0325-deterministic-fixturevalue-0326-deterministic-fixturevalue-0327-deterministic-fixturevalue-0328-deterministic-fixturevalue-0330-deterministic-fixturevalue-0331-deterministic-fixturevalue-0332-deterministic-fixturevalue-0333-deterministic-fixturevalue-0334-deterministic-fixturevalue-0335-deterministic-fixturevalue-0337-deterministic-fixturevalue-0338-deterministic-fixturevalue-0339-deterministic-fixturevalue-0340-deterministic-fixturevalue-0341-deterministic-fixturevalue-0342-deterministic-fixturevalue-0344-deterministic-fixturevalue-0345-deterministic-fixturevalue-0346-deterministic-fixturevalue-0347-deterministic-fixturevalue-0348-deterministic-fixturevalue-0349-deterministic-fixturevalue-0351-deterministic-fixturevalue-0352-deterministic-fixturevalue-0353-deterministic-fixturevalue-0354-deterministic-fixturevalue-0355-deterministic-fixturevalue-0356-deterministic-fixturevalue-0358-deterministic-fixturevalue-0359-deterministic-fixturevalue-0360-deterministic-fixturevalue-0361-deterministic-fixturevalue-0362-deterministic-fixturevalue-0363-deterministic-fixturevalue-0365-deterministic-fixturevalue-0366-deterministic-fixturevalue-0367-deterministic-fixturevalue-0368-deterministic-fixturevalue-0369-deterministic-fixturevalue-0370-deterministic-fixturevalue-0372-deterministic-fixturevalue-0373-deterministic-fixturevalue-0374-deterministic-fixturevalue-0375-deterministic-fixturevalue-0376-deterministic-fixturevalue-0377-deterministic-fixturevalue-0379-deterministic-fixturevalue-0380-deterministic-fixturevalue-0381-deterministic-fixturevalue-0382-deterministic-fixturevalue-0383-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-0384-deterministic-fixturevalue-0386-deterministic-fixturevalue-0387-deterministic-fixturevalue-0388-deterministic-fixturevalue-0389-deterministic-fixturevalue-0390-deterministic-fixturevalue-0391-deterministic-fixturevalue-0393-deterministic-fixturevalue-0394-deterministic-fixturevalue-0395-deterministic-fixturevalue-0396-deterministic-fixturevalue-0397-deterministic-fixturevalue-0398-deterministic-fixturevalue-0400-deterministic-fixturevalue-0401-deterministic-fixturevalue-0402-deterministic-fixturevalue-0403-deterministic-fixturevalue-0404-deterministic-fixturevalue-0405-deterministic-fixturevalue-0407-deterministic-fixturevalue-0408-deterministic-fixturevalue-0409-deterministic-fixturevalue-0410-deterministic-fixturevalue-0411-deterministic-fixturevalue-0412-deterministic-fixturevalue-0414-deterministic-fixturevalue-0415-deterministic-fixturevalue-0416-deterministic-fixturevalue-0417-deterministic-fixturevalue-0418-deterministic-fixturevalue-0419-deterministic-fixturevalue-0421-deterministic-fixturevalue-0422-deterministic-fixturevalue-0423-deterministic-fixturevalue-0424-deterministic-fixturevalue-0425-deterministic-fixturevalue-0426-deterministic-fixturevalue-0428-deterministic-fixturevalue-0429-deterministic-fixturevalue-0430-deterministic-fixturevalue-0431-deterministic-fixturevalue-0432-deterministic-fixturevalue-0433-deterministic-fixturevalue-0435-deterministic-fixturevalue-0436-deterministic-fixturevalue-0437-deterministic-fixturevalue-0438-deterministic-fixturevalue-0439-deterministic-fixturevalue-0440-deterministic-fixturevalue-0442-deterministic-fixturevalue-0443-deterministic-fixturevalue-0444-deterministic-fixturevalue-0445-deterministic-fixturevalue-0446-deterministic-fixturevalue-0447-deterministic-fixturevalue-0449-deterministic-fixturevalue-0450-deterministic-fixturevalue-0451-deterministic-fixturevalue-0452-deterministic-fixturevalue-0453-deterministic-fixturevalue-0454-deterministic-fixturevalue-0456-deterministic-fixturevalue-0457-deterministic-fixturevalue-0458-deterministic-fixturevalue-0459-deterministic-fixturevalue-0460-deterministic-fixturevalue-0461-deterministic-fixturevalue-0463-deterministic-fixturevalue-0464-deterministic-fixturevalue-0465-deterministic-fixturevalue-0466-deterministic-fixturevalue-0467-deterministic-fixturevalue-0468-deterministic-fixturevalue-0470-deterministic-fixturevalue-0471-deterministic-fixturevalue-0472-deterministic-fixturevalue-0473-deterministic-fixturevalue-0474-deterministic-fixturevalue-0475-deterministic-fixturevalue-0477-deterministic-fixturevalue-0478-deterministic-fixturevalue-0479-deterministic-fixturevalue-0480-deterministic-fixturevalue-0481-deterministic-fixturevalue-0482-deterministic-fixturevalue-0484-deterministic-fixturevalue-0485-deterministic-fixturevalue-0486-deterministic-fixturevalue-0487-deterministic-fixturevalue-0488-deterministic-fixturevalue-0489-deterministic-fixturevalue-0491-deterministic-fixturevalue-0492-deterministic-fixturevalue-0493-deterministic-fixturevalue-0494-deterministic-fixturevalue-0495-deterministic-fixturevalue-0496-deterministic-fixturevalue-0498-deterministic-fixturevalue-0499-deterministic-fixturevalue-0500-deterministic-fixturevalue-0501-deterministic-fixturevalue-0502-deterministic-fixturevalue-0503-deterministic-fixturevalue-0505-deterministic-fixturevalue-0506-deterministic-fixturevalue-0507-deterministic-fixturevalue-0508-deterministic-fixturevalue-0509-deterministic-fixturevalue-0510-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0512-deterministic-fixturevalue-0513-deterministic-fixturevalue-0514-deterministic-fixturevalue-0515-deterministic-fixturevalue-0516-deterministic-fixturevalue-0517-deterministic-fixturevalue-0519-deterministic-fixturevalue-0520-deterministic-fixturevalue-0521-deterministic-fixturevalue-0522-deterministic-fixturevalue-0523-deterministic-fixturevalue-0524-deterministic-fixturevalue-0526-deterministic-fixturevalue-0527-deterministic-fixturevalue-0528-deterministic-fixturevalue-0529-deterministic-fixturevalue-0530-deterministic-fixturevalue-0531-deterministic-fixturevalue-0533-deterministic-fixturevalue-0534-deterministic-fixturevalue-0535-deterministic-fixturevalue-0536-deterministic-fixturevalue-0537-deterministic-fixturevalue-0538-deterministic-fixturevalue-0540-deterministic-fixturevalue-0541-deterministic-fixturevalue-0542-deterministic-fixturevalue-0543-deterministic-fixturevalue-0544-deterministic-fixturevalue-0545-deterministic-fixturevalue-0547-deterministic-fixturevalue-0548-deterministic-fixturevalue-0549-deterministic-fixturevalue-0550-deterministic-fixturevalue-0551-deterministic-fixturevalue-0552-deterministic-fixturevalue-0554-deterministic-fixturevalue-0555-deterministic-fixturevalue-0556-deterministic-fixturevalue-0557-deterministic-fixturevalue-0558-deterministic-fixturevalue-0559-deterministic-fixturevalue-0561-deterministic-fixturevalue-0562-deterministic-fixturevalue-0563-deterministic-fixturevalue-0564-deterministic-fixturevalue-0565-deterministic-fixturevalue-0566-deterministic-fixturevalue-0568-deterministic-fixturevalue-0569-deterministic-fixturevalue-0570-deterministic-fixturevalue-0571-deterministic-fixturevalue-0572-deterministic-fixturevalue-0573-deterministic-fixturevalue-0575-deterministic-fixturevalue-0576-deterministic-fixturevalue-0577-deterministic-fixturevalue-0578-deterministic-fixturevalue-0579-deterministic-fixturevalue-0580-deterministic-fixturevalue-0582-deterministic-fixturevalue-0583-deterministic-fixturevalue-0584-deterministic-fixturevalue-0585-deterministic-fixturevalue-0586-deterministic-fixturevalue-0587-deterministic-fixturevalue-0589-deterministic-fixturevalue-0590-deterministic-fixturevalue-0591-deterministic-fixturevalue-0592-deterministic-fixturevalue-0593-deterministic-fixturevalue-0594-deterministic-fixturevalue-0596-deterministic-fixturevalue-0597-deterministic-fixturevalue-0598-deterministic-fixturevalue-0599-deterministic-fixturevalue-0600-deterministic-fixturevalue-0601-deterministic-fixturevalue-0603-deterministic-fixturevalue-0604-deterministic-fixturevalue-0605-deterministic-fixturevalue-0606-deterministic-fixturevalue-0607-deterministic-fixturevalue-0608-deterministic-fixturevalue-0610-deterministic-fixturevalue-0611-deterministic-fixturevalue-0612-deterministic-fixturevalue-0613-deterministic-fixturevalue-0614-deterministic-fixturevalue-0615-deterministic-fixturevalue-0617-deterministic-fixturevalue-0618-deterministic-fixturevalue-0619-deterministic-fixturevalue-0620-deterministic-fixturevalue-0621-deterministic-fixturevalue-0622-deterministic-fixturevalue-0624-deterministic-fixturevalue-0625-deterministic-fixturevalue-0626-deterministic-fixturevalue-0627-deterministic-fixturevalue-0628-deterministic-fixturevalue-0629-deterministic-fixturevalue-0631-deterministic-fixturevalue-0632-deterministic-fixturevalue-0633-deterministic-fixturevalue-0634-deterministic-fixturevalue-0635-deterministic-fixturevalue-0636-deterministic-fixturevalue-0638-deterministic-fixturevalue-0639-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-0640-deterministic-fixturevalue-0641-deterministic-fixturevalue-0642-deterministic-fixturevalue-0643-deterministic-fixturevalue-0645-deterministic-fixturevalue-0646-deterministic-fixturevalue-0647-deterministic-fixturevalue-0648-deterministic-fixturevalue-0649-deterministic-fixturevalue-0650-deterministic-fixturevalue-0652-deterministic-fixturevalue-0653-deterministic-fixturevalue-0654-deterministic-fixturevalue-0655-deterministic-fixturevalue-0656-deterministic-fixturevalue-0657-deterministic-fixturevalue-0659-deterministic-fixturevalue-0660-deterministic-fixturevalue-0661-deterministic-fixturevalue-0662-deterministic-fixturevalue-0663-deterministic-fixturevalue-0664-deterministic-fixturevalue-0666-deterministic-fixturevalue-0667-deterministic-fixturevalue-0668-deterministic-fixturevalue-0669-deterministic-fixturevalue-0670-deterministic-fixturevalue-0671-deterministic-fixturevalue-0673-deterministic-fixturevalue-0674-deterministic-fixturevalue-0675-deterministic-fixturevalue-0676-deterministic-fixturevalue-0677-deterministic-fixturevalue-0678-deterministic-fixturevalue-0680-deterministic-fixturevalue-0681-deterministic-fixturevalue-0682-deterministic-fixturevalue-0683-deterministic-fixturevalue-0684-deterministic-fixturevalue-0685-deterministic-fixturevalue-0687-deterministic-fixturevalue-0688-deterministic-fixturevalue-0689-deterministic-fixturevalue-0690-deterministic-fixturevalue-0691-deterministic-fixturevalue-0692-deterministic-fixturevalue-0694-deterministic-fixturevalue-0695-deterministic-fixturevalue-0696-deterministic-fixturevalue-0697-deterministic-fixturevalue-0698-deterministic-fixturevalue-0699-deterministic-fixturevalue-0701-deterministic-fixturevalue-0702-deterministic-fixturevalue-0703-deterministic-fixturevalue-0704-deterministic-fixturevalue-0705-deterministic-fixturevalue-0706-deterministic-fixturevalue-0708-deterministic-fixturevalue-0709-deterministic-fixturevalue-0710-deterministic-fixturevalue-0711-deterministic-fixturevalue-0712-deterministic-fixturevalue-0713-deterministic-fixturevalue-0715-deterministic-fixturevalue-0716-deterministic-fixturevalue-0717-deterministic-fixturevalue-0718-deterministic-fixturevalue-0719-deterministic-fixturevalue-0720-deterministic-fixturevalue-0722-deterministic-fixturevalue-0723-deterministic-fixturevalue-0724-deterministic-fixturevalue-0725-deterministic-fixturevalue-0726-deterministic-fixturevalue-0727-deterministic-fixturevalue-0729-deterministic-fixturevalue-0730-deterministic-fixturevalue-0731-deterministic-fixturevalue-0732-deterministic-fixturevalue-0733-deterministic-fixturevalue-0734-deterministic-fixturevalue-0736-deterministic-fixturevalue-0737-deterministic-fixturevalue-0738-deterministic-fixturevalue-0739-deterministic-fixturevalue-0740-deterministic-fixturevalue-0741-deterministic-fixturevalue-0743-deterministic-fixturevalue-0744-deterministic-fixturevalue-0745-deterministic-fixturevalue-0746-deterministic-fixturevalue-0747-deterministic-fixturevalue-0748-deterministic-fixturevalue-0750-deterministic-fixturevalue-0751-deterministic-fixturevalue-0752-deterministic-fixturevalue-0753-deterministic-fixturevalue-0754-deterministic-fixturevalue-0755-deterministic-fixturevalue-0757-deterministic-fixturevalue-0758-deterministic-fixturevalue-0759-deterministic-fixturevalue-0760-deterministic-fixturevalue-0761-deterministic-fixturevalue-0762-deterministic-fixturevalue-0764-deterministic-fixturevalue-0765-deterministic-fixturevalue-0766-deterministic-fixturevalue-0767-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-0768-deterministic-fixturevalue-0769-deterministic-fixturevalue-0771-deterministic-fixturevalue-0772-deterministic-fixturevalue-0773-deterministic-fixturevalue-0774-deterministic-fixturevalue-0775-deterministic-fixturevalue-0776-deterministic-fixturevalue-0778-deterministic-fixturevalue-0779-deterministic-fixturevalue-0780-deterministic-fixturevalue-0781-deterministic-fixturevalue-0782-deterministic-fixturevalue-0783-deterministic-fixturevalue-0785-deterministic-fixturevalue-0786-deterministic-fixturevalue-0787-deterministic-fixturevalue-0788-deterministic-fixturevalue-0789-deterministic-fixturevalue-0790-deterministic-fixturevalue-0792-deterministic-fixturevalue-0793-deterministic-fixturevalue-0794-deterministic-fixturevalue-0795-deterministic-fixturevalue-0796-deterministic-fixturevalue-0797-deterministic-fixturevalue-0799-deterministic-fixturevalue-0800-deterministic-fixturevalue-0801-deterministic-fixturevalue-0802-deterministic-fixturevalue-0803-deterministic-fixturevalue-0804-deterministic-fixturevalue-0806-deterministic-fixturevalue-0807-deterministic-fixturevalue-0808-deterministic-fixturevalue-0809-deterministic-fixturevalue-0810-deterministic-fixturevalue-0811-deterministic-fixturevalue-0813-deterministic-fixturevalue-0814-deterministic-fixturevalue-0815-deterministic-fixturevalue-0816-deterministic-fixturevalue-0817-deterministic-fixturevalue-0818-deterministic-fixturevalue-0820-deterministic-fixturevalue-0821-deterministic-fixturevalue-0822-deterministic-fixturevalue-0823-deterministic-fixturevalue-0824-deterministic-fixturevalue-0825-deterministic-fixturevalue-0827-deterministic-fixturevalue-0828-deterministic-fixturevalue-0829-deterministic-fixturevalue-0830-deterministic-fixturevalue-0831-deterministic-fixturevalue-0832-deterministic-fixturevalue-0834-deterministic-fixturevalue-0835-deterministic-fixturevalue-0836-deterministic-fixturevalue-0837-deterministic-fixturevalue-0838-deterministic-fixturevalue-0839-deterministic-fixturevalue-0841-deterministic-fixturevalue-0842-deterministic-fixturevalue-0843-deterministic-fixturevalue-0844-deterministic-fixturevalue-0845-deterministic-fixturevalue-0846-deterministic-fixturevalue-0848-deterministic-fixturevalue-0849-deterministic-fixturevalue-0850-deterministic-fixturevalue-0851-deterministic-fixturevalue-0852-deterministic-fixturevalue-0853-deterministic-fixturevalue-0855-deterministic-fixturevalue-0856-deterministic-fixturevalue-0857-deterministic-fixturevalue-0858-deterministic-fixturevalue-0859-deterministic-fixturevalue-0860-deterministic-fixturevalue-0862-deterministic-fixturevalue-0863-deterministic-fixturevalue-0864-deterministic-fixturevalue-0865-deterministic-fixturevalue-0866-deterministic-fixturevalue-0867-deterministic-fixturevalue-0869-deterministic-fixturevalue-0870-deterministic-fixturevalue-0871-deterministic-fixturevalue-0872-deterministic-fixturevalue-0873-deterministic-fixturevalue-0874-deterministic-fixturevalue-0876-deterministic-fixturevalue-0877-deterministic-fixturevalue-0878-deterministic-fixturevalue-0879-deterministic-fixturevalue-0880-deterministic-fixturevalue-0881-deterministic-fixturevalue-0883-deterministic-fixturevalue-0884-deterministic-fixturevalue-0885-deterministic-fixturevalue-0886-deterministic-fixturevalue-0887-deterministic-fixturevalue-0888-deterministic-fixturevalue-0890-deterministic-fixturevalue-0891-deterministic-fixturevalue-0892-deterministic-fixturevalue-0893-deterministic-fixturevalue-0894-deterministic-fixturevalue-0895-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0897-deterministic-fixturevalue-0898-deterministic-fixturevalue-0899-deterministic-fixturevalue-0900-deterministic-fixturevalue-0901-deterministic-fixturevalue-0902-deterministic-fixturevalue-0904-deterministic-fixturevalue-0905-deterministic-fixturevalue-0906-deterministic-fixturevalue-0907-deterministic-fixturevalue-0908-deterministic-fixturevalue-0909-deterministic-fixturevalue-0911-deterministic-fixturevalue-0912-deterministic-fixturevalue-0913-deterministic-fixturevalue-0914-deterministic-fixturevalue-0915-deterministic-fixturevalue-0916-deterministic-fixturevalue-0918-deterministic-fixturevalue-0919-deterministic-fixturevalue-0920-deterministic-fixturevalue-0921-deterministic-fixturevalue-0922-deterministic-fixturevalue-0923-deterministic-fixturevalue-0925-deterministic-fixturevalue-0926-deterministic-fixturevalue-0927-deterministic-fixturevalue-0928-deterministic-fixturevalue-0929-deterministic-fixturevalue-0930-deterministic-fixturevalue-0932-deterministic-fixturevalue-0933-deterministic-fixturevalue-0934-deterministic-fixturevalue-0935-deterministic-fixturevalue-0936-deterministic-fixturevalue-0937-deterministic-fixturevalue-0939-deterministic-fixturevalue-0940-deterministic-fixturevalue-0941-deterministic-fixturevalue-0942-deterministic-fixturevalue-0943-deterministic-fixturevalue-0944-deterministic-fixturevalue-0946-deterministic-fixturevalue-0947-deterministic-fixturevalue-0948-deterministic-fixturevalue-0949-deterministic-fixturevalue-0950-deterministic-fixturevalue-0951-deterministic-fixturevalue-0953-deterministic-fixturevalue-0954-deterministic-fixturevalue-0955-deterministic-fixturevalue-0956-deterministic-fixturevalue-0957-deterministic-fixturevalue-0958-deterministic-fixturevalue-0960-deterministic-fixturevalue-0961-deterministic-fixturevalue-0962-deterministic-fixturevalue-0963-deterministic-fixturevalue-0964-deterministic-fixturevalue-0965-deterministic-fixturevalue-0967-deterministic-fixturevalue-0968-deterministic-fixturevalue-0969-deterministic-fixturevalue-0970-deterministic-fixturevalue-0971-deterministic-fixturevalue-0972-deterministic-fixturevalue-0974-deterministic-fixturevalue-0975-deterministic-fixturevalue-0976-deterministic-fixturevalue-0977-deterministic-fixturevalue-0978-deterministic-fixturevalue-0979-deterministic-fixturevalue-0981-deterministic-fixturevalue-0982-deterministic-fixturevalue-0983-deterministic-fixturevalue-0984-deterministic-fixturevalue-0985-deterministic-fixturevalue-0986-deterministic-fixturevalue-0988-deterministic-fixturevalue-0989-deterministic-fixturevalue-0990-deterministic-fixturevalue-0991-deterministic-fixturevalue-0992-deterministic-fixturevalue-0993-deterministic-fixturevalue-0995-deterministic-fixturevalue-0996-deterministic-fixturevalue-0997-deterministic-fixturevalue-0998-deterministic-fixturevalue-0999-deterministic-fixturevalue-1000-deterministic-fixturevalue-1002-deterministic-fixturevalue-1003-deterministic-fixturevalue-1004-deterministic-fixturevalue-1005-deterministic-fixturevalue-1006-deterministic-fixturevalue-1007-deterministic-fixturevalue-1009-deterministic-fixturevalue-1010-deterministic-fixturevalue-1011-deterministic-fixturevalue-1012-deterministic-fixturevalue-1013-deterministic-fixturevalue-1014-deterministic-fixturevalue-1016-deterministic-fixturevalue-1017-deterministic-fixturevalue-1018-deterministic-fixturevalue-1019-deterministic-fixturevalue-1020-deterministic-fixturevalue-1021-deterministic-fixturevalue-1023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH::HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH#F$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I@ @@ @!@@@ @@ @!@@ H@ @@ h/FAG0H @0H 01I@ KPC2K`3L @3AM +X4NC pNA#4ҋ`Nvq#8 bTv #@c`w"# efw$$Hfl#x&1$L" hrCx(a$P2ixy*$TBl~z,$P nz.$\ro{0Q%` qh{0()H8MRld{h)KDRp$|4)NP St|6)Q\MS||8)TtSD}:)W } *Z T~> *]MT d~@I*`T"~D!i*fT#$)*i8Wpب;L@X;NHBYPPZ=RX[=T`B\ >Vp]DPXx]h?Z^?\B_ @^` E B L'RX'b Úd!(r|Q(ق (ݐ#(ࢍD( + d) ҍѤq) ěġ)  ,$NX dI,'XĄi,*$,-YЄ ,0NY",3YD$,6Y& -90H )-<@NZ`d*(@-?PZxć.0-E[52Od#56APC68QpCc6:Qt7qT|@Æ8@T9BтUDBV":FaW2@c:H{ %r1|ȍ%D t|%uA } & wQ)} &dxYI~& z`&$ }i&~q(N' yɀ4'D@'Ť$H*l0U%J*o@U&LB+r@V'N+u`(+xpV)(RB+{V*XT+~@W+pV+W-+.\B+W/^@b)` &d*a ,Af1*b"!2DBha*c0!8dCj*dB!>l*eR"DDn*fb"JDp!+f"PEr+g"V$Et+h#\Dx+i􄈕-䈖-[E-[ +. \ *.O\ eJ.$Ŋ!j.0\%.<]1.HO]勤9.T!A +/"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$d @@ @H@ @@ @!@@@ @@ @!@@  rh4yQOk5zOn{OqD5|Otd5}Pw5~APzP}5ހP6߂Q$61Q#D6@4b`h4` f@5!a@s @6QalT 7and %A8"apt EA9Rar &eA:b FA;Abv f=qbz A>Bb| A?rbr",&L2r$%,&GXLbr&5,&gXLr*E,&XL„s,U,&§Xs.e,&M"s0u,&XMRs,&YMt4,&'Yt6,&GYMt8,&ԇY7퐡Sǰ7S"ʳ8T$ж$81T&#ӹaT(Cּ8TPٿ8T,c8!U0sQU29U$9UAe!FCLqe!CMe!CNBe"DOrf"P1f#"EDQaf0"&eDS2f C"FDTbf4S"fDUfDs"DVQgT"'[OvR-''[wV-'OwX-'h[Ow-'[OBw\-( +[rx^-( [Px`-(P҉xb.(\Px .(H\Pbxf5.(h\aQ3⅑Q +C6QP6Qc6!Rs6QR$7花RD7Rd7S7ASqS!%B@1c#!Aac3!eBBc@!BC2c$S!FBDbc@c!̆BE!dTs!ϦFQdd!CH"dt!%CIRd!ECJd!&KMBt:,&קYMrt,'YNu@,'Y҆uB-'NuD-''ZN2u -'GZNbuH5-'gZvJE-'ZN‡vLU-'OvNu-'ZORv-6D9U8d9V:AV<ڤ9qV9V@:WB1WF#D:aW0d:WJC:WLS :EWgd"%EX"gp"EEYRg" Zh"E[Ah"gE\qh"E^Bh"E_rh#`i$# FFabi0##fFbixhE.(PŠylU.("\Py`.(%\Q"ypu.((\Ryr.(+]Qzt.(.Qzv.(1H]Qz.(4h]Qzz.(7]rz|.):R{~.)=]$ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ @ @@ H@ @@@ @ cM1cNҕ2P2(dQ2 2HdRbE2܈dS–U2Te2dU"p2dVR2eW2(eXCV7Kos7koZ7 Cӧ\8 + ps^8 )p3`&8 Ipc68+ip3f'F8K cèh7V8 pjGp8pSP8"={={={={={>|>|>|>>|>|`3,ff#v32fg S35 gh +38Igi 3;!igj 3>$gkC3A*gls3D- m 3G0gnӛ4J3 ho$4MF9F+r3ë'P9I rc7f9Lkrœ#Pv9OrSg9R +#w9U *sS9[ Jsʃ9^+js˳9aK +s9dks9g sC~??~?~?~??????2Y◲2 he[2e\B2e]r2^Ә3 f_3)f`3&3 Ifac63# ifb +F3& cÙ V3)fd +qng8% +#pw8(JqSr8+ jqt8.KqC81kqsx84 +C|8: qsӪ~9= +r&9@Jrc69C +>|>|>>}>}>}>>}>}?~?6)hq3&&4S9Ihrc(64V< s*V4Y?ht`4\Bhu#.v4_EhvS24bK w44eN)ix4hQIiy84kTiiz:4nWi|Cӭ:j +ts:m*tУ3&:p +c6:s jt@:v tîV:| tf: t#: +u: *u: Ju: ??HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHxwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@ @@ @@ @@ @@ aaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $@ d@@@$$A,dA4A?%HA eHCHE+HG;%IIKeIK[IMkIO{%JQeJSJUJW%KYeK[ۥK]K_%La eLcLe+Lg;%MiKeMk[MmkMo{%NqeNsNuNw%OyeO{ۥO}ObPPQ6Q6$mD64mɣFR6DmHS6S6+GPre+/@Gr+kGr%,/Gs,˹GsE-/Gs-Gt.+GPte.KH. +k Ht %/0Hu `e: +#0(ve< S0)e$= 0*e*? 0%+e6@I0)e:pWz@pW{A qW'{C cqW7{Dk:“qWG{F:qXW{Iqe//m"/q(/y4/:/@/F/L/R/ G .U +gN ǵn! ''*U-G 0"3X'\K*;cX0^k2ʓXG_XWaBXgb J;#Xd R̃Xe+Z;ͳYgKYj jYǸkr;C(Yt|/t|/t|/t|/t|/u|/u|/u|/u|/v|/v|/-TfMm&F.Ug@ n ' +XaJ XiLҺ8XlM:DXqOPXuP\XyRhX|S:tX́UX̅V +X̉YX͍[g{X/#rw{^/ Sr{d/ r{j/+r{v/Kr{|/ks{/ss{/s{/s|/ t|/ 36&9(<*?,B0H2K4N6QT @YUŝgNLYǸXY̝;dYU繢pY՝G;|YYݝ.Yg;NYǻʻ`Y'msv|nУv|pv}q w}s+3w0}tkғwG}vwW}w wg}z#xp}|Sx}} Ճx///HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@aaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH (8HXhx + + Ȉ ؈ (8HXhxȉ؉ +)*9:IJYZijyz + ɘ ٘   +)*9:IJYZijyzəٙ +;K[k{ + ˸ ۸븨 +;K[k{˹۹빩@@AAB(BC8CDHDEXEFhFGxGHHIIJJKKLȌLM،MNNOOPPQQR(RS8STHTUXUVhVWxWXXYYZZ[[\ȍ\]؍]^^__ @ +A)B*9C:IDJYEZiFjyGzHIJKɜLٜMNO P +Q)R*9S:ITJYUZiVjyWzXYZ[ɝ\ٝ]^_@ AB+C;DKE[FkG{HIJKL˼MۼN뼬OP QR+S;TKU[VkW{XYZ[\˽]۽^뽭_  !!"("#8#$H$%X%&h&'x'(())**++,Ȋ,-؊-..//00112(23834H45X56h67x78899::;;<ȋ<=؋=>>?? +!)"*9#:I$JY%Zi&jy'z()*+ɚ,ٚ-./ 0 +1)2*93:I4JY5Zi6jy7z89:;ɛ<ٛ=>? !"+#;$K%[&k'{()*+,˺-ۺ.뺪/0 12+3;4K5[6k7{89:;<˻=ۻ>뻫?``aab(bc8cdHdeXefhfgxghhiijjkklȎlm؎mnnooppqqr(rs8stHtuXuvhvwxwxxyyzz{{|ȏ|}؏}~~ ` +a)b*9c:IdJYeZifjygzhijkɞlٞmno p +q)r*9s:ItJYuZivjywzxyz{ɟ|ٟ}~` ab+c;dKe[fkg{hijkl˾m۾n뾮op qr+s;tKu[vkw{xyz{|˿}ۿ~뿯HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH g g HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2048-deterministic-fixturevalue-2049-deterministic-fixturevalue-2050-deterministic-fixturevalue-2052-deterministic-fixturevalue-2053-deterministic-fixturevalue-2054-deterministic-fixturevalue-2055-deterministic-fixturevalue-2056-deterministic-fixturevalue-2057-deterministic-fixturevalue-2059-deterministic-fixturevalue-2060-deterministic-fixturevalue-2061-deterministic-fixturevalue-2062-deterministic-fixturevalue-2063-deterministic-fixturevalue-2064-deterministic-fixturevalue-2066-deterministic-fixturevalue-2067-deterministic-fixturevalue-2068-deterministic-fixturevalue-2069-deterministic-fixturevalue-2070-deterministic-fixturevalue-2071-deterministic-fixturevalue-2073-deterministic-fixturevalue-2074-deterministic-fixturevalue-2075-deterministic-fixturevalue-2076-deterministic-fixturevalue-2077-deterministic-fixturevalue-2078-deterministic-fixturevalue-2080-deterministic-fixturevalue-2081-deterministic-fixturevalue-2082-deterministic-fixturevalue-2083-deterministic-fixturevalue-2084-deterministic-fixturevalue-2085-deterministic-fixturevalue-2087-deterministic-fixturevalue-2088-deterministic-fixturevalue-2089-deterministic-fixturevalue-2090-deterministic-fixturevalue-2091-deterministic-fixturevalue-2092-deterministic-fixturevalue-2094-deterministic-fixturevalue-2095-deterministic-fixturevalue-2096-deterministic-fixturevalue-2097-deterministic-fixturevalue-2098-deterministic-fixturevalue-2099-deterministic-fixturevalue-2101-deterministic-fixturevalue-2102-deterministic-fixturevalue-2103-deterministic-fixturevalue-2104-deterministic-fixturevalue-2105-deterministic-fixturevalue-2106-deterministic-fixturevalue-2108-deterministic-fixturevalue-2109-deterministic-fixturevalue-2110-deterministic-fixturevalue-2111-deterministic-fixturevalue-2112-deterministic-fixturevalue-2113-deterministic-fixturevalue-2115-deterministic-fixturevalue-2116-deterministic-fixturevalue-2117-deterministic-fixturevalue-2118-deterministic-fixturevalue-2119-deterministic-fixturevalue-2120-deterministic-fixturevalue-2122-deterministic-fixturevalue-2123-deterministic-fixturevalue-2124-deterministic-fixturevalue-2125-deterministic-fixturevalue-2126-deterministic-fixturevalue-2127-deterministic-fixturevalue-2129-deterministic-fixturevalue-2130-deterministic-fixturevalue-2131-deterministic-fixturevalue-2132-deterministic-fixturevalue-2133-deterministic-fixturevalue-2134-deterministic-fixturevalue-2136-deterministic-fixturevalue-2137-deterministic-fixturevalue-2138-deterministic-fixturevalue-2139-deterministic-fixturevalue-2140-deterministic-fixturevalue-2141-deterministic-fixturevalue-2143-deterministic-fixturevalue-2144-deterministic-fixturevalue-2145-deterministic-fixturevalue-2146-deterministic-fixturevalue-2147-deterministic-fixturevalue-2148-deterministic-fixturevalue-2150-deterministic-fixturevalue-2151-deterministic-fixturevalue-2152-deterministic-fixturevalue-2153-deterministic-fixturevalue-2154-deterministic-fixturevalue-2155-deterministic-fixturevalue-2157-deterministic-fixturevalue-2158-deterministic-fixturevalue-2159-deterministic-fixturevalue-2160-deterministic-fixturevalue-2161-deterministic-fixturevalue-2162-deterministic-fixturevalue-2164-deterministic-fixturevalue-2165-deterministic-fixturevalue-2166-deterministic-fixturevalue-2167-deterministic-fixturevalue-2168-deterministic-fixturevalue-2169-deterministic-fixturevalue-2171-deterministic-fixturevalue-2172-deterministic-fixturevalue-2173-deterministic-fixturevalue-2174-deterministic-fixturevalue-2175-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-2176-deterministic-fixturevalue-2178-deterministic-fixturevalue-2179-deterministic-fixturevalue-2180-deterministic-fixturevalue-2181-deterministic-fixturevalue-2182-deterministic-fixturevalue-2183-deterministic-fixturevalue-2185-deterministic-fixturevalue-2186-deterministic-fixturevalue-2187-deterministic-fixturevalue-2188-deterministic-fixturevalue-2189-deterministic-fixturevalue-2190-deterministic-fixturevalue-2192-deterministic-fixturevalue-2193-deterministic-fixturevalue-2194-deterministic-fixturevalue-2195-deterministic-fixturevalue-2196-deterministic-fixturevalue-2197-deterministic-fixturevalue-2199-deterministic-fixturevalue-2200-deterministic-fixturevalue-2201-deterministic-fixturevalue-2202-deterministic-fixturevalue-2203-deterministic-fixturevalue-2204-deterministic-fixturevalue-2206-deterministic-fixturevalue-2207-deterministic-fixturevalue-2208-deterministic-fixturevalue-2209-deterministic-fixturevalue-2210-deterministic-fixturevalue-2211-deterministic-fixturevalue-2213-deterministic-fixturevalue-2214-deterministic-fixturevalue-2215-deterministic-fixturevalue-2216-deterministic-fixturevalue-2217-deterministic-fixturevalue-2218-deterministic-fixturevalue-2220-deterministic-fixturevalue-2221-deterministic-fixturevalue-2222-deterministic-fixturevalue-2223-deterministic-fixturevalue-2224-deterministic-fixturevalue-2225-deterministic-fixturevalue-2227-deterministic-fixturevalue-2228-deterministic-fixturevalue-2229-deterministic-fixturevalue-2230-deterministic-fixturevalue-2231-deterministic-fixturevalue-2232-deterministic-fixturevalue-2234-deterministic-fixturevalue-2235-deterministic-fixturevalue-2236-deterministic-fixturevalue-2237-deterministic-fixturevalue-2238-deterministic-fixturevalue-2239-deterministic-fixturevalue-2241-deterministic-fixturevalue-2242-deterministic-fixturevalue-2243-deterministic-fixturevalue-2244-deterministic-fixturevalue-2245-deterministic-fixturevalue-2246-deterministic-fixturevalue-2248-deterministic-fixturevalue-2249-deterministic-fixturevalue-2250-deterministic-fixturevalue-2251-deterministic-fixturevalue-2252-deterministic-fixturevalue-2253-deterministic-fixturevalue-2255-deterministic-fixturevalue-2256-deterministic-fixturevalue-2257-deterministic-fixturevalue-2258-deterministic-fixturevalue-2259-deterministic-fixturevalue-2260-deterministic-fixturevalue-2262-deterministic-fixturevalue-2263-deterministic-fixturevalue-2264-deterministic-fixturevalue-2265-deterministic-fixturevalue-2266-deterministic-fixturevalue-2267-deterministic-fixturevalue-2269-deterministic-fixturevalue-2270-deterministic-fixturevalue-2271-deterministic-fixturevalue-2272-deterministic-fixturevalue-2273-deterministic-fixturevalue-2274-deterministic-fixturevalue-2276-deterministic-fixturevalue-2277-deterministic-fixturevalue-2278-deterministic-fixturevalue-2279-deterministic-fixturevalue-2280-deterministic-fixturevalue-2281-deterministic-fixturevalue-2283-deterministic-fixturevalue-2284-deterministic-fixturevalue-2285-deterministic-fixturevalue-2286-deterministic-fixturevalue-2287-deterministic-fixturevalue-2288-deterministic-fixturevalue-2290-deterministic-fixturevalue-2291-deterministic-fixturevalue-2292-deterministic-fixturevalue-2293-deterministic-fixturevalue-2294-deterministic-fixturevalue-2295-deterministic-fixturevalue-2297-deterministic-fixturevalue-2298-deterministic-fixturevalue-2299-deterministic-fixturevalue-2300-deterministic-fixturevalue-2301-deterministic-fixturevalue-2302-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2304-deterministic-fixturevalue-2305-deterministic-fixturevalue-2306-deterministic-fixturevalue-2307-deterministic-fixturevalue-2308-deterministic-fixturevalue-2309-deterministic-fixturevalue-2311-deterministic-fixturevalue-2312-deterministic-fixturevalue-2313-deterministic-fixturevalue-2314-deterministic-fixturevalue-2315-deterministic-fixturevalue-2316-deterministic-fixturevalue-2318-deterministic-fixturevalue-2319-deterministic-fixturevalue-2320-deterministic-fixturevalue-2321-deterministic-fixturevalue-2322-deterministic-fixturevalue-2323-deterministic-fixturevalue-2325-deterministic-fixturevalue-2326-deterministic-fixturevalue-2327-deterministic-fixturevalue-2328-deterministic-fixturevalue-2329-deterministic-fixturevalue-2330-deterministic-fixturevalue-2332-deterministic-fixturevalue-2333-deterministic-fixturevalue-2334-deterministic-fixturevalue-2335-deterministic-fixturevalue-2336-deterministic-fixturevalue-2337-deterministic-fixturevalue-2339-deterministic-fixturevalue-2340-deterministic-fixturevalue-2341-deterministic-fixturevalue-2342-deterministic-fixturevalue-2343-deterministic-fixturevalue-2344-deterministic-fixturevalue-2346-deterministic-fixturevalue-2347-deterministic-fixturevalue-2348-deterministic-fixturevalue-2349-deterministic-fixturevalue-2350-deterministic-fixturevalue-2351-deterministic-fixturevalue-2353-deterministic-fixturevalue-2354-deterministic-fixturevalue-2355-deterministic-fixturevalue-2356-deterministic-fixturevalue-2357-deterministic-fixturevalue-2358-deterministic-fixturevalue-2360-deterministic-fixturevalue-2361-deterministic-fixturevalue-2362-deterministic-fixturevalue-2363-deterministic-fixturevalue-2364-deterministic-fixturevalue-2365-deterministic-fixturevalue-2367-deterministic-fixturevalue-2368-deterministic-fixturevalue-2369-deterministic-fixturevalue-2370-deterministic-fixturevalue-2371-deterministic-fixturevalue-2372-deterministic-fixturevalue-2374-deterministic-fixturevalue-2375-deterministic-fixturevalue-2376-deterministic-fixturevalue-2377-deterministic-fixturevalue-2378-deterministic-fixturevalue-2379-deterministic-fixturevalue-2381-deterministic-fixturevalue-2382-deterministic-fixturevalue-2383-deterministic-fixturevalue-2384-deterministic-fixturevalue-2385-deterministic-fixturevalue-2386-deterministic-fixturevalue-2388-deterministic-fixturevalue-2389-deterministic-fixturevalue-2390-deterministic-fixturevalue-2391-deterministic-fixturevalue-2392-deterministic-fixturevalue-2393-deterministic-fixturevalue-2395-deterministic-fixturevalue-2396-deterministic-fixturevalue-2397-deterministic-fixturevalue-2398-deterministic-fixturevalue-2399-deterministic-fixturevalue-2400-deterministic-fixturevalue-2402-deterministic-fixturevalue-2403-deterministic-fixturevalue-2404-deterministic-fixturevalue-2405-deterministic-fixturevalue-2406-deterministic-fixturevalue-2407-deterministic-fixturevalue-2409-deterministic-fixturevalue-2410-deterministic-fixturevalue-2411-deterministic-fixturevalue-2412-deterministic-fixturevalue-2413-deterministic-fixturevalue-2414-deterministic-fixturevalue-2416-deterministic-fixturevalue-2417-deterministic-fixturevalue-2418-deterministic-fixturevalue-2419-deterministic-fixturevalue-2420-deterministic-fixturevalue-2421-deterministic-fixturevalue-2423-deterministic-fixturevalue-2424-deterministic-fixturevalue-2425-deterministic-fixturevalue-2426-deterministic-fixturevalue-2427-deterministic-fixturevalue-2428-deterministic-fixturevalue-2430-deterministic-fixturevalue-2431-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-2432-deterministic-fixturevalue-2433-deterministic-fixturevalue-2434-deterministic-fixturevalue-2435-deterministic-fixturevalue-2437-deterministic-fixturevalue-2438-deterministic-fixturevalue-2439-deterministic-fixturevalue-2440-deterministic-fixturevalue-2441-deterministic-fixturevalue-2442-deterministic-fixturevalue-2444-deterministic-fixturevalue-2445-deterministic-fixturevalue-2446-deterministic-fixturevalue-2447-deterministic-fixturevalue-2448-deterministic-fixturevalue-2449-deterministic-fixturevalue-2451-deterministic-fixturevalue-2452-deterministic-fixturevalue-2453-deterministic-fixturevalue-2454-deterministic-fixturevalue-2455-deterministic-fixturevalue-2456-deterministic-fixturevalue-2458-deterministic-fixturevalue-2459-deterministic-fixturevalue-2460-deterministic-fixturevalue-2461-deterministic-fixturevalue-2462-deterministic-fixturevalue-2463-deterministic-fixturevalue-2465-deterministic-fixturevalue-2466-deterministic-fixturevalue-2467-deterministic-fixturevalue-2468-deterministic-fixturevalue-2469-deterministic-fixturevalue-2470-deterministic-fixturevalue-2472-deterministic-fixturevalue-2473-deterministic-fixturevalue-2474-deterministic-fixturevalue-2475-deterministic-fixturevalue-2476-deterministic-fixturevalue-2477-deterministic-fixturevalue-2479-deterministic-fixturevalue-2480-deterministic-fixturevalue-2481-deterministic-fixturevalue-2482-deterministic-fixturevalue-2483-deterministic-fixturevalue-2484-deterministic-fixturevalue-2486-deterministic-fixturevalue-2487-deterministic-fixturevalue-2488-deterministic-fixturevalue-2489-deterministic-fixturevalue-2490-deterministic-fixturevalue-2491-deterministic-fixturevalue-2493-deterministic-fixturevalue-2494-deterministic-fixturevalue-2495-deterministic-fixturevalue-2496-deterministic-fixturevalue-2497-deterministic-fixturevalue-2498-deterministic-fixturevalue-2500-deterministic-fixturevalue-2501-deterministic-fixturevalue-2502-deterministic-fixturevalue-2503-deterministic-fixturevalue-2504-deterministic-fixturevalue-2505-deterministic-fixturevalue-2507-deterministic-fixturevalue-2508-deterministic-fixturevalue-2509-deterministic-fixturevalue-2510-deterministic-fixturevalue-2511-deterministic-fixturevalue-2512-deterministic-fixturevalue-2514-deterministic-fixturevalue-2515-deterministic-fixturevalue-2516-deterministic-fixturevalue-2517-deterministic-fixturevalue-2518-deterministic-fixturevalue-2519-deterministic-fixturevalue-2521-deterministic-fixturevalue-2522-deterministic-fixturevalue-2523-deterministic-fixturevalue-2524-deterministic-fixturevalue-2525-deterministic-fixturevalue-2526-deterministic-fixturevalue-2528-deterministic-fixturevalue-2529-deterministic-fixturevalue-2530-deterministic-fixturevalue-2531-deterministic-fixturevalue-2532-deterministic-fixturevalue-2533-deterministic-fixturevalue-2535-deterministic-fixturevalue-2536-deterministic-fixturevalue-2537-deterministic-fixturevalue-2538-deterministic-fixturevalue-2539-deterministic-fixturevalue-2540-deterministic-fixturevalue-2542-deterministic-fixturevalue-2543-deterministic-fixturevalue-2544-deterministic-fixturevalue-2545-deterministic-fixturevalue-2546-deterministic-fixturevalue-2547-deterministic-fixturevalue-2549-deterministic-fixturevalue-2550-deterministic-fixturevalue-2551-deterministic-fixturevalue-2552-deterministic-fixturevalue-2553-deterministic-fixturevalue-2554-deterministic-fixturevalue-2556-deterministic-fixturevalue-2557-deterministic-fixturevalue-2558-deterministic-fixturevalue-2559-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-2560-deterministic-fixturevalue-2561-deterministic-fixturevalue-2563-deterministic-fixturevalue-2564-deterministic-fixturevalue-2565-deterministic-fixturevalue-2566-deterministic-fixturevalue-2567-deterministic-fixturevalue-2568-deterministic-fixturevalue-2570-deterministic-fixturevalue-2571-deterministic-fixturevalue-2572-deterministic-fixturevalue-2573-deterministic-fixturevalue-2574-deterministic-fixturevalue-2575-deterministic-fixturevalue-2577-deterministic-fixturevalue-2578-deterministic-fixturevalue-2579-deterministic-fixturevalue-2580-deterministic-fixturevalue-2581-deterministic-fixturevalue-2582-deterministic-fixturevalue-2584-deterministic-fixturevalue-2585-deterministic-fixturevalue-2586-deterministic-fixturevalue-2587-deterministic-fixturevalue-2588-deterministic-fixturevalue-2589-deterministic-fixturevalue-2591-deterministic-fixturevalue-2592-deterministic-fixturevalue-2593-deterministic-fixturevalue-2594-deterministic-fixturevalue-2595-deterministic-fixturevalue-2596-deterministic-fixturevalue-2598-deterministic-fixturevalue-2599-deterministic-fixturevalue-2600-deterministic-fixturevalue-2601-deterministic-fixturevalue-2602-deterministic-fixturevalue-2603-deterministic-fixturevalue-2605-deterministic-fixturevalue-2606-deterministic-fixturevalue-2607-deterministic-fixturevalue-2608-deterministic-fixturevalue-2609-deterministic-fixturevalue-2610-deterministic-fixturevalue-2612-deterministic-fixturevalue-2613-deterministic-fixturevalue-2614-deterministic-fixturevalue-2615-deterministic-fixturevalue-2616-deterministic-fixturevalue-2617-deterministic-fixturevalue-2619-deterministic-fixturevalue-2620-deterministic-fixturevalue-2621-deterministic-fixturevalue-2622-deterministic-fixturevalue-2623-deterministic-fixturevalue-2624-deterministic-fixturevalue-2626-deterministic-fixturevalue-2627-deterministic-fixturevalue-2628-deterministic-fixturevalue-2629-deterministic-fixturevalue-2630-deterministic-fixturevalue-2631-deterministic-fixturevalue-2633-deterministic-fixturevalue-2634-deterministic-fixturevalue-2635-deterministic-fixturevalue-2636-deterministic-fixturevalue-2637-deterministic-fixturevalue-2638-deterministic-fixturevalue-2640-deterministic-fixturevalue-2641-deterministic-fixturevalue-2642-deterministic-fixturevalue-2643-deterministic-fixturevalue-2644-deterministic-fixturevalue-2645-deterministic-fixturevalue-2647-deterministic-fixturevalue-2648-deterministic-fixturevalue-2649-deterministic-fixturevalue-2650-deterministic-fixturevalue-2651-deterministic-fixturevalue-2652-deterministic-fixturevalue-2654-deterministic-fixturevalue-2655-deterministic-fixturevalue-2656-deterministic-fixturevalue-2657-deterministic-fixturevalue-2658-deterministic-fixturevalue-2659-deterministic-fixturevalue-2661-deterministic-fixturevalue-2662-deterministic-fixturevalue-2663-deterministic-fixturevalue-2664-deterministic-fixturevalue-2665-deterministic-fixturevalue-2666-deterministic-fixturevalue-2668-deterministic-fixturevalue-2669-deterministic-fixturevalue-2670-deterministic-fixturevalue-2671-deterministic-fixturevalue-2672-deterministic-fixturevalue-2673-deterministic-fixturevalue-2675-deterministic-fixturevalue-2676-deterministic-fixturevalue-2677-deterministic-fixturevalue-2678-deterministic-fixturevalue-2679-deterministic-fixturevalue-2680-deterministic-fixturevalue-2682-deterministic-fixturevalue-2683-deterministic-fixturevalue-2684-deterministic-fixturevalue-2685-deterministic-fixturevalue-2686-deterministic-fixturevalue-2687-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2689-deterministic-fixturevalue-2690-deterministic-fixturevalue-2691-deterministic-fixturevalue-2692-deterministic-fixturevalue-2693-deterministic-fixturevalue-2694-deterministic-fixturevalue-2696-deterministic-fixturevalue-2697-deterministic-fixturevalue-2698-deterministic-fixturevalue-2699-deterministic-fixturevalue-2700-deterministic-fixturevalue-2701-deterministic-fixturevalue-2703-deterministic-fixturevalue-2704-deterministic-fixturevalue-2705-deterministic-fixturevalue-2706-deterministic-fixturevalue-2707-deterministic-fixturevalue-2708-deterministic-fixturevalue-2710-deterministic-fixturevalue-2711-deterministic-fixturevalue-2712-deterministic-fixturevalue-2713-deterministic-fixturevalue-2714-deterministic-fixturevalue-2715-deterministic-fixturevalue-2717-deterministic-fixturevalue-2718-deterministic-fixturevalue-2719-deterministic-fixturevalue-2720-deterministic-fixturevalue-2721-deterministic-fixturevalue-2722-deterministic-fixturevalue-2724-deterministic-fixturevalue-2725-deterministic-fixturevalue-2726-deterministic-fixturevalue-2727-deterministic-fixturevalue-2728-deterministic-fixturevalue-2729-deterministic-fixturevalue-2731-deterministic-fixturevalue-2732-deterministic-fixturevalue-2733-deterministic-fixturevalue-2734-deterministic-fixturevalue-2735-deterministic-fixturevalue-2736-deterministic-fixturevalue-2738-deterministic-fixturevalue-2739-deterministic-fixturevalue-2740-deterministic-fixturevalue-2741-deterministic-fixturevalue-2742-deterministic-fixturevalue-2743-deterministic-fixturevalue-2745-deterministic-fixturevalue-2746-deterministic-fixturevalue-2747-deterministic-fixturevalue-2748-deterministic-fixturevalue-2749-deterministic-fixturevalue-2750-deterministic-fixturevalue-2752-deterministic-fixturevalue-2753-deterministic-fixturevalue-2754-deterministic-fixturevalue-2755-deterministic-fixturevalue-2756-deterministic-fixturevalue-2757-deterministic-fixturevalue-2759-deterministic-fixturevalue-2760-deterministic-fixturevalue-2761-deterministic-fixturevalue-2762-deterministic-fixturevalue-2763-deterministic-fixturevalue-2764-deterministic-fixturevalue-2766-deterministic-fixturevalue-2767-deterministic-fixturevalue-2768-deterministic-fixturevalue-2769-deterministic-fixturevalue-2770-deterministic-fixturevalue-2771-deterministic-fixturevalue-2773-deterministic-fixturevalue-2774-deterministic-fixturevalue-2775-deterministic-fixturevalue-2776-deterministic-fixturevalue-2777-deterministic-fixturevalue-2778-deterministic-fixturevalue-2780-deterministic-fixturevalue-2781-deterministic-fixturevalue-2782-deterministic-fixturevalue-2783-deterministic-fixturevalue-2784-deterministic-fixturevalue-2785-deterministic-fixturevalue-2787-deterministic-fixturevalue-2788-deterministic-fixturevalue-2789-deterministic-fixturevalue-2790-deterministic-fixturevalue-2791-deterministic-fixturevalue-2792-deterministic-fixturevalue-2794-deterministic-fixturevalue-2795-deterministic-fixturevalue-2796-deterministic-fixturevalue-2797-deterministic-fixturevalue-2798-deterministic-fixturevalue-2799-deterministic-fixturevalue-2801-deterministic-fixturevalue-2802-deterministic-fixturevalue-2803-deterministic-fixturevalue-2804-deterministic-fixturevalue-2805-deterministic-fixturevalue-2806-deterministic-fixturevalue-2808-deterministic-fixturevalue-2809-deterministic-fixturevalue-2810-deterministic-fixturevalue-2811-deterministic-fixturevalue-2812-deterministic-fixturevalue-2813-deterministic-fixturevalue-2815-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-2816-deterministic-fixturevalue-2817-deterministic-fixturevalue-2818-deterministic-fixturevalue-2819-deterministic-fixturevalue-2820-deterministic-fixturevalue-2822-deterministic-fixturevalue-2823-deterministic-fixturevalue-2824-deterministic-fixturevalue-2825-deterministic-fixturevalue-2826-deterministic-fixturevalue-2827-deterministic-fixturevalue-2829-deterministic-fixturevalue-2830-deterministic-fixturevalue-2831-deterministic-fixturevalue-2832-deterministic-fixturevalue-2833-deterministic-fixturevalue-2834-deterministic-fixturevalue-2836-deterministic-fixturevalue-2837-deterministic-fixturevalue-2838-deterministic-fixturevalue-2839-deterministic-fixturevalue-2840-deterministic-fixturevalue-2841-deterministic-fixturevalue-2843-deterministic-fixturevalue-2844-deterministic-fixturevalue-2845-deterministic-fixturevalue-2846-deterministic-fixturevalue-2847-deterministic-fixturevalue-2848-deterministic-fixturevalue-2850-deterministic-fixturevalue-2851-deterministic-fixturevalue-2852-deterministic-fixturevalue-2853-deterministic-fixturevalue-2854-deterministic-fixturevalue-2855-deterministic-fixturevalue-2857-deterministic-fixturevalue-2858-deterministic-fixturevalue-2859-deterministic-fixturevalue-2860-deterministic-fixturevalue-2861-deterministic-fixturevalue-2862-deterministic-fixturevalue-2864-deterministic-fixturevalue-2865-deterministic-fixturevalue-2866-deterministic-fixturevalue-2867-deterministic-fixturevalue-2868-deterministic-fixturevalue-2869-deterministic-fixturevalue-2871-deterministic-fixturevalue-2872-deterministic-fixturevalue-2873-deterministic-fixturevalue-2874-deterministic-fixturevalue-2875-deterministic-fixturevalue-2876-deterministic-fixturevalue-2878-deterministic-fixturevalue-2879-deterministic-fixturevalue-2880-deterministic-fixturevalue-2881-deterministic-fixturevalue-2882-deterministic-fixturevalue-2883-deterministic-fixturevalue-2885-deterministic-fixturevalue-2886-deterministic-fixturevalue-2887-deterministic-fixturevalue-2888-deterministic-fixturevalue-2889-deterministic-fixturevalue-2890-deterministic-fixturevalue-2892-deterministic-fixturevalue-2893-deterministic-fixturevalue-2894-deterministic-fixturevalue-2895-deterministic-fixturevalue-2896-deterministic-fixturevalue-2897-deterministic-fixturevalue-2899-deterministic-fixturevalue-2900-deterministic-fixturevalue-2901-deterministic-fixturevalue-2902-deterministic-fixturevalue-2903-deterministic-fixturevalue-2904-deterministic-fixturevalue-2906-deterministic-fixturevalue-2907-deterministic-fixturevalue-2908-deterministic-fixturevalue-2909-deterministic-fixturevalue-2910-deterministic-fixturevalue-2911-deterministic-fixturevalue-2913-deterministic-fixturevalue-2914-deterministic-fixturevalue-2915-deterministic-fixturevalue-2916-deterministic-fixturevalue-2917-deterministic-fixturevalue-2918-deterministic-fixturevalue-2920-deterministic-fixturevalue-2921-deterministic-fixturevalue-2922-deterministic-fixturevalue-2923-deterministic-fixturevalue-2924-deterministic-fixturevalue-2925-deterministic-fixturevalue-2927-deterministic-fixturevalue-2928-deterministic-fixturevalue-2929-deterministic-fixturevalue-2930-deterministic-fixturevalue-2931-deterministic-fixturevalue-2932-deterministic-fixturevalue-2934-deterministic-fixturevalue-2935-deterministic-fixturevalue-2936-deterministic-fixturevalue-2937-deterministic-fixturevalue-2938-deterministic-fixturevalue-2939-deterministic-fixturevalue-2941-deterministic-fixturevalue-2942-deterministic-fixturevalue-2943-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2944-deterministic-fixturevalue-2945-deterministic-fixturevalue-2946-deterministic-fixturevalue-2948-deterministic-fixturevalue-2949-deterministic-fixturevalue-2950-deterministic-fixturevalue-2951-deterministic-fixturevalue-2952-deterministic-fixturevalue-2953-deterministic-fixturevalue-2955-deterministic-fixturevalue-2956-deterministic-fixturevalue-2957-deterministic-fixturevalue-2958-deterministic-fixturevalue-2959-deterministic-fixturevalue-2960-deterministic-fixturevalue-2962-deterministic-fixturevalue-2963-deterministic-fixturevalue-2964-deterministic-fixturevalue-2965-deterministic-fixturevalue-2966-deterministic-fixturevalue-2967-deterministic-fixturevalue-2969-deterministic-fixturevalue-2970-deterministic-fixturevalue-2971-deterministic-fixturevalue-2972-deterministic-fixturevalue-2973-deterministic-fixturevalue-2974-deterministic-fixturevalue-2976-deterministic-fixturevalue-2977-deterministic-fixturevalue-2978-deterministic-fixturevalue-2979-deterministic-fixturevalue-2980-deterministic-fixturevalue-2981-deterministic-fixturevalue-2983-deterministic-fixturevalue-2984-deterministic-fixturevalue-2985-deterministic-fixturevalue-2986-deterministic-fixturevalue-2987-deterministic-fixturevalue-2988-deterministic-fixturevalue-2990-deterministic-fixturevalue-2991-deterministic-fixturevalue-2992-deterministic-fixturevalue-2993-deterministic-fixturevalue-2994-deterministic-fixturevalue-2995-deterministic-fixturevalue-2997-deterministic-fixturevalue-2998-deterministic-fixturevalue-2999-deterministic-fixturevalue-3000-deterministic-fixturevalue-3001-deterministic-fixturevalue-3002-deterministic-fixturevalue-3004-deterministic-fixturevalue-3005-deterministic-fixturevalue-3006-deterministic-fixturevalue-3007-deterministic-fixturevalue-3008-deterministic-fixturevalue-3009-deterministic-fixturevalue-3011-deterministic-fixturevalue-3012-deterministic-fixturevalue-3013-deterministic-fixturevalue-3014-deterministic-fixturevalue-3015-deterministic-fixturevalue-3016-deterministic-fixturevalue-3018-deterministic-fixturevalue-3019-deterministic-fixturevalue-3020-deterministic-fixturevalue-3021-deterministic-fixturevalue-3022-deterministic-fixturevalue-3023-deterministic-fixturevalue-3025-deterministic-fixturevalue-3026-deterministic-fixturevalue-3027-deterministic-fixturevalue-3028-deterministic-fixturevalue-3029-deterministic-fixturevalue-3030-deterministic-fixturevalue-3032-deterministic-fixturevalue-3033-deterministic-fixturevalue-3034-deterministic-fixturevalue-3035-deterministic-fixturevalue-3036-deterministic-fixturevalue-3037-deterministic-fixturevalue-3039-deterministic-fixturevalue-3040-deterministic-fixturevalue-3041-deterministic-fixturevalue-3042-deterministic-fixturevalue-3043-deterministic-fixturevalue-3044-deterministic-fixturevalue-3046-deterministic-fixturevalue-3047-deterministic-fixturevalue-3048-deterministic-fixturevalue-3049-deterministic-fixturevalue-3050-deterministic-fixturevalue-3051-deterministic-fixturevalue-3053-deterministic-fixturevalue-3054-deterministic-fixturevalue-3055-deterministic-fixturevalue-3056-deterministic-fixturevalue-3057-deterministic-fixturevalue-3058-deterministic-fixturevalue-3060-deterministic-fixturevalue-3061-deterministic-fixturevalue-3062-deterministic-fixturevalue-3063-deterministic-fixturevalue-3064-deterministic-fixturevalue-3065-deterministic-fixturevalue-3067-deterministic-fixturevalue-3068-deterministic-fixturevalue-3069-deterministic-fixturevalue-3070-deterministic-fixturevalue-3071-deterministic-fixture::HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$!@@@ @@ @!@@ H@ @@ @@ @H@  H]xi ~ uX hP p ( +#  H+  +;0 hCPBFdȋ!ҌBGDdЋ!լGtd苡CHd!Hԣd CId!,I4d LCJdd8!lJdX CKdh!K$dxcF|LҖFڂ5PّӦF򈵍\QԶ@G +dG"5lޑֽG:5pQ#HR5|@HjͣH5HQ&#I5.JEH0JH2 JK40 JNXh@ JQ8 JT: JW< JZ H J`!B +Jc"D0 +JfXqe8!lR}eHڡ CSĉeX!Se`CT$ex!TTe CUe! UemCVe!VDeWte 6 dL  L6 R$M6 dMM*6!M0R%$N66)dN<NB61NHRK! ` K$ (K'&2K*X6J$K-FHb ,K0z 4K3Xfh$P8K6vȪ&pDK9(*TK?\KBXKLdSCMd!MekCNe!sN)e{CO5e!OAeơCPMe!PYeΡCQee!CRTcĵ ф5,񑴥#5 c5l5D،赬Qt#5c 5,4 LRdF#IFJiV$IHJl%JJov&J*LJrކ'JBNJuߖ(JZPJxক)KrRJ{+KVJƕ,KXJ֕-KZJ.L\hDP +f" p + f(D +%f" +1fX+D +=f" 3 If!;D0 UfX""h!CP af&!p mf*"("SD yf,""[İ fX5NXԩ9T Y=Z -!Y4(` M%ZdI8l &ZMHr)[ĪQ`x+[$x~-\TY .]]-1]a$O +6dKƖO"6lK֖O:⶟tKRPR6|KdPjⶠKP6KPⶡKR&Q6K6dQⶢKVQ6KfQ6K"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd@@ @!@@@ @@ @!@@ H@ @@ @iePxRjnԔ{>'jqդ|?Gktִ} AgRkw~AkBzԀBkrICl DSl҆!P30xf:(4x|R1Ԙ215JgR@ +ˇRT2U˧R3eJ"4u +RTRS5'S5JGS6 +T+V+ V +VI" + $+!V0&+)VI<(+1V H,S+œ9VT!+`0)+ȜQVIx2,+YV ĞDMpoBPN=uo\O=yphP>}ptQp2$RG>qb 0Sg>qDS>qTT>q)dUr"1tV>yT(ޞE+ݜVÍI+ ؁DSM+朙VFT+VHY+VIJ]+rR9V'?r够WG?rI崟XsQğY?sBYПZ?sra[?t\?tq(\t@]'@t2 L^G@ubDXQV +yT(BWy(CQXy(DQY +yT( DQZz(EQ\zԢ(E]Jz(Q^ +zT(FQ_z(GQ`zԣ( GaJz+WUV+"WeV+"*WuV+RȂW+:WԂ+BWGW+JWgW+ūW+BbWիW+rjWW+rW2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 2@ H@ @@@ @!@ UޕB VǢРJW  +X Y(2٠%JZHb 5 +[hUɒ⠒E[ U\ `J]" u +^ĉTi-ЉVi-[܉X-[Zj-[j. +\^j.  bj .\d .\$j.*\@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ,<L\l| + +  ,<L\l| -.=>MN]^mn}~ +  -.=>MN]^mn}~/?O_o + /?O_o@ @AAB,BCMDN]E^mFn}G~HIJKLMNO PQ-R.=S>MTN]U^mVn}W~XYZ[\]^_@AB/C?DOE_FoGHIJKLMNOPQR/S?TOU_VoWXYZ[\]^_ !!","#<#$L$%\%&l&'|'(())**++,,--..//0 0112,23<34L45\56l67|78899::;;<<==>>?? !-".=#>M$N]%^m&n}'~()*+,-./ 01-2.=3>M4N]5^m6n}7~89:;<=>? !"/#?$O%_&o'()*+,-./012/3?4O5_6o789:;<=>?` `aab,bcMdN]e^mfn}g~hijklmno pq-r.=s>MtN]u^mvn}w~xyz{|}~`ab/c?dOe_foghijklmnopqr/s?tOu_vowxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3072-deterministic-fixturevalue-3074-deterministic-fixturevalue-3075-deterministic-fixturevalue-3076-deterministic-fixturevalue-3077-deterministic-fixturevalue-3078-deterministic-fixturevalue-3079-deterministic-fixturevalue-3081-deterministic-fixturevalue-3082-deterministic-fixturevalue-3083-deterministic-fixturevalue-3084-deterministic-fixturevalue-3085-deterministic-fixturevalue-3086-deterministic-fixturevalue-3088-deterministic-fixturevalue-3089-deterministic-fixturevalue-3090-deterministic-fixturevalue-3091-deterministic-fixturevalue-3092-deterministic-fixturevalue-3093-deterministic-fixturevalue-3095-deterministic-fixturevalue-3096-deterministic-fixturevalue-3097-deterministic-fixturevalue-3098-deterministic-fixturevalue-3099-deterministic-fixturevalue-3100-deterministic-fixturevalue-3102-deterministic-fixturevalue-3103-deterministic-fixturevalue-3104-deterministic-fixturevalue-3105-deterministic-fixturevalue-3106-deterministic-fixturevalue-3107-deterministic-fixturevalue-3109-deterministic-fixturevalue-3110-deterministic-fixturevalue-3111-deterministic-fixturevalue-3112-deterministic-fixturevalue-3113-deterministic-fixturevalue-3114-deterministic-fixturevalue-3116-deterministic-fixturevalue-3117-deterministic-fixturevalue-3118-deterministic-fixturevalue-3119-deterministic-fixturevalue-3120-deterministic-fixturevalue-3121-deterministic-fixturevalue-3123-deterministic-fixturevalue-3124-deterministic-fixturevalue-3125-deterministic-fixturevalue-3126-deterministic-fixturevalue-3127-deterministic-fixturevalue-3128-deterministic-fixturevalue-3130-deterministic-fixturevalue-3131-deterministic-fixturevalue-3132-deterministic-fixturevalue-3133-deterministic-fixturevalue-3134-deterministic-fixturevalue-3135-deterministic-fixturevalue-3137-deterministic-fixturevalue-3138-deterministic-fixturevalue-3139-deterministic-fixturevalue-3140-deterministic-fixturevalue-3141-deterministic-fixturevalue-3142-deterministic-fixturevalue-3144-deterministic-fixturevalue-3145-deterministic-fixturevalue-3146-deterministic-fixturevalue-3147-deterministic-fixturevalue-3148-deterministic-fixturevalue-3149-deterministic-fixturevalue-3151-deterministic-fixturevalue-3152-deterministic-fixturevalue-3153-deterministic-fixturevalue-3154-deterministic-fixturevalue-3155-deterministic-fixturevalue-3156-deterministic-fixturevalue-3158-deterministic-fixturevalue-3159-deterministic-fixturevalue-3160-deterministic-fixturevalue-3161-deterministic-fixturevalue-3162-deterministic-fixturevalue-3163-deterministic-fixturevalue-3165-deterministic-fixturevalue-3166-deterministic-fixturevalue-3167-deterministic-fixturevalue-3168-deterministic-fixturevalue-3169-deterministic-fixturevalue-3170-deterministic-fixturevalue-3172-deterministic-fixturevalue-3173-deterministic-fixturevalue-3174-deterministic-fixturevalue-3175-deterministic-fixturevalue-3176-deterministic-fixturevalue-3177-deterministic-fixturevalue-3179-deterministic-fixturevalue-3180-deterministic-fixturevalue-3181-deterministic-fixturevalue-3182-deterministic-fixturevalue-3183-deterministic-fixturevalue-3184-deterministic-fixturevalue-3186-deterministic-fixturevalue-3187-deterministic-fixturevalue-3188-deterministic-fixturevalue-3189-deterministic-fixturevalue-3190-deterministic-fixturevalue-3191-deterministic-fixturevalue-3193-deterministic-fixturevalue-3194-deterministic-fixturevalue-3195-deterministic-fixturevalue-3196-deterministic-fixturevalue-3197-deterministic-fixturevalue-3198-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3200-deterministic-fixturevalue-3201-deterministic-fixturevalue-3202-deterministic-fixturevalue-3203-deterministic-fixturevalue-3204-deterministic-fixturevalue-3205-deterministic-fixturevalue-3207-deterministic-fixturevalue-3208-deterministic-fixturevalue-3209-deterministic-fixturevalue-3210-deterministic-fixturevalue-3211-deterministic-fixturevalue-3212-deterministic-fixturevalue-3214-deterministic-fixturevalue-3215-deterministic-fixturevalue-3216-deterministic-fixturevalue-3217-deterministic-fixturevalue-3218-deterministic-fixturevalue-3219-deterministic-fixturevalue-3221-deterministic-fixturevalue-3222-deterministic-fixturevalue-3223-deterministic-fixturevalue-3224-deterministic-fixturevalue-3225-deterministic-fixturevalue-3226-deterministic-fixturevalue-3228-deterministic-fixturevalue-3229-deterministic-fixturevalue-3230-deterministic-fixturevalue-3231-deterministic-fixturevalue-3232-deterministic-fixturevalue-3233-deterministic-fixturevalue-3235-deterministic-fixturevalue-3236-deterministic-fixturevalue-3237-deterministic-fixturevalue-3238-deterministic-fixturevalue-3239-deterministic-fixturevalue-3240-deterministic-fixturevalue-3242-deterministic-fixturevalue-3243-deterministic-fixturevalue-3244-deterministic-fixturevalue-3245-deterministic-fixturevalue-3246-deterministic-fixturevalue-3247-deterministic-fixturevalue-3249-deterministic-fixturevalue-3250-deterministic-fixturevalue-3251-deterministic-fixturevalue-3252-deterministic-fixturevalue-3253-deterministic-fixturevalue-3254-deterministic-fixturevalue-3256-deterministic-fixturevalue-3257-deterministic-fixturevalue-3258-deterministic-fixturevalue-3259-deterministic-fixturevalue-3260-deterministic-fixturevalue-3261-deterministic-fixturevalue-3263-deterministic-fixturevalue-3264-deterministic-fixturevalue-3265-deterministic-fixturevalue-3266-deterministic-fixturevalue-3267-deterministic-fixturevalue-3268-deterministic-fixturevalue-3270-deterministic-fixturevalue-3271-deterministic-fixturevalue-3272-deterministic-fixturevalue-3273-deterministic-fixturevalue-3274-deterministic-fixturevalue-3275-deterministic-fixturevalue-3277-deterministic-fixturevalue-3278-deterministic-fixturevalue-3279-deterministic-fixturevalue-3280-deterministic-fixturevalue-3281-deterministic-fixturevalue-3282-deterministic-fixturevalue-3284-deterministic-fixturevalue-3285-deterministic-fixturevalue-3286-deterministic-fixturevalue-3287-deterministic-fixturevalue-3288-deterministic-fixturevalue-3289-deterministic-fixturevalue-3291-deterministic-fixturevalue-3292-deterministic-fixturevalue-3293-deterministic-fixturevalue-3294-deterministic-fixturevalue-3295-deterministic-fixturevalue-3296-deterministic-fixturevalue-3298-deterministic-fixturevalue-3299-deterministic-fixturevalue-3300-deterministic-fixturevalue-3301-deterministic-fixturevalue-3302-deterministic-fixturevalue-3303-deterministic-fixturevalue-3305-deterministic-fixturevalue-3306-deterministic-fixturevalue-3307-deterministic-fixturevalue-3308-deterministic-fixturevalue-3309-deterministic-fixturevalue-3310-deterministic-fixturevalue-3312-deterministic-fixturevalue-3313-deterministic-fixturevalue-3314-deterministic-fixturevalue-3315-deterministic-fixturevalue-3316-deterministic-fixturevalue-3317-deterministic-fixturevalue-3319-deterministic-fixturevalue-3320-deterministic-fixturevalue-3321-deterministic-fixturevalue-3322-deterministic-fixturevalue-3323-deterministic-fixturevalue-3324-deterministic-fixturevalue-3326-deterministic-fixturevalue-3327-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-3328-deterministic-fixturevalue-3329-deterministic-fixturevalue-3330-deterministic-fixturevalue-3331-deterministic-fixturevalue-3333-deterministic-fixturevalue-3334-deterministic-fixturevalue-3335-deterministic-fixturevalue-3336-deterministic-fixturevalue-3337-deterministic-fixturevalue-3338-deterministic-fixturevalue-3340-deterministic-fixturevalue-3341-deterministic-fixturevalue-3342-deterministic-fixturevalue-3343-deterministic-fixturevalue-3344-deterministic-fixturevalue-3345-deterministic-fixturevalue-3347-deterministic-fixturevalue-3348-deterministic-fixturevalue-3349-deterministic-fixturevalue-3350-deterministic-fixturevalue-3351-deterministic-fixturevalue-3352-deterministic-fixturevalue-3354-deterministic-fixturevalue-3355-deterministic-fixturevalue-3356-deterministic-fixturevalue-3357-deterministic-fixturevalue-3358-deterministic-fixturevalue-3359-deterministic-fixturevalue-3361-deterministic-fixturevalue-3362-deterministic-fixturevalue-3363-deterministic-fixturevalue-3364-deterministic-fixturevalue-3365-deterministic-fixturevalue-3366-deterministic-fixturevalue-3368-deterministic-fixturevalue-3369-deterministic-fixturevalue-3370-deterministic-fixturevalue-3371-deterministic-fixturevalue-3372-deterministic-fixturevalue-3373-deterministic-fixturevalue-3375-deterministic-fixturevalue-3376-deterministic-fixturevalue-3377-deterministic-fixturevalue-3378-deterministic-fixturevalue-3379-deterministic-fixturevalue-3380-deterministic-fixturevalue-3382-deterministic-fixturevalue-3383-deterministic-fixturevalue-3384-deterministic-fixturevalue-3385-deterministic-fixturevalue-3386-deterministic-fixturevalue-3387-deterministic-fixturevalue-3389-deterministic-fixturevalue-3390-deterministic-fixturevalue-3391-deterministic-fixturevalue-3392-deterministic-fixturevalue-3393-deterministic-fixturevalue-3394-deterministic-fixturevalue-3396-deterministic-fixturevalue-3397-deterministic-fixturevalue-3398-deterministic-fixturevalue-3399-deterministic-fixturevalue-3400-deterministic-fixturevalue-3401-deterministic-fixturevalue-3403-deterministic-fixturevalue-3404-deterministic-fixturevalue-3405-deterministic-fixturevalue-3406-deterministic-fixturevalue-3407-deterministic-fixturevalue-3408-deterministic-fixturevalue-3410-deterministic-fixturevalue-3411-deterministic-fixturevalue-3412-deterministic-fixturevalue-3413-deterministic-fixturevalue-3414-deterministic-fixturevalue-3415-deterministic-fixturevalue-3417-deterministic-fixturevalue-3418-deterministic-fixturevalue-3419-deterministic-fixturevalue-3420-deterministic-fixturevalue-3421-deterministic-fixturevalue-3422-deterministic-fixturevalue-3424-deterministic-fixturevalue-3425-deterministic-fixturevalue-3426-deterministic-fixturevalue-3427-deterministic-fixturevalue-3428-deterministic-fixturevalue-3429-deterministic-fixturevalue-3431-deterministic-fixturevalue-3432-deterministic-fixturevalue-3433-deterministic-fixturevalue-3434-deterministic-fixturevalue-3435-deterministic-fixturevalue-3436-deterministic-fixturevalue-3438-deterministic-fixturevalue-3439-deterministic-fixturevalue-3440-deterministic-fixturevalue-3441-deterministic-fixturevalue-3442-deterministic-fixturevalue-3443-deterministic-fixturevalue-3445-deterministic-fixturevalue-3446-deterministic-fixturevalue-3447-deterministic-fixturevalue-3448-deterministic-fixturevalue-3449-deterministic-fixturevalue-3450-deterministic-fixturevalue-3452-deterministic-fixturevalue-3453-deterministic-fixturevalue-3454-deterministic-fixturevalue-3455-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-3456-deterministic-fixturevalue-3457-deterministic-fixturevalue-3459-deterministic-fixturevalue-3460-deterministic-fixturevalue-3461-deterministic-fixturevalue-3462-deterministic-fixturevalue-3463-deterministic-fixturevalue-3464-deterministic-fixturevalue-3466-deterministic-fixturevalue-3467-deterministic-fixturevalue-3468-deterministic-fixturevalue-3469-deterministic-fixturevalue-3470-deterministic-fixturevalue-3471-deterministic-fixturevalue-3473-deterministic-fixturevalue-3474-deterministic-fixturevalue-3475-deterministic-fixturevalue-3476-deterministic-fixturevalue-3477-deterministic-fixturevalue-3478-deterministic-fixturevalue-3480-deterministic-fixturevalue-3481-deterministic-fixturevalue-3482-deterministic-fixturevalue-3483-deterministic-fixturevalue-3484-deterministic-fixturevalue-3485-deterministic-fixturevalue-3487-deterministic-fixturevalue-3488-deterministic-fixturevalue-3489-deterministic-fixturevalue-3490-deterministic-fixturevalue-3491-deterministic-fixturevalue-3492-deterministic-fixturevalue-3494-deterministic-fixturevalue-3495-deterministic-fixturevalue-3496-deterministic-fixturevalue-3497-deterministic-fixturevalue-3498-deterministic-fixturevalue-3499-deterministic-fixturevalue-3501-deterministic-fixturevalue-3502-deterministic-fixturevalue-3503-deterministic-fixturevalue-3504-deterministic-fixturevalue-3505-deterministic-fixturevalue-3506-deterministic-fixturevalue-3508-deterministic-fixturevalue-3509-deterministic-fixturevalue-3510-deterministic-fixturevalue-3511-deterministic-fixturevalue-3512-deterministic-fixturevalue-3513-deterministic-fixturevalue-3515-deterministic-fixturevalue-3516-deterministic-fixturevalue-3517-deterministic-fixturevalue-3518-deterministic-fixturevalue-3519-deterministic-fixturevalue-3520-deterministic-fixturevalue-3522-deterministic-fixturevalue-3523-deterministic-fixturevalue-3524-deterministic-fixturevalue-3525-deterministic-fixturevalue-3526-deterministic-fixturevalue-3527-deterministic-fixturevalue-3529-deterministic-fixturevalue-3530-deterministic-fixturevalue-3531-deterministic-fixturevalue-3532-deterministic-fixturevalue-3533-deterministic-fixturevalue-3534-deterministic-fixturevalue-3536-deterministic-fixturevalue-3537-deterministic-fixturevalue-3538-deterministic-fixturevalue-3539-deterministic-fixturevalue-3540-deterministic-fixturevalue-3541-deterministic-fixturevalue-3543-deterministic-fixturevalue-3544-deterministic-fixturevalue-3545-deterministic-fixturevalue-3546-deterministic-fixturevalue-3547-deterministic-fixturevalue-3548-deterministic-fixturevalue-3550-deterministic-fixturevalue-3551-deterministic-fixturevalue-3552-deterministic-fixturevalue-3553-deterministic-fixturevalue-3554-deterministic-fixturevalue-3555-deterministic-fixturevalue-3557-deterministic-fixturevalue-3558-deterministic-fixturevalue-3559-deterministic-fixturevalue-3560-deterministic-fixturevalue-3561-deterministic-fixturevalue-3562-deterministic-fixturevalue-3564-deterministic-fixturevalue-3565-deterministic-fixturevalue-3566-deterministic-fixturevalue-3567-deterministic-fixturevalue-3568-deterministic-fixturevalue-3569-deterministic-fixturevalue-3571-deterministic-fixturevalue-3572-deterministic-fixturevalue-3573-deterministic-fixturevalue-3574-deterministic-fixturevalue-3575-deterministic-fixturevalue-3576-deterministic-fixturevalue-3578-deterministic-fixturevalue-3579-deterministic-fixturevalue-3580-deterministic-fixturevalue-3581-deterministic-fixturevalue-3582-deterministic-fixturevalue-3583-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3585-deterministic-fixturevalue-3586-deterministic-fixturevalue-3587-deterministic-fixturevalue-3588-deterministic-fixturevalue-3589-deterministic-fixturevalue-3590-deterministic-fixturevalue-3592-deterministic-fixturevalue-3593-deterministic-fixturevalue-3594-deterministic-fixturevalue-3595-deterministic-fixturevalue-3596-deterministic-fixturevalue-3597-deterministic-fixturevalue-3599-deterministic-fixturevalue-3600-deterministic-fixturevalue-3601-deterministic-fixturevalue-3602-deterministic-fixturevalue-3603-deterministic-fixturevalue-3604-deterministic-fixturevalue-3606-deterministic-fixturevalue-3607-deterministic-fixturevalue-3608-deterministic-fixturevalue-3609-deterministic-fixturevalue-3610-deterministic-fixturevalue-3611-deterministic-fixturevalue-3613-deterministic-fixturevalue-3614-deterministic-fixturevalue-3615-deterministic-fixturevalue-3616-deterministic-fixturevalue-3617-deterministic-fixturevalue-3618-deterministic-fixturevalue-3620-deterministic-fixturevalue-3621-deterministic-fixturevalue-3622-deterministic-fixturevalue-3623-deterministic-fixturevalue-3624-deterministic-fixturevalue-3625-deterministic-fixturevalue-3627-deterministic-fixturevalue-3628-deterministic-fixturevalue-3629-deterministic-fixturevalue-3630-deterministic-fixturevalue-3631-deterministic-fixturevalue-3632-deterministic-fixturevalue-3634-deterministic-fixturevalue-3635-deterministic-fixturevalue-3636-deterministic-fixturevalue-3637-deterministic-fixturevalue-3638-deterministic-fixturevalue-3639-deterministic-fixturevalue-3641-deterministic-fixturevalue-3642-deterministic-fixturevalue-3643-deterministic-fixturevalue-3644-deterministic-fixturevalue-3645-deterministic-fixturevalue-3646-deterministic-fixturevalue-3648-deterministic-fixturevalue-3649-deterministic-fixturevalue-3650-deterministic-fixturevalue-3651-deterministic-fixturevalue-3652-deterministic-fixturevalue-3653-deterministic-fixturevalue-3655-deterministic-fixturevalue-3656-deterministic-fixturevalue-3657-deterministic-fixturevalue-3658-deterministic-fixturevalue-3659-deterministic-fixturevalue-3660-deterministic-fixturevalue-3662-deterministic-fixturevalue-3663-deterministic-fixturevalue-3664-deterministic-fixturevalue-3665-deterministic-fixturevalue-3666-deterministic-fixturevalue-3667-deterministic-fixturevalue-3669-deterministic-fixturevalue-3670-deterministic-fixturevalue-3671-deterministic-fixturevalue-3672-deterministic-fixturevalue-3673-deterministic-fixturevalue-3674-deterministic-fixturevalue-3676-deterministic-fixturevalue-3677-deterministic-fixturevalue-3678-deterministic-fixturevalue-3679-deterministic-fixturevalue-3680-deterministic-fixturevalue-3681-deterministic-fixturevalue-3683-deterministic-fixturevalue-3684-deterministic-fixturevalue-3685-deterministic-fixturevalue-3686-deterministic-fixturevalue-3687-deterministic-fixturevalue-3688-deterministic-fixturevalue-3690-deterministic-fixturevalue-3691-deterministic-fixturevalue-3692-deterministic-fixturevalue-3693-deterministic-fixturevalue-3694-deterministic-fixturevalue-3695-deterministic-fixturevalue-3697-deterministic-fixturevalue-3698-deterministic-fixturevalue-3699-deterministic-fixturevalue-3700-deterministic-fixturevalue-3701-deterministic-fixturevalue-3702-deterministic-fixturevalue-3704-deterministic-fixturevalue-3705-deterministic-fixturevalue-3706-deterministic-fixturevalue-3707-deterministic-fixturevalue-3708-deterministic-fixturevalue-3709-deterministic-fixturevalue-3711-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-3712-deterministic-fixturevalue-3713-deterministic-fixturevalue-3714-deterministic-fixturevalue-3715-deterministic-fixturevalue-3716-deterministic-fixturevalue-3718-deterministic-fixturevalue-3719-deterministic-fixturevalue-3720-deterministic-fixturevalue-3721-deterministic-fixturevalue-3722-deterministic-fixturevalue-3723-deterministic-fixturevalue-3725-deterministic-fixturevalue-3726-deterministic-fixturevalue-3727-deterministic-fixturevalue-3728-deterministic-fixturevalue-3729-deterministic-fixturevalue-3730-deterministic-fixturevalue-3732-deterministic-fixturevalue-3733-deterministic-fixturevalue-3734-deterministic-fixturevalue-3735-deterministic-fixturevalue-3736-deterministic-fixturevalue-3737-deterministic-fixturevalue-3739-deterministic-fixturevalue-3740-deterministic-fixturevalue-3741-deterministic-fixturevalue-3742-deterministic-fixturevalue-3743-deterministic-fixturevalue-3744-deterministic-fixturevalue-3746-deterministic-fixturevalue-3747-deterministic-fixturevalue-3748-deterministic-fixturevalue-3749-deterministic-fixturevalue-3750-deterministic-fixturevalue-3751-deterministic-fixturevalue-3753-deterministic-fixturevalue-3754-deterministic-fixturevalue-3755-deterministic-fixturevalue-3756-deterministic-fixturevalue-3757-deterministic-fixturevalue-3758-deterministic-fixturevalue-3760-deterministic-fixturevalue-3761-deterministic-fixturevalue-3762-deterministic-fixturevalue-3763-deterministic-fixturevalue-3764-deterministic-fixturevalue-3765-deterministic-fixturevalue-3767-deterministic-fixturevalue-3768-deterministic-fixturevalue-3769-deterministic-fixturevalue-3770-deterministic-fixturevalue-3771-deterministic-fixturevalue-3772-deterministic-fixturevalue-3774-deterministic-fixturevalue-3775-deterministic-fixturevalue-3776-deterministic-fixturevalue-3777-deterministic-fixturevalue-3778-deterministic-fixturevalue-3779-deterministic-fixturevalue-3781-deterministic-fixturevalue-3782-deterministic-fixturevalue-3783-deterministic-fixturevalue-3784-deterministic-fixturevalue-3785-deterministic-fixturevalue-3786-deterministic-fixturevalue-3788-deterministic-fixturevalue-3789-deterministic-fixturevalue-3790-deterministic-fixturevalue-3791-deterministic-fixturevalue-3792-deterministic-fixturevalue-3793-deterministic-fixturevalue-3795-deterministic-fixturevalue-3796-deterministic-fixturevalue-3797-deterministic-fixturevalue-3798-deterministic-fixturevalue-3799-deterministic-fixturevalue-3800-deterministic-fixturevalue-3802-deterministic-fixturevalue-3803-deterministic-fixturevalue-3804-deterministic-fixturevalue-3805-deterministic-fixturevalue-3806-deterministic-fixturevalue-3807-deterministic-fixturevalue-3809-deterministic-fixturevalue-3810-deterministic-fixturevalue-3811-deterministic-fixturevalue-3812-deterministic-fixturevalue-3813-deterministic-fixturevalue-3814-deterministic-fixturevalue-3816-deterministic-fixturevalue-3817-deterministic-fixturevalue-3818-deterministic-fixturevalue-3819-deterministic-fixturevalue-3820-deterministic-fixturevalue-3821-deterministic-fixturevalue-3823-deterministic-fixturevalue-3824-deterministic-fixturevalue-3825-deterministic-fixturevalue-3826-deterministic-fixturevalue-3827-deterministic-fixturevalue-3828-deterministic-fixturevalue-3830-deterministic-fixturevalue-3831-deterministic-fixturevalue-3832-deterministic-fixturevalue-3833-deterministic-fixturevalue-3834-deterministic-fixturevalue-3835-deterministic-fixturevalue-3837-deterministic-fixturevalue-3838-deterministic-fixturevalue-3839-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-3840-deterministic-fixturevalue-3841-deterministic-fixturevalue-3842-deterministic-fixturevalue-3844-deterministic-fixturevalue-3845-deterministic-fixturevalue-3846-deterministic-fixturevalue-3847-deterministic-fixturevalue-3848-deterministic-fixturevalue-3849-deterministic-fixturevalue-3851-deterministic-fixturevalue-3852-deterministic-fixturevalue-3853-deterministic-fixturevalue-3854-deterministic-fixturevalue-3855-deterministic-fixturevalue-3856-deterministic-fixturevalue-3858-deterministic-fixturevalue-3859-deterministic-fixturevalue-3860-deterministic-fixturevalue-3861-deterministic-fixturevalue-3862-deterministic-fixturevalue-3863-deterministic-fixturevalue-3865-deterministic-fixturevalue-3866-deterministic-fixturevalue-3867-deterministic-fixturevalue-3868-deterministic-fixturevalue-3869-deterministic-fixturevalue-3870-deterministic-fixturevalue-3872-deterministic-fixturevalue-3873-deterministic-fixturevalue-3874-deterministic-fixturevalue-3875-deterministic-fixturevalue-3876-deterministic-fixturevalue-3877-deterministic-fixturevalue-3879-deterministic-fixturevalue-3880-deterministic-fixturevalue-3881-deterministic-fixturevalue-3882-deterministic-fixturevalue-3883-deterministic-fixturevalue-3884-deterministic-fixturevalue-3886-deterministic-fixturevalue-3887-deterministic-fixturevalue-3888-deterministic-fixturevalue-3889-deterministic-fixturevalue-3890-deterministic-fixturevalue-3891-deterministic-fixturevalue-3893-deterministic-fixturevalue-3894-deterministic-fixturevalue-3895-deterministic-fixturevalue-3896-deterministic-fixturevalue-3897-deterministic-fixturevalue-3898-deterministic-fixturevalue-3900-deterministic-fixturevalue-3901-deterministic-fixturevalue-3902-deterministic-fixturevalue-3903-deterministic-fixturevalue-3904-deterministic-fixturevalue-3905-deterministic-fixturevalue-3907-deterministic-fixturevalue-3908-deterministic-fixturevalue-3909-deterministic-fixturevalue-3910-deterministic-fixturevalue-3911-deterministic-fixturevalue-3912-deterministic-fixturevalue-3914-deterministic-fixturevalue-3915-deterministic-fixturevalue-3916-deterministic-fixturevalue-3917-deterministic-fixturevalue-3918-deterministic-fixturevalue-3919-deterministic-fixturevalue-3921-deterministic-fixturevalue-3922-deterministic-fixturevalue-3923-deterministic-fixturevalue-3924-deterministic-fixturevalue-3925-deterministic-fixturevalue-3926-deterministic-fixturevalue-3928-deterministic-fixturevalue-3929-deterministic-fixturevalue-3930-deterministic-fixturevalue-3931-deterministic-fixturevalue-3932-deterministic-fixturevalue-3933-deterministic-fixturevalue-3935-deterministic-fixturevalue-3936-deterministic-fixturevalue-3937-deterministic-fixturevalue-3938-deterministic-fixturevalue-3939-deterministic-fixturevalue-3940-deterministic-fixturevalue-3942-deterministic-fixturevalue-3943-deterministic-fixturevalue-3944-deterministic-fixturevalue-3945-deterministic-fixturevalue-3946-deterministic-fixturevalue-3947-deterministic-fixturevalue-3949-deterministic-fixturevalue-3950-deterministic-fixturevalue-3951-deterministic-fixturevalue-3952-deterministic-fixturevalue-3953-deterministic-fixturevalue-3954-deterministic-fixturevalue-3956-deterministic-fixturevalue-3957-deterministic-fixturevalue-3958-deterministic-fixturevalue-3959-deterministic-fixturevalue-3960-deterministic-fixturevalue-3961-deterministic-fixturevalue-3963-deterministic-fixturevalue-3964-deterministic-fixturevalue-3965-deterministic-fixturevalue-3966-deterministic-fixturevalue-3967-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3968-deterministic-fixturevalue-3970-deterministic-fixturevalue-3971-deterministic-fixturevalue-3972-deterministic-fixturevalue-3973-deterministic-fixturevalue-3974-deterministic-fixturevalue-3975-deterministic-fixturevalue-3977-deterministic-fixturevalue-3978-deterministic-fixturevalue-3979-deterministic-fixturevalue-3980-deterministic-fixturevalue-3981-deterministic-fixturevalue-3982-deterministic-fixturevalue-3984-deterministic-fixturevalue-3985-deterministic-fixturevalue-3986-deterministic-fixturevalue-3987-deterministic-fixturevalue-3988-deterministic-fixturevalue-3989-deterministic-fixturevalue-3991-deterministic-fixturevalue-3992-deterministic-fixturevalue-3993-deterministic-fixturevalue-3994-deterministic-fixturevalue-3995-deterministic-fixturevalue-3996-deterministic-fixturevalue-3998-deterministic-fixturevalue-3999-deterministic-fixturevalue-4000-deterministic-fixturevalue-4001-deterministic-fixturevalue-4002-deterministic-fixturevalue-4003-deterministic-fixturevalue-4005-deterministic-fixturevalue-4006-deterministic-fixturevalue-4007-deterministic-fixturevalue-4008-deterministic-fixturevalue-4009-deterministic-fixturevalue-4010-deterministic-fixturevalue-4012-deterministic-fixturevalue-4013-deterministic-fixturevalue-4014-deterministic-fixturevalue-4015-deterministic-fixturevalue-4016-deterministic-fixturevalue-4017-deterministic-fixturevalue-4019-deterministic-fixturevalue-4020-deterministic-fixturevalue-4021-deterministic-fixturevalue-4022-deterministic-fixturevalue-4023-deterministic-fixturevalue-4024-deterministic-fixturevalue-4026-deterministic-fixturevalue-4027-deterministic-fixturevalue-4028-deterministic-fixturevalue-4029-deterministic-fixturevalue-4030-deterministic-fixturevalue-4031-deterministic-fixturevalue-4033-deterministic-fixturevalue-4034-deterministic-fixturevalue-4035-deterministic-fixturevalue-4036-deterministic-fixturevalue-4037-deterministic-fixturevalue-4038-deterministic-fixturevalue-4040-deterministic-fixturevalue-4041-deterministic-fixturevalue-4042-deterministic-fixturevalue-4043-deterministic-fixturevalue-4044-deterministic-fixturevalue-4045-deterministic-fixturevalue-4047-deterministic-fixturevalue-4048-deterministic-fixturevalue-4049-deterministic-fixturevalue-4050-deterministic-fixturevalue-4051-deterministic-fixturevalue-4052-deterministic-fixturevalue-4054-deterministic-fixturevalue-4055-deterministic-fixturevalue-4056-deterministic-fixturevalue-4057-deterministic-fixturevalue-4058-deterministic-fixturevalue-4059-deterministic-fixturevalue-4061-deterministic-fixturevalue-4062-deterministic-fixturevalue-4063-deterministic-fixturevalue-4064-deterministic-fixturevalue-4065-deterministic-fixturevalue-4066-deterministic-fixturevalue-4068-deterministic-fixturevalue-4069-deterministic-fixturevalue-4070-deterministic-fixturevalue-4071-deterministic-fixturevalue-4072-deterministic-fixturevalue-4073-deterministic-fixturevalue-4075-deterministic-fixturevalue-4076-deterministic-fixturevalue-4077-deterministic-fixturevalue-4078-deterministic-fixturevalue-4079-deterministic-fixturevalue-4080-deterministic-fixturevalue-4082-deterministic-fixturevalue-4083-deterministic-fixturevalue-4084-deterministic-fixturevalue-4085-deterministic-fixturevalue-4086-deterministic-fixturevalue-4087-deterministic-fixturevalue-4089-deterministic-fixturevalue-4090-deterministic-fixturevalue-4091-deterministic-fixturevalue-4092-deterministic-fixturevalue-4093-deterministic-fixturevalue-4094-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH::HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$dd @@ @H@ @@ @!@@@ @@ @!@@ " sL# $ $X $) 3 $&%  c 6L&i +X $@' $V( f L)#$v + * XS * 421QjdŜ924QrdŨz=27Qƴ|A2:Rd~D2=Rd̖I2@RdؖM2CRd䖄Q2FR𖈌U2LSdX2OSda2RSd&uь):tuLI;t&v Y;Cuv̱;su&ьɣA5]$j @E5`j!L̟M5c$j" DXQ5fU"#Vǰ4U*#vǼ5鎙SV2$6 V:6)V%8IVJ&9 WR&9CWZ:sWb';ɏWr((<鏽Wz(4e@2y&eԌ@2|eL—A2&e ×A2&׌ėA2&eLŗB2&e Ɨ@B2eƘB2&e܌ǘB2eLȘC2&e ɘYC 5yk ӀI5cy6ӌi5ykFӘ 5ykVӤ5y"k`Ӱɦ5#z*kvӼ香5SzȠ 5z:kԠ 5zJkI5zRki5{!$ +)$ L,I'$ - XC*-s-$. 0$  0"X3 0 6$!1 (3Sw& Y>w|̺)>w}L ?w&~ iY?x&̽?Cx&Ҍ?sxLɥ@x& Y@xӢj#FT5i$j$HY5l$j%LJ]5o$& a5r$j' Nd5uj(L,Pi5x$j* 8TXm5~j+Dq5$,PXy5j-LhZ|5$j. t\X< X)&@=IcX0L>iX*FX?X+Vd@ X+fpAɐ#Yp|BSY,ȈB Y-ȔC)Y-ȬD Y.ȸEiZ2eɘC2&eʘC2L˘YD2&e̘ęD2&e͘D2eLΘE2&e ϘYE2ϘΙE2&eИE2eLјF2&e 5s{bk 5{jk駹5{rk( 5|zk4)53|&@I5c|k6L 5|kFX5|kPp5|f|ɨ5#}kԈ 5"Dd2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2DH@ @@ @@ @H@ @@ @!@@@ @@ @ie(Vx +@jnԖ{> +jqզ|M@Jkt(} AjZkwAkC}(րBksMC +l( D[lӆ("p308*p4޴| +Ҁ*1' +ܘc17Nj*2GˊҜû*3W̪ܙ3gN#*4w +\S4 +Ӝ**ܚ5NJ*6j\;v;( +v ;vM"[ ;v$$;(0;*vM<*;(2v H,[;(:vT.%;(l);ȨRvMx2-;(Zv DM +posPNʽuo\OypӜhP +}ptQ*p3&߀R +qc +6ߌSjq@ߘSqÝV߰Tq*f߼Uʾr#2V +\8*pKک8*=pL۩8*>pMݩ\8>Nݩ8*qPީܟ8?qQNߩ8*@qR੤\8@S੦8*AqTᩨܠ8*qUN⩪8*Bv*a; Je;v, +h;v8m;CvDq;svPu;\};vt +;w뀲*օ;3 +w0댲j։;G똲D +l(&EJlc6MF + +m(F G mÒ(PHmfMIn#(v J[nS܆J +n(ݖK*n(MLJo(޶ Mj@oA8?~E8p@̨H8pAبM8ÚpBQ8ƚpC𨆼U8ɛEY8ϛpF\8қpGe8՛pH,i8؛I8m8ۜpJ6ѪӜs*7ܛ*N\ӽ8 +Ԁ*9 +ܜ39'NJc*0j\:G׊Ԁþ*;Wتܝ;gN +#*< +Մ4[1;Ψbv͐65;(jv89;( @=;רzvʹ<@;(v@E;(vM̱BI;(v ر@Q;樢vFU;(vHX;쨲vMJ];(r:V*rBWJsYjsRY +sCZZssb[ʿtj\꿽tӟz4\ +t@] +t36L^ju@XqV@8BqW㩮ܡ8CXN婰8+qY橲\8 Dq[8+Eq\窸ܢ8Eq]N誺8+Fq^骼\8+q_骾8+Gq`8 GqaN֍;wW뤲 +;"wg밲֕;#*wp뼲֙;S2wȲ +ם;Բ*ס;Bw +;Rwj׭;Zwױ;C׵;jw +;rw2"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"!@@ @@ H@ @@ʬUޝs,VǣЬNW@ӬX ,Y+3, NZKc,7[k@ɓ⬒G[,W\ 謓gN]#,^ Ĺ={йV=ܹX={Z={\> +|`> +| b >d>"|0f>*|<>2|Hj>@ / / _ / O / _/ / O / ^-O- ^ !- $ O'-^*-- 0O"g L⚚WgQ`iU#㪜wjYSj ]΃㺝k `㝠lKi8mkmDmqCPn tϣ\oy>C/|>F/|Ï>I|O>L/_>R/|>U/|ȏ>X|O>[/_>^|>a/|̏>d|O           ]ʃ^+ʳ,_ ,ak],aC,bsOc̣-d ^d  -e+3 7Ofk͓-GTl!>"B|`n$>%J|lp)>(R|x->+Z|v1>1x5>4j|z<>7z|A>:|~E>=̺I>@|غ _ O / @ / /O / @ / O 3-$6-&9O<-,^B.E-0HOK-4^N6Q-8T-Ohp }tq+3'rKc +7s ѓGsPtgu#2wvS:v+ҳBwKJxk嗢@>g|>j/Ϗ>m/}O>s/}>v/}>y/>|/}>/}>/}>/}>/   HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHPHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH((value-4096-deterministic-fixtureHHHHHHHHHHHHHHHH@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  0HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +id *int328 +Cname *string08R" +lance-encoding:compressionnone +items *list08 +item *int3208 +ecategory *dict:string:int8:false08BR. +&lance-encoding:dict-values-compressionnone +Bblob * large_binary08R +lance-encoding:blobtrue +)' +% +/lance.encodings.ColumnEncoding +G + +"53 +1 +/lance.encodings21.PageLayout +* 28HJ +  "53 +1 +/lance.encodings21.PageLayout +* 28H(J + "53 +1 +/lance.encodings21.PageLayout +* 28H(J + "53 +1 +/lance.encodings21.PageLayout +* 28H(G +"42 +0 +/lance.encodings21.PageLayout +  + 28H( +)' +% +/lance.encodings.ColumnEncoding +R +"?= +; +/lance.encodings21.PageLayout +* + + 28HU + "?= +; +/lance.encodings21.PageLayout +* + + 28H(U +"?= +; +/lance.encodings21.PageLayout +* + + 28H(U +"?= +; +/lance.encodings21.PageLayout +* + + 28H(K +0"86 +4 +/lance.encodings21.PageLayout + + + 28H( +)' +% +/lance.encodings.ColumnEncoding +f + .0"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@Hi + 20"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@H(i + 40"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@H(i + 50"PN +L +/lance.encodings21.PageLayout+ +) + +" + +" +* 28@H(Z + ("CA +? +/lance.encodings21.PageLayout + + +* + 28@H( +)' +% +/lance.encodings.ColumnEncoding +] + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H` + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H(` + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H(` + $"GE +C +/lance.encodings21.PageLayout" + **" + + (28H(W + $"@> +< +/lance.encodings21.PageLayout + +" + + (28H( +)' +% +/lance.encodings.ColumnEncoding +X +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H[ +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H([ +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H([ +"EC +A +/lance.encodings21.PageLayout " + + b +@@ +28H(W +"DB +@ +/lance.encodings21.PageLayout" + + b +@@ +28H( (0) 2@G'wLANC \ No newline at end of file diff --git a/rust/lance-file/test_data/exact_versions/v2_2.lance b/rust/lance-file/test_data/exact_versions/v2_2.lance new file mode 100644 index 00000000000..683c06cd2fe --- /dev/null +++ b/rust/lance-file/test_data/exact_versions/v2_2.lance @@ -0,0 +1,633 @@ +blob-0000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-0999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH +P" 2BRPbr  $P +( , 0 4P8<@DQH"L2PBTRQXb\r`dQhlptQx|` +`` +*`:aJaZajazb"b &b +*b .c 2c 6c:c>dB +dFdJ*dN:eRJeVZeZje^zfbfffjfngrgvgzg~0@pABC 0DpEFG0H pI$J(K,0L0pM4N8O<1P@qQDRHSL1TPqUTVXW\1X`qYdZh[l1\pq]t^x_|p@TpA#pB3pCCqDSTqEcqFsqGrHTrIrJrKsLTsMsNsOtPUtQ#tR3tSCuTSUuUcuVsuWvXUvYvZv[w\Uw]w^w_ 4t ++;K4[tk{"4&t*.246t:>B 5FuJ+N;RK5V[uZk^{b5fujnr5vuz~ !R""#2$B%RR&b'r()R*+,-R./01S2"324B5RS6b7r89S:袓;<=S>?h +h!h"*h#:i$Ji%Zi&ji'zj(j)j*j+k,k-k.k/l0 +l1l2*l3:m4Jm5Zm6jm7zn8n9n:n;oo?2`rabc2drefg2hrijk2lrmno3psqijrs3tsuԳvw3xsyz{3|s}~x`Vxa#xb3xcCydSVyecyfsygzhVzizjzk{lV{m{n{o|pW|q#|r3|sC}tSW}uc}vs}w~xW~y~z~{|W}~ 6v+;K6[vk{6v6v 7w+;K7[wk{7w꫷7wHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0001-deterministic-fixturevalue-0002-deterministic-fixturevalue-0003-deterministic-fixturevalue-0004-deterministic-fixturevalue-0005-deterministic-fixturevalue-0006-deterministic-fixturevalue-0008-deterministic-fixturevalue-0009-deterministic-fixturevalue-0010-deterministic-fixturevalue-0011-deterministic-fixturevalue-0012-deterministic-fixturevalue-0013-deterministic-fixturevalue-0015-deterministic-fixturevalue-0016-deterministic-fixturevalue-0017-deterministic-fixturevalue-0018-deterministic-fixturevalue-0019-deterministic-fixturevalue-0020-deterministic-fixturevalue-0022-deterministic-fixturevalue-0023-deterministic-fixturevalue-0024-deterministic-fixturevalue-0025-deterministic-fixturevalue-0026-deterministic-fixturevalue-0027-deterministic-fixturevalue-0029-deterministic-fixturevalue-0030-deterministic-fixturevalue-0031-deterministic-fixturevalue-0032-deterministic-fixturevalue-0033-deterministic-fixturevalue-0034-deterministic-fixturevalue-0036-deterministic-fixturevalue-0037-deterministic-fixturevalue-0038-deterministic-fixturevalue-0039-deterministic-fixturevalue-0040-deterministic-fixturevalue-0041-deterministic-fixturevalue-0043-deterministic-fixturevalue-0044-deterministic-fixturevalue-0045-deterministic-fixturevalue-0046-deterministic-fixturevalue-0047-deterministic-fixturevalue-0048-deterministic-fixturevalue-0050-deterministic-fixturevalue-0051-deterministic-fixturevalue-0052-deterministic-fixturevalue-0053-deterministic-fixturevalue-0054-deterministic-fixturevalue-0055-deterministic-fixturevalue-0057-deterministic-fixturevalue-0058-deterministic-fixturevalue-0059-deterministic-fixturevalue-0060-deterministic-fixturevalue-0061-deterministic-fixturevalue-0062-deterministic-fixturevalue-0064-deterministic-fixturevalue-0065-deterministic-fixturevalue-0066-deterministic-fixturevalue-0067-deterministic-fixturevalue-0068-deterministic-fixturevalue-0069-deterministic-fixturevalue-0071-deterministic-fixturevalue-0072-deterministic-fixturevalue-0073-deterministic-fixturevalue-0074-deterministic-fixturevalue-0075-deterministic-fixturevalue-0076-deterministic-fixturevalue-0078-deterministic-fixturevalue-0079-deterministic-fixturevalue-0080-deterministic-fixturevalue-0081-deterministic-fixturevalue-0082-deterministic-fixturevalue-0083-deterministic-fixturevalue-0085-deterministic-fixturevalue-0086-deterministic-fixturevalue-0087-deterministic-fixturevalue-0088-deterministic-fixturevalue-0089-deterministic-fixturevalue-0090-deterministic-fixturevalue-0092-deterministic-fixturevalue-0093-deterministic-fixturevalue-0094-deterministic-fixturevalue-0095-deterministic-fixturevalue-0096-deterministic-fixturevalue-0097-deterministic-fixturevalue-0099-deterministic-fixturevalue-0100-deterministic-fixturevalue-0101-deterministic-fixturevalue-0102-deterministic-fixturevalue-0103-deterministic-fixturevalue-0104-deterministic-fixturevalue-0106-deterministic-fixturevalue-0107-deterministic-fixturevalue-0108-deterministic-fixturevalue-0109-deterministic-fixturevalue-0110-deterministic-fixturevalue-0111-deterministic-fixturevalue-0113-deterministic-fixturevalue-0114-deterministic-fixturevalue-0115-deterministic-fixturevalue-0116-deterministic-fixturevalue-0117-deterministic-fixturevalue-0118-deterministic-fixturevalue-0120-deterministic-fixturevalue-0121-deterministic-fixturevalue-0122-deterministic-fixturevalue-0123-deterministic-fixturevalue-0124-deterministic-fixturevalue-0125-deterministic-fixturevalue-0127-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-0128-deterministic-fixturevalue-0129-deterministic-fixturevalue-0130-deterministic-fixturevalue-0131-deterministic-fixturevalue-0132-deterministic-fixturevalue-0134-deterministic-fixturevalue-0135-deterministic-fixturevalue-0136-deterministic-fixturevalue-0137-deterministic-fixturevalue-0138-deterministic-fixturevalue-0139-deterministic-fixturevalue-0141-deterministic-fixturevalue-0142-deterministic-fixturevalue-0143-deterministic-fixturevalue-0144-deterministic-fixturevalue-0145-deterministic-fixturevalue-0146-deterministic-fixturevalue-0148-deterministic-fixturevalue-0149-deterministic-fixturevalue-0150-deterministic-fixturevalue-0151-deterministic-fixturevalue-0152-deterministic-fixturevalue-0153-deterministic-fixturevalue-0155-deterministic-fixturevalue-0156-deterministic-fixturevalue-0157-deterministic-fixturevalue-0158-deterministic-fixturevalue-0159-deterministic-fixturevalue-0160-deterministic-fixturevalue-0162-deterministic-fixturevalue-0163-deterministic-fixturevalue-0164-deterministic-fixturevalue-0165-deterministic-fixturevalue-0166-deterministic-fixturevalue-0167-deterministic-fixturevalue-0169-deterministic-fixturevalue-0170-deterministic-fixturevalue-0171-deterministic-fixturevalue-0172-deterministic-fixturevalue-0173-deterministic-fixturevalue-0174-deterministic-fixturevalue-0176-deterministic-fixturevalue-0177-deterministic-fixturevalue-0178-deterministic-fixturevalue-0179-deterministic-fixturevalue-0180-deterministic-fixturevalue-0181-deterministic-fixturevalue-0183-deterministic-fixturevalue-0184-deterministic-fixturevalue-0185-deterministic-fixturevalue-0186-deterministic-fixturevalue-0187-deterministic-fixturevalue-0188-deterministic-fixturevalue-0190-deterministic-fixturevalue-0191-deterministic-fixturevalue-0192-deterministic-fixturevalue-0193-deterministic-fixturevalue-0194-deterministic-fixturevalue-0195-deterministic-fixturevalue-0197-deterministic-fixturevalue-0198-deterministic-fixturevalue-0199-deterministic-fixturevalue-0200-deterministic-fixturevalue-0201-deterministic-fixturevalue-0202-deterministic-fixturevalue-0204-deterministic-fixturevalue-0205-deterministic-fixturevalue-0206-deterministic-fixturevalue-0207-deterministic-fixturevalue-0208-deterministic-fixturevalue-0209-deterministic-fixturevalue-0211-deterministic-fixturevalue-0212-deterministic-fixturevalue-0213-deterministic-fixturevalue-0214-deterministic-fixturevalue-0215-deterministic-fixturevalue-0216-deterministic-fixturevalue-0218-deterministic-fixturevalue-0219-deterministic-fixturevalue-0220-deterministic-fixturevalue-0221-deterministic-fixturevalue-0222-deterministic-fixturevalue-0223-deterministic-fixturevalue-0225-deterministic-fixturevalue-0226-deterministic-fixturevalue-0227-deterministic-fixturevalue-0228-deterministic-fixturevalue-0229-deterministic-fixturevalue-0230-deterministic-fixturevalue-0232-deterministic-fixturevalue-0233-deterministic-fixturevalue-0234-deterministic-fixturevalue-0235-deterministic-fixturevalue-0236-deterministic-fixturevalue-0237-deterministic-fixturevalue-0239-deterministic-fixturevalue-0240-deterministic-fixturevalue-0241-deterministic-fixturevalue-0242-deterministic-fixturevalue-0243-deterministic-fixturevalue-0244-deterministic-fixturevalue-0246-deterministic-fixturevalue-0247-deterministic-fixturevalue-0248-deterministic-fixturevalue-0249-deterministic-fixturevalue-0250-deterministic-fixturevalue-0251-deterministic-fixturevalue-0253-deterministic-fixturevalue-0254-deterministic-fixturevalue-0255-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-0256-deterministic-fixturevalue-0257-deterministic-fixturevalue-0258-deterministic-fixturevalue-0260-deterministic-fixturevalue-0261-deterministic-fixturevalue-0262-deterministic-fixturevalue-0263-deterministic-fixturevalue-0264-deterministic-fixturevalue-0265-deterministic-fixturevalue-0267-deterministic-fixturevalue-0268-deterministic-fixturevalue-0269-deterministic-fixturevalue-0270-deterministic-fixturevalue-0271-deterministic-fixturevalue-0272-deterministic-fixturevalue-0274-deterministic-fixturevalue-0275-deterministic-fixturevalue-0276-deterministic-fixturevalue-0277-deterministic-fixturevalue-0278-deterministic-fixturevalue-0279-deterministic-fixturevalue-0281-deterministic-fixturevalue-0282-deterministic-fixturevalue-0283-deterministic-fixturevalue-0284-deterministic-fixturevalue-0285-deterministic-fixturevalue-0286-deterministic-fixturevalue-0288-deterministic-fixturevalue-0289-deterministic-fixturevalue-0290-deterministic-fixturevalue-0291-deterministic-fixturevalue-0292-deterministic-fixturevalue-0293-deterministic-fixturevalue-0295-deterministic-fixturevalue-0296-deterministic-fixturevalue-0297-deterministic-fixturevalue-0298-deterministic-fixturevalue-0299-deterministic-fixturevalue-0300-deterministic-fixturevalue-0302-deterministic-fixturevalue-0303-deterministic-fixturevalue-0304-deterministic-fixturevalue-0305-deterministic-fixturevalue-0306-deterministic-fixturevalue-0307-deterministic-fixturevalue-0309-deterministic-fixturevalue-0310-deterministic-fixturevalue-0311-deterministic-fixturevalue-0312-deterministic-fixturevalue-0313-deterministic-fixturevalue-0314-deterministic-fixturevalue-0316-deterministic-fixturevalue-0317-deterministic-fixturevalue-0318-deterministic-fixturevalue-0319-deterministic-fixturevalue-0320-deterministic-fixturevalue-0321-deterministic-fixturevalue-0323-deterministic-fixturevalue-0324-deterministic-fixturevalue-0325-deterministic-fixturevalue-0326-deterministic-fixturevalue-0327-deterministic-fixturevalue-0328-deterministic-fixturevalue-0330-deterministic-fixturevalue-0331-deterministic-fixturevalue-0332-deterministic-fixturevalue-0333-deterministic-fixturevalue-0334-deterministic-fixturevalue-0335-deterministic-fixturevalue-0337-deterministic-fixturevalue-0338-deterministic-fixturevalue-0339-deterministic-fixturevalue-0340-deterministic-fixturevalue-0341-deterministic-fixturevalue-0342-deterministic-fixturevalue-0344-deterministic-fixturevalue-0345-deterministic-fixturevalue-0346-deterministic-fixturevalue-0347-deterministic-fixturevalue-0348-deterministic-fixturevalue-0349-deterministic-fixturevalue-0351-deterministic-fixturevalue-0352-deterministic-fixturevalue-0353-deterministic-fixturevalue-0354-deterministic-fixturevalue-0355-deterministic-fixturevalue-0356-deterministic-fixturevalue-0358-deterministic-fixturevalue-0359-deterministic-fixturevalue-0360-deterministic-fixturevalue-0361-deterministic-fixturevalue-0362-deterministic-fixturevalue-0363-deterministic-fixturevalue-0365-deterministic-fixturevalue-0366-deterministic-fixturevalue-0367-deterministic-fixturevalue-0368-deterministic-fixturevalue-0369-deterministic-fixturevalue-0370-deterministic-fixturevalue-0372-deterministic-fixturevalue-0373-deterministic-fixturevalue-0374-deterministic-fixturevalue-0375-deterministic-fixturevalue-0376-deterministic-fixturevalue-0377-deterministic-fixturevalue-0379-deterministic-fixturevalue-0380-deterministic-fixturevalue-0381-deterministic-fixturevalue-0382-deterministic-fixturevalue-0383-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-0384-deterministic-fixturevalue-0386-deterministic-fixturevalue-0387-deterministic-fixturevalue-0388-deterministic-fixturevalue-0389-deterministic-fixturevalue-0390-deterministic-fixturevalue-0391-deterministic-fixturevalue-0393-deterministic-fixturevalue-0394-deterministic-fixturevalue-0395-deterministic-fixturevalue-0396-deterministic-fixturevalue-0397-deterministic-fixturevalue-0398-deterministic-fixturevalue-0400-deterministic-fixturevalue-0401-deterministic-fixturevalue-0402-deterministic-fixturevalue-0403-deterministic-fixturevalue-0404-deterministic-fixturevalue-0405-deterministic-fixturevalue-0407-deterministic-fixturevalue-0408-deterministic-fixturevalue-0409-deterministic-fixturevalue-0410-deterministic-fixturevalue-0411-deterministic-fixturevalue-0412-deterministic-fixturevalue-0414-deterministic-fixturevalue-0415-deterministic-fixturevalue-0416-deterministic-fixturevalue-0417-deterministic-fixturevalue-0418-deterministic-fixturevalue-0419-deterministic-fixturevalue-0421-deterministic-fixturevalue-0422-deterministic-fixturevalue-0423-deterministic-fixturevalue-0424-deterministic-fixturevalue-0425-deterministic-fixturevalue-0426-deterministic-fixturevalue-0428-deterministic-fixturevalue-0429-deterministic-fixturevalue-0430-deterministic-fixturevalue-0431-deterministic-fixturevalue-0432-deterministic-fixturevalue-0433-deterministic-fixturevalue-0435-deterministic-fixturevalue-0436-deterministic-fixturevalue-0437-deterministic-fixturevalue-0438-deterministic-fixturevalue-0439-deterministic-fixturevalue-0440-deterministic-fixturevalue-0442-deterministic-fixturevalue-0443-deterministic-fixturevalue-0444-deterministic-fixturevalue-0445-deterministic-fixturevalue-0446-deterministic-fixturevalue-0447-deterministic-fixturevalue-0449-deterministic-fixturevalue-0450-deterministic-fixturevalue-0451-deterministic-fixturevalue-0452-deterministic-fixturevalue-0453-deterministic-fixturevalue-0454-deterministic-fixturevalue-0456-deterministic-fixturevalue-0457-deterministic-fixturevalue-0458-deterministic-fixturevalue-0459-deterministic-fixturevalue-0460-deterministic-fixturevalue-0461-deterministic-fixturevalue-0463-deterministic-fixturevalue-0464-deterministic-fixturevalue-0465-deterministic-fixturevalue-0466-deterministic-fixturevalue-0467-deterministic-fixturevalue-0468-deterministic-fixturevalue-0470-deterministic-fixturevalue-0471-deterministic-fixturevalue-0472-deterministic-fixturevalue-0473-deterministic-fixturevalue-0474-deterministic-fixturevalue-0475-deterministic-fixturevalue-0477-deterministic-fixturevalue-0478-deterministic-fixturevalue-0479-deterministic-fixturevalue-0480-deterministic-fixturevalue-0481-deterministic-fixturevalue-0482-deterministic-fixturevalue-0484-deterministic-fixturevalue-0485-deterministic-fixturevalue-0486-deterministic-fixturevalue-0487-deterministic-fixturevalue-0488-deterministic-fixturevalue-0489-deterministic-fixturevalue-0491-deterministic-fixturevalue-0492-deterministic-fixturevalue-0493-deterministic-fixturevalue-0494-deterministic-fixturevalue-0495-deterministic-fixturevalue-0496-deterministic-fixturevalue-0498-deterministic-fixturevalue-0499-deterministic-fixturevalue-0500-deterministic-fixturevalue-0501-deterministic-fixturevalue-0502-deterministic-fixturevalue-0503-deterministic-fixturevalue-0505-deterministic-fixturevalue-0506-deterministic-fixturevalue-0507-deterministic-fixturevalue-0508-deterministic-fixturevalue-0509-deterministic-fixturevalue-0510-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0512-deterministic-fixturevalue-0513-deterministic-fixturevalue-0514-deterministic-fixturevalue-0515-deterministic-fixturevalue-0516-deterministic-fixturevalue-0517-deterministic-fixturevalue-0519-deterministic-fixturevalue-0520-deterministic-fixturevalue-0521-deterministic-fixturevalue-0522-deterministic-fixturevalue-0523-deterministic-fixturevalue-0524-deterministic-fixturevalue-0526-deterministic-fixturevalue-0527-deterministic-fixturevalue-0528-deterministic-fixturevalue-0529-deterministic-fixturevalue-0530-deterministic-fixturevalue-0531-deterministic-fixturevalue-0533-deterministic-fixturevalue-0534-deterministic-fixturevalue-0535-deterministic-fixturevalue-0536-deterministic-fixturevalue-0537-deterministic-fixturevalue-0538-deterministic-fixturevalue-0540-deterministic-fixturevalue-0541-deterministic-fixturevalue-0542-deterministic-fixturevalue-0543-deterministic-fixturevalue-0544-deterministic-fixturevalue-0545-deterministic-fixturevalue-0547-deterministic-fixturevalue-0548-deterministic-fixturevalue-0549-deterministic-fixturevalue-0550-deterministic-fixturevalue-0551-deterministic-fixturevalue-0552-deterministic-fixturevalue-0554-deterministic-fixturevalue-0555-deterministic-fixturevalue-0556-deterministic-fixturevalue-0557-deterministic-fixturevalue-0558-deterministic-fixturevalue-0559-deterministic-fixturevalue-0561-deterministic-fixturevalue-0562-deterministic-fixturevalue-0563-deterministic-fixturevalue-0564-deterministic-fixturevalue-0565-deterministic-fixturevalue-0566-deterministic-fixturevalue-0568-deterministic-fixturevalue-0569-deterministic-fixturevalue-0570-deterministic-fixturevalue-0571-deterministic-fixturevalue-0572-deterministic-fixturevalue-0573-deterministic-fixturevalue-0575-deterministic-fixturevalue-0576-deterministic-fixturevalue-0577-deterministic-fixturevalue-0578-deterministic-fixturevalue-0579-deterministic-fixturevalue-0580-deterministic-fixturevalue-0582-deterministic-fixturevalue-0583-deterministic-fixturevalue-0584-deterministic-fixturevalue-0585-deterministic-fixturevalue-0586-deterministic-fixturevalue-0587-deterministic-fixturevalue-0589-deterministic-fixturevalue-0590-deterministic-fixturevalue-0591-deterministic-fixturevalue-0592-deterministic-fixturevalue-0593-deterministic-fixturevalue-0594-deterministic-fixturevalue-0596-deterministic-fixturevalue-0597-deterministic-fixturevalue-0598-deterministic-fixturevalue-0599-deterministic-fixturevalue-0600-deterministic-fixturevalue-0601-deterministic-fixturevalue-0603-deterministic-fixturevalue-0604-deterministic-fixturevalue-0605-deterministic-fixturevalue-0606-deterministic-fixturevalue-0607-deterministic-fixturevalue-0608-deterministic-fixturevalue-0610-deterministic-fixturevalue-0611-deterministic-fixturevalue-0612-deterministic-fixturevalue-0613-deterministic-fixturevalue-0614-deterministic-fixturevalue-0615-deterministic-fixturevalue-0617-deterministic-fixturevalue-0618-deterministic-fixturevalue-0619-deterministic-fixturevalue-0620-deterministic-fixturevalue-0621-deterministic-fixturevalue-0622-deterministic-fixturevalue-0624-deterministic-fixturevalue-0625-deterministic-fixturevalue-0626-deterministic-fixturevalue-0627-deterministic-fixturevalue-0628-deterministic-fixturevalue-0629-deterministic-fixturevalue-0631-deterministic-fixturevalue-0632-deterministic-fixturevalue-0633-deterministic-fixturevalue-0634-deterministic-fixturevalue-0635-deterministic-fixturevalue-0636-deterministic-fixturevalue-0638-deterministic-fixturevalue-0639-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-0640-deterministic-fixturevalue-0641-deterministic-fixturevalue-0642-deterministic-fixturevalue-0643-deterministic-fixturevalue-0645-deterministic-fixturevalue-0646-deterministic-fixturevalue-0647-deterministic-fixturevalue-0648-deterministic-fixturevalue-0649-deterministic-fixturevalue-0650-deterministic-fixturevalue-0652-deterministic-fixturevalue-0653-deterministic-fixturevalue-0654-deterministic-fixturevalue-0655-deterministic-fixturevalue-0656-deterministic-fixturevalue-0657-deterministic-fixturevalue-0659-deterministic-fixturevalue-0660-deterministic-fixturevalue-0661-deterministic-fixturevalue-0662-deterministic-fixturevalue-0663-deterministic-fixturevalue-0664-deterministic-fixturevalue-0666-deterministic-fixturevalue-0667-deterministic-fixturevalue-0668-deterministic-fixturevalue-0669-deterministic-fixturevalue-0670-deterministic-fixturevalue-0671-deterministic-fixturevalue-0673-deterministic-fixturevalue-0674-deterministic-fixturevalue-0675-deterministic-fixturevalue-0676-deterministic-fixturevalue-0677-deterministic-fixturevalue-0678-deterministic-fixturevalue-0680-deterministic-fixturevalue-0681-deterministic-fixturevalue-0682-deterministic-fixturevalue-0683-deterministic-fixturevalue-0684-deterministic-fixturevalue-0685-deterministic-fixturevalue-0687-deterministic-fixturevalue-0688-deterministic-fixturevalue-0689-deterministic-fixturevalue-0690-deterministic-fixturevalue-0691-deterministic-fixturevalue-0692-deterministic-fixturevalue-0694-deterministic-fixturevalue-0695-deterministic-fixturevalue-0696-deterministic-fixturevalue-0697-deterministic-fixturevalue-0698-deterministic-fixturevalue-0699-deterministic-fixturevalue-0701-deterministic-fixturevalue-0702-deterministic-fixturevalue-0703-deterministic-fixturevalue-0704-deterministic-fixturevalue-0705-deterministic-fixturevalue-0706-deterministic-fixturevalue-0708-deterministic-fixturevalue-0709-deterministic-fixturevalue-0710-deterministic-fixturevalue-0711-deterministic-fixturevalue-0712-deterministic-fixturevalue-0713-deterministic-fixturevalue-0715-deterministic-fixturevalue-0716-deterministic-fixturevalue-0717-deterministic-fixturevalue-0718-deterministic-fixturevalue-0719-deterministic-fixturevalue-0720-deterministic-fixturevalue-0722-deterministic-fixturevalue-0723-deterministic-fixturevalue-0724-deterministic-fixturevalue-0725-deterministic-fixturevalue-0726-deterministic-fixturevalue-0727-deterministic-fixturevalue-0729-deterministic-fixturevalue-0730-deterministic-fixturevalue-0731-deterministic-fixturevalue-0732-deterministic-fixturevalue-0733-deterministic-fixturevalue-0734-deterministic-fixturevalue-0736-deterministic-fixturevalue-0737-deterministic-fixturevalue-0738-deterministic-fixturevalue-0739-deterministic-fixturevalue-0740-deterministic-fixturevalue-0741-deterministic-fixturevalue-0743-deterministic-fixturevalue-0744-deterministic-fixturevalue-0745-deterministic-fixturevalue-0746-deterministic-fixturevalue-0747-deterministic-fixturevalue-0748-deterministic-fixturevalue-0750-deterministic-fixturevalue-0751-deterministic-fixturevalue-0752-deterministic-fixturevalue-0753-deterministic-fixturevalue-0754-deterministic-fixturevalue-0755-deterministic-fixturevalue-0757-deterministic-fixturevalue-0758-deterministic-fixturevalue-0759-deterministic-fixturevalue-0760-deterministic-fixturevalue-0761-deterministic-fixturevalue-0762-deterministic-fixturevalue-0764-deterministic-fixturevalue-0765-deterministic-fixturevalue-0766-deterministic-fixturevalue-0767-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-0768-deterministic-fixturevalue-0769-deterministic-fixturevalue-0771-deterministic-fixturevalue-0772-deterministic-fixturevalue-0773-deterministic-fixturevalue-0774-deterministic-fixturevalue-0775-deterministic-fixturevalue-0776-deterministic-fixturevalue-0778-deterministic-fixturevalue-0779-deterministic-fixturevalue-0780-deterministic-fixturevalue-0781-deterministic-fixturevalue-0782-deterministic-fixturevalue-0783-deterministic-fixturevalue-0785-deterministic-fixturevalue-0786-deterministic-fixturevalue-0787-deterministic-fixturevalue-0788-deterministic-fixturevalue-0789-deterministic-fixturevalue-0790-deterministic-fixturevalue-0792-deterministic-fixturevalue-0793-deterministic-fixturevalue-0794-deterministic-fixturevalue-0795-deterministic-fixturevalue-0796-deterministic-fixturevalue-0797-deterministic-fixturevalue-0799-deterministic-fixturevalue-0800-deterministic-fixturevalue-0801-deterministic-fixturevalue-0802-deterministic-fixturevalue-0803-deterministic-fixturevalue-0804-deterministic-fixturevalue-0806-deterministic-fixturevalue-0807-deterministic-fixturevalue-0808-deterministic-fixturevalue-0809-deterministic-fixturevalue-0810-deterministic-fixturevalue-0811-deterministic-fixturevalue-0813-deterministic-fixturevalue-0814-deterministic-fixturevalue-0815-deterministic-fixturevalue-0816-deterministic-fixturevalue-0817-deterministic-fixturevalue-0818-deterministic-fixturevalue-0820-deterministic-fixturevalue-0821-deterministic-fixturevalue-0822-deterministic-fixturevalue-0823-deterministic-fixturevalue-0824-deterministic-fixturevalue-0825-deterministic-fixturevalue-0827-deterministic-fixturevalue-0828-deterministic-fixturevalue-0829-deterministic-fixturevalue-0830-deterministic-fixturevalue-0831-deterministic-fixturevalue-0832-deterministic-fixturevalue-0834-deterministic-fixturevalue-0835-deterministic-fixturevalue-0836-deterministic-fixturevalue-0837-deterministic-fixturevalue-0838-deterministic-fixturevalue-0839-deterministic-fixturevalue-0841-deterministic-fixturevalue-0842-deterministic-fixturevalue-0843-deterministic-fixturevalue-0844-deterministic-fixturevalue-0845-deterministic-fixturevalue-0846-deterministic-fixturevalue-0848-deterministic-fixturevalue-0849-deterministic-fixturevalue-0850-deterministic-fixturevalue-0851-deterministic-fixturevalue-0852-deterministic-fixturevalue-0853-deterministic-fixturevalue-0855-deterministic-fixturevalue-0856-deterministic-fixturevalue-0857-deterministic-fixturevalue-0858-deterministic-fixturevalue-0859-deterministic-fixturevalue-0860-deterministic-fixturevalue-0862-deterministic-fixturevalue-0863-deterministic-fixturevalue-0864-deterministic-fixturevalue-0865-deterministic-fixturevalue-0866-deterministic-fixturevalue-0867-deterministic-fixturevalue-0869-deterministic-fixturevalue-0870-deterministic-fixturevalue-0871-deterministic-fixturevalue-0872-deterministic-fixturevalue-0873-deterministic-fixturevalue-0874-deterministic-fixturevalue-0876-deterministic-fixturevalue-0877-deterministic-fixturevalue-0878-deterministic-fixturevalue-0879-deterministic-fixturevalue-0880-deterministic-fixturevalue-0881-deterministic-fixturevalue-0883-deterministic-fixturevalue-0884-deterministic-fixturevalue-0885-deterministic-fixturevalue-0886-deterministic-fixturevalue-0887-deterministic-fixturevalue-0888-deterministic-fixturevalue-0890-deterministic-fixturevalue-0891-deterministic-fixturevalue-0892-deterministic-fixturevalue-0893-deterministic-fixturevalue-0894-deterministic-fixturevalue-0895-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-0897-deterministic-fixturevalue-0898-deterministic-fixturevalue-0899-deterministic-fixturevalue-0900-deterministic-fixturevalue-0901-deterministic-fixturevalue-0902-deterministic-fixturevalue-0904-deterministic-fixturevalue-0905-deterministic-fixturevalue-0906-deterministic-fixturevalue-0907-deterministic-fixturevalue-0908-deterministic-fixturevalue-0909-deterministic-fixturevalue-0911-deterministic-fixturevalue-0912-deterministic-fixturevalue-0913-deterministic-fixturevalue-0914-deterministic-fixturevalue-0915-deterministic-fixturevalue-0916-deterministic-fixturevalue-0918-deterministic-fixturevalue-0919-deterministic-fixturevalue-0920-deterministic-fixturevalue-0921-deterministic-fixturevalue-0922-deterministic-fixturevalue-0923-deterministic-fixturevalue-0925-deterministic-fixturevalue-0926-deterministic-fixturevalue-0927-deterministic-fixturevalue-0928-deterministic-fixturevalue-0929-deterministic-fixturevalue-0930-deterministic-fixturevalue-0932-deterministic-fixturevalue-0933-deterministic-fixturevalue-0934-deterministic-fixturevalue-0935-deterministic-fixturevalue-0936-deterministic-fixturevalue-0937-deterministic-fixturevalue-0939-deterministic-fixturevalue-0940-deterministic-fixturevalue-0941-deterministic-fixturevalue-0942-deterministic-fixturevalue-0943-deterministic-fixturevalue-0944-deterministic-fixturevalue-0946-deterministic-fixturevalue-0947-deterministic-fixturevalue-0948-deterministic-fixturevalue-0949-deterministic-fixturevalue-0950-deterministic-fixturevalue-0951-deterministic-fixturevalue-0953-deterministic-fixturevalue-0954-deterministic-fixturevalue-0955-deterministic-fixturevalue-0956-deterministic-fixturevalue-0957-deterministic-fixturevalue-0958-deterministic-fixturevalue-0960-deterministic-fixturevalue-0961-deterministic-fixturevalue-0962-deterministic-fixturevalue-0963-deterministic-fixturevalue-0964-deterministic-fixturevalue-0965-deterministic-fixturevalue-0967-deterministic-fixturevalue-0968-deterministic-fixturevalue-0969-deterministic-fixturevalue-0970-deterministic-fixturevalue-0971-deterministic-fixturevalue-0972-deterministic-fixturevalue-0974-deterministic-fixturevalue-0975-deterministic-fixturevalue-0976-deterministic-fixturevalue-0977-deterministic-fixturevalue-0978-deterministic-fixturevalue-0979-deterministic-fixturevalue-0981-deterministic-fixturevalue-0982-deterministic-fixturevalue-0983-deterministic-fixturevalue-0984-deterministic-fixturevalue-0985-deterministic-fixturevalue-0986-deterministic-fixturevalue-0988-deterministic-fixturevalue-0989-deterministic-fixturevalue-0990-deterministic-fixturevalue-0991-deterministic-fixturevalue-0992-deterministic-fixturevalue-0993-deterministic-fixturevalue-0995-deterministic-fixturevalue-0996-deterministic-fixturevalue-0997-deterministic-fixturevalue-0998-deterministic-fixturevalue-0999-deterministic-fixturevalue-1000-deterministic-fixturevalue-1002-deterministic-fixturevalue-1003-deterministic-fixturevalue-1004-deterministic-fixturevalue-1005-deterministic-fixturevalue-1006-deterministic-fixturevalue-1007-deterministic-fixturevalue-1009-deterministic-fixturevalue-1010-deterministic-fixturevalue-1011-deterministic-fixturevalue-1012-deterministic-fixturevalue-1013-deterministic-fixturevalue-1014-deterministic-fixturevalue-1016-deterministic-fixturevalue-1017-deterministic-fixturevalue-1018-deterministic-fixturevalue-1019-deterministic-fixturevalue-1020-deterministic-fixturevalue-1021-deterministic-fixturevalue-1023-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJJ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH#F$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I@ @@ @!@@@ @@ @!@@ H@ @@ h/FAG0H @0H 01I@ KPC2K`3L @3AM +X4NC pNA#4ҋ`Nvq#8 bTv #@c`w"# efw$$Hfl#x&1$L" hrCx(a$P2ixy*$TBl~z,$P nz.$\ro{0Q%` qh{0()H8MRld{h)KDRp$|4)NP St|6)Q\MS||8)TtSD}:)W } *Z T~> *]MT d~@I*`T"~D!i*fT#$)*i8Wpب;L@X;NHBYPPZ=RX[=T`B\ >Vp]DPXx]h?Z^?\B_ @^` E B L'RX'b Úd!(r|Q(ق (ݐ#(ࢍD( + d) ҍѤq) ěġ)  ,$NX dI,'XĄi,*$,-YЄ ,0NY",3YD$,6Y& -90H )-<@NZ`d*(@-?PZxć.0-E[52Od#56APC68QpCc6:Qt7qT|@Æ8@T9BтUDBV":FaW2@c:H{ %r1|ȍ%D t|%uA } & wQ)} &dxYI~& z`&$ }i&~q(N' yɀ4'D@'Ť$H*l0U%J*o@U&LB+r@V'N+u`(+xpV)(RB+{V*XT+~@W+pV+W-+.\B+W/^@b)` &d*a ,Af1*b"!2DBha*c0!8dCj*dB!>l*eR"DDn*fb"JDp!+f"PEr+g"V$Et+h#\Dx+i􄈕-䈖-[E-[ +. \ *.O\ eJ.$Ŋ!j.0\%.<]1.HO]勤9.T!A +/"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$d @@ @H@ @@ @!@@@ @@ @!@@  rh4yQOk5zOn{OqD5|Otd5}Pw5~APzP}5ހP6߂Q$61Q#D6@4b`h4` f@5!a@s @6QalT 7and %A8"apt EA9Rar &eA:b FA;Abv f=qbz A>Bb| A?rbr",&L2r$%,&GXLbr&5,&gXLr*E,&XL„s,U,&§Xs.e,&M"s0u,&XMRs,&YMt4,&'Yt6,&GYMt8,&ԇY7퐡Sǰ7S"ʳ8T$ж$81T&#ӹaT(Cּ8TPٿ8T,c8!U0sQU29U$9UAe!FCLqe!CMe!CNBe"DOrf"P1f#"EDQaf0"&eDS2f C"FDTbf4S"fDUfDs"DVQgT"'[OvR-''[wV-'OwX-'h[Ow-'[OBw\-( +[rx^-( [Px`-(P҉xb.(\Px .(H\Pbxf5.(h\aQ3⅑Q +C6QP6Qc6!Rs6QR$7花RD7Rd7S7ASqS!%B@1c#!Aac3!eBBc@!BC2c$S!FBDbc@c!̆BE!dTs!ϦFQdd!CH"dt!%CIRd!ECJd!&KMBt:,&קYMrt,'YNu@,'Y҆uB-'NuD-''ZN2u -'GZNbuH5-'gZvJE-'ZN‡vLU-'OvNu-'ZORv-6D9U8d9V:AV<ڤ9qV9V@:WB1WF#D:aW0d:WJC:WLS :EWgd"%EX"gp"EEYRg" Zh"E[Ah"gE\qh"E^Bh"E_rh#`i$# FFabi0##fFbixhE.(PŠylU.("\Py`.(%\Q"ypu.((\Ryr.(+]Qzt.(.Qzv.(1H]Qz.(4h]Qzz.(7]rz|.):R{~.)=]$ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ @ @@ H@ @@@ @ cM1cNҕ2P2(dQ2 2HdRbE2܈dS–U2Te2dU"p2dVR2eW2(eXCV7Kos7koZ7 Cӧ\8 + ps^8 )p3`&8 Ipc68+ip3f'F8K cèh7V8 pjGp8pSP8"={={={={={>|>|>|>>|>|`3,ff#v32fg S35 gh +38Igi 3;!igj 3>$gkC3A*gls3D- m 3G0gnӛ4J3 ho$4MF9F+r3ë'P9I rc7f9Lkrœ#Pv9OrSg9R +#w9U *sS9[ Jsʃ9^+js˳9aK +s9dks9g sC~??~?~?~??????2Y◲2 he[2e\B2e]r2^Ә3 f_3)f`3&3 Ifac63# ifb +F3& cÙ V3)fd +qng8% +#pw8(JqSr8+ jqt8.KqC81kqsx84 +C|8: qsӪ~9= +r&9@Jrc69C +>|>|>>}>}>}>>}>}?~?6)hq3&&4S9Ihrc(64V< s*V4Y?ht`4\Bhu#.v4_EhvS24bK w44eN)ix4hQIiy84kTiiz:4nWi|Cӭ:j +ts:m*tУ3&:p +c6:s jt@:v tîV:| tf: t#: +u: *u: Ju: ??HHHHHHHHHHHHHHHHHHHHHHHHxwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@ @@ @@ @@ @@ aaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-1999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH $@ d@@@$$A,dA4A?%HA eHCHE+HG;%IIKeIK[IMkIO{%JQeJSJUJW%KYeK[ۥK]K_%La eLcLe+Lg;%MiKeMk[MmkMo{%NqeNsNuNw%OyeO{ۥO}ObPPQ6Q6$mD64mɣFR6DmHS6S6+GPre+/@Gr+kGr%,/Gs,˹GsE-/Gs-Gt.+GPte.KH. +k Ht %/0Hu `e: +#0(ve< S0)e$= 0*e*? 0%+e6@I0)e:pWz@pW{A qW'{C cqW7{Dk:“qWG{F:qXW{Iqe//m"/q(/y4/:/@/F/L/R/ G .U +gN ǵn! ''*U-G 0"3X'\K*;cX0^k2ʓXG_XWaBXgb J;#Xd R̃Xe+Z;ͳYgKYj jYǸkr;C(Yt|/t|/t|/t|/t|/u|/u|/u|/u|/v|/v|/-TfMm&F.Ug@ n ' +XaJ XiLҺ8XlM:DXqOPXuP\XyRhX|S:tX́UX̅V +X̉YX͍[g{X/#rw{^/ Sr{d/ r{j/+r{v/Kr{|/ks{/ss{/s{/s|/ t|/ 36&9(<*?,B0H2K4N6QT @YUŝgNLYǸXY̝;dYU繢pY՝G;|YYݝ.Yg;NYǻʻ`Y'msv|nУv|pv}q w}s+3w0}tkғwG}vwW}w wg}z#xp}|Sx}} Ճx///HHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@aaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-2999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH (8HXhx + + Ȉ ؈ (8HXhxȉ؉ +)*9:IJYZijyz + ɘ ٘   +)*9:IJYZijyzəٙ +;K[k{ + ˸ ۸븨 +;K[k{˹۹빩@@AAB(BC8CDHDEXEFhFGxGHHIIJJKKLȌLM،MNNOOPPQQR(RS8STHTUXUVhVWxWXXYYZZ[[\ȍ\]؍]^^__ @ +A)B*9C:IDJYEZiFjyGzHIJKɜLٜMNO P +Q)R*9S:ITJYUZiVjyWzXYZ[ɝ\ٝ]^_@ AB+C;DKE[FkG{HIJKL˼MۼN뼬OP QR+S;TKU[VkW{XYZ[\˽]۽^뽭_  !!"("#8#$H$%X%&h&'x'(())**++,Ȋ,-؊-..//00112(23834H45X56h67x78899::;;<ȋ<=؋=>>?? +!)"*9#:I$JY%Zi&jy'z()*+ɚ,ٚ-./ 0 +1)2*93:I4JY5Zi6jy7z89:;ɛ<ٛ=>? !"+#;$K%[&k'{()*+,˺-ۺ.뺪/0 12+3;4K5[6k7{89:;<˻=ۻ>뻫?``aab(bc8cdHdeXefhfgxghhiijjkklȎlm؎mnnooppqqr(rs8stHtuXuvhvwxwxxyyzz{{|ȏ|}؏}~~ ` +a)b*9c:IdJYeZifjygzhijkɞlٞmno p +q)r*9s:ItJYuZivjywzxyz{ɟ|ٟ}~` ab+c;dKe[fkg{hijkl˾m۾n뾮op qr+s;tKu[vkw{xyz{|˿}ۿ~뿯HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH g g HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2048-deterministic-fixturevalue-2049-deterministic-fixturevalue-2050-deterministic-fixturevalue-2052-deterministic-fixturevalue-2053-deterministic-fixturevalue-2054-deterministic-fixturevalue-2055-deterministic-fixturevalue-2056-deterministic-fixturevalue-2057-deterministic-fixturevalue-2059-deterministic-fixturevalue-2060-deterministic-fixturevalue-2061-deterministic-fixturevalue-2062-deterministic-fixturevalue-2063-deterministic-fixturevalue-2064-deterministic-fixturevalue-2066-deterministic-fixturevalue-2067-deterministic-fixturevalue-2068-deterministic-fixturevalue-2069-deterministic-fixturevalue-2070-deterministic-fixturevalue-2071-deterministic-fixturevalue-2073-deterministic-fixturevalue-2074-deterministic-fixturevalue-2075-deterministic-fixturevalue-2076-deterministic-fixturevalue-2077-deterministic-fixturevalue-2078-deterministic-fixturevalue-2080-deterministic-fixturevalue-2081-deterministic-fixturevalue-2082-deterministic-fixturevalue-2083-deterministic-fixturevalue-2084-deterministic-fixturevalue-2085-deterministic-fixturevalue-2087-deterministic-fixturevalue-2088-deterministic-fixturevalue-2089-deterministic-fixturevalue-2090-deterministic-fixturevalue-2091-deterministic-fixturevalue-2092-deterministic-fixturevalue-2094-deterministic-fixturevalue-2095-deterministic-fixturevalue-2096-deterministic-fixturevalue-2097-deterministic-fixturevalue-2098-deterministic-fixturevalue-2099-deterministic-fixturevalue-2101-deterministic-fixturevalue-2102-deterministic-fixturevalue-2103-deterministic-fixturevalue-2104-deterministic-fixturevalue-2105-deterministic-fixturevalue-2106-deterministic-fixturevalue-2108-deterministic-fixturevalue-2109-deterministic-fixturevalue-2110-deterministic-fixturevalue-2111-deterministic-fixturevalue-2112-deterministic-fixturevalue-2113-deterministic-fixturevalue-2115-deterministic-fixturevalue-2116-deterministic-fixturevalue-2117-deterministic-fixturevalue-2118-deterministic-fixturevalue-2119-deterministic-fixturevalue-2120-deterministic-fixturevalue-2122-deterministic-fixturevalue-2123-deterministic-fixturevalue-2124-deterministic-fixturevalue-2125-deterministic-fixturevalue-2126-deterministic-fixturevalue-2127-deterministic-fixturevalue-2129-deterministic-fixturevalue-2130-deterministic-fixturevalue-2131-deterministic-fixturevalue-2132-deterministic-fixturevalue-2133-deterministic-fixturevalue-2134-deterministic-fixturevalue-2136-deterministic-fixturevalue-2137-deterministic-fixturevalue-2138-deterministic-fixturevalue-2139-deterministic-fixturevalue-2140-deterministic-fixturevalue-2141-deterministic-fixturevalue-2143-deterministic-fixturevalue-2144-deterministic-fixturevalue-2145-deterministic-fixturevalue-2146-deterministic-fixturevalue-2147-deterministic-fixturevalue-2148-deterministic-fixturevalue-2150-deterministic-fixturevalue-2151-deterministic-fixturevalue-2152-deterministic-fixturevalue-2153-deterministic-fixturevalue-2154-deterministic-fixturevalue-2155-deterministic-fixturevalue-2157-deterministic-fixturevalue-2158-deterministic-fixturevalue-2159-deterministic-fixturevalue-2160-deterministic-fixturevalue-2161-deterministic-fixturevalue-2162-deterministic-fixturevalue-2164-deterministic-fixturevalue-2165-deterministic-fixturevalue-2166-deterministic-fixturevalue-2167-deterministic-fixturevalue-2168-deterministic-fixturevalue-2169-deterministic-fixturevalue-2171-deterministic-fixturevalue-2172-deterministic-fixturevalue-2173-deterministic-fixturevalue-2174-deterministic-fixturevalue-2175-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-2176-deterministic-fixturevalue-2178-deterministic-fixturevalue-2179-deterministic-fixturevalue-2180-deterministic-fixturevalue-2181-deterministic-fixturevalue-2182-deterministic-fixturevalue-2183-deterministic-fixturevalue-2185-deterministic-fixturevalue-2186-deterministic-fixturevalue-2187-deterministic-fixturevalue-2188-deterministic-fixturevalue-2189-deterministic-fixturevalue-2190-deterministic-fixturevalue-2192-deterministic-fixturevalue-2193-deterministic-fixturevalue-2194-deterministic-fixturevalue-2195-deterministic-fixturevalue-2196-deterministic-fixturevalue-2197-deterministic-fixturevalue-2199-deterministic-fixturevalue-2200-deterministic-fixturevalue-2201-deterministic-fixturevalue-2202-deterministic-fixturevalue-2203-deterministic-fixturevalue-2204-deterministic-fixturevalue-2206-deterministic-fixturevalue-2207-deterministic-fixturevalue-2208-deterministic-fixturevalue-2209-deterministic-fixturevalue-2210-deterministic-fixturevalue-2211-deterministic-fixturevalue-2213-deterministic-fixturevalue-2214-deterministic-fixturevalue-2215-deterministic-fixturevalue-2216-deterministic-fixturevalue-2217-deterministic-fixturevalue-2218-deterministic-fixturevalue-2220-deterministic-fixturevalue-2221-deterministic-fixturevalue-2222-deterministic-fixturevalue-2223-deterministic-fixturevalue-2224-deterministic-fixturevalue-2225-deterministic-fixturevalue-2227-deterministic-fixturevalue-2228-deterministic-fixturevalue-2229-deterministic-fixturevalue-2230-deterministic-fixturevalue-2231-deterministic-fixturevalue-2232-deterministic-fixturevalue-2234-deterministic-fixturevalue-2235-deterministic-fixturevalue-2236-deterministic-fixturevalue-2237-deterministic-fixturevalue-2238-deterministic-fixturevalue-2239-deterministic-fixturevalue-2241-deterministic-fixturevalue-2242-deterministic-fixturevalue-2243-deterministic-fixturevalue-2244-deterministic-fixturevalue-2245-deterministic-fixturevalue-2246-deterministic-fixturevalue-2248-deterministic-fixturevalue-2249-deterministic-fixturevalue-2250-deterministic-fixturevalue-2251-deterministic-fixturevalue-2252-deterministic-fixturevalue-2253-deterministic-fixturevalue-2255-deterministic-fixturevalue-2256-deterministic-fixturevalue-2257-deterministic-fixturevalue-2258-deterministic-fixturevalue-2259-deterministic-fixturevalue-2260-deterministic-fixturevalue-2262-deterministic-fixturevalue-2263-deterministic-fixturevalue-2264-deterministic-fixturevalue-2265-deterministic-fixturevalue-2266-deterministic-fixturevalue-2267-deterministic-fixturevalue-2269-deterministic-fixturevalue-2270-deterministic-fixturevalue-2271-deterministic-fixturevalue-2272-deterministic-fixturevalue-2273-deterministic-fixturevalue-2274-deterministic-fixturevalue-2276-deterministic-fixturevalue-2277-deterministic-fixturevalue-2278-deterministic-fixturevalue-2279-deterministic-fixturevalue-2280-deterministic-fixturevalue-2281-deterministic-fixturevalue-2283-deterministic-fixturevalue-2284-deterministic-fixturevalue-2285-deterministic-fixturevalue-2286-deterministic-fixturevalue-2287-deterministic-fixturevalue-2288-deterministic-fixturevalue-2290-deterministic-fixturevalue-2291-deterministic-fixturevalue-2292-deterministic-fixturevalue-2293-deterministic-fixturevalue-2294-deterministic-fixturevalue-2295-deterministic-fixturevalue-2297-deterministic-fixturevalue-2298-deterministic-fixturevalue-2299-deterministic-fixturevalue-2300-deterministic-fixturevalue-2301-deterministic-fixturevalue-2302-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2304-deterministic-fixturevalue-2305-deterministic-fixturevalue-2306-deterministic-fixturevalue-2307-deterministic-fixturevalue-2308-deterministic-fixturevalue-2309-deterministic-fixturevalue-2311-deterministic-fixturevalue-2312-deterministic-fixturevalue-2313-deterministic-fixturevalue-2314-deterministic-fixturevalue-2315-deterministic-fixturevalue-2316-deterministic-fixturevalue-2318-deterministic-fixturevalue-2319-deterministic-fixturevalue-2320-deterministic-fixturevalue-2321-deterministic-fixturevalue-2322-deterministic-fixturevalue-2323-deterministic-fixturevalue-2325-deterministic-fixturevalue-2326-deterministic-fixturevalue-2327-deterministic-fixturevalue-2328-deterministic-fixturevalue-2329-deterministic-fixturevalue-2330-deterministic-fixturevalue-2332-deterministic-fixturevalue-2333-deterministic-fixturevalue-2334-deterministic-fixturevalue-2335-deterministic-fixturevalue-2336-deterministic-fixturevalue-2337-deterministic-fixturevalue-2339-deterministic-fixturevalue-2340-deterministic-fixturevalue-2341-deterministic-fixturevalue-2342-deterministic-fixturevalue-2343-deterministic-fixturevalue-2344-deterministic-fixturevalue-2346-deterministic-fixturevalue-2347-deterministic-fixturevalue-2348-deterministic-fixturevalue-2349-deterministic-fixturevalue-2350-deterministic-fixturevalue-2351-deterministic-fixturevalue-2353-deterministic-fixturevalue-2354-deterministic-fixturevalue-2355-deterministic-fixturevalue-2356-deterministic-fixturevalue-2357-deterministic-fixturevalue-2358-deterministic-fixturevalue-2360-deterministic-fixturevalue-2361-deterministic-fixturevalue-2362-deterministic-fixturevalue-2363-deterministic-fixturevalue-2364-deterministic-fixturevalue-2365-deterministic-fixturevalue-2367-deterministic-fixturevalue-2368-deterministic-fixturevalue-2369-deterministic-fixturevalue-2370-deterministic-fixturevalue-2371-deterministic-fixturevalue-2372-deterministic-fixturevalue-2374-deterministic-fixturevalue-2375-deterministic-fixturevalue-2376-deterministic-fixturevalue-2377-deterministic-fixturevalue-2378-deterministic-fixturevalue-2379-deterministic-fixturevalue-2381-deterministic-fixturevalue-2382-deterministic-fixturevalue-2383-deterministic-fixturevalue-2384-deterministic-fixturevalue-2385-deterministic-fixturevalue-2386-deterministic-fixturevalue-2388-deterministic-fixturevalue-2389-deterministic-fixturevalue-2390-deterministic-fixturevalue-2391-deterministic-fixturevalue-2392-deterministic-fixturevalue-2393-deterministic-fixturevalue-2395-deterministic-fixturevalue-2396-deterministic-fixturevalue-2397-deterministic-fixturevalue-2398-deterministic-fixturevalue-2399-deterministic-fixturevalue-2400-deterministic-fixturevalue-2402-deterministic-fixturevalue-2403-deterministic-fixturevalue-2404-deterministic-fixturevalue-2405-deterministic-fixturevalue-2406-deterministic-fixturevalue-2407-deterministic-fixturevalue-2409-deterministic-fixturevalue-2410-deterministic-fixturevalue-2411-deterministic-fixturevalue-2412-deterministic-fixturevalue-2413-deterministic-fixturevalue-2414-deterministic-fixturevalue-2416-deterministic-fixturevalue-2417-deterministic-fixturevalue-2418-deterministic-fixturevalue-2419-deterministic-fixturevalue-2420-deterministic-fixturevalue-2421-deterministic-fixturevalue-2423-deterministic-fixturevalue-2424-deterministic-fixturevalue-2425-deterministic-fixturevalue-2426-deterministic-fixturevalue-2427-deterministic-fixturevalue-2428-deterministic-fixturevalue-2430-deterministic-fixturevalue-2431-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-2432-deterministic-fixturevalue-2433-deterministic-fixturevalue-2434-deterministic-fixturevalue-2435-deterministic-fixturevalue-2437-deterministic-fixturevalue-2438-deterministic-fixturevalue-2439-deterministic-fixturevalue-2440-deterministic-fixturevalue-2441-deterministic-fixturevalue-2442-deterministic-fixturevalue-2444-deterministic-fixturevalue-2445-deterministic-fixturevalue-2446-deterministic-fixturevalue-2447-deterministic-fixturevalue-2448-deterministic-fixturevalue-2449-deterministic-fixturevalue-2451-deterministic-fixturevalue-2452-deterministic-fixturevalue-2453-deterministic-fixturevalue-2454-deterministic-fixturevalue-2455-deterministic-fixturevalue-2456-deterministic-fixturevalue-2458-deterministic-fixturevalue-2459-deterministic-fixturevalue-2460-deterministic-fixturevalue-2461-deterministic-fixturevalue-2462-deterministic-fixturevalue-2463-deterministic-fixturevalue-2465-deterministic-fixturevalue-2466-deterministic-fixturevalue-2467-deterministic-fixturevalue-2468-deterministic-fixturevalue-2469-deterministic-fixturevalue-2470-deterministic-fixturevalue-2472-deterministic-fixturevalue-2473-deterministic-fixturevalue-2474-deterministic-fixturevalue-2475-deterministic-fixturevalue-2476-deterministic-fixturevalue-2477-deterministic-fixturevalue-2479-deterministic-fixturevalue-2480-deterministic-fixturevalue-2481-deterministic-fixturevalue-2482-deterministic-fixturevalue-2483-deterministic-fixturevalue-2484-deterministic-fixturevalue-2486-deterministic-fixturevalue-2487-deterministic-fixturevalue-2488-deterministic-fixturevalue-2489-deterministic-fixturevalue-2490-deterministic-fixturevalue-2491-deterministic-fixturevalue-2493-deterministic-fixturevalue-2494-deterministic-fixturevalue-2495-deterministic-fixturevalue-2496-deterministic-fixturevalue-2497-deterministic-fixturevalue-2498-deterministic-fixturevalue-2500-deterministic-fixturevalue-2501-deterministic-fixturevalue-2502-deterministic-fixturevalue-2503-deterministic-fixturevalue-2504-deterministic-fixturevalue-2505-deterministic-fixturevalue-2507-deterministic-fixturevalue-2508-deterministic-fixturevalue-2509-deterministic-fixturevalue-2510-deterministic-fixturevalue-2511-deterministic-fixturevalue-2512-deterministic-fixturevalue-2514-deterministic-fixturevalue-2515-deterministic-fixturevalue-2516-deterministic-fixturevalue-2517-deterministic-fixturevalue-2518-deterministic-fixturevalue-2519-deterministic-fixturevalue-2521-deterministic-fixturevalue-2522-deterministic-fixturevalue-2523-deterministic-fixturevalue-2524-deterministic-fixturevalue-2525-deterministic-fixturevalue-2526-deterministic-fixturevalue-2528-deterministic-fixturevalue-2529-deterministic-fixturevalue-2530-deterministic-fixturevalue-2531-deterministic-fixturevalue-2532-deterministic-fixturevalue-2533-deterministic-fixturevalue-2535-deterministic-fixturevalue-2536-deterministic-fixturevalue-2537-deterministic-fixturevalue-2538-deterministic-fixturevalue-2539-deterministic-fixturevalue-2540-deterministic-fixturevalue-2542-deterministic-fixturevalue-2543-deterministic-fixturevalue-2544-deterministic-fixturevalue-2545-deterministic-fixturevalue-2546-deterministic-fixturevalue-2547-deterministic-fixturevalue-2549-deterministic-fixturevalue-2550-deterministic-fixturevalue-2551-deterministic-fixturevalue-2552-deterministic-fixturevalue-2553-deterministic-fixturevalue-2554-deterministic-fixturevalue-2556-deterministic-fixturevalue-2557-deterministic-fixturevalue-2558-deterministic-fixturevalue-2559-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-2560-deterministic-fixturevalue-2561-deterministic-fixturevalue-2563-deterministic-fixturevalue-2564-deterministic-fixturevalue-2565-deterministic-fixturevalue-2566-deterministic-fixturevalue-2567-deterministic-fixturevalue-2568-deterministic-fixturevalue-2570-deterministic-fixturevalue-2571-deterministic-fixturevalue-2572-deterministic-fixturevalue-2573-deterministic-fixturevalue-2574-deterministic-fixturevalue-2575-deterministic-fixturevalue-2577-deterministic-fixturevalue-2578-deterministic-fixturevalue-2579-deterministic-fixturevalue-2580-deterministic-fixturevalue-2581-deterministic-fixturevalue-2582-deterministic-fixturevalue-2584-deterministic-fixturevalue-2585-deterministic-fixturevalue-2586-deterministic-fixturevalue-2587-deterministic-fixturevalue-2588-deterministic-fixturevalue-2589-deterministic-fixturevalue-2591-deterministic-fixturevalue-2592-deterministic-fixturevalue-2593-deterministic-fixturevalue-2594-deterministic-fixturevalue-2595-deterministic-fixturevalue-2596-deterministic-fixturevalue-2598-deterministic-fixturevalue-2599-deterministic-fixturevalue-2600-deterministic-fixturevalue-2601-deterministic-fixturevalue-2602-deterministic-fixturevalue-2603-deterministic-fixturevalue-2605-deterministic-fixturevalue-2606-deterministic-fixturevalue-2607-deterministic-fixturevalue-2608-deterministic-fixturevalue-2609-deterministic-fixturevalue-2610-deterministic-fixturevalue-2612-deterministic-fixturevalue-2613-deterministic-fixturevalue-2614-deterministic-fixturevalue-2615-deterministic-fixturevalue-2616-deterministic-fixturevalue-2617-deterministic-fixturevalue-2619-deterministic-fixturevalue-2620-deterministic-fixturevalue-2621-deterministic-fixturevalue-2622-deterministic-fixturevalue-2623-deterministic-fixturevalue-2624-deterministic-fixturevalue-2626-deterministic-fixturevalue-2627-deterministic-fixturevalue-2628-deterministic-fixturevalue-2629-deterministic-fixturevalue-2630-deterministic-fixturevalue-2631-deterministic-fixturevalue-2633-deterministic-fixturevalue-2634-deterministic-fixturevalue-2635-deterministic-fixturevalue-2636-deterministic-fixturevalue-2637-deterministic-fixturevalue-2638-deterministic-fixturevalue-2640-deterministic-fixturevalue-2641-deterministic-fixturevalue-2642-deterministic-fixturevalue-2643-deterministic-fixturevalue-2644-deterministic-fixturevalue-2645-deterministic-fixturevalue-2647-deterministic-fixturevalue-2648-deterministic-fixturevalue-2649-deterministic-fixturevalue-2650-deterministic-fixturevalue-2651-deterministic-fixturevalue-2652-deterministic-fixturevalue-2654-deterministic-fixturevalue-2655-deterministic-fixturevalue-2656-deterministic-fixturevalue-2657-deterministic-fixturevalue-2658-deterministic-fixturevalue-2659-deterministic-fixturevalue-2661-deterministic-fixturevalue-2662-deterministic-fixturevalue-2663-deterministic-fixturevalue-2664-deterministic-fixturevalue-2665-deterministic-fixturevalue-2666-deterministic-fixturevalue-2668-deterministic-fixturevalue-2669-deterministic-fixturevalue-2670-deterministic-fixturevalue-2671-deterministic-fixturevalue-2672-deterministic-fixturevalue-2673-deterministic-fixturevalue-2675-deterministic-fixturevalue-2676-deterministic-fixturevalue-2677-deterministic-fixturevalue-2678-deterministic-fixturevalue-2679-deterministic-fixturevalue-2680-deterministic-fixturevalue-2682-deterministic-fixturevalue-2683-deterministic-fixturevalue-2684-deterministic-fixturevalue-2685-deterministic-fixturevalue-2686-deterministic-fixturevalue-2687-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-2689-deterministic-fixturevalue-2690-deterministic-fixturevalue-2691-deterministic-fixturevalue-2692-deterministic-fixturevalue-2693-deterministic-fixturevalue-2694-deterministic-fixturevalue-2696-deterministic-fixturevalue-2697-deterministic-fixturevalue-2698-deterministic-fixturevalue-2699-deterministic-fixturevalue-2700-deterministic-fixturevalue-2701-deterministic-fixturevalue-2703-deterministic-fixturevalue-2704-deterministic-fixturevalue-2705-deterministic-fixturevalue-2706-deterministic-fixturevalue-2707-deterministic-fixturevalue-2708-deterministic-fixturevalue-2710-deterministic-fixturevalue-2711-deterministic-fixturevalue-2712-deterministic-fixturevalue-2713-deterministic-fixturevalue-2714-deterministic-fixturevalue-2715-deterministic-fixturevalue-2717-deterministic-fixturevalue-2718-deterministic-fixturevalue-2719-deterministic-fixturevalue-2720-deterministic-fixturevalue-2721-deterministic-fixturevalue-2722-deterministic-fixturevalue-2724-deterministic-fixturevalue-2725-deterministic-fixturevalue-2726-deterministic-fixturevalue-2727-deterministic-fixturevalue-2728-deterministic-fixturevalue-2729-deterministic-fixturevalue-2731-deterministic-fixturevalue-2732-deterministic-fixturevalue-2733-deterministic-fixturevalue-2734-deterministic-fixturevalue-2735-deterministic-fixturevalue-2736-deterministic-fixturevalue-2738-deterministic-fixturevalue-2739-deterministic-fixturevalue-2740-deterministic-fixturevalue-2741-deterministic-fixturevalue-2742-deterministic-fixturevalue-2743-deterministic-fixturevalue-2745-deterministic-fixturevalue-2746-deterministic-fixturevalue-2747-deterministic-fixturevalue-2748-deterministic-fixturevalue-2749-deterministic-fixturevalue-2750-deterministic-fixturevalue-2752-deterministic-fixturevalue-2753-deterministic-fixturevalue-2754-deterministic-fixturevalue-2755-deterministic-fixturevalue-2756-deterministic-fixturevalue-2757-deterministic-fixturevalue-2759-deterministic-fixturevalue-2760-deterministic-fixturevalue-2761-deterministic-fixturevalue-2762-deterministic-fixturevalue-2763-deterministic-fixturevalue-2764-deterministic-fixturevalue-2766-deterministic-fixturevalue-2767-deterministic-fixturevalue-2768-deterministic-fixturevalue-2769-deterministic-fixturevalue-2770-deterministic-fixturevalue-2771-deterministic-fixturevalue-2773-deterministic-fixturevalue-2774-deterministic-fixturevalue-2775-deterministic-fixturevalue-2776-deterministic-fixturevalue-2777-deterministic-fixturevalue-2778-deterministic-fixturevalue-2780-deterministic-fixturevalue-2781-deterministic-fixturevalue-2782-deterministic-fixturevalue-2783-deterministic-fixturevalue-2784-deterministic-fixturevalue-2785-deterministic-fixturevalue-2787-deterministic-fixturevalue-2788-deterministic-fixturevalue-2789-deterministic-fixturevalue-2790-deterministic-fixturevalue-2791-deterministic-fixturevalue-2792-deterministic-fixturevalue-2794-deterministic-fixturevalue-2795-deterministic-fixturevalue-2796-deterministic-fixturevalue-2797-deterministic-fixturevalue-2798-deterministic-fixturevalue-2799-deterministic-fixturevalue-2801-deterministic-fixturevalue-2802-deterministic-fixturevalue-2803-deterministic-fixturevalue-2804-deterministic-fixturevalue-2805-deterministic-fixturevalue-2806-deterministic-fixturevalue-2808-deterministic-fixturevalue-2809-deterministic-fixturevalue-2810-deterministic-fixturevalue-2811-deterministic-fixturevalue-2812-deterministic-fixturevalue-2813-deterministic-fixturevalue-2815-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-2816-deterministic-fixturevalue-2817-deterministic-fixturevalue-2818-deterministic-fixturevalue-2819-deterministic-fixturevalue-2820-deterministic-fixturevalue-2822-deterministic-fixturevalue-2823-deterministic-fixturevalue-2824-deterministic-fixturevalue-2825-deterministic-fixturevalue-2826-deterministic-fixturevalue-2827-deterministic-fixturevalue-2829-deterministic-fixturevalue-2830-deterministic-fixturevalue-2831-deterministic-fixturevalue-2832-deterministic-fixturevalue-2833-deterministic-fixturevalue-2834-deterministic-fixturevalue-2836-deterministic-fixturevalue-2837-deterministic-fixturevalue-2838-deterministic-fixturevalue-2839-deterministic-fixturevalue-2840-deterministic-fixturevalue-2841-deterministic-fixturevalue-2843-deterministic-fixturevalue-2844-deterministic-fixturevalue-2845-deterministic-fixturevalue-2846-deterministic-fixturevalue-2847-deterministic-fixturevalue-2848-deterministic-fixturevalue-2850-deterministic-fixturevalue-2851-deterministic-fixturevalue-2852-deterministic-fixturevalue-2853-deterministic-fixturevalue-2854-deterministic-fixturevalue-2855-deterministic-fixturevalue-2857-deterministic-fixturevalue-2858-deterministic-fixturevalue-2859-deterministic-fixturevalue-2860-deterministic-fixturevalue-2861-deterministic-fixturevalue-2862-deterministic-fixturevalue-2864-deterministic-fixturevalue-2865-deterministic-fixturevalue-2866-deterministic-fixturevalue-2867-deterministic-fixturevalue-2868-deterministic-fixturevalue-2869-deterministic-fixturevalue-2871-deterministic-fixturevalue-2872-deterministic-fixturevalue-2873-deterministic-fixturevalue-2874-deterministic-fixturevalue-2875-deterministic-fixturevalue-2876-deterministic-fixturevalue-2878-deterministic-fixturevalue-2879-deterministic-fixturevalue-2880-deterministic-fixturevalue-2881-deterministic-fixturevalue-2882-deterministic-fixturevalue-2883-deterministic-fixturevalue-2885-deterministic-fixturevalue-2886-deterministic-fixturevalue-2887-deterministic-fixturevalue-2888-deterministic-fixturevalue-2889-deterministic-fixturevalue-2890-deterministic-fixturevalue-2892-deterministic-fixturevalue-2893-deterministic-fixturevalue-2894-deterministic-fixturevalue-2895-deterministic-fixturevalue-2896-deterministic-fixturevalue-2897-deterministic-fixturevalue-2899-deterministic-fixturevalue-2900-deterministic-fixturevalue-2901-deterministic-fixturevalue-2902-deterministic-fixturevalue-2903-deterministic-fixturevalue-2904-deterministic-fixturevalue-2906-deterministic-fixturevalue-2907-deterministic-fixturevalue-2908-deterministic-fixturevalue-2909-deterministic-fixturevalue-2910-deterministic-fixturevalue-2911-deterministic-fixturevalue-2913-deterministic-fixturevalue-2914-deterministic-fixturevalue-2915-deterministic-fixturevalue-2916-deterministic-fixturevalue-2917-deterministic-fixturevalue-2918-deterministic-fixturevalue-2920-deterministic-fixturevalue-2921-deterministic-fixturevalue-2922-deterministic-fixturevalue-2923-deterministic-fixturevalue-2924-deterministic-fixturevalue-2925-deterministic-fixturevalue-2927-deterministic-fixturevalue-2928-deterministic-fixturevalue-2929-deterministic-fixturevalue-2930-deterministic-fixturevalue-2931-deterministic-fixturevalue-2932-deterministic-fixturevalue-2934-deterministic-fixturevalue-2935-deterministic-fixturevalue-2936-deterministic-fixturevalue-2937-deterministic-fixturevalue-2938-deterministic-fixturevalue-2939-deterministic-fixturevalue-2941-deterministic-fixturevalue-2942-deterministic-fixturevalue-2943-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-2944-deterministic-fixturevalue-2945-deterministic-fixturevalue-2946-deterministic-fixturevalue-2948-deterministic-fixturevalue-2949-deterministic-fixturevalue-2950-deterministic-fixturevalue-2951-deterministic-fixturevalue-2952-deterministic-fixturevalue-2953-deterministic-fixturevalue-2955-deterministic-fixturevalue-2956-deterministic-fixturevalue-2957-deterministic-fixturevalue-2958-deterministic-fixturevalue-2959-deterministic-fixturevalue-2960-deterministic-fixturevalue-2962-deterministic-fixturevalue-2963-deterministic-fixturevalue-2964-deterministic-fixturevalue-2965-deterministic-fixturevalue-2966-deterministic-fixturevalue-2967-deterministic-fixturevalue-2969-deterministic-fixturevalue-2970-deterministic-fixturevalue-2971-deterministic-fixturevalue-2972-deterministic-fixturevalue-2973-deterministic-fixturevalue-2974-deterministic-fixturevalue-2976-deterministic-fixturevalue-2977-deterministic-fixturevalue-2978-deterministic-fixturevalue-2979-deterministic-fixturevalue-2980-deterministic-fixturevalue-2981-deterministic-fixturevalue-2983-deterministic-fixturevalue-2984-deterministic-fixturevalue-2985-deterministic-fixturevalue-2986-deterministic-fixturevalue-2987-deterministic-fixturevalue-2988-deterministic-fixturevalue-2990-deterministic-fixturevalue-2991-deterministic-fixturevalue-2992-deterministic-fixturevalue-2993-deterministic-fixturevalue-2994-deterministic-fixturevalue-2995-deterministic-fixturevalue-2997-deterministic-fixturevalue-2998-deterministic-fixturevalue-2999-deterministic-fixturevalue-3000-deterministic-fixturevalue-3001-deterministic-fixturevalue-3002-deterministic-fixturevalue-3004-deterministic-fixturevalue-3005-deterministic-fixturevalue-3006-deterministic-fixturevalue-3007-deterministic-fixturevalue-3008-deterministic-fixturevalue-3009-deterministic-fixturevalue-3011-deterministic-fixturevalue-3012-deterministic-fixturevalue-3013-deterministic-fixturevalue-3014-deterministic-fixturevalue-3015-deterministic-fixturevalue-3016-deterministic-fixturevalue-3018-deterministic-fixturevalue-3019-deterministic-fixturevalue-3020-deterministic-fixturevalue-3021-deterministic-fixturevalue-3022-deterministic-fixturevalue-3023-deterministic-fixturevalue-3025-deterministic-fixturevalue-3026-deterministic-fixturevalue-3027-deterministic-fixturevalue-3028-deterministic-fixturevalue-3029-deterministic-fixturevalue-3030-deterministic-fixturevalue-3032-deterministic-fixturevalue-3033-deterministic-fixturevalue-3034-deterministic-fixturevalue-3035-deterministic-fixturevalue-3036-deterministic-fixturevalue-3037-deterministic-fixturevalue-3039-deterministic-fixturevalue-3040-deterministic-fixturevalue-3041-deterministic-fixturevalue-3042-deterministic-fixturevalue-3043-deterministic-fixturevalue-3044-deterministic-fixturevalue-3046-deterministic-fixturevalue-3047-deterministic-fixturevalue-3048-deterministic-fixturevalue-3049-deterministic-fixturevalue-3050-deterministic-fixturevalue-3051-deterministic-fixturevalue-3053-deterministic-fixturevalue-3054-deterministic-fixturevalue-3055-deterministic-fixturevalue-3056-deterministic-fixturevalue-3057-deterministic-fixturevalue-3058-deterministic-fixturevalue-3060-deterministic-fixturevalue-3061-deterministic-fixturevalue-3062-deterministic-fixturevalue-3063-deterministic-fixturevalue-3064-deterministic-fixturevalue-3065-deterministic-fixturevalue-3067-deterministic-fixturevalue-3068-deterministic-fixturevalue-3069-deterministic-fixturevalue-3070-deterministic-fixturevalue-3071-deterministic-fixtureJJ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"D$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$!@@@ @@ @!@@ H@ @@ @@ @H@  H]xi ~ uX hP p ( +#  H+  +;0 hCPBFdȋ!ҌBGDdЋ!լGtd苡CHd!Hԣd CId!,I4d LCJdd8!lJdX CKdh!K$dxcF|LҖFڂ5PّӦF򈵍\QԶ@G +dG"5lޑֽG:5pQ#HR5|@HjͣH5HQ&#I5.JEH0JH2 JK40 JNXh@ JQ8 JT: JW< JZ H J`!B +Jc"D0 +JfXqe8!lR}eHڡ CSĉeX!Se`CT$ex!TTe CUe! UemCVe!VDeWte 6 dL  L6 R$M6 dMM*6!M0R%$N66)dN<NB61NHRK! ` K$ (K'&2K*X6J$K-FHb ,K0z 4K3Xfh$P8K6vȪ&pDK9(*TK?\KBXKLdSCMd!MekCNe!sN)e{CO5e!OAeơCPMe!PYeΡCQee!CRTcĵ ф5,񑴥#5 c5l5D،赬Qt#5c 5,4 LRdF#IFJiV$IHJl%JJov&J*LJrކ'JBNJuߖ(JZPJxক)KrRJ{+KVJƕ,KXJ֕-KZJ.L\hDP +f" p + f(D +%f" +1fX+D +=f" 3 If!;D0 UfX""h!CP af&!p mf*"("SD yf,""[İ fX5NXԩ9T Y=Z -!Y4(` M%ZdI8l &ZMHr)[ĪQ`x+[$x~-\TY .]]-1]a$O +6dKƖO"6lK֖O:⶟tKRPR6|KdPjⶠKP6KPⶡKR&Q6K6dQⶢKVQ6KfQ6K"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd@@ @!@@@ @@ @!@@ H@ @@ @iePxRjnԔ{>'jqդ|?Gktִ} AgRkw~AkBzԀBkrICl DSl҆!P30xf:(4x|R1Ԙ215JgR@ +ˇRT2U˧R3eJ"4u +RTRS5'S5JGS6 +T+V+ V +VI" + $+!V0&+)VI<(+1V H,S+œ9VT!+`0)+ȜQVIx2,+YV ĞDMpoBPN=uo\O=yphP>}ptQp2$RG>qb 0Sg>qDS>qTT>q)dUr"1tV>yT(ޞE+ݜVÍI+ ؁DSM+朙VFT+VHY+VIJ]+rR9V'?r够WG?rI崟XsQğY?sBYПZ?sra[?t\?tq(\t@]'@t2 L^G@ubDXQV +yT(BWy(CQXy(DQY +yT( DQZz(EQ\zԢ(E]Jz(Q^ +zT(FQ_z(GQ`zԣ( GaJz+WUV+"WeV+"*WuV+RȂW+:WԂ+BWGW+JWgW+ūW+BbWիW+rjWW+rW2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 2@ H@ @@@ @!@ UޕB VǢРJW  +X Y(2٠%JZHb 5 +[hUɒ⠒E[ U\ `J]" u +^ĉTi-ЉVi-[܉X-[Zj-[j. +\^j.  bj .\d .\$j.*\@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3097-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3098-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3099-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3100-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3101-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3102-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3103-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3104-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3105-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3106-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3107-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3108-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3109-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3110-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3111-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3112-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3113-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3114-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3115-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3116-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3117-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3118-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3119-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3120-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3121-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3122-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3123-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3124-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3125-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3126-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3127-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3128-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3129-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3130-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3131-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3132-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3133-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3134-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3135-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3136-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3137-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3138-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3139-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3140-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3141-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3142-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3143-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3144-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3145-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3146-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3147-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3148-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3149-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3150-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3151-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3152-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3153-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3154-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3155-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3156-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3157-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3158-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3159-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3160-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3161-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3162-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3163-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3164-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3165-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3166-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3167-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3168-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3169-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3170-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3171-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3172-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3173-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3174-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3175-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3176-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3177-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3178-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3179-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3180-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3181-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3182-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3183-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3184-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3185-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3186-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3187-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3188-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3189-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3190-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3191-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3192-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3193-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3194-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3195-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3196-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3197-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3198-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3199-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3200-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3201-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3202-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3203-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3204-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3205-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3206-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3207-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3208-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3209-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3210-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3211-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3212-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3213-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3214-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3215-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3216-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3217-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3218-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3219-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3220-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3221-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3222-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3223-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3224-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3225-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3226-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3227-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3228-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3229-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3230-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3231-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3232-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3233-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3234-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3235-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3236-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3237-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3238-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3239-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3240-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3241-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3242-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3243-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3244-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3245-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3246-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3247-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3248-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3249-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3250-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3251-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3252-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3253-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3254-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3255-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3256-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3257-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3258-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3259-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3260-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3261-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3262-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3263-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3264-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3265-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3266-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3267-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3268-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3269-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3270-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3271-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3272-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3273-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3274-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3275-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3276-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3277-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3278-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3279-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3280-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3281-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3282-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3283-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3284-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3285-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3286-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3287-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3288-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3289-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3290-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3291-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3292-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3293-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3294-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3295-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3296-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3297-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3298-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3299-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3300-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3301-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3302-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3303-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3304-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3305-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3306-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3307-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3308-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3309-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3310-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3311-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3312-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3313-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3314-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3315-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3316-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3317-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3318-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3319-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3320-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3321-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3322-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3323-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3324-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3325-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3326-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3327-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3328-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3329-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3330-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3331-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3332-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3333-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3334-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3335-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3336-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3337-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3338-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3339-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3340-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3341-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3342-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3343-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3344-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3345-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3346-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3347-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3348-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3349-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3350-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3351-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3352-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3353-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3354-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3355-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3356-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3357-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3358-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3359-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3360-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3361-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3362-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3363-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3364-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3365-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3366-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3367-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3368-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3369-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3370-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3371-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3372-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3373-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3374-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3375-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3376-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3377-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3378-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3379-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3380-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3381-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3382-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3383-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3384-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3385-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3386-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3387-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3388-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3389-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3390-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3391-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3392-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3393-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3394-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3395-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3396-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3397-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3398-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3399-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3400-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3401-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3402-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3403-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3404-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3405-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3406-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3407-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3408-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3409-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3410-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3411-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3412-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3413-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3414-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3415-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3416-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3417-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3418-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3419-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3420-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3421-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3422-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3423-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3424-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3425-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3426-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3427-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3428-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3429-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3430-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3431-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3432-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3433-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3434-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3435-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3436-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3437-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3438-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3439-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3440-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3441-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3442-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3443-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3444-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3445-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3446-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3447-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3448-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3449-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3450-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3451-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3452-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3453-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3454-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3455-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3456-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3457-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3458-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3459-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3460-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3461-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3462-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3463-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3464-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3465-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3466-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3467-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3468-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3469-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3470-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3471-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3472-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3473-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3474-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3475-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3476-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3477-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3478-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3479-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3480-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3481-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3482-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3483-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3484-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3485-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3486-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3487-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3488-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3489-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3490-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3491-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3492-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3493-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3494-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3495-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3496-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3497-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3498-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3499-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3500-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3501-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3502-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3503-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3504-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3505-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3506-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3507-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3508-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3509-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3510-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3511-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3512-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3513-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3514-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3515-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3516-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3517-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3518-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3519-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3520-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3521-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3522-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3523-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3524-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3525-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3526-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3527-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3528-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3529-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3530-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3531-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3532-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3533-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3534-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3535-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3536-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3537-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3538-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3539-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3540-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3541-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3542-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3543-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3544-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3545-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3546-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3547-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3548-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3549-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3550-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3551-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3552-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3553-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3554-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3555-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3556-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3557-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3558-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3559-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3560-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3561-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3562-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3563-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3564-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3565-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3566-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3567-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3568-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3569-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3570-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3571-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3572-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3573-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3574-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3575-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3576-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3577-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3578-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3579-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3580-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3581-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3582-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3583-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3584-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3585-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3586-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3587-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3588-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3589-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3590-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3591-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3592-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3593-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3594-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3595-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3596-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3597-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3598-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3599-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3600-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3601-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3602-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3603-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3604-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3605-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3606-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3607-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3608-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3609-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3610-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3611-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3612-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3613-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3614-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3615-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3616-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3617-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3618-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3619-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3620-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3621-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3622-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3623-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3624-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3625-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3626-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3627-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3628-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3629-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3630-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3631-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3632-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3633-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3634-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3635-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3636-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3637-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3638-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3639-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3640-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3641-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3642-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3643-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3644-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3645-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3646-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3647-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3648-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3649-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3650-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3651-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3652-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3653-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3654-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3655-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3656-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3657-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3658-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3659-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3660-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3661-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3662-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3663-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3664-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3665-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3666-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3667-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3668-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3669-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3670-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3671-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3672-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3673-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3674-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3675-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3676-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3677-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3678-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3679-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3680-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3681-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3682-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3683-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3684-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3685-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3686-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3687-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3688-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3689-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3690-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3691-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3692-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3693-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3694-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3695-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3696-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3697-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3698-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3699-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3700-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3701-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3702-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3703-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3704-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3705-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3706-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3707-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3708-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3709-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3710-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3711-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3712-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3713-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3714-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3715-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3716-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3717-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3718-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3719-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3720-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3721-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3722-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3723-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3724-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3725-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3726-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3727-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3728-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3729-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3730-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3731-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3732-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3733-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3734-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3735-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3736-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3737-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3738-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3739-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3740-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3741-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3742-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3743-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3744-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3745-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3746-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3747-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3748-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3749-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3750-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3751-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3752-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3753-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3754-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3755-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3756-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3757-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3758-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3759-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3760-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3761-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3762-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3763-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3764-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3765-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3766-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3767-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3768-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3769-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3770-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3771-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3772-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3773-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3774-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3775-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3776-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3777-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3778-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3779-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3780-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3781-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3782-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3783-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3784-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3785-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3786-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3787-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3788-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3789-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3790-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3791-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3792-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3793-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3794-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3795-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3796-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3797-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3798-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3799-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3800-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3801-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3802-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3803-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3804-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3805-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3806-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3807-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3808-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3809-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3810-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3811-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3812-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3813-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3814-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3815-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3816-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3817-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3818-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3819-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3820-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3821-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3822-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3823-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3824-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3825-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3826-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3827-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3828-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3829-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3830-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3831-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3832-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3833-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3834-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3835-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3836-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3837-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3838-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3839-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3840-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3841-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3842-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3843-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3844-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3845-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3846-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3847-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3848-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3849-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3850-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3851-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3852-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3853-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3854-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3855-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3856-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3857-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3858-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3859-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3860-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3861-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3862-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3863-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3864-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3865-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3866-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3867-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3868-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3869-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3870-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3871-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3872-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3873-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3874-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3875-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3876-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3877-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3878-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3879-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3880-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3881-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3882-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3883-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3884-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3885-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3886-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3887-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3888-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3889-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3890-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3891-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3892-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3893-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3894-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3895-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3896-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3897-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3898-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3899-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3900-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3901-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3902-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3903-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3904-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3905-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3906-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3907-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3908-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3909-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3910-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3911-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3912-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3913-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3914-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3915-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3916-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3917-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3918-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3919-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3920-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3921-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3922-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3923-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3924-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3925-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3926-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3927-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3928-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3929-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3930-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3931-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3932-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3933-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3934-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3935-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3936-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3937-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3938-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3939-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3940-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3941-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3942-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3943-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3944-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3945-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3946-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3947-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3948-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3949-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3950-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3951-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3952-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3953-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3954-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3955-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3956-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3957-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3958-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3959-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3960-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3961-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3962-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3963-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3964-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3965-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3966-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3967-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3968-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3969-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3970-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3971-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3972-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3973-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3974-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3975-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3976-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3977-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3978-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3979-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3980-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3981-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3982-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3983-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3984-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3985-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3986-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3987-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3988-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3989-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3990-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3991-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3992-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3993-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3994-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3995-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3996-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3997-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3998-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-3999-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4000-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4001-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4002-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4003-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4004-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4005-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4006-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4007-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4008-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4009-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4010-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4011-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4012-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4013-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4014-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4015-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4016-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4017-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4018-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4019-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4020-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4021-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4022-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4023-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4024-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4025-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4026-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4027-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4028-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4029-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4030-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4031-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4032-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4033-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4034-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4035-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4036-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4037-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4038-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4039-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4040-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4041-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4042-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4043-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4044-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4045-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4046-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4047-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4048-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4049-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4050-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4051-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4052-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4053-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4054-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4055-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4056-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4057-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4058-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4059-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4060-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4061-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4062-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4063-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4064-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4065-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4066-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4067-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4068-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4069-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4070-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4071-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4072-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4073-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4074-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4075-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4076-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4077-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4078-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4079-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4080-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4081-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4082-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4083-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4084-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4085-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4086-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4087-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4088-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4089-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4090-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4091-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4092-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4093-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4094-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4095-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH ,<L\l| + +  ,<L\l| -.=>MN]^mn}~ +  -.=>MN]^mn}~/?O_o + /?O_o@ @AAB,BCMDN]E^mFn}G~HIJKLMNO PQ-R.=S>MTN]U^mVn}W~XYZ[\]^_@AB/C?DOE_FoGHIJKLMNOPQR/S?TOU_VoWXYZ[\]^_ !!","#<#$L$%\%&l&'|'(())**++,,--..//0 0112,23<34L45\56l67|78899::;;<<==>>?? !-".=#>M$N]%^m&n}'~()*+,-./ 01-2.=3>M4N]5^m6n}7~89:;<=>? !"/#?$O%_&o'()*+,-./012/3?4O5_6o789:;<=>?` `aab,bcMdN]e^mfn}g~hijklmno pq-r.=s>MtN]u^mvn}w~xyz{|}~`ab/c?dOe_foghijklmnopqr/s?tOu_vowxyz{|}~HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHg g ` HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3072-deterministic-fixturevalue-3074-deterministic-fixturevalue-3075-deterministic-fixturevalue-3076-deterministic-fixturevalue-3077-deterministic-fixturevalue-3078-deterministic-fixturevalue-3079-deterministic-fixturevalue-3081-deterministic-fixturevalue-3082-deterministic-fixturevalue-3083-deterministic-fixturevalue-3084-deterministic-fixturevalue-3085-deterministic-fixturevalue-3086-deterministic-fixturevalue-3088-deterministic-fixturevalue-3089-deterministic-fixturevalue-3090-deterministic-fixturevalue-3091-deterministic-fixturevalue-3092-deterministic-fixturevalue-3093-deterministic-fixturevalue-3095-deterministic-fixturevalue-3096-deterministic-fixturevalue-3097-deterministic-fixturevalue-3098-deterministic-fixturevalue-3099-deterministic-fixturevalue-3100-deterministic-fixturevalue-3102-deterministic-fixturevalue-3103-deterministic-fixturevalue-3104-deterministic-fixturevalue-3105-deterministic-fixturevalue-3106-deterministic-fixturevalue-3107-deterministic-fixturevalue-3109-deterministic-fixturevalue-3110-deterministic-fixturevalue-3111-deterministic-fixturevalue-3112-deterministic-fixturevalue-3113-deterministic-fixturevalue-3114-deterministic-fixturevalue-3116-deterministic-fixturevalue-3117-deterministic-fixturevalue-3118-deterministic-fixturevalue-3119-deterministic-fixturevalue-3120-deterministic-fixturevalue-3121-deterministic-fixturevalue-3123-deterministic-fixturevalue-3124-deterministic-fixturevalue-3125-deterministic-fixturevalue-3126-deterministic-fixturevalue-3127-deterministic-fixturevalue-3128-deterministic-fixturevalue-3130-deterministic-fixturevalue-3131-deterministic-fixturevalue-3132-deterministic-fixturevalue-3133-deterministic-fixturevalue-3134-deterministic-fixturevalue-3135-deterministic-fixturevalue-3137-deterministic-fixturevalue-3138-deterministic-fixturevalue-3139-deterministic-fixturevalue-3140-deterministic-fixturevalue-3141-deterministic-fixturevalue-3142-deterministic-fixturevalue-3144-deterministic-fixturevalue-3145-deterministic-fixturevalue-3146-deterministic-fixturevalue-3147-deterministic-fixturevalue-3148-deterministic-fixturevalue-3149-deterministic-fixturevalue-3151-deterministic-fixturevalue-3152-deterministic-fixturevalue-3153-deterministic-fixturevalue-3154-deterministic-fixturevalue-3155-deterministic-fixturevalue-3156-deterministic-fixturevalue-3158-deterministic-fixturevalue-3159-deterministic-fixturevalue-3160-deterministic-fixturevalue-3161-deterministic-fixturevalue-3162-deterministic-fixturevalue-3163-deterministic-fixturevalue-3165-deterministic-fixturevalue-3166-deterministic-fixturevalue-3167-deterministic-fixturevalue-3168-deterministic-fixturevalue-3169-deterministic-fixturevalue-3170-deterministic-fixturevalue-3172-deterministic-fixturevalue-3173-deterministic-fixturevalue-3174-deterministic-fixturevalue-3175-deterministic-fixturevalue-3176-deterministic-fixturevalue-3177-deterministic-fixturevalue-3179-deterministic-fixturevalue-3180-deterministic-fixturevalue-3181-deterministic-fixturevalue-3182-deterministic-fixturevalue-3183-deterministic-fixturevalue-3184-deterministic-fixturevalue-3186-deterministic-fixturevalue-3187-deterministic-fixturevalue-3188-deterministic-fixturevalue-3189-deterministic-fixturevalue-3190-deterministic-fixturevalue-3191-deterministic-fixturevalue-3193-deterministic-fixturevalue-3194-deterministic-fixturevalue-3195-deterministic-fixturevalue-3196-deterministic-fixturevalue-3197-deterministic-fixturevalue-3198-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3200-deterministic-fixturevalue-3201-deterministic-fixturevalue-3202-deterministic-fixturevalue-3203-deterministic-fixturevalue-3204-deterministic-fixturevalue-3205-deterministic-fixturevalue-3207-deterministic-fixturevalue-3208-deterministic-fixturevalue-3209-deterministic-fixturevalue-3210-deterministic-fixturevalue-3211-deterministic-fixturevalue-3212-deterministic-fixturevalue-3214-deterministic-fixturevalue-3215-deterministic-fixturevalue-3216-deterministic-fixturevalue-3217-deterministic-fixturevalue-3218-deterministic-fixturevalue-3219-deterministic-fixturevalue-3221-deterministic-fixturevalue-3222-deterministic-fixturevalue-3223-deterministic-fixturevalue-3224-deterministic-fixturevalue-3225-deterministic-fixturevalue-3226-deterministic-fixturevalue-3228-deterministic-fixturevalue-3229-deterministic-fixturevalue-3230-deterministic-fixturevalue-3231-deterministic-fixturevalue-3232-deterministic-fixturevalue-3233-deterministic-fixturevalue-3235-deterministic-fixturevalue-3236-deterministic-fixturevalue-3237-deterministic-fixturevalue-3238-deterministic-fixturevalue-3239-deterministic-fixturevalue-3240-deterministic-fixturevalue-3242-deterministic-fixturevalue-3243-deterministic-fixturevalue-3244-deterministic-fixturevalue-3245-deterministic-fixturevalue-3246-deterministic-fixturevalue-3247-deterministic-fixturevalue-3249-deterministic-fixturevalue-3250-deterministic-fixturevalue-3251-deterministic-fixturevalue-3252-deterministic-fixturevalue-3253-deterministic-fixturevalue-3254-deterministic-fixturevalue-3256-deterministic-fixturevalue-3257-deterministic-fixturevalue-3258-deterministic-fixturevalue-3259-deterministic-fixturevalue-3260-deterministic-fixturevalue-3261-deterministic-fixturevalue-3263-deterministic-fixturevalue-3264-deterministic-fixturevalue-3265-deterministic-fixturevalue-3266-deterministic-fixturevalue-3267-deterministic-fixturevalue-3268-deterministic-fixturevalue-3270-deterministic-fixturevalue-3271-deterministic-fixturevalue-3272-deterministic-fixturevalue-3273-deterministic-fixturevalue-3274-deterministic-fixturevalue-3275-deterministic-fixturevalue-3277-deterministic-fixturevalue-3278-deterministic-fixturevalue-3279-deterministic-fixturevalue-3280-deterministic-fixturevalue-3281-deterministic-fixturevalue-3282-deterministic-fixturevalue-3284-deterministic-fixturevalue-3285-deterministic-fixturevalue-3286-deterministic-fixturevalue-3287-deterministic-fixturevalue-3288-deterministic-fixturevalue-3289-deterministic-fixturevalue-3291-deterministic-fixturevalue-3292-deterministic-fixturevalue-3293-deterministic-fixturevalue-3294-deterministic-fixturevalue-3295-deterministic-fixturevalue-3296-deterministic-fixturevalue-3298-deterministic-fixturevalue-3299-deterministic-fixturevalue-3300-deterministic-fixturevalue-3301-deterministic-fixturevalue-3302-deterministic-fixturevalue-3303-deterministic-fixturevalue-3305-deterministic-fixturevalue-3306-deterministic-fixturevalue-3307-deterministic-fixturevalue-3308-deterministic-fixturevalue-3309-deterministic-fixturevalue-3310-deterministic-fixturevalue-3312-deterministic-fixturevalue-3313-deterministic-fixturevalue-3314-deterministic-fixturevalue-3315-deterministic-fixturevalue-3316-deterministic-fixturevalue-3317-deterministic-fixturevalue-3319-deterministic-fixturevalue-3320-deterministic-fixturevalue-3321-deterministic-fixturevalue-3322-deterministic-fixturevalue-3323-deterministic-fixturevalue-3324-deterministic-fixturevalue-3326-deterministic-fixturevalue-3327-deterministic-fixture$Dd$DDd$Dd$Dd$DDd$Dd$Dd $ D D d  + +$ +D +d + + + + + + $ D d  $ D D d   $ D d $Dd$DDdvalue-3328-deterministic-fixturevalue-3329-deterministic-fixturevalue-3330-deterministic-fixturevalue-3331-deterministic-fixturevalue-3333-deterministic-fixturevalue-3334-deterministic-fixturevalue-3335-deterministic-fixturevalue-3336-deterministic-fixturevalue-3337-deterministic-fixturevalue-3338-deterministic-fixturevalue-3340-deterministic-fixturevalue-3341-deterministic-fixturevalue-3342-deterministic-fixturevalue-3343-deterministic-fixturevalue-3344-deterministic-fixturevalue-3345-deterministic-fixturevalue-3347-deterministic-fixturevalue-3348-deterministic-fixturevalue-3349-deterministic-fixturevalue-3350-deterministic-fixturevalue-3351-deterministic-fixturevalue-3352-deterministic-fixturevalue-3354-deterministic-fixturevalue-3355-deterministic-fixturevalue-3356-deterministic-fixturevalue-3357-deterministic-fixturevalue-3358-deterministic-fixturevalue-3359-deterministic-fixturevalue-3361-deterministic-fixturevalue-3362-deterministic-fixturevalue-3363-deterministic-fixturevalue-3364-deterministic-fixturevalue-3365-deterministic-fixturevalue-3366-deterministic-fixturevalue-3368-deterministic-fixturevalue-3369-deterministic-fixturevalue-3370-deterministic-fixturevalue-3371-deterministic-fixturevalue-3372-deterministic-fixturevalue-3373-deterministic-fixturevalue-3375-deterministic-fixturevalue-3376-deterministic-fixturevalue-3377-deterministic-fixturevalue-3378-deterministic-fixturevalue-3379-deterministic-fixturevalue-3380-deterministic-fixturevalue-3382-deterministic-fixturevalue-3383-deterministic-fixturevalue-3384-deterministic-fixturevalue-3385-deterministic-fixturevalue-3386-deterministic-fixturevalue-3387-deterministic-fixturevalue-3389-deterministic-fixturevalue-3390-deterministic-fixturevalue-3391-deterministic-fixturevalue-3392-deterministic-fixturevalue-3393-deterministic-fixturevalue-3394-deterministic-fixturevalue-3396-deterministic-fixturevalue-3397-deterministic-fixturevalue-3398-deterministic-fixturevalue-3399-deterministic-fixturevalue-3400-deterministic-fixturevalue-3401-deterministic-fixturevalue-3403-deterministic-fixturevalue-3404-deterministic-fixturevalue-3405-deterministic-fixturevalue-3406-deterministic-fixturevalue-3407-deterministic-fixturevalue-3408-deterministic-fixturevalue-3410-deterministic-fixturevalue-3411-deterministic-fixturevalue-3412-deterministic-fixturevalue-3413-deterministic-fixturevalue-3414-deterministic-fixturevalue-3415-deterministic-fixturevalue-3417-deterministic-fixturevalue-3418-deterministic-fixturevalue-3419-deterministic-fixturevalue-3420-deterministic-fixturevalue-3421-deterministic-fixturevalue-3422-deterministic-fixturevalue-3424-deterministic-fixturevalue-3425-deterministic-fixturevalue-3426-deterministic-fixturevalue-3427-deterministic-fixturevalue-3428-deterministic-fixturevalue-3429-deterministic-fixturevalue-3431-deterministic-fixturevalue-3432-deterministic-fixturevalue-3433-deterministic-fixturevalue-3434-deterministic-fixturevalue-3435-deterministic-fixturevalue-3436-deterministic-fixturevalue-3438-deterministic-fixturevalue-3439-deterministic-fixturevalue-3440-deterministic-fixturevalue-3441-deterministic-fixturevalue-3442-deterministic-fixturevalue-3443-deterministic-fixturevalue-3445-deterministic-fixturevalue-3446-deterministic-fixturevalue-3447-deterministic-fixturevalue-3448-deterministic-fixturevalue-3449-deterministic-fixturevalue-3450-deterministic-fixturevalue-3452-deterministic-fixturevalue-3453-deterministic-fixturevalue-3454-deterministic-fixturevalue-3455-deterministic-fixture$DDd$Dd$Dd$DDd$Dd$Dd$DDd  $ D d  +$ +D +d + + + + + + $ D D d   $ D d  $ D d $DDd$Ddvalue-3456-deterministic-fixturevalue-3457-deterministic-fixturevalue-3459-deterministic-fixturevalue-3460-deterministic-fixturevalue-3461-deterministic-fixturevalue-3462-deterministic-fixturevalue-3463-deterministic-fixturevalue-3464-deterministic-fixturevalue-3466-deterministic-fixturevalue-3467-deterministic-fixturevalue-3468-deterministic-fixturevalue-3469-deterministic-fixturevalue-3470-deterministic-fixturevalue-3471-deterministic-fixturevalue-3473-deterministic-fixturevalue-3474-deterministic-fixturevalue-3475-deterministic-fixturevalue-3476-deterministic-fixturevalue-3477-deterministic-fixturevalue-3478-deterministic-fixturevalue-3480-deterministic-fixturevalue-3481-deterministic-fixturevalue-3482-deterministic-fixturevalue-3483-deterministic-fixturevalue-3484-deterministic-fixturevalue-3485-deterministic-fixturevalue-3487-deterministic-fixturevalue-3488-deterministic-fixturevalue-3489-deterministic-fixturevalue-3490-deterministic-fixturevalue-3491-deterministic-fixturevalue-3492-deterministic-fixturevalue-3494-deterministic-fixturevalue-3495-deterministic-fixturevalue-3496-deterministic-fixturevalue-3497-deterministic-fixturevalue-3498-deterministic-fixturevalue-3499-deterministic-fixturevalue-3501-deterministic-fixturevalue-3502-deterministic-fixturevalue-3503-deterministic-fixturevalue-3504-deterministic-fixturevalue-3505-deterministic-fixturevalue-3506-deterministic-fixturevalue-3508-deterministic-fixturevalue-3509-deterministic-fixturevalue-3510-deterministic-fixturevalue-3511-deterministic-fixturevalue-3512-deterministic-fixturevalue-3513-deterministic-fixturevalue-3515-deterministic-fixturevalue-3516-deterministic-fixturevalue-3517-deterministic-fixturevalue-3518-deterministic-fixturevalue-3519-deterministic-fixturevalue-3520-deterministic-fixturevalue-3522-deterministic-fixturevalue-3523-deterministic-fixturevalue-3524-deterministic-fixturevalue-3525-deterministic-fixturevalue-3526-deterministic-fixturevalue-3527-deterministic-fixturevalue-3529-deterministic-fixturevalue-3530-deterministic-fixturevalue-3531-deterministic-fixturevalue-3532-deterministic-fixturevalue-3533-deterministic-fixturevalue-3534-deterministic-fixturevalue-3536-deterministic-fixturevalue-3537-deterministic-fixturevalue-3538-deterministic-fixturevalue-3539-deterministic-fixturevalue-3540-deterministic-fixturevalue-3541-deterministic-fixturevalue-3543-deterministic-fixturevalue-3544-deterministic-fixturevalue-3545-deterministic-fixturevalue-3546-deterministic-fixturevalue-3547-deterministic-fixturevalue-3548-deterministic-fixturevalue-3550-deterministic-fixturevalue-3551-deterministic-fixturevalue-3552-deterministic-fixturevalue-3553-deterministic-fixturevalue-3554-deterministic-fixturevalue-3555-deterministic-fixturevalue-3557-deterministic-fixturevalue-3558-deterministic-fixturevalue-3559-deterministic-fixturevalue-3560-deterministic-fixturevalue-3561-deterministic-fixturevalue-3562-deterministic-fixturevalue-3564-deterministic-fixturevalue-3565-deterministic-fixturevalue-3566-deterministic-fixturevalue-3567-deterministic-fixturevalue-3568-deterministic-fixturevalue-3569-deterministic-fixturevalue-3571-deterministic-fixturevalue-3572-deterministic-fixturevalue-3573-deterministic-fixturevalue-3574-deterministic-fixturevalue-3575-deterministic-fixturevalue-3576-deterministic-fixturevalue-3578-deterministic-fixturevalue-3579-deterministic-fixturevalue-3580-deterministic-fixturevalue-3581-deterministic-fixturevalue-3582-deterministic-fixturevalue-3583-deterministic-fixture$Dd$Dd$DDd$Dd$Dd$DDd$Dd $ D d  +$ +D +D +d + + + + +  $ D d  $ D d  $ D D d $Dd$Ddvalue-3585-deterministic-fixturevalue-3586-deterministic-fixturevalue-3587-deterministic-fixturevalue-3588-deterministic-fixturevalue-3589-deterministic-fixturevalue-3590-deterministic-fixturevalue-3592-deterministic-fixturevalue-3593-deterministic-fixturevalue-3594-deterministic-fixturevalue-3595-deterministic-fixturevalue-3596-deterministic-fixturevalue-3597-deterministic-fixturevalue-3599-deterministic-fixturevalue-3600-deterministic-fixturevalue-3601-deterministic-fixturevalue-3602-deterministic-fixturevalue-3603-deterministic-fixturevalue-3604-deterministic-fixturevalue-3606-deterministic-fixturevalue-3607-deterministic-fixturevalue-3608-deterministic-fixturevalue-3609-deterministic-fixturevalue-3610-deterministic-fixturevalue-3611-deterministic-fixturevalue-3613-deterministic-fixturevalue-3614-deterministic-fixturevalue-3615-deterministic-fixturevalue-3616-deterministic-fixturevalue-3617-deterministic-fixturevalue-3618-deterministic-fixturevalue-3620-deterministic-fixturevalue-3621-deterministic-fixturevalue-3622-deterministic-fixturevalue-3623-deterministic-fixturevalue-3624-deterministic-fixturevalue-3625-deterministic-fixturevalue-3627-deterministic-fixturevalue-3628-deterministic-fixturevalue-3629-deterministic-fixturevalue-3630-deterministic-fixturevalue-3631-deterministic-fixturevalue-3632-deterministic-fixturevalue-3634-deterministic-fixturevalue-3635-deterministic-fixturevalue-3636-deterministic-fixturevalue-3637-deterministic-fixturevalue-3638-deterministic-fixturevalue-3639-deterministic-fixturevalue-3641-deterministic-fixturevalue-3642-deterministic-fixturevalue-3643-deterministic-fixturevalue-3644-deterministic-fixturevalue-3645-deterministic-fixturevalue-3646-deterministic-fixturevalue-3648-deterministic-fixturevalue-3649-deterministic-fixturevalue-3650-deterministic-fixturevalue-3651-deterministic-fixturevalue-3652-deterministic-fixturevalue-3653-deterministic-fixturevalue-3655-deterministic-fixturevalue-3656-deterministic-fixturevalue-3657-deterministic-fixturevalue-3658-deterministic-fixturevalue-3659-deterministic-fixturevalue-3660-deterministic-fixturevalue-3662-deterministic-fixturevalue-3663-deterministic-fixturevalue-3664-deterministic-fixturevalue-3665-deterministic-fixturevalue-3666-deterministic-fixturevalue-3667-deterministic-fixturevalue-3669-deterministic-fixturevalue-3670-deterministic-fixturevalue-3671-deterministic-fixturevalue-3672-deterministic-fixturevalue-3673-deterministic-fixturevalue-3674-deterministic-fixturevalue-3676-deterministic-fixturevalue-3677-deterministic-fixturevalue-3678-deterministic-fixturevalue-3679-deterministic-fixturevalue-3680-deterministic-fixturevalue-3681-deterministic-fixturevalue-3683-deterministic-fixturevalue-3684-deterministic-fixturevalue-3685-deterministic-fixturevalue-3686-deterministic-fixturevalue-3687-deterministic-fixturevalue-3688-deterministic-fixturevalue-3690-deterministic-fixturevalue-3691-deterministic-fixturevalue-3692-deterministic-fixturevalue-3693-deterministic-fixturevalue-3694-deterministic-fixturevalue-3695-deterministic-fixturevalue-3697-deterministic-fixturevalue-3698-deterministic-fixturevalue-3699-deterministic-fixturevalue-3700-deterministic-fixturevalue-3701-deterministic-fixturevalue-3702-deterministic-fixturevalue-3704-deterministic-fixturevalue-3705-deterministic-fixturevalue-3706-deterministic-fixturevalue-3707-deterministic-fixturevalue-3708-deterministic-fixturevalue-3709-deterministic-fixturevalue-3711-deterministic-fixture$Dd$Ddd$$Dd$Dd$Ddd$$Dd$Dd $ D d d  +$ +$ +D +d + + + + + + $ D d  $ D d d  $ $ D d $Dd$Dddvalue-3712-deterministic-fixturevalue-3713-deterministic-fixturevalue-3714-deterministic-fixturevalue-3715-deterministic-fixturevalue-3716-deterministic-fixturevalue-3718-deterministic-fixturevalue-3719-deterministic-fixturevalue-3720-deterministic-fixturevalue-3721-deterministic-fixturevalue-3722-deterministic-fixturevalue-3723-deterministic-fixturevalue-3725-deterministic-fixturevalue-3726-deterministic-fixturevalue-3727-deterministic-fixturevalue-3728-deterministic-fixturevalue-3729-deterministic-fixturevalue-3730-deterministic-fixturevalue-3732-deterministic-fixturevalue-3733-deterministic-fixturevalue-3734-deterministic-fixturevalue-3735-deterministic-fixturevalue-3736-deterministic-fixturevalue-3737-deterministic-fixturevalue-3739-deterministic-fixturevalue-3740-deterministic-fixturevalue-3741-deterministic-fixturevalue-3742-deterministic-fixturevalue-3743-deterministic-fixturevalue-3744-deterministic-fixturevalue-3746-deterministic-fixturevalue-3747-deterministic-fixturevalue-3748-deterministic-fixturevalue-3749-deterministic-fixturevalue-3750-deterministic-fixturevalue-3751-deterministic-fixturevalue-3753-deterministic-fixturevalue-3754-deterministic-fixturevalue-3755-deterministic-fixturevalue-3756-deterministic-fixturevalue-3757-deterministic-fixturevalue-3758-deterministic-fixturevalue-3760-deterministic-fixturevalue-3761-deterministic-fixturevalue-3762-deterministic-fixturevalue-3763-deterministic-fixturevalue-3764-deterministic-fixturevalue-3765-deterministic-fixturevalue-3767-deterministic-fixturevalue-3768-deterministic-fixturevalue-3769-deterministic-fixturevalue-3770-deterministic-fixturevalue-3771-deterministic-fixturevalue-3772-deterministic-fixturevalue-3774-deterministic-fixturevalue-3775-deterministic-fixturevalue-3776-deterministic-fixturevalue-3777-deterministic-fixturevalue-3778-deterministic-fixturevalue-3779-deterministic-fixturevalue-3781-deterministic-fixturevalue-3782-deterministic-fixturevalue-3783-deterministic-fixturevalue-3784-deterministic-fixturevalue-3785-deterministic-fixturevalue-3786-deterministic-fixturevalue-3788-deterministic-fixturevalue-3789-deterministic-fixturevalue-3790-deterministic-fixturevalue-3791-deterministic-fixturevalue-3792-deterministic-fixturevalue-3793-deterministic-fixturevalue-3795-deterministic-fixturevalue-3796-deterministic-fixturevalue-3797-deterministic-fixturevalue-3798-deterministic-fixturevalue-3799-deterministic-fixturevalue-3800-deterministic-fixturevalue-3802-deterministic-fixturevalue-3803-deterministic-fixturevalue-3804-deterministic-fixturevalue-3805-deterministic-fixturevalue-3806-deterministic-fixturevalue-3807-deterministic-fixturevalue-3809-deterministic-fixturevalue-3810-deterministic-fixturevalue-3811-deterministic-fixturevalue-3812-deterministic-fixturevalue-3813-deterministic-fixturevalue-3814-deterministic-fixturevalue-3816-deterministic-fixturevalue-3817-deterministic-fixturevalue-3818-deterministic-fixturevalue-3819-deterministic-fixturevalue-3820-deterministic-fixturevalue-3821-deterministic-fixturevalue-3823-deterministic-fixturevalue-3824-deterministic-fixturevalue-3825-deterministic-fixturevalue-3826-deterministic-fixturevalue-3827-deterministic-fixturevalue-3828-deterministic-fixturevalue-3830-deterministic-fixturevalue-3831-deterministic-fixturevalue-3832-deterministic-fixturevalue-3833-deterministic-fixturevalue-3834-deterministic-fixturevalue-3835-deterministic-fixturevalue-3837-deterministic-fixturevalue-3838-deterministic-fixturevalue-3839-deterministic-fixture$Ddd$$Dd$Dd$Ddd$$Dd$Dd$Ddd $ $ D d  +$ +D +d + + + + + + $ D d d  $ $ D d  $ D d $Ddd$$Ddvalue-3840-deterministic-fixturevalue-3841-deterministic-fixturevalue-3842-deterministic-fixturevalue-3844-deterministic-fixturevalue-3845-deterministic-fixturevalue-3846-deterministic-fixturevalue-3847-deterministic-fixturevalue-3848-deterministic-fixturevalue-3849-deterministic-fixturevalue-3851-deterministic-fixturevalue-3852-deterministic-fixturevalue-3853-deterministic-fixturevalue-3854-deterministic-fixturevalue-3855-deterministic-fixturevalue-3856-deterministic-fixturevalue-3858-deterministic-fixturevalue-3859-deterministic-fixturevalue-3860-deterministic-fixturevalue-3861-deterministic-fixturevalue-3862-deterministic-fixturevalue-3863-deterministic-fixturevalue-3865-deterministic-fixturevalue-3866-deterministic-fixturevalue-3867-deterministic-fixturevalue-3868-deterministic-fixturevalue-3869-deterministic-fixturevalue-3870-deterministic-fixturevalue-3872-deterministic-fixturevalue-3873-deterministic-fixturevalue-3874-deterministic-fixturevalue-3875-deterministic-fixturevalue-3876-deterministic-fixturevalue-3877-deterministic-fixturevalue-3879-deterministic-fixturevalue-3880-deterministic-fixturevalue-3881-deterministic-fixturevalue-3882-deterministic-fixturevalue-3883-deterministic-fixturevalue-3884-deterministic-fixturevalue-3886-deterministic-fixturevalue-3887-deterministic-fixturevalue-3888-deterministic-fixturevalue-3889-deterministic-fixturevalue-3890-deterministic-fixturevalue-3891-deterministic-fixturevalue-3893-deterministic-fixturevalue-3894-deterministic-fixturevalue-3895-deterministic-fixturevalue-3896-deterministic-fixturevalue-3897-deterministic-fixturevalue-3898-deterministic-fixturevalue-3900-deterministic-fixturevalue-3901-deterministic-fixturevalue-3902-deterministic-fixturevalue-3903-deterministic-fixturevalue-3904-deterministic-fixturevalue-3905-deterministic-fixturevalue-3907-deterministic-fixturevalue-3908-deterministic-fixturevalue-3909-deterministic-fixturevalue-3910-deterministic-fixturevalue-3911-deterministic-fixturevalue-3912-deterministic-fixturevalue-3914-deterministic-fixturevalue-3915-deterministic-fixturevalue-3916-deterministic-fixturevalue-3917-deterministic-fixturevalue-3918-deterministic-fixturevalue-3919-deterministic-fixturevalue-3921-deterministic-fixturevalue-3922-deterministic-fixturevalue-3923-deterministic-fixturevalue-3924-deterministic-fixturevalue-3925-deterministic-fixturevalue-3926-deterministic-fixturevalue-3928-deterministic-fixturevalue-3929-deterministic-fixturevalue-3930-deterministic-fixturevalue-3931-deterministic-fixturevalue-3932-deterministic-fixturevalue-3933-deterministic-fixturevalue-3935-deterministic-fixturevalue-3936-deterministic-fixturevalue-3937-deterministic-fixturevalue-3938-deterministic-fixturevalue-3939-deterministic-fixturevalue-3940-deterministic-fixturevalue-3942-deterministic-fixturevalue-3943-deterministic-fixturevalue-3944-deterministic-fixturevalue-3945-deterministic-fixturevalue-3946-deterministic-fixturevalue-3947-deterministic-fixturevalue-3949-deterministic-fixturevalue-3950-deterministic-fixturevalue-3951-deterministic-fixturevalue-3952-deterministic-fixturevalue-3953-deterministic-fixturevalue-3954-deterministic-fixturevalue-3956-deterministic-fixturevalue-3957-deterministic-fixturevalue-3958-deterministic-fixturevalue-3959-deterministic-fixturevalue-3960-deterministic-fixturevalue-3961-deterministic-fixturevalue-3963-deterministic-fixturevalue-3964-deterministic-fixturevalue-3965-deterministic-fixturevalue-3966-deterministic-fixturevalue-3967-deterministic-fixture$$Dd$Dd$Ddd$$Dd$Dd$Ddd$$Dd $ D d  +$ +D +d +d + + + + + $ $ D d  $ D d  $ D d d $$Dd$Ddvalue-3968-deterministic-fixturevalue-3970-deterministic-fixturevalue-3971-deterministic-fixturevalue-3972-deterministic-fixturevalue-3973-deterministic-fixturevalue-3974-deterministic-fixturevalue-3975-deterministic-fixturevalue-3977-deterministic-fixturevalue-3978-deterministic-fixturevalue-3979-deterministic-fixturevalue-3980-deterministic-fixturevalue-3981-deterministic-fixturevalue-3982-deterministic-fixturevalue-3984-deterministic-fixturevalue-3985-deterministic-fixturevalue-3986-deterministic-fixturevalue-3987-deterministic-fixturevalue-3988-deterministic-fixturevalue-3989-deterministic-fixturevalue-3991-deterministic-fixturevalue-3992-deterministic-fixturevalue-3993-deterministic-fixturevalue-3994-deterministic-fixturevalue-3995-deterministic-fixturevalue-3996-deterministic-fixturevalue-3998-deterministic-fixturevalue-3999-deterministic-fixturevalue-4000-deterministic-fixturevalue-4001-deterministic-fixturevalue-4002-deterministic-fixturevalue-4003-deterministic-fixturevalue-4005-deterministic-fixturevalue-4006-deterministic-fixturevalue-4007-deterministic-fixturevalue-4008-deterministic-fixturevalue-4009-deterministic-fixturevalue-4010-deterministic-fixturevalue-4012-deterministic-fixturevalue-4013-deterministic-fixturevalue-4014-deterministic-fixturevalue-4015-deterministic-fixturevalue-4016-deterministic-fixturevalue-4017-deterministic-fixturevalue-4019-deterministic-fixturevalue-4020-deterministic-fixturevalue-4021-deterministic-fixturevalue-4022-deterministic-fixturevalue-4023-deterministic-fixturevalue-4024-deterministic-fixturevalue-4026-deterministic-fixturevalue-4027-deterministic-fixturevalue-4028-deterministic-fixturevalue-4029-deterministic-fixturevalue-4030-deterministic-fixturevalue-4031-deterministic-fixturevalue-4033-deterministic-fixturevalue-4034-deterministic-fixturevalue-4035-deterministic-fixturevalue-4036-deterministic-fixturevalue-4037-deterministic-fixturevalue-4038-deterministic-fixturevalue-4040-deterministic-fixturevalue-4041-deterministic-fixturevalue-4042-deterministic-fixturevalue-4043-deterministic-fixturevalue-4044-deterministic-fixturevalue-4045-deterministic-fixturevalue-4047-deterministic-fixturevalue-4048-deterministic-fixturevalue-4049-deterministic-fixturevalue-4050-deterministic-fixturevalue-4051-deterministic-fixturevalue-4052-deterministic-fixturevalue-4054-deterministic-fixturevalue-4055-deterministic-fixturevalue-4056-deterministic-fixturevalue-4057-deterministic-fixturevalue-4058-deterministic-fixturevalue-4059-deterministic-fixturevalue-4061-deterministic-fixturevalue-4062-deterministic-fixturevalue-4063-deterministic-fixturevalue-4064-deterministic-fixturevalue-4065-deterministic-fixturevalue-4066-deterministic-fixturevalue-4068-deterministic-fixturevalue-4069-deterministic-fixturevalue-4070-deterministic-fixturevalue-4071-deterministic-fixturevalue-4072-deterministic-fixturevalue-4073-deterministic-fixturevalue-4075-deterministic-fixturevalue-4076-deterministic-fixturevalue-4077-deterministic-fixturevalue-4078-deterministic-fixturevalue-4079-deterministic-fixturevalue-4080-deterministic-fixturevalue-4082-deterministic-fixturevalue-4083-deterministic-fixturevalue-4084-deterministic-fixturevalue-4085-deterministic-fixturevalue-4086-deterministic-fixturevalue-4087-deterministic-fixturevalue-4089-deterministic-fixturevalue-4090-deterministic-fixturevalue-4091-deterministic-fixturevalue-4092-deterministic-fixturevalue-4093-deterministic-fixturevalue-4094-deterministic-fixtureHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHJJ HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH"Ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$dd @@ @H@ @@ @!@@@ @@ @!@@ " sL# $ $X $) 3 $&%  c 6L&i +X $@' $V( f L)#$v + * XS * 421QjdŜ924QrdŨz=27Qƴ|A2:Rd~D2=Rd̖I2@RdؖM2CRd䖄Q2FR𖈌U2LSdX2OSda2RSd&uь):tuLI;t&v Y;Cuv̱;su&ьɣA5]$j @E5`j!L̟M5c$j" DXQ5fU"#Vǰ4U*#vǼ5鎙SV2$6 V:6)V%8IVJ&9 WR&9CWZ:sWb';ɏWr((<鏽Wz(4e@2y&eԌ@2|eL—A2&e ×A2&׌ėA2&eLŗB2&e Ɨ@B2eƘB2&e܌ǘB2eLȘC2&e ɘYC 5yk ӀI5cy6ӌi5ykFӘ 5ykVӤ5y"k`Ӱɦ5#z*kvӼ香5SzȠ 5z:kԠ 5zJkI5zRki5{!$ +)$ L,I'$ - XC*-s-$. 0$  0"X3 0 6$!1 (3Sw& Y>w|̺)>w}L ?w&~ iY?x&̽?Cx&Ҍ?sxLɥ@x& Y@xӢj#FT5i$j$HY5l$j%LJ]5o$& a5r$j' Nd5uj(L,Pi5x$j* 8TXm5~j+Dq5$,PXy5j-LhZ|5$j. t\X< X)&@=IcX0L>iX*FX?X+Vd@ X+fpAɐ#Yp|BSY,ȈB Y-ȔC)Y-ȬD Y.ȸEiZ2eɘC2&eʘC2L˘YD2&e̘ęD2&e͘D2eLΘE2&e ϘYE2ϘΙE2&eИE2eLјF2&e 5s{bk 5{jk駹5{rk( 5|zk4)53|&@I5c|k6L 5|kFX5|kPp5|f|ɨ5#}kԈ 5"Dd2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2D2D2D"D"L"L"L&L&H&H&H$H$I$I$ɒ$$ddd2d2DH@ @@ @@ @H@ @@ @!@@@ @@ @ie(Vx +@jnԖ{> +jqզ|M@Jkt(} AjZkwAkC}(րBksMC +l( D[lӆ("p308*p4޴| +Ҁ*1' +ܘc17Nj*2GˊҜû*3W̪ܙ3gN#*4w +\S4 +Ӝ**ܚ5NJ*6j\;v;( +v ;vM"[ ;v$$;(0;*vM<*;(2v H,[;(:vT.%;(l);ȨRvMx2-;(Zv DM +posPNʽuo\OypӜhP +}ptQ*p3&߀R +qc +6ߌSjq@ߘSqÝV߰Tq*f߼Uʾr#2V +\8*pKک8*=pL۩8*>pMݩ\8>Nݩ8*qPީܟ8?qQNߩ8*@qR੤\8@S੦8*AqTᩨܠ8*qUN⩪8*Bv*a; Je;v, +h;v8m;CvDq;svPu;\};vt +;w뀲*օ;3 +w0댲j։;G똲D +l(&EJlc6MF + +m(F G mÒ(PHmfMIn#(v J[nS܆J +n(ݖK*n(MLJo(޶ Mj@oA8?~E8p@̨H8pAبM8ÚpBQ8ƚpC𨆼U8ɛEY8ϛpF\8қpGe8՛pH,i8؛I8m8ۜpJ6ѪӜs*7ܛ*N\ӽ8 +Ԁ*9 +ܜ39'NJc*0j\:G׊Ԁþ*;Wتܝ;gN +#*< +Մ4[1;Ψbv͐65;(jv89;( @=;רzvʹ<@;(v@E;(vM̱BI;(v ر@Q;樢vFU;(vHX;쨲vMJ];(r:V*rBWJsYjsRY +sCZZssb[ʿtj\꿽tӟz4\ +t@] +t36L^ju@XqV@8BqW㩮ܡ8CXN婰8+qY橲\8 Dq[8+Eq\窸ܢ8Eq]N誺8+Fq^骼\8+q_骾8+Gq`8 GqaN֍;wW뤲 +;"wg밲֕;#*wp뼲֙;S2wȲ +ם;Բ*ס;Bw +;Rwj׭;Zwױ;C׵;jw +;rw2"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"" " " & &&&$$ $ $ $ $ $ $ $ 2$ 2 22"!@@ @@ H@ @@ʬUޝs,VǣЬNW@ӬX ,Y+3, NZKc,7[k@ɓ⬒G[,W\ 謓gN]#,^ Ĺ={йV=ܹX={Z={\> +|`> +| b >d>"|0f>*|<>2|Hj>@ / / _ / O / _/ / O / ^-O- ^ !- $ O'-^*-- 0O"g L⚚WgQ`iU#㪜wjYSj ]΃㺝k `㝠lKi8mkmDmqCPn tϣ\oy>C/|>F/|Ï>I|O>L/_>R/|>U/|ȏ>X|O>[/_>^|>a/|̏>d|O           ]ʃ^+ʳ,_ ,ak],aC,bsOc̣-d ^d  -e+3 7Ofk͓-GTl!>"B|`n$>%J|lp)>(R|x->+Z|v1>1x5>4j|z<>7z|A>:|~E>=̺I>@|غ _ O / @ / /O / @ / O 3-$6-&9O<-,^B.E-0HOK-4^N6Q-8T-Ohp }tq+3'rKc +7s ѓGsPtgu#2wvS:v+ҳBwKJxk嗢@>g|>j/Ϗ>m/}O>s/}>v/}>y/>|/}>/}>/}>/}>/   HHHHHHHHHHHHHHHHHHHHHHHHwwHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH @@ @@ @@ @@ @@!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaaaAaa`a!aaaaaaaAaa`a!aaaaaaaAaa`a!aaaaHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH    HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @    +@ + + + @    @    @   @@@@@@@@@@@@@@@@@@ @   !@!!!"@"""#@###$@$$$%@%%%&@&&&'@'''(@((()@)))*@***+@+++,@,,,-@---.@.../@///0@0001@1112@2223@3334@4445@5556@6667@7778@8889@999:@:::;@;;;<@<<<=@===>@>>>?@???@@@@@A@AAAB@BBBC@CCCD@DDDE@EEEF@FFFG@GGGH@HHHI@IIIJ@JJJK@KKKL@LLLM@MMMN@NNNO@OOOP@PPPQ@QQQR@RRRS@SSST@TTTU@UUUV@VVVW@WWWX@XXXY@YYYZ@ZZZ[@[[[\@\\\]@]]]^@^^^_@___`@```a@aaab@bbbc@cccd@ddde@eeef@fffg@gggh@hhhi@iiij@jjjk@kkkl@lllm@mmmn@nnno@ooop@pppq@qqqr@rrrs@ssst@tttu@uuuv@vvvw@wwwx@xxxy@yyyz@zzz{@{{{|@|||}@}}}~@~~~@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHblob-4096-deterministic-payloadHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH value-4096-deterministic-fixtureHHHHHHHHHHHHPHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  0HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH  greenblueredHHHHHHHHHHHHHHHHHHHHHHHHHHHH HHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHH + +id *int328 +Cname *string08R" +lance-encoding:compressionnone +items *list08 +item *int3208 +ecategory *dict:string:int8:false08BR. +&lance-encoding:dict-values-compressionnone +Bblob * large_binary08R +lance-encoding:blobtrue +)' +% +/lance.encodings.ColumnEncoding +I + +"75 +3 +/lance.encodings21.PageLayout +* 28HPL +  "75 +3 +/lance.encodings21.PageLayout +* 28HP(L + "75 +3 +/lance.encodings21.PageLayout +* 28HP(L + "75 +3 +/lance.encodings21.PageLayout +* 28HP(7"0. +, +/lance.encodings21.PageLayout  *2( +)' +% +/lance.encodings.ColumnEncoding +T + "A? += +/lance.encodings21.PageLayout +* + + 28HPW +  "A? += +/lance.encodings21.PageLayout +* + + 28HP(W + "A? += +/lance.encodings21.PageLayout +* + + 28HP(W + "A? += +/lance.encodings21.PageLayout +* + + 28HP(9 +4"*( +& +/lance.encodings21.PageLayout*( +)' +% +/lance.encodings.ColumnEncoding +h +  .0"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HPk +  20"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HP(k +  40"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HP(k +  50"RP +N +/lance.encodings21.PageLayout- ++ + +" + +" +* 28@HP(\ + 0"EC +A +/lance.encodings21.PageLayout + + +* + 28@HP( +)' +% +/lance.encodings.ColumnEncoding +_ + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HPb + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HP(b + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HP(b + $"IG +E +/lance.encodings21.PageLayout$ +"**" + + (28HP(Y + $"B@ +> +/lance.encodings21.PageLayout + +" + + (28HP( +)' +% +/lance.encodings.ColumnEncoding +Z +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP] +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP(] +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP(] +"GE +C +/lance.encodings21.PageLayout"" + + b +@@ +28HP(Y +"FD +B +/lance.encodings21.PageLayout!" + + b +@@ +28HP( ǚ`':atGǚsäLANC \ No newline at end of file diff --git a/rust/lance-geo/src/bbox.rs b/rust/lance-geo/src/bbox.rs index 71537683bf6..da891335318 100644 --- a/rust/lance-geo/src/bbox.rs +++ b/rust/lance-geo/src/bbox.rs @@ -16,6 +16,18 @@ use geoarrow_schema::{BoxType, Dimension}; use lance_core::error::ArrowResult; use serde::{Deserialize, Serialize}; +/// Returns whether a rectangle has inverted bounds and therefore no extent. +/// +/// ``` +/// # use lance_geo::bbox::{BoundingBox, is_empty_rect}; +/// assert!(is_empty_rect(&BoundingBox::new())); +/// ``` +pub fn is_empty_rect(rect: &impl RectTrait) -> bool { + let min = rect.min(); + let max = rect.max(); + min.x() > max.x() || min.y() > max.y() +} + /// Inspired by #[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct BoundingBox { @@ -97,8 +109,15 @@ impl BoundingBox { } pub fn add_rect(&mut self, rect: &impl RectTrait) { - self.add_coord(&rect.min()); - self.add_coord(&rect.max()); + // Empty bounding boxes use inverted bounds, so they are the identity for a union. + if is_empty_rect(rect) { + return; + } + + let min = rect.min(); + let max = rect.max(); + self.add_coord(&min); + self.add_coord(&max); } pub fn add_polygon(&mut self, polygon: &impl PolygonTrait) { @@ -331,3 +350,30 @@ fn impl_total_bounds<'a>(arr: &'a impl GeoArrowArrayAccessor<'a>) -> ArrowResult Ok(bbox) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_add_empty_rect_is_noop() { + let valid = BoundingBox::new_with_coords(&[ + geo_types::coord! { x: -1.0, y: -2.0 }, + geo_types::coord! { x: 3.0, y: 4.0 }, + ]); + let empty = BoundingBox::new(); + + let mut valid_then_empty = valid; + valid_then_empty.add_rect(&empty); + + let mut empty_then_valid = empty; + empty_then_valid.add_rect(&valid); + + for bbox in [valid_then_empty, empty_then_valid] { + assert_eq!(bbox.minx(), -1.0); + assert_eq!(bbox.miny(), -2.0); + assert_eq!(bbox.maxx(), 3.0); + assert_eq!(bbox.maxy(), 4.0); + } + } +} diff --git a/rust/lance-index-core/Cargo.toml b/rust/lance-index-core/Cargo.toml new file mode 100644 index 00000000000..5024f2eee0b --- /dev/null +++ b/rust/lance-index-core/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "lance-index-core" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +readme = "README.md" +description = "Core traits and types for Lance index plugins" +keywords.workspace = true +categories.workspace = true +rust-version.workspace = true + +[dependencies] +async-trait.workspace = true +arrow-array.workspace = true +arrow-schema.workspace = true +arrow-select.workspace = true +bytes.workspace = true +datafusion-common.workspace = true +datafusion-expr.workspace = true +datafusion.workspace = true +futures.workspace = true +lance-core.workspace = true +lance-io.workspace = true +lance-select.workspace = true +prost-types.workspace = true +roaring.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/rust/lance-index-core/README.md b/rust/lance-index-core/README.md new file mode 100644 index 00000000000..a954522a283 --- /dev/null +++ b/rust/lance-index-core/README.md @@ -0,0 +1,6 @@ +# lance-index-core + +`lance-index-core` is an internal sub-crate, containing the core traits and types used to +implement index plugins for [Lance](https://github.com/lance-format/lance). + +**Important Note**: This crate is **not intended for external usage**. diff --git a/rust/lance-index-core/src/lib.rs b/rust/lance-index-core/src/lib.rs new file mode 100644 index 00000000000..c949387b0b2 --- /dev/null +++ b/rust/lance-index-core/src/lib.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{any::Any, sync::Arc}; + +use async_trait::async_trait; +use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; +use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; + +pub mod metrics; +pub mod scalar; + +/// Generic methods common across all types of secondary indices +/// +#[async_trait] +pub trait Index: Send + Sync + DeepSizeOf { + /// Cast to [Any]. + fn as_any(&self) -> &dyn Any; + + /// Cast to [Index] + fn as_index(self: Arc) -> Arc; + + /// Retrieve index statistics as a JSON Value + fn statistics(&self) -> Result; + + /// Prewarm the index. + /// + /// This will load the index into memory and cache it. + async fn prewarm(&self) -> Result<()>; + + /// Get the type of the index + fn index_type(&self) -> IndexType; + + /// Read through the index and determine which fragment ids are covered by the index + /// + /// This is a kind of slow operation. It's better to use the fragment_bitmap. This + /// only exists for cases where the fragment_bitmap has become corrupted or missing. + async fn calculate_included_frags(&self) -> Result; +} + +/// Index Type +#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf, Serialize, Deserialize)] +pub enum IndexType { + // Preserve 0-100 for simple indices. + Scalar = 0, // Legacy scalar index, alias to BTree + + BTree = 1, // BTree + + Bitmap = 2, // Bitmap + + LabelList = 3, // LabelList + + Inverted = 4, // Inverted + + NGram = 5, // NGram + + FragmentReuse = 6, + + MemWal = 7, + + ZoneMap = 8, // ZoneMap + + BloomFilter = 9, // Bloom filter + + RTree = 10, // RTree + + Fm = 11, // FM-Index + + // 100+ and up for vector index. + /// Flat vector index. + Vector = 100, // Legacy vector index, alias to IvfPq + IvfFlat = 101, + IvfSq = 102, + IvfPq = 103, + IvfHnswSq = 104, + IvfHnswPq = 105, + IvfHnswFlat = 106, + IvfRq = 107, +} + +impl std::fmt::Display for IndexType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Self::Scalar | Self::BTree => write!(f, "BTree"), + Self::Bitmap => write!(f, "Bitmap"), + Self::LabelList => write!(f, "LabelList"), + Self::Inverted => write!(f, "Inverted"), + Self::NGram => write!(f, "NGram"), + Self::FragmentReuse => write!(f, "FragmentReuse"), + Self::MemWal => write!(f, "MemWal"), + Self::ZoneMap => write!(f, "ZoneMap"), + Self::BloomFilter => write!(f, "BloomFilter"), + Self::RTree => write!(f, "RTree"), + Self::Fm => write!(f, "Fm"), + Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"), + Self::IvfFlat => write!(f, "IVF_FLAT"), + Self::IvfSq => write!(f, "IVF_SQ"), + Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"), + Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"), + Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"), + Self::IvfRq => write!(f, "IVF_RQ"), + } + } +} + +impl TryFrom for IndexType { + type Error = Error; + + fn try_from(value: i32) -> Result { + match value { + v if v == Self::Scalar as i32 => Ok(Self::Scalar), + v if v == Self::BTree as i32 => Ok(Self::BTree), + v if v == Self::Bitmap as i32 => Ok(Self::Bitmap), + v if v == Self::LabelList as i32 => Ok(Self::LabelList), + v if v == Self::NGram as i32 => Ok(Self::NGram), + v if v == Self::Inverted as i32 => Ok(Self::Inverted), + v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse), + v if v == Self::MemWal as i32 => Ok(Self::MemWal), + v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap), + v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter), + v if v == Self::RTree as i32 => Ok(Self::RTree), + v if v == Self::Fm as i32 => Ok(Self::Fm), + v if v == Self::Vector as i32 => Ok(Self::Vector), + v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat), + v if v == Self::IvfSq as i32 => Ok(Self::IvfSq), + v if v == Self::IvfPq as i32 => Ok(Self::IvfPq), + v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq), + v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq), + v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat), + v if v == Self::IvfRq as i32 => Ok(Self::IvfRq), + _ => Err(Error::invalid_input_source( + format!("the input value {} is not a valid IndexType", value).into(), + )), + } + } +} + +impl TryFrom<&str> for IndexType { + type Error = Error; + + fn try_from(value: &str) -> Result { + match value { + "BTree" | "BTREE" => Ok(Self::BTree), + "Bitmap" | "BITMAP" => Ok(Self::Bitmap), + "LabelList" | "LABELLIST" => Ok(Self::LabelList), + "Inverted" | "INVERTED" => Ok(Self::Inverted), + "NGram" | "NGRAM" => Ok(Self::NGram), + "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap), + "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter), + "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree), + "Fm" | "FM" => Ok(Self::Fm), + "Vector" | "VECTOR" => Ok(Self::Vector), + "IVF_FLAT" => Ok(Self::IvfFlat), + "IVF_SQ" => Ok(Self::IvfSq), + "IVF_PQ" => Ok(Self::IvfPq), + "IVF_RQ" => Ok(Self::IvfRq), + "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat), + "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq), + "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq), + "FragmentReuse" => Ok(Self::FragmentReuse), + "MemWal" => Ok(Self::MemWal), + _ => Err(Error::invalid_input(format!( + "invalid index type: {}", + value + ))), + } + } +} + +impl IndexType { + pub fn is_scalar(&self) -> bool { + matches!( + self, + Self::Scalar + | Self::BTree + | Self::Bitmap + | Self::LabelList + | Self::Inverted + | Self::NGram + | Self::ZoneMap + | Self::BloomFilter + | Self::RTree + | Self::Fm, + ) + } + + pub fn is_vector(&self) -> bool { + matches!( + self, + Self::Vector + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat + | Self::IvfFlat + | Self::IvfSq + | Self::IvfRq + ) + } + + pub fn is_system(&self) -> bool { + matches!(self, Self::FragmentReuse | Self::MemWal) + } + + /// Returns the current format version of the index type, + /// bump this when the index format changes. + /// Indices which higher version than these will be ignored for compatibility, + /// This would happen when creating index in a newer version of Lance, + /// but then opening the index in older version of Lance + pub fn version(&self) -> i32 { + match self { + Self::Scalar => 0, + Self::BTree => 0, + Self::Bitmap => 0, + Self::LabelList => 0, + Self::Inverted => 0, + Self::NGram => 0, + Self::FragmentReuse => 0, + Self::MemWal => 0, + Self::ZoneMap => 0, + Self::BloomFilter => 0, + Self::RTree => 0, + Self::Fm => 0, + + // IMPORTANT: if any vector index subtype needs a format bump that is + // not backward compatible, its new version must be set to + // (current max vector index version + 1), even if only one subtype + // changed. Compatibility filtering currently cannot distinguish vector + // subtypes from details-only metadata, so vector versions effectively + // share one global monotonic compatibility level. + Self::Vector + | Self::IvfFlat + | Self::IvfSq + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat => 1, + Self::IvfRq => 2, + } + } + + /// Returns the target partition size for the index type. + /// + /// This is used to compute the number of partitions for the index. + /// The partition size is optimized for the best performance of the index. + /// + /// This is for vector indices only. + pub fn target_partition_size(&self) -> usize { + match self { + Self::Vector => 8192, + Self::IvfFlat => 4096, + Self::IvfSq => 8192, + Self::IvfPq => 8192, + Self::IvfRq => 4096, + Self::IvfHnswFlat => 1 << 20, + Self::IvfHnswSq => 1 << 20, + Self::IvfHnswPq => 1 << 20, + _ => 8192, + } + } + + /// Returns the highest supported vector index version in this Lance build. + pub fn max_vector_version() -> u32 { + [ + Self::Vector, + Self::IvfFlat, + Self::IvfSq, + Self::IvfPq, + Self::IvfHnswSq, + Self::IvfHnswPq, + Self::IvfHnswFlat, + Self::IvfRq, + ] + .into_iter() + .map(|index_type| index_type.version() as u32) + .max() + .unwrap_or(1) + } + + pub fn matches_details(&self, details: &prost_types::Any) -> bool { + let url = &details.type_url; + match self { + Self::Scalar | Self::BTree => url.ends_with("BTreeIndexDetails"), + Self::Bitmap => url.ends_with("BitmapIndexDetails"), + Self::LabelList => url.ends_with("LabelListIndexDetails"), + Self::Inverted => url.ends_with("InvertedIndexDetails"), + Self::NGram => url.ends_with("NGramIndexDetails"), + Self::ZoneMap => url.ends_with("ZoneMapIndexDetails"), + Self::BloomFilter => url.ends_with("BloomFilterIndexDetails"), + Self::RTree => url.ends_with("RTreeIndexDetails"), + Self::Fm => url.ends_with("FMIndexDetails"), + Self::FragmentReuse => url.ends_with("FragmentReuseIndexDetails"), + Self::MemWal => url.ends_with("MemWalIndexDetails"), + Self::Vector + | Self::IvfFlat + | Self::IvfSq + | Self::IvfPq + | Self::IvfHnswSq + | Self::IvfHnswPq + | Self::IvfHnswFlat + | Self::IvfRq => url.ends_with("VectorIndexDetails"), + } + } +} + +pub trait IndexParams: Send + Sync { + fn as_any(&self) -> &dyn Any; + + fn index_name(&self) -> &str; +} diff --git a/rust/lance-index-core/src/metrics.rs b/rust/lance-index-core/src/metrics.rs new file mode 100644 index 00000000000..f2dce6ee407 --- /dev/null +++ b/rust/lance-index-core/src/metrics.rs @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// A trait used by the index to report metrics +/// +/// Callers can implement this trait to collect metrics. Production collectors +/// must stay coarse-grained: do not record per-document, posting, candidate, or +/// window events here. Benchmark-only instrumentation belongs in test- or +/// benchmark-local hooks. +pub trait MetricsCollector: Send + Sync { + /// Record partition loads + /// + /// Many indices consist of partitions that may need to be loaded + /// into cache. For example, an inverted index or ngram index has a + /// posting list for each token. + /// + /// In the ideal case, these shards are in the cache and will not need + /// to be loaded from disk. This method should not be called if the + /// shard is in the cache. + fn record_parts_loaded(&self, num_parts: usize); + + /// Record a shard load + fn record_part_load(&self) { + self.record_parts_loaded(1); + } + + /// Record an index load + /// + /// This should be called when a scalar index is loaded from storage. + /// It should not be called if the index is already in memory. + fn record_index_loads(&self, num_indexes: usize); + + /// Record an index load + fn record_index_load(&self) { + self.record_index_loads(1); + } + + /// Record the number of "comparisons" made by the index + /// + /// What exactly constitutes a comparison depends on the index type. + /// For example, a B-tree index may make comparisons while searching for a value. + /// On the other hand, a bitmap index makes comparisons when computing the intersection + /// of two bitmaps. + /// + /// The goal is to provide some visibility into the compute cost of the search + fn record_comparisons(&self, num_comparisons: usize); + + /// Record index cache hits observed while serving this query. + /// + /// A "hit" is one page-level lookup (partition, posting list, BTree page, etc.) + /// that was served from the in-memory index cache without touching storage. + fn record_index_cache_hits(&self, _num_hits: usize) {} + + /// Convenience for a single cache hit. + fn record_index_cache_hit(&self) { + self.record_index_cache_hits(1); + } + + /// Record index cache misses observed while serving this query. + /// + /// A "miss" is one page-level lookup that had to be loaded from storage + /// because it was not present in the cache. + fn record_index_cache_misses(&self, _num_misses: usize) {} + + /// Convenience for a single cache miss. + fn record_index_cache_miss(&self) { + self.record_index_cache_misses(1); + } + + /// Returns an optional sink for recording exact I/O statistics (bytes read, + /// IOPS, and requests) performed on behalf of this collector. + /// + /// Index implementations that read from a + /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to + /// their file readers so the I/O performed for a single query is measured + /// and attributed here. The default returns `None`, meaning the caller does + /// not want I/O measured (and index implementations should then take their + /// normal, uninstrumented read path). + fn io_stats(&self) -> Option { + None + } +} + +/// A no-op metrics collector that does nothing +pub struct NoOpMetricsCollector; + +impl MetricsCollector for NoOpMetricsCollector { + fn record_parts_loaded(&self, _num_parts: usize) {} + fn record_index_loads(&self, _num_indexes: usize) {} + fn record_comparisons(&self, _num_comparisons: usize) {} +} + +#[derive(Default)] +pub struct LocalMetricsCollector { + pub parts_loaded: AtomicUsize, + pub index_loads: AtomicUsize, + pub comparisons: AtomicUsize, + // Kept `pub(crate)` so that adding new metric fields to this public struct + // does not break downstream callers that construct or destructure the + // existing three fields. Callers can still read cumulative values via + // [`Self::index_cache_hits`] / [`Self::index_cache_misses`]. + pub(crate) index_cache_hits: AtomicUsize, + pub(crate) index_cache_misses: AtomicUsize, +} + +impl LocalMetricsCollector { + pub fn dump_into(self, other: &dyn MetricsCollector) { + other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed)); + other.record_index_loads(self.index_loads.load(Ordering::Relaxed)); + other.record_comparisons(self.comparisons.load(Ordering::Relaxed)); + other.record_index_cache_hits(self.index_cache_hits.load(Ordering::Relaxed)); + other.record_index_cache_misses(self.index_cache_misses.load(Ordering::Relaxed)); + } + + /// Cumulative index cache hits recorded so far. + pub fn index_cache_hits(&self) -> usize { + self.index_cache_hits.load(Ordering::Relaxed) + } + + /// Cumulative index cache misses recorded so far. + pub fn index_cache_misses(&self) -> usize { + self.index_cache_misses.load(Ordering::Relaxed) + } +} + +impl MetricsCollector for LocalMetricsCollector { + fn record_parts_loaded(&self, num_parts: usize) { + self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed); + } + + fn record_index_loads(&self, num_indexes: usize) { + self.index_loads.fetch_add(num_indexes, Ordering::Relaxed); + } + + fn record_comparisons(&self, num_comparisons: usize) { + self.comparisons + .fetch_add(num_comparisons, Ordering::Relaxed); + } + + fn record_index_cache_hits(&self, num_hits: usize) { + self.index_cache_hits.fetch_add(num_hits, Ordering::Relaxed); + } + + fn record_index_cache_misses(&self, num_misses: usize) { + self.index_cache_misses + .fetch_add(num_misses, Ordering::Relaxed); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct SumSink { + parts: AtomicUsize, + loads: AtomicUsize, + comparisons: AtomicUsize, + hits: AtomicUsize, + misses: AtomicUsize, + } + + impl MetricsCollector for SumSink { + fn record_parts_loaded(&self, n: usize) { + self.parts.fetch_add(n, Ordering::Relaxed); + } + fn record_index_loads(&self, n: usize) { + self.loads.fetch_add(n, Ordering::Relaxed); + } + fn record_comparisons(&self, n: usize) { + self.comparisons.fetch_add(n, Ordering::Relaxed); + } + fn record_index_cache_hits(&self, n: usize) { + self.hits.fetch_add(n, Ordering::Relaxed); + } + fn record_index_cache_misses(&self, n: usize) { + self.misses.fetch_add(n, Ordering::Relaxed); + } + } + + #[test] + fn local_metrics_collector_forwards_cache_counts() { + let local = LocalMetricsCollector::default(); + local.record_index_cache_hit(); + local.record_index_cache_hit(); + local.record_index_cache_misses(3); + local.record_part_load(); + local.record_index_load(); + local.record_comparisons(5); + + let sink = SumSink { + parts: AtomicUsize::new(0), + loads: AtomicUsize::new(0), + comparisons: AtomicUsize::new(0), + hits: AtomicUsize::new(0), + misses: AtomicUsize::new(0), + }; + local.dump_into(&sink); + + assert_eq!(sink.parts.load(Ordering::Relaxed), 1); + assert_eq!(sink.loads.load(Ordering::Relaxed), 1); + assert_eq!(sink.comparisons.load(Ordering::Relaxed), 5); + assert_eq!(sink.hits.load(Ordering::Relaxed), 2); + assert_eq!(sink.misses.load(Ordering::Relaxed), 3); + } + + #[test] + fn no_op_metrics_collector_ignores_cache_counts() { + // Ensures existing implementors that do not override cache-count methods + // remain sound (default impl is a no-op). + let collector = NoOpMetricsCollector; + collector.record_index_cache_hit(); + collector.record_index_cache_miss(); + collector.record_index_cache_hits(10); + collector.record_index_cache_misses(20); + } +} diff --git a/rust/lance-index-core/src/scalar.rs b/rust/lance-index-core/src/scalar.rs new file mode 100644 index 00000000000..fa597c5cd18 --- /dev/null +++ b/rust/lance-index-core/src/scalar.rs @@ -0,0 +1,639 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Abstract scalar index traits and types for Lance index plugins + +use arrow_array::{BooleanArray, RecordBatch, UInt64Array}; +use arrow_schema::{DataType, Schema}; +use async_trait::async_trait; +use bytes::Bytes; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion_common::scalar::ScalarValue; +use datafusion_expr::Expr; +use lance_core::deepsize::DeepSizeOf; +use lance_core::utils::row_addr_remap::RowAddrRemap; +use lance_core::{Error, Result}; +use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; +use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps}; +use roaring::{RoaringBitmap, RoaringTreemap}; +use serde::Serialize; +use std::collections::HashMap; +use std::fmt::Debug; +use std::pin::Pin; +use std::{any::Any, sync::Arc}; + +use crate::metrics::MetricsCollector; +use crate::{Index, IndexParams, IndexType}; + +/// Metadata about a single file within an index segment. +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub struct IndexFile { + /// Path relative to the index directory (e.g., "index.idx", "auxiliary.idx") + pub path: String, + /// Size of the file in bytes + pub size_bytes: u64, +} + +pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index"; + +/// Builtin index types supported by the Lance library +/// +/// This is primarily for convenience to avoid a bunch of string +/// constants and provide some auto-complete. This type should not +/// be used in the manifest as plugins cannot add new entries. +#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] +pub enum BuiltinIndexType { + BTree, + Bitmap, + LabelList, + NGram, + ZoneMap, + BloomFilter, + RTree, + Inverted, + Fm, +} + +impl BuiltinIndexType { + pub fn as_str(&self) -> &str { + match self { + Self::BTree => "btree", + Self::Bitmap => "bitmap", + Self::LabelList => "labellist", + Self::NGram => "ngram", + Self::ZoneMap => "zonemap", + Self::Inverted => "inverted", + Self::BloomFilter => "bloomfilter", + Self::RTree => "rtree", + Self::Fm => "fm", + } + } +} + +impl TryFrom for BuiltinIndexType { + type Error = Error; + + fn try_from(value: IndexType) -> Result { + match value { + IndexType::BTree => Ok(Self::BTree), + IndexType::Bitmap => Ok(Self::Bitmap), + IndexType::LabelList => Ok(Self::LabelList), + IndexType::NGram => Ok(Self::NGram), + IndexType::ZoneMap => Ok(Self::ZoneMap), + IndexType::Inverted => Ok(Self::Inverted), + IndexType::BloomFilter => Ok(Self::BloomFilter), + IndexType::RTree => Ok(Self::RTree), + IndexType::Fm => Ok(Self::Fm), + _ => Err(Error::index("Invalid index type".to_string())), + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ScalarIndexParams { + /// The type of index to create + /// + /// Plugins may add additional index types. Index type lookup is case-insensitive. + pub index_type: String, + /// The parameters to train the index + /// + /// This should be a JSON string. The contents of the JSON string will be specific to the + /// index type. If not set, then default parameters will be used for the index type. + pub params: Option, +} + +impl Default for ScalarIndexParams { + fn default() -> Self { + Self { + index_type: BuiltinIndexType::BTree.as_str().to_string(), + params: None, + } + } +} + +impl ScalarIndexParams { + /// Creates a new ScalarIndexParams from one of the builtin index types + pub fn for_builtin(index_type: BuiltinIndexType) -> Self { + Self { + index_type: index_type.as_str().to_string(), + params: None, + } + } + + /// Create a new ScalarIndexParams with the given index type + pub fn new(index_type: String) -> Self { + Self { + index_type, + params: None, + } + } + + /// Set the parameters for the index + pub fn with_params(mut self, params: &ParamsType) -> Self { + self.params = Some(serde_json::to_string(params).unwrap()); + self + } +} + +impl IndexParams for ScalarIndexParams { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn index_name(&self) -> &str { + LANCE_SCALAR_INDEX + } +} + +/// Trait for storing an index (or parts of an index) into storage +#[async_trait] +pub trait IndexWriter: Send { + /// Writes a record batch into the file, returning the 0-based index of the batch in the file + /// + /// E.g. if this is the third time this is called this method will return 2 + async fn write_record_batch(&mut self, batch: RecordBatch) -> Result; + /// Adds a global buffer and returns its index. + async fn add_global_buffer(&mut self, _data: Bytes) -> Result { + Err(Error::not_supported( + "global buffers are not supported by this index writer", + )) + } + /// Finishes writing the file and closes the file + async fn finish(&mut self) -> Result; + /// Finishes writing the file and closes the file with additional metadata + async fn finish_with_metadata( + &mut self, + metadata: HashMap, + ) -> Result; +} + +/// Trait for reading an index (or parts of an index) from storage +#[async_trait] +pub trait IndexReader: Send + Sync { + /// Read the n-th record batch from the file + async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result; + /// Reads a global buffer by index. + async fn read_global_buffer(&self, _index: u32) -> Result { + Err(Error::not_supported( + "global buffers are not supported by this index reader", + )) + } + /// Read the range of rows from the file. + /// If projection is Some, only return the columns in the projection, + /// nested columns like Some(&["x.y"]) are not supported. + /// If projection is None, return all columns. + async fn read_range( + &self, + range: std::ops::Range, + projection: Option<&[&str]>, + ) -> Result; + /// Read multiple ranges and concatenate into a single batch. + /// Default impl runs `read_range`s in parallel via `try_join_all`. + async fn read_ranges( + &self, + ranges: &[std::ops::Range], + projection: Option<&[&str]>, + ) -> Result { + if ranges.is_empty() { + return self.read_range(0..0, projection).await; + } + let futures = ranges + .iter() + .map(|r| self.read_range(r.clone(), projection)); + let batches = futures::future::try_join_all(futures).await?; + let schema = batches[0].schema(); + Ok(arrow_select::concat::concat_batches(&schema, &batches)?) + } + /// Read a range of rows as a stream of record batches. + /// + /// This allows the caller to process rows incrementally without loading the + /// entire range into memory at once. + /// + /// The default implementation falls back to [`Self::read_range`] and wraps + /// the result in a single-item stream. + async fn read_range_stream( + &self, + range: std::ops::Range, + projection: Option<&[&str]>, + ) -> Result>> { + let batch = self.read_range(range, projection).await?; + let schema = batch.schema(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok(batch) }), + ))) + } + /// Return the number of batches in the file + async fn num_batches(&self, batch_size: u64) -> u32; + /// Return the number of rows in the file + fn num_rows(&self) -> usize; + /// Return the metadata of the file + fn schema(&self) -> &lance_core::datatypes::Schema; + /// Best-effort on-disk byte size of the file when the reader already knows it + /// without extra I/O, else `None`. Used to size prewarm chunks. + fn file_size_bytes(&self) -> Option { + None + } +} + +/// Trait abstracting I/O away from index logic +/// +/// Scalar indices are currently serialized as indexable arrow record batches stored in +/// named "files". The index store is responsible for serializing and deserializing +/// these batches into file data (e.g. as .lance files or .parquet files, etc.) +#[async_trait] +pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf { + fn as_any(&self) -> &dyn Any; + fn clone_arc(&self) -> Arc; + + /// Suggested I/O parallelism for the store + fn io_parallelism(&self) -> usize; + + /// Create a new file and return a writer to store data in the file + async fn new_index_file(&self, name: &str, schema: Arc) + -> Result>; + + /// Open an existing file for retrieval + async fn open_index_file(&self, name: &str) -> Result>; + + /// Return a store that submits its I/O at the given base priority. + fn with_io_priority(&self, io_priority: u64) -> Arc; + + /// Copy a range of batches from an index file from this store to another + /// + /// This is often useful when remapping or updating + async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result; + + /// Copy an index file from this store to a new name in another store, leaving the source intact + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + if name == new_name { + self.copy_index_file(name, dest_store).await + } else { + Err(Error::not_supported(format!( + "copying index file {name} to {new_name} is not supported by this index store" + ))) + } + } + + /// Rename an index file + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result; + + /// Delete an index file (used in the tmp spill store to keep tmp size down) + async fn delete_index_file(&self, name: &str) -> Result<()>; + + /// List all files in the index directory with their sizes. + /// + /// Returns a list of (relative_path, size_bytes) tuples. + /// Used to capture file metadata after index creation/modification. + async fn list_files_with_sizes(&self) -> Result>; +} + +/// Different scalar indices may support different kinds of queries +/// +/// For example, a btree index can support a wide range of queries (e.g. x > 7) +/// while an index based on FTS only supports queries like "x LIKE 'foo'" +/// +/// This trait is used when we need an object that can represent any kind of query +/// +/// Note: if you are implementing this trait for a query type then you probably also +/// need to implement the scalar query parser trait to create instances of your query at parse time. +pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync { + /// Cast the query as Any to allow for downcasting + fn as_any(&self) -> &dyn Any; + /// Format the query as a string for display purposes + fn format(&self, col: &str) -> String; + /// Convert the query to a datafusion expression + fn to_expr(&self, col: String) -> Expr; + /// Compare this query to another query + fn dyn_eq(&self, other: &dyn AnyQuery) -> bool; +} + +impl PartialEq for dyn AnyQuery { + fn eq(&self, other: &Self) -> bool { + self.dyn_eq(other) + } +} + +/// The result of a search operation against a scalar index +#[derive(Debug, PartialEq)] +pub enum SearchResult { + /// The exact row ids that satisfy the query + Exact(NullableRowAddrSet), + /// Any row id satisfying the query will be in this set but not every + /// row id in this set will satisfy the query, a further recheck step + /// is needed + AtMost(NullableRowAddrSet), + /// All of the given row ids satisfy the query but there may be more + /// + /// No scalar index actually returns this today but it can arise from + /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x))) + AtLeast(NullableRowAddrSet), +} + +impl SearchResult { + pub fn exact(row_ids: impl Into) -> Self { + Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default())) + } + + pub fn at_most(row_ids: impl Into) -> Self { + Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default())) + } + + pub fn at_least(row_ids: impl Into) -> Self { + Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default())) + } + + pub fn with_nulls(self, nulls: impl Into) -> Self { + match self { + Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())), + Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())), + Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())), + } + } + + pub fn row_addrs(&self) -> &NullableRowAddrSet { + match self { + Self::Exact(row_addrs) => row_addrs, + Self::AtMost(row_addrs) => row_addrs, + Self::AtLeast(row_addrs) => row_addrs, + } + } + + pub fn is_exact(&self) -> bool { + matches!(self, Self::Exact(_)) + } +} + +/// Brief information about an index that was created +pub struct CreatedIndex { + /// The details of the index that was created + /// + /// These should be stored somewhere as they will be needed to + /// load the index later. + pub index_details: prost_types::Any, + /// The version of the index that was created + /// + /// This can be used to determine if a reader is able to load the index. + pub index_version: u32, + /// List of files and their sizes for this index + /// + /// This enables skipping HEAD calls when opening indices and provides + /// visibility into index storage size via describe_indices(). + pub files: Vec, +} + +/// The ordering that training data must satisfy +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrainingOrdering { + /// The input will arrive sorted by the value column in ascending order + Values, + /// The input will arrive sorted by the address column in ascending order + Addresses, + /// The input will arrive in an arbitrary order + None, +} + +#[derive(Debug, Clone)] +pub struct TrainingCriteria { + pub ordering: TrainingOrdering, + pub needs_row_ids: bool, + pub needs_row_addrs: bool, +} + +impl TrainingCriteria { + pub fn new(ordering: TrainingOrdering) -> Self { + Self { + ordering, + needs_row_ids: false, + needs_row_addrs: false, + } + } + + pub fn with_row_id(mut self) -> Self { + self.needs_row_ids = true; + self + } + + pub fn with_row_addr(mut self) -> Self { + self.needs_row_addrs = true; + self + } +} + +/// The criteria that specifies how to update an index +pub struct UpdateCriteria { + /// If true, then we need to read the old data to update the index + /// + /// This should be avoided if possible but is left in for some legacy paths + pub requires_old_data: bool, + /// The criteria required for data (both old and new) + pub data_criteria: TrainingCriteria, +} + +/// Filter used when merging existing scalar-index rows during update. +/// +/// The caller must pick a filter mode that matches the row-id semantics of the +/// dataset: +/// - address-style row IDs: fragment filtering is valid +/// - stable row IDs: use exact row-id membership instead +#[derive(Debug, Clone)] +pub enum OldIndexDataFilter { + /// Keeps track of which fragments are still valid and which are no longer valid. + /// + /// This is valid for address-style row IDs. + Fragments { + to_keep: RoaringBitmap, + to_remove: RoaringBitmap, + }, + /// Keep old rows whose row IDs are in this exact allow-list. + /// + /// This is required for stable row IDs, where row IDs are opaque and + /// should not be interpreted as encoded row addresses. + RowIds(RowAddrTreeMap), +} + +impl OldIndexDataFilter { + /// Build a boolean mask that keeps only row IDs selected by this filter. + pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray { + match self { + Self::Fragments { to_keep, .. } => row_ids + .iter() + .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32))) + .collect(), + Self::RowIds(valid_row_ids) => row_ids + .iter() + .map(|id| id.map(|id| valid_row_ids.contains(id))) + .collect(), + } + } + + /// Apply this filter in place to a set of existing (old) row ids/addresses, + /// retaining only the rows the filter selects to keep. Used by index types + /// that merge old postings directly (e.g. bitmap) instead of re-scanning a + /// row-id array through [`Self::filter_row_ids`]. + pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) { + match self { + Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()), + Self::RowIds(valid_row_ids) => *rows &= valid_row_ids, + } + } +} + +impl UpdateCriteria { + pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self { + Self { + requires_old_data: true, + data_criteria, + } + } + + pub fn only_new_data(data_criteria: TrainingCriteria) -> Self { + Self { + requires_old_data: false, + data_criteria, + } + } +} + +/// Execution-time options for scalar index searches. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SearchOptions { + /// Preserve rows where the query evaluates to NULL. + /// + /// Callers may disable this only when NULL rows cannot affect the final + /// result, such as a top-level filter whose NULL results will be discarded. + track_nulls: bool, +} + +impl Default for SearchOptions { + fn default() -> Self { + Self { track_nulls: true } + } +} + +impl SearchOptions { + /// Configure whether searches preserve rows where the query evaluates to + /// NULL. When disabled, implementations return only TRUE rows. + pub fn with_track_nulls(mut self, track_nulls: bool) -> Self { + self.track_nulls = track_nulls; + self + } + + /// Whether searches preserve rows where the query evaluates to NULL. + pub fn track_nulls(&self) -> bool { + self.track_nulls + } +} + +/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries +#[async_trait] +pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf { + /// Search the scalar index + /// + /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered + async fn search( + &self, + query: &dyn AnyQuery, + metrics: &dyn MetricsCollector, + ) -> Result; + + /// Search the scalar index with execution-time options. + /// + /// Index implementations that do not need the options can rely on this + /// default implementation. The default preserves the behavior of + /// [`Self::search`]. + async fn search_with_options( + &self, + query: &dyn AnyQuery, + _options: SearchOptions, + metrics: &dyn MetricsCollector, + ) -> Result { + self.search(query, metrics).await + } + + /// Returns true if this index reports matches as physical row addresses + /// (`fragment_id << 32 | offset`) rather than row ids + /// + /// Address-domain indices (e.g. zone map, bloom filter) are built over the + /// `_rowaddr` column. On a dataset with stable row ids the address and + /// row-id domains diverge, so these results must be translated back to row + /// ids (via the per-fragment row-id sequences, known only at the dataset + /// layer) before they are combined with row-id results or handed to the + /// scan. The default (row-id domain) needs no translation. + fn results_are_row_addresses(&self) -> bool { + false + } + + /// Returns true if the remap operation is supported + fn can_remap(&self) -> bool; + + /// Remap the row ids, creating a new remapped version of this index in `dest_store` + async fn remap( + &self, + mapping: &RowAddrRemap, + dest_store: &dyn IndexStore, + ) -> Result; + + /// Add the new data into the index, creating an updated version of the index in `dest_store` + /// + /// If `old_data_filter` is provided, old index data will be filtered before + /// merge according to the chosen filter mode. + async fn update( + &self, + new_data: SendableRecordBatchStream, + dest_store: &dyn IndexStore, + old_data_filter: Option, + ) -> Result; + + /// Returns the criteria that will be used to update the index + fn update_criteria(&self) -> UpdateCriteria; + + /// Derive the index parameters from the current index + /// + /// This returns a ScalarIndexParams that can be used to recreate an index + /// with the same configuration on another dataset. + fn derive_index_params(&self) -> Result; + + /// Returns the value type expected by [`Self::update`], when the index has + /// a durable type contract for its training data. + /// + /// Wrapper indices use this to transform new data to the same type as the + /// loaded index instead of inferring a potentially different type from an + /// update batch. Index types without such a contract may return `None`. + fn training_data_type(&self) -> Option { + None + } + + /// Global `[min, max]` of the indexed column from index metadata, without a + /// scan, or `None` if this index type cannot supply a sound bound. When + /// `Some`, the range is a superset of live values (conservative under + /// deletes): safe to prune with, not guaranteed tight. + fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { + None + } +} + +/// Abstraction over any type that can remap row IDs during index loading. +/// +/// This decouples scalar index plugins from the table-level frag reuse index type. +/// The frag reuse index implements this trait, but callers may also supply custom +/// implementations for testing or other remapping strategies. +pub trait RowIdRemapper: Send + Sync + std::fmt::Debug { + /// Remap a single row id. Returns `None` if the row was deleted. + fn remap_row_id(&self, row_id: u64) -> Option; + /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows. + fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap; + /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows. + fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap; + /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows. + fn remap_row_ids_record_batch( + &self, + batch: RecordBatch, + row_id_idx: usize, + ) -> Result; +} diff --git a/rust/lance-index/Cargo.toml b/rust/lance-index/Cargo.toml index b9d1a5fde29..ab8ad36783c 100644 --- a/rust/lance-index/Cargo.toml +++ b/rust/lance-index/Cargo.toml @@ -12,9 +12,10 @@ categories.workspace = true rust-version.workspace = true [dependencies] -arc-swap.workspace = true +arc-swap = { workspace = true, features = ["weak"] } arrow.workspace = true arrow-array.workspace = true +arrow-ipc.workspace = true arrow-ord.workspace = true arrow-schema.workspace = true arrow-select.workspace = true @@ -39,6 +40,7 @@ jsonb.workspace = true lance-arrow.workspace = true lance-arrow-stats.workspace = true lance-core.workspace = true +lance-index-core.workspace = true lance-datafusion.workspace = true lance-encoding.workspace = true lance-file.workspace = true @@ -68,23 +70,24 @@ tracing.workspace = true tempfile.workspace = true crossbeam-queue.workspace = true bytes.workspace = true -chrono.workspace = true -uuid.workspace = true async-channel = "2.3.1" rand_distr.workspace = true -lance-datagen.workspace = true rangemap.workspace = true [dev-dependencies] approx.workspace = true criterion.workspace = true -env_logger = "0.11.6" +env_logger.workspace = true geo-traits.workspace = true lance-datagen.workspace = true +lance-datafusion = { workspace = true, features = ["datagen"] } lance-testing.workspace = true +libc.workspace = true test-log.workspace = true rstest.workspace = true +serial_test.workspace = true chrono.workspace = true +uuid.workspace = true [features] geo = ["dep:lance-geo", "lance-geo/geo", "dep:geoarrow-array", "dep:geoarrow-schema", "dep:geo-types"] @@ -96,7 +99,7 @@ tokenizer-jieba = ["dep:jieba-rs", "lance-tokenizer/tokenizer-jieba"] [build-dependencies] prost-build.workspace = true -protobuf-src = { version = "2.1", optional = true } +protobuf-src = { workspace = true, optional = true } [package.metadata.docs.rs] # docs.rs uses an older version of Ubuntu that does not have the necessary protoc version @@ -167,5 +170,9 @@ required-features = ["geo"] name = "residual_transform" harness = false +[[bench]] +name = "two_file_shuffle_read" +harness = false + [lints] workspace = true diff --git a/rust/lance-index/README.md b/rust/lance-index/README.md index 12f1f824936..fc7cf0e9517 100644 --- a/rust/lance-index/README.md +++ b/rust/lance-index/README.md @@ -1,7 +1,7 @@ # lance-index `lance-index` is an internal sub-crate, containing various vector index implementations -used in [Lance](https://github.com/lancedb/lance). +used in [Lance](https://github.com/lance-format/lance). **Important Note**: This crate is **not intended for external usage**. diff --git a/rust/lance-index/benches/hnsw.rs b/rust/lance-index/benches/hnsw.rs index 0a9b10bf42c..5884410ad9a 100644 --- a/rust/lance-index/benches/hnsw.rs +++ b/rust/lance-index/benches/hnsw.rs @@ -53,6 +53,7 @@ fn bench_hnsw(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -81,6 +82,7 @@ fn bench_hnsw(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -141,6 +143,7 @@ fn bench_hnsw_load(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -206,6 +209,7 @@ fn bench_hnsw_sq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -235,6 +239,7 @@ fn bench_hnsw_sq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -302,6 +307,7 @@ fn bench_hnsw_pq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), @@ -331,6 +337,7 @@ fn bench_hnsw_pq(c: &mut Criterion) { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, vectors.as_ref(), diff --git a/rust/lance-index/benches/kmeans.rs b/rust/lance-index/benches/kmeans.rs index 0beaf5448de..ef7e4df42b2 100644 --- a/rust/lance-index/benches/kmeans.rs +++ b/rust/lance-index/benches/kmeans.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::sync::OnceLock; + use arrow::array::AsArray; use arrow::datatypes::Float32Type; use arrow_array::FixedSizeListArray; @@ -14,6 +16,7 @@ use lance_testing::pprof::{Output, PProfProfiler}; use lance_index::vector::kmeans::{ KMeans, KMeansAlgo, KMeansAlgoFloat, KMeansParams, compute_partitions_arrow_array, }; +use lance_index::vector::pq::PQBuildParams; use lance_linalg::distance::DistanceType; use lance_testing::datagen::generate_random_array; @@ -29,17 +32,17 @@ fn bench_train(c: &mut Criterion) { ]; for (n, dimension) in params { let k = n / 256; - - let values = generate_random_array(n * dimension as usize); - let data = FixedSizeListArray::try_new_from_values(values, dimension).unwrap(); - - let values = generate_random_array(k * dimension as usize); - let centroids = FixedSizeListArray::try_new_from_values(values, dimension).unwrap(); + let data: OnceLock = OnceLock::new(); + let centroids: OnceLock = OnceLock::new(); c.bench_function(&format!("train_{}d_{}k", dimension, k), |b| { let params = KMeansParams::default().with_hierarchical_k(0); b.iter(|| { - KMeans::new_with_params(&data, k, ¶ms).ok().unwrap(); + let data = data.get_or_init(|| { + let values = generate_random_array(n * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + KMeans::new_with_params(data, k, ¶ms).ok().unwrap(); }) }); @@ -52,7 +55,13 @@ fn bench_train(c: &mut Criterion) { dimension, k, hierarchical_k ), |b| { - b.iter(|| KMeans::new_with_params(&data, k, ¶ms).ok().unwrap()); + b.iter(|| { + let data = data.get_or_init(|| { + let values = generate_random_array(n * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + KMeans::new_with_params(data, k, ¶ms).ok().unwrap() + }); }, ); } @@ -61,19 +70,40 @@ fn bench_train(c: &mut Criterion) { let mut group = c.benchmark_group(format!("compute_membership_{}d_{}k", dimension, k)); group.bench_function("flat", |b| { - b.iter(|| compute_partitions_arrow_array(¢roids, &data, DistanceType::L2)) + b.iter(|| { + let data = data.get_or_init(|| { + let values = generate_random_array(n * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + let centroids = centroids.get_or_init(|| { + let values = generate_random_array(k * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + compute_partitions_arrow_array(centroids, data, DistanceType::L2) + }) }); if k * dimension as usize >= 1_000_000 { - let index = SimpleIndex::may_train_index( - centroids.values().clone(), - dimension as usize, - DistanceType::L2, - ) - .unwrap() - .unwrap(); + let index: OnceLock = OnceLock::new(); group.bench_function("with_index", |b| { b.iter(|| { + let data = data.get_or_init(|| { + let values = generate_random_array(n * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + let centroids = centroids.get_or_init(|| { + let values = generate_random_array(k * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + let index = index.get_or_init(|| { + SimpleIndex::may_train_index( + centroids.values().clone(), + dimension as usize, + DistanceType::L2, + ) + .unwrap() + .unwrap() + }); KMeansAlgoFloat::::compute_membership_and_loss( centroids.values().as_primitive::().values(), data.values().as_primitive::().values(), @@ -81,7 +111,7 @@ fn bench_train(c: &mut Criterion) { DistanceType::L2, 0.0, None, - Some(&index), + Some(index), ) }) }); @@ -89,17 +119,63 @@ fn bench_train(c: &mut Criterion) { } } +fn bench_pq_build(c: &mut Criterion) { + let (dimension, num_sub_vectors) = (128, 16); + let mut group = c.benchmark_group(format!( + "pq_build_sampled_{}d_{}m", + dimension, num_sub_vectors + )); + for num_bits in [4, 8] { + let data = OnceLock::new(); + let params = PQBuildParams::new(num_sub_vectors, num_bits); + group.bench_function(format!("{}bit", num_bits), |b| { + b.iter(|| { + let data = data.get_or_init(|| { + let n = 256 * (1 << num_bits); + let values = generate_random_array(n * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + params.build(data, DistanceType::L2).unwrap() + }) + }); + } + group.finish(); + + // Callers may pass more rows than the kmeans sample cap + // (`sample_rate * num_centroids`). The old sub-vector division copied the + // whole input while training only read a prefix. + let mut group = c.benchmark_group(format!( + "pq_build_oversampled_{}d_{}m", + dimension, num_sub_vectors + )); + for num_bits in [4, 8] { + let data = OnceLock::new(); + let params = PQBuildParams::new(num_sub_vectors, num_bits); + group.bench_function(format!("{}bit", num_bits), |b| { + b.iter(|| { + let data = data.get_or_init(|| { + let n = 4 * 1024 * 1024; + let values = generate_random_array(n * dimension as usize); + FixedSizeListArray::try_new_from_values(values, dimension).unwrap() + }); + params.build(data, DistanceType::L2).unwrap() + }) + }); + } + group.finish(); +} + #[cfg(target_os = "linux")] criterion_group!( name=benches; config = Criterion::default().significance_level(0.1).sample_size(10) .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); - targets = bench_train); + targets = bench_train, bench_pq_build); // Non-linux version does not support pprof. #[cfg(not(target_os = "linux"))] criterion_group!( name=benches; config = Criterion::default().significance_level(0.1).sample_size(10); - targets = bench_train); + targets = bench_train, bench_pq_build); criterion_main!(benches); diff --git a/rust/lance-index/benches/two_file_shuffle_read.rs b/rust/lance-index/benches/two_file_shuffle_read.rs new file mode 100644 index 00000000000..6cd82dee857 --- /dev/null +++ b/rust/lance-index/benches/two_file_shuffle_read.rs @@ -0,0 +1,751 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Reproducible read benchmark for the two-file IVF shuffle format. +//! +//! The fixture is deliberately generated once, before Criterion starts timing, +//! and is then reopened for every end-to-end sample. Each input `RecordBatch` is forced to +//! become one flush group, so both scenarios contain exactly 20 physical groups. +//! The payload has the same column topology as a 5-bit RQ build (row ID, binary +//! and extra codes, and five floating-point factors), scaled to 256 dimensions +//! so the benchmark remains practical on a developer laptop. +//! +//! Run both the benchmark-local pre-change reader and the current reader against +//! one persistent fixture: +//! +//! ```text +//! LANCE_SHUFFLE_BENCH_FIXTURE_ROOT=/tmp/lance-two-file-fixture \ +//! cargo bench --profile release-with-debug -p lance-index \ +//! --bench two_file_shuffle_read +//! ``` +//! +//! The optional fixture root makes later invocations reopen the exact same files +//! and manifest. Without it, each process regenerates byte-equivalent +//! deterministic fixtures in temporary directories. +//! A benchmark-local copy of the pre-change on-demand offsets reader runs next +//! to the current reader, so every invocation also provides a same-process, +//! same-file baseline/current comparison. +//! +//! Criterion reports separate reopen, read-only, and reopen-plus-read timings; +//! the latter two report rows/s. Before each scenario it also prints one +//! cache-hot diagnostic pass containing separate init/read/total wall time, +//! process CPU time, peak RSS, scheduler IOPS, scheduler bytes read, and the +//! number of logical data ranges requested. Scheduler counters are split by +//! phase but aggregate data and offsets files; exact per-file calls require the +//! scheduler's per-file trace events. For kernel syscall counts on Linux, wrap +//! either command with `strace -f -c -e pread64`. +//! `LANCE_SHUFFLE_BENCH_DISTRIBUTION` (`uniform` or `hotspot`) and +//! `LANCE_SHUFFLE_BENCH_IMPLEMENTATION` (`baseline` or `current`) can isolate a +//! scenario in a fresh process so peak RSS is comparable. + +use std::hint::black_box; +use std::io::Write; +use std::ops::Range; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use arrow::{array::AsArray, compute::concat_batches, datatypes::UInt64Type}; +use arrow_array::{ + ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, UInt8Array, UInt32Array, UInt64Array, +}; +use arrow_schema::Schema; +use async_trait::async_trait; +use criterion::{Criterion, Throughput}; +use futures::{StreamExt, TryStreamExt, stream}; +use lance_arrow::FixedSizeListArrayExt; +use lance_core::cache::LanceCache; +use lance_core::utils::tempfile::TempDir; +use lance_core::utils::tokio::get_num_compute_intensive_cpus; +use lance_core::{Error, ROW_ID}; +use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; +use lance_file::reader::{FileReader, FileReaderOptions}; +use lance_index::vector::PART_ID_COLUMN; +use lance_index::vector::bq::ex_dot::blocked_ex_code_bytes; +use lance_index::vector::bq::storage::{RABIT_BLOCKED_EX_CODE_COLUMN, RABIT_CODE_COLUMN}; +use lance_index::vector::bq::transform::{ + ADD_FACTORS_COLUMN, ERROR_FACTORS_COLUMN, EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN, + SCALE_FACTORS_COLUMN, +}; +use lance_index::vector::bq::{rabit_binary_code_bytes, rabit_ex_bits}; +use lance_index::vector::v3::shuffle_bench::{ + TwoFileShuffleFixtureManifest, open_two_file_shuffle_fixture, +}; +use lance_index::vector::v3::shuffler::{ + DEFAULT_PARTITION_WINDOW_BYTES, ShuffleReader, Shuffler, TwoFileShuffler, +}; +use lance_io::ReadBatchParams; +use lance_io::object_store::ObjectStore; +use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; +use lance_io::scheduler::{bytes_read_counter, iops_counter}; +use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; +use lance_io::utils::CachedFileSize; +use object_store::path::Path; + +const NUM_FLUSH_GROUPS: usize = 20; +const NUM_PARTITIONS: usize = 4_096; +const ROWS_PER_PARTITION: usize = 64; +const RQ_DIMENSION: usize = 256; +const RQ_NUM_BITS: u8 = 5; +const HOTSPOT_TARGET_CV: f64 = 3.6; + +#[derive(Clone, Copy, Debug)] +enum Distribution { + Uniform, + Hotspot, +} + +#[derive(Clone, Copy, Debug)] +enum ReaderImplementation { + Baseline, + Current, +} + +impl ReaderImplementation { + fn name(self) -> &'static str { + match self { + Self::Baseline => "baseline_on_demand_offsets", + Self::Current => "current", + } + } +} + +/// The pre-change two-file reader, kept local to the benchmark so baseline and +/// current read the exact same files in the same process. +struct BaselineTwoFileShuffleReader { + _scheduler: Arc, + file_reader: FileReader, + offsets_reader: FileReader, + num_partitions: usize, + num_flush_groups: u64, + partition_counts: Vec, + total_loss: f64, +} + +impl BaselineTwoFileShuffleReader { + async fn try_new( + output_dir: Path, + manifest: &TwoFileShuffleFixtureManifest, + ) -> lance_core::Result> { + let object_store = Arc::new(ObjectStore::local()); + let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + + let data_path = output_dir.clone().join("shuffle_data.lance"); + let file_reader = FileReader::try_open( + scheduler + .open_file(&data_path, &CachedFileSize::unknown()) + .await?, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await?; + + let offsets_path = output_dir.join("shuffle_offsets.lance"); + let offsets_reader = FileReader::try_open( + scheduler + .open_file(&offsets_path, &CachedFileSize::unknown()) + .await?, + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await?; + + Ok(Arc::new(Self { + _scheduler: scheduler, + file_reader, + offsets_reader, + num_partitions: manifest.num_partitions, + num_flush_groups: manifest.num_flush_groups, + partition_counts: manifest.partition_counts.clone(), + total_loss: manifest.total_loss, + })) + } + + async fn partition_ranges(&self, partition_id: usize) -> lance_core::Result>> { + let mut positions = Vec::with_capacity(self.num_flush_groups as usize * 2); + for group in 0..self.num_flush_groups { + let end_position = u32::try_from(group as usize * self.num_partitions + partition_id) + .map_err(|_| { + Error::invalid_input( + "There are more than 2^32 partition offsets in the spill file. Need to support 64-bit take", + ) + })?; + if end_position != 0 { + positions.push(end_position - 1); + } + positions.push(end_position); + } + + let num_positions = positions.len() as u32; + let positions = UInt32Array::from(positions); + let offsets_stream = self + .offsets_reader + .read_stream( + ReadBatchParams::Indices(positions), + num_positions, + 1, + FilterExpression::no_filter(), + ) + .await?; + let schema = offsets_stream.schema().clone(); + let offsets = offsets_stream.try_collect::>().await?; + let offsets = if offsets.len() == 1 { + offsets.into_iter().next().expect("one offsets batch") + } else { + concat_batches(&schema, &offsets)? + }; + let offsets = offsets.column(0).as_primitive::(); + let mut offsets_iter = offsets.values().iter().copied(); + + let mut ranges = Vec::with_capacity(self.num_flush_groups as usize); + for group in 0..self.num_flush_groups { + if group == 0 && partition_id == 0 { + ranges.push(0..offsets_iter.next().expect("partition end offset")); + } else { + ranges.push( + offsets_iter.next().expect("partition start offset") + ..offsets_iter.next().expect("partition end offset"), + ); + } + } + Ok(ranges) + } +} + +#[async_trait] +impl ShuffleReader for BaselineTwoFileShuffleReader { + async fn read_partition( + &self, + partition_id: usize, + ) -> lance_core::Result>> { + if partition_id >= self.num_partitions || self.partition_counts[partition_id] == 0 { + return Ok(None); + } + + let ranges = self.partition_ranges(partition_id).await?; + let schema: Schema = self.file_reader.schema().as_ref().into(); + let stream = self + .file_reader + .read_stream( + ReadBatchParams::Ranges(ranges.into()), + u32::MAX, + 16, + FilterExpression::no_filter(), + ) + .await?; + Ok(Some(Box::new(RecordBatchStreamAdapter::new( + Arc::new(schema), + stream, + )))) + } + + fn partition_size(&self, partition_id: usize) -> lance_core::Result { + Ok(self + .partition_counts + .get(partition_id) + .copied() + .unwrap_or(0) as usize) + } + + fn total_loss(&self) -> Option { + Some(self.total_loss) + } +} + +impl Distribution { + fn name(self) -> &'static str { + match self { + Self::Uniform => "uniform", + Self::Hotspot => "hotspot_cv_3_6", + } + } +} + +struct Fixture { + _temporary_directory: Option, + output_dir: Path, + manifest_path: PathBuf, + manifest: TwoFileShuffleFixtureManifest, + baseline_reader: Arc, + current_reader: Arc, + total_rows: u64, + coefficient_of_variation: f64, +} + +fn partition_counts(distribution: Distribution) -> Vec { + match distribution { + Distribution::Uniform => vec![ROWS_PER_PARTITION; NUM_PARTITIONS], + Distribution::Hotspot => { + let total_rows = NUM_PARTITIONS * ROWS_PER_PARTITION; + let hotspot_rows = (ROWS_PER_PARTITION as f64 + * (1.0 + HOTSPOT_TARGET_CV * ((NUM_PARTITIONS - 1) as f64).sqrt())) + .round() as usize; + let other_rows = total_rows - hotspot_rows; + let base = other_rows / (NUM_PARTITIONS - 1); + let remainder = other_rows % (NUM_PARTITIONS - 1); + + let hotspot_partition = NUM_PARTITIONS / 2; + let mut counts = Vec::with_capacity(NUM_PARTITIONS); + for partition_id in 0..NUM_PARTITIONS { + if partition_id == hotspot_partition { + counts.push(hotspot_rows); + } else { + let non_hotspot_index = if partition_id < hotspot_partition { + partition_id + } else { + partition_id - 1 + }; + counts.push(base + usize::from(non_hotspot_index < remainder)); + } + } + counts + } + } +} + +fn coefficient_of_variation(counts: &[usize]) -> f64 { + let mean = counts.iter().sum::() as f64 / counts.len() as f64; + let variance = counts + .iter() + .map(|&count| { + let delta = count as f64 - mean; + delta * delta + }) + .sum::() + / counts.len() as f64; + variance.sqrt() / mean +} + +fn rows_in_flush_group(partition_rows: usize, partition_id: usize, group: usize) -> usize { + let base = partition_rows / NUM_FLUSH_GROUPS; + let remainder = partition_rows % NUM_FLUSH_GROUPS; + base + usize::from((group + partition_id * 7) % NUM_FLUSH_GROUPS < remainder) +} + +fn make_rq5_like_batch(counts: &[usize], group: usize, next_row_id: &mut u64) -> RecordBatch { + let num_rows = counts + .iter() + .enumerate() + .map(|(partition_id, &rows)| rows_in_flush_group(rows, partition_id, group)) + .sum::(); + let mut partition_ids = Vec::with_capacity(num_rows); + let mut row_ids = Vec::with_capacity(num_rows); + + // 4051 is coprime to 4096, so every group visits each partition exactly + // once but starts with a different deterministic, non-sorted order. + for slot in 0..NUM_PARTITIONS { + let partition_id = (slot * 4_051 + group * 997) % NUM_PARTITIONS; + let group_rows = rows_in_flush_group(counts[partition_id], partition_id, group); + for _ in 0..group_rows { + partition_ids.push(partition_id as u32); + row_ids.push(*next_row_id); + *next_row_id += 1; + } + } + + let binary_code_bytes = rabit_binary_code_bytes(RQ_DIMENSION); + let ex_bits = rabit_ex_bits(RQ_NUM_BITS).expect("RQ5 must be a valid configuration"); + let ex_code_bytes = blocked_ex_code_bytes(RQ_DIMENSION, ex_bits); + let make_codes = |width: usize, salt: u8| { + let values = (0..num_rows * width) + .map(|index| (index as u8).wrapping_mul(31).wrapping_add(salt)) + .collect::>(); + Arc::new( + FixedSizeListArray::try_new_from_values(UInt8Array::from(values), width as i32) + .expect("valid fixed-size RQ code array"), + ) as ArrayRef + }; + let make_factors = |salt: f32| { + Arc::new(Float32Array::from_iter_values( + (0..num_rows).map(|row| salt + (row % 257) as f32 / 257.0), + )) as ArrayRef + }; + + RecordBatch::try_from_iter(vec![ + ( + PART_ID_COLUMN, + Arc::new(UInt32Array::from(partition_ids)) as ArrayRef, + ), + (ROW_ID, Arc::new(UInt64Array::from(row_ids)) as ArrayRef), + (RABIT_CODE_COLUMN, make_codes(binary_code_bytes, 11)), + (ADD_FACTORS_COLUMN, make_factors(1.0)), + (SCALE_FACTORS_COLUMN, make_factors(2.0)), + (ERROR_FACTORS_COLUMN, make_factors(3.0)), + (RABIT_BLOCKED_EX_CODE_COLUMN, make_codes(ex_code_bytes, 19)), + (EX_ADD_FACTORS_COLUMN, make_factors(4.0)), + (EX_SCALE_FACTORS_COLUMN, make_factors(5.0)), + ]) + .expect("all deterministic fixture columns have equal length") +} + +fn batches_to_stream(batches: Vec) -> Box { + let schema = batches + .first() + .expect("the fixture always has 20 flush groups") + .schema(); + Box::new(RecordBatchStreamAdapter::new( + schema, + stream::iter(batches.into_iter().map(Ok)), + )) +} + +async fn build_fixture(distribution: Distribution) -> Fixture { + let counts = partition_counts(distribution); + let coefficient_of_variation = coefficient_of_variation(&counts); + match distribution { + Distribution::Uniform => assert_eq!(coefficient_of_variation, 0.0), + Distribution::Hotspot => assert!( + (coefficient_of_variation - HOTSPOT_TARGET_CV).abs() < 0.01, + "hotspot fixture CV was {coefficient_of_variation}" + ), + } + + let total_rows = counts.iter().sum::() as u64; + let manifest = TwoFileShuffleFixtureManifest { + num_partitions: NUM_PARTITIONS, + num_flush_groups: NUM_FLUSH_GROUPS as u64, + partition_counts: counts.iter().map(|&count| count as u64).collect(), + total_loss: 0.0, + }; + + let (temporary_directory, fixture_path) = + if let Some(root) = std::env::var_os("LANCE_SHUFFLE_BENCH_FIXTURE_ROOT") { + let fixture_path = PathBuf::from(root).join(distribution.name()); + std::fs::create_dir_all(&fixture_path).expect("create persistent shuffle fixture root"); + (None, fixture_path) + } else { + let directory = TempDir::default(); + let fixture_path = directory.std_path().to_owned(); + (Some(directory), fixture_path) + }; + let output_dir = Path::from_filesystem_path(&fixture_path) + .expect("shuffle fixture path must be a valid object-store path"); + let manifest_path = fixture_path.join("shuffle_manifest.json"); + + if manifest_path.exists() { + let stored: TwoFileShuffleFixtureManifest = serde_json::from_slice( + &std::fs::read(&manifest_path).expect("read existing shuffle fixture manifest"), + ) + .expect("parse existing shuffle fixture manifest"); + assert_eq!( + stored.num_partitions, manifest.num_partitions, + "existing fixture has a different partition count" + ); + assert_eq!( + stored.num_flush_groups, manifest.num_flush_groups, + "existing fixture has a different flush-group count" + ); + assert_eq!( + stored.partition_counts, manifest.partition_counts, + "existing fixture has a different partition distribution" + ); + } else { + let mut next_row_id = 0; + let batches = (0..NUM_FLUSH_GROUPS) + .map(|group| make_rq5_like_batch(&counts, group, &mut next_row_id)) + .collect::>(); + assert_eq!(next_row_id, total_rows); + + let shuffler = TwoFileShuffler::new(output_dir.clone(), NUM_PARTITIONS); + let initial_reader = shuffler + .shuffle(batches_to_stream(batches)) + .await + .expect("write and reopen deterministic two-file shuffle fixture"); + drop(initial_reader); + std::fs::write( + &manifest_path, + serde_json::to_vec_pretty(&manifest).expect("serialize deterministic shuffle manifest"), + ) + .expect("write deterministic shuffle manifest"); + } + + let current_reader = open_two_file_shuffle_fixture(output_dir.clone(), &manifest) + .await + .expect("reopen frozen two-file shuffle fixture"); + let baseline_reader = BaselineTwoFileShuffleReader::try_new(output_dir.clone(), &manifest) + .await + .expect("reopen frozen fixture with baseline reader"); + + Fixture { + _temporary_directory: temporary_directory, + output_dir, + manifest_path, + manifest, + baseline_reader, + current_reader: Arc::from(current_reader), + total_rows, + coefficient_of_variation, + } +} + +async fn reopen_fixture( + fixture: &Fixture, + implementation: ReaderImplementation, +) -> Arc { + let manifest_bytes = + std::fs::read(&fixture.manifest_path).expect("read frozen shuffle fixture manifest"); + let manifest: TwoFileShuffleFixtureManifest = + serde_json::from_slice(&manifest_bytes).expect("parse frozen shuffle fixture manifest"); + assert_eq!(manifest.num_partitions, fixture.manifest.num_partitions); + match implementation { + ReaderImplementation::Baseline => { + BaselineTwoFileShuffleReader::try_new(fixture.output_dir.clone(), &manifest) + .await + .expect("reopen fixture with baseline reader") + } + ReaderImplementation::Current => Arc::from( + open_two_file_shuffle_fixture(fixture.output_dir.clone(), &manifest) + .await + .expect("reopen fixture with current reader"), + ), + } +} + +struct ReadResult { + rows: u64, + windows: usize, +} + +async fn read_all_partitions_baseline( + reader: Arc, + concurrency: usize, +) -> lance_core::Result { + let rows = stream::iter(0..NUM_PARTITIONS) + .map(|partition_id| { + let reader = reader.clone(); + async move { + let Some(mut batches) = reader.read_partition(partition_id).await? else { + return Ok::<_, lance_core::Error>(0u64); + }; + let mut rows = 0u64; + while let Some(batch) = batches.try_next().await? { + rows += batch.num_rows() as u64; + black_box(batch); + } + Ok(rows) + } + }) + .buffered(concurrency) + .try_fold(0u64, |total, rows| async move { Ok(total + rows) }) + .await?; + Ok(ReadResult { + rows, + windows: NUM_PARTITIONS, + }) +} + +async fn read_all_partitions_current( + reader: Arc, +) -> lance_core::Result { + let mut rows = 0u64; + let mut windows = 0usize; + let mut next_partition_id = 0usize; + while next_partition_id < NUM_PARTITIONS { + let window = reader + .read_partition_window(next_partition_id, DEFAULT_PARTITION_WINDOW_BYTES) + .await?; + assert_eq!(window.partition_range.start, next_partition_id); + assert!(window.partition_range.end > next_partition_id); + assert!(window.partition_range.end <= NUM_PARTITIONS); + assert_eq!(window.partition_range.len(), window.partitions.len()); + next_partition_id = window.partition_range.end; + windows += 1; + + for partition in window.partitions { + if let Some(mut batches) = partition.data { + while let Some(batch) = batches.try_next().await? { + rows += batch.num_rows() as u64; + black_box(batch); + } + } + } + } + Ok(ReadResult { rows, windows }) +} + +async fn read_all_partitions( + reader: Arc, + implementation: ReaderImplementation, + concurrency: usize, +) -> lance_core::Result { + match implementation { + ReaderImplementation::Baseline => read_all_partitions_baseline(reader, concurrency).await, + ReaderImplementation::Current => read_all_partitions_current(reader).await, + } +} + +#[cfg(unix)] +fn process_resources() -> (f64, u64) { + // SAFETY: getrusage initializes the provided rusage value and does not + // retain its pointer. RUSAGE_SELF is valid on all Unix targets. + unsafe { + let mut usage: libc::rusage = std::mem::zeroed(); + if libc::getrusage(libc::RUSAGE_SELF, &mut usage) != 0 { + return (0.0, 0); + } + let user_seconds = + usage.ru_utime.tv_sec as f64 + usage.ru_utime.tv_usec as f64 / 1_000_000.0; + let system_seconds = + usage.ru_stime.tv_sec as f64 + usage.ru_stime.tv_usec as f64 / 1_000_000.0; + #[cfg(target_os = "macos")] + let peak_rss_bytes = usage.ru_maxrss as u64; + #[cfg(not(target_os = "macos"))] + let peak_rss_bytes = usage.ru_maxrss as u64 * 1024; + (user_seconds + system_seconds, peak_rss_bytes) + } +} + +#[cfg(not(unix))] +fn process_resources() -> (f64, u64) { + (0.0, 0) +} + +async fn print_diagnostic( + distribution: Distribution, + implementation: ReaderImplementation, + fixture: &Fixture, + concurrency: usize, +) { + let init_iops_before = iops_counter(); + let init_bytes_before = bytes_read_counter(); + let (cpu_before, _) = process_resources(); + let started = Instant::now(); + let reader = reopen_fixture(fixture, implementation).await; + let init_elapsed = started.elapsed(); + let init_iops = iops_counter() - init_iops_before; + let init_bytes_read = bytes_read_counter() - init_bytes_before; + + let read_iops_before = iops_counter(); + let read_bytes_before = bytes_read_counter(); + let read_started = Instant::now(); + let read_result = read_all_partitions(reader, implementation, concurrency) + .await + .expect("read every deterministic partition"); + let read_elapsed = read_started.elapsed(); + let total_elapsed = started.elapsed(); + let (cpu_after, peak_rss_bytes) = process_resources(); + assert_eq!(read_result.rows, fixture.total_rows); + + writeln!( + std::io::stderr().lock(), + "two_file_shuffle_read scenario={} implementation={} flush_groups={} partitions={} rows={} cv={:.4} concurrency={} init_ms={:.3} read_ms={:.3} total_ms={:.3} rows_per_second={:.0} cpu_seconds={:.6} peak_rss_mib={:.1} init_scheduler_iops={} init_scheduler_bytes_read={} read_scheduler_iops={} read_scheduler_bytes_read={} logical_partition_reads={} logical_data_ranges={} offset_entries={} per_file_read_calls=unavailable_use_scheduler_trace", + distribution.name(), + implementation.name(), + NUM_FLUSH_GROUPS, + NUM_PARTITIONS, + read_result.rows, + fixture.coefficient_of_variation, + concurrency, + init_elapsed.as_secs_f64() * 1_000.0, + read_elapsed.as_secs_f64() * 1_000.0, + total_elapsed.as_secs_f64() * 1_000.0, + read_result.rows as f64 / total_elapsed.as_secs_f64(), + cpu_after - cpu_before, + peak_rss_bytes as f64 / (1024.0 * 1024.0), + init_iops, + init_bytes_read, + iops_counter() - read_iops_before, + bytes_read_counter() - read_bytes_before, + read_result.windows, + read_result.windows * NUM_FLUSH_GROUPS, + NUM_PARTITIONS * NUM_FLUSH_GROUPS, + ) + .expect("write shuffle benchmark diagnostic"); +} + +fn bench_two_file_shuffle_read(criterion: &mut Criterion) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("create benchmark runtime"); + let concurrency = get_num_compute_intensive_cpus(); + + let distributions = match std::env::var("LANCE_SHUFFLE_BENCH_DISTRIBUTION").as_deref() { + Ok("uniform") => vec![Distribution::Uniform], + Ok("hotspot") => vec![Distribution::Hotspot], + Ok(value) => panic!("unknown LANCE_SHUFFLE_BENCH_DISTRIBUTION={value}"), + Err(_) => vec![Distribution::Uniform, Distribution::Hotspot], + }; + let implementations = match std::env::var("LANCE_SHUFFLE_BENCH_IMPLEMENTATION").as_deref() { + Ok("baseline") => vec![ReaderImplementation::Baseline], + Ok("current") => vec![ReaderImplementation::Current], + Ok(value) => panic!("unknown LANCE_SHUFFLE_BENCH_IMPLEMENTATION={value}"), + Err(_) => vec![ + ReaderImplementation::Baseline, + ReaderImplementation::Current, + ], + }; + + for distribution in distributions { + let fixture = runtime.block_on(build_fixture(distribution)); + for implementation in implementations.iter().copied() { + runtime.block_on(print_diagnostic( + distribution, + implementation, + &fixture, + concurrency, + )); + let benchmark_id = format!("{}/{}", distribution.name(), implementation.name()); + + let mut reopen_group = criterion.benchmark_group("two_file_shuffle_reopen"); + reopen_group.bench_function(&benchmark_id, |bencher| { + bencher + .to_async(&runtime) + .iter(|| async { black_box(reopen_fixture(&fixture, implementation).await) }); + }); + reopen_group.finish(); + + let reader = match implementation { + ReaderImplementation::Baseline => fixture.baseline_reader.clone(), + ReaderImplementation::Current => fixture.current_reader.clone(), + }; + let mut read_group = criterion.benchmark_group("two_file_shuffle_read_only"); + read_group.throughput(Throughput::Elements(fixture.total_rows)); + read_group.bench_function(&benchmark_id, |bencher| { + bencher.to_async(&runtime).iter(|| async { + let result = read_all_partitions(reader.clone(), implementation, concurrency) + .await + .expect("read every deterministic partition"); + assert_eq!(result.rows, fixture.total_rows); + black_box(result.rows) + }); + }); + read_group.finish(); + + let mut total_group = criterion.benchmark_group("two_file_shuffle_reopen_and_read"); + total_group.throughput(Throughput::Elements(fixture.total_rows)); + total_group.bench_function(&benchmark_id, |bencher| { + bencher.to_async(&runtime).iter(|| async { + let reader = reopen_fixture(&fixture, implementation).await; + let result = read_all_partitions(reader, implementation, concurrency) + .await + .expect("read every deterministic partition"); + assert_eq!(result.rows, fixture.total_rows); + black_box(result.rows) + }); + }); + total_group.finish(); + } + } +} + +fn main() { + // SAFETY: this is the first action in this single-purpose benchmark binary, + // before the Tokio runtime or any other worker threads are created. + unsafe { + std::env::set_var("LANCE_SHUFFLE_BATCH_BYTES", "1"); + } + + let mut criterion = Criterion::default() + .sample_size(10) + .warm_up_time(Duration::from_secs(2)) + .measurement_time(Duration::from_secs(8)) + .configure_from_args(); + bench_two_file_shuffle_read(&mut criterion); + criterion.final_summary(); +} diff --git a/rust/lance-index/examples/acorn_bench.rs b/rust/lance-index/examples/acorn_bench.rs new file mode 100644 index 00000000000..8a24088e89b --- /dev/null +++ b/rust/lance-index/examples/acorn_bench.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Compare the current mask-aware HNSW traversal (`search_basic`) against an +//! ACORN-1 style traversal (`search_acorn`) and a flat scan over matching +//! rows, across filter selectivities and mask shapes: +//! +//! - `corr-in`: mask is a cluster in embedding space, query is inside it +//! (e.g. "filter to courses" with a course query) +//! - `corr-out`: mask is a cluster, query is unrelated +//! - `random`: uniform random mask (the easy case) +//! +//! Run: cargo run --release -p lance-index --example acorn_bench + +#![allow(clippy::print_stdout)] + +use std::sync::Arc; +use std::time::Instant; + +use arrow_array::{Array, FixedSizeListArray, types::Float32Type}; +use lance_arrow::FixedSizeListArrayExt; +use lance_index::vector::flat::storage::FlatFloatStorage; +use lance_index::vector::graph::VisitedGenerator; +use lance_index::vector::hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams}; +use lance_index::vector::storage::{DistCalculator, VectorStore}; +use lance_index::vector::v3::subindex::IvfSubIndex; +use lance_linalg::distance::DistanceType; +use lance_testing::datagen::generate_random_array_with_seed; +use rand::rngs::SmallRng; +use rand::seq::{IndexedRandom, SliceRandom}; +use rand::{Rng, SeedableRng}; + +const TOTAL: usize = 100_000; +const DIMENSION: usize = 768; +const K: usize = 10; +const EF: usize = 100; +const EF_HI: usize = 400; +const QUERIES_PER_CASE: usize = 20; +const SEED: [u8; 32] = [42; 32]; + +fn all_distances(storage: &FlatFloatStorage, query: Arc) -> Vec { + let dist_calc = storage.dist_calculator(query, 0.0); + (0..TOTAL as u32).map(|i| dist_calc.distance(i)).collect() +} + +/// Node ids sorted ascending by distance to `query`, restricted to `mask`. +fn ground_truth(storage: &FlatFloatStorage, query: Arc, mask: &[bool]) -> Vec { + let dists = all_distances(storage, query); + let mut ids: Vec = (0..TOTAL as u32).filter(|&i| mask[i as usize]).collect(); + ids.sort_by(|&a, &b| dists[a as usize].partial_cmp(&dists[b as usize]).unwrap()); + ids.truncate(K); + ids +} + +fn recall(got: &[u32], truth: &[u32]) -> f64 { + let hits = truth.iter().filter(|id| got.contains(id)).count(); + hits as f64 / truth.len().max(1) as f64 +} + +struct CaseResult { + latency_us: Vec, + recalls: Vec, +} + +impl CaseResult { + fn new() -> Self { + Self { + latency_us: vec![], + recalls: vec![], + } + } + fn median_latency_ms(&mut self) -> f64 { + self.latency_us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + self.latency_us[self.latency_us.len() / 2] / 1000.0 + } + fn mean_recall(&self) -> f64 { + self.recalls.iter().sum::() / self.recalls.len() as f64 + } +} + +fn main() { + println!("generating {TOTAL} x {DIMENSION} vectors..."); + let data = generate_random_array_with_seed::(TOTAL * DIMENSION, SEED); + let fsl = FixedSizeListArray::try_new_from_values(data, DIMENSION as i32).unwrap(); + let storage = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + + println!("building HNSW graph..."); + let build_start = Instant::now(); + let hnsw = HNSW::index_vectors(storage.as_ref(), HnswBuildParams::default()).unwrap(); + println!("built in {:.1}s", build_start.elapsed().as_secs_f32()); + + let mut rng = SmallRng::seed_from_u64(7); + let params = HnswQueryParams { + ef: EF, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let params_hi = HnswQueryParams { + ef: EF_HI, + ..params + }; + + // unfiltered control: this graph's recall ceiling at ef=EF + { + let mut control = CaseResult::new(); + let all_mask = vec![true; TOTAL]; + for _ in 0..QUERIES_PER_CASE { + let query_id = rng.random_range(0..TOTAL as u32); + let query = fsl.value(query_id as usize); + let truth = ground_truth(&storage, query.clone(), &all_mask); + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms, None, storage.as_ref()) + .unwrap(); + control.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + control.recalls.push(recall(&got, &truth)); + } + println!( + "\nunfiltered control (ef={EF}): {:.2} ms, recall@{K} {:.3}", + control.median_latency_ms(), + control.mean_recall() + ); + } + + println!( + "\n{:<10} {:>5} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7} | {:>9} {:>7}", + "mask", + "sel%", + "basic ms", + "recall", + "basic4x ms", + "recall", + "acorn ms", + "recall", + "flat ms", + "recall" + ); + + for mask_kind in ["corr-in", "corr-out", "random"] { + for selectivity in [0.02f64, 0.05, 0.10, 0.25, 0.50] { + let mask_size = (TOTAL as f64 * selectivity) as usize; + let mut basic = CaseResult::new(); + let mut basic_hi = CaseResult::new(); + let mut acorn = CaseResult::new(); + let mut flat = CaseResult::new(); + let mut mask_generator = VisitedGenerator::new(TOTAL); + + for _ in 0..QUERIES_PER_CASE { + let mut mask = vec![false; TOTAL]; + let anchor_id = rng.random_range(0..TOTAL as u32); + let member_ids: Vec = match mask_kind { + "random" => { + let mut ids: Vec = (0..TOTAL as u32).collect(); + ids.shuffle(&mut rng); + ids.truncate(mask_size); + ids + } + _ => { + // cluster: the nearest nodes to a random anchor + let dists = all_distances(&storage, fsl.value(anchor_id as usize)); + let mut ids: Vec = (0..TOTAL as u32).collect(); + ids.sort_by(|&a, &b| { + dists[a as usize].partial_cmp(&dists[b as usize]).unwrap() + }); + ids.truncate(mask_size); + ids + } + }; + for &id in &member_ids { + mask[id as usize] = true; + } + + let query_id = match mask_kind { + "corr-in" => *member_ids.choose(&mut rng).unwrap(), + _ => rng.random_range(0..TOTAL as u32), + }; + let query = fsl.value(query_id as usize); + let truth = ground_truth(&storage, query.clone(), &mask); + + // search_basic (current traversal) + { + let mut bitset = mask_generator.generate(TOTAL); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms, Some(bitset), storage.as_ref()) + .unwrap(); + basic.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic.recalls.push(recall(&got, &truth)); + } + + // search_basic at 4x ef (recall-equalizing baseline) + { + let mut bitset = mask_generator.generate(TOTAL); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms_hi, Some(bitset), storage.as_ref()) + .unwrap(); + basic_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic_hi.recalls.push(recall(&got, &truth)); + } + + // search_acorn + { + let mut bitset = mask_generator.generate(TOTAL); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_acorn(query.clone(), K, ¶ms, &bitset, storage.as_ref()) + .unwrap(); + acorn.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + acorn.recalls.push(recall(&got, &truth)); + } + + // flat scan over matching rows + { + let t = Instant::now(); + let dist_calc = storage.dist_calculator(query.clone(), 0.0); + let mut scored: Vec<(f32, u32)> = member_ids + .iter() + .map(|&id| (dist_calc.distance(id), id)) + .collect(); + scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + scored.truncate(K); + flat.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = scored.iter().map(|&(_, id)| id).collect(); + flat.recalls.push(recall(&got, &truth)); + } + } + + println!( + "{:<10} {:>5.0} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3} | {:>9.2} {:>7.3}", + mask_kind, + selectivity * 100.0, + basic.median_latency_ms(), + basic.mean_recall(), + basic_hi.median_latency_ms(), + basic_hi.mean_recall(), + acorn.median_latency_ms(), + acorn.mean_recall(), + flat.median_latency_ms(), + flat.mean_recall(), + ); + } + } +} diff --git a/rust/lance-index/examples/acorn_bench_sift.rs b/rust/lance-index/examples/acorn_bench_sift.rs new file mode 100644 index 00000000000..c781ae12577 --- /dev/null +++ b/rust/lance-index/examples/acorn_bench_sift.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! SIFT1M variant of `acorn_bench`: compares `search_basic`, `search_acorn`, +//! and a flat scan on the ANN_SIFT1M dataset (1M x 128, L2) with synthetic +//! filter masks. The unfiltered control is checked against the official +//! ground truth. +//! +//! Run: SIFT_DIR=/path/to/sift cargo run --release -p lance-index --example acorn_bench_sift +//! Works with any texmex-format dataset, e.g. SIFT_DIR=/path/to/gist for GIST1M. + +#![allow(clippy::print_stdout)] + +use std::sync::Arc; +use std::time::Instant; + +use arrow_array::{Array, FixedSizeListArray, Float32Array}; +use lance_arrow::FixedSizeListArrayExt; +use lance_index::vector::flat::storage::FlatFloatStorage; +use lance_index::vector::graph::VisitedGenerator; +use lance_index::vector::hnsw::builder::{HNSW, HnswBuildParams, HnswQueryParams}; +use lance_index::vector::storage::{DistCalculator, VectorStore}; +use lance_index::vector::v3::subindex::IvfSubIndex; +use lance_linalg::distance::DistanceType; +use rand::rngs::SmallRng; +use rand::seq::SliceRandom; +use rand::{Rng, SeedableRng}; + +const K: usize = 10; +const EF: usize = 100; +const EF_HI: usize = 400; +const QUERIES_PER_CASE: usize = 20; + +/// (flat values, dim, count) +fn read_fvecs(path: &str) -> (Vec, usize, usize) { + let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path}: {e}")); + let dim = i32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize; + let record = 4 + dim * 4; + assert_eq!(bytes.len() % record, 0); + let count = bytes.len() / record; + let mut values = Vec::with_capacity(count * dim); + for row in 0..count { + let start = row * record + 4; + for i in 0..dim { + let offset = start + i * 4; + values.push(f32::from_le_bytes( + bytes[offset..offset + 4].try_into().unwrap(), + )); + } + } + (values, dim, count) +} + +fn read_ivecs(path: &str) -> Vec> { + let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {path}: {e}")); + let dim = i32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize; + let record = 4 + dim * 4; + assert_eq!(bytes.len() % record, 0); + (0..bytes.len() / record) + .map(|row| { + let start = row * record + 4; + (0..dim) + .map(|i| { + let offset = start + i * 4; + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) + }) + .collect() + }) + .collect() +} + +fn all_distances(storage: &FlatFloatStorage, total: usize, query: Arc) -> Vec { + let dist_calc = storage.dist_calculator(query, 0.0); + (0..total as u32).map(|i| dist_calc.distance(i)).collect() +} + +fn ground_truth( + storage: &FlatFloatStorage, + total: usize, + query: Arc, + mask: &[bool], +) -> Vec { + let dists = all_distances(storage, total, query); + let mut ids: Vec = (0..total as u32).filter(|&i| mask[i as usize]).collect(); + ids.sort_by(|&a, &b| dists[a as usize].partial_cmp(&dists[b as usize]).unwrap()); + ids.truncate(K); + ids +} + +fn recall(got: &[u32], truth: &[u32]) -> f64 { + let hits = truth.iter().filter(|id| got.contains(id)).count(); + hits as f64 / truth.len().max(1) as f64 +} + +struct CaseResult { + latency_us: Vec, + recalls: Vec, +} + +impl CaseResult { + fn new() -> Self { + Self { + latency_us: vec![], + recalls: vec![], + } + } + fn median_latency_ms(&mut self) -> f64 { + self.latency_us.sort_by(|a, b| a.partial_cmp(b).unwrap()); + self.latency_us[self.latency_us.len() / 2] / 1000.0 + } + fn mean_recall(&self) -> f64 { + self.recalls.iter().sum::() / self.recalls.len() as f64 + } +} + +fn main() { + let sift_dir = std::env::var("SIFT_DIR").expect("set SIFT_DIR to the extracted dataset dir"); + let prefix = std::path::Path::new(&sift_dir) + .file_name() + .and_then(|name| name.to_str()) + .expect("SIFT_DIR must end in the dataset name, e.g. sift or gist") + .to_string(); + println!("loading {prefix} from {sift_dir}..."); + let (base, dim, total) = read_fvecs(&format!("{sift_dir}/{prefix}_base.fvecs")); + let (queries, query_dim, num_queries) = read_fvecs(&format!("{sift_dir}/{prefix}_query.fvecs")); + let official_gt = read_ivecs(&format!("{sift_dir}/{prefix}_groundtruth.ivecs")); + assert_eq!(dim, query_dim); + println!("base {total} x {dim}, {num_queries} queries"); + + let fsl = + FixedSizeListArray::try_new_from_values(Float32Array::from(base), dim as i32).unwrap(); + let storage = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let query_vec = |i: usize| -> Arc { + Arc::new(Float32Array::from(queries[i * dim..(i + 1) * dim].to_vec())) + }; + + println!("building HNSW graph..."); + let build_start = Instant::now(); + let hnsw = HNSW::index_vectors(storage.as_ref(), HnswBuildParams::default()).unwrap(); + println!("built in {:.1}s", build_start.elapsed().as_secs_f32()); + + let mut rng = SmallRng::seed_from_u64(7); + let params = HnswQueryParams { + ef: EF, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let params_hi = HnswQueryParams { + ef: EF_HI, + ..params + }; + + // unfiltered control against the official ground truth + { + let mut control = CaseResult::new(); + for _ in 0..QUERIES_PER_CASE { + let query_id = rng.random_range(0..num_queries); + let query = query_vec(query_id); + let truth: Vec = official_gt[query_id][..K].to_vec(); + let t = Instant::now(); + let nodes = hnsw + .search_basic(query, K, ¶ms, None, storage.as_ref()) + .unwrap(); + control.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + control.recalls.push(recall(&got, &truth)); + } + println!( + "\nunfiltered control (ef={EF}, official GT): {:.2} ms, recall@{K} {:.3}", + control.median_latency_ms(), + control.mean_recall() + ); + } + + println!( + "\n{:<10} {:>5} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7} | {:>10} {:>7} | {:>9} {:>7}", + "mask", + "sel%", + "basic ms", + "recall", + "basic4x ms", + "recall", + "acorn ms", + "recall", + "acorn4x ms", + "recall", + "flat ms", + "recall" + ); + + for mask_kind in ["corr-in", "corr-out", "random"] { + for selectivity in [0.02f64, 0.05, 0.10, 0.25, 0.50] { + let mask_size = (total as f64 * selectivity) as usize; + let mut basic = CaseResult::new(); + let mut basic_hi = CaseResult::new(); + let mut acorn = CaseResult::new(); + let mut acorn_hi = CaseResult::new(); + let mut flat = CaseResult::new(); + let mut mask_generator = VisitedGenerator::new(total); + + for _ in 0..QUERIES_PER_CASE { + let query_id = rng.random_range(0..num_queries); + let query = query_vec(query_id); + + // corr-in: cluster around the query's own neighborhood + // corr-out: cluster around an unrelated base vector + let mut mask = vec![false; total]; + let member_ids: Vec = match mask_kind { + "random" => { + let mut ids: Vec = (0..total as u32).collect(); + ids.shuffle(&mut rng); + ids.truncate(mask_size); + ids + } + _ => { + let anchor: Arc = match mask_kind { + "corr-in" => query.clone(), + _ => fsl.value(rng.random_range(0..total)), + }; + let dists = all_distances(&storage, total, anchor); + let mut ids: Vec = (0..total as u32).collect(); + ids.sort_by(|&a, &b| { + dists[a as usize].partial_cmp(&dists[b as usize]).unwrap() + }); + ids.truncate(mask_size); + ids + } + }; + for &id in &member_ids { + mask[id as usize] = true; + } + let truth = ground_truth(&storage, total, query.clone(), &mask); + + // search_basic (current traversal) + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms, Some(bitset), storage.as_ref()) + .unwrap(); + basic.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic.recalls.push(recall(&got, &truth)); + } + + // search_basic at 4x ef (recall-equalizing baseline) + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_basic(query.clone(), K, ¶ms_hi, Some(bitset), storage.as_ref()) + .unwrap(); + basic_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + basic_hi.recalls.push(recall(&got, &truth)); + } + + // search_acorn + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_acorn(query.clone(), K, ¶ms, &bitset, storage.as_ref()) + .unwrap(); + acorn.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + acorn.recalls.push(recall(&got, &truth)); + } + + // search_acorn at 4x ef + { + let mut bitset = mask_generator.generate(total); + for &id in &member_ids { + bitset.insert(id); + } + let t = Instant::now(); + let nodes = hnsw + .search_acorn(query.clone(), K, ¶ms_hi, &bitset, storage.as_ref()) + .unwrap(); + acorn_hi.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = nodes.iter().map(|n| n.id).collect(); + acorn_hi.recalls.push(recall(&got, &truth)); + } + + // flat scan over matching rows + { + let t = Instant::now(); + let dist_calc = storage.dist_calculator(query.clone(), 0.0); + let mut scored: Vec<(f32, u32)> = member_ids + .iter() + .map(|&id| (dist_calc.distance(id), id)) + .collect(); + scored.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + scored.truncate(K); + flat.latency_us.push(t.elapsed().as_secs_f64() * 1e6); + let got: Vec = scored.iter().map(|&(_, id)| id).collect(); + flat.recalls.push(recall(&got, &truth)); + } + } + + println!( + "{:<10} {:>5.0} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3} | {:>10.2} {:>7.3} | {:>9.2} {:>7.3}", + mask_kind, + selectivity * 100.0, + basic.median_latency_ms(), + basic.mean_recall(), + basic_hi.median_latency_ms(), + basic_hi.mean_recall(), + acorn.median_latency_ms(), + acorn.mean_recall(), + acorn_hi.median_latency_ms(), + acorn_hi.mean_recall(), + flat.median_latency_ms(), + flat.mean_recall(), + ); + } + } +} diff --git a/rust/lance-index/protos-cache/cache.proto b/rust/lance-index/protos-cache/cache.proto index b24a27055d7..efe16e92ea0 100644 --- a/rust/lance-index/protos-cache/cache.proto +++ b/rust/lance-index/protos-cache/cache.proto @@ -28,6 +28,11 @@ message CompressedPostingHeader { PositionStorage position_storage = 4; // Only meaningful when position_storage == POSITION_STORAGE_SHARED. PositionStreamCodec position_stream_codec = 5; + // Number of documents in each compressed posting block. Older cache entries + // omit this field and decode as the legacy 128-doc block size. + uint32 block_size = 6; + // Whether an impact IPC section follows the posting/position sections. + bool has_impacts = 7; } // Header for a serialized `PlainPostingList` cache entry. Followed by an Arrow @@ -164,6 +169,9 @@ message SqPartitionHeader { // three raw blobs: the IVF model protobuf, the quantizer's extra-metadata // buffer (may be empty), and the auxiliary IVF model protobuf. message IvfStateHeader { + reserved 8; + reserved "cache_key_prefix"; + string index_file_path = 1; string uuid = 2; string distance_type = 3; @@ -174,7 +182,6 @@ message IvfStateHeader { // type is generic over the quantizer; the proto envelope still provides // additive evolution for the surrounding fields. string quantizer_metadata_json = 7; - string cache_key_prefix = 8; uint64 index_file_size = 9; uint64 aux_file_size = 10; } diff --git a/rust/lance-index/src/frag_reuse.rs b/rust/lance-index/src/frag_reuse.rs index 4c70db44094..12cd490e5e3 100644 --- a/rust/lance-index/src/frag_reuse.rs +++ b/rust/lance-index/src/frag_reuse.rs @@ -5,14 +5,16 @@ //! //! The data structures and table-format logic live in //! [`lance_table::system_index::frag_reuse`]; this module re-exports them and -//! implements the local [`Index`] trait for [`FragReuseIndex`]. +//! provides newtype wrappers that implement the [`Index`] and [`RowIdRemapper`] +//! traits. use std::any::Any; use std::sync::Arc; use arrow_array::RecordBatch; use async_trait::async_trait; -use lance_core::{Error, Result}; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; use lance_select::RowAddrTreeMap; use roaring::{RoaringBitmap, RoaringTreemap}; use serde::Serialize; @@ -22,17 +24,80 @@ pub use lance_table::system_index::frag_reuse::*; use crate::scalar::RowIdRemapper; use crate::{Index, IndexType}; -impl RowIdRemapper for FragReuseIndex { +/// Newtype wrapping [`FragReuseIndex`] so that `lance-index` can implement +/// the `Index` and `RowIdRemapper` traits (orphan rules prevent implementing +/// them directly in `lance-table`). +pub struct FragReuseIndexHandle(pub Arc); + +/// Adapter for the compact runtime representation loaded from persisted FRI details. +#[doc(hidden)] +pub struct CompactFragReuseIndexHandle(pub Arc); + +impl std::fmt::Debug for FragReuseIndexHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("FragReuseIndexHandle") + .field(&self.0) + .finish() + } +} + +impl DeepSizeOf for FragReuseIndexHandle { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.0.deep_size_of_children(context) + } +} + +#[derive(Serialize)] +struct FragReuseStatistics { + num_versions: usize, +} + +#[async_trait] +impl Index for FragReuseIndexHandle { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_index(self: Arc) -> Arc { + self + } + + fn statistics(&self) -> Result { + let stats = FragReuseStatistics { + num_versions: self.0.details.versions.len(), + }; + serde_json::to_value(stats).map_err(|e| { + lance_core::Error::internal(format!( + "failed to serialize fragment reuse index statistics: {}", + e + )) + }) + } + + async fn prewarm(&self) -> Result<()> { + Ok(()) + } + + fn index_type(&self) -> IndexType { + IndexType::FragmentReuse + } + + async fn calculate_included_frags(&self) -> Result { + unimplemented!() + } +} + +impl RowIdRemapper for FragReuseIndexHandle { fn remap_row_id(&self, row_id: u64) -> Option { - self.remap_row_id(row_id) + self.0.remap_row_id(row_id) } fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { - self.remap_row_addrs_tree_map(row_addrs) + self.0.remap_row_addrs_tree_map(row_addrs) } fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap { - self.remap_row_ids_roaring_tree_map(row_ids) + self.0.remap_row_ids_roaring_tree_map(row_ids) } fn remap_row_ids_record_batch( @@ -40,17 +105,26 @@ impl RowIdRemapper for FragReuseIndex { batch: RecordBatch, row_id_idx: usize, ) -> Result { - self.remap_row_ids_record_batch(batch, row_id_idx) + self.0.remap_row_ids_record_batch(batch, row_id_idx) } } -#[derive(Serialize)] -struct FragReuseStatistics { - num_versions: usize, +impl std::fmt::Debug for CompactFragReuseIndexHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("CompactFragReuseIndexHandle") + .field(&self.0) + .finish() + } +} + +impl DeepSizeOf for CompactFragReuseIndexHandle { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.0.deep_size_of_children(context) + } } #[async_trait] -impl Index for FragReuseIndex { +impl Index for CompactFragReuseIndexHandle { fn as_any(&self) -> &dyn Any { self } @@ -61,10 +135,10 @@ impl Index for FragReuseIndex { fn statistics(&self) -> Result { let stats = FragReuseStatistics { - num_versions: self.details.versions.len(), + num_versions: self.0.details.versions.len(), }; serde_json::to_value(stats).map_err(|e| { - Error::internal(format!( + lance_core::Error::internal(format!( "failed to serialize fragment reuse index statistics: {}", e )) @@ -83,3 +157,25 @@ impl Index for FragReuseIndex { unimplemented!() } } + +impl RowIdRemapper for CompactFragReuseIndexHandle { + fn remap_row_id(&self, row_id: u64) -> Option { + self.0.remap_row_id(row_id) + } + + fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { + self.0.remap_row_addrs_tree_map(row_addrs) + } + + fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap { + self.0.remap_row_ids_roaring_tree_map(row_ids) + } + + fn remap_row_ids_record_batch( + &self, + batch: RecordBatch, + row_id_idx: usize, + ) -> Result { + self.0.remap_row_ids_record_batch(batch, row_id_idx) + } +} diff --git a/rust/lance-index/src/lib.rs b/rust/lance-index/src/lib.rs index 61b45550367..52fa9e5a5de 100644 --- a/rust/lance-index/src/lib.rs +++ b/rust/lance-index/src/lib.rs @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +#![cfg_attr(coverage, feature(coverage_attribute))] //! Lance secondary index library //! @@ -9,16 +10,9 @@ //! API stability is not guaranteed. //! -use std::{any::Any, sync::Arc}; - use crate::frag_reuse::FRAG_REUSE_INDEX_NAME; use crate::mem_wal::MEM_WAL_INDEX_NAME; -use async_trait::async_trait; -use lance_core::deepsize::DeepSizeOf; -use lance_core::{Error, Result}; -use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; -use std::convert::TryFrom; pub mod frag_reuse; pub mod mem_wal; @@ -33,6 +27,9 @@ pub mod vector; pub use crate::traits::*; +// Re-export core traits from lance-index-core +pub use lance_index_core::{Index, IndexParams, IndexType}; + pub const INDEX_FILE_NAME: &str = "index.idx"; /// The name of the auxiliary index file. /// @@ -75,280 +72,6 @@ pub mod cache_pb { include!(concat!(env!("OUT_DIR"), "/lance.index.cache.rs")); } -/// Generic methods common across all types of secondary indices -/// -#[async_trait] -pub trait Index: Send + Sync + DeepSizeOf { - /// Cast to [Any]. - fn as_any(&self) -> &dyn Any; - - /// Cast to [Index] - fn as_index(self: Arc) -> Arc; - - /// Retrieve index statistics as a JSON Value - fn statistics(&self) -> Result; - - /// Prewarm the index. - /// - /// This will load the index into memory and cache it. - async fn prewarm(&self) -> Result<()>; - - /// Get the type of the index - fn index_type(&self) -> IndexType; - - /// Read through the index and determine which fragment ids are covered by the index - /// - /// This is a kind of slow operation. It's better to use the fragment_bitmap. This - /// only exists for cases where the fragment_bitmap has become corrupted or missing. - async fn calculate_included_frags(&self) -> Result; -} - -/// Index Type -#[derive(Debug, PartialEq, Eq, Copy, Hash, Clone, DeepSizeOf)] -pub enum IndexType { - // Preserve 0-100 for simple indices. - Scalar = 0, // Legacy scalar index, alias to BTree - - BTree = 1, // BTree - - Bitmap = 2, // Bitmap - - LabelList = 3, // LabelList - - Inverted = 4, // Inverted - - NGram = 5, // NGram - - FragmentReuse = 6, - - MemWal = 7, - - ZoneMap = 8, // ZoneMap - - BloomFilter = 9, // Bloom filter - - RTree = 10, // RTree - - Fm = 11, // FM-Index - - // 100+ and up for vector index. - /// Flat vector index. - Vector = 100, // Legacy vector index, alias to IvfPq - IvfFlat = 101, - IvfSq = 102, - IvfPq = 103, - IvfHnswSq = 104, - IvfHnswPq = 105, - IvfHnswFlat = 106, - IvfRq = 107, -} - -impl std::fmt::Display for IndexType { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Self::Scalar | Self::BTree => write!(f, "BTree"), - Self::Bitmap => write!(f, "Bitmap"), - Self::LabelList => write!(f, "LabelList"), - Self::Inverted => write!(f, "Inverted"), - Self::NGram => write!(f, "NGram"), - Self::FragmentReuse => write!(f, "FragmentReuse"), - Self::MemWal => write!(f, "MemWal"), - Self::ZoneMap => write!(f, "ZoneMap"), - Self::BloomFilter => write!(f, "BloomFilter"), - Self::RTree => write!(f, "RTree"), - Self::Fm => write!(f, "Fm"), - Self::Vector | Self::IvfPq => write!(f, "IVF_PQ"), - Self::IvfFlat => write!(f, "IVF_FLAT"), - Self::IvfSq => write!(f, "IVF_SQ"), - Self::IvfHnswSq => write!(f, "IVF_HNSW_SQ"), - Self::IvfHnswPq => write!(f, "IVF_HNSW_PQ"), - Self::IvfHnswFlat => write!(f, "IVF_HNSW_FLAT"), - Self::IvfRq => write!(f, "IVF_RQ"), - } - } -} - -impl TryFrom for IndexType { - type Error = Error; - - fn try_from(value: i32) -> Result { - match value { - v if v == Self::Scalar as i32 => Ok(Self::Scalar), - v if v == Self::BTree as i32 => Ok(Self::BTree), - v if v == Self::Bitmap as i32 => Ok(Self::Bitmap), - v if v == Self::LabelList as i32 => Ok(Self::LabelList), - v if v == Self::NGram as i32 => Ok(Self::NGram), - v if v == Self::Inverted as i32 => Ok(Self::Inverted), - v if v == Self::FragmentReuse as i32 => Ok(Self::FragmentReuse), - v if v == Self::MemWal as i32 => Ok(Self::MemWal), - v if v == Self::ZoneMap as i32 => Ok(Self::ZoneMap), - v if v == Self::BloomFilter as i32 => Ok(Self::BloomFilter), - v if v == Self::RTree as i32 => Ok(Self::RTree), - v if v == Self::Fm as i32 => Ok(Self::Fm), - v if v == Self::Vector as i32 => Ok(Self::Vector), - v if v == Self::IvfFlat as i32 => Ok(Self::IvfFlat), - v if v == Self::IvfSq as i32 => Ok(Self::IvfSq), - v if v == Self::IvfPq as i32 => Ok(Self::IvfPq), - v if v == Self::IvfHnswSq as i32 => Ok(Self::IvfHnswSq), - v if v == Self::IvfHnswPq as i32 => Ok(Self::IvfHnswPq), - v if v == Self::IvfHnswFlat as i32 => Ok(Self::IvfHnswFlat), - v if v == Self::IvfRq as i32 => Ok(Self::IvfRq), - _ => Err(Error::invalid_input_source( - format!("the input value {} is not a valid IndexType", value).into(), - )), - } - } -} - -impl TryFrom<&str> for IndexType { - type Error = Error; - - fn try_from(value: &str) -> Result { - match value { - "BTree" | "BTREE" => Ok(Self::BTree), - "Bitmap" | "BITMAP" => Ok(Self::Bitmap), - "LabelList" | "LABELLIST" => Ok(Self::LabelList), - "Inverted" | "INVERTED" => Ok(Self::Inverted), - "NGram" | "NGRAM" => Ok(Self::NGram), - "ZoneMap" | "ZONEMAP" => Ok(Self::ZoneMap), - "BloomFilter" | "BLOOMFILTER" | "BLOOM_FILTER" => Ok(Self::BloomFilter), - "RTree" | "RTREE" | "R_TREE" => Ok(Self::RTree), - "Fm" | "FM" => Ok(Self::Fm), - "Vector" | "VECTOR" => Ok(Self::Vector), - "IVF_FLAT" => Ok(Self::IvfFlat), - "IVF_SQ" => Ok(Self::IvfSq), - "IVF_PQ" => Ok(Self::IvfPq), - "IVF_RQ" => Ok(Self::IvfRq), - "IVF_HNSW_FLAT" => Ok(Self::IvfHnswFlat), - "IVF_HNSW_SQ" => Ok(Self::IvfHnswSq), - "IVF_HNSW_PQ" => Ok(Self::IvfHnswPq), - "FragmentReuse" => Ok(Self::FragmentReuse), - "MemWal" => Ok(Self::MemWal), - _ => Err(Error::invalid_input(format!( - "invalid index type: {}", - value - ))), - } - } -} - -impl IndexType { - pub fn is_scalar(&self) -> bool { - matches!( - self, - Self::Scalar - | Self::BTree - | Self::Bitmap - | Self::LabelList - | Self::Inverted - | Self::NGram - | Self::ZoneMap - | Self::BloomFilter - | Self::RTree - | Self::Fm, - ) - } - - pub fn is_vector(&self) -> bool { - matches!( - self, - Self::Vector - | Self::IvfPq - | Self::IvfHnswSq - | Self::IvfHnswPq - | Self::IvfHnswFlat - | Self::IvfFlat - | Self::IvfSq - | Self::IvfRq - ) - } - - pub fn is_system(&self) -> bool { - matches!(self, Self::FragmentReuse | Self::MemWal) - } - - /// Returns the current format version of the index type, - /// bump this when the index format changes. - /// Indices which higher version than these will be ignored for compatibility, - /// This would happen when creating index in a newer version of Lance, - /// but then opening the index in older version of Lance - pub fn version(&self) -> i32 { - match self { - Self::Scalar => 0, - Self::BTree => 0, - Self::Bitmap => 0, - Self::LabelList => 0, - Self::Inverted => 0, - Self::NGram => 0, - Self::FragmentReuse => 0, - Self::MemWal => 0, - Self::ZoneMap => 0, - Self::BloomFilter => 0, - Self::RTree => 0, - Self::Fm => 0, - - // IMPORTANT: if any vector index subtype needs a format bump that is - // not backward compatible, its new version must be set to - // (current max vector index version + 1), even if only one subtype - // changed. Compatibility filtering currently cannot distinguish vector - // subtypes from details-only metadata, so vector versions effectively - // share one global monotonic compatibility level. - Self::Vector - | Self::IvfFlat - | Self::IvfSq - | Self::IvfPq - | Self::IvfHnswSq - | Self::IvfHnswPq - | Self::IvfHnswFlat => VECTOR_INDEX_VERSION as i32, - Self::IvfRq => IVF_RQ_INDEX_VERSION as i32, - } - } - - /// Returns the target partition size for the index type. - /// - /// This is used to compute the number of partitions for the index. - /// The partition size is optimized for the best performance of the index. - /// - /// This is for vector indices only. - pub fn target_partition_size(&self) -> usize { - match self { - Self::Vector => 8192, - Self::IvfFlat => 4096, - Self::IvfSq => 8192, - Self::IvfPq => 8192, - Self::IvfRq => 4096, - Self::IvfHnswFlat => 1 << 20, - Self::IvfHnswSq => 1 << 20, - Self::IvfHnswPq => 1 << 20, - _ => 8192, - } - } - - /// Returns the highest supported vector index version in this Lance build. - pub fn max_vector_version() -> u32 { - [ - Self::Vector, - Self::IvfFlat, - Self::IvfSq, - Self::IvfPq, - Self::IvfHnswSq, - Self::IvfHnswPq, - Self::IvfHnswFlat, - Self::IvfRq, - ] - .into_iter() - .map(|index_type| index_type.version() as u32) - .max() - .unwrap_or(VECTOR_INDEX_VERSION) - } -} - -pub trait IndexParams: Send + Sync { - fn as_any(&self) -> &dyn Any; - - fn index_name(&self) -> &str; -} - #[derive(Serialize, Deserialize, Debug)] pub struct IndexMetadata { #[serde(rename = "type")] @@ -356,9 +79,7 @@ pub struct IndexMetadata { pub distance_type: String, } -pub fn is_system_index(index_meta: &lance_table::format::IndexMetadata) -> bool { - index_meta.name == FRAG_REUSE_INDEX_NAME || index_meta.name == MEM_WAL_INDEX_NAME -} +pub use lance_table::system_index::is_system_index; pub fn infer_system_index_type( index_meta: &lance_table::format::IndexMetadata, diff --git a/rust/lance-index/src/mem_wal.rs b/rust/lance-index/src/mem_wal.rs index 9bd72ff7866..879f4757919 100644 --- a/rust/lance-index/src/mem_wal.rs +++ b/rust/lance-index/src/mem_wal.rs @@ -5,13 +5,14 @@ //! //! The data structures and table-format logic live in //! [`lance_table::system_index::mem_wal`]; this module re-exports them and -//! implements the local [`Index`] trait for [`MemWalIndex`]. +//! provides a newtype wrapper that implements the [`Index`] trait. use std::any::Any; use std::sync::Arc; use async_trait::async_trait; -use lance_core::Error; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; use roaring::RoaringBitmap; use serde::Serialize; @@ -19,17 +20,28 @@ pub use lance_table::system_index::mem_wal::*; use crate::{Index, IndexType}; +/// Newtype wrapping [`MemWalIndex`] so that `lance-index` can implement +/// the `Index` trait (orphan rules prevent implementing it directly in +/// `lance-table`). +pub struct MemWalIndexHandle(pub Arc); + +impl DeepSizeOf for MemWalIndexHandle { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.0.deep_size_of_children(context) + } +} + #[derive(Serialize)] struct MemWalStatistics { num_shards: u32, - num_merged_generations: usize, + num_compacted_sstables: usize, num_shard_specs: usize, num_maintained_indexes: usize, num_index_catchup_entries: usize, } #[async_trait] -impl Index for MemWalIndex { +impl Index for MemWalIndexHandle { fn as_any(&self) -> &dyn Any { self } @@ -38,23 +50,23 @@ impl Index for MemWalIndex { self } - fn statistics(&self) -> lance_core::Result { + fn statistics(&self) -> Result { let stats = MemWalStatistics { - num_shards: self.details.num_shards, - num_merged_generations: self.details.merged_generations.len(), - num_shard_specs: self.details.sharding_specs.len(), - num_maintained_indexes: self.details.maintained_indexes.len(), - num_index_catchup_entries: self.details.index_catchup.len(), + num_shards: self.0.details.num_shards, + num_compacted_sstables: self.0.details.compacted_sstables.len(), + num_shard_specs: self.0.details.sharding_specs.len(), + num_maintained_indexes: self.0.details.maintained_indexes.len(), + num_index_catchup_entries: self.0.details.index_catchup.len(), }; serde_json::to_value(stats).map_err(|e| { - Error::internal(format!( + lance_core::Error::internal(format!( "failed to serialize MemWAL index statistics: {}", e )) }) } - async fn prewarm(&self) -> lance_core::Result<()> { + async fn prewarm(&self) -> Result<()> { Ok(()) } @@ -62,7 +74,7 @@ impl Index for MemWalIndex { IndexType::MemWal } - async fn calculate_included_frags(&self) -> lance_core::Result { + async fn calculate_included_frags(&self) -> Result { Ok(RoaringBitmap::new()) } } diff --git a/rust/lance-index/src/metrics.rs b/rust/lance-index/src/metrics.rs index 8c0c119a3c3..7d1ea2964c6 100644 --- a/rust/lance-index/src/metrics.rs +++ b/rust/lance-index/src/metrics.rs @@ -1,117 +1,4 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::atomic::{AtomicUsize, Ordering}; - -pub const AND_CANDIDATES_SEEN_METRIC: &str = "and_candidates_seen"; -pub const AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC: &str = "and_candidates_pruned_before_return"; -pub const AND_FULL_SCORES_METRIC: &str = "and_full_scores"; -pub const FREQS_COLLECTED_METRIC: &str = "freqs_collected"; - -/// A trait used by the index to report metrics -/// -/// Callers can implement this trait to collect metrics -pub trait MetricsCollector: Send + Sync { - /// Record partition loads - /// - /// Many indices consist of partitions that may need to be loaded - /// into cache. For example, an inverted index or ngram index has a - /// posting list for each token. - /// - /// In the ideal case, these shards are in the cache and will not need - /// to be loaded from disk. This method should not be called if the - /// shard is in the cache. - fn record_parts_loaded(&self, num_parts: usize); - - /// Record a shard load - fn record_part_load(&self) { - self.record_parts_loaded(1); - } - - /// Record an index load - /// - /// This should be called when a scalar index is loaded from storage. - /// It should not be called if the index is already in memory. - fn record_index_loads(&self, num_indexes: usize); - - /// Record an index load - fn record_index_load(&self) { - self.record_index_loads(1); - } - - /// Record the number of "comparisons" made by the index - /// - /// What exactly constitutes a comparison depends on the index type. - /// For example, a B-tree index may make comparisons while searching for a value. - /// On the other hand, a bitmap index makes comparisons when computing the intersection - /// of two bitmaps. - /// - /// The goal is to provide some visibility into the compute cost of the search - fn record_comparisons(&self, num_comparisons: usize); - - /// Record AND candidates returned from WAND alignment to the scoring loop. - /// - /// This excludes candidates pruned before `next()` returns. Use this with - /// `record_and_candidates_pruned_before_return` to recover total aligned - /// AND candidates. - fn record_and_candidates_seen(&self, _num_candidates: usize) {} - - /// Record AND candidates pruned during WAND alignment before `next()` returns. - fn record_and_candidates_pruned_before_return(&self, _num_candidates: usize) {} - - fn record_and_full_scores(&self, _num_scores: usize) {} - - fn record_freqs_collected(&self, _num_collections: usize) {} - - /// Returns an optional sink for recording exact I/O statistics (bytes read, - /// IOPS, and requests) performed on behalf of this collector. - /// - /// Index implementations that read from a - /// [`lance_io::scheduler::ScanScheduler`] can attach the returned handle to - /// their file readers so the I/O performed for a single query is measured - /// and attributed here. The default returns `None`, meaning the caller does - /// not want I/O measured (and index implementations should then take their - /// normal, uninstrumented read path). - fn io_stats(&self) -> Option { - None - } -} - -/// A no-op metrics collector that does nothing -pub struct NoOpMetricsCollector; - -impl MetricsCollector for NoOpMetricsCollector { - fn record_parts_loaded(&self, _num_parts: usize) {} - fn record_index_loads(&self, _num_indexes: usize) {} - fn record_comparisons(&self, _num_comparisons: usize) {} -} - -#[derive(Default)] -pub struct LocalMetricsCollector { - pub parts_loaded: AtomicUsize, - pub index_loads: AtomicUsize, - pub comparisons: AtomicUsize, -} - -impl LocalMetricsCollector { - pub fn dump_into(self, other: &dyn MetricsCollector) { - other.record_parts_loaded(self.parts_loaded.load(Ordering::Relaxed)); - other.record_index_loads(self.index_loads.load(Ordering::Relaxed)); - other.record_comparisons(self.comparisons.load(Ordering::Relaxed)); - } -} - -impl MetricsCollector for LocalMetricsCollector { - fn record_parts_loaded(&self, num_parts: usize) { - self.parts_loaded.fetch_add(num_parts, Ordering::Relaxed); - } - - fn record_index_loads(&self, num_indexes: usize) { - self.index_loads.fetch_add(num_indexes, Ordering::Relaxed); - } - - fn record_comparisons(&self, num_comparisons: usize) { - self.comparisons - .fetch_add(num_comparisons, Ordering::Relaxed); - } -} +pub use lance_index_core::metrics::*; diff --git a/rust/lance-index/src/prefilter.rs b/rust/lance-index/src/prefilter.rs index 6671bac0a81..0fb131a2381 100644 --- a/rust/lance-index/src/prefilter.rs +++ b/rust/lance-index/src/prefilter.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use async_trait::async_trait; use lance_core::Result; -use lance_select::RowAddrMask; +use lance_select::{RowAddrMask, RowAddrTreeMap}; /// A trait to be implemented by anything supplying a prefilter row addr mask /// @@ -36,6 +36,23 @@ pub trait PreFilter: Send + Sync { /// If the filter is empty. fn is_empty(&self) -> bool; + /// Whether partition-local row coverage is needed to determine if this filter + /// can be replaced by [`NoFilter`]. + /// + /// Most filters cannot make this proof and must not make an IVF search + /// enumerate a partition's row addresses just to call [`Self::is_empty_for`]. + fn needs_partition_row_ids(&self) -> bool { + false + } + + /// Whether this filter selects every row in a known partition. + /// + /// Callers must first call [`Self::wait_for_ready`]. Implementations that + /// cannot prove partition-local emptiness fall back to the global answer. + fn is_empty_for(&self, _rows: &RowAddrTreeMap) -> bool { + self.is_empty() + } + /// Get the row addr mask for this prefilter /// /// This method must be called after `wait_for_ready` diff --git a/rust/lance-index/src/registry.rs b/rust/lance-index/src/registry.rs index bd7448240fe..a0f3cd96df2 100644 --- a/rust/lance-index/src/registry.rs +++ b/rust/lance-index/src/registry.rs @@ -1,6 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use lance_core::{Error, Result}; @@ -16,6 +19,25 @@ use crate::{ }, }; +/// Scalar detail package emitted by Lance 0.36 before the messages moved back +/// to `lance.table` for forward compatibility. +const V036_SCALAR_DETAILS_PACKAGE: &str = "lance.index.pb"; + +/// Derive the scalar index plugin name from a details type URL. +/// +/// Takes the last `.`-separated segment, lowercases it, and strips any trailing +/// `"indexdetails"` suffix so the result matches the plugin name used in +/// [`IndexPluginRegistry`]. For example, `/lance.index.pb.ZoneMapIndexDetails` +/// yields `"zonemap"`. +pub fn plugin_name_from_details_url(type_url: &str) -> String { + let segment = type_url.split('.').next_back().unwrap_or(type_url); + let lower = segment.to_lowercase(); + lower + .strip_suffix("indexdetails") + .map(|s| s.to_string()) + .unwrap_or(lower) +} + /// Derive a human-readable index type name from a details type URL. /// /// The display name is the final `.`-separated segment of the type URL with any @@ -34,6 +56,7 @@ pub fn display_type_from_url(type_url: &str) -> &str { /// A registry of index plugins pub struct IndexPluginRegistry { plugins: HashMap>, + details_type_names: HashSet, } impl IndexPluginRegistry { @@ -42,12 +65,7 @@ impl IndexPluginRegistry { } fn get_plugin_name_from_details_name(&self, details_name: &str) -> String { - let details_name = Self::normalize_plugin_name(details_name); - if details_name.ends_with("indexdetails") { - details_name.replace("indexdetails", "") - } else { - details_name - } + plugin_name_from_details_url(details_name) } /// Adds a plugin to the registry, using the name of the details message to determine @@ -65,14 +83,22 @@ impl IndexPluginRegistry { &mut self, ) { let plugin_name = self.get_plugin_name_from_details_name(DetailsType::NAME); + self.details_type_names + .insert(DetailsType::full_name().to_ascii_lowercase()); self.plugins .insert(plugin_name, Box::new(PluginType::default())); } + fn add_details_type_alias(&mut self, package: &str) { + self.details_type_names + .insert(format!("{}.{}", package, DetailsType::NAME).to_ascii_lowercase()); + } + /// Create a registry with the default plugins pub fn with_default_plugins() -> Arc { let mut registry = Self { plugins: HashMap::new(), + details_type_names: HashSet::new(), }; registry.add_plugin::(); registry.add_plugin::(); @@ -86,6 +112,17 @@ impl IndexPluginRegistry { #[cfg(feature = "geo")] registry.add_plugin::(); + // Lance 0.36 released these scalar detail messages in the index package. + // Register only those historical identities, not arbitrary packages + // carrying the same terminal message names. + registry.add_details_type_alias::(V036_SCALAR_DETAILS_PACKAGE); + registry.add_details_type_alias::(V036_SCALAR_DETAILS_PACKAGE); + registry + .add_details_type_alias::(V036_SCALAR_DETAILS_PACKAGE); + registry.add_details_type_alias::(V036_SCALAR_DETAILS_PACKAGE); + registry.add_details_type_alias::(V036_SCALAR_DETAILS_PACKAGE); + registry.add_details_type_alias::(V036_SCALAR_DETAILS_PACKAGE); + let registry = Arc::new(registry); for plugin in registry.plugins.values() { plugin.attach_registry(registry.clone()); @@ -94,6 +131,23 @@ impl IndexPluginRegistry { registry } + /// Returns whether the complete protobuf type name in `details` belongs to + /// a registered scalar index reader. + /// + /// Type URL authorities may vary, so matching uses the fully qualified + /// message name after the final slash. The table format requires index type + /// URL comparisons to be case-insensitive. + pub fn supports_details(&self, details: &prost_types::Any) -> bool { + let Some((_, details_type_name)) = details.type_url.rsplit_once('/') else { + return false; + }; + if details_type_name.is_empty() || details_type_name.starts_with('.') { + return false; + } + self.details_type_names + .contains(&details_type_name.to_ascii_lowercase()) + } + /// Get an index plugin suitable for training an index with the given parameters pub fn get_plugin_by_name(&self, name: &str) -> Result<&dyn ScalarIndexPlugin> { let plugin_name = Self::normalize_plugin_name(name); @@ -163,4 +217,36 @@ mod tests { assert_eq!(plugin.name(), expected_name); } } + + #[test] + fn test_supports_details_matches_complete_type_name_case_insensitively() { + let registry = IndexPluginRegistry::with_default_plugins(); + + for type_url in [ + "/lance.table.BTreeIndexDetails", + "type.googleapis.com/LANCE.TABLE.BTREEINDEXDETAILS", + "/lance.index.pb.BTreeIndexDetails", + "/lance.index.pb.BitmapIndexDetails", + "/lance.index.pb.LabelListIndexDetails", + "/lance.index.pb.NGramIndexDetails", + "/lance.index.pb.ZoneMapIndexDetails", + "/lance.index.pb.InvertedIndexDetails", + ] { + assert!(registry.supports_details(&prost_types::Any { + type_url: type_url.to_string(), + value: Vec::new(), + })); + } + + for type_url in [ + "type.googleapis.com/example.BTreeIndexDetails", + "BTreeIndexDetails", + "/.lance.table.BTreeIndexDetails", + ] { + assert!(!registry.supports_details(&prost_types::Any { + type_url: type_url.to_string(), + value: Vec::new(), + })); + } + } } diff --git a/rust/lance-index/src/scalar.rs b/rust/lance-index/src/scalar.rs index f7b158ed558..6dcfc9018e0 100644 --- a/rust/lance-index/src/scalar.rs +++ b/rust/lance-index/src/scalar.rs @@ -4,19 +4,13 @@ //! Scalar indices for metadata search & filtering use arrow::buffer::{OffsetBuffer, ScalarBuffer}; -use arrow_array::{BooleanArray, ListArray, RecordBatch, UInt64Array}; -use arrow_schema::{Field, Schema}; -use async_trait::async_trait; -use bytes::Bytes; +use arrow_array::ListArray; +use arrow_schema::Field; use datafusion::functions::regex::regexplike::RegexpLikeFunc; use datafusion::functions::string::contains::ContainsFunc; use datafusion::functions_nested::array_has; -use datafusion::physical_plan::SendableRecordBatchStream; use datafusion_common::{Column, scalar::ScalarValue}; -use lance_core::utils::row_addr_remap::RowAddrRemap; -use std::collections::{HashMap, HashSet}; -use std::fmt::Debug; -use std::pin::Pin; +use std::collections::HashSet; use std::{any::Any, ops::Bound, sync::Arc}; use datafusion_expr::{ @@ -24,17 +18,17 @@ use datafusion_expr::{ expr::{Like, ScalarFunction}, }; use inverted::query::{FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, fill_fts_query_column}; -use lance_core::deepsize::DeepSizeOf; -use lance_core::{Error, Result}; -use lance_io::stream::{RecordBatchStream, RecordBatchStreamAdapter}; -use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps}; -use roaring::{RoaringBitmap, RoaringTreemap}; -use serde::Serialize; - -use crate::metrics::MetricsCollector; -use crate::scalar::registry::TrainingCriteria; -use crate::{Index, IndexParams, IndexType}; -pub use lance_table::format::IndexFile; +use lance_core::Result; + +use lance_datafusion::udf::CONTAINS_TOKENS_UDF; + +use crate::IndexParams; +pub use crate::metrics::MetricsCollector; +pub use lance_index_core::scalar::{ + AnyQuery, BuiltinIndexType, CreatedIndex, IndexFile, IndexReader, IndexStore, IndexWriter, + LANCE_SCALAR_INDEX, OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, + SearchOptions, SearchResult, TrainingCriteria, TrainingOrdering, UpdateCriteria, +}; pub mod bitmap; pub mod bloomfilter; @@ -49,121 +43,44 @@ pub mod ngram; pub mod registry; #[cfg(feature = "geo")] pub mod rtree; +pub mod seed; pub mod zoned; pub mod zonemap; pub use inverted::tokenizer::InvertedIndexParams; -use lance_datafusion::udf::CONTAINS_TOKENS_UDF; -pub const LANCE_SCALAR_INDEX: &str = "__lance_scalar_index"; - -/// Builtin index types supported by the Lance library +/// Convert a `Vec<`[`lance_index_core::scalar::IndexFile`]`>` to a +/// `Vec<`[`lance_table::format::IndexFile`]`>`. /// -/// This is primarily for convenience to avoid a bunch of string -/// constants and provide some auto-complete. This type should not -/// be used in the manifest as plugins cannot add new entries. -#[derive(Debug, Clone, PartialEq, Eq, DeepSizeOf)] -pub enum BuiltinIndexType { - BTree, - Bitmap, - LabelList, - NGram, - ZoneMap, - BloomFilter, - RTree, - Inverted, - Fm, -} - -impl BuiltinIndexType { - pub fn as_str(&self) -> &str { - match self { - Self::BTree => "btree", - Self::Bitmap => "bitmap", - Self::LabelList => "labellist", - Self::NGram => "ngram", - Self::ZoneMap => "zonemap", - Self::Inverted => "inverted", - Self::BloomFilter => "bloomfilter", - Self::RTree => "rtree", - Self::Fm => "fm", - } - } -} - -impl TryFrom for BuiltinIndexType { - type Error = Error; - - fn try_from(value: IndexType) -> Result { - match value { - IndexType::BTree => Ok(Self::BTree), - IndexType::Bitmap => Ok(Self::Bitmap), - IndexType::LabelList => Ok(Self::LabelList), - IndexType::NGram => Ok(Self::NGram), - IndexType::ZoneMap => Ok(Self::ZoneMap), - IndexType::Inverted => Ok(Self::Inverted), - IndexType::BloomFilter => Ok(Self::BloomFilter), - IndexType::RTree => Ok(Self::RTree), - IndexType::Fm => Ok(Self::Fm), - _ => Err(Error::index("Invalid index type".to_string())), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ScalarIndexParams { - /// The type of index to create - /// - /// Plugins may add additional index types. Index type lookup is case-insensitive. - pub index_type: String, - /// The parameters to train the index - /// - /// This should be a JSON string. The contents of the JSON string will be specific to the - /// index type. If not set, then default parameters will be used for the index type. - pub params: Option, -} - -impl Default for ScalarIndexParams { - fn default() -> Self { - Self { - index_type: BuiltinIndexType::BTree.as_str().to_string(), - params: None, - } - } -} - -impl ScalarIndexParams { - /// Creates a new ScalarIndexParams from one of the builtin index types - pub fn for_builtin(index_type: BuiltinIndexType) -> Self { - Self { - index_type: index_type.as_str().to_string(), - params: None, - } - } - - /// Create a new ScalarIndexParams with the given index type - pub fn new(index_type: String) -> Self { - Self { - index_type, - params: None, - } - } - - /// Set the parameters for the index - pub fn with_params(mut self, params: &ParamsType) -> Self { - self.params = Some(serde_json::to_string(params).unwrap()); - self - } -} - -impl IndexParams for ScalarIndexParams { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn index_name(&self) -> &str { - LANCE_SCALAR_INDEX - } +/// These two structs have identical fields; this helper bridges the crate +/// boundary without relying on orphan-rule–violating `From` impls. +pub fn index_files_to_table( + files: Vec, +) -> Vec { + files + .into_iter() + .map(|f| lance_table::format::IndexFile { + path: f.path, + size_bytes: f.size_bytes, + }) + .collect() +} + +/// Convert a `Vec<`[`lance_table::format::IndexFile`]`>` to a +/// `Vec<`[`lance_index_core::scalar::IndexFile`]`>`. +/// +/// These two structs have identical fields; this helper bridges the crate +/// boundary without relying on orphan-rule–violating `From` impls. +pub fn table_files_to_index( + files: Vec, +) -> Vec { + files + .into_iter() + .map(|f| lance_index_core::scalar::IndexFile { + path: f.path, + size_bytes: f.size_bytes, + }) + .collect() } impl IndexParams for InvertedIndexParams { @@ -176,180 +93,6 @@ impl IndexParams for InvertedIndexParams { } } -/// Trait for storing an index (or parts of an index) into storage -#[async_trait] -pub trait IndexWriter: Send { - /// Writes a record batch into the file, returning the 0-based index of the batch in the file - /// - /// E.g. if this is the third time this is called this method will return 2 - async fn write_record_batch(&mut self, batch: RecordBatch) -> Result; - /// Adds a global buffer and returns its index. - async fn add_global_buffer(&mut self, _data: Bytes) -> Result { - Err(Error::not_supported( - "global buffers are not supported by this index writer", - )) - } - /// Finishes writing the file and closes the file - async fn finish(&mut self) -> Result; - /// Finishes writing the file and closes the file with additional metadata - async fn finish_with_metadata( - &mut self, - metadata: HashMap, - ) -> Result; -} - -/// Trait for reading an index (or parts of an index) from storage -#[async_trait] -pub trait IndexReader: Send + Sync { - /// Read the n-th record batch from the file - async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result; - /// Reads a global buffer by index. - async fn read_global_buffer(&self, _index: u32) -> Result { - Err(Error::not_supported( - "global buffers are not supported by this index reader", - )) - } - /// Read the range of rows from the file. - /// If projection is Some, only return the columns in the projection, - /// nested columns like Some(&["x.y"]) are not supported. - /// If projection is None, return all columns. - async fn read_range( - &self, - range: std::ops::Range, - projection: Option<&[&str]>, - ) -> Result; - /// Read multiple ranges and concatenate into a single batch. - /// Default impl runs `read_range`s in parallel via `try_join_all`. - async fn read_ranges( - &self, - ranges: &[std::ops::Range], - projection: Option<&[&str]>, - ) -> Result { - if ranges.is_empty() { - return self.read_range(0..0, projection).await; - } - let futures = ranges - .iter() - .map(|r| self.read_range(r.clone(), projection)); - let batches = futures::future::try_join_all(futures).await?; - let schema = batches[0].schema(); - Ok(arrow_select::concat::concat_batches(&schema, &batches)?) - } - /// Read a range of rows as a stream of record batches. - /// - /// This allows the caller to process rows incrementally without loading the - /// entire range into memory at once. - /// - /// The default implementation falls back to [`Self::read_range`] and wraps - /// the result in a single-item stream. - async fn read_range_stream( - &self, - range: std::ops::Range, - projection: Option<&[&str]>, - ) -> Result>> { - let batch = self.read_range(range, projection).await?; - let schema = batch.schema(); - Ok(Box::pin(RecordBatchStreamAdapter::new( - schema, - futures::stream::once(async move { Ok(batch) }), - ))) - } - /// Return the number of batches in the file - async fn num_batches(&self, batch_size: u64) -> u32; - /// Return the number of rows in the file - fn num_rows(&self) -> usize; - /// Return the metadata of the file - fn schema(&self) -> &lance_core::datatypes::Schema; - /// Best-effort on-disk byte size of the file when the reader already knows it - /// without extra I/O, else `None`. Used to size prewarm chunks. - fn file_size_bytes(&self) -> Option { - None - } -} - -/// Trait abstracting I/O away from index logic -/// -/// Scalar indices are currently serialized as indexable arrow record batches stored in -/// named "files". The index store is responsible for serializing and deserializing -/// these batches into file data (e.g. as .lance files or .parquet files, etc.) -#[async_trait] -pub trait IndexStore: std::fmt::Debug + Send + Sync + DeepSizeOf { - fn as_any(&self) -> &dyn Any; - fn clone_arc(&self) -> Arc; - - /// Suggested I/O parallelism for the store - fn io_parallelism(&self) -> usize; - - /// Create a new file and return a writer to store data in the file - async fn new_index_file(&self, name: &str, schema: Arc) - -> Result>; - - /// Open an existing file for retrieval - async fn open_index_file(&self, name: &str) -> Result>; - - /// Return a store that submits its I/O at the given base priority. - fn with_io_priority(&self, io_priority: u64) -> Arc; - - /// Copy a range of batches from an index file from this store to another - /// - /// This is often useful when remapping or updating - async fn copy_index_file(&self, name: &str, dest_store: &dyn IndexStore) -> Result; - - /// Copy an index file from this store to a new name in another store, leaving the source intact - async fn copy_index_file_to( - &self, - name: &str, - new_name: &str, - dest_store: &dyn IndexStore, - ) -> Result { - if name == new_name { - self.copy_index_file(name, dest_store).await - } else { - Err(Error::not_supported(format!( - "copying index file {name} to {new_name} is not supported by this index store" - ))) - } - } - - /// Rename an index file - async fn rename_index_file(&self, name: &str, new_name: &str) -> Result; - - /// Delete an index file (used in the tmp spill store to keep tmp size down) - async fn delete_index_file(&self, name: &str) -> Result<()>; - - /// List all files in the index directory with their sizes. - /// - /// Returns a list of (relative_path, size_bytes) tuples. - /// Used to capture file metadata after index creation/modification. - async fn list_files_with_sizes(&self) -> Result>; -} - -/// Different scalar indices may support different kinds of queries -/// -/// For example, a btree index can support a wide range of queries (e.g. x > 7) -/// while an index based on FTS only supports queries like "x LIKE 'foo'" -/// -/// This trait is used when we need an object that can represent any kind of query -/// -/// Note: if you are implementing this trait for a query type then you probably also -/// need to implement the [crate::scalar::expression::ScalarQueryParser] trait to -/// create instances of your query at parse time. -pub trait AnyQuery: std::fmt::Debug + Any + Send + Sync { - /// Cast the query as Any to allow for downcasting - fn as_any(&self) -> &dyn Any; - /// Format the query as a string for display purposes - fn format(&self, col: &str) -> String; - /// Convert the query to a datafusion expression - fn to_expr(&self, col: String) -> Expr; - /// Compare this query to another query - fn dyn_eq(&self, other: &dyn AnyQuery) -> bool; -} - -impl PartialEq for dyn AnyQuery { - fn eq(&self, other: &Self) -> bool { - self.dyn_eq(other) - } -} /// A full text search query #[derive(Debug, Clone, PartialEq)] pub struct FullTextSearchQuery { @@ -892,149 +635,6 @@ impl AnyQuery for GeoQuery { } } -/// The result of a search operation against a scalar index -#[derive(Debug, PartialEq)] -pub enum SearchResult { - /// The exact row ids that satisfy the query - Exact(NullableRowAddrSet), - /// Any row id satisfying the query will be in this set but not every - /// row id in this set will satisfy the query, a further recheck step - /// is needed - AtMost(NullableRowAddrSet), - /// All of the given row ids satisfy the query but there may be more - /// - /// No scalar index actually returns this today but it can arise from - /// boolean operations (e.g. NOT(AtMost(x)) == AtLeast(NOT(x))) - AtLeast(NullableRowAddrSet), -} - -impl SearchResult { - pub fn exact(row_ids: impl Into) -> Self { - Self::Exact(NullableRowAddrSet::new(row_ids.into(), Default::default())) - } - - pub fn at_most(row_ids: impl Into) -> Self { - Self::AtMost(NullableRowAddrSet::new(row_ids.into(), Default::default())) - } - - pub fn at_least(row_ids: impl Into) -> Self { - Self::AtLeast(NullableRowAddrSet::new(row_ids.into(), Default::default())) - } - - pub fn with_nulls(self, nulls: impl Into) -> Self { - match self { - Self::Exact(row_ids) => Self::Exact(row_ids.with_nulls(nulls.into())), - Self::AtMost(row_ids) => Self::AtMost(row_ids.with_nulls(nulls.into())), - Self::AtLeast(row_ids) => Self::AtLeast(row_ids.with_nulls(nulls.into())), - } - } - - pub fn row_addrs(&self) -> &NullableRowAddrSet { - match self { - Self::Exact(row_addrs) => row_addrs, - Self::AtMost(row_addrs) => row_addrs, - Self::AtLeast(row_addrs) => row_addrs, - } - } - - pub fn is_exact(&self) -> bool { - matches!(self, Self::Exact(_)) - } -} - -/// Brief information about an index that was created -pub struct CreatedIndex { - /// The details of the index that was created - /// - /// These should be stored somewhere as they will be needed to - /// load the index later. - pub index_details: prost_types::Any, - /// The version of the index that was created - /// - /// This can be used to determine if a reader is able to load the index. - pub index_version: u32, - /// List of files and their sizes for this index - /// - /// This enables skipping HEAD calls when opening indices and provides - /// visibility into index storage size via describe_indices(). - pub files: Vec, -} - -/// The criteria that specifies how to update an index -pub struct UpdateCriteria { - /// If true, then we need to read the old data to update the index - /// - /// This should be avoided if possible but is left in for some legacy paths - pub requires_old_data: bool, - /// The criteria required for data (both old and new) - pub data_criteria: TrainingCriteria, -} - -/// Filter used when merging existing scalar-index rows during update. -/// -/// The caller must pick a filter mode that matches the row-id semantics of the -/// dataset: -/// - address-style row IDs: fragment filtering is valid -/// - stable row IDs: use exact row-id membership instead -#[derive(Debug, Clone)] -pub enum OldIndexDataFilter { - /// Keeps track of which fragments are still valid and which are no longer valid. - /// - /// This is valid for address-style row IDs. - Fragments { - to_keep: RoaringBitmap, - to_remove: RoaringBitmap, - }, - /// Keep old rows whose row IDs are in this exact allow-list. - /// - /// This is required for stable row IDs, where row IDs are opaque and - /// should not be interpreted as encoded row addresses. - RowIds(RowAddrTreeMap), -} - -impl OldIndexDataFilter { - /// Build a boolean mask that keeps only row IDs selected by this filter. - pub fn filter_row_ids(&self, row_ids: &UInt64Array) -> BooleanArray { - match self { - Self::Fragments { to_keep, .. } => row_ids - .iter() - .map(|id| id.map(|id| to_keep.contains((id >> 32) as u32))) - .collect(), - Self::RowIds(valid_row_ids) => row_ids - .iter() - .map(|id| id.map(|id| valid_row_ids.contains(id))) - .collect(), - } - } - - /// Apply this filter in place to a set of existing (old) row ids/addresses, - /// retaining only the rows the filter selects to keep. Used by index types - /// that merge old postings directly (e.g. bitmap) instead of re-scanning a - /// row-id array through [`Self::filter_row_ids`]. - pub fn retain_old_rows(&self, rows: &mut RowAddrTreeMap) { - match self { - Self::Fragments { to_keep, .. } => rows.retain_fragments(to_keep.iter()), - Self::RowIds(valid_row_ids) => *rows &= valid_row_ids, - } - } -} - -impl UpdateCriteria { - pub fn requires_old_data(data_criteria: TrainingCriteria) -> Self { - Self { - requires_old_data: true, - data_criteria, - } - } - - pub fn only_new_data(data_criteria: TrainingCriteria) -> Self { - Self { - requires_old_data: false, - data_criteria, - } - } -} - /// Compute the lexicographically next prefix by incrementing the last character's code point. /// Returns None if no valid upper bound exists. /// @@ -1091,77 +691,6 @@ fn next_unicode_char(c: char) -> Option { char::from_u32(next_cp) } -/// A trait for a scalar index, a structure that can determine row ids that satisfy scalar queries -#[async_trait] -pub trait ScalarIndex: Send + Sync + std::fmt::Debug + Index + DeepSizeOf { - /// Search the scalar index - /// - /// Returns all row ids that satisfy the query, these row ids are not necessarily ordered - async fn search( - &self, - query: &dyn AnyQuery, - metrics: &dyn MetricsCollector, - ) -> Result; - - /// Returns true if the remap operation is supported - fn can_remap(&self) -> bool; - - /// Remap the row ids, creating a new remapped version of this index in `dest_store` - async fn remap( - &self, - mapping: &RowAddrRemap, - dest_store: &dyn IndexStore, - ) -> Result; - - /// Add the new data into the index, creating an updated version of the index in `dest_store` - /// - /// If `old_data_filter` is provided, old index data will be filtered before - /// merge according to the chosen filter mode. - async fn update( - &self, - new_data: SendableRecordBatchStream, - dest_store: &dyn IndexStore, - old_data_filter: Option, - ) -> Result; - - /// Returns the criteria that will be used to update the index - fn update_criteria(&self) -> UpdateCriteria; - - /// Derive the index parameters from the current index - /// - /// This returns a ScalarIndexParams that can be used to recreate an index - /// with the same configuration on another dataset. - fn derive_index_params(&self) -> Result; - - /// Global `[min, max]` of the indexed column from index metadata, without a - /// scan, or `None` if this index type cannot supply a sound bound. When - /// `Some`, the range is a superset of live values (conservative under - /// deletes): safe to prune with, not guaranteed tight. - fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { - None - } -} - -/// Abstraction over any type that can remap row IDs during index loading. -/// -/// This decouples scalar index plugins from the table-level [`crate::frag_reuse::FragReuseIndex`] -/// type. [`crate::frag_reuse::FragReuseIndex`] implements this trait, but callers may also -/// supply custom implementations for testing or other remapping strategies. -pub trait RowIdRemapper: Send + Sync + std::fmt::Debug { - /// Remap a single row id. Returns `None` if the row was deleted. - fn remap_row_id(&self, row_id: u64) -> Option; - /// Remap all addresses in a [`RowAddrTreeMap`], dropping deleted rows. - fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap; - /// Remap all row ids in a [`RoaringTreemap`], dropping deleted rows. - fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap; - /// Remap the row-id column at `row_id_idx` inside `batch`, dropping deleted rows. - fn remap_row_ids_record_batch( - &self, - batch: RecordBatch, - row_id_idx: usize, - ) -> Result; -} - #[cfg(test)] mod tests { use super::*; diff --git a/rust/lance-index/src/scalar/bitmap.rs b/rust/lance-index/src/scalar/bitmap.rs index 98a07ff827d..8b92b9eddca 100644 --- a/rust/lance-index/src/scalar/bitmap.rs +++ b/rust/lance-index/src/scalar/bitmap.rs @@ -23,8 +23,8 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::{ Error, ROW_ID, Result, cache::{ - CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache, - WeakLanceCache, + CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, CacheKeySchema, + KeyBuilder, LanceCache, WeakLanceCache, }, error::LanceOptionExt, utils::tokio::get_num_compute_intensive_cpus, @@ -36,7 +36,7 @@ use roaring::RoaringBitmap; use serde::{Deserialize, Serialize}; use tracing::{instrument, warn}; -use super::{AnyQuery, IndexFile, IndexStore, ScalarIndex}; +use super::{AnyQuery, IndexFile, IndexStore, ScalarIndex, SearchOptions}; use super::{ BuiltinIndexType, SargableQuery, ScalarIndexParams, SearchResult, btree::OrderableScalarValue, }; @@ -132,20 +132,39 @@ pub struct BitmapIndex { #[derive(Debug, Clone)] pub struct BitmapKey { - value: OrderableScalarValue, + row_offset: u64, +} + +impl BitmapKey { + fn try_new(row_offset: usize) -> Result { + let row_offset = u64::try_from(row_offset).map_err(|_| { + Error::internal(format!( + "bitmap row offset {row_offset} does not fit in u64" + )) + })?; + Ok(Self { row_offset }) + } } impl CacheKey for BitmapKey { type ValueType = RowAddrTreeMap; fn key(&self) -> std::borrow::Cow<'_, str> { - format!("{}", self.value.0).into() + self.row_offset.to_string().into() } fn type_name() -> &'static str { "Bitmap" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.bitmap-row-offset-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.row_offset); + } + fn codec() -> Option { Some(CacheCodec::from_impl::()) } @@ -335,6 +354,14 @@ impl CacheKey for BitmapIndexStateKey { "BitmapIndexState" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.bitmap-index-state-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_variant(0); + } + fn codec() -> Option { Some(CacheCodec::from_impl::()) } @@ -446,22 +473,30 @@ impl BitmapIndex { return Ok(self.null_map.clone()); } - let cache_key = BitmapKey { value: key.clone() }; + // A value that isn't in `index_map` never reaches the loader or the + // cache, so it should not touch the per-query cache counters either. + // Checking here (before the cached-lookup fast path) also avoids + // returning an unmapped-value response as a spurious cache hit if a + // prior insert somehow ended up under `cache_key`. + let row_offset = match self.index_map.get(key) { + Some(loc) => *loc, + None => return Ok(Arc::new(RowAddrTreeMap::default())), + }; + let cache_key = BitmapKey::try_new(row_offset)?; if let Some(cached) = self.index_cache.get_with_key(&cache_key).await { + if let Some(metrics) = metrics { + metrics.record_index_cache_hit(); + } return Ok(cached); } // Record that we're loading a partition from disk if let Some(metrics) = metrics { + metrics.record_index_cache_miss(); metrics.record_part_load(); } - let row_offset = match self.index_map.get(key) { - Some(loc) => *loc, - None => return Ok(Arc::new(RowAddrTreeMap::default())), - }; - let page_lookup_file = self.lazy_reader.get().await?; let batch = page_lookup_file .read_range(row_offset..row_offset + 1, Some(&["bitmaps"])) @@ -602,7 +637,12 @@ impl Index for BitmapIndex { bitmap = frag_reuse_index_ref.remap_row_addrs_tree_map(&bitmap); } - let cache_key = BitmapKey { value: key }; + let row_offset = start_row.checked_add(idx).ok_or_else(|| { + Error::internal(format!( + "bitmap row offset overflow: start_row={start_row}, idx={idx}" + )) + })?; + let cache_key = BitmapKey::try_new(row_offset)?; self.index_cache .insert_with_key(&cache_key, Arc::new(bitmap)) .await; @@ -640,9 +680,27 @@ impl ScalarIndex for BitmapIndex { &self, query: &dyn AnyQuery, metrics: &dyn MetricsCollector, + ) -> Result { + self.search_with_options(query, SearchOptions::default(), metrics) + .await + } + + async fn search_with_options( + &self, + query: &dyn AnyQuery, + options: SearchOptions, + metrics: &dyn MetricsCollector, ) -> Result { let query = query.as_any().downcast_ref::().unwrap(); + let tracked_null_rows = || { + if options.track_nulls() && !self.null_map.is_empty() { + Some((*self.null_map).clone()) + } else { + None + } + }; + let (row_ids, null_row_ids) = match query { SargableQuery::Equals(val) => { metrics.record_comparisons(1); @@ -652,12 +710,7 @@ impl ScalarIndex for BitmapIndex { } else { let key = OrderableScalarValue(val.clone()); let bitmap = self.load_bitmap(&key, Some(metrics)).await?; - let null_rows = if !self.null_map.is_empty() { - Some((*self.null_map).clone()) - } else { - None - }; - ((*bitmap).clone(), null_rows) + ((*bitmap).clone(), tracked_null_rows()) } } SargableQuery::Range(start, end) => { @@ -698,7 +751,7 @@ impl ScalarIndex for BitmapIndex { } else { let bitmaps: Vec<_> = stream::iter( keys.into_iter() - .map(|key| async move { self.load_bitmap(&key, None).await }), + .map(|key| async move { self.load_bitmap(&key, Some(metrics)).await }), ) .buffer_unordered(get_num_compute_intensive_cpus()) .try_collect() @@ -708,12 +761,7 @@ impl ScalarIndex for BitmapIndex { RowAddrTreeMap::union_all(&bitmap_refs) }; - let null_rows = if !self.null_map.is_empty() { - Some((*self.null_map).clone()) - } else { - None - }; - (result, null_rows) + (result, tracked_null_rows()) } SargableQuery::IsIn(values) => { metrics.record_comparisons(values.len()); @@ -740,7 +788,7 @@ impl ScalarIndex for BitmapIndex { // Load bitmaps in parallel let mut bitmaps: Vec<_> = stream::iter( keys.into_iter() - .map(|key| async move { self.load_bitmap(&key, None).await }), + .map(|key| async move { self.load_bitmap(&key, Some(metrics)).await }), ) .buffer_unordered(get_num_compute_intensive_cpus()) .try_collect() @@ -761,11 +809,7 @@ impl ScalarIndex for BitmapIndex { // If the query explicitly includes null, then nulls are TRUE (not NULL) // Otherwise, nulls remain NULL (unknown) - let null_rows = if !has_null && !self.null_map.is_empty() { - Some((*self.null_map).clone()) - } else { - None - }; + let null_rows = if has_null { None } else { tracked_null_rows() }; (result, null_rows) } SargableQuery::IsNull() => { @@ -891,8 +935,7 @@ impl BitmapBatchWriter { return Ok(()); } let keys_array = - ScalarValue::iter_to_array(self.keys.drain(..).collect::>().into_iter()) - .unwrap(); + ScalarValue::iter_to_array(self.keys.drain(..).collect::>()).unwrap(); let total_size: usize = self.serialized.iter().map(|b| b.len()).sum(); let mut binary_builder = BinaryBuilder::with_capacity(self.serialized.len(), total_size); for b in self.serialized.drain(..) { @@ -1123,10 +1166,7 @@ async fn drain_same_key_bitmaps( let merged_key = OrderableScalarValue(key); advance_cursor_and_push(cursors, heap, item.shard_idx).await?; - loop { - let Some(Reverse(next_item)) = heap.peek() else { - break; - }; + while let Some(Reverse(next_item)) = heap.peek() { if next_item.key != merged_key { break; } @@ -1280,7 +1320,7 @@ impl BitmapIndexPlugin { let bitmap_size = bytes.len(); if cur_bytes + bitmap_size > MAX_BITMAP_ARRAY_LENGTH { - let keys_array = ScalarValue::iter_to_array(cur_keys.clone().into_iter()).unwrap(); + let keys_array = ScalarValue::iter_to_array(cur_keys.clone()).unwrap(); let mut binary_builder = BinaryBuilder::new(); for b in &cur_bitmaps { binary_builder.append_value(b); @@ -1882,7 +1922,7 @@ impl ScalarIndexPlugin for BitmapIndexPlugin { #[cfg(test)] mod tests { use super::*; - use crate::metrics::NoOpMetricsCollector; + use crate::metrics::{LocalMetricsCollector, NoOpMetricsCollector}; use crate::scalar::lance_format::LanceIndexStore; use arrow_array::{RecordBatch, StringArray, UInt64Array, record_batch}; use arrow_schema::{DataType, Field, Schema}; @@ -1997,6 +2037,19 @@ mod tests { } } + #[tokio::test] + async fn test_bitmap_cache_key_uses_row_offset_identity() { + let cache = LanceCache::with_capacity(1024); + let first = BitmapKey::try_new(3).unwrap(); + let second = BitmapKey::try_new(4).unwrap(); + + cache + .insert_with_key(&first, Arc::new(RowAddrTreeMap::default())) + .await; + + assert!(cache.get_with_key(&second).await.is_none()); + } + #[tokio::test] async fn test_bitmap_lazy_loading_and_cache() { // Create a temporary directory for the index @@ -2132,6 +2185,95 @@ mod tests { } } + /// Regression test for the review fix that gates `load_bitmap` on + /// `index_map.contains_key` before recording a miss: a value that is + /// not present in the index must short-circuit before touching the + /// per-query cache counters. Previously an Equals query for a missing + /// value would silently bump `index_cache_misses` and `parts_loaded` + /// on every call even though no bitmap page was actually loaded. + #[tokio::test] + async fn test_bitmap_absent_value_records_no_cache_activity() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let colors = vec!["red", "blue", "green", "yellow"]; + let row_ids = (0u64..4u64).collect::>(); + let schema = Arc::new(Schema::new(vec![ + Field::new("value", DataType::Utf8, false), + Field::new("_rowid", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(colors)), + Arc::new(UInt64Array::from(row_ids)), + ], + ) + .unwrap(); + let batch = sort_batch_by_value(&batch); + let stream = stream::once(async move { Ok(batch) }); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + BitmapIndexPlugin::train_bitmap_index(stream, store.as_ref()) + .await + .unwrap(); + + // Keep the `LanceCache` alive in test scope so the `WeakLanceCache` + // inside `BitmapIndex` can upgrade during search. + let cache = LanceCache::with_capacity(1024 * 1024); + let index = BitmapIndex::load(store.clone(), None, &cache) + .await + .unwrap(); + + // Equals on a value that is not in `index_map` must not touch + // the cache counters and must not report a part load. + let metrics = LocalMetricsCollector::default(); + let query = SargableQuery::Equals(ScalarValue::Utf8(Some("purple".to_string()))); + let result = index.search(&query, &metrics).await.unwrap(); + if let SearchResult::Exact(row_ids) = result { + assert!(row_ids.true_rows().is_empty()); + } else { + panic!("Expected exact search result"); + } + assert_eq!( + metrics.index_cache_hits(), + 0, + "absent value must not record any cache hits", + ); + assert_eq!( + metrics.index_cache_misses(), + 0, + "absent value must not record a cache miss (no loader ran)", + ); + + // IsIn covering only absent values also stays at 0/0. + let metrics = LocalMetricsCollector::default(); + let query = SargableQuery::IsIn(vec![ + ScalarValue::Utf8(Some("purple".to_string())), + ScalarValue::Utf8(Some("teal".to_string())), + ]); + let result = index.search(&query, &metrics).await.unwrap(); + if let SearchResult::Exact(row_ids) = result { + assert!(row_ids.true_rows().is_empty()); + } else { + panic!("Expected exact search result"); + } + assert_eq!(metrics.index_cache_hits(), 0); + assert_eq!(metrics.index_cache_misses(), 0); + + // Sanity: a present value on the same cold cache still records + // exactly one miss, proving the counters are wired up and the + // absent-value path above is not silently no-op. + let metrics = LocalMetricsCollector::default(); + let query = SargableQuery::Equals(ScalarValue::Utf8(Some("red".to_string()))); + index.search(&query, &metrics).await.unwrap(); + assert_eq!(metrics.index_cache_hits(), 0); + assert_eq!(metrics.index_cache_misses(), 1); + } + // Regression test for the O(N log N) warm-cache rebuild introduced in // commit 4de5ce67d. BitmapIndexState now caches the parsed Arc // so that get_from_cache skips parse_lookup_batch on warm hits. @@ -2401,12 +2543,10 @@ mod tests { .unwrap(); // Verify no bitmaps are cached yet - let cache_key_red = BitmapKey { - value: OrderableScalarValue(ScalarValue::Utf8(Some("red".to_string()))), - }; - let cache_key_blue = BitmapKey { - value: OrderableScalarValue(ScalarValue::Utf8(Some("blue".to_string()))), - }; + let red = OrderableScalarValue(ScalarValue::Utf8(Some("red".to_string()))); + let blue = OrderableScalarValue(ScalarValue::Utf8(Some("blue".to_string()))); + let cache_key_red = BitmapKey::try_new(*index.index_map.get(&red).unwrap()).unwrap(); + let cache_key_blue = BitmapKey::try_new(*index.index_map.get(&blue).unwrap()).unwrap(); assert!( cache @@ -2677,8 +2817,32 @@ mod tests { .await .unwrap(); - // Test 1: Search for value 5 - should return allow=[1], null=[2] + // Test 1: A caller that does not need NULL bookkeeping should receive + // the same true rows without cloning the null bitmap. let query = SargableQuery::Equals(ScalarValue::Int64(Some(5))); + let result = index + .search_with_options( + &query, + SearchOptions::default().with_track_nulls(false), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + match result { + SearchResult::Exact(row_ids) => { + let actual_rows: Vec = row_ids + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .collect(); + assert_eq!(actual_rows, vec![1]); + assert!(row_ids.null_rows().is_empty()); + } + _ => panic!("Expected Exact search result"), + } + + // The existing API keeps NULL rows for three-valued logic. let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); match result { @@ -2700,6 +2864,7 @@ mod tests { } _ => panic!("Expected Exact search result"), } + let entries_after_value_lookup = cache.size().await; // Test 2: Search for null values - should return allow=[2], null=None let query = SargableQuery::IsNull(); @@ -2728,6 +2893,11 @@ mod tests { } _ => panic!("Expected Exact search result"), } + assert_eq!( + cache.size().await, + entries_after_value_lookup, + "null bitmap lookup should bypass the per-value cache" + ); // Test 3: Range query - should return matching rows and null_list let query = SargableQuery::Range( diff --git a/rust/lance-index/src/scalar/bloomfilter.rs b/rust/lance-index/src/scalar/bloomfilter.rs index 41bcb5a8b11..6235e1e2199 100644 --- a/rust/lance-index/src/scalar/bloomfilter.rs +++ b/rust/lance-index/src/scalar/bloomfilter.rs @@ -7,6 +7,7 @@ //! It is a space-efficient data structure that can be used to test whether an element is a member of a set. //! It's an inexact filter - they may include false positives that require rechecking. +use crate::pb; use crate::scalar::expression::{BloomFilterQueryParser, ScalarQueryParser}; use crate::scalar::registry::{ BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest, @@ -14,15 +15,18 @@ use crate::scalar::registry::{ use crate::scalar::{ BloomFilterQuery, BuiltinIndexType, CreatedIndex, IndexFile, ScalarIndexParams, UpdateCriteria, }; -use crate::{Any, pb}; use arrow_array::{Array, UInt64Array}; -use arrow_schema::{DataType, Field}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use futures::TryStreamExt; use lance_arrow_stats::StatisticsAccumulator; use lance_core::utils::bloomfilter::as_bytes; use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder}; use lance_core::utils::row_addr_remap::RowAddrRemap; +use lance_select::RowAddrTreeMap; use serde::{Deserialize, Serialize}; +use std::any::Any; +use std::collections::HashMap; use std::sync::LazyLock; use datafusion::execution::SendableRecordBatchStream; @@ -44,7 +48,13 @@ use super::zoned::{ZoneBound, ZoneProcessor, ZoneTrainer, rebuild_zones, search_ const BLOOMFILTER_FILENAME: &str = "bloomfilter.lance"; const BLOOMFILTER_ITEM_META_KEY: &str = "bloomfilter_item"; +const NULL_BITMAP_META_KEY: &str = "null_bitmap"; const BLOOMFILTER_PROBABILITY_META_KEY: &str = "bloomfilter_probability"; +/// Upper bound on the total serialized bytes packed into a single bloom filter +/// `BinaryArray`. Its offsets are `i32`, so the concatenated payload cannot exceed +/// `i32::MAX`. We reserve a 1 MiB margin below that hard limit so per-row Arrow +/// bookkeeping (offset and validity buffers) cannot push a batch over the edge. +const MAX_BLOOMFILTER_ARRAY_LENGTH: usize = i32::MAX as usize - 1024 * 1024; const BLOOMFILTER_INDEX_VERSION: u32 = 0; #[derive(Debug, Clone)] @@ -80,24 +90,34 @@ pub struct BloomFilterIndex { number_of_items: u64, // Probability of false positives, fraction between 0 and 1 probability: f64, + // Exact set of null row addresses; None for older indices without this bitmap. + null_rows: Option, + frag_reuse_index: Option>, } impl DeepSizeOf for BloomFilterIndex { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.zones.deep_size_of_children(context) + self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context) } } impl BloomFilterIndex { async fn load( store: Arc, - _fri: Option>, + fri: Option>, + index_cache: &LanceCache, + ) -> Result> { + Self::load_with_max_array_length(store, fri, index_cache, MAX_BLOOMFILTER_ARRAY_LENGTH) + .await + } + + async fn load_with_max_array_length( + store: Arc, + fri: Option>, _index_cache: &LanceCache, + max_array_length: usize, ) -> Result> { let index_file = store.open_index_file(BLOOMFILTER_FILENAME).await?; - let bloom_data = index_file - .read_range(0..index_file.num_rows(), None) - .await?; let file_schema = index_file.schema(); let number_of_items: u64 = file_schema @@ -112,25 +132,65 @@ impl BloomFilterIndex { .and_then(|bs| bs.parse().ok()) .unwrap_or(*DEFAULT_PROBABILITY); - Ok(Arc::new(Self::try_from_serialized( - bloom_data, + let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) { + let idx = idx_str.parse::().map_err(|e| { + Error::invalid_input(format!("invalid null bitmap buffer index: {e}")) + })?; + let bytes = index_file.read_global_buffer(idx).await?; + Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?) + } else { + None + }; + + let read_batch_size = + Self::read_batch_size(number_of_items, probability, max_array_length)?; + + let mut zones = Vec::with_capacity(index_file.num_rows()); + for start in (0..index_file.num_rows()).step_by(read_batch_size) { + let end = (start + read_batch_size).min(index_file.num_rows()); + let mut bloom_data = index_file.read_range_stream(start..end, None).await?; + while let Some(batch) = bloom_data.try_next().await? { + zones.extend(Self::try_from_serialized(batch, max_array_length)?); + } + } + + Ok(Arc::new(Self { + zones, number_of_items, probability, - )?)) + null_rows, + frag_reuse_index: fri, + })) } - fn try_from_serialized( - data: RecordBatch, + fn read_batch_size( number_of_items: u64, probability: f64, - ) -> Result { + max_array_length: usize, + ) -> Result { + // Bloom filters are stored in an Arrow BinaryArray, whose offsets are i32. + // The serialized filter size is fixed by the index parameters, so bound + // reads by total serialized bytes instead of row count alone. + let params = BloomFilterIndexBuilderParams { + number_of_items, + probability, + }; + let filter_size = BloomFilterProcessor::build_filter(¶ms)?.size_bytes(); + if filter_size > max_array_length { + return Err(Error::invalid_input(format!( + "Serialized bloom filter size {} exceeds max supported batch bytes {}", + filter_size, max_array_length + ))); + } + Ok((max_array_length / filter_size).max(1)) + } + + fn try_from_serialized( + data: RecordBatch, + max_array_length: usize, + ) -> Result> { if data.num_rows() == 0 { - // Return empty index for empty data - return Ok(Self { - zones: Vec::new(), - number_of_items, - probability, - }); + return Ok(Vec::new()); } let fragment_id_col = data @@ -171,6 +231,18 @@ impl BloomFilterIndex { Error::invalid_input("BloomFilterIndex: 'bloom_filter_data' column is not Binary") })?; + // Enforce the i32-offset cap on read, symmetric to the write side. A batch this + // large means the read chunking was bypassed; reject it before it overflows the + // BinaryArray offsets instead of panicking deep inside Arrow. + let offsets = bloom_filter_data_col.value_offsets(); + let batch_bytes = (offsets[offsets.len() - 1] - offsets[0]) as usize; + if batch_bytes > max_array_length { + return Err(Error::invalid_input(format!( + "Serialized bloom filter batch size {} exceeds max supported batch bytes {}", + batch_bytes, max_array_length + ))); + } + let has_null_col = data .column_by_name("has_null") .ok_or_else(|| Error::invalid_input("BloomFilterIndex: missing 'has_null' column"))? @@ -190,7 +262,6 @@ impl BloomFilterIndex { Vec::new() }; - // Convert bytes back to Sbbf let bloom_filter = Sbbf::new(&bloom_filter_bytes).map_err(|e| { Error::invalid_input(format!("Failed to deserialize bloom filter: {:?}", e)) })?; @@ -206,11 +277,7 @@ impl BloomFilterIndex { }); } - Ok(Self { - zones: blocks, - number_of_items, - probability, - }) + Ok(blocks) } fn evaluate_block_against_query( @@ -415,11 +482,33 @@ impl ScalarIndex for BloomFilterIndex { metrics: &dyn MetricsCollector, ) -> Result { let query = query.as_any().downcast_ref::().unwrap(); - search_zones(&self.zones, metrics, |block| { - self.evaluate_block_against_query(block, query) + let result = if let BloomFilterQuery::IsNull() = query + && let Some(null_rows) = &self.null_rows + { + SearchResult::exact(null_rows.clone()) + } else { + search_zones(&self.zones, metrics, |block| { + self.evaluate_block_against_query(block, query) + })? + }; + + let Some(remapper) = &self.frag_reuse_index else { + return Ok(result); + }; + let selected = remapper.remap_row_addrs_tree_map(result.row_addrs().selected_rows()); + let nulls = remapper.remap_row_addrs_tree_map(result.row_addrs().null_rows()); + + Ok(match result { + SearchResult::Exact(_) => SearchResult::exact(selected).with_nulls(nulls), + SearchResult::AtMost(_) => SearchResult::at_most(selected).with_nulls(nulls), + SearchResult::AtLeast(_) => SearchResult::at_least(selected).with_nulls(nulls), }) } + fn results_are_row_addresses(&self) -> bool { + true + } + fn can_remap(&self) -> bool { false } @@ -448,18 +537,28 @@ impl ScalarIndex for BloomFilterIndex { let processor = BloomFilterProcessor::new(params.clone())?; let trainer = ZoneTrainer::new(processor, params.number_of_items)?; - let updated_blocks = rebuild_zones(&self.zones, trainer, new_data).await?; + let (updated_blocks, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?; + + // Merge existing and new null rows. If the existing index had no null bitmap + // (legacy format — null positions unknown), preserve that None: updating cannot + // recover the missing information, and claiming the result has zero nulls would + // be a false negative. Only a full retrain produces a fresh, complete bitmap. + let merged_null_rows = self.null_rows.as_ref().map(|existing| { + let mut merged = existing.clone(); + merged |= &new_null_rows; + merged + }); // Write the combined zones back to storage let mut builder = BloomFilterIndexBuilder::try_new(params)?; builder.blocks = updated_blocks; - let file = builder.write_index(dest_store).await?; + builder.null_rows = merged_null_rows; + let files = builder.write_index(dest_store).await?; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default()) - .unwrap(), + index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())?, index_version: BLOOMFILTER_INDEX_VERSION, - files: vec![file], + files, }) } @@ -478,6 +577,126 @@ impl ScalarIndex for BloomFilterIndex { } } +fn remap_zone( + zone: &BloomFilterStatistics, + remapper: &dyn RowIdRemapper, +) -> Vec { + let zone_start = (zone.bound.fragment_id << 32).saturating_add(zone.bound.start); + let mut remapped = (0..zone.bound.length as u64) + .filter_map(|offset| remapper.remap_row_id(zone_start.saturating_add(offset))) + .collect::>(); + remapped.sort_unstable(); + remapped.dedup(); + + let mut zones = Vec::new(); + let mut run_start = None; + let mut previous = 0u64; + for row_id in remapped { + if run_start.is_none() { + run_start = Some(row_id); + } else if row_id != previous.saturating_add(1) || row_id >> 32 != previous >> 32 { + let start = run_start.take().unwrap(); + zones.push(BloomFilterStatistics { + bound: ZoneBound { + fragment_id: start >> 32, + start: start & u64::from(u32::MAX), + length: (previous - start + 1) as usize, + }, + has_null: zone.has_null, + bloom_filter: zone.bloom_filter.clone(), + }); + run_start = Some(row_id); + } + previous = row_id; + } + if let Some(start) = run_start { + zones.push(BloomFilterStatistics { + bound: ZoneBound { + fragment_id: start >> 32, + start: start & u64::from(u32::MAX), + length: (previous - start + 1) as usize, + }, + has_null: zone.has_null, + bloom_filter: zone.bloom_filter.clone(), + }); + } + zones +} + +/// Merge caller-selected BloomFilter segments into one self-contained segment. +pub async fn merge_bloomfilter_indices( + source_indices: &[(&BloomFilterIndex, &RoaringBitmap)], + dest_store: &dyn IndexStore, +) -> Result { + let first = source_indices + .iter() + .find(|(_, fragment_filter)| !fragment_filter.is_empty()) + .or_else(|| source_indices.first()) + .ok_or_else(|| { + Error::invalid_input("merge_bloomfilter_indices requires at least one source index") + })?; + let params = BloomFilterIndexBuilderParams { + number_of_items: first.0.number_of_items, + probability: first.0.probability, + }; + + let mut blocks = Vec::new(); + let mut merged_null_rows = RowAddrTreeMap::new(); + let mut has_missing_null_bitmap = false; + for (source, fragment_filter) in source_indices { + if fragment_filter.is_empty() { + continue; + } + if source.number_of_items != params.number_of_items + || source.probability != params.probability + { + return Err(Error::invalid_input(format!( + "cannot merge BloomFilter segments with different parameters: \ + number_of_items={}, probability={} and number_of_items={}, probability={}", + params.number_of_items, + params.probability, + source.number_of_items, + source.probability + ))); + } + let source_zones = source.zones.iter().flat_map(|block| { + source.frag_reuse_index.as_deref().map_or_else( + || vec![block.clone()], + |remapper| remap_zone(block, remapper), + ) + }); + blocks.extend(source_zones.filter(|block| { + u32::try_from(block.bound.fragment_id) + .is_ok_and(|fragment_id| fragment_filter.contains(fragment_id)) + })); + match &source.null_rows { + Some(null_rows) => { + let mut filtered = source.frag_reuse_index.as_deref().map_or_else( + || null_rows.clone(), + |remapper| remapper.remap_row_addrs_tree_map(null_rows), + ); + filtered.retain_fragments(fragment_filter.iter()); + merged_null_rows |= &filtered; + } + None => has_missing_null_bitmap = true, + } + } + blocks.sort_by_key(|block| (block.bound.fragment_id, block.bound.start)); + + let mut builder = BloomFilterIndexBuilder::try_new(params)?; + builder.blocks = blocks; + if !has_missing_null_bitmap { + builder.null_rows = Some(merged_null_rows); + } + let files = builder.write_index(dest_store).await?; + + Ok(CreatedIndex { + index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default())?, + index_version: BLOOMFILTER_INDEX_VERSION, + files, + }) +} + fn default_number_of_items() -> u64 { *DEFAULT_NUMBER_OF_ITEMS } @@ -486,6 +705,17 @@ fn default_probability() -> f64 { *DEFAULT_PROBABILITY } +/// Schema of the per-zone bloom filter statistics batch. +static BLOOMFILTER_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![ + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + Field::new("has_null", DataType::Boolean, false), + Field::new("bloom_filter_data", DataType::Binary, false), + ])) +}); + // NumberOfItems: 8192 + Probability: 0.00057(1 in 1754) -> NumberOfBytes: 16384(16KiB) + 8 SALT values // reference: https://hur.st/bloomfilter/?n=8192&p=&m=16KiB&k=8 static DEFAULT_NUMBER_OF_ITEMS: LazyLock = LazyLock::new(|| { @@ -542,6 +772,10 @@ impl BloomFilterIndexBuilderParams { pub struct BloomFilterIndexBuilder { params: BloomFilterIndexBuilderParams, blocks: Vec, + // None means "legacy index — null positions unknown"; Some means a complete bitmap. + // write_index omits the null-bitmap global buffer when this is None, preserving the + // legacy format so that downstream searches remain conservative. + null_rows: Option, } impl BloomFilterIndexBuilder { @@ -549,6 +783,7 @@ impl BloomFilterIndexBuilder { Ok(Self { params, blocks: Vec::new(), + null_rows: None, }) } @@ -558,37 +793,26 @@ impl BloomFilterIndexBuilder { pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> { let processor = BloomFilterProcessor::new(self.params.clone())?; let trainer = ZoneTrainer::new(processor, self.params.number_of_items)?; - self.blocks = trainer.train(batches_source).await?; + let (blocks, null_rows) = trainer.train(batches_source).await?; + self.blocks = blocks; + self.null_rows = Some(null_rows); Ok(()) } - fn bloomfilter_stats_as_batch(&self) -> Result { - let fragment_ids = - UInt64Array::from_iter_values(self.blocks.iter().map(|block| block.bound.fragment_id)); - - let zone_starts = - UInt64Array::from_iter_values(self.blocks.iter().map(|block| block.bound.start)); - - let zone_lengths = UInt64Array::from_iter_values( - self.blocks.iter().map(|block| block.bound.length as u64), - ); - - let has_nulls = arrow_array::BooleanArray::from( - self.blocks - .iter() - .map(|block| block.has_null) - .collect::>(), - ); - - // Convert bloom filters to binary data for serialization - let bloom_filter_data = if self.blocks.is_empty() { + fn bloomfilter_stats_as_batch( + fragment_ids: Vec, + zone_starts: Vec, + zone_lengths: Vec, + has_nulls: Vec, + binary_data: Vec>, + ) -> Result { + let fragment_ids = UInt64Array::from(fragment_ids); + let zone_starts = UInt64Array::from(zone_starts); + let zone_lengths = UInt64Array::from(zone_lengths); + let has_nulls = arrow_array::BooleanArray::from(has_nulls); + let bloom_filter_data = if binary_data.is_empty() { Arc::new(arrow_array::BinaryArray::new_null(0)) as ArrayRef } else { - let binary_data: Vec> = self - .blocks - .iter() - .map(|block| block.bloom_filter.to_bytes()) - .collect(); let binary_refs: Vec> = binary_data .iter() .map(|bytes| Some(bytes.as_slice())) @@ -596,14 +820,6 @@ impl BloomFilterIndexBuilder { Arc::new(arrow_array::BinaryArray::from_opt_vec(binary_refs)) as ArrayRef }; - let schema = Arc::new(arrow_schema::Schema::new(vec![ - Field::new("fragment_id", DataType::UInt64, false), - Field::new("zone_start", DataType::UInt64, false), - Field::new("zone_length", DataType::UInt64, false), - Field::new("has_null", DataType::Boolean, false), - Field::new("bloom_filter_data", DataType::Binary, false), - ])); - let columns: Vec = vec![ Arc::new(fragment_ids) as ArrayRef, Arc::new(zone_starts) as ArrayRef, @@ -612,28 +828,160 @@ impl BloomFilterIndexBuilder { bloom_filter_data, ]; - Ok(RecordBatch::try_new(schema, columns)?) + Ok(RecordBatch::try_new(BLOOMFILTER_SCHEMA.clone(), columns)?) } - pub async fn write_index(self, index_store: &dyn IndexStore) -> Result { - let record_batch = self.bloomfilter_stats_as_batch()?; + /// Serialize the trained bloom filter zone statistics into an index file in + /// `index_store`, returning the resulting [`IndexFile`]s. + /// + /// Zones are flushed as one or more record batches, each bounded by + /// `MAX_BLOOMFILTER_ARRAY_LENGTH` serialized bytes so the underlying Arrow + /// `BinaryArray` never overflows its `i32` offsets. Any optional null-row bitmap + /// is persisted as a global buffer on the same [`IndexFile`] via [`IndexStore`]. + pub async fn write_index(self, index_store: &dyn IndexStore) -> Result> { + self.write_index_with_max_array_length(index_store, MAX_BLOOMFILTER_ARRAY_LENGTH) + .await + } - let mut file_schema = record_batch.schema().as_ref().clone(); + async fn write_index_with_max_array_length( + self, + index_store: &dyn IndexStore, + max_array_length: usize, + ) -> Result> { + let mut file_schema = BLOOMFILTER_SCHEMA.as_ref().clone(); file_schema.metadata.insert( BLOOMFILTER_ITEM_META_KEY.to_string(), self.params.number_of_items.to_string(), ); - file_schema.metadata.insert( BLOOMFILTER_PROBABILITY_META_KEY.to_string(), self.params.probability.to_string(), ); - let mut index_file = index_store + let index_file = index_store .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema)) .await?; - index_file.write_record_batch(record_batch).await?; - index_file.finish().await + + let mut writer = BloomFilterBatchWriter::new(index_file, max_array_length); + for block in self.blocks { + writer.emit(block).await?; + } + let bloomfilter_file = writer.finish(self.null_rows).await?; + Ok(vec![bloomfilter_file]) + } +} + +/// Buffers serialized bloom filter zone statistics and flushes them as record batches +/// to the index file, respecting the `max_array_length` limit. +struct BloomFilterBatchWriter { + file: Box, + max_array_length: usize, + fragment_ids: Vec, + zone_starts: Vec, + zone_lengths: Vec, + has_nulls: Vec, + bloom_filter_data: Vec>, + current_bytes: usize, + has_written: bool, +} + +impl BloomFilterBatchWriter { + fn new(file: Box, max_array_length: usize) -> Self { + Self { + file, + max_array_length, + fragment_ids: Vec::new(), + zone_starts: Vec::new(), + zone_lengths: Vec::new(), + has_nulls: Vec::new(), + bloom_filter_data: Vec::new(), + current_bytes: 0, + has_written: false, + } + } + + async fn emit(&mut self, block: BloomFilterStatistics) -> Result<()> { + let serialized_filter = block.bloom_filter.to_bytes(); + let serialized_len = serialized_filter.len(); + + if serialized_len > self.max_array_length { + return Err(Error::invalid_input(format!( + "Serialized bloom filter size {} exceeds max supported batch bytes {}", + serialized_len, self.max_array_length + ))); + } + + let next_bytes = self + .current_bytes + .checked_add(serialized_len) + .ok_or_else(|| { + Error::invalid_input(format!( + "Bloom filter batch size overflow when adding {} bytes to {} bytes", + serialized_len, self.current_bytes + )) + })?; + + if !self.bloom_filter_data.is_empty() && next_bytes > self.max_array_length { + self.flush().await?; + } + + self.fragment_ids.push(block.bound.fragment_id); + self.zone_starts.push(block.bound.start); + self.zone_lengths.push(block.bound.length as u64); + self.has_nulls.push(block.has_null); + self.bloom_filter_data.push(serialized_filter); + self.current_bytes += serialized_len; + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + if self.bloom_filter_data.is_empty() { + return Ok(()); + } + + let batch = BloomFilterIndexBuilder::bloomfilter_stats_as_batch( + std::mem::take(&mut self.fragment_ids), + std::mem::take(&mut self.zone_starts), + std::mem::take(&mut self.zone_lengths), + std::mem::take(&mut self.has_nulls), + std::mem::take(&mut self.bloom_filter_data), + )?; + self.file.write_record_batch(batch).await?; + self.current_bytes = 0; + self.has_written = true; + Ok(()) + } + + async fn finish(mut self, null_rows: Option) -> Result { + self.flush().await?; + if !self.has_written { + self.file + .write_record_batch(BloomFilterIndexBuilder::bloomfilter_stats_as_batch( + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + )?) + .await?; + } + + if let Some(null_rows) = null_rows { + let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size()); + null_rows.serialize_into(&mut null_bitmap_bytes)?; + let null_bitmap_idx = self + .file + .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes)) + .await?; + self.file + .finish_with_metadata(HashMap::from([( + NULL_BITMAP_META_KEY.to_string(), + null_bitmap_idx.to_string(), + )])) + .await + } else { + self.file.finish_with_metadata(HashMap::new()).await + } } } @@ -980,7 +1328,7 @@ impl BloomFilterIndexPlugin { batches_source: SendableRecordBatchStream, index_store: &dyn IndexStore, options: Option, - ) -> Result { + ) -> Result> { let mut builder = BloomFilterIndexBuilder::try_new(options.unwrap_or_default())?; builder.train(batches_source).await?; @@ -1049,15 +1397,9 @@ impl BasicTrainer for BloomFilterIndexPlugin { data: SendableRecordBatchStream, index_store: &dyn IndexStore, request: Box, - fragment_ids: Option>, + _fragment_ids: Option>, _progress: Arc, ) -> Result { - if fragment_ids.is_some() { - return Err(Error::invalid_input_source( - "BloomFilter index does not support fragment training".into(), - )); - } - let request = (request as Box) .downcast::() .map_err(|_| { @@ -1065,12 +1407,12 @@ impl BasicTrainer for BloomFilterIndexPlugin { "must provide training request created by new_training_request".into(), ) })?; - let file = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?; + let files = Self::train_bloomfilter_index(data, index_store, Some(request.params)).await?; Ok(CreatedIndex { index_details: prost_types::Any::from_msg(&pb::BloomFilterIndexDetails::default()) .unwrap(), index_version: BLOOMFILTER_INDEX_VERSION, - files: vec![file], + files, }) } } @@ -1154,7 +1496,9 @@ impl TrainingRequest for BloomFilterIndexTrainingRequest { #[cfg(test)] mod tests { + use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle}; use crate::scalar::registry::VALUE_COLUMN_NAME; + use std::collections::HashMap; use std::sync::Arc; use crate::scalar::bloomfilter::BloomFilterIndexPlugin; @@ -1164,15 +1508,19 @@ mod tests { use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion_common::ScalarValue; use futures::{StreamExt, stream}; - use lance_core::{ROW_ADDR, cache::LanceCache, utils::tempfile::TempObjDir}; + use lance_core::{Error, ROW_ADDR, cache::LanceCache, utils::tempfile::TempObjDir}; use lance_io::object_store::ObjectStore; use lance_select::RowAddrTreeMap; use crate::scalar::{ - BloomFilterQuery, ScalarIndex, SearchResult, - bloomfilter::{BloomFilterIndex, BloomFilterIndexBuilderParams}, + BloomFilterQuery, IndexStore, ScalarIndex, SearchResult, + bloomfilter::{ + BloomFilterIndex, BloomFilterIndexBuilder, BloomFilterIndexBuilderParams, + merge_bloomfilter_indices, + }, lance_format::LanceIndexStore, }; + use lance_core::utils::bloomfilter::sbbf::Sbbf; use crate::Index; // Import Index trait to access calculate_included_frags use crate::metrics::NoOpMetricsCollector; @@ -2037,10 +2385,10 @@ mod tests { expected.insert_range(500..750); // Should match the zone containing 500 assert_eq!(result, SearchResult::at_most(expected)); - // Test IsNull query + // Test IsNull query (no nulls in data, should return exact empty set) let query = BloomFilterQuery::IsNull(); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); // No nulls in the data + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new())); // Test IsIn query let query = BloomFilterQuery::IsIn(vec![ @@ -2133,4 +2481,523 @@ mod tests { _ => panic!("Expected AtMost search result from bloomfilter"), } } + + #[test] + fn test_bloomfilter_read_batch_size_is_byte_bounded() { + let number_of_items = 1; + let probability = 0.25; + let max_test_batch_bytes = 48; + let filter_bytes = Sbbf::with_ndv_fpp(number_of_items, probability) + .unwrap() + .to_bytes(); + + assert!(filter_bytes.len() <= max_test_batch_bytes); + assert!(filter_bytes.len() * 2 > max_test_batch_bytes); + + assert_eq!( + BloomFilterIndex::read_batch_size(number_of_items, probability, max_test_batch_bytes,) + .unwrap(), + 1 + ); + } + + #[test] + fn test_bloomfilter_read_batch_size_rejects_oversized_filter() { + let number_of_items = 1; + let probability = 0.25; + let filter_bytes = Sbbf::with_ndv_fpp(number_of_items, probability) + .unwrap() + .to_bytes(); + + let error = + BloomFilterIndex::read_batch_size(number_of_items, probability, filter_bytes.len() - 1) + .unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "unexpected error variant: {error:?}" + ); + assert!( + error + .to_string() + .contains("exceeds max supported batch bytes"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_bloomfilter_chunked_write_and_load() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let row_count = 5_000; + // Sprinkle nulls at known positions (none of which are asserted individually + // below) so the chunked write must carry a non-empty null-row bitmap through + // multiple flushes and reload it correctly via add_global_buffer. + let expected_null_count = (0..row_count).filter(|&i| i % 100 == 50).count(); + let values = (0..row_count) + .map(|i| (i % 100 != 50).then_some(i)) + .collect::>(); + let data = record_batch!((VALUE_COLUMN_NAME, Int32, values)).unwrap(); + let schema = data.schema(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + let data_stream = add_row_addr(data_stream); + + let mut builder = + BloomFilterIndexBuilder::try_new(BloomFilterIndexBuilderParams::new(1, 0.25)).unwrap(); + builder.train(data_stream).await.unwrap(); + assert_eq!(builder.blocks.len(), row_count as usize); + + // Small byte limit forces the writer to emit many on-disk batches and the + // reader to chunk its reads. The total payload is far larger than the limit, + // so a load that concatenated everything into one BinaryArray would trip the + // read-side cap; a correctly chunked load must still succeed. + let max_array_length = 64; + let filter_bytes = Sbbf::with_ndv_fpp(1, 0.25).unwrap().size_bytes(); + assert!(filter_bytes * row_count as usize > max_array_length); + + builder + .write_index_with_max_array_length(test_store.as_ref(), max_array_length) + .await + .unwrap(); + + let index = BloomFilterIndex::load_with_max_array_length( + test_store.clone(), + None, + &LanceCache::no_cache(), + max_array_length, + ) + .await + .expect("Failed to load chunked BloomFilterIndex"); + + assert_eq!(index.zones.len(), row_count as usize); + assert_eq!(index.zones[0].bound.start, 0); + assert_eq!(index.zones[4096].bound.start, 4096); + assert_eq!(index.zones[4999].bound.start, 4999); + assert_eq!(index.zones[4999].bound.length, 1); + assert!(!index.zones[4096].bloom_filter.to_bytes().is_empty()); + + // The null-row bitmap is stored as a global buffer, independent of the chunked + // zone batches. Verify it survives the multi-flush write and reloads intact. + let null_rows = index + .null_rows + .as_ref() + .expect("chunked write must preserve the null-row bitmap"); + let loaded_null_count = null_rows + .row_addrs() + .map(|addrs| addrs.count()) + .unwrap_or(0); + assert_eq!(loaded_null_count, expected_null_count); + } + + #[tokio::test] + async fn test_bloomfilter_load_rejects_unchunked_oversized_read() { + // Guards the read-side invariant directly: a batch whose concatenated filter + // payload exceeds the cap must be rejected rather than building an oversized + // BinaryArray. This is what protects `load` if its chunking ever regresses. + let filter_bytes = Sbbf::with_ndv_fpp(1, 0.25).unwrap().to_bytes(); + let batch = BloomFilterIndexBuilder::bloomfilter_stats_as_batch( + vec![0, 0], + vec![0, 1], + vec![1, 1], + vec![false, false], + vec![filter_bytes.clone(), filter_bytes.clone()], + ) + .unwrap(); + + // One filter fits, two together do not. + let max_array_length = filter_bytes.len(); + let error = BloomFilterIndex::try_from_serialized(batch, max_array_length).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "unexpected error variant: {error:?}" + ); + assert!( + error + .to_string() + .contains("exceeds max supported batch bytes"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_bloomfilter_chunked_write_rejects_oversized_filter() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let data = record_batch!((VALUE_COLUMN_NAME, Int32, [0])).unwrap(); + let schema = data.schema(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + let data_stream = add_row_addr(data_stream); + + let mut builder = + BloomFilterIndexBuilder::try_new(BloomFilterIndexBuilderParams::new(1, 0.25)).unwrap(); + builder.train(data_stream).await.unwrap(); + + let error = builder + .write_index_with_max_array_length(test_store.as_ref(), 16) + .await + .unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "unexpected error variant: {error:?}" + ); + assert!( + error + .to_string() + .contains("exceeds max supported batch bytes 16"), + "unexpected error: {error}" + ); + } + + // Writes a bloomfilter file in the legacy format (no null bitmap global buffer), + // simulating an index created before the null bitmap feature was added. + async fn write_legacy_bloomfilter(store: &dyn IndexStore, has_null: bool) { + use crate::scalar::bloomfilter::{ + BLOOMFILTER_FILENAME, BLOOMFILTER_ITEM_META_KEY, BLOOMFILTER_PROBABILITY_META_KEY, + }; + use arrow_array::BooleanArray; + let schema = Arc::new(Schema::new(vec![ + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + Field::new("has_null", DataType::Boolean, false), + Field::new("bloom_filter_data", DataType::Binary, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![3u64])) as _, + Arc::new(BooleanArray::from(vec![has_null])) as _, + Arc::new(arrow_array::BinaryArray::from_vec(vec![b"".as_ref()])) as _, + ], + ) + .unwrap(); + let mut file_schema = schema.as_ref().clone(); + file_schema + .metadata + .insert(BLOOMFILTER_ITEM_META_KEY.to_string(), "1000".to_string()); + file_schema.metadata.insert( + BLOOMFILTER_PROBABILITY_META_KEY.to_string(), + "0.01".to_string(), + ); + let mut writer = store + .new_index_file(BLOOMFILTER_FILENAME, Arc::new(file_schema)) + .await + .unwrap(); + writer.write_record_batch(batch).await.unwrap(); + writer.finish().await.unwrap(); + } + + // Updating a legacy (null_rows = None) index must not silently treat None as + // "no nulls". The bug: `self.null_rows.clone().unwrap_or_default()` collapses + // None into an empty RowAddrTreeMap; after the merge the updated index has + // `null_rows = Some(empty)`, so an IsNull search returns `exact(empty)` — a + // false negative even though the legacy zone recorded has_null = true. + #[tokio::test] + async fn test_update_legacy_none_null_rows_not_treated_as_no_nulls() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Write a legacy-format index (no null bitmap) with has_null=true in its zone. + write_legacy_bloomfilter(store.as_ref(), true).await; + + let index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + assert!( + index.null_rows.is_none(), + "precondition: legacy null_rows is None" + ); + + // Update with new data from fragment 1 (no nulls). The destination is the + // same store so we can reload from it afterwards. + let new_schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Int32, true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let new_batch = RecordBatch::try_new( + new_schema.clone(), + vec![ + Arc::new(arrow_array::Int32Array::from(vec![ + Some(10i32), + Some(20), + Some(30), + ])) as _, + Arc::new(UInt64Array::from_iter_values( + (0u64..3).map(|i| (1u64 << 32) | i), + )) as _, + ], + ) + .unwrap(); + let new_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + new_schema, + stream::once(std::future::ready(Ok(new_batch))), + )); + + index + .update(new_stream, store.as_ref(), None) + .await + .unwrap(); + + let updated_index = BloomFilterIndex::load(store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + + // The legacy zone had has_null=true, so there ARE nulls at unknown positions. + // An IsNull search on the updated index must NOT claim "no nulls" (exact empty). + // It must be conservative and return AtMost, falling back to the has_null scan. + let result = updated_index + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + + // With the bug: null_rows = Some(empty) → returns exact(empty) ← FALSE NEGATIVE + // With the fix: null_rows = None → falls through to has_null scan → AtMost + assert!( + !result.is_exact(), + "IsNull on an updated legacy index must not return exact(empty); \ + the legacy zone had has_null=true so nulls exist at unknown positions" + ); + } + + #[tokio::test] + async fn test_legacy_bloomfilter_no_null_bitmap() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + write_legacy_bloomfilter(store.as_ref(), true).await; + + let index = BloomFilterIndex::load(store, None, &LanceCache::no_cache()) + .await + .expect("failed to load legacy bloomfilter"); + + assert!( + index.null_rows.is_none(), + "legacy index should have no null bitmap" + ); + + // IS NULL should fall back to the has_null zone scan and return AtMost, not Exact. + let result = index + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on a legacy index should not be exact" + ); + } + + #[tokio::test] + async fn test_merge_bloomfilter_indices_preserves_exact_and_legacy_nulls() { + fn create_store() -> (TempObjDir, Arc) { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + (tmpdir, store) + } + + let (_first_tmpdir, first_store) = create_store(); + let (_second_tmpdir, second_store) = create_store(); + let (_merged_tmpdir, merged_store) = create_store(); + let params = BloomFilterIndexBuilderParams::new(1000, 0.01); + for (fragment_id, store) in [(0_u64, &first_store), (1_u64, &second_store)] { + let row_base = fragment_id << 32; + let batch = record_batch!( + (VALUE_COLUMN_NAME, Int64, [Some(10), None, Some(30)]), + (ROW_ADDR, UInt64, [row_base, row_base + 1, row_base + 2]) + ) + .unwrap(); + let schema = batch.schema(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(async move { Ok(batch) }), + )); + BloomFilterIndexPlugin::train_bloomfilter_index( + stream, + store.as_ref(), + Some(params.clone()), + ) + .await + .unwrap(); + } + + let first = BloomFilterIndex::load(first_store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + let second = BloomFilterIndex::load(second_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + merge_bloomfilter_indices( + &[ + (first.as_ref(), &RoaringBitmap::from_iter([0])), + (second.as_ref(), &RoaringBitmap::from_iter([1])), + ], + merged_store.as_ref(), + ) + .await + .unwrap(); + let merged = BloomFilterIndex::load(merged_store.clone(), None, &LanceCache::no_cache()) + .await + .unwrap(); + let mut expected_nulls = RowAddrTreeMap::new(); + expected_nulls.insert(1); + expected_nulls.insert((1_u64 << 32) + 1); + assert_eq!( + merged + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(expected_nulls) + ); + + let remapped_base = 2_u64 << 32; + let remapper = FragReuseIndexHandle(Arc::new(FragReuseIndex::new( + uuid::Uuid::new_v4(), + vec![HashMap::from([ + (0, Some(remapped_base)), + (1, Some(remapped_base + 1)), + (2, Some(remapped_base + 2)), + ])], + FragReuseIndexDetails { versions: vec![] }, + ))); + let remapped_first = BloomFilterIndex::load( + first_store, + Some(Arc::new(remapper)), + &LanceCache::no_cache(), + ) + .await + .unwrap(); + let (_remapped_tmpdir, remapped_store) = create_store(); + merge_bloomfilter_indices( + &[(remapped_first.as_ref(), &RoaringBitmap::from_iter([2]))], + remapped_store.as_ref(), + ) + .await + .unwrap(); + let remapped = BloomFilterIndex::load(remapped_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + let mut expected_remapped_nulls = RowAddrTreeMap::new(); + expected_remapped_nulls.insert(remapped_base + 1); + assert_eq!( + remapped + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(expected_remapped_nulls) + ); + let candidates = remapped + .search( + &BloomFilterQuery::Equals(ScalarValue::Int64(Some(10))), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert!( + candidates + .row_addrs() + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .any(|row_id| row_id == remapped_base) + ); + + let (_legacy_tmpdir, legacy_store) = create_store(); + let (_legacy_merged_tmpdir, legacy_merged_store) = create_store(); + write_legacy_bloomfilter(legacy_store.as_ref(), true).await; + let legacy = BloomFilterIndex::load(legacy_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + merge_bloomfilter_indices( + &[ + (legacy.as_ref(), &RoaringBitmap::from_iter([0])), + (second.as_ref(), &RoaringBitmap::from_iter([1])), + ], + legacy_merged_store.as_ref(), + ) + .await + .unwrap(); + let legacy_merged = + BloomFilterIndex::load(legacy_merged_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + assert!( + !legacy_merged + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap() + .is_exact() + ); + + let (_trimmed_tmpdir, trimmed_store) = create_store(); + let mut ignored_legacy = legacy.as_ref().clone(); + ignored_legacy.probability = 0.5; + merge_bloomfilter_indices( + &[ + (&ignored_legacy, &RoaringBitmap::new()), + (second.as_ref(), &RoaringBitmap::from_iter([1])), + ], + trimmed_store.as_ref(), + ) + .await + .unwrap(); + let trimmed = BloomFilterIndex::load(trimmed_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + let mut expected_nulls = RowAddrTreeMap::new(); + expected_nulls.insert((1_u64 << 32) + 1); + assert_eq!( + trimmed + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(expected_nulls) + ); + + let (_empty_tmpdir, empty_store) = create_store(); + merge_bloomfilter_indices( + &[(&ignored_legacy, &RoaringBitmap::new())], + empty_store.as_ref(), + ) + .await + .unwrap(); + let empty = BloomFilterIndex::load(empty_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + assert_eq!( + empty + .search(&BloomFilterQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(), + SearchResult::exact(RowAddrTreeMap::new()) + ); + } } diff --git a/rust/lance-index/src/scalar/btree.rs b/rust/lance-index/src/scalar/btree.rs index 7201574d58a..f58548f00f9 100644 --- a/rust/lance-index/src/scalar/btree.rs +++ b/rust/lance-index/src/scalar/btree.rs @@ -13,7 +13,7 @@ use std::{ use super::{ AnyQuery, BuiltinIndexType, IndexFile, IndexReader, IndexStore, IndexWriter, MetricsCollector, - OldIndexDataFilter, SargableQuery, ScalarIndex, ScalarIndexParams, SearchResult, + OldIndexDataFilter, SargableQuery, ScalarIndex, ScalarIndexParams, SearchOptions, SearchResult, compute_next_prefix, }; use crate::cache_pb::{BTreeIndexHeader, RangeToFile}; @@ -63,8 +63,8 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::{ Error, ROW_ID, Result, cache::{ - CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache, - WeakLanceCache, + CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, CacheKeySchema, + KeyBuilder, LanceCache, WeakLanceCache, }, error::LanceOptionExt, utils::{ @@ -76,7 +76,7 @@ use lance_datafusion::{ chunker::chunk_concat_stream, exec::{LanceExecutionOptions, OneShotExec, execute_plan}, }; -use lance_select::{NullableRowAddrSet, RowSetOps}; +use lance_select::{NullableRowAddrSet, RowAddrTreeMap, RowSetOps}; use log::{debug, warn}; use object_store::Error as ObjectStoreError; use rangemap::RangeInclusiveMap; @@ -397,6 +397,8 @@ impl Ord for OrderableScalarValue { panic!("Attempt to compare List with non-List") } (LargeList(_), _) => todo!(), + (ListView(_), _) => todo!(), + (LargeListView(_), _) => todo!(), (Map(_), Map(_)) => todo!(), (Map(left), Null) => { if left.is_null(0) { @@ -1363,6 +1365,14 @@ impl CacheKey for BTreePageKey { "BTreePage" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.btree-page-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u32(self.page_number); + } + fn codec() -> Option { // Pages are cached as `FlatIndex` values (see `ValueType` above). Some(CacheCodec::from_impl::()) @@ -1489,6 +1499,14 @@ impl CacheKey for BTreeIndexStateKey { "BTreeIndexState" } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.scalar.btree-index-state-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_variant(0); + } + fn codec() -> Option { Some(CacheCodec::from_impl::()) } @@ -1633,11 +1651,17 @@ impl BTreeIndex { index_reader: LazyIndexReader, metrics: &dyn MetricsCollector, ) -> Result> { - self.index_cache - .get_or_insert_with_key(BTreePageKey { page_number }, move || async move { + let result = self + .index_cache + .get_or_insert_with_key_hit(BTreePageKey { page_number }, move || async move { self.read_page(page_number, index_reader, metrics).await }) - .await + .await; + match &result { + Ok((_, true)) => metrics.record_index_cache_hit(), + _ => metrics.record_index_cache_miss(), + } + result.map(|(page, _)| page) } #[instrument(level = "debug", skip_all)] @@ -1682,6 +1706,7 @@ impl BTreeIndex { matches: Matches, index_reader: LazyIndexReader, prebuilt: Option<&Arc>, + track_nulls: bool, metrics: &dyn MetricsCollector, ) -> Result { let subindex = self @@ -1692,13 +1717,14 @@ impl BTreeIndex { // For a large IsIn the predicate is compiled once (see `search`) and // reused here, instead of rebuilding the whole IN-list per page. Matches::Some(_) => match prebuilt { - Some(expr) => subindex.search_prebuilt(expr, metrics), - None => subindex.search(query, metrics), + Some(expr) => subindex.search_prebuilt(expr, track_nulls, metrics), + None => subindex.search(query, track_nulls, metrics), }, Matches::All(_) => Ok(match query { // This means we hit an all-null page so just grab all row ids as true SargableQuery::IsNull() => subindex.all_ignore_nulls(), - _ => subindex.all(), + _ if track_nulls => subindex.all(), + _ => subindex.all_non_null(), }), } } @@ -2102,6 +2128,16 @@ impl ScalarIndex for BTreeIndex { &self, query: &dyn AnyQuery, metrics: &dyn MetricsCollector, + ) -> Result { + self.search_with_options(query, SearchOptions::default(), metrics) + .await + } + + async fn search_with_options( + &self, + query: &dyn AnyQuery, + options: SearchOptions, + metrics: &dyn MetricsCollector, ) -> Result { let query = query.as_any().downcast_ref::().unwrap(); let mut pages = match query { @@ -2169,7 +2205,7 @@ impl ScalarIndex for BTreeIndex { // page with zero nulls is a true Matches::All, while one with nulls needs // Matches::Some only to track the null rows; surfacing `null_count` here // could refine that classification (see #6802). - if !matches!(query, SargableQuery::IsNull()) { + if options.track_nulls() && !matches!(query, SargableQuery::IsNull()) { let existing: HashSet = pages.iter().map(|m| m.page_id()).collect(); for &page_id in self .page_lookup @@ -2201,6 +2237,7 @@ impl ScalarIndex for BTreeIndex { page_index, lazy_index_reader.clone(), prebuilt.as_ref(), + options.track_nulls(), metrics, ) .boxed() @@ -2216,8 +2253,18 @@ impl ScalarIndex for BTreeIndex { .try_collect() .await?; - // Merge matching row IDs - let selection = NullableRowAddrSet::union_all(&results); + let selection = if options.track_nulls() { + NullableRowAddrSet::union_all(&results) + } else { + let selected_rows = results + .iter() + .map(NullableRowAddrSet::selected_rows) + .collect::>(); + NullableRowAddrSet::new( + RowAddrTreeMap::union_all(&selected_rows), + Default::default(), + ) + }; Ok(SearchResult::Exact(selection)) } @@ -2344,6 +2391,10 @@ impl ScalarIndex for BTreeIndex { })?; Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::BTree).with_params(¶ms)) } + + fn training_data_type(&self) -> Option { + Some(self.data_type.clone()) + } } struct BatchStats { @@ -2562,9 +2613,7 @@ pub async fn train_btree_index( Ok(vec![pages_file, lookup_file]) } -fn find_single_partition_files( - files: &[lance_table::format::IndexFile], -) -> Result> { +fn find_single_partition_files(files: &[super::IndexFile]) -> Result> { let lookup_files = files .iter() .filter_map(|file| { @@ -3384,7 +3433,8 @@ mod tests { use crate::{ metrics::NoOpMetricsCollector, scalar::{ - IndexStore, OldIndexDataFilter, SargableQuery, ScalarIndex, SearchResult, + IndexStore, OldIndexDataFilter, SargableQuery, ScalarIndex, SearchOptions, + SearchResult, btree::{BTREE_PAGES_NAME, BTreeIndex}, lance_format::LanceIndexStore, }, @@ -3704,6 +3754,53 @@ mod tests { assert_eq!(metrics.parts_loaded.load(Ordering::Relaxed), 1); } + #[tokio::test] + async fn test_page_cache_hit_miss_counts() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let data = gen_batch() + .col("value", array::step::()) + .col("_rowid", array::step::()) + .into_df_exec(RowCount::from(1000), BatchCount::from(10)); + let schema = data.schema(); + let sort_expr = PhysicalSortExpr::new_default(col("value", schema.as_ref()).unwrap()); + let plan = Arc::new(SortExec::new([sort_expr].into(), data)); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let stream = break_stream(stream, 64); + let stream = stream.map_err(DataFusionError::from); + let stream = + Box::pin(RecordBatchStreamAdapter::new(schema, stream)) as SendableRecordBatchStream; + + train_btree_index(stream, test_store.as_ref(), 64, None, None) + .await + .unwrap(); + + let cache = Arc::new(LanceCache::with_capacity(100 * 1024 * 1024)); + let index = BTreeIndex::load(test_store, None, cache.as_ref()) + .await + .unwrap(); + + // First search: cold cache — the page fetch must miss. + let query = SargableQuery::Equals(ScalarValue::Float32(Some(0.0))); + let cold = LocalMetricsCollector::default(); + index.search(&query, &cold).await.unwrap(); + assert_eq!(cold.index_cache_hits(), 0); + assert_eq!(cold.index_cache_misses(), 1); + assert_eq!(cold.parts_loaded.load(Ordering::Relaxed), 1); + + // Second search: same key, page must now be served from cache. + let warm = LocalMetricsCollector::default(); + index.search(&query, &warm).await.unwrap(); + assert_eq!(warm.index_cache_hits(), 1); + assert_eq!(warm.index_cache_misses(), 0); + assert_eq!(warm.parts_loaded.load(Ordering::Relaxed), 0); + } + #[tokio::test] async fn test_like_prefix_search() { use arrow::datatypes::DataType; @@ -5049,7 +5146,10 @@ mod tests { assert_eq!(original_data, remapped_data); } + // Spill-enabled index builds share the cached DataFusion memory pool within the + // test process, so keep them in one resource group. #[tokio::test] + #[serial_test::serial(LANCE_DF_SPILL_POOL)] async fn test_update_ranged_index() { // Setup stores for both indexes let old_tmpdir = TempObjDir::default(); @@ -5200,6 +5300,7 @@ mod tests { } #[tokio::test] + #[serial_test::serial(LANCE_DF_SPILL_POOL)] async fn test_update_with_exact_row_id_filter() { let old_tmpdir = TempObjDir::default(); let old_store = Arc::new(LanceIndexStore::new( @@ -5327,12 +5428,8 @@ mod tests { mapping.insert(old_id, None); } - let mut new_id_counter = 100_000; - // Remap all other rows - for old_id in (0..1000).chain(10000..15000) { - let new_id = new_id_counter; - new_id_counter += 1; + for (new_id, old_id) in (100_000..).zip((0..1000).chain(10000..15000)) { mapping.insert(old_id, Some(new_id)); } @@ -5400,13 +5497,10 @@ mod tests { } } - /// Regression test: BTree search must track null row IDs for non-IsNull - /// queries, even when no pages match the queried value. - /// - /// Without this, `NOT(x = val)` when `val` is absent from the data would - /// produce an empty null set, causing NULL rows to incorrectly pass. + /// Regression test: BTree search skips null pages only when the caller + /// explicitly opts out of NULL tracking. #[tokio::test] - async fn test_search_tracks_nulls_for_absent_value() { + async fn test_search_null_tracking_options_for_absent_value() { use arrow_array::{Int32Array, UInt64Array}; let tmpdir = TempObjDir::default(); @@ -5457,16 +5551,26 @@ mod tests { index.page_lookup.all_null_pages.len(), ); - let metrics = NoOpMetricsCollector; + let query = SargableQuery::Equals(ScalarValue::Int32(Some(0))); - // Search for Equals(0) — value 0 doesn't exist in any page + // A top-level positive filter can discard NULL results. No BTree page + // should be read when the searched value is absent. + let metrics = LocalMetricsCollector::default(); let result = index - .search( - &SargableQuery::Equals(ScalarValue::Int32(Some(0))), + .search_with_options( + &query, + SearchOptions::default().with_track_nulls(false), &metrics, ) .await .unwrap(); + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::default())); + assert_eq!(metrics.parts_loaded.load(Ordering::Relaxed), 0); + assert_eq!(metrics.comparisons.load(Ordering::Relaxed), 0); + + // The existing search API keeps its NULL-preserving behavior for + // callers that need three-valued logic. + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); match result { SearchResult::Exact(set) => { @@ -5488,7 +5592,7 @@ mod tests { std::ops::Bound::Unbounded, std::ops::Bound::Excluded(ScalarValue::Int32(Some(50))), ), - &metrics, + &NoOpMetricsCollector, ) .await .unwrap(); @@ -5505,6 +5609,57 @@ mod tests { } } + /// Regression test: disabling NULL tracking also omits NULL rows from a + /// candidate page that contains both matching and NULL values. + #[tokio::test] + async fn test_search_without_null_tracking_on_mixed_page() { + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::memory()), + Path::default(), + Arc::new(LanceCache::no_cache()), + )); + let data = record_batch!( + ("value", Int32, [None, Some(5), Some(7)]), + ("_rowid", UInt64, [0, 1, 2]) + ) + .unwrap(); + let schema = data.schema(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(async { Ok(data) }), + )); + train_btree_index(stream, test_store.as_ref(), 3, None, None) + .await + .unwrap(); + + let index = BTreeIndex::load(test_store, None, &LanceCache::no_cache()) + .await + .unwrap(); + assert_eq!(index.page_lookup.null_pages, vec![0]); + + let query = SargableQuery::Equals(ScalarValue::Int32(Some(5))); + let result = index + .search_with_options( + &query, + SearchOptions::default().with_track_nulls(false), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let SearchResult::Exact(row_ids) = result else { + panic!("BTree search should be exact"); + }; + assert_eq!(row_ids.true_rows(), RowAddrTreeMap::from_iter([1])); + assert!(row_ids.null_rows().is_empty()); + + let tracked = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + let SearchResult::Exact(tracked) = tracked else { + panic!("BTree search should be exact"); + }; + assert_eq!(tracked.true_rows(), RowAddrTreeMap::from_iter([1])); + assert_eq!(tracked.null_rows(), &RowAddrTreeMap::from_iter([0])); + } + fn sample_lookup_batch() -> RecordBatch { record_batch!( ("min", Int32, [Some(0), Some(10), Some(20)]), @@ -6289,7 +6444,7 @@ mod tests { #[tokio::test] async fn test_btree_index_state_reconstruct_applies_frag_reuse_index() { - use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails}; + use crate::frag_reuse::{FragReuseIndex, FragReuseIndexDetails, FragReuseIndexHandle}; use std::collections::HashMap; use uuid::Uuid; @@ -6322,11 +6477,11 @@ mod tests { // Querying for value == 0 should now return row 5000, confirming reconstruct threaded // the FragReuseIndex through to the rebuilt BTreeIndex. let frag_reuse_index: Arc = - Arc::new(FragReuseIndex::new( + Arc::new(FragReuseIndexHandle(Arc::new(FragReuseIndex::new( Uuid::new_v4(), vec![HashMap::from([(0u64, Some(5000u64))])], FragReuseIndexDetails { versions: vec![] }, - )); + )))); let reconstructed = state .reconstruct( test_store.clone(), diff --git a/rust/lance-index/src/scalar/btree/flat.rs b/rust/lance-index/src/scalar/btree/flat.rs index 7eb9f0422f3..6322d3e280a 100644 --- a/rust/lance-index/src/scalar/btree/flat.rs +++ b/rust/lance-index/src/scalar/btree/flat.rs @@ -134,6 +134,14 @@ impl FlatIndex { NullableRowAddrSet::new(self.all_addrs_map.clone(), Default::default()) } + /// Return every non-null row as TRUE without preserving NULL rows. + pub fn all_non_null(&self) -> NullableRowAddrSet { + NullableRowAddrSet::new( + self.all_addrs_map.clone() - &self.null_addrs_map, + Default::default(), + ) + } + pub fn remap_batch(batch: RecordBatch, mapping: &RowAddrRemap) -> Result { let row_ids = batch.column(IDS_COL_IDX).as_primitive::(); let val_idx_and_new_id = row_ids @@ -176,6 +184,7 @@ impl FlatIndex { pub fn search( &self, query: &dyn AnyQuery, + track_nulls: bool, metrics: &dyn MetricsCollector, ) -> Result { metrics.record_comparisons(self.data.num_rows()); @@ -189,10 +198,11 @@ impl FlatIndex { SargableQuery::Equals(value) => { if value.is_null() { // if we have x = NULL then the correct SQL behavior is to return all NULLs - return Ok(NullableRowAddrSet::new( - Default::default(), - self.all_addrs_map.clone(), - )); + return Ok(if track_nulls { + NullableRowAddrSet::new(Default::default(), self.all_addrs_map.clone()) + } else { + NullableRowAddrSet::empty() + }); } } // x IS NULL we can use pre-computed nulls @@ -212,19 +222,21 @@ impl FlatIndex { } (Bound::Unbounded, Bound::Included(upper) | Bound::Excluded(upper)) => { if upper.is_null() { - return Ok(NullableRowAddrSet::new( - Default::default(), - self.all_addrs_map.clone(), - )); + return Ok(if track_nulls { + NullableRowAddrSet::new(Default::default(), self.all_addrs_map.clone()) + } else { + NullableRowAddrSet::empty() + }); } } - (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded) => { - if lower.is_null() { - return Ok(NullableRowAddrSet::new( - Default::default(), - self.all_addrs_map.clone(), - )); - } + (Bound::Included(lower) | Bound::Excluded(lower), Bound::Unbounded) + if lower.is_null() => + { + return Ok(if track_nulls { + NullableRowAddrSet::new(Default::default(), self.all_addrs_map.clone()) + } else { + NullableRowAddrSet::empty() + }); } _ => {} }, @@ -234,7 +246,7 @@ impl FlatIndex { // No shortcut possible, need to actually evaluate the query let expr = query.to_expr(BTREE_VALUES_COLUMN.to_string()); let expr = create_physical_expr(&expr, &self.df_schema, &ExecutionProps::default())?; - self.eval_expr(&expr) + self.eval_expr(&expr, track_nulls) } /// Evaluate a predicate compiled once by the caller. Lets a large IsIn that @@ -243,20 +255,24 @@ impl FlatIndex { pub fn search_prebuilt( &self, expr: &Arc, + track_nulls: bool, metrics: &dyn MetricsCollector, ) -> Result { metrics.record_comparisons(self.data.num_rows()); - self.eval_expr(expr) + self.eval_expr(expr, track_nulls) } - fn eval_expr(&self, expr: &Arc) -> Result { + fn eval_expr( + &self, + expr: &Arc, + track_nulls: bool, + ) -> Result { let predicate = expr.evaluate(&self.data)?; let predicate = predicate.into_array(self.data.num_rows())?; let predicate = predicate .as_any() .downcast_ref::() .expect("Predicate should return boolean array"); - let nulls = arrow::compute::is_null(&predicate)?; let matching_ids = arrow_select::filter::filter(self.ids(), predicate)?; let matching_ids = matching_ids @@ -265,6 +281,11 @@ impl FlatIndex { .expect("Result of arrow_select::filter::filter did not match input type"); let selected = RowAddrTreeMap::from_sorted_iter(matching_ids.values().iter().copied())?; + if !track_nulls { + return Ok(NullableRowAddrSet::new(selected, Default::default())); + } + + let nulls = arrow::compute::is_null(&predicate)?; let null_row_ids = arrow_select::filter::filter(self.ids(), &nulls)?; let null_row_ids = null_row_ids .as_any() @@ -364,7 +385,7 @@ mod tests { async fn check_index(query: &SargableQuery, expected: &[u64]) { let index = example_index(); - let actual = index.search(query, &NoOpMetricsCollector).unwrap(); + let actual = index.search(query, true, &NoOpMetricsCollector).unwrap(); let expected = NullableRowAddrSet::new(RowAddrTreeMap::from_iter(expected), Default::default()); assert_eq!(actual, expected); @@ -537,7 +558,7 @@ mod tests { let index = FlatIndex::try_new(batch).unwrap(); let check = |query: SargableQuery, true_ids: &[u64], null_ids: &[u64]| { - let actual = index.search(&query, &NoOpMetricsCollector).unwrap(); + let actual = index.search(&query, true, &NoOpMetricsCollector).unwrap(); let expected = NullableRowAddrSet::new( RowAddrTreeMap::from_iter(true_ids), RowAddrTreeMap::from_iter(null_ids), @@ -598,4 +619,80 @@ mod tests { &[0, 1, 2], ); } + + /// Row addresses pack `(fragment_id << 32) | offset`, so a page spanning + /// several fragments must report exactly those fragments, deduped and + /// sorted. Every other test in this module uses offsets inside fragment 0, + /// which never exercises the shift. + #[test] + fn test_calculate_included_frags_spans_fragments() { + let addr = |frag: u32, offset: u32| u64::from(RowAddress::new_from_parts(frag, offset)); + let batch = record_batch!( + ( + BTREE_VALUES_COLUMN, + Int32, + [Some(1), Some(2), Some(3), Some(4)] + ), + ( + BTREE_IDS_COLUMN, + UInt64, + [addr(0, 0), addr(2, 7), addr(0, 1), addr(5, 3)] + ) + ) + .unwrap(); + let index = FlatIndex::try_new(batch).unwrap(); + + assert_eq!( + index.calculate_included_frags().unwrap(), + RoaringBitmap::from_iter([0u32, 2, 5]) + ); + + // A hit still carries the full 64-bit address, not a bare offset. + let hit = index + .search( + &SargableQuery::Equals(ScalarValue::from(2)), + true, + &NoOpMetricsCollector, + ) + .unwrap(); + assert_eq!( + hit, + NullableRowAddrSet::new(RowAddrTreeMap::from_iter(&[addr(2, 7)]), Default::default()) + ); + } + + /// A zero-row page has to answer queries with empty sets rather than + /// panicking inside the Arrow predicate evaluation. The roundtrip test + /// builds an empty index but never queries one. Both `track_nulls` modes + /// take different shortcuts, and neither has any row to return here. + #[test] + fn test_empty_index_answers_queries_with_empty_sets() { + let empty = RecordBatch::new_empty(example_index().data.schema()); + let index = FlatIndex::try_new(empty).unwrap(); + let nothing = NullableRowAddrSet::new(RowAddrTreeMap::new(), RowAddrTreeMap::new()); + + for query in [ + SargableQuery::Equals(ScalarValue::from(10)), + SargableQuery::Equals(ScalarValue::Int32(None)), + SargableQuery::IsNull(), + SargableQuery::IsIn(vec![ScalarValue::from(10), ScalarValue::from(20)]), + SargableQuery::Range(Bound::Unbounded, Bound::Unbounded), + ] { + for track_nulls in [true, false] { + assert_eq!( + index + .search(&query, track_nulls, &NoOpMetricsCollector) + .unwrap(), + nothing, + "query: {query:?}, track_nulls: {track_nulls}" + ); + } + } + + assert!(index.all().true_rows().is_empty()); + assert_eq!( + index.calculate_included_frags().unwrap(), + RoaringBitmap::new() + ); + } } diff --git a/rust/lance-index/src/scalar/expression.rs b/rust/lance-index/src/scalar/expression.rs index e495d5b701f..3f171d0e008 100644 --- a/rust/lance-index/src/scalar/expression.rs +++ b/rust/lance-index/src/scalar/expression.rs @@ -1,7 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{ops::Bound, sync::Arc}; +use std::{ + cmp::Ordering, + collections::{HashMap, HashSet, hash_map::Entry}, + ops::Bound, + sync::Arc, +}; use arrow_schema::{DataType, Field}; use async_recursion::async_recursion; @@ -15,7 +20,7 @@ use tokio::try_join; use super::{ AnyQuery, BloomFilterQuery, LabelListQuery, MetricsCollector, SargableQuery, ScalarIndex, - SearchResult, TextQuery, TokenQuery, + SearchOptions, SearchResult, TextQuery, TokenQuery, label_list::validate_label_list_data_type, }; #[cfg(feature = "geo")] use super::{GeoQuery, RelationQuery}; @@ -269,6 +274,10 @@ pub struct SargableQueryParser { index_name: String, index_type: String, needs_recheck: bool, + /// Whether IS NULL queries need post-filter rechecking. May be false even + /// when `needs_recheck` is true for indexes that track exact null positions + /// (e.g. zone maps with a null bitmap). + is_null_needs_recheck: bool, supports_like_prefix: bool, } @@ -278,6 +287,7 @@ impl SargableQueryParser { index_name, index_type, needs_recheck, + is_null_needs_recheck: needs_recheck, supports_like_prefix: true, } } @@ -289,6 +299,14 @@ impl SargableQueryParser { self.supports_like_prefix = false; self } + + /// Mark IS NULL as exact so that IS NOT NULL can be served by the index + /// without a post-filter. Use this when the index tracks null row addresses + /// precisely (e.g. a zone map with a null bitmap). + pub fn with_exact_null_tracking(mut self) -> Self { + self.is_null_needs_recheck = false; + self + } } impl ScalarQueryParser for SargableQueryParser { @@ -357,7 +375,7 @@ impl ScalarQueryParser for SargableQueryParser { self.index_name.clone(), self.index_type.clone(), Arc::new(SargableQuery::IsNull()), - self.needs_recheck, + self.is_null_needs_recheck, )) } @@ -769,6 +787,11 @@ impl ScalarQueryParser for LabelListQueryParser { if args.len() != 2 { return None; } + // LABEL_LIST stores unnested items as bitmap keys. Nested items cannot be + // ordered by the bitmap lookup, so legacy indexes must fall back to a scan. + if validate_label_list_data_type(data_type).is_err() { + return None; + } // DataFusion normalizes array_contains to array_has if func.name() == "array_has" { let inner_type = match data_type { @@ -791,34 +814,34 @@ impl ScalarQueryParser for LabelListQueryParser { } let label_list = maybe_scalar(&args[1], data_type)?; - if let ScalarValue::List(list_arr) = label_list { - let list_values = list_arr.values(); - if list_values.is_empty() { - return None; - } - let mut scalars = Vec::with_capacity(list_values.len()); - for idx in 0..list_values.len() { - scalars.push(ScalarValue::try_from_array(list_values.as_ref(), idx).ok()?); - } - if func.name() == "array_has_all" { - let query = LabelListQuery::HasAllLabels(scalars); - Some(IndexedExpression::index_query( - column.to_string(), - self.index_name.clone(), - self.index_type.clone(), - Arc::new(query), - )) - } else if func.name() == "array_has_any" { - let query = LabelListQuery::HasAnyLabel(scalars); - Some(IndexedExpression::index_query( - column.to_string(), - self.index_name.clone(), - self.index_type.clone(), - Arc::new(query), - )) - } else { - None - } + let list_values = match label_list { + ScalarValue::List(list_arr) => list_arr.values().clone(), + ScalarValue::LargeList(list_arr) => list_arr.values().clone(), + _ => return None, + }; + if list_values.is_empty() { + return None; + } + let mut scalars = Vec::with_capacity(list_values.len()); + for idx in 0..list_values.len() { + scalars.push(ScalarValue::try_from_array(list_values.as_ref(), idx).ok()?); + } + if func.name() == "array_has_all" { + let query = LabelListQuery::HasAllLabels(scalars); + Some(IndexedExpression::index_query( + column.to_string(), + self.index_name.clone(), + self.index_type.clone(), + Arc::new(query), + )) + } else if func.name() == "array_has_any" { + let query = LabelListQuery::HasAnyLabel(scalars); + Some(IndexedExpression::index_query( + column.to_string(), + self.index_name.clone(), + self.index_type.clone(), + Arc::new(query), + )) } else { None } @@ -833,6 +856,10 @@ pub struct TextQueryParser { index_type: String, needs_recheck: bool, supports_regex: bool, + /// The shortest `contains` pattern (in characters, not bytes) the index can + /// say anything useful about. Shorter patterns bypass the index entirely so + /// they are answered by a full scan. Use 0 for indices with no such limit. + min_contains_chars: usize, } impl TextQueryParser { @@ -841,12 +868,14 @@ impl TextQueryParser { index_type: String, needs_recheck: bool, supports_regex: bool, + min_contains_chars: usize, ) -> Self { Self { index_name, index_type, needs_recheck, supports_regex, + min_contains_chars, } } } @@ -903,7 +932,16 @@ impl ScalarQueryParser for TextQueryParser { }; let query = match func.name() { - "contains" if args.len() == 2 => TextQuery::StringContains(pattern), + "contains" if args.len() == 2 => { + // A pattern shorter than the index's token width produces no + // tokens, so the index can only answer "recheck everything". + // Leave it to a full scan instead. The count is in characters + // because tokenizers split on characters, not bytes. + if pattern.chars().count() < self.min_contains_chars { + return None; + } + TextQuery::StringContains(pattern) + } "regexp_like" | "regexp_match" if self.supports_regex => { let pattern = match args.get(2) { Some(flags_expr) => apply_regex_flags(&pattern, flags_expr)?, @@ -1402,6 +1440,20 @@ pub trait ScalarIndexLoader: Send + Sync { index_name: &str, metrics: &dyn MetricsCollector, ) -> Result>; + + /// Translate an address-domain index result into the row-id domain + /// + /// Address-domain indices (see [`ScalarIndex::results_are_row_addresses`]) + /// report matches as physical row addresses. The default returns `result` + /// unchanged, which is correct when addresses and row ids coincide (no + /// stable row ids). A dataset with stable row ids overrides this to remap + /// addresses to stable row ids via its per-fragment row-id sequences. + async fn row_addr_result_to_row_ids( + &self, + result: NullableIndexExprResult, + ) -> Result { + Ok(result) + } } /// This represents a search into a scalar index @@ -1462,173 +1514,386 @@ impl PartialEq for ScalarIndexExpr { } } +/// Conservative grouping key for rewrites targeting the same parsed scalar index. +/// `index_type` is display metadata, but including it prevents rewrites across +/// index implementations that may not share query semantics. +type ScalarIndexQueryKey = (String, String, String); + /// Returns the tighter (more restrictive) lower bound. -/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value. -fn tighter_lower_bound(a: &Bound, b: &Bound) -> Bound { +/// +/// Returns `None` if the bound values cannot be compared. In that case callers +/// keep the original predicates rather than inventing ordering semantics. +fn tighter_lower_bound( + a: &Bound, + b: &Bound, +) -> Option> { match (a, b) { - (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(), - (Bound::Included(va), Bound::Included(vb)) => { - if va >= vb { - Bound::Included(va.clone()) - } else { - Bound::Included(vb.clone()) - } - } - (Bound::Excluded(va), Bound::Excluded(vb)) => { - if va >= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Excluded(vb.clone()) - } - } - (Bound::Excluded(va), Bound::Included(vb)) => { - if va >= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Included(vb.clone()) - } - } - (Bound::Included(va), Bound::Excluded(vb)) => { - if vb >= va { - Bound::Excluded(vb.clone()) - } else { - Bound::Included(va.clone()) - } - } + (Bound::Unbounded, Bound::Unbounded) => Some(Bound::Unbounded), + (Bound::Unbounded, other) | (other, Bound::Unbounded) => Some(other.clone()), + ( + Bound::Included(a_value) | Bound::Excluded(a_value), + Bound::Included(b_value) | Bound::Excluded(b_value), + ) => match a_value.partial_cmp(b_value)? { + Ordering::Less => Some(b.clone()), + Ordering::Equal => Some(stricter_bound_for_equal_value(a_value, a, b)), + Ordering::Greater => Some(a.clone()), + }, } } /// Returns the tighter (more restrictive) upper bound. -/// Priority: Included/Excluded > Unbounded; Excluded > Included for same value. -fn tighter_upper_bound(a: &Bound, b: &Bound) -> Bound { +/// +/// Returns `None` if the bound values cannot be compared. In that case callers +/// keep the original predicates rather than inventing ordering semantics. +fn tighter_upper_bound( + a: &Bound, + b: &Bound, +) -> Option> { match (a, b) { - (Bound::Unbounded, other) | (other, Bound::Unbounded) => other.clone(), - (Bound::Included(va), Bound::Included(vb)) => { - if va <= vb { - Bound::Included(va.clone()) - } else { - Bound::Included(vb.clone()) - } - } - (Bound::Excluded(va), Bound::Excluded(vb)) => { - if va <= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Excluded(vb.clone()) + (Bound::Unbounded, Bound::Unbounded) => Some(Bound::Unbounded), + (Bound::Unbounded, other) | (other, Bound::Unbounded) => Some(other.clone()), + ( + Bound::Included(a_value) | Bound::Excluded(a_value), + Bound::Included(b_value) | Bound::Excluded(b_value), + ) => match a_value.partial_cmp(b_value)? { + Ordering::Less => Some(a.clone()), + Ordering::Equal => Some(stricter_bound_for_equal_value(a_value, a, b)), + Ordering::Greater => Some(b.clone()), + }, + } +} + +fn is_excluded_bound(bound: &Bound) -> bool { + matches!(bound, Bound::Excluded(_)) +} + +/// For an equal scalar value, an excluded bound is stricter than an included +/// bound. This handles cases like `x >= 5 AND x > 5`. +fn stricter_bound_for_equal_value( + value: &ScalarValue, + lhs: &Bound, + rhs: &Bound, +) -> Bound { + if is_excluded_bound(lhs) || is_excluded_bound(rhs) { + Bound::Excluded(value.clone()) + } else { + Bound::Included(value.clone()) + } +} + +fn range_has_non_null_bound(lower: &Bound, upper: &Bound) -> bool { + let mut has_bound = false; + for bound in [lower, upper] { + match bound { + Bound::Included(value) | Bound::Excluded(value) => { + if value.is_null() { + return false; + } + has_bound = true; } + Bound::Unbounded => {} } - (Bound::Excluded(va), Bound::Included(vb)) => { - if va <= vb { - Bound::Excluded(va.clone()) - } else { - Bound::Included(vb.clone()) - } + } + has_bound +} + +/// Null bounds are skipped by range optimization. Comparisons with NULL should +/// already be rejected by normal parsing, but this keeps manually constructed +/// ScalarIndexExpr values from being rewritten into misleading ranges. +fn range_has_null_bound(lower: &Bound, upper: &Bound) -> bool { + [lower, upper].iter().any(|bound| match bound { + Bound::Included(value) | Bound::Excluded(value) => value.is_null(), + Bound::Unbounded => false, + }) +} + +impl ScalarIndexSearch { + /// Only SargableQuery participates in these expression-level rewrites. + /// Other AnyQuery implementations may have different null/range semantics. + fn sargable_query(&self) -> Option<&SargableQuery> { + self.query.as_any().downcast_ref::() + } + + /// Require a concrete SargableQuery before producing a key so callers do not + /// accidentally group unrelated AnyQuery implementations by metadata alone. + fn sargable_query_key(&self) -> Option { + self.sargable_query()?; + Some(( + self.column.clone(), + self.index_name.clone(), + self.index_type.clone(), + )) + } + + /// Only exact SargableQuery values may drive NULL-elimination rewrites. + /// Range merging also accepts inexact queries and preserves their recheck. + fn exact_sargable_query(&self) -> Option<&SargableQuery> { + if self.needs_recheck { + return None; } - (Bound::Included(va), Bound::Excluded(vb)) => { - if vb <= va { - Bound::Excluded(vb.clone()) - } else { - Bound::Included(va.clone()) + self.sargable_query() + } + + /// Return a scalar-index key only when the query is an exact SargableQuery. + fn exact_sargable_query_key(&self) -> Option { + self.exact_sargable_query()?; + self.sargable_query_key() + } + + /// A query is null-intolerant if the original predicate cannot match NULL. + /// Only exact queries can make `IS NOT NULL` redundant in the index expression. + fn is_null_intolerant_sargable_query(&self) -> bool { + match self.exact_sargable_query() { + Some(SargableQuery::Range(lower, upper)) => range_has_non_null_bound(lower, upper), + Some(SargableQuery::Equals(value)) => !value.is_null(), + Some(SargableQuery::IsIn(values)) => { + !values.is_empty() && values.iter().all(|value| !value.is_null()) } + _ => false, } } } impl ScalarIndexExpr { - /// Optimize the expression tree by merging range queries on the same index. + /// Apply scalar-index optimizer rules within each contiguous AND region. + /// + /// Range queries for the same parsed scalar index are intersected and retain + /// `needs_recheck` if any input range requires it. An exact `IS NOT NULL` + /// query is removed when another exact same-index query is null-intolerant. + /// OR branches are optimized independently. Range queries are still merged + /// under NOT, but `IS NOT NULL` elimination is disabled there to preserve + /// SQL NULL semantics. + /// + /// Compatible [`SargableQuery::Range`] values carried by + /// [`ScalarIndexSearch`] are merged into one query. + /// + /// # Example /// - /// This collects all leaf Range queries from the AND tree, groups them by - /// index name, merges overlapping ranges into a single closed-range query, - /// and rebuilds the tree. This handles the case where `log_time >= X` and - /// `log_time <= Y` end up in different branches of a nested AND tree. + /// ``` + /// # use std::{ops::Bound, sync::Arc}; + /// # use lance_index::scalar::{SargableQuery, expression::{ScalarIndexExpr, ScalarIndexSearch}}; + /// # let range = |lower, upper| ScalarIndexExpr::Query(ScalarIndexSearch { + /// # column: "x".into(), + /// # index_name: "x_idx".into(), + /// # index_type: "BTree".into(), + /// # query: Arc::new(SargableQuery::Range(lower, upper)), + /// # needs_recheck: false, + /// # fragment_bitmap: None, + /// # }); + /// let optimized = ScalarIndexExpr::And( + /// Box::new(range(Bound::Included(10_i64.into()), Bound::Unbounded)), + /// Box::new(range(Bound::Unbounded, Bound::Included(20_i64.into()))), + /// ) + /// .optimize(); + /// + /// assert!(matches!(optimized, ScalarIndexExpr::Query(_))); + /// ``` pub fn optimize(self) -> Self { + self.optimize_with_context(true) + } + + fn optimize_with_context(self, is_not_null_elidable: bool) -> Self { match self { - Self::And(_, _) => self.optimize_and_tree(), - Self::Or(lhs, rhs) => Self::Or(Box::new(lhs.optimize()), Box::new(rhs.optimize())), - Self::Not(inner) => Self::Not(Box::new(inner.optimize())), + Self::And(_, _) => self.optimize_and_tree(is_not_null_elidable), + Self::Or(lhs, rhs) => Self::Or( + Box::new(lhs.optimize_with_context(is_not_null_elidable)), + Box::new(rhs.optimize_with_context(is_not_null_elidable)), + ), + Self::Not(inner) => Self::Not(Box::new(inner.optimize_with_context(false))), other => other, } } - /// Flatten an AND tree, merge ranges on same index, rebuild. - fn optimize_and_tree(self) -> Self { + /// Flatten one contiguous AND region, apply local optimizer rules, and + /// rebuild a balanced AND tree. + fn optimize_and_tree(self, is_not_null_elidable: bool) -> Self { let mut leaves = Vec::new(); - self.collect_and_leaves(&mut leaves); + self.collect_and_leaves(&mut leaves, is_not_null_elidable); + let leaves = Self::optimize_and_leaves(leaves, is_not_null_elidable); + + Self::rebuild_and_tree(leaves).expect("AND tree optimization should keep at least one leaf") + } + + /// Apply optimizer rules within one AND region only. OR branches have already + /// been kept as opaque leaves, so range and null-intolerance rewrites cannot + /// cross boolean boundaries. + fn optimize_and_leaves(leaves: Vec, is_not_null_elidable: bool) -> Vec { + let null_intolerant_keys = if !is_not_null_elidable { + HashSet::new() + } else { + let is_not_null_keys = leaves + .iter() + .filter_map(Self::is_not_null_query_key) + .collect::>(); + if is_not_null_keys.is_empty() { + HashSet::new() + } else { + leaves + .iter() + .filter_map(Self::null_intolerant_query_key) + .filter(|key| is_not_null_keys.contains(key)) + .collect() + } + }; - // Try to merge Range queries on the same index - let mut merged_indices: Vec = vec![false; leaves.len()]; - let mut result_leaves: Vec = Vec::new(); + let mut optimized = Vec::with_capacity(leaves.len()); + let mut range_positions = HashMap::new(); - for i in 0..leaves.len() { - if merged_indices[i] { + for leaf in leaves { + if let Some(key) = leaf.is_not_null_query_key() + && null_intolerant_keys.contains(&key) + { continue; } - let mut current = leaves[i].clone(); - // Try to merge with subsequent leaves on the same index - for j in (i + 1)..leaves.len() { - if merged_indices[j] { - continue; - } - if let Some(merged) = try_merge_range_pair(¤t, &leaves[j]) { - current = merged; - merged_indices[j] = true; + if let Some(key) = leaf.range_query_key() { + match range_positions.entry(key) { + Entry::Vacant(entry) => { + entry.insert(optimized.len()); + optimized.push(leaf); + } + Entry::Occupied(entry) => { + if !optimized[*entry.get()].try_merge_range(&leaf) { + optimized.push(leaf); + } + } } + } else { + optimized.push(leaf); } - - result_leaves.push(current); } - // Rebuild the AND tree from remaining leaves - let mut iter = result_leaves.into_iter(); - let first = iter.next().expect("AND tree must have at least one leaf"); - iter.fold(first, |acc, leaf| Self::And(Box::new(acc), Box::new(leaf))) + optimized } - /// Recursively collect all leaf nodes from an AND tree. - fn collect_and_leaves(self, leaves: &mut Vec) { + /// Recursively collect all leaf nodes from an AND tree. OR branches are + /// optimized independently with the current null-elision context. NOT + /// branches remain leaves while their children are optimized with null + /// elimination disabled. + fn collect_and_leaves(self, leaves: &mut Vec, is_not_null_elidable: bool) { match self { Self::And(lhs, rhs) => { - lhs.collect_and_leaves(leaves); - rhs.collect_and_leaves(leaves); + lhs.collect_and_leaves(leaves, is_not_null_elidable); + rhs.collect_and_leaves(leaves, is_not_null_elidable); + } + Self::Or(lhs, rhs) => { + leaves.push(Self::Or( + Box::new(lhs.optimize_with_context(is_not_null_elidable)), + Box::new(rhs.optimize_with_context(is_not_null_elidable)), + )); + } + Self::Not(inner) => { + leaves.push(Self::Not(Box::new(inner.optimize_with_context(false)))) } other => leaves.push(other), } } -} -/// Try to merge two ScalarIndexExpr nodes if they are both Range queries on the same index. -fn try_merge_range_pair(lhs: &ScalarIndexExpr, rhs: &ScalarIndexExpr) -> Option { - let (ScalarIndexExpr::Query(l), ScalarIndexExpr::Query(r)) = (lhs, rhs) else { - return None; - }; - if l.index_name != r.index_name || l.column != r.column { - return None; + /// Rebuild as a balanced tree so large planner-generated conjunctions do not + /// become deep left-leaning trees after optimization. + fn rebuild_and_tree(leaves: Vec) -> Option { + let mut leaves = leaves; + if leaves.is_empty() { + return None; + } + + while leaves.len() > 1 { + let mut next = Vec::with_capacity(leaves.len().div_ceil(2)); + let mut iter = leaves.into_iter(); + while let Some(lhs) = iter.next() { + if let Some(rhs) = iter.next() { + next.push(Self::And(Box::new(lhs), Box::new(rhs))); + } else { + next.push(lhs); + } + } + leaves = next; + } + + leaves.pop() } - let l_query = l.query.as_any().downcast_ref::()?; - let r_query = r.query.as_any().downcast_ref::()?; + /// Detect the scalar-index representation of `col IS NOT NULL`, which is + /// stored as `NOT(col IS NULL)`. + fn is_not_null_query_key(&self) -> Option { + match self { + Self::Not(inner) => match inner.as_ref() { + Self::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) => + { + search.exact_sargable_query_key() + } + _ => None, + }, + _ => None, + } + } - let (SargableQuery::Range(l_low, l_high), SargableQuery::Range(r_low, r_high)) = - (l_query, r_query) - else { - return None; - }; + /// Return the key of a same-column predicate that makes `IS NOT NULL` + /// redundant in the same AND region. + fn null_intolerant_query_key(&self) -> Option { + match self { + Self::Query(search) if search.is_null_intolerant_sargable_query() => { + search.exact_sargable_query_key() + } + Self::Not(inner) => match inner.as_ref() { + // `NOT (predicate)` is NULL wherever the predicate is, so it + // cannot match a NULL row either, and `IS NOT NULL` adds nothing. + Self::Query(search) if search.is_null_intolerant_sargable_query() => { + search.exact_sargable_query_key() + } + _ => None, + }, + _ => None, + } + } + + /// Return the grouping key for an optimizable range query. Ranges with NULL + /// bounds are left unchanged. + fn range_query_key(&self) -> Option { + let Self::Query(search) = self else { + return None; + }; + let SargableQuery::Range(lower, upper) = search.sargable_query()? else { + return None; + }; + if range_has_null_bound(lower, upper) { + return None; + } + search.sargable_query_key() + } + + /// Merge another compatible range into this expression. + /// + /// The caller must first group both expressions by [`ScalarIndexQueryKey`]. + /// Different fragment coverage or incomparable bounds leave both predicates + /// unchanged. Empty intersections are retained as ranges. + fn try_merge_range(&mut self, other: &Self) -> bool { + let (Self::Query(search), Self::Query(other_search)) = (self, other) else { + return false; + }; + if search.fragment_bitmap != other_search.fragment_bitmap { + return false; + } + + let ( + Some(SargableQuery::Range(lower, upper)), + Some(SargableQuery::Range(other_lower, other_upper)), + ) = (search.sargable_query(), other_search.sargable_query()) + else { + return false; + }; + let Some(lower) = tighter_lower_bound(lower, other_lower) else { + return false; + }; + let Some(upper) = tighter_upper_bound(upper, other_upper) else { + return false; + }; - let merged_low = tighter_lower_bound(l_low, r_low); - let merged_high = tighter_upper_bound(l_high, r_high); - - Some(ScalarIndexExpr::Query(ScalarIndexSearch { - column: l.column.clone(), - index_name: l.index_name.clone(), - index_type: l.index_type.clone(), - query: Arc::new(SargableQuery::Range(merged_low, merged_high)), - needs_recheck: l.needs_recheck || r.needs_recheck, - // Both queries target the same index (checked above), so they share - // the same fragment coverage; carry it over to keep the merged query - // usable by coverage-dependent optimizer rules. - fragment_bitmap: l.fragment_bitmap.clone(), - })) + search.query = Arc::new(SargableQuery::Range(lower, upper)); + search.needs_recheck |= other_search.needs_recheck; + true + } } impl std::fmt::Display for ScalarIndexExpr { @@ -1648,12 +1913,16 @@ impl std::fmt::Display for ScalarIndexExpr { } } -impl From for NullableIndexExprResult { - fn from(result: SearchResult) -> Self { - match result { - SearchResult::Exact(mask) => Self::exact(NullableRowAddrMask::AllowList(mask)), - SearchResult::AtMost(mask) => Self::at_most(NullableRowAddrMask::AllowList(mask)), - SearchResult::AtLeast(mask) => Self::at_least(NullableRowAddrMask::AllowList(mask)), +fn search_result_to_nullable(result: SearchResult) -> NullableIndexExprResult { + match result { + SearchResult::Exact(mask) => { + NullableIndexExprResult::exact(NullableRowAddrMask::AllowList(mask)) + } + SearchResult::AtMost(mask) => { + NullableIndexExprResult::at_most(NullableRowAddrMask::AllowList(mask)) + } + SearchResult::AtLeast(mask) => { + NullableIndexExprResult::at_least(NullableRowAddrMask::AllowList(mask)) } } } @@ -1665,26 +1934,40 @@ impl ScalarIndexExpr { /// /// TODO: We could potentially try and be smarter about reusing loaded indices for /// any situations where the session cache has been disabled. - #[async_recursion] pub async fn evaluate_nullable( &self, index_loader: &dyn ScalarIndexLoader, metrics: &dyn MetricsCollector, + ) -> Result { + self.evaluate_with_options(index_loader, metrics, true) + .await + } + + #[async_recursion] + async fn evaluate_with_options( + &self, + index_loader: &dyn ScalarIndexLoader, + metrics: &dyn MetricsCollector, + track_nulls: bool, ) -> Result { match self { Self::Not(inner) => { - let result = inner.evaluate_nullable(index_loader, metrics).await?; + // NOT needs the child's NULL rows to preserve SQL three-valued + // logic. Once enabled, keep tracking through the whole subtree. + let result = inner + .evaluate_with_options(index_loader, metrics, true) + .await?; Ok(!result) } Self::And(lhs, rhs) => { - let lhs_result = lhs.evaluate_nullable(index_loader, metrics); - let rhs_result = rhs.evaluate_nullable(index_loader, metrics); + let lhs_result = lhs.evaluate_with_options(index_loader, metrics, track_nulls); + let rhs_result = rhs.evaluate_with_options(index_loader, metrics, track_nulls); let (lhs_result, rhs_result) = try_join!(lhs_result, rhs_result)?; Ok(lhs_result & rhs_result) } Self::Or(lhs, rhs) => { - let lhs_result = lhs.evaluate_nullable(index_loader, metrics); - let rhs_result = rhs.evaluate_nullable(index_loader, metrics); + let lhs_result = lhs.evaluate_with_options(index_loader, metrics, track_nulls); + let rhs_result = rhs.evaluate_with_options(index_loader, metrics, track_nulls); let (lhs_result, rhs_result) = try_join!(lhs_result, rhs_result)?; Ok(lhs_result | rhs_result) } @@ -1692,8 +1975,22 @@ impl ScalarIndexExpr { let index = index_loader .load_index(&search.column, &search.index_name, metrics) .await?; - let search_result = index.search(search.query.as_ref(), metrics).await?; - Ok(search_result.into()) + let search_result = index + .search_with_options( + search.query.as_ref(), + SearchOptions::default().with_track_nulls(track_nulls), + metrics, + ) + .await?; + let result = search_result_to_nullable(search_result); + if index.results_are_row_addresses() { + // Translate address-domain results to the row-id domain + // before combining or scanning; otherwise stable-row-id + // datasets silently drop matches (issue #7434). + index_loader.row_addr_result_to_row_ids(result).await + } else { + Ok(result) + } } } } @@ -1705,7 +2002,7 @@ impl ScalarIndexExpr { metrics: &dyn MetricsCollector, ) -> Result { Ok(self - .evaluate_nullable(index_loader, metrics) + .evaluate_with_options(index_loader, metrics, false) .await? .drop_nulls()) } @@ -1744,27 +2041,30 @@ fn maybe_column(expr: &Expr) -> Option<&str> { } } -// Extract the full nested column path from a get_field expression chain -// For example: get_field(get_field(metadata, "status"), "code") -> "metadata.status.code" +// Extract the full nested column path from chained or variadic get_field expressions. +// For example, both get_field(get_field(metadata, "status"), "code") and +// get_field(metadata, "status", "code") produce "metadata.status.code". fn extract_nested_column_path(expr: &Expr) -> Option { let mut current_expr = expr; let mut parts = Vec::new(); - // Walk up the get_field chain + // Walk up the get_field chain. Add variadic arguments in reverse because all + // parts are reversed after reaching the base column. loop { match current_expr { Expr::ScalarFunction(udf) if udf.name() == "get_field" => { - if udf.args.len() != 2 { + if udf.args.len() < 2 { return None; } - // Extract the field name from the second argument - // The Literal now has two fields: ScalarValue and Option - if let Expr::Literal(ScalarValue::Utf8(Some(field_name)), _) = &udf.args[1] { - parts.push(field_name.clone()); - } else { - return None; + + for field_expr in udf.args[1..].iter().rev() { + if let Expr::Literal(ScalarValue::Utf8(Some(field_name)), _) = field_expr { + parts.push(field_name.clone()); + } else { + return None; + } } - // Move up to the parent expression + current_expr = &udf.args[0]; } Expr::Column(col) => { @@ -1838,7 +2138,7 @@ fn maybe_scalar(expr: &Expr, expected_type: &DataType) -> Option { // In this case we need to extract the value, apply the cast, and then test the casted value Expr::Cast(cast) => match cast.expr.as_ref() { Expr::Literal(value, _) => { - let casted = value.cast_to(&cast.data_type).ok()?; + let casted = value.cast_to(cast.field.data_type()).ok()?; safe_coerce_scalar(&casted, expected_type) } _ => None, @@ -2352,13 +2652,14 @@ mod tests { use std::collections::HashMap; use arrow_array::Array; - use arrow_schema::{Field, Schema}; + use arrow_schema::{Field, Fields, Schema}; use chrono::Utc; use datafusion_common::{Column, DFSchema}; use datafusion_expr::simplify::SimplifyContext; use lance_datafusion::exec::{LanceExecutionOptions, get_session_context}; use lance_select::result::IndexExprResultWireFormat; use roaring::RoaringBitmap; + use rstest::rstest; use crate::scalar::json::{JsonQuery, JsonQueryParser}; @@ -2420,15 +2721,26 @@ mod tests { Field::new("price", DataType::Float32, false), Field::new("json", DataType::LargeBinary, false), ]); + check_with_schema(index_info, expr, expected, optimize, schema); + } + + fn check_with_schema( + index_info: &dyn IndexInformationProvider, + expr: &str, + expected: Option, + optimize: bool, + schema: Schema, + ) { let df_schema: DFSchema = schema.try_into().unwrap(); let ctx = get_session_context(&LanceExecutionOptions::default()); let state = ctx.state(); let mut expr = state.create_logical_expr(expr, &df_schema).unwrap(); if optimize { - let simplify_context = SimplifyContext::default() + let simplify_context = SimplifyContext::builder() .with_schema(Arc::new(df_schema)) - .with_query_execution_start_time(Some(Utc::now())); + .with_query_execution_start_time(Some(Utc::now())) + .build(); let simplifier = datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context); expr = simplifier.simplify(expr).unwrap(); @@ -2508,6 +2820,117 @@ mod tests { ) } + #[rstest] + #[case::list(DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))))] + #[case::large_list(DataType::LargeList(Arc::new(Field::new("item", DataType::Utf8, true))))] + fn test_label_list_query_parser(#[case] label_type: DataType) { + let index_info = MockIndexInfoProvider::new(vec![( + "labels", + ColInfo::new( + label_type.clone(), + Box::new(LabelListQueryParser::new( + "labels_idx".to_string(), + "LabelList".to_string(), + )), + ), + )]); + let schema = Schema::new(vec![Field::new("labels", label_type, true)]); + + check_with_schema( + &index_info, + "array_has_any(labels, ['distributed'])", + Some(IndexedExpression::index_query( + "labels".to_string(), + "labels_idx".to_string(), + "LabelList".to_string(), + Arc::new(LabelListQuery::HasAnyLabel(vec![ScalarValue::Utf8(Some( + "distributed".to_string(), + ))])), + )), + true, + schema, + ); + } + + #[rstest] + #[case::list(DataType::List(Arc::new(Field::new( + "item", + DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + true, + ))))] + #[case::large_list(DataType::LargeList(Arc::new(Field::new( + "item", + DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + true, + ))))] + fn test_label_list_nested_item(#[case] label_type: DataType) { + let index_info = MockIndexInfoProvider::new(vec![( + "labels", + ColInfo::new( + label_type.clone(), + Box::new(LabelListQueryParser::new( + "labels_idx".to_string(), + "LabelList".to_string(), + )), + ), + )]); + let schema = Schema::new(vec![Field::new("labels", label_type, true)]); + + // Nested item types must not produce a scalar index query. + check_with_schema( + &index_info, + "array_has_any(labels, [[1]])", + None, + true, + schema, + ); + } + + #[test] + fn test_nested_column_index() { + let column = "user_profile.basic.age"; + let index_info = MockIndexInfoProvider::new(vec![( + column, + ColInfo::new( + DataType::Int32, + Box::new(SargableQueryParser::new( + format!("{column}_idx"), + "BTree".to_string(), + false, + )), + ), + )]); + let schema = Schema::new(vec![Field::new( + "user_profile", + DataType::Struct(Fields::from(vec![Field::new( + "basic", + DataType::Struct(Fields::from(vec![Field::new("age", DataType::Int32, true)])), + true, + )])), + true, + )]); + + let planner = Planner::new(Arc::new(schema)); + let filter = planner + .parse_filter("user_profile.basic.age > 900") + .unwrap(); + let plan = planner + .create_filter_plan(filter, &index_info, true) + .unwrap(); + let expected = IndexedExpression::index_query( + column.to_string(), + format!("{column}_idx"), + "BTree".to_string(), + Arc::new(SargableQuery::Range( + Bound::Excluded(ScalarValue::Int32(Some(900))), + Bound::Unbounded, + )), + ); + + assert_eq!(plan.index_query, expected.scalar_query); + assert!(plan.refine_expr.is_none()); + } + #[test] fn test_expressions() { let index_info = MockIndexInfoProvider::new(vec![ @@ -3302,9 +3725,10 @@ mod tests { .unwrap(); // Apply DataFusion simplification (this may convert starts_with to LIKE) - let simplify_context = SimplifyContext::default() + let simplify_context = SimplifyContext::builder() .with_schema(Arc::new(df_schema)) - .with_query_execution_start_time(Some(Utc::now())); + .with_query_execution_start_time(Some(Utc::now())) + .build(); let simplifier = datafusion::optimizer::simplify_expressions::ExprSimplifier::new(simplify_context); let simplified_expr = simplifier.simplify(expr).unwrap(); @@ -3871,7 +4295,7 @@ mod tests { // and the other queries as separate leaves // Let's verify by collecting leaves let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); // Should have 3 leaves: fqdn, merged_time, channel assert_eq!( @@ -3943,7 +4367,7 @@ mod tests { // Should remain as two separate leaves (not merged) let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 2); } @@ -3980,7 +4404,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 2, "Equals + Range should not merge"); } @@ -4020,7 +4444,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { @@ -4099,7 +4523,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); // Should have 2 leaves: OR node (preserved) + merged range assert_eq!(leaves.len(), 2); @@ -4129,52 +4553,695 @@ mod tests { } #[test] - fn test_optimize_needs_recheck_preserved() { - use super::{ScalarIndexExpr, ScalarIndexSearch}; - use crate::scalar::SargableQuery; - use datafusion_common::ScalarValue; - use std::ops::Bound; - use std::sync::Arc; + fn test_optimize_respects_fragment_coverage_when_merging_ranges() { + let fragments = RoaringBitmap::from_iter([1, 3, 5]); + let lower = test_scalar_range_with_metadata( + Bound::Included(ScalarValue::Int64(Some(1))), + Bound::Unbounded, + true, + Some(fragments.clone()), + ); + let upper = test_scalar_range_with_metadata( + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(99))), + false, + Some(fragments.clone()), + ); - // If either range has needs_recheck=true, merged result should too - let range_a = ScalarIndexExpr::Query(ScalarIndexSearch { - column: "x".to_string(), - index_name: "idx_x".to_string(), - index_type: "".to_string(), - query: Arc::new(SargableQuery::Range( - Bound::Included(ScalarValue::Int64(Some(1))), + let leaves = collect_test_and_leaves(test_and_terms(vec![lower.clone(), upper]).optimize()); + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if search.needs_recheck + && search.fragment_bitmap.as_ref() == Some(&fragments) + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(1))), + Bound::Included(ScalarValue::Int64(Some(99))), + )) + ) + )); + + let upper_without_coverage = test_scalar_range_with_metadata( + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(99))), + false, + None, + ); + for (lhs, rhs) in [ + (lower.clone(), upper_without_coverage.clone()), + (upper_without_coverage, lower), + ] { + let optimized = ScalarIndexExpr::And(Box::new(lhs), Box::new(rhs)).optimize(); + let leaves = collect_test_and_leaves(optimized); + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!(leaf, ScalarIndexExpr::Query(search) + if search.needs_recheck + && search.fragment_bitmap.as_ref() == Some(&fragments)) + })); + assert!(leaves.iter().any(|leaf| { + matches!(leaf, ScalarIndexExpr::Query(search) + if !search.needs_recheck && search.fragment_bitmap.is_none()) + })); + } + } + + #[test] + fn test_optimize_merges_recheck_ranges_into_empty_range() { + let lower = test_scalar_query_with_recheck( + "x", + "idx_x", + SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(200))), Bound::Unbounded, - )), + ), + ); + let upper = test_scalar_query_with_recheck( + "x", + "idx_x", + SargableQuery::Range( + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(100))), + ), + ); + + let leaves = collect_test_and_leaves(test_and_terms(vec![lower, upper]).optimize()); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if search.needs_recheck + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(200))), + Bound::Included(ScalarValue::Int64(Some(100))), + )) + ) + )); + } + + fn test_scalar_query(column: &str, index_name: &str, query: SargableQuery) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: column.to_string(), + index_name: index_name.to_string(), + index_type: "BTree".to_string(), + query: Arc::new(query), + needs_recheck: false, + fragment_bitmap: None, + }) + } + + fn test_scalar_query_with_recheck( + column: &str, + index_name: &str, + query: SargableQuery, + ) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: column.to_string(), + index_name: index_name.to_string(), + index_type: "ZoneMap".to_string(), + query: Arc::new(query), needs_recheck: true, fragment_bitmap: None, - }); - let range_b = ScalarIndexExpr::Query(ScalarIndexSearch { + }) + } + + fn test_scalar_range( + column: &str, + index_name: &str, + lower: Bound, + upper: Bound, + ) -> ScalarIndexExpr { + test_scalar_query(column, index_name, SargableQuery::Range(lower, upper)) + } + + fn test_scalar_range_with_metadata( + lower: Bound, + upper: Bound, + needs_recheck: bool, + fragment_bitmap: Option, + ) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { column: "x".to_string(), index_name: "idx_x".to_string(), - index_type: "".to_string(), - query: Arc::new(SargableQuery::Range( - Bound::Unbounded, - Bound::Included(ScalarValue::Int64(Some(99))), - )), - needs_recheck: false, - fragment_bitmap: None, - }); + index_type: "ZoneMap".to_string(), + query: Arc::new(SargableQuery::Range(lower, upper)), + needs_recheck, + fragment_bitmap, + }) + } - let expr = ScalarIndexExpr::And(Box::new(range_a), Box::new(range_b)); - let optimized = expr.optimize(); + fn test_and_terms(mut terms: Vec) -> ScalarIndexExpr { + assert!(!terms.is_empty()); + while terms.len() > 1 { + let mut next = Vec::with_capacity(terms.len().div_ceil(2)); + let mut iter = terms.into_iter(); + while let Some(lhs) = iter.next() { + if let Some(rhs) = iter.next() { + next.push(ScalarIndexExpr::And(Box::new(lhs), Box::new(rhs))); + } else { + next.push(lhs); + } + } + terms = next; + } + terms.pop().unwrap() + } + fn collect_test_and_leaves(expr: ScalarIndexExpr) -> Vec { let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); - assert_eq!(leaves.len(), 1); + expr.collect_and_leaves(&mut leaves, true); + leaves + } - if let ScalarIndexExpr::Query(s) = &leaves[0] { - assert!( - s.needs_recheck, - "Merged query should preserve needs_recheck=true" - ); - } else { - panic!("Expected a Query leaf"); + fn test_and_depth(expr: &ScalarIndexExpr) -> usize { + match expr { + ScalarIndexExpr::And(lhs, rhs) => 1 + test_and_depth(lhs).max(test_and_depth(rhs)), + ScalarIndexExpr::Or(lhs, rhs) => test_and_depth(lhs).max(test_and_depth(rhs)), + ScalarIndexExpr::Not(inner) => test_and_depth(inner), + ScalarIndexExpr::Query(_) => 0, + } + } + + fn balanced_depth_bound(mut term_count: usize) -> usize { + let mut depth = 0; + while term_count > 1 { + term_count = term_count.div_ceil(2); + depth += 1; } + depth + } + + fn int64_index_info(index_type: &str, needs_recheck: bool) -> MockIndexInfoProvider { + let parser = |column: &str| { + ColInfo::new( + DataType::Int64, + Box::new(SargableQueryParser::new( + format!("{}_idx", column), + index_type.to_string(), + needs_recheck, + )), + ) + }; + MockIndexInfoProvider::new(vec![("x", parser("x")), ("y", parser("y"))]) + } + + fn parse_int64_filter( + expr: &str, + index_info: &dyn IndexInformationProvider, + ) -> IndexedExpression { + let schema = Schema::new(vec![ + Field::new("x", DataType::Int64, true), + Field::new("y", DataType::Int64, true), + ]); + let df_schema: DFSchema = schema.try_into().unwrap(); + let ctx = get_session_context(&LanceExecutionOptions::default()); + let state = ctx.state(); + let expr = state.create_logical_expr(expr, &df_schema).unwrap(); + apply_scalar_indices(expr, index_info).unwrap() + } + + fn optimize_parsed_scalar_filter( + expr: &str, + index_info: &dyn IndexInformationProvider, + ) -> Vec { + let indexed = parse_int64_filter(expr, index_info); + collect_test_and_leaves(indexed.scalar_query.unwrap().optimize()) + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_for_recheck_range() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let mut range = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let ScalarIndexExpr::Query(search) = &mut range else { + panic!("expected a range query"); + }; + search.needs_recheck = true; + + let leaves = collect_test_and_leaves(test_and_terms(vec![is_not_null, range]).optimize()); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) + && !search.needs_recheck) + ) + })); + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_across_or() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let range = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let other = test_scalar_query( + "y", + "idx_y", + SargableQuery::Equals(ScalarValue::Int64(Some(1))), + ); + let disjunction = ScalarIndexExpr::Or(Box::new(range), Box::new(other)); + + let leaves = collect_test_and_leaves( + ScalarIndexExpr::And(Box::new(is_not_null), Box::new(disjunction)).optimize(), + ); + + assert_eq!(leaves.len(), 2); + assert!( + leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_))) + ); + assert!( + leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Or(_, _))) + ); + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_for_different_index() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let range = test_scalar_range( + "x", + "idx_x_other", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + + let leaves = collect_test_and_leaves( + ScalarIndexExpr::And(Box::new(is_not_null), Box::new(range)).optimize(), + ); + + assert_eq!(leaves.len(), 2); + assert!( + leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_))) + ); + } + + #[test] + fn test_optimize_does_not_merge_different_index_types() { + let range = |index_type: &str, lower, upper| { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: "x".to_string(), + index_name: "idx_x".to_string(), + index_type: index_type.to_string(), + query: Arc::new(SargableQuery::Range(lower, upper)), + needs_recheck: false, + fragment_bitmap: None, + }) + }; + let btree = range( + "BTree", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let zone_map = range( + "ZoneMap", + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(20))), + ); + + let leaves = collect_test_and_leaves(test_and_terms(vec![btree, zone_map]).optimize()); + + assert_eq!(leaves.len(), 2); + } + + #[test] + fn test_optimize_parser_exact_removes_is_not_null_and_merges_ranges() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter( + "x IS NOT NULL AND y = 1 AND x >= 10 AND x <= 20", + &index_info, + ); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "x" + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Included(ScalarValue::Int64(Some(20))), + )) + ) + ) + })); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "y" + && matches!( + search.sargable_query(), + Some(SargableQuery::Equals(ScalarValue::Int64(Some(1)))) + ) + ) + })); + assert!( + !leaves + .iter() + .any(|leaf| matches!(leaf, ScalarIndexExpr::Not(_))) + ); + } + + #[test] + fn test_optimize_parser_merges_ranges_for_multiple_columns() { + let index_info = int64_index_info("BTree", false); + + let leaves = + optimize_parsed_scalar_filter("x > 10 AND x < 20 AND y > 30 AND y < 40", &index_info); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "x" + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Excluded(ScalarValue::Int64(Some(10))), + Bound::Excluded(ScalarValue::Int64(Some(20))), + )) + ) + ) + })); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "y" + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Excluded(ScalarValue::Int64(Some(30))), + Bound::Excluded(ScalarValue::Int64(Some(40))), + )) + ) + ) + })); + } + + #[test] + fn test_optimize_parser_preserves_standalone_null_checks() { + let index_info = int64_index_info("BTree", false); + + let is_not_null = optimize_parsed_scalar_filter("x IS NOT NULL", &index_info); + assert_eq!(is_not_null.len(), 1); + assert!(matches!(&is_not_null[0], ScalarIndexExpr::Not(_))); + + let is_null = optimize_parsed_scalar_filter("x IS NULL", &index_info); + assert_eq!(is_null.len(), 1); + assert!(matches!( + &is_null[0], + ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull())) + )); + } + + #[test] + fn test_optimize_parser_does_not_remove_is_not_null_for_different_column() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND y >= 10", &index_info); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if search.column == "x" + && matches!(search.sargable_query(), Some(SargableQuery::IsNull()))) + ) + })); + } + + #[test] + fn test_optimize_parser_handles_in_list_null_semantics() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND x IN (1, 2)", &index_info); + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsIn(values)) if values.len() == 2) + )); + + let indexed = parse_int64_filter("x IS NOT NULL AND x IN (1, NULL)", &index_info); + assert!(indexed.refine_expr.is_some()); + let leaves = collect_test_and_leaves(indexed.scalar_query.unwrap().optimize()); + assert_eq!(leaves.len(), 1); + assert!(matches!(&leaves[0], ScalarIndexExpr::Not(_))); + } + + #[test] + fn test_optimize_parser_removes_is_not_null_from_not_equal() { + let index_info = int64_index_info("BTree", false); + + let leaves = optimize_parsed_scalar_filter("x IS NOT NULL AND x != 5", &index_info); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::Equals(value)) + if *value == ScalarValue::Int64(Some(5)))) + )); + } + + #[test] + fn test_optimize_parser_removes_is_not_null_from_not_in_list() { + let index_info = int64_index_info("BTree", false); + + // The signed-zero rewrite turns `x != 0.0` into this shape, and it is just + // as null-intolerant as `x != 5`. + let leaves = + optimize_parsed_scalar_filter("x IS NOT NULL AND x NOT IN (1, 2)", &index_info); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsIn(values)) + if values.len() == 2)) + )); + } + + #[test] + fn test_optimize_parser_merges_recheck_ranges() { + let index_info = int64_index_info("ZoneMap", true); + + let leaves = optimize_parsed_scalar_filter("x >= 10 AND y = 1 AND x <= 20", &index_info); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Query(search) + if search.column == "x" + && search.needs_recheck + && matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Included(ScalarValue::Int64(Some(20))), + )) + ) + ) + })); + } + + #[test] + fn test_optimize_parser_keeps_recheck_is_not_null_as_refine() { + let index_info = int64_index_info("ZoneMap", true); + + let indexed = parse_int64_filter("x IS NOT NULL AND x >= 10", &index_info); + + assert!(indexed.scalar_query.is_some()); + assert!(matches!( + indexed.refine_expr.as_ref(), + Some(Expr::IsNotNull(expr)) + if matches!(expr.as_ref(), Expr::Column(column) if column.name == "x") + )); + let leaves = collect_test_and_leaves(indexed.scalar_query.unwrap().optimize()); + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if search.needs_recheck + && matches!(search.sargable_query(), Some(SargableQuery::Range(_, _))) + )); + } + + #[test] + fn test_optimize_preserves_balanced_depth_for_unmerged_terms() { + let term_count = 2048; + let terms = (0..term_count) + .map(|value| { + test_scalar_query( + "x", + "idx_x", + SargableQuery::Equals(ScalarValue::Int64(Some(value))), + ) + }) + .collect::>(); + + let optimized = test_and_terms(terms).optimize(); + + assert_eq!( + collect_test_and_leaves(optimized.clone()).len(), + term_count as usize + ); + assert!( + test_and_depth(&optimized) <= balanced_depth_bound(term_count as usize), + "optimized AND depth should stay balanced, got {} for {} terms", + test_and_depth(&optimized), + term_count + ); + } + + #[test] + fn test_optimize_merges_ranges_inside_not() { + let lower = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let upper = test_scalar_range( + "x", + "idx_x", + Bound::Unbounded, + Bound::Included(ScalarValue::Int64(Some(20))), + ); + let sibling = test_scalar_query( + "y", + "idx_y", + SargableQuery::Equals(ScalarValue::Int64(Some(1))), + ); + + let nested_not = ScalarIndexExpr::Not(Box::new(test_and_terms(vec![lower, upper]))); + let optimized = ScalarIndexExpr::And(Box::new(nested_not), Box::new(sibling)).optimize(); + + let ScalarIndexExpr::And(nested_not, _) = optimized else { + panic!("expected outer AND expression"); + }; + let ScalarIndexExpr::Not(inner) = *nested_not else { + panic!("expected NOT expression"); + }; + let leaves = collect_test_and_leaves(*inner); + + assert_eq!(leaves.len(), 1); + assert!(matches!( + &leaves[0], + ScalarIndexExpr::Query(search) + if matches!( + search.sargable_query(), + Some(SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Included(ScalarValue::Int64(Some(20))), + )) + ) + )); + } + + #[test] + fn test_optimize_does_not_remove_is_not_null_inside_not() { + let is_not_null = ScalarIndexExpr::Not(Box::new(test_scalar_query( + "x", + "idx_x", + SargableQuery::IsNull(), + ))); + let range = test_scalar_range( + "x", + "idx_x", + Bound::Included(ScalarValue::Int64(Some(10))), + Bound::Unbounded, + ); + let guarded_range = test_and_terms(vec![is_not_null, range]); + let alternative = test_scalar_query( + "y", + "idx_y", + SargableQuery::Equals(ScalarValue::Int64(Some(1))), + ); + let or = ScalarIndexExpr::Or(Box::new(guarded_range), Box::new(alternative)); + let sibling = test_scalar_query( + "z", + "idx_z", + SargableQuery::Equals(ScalarValue::Int64(Some(2))), + ); + let outer_sibling = test_scalar_query( + "w", + "idx_w", + SargableQuery::Equals(ScalarValue::Int64(Some(3))), + ); + + let nested_not = ScalarIndexExpr::Not(Box::new(test_and_terms(vec![or, sibling]))); + let optimized = + ScalarIndexExpr::And(Box::new(nested_not), Box::new(outer_sibling)).optimize(); + + let ScalarIndexExpr::And(nested_not, _) = optimized else { + panic!("expected outer AND expression"); + }; + let ScalarIndexExpr::Not(inner) = *nested_not else { + panic!("expected NOT expression"); + }; + let ScalarIndexExpr::And(or, _) = *inner else { + panic!("expected AND expression"); + }; + let ScalarIndexExpr::Or(lhs, _) = *or else { + panic!("expected OR expression"); + }; + let leaves = collect_test_and_leaves(*lhs); + + assert_eq!(leaves.len(), 2); + assert!(leaves.iter().any(|leaf| { + matches!( + leaf, + ScalarIndexExpr::Not(inner) + if matches!(inner.as_ref(), ScalarIndexExpr::Query(search) + if matches!(search.sargable_query(), Some(SargableQuery::IsNull()))) + ) + })); } #[test] @@ -4268,7 +5335,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1, "All 4 ranges should merge into 1"); if let ScalarIndexExpr::Query(s) = &leaves[0] { @@ -4305,7 +5372,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!( leaves.len(), 2, @@ -4353,7 +5420,7 @@ mod tests { if let ScalarIndexExpr::Or(lhs, rhs) = &optimized { // Each branch should be a single merged range let mut left_leaves = Vec::new(); - lhs.clone().collect_and_leaves(&mut left_leaves); + lhs.clone().collect_and_leaves(&mut left_leaves, true); assert_eq!( left_leaves.len(), 1, @@ -4371,7 +5438,7 @@ mod tests { } let mut right_leaves = Vec::new(); - rhs.clone().collect_and_leaves(&mut right_leaves); + rhs.clone().collect_and_leaves(&mut right_leaves, true); assert_eq!( right_leaves.len(), 1, @@ -4429,7 +5496,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!( leaves.len(), 2, @@ -4502,7 +5569,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 3, "fqdn + merged_time + channel = 3 leaves"); let time_leaf = leaves @@ -4541,7 +5608,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { let range = s.query.as_any().downcast_ref::().unwrap(); @@ -4575,7 +5642,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { let range = s.query.as_any().downcast_ref::().unwrap(); @@ -4610,7 +5677,7 @@ mod tests { let optimized = expr.optimize(); let mut leaves = Vec::new(); - optimized.collect_and_leaves(&mut leaves); + optimized.collect_and_leaves(&mut leaves, true); assert_eq!(leaves.len(), 1); if let ScalarIndexExpr::Query(s) = &leaves[0] { let range = s.query.as_any().downcast_ref::().unwrap(); diff --git a/rust/lance-index/src/scalar/fmindex.rs b/rust/lance-index/src/scalar/fmindex.rs index 7b87e492ec2..aa07e550b52 100644 --- a/rust/lance-index/src/scalar/fmindex.rs +++ b/rust/lance-index/src/scalar/fmindex.rs @@ -23,15 +23,16 @@ use std::cmp::Reverse; use std::collections::{BinaryHeap, HashMap}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, OnceLock}; +use std::sync::{Arc, LazyLock, OnceLock}; use arrow_array::RecordBatch; -use arrow_schema::{DataType, Field}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; use futures::{StreamExt, TryStreamExt}; use lance_core::cache::LanceCache; use lance_core::deepsize::DeepSizeOf; +use lance_core::utils::parse::str_is_truthy; use lance_core::utils::row_addr_remap::RowAddrRemap; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; use lance_core::{Error, ROW_ADDR, Result}; @@ -50,6 +51,17 @@ use crate::scalar::{ }; use crate::{Index, IndexType}; +/// Schema of one FM-index block batch. +static BLOCK_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![ + Field::new("node_id", DataType::UInt32, false), + Field::new("block_id", DataType::UInt32, false), + Field::new("words", DataType::LargeBinary, false), + Field::new("prefix_rank", DataType::UInt64, false), + Field::new("bit_len", DataType::UInt64, false), + ])) +}); + const FMINDEX_INDEX_VERSION: u32 = 10; const BLOCK_WORDS: usize = 4096; const PARTITION_SIZE: usize = 10_000; @@ -90,12 +102,7 @@ static LANCE_FMINDEX_WRITE_QUEUE_SIZE: std::sync::LazyLock = static LANCE_FMINDEX_RESUME_EXISTING_PARTITIONS: std::sync::LazyLock = std::sync::LazyLock::new(|| { std::env::var("LANCE_FMINDEX_RESUME_EXISTING_PARTITIONS") - .map(|value| { - matches!( - value.as_str(), - "1" | "true" | "TRUE" | "True" | "yes" | "YES" - ) - }) + .map(|value| str_is_truthy(&value)) .unwrap_or(false) }); static LANCE_FMINDEX_PREWARM_CHUNK_BYTES: std::sync::LazyLock = @@ -178,7 +185,6 @@ struct RankBitVec { len: usize, } -#[allow(dead_code)] impl RankBitVec { fn new(len: usize) -> Self { Self { @@ -193,11 +199,6 @@ impl RankBitVec { self.words[pos / 64] |= 1u64 << (pos % 64); } - #[inline] - fn get(&self, pos: usize) -> bool { - (self.words[pos / 64] >> (pos % 64)) & 1 != 0 - } - fn build_rank_index(&mut self) { let num_sb = self.words.len().div_ceil(WORDS_PER_SUPERBLOCK) + 1; self.superblocks = Vec::with_capacity(num_sb); @@ -211,29 +212,6 @@ impl RankBitVec { self.superblocks.push(cum); } - #[inline] - fn rank1(&self, pos: usize) -> usize { - if pos == 0 { - return 0; - } - let word_idx = pos / 64; - let bit_idx = pos % 64; - let sb_idx = word_idx / WORDS_PER_SUPERBLOCK; - let mut count = self.superblocks[sb_idx] as usize; - for i in (sb_idx * WORDS_PER_SUPERBLOCK)..word_idx { - count += self.words[i].count_ones() as usize; - } - if bit_idx > 0 { - count += (self.words[word_idx] & ((1u64 << bit_idx) - 1)).count_ones() as usize; - } - count - } - - #[inline] - fn rank0(&self, pos: usize) -> usize { - pos - self.rank1(pos) - } - fn deep_size(&self) -> usize { self.words.len() * 8 + self.superblocks.len() * 4 } @@ -285,7 +263,6 @@ impl Ord for HuffNode { } } -#[allow(dead_code)] impl HuffmanWaveletTree { fn build(data: &[u8]) -> Self { let n = data.len(); @@ -403,73 +380,6 @@ impl HuffmanWaveletTree { } } - /// Retrieve the byte at position `pos` in the original BWT. - #[inline] - fn access(&self, mut pos: usize) -> u8 { - if self.nodes.is_empty() { - return 0; - } - let mut node_idx = 0; - loop { - let bit = self.nodes[node_idx].get(pos); - let (ref left, ref right) = self.children[node_idx]; - if bit { - pos = self.nodes[node_idx].rank1(pos); - match right { - WaveletChild::Leaf(b) => return *b, - WaveletChild::Node(next) => node_idx = *next, - } - } else { - pos = self.nodes[node_idx].rank0(pos); - match left { - WaveletChild::Leaf(b) => return *b, - WaveletChild::Node(next) => node_idx = *next, - } - } - } - } - - /// Count occurrences of byte `c` in positions `[0, pos)`. - #[inline] - fn rank(&self, c: u8, pos: usize) -> usize { - let code = &self.codes[c as usize]; - if code.length == 0 { - return 0; - } - let (mut lo, mut hi) = (0, pos); - for (level, &nid) in code.node_path.iter().enumerate() { - if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 { - lo = self.nodes[nid].rank0(lo); - hi = self.nodes[nid].rank0(hi); - } else { - lo = self.nodes[nid].rank1(lo); - hi = self.nodes[nid].rank1(hi); - } - } - hi - lo - } - - #[inline] - fn rank_pair(&self, c: u8, lo: usize, hi: usize) -> (usize, usize) { - let code = &self.codes[c as usize]; - if code.length == 0 { - return (0, 0); - } - let (mut s, mut l, mut h) = (0, lo, hi); - for (level, &nid) in code.node_path.iter().enumerate() { - if (code.bits >> (code.length - 1 - level as u8)) & 1 == 0 { - s = self.nodes[nid].rank0(s); - l = self.nodes[nid].rank0(l); - h = self.nodes[nid].rank0(h); - } else { - s = self.nodes[nid].rank1(s); - l = self.nodes[nid].rank1(l); - h = self.nodes[nid].rank1(h); - } - } - (l - s, h - s) - } - fn deep_size(&self) -> usize { self.nodes.iter().map(|n| n.deep_size()).sum::() + self @@ -1008,7 +918,6 @@ impl DeepSizeOf for FMIndex { } } -#[allow(dead_code)] impl FMIndex { fn build(texts: &[(u64, &[u8])]) -> Result { if texts.is_empty() { @@ -1080,89 +989,6 @@ impl FMIndex { }) } - /// Locate: resolve SA[pos] by walking LF-mapping until hitting a sampled position. - /// For large data (N >> SA_SAMPLE_RATE), converges within SA_SAMPLE_RATE steps. - /// For small data with short LF cycles, may need up to N steps. - #[inline] - fn locate(&self, mut pos: usize) -> usize { - let mut steps = 0; - let n = self.wavelet.len; - loop { - if pos.is_multiple_of(SA_SAMPLE_RATE) && (pos / SA_SAMPLE_RATE) < self.sa_samples.len() - { - return (self.sa_samples[pos / SA_SAMPLE_RATE] as usize + steps) % n; - } - let c = self.wavelet.access(pos); - pos = self.c_table[c as usize] + self.wavelet.rank(c, pos); - steps += 1; - if steps >= n { - log::warn!("FM-Index SA locate exceeded {n} steps, possible index corruption"); - return 0; - } - } - } - - /// Map a text position to document index via binary search on doc_start_positions. - #[inline] - fn doc_for_position(&self, text_pos: usize) -> usize { - let tp = text_pos as u64; - match self.doc_start_positions.binary_search(&tp) { - Ok(idx) => idx, - Err(idx) => idx - 1, - } - } - - fn backward_search(&self, pattern: &[u8]) -> (usize, usize) { - if pattern.is_empty() || self.wavelet.len == 0 { - return (0, 0); - } - let (mut lo, mut hi) = (0, self.wavelet.len); - for &b in pattern.iter().rev() { - let c = self.c_table[b as usize]; - let (occ_lo, occ_hi) = self.wavelet.rank_pair(b, lo, hi); - lo = c + occ_lo; - hi = c + occ_hi; - if lo >= hi { - return (0, 0); - } - } - (lo, hi) - } - - #[cfg(test)] - fn search(&self, pattern: &[u8]) -> RoaringBitmap { - let (lo, hi) = self.backward_search(pattern); - if lo >= hi { - return RoaringBitmap::new(); - } - let mut result = RoaringBitmap::new(); - for i in lo..hi { - let text_pos = self.locate(i); - let doc_idx = self.doc_for_position(text_pos); - result.insert(self.row_ids[doc_idx] as u32); - } - result - } - - /// Search returning full u64 row addresses (preserving fragment ID in upper bits). - fn search_row_addrs(&self, pattern: &[u8]) -> Vec { - let (lo, hi) = self.backward_search(pattern); - if lo >= hi { - return Vec::new(); - } - let mut seen = std::collections::HashSet::new(); - let mut result = Vec::new(); - for i in lo..hi { - let text_pos = self.locate(i); - let doc_idx = self.doc_for_position(text_pos); - let row_addr = self.row_ids[doc_idx]; - if seen.insert(row_addr) { - result.push(row_addr); - } - } - result - } - fn serialize_huffman_codes(&self) -> Vec { let mut buf = Vec::new(); for code in &self.wavelet.codes { @@ -1289,7 +1115,7 @@ impl FMIndex { } } let refs: Vec<&[u8]> = words_b.iter().map(|v| v.as_slice()).collect(); - let schema = Arc::new(Self::block_schema()); + let schema = BLOCK_SCHEMA.clone(); Ok(RecordBatch::try_new( schema, vec![ @@ -1301,16 +1127,6 @@ impl FMIndex { ], )?) } - - fn block_schema() -> arrow_schema::Schema { - arrow_schema::Schema::new(vec![ - Field::new("node_id", DataType::UInt32, false), - Field::new("block_id", DataType::UInt32, false), - Field::new("words", DataType::LargeBinary, false), - Field::new("prefix_rank", DataType::UInt64, false), - Field::new("bit_len", DataType::UInt64, false), - ]) - } } // ── Lazy FM-Index ──────────────────────────────────────────────────────────── @@ -1701,7 +1517,7 @@ impl FMIndexScalarIndex { } pfiles.sort_by_key(|(id, _)| *id); let io_parallelism = store.io_parallelism().max(1); - let mut parts = futures::stream::iter(pfiles.into_iter()) + let mut parts = futures::stream::iter(pfiles) .map(|(id, name)| { let store = Arc::clone(&store); async move { @@ -2285,7 +2101,7 @@ async fn write_fmindex( filename: &str, partition_fingerprint: Option<&str>, ) -> Result { - let schema = Arc::new(FMIndex::block_schema()); + let schema = BLOCK_SCHEMA.clone(); let mut writer = store.new_index_file(filename, schema.clone()).await?; @@ -2492,6 +2308,8 @@ impl ScalarIndexPlugin for FMIndexPlugin { false, // supports_regex: regex acceleration is only implemented for ngram. false, + // min_contains_chars: the FM-index can match a needle of any length. + 0, ))) } async fn load_index( @@ -2616,107 +2434,6 @@ mod tests { .sum() } - #[test] - fn test_fmindex_build_and_search() { - let texts: Vec<(u64, &[u8])> = vec![ - (0, b"hello world"), - (1, b"hello rust"), - (2, b"goodbye world"), - ]; - let fm = FMIndex::build(&texts).unwrap(); - - let r = fm.search(b"hello"); - assert!(r.contains(0)); - assert!(r.contains(1)); - assert!(!r.contains(2)); - - let r = fm.search(b"world"); - assert!(r.contains(0)); - assert!(!r.contains(1)); - assert!(r.contains(2)); - - let r = fm.search(b"goodbye"); - assert!(!r.contains(0)); - assert!(!r.contains(1)); - assert!(r.contains(2)); - - assert!(fm.search(b"xyz").is_empty()); - } - - #[test] - fn test_fmindex_empty() { - let fm = FMIndex::build(&[]).unwrap(); - assert!(fm.search(b"anything").is_empty()); - } - - #[test] - fn test_fmindex_single_char_search() { - let texts: Vec<(u64, &[u8])> = vec![(0, b"abc"), (1, b"def")]; - let fm = FMIndex::build(&texts).unwrap(); - assert!(fm.search(b"a").contains(0)); - assert!(!fm.search(b"a").contains(1)); - assert!(!fm.search(b"d").contains(0)); - assert!(fm.search(b"d").contains(1)); - } - - #[test] - fn test_fmindex_repeated_pattern() { - let texts: Vec<(u64, &[u8])> = vec![(0, b"ababab"), (1, b"cdcd")]; - let fm = FMIndex::build(&texts).unwrap(); - assert!(fm.search(b"ab").contains(0)); - assert!(!fm.search(b"ab").contains(1)); - assert!(!fm.search(b"cd").contains(0)); - assert!(fm.search(b"cd").contains(1)); - } - - #[test] - fn test_early_exit_all_docs_match() { - let texts: Vec<(u64, &[u8])> = vec![(0, b"the cat"), (1, b"the dog"), (2, b"the bird")]; - let fm = FMIndex::build(&texts).unwrap(); - assert_eq!(fm.search(b"the").len(), 3); - } - - #[test] - fn test_locate_correctness() { - let texts: Vec<(u64, &[u8])> = vec![ - (0, b"the quick brown fox jumps over the lazy dog"), - (1, b"pack my box with five dozen liquor jugs"), - (2, b"how vexingly quick daft zebras jump"), - ]; - let fm = FMIndex::build(&texts).unwrap(); - - let r = fm.search(b"quick"); - assert!(r.contains(0)); - assert!(!r.contains(1)); - assert!(r.contains(2)); - - let r = fm.search(b"the"); - assert!(r.contains(0)); - assert!(!r.contains(1)); - assert!(!r.contains(2)); - - let r = fm.search(b"jump"); - assert!(r.contains(0)); - assert!(r.contains(2)); - } - - #[test] - fn test_many_documents() { - let docs: Vec> = (0..100) - .map(|i| format!("document number {} with hello world data xyz", i).into_bytes()) - .collect(); - let texts: Vec<(u64, &[u8])> = docs - .iter() - .enumerate() - .map(|(i, d)| (i as u64, d.as_slice())) - .collect(); - let fm = FMIndex::build(&texts).unwrap(); - - assert_eq!(fm.search(b"hello world").len(), 100); - assert_eq!(fm.search(b"document number 42").len(), 1); - assert_eq!(fm.search(b"nonexistent").len(), 0); - } - #[test] fn test_index_size_ratio() { let docs: Vec> = (0..200) @@ -2747,42 +2464,6 @@ mod tests { ); } - #[test] - fn test_wavelet_access_consistency() { - let docs: Vec> = (0..50) - .map(|i| format!("document {i} hello world test").into_bytes()) - .collect(); - let texts: Vec<(u64, &[u8])> = docs - .iter() - .enumerate() - .map(|(i, d)| (i as u64, d.as_slice())) - .collect(); - - let mut concat = Vec::new(); - for (_, text) in &texts { - concat.extend_from_slice(text); - concat.push(SENTINEL_BYTE); - } - concat.push(0x00); - let sa = build_suffix_array(&concat); - let n = concat.len(); - let bwt: Vec = sa - .iter() - .map(|&pos| { - if pos == 0 { - concat[n - 1] - } else { - concat[pos - 1] - } - }) - .collect(); - let wavelet = HuffmanWaveletTree::build(&bwt); - - for (i, &expected) in bwt.iter().enumerate().take(n.min(500)) { - assert_eq!(wavelet.access(i), expected, "access mismatch at {i}"); - } - } - #[test] fn test_serialization_roundtrip() { let texts: Vec<(u64, &[u8])> = vec![ @@ -2827,43 +2508,6 @@ mod tests { assert!(!fmindex_partition_limit_reached(3, 99, 10, 100)); } - #[test] - fn test_sentinel_sanitization() { - // Text containing \xFF should be sanitized to space during training. - let texts: Vec<(u64, &[u8])> = vec![(0, b"hello\xFFworld")]; - let fm = FMIndex::build(&texts).unwrap(); - // Build itself does not sanitize, but search should still work. - let r = fm.search(b"hello"); - assert!(r.contains(0)); - } - - #[test] - fn test_wavelet_rank_pair_consistency() { - let docs: Vec> = (0..30) - .map(|i| format!("doc {i} with repeated words hello world test data").into_bytes()) - .collect(); - let texts: Vec<(u64, &[u8])> = docs - .iter() - .enumerate() - .map(|(i, d)| (i as u64, d.as_slice())) - .collect(); - let fm = FMIndex::build(&texts).unwrap(); - - let n = fm.wavelet.len; - for b in [b'a', b'e', b' ', SENTINEL_BYTE] { - for &(lo, hi) in &[(0usize, 1usize), (0, n), (n / 4, n / 2)] { - if lo >= n || hi > n || lo >= hi { - continue; - } - let (pl, ph) = fm.wavelet.rank_pair(b, lo, hi); - let rl = fm.wavelet.rank(b, lo); - let rh = fm.wavelet.rank(b, hi); - assert_eq!(pl, rl, "rank_pair lo mismatch for b={b} [{lo},{hi})"); - assert_eq!(ph, rh, "rank_pair hi mismatch for b={b} [{lo},{hi})"); - } - } - } - #[test] fn test_large_sa_sampling() { // Test with enough documents to have multiple SA sample points @@ -2884,9 +2528,6 @@ mod tests { let fm = FMIndex::build(&texts).unwrap(); assert!(fm.sa_samples.len() > 1, "should have multiple SA samples"); - assert_eq!(fm.search(b"document number 25").len(), 1); - assert_eq!(fm.search(b"document number").len(), 50); - assert_eq!(fm.search(b"nonexistent pattern").len(), 0); } #[tokio::test(flavor = "multi_thread")] diff --git a/rust/lance-index/src/scalar/inverted.rs b/rust/lance-index/src/scalar/inverted.rs index 6c9a5ad2947..ec7c22021cc 100644 --- a/rust/lance-index/src/scalar/inverted.rs +++ b/rust/lance-index/src/scalar/inverted.rs @@ -3,23 +3,34 @@ pub mod builder; mod cache_codec; +mod compound; +mod cross_column; +mod documents; mod encoding; +mod impact; mod index; mod iter; pub mod json; -mod lazy_docset; pub mod parser; pub mod query; mod scorer; pub mod tokenizer; mod wand; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::Arc; use arrow_schema::{DataType, Field}; use async_trait::async_trait; pub use builder::InvertedIndexBuilder; +pub use compound::{ + compound_search, compound_search_prepared_match, + compound_search_prepared_match_with_score_floor, compound_search_with_base_scorer, + compound_search_with_base_scorer_and_score_floor, exclusive_scaled_score_floor, + materialized_compound_top_k, +}; +#[doc(hidden)] +pub use cross_column::cross_column_compound_search; use datafusion::execution::SendableRecordBatchStream; pub use index::*; use lance_core::{Result, cache::LanceCache}; @@ -27,87 +38,243 @@ pub use lance_tokenizer::Language; pub use scorer::{MemBM25Scorer, Scorer}; pub use tokenizer::*; -use crate::scalar::inverted::query::{FtsSearchParams, Tokens}; +use crate::scalar::inverted::query::{FtsSearchParams, Tokens, uses_fuzzy_expansion}; -/// Collect the unique terms needed to build a shared BM25 scorer. +/// Canonical token vocabulary and BM25 statistics for one indexed query leaf. /// -/// The scorer only needs corpus-level document frequencies, so we keep a -/// deduplicated term list here instead of constructing a full `Tokens` -/// object with positions. When fuzziness is enabled, each segment may -/// contribute additional terms (via `expand_fuzzy_tokens`); the union of -/// those terms is what the global scorer must cover. -fn scorer_terms( +/// Keeping these values together prevents a search path from expanding one +/// vocabulary while scoring another. Positions on `tokens` identify fuzzy +/// alternatives belonging to the same original query position. +#[doc(hidden)] +#[derive(Clone)] +pub struct PreparedBm25Query { + tokens: Arc, + scorer: Arc, + has_all_query_positions: bool, +} + +impl std::fmt::Debug for PreparedBm25Query { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PreparedBm25Query") + .field("token_count", &self.tokens.len()) + .field("has_all_query_positions", &self.has_all_query_positions) + .field("scorer", &self.scorer) + .finish() + } +} + +impl PreparedBm25Query { + pub(crate) fn from_parts( + tokens: Arc, + scorer: Arc, + has_all_query_positions: bool, + ) -> Self { + Self { + tokens, + scorer, + has_all_query_positions, + } + } + + #[doc(hidden)] + pub fn tokens(&self) -> &Arc { + &self.tokens + } + + #[doc(hidden)] + pub fn scorer(&self) -> &Arc { + &self.scorer + } + + pub(crate) fn has_all_query_positions(&self) -> bool { + self.has_all_query_positions + } +} + +pub(crate) fn final_query_tokens( indices: &[Arc], query_tokens: &Tokens, params: &FtsSearchParams, -) -> Result> { - let mut terms = Vec::new(); - let mut seen = HashSet::new(); +) -> Result { + if !uses_fuzzy_expansion(params.fuzziness) { + return Ok(query_tokens.clone()); + } - if !matches!(params.fuzziness, Some(n) if n != 0) { - for token in query_tokens { - if seen.insert(token.to_string()) { - terms.push(token.to_string()); + let initial_capacity = query_tokens.len().min(params.max_expansions); + let mut expanded_tokens = Vec::with_capacity(initial_capacity); + let mut expanded_positions = Vec::with_capacity(initial_capacity); + let mut seen = HashSet::new(); + let mut source_terms_by_position = BTreeMap::>::new(); + for token_idx in 0..query_tokens.len() { + source_terms_by_position + .entry(query_tokens.position(token_idx)) + .or_default() + .push(query_tokens.get_token(token_idx)); + } + for (position, source_terms) in source_terms_by_position { + let remaining = params.max_expansions.saturating_sub(expanded_tokens.len()); + if remaining == 0 { + break; + } + let mut candidates = BTreeSet::new(); + let mut seen_source_terms = HashSet::new(); + for source_term in source_terms { + if !seen_source_terms.insert(source_term) { + continue; + } + // One source token has one canonical automaton across every + // selected segment. Drop it after this source term so peak DFA + // memory is independent of the number of query terms. + let automaton = FuzzyAutomaton::new(source_term, query_tokens.token_type(), params)?; + for index in indices { + index.collect_fuzzy_candidates_with_automaton( + &automaton, + remaining, + &mut candidates, + )?; + } + } + for candidate in candidates { + if expanded_tokens.len() >= params.max_expansions { + break; + } + if seen.insert((candidate.clone(), position)) { + expanded_tokens.push(candidate); + expanded_positions.push(position); } } - return Ok(terms); } + Ok(Tokens::with_positions( + expanded_tokens, + expanded_positions, + query_tokens.token_type().clone(), + )) +} - for index in indices { - let expanded = index.expand_fuzzy_tokens(query_tokens, params)?; - for idx in 0..expanded.len() { - let token = expanded.get_token(idx); - if seen.insert(token.to_string()) { - terms.push(token.to_string()); - } +fn unique_terms(tokens: &Tokens) -> Vec { + let mut terms = Vec::with_capacity(tokens.len()); + let mut seen = HashSet::new(); + for token in tokens { + if seen.insert(token.clone()) { + terms.push(token.clone()); } } - Ok(terms) + terms } -/// Build a shared [`MemBM25Scorer`] across a set of FTS index segments. +pub(crate) fn has_all_query_positions(query_tokens: &Tokens, final_tokens: &Tokens) -> bool { + let surviving_positions = (0..final_tokens.len()) + .map(|index| final_tokens.position(index)) + .collect::>(); + (0..query_tokens.len()).all(|index| surviving_positions.contains(&query_tokens.position(index))) +} + +/// Expand and score one indexed query leaf exactly once across all segments. /// -/// Aggregates each segment's `(total_tokens, num_docs, per_term_doc_freq)` -/// statistics — obtained via [`InvertedIndex::bm25_stats_for_terms`] — into a -/// single corpus-wide scorer, so that BM25 IDF scoring uses *global* -/// statistics rather than per-segment statistics. Computes the union of -/// fuzzy-expanded terms when `params.fuzziness` is set. +/// Expansion consumes one deterministic `max_expansions` budget in query +/// position order, with terms ordered lexicographically across every physical +/// segment and partition and deduplicated by `(term, position)`. The scorer's +/// document frequencies are then merged for exactly those final terms. /// -/// Public as the canonical producer paired with the `with_base_scorer` -/// consumer on FTS exec types: callers holding `Arc` segment -/// handles locally can construct an injectable scorer without reimplementing -/// per-segment stat aggregation, term deduplication, and fuzzy-expansion -/// union. Keeps a single source of truth for BM25 IDF arithmetic across -/// segments. -pub async fn build_global_bm25_scorer( +/// `base_scorer` is an API-compatibility hook for distributed/mixed callers +/// that already own corpus-wide statistics. It is validated against the final +/// vocabulary before being paired with the tokens. +#[doc(hidden)] +pub async fn prepare_bm25_query( indices: &[Arc], - query_tokens: &Tokens, + query_tokens: Tokens, params: &FtsSearchParams, -) -> Result { - let terms = scorer_terms(indices, query_tokens, params)?; + metrics: Option<&dyn crate::scalar::MetricsCollector>, + base_scorer: Option>, +) -> Result { let first_index = indices.first().ok_or_else(|| { lance_core::Error::invalid_input("FTS index requires at least one segment") })?; - let (mut total_tokens, mut num_docs, first_token_docs) = - first_index.bm25_stats_for_terms(&terms).await?; - let mut token_docs = HashMap::with_capacity(terms.len()); - for (term, count) in terms.iter().cloned().zip(first_token_docs.into_iter()) { - token_docs.insert(term, count); - } + let (tokens, has_all_query_positions) = if uses_fuzzy_expansion(params.fuzziness) { + let tokens = Arc::new(final_query_tokens(indices, &query_tokens, params)?); + let has_all_query_positions = has_all_query_positions(&query_tokens, tokens.as_ref()); + (tokens, has_all_query_positions) + } else { + (Arc::new(query_tokens), true) + }; + let terms = unique_terms(tokens.as_ref()); + let scorer = if let Some(scorer) = base_scorer { + if let Some(missing) = terms + .iter() + .find(|term| !scorer.token_docs.contains_key(term.as_str())) + { + return Err(lance_core::Error::invalid_input(format!( + "injected BM25 scorer is missing compound FTS token '{missing}'" + ))); + } + scorer + } else { + let (mut total_tokens, mut num_docs, first_token_docs) = + first_index.bm25_stats_for_terms(&terms, metrics).await?; + let mut token_docs = HashMap::with_capacity(terms.len()); + for (term, count) in terms.iter().cloned().zip(first_token_docs) { + token_docs.insert(term, count); + } - for index in indices.iter().skip(1) { - let (segment_total_tokens, segment_num_docs, segment_token_docs) = - index.bm25_stats_for_terms(&terms).await?; - total_tokens += segment_total_tokens; - num_docs += segment_num_docs; - for (term, count) in terms.iter().zip(segment_token_docs.into_iter()) { - *token_docs - .get_mut(term) - .expect("global scorer terms should already be initialized") += count; + for index in indices.iter().skip(1) { + let (segment_total_tokens, segment_num_docs, segment_token_docs) = + index.bm25_stats_for_terms(&terms, metrics).await?; + total_tokens = total_tokens + .checked_add(segment_total_tokens) + .ok_or_else(|| lance_core::Error::index("FTS corpus token count overflows u64"))?; + num_docs = num_docs.checked_add(segment_num_docs).ok_or_else(|| { + lance_core::Error::index("FTS corpus document count overflows usize") + })?; + for (term, count) in terms.iter().zip(segment_token_docs) { + let total = token_docs.get_mut(term).ok_or_else(|| { + lance_core::Error::internal(format!( + "global scorer term '{term}' was not initialized" + )) + })?; + *total = total.checked_add(count).ok_or_else(|| { + lance_core::Error::index(format!( + "FTS document frequency for term '{term}' overflows usize" + )) + })?; + } } - } + Arc::new(MemBM25Scorer::new(total_tokens, num_docs, token_docs)) + }; + + Ok(PreparedBm25Query { + tokens, + scorer, + has_all_query_positions, + }) +} - Ok(MemBM25Scorer::new(total_tokens, num_docs, token_docs)) +/// Build a shared [`MemBM25Scorer`] across a set of FTS index segments. +/// +/// Compatibility wrapper for callers that only need statistics. Indexed +/// execution should retain the [`PreparedBm25Query`] returned by +/// [`prepare_bm25_query`] so the same final vocabulary reaches search. +/// +/// Aggregates each segment's `(total_tokens, num_docs, per_term_doc_freq)` +/// statistics into a single corpus-wide scorer. +/// +/// `metrics`, when provided, is forwarded to the per-token metadata cache +/// boundary on each segment so callers running under an `ExecutionPlan` +/// (e.g. `MatchQueryExec`) see the reads triggered here in their per-query +/// `index_cache_hits`/`index_cache_misses` counters. +/// +/// For exact queries this remains the compatibility producer paired with the +/// `with_base_scorer` consumer on FTS exec types. Fuzzy distributed execution +/// must retain the full [`PreparedBm25Query`] from [`prepare_bm25_query`]; a +/// scorer alone cannot preserve the canonical expansion vocabulary. +pub async fn build_global_bm25_scorer( + indices: &[Arc], + query_tokens: &Tokens, + params: &FtsSearchParams, + metrics: Option<&dyn crate::scalar::MetricsCollector>, +) -> Result { + let prepared = prepare_bm25_query(indices, query_tokens.clone(), params, metrics, None).await?; + Ok(prepared.scorer.as_ref().clone()) } use lance_core::Error; @@ -118,7 +285,8 @@ use crate::scalar::{ CreatedIndex, RowIdRemapper, ScalarIndex, expression::{FtsQueryParser, ScalarQueryParser}, registry::{ - BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest, + BasicTrainer, ScalarIndexCacheKey, ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, + TrainingOrdering, TrainingRequest, }, }; @@ -146,7 +314,9 @@ impl InvertedIndexPlugin { } }); + params.validate_format_version()?; let format_version = params.resolved_format_version(); + let is_element_document = params.get_document_granularity().is_list_element(); let details = pbold::InvertedIndexDetails::try_from(¶ms)?; let mut inverted_index = InvertedIndexBuilder::new_with_fragment_mask(params, fragment_mask) @@ -154,7 +324,11 @@ impl InvertedIndexPlugin { let files = inverted_index.update(data, index_store, None).await?; Ok(CreatedIndex { index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: format_version.index_version(), + index_version: if is_element_document { + INVERTED_INDEX_VERSION_V3 + } else { + format_version.index_version() + }, files, }) } @@ -211,7 +385,7 @@ impl BasicTrainer for InvertedIndexPlugin { .into())) } - let params = serde_json::from_str::(params)?; + let params = InvertedIndexParams::from_training_json(params)?; Ok(Box::new(InvertedIndexTrainingRequest::new(params))) } @@ -266,7 +440,7 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { } fn version(&self) -> u32 { - max_supported_fts_format_version().index_version() + INVERTED_INDEX_VERSION_V3 } fn new_query_parser( @@ -295,33 +469,95 @@ impl ScalarIndexPlugin for InvertedIndexPlugin { async fn load_index( &self, index_store: Arc, - _index_details: &prost_types::Any, + index_details: &prost_types::Any, frag_reuse_index: Option>, cache: &LanceCache, ) -> Result> { - Ok( - InvertedIndex::load(index_store, frag_reuse_index, cache).await? - as Arc, - ) + let index = InvertedIndex::load(index_store, frag_reuse_index, cache).await?; + let details = index_details.to_msg::()?; + let expected_granularity = DocumentGranularity::try_from(details.document_granularity)?; + let physical_granularity = index.params().get_document_granularity(); + if physical_granularity != expected_granularity { + return Err(Error::index(format!( + "FTS document granularity in index details is {expected_granularity:?}, but the physical document schema implies {physical_granularity:?}" + ))); + } + Ok(index as Arc) + } + + async fn get_or_insert_in_cache( + &self, + _index_store: Arc, + _frag_reuse_index: Option>, + cache: &LanceCache, + load: ScalarIndexLoad<'_>, + ) -> Result> { + cache + .get_or_insert_unsized_with_key(ScalarIndexCacheKey, || load) + .await } fn details_as_json(&self, details: &prost_types::Any) -> Result { let index_details = details.to_msg::()?; let index_params = InvertedIndexParams::try_from(&index_details)?; - Ok(serde_json::json!(&index_params)) + Ok(index_params.to_details_json()?) } } #[cfg(test)] mod tests { use super::*; + use crate::scalar::{BuiltinIndexType, ScalarIndexParams}; + + #[test] + fn test_plugin_version_tracks_v3_capability_gate() { + let plugin = InvertedIndexPlugin; + assert_eq!(plugin.version(), INVERTED_INDEX_VERSION_V3); + } #[test] - fn test_plugin_version_tracks_max_supported_format() { + fn test_details_json_includes_document_granularity() { + let details = pbold::InvertedIndexDetails { + document_granularity: pbold::inverted_index_details::DocumentGranularity::ListElement + as i32, + ..Default::default() + }; + let details = prost_types::Any::from_msg(&details).unwrap(); + + let json = InvertedIndexPlugin.details_as_json(&details).unwrap(); + + assert_eq!(json["document_granularity"], "list_element"); + } + + #[test] + fn test_new_training_request_defaults_missing_block_size_to_128() { let plugin = InvertedIndexPlugin; - assert_eq!( - plugin.version(), - max_supported_fts_format_version().index_version() - ); + let field = Field::new("text", DataType::Utf8, true); + + let cases = [ + ( + ScalarIndexParams::for_builtin(BuiltinIndexType::Inverted), + false, + ), + (ScalarIndexParams::new("inverted".to_string()), false), + ( + ScalarIndexParams::new("inverted".to_string()) + .with_params(&serde_json::json!({ "with_position": true })), + true, + ), + ]; + + for (params, expected_with_position) in cases { + let request = plugin + .new_training_request(params.params.as_deref().unwrap_or("{}"), &field) + .unwrap(); + let request = request + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(request.parameters.posting_block_size(), DEFAULT_BLOCK_SIZE); + assert_eq!(request.parameters.has_positions(), expected_with_position); + } } } diff --git a/rust/lance-index/src/scalar/inverted/builder.rs b/rust/lance-index/src/scalar/inverted/builder.rs index ede00ad43ee..7b48ae44ad3 100644 --- a/rust/lance-index/src/scalar/inverted/builder.rs +++ b/rust/lance-index/src/scalar/inverted/builder.rs @@ -1,10 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use super::encoding::encode_group_starts; use super::{InvertedIndexParams, index::*}; use crate::scalar::inverted::document_tokenizer::DocType; use crate::scalar::inverted::json::JsonTextStream; +use crate::scalar::inverted::tokenizer::LEGACY_BLOCK_SIZE; use crate::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer; #[cfg(test)] use crate::scalar::lance_format::LanceIndexStore; @@ -13,12 +13,11 @@ use crate::vector::graph::OrderedFloat; use crate::{progress::IndexBuildProgress, progress::noop_progress}; use arrow::array::AsArray; use arrow::datatypes; -use arrow_array::{Array, BinaryArray, RecordBatch, UInt64Array}; +use arrow_array::{Array, BinaryArray, RecordBatch}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; -use bytes::Bytes; -use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream}; +use datafusion::execution::SendableRecordBatchStream; use fst::Streamer; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::{StreamExt, TryStreamExt}; use lance_arrow::json::JSON_EXT_NAME; use lance_arrow::{ARROW_EXT_NAME_KEY, iter_str_array}; use lance_bitpacking::{BitPacker, BitPacker4x}; @@ -27,24 +26,23 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::error::LanceOptionExt; use lance_core::utils::row_addr_remap::RowAddrRemap; use lance_core::utils::tokio::{IO_CORE_RESERVATION, get_num_compute_intensive_cpus, spawn_cpu}; -use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result}; +use lance_core::{Error, ROW_ID, Result}; use lance_io::object_store::ObjectStore; use lance_select::RowSetOps; use object_store::path::Path; use roaring::RoaringBitmap; use smallvec::SmallVec; use std::collections::HashMap; -use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; -use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; use std::{fmt::Debug, sync::atomic::AtomicU64}; use tracing::instrument; -// the number of elements in each block -// each block contains 128 row ids and 128 frequencies -// WARNING: changing this value will break the compatibility with existing indexes +// The legacy bitpacking block size. Position streams still use this block size; +// FTS posting blocks choose their physical bitpacker from the configured +// InvertedIndexParams::block_size. pub const BLOCK_SIZE: usize = BitPacker4x::BLOCK_LEN; // The default number of workers to use for FTS builds. @@ -75,92 +73,8 @@ static LANCE_FTS_POSTING_BATCH_ROWS: LazyLock = LazyLock::new(|| { .parse() .expect("failed to parse LANCE_FTS_POSTING_BATCH_ROWS") }); -// Target serialized byte size of a posting-list cache group. Consecutive -// posting lists are grouped into a single cache entry until their combined -// serialized size reaches this target, amortizing per-entry overhead across -// small (Zipfian-rare) terms. See issue #7040. -static LANCE_FTS_POSTING_GROUP_TARGET_BYTES: LazyLock = LazyLock::new(|| { - std::env::var("LANCE_FTS_POSTING_GROUP_TARGET_BYTES") - .unwrap_or_else(|_| "4096".to_string()) - .parse() - .expect("failed to parse LANCE_FTS_POSTING_GROUP_TARGET_BYTES") -}); -// Maximum number of posting lists in a single cache group, regardless of byte -// size. Caps the work and memory of a single group read for corpora with many -// tiny terms. -static LANCE_FTS_POSTING_GROUP_MAX_TOKENS: LazyLock = LazyLock::new(|| { - std::env::var("LANCE_FTS_POSTING_GROUP_MAX_TOKENS") - .unwrap_or_else(|_| "256".to_string()) - .parse() - .expect("failed to parse LANCE_FTS_POSTING_GROUP_MAX_TOKENS") -}); const MAX_RETAINED_TOKEN_IDS: usize = 8 * 1024; -/// Write-time configuration controlling how consecutive posting lists are -/// grouped into a single read-path cache entry (issue #7040). Defaults come -/// from the `LANCE_FTS_POSTING_GROUP_*` environment variables. -#[derive(Debug, Clone, Copy)] -pub(crate) struct PostingGroupConfig { - pub(crate) target_bytes: usize, - pub(crate) max_tokens: usize, -} - -impl Default for PostingGroupConfig { - fn default() -> Self { - Self { - target_bytes: (*LANCE_FTS_POSTING_GROUP_TARGET_BYTES).max(1), - max_tokens: (*LANCE_FTS_POSTING_GROUP_MAX_TOKENS).max(1), - } - } -} - -/// Accumulates posting-list group boundaries at write time. Tokens are pushed -/// in row order; a group is cut once its serialized bytes reach -/// `target_bytes` or it holds `max_tokens` posting lists. A posting list -/// larger than the target that *starts* a group occupies that group alone (the -/// clamp case); one encountered mid-group is absorbed and closes that group, so -/// a single term is never split across groups. -#[derive(Debug)] -pub(crate) struct PostingGroupAccumulator { - config: PostingGroupConfig, - starts: Vec, - next_token: u32, - current_bytes: usize, - current_tokens: usize, -} - -impl PostingGroupAccumulator { - pub(crate) fn new(config: PostingGroupConfig) -> Self { - Self { - config, - starts: Vec::new(), - next_token: 0, - current_bytes: 0, - current_tokens: 0, - } - } - - /// Record the next posting list in row order, given its serialized byte size. - pub(crate) fn push(&mut self, posting_bytes: usize) { - if self.current_tokens == 0 { - self.starts.push(self.next_token); - } - self.current_bytes += posting_bytes; - self.current_tokens += 1; - self.next_token += 1; - if self.current_bytes >= self.config.target_bytes - || self.current_tokens >= self.config.max_tokens - { - self.current_bytes = 0; - self.current_tokens = 0; - } - } - - pub(crate) fn into_starts(self) -> Vec { - self.starts - } -} - fn default_num_workers() -> usize { let total_cpus = get_num_compute_intensive_cpus() + *IO_CORE_RESERVATION; std::cmp::max(1, total_cpus / 2) @@ -195,9 +109,13 @@ fn merge_all_tail_partitions( ) -> Result> { let mut merged_builders: Vec = Vec::new(); let mut merged: Option = None; + let mut empty_coordinate_builder: Option = None; for tail in tails { let builder = tail.builder; if builder.is_empty() { + if builder.docs.coordinate_rank() > 0 && empty_coordinate_builder.is_none() { + empty_coordinate_builder = Some(builder); + } continue; } match &mut merged { @@ -219,6 +137,11 @@ fn merge_all_tail_partitions( if let Some(builder) = merged { merged_builders.push(builder); } + if merged_builders.is_empty() + && let Some(builder) = empty_coordinate_builder + { + merged_builders.push(builder); + } Ok(merged_builders) } @@ -282,8 +205,11 @@ impl InvertedIndexBuilder { } pub fn with_posting_tail_codec(mut self, posting_tail_codec: PostingTailCodec) -> Self { - self.format_version = - InvertedListFormatVersion::from_posting_tail_codec(posting_tail_codec); + self.format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size( + posting_tail_codec, + self.params.block_size, + ) + .expect("invalid posting tail codec for posting block size"); self.posting_tail_codec = posting_tail_codec; self } @@ -310,6 +236,7 @@ impl InvertedIndexBuilder { dest_store: &dyn IndexStore, old_data_filter: Option, ) -> Result> { + validate_format_version_block_size(self.format_version, self.params.block_size)?; let schema = new_data.schema(); let doc_col = schema.field(0).name(); @@ -344,6 +271,7 @@ impl InvertedIndexBuilder { old_segments: &[Arc], old_data_filter: Option, ) -> Result> { + validate_format_version_block_size(self.format_version, self.params.block_size)?; let schema = new_data.schema(); let doc_col = schema.field(0).name(); @@ -392,22 +320,36 @@ impl InvertedIndexBuilder { if partition_builder.is_empty() { continue; } - match &mut merged { - Some(merged) => { - let would_exceed_memory = merged + match merged.take() { + Some(mut accumulated) => { + let would_exceed_memory = accumulated .memory_size() .saturating_add(partition_builder.memory_size()) >= memory_limit_bytes; - let would_exceed_doc_ids = merged + let would_exceed_doc_ids = accumulated .docs .len() .saturating_add(partition_builder.docs.len()) > u32::MAX as usize; if would_exceed_memory || would_exceed_doc_ids { - let builder = std::mem::replace(merged, partition_builder); - files.extend(self.write_new_partition(dest_store, builder).await?); + merged = Some(partition_builder); + files.extend(self.write_new_partition(dest_store, accumulated).await?); } else { - merged.merge_from(partition_builder)?; + // `merge_from` remaps token ids into a unified + // dictionary and concatenates posting lists across + // builders holding up to LANCE_FTS_PARTITION_SIZE of + // state, so it runs for seconds at a time. Inline it + // would occupy a runtime worker for that whole span, + // starving the tasks driving in-flight uploads; the + // upload's whole-request timeout keeps running while + // its task waits to be polled. The builder is moved + // in and handed back so ownership survives the hop. + accumulated = spawn_cpu(move || { + accumulated.merge_from(partition_builder)?; + Result::Ok(accumulated) + }) + .await?; + merged = Some(accumulated); } } None => merged = Some(partition_builder), @@ -469,6 +411,8 @@ impl InvertedIndexBuilder { fragment_mask: self.fragment_mask, token_set_format: self.token_set_format, worker_memory_limit_bytes, + block_size: self.params.block_size, + coordinate_rank: document_coordinate_rank(&stream.schema()), }; let next_id = self.next_partition_id(); let id_alloc = Arc::new(AtomicU64::new(next_id)); @@ -619,6 +563,7 @@ impl InvertedIndexBuilder { dest_store: &dyn IndexStore, partitions: &[u64], ) -> Result { + validate_format_version_block_size(self.format_version, self.params.block_size)?; let mut serialized_deleted_fragments = Vec::with_capacity(self.deleted_fragments.serialized_size()); self.deleted_fragments @@ -635,6 +580,14 @@ impl InvertedIndexBuilder { POSTING_TAIL_CODEC_KEY.to_owned(), self.posting_tail_codec.as_str().to_owned(), ), + ( + FTS_FORMAT_VERSION_KEY.to_owned(), + self.format_version.index_version().to_string(), + ), + ( + POSTING_BLOCK_SIZE_KEY.to_owned(), + self.params.block_size.to_string(), + ), ]); if self.params.with_position && self.format_version.uses_shared_position_stream() { @@ -677,9 +630,19 @@ impl InvertedIndexBuilder { pub(crate) async fn write_part_metadata( &self, dest_store: &dyn IndexStore, - partition: u64, // Modify parameter type + partition: u64, + ) -> Result { + self.write_staged_metadata(dest_store, part_metadata_file_path(partition), &[partition]) + .await + } + + async fn write_staged_metadata( + &self, + dest_store: &dyn IndexStore, + file_name: String, + partitions: &[u64], ) -> Result { - let partitions = vec![partition]; + validate_format_version_block_size(self.format_version, self.params.block_size)?; let mut metadata = HashMap::from_iter(vec![ ("partitions".to_owned(), serde_json::to_string(&partitions)?), ("params".to_owned(), serde_json::to_string(&self.params)?), @@ -691,6 +654,14 @@ impl InvertedIndexBuilder { POSTING_TAIL_CODEC_KEY.to_owned(), self.posting_tail_codec.as_str().to_owned(), ), + ( + FTS_FORMAT_VERSION_KEY.to_owned(), + self.format_version.index_version().to_string(), + ), + ( + POSTING_BLOCK_SIZE_KEY.to_owned(), + self.params.block_size.to_string(), + ), ]); if self.params.with_position && self.format_version.uses_shared_position_stream() { metadata.insert( @@ -706,8 +677,6 @@ impl InvertedIndexBuilder { .to_owned(), ); } - // Use partition ID to generate a unique temporary filename - let file_name = part_metadata_file_path(partition); let mut writer = dest_store .new_index_file(&file_name, Arc::new(Schema::empty())) .await?; @@ -719,26 +688,39 @@ impl InvertedIndexBuilder { dest_store: &dyn IndexStore, partitions: &[u64], ) -> Result> { - let total = if self.fragment_mask.is_none() { - Some(1) - } else { - Some(partitions.len() as u64) - }; + let total = Some(partitions.len().max(1) as u64); let mut files = Vec::new(); self.progress .stage_start("write_metadata", total, "files") .await?; - if self.fragment_mask.is_none() { - files.push(self.write_metadata(dest_store, partitions).await?); - self.progress.stage_progress("write_metadata", 1).await?; - } else { - let mut completed = 0; - for &partition_id in partitions { - files.push(self.write_part_metadata(dest_store, partition_id).await?); - completed += 1; - self.progress - .stage_progress("write_metadata", completed) - .await?; + match self.fragment_mask { + None => { + files.push(self.write_metadata(dest_store, partitions).await?); + self.progress.stage_progress("write_metadata", 1).await?; + } + Some(fragment_mask) if partitions.is_empty() => { + // Root metadata is the finalization marker for the shared index directory. An + // empty shard must publish only staged metadata so sibling partitions are still + // finalized. + files.push( + self.write_staged_metadata( + dest_store, + empty_part_metadata_file_path(fragment_mask), + partitions, + ) + .await?, + ); + self.progress.stage_progress("write_metadata", 1).await?; + } + Some(_) => { + let mut completed = 0; + for &partition_id in partitions { + files.push(self.write_part_metadata(dest_store, partition_id).await?); + completed += 1; + self.progress + .stage_progress("write_metadata", completed) + .await?; + } } } self.progress.stage_complete("write_metadata").await?; @@ -827,10 +809,10 @@ pub struct InnerBuilder { token_set_format: TokenSetFormat, format_version: InvertedListFormatVersion, posting_tail_codec: PostingTailCodec, + block_size: usize, pub(crate) tokens: TokenSet, pub(crate) posting_lists: Vec, pub(crate) docs: DocSet, - pub(crate) group_config: PostingGroupConfig, } impl InnerBuilder { @@ -849,16 +831,51 @@ impl InnerBuilder { token_set_format: TokenSetFormat, format_version: InvertedListFormatVersion, ) -> Self { + Self::new_with_format_version_and_block_size( + id, + with_position, + token_set_format, + format_version, + LEGACY_BLOCK_SIZE, + ) + } + + pub fn new_with_block_size( + id: u64, + with_position: bool, + token_set_format: TokenSetFormat, + block_size: usize, + ) -> Self { + let format_version = default_fts_format_version_for_block_size(block_size) + .expect("invalid posting list block size"); + Self::new_with_format_version_and_block_size( + id, + with_position, + token_set_format, + format_version, + block_size, + ) + } + + pub fn new_with_format_version_and_block_size( + id: u64, + with_position: bool, + token_set_format: TokenSetFormat, + format_version: InvertedListFormatVersion, + block_size: usize, + ) -> Self { + validate_format_version_block_size(format_version, block_size) + .expect("invalid FTS format version for posting block size"); Self { id, with_position, token_set_format, format_version, posting_tail_codec: format_version.posting_tail_codec(), + block_size, tokens: TokenSet::default(), posting_lists: Vec::new(), docs: DocSet::default(), - group_config: PostingGroupConfig::default(), } } @@ -868,13 +885,34 @@ impl InnerBuilder { token_set_format: TokenSetFormat, posting_tail_codec: PostingTailCodec, ) -> Self { - let format_version = if posting_tail_codec == PostingTailCodec::Fixed32 { - InvertedListFormatVersion::V1 - } else { - InvertedListFormatVersion::V2 - }; - let mut builder = - Self::new_with_format_version(id, with_position, token_set_format, format_version); + Self::new_with_posting_tail_codec_and_block_size( + id, + with_position, + token_set_format, + posting_tail_codec, + LEGACY_BLOCK_SIZE, + ) + } + + pub fn new_with_posting_tail_codec_and_block_size( + id: u64, + with_position: bool, + token_set_format: TokenSetFormat, + posting_tail_codec: PostingTailCodec, + block_size: usize, + ) -> Self { + let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size( + posting_tail_codec, + block_size, + ) + .expect("invalid posting tail codec for posting block size"); + let mut builder = Self::new_with_format_version_and_block_size( + id, + with_position, + token_set_format, + format_version, + block_size, + ); builder.posting_tail_codec = posting_tail_codec; builder } @@ -955,10 +993,10 @@ impl InnerBuilder { token_set_format, format_version, posting_tail_codec, + block_size, tokens, posting_lists, docs, - group_config: _, } = other; if self.with_position != with_position { @@ -985,6 +1023,12 @@ impl InnerBuilder { self.posting_tail_codec, posting_tail_codec ))); } + if self.block_size != block_size { + return Err(Error::index(format!( + "cannot merge partitions with mismatched FTS block sizes: {} vs {}", + self.block_size, block_size + ))); + } let mut token_id_map = vec![u32::MAX; posting_lists.len()]; match tokens.tokens { @@ -1006,11 +1050,23 @@ impl InnerBuilder { } let doc_id_offset = self.docs.len() as u32; - for (row_id, num_tokens) in docs.iter() { - self.docs.append(*row_id, *num_tokens); + for doc_id in 0..docs.len() as u32 { + let row_id = docs.row_id(doc_id); + let num_tokens = docs.num_tokens(doc_id); + let doc_index = docs.doc_index(doc_id); + if doc_index.is_empty() { + self.docs.append(row_id, num_tokens); + } else { + self.docs + .append_with_doc_index(row_id, num_tokens, &doc_index)?; + } } self.posting_lists.resize_with(self.tokens.len(), || { - PostingListBuilder::new_with_posting_tail_codec(with_position, self.posting_tail_codec) + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + with_position, + self.posting_tail_codec, + self.block_size, + ) }); for (token_id, posting_list) in posting_lists.into_iter().enumerate() { @@ -1077,7 +1133,11 @@ impl InnerBuilder { let mut writer = store .new_index_file( path, - inverted_list_schema_for_version(self.with_position, self.format_version), + inverted_list_schema_for_version_with_block_size( + self.with_position, + self.format_version, + self.block_size, + ), ) .await?; let posting_lists = std::mem::take(&mut self.posting_lists); @@ -1090,8 +1150,11 @@ impl InnerBuilder { ); let with_position = self.with_position; let format_version = self.format_version; - let group_config = self.group_config; - let schema = inverted_list_schema_for_version(self.with_position, self.format_version); + let schema = inverted_list_schema_for_version_with_block_size( + self.with_position, + self.format_version, + self.block_size, + ); let docs_for_batches = docs.clone(); let schema_for_batches = schema.clone(); let batch_rows = *LANCE_FTS_POSTING_BATCH_ROWS; @@ -1111,15 +1174,15 @@ impl InnerBuilder { with_position, format_version, batch_rows, - group_config, ); let mut posting_lists = posting_lists.into_iter(); + let mut encode_elapsed = Duration::ZERO; loop { let docs_for_batches = docs_for_batches.clone(); + let encode_started = Instant::now(); // Build the next batch on the CPU pool. The builder and the // remaining posting lists are moved in and handed back so state - // persists across batches -- notably the cache-group accumulator, - // which spans every batch this builder produces. + // persists across batches. let (next_builder, next_posting_lists, batch) = spawn_cpu(move || { let mut batch_builder = batch_builder; let mut posting_lists = posting_lists; @@ -1141,6 +1204,7 @@ impl InnerBuilder { Result::Ok((batch_builder, posting_lists, batch)) }) .await?; + encode_elapsed += encode_started.elapsed(); batch_builder = next_builder; posting_lists = next_posting_lists; @@ -1155,11 +1219,15 @@ impl InnerBuilder { } } - Result::Ok(batch_builder.into_group_starts()) + Result::Ok(encode_elapsed) }); + let mut write_elapsed = Duration::ZERO; while let Ok(batch) = rx.recv().await { - if let Err(err) = writer.write_record_batch(batch).await { + let write_started = Instant::now(); + let result = writer.write_record_batch(batch).await; + write_elapsed += write_started.elapsed(); + if let Err(err) = result { drop(rx); // Wait for producer to stop; preserve the write error as the primary failure. let _ = producer.await; @@ -1167,22 +1235,21 @@ impl InnerBuilder { } } drop(rx); - let group_starts = producer.await??; - - // Persist the posting-list cache-group boundaries as a global buffer, - // recording its 1-indexed id in schema metadata so the reader can group - // small posting lists into a single cache entry (issue #7040). Empty - // partitions skip this entirely and fall back to the per-token path. - let mut extra_metadata = HashMap::new(); - if !group_starts.is_empty() { - let encoded = encode_group_starts(&group_starts); - let buffer_id = writer.add_global_buffer(Bytes::from(encoded)).await?; - extra_metadata.insert( - POSTING_GROUP_OFFSETS_BUF_KEY.to_owned(), - buffer_id.to_string(), - ); - } - writer.finish_with_metadata(extra_metadata).await + let encode_elapsed = producer.await??; + let finish_started = Instant::now(); + let file = writer.finish().await?; + write_elapsed += finish_started.elapsed(); + + // Splits the cost of a partition write into the two halves that are + // otherwise indistinguishable from the outside, so a build that fails on + // an upload timeout shows whether encoding or the upload dominated. + log::info!( + "wrote posting lists of partition {}: {:.1?} encoding, {:.1?} writing", + id, + encode_elapsed, + write_elapsed + ); + Ok(file) } #[instrument(level = "debug", skip_all)] @@ -1206,7 +1273,12 @@ impl InnerBuilder { let batch = docs.to_batch()?; let mut writer = store.new_index_file(path, batch.schema()).await?; writer.write_record_batch(batch).await?; - writer.finish().await + writer + .finish_with_metadata(HashMap::from([( + super::documents::TOTAL_TOKENS_KEY.to_owned(), + docs.total_tokens_num().to_string(), + )])) + .await } } @@ -1252,6 +1324,7 @@ struct IndexWorker { token_set_format: TokenSetFormat, token_ids: Vec, last_token_count: usize, + coordinate_rank: usize, } struct TailPartition { @@ -1264,6 +1337,11 @@ struct WorkerOutput { tail_partition: Option, } +enum DocumentSource<'a> { + Text(&'a str), + StringList(&'a dyn Array), +} + #[derive(Debug, Clone, Copy)] struct IndexWorkerConfig { with_position: bool, @@ -1271,6 +1349,8 @@ struct IndexWorkerConfig { fragment_mask: Option, token_set_format: TokenSetFormat, worker_memory_limit_bytes: u64, + block_size: usize, + coordinate_rank: usize, } impl IndexWorker { @@ -1314,18 +1394,26 @@ impl IndexWorker { id_alloc: Arc, config: IndexWorkerConfig, ) -> Result { - let schema = inverted_list_schema_for_version(config.with_position, config.format_version); + let schema = inverted_list_schema_for_version_with_block_size( + config.with_position, + config.format_version, + config.block_size, + ); + + let mut builder = InnerBuilder::new_with_format_version_and_block_size( + id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + | config.fragment_mask.unwrap_or(0), + config.with_position, + config.token_set_format, + config.format_version, + config.block_size, + ); + builder.docs = DocSet::with_coordinate_rank(config.coordinate_rank); Ok(Self { tokenizer, dest_store, - builder: InnerBuilder::new_with_format_version( - id_alloc.fetch_add(1, std::sync::atomic::Ordering::Relaxed) - | config.fragment_mask.unwrap_or(0), - config.with_position, - config.token_set_format, - config.format_version, - ), + builder, partitions: Vec::new(), files: Vec::new(), id_alloc, @@ -1337,6 +1425,7 @@ impl IndexWorker { token_set_format: config.token_set_format, token_ids: Vec::new(), last_token_count: 0, + coordinate_rank: config.coordinate_rank, }) } @@ -1349,147 +1438,301 @@ impl IndexWorker { async fn process_batch(&mut self, batch: RecordBatch) -> Result<()> { let doc_col = batch.column(0); - let doc_iter = iter_str_array(doc_col); let row_id_col = batch[ROW_ID].as_primitive::(); - let docs = doc_iter - .zip(row_id_col.values().iter()) - .filter_map(|(doc, row_id)| doc.map(|doc| (doc, *row_id))); + let doc_index_columns = (0..self.coordinate_rank) + .map(|rank| { + let column_name = doc_index_storage_column(rank); + batch + .column_by_name(&column_name) + .ok_or_else(|| { + Error::index(format!( + "FTS document input is missing coordinate column {column_name}" + )) + }) + .map(|column| column.as_primitive::()) + }) + .collect::>>()?; + match doc_col.data_type() { + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => { + for (row_index, (doc, row_id)) in iter_str_array(doc_col.as_ref()) + .zip(row_id_col.values().iter()) + .enumerate() + { + let doc = match doc { + Some(doc) => doc, + None if self.coordinate_rank > 0 => "", + None => continue, + }; + let doc_index = doc_index_columns + .iter() + .map(|column| column.value(row_index)) + .collect::>(); + self.process_document(*row_id, DocumentSource::Text(doc), &doc_index) + .await?; + } + } + DataType::List(_) => { + if self.coordinate_rank > 0 { + return Err(Error::index( + "ListElement FTS input must be expanded to string documents before indexing" + .to_string(), + )); + } + self.process_string_list_batch::(doc_col, row_id_col) + .await?; + } + DataType::LargeList(_) => { + if self.coordinate_rank > 0 { + return Err(Error::index( + "ListElement FTS input must be expanded to string documents before indexing" + .to_string(), + )); + } + self.process_string_list_batch::(doc_col, row_id_col) + .await?; + } + data_type => { + return Err(Error::index(format!( + "expect data type String, LargeString, List(String), or LargeList(String) but got {}", + data_type + ))); + } + } + Ok(()) + } + + async fn process_string_list_batch( + &mut self, + doc_col: &Arc, + row_id_col: &arrow_array::PrimitiveArray, + ) -> Result<()> { + let docs = doc_col.as_list::(); + match docs.value_type() { + datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => {} + data_type => { + return Err(Error::index(format!( + "expect list item data type String or LargeString but got {}", + data_type + ))); + } + } + + for (doc, row_id) in docs.iter().zip(row_id_col.values().iter()) { + let Some(doc) = doc else { + continue; + }; + + self.process_document(*row_id, DocumentSource::StringList(doc.as_ref()), &[]) + .await?; + } + + Ok(()) + } + + fn checked_token_position(row_id: u64, token_position: usize) -> Result { + u32::try_from(token_position).map_err(|_| { + Error::invalid_input(format!( + "token position overflow for row_id={row_id}: token_position={token_position}" + )) + }) + } + + fn materialize_string_list(elements: &dyn Array) -> String { + let mut doc = String::new(); + for element in iter_str_array(elements).flatten() { + if !doc.is_empty() { + doc.push(' '); + } + doc.push_str(element); + } + doc + } + + async fn process_document( + &mut self, + row_id: u64, + document: DocumentSource<'_>, + doc_index: &[u32], + ) -> Result<()> { let with_position = self.has_position(); - for (doc, row_id) in docs { - let builder_was_empty = self.builder.docs.is_empty(); - let old_temporary_memory_size = self.temporary_memory_size(); - let old_token_memory_size = self.builder.tokens.memory_size() as u64; - let doc_id = self.builder.docs.len() as u32; - let mut token_num: u32 = 0; - let mut posting_memory_delta = 0i64; - if with_position { + let builder_was_empty = self.builder.docs.is_empty(); + let old_temporary_memory_size = self.temporary_memory_size(); + let old_token_memory_size = self.builder.tokens.memory_size() as u64; + let doc_id = self.builder.docs.len() as u32; + let mut token_num: u32 = 0; + let mut doc_length_bytes = 0usize; + let mut posting_memory_delta = 0i64; + if with_position { + { if self.token_ids.capacity() < self.last_token_count { self.token_ids .reserve(self.last_token_count - self.token_ids.capacity()); } self.token_ids.clear(); + let tokenizer = &mut self.tokenizer; let builder = &mut self.builder; let token_ids = &mut self.token_ids; let memory_size = &mut self.memory_size; let posting_tail_codec = builder.posting_tail_codec; - let mut token_stream = self.tokenizer.token_stream_for_doc(doc); - while token_stream.advance() { - let token = token_stream.token(); - let token_id = builder.tokens.get_or_add(&token.text); - if token_id as usize == builder.posting_lists.len() { - let old_posting_lists_overhead_size = (builder.posting_lists.capacity() - * std::mem::size_of::()) - as u64; - builder.posting_lists.push( - PostingListBuilder::new_with_posting_tail_codec( - true, - posting_tail_codec, - ), - ); - let new_posting_lists_overhead_size = (builder.posting_lists.capacity() - * std::mem::size_of::()) - as u64; - Self::adjust_tracked_value( - memory_size, - old_posting_lists_overhead_size, - new_posting_lists_overhead_size, - ); + let block_size = builder.block_size; + let mut process_text = |text: &str| -> Result<()> { + doc_length_bytes += text.len(); + let mut token_stream = tokenizer.token_stream_for_doc(text); + while token_stream.advance() { + let token = token_stream.token(); + let position = Self::checked_token_position(row_id, token.position)?; + let token_id = builder.tokens.get_or_add(&token.text); + if token_id as usize == builder.posting_lists.len() { + let old_posting_lists_overhead_size = (builder.posting_lists.capacity() + * std::mem::size_of::()) + as u64; + builder.posting_lists.push( + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + true, + posting_tail_codec, + block_size, + ), + ); + let new_posting_lists_overhead_size = (builder.posting_lists.capacity() + * std::mem::size_of::()) + as u64; + Self::adjust_tracked_value( + memory_size, + old_posting_lists_overhead_size, + new_posting_lists_overhead_size, + ); + } + let posting_list = &mut builder.posting_lists[token_id as usize]; + let old_posting_memory_size = posting_list.size(); + if posting_list.add_occurrence(doc_id, position)? { + token_ids.push(token_id); + } + let new_posting_memory_size = posting_list.size(); + posting_memory_delta += + new_posting_memory_size as i64 - old_posting_memory_size as i64; + token_num += 1; } - let posting_list = &mut builder.posting_lists[token_id as usize]; - let old_posting_memory_size = posting_list.size(); - if posting_list.add_occurrence(doc_id, token.position as u32)? { - token_ids.push(token_id); + Ok(()) + }; + + match document { + DocumentSource::Text(doc) => { + process_text(doc)?; + } + DocumentSource::StringList(elements) => { + let doc = Self::materialize_string_list(elements); + process_text(&doc)?; } - let new_posting_memory_size = posting_list.size(); - posting_memory_delta += - new_posting_memory_size as i64 - old_posting_memory_size as i64; - token_num += 1; } - } else { + } + } else { + { if self.token_ids.capacity() < self.last_token_count { self.token_ids .reserve(self.last_token_count - self.token_ids.capacity()); } self.token_ids.clear(); - let mut token_stream = self.tokenizer.token_stream_for_doc(doc); - while token_stream.advance() { - let token_id = self.builder.tokens.get_or_add(&token_stream.token().text); - self.token_ids.push(token_id); - token_num += 1; - } - } - self.adjust_tracked_memory_size( - old_token_memory_size, - self.builder.tokens.memory_size() as u64, - ); + let tokenizer = &mut self.tokenizer; + let builder = &mut self.builder; + let token_ids = &mut self.token_ids; + let mut process_text = |text: &str| { + doc_length_bytes += text.len(); + let mut token_stream = tokenizer.token_stream_for_doc(text); + while token_stream.advance() { + let token_id = builder.tokens.get_or_add(&token_stream.token().text); + token_ids.push(token_id); + token_num += 1; + } + }; - if !with_position { - let old_posting_lists_overhead_size = self.posting_lists_overhead_size(); - self.builder - .posting_lists - .resize_with(self.builder.tokens.len(), || { - PostingListBuilder::new_with_posting_tail_codec( - false, - self.builder.posting_tail_codec, - ) - }); - let new_posting_lists_overhead_size = self.posting_lists_overhead_size(); - Self::adjust_tracked_value( - &mut self.memory_size, - old_posting_lists_overhead_size, - new_posting_lists_overhead_size, - ); + match document { + DocumentSource::Text(doc) => process_text(doc), + DocumentSource::StringList(elements) => { + let doc = Self::materialize_string_list(elements); + process_text(&doc); + } + } } + } + self.adjust_tracked_memory_size( + old_token_memory_size, + self.builder.tokens.memory_size() as u64, + ); - let old_doc_memory_size = self.builder.docs.memory_size() as u64; - let appended_doc_id = self.builder.docs.append(row_id, token_num); - debug_assert_eq!(appended_doc_id, doc_id); + // Row indexes omit zero-token documents from corpus statistics. + // ListElement indexes retain them so their physical coordinates remain + // part of the document corpus even when they cannot match a term. + if token_num == 0 && self.coordinate_rank == 0 { + self.last_token_count = 0; + self.trim_temporary_buffers(); self.adjust_tracked_memory_size( - old_doc_memory_size, - self.builder.docs.memory_size() as u64, + old_temporary_memory_size, + self.temporary_memory_size(), ); - self.total_doc_length += doc.len(); + return Ok(()); + } - if with_position { - for &token_id in &self.token_ids { - let (old_posting_memory_size, new_posting_memory_size) = { - let posting_list = &mut self.builder.posting_lists[token_id as usize]; - let old_posting_memory_size = posting_list.size(); - posting_list.finish_open_doc(doc_id)?; - let new_posting_memory_size = posting_list.size(); - (old_posting_memory_size, new_posting_memory_size) - }; - posting_memory_delta += - new_posting_memory_size as i64 - old_posting_memory_size as i64; - } - Self::apply_delta(&mut self.memory_size, posting_memory_delta); - } else if token_num > 0 { - self.token_ids.sort_unstable(); - let mut iter = self.token_ids.iter(); - let mut current = *iter.next().unwrap(); - let mut count = 1u32; - for &token_id in iter { - if token_id == current { - count += 1; - continue; - } + if !with_position { + let old_posting_lists_overhead_size = self.posting_lists_overhead_size(); + self.builder + .posting_lists + .resize_with(self.builder.tokens.len(), || { + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + false, + self.builder.posting_tail_codec, + self.builder.block_size, + ) + }); + let new_posting_lists_overhead_size = self.posting_lists_overhead_size(); + Self::adjust_tracked_value( + &mut self.memory_size, + old_posting_lists_overhead_size, + new_posting_lists_overhead_size, + ); + } - let (old_posting_memory_size, new_posting_memory_size) = { - let posting_list = &mut self.builder.posting_lists[current as usize]; - let old_posting_memory_size = posting_list.size(); - posting_list.add(doc_id, PositionRecorder::Count(count)); - let new_posting_memory_size = posting_list.size(); - (old_posting_memory_size, new_posting_memory_size) - }; - posting_memory_delta += - new_posting_memory_size as i64 - old_posting_memory_size as i64; + let old_doc_memory_size = self.builder.docs.memory_size() as u64; + let appended_doc_id = if doc_index.is_empty() { + self.builder.docs.append(row_id, token_num) + } else { + self.builder + .docs + .append_with_doc_index(row_id, token_num, doc_index)? + }; + debug_assert_eq!(appended_doc_id, doc_id); + self.adjust_tracked_memory_size( + old_doc_memory_size, + self.builder.docs.memory_size() as u64, + ); + self.total_doc_length += doc_length_bytes; - current = token_id; - count = 1; + if with_position { + for &token_id in &self.token_ids { + let (old_posting_memory_size, new_posting_memory_size) = { + let posting_list = &mut self.builder.posting_lists[token_id as usize]; + let old_posting_memory_size = posting_list.size(); + posting_list.finish_open_doc(doc_id)?; + let new_posting_memory_size = posting_list.size(); + (old_posting_memory_size, new_posting_memory_size) + }; + posting_memory_delta += + new_posting_memory_size as i64 - old_posting_memory_size as i64; + } + Self::apply_delta(&mut self.memory_size, posting_memory_delta); + } else if token_num > 0 { + self.token_ids.sort_unstable(); + let mut iter = self.token_ids.iter(); + let mut current = *iter.next().unwrap(); + let mut count = 1u32; + for &token_id in iter { + if token_id == current { + count += 1; + continue; } + let (old_posting_memory_size, new_posting_memory_size) = { let posting_list = &mut self.builder.posting_lists[current as usize]; let old_posting_memory_size = posting_list.size(); @@ -1499,27 +1742,35 @@ impl IndexWorker { }; posting_memory_delta += new_posting_memory_size as i64 - old_posting_memory_size as i64; - Self::apply_delta(&mut self.memory_size, posting_memory_delta); - } - self.last_token_count = self.token_ids.len(); - self.trim_temporary_buffers(); - self.adjust_tracked_memory_size( - old_temporary_memory_size, - self.temporary_memory_size(), - ); - if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes { - return Err(Error::invalid_input(format!( - "single document row_id={} exceeds worker memory limit: {} > {} bytes", - row_id, self.memory_size, self.worker_memory_limit_bytes - ))); + current = token_id; + count = 1; } + let (old_posting_memory_size, new_posting_memory_size) = { + let posting_list = &mut self.builder.posting_lists[current as usize]; + let old_posting_memory_size = posting_list.size(); + posting_list.add(doc_id, PositionRecorder::Count(count)); + let new_posting_memory_size = posting_list.size(); + (old_posting_memory_size, new_posting_memory_size) + }; + posting_memory_delta += new_posting_memory_size as i64 - old_posting_memory_size as i64; + Self::apply_delta(&mut self.memory_size, posting_memory_delta); + } + self.last_token_count = self.token_ids.len(); + self.trim_temporary_buffers(); + self.adjust_tracked_memory_size(old_temporary_memory_size, self.temporary_memory_size()); - if self.builder.docs.len() as u32 == u32::MAX - || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes) - { - self.flush().await?; - } + if self.builder.docs.len() == 1 && self.memory_size > self.worker_memory_limit_bytes { + return Err(Error::invalid_input(format!( + "single document row_id={} exceeds worker memory limit: {} > {} bytes", + row_id, self.memory_size, self.worker_memory_limit_bytes + ))); + } + + if self.builder.docs.len() as u32 == u32::MAX + || (!builder_was_empty && self.memory_size >= self.worker_memory_limit_bytes) + { + self.flush().await?; } Ok(()) @@ -1527,7 +1778,7 @@ impl IndexWorker { #[instrument(level = "debug", skip_all)] async fn flush(&mut self) -> Result<()> { - if self.builder.tokens.is_empty() { + if self.builder.docs.is_empty() { return Ok(()); } @@ -1538,17 +1789,18 @@ impl IndexWorker { self.memory_size = self.temporary_memory_size(); let with_position = self.has_position(); let format_version = self.builder.format_version; - let builder = std::mem::replace( - &mut self.builder, - InnerBuilder::new_with_format_version( - self.id_alloc - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - | self.fragment_mask.unwrap_or(0), - with_position, - self.token_set_format, - format_version, - ), + let block_size = self.builder.block_size; + let mut replacement = InnerBuilder::new_with_format_version_and_block_size( + self.id_alloc + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + | self.fragment_mask.unwrap_or(0), + with_position, + self.token_set_format, + format_version, + block_size, ); + replacement.docs = DocSet::with_coordinate_rank(self.coordinate_rank); + let builder = std::mem::replace(&mut self.builder, replacement); let written_partition_id = builder.id(); let mut builder = builder; let target = if self.fragment_mask.is_some() { @@ -1571,7 +1823,7 @@ impl IndexWorker { } async fn finish(self) -> Result { - let tail_partition = if self.builder.tokens.is_empty() { + let tail_partition = if self.builder.docs.is_empty() && self.coordinate_rank == 0 { None } else { Some(TailPartition { @@ -1615,6 +1867,7 @@ impl PositionRecorder { #[derive(Debug, Eq, PartialEq, Clone, DeepSizeOf)] pub struct ScoredDoc { pub row_id: u64, + pub doc_index: Vec, pub score: OrderedFloat, } @@ -1622,6 +1875,15 @@ impl ScoredDoc { pub fn new(row_id: u64, score: f32) -> Self { Self { row_id, + doc_index: Vec::new(), + score: OrderedFloat(score), + } + } + + pub fn with_doc_index(row_id: u64, doc_index: Vec, score: f32) -> Self { + Self { + row_id, + doc_index, score: OrderedFloat(score), } } @@ -1666,17 +1928,56 @@ pub fn inverted_list_schema_for_version( with_position: bool, format_version: InvertedListFormatVersion, ) -> SchemaRef { + inverted_list_schema_for_version_with_block_size( + with_position, + format_version, + LEGACY_BLOCK_SIZE, + ) +} + +pub fn inverted_list_schema_for_version_with_block_size( + with_position: bool, + format_version: InvertedListFormatVersion, + block_size: usize, +) -> SchemaRef { + inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + format_version, + block_size, + true, + ) +} + +pub(crate) fn inverted_list_schema_for_version_with_block_size_and_impacts( + with_position: bool, + format_version: InvertedListFormatVersion, + block_size: usize, + with_impacts: bool, +) -> SchemaRef { + validate_format_version_block_size(format_version, block_size) + .expect("invalid FTS format version for posting block size"); match format_version { - InvertedListFormatVersion::V1 => inverted_list_schema_v1(with_position), - InvertedListFormatVersion::V2 => inverted_list_schema_with_tail_codec_and_position_codec( - with_position, - PostingTailCodec::VarintDelta, - Some(PositionStreamCodec::PackedDelta), - ), + InvertedListFormatVersion::V1 => { + inverted_list_schema_v1(with_position, block_size, with_impacts) + } + InvertedListFormatVersion::V2 | InvertedListFormatVersion::V3 => { + inverted_list_schema_with_tail_codec_and_position_codec( + with_position, + format_version, + PostingTailCodec::VarintDelta, + Some(PositionStreamCodec::PackedDelta), + block_size, + with_impacts, + ) + } } } -fn inverted_list_schema_v1(with_position: bool) -> SchemaRef { +fn inverted_list_schema_v1( + with_position: bool, + block_size: usize, + with_impacts: bool, +) -> SchemaRef { let mut fields = vec![ arrow_schema::Field::new( POSTING_COL, @@ -1690,6 +1991,17 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef { arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false), arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false), ]; + if with_impacts { + fields.push(arrow_schema::Field::new( + IMPACT_COL, + datatypes::DataType::List(Arc::new(Field::new( + "item", + datatypes::DataType::LargeBinary, + true, + ))), + false, + )); + } if with_position { fields.push(arrow_schema::Field::new( POSITION_COL, @@ -1705,24 +2017,44 @@ fn inverted_list_schema_v1(with_position: bool) -> SchemaRef { false, )); } - Arc::new(arrow_schema::Schema::new(fields)) + Arc::new(arrow_schema::Schema::new_with_metadata( + fields, + HashMap::from([ + (POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()), + ( + FTS_FORMAT_VERSION_KEY.to_owned(), + InvertedListFormatVersion::V1.index_version().to_string(), + ), + ]), + )) } pub fn inverted_list_schema_with_tail_codec( with_position: bool, posting_tail_codec: PostingTailCodec, ) -> SchemaRef { + let format_version = InvertedListFormatVersion::from_posting_tail_codec_and_block_size( + posting_tail_codec, + LEGACY_BLOCK_SIZE, + ) + .expect("invalid posting tail codec for posting block size"); inverted_list_schema_with_tail_codec_and_position_codec( with_position, + format_version, posting_tail_codec, Some(PositionStreamCodec::PackedDelta), + LEGACY_BLOCK_SIZE, + false, ) } fn inverted_list_schema_with_tail_codec_and_position_codec( with_position: bool, + format_version: InvertedListFormatVersion, posting_tail_codec: PostingTailCodec, position_codec: Option, + block_size: usize, + with_impacts: bool, ) -> SchemaRef { let mut fields = vec![ // we compress the posting lists (including row ids and frequencies), @@ -1739,6 +2071,17 @@ fn inverted_list_schema_with_tail_codec_and_position_codec( arrow_schema::Field::new(MAX_SCORE_COL, datatypes::DataType::Float32, false), arrow_schema::Field::new(LENGTH_COL, datatypes::DataType::UInt32, false), ]; + if with_impacts { + fields.push(arrow_schema::Field::new( + IMPACT_COL, + datatypes::DataType::List(Arc::new(Field::new( + "item", + datatypes::DataType::LargeBinary, + true, + ))), + false, + )); + } if with_position { fields.push(arrow_schema::Field::new( COMPRESSED_POSITION_COL, @@ -1759,6 +2102,11 @@ fn inverted_list_schema_with_tail_codec_and_position_codec( POSTING_TAIL_CODEC_KEY.to_owned(), posting_tail_codec.as_str().to_owned(), )]); + metadata.insert( + FTS_FORMAT_VERSION_KEY.to_owned(), + format_version.index_version().to_string(), + ); + metadata.insert(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string()); if let Some(position_codec) = position_codec.filter(|_| with_position) { metadata.insert( POSITIONS_LAYOUT_KEY.to_owned(), @@ -1772,129 +2120,6 @@ fn inverted_list_schema_with_tail_codec_and_position_codec( Arc::new(arrow_schema::Schema::new_with_metadata(fields, metadata)) } -/// Flatten the string list stream into a string stream -pub struct FlattenStream { - /// Inner record batch stream with 2 columns: - /// 1. doc_col: List(Utf8) or List(LargeUtf8) - /// 2. row_id_col: UInt64 - inner: SendableRecordBatchStream, - field_type: DataType, - data_type: DataType, -} - -impl FlattenStream { - pub fn new(input: SendableRecordBatchStream) -> Self { - let schema = input.schema(); - let field = schema.field(0); - let data_type = match field.data_type() { - DataType::List(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8, - DataType::List(f) if matches!(f.data_type(), DataType::LargeUtf8) => { - DataType::LargeUtf8 - } - DataType::LargeList(f) if matches!(f.data_type(), DataType::Utf8) => DataType::Utf8, - DataType::LargeList(f) if matches!(f.data_type(), DataType::LargeUtf8) => { - DataType::LargeUtf8 - } - _ => panic!( - "expect data type List(Utf8) or List(LargeUtf8) but got {:?}", - field.data_type() - ), - }; - Self { - inner: input, - field_type: field.data_type().clone(), - data_type, - } - } -} - -impl Stream for FlattenStream { - type Item = datafusion_common::Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - match Pin::new(&mut self.inner).poll_next(cx) { - Poll::Ready(Some(Ok(batch))) => { - let doc_col = batch.column(0); - let batch = match self.field_type { - DataType::List(_) => flatten_string_list::(&batch, doc_col).map_err(|e| { - datafusion_common::error::DataFusionError::Execution(format!( - "flatten string list error: {}", - e - )) - }), - DataType::LargeList(_) => { - flatten_string_list::(&batch, doc_col).map_err(|e| { - datafusion_common::error::DataFusionError::Execution(format!( - "flatten string list error: {}", - e - )) - }) - } - _ => unreachable!( - "expect data type List or LargeList but got {:?}", - self.field_type - ), - }; - Poll::Ready(Some(batch)) - } - Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -impl RecordBatchStream for FlattenStream { - fn schema(&self) -> SchemaRef { - let schema = Schema::new(vec![ - Field::new( - self.inner.schema().field(0).name(), - self.data_type.clone(), - true, - ), - ROW_ID_FIELD.clone(), - ]); - - Arc::new(schema) - } -} - -fn flatten_string_list( - batch: &RecordBatch, - doc_col: &Arc, -) -> Result { - let docs = doc_col.as_list::(); - let row_ids = batch[ROW_ID].as_primitive::(); - - let row_ids = row_ids - .values() - .iter() - .zip(docs.iter()) - .flat_map(|(row_id, doc)| std::iter::repeat_n(*row_id, doc.map(|d| d.len()).unwrap_or(0))); - - let row_ids = Arc::new(UInt64Array::from_iter_values(row_ids)); - let docs = match docs.value_type() { - datatypes::DataType::Utf8 | datatypes::DataType::LargeUtf8 => docs.values().clone(), - _ => { - return Err(Error::index(format!( - "expect data type String or LargeString but got {}", - docs.value_type() - ))); - } - }; - - let schema = Schema::new(vec![ - Field::new( - batch.schema().field(0).name(), - docs.data_type().clone(), - true, - ), - ROW_ID_FIELD.clone(), - ]); - let batch = RecordBatch::try_new(Arc::new(schema), vec![docs, row_ids])?; - Ok(batch) -} - pub(crate) fn token_file_path(partition_id: u64) -> String { format!("part_{}_{}", partition_id, TOKENS_FILE) } @@ -1911,6 +2136,10 @@ pub(crate) fn part_metadata_file_path(partition_id: u64) -> String { staged_partition_file_path(partition_id, METADATA_FILE) } +fn empty_part_metadata_file_path(fragment_mask: u64) -> String { + format!("{STAGED_PARTITION_DIR}/part_empty_{fragment_mask}_{METADATA_FILE}") +} + const PARTITION_FILE_SUFFIXES: [&str; 3] = [TOKENS_FILE, INVERT_LIST_FILE, DOCS_FILE]; const STAGED_PARTITION_DIR: &str = "staging"; @@ -2015,7 +2244,6 @@ async fn merge_metadata_files( let mut params = None; let mut token_set_format = None; let mut format_version = None; - let mut posting_tail_codec = None; let mut deleted_fragments = RoaringBitmap::new(); progress .stage_start( @@ -2057,9 +2285,6 @@ async fn merge_metadata_files( if format_version.is_none() { format_version = Some(parse_format_version_from_metadata(metadata)?); } - if posting_tail_codec.is_none() { - posting_tail_codec = Some(parse_posting_tail_codec(metadata)?); - } if reader.num_rows() > 0 { let metadata_batch = reader.read_range(0..1, None).await?; @@ -2125,8 +2350,7 @@ async fn merge_metadata_files( None, deleted_fragments, ) - .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1)) - .with_posting_tail_codec(posting_tail_codec.unwrap_or(PostingTailCodec::Fixed32)); + .with_format_version(format_version.unwrap_or(InvertedListFormatVersion::V1)); progress .stage_start("write_merged_metadata", Some(1), "files") .await?; @@ -2164,11 +2388,11 @@ pub fn document_input( let schema = input.schema(); let field = schema.column_with_name(column).expect_ok()?.1; match field.data_type() { - DataType::Utf8 | DataType::LargeUtf8 => Ok(input), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(input), DataType::List(field) | DataType::LargeList(field) if matches!(field.data_type(), DataType::Utf8 | DataType::LargeUtf8) => { - Ok(Box::pin(FlattenStream::new(input))) + Ok(input) } DataType::LargeBinary => match field.metadata().get(ARROW_EXT_NAME_KEY) { Some(name) if name.as_str() == JSON_EXT_NAME => { @@ -2192,10 +2416,12 @@ pub fn document_input( #[cfg(test)] mod tests { use super::*; + use crate::Index; use crate::metrics::NoOpMetricsCollector; use crate::progress::IndexBuildProgress; + use crate::scalar::inverted::{MemBM25Scorer, Scorer}; use crate::scalar::{IndexFile, IndexReader, IndexWriter, ScalarIndex}; - use arrow_array::{RecordBatch, StringArray, UInt64Array}; + use arrow_array::{RecordBatch, StringArray, UInt32Array, UInt64Array}; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; use bytes::Bytes; @@ -2226,6 +2452,17 @@ mod tests { RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap() } + fn make_doc_batch_from_docs(docs: Vec>) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("doc", DataType::Utf8, true), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let num_rows = docs.len(); + let docs = Arc::new(StringArray::from(docs)); + let row_ids = Arc::new(UInt64Array::from_iter_values(0..num_rows as u64)); + RecordBatch::try_new(schema, vec![docs, row_ids]).unwrap() + } + struct FailingListObjectStore { inner: InMemory, } @@ -2309,6 +2546,77 @@ mod tests { } } + #[derive(Debug)] + struct CopyFailingObjectStore { + inner: InMemory, + copy_count: Arc, + } + + impl Display for CopyFailingObjectStore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "CopyFailingObjectStore") + } + } + + #[async_trait] + impl OSObjectStore for CopyFailingObjectStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + self.inner.put_opts(location, bytes, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + self.inner.get_opts(location, options).await + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + self.inner.get_ranges(location, ranges).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + self.inner.list(prefix) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, OSResult> { + self.inner.list_with_offset(prefix, offset) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> { + self.copy_count.fetch_add(1, Ordering::SeqCst); + Err(object_store::Error::Generic { + store: "CopyFailingObjectStore", + source: "native copy disabled in test".into(), + }) + } + } + #[tokio::test] async fn test_list_metadata_files_propagates_list_error() -> Result<()> { let mut object_store = ObjectStore::memory(); @@ -2613,8 +2921,7 @@ mod tests { } async fn add_global_buffer(&mut self, _data: Bytes) -> Result { - // The posting-list writer stores the group offsets as a global - // buffer; mirror the real writer's 1-indexed return value. + // Mirror the real writer's 1-indexed return value. Ok(1) } @@ -2699,63 +3006,6 @@ mod tests { } } - fn collect_group_starts(config: PostingGroupConfig, sizes: &[usize]) -> Vec { - let mut acc = PostingGroupAccumulator::new(config); - for &size in sizes { - acc.push(size); - } - acc.into_starts() - } - - #[test] - fn test_group_accumulator_cuts_on_target_bytes() { - let config = PostingGroupConfig { - target_bytes: 100, - max_tokens: 1000, - }; - // 40+40 -> cut at 80? no, 80 < 100; third 40 reaches 120 >= 100 -> cut. - // So group 0 = tokens [0,3), then a new group starts at token 3. - let starts = collect_group_starts(config, &[40, 40, 40, 10, 10]); - assert_eq!(starts, vec![0, 3]); - } - - #[test] - fn test_group_accumulator_cuts_on_max_tokens() { - let config = PostingGroupConfig { - target_bytes: 1_000_000, - max_tokens: 2, - }; - // Byte target never reached; cap of 2 forces a cut every 2 tokens. - let starts = collect_group_starts(config, &[1, 1, 1, 1, 1]); - assert_eq!(starts, vec![0, 2, 4]); - } - - #[test] - fn test_group_accumulator_clamps_oversized_term() { - let config = PostingGroupConfig { - target_bytes: 100, - max_tokens: 64, - }; - // A term larger than the target that *starts* a group occupies that - // group alone ([1, 2) here), so a single huge posting list is never - // forced to share a cache entry. Token 0 (==100) closes its own group - // first; the trailing small terms regroup after the big one. - let starts = collect_group_starts(config, &[100, 5000, 10, 10]); - assert_eq!(starts, vec![0, 1, 2]); - - // A huge term encountered mid-group is absorbed and closes that group; - // we never split one term across groups. - let starts = collect_group_starts(config, &[10, 10, 5000, 10, 10]); - assert_eq!(starts, vec![0, 3]); - } - - #[test] - fn test_group_accumulator_empty_and_single() { - let config = PostingGroupConfig::default(); - assert_eq!(collect_group_starts(config, &[]), Vec::::new()); - assert_eq!(collect_group_starts(config, &[10]), vec![0]); - } - #[tokio::test] async fn test_write_posting_lists_batches_multiple_rows() -> Result<()> { let mut builder = InnerBuilder::new(0, false, TokenSetFormat::default()); @@ -2890,6 +3140,10 @@ mod tests { expected_partitions.dedup(); let remapped_partitions = (0..expected_partitions.len() as u64).collect::>(); assert_eq!(written_partitions, remapped_partitions); + assert_eq!( + parse_format_version_from_metadata(metadata)?, + InvertedListFormatVersion::V2 + ); for (new_id, old_id) in expected_partitions.iter().enumerate() { assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?; @@ -2914,6 +3168,99 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_fts_remap_streams_files_when_native_copy_fails() -> Result<()> { + let copy_count = Arc::new(AtomicUsize::new(0)); + let mut object_store = ObjectStore::memory(); + object_store.inner = Arc::new(CopyFailingObjectStore { + inner: InMemory::new(), + copy_count: copy_count.clone(), + }); + let object_store = Arc::new(object_store); + let index_path = Path::from("index"); + let base_store: Arc = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_path.clone(), + Arc::new(LanceCache::no_cache()), + )); + let store = Arc::new(NoRenameStore::new(base_store.clone())); + let partitions = vec![5_u64, 1_u64]; + let metadata_builder = InvertedIndexBuilder::from_existing_index( + InvertedIndexParams::default(), + None, + Vec::new(), + TokenSetFormat::default(), + None, + RoaringBitmap::new(), + ); + + for partition_id in &partitions { + write_partition_files( + base_store.as_ref(), + *partition_id, + PartitionWriteTarget::Staged, + ) + .await?; + metadata_builder + .write_part_metadata(base_store.as_ref(), *partition_id) + .await?; + } + + let probe_source = staged_partition_file_path(partitions[0], TOKENS_FILE); + let probe_source_size = base_store + .open_index_file(&probe_source) + .await? + .file_size_bytes() + .expect("written index file should report its size"); + let copied = base_store + .copy_index_file_to(&probe_source, "probe.lance", base_store.as_ref()) + .await?; + assert_eq!(copied.path, "probe.lance"); + assert_eq!(copied.size_bytes, probe_source_size); + assert_eq!( + read_partition_file_marker(base_store.as_ref(), "probe.lance").await?, + partitions[0] + ); + + let renamed = base_store + .rename_index_file("probe.lance", "renamed-probe.lance") + .await?; + assert_eq!(renamed.path, "renamed-probe.lance"); + assert_eq!(renamed.size_bytes, probe_source_size); + assert!(base_store.open_index_file("probe.lance").await.is_err()); + assert_eq!( + read_partition_file_marker(base_store.as_ref(), "renamed-probe.lance").await?, + partitions[0] + ); + + let progress = Arc::new(RecordingProgress::default()); + merge_index_files(object_store.as_ref(), &index_path, store, progress.clone()).await?; + + let mut expected_partitions = partitions; + expected_partitions.sort_unstable(); + for (new_id, old_id) in expected_partitions.iter().enumerate() { + assert_partition_file_markers(base_store.as_ref(), new_id as u64, *old_id).await?; + } + let remap_progress = progress + .recorded_events() + .into_iter() + .filter_map(|(kind, stage, completed)| { + (kind == "progress" && stage == "remap_partition_files").then_some(completed) + }) + .collect::>(); + assert_eq!( + remap_progress.last().copied(), + Some((expected_partitions.len() * PARTITION_FILE_SUFFIXES.len()) as u64) + ); + assert_eq!( + copy_count.load(Ordering::SeqCst), + 0, + "bulk index movement must not invoke native object-store copy" + ); + + Ok(()) + } + #[tokio::test] async fn test_merge_index_files_rewrites_partial_final_files_from_staging() -> Result<()> { let index_dir = TempDir::default(); @@ -3179,6 +3526,80 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_distributed_empty_build_does_not_finalize_shared_directory() -> Result<()> { + let index_dir = TempDir::default(); + let object_store = Arc::new(ObjectStore::local()); + let store = Arc::new(LanceIndexStore::new( + object_store.clone(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let empty_fragment_mask = 7_u64 << 32; + let batch = make_doc_batch_from_docs(vec![None, None]); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = InvertedIndexParams { + lance_tokenizer: Some("text".to_string()), + with_position: true, + ..Default::default() + }; + let mut builder = + InvertedIndexBuilder::new_with_fragment_mask(params.clone(), Some(empty_fragment_mask)); + + let files = builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + assert_eq!(files.len(), 1); + let empty_metadata_path = empty_part_metadata_file_path(empty_fragment_mask); + assert_eq!(files[0].path, empty_metadata_path); + assert!( + store.open_index_file(METADATA_FILE).await.is_err(), + "an empty shard must not finalize the shared directory" + ); + let reader = store.open_index_file(&empty_metadata_path).await?; + let metadata = &reader.schema().metadata; + let partitions: Vec = serde_json::from_str( + metadata + .get("partitions") + .expect("partitions missing from metadata"), + )?; + assert!(partitions.is_empty()); + let written_params: InvertedIndexParams = serde_json::from_str( + metadata + .get("params") + .expect("params missing from metadata"), + )?; + assert_eq!(written_params, params); + + let non_empty_fragment_mask = 8_u64 << 32; + let batch = make_doc_batch("searchable text", non_empty_fragment_mask); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let mut builder = + InvertedIndexBuilder::new_with_fragment_mask(params, Some(non_empty_fragment_mask)); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let staged_metadata = + list_metadata_files(object_store.as_ref(), &index_dir.obj_path()).await?; + assert_eq!(staged_metadata.len(), 2); + + merge_index_files( + object_store.as_ref(), + &index_dir.obj_path(), + store.clone(), + noop_progress(), + ) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + assert_eq!(index.partition_count(), 1); + + Ok(()) + } + #[tokio::test] async fn test_merge_index_files_is_noop_when_metadata_exists() -> Result<()> { let index_dir = TempDir::default(); @@ -3241,6 +3662,8 @@ mod tests { fragment_mask: None, token_set_format, worker_memory_limit_bytes: u64::MAX, + block_size: params.block_size, + coordinate_rank: 0, }, ) .await?; @@ -3264,6 +3687,8 @@ mod tests { fragment_mask: None, token_set_format, worker_memory_limit_bytes: u64::MAX, + block_size: params.block_size, + coordinate_rank: 0, }, ) .await?; @@ -3411,6 +3836,39 @@ mod tests { assert_eq!(builder.posting_tail_codec, PostingTailCodec::VarintDelta); } + #[test] + fn test_v3_128_reuses_v2_physical_layout() { + for with_position in [false, true] { + for with_impacts in [false, true] { + let v2 = inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + InvertedListFormatVersion::V2, + LEGACY_BLOCK_SIZE, + with_impacts, + ); + let v3 = inverted_list_schema_for_version_with_block_size_and_impacts( + with_position, + InvertedListFormatVersion::V3, + LEGACY_BLOCK_SIZE, + with_impacts, + ); + + assert_eq!(v2.fields(), v3.fields()); + let mut v2_metadata = v2.metadata.clone(); + let mut v3_metadata = v3.metadata.clone(); + assert_eq!( + v2_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(), + Some("2") + ); + assert_eq!( + v3_metadata.remove(FTS_FORMAT_VERSION_KEY).as_deref(), + Some("3") + ); + assert_eq!(v2_metadata, v3_metadata); + } + } + } + #[tokio::test] async fn test_inverted_index_without_positions_tracks_frequency() -> Result<()> { let index_dir = TempDir::default(); @@ -3459,6 +3917,177 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_zero_token_string_documents_are_skipped_in_corpus_stats() -> Result<()> { + let index_dir = TempDir::default(); + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let batch = make_doc_batch_from_docs(vec![ + Some(""), + Some(" "), + Some("the"), + Some("overlength"), + None, + Some("hello"), + ]); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .with_position(false) + .remove_stop_words(true) + .stem(false) + .max_token_length(Some(6)) + .num_workers(1); + + let mut builder = InvertedIndexBuilder::new(params); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + let (total_tokens, num_docs, token_docs) = index + .bm25_stats_for_terms(&["hello".to_string()], None) + .await?; + assert_eq!(total_tokens, 1); + assert_eq!(num_docs, 1); + assert_eq!(token_docs, vec![1]); + + let actual_scorer = MemBM25Scorer::new( + total_tokens, + num_docs, + HashMap::from([("hello".to_string(), token_docs[0])]), + ); + let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)])); + assert_eq!( + actual_scorer.avg_doc_length(), + expected_scorer.avg_doc_length() + ); + assert_eq!( + actual_scorer.query_weight("hello"), + expected_scorer.query_weight("hello") + ); + + Ok(()) + } + + #[tokio::test] + async fn test_all_empty_string_documents_build_empty_index() -> Result<()> { + let index_dir = TempDir::default(); + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + + let batch = make_doc_batch_from_docs(vec![Some(""), Some(" "), None]); + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .with_position(false) + .remove_stop_words(false) + .stem(false) + .max_token_length(None) + .num_workers(1); + + let mut builder = InvertedIndexBuilder::new(params); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + assert!(index.partitions.is_empty()); + let statistics = index.statistics()?; + assert_eq!(statistics["num_tokens"], 0); + assert_eq!(statistics["num_docs"], 0); + + Ok(()) + } + + #[tokio::test] + async fn test_zero_token_coordinate_documents_are_preserved_in_corpus_stats() -> Result<()> { + let index_dir = TempDir::default(); + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + index_dir.obj_path(), + Arc::new(LanceCache::no_cache()), + )); + let schema = Arc::new(Schema::new(vec![ + Field::new("doc", DataType::Utf8, true), + Field::new(doc_index_storage_column(0), DataType::UInt32, false), + Field::new(ROW_ID, DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(StringArray::from(vec![ + None, + Some(""), + Some(" "), + Some("the"), + Some("overlength"), + ])), + Arc::new(UInt32Array::from(vec![0, 1, 2, 3, 4])), + Arc::new(UInt64Array::from(vec![7, 7, 7, 7, 7])), + ], + )?; + let stream = RecordBatchStreamAdapter::new(batch.schema(), stream::iter(vec![Ok(batch)])); + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .with_position(false) + .remove_stop_words(true) + .stem(false) + .max_token_length(Some(6)) + .num_workers(1); + + let mut builder = InvertedIndexBuilder::new(params); + builder + .update(Box::pin(stream), store.as_ref(), None) + .await?; + + let index = InvertedIndex::load(store, None, &LanceCache::no_cache()).await?; + let statistics = index.statistics()?; + assert_eq!(statistics["num_tokens"], 0); + assert_eq!(statistics["num_docs"], 5); + + Ok(()) + } + + #[tokio::test] + async fn test_all_empty_string_documents_do_not_create_tail_partition() -> Result<()> { + let tokenizer = InvertedIndexParams::default().build()?; + let store = Arc::new(CountingStore::new()); + let id_alloc = Arc::new(AtomicU64::new(0)); + let mut worker = IndexWorker::new( + tokenizer, + store, + id_alloc, + IndexWorkerConfig { + with_position: false, + format_version: InvertedListFormatVersion::V1, + fragment_mask: None, + token_set_format: TokenSetFormat::default(), + worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, + }, + ) + .await?; + + worker + .process_batch(make_doc_batch_from_docs(vec![Some(""), Some(" "), None])) + .await?; + let output = worker.finish().await?; + + assert!(output.partitions.is_empty()); + assert!(output.tail_partition.is_none()); + + Ok(()) + } + lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>); #[derive(Debug, Default)] @@ -3746,6 +4375,8 @@ mod tests { fragment_mask: None, token_set_format: TokenSetFormat::default(), worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, }, ) .await?; @@ -3777,6 +4408,8 @@ mod tests { fragment_mask: None, token_set_format: TokenSetFormat::default(), worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, }, ) .await?; @@ -3815,6 +4448,8 @@ mod tests { fragment_mask: None, token_set_format: TokenSetFormat::default(), worker_memory_limit_bytes: u64::MAX, + block_size: InvertedIndexParams::default().block_size, + coordinate_rank: 0, }, ) .await?; @@ -4131,6 +4766,11 @@ mod tests { first.posting_lists.remove(1); assert_eq!(first.tokens.len(), first.posting_lists.len()); + // Mimic a token set persisted by a writer from before #7115. Converting the + // loaded set for mutation must restore the dense token-id invariant. + first.tokens.next_id = 9; + first.tokens = std::mem::take(&mut first.tokens).into_mutable(); + // `second` contributes a brand-new token absent from `first`. Before the fix, // get_or_add returned the stale next_id, indexing past posting_lists. let mut second = InnerBuilder::new(1, false, TokenSetFormat::default()); diff --git a/rust/lance-index/src/scalar/inverted/cache_codec.rs b/rust/lance-index/src/scalar/inverted/cache_codec.rs index a676455d5c9..3ea4557cb1a 100644 --- a/rust/lance-index/src/scalar/inverted/cache_codec.rs +++ b/rust/lance-index/src/scalar/inverted/cache_codec.rs @@ -14,22 +14,26 @@ //! - the compressed posting list: an IPC section for `blocks`, then the //! position sections (legacy IPC, or shared block-offsets IPC + a raw blob of //! the [`SharedPositionStream`] byte buffer, which has its own portable -//! encoding); +//! encoding), then an optional impact IPC section; //! - the plain posting list: an IPC section of `(row_ids, frequencies)`, then //! an optional legacy position IPC section; +//! - a packed posting-list group: one IPC section containing the original +//! `List` posting rows and optional impact rows; prewarmed groups +//! omit score/length metadata and inject it from the posting reader into +//! query-local views; //! - the standalone [`Positions`] codec: the position sections alone. //! //! All sections read back zero-copy via [`lance_arrow::ipc`]. This is the FTS //! counterpart of `partition_serde.rs` for vector indices. -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use arrow_array::cast::AsArray; use arrow_array::types::{Float32Type, UInt32Type, UInt64Type}; use arrow_array::{ Array, Float32Array, LargeBinaryArray, ListArray, RecordBatch, UInt32Array, UInt64Array, }; -use arrow_schema::{DataType, Field, Schema}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter}; use lance_core::{Error, Result}; @@ -39,10 +43,13 @@ use crate::cache_pb::{ PostingTailCodec as PbPostingTailCodec, }; +use super::impact::ImpactSkipData; use super::index::{ CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec, - Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream, + Positions, PostingList, PostingListGroup, PostingListGroupStorage, PostingTailCodec, + SharedPositionStream, }; +use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size}; // --------------------------------------------------------------------------- // Tags @@ -50,6 +57,46 @@ use super::index::{ const POSTING_VARIANT_PLAIN: u8 = 0; const POSTING_VARIANT_COMPRESSED: u8 = 1; +const GROUP_VARIANT_MATERIALIZED: u8 = 0; +const GROUP_VARIANT_PACKED: u8 = 1; + +// --------------------------------------------------------------------------- +// Section schemas +// --------------------------------------------------------------------------- + +// One posting list is written per token, so these are built once rather than +// per section. + +static BLOCK_OFFSETS_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![Field::new( + BLOCK_OFFSETS_COLUMN, + DataType::UInt32, + false, + )])) +}); + +static PLAIN_POSTING_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![ + Field::new(ROW_IDS_COLUMN, DataType::UInt64, false), + Field::new(FREQUENCIES_COLUMN, DataType::Float32, false), + ])) +}); + +static BLOCKS_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![Field::new( + BLOCKS_COLUMN, + DataType::LargeBinary, + false, + )])) +}); + +static IMPACTS_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![Field::new( + IMPACTS_COLUMN, + DataType::LargeBinary, + false, + )])) +}); // --------------------------------------------------------------------------- // Codec enum mappings @@ -72,6 +119,23 @@ fn proto_to_posting_tail_codec(c: PbPostingTailCodec) -> PostingTailCodec { } } +fn posting_tail_codec_to_tag(c: PostingTailCodec) -> u8 { + match c { + PostingTailCodec::Fixed32 => 0, + PostingTailCodec::VarintDelta => 1, + } +} + +fn posting_tail_codec_from_tag(tag: u8) -> Result { + match tag { + 0 => Ok(PostingTailCodec::Fixed32), + 1 => Ok(PostingTailCodec::VarintDelta), + other => Err(Error::io(format!( + "unknown packed posting tail codec: {other}" + ))), + } +} + fn position_stream_codec_to_proto(c: PositionStreamCodec) -> PbPositionStreamCodec { match c { PositionStreamCodec::VarintDocDelta => PbPositionStreamCodec::VarintDocDelta, @@ -95,6 +159,7 @@ const BLOCK_OFFSETS_COLUMN: &str = "block_offsets"; const ROW_IDS_COLUMN: &str = "row_ids"; const FREQUENCIES_COLUMN: &str = "frequencies"; const BLOCKS_COLUMN: &str = "blocks"; +const IMPACTS_COLUMN: &str = "impacts"; fn legacy_positions_batch(list: &ListArray) -> Result { let schema = Arc::new(Schema::new(vec![Field::new( @@ -108,7 +173,8 @@ fn legacy_positions_batch(list: &ListArray) -> Result { fn read_legacy_positions(r: &mut CacheEntryReader<'_>) -> Result { let batch = r.read_ipc()?; Ok(batch - .column(0) + .column_by_name(POSITION_LIST_COLUMN) + .ok_or_else(|| Error::io("legacy position column is missing".to_string()))? .as_any() .downcast_ref::() .ok_or_else(|| Error::io("legacy position column is not a ListArray".to_string()))? @@ -127,12 +193,8 @@ fn write_position_sections( } CompressedPositionStorage::SharedStream(stream) => { let offsets = UInt32Array::from(stream.block_offsets().to_vec()); - let schema = Arc::new(Schema::new(vec![Field::new( - BLOCK_OFFSETS_COLUMN, - DataType::UInt32, - false, - )])); - let batch = RecordBatch::try_new(schema, vec![Arc::new(offsets)])?; + let batch = + RecordBatch::try_new(BLOCK_OFFSETS_SCHEMA.clone(), vec![Arc::new(offsets)])?; w.write_ipc(&batch)?; w.write_raw(stream.bytes())?; } @@ -156,7 +218,8 @@ fn read_position_sections( PbPositionStorage::Shared => { let batch = r.read_ipc()?; let block_offsets = batch - .column(0) + .column_by_name(BLOCK_OFFSETS_COLUMN) + .ok_or_else(|| Error::io("block_offsets column is missing".to_string()))? .as_primitive_opt::() .ok_or_else(|| Error::io("block_offsets column is not UInt32".to_string()))? .values() @@ -178,7 +241,11 @@ fn read_position_sections( impl CacheCodecImpl for PostingList { const TYPE_ID: &'static str = "lance.fts.PostingList"; - const CURRENT_VERSION: u32 = 1; + // Version 3 adds the optional impact IPC section. Main already used v2 for + // configurable posting block sizes, so impact data needs a distinct + // version to keep older readers from accepting a body with an extra + // section they cannot consume. + const CURRENT_VERSION: u32 = 3; fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { match self { @@ -194,15 +261,24 @@ impl CacheCodecImpl for PostingList { } fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { - let variant = r.read_u8()?; - match variant { - POSTING_VARIANT_PLAIN => Ok(Self::Plain(deserialize_plain(r)?)), - POSTING_VARIANT_COMPRESSED => Ok(Self::Compressed(deserialize_compressed(r)?)), - other => Err(Error::io(format!("unknown PostingList variant: {other}"))), + match r.version() { + 1 | 2 | Self::CURRENT_VERSION => deserialize_posting_list_body(r), + other => Err(Error::io(format!( + "unsupported PostingList cache version: {other}" + ))), } } } +fn deserialize_posting_list_body(r: &mut CacheEntryReader<'_>) -> Result { + let variant = r.read_u8()?; + match variant { + POSTING_VARIANT_PLAIN => Ok(PostingList::Plain(deserialize_plain(r)?)), + POSTING_VARIANT_COMPRESSED => Ok(PostingList::Compressed(deserialize_compressed(r)?)), + other => Err(Error::io(format!("unknown PostingList variant: {other}"))), + } +} + fn serialize_plain(w: &mut CacheEntryWriter<'_>, plain: &PlainPostingList) -> Result<()> { // Plain postings carry only per-doc legacy positions (or none). let position_storage = if plain.positions.is_some() { @@ -218,11 +294,10 @@ fn serialize_plain(w: &mut CacheEntryWriter<'_>, plain: &PlainPostingList) -> Re let row_ids = UInt64Array::new(plain.row_ids.clone(), None); let frequencies = Float32Array::new(plain.frequencies.clone(), None); - let schema = Arc::new(Schema::new(vec![ - Field::new(ROW_IDS_COLUMN, DataType::UInt64, false), - Field::new(FREQUENCIES_COLUMN, DataType::Float32, false), - ])); - let batch = RecordBatch::try_new(schema, vec![Arc::new(row_ids), Arc::new(frequencies)])?; + let batch = RecordBatch::try_new( + PLAIN_POSTING_SCHEMA.clone(), + vec![Arc::new(row_ids), Arc::new(frequencies)], + )?; w.write_ipc(&batch)?; if let Some(list) = &plain.positions { @@ -236,13 +311,15 @@ fn deserialize_plain(r: &mut CacheEntryReader<'_>) -> Result { let batch = r.read_ipc()?; let row_ids = batch - .column(0) + .column_by_name(ROW_IDS_COLUMN) + .ok_or_else(|| Error::io("row_ids column is missing".to_string()))? .as_primitive_opt::() .ok_or_else(|| Error::io("row_ids column is not UInt64".to_string()))? .values() .clone(); let frequencies = batch - .column(1) + .column_by_name(FREQUENCIES_COLUMN) + .ok_or_else(|| Error::io("frequencies column is missing".to_string()))? .as_primitive_opt::() .ok_or_else(|| Error::io("frequencies column is not Float32".to_string()))? .values() @@ -291,20 +368,27 @@ fn serialize_compressed( posting_tail_codec: posting_tail_codec_to_proto(posting.posting_tail_codec) as i32, position_storage: position_storage as i32, position_stream_codec: position_stream_codec as i32, + block_size: posting.block_size as u32, + has_impacts: posting.impacts.is_some(), }; w.write_header(&header)?; - let schema = Arc::new(Schema::new(vec![Field::new( - BLOCKS_COLUMN, - DataType::LargeBinary, - false, - )])); - let batch = RecordBatch::try_new(schema, vec![Arc::new(posting.blocks.clone())])?; + let batch = RecordBatch::try_new( + BLOCKS_SCHEMA.clone(), + vec![Arc::new(posting.blocks.clone())], + )?; w.write_ipc(&batch)?; if let Some(storage) = &posting.positions { write_position_sections(w, storage)?; } + if let Some(impacts) = &posting.impacts { + let batch = RecordBatch::try_new( + IMPACTS_SCHEMA.clone(), + vec![Arc::new(impacts.entries().clone())], + )?; + w.write_ipc(&batch)?; + } Ok(()) } @@ -314,7 +398,8 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result() .ok_or_else(|| Error::io("blocks column is not a LargeBinaryArray".to_string()))? @@ -322,13 +407,33 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result= 3 && header.has_impacts { + let batch = r.read_ipc()?; + let entries = batch + .column_by_name(IMPACTS_COLUMN) + .ok_or_else(|| Error::io("impacts column is missing".to_string()))? + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::io("impacts column is not a LargeBinaryArray".to_string()))? + .clone(); + Some(ImpactSkipData::new(entries, blocks.len())?) + } else { + None + }; Ok(CompressedPostingList::new( blocks, header.max_score, header.length, posting_tail_codec, + block_size, positions, + impacts, )) } @@ -336,34 +441,79 @@ fn deserialize_compressed(r: &mut CacheEntryReader<'_>) -> Result) -> Result<()> { - let count = u32::try_from(self.posting_lists.len()) + let count = u32::try_from(self.len()) .map_err(|_| Error::io("posting list group too large to serialize".to_string()))?; - w.write_header(&PostingListGroupHeader { count })?; - for posting in &self.posting_lists { - posting.serialize(w)?; + match &self.storage { + PostingListGroupStorage::Materialized(posting_lists) => { + w.write_u8(GROUP_VARIANT_MATERIALIZED)?; + w.write_header(&PostingListGroupHeader { count })?; + for posting in posting_lists { + posting.serialize(w)?; + } + } + PostingListGroupStorage::Packed(group) => { + w.write_u8(GROUP_VARIANT_PACKED)?; + w.write_header(&PostingListGroupHeader { count })?; + w.write_u8(posting_tail_codec_to_tag(group.posting_tail_codec))?; + w.write_ipc(&group.batch)?; + } } Ok(()) } fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { - let header: PostingListGroupHeader = r.read_header()?; - let mut posting_lists = Vec::with_capacity(header.count as usize); - for _ in 0..header.count { - posting_lists.push(PostingList::deserialize(r)?); + match r.version() { + 1 => return deserialize_materialized_group(r), + 2 | 3 | Self::CURRENT_VERSION => {} + other => { + return Err(Error::io(format!( + "unsupported PostingListGroup cache version: {other}" + ))); + } } - Ok(Self::new(posting_lists)) + + let variant = r.read_u8()?; + match variant { + GROUP_VARIANT_MATERIALIZED => deserialize_materialized_group(r), + GROUP_VARIANT_PACKED => { + let header: PostingListGroupHeader = r.read_header()?; + let posting_tail_codec = posting_tail_codec_from_tag(r.read_u8()?)?; + let batch = r.read_ipc()?; + if batch.num_rows() != header.count as usize { + return Err(Error::io(format!( + "packed posting group row count {} does not match header count {}", + batch.num_rows(), + header.count + ))); + } + Self::new_packed(batch, posting_tail_codec) + } + other => Err(Error::io(format!( + "unknown PostingListGroup variant: {other}" + ))), + } + } +} + +fn deserialize_materialized_group(r: &mut CacheEntryReader<'_>) -> Result { + let header: PostingListGroupHeader = r.read_header()?; + let mut posting_lists = Vec::with_capacity(header.count as usize); + for _ in 0..header.count { + posting_lists.push(deserialize_posting_list_body(r)?); } + Ok(PostingListGroup::new(posting_lists)) } // --------------------------------------------------------------------------- @@ -409,17 +559,26 @@ impl CacheCodecImpl for Positions { #[cfg(test)] mod tests { + use std::collections::HashMap; + use std::sync::Arc; + use arrow::buffer::ScalarBuffer; - use arrow_array::LargeBinaryArray; - use arrow_array::builder::{Int32Builder, ListBuilder}; + use arrow_array::builder::{Int32Builder, LargeBinaryBuilder, ListBuilder}; + use arrow_array::{Array, LargeBinaryArray, RecordBatch}; + use arrow_schema::{Field, Schema}; use bytes::Bytes; use lance_core::Result; use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter}; + use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec}; + + use super::super::impact::{ImpactSkipData, ImpactSkipDataBuilder}; use super::super::index::{ - CompressedPositionStorage, CompressedPostingList, PlainPostingList, PositionStreamCodec, - Positions, PostingList, PostingListGroup, PostingTailCodec, SharedPositionStream, + CompressedPositionStorage, CompressedPostingList, IMPACT_COL, POSTING_BLOCK_SIZE_KEY, + POSTING_COL, PlainPostingList, PositionStreamCodec, Positions, PostingList, + PostingListGroup, PostingTailCodec, SharedPositionStream, }; + use super::super::tokenizer::LEGACY_BLOCK_SIZE; fn legacy_positions(rows: &[&[i32]]) -> arrow_array::ListArray { let mut builder = ListBuilder::new(Int32Builder::new()); @@ -432,6 +591,73 @@ mod tests { builder.finish() } + fn packed_batch(postings: &[Vec>], block_size: Option) -> RecordBatch { + let mut builder = ListBuilder::new(LargeBinaryBuilder::new()); + for posting in postings { + for block in posting { + builder.values().append_value(block); + } + builder.append(true); + } + let postings = builder.finish(); + let fields = vec![Field::new(POSTING_COL, postings.data_type().clone(), false)]; + let schema = Arc::new(match block_size { + Some(block_size) => Schema::new_with_metadata( + fields, + HashMap::from([(POSTING_BLOCK_SIZE_KEY.to_owned(), block_size.to_string())]), + ), + None => Schema::new(fields), + }); + RecordBatch::try_new(schema, vec![Arc::new(postings)]).unwrap() + } + + fn packed_group( + postings: &[Vec>], + posting_tail_codec: PostingTailCodec, + block_size: Option, + ) -> PostingListGroup { + PostingListGroup::new_packed(packed_batch(postings, block_size), posting_tail_codec) + .unwrap() + } + + fn packed_group_with_impacts( + postings: &[Vec>], + impacts: &[ImpactSkipData], + posting_tail_codec: PostingTailCodec, + block_size: usize, + ) -> PostingListGroup { + assert_eq!(postings.len(), impacts.len()); + let posting_batch = packed_batch(postings, Some(block_size)); + let mut impacts_builder = ListBuilder::new(LargeBinaryBuilder::new()); + for impacts in impacts { + for entry_idx in 0..impacts.entries().len() { + impacts_builder + .values() + .append_value(impacts.entries().value(entry_idx)); + } + impacts_builder.append(true); + } + let impacts = impacts_builder.finish(); + let fields = vec![ + Field::new( + POSTING_COL, + posting_batch.column(0).data_type().clone(), + false, + ), + Field::new(IMPACT_COL, impacts.data_type().clone(), false), + ]; + let schema = Arc::new(Schema::new_with_metadata( + fields, + posting_batch.schema_ref().metadata().clone(), + )); + let batch = RecordBatch::try_new( + schema, + vec![posting_batch.column(0).clone(), Arc::new(impacts)], + ) + .unwrap(); + PostingListGroup::new_packed(batch, posting_tail_codec).unwrap() + } + fn assert_plain_eq(a: &PlainPostingList, b: &PlainPostingList) { assert_eq!(a.row_ids.as_ref(), b.row_ids.as_ref()); assert_eq!(a.frequencies.as_ref(), b.frequencies.as_ref()); @@ -461,6 +687,20 @@ mod tests { } } + fn impact_skip_data(level0_len: usize, block_size: usize) -> ImpactSkipData { + let mut builder = ImpactSkipDataBuilder::with_capacity(level0_len, block_size); + for block_idx in 0..level0_len { + let doc_base = block_idx as u32 * 10; + builder + .append_block(&[ + (doc_base + 1, block_idx as u32 + 1, 10), + (doc_base + 9, block_idx as u32 + 2, 8), + ]) + .unwrap(); + } + builder.finish().unwrap() + } + /// Serialize a codec body (no envelope) into a standalone buffer. fn body_bytes(entry: &T) -> Bytes { let mut buf = Vec::new(); @@ -475,6 +715,34 @@ mod tests { T::deserialize(&mut r) } + fn from_body_version(data: &Bytes, version: u32) -> Result { + let mut r = CacheEntryReader::new(data, 0, version); + T::deserialize(&mut r) + } + + fn compressed_body_with_ipc_sections( + blocks: &RecordBatch, + impacts: Option<&RecordBatch>, + ) -> Bytes { + let mut buf = Vec::new(); + let mut w = CacheEntryWriter::new(&mut buf); + w.write_u8(super::POSTING_VARIANT_COMPRESSED).unwrap(); + w.write_header(&CompressedPostingHeader { + max_score: 1.0, + length: 1, + posting_tail_codec: PbPostingTailCodec::VarintDelta as i32, + block_size: 256, + has_impacts: impacts.is_some(), + ..Default::default() + }) + .unwrap(); + w.write_ipc(blocks).unwrap(); + if let Some(impacts) = impacts { + w.write_ipc(impacts).unwrap(); + } + Bytes::from(buf) + } + fn roundtrip_posting_list(entry: &PostingList) -> PostingList { from_body::(&body_bytes(entry)).unwrap() } @@ -532,14 +800,22 @@ mod tests { Some(&[1u8, 2, 3, 4, 5][..]), Some(&[6, 7, 8, 9, 10][..]), ]); - let posting = - CompressedPostingList::new(blocks, 3.5, 42, PostingTailCodec::VarintDelta, None); + let posting = CompressedPostingList::new( + blocks, + 3.5, + 42, + PostingTailCodec::VarintDelta, + 256, + None, + None, + ); let entry = PostingList::Compressed(posting.clone()); match roundtrip_posting_list(&entry) { PostingList::Compressed(restored) => { assert_eq!(restored.max_score, posting.max_score); assert_eq!(restored.length, posting.length); assert_eq!(restored.posting_tail_codec, posting.posting_tail_codec); + assert_eq!(restored.block_size, posting.block_size); assert_eq!(restored.blocks, posting.blocks); assert!(restored.positions.is_none()); } @@ -547,6 +823,73 @@ mod tests { } } + #[test] + fn compressed_posting_list_impacts_roundtrip() { + let blocks = LargeBinaryArray::from_opt_vec(vec![ + Some(&[1u8, 2, 3, 4, 5][..]), + Some(&[6, 7, 8, 9, 10][..]), + ]); + let impacts = impact_skip_data(blocks.len(), 256); + let posting = CompressedPostingList::new( + blocks, + 3.5, + 42, + PostingTailCodec::VarintDelta, + 256, + None, + Some(impacts.clone()), + ); + let entry = PostingList::Compressed(posting); + match roundtrip_posting_list(&entry) { + PostingList::Compressed(restored) => { + let restored = restored.impacts.expect("impacts should roundtrip"); + assert_eq!(restored.level0_len(), impacts.level0_len()); + assert_eq!(restored.level1_len(), impacts.level1_len()); + assert_eq!(restored.entries(), impacts.entries()); + } + PostingList::Plain(_) => panic!("expected Compressed variant"), + } + } + + #[test] + fn compressed_posting_list_missing_ipc_columns_returns_error() { + let empty = RecordBatch::new_empty(Arc::new(Schema::empty())); + assert!( + from_body::(&compressed_body_with_ipc_sections(&empty, None)).is_err() + ); + + let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1_u8, 2, 3][..])]); + let schema = Arc::new(Schema::new(vec![Field::new( + super::BLOCKS_COLUMN, + blocks.data_type().clone(), + false, + )])); + let blocks = RecordBatch::try_new(schema, vec![Arc::new(blocks)]).unwrap(); + assert!( + from_body::(&compressed_body_with_ipc_sections(&blocks, Some(&empty))) + .is_err() + ); + } + + #[test] + fn compressed_posting_list_v1_cache_without_impacts_decodes() { + let posting = CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]), + 1.25, + 5, + PostingTailCodec::Fixed32, + crate::scalar::inverted::LEGACY_BLOCK_SIZE, + None, + None, + ); + let data = body_bytes(&PostingList::Compressed(posting)); + let restored = from_body_version::(&data, 1).unwrap(); + let PostingList::Compressed(restored) = restored else { + panic!("expected Compressed variant"); + }; + assert!(restored.impacts.is_none()); + } + #[test] fn compressed_posting_list_legacy_positions_roundtrip() { let blocks = LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..])]); @@ -555,9 +898,11 @@ mod tests { 1.25, 5, PostingTailCodec::Fixed32, + crate::scalar::inverted::LEGACY_BLOCK_SIZE, Some(CompressedPositionStorage::LegacyPerDoc(legacy_positions( &[&[0, 4, 8]], ))), + None, ); let entry = PostingList::Compressed(posting.clone()); match roundtrip_posting_list(&entry) { @@ -589,7 +934,9 @@ mod tests { 7.0, 3, PostingTailCodec::VarintDelta, + 256, Some(CompressedPositionStorage::SharedStream(stream)), + None, ); let entry = PostingList::Compressed(posting.clone()); match roundtrip_posting_list(&entry) { @@ -617,9 +964,11 @@ mod tests { 7.0, 3, PostingTailCodec::VarintDelta, + 256, Some(CompressedPositionStorage::SharedStream( expected_stream.clone(), )), + None, ); let serialized = body_bytes(&PostingList::Compressed(posting)); @@ -651,6 +1000,8 @@ mod tests { 2.5, 7, PostingTailCodec::VarintDelta, + 256, + None, None, )); @@ -661,9 +1012,11 @@ mod tests { ] { let group = PostingListGroup::new(members.clone()); let restored = from_body::(&body_bytes(&group)).unwrap(); - assert_eq!(restored.posting_lists.len(), members.len()); - for (a, b) in members.iter().zip(restored.posting_lists.iter()) { - match (a, b) { + assert!(!restored.is_packed()); + assert_eq!(restored.len(), members.len()); + for (index, a) in members.iter().enumerate() { + let b = restored.posting_list(index, None, None).unwrap().unwrap(); + match (a, &b) { (PostingList::Plain(x), PostingList::Plain(y)) => assert_plain_eq(x, y), (PostingList::Compressed(x), PostingList::Compressed(y)) => { assert_eq!(x.blocks, y.blocks); @@ -676,6 +1029,167 @@ mod tests { } } + #[test] + fn packed_posting_list_group_roundtrip_and_v1_fallback() { + let group = packed_group( + &[vec![vec![1, 2, 3], vec![4, 5]], vec![vec![7; 16 * 1024]]], + PostingTailCodec::VarintDelta, + Some(256), + ); + let restored = from_body::(&body_bytes(&group)).unwrap(); + assert!(restored.is_packed()); + assert_eq!(restored.len(), 2); + for slot in 0..2 { + let max_score = [1.5, 3.25][slot]; + let length = [3, 4096][slot]; + let expected = group + .posting_list(slot, Some(max_score), Some(length)) + .unwrap() + .unwrap(); + let actual = restored + .posting_list(slot, Some(max_score), Some(length)) + .unwrap() + .unwrap(); + let (PostingList::Compressed(expected), PostingList::Compressed(actual)) = + (expected, actual) + else { + panic!("expected compressed packed posting views"); + }; + assert_eq!(actual.blocks, expected.blocks); + assert_eq!(actual.max_score, expected.max_score); + assert_eq!(actual.length, expected.length); + assert_eq!(actual.posting_tail_codec, expected.posting_tail_codec); + assert_eq!(actual.block_size, 256); + } + + let legacy_packed = + packed_group(&[vec![vec![9, 8, 7]]], PostingTailCodec::VarintDelta, None); + let restored = from_body::(&body_bytes(&legacy_packed)).unwrap(); + let PostingList::Compressed(posting) = restored + .posting_list(0, Some(2.0), Some(3)) + .unwrap() + .unwrap() + else { + panic!("expected compressed legacy packed posting"); + }; + assert_eq!(posting.block_size, LEGACY_BLOCK_SIZE); + + let legacy_member = PostingList::Compressed(CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]), + 2.0, + 3, + PostingTailCodec::VarintDelta, + LEGACY_BLOCK_SIZE, + None, + None, + )); + let mut legacy_body = Vec::new(); + let mut writer = CacheEntryWriter::new(&mut legacy_body); + writer + .write_header(&crate::cache_pb::PostingListGroupHeader { count: 1 }) + .unwrap(); + legacy_member.serialize(&mut writer).unwrap(); + let legacy_body = Bytes::from(legacy_body); + let mut reader = CacheEntryReader::new(&legacy_body, 0, 1); + let restored = PostingListGroup::deserialize(&mut reader).unwrap(); + assert!(!restored.is_packed()); + assert_eq!(restored.len(), 1); + } + + #[test] + fn posting_list_group_impacted_compressed_members_roundtrip() { + let first = CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[1u8, 2, 3][..]), Some(&[4u8, 5, 6][..])]), + 3.0, + 256, + PostingTailCodec::VarintDelta, + LEGACY_BLOCK_SIZE, + None, + Some(impact_skip_data(2, LEGACY_BLOCK_SIZE)), + ); + let second = CompressedPostingList::new( + LargeBinaryArray::from_opt_vec(vec![Some(&[7u8, 8, 9][..])]), + 5.0, + 128, + PostingTailCodec::Fixed32, + 256, + Some(CompressedPositionStorage::SharedStream( + SharedPositionStream::new( + PositionStreamCodec::PackedDelta, + vec![0u32, 12], + Bytes::from(vec![0xABu8; 32]), + ), + )), + Some(impact_skip_data(1, 256)), + ); + let members = vec![ + PostingList::Compressed(first.clone()), + PostingList::Compressed(second.clone()), + ]; + let group = PostingListGroup::new(members); + let restored = from_body::(&body_bytes(&group)).unwrap(); + assert!(!restored.is_packed()); + assert_eq!(restored.len(), 2); + + let expected = [&first, &second]; + for (slot, expected) in expected.iter().enumerate() { + let restored = restored.posting_list(slot, None, None).unwrap().unwrap(); + let PostingList::Compressed(restored) = restored else { + panic!("expected compressed member"); + }; + assert_eq!(restored.blocks, expected.blocks); + assert_eq!(restored.length, expected.length); + assert_eq!(restored.max_score, expected.max_score); + assert_eq!(restored.posting_tail_codec, expected.posting_tail_codec); + assert_eq!(restored.block_size, expected.block_size); + assert_eq!( + restored.impacts.as_ref().unwrap().entries(), + expected.impacts.as_ref().unwrap().entries() + ); + match (&expected.positions, &restored.positions) { + (Some(expected), Some(restored)) => { + assert_position_storage_eq(expected, restored); + } + (None, None) => {} + _ => panic!("position storage mismatch"), + } + } + } + + #[test] + fn packed_posting_list_group_impacts_roundtrip() { + let postings = vec![vec![vec![1, 2, 3], vec![4, 5, 6]], vec![vec![7, 8, 9]]]; + let expected_impacts = vec![impact_skip_data(2, 256), impact_skip_data(1, 256)]; + let group = packed_group_with_impacts( + &postings, + &expected_impacts, + PostingTailCodec::VarintDelta, + 256, + ); + + let restored = from_body::(&body_bytes(&group)).unwrap(); + assert!(restored.is_packed()); + assert_eq!(restored.len(), expected_impacts.len()); + for (slot, expected) in expected_impacts.iter().enumerate() { + let posting = restored + .posting_list(slot, Some(3.0), Some(256)) + .unwrap() + .unwrap(); + let PostingList::Compressed(posting) = posting else { + panic!("expected compressed packed posting"); + }; + let actual = posting.impacts.as_ref().expect("impacts should roundtrip"); + assert_eq!(actual.entries(), expected.entries()); + assert_eq!(actual.level0_len(), expected.level0_len()); + assert_eq!( + actual.level1_doc_up_to(0), + expected.level1_doc_up_to(0), + "impact entries should remain decodable with the packed block size", + ); + assert!(actual.level1_doc_up_to(0).is_some()); + } + } + #[test] fn positions_legacy_roundtrip() { let positions = Positions(CompressedPositionStorage::LegacyPerDoc(legacy_positions( @@ -717,26 +1231,130 @@ mod tests { use std::sync::Arc; use arrow_array::Array; - use lance_core::cache::CacheCodec; + use arrow_schema::DataType; + use lance_core::cache::{ + CacheCodec, CacheCodecImpl, CacheDecode, CacheEntryReader, CacheEntryWriter, + CacheMissReason, + }; + use lance_core::{Error, Result}; use prost::Message; + use super::super::{ + BLOCKS_COLUMN, GROUP_VARIANT_PACKED, POSTING_VARIANT_COMPRESSED, + posting_tail_codec_to_tag, + }; use super::*; - use crate::cache_pb::{CompressedPostingHeader, PostingTailCodec as PbPostingTailCodec}; + use crate::cache_pb::{ + CompressedPostingHeader, PostingListGroupHeader, PostingTailCodec as PbPostingTailCodec, + }; type ArcAny = Arc; + struct PostingListV2Codec(PostingList); + + impl CacheCodecImpl for PostingListV2Codec { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 2; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + self.0.serialize(w) + } + + fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { + PostingList::deserialize(r).map(Self) + } + } + + struct PostingListGroupV3Codec(PostingListGroup); + + impl CacheCodecImpl for PostingListGroupV3Codec { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 3; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + self.0.serialize(w) + } + + fn deserialize(r: &mut CacheEntryReader<'_>) -> Result { + PostingListGroup::deserialize(r).map(Self) + } + } + + struct LegacyCompressedPostingV1 { + blocks: LargeBinaryArray, + } + + impl CacheCodecImpl for LegacyCompressedPostingV1 { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 1; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + w.write_u8(POSTING_VARIANT_COMPRESSED)?; + w.write_header(&CompressedPostingHeader { + max_score: 2.0, + length: 3, + posting_tail_codec: PbPostingTailCodec::VarintDelta as i32, + ..Default::default() + })?; + let schema = Arc::new(Schema::new(vec![Field::new( + BLOCKS_COLUMN, + DataType::LargeBinary, + false, + )])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(self.blocks.clone())])?; + w.write_ipc(&batch) + } + + fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result { + Err(Error::io( + "LegacyCompressedPostingV1 is a writer-only test codec".to_string(), + )) + } + } + + struct LegacyPackedGroupV2 { + batch: RecordBatch, + posting_tail_codec: PostingTailCodec, + } + + impl CacheCodecImpl for LegacyPackedGroupV2 { + const TYPE_ID: &'static str = ::TYPE_ID; + const CURRENT_VERSION: u32 = 2; + + fn serialize(&self, w: &mut CacheEntryWriter<'_>) -> Result<()> { + w.write_u8(GROUP_VARIANT_PACKED)?; + let count = u32::try_from(self.batch.num_rows()) + .map_err(|_| Error::io("legacy packed group is too large".to_string()))?; + w.write_header(&PostingListGroupHeader { count })?; + w.write_u8(posting_tail_codec_to_tag(self.posting_tail_codec))?; + w.write_ipc(&self.batch) + } + + fn deserialize(_r: &mut CacheEntryReader<'_>) -> Result { + Err(Error::io( + "LegacyPackedGroupV2 is a writer-only test codec".to_string(), + )) + } + } + fn codec() -> CacheCodec { CacheCodec::from_impl::() } - /// Serialize an entry through the full codec (envelope + body). - fn serialize_entry(entry: PostingList) -> Vec { + fn serialize_typed_entry(entry: T) -> Vec { let any: ArcAny = Arc::new(entry); let mut buf = Vec::new(); - codec().serialize(&any, &mut buf).unwrap(); + CacheCodec::from_impl::() + .serialize(&any, &mut buf) + .unwrap(); buf } + /// Serialize an entry through the full codec (envelope + body). + fn serialize_entry(entry: PostingList) -> Vec { + serialize_typed_entry(entry) + } + /// A `Bytes` whose base address is 64-byte aligned, modelling a backend /// that reads cache entries into an aligned buffer. fn aligned_bytes(payload: &[u8]) -> Bytes { @@ -760,7 +1378,9 @@ mod tests { 7.0, 3, PostingTailCodec::VarintDelta, + 256, Some(CompressedPositionStorage::SharedStream(stream)), + None, )) } @@ -795,25 +1415,17 @@ mod tests { assert!(points_in(stream.bytes().as_ptr() as usize)); } - /// Every member of a `PostingListGroup` must also decode zero-copy. The - /// group writes its members inline so each member's IPC sections stay - /// 64-byte aligned within the entry; embedding members in per-member - /// sub-buffers would land them at arbitrary offsets and force a - /// realigning memcpy on load. + /// A packed group's single IPC batch and all posting views decoded from + /// it must borrow the cache entry's aligned input buffer. #[test] - fn group_member_sections_are_zero_copy_through_envelope() { - let make_member = |fill: u8| { - let blocks = - LargeBinaryArray::from_opt_vec(vec![Some(&[fill; 48][..]), Some(&[fill; 48])]); - PostingList::Compressed(CompressedPostingList::new( - blocks, - 7.0, - 3, - PostingTailCodec::VarintDelta, - None, - )) - }; - let group = PostingListGroup::new(vec![make_member(9), make_member(1)]); + fn packed_group_sections_are_zero_copy_through_envelope() { + let postings = vec![ + vec![vec![9; 48], vec![9; 48]], + vec![vec![1; 48], vec![1; 48]], + ]; + let impacts = vec![impact_skip_data(2, 256), impact_skip_data(2, 256)]; + let group = + packed_group_with_impacts(&postings, &impacts, PostingTailCodec::VarintDelta, 256); let group_codec = CacheCodec::from_impl::(); let any: ArcAny = Arc::new(group); @@ -828,8 +1440,13 @@ mod tests { let end = base + serialized.len(); let points_in = |ptr: usize| ptr >= base && ptr < end; - assert_eq!(restored.posting_lists.len(), 2); - for member in &restored.posting_lists { + assert!(restored.is_packed()); + assert_eq!(restored.len(), 2); + for slot in 0..restored.len() { + let member = restored + .posting_list(slot, Some(7.0), Some(3)) + .unwrap() + .unwrap(); let PostingList::Compressed(member) = member else { panic!("expected Compressed member"); }; @@ -837,7 +1454,17 @@ mod tests { assert!( points_in(buf.as_ptr() as usize), "group member blocks buffer was realigned out of the input — \ - misaligned IPC section", + misaligned IPC section", + ); + } + let impacts = member + .impacts + .as_ref() + .expect("packed impacts should decode"); + for buf in impacts.entries().to_data().buffers() { + assert!( + points_in(buf.as_ptr() as usize), + "group member impact buffer was realigned out of the input", ); } } @@ -915,6 +1542,110 @@ mod tests { assert!(codec().deserialize(&Bytes::from(buf)).hit().is_none()); } + #[test] + fn old_codecs_reject_new_impact_envelopes_as_version_too_new() { + let posting = Bytes::from(serialize_entry(compressed_with_shared_positions())); + match CacheCodec::from_impl::().deserialize(&posting) { + CacheDecode::Miss(reason) => { + assert_eq!(reason, CacheMissReason::VersionTooNew) + } + CacheDecode::Hit(_) => panic!("v2 PostingList codec accepted a v3 envelope"), + } + + let group = packed_group( + &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], + PostingTailCodec::VarintDelta, + Some(256), + ); + let group = Bytes::from(serialize_typed_entry(group)); + match CacheCodec::from_impl::().deserialize(&group) { + CacheDecode::Miss(reason) => { + assert_eq!(reason, CacheMissReason::VersionTooNew) + } + CacheDecode::Hit(_) => { + panic!("v3 PostingListGroup codec accepted a v4 envelope") + } + } + } + + #[test] + fn current_codecs_read_previous_main_versions() { + let previous_posting = PostingListV2Codec(compressed_with_shared_positions()); + let previous_posting = Bytes::from(serialize_typed_entry(previous_posting)); + let restored = codec().deserialize(&previous_posting).hit().unwrap(); + let restored = restored.downcast::().unwrap(); + let PostingList::Compressed(restored) = restored.as_ref() else { + panic!("expected compressed posting"); + }; + assert_eq!(restored.block_size, 256); + assert!(restored.impacts.is_none()); + + let previous_group = PostingListGroupV3Codec(packed_group( + &[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], + PostingTailCodec::VarintDelta, + Some(256), + )); + let previous_group = Bytes::from(serialize_typed_entry(previous_group)); + let restored = CacheCodec::from_impl::() + .deserialize(&previous_group) + .hit() + .unwrap() + .downcast::() + .unwrap(); + assert!(restored.is_packed()); + assert_eq!(restored.len(), 2); + let PostingList::Compressed(restored) = restored + .posting_list(0, Some(2.0), Some(3)) + .unwrap() + .unwrap() + else { + panic!("expected compressed packed posting"); + }; + assert_eq!(restored.block_size, 256); + assert!(restored.impacts.is_none()); + } + + #[test] + fn current_codecs_read_legacy_payloads_without_block_size() { + let legacy_posting = LegacyCompressedPostingV1 { + blocks: LargeBinaryArray::from_opt_vec(vec![Some(&[9u8, 8, 7][..])]), + }; + let legacy_posting = Bytes::from(serialize_typed_entry(legacy_posting)); + let restored = codec().deserialize(&legacy_posting).hit().unwrap(); + let restored = restored.downcast::().unwrap(); + let PostingList::Compressed(restored) = restored.as_ref() else { + panic!("expected a compressed legacy posting"); + }; + assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE); + + let legacy_batch = packed_batch(&[vec![vec![1, 2, 3]], vec![vec![4, 5, 6]]], None); + assert!( + !legacy_batch + .schema_ref() + .metadata() + .contains_key(POSTING_BLOCK_SIZE_KEY) + ); + let legacy_group = LegacyPackedGroupV2 { + batch: legacy_batch, + posting_tail_codec: PostingTailCodec::VarintDelta, + }; + let legacy_group = Bytes::from(serialize_typed_entry(legacy_group)); + let restored = CacheCodec::from_impl::() + .deserialize(&legacy_group) + .hit() + .unwrap() + .downcast::() + .unwrap(); + let PostingList::Compressed(restored) = restored + .posting_list(0, Some(2.0), Some(3)) + .unwrap() + .unwrap() + else { + panic!("expected a compressed legacy packed posting"); + }; + assert_eq!(restored.block_size, LEGACY_BLOCK_SIZE); + } + /// A pre-stabilization blob (no magic) self-heals to a miss. #[test] fn pre_stabilization_blob_is_miss() { diff --git a/rust/lance-index/src/scalar/inverted/compound.rs b/rust/lance-index/src/scalar/inverted/compound.rs new file mode 100644 index 00000000000..7bbba53ee52 --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/compound.rs @@ -0,0 +1,7024 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +mod should_maxscore; + +use std::cell::RefCell; +use std::cmp::{Ordering, Reverse}; +use std::collections::{BinaryHeap, HashSet}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering as AtomicOrdering}; + +use futures::{StreamExt, TryStreamExt, stream}; +use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; +use lance_core::{Error, Result}; +use lance_select::RowAddrMask; +use lance_tokenizer::{SimpleTokenizer, TextAnalyzer}; + +use super::{ + InvertedIndex, PreparedBm25Query, + document_tokenizer::{DocType, JsonTokenizer, LanceTokenizer}, + documents::{ + CachedRowAddressOrder, DocId, DocLengths, DocVisibility, OrderedRowAddressProjection, + PartitionDocuments, ResidentAddressProjection, RowAddressProjectionOrderError, + }, + index::{DocSet, InvertedPartition}, + prepare_bm25_query, + query::{ + FtsQuery, FtsSearchParams, MatchQuery, Operator, PhraseQuery, Tokens, collect_query_tokens, + }, + scorer::MemBM25Scorer, + tokenizer::document_tokenizer::TextTokenizer, + wand::{ + FLAT_SEARCH_PERCENT_THRESHOLD, LegacyWandDocuments, ModernWandDocuments, PostingIterator, + WandCursor, WandDocuments, score_sum_upper_bound_factor, + }, +}; +use crate::{metrics::MetricsCollector, prefilter::PreFilter}; + +use self::should_maxscore::ShouldMaxScoreScorer; + +const DEFAULT_BLOCK_SIZE: usize = 128; +const SCORE_FLOOR_RESOLUTION_BATCH_SIZE: usize = DEFAULT_BLOCK_SIZE; + +/// One exact FTS result in a compound collector's candidate domain. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) struct ScoredRow { + pub row_id: K, + pub score: f32, +} + +#[cfg(test)] +impl ScoredRow { + pub(super) fn new(row_id: u64, score: f32) -> Result { + if !score.is_finite() { + return Err(Error::invalid_input(format!( + "FTS score for row_id={row_id} must be finite, got {score}" + ))); + } + Ok(Self { row_id, score }) + } +} + +/// Conservative score bounds for a document range. +/// +/// The lower bound is needed by signed compositions such as [`BoostScorer`]. +/// Arithmetic widens both sides by one representable `f32` so nested +/// operations cannot round an upper bound below an exact score. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) struct ScoreBounds { + lower: f32, + upper: f32, +} + +impl ScoreBounds { + pub(super) const ZERO: Self = Self { + lower: 0.0, + upper: 0.0, + }; + pub(super) const UNBOUNDED: Self = Self { + lower: f32::NEG_INFINITY, + upper: f32::INFINITY, + }; + + pub(super) fn try_new(lower: f32, upper: f32) -> Result { + let bounds = Self { lower, upper }; + if !bounds.is_valid_for_finite_scores() { + return Err(Error::invalid_input(format!( + "FTS score bounds require an ordered interval that can contain finite scores, got [{lower}, {upper}]" + ))); + } + Ok(bounds) + } + + #[cfg(test)] + pub(super) fn lower(self) -> f32 { + self.lower + } + + #[cfg(test)] + pub(super) fn upper(self) -> f32 { + self.upper + } + + fn is_valid_for_finite_scores(self) -> bool { + !self.lower.is_nan() + && !self.upper.is_nan() + && self.lower <= self.upper + && self.lower != f32::INFINITY + && self.upper != f32::NEG_INFINITY + } + + pub(super) fn contains(self, score: f32) -> bool { + score.is_finite() && self.lower <= score && score <= self.upper + } + + fn point(score: f32) -> Result { + if !score.is_finite() { + return Err(Error::invalid_input(format!( + "FTS score bounds require a finite score, got {score}" + ))); + } + Ok(Self { + lower: score, + upper: score, + }) + } + + fn scale_non_negative(self, factor: f32) -> Self { + debug_assert!(factor.is_finite() && factor >= 0.0); + if factor == 0.0 { + return Self::ZERO; + } + if !self.lower.is_finite() || !self.upper.is_finite() { + return Self::UNBOUNDED; + } + Self { + lower: next_down(self.lower * factor), + upper: next_up(self.upper * factor), + } + } + + fn include_zero(self) -> Self { + Self { + lower: self.lower.min(0.0), + upper: self.upper.max(0.0), + } + } + + fn add(self, other: Self) -> Self { + if !self.lower.is_finite() + || !self.upper.is_finite() + || !other.lower.is_finite() + || !other.upper.is_finite() + { + return Self::UNBOUNDED; + } + Self { + lower: next_down(self.lower + other.lower), + upper: next_up(self.upper + other.upper), + } + } + + fn subtract_scaled(self, other: Self, factor: f32) -> Self { + let penalty = other.scale_non_negative(factor); + if !self.lower.is_finite() + || !self.upper.is_finite() + || !penalty.lower.is_finite() + || !penalty.upper.is_finite() + { + return Self::UNBOUNDED; + } + Self { + lower: next_down(self.lower - penalty.upper), + upper: next_up(self.upper - penalty.lower), + } + } +} + +fn next_up(value: f32) -> f32 { + if !value.is_finite() { + return value; + } + if value == 0.0 { + return f32::from_bits(1); + } + let bits = value.to_bits(); + if value > 0.0 { + f32::from_bits(bits + 1) + } else { + f32::from_bits(bits - 1) + } +} + +fn next_down(value: f32) -> f32 { + if !value.is_finite() { + return value; + } + if value == 0.0 { + return f32::from_bits((1_u32 << 31) | 1); + } + let bits = value.to_bits(); + if value > 0.0 { + f32::from_bits(bits - 1) + } else { + f32::from_bits(bits + 1) + } +} + +/// Find the greatest finite non-negative raw score whose actual `f32` +/// multiplication remains strictly below an inclusive scaled score floor. +/// +/// Non-negative finite `f32` values have the same order as their bit patterns, +/// and multiplication by a finite positive factor is monotonic. Binary search +/// therefore proves the returned exclusive child floor cannot discard a raw +/// score whose scaled value is equal to `scaled_score_floor`. +#[doc(hidden)] +pub fn exclusive_scaled_score_floor(scaled_score_floor: f32, factor: f32) -> Option { + if !scaled_score_floor.is_finite() + || scaled_score_floor <= 0.0 + || !factor.is_finite() + || factor <= 0.0 + { + return None; + } + + let mut lower_bits = 0_u32; + let mut upper_bits = f32::MAX.to_bits(); + while lower_bits < upper_bits { + let midpoint = lower_bits + (upper_bits - lower_bits).div_ceil(2); + let raw_score = f32::from_bits(midpoint); + if raw_score * factor < scaled_score_floor { + lower_bits = midpoint; + } else { + upper_bits = midpoint - 1; + } + } + Some(f32::from_bits(lower_bits)) +} + +fn checked_score(score: f32, context: &str) -> Result { + if score.is_finite() { + Ok(score) + } else { + Err(Error::invalid_input(format!( + "{context} produced a non-finite FTS score: {score}" + ))) + } +} + +/// Internal document-at-a-time scorer protocol for compound FTS. +/// +/// Implementations iterate matching partition-local document ids in ascending +/// order and expose the corresponding candidate key separately. A collector +/// may shallow-advance independently of the exact iterator, inspect a +/// conservative range bound, and monotonically raise the competitive score. +/// `matches` is the optional two-phase confirmation hook: cheap approximations +/// return a candidate from `next` / `advance` and defer expensive checks such +/// as phrase positions until confirmation. +pub(super) trait ComposableScorer: Send { + fn doc(&self) -> Option; + fn document_key(&self) -> Option { + self.doc() + } + fn next(&mut self) -> Result>; + fn advance(&mut self, target: u64) -> Result>; + fn cost(&self) -> usize; + fn score(&mut self) -> Result; + fn advance_shallow(&mut self, target: u64) -> Result; + fn score_bounds(&mut self, up_to: u64) -> Result; + /// Conservative list-wide score upper bound, independent of iterator + /// position. `None` keeps the scorer on exact eager composition paths. + fn global_score_upper_bound(&self) -> Option { + None + } + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()>; + + /// Conservative score upper bound for the current approximation without + /// running two-phase confirmation. `None` disables doc-local pruning. + fn current_score_upper_bound(&mut self) -> Result> { + Ok(None) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + false + } + + fn matches(&mut self) -> Result { + Ok(true) + } + + /// Estimated relative cost of [`Self::matches`], stable for this scorer's + /// lifetime. `None` means no ordering hint, not that confirmation may be skipped. + fn match_cost(&self) -> Option { + None + } + + fn scores_non_negative(&self) -> bool { + false + } +} + +pub(super) type BoxScorer<'a> = Box; + +fn sum_global_score_upper_bounds(children: &[BoxScorer<'_>]) -> Option { + children.iter().try_fold(0.0, |upper, child| { + let child_upper = child.global_score_upper_bound()?; + if !child_upper.is_finite() || child_upper < 0.0 { + return None; + } + let combined = ScoreBounds { lower: 0.0, upper } + .add(ScoreBounds { + lower: 0.0, + upper: child_upper, + }) + .upper; + combined.is_finite().then_some(combined) + }) +} + +/// Posting-payload-independent metadata for one semantic leaf in a staged +/// search task. +/// +/// Bounds describe the unboosted leaf score. [`CompoundScorerPlan`] applies +/// the `MatchQuery` boost recorded in its leaf node while composing the root +/// interval. An impossible leaf contributes neither candidates nor score. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(super) struct CompoundLeafPlanInput { + pub(super) possible: bool, + pub(super) cost: usize, + pub(super) bounds: ScoreBounds, +} + +impl CompoundLeafPlanInput { + pub(super) fn new(possible: bool, cost: usize, bounds: ScoreBounds) -> Self { + Self { + possible, + cost, + bounds, + } + } +} + +/// Pure metadata analysis used before staged source I/O begins. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct CompoundPlanAnalysis { + pub(super) possible: bool, + pub(super) bounds: ScoreBounds, + pub(super) generator_cost: usize, + pub(super) generator_leaves: Vec, +} + +#[derive(Debug)] +struct NodePlanAnalysis { + possible: bool, + bounds: ScoreBounds, + generator_cost: usize, + generator_leaves: Vec, +} + +impl NodePlanAnalysis { + fn impossible() -> Self { + Self { + possible: false, + bounds: ScoreBounds::ZERO, + generator_cost: 0, + generator_leaves: Vec::new(), + } + } +} + +#[derive(Debug, Clone)] +pub(super) enum CompoundScorerPlan { + Leaf { + index: usize, + boost: f32, + }, + Boost { + positive: Box, + negative: Box, + negative_boost: f32, + }, + MultiMatch(Vec), + Boolean { + should: Vec, + must: Vec, + must_not: Vec, + }, +} + +impl CompoundScorerPlan { + pub(super) fn leaf_count(&self) -> usize { + match self { + Self::Leaf { .. } => 1, + Self::Boost { + positive, negative, .. + } => positive.leaf_count().saturating_add(negative.leaf_count()), + Self::MultiMatch(children) => children + .iter() + .fold(0, |count, child| count.saturating_add(child.leaf_count())), + Self::Boolean { + should, + must, + must_not, + } => should + .iter() + .chain(must) + .chain(must_not) + .fold(0, |count, child| count.saturating_add(child.leaf_count())), + } + } + + /// Select a complete positive generator cover and compose conservative + /// root score bounds without constructing or loading any scorer. + pub(super) fn analyze_leaves( + &self, + leaves: &[CompoundLeafPlanInput], + ) -> Result { + let leaf_count = self.leaf_count(); + if leaves.len() != leaf_count { + return Err(Error::internal(format!( + "compound FTS plan has {leaf_count} leaves but received {} staged leaf inputs", + leaves.len() + ))); + } + for (index, leaf) in leaves.iter().enumerate() { + if !leaf.bounds.is_valid_for_finite_scores() { + return Err(Error::internal(format!( + "compound FTS staged leaf {index} reported invalid score bounds [{}, {}]", + leaf.bounds.lower, leaf.bounds.upper + ))); + } + } + + let mut seen = vec![false; leaf_count]; + self.validate_leaf_indices(&mut seen)?; + if let Some(missing) = seen.iter().position(|seen| !*seen) { + return Err(Error::internal(format!( + "compound FTS plan does not reference staged leaf {missing}" + ))); + } + + let mut node = self.analyze_node(leaves)?; + node.generator_leaves.sort_unstable(); + node.generator_leaves.dedup(); + Ok(CompoundPlanAnalysis { + possible: node.possible, + bounds: node.bounds, + generator_cost: node.generator_cost, + generator_leaves: node.generator_leaves, + }) + } + + fn validate_leaf_indices(&self, seen: &mut [bool]) -> Result<()> { + match self { + Self::Leaf { index, .. } => { + let slot_count = seen.len(); + let slot = seen.get_mut(*index).ok_or_else(|| { + Error::internal(format!( + "compound FTS plan references staged leaf {index}, but only {} slots exist", + slot_count + )) + })?; + if *slot { + return Err(Error::internal(format!( + "compound FTS plan references staged leaf {index} more than once" + ))); + } + *slot = true; + Ok(()) + } + Self::Boost { + positive, negative, .. + } => { + positive.validate_leaf_indices(seen)?; + negative.validate_leaf_indices(seen) + } + Self::MultiMatch(children) => { + for child in children { + child.validate_leaf_indices(seen)?; + } + Ok(()) + } + Self::Boolean { + should, + must, + must_not, + } => { + for child in should.iter().chain(must).chain(must_not) { + child.validate_leaf_indices(seen)?; + } + Ok(()) + } + } + } + + fn analyze_node(&self, leaves: &[CompoundLeafPlanInput]) -> Result { + match self { + Self::Leaf { index, boost } => { + if !boost.is_finite() || *boost < 0.0 { + return Err(Error::invalid_input(format!( + "MatchQuery boost must be finite and non-negative, got {boost}" + ))); + } + let leaf = leaves.get(*index).ok_or_else(|| { + Error::internal(format!( + "compound FTS plan references missing staged leaf {index}" + )) + })?; + if !leaf.possible { + return Ok(NodePlanAnalysis::impossible()); + } + Ok(NodePlanAnalysis { + possible: true, + bounds: leaf.bounds.scale_non_negative(*boost), + generator_cost: leaf.cost, + generator_leaves: vec![*index], + }) + } + Self::Boost { + positive, + negative, + negative_boost, + } => { + if !negative_boost.is_finite() || *negative_boost < 0.0 { + return Err(Error::invalid_input(format!( + "BoostQuery negative_boost must be finite and non-negative, got {negative_boost}" + ))); + } + let positive = positive.analyze_node(leaves)?; + let negative = negative.analyze_node(leaves)?; + if !positive.possible { + return Ok(NodePlanAnalysis::impossible()); + } + let bounds = if negative.possible { + positive + .bounds + .subtract_scaled(negative.bounds.include_zero(), *negative_boost) + } else { + positive.bounds + }; + Ok(NodePlanAnalysis { + possible: true, + bounds, + generator_cost: positive.generator_cost, + generator_leaves: positive.generator_leaves, + }) + } + Self::MultiMatch(children) => { + let children = children + .iter() + .map(|child| child.analyze_node(leaves)) + .collect::>>()?; + let mut possible = children.iter().filter(|child| child.possible); + let Some(first) = possible.next() else { + return Ok(NodePlanAnalysis::impossible()); + }; + let mut bounds = first.bounds; + for child in possible { + bounds.lower = bounds.lower.min(child.bounds.lower); + bounds.upper = bounds.upper.max(child.bounds.upper); + } + let mut generator_cost = 0_usize; + let mut generator_leaves = Vec::new(); + for child in children.into_iter().filter(|child| child.possible) { + generator_cost = generator_cost.saturating_add(child.generator_cost); + generator_leaves.extend(child.generator_leaves); + } + Ok(NodePlanAnalysis { + possible: true, + bounds, + generator_cost, + generator_leaves, + }) + } + Self::Boolean { + should, + must, + must_not, + } => { + let should = should + .iter() + .map(|child| child.analyze_node(leaves)) + .collect::>>()?; + let must = must + .iter() + .map(|child| child.analyze_node(leaves)) + .collect::>>()?; + for child in must_not { + child.analyze_node(leaves)?; + } + + if !must.is_empty() { + if must.iter().any(|child| !child.possible) { + return Ok(NodePlanAnalysis::impossible()); + } + let mut bounds = ScoreBounds::ZERO; + for child in &must { + bounds = bounds.add(child.bounds); + } + for child in should.iter().filter(|child| child.possible) { + bounds = bounds.add(child.bounds.include_zero()); + } + let mut must = must.into_iter(); + let mut generator = must.next().ok_or_else(|| { + Error::internal("compound FTS Boolean MUST analysis lost its generator") + })?; + for child in must { + if child.generator_cost < generator.generator_cost { + generator = child; + } + } + return Ok(NodePlanAnalysis { + possible: true, + bounds, + generator_cost: generator.generator_cost, + generator_leaves: generator.generator_leaves, + }); + } + + let possible_should = should + .into_iter() + .filter(|child| child.possible) + .collect::>(); + if possible_should.is_empty() { + return Ok(NodePlanAnalysis::impossible()); + } + let mut bounds = ScoreBounds::ZERO; + let mut generator_cost = 0_usize; + let mut generator_leaves = Vec::new(); + for child in possible_should { + bounds = bounds.add(child.bounds.include_zero()); + generator_cost = generator_cost.saturating_add(child.generator_cost); + generator_leaves.extend(child.generator_leaves); + } + Ok(NodePlanAnalysis { + possible: true, + bounds, + generator_cost, + generator_leaves, + }) + } + } + } + + pub(super) fn from_query(query: &FtsQuery, num_leaves: &mut usize) -> Result { + match query { + FtsQuery::Match(query) => { + let index = *num_leaves; + *num_leaves += 1; + Ok(Self::Leaf { + index, + boost: query.boost, + }) + } + FtsQuery::Phrase(_) => { + let index = *num_leaves; + *num_leaves += 1; + Ok(Self::Leaf { index, boost: 1.0 }) + } + FtsQuery::Boost(query) => Ok(Self::Boost { + positive: Box::new(Self::from_query(&query.positive, num_leaves)?), + negative: Box::new(Self::from_query(&query.negative, num_leaves)?), + negative_boost: query.negative_boost, + }), + FtsQuery::MultiMatch(query) => Ok(Self::MultiMatch( + query + .match_queries + .iter() + .map(|query| Self::from_query(&FtsQuery::Match(query.clone()), num_leaves)) + .collect::>>()?, + )), + FtsQuery::Boolean(query) => Ok(Self::Boolean { + should: query + .should + .iter() + .map(|query| Self::from_query(query, num_leaves)) + .collect::>>()?, + must: query + .must + .iter() + .map(|query| Self::from_query(query, num_leaves)) + .collect::>>()?, + must_not: query + .must_not + .iter() + .map(|query| Self::from_query(query, num_leaves)) + .collect::>>()?, + }), + } + } + + pub(super) fn build<'a>(&self, leaves: &mut [Option>]) -> Result> { + match self { + Self::Leaf { index, boost } => { + let leaf = leaves + .get_mut(*index) + .and_then(Option::take) + .ok_or_else(|| { + Error::internal(format!( + "compound FTS scorer references missing leaf index {index}" + )) + })?; + Ok(Box::new(ScaleScorer::try_new(leaf, *boost)?)) + } + Self::Boost { + positive, + negative, + negative_boost, + } => Ok(Box::new(BoostScorer::try_new( + positive.build(leaves)?, + negative.build(leaves)?, + *negative_boost, + )?)), + Self::MultiMatch(children) => Ok(Box::new(DisjunctionScorer::try_new( + children + .iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + DisjunctionScore::Max, + )?)), + Self::Boolean { + should, + must, + must_not, + } => Ok(Box::new(BooleanScorer::try_new( + should + .iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + must.iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + must_not + .iter() + .map(|child| child.build(leaves)) + .collect::>>()?, + )?)), + } + } +} + +impl ComposableScorer for WandCursor<'_, D> { + fn doc(&self) -> Option { + self.doc() + } + + fn document_key(&self) -> Option { + self.document_key() + } + + fn next(&mut self) -> Result> { + self.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.advance(target) + } + + fn cost(&self) -> usize { + self.cost() + } + + fn score(&mut self) -> Result { + self.current_score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + Ok(ScoreBounds { + lower: 0.0, + upper: self.score_upper_bound(up_to)?, + }) + } + + fn global_score_upper_bound(&self) -> Option { + WandCursor::global_score_upper_bound(self) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.set_min_competitive_score(min_score) + } + + fn current_score_upper_bound(&mut self) -> Result> { + self.current_score().map(Some) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.match_cost().is_some() + } + + fn matches(&mut self) -> Result { + self.matches() + } + + fn match_cost(&self) -> Option { + self.match_cost() + } + + fn scores_non_negative(&self) -> bool { + true + } +} + +#[derive(Debug, Clone, Copy)] +struct ShallowRange { + target: u64, + up_to: u64, + block_index: Option, +} + +/// Exact in-memory scorer used for unordered-address fallbacks and unit tests. +pub(super) struct MaterializedScorer { + rows: Vec, + block_size: usize, + block_bounds: Box<[ScoreBounds]>, + global_score_upper_bound: f32, + index: Option, + shallow: Option, + min_competitive_score: f32, + scores_non_negative: bool, + #[cfg(test)] + bound_score_visits: usize, +} + +impl MaterializedScorer { + pub(super) fn try_new(mut rows: Vec) -> Result { + rows.sort_unstable_by_key(|row| row.row_id); + for row in &rows { + ScoreBounds::point(row.score)?; + } + for pair in rows.windows(2) { + if pair[0].row_id == pair[1].row_id { + return Err(Error::internal(format!( + "FTS leaf scorer produced duplicate row_id={}", + pair[0].row_id + ))); + } + } + let block_size = DEFAULT_BLOCK_SIZE; + let block_bounds = Self::build_block_bounds(&rows, block_size); + let global_score_upper_bound = Self::global_upper_bound(&block_bounds); + let scores_non_negative = rows.iter().all(|row| row.score >= 0.0); + #[cfg(test)] + let bound_score_visits = rows.len(); + Ok(Self { + rows, + block_size, + block_bounds, + global_score_upper_bound, + index: None, + shallow: None, + min_competitive_score: f32::NEG_INFINITY, + scores_non_negative, + #[cfg(test)] + bound_score_visits, + }) + } + + #[cfg(test)] + fn with_block_size(mut self, block_size: usize) -> Self { + assert!(block_size > 0); + self.block_size = block_size; + self.block_bounds = Self::build_block_bounds(&self.rows, block_size); + self.global_score_upper_bound = Self::global_upper_bound(&self.block_bounds); + self.bound_score_visits = self.rows.len(); + self + } + + fn build_block_bounds(rows: &[ScoredRow], block_size: usize) -> Box<[ScoreBounds]> { + debug_assert!(block_size > 0); + rows.chunks(block_size) + .map(|block| { + let first = block[0].score; + let mut bounds = ScoreBounds { + lower: first, + upper: first, + }; + for row in &block[1..] { + bounds.lower = bounds.lower.min(row.score); + bounds.upper = bounds.upper.max(row.score); + } + bounds + }) + .collect() + } + + fn global_upper_bound(block_bounds: &[ScoreBounds]) -> f32 { + block_bounds + .iter() + .map(|bounds| bounds.upper) + .max_by(f32::total_cmp) + .unwrap_or(0.0) + } + + fn block_bounds_at(&self, block_index: usize) -> Result { + self.block_bounds.get(block_index).copied().ok_or_else(|| { + Error::internal(format!( + "materialized FTS scorer has no score bounds for block {block_index}" + )) + }) + } + + #[cfg(test)] + fn bound_score_visits(&self) -> usize { + self.bound_score_visits + } + + #[cfg(test)] + fn num_bound_blocks(&self) -> usize { + self.block_bounds.len() + } + + fn block_end(&self, block_index: usize) -> usize { + (block_index + 1) + .saturating_mul(self.block_size) + .min(self.rows.len()) + } + + fn block_index(&self, row_index: usize) -> usize { + row_index / self.block_size + } + + fn skip_non_competitive_block(&self, row_index: usize) -> Result> { + let block_index = self.block_index(row_index); + if self.block_bounds_at(block_index)?.upper < self.min_competitive_score { + Ok(Some(self.block_end(block_index))) + } else { + Ok(None) + } + } + + fn position_at(&mut self, mut index: usize) -> Result> { + while index < self.rows.len() { + if let Some(next_index) = self.skip_non_competitive_block(index)? { + index = next_index; + continue; + } + self.index = Some(index); + self.shallow = None; + return Ok(Some(self.rows[index].row_id)); + } + self.index = None; + self.shallow = None; + Ok(None) + } +} + +impl ComposableScorer for MaterializedScorer { + fn doc(&self) -> Option { + self.index.map(|index| self.rows[index].row_id) + } + + fn next(&mut self) -> Result> { + self.position_at(self.index.map_or(0, |index| index + 1)) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.doc().is_some_and(|doc| doc >= target) { + return Ok(self.doc()); + } + let start = self.index.map_or(0, |index| index + 1); + let offset = self.rows[start..].partition_point(|row| row.row_id < target); + self.position_at(start + offset) + } + + fn cost(&self) -> usize { + self.rows.len() + } + + fn score(&mut self) -> Result { + self.index + .map(|index| self.rows[index].score) + .ok_or_else(|| Error::internal("FTS scorer is not positioned on a document")) + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let start = self.rows.partition_point(|row| row.row_id < target); + if start == self.rows.len() { + self.shallow = Some(ShallowRange { + target, + up_to: u64::MAX, + block_index: None, + }); + return Ok(u64::MAX); + } + let block_index = self.block_index(start); + let end = self.block_end(block_index); + let up_to = self + .rows + .get(end) + .map(|next| next.row_id.saturating_sub(1)) + .unwrap_or(u64::MAX); + self.shallow = Some(ShallowRange { + target, + up_to, + block_index: Some(block_index), + }); + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let shallow = self.shallow.ok_or_else(|| { + Error::internal("score_bounds requires advance_shallow on the FTS scorer") + })?; + if up_to < shallow.target || up_to > shallow.up_to { + return Err(Error::internal(format!( + "FTS score bound up_to={up_to} is outside shallow range [{}, {}]", + shallow.target, shallow.up_to + ))); + } + shallow + .block_index + .map_or(Ok(ScoreBounds::ZERO), |block_index| { + self.block_bounds_at(block_index) + }) + } + + fn global_score_upper_bound(&self) -> Option { + Some(self.global_score_upper_bound) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score > self.min_competitive_score { + self.min_competitive_score = min_score; + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + self.score().map(Some) + } + + fn scores_non_negative(&self) -> bool { + self.scores_non_negative + } +} + +#[derive(Debug, Clone, Copy)] +struct MappedShallowRange { + target: u64, + up_to: u64, + source_target: u64, + source_up_to: u64, + has_source_docs: bool, +} + +/// Project a strictly ordered partition-local scorer into the shared physical +/// row-address domain. +struct RowAddressScorer<'a> { + source: BoxScorer<'a>, + projection: OrderedRowAddressProjection, + current: Option, + exhausted: bool, + shallow: Option, +} + +impl<'a> RowAddressScorer<'a> { + fn new(source: BoxScorer<'a>, projection: OrderedRowAddressProjection) -> Self { + Self { + source, + projection, + current: None, + exhausted: false, + shallow: None, + } + } + + fn set_source_position(&mut self, source_doc: Option) -> Result> { + self.shallow = None; + let Some(source_doc) = source_doc else { + self.current = None; + self.exhausted = true; + return Ok(None); + }; + if self.source.doc() != Some(source_doc) { + return Err(Error::internal(format!( + "FTS source returned local document {source_doc} but reported position {:?}", + self.source.doc() + ))); + } + let row_address = self.projection.address(source_doc).ok_or_else(|| { + Error::internal(format!( + "FTS source returned non-live or out-of-range local document {source_doc} for a projection with {} slots", + self.projection.len() + )) + })?; + if self.current.is_some_and(|current| row_address <= current) { + return Err(Error::internal(format!( + "FTS row-address projection moved from {:?} to non-increasing address {row_address}", + self.current + ))); + } + self.current = Some(row_address); + Ok(self.current) + } + + fn ensure_positioned(&self) -> Result<()> { + if self.current.is_none() { + Err(Error::internal( + "row-address FTS scorer is not positioned on a document", + )) + } else { + Ok(()) + } + } + + fn local_upper_bound(&self, global_up_to: u64, shallow: MappedShallowRange) -> Option { + if global_up_to >= shallow.up_to { + return Some(shallow.source_up_to); + } + let next_global = global_up_to.checked_add(1)?; + match self.projection.lower_bound(next_global) { + Some(next_local) => next_local + .checked_sub(1) + .map(|local| local.min(shallow.source_up_to)), + None => Some(shallow.source_up_to), + } + } +} + +impl ComposableScorer for RowAddressScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.current + } + + fn next(&mut self) -> Result> { + if self.exhausted { + return Ok(None); + } + let source_doc = self.source.next()?; + self.set_source_position(source_doc) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + if self.exhausted { + return Ok(None); + } + let Some(source_target) = self.projection.lower_bound(target) else { + self.current = None; + self.exhausted = true; + self.shallow = None; + return Ok(None); + }; + let source_doc = self.source.advance(source_target)?; + if source_doc.is_some_and(|doc| doc < source_target) { + return Err(Error::internal(format!( + "FTS source advanced to local document {:?} before target {source_target}", + source_doc + ))); + } + let row_address = self.set_source_position(source_doc)?; + if row_address.is_some_and(|address| address < target) { + return Err(Error::internal(format!( + "FTS projection advanced to row address {:?} before target {target}", + row_address + ))); + } + Ok(row_address) + } + + fn cost(&self) -> usize { + self.source.cost().min(self.projection.live_len()) + } + + fn score(&mut self) -> Result { + self.ensure_positioned()?; + self.source.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + if self.exhausted { + self.shallow = Some(MappedShallowRange { + target, + up_to: u64::MAX, + source_target: 0, + source_up_to: 0, + has_source_docs: false, + }); + return Ok(u64::MAX); + } + let Some(source_target) = self.projection.lower_bound(target) else { + self.shallow = Some(MappedShallowRange { + target, + up_to: u64::MAX, + source_target: 0, + source_up_to: 0, + has_source_docs: false, + }); + return Ok(u64::MAX); + }; + let source_target = self + .source + .doc() + .map_or(source_target, |doc| source_target.max(doc)); + let source_up_to = self.source.advance_shallow(source_target)?; + if source_up_to < source_target { + return Err(Error::internal(format!( + "FTS source returned shallow range ending at local document {source_up_to} before target {source_target}" + ))); + } + let up_to = self + .projection + .next_address(source_up_to) + .map(|next| next.saturating_sub(1)) + .unwrap_or(u64::MAX); + if up_to < target { + return Err(Error::internal(format!( + "FTS projection mapped local shallow end {source_up_to} to row address {up_to} before target {target}" + ))); + } + self.shallow = Some(MappedShallowRange { + target, + up_to, + source_target, + source_up_to, + has_source_docs: true, + }); + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let shallow = self.shallow.ok_or_else(|| { + Error::internal("score_bounds requires advance_shallow on the row-address FTS scorer") + })?; + if up_to < shallow.target || up_to > shallow.up_to { + return Err(Error::internal(format!( + "FTS row-address score bound up_to={up_to} is outside shallow range [{}, {}]", + shallow.target, shallow.up_to + ))); + } + if !shallow.has_source_docs { + return Ok(ScoreBounds::ZERO); + } + let Some(source_up_to) = self.local_upper_bound(up_to, shallow) else { + return Ok(ScoreBounds::ZERO); + }; + if source_up_to < shallow.source_target { + return Ok(ScoreBounds::ZERO); + } + self.source.score_bounds(source_up_to) + } + + fn global_score_upper_bound(&self) -> Option { + self.source.global_score_upper_bound() + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.source.set_min_competitive_score(min_score) + } + + fn current_score_upper_bound(&mut self) -> Result> { + self.ensure_positioned()?; + self.source.current_score_upper_bound() + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.source.supports_doc_local_confirmation_pruning() + } + + fn matches(&mut self) -> Result { + self.ensure_positioned()?; + self.source.matches() + } + + fn match_cost(&self) -> Option { + self.source.match_cost() + } + + fn scores_non_negative(&self) -> bool { + self.source.scores_non_negative() + } +} + +fn projected_address(projection: &ResidentAddressProjection, source_doc: u64) -> Result { + let source_doc = u32::try_from(source_doc).map_err(|_| { + Error::index(format!( + "FTS local document {source_doc} exceeds the modern u32 domain" + )) + })?; + projection.address(DocId::new(source_doc)).ok_or_else(|| { + Error::internal(format!( + "FTS source returned non-live local document {source_doc} while materializing row addresses" + )) + }) +} + +fn materialize_row_address_scorer<'a>( + source: BoxScorer<'a>, + projection: &ResidentAddressProjection, + collisions: &MaterializedProjectionCollisions, +) -> Result>> { + let mut mapped_documents = Vec::with_capacity(source.cost().min(DEFAULT_BLOCK_SIZE)); + let scorer = materialize_mapped_scorer(source, |local_doc| { + let row_address = projected_address(projection, local_doc)?; + mapped_documents.push((row_address, local_doc)); + Ok(row_address) + })?; + + // Constructing the materialized scorer first preserves its more local + // duplicate diagnostic when one leaf produces both colliding documents. + // Only a valid leaf is then published to the source-wide tracker. + collisions.register(&mapped_documents)?; + Ok(scorer) +} + +fn materialize_mapped_scorer<'a>( + mut source: BoxScorer<'a>, + mut map_document: impl FnMut(u64) -> Result, +) -> Result>> { + let mut rows = Vec::with_capacity(source.cost().min(DEFAULT_BLOCK_SIZE)); + let mut source_doc = source.next()?; + while let Some(doc) = source_doc { + if source.matches()? { + rows.push(ScoredRow { + row_id: map_document(doc)?, + score: checked_score(source.score()?, "materialized row-address FTS scorer")?, + }); + } + source_doc = source.next()?; + } + let scorer = MaterializedScorer::try_new(rows)?; + let min_possible_row_address = scorer.rows.first().map(|row| row.row_id); + Ok( + min_possible_row_address.map(|min_possible_row_address| RowAddressSource { + min_possible_row_address, + scorer: Box::new(scorer), + }), + ) +} + +#[derive(Debug, Default)] +struct MaterializedProjectionCollisions { + local_doc_by_address: RefCell>, +} + +impl MaterializedProjectionCollisions { + fn register(&self, mapped_documents: &[(u64, u64)]) -> Result<()> { + let mut local_doc_by_address = + self.local_doc_by_address.try_borrow_mut().map_err(|_| { + Error::internal( + "materialized FTS row-address collision tracker is already borrowed", + ) + })?; + + for &(row_address, local_doc) in mapped_documents { + if let Some(&existing_local_doc) = local_doc_by_address.get(&row_address) + && existing_local_doc != local_doc + { + return Err(Error::index(format!( + "FTS row address {row_address} maps to distinct local documents {existing_local_doc} and {local_doc} in one physical source" + ))); + } + } + for &(row_address, local_doc) in mapped_documents { + local_doc_by_address.insert(row_address, local_doc); + } + Ok(()) + } +} + +/// Query-scoped projection state shared by every leaf from one physical +/// source. Preparation is O(1); ordered validation is triggered lazily only +/// when a dense source makes streaming cheaper than candidate materialization. +#[derive(Debug)] +pub(super) struct PreparedRowAddressProjection { + projection: ResidentAddressProjection, + collisions: MaterializedProjectionCollisions, +} + +pub(super) fn prepare_row_address_projection( + projection: &ResidentAddressProjection, +) -> PreparedRowAddressProjection { + PreparedRowAddressProjection { + projection: projection.clone(), + collisions: MaterializedProjectionCollisions::default(), + } +} + +fn invalid_row_address_projection(error: RowAddressProjectionOrderError) -> Error { + Error::index(format!("invalid FTS row-address projection: {error}")) +} + +fn map_validated_scorer_to_row_addresses<'a>( + source: BoxScorer<'a>, + projection: &PreparedRowAddressProjection, + validation: std::result::Result, + local_document_lower_bound: u64, +) -> Result>> { + match validation { + Ok(ordered) => { + let Some(first_row_address) = ordered + .address(local_document_lower_bound) + .or_else(|| ordered.next_address(local_document_lower_bound)) + else { + return Ok(None); + }; + Ok(Some(RowAddressSource::new( + first_row_address, + Box::new(RowAddressScorer::new(source, ordered)), + ))) + } + Err(error @ RowAddressProjectionOrderError::Duplicate { .. }) => { + Err(invalid_row_address_projection(error)) + } + Err(RowAddressProjectionOrderError::OutOfOrder { .. }) => { + materialize_row_address_scorer(source, &projection.projection, &projection.collisions) + } + } +} + +/// A lazily initialized scorer source and a conservative lower bound on its +/// first possible row address. +pub(super) struct RowAddressSource<'a> { + min_possible_row_address: u64, + scorer: BoxScorer<'a>, +} + +impl<'a> RowAddressSource<'a> { + pub(super) fn new(min_possible_row_address: u64, scorer: BoxScorer<'a>) -> Self { + Self { + min_possible_row_address, + scorer, + } + } + + pub(super) fn into_scorer(self) -> BoxScorer<'a> { + self.scorer + } +} + +/// Map a partition-local scorer into the shared row-address domain. +/// +/// Strictly ordered projections stay streaming. Descending or sufficiently +/// sparse unknown projections use an exact materialized fallback. Duplicate +/// projections are rejected because two local documents cannot share one row. +pub(super) fn map_scorer_to_row_addresses<'a>( + source: BoxScorer<'a>, + projection: &PreparedRowAddressProjection, + local_document_lower_bound: u64, +) -> Result>> { + map_scorer_to_row_addresses_with_threshold( + source, + projection, + local_document_lower_bound, + *FLAT_SEARCH_PERCENT_THRESHOLD, + ) +} + +fn map_scorer_to_row_addresses_with_threshold<'a>( + source: BoxScorer<'a>, + projection: &PreparedRowAddressProjection, + local_document_lower_bound: u64, + flat_search_percent_threshold: u64, +) -> Result>> { + match projection.projection.cached_row_address_order() { + CachedRowAddressOrder::Ordered => map_validated_scorer_to_row_addresses( + source, + projection, + projection.projection.try_ordered_row_addresses(), + local_document_lower_bound, + ), + CachedRowAddressOrder::Duplicate => map_validated_scorer_to_row_addresses( + source, + projection, + projection.projection.try_ordered_row_addresses(), + local_document_lower_bound, + ), + CachedRowAddressOrder::OutOfOrder => { + materialize_row_address_scorer(source, &projection.projection, &projection.collisions) + } + CachedRowAddressOrder::Unknown + if projection.projection.should_materialize_unknown_projection( + source.cost(), + flat_search_percent_threshold, + ) => + { + materialize_row_address_scorer(source, &projection.projection, &projection.collisions) + } + CachedRowAddressOrder::Unknown => map_validated_scorer_to_row_addresses( + source, + projection, + projection.projection.try_ordered_row_addresses(), + local_document_lower_bound, + ), + } +} + +#[derive(Debug, Clone, Copy)] +enum MergeShallowBounds { + Current { source_index: usize }, + Global(ScoreBounds), + Empty, +} + +#[derive(Debug, Clone, Copy)] +struct MergeShallowRange { + target: u64, + up_to: u64, + bounds: MergeShallowBounds, +} + +/// Merge disjoint sources for one semantic leaf in their shared row-address +/// domain. +/// +/// Exactly one source must own each row address. Keeping the current source +/// outside the heap makes both `next` and one-source `advance` O(log P), where +/// P is the number of sources, instead of scanning every source per hit. +pub(super) struct RowAddressMergeScorer<'a> { + sources: Vec>, + source_minimums: Vec, + pending: BinaryHeap>, + heads: BinaryHeap>, + active_sources: HashSet, + current: Option<(u64, usize)>, + shallow: Option, + min_competitive_score: f32, +} + +impl<'a> RowAddressMergeScorer<'a> { + pub(super) fn try_new(sources: Vec>) -> Result { + if sources.is_empty() { + return Err(Error::internal( + "row-address merge scorer requires at least one source", + )); + } + let mut pending = BinaryHeap::with_capacity(sources.len()); + let mut source_minimums = Vec::with_capacity(sources.len()); + let mut scorers = Vec::with_capacity(sources.len()); + for (source_index, source) in sources.into_iter().enumerate() { + pending.push(Reverse((source.min_possible_row_address, source_index))); + source_minimums.push(source.min_possible_row_address); + scorers.push(source.scorer); + } + Ok(Self { + heads: BinaryHeap::with_capacity(scorers.len()), + active_sources: HashSet::with_capacity(scorers.len()), + sources: scorers, + source_minimums, + pending, + current: None, + shallow: None, + min_competitive_score: f32::NEG_INFINITY, + }) + } + + fn push_positioned_source(&mut self, source_index: usize, doc: u64) -> Result<()> { + if self.sources[source_index].doc() != Some(doc) { + return Err(Error::internal(format!( + "FTS source {source_index} returned row address {doc} but reported position {:?}", + self.sources[source_index].doc() + ))); + } + self.heads.push(Reverse((doc, source_index))); + Ok(()) + } + + fn initialize_source(&mut self, source_index: usize, target: u64) -> Result<()> { + if self.min_competitive_score > f32::NEG_INFINITY { + self.sources[source_index].set_min_competitive_score(self.min_competitive_score)?; + } + let source_target = target.max(self.source_minimums[source_index]); + if let Some(doc) = self.sources[source_index].advance(source_target)? { + if doc < source_target { + return Err(Error::internal(format!( + "FTS source {source_index} initialized at row address {doc} before target {source_target}" + ))); + } + self.active_sources.insert(source_index); + self.push_positioned_source(source_index, doc)?; + } + Ok(()) + } + + fn ensure_candidate_head(&mut self, target: u64) -> Result<()> { + loop { + let actual_head = self.heads.peek().map(|Reverse((doc, _))| *doc); + let pending_head = self.pending.peek().map(|Reverse((minimum, _))| *minimum); + let should_initialize = match (actual_head, pending_head) { + (_, None) => false, + (None, Some(_)) => true, + (Some(actual), Some(pending)) => pending <= actual, + }; + if !should_initialize { + return Ok(()); + } + let Reverse((_, source_index)) = self.pending.pop().ok_or_else(|| { + Error::internal("FTS pending source heap unexpectedly became empty") + })?; + self.initialize_source(source_index, target)?; + } + } + + fn select_current(&mut self, target: u64) -> Result> { + self.shallow = None; + self.ensure_candidate_head(target)?; + let Some(Reverse((doc, source_index))) = self.heads.pop() else { + self.current = None; + return Ok(None); + }; + if let Some(Reverse((duplicate, duplicate_source))) = self.heads.peek() + && *duplicate == doc + { + self.current = None; + return Err(Error::internal(format!( + "FTS sources {source_index} and {duplicate_source} produced duplicate row address {doc}" + ))); + } + self.current = Some((doc, source_index)); + Ok(Some(doc)) + } + + fn advance_source(&mut self, source_index: usize, target: u64) -> Result<()> { + if let Some(doc) = self.sources[source_index].advance(target)? { + if doc < target { + return Err(Error::internal(format!( + "FTS source {source_index} advanced to row address {doc} before target {target}" + ))); + } + self.push_positioned_source(source_index, doc)?; + } else { + self.active_sources.remove(&source_index); + } + Ok(()) + } + + fn next_source(&mut self, source_index: usize) -> Result<()> { + if let Some(doc) = self.sources[source_index].next()? { + self.push_positioned_source(source_index, doc)?; + } else { + self.active_sources.remove(&source_index); + } + Ok(()) + } + + fn current_source_mut(&mut self) -> Result<&mut BoxScorer<'a>> { + let (_, source_index) = self.current.ok_or_else(|| { + Error::internal("row-address merge scorer is not positioned on a document") + })?; + Ok(&mut self.sources[source_index]) + } + + fn next_source_boundary(&self) -> Option { + self.heads + .peek() + .map(|Reverse((doc, _))| *doc) + .into_iter() + .chain(self.pending.peek().map(|Reverse((minimum, _))| *minimum)) + .min() + } + + fn global_range_bounds(&self) -> ScoreBounds { + ScoreBounds { + lower: if self.scores_non_negative() { + 0.0 + } else { + f32::NEG_INFINITY + }, + upper: self.global_score_upper_bound().unwrap_or(f32::INFINITY), + } + } +} + +impl ComposableScorer for RowAddressMergeScorer<'_> { + fn doc(&self) -> Option { + self.current.map(|(doc, _)| doc) + } + + fn document_key(&self) -> Option { + self.doc() + } + + fn next(&mut self) -> Result> { + let target = match self.current.take() { + Some((u64::MAX, source_index)) => { + self.active_sources.remove(&source_index); + self.shallow = None; + return Ok(None); + } + Some((doc, source_index)) => { + self.next_source(source_index)?; + doc + 1 + } + None if self.heads.is_empty() && self.pending.is_empty() => return Ok(None), + None => 0, + }; + self.select_current(target) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.doc().is_some_and(|doc| doc >= target) { + return Ok(self.doc()); + } + if let Some((_, source_index)) = self.current.take() { + self.advance_source(source_index, target)?; + } + while let Some(Reverse((doc, _))) = self.heads.peek() { + if *doc >= target { + break; + } + let Reverse((_, source_index)) = self + .heads + .pop() + .ok_or_else(|| Error::internal("FTS source heap unexpectedly became empty"))?; + self.advance_source(source_index, target)?; + } + self.select_current(target) + } + + fn cost(&self) -> usize { + self.sources + .iter() + .map(|source| source.cost()) + .fold(0, usize::saturating_add) + } + + fn score(&mut self) -> Result { + self.current_source_mut()?.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let Some((current, source_index)) = self.current else { + self.shallow = Some(MergeShallowRange { + target, + up_to: u64::MAX, + bounds: MergeShallowBounds::Empty, + }); + return Ok(u64::MAX); + }; + let next_source = self.next_source_boundary(); + if next_source.is_some_and(|boundary| boundary <= target) { + let bounds = self.global_range_bounds(); + self.shallow = Some(MergeShallowRange { + target, + up_to: target, + bounds: MergeShallowBounds::Global(bounds), + }); + return Ok(target); + } + + let source_target = target.max(current); + let source_up_to = self.sources[source_index].advance_shallow(source_target)?; + if source_up_to < source_target { + return Err(Error::internal(format!( + "FTS source {source_index} returned shallow range ending at {source_up_to} before target {source_target}" + ))); + } + let up_to = next_source + .map(|boundary| source_up_to.min(boundary.saturating_sub(1))) + .unwrap_or(source_up_to); + self.shallow = Some(MergeShallowRange { + target, + up_to, + bounds: MergeShallowBounds::Current { source_index }, + }); + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let shallow = self.shallow.ok_or_else(|| { + Error::internal("score_bounds requires advance_shallow on the row-address merge scorer") + })?; + if up_to < shallow.target || up_to > shallow.up_to { + return Err(Error::internal(format!( + "FTS row-address merge bound up_to={up_to} is outside shallow range [{}, {}]", + shallow.target, shallow.up_to + ))); + } + match shallow.bounds { + MergeShallowBounds::Current { source_index } => { + self.sources[source_index].score_bounds(up_to) + } + MergeShallowBounds::Global(bounds) => Ok(bounds), + MergeShallowBounds::Empty => Ok(ScoreBounds::ZERO), + } + } + + fn global_score_upper_bound(&self) -> Option { + self.sources + .iter() + .map(|source| source.global_score_upper_bound()) + .try_fold(f32::NEG_INFINITY, |upper, source_upper| { + let source_upper = source_upper?; + source_upper.is_finite().then_some(upper.max(source_upper)) + }) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score <= self.min_competitive_score { + return Ok(()); + } + for source_index in self.active_sources.iter().copied() { + self.sources[source_index].set_min_competitive_score(min_score)?; + } + self.min_competitive_score = min_score; + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + self.current_source_mut()?.current_score_upper_bound() + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.sources + .iter() + .any(|source| source.supports_doc_local_confirmation_pruning()) + } + + fn matches(&mut self) -> Result { + self.current_source_mut()?.matches() + } + + fn match_cost(&self) -> Option { + self.sources + .iter() + .map(|source| source.match_cost()) + .try_fold(0.0_f32, |cost, source_cost| { + source_cost.map(|source_cost| cost.max(source_cost)) + }) + } + + fn scores_non_negative(&self) -> bool { + self.sources + .iter() + .all(|source| source.scores_non_negative()) + } +} + +/// Monotonic score-only floor shared by partition-local top-k collectors. +/// +/// Equal-score candidates are never pruned because final ordering also uses +/// row id. The score-only floor is therefore a safe lower bound even when +/// partitions encounter ties in different orders. +#[derive(Debug)] +pub(super) struct CompetitiveScore { + bits: AtomicU32, +} + +impl Default for CompetitiveScore { + fn default() -> Self { + Self { + bits: AtomicU32::new(f32::NEG_INFINITY.to_bits()), + } + } +} + +impl CompetitiveScore { + fn get(&self) -> f32 { + f32::from_bits(self.bits.load(AtomicOrdering::Relaxed)) + } + + fn raise(&self, score: f32) { + debug_assert!(!score.is_nan()); + let mut current = self.bits.load(AtomicOrdering::Relaxed); + while score > f32::from_bits(current) { + match self.bits.compare_exchange_weak( + current, + score.to_bits(), + AtomicOrdering::Relaxed, + AtomicOrdering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + } +} + +#[derive(Debug, Clone, Copy)] +struct HeapRow(ScoredRow); + +impl PartialEq for HeapRow { + fn eq(&self, other: &Self) -> bool { + self.0.row_id == other.0.row_id && self.0.score.to_bits() == other.0.score.to_bits() + } +} + +impl Eq for HeapRow {} + +impl PartialOrd for HeapRow { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HeapRow { + fn cmp(&self, other: &Self) -> Ordering { + // The worst result is the heap maximum: lower score, then higher row id. + other + .0 + .score + .total_cmp(&self.0.score) + .then_with(|| self.0.row_id.cmp(&other.0.row_id)) + } +} + +fn compare_scored_rows(left: &ScoredRow, right: &ScoredRow) -> Ordering { + right + .score + .total_cmp(&left.score) + .then_with(|| left.row_id.cmp(&right.row_id)) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CollectionStatus { + Complete, + ScoreFloorOverflow, +} + +#[derive(Debug, Clone, Copy)] +enum TieHandling { + ResolveByKey, + RetainScoreFloor { max_buffered: usize }, +} + +/// The sole owner of top-k state for a compound scorer tree. +/// +/// Resolved document keys use the normal `(score DESC, row_id ASC)` ordering +/// and keep exactly `limit` rows. An unresolved partition temporarily retains +/// its kth-score floor, but stops once the bounded resolution buffer fills. +pub(super) struct TopKCollector { + limit: usize, + heap: BinaryHeap>, + competitive_score: Arc, + tie_handling: TieHandling, +} + +impl TopKCollector { + pub(super) fn new(limit: usize) -> Self { + Self::with_competitive_score(limit, Arc::new(CompetitiveScore::default())) + } + + pub(super) fn with_competitive_score( + limit: usize, + competitive_score: Arc, + ) -> Self { + Self::with_tie_handling(limit, competitive_score, TieHandling::ResolveByKey) + } + + fn retaining_score_floor( + limit: usize, + competitive_score: Arc, + max_buffered: usize, + ) -> Self { + debug_assert!(max_buffered >= limit); + Self::with_tie_handling( + limit, + competitive_score, + TieHandling::RetainScoreFloor { max_buffered }, + ) + } + + fn with_tie_handling( + limit: usize, + competitive_score: Arc, + tie_handling: TieHandling, + ) -> Self { + Self { + limit, + heap: BinaryHeap::with_capacity(limit.min(DEFAULT_BLOCK_SIZE)), + competitive_score, + tie_handling, + } + } + + fn insert(&mut self, row: ScoredRow) -> CollectionStatus { + if self.limit == 0 { + return CollectionStatus::Complete; + } + if self.heap.len() < self.limit { + self.heap.push(HeapRow(row)); + } else { + let worst = self.heap.peek().expect("a full top-k heap is non-empty").0; + match row.score.total_cmp(&worst.score) { + Ordering::Less => {} + Ordering::Equal => match self.tie_handling { + TieHandling::ResolveByKey => { + if row.row_id < worst.row_id { + self.heap.pop(); + self.heap.push(HeapRow(row)); + } + } + TieHandling::RetainScoreFloor { max_buffered } => { + if self.heap.len() >= max_buffered { + self.raise_competitive_score(); + return CollectionStatus::ScoreFloorOverflow; + } + self.heap.push(HeapRow(row)); + } + }, + Ordering::Greater => { + match self.tie_handling { + TieHandling::ResolveByKey => { + self.heap.pop(); + self.heap.push(HeapRow(row)); + } + TieHandling::RetainScoreFloor { max_buffered } => { + self.heap.push(HeapRow(row)); + self.prune_obsolete_score_floors(); + if self.heap.len() > max_buffered { + // This collector is discarded and the partition is + // retried with resolved keys. Keep its observable + // working set within the advertised bound meanwhile. + self.heap.pop(); + self.raise_competitive_score(); + return CollectionStatus::ScoreFloorOverflow; + } + } + } + } + } + } + self.raise_competitive_score(); + CollectionStatus::Complete + } + + fn raise_competitive_score(&self) { + if self.heap.len() >= self.limit + && let Some(worst) = self.heap.peek() + { + self.competitive_score.raise(worst.0.score); + } + } + + fn prune_obsolete_score_floors(&mut self) { + while self.heap.len() > self.limit { + let floor = self + .heap + .peek() + .expect("a non-empty top-k heap has a score floor") + .0 + .score; + let floor_count = self + .heap + .iter() + .filter(|row| row.0.score.total_cmp(&floor) == Ordering::Equal) + .count(); + if self.heap.len() - floor_count < self.limit { + break; + } + self.heap + .retain(|row| row.0.score.total_cmp(&floor) != Ordering::Equal); + } + } + + pub(super) fn collect_mapped( + &mut self, + scorer: &mut dyn ComposableScorer, + mut map_document: impl FnMut(u64) -> Result, + ) -> Result { + if self.limit == 0 { + return Ok(CollectionStatus::Complete); + } + let capacity_limit = match self.tie_handling { + TieHandling::ResolveByKey => self.limit, + TieHandling::RetainScoreFloor { max_buffered } => max_buffered, + }; + let expected = scorer.cost().min(capacity_limit); + self.heap + .reserve(expected.saturating_sub(self.heap.capacity())); + + scorer.set_min_competitive_score(self.competitive_score.get())?; + let mut doc = scorer.next()?; + while let Some(doc_id) = doc { + let min_score = self.competitive_score.get(); + scorer.set_min_competitive_score(min_score)?; + let up_to = scorer.advance_shallow(doc_id)?; + let bounds = scorer.score_bounds(up_to)?; + if bounds.upper < min_score { + doc = if up_to == u64::MAX { + None + } else { + scorer.advance(up_to + 1)? + }; + continue; + } + + // Phrase leaves expose their score from posting frequencies before + // positions are decoded. Composite scorers combine those doc-local + // uppers with sibling residuals, so a strict miss can bypass every + // pending position confirmation. Equality remains live because row + // id is the final top-k tie breaker. + if min_score.is_finite() + && scorer.supports_doc_local_confirmation_pruning() + && scorer + .current_score_upper_bound()? + .is_some_and(|upper| upper < min_score) + { + doc = scorer.next()?; + continue; + } + + if let Some(match_cost) = scorer.match_cost() + && (!match_cost.is_finite() || match_cost < 0.0) + { + return Err(Error::internal(format!( + "FTS scorer reported invalid two-phase match cost: {match_cost}" + ))); + } + if scorer.matches()? { + let score = checked_score(scorer.score()?, "compound scorer")?; + // A shared partition floor is already known to be globally + // competitive. Scores strictly below it cannot enter final top-k. + if score >= self.competitive_score.get() { + let document_key = scorer.document_key().ok_or_else(|| { + Error::internal( + "compound FTS scorer did not expose its current document key", + ) + })?; + let status = self.insert(ScoredRow { + row_id: map_document(document_key)?, + score, + }); + if status == CollectionStatus::ScoreFloorOverflow { + return Ok(status); + } + } + } + doc = scorer.next()?; + } + + Ok(CollectionStatus::Complete) + } + + fn into_candidates(self) -> Vec> { + let mut rows = self.heap.into_iter().map(|row| row.0).collect::>(); + rows.sort_unstable_by(compare_scored_rows); + rows + } + + pub(super) fn into_rows(self) -> Vec> { + let limit = self.limit; + let mut rows = self.into_candidates(); + rows.truncate(limit); + rows + } +} + +impl TopKCollector { + pub(super) fn collect(mut self, scorer: &mut dyn ComposableScorer) -> Result> { + self.collect_mapped(scorer, Ok)?; + Ok(self.into_rows()) + } +} + +/// Evaluate a compound query over exact, materialized leaf result sets. +/// +/// This is the bridge used by query-local residual postings: it keeps Boolean, +/// Boost, and MultiMatch semantics in the same scorer tree as the on-disk +/// compound path while allowing a different posting source. +#[doc(hidden)] +pub fn materialized_compound_top_k( + query: &FtsQuery, + leaves: Vec>, + limit: usize, +) -> Result<(Vec, Vec)> { + let mut leaf_count = 0; + let plan = CompoundScorerPlan::from_query(query, &mut leaf_count)?; + if leaf_count != leaves.len() { + return Err(Error::internal(format!( + "compound FTS planned {leaf_count} leaves but received {} materialized leaves", + leaves.len() + ))); + } + let mut scorers = leaves + .into_iter() + .map(|rows| { + let rows = rows + .into_iter() + .map(|(row_id, score)| ScoredRow { row_id, score }) + .collect(); + MaterializedScorer::try_new(rows).map(|scorer| Some(Box::new(scorer) as BoxScorer<'_>)) + }) + .collect::>>()?; + let mut scorer = plan.build(&mut scorers)?; + let rows = TopKCollector::new(limit).collect(scorer.as_mut())?; + Ok(rows.into_iter().map(|row| (row.row_id, row.score)).unzip()) +} + +#[derive(Debug, Clone, Copy)] +pub(super) enum DisjunctionScore { + Sum, + Max, +} + +pub(super) struct EmptyScorer; + +impl ComposableScorer for EmptyScorer { + fn doc(&self) -> Option { + None + } + + fn next(&mut self) -> Result> { + Ok(None) + } + + fn advance(&mut self, _target: u64) -> Result> { + Ok(None) + } + + fn cost(&self) -> usize { + 0 + } + + fn score(&mut self) -> Result { + Err(Error::internal( + "score requested from an empty compound FTS scorer", + )) + } + + fn advance_shallow(&mut self, _target: u64) -> Result { + Ok(u64::MAX) + } + + fn score_bounds(&mut self, _up_to: u64) -> Result { + Ok(ScoreBounds::ZERO) + } + + fn global_score_upper_bound(&self) -> Option { + Some(0.0) + } + + fn set_min_competitive_score(&mut self, _min_score: f32) -> Result<()> { + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + Ok(Some(0.0)) + } + + fn scores_non_negative(&self) -> bool { + true + } +} + +struct ScaleScorer<'a> { + child: BoxScorer<'a>, + factor: f32, + last_parent_score_floor: f32, + #[cfg(test)] + score_floor_translations: usize, +} + +impl<'a> ScaleScorer<'a> { + fn try_new(child: BoxScorer<'a>, factor: f32) -> Result { + if !factor.is_finite() || factor < 0.0 { + return Err(Error::invalid_input(format!( + "MatchQuery boost must be finite and non-negative, got {factor}" + ))); + } + Ok(Self { + child, + factor, + last_parent_score_floor: f32::NEG_INFINITY, + #[cfg(test)] + score_floor_translations: 0, + }) + } +} + +impl ComposableScorer for ScaleScorer<'_> { + fn doc(&self) -> Option { + self.child.doc() + } + + fn document_key(&self) -> Option { + self.child.document_key() + } + + fn next(&mut self) -> Result> { + self.child.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.child.advance(target) + } + + fn cost(&self) -> usize { + self.child.cost() + } + + fn score(&mut self) -> Result { + checked_score(self.child.score()? * self.factor, "MatchQuery boost") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.child.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + Ok(self + .child + .score_bounds(up_to)? + .scale_non_negative(self.factor)) + } + + fn global_score_upper_bound(&self) -> Option { + self.child + .global_score_upper_bound() + .map(|upper| { + ScoreBounds { lower: 0.0, upper } + .scale_non_negative(self.factor) + .upper + }) + .filter(|upper| upper.is_finite() && *upper >= 0.0) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive MatchQuery score cannot be NaN", + )); + } + if min_score <= self.last_parent_score_floor { + return Ok(()); + } + #[cfg(test)] + { + self.score_floor_translations += 1; + } + if let Some(child_floor) = exclusive_scaled_score_floor(min_score, self.factor) { + self.child.set_min_competitive_score(child_floor)?; + } + self.last_parent_score_floor = min_score; + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + Ok(self.child.current_score_upper_bound()?.map(|upper| { + ScoreBounds { lower: 0.0, upper } + .scale_non_negative(self.factor) + .upper + })) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.child.supports_doc_local_confirmation_pruning() + } + + fn matches(&mut self) -> Result { + self.child.matches() + } + + fn match_cost(&self) -> Option { + self.child.match_cost() + } + + fn scores_non_negative(&self) -> bool { + self.child.scores_non_negative() + } +} + +/// Union scorer used for Boolean SHOULD sums and MultiMatch DisMax. +pub(super) struct DisjunctionScorer<'a> { + children: Vec>, + mode: DisjunctionScore, + current: Option, + confirmed_doc: Option, + confirmed: Vec, + min_competitive_score: f32, +} + +impl<'a> DisjunctionScorer<'a> { + pub(super) fn try_new(children: Vec>, mode: DisjunctionScore) -> Result { + if children.is_empty() { + return Err(Error::internal( + "FTS disjunction scorer requires at least one child", + )); + } + let confirmed = vec![false; children.len()]; + Ok(Self { + children, + mode, + current: None, + confirmed_doc: None, + confirmed, + min_competitive_score: f32::NEG_INFINITY, + }) + } + + fn set_current_from_children(&mut self) -> Option { + self.current = self.children.iter().filter_map(|child| child.doc()).min(); + self.confirmed_doc = None; + self.confirmed.fill(false); + self.current + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed.iter().any(|matched| *matched)); + } + self.confirmed.fill(false); + for (matched, child) in self.confirmed.iter_mut().zip(&mut self.children) { + if child.doc() == Some(current) { + *matched = child.matches()?; + } + } + self.confirmed_doc = Some(current); + Ok(self.confirmed.iter().any(|matched| *matched)) + } +} + +impl ComposableScorer for DisjunctionScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + let current = self.current?; + self.children + .iter() + .find(|child| child.doc() == Some(current)) + .and_then(|child| child.document_key()) + } + + fn next(&mut self) -> Result> { + match self.current { + None => { + for child in &mut self.children { + child.next()?; + } + } + Some(current) => { + for child in &mut self.children { + if child.doc() == Some(current) { + child.next()?; + } + } + } + } + Ok(self.set_current_from_children()) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + for child in &mut self.children { + if child.doc().is_none_or(|doc| doc < target) { + child.advance(target)?; + } + } + Ok(self.set_current_from_children()) + } + + fn cost(&self) -> usize { + self.children + .iter() + .map(|child| child.cost()) + .fold(0, usize::saturating_add) + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "FTS disjunction score requested for an unconfirmed document", + )); + } + let mut score = match self.mode { + DisjunctionScore::Sum => 0.0_f32, + DisjunctionScore::Max => f32::NEG_INFINITY, + }; + for (matched, child) in self.confirmed.iter().zip(&mut self.children) { + if !matched { + continue; + } + let child_score = child.score()?; + score = match self.mode { + DisjunctionScore::Sum => score + child_score, + DisjunctionScore::Max => score.max(child_score), + }; + } + checked_score(score, "FTS disjunction") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = u64::MAX; + for child in &mut self.children { + if let Some(doc) = child.doc() { + up_to = up_to.min(child.advance_shallow(target.max(doc))?); + } + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let mut bounds = match self.mode { + DisjunctionScore::Sum => ScoreBounds::ZERO, + DisjunctionScore::Max => ScoreBounds { + lower: f32::INFINITY, + upper: f32::NEG_INFINITY, + }, + }; + for child in &mut self.children { + let child_bounds = if child.doc().is_some_and(|doc| doc <= up_to) { + child.score_bounds(up_to)? + } else { + ScoreBounds::ZERO + }; + bounds = match self.mode { + DisjunctionScore::Sum => bounds.add(child_bounds.include_zero()), + DisjunctionScore::Max => ScoreBounds { + lower: bounds.lower.min(child_bounds.lower), + upper: bounds.upper.max(child_bounds.upper), + }, + }; + } + if bounds.lower == f32::INFINITY { + Ok(ScoreBounds::ZERO) + } else { + Ok(bounds) + } + } + + fn global_score_upper_bound(&self) -> Option { + match self.mode { + DisjunctionScore::Sum => sum_global_score_upper_bounds(&self.children), + DisjunctionScore::Max => self.children.iter().try_fold(0.0_f32, |upper, child| { + let child_upper = child.global_score_upper_bound()?; + child_upper.is_finite().then_some(upper.max(child_upper)) + }), + } + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score <= self.min_competitive_score { + return Ok(()); + } + self.min_competitive_score = min_score; + // A child below a DisMax threshold cannot affect a competitive max. + // Sum scorers need sibling-global bounds before translating the floor, + // so they keep it at this node and prune from their combined block bound. + if matches!(self.mode, DisjunctionScore::Max) { + for child in &mut self.children { + child.set_min_competitive_score(min_score)?; + } + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + let Some(current) = self.current else { + return Ok(None); + }; + let mut upper = 0.0_f32; + for child in &mut self.children { + if child.doc() != Some(current) { + continue; + } + let Some(child_upper) = child.current_score_upper_bound()? else { + return Ok(None); + }; + if !child_upper.is_finite() { + return Ok(None); + } + upper = match self.mode { + DisjunctionScore::Sum => { + ScoreBounds { lower: 0.0, upper } + .add(ScoreBounds { + lower: 0.0, + upper: child_upper.max(0.0), + }) + .upper + } + DisjunctionScore::Max => upper.max(child_upper), + }; + } + Ok(upper.is_finite().then_some(upper)) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.children + .iter() + .any(|child| child.supports_doc_local_confirmation_pruning()) + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.children + .iter() + .filter_map(|child| child.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + self.children + .iter() + .all(|child| child.scores_non_negative()) + } +} + +/// Intersection scorer that requires and scores every Boolean MUST child. +pub(super) struct RequiredConjunctionScorer<'a> { + children: Vec>, + /// Child indices sorted by approximation cost, omitted when query order is + /// already cheapest-first. `children` remains in query order so scoring and + /// score-bound arithmetic stay bit-for-bit stable. + approximation_order: Option>, + /// Child indices sorted by two-phase confirmation cost. Children without a + /// cost hint remain in query order after costed confirmations. + confirmation_order: Option>, + current: Option, + confirmed_doc: Option, + confirmed: bool, +} + +fn align_conjunction_children( + children: &mut [BoxScorer<'_>], + mut target: u64, + child_index: impl Fn(usize) -> usize, +) -> Result> { + loop { + for position in 0..children.len() { + let child = &mut children[child_index(position)]; + if child.doc().is_none_or(|doc| doc < target) { + let Some(doc) = child.advance(target)? else { + return Ok(None); + }; + target = target.max(doc); + } + } + let min_doc = children.iter().filter_map(|child| child.doc()).min(); + let max_doc = children.iter().filter_map(|child| child.doc()).max(); + if min_doc == max_doc { + return Ok(min_doc); + } + target = max_doc.ok_or_else(|| { + Error::internal("FTS conjunction lost a child while aligning scorers") + })?; + } +} + +fn compare_confirmation_cost( + left: &dyn ComposableScorer, + right: &dyn ComposableScorer, +) -> Ordering { + match (left.match_cost(), right.match_cost()) { + (Some(left), Some(right)) => left.total_cmp(&right), + (Some(_), None) => Ordering::Less, + (None, Some(_)) => Ordering::Greater, + (None, None) => Ordering::Equal, + } +} + +fn confirm_conjunction_children( + children: &mut [BoxScorer<'_>], + child_index: impl Fn(usize) -> usize, +) -> Result { + for position in 0..children.len() { + if !children[child_index(position)].matches()? { + return Ok(false); + } + } + Ok(true) +} + +impl<'a> RequiredConjunctionScorer<'a> { + pub(super) fn try_new(children: Vec>) -> Result { + if children.is_empty() { + return Err(Error::internal( + "FTS conjunction scorer requires at least one child", + )); + } + let approximation_order = if children + .windows(2) + .all(|pair| pair[0].cost() <= pair[1].cost()) + { + None + } else { + let mut order = (0..children.len()).collect::>(); + order.sort_by_key(|&index| (children[index].cost(), index)); + Some(order) + }; + for (index, child) in children.iter().enumerate() { + if let Some(match_cost) = child.match_cost() + && (!match_cost.is_finite() || match_cost < 0.0) + { + return Err(Error::internal(format!( + "FTS conjunction child {index} reported invalid two-phase match cost: {match_cost}" + ))); + } + } + let confirmation_order = if children.windows(2).all(|pair| { + compare_confirmation_cost(pair[0].as_ref(), pair[1].as_ref()) != Ordering::Greater + }) { + None + } else { + let mut order = (0..children.len()).collect::>(); + order.sort_by(|&left, &right| { + compare_confirmation_cost(children[left].as_ref(), children[right].as_ref()) + .then_with(|| left.cmp(&right)) + }); + Some(order) + }; + Ok(Self { + children, + approximation_order, + confirmation_order, + current: None, + confirmed_doc: None, + confirmed: false, + }) + } + + fn align(&mut self, target: u64) -> Result> { + self.current = if let Some(order) = &self.approximation_order { + align_conjunction_children(&mut self.children, target, |position| order[position])? + } else { + align_conjunction_children(&mut self.children, target, |position| position)? + }; + if self.current.is_some() { + self.confirmed_doc = None; + self.confirmed = false; + } + Ok(self.current) + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed); + } + self.confirmed = if let Some(order) = &self.confirmation_order { + confirm_conjunction_children(&mut self.children, |position| order[position])? + } else { + confirm_conjunction_children(&mut self.children, |position| position)? + }; + self.confirmed_doc = Some(current); + Ok(self.confirmed) + } +} + +impl ComposableScorer for RequiredConjunctionScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.children.first().and_then(|child| child.document_key()) + } + + fn next(&mut self) -> Result> { + let target = match self.current { + None => 0, + Some(u64::MAX) => return Ok(None), + Some(current) => current + 1, + }; + self.align(target) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + self.align(target) + } + + fn cost(&self) -> usize { + self.children + .iter() + .map(|child| child.cost()) + .min() + .unwrap_or(0) + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "FTS conjunction score requested for an unconfirmed document", + )); + } + let mut score = 0.0_f32; + for child in &mut self.children { + score += child.score()?; + } + checked_score(score, "FTS conjunction") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = u64::MAX; + for child in &mut self.children { + let child_target = child.doc().map_or(target, |doc| target.max(doc)); + up_to = up_to.min(child.advance_shallow(child_target)?); + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let mut bounds = ScoreBounds::ZERO; + for child in &mut self.children { + bounds = bounds.add(child.score_bounds(up_to)?); + } + Ok(bounds) + } + + fn global_score_upper_bound(&self) -> Option { + self.scores_non_negative() + .then(|| sum_global_score_upper_bounds(&self.children)) + .flatten() + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + // Propagating the full conjunction floor to one child is unsafe because + // individually sub-threshold MUST scores may sum to a competitive hit. + if self.children.len() == 1 { + self.children[0].set_min_competitive_score(min_score)?; + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + let Some(current) = self.current else { + return Ok(None); + }; + let mut bounds = ScoreBounds::ZERO; + for child in &mut self.children { + if child.doc() != Some(current) { + return Ok(None); + } + let Some(upper) = child.current_score_upper_bound()? else { + return Ok(None); + }; + if !upper.is_finite() { + return Ok(None); + } + bounds = bounds.add(ScoreBounds { + lower: 0.0, + upper: upper.max(0.0), + }); + } + Ok(bounds.upper.is_finite().then_some(bounds.upper)) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.children + .iter() + .any(|child| child.supports_doc_local_confirmation_pruning()) + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.children + .iter() + .filter_map(|child| child.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + self.children + .iter() + .all(|child| child.scores_non_negative()) + } +} + +/// Positive-driven Boost scorer with signed conservative bounds. +pub(super) struct BoostScorer<'a> { + positive: BoxScorer<'a>, + negative: BoxScorer<'a>, + negative_boost: f32, + negative_matches_doc: Option, + negative_matches: bool, +} + +impl<'a> BoostScorer<'a> { + pub(super) fn try_new( + positive: BoxScorer<'a>, + negative: BoxScorer<'a>, + negative_boost: f32, + ) -> Result { + if !negative_boost.is_finite() || negative_boost < 0.0 { + return Err(Error::invalid_input(format!( + "BoostQuery negative_boost must be finite and non-negative, got {negative_boost}" + ))); + } + Ok(Self { + positive, + negative, + negative_boost, + negative_matches_doc: None, + negative_matches: false, + }) + } + + fn reset_confirmation(&mut self) { + self.negative_matches_doc = None; + self.negative_matches = false; + } + + fn confirm_negative(&mut self) -> Result { + let Some(current) = self.positive.doc() else { + return Ok(false); + }; + if self.negative_matches_doc == Some(current) { + return Ok(self.negative_matches); + } + self.negative_matches = + self.negative.advance(current)? == Some(current) && self.negative.matches()?; + self.negative_matches_doc = Some(current); + Ok(self.negative_matches) + } +} + +impl ComposableScorer for BoostScorer<'_> { + fn doc(&self) -> Option { + self.positive.doc() + } + + fn document_key(&self) -> Option { + self.positive.document_key() + } + + fn next(&mut self) -> Result> { + self.reset_confirmation(); + self.positive.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.reset_confirmation(); + self.positive.advance(target) + } + + fn cost(&self) -> usize { + self.positive.cost() + } + + fn score(&mut self) -> Result { + let positive = self.positive.score()?; + let score = if self.confirm_negative()? { + positive - self.negative_boost * self.negative.score()? + } else { + positive + }; + checked_score(score, "BoostQuery scorer") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = self.positive.advance_shallow(target)?; + if self.negative.doc().is_none_or(|doc| doc < target) { + self.negative.advance(target)?; + } + if let Some(doc) = self.negative.doc() { + up_to = up_to.min(self.negative.advance_shallow(target.max(doc))?); + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let positive = self.positive.score_bounds(up_to)?; + let negative = if self.negative.doc().is_some_and(|doc| doc <= up_to) { + self.negative.score_bounds(up_to)?.include_zero() + } else { + ScoreBounds::ZERO + }; + Ok(positive.subtract_scaled(negative, self.negative_boost)) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + // With a non-negative negative scorer, Boost can only demote the + // positive score, so the parent's floor is safe for the positive side. + if self.negative.scores_non_negative() { + self.positive.set_min_competitive_score(min_score)?; + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + if self.negative.scores_non_negative() { + self.positive.current_score_upper_bound() + } else { + Ok(None) + } + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.positive.supports_doc_local_confirmation_pruning() + } + + fn matches(&mut self) -> Result { + self.positive.matches() + } + + fn match_cost(&self) -> Option { + self.positive.match_cost() + } +} + +/// Required-plus-optional scorer that only touches the optional side when it +/// can change competitiveness or an exact score is requested. +/// +/// The required scorer drives iteration. Once a score floor is available, +/// block bounds may either skip the whole range or temporarily turn the +/// optional approximation into a required iterator when the required score +/// cannot reach the floor on its own. +#[derive(Clone, Copy)] +struct ReqOptBounds { + up_to: u64, + required: ScoreBounds, + combined: ScoreBounds, +} + +struct ReqOptScorer<'a> { + required: BoxScorer<'a>, + optional: BoxScorer<'a>, + current: Option, + exhausted: bool, + optional_initialized: bool, + optional_is_required: bool, + optional_checked_doc: Option, + optional_matches: bool, + confirmed_doc: Option, + confirmed: bool, + min_competitive_score: f32, + shallow_bounds: Option, +} + +impl<'a> ReqOptScorer<'a> { + fn new(required: BoxScorer<'a>, optional: BoxScorer<'a>) -> Self { + debug_assert!(required.scores_non_negative()); + debug_assert!(optional.scores_non_negative()); + Self { + required, + optional, + current: None, + exhausted: false, + optional_initialized: false, + optional_is_required: false, + optional_checked_doc: None, + optional_matches: false, + confirmed_doc: None, + confirmed: false, + min_competitive_score: f32::NEG_INFINITY, + shallow_bounds: None, + } + } + + fn set_current(&mut self, current: Option) { + if self.current != current { + self.optional_checked_doc = None; + self.optional_matches = false; + self.confirmed_doc = None; + self.confirmed = false; + } + self.current = current; + self.optional_is_required = false; + } + + fn set_optional_required(&mut self, required: bool) { + if self.optional_is_required != required { + self.confirmed_doc = None; + self.confirmed = false; + } + self.optional_is_required = required; + } + + fn exhaust(&mut self) -> Option { + self.exhausted = true; + self.set_current(None); + None + } + + fn ensure_optional_at_or_after(&mut self, target: u64) -> Result> { + if !self.optional_initialized || self.optional.doc().is_some_and(|doc| doc < target) { + self.optional.advance(target)?; + self.optional_initialized = true; + } + Ok(self.optional.doc()) + } + + fn optional_matches_current(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.optional_checked_doc == Some(current) { + return Ok(self.optional_matches); + } + self.optional_matches = self.ensure_optional_at_or_after(current)? == Some(current) + && self.optional.matches()?; + self.optional_checked_doc = Some(current); + Ok(self.optional_matches) + } + + fn usable_bounds(bounds: ScoreBounds) -> bool { + bounds.lower.is_finite() && bounds.upper.is_finite() && bounds.lower <= bounds.upper + } + + fn bounds(&mut self, up_to: u64) -> Result { + if let Some(bounds) = self.shallow_bounds + && bounds.up_to == up_to + { + return Ok(bounds); + } + + let required = self.required.score_bounds(up_to)?; + let optional = if self.optional.doc().is_some_and(|doc| doc <= up_to) { + let bounds = self.optional.score_bounds(up_to)?; + if Self::usable_bounds(bounds) { + bounds.include_zero() + } else { + ScoreBounds::UNBOUNDED + } + } else { + ScoreBounds::ZERO + }; + let combined = required.add(optional); + let bounds = ReqOptBounds { + up_to, + required, + combined, + }; + self.shallow_bounds = Some(bounds); + Ok(bounds) + } + + fn position(&mut self, mut target: u64) -> Result> { + if self.exhausted { + return Ok(None); + } + + 'search: loop { + let bounds = self.shallow_bounds.filter(|bounds| target <= bounds.up_to); + + if self.min_competitive_score.is_finite() + && self.min_competitive_score > 0.0 + && let Some(bounds) = bounds + && Self::usable_bounds(bounds.required) + && Self::usable_bounds(bounds.combined) + { + if bounds.combined.upper < self.min_competitive_score { + if bounds.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = bounds.up_to + 1; + self.shallow_bounds = None; + continue; + } + + if bounds.required.upper < self.min_competitive_score { + // The optional contribution is necessary throughout this + // cached shallow range. Intersect approximations until + // both sides agree or the range is exhausted. + let Some(mut required_doc) = self.required.advance(target)? else { + return Ok(self.exhaust()); + }; + if required_doc > bounds.up_to { + target = required_doc; + self.shallow_bounds = None; + continue; + } + self.set_current(Some(required_doc)); + self.set_optional_required(true); + loop { + self.set_current(Some(required_doc)); + self.set_optional_required(true); + let Some(optional_doc) = self.ensure_optional_at_or_after(required_doc)? + else { + if bounds.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = bounds.up_to + 1; + self.shallow_bounds = None; + continue 'search; + }; + if optional_doc > bounds.up_to { + if bounds.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = bounds.up_to + 1; + self.shallow_bounds = None; + continue 'search; + } + if optional_doc == required_doc { + return Ok(self.current); + } + let Some(next_required) = self.required.advance(optional_doc)? else { + return Ok(self.exhaust()); + }; + if next_required > bounds.up_to { + target = next_required; + self.shallow_bounds = None; + continue 'search; + } + required_doc = next_required; + } + } + } + + let Some(required_doc) = self.required.advance(target)? else { + return Ok(self.exhaust()); + }; + self.set_current(Some(required_doc)); + self.set_optional_required(false); + return Ok(self.current); + } + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed); + } + self.confirmed = self.required.matches()? + && (!self.optional_is_required || self.optional_matches_current()?); + self.confirmed_doc = Some(current); + Ok(self.confirmed) + } +} + +impl ComposableScorer for ReqOptScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.required.document_key() + } + + fn next(&mut self) -> Result> { + let target = match self.current { + None => 0, + Some(u64::MAX) => return Ok(self.exhaust()), + Some(current) => current + 1, + }; + self.position(target) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + self.position(target) + } + + fn cost(&self) -> usize { + self.required.cost() + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "required-plus-optional FTS score requested for an unconfirmed document", + )); + } + let mut score = self.required.score()?; + if self.optional_matches_current()? { + score += self.optional.score()?; + } + checked_score(score, "required-plus-optional FTS scorer") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let target = self.current.map_or(target, |current| target.max(current)); + if let Some(bounds) = self.shallow_bounds + && target <= bounds.up_to + { + return Ok(bounds.up_to); + } + self.shallow_bounds = None; + let mut up_to = self.required.advance_shallow(target)?; + match self.ensure_optional_at_or_after(target)? { + Some(optional_doc) if optional_doc <= target => { + up_to = up_to.min(self.optional.advance_shallow(target)?); + } + Some(optional_doc) => { + up_to = up_to.min(optional_doc.saturating_sub(1)); + } + None => {} + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + Ok(self.bounds(up_to)?.combined) + } + + fn global_score_upper_bound(&self) -> Option { + let required = self.required.global_score_upper_bound()?; + let optional = self.optional.global_score_upper_bound()?; + let combined = ScoreBounds { + lower: 0.0, + upper: required, + } + .add(ScoreBounds { + lower: 0.0, + upper: optional, + }) + .upper; + combined.is_finite().then_some(combined) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score > self.min_competitive_score { + self.min_competitive_score = min_score; + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + let Some(current) = self.current else { + return Ok(None); + }; + let Some(required_upper) = self.required.current_score_upper_bound()? else { + return Ok(None); + }; + if !required_upper.is_finite() { + return Ok(None); + } + if required_upper >= self.min_competitive_score { + // The required side alone keeps this document competitive. Keep + // the optional iterator lazy; scoring will align it only for a + // surviving candidate. + return Ok(Some(f32::INFINITY)); + } + let optional_upper = if self.ensure_optional_at_or_after(current)? == Some(current) { + let Some(optional_upper) = self.optional.current_score_upper_bound()? else { + return Ok(None); + }; + if !optional_upper.is_finite() { + return Ok(None); + } + optional_upper.max(0.0) + } else { + 0.0 + }; + let upper = ScoreBounds { + lower: 0.0, + upper: required_upper.max(0.0), + } + .add(ScoreBounds { + lower: 0.0, + upper: optional_upper, + }) + .upper; + Ok(upper.is_finite().then_some(upper)) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.required.supports_doc_local_confirmation_pruning() + || self.optional.supports_doc_local_confirmation_pruning() + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.required + .match_cost() + .into_iter() + .chain(self.optional.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + true + } +} + +/// Boolean scorer preserving the current membership and score semantics. +pub(super) struct BooleanScorer<'a> { + driver: BoxScorer<'a>, + optional: Option>, + prohibited: Option>, + current: Option, + confirmed_doc: Option, + confirmed: bool, + optional_matches: bool, + defer_confirmation: bool, +} + +impl<'a> BooleanScorer<'a> { + pub(super) fn try_new( + should: Vec>, + must: Vec>, + must_not: Vec>, + ) -> Result { + let (driver, optional) = if must.is_empty() { + if should.is_empty() { + return Err(Error::invalid_input( + "boolean query must have at least one should/must query", + )); + } + let driver = if let Some(global_bounds) = ShouldMaxScoreScorer::global_bounds(&should) { + Box::new(ShouldMaxScoreScorer::new(should, global_bounds)) as BoxScorer<'a> + } else { + Box::new(DisjunctionScorer::try_new(should, DisjunctionScore::Sum)?) + as BoxScorer<'a> + }; + (driver, None) + } else { + let mut optional = if should.is_empty() { + None + } else { + Some( + Box::new(DisjunctionScorer::try_new(should, DisjunctionScore::Sum)?) + as BoxScorer<'a>, + ) + }; + let required = Box::new(RequiredConjunctionScorer::try_new(must)?) as BoxScorer<'a>; + let driver = if required.scores_non_negative() + && optional + .as_ref() + .is_some_and(|optional| optional.scores_non_negative()) + { + Box::new(ReqOptScorer::new( + required, + optional + .take() + .expect("checked that the optional scorer is present"), + )) as BoxScorer<'a> + } else { + required + }; + (driver, optional) + }; + let prohibited = if must_not.is_empty() { + None + } else { + Some( + Box::new(DisjunctionScorer::try_new(must_not, DisjunctionScore::Max)?) + as BoxScorer<'a>, + ) + }; + let scores_non_negative = driver.scores_non_negative() + && optional + .as_ref() + .is_none_or(|optional| optional.scores_non_negative()); + let has_doc_local_confirmation = driver.supports_doc_local_confirmation_pruning() + || optional + .as_ref() + .is_some_and(|optional| optional.supports_doc_local_confirmation_pruning()) + || prohibited + .as_ref() + .is_some_and(|prohibited| prohibited.supports_doc_local_confirmation_pruning()); + Ok(Self { + driver, + optional, + prohibited, + current: None, + confirmed_doc: None, + confirmed: false, + optional_matches: false, + defer_confirmation: scores_non_negative && has_doc_local_confirmation, + }) + } + + fn set_current(&mut self, current: Option) -> Option { + if self.current != current { + self.confirmed_doc = None; + self.confirmed = false; + self.optional_matches = false; + } + self.current = current; + current + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed); + } + if !self.driver.matches()? { + self.confirmed_doc = Some(current); + self.confirmed = false; + return Ok(false); + } + if let Some(prohibited) = &mut self.prohibited + && prohibited.advance(current)? == Some(current) + && prohibited.matches()? + { + self.confirmed_doc = Some(current); + self.confirmed = false; + return Ok(false); + } + self.optional_matches = if let Some(optional) = &mut self.optional { + optional.advance(current)? == Some(current) && optional.matches()? + } else { + false + }; + self.confirmed_doc = Some(current); + self.confirmed = true; + Ok(true) + } + + fn next_candidate(&mut self, target: Option) -> Result> { + let mut candidate = match target { + Some(target) => self.driver.advance(target)?, + None => self.driver.next()?, + }; + loop { + self.set_current(candidate); + if self.defer_confirmation || candidate.is_none() || self.ensure_confirmed()? { + return Ok(self.current); + } + candidate = self.driver.next()?; + } + } +} + +impl ComposableScorer for BooleanScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + self.driver.document_key() + } + + fn next(&mut self) -> Result> { + self.next_candidate(None) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + self.next_candidate(Some(target)) + } + + fn cost(&self) -> usize { + self.driver.cost() + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "Boolean FTS score requested for an unconfirmed document", + )); + } + let mut score = self.driver.score()?; + if self.optional_matches + && let Some(optional) = &mut self.optional + { + score += optional.score()?; + } + checked_score(score, "BooleanQuery scorer") + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let mut up_to = self.driver.advance_shallow(target)?; + if let Some(optional) = &mut self.optional + && let Some(doc) = optional.doc() + { + up_to = up_to.min(optional.advance_shallow(target.max(doc))?); + } + Ok(up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let mut bounds = self.driver.score_bounds(up_to)?; + if let Some(optional) = &mut self.optional + && optional.doc().is_some_and(|doc| doc <= up_to) + { + bounds = bounds.add(optional.score_bounds(up_to)?.include_zero()); + } + Ok(bounds) + } + + fn global_score_upper_bound(&self) -> Option { + if !self.scores_non_negative() { + return None; + } + let driver = self.driver.global_score_upper_bound()?; + let combined = if let Some(optional) = &self.optional { + let optional = optional.global_score_upper_bound()?; + ScoreBounds { + lower: 0.0, + upper: driver, + } + .add(ScoreBounds { + lower: 0.0, + upper: optional, + }) + .upper + } else { + driver + }; + (combined.is_finite() && combined >= 0.0).then_some(combined) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + // When SHOULD is also present, a global sibling bound is required to + // translate the parent threshold safely. The combined block bound still + // prunes at this node. Without SHOULD, driver score is the full score. + if self.optional.is_none() { + self.driver.set_min_competitive_score(min_score)?; + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + let Some(current) = self.current else { + return Ok(None); + }; + if !self.scores_non_negative() { + return Ok(None); + } + let Some(driver_upper) = self.driver.current_score_upper_bound()? else { + return Ok(None); + }; + if !driver_upper.is_finite() { + return Ok(None); + } + let optional_upper = if let Some(optional) = &mut self.optional { + if optional.advance(current)? == Some(current) { + let Some(optional_upper) = optional.current_score_upper_bound()? else { + return Ok(None); + }; + if !optional_upper.is_finite() { + return Ok(None); + } + optional_upper.max(0.0) + } else { + 0.0 + } + } else { + 0.0 + }; + let upper = ScoreBounds { + lower: 0.0, + upper: driver_upper, + } + .add(ScoreBounds { + lower: 0.0, + upper: optional_upper, + }) + .upper; + Ok(upper.is_finite().then_some(upper)) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.defer_confirmation + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.driver + .match_cost() + .into_iter() + .chain( + self.optional + .as_ref() + .and_then(|optional| optional.match_cost()), + ) + .chain( + self.prohibited + .as_ref() + .and_then(|prohibited| prohibited.match_cost()), + ) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + self.driver.scores_non_negative() + && self + .optional + .as_ref() + .is_none_or(|optional| optional.scores_non_negative()) + } +} + +#[derive(Clone)] +pub(super) enum LeafQuery { + Match(MatchQuery), + Phrase(PhraseQuery), +} + +impl LeafQuery { + pub(super) fn terms(&self) -> &str { + match self { + Self::Match(query) => &query.terms, + Self::Phrase(query) => &query.terms, + } + } + + pub(super) fn operator(&self) -> Operator { + match self { + Self::Match(query) => query.operator, + Self::Phrase(_) => Operator::And, + } + } + + pub(super) fn effective_params(&self, params: &FtsSearchParams) -> FtsSearchParams { + match self { + Self::Match(query) => params + .clone() + .with_limit(None) + .with_phrase_slop(None) + .with_fuzziness(query.fuzziness) + .with_max_expansions(query.max_expansions) + .with_prefix_length(query.prefix_length), + Self::Phrase(query) => params + .clone() + .with_limit(None) + .with_phrase_slop(Some(query.slop)), + } + } +} + +pub(super) fn collect_leaf_queries(query: &FtsQuery, leaves: &mut Vec) -> Result<()> { + match query { + FtsQuery::Match(query) => leaves.push(LeafQuery::Match(query.clone())), + FtsQuery::Phrase(query) => leaves.push(LeafQuery::Phrase(query.clone())), + FtsQuery::Boost(query) => { + collect_leaf_queries(&query.positive, leaves)?; + collect_leaf_queries(&query.negative, leaves)?; + } + FtsQuery::MultiMatch(query) => { + leaves.extend(query.match_queries.iter().cloned().map(LeafQuery::Match)); + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + collect_leaf_queries(child, leaves)?; + } + } + } + Ok(()) +} + +struct PreparedLeaf { + query: Arc, + params: Arc, + operator: Operator, +} + +pub(super) fn tokenize_leaf( + index: &InvertedIndex, + leaf: &LeafQuery, + params: &FtsSearchParams, +) -> Tokens { + // Keep the legacy explicit-fuzzy rewrite independent of index analysis. + // AUTO fuzziness still expands later, but its source terms must first use + // the same normalization and filtering as the indexed vocabulary. + let is_explicit_fuzzy_match = matches!(leaf, LeafQuery::Match(_)) + && matches!(params.fuzziness, Some(distance) if distance > 0); + let mut tokenizer = if is_explicit_fuzzy_match { + let analyzer = TextAnalyzer::from(SimpleTokenizer::default()); + match index.tokenizer().doc_type() { + DocType::Text => Box::new(TextTokenizer::new(analyzer)) as Box, + DocType::Json => Box::new(JsonTokenizer::new(analyzer)) as Box, + } + } else { + index.tokenizer() + }; + collect_query_tokens(leaf.terms(), &mut tokenizer) +} + +async fn prepare_compound_query( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + metrics: &dyn MetricsCollector, + base_scorer: Option>, + prepared_match: Option>, +) -> Result<(CompoundScorerPlan, Vec)> { + let first_index = indices + .first() + .ok_or_else(|| Error::invalid_input("compound FTS requires at least one index segment"))?; + let mut leaf_queries = Vec::new(); + collect_leaf_queries(query, &mut leaf_queries)?; + let mut num_plan_leaves = 0; + let plan = CompoundScorerPlan::from_query(query, &mut num_plan_leaves)?; + if num_plan_leaves != leaf_queries.len() { + return Err(Error::internal(format!( + "compound FTS planned {num_plan_leaves} leaves but prepared {}", + leaf_queries.len() + ))); + } + + let mut leaves = Vec::with_capacity(leaf_queries.len()); + if prepared_match.is_some() && leaf_queries.len() != 1 { + return Err(Error::internal( + "prepared Match replay requires exactly one compound FTS leaf", + )); + } + for leaf in leaf_queries { + let effective_params = leaf.effective_params(params); + let tokens = tokenize_leaf(first_index, &leaf, &effective_params); + let prepared = match &prepared_match { + Some(prepared) => prepared.clone(), + None => Arc::new( + prepare_bm25_query( + indices, + tokens, + &effective_params, + Some(metrics), + base_scorer.clone(), + ) + .await?, + ), + }; + leaves.push(PreparedLeaf { + query: prepared, + params: Arc::new(effective_params), + operator: leaf.operator(), + }); + } + Ok((plan, leaves)) +} + +struct LoadedLeaf { + postings: Vec, + params: Arc, + operator: Operator, + scorer: Arc, +} + +enum LoadedDocuments { + Legacy(Arc), + Modern { + documents: Arc, + lengths: Arc, + visibility: DocVisibility, + projection: Option, + }, +} + +struct LoadedPartition { + segment_ordinal: usize, + partition_ordinal: usize, + partition: Arc, + documents: LoadedDocuments, + leaves: Vec, +} + +async fn load_compound_partition( + segment_ordinal: usize, + partition_ordinal: usize, + partition: Arc, + leaves: &[PreparedLeaf], + mask: Arc, + metrics: Arc, +) -> Result> { + let leaf_loads = leaves.iter().map(|leaf| { + let partition = partition.clone(); + let tokens = leaf.query.tokens().clone(); + let params = leaf.params.clone(); + let scorer = leaf.query.scorer().clone(); + let metrics = metrics.clone(); + let operator = leaf.operator; + let has_all_query_positions = leaf.query.has_all_query_positions(); + async move { + let postings = if tokens.is_empty() + || ((operator == Operator::And || params.phrase_slop.is_some()) + && !has_all_query_positions) + { + Vec::new() + } else { + partition + .load_posting_lists( + tokens.as_ref(), + params.as_ref(), + operator, + scorer.as_ref(), + metrics.as_ref(), + true, + ) + .await? + .postings + }; + Result::Ok(LoadedLeaf { + postings, + params, + operator, + scorer, + }) + } + }); + let leaves = futures::future::try_join_all(leaf_loads).await?; + + let documents = if let Some(docs) = partition.docs.legacy() { + LoadedDocuments::Legacy(docs.clone()) + } else { + let documents = partition.docs.modern().cloned().ok_or_else(|| { + Error::internal("FTS partition contains neither legacy nor modern documents") + })?; + let materialize_selected = mask.max_len().is_some_and(|selected| { + u128::from(selected).saturating_mul(100) + <= u128::from(*FLAT_SEARCH_PERCENT_THRESHOLD) + .saturating_mul(documents.len() as u128) + }); + let visibility = match documents.immediate_visibility(mask.clone(), materialize_selected) { + Some(visibility) => visibility, + None => { + documents + .visibility(mask.clone(), materialize_selected) + .await? + } + }; + if visibility.is_empty() { + return Ok(None); + } + let lengths = match documents.cached_lengths() { + Some(lengths) => lengths, + None => documents.lengths().await?, + }; + let projection = documents.resident_address_projection(); + LoadedDocuments::Modern { + documents, + lengths, + visibility, + projection, + } + }; + + Ok(Some(LoadedPartition { + segment_ordinal, + partition_ordinal, + partition, + documents, + leaves, + })) +} + +struct DeferredCompoundRows { + documents: Arc, + rows: Vec>, +} + +struct OverflowedCompoundPartition { + segment_ordinal: usize, + partition_ordinal: usize, + partition: Arc, + documents: Arc, +} + +enum PartitionCollectionBoundary { + Deferred(DeferredCompoundRows), + Overflow(OverflowedCompoundPartition), +} + +struct CollectedPartitions { + collector: TopKCollector, + remaining: Vec, + boundary: Option, +} + +fn collect_partition_with_documents( + documents: &D, + leaves: Vec, + plan: &CompoundScorerPlan, + metrics: &dyn MetricsCollector, + collector: &mut TopKCollector, + mut map_document: impl FnMut(u64) -> Result, +) -> Result +where + D: WandDocuments + Sync, + K: Copy + Ord, +{ + let mut leaf_scorers = leaves + .into_iter() + .map(|leaf| { + let scorer: BoxScorer<'_> = if leaf.postings.is_empty() { + Box::new(EmptyScorer) + } else { + Box::new(WandCursor::new( + leaf.operator, + leaf.postings, + documents, + leaf.scorer, + leaf.params.as_ref(), + metrics, + )) + }; + Some(scorer) + }) + .collect::>(); + let mut scorer = plan.build(&mut leaf_scorers)?; + if leaf_scorers.iter().any(Option::is_some) { + return Err(Error::internal( + "compound FTS scorer did not consume every prepared leaf", + )); + } + collector.collect_mapped(scorer.as_mut(), &mut map_document) +} + +fn collect_loaded_partitions( + partitions: Vec, + plan: &CompoundScorerPlan, + mask: &RowAddrMask, + metrics: &dyn MetricsCollector, + mut collector: TopKCollector, +) -> Result { + let mut partitions = partitions.into_iter(); + while let Some(partition) = partitions.next() { + let LoadedPartition { + segment_ordinal, + partition_ordinal, + partition: source, + documents, + leaves, + } = partition; + match documents { + LoadedDocuments::Legacy(docs) => { + let documents = LegacyWandDocuments::new(docs.as_ref(), mask); + let status = collect_partition_with_documents( + &documents, + leaves, + plan, + metrics, + &mut collector, + Ok, + )?; + debug_assert_eq!(status, CollectionStatus::Complete); + } + LoadedDocuments::Modern { + documents: partition_documents, + lengths, + visibility, + projection, + } => { + let documents = ModernWandDocuments::filtered(lengths.as_ref(), &visibility); + if let Some(projection) = projection { + let status = collect_partition_with_documents( + &documents, + leaves, + plan, + metrics, + &mut collector, + |doc_id| { + let doc_id = DocId::new(u32::try_from(doc_id).map_err(|_| { + Error::index(format!( + "FTS DocId {doc_id} exceeds the modern u32 domain" + )) + })?); + let row_id = projection.address(doc_id).ok_or_else(|| { + Error::internal(format!( + "compound FTS scorer returned non-visible DocId {} in segment {segment_ordinal}, partition {partition_ordinal}", + doc_id.get() + )) + })?; + Ok(row_id) + }, + )?; + debug_assert_eq!(status, CollectionStatus::Complete); + } else { + let max_buffered = collector + .limit + .saturating_add(SCORE_FLOOR_RESOLUTION_BATCH_SIZE); + let mut local_collector = TopKCollector::retaining_score_floor( + collector.limit, + collector.competitive_score.clone(), + max_buffered, + ); + let status = collect_partition_with_documents( + &documents, + leaves, + plan, + metrics, + &mut local_collector, + |doc_id| { + Ok(DocId::new(u32::try_from(doc_id).map_err(|_| { + Error::index(format!( + "FTS DocId {doc_id} exceeds the modern u32 domain" + )) + })?)) + }, + )?; + let boundary = match status { + CollectionStatus::Complete => { + PartitionCollectionBoundary::Deferred(DeferredCompoundRows { + documents: partition_documents, + rows: local_collector.into_candidates(), + }) + } + CollectionStatus::ScoreFloorOverflow => { + PartitionCollectionBoundary::Overflow(OverflowedCompoundPartition { + segment_ordinal, + partition_ordinal, + partition: source, + documents: partition_documents, + }) + } + }; + return Ok(CollectedPartitions { + collector, + remaining: partitions.collect(), + boundary: Some(boundary), + }); + } + } + } + } + Ok(CollectedPartitions { + collector, + remaining: Vec::new(), + boundary: None, + }) +} + +async fn merge_resolved_compound_rows( + collector: &mut TopKCollector, + deferred: DeferredCompoundRows, +) -> Result<()> { + for rows in deferred.rows.chunks(SCORE_FLOOR_RESOLUTION_BATCH_SIZE) { + let doc_ids = rows.iter().map(|row| row.row_id).collect::>(); + let addresses = deferred.documents.resolve_addresses(&doc_ids).await?; + if addresses.len() != rows.len() { + return Err(Error::internal(format!( + "compound FTS resolved {} addresses for {} DocIds", + addresses.len(), + rows.len() + ))); + } + for (row, row_id) in rows.iter().zip(addresses) { + let status = collector.insert(ScoredRow { + row_id, + score: row.score, + }); + debug_assert_eq!(status, CollectionStatus::Complete); + } + } + Ok(()) +} + +async fn reload_compound_partition_with_projection( + overflow: OverflowedCompoundPartition, + leaves: &[PreparedLeaf], + mask: Arc, + metrics: Arc, +) -> Result { + let projection = overflow.documents.address_projection().await?; + let mut loaded = load_compound_partition( + overflow.segment_ordinal, + overflow.partition_ordinal, + overflow.partition, + leaves, + mask, + metrics, + ) + .await? + .ok_or_else(|| { + Error::internal(format!( + "compound FTS retry lost visible documents in segment {}, partition {}", + overflow.segment_ordinal, overflow.partition_ordinal + )) + })?; + match &mut loaded.documents { + LoadedDocuments::Modern { + projection: loaded_projection, + .. + } => *loaded_projection = Some(projection), + LoadedDocuments::Legacy(_) => { + return Err(Error::internal(format!( + "compound FTS retry changed segment {}, partition {} from modern to legacy documents", + overflow.segment_ordinal, overflow.partition_ordinal + ))); + } + } + Ok(loaded) +} + +/// Search one-column compound FTS directly over posting-backed scorers. +/// +/// The caller must provide all committed index segments for the column and a +/// ready prefilter. One collector owns the global top-k heap and propagates its +/// score floor through every partition-local scorer tree. Modern partitions +/// resolve candidates in bounded batches; an oversized kth-score tie is retried +/// against a resident row-address projection so final row-id ordering stays exact. +pub async fn compound_search( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, +) -> Result<(Vec, Vec)> { + compound_search_impl(indices, query, params, prefilter, metrics, None, None, None).await +} + +/// Search one-column compound FTS with caller-supplied corpus-wide BM25 statistics. +/// +/// The scorer must contain an entry for every token used by every query leaf, +/// including terms produced by fuzzy expansion. An incomplete scorer is rejected +/// instead of treating missing token statistics as zero. +pub async fn compound_search_with_base_scorer( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + base_scorer: Arc, +) -> Result<(Vec, Vec)> { + compound_search_impl( + indices, + query, + params, + prefilter, + metrics, + Some(base_scorer), + None, + None, + ) + .await +} + +/// Search one-column compound FTS with corpus-wide BM25 statistics and an +/// inclusive initial score floor. +/// +/// The floor may only remove scores strictly below it. Equal-score rows must +/// still be visited because final ordering uses row id as its secondary key. +pub async fn compound_search_with_base_scorer_and_score_floor( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + base_scorer: Arc, + score_floor: f32, +) -> Result<(Vec, Vec)> { + compound_search_impl( + indices, + query, + params, + prefilter, + metrics, + Some(base_scorer), + Some(score_floor), + None, + ) + .await +} + +/// Replay one root Match query with the exact vocabulary/scorer pair used by +/// an earlier bounded WAND probe. +#[doc(hidden)] +pub async fn compound_search_prepared_match( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + prepared_match: Arc, +) -> Result<(Vec, Vec)> { + if !matches!(query, FtsQuery::Match(_)) { + return Err(Error::invalid_input( + "prepared Match replay requires a root Match query", + )); + } + compound_search_impl( + indices, + query, + params, + prefilter, + metrics, + None, + None, + Some(prepared_match), + ) + .await +} + +/// Replay one root Match query with a prepared vocabulary/scorer pair and an +/// inclusive initial score floor. +#[doc(hidden)] +pub async fn compound_search_prepared_match_with_score_floor( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + prepared_match: Arc, + score_floor: f32, +) -> Result<(Vec, Vec)> { + if !matches!(query, FtsQuery::Match(_)) { + return Err(Error::invalid_input( + "prepared Match replay requires a root Match query", + )); + } + compound_search_impl( + indices, + query, + params, + prefilter, + metrics, + None, + Some(score_floor), + Some(prepared_match), + ) + .await +} + +// These arguments keep the public entry points explicit while centralizing the +// shared search loop; bundling them would only move the same independent inputs +// into an internal forwarding struct. +#[allow(clippy::too_many_arguments)] +async fn compound_search_impl( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + base_scorer: Option>, + initial_score_floor: Option, + prepared_match: Option>, +) -> Result<(Vec, Vec)> { + let limit = params.limit.unwrap_or(usize::MAX); + if limit == 0 { + return Ok((Vec::new(), Vec::new())); + } + let (plan, leaves) = prepare_compound_query( + indices, + query, + params, + metrics.as_ref(), + base_scorer, + prepared_match, + ) + .await?; + prefilter.wait_for_ready().await?; + let mask = prefilter.mask(); + let competitive_score = Arc::new(CompetitiveScore::default()); + if let Some(score_floor) = initial_score_floor { + competitive_score.raise(checked_score(score_floor, "initial compound score floor")?); + } + let mut collector = TopKCollector::with_competitive_score(limit, competitive_score); + + for (segment_ordinal, index) in indices.iter().enumerate() { + let loads = + index + .partitions + .iter() + .cloned() + .enumerate() + .map(|(partition_ordinal, partition)| { + load_compound_partition( + segment_ordinal, + partition_ordinal, + partition, + &leaves, + mask.clone(), + metrics.clone(), + ) + }); + let mut partitions = stream::iter(loads) + .buffer_unordered(get_num_compute_intensive_cpus().clamp(1, 32)) + .try_collect::>() + .await? + .into_iter() + .flatten() + .collect::>(); + while !partitions.is_empty() { + let cpu_plan = plan.clone(); + let cpu_mask = mask.clone(); + let cpu_metrics = metrics.clone(); + let collected = spawn_cpu(move || { + collect_loaded_partitions( + partitions, + &cpu_plan, + cpu_mask.as_ref(), + cpu_metrics.as_ref(), + collector, + ) + }) + .await?; + collector = collected.collector; + partitions = collected.remaining; + match collected.boundary { + Some(PartitionCollectionBoundary::Deferred(deferred)) => { + merge_resolved_compound_rows(&mut collector, deferred).await?; + } + Some(PartitionCollectionBoundary::Overflow(overflow)) => { + let retry = reload_compound_partition_with_projection( + overflow, + &leaves, + mask.clone(), + metrics.clone(), + ) + .await?; + partitions.insert(0, retry); + } + None => debug_assert!(partitions.is_empty()), + } + } + } + + let rows = collector.into_rows(); + Ok(rows.into_iter().map(|row| (row.row_id, row.score)).unzip()) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::atomic::AtomicUsize; + + use arrow::buffer::ScalarBuffer; + use rand::{Rng, SeedableRng, rngs::SmallRng}; + + use super::super::documents::{ + ordered_row_address_projection_for_test, resident_row_address_projection_for_test, + }; + use super::super::index::{PlainPostingList, PostingList}; + use super::super::scorer::Scorer; + use super::*; + use crate::metrics::NoOpMetricsCollector; + use crate::scalar::inverted::query::MultiMatchQuery; + + fn rows(values: &[(u64, f32)]) -> Vec { + values + .iter() + .map(|(row_id, score)| ScoredRow::new(*row_id, *score).unwrap()) + .collect() + } + + fn materialized(values: &[(u64, f32)]) -> Box { + Box::new(MaterializedScorer::try_new(rows(values)).unwrap()) + } + + #[test] + fn materialized_compound_top_k_preserves_multimatch_and_tie_order() { + let query = FtsQuery::MultiMatch(MultiMatchQuery { + match_queries: vec![ + MatchQuery::new("alpha".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("alpha".to_string()).with_column(Some("text".to_string())), + ], + }); + let (row_ids, scores) = materialized_compound_top_k( + &query, + vec![vec![(7, 1.0), (3, 2.0)], vec![(7, 3.0), (5, 3.0)]], + 2, + ) + .unwrap(); + + assert_eq!(row_ids, vec![5, 7]); + assert_eq!(scores, vec![3.0, 3.0]); + } + + fn zero_weight_wand<'a>( + documents: &'a DocSet, + scorer: Arc, + params: &'a FtsSearchParams, + metrics: &'a dyn MetricsCollector, + ) -> BoxScorer<'a> { + let query_weight = scorer.query_weight("common"); + assert_eq!(query_weight, 0.0); + let posting = PostingIterator::with_query_weight( + "common".to_owned(), + 0, + 0, + query_weight, + PostingList::Plain(PlainPostingList::new( + ScalarBuffer::from(vec![0_u64]), + ScalarBuffer::from(vec![1.0_f32]), + Some(0.0), + None, + )), + 1, + ); + Box::new(WandCursor::new( + Operator::Or, + vec![posting], + documents, + scorer, + params, + metrics, + )) + } + + fn mapping_error(result: Result>>) -> Error { + match result { + Err(error) => error, + Ok(_) => panic!("row-address mapping unexpectedly succeeded"), + } + } + + fn should_maxscore<'a>(children: Vec>) -> ShouldMaxScoreScorer<'a> { + let global_bounds = ShouldMaxScoreScorer::global_bounds(&children).unwrap(); + ShouldMaxScoreScorer::new(children, global_bounds) + } + + #[test] + fn score_bounds_are_conservative_under_nested_sum_and_boost() { + let should = DisjunctionScorer::try_new( + vec![ + materialized(&[(0, 0.1), (2, 2.0)]), + materialized(&[(0, 0.2)]), + ], + DisjunctionScore::Sum, + ) + .unwrap(); + let negative = materialized(&[(0, 0.3), (2, 5.0)]); + let mut scorer = BoostScorer::try_new(Box::new(should), negative, 0.5).unwrap(); + + assert_eq!(scorer.next().unwrap(), Some(0)); + let up_to = scorer.advance_shallow(0).unwrap(); + let bounds = scorer.score_bounds(up_to).unwrap(); + let first_score = scorer.score().unwrap(); + assert!(bounds.lower <= first_score); + assert!(bounds.upper >= first_score); + + assert_eq!(scorer.next().unwrap(), Some(2)); + let second_score = scorer.score().unwrap(); + assert!(second_score.is_sign_negative()); + let up_to = scorer.advance_shallow(2).unwrap(); + let bounds = scorer.score_bounds(up_to).unwrap(); + assert!(bounds.lower <= second_score); + assert!(bounds.upper >= second_score); + } + + #[test] + fn scaled_score_floor_is_maximal_and_preserves_equalities() { + let cases = [ + (3.75_f32, 2.5_f32), + (f32::from_bits(1.0_f32.to_bits() + 1), 1.000_000_2_f32), + (2.0_f32, f32::MIN_POSITIVE), + (1.0_f32, f32::from_bits(1)), + ]; + for (raw_score, factor) in cases { + let scaled_score = raw_score * factor; + assert!(scaled_score.is_finite() && scaled_score > 0.0); + let child_floor = exclusive_scaled_score_floor(scaled_score, factor).unwrap(); + assert!(child_floor * factor < scaled_score); + assert!(child_floor < raw_score); + let next_raw = f32::from_bits(child_floor.to_bits() + 1); + assert!(next_raw * factor >= scaled_score); + + let mut scorer = ScaleScorer::try_new(materialized(&[(0, raw_score)]), factor).unwrap(); + scorer.set_min_competitive_score(scaled_score).unwrap(); + assert_eq!(scorer.next().unwrap(), Some(0)); + assert_eq!(scorer.score().unwrap(), scaled_score); + } + + let subnormal_factor = f32::from_bits(1); + let raw_score = 0.25_f32; + let scaled_score = raw_score * subnormal_factor; + assert_eq!(scaled_score, 0.0); + assert_eq!( + exclusive_scaled_score_floor(scaled_score, subnormal_factor), + None + ); + let mut scorer = + ScaleScorer::try_new(materialized(&[(0, raw_score)]), subnormal_factor).unwrap(); + scorer.set_min_competitive_score(scaled_score).unwrap(); + assert_eq!(scorer.next().unwrap(), Some(0)); + assert_eq!(scorer.score().unwrap(), 0.0); + } + + #[test] + fn scale_scorer_only_translates_strictly_higher_floors() { + let (child, work) = instrumented(materialized(&[(0, 10.0)])); + let mut scorer = ScaleScorer::try_new(child, 2.0).unwrap(); + + scorer.set_min_competitive_score(4.0).unwrap(); + scorer.set_min_competitive_score(4.0).unwrap(); + scorer.set_min_competitive_score(3.0).unwrap(); + assert_eq!(scorer.score_floor_translations, 1); + assert_eq!(work.floors.load(AtomicOrdering::Relaxed), 1); + + scorer.set_min_competitive_score(5.0).unwrap(); + assert_eq!(scorer.score_floor_translations, 2); + assert_eq!(work.floors.load(AtomicOrdering::Relaxed), 2); + + let error = scorer.set_min_competitive_score(f32::NAN).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("cannot be NaN")); + assert_eq!(scorer.score_floor_translations, 2); + + let subnormal_factor = f32::from_bits(1); + let (child, work) = instrumented(materialized(&[(0, 1.0)])); + let mut scorer = ScaleScorer::try_new(child, subnormal_factor).unwrap(); + scorer.set_min_competitive_score(0.0).unwrap(); + scorer.set_min_competitive_score(0.0).unwrap(); + assert_eq!(scorer.score_floor_translations, 1); + assert_eq!(work.floors.load(AtomicOrdering::Relaxed), 0); + + scorer.set_min_competitive_score(f32::from_bits(1)).unwrap(); + assert_eq!(scorer.score_floor_translations, 2); + assert_eq!(work.floors.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn materialized_scorer_precomputes_multi_block_bounds() { + assert_eq!(std::mem::size_of::(), 8); + let values = [ + (0, 5.0), + (1, 1.0), + (2, 3.0), + (3, -2.0), + (4, 7.0), + (5, 4.0), + (6, 8.0), + (7, 6.0), + (8, 9.0), + (9, 0.0), + ]; + let mut scorer = MaterializedScorer::try_new(rows(&values)) + .unwrap() + .with_block_size(3); + + assert_eq!(scorer.num_bound_blocks(), 4); + assert_eq!(scorer.bound_score_visits(), values.len()); + assert_eq!(scorer.global_score_upper_bound(), Some(9.0)); + assert_eq!(scorer.advance_shallow(4).unwrap(), 5); + assert_eq!( + scorer.score_bounds(4).unwrap(), + ScoreBounds { + lower: -2.0, + upper: 7.0, + } + ); + for _ in 0..100 { + assert_eq!(scorer.score_bounds(5).unwrap().upper, 7.0); + } + assert_eq!(scorer.bound_score_visits(), values.len()); + + scorer.set_min_competitive_score(8.0).unwrap(); + assert_eq!(scorer.next().unwrap(), Some(6)); + assert_eq!(scorer.bound_score_visits(), values.len()); + } + + #[test] + fn collector_propagates_threshold_across_partitions_and_keeps_ties() { + let mut collector = TopKCollector::new(2); + let mut first = MaterializedScorer::try_new(rows(&[(8, 9.0), (4, 10.0), (3, 9.0)])) + .unwrap() + .with_block_size(1); + collector.collect_mapped(&mut first, Ok).unwrap(); + assert_eq!(collector.competitive_score.get(), 9.0); + + let mut second = MaterializedScorer::try_new(rows(&[(1, 1.0), (2, 9.0)])) + .unwrap() + .with_block_size(1); + collector.collect_mapped(&mut second, Ok).unwrap(); + assert_eq!( + collector.into_rows(), + vec![ + ScoredRow { + row_id: 4, + score: 10.0 + }, + ScoredRow { + row_id: 2, + score: 9.0 + } + ] + ); + } + + #[test] + fn seeded_collector_keeps_floor_equalities_for_row_id_ordering() { + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(5.0); + let mut collector = TopKCollector::with_competitive_score(2, competitive_score); + + let mut later_segment = MaterializedScorer::try_new(rows(&[(2, 4.0), (99, 5.0)])).unwrap(); + collector.collect_mapped(&mut later_segment, Ok).unwrap(); + let mut earlier_segment = + MaterializedScorer::try_new(rows(&[(1, 5.0), (50, 6.0)])).unwrap(); + collector.collect_mapped(&mut earlier_segment, Ok).unwrap(); + + assert_eq!(collector.into_rows(), rows(&[(50, 6.0), (1, 5.0)])); + } + + #[test] + fn collector_bounds_equal_score_candidates() { + let limit = 1; + let num_candidates = DEFAULT_BLOCK_SIZE * 4; + let values = (0..num_candidates) + .map(|row_id| (row_id as u64, 1.0)) + .collect::>(); + let mut scorer = MaterializedScorer::try_new(rows(&values)).unwrap(); + let max_buffered = limit + SCORE_FLOOR_RESOLUTION_BATCH_SIZE; + let mut collector = TopKCollector::retaining_score_floor( + limit, + Arc::new(CompetitiveScore::default()), + max_buffered, + ); + + let status = collector.collect_mapped(&mut scorer, Ok).unwrap(); + + assert_eq!(status, CollectionStatus::ScoreFloorOverflow); + assert_eq!(collector.heap.len(), max_buffered); + } + + #[test] + fn collector_reclaims_obsolete_score_floor_before_overflowing() { + let limit = 2; + let max_buffered = 4; + let values = [ + (0, 1.0), + (1, 1.0), + (2, 1.0), + (3, 2.0), + (4, 2.0), + (5, 2.0), + (6, 2.0), + (7, 2.0), + ]; + let mut scorer = MaterializedScorer::try_new(rows(&values)).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + let mut collector = + TopKCollector::retaining_score_floor(limit, competitive_score.clone(), max_buffered); + + let status = collector.collect_mapped(&mut scorer, Ok).unwrap(); + + assert_eq!(status, CollectionStatus::ScoreFloorOverflow); + assert_eq!(collector.heap.len(), max_buffered); + assert!(collector.heap.iter().all(|row| row.0.score == 2.0)); + assert_eq!(competitive_score.get(), 2.0); + } + + struct TwoPhaseScorer { + inner: MaterializedScorer, + accepted: Vec, + match_cost: Option, + has_doc_upper: bool, + approximations: Arc, + confirmations: Arc, + } + + impl ComposableScorer for TwoPhaseScorer { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn next(&mut self) -> Result> { + let doc = self.inner.next()?; + if doc.is_some() { + self.approximations.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn advance(&mut self, target: u64) -> Result> { + let doc = self.inner.advance(target)?; + if doc.is_some() { + self.approximations.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn cost(&self) -> usize { + self.inner.cost() + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + self.inner.score_bounds(up_to) + } + + fn global_score_upper_bound(&self) -> Option { + self.inner.global_score_upper_bound() + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn current_score_upper_bound(&mut self) -> Result> { + if self.has_doc_upper { + self.inner.score().map(Some) + } else { + Ok(None) + } + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + true + } + + fn matches(&mut self) -> Result { + self.confirmations.fetch_add(1, AtomicOrdering::Relaxed); + Ok(self + .doc() + .is_some_and(|doc| self.accepted.binary_search(&doc).is_ok())) + } + + fn match_cost(&self) -> Option { + self.match_cost + } + + fn scores_non_negative(&self) -> bool { + true + } + } + + fn two_phase( + values: &[(u64, f32)], + accepted: Vec, + match_cost: Option, + ) -> ( + Box, + Arc, + Arc, + ) { + let approximations = Arc::new(AtomicUsize::new(0)); + let confirmations = Arc::new(AtomicUsize::new(0)); + let scorer = TwoPhaseScorer { + inner: MaterializedScorer::try_new(rows(values)).unwrap(), + accepted, + match_cost, + has_doc_upper: true, + approximations: approximations.clone(), + confirmations: confirmations.clone(), + }; + (Box::new(scorer), approximations, confirmations) + } + + fn two_phase_without_doc_upper( + values: &[(u64, f32)], + accepted: Vec, + ) -> ( + Box, + Arc, + Arc, + ) { + let approximations = Arc::new(AtomicUsize::new(0)); + let confirmations = Arc::new(AtomicUsize::new(0)); + let scorer = TwoPhaseScorer { + inner: MaterializedScorer::try_new(rows(values)).unwrap(), + accepted, + match_cost: Some(1.0), + has_doc_upper: false, + approximations: approximations.clone(), + confirmations: confirmations.clone(), + }; + (Box::new(scorer), approximations, confirmations) + } + + #[derive(Default)] + struct ScorerWork { + advances: AtomicUsize, + confirmations: AtomicUsize, + shallow_advances: AtomicUsize, + bounds: AtomicUsize, + floors: AtomicUsize, + } + + struct InstrumentedScorer<'a> { + inner: BoxScorer<'a>, + work: Arc, + } + + impl ComposableScorer for InstrumentedScorer<'_> { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn document_key(&self) -> Option { + self.inner.document_key() + } + + fn next(&mut self) -> Result> { + let doc = self.inner.next()?; + if doc.is_some() { + self.work.advances.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn advance(&mut self, target: u64) -> Result> { + let doc = self.inner.advance(target)?; + if doc.is_some() { + self.work.advances.fetch_add(1, AtomicOrdering::Relaxed); + } + Ok(doc) + } + + fn cost(&self) -> usize { + self.inner.cost() + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.work + .shallow_advances + .fetch_add(1, AtomicOrdering::Relaxed); + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + self.work.bounds.fetch_add(1, AtomicOrdering::Relaxed); + self.inner.score_bounds(up_to) + } + + fn global_score_upper_bound(&self) -> Option { + self.inner.global_score_upper_bound() + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.work.floors.fetch_add(1, AtomicOrdering::Relaxed); + self.inner.set_min_competitive_score(min_score) + } + + fn current_score_upper_bound(&mut self) -> Result> { + self.inner.current_score_upper_bound() + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.inner.supports_doc_local_confirmation_pruning() + } + + fn matches(&mut self) -> Result { + self.work + .confirmations + .fetch_add(1, AtomicOrdering::Relaxed); + self.inner.matches() + } + + fn match_cost(&self) -> Option { + self.inner.match_cost() + } + + fn scores_non_negative(&self) -> bool { + self.inner.scores_non_negative() + } + } + + fn instrumented<'a>(inner: BoxScorer<'a>) -> (BoxScorer<'a>, Arc) { + let work = Arc::new(ScorerWork::default()); + ( + Box::new(InstrumentedScorer { + inner, + work: work.clone(), + }), + work, + ) + } + + fn row_address_source(values: &[(u64, f32)]) -> RowAddressSource<'static> { + let min_possible_row_address = values + .iter() + .map(|(row_address, _)| *row_address) + .min() + .unwrap(); + RowAddressSource::new(min_possible_row_address, materialized(values)) + } + + #[test] + fn row_address_scorer_maps_gaps_advance_and_shallow_bounds() { + let projection = ordered_row_address_projection_for_test(vec![10, 20, 50, 100, 200]); + let source = Box::new( + MaterializedScorer::try_new(rows(&[(0, 1.0), (2, 5.0), (4, 9.0)])) + .unwrap() + .with_block_size(2), + ); + let mut scorer = RowAddressScorer::new(source, projection); + + assert_eq!(scorer.next().unwrap(), Some(10)); + assert_eq!(scorer.document_key(), Some(10)); + let up_to = scorer.advance_shallow(10).unwrap(); + assert_eq!(up_to, 199); + assert_eq!( + scorer.score_bounds(150).unwrap(), + ScoreBounds { + lower: 1.0, + upper: 5.0, + } + ); + assert_eq!(scorer.global_score_upper_bound(), Some(9.0)); + + assert_eq!(scorer.advance(21).unwrap(), Some(50)); + assert_eq!(scorer.score().unwrap(), 5.0); + assert_eq!(scorer.advance(51).unwrap(), Some(200)); + assert_eq!(scorer.score().unwrap(), 9.0); + assert_eq!(scorer.next().unwrap(), None); + } + + #[test] + fn prepared_row_address_projection_is_reusable_across_leaf_scorers() { + let ordered_projection = resident_row_address_projection_for_test(vec![10, 20, 50]); + let ordered = prepare_row_address_projection(&ordered_projection); + assert_eq!( + ordered_projection.cached_row_address_order(), + CachedRowAddressOrder::Unknown + ); + assert_eq!(ordered_projection.ordered_validation_visited_docs(), 0); + + let first = map_scorer_to_row_addresses_with_threshold( + materialized(&[(0, 1.0), (2, 3.0)]), + &ordered, + 0, + 10, + ) + .unwrap() + .unwrap(); + assert_eq!( + ordered_projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + assert_eq!(ordered_projection.ordered_validation_visited_docs(), 3); + let second = map_scorer_to_row_addresses(materialized(&[(1, 2.0)]), &ordered, 0) + .unwrap() + .unwrap(); + assert_eq!(ordered_projection.ordered_validation_visited_docs(), 3); + let first = TopKCollector::new(10) + .collect(first.into_scorer().as_mut()) + .unwrap(); + let second = TopKCollector::new(10) + .collect(second.into_scorer().as_mut()) + .unwrap(); + assert_eq!(first, rows(&[(50, 3.0), (10, 1.0)])); + assert_eq!(second, rows(&[(20, 2.0)])); + + let delayed = map_scorer_to_row_addresses(materialized(&[(2, 3.0)]), &ordered, 2) + .unwrap() + .unwrap(); + assert_eq!(delayed.min_possible_row_address, 50); + + let unordered_projection = resident_row_address_projection_for_test(vec![30, 10, 20]); + let unordered = prepare_row_address_projection(&unordered_projection); + let first = map_scorer_to_row_addresses_with_threshold( + materialized(&[(0, 1.0), (1, 2.0)]), + &unordered, + 0, + 10, + ) + .unwrap() + .unwrap(); + assert!(matches!( + unordered_projection.cached_row_address_order(), + CachedRowAddressOrder::OutOfOrder + )); + assert_eq!(unordered_projection.ordered_validation_visited_docs(), 2); + let second = map_scorer_to_row_addresses(materialized(&[(2, 4.0)]), &unordered, 0) + .unwrap() + .unwrap(); + assert_eq!(unordered_projection.ordered_validation_visited_docs(), 2); + let first = TopKCollector::new(10) + .collect(first.into_scorer().as_mut()) + .unwrap(); + let second = TopKCollector::new(10) + .collect(second.into_scorer().as_mut()) + .unwrap(); + assert_eq!(first, rows(&[(10, 2.0), (30, 1.0)])); + assert_eq!(second, rows(&[(20, 4.0)])); + } + + #[test] + fn adaptive_projection_materializes_sparse_unknown_without_validation() { + let projection = resident_row_address_projection_for_test( + (0..100).map(|local_doc| local_doc * 10).collect(), + ); + let prepared = prepare_row_address_projection(&projection); + + let source = map_scorer_to_row_addresses_with_threshold( + materialized(&[(73, 4.0)]), + &prepared, + 73, + 10, + ) + .unwrap() + .unwrap(); + + assert_eq!(projection.ordered_validation_visited_docs(), 0); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Unknown + ); + let result = TopKCollector::new(1) + .collect(source.into_scorer().as_mut()) + .unwrap(); + assert_eq!(result, rows(&[(730, 4.0)])); + } + + #[test] + fn adaptive_projection_amortizes_repeated_sparse_materialization() { + let projection = resident_row_address_projection_for_test( + (0..100).map(|local_doc| local_doc * 10).collect(), + ); + let prepared = prepare_row_address_projection(&projection); + + for _ in 0..10 { + map_scorer_to_row_addresses_with_threshold( + materialized(&[(73, 4.0)]), + &prepared, + 73, + 10, + ) + .unwrap(); + } + assert_eq!(projection.ordered_validation_visited_docs(), 0); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Unknown + ); + + map_scorer_to_row_addresses_with_threshold(materialized(&[(73, 4.0)]), &prepared, 73, 10) + .unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 100); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + + map_scorer_to_row_addresses_with_threshold(materialized(&[(73, 4.0)]), &prepared, 73, 10) + .unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 100); + } + + #[test] + fn adaptive_projection_validates_dense_unknown_once() { + let projection = resident_row_address_projection_for_test(vec![10, 20, 30, 40]); + let prepared = prepare_row_address_projection(&projection); + + map_scorer_to_row_addresses_with_threshold( + materialized(&[(0, 1.0), (1, 1.0), (2, 1.0), (3, 1.0)]), + &prepared, + 0, + 10, + ) + .unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 4); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + + map_scorer_to_row_addresses_with_threshold( + materialized(&[(0, 2.0), (1, 2.0), (2, 2.0), (3, 2.0)]), + &prepared, + 0, + 10, + ) + .unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 4); + } + + #[test] + fn adaptive_projection_tracks_unknown_duplicates_across_leaves() { + let projection = resident_row_address_projection_for_test(vec![10, 10]); + let prepared = prepare_row_address_projection(&projection); + + map_scorer_to_row_addresses_with_threshold(materialized(&[(0, 1.0)]), &prepared, 0, 1000) + .unwrap(); + // The same physical local document can match more than one leaf. + map_scorer_to_row_addresses_with_threshold(materialized(&[(0, 2.0)]), &prepared, 0, 1000) + .unwrap(); + let error = mapping_error(map_scorer_to_row_addresses_with_threshold( + materialized(&[(1, 3.0)]), + &prepared, + 1, + 1000, + )); + + assert!( + error + .to_string() + .contains("distinct local documents 0 and 1") + ); + assert_eq!(projection.ordered_validation_visited_docs(), 0); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Unknown + ); + } + + #[test] + fn adaptive_projection_tracks_out_of_order_duplicates_across_leaves() { + let projection = resident_row_address_projection_for_test(vec![10, 30, 10]); + let prepared = prepare_row_address_projection(&projection); + + map_scorer_to_row_addresses_with_threshold(materialized(&[(0, 1.0)]), &prepared, 0, 0) + .unwrap(); + assert!(matches!( + projection.cached_row_address_order(), + CachedRowAddressOrder::OutOfOrder + )); + assert_eq!(projection.ordered_validation_visited_docs(), 3); + + let error = mapping_error(map_scorer_to_row_addresses_with_threshold( + materialized(&[(2, 2.0)]), + &prepared, + 2, + 100, + )); + assert!( + error + .to_string() + .contains("distinct local documents 0 and 2") + ); + assert_eq!(projection.ordered_validation_visited_docs(), 3); + } + + #[test] + fn adaptive_projection_rejects_cached_duplicates_without_materializing() { + let projection = resident_row_address_projection_for_test(vec![10, 10]); + let prepared = prepare_row_address_projection(&projection); + + let first_error = mapping_error(map_scorer_to_row_addresses_with_threshold( + materialized(&[(0, 1.0)]), + &prepared, + 0, + 0, + )); + assert!( + first_error + .to_string() + .contains("shared by local documents 0 and 1") + ); + assert_eq!(projection.ordered_validation_visited_docs(), 2); + assert!(matches!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Duplicate + )); + + let second_error = mapping_error(map_scorer_to_row_addresses_with_threshold( + materialized(&[(1, 2.0)]), + &prepared, + 1, + 100, + )); + assert!( + second_error + .to_string() + .contains("shared by local documents 0 and 1") + ); + // The atomic cache stores only the invalid category. Reconstructing + // exact duplicate diagnostics is a rare error-path rescan. + assert_eq!(projection.ordered_validation_visited_docs(), 4); + } + + #[test] + fn materialized_projection_reports_single_leaf_duplicates_locally() { + let projection = resident_row_address_projection_for_test(vec![10, 10]); + let prepared = prepare_row_address_projection(&projection); + let error = mapping_error(map_scorer_to_row_addresses_with_threshold( + materialized(&[(0, 1.0), (1, 2.0)]), + &prepared, + 0, + 100, + )); + + assert!(error.to_string().contains("duplicate row_id=10")); + assert_eq!(projection.ordered_validation_visited_docs(), 0); + } + + #[test] + fn row_address_merge_orders_gapped_sources_and_keeps_score_ties() { + let mut scorer = RowAddressMergeScorer::try_new(vec![ + row_address_source(&[(10, 5.0), (100, 2.0)]), + row_address_source(&[(20, 5.0), (70, 8.0)]), + ]) + .unwrap(); + + let results = TopKCollector::new(10).collect(&mut scorer).unwrap(); + assert_eq!( + results, + rows(&[(70, 8.0), (10, 5.0), (20, 5.0), (100, 2.0)]) + ); + } + + #[test] + fn row_address_merge_advance_skips_sources_with_a_heap() { + let mut scorer = RowAddressMergeScorer::try_new(vec![ + row_address_source(&[(10, 1.0), (100, 2.0)]), + row_address_source(&[(20, 3.0), (70, 4.0)]), + row_address_source(&[(30, 5.0), (90, 6.0)]), + ]) + .unwrap(); + + assert_eq!(scorer.advance(65).unwrap(), Some(70)); + assert_eq!(scorer.next().unwrap(), Some(90)); + assert_eq!(scorer.next().unwrap(), Some(100)); + assert_eq!(scorer.next().unwrap(), None); + } + + #[test] + fn row_address_merge_delays_pending_sources_and_shallow_work() { + let first = Box::new( + MaterializedScorer::try_new(rows(&[(10, 1.0), (20, 4.0), (2_000, 8.0)])) + .unwrap() + .with_block_size(3), + ); + let second = Box::new( + MaterializedScorer::try_new(rows(&[(1_000, 10.0)])) + .unwrap() + .with_block_size(1), + ); + let (first, first_work) = instrumented(first); + let (second, second_work) = instrumented(second); + let mut scorer = RowAddressMergeScorer::try_new(vec![ + RowAddressSource::new(10, first), + RowAddressSource::new(1_000, second), + ]) + .unwrap(); + + assert_eq!(scorer.next().unwrap(), Some(10)); + assert_eq!(first_work.advances.load(AtomicOrdering::Relaxed), 1); + assert_eq!(second_work.advances.load(AtomicOrdering::Relaxed), 0); + + let up_to = scorer.advance_shallow(10).unwrap(); + assert_eq!(up_to, 999); + // Materialized bounds conservatively cover the whole source block, + // including its row at 2_000, while the merge window still stops + // before the pending source at 1_000. + assert_eq!(scorer.score_bounds(up_to).unwrap().upper, 8.0); + assert_eq!(first_work.shallow_advances.load(AtomicOrdering::Relaxed), 1); + assert_eq!(first_work.bounds.load(AtomicOrdering::Relaxed), 1); + assert_eq!( + second_work.shallow_advances.load(AtomicOrdering::Relaxed), + 0 + ); + assert_eq!(second_work.bounds.load(AtomicOrdering::Relaxed), 0); + assert_eq!(scorer.global_score_upper_bound(), Some(10.0)); + } + + #[test] + fn row_address_merge_pushes_new_floors_only_to_active_sources() { + let (first, first_work) = instrumented(materialized(&[(10, 1.0), (20, 2.0)])); + let (second, second_work) = instrumented(materialized(&[(1_000, 10.0)])); + let mut scorer = RowAddressMergeScorer::try_new(vec![ + RowAddressSource::new(10, first), + RowAddressSource::new(1_000, second), + ]) + .unwrap(); + + assert_eq!(scorer.next().unwrap(), Some(10)); + scorer.set_min_competitive_score(5.0).unwrap(); + scorer.set_min_competitive_score(5.0).unwrap(); + assert_eq!(first_work.floors.load(AtomicOrdering::Relaxed), 1); + assert_eq!(second_work.floors.load(AtomicOrdering::Relaxed), 0); + + assert_eq!(scorer.advance(1_000).unwrap(), Some(1_000)); + assert_eq!(second_work.floors.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn row_address_merge_rejects_duplicate_source_addresses() { + let mut scorer = RowAddressMergeScorer::try_new(vec![ + row_address_source(&[(20, 1.0)]), + row_address_source(&[(20, 2.0)]), + ]) + .unwrap(); + + let error = scorer.next().unwrap_err(); + assert!(error.to_string().contains("duplicate row address 20")); + } + + #[test] + fn materialized_address_fallback_sorts_nonmonotonic_projection() { + let addresses = [30, 10, 20]; + let source = materialized(&[(0, 1.0), (1, 2.0), (2, 3.0)]); + let source = materialize_mapped_scorer(source, |doc| { + addresses + .get(doc as usize) + .copied() + .ok_or_else(|| Error::internal(format!("missing test address for document {doc}"))) + }) + .unwrap() + .unwrap(); + let mut scorer = source.into_scorer(); + + assert_eq!(scorer.next().unwrap(), Some(10)); + assert_eq!(scorer.score().unwrap(), 2.0); + assert_eq!(scorer.next().unwrap(), Some(20)); + assert_eq!(scorer.score().unwrap(), 3.0); + assert_eq!(scorer.next().unwrap(), Some(30)); + assert_eq!(scorer.score().unwrap(), 1.0); + assert_eq!(scorer.next().unwrap(), None); + } + + #[test] + fn materialized_address_fallback_rejects_duplicate_matches() { + let source = materialized(&[(0, 1.0), (1, 2.0)]); + let Err(error) = materialize_mapped_scorer(source, |_| Ok(10)) else { + panic!("duplicate projected addresses must fail"); + }; + + assert!(error.to_string().contains("duplicate row_id=10")); + } + + struct UnboundedScorer { + inner: MaterializedScorer, + } + + impl ComposableScorer for UnboundedScorer { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn next(&mut self) -> Result> { + self.inner.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.inner.advance(target) + } + + fn cost(&self) -> usize { + self.inner.cost() + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, _up_to: u64) -> Result { + Ok(ScoreBounds::UNBOUNDED) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn scores_non_negative(&self) -> bool { + true + } + } + + struct CountingScorer { + inner: MaterializedScorer, + cost: usize, + advance_calls: Arc, + } + + impl ComposableScorer for CountingScorer { + fn doc(&self) -> Option { + self.inner.doc() + } + + fn next(&mut self) -> Result> { + self.inner.next() + } + + fn advance(&mut self, target: u64) -> Result> { + self.advance_calls.fetch_add(1, AtomicOrdering::Relaxed); + self.inner.advance(target) + } + + fn cost(&self) -> usize { + self.cost + } + + fn score(&mut self) -> Result { + self.inner.score() + } + + fn advance_shallow(&mut self, target: u64) -> Result { + self.inner.advance_shallow(target) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + self.inner.score_bounds(up_to) + } + + fn global_score_upper_bound(&self) -> Option { + self.inner.global_score_upper_bound() + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + self.inner.set_min_competitive_score(min_score) + } + + fn matches(&mut self) -> Result { + self.inner.matches() + } + + fn scores_non_negative(&self) -> bool { + self.inner.scores_non_negative() + } + } + + fn counting( + values: &[(u64, f32)], + cost: usize, + ) -> (Box, Arc) { + let advance_calls = Arc::new(AtomicUsize::new(0)); + let scorer = CountingScorer { + inner: MaterializedScorer::try_new(rows(values)).unwrap(), + cost, + advance_calls: advance_calls.clone(), + }; + (Box::new(scorer), advance_calls) + } + + #[test] + fn collector_confirms_two_phase_matches_without_a_cost_hint() { + let (mut scorer, approximations, confirmations) = + two_phase(&[(1, 100.0), (2, 2.0), (3, 1.0)], vec![2, 3], None); + let results = TopKCollector::new(2).collect(scorer.as_mut()).unwrap(); + assert_eq!(results, rows(&[(2, 2.0), (3, 1.0)])); + assert_eq!(approximations.load(AtomicOrdering::Relaxed), 3); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 3); + assert_eq!(scorer.match_cost(), None); + } + + #[test] + fn required_conjunction_confirms_cheapest_first_and_short_circuits() { + let values = (0..100).map(|doc| (doc, 1.0)).collect::>(); + let accepted_by_cheap = (0..100).step_by(5).collect::>(); + + let (expensive, expensive_approximations, expensive_confirmations) = + two_phase(&values, (0..100).collect(), Some(10.0)); + let (cheap, cheap_approximations, cheap_confirmations) = + two_phase(&values, accepted_by_cheap.clone(), Some(1.0)); + let mut scorer = RequiredConjunctionScorer::try_new(vec![expensive, cheap]).unwrap(); + assert_eq!(scorer.confirmation_order.as_deref(), Some(&[1, 0][..])); + + let results = TopKCollector::new(100).collect(&mut scorer).unwrap(); + let expected = accepted_by_cheap + .iter() + .map(|doc| (*doc, 2.0)) + .collect::>(); + assert_eq!(results, rows(&expected)); + assert_eq!(cheap_confirmations.load(AtomicOrdering::Relaxed), 100); + assert_eq!(expensive_confirmations.load(AtomicOrdering::Relaxed), 20); + let approximations = cheap_approximations.load(AtomicOrdering::Relaxed) + + expensive_approximations.load(AtomicOrdering::Relaxed); + let confirmations = cheap_confirmations.load(AtomicOrdering::Relaxed) + + expensive_confirmations.load(AtomicOrdering::Relaxed); + assert_eq!(approximations, 200); + assert_eq!(confirmations, 120); + assert!( + confirmations * 5 <= approximations * 4, + "confirmation ordering should reduce work by at least 20%: {confirmations}/{approximations}" + ); + + let (cheap, _, _) = two_phase(&values, accepted_by_cheap, Some(1.0)); + let (expensive, _, _) = two_phase(&values, (0..100).collect(), Some(10.0)); + let scorer = RequiredConjunctionScorer::try_new(vec![cheap, expensive]).unwrap(); + assert!(scorer.confirmation_order.is_none()); + } + + #[test] + fn required_conjunction_confirms_children_without_cost_hints() { + let (unknown, _, unknown_confirmations) = two_phase(&[(0, 1.0)], Vec::new(), None); + let (costed, _, costed_confirmations) = two_phase(&[(0, 1.0)], vec![0], Some(1.0)); + let mut scorer = RequiredConjunctionScorer::try_new(vec![unknown, costed]).unwrap(); + + assert_eq!(scorer.confirmation_order.as_deref(), Some(&[1, 0][..])); + assert!( + TopKCollector::new(1) + .collect(&mut scorer) + .unwrap() + .is_empty() + ); + assert_eq!(costed_confirmations.load(AtomicOrdering::Relaxed), 1); + assert_eq!(unknown_confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn required_conjunction_rejects_invalid_match_cost() { + let (invalid, _, _) = two_phase(&[(0, 1.0)], vec![0], Some(f32::NAN)); + + let error = RequiredConjunctionScorer::try_new(vec![invalid]) + .err() + .unwrap(); + assert!(matches!(error, Error::Internal { .. })); + assert!( + error + .to_string() + .contains("child 0 reported invalid two-phase match cost: NaN") + ); + } + + #[test] + fn required_conjunction_uses_all_must_scores_for_competitive_bounds() { + let left = Box::new( + MaterializedScorer::try_new(rows(&[(1, 3.0), (3, 1.0)])) + .unwrap() + .with_block_size(1), + ); + let right = Box::new( + MaterializedScorer::try_new(rows(&[(1, 30.0), (3, 10.0)])) + .unwrap() + .with_block_size(1), + ); + let mut scorer = RequiredConjunctionScorer::try_new(vec![left, right]).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + let results = TopKCollector::with_competitive_score(10, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(1, 33.0), (3, 11.0)])); + } + + #[test] + fn required_conjunction_aligns_cheapest_clause_first() { + let dense_rows = (0..=100).map(|row_id| (row_id, 1.0)).collect::>(); + let (dense, dense_advance_calls) = counting(&dense_rows, 101); + let (rare, rare_advance_calls) = counting(&[(50, 1.0)], 1); + let mut scorer = RequiredConjunctionScorer::try_new(vec![dense, rare]).unwrap(); + assert_eq!(scorer.approximation_order.as_deref(), Some(&[1, 0][..])); + + assert_eq!(scorer.next().unwrap(), Some(50)); + assert_eq!(scorer.next().unwrap(), None); + assert_eq!(dense_advance_calls.load(AtomicOrdering::Relaxed), 1); + assert_eq!(rare_advance_calls.load(AtomicOrdering::Relaxed), 2); + + let (rare, _) = counting(&[(50, 1.0)], 1); + let (dense, _) = counting(&dense_rows, 101); + let scorer = RequiredConjunctionScorer::try_new(vec![rare, dense]).unwrap(); + assert!(scorer.approximation_order.is_none()); + } + + #[test] + fn required_conjunction_preserves_query_score_order() { + let (large, _) = counting(&[(0, 16_777_216.0)], 3); + let (first_small, _) = counting(&[(0, 1.0)], 1); + let (second_small, _) = counting(&[(0, 1.0)], 2); + let mut scorer = + RequiredConjunctionScorer::try_new(vec![large, first_small, second_small]).unwrap(); + + assert_eq!(scorer.next().unwrap(), Some(0)); + assert_eq!(scorer.score().unwrap(), 16_777_216.0); + } + + #[test] + fn boolean_sums_all_matching_clause_scores() { + let must = vec![ + materialized(&[(1, 3.0), (2, 2.0), (3, 1.0)]), + materialized(&[(1, 30.0), (3, 10.0)]), + ]; + let should = vec![ + materialized(&[(1, 0.5), (3, 4.0)]), + materialized(&[(3, 2.0)]), + ]; + let must_not = vec![materialized(&[(1, 9.0)])]; + let mut boolean = BooleanScorer::try_new(should, must, must_not).unwrap(); + let results = TopKCollector::new(10).collect(&mut boolean).unwrap(); + assert_eq!( + results, + vec![ScoredRow { + row_id: 3, + score: 17.0 + }] + ); + + let mut dismax = DisjunctionScorer::try_new( + vec![ + materialized(&[(1, 2.0), (3, 3.0)]), + materialized(&[(1, 4.0), (2, 4.0)]), + ], + DisjunctionScore::Max, + ) + .unwrap(); + let results = TopKCollector::new(2).collect(&mut dismax).unwrap(); + assert_eq!(results, rows(&[(1, 4.0), (2, 4.0)])); + } + + #[test] + fn boolean_preserves_zero_weight_required_and_prohibited_membership() { + let mut token_docs = HashMap::new(); + token_docs.insert("common".to_owned(), 10_000_000); + let scorer = Arc::new(MemBM25Scorer::new(10_000_000, 10_000_000, token_docs)); + assert_eq!(scorer.query_weight("common"), 0.0); + + let mut documents = DocSet::default(); + documents.append(0, 1); + let params = FtsSearchParams::default(); + let metrics = NoOpMetricsCollector; + + let mut required = BooleanScorer::try_new( + Vec::new(), + vec![zero_weight_wand( + &documents, + scorer.clone(), + ¶ms, + &metrics, + )], + Vec::new(), + ) + .unwrap(); + assert_eq!(required.next().unwrap(), Some(0)); + assert!(required.matches().unwrap()); + assert_eq!(required.score().unwrap(), 0.0); + drop(required); + + let mut excluded = BooleanScorer::try_new( + Vec::new(), + vec![materialized(&[(0, 1.0)])], + vec![zero_weight_wand(&documents, scorer, ¶ms, &metrics)], + ) + .unwrap(); + assert_eq!(excluded.next().unwrap(), None); + } + + #[test] + fn phrase_doc_bound_records_exact_and_sloppy_confirmation_avoidance() { + let mut token_docs = HashMap::new(); + token_docs.insert("common".to_owned(), 10_000_000); + let scorer = Arc::new(MemBM25Scorer::new(10_000_000, 10_000_000, token_docs)); + let mut documents = DocSet::default(); + documents.append(0, 1); + + for slop in [0, 2] { + let params = FtsSearchParams::default().with_phrase_slop(Some(slop)); + let metrics = NoOpMetricsCollector; + let phrase = zero_weight_wand(&documents, scorer.clone(), ¶ms, &metrics); + // The high-scoring sibling keeps the shared block competitive, while + // doc 0 still has a combined doc-local upper below the score floor. + let mut phrase = DisjunctionScorer::try_new( + vec![phrase, materialized(&[(0, 0.0), (1, 2.0)])], + DisjunctionScore::Sum, + ) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(1.0); + + assert_eq!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut phrase) + .unwrap(), + rows(&[(1, 2.0)]) + ); + } + } + + #[test] + fn phrase_positive_boost_uses_positive_upper_before_confirmation() { + let (phrase, _, confirmations) = two_phase(&[(0, 2.0), (1, 10.0)], vec![1], Some(1.0)); + let mut scorer = BoostScorer::try_new(phrase, materialized(&[(1, 1.0)]), 0.5).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(9.5); + + assert_eq!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(), + rows(&[(1, 9.5)]) + ); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn phrase_must_not_never_contributes_to_score_upper_bound() { + let (prohibited, approximations, confirmations) = + two_phase(&[(0, 100.0)], Vec::new(), Some(1.0)); + let mut scorer = BooleanScorer::try_new( + Vec::new(), + vec![materialized(&[(0, 1.0), (1, 10.0)])], + vec![prohibited], + ) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + assert_eq!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(), + rows(&[(1, 10.0)]) + ); + // Doc 0 is rejected from the positive score alone. The prohibited + // phrase is never advanced, so it cannot inflate that upper bound. + assert_eq!(approximations.load(AtomicOrdering::Relaxed), 0); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 0); + } + + #[test] + fn signed_and_unknown_doc_bounds_use_exact_confirmation_fallback() { + let (signed_phrase, _, signed_confirmations) = two_phase(&[(0, 2.0)], vec![0], Some(1.0)); + let signed_optional = + Box::new(BoostScorer::try_new(signed_phrase, materialized(&[(0, 2.0)]), 2.0).unwrap()); + let mut signed = BooleanScorer::try_new( + vec![signed_optional], + vec![materialized(&[(0, 1.0)])], + Vec::new(), + ) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + assert!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut signed) + .unwrap() + .is_empty() + ); + assert_eq!(signed_confirmations.load(AtomicOrdering::Relaxed), 1); + let (unknown_phrase, approximations, confirmations) = + two_phase_without_doc_upper(&[(0, 1.0), (1, 100.0)], vec![1]); + let mut unknown = BooleanScorer::try_new( + vec![unknown_phrase, materialized(&[(0, 0.0), (1, 0.0)])], + Vec::new(), + Vec::new(), + ) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(50.0); + + assert_eq!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut unknown) + .unwrap(), + rows(&[(1, 100.0)]) + ); + assert_eq!(approximations.load(AtomicOrdering::Relaxed), 2); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 2); + } + + #[test] + fn row_address_merge_forwards_phrase_doc_bound_and_avoidance() { + // Cross-column eager execution maps each leaf into row-address space + // and merges disjoint sources through this same wrapper stack. + let (phrase, approximations, confirmations) = + two_phase(&[(0, 2.0), (1, 10.0)], vec![1], Some(1.0)); + let mapped_phrase = Box::new(RowAddressScorer::new( + phrase, + ordered_row_address_projection_for_test(vec![100, 200]), + )); + let mut scorer = RowAddressMergeScorer::try_new(vec![ + RowAddressSource::new(100, mapped_phrase), + row_address_source(&[(300, 100.0)]), + ]) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + assert_eq!( + TopKCollector::with_competitive_score(2, competitive_score) + .collect(&mut scorer) + .unwrap(), + rows(&[(300, 100.0), (200, 10.0)]) + ); + assert_eq!(approximations.load(AtomicOrdering::Relaxed), 2); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn phrase_outward_upper_equal_to_ulp_floor_still_confirms() { + let exact_score = next_down(1.0); + let floor = 1.0; + assert_eq!( + ScoreBounds::ZERO + .add(ScoreBounds::point(exact_score).unwrap()) + .upper(), + floor + ); + let (phrase, _, confirmations) = two_phase(&[(7, exact_score)], vec![7], Some(1.0)); + let mut phrase = DisjunctionScorer::try_new(vec![phrase], DisjunctionScore::Sum).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(floor); + + assert!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut phrase) + .unwrap() + .is_empty() + ); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn surviving_nested_phrase_is_confirmed_exactly_once() { + let (phrase, _, confirmations) = two_phase(&[(3, 5.0)], vec![3], Some(1.0)); + let nested = Box::new( + DisjunctionScorer::try_new( + vec![phrase, materialized(&[(3, 0.0)])], + DisjunctionScore::Sum, + ) + .unwrap(), + ); + let mut scorer = BooleanScorer::try_new(vec![nested], Vec::new(), Vec::new()).unwrap(); + + assert_eq!( + TopKCollector::new(1).collect(&mut scorer).unwrap(), + rows(&[(3, 5.0)]) + ); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn reqopt_delays_sparse_optional_probes() { + let values = (0..100).map(|doc| (doc, 1.0)).collect::>(); + let build = || { + let required = + Box::new(RequiredConjunctionScorer::try_new(vec![materialized(&values)]).unwrap()); + let optional = Box::new( + DisjunctionScorer::try_new( + vec![materialized(&[(99, 100.0)])], + DisjunctionScore::Sum, + ) + .unwrap(), + ); + let (required, required_work) = instrumented(required); + let (optional, optional_work) = instrumented(optional); + (required, required_work, optional, optional_work) + }; + + let (required, _, optional, eager_optional_work) = build(); + let mut eager = BooleanScorer { + driver: required, + optional: Some(optional), + prohibited: None, + current: None, + confirmed_doc: None, + confirmed: false, + optional_matches: false, + defer_confirmation: false, + }; + let eager_results = TopKCollector::new(1).collect(&mut eager).unwrap(); + + let (required, required_work, optional, optional_work) = build(); + let mut scorer = ReqOptScorer::new(required, optional); + let results = TopKCollector::new(1).collect(&mut scorer).unwrap(); + + assert_eq!(eager_results, rows(&[(99, 101.0)])); + assert_eq!(results, rows(&[(99, 101.0)])); + let required_advances = required_work.advances.load(AtomicOrdering::Relaxed); + let eager_optional_probes = eager_optional_work.advances.load(AtomicOrdering::Relaxed); + let optional_probes = optional_work.advances.load(AtomicOrdering::Relaxed); + assert_eq!(required_advances, 100); + assert_eq!(eager_optional_probes, 100); + assert_eq!(optional_probes, 1); + assert_eq!( + required_work.shallow_advances.load(AtomicOrdering::Relaxed), + 2 + ); + assert_eq!(required_work.bounds.load(AtomicOrdering::Relaxed), 2); + assert_eq!( + required_work.confirmations.load(AtomicOrdering::Relaxed), + 100 + ); + assert_eq!(optional_work.confirmations.load(AtomicOrdering::Relaxed), 1); + assert!( + optional_probes * 5 <= eager_optional_probes * 4, + "lazy required-plus-optional scoring should reduce optional probes by at least 20%: \ + {optional_probes}/{eager_optional_probes}" + ); + } + + #[test] + fn reqopt_temporarily_requires_optional_contribution() { + let values = (0..100).map(|doc| (doc, 1.0)).collect::>(); + let (required, required_work) = instrumented(materialized(&values)); + let (optional, optional_work) = instrumented(materialized(&[(50, 10.0)])); + let mut scorer = ReqOptScorer::new(required, optional); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + + let results = TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(50, 11.0)])); + assert!(required_work.advances.load(AtomicOrdering::Relaxed) < 10); + assert_eq!(required_work.confirmations.load(AtomicOrdering::Relaxed), 1); + assert_eq!(optional_work.advances.load(AtomicOrdering::Relaxed), 1); + assert_eq!(optional_work.confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn boolean_reqopt_keeps_current_confirmation_stable_after_bounds() { + let mut scorer = BooleanScorer::try_new( + vec![materialized(&[(1, 10.0)])], + vec![materialized(&[(0, 1.0), (1, 1.0)])], + Vec::new(), + ) + .unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(5.0); + + let results = TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(1, 11.0)])); + } + + #[test] + fn boolean_gates_reqopt_and_signed_boost_uses_exact_fallback() { + let supported = BooleanScorer::try_new( + vec![materialized(&[(1, 4.0)])], + vec![materialized(&[(1, 2.0)])], + Vec::new(), + ) + .unwrap(); + assert!(supported.optional.is_none()); + + let signed_optional = Box::new( + BoostScorer::try_new( + materialized(&[(1, 4.0), (2, 1.0)]), + materialized(&[(1, 1.0), (2, 4.0)]), + 0.5, + ) + .unwrap(), + ); + let mut fallback = BooleanScorer::try_new( + vec![signed_optional], + vec![materialized(&[(1, 2.0), (2, 2.0)])], + Vec::new(), + ) + .unwrap(); + assert!(fallback.optional.is_some()); + assert_eq!( + TopKCollector::new(2).collect(&mut fallback).unwrap(), + rows(&[(1, 5.5), (2, 1.0)]) + ); + } + + #[test] + fn reqopt_uses_exact_iteration_for_unbounded_scorers() { + let required = Box::new(UnboundedScorer { + inner: MaterializedScorer::try_new(rows(&[(1, 1.0), (2, 1.0)])).unwrap(), + }); + let optional = materialized(&[(2, 10.0)]); + let mut scorer = ReqOptScorer::new(required, optional); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(5.0); + + let results = TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(); + + assert_eq!(results, rows(&[(2, 11.0)])); + } + + fn pure_should_canary_children() -> (Vec>, Vec>) { + let mut children = Vec::new(); + let mut work = Vec::new(); + let mut push = |child: BoxScorer<'static>| { + let (child, child_work) = instrumented(child); + children.push(child); + work.push(child_work); + }; + + push(materialized(&[(0, 2.0)])); + let dense = (1..=1024).map(|doc| (doc, 0.125)).collect::>(); + for _ in 0..8 { + push(materialized(&dense)); + } + let sparse = (127..=1023) + .step_by(128) + .map(|doc| (doc, 1.5)) + .collect::>(); + push(materialized(&sparse)); + (children, work) + } + + fn scorer_advances(work: &[Arc]) -> usize { + work.iter() + .map(|work| work.advances.load(AtomicOrdering::Relaxed)) + .sum() + } + + fn exhaustive_should_top_k(children: &[Vec<(u64, f32)>], limit: usize) -> Vec { + let mut scores = HashMap::::new(); + for child in children { + for (doc, score) in child { + *scores.entry(*doc).or_default() += *score; + } + } + let mut rows = scores + .into_iter() + .map(|(doc, score)| ScoredRow::new(doc, score).unwrap()) + .collect::>(); + rows.sort_unstable_by(compare_scored_rows); + rows.truncate(limit); + rows + } + + fn exhaustive_compound_scores( + plan: &CompoundScorerPlan, + leaves: &[HashMap], + ) -> HashMap { + match plan { + CompoundScorerPlan::Leaf { index, boost } => leaves[*index] + .iter() + .map(|(row_address, score)| (*row_address, *score * *boost)) + .collect(), + CompoundScorerPlan::Boost { + positive, + negative, + negative_boost, + } => { + let mut positive = exhaustive_compound_scores(positive, leaves); + let negative = exhaustive_compound_scores(negative, leaves); + for (row_address, score) in &mut positive { + if let Some(negative_score) = negative.get(row_address) { + *score -= *negative_boost * *negative_score; + } + } + positive + } + CompoundScorerPlan::MultiMatch(children) => { + let mut scores = HashMap::::new(); + for child in children { + for (row_address, score) in exhaustive_compound_scores(child, leaves) { + scores + .entry(row_address) + .and_modify(|current| *current = current.max(score)) + .or_insert(score); + } + } + scores + } + CompoundScorerPlan::Boolean { + should, + must, + must_not, + } => { + let mut scores = if let Some((first, remaining)) = must.split_first() { + let mut scores = exhaustive_compound_scores(first, leaves); + for child in remaining { + let required = exhaustive_compound_scores(child, leaves); + scores.retain(|row_address, score| { + if let Some(required_score) = required.get(row_address) { + *score += required_score; + true + } else { + false + } + }); + } + for child in should { + for (row_address, optional_score) in + exhaustive_compound_scores(child, leaves) + { + if let Some(score) = scores.get_mut(&row_address) { + *score += optional_score; + } + } + } + scores + } else { + let mut scores = HashMap::::new(); + for child in should { + for (row_address, score) in exhaustive_compound_scores(child, leaves) { + *scores.entry(row_address).or_default() += score; + } + } + scores + }; + + for child in must_not { + for row_address in exhaustive_compound_scores(child, leaves).into_keys() { + scores.remove(&row_address); + } + } + scores + } + } + } + + fn exhaustive_compound_top_k( + plan: &CompoundScorerPlan, + leaves: &[HashMap], + limit: usize, + ) -> Vec { + let mut scores = exhaustive_compound_scores(plan, leaves) + .into_iter() + .collect::>(); + scores.sort_unstable_by(|(left_row, left_score), (right_row, right_score)| { + right_score + .total_cmp(left_score) + .then_with(|| left_row.cmp(right_row)) + }); + scores.truncate(limit); + scores + .into_iter() + .map(|(row_address, score)| ScoredRow::new(row_address, score).unwrap()) + .collect() + } + + fn randomized_mapped_leaf( + scores: &HashMap, + canonical_row_addresses: &[u64], + rng: &mut SmallRng, + ) -> BoxScorer<'static> { + let source_count = rng.random_range(2..=3); + let source_by_row = (0..256) + .find_map(|_| { + let source_by_row = (0..canonical_row_addresses.len()) + .map(|_| rng.random_range(0..source_count)) + .collect::>(); + let mut projection_lengths = vec![0_u64; source_count]; + let mut local_matches = vec![Vec::::new(); source_count]; + for (row_index, row_address) in canonical_row_addresses.iter().enumerate() { + let source_index = source_by_row[row_index]; + let local_doc = projection_lengths[source_index]; + projection_lengths[source_index] += 1; + if scores.contains_key(row_address) { + local_matches[source_index].push(local_doc); + } + } + + let local_gap_patterns = local_matches + .iter() + .map(|matches| { + matches + .windows(2) + .map(|pair| pair[1] - pair[0]) + .collect::>() + }) + .collect::>(); + let has_distinct_gapped_sources = + local_matches + .iter() + .enumerate() + .all(|(source_index, matches)| { + matches.len() >= 2 + && matches.windows(2).any(|pair| pair[1] > pair[0] + 1) + && matches.len() * 4 > projection_lengths[source_index] as usize + }) + && local_gap_patterns + .iter() + .enumerate() + .all(|(source_index, gaps)| { + local_gap_patterns[..source_index] + .iter() + .all(|previous| previous != gaps) + }); + has_distinct_gapped_sources.then_some(source_by_row) + }) + .expect("randomized physical sources should have distinct local-document gaps"); + + let mut sources = Vec::with_capacity(source_count); + for source_index in 0..source_count { + let projection_addresses = canonical_row_addresses + .iter() + .enumerate() + .filter_map(|(row_index, row_address)| { + (source_by_row[row_index] == source_index).then_some(*row_address) + }) + .collect::>(); + let local_rows = projection_addresses + .iter() + .enumerate() + .filter_map(|(local_doc, row_address)| { + scores + .get(row_address) + .map(|score| (local_doc as u64, *score)) + }) + .collect::>(); + let first_match = local_rows + .first() + .expect("every randomized physical source should have postings") + .0; + let local_document_lower_bound = rng.random_range(0..=first_match); + let projection = resident_row_address_projection_for_test(projection_addresses.clone()); + let prepared = prepare_row_address_projection(&projection); + let source = map_scorer_to_row_addresses( + materialized(&local_rows), + &prepared, + local_document_lower_bound, + ) + .unwrap() + .expect("a randomized physical source with postings should map"); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + assert_eq!( + projection.ordered_validation_visited_docs(), + projection_addresses.len() + ); + sources.push(source); + } + + Box::new(RowAddressMergeScorer::try_new(sources).unwrap()) + } + + fn plan_leaf(index: usize) -> CompoundScorerPlan { + CompoundScorerPlan::Leaf { index, boost: 1.0 } + } + + #[test] + fn phrase_doc_bounds_match_exhaustive_oracle_across_boolean_shapes() { + // Leaf 0 models an exact phrase and leaf 1 a sloppy phrase. Their + // approximation scores include false position candidates, including a + // high-scoring false candidate at doc 4. Docs 1 and 3 deliberately tie. + let exact_approximations = [(0, 2.0), (1, 4.0), (2, 3.0), (3, 4.0), (4, 100.0)]; + let sloppy_approximations = [(0, 1.0), (1, 2.0), (2, 3.0), (3, 2.0), (4, 1.0)]; + let exact_matches = HashMap::from([(1, 4.0), (3, 4.0)]); + let sloppy_matches = HashMap::from([(1, 2.0), (2, 3.0), (3, 2.0)]); + let optional = HashMap::from([(0, 1.0), (1, 4.0), (2, 2.0), (3, 4.0), (4, 1.0)]); + let required = HashMap::from([(0, 1.0), (1, 1.0), (2, 1.0), (3, 1.0), (4, 1.0)]); + let oracle_leaves = vec![exact_matches, sloppy_matches, optional.clone(), required]; + + let pure_should = CompoundScorerPlan::Boolean { + should: vec![plan_leaf(0), plan_leaf(1), plan_leaf(2)], + must: Vec::new(), + must_not: Vec::new(), + }; + let must_should = CompoundScorerPlan::Boolean { + should: vec![plan_leaf(0), plan_leaf(1), plan_leaf(2)], + must: vec![plan_leaf(3)], + must_not: Vec::new(), + }; + let nested = CompoundScorerPlan::Boolean { + should: vec![pure_should.clone()], + must: vec![plan_leaf(3)], + must_not: Vec::new(), + }; + + for (shape, plan, floor) in [ + ("should", pure_should, 10.0), + ("must_should", must_should, 11.0), + ("nested", nested, 11.0), + ] { + let (exact, _, exact_confirmations) = + two_phase(&exact_approximations, vec![1, 3], Some(1.0)); + let (sloppy, _, sloppy_confirmations) = + two_phase(&sloppy_approximations, vec![1, 2, 3], Some(2.0)); + let mut leaves = vec![ + Some(exact), + Some(sloppy), + Some(materialized( + &optional + .iter() + .map(|(doc, score)| (*doc, *score)) + .collect::>(), + )), + Some(materialized(&[ + (0, 1.0), + (1, 1.0), + (2, 1.0), + (3, 1.0), + (4, 1.0), + ])), + ]; + let mut scorer = plan.build(&mut leaves).unwrap(); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(floor); + let actual = TopKCollector::with_competitive_score(2, competitive_score) + .collect(scorer.as_mut()) + .unwrap(); + let expected = exhaustive_compound_top_k(&plan, &oracle_leaves, 2); + + assert_eq!(actual, expected, "shape={shape}"); + assert_eq!(actual, rows(&[(1, floor), (3, floor)]), "shape={shape}"); + assert!( + exact_confirmations.load(AtomicOrdering::Relaxed) > 0 + && sloppy_confirmations.load(AtomicOrdering::Relaxed) > 0, + "shape={shape} must confirm surviving equal-floor phrase candidates" + ); + } + } + + fn plan_input(possible: bool, cost: usize, lower: f32, upper: f32) -> CompoundLeafPlanInput { + CompoundLeafPlanInput::new(possible, cost, ScoreBounds::try_new(lower, upper).unwrap()) + } + + #[test] + fn staged_plan_analysis_selects_stable_must_generator() { + let plan = CompoundScorerPlan::Boolean { + should: vec![plan_leaf(0)], + must: vec![ + CompoundScorerPlan::MultiMatch(vec![plan_leaf(1), plan_leaf(2)]), + CompoundScorerPlan::Boost { + positive: Box::new(plan_leaf(3)), + negative: Box::new(plan_leaf(4)), + negative_boost: 0.5, + }, + ], + must_not: vec![plan_leaf(5)], + }; + let analysis = plan + .analyze_leaves(&[ + plan_input(true, 9, 1.0, 2.0), + plan_input(true, 2, 2.0, 4.0), + plan_input(true, 2, -1.0, 3.0), + plan_input(true, 4, 5.0, 6.0), + plan_input(true, 1, 1.0, 2.0), + plan_input(true, 1, 0.0, 10.0), + ]) + .unwrap(); + + assert_eq!(plan.leaf_count(), 6); + assert!(analysis.possible); + assert_eq!(analysis.generator_cost, 4); + // Equal-cost MUST covers retain query order, then expose a canonical + // sorted/deduplicated leaf list to the I/O scheduler. + assert_eq!(analysis.generator_leaves, vec![1, 2]); + assert!(analysis.bounds.lower() <= 3.0); + assert!(analysis.bounds.upper() >= 12.0); + } + + #[test] + fn staged_plan_analysis_handles_optional_missing_and_impossible_required() { + let pure_should = CompoundScorerPlan::Boolean { + // Deliberately use non-canonical traversal order so the public + // generator list must sort independently of query-tree layout. + should: vec![plan_leaf(2), plan_leaf(0), plan_leaf(1)], + must: Vec::new(), + must_not: Vec::new(), + }; + let analysis = pure_should + .analyze_leaves(&[ + plan_input(true, 7, 1.0, 2.0), + plan_input(false, 1, 100.0, 200.0), + plan_input(true, 3, -4.0, -1.0), + ]) + .unwrap(); + assert!(analysis.possible); + assert_eq!(analysis.generator_cost, 10); + assert_eq!(analysis.generator_leaves, vec![0, 2]); + assert!(analysis.bounds.lower() <= -4.0); + assert!(analysis.bounds.upper() >= 2.0); + + let missing_must = CompoundScorerPlan::Boolean { + should: vec![plan_leaf(0)], + must: vec![plan_leaf(1)], + must_not: Vec::new(), + }; + let analysis = missing_must + .analyze_leaves(&[ + plan_input(true, 1, 0.0, 10.0), + plan_input(false, 1, 0.0, 10.0), + ]) + .unwrap(); + assert!(!analysis.possible); + assert_eq!(analysis.bounds, ScoreBounds::ZERO); + assert!(analysis.generator_leaves.is_empty()); + + let only_must_not = CompoundScorerPlan::Boolean { + should: Vec::new(), + must: Vec::new(), + must_not: vec![plan_leaf(0)], + }; + let analysis = only_must_not + .analyze_leaves(&[plan_input(true, 1, 0.0, 10.0)]) + .unwrap(); + assert!(!analysis.possible); + assert!(analysis.generator_leaves.is_empty()); + } + + #[test] + fn staged_plan_analysis_composes_signed_nested_boost_and_unbounded_inputs() { + let plan = CompoundScorerPlan::Boost { + positive: Box::new(plan_leaf(0)), + negative: Box::new(CompoundScorerPlan::Boost { + positive: Box::new(plan_leaf(1)), + negative: Box::new(plan_leaf(2)), + negative_boost: 1.0, + }), + negative_boost: 0.5, + }; + let analysis = plan + .analyze_leaves(&[ + plan_input(true, 3, 2.0, 3.0), + plan_input(true, 1, 1.0, 2.0), + plan_input(true, 1, 4.0, 5.0), + ]) + .unwrap(); + assert_eq!(analysis.generator_leaves, vec![0]); + // The nested negative can itself be negative, so subtracting it may + // increase the outer Boost score. + assert!(analysis.bounds.lower() <= 1.0); + assert!(analysis.bounds.upper() >= 5.0); + + let unbounded = CompoundScorerPlan::Boolean { + should: vec![plan_leaf(0), plan_leaf(1)], + must: Vec::new(), + must_not: Vec::new(), + } + .analyze_leaves(&[ + CompoundLeafPlanInput::new(true, 1, ScoreBounds::UNBOUNDED), + plan_input(true, 1, 0.0, 1.0), + ]) + .unwrap(); + assert_eq!(unbounded.bounds, ScoreBounds::UNBOUNDED); + + assert!(ScoreBounds::try_new(f32::NAN, 1.0).is_err()); + let invalid = [CompoundLeafPlanInput { + possible: true, + cost: 1, + bounds: ScoreBounds { + lower: 2.0, + upper: 1.0, + }, + }]; + assert!(plan_leaf(0).analyze_leaves(&invalid).is_err()); + } + + fn random_staged_plan( + rng: &mut SmallRng, + depth: usize, + next_leaf: &mut usize, + ) -> CompoundScorerPlan { + if depth == 0 || rng.random_bool(0.35) { + let index = *next_leaf; + *next_leaf += 1; + return CompoundScorerPlan::Leaf { + index, + boost: [0.0, 0.5, 1.0, 2.0][rng.random_range(0..4)], + }; + } + match rng.random_range(0..3) { + 0 => CompoundScorerPlan::Boost { + positive: Box::new(random_staged_plan(rng, depth - 1, next_leaf)), + negative: Box::new(random_staged_plan(rng, depth - 1, next_leaf)), + negative_boost: [0.0, 0.5, 1.0, 1.5][rng.random_range(0..4)], + }, + 1 => CompoundScorerPlan::MultiMatch( + (0..rng.random_range(1..=3)) + .map(|_| random_staged_plan(rng, depth - 1, next_leaf)) + .collect(), + ), + _ => { + let must_count = rng.random_range(0..=2); + let should_count = if must_count == 0 { + rng.random_range(1..=3) + } else { + rng.random_range(0..=2) + }; + let should = (0..should_count) + .map(|_| random_staged_plan(rng, depth - 1, next_leaf)) + .collect(); + let must = (0..must_count) + .map(|_| random_staged_plan(rng, depth - 1, next_leaf)) + .collect(); + let must_not = (0..rng.random_range(0..=2)) + .map(|_| random_staged_plan(rng, depth - 1, next_leaf)) + .collect(); + CompoundScorerPlan::Boolean { + should, + must, + must_not, + } + } + } + } + + #[test] + fn randomized_staged_plan_bounds_and_generator_cover_exact_matches() { + for seed in 0..64 { + let mut rng = SmallRng::seed_from_u64(seed); + let mut leaf_count = 0; + let plan = random_staged_plan(&mut rng, 3, &mut leaf_count); + assert_eq!(plan.leaf_count(), leaf_count, "seed={seed}"); + let inputs = (0..leaf_count) + .map(|_| { + let possible = rng.random_bool(0.8); + let lower = rng.random_range(-4..=2) as f32; + let upper = rng.random_range(lower as i32..=5) as f32; + plan_input(possible, rng.random_range(1..=32), lower, upper) + }) + .collect::>(); + let analysis = plan.analyze_leaves(&inputs).unwrap(); + assert!( + analysis + .generator_leaves + .windows(2) + .all(|pair| pair[0] < pair[1]) + ); + + for document in 0..128_u64 { + let mut leaves = vec![HashMap::new(); leaf_count]; + for (leaf_index, input) in inputs.iter().enumerate() { + if input.possible && rng.random_bool(0.5) { + let score = rng.random_range(input.bounds.lower()..=input.bounds.upper()); + leaves[leaf_index].insert(document, score); + } + } + if let Some(score) = exhaustive_compound_scores(&plan, &leaves).get(&document) { + assert!(analysis.possible, "seed={seed}, document={document}"); + assert!( + analysis.bounds.lower() <= *score && *score <= analysis.bounds.upper(), + "seed={seed}, document={document}, score={score}, bounds={:?}", + analysis.bounds + ); + assert!( + analysis + .generator_leaves + .iter() + .any(|leaf| leaves[*leaf].contains_key(&document)), + "seed={seed}, document={document}, generators={:?}", + analysis.generator_leaves + ); + } + } + } + } + + #[test] + fn randomized_mapped_sources_match_recursive_exhaustive_oracle() { + let plans = [ + ( + "should_sum", + CompoundScorerPlan::Boolean { + should: (0..4).map(plan_leaf).collect(), + must: Vec::new(), + must_not: Vec::new(), + }, + ), + ( + "multimatch_max", + CompoundScorerPlan::MultiMatch((0..4).map(plan_leaf).collect()), + ), + ( + "must_sum", + CompoundScorerPlan::Boolean { + should: Vec::new(), + must: (0..4).map(plan_leaf).collect(), + must_not: Vec::new(), + }, + ), + ( + "required_optional", + CompoundScorerPlan::Boolean { + should: vec![plan_leaf(1), plan_leaf(2), plan_leaf(3)], + must: vec![plan_leaf(0)], + must_not: Vec::new(), + }, + ), + ( + "signed_boost", + CompoundScorerPlan::Boost { + positive: Box::new(CompoundScorerPlan::MultiMatch(vec![ + plan_leaf(0), + CompoundScorerPlan::Leaf { + index: 1, + boost: 0.5, + }, + ])), + negative: Box::new(CompoundScorerPlan::Boolean { + should: vec![plan_leaf(2), plan_leaf(3)], + must: Vec::new(), + must_not: Vec::new(), + }), + negative_boost: 1.5, + }, + ), + ( + "must_not", + CompoundScorerPlan::Boolean { + should: vec![plan_leaf(1)], + must: vec![plan_leaf(0)], + must_not: vec![CompoundScorerPlan::MultiMatch(vec![ + plan_leaf(2), + plan_leaf(3), + ])], + }, + ), + ]; + + for seed in 0..12 { + let mut rng = SmallRng::seed_from_u64(seed); + let num_rows = rng.random_range(32..=64); + let mut next_row_address = (seed + 1) << 32; + let canonical_row_addresses = (0..num_rows) + .map(|_| { + next_row_address += rng.random_range(1..=16); + next_row_address + }) + .collect::>(); + let mut leaves = vec![HashMap::::new(); 4]; + for (row_index, row_address) in canonical_row_addresses.iter().enumerate() { + let mut matched = false; + for (leaf_index, leaf) in leaves.iter_mut().enumerate() { + let is_required_only_canary = row_index < 16 && leaf_index < 2; + let is_random_match = row_index >= 16 && rng.random_bool(0.5); + if row_index < 8 || is_required_only_canary || is_random_match { + let score = if row_index < 8 { + 2.0 + } else { + rng.random_range(1..=4) as f32 * 0.5 + }; + leaf.insert(*row_address, score); + matched = true; + } + } + if !matched { + let leaf_index = rng.random_range(0..leaves.len()); + let score = rng.random_range(1..=4) as f32 * 0.5; + leaves[leaf_index].insert(*row_address, score); + } + } + + let max_scores = exhaustive_compound_scores(&plans[1].1, &leaves); + let max_score = max_scores.values().copied().max_by(f32::total_cmp).unwrap(); + assert!( + max_scores + .values() + .filter(|score| **score == max_score) + .count() + >= 8, + "seed={seed} should retain enough top-score ties to cross every tested limit" + ); + assert!( + exhaustive_compound_scores(&plans[4].1, &leaves) + .values() + .any(|score| *score < 0.0), + "seed={seed} should exercise signed Boost scores" + ); + let must_not_scores = exhaustive_compound_scores(&plans[5].1, &leaves); + assert!( + canonical_row_addresses[..8] + .iter() + .all(|row_address| !must_not_scores.contains_key(row_address)) + && canonical_row_addresses[8..16] + .iter() + .all(|row_address| must_not_scores.contains_key(row_address)), + "seed={seed} should exercise both prohibited and retained candidates" + ); + + for (shape, plan) in &plans { + for limit in [1, 3, 7] { + let expected = exhaustive_compound_top_k(plan, &leaves, limit); + let mut mapped_leaves = leaves + .iter() + .map(|leaf| { + Some(randomized_mapped_leaf( + leaf, + &canonical_row_addresses, + &mut rng, + )) + }) + .collect::>(); + let mut scorer = plan.build(&mut mapped_leaves).unwrap(); + assert!(mapped_leaves.iter().all(Option::is_none)); + let actual = TopKCollector::new(limit).collect(scorer.as_mut()).unwrap(); + assert_eq!(actual, expected, "seed={seed} shape={shape} limit={limit}"); + } + } + } + } + + #[test] + fn pure_should_maxscore_reduces_posting_comparisons() { + let (children, eager_work) = pure_should_canary_children(); + let mut eager = DisjunctionScorer::try_new(children, DisjunctionScore::Sum).unwrap(); + let eager_results = TopKCollector::new(1).collect(&mut eager).unwrap(); + let eager_comparisons = scorer_advances(&eager_work); + + let (children, optimized_work) = pure_should_canary_children(); + let optimized_results = { + let mut optimized = should_maxscore(children); + TopKCollector::new(1).collect(&mut optimized).unwrap() + }; + let optimized_comparisons = scorer_advances(&optimized_work); + + assert_eq!(eager_results, rows(&[(127, 2.5)])); + assert_eq!(optimized_results, eager_results); + assert!(eager_comparisons > 0); + assert!( + optimized_comparisons * 5 <= eager_comparisons * 4, + "pure-SHOULD MAXSCORE should reduce posting candidate probes by at least 20%: \ + optimized={optimized_comparisons} eager={eager_comparisons}" + ); + } + + #[test] + fn pure_should_maxscore_matches_randomized_exhaustive_top_k() { + for seed in 0..8 { + let mut rng = SmallRng::seed_from_u64(seed); + let children = (0..6) + .map(|_| { + (0..256) + .filter_map(|doc| { + if rng.random_bool(0.35) { + let score = rng.random_range(1..=16) as f32 * 0.25; + Some((doc, score)) + } else { + None + } + }) + .collect::>() + }) + .collect::>(); + + for limit in [1, 7, 31, 512] { + let expected = exhaustive_should_top_k(&children, limit); + + let mut optimized = + should_maxscore(children.iter().map(|values| materialized(values)).collect()); + let actual = TopKCollector::new(limit).collect(&mut optimized).unwrap(); + assert_eq!(actual, expected, "seed={seed} limit={limit}"); + } + } + } + + #[test] + fn pure_should_maxscore_confirms_two_phase_children_before_scoring() { + let (phrase, _, confirmations) = two_phase(&[(1, 100.0)], Vec::new(), Some(10.0)); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(10.0); + let results = { + let mut scorer = should_maxscore(vec![ + materialized(&[(0, 10.0)]), + phrase, + materialized(&[(1, 6.0)]), + materialized(&[(1, 5.0)]), + ]); + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap() + }; + + assert_eq!(results, rows(&[(1, 11.0)])); + assert_eq!(confirmations.load(AtomicOrdering::Relaxed), 1); + } + + #[test] + fn pure_should_maxscore_preserves_query_score_order_and_terminal_doc() { + let mut scorer = should_maxscore(vec![ + materialized(&[(u64::MAX, 16_777_216.0)]), + materialized(&[(u64::MAX, 1.0)]), + materialized(&[(u64::MAX, 1.0)]), + materialized(&[]), + ]); + + assert_eq!( + TopKCollector::new(1).collect(&mut scorer).unwrap(), + rows(&[(u64::MAX, 16_777_216.0)]) + ); + } + + #[test] + fn pure_should_maxscore_keeps_equal_floor_across_bound_ordering() { + let scores = [ + f32::from_bits(0x4783_798b), + f32::from_bits(0x4dd3_8b75), + f32::from_bits(0x48e7_7236), + f32::from_bits(0x418e_5b26), + f32::from_bits(0x4241_b1eb), + ]; + let exact_score = scores + .into_iter() + .fold(0.0_f32, |total, score| total + score); + assert_eq!(exact_score.to_bits(), 0x4dd3_cd8d); + + let mut scorer = should_maxscore( + scores + .into_iter() + .map(|score| materialized(&[(7, score)])) + .collect(), + ); + let competitive_score = Arc::new(CompetitiveScore::default()); + competitive_score.raise(exact_score); + + assert_eq!( + TopKCollector::with_competitive_score(1, competitive_score) + .collect(&mut scorer) + .unwrap(), + rows(&[(7, exact_score)]) + ); + } + + #[test] + fn pure_should_maxscore_supports_nested_non_negative_children() { + let nested_dismax = Box::new( + DisjunctionScorer::try_new( + vec![ + materialized(&[(0, 1.0), (1, 5.0)]), + materialized(&[(0, 3.0), (2, 4.0)]), + ], + DisjunctionScore::Max, + ) + .unwrap(), + ); + let nested_boolean = Box::new( + BooleanScorer::try_new( + Vec::new(), + vec![materialized(&[(0, 2.0), (1, 2.0), (2, 2.0)])], + vec![materialized(&[(1, 0.0)])], + ) + .unwrap(), + ); + let results = { + let mut scorer = BooleanScorer::try_new( + vec![ + nested_dismax, + nested_boolean, + materialized(&[(0, 0.5), (1, 0.5), (2, 0.5)]), + ], + Vec::new(), + Vec::new(), + ) + .unwrap(); + TopKCollector::new(1).collect(&mut scorer).unwrap() + }; + + assert_eq!(results, rows(&[(2, 6.5)])); + } + + #[test] + fn pure_should_maxscore_applies_must_not_before_raising_the_floor() { + let results = { + let mut scorer = BooleanScorer::try_new( + vec![ + materialized(&[(0, 10.0), (1, 5.0)]), + materialized(&[(0, 1.0), (1, 1.0)]), + materialized(&[(2, 8.0)]), + ], + Vec::new(), + vec![materialized(&[(0, 1.0)])], + ) + .unwrap(); + TopKCollector::new(1).collect(&mut scorer).unwrap() + }; + + assert_eq!(results, rows(&[(2, 8.0)])); + } + + #[test] + fn pure_should_uses_exact_fallback_for_unsupported_shapes() { + let signed_results = { + let signed = Box::new( + BoostScorer::try_new( + materialized(&[(0, 5.0), (1, 1.0)]), + materialized(&[(0, 2.0), (1, 4.0)]), + 1.0, + ) + .unwrap(), + ); + let mut scorer = BooleanScorer::try_new( + vec![ + signed, + materialized(&[(0, 1.0), (1, 1.0)]), + materialized(&[(1, 5.0)]), + ], + Vec::new(), + Vec::new(), + ) + .unwrap(); + TopKCollector::new(2).collect(&mut scorer).unwrap() + }; + assert_eq!(signed_results, rows(&[(0, 4.0), (1, 3.0)])); + + let unbounded_results = { + let unbounded = Box::new(UnboundedScorer { + inner: MaterializedScorer::try_new(rows(&[(0, 1.0), (2, 3.0)])).unwrap(), + }); + let mut scorer = BooleanScorer::try_new( + vec![ + unbounded, + materialized(&[(0, 2.0), (1, 2.0)]), + materialized(&[(1, 4.0)]), + ], + Vec::new(), + Vec::new(), + ) + .unwrap(); + TopKCollector::new(3).collect(&mut scorer).unwrap() + }; + assert_eq!(unbounded_results, rows(&[(1, 6.0), (0, 3.0), (2, 3.0)])); + + { + let mut scorer = BooleanScorer::try_new( + vec![materialized(&[(0, 1.0)]), materialized(&[(1, 2.0)])], + Vec::new(), + Vec::new(), + ) + .unwrap(); + assert_eq!( + TopKCollector::new(2).collect(&mut scorer).unwrap(), + rows(&[(1, 2.0), (0, 1.0)]) + ); + } + + let large_score = f32::MAX / 2.0; + { + let mut scorer = BooleanScorer::try_new( + vec![ + materialized(&[(0, large_score)]), + materialized(&[(1, large_score)]), + materialized(&[(2, large_score)]), + ], + Vec::new(), + Vec::new(), + ) + .unwrap(); + assert_eq!( + TopKCollector::new(3).collect(&mut scorer).unwrap(), + rows(&[(0, large_score), (1, large_score), (2, large_score)]) + ); + } + } +} diff --git a/rust/lance-index/src/scalar/inverted/compound/should_maxscore.rs b/rust/lance-index/src/scalar/inverted/compound/should_maxscore.rs new file mode 100644 index 00000000000..1b71f66779f --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/compound/should_maxscore.rs @@ -0,0 +1,596 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use super::*; + +/// Below three clauses, maintaining MAXSCORE windows costs more than the +/// generic document-at-a-time union is likely to save. +const MIN_SHOULD_MAXSCORE_CLAUSES: usize = 3; + +#[derive(Clone, Copy)] +struct WindowBounds { + start: u64, + up_to: u64, + floor: f32, + combined_upper: f32, +} + +#[derive(Clone, Copy)] +struct ReportedBounds { + target: u64, + up_to: u64, + bounds: ScoreBounds, +} + +/// Exact windowed MAXSCORE scorer for same-column Boolean SHOULD sums. +/// +/// List-wide maxima split clauses into a non-essential prefix whose total +/// score cannot reach the current floor and an essential suffix that drives +/// candidate iteration and shallow-window boundaries. Non-essential clauses +/// are only probed when they can still make an essential candidate competitive. +pub(super) struct ShouldMaxScoreScorer<'a> { + children: Vec>, + initialized: bool, + exhausted: bool, + current: Option, + confirmed_doc: Option, + confirmed: bool, + current_score: Option, + min_competitive_score: f32, + window: Option, + reported_bounds: Option, + global_upper_bounds: Vec, + global_score_upper_bound: f32, + child_upper_bounds: Vec, + essential: Vec, + bound_order: Vec, + child_scores: Vec>, +} + +impl<'a> ShouldMaxScoreScorer<'a> { + pub(super) fn global_bounds(children: &[BoxScorer<'a>]) -> Option> { + if children.len() < MIN_SHOULD_MAXSCORE_CLAUSES { + return None; + } + if !children.iter().all(|child| child.scores_non_negative()) { + return None; + } + let bounds = children + .iter() + .map(|child| { + child + .global_score_upper_bound() + .filter(|upper| upper.is_finite() && *upper >= 0.0) + }) + .collect::>>()?; + Self::sum_uppers(bounds.iter()) + .is_finite() + .then_some(bounds) + } + + pub(super) fn new(children: Vec>, global_upper_bounds: Vec) -> Self { + debug_assert_eq!(children.len(), global_upper_bounds.len()); + let num_children = children.len(); + let global_score_upper_bound = Self::sum_uppers(global_upper_bounds.iter()); + let mut bound_order = (0..num_children).collect::>(); + bound_order.sort_by(|left, right| { + global_upper_bounds[*left] + .total_cmp(&global_upper_bounds[*right]) + .then_with(|| left.cmp(right)) + }); + Self { + children, + initialized: false, + exhausted: false, + current: None, + confirmed_doc: None, + confirmed: false, + current_score: None, + min_competitive_score: f32::NEG_INFINITY, + window: None, + reported_bounds: None, + global_upper_bounds, + global_score_upper_bound, + child_upper_bounds: vec![0.0; num_children], + essential: vec![true; num_children], + bound_order, + child_scores: vec![None; num_children], + } + } + + fn reset_current(&mut self) { + self.current = None; + self.confirmed_doc = None; + self.confirmed = false; + self.current_score = None; + self.child_scores.fill(None); + self.reported_bounds = None; + } + + fn set_current(&mut self, current: u64) { + self.reset_current(); + self.current = Some(current); + } + + fn exhaust(&mut self) -> Option { + self.exhausted = true; + self.reset_current(); + self.window = None; + None + } + + fn initialize_next(&mut self) -> Result<()> { + if self.initialized { + return Ok(()); + } + for child in &mut self.children { + child.next()?; + } + self.initialized = true; + Ok(()) + } + + fn initialize_advance(&mut self, target: u64) -> Result<()> { + if self.initialized { + return Ok(()); + } + for child in &mut self.children { + child.advance(target)?; + } + self.initialized = true; + Ok(()) + } + + fn align_all_children(&mut self, target: u64) -> Result<()> { + for child in &mut self.children { + if child.doc().is_some_and(|doc| doc < target) { + child.advance(target)?; + } + } + Ok(()) + } + + fn select_essential_children(&mut self) { + self.essential.fill(false); + let floor = self.min_competitive_score; + if floor <= 0.0 || floor.is_nan() { + for (is_essential, child) in self.essential.iter_mut().zip(&self.children) { + *is_essential = child.doc().is_some(); + } + return; + } + + let mut non_essential = 0.0_f64; + let mut num_non_essential = 0; + let mut found_essential = false; + for index in &self.bound_order { + if self.children[*index].doc().is_none() { + continue; + } + if found_essential { + self.essential[*index] = true; + continue; + } + let next = non_essential + f64::from(self.global_upper_bounds[*index]); + let widened_exact = next * score_sum_upper_bound_factor(num_non_essential + 1); + let rounded = widened_exact as f32; + let widened = if f64::from(rounded) < widened_exact { + next_up(rounded) + } else { + rounded + }; + if widened < floor { + non_essential = next; + num_non_essential += 1; + } else { + self.essential[*index] = true; + found_essential = true; + } + } + } + + fn usable_bounds(bounds: ScoreBounds) -> bool { + bounds.lower.is_finite() + && bounds.upper.is_finite() + && bounds.lower <= bounds.upper + && bounds.upper >= 0.0 + } + + fn add_upper(bounds: ScoreBounds, upper: f32) -> ScoreBounds { + bounds.add(ScoreBounds { lower: 0.0, upper }) + } + + fn sum_uppers<'b>(uppers: impl Iterator) -> f32 { + uppers + .fold(ScoreBounds::ZERO, |sum, upper| Self::add_upper(sum, *upper)) + .upper + } + + fn prepare_window(&mut self, target: u64) -> Result<()> { + self.child_upper_bounds.fill(0.0); + self.reported_bounds = None; + + self.select_essential_children(); + if !self.essential.iter().any(|is_essential| *is_essential) { + self.exhaust(); + return Ok(()); + } + + let mut up_to = u64::MAX; + let mut has_active_child = false; + for (child, is_essential) in self.children.iter_mut().zip(&mut self.essential) { + if !*is_essential { + continue; + } + if child.doc().is_some_and(|doc| doc < target) { + child.advance(target)?; + } + if let Some(doc) = child.doc() { + has_active_child = true; + let child_target = target.max(doc); + let child_up_to = child.advance_shallow(child_target)?; + if child_up_to < child_target { + return Err(Error::internal(format!( + "FTS SHOULD child returned shallow range ending at {child_up_to} before target {child_target}" + ))); + } + up_to = up_to.min(child_up_to); + } else { + *is_essential = false; + } + } + if !has_active_child { + self.exhaust(); + return Ok(()); + } + + for (index, child) in self.children.iter().enumerate() { + if !self.essential[index] && child.doc().is_some() { + self.child_upper_bounds[index] = self.global_upper_bounds[index]; + } + } + for (index, child) in self.children.iter_mut().enumerate() { + if self.essential[index] && child.doc().is_some_and(|doc| doc <= up_to) { + let bounds = child.score_bounds(up_to)?; + if Self::usable_bounds(bounds) { + self.child_upper_bounds[index] = + bounds.upper.max(0.0).min(self.global_upper_bounds[index]); + } else { + self.child_upper_bounds[index] = self.global_upper_bounds[index]; + } + } + } + + let combined_upper = Self::sum_uppers(self.child_upper_bounds.iter()); + let floor = self.min_competitive_score; + self.window = Some(WindowBounds { + start: target, + up_to, + floor, + combined_upper, + }); + Ok(()) + } + + fn position(&mut self, mut target: u64) -> Result> { + if self.exhausted { + return Ok(None); + } + + loop { + let needs_window = self.window.is_none_or(|window| { + target < window.start + || target > window.up_to + || self.min_competitive_score > window.floor + }); + if needs_window { + self.window = None; + self.prepare_window(target)?; + if self.exhausted { + return Ok(None); + } + } + let window = self + .window + .ok_or_else(|| Error::internal("FTS SHOULD scorer did not prepare a window"))?; + + if window.combined_upper < self.min_competitive_score { + if window.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = window.up_to + 1; + self.window = None; + continue; + } + + let next = self + .children + .iter() + .zip(&self.essential) + .filter_map(|(child, is_essential)| { + (*is_essential) + .then(|| child.doc()) + .flatten() + .filter(|doc| *doc >= target && *doc <= window.up_to) + }) + .min(); + if let Some(next) = next { + self.set_current(next); + return Ok(self.current); + } + + if window.up_to == u64::MAX { + return Ok(self.exhaust()); + } + target = window.up_to + 1; + self.window = None; + } + } + + fn partial_score_upper(&self) -> f32 { + let mut bounds = ScoreBounds::ZERO; + for (index, score) in self.child_scores.iter().enumerate() { + if let Some(score) = score { + bounds = bounds.add(ScoreBounds { + lower: *score, + upper: *score, + }); + } else if !self.essential[index] { + bounds = Self::add_upper(bounds, self.child_upper_bounds[index]); + } + } + bounds.upper + } + + fn ensure_confirmed(&mut self) -> Result { + let Some(current) = self.current else { + return Ok(false); + }; + if self.confirmed_doc == Some(current) { + return Ok(self.confirmed); + } + + self.child_scores.fill(None); + for index in 0..self.children.len() { + if !self.essential[index] || self.children[index].doc() != Some(current) { + continue; + } + if self.children[index].matches()? { + self.child_scores[index] = Some(self.children[index].score()?); + } + } + + if self.partial_score_upper() >= self.min_competitive_score { + for index in 0..self.children.len() { + if self.essential[index] || self.child_upper_bounds[index] == 0.0 { + continue; + } + if self.children[index].doc().is_some_and(|doc| doc < current) { + self.children[index].advance(current)?; + } + if self.children[index].doc() == Some(current) && self.children[index].matches()? { + self.child_scores[index] = Some(self.children[index].score()?); + } + } + } + + let mut has_match = false; + let mut score = 0.0_f32; + for child_score in self.child_scores.iter().flatten() { + has_match = true; + score += *child_score; + } + score = checked_score(score, "FTS SHOULD MAXSCORE")?; + self.confirmed = has_match && score >= self.min_competitive_score; + self.current_score = self.confirmed.then_some(score); + self.confirmed_doc = Some(current); + Ok(self.confirmed) + } + + fn combined_shallow_bounds(&self, target: u64) -> ReportedBounds { + ReportedBounds { + target, + up_to: u64::MAX, + bounds: ScoreBounds { + lower: 0.0, + upper: self.global_score_upper_bound, + }, + } + } +} + +impl ComposableScorer for ShouldMaxScoreScorer<'_> { + fn doc(&self) -> Option { + self.current + } + + fn document_key(&self) -> Option { + let current = self.current?; + self.children + .iter() + .find(|child| child.doc() == Some(current)) + .and_then(|child| child.document_key()) + } + + fn next(&mut self) -> Result> { + if self.exhausted { + return Ok(None); + } + if !self.initialized { + self.initialize_next()?; + return self.position(0); + } + let Some(current) = self.current else { + return Ok(self.exhaust()); + }; + if current == u64::MAX { + return Ok(self.exhaust()); + } + for child in &mut self.children { + if child.doc() == Some(current) { + child.next()?; + } + } + self.reset_current(); + self.position(current + 1) + } + + fn advance(&mut self, target: u64) -> Result> { + if self.current.is_some_and(|current| current >= target) { + return Ok(self.current); + } + if self.exhausted { + return Ok(None); + } + self.initialize_advance(target)?; + self.align_all_children(target)?; + self.reset_current(); + self.window = None; + self.position(target) + } + + fn cost(&self) -> usize { + self.children + .iter() + .map(|child| child.cost()) + .fold(0, usize::saturating_add) + } + + fn score(&mut self) -> Result { + if !self.ensure_confirmed()? { + return Err(Error::internal( + "score requested from an unconfirmed FTS SHOULD MAXSCORE document", + )); + } + self.current_score.ok_or_else(|| { + Error::internal("confirmed FTS SHOULD MAXSCORE document has no exact score") + }) + } + + fn advance_shallow(&mut self, target: u64) -> Result { + let target = self.current.map_or(target, |current| target.max(current)); + let reported = if let Some(window) = self.window + && target >= window.start + && target <= window.up_to + { + ReportedBounds { + target, + up_to: window.up_to, + bounds: ScoreBounds { + lower: 0.0, + upper: window.combined_upper, + }, + } + } else { + self.combined_shallow_bounds(target) + }; + self.reported_bounds = Some(reported); + Ok(reported.up_to) + } + + fn score_bounds(&mut self, up_to: u64) -> Result { + let reported = self.reported_bounds.ok_or_else(|| { + Error::internal("score_bounds requires advance_shallow on the FTS SHOULD scorer") + })?; + if up_to < reported.target || up_to > reported.up_to { + return Err(Error::internal(format!( + "FTS SHOULD score bound up_to={up_to} is outside shallow range [{}, {}]", + reported.target, reported.up_to + ))); + } + Ok(reported.bounds) + } + + fn global_score_upper_bound(&self) -> Option { + Some(self.global_score_upper_bound) + } + + fn set_min_competitive_score(&mut self, min_score: f32) -> Result<()> { + if min_score.is_nan() { + return Err(Error::invalid_input( + "minimum competitive FTS score cannot be NaN", + )); + } + if min_score > self.min_competitive_score { + self.min_competitive_score = min_score; + } + Ok(()) + } + + fn current_score_upper_bound(&mut self) -> Result> { + let Some(current) = self.current else { + return Ok(None); + }; + self.child_scores.fill(None); + for (index, child) in self.children.iter_mut().enumerate() { + if self.essential[index] { + if child.doc() != Some(current) { + self.child_scores[index] = Some(0.0); + continue; + } + let Some(upper) = child.current_score_upper_bound()? else { + return Ok(None); + }; + if !upper.is_finite() { + return Ok(None); + } + self.child_scores[index] = Some(upper.max(0.0)); + } + } + let mut upper = self.partial_score_upper(); + if upper < self.min_competitive_score { + return Ok(Some(upper)); + } + + // The residual range bound was inconclusive. Tighten it with each + // non-essential posting approximation for this document, largest + // global bound first. Stop as soon as the unresolved residual can no + // longer reach the floor, still without touching phrase positions. + for index in self.bound_order.iter().rev().copied() { + if self.essential[index] { + continue; + } + let child = &mut self.children[index]; + if child.doc().is_some_and(|doc| doc < current) { + child.advance(current)?; + } + self.child_scores[index] = if child.doc() == Some(current) { + let Some(upper) = child.current_score_upper_bound()? else { + return Ok(None); + }; + if !upper.is_finite() { + return Ok(None); + } + Some(upper.max(0.0)) + } else { + Some(0.0) + }; + upper = self.partial_score_upper(); + if upper < self.min_competitive_score { + return Ok(Some(upper)); + } + } + Ok(upper.is_finite().then_some(upper)) + } + + fn supports_doc_local_confirmation_pruning(&self) -> bool { + self.children + .iter() + .any(|child| child.supports_doc_local_confirmation_pruning()) + } + + fn matches(&mut self) -> Result { + self.ensure_confirmed() + } + + fn match_cost(&self) -> Option { + self.children + .iter() + .filter_map(|child| child.match_cost()) + .reduce(|left, right| left + right) + } + + fn scores_non_negative(&self) -> bool { + true + } +} diff --git a/rust/lance-index/src/scalar/inverted/cross_column.rs b/rust/lance-index/src/scalar/inverted/cross_column.rs new file mode 100644 index 00000000000..b5c6dece8a0 --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/cross_column.rs @@ -0,0 +1,2147 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Posting-backed compound FTS over more than one indexed column. +//! +//! A partition-local FTS scorer iterates `DocId`s, but `DocId` is only stable +//! within one partition of one column. This module maps every leaf source to +//! the dataset row-address domain before composing the query. Consequently, +//! columns may have different segment and partition boundaries without an +//! intermediate hash join or a materialized result set per query node. + +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use futures::{StreamExt, TryStreamExt, stream}; +use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; +use lance_core::{Error, Result}; +use lance_select::RowAddrMask; +use roaring::{RoaringBitmap, RoaringTreemap}; + +use super::compound::{ + BoxScorer, ComposableScorer, CompoundLeafPlanInput, CompoundPlanAnalysis, CompoundScorerPlan, + EmptyScorer, LeafQuery, MaterializedScorer, RowAddressMergeScorer, RowAddressSource, + ScoreBounds, ScoredRow, TopKCollector, collect_leaf_queries, map_scorer_to_row_addresses, + prepare_row_address_projection, tokenize_leaf, +}; +use super::documents::{ + DocId, DocLengths, DocVisibility, PartitionDocuments, ResidentAddressProjection, +}; +use super::index::{InvertedPartition, PostingLoadOptions}; +use super::query::{FtsQuery, FtsSearchParams, Operator}; +use super::scorer::MemBM25Scorer; +use super::wand::{ + FLAT_SEARCH_PERCENT_THRESHOLD, FlatDocuments, PostingIterator, WandCursor, WandDocuments, +}; +use super::{ + DocInfo, DocumentGranularity, InvertedIndex, PreparedBm25Query, final_query_tokens, + has_all_query_positions, +}; +use crate::metrics::MetricsCollector; +use crate::prefilter::PreFilter; + +const MAX_CONCURRENT_SOURCE_LOADS: usize = 32; + +/// Stage optional/prohibited source I/O only when the positive generator is +/// expected to match at most this percentage of its column corpus. +const MAX_STAGED_GENERATOR_PERCENT: usize = 1; + +/// Very small candidate sets are cheap enough to stage regardless of corpus +/// size and keep the path useful for small shards. +const MIN_STAGED_GENERATOR_CANDIDATES: usize = 128; + +/// Bound the row-address candidate buffer even for very large corpora. +const MAX_STAGED_GENERATOR_CANDIDATES: usize = 1_000_000; + +struct PreparedCrossColumnLeaf { + column_ordinal: usize, + query: Arc, + params: Arc, + operator: Operator, +} + +struct LoadedCrossColumnLeaf { + leaf_ordinal: usize, + postings: Vec, + params: Arc, + operator: Operator, + scorer: Arc, +} + +fn local_candidate_lower_bound(operator: Operator, postings: &[PostingIterator]) -> Option { + match operator { + Operator::Or => postings + .iter() + .filter_map(PostingIterator::current_doc_id) + .min(), + Operator::And => postings + .iter() + .map(PostingIterator::current_doc_id) + .try_fold(0, |lower_bound, doc| doc.map(|doc| lower_bound.max(doc))), + } +} + +fn compare_leaf_mapping_priority( + left_leaf_ordinal: usize, + left_cost: usize, + right_leaf_ordinal: usize, + right_cost: usize, +) -> Ordering { + right_cost + .cmp(&left_cost) + .then_with(|| left_leaf_ordinal.cmp(&right_leaf_ordinal)) +} + +struct LoadedCrossColumnSource { + num_docs: usize, + lengths: LoadedScoringLengths, + visibility: DocVisibility, + projection: ResidentAddressProjection, + leaves: Vec, +} + +enum LoadedScoringLengths { + Dense(Arc), + Sparse(Vec), +} + +#[derive(Clone, Copy)] +struct ResolvedCandidateDocument { + doc_id: u32, + row_address: u64, + scoring_length: u32, +} + +struct LoadedGeneratorSource { + documents: Arc, + visibility: DocVisibility, + leaves: Vec, +} + +/// Document view used by the two CPU phases of staged generation. +/// +/// The membership pass supplies no lengths and runs with a negative WAND +/// floor, so every exact match remains visible. The scoring pass supplies +/// lengths only for selected generator candidates; non-selected posting docs +/// use zero solely while they are advanced past and can never escape through +/// `document_key`. +#[derive(Clone, Copy)] +enum ScoringLengths<'a> { + Missing, + Dense(&'a DocLengths), + Sparse(&'a [ResolvedCandidateDocument]), +} + +struct StagedWandDocuments<'a> { + num_docs: usize, + visibility: &'a DocVisibility, + scoring_lengths: ScoringLengths<'a>, +} + +impl StagedWandDocuments<'_> { + fn scoring_length(&self, doc_id: u32) -> u32 { + match self.scoring_lengths { + ScoringLengths::Missing => 0, + ScoringLengths::Dense(lengths) => lengths.scoring(DocId::new(doc_id)), + ScoringLengths::Sparse(documents) => documents + .binary_search_by_key(&doc_id, |document| document.doc_id) + .ok() + .map(|index| documents[index].scoring_length) + .unwrap_or_default(), + } + } + + fn row_address(&self, doc_id: u32) -> Option { + let ScoringLengths::Sparse(documents) = self.scoring_lengths else { + return None; + }; + documents + .binary_search_by_key(&doc_id, |document| document.doc_id) + .ok() + .map(|index| documents[index].row_address) + } +} + +impl WandDocuments for StagedWandDocuments<'_> { + type Candidate = DocId; + + fn len(&self) -> usize { + self.num_docs + } + + fn visible_cost_upper_bound(&self) -> usize { + self.visibility.len(self.num_docs) + } + + fn scoring_norms(&self) -> Option<&[u8]> { + match self.scoring_lengths { + ScoringLengths::Dense(lengths) => lengths.scoring_norms(), + ScoringLengths::Missing | ScoringLengths::Sparse(_) => None, + } + } + + fn scoring_num_tokens(&self, doc_id: u32) -> u32 { + self.scoring_length(doc_id) + } + + fn doc_length(&self, doc: &DocInfo) -> u32 { + match doc { + DocInfo::Raw(doc) => self.scoring_length(doc.doc_id), + DocInfo::Located(_) => unreachable!("modern posting lists contain dense DocIds"), + } + } + + fn document_key(&self, doc: &DocInfo) -> Option { + match doc { + DocInfo::Raw(doc) if self.visibility.selected(DocId::new(doc.doc_id)) => { + Some(u64::from(doc.doc_id)) + } + DocInfo::Raw(_) => None, + DocInfo::Located(_) => unreachable!("modern posting lists contain dense DocIds"), + } + } + + fn document_key_for_doc_id(&self, doc_id: u32) -> Option { + self.visibility + .selected(DocId::new(doc_id)) + .then_some(u64::from(doc_id)) + } + + fn candidate_from_key(&self, key: u64) -> Self::Candidate { + DocId::new(key as u32) + } + + fn flat_documents(&self) -> Option> { + if matches!(self.scoring_lengths, ScoringLengths::Missing) { + return None; + } + self.visibility.iter().map(|doc_ids| { + let len = self.visibility.len(self.num_docs); + let docs = doc_ids.map(|doc_id| { + let value = u64::from(doc_id.get()); + (value, value) + }); + (len, Box::new(docs) as Box>) + }) + } + + fn flat_doc_length(&self, doc_id: u64, _document_key: u64, _compressed: bool) -> u32 { + u32::try_from(doc_id) + .ok() + .map(|doc_id| self.scoring_length(doc_id)) + .unwrap_or_default() + } +} + +#[derive(Clone)] +struct SourceDescriptor { + column_ordinal: usize, + segment_ordinal: usize, + partition: Arc, +} + +fn staged_candidate_budget(num_docs: usize, limit: usize) -> usize { + let percentage_budget = num_docs + .saturating_mul(MAX_STAGED_GENERATOR_PERCENT) + .div_ceil(100); + percentage_budget.max(limit).clamp( + MIN_STAGED_GENERATOR_CANDIDATES, + MAX_STAGED_GENERATOR_CANDIDATES, + ) +} + +fn leaf_plan_input(leaf: &PreparedCrossColumnLeaf) -> Result { + let mut seen_terms = HashSet::<(u32, String)>::new(); + let mut costs_by_position = HashMap::::new(); + let tokens = leaf.query.tokens(); + for token_index in 0..tokens.len() { + let position = tokens.position(token_index); + let token = tokens.get_token(token_index); + if seen_terms.insert((position, token.to_owned())) { + let frequency = leaf.query.scorer().num_docs_containing_token(token); + let position_cost = costs_by_position.entry(position).or_default(); + *position_cost = position_cost.saturating_add(frequency); + } + } + + let requires_every_position = + leaf.operator == Operator::And || leaf.params.phrase_slop.is_some(); + let possible = (!requires_every_position || leaf.query.has_all_query_positions()) + && !costs_by_position.is_empty() + && if requires_every_position { + costs_by_position.values().all(|cost| *cost > 0) + } else { + costs_by_position.values().any(|cost| *cost > 0) + }; + if !possible { + return Ok(CompoundLeafPlanInput::new(false, 0, ScoreBounds::ZERO)); + } + + // Position alternatives can overlap, so this is deliberately an upper + // estimate. Overestimating only disables staging; the actual candidate + // count is guarded independently before any probe source is skipped. + let cost = if requires_every_position { + costs_by_position.values().copied().min().unwrap_or(0) + } else { + costs_by_position + .values() + .copied() + .fold(0, usize::saturating_add) + } + .min(leaf.query.scorer().num_docs()); + Ok(CompoundLeafPlanInput::new( + true, + cost, + ScoreBounds::try_new(0.0, f32::INFINITY)?, + )) +} + +fn staged_generator( + analysis: &CompoundPlanAnalysis, + inputs: &[CompoundLeafPlanInput], + leaves: &[PreparedCrossColumnLeaf], + limit: usize, +) -> Option<(Vec, usize)> { + if analysis.generator_leaves.is_empty() { + return None; + } + let generator_leaves = analysis + .generator_leaves + .iter() + .copied() + .collect::>(); + if !inputs + .iter() + .enumerate() + .any(|(leaf_ordinal, input)| input.possible && !generator_leaves.contains(&leaf_ordinal)) + { + return None; + } + let num_docs = analysis + .generator_leaves + .iter() + .map(|&leaf_ordinal| { + leaves + .get(leaf_ordinal) + .map(|leaf| leaf.query.scorer().num_docs()) + }) + .collect::>>()? + .into_iter() + .max() + .unwrap_or(0); + let candidate_budget = staged_candidate_budget(num_docs, limit); + (analysis.generator_cost <= candidate_budget) + .then_some((analysis.generator_leaves.clone(), candidate_budget)) +} + +fn query_state_is_prewarmed( + columns: &[(String, Vec>)], + leaves: &[PreparedCrossColumnLeaf], +) -> bool { + columns + .iter() + .enumerate() + .all(|(column_ordinal, (_, indices))| { + let with_position = leaves.iter().any(|leaf| { + leaf.column_ordinal == column_ordinal && leaf.params.phrase_slop.is_some() + }); + indices + .iter() + .all(|index| index.prewarmed_query_state_ready(with_position)) + }) +} + +fn leaf_column(leaf: &LeafQuery) -> Option<&str> { + match leaf { + LeafQuery::Match(query) => query.column.as_deref(), + LeafQuery::Phrase(query) => query.column.as_deref(), + } +} + +fn validate_row_leaf_granularities(leaves: &[LeafQuery]) -> Result<()> { + for (leaf_ordinal, leaf) in leaves.iter().enumerate() { + let (leaf_kind, column, document_granularity) = match leaf { + LeafQuery::Match(query) => { + ("Match", query.column.as_deref(), query.document_granularity) + } + LeafQuery::Phrase(query) => ( + "Phrase", + query.column.as_deref(), + query.document_granularity, + ), + }; + if document_granularity == Some(DocumentGranularity::ListElement) { + let column = column.unwrap_or(""); + return Err(Error::invalid_input(format!( + "cross-column compound FTS {leaf_kind} leaf {leaf_ordinal} for column '{column}' requested ListElement document granularity, but only Row is supported" + ))); + } + } + Ok(()) +} + +/// Validate the query-local column domain and return the input column ordinal +/// for every query leaf. Keeping this separate from index validation makes it +/// possible to test plan/leaf alignment without constructing index fixtures. +fn resolve_leaf_columns(column_names: &[String], leaves: &[LeafQuery]) -> Result> { + if column_names.len() < 2 { + return Err(Error::invalid_input( + "cross-column compound FTS requires at least two columns", + )); + } + + let mut columns_by_name = HashMap::with_capacity(column_names.len()); + for (column_ordinal, column) in column_names.iter().enumerate() { + if column.is_empty() { + return Err(Error::invalid_input( + "cross-column compound FTS column names cannot be empty", + )); + } + if columns_by_name + .insert(column.as_str(), column_ordinal) + .is_some() + { + return Err(Error::invalid_input(format!( + "cross-column compound FTS received duplicate column '{column}'" + ))); + } + } + + let mut referenced_columns = HashSet::with_capacity(column_names.len()); + let mut leaf_columns = Vec::with_capacity(leaves.len()); + for (leaf_ordinal, leaf) in leaves.iter().enumerate() { + let column = leaf_column(leaf).ok_or_else(|| { + Error::invalid_input(format!( + "cross-column compound FTS leaf {leaf_ordinal} is missing a column" + )) + })?; + let column_ordinal = columns_by_name.get(column).copied().ok_or_else(|| { + Error::invalid_input(format!( + "cross-column compound FTS leaf {leaf_ordinal} references column '{column}', which has no supplied index" + )) + })?; + referenced_columns.insert(column_ordinal); + leaf_columns.push(column_ordinal); + } + + if referenced_columns.len() != column_names.len() { + let unused = column_names + .iter() + .enumerate() + .filter(|(ordinal, _)| !referenced_columns.contains(ordinal)) + .map(|(_, column)| column.as_str()) + .collect::>(); + return Err(Error::invalid_input(format!( + "cross-column compound FTS received indices for unreferenced columns: {}", + unused.join(", ") + ))); + } + + Ok(leaf_columns) +} + +fn validate_modern_row_indices(columns: &[(String, Vec>)]) -> Result<()> { + for (column, indices) in columns { + if indices.is_empty() { + return Err(Error::invalid_input(format!( + "cross-column compound FTS column '{column}' has no index segments" + ))); + } + for (segment_ordinal, index) in indices.iter().enumerate() { + if index.is_legacy() { + return Err(Error::invalid_input(format!( + "cross-column compound FTS requires modern indices, but column '{column}' segment {segment_ordinal} is legacy" + ))); + } + for (partition_ordinal, partition) in index.partitions.iter().enumerate() { + if partition.docs.modern().is_none() { + return Err(Error::invalid_input(format!( + "cross-column compound FTS requires modern documents, but column '{column}' segment {segment_ordinal} partition {partition_ordinal} is legacy" + ))); + } + if partition.docs.coordinate_rank() != 0 { + return Err(Error::invalid_input(format!( + "cross-column compound FTS only supports row documents, but column '{column}' segment {segment_ordinal} partition {partition_ordinal} has coordinate rank {}", + partition.docs.coordinate_rank() + ))); + } + } + } + } + Ok(()) +} + +async fn build_column_scorer( + indices: &[Arc], + terms: Vec, + metrics: Arc, +) -> Result> { + let terms = Arc::new(terms); + let parallelism = get_num_compute_intensive_cpus() + .clamp(1, MAX_CONCURRENT_SOURCE_LOADS) + .min(indices.len().max(1)); + let stats = stream::iter(indices.iter().cloned().map(|index| { + let terms = terms.clone(); + let metrics = metrics.clone(); + async move { + index + .bm25_stats_for_terms(terms.as_ref(), Some(metrics.as_ref())) + .await + } + })) + .buffer_unordered(parallelism) + .try_collect::>() + .await?; + + let mut total_tokens = 0_u64; + let mut num_docs = 0_usize; + let mut term_doc_freqs = vec![0_usize; terms.len()]; + for (segment_total_tokens, segment_num_docs, segment_term_doc_freqs) in stats { + if segment_term_doc_freqs.len() != terms.len() { + return Err(Error::internal(format!( + "FTS segment returned {} document frequencies for {} requested terms", + segment_term_doc_freqs.len(), + terms.len() + ))); + } + total_tokens = total_tokens + .checked_add(segment_total_tokens) + .ok_or_else(|| Error::index("cross-column FTS corpus token count overflows u64"))?; + num_docs = num_docs.checked_add(segment_num_docs).ok_or_else(|| { + Error::index("cross-column FTS corpus document count overflows usize") + })?; + for (total, segment) in term_doc_freqs.iter_mut().zip(segment_term_doc_freqs) { + *total = total.checked_add(segment).ok_or_else(|| { + Error::index("cross-column FTS term document frequency overflows usize") + })?; + } + } + + let token_docs = terms + .iter() + .cloned() + .zip(term_doc_freqs) + .collect::>(); + Ok(Arc::new(MemBM25Scorer::new( + total_tokens, + num_docs, + token_docs, + ))) +} + +async fn prepare_column_leaves( + column_ordinal: usize, + indices: &[Arc], + leaf_queries: &[(usize, LeafQuery)], + params: &FtsSearchParams, + metrics: Arc, +) -> Result> { + let first_index = indices.first().ok_or_else(|| { + Error::invalid_input("cross-column compound FTS requires at least one index segment") + })?; + let mut leaf_metadata = Vec::with_capacity(leaf_queries.len()); + let mut union_terms = Vec::new(); + let mut seen_terms = HashSet::new(); + + for (leaf_ordinal, leaf) in leaf_queries { + let effective_params = leaf.effective_params(params); + let tokens = tokenize_leaf(first_index, leaf, &effective_params); + let final_tokens = Arc::new(final_query_tokens(indices, &tokens, &effective_params)?); + for token in final_tokens.as_ref() { + if seen_terms.insert(token.clone()) { + union_terms.push(token.clone()); + } + } + let has_all_positions = has_all_query_positions(&tokens, final_tokens.as_ref()); + leaf_metadata.push(( + *leaf_ordinal, + final_tokens, + has_all_positions, + Arc::new(effective_params), + leaf.operator(), + )); + } + + // One union-term scorer per column preserves the existing metadata-I/O + // boundary while every leaf keeps its own canonical vocabulary budget. + let scorer = build_column_scorer(indices, union_terms, metrics).await?; + Ok(leaf_metadata + .into_iter() + .map( + |(leaf_ordinal, tokens, has_all_positions, params, operator)| { + ( + leaf_ordinal, + PreparedCrossColumnLeaf { + column_ordinal, + query: Arc::new(PreparedBm25Query::from_parts( + tokens, + scorer.clone(), + has_all_positions, + )), + params, + operator, + }, + ) + }, + ) + .collect()) +} + +fn viable_leaf_ordinals( + column_ordinal: usize, + _segment_ordinal: usize, + partition: &InvertedPartition, + prepared_leaves: &[PreparedCrossColumnLeaf], + leaf_ordinals: &[usize], +) -> Result> { + let mut viable_leaf_ordinals = Vec::with_capacity(leaf_ordinals.len()); + for &leaf_ordinal in leaf_ordinals { + let leaf = prepared_leaves.get(leaf_ordinal).ok_or_else(|| { + Error::internal(format!( + "cross-column FTS source references missing leaf {leaf_ordinal}" + )) + })?; + if leaf.column_ordinal != column_ordinal { + return Err(Error::internal(format!( + "cross-column FTS leaf {leaf_ordinal} belongs to column {}, not {column_ordinal}", + leaf.column_ordinal + ))); + } + let tokens = leaf.query.tokens(); + if partition.may_match_tokens( + tokens.as_ref(), + leaf.operator, + leaf.params.phrase_slop.is_some(), + ) { + viable_leaf_ordinals.push(leaf_ordinal); + } + } + Ok(viable_leaf_ordinals) +} + +async fn load_source_leaves( + partition: Arc, + _segment_ordinal: usize, + viable_leaf_ordinals: Vec, + prepared_leaves: Arc>, + metrics: Arc, +) -> Result> { + let leaf_parallelism = partition + .store() + .io_parallelism() + .max(1) + .min(viable_leaf_ordinals.len()); + let leaves = stream::iter(viable_leaf_ordinals.into_iter().map(|leaf_ordinal| { + let partition = partition.clone(); + let prepared_leaves = prepared_leaves.clone(); + let metrics = metrics.clone(); + async move { + let leaf = prepared_leaves.get(leaf_ordinal).ok_or_else(|| { + Error::internal(format!( + "cross-column FTS source references missing leaf {leaf_ordinal}" + )) + })?; + let tokens = leaf.query.tokens(); + let postings = if tokens.is_empty() + || ((leaf.operator == Operator::And || leaf.params.phrase_slop.is_some()) + && !leaf.query.has_all_query_positions()) + { + Vec::new() + } else { + partition + .load_posting_lists_with_policy( + tokens.as_ref(), + leaf.params.as_ref(), + leaf.operator, + leaf.query.scorer().as_ref(), + metrics.as_ref(), + PostingLoadOptions::cache_aware_exact(true), + ) + .await? + .postings + }; + Result::Ok(LoadedCrossColumnLeaf { + leaf_ordinal, + postings, + params: leaf.params.clone(), + operator: leaf.operator, + scorer: leaf.query.scorer().clone(), + }) + } + })) + .buffer_unordered(leaf_parallelism) + .try_collect::>() + .await? + .into_iter() + .filter(|leaf| !leaf.postings.is_empty()) + .collect::>(); + Ok(leaves) +} + +async fn source_visibility( + documents: &Arc, + mask: Arc, +) -> Result { + let materialize_selected = mask.max_len().is_some_and(|selected| { + u128::from(selected).saturating_mul(100) + <= u128::from(*FLAT_SEARCH_PERCENT_THRESHOLD).saturating_mul(documents.len() as u128) + }); + match documents.immediate_visibility(mask.clone(), materialize_selected) { + Some(visibility) => Ok(visibility), + None => documents.visibility(mask, materialize_selected).await, + } +} + +async fn load_masked_cross_column_source_for_leaves( + descriptor: SourceDescriptor, + prepared_leaves: Arc>, + leaf_ordinals: Vec, + mask: Arc, + metrics: Arc, +) -> Result> { + let SourceDescriptor { + column_ordinal, + segment_ordinal, + partition, + } = descriptor; + let viable_leaf_ordinals = viable_leaf_ordinals( + column_ordinal, + segment_ordinal, + partition.as_ref(), + prepared_leaves.as_ref(), + &leaf_ordinals, + )?; + if viable_leaf_ordinals.is_empty() { + return Ok(None); + } + + let documents = partition.docs.modern().cloned().ok_or_else(|| { + Error::internal("cross-column FTS source changed from modern to legacy documents") + })?; + let visibility = source_visibility(&documents, mask).await?; + if visibility.is_empty() { + return Ok(None); + } + + // Visibility is resolved before posting reads so a filtered-out source + // never touches its posting payloads. + let leaves = load_source_leaves( + partition, + segment_ordinal, + viable_leaf_ordinals, + prepared_leaves, + metrics, + ) + .await?; + if leaves.is_empty() { + return Ok(None); + } + + // Only sources with at least one matching posting need scoring lengths or + // row-address projection. These independent document columns load once and + // in parallel. + let lengths = async { + match documents.cached_lengths() { + Some(lengths) => Ok(lengths), + None => documents.lengths().await, + } + }; + let projection = documents.address_projection(); + let (lengths, projection) = futures::try_join!(lengths, projection)?; + + Ok(Some(LoadedCrossColumnSource { + num_docs: documents.len(), + lengths: LoadedScoringLengths::Dense(lengths), + visibility, + projection, + leaves, + })) +} + +async fn load_masked_generator_source( + descriptor: SourceDescriptor, + prepared_leaves: Arc>, + leaf_ordinals: Vec, + mask: Arc, + metrics: Arc, +) -> Result> { + let SourceDescriptor { + column_ordinal, + segment_ordinal, + partition, + } = descriptor; + let viable_leaf_ordinals = viable_leaf_ordinals( + column_ordinal, + segment_ordinal, + partition.as_ref(), + prepared_leaves.as_ref(), + &leaf_ordinals, + )?; + if viable_leaf_ordinals.is_empty() { + return Ok(None); + } + + let documents = partition.docs.modern().cloned().ok_or_else(|| { + Error::internal("cross-column FTS source changed from modern to legacy documents") + })?; + let visibility = source_visibility(&documents, mask).await?; + if visibility.is_empty() { + return Ok(None); + } + let leaves = load_source_leaves( + partition, + segment_ordinal, + viable_leaf_ordinals, + prepared_leaves, + metrics, + ) + .await?; + if leaves.is_empty() { + return Ok(None); + } + Ok(Some(LoadedGeneratorSource { + documents, + visibility, + leaves, + })) +} + +async fn load_masked_cross_column_source( + descriptor: SourceDescriptor, + prepared_leaves: Arc>, + leaves_by_column: Arc>>, + mask: Arc, + metrics: Arc, +) -> Result> { + let leaf_ordinals = leaves_by_column + .get(descriptor.column_ordinal) + .cloned() + .ok_or_else(|| Error::internal("cross-column FTS source references a missing column"))?; + load_masked_cross_column_source_for_leaves( + descriptor, + prepared_leaves, + leaf_ordinals, + mask, + metrics, + ) + .await +} + +async fn load_candidate_cross_column_source( + descriptor: SourceDescriptor, + prepared_leaves: Arc>, + leaves_by_column: Arc>>, + candidates: Arc>, + metrics: Arc, +) -> Result> { + let SourceDescriptor { + column_ordinal, + segment_ordinal, + partition, + } = descriptor; + let leaf_ordinals = leaves_by_column + .get(column_ordinal) + .ok_or_else(|| Error::internal("cross-column FTS source references a missing column"))?; + let viable_leaf_ordinals = viable_leaf_ordinals( + column_ordinal, + segment_ordinal, + partition.as_ref(), + prepared_leaves.as_ref(), + leaf_ordinals, + )?; + if viable_leaf_ordinals.is_empty() { + return Ok(None); + } + + let documents = partition.docs.modern().cloned().ok_or_else(|| { + Error::internal("cross-column FTS source changed from modern to legacy documents") + })?; + let projection = documents.address_projection().await?; + let selected = projection + .select_sorted_addresses(candidates.as_slice()) + .await?; + let candidate_doc_ids = selected.iter().map(DocId::new).collect::>(); + let visibility = DocVisibility::Selected(selected); + if visibility.is_empty() { + return Ok(None); + } + + // Candidate projection is exact in this source's local DocId domain. Only + // a non-empty intersection is allowed to trigger posting or length I/O. + let leaves = load_source_leaves( + partition, + segment_ordinal, + viable_leaf_ordinals, + prepared_leaves, + metrics, + ) + .await?; + if leaves.is_empty() { + return Ok(None); + } + let lengths = match documents.cached_lengths() { + Some(lengths) => LoadedScoringLengths::Dense(lengths), + None if documents.prefer_sparse_document_read(candidate_doc_ids.len()) => { + let resolved = documents + .resolve_scoring_documents(&candidate_doc_ids) + .await? + .into_iter() + .map( + |(doc_id, row_address, scoring_length)| ResolvedCandidateDocument { + doc_id, + row_address, + scoring_length, + }, + ) + .collect(); + LoadedScoringLengths::Sparse(resolved) + } + None => LoadedScoringLengths::Dense(documents.lengths().await?), + }; + + Ok(Some(LoadedCrossColumnSource { + num_docs: documents.len(), + lengths, + visibility, + projection, + leaves, + })) +} + +struct StagedGeneratorCandidates { + addresses: Vec, + materialized_leaves: Vec<(usize, Vec)>, +} + +struct LocalGeneratorLeaf { + leaf_ordinal: usize, + postings: Vec, + params: Arc, + operator: Operator, + scorer: Arc, + candidate_docs: RoaringBitmap, +} + +struct LocalGeneratorCandidates { + documents: Arc, + leaves: Vec, + candidate_docs: Vec, +} + +struct ResolvedGeneratorCandidates { + num_docs: usize, + leaves: Vec, + /// Sorted by local DocId. + documents: Vec, +} + +fn collect_local_generator_candidates( + sources: Vec, + generator_leaf_ordinals: Vec, + max_candidates: usize, + metrics: Arc, +) -> Result>> { + let mut materialized_row_count = 0_usize; + let generator_leaf_set = generator_leaf_ordinals + .iter() + .copied() + .collect::>(); + let mut collected = Vec::with_capacity(sources.len()); + for source in sources { + let documents = StagedWandDocuments { + num_docs: source.documents.len(), + visibility: &source.visibility, + scoring_lengths: ScoringLengths::Missing, + }; + let mut candidate_docs = RoaringBitmap::new(); + let mut collected_leaves = Vec::with_capacity(source.leaves.len()); + for leaf in source.leaves { + if !generator_leaf_set.contains(&leaf.leaf_ordinal) { + return Err(Error::internal(format!( + "staged cross-column FTS loaded non-generator leaf {}", + leaf.leaf_ordinal + ))); + } + let Some(local_document_lower_bound) = + local_candidate_lower_bound(leaf.operator, &leaf.postings) + else { + continue; + }; + let score_postings = leaf + .postings + .iter() + .map(PostingIterator::fork_from_start) + .collect::>(); + let mut local_scorer = WandCursor::new( + leaf.operator, + leaf.postings, + &documents, + leaf.scorer.clone(), + leaf.params.as_ref(), + metrics.as_ref(), + ); + let mut leaf_candidate_docs = RoaringBitmap::new(); + let mut candidate = local_scorer.advance(local_document_lower_bound)?; + while let Some(local_doc) = candidate { + if local_scorer.matches()? { + let local_doc = u32::try_from(local_doc).map_err(|_| { + Error::index(format!( + "staged cross-column FTS local document {local_doc} exceeds the modern u32 domain" + )) + })?; + if leaf_candidate_docs.insert(local_doc) { + materialized_row_count = materialized_row_count.saturating_add(1); + } + candidate_docs.insert(local_doc); + if materialized_row_count > max_candidates { + // A partial candidate set is never used. Returning + // None makes the async caller run the complete eager + // execution path. + return Ok(None); + } + } + candidate = local_scorer.next()?; + } + if !leaf_candidate_docs.is_empty() { + collected_leaves.push(LocalGeneratorLeaf { + leaf_ordinal: leaf.leaf_ordinal, + postings: score_postings, + params: leaf.params, + operator: leaf.operator, + scorer: leaf.scorer, + candidate_docs: leaf_candidate_docs, + }); + } + } + if !candidate_docs.is_empty() { + collected.push(LocalGeneratorCandidates { + documents: source.documents, + leaves: collected_leaves, + candidate_docs: candidate_docs.iter().map(DocId::new).collect(), + }); + } + } + Ok(Some(collected)) +} + +async fn resolve_generator_candidates( + source: LocalGeneratorCandidates, +) -> Result { + let documents = source + .documents + .resolve_scoring_documents(&source.candidate_docs) + .await? + .into_iter() + .map( + |(doc_id, row_address, scoring_length)| ResolvedCandidateDocument { + doc_id, + row_address, + scoring_length, + }, + ) + .collect(); + Ok(ResolvedGeneratorCandidates { + num_docs: source.documents.len(), + leaves: source.leaves, + documents, + }) +} + +fn score_generator_candidates( + sources: Vec, + generator_leaf_ordinals: Vec, + max_candidates: usize, + metrics: Arc, +) -> Result> { + let mut addresses = RoaringTreemap::new(); + let mut materialized_row_count = 0_usize; + let mut scored_rows_by_leaf = generator_leaf_ordinals + .iter() + .map(|&leaf_ordinal| { + ( + leaf_ordinal, + ( + RoaringTreemap::new(), + Vec::with_capacity(max_candidates.min(MIN_STAGED_GENERATOR_CANDIDATES)), + ), + ) + }) + .collect::>(); + + for source in sources { + let mut source_address_owners = HashMap::::new(); + for leaf in source.leaves { + let visibility = DocVisibility::Selected(leaf.candidate_docs.clone()); + let documents = StagedWandDocuments { + num_docs: source.num_docs, + visibility: &visibility, + scoring_lengths: ScoringLengths::Sparse(&source.documents), + }; + let expected_matches = leaf.candidate_docs.len(); + let mut actual_matches = 0_u64; + let mut local_scorer = WandCursor::new( + leaf.operator, + leaf.postings, + &documents, + leaf.scorer, + leaf.params.as_ref(), + metrics.as_ref(), + ); + for local_doc in leaf.candidate_docs.iter() { + let positioned = local_scorer.advance(u64::from(local_doc))?; + if positioned != Some(u64::from(local_doc)) || !local_scorer.matches()? { + return Err(Error::internal(format!( + "staged cross-column FTS could not reproduce generator leaf {} candidate {local_doc}", + leaf.leaf_ordinal + ))); + } + let row_address = documents.row_address(local_doc).ok_or_else(|| { + Error::internal(format!( + "staged cross-column FTS did not resolve generator local document {local_doc}" + )) + })?; + if let Some(existing_doc) = source_address_owners.insert(row_address, local_doc) + && existing_doc != local_doc + { + return Err(Error::index(format!( + "invalid FTS row-address projection: row address {row_address} is shared by local documents {existing_doc} and {local_doc}" + ))); + } + let (leaf_addresses, scored_rows) = scored_rows_by_leaf + .get_mut(&leaf.leaf_ordinal) + .ok_or_else(|| { + Error::internal(format!( + "staged cross-column FTS lost generator leaf {}", + leaf.leaf_ordinal + )) + })?; + if !leaf_addresses.insert(row_address) { + return Err(Error::internal(format!( + "cross-column FTS generator leaf {} produced duplicate row address {row_address}", + leaf.leaf_ordinal + ))); + } + scored_rows.push(ScoredRow { + row_id: row_address, + score: local_scorer.score()?, + }); + materialized_row_count = materialized_row_count.saturating_add(1); + addresses.insert(row_address); + actual_matches += 1; + if addresses.len() > max_candidates as u64 + || materialized_row_count > max_candidates + { + return Ok(None); + } + } + if actual_matches != expected_matches { + return Err(Error::internal(format!( + "staged cross-column FTS rescored {actual_matches} of {expected_matches} candidates for generator leaf {}", + leaf.leaf_ordinal + ))); + } + } + } + let mut materialized_leaves = scored_rows_by_leaf + .into_iter() + .map(|(leaf_ordinal, (_, mut rows))| { + rows.sort_unstable_by_key(|row| row.row_id); + (leaf_ordinal, rows) + }) + .collect::>(); + materialized_leaves.sort_unstable_by_key(|(leaf_ordinal, _)| *leaf_ordinal); + Ok(Some(StagedGeneratorCandidates { + addresses: addresses.into_iter().collect(), + materialized_leaves, + })) +} + +struct SourceDocuments { + num_docs: usize, + lengths: LoadedScoringLengths, + visibility: DocVisibility, + projection: ResidentAddressProjection, +} + +fn score_cross_column_sources( + sources: Vec, + plan: CompoundScorerPlan, + plan_bounds: ScoreBounds, + num_leaves: usize, + materialized_leaves: Vec<(usize, Vec)>, + limit: usize, + metrics: Arc, +) -> Result<(Vec, Vec)> { + let mut source_documents = Vec::with_capacity(sources.len()); + let mut source_leaves = Vec::with_capacity(sources.len()); + for source in sources { + source_documents.push(SourceDocuments { + num_docs: source.num_docs, + lengths: source.lengths, + visibility: source.visibility, + projection: source.projection, + }); + source_leaves.push(source.leaves); + } + let wand_documents = source_documents + .iter() + .map(|source| StagedWandDocuments { + num_docs: source.num_docs, + visibility: &source.visibility, + scoring_lengths: match &source.lengths { + LoadedScoringLengths::Dense(lengths) => ScoringLengths::Dense(lengths.as_ref()), + LoadedScoringLengths::Sparse(documents) => { + ScoringLengths::Sparse(documents.as_slice()) + } + }, + }) + .collect::>(); + let address_projections = source_documents + .iter() + .map(|source| prepare_row_address_projection(&source.projection)) + .collect::>(); + + let mut sources_by_leaf = (0..num_leaves) + .map(|_| Vec::>::new()) + .collect::>(); + for (source_ordinal, leaves) in source_leaves.into_iter().enumerate() { + let mut local_scorers = Vec::with_capacity(leaves.len()); + for leaf in leaves { + if leaf.postings.is_empty() { + continue; + } + let Some(local_document_lower_bound) = + local_candidate_lower_bound(leaf.operator, &leaf.postings) + else { + continue; + }; + let local_scorer: BoxScorer<'_> = Box::new(WandCursor::new( + leaf.operator, + leaf.postings, + &wand_documents[source_ordinal], + leaf.scorer, + leaf.params.as_ref(), + metrics.as_ref(), + )); + let cost = local_scorer.cost(); + local_scorers.push(( + leaf.leaf_ordinal, + cost, + local_document_lower_bound, + local_scorer, + )); + } + + // A dense leaf validates an unknown projection once and caches the + // ordered result for sparse siblings. Mapping high-cost leaves first + // therefore avoids materializing sparse leaves before that validation. + local_scorers.sort_unstable_by(|left, right| { + compare_leaf_mapping_priority(left.0, left.1, right.0, right.1) + }); + for (leaf_ordinal, _, local_document_lower_bound, local_scorer) in local_scorers { + let Some(address_source) = map_scorer_to_row_addresses( + local_scorer, + &address_projections[source_ordinal], + local_document_lower_bound, + )? + else { + continue; + }; + sources_by_leaf + .get_mut(leaf_ordinal) + .ok_or_else(|| { + Error::internal(format!( + "cross-column FTS loaded unexpected leaf {}", + leaf_ordinal + )) + })? + .push(address_source); + } + } + // Materialized fallbacks retain a source-wide collision map only while + // sibling leaves are being mapped. Scorers no longer borrow this state. + drop(address_projections); + + let mut materialized_leaf_ordinals = vec![false; num_leaves]; + for (leaf_ordinal, _) in &materialized_leaves { + let materialized = materialized_leaf_ordinals + .get_mut(*leaf_ordinal) + .ok_or_else(|| { + Error::internal(format!( + "staged cross-column FTS materialized unexpected leaf {leaf_ordinal}" + )) + })?; + if std::mem::replace(materialized, true) { + return Err(Error::internal(format!( + "staged cross-column FTS materialized leaf {leaf_ordinal} more than once" + ))); + } + } + let mut leaf_scorers = sources_by_leaf + .into_iter() + .enumerate() + .map(|(leaf_ordinal, mut sources)| { + if materialized_leaf_ordinals[leaf_ordinal] { + if !sources.is_empty() { + return Err(Error::internal(format!( + "staged cross-column FTS loaded materialized generator leaf {leaf_ordinal} twice" + ))); + } + return Ok(None); + } + let scorer: BoxScorer<'_> = match sources.len() { + 0 => Box::new(EmptyScorer), + 1 => sources + .pop() + .ok_or_else(|| Error::internal("cross-column FTS lost its only leaf source"))? + .into_scorer(), + _ => Box::new(RowAddressMergeScorer::try_new(sources)?), + }; + Ok(Some(scorer)) + }) + .collect::>>()?; + for (leaf_ordinal, rows) in materialized_leaves { + let slot = leaf_scorers.get_mut(leaf_ordinal).ok_or_else(|| { + Error::internal(format!( + "staged cross-column FTS materialized unexpected leaf {leaf_ordinal}" + )) + })?; + debug_assert!(slot.is_none()); + *slot = Some(Box::new(MaterializedScorer::try_new(rows)?)); + } + let mut scorer = plan.build(&mut leaf_scorers)?; + if leaf_scorers.iter().any(Option::is_some) { + return Err(Error::internal( + "cross-column compound FTS scorer did not consume every prepared leaf", + )); + } + let rows = TopKCollector::new(limit).collect(scorer.as_mut())?; + if let Some(row) = rows.iter().find(|row| !plan_bounds.contains(row.score)) { + return Err(Error::internal(format!( + "cross-column compound FTS score {} for row address {} escaped plan bounds {plan_bounds:?}", + row.score, row.row_id + ))); + } + Ok(rows.into_iter().map(|row| (row.row_id, row.score)).unzip()) +} + +/// Internal cross-crate hook for a bounded compound FTS query over multiple +/// indexed columns. +/// +/// Each leaf is scored with corpus statistics from its own column. Partition +/// scorers are mapped to current row addresses and heap-merged before the +/// Boolean/Boost/MultiMatch tree is composed, so one exact global collector +/// owns both the competitive score and final row-address tie breaking. +/// +/// Every Match and Phrase leaf must omit document granularity or request +/// [`DocumentGranularity::Row`]. `columns` must contain at least two modern, +/// row-document indices and every supplied column must be referenced. `params` +/// must specify a bounded result limit. Invalid query or index shapes return +/// [`Error::InvalidInput`]; list-element requests are rejected before any +/// prefilter or index work begins. +#[doc(hidden)] +pub async fn cross_column_compound_search( + columns: &[(String, Vec>)], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, +) -> Result<(Vec, Vec)> { + let mut leaf_queries = Vec::new(); + collect_leaf_queries(query, &mut leaf_queries)?; + validate_row_leaf_granularities(&leaf_queries)?; + + let limit = params.limit.ok_or_else(|| { + Error::invalid_input("cross-column compound FTS requires a bounded result limit") + })?; + if limit == 0 { + return Ok((Vec::new(), Vec::new())); + } + + let column_names = columns + .iter() + .map(|(column, _)| column.clone()) + .collect::>(); + let leaf_columns = resolve_leaf_columns(&column_names, &leaf_queries)?; + validate_modern_row_indices(columns)?; + + let mut num_plan_leaves = 0; + let plan = CompoundScorerPlan::from_query(query, &mut num_plan_leaves)?; + if num_plan_leaves != leaf_queries.len() { + return Err(Error::internal(format!( + "cross-column compound FTS planned {num_plan_leaves} leaves but prepared {}", + leaf_queries.len() + ))); + } + + // The prefilter owns potentially asynchronous deletion/filter work. It is + // shared by every column but reaches readiness exactly once here. An empty + // filter avoids all token-statistics and posting I/O. + prefilter.wait_for_ready().await?; + let mask = prefilter.mask(); + if mask.max_len() == Some(0) { + return Ok((Vec::new(), Vec::new())); + } + + let mut queries_by_column = (0..columns.len()) + .map(|_| Vec::<(usize, LeafQuery)>::new()) + .collect::>(); + for (leaf_ordinal, (leaf, column_ordinal)) in + leaf_queries.into_iter().zip(leaf_columns).enumerate() + { + queries_by_column[column_ordinal].push((leaf_ordinal, leaf)); + } + + // Own the work items before building the stream. Besides keeping this + // future `Send`, it prevents borrowed iterator closure types from leaking + // into callers that box their execution stream. + let preparation_work = columns + .iter() + .enumerate() + .map(|(column_ordinal, (_, indices))| { + ( + column_ordinal, + indices.clone(), + queries_by_column[column_ordinal].clone(), + ) + }) + .collect::>(); + let preparation_parallelism = get_num_compute_intensive_cpus() + .clamp(1, MAX_CONCURRENT_SOURCE_LOADS) + .min(preparation_work.len()); + let prepared_by_column = stream::iter(preparation_work.into_iter().map( + |(column_ordinal, indices, leaf_queries)| { + let params = params.clone(); + let metrics = metrics.clone(); + async move { + prepare_column_leaves(column_ordinal, &indices, &leaf_queries, ¶ms, metrics) + .await + } + }, + )) + .buffer_unordered(preparation_parallelism) + .try_collect::>() + .await?; + let mut prepared_leaves = (0..num_plan_leaves) + .map(|_| None) + .collect::>>(); + for column_leaves in prepared_by_column { + for (leaf_ordinal, leaf) in column_leaves { + let slot = prepared_leaves.get_mut(leaf_ordinal).ok_or_else(|| { + Error::internal(format!( + "cross-column FTS prepared unexpected leaf {leaf_ordinal}" + )) + })?; + if slot.replace(leaf).is_some() { + return Err(Error::internal(format!( + "cross-column FTS prepared leaf {leaf_ordinal} more than once" + ))); + } + } + } + let prepared_leaves = prepared_leaves + .into_iter() + .enumerate() + .map(|(leaf_ordinal, leaf)| { + leaf.ok_or_else(|| { + Error::internal(format!( + "cross-column FTS did not prepare leaf {leaf_ordinal}" + )) + }) + }) + .collect::>>()?; + let plan_inputs = prepared_leaves + .iter() + .map(leaf_plan_input) + .collect::>>()?; + let plan_analysis = plan.analyze_leaves(&plan_inputs)?; + if !plan_analysis.possible { + return Ok((Vec::new(), Vec::new())); + } + // Staging trades a second bounded CPU pass for deferred cold I/O. Once an + // explicit prewarm has made every selected index query-ready, that trade is + // strictly worse: the original bounded coordinator can consume resident + // postings and document columns directly. The hint is checked without I/O; + // any mixed or uncertain state conservatively keeps the staged cold path. + let staged_generator = (!query_state_is_prewarmed(columns, &prepared_leaves)) + .then(|| staged_generator(&plan_analysis, &plan_inputs, &prepared_leaves, limit)) + .flatten(); + let leaves_by_column = prepared_leaves.iter().enumerate().fold( + (0..columns.len()).map(|_| Vec::new()).collect::>(), + |mut by_column, (leaf_ordinal, leaf)| { + by_column[leaf.column_ordinal].push(leaf_ordinal); + by_column + }, + ); + + let prepared_leaves = Arc::new(prepared_leaves); + let leaves_by_column = Arc::new(leaves_by_column); + let descriptors = columns + .iter() + .enumerate() + .flat_map(|(column_ordinal, (_, indices))| { + indices + .iter() + .enumerate() + .flat_map(move |(segment_ordinal, index)| { + index + .partitions + .iter() + .cloned() + .map(move |partition| SourceDescriptor { + column_ordinal, + segment_ordinal, + partition, + }) + }) + }) + .collect::>(); + let parallelism = get_num_compute_intensive_cpus().clamp(1, MAX_CONCURRENT_SOURCE_LOADS); + let staged_candidates = if let Some((generator_leaf_ordinals, candidate_budget)) = + staged_generator + { + let generator_leaf_set = generator_leaf_ordinals + .iter() + .copied() + .collect::>(); + let generator_leaves_by_column = Arc::new( + leaves_by_column + .iter() + .map(|leaves| { + leaves + .iter() + .copied() + .filter(|leaf| generator_leaf_set.contains(leaf)) + .collect::>() + }) + .collect::>(), + ); + let generator_descriptors = descriptors + .iter() + .filter(|descriptor| { + generator_leaves_by_column + .get(descriptor.column_ordinal) + .is_some_and(|leaves| !leaves.is_empty()) + }) + .cloned() + .collect::>(); + let generator_sources = stream::iter(generator_descriptors.into_iter().map(|descriptor| { + let leaf_ordinals = generator_leaves_by_column + .get(descriptor.column_ordinal) + .cloned() + .unwrap_or_default(); + load_masked_generator_source( + descriptor, + prepared_leaves.clone(), + leaf_ordinals, + mask.clone(), + metrics.clone(), + ) + })) + .buffer_unordered(parallelism) + .try_collect::>() + .await? + .into_iter() + .flatten() + .collect::>(); + let candidate_metrics = metrics.clone(); + let collection_leaf_ordinals = generator_leaf_ordinals.clone(); + let local_candidates = spawn_cpu(move || { + collect_local_generator_candidates( + generator_sources, + collection_leaf_ordinals, + candidate_budget, + candidate_metrics, + ) + }) + .await?; + if let Some(local_candidates) = local_candidates { + let resolved = stream::iter( + local_candidates + .into_iter() + .map(resolve_generator_candidates), + ) + .buffer_unordered(parallelism) + .try_collect::>() + .await?; + let candidate_metrics = metrics.clone(); + spawn_cpu(move || { + score_generator_candidates( + resolved, + generator_leaf_ordinals, + candidate_budget, + candidate_metrics, + ) + }) + .await? + } else { + None + } + } else { + None + }; + + let (sources, materialized_leaves) = if let Some(candidates) = staged_candidates { + if candidates.addresses.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + let StagedGeneratorCandidates { + addresses, + materialized_leaves, + } = candidates; + let generator_leaf_set = materialized_leaves + .iter() + .map(|(leaf_ordinal, _)| *leaf_ordinal) + .collect::>(); + let candidates = Arc::new(addresses); + let deferred_leaves_by_column = Arc::new( + leaves_by_column + .iter() + .map(|leaves| { + leaves + .iter() + .copied() + .filter(|leaf| !generator_leaf_set.contains(leaf)) + .collect::>() + }) + .collect::>(), + ); + let sources = stream::iter(descriptors.into_iter().map(|descriptor| { + load_candidate_cross_column_source( + descriptor, + prepared_leaves.clone(), + deferred_leaves_by_column.clone(), + candidates.clone(), + metrics.clone(), + ) + })) + .buffer_unordered(parallelism) + .try_collect::>() + .await? + .into_iter() + .flatten() + .collect::>(); + (sources, materialized_leaves) + } else { + let sources = stream::iter(descriptors.into_iter().map(|descriptor| { + load_masked_cross_column_source( + descriptor, + prepared_leaves.clone(), + leaves_by_column.clone(), + mask.clone(), + metrics.clone(), + ) + })) + .buffer_unordered(parallelism) + .try_collect::>() + .await? + .into_iter() + .flatten() + .collect::>(); + (sources, Vec::new()) + }; + + spawn_cpu(move || { + score_cross_column_sources( + sources, + plan, + plan_analysis.bounds, + num_plan_leaves, + materialized_leaves, + limit, + metrics, + ) + }) + .await +} + +#[cfg(test)] +mod tests { + use arrow::buffer::ScalarBuffer; + + use super::*; + use crate::metrics::NoOpMetricsCollector; + use crate::prefilter::NoFilter; + use crate::scalar::inverted::encoding::compress_posting_list; + use crate::scalar::inverted::query::{ + BooleanQuery, MatchQuery, MultiMatchQuery, Occur, PhraseQuery, Tokens, + }; + use crate::scalar::inverted::tokenizer::document_tokenizer::DocType; + use crate::scalar::inverted::{ + CompressedPostingList, DocumentGranularity, LEGACY_BLOCK_SIZE, PlainPostingList, + PostingList, PostingTailCodec, + }; + + fn match_query(column: Option<&str>, terms: &str) -> FtsQuery { + FtsQuery::Match( + MatchQuery::new(terms.to_owned()).with_column(column.map(ToOwned::to_owned)), + ) + } + + fn posting(doc_ids: &[u64]) -> PostingIterator { + if doc_ids.is_empty() { + return PostingIterator::new( + "term".to_owned(), + 0, + 0, + PostingList::Plain(PlainPostingList::new( + ScalarBuffer::from(Vec::::new()), + ScalarBuffer::from(Vec::::new()), + Some(0.0), + None, + )), + 100, + ); + } + let doc_ids = doc_ids + .iter() + .map(|&doc_id| u32::try_from(doc_id).unwrap()) + .collect::>(); + let frequencies = vec![1_u32; doc_ids.len()]; + let blocks = compress_posting_list( + doc_ids.len(), + doc_ids.iter(), + frequencies.iter(), + vec![1.0; doc_ids.len()].into_iter(), + ) + .unwrap(); + PostingIterator::new( + "term".to_owned(), + 0, + 0, + PostingList::Compressed(CompressedPostingList::new( + blocks, + 1.0, + doc_ids.len() as u32, + PostingTailCodec::VarintDelta, + LEGACY_BLOCK_SIZE, + None, + None, + )), + 100, + ) + } + + fn prepared_leaf( + tokens_by_segment: Vec, + operator: Operator, + phrase_slop: Option, + num_docs: usize, + token_docs: impl IntoIterator, + ) -> PreparedCrossColumnLeaf { + let mut segment_tokens = tokens_by_segment.into_iter(); + let tokens = Arc::new(segment_tokens.next().unwrap()); + for segment_tokens in segment_tokens { + assert_eq!(segment_tokens.len(), tokens.len()); + for index in 0..segment_tokens.len() { + assert_eq!(segment_tokens.get_token(index), tokens.get_token(index)); + assert_eq!(segment_tokens.position(index), tokens.position(index)); + } + } + let scorer = Arc::new(MemBM25Scorer::new( + num_docs as u64, + num_docs, + token_docs + .into_iter() + .map(|(token, count)| (token.to_owned(), count)) + .collect(), + )); + PreparedCrossColumnLeaf { + column_ordinal: 0, + query: Arc::new(PreparedBm25Query::from_parts(tokens, scorer, true)), + params: Arc::new(FtsSearchParams::new().with_phrase_slop(phrase_slop)), + operator, + } + } + + fn loaded_generator_source( + leaf_ordinal: usize, + addresses: Vec, + matching_docs: &[u64], + ) -> ResolvedGeneratorCandidates { + let num_docs = addresses.len(); + let candidate_docs = matching_docs + .iter() + .map(|&doc_id| u32::try_from(doc_id).unwrap()) + .collect::(); + ResolvedGeneratorCandidates { + num_docs, + documents: candidate_docs + .iter() + .map(|doc_id| ResolvedCandidateDocument { + doc_id, + row_address: addresses[doc_id as usize], + scoring_length: 1, + }) + .collect(), + leaves: vec![LoadedCrossColumnLeaf { + leaf_ordinal, + postings: vec![posting(matching_docs)], + params: Arc::new(FtsSearchParams::new()), + operator: Operator::Or, + scorer: Arc::new(MemBM25Scorer::new( + num_docs as u64, + num_docs, + HashMap::from([("term".to_owned(), matching_docs.len())]), + )), + }] + .into_iter() + .map(|leaf| LocalGeneratorLeaf { + leaf_ordinal: leaf.leaf_ordinal, + postings: leaf.postings, + params: leaf.params, + operator: leaf.operator, + scorer: leaf.scorer, + candidate_docs: candidate_docs.clone(), + }) + .collect(), + } + } + + #[tokio::test] + async fn rejects_list_element_leaves_before_column_or_index_validation() { + let mut multi_match = MultiMatchQuery::try_new( + "term".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap(); + multi_match.match_queries[1].document_granularity = Some(DocumentGranularity::ListElement); + let cases = [ + ( + "Match leaf 0 for column 'title'", + FtsQuery::Match( + MatchQuery::new("term".to_owned()) + .with_column(Some("title".to_owned())) + .with_document_granularity(DocumentGranularity::ListElement), + ), + ), + ( + "Phrase leaf 0 for column 'body'", + FtsQuery::Phrase( + PhraseQuery::new("two terms".to_owned()) + .with_column(Some("body".to_owned())) + .with_document_granularity(DocumentGranularity::ListElement), + ), + ), + ( + "Match leaf 1 for column 'body'", + FtsQuery::MultiMatch(multi_match), + ), + ]; + + for (leaf_context, query) in cases { + let error = cross_column_compound_search( + &[], + &query, + &FtsSearchParams::new().with_limit(Some(10)), + Arc::new(NoFilter), + Arc::new(NoOpMetricsCollector), + ) + .await + .unwrap_err(); + + assert!(matches!(&error, Error::InvalidInput { .. })); + let message = error.to_string(); + assert!( + message.contains(leaf_context), + "unexpected error: {message}" + ); + assert!( + message.contains( + "requested ListElement document granularity, but only Row is supported" + ), + "unexpected error: {message}" + ); + } + } + + #[tokio::test] + async fn permits_unspecified_and_row_leaf_granularity() { + for document_granularity in [None, Some(DocumentGranularity::Row)] { + let mut match_query = + MatchQuery::new("term".to_owned()).with_column(Some("title".to_owned())); + match_query.document_granularity = document_granularity; + let error = cross_column_compound_search( + &[], + &FtsQuery::Match(match_query), + &FtsSearchParams::new().with_limit(Some(10)), + Arc::new(NoFilter), + Arc::new(NoOpMetricsCollector), + ) + .await + .unwrap_err(); + + assert!(matches!(&error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("requires at least two columns")); + } + } + + #[test] + fn derives_conservative_local_candidate_lower_bound() { + let postings = vec![posting(&[10, 20]), posting(&[5, 30])]; + assert_eq!( + local_candidate_lower_bound(Operator::Or, &postings), + Some(5) + ); + assert_eq!( + local_candidate_lower_bound(Operator::And, &postings), + Some(10) + ); + + let postings_with_empty = vec![posting(&[10]), posting(&[])]; + assert_eq!( + local_candidate_lower_bound(Operator::Or, &postings_with_empty), + Some(10) + ); + assert_eq!( + local_candidate_lower_bound(Operator::And, &postings_with_empty), + None + ); + } + + #[test] + fn maps_dense_leaves_before_sparse_siblings() { + let mut leaves = vec![(3, 10), (2, 100), (1, 10), (0, 100)]; + leaves.sort_unstable_by(|left, right| { + compare_leaf_mapping_priority(left.0, left.1, right.0, right.1) + }); + + assert_eq!(leaves, vec![(0, 100), (2, 100), (1, 10), (3, 10)]); + } + + #[test] + fn estimates_generator_cost_by_unique_query_positions() { + let tokens = Tokens::with_positions( + vec![ + "rare".to_owned(), + "rare_alt".to_owned(), + "common".to_owned(), + ], + vec![0, 0, 1], + DocType::Text, + ); + let leaf = prepared_leaf( + vec![tokens.clone(), tokens], + Operator::And, + None, + 10_000, + [("rare", 7), ("rare_alt", 3), ("common", 1_000)], + ); + let input = leaf_plan_input(&leaf).unwrap(); + assert!(input.possible); + assert_eq!(input.cost, 10); + assert_eq!(input.bounds.lower(), 0.0); + assert_eq!(input.bounds.upper(), f32::INFINITY); + + let missing = prepared_leaf( + vec![Tokens::with_positions( + vec!["rare".to_owned(), "missing".to_owned()], + vec![0, 1], + DocType::Text, + )], + Operator::And, + None, + 10_000, + [("rare", 7), ("missing", 0)], + ); + assert!(!leaf_plan_input(&missing).unwrap().possible); + } + + #[test] + fn stages_only_selective_single_leaf_generators_with_probe_work() { + let leaves = vec![ + prepared_leaf( + vec![Tokens::new(vec!["rare".to_owned()], DocType::Text)], + Operator::Or, + None, + 10_000, + [("rare", 100)], + ), + prepared_leaf( + vec![Tokens::new(vec!["optional".to_owned()], DocType::Text)], + Operator::Or, + None, + 10_000, + [("optional", 2_000)], + ), + prepared_leaf( + vec![Tokens::new(vec!["probe".to_owned()], DocType::Text)], + Operator::Or, + None, + 10_000, + [("probe", 500)], + ), + ]; + let inputs = leaves + .iter() + .map(leaf_plan_input) + .collect::>>() + .unwrap(); + let analysis = CompoundPlanAnalysis { + possible: true, + bounds: ScoreBounds::UNBOUNDED, + generator_cost: 100, + generator_leaves: vec![0], + }; + assert_eq!( + staged_generator(&analysis, &inputs, &leaves, 10), + Some((vec![0], 128)) + ); + let no_probe_inputs = vec![inputs[0]]; + assert_eq!( + staged_generator(&analysis, &no_probe_inputs, &leaves[..1], 10), + None + ); + + let multi_generator = CompoundPlanAnalysis { + generator_cost: 100, + generator_leaves: vec![0, 1], + ..analysis + }; + assert_eq!( + staged_generator(&multi_generator, &inputs, &leaves, 10), + Some((vec![0, 1], 128)) + ); + + let too_dense = CompoundPlanAnalysis { + generator_cost: 129, + ..multi_generator + }; + assert_eq!(staged_generator(&too_dense, &inputs, &leaves, 10), None); + } + + #[test] + fn generator_collection_is_complete_and_overflow_falls_back() { + let make_sources = || { + vec![ + loaded_generator_source(1, vec![10, 20, 30], &[0, 2]), + loaded_generator_source(1, vec![40, 50], &[1]), + ] + }; + let metrics: Arc = Arc::new(NoOpMetricsCollector); + let candidates = score_generator_candidates(make_sources(), vec![1], 3, metrics.clone()) + .unwrap() + .unwrap(); + assert_eq!(candidates.addresses, vec![10, 30, 50]); + assert_eq!(candidates.materialized_leaves.len(), 1); + let (leaf_ordinal, scored_rows) = &candidates.materialized_leaves[0]; + assert_eq!(*leaf_ordinal, 1); + assert_eq!( + scored_rows.iter().map(|row| row.row_id).collect::>(), + vec![10, 30, 50] + ); + assert!( + scored_rows + .iter() + .all(|row| row.score.is_finite() && row.score > 0.0) + ); + + assert!( + score_generator_candidates(make_sources(), vec![1], 2, metrics) + .unwrap() + .is_none(), + "overflow must abandon the entire staged candidate set" + ); + } + + #[test] + fn generator_collection_materializes_every_leaf_in_a_union_cover() { + let sources = vec![ + loaded_generator_source(0, vec![10, 20], &[0]), + loaded_generator_source(1, vec![10, 20], &[0, 1]), + ]; + let metrics: Arc = Arc::new(NoOpMetricsCollector); + let candidates = score_generator_candidates(sources, vec![0, 1], 3, metrics) + .unwrap() + .unwrap(); + + assert_eq!(candidates.addresses, vec![10, 20]); + assert_eq!( + candidates + .materialized_leaves + .iter() + .map(|(leaf, rows)| { + (*leaf, rows.iter().map(|row| row.row_id).collect::>()) + }) + .collect::>(), + vec![(0, vec![10]), (1, vec![10, 20])] + ); + } + + #[test] + fn generator_collection_rejects_same_leaf_duplicates_across_sources() { + let sources = vec![ + loaded_generator_source(0, vec![10], &[0]), + loaded_generator_source(0, vec![10], &[0]), + ]; + let metrics: Arc = Arc::new(NoOpMetricsCollector); + + let Err(error) = score_generator_candidates(sources, vec![0], 2, metrics) else { + panic!("duplicate row addresses should be rejected"); + }; + assert!( + error + .to_string() + .contains("generator leaf 0 produced duplicate row address 10") + ); + } + + #[test] + fn resolves_leaf_columns_in_compound_plan_order() { + let query = FtsQuery::Boolean(BooleanQuery::new([ + (Occur::Should, match_query(Some("body"), "optional")), + ( + Occur::Must, + FtsQuery::Phrase( + PhraseQuery::new("required phrase".to_owned()) + .with_column(Some("title".to_owned())), + ), + ), + (Occur::MustNot, match_query(Some("body"), "blocked")), + ])); + let mut leaves = Vec::new(); + collect_leaf_queries(&query, &mut leaves).unwrap(); + + assert_eq!( + resolve_leaf_columns(&["title".to_owned(), "body".to_owned()], &leaves).unwrap(), + vec![1, 0, 1] + ); + let mut num_plan_leaves = 0; + CompoundScorerPlan::from_query(&query, &mut num_plan_leaves).unwrap(); + assert_eq!(num_plan_leaves, leaves.len()); + } + + #[test] + fn rejects_missing_duplicate_and_unreferenced_columns() { + let leaves = vec![LeafQuery::Match( + MatchQuery::new("term".to_owned()).with_column(Some("title".to_owned())), + )]; + + let error = resolve_leaf_columns(&["title".to_owned()], &leaves).unwrap_err(); + assert!(error.to_string().contains("at least two columns")); + + let error = + resolve_leaf_columns(&["title".to_owned(), "title".to_owned()], &leaves).unwrap_err(); + assert!(error.to_string().contains("duplicate column 'title'")); + + let error = + resolve_leaf_columns(&["title".to_owned(), "body".to_owned()], &leaves).unwrap_err(); + assert!(error.to_string().contains("unreferenced columns: body")); + } + + #[test] + fn rejects_leaf_without_a_supplied_column_index() { + let leaves = vec![LeafQuery::Match(MatchQuery::new("term".to_owned()))]; + let error = + resolve_leaf_columns(&["title".to_owned(), "body".to_owned()], &leaves).unwrap_err(); + assert!(error.to_string().contains("leaf 0 is missing a column")); + + let leaves = vec![LeafQuery::Match( + MatchQuery::new("term".to_owned()).with_column(Some("summary".to_owned())), + )]; + let error = + resolve_leaf_columns(&["title".to_owned(), "body".to_owned()], &leaves).unwrap_err(); + assert!(error.to_string().contains("no supplied index")); + } +} diff --git a/rust/lance-index/src/scalar/inverted/documents.rs b/rust/lance-index/src/scalar/inverted/documents.rs new file mode 100644 index 00000000000..f7631355b8b --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/documents.rs @@ -0,0 +1,3355 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Typed document-side state for partitioned FTS indices. +//! +//! Posting lists use dense, partition-local document identifiers. This module +//! keeps that identity separate from dataset-version row addresses so scoring +//! never has to infer which value a numeric slot represents. + +use std::borrow::Cow; +use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering as AtomicOrdering}; +use std::sync::{Arc, OnceLock, Weak}; + +use arc_swap::ArcSwapWeak; +use arrow::buffer::ScalarBuffer; +use arrow_array::{Array, RecordBatch, UInt32Array, UInt64Array}; +use lance_core::cache::{CacheKey, WeakLanceCache}; +use lance_core::deepsize::DeepSizeOf; +use lance_core::utils::address::RowAddress; +use lance_core::utils::tokio::spawn_cpu; +use lance_core::{Error, ROW_ID, Result}; +use lance_select::{RowAddrMask, RowAddrSelection, RowAddrTreeMap}; +use object_store::path::Path; +use roaring::RoaringBitmap; +use tokio::sync::OnceCell; + +use crate::FtsPrewarmDocumentStatus; +use crate::scalar::{IndexReader, IndexStore, RowIdRemapper}; + +use super::index::{ + DocSet, NUM_TOKEN_COL, dequantize_doc_length, doc_index_storage_column, + document_coordinate_rank, quantize_doc_length, +}; + +/// Schema metadata key persisted in every modern `docs.lance` partition. +pub(super) const TOTAL_TOKENS_KEY: &str = "total_tokens"; + +/// Candidate-side document reads stay sparse below this share of a partition. +/// Larger selections amortize one dense column read and populate the reusable +/// document cache for subsequent queries. +const SPARSE_DOCUMENT_READ_PERCENT: usize = 10; + +/// Dense, immutable document identity inside one FTS partition. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(super) struct DocId(u32); + +impl DocId { + pub(crate) fn new(value: u32) -> Self { + Self(value) + } + + pub(crate) fn get(self) -> u32 { + self.0 + } + + fn as_usize(self) -> usize { + self.0 as usize + } +} + +/// Immutable corpus statistics for one partition. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct PartitionStats { + pub(crate) num_docs: usize, + pub(crate) total_tokens: u64, +} + +/// One partition's immutable `DocId -> row address` column. +/// +/// The independently weighed cache entry is the only long-lived owner. The +/// address projection keeps a weak handle and query-local guards upgrade it, +/// so cache eviction releases the column once in-flight queries finish. +#[derive(Debug)] +pub(super) struct CachedDocRowIds { + pub(crate) row_ids: Arc, +} + +impl DeepSizeOf for CachedDocRowIds { + fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { + self.row_ids.len() * std::mem::size_of::() + } +} + +/// Cache key for one partition's [`CachedDocRowIds`]. +#[derive(Debug, Clone)] +pub(super) struct DocRowIdsKey { + pub(crate) partition_id: u64, +} + +impl CacheKey for DocRowIdsKey { + type ValueType = CachedDocRowIds; + + fn key(&self) -> Cow<'_, str> { + format!("doc-row-ids-{}", self.partition_id).into() + } + + fn type_name() -> &'static str { + "DocRowIds" + } +} + +/// Exact document lengths plus the optional quantized scoring representation. +#[derive(Debug)] +pub(super) struct DocLengths { + values: ScalarBuffer, + total_tokens: u64, + quantized_scoring: bool, + norms: OnceLock>, +} + +impl DeepSizeOf for DocLengths { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.values.deep_size_of_children(context) + + self + .norms + .get() + .map(|norms| std::mem::size_of_val(norms.as_ref())) + .unwrap_or(0) + } +} + +impl DocLengths { + fn try_new( + values: ScalarBuffer, + expected_num_docs: usize, + persisted_total_tokens: Option, + quantized_scoring: bool, + path: &str, + ) -> Result { + if values.len() != expected_num_docs { + return Err(corrupt_docs( + path, + format!( + "{NUM_TOKEN_COL} has {} rows but the file footer reports {expected_num_docs}", + values.len() + ), + )); + } + let total_tokens = values.iter().try_fold(0_u64, |total, &value| { + total + .checked_add(u64::from(value)) + .ok_or_else(|| corrupt_docs(path, format!("{NUM_TOKEN_COL} sum overflows u64"))) + })?; + if let Some(expected) = persisted_total_tokens + && expected != total_tokens + { + return Err(corrupt_docs( + path, + format!( + "{TOTAL_TOKENS_KEY} metadata is {expected}, but {NUM_TOKEN_COL} sums to {total_tokens}" + ), + )); + } + Ok(Self { + values, + total_tokens, + quantized_scoring, + norms: OnceLock::new(), + }) + } + + pub(crate) fn len(&self) -> usize { + self.values.len() + } + + pub(crate) fn total_tokens(&self) -> u64 { + self.total_tokens + } + + #[inline] + pub(crate) fn exact(&self, doc_id: DocId) -> u32 { + self.values[doc_id.as_usize()] + } + + pub(crate) fn scoring_norms(&self) -> Option<&[u8]> { + if !self.quantized_scoring { + return None; + } + Some( + self.norms + .get_or_init(|| { + self.values + .iter() + .map(|&length| quantize_doc_length(length)) + .collect() + }) + .as_ref(), + ) + } + + fn scoring_ready(&self) -> bool { + !self.quantized_scoring || self.norms.get().is_some() + } + + #[inline] + pub(crate) fn scoring(&self, doc_id: DocId) -> u32 { + match self.scoring_norms() { + Some(norms) => dequantize_doc_length(norms[doc_id.as_usize()]), + None => self.exact(doc_id), + } + } +} + +#[derive(Debug)] +enum AddressValues { + Shared { len: usize }, + Owned(Arc>), +} + +impl AddressValues { + fn len(&self) -> usize { + match self { + Self::Shared { len } => *len, + Self::Owned(values) => values.len(), + } + } +} + +impl DeepSizeOf for AddressValues { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + match self { + Self::Shared { .. } => 0, + Self::Owned(values) => values.deep_size_of_children(context), + } + } +} + +/// A compact address-sorted view of the live DocIds in a projection. +/// +/// Most newly built partitions already store addresses in DocId order, so the +/// identity variant adds no per-document memory. Remapped or otherwise +/// unsorted projections keep only a u32 permutation instead of duplicating +/// the u64 address column. +#[derive(Debug)] +enum AddressDocIdLookup { + Identity, + Sorted(Box<[u32]>), +} + +impl DeepSizeOf for AddressDocIdLookup { + fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { + match self { + Self::Identity => 0, + Self::Sorted(doc_ids) => std::mem::size_of_val(doc_ids.as_ref()), + } + } +} + +impl AddressDocIdLookup { + fn build(projection: &ResidentAddressProjection) -> Self { + if projection.projection.live_docs.is_none() + && (1..projection.len()).all(|index| { + projection.stored_address(index - 1) <= projection.stored_address(index) + }) + { + return Self::Identity; + } + + let mut doc_ids = match projection.projection.live_docs.as_ref() { + Some(live_docs) => live_docs.iter().collect::>(), + None => (0..projection.len() as u32).collect::>(), + }; + doc_ids + .sort_unstable_by_key(|&doc_id| (projection.stored_address(doc_id as usize), doc_id)); + Self::Sorted(doc_ids.into_boxed_slice()) + } + + fn len(&self, projection: &ResidentAddressProjection) -> usize { + match self { + Self::Identity => projection.len(), + Self::Sorted(doc_ids) => doc_ids.len(), + } + } + + fn doc_id_at(&self, position: usize) -> u32 { + match self { + Self::Identity => position as u32, + Self::Sorted(doc_ids) => doc_ids[position], + } + } + + fn address_at(&self, projection: &ResidentAddressProjection, position: usize) -> u64 { + projection.stored_address(self.doc_id_at(position) as usize) + } + + fn partition_point( + &self, + projection: &ResidentAddressProjection, + mut predicate: impl FnMut(u64) -> bool, + ) -> usize { + let mut left = 0; + let mut right = self.len(projection); + while left < right { + let middle = left + (right - left) / 2; + if predicate(self.address_at(projection, middle)) { + left = middle + 1; + } else { + right = middle; + } + } + left + } + + fn insert_address_range( + &self, + projection: &ResidentAddressProjection, + start: u64, + end: u64, + selected: &mut RoaringBitmap, + ) { + let first = self.partition_point(projection, |address| address < start); + let after_last = self.partition_point(projection, |address| address <= end); + for position in first..after_last { + selected.insert(self.doc_id_at(position)); + } + } + + fn matching_doc_ids( + &self, + projection: &ResidentAddressProjection, + addresses: &RowAddrTreeMap, + ) -> RoaringBitmap { + let mut selected = RoaringBitmap::new(); + for (&fragment_id, selection) in addresses.iter() { + match selection { + RowAddrSelection::Full => { + let start = u64::from(RowAddress::new_from_parts(fragment_id, 0)); + let end = u64::from(RowAddress::new_from_parts(fragment_id, u32::MAX)); + self.insert_address_range(projection, start, end, &mut selected); + } + RowAddrSelection::Partial(offsets) => { + let mut offsets = offsets.iter(); + while let Some(range) = offsets.next_range() { + let start = + u64::from(RowAddress::new_from_parts(fragment_id, *range.start())); + let end = u64::from(RowAddress::new_from_parts(fragment_id, *range.end())); + self.insert_address_range(projection, start, end, &mut selected); + } + } + } + } + selected + } + + fn matching_sorted_addresses( + &self, + projection: &ResidentAddressProjection, + addresses: &[u64], + ) -> std::result::Result { + let mut selected = RoaringBitmap::new(); + for &address in addresses { + let first = self.partition_point(projection, |candidate| candidate < address); + let after_last = self.partition_point(projection, |candidate| candidate <= address); + if after_last.saturating_sub(first) > 1 { + return Err(RowAddressProjectionOrderError::Duplicate { + first_doc_id: DocId::new(self.doc_id_at(first)), + duplicate_doc_id: DocId::new(self.doc_id_at(first + 1)), + address: RowAddress::new_from_u64(address), + }); + } + if first < after_last { + selected.insert(self.doc_id_at(first)); + } + } + Ok(selected) + } + + fn visibility( + &self, + projection: &ResidentAddressProjection, + mask: &RowAddrMask, + ) -> DocVisibility { + match mask { + RowAddrMask::AllowList(allowed) => { + DocVisibility::Selected(self.matching_doc_ids(projection, allowed)) + } + RowAddrMask::BlockList(blocked) => { + let blocked = self.matching_doc_ids(projection, blocked); + let mut selected = projection.live_doc_ids(); + selected -= &blocked; + DocVisibility::Selected(selected) + } + } + } +} + +/// Addresses projected into the dataset version that opened the index. +#[derive(Debug)] +pub(super) struct VersionAddressProjection { + addresses: AddressValues, + /// `None` means every slot is live. Deleted documents retain their DocId + /// slot but are absent from this bitmap. + live_docs: Option, + doc_ids_by_address: OnceCell>, + ordered_validation: AtomicU8, + /// Upper-bound candidate work materialized while order is still unknown. + /// Once this exceeds the normal one-query flat-search budget, the next + /// mapper pays for one reusable ordered validation instead. + materialized_candidate_cost: AtomicUsize, + #[cfg(test)] + ordered_validation_visited_docs: AtomicUsize, +} + +/// A query-scoped projection guard. Shared addresses remain alive only while +/// this guard or their independently weighed cache entry owns the Arrow column. +#[derive(Debug, Clone)] +pub(super) struct ResidentAddressProjection { + projection: Arc, + addresses: ResidentAddressValues, +} + +#[derive(Debug, Clone)] +enum ResidentAddressValues { + Shared(Arc), + Owned(Arc>), +} + +/// A row-granularity projection whose live row addresses are strictly ordered +/// by partition-local document id. +/// +/// Cross-column scorers can use this view to translate their local document +/// domain into the shared row-address domain without materializing all hits. +/// Construction deliberately rejects duplicate or descending live addresses; +/// callers can distinguish those cases and select a materialized fallback. +#[derive(Debug, Clone)] +pub(super) struct OrderedRowAddressProjection { + projection: ResidentAddressProjection, +} + +/// Why a resident address projection cannot be streamed in local DocId order. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RowAddressProjectionOrderError { + Duplicate { + first_doc_id: DocId, + duplicate_doc_id: DocId, + address: RowAddress, + }, + OutOfOrder { + previous_doc_id: DocId, + previous_address: RowAddress, + doc_id: DocId, + address: RowAddress, + }, +} + +/// Non-triggering view of the immutable ordered-validation cache. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CachedRowAddressOrder { + Unknown = 0, + Ordered = 1, + Duplicate = 2, + OutOfOrder = 3, +} + +impl CachedRowAddressOrder { + fn from_raw(value: u8) -> Self { + match value { + value if value == Self::Ordered as u8 => Self::Ordered, + value if value == Self::Duplicate as u8 => Self::Duplicate, + value if value == Self::OutOfOrder as u8 => Self::OutOfOrder, + value => { + debug_assert_eq!( + value, + Self::Unknown as u8, + "ordered row-address validation cache contains invalid state {value}" + ); + // Treat impossible/corrupt state as cold in release builds so + // callers safely recompute the immutable projection order. + Self::Unknown + } + } + } + + fn from_validation( + validation: &std::result::Result<(), RowAddressProjectionOrderError>, + ) -> Self { + match validation { + Ok(()) => Self::Ordered, + Err(RowAddressProjectionOrderError::Duplicate { .. }) => Self::Duplicate, + Err(RowAddressProjectionOrderError::OutOfOrder { .. }) => Self::OutOfOrder, + } + } +} + +impl std::fmt::Display for RowAddressProjectionOrderError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Duplicate { + first_doc_id, + duplicate_doc_id, + address, + } => write!( + formatter, + "row address {} is shared by local documents {} and {}", + u64::from(*address), + first_doc_id.get(), + duplicate_doc_id.get() + ), + Self::OutOfOrder { + previous_doc_id, + previous_address, + doc_id, + address, + } => write!( + formatter, + "row address {} for local document {} follows larger address {} for local document {}", + u64::from(*address), + doc_id.get(), + u64::from(*previous_address), + previous_doc_id.get() + ), + } + } +} + +impl std::error::Error for RowAddressProjectionOrderError {} + +impl OrderedRowAddressProjection { + fn validate_doc_ids( + projection: &ResidentAddressProjection, + doc_ids: impl Iterator, + ) -> std::result::Result<(), RowAddressProjectionOrderError> { + let mut previous = None; + for doc_id in doc_ids { + #[cfg(test)] + projection + .projection + .ordered_validation_visited_docs + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + let doc_id = DocId::new(doc_id); + let address = projection.stored_address(doc_id.as_usize()); + if let Some((previous_doc_id, previous_address)) = previous { + if address == previous_address { + return Err(RowAddressProjectionOrderError::Duplicate { + first_doc_id: previous_doc_id, + duplicate_doc_id: doc_id, + address: RowAddress::new_from_u64(address), + }); + } + if address < previous_address { + return Err(RowAddressProjectionOrderError::OutOfOrder { + previous_doc_id, + previous_address: RowAddress::new_from_u64(previous_address), + doc_id, + address: RowAddress::new_from_u64(address), + }); + } + } + previous = Some((doc_id, address)); + } + Ok(()) + } + + fn validate( + projection: &ResidentAddressProjection, + ) -> std::result::Result<(), RowAddressProjectionOrderError> { + match projection.projection.live_docs.as_ref() { + Some(live_docs) => Self::validate_doc_ids(projection, live_docs.iter()), + None => Self::validate_doc_ids(projection, 0..projection.len() as u32), + } + } + + fn try_new( + projection: &ResidentAddressProjection, + ) -> std::result::Result { + match projection.cached_row_address_order() { + CachedRowAddressOrder::Ordered => {} + CachedRowAddressOrder::Unknown => { + // Validation intentionally runs before the atomic publish. A + // racing query may repeat this work, but never waits for a + // query holding a lock while occupying a CPU worker. + let validation = projection.compute_ordered_validation(); + projection.publish_ordered_validation(&validation); + validation?; + } + cached @ (CachedRowAddressOrder::Duplicate | CachedRowAddressOrder::OutOfOrder) => { + // The compact cache intentionally stores only the category. + // Reconstruct the exact diagnostics only for callers that ask + // for an ordered view after learning the projection is invalid. + let validation = projection.compute_ordered_validation(); + debug_assert_eq!(CachedRowAddressOrder::from_validation(&validation), cached); + validation?; + } + } + + Ok(Self { + projection: projection.clone(), + }) + } + + fn has_sparse_live_docs(&self) -> bool { + self.projection + .projection + .live_docs + .as_ref() + .is_some_and(|live_docs| live_docs.len() as usize != self.len()) + } + + fn doc_id_at(&self, position: usize) -> Option { + if self.has_sparse_live_docs() { + self.projection + .projection + .live_docs + .as_ref()? + .select(u32::try_from(position).ok()?) + } else { + let doc_id = u32::try_from(position).ok()?; + (position < self.len()).then_some(doc_id) + } + } + + fn first_after(&self, local_doc: u64) -> Option { + let next_doc = u32::try_from(local_doc.checked_add(1)?).ok()?; + if self.has_sparse_live_docs() { + self.projection + .projection + .live_docs + .as_ref()? + .range(next_doc..) + .next() + } else { + ((next_doc as usize) < self.len()).then_some(next_doc) + } + } + + /// Number of slots in the partition-local DocId domain, including deleted + /// slots. This is the terminal boundary used by shallow-advance mapping. + pub(super) fn len(&self) -> usize { + self.projection.len() + } + + pub(super) fn live_len(&self) -> usize { + self.projection.live_len() + } + + /// Inclusive minimum and maximum row addresses among live documents. + /// + /// Strict ordering makes this an O(1) lookup apart from sparse-bitmap + /// selection. Deleted DocId slots never contribute to the hull. + #[cfg(test)] + pub(super) fn live_address_hull(&self) -> Option<(u64, u64)> { + let first_doc_id = self.doc_id_at(0)?; + let last_position = self.live_len().checked_sub(1)?; + let last_doc_id = self.doc_id_at(last_position)?; + Some(( + self.projection.stored_address(first_doc_id as usize), + self.projection.stored_address(last_doc_id as usize), + )) + } + + /// Map sorted, deduplicated row-address candidates into live local DocIds. + /// + /// Each candidate uses binary search over live documents. Independent + /// lookups also keep the result exact if an internal caller accidentally + /// supplies duplicate or out-of-order candidates. + pub(super) fn select_sorted_addresses(&self, addresses: &[u64]) -> RoaringBitmap { + addresses + .iter() + .filter_map(|&address| { + let local_doc = self.lower_bound(address)?; + (self.address(local_doc) == Some(address)).then_some(local_doc as u32) + }) + .collect() + } + + /// Translate a live partition-local document into its row address. + pub(super) fn address(&self, local_doc: u64) -> Option { + let local_doc = u32::try_from(local_doc).ok()?; + if local_doc as usize >= self.len() { + return None; + } + self.projection.address(DocId::new(local_doc)) + } + + /// Return the first live local document whose row address is at least the + /// requested global row address. + pub(super) fn lower_bound(&self, global_row_address: u64) -> Option { + let mut left = 0; + let mut right = self.live_len(); + while left < right { + let middle = left + (right - left) / 2; + let doc_id = self.doc_id_at(middle)?; + if self.projection.stored_address(doc_id as usize) < global_row_address { + left = middle + 1; + } else { + right = middle; + } + } + self.doc_id_at(left).map(u64::from) + } + + /// Return the address of the first live document after `local_doc`. + /// + /// Deleted slots are skipped, so this can turn an inclusive local shallow + /// endpoint into an exclusive boundary in the shared row-address domain. + pub(super) fn next_address(&self, local_doc: u64) -> Option { + let next_doc = self.first_after(local_doc)?; + Some(self.projection.stored_address(next_doc as usize)) + } +} + +#[cfg(test)] +pub(super) fn resident_row_address_projection_for_test( + addresses: Vec, +) -> ResidentAddressProjection { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(addresses)), + live_docs: None, + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + projection + .resident(None) + .expect("owned test row addresses must be resident") +} + +#[cfg(test)] +pub(super) fn ordered_row_address_projection_for_test( + addresses: Vec, +) -> OrderedRowAddressProjection { + resident_row_address_projection_for_test(addresses) + .try_ordered_row_addresses() + .expect("test row addresses must be strictly increasing and unique") +} + +impl DeepSizeOf for VersionAddressProjection { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.addresses.deep_size_of_children(context) + + self + .live_docs + .as_ref() + .map(|live| live.serialized_size()) + .unwrap_or(0) + + self + .doc_ids_by_address + .get() + .map(|lookup| lookup.deep_size_of_children(context)) + .unwrap_or(0) + } +} + +impl VersionAddressProjection { + fn try_new( + raw: &UInt64Array, + expected_num_docs: usize, + remapper: Option<&dyn RowIdRemapper>, + path: &str, + ) -> Result { + if raw.len() != expected_num_docs { + return Err(corrupt_docs( + path, + format!( + "{ROW_ID} has {} rows but the file footer reports {expected_num_docs}", + raw.len() + ), + )); + } + if raw.null_count() != 0 { + return Err(corrupt_docs(path, format!("{ROW_ID} contains null values"))); + } + + let Some(remapper) = remapper else { + return Ok(Self { + addresses: AddressValues::Shared { len: raw.len() }, + live_docs: None, + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + #[cfg(test)] + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + }; + + let mut addresses = Vec::with_capacity(raw.len()); + let mut live_docs = RoaringBitmap::new(); + for (doc_id, &address) in raw.values().iter().enumerate() { + match remapper.remap_row_id(address) { + Some(current) => { + addresses.push(current); + live_docs.insert(doc_id as u32); + } + None => { + // The value in a dead slot is intentionally meaningless; + // callers must consult `live_docs` before reading it. + addresses.push(0); + } + } + } + Ok(Self { + addresses: AddressValues::Owned(Arc::new(addresses)), + live_docs: Some(live_docs), + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + #[cfg(test)] + ordered_validation_visited_docs: AtomicUsize::new(0), + }) + } + + fn resident( + self: &Arc, + shared_addresses: Option>, + ) -> Option { + match &self.addresses { + AddressValues::Shared { len } => { + let shared_addresses = shared_addresses?; + debug_assert_eq!(shared_addresses.len(), *len); + Some(ResidentAddressProjection { + projection: self.clone(), + addresses: ResidentAddressValues::Shared(shared_addresses), + }) + } + AddressValues::Owned(values) => Some(ResidentAddressProjection { + projection: self.clone(), + addresses: ResidentAddressValues::Owned(values.clone()), + }), + } + } +} + +impl ResidentAddressProjection { + fn len(&self) -> usize { + self.projection.addresses.len() + } + + pub(super) fn live_len(&self) -> usize { + self.projection + .live_docs + .as_ref() + .map_or(self.len(), |live_docs| live_docs.len() as usize) + } + + /// Inclusive minimum and maximum row addresses among live documents. + /// + /// Unlike [`OrderedRowAddressProjection::live_address_hull`], this scans + /// the live projection because remapping may reorder addresses. Deleted + /// DocId slots never contribute to the hull. + #[cfg(test)] + pub(super) fn live_address_hull(&self) -> Option<(u64, u64)> { + let mut hull: Option<(u64, u64)> = None; + let mut include_doc = |doc_id: u32| { + let address = self.stored_address(doc_id as usize); + hull = Some(match hull { + Some((minimum, maximum)) => (minimum.min(address), maximum.max(address)), + None => (address, address), + }); + }; + match self.projection.live_docs.as_ref() { + Some(live_docs) => live_docs.iter().for_each(&mut include_doc), + None => (0..self.len() as u32).for_each(include_doc), + } + hull + } + + /// Decide whether another unknown-order source should be materialized. + /// + /// A single sparse query avoids an O(all documents) validation. Repeated + /// queries share this lock-free budget through the immutable version + /// projection, so materialization cannot remain the permanent execution + /// mode once its cumulative upper-bound cost exceeds one validation + /// threshold. + pub(super) fn should_materialize_unknown_projection( + &self, + source_cost: usize, + flat_search_percent_threshold: u64, + ) -> bool { + let mut current = self + .projection + .materialized_candidate_cost + .load(AtomicOrdering::Relaxed); + let cumulative_cost = loop { + let next = current.saturating_add(source_cost); + match self + .projection + .materialized_candidate_cost + .compare_exchange_weak( + current, + next, + AtomicOrdering::Relaxed, + AtomicOrdering::Relaxed, + ) { + Ok(_) => break next, + Err(observed) => current = observed, + } + }; + + (cumulative_cost as u128).saturating_mul(100) + <= u128::from(flat_search_percent_threshold).saturating_mul(self.live_len() as u128) + } + + #[cfg(test)] + pub(super) fn ordered_validation_visited_docs(&self) -> usize { + self.projection + .ordered_validation_visited_docs + .load(std::sync::atomic::Ordering::Relaxed) + } + + /// Inspect ordered-validation state without starting validation or waiting + /// for another query's validation. + pub(super) fn cached_row_address_order(&self) -> CachedRowAddressOrder { + CachedRowAddressOrder::from_raw( + self.projection + .ordered_validation + .load(AtomicOrdering::Acquire), + ) + } + + fn compute_ordered_validation( + &self, + ) -> std::result::Result<(), RowAddressProjectionOrderError> { + OrderedRowAddressProjection::validate(self) + } + + fn publish_ordered_validation( + &self, + validation: &std::result::Result<(), RowAddressProjectionOrderError>, + ) { + let status = CachedRowAddressOrder::from_validation(validation); + // All validation work happened before this non-blocking publication. + // Immutable projections make racing results deterministic, so losing + // the compare-exchange requires no reconciliation or wait. + if let Err(observed) = self.projection.ordered_validation.compare_exchange( + CachedRowAddressOrder::Unknown as u8, + status as u8, + AtomicOrdering::Release, + AtomicOrdering::Relaxed, + ) { + debug_assert_eq!( + observed, status as u8, + "immutable row-address projection published conflicting validation states" + ); + } + } + + /// Validate that this row-granularity projection can be streamed in the + /// shared row-address domain. + pub(super) fn try_ordered_row_addresses( + &self, + ) -> std::result::Result { + OrderedRowAddressProjection::try_new(self) + } + + fn stored_address(&self, index: usize) -> u64 { + match &self.addresses { + ResidentAddressValues::Shared(values) => values.value(index), + ResidentAddressValues::Owned(values) => values[index], + } + } + + pub(super) fn address(&self, doc_id: DocId) -> Option { + if self + .projection + .live_docs + .as_ref() + .is_some_and(|live| !live.contains(doc_id.get())) + { + None + } else { + Some(self.stored_address(doc_id.as_usize())) + } + } + + fn live_doc_ids(&self) -> RoaringBitmap { + self.projection + .live_docs + .clone() + .unwrap_or_else(|| (0..self.len() as u32).collect()) + } + + async fn doc_ids_by_address(&self) -> Result> { + self.projection + .doc_ids_by_address + .get_or_try_init(|| { + let projection = self.clone(); + async move { + spawn_cpu(move || Result::Ok(Arc::new(AddressDocIdLookup::build(&projection)))) + .await + } + }) + .await + .cloned() + } + + /// Map sorted, deduplicated row-address candidates into live local DocIds. + /// + /// The reusable reverse lookup handles unordered remapped projections. A + /// candidate that resolves to multiple live DocIds is rejected as an + /// invalid FTS row-address projection instead of silently merging them. + /// The lookup is exact for any candidate order; sorting only avoids + /// redundant caller work. + pub(super) async fn select_sorted_addresses(&self, addresses: &[u64]) -> Result { + if addresses.is_empty() || self.live_len() == 0 { + return Ok(RoaringBitmap::new()); + } + + let ordered = match self.cached_row_address_order() { + CachedRowAddressOrder::Ordered => { + Some(self.try_ordered_row_addresses().map_err(|error| { + Error::index(format!("invalid FTS row-address projection: {error}")) + })?) + } + CachedRowAddressOrder::OutOfOrder => None, + CachedRowAddressOrder::Unknown | CachedRowAddressOrder::Duplicate => { + let projection = self.clone(); + match spawn_cpu(move || Result::Ok(projection.try_ordered_row_addresses())).await? { + Ok(ordered) => Some(ordered), + Err(error @ RowAddressProjectionOrderError::Duplicate { .. }) => { + return Err(Error::index(format!( + "invalid FTS row-address projection: {error}" + ))); + } + Err(RowAddressProjectionOrderError::OutOfOrder { .. }) => None, + } + } + }; + if let Some(ordered) = ordered { + return Ok(ordered.select_sorted_addresses(addresses)); + } + + let lookup = self.doc_ids_by_address().await?; + lookup + .matching_sorted_addresses(self, addresses) + .map_err(|error| Error::index(format!("invalid FTS row-address projection: {error}"))) + } + + async fn materialize_visibility(self, mask: Arc) -> Result { + let lookup = self.doc_ids_by_address().await?; + let projection = self; + spawn_cpu(move || Result::Ok(lookup.visibility(&projection, &mask))).await + } +} + +/// Query-local selection in the partition-local DocId domain. +#[derive(Debug, Clone)] +pub(super) enum DocVisibility { + All, + Selected(RoaringBitmap), + Filtered { + projection: ResidentAddressProjection, + mask: Arc, + }, +} + +impl DocVisibility { + pub(crate) fn is_all(&self) -> bool { + matches!(self, Self::All) + } + + #[inline] + pub(crate) fn selected(&self, doc_id: DocId) -> bool { + match self { + Self::All => true, + Self::Selected(selected) => selected.contains(doc_id.get()), + Self::Filtered { projection, mask } => projection + .address(doc_id) + .is_some_and(|address| mask.selected(address)), + } + } + + pub(crate) fn len(&self, total_docs: usize) -> usize { + match self { + Self::All => total_docs, + Self::Selected(selected) => selected.len() as usize, + Self::Filtered { .. } => total_docs, + } + } + + pub(crate) fn is_empty(&self) -> bool { + matches!(self, Self::Selected(selected) if selected.is_empty()) + } + + pub(crate) fn iter(&self) -> Option + '_> { + match self { + Self::Selected(selected) => Some(selected.iter().map(DocId::new)), + Self::All | Self::Filtered { .. } => None, + } + } +} + +/// Modern query-side document state for one partition. +pub(super) struct PartitionDocuments { + store: Arc, + path: String, + partition_id: u64, + index_cache: WeakLanceCache, + num_docs: usize, + coordinate_rank: usize, + persisted_total_tokens: Option, + quantized_scoring: bool, + remapper: Option>, + lengths: OnceCell>, + projection: OnceCell>, + shared_addresses: ArcSwapWeak, + prewarm_complete: OnceCell<()>, +} + +/// Load-boundary discriminator between the read-only legacy representation and +/// the typed partitioned representation. Query code dispatches on this enum +/// once; modern scoring never receives a partial legacy [`DocSet`]. +#[derive(Debug, Clone)] +pub(super) enum PartitionDocumentStore { + Legacy(Arc), + Modern(Arc), +} + +impl DeepSizeOf for PartitionDocumentStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + match self { + Self::Legacy(docs) => docs.deep_size_of_children(context), + Self::Modern(docs) => docs.deep_size_of_children(context), + } + } +} + +impl PartitionDocumentStore { + pub(crate) fn len(&self) -> usize { + match self { + Self::Legacy(docs) => docs.len(), + Self::Modern(docs) => docs.len(), + } + } + + pub(crate) fn coordinate_rank(&self) -> usize { + match self { + Self::Legacy(docs) => docs.coordinate_rank(), + Self::Modern(docs) => docs.coordinate_rank(), + } + } + + pub(crate) fn legacy(&self) -> Option<&Arc> { + match self { + Self::Legacy(docs) => Some(docs), + Self::Modern(_) => None, + } + } + + pub(crate) fn modern(&self) -> Option<&Arc> { + match self { + Self::Legacy(_) => None, + Self::Modern(docs) => Some(docs), + } + } + + pub(crate) async fn stats(&self) -> Result { + match self { + Self::Legacy(docs) => Ok(PartitionStats { + num_docs: docs.len(), + total_tokens: docs.total_tokens_num(), + }), + Self::Modern(docs) => docs.stats().await, + } + } + + pub(crate) fn cached_stats(&self) -> Option { + match self { + Self::Legacy(docs) => Some(PartitionStats { + num_docs: docs.len(), + total_tokens: docs.total_tokens_num(), + }), + Self::Modern(docs) => docs.cached_stats(), + } + } + + pub(crate) async fn prewarm(&self) -> Result<()> { + match self { + Self::Legacy(_) => Ok(()), + Self::Modern(docs) => docs.prewarm().await, + } + } + + #[cfg(test)] + pub(crate) fn query_ready(&self) -> bool { + match self { + Self::Legacy(_) => true, + Self::Modern(docs) => docs.query_ready(), + } + } + + pub(crate) fn prewarm_status(&self) -> FtsPrewarmDocumentStatus { + match self { + Self::Legacy(_) => FtsPrewarmDocumentStatus { + prewarm_complete: true, + scoring_ready: true, + reverse_lookup_ready: true, + projection_resident: true, + }, + Self::Modern(docs) => docs.prewarm_status(), + } + } + + pub(crate) async fn load_build_docset(&self) -> Result { + match self { + Self::Legacy(docs) => Ok((**docs).clone()), + Self::Modern(docs) => docs.load_build_docset().await, + } + } +} + +impl std::fmt::Debug for PartitionDocuments { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PartitionDocuments") + .field("path", &self.path) + .field("num_docs", &self.num_docs) + .field("coordinate_rank", &self.coordinate_rank) + .field("persisted_total_tokens", &self.persisted_total_tokens) + .field("lengths_loaded", &self.lengths.initialized()) + .field("projection_loaded", &self.projection.initialized()) + .finish() + } +} + +impl DeepSizeOf for PartitionDocuments { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.lengths + .get() + .map(|lengths| lengths.deep_size_of_children(context)) + .unwrap_or(0) + + self + .projection + .get() + .map(|projection| projection.deep_size_of_children(context)) + .unwrap_or(0) + } +} + +impl PartitionDocuments { + pub(crate) fn try_new( + store: Arc, + path: String, + partition_id: u64, + index_cache: WeakLanceCache, + reader: &dyn IndexReader, + remapper: Option>, + quantized_scoring: bool, + ) -> Result { + let num_docs = reader.num_rows(); + if num_docs > u32::MAX as usize { + return Err(corrupt_docs( + &path, + format!("document count {num_docs} exceeds dense DocId capacity"), + )); + } + let persisted_total_tokens = reader + .schema() + .metadata + .get(TOTAL_TOKENS_KEY) + .map(|value| { + value.parse::().map_err(|error| { + corrupt_docs( + &path, + format!("invalid {TOTAL_TOKENS_KEY} metadata value {value:?}: {error}"), + ) + }) + }) + .transpose()?; + let coordinate_rank = + document_coordinate_rank(&arrow_schema::Schema::from(reader.schema())); + Ok(Self { + store, + path, + partition_id, + index_cache, + num_docs, + coordinate_rank, + persisted_total_tokens, + quantized_scoring, + remapper, + lengths: OnceCell::new(), + projection: OnceCell::new(), + shared_addresses: ArcSwapWeak::from(Weak::new()), + prewarm_complete: OnceCell::new(), + }) + } + + pub(crate) fn len(&self) -> usize { + self.num_docs + } + + pub(crate) fn coordinate_rank(&self) -> usize { + self.coordinate_rank + } + + #[cfg(test)] + pub(crate) fn lengths_loaded(&self) -> bool { + self.lengths.initialized() + } + + #[cfg(test)] + pub(crate) fn projection_loaded(&self) -> bool { + self.projection.initialized() + } + + #[cfg(test)] + pub(crate) fn address_buffer_handle(&self) -> Weak { + self.shared_addresses.load_full() + } + + pub(crate) fn projection_resident(&self) -> bool { + self.resident_address_projection().is_some() + } + + #[cfg(test)] + pub(crate) fn query_ready(&self) -> bool { + self.prewarm_status().query_ready() + } + + pub(crate) fn prewarm_status(&self) -> FtsPrewarmDocumentStatus { + FtsPrewarmDocumentStatus { + prewarm_complete: self.prewarm_complete.initialized(), + scoring_ready: self + .lengths + .get() + .is_some_and(|lengths| lengths.scoring_ready()), + reverse_lookup_ready: self + .projection + .get() + .is_some_and(|projection| projection.doc_ids_by_address.initialized()), + projection_resident: self.projection_resident(), + } + } + + async fn reader(&self) -> Result> { + self.store.open_index_file(&self.path).await + } + + async fn row_ids_column(&self) -> Result> { + let store = self.store.clone(); + let path = self.path.clone(); + let num_docs = self.num_docs; + let cached = self + .index_cache + .get_or_insert_with_key( + DocRowIdsKey { + partition_id: self.partition_id, + }, + || async move { + let reader = store.open_index_file(&path).await?; + let batch = reader.read_range(0..num_docs, Some(&[ROW_ID])).await?; + let row_ids = required_u64_column(&batch, ROW_ID, &path)?; + if row_ids.null_count() != 0 { + return Err(corrupt_docs( + &path, + format!("{ROW_ID} contains null values"), + )); + } + if row_ids.len() != num_docs { + return Err(corrupt_docs( + &path, + format!( + "{ROW_ID} has {} rows but the file footer reports {num_docs}", + row_ids.len() + ), + )); + } + Ok(CachedDocRowIds { + row_ids: Arc::new(row_ids.clone()), + }) + }, + ) + .await?; + let row_ids = cached.row_ids.clone(); + self.shared_addresses.store(Arc::downgrade(&row_ids)); + Ok(row_ids) + } + + fn lengths_from_batch(&self, batch: &RecordBatch) -> Result> { + let column = required_u32_column(batch, NUM_TOKEN_COL, &self.path)?; + if column.null_count() != 0 { + return Err(corrupt_docs( + &self.path, + format!("{NUM_TOKEN_COL} contains null values"), + )); + } + Ok(Arc::new(DocLengths::try_new( + column.values().clone(), + self.num_docs, + self.persisted_total_tokens, + self.quantized_scoring, + &self.path, + )?)) + } + + pub(crate) async fn stats(&self) -> Result { + let total_tokens = match self.persisted_total_tokens { + Some(total_tokens) => total_tokens, + None => self.lengths().await?.total_tokens(), + }; + Ok(PartitionStats { + num_docs: self.num_docs, + total_tokens, + }) + } + + pub(crate) fn cached_stats(&self) -> Option { + self.persisted_total_tokens + .or_else(|| self.lengths.get().map(|lengths| lengths.total_tokens())) + .map(|total_tokens| PartitionStats { + num_docs: self.num_docs, + total_tokens, + }) + } + + pub(crate) async fn lengths(&self) -> Result> { + self.lengths + .get_or_try_init(|| async { + let reader = self.reader().await?; + let batch = reader + .read_range(0..self.num_docs, Some(&[NUM_TOKEN_COL])) + .await?; + self.lengths_from_batch(&batch) + }) + .await + .cloned() + } + + /// Return resident lengths without entering the asynchronous singleflight path. + pub(crate) fn cached_lengths(&self) -> Option> { + self.lengths.get().cloned() + } + + pub(super) fn resident_address_projection(&self) -> Option { + let projection = self.projection.get()?.clone(); + let shared_addresses = match &projection.addresses { + AddressValues::Shared { .. } => self.shared_addresses.load().upgrade(), + AddressValues::Owned(_) => None, + }; + projection.resident(shared_addresses) + } + + pub(crate) async fn address_projection(&self) -> Result { + if let Some(projection) = self.resident_address_projection() { + return Ok(projection); + } + + let row_ids = self.row_ids_column().await?; + let projection = self + .projection + .get_or_try_init(|| async { + Result::Ok(Arc::new(VersionAddressProjection::try_new( + row_ids.as_ref(), + self.num_docs, + self.remapper.as_deref(), + &self.path, + )?)) + }) + .await + .cloned()?; + projection.resident(Some(row_ids)).ok_or_else(|| { + Error::internal(format!( + "address projection for {} could not bind its cache-managed ROW_ID column", + self.path + )) + }) + } + + pub(crate) async fn visibility( + &self, + mask: Arc, + materialize_selected: bool, + ) -> Result { + if let Some(visibility) = self.immediate_visibility(mask.clone(), materialize_selected) { + return Ok(visibility); + } + + let projection = self.address_projection().await?; + if mask.is_select_all() { + return Ok(DocVisibility::Selected(projection.live_doc_ids())); + } + if materialize_selected { + projection.materialize_visibility(mask).await + } else { + Ok(DocVisibility::Filtered { projection, mask }) + } + } + + /// Resolve visibility without I/O or CPU-pool work when all required state is resident. + pub(crate) fn immediate_visibility( + &self, + mask: Arc, + materialize_selected: bool, + ) -> Option { + if mask.max_len() == Some(0) { + return Some(DocVisibility::Selected(RoaringBitmap::new())); + } + if mask.is_select_all() && self.remapper.is_none() { + return Some(DocVisibility::All); + } + + let projection = self.resident_address_projection()?; + if mask.is_select_all() { + return Some(DocVisibility::Selected(projection.live_doc_ids())); + } + if materialize_selected { + None + } else { + Some(DocVisibility::Filtered { projection, mask }) + } + } + + /// Resolve final global top-k DocIds to current row addresses. + pub(crate) async fn resolve_addresses(&self, doc_ids: &[DocId]) -> Result> { + if doc_ids.is_empty() { + return Ok(Vec::new()); + } + self.validate_doc_ids(doc_ids)?; + if let Some(projection) = self.resident_address_projection() { + return self.resolve_projected_addresses(&projection, doc_ids); + } + if self.remapper.is_some() { + let projection = self.address_projection().await?; + return self.resolve_projected_addresses(&projection, doc_ids); + } + + let row_ids = self.row_ids_column().await?; + Ok(doc_ids + .iter() + .map(|doc_id| row_ids.value(doc_id.as_usize())) + .collect()) + } + + /// Load scoring document lengths for a bounded candidate set. + /// + /// The current format stores lengths in an independent dense column. Cold + /// selective staged queries read only candidate rows; resident or dense + /// queries reuse/populate the normal full-column cache. Returned values + /// exactly match [`DocLengths::scoring`], including the quantized V3 path. + pub(crate) async fn resolve_scoring_lengths(&self, doc_ids: &[DocId]) -> Result> { + if doc_ids.is_empty() { + return Ok(Vec::new()); + } + self.validate_doc_ids(doc_ids)?; + if let Some(lengths) = self.cached_lengths() { + return Ok(doc_ids + .iter() + .map(|&doc_id| lengths.scoring(doc_id)) + .collect()); + } + if !self.prefer_sparse_document_read(doc_ids.len()) { + let lengths = self.lengths().await?; + return Ok(doc_ids + .iter() + .map(|&doc_id| lengths.scoring(doc_id)) + .collect()); + } + + let ranges = doc_ids + .iter() + .map(|doc_id| { + let index = doc_id.as_usize(); + index..index + 1 + }) + .collect::>(); + let batch = self + .reader() + .await? + .read_ranges(&ranges, Some(&[NUM_TOKEN_COL])) + .await?; + let lengths = required_u32_column(&batch, NUM_TOKEN_COL, &self.path)?; + if lengths.null_count() != 0 || lengths.len() != doc_ids.len() { + return Err(corrupt_docs( + &self.path, + format!( + "sparse {NUM_TOKEN_COL} projection returned {} rows with {} nulls for {} candidates", + lengths.len(), + lengths.null_count(), + doc_ids.len() + ), + )); + } + Ok(lengths + .values() + .iter() + .map(|&length| { + if self.quantized_scoring { + dequantize_doc_length(quantize_doc_length(length)) + } else { + length + } + }) + .collect()) + } + + /// Resolve the row address and scoring length for a bounded candidate set. + /// + /// When neither document column is resident and the selection is sparse, + /// both columns are projected by one `read_ranges` call. This avoids + /// opening and scheduling the same document file twice during staged + /// cross-column execution. Asymmetric cache states continue to reuse the + /// resident side through the existing typed resolvers. + pub(crate) async fn resolve_scoring_documents( + &self, + doc_ids: &[DocId], + ) -> Result> { + if doc_ids.is_empty() { + return Ok(Vec::new()); + } + self.validate_doc_ids(doc_ids)?; + + let addresses_need_sparse_read = self.resident_address_projection().is_none() + && self.remapper.is_none() + && self.shared_addresses.load().upgrade().is_none() + && self.prefer_sparse_document_read(doc_ids.len()); + let lengths_need_sparse_read = + self.cached_lengths().is_none() && self.prefer_sparse_document_read(doc_ids.len()); + if addresses_need_sparse_read && lengths_need_sparse_read { + let ranges = doc_ids + .iter() + .map(|doc_id| { + let index = doc_id.as_usize(); + index..index + 1 + }) + .collect::>(); + let batch = self + .reader() + .await? + .read_ranges(&ranges, Some(&[ROW_ID, NUM_TOKEN_COL])) + .await?; + let row_ids = required_u64_column(&batch, ROW_ID, &self.path)?; + let lengths = required_u32_column(&batch, NUM_TOKEN_COL, &self.path)?; + if row_ids.null_count() != 0 + || lengths.null_count() != 0 + || row_ids.len() != doc_ids.len() + || lengths.len() != doc_ids.len() + { + return Err(corrupt_docs( + &self.path, + format!( + "sparse document projection returned {} row addresses ({} nulls) and {} lengths ({} nulls) for {} candidates", + row_ids.len(), + row_ids.null_count(), + lengths.len(), + lengths.null_count(), + doc_ids.len() + ), + )); + } + return Ok(doc_ids + .iter() + .zip(row_ids.values()) + .zip(lengths.values()) + .map(|((&doc_id, &row_address), &length)| { + let scoring_length = if self.quantized_scoring { + dequantize_doc_length(quantize_doc_length(length)) + } else { + length + }; + (doc_id.get(), row_address, scoring_length) + }) + .collect()); + } + + let (row_addresses, scoring_lengths) = futures::try_join!( + self.resolve_addresses(doc_ids), + self.resolve_scoring_lengths(doc_ids) + )?; + if row_addresses.len() != doc_ids.len() || scoring_lengths.len() != doc_ids.len() { + return Err(Error::internal(format!( + "resolved {} row addresses and {} lengths for {} FTS candidates", + row_addresses.len(), + scoring_lengths.len(), + doc_ids.len() + ))); + } + Ok(doc_ids + .iter() + .zip(row_addresses) + .zip(scoring_lengths) + .map(|((&doc_id, row_address), scoring_length)| { + (doc_id.get(), row_address, scoring_length) + }) + .collect()) + } + + pub(crate) fn prefer_sparse_document_read(&self, selected: usize) -> bool { + (selected as u128).saturating_mul(100) + <= (SPARSE_DOCUMENT_READ_PERCENT as u128).saturating_mul(self.num_docs as u128) + } + + /// Resolve final global top-k DocIds to their logical FTS document keys. + pub(crate) async fn resolve_document_keys( + &self, + doc_ids: &[DocId], + ) -> Result)>> { + let row_ids = self.resolve_addresses(doc_ids).await?; + if self.coordinate_rank == 0 { + return Ok(row_ids + .into_iter() + .map(|row_id| (row_id, Vec::new())) + .collect()); + } + + let ranges = doc_ids + .iter() + .map(|doc_id| { + let index = doc_id.as_usize(); + index..index + 1 + }) + .collect::>(); + let coordinate_names = (0..self.coordinate_rank) + .map(doc_index_storage_column) + .collect::>(); + let projection = coordinate_names + .iter() + .map(String::as_str) + .collect::>(); + let batch = self + .reader() + .await? + .read_ranges(&ranges, Some(&projection)) + .await?; + if batch.num_rows() != row_ids.len() { + return Err(corrupt_docs( + &self.path, + format!( + "document coordinate projection returned {} rows for {} candidates", + batch.num_rows(), + row_ids.len() + ), + )); + } + let coordinate_columns = coordinate_names + .iter() + .map(|name| { + let column = required_u32_column(&batch, name, &self.path)?; + if column.null_count() != 0 { + return Err(corrupt_docs( + &self.path, + format!("document coordinate column {name} contains null values"), + )); + } + Ok(column) + }) + .collect::>>()?; + + Ok(row_ids + .into_iter() + .enumerate() + .map(|(index, row_id)| { + ( + row_id, + coordinate_columns + .iter() + .map(|column| column.value(index)) + .collect(), + ) + }) + .collect()) + } + + fn validate_doc_ids(&self, doc_ids: &[DocId]) -> Result<()> { + for doc_id in doc_ids { + if doc_id.as_usize() >= self.num_docs { + return Err(corrupt_docs( + &self.path, + format!( + "candidate DocId {} is outside [0, {})", + doc_id.get(), + self.num_docs + ), + )); + } + } + Ok(()) + } + + fn resolve_projected_addresses( + &self, + projection: &ResidentAddressProjection, + doc_ids: &[DocId], + ) -> Result> { + doc_ids + .iter() + .map(|&doc_id| { + projection.address(doc_id).ok_or_else(|| { + corrupt_docs( + &self.path, + format!("candidate DocId {} is not live", doc_id.get()), + ) + }) + }) + .collect() + } + + /// Resolve addresses synchronously when the cache-managed projection buffer + /// is resident. The returned `None` asks the caller to use the async reload path. + pub(crate) fn cached_row_addresses(&self, doc_ids: &[DocId]) -> Result>> { + self.validate_doc_ids(doc_ids)?; + let Some(projection) = self.resident_address_projection() else { + return Ok(None); + }; + self.resolve_projected_addresses(&projection, doc_ids) + .map(Some) + } + + /// Estimated Arrow payload retained while loading this partition's cached + /// row-address column. + /// The estimate is used to cap cross-partition read concurrency; a single + /// oversized partition is still allowed to make progress. + pub(crate) fn estimated_address_read_bytes(&self, doc_ids: &[DocId]) -> usize { + if doc_ids.is_empty() || self.projection_resident() { + return 0; + } + self.num_docs.saturating_mul(std::mem::size_of::()) + } + + /// Materialize the build-side table for rewrite/update operations. + pub(crate) async fn load_build_docset(&self) -> Result { + DocSet::load(self.reader().await?, false, self.remapper.clone()).await + } + + pub(crate) async fn prewarm(&self) -> Result<()> { + self.prewarm_complete + .get_or_try_init(|| async { + if self.lengths.get().is_none() && self.projection.get().is_none() { + let reader = self.reader().await?; + let batch = reader + .read_range(0..self.num_docs, Some(&[ROW_ID, NUM_TOKEN_COL])) + .await?; + let lengths = self.lengths_from_batch(&batch)?; + let row_ids = + Arc::new(required_u64_column(&batch, ROW_ID, &self.path)?.clone()); + if row_ids.null_count() != 0 { + return Err(corrupt_docs( + &self.path, + format!("{ROW_ID} contains null values"), + )); + } + let projection = Arc::new(VersionAddressProjection::try_new( + row_ids.as_ref(), + self.num_docs, + self.remapper.as_deref(), + &self.path, + )?); + let cached_row_ids = Arc::new(CachedDocRowIds { + row_ids: row_ids.clone(), + }); + self.index_cache + .insert_with_key( + &DocRowIdsKey { + partition_id: self.partition_id, + }, + cached_row_ids, + ) + .await; + self.shared_addresses.store(Arc::downgrade(&row_ids)); + + // A concurrent single-column request may win either OnceCell. + // Awaiting the accessors below joins that initialization without + // replacing the already-published value. + let _ = self.lengths.set(lengths); + let _ = self.projection.set(projection); + } + + let lengths = self.lengths().await?; + spawn_cpu(move || { + let _ = lengths.scoring_norms(); + Result::Ok(()) + }) + .await?; + self.address_projection() + .await? + .doc_ids_by_address() + .await?; + Result::Ok(()) + }) + .await?; + if !self.projection_resident() { + self.address_projection().await?; + } + Ok(()) + } +} + +fn required_u32_column<'a>( + batch: &'a RecordBatch, + name: &str, + path: &str, +) -> Result<&'a UInt32Array> { + let column = batch + .column_by_name(name) + .ok_or_else(|| corrupt_docs(path, format!("required column {name} is missing")))?; + column + .as_any() + .downcast_ref::() + .ok_or_else(|| { + corrupt_docs( + path, + format!( + "column {name} has type {}, expected UInt32", + column.data_type() + ), + ) + }) +} + +fn required_u64_column<'a>( + batch: &'a RecordBatch, + name: &str, + path: &str, +) -> Result<&'a UInt64Array> { + let column = batch + .column_by_name(name) + .ok_or_else(|| corrupt_docs(path, format!("required column {name} is missing")))?; + column + .as_any() + .downcast_ref::() + .ok_or_else(|| { + corrupt_docs( + path, + format!( + "column {name} has type {}, expected UInt64", + column.data_type() + ), + ) + }) +} + +fn corrupt_docs(path: &str, message: impl Into) -> Error { + Error::corrupt_file(Path::from(path), message) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::ops::Range; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use arrow_array::{ArrayRef, RecordBatch}; + use arrow_schema::{DataType, Field, Schema}; + use async_trait::async_trait; + use lance_core::cache::{LanceCache, QuickCacheBackend}; + use lance_core::utils::tempfile::TempObjDir; + use lance_io::object_store::ObjectStore; + use lance_select::RowAddrTreeMap; + use roaring::RoaringTreemap; + use tokio::sync::Notify; + + use crate::scalar::lance_format::LanceIndexStore; + use crate::scalar::{IndexFile, IndexWriter}; + + use super::*; + + #[derive(Debug, Default)] + struct DocumentReadCounts { + open_calls: AtomicUsize, + range_calls: AtomicUsize, + ranges_calls: AtomicUsize, + rows: AtomicUsize, + length_rows: AtomicUsize, + address_rows: AtomicUsize, + } + + impl DocumentReadCounts { + fn record(&self, rows: usize, projection: Option<&[&str]>) { + self.rows.fetch_add(rows, Ordering::Relaxed); + if projection.is_none_or(|columns| columns.contains(&NUM_TOKEN_COL)) { + self.length_rows.fetch_add(rows, Ordering::Relaxed); + } + if projection.is_none_or(|columns| columns.contains(&ROW_ID)) { + self.address_rows.fetch_add(rows, Ordering::Relaxed); + } + } + } + + const PAUSE_ONCE: usize = 1; + const FAIL_ONCE: usize = 2; + + #[derive(Debug, Default)] + struct ReadFault { + action: AtomicUsize, + started: Notify, + } + + impl ReadFault { + async fn apply(&self) -> Result<()> { + match self.action.swap(0, Ordering::AcqRel) { + PAUSE_ONCE => { + self.started.notify_one(); + std::future::pending::>().await + } + FAIL_ONCE => Err(Error::io("injected document read failure")), + _ => Ok(()), + } + } + } + + struct CountingReader { + inner: Arc, + counts: Arc, + fault: Option>, + } + + #[async_trait] + impl IndexReader for CountingReader { + async fn read_record_batch(&self, n: u64, batch_size: u64) -> Result { + self.inner.read_record_batch(n, batch_size).await + } + + async fn read_global_buffer(&self, index: u32) -> Result { + self.inner.read_global_buffer(index).await + } + + async fn read_range( + &self, + range: Range, + projection: Option<&[&str]>, + ) -> Result { + self.counts.range_calls.fetch_add(1, Ordering::Relaxed); + self.counts.record(range.len(), projection); + if let Some(fault) = &self.fault { + fault.apply().await?; + } + self.inner.read_range(range, projection).await + } + + async fn read_ranges( + &self, + ranges: &[Range], + projection: Option<&[&str]>, + ) -> Result { + self.counts.ranges_calls.fetch_add(1, Ordering::Relaxed); + self.counts + .record(ranges.iter().map(Range::len).sum(), projection); + if let Some(fault) = &self.fault { + fault.apply().await?; + } + self.inner.read_ranges(ranges, projection).await + } + + async fn num_batches(&self, batch_size: u64) -> u32 { + self.inner.num_batches(batch_size).await + } + + fn num_rows(&self) -> usize { + self.inner.num_rows() + } + + fn schema(&self) -> &lance_core::datatypes::Schema { + self.inner.schema() + } + + fn file_size_bytes(&self) -> Option { + self.inner.file_size_bytes() + } + } + + #[derive(Debug)] + struct CountingStore { + inner: Arc, + target: String, + counts: Arc, + fault: Option>, + } + + impl DeepSizeOf for CountingStore { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.inner.deep_size_of_children(context) + } + } + + #[async_trait] + impl IndexStore for CountingStore { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn clone_arc(&self) -> Arc { + Arc::new(Self { + inner: self.inner.clone(), + target: self.target.clone(), + counts: self.counts.clone(), + fault: self.fault.clone(), + }) + } + + fn io_parallelism(&self) -> usize { + self.inner.io_parallelism() + } + + async fn new_index_file( + &self, + name: &str, + schema: Arc, + ) -> Result> { + self.inner.new_index_file(name, schema).await + } + + async fn open_index_file(&self, name: &str) -> Result> { + let reader = self.inner.open_index_file(name).await?; + if name == self.target { + self.counts.open_calls.fetch_add(1, Ordering::Relaxed); + Ok(Arc::new(CountingReader { + inner: reader, + counts: self.counts.clone(), + fault: self.fault.clone(), + })) + } else { + Ok(reader) + } + } + + fn with_io_priority(&self, io_priority: u64) -> Arc { + Arc::new(Self { + inner: self.inner.with_io_priority(io_priority), + target: self.target.clone(), + counts: self.counts.clone(), + fault: self.fault.clone(), + }) + } + + async fn copy_index_file( + &self, + name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner.copy_index_file(name, dest_store).await + } + + async fn copy_index_file_to( + &self, + name: &str, + new_name: &str, + dest_store: &dyn IndexStore, + ) -> Result { + self.inner + .copy_index_file_to(name, new_name, dest_store) + .await + } + + async fn rename_index_file(&self, name: &str, new_name: &str) -> Result { + self.inner.rename_index_file(name, new_name).await + } + + async fn delete_index_file(&self, name: &str) -> Result<()> { + self.inner.delete_index_file(name).await + } + + async fn list_files_with_sizes(&self) -> Result> { + self.inner.list_files_with_sizes().await + } + } + + fn test_store() -> (TempObjDir, Arc, Arc) { + let directory = TempObjDir::default(); + let cache = Arc::new(LanceCache::with_capacity(1024 * 1024)); + test_store_with_cache(directory, cache) + } + + fn eviction_test_store() -> (TempObjDir, Arc, Arc) { + let directory = TempObjDir::default(); + let cache = Arc::new(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(1024 * 1024), + ))); + test_store_with_cache(directory, cache) + } + + fn test_store_with_cache( + directory: TempObjDir, + cache: Arc, + ) -> (TempObjDir, Arc, Arc) { + let store = Arc::new(LanceIndexStore::new( + ObjectStore::local().into(), + directory.clone(), + cache.clone(), + )); + (directory, store, cache) + } + + async fn write_documents( + store: &dyn IndexStore, + path: &str, + addresses: UInt64Array, + lengths: UInt32Array, + total_tokens: Option<&str>, + ) { + let schema = Arc::new(Schema::new(vec![ + Field::new(ROW_ID, DataType::UInt64, addresses.null_count() != 0), + Field::new(NUM_TOKEN_COL, DataType::UInt32, lengths.null_count() != 0), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(addresses) as ArrayRef, + Arc::new(lengths) as ArrayRef, + ], + ) + .unwrap(); + let mut writer = store.new_index_file(path, schema).await.unwrap(); + writer.write_record_batch(batch).await.unwrap(); + if let Some(total_tokens) = total_tokens { + writer + .finish_with_metadata(HashMap::from([( + TOTAL_TOKENS_KEY.to_owned(), + total_tokens.to_owned(), + )])) + .await + .unwrap(); + } else { + writer.finish().await.unwrap(); + } + } + + async fn open_documents( + store: Arc, + path: &str, + index_cache: &LanceCache, + remapper: Option>, + ) -> Result { + let reader = store.open_index_file(path).await?; + PartitionDocuments::try_new( + store, + path.to_owned(), + 0, + WeakLanceCache::from(index_cache), + reader.as_ref(), + remapper, + false, + ) + } + + fn counted_store( + inner: Arc, + target: &str, + ) -> (Arc, Arc) { + let counts = Arc::new(DocumentReadCounts::default()); + ( + Arc::new(CountingStore { + inner, + target: target.to_owned(), + counts: counts.clone(), + fault: None, + }), + counts, + ) + } + + fn faulting_store( + inner: Arc, + target: &str, + action: usize, + ) -> (Arc, Arc) { + let fault = Arc::new(ReadFault { + action: AtomicUsize::new(action), + started: Notify::new(), + }); + ( + Arc::new(CountingStore { + inner, + target: target.to_owned(), + counts: Arc::new(DocumentReadCounts::default()), + fault: Some(fault.clone()), + }), + fault, + ) + } + + #[derive(Debug)] + struct TestRemapper { + mapping: HashMap>, + } + + impl RowIdRemapper for TestRemapper { + fn remap_row_id(&self, row_id: u64) -> Option { + self.mapping.get(&row_id).copied().unwrap_or(Some(row_id)) + } + + fn remap_row_addrs_tree_map(&self, _: &RowAddrTreeMap) -> RowAddrTreeMap { + unreachable!("not used by document projection tests") + } + + fn remap_row_ids_roaring_tree_map(&self, _: &RoaringTreemap) -> RoaringTreemap { + unreachable!("not used by document projection tests") + } + + fn remap_row_ids_record_batch(&self, _: RecordBatch, _: usize) -> Result { + unreachable!("not used by document projection tests") + } + } + + #[test] + fn required_document_columns_validate_name_and_type() { + let schema = Arc::new(Schema::new(vec![Field::new( + NUM_TOKEN_COL, + DataType::UInt64, + false, + )])); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(UInt64Array::from(vec![1])) as ArrayRef], + ) + .unwrap(); + + let missing = required_u64_column(&batch, ROW_ID, "docs.lance").unwrap_err(); + assert!( + missing + .to_string() + .contains("required column _rowid is missing") + ); + let wrong_type = required_u32_column(&batch, NUM_TOKEN_COL, "docs.lance").unwrap_err(); + assert!(wrong_type.to_string().contains("expected UInt32")); + } + + #[test] + fn live_documents_use_doc_ids() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 20, 30])), + live_docs: Some(RoaringBitmap::from_iter([0, 2])), + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let projection = projection.resident(None).unwrap(); + let selected = projection.live_doc_ids(); + assert_eq!(selected.iter().collect::>(), vec![0, 2]); + assert_eq!(projection.live_address_hull(), Some((10, 30))); + + let empty = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 20, 30])), + live_docs: Some(RoaringBitmap::new()), + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + assert_eq!(empty.resident(None).unwrap().live_address_hull(), None); + } + + #[test] + fn ordered_row_address_projection_maps_identity_domain() { + let addresses = Arc::new(UInt64Array::from(vec![10, 20, 30])); + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Shared { + len: addresses.len(), + }, + live_docs: None, + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let projection = projection.resident(Some(addresses)).unwrap(); + let ordered = projection.try_ordered_row_addresses().unwrap(); + + assert_eq!(ordered.len(), 3); + assert_eq!(ordered.live_len(), 3); + assert_eq!(ordered.live_address_hull(), Some((10, 30))); + assert_eq!( + ordered + .select_sorted_addresses(&[5, 10, 25, 30, 50]) + .iter() + .collect::>(), + vec![0, 2] + ); + assert!(ordered.select_sorted_addresses(&[]).is_empty()); + assert_eq!(ordered.address(0), Some(10)); + assert_eq!(ordered.address(2), Some(30)); + assert_eq!(ordered.address(3), None); + assert_eq!(ordered.lower_bound(0), Some(0)); + assert_eq!(ordered.lower_bound(10), Some(0)); + assert_eq!(ordered.lower_bound(11), Some(1)); + assert_eq!(ordered.lower_bound(30), Some(2)); + assert_eq!(ordered.lower_bound(31), None); + assert_eq!(ordered.next_address(0), Some(20)); + assert_eq!(ordered.next_address(1), Some(30)); + assert_eq!(ordered.next_address(2), None); + assert_eq!(ordered.next_address(u64::MAX), None); + } + + #[test] + fn ordered_row_address_projection_caches_successful_validation() { + let projection = resident_row_address_projection_for_test(vec![10, 20, 30, 40]); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Unknown + ); + assert_eq!(projection.ordered_validation_visited_docs(), 0); + + let first = projection.try_ordered_row_addresses().unwrap(); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + assert_eq!(projection.ordered_validation_visited_docs(), 4); + assert_eq!(first.lower_bound(25), Some(2)); + + let second_query = projection.projection.resident(None).unwrap(); + let second = second_query.try_ordered_row_addresses().unwrap(); + assert_eq!(second_query.ordered_validation_visited_docs(), 4); + assert_eq!(second.lower_bound(25), Some(2)); + } + + #[test] + fn ordered_validation_computes_before_short_cache_publication() { + let projection = resident_row_address_projection_for_test(vec![10, 20, 30, 40]); + + let validation = projection.compute_ordered_validation(); + assert_eq!(validation, Ok(())); + assert_eq!(projection.ordered_validation_visited_docs(), 4); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Unknown + ); + + projection.publish_ordered_validation(&validation); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + assert_eq!(projection.ordered_validation_visited_docs(), 4); + } + + #[test] + fn ordered_validation_concurrent_race_publishes_one_stable_state() { + let projection = resident_row_address_projection_for_test(vec![10, 20, 30, 40]); + let barrier = Arc::new(std::sync::Barrier::new(3)); + let handles = (0..2) + .map(|_| { + let projection = projection.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + projection.try_ordered_row_addresses().map(drop) + }) + }) + .collect::>(); + + barrier.wait(); + for handle in handles { + handle.join().unwrap().unwrap(); + } + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Ordered + ); + assert!(matches!( + projection.ordered_validation_visited_docs(), + 4 | 8 + )); + } + + #[test] + fn ordered_row_address_projection_skips_deleted_slots() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 0, 30, 0, 50])), + live_docs: Some(RoaringBitmap::from_iter([0, 2, 4])), + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let projection = projection.resident(None).unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 0); + let ordered = projection.try_ordered_row_addresses().unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 3); + projection.try_ordered_row_addresses().unwrap(); + assert_eq!(projection.ordered_validation_visited_docs(), 3); + + assert_eq!(ordered.len(), 5); + assert_eq!(ordered.live_len(), 3); + assert_eq!(ordered.live_address_hull(), Some((10, 50))); + assert_eq!( + ordered + .select_sorted_addresses(&[0, 10, 30, 40, 50, 60]) + .iter() + .collect::>(), + vec![0, 2, 4] + ); + assert_eq!(ordered.address(0), Some(10)); + assert_eq!(ordered.address(1), None); + assert_eq!(ordered.address(2), Some(30)); + assert_eq!(ordered.lower_bound(11), Some(2)); + assert_eq!(ordered.lower_bound(30), Some(2)); + assert_eq!(ordered.lower_bound(31), Some(4)); + assert_eq!(ordered.lower_bound(51), None); + assert_eq!(ordered.next_address(0), Some(30)); + assert_eq!(ordered.next_address(1), Some(30)); + assert_eq!(ordered.next_address(2), Some(50)); + assert_eq!(ordered.next_address(4), None); + assert_eq!( + ordered.address(1).or_else(|| ordered.next_address(1)), + Some(30) + ); + } + + #[test] + fn ordered_row_address_projection_rejects_nonmonotonic_remap() { + let raw = UInt64Array::from(vec![10, 20, 30, 40]); + let remapper = TestRemapper { + mapping: HashMap::from([(10, Some(100)), (20, None), (30, Some(300))]), + }; + let projection = Arc::new( + VersionAddressProjection::try_new(&raw, 4, Some(&remapper), "docs") + .expect("valid projection"), + ); + let projection = projection.resident(None).unwrap(); + + let expected = RowAddressProjectionOrderError::OutOfOrder { + previous_doc_id: DocId::new(2), + previous_address: RowAddress::new_from_u64(300), + doc_id: DocId::new(3), + address: RowAddress::new_from_u64(40), + }; + assert_eq!( + projection.try_ordered_row_addresses().unwrap_err(), + expected + ); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::OutOfOrder + ); + assert_eq!(projection.ordered_validation_visited_docs(), 3); + assert_eq!( + projection.try_ordered_row_addresses().unwrap_err(), + expected + ); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::OutOfOrder + ); + assert_eq!(projection.ordered_validation_visited_docs(), 6); + } + + #[test] + fn ordered_row_address_projection_rejects_duplicate_address() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 20, 20, 30])), + live_docs: None, + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let projection = projection.resident(None).unwrap(); + + let expected = RowAddressProjectionOrderError::Duplicate { + first_doc_id: DocId::new(1), + duplicate_doc_id: DocId::new(2), + address: RowAddress::new_from_u64(20), + }; + assert_eq!( + projection.try_ordered_row_addresses().unwrap_err(), + expected + ); + assert_eq!( + projection.cached_row_address_order(), + CachedRowAddressOrder::Duplicate + ); + assert_eq!(projection.ordered_validation_visited_docs(), 3); + assert_eq!( + projection.try_ordered_row_addresses().unwrap_err(), + expected + ); + assert_eq!(projection.ordered_validation_visited_docs(), 6); + } + + #[tokio::test] + async fn remap_preserves_doc_id_slots_and_filters_in_current_address_domain() { + let raw = UInt64Array::from(vec![10, 20, 30, 40]); + let remapper = TestRemapper { + mapping: HashMap::from([(10, Some(100)), (20, None), (30, Some(300))]), + }; + let projection = Arc::new( + VersionAddressProjection::try_new(&raw, 4, Some(&remapper), "docs") + .expect("valid projection"), + ); + let projection = projection.resident(None).unwrap(); + + assert_eq!(projection.address(DocId::new(0)), Some(100)); + assert_eq!(projection.address(DocId::new(1)), None); + assert_eq!(projection.address(DocId::new(2)), Some(300)); + assert_eq!(projection.address(DocId::new(3)), Some(40)); + + let all_live = projection.live_doc_ids(); + assert_eq!(all_live.iter().collect::>(), vec![0, 2, 3]); + assert_eq!(projection.live_address_hull(), Some((40, 300))); + + let allowed = Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + 100, 40, + ]))); + let DocVisibility::Selected(selected) = projection + .clone() + .materialize_visibility(allowed) + .await + .expect("valid allow-list") + else { + panic!("allow-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![0, 3]); + let candidate_selected = projection + .select_sorted_addresses(&[10, 40, 100]) + .await + .expect("valid candidate projection"); + assert_eq!(candidate_selected, selected); + + let first_lookup = projection + .doc_ids_by_address() + .await + .expect("cached address lookup"); + let blocked = Arc::new(RowAddrMask::from_block(RowAddrTreeMap::from_iter([300]))); + let DocVisibility::Selected(selected) = projection + .clone() + .materialize_visibility(blocked) + .await + .expect("valid block-list") + else { + panic!("block-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![0, 3]); + let second_lookup = projection + .doc_ids_by_address() + .await + .expect("cached address lookup"); + assert!(Arc::ptr_eq(&first_lookup, &second_lookup)); + } + + #[tokio::test] + async fn materialized_visibility_handles_unsorted_duplicate_addresses_and_full_fragments() { + let row_address = |fragment_id, row_offset| { + u64::from(RowAddress::new_from_parts(fragment_id, row_offset)) + }; + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![ + row_address(2, 5), + row_address(1, 2), + row_address(1, 2), + row_address(1, 4), + ])), + live_docs: None, + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let projection = projection.resident(None).unwrap(); + + let duplicate = projection + .select_sorted_addresses(&[row_address(1, 2)]) + .await + .unwrap_err(); + assert!(matches!(duplicate, Error::Index { .. })); + assert!( + duplicate + .to_string() + .contains("row address 4294967298 is shared by local documents 1 and 2") + ); + + let allowed = Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + row_address(1, 2), + row_address(1, 3), + row_address(1, 4), + ]))); + let DocVisibility::Selected(selected) = projection + .clone() + .materialize_visibility(allowed) + .await + .expect("valid allow-list") + else { + panic!("allow-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![1, 2, 3]); + + let mut full_fragment = RowAddrTreeMap::new(); + full_fragment.insert_fragment(2); + let DocVisibility::Selected(selected) = projection + .materialize_visibility(Arc::new(RowAddrMask::from_allowed(full_fragment))) + .await + .expect("valid full-fragment allow-list") + else { + panic!("allow-list must compile to DocIds") + }; + assert_eq!(selected.iter().collect::>(), vec![0]); + } + + #[tokio::test] + async fn candidate_projection_ignores_deleted_duplicate_addresses() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![30, 10, 10, 20])), + live_docs: Some(RoaringBitmap::from_iter([0, 1, 3])), + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let projection = projection.resident(None).unwrap(); + + assert_eq!(projection.live_address_hull(), Some((10, 30))); + let selected = projection + .select_sorted_addresses(&[10, 20, 25, 30]) + .await + .expect("deleted duplicate does not collide"); + assert_eq!(selected.iter().collect::>(), vec![0, 1, 3]); + } + + #[test] + fn lazy_visibility_projects_only_candidate_doc_ids() { + let projection = Arc::new(VersionAddressProjection { + addresses: AddressValues::Owned(Arc::new(vec![10, 20, 30])), + live_docs: Some(RoaringBitmap::from_iter([0, 2])), + doc_ids_by_address: OnceCell::new(), + ordered_validation: AtomicU8::new(CachedRowAddressOrder::Unknown as u8), + materialized_candidate_cost: AtomicUsize::new(0), + ordered_validation_visited_docs: AtomicUsize::new(0), + }); + let resident = projection.resident(None).unwrap(); + let visibility = DocVisibility::Filtered { + projection: resident, + mask: Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + 20, 30, + ]))), + }; + + assert!(!visibility.selected(DocId::new(0))); + assert!(!visibility.selected(DocId::new(1))); + assert!(visibility.selected(DocId::new(2))); + assert!(!projection.doc_ids_by_address.initialized()); + } + + #[test] + fn doc_lengths_validate_shape_total_and_memory() { + let mismatch = DocLengths::try_new(ScalarBuffer::from(vec![2, 3]), 3, None, false, "docs") + .unwrap_err(); + assert!(mismatch.to_string().contains("2 rows")); + + let mismatch = DocLengths::try_new( + ScalarBuffer::from(vec![2, 3, 5]), + 3, + Some(11), + false, + "docs", + ) + .unwrap_err(); + assert!(mismatch.to_string().contains("sums to 10")); + + let lengths = + DocLengths::try_new(ScalarBuffer::from(vec![2, 3, 5]), 3, Some(10), true, "docs") + .unwrap(); + let before_norms = lengths.deep_size_of(); + assert_eq!(lengths.total_tokens(), 10); + assert_eq!(lengths.scoring_norms().unwrap().len(), 3); + assert_eq!(lengths.deep_size_of() - before_norms, 3); + } + + #[tokio::test] + async fn persisted_stats_are_footer_only_and_compatible_with_full_docset_reader() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + + let reader = store.open_index_file(path).await.unwrap(); + assert_eq!( + reader.schema().metadata.get(TOTAL_TOKENS_KEY), + Some(&"10".to_owned()) + ); + let complete = DocSet::load(reader, false, None).await.unwrap(); + assert_eq!(complete.len(), 3); + assert_eq!(complete.row_id(1), 20); + assert_eq!(complete.total_tokens_num(), 10); + + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + assert_eq!( + documents.stats().await.unwrap(), + PartitionStats { + num_docs: 3, + total_tokens: 10, + } + ); + assert_eq!(counts.rows.load(Ordering::Relaxed), 0); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn missing_stats_fall_back_once_to_lengths() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + None, + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + assert_eq!(documents.stats().await.unwrap().total_tokens, 10); + assert_eq!(documents.stats().await.unwrap().total_tokens, 10); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 0); + assert!(documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn prewarm_loads_document_columns_and_address_lookup_once() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + futures::future::join_all((0..8).map(|_| documents.prewarm())) + .await + .into_iter() + .collect::>>() + .unwrap(); + let projection = documents.address_projection().await.unwrap(); + let first_lookup = projection.doc_ids_by_address().await.unwrap(); + documents.prewarm().await.unwrap(); + let second_lookup = documents + .address_projection() + .await + .unwrap() + .doc_ids_by_address() + .await + .unwrap(); + + assert!(documents.lengths_loaded()); + assert!(documents.projection_loaded()); + assert!(documents.query_ready()); + assert_eq!(documents.cached_lengths().unwrap().total_tokens(), 10); + assert!(matches!( + documents.immediate_visibility(Arc::new(RowAddrMask::all_rows()), false), + Some(DocVisibility::All) + )); + assert!(matches!( + first_lookup.as_ref(), + AddressDocIdLookup::Identity + )); + assert!(Arc::ptr_eq(&first_lookup, &second_lookup)); + assert_eq!( + documents.cached_row_addresses(&[DocId::new(1)]).unwrap(), + Some(vec![20]) + ); + assert_eq!(counts.open_calls.load(Ordering::Relaxed), 2); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 3); + assert!( + cache + .get_with_key(&DocRowIdsKey { partition_id: 0 }) + .await + .is_some() + ); + } + + #[tokio::test] + async fn filtered_visibility_releases_addresses_after_cache_eviction() { + let (_directory, store, cache) = eviction_test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + let mask = Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([20]))); + + let visibility = documents.visibility(mask.clone(), false).await.unwrap(); + assert!(!visibility.selected(DocId::new(0))); + assert!(visibility.selected(DocId::new(1))); + assert!(!visibility.selected(DocId::new(2))); + + let weak_addresses = documents.address_buffer_handle(); + + cache.clear().await; + assert!(weak_addresses.upgrade().is_some()); + assert!(documents.projection_resident()); + + drop(visibility); + assert!(weak_addresses.upgrade().is_none()); + assert!(!documents.projection_resident()); + + let reloaded = documents.visibility(mask, false).await.unwrap(); + assert!(reloaded.selected(DocId::new(1))); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 6); + } + + #[tokio::test] + async fn prewarm_reloads_addresses_after_cache_eviction() { + let (_directory, store, cache) = eviction_test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + let weak_addresses = documents.address_buffer_handle(); + + cache.clear().await; + assert!(weak_addresses.upgrade().is_none()); + assert!(documents.projection_loaded()); + assert!(!documents.projection_resident()); + assert!(!documents.query_ready()); + + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + assert_eq!( + documents + .cached_row_addresses(&[DocId::new(2), DocId::new(0)]) + .unwrap(), + Some(vec![30, 10]) + ); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 6); + } + + #[tokio::test] + async fn prewarm_materializes_quantized_norms_before_becoming_query_ready() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 300, 5]), + Some("307"), + ) + .await; + let reader = store.open_index_file(path).await.unwrap(); + let documents = PartitionDocuments::try_new( + store, + path.to_owned(), + 0, + WeakLanceCache::from(cache.as_ref()), + reader.as_ref(), + None, + true, + ) + .unwrap(); + + assert!(!documents.query_ready()); + documents.prewarm().await.unwrap(); + let lengths = documents.lengths().await.unwrap(); + assert!(lengths.scoring_ready()); + assert_eq!(lengths.scoring_norms().unwrap().len(), 3); + assert!(documents.query_ready()); + } + + #[tokio::test] + async fn cancelled_or_failed_prewarm_can_retry_without_partial_publication() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + + let (pausing, pause) = faulting_store(store.clone(), path, PAUSE_ONCE); + let documents = Arc::new( + open_documents(pausing, path, cache.as_ref(), None) + .await + .unwrap(), + ); + let task = tokio::spawn({ + let documents = documents.clone(); + async move { documents.prewarm().await } + }); + tokio::time::timeout(Duration::from_secs(5), pause.started.notified()) + .await + .expect("prewarm should reach the injected pending read"); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + assert!(!documents.query_ready()); + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + + let (failing, _fault) = faulting_store(store, path, FAIL_ONCE); + let documents = open_documents(failing, path, cache.as_ref(), None) + .await + .unwrap(); + let error = documents.prewarm().await.unwrap_err(); + assert!(error.to_string().contains("injected document read failure")); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + assert!(!documents.query_ready()); + documents.prewarm().await.unwrap(); + assert!(documents.query_ready()); + } + + #[tokio::test] + async fn prewarm_reuses_an_already_loaded_document_column() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + documents.lengths().await.unwrap(); + documents.prewarm().await.unwrap(); + documents.prewarm().await.unwrap(); + + assert_eq!(counts.open_calls.load(Ordering::Relaxed), 3); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 3); + } + + #[tokio::test] + async fn invalid_or_mismatched_stats_are_corruption_not_fallback() { + let (_directory, store, cache) = test_store(); + write_documents( + store.as_ref(), + "invalid.lance", + UInt64Array::from(vec![10]), + UInt32Array::from(vec![2]), + Some("not-a-u64"), + ) + .await; + let error = open_documents(store.clone(), "invalid.lance", cache.as_ref(), None) + .await + .unwrap_err(); + assert!(error.to_string().contains("invalid total_tokens")); + + write_documents( + store.as_ref(), + "mismatch.lance", + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("11"), + ) + .await; + let (counting, counts) = counted_store(store, "mismatch.lance"); + let documents = open_documents(counting, "mismatch.lance", cache.as_ref(), None) + .await + .unwrap(); + assert_eq!(documents.stats().await.unwrap().total_tokens, 11); + let error = documents.lengths().await.unwrap_err(); + assert!(error.to_string().contains("sums to 10")); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), 3); + assert!(!documents.lengths_loaded()); + } + + #[tokio::test] + async fn final_address_resolution_reuses_the_cached_row_id_column() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + let num_docs = 600_u64; + write_documents( + store.as_ref(), + path, + UInt64Array::from_iter_values((0..num_docs).map(|id| id + 1_000)), + UInt32Array::from_iter_values((0..num_docs).map(|_| 1)), + Some("600"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + + assert!(documents.resolve_addresses(&[]).await.unwrap().is_empty()); + assert_eq!(counts.rows.load(Ordering::Relaxed), 0); + + let point_ids = [DocId::new(5), DocId::new(6), DocId::new(10), DocId::new(5)]; + assert_eq!( + documents.estimated_address_read_bytes(&point_ids), + num_docs as usize * std::mem::size_of::() + ); + assert_eq!( + documents.resolve_addresses(&point_ids).await.unwrap(), + vec![1005, 1006, 1010, 1005] + ); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 600); + assert!( + cache + .get_with_key(&DocRowIdsKey { partition_id: 0 }) + .await + .is_some() + ); + + let bulk_ids = (0..=512).step_by(2).map(DocId::new).collect::>(); + assert_eq!( + documents.estimated_address_read_bytes(&bulk_ids), + num_docs as usize * std::mem::size_of::() + ); + let resolved = documents.resolve_addresses(&bulk_ids).await.unwrap(); + assert_eq!(resolved.first(), Some(&1000)); + assert_eq!(resolved.last(), Some(&1512)); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 600); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn selective_scoring_lengths_read_only_candidate_rows() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + let num_docs = 600_u32; + write_documents( + store.as_ref(), + path, + UInt64Array::from_iter_values((0..num_docs).map(u64::from)), + UInt32Array::from_iter_values(1..=num_docs), + Some( + &u64::from(num_docs) + .saturating_mul(u64::from(num_docs + 1)) + .div_ceil(2) + .to_string(), + ), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + let doc_ids = [DocId::new(2), DocId::new(10), DocId::new(2)]; + + assert_eq!( + documents.resolve_scoring_lengths(&doc_ids).await.unwrap(), + vec![3, 11, 3] + ); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), doc_ids.len()); + assert!(!documents.lengths_loaded()); + } + + #[tokio::test] + async fn selective_scoring_documents_share_one_candidate_read() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + let num_docs = 600_u32; + write_documents( + store.as_ref(), + path, + UInt64Array::from_iter_values((0..num_docs).map(|doc_id| 10_000 + u64::from(doc_id))), + UInt32Array::from_iter_values((0..num_docs).map(|doc_id| doc_id + 1)), + Some("180300"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let documents = open_documents(counting, path, cache.as_ref(), None) + .await + .unwrap(); + let doc_ids = [DocId::new(2), DocId::new(10), DocId::new(2)]; + + assert_eq!( + documents.resolve_scoring_documents(&doc_ids).await.unwrap(), + vec![(2, 10_002, 3), (10, 10_010, 11), (2, 10_002, 3)] + ); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), doc_ids.len()); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), doc_ids.len()); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn selective_scoring_documents_preserve_quantized_lengths() { + let (_directory, store, cache) = test_store(); + let path = "docs.lance"; + let num_docs = 600_u32; + let lengths = (0..num_docs) + .map(|doc_id| if doc_id == 10 { 300 } else { doc_id + 1 }) + .collect::>(); + let total_tokens = lengths + .iter() + .map(|&length| u64::from(length)) + .sum::() + .to_string(); + write_documents( + store.as_ref(), + path, + UInt64Array::from_iter_values((0..num_docs).map(|doc_id| 20_000 + u64::from(doc_id))), + UInt32Array::from(lengths.clone()), + Some(&total_tokens), + ) + .await; + let (counting, counts) = counted_store(store, path); + let reader = counting.open_index_file(path).await.unwrap(); + let documents = PartitionDocuments::try_new( + counting, + path.to_owned(), + 0, + WeakLanceCache::from(cache.as_ref()), + reader.as_ref(), + None, + true, + ) + .unwrap(); + let doc_ids = [DocId::new(10), DocId::new(500)]; + + assert_eq!( + documents.resolve_scoring_documents(&doc_ids).await.unwrap(), + vec![ + ( + 10, + 20_010, + dequantize_doc_length(quantize_doc_length(lengths[10])), + ), + ( + 500, + 20_500, + dequantize_doc_length(quantize_doc_length(lengths[500])), + ), + ] + ); + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 1); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), doc_ids.len()); + assert_eq!(counts.length_rows.load(Ordering::Relaxed), doc_ids.len()); + assert!(!documents.lengths_loaded()); + } + + #[tokio::test] + async fn final_address_resolution_reloads_after_cache_eviction() { + let (_directory, store, _cache) = test_store(); + let path = "docs.lance"; + write_documents( + store.as_ref(), + path, + UInt64Array::from(vec![10, 20, 30]), + UInt32Array::from(vec![2, 3, 5]), + Some("10"), + ) + .await; + let (counting, counts) = counted_store(store, path); + let no_retention_cache = LanceCache::no_cache(); + let documents = open_documents(counting, path, &no_retention_cache, None) + .await + .unwrap(); + + for _ in 0..2 { + assert_eq!( + documents + .resolve_addresses(&[DocId::new(2), DocId::new(0)]) + .await + .unwrap(), + vec![30, 10] + ); + } + assert_eq!(counts.ranges_calls.load(Ordering::Relaxed), 0); + assert_eq!(counts.range_calls.load(Ordering::Relaxed), 2); + assert_eq!(counts.address_rows.load(Ordering::Relaxed), 6); + assert!(!documents.projection_loaded()); + } + + #[tokio::test] + async fn document_column_nulls_are_reported_as_corruption() { + let (_directory, store, cache) = test_store(); + write_documents( + store.as_ref(), + "null-length.lance", + UInt64Array::from(vec![Some(10), Some(20)]), + UInt32Array::from(vec![Some(2), None]), + Some("2"), + ) + .await; + let documents = open_documents(store.clone(), "null-length.lance", cache.as_ref(), None) + .await + .unwrap(); + assert!( + documents + .lengths() + .await + .unwrap_err() + .to_string() + .contains("_num_tokens contains null") + ); + + write_documents( + store.as_ref(), + "null-address.lance", + UInt64Array::from(vec![Some(10), None]), + UInt32Array::from(vec![2, 3]), + Some("5"), + ) + .await; + let documents = open_documents(store, "null-address.lance", cache.as_ref(), None) + .await + .unwrap(); + assert!( + documents + .resolve_addresses(&[DocId::new(1)]) + .await + .unwrap_err() + .to_string() + .contains("_rowid contains null") + ); + assert!( + documents + .prewarm() + .await + .unwrap_err() + .to_string() + .contains("_rowid contains null") + ); + assert!(!documents.lengths_loaded()); + assert!(!documents.projection_loaded()); + } +} diff --git a/rust/lance-index/src/scalar/inverted/encoding.rs b/rust/lance-index/src/scalar/inverted/encoding.rs index d75fa742ee7..6d57b032650 100644 --- a/rust/lance-index/src/scalar/inverted/encoding.rs +++ b/rust/lance-index/src/scalar/inverted/encoding.rs @@ -5,10 +5,15 @@ use std::io::Write; use super::builder::BLOCK_SIZE; use super::index::{PositionStreamCodec, PostingTailCodec}; +#[cfg(test)] +use super::tokenizer::LEGACY_BLOCK_SIZE; +use super::tokenizer::validate_block_size; use arrow::array::LargeBinaryBuilder; -use lance_bitpacking::{BitPacker, BitPacker4x}; +use lance_bitpacking::{BitPacker, BitPacker4x, BitPacker8x}; use lance_core::{Error, Result}; +pub const MAX_POSTING_BLOCK_SIZE: usize = BitPacker8x::BLOCK_LEN; + // we compress the posting list to multiple blocks of fixed number of elements (BLOCK_SIZE), // returns a LargeBinaryArray, where each binary is a compressed block (128 row ids + 128 frequencies) // each block is: @@ -41,18 +46,40 @@ pub fn compress_posting_list<'a>( #[cfg(test)] pub fn compress_posting_list_with_tail_codec<'a>( + length: usize, + doc_ids: impl Iterator, + frequencies: impl Iterator, + block_max_scores: impl Iterator, + tail_codec: PostingTailCodec, +) -> Result { + compress_posting_list_with_tail_codec_and_block_size( + length, + doc_ids, + frequencies, + block_max_scores, + tail_codec, + LEGACY_BLOCK_SIZE, + ) +} + +#[cfg(test)] +pub fn compress_posting_list_with_tail_codec_and_block_size<'a>( length: usize, doc_ids: impl Iterator, frequencies: impl Iterator, mut block_max_scores: impl Iterator, tail_codec: PostingTailCodec, + block_size: usize, ) -> Result { - if length < BLOCK_SIZE { + let block_size = validate_block_size(block_size)?; + if length < block_size { // directly do remainder compression to avoid overhead of creating buffer let mut builder = LargeBinaryBuilder::with_capacity(1, length * 4 * 2 + 1); - // write the max score of the block - let max_score = block_max_scores.next().unwrap(); - let _ = builder.write(max_score.to_le_bytes().as_ref())?; + // write the max score of the block (128-doc blocks only) + if posting_block_score_prefix_len(block_size) > 0 { + let max_score = block_max_scores.next().unwrap(); + let _ = builder.write(max_score.to_le_bytes().as_ref())?; + } compress_posting_remainder( doc_ids.copied().collect::>().as_slice(), frequencies.copied().collect::>().as_slice(), @@ -63,27 +90,25 @@ pub fn compress_posting_list_with_tail_codec<'a>( return Ok(builder.finish()); } - let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(BLOCK_SIZE), length * 3); - let mut buffer = [0u8; BLOCK_SIZE * 4 + 5]; - let mut doc_id_buffer = Vec::with_capacity(BLOCK_SIZE); - let mut freq_buffer = Vec::with_capacity(BLOCK_SIZE); + let mut builder = LargeBinaryBuilder::with_capacity(length.div_ceil(block_size), length * 3); + let mut doc_id_buffer = Vec::with_capacity(block_size); + let mut freq_buffer = Vec::with_capacity(block_size); for (doc_id, freq) in std::iter::zip(doc_ids, frequencies) { doc_id_buffer.push(*doc_id); freq_buffer.push(*freq); - if doc_id_buffer.len() < BLOCK_SIZE { + if doc_id_buffer.len() < block_size { continue; } - assert_eq!(doc_id_buffer.len(), BLOCK_SIZE); + assert_eq!(doc_id_buffer.len(), block_size); - // write the max score of the block - let max_score = block_max_scores.next().unwrap(); - let _ = builder.write(max_score.to_le_bytes().as_ref())?; - // delta encoding + bitpacking for doc ids - compress_sorted_block(&doc_id_buffer, &mut buffer, &mut builder)?; - // bitpacking for frequencies - compress_block(&freq_buffer, &mut buffer, &mut builder)?; + // write the max score of the block (128-doc blocks only) + if posting_block_score_prefix_len(block_size) > 0 { + let max_score = block_max_scores.next().unwrap(); + let _ = builder.write(max_score.to_le_bytes().as_ref())?; + } + encode_posting_block_payload(&doc_id_buffer, &freq_buffer, &mut builder)?; builder.append_value(""); doc_id_buffer.clear(); freq_buffer.clear(); @@ -91,44 +116,187 @@ pub fn compress_posting_list_with_tail_codec<'a>( // we don't compress the last block if it is not full if !doc_id_buffer.is_empty() { - // write the max score of the block - let max_score = block_max_scores.next().unwrap(); - let _ = builder.write(max_score.to_le_bytes().as_ref())?; + // write the max score of the block (128-doc blocks only) + if posting_block_score_prefix_len(block_size) > 0 { + let max_score = block_max_scores.next().unwrap(); + let _ = builder.write(max_score.to_le_bytes().as_ref())?; + } compress_posting_remainder(&doc_id_buffer, &freq_buffer, tail_codec, &mut builder)?; builder.append_value(""); } Ok(builder.finish()) } +/// Byte length of the block-max-score prefix on posting blocks. 128-doc +/// blocks store a per-block max score, patched in at build time; 256-document +/// blocks always carry impact skip data, which supersedes it, so they +/// store none. +#[inline] +pub fn posting_block_score_prefix_len(block_size: usize) -> usize { + if block_size == MAX_POSTING_BLOCK_SIZE { + 0 + } else { + 4 + } +} + pub fn encode_full_posting_block_into( doc_ids: &[u32], frequencies: &[u32], block: &mut Vec, ) -> Result<()> { - debug_assert_eq!(doc_ids.len(), BLOCK_SIZE); - debug_assert_eq!(frequencies.len(), BLOCK_SIZE); - block.extend_from_slice(&0f32.to_le_bytes()); - let mut buffer = [0u8; BLOCK_SIZE * 4 + 5]; - compress_sorted_block(doc_ids, &mut buffer, block)?; - compress_block(frequencies, &mut buffer, block)?; + validate_block_size(doc_ids.len())?; + debug_assert_eq!(doc_ids.len(), frequencies.len()); + if posting_block_score_prefix_len(doc_ids.len()) > 0 { + block.extend_from_slice(&0f32.to_le_bytes()); + } + encode_posting_block_payload(doc_ids, frequencies, block)?; Ok(()) } +fn encode_posting_block_payload( + doc_ids: &[u32], + frequencies: &[u32], + block: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(doc_ids.len(), frequencies.len()); + validate_block_size(doc_ids.len())?; + let mut buffer = [0u8; MAX_POSTING_BLOCK_SIZE * 4 + 5]; + match doc_ids.len() { + BitPacker4x::BLOCK_LEN => { + compress_sorted_block_with::(doc_ids, &mut buffer, block)?; + compress_block_with::(frequencies, &mut buffer, block)?; + } + // 256-document blocks store frequencies with patched FOR: + // outliers no longer widen the whole block, which matters because one + // large tf per block otherwise doubles the frequency payload. + BitPacker8x::BLOCK_LEN => { + compress_sorted_block_with::(doc_ids, &mut buffer, block)?; + compress_pfor_block_with::(frequencies, &mut buffer, block)?; + } + _ => unreachable!("validated posting block size should be supported"), + } + Ok(()) +} + +/// Patched FOR (Lucene PForUtil style): pick the body bit width that +/// minimizes total bytes, pack all values masked to that width, and append +/// up to [`PFOR_MAX_EXCEPTIONS`] exceptions as (index u8, high-bits varint). +const PFOR_MAX_EXCEPTIONS: usize = 31; + +#[inline] +fn u32_bits(value: u32) -> usize { + (32 - value.leading_zeros()) as usize +} + +#[inline] +fn varint_u32_len(value: u32) -> usize { + u32_bits(value).max(1).div_ceil(7) +} + +fn compress_pfor_block_with( + data: &[u32], + buffer: &mut [u8], + builder: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(data.len(), P::BLOCK_LEN); + let max_bits = data.iter().map(|&v| u32_bits(v)).max().unwrap_or(0); + let mut best_width = max_bits; + let mut best_cost = P::BLOCK_LEN * max_bits / 8; + for width in (0..max_bits).rev() { + let mut exceptions = 0usize; + let mut exception_bytes = 0usize; + for &value in data { + if u32_bits(value) > width { + exceptions += 1; + exception_bytes += 1 + varint_u32_len(value >> width); + } + } + if exceptions > PFOR_MAX_EXCEPTIONS { + break; + } + let cost = P::BLOCK_LEN * width / 8 + exception_bytes; + if cost < best_cost { + best_cost = cost; + best_width = width; + } + } + + let mask = if best_width >= 32 { + u32::MAX + } else { + (1u32 << best_width) - 1 + }; + let mut body = [0u32; MAX_POSTING_BLOCK_SIZE]; + let mut exception_buf = Vec::new(); + let mut exception_count = 0u8; + for (index, &value) in data.iter().enumerate() { + body[index] = value & mask; + if u32_bits(value) > best_width { + exception_buf.push(index as u8); + encode_varint_u32(&mut exception_buf, value >> best_width); + exception_count += 1; + } + } + let compressor = P::new(); + let num_bytes = compressor.compress(&body[..P::BLOCK_LEN], buffer, best_width as u8); + let _ = builder.write(&[best_width as u8, exception_count])?; + let _ = builder.write(&buffer[..num_bytes])?; + let _ = builder.write(&exception_buf)?; + Ok(()) +} + +fn decompress_pfor_block_with( + block: &[u8], + buffer: &mut [u32], + res: &mut Vec, +) -> usize { + debug_assert!(buffer.len() >= P::BLOCK_LEN); + let buffer = &mut buffer[..P::BLOCK_LEN]; + let width = block[0]; + let exception_count = block[1] as usize; + let compressor = P::new(); + let num_bytes = compressor.decompress(&block[2..], buffer, width); + let mut offset = 2 + num_bytes; + for _ in 0..exception_count { + let index = block[offset] as usize; + offset += 1; + let high = decode_varint_u32(block, &mut offset) + .expect("pfor exception high bits should be a valid varint"); + buffer[index] |= high << width; + } + res.extend_from_slice(buffer); + offset +} + pub fn encode_remainder_posting_block_into( doc_ids: &[u32], frequencies: &[u32], codec: PostingTailCodec, + block_size: usize, block: &mut Vec, ) -> Result<()> { debug_assert_eq!(doc_ids.len(), frequencies.len()); - block.extend_from_slice(&0f32.to_le_bytes()); + if posting_block_score_prefix_len(block_size) > 0 { + block.extend_from_slice(&0f32.to_le_bytes()); + } compress_posting_remainder(doc_ids, frequencies, codec, block)?; Ok(()) } #[inline] fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> { - let compressor = BitPacker4x::new(); + compress_sorted_block_with::(data, buffer, builder) +} + +#[inline] +fn compress_sorted_block_with( + data: &[u32], + buffer: &mut [u8], + builder: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(data.len(), P::BLOCK_LEN); + let compressor = P::new(); let num_bits = compressor.num_bits_sorted(data[0], data); let num_bytes = compressor.compress_sorted(data[0], data, buffer, num_bits); let _ = builder.write(data[0].to_le_bytes().as_ref())?; @@ -139,7 +307,17 @@ fn compress_sorted_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Wri #[inline] fn compress_block(data: &[u32], buffer: &mut [u8], builder: &mut impl Write) -> Result<()> { - let compressor = BitPacker4x::new(); + compress_block_with::(data, buffer, builder) +} + +#[inline] +fn compress_block_with( + data: &[u32], + buffer: &mut [u8], + builder: &mut impl Write, +) -> Result<()> { + debug_assert_eq!(data.len(), P::BLOCK_LEN); + let compressor = P::new(); let num_bits = compressor.num_bits(data); let num_bytes = compressor.compress(data, buffer, num_bits); let _ = builder.write(&[num_bits])?; @@ -236,7 +414,7 @@ pub fn compress_positions(positions: &[u32]) -> Result, mut value: u32) { +pub fn encode_varint_u32(dst: &mut Vec, mut value: u32) { while value >= 0x80 { dst.push((value as u8) | 0x80); value >>= 7; @@ -244,39 +422,6 @@ fn encode_varint_u32(dst: &mut Vec, mut value: u32) { dst.push(value as u8); } -/// Encode a monotonically increasing sequence of `group_starts` (the first -/// row of each posting-list cache group) as varint-encoded deltas. The first -/// value is stored as-is and each subsequent value as its delta from the -/// previous one; since deltas are group sizes (1..=cap) they fit in ~1 byte, -/// keeping the buffer tiny even for indexes with millions of tokens. See -/// issue #7040. -pub(super) fn encode_group_starts(group_starts: &[u32]) -> Vec { - let mut dst = Vec::with_capacity(group_starts.len()); - let mut previous = 0u32; - for &start in group_starts { - debug_assert!(start >= previous, "group_starts must be monotonic"); - encode_varint_u32(&mut dst, start - previous); - previous = start; - } - dst -} - -/// Decode the buffer produced by [`encode_group_starts`] back into the -/// absolute `group_starts` values. -pub(super) fn decode_group_starts(src: &[u8]) -> Result> { - let mut group_starts = Vec::new(); - let mut offset = 0; - let mut previous = 0u32; - while offset < src.len() { - let delta = decode_varint_u32(src, &mut offset)?; - previous = previous - .checked_add(delta) - .ok_or_else(|| Error::index("group_starts delta decode overflowed u32".to_owned()))?; - group_starts.push(previous); - } - Ok(group_starts) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct PositionBlockBuilder { codec: PositionStreamCodec, @@ -379,7 +524,7 @@ impl PositionBlockBuilder { } #[inline] -fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result { +pub fn decode_varint_u32(src: &[u8], offset: &mut usize) -> Result { let mut value = 0u32; let mut shift = 0u32; while *offset < src.len() { @@ -557,15 +702,10 @@ fn decode_position_stream_packed_block( let mut deltas = Vec::with_capacity(total_positions); for _ in 0..full_delta_blocks { - if packed_offset >= src.len() { - return Err(Error::index( - "unexpected EOF while decoding packed position stream".to_owned(), - )); - } - let num_bits = src[packed_offset]; - packed_offset += 1; - let consumed = compressor.decompress(&src[packed_offset..], &mut packed_values, num_bits); - packed_offset += consumed; + let (num_bits, payload, next_offset) = packed_position_group(src, packed_offset)?; + let consumed = compressor.decompress(payload, &mut packed_values, num_bits); + debug_assert_eq!(consumed, payload.len()); + packed_offset = next_offset; deltas.extend_from_slice(&packed_values); } @@ -601,6 +741,144 @@ fn decode_position_stream_packed_block( Ok(()) } +fn packed_position_group(src: &[u8], offset: usize) -> Result<(u8, &[u8], usize)> { + let num_bits = *src.get(offset).ok_or_else(|| { + Error::index(format!( + "unexpected EOF reading packed position group header at byte offset {offset}; \ + stream length is {}", + src.len() + )) + })?; + if num_bits > u32::BITS as u8 { + return Err(Error::index(format!( + "invalid packed position group bit width {num_bits} at byte offset {offset}; \ + expected at most {}", + u32::BITS + ))); + } + + let payload_start = offset + .checked_add(1) + .ok_or_else(|| Error::index("packed position group offset overflow".to_owned()))?; + let payload_len = usize::from(num_bits) * BLOCK_SIZE / 8; + let payload_end = payload_start + .checked_add(payload_len) + .ok_or_else(|| Error::index("packed position group length overflow".to_owned()))?; + let payload = src.get(payload_start..payload_end).ok_or_else(|| { + Error::index(format!( + "unexpected EOF reading packed position group payload at byte offset {offset}; \ + need {payload_len} bytes after the header but stream length is {}", + src.len() + )) + })?; + Ok((num_bits, payload, payload_end)) +} + +/// Decode one document's positions out of a PackedDelta position block +/// without decoding the rest of the block. Full 128-delta groups are +/// self-describing (`[num_bits u8][16 * num_bits packed bytes]`), so group +/// byte offsets are recovered by hopping headers — no format change is +/// involved. `delta_range` is the doc's range in the block-wide delta stream +/// (from the frequency prefix sums); per-doc deltas reset at document +/// boundaries, so decoding starts cleanly at `delta_range.start`. +/// +/// The caller passes per-block scratch state that this function maintains: +/// `group_offsets` (lazily extended header index, seeded with `[0]`), +/// `unpacked_group`/`unpacked_group_idx` (the last unpacked group), and +/// `tail_cache` (the varint tail, decoded in full on first touch). All are +/// reset by the caller when the block cursor moves. +#[allow(clippy::too_many_arguments)] +pub(super) fn seek_packed_doc_positions( + src: &[u8], + total_deltas: usize, + delta_range: std::ops::Range, + group_offsets: &mut Vec, + unpacked_group: &mut [u32; BLOCK_SIZE], + unpacked_group_idx: &mut Option, + tail_cache: &mut Vec, + dst: &mut Vec, +) -> Result<()> { + dst.clear(); + if delta_range.start > delta_range.end || delta_range.end > total_deltas { + return Err(Error::index(format!( + "invalid packed position delta range {}..{} for {total_deltas} total deltas", + delta_range.start, delta_range.end + ))); + } + if delta_range.is_empty() { + return Ok(()); + } + let num_full_groups = total_deltas / BLOCK_SIZE; + let packed_deltas_end = num_full_groups * BLOCK_SIZE; + + // Extend the header index far enough for this range (tail needs the + // offset one past the last full group). + let last_needed_group = if delta_range.end > packed_deltas_end { + num_full_groups + } else { + (delta_range.end - 1) / BLOCK_SIZE + }; + while group_offsets.len() <= last_needed_group { + let last = *group_offsets + .last() + .ok_or_else(|| Error::index("packed position group offsets are empty".to_owned()))?; + let (_, _, next_offset) = packed_position_group(src, last)?; + group_offsets.push(next_offset); + } + + let mut previous = 0u32; + let mut first = true; + let mut push_delta = |delta: u32, dst: &mut Vec| -> Result<()> { + let position = if first { + first = false; + delta + } else { + previous + .checked_add(delta) + .ok_or_else(|| Error::index("position stream overflow while decoding".to_owned()))? + }; + dst.push(position); + previous = position; + Ok(()) + }; + + for index in delta_range.start..delta_range.end.min(packed_deltas_end) { + let group = index / BLOCK_SIZE; + if *unpacked_group_idx != Some(group) { + let offset = *group_offsets.get(group).ok_or_else(|| { + Error::index(format!( + "missing packed position group offset for group {group}; have {} offsets", + group_offsets.len() + )) + })?; + let (num_bits, payload, _) = packed_position_group(src, offset)?; + BitPacker4x::new().decompress(payload, unpacked_group, num_bits); + *unpacked_group_idx = Some(group); + } + push_delta(unpacked_group[index % BLOCK_SIZE], dst)?; + } + + if delta_range.end > packed_deltas_end { + let tail_len = total_deltas - packed_deltas_end; + if tail_cache.len() != tail_len { + tail_cache.clear(); + tail_cache.reserve(tail_len); + let mut offset = *group_offsets.get(num_full_groups).ok_or_else(|| { + Error::index(format!( + "missing packed position tail offset after {num_full_groups} full groups" + )) + })?; + for _ in 0..tail_len { + tail_cache.push(decode_varint_u32(src, &mut offset)?); + } + } + for index in delta_range.start.max(packed_deltas_end)..delta_range.end { + push_delta(tail_cache[index - packed_deltas_end], dst)?; + } + } + Ok(()) +} + #[cfg(test)] pub fn encode_position_stream_block_into( positions: &[u32], @@ -650,23 +928,46 @@ pub fn decompress_posting_list_with_tail_codec( posting_list: &arrow::array::LargeBinaryArray, tail_codec: PostingTailCodec, ) -> Result<(Vec, Vec)> { + decompress_posting_list_with_tail_codec_and_block_size( + num_docs, + posting_list, + tail_codec, + LEGACY_BLOCK_SIZE, + ) +} + +#[cfg(test)] +pub fn decompress_posting_list_with_tail_codec_and_block_size( + num_docs: u32, + posting_list: &arrow::array::LargeBinaryArray, + tail_codec: PostingTailCodec, + block_size: usize, +) -> Result<(Vec, Vec)> { + let block_size = validate_block_size(block_size)?; let mut doc_ids: Vec = Vec::with_capacity(num_docs as usize); let mut frequencies: Vec = Vec::with_capacity(num_docs as usize); - let mut buffer = [0u32; BLOCK_SIZE]; - let bitpacking_blocks = num_docs as usize / BLOCK_SIZE; + let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE]; + let bitpacking_blocks = num_docs as usize / block_size; for compressed in posting_list.iter().take(bitpacking_blocks) { let compressed = compressed.unwrap(); - decompress_posting_block(compressed, &mut buffer, &mut doc_ids, &mut frequencies); + decompress_posting_block( + compressed, + &mut buffer, + &mut doc_ids, + &mut frequencies, + block_size, + ); } - let remainder = num_docs as usize % BLOCK_SIZE; + let remainder = num_docs as usize % block_size; if remainder > 0 { let compressed = posting_list.value(bitpacking_blocks); decompress_posting_remainder( compressed, remainder, tail_codec, + block_size, &mut doc_ids, &mut frequencies, ); @@ -701,24 +1002,46 @@ pub fn read_num_positions(compressed: &arrow::array::LargeBinaryArray) -> u32 { pub fn decompress_posting_block( block: &[u8], - buffer: &mut [u32; BLOCK_SIZE], + buffer: &mut [u32], doc_ids: &mut Vec, frequencies: &mut Vec, + block_size: usize, ) { - // skip the first 4 bytes for the max block score - let block = &block[4..]; - let num_bytes = decompress_sorted_block(block, buffer, doc_ids); - decompress_block(&block[num_bytes..], buffer, frequencies); + debug_assert!(validate_block_size(block_size).is_ok()); + debug_assert!(buffer.len() >= block_size); + // skip the block max score prefix (128-doc blocks only) + let mut block = &block[posting_block_score_prefix_len(block_size)..]; + match block_size { + BitPacker4x::BLOCK_LEN => { + let num_bytes = decompress_sorted_block_with::(block, buffer, doc_ids); + block = &block[num_bytes..]; + let num_bytes = decompress_block_with::(block, buffer, frequencies); + block = &block[num_bytes..]; + } + BitPacker8x::BLOCK_LEN => { + let num_bytes = decompress_sorted_block_with::(block, buffer, doc_ids); + block = &block[num_bytes..]; + let num_bytes = decompress_pfor_block_with::(block, buffer, frequencies); + block = &block[num_bytes..]; + } + _ => unreachable!("validated posting block size should be supported"), + } + debug_assert!( + block.is_empty(), + "posting block has {} trailing bytes after decoding", + block.len() + ); } pub fn decompress_posting_remainder( block: &[u8], n: usize, codec: PostingTailCodec, + block_size: usize, doc_ids: &mut Vec, frequencies: &mut Vec, ) { - let block = &block[4..]; + let block = &block[posting_block_score_prefix_len(block_size)..]; match codec { PostingTailCodec::Fixed32 => { decompress_raw_remainder(block, n, doc_ids); @@ -755,9 +1078,14 @@ pub fn decompress_posting_remainder( } } -pub fn decode_full_posting_block(block: &[u8], doc_ids: &mut Vec, frequencies: &mut Vec) { - let mut buffer = [0u32; BLOCK_SIZE]; - decompress_posting_block(block, &mut buffer, doc_ids, frequencies); +pub fn decode_full_posting_block( + block: &[u8], + doc_ids: &mut Vec, + frequencies: &mut Vec, + block_size: usize, +) { + let mut buffer = [0u32; MAX_POSTING_BLOCK_SIZE]; + decompress_posting_block(block, &mut buffer, doc_ids, frequencies, block_size); } pub fn decompress_sorted_block( @@ -765,7 +1093,17 @@ pub fn decompress_sorted_block( buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec, ) -> usize { - let compressor = BitPacker4x::new(); + decompress_sorted_block_with::(block, buffer, res) +} + +fn decompress_sorted_block_with( + block: &[u8], + buffer: &mut [u32], + res: &mut Vec, +) -> usize { + debug_assert!(buffer.len() >= P::BLOCK_LEN); + let buffer = &mut buffer[..P::BLOCK_LEN]; + let compressor = P::new(); let initial = u32::from_le_bytes(block[0..4].try_into().unwrap()); let num_bits = block[4]; let num_bytes = compressor.decompress_sorted(initial, &block[5..], buffer, num_bits); @@ -773,11 +1111,18 @@ pub fn decompress_sorted_block( 5 + num_bytes } -fn decompress_block(block: &[u8], buffer: &mut [u32; BLOCK_SIZE], res: &mut Vec) { - let compressor = BitPacker4x::new(); +fn decompress_block_with( + block: &[u8], + buffer: &mut [u32], + res: &mut Vec, +) -> usize { + debug_assert!(buffer.len() >= P::BLOCK_LEN); + let buffer = &mut buffer[..P::BLOCK_LEN]; + let compressor = P::new(); let num_bits = block[0]; - compressor.decompress(&block[1..], buffer, num_bits); + let num_bytes = compressor.decompress(&block[1..], buffer, num_bits); res.extend_from_slice(&buffer[..]); + 1 + num_bytes } pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec) { @@ -787,11 +1132,18 @@ pub fn decompress_raw_remainder(compressed: &[u8], n: usize, dest: &mut Vec } } -pub fn read_posting_tail_first_doc(block: &[u8], codec: PostingTailCodec) -> u32 { +pub fn read_posting_tail_first_doc( + block: &[u8], + codec: PostingTailCodec, + block_size: usize, +) -> u32 { + let prefix = posting_block_score_prefix_len(block_size); match codec { - PostingTailCodec::Fixed32 => u32::from_le_bytes(block[4..8].try_into().unwrap()), + PostingTailCodec::Fixed32 => { + u32::from_le_bytes(block[prefix..prefix + 4].try_into().unwrap()) + } PostingTailCodec::VarintDelta => { - let mut offset = 4usize; + let mut offset = prefix; decode_varint_u32(block, &mut offset) .expect("posting tail block should contain a valid first doc id") } @@ -805,31 +1157,6 @@ mod tests { use itertools::Itertools; use rand::Rng; - #[test] - fn test_group_starts_codec_roundtrip() { - for case in [ - vec![], - vec![0u32], - vec![0, 1, 2, 3], - // realistic: monotonic with varied group sizes and a large jump - vec![0, 64, 128, 129, 4096, 1_000_000], - ] { - let encoded = encode_group_starts(&case); - let decoded = decode_group_starts(&encoded).unwrap(); - assert_eq!(decoded, case, "roundtrip mismatch for {case:?}"); - } - } - - #[test] - fn test_decode_group_starts_rejects_overflow() { - // A crafted buffer whose deltas sum past u32::MAX must error rather - // than wrap. Encodes u32::MAX followed by a +1 delta. - let mut buf = Vec::new(); - encode_varint_u32(&mut buf, u32::MAX); - encode_varint_u32(&mut buf, 1); - assert!(decode_group_starts(&buf).is_err()); - } - #[test] fn test_compress_posting_list() -> Result<()> { let num_rows: usize = BLOCK_SIZE * 1024 - 7; @@ -867,6 +1194,84 @@ mod tests { Ok(()) } + #[test] + fn test_compress_posting_list_supported_block_sizes() -> Result<()> { + for block_size in [128, 256] { + let num_rows: usize = block_size * 2 + 7; + let doc_ids = (0..num_rows as u32).collect::>(); + let frequencies = (0..num_rows as u32) + .map(|value| value % 7 + 1) + .collect::>(); + let block_max_scores = + (0..num_rows.div_ceil(block_size)).map(|value| value as f32 + 1.0); + + let posting_list = compress_posting_list_with_tail_codec_and_block_size( + doc_ids.len(), + doc_ids.iter(), + frequencies.iter(), + block_max_scores, + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(posting_list.len(), num_rows.div_ceil(block_size)); + + let (decoded_doc_ids, decoded_frequencies) = + decompress_posting_list_with_tail_codec_and_block_size( + num_rows as u32, + &posting_list, + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(decoded_doc_ids, doc_ids); + assert_eq!(decoded_frequencies, frequencies); + } + Ok(()) + } + + #[test] + fn test_256_posting_block_uses_single_physical_bitpack_chunk() -> Result<()> { + let block_size = BitPacker8x::BLOCK_LEN; + let doc_ids = (0..block_size as u32).collect::>(); + let frequencies = (0..block_size as u32) + .map(|value| value % 13 + 1) + .collect::>(); + + let posting_list = compress_posting_list_with_tail_codec_and_block_size( + doc_ids.len(), + doc_ids.iter(), + frequencies.iter(), + std::iter::once(1.0), + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(posting_list.len(), 1); + + let block = posting_list.value(0); + // 256-doc blocks carry no block-max-score prefix (impacts supply the + // per-block bound): [first_doc u32][doc num_bits u8][doc payload]... + let doc_num_bits = block[4]; + let doc_bytes = BitPacker8x::compressed_block_size(doc_num_bits); + let freq_header_offset = 5 + doc_bytes; + // 256-doc blocks use patched FOR for frequencies: + // [width u8][exception_count u8][body][exceptions...] + let freq_num_bits = block[freq_header_offset]; + let exception_count = block[freq_header_offset + 1] as usize; + assert_eq!(exception_count, 0, "uniform freqs need no exceptions"); + let freq_bytes = BitPacker8x::compressed_block_size(freq_num_bits); + assert_eq!(block.len(), freq_header_offset + 2 + freq_bytes); + + let (decoded_doc_ids, decoded_frequencies) = + decompress_posting_list_with_tail_codec_and_block_size( + doc_ids.len() as u32, + &posting_list, + PostingTailCodec::VarintDelta, + block_size, + )?; + assert_eq!(decoded_doc_ids, doc_ids); + assert_eq!(decoded_frequencies, frequencies); + Ok(()) + } + #[test] fn test_compress_posting_list_fixed32_tail_still_roundtrips() -> Result<()> { let doc_ids = vec![3_u32, 10_u32, 24_u32]; @@ -913,6 +1318,116 @@ mod tests { Ok(()) } + /// Per-doc seek decoding of a PackedDelta position block must return + /// exactly the same positions as decoding the whole block, for every doc, + /// across group-boundary-straddling docs and varint tails. + #[test] + fn test_packed_position_doc_seek_matches_block_decode() -> Result<()> { + let mut rng = rand::rng(); + // Frequency shapes: tiny blocks (tail only), exactly one group, a doc + // straddling group boundaries, and a large multi-group block. + let freq_shapes: Vec> = vec![ + vec![1], + vec![3, 1, 5], + vec![64, 64], + vec![100, 60, 40], + (0..256u32).map(|i| (i % 7) + 1).collect(), + vec![300, 2, 129, 1, 77], + ]; + for frequencies in freq_shapes { + let total: usize = frequencies.iter().map(|&f| f as usize).sum(); + // Positions ascend within each doc; docs are independent. + let mut positions = Vec::with_capacity(total); + for &freq in &frequencies { + let mut current = rng.random_range(0..1000u32); + for _ in 0..freq { + positions.push(current); + current += rng.random_range(1..50u32); + } + } + + let mut encoded = Vec::new(); + encode_position_stream_block_into( + &positions, + &frequencies, + PositionStreamCodec::PackedDelta, + &mut encoded, + )?; + + let mut whole = Vec::new(); + decode_position_stream_block( + &encoded, + &frequencies, + PositionStreamCodec::PackedDelta, + &mut whole, + )?; + assert_eq!(whole, positions); + + let mut group_offsets = vec![0usize]; + let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]); + let mut unpacked_group_idx = None; + let mut tail_cache = Vec::new(); + let mut scratch = Vec::new(); + let mut delta_start = 0usize; + for &freq in &frequencies { + let delta_end = delta_start + freq as usize; + seek_packed_doc_positions( + &encoded, + total, + delta_start..delta_end, + &mut group_offsets, + &mut unpacked_group, + &mut unpacked_group_idx, + &mut tail_cache, + &mut scratch, + )?; + assert_eq!( + scratch, + whole[delta_start..delta_end], + "doc positions mismatch for range {delta_start}..{delta_end} freqs={frequencies:?}" + ); + delta_start = delta_end; + } + } + Ok(()) + } + + #[test] + fn test_packed_position_decoders_reject_malformed_groups() { + let frequencies = [BLOCK_SIZE as u32]; + for encoded in [&[1_u8][..], &[33_u8][..]] { + let mut decoded = Vec::new(); + assert!( + decode_position_stream_block( + encoded, + &frequencies, + PositionStreamCodec::PackedDelta, + &mut decoded, + ) + .is_err() + ); + + let mut group_offsets = vec![0usize]; + let mut unpacked_group = Box::new([0u32; BLOCK_SIZE]); + let mut unpacked_group_idx = None; + let mut tail_cache = Vec::new(); + let mut scratch = Vec::new(); + assert!( + seek_packed_doc_positions( + encoded, + BLOCK_SIZE, + 0..BLOCK_SIZE, + &mut group_offsets, + &mut unpacked_group, + &mut unpacked_group_idx, + &mut tail_cache, + &mut scratch, + ) + .is_err() + ); + } + } + #[test] fn test_encode_position_stream_block_roundtrip() -> Result<()> { let frequencies = vec![1, 3, 2, 4]; diff --git a/rust/lance-index/src/scalar/inverted/impact.rs b/rust/lance-index/src/scalar/inverted/impact.rs new file mode 100644 index 00000000000..848c2376459 --- /dev/null +++ b/rust/lance-index/src/scalar/inverted/impact.rs @@ -0,0 +1,1055 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::mem::size_of; +use std::sync::{Arc, Mutex, MutexGuard}; + +use arrow_array::builder::LargeBinaryBuilder; +use arrow_array::{Array, LargeBinaryArray}; +use lance_core::{Error, Result}; + +use super::scorer::Scorer; + +pub const IMPACT_LEVEL1_BLOCKS: usize = 32; +const SMALL_FRONTIER_FREQ_LIMIT: usize = 256; + +/// On-disk encoding of one impact entry, shared by every posting block size. +/// +/// Entries contain `[doc_up_to varint][pair_count varint][pairs...]`. Each +/// pair stores a varint whose high bits are `freq_delta - 1` and whose low bit +/// reports whether a one-byte norm delta follows. The norm itself is the +/// quantized `u8` document-length code; the common `norm_delta == 1` case needs +/// no norm byte. +#[derive(Debug, Clone)] +pub struct ImpactSkipData { + entries: LargeBinaryArray, + level0_len: usize, + // Last doc id covered by each entry (level0 entries then level1 entries), + // decoded once at construction. Level1 markers are fully validated because + // WAND may use them to skip a group; u32::MAX marks malformed entries. + entry_doc_up_tos: Arc<[u32]>, + // The most recently baked bounds with a stable scorer key. Each query holds + // its own Arc in ImpactScoreCache, so replacing this slot for another scorer + // cannot change bounds already in use. Scorers without a key never enter the + // shared slot. Malformed entries bake to INFINITY so pruning stays safe. + last_keyed_bounds: Arc>, +} + +impl PartialEq for ImpactSkipData { + fn eq(&self, other: &Self) -> bool { + self.entries == other.entries && self.level0_len == other.level0_len + } +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy)] +pub struct ImpactScore { + pub score: f32, + pub entries_scanned: usize, +} + +#[derive(Debug)] +struct ImpactBounds { + per_entry: Box<[f32]>, + global: f32, +} + +type LastKeyedImpactBounds = Option<(u64, Arc)>; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ImpactBoundsCacheKey { + Keyed(u64), + QueryLocal, +} + +#[derive(Debug, Default, Clone)] +pub struct ImpactScoreCache { + key: Option, + bounds: Option>, +} + +impl ImpactScoreCache { + fn bounds<'a, S: Scorer + ?Sized>( + &'a mut self, + impacts: &ImpactSkipData, + scorer: &S, + ) -> &'a ImpactBounds { + let scorer_key = scorer.doc_weight_cache_key(); + let cache_key = scorer_key + .map(ImpactBoundsCacheKey::Keyed) + .unwrap_or(ImpactBoundsCacheKey::QueryLocal); + if self.key != Some(cache_key) { + self.key = Some(cache_key); + self.bounds = None; + } + + self.bounds + .get_or_insert_with(|| impacts.bounds_for_scorer(scorer, scorer_key)) + } + + fn entry_score( + &mut self, + impacts: &ImpactSkipData, + entry_idx: usize, + query_weight: f32, + scorer: &S, + ) -> f32 { + if query_weight <= 0.0 { + return 0.0; + } + query_weight * self.bounds(impacts, scorer).per_entry[entry_idx] + } +} + +impl ImpactSkipData { + pub fn new(entries: LargeBinaryArray, level0_len: usize) -> Result { + let expected_len = level0_len + level1_len(level0_len); + if entries.len() != expected_len { + return Err(Error::index(format!( + "impact entry count mismatch: got {}, expected {} for {} level0 blocks", + entries.len(), + expected_len, + level0_len + ))); + } + let entry_doc_up_tos = (0..entries.len()) + .map(|entry_idx| { + if entries.is_null(entry_idx) { + return u32::MAX; + } + let bytes = entries.value(entry_idx); + let doc_up_to = if entry_idx < level0_len { + decode_level0_entry_doc_up_to(bytes) + } else { + decode_entry_doc_up_to(bytes) + }; + doc_up_to.unwrap_or(u32::MAX) + }) + .collect::>(); + Ok(Self { + entries, + level0_len, + entry_doc_up_tos, + last_keyed_bounds: Arc::new(Mutex::new(None)), + }) + } + + fn keyed_bounds_guard(&self) -> MutexGuard<'_, LastKeyedImpactBounds> { + match self.last_keyed_bounds.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } + + fn bounds_for_scorer( + &self, + scorer: &S, + scorer_key: Option, + ) -> Arc { + let Some(scorer_key) = scorer_key else { + return Arc::new(self.compute_bounds(scorer)); + }; + + { + let cached = self.keyed_bounds_guard(); + if let Some((cached_key, bounds)) = cached.as_ref() + && *cached_key == scorer_key + { + return bounds.clone(); + } + } + + // Compute outside the mutex. Concurrent misses may duplicate this work, + // but only the short publication/check below holds the shared lock. + let computed = Arc::new(self.compute_bounds(scorer)); + let mut cached = self.keyed_bounds_guard(); + if let Some((cached_key, bounds)) = cached.as_ref() + && *cached_key == scorer_key + { + return bounds.clone(); + } + *cached = Some((scorer_key, computed.clone())); + computed + } + + fn compute_bounds(&self, scorer: &S) -> ImpactBounds { + let per_entry = (0..self.entries.len()) + .map(|entry_idx| { + if self.entries.is_null(entry_idx) { + return f32::INFINITY; + } + let bytes = self.entries.value(entry_idx); + let mut max_doc_weight = 0.0_f32; + match for_each_entry_pair(bytes, |freq, doc_len| { + max_doc_weight = max_doc_weight.max(scorer.doc_weight(freq, doc_len)); + }) { + Ok(()) => max_doc_weight, + Err(_) => f32::INFINITY, + } + }) + .collect::>(); + // The level1 entries cover every block, so their max is the list-wide + // max doc weight; zero-entry lists fall back to the empty level0 slab. + let global = if per_entry.len() > self.level0_len { + per_entry[self.level0_len..] + .iter() + .copied() + .fold(0.0_f32, f32::max) + } else { + per_entry.iter().copied().fold(0.0_f32, f32::max) + }; + ImpactBounds { per_entry, global } + } + + /// List-wide max doc weight, from the scorer-specific cached bounds. The + /// tightest valid global score bound is `query_weight * this`, matching what + /// the non-impact format stores as `max_score` at build time. + pub fn global_max_doc_weight_cached( + &self, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> f32 { + cache.bounds(self, scorer).global + } + + /// Cached per-block max doc weights (level0 entries only), for bulk skip + /// scans over dead ranges without per-block window bookkeeping. + pub(crate) fn level0_doc_weight_bounds_cached<'a, S: Scorer + ?Sized>( + &self, + scorer: &S, + cache: &'a mut ImpactScoreCache, + ) -> &'a [f32] { + &cache.bounds(self, scorer).per_entry[..self.level0_len] + } + + pub fn entries(&self) -> &LargeBinaryArray { + &self.entries + } + + /// Conservative heap charge for query-independent derived state and one + /// shared keyed-bound slab, whether or not that slab has been initialized + /// yet. The Arrow impact entries are owned by the enclosing batch and are + /// deliberately excluded so packed-group cache accounting counts them once. + pub(crate) fn derived_cache_bytes(&self) -> usize { + Self::derived_cache_bytes_for_entries(self.entries.len()) + } + + pub(crate) fn derived_cache_bytes_for_entries(entry_count: usize) -> usize { + entry_count * size_of::() + + size_of::>() + + size_of::() + + entry_count * size_of::() + } + + #[cfg(test)] + pub(crate) fn shares_derived_state_with(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.entry_doc_up_tos, &other.entry_doc_up_tos) + && Arc::ptr_eq(&self.last_keyed_bounds, &other.last_keyed_bounds) + } + + #[cfg(test)] + pub fn level0_len(&self) -> usize { + self.level0_len + } + + #[cfg(test)] + pub fn level1_len(&self) -> usize { + level1_len(self.level0_len) + } + + pub(crate) fn level1_doc_up_to(&self, group_idx: usize) -> Option { + if group_idx >= level1_len(self.level0_len) { + return None; + } + match self.entry_doc_up_tos[self.level0_len + group_idx] { + u32::MAX => None, + doc_up_to => Some(doc_up_to), + } + } + + /// Last doc id covered by the level0 entry of `block_idx`, or `None` when + /// the entry is missing or malformed. + pub(crate) fn level0_doc_up_to(&self, block_idx: usize) -> Option { + if block_idx >= self.level0_len { + return None; + } + match self.entry_doc_up_tos[block_idx] { + u32::MAX => None, + doc_up_to => Some(doc_up_to), + } + } + + pub fn level0_score_cached( + &self, + block_idx: usize, + query_weight: f32, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> f32 { + if block_idx >= self.level0_len { + return 0.0; + } + cache.entry_score(self, block_idx, query_weight, scorer) + } + + /// Max score of the docs covered by the level1 entry of `group_idx`, + /// answered from the scorer-specific cached bounds slab. + pub(crate) fn level1_score_cached( + &self, + group_idx: usize, + query_weight: f32, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> f32 { + if group_idx >= level1_len(self.level0_len) { + return 0.0; + } + cache.entry_score(self, self.level0_len + group_idx, query_weight, scorer) + } + + #[cfg(test)] + pub fn max_score_up_to_cached( + &self, + start_block_idx: usize, + up_to: u64, + query_weight: f32, + scorer: &S, + cache: &mut ImpactScoreCache, + ) -> ImpactScore + where + S: Scorer + ?Sized, + { + let mut block_idx = start_block_idx; + let mut max_score = 0.0_f32; + let mut entries_scanned = 0usize; + + while block_idx < self.level0_len { + let group_idx = block_idx / IMPACT_LEVEL1_BLOCKS; + let group_start = group_idx * IMPACT_LEVEL1_BLOCKS; + let group_end = ((group_idx + 1) * IMPACT_LEVEL1_BLOCKS).min(self.level0_len); + if block_idx == group_start { + let level1_entry_idx = self.level0_len + group_idx; + match self.entry_doc_up_tos[level1_entry_idx] { + u32::MAX => { + return ImpactScore { + score: f32::INFINITY, + entries_scanned: entries_scanned + 1, + }; + } + doc_up_to if u64::from(doc_up_to) <= up_to => { + max_score = max_score.max(cache.entry_score( + self, + level1_entry_idx, + query_weight, + scorer, + )); + entries_scanned += 1; + block_idx = group_end; + continue; + } + _ => {} + } + } + + max_score = max_score.max(cache.entry_score(self, block_idx, query_weight, scorer)); + entries_scanned += 1; + match self.entry_doc_up_tos[block_idx] { + u32::MAX => { + return ImpactScore { + score: f32::INFINITY, + entries_scanned, + }; + } + doc_up_to if u64::from(doc_up_to) >= up_to => break, + _ => {} + } + block_idx += 1; + } + + ImpactScore { + score: max_score, + entries_scanned, + } + } +} + +pub struct ImpactSkipDataBuilder { + entries: LargeBinaryBuilder, + level0_len: usize, + level1_entries: Vec>, + level1_docs: Vec<(u32, u32, u32)>, +} + +impl ImpactSkipDataBuilder { + pub fn with_capacity(level0_blocks: usize, block_size: usize) -> Self { + Self { + entries: LargeBinaryBuilder::with_capacity( + level0_blocks + level1_len(level0_blocks), + 0, + ), + level0_len: 0, + level1_entries: Vec::with_capacity(level1_len(level0_blocks)), + level1_docs: Vec::with_capacity(IMPACT_LEVEL1_BLOCKS * block_size), + } + } + + pub fn append_block(&mut self, docs: &[(u32, u32, u32)]) -> Result<()> { + let bytes = encode_impact_entry(docs)?; + self.entries.append_value(bytes.as_slice()); + self.level0_len += 1; + self.level1_docs.extend_from_slice(docs); + if self.level0_len.is_multiple_of(IMPACT_LEVEL1_BLOCKS) { + self.flush_level1()?; + } + Ok(()) + } + + pub fn finish(mut self) -> Result { + if !self.level1_docs.is_empty() { + self.flush_level1()?; + } + for entry in self.level1_entries { + self.entries.append_value(entry.as_slice()); + } + ImpactSkipData::new(self.entries.finish(), self.level0_len) + } + + fn flush_level1(&mut self) -> Result<()> { + let bytes = encode_impact_entry(self.level1_docs.as_slice())?; + self.level1_entries.push(bytes); + self.level1_docs.clear(); + Ok(()) + } +} + +#[cfg(test)] +pub fn build_impact_skip_data(blocks: &[Vec<(u32, u32, u32)>]) -> Result { + let block_size = blocks.iter().map(Vec::len).max().unwrap_or(0).max(1); + let mut builder = ImpactSkipDataBuilder::with_capacity(blocks.len(), block_size); + for block in blocks { + builder.append_block(block)?; + } + builder.finish() +} + +fn encode_impact_entry(docs: &[(u32, u32, u32)]) -> Result> { + if docs.is_empty() { + return Err(Error::index( + "cannot encode an empty impact entry".to_owned(), + )); + } + let doc_up_to = docs + .last() + .map(|(doc_id, _, _)| *doc_id) + .expect("non-empty impact entry was validated above"); + let frontier = quantized_impact_frontier(docs); + let pair_count = u32::try_from(frontier.len()).map_err(|_| { + Error::index("impact frontier too large to encode as u32 pair count".to_string()) + })?; + let mut bytes = Vec::with_capacity(5 + frontier.len() * 2); + super::encoding::encode_varint_u32(&mut bytes, doc_up_to); + super::encoding::encode_varint_u32(&mut bytes, pair_count); + let mut previous_freq = 0u32; + let mut previous_norm = 0u8; + for (pair_idx, (freq, norm)) in frontier.into_iter().enumerate() { + let freq_delta_minus_one = freq + .checked_sub(previous_freq) + .and_then(|delta| delta.checked_sub(1)) + .ok_or_else(|| { + Error::index(format!( + "impact frequencies must be positive and strictly increasing: previous={previous_freq}, current={freq}" + )) + })?; + let norm_delta = norm.checked_sub(previous_norm).ok_or_else(|| { + Error::index(format!( + "impact norms must be non-decreasing: previous={previous_norm}, current={norm}" + )) + })?; + if pair_idx > 0 && norm_delta == 0 { + return Err(Error::index(format!( + "impact norms must be strictly increasing after quantization: norm={norm}" + ))); + } + + let has_explicit_norm_delta = norm_delta != 1; + let packed_freq_delta = + (u64::from(freq_delta_minus_one) << 1) | u64::from(has_explicit_norm_delta); + encode_varint_u64(&mut bytes, packed_freq_delta); + if has_explicit_norm_delta { + bytes.push(norm_delta); + } + previous_freq = freq; + previous_norm = norm; + } + Ok(bytes) +} + +fn decode_entry_doc_up_to(bytes: &[u8]) -> Result { + let mut offset = 0usize; + let doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?; + // Level-1 doc ids drive whole-group skips, so only publish a doc id after + // validating the complete entry. A truncated entry may still have a valid + // first varint. + for_each_entry_pair(bytes, |_, _| {})?; + Ok(doc_up_to) +} + +fn decode_level0_entry_doc_up_to(bytes: &[u8]) -> Result { + // A malformed level0 frontier bakes an INFINITY score before this marker + // can terminate a range scan, so avoid parsing every frontier twice on the + // query-load path. Level1 entries are fully validated. + let mut offset = 0usize; + super::encoding::decode_varint_u32(bytes, &mut offset) +} + +/// Walk an entry's (freq, doc_len) frontier pairs, validating the layout. +fn for_each_entry_pair(bytes: &[u8], mut visit: impl FnMut(u32, u32)) -> Result<()> { + let mut offset = 0usize; + let _doc_up_to = super::encoding::decode_varint_u32(bytes, &mut offset)?; + let pair_count = super::encoding::decode_varint_u32(bytes, &mut offset)?; + if pair_count == 0 { + return Err(Error::index( + "impact entry must contain at least one frontier pair".to_owned(), + )); + } + + let mut previous_freq = 0u32; + let mut previous_norm = 0u16; + for pair_idx in 0..pair_count { + let packed_freq_delta = decode_varint_u64(bytes, &mut offset)?; + let freq_delta_minus_one = u32::try_from(packed_freq_delta >> 1) + .map_err(|_| Error::index("impact freq delta exceeds u32".to_owned()))?; + let freq_delta = freq_delta_minus_one + .checked_add(1) + .ok_or_else(|| Error::index("impact freq delta overflow".to_owned()))?; + let freq = previous_freq + .checked_add(freq_delta) + .ok_or_else(|| Error::index("impact frequency overflow".to_owned()))?; + + let has_explicit_norm_delta = packed_freq_delta & 1 != 0; + let norm_delta = if has_explicit_norm_delta { + let norm_delta = bytes.get(offset).copied().ok_or_else(|| { + Error::index("unexpected EOF while decoding impact norm delta".to_owned()) + })?; + offset += 1; + norm_delta + } else { + 1 + }; + if pair_idx > 0 && norm_delta == 0 { + return Err(Error::index( + "impact norms must be strictly increasing".to_owned(), + )); + } + + let norm = previous_norm + .checked_add(u16::from(norm_delta)) + .filter(|norm| *norm <= u16::from(u8::MAX)) + .ok_or_else(|| Error::index("impact norm delta overflow".to_owned()))?; + let norm = norm as u8; + visit(freq, super::index::dequantize_doc_length(norm)); + previous_freq = freq; + previous_norm = u16::from(norm); + } + if offset != bytes.len() { + return Err(Error::index(format!( + "impact entry has {} trailing bytes", + bytes.len() - offset + ))); + } + Ok(()) +} + +#[inline] +fn encode_varint_u64(dst: &mut Vec, mut value: u64) { + while value >= 0x80 { + dst.push((value as u8) | 0x80); + value >>= 7; + } + dst.push(value as u8); +} + +#[inline] +fn decode_varint_u64(src: &[u8], offset: &mut usize) -> Result { + let mut value = 0u64; + let mut shift = 0u32; + while *offset < src.len() { + let byte = src[*offset]; + *offset += 1; + if shift == 63 && byte & 0xFE != 0 { + return Err(Error::index( + "invalid u64 varint in impact entry".to_owned(), + )); + } + value |= u64::from(byte & 0x7F) << shift; + if byte & 0x80 == 0 { + return Ok(value); + } + shift += 7; + if shift > 63 { + return Err(Error::index( + "invalid u64 varint in impact entry".to_owned(), + )); + } + } + Err(Error::index( + "unexpected EOF while decoding impact entry".to_owned(), + )) +} + +fn quantized_impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u8)> { + let raw_frontier = impact_frontier(docs); + let mut frontier: Vec<(u32, u8)> = Vec::with_capacity(raw_frontier.len()); + for (freq, doc_len) in raw_frontier { + let norm = super::index::quantize_doc_length(doc_len); + match frontier.last_mut() { + Some((last_freq, last_norm)) if *last_norm == norm => { + // At the same quantized norm, the larger frequency dominates. + *last_freq = freq; + } + Some((_, last_norm)) => { + debug_assert!( + *last_norm < norm, + "raw impact frontier document lengths must be increasing" + ); + frontier.push((freq, norm)); + } + None => frontier.push((freq, norm)), + } + } + frontier +} + +fn impact_frontier(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> { + let max_freq = docs.iter().map(|(_, freq, _)| *freq).max().unwrap_or(0) as usize; + if max_freq <= SMALL_FRONTIER_FREQ_LIMIT { + return impact_frontier_small_freq(docs, max_freq); + } + + impact_frontier_sparse_freq(docs) +} + +fn impact_frontier_small_freq(docs: &[(u32, u32, u32)], max_freq: usize) -> Vec<(u32, u32)> { + let mut min_doc_len_by_freq = [u32::MAX; SMALL_FRONTIER_FREQ_LIMIT + 1]; + for (_, freq, doc_len) in docs { + min_doc_len_by_freq[*freq as usize] = min_doc_len_by_freq[*freq as usize].min(*doc_len); + } + + let min_doc_lens = min_doc_len_by_freq[..=max_freq] + .iter() + .enumerate() + .filter_map(|(freq, doc_len)| (*doc_len != u32::MAX).then_some((freq as u32, *doc_len))) + .collect::>(); + frontier_from_min_doc_lens(min_doc_lens) +} + +fn impact_frontier_sparse_freq(docs: &[(u32, u32, u32)]) -> Vec<(u32, u32)> { + let mut pairs = docs + .iter() + .map(|(_, freq, doc_len)| (*freq, *doc_len)) + .collect::>(); + pairs.sort_unstable_by_key(|(freq, _)| *freq); + + let mut min_doc_lens: Vec<(u32, u32)> = Vec::with_capacity(pairs.len()); + for (freq, doc_len) in pairs { + match min_doc_lens.last_mut() { + Some((last_freq, last_doc_len)) if *last_freq == freq => { + *last_doc_len = (*last_doc_len).min(doc_len); + } + _ => min_doc_lens.push((freq, doc_len)), + } + } + + frontier_from_min_doc_lens(min_doc_lens) +} + +fn frontier_from_min_doc_lens(min_doc_lens: Vec<(u32, u32)>) -> Vec<(u32, u32)> { + let mut best_doc_len = u32::MAX; + let mut frontier = Vec::with_capacity(min_doc_lens.len()); + for (freq, doc_len) in min_doc_lens.into_iter().rev() { + if doc_len < best_doc_len { + frontier.push((freq, doc_len)); + best_doc_len = doc_len; + } + } + frontier.reverse(); + frontier +} + +fn level1_len(level0_len: usize) -> usize { + level0_len.div_ceil(IMPACT_LEVEL1_BLOCKS) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer}; + + use super::*; + use crate::scalar::inverted::scorer::{MemBM25Scorer, Scorer}; + + struct KeyedCountingScorer { + key: u64, + calls: Arc, + } + + impl Scorer for KeyedCountingScorer { + fn query_weight(&self, _token: &str) -> f32 { + 1.0 + } + + fn doc_weight(&self, freq: u32, doc_tokens: u32) -> f32 { + self.calls.fetch_add(1, Ordering::Relaxed); + freq as f32 / doc_tokens as f32 + } + + fn doc_weight_cache_key(&self) -> Option { + Some(self.key) + } + } + + #[test] + fn impact_entry_frontier_drops_dominated_pairs() { + let docs = vec![(0, 1, 10), (1, 1, 8), (2, 2, 9), (3, 3, 20)]; + assert_eq!(impact_frontier(&docs), vec![(1, 8), (2, 9), (3, 20)]); + } + + #[test] + fn impact_entry_frontier_handles_sparse_large_frequencies() { + let docs = vec![ + (0, 1, 100), + (1, 1, 80), + (2, 512, 90), + (3, 1_000, 120), + (4, 1_000, 110), + ]; + assert_eq!( + impact_frontier(&docs), + vec![(1, 80), (512, 90), (1_000, 110)] + ); + } + + #[test] + fn quantized_impact_frontier_drops_equal_norms() { + let docs = vec![(0, 1, 16), (1, 2, 17), (2, 3, 24)]; + assert_eq!( + quantized_impact_frontier(&docs), + vec![ + (2, super::super::index::quantize_doc_length(17)), + (3, super::super::index::quantize_doc_length(24)), + ] + ); + } + + #[test] + fn impact_max_score_can_use_level1_entry() { + let blocks = (0..40) + .map(|block| vec![(block as u32, 1 + block as u32 % 3, 10)]) + .collect::>(); + let impacts = build_impact_skip_data(&blocks).unwrap(); + assert_eq!(impacts.level0_len(), 40); + assert_eq!(impacts.level1_len(), 2); + let scorer = MemBM25Scorer::new(400, 40, HashMap::from([(String::from("token"), 40usize)])); + let mut cache = ImpactScoreCache::default(); + let score = impacts.max_score_up_to_cached(0, 31, 1.0, &scorer, &mut cache); + assert!(score.entries_scanned < IMPACT_LEVEL1_BLOCKS); + assert!(score.score > 0.0); + } + + #[test] + fn impact_level1_doc_up_to_reports_full_and_partial_groups() { + let blocks = (0..40) + .map(|block| vec![(block as u32, 1, 10)]) + .collect::>(); + let impacts = build_impact_skip_data(&blocks).unwrap(); + + assert_eq!( + impacts.level1_doc_up_to(0), + Some((IMPACT_LEVEL1_BLOCKS - 1) as u32) + ); + assert_eq!(impacts.level1_doc_up_to(1), Some(39)); + assert_eq!(impacts.level1_doc_up_to(2), None); + } + + #[test] + fn impact_level1_doc_up_to_returns_none_for_malformed_entry() { + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let malformed_level1 = vec![1, 2, 3]; + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0.as_slice()), + Some(malformed_level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + } + + #[test] + fn impact_level1_doc_up_to_validates_complete_entry() { + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + // A complete first varint is not enough: the pair count and frontier + // are required before this doc id can safely drive a group skip. + let truncated_level1 = [31_u8]; + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0.as_slice()), + Some(truncated_level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + } + + #[test] + fn empty_impact_frontiers_are_malformed_bounds() { + let scorer = MemBM25Scorer::new(10, 1, HashMap::new()); + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let empty_level1 = [0, 0]; + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0.as_slice()), + Some(empty_level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + let mut cache = ImpactScoreCache::default(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + assert!( + impacts + .global_max_doc_weight_cached(&scorer, &mut cache) + .is_infinite() + ); + assert!( + ImpactSkipDataBuilder::with_capacity(1, 128) + .append_block(&[]) + .is_err() + ); + } + + #[test] + fn null_impact_entry_is_an_infinite_bound_even_with_hidden_bytes() { + let level0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let level1 = encode_impact_entry(&[(0, 2, 8)]).unwrap(); + let mut values = level0.clone(); + values.extend_from_slice(&level1); + let entries = LargeBinaryArray::new( + OffsetBuffer::new(ScalarBuffer::from(vec![ + 0_i64, + level0.len() as i64, + values.len() as i64, + ])), + Buffer::from_vec(values), + Some(NullBuffer::from(vec![true, false])), + ); + let impacts = ImpactSkipData::new(entries, 1).unwrap(); + let scorer = MemBM25Scorer::new(10, 1, HashMap::new()); + let mut cache = ImpactScoreCache::default(); + + assert_eq!(impacts.level1_doc_up_to(0), None); + assert!( + impacts + .global_max_doc_weight_cached(&scorer, &mut cache) + .is_infinite() + ); + } + + #[test] + fn impact_bounds_follow_changed_bm25_average_doc_length() { + let impacts = build_impact_skip_data(&[vec![(0, 1, 100)]]).unwrap(); + let low_avgdl = MemBM25Scorer::new(1, 1, HashMap::new()); + let high_avgdl = MemBM25Scorer::new(100, 1, HashMap::new()); + let mut low_cache = ImpactScoreCache::default(); + let mut high_cache = ImpactScoreCache::default(); + + let low_bound = impacts.global_max_doc_weight_cached(&low_avgdl, &mut low_cache); + let high_bound = impacts.global_max_doc_weight_cached(&high_avgdl, &mut high_cache); + let quantized_doc_length = super::super::index::dequantize_doc_length( + super::super::index::quantize_doc_length(100), + ); + + assert!((low_bound - low_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6); + assert!((high_bound - high_avgdl.doc_weight(1, quantized_doc_length)).abs() < 1e-6); + assert!(low_bound >= low_avgdl.doc_weight(1, 100)); + assert!(high_bound >= high_avgdl.doc_weight(1, 100)); + assert!( + high_bound > low_bound, + "larger avgdl must recompute a larger bound: low={low_bound}, high={high_bound}" + ); + } + + #[test] + fn impact_bounds_reuse_same_scorer_key_across_queries() { + let impacts = build_impact_skip_data(&[vec![(0, 2, 10)]]).unwrap(); + let cloned = impacts.clone(); + assert!(impacts.shares_derived_state_with(&cloned)); + let calls = Arc::new(AtomicUsize::new(0)); + let scorer = KeyedCountingScorer { + key: 7, + calls: calls.clone(), + }; + let mut first_query = ImpactScoreCache::default(); + let mut second_query = ImpactScoreCache::default(); + + let first = impacts.global_max_doc_weight_cached(&scorer, &mut first_query); + let baked_calls = calls.load(Ordering::Relaxed); + assert!(baked_calls > 0); + let second = cloned.global_max_doc_weight_cached(&scorer, &mut second_query); + + assert_eq!(second, first); + assert_eq!( + calls.load(Ordering::Relaxed), + baked_calls, + "the same keyed bounds should be shared across query caches" + ); + } + + #[test] + fn malformed_unscanned_entry_does_not_poison_range_score() { + let level0_0 = encode_impact_entry(&[(0, 1, 10)]).unwrap(); + let malformed_level0_1 = vec![1, 2, 3]; + let level1 = encode_impact_entry(&[(0, 1, 10), (1, 1, 10)]).unwrap(); + let entries = LargeBinaryArray::from_opt_vec(vec![ + Some(level0_0.as_slice()), + Some(malformed_level0_1.as_slice()), + Some(level1.as_slice()), + ]); + let impacts = ImpactSkipData::new(entries, 2).unwrap(); + let scorer = MemBM25Scorer::new(10, 10, HashMap::from([(String::from("token"), 2usize)])); + let mut cache = ImpactScoreCache::default(); + + let score = impacts.max_score_up_to_cached(0, 0, 1.0, &scorer, &mut cache); + assert!(score.score.is_finite()); + assert_eq!(score.entries_scanned, 1); + + assert_eq!( + impacts.level0_score_cached(1, 1.0, &scorer, &mut cache), + f32::INFINITY + ); + } + + #[test] + fn impact_entries_store_quantized_norm_deltas() { + let docs = vec![(7, 1, 1), (9, 2, 2), (12, 3, 5)]; + let encoded = encode_impact_entry(&docs).unwrap(); + + // doc_up_to=12, pair_count=3, two implicit +1 norm deltas, then an + // explicit +3 norm delta folded behind the frequency varint's low bit. + assert_eq!(encoded, vec![12, 3, 0, 0, 1, 3]); + + let mut pairs = Vec::new(); + for_each_entry_pair(&encoded, |freq, doc_len| pairs.push((freq, doc_len))).unwrap(); + assert_eq!(pairs, vec![(1, 1), (2, 2), (3, 5)]); + } + + #[test] + fn malformed_norm_deltas_are_rejected() { + // doc_up_to=0, pair_count=1, and the pair flag promises a norm byte + // that is not present. + let truncated = [0, 1, 1]; + let error = for_each_entry_pair(&truncated, |_, _| {}).unwrap_err(); + assert!(matches!(&error, Error::Index { .. })); + assert!(error.to_string().contains("impact norm delta")); + + // The first pair reaches norm 255, so an implicit +1 on the second + // pair must fail instead of wrapping back to zero. + let overflowing = [0, 2, 1, 255, 0]; + let error = for_each_entry_pair(&overflowing, |_, _| {}).unwrap_err(); + assert!(matches!(&error, Error::Index { .. })); + assert!(error.to_string().contains("impact norm delta overflow")); + } + + #[test] + fn impact_entries_roundtrip_quantized_frontier() { + let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80), (4095, 130, 900)]; + let encoded = encode_impact_entry(&docs).unwrap(); + assert_eq!(decode_entry_doc_up_to(&encoded).unwrap(), 4095); + let mut decoded_pairs = Vec::new(); + for_each_entry_pair(&encoded, |freq, doc_len| { + decoded_pairs.push((freq, doc_len)) + }) + .unwrap(); + let expected_pairs = quantized_impact_frontier(&docs) + .into_iter() + .map(|(freq, norm)| (freq, super::super::index::dequantize_doc_length(norm))) + .collect::>(); + assert_eq!(decoded_pairs, expected_pairs); + assert!(!decoded_pairs.is_empty()); + + // A 256-doc-block skip data goes through the shared codec end to end. + let blocks: Vec> = (0..3) + .map(|b| (0..256).map(|i| (b * 256 + i, 1 + i % 5, 10)).collect()) + .collect(); + let impacts = build_impact_skip_data(&blocks).unwrap(); + assert_eq!(impacts.level1_doc_up_to(0), Some(767)); + let scorer = MemBM25Scorer::new(400, 768, HashMap::from([(String::from("t"), 768usize)])); + let mut cache = ImpactScoreCache::default(); + assert!( + impacts + .level0_score_cached(0, 1.0, &scorer, &mut cache) + .is_finite() + ); + let level1 = impacts.max_score_up_to_cached(0, 767, 1.0, &scorer, &mut cache); + assert!(level1.score.is_finite() && level1.score > 0.0); + } + + #[test] + fn posting_block_sizes_use_identical_impact_encoding() { + let docs = vec![(3, 1, 100), (9, 2, 40), (200, 7, 80)]; + let mut v2_builder = ImpactSkipDataBuilder::with_capacity(1, 128); + v2_builder.append_block(&docs).unwrap(); + let v2 = v2_builder.finish().unwrap(); + let mut v3_builder = ImpactSkipDataBuilder::with_capacity(1, 256); + v3_builder.append_block(&docs).unwrap(); + let v3 = v3_builder.finish().unwrap(); + + assert_eq!(v2.entries(), v3.entries()); + } + + #[test] + fn impact_upper_bound_covers_real_scores() { + let blocks = vec![ + vec![(0, 1, 100), (3, 2, 40), (7, 4, 80)], + vec![(9, 3, 15), (10, 1, 5), (12, 5, 30)], + vec![(16, 2, 10), (18, 6, 70), (21, 3, 12)], + vec![(24, 1, 4), (28, 7, 100), (30, 2, 8)], + ]; + let impacts = build_impact_skip_data(&blocks).unwrap(); + let scorer = MemBM25Scorer::new(474, 31, HashMap::from([(String::from("token"), 4usize)])); + let query_weight = scorer.query_weight("token"); + let mut cache = ImpactScoreCache::default(); + + for start_block_idx in 0..blocks.len() { + let up_to = blocks + .iter() + .skip(start_block_idx) + .take(2) + .flatten() + .map(|(doc_id, _, _)| *doc_id) + .max() + .unwrap(); + let upper_bound = impacts.max_score_up_to_cached( + start_block_idx, + u64::from(up_to), + query_weight, + &scorer, + &mut cache, + ); + let exact_max = blocks + .iter() + .skip(start_block_idx) + .flatten() + .take_while(|(doc_id, _, _)| *doc_id <= up_to) + .map(|(_, freq, doc_len)| query_weight * scorer.doc_weight(*freq, *doc_len)) + .fold(0.0_f32, f32::max); + assert!( + upper_bound.score + 1e-6 >= exact_max, + "upper bound {} should cover exact max {} from block {} up to doc {}", + upper_bound.score, + exact_max, + start_block_idx, + up_to + ); + } + } +} diff --git a/rust/lance-index/src/scalar/inverted/index.rs b/rust/lance-index/src/scalar/inverted/index.rs index d5a513c7112..660339fefd3 100644 --- a/rust/lance-index/src/scalar/inverted/index.rs +++ b/rust/lance-index/src/scalar/inverted/index.rs @@ -3,14 +3,14 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::fmt::{Debug, Display}; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; use std::{ cmp::{Reverse, min}, collections::BinaryHeap, }; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, ops::Range, time::Instant, }; @@ -18,7 +18,8 @@ use std::{ use crate::metrics::NoOpMetricsCollector; use crate::prefilter::NoFilter; use crate::scalar::registry::{TrainingCriteria, TrainingOrdering}; -use arrow::array::{FixedSizeListBuilder, Float32Builder}; +use crate::vector::graph::OrderedFloat; +use arrow::array::{BooleanBuilder, FixedSizeListBuilder, Float32Builder, Int32Builder}; use arrow::datatypes::{self, Float32Type, Int32Type, UInt64Type}; use arrow::{ array::{ @@ -28,8 +29,8 @@ use arrow::{ }; use arrow::{buffer::ScalarBuffer, datatypes::UInt32Type}; use arrow_array::{ - Array, ArrayRef, Float32Array, LargeBinaryArray, ListArray, OffsetSizeTrait, RecordBatch, - UInt32Array, UInt64Array, + Array, ArrayRef, BooleanArray, Float32Array, LargeBinaryArray, ListArray, OffsetSizeTrait, + RecordBatch, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use async_trait::async_trait; @@ -40,7 +41,9 @@ use fst::{Automaton, IntoStreamer, Streamer}; use futures::{FutureExt, Stream, StreamExt, TryStreamExt, stream}; use itertools::{Either, Itertools}; use lance_arrow::{RecordBatchExt, iter_str_array}; -use lance_core::cache::{CacheCodec, CacheKey, LanceCache, WeakLanceCache}; +use lance_core::cache::{ + CacheCodec, CacheKey, CacheKeySchema, KeyBuilder, LanceCache, WeakLanceCache, +}; use lance_core::deepsize::DeepSizeOf; use lance_core::error::{DataFusionResult, LanceOptionExt}; use lance_core::utils::address::RowAddress; @@ -50,17 +53,25 @@ use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result}; use lance_select::{RowAddrMask, RowAddrTreeMap}; use roaring::RoaringBitmap; use std::sync::LazyLock; -use tokio::{sync::OnceCell, task::spawn_blocking}; -use tracing::{info, instrument}; +use tokio::{ + sync::{Mutex, OnceCell}, + task::spawn_blocking, +}; +use tracing::{debug, info, instrument, warn}; -use super::encoding::{PositionBlockBuilder, decode_group_starts}; +use super::documents::{ + DocId, DocLengths, DocVisibility, PartitionDocumentStore, PartitionDocuments, +}; +use super::encoding::{MAX_POSTING_BLOCK_SIZE, PositionBlockBuilder}; +use super::impact::{IMPACT_LEVEL1_BLOCKS, ImpactSkipData, ImpactSkipDataBuilder}; use super::iter::PostingListIterator; -use super::lazy_docset::LazyDocSet; -use super::{InvertedIndexBuilder, InvertedIndexParams, wand::*}; +use super::tokenizer::{LEGACY_BLOCK_SIZE, validate_block_size}; +use super::{DocumentGranularity, InvertedIndexBuilder, InvertedIndexParams, wand::*}; use super::{ builder::{ - BLOCK_SIZE, PostingGroupAccumulator, PostingGroupConfig, ScoredDoc, doc_file_path, - inverted_list_schema_for_version, posting_file_path, token_file_path, + BLOCK_SIZE, ScoredDoc, doc_file_path, + inverted_list_schema_for_version_with_block_size_and_impacts, posting_file_path, + token_file_path, }, iter::PlainPostingListIterator, query::*, @@ -79,9204 +90,41 @@ use crate::scalar::{ OldIndexDataFilter, RowIdRemapper, ScalarIndex, ScalarIndexParams, SearchResult, TokenQuery, UpdateCriteria, }; -use crate::{FtsPrewarmOptions, Index}; +use crate::{ + FtsPrewarmDiagnostics, FtsPrewarmOptions, FtsPrewarmPartitionStatus, FtsPrewarmResult, Index, +}; use crate::{prefilter::PreFilter, scalar::inverted::iter::take_fst_keys}; use std::str::FromStr; -// Version 0: Arrow TokenSetFormat (legacy) -// Version 1: Fst TokenSetFormat with per-doc compressed positions -// Version 2: Fst TokenSetFormat with shared posting-list position streams. -pub const INVERTED_INDEX_VERSION_V1: u32 = 1; -pub const INVERTED_INDEX_VERSION_V2: u32 = 2; -pub const TOKENS_FILE: &str = "tokens.lance"; -pub const INVERT_LIST_FILE: &str = "invert.lance"; -pub const DOCS_FILE: &str = "docs.lance"; -pub const METADATA_FILE: &str = "metadata.lance"; - -pub const TOKEN_COL: &str = "_token"; -pub const TOKEN_ID_COL: &str = "_token_id"; -pub const TOKEN_FST_BYTES_COL: &str = "_token_fst_bytes"; -pub const TOKEN_NEXT_ID_COL: &str = "_token_next_id"; -pub const TOKEN_TOTAL_LENGTH_COL: &str = "_token_total_length"; -pub const FREQUENCY_COL: &str = "_frequency"; -pub const POSITION_COL: &str = "_position"; -pub const COMPRESSED_POSITION_COL: &str = "_compressed_position"; -pub const POSITION_BLOCK_OFFSET_COL: &str = "_position_block_offset"; -pub const POSTING_COL: &str = "_posting"; -pub const MAX_SCORE_COL: &str = "_max_score"; -pub const LENGTH_COL: &str = "_length"; -pub const BLOCK_MAX_SCORE_COL: &str = "_block_max_score"; -pub const NUM_TOKEN_COL: &str = "_num_tokens"; -pub const SCORE_COL: &str = "_score"; -pub const TOKEN_SET_FORMAT_KEY: &str = "token_set_format"; -pub const POSTING_TAIL_CODEC_KEY: &str = "posting_tail_codec"; -pub const POSITIONS_LAYOUT_KEY: &str = "positions_layout"; -pub const POSITIONS_CODEC_KEY: &str = "positions_codec"; -/// Schema-metadata key holding the 1-indexed global-buffer id of the -/// varint-delta-encoded posting-list cache-group boundaries (issue #7040). -/// Absent on indexes written before grouping was introduced, which fall back -/// to the per-token cache path. -pub const POSTING_GROUP_OFFSETS_BUF_KEY: &str = "posting_group_offsets_buf"; -pub const POSTING_TAIL_CODEC_FIXED32_V1: &str = "fixed32_v1"; -pub const POSTING_TAIL_CODEC_VARINT_DELTA_V1: &str = "varint_delta_v1"; -pub const POSITIONS_LAYOUT_SHARED_STREAM_V2: &str = "shared_stream_v2"; -pub const POSITIONS_CODEC_VARINT_DOC_DELTA_V2: &str = "varint_doc_delta_v2"; -pub const POSITIONS_CODEC_PACKED_DELTA_V1: &str = "packed_delta_v1"; -pub const DELETED_FRAGMENTS_COL: &str = "deleted_fragments"; - -// Just a heuristic when we need to pre-allocate memory for tokens -pub const ESTIMATED_MAX_TOKENS_PER_ROW: usize = 4 * 1024; - -pub static SCORE_FIELD: LazyLock = - LazyLock::new(|| Field::new(SCORE_COL, DataType::Float32, true)); -pub static FTS_SCHEMA: LazyLock = - LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone(), SCORE_FIELD.clone()]))); -static ROW_ID_SCHEMA: LazyLock = - LazyLock::new(|| Arc::new(Schema::new(vec![ROW_ID_FIELD.clone()]))); - -pub fn resolve_fts_format_version( - value: Option<&str>, -) -> std::result::Result { - match value { - Some(value) => value.parse(), - None => Ok(default_fts_format_version()), - } -} - -pub fn default_fts_format_version() -> InvertedListFormatVersion { - InvertedListFormatVersion::V2 -} - -pub fn current_fts_format_version() -> InvertedListFormatVersion { - default_fts_format_version() -} - -pub fn max_supported_fts_format_version() -> InvertedListFormatVersion { - InvertedListFormatVersion::V2 -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum InvertedListFormatVersion { - V1, - #[default] - V2, -} - -impl InvertedListFormatVersion { - pub fn from_posting_tail_codec(codec: PostingTailCodec) -> Self { - match codec { - PostingTailCodec::Fixed32 => Self::V1, - PostingTailCodec::VarintDelta => Self::V2, - } - } - - pub fn index_version(self) -> u32 { - match self { - Self::V1 => INVERTED_INDEX_VERSION_V1, - Self::V2 => INVERTED_INDEX_VERSION_V2, - } - } - - pub fn posting_tail_codec(self) -> PostingTailCodec { - match self { - Self::V1 => PostingTailCodec::Fixed32, - Self::V2 => PostingTailCodec::VarintDelta, - } - } - - pub fn position_codec(self) -> Option { - match self { - Self::V1 => None, - Self::V2 => Some(PositionStreamCodec::PackedDelta), - } - } - - pub fn uses_shared_position_stream(self) -> bool { - matches!(self, Self::V2) - } -} - -impl FromStr for InvertedListFormatVersion { - type Err = Error; - - fn from_str(s: &str) -> std::result::Result { - match s.trim() { - "1" | "v1" | "V1" => Ok(Self::V1), - "2" | "v2" | "V2" => Ok(Self::V2), - other => Err(Error::index(format!( - "unsupported FTS format version {}, expected 1 or 2", - other - ))), - } - } -} - -#[derive(Debug)] -struct PartitionCandidates { - tokens_by_position: Vec, - grouped_expansions: Vec, - candidates: Vec, -} - -impl PartitionCandidates { - fn empty() -> Self { - Self { - tokens_by_position: Vec::new(), - grouped_expansions: Vec::new(), - candidates: Vec::new(), - } - } -} - -#[derive(Debug)] -struct LoadedPostings { - postings: Vec, - grouped_expansions: Vec, -} - -impl LoadedPostings { - fn empty() -> Self { - Self { - postings: Vec::new(), - grouped_expansions: Vec::new(), - } - } -} - -#[derive(Debug)] -struct GroupedExpansionTerms { - position: u32, - terms: Vec, -} - -fn grouped_rescore_wand_limit( - limit: Option, - grouped_expansions: &[GroupedExpansionTerms], -) -> Option { - let limit = limit?; - // Grouped fuzzy AND rescoring needs a small candidate cushion because WAND - // ranks by the unioned group posting first and the exact expansion IDF later. - let expansion_terms = grouped_expansions - .iter() - .map(|group| group.terms.len()) - .sum::() - .max(1); - Some(limit.saturating_mul(expansion_terms)) -} - -#[derive(Debug)] -struct ExpansionTermFreqs { - token: String, - freqs_by_posting_doc_id: Vec<(u64, u32)>, -} - -impl ExpansionTermFreqs { - fn new(token: String, posting: &PostingList) -> Self { - let freqs_by_posting_doc_id = posting - .iter() - .map(|(posting_doc_id, freq, _)| (posting_doc_id, freq)) - .collect(); - Self { - token, - freqs_by_posting_doc_id, - } - } - - fn frequency(&self, posting_doc_id: u64) -> Option { - self.freqs_by_posting_doc_id - .binary_search_by_key(&posting_doc_id, |(doc_id, _)| *doc_id) - .ok() - .map(|idx| self.freqs_by_posting_doc_id[idx].1) - } -} - -#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Default)] -pub enum TokenSetFormat { - Arrow, - #[default] - Fst, -} - -impl Display for TokenSetFormat { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Arrow => f.write_str("arrow"), - Self::Fst => f.write_str("fst"), - } - } -} - -impl FromStr for TokenSetFormat { - type Err = Error; - - fn from_str(s: &str) -> std::result::Result { - match s.trim() { - "" => Ok(Self::Arrow), - "arrow" => Ok(Self::Arrow), - "fst" => Ok(Self::Fst), - other => Err(Error::index(format!( - "unsupported token set format {}", - other - ))), - } - } -} - -impl DeepSizeOf for TokenSetFormat { - fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize { - 0 - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum PositionStreamCodec { - VarintDocDelta, - #[default] - PackedDelta, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] -pub enum PostingTailCodec { - Fixed32, - #[default] - VarintDelta, -} - -impl PostingTailCodec { - pub fn as_str(self) -> &'static str { - match self { - Self::Fixed32 => POSTING_TAIL_CODEC_FIXED32_V1, - Self::VarintDelta => POSTING_TAIL_CODEC_VARINT_DELTA_V1, - } - } - - fn from_metadata_value(value: &str) -> Result { - match value.trim() { - POSTING_TAIL_CODEC_FIXED32_V1 => Ok(Self::Fixed32), - POSTING_TAIL_CODEC_VARINT_DELTA_V1 => Ok(Self::VarintDelta), - other => Err(Error::index(format!( - "unsupported posting tail codec {}", - other - ))), - } - } -} - -pub(super) fn parse_posting_tail_codec( - metadata: &HashMap, -) -> Result { - Ok(metadata - .get(POSTING_TAIL_CODEC_KEY) - .map(|codec| PostingTailCodec::from_metadata_value(codec)) - .transpose()? - .unwrap_or(PostingTailCodec::Fixed32)) -} - -impl PositionStreamCodec { - pub fn as_str(self) -> &'static str { - match self { - Self::VarintDocDelta => POSITIONS_CODEC_VARINT_DOC_DELTA_V2, - Self::PackedDelta => POSITIONS_CODEC_PACKED_DELTA_V1, - } - } - - fn from_metadata_value(value: &str) -> Result { - match value.trim() { - POSITIONS_CODEC_VARINT_DOC_DELTA_V2 => Ok(Self::VarintDocDelta), - POSITIONS_CODEC_PACKED_DELTA_V1 => Ok(Self::PackedDelta), - other => Err(Error::index(format!( - "unsupported positions codec {}", - other - ))), - } - } -} - -fn parse_shared_position_codec(metadata: &HashMap) -> Result { - if let Some(codec) = metadata.get(POSITIONS_CODEC_KEY) { - return PositionStreamCodec::from_metadata_value(codec); - } - - match metadata - .get(POSITIONS_LAYOUT_KEY) - .map(|layout| layout.as_str()) - { - Some(POSITIONS_LAYOUT_SHARED_STREAM_V2) => Ok(PositionStreamCodec::VarintDocDelta), - _ => Ok(PositionStreamCodec::VarintDocDelta), - } -} - -pub(super) fn parse_format_version_from_metadata( - metadata: &HashMap, -) -> Result { - if metadata.contains_key(POSITIONS_CODEC_KEY) || metadata.contains_key(POSITIONS_LAYOUT_KEY) { - return Ok(InvertedListFormatVersion::V2); - } - if parse_posting_tail_codec(metadata)? == PostingTailCodec::VarintDelta { - Ok(InvertedListFormatVersion::V2) - } else { - Ok(InvertedListFormatVersion::V1) - } -} - -#[derive(Clone)] -pub struct InvertedIndex { - params: InvertedIndexParams, - store: Arc, - tokenizer: Box, - token_set_format: TokenSetFormat, - format_version: InvertedListFormatVersion, - pub(crate) partitions: Vec>, - corpus_stats: Arc>, - // Fragments which are contained in the index, but no longer in the dataset. - // These should be pruned at search time since we don't prune them at update time. - deleted_fragments: RoaringBitmap, -} - -impl Debug for InvertedIndex { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("InvertedIndex") - .field("params", &self.params) - .field("token_set_format", &self.token_set_format) - .field("format_version", &self.format_version) - .field("partitions", &self.partitions) - .field("deleted_fragments", &self.deleted_fragments) - .finish() - } -} - -impl DeepSizeOf for InvertedIndex { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.partitions.deep_size_of_children(context) - } -} - -/// Resolve any `Pending` candidates that wand emitted via the -/// deferred-row_id path. After this returns, every entry in -/// `candidates` carries a real row_id. -async fn resolve_deferred_candidates( - docs: &LazyDocSet, - candidates: &mut [DocCandidate], -) -> Result<()> { - let pending: Vec = candidates - .iter() - .filter_map(|c| match c.addr { - CandidateAddr::Pending(d) => Some(d), - CandidateAddr::RowId(_) => None, - }) - .collect(); - if pending.is_empty() { - return Ok(()); - } - let mut iter = docs.resolve_row_ids(&pending).await?.into_iter(); - for c in candidates { - if matches!(c.addr, CandidateAddr::Pending(_)) { - let r = iter.next().ok_or_else(|| { - Error::internal("resolve_row_ids returned fewer items than requested") - })?; - c.addr = CandidateAddr::RowId(r); - } - } - Ok(()) -} - -impl InvertedIndex { - fn format_version(&self) -> InvertedListFormatVersion { - self.format_version - } - - fn index_version(&self) -> u32 { - match self.token_set_format { - TokenSetFormat::Arrow => 0, - TokenSetFormat::Fst => self.format_version().index_version(), - } - } - - fn posting_tail_codec(&self) -> PostingTailCodec { - self.partitions - .first() - .map(|partition| partition.inverted_list.posting_tail_codec()) - .unwrap_or_default() - } - - fn to_builder(&self) -> InvertedIndexBuilder { - self.to_builder_with_offset(None) - } - - fn to_builder_with_offset(&self, fragment_mask: Option) -> InvertedIndexBuilder { - if self.is_legacy() { - // for legacy format, we re-create the index in the new format - InvertedIndexBuilder::from_existing_index( - self.params.clone(), - None, - Vec::new(), - self.token_set_format, - fragment_mask, - self.deleted_fragments.clone(), - ) - .with_posting_tail_codec(self.posting_tail_codec()) - } else { - let partitions = match fragment_mask { - Some(fragment_mask) => self - .partitions - .iter() - // Filter partitions that belong to the specified fragment - // The mask contains fragment_id in high 32 bits, we check if partition's - // fragment_id matches by comparing the masked result with the original mask - .filter(|part| part.belongs_to_fragment(fragment_mask)) - .map(|part| part.id()) - .collect(), - None => self.partitions.iter().map(|part| part.id()).collect(), - }; - - InvertedIndexBuilder::from_existing_index( - self.params.clone(), - Some(self.store.clone()), - partitions, - self.token_set_format, - fragment_mask, - self.deleted_fragments.clone(), - ) - .with_format_version(self.format_version()) - } - } - - pub fn tokenizer(&self) -> Box { - self.tokenizer.clone() - } - - pub fn params(&self) -> &InvertedIndexParams { - &self.params - } - - /// Returns the number of partitions in this inverted index. - pub fn partition_count(&self) -> usize { - self.partitions.len() - } - /// Returns the set of fragments which are contained in the index, but no longer in the dataset. - /// - /// Most other indices remove data from deleted fragments when the index updates (copy-on-write). - /// However, this would require an expensive copy of the FTS index. Instead, we track the deleted - /// fragments and prune them at search time (merge-on-read). - pub fn deleted_fragments(&self) -> &RoaringBitmap { - &self.deleted_fragments - } - - pub async fn merge_segments( - segments: &[Arc], - new_data: SendableRecordBatchStream, - dest_store: &dyn IndexStore, - old_data_filter: Option, - progress: Arc, - ) -> Result { - let Some(first) = segments.first() else { - return Err(Error::invalid_input( - "cannot merge inverted index without at least one source segment".to_string(), - )); - }; - - for segment in segments.iter().skip(1) { - if segment.params != first.params { - return Err(Error::index( - "cannot merge inverted index segments with different parameters".to_string(), - )); - } - if segment.token_set_format != first.token_set_format { - return Err(Error::index( - "cannot merge inverted index segments with different token set formats" - .to_string(), - )); - } - if segment.format_version() != first.format_version() { - return Err(Error::index( - "cannot merge inverted index segments with different format versions" - .to_string(), - )); - } - if segment.posting_tail_codec() != first.posting_tail_codec() { - return Err(Error::index( - "cannot merge inverted index segments with different posting tail codecs" - .to_string(), - )); - } - } - - let mut builder = InvertedIndexBuilder::new(first.params.clone()).with_progress(progress); - builder = builder - .with_token_set_format(first.token_set_format) - .with_format_version(first.format_version()) - .with_posting_tail_codec(first.posting_tail_codec()); - let files = builder - .update_from_segments(new_data, dest_store, segments, old_data_filter) - .await?; - - let details = pbold::InvertedIndexDetails::try_from(&first.params)?; - - Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: first.index_version(), - files, - }) - } - - /// Build a single-segment [`MemBM25Scorer`] whose per-term IDF table - /// covers every token that the per-partition scoring loop will look - /// up. For fuzzy queries that means the union of Levenshtein - /// expansions, not just the raw query tokens — otherwise - /// `query_weight(expanded_token)` returns 0 and the BM25 contribution - /// of every expanded match is discarded. - pub async fn bm25_base_scorer( - &self, - query_tokens: &Tokens, - params: &FtsSearchParams, - ) -> Result { - let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?; - let mut terms: Vec = Vec::new(); - let mut seen = HashSet::new(); - if matches!(params.fuzziness, Some(n) if n != 0) { - let expanded = self.expand_fuzzy_tokens(query_tokens, params)?; - for idx in 0..expanded.len() { - let token = expanded.get_token(idx); - if seen.insert(token.to_string()) { - terms.push(token.to_string()); - } - } - } else { - for token in query_tokens { - if seen.insert(token.to_string()) { - terms.push(token.to_string()); - } - } - } - let mut token_docs = HashMap::with_capacity(terms.len()); - for term in &terms { - let df = self.df_for_term(term).await?; - token_docs.insert(term.clone(), df); - } - Ok(MemBM25Scorer::new(total_tokens, num_docs, token_docs)) - } - - pub async fn bm25_stats_for_terms(&self, terms: &[String]) -> Result<(u64, usize, Vec)> { - let (total_tokens, num_docs) = self.aggregate_corpus_stats().await?; - let token_docs = - futures::future::try_join_all(terms.iter().map(|term| self.df_for_term(term))).await?; - Ok((total_tokens, num_docs, token_docs)) - } - - /// Aggregate per-partition `total_tokens` and `num_docs` across the - /// index. `len` is cheap (no IO); `total_tokens_num` reads only the - /// num_tokens column the first time per partition and caches it on - /// `LazyDocSet`. Avoids materializing the full DocSet just to get - /// these two scalars. - async fn aggregate_corpus_stats(&self) -> Result<(u64, usize)> { - self.corpus_stats - .get_or_try_init(|| async { - let io_parallelism = self.store.io_parallelism(); - let num_docs: usize = self.partitions.iter().map(|p| p.docs.len()).sum(); - let futures = self - .partitions - .iter() - .map(|p| { - let docs = p.docs.clone(); - async move { docs.total_tokens_num().await } - }) - .collect::>(); - let totals: Vec = stream::iter(futures) - .buffer_unordered(io_parallelism) - .try_collect() - .await?; - Ok((totals.into_iter().sum(), num_docs)) - }) - .await - .copied() - } - - /// Sum the posting-list length for `term` across this index's partitions - /// via single-row reads, with partition lookups bounded by the store's - /// `io_parallelism()`. - async fn df_for_term(&self, term: &str) -> Result { - let io_parallelism = self.store.io_parallelism(); - let futures = self - .partitions - .iter() - .map(|part| { - let part = part.clone(); - async move { - match part.tokens.get(term) { - Some(token_id) => part.inverted_list.posting_len_for_token(token_id).await, - None => Ok(0), - } - } - }) - .collect::>(); - let dfs: Vec = stream::iter(futures) - .buffer_unordered(io_parallelism) - .try_collect() - .await?; - Ok(dfs.into_iter().sum()) - } - - /// Expand fuzzy query tokens against all partitions in this segment. - pub fn expand_fuzzy_tokens(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result { - let mut expanded_tokens = Vec::new(); - let mut expanded_positions = Vec::new(); - let mut seen = HashSet::new(); - for partition in &self.partitions { - let expanded = partition.expand_fuzzy(tokens, params)?; - for idx in 0..expanded.len() { - let token = expanded.get_token(idx); - let position = expanded.position(idx); - if seen.insert((token.to_string(), position)) { - expanded_tokens.push(token.to_string()); - expanded_positions.push(position); - } - } - } - Ok(Tokens::with_positions( - expanded_tokens, - expanded_positions, - tokens.token_type().clone(), - )) - } - - /// Search documents that match the query and return row ids sorted by BM25 score. - /// - /// When `base_scorer` is provided, search uses those corpus-level BM25 statistics - /// instead of deriving them from this segment alone. - #[instrument(level = "debug", skip_all)] - pub async fn bm25_search( - &self, - tokens: Arc, - params: Arc, - operator: Operator, - prefilter: Arc, - metrics: Arc, - base_scorer: Option<&MemBM25Scorer>, - ) -> Result<(Vec, Vec)> { - // The wand only consults `scorer.doc_weight`, which is metadata-free. - // The outer aggregation below consults `scorer.query_weight`, which - // hits per-token `posting_len`; building a `MemBM25Scorer` with - // precomputed per-term IDFs avoids the v2 bulk metadata pull. - let local_scorer; - let scorer: &dyn Scorer = if let Some(base_scorer) = base_scorer { - base_scorer - } else { - local_scorer = self - .bm25_base_scorer(tokens.as_ref(), params.as_ref()) - .await?; - &local_scorer - }; - - let limit = params.limit.unwrap_or(usize::MAX); - if limit == 0 { - return Ok((Vec::new(), Vec::new())); - } - - fn push_scored_candidate( - candidates: &mut BinaryHeap>, - limit: usize, - addr: CandidateAddr, - score: f32, - ) -> Result<()> { - // resolve_deferred_candidates ran upstream, so every candidate - // carries a real row_id at this point. - let row_id = match addr { - CandidateAddr::RowId(r) => r, - CandidateAddr::Pending(_) => { - return Err(Error::internal( - "bm25_search post-condition: deferred candidate left unresolved", - )); - } - }; - - if candidates.len() < limit { - candidates.push(Reverse(ScoredDoc::new(row_id, score))); - } else if candidates.peek().unwrap().0.score.0 < score { - candidates.pop(); - candidates.push(Reverse(ScoredDoc::new(row_id, score))); - } - Ok(()) - } - - let mask = prefilter.mask(); - - let mut candidates = BinaryHeap::new(); - // Shared top-k floor across this query's partitions. Seeded to -inf so - // the first real score wins; each partition publishes its local k-th - // and prunes against the running global k-th (a lower bound on the true - // global k-th — see `Wand::shared_threshold`). - let shared_threshold = Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits())); - let parts = self - .partitions - .iter() - .map(|part| { - let part = part.clone(); - let tokens = tokens.clone(); - let params = params.clone(); - let mask = mask.clone(); - let metrics = metrics.clone(); - let shared_threshold = shared_threshold.clone(); - async move { - let loaded_postings = part - .load_posting_lists( - tokens.as_ref(), - params.as_ref(), - operator, - metrics.as_ref(), - ) - .await?; - let LoadedPostings { - postings, - grouped_expansions, - } = loaded_postings; - if postings.is_empty() { - // No hits in this partition; its DocSet stays - // unloaded, so we never pay the per-doc - // row_id/num_tokens download for it. - return Result::Ok(PartitionCandidates::empty()); - } - let docs_for_wand = part.docs.docs_for_wand(mask.as_ref()).await?; - let max_position = postings - .iter() - .map(|posting| posting.term_index() as usize) - .max() - .unwrap_or_default(); - let mut tokens_by_position = vec![String::new(); max_position + 1]; - for posting in &postings { - let idx = posting.term_index() as usize; - tokens_by_position[idx] = posting.token().to_owned(); - } - let params = params.clone(); - let mask = mask.clone(); - let metrics = metrics.clone(); - let part_for_wand = part.clone(); - let has_grouped_expansions = !grouped_expansions.is_empty(); - let wand_params = if has_grouped_expansions { - let mut rescoring_params = params.as_ref().clone(); - rescoring_params.limit = - grouped_rescore_wand_limit(params.limit, &grouped_expansions); - Arc::new(rescoring_params) - } else { - params.clone() - }; - let partition_threshold = if has_grouped_expansions { - Arc::new(AtomicU32::new(f32::NEG_INFINITY.to_bits())) - } else { - shared_threshold - }; - let candidates = spawn_cpu(move || { - let candidates = part_for_wand.bm25_search( - docs_for_wand.as_ref(), - wand_params.as_ref(), - operator, - mask, - postings, - metrics.as_ref(), - partition_threshold, - )?; - std::result::Result::<_, Error>::Ok(candidates) - }) - .await?; - let mut partition_result = PartitionCandidates { - tokens_by_position, - grouped_expansions, - candidates, - }; - resolve_deferred_candidates(&part.docs, &mut partition_result.candidates) - .await?; - Result::Ok(partition_result) - } - }) - .collect::>(); - let mut parts = stream::iter(parts).buffer_unordered(get_num_compute_intensive_cpus()); - let mut idf_cache: HashMap = HashMap::new(); - while let Some(res) = parts.try_next().await? { - if res.candidates.is_empty() { - continue; - } - let PartitionCandidates { - tokens_by_position, - grouped_expansions, - candidates: part_candidates, - } = res; - let mut idf_by_position = Vec::with_capacity(tokens_by_position.len()); - for token in &tokens_by_position { - let idf_weight = match idf_cache.get(token) { - Some(weight) => *weight, - None => { - let weight = scorer.query_weight(token); - idf_cache.insert(token.clone(), weight); - weight - } - }; - idf_by_position.push(idf_weight); - } - - if grouped_expansions.is_empty() { - for DocCandidate { - addr, - freqs, - doc_length, - .. - } in part_candidates - { - let mut score = 0.0; - for (term_index, freq) in freqs.into_iter() { - debug_assert!((term_index as usize) < idf_by_position.len()); - score += idf_by_position[term_index as usize] - * scorer.doc_weight(freq, doc_length); - } - push_scored_candidate(&mut candidates, limit, addr, score)?; - } - } else { - let grouped_positions = grouped_expansions - .iter() - .map(|group| group.position) - .collect::>(); - for DocCandidate { - addr, - posting_doc_id, - freqs, - doc_length, - } in part_candidates - { - let mut score = 0.0; - for (term_index, freq) in freqs.into_iter() { - if grouped_positions.contains(&term_index) { - continue; - } - debug_assert!((term_index as usize) < idf_by_position.len()); - score += idf_by_position[term_index as usize] - * scorer.doc_weight(freq, doc_length); - } - for group in &grouped_expansions { - for term in &group.terms { - let Some(freq) = term.frequency(posting_doc_id) else { - continue; - }; - let idf_weight = match idf_cache.get(&term.token) { - Some(weight) => *weight, - None => { - let weight = scorer.query_weight(&term.token); - idf_cache.insert(term.token.clone(), weight); - weight - } - }; - score += idf_weight * scorer.doc_weight(freq, doc_length); - } - } - push_scored_candidate(&mut candidates, limit, addr, score)?; - } - } - } - - Ok(candidates - .into_sorted_vec() - .into_iter() - .map(|Reverse(doc)| (doc.row_id, doc.score.0)) - .unzip()) - } - - async fn load_legacy_index( - store: Arc, - frag_reuse_index: Option>, - index_cache: &LanceCache, - ) -> Result> { - log::warn!("loading legacy FTS index"); - let tokens_fut = tokio::spawn({ - let store = store.clone(); - async move { - let token_reader = store.open_index_file(TOKENS_FILE).await?; - let tokenizer = token_reader - .schema() - .metadata - .get("tokenizer") - .map(|s| serde_json::from_str::(s)) - .transpose()? - .unwrap_or_default(); - let tokens = TokenSet::load(token_reader, TokenSetFormat::Arrow).await?; - Result::Ok((tokenizer, tokens)) - } - }); - let invert_list_fut = tokio::spawn({ - let store = store.clone(); - let index_cache_clone = index_cache.clone(); - async move { - let invert_list_reader = store.open_index_file(INVERT_LIST_FILE).await?; - let invert_list = - PostingListReader::try_new(invert_list_reader, &index_cache_clone).await?; - Result::Ok(Arc::new(invert_list)) - } - }); - let docs_fut = tokio::spawn({ - let store = store.clone(); - async move { - let docs_reader = store.open_index_file(DOCS_FILE).await?; - let docs = DocSet::load(docs_reader, true, frag_reuse_index).await?; - Result::Ok(docs) - } - }); - - let (tokenizer_config, tokens) = tokens_fut.await??; - let inverted_list = invert_list_fut.await??; - let docs = docs_fut.await??; - - let tokenizer = tokenizer_config.build()?; - - Ok(Arc::new(Self { - params: tokenizer_config, - store: store.clone(), - tokenizer, - token_set_format: TokenSetFormat::Arrow, - format_version: InvertedListFormatVersion::V1, - partitions: vec![Arc::new(InvertedPartition { - id: 0, - store, - tokens, - inverted_list, - docs: Arc::new(LazyDocSet::from_loaded(docs)), - token_set_format: TokenSetFormat::Arrow, - })], - corpus_stats: Arc::new(OnceCell::new()), - deleted_fragments: RoaringBitmap::new(), - })) - } - - pub fn is_legacy(&self) -> bool { - self.partitions.len() == 1 && self.partitions[0].is_legacy() - } - - pub async fn load( - store: Arc, - frag_reuse_index: Option>, - index_cache: &LanceCache, - ) -> Result> - where - Self: Sized, - { - // for new index format, there is a metadata file and multiple partitions, - // each partition is a separate index containing tokens, inverted list and docs. - // for old index format, there is no metadata file, and it's just like a single partition - - match store.open_index_file(METADATA_FILE).await { - Ok(reader) => { - let params = reader - .schema() - .metadata - .get("params") - .ok_or(Error::index("params not found in metadata".to_owned()))?; - let params = serde_json::from_str::(params)?; - let partitions = reader - .schema() - .metadata - .get("partitions") - .ok_or(Error::index("partitions not found in metadata".to_owned()))?; - let partitions: Vec = serde_json::from_str(partitions)?; - let token_set_format = reader - .schema() - .metadata - .get(TOKEN_SET_FORMAT_KEY) - .map(|name| TokenSetFormat::from_str(name)) - .transpose()? - .unwrap_or(TokenSetFormat::Arrow); - let format_version = parse_format_version_from_metadata(&reader.schema().metadata)?; - - // Load deleted_fragments if present (optional for backward compatibility) - let deleted_fragments = if reader.num_rows() > 0 { - let metadata_batch = reader.read_range(0..1, None).await?; - if let Some(col) = metadata_batch.column_by_name(DELETED_FRAGMENTS_COL) { - let arr = col.as_binary_opt::().expect_ok()?; - RoaringBitmap::deserialize_from(arr.value(0))? - } else { - RoaringBitmap::new() - } - } else { - RoaringBitmap::new() - }; - - let format = token_set_format; - let partitions = partitions.into_iter().enumerate().map(|(priority, id)| { - let store = store.with_io_priority(priority as u64); - let frag_reuse_index_clone = frag_reuse_index.clone(); - let index_cache_for_part = - index_cache.with_key_prefix(format!("part-{}", id).as_str()); - let token_set_format = format; - async move { - Result::Ok(Arc::new( - InvertedPartition::load( - store, - id, - frag_reuse_index_clone, - &index_cache_for_part, - token_set_format, - ) - .await?, - )) - } - }); - let partitions = stream::iter(partitions) - .buffer_unordered(store.io_parallelism()) - .try_collect::>() - .await?; - - let tokenizer = params.build()?; - Ok(Arc::new(Self { - params, - store, - tokenizer, - token_set_format, - format_version, - partitions, - corpus_stats: Arc::new(OnceCell::new()), - deleted_fragments, - })) - } - Err(_) => { - // old index format - Self::load_legacy_index(store, frag_reuse_index, index_cache).await - } - } - } -} - -#[async_trait] -impl Index for InvertedIndex { - fn as_any(&self) -> &dyn std::any::Any { - self - } - - fn as_index(self: Arc) -> Arc { - self - } - - fn statistics(&self) -> Result { - let num_tokens = self - .partitions - .iter() - .map(|part| part.tokens.len()) - .sum::(); - let num_docs = self - .partitions - .iter() - .map(|part| part.docs.len()) - .sum::(); - Ok(serde_json::json!({ - "params": self.params, - "num_tokens": num_tokens, - "num_docs": num_docs, - })) - } - - async fn prewarm(&self) -> Result<()> { - self.prewarm_with_options(&FtsPrewarmOptions::default()) - .await - } - - fn index_type(&self) -> crate::IndexType { - crate::IndexType::Inverted - } - - async fn calculate_included_frags(&self) -> Result { - unimplemented!() - } -} - -/// Target on-disk size of one prewarm chunk. Keep this large enough that cloud -/// stores do not spend prewarm time on thousands of tiny range reads, but still -/// bounded so one large partition is not materialized all at once. -const PREWARM_CHUNK_TARGET_BYTES: u64 = 128 << 20; - -/// Cap on token rows per chunk, bounding the built `Vec` when posting lists are tiny. -const PREWARM_MAX_CHUNK_TOKENS: usize = 256 * 1024; - -/// Floor on token rows per chunk, so a partition always makes progress. -const PREWARM_MIN_CHUNK_TOKENS: usize = 1; - -/// Token rows per chunk: byte target / average bytes-per-token, clamped to `[MIN, MAX]`. -fn prewarm_chunk_tokens(token_count: usize, file_size_bytes: u64) -> usize { - if token_count == 0 { - return PREWARM_MIN_CHUNK_TOKENS; - } - let bytes_per_token = (file_size_bytes / token_count as u64).max(1); // >= 1: no div-by-zero - let by_bytes = (PREWARM_CHUNK_TARGET_BYTES / bytes_per_token) as usize; - by_bytes.clamp(PREWARM_MIN_CHUNK_TOKENS, PREWARM_MAX_CHUNK_TOKENS) -} - -/// Snap a chunk's exclusive token end back to a posting-group boundary so no group -/// straddles chunks. Returns the largest group boundary in `(tok_start, desired_end]`, -/// or the next boundary past an oversized group so it runs as one solo chunk. -fn group_aligned_chunk_end( - starts: &[u32], - token_count: usize, - tok_start: usize, - desired_end: usize, -) -> usize { - if desired_end >= token_count { - return token_count; - } - - let first_after_start = starts.partition_point(|&start| start as usize <= tok_start); - let first_after_desired = starts.partition_point(|&start| start as usize <= desired_end); - if first_after_desired > first_after_start { - return starts[first_after_desired - 1] as usize; - } - - // Oversized group: extend to its end so it runs as one chunk. - starts - .get(first_after_start) - .map(|&start| start as usize) - .unwrap_or(token_count) -} - -fn prewarm_chunk_ranges( - group_starts: Option<&[u32]>, - token_count: usize, - chunk_tokens: usize, -) -> Vec<(usize, usize)> { - let mut ranges = Vec::new(); - let mut tok_start = 0usize; - while tok_start < token_count { - let mut tok_end = (tok_start + chunk_tokens).min(token_count); - // `tok_start` is always a group boundary; snap `tok_end` back to one too. - if let Some(starts) = group_starts { - tok_end = group_aligned_chunk_end(starts, token_count, tok_start, tok_end); - } - ranges.push((tok_start, tok_end)); - tok_start = tok_end; - } - ranges -} - -fn group_start_indices_for_chunk(starts: &[u32], tok_start: usize, tok_end: usize) -> Range { - let first = starts.partition_point(|&start| (start as usize) < tok_start); - let end = starts.partition_point(|&start| (start as usize) < tok_end); - first..end -} - -fn group_range_for_start_index(starts: &[u32], token_count: usize, group_idx: usize) -> (u32, u32) { - let start = starts[group_idx]; - let end = starts - .get(group_idx + 1) - .copied() - .unwrap_or(token_count as u32); - (start, end) -} - -impl InvertedIndex { - pub async fn prewarm_with_options(&self, options: &FtsPrewarmOptions) -> Result<()> { - let with_position = options.with_position; - let chunk_concurrency = self.store.io_parallelism().max(1); - for part in &self.partitions { - part.inverted_list - .prewarm_posting_lists(with_position, chunk_concurrency) - .await?; - // Materialize the deferred DocSet too: prewarm's contract is - // that subsequent queries do no IO, so the per-doc row_ids / - // num_tokens must be resident, not lazily faulted in at query - // time. `ensure_loaded` opens, reads, and drops the reader. - part.docs.ensure_loaded().await?; - } - Ok(()) - } - /// Search docs match the input text. - async fn do_search(&self, text: &str) -> Result { - let params = FtsSearchParams::new(); - let mut tokenizer = self.tokenizer.clone(); - let tokens = collect_query_tokens(text, &mut tokenizer); - - let (doc_ids, _) = self - .bm25_search( - Arc::new(tokens), - params.into(), - Operator::And, - Arc::new(NoFilter), - Arc::new(NoOpMetricsCollector), - None, - ) - .boxed() - .await?; - - Ok(RecordBatch::try_new( - ROW_ID_SCHEMA.clone(), - vec![Arc::new(UInt64Array::from(doc_ids))], - )?) - } -} - -#[async_trait] -impl ScalarIndex for InvertedIndex { - // return the row ids of the documents that contain the query - #[instrument(level = "debug", skip_all)] - async fn search( - &self, - query: &dyn AnyQuery, - _metrics: &dyn MetricsCollector, - ) -> Result { - let query = query.as_any().downcast_ref::().unwrap(); - - match query { - TokenQuery::TokensContains(text) => { - let records = self.do_search(text).await?; - let row_ids = records - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let row_ids = row_ids.iter().flatten().collect_vec(); - Ok(SearchResult::at_most(RowAddrTreeMap::from_iter(row_ids))) - } - } - } - - fn can_remap(&self) -> bool { - true - } - - async fn remap( - &self, - mapping: &RowAddrRemap, - dest_store: &dyn IndexStore, - ) -> Result { - let files = self - .to_builder() - .remap(mapping, self.store.clone(), dest_store) - .await?; - - let details = pbold::InvertedIndexDetails::try_from(&self.params)?; - - Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: self.index_version(), - files, - }) - } - - async fn update( - &self, - new_data: SendableRecordBatchStream, - dest_store: &dyn IndexStore, - old_data_filter: Option, - ) -> Result { - let files = self - .to_builder() - .update(new_data, dest_store, old_data_filter) - .await?; - - let details = pbold::InvertedIndexDetails::try_from(&self.params)?; - - Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&details).unwrap(), - index_version: self.index_version(), - files, - }) - } - - fn update_criteria(&self) -> UpdateCriteria { - let criteria = TrainingCriteria::new(TrainingOrdering::None).with_row_id(); - if self.is_legacy() { - UpdateCriteria::requires_old_data(criteria) - } else { - UpdateCriteria::only_new_data(criteria) - } - } - - fn derive_index_params(&self) -> Result { - let mut params = self.params.clone(); - if params.base_tokenizer.is_empty() { - // Empty tokenizer metadata only appears in legacy simple-tokenizer indexes. - params.base_tokenizer = "simple".to_string(); - } - params = params.format_version(self.format_version()); - - let params_json = params.to_training_json()?.to_string(); - - Ok(ScalarIndexParams { - index_type: BuiltinIndexType::Inverted.as_str().to_string(), - params: Some(params_json), - }) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -pub struct InvertedPartition { - // 0 for legacy format - id: u64, - store: Arc, - pub(crate) tokens: TokenSet, - pub(crate) inverted_list: Arc, - /// Per-doc row_id + num_tokens. Wrapped in `LazyDocSet` so partitions - /// that don't contribute hits to a query never pay the full-array - /// download. Scoring paths call `ensure_loaded` before walking wand. - pub(crate) docs: Arc, - token_set_format: TokenSetFormat, -} - -impl InvertedPartition { - /// Check if this partition belongs to the specified fragment. - /// - /// This method encapsulates the bit manipulation logic for fragment filtering - /// in distributed indexing scenarios. - /// - /// # Arguments - /// * `fragment_mask` - A mask with fragment_id in high 32 bits - /// - /// # Returns - /// * `true` if the partition belongs to the fragment, `false` otherwise - pub fn belongs_to_fragment(&self, fragment_mask: u64) -> bool { - (self.id() & fragment_mask) == fragment_mask - } - - pub fn id(&self) -> u64 { - self.id - } - - pub fn store(&self) -> &dyn IndexStore { - self.store.as_ref() - } - - pub fn is_legacy(&self) -> bool { - self.inverted_list.is_legacy_layout() - } - - pub async fn load( - store: Arc, - id: u64, - frag_reuse_index: Option>, - index_cache: &LanceCache, - token_set_format: TokenSetFormat, - ) -> Result { - let token_file = store.open_index_file(&token_file_path(id)).await?; - let tokens = TokenSet::load(token_file, token_set_format).await?; - let invert_list_file = store.open_index_file(&posting_file_path(id)).await?; - let inverted_list = PostingListReader::try_new(invert_list_file, index_cache).await?; - // Defer the per-doc row_id/num_tokens read. Construction reads only - // the doc count (one footer read) and then drops the reader; the bulk - // load happens on first scoring use, re-opening the docs file on - // demand, and partitions that never score skip it entirely. Storing - // the store + path instead of an open reader keeps a cached partition - // from pinning a docs-file handle for its whole lifetime. - let docs_path = doc_file_path(id); - let num_docs = store.open_index_file(&docs_path).await?.num_rows(); - let docs = Arc::new(LazyDocSet::new( - store.clone(), - docs_path, - num_docs, - false, - frag_reuse_index, - )); - - Ok(Self { - id, - store, - tokens, - inverted_list: Arc::new(inverted_list), - docs, - token_set_format, - }) - } - - fn map(&self, token: &str) -> Option { - self.tokens.get(token) - } - - pub fn expand_fuzzy(&self, tokens: &Tokens, params: &FtsSearchParams) -> Result { - let mut new_tokens = Vec::with_capacity(min(tokens.len(), params.max_expansions)); - let mut new_positions = Vec::with_capacity(new_tokens.capacity()); - let mut seen = HashSet::new(); - for token_idx in 0..tokens.len() { - if new_tokens.len() >= params.max_expansions { - break; - } - let token = tokens.get_token(token_idx); - let position = tokens.position(token_idx); - let fuzziness = match params.fuzziness { - Some(fuzziness) => fuzziness, - None => MatchQuery::auto_fuzziness(token), - }; - let lev = fst::automaton::Levenshtein::new(token, fuzziness) - .map_err(|e| Error::index(format!("failed to construct the fuzzy query: {}", e)))?; - - let base_len = tokens.token_type().prefix_len(token) as u32; - if let TokenMap::Fst(ref map) = self.tokens.tokens { - let mut expanded = Vec::new(); - let remaining = params.max_expansions - new_tokens.len(); - match base_len + params.prefix_length { - 0 => take_fst_keys(map.search(lev), &mut expanded, remaining), - prefix_length => { - let prefix = &token[..min(prefix_length as usize, token.len())]; - let prefix = fst::automaton::Str::new(prefix).starts_with(); - take_fst_keys( - map.search(lev.intersection(prefix)), - &mut expanded, - remaining, - ) - } - } - for token in expanded { - if seen.insert((token.clone(), position)) { - new_tokens.push(token); - new_positions.push(position); - if new_tokens.len() >= params.max_expansions { - break; - } - } - } - } else { - return Err(Error::index( - "tokens is not fst, which is not expected".to_owned(), - )); - } - } - Ok(Tokens::with_positions( - new_tokens, - new_positions, - tokens.token_type().clone(), - )) - } - - fn union_plain_posting_lists(postings: Vec) -> Result { - let mut freqs_by_row_id = BTreeMap::new(); - for posting in postings { - for (row_id, freq, _) in posting.iter() { - let entry = freqs_by_row_id.entry(row_id).or_insert(0u32); - *entry = entry.checked_add(freq).ok_or_else(|| { - Error::index(format!("posting frequency overflow for row id {}", row_id)) - })?; - } - } - let mut row_ids = Vec::with_capacity(freqs_by_row_id.len()); - let mut frequencies = Vec::with_capacity(freqs_by_row_id.len()); - for (row_id, freq) in freqs_by_row_id { - row_ids.push(row_id); - frequencies.push(freq as f32); - } - Ok(PostingList::Plain(PlainPostingList::new( - ScalarBuffer::from(row_ids), - ScalarBuffer::from(frequencies), - None, - None, - ))) - } - - fn union_compressed_posting_lists( - postings: Vec, - docs: &DocSet, - ) -> Result { - let mut freqs_by_doc_id = BTreeMap::new(); - for posting in postings { - for (doc_id, freq, _) in posting.iter() { - let doc_id = u32::try_from(doc_id).map_err(|_| { - Error::index(format!( - "compressed posting doc id {} exceeds u32::MAX", - doc_id - )) - })?; - let entry = freqs_by_doc_id.entry(doc_id).or_insert(0u32); - *entry = entry.checked_add(freq).ok_or_else(|| { - Error::index(format!("posting frequency overflow for doc id {}", doc_id)) - })?; - } - } - if freqs_by_doc_id.is_empty() { - return Ok(PostingList::Plain(PlainPostingList::new( - ScalarBuffer::from(Vec::::new()), - ScalarBuffer::from(Vec::::new()), - None, - None, - ))); - } - - let mut builder = PostingListBuilder::new(false); - let mut doc_ids = Vec::with_capacity(freqs_by_doc_id.len()); - let mut frequencies = Vec::with_capacity(freqs_by_doc_id.len()); - for (doc_id, freq) in freqs_by_doc_id { - builder.add(doc_id, PositionRecorder::Count(freq)); - doc_ids.push(doc_id); - frequencies.push(freq); - } - let block_max_scores = docs.calculate_block_max_scores(doc_ids.iter(), frequencies.iter()); - let batch = builder.to_batch(block_max_scores)?; - let max_score = batch[MAX_SCORE_COL].as_primitive::().value(0); - let length = batch[LENGTH_COL].as_primitive::().value(0); - PostingList::from_batch(&batch, Some(max_score), Some(length)) - } - - fn union_posting_lists(postings: Vec, docs: &DocSet) -> Result { - let has_plain = postings - .iter() - .any(|posting| matches!(posting, PostingList::Plain(_))); - let has_compressed = postings - .iter() - .any(|posting| matches!(posting, PostingList::Compressed(_))); - match (has_plain, has_compressed) { - (true, true) => Err(Error::index( - "cannot union mixed plain and compressed posting lists".to_owned(), - )), - (true, false) => Self::union_plain_posting_lists(postings), - (false, true) => Self::union_compressed_posting_lists(postings, docs), - (false, false) => Ok(PostingList::Plain(PlainPostingList::new( - ScalarBuffer::from(Vec::::new()), - ScalarBuffer::from(Vec::::new()), - None, - None, - ))), - } - } - - // search the documents that contain the query - // return the doc info and the doc length - // ref: https://en.wikipedia.org/wiki/Okapi_BM25 - #[instrument(level = "debug", skip_all)] - async fn load_posting_lists( - &self, - tokens: &Tokens, - params: &FtsSearchParams, - operator: Operator, - metrics: &dyn MetricsCollector, - ) -> Result { - let is_fuzzy = matches!(params.fuzziness, Some(n) if n != 0); - let is_phrase_query = params.phrase_slop.is_some(); - let is_and_query = operator == Operator::And; - let required_positions = (is_and_query || is_phrase_query).then(|| { - (0..tokens.len()) - .map(|index| tokens.position(index)) - .collect::>() - }); - let tokens = match is_fuzzy { - true => self.expand_fuzzy(tokens, params)?, - false => tokens.clone(), - }; - let token_positions = (0..tokens.len()) - .map(|index| tokens.position(index)) - .collect::>(); - let mut token_ids = Vec::with_capacity(tokens.len()); - let mut matched_positions = required_positions.as_ref().map(|_| HashSet::new()); - for (index, token) in tokens.into_iter().enumerate() { - let token_id = self.map(&token); - if let Some(token_id) = token_id { - let position = token_positions[index]; - if let Some(matched_positions) = matched_positions.as_mut() { - matched_positions.insert(position); - } - token_ids.push((token_id, token, position)); - } else if is_phrase_query || is_and_query { - // if the token is not found, we can't do phrase or AND query - return Ok(LoadedPostings::empty()); - } - } - if token_ids.is_empty() { - return Ok(LoadedPostings::empty()); - } - if let Some(required_positions) = required_positions.as_ref() - && let Some(matched_positions) = matched_positions.as_ref() - && !required_positions.is_subset(matched_positions) - { - return Ok(LoadedPostings::empty()); - } - - let is_fuzzy_and_query = is_fuzzy && is_and_query && !is_phrase_query; - if !is_phrase_query { - if is_fuzzy_and_query { - token_ids.sort_unstable_by_key(|(token_id, _, position)| (*position, *token_id)); - token_ids.dedup_by(|lhs, rhs| lhs.0 == rhs.0 && lhs.2 == rhs.2); - } else { - token_ids.sort_unstable_by_key(|(token_id, _, _)| *token_id); - token_ids.dedup_by_key(|(token_id, _, _)| *token_id); - } - } - - let num_docs = self.docs.len(); - let loaded_postings = stream::iter(token_ids) - .map(|(token_id, token, position)| async move { - let posting = self - .inverted_list - .posting_list(token_id, is_phrase_query, metrics) - .await?; - - Result::Ok((token_id, token, position, posting)) - }) - .buffered(self.store.io_parallelism()) - .try_collect::>() - .await?; - - if (is_and_query || is_phrase_query) - && !is_fuzzy_and_query - && loaded_postings - .iter() - .any(|(_, _, _, posting)| posting.is_empty()) - { - return Ok(LoadedPostings::empty()); - } - - if !is_fuzzy_and_query { - return Ok(LoadedPostings { - postings: loaded_postings - .into_iter() - .map(|(token_id, token, position, posting)| { - let query_weight = idf(posting.len(), num_docs); - PostingIterator::with_query_weight( - token, - token_id, - position, - query_weight, - posting, - num_docs, - ) - }) - .collect(), - grouped_expansions: Vec::new(), - }); - } - - let needs_union = loaded_postings - .windows(2) - .any(|window| window[0].2 == window[1].2); - let docs_for_union = if needs_union { - Some(self.docs.ensure_num_tokens_loaded().await?) - } else { - None - }; - - // WAND's AND mode treats every iterator as required, so expansions from - // one original query position must be merged before scoring. - let mut grouped_postings = Vec::new(); - let mut grouped_expansions = Vec::new(); - let mut iter = loaded_postings.into_iter().peekable(); - while let Some((token_id, token, position, posting)) = iter.next() { - let mut group = vec![(token_id, token, posting)]; - while matches!(iter.peek(), Some((_, _, next_position, _)) if *next_position == position) - { - let (token_id, token, _, posting) = iter.next().expect("peeked item must exist"); - group.push((token_id, token, posting)); - } - - let (token_id, token, posting) = if group.len() == 1 { - group.pop().expect("single-item group must exist") - } else { - let token_id = group[0].0; - let token = group[0].1.clone(); - grouped_expansions.push(GroupedExpansionTerms { - position, - terms: group - .iter() - .map(|(_, token, posting)| ExpansionTermFreqs::new(token.clone(), posting)) - .collect(), - }); - let postings = group - .into_iter() - .map(|(_, _, posting)| posting) - .collect::>(); - let posting = Self::union_posting_lists( - postings, - docs_for_union - .as_deref() - .expect("union docs must be loaded for grouped fuzzy AND"), - )?; - (token_id, token, posting) - }; - if posting.is_empty() { - return Ok(LoadedPostings::empty()); - } - - let query_weight = idf(posting.len(), num_docs); - grouped_postings.push(PostingIterator::with_query_weight( - token, - token_id, - position, - query_weight, - posting, - num_docs, - )); - } - - Ok(LoadedPostings { - postings: grouped_postings, - grouped_expansions, - }) - } - - #[instrument(level = "debug", skip_all)] - // Deferred-DocSet adds the `docs` param (caller materializes it) on top of - // the cross-partition `shared_threshold`, tipping this hot-path search fn - // one over the limit. Bundling args isn't worth the churn here. - #[allow(clippy::too_many_arguments)] - pub fn bm25_search( - &self, - docs: &DocSet, - params: &FtsSearchParams, - operator: Operator, - mask: Arc, - postings: Vec, - metrics: &dyn MetricsCollector, - shared_threshold: Arc, - ) -> Result> { - if postings.is_empty() { - return Ok(Vec::new()); - } - - // Caller selects the DocSet shape via `LazyDocSet::docs_for_wand` - // and passes it in here; wand uses `docs.has_row_ids()` to - // handle the num_tokens-only case. - let scorer = IndexBM25Scorer::new(std::iter::once(self)); - let mut wand = Wand::new(operator, postings.into_iter(), docs, scorer) - .with_shared_threshold(shared_threshold); - let hits = wand.search(params, mask, metrics)?; - Ok(hits) - } - - pub async fn into_builder(self) -> Result { - let mut builder = InnerBuilder::new_with_posting_tail_codec( - self.id, - self.inverted_list.has_positions(), - self.token_set_format, - self.inverted_list.posting_tail_codec(), - ); - builder.tokens = self.tokens.into_mutable(); - // into_builder rewrites every doc, so materialize the full - // DocSet now and clone it out of the Arc. - let docs_arc = self.docs.ensure_loaded().await?; - builder.docs = (*docs_arc).clone(); - - builder - .posting_lists - .reserve_exact(self.inverted_list.len()); - for posting_list in self - .inverted_list - .read_all(self.inverted_list.has_positions()) - .await? - { - let posting_list = posting_list?; - builder - .posting_lists - .push(posting_list.into_builder(&builder.docs)); - } - Ok(builder) - } -} - -// at indexing, we use HashMap because we need it to be mutable, -// at searching, we use fst::Map because it's more efficient -#[derive(Debug, Clone)] -pub enum TokenMap { - HashMap(HashMap), - Fst(fst::Map>), -} - -impl Default for TokenMap { - fn default() -> Self { - Self::HashMap(HashMap::new()) - } -} - -impl DeepSizeOf for TokenMap { - fn deep_size_of_children(&self, ctx: &mut lance_core::deepsize::Context) -> usize { - match self { - Self::HashMap(map) => map.deep_size_of_children(ctx), - Self::Fst(map) => map.as_fst().size(), - } - } -} - -impl TokenMap { - pub fn len(&self) -> usize { - match self { - Self::HashMap(map) => map.len(), - Self::Fst(map) => map.len(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } -} - -// TokenSet is a mapping from tokens to token ids -#[derive(Debug, Clone, Default, DeepSizeOf)] -pub struct TokenSet { - // token -> token_id - pub(crate) tokens: TokenMap, - pub(crate) next_id: u32, - total_length: usize, -} - -impl TokenSet { - pub fn into_mut(self) -> Self { - let tokens = match self.tokens { - TokenMap::HashMap(map) => map, - TokenMap::Fst(map) => { - let mut new_map = HashMap::with_capacity(map.len()); - let mut stream = map.into_stream(); - while let Some((token, token_id)) = stream.next() { - new_map.insert(String::from_utf8_lossy(token).into_owned(), token_id as u32); - } - - new_map - } - }; - - Self { - tokens: TokenMap::HashMap(tokens), - next_id: self.next_id, - total_length: self.total_length, - } - } - - pub fn len(&self) -> usize { - self.tokens.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn to_batch(self, format: TokenSetFormat) -> Result { - match format { - TokenSetFormat::Arrow => self.into_arrow_batch(), - TokenSetFormat::Fst => self.into_fst_batch(), - } - } - - fn into_arrow_batch(self) -> Result { - let mut token_builder = StringBuilder::with_capacity(self.tokens.len(), self.total_length); - let mut token_id_builder = UInt32Builder::with_capacity(self.tokens.len()); - - match self.tokens { - TokenMap::Fst(map) => { - let mut stream = map.stream(); - while let Some((token, token_id)) = stream.next() { - token_builder.append_value(String::from_utf8_lossy(token)); - token_id_builder.append_value(token_id as u32); - } - } - TokenMap::HashMap(map) => { - for (token, token_id) in map.into_iter().sorted_unstable() { - token_builder.append_value(token); - token_id_builder.append_value(token_id); - } - } - } - - let token_col = token_builder.finish(); - let token_id_col = token_id_builder.finish(); - - let schema = arrow_schema::Schema::new(vec![ - arrow_schema::Field::new(TOKEN_COL, DataType::Utf8, false), - arrow_schema::Field::new(TOKEN_ID_COL, DataType::UInt32, false), - ]); - - let batch = RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(token_col) as ArrayRef, - Arc::new(token_id_col) as ArrayRef, - ], - )?; - Ok(batch) - } - - fn into_fst_batch(mut self) -> Result { - let fst_map = match std::mem::take(&mut self.tokens) { - TokenMap::Fst(map) => map, - TokenMap::HashMap(map) => Self::build_fst_from_map(map)?, - }; - let bytes = fst_map.into_fst().into_inner(); - - let mut fst_builder = LargeBinaryBuilder::with_capacity(1, bytes.len()); - fst_builder.append_value(bytes); - let fst_col = fst_builder.finish(); - - let mut next_id_builder = UInt32Builder::with_capacity(1); - next_id_builder.append_value(self.next_id); - let next_id_col = next_id_builder.finish(); - - let mut total_length_builder = UInt64Builder::with_capacity(1); - total_length_builder.append_value(self.total_length as u64); - let total_length_col = total_length_builder.finish(); - - let schema = arrow_schema::Schema::new(vec![ - arrow_schema::Field::new(TOKEN_FST_BYTES_COL, DataType::LargeBinary, false), - arrow_schema::Field::new(TOKEN_NEXT_ID_COL, DataType::UInt32, false), - arrow_schema::Field::new(TOKEN_TOTAL_LENGTH_COL, DataType::UInt64, false), - ]); - - let batch = RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(fst_col) as ArrayRef, - Arc::new(next_id_col) as ArrayRef, - Arc::new(total_length_col) as ArrayRef, - ], - )?; - Ok(batch) - } - - fn build_fst_from_map(map: HashMap) -> Result>> { - let mut entries: Vec<_> = map.into_iter().collect(); - entries.sort_unstable_by(|(lhs, _), (rhs, _)| lhs.cmp(rhs)); - let mut builder = fst::MapBuilder::memory(); - for (token, token_id) in entries { - builder - .insert(&token, token_id as u64) - .map_err(|e| Error::index(format!("failed to insert token {}: {}", token, e)))?; - } - Ok(builder.into_map()) - } - - pub async fn load(reader: Arc, format: TokenSetFormat) -> Result { - match format { - TokenSetFormat::Arrow => Self::load_arrow(reader).await, - TokenSetFormat::Fst => Self::load_fst(reader).await, - } - } - - async fn load_arrow(reader: Arc) -> Result { - let batch = reader.read_range(0..reader.num_rows(), None).await?; - - let (tokens, next_id, total_length) = spawn_blocking(move || { - let mut next_id = 0; - let mut total_length = 0; - let mut tokens = fst::MapBuilder::memory(); - - let token_col = batch[TOKEN_COL].as_string::(); - let token_id_col = batch[TOKEN_ID_COL].as_primitive::(); - - for (token, &token_id) in token_col.iter().zip(token_id_col.values().iter()) { - let token = - token.ok_or(Error::index("found null token in token set".to_owned()))?; - next_id = next_id.max(token_id + 1); - total_length += token.len(); - tokens.insert(token, token_id as u64).map_err(|e| { - Error::index(format!("failed to insert token {}: {}", token, e)) - })?; - } - - Ok::<_, Error>((tokens.into_map(), next_id, total_length)) - }) - .await - .map_err(|err| Error::execution(format!("failed to spawn blocking task: {}", err)))??; - - Ok(Self { - tokens: TokenMap::Fst(tokens), - next_id, - total_length, - }) - } - - async fn load_fst(reader: Arc) -> Result { - let batch = reader.read_range(0..reader.num_rows(), None).await?; - if batch.num_rows() == 0 { - return Err(Error::index("token set batch is empty".to_owned())); - } - - let fst_col = batch[TOKEN_FST_BYTES_COL].as_binary::(); - let bytes = fst_col.value(0); - let map = fst::Map::new(bytes.to_vec()) - .map_err(|e| Error::index(format!("failed to load fst tokens: {}", e)))?; - - let total_length_col = - batch[TOKEN_TOTAL_LENGTH_COL].as_primitive::(); - - // Token ids are dense `[0, len)`, so `next_id` must equal the token count. Recompute - // it instead of trusting the persisted value, which writers before #7115 could leave - // stale. Mirrors `load_arrow`. - let next_id = map.len() as u32; - - let total_length = total_length_col - .values() - .first() - .copied() - .ok_or(Error::index( - "token total length column is empty".to_owned(), - ))?; - - Ok(Self { - tokens: TokenMap::Fst(map), - next_id, - total_length: usize::try_from(total_length).map_err(|_| { - Error::index(format!( - "token total length {} overflows usize", - total_length - )) - })?, - }) - } - - pub fn add(&mut self, token: String) -> u32 { - let next_id = self.next_id(); - let len = token.len(); - let token_id = match self.tokens { - TokenMap::HashMap(ref mut map) => *map.entry(token).or_insert(next_id), - _ => unreachable!("tokens must be HashMap while indexing"), - }; - - // add token if it doesn't exist - if token_id == next_id { - self.next_id += 1; - self.total_length += len; - } - - token_id - } - - pub(crate) fn get_or_add(&mut self, token: &str) -> u32 { - let next_id = self.next_id; - match self.tokens { - TokenMap::HashMap(ref mut map) => { - if let Some(&token_id) = map.get(token) { - return token_id; - } - - map.insert(token.to_owned(), next_id); - } - _ => unreachable!("tokens must be HashMap while indexing"), - } - - self.next_id += 1; - self.total_length += token.len(); - next_id - } - - pub(crate) fn into_mutable(self) -> Self { - let Self { - tokens, - next_id, - total_length, - } = self; - match tokens { - TokenMap::HashMap(_) => Self { - tokens, - next_id, - total_length, - }, - TokenMap::Fst(map) => { - let mut mutable = HashMap::new(); - let mut stream = map.stream(); - while let Some((token, token_id)) = stream.next() { - mutable.insert(String::from_utf8_lossy(token).into_owned(), token_id as u32); - } - Self { - tokens: TokenMap::HashMap(mutable), - next_id, - total_length, - } - } - } - } - - pub fn get(&self, token: &str) -> Option { - match self.tokens { - TokenMap::HashMap(ref map) => map.get(token).copied(), - TokenMap::Fst(ref map) => map.get(token).map(|id| id as u32), - } - } - - // the `removed_token_ids` must be sorted - pub fn remap(&mut self, removed_token_ids: &[u32]) { - if removed_token_ids.is_empty() { - return; - } - - let mut map = match std::mem::take(&mut self.tokens) { - TokenMap::HashMap(map) => map, - TokenMap::Fst(map) => { - let mut new_map = HashMap::with_capacity(map.len()); - let mut stream = map.into_stream(); - while let Some((token, token_id)) = stream.next() { - new_map.insert(String::from_utf8_lossy(token).into_owned(), token_id as u32); - } - - new_map - } - }; - - let mut retained_length = 0; - map.retain( - |token, token_id| match removed_token_ids.binary_search(token_id) { - Ok(_) => false, - Err(index) => { - *token_id -= index as u32; - retained_length += token.len(); - true - } - }, - ); - - self.tokens = TokenMap::HashMap(map); - - // The retain above compacts the surviving token ids into a dense `[0, len)` - // range, so `next_id` (handed to the next new token) must follow them down. - // `total_length` likewise must drop the removed tokens' bytes; it is persisted - // and feeds memory accounting, so a stale value drifts across remap/merge cycles. - self.next_id = self.tokens.len() as u32; - self.total_length = retained_length; - } - - pub fn next_id(&self) -> u32 { - self.next_id - } - - pub(crate) fn memory_size(&self) -> usize { - match &self.tokens { - TokenMap::HashMap(map) => { - self.total_length - + map.capacity() - * (std::mem::size_of::() - + std::mem::size_of::() - + std::mem::size_of::()) - } - TokenMap::Fst(map) => map.as_fst().size(), - } - } -} - -pub struct PostingListReader { - reader: Arc, - - /// Layout-specific metadata. V2 keeps its per-token max-score and - /// length columns lazy so opening a partition doesn't drag O(num_tokens) - /// bytes off cold storage when the caller only needs `df` for a few terms. - metadata: PostingMetadata, - - has_position: bool, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, - - /// First row of each posting-list cache group, decoded at open from the - /// global buffer named by [`POSTING_GROUP_OFFSETS_BUF_KEY`] (issue #7040). - /// `None` for indexes written before grouping; those use the per-token - /// cache path. Always present for grouped v2 indexes with `>0` rows. - group_starts: Option>, - - index_cache: WeakLanceCache, -} - -/// Per-token metadata (max_score, length) needed by the BM25 query and stats -/// paths. The legacy and v2 formats store this metadata in different -/// places, with very different cost profiles for cold-load: the variants -/// surface that asymmetry so callers can choose a per-token or bulk access -/// pattern. -enum PostingMetadata { - /// Legacy v1: offsets and max_scores are encoded in the file's schema - /// metadata, so they are already in memory by the time `try_new` returns. - LegacyV1 { - offsets: Vec, - max_scores: Option>, - }, - /// V2: per-token `max_score` and `length` live as columns in the - /// posting file. The bulk vectors are filled lazily by - /// `ensure_metadata_loaded`, and the stats path can also fetch a single - /// token via `posting_len_for_token` without forcing the bulk load. - V2 { - metadata: OnceCell, - }, -} - -#[derive(Debug, Clone)] -struct LoadedPostingMetadata { - max_scores: Vec, - lengths: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PositionsLayout { - None, - LegacyPerDoc, - SharedStream(PositionStreamCodec), -} - -impl std::fmt::Debug for PostingListReader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let mut s = f.debug_struct("InvertedListReader"); - match &self.metadata { - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } => { - s.field("layout", &"legacy_v1") - .field("offsets", offsets) - .field("max_scores", max_scores); - } - PostingMetadata::V2 { metadata } => { - s.field("layout", &"v2") - .field("metadata_loaded", &metadata.initialized()); - } - } - s.finish() - } -} - -impl DeepSizeOf for PostingListReader { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - let metadata_size = match &self.metadata { - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } => offsets.deep_size_of_children(context) + max_scores.deep_size_of_children(context), - PostingMetadata::V2 { metadata } => metadata - .get() - .map(|loaded| { - loaded.max_scores.deep_size_of_children(context) - + loaded.lengths.deep_size_of_children(context) - }) - .unwrap_or(0), - }; - metadata_size + self.group_starts.deep_size_of_children(context) - } -} - -impl PostingListReader { - pub(crate) async fn try_new( - reader: Arc, - index_cache: &LanceCache, - ) -> Result { - let positions_layout = if reader.schema().field(COMPRESSED_POSITION_COL).is_some() { - PositionsLayout::SharedStream(parse_shared_position_codec(&reader.schema().metadata)?) - } else if reader.schema().field(POSITION_COL).is_some() { - PositionsLayout::LegacyPerDoc - } else { - PositionsLayout::None - }; - let posting_tail_codec = parse_posting_tail_codec(&reader.schema().metadata)?; - let has_position = positions_layout != PositionsLayout::None; - let metadata = if reader.schema().field(POSTING_COL).is_none() { - let (offsets, max_scores) = Self::load_metadata(reader.schema())?; - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } - } else { - PostingMetadata::V2 { - metadata: OnceCell::new(), - } - }; - - let group_starts = Self::load_group_starts(reader.as_ref()).await?; - - Ok(Self { - reader, - metadata, - has_position, - posting_tail_codec, - positions_layout, - group_starts, - index_cache: WeakLanceCache::from(index_cache), - }) - } - - /// Decode the posting-list cache-group boundaries from the global buffer - /// recorded in schema metadata, if present (issue #7040). Returns `None` - /// for indexes written before grouping was introduced. - async fn load_group_starts(reader: &dyn IndexReader) -> Result>> { - let Some(buf_id) = reader.schema().metadata.get(POSTING_GROUP_OFFSETS_BUF_KEY) else { - return Ok(None); - }; - let buf_id: u32 = buf_id.parse().map_err(|e| { - Error::index(format!( - "invalid {POSTING_GROUP_OFFSETS_BUF_KEY} metadata value {buf_id:?}: {e}" - )) - })?; - let bytes = reader.read_global_buffer(buf_id).await?; - let group_starts = decode_group_starts(&bytes)?; - Ok(Some(group_starts)) - } - - // for legacy format - // returns the offsets and max scores - fn load_metadata( - schema: &lance_core::datatypes::Schema, - ) -> Result<(Vec, Option>)> { - let offsets = schema - .metadata - .get("offsets") - .ok_or(Error::index("offsets not found in metadata".to_owned()))?; - let offsets = serde_json::from_str(offsets)?; - - let max_scores = schema - .metadata - .get("max_scores") - .map(|max_scores| serde_json::from_str(max_scores)) - .transpose()?; - Ok((offsets, max_scores)) - } - - // the number of posting lists - pub fn len(&self) -> usize { - match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => offsets.len(), - PostingMetadata::V2 { .. } => self.reader.num_rows(), - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub(crate) fn has_positions(&self) -> bool { - self.has_position - } - - pub(crate) fn posting_tail_codec(&self) -> PostingTailCodec { - self.posting_tail_codec - } - - fn is_legacy_layout(&self) -> bool { - matches!(self.metadata, PostingMetadata::LegacyV1 { .. }) - } - - /// Sync access to `posting_len`. Requires v2 metadata to already be - /// loaded via [`ensure_metadata_loaded`]; the bm25 scoring path enforces - /// that contract before kicking off wand. The stats path uses - /// [`Self::posting_len_for_token`] instead, which avoids the bulk load. - pub(crate) fn posting_len(&self, token_id: u32) -> usize { - let token_id = token_id as usize; - match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => { - let next_offset = offsets - .get(token_id + 1) - .copied() - .unwrap_or(self.reader.num_rows()); - next_offset - offsets[token_id] - } - PostingMetadata::V2 { metadata } => { - let metadata = metadata - .get() - .expect("v2 posting metadata must be bulk-loaded before sync posting_len; call ensure_metadata_loaded first"); - metadata.lengths[token_id] as usize - } - } - } - - /// Async access to a single token's posting list length. For v2 - /// indexes this reads one row of posting metadata if the bulk metadata has - /// not been loaded yet, and never triggers the bulk load itself. The stats - /// path uses this so a single-term `df` lookup costs O(1) bytes rather - /// than O(num_unique_tokens). - pub(crate) async fn posting_len_for_token(&self, token_id: u32) -> Result { - match &self.metadata { - PostingMetadata::LegacyV1 { .. } => Ok(self.posting_len(token_id)), - PostingMetadata::V2 { metadata } => { - if let Some(metadata) = metadata.get() { - return Ok(metadata.lengths[token_id as usize] as usize); - } - let (_, length) = self.posting_metadata_for_token(token_id).await?; - length - .map(|len| len as usize) - .ok_or_else(|| Error::index("posting length metadata missing".to_string())) - } - } - } - - /// Async access to a single token's `(max_score, length)` pair. Mirrors - /// [`Self::posting_len_for_token`] but covers both columns the scoring - /// path needs, in one read. For v2 indexes that have not been - /// bulk-loaded this issues one `read_range(token..token+1, [MAX_SCORE, - /// LENGTH])`; for legacy v1 the values come from in-memory schema - /// metadata. - pub(crate) async fn posting_metadata_for_token( - &self, - token_id: u32, - ) -> Result<(Option, Option)> { - match &self.metadata { - PostingMetadata::LegacyV1 { max_scores, .. } => { - Ok((max_scores.as_ref().map(|m| m[token_id as usize]), None)) - } - PostingMetadata::V2 { metadata } => { - if let Some(loaded) = metadata.get() { - return Ok(( - Some(loaded.max_scores[token_id as usize]), - Some(loaded.lengths[token_id as usize]), - )); - } - let metadata = self - .index_cache - .get_or_insert_with_key(PostingMetadataKey { token_id }, || async move { - let token_id = token_id as usize; - let batch = self - .reader - .read_range(token_id..token_id + 1, Some(&[MAX_SCORE_COL, LENGTH_COL])) - .await?; - let max_score = batch[MAX_SCORE_COL].as_primitive::().value(0); - let length = batch[LENGTH_COL].as_primitive::().value(0); - Ok(PostingMetadataValue { max_score, length }) - }) - .await?; - Ok((Some(metadata.max_score), Some(metadata.length))) - } - } - } - - /// Force the v2 bulk metadata (`max_scores`, `lengths`) into - /// memory. Cheap to call repeatedly; no-op for legacy v1 indexes whose - /// metadata is already populated from schema metadata at `try_new` time. - pub(crate) async fn ensure_metadata_loaded(&self) -> Result<()> { - let PostingMetadata::V2 { metadata } = &self.metadata else { - return Ok(()); - }; - metadata - .get_or_try_init(|| async { - let batch = self - .reader - .read_range( - 0..self.reader.num_rows(), - Some(&[MAX_SCORE_COL, LENGTH_COL]), - ) - .await?; - let max_scores = batch[MAX_SCORE_COL] - .as_primitive::() - .values() - .to_vec(); - let lengths = batch[LENGTH_COL] - .as_primitive::() - .values() - .to_vec(); - Ok::(LoadedPostingMetadata { - max_scores, - lengths, - }) - }) - .await?; - Ok(()) - } - - pub(crate) async fn posting_batch( - &self, - token_id: u32, - with_position: bool, - ) -> Result { - if self.is_legacy_layout() { - self.posting_batch_legacy(token_id, with_position).await - } else { - let token_id = token_id as usize; - let columns = if with_position { - match self.positions_layout { - PositionsLayout::SharedStream(_) => { - vec![ - POSTING_COL, - COMPRESSED_POSITION_COL, - POSITION_BLOCK_OFFSET_COL, - ] - } - PositionsLayout::LegacyPerDoc => vec![POSTING_COL, POSITION_COL], - PositionsLayout::None => vec![POSTING_COL], - } - } else { - vec![POSTING_COL] - }; - let batch = self - .reader - .read_range(token_id..token_id + 1, Some(&columns)) - .await?; - Ok(batch) - } - } - - async fn posting_batch_legacy( - &self, - token_id: u32, - with_position: bool, - ) -> Result { - let mut columns = vec![ROW_ID, FREQUENCY_COL]; - if with_position { - columns.push(POSITION_COL); - } - - let length = self.posting_len(token_id); - let PostingMetadata::LegacyV1 { offsets, .. } = &self.metadata else { - unreachable!("posting_batch_legacy is only reachable on legacy v1 layout"); - }; - let token_id = token_id as usize; - let offset = offsets[token_id]; - let batch = self - .reader - .read_range(offset..offset + length, Some(&columns)) - .await?; - Ok(batch) - } - - #[instrument(level = "debug", skip(self, metrics))] - pub(crate) async fn posting_list( - &self, - token_id: u32, - is_phrase_query: bool, - metrics: &dyn MetricsCollector, - ) -> Result { - let mut posting = match self.group_range_for_token(token_id) { - // Grouped path (issue #7040): one cache entry covers rows - // [start, end), so neighbouring rare terms share a single read. - Some((start, end)) => { - let group = self - .index_cache - .get_or_insert_with_key(PostingListGroupKey { start, end }, || async move { - metrics.record_part_load(); - info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=start); - self.load_posting_list_group(start, end).await - }) - .await?; - let slot = (token_id - start) as usize; - group - .get(slot) - .ok_or_else(|| { - Error::index(format!( - "token {token_id} maps to slot {slot} outside posting group [{start}, {end})" - )) - })? - .clone() - } - // Fallback for indexes written before grouping: one cache entry - // per token. - None => self - .index_cache - .get_or_insert_with_key(PostingListKey { token_id }, || async move { - metrics.record_part_load(); - info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_SCALAR_PART, index_type="inverted", part_id=token_id); - // Fetch the posting batch and this token's (max_score, - // length) in parallel; for cold v2 partitions this is one - // single-row metadata read plus one posting-row read, - // instead of pulling the full per-token metadata table. - let (batch, (max_score, length)) = futures::try_join!( - self.posting_batch(token_id, false), - self.posting_metadata_for_token(token_id), - )?; - self.posting_list_from_batch(&batch, max_score, length) - }) - .await? - .as_ref() - .clone(), - }; - - if is_phrase_query && !posting.has_position() { - // hit the cache and when the cache was populated, the positions column was not loaded - let positions = self.read_positions(token_id).await?; - posting.set_positions(positions); - } - - Ok(posting) - } - - /// Map a token id to its cache group's row range `[start, end)`, or `None` - /// when grouping is not available (pre-grouping indexes) so the caller - /// falls back to the per-token path. In v2 the token id is the row offset, - /// so the group range is also the physical row range. - fn group_range_for_token(&self, token_id: u32) -> Option<(u32, u32)> { - let starts = self.group_starts.as_ref()?; - // partition_point returns the count of group starts <= token_id, so the - // owning group begins at index k - 1 and the next start (if any) is its - // exclusive end. - let k = starts.partition_point(|&s| s <= token_id); - // k == 0 means token_id precedes the first group start, which cannot - // happen for a valid token in a grouped index (the first group starts - // at row 0); guard anyway and fall back to the per-token path. - if k == 0 { - return None; - } - let start = starts[k - 1]; - // The last group runs to the final posting list. `self.len()` is the - // authoritative posting-list count (offsets length for v1, row count for - // v2), and prewarm derives the same `end` from it — so warm- and - // cold-cache group keys are identical by construction, not by the - // incidental v2 `num_rows == token_count` equality. - let end = starts.get(k).copied().unwrap_or(self.len() as u32); - Some((start, end)) - } - - /// Read rows `[start, end)` of the posting file and decode them into a - /// [`PostingListGroup`] cache value (issue #7040). Positions are excluded; - /// phrase queries load them on demand via [`Self::read_positions`]. - async fn load_posting_list_group(&self, start: u32, end: u32) -> Result { - let batch = self - .reader - .read_range( - start as usize..end as usize, - Some(&[POSTING_COL, MAX_SCORE_COL, LENGTH_COL]), - ) - .await?; - let max_scores = batch[MAX_SCORE_COL].as_primitive::(); - let lengths = batch[LENGTH_COL].as_primitive::(); - let mut posting_lists = Vec::with_capacity(batch.num_rows()); - for i in 0..batch.num_rows() { - let row = batch.slice(i, 1); - let posting = self.posting_list_from_batch( - &row, - Some(max_scores.value(i)), - Some(lengths.value(i)), - )?; - posting_lists.push(posting); - } - Ok(PostingListGroup::new(posting_lists)) - } - - fn posting_list_from_batch_parts( - batch: &RecordBatch, - max_score: Option, - length: Option, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, - ) -> Result { - let posting_list = PostingList::from_batch_with_tail_codec_and_positions_layout( - batch, - max_score, - length, - posting_tail_codec, - positions_layout, - )?; - Ok(posting_list) - } - - pub(crate) fn posting_list_from_batch( - &self, - batch: &RecordBatch, - max_score: Option, - length: Option, - ) -> Result { - Self::posting_list_from_batch_parts( - batch, - max_score, - length, - self.posting_tail_codec, - self.positions_layout, - ) - } - - /// Build posting lists for one chunk's token range from `chunk_batch`, rebasing - /// global offsets to chunk-local rows. Returns `(global token_id, PostingList)` - /// pairs identical to the whole-file path, only bounded to one chunk. - fn build_prewarm_posting_lists_chunk( - chunk_batch: RecordBatch, - chunk: PrewarmChunk<'_>, - ctx: &PrewarmBuildCtx<'_>, - ) -> Result> { - let mut posting_lists = Vec::with_capacity(chunk.token_count); - for local in 0..chunk.token_count { - let global = chunk.tok_start + local; - let row_batch = if let Some(chunk_offsets) = chunk.offsets { - // Legacy v1: rebase global offsets to chunk row 0; the last token - // ends at `chunk.end_row` (no trailing sentinel in chunk_offsets). - let base = chunk_offsets[0]; - let start = chunk_offsets[local] - base; - let end = if local + 1 < chunk_offsets.len() { - chunk_offsets[local + 1] - base - } else { - chunk.end_row - base - }; - chunk_batch.slice(start, end - start) - } else { - // V2: one posting row per token; row `local` within the chunk. - chunk_batch.slice(local, 1) - }; - let row_batch = row_batch.shrink_to_fit()?; - let posting_list = Self::posting_list_from_batch_parts( - &row_batch, - ctx.max_scores.map(|scores| scores[global]), - ctx.lengths.map(|lengths| lengths[global]), - ctx.posting_tail_codec, - ctx.positions_layout, - )?; - posting_lists.push((global as u32, posting_list)); - } - - Ok(posting_lists) - } - - /// Read the posting rows for token ids `[tok_start, tok_end)` into one RecordBatch. - /// For v2 the token range is the row range; for v1 it's derived from the offsets. - async fn read_chunk_batch( - &self, - tok_start: usize, - tok_end: usize, - with_position: bool, - ) -> Result { - let columns = self.posting_columns(with_position); - let row_range = match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => { - let start = offsets[tok_start]; - let end = offsets - .get(tok_end) - .copied() - .unwrap_or_else(|| self.reader.num_rows()); - start..end - } - PostingMetadata::V2 { .. } => tok_start..tok_end, - }; - let batch = self.reader.read_range(row_range, Some(&columns)).await?; - Ok(batch) - } - - async fn prewarm_posting_lists( - &self, - with_position: bool, - chunk_concurrency: usize, - ) -> Result<()> { - self.prewarm_posting_lists_chunked(with_position, None, chunk_concurrency) - .await?; - Ok(()) - } - - /// Stream the partition's posting lists into the cache in bounded token-row chunks - /// (read -> build -> insert -> drop), so peak resident set is ~one chunk. Returns - /// the chunk count (tests assert it split). `chunk_tokens_override` is test-only. - async fn prewarm_posting_lists_chunked( - &self, - with_position: bool, - chunk_tokens_override: Option, - chunk_concurrency: usize, - ) -> Result { - if with_position && !self.has_positions() { - return Err(Error::invalid_input( - "cannot prewarm positions for an inverted index that was built without positions; recreate the index with with_position=true".to_owned(), - )); - } - - // Make sure max_scores/lengths are populated before we clone them into - // the blocking task; otherwise the v2 branch would unwrap empty - // OnceCells. - self.ensure_metadata_loaded().await?; - - let state = self.chunk_build_state(); - // With grouping the cache stores one entry per group, so a group's posting - // lists must all be resident at once: align chunk boundaries to whole - // groups. Without grouping, chunks are plain token ranges. - let group_starts = self.group_starts.clone(); - let token_count = self.len(); - let posting_data_size_bytes = self.posting_data_size_bytes(); - let chunk_tokens = chunk_tokens_override - .unwrap_or_else(|| prewarm_chunk_tokens(token_count, posting_data_size_bytes)) - .max(1); - let chunk_ranges = prewarm_chunk_ranges(group_starts.as_deref(), token_count, chunk_tokens); - let chunk_count = chunk_ranges.len(); - let chunk_concurrency = chunk_concurrency.max(1); - - let read_build_start = Instant::now(); - stream::iter(chunk_ranges) - .map(|(tok_start, tok_end)| { - let state = &state; - let group_starts = group_starts.as_deref(); - async move { - let posting_lists = self - .build_chunk_postings(tok_start, tok_end, with_position, state) - .await?; - self.publish_chunk_postings( - posting_lists, - group_starts, - tok_start, - tok_end, - token_count, - with_position, - ) - .await; - Result::Ok(()) - } - }) - .buffer_unordered(chunk_concurrency) - .try_collect::<()>() - .await?; - let read_build_elapsed = read_build_start.elapsed(); - - info!( - legacy_layout = self.is_legacy_layout(), - with_position, - token_count, - chunk_count, - chunk_tokens, - chunk_concurrency, - posting_data_size_bytes, - read_build_ms = read_build_elapsed.as_secs_f64() * 1000.0, - "posting list prewarm timing" - ); - - Ok(chunk_count) - } - - /// Loop-invariant inputs shared by every chunk build: the metadata vecs - /// (`Arc`d so chunks share them without re-cloning) plus codec/layout. - fn chunk_build_state(&self) -> ChunkBuildState { - let (offsets, max_scores, lengths) = match &self.metadata { - PostingMetadata::LegacyV1 { - offsets, - max_scores, - } => (Some(offsets.clone()), max_scores.clone(), None), - PostingMetadata::V2 { metadata } => ( - None, - metadata.get().map(|loaded| loaded.max_scores.clone()), - metadata.get().map(|loaded| loaded.lengths.clone()), - ), - }; - ChunkBuildState { - offsets: offsets.map(Arc::new), - max_scores: max_scores.map(Arc::new), - lengths: lengths.map(Arc::new), - posting_tail_codec: self.posting_tail_codec, - positions_layout: self.positions_layout, - } - } - - /// Read one token-row chunk and build its posting lists off the runtime thread. - /// The large batch is dropped inside the blocking task once built, bounding - /// resident memory to one chunk. - async fn build_chunk_postings( - &self, - tok_start: usize, - tok_end: usize, - with_position: bool, - state: &ChunkBuildState, - ) -> Result> { - let chunk_token_count = tok_end - tok_start; - let chunk_batch = self - .read_chunk_batch(tok_start, tok_end, with_position) - .await?; - - let (chunk_offsets, chunk_end_row) = match state.offsets.as_ref() { - Some(offsets) => { - let end_row = offsets - .get(tok_end) - .copied() - .unwrap_or_else(|| self.reader.num_rows()); - (Some(offsets[tok_start..tok_end].to_vec()), end_row) - } - // V2 doesn't use chunk_end_row (one row per token); pass tok_end. - None => (None, tok_end), - }; - let max_scores = state.max_scores.clone(); - let lengths = state.lengths.clone(); - let posting_tail_codec = state.posting_tail_codec; - let positions_layout = state.positions_layout; - let posting_lists = spawn_blocking(move || { - let ctx = PrewarmBuildCtx { - max_scores: max_scores.as_deref().map(|v| v.as_slice()), - lengths: lengths.as_deref().map(|v| v.as_slice()), - posting_tail_codec, - positions_layout, - }; - let chunk = PrewarmChunk { - tok_start, - token_count: chunk_token_count, - offsets: chunk_offsets.as_deref(), - end_row: chunk_end_row, - }; - Self::build_prewarm_posting_lists_chunk(chunk_batch, chunk, &ctx) - }) - .await - .map_err(|err| { - Error::internal(format!( - "Failed to build prewarm posting lists in blocking task: {err}" - )) - })??; - // The chunk yields its token range as contiguous ascending ids from - // `tok_start`; the group publish path relies on this to index the lists. - debug_assert_eq!(posting_lists.len(), chunk_token_count); - debug_assert!( - posting_lists - .iter() - .enumerate() - .all(|(i, (token_id, _))| *token_id as usize == tok_start + i) - ); - Ok(posting_lists) - } - - /// Strip positions into their own per-token cache entries (the posting cache - /// holds positions-free lists), then populate the same cache keys the read - /// path uses: grouped entries when grouping is active, per-token entries - /// otherwise. Called once per chunk; the chunk's lists drop on return. - async fn publish_chunk_postings( - &self, - posting_lists: Vec<(u32, PostingList)>, - group_starts: Option<&[u32]>, - tok_start: usize, - tok_end: usize, - token_count: usize, - with_position: bool, - ) { - match group_starts { - Some(starts) => { - let mut chunk_postings = Vec::with_capacity(posting_lists.len()); - for (token_id, mut posting_list) in posting_lists { - self.cache_positions(&mut posting_list, token_id, with_position) - .await; - chunk_postings.push(posting_list); - } - // Chunk is group-aligned, so every group starting in it also ends - // in it; `chunk_postings[i]` is token `tok_start + i`. The last - // group's `end` derives from `token_count`, matching the read path - // so both produce identical `PostingListGroupKey`s. - for group_idx in group_start_indices_for_chunk(starts, tok_start, tok_end) { - let (start, end) = group_range_for_start_index(starts, token_count, group_idx); - let start_usize = start as usize; - let lo = start_usize - tok_start; - let hi = end as usize - tok_start; - let group = PostingListGroup::new(chunk_postings[lo..hi].to_vec()); - self.index_cache - .insert_with_key(&PostingListGroupKey { start, end }, Arc::new(group)) - .await; - } - } - None => { - for (token_id, mut posting_list) in posting_lists { - self.cache_positions(&mut posting_list, token_id, with_position) - .await; - self.index_cache - .insert_with_key(&PostingListKey { token_id }, Arc::new(posting_list)) - .await; - } - } - } - } - - /// Move a posting list's positions (when present and requested) into the - /// dedicated per-token position cache, leaving the posting list positions-free. - async fn cache_positions( - &self, - posting_list: &mut PostingList, - token_id: u32, - with_position: bool, - ) { - if with_position && let Some(positions) = posting_list.take_positions() { - self.index_cache - .insert_with_key(&PositionKey { token_id }, Arc::new(Positions(positions))) - .await; - } - } - - /// Cheap `invert.lance` size estimate (file length from object metadata, no - /// data read), used only to size prewarm chunks. Falls back to a row-count - /// proxy when the reader can't surface the length (legacy v1). - pub(crate) fn posting_data_size_bytes(&self) -> u64 { - if let Some(size) = self.reader.file_size_bytes() { - return size; - } - // Fallback proxy for readers that don't cache their file length: just needs - // to be monotonic in partition size. - const ESTIMATED_BYTES_PER_ROW: u64 = 16; - (self.reader.num_rows() as u64).saturating_mul(ESTIMATED_BYTES_PER_ROW) - } - - pub(crate) async fn read_batch(&self, with_position: bool) -> Result { - let columns = self.posting_columns(with_position); - let batch = self - .reader - .read_range(0..self.reader.num_rows(), Some(&columns)) - .await?; - Ok(batch) - } - - pub(crate) async fn read_all( - &self, - with_position: bool, - ) -> Result> + '_> { - // read_all walks every posting list; the bulk metadata is paid for - // unconditionally, so just load it once up front and index into it - // synchronously below. - self.ensure_metadata_loaded().await?; - let batch = self.read_batch(with_position).await?; - Ok((0..self.len()).map(move |i| { - let token_id = i as u32; - let range = self.posting_list_range(token_id); - let batch = batch.slice(i, range.end - range.start); - let (max_score, length) = self.bulk_metadata_for_token(token_id); - self.posting_list_from_batch(&batch, max_score, length) - })) - } - - /// Sync lookup of `(max_score, length)` from the bulk-loaded metadata. - /// Only safe after [`Self::ensure_metadata_loaded`]; callers that hold - /// the OnceCell-loaded reference (e.g. read_all, prewarm) use this to - /// avoid the per-token IO path. - fn bulk_metadata_for_token(&self, token_id: u32) -> (Option, Option) { - match &self.metadata { - PostingMetadata::LegacyV1 { max_scores, .. } => { - (max_scores.as_ref().map(|m| m[token_id as usize]), None) - } - PostingMetadata::V2 { metadata } => { - let loaded = metadata.get().expect( - "v2 metadata must be bulk-loaded before bulk_metadata_for_token; call ensure_metadata_loaded first", - ); - ( - Some(loaded.max_scores[token_id as usize]), - Some(loaded.lengths[token_id as usize]), - ) - } - } - } - - async fn read_positions(&self, token_id: u32) -> Result { - let positions = self.index_cache.get_or_insert_with_key(PositionKey { token_id }, || async move { - let positions = match self.positions_layout { - PositionsLayout::None => { - return Err(Error::invalid_input( - "position is not found but required for phrase queries, try recreating the index with position".to_owned(), - )); - } - PositionsLayout::LegacyPerDoc => { - let batch = self - .reader - .read_range(self.posting_list_range(token_id), Some(&[POSITION_COL])) - .await - .map_err(|e| match e { - Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()), - e => e, - })?; - CompressedPositionStorage::LegacyPerDoc( - batch[POSITION_COL].as_list::().value(0).as_list::().clone(), - ) - } - PositionsLayout::SharedStream(codec) => { - let batch = self - .reader - .read_range( - self.posting_list_range(token_id), - Some(&[COMPRESSED_POSITION_COL, POSITION_BLOCK_OFFSET_COL]), - ) - .await - .map_err(|e| match e { - Error::Schema { .. } => Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position".to_owned()), - e => e, - })?; - let bytes = bytes::Bytes::from( - batch[COMPRESSED_POSITION_COL] - .as_binary::() - .value(0) - .to_vec(), - ); - let block_offsets = batch[POSITION_BLOCK_OFFSET_COL] - .as_list::() - .value(0) - .as_primitive::() - .values() - .to_vec(); - CompressedPositionStorage::SharedStream(SharedPositionStream::new( - codec, - block_offsets, - bytes, - )) - } - }; - Result::Ok(Positions(positions)) - }).await?; - Ok(positions.0.clone()) - } - - fn posting_list_range(&self, token_id: u32) -> Range { - match &self.metadata { - PostingMetadata::LegacyV1 { offsets, .. } => { - let offset = offsets[token_id as usize]; - let posting_len = self.posting_len(token_id); - offset..offset + posting_len - } - PostingMetadata::V2 { .. } => { - let token_id = token_id as usize; - token_id..token_id + 1 - } - } - } - - fn posting_columns(&self, with_position: bool) -> Vec<&'static str> { - let mut base_columns = if self.is_legacy_layout() { - vec![ROW_ID, FREQUENCY_COL] - } else { - vec![POSTING_COL] - }; - if with_position { - match self.positions_layout { - PositionsLayout::None => {} - PositionsLayout::LegacyPerDoc => base_columns.push(POSITION_COL), - PositionsLayout::SharedStream(_) => { - base_columns.push(COMPRESSED_POSITION_COL); - base_columns.push(POSITION_BLOCK_OFFSET_COL); - } - } - } - base_columns - } -} - -/// Loop-invariant state for [`InvertedPartition::build_chunk_postings`]. The -/// metadata vecs are `Arc`d so each chunk's blocking build shares them cheaply. -struct ChunkBuildState { - offsets: Option>>, - max_scores: Option>>, - lengths: Option>>, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, -} - -/// Chunk-invariant inputs to [`InvertedPartition::build_prewarm_posting_lists_chunk`]: -/// the per-partition codec/layout and the (shared, whole-partition) metadata -/// slices indexed by global token id. These don't change across chunks. -struct PrewarmBuildCtx<'a> { - max_scores: Option<&'a [f32]>, - lengths: Option<&'a [u32]>, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, -} - -/// Per-chunk inputs to [`InvertedPartition::build_prewarm_posting_lists_chunk`]: -/// the token sub-range `[tok_start, tok_start + token_count)` and, for legacy -/// v1, the rebased offset slice plus the chunk's end row. -struct PrewarmChunk<'a> { - tok_start: usize, - token_count: usize, - /// Legacy v1 only: `offsets[tok_start..tok_start+token_count]` (no sentinel). - offsets: Option<&'a [usize]>, - /// Legacy v1 only: global row at which this chunk's posting rows end. - end_row: usize, -} - -/// New type just to allow Positions implement DeepSizeOf so it can be put -/// in the cache. -#[derive(Clone)] -pub struct Positions(pub(super) CompressedPositionStorage); - -/// Slice-aware cache-size charge for the Arrow array shapes stored in posting -/// caches. [`Array::get_buffer_memory_size`] reports the full capacity of shared -/// backing buffers; cached posting lists often reference only a small slice of a -/// group read. Count the referenced span for the known posting-list types and -/// fall back to Arrow's full-buffer size for anything else. -fn sliced_cache_bytes(array: &dyn Array) -> usize { - let validity_bytes = array - .nulls() - .map(|nulls| nulls.len().div_ceil(8)) - .unwrap_or(0); - match array.data_type() { - DataType::LargeBinary => { - let array = array.as_binary::(); - let data_bytes = if array.is_empty() { - 0 - } else { - let offsets = array.value_offsets(); - (offsets[array.len()] - offsets[0]) as usize - }; - data_bytes + (array.len() + 1) * std::mem::size_of::() + validity_bytes - } - DataType::List(_) => { - let array = array.as_list::(); - let (child_start, child_end) = if array.is_empty() { - (0, 0) - } else { - let offsets = array.value_offsets(); - (offsets[0] as usize, offsets[array.len()] as usize) - }; - let offset_bytes = (array.len() + 1) * std::mem::size_of::(); - let child = array.values().slice(child_start, child_end - child_start); - offset_bytes + validity_bytes + sliced_cache_bytes(child.as_ref()) - } - // Fixed-width primitives hold exactly `len * width` bytes regardless of - // buffer capacity, so this is already slice-aware. Any other type falls - // back to the full-buffer size. - other => match other.primitive_width() { - Some(width) => array.len() * width + validity_bytes, - None => array.get_buffer_memory_size(), - }, - } -} - -impl DeepSizeOf for Positions { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.0.deep_size_of_children(context) - } -} - -// Cache key implementations for type-safe cache access -#[derive(Debug, Clone)] -pub struct PostingListKey { - pub token_id: u32, -} - -impl CacheKey for PostingListKey { - type ValueType = PostingList; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("postings-{}", self.token_id).into() - } - - fn type_name() -> &'static str { - "PostingList" - } - - fn codec() -> Option { - Some(CacheCodec::from_impl::()) - } -} - -/// Cache key for a group of consecutive posting lists stored as a single -/// entry, covering rows `[start, end)` (issue #7040). The range, not a token -/// id, is the key so that a write-time config change that reshapes groups -/// simply misses old entries instead of serving a differently-shaped group. -#[derive(Debug, Clone)] -pub struct PostingListGroupKey { - pub start: u32, - pub end: u32, -} - -impl CacheKey for PostingListGroupKey { - type ValueType = PostingListGroup; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("postings-{}-{}", self.start, self.end).into() - } - - fn type_name() -> &'static str { - "PostingListGroup" - } - - fn codec() -> Option { - Some(CacheCodec::from_impl::()) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -struct PostingMetadataValue { - max_score: f32, - length: u32, -} - -#[derive(Debug, Clone)] -struct PostingMetadataKey { - token_id: u32, -} - -impl CacheKey for PostingMetadataKey { - type ValueType = PostingMetadataValue; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("posting-metadata-{}", self.token_id).into() - } - - fn type_name() -> &'static str { - "PostingMetadata" - } -} - -#[derive(Debug, Clone)] -pub struct PositionKey { - pub token_id: u32, -} - -impl CacheKey for PositionKey { - type ValueType = Positions; - - fn key(&self) -> std::borrow::Cow<'_, str> { - format!("positions-{}", self.token_id).into() - } - - fn type_name() -> &'static str { - "Position" - } - - fn codec() -> Option { - Some(CacheCodec::from_impl::()) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub enum CompressedPositionStorage { - LegacyPerDoc(ListArray), - SharedStream(SharedPositionStream), -} - -impl DeepSizeOf for CompressedPositionStorage { - fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { - match self { - Self::LegacyPerDoc(positions) => sliced_cache_bytes(positions), - Self::SharedStream(stream) => stream.size(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct SharedPositionStream { - codec: PositionStreamCodec, - block_offsets: Arc<[u32]>, - // Stored with shared ownership so cache hits can clone position streams - // without copying either offsets or bytes. - bytes: bytes::Bytes, -} - -impl SharedPositionStream { - pub fn new(codec: PositionStreamCodec, block_offsets: Vec, bytes: bytes::Bytes) -> Self { - Self { - codec, - block_offsets: Arc::from(block_offsets.into_boxed_slice()), - bytes, - } - } - - pub fn codec(&self) -> PositionStreamCodec { - self.codec - } - - pub fn block_count(&self) -> usize { - self.block_offsets.len() - } - - pub fn block_range(&self, index: usize) -> Range { - let start = self.block_offsets[index] as usize; - let end = self - .block_offsets - .get(index + 1) - .map(|offset| *offset as usize) - .unwrap_or(self.bytes.len()); - start..end - } - - pub fn block(&self, index: usize) -> &[u8] { - let range = self.block_range(index); - &self.bytes[range] - } - - pub fn bytes(&self) -> &[u8] { - &self.bytes - } - - pub fn block_offsets(&self) -> &[u32] { - self.block_offsets.as_ref() - } - - pub fn size(&self) -> usize { - self.block_offsets.len() * std::mem::size_of::() + self.bytes.len() - } -} - -/// A group of consecutive posting lists held in a single cache entry, in row -/// order (issue #7040). `posting_lists[i]` corresponds to row `start + i`, -/// where `start` is the group's first row from [`PostingListGroupKey`]. -#[derive(Debug, Clone, DeepSizeOf)] -pub struct PostingListGroup { - pub(super) posting_lists: Vec, -} - -impl PostingListGroup { - pub(super) fn new(posting_lists: Vec) -> Self { - Self { posting_lists } - } - - /// Borrow the posting list at offset `slot` within the group (i.e. - /// `token_id - start`). - pub(super) fn get(&self, slot: usize) -> Option<&PostingList> { - self.posting_lists.get(slot) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -pub enum PostingList { - Plain(PlainPostingList), - Compressed(CompressedPostingList), -} - -impl PostingList { - pub fn from_batch( - batch: &RecordBatch, - max_score: Option, - length: Option, - ) -> Result { - let posting_tail_codec = parse_posting_tail_codec(batch.schema_ref().metadata())?; - Self::from_batch_with_tail_codec(batch, max_score, length, posting_tail_codec) - } - - pub fn from_batch_with_tail_codec( - batch: &RecordBatch, - max_score: Option, - length: Option, - posting_tail_codec: PostingTailCodec, - ) -> Result { - let positions_layout = if batch.column_by_name(COMPRESSED_POSITION_COL).is_some() { - PositionsLayout::SharedStream(parse_shared_position_codec( - batch.schema_ref().metadata(), - )?) - } else if batch.column_by_name(POSITION_COL).is_some() { - PositionsLayout::LegacyPerDoc - } else { - PositionsLayout::None - }; - Self::from_batch_with_tail_codec_and_positions_layout( - batch, - max_score, - length, - posting_tail_codec, - positions_layout, - ) - } - - fn from_batch_with_tail_codec_and_positions_layout( - batch: &RecordBatch, - max_score: Option, - length: Option, - posting_tail_codec: PostingTailCodec, - positions_layout: PositionsLayout, - ) -> Result { - match batch.column_by_name(POSTING_COL) { - Some(_) => { - debug_assert!(max_score.is_some() && length.is_some()); - let shared_position_codec = match positions_layout { - PositionsLayout::SharedStream(codec) => Some(codec), - _ => None, - }; - let posting = CompressedPostingList::from_batch( - batch, - max_score.unwrap(), - length.unwrap(), - posting_tail_codec, - shared_position_codec, - ); - Ok(Self::Compressed(posting)) - } - None => { - let posting = PlainPostingList::from_batch(batch, max_score); - Ok(Self::Plain(posting)) - } - } - } - - pub fn iter(&self) -> PostingListIterator<'_> { - PostingListIterator::new(self) - } - - pub fn has_position(&self) -> bool { - match self { - Self::Plain(posting) => posting.positions.is_some(), - Self::Compressed(posting) => posting.positions.is_some(), - } - } - - pub fn set_positions(&mut self, positions: CompressedPositionStorage) { - match self { - Self::Plain(posting) => match positions { - CompressedPositionStorage::LegacyPerDoc(positions) => { - posting.positions = Some(positions) - } - CompressedPositionStorage::SharedStream(_) => { - unreachable!("shared position stream is not supported for plain postings") - } - }, - Self::Compressed(posting) => { - posting.positions = Some(positions); - } - } - } - - pub fn take_positions(&mut self) -> Option { - match self { - Self::Plain(posting) => posting - .positions - .take() - .map(CompressedPositionStorage::LegacyPerDoc), - Self::Compressed(posting) => posting.positions.take(), - } - } - - pub fn max_score(&self) -> Option { - match self { - Self::Plain(posting) => posting.max_score, - Self::Compressed(posting) => Some(posting.max_score), - } - } - - pub fn len(&self) -> usize { - match self { - Self::Plain(posting) => posting.len(), - Self::Compressed(posting) => posting.length as usize, - } - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn into_builder(self, docs: &DocSet) -> PostingListBuilder { - let posting_tail_codec = match &self { - Self::Plain(_) => PostingTailCodec::Fixed32, - Self::Compressed(posting) => posting.posting_tail_codec, - }; - let mut builder = PostingListBuilder::new_with_posting_tail_codec( - self.has_position(), - posting_tail_codec, - ); - match self { - // legacy format - Self::Plain(posting) => { - // convert the posting list to the new format: - // 1. map row ids to doc ids - // 2. sort the posting list by doc ids - struct Item { - doc_id: u32, - positions: PositionRecorder, - } - let doc_ids = docs - .row_ids - .iter() - .enumerate() - .map(|(doc_id, row_id)| (*row_id, doc_id as u32)) - .collect::>(); - let mut items = Vec::with_capacity(posting.len()); - for (row_id, freq, positions) in posting.iter() { - let freq = freq as u32; - let positions = match positions { - Some(positions) => { - PositionRecorder::Position(positions.collect::>().into()) - } - None => PositionRecorder::Count(freq), - }; - items.push(Item { - doc_id: doc_ids[&row_id], - positions, - }); - } - items.sort_unstable_by_key(|item| item.doc_id); - for item in items { - builder.add(item.doc_id, item.positions); - } - } - Self::Compressed(posting) => { - posting.iter().for_each(|(doc_id, freq, positions)| { - let positions = match positions { - Some(positions) => { - PositionRecorder::Position(positions.collect::>().into()) - } - None => PositionRecorder::Count(freq), - }; - builder.add(doc_id, positions); - }); - } - } - builder - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct PlainPostingList { - pub row_ids: ScalarBuffer, - pub frequencies: ScalarBuffer, - pub max_score: Option, - pub positions: Option, // List of Int32 -} - -impl DeepSizeOf for PlainPostingList { - fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { - self.row_ids.len() * std::mem::size_of::() - + self.frequencies.len() * std::mem::size_of::() - + self - .positions - .as_ref() - .map(|positions| sliced_cache_bytes(positions)) - .unwrap_or(0) - } -} - -impl PlainPostingList { - pub fn new( - row_ids: ScalarBuffer, - frequencies: ScalarBuffer, - max_score: Option, - positions: Option, - ) -> Self { - Self { - row_ids, - frequencies, - max_score, - positions, - } - } - - pub fn from_batch(batch: &RecordBatch, max_score: Option) -> Self { - let row_ids = batch[ROW_ID].as_primitive::().values().clone(); - let frequencies = batch[FREQUENCY_COL] - .as_primitive::() - .values() - .clone(); - let positions = batch - .column_by_name(POSITION_COL) - .map(|col| col.as_list::().clone()); - - Self::new(row_ids, frequencies, max_score, positions) - } - - pub fn len(&self) -> usize { - self.row_ids.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn iter(&self) -> PlainPostingListIterator<'_> { - Box::new( - self.row_ids - .iter() - .zip(self.frequencies.iter()) - .enumerate() - .map(|(idx, (doc_id, freq))| { - ( - *doc_id, - *freq, - self.positions.as_ref().map(|p| { - let start = p.value_offsets()[idx] as usize; - let end = p.value_offsets()[idx + 1] as usize; - Box::new( - p.values().as_primitive::().values()[start..end] - .iter() - .map(|pos| *pos as u32), - ) as _ - }), - ) - }), - ) - } - - #[inline] - pub fn doc(&self, i: usize) -> LocatedDocInfo { - LocatedDocInfo::new(self.row_ids[i], self.frequencies[i]) - } - - pub fn positions(&self, index: usize) -> Option> { - self.positions - .as_ref() - .map(|positions| positions.value(index)) - } - - pub fn max_score(&self) -> Option { - self.max_score - } - - pub fn row_id(&self, i: usize) -> u64 { - self.row_ids[i] - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct CompressedPostingList { - pub max_score: f32, - pub length: u32, - // each binary is a block of compressed data - // that contains `BLOCK_SIZE` doc ids and then `BLOCK_SIZE` frequencies - pub blocks: LargeBinaryArray, - pub posting_tail_codec: PostingTailCodec, - pub positions: Option, -} - -impl DeepSizeOf for CompressedPostingList { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - sliced_cache_bytes(&self.blocks) - + self - .positions - .as_ref() - .map(|positions| positions.deep_size_of_children(context)) - .unwrap_or(0) - } -} - -impl CompressedPostingList { - pub fn new( - blocks: LargeBinaryArray, - max_score: f32, - length: u32, - posting_tail_codec: PostingTailCodec, - positions: Option, - ) -> Self { - Self { - max_score, - length, - blocks, - posting_tail_codec, - positions, - } - } - - pub fn from_batch( - batch: &RecordBatch, - max_score: f32, - length: u32, - posting_tail_codec: PostingTailCodec, - shared_position_codec: Option, - ) -> Self { - debug_assert_eq!(batch.num_rows(), 1); - let blocks = batch[POSTING_COL] - .as_list::() - .value(0) - .as_binary::() - .clone(); - let positions = if let Some(col) = batch.column_by_name(COMPRESSED_POSITION_COL) { - let bytes = bytes::Bytes::from(col.as_binary::().value(0).to_vec()); - let block_offsets = batch[POSITION_BLOCK_OFFSET_COL] - .as_list::() - .value(0) - .as_primitive::() - .values() - .to_vec(); - let codec = shared_position_codec.unwrap_or_else(|| { - parse_shared_position_codec(batch.schema_ref().metadata()) - .expect("shared position stream codec metadata should be valid") - }); - Some(CompressedPositionStorage::SharedStream( - SharedPositionStream::new(codec, block_offsets, bytes), - )) - } else { - batch.column_by_name(POSITION_COL).map(|col| { - CompressedPositionStorage::LegacyPerDoc( - col.as_list::().value(0).as_list::().clone(), - ) - }) - }; - - Self { - max_score, - length, - blocks, - posting_tail_codec, - positions, - } - } - - pub fn iter(&self) -> CompressedPostingListIterator { - CompressedPostingListIterator::new( - self.length as usize, - self.blocks.clone(), - self.posting_tail_codec, - self.positions.clone(), - ) - } - - pub fn block_max_score(&self, block_idx: usize) -> f32 { - let block = self.blocks.value(block_idx); - block[0..4].try_into().map(f32::from_le_bytes).unwrap() - } - - pub fn block_least_doc_id(&self, block_idx: usize) -> u32 { - let block = self.blocks.value(block_idx); - let remainder = self.length as usize % BLOCK_SIZE; - let is_remainder_block = remainder > 0 && block_idx + 1 == self.blocks.len(); - if is_remainder_block { - super::encoding::read_posting_tail_first_doc(block, self.posting_tail_codec) - } else { - block[4..8].try_into().map(u32::from_le_bytes).unwrap() - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -struct EncodedBlocks { - offsets: Vec, - bytes: Vec, -} - -impl EncodedBlocks { - fn len(&self) -> usize { - self.offsets.len() - } - - fn size(&self) -> usize { - self.offsets.capacity() * std::mem::size_of::() + self.bytes.capacity() - } - - fn push_full_block(&mut self, doc_ids: &[u32], frequencies: &[u32]) -> Result { - let start = self.bytes.len(); - self.offsets.push(start as u32); - super::encoding::encode_full_posting_block_into(doc_ids, frequencies, &mut self.bytes)?; - Ok(self.bytes.len() - start) - } - - fn block(&self, index: usize) -> &[u8] { - let (start, end) = self.block_range(index); - &self.bytes[start..end] - } - - fn block_range(&self, index: usize) -> (usize, usize) { - let start = self.offsets[index] as usize; - let end = self - .offsets - .get(index + 1) - .map(|offset| *offset as usize) - .unwrap_or(self.bytes.len()); - (start, end) - } - - fn set_block_score(&mut self, index: usize, score: f32) { - let (start, _) = self.block_range(index); - self.bytes[start..start + 4].copy_from_slice(&score.to_le_bytes()); - } - - fn append_remainder_block_with_codec( - &mut self, - doc_ids: &[u32], - frequencies: &[u32], - codec: PostingTailCodec, - ) -> Result<()> { - self.offsets.push(self.bytes.len() as u32); - super::encoding::encode_remainder_posting_block_into( - doc_ids, - frequencies, - codec, - &mut self.bytes, - ) - } - - fn into_array(mut self) -> LargeBinaryArray { - let mut offsets = Vec::with_capacity(self.offsets.len() + 1); - offsets.extend(self.offsets.into_iter().map(i64::from)); - offsets.push(self.bytes.len() as i64); - LargeBinaryArray::new( - OffsetBuffer::new(ScalarBuffer::from(offsets)), - Buffer::from_vec(std::mem::take(&mut self.bytes)), - None, - ) - } - - fn iter(&self) -> impl Iterator { - (0..self.len()).map(|index| self.block(index)) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Default)] -struct EncodedPositionBlocks { - offsets: Vec, - bytes: Vec, -} - -impl EncodedPositionBlocks { - fn size(&self) -> usize { - self.offsets.capacity() * std::mem::size_of::() + self.bytes.capacity() - } - - fn block(&self, index: usize) -> &[u8] { - let start = self.offsets[index] as usize; - let end = self - .offsets - .get(index + 1) - .map(|offset| *offset as usize) - .unwrap_or(self.bytes.len()); - &self.bytes[start..end] - } - - fn push_encoded_block(&mut self, block: &[u8]) -> usize { - let start = self.bytes.len(); - self.offsets.push(start as u32); - self.bytes.extend_from_slice(block); - self.bytes.len() - start - } - - fn into_stream(self) -> SharedPositionStream { - SharedPositionStream::new( - PositionStreamCodec::PackedDelta, - self.offsets, - bytes::Bytes::from(self.bytes), - ) - } -} - -#[derive(Debug)] -pub struct PostingListBuilder { - with_positions: bool, - posting_tail_codec: PostingTailCodec, - encoded_blocks: Option>, - encoded_position_blocks: Option>, - tail_entries: Vec, - tail_positions: PositionBlockBuilder, - open_doc_id: Option, - open_doc_frequency: u32, - open_doc_last_position: Option, - memory_size_bytes: u32, - len: u32, -} - -pub(super) struct PostingListBatchBuilder { - schema: SchemaRef, - postings: ListBuilder, - max_scores: Float32Builder, - lengths: UInt32Builder, - positions: BatchPositionsBuilder, - len: usize, - /// Tracks posting-list cache-group boundaries in row order across all - /// batches this builder produces (issue #7040). Outlives `finish`, which - /// only resets the per-batch column builders. - group_accumulator: PostingGroupAccumulator, -} - -enum BatchPositionsBuilder { - None, - Legacy(ListBuilder>), - Shared { - bytes: LargeBinaryBuilder, - block_offsets: ListBuilder, - }, -} - -struct PostingListParts<'a> { - with_positions: bool, - posting_tail_codec: PostingTailCodec, - length: usize, - encoded_blocks: EncodedBlocks, - encoded_position_blocks: EncodedPositionBlocks, - tail_entries: &'a [RawDocInfo], - tail_position_block: Option>, -} - -impl PostingListBatchBuilder { - pub fn new( - schema: SchemaRef, - with_positions: bool, - format_version: InvertedListFormatVersion, - capacity: usize, - group_config: PostingGroupConfig, - ) -> Self { - let positions = if !with_positions { - BatchPositionsBuilder::None - } else if format_version.uses_shared_position_stream() { - BatchPositionsBuilder::Shared { - bytes: LargeBinaryBuilder::with_capacity(capacity, 0), - block_offsets: ListBuilder::with_capacity(UInt32Builder::new(), capacity), - } - } else { - BatchPositionsBuilder::Legacy(ListBuilder::with_capacity( - ListBuilder::new(LargeBinaryBuilder::new()), - capacity, - )) - }; - Self { - schema, - postings: ListBuilder::with_capacity(LargeBinaryBuilder::new(), capacity), - max_scores: Float32Builder::with_capacity(capacity), - lengths: UInt32Builder::with_capacity(capacity), - positions, - len: 0, - group_accumulator: PostingGroupAccumulator::new(group_config), - } - } - - pub fn len(&self) -> usize { - self.len - } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - fn append( - &mut self, - compressed: LargeBinaryArray, - max_score: f32, - length: u32, - positions: Option<&CompressedPositionStorage>, - ) -> Result<()> { - let posting_bytes = compressed.value_data().len(); - { - let values = self.postings.values(); - for index in 0..compressed.len() { - values.append_value(compressed.value(index)); - } - } - self.postings.append(true); - self.group_accumulator.push(posting_bytes); - self.max_scores.append_value(max_score); - self.lengths.append_value(length); - - match &mut self.positions { - BatchPositionsBuilder::None => {} - BatchPositionsBuilder::Shared { - bytes, - block_offsets, - } => { - let positions = positions.ok_or_else(|| { - Error::index(format!( - "positions builder missing position data for posting length {}", - length - )) - })?; - let CompressedPositionStorage::SharedStream(positions) = positions else { - return Err(Error::index( - "shared positions builder received legacy positions".to_owned(), - )); - }; - bytes.append_value(positions.bytes()); - let offsets_builder = block_offsets.values(); - for &offset in positions.block_offsets() { - offsets_builder.append_value(offset); - } - block_offsets.append(true); - } - BatchPositionsBuilder::Legacy(position_lists) => { - let positions = positions.ok_or_else(|| { - Error::index(format!( - "positions builder missing position data for posting length {}", - length - )) - })?; - let CompressedPositionStorage::LegacyPerDoc(positions) = positions else { - return Err(Error::index( - "legacy positions builder received shared position stream".to_owned(), - )); - }; - let docs_builder = position_lists.values(); - for doc_idx in 0..positions.len() { - let doc_positions = positions.value(doc_idx); - let compressed_positions = doc_positions.as_binary::(); - for block_idx in 0..compressed_positions.len() { - docs_builder - .values() - .append_value(compressed_positions.value(block_idx)); - } - docs_builder.append(true); - } - position_lists.append(true); - } - } - - self.len += 1; - Ok(()) - } - - pub fn finish(&mut self) -> Result { - let mut columns = vec![ - Arc::new(self.postings.finish()) as ArrayRef, - Arc::new(self.max_scores.finish()) as ArrayRef, - Arc::new(self.lengths.finish()) as ArrayRef, - ]; - match &mut self.positions { - BatchPositionsBuilder::None => {} - BatchPositionsBuilder::Legacy(position_lists) => { - columns.push(Arc::new(position_lists.finish()) as ArrayRef); - } - BatchPositionsBuilder::Shared { - bytes, - block_offsets, - } => { - columns.push(Arc::new(bytes.finish()) as ArrayRef); - columns.push(Arc::new(block_offsets.finish()) as ArrayRef); - } - } - self.len = 0; - RecordBatch::try_new(self.schema.clone(), columns).map_err(Error::from) - } - - /// Consume the builder and return the posting-list cache-group boundaries - /// accumulated across all batches (issue #7040). Each entry is the first - /// row of a group; the sequence is monotonically increasing. - pub fn into_group_starts(self) -> Vec { - self.group_accumulator.into_starts() - } -} - -impl PostingListBuilder { - pub fn size(&self) -> u64 { - self.memory_size_bytes as u64 - } - - pub fn has_positions(&self) -> bool { - self.with_positions - } - - pub fn new(with_position: bool) -> Self { - Self::new_with_posting_tail_codec( - with_position, - current_fts_format_version().posting_tail_codec(), - ) - } - - pub fn new_with_posting_tail_codec( - with_position: bool, - posting_tail_codec: PostingTailCodec, - ) -> Self { - Self { - with_positions: with_position, - posting_tail_codec, - encoded_blocks: None, - encoded_position_blocks: None, - tail_entries: Vec::new(), - tail_positions: PositionBlockBuilder::default(), - open_doc_id: None, - open_doc_frequency: 0, - open_doc_last_position: None, - len: 0, - memory_size_bytes: 0, - } - } - - pub fn len(&self) -> usize { - self.len as usize - } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - pub fn iter(&self) -> std::vec::IntoIter<(u32, u32, Option>)> { - self.collect_entries().into_iter() - } - - pub fn for_each_entry( - &self, - mut visit: impl FnMut(u32, u32, Option>) -> std::result::Result<(), E>, - ) -> std::result::Result<(), E> { - let mut doc_ids = Vec::with_capacity(BLOCK_SIZE); - let mut frequencies = Vec::with_capacity(BLOCK_SIZE); - let mut decoded_positions = Vec::new(); - let mut position_block_index = 0usize; - - if let Some(encoded_blocks) = self.encoded_blocks.as_deref() { - for block in encoded_blocks.iter() { - doc_ids.clear(); - frequencies.clear(); - super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies); - decoded_positions.clear(); - if self.with_positions { - let position_blocks = self - .encoded_position_blocks - .as_deref() - .expect("positions must exist for posting list"); - super::encoding::decode_position_stream_block( - position_blocks.block(position_block_index), - &frequencies, - PositionStreamCodec::PackedDelta, - &mut decoded_positions, - ) - .expect("position stream decoding should succeed"); - position_block_index += 1; - } - let mut offset = 0usize; - for (doc_id, frequency) in doc_ids.iter().copied().zip(frequencies.iter().copied()) - { - let positions = self.with_positions.then(|| { - let end = offset + frequency as usize; - let doc_positions = decoded_positions[offset..end].to_vec(); - offset = end; - doc_positions - }); - visit(doc_id, frequency, positions)?; - } - } - } - - let mut decoded_tail_positions = Vec::new(); - if self.with_positions && !self.tail_entries.is_empty() { - let tail_frequencies = self - .tail_entries - .iter() - .map(|entry| entry.frequency) - .collect::>(); - self.tail_positions - .decode_into(tail_frequencies.as_slice(), &mut decoded_tail_positions) - .expect("tail position stream decoding should succeed"); - } - let mut tail_offset = 0usize; - for entry in &self.tail_entries { - let positions = self.with_positions.then(|| { - let end = tail_offset + entry.frequency as usize; - let doc_positions = decoded_tail_positions[tail_offset..end].to_vec(); - tail_offset = end; - doc_positions - }); - visit(entry.doc_id, entry.frequency, positions)?; - } - - Ok(()) - } - - pub fn add(&mut self, doc_id: u32, term_positions: PositionRecorder) { - debug_assert!( - self.open_doc_id.is_none(), - "cannot add closed doc while a positions doc is still open" - ); - let tail_entries_capacity_before = self.tail_entries.capacity(); - self.tail_entries - .push(RawDocInfo::new(doc_id, term_positions.len())); - let tail_entries_capacity_after = self.tail_entries.capacity(); - if tail_entries_capacity_after > tail_entries_capacity_before { - self.add_memory_bytes( - (tail_entries_capacity_after - tail_entries_capacity_before) - * std::mem::size_of::(), - ); - } - if let PositionRecorder::Position(positions_in_doc) = term_positions { - debug_assert!(self.with_positions); - let old_size = self.tail_positions.size(); - self.tail_positions - .append_doc_positions(positions_in_doc.as_slice()) - .expect("position stream encoding should succeed"); - self.adjust_tail_positions_size(old_size); - } - self.len += 1; - - if self.tail_entries.len() == BLOCK_SIZE { - self.flush_tail_block() - .expect("posting list block compression should succeed"); - } - } - - pub fn add_occurrence(&mut self, doc_id: u32, position: u32) -> Result { - if !self.with_positions { - return Err(Error::index( - "cannot append streamed positions to a posting list without positions".to_owned(), - )); - } - - match self.open_doc_id { - Some(open_doc_id) if open_doc_id == doc_id => { - let old_size = self.tail_positions.size(); - self.tail_positions - .append_position(position, self.open_doc_last_position)?; - self.adjust_tail_positions_size(old_size); - self.open_doc_frequency += 1; - self.open_doc_last_position = Some(position); - Ok(false) - } - Some(open_doc_id) => Err(Error::index(format!( - "posting list received doc {} before finishing open doc {}", - doc_id, open_doc_id - ))), - None => { - let old_size = self.tail_positions.size(); - self.tail_positions.append_position(position, None)?; - self.adjust_tail_positions_size(old_size); - self.open_doc_id = Some(doc_id); - self.open_doc_frequency = 1; - self.open_doc_last_position = Some(position); - self.len += 1; - Ok(true) - } - } - } - - pub fn finish_open_doc(&mut self, doc_id: u32) -> Result<()> { - if !self.with_positions { - return Ok(()); - } - match self.open_doc_id { - Some(open_doc_id) if open_doc_id == doc_id => { - let tail_entries_capacity_before = self.tail_entries.capacity(); - self.tail_entries - .push(RawDocInfo::new(doc_id, self.open_doc_frequency)); - let tail_entries_capacity_after = self.tail_entries.capacity(); - if tail_entries_capacity_after > tail_entries_capacity_before { - self.add_memory_bytes( - (tail_entries_capacity_after - tail_entries_capacity_before) - * std::mem::size_of::(), - ); - } - self.open_doc_id = None; - self.open_doc_frequency = 0; - self.open_doc_last_position = None; - if self.tail_entries.len() == BLOCK_SIZE { - self.flush_tail_block()?; - } - Ok(()) - } - Some(open_doc_id) => Err(Error::index(format!( - "attempted to finish doc {} while doc {} is still open", - doc_id, open_doc_id - ))), - None => Ok(()), - } - } - - fn collect_entries(&self) -> Vec<(u32, u32, Option>)> { - let mut entries = Vec::with_capacity(self.len()); - self.for_each_entry(|doc_id, frequency, positions| { - entries.push((doc_id, frequency, positions)); - Ok::<(), ()>(()) - }) - .expect("collecting posting list entries should not fail"); - entries - } - - fn encoded_blocks_mut(&mut self) -> &mut EncodedBlocks { - if self.encoded_blocks.is_none() { - self.encoded_blocks = Some(Box::default()); - self.add_memory_bytes(std::mem::size_of::()); - } - self.encoded_blocks - .as_deref_mut() - .expect("encoded blocks must exist") - } - - fn encoded_position_blocks_mut(&mut self) -> &mut EncodedPositionBlocks { - if self.encoded_position_blocks.is_none() { - self.encoded_position_blocks = Some(Box::default()); - self.add_memory_bytes(std::mem::size_of::()); - } - self.encoded_position_blocks - .as_deref_mut() - .expect("encoded position blocks must exist") - } - - fn flush_tail_block(&mut self) -> Result<()> { - if self.tail_entries.is_empty() { - return Ok(()); - } - debug_assert!( - self.open_doc_id.is_none(), - "cannot flush a posting block while a document is still open" - ); - debug_assert_eq!(self.tail_entries.len(), BLOCK_SIZE); - let mut doc_ids = [0u32; BLOCK_SIZE]; - let mut frequencies = [0u32; BLOCK_SIZE]; - for (index, entry) in self.tail_entries.iter().enumerate() { - doc_ids[index] = entry.doc_id; - frequencies[index] = entry.frequency; - } - let encoded_blocks_size_before = self - .encoded_blocks - .as_ref() - .map(|encoded_blocks| encoded_blocks.size()) - .unwrap_or(0usize); - self.encoded_blocks_mut() - .push_full_block(&doc_ids, &frequencies)?; - let encoded_blocks_size_after = self - .encoded_blocks - .as_ref() - .map(|encoded_blocks| encoded_blocks.size()) - .unwrap_or(0usize); - if encoded_blocks_size_after > encoded_blocks_size_before { - self.add_memory_bytes(encoded_blocks_size_after - encoded_blocks_size_before); - } - if self.with_positions { - let encoded_positions_size_before = self - .encoded_position_blocks - .as_ref() - .map(|encoded| encoded.size()) - .unwrap_or(0usize); - let released_tail_positions_bytes = self.tail_positions.size(); - let tail_position_block = std::mem::take(&mut self.tail_positions).finish(); - self.encoded_position_blocks_mut() - .push_encoded_block(tail_position_block.as_slice()); - let encoded_positions_size_after = self - .encoded_position_blocks - .as_ref() - .map(|encoded| encoded.size()) - .unwrap_or(0usize); - if released_tail_positions_bytes > 0 { - self.subtract_memory_bytes(released_tail_positions_bytes); - } - if encoded_positions_size_after > encoded_positions_size_before { - self.add_memory_bytes(encoded_positions_size_after - encoded_positions_size_before); - } - } - self.tail_entries.clear(); - Ok(()) - } - - fn adjust_tail_positions_size(&mut self, old_size: usize) { - let new_size = self.tail_positions.size(); - if new_size > old_size { - self.add_memory_bytes(new_size - old_size); - } else if old_size > new_size { - self.subtract_memory_bytes(old_size - new_size); - } - } - - fn add_memory_bytes(&mut self, bytes: usize) { - self.memory_size_bytes = self - .memory_size_bytes - .checked_add( - u32::try_from(bytes).expect("posting list memory size delta overflowed u32"), - ) - .expect("posting list memory size overflowed u32"); - } - - fn subtract_memory_bytes(&mut self, bytes: usize) { - self.memory_size_bytes = self - .memory_size_bytes - .checked_sub( - u32::try_from(bytes).expect("posting list memory size delta overflowed u32"), - ) - .expect("posting list memory size underflowed u32"); - } - - fn build_position_columns( - positions: Option, - ) -> Result> { - let Some(positions) = positions else { - return Ok(Vec::new()); - }; - match positions { - CompressedPositionStorage::LegacyPerDoc(positions) => { - Ok(vec![Arc::new(ListArray::try_new( - Arc::new(Field::new("item", positions.data_type().clone(), true)), - OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, positions.len() as i32])), - Arc::new(positions) as ArrayRef, - None, - )?) as ArrayRef]) - } - CompressedPositionStorage::SharedStream(positions) => { - let mut columns = Vec::with_capacity(2); - columns.push( - Arc::new(LargeBinaryArray::from(vec![Some(positions.bytes())])) as ArrayRef, - ); - - let mut offsets_builder = ListBuilder::new(UInt32Builder::new()); - for &offset in positions.block_offsets() { - offsets_builder.values().append_value(offset); - } - offsets_builder.append(true); - columns.push(Arc::new(offsets_builder.finish()) as ArrayRef); - Ok(columns) - } - } - } - - fn build_batch( - self, - compressed: LargeBinaryArray, - max_score: f32, - schema: SchemaRef, - positions: Option, - ) -> Result { - let length = self.len(); - let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, compressed.len() as i32])); - let mut columns = vec![ - Arc::new(ListArray::try_new( - Arc::new(Field::new("item", datatypes::DataType::LargeBinary, true)), - offsets, - Arc::new(compressed), - None, - )?) as ArrayRef, - Arc::new(Float32Array::from_iter_values(std::iter::once(max_score))) as ArrayRef, - Arc::new(UInt32Array::from_iter_values(std::iter::once( - length as u32, - ))) as ArrayRef, - ]; - columns.extend(Self::build_position_columns(positions)?); - - let batch = RecordBatch::try_new(schema, columns)?; - Ok(batch) - } - - fn build_legacy_positions(&self) -> Result { - let mut positions_builder = ListBuilder::new(LargeBinaryBuilder::new()); - self.for_each_entry(|_doc_id, frequency, positions| { - let positions = positions.ok_or_else(|| { - Error::index(format!( - "legacy position writer missing positions for frequency {}", - frequency - )) - })?; - let compressed = super::encoding::compress_positions(positions.as_slice())?; - for block_idx in 0..compressed.len() { - positions_builder - .values() - .append_value(compressed.value(block_idx)); - } - positions_builder.append(true); - Ok::<(), Error>(()) - })?; - Ok(positions_builder.finish()) - } - - pub(super) fn append_to_batch_with_docs( - self, - docs: &DocSet, - batch_builder: &mut PostingListBatchBuilder, - format_version: InvertedListFormatVersion, - ) -> Result<()> { - let legacy_positions = - if self.with_positions && !format_version.uses_shared_position_stream() { - Some(self.build_legacy_positions()?) - } else { - None - }; - let Self { - with_positions, - posting_tail_codec, - encoded_blocks, - encoded_position_blocks, - tail_entries, - tail_positions, - open_doc_id, - open_doc_frequency, - open_doc_last_position, - len, - .. - } = self; - debug_assert!(open_doc_id.is_none()); - debug_assert_eq!(open_doc_frequency, 0); - debug_assert!(open_doc_last_position.is_none()); - let parts = PostingListParts { - with_positions, - posting_tail_codec, - length: len as usize, - encoded_blocks: encoded_blocks - .map(|encoded_blocks| *encoded_blocks) - .unwrap_or_default(), - encoded_position_blocks: encoded_position_blocks - .map(|encoded_positions| *encoded_positions) - .unwrap_or_default(), - tail_entries: tail_entries.as_slice(), - tail_position_block: with_positions.then(|| tail_positions.finish()), - }; - let (compressed, shared_positions, max_score) = - Self::build_compressed_with_scores_from_parts(parts, docs)?; - let positions = match legacy_positions { - Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)), - None => shared_positions.map(CompressedPositionStorage::SharedStream), - }; - batch_builder.append(compressed, max_score, len, positions.as_ref()) - } - - fn extend_tail_components( - tail_entries: &[RawDocInfo], - doc_ids: &mut Vec, - frequencies: &mut Vec, - ) { - doc_ids.clear(); - frequencies.clear(); - doc_ids.extend(tail_entries.iter().map(|entry| entry.doc_id)); - frequencies.extend(tail_entries.iter().map(|entry| entry.frequency)); - } - - fn build_compressed_with_scores_from_parts( - parts: PostingListParts<'_>, - docs: &DocSet, - ) -> Result<(LargeBinaryArray, Option, f32)> { - let PostingListParts { - with_positions, - posting_tail_codec, - length, - mut encoded_blocks, - mut encoded_position_blocks, - tail_entries, - tail_position_block, - } = parts; - let avgdl = docs.average_length(); - let idf_scale = idf(length, docs.len()) * (K1 + 1.0); - let mut max_score = f32::MIN; - let mut doc_ids = Vec::with_capacity(BLOCK_SIZE); - let mut frequencies = Vec::with_capacity(BLOCK_SIZE); - - for index in 0..encoded_blocks.len() { - let block = encoded_blocks.block(index); - doc_ids.clear(); - frequencies.clear(); - super::encoding::decode_full_posting_block(block, &mut doc_ids, &mut frequencies); - let block_score = compute_block_score( - docs, - avgdl, - idf_scale, - doc_ids.iter().copied(), - frequencies.iter().copied(), - ); - max_score = max_score.max(block_score); - encoded_blocks.set_block_score(index, block_score); - } - - if !tail_entries.is_empty() { - Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies); - let block_score = compute_block_score( - docs, - avgdl, - idf_scale, - doc_ids.iter().copied(), - frequencies.iter().copied(), - ); - max_score = max_score.max(block_score); - encoded_blocks.append_remainder_block_with_codec( - doc_ids.as_slice(), - frequencies.as_slice(), - posting_tail_codec, - )?; - encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score); - if with_positions { - encoded_position_blocks.push_encoded_block( - tail_position_block - .as_deref() - .expect("tail position block must exist for postings with positions"), - ); - } - } - - Ok(( - encoded_blocks.into_array(), - with_positions.then(|| encoded_position_blocks.into_stream()), - max_score, - )) - } - - fn build_compressed_with_block_scores_from_parts( - with_positions: bool, - posting_tail_codec: PostingTailCodec, - mut encoded_blocks: EncodedBlocks, - mut encoded_position_blocks: EncodedPositionBlocks, - tail_entries: &[RawDocInfo], - tail_position_block: Option>, - mut block_max_scores: impl Iterator, - ) -> Result<(LargeBinaryArray, Option, f32)> { - let mut max_score = f32::MIN; - let mut doc_ids = Vec::with_capacity(BLOCK_SIZE); - let mut frequencies = Vec::with_capacity(BLOCK_SIZE); - - for index in 0..encoded_blocks.len() { - let block_score = block_max_scores - .next() - .ok_or_else(|| Error::index("missing block max score".to_owned()))?; - max_score = max_score.max(block_score); - encoded_blocks.set_block_score(index, block_score); - } - - if !tail_entries.is_empty() { - let block_score = block_max_scores - .next() - .ok_or_else(|| Error::index("missing tail block max score".to_owned()))?; - max_score = max_score.max(block_score); - Self::extend_tail_components(tail_entries, &mut doc_ids, &mut frequencies); - encoded_blocks.append_remainder_block_with_codec( - doc_ids.as_slice(), - frequencies.as_slice(), - posting_tail_codec, - )?; - encoded_blocks.set_block_score(encoded_blocks.len() - 1, block_score); - if with_positions { - encoded_position_blocks.push_encoded_block( - tail_position_block - .as_deref() - .expect("tail position block must exist for postings with positions"), - ); - } - } - - Ok(( - encoded_blocks.into_array(), - with_positions.then(|| encoded_position_blocks.into_stream()), - max_score, - )) - } - - pub fn to_batch(self, block_max_scores: Vec) -> Result { - let format_version = if self.posting_tail_codec == PostingTailCodec::Fixed32 { - InvertedListFormatVersion::V1 - } else { - InvertedListFormatVersion::V2 - }; - let schema = inverted_list_schema_for_version(self.has_positions(), format_version); - let legacy_positions = - if self.with_positions && !format_version.uses_shared_position_stream() { - Some(self.build_legacy_positions()?) - } else { - None - }; - let Self { - with_positions, - posting_tail_codec, - encoded_blocks, - encoded_position_blocks, - tail_entries, - tail_positions, - open_doc_id, - open_doc_frequency, - open_doc_last_position, - len, - .. - } = self; - debug_assert!(open_doc_id.is_none()); - debug_assert_eq!(open_doc_frequency, 0); - debug_assert!(open_doc_last_position.is_none()); - let (compressed, shared_positions, max_score) = - Self::build_compressed_with_block_scores_from_parts( - with_positions, - posting_tail_codec, - encoded_blocks - .map(|encoded_blocks| *encoded_blocks) - .unwrap_or_default(), - encoded_position_blocks - .map(|encoded_positions| *encoded_positions) - .unwrap_or_default(), - tail_entries.as_slice(), - with_positions.then(|| tail_positions.finish()), - block_max_scores.into_iter(), - )?; - let builder = Self { - with_positions, - posting_tail_codec, - encoded_blocks: None, - encoded_position_blocks: None, - tail_entries: Vec::new(), - tail_positions: PositionBlockBuilder::default(), - open_doc_id: None, - open_doc_frequency: 0, - open_doc_last_position: None, - memory_size_bytes: 0, - len, - }; - let positions = match legacy_positions { - Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)), - None => shared_positions.map(CompressedPositionStorage::SharedStream), - }; - builder.build_batch(compressed, max_score, schema, positions) - } - - pub fn to_batch_with_docs(self, docs: &DocSet, schema: SchemaRef) -> Result { - let format_version = if schema.column_with_name(POSITION_COL).is_some() - && schema.column_with_name(COMPRESSED_POSITION_COL).is_none() - { - InvertedListFormatVersion::V1 - } else { - InvertedListFormatVersion::V2 - }; - let legacy_positions = - if self.with_positions && !format_version.uses_shared_position_stream() { - Some(self.build_legacy_positions()?) - } else { - None - }; - let Self { - with_positions, - posting_tail_codec, - encoded_blocks, - encoded_position_blocks, - tail_entries, - tail_positions, - open_doc_id, - open_doc_frequency, - open_doc_last_position, - len, - .. - } = self; - debug_assert!(open_doc_id.is_none()); - debug_assert_eq!(open_doc_frequency, 0); - debug_assert!(open_doc_last_position.is_none()); - let parts = PostingListParts { - with_positions, - posting_tail_codec, - length: len as usize, - encoded_blocks: encoded_blocks - .map(|encoded_blocks| *encoded_blocks) - .unwrap_or_default(), - encoded_position_blocks: encoded_position_blocks - .map(|encoded_positions| *encoded_positions) - .unwrap_or_default(), - tail_entries: tail_entries.as_slice(), - tail_position_block: with_positions.then(|| tail_positions.finish()), - }; - let (compressed, shared_positions, max_score) = - Self::build_compressed_with_scores_from_parts(parts, docs)?; - let builder = Self { - with_positions, - posting_tail_codec, - encoded_blocks: None, - encoded_position_blocks: None, - tail_entries: Vec::new(), - tail_positions: PositionBlockBuilder::default(), - open_doc_id: None, - open_doc_frequency: 0, - open_doc_last_position: None, - memory_size_bytes: 0, - len, - }; - let positions = match legacy_positions { - Some(positions) => Some(CompressedPositionStorage::LegacyPerDoc(positions)), - None => shared_positions.map(CompressedPositionStorage::SharedStream), - }; - builder.build_batch(compressed, max_score, schema, positions) - } - - pub fn remap(&mut self, removed: &[u32]) { - let mut cursor = 0; - let mut new_builder = - Self::new_with_posting_tail_codec(self.has_positions(), self.posting_tail_codec); - for (doc_id, freq, positions) in self.iter() { - while cursor < removed.len() && removed[cursor] < doc_id { - cursor += 1; - } - if cursor < removed.len() && removed[cursor] == doc_id { - continue; - } - let positions = match positions { - Some(positions) => PositionRecorder::Position(positions.into()), - None => PositionRecorder::Count(freq), - }; - new_builder.add(doc_id - cursor as u32, positions); - } - - *self = new_builder; - } -} - -fn compute_block_score( - docs: &DocSet, - avgdl: f32, - idf_scale: f32, - doc_ids: impl Iterator, - frequencies: impl Iterator, -) -> f32 { - let mut block_max_score = f32::MIN; - for (doc_id, freq) in doc_ids.zip(frequencies) { - let doc_norm = K1 * (1.0 - B + B * docs.num_tokens(doc_id) as f32 / avgdl); - let freq = freq as f32; - let score = freq / (freq + doc_norm); - block_max_score = block_max_score.max(score); - } - block_max_score * idf_scale -} - -#[derive(Debug, Clone, DeepSizeOf, Copy)] -pub enum DocInfo { - Located(LocatedDocInfo), - Raw(RawDocInfo), -} - -impl DocInfo { - pub fn doc_id(&self) -> u64 { - match self { - Self::Raw(info) => info.doc_id as u64, - Self::Located(info) => info.row_id, - } - } - - pub fn frequency(&self) -> u32 { - match self { - Self::Raw(info) => info.frequency, - Self::Located(info) => info.frequency as u32, - } - } -} - -impl Eq for DocInfo {} - -impl PartialEq for DocInfo { - fn eq(&self, other: &Self) -> bool { - self.doc_id() == other.doc_id() - } -} - -impl PartialOrd for DocInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for DocInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.doc_id().cmp(&other.doc_id()) - } -} - -#[derive(Debug, Clone, Default, DeepSizeOf, Copy)] -pub struct LocatedDocInfo { - pub row_id: u64, - pub frequency: f32, -} - -impl LocatedDocInfo { - pub fn new(row_id: u64, frequency: f32) -> Self { - Self { row_id, frequency } - } -} - -impl Eq for LocatedDocInfo {} - -impl PartialEq for LocatedDocInfo { - fn eq(&self, other: &Self) -> bool { - self.row_id == other.row_id - } -} - -impl PartialOrd for LocatedDocInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for LocatedDocInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.row_id.cmp(&other.row_id) - } -} - -#[derive(Debug, Clone, Default, DeepSizeOf, Copy)] -pub struct RawDocInfo { - pub doc_id: u32, - pub frequency: u32, -} - -impl RawDocInfo { - pub fn new(doc_id: u32, frequency: u32) -> Self { - Self { doc_id, frequency } - } -} - -impl Eq for RawDocInfo {} - -impl PartialEq for RawDocInfo { - fn eq(&self, other: &Self) -> bool { - self.doc_id == other.doc_id - } -} - -impl PartialOrd for RawDocInfo { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for RawDocInfo { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.doc_id.cmp(&other.doc_id) - } -} - -// DocSet is a mapping from row ids to the number of tokens in the document -// It's used to sort the documents by the bm25 score -#[derive(Debug, Clone, Default, DeepSizeOf)] -pub struct DocSet { - row_ids: Vec, - num_tokens: Vec, - // (row_id, doc_id) pairs sorted by row_id - inv: Vec<(u64, u32)>, - - total_tokens: u64, -} - -impl DocSet { - #[inline] - pub fn len(&self) -> usize { - // Use num_tokens instead of row_ids so the deferred-row_ids - // scoring path (which constructs a DocSet via - // [`Self::from_num_tokens_only`]) still reports the right doc - // count. - self.num_tokens.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// True iff the per-doc `row_id` array is populated. The - /// deferred-row_id scoring path constructs DocSets with the array - /// left empty so wand can skip the load; callers that need to do - /// row_id lookups in the inner loop must check this and fall back - /// to async resolution otherwise. - #[inline] - pub fn has_row_ids(&self) -> bool { - !self.row_ids.is_empty() - } - - pub fn iter(&self) -> impl Iterator { - self.row_ids.iter().zip(self.num_tokens.iter()) - } - - pub fn row_id(&self, doc_id: u32) -> u64 { - self.row_ids[doc_id as usize] - } - - /// Resolve a `row_id` to every `doc_id` it owns. - /// - /// A scalar column maps each row to a single document, but a - /// `list` column indexes every element as its own document, so a - /// single `row_id` can own several `doc_id`s sharing that key in `inv`. - /// The prefilter path (`flat_search`) walks an allow-list of row_ids and - /// must evaluate *all* of a row's documents; resolving to one `doc_id` - /// silently drops matches at non-last list positions (lancedb#3352). - pub fn doc_ids(&self, row_id: u64) -> impl Iterator + '_ { - if self.inv.is_empty() { - // in legacy format, the row id is doc id (one document per row) - let found = self.row_ids.binary_search(&row_id).is_ok(); - Either::Left(found.then_some(row_id).into_iter()) - } else { - // `inv` is sorted by row_id, so the entries sharing this key form a - // contiguous run; yield the doc_id of each. - let lo = self.inv.partition_point(|entry| entry.0 < row_id); - let hi = self.inv.partition_point(|entry| entry.0 <= row_id); - Either::Right(self.inv[lo..hi].iter().map(|entry| entry.1 as u64)) - } - } - pub fn total_tokens_num(&self) -> u64 { - self.total_tokens - } - - #[inline] - pub fn average_length(&self) -> f32 { - self.total_tokens as f32 / self.len() as f32 - } - - pub fn calculate_block_max_scores<'a>( - &self, - doc_ids: impl Iterator, - freqs: impl Iterator, - ) -> Vec { - let avgdl = self.average_length(); - let length = doc_ids.size_hint().0; - let num_blocks = length.div_ceil(BLOCK_SIZE); - let mut block_max_scores = Vec::with_capacity(num_blocks); - let idf_scale = idf(length, self.len()) * (K1 + 1.0); - let mut max_score = f32::MIN; - for (i, (doc_id, freq)) in doc_ids.zip(freqs).enumerate() { - let doc_norm = K1 * (1.0 - B + B * self.num_tokens(*doc_id) as f32 / avgdl); - let freq = *freq as f32; - let score = freq / (freq + doc_norm); - if score > max_score { - max_score = score; - } - if (i + 1) % BLOCK_SIZE == 0 { - max_score *= idf_scale; - block_max_scores.push(max_score); - max_score = f32::MIN; - } - } - if !length.is_multiple_of(BLOCK_SIZE) { - max_score *= idf_scale; - block_max_scores.push(max_score); - } - block_max_scores - } - - pub fn to_batch(&self) -> Result { - let row_id_col = UInt64Array::from_iter_values(self.row_ids.iter().cloned()); - let num_tokens_col = UInt32Array::from_iter_values(self.num_tokens.iter().cloned()); - - let schema = arrow_schema::Schema::new(vec![ - arrow_schema::Field::new(ROW_ID, DataType::UInt64, false), - arrow_schema::Field::new(NUM_TOKEN_COL, DataType::UInt32, false), - ]); - - let batch = RecordBatch::try_new( - Arc::new(schema), - vec![ - Arc::new(row_id_col) as ArrayRef, - Arc::new(num_tokens_col) as ArrayRef, - ], - )?; - Ok(batch) - } - - pub async fn load( - reader: Arc, - is_legacy: bool, - frag_reuse_index: Option>, - ) -> Result { - let batch = reader.read_range(0..reader.num_rows(), None).await?; - let row_id_col = batch[ROW_ID].as_primitive::(); - let num_tokens_col = batch[NUM_TOKEN_COL].as_primitive::(); - Self::from_columns(row_id_col, num_tokens_col, is_legacy, frag_reuse_index) - } - - /// Build a `DocSet` carrying only the per-doc `num_tokens` array; - /// `row_ids` and `inv` are left empty. Used by the deferred-row_id - /// scoring path: wand checks `has_row_ids()` to skip `row_id` / - /// `num_tokens_by_row_id` calls, and the per-partition caller - /// resolves doc_id → row_id for the surviving top-K post-wand. - pub fn from_num_tokens_only(num_tokens_col: &arrow_array::UInt32Array) -> Self { - let num_tokens = num_tokens_col.values().to_vec(); - let total_tokens = num_tokens.iter().map(|&n| n as u64).sum(); - Self { - row_ids: Vec::new(), - num_tokens, - inv: Vec::new(), - total_tokens, - } - } - - /// Build a `DocSet` from already-loaded `row_id` and `num_tokens` - /// arrow columns. Lets callers that have one column already in hand - /// (e.g. `LazyDocSet` after `total_tokens_num` pre-fetched - /// `num_tokens`) skip re-reading that column. - pub fn from_columns( - row_id_col: &UInt64Array, - num_tokens_col: &arrow_array::UInt32Array, - is_legacy: bool, - frag_reuse_index: Option>, - ) -> Result { - // for legacy format, the row id is doc id; sorting keeps binary search viable - if is_legacy { - let (row_ids, num_tokens): (Vec<_>, Vec<_>) = row_id_col - .values() - .iter() - .filter_map(|id| { - if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() { - frag_reuse_index_ref.remap_row_id(*id) - } else { - Some(*id) - } - }) - .zip(num_tokens_col.values().iter()) - .sorted_unstable_by_key(|x| x.0) - .unzip(); - - let total_tokens = num_tokens.iter().map(|&x| x as u64).sum(); - return Ok(Self { - row_ids, - num_tokens, - inv: Vec::new(), - total_tokens, - }); - } - - // If frag reuse happened, remap the row_ids through it. Crucially we - // must NOT drop the rows the reuse index deleted, because the posting - // lists reference doc_ids *positionally* (a doc_id is an index into - // these arrays, fixed at build time). Dropping deleted rows would - // renumber every later doc_id and desync the posting lists, so wand - // would index `num_tokens`/`row_ids` out of bounds or score the wrong - // doc. Instead we tombstone deleted rows in place: their slot survives - // (so doc_ids stay aligned with the posting lists) carrying - // `RowAddress::TOMBSTONE_ROW`, which wand skips, and they are left out - // of `inv` so a row_id lookup never resolves to a deleted doc. The - // heavyweight physical remap (`DocSet::remap`) is what actually - // renumbers and compacts; this load-time path only has to stay - // consistent until then. - if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() { - let mut row_ids = Vec::with_capacity(row_id_col.len()); - let num_tokens = num_tokens_col.values().to_vec(); - let mut inv = Vec::with_capacity(row_id_col.len()); - for (doc_id, row_id) in row_id_col.values().iter().enumerate() { - match frag_reuse_index_ref.remap_row_id(*row_id) { - Some(new_row_id) => { - row_ids.push(new_row_id); - inv.push((new_row_id, doc_id as u32)); - } - None => { - // Deleted: keep the slot (doc_ids must not shift) but - // tombstone it and leave it out of `inv`. - row_ids.push(RowAddress::TOMBSTONE_ROW); - } - } - } - inv.sort_unstable_by_key(|entry| entry.0); - - let total_tokens = num_tokens.iter().map(|&x| x as u64).sum(); - return Ok(Self { - row_ids, - num_tokens, - inv, - total_tokens, - }); - } - - let row_ids = row_id_col.values().to_vec(); - let num_tokens = num_tokens_col.values().to_vec(); - let mut inv: Vec<(u64, u32)> = row_ids - .iter() - .enumerate() - .map(|(doc_id, row_id)| (*row_id, doc_id as u32)) - .collect(); - if !row_ids.is_sorted() { - inv.sort_unstable_by_key(|entry| entry.0); - } - let total_tokens = num_tokens.iter().map(|&x| x as u64).sum(); - Ok(Self { - row_ids, - num_tokens, - inv, - total_tokens, - }) - } - - // remap the row ids to the new row ids - // returns the removed doc ids - pub fn remap(&mut self, mapping: &RowAddrRemap) -> Vec { - let mut removed = Vec::new(); - let len = self.len(); - let row_ids = std::mem::replace(&mut self.row_ids, Vec::with_capacity(len)); - let num_tokens = std::mem::replace(&mut self.num_tokens, Vec::with_capacity(len)); - self.total_tokens = 0; - for (doc_id, (row_id, num_token)) in std::iter::zip(row_ids, num_tokens).enumerate() { - match mapping.get(row_id) { - Some(Some(new_row_id)) => { - self.row_ids.push(new_row_id); - self.num_tokens.push(num_token); - self.total_tokens += num_token as u64; - } - Some(None) => { - removed.push(doc_id as u32); - } - None => { - self.row_ids.push(row_id); - self.num_tokens.push(num_token); - self.total_tokens += num_token as u64; - } - } - } - removed - } - - #[inline] - pub fn num_tokens(&self, doc_id: u32) -> u32 { - self.num_tokens[doc_id as usize] - } - - // this can be used only if it's a legacy format, - // which store the sorted row ids so that we can use binary search - #[inline] - pub fn num_tokens_by_row_id(&self, row_id: u64) -> u32 { - self.row_ids - .binary_search(&row_id) - .map(|idx| self.num_tokens[idx]) - .unwrap_or(0) - } - - // append a document to the doc set - // returns the doc_id (the number of documents before appending) - pub fn append(&mut self, row_id: u64, num_tokens: u32) -> u32 { - self.row_ids.push(row_id); - self.num_tokens.push(num_tokens); - self.total_tokens += num_tokens as u64; - self.row_ids.len() as u32 - 1 - } - - pub(crate) fn memory_size(&self) -> usize { - self.row_ids.capacity() * std::mem::size_of::() - + self.num_tokens.capacity() * std::mem::size_of::() - + self.inv.capacity() * std::mem::size_of::<(u64, u32)>() - } -} - -pub fn flat_full_text_search( - batches: &[&RecordBatch], - doc_col: &str, - query: &str, - tokenizer: Option>, -) -> Result> { - if batches.is_empty() { - return Ok(vec![]); - } - - if is_phrase_query(query) { - return Err(Error::invalid_input( - "phrase query is not supported for flat full text search, try using FTS index", - )); - } - - match batches[0][doc_col].data_type() { - DataType::Utf8 => do_flat_full_text_search::(batches, doc_col, query, tokenizer), - DataType::LargeUtf8 => do_flat_full_text_search::(batches, doc_col, query, tokenizer), - data_type => Err(Error::invalid_input(format!( - "unsupported data type {} for inverted index", - data_type - ))), - } -} - -fn do_flat_full_text_search( - batches: &[&RecordBatch], - doc_col: &str, - query: &str, - tokenizer: Option>, -) -> Result> { - let mut results = Vec::new(); - let mut tokenizer = - tokenizer.unwrap_or_else(|| InvertedIndexParams::default().build().unwrap()); - let query_tokens = collect_query_tokens(query, &mut tokenizer); - - for batch in batches { - let row_id_array = batch[ROW_ID].as_primitive::(); - let doc_array = batch[doc_col].as_string::(); - for i in 0..row_id_array.len() { - let doc = doc_array.value(i); - if has_query_token(doc, &mut tokenizer, &query_tokens) { - results.push(row_id_array.value(i)); - // What is this assertion for? Why would doc contain query? Don't we reach - // here only if they share at least one token? Why is it not debug_assert? - assert!(doc.contains(query)); - } - } - } - - Ok(results) -} - -const FLAT_ROW_ID_COL_IDX: usize = 0; -const FLAT_ALL_TOKENS_COL_IDX: usize = 1; -const FLAT_QUERY_TOKEN_COUNTS_COL_IDX: usize = 2; - -/// If we accumulate this many bytes we warn the user they probably want to use an FTS index instead. -const BYTES_ACCUMULATED_WARNING_THRESHOLD: u64 = 1024 * 1024 * 1024; // 1GB - -/// Consumes a stream of record batches and produces token counts -/// -/// The resulting batch will have three columns: -/// - row_id: the row id of the document -/// - all_tokens: the total number of tokens in the document -/// - query_token_counts: a fixed size list of the count of each query token in the document -/// -/// This is an unbounded accumulation, however, for most queries, the per-row -/// growth will be fairly small. As a result we can process millions of tokens -/// with fairly modest memory usage. -/// -/// However, it is unwise to do a flat search across billions of rows. An FTS -/// index should be created instead. -async fn tokenize_and_count( - input: impl Stream> + Send, - tokenizer: Box, - query_tokens: Arc, - doc_col_idx: usize, - elapsed_compute: Option

    ( existing: &[P::ZoneStatistics], trainer: ZoneTrainer

    , stream: SendableRecordBatchStream, -) -> Result> +) -> Result<(Vec, RowAddrTreeMap)> where P: ZoneProcessor, P::ZoneStatistics: Clone, { let mut combined = existing.to_vec(); - let mut new_zones = trainer.train(stream).await?; + let (mut new_zones, null_rows) = trainer.train(stream).await?; combined.append(&mut new_zones); - Ok(combined) + Ok((combined, null_rows)) } #[cfg(test)] @@ -362,7 +373,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 4).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Three zones: offsets [0..=3], [4..=7], [8..=9] assert_eq!(stats.len(), 3); @@ -393,7 +404,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Two zones, one per fragment (capacity=10 is large enough) assert_eq!(stats.len(), 2); @@ -447,7 +458,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // One zone containing the 3 valid rows (empty batches skipped) assert_eq!(stats.len(), 1); @@ -469,7 +480,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 1).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Three zones, one per row (capacity=1) assert_eq!(stats.len(), 3); @@ -494,7 +505,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10000).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // One zone containing all 100 rows (capacity is large enough) assert_eq!(stats.len(), 1); @@ -530,7 +541,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 4).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Two zones: first 4 rows, then remaining 2 rows assert_eq!(stats.len(), 2); @@ -561,7 +572,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 3).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Three zones: frag 0 full zone, frag 0 partial (flushed at boundary), frag 1 assert_eq!(stats.len(), 3); @@ -602,7 +613,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 4).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Should create 2 zones (capacity=4): // Zone 0: rows at offsets [0, 1, 5, 7] (4 rows) @@ -637,7 +648,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // One zone with 3 rows, but offset span [0..=200] so length=201 due to large gaps assert_eq!(stats.len(), 1); @@ -663,7 +674,7 @@ mod tests { let processor = MockProcessor::new(); let trainer = ZoneTrainer::new(processor, 10).unwrap(); - let stats = trainer.train(stream).await.unwrap(); + let (stats, _) = trainer.train(stream).await.unwrap(); // Should create 3 zones (one per fragment) assert_eq!(stats.len(), 3); @@ -810,7 +821,7 @@ mod tests { )); let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap(); - let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap(); + let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap(); // Existing zone should remain unchanged and new stats appended afterwards assert_eq!(rebuilt.len(), 2); assert_eq!(rebuilt[0].sum, 50); @@ -840,7 +851,7 @@ mod tests { )); let trainer = ZoneTrainer::new(MockProcessor::new(), 2).unwrap(); - let rebuilt = rebuild_zones(&existing, trainer, stream).await.unwrap(); + let (rebuilt, _) = rebuild_zones(&existing, trainer, stream).await.unwrap(); // Existing zone plus two new fragments should yield three total zones assert_eq!(rebuilt.len(), 3); assert_eq!(rebuilt[0].bound.fragment_id, 0); diff --git a/rust/lance-index/src/scalar/zonemap.rs b/rust/lance-index/src/scalar/zonemap.rs index 770188f3378..8c1a195f007 100644 --- a/rust/lance-index/src/scalar/zonemap.rs +++ b/rust/lance-index/src/scalar/zonemap.rs @@ -12,12 +12,12 @@ //! false positives that require rechecking. //! //! -use crate::Any; use crate::pbold; use crate::scalar::expression::{SargableQueryParser, ScalarQueryParser}; use crate::scalar::registry::{ BasicTrainer, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, TrainingRequest, }; +use crate::scalar::seed::IndexSeedWriter; use crate::scalar::{ BuiltinIndexType, CreatedIndex, IndexFile, SargableQuery, ScalarIndexParams, UpdateCriteria, compute_next_prefix, @@ -26,6 +26,7 @@ use lance_arrow_stats::StatisticsAccumulator; use lance_core::cache::{LanceCache, WeakLanceCache}; use lance_core::utils::row_addr_remap::RowAddrRemap; use serde::{Deserialize, Serialize}; +use std::any::Any; use std::sync::LazyLock; use arrow_array::{ @@ -34,7 +35,8 @@ use arrow_array::{ use arrow_schema::{DataType, Field}; use datafusion::execution::SendableRecordBatchStream; use datafusion_common::ScalarValue; -use std::sync::Arc; +use lance_select::{RowAddrTreeMap, RowSetOps}; +use std::{collections::HashMap, sync::Arc}; use super::{AnyQuery, IndexStore, MetricsCollector, ScalarIndex, SearchResult}; use crate::scalar::RowIdRemapper; @@ -50,11 +52,13 @@ const ROWS_PER_ZONE_DEFAULT: u64 = 8192; // 1 zone every two batches const ZONEMAP_FILENAME: &str = "zonemap.lance"; const ZONEMAP_SIZE_META_KEY: &str = "rows_per_zone"; +const NULL_BITMAP_META_KEY: &str = "null_bitmap"; +const SEED_NULL_BITMAP_META_KEY: &str = "seed_null_bitmap"; const ZONEMAP_INDEX_VERSION: u32 = 0; /// Basic stats about zonemap index #[derive(Debug, PartialEq, Clone)] -struct ZoneMapStatistics { +pub(crate) struct ZoneMapStatistics { min: ScalarValue, max: ScalarValue, null_count: u32, @@ -107,9 +111,13 @@ pub struct ZoneMapIndex { data_type: DataType, // The maximum rows per zone provided by user rows_per_zone: u64, + use_seeds: bool, store: Arc, fri: Option>, index_cache: WeakLanceCache, + // Exact set of null row addresses across all zones; None when loaded from an + // older index that did not persist this bitmap. + null_rows: Option, } impl std::fmt::Debug for ZoneMapIndex { @@ -118,6 +126,7 @@ impl std::fmt::Debug for ZoneMapIndex { .field("zones", &self.zones) .field("data_type", &self.data_type) .field("rows_per_zone", &self.rows_per_zone) + .field("use_seeds", &self.use_seeds) .field("store", &self.store) .field("fri", &self.fri) .field("index_cache", &self.index_cache) @@ -127,11 +136,16 @@ impl std::fmt::Debug for ZoneMapIndex { impl DeepSizeOf for ZoneMapIndex { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.zones.deep_size_of_children(context) + self.zones.deep_size_of_children(context) + self.null_rows.deep_size_of_children(context) } } impl ZoneMapIndex { + /// Returns the rows-per-zone parameter for this index. + pub fn rows_per_zone(&self) -> u64 { + self.rows_per_zone + } + fn scalar_is_nan(value: &ScalarValue) -> bool { match value { ScalarValue::Float16(Some(value)) => value.is_nan(), @@ -151,6 +165,16 @@ impl ZoneMapIndex { Self::zone_has_finite_min(zone) && !(zone.max.is_null() || Self::scalar_is_nan(&zone.max)) } + fn zone_has_missing_extrema(zone: &ZoneMapStatistics) -> bool { + zone.min.is_null() || zone.max.is_null() + } + + /// Counts prove whether missing extrema mean "no comparable values" or + /// "bounds unknown". Only the latter must conservatively retain the zone. + fn zone_has_comparable_values(zone: &ZoneMapStatistics) -> bool { + zone.bound.length as u128 > u128::from(zone.null_count) + u128::from(zone.nan_count) + } + /// Global `[min, max]` folded across one or more ZoneMap segments (the /// disjoint per-column segments of a multi-segment index), without a scan. /// @@ -170,19 +194,30 @@ impl ZoneMapIndex { ) -> Option<(ScalarValue, ScalarValue)> { let mut min: Option<&ScalarValue> = None; let mut max: Option<&ScalarValue> = None; - for zone in segments.into_iter().flat_map(|seg| seg.zones.iter()) { - if Self::scalar_is_nan(&zone.max) { + for seg in segments.into_iter() { + // Nested types have no meaningful ordering + if seg.data_type.is_nested() { return None; } - if Self::scalar_is_finite_bound(&zone.min) - && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt())) - { - min = Some(&zone.min); - } - if Self::scalar_is_finite_bound(&zone.max) - && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt())) - { - max = Some(&zone.max); + for zone in seg.zones.iter() { + // Legacy Decimal zones can contain comparable values even though their + // extrema were never written, so skipping them would produce a subset. + if Self::zone_has_missing_extrema(zone) && Self::zone_has_comparable_values(zone) { + return None; + } + if Self::scalar_is_nan(&zone.max) { + return None; + } + if Self::scalar_is_finite_bound(&zone.min) + && min.is_none_or(|cur| zone.min.partial_cmp(cur).is_some_and(|o| o.is_lt())) + { + min = Some(&zone.min); + } + if Self::scalar_is_finite_bound(&zone.max) + && max.is_none_or(|cur| zone.max.partial_cmp(cur).is_some_and(|o| o.is_gt())) + { + max = Some(&zone.max); + } } } Some((min?.clone(), max?.clone())) @@ -205,6 +240,29 @@ impl ZoneMapIndex { ) -> Result { use std::ops::Bound; + // For nested types we only track null_count; prune only when certain. + if self.data_type.is_nested() { + let all_null = zone.null_count as usize == zone.bound.length; + return match query { + SargableQuery::IsNull() => Ok(zone.null_count > 0), + SargableQuery::Equals(target) => { + if target.is_null() { + Ok(zone.null_count > 0) + } else { + Ok(!all_null) + } + } + SargableQuery::IsIn(values) => { + if values.iter().any(|v| !v.is_null()) { + Ok(!all_null) + } else { + Ok(zone.null_count > 0) + } + } + _ => Ok(!all_null), + }; + } + match query { SargableQuery::IsNull() => { // Zone contains matching values if it has any null values @@ -229,8 +287,8 @@ impl ZoneMapIndex { return Ok(zone.nan_count > 0); } - if !Self::zone_has_finite_min(zone) { - return Ok(false); + if Self::zone_has_missing_extrema(zone) { + return Ok(Self::zone_has_comparable_values(zone)); } Ok(target >= &zone.min && target <= &zone.max) @@ -238,8 +296,8 @@ impl ZoneMapIndex { SargableQuery::Range(start, end) => { // Zone overlaps with query range if there's any intersection between // the zone's [min, max] and the query's range - if !Self::zone_has_finite_min(zone) { - return Ok(false); + if Self::zone_has_missing_extrema(zone) { + return Ok(Self::zone_has_comparable_values(zone)); } let zone_min = &zone.min; @@ -260,10 +318,8 @@ impl ZoneMapIndex { return Ok(zone.nan_count > 0); } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(zone.nan_count > 0); - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(zone.nan_count > 0); } _ => {} } @@ -290,10 +346,8 @@ impl ZoneMapIndex { return Ok(false); // Nothing is greater than NaN } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(false); // Nothing is greater than NaN - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(false); // Nothing is greater than NaN } _ => {} } @@ -317,10 +371,8 @@ impl ZoneMapIndex { return Ok(zone.nan_count > 0 || zone_min <= e); } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(zone.nan_count > 0 || zone_min <= e); - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(zone.nan_count > 0 || zone_min <= e); } _ => {} } @@ -340,10 +392,8 @@ impl ZoneMapIndex { return Ok(true); } } - ScalarValue::Float64(Some(f)) => { - if f.is_nan() { - return Ok(true); - } + ScalarValue::Float64(Some(f)) if f.is_nan() => { + return Ok(true); } _ => {} } @@ -363,6 +413,8 @@ impl ZoneMapIndex { ScalarValue::Float16(Some(f)) => { if f.is_nan() { zone.nan_count > 0 + } else if Self::zone_has_missing_extrema(zone) { + Self::zone_has_comparable_values(zone) } else if !Self::zone_has_finite_min(zone) { false } else { @@ -372,6 +424,8 @@ impl ZoneMapIndex { ScalarValue::Float32(Some(f)) => { if f.is_nan() { zone.nan_count > 0 + } else if Self::zone_has_missing_extrema(zone) { + Self::zone_has_comparable_values(zone) } else if !Self::zone_has_finite_min(zone) { false } else { @@ -381,6 +435,8 @@ impl ZoneMapIndex { ScalarValue::Float64(Some(f)) => { if f.is_nan() { zone.nan_count > 0 + } else if Self::zone_has_missing_extrema(zone) { + Self::zone_has_comparable_values(zone) } else if !Self::zone_has_finite_min(zone) { false } else { @@ -388,9 +444,13 @@ impl ZoneMapIndex { } } _ => { - Self::zone_has_finite_extrema(zone) - && value >= &zone.min - && value <= &zone.max + if Self::zone_has_missing_extrema(zone) { + Self::zone_has_comparable_values(zone) + } else { + Self::zone_has_finite_extrema(zone) + && value >= &zone.min + && value <= &zone.max + } } } } @@ -454,6 +514,7 @@ impl ZoneMapIndex { store: Arc, fri: Option>, index_cache: &LanceCache, + use_seeds: bool, ) -> Result> where Self: Sized, @@ -469,12 +530,25 @@ impl ZoneMapIndex { .get(ZONEMAP_SIZE_META_KEY) .and_then(|bs| bs.parse().ok()) .unwrap_or(ROWS_PER_ZONE_DEFAULT); + + let null_rows = if let Some(idx_str) = file_schema.metadata.get(NULL_BITMAP_META_KEY) { + let idx = idx_str.parse::().map_err(|e| { + Error::invalid_input(format!("invalid null bitmap buffer index: {e}")) + })?; + let bytes = index_file.read_global_buffer(idx).await?; + Some(RowAddrTreeMap::deserialize_from(bytes.as_ref())?) + } else { + None + }; + Ok(Arc::new(Self::try_from_serialized( zone_maps, store, fri, index_cache, rows_per_zone, + null_rows, + use_seeds, )?)) } @@ -484,6 +558,8 @@ impl ZoneMapIndex { fri: Option>, index_cache: &LanceCache, rows_per_zone: u64, + null_rows: Option, + use_seeds: bool, ) -> Result { // The RecordBatch should have columns: min, max, null_count let min_col = data @@ -542,9 +618,11 @@ impl ZoneMapIndex { zones: Vec::new(), data_type, rows_per_zone, + use_seeds, store, fri, index_cache: WeakLanceCache::from(index_cache), + null_rows, }); } @@ -573,9 +651,11 @@ impl ZoneMapIndex { zones, data_type, rows_per_zone, + use_seeds, store, fri, index_cache: WeakLanceCache::from(index_cache), + null_rows, }) } } @@ -626,11 +706,33 @@ impl ScalarIndex for ZoneMapIndex { metrics: &dyn MetricsCollector, ) -> Result { let query = query.as_any().downcast_ref::().unwrap(); - search_zones(&self.zones, metrics, |zone| { - self.evaluate_zone_against_query(zone, query) + let result = if let SargableQuery::IsNull() = query + && let Some(null_rows) = &self.null_rows + { + SearchResult::exact(null_rows.clone()) + } else { + search_zones(&self.zones, metrics, |zone| { + self.evaluate_zone_against_query(zone, query) + })? + }; + + let Some(remapper) = &self.fri else { + return Ok(result); + }; + let selected = remapper.remap_row_addrs_tree_map(result.row_addrs().selected_rows()); + let nulls = remapper.remap_row_addrs_tree_map(result.row_addrs().null_rows()); + + Ok(match result { + SearchResult::Exact(_) => SearchResult::exact(selected).with_nulls(nulls), + SearchResult::AtMost(_) => SearchResult::at_most(selected).with_nulls(nulls), + SearchResult::AtLeast(_) => SearchResult::at_least(selected).with_nulls(nulls), }) } + fn results_are_row_addresses(&self) -> bool { + true + } + fn can_remap(&self) -> bool { false } @@ -660,19 +762,34 @@ impl ScalarIndex for ZoneMapIndex { let options = ZoneMapIndexBuilderParams::new(self.rows_per_zone); let processor = ZoneMapProcessor::new(value_type.clone())?; let trainer = ZoneTrainer::new(processor, self.rows_per_zone)?; - let updated_zones = rebuild_zones(&self.zones, trainer, new_data).await?; + let (updated_zones, new_null_rows) = rebuild_zones(&self.zones, trainer, new_data).await?; + + // Merge existing and new null rows. If the existing index had no null bitmap + // (legacy format — null positions unknown), preserve that None: updating cannot + // recover the missing information, and claiming the result has zero nulls would + // be a false negative. Only a full retrain produces a fresh, complete bitmap. + let merged_null_rows = self.null_rows.as_ref().map(|existing| { + let mut merged = existing.clone(); + merged |= &new_null_rows; + merged + }); // Serialize the combined zones back into the index file let mut builder = ZoneMapIndexBuilder::try_new(options, self.data_type.clone())?; builder.options.rows_per_zone = self.rows_per_zone; builder.maps = updated_zones; - let file = builder.write_index(dest_store).await?; + builder.null_rows = merged_null_rows; + let has_null_bitmap = builder.null_rows.is_some(); + let files = builder.write_index(dest_store).await?; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()) - .unwrap(), + index_details: make_zone_map_index_details( + self.rows_per_zone, + self.use_seeds, + has_null_bitmap, + ), index_version: ZONEMAP_INDEX_VERSION, - files: vec![file], + files, }) } @@ -683,17 +800,184 @@ impl ScalarIndex for ZoneMapIndex { } fn derive_index_params(&self) -> Result { - let params = serde_json::to_value(ZoneMapIndexBuilderParams::new(self.rows_per_zone))?; + let params = serde_json::to_value(ZoneMapIndexBuilderParams { + rows_per_zone: self.rows_per_zone, + use_seeds: Some(self.use_seeds), + })?; Ok(ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap).with_params(¶ms)) } /// Single-segment `[min, max]` folded from this index's zones; see /// [`value_range_over`](Self::value_range_over) for the full contract. fn value_range(&self) -> Option<(ScalarValue, ScalarValue)> { + // We don't record min/max for nested types + if self.data_type.is_nested() { + return None; + } Self::value_range_over([self]) } } +impl ZoneMapIndex { + async fn try_update_with_seeds( + &self, + seeds: &[crate::scalar::seed::FragmentSeed], + dest_store: &dyn IndexStore, + ) -> Result> { + let mut new_zones = self.zones.clone(); + let mut merged_null_rows = self.null_rows.clone(); + let mut any_missing_bitmap = self.null_rows.is_none(); + + for seed in seeds { + let (mut zones, seed_null_bitmap) = ZoneMapSeedWriter::deserialize_seed( + seed.fragment_id, + &seed.bytes, + self.rows_per_zone, + )?; + new_zones.append(&mut zones); + + if !any_missing_bitmap { + match seed_null_bitmap { + Some(bitmap) => { + let frag_id = u32::try_from(seed.fragment_id).map_err(|_| { + Error::invalid_input(format!( + "fragment_id {} exceeds u32::MAX", + seed.fragment_id + )) + })?; + merged_null_rows + .as_mut() + .unwrap() + .insert_bitmap(frag_id, bitmap); + } + None => { + any_missing_bitmap = true; + merged_null_rows = None; + } + } + } + } + new_zones.sort_by_key(|z| (z.bound.fragment_id, z.bound.start)); + + let mut builder = ZoneMapIndexBuilder::try_new( + ZoneMapIndexBuilderParams::new(self.rows_per_zone), + self.data_type.clone(), + )?; + builder.maps = new_zones; + builder.null_rows = merged_null_rows; + let has_null_bitmap = builder.null_rows.is_some(); + let files = builder.write_index(dest_store).await?; + + Ok(Some(CreatedIndex { + index_details: make_zone_map_index_details( + self.rows_per_zone, + self.use_seeds, + has_null_bitmap, + ), + index_version: ZONEMAP_INDEX_VERSION, + files, + })) + } +} + +fn remap_zone( + zone: &ZoneMapStatistics, + remapper: &dyn RowIdRemapper, + remapped_null_rows: Option<&RowAddrTreeMap>, + is_nested: bool, +) -> Result> { + let zone_start = (zone.bound.fragment_id << 32).saturating_add(zone.bound.start); + let mut remapped = (0..zone.bound.length as u64) + .filter_map(|offset| remapper.remap_row_id(zone_start.saturating_add(offset))) + .collect::>(); + remapped.sort_unstable(); + remapped.dedup(); + + let make_zone = |start: u64, end: u64| -> Result { + let length = (end - start + 1) as usize; + let null_count = if let Some(null_rows) = remapped_null_rows { + u32::try_from( + (start..=end) + .filter(|row_id| null_rows.contains(*row_id)) + .count(), + ) + .map_err(|_| { + Error::invalid_input(format!( + "remapped ZoneMap zone has more null rows than can be represented: \ + fragment_id={}, start={}, length={}", + start >> 32, + start & u64::from(u32::MAX), + length + )) + })? + } else if length == zone.bound.length { + zone.null_count + } else if zone.null_count == 0 { + 0 + } else if zone.null_count as usize == zone.bound.length { + u32::try_from(length).map_err(|_| { + Error::invalid_input(format!( + "remapped all-null ZoneMap zone length cannot be represented: \ + fragment_id={}, start={}, length={}", + start >> 32, + start & u64::from(u32::MAX), + length + )) + })? + } else if length > 1 || (!is_nested && !ZoneMapIndex::zone_has_missing_extrema(zone)) { + // Without exact null positions, one null conservatively preserves both + // null and non-null candidates when the run has multiple rows. A scalar + // singleton is also safe when its extrema independently prove that it + // may contain a comparable value. Otherwise null_count == length would + // fabricate an all-null zone and could prune live non-null rows. + 1 + } else { + return Err(Error::not_supported(format!( + "cannot safely remap a mixed-null ZoneMap zone without an exact null bitmap: \ + fragment_id={}, start={}, original_length={}, null_count={}, remapped_length={}, \ + nested={}, missing_extrema={}", + zone.bound.fragment_id, + zone.bound.start, + zone.bound.length, + zone.null_count, + length, + is_nested, + ZoneMapIndex::zone_has_missing_extrema(zone) + ))); + }; + + Ok(ZoneMapStatistics { + min: zone.min.clone(), + max: zone.max.clone(), + null_count, + nan_count: zone.nan_count, + bound: ZoneBound { + fragment_id: start >> 32, + start: start & u64::from(u32::MAX), + length, + }, + }) + }; + let mut zones = Vec::new(); + let mut run_start = None; + let mut previous = 0u64; + for row_id in remapped { + if run_start.is_none() { + run_start = Some(row_id); + } else if row_id != previous.saturating_add(1) || row_id >> 32 != previous >> 32 { + if let Some(start) = run_start { + zones.push(make_zone(start, previous)?); + } + run_start = Some(row_id); + } + previous = row_id; + } + if let Some(start) = run_start { + zones.push(make_zone(start, previous)?); + } + Ok(zones) +} + /// Merge caller-selected ZoneMap segments into one self-contained segment. pub async fn merge_zonemap_indices( source_indices: &[&ZoneMapIndex], @@ -704,9 +988,12 @@ pub async fn merge_zonemap_indices( Error::invalid_input("merge_zonemap_indices requires at least one source index") })?; let rows_per_zone = first.rows_per_zone; + let use_seeds = first.use_seeds; let data_type = first.data_type.clone(); let mut zones = Vec::new(); + let mut merged_null_rows = RowAddrTreeMap::new(); + let mut any_missing_bitmap = false; for source in source_indices { if source.rows_per_zone != rows_per_zone { return Err(Error::invalid_input(format!( @@ -720,28 +1007,52 @@ pub async fn merge_zonemap_indices( data_type, source.data_type ))); } - zones.extend( - source - .zones - .iter() - .filter(|zone| { - u32::try_from(zone.bound.fragment_id) - .is_ok_and(|fragment_id| fragment_filter.contains(fragment_id)) - }) - .cloned(), - ); + let remapped_null_rows = source.null_rows.as_ref().map(|null_rows| { + source.fri.as_deref().map_or_else( + || null_rows.clone(), + |remapper| remapper.remap_row_addrs_tree_map(null_rows), + ) + }); + for zone in &source.zones { + let source_zones = source.fri.as_deref().map_or_else( + || Ok(vec![zone.clone()]), + |remapper| { + remap_zone( + zone, + remapper, + remapped_null_rows.as_ref(), + source.data_type.is_nested(), + ) + }, + )?; + zones.extend(source_zones.into_iter().filter(|zone| { + u32::try_from(zone.bound.fragment_id) + .is_ok_and(|fragment_id| fragment_filter.contains(fragment_id)) + })); + } + match remapped_null_rows { + Some(mut filtered) => { + filtered.retain_fragments(fragment_filter.iter()); + merged_null_rows |= &filtered; + } + None => any_missing_bitmap = true, + } } zones.sort_by_key(|zone| (zone.bound.fragment_id, zone.bound.start)); let mut builder = ZoneMapIndexBuilder::try_new(ZoneMapIndexBuilderParams::new(rows_per_zone), data_type)?; builder.maps = zones; - builder.write_index(dest_store).await?; + if !any_missing_bitmap { + builder.null_rows = Some(merged_null_rows); + } + let has_null_bitmap = builder.null_rows.is_some(); + let files = builder.write_index(dest_store).await?; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()).unwrap(), + index_details: make_zone_map_index_details(rows_per_zone, use_seeds, has_null_bitmap), index_version: ZONEMAP_INDEX_VERSION, - files: dest_store.list_files_with_sizes().await?, + files, }) } @@ -753,6 +1064,12 @@ fn default_rows_per_zone() -> u64 { pub struct ZoneMapIndexBuilderParams { #[serde(default = "default_rows_per_zone")] rows_per_zone: u64, + /// Whether to embed per-fragment seed buffers in data files for use during + /// incremental index updates. `None` means auto-detect based on column type + /// (see [`default_use_seeds`]). Resolved to a concrete `bool` during + /// training in [`ZoneMapIndexPlugin::new_training_request`]. + #[serde(default)] + use_seeds: Option, } static DEFAULT_ROWS_PER_ZONE: LazyLock = LazyLock::new(|| { @@ -766,13 +1083,17 @@ impl Default for ZoneMapIndexBuilderParams { fn default() -> Self { Self { rows_per_zone: *DEFAULT_ROWS_PER_ZONE, + use_seeds: None, } } } impl ZoneMapIndexBuilderParams { pub fn new(rows_per_zone: u64) -> Self { - Self { rows_per_zone } + Self { + rows_per_zone, + use_seeds: None, + } } pub fn rows_per_zone(&self) -> u64 { @@ -786,6 +1107,10 @@ pub struct ZoneMapIndexBuilder { items_type: DataType, maps: Vec, + // None means "legacy index — null positions unknown"; Some means a complete bitmap. + // write_index omits the null-bitmap global buffer when this is None, preserving the + // legacy format so that downstream searches remain conservative. + null_rows: Option, } impl ZoneMapIndexBuilder { @@ -794,6 +1119,7 @@ impl ZoneMapIndexBuilder { options, items_type, maps: Vec::new(), + null_rows: None, }) } @@ -803,7 +1129,9 @@ impl ZoneMapIndexBuilder { pub async fn train(&mut self, batches_source: SendableRecordBatchStream) -> Result<()> { let processor = ZoneMapProcessor::new(self.items_type.clone())?; let trainer = ZoneTrainer::new(processor, self.options.rows_per_zone)?; - self.maps = trainer.train(batches_source).await?; + let (maps, null_rows) = trainer.train(batches_source).await?; + self.maps = maps; + self.null_rows = Some(null_rows); Ok(()) } @@ -856,7 +1184,7 @@ impl ZoneMapIndexBuilder { Ok(RecordBatch::try_new(schema, columns)?) } - pub async fn write_index(self, index_store: &dyn IndexStore) -> Result { + pub async fn write_index(self, index_store: &dyn IndexStore) -> Result> { let record_batch = self.zonemap_stats_as_batch()?; let mut file_schema = record_batch.schema().as_ref().clone(); @@ -869,12 +1197,34 @@ impl ZoneMapIndexBuilder { .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema)) .await?; index_file.write_record_batch(record_batch).await?; - index_file.finish().await + + let zonemap_file = if let Some(null_rows) = self.null_rows { + let mut null_bitmap_bytes = Vec::with_capacity(null_rows.serialized_size()); + null_rows.serialize_into(&mut null_bitmap_bytes)?; + let null_bitmap_idx = index_file + .add_global_buffer(bytes::Bytes::from(null_bitmap_bytes)) + .await?; + index_file + .finish_with_metadata(HashMap::from([( + NULL_BITMAP_META_KEY.to_string(), + null_bitmap_idx.to_string(), + )])) + .await? + } else { + index_file.finish_with_metadata(HashMap::new()).await? + }; + + Ok(vec![zonemap_file]) } } -/// Index-specific processor that computes min/max statistics for each zone while the -/// trainer takes care of chunking and fragment boundaries. +/// Index-specific processor that computes zone statistics while the trainer +/// handles chunking and fragment boundaries. +/// +/// For non-nested types, tracks min, max, null_count, and nan_count. +/// For nested types (List, FixedSizeList, Struct, Map, etc.), tracks only +/// null_count; min and max are stored as typed null values. +#[derive(Debug)] struct ZoneMapProcessor { data_type: DataType, statistics: StatisticsAccumulator, @@ -882,9 +1232,10 @@ struct ZoneMapProcessor { impl ZoneMapProcessor { fn new(data_type: DataType) -> Result { + let statistics = StatisticsAccumulator::new(&data_type); Ok(Self { - statistics: StatisticsAccumulator::new(&data_type), data_type, + statistics, }) } @@ -943,6 +1294,19 @@ impl ZoneProcessor for ZoneMapProcessor { fn finish_zone(&mut self, bound: ZoneBound) -> Result { let statistics = self.statistics.statistics(); + let null_count = Self::stat_count_to_u32("null_count", statistics.null_count)?; + + // For nested types, only null_count is meaningful; store null min/max. + if self.data_type.is_nested() { + return Ok(ZoneMapStatistics { + min: ScalarValue::try_new_null(&self.data_type)?, + max: ScalarValue::try_new_null(&self.data_type)?, + null_count, + nan_count: 0, + bound, + }); + } + let nan_count = Self::stat_count_to_u32("nan_count", statistics.nan_count.unwrap_or(0))?; Ok(ZoneMapStatistics { min: Self::scalar_value_from_stat( @@ -954,7 +1318,7 @@ impl ZoneProcessor for ZoneMapProcessor { &self.data_type, nan_count, )?, - null_count: Self::stat_count_to_u32("null_count", statistics.null_count)?, + null_count, nan_count, bound, }) @@ -966,6 +1330,44 @@ impl ZoneProcessor for ZoneMapProcessor { } } +fn make_zone_map_index_details( + rows_per_zone: u64, + use_seeds: bool, + has_null_bitmap: bool, +) -> prost_types::Any { + prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails { + rows_per_zone: Some(rows_per_zone), + use_seeds: Some(use_seeds), + has_null_bitmap: Some(has_null_bitmap), + }) + .unwrap() +} + +/// Returns true when seed-based incremental updates should be enabled by +/// default for the given column type. +/// +/// Seeds pay off for variable-length types (strings, binary) — which can be +/// arbitrarily wide — and fixed-width types wider than 8 bytes (e.g. +/// Decimal128, FixedSizeBinary tensors). Fixed-width types ≤ 8 bytes (Int64, +/// Float64, …) scan fast enough that the seed overhead is not worth it. +fn default_use_seeds(data_type: &DataType) -> bool { + match data_type { + // Variable-length: width is unbounded, skipping scans is always valuable. + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + | DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView => true, + // Fixed-width types wider than 8 bytes. + DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => true, + DataType::FixedSizeBinary(n) => *n > 8, + // Nested types (FSL, List, Struct, Map, …): typically wide. + _ if data_type.is_nested() => true, + _ => false, + } +} + #[derive(Debug, Default)] pub struct ZoneMapIndexPlugin; @@ -974,8 +1376,7 @@ impl ZoneMapIndexPlugin { batches_source: SendableRecordBatchStream, index_store: &dyn IndexStore, options: Option, - ) -> Result { - // train_zonemap_index: calling scan_aligned_chunks + ) -> Result> { let value_type = batches_source.schema().field(0).data_type().clone(); let mut builder = ZoneMapIndexBuilder::try_new(options.unwrap_or_default(), value_type)?; @@ -1016,14 +1417,12 @@ impl BasicTrainer for ZoneMapIndexPlugin { params: &str, field: &Field, ) -> Result> { - if field.data_type().is_nested() { - return Err(Error::invalid_input_source( - "A zone map index can only be created on a non-nested field.".into(), - )); + let mut params = serde_json::from_str::(params)?; + // Resolve None → type-based default so train_index always sees Some(bool). + if params.use_seeds.is_none() { + params.use_seeds = Some(default_use_seeds(field.data_type())); } - let params = serde_json::from_str::(params)?; - Ok(Box::new(ZoneMapIndexTrainingRequest::new(params))) } @@ -1042,12 +1441,15 @@ impl BasicTrainer for ZoneMapIndexPlugin { "must provide training request created by new_training_request".into(), ) })?; - let file = Self::train_zonemap_index(data, index_store, Some(request.params)).await?; + let rows_per_zone = request.params.rows_per_zone; + let use_seeds = request.params.use_seeds.unwrap_or(false); + let files = Self::train_zonemap_index(data, index_store, Some(request.params)).await?; + // Training a new index will always populate the null bitmap + let has_null_bitmap = true; Ok(CreatedIndex { - index_details: prost_types::Any::from_msg(&pbold::ZoneMapIndexDetails::default()) - .unwrap(), + index_details: make_zone_map_index_details(rows_per_zone, use_seeds, has_null_bitmap), index_version: ZONEMAP_INDEX_VERSION, - files: vec![file], + files, }) } } @@ -1073,671 +1475,827 @@ impl ScalarIndexPlugin for ZoneMapIndexPlugin { fn new_query_parser( &self, index_name: String, - _index_details: &prost_types::Any, + index_details: &prost_types::Any, ) -> Option> { - Some(Box::new(SargableQueryParser::new( - index_name, - self.name().to_string(), - true, - ))) + let has_null_bitmap = index_details + .to_msg::() + .ok() + .and_then(|d| d.has_null_bitmap) + .unwrap_or(false); + let parser = SargableQueryParser::new(index_name, self.name().to_string(), true); + let parser = if has_null_bitmap { + parser.with_exact_null_tracking() + } else { + parser + }; + Some(Box::new(parser)) } async fn load_index( &self, index_store: Arc, - _index_details: &prost_types::Any, + index_details: &prost_types::Any, frag_reuse_index: Option>, cache: &LanceCache, ) -> Result> { - Ok(ZoneMapIndex::load(index_store, frag_reuse_index, cache).await? as Arc) + let use_seeds = index_details + .to_msg::() + .ok() + .and_then(|d| d.use_seeds) + .unwrap_or(false); + Ok( + ZoneMapIndex::load(index_store, frag_reuse_index, cache, use_seeds).await? + as Arc, + ) } -} -#[cfg(test)] -mod tests { - use crate::scalar::registry::VALUE_COLUMN_NAME; - use crate::scalar::{IndexStore, zonemap::ROWS_PER_ZONE_DEFAULT}; - use std::sync::Arc; + fn might_use_seeds(&self, index_details: &prost_types::Any) -> bool { + index_details + .to_msg::() + .ok() + .and_then(|d| d.use_seeds) + .unwrap_or(false) + } - use crate::scalar::zoned::ZoneBound; - use crate::scalar::zonemap::{ZoneMapIndexPlugin, ZoneMapStatistics}; - use arrow::datatypes::{ArrowPrimitiveType, Float32Type, Int64Type}; - use arrow_array::{Array, PrimitiveArray, RecordBatch, UInt64Array, record_batch}; - use arrow_schema::{DataType, Field, Schema}; - use datafusion::execution::SendableRecordBatchStream; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; - use datafusion_common::ScalarValue; - use futures::{StreamExt, TryStreamExt, stream}; - use lance_core::utils::tempfile::TempObjDir; - use lance_core::{ - ROW_ADDR, - cache::{LanceCache, WeakLanceCache}, - }; - use lance_datafusion::datagen::DatafusionDatagenExt; - use lance_datagen::ArrayGeneratorExt; - use lance_datagen::{BatchCount, RowCount, array}; - use lance_io::object_store::ObjectStore; - use lance_select::{NullableRowAddrSet, RowAddrTreeMap}; + async fn create_seed_writer( + &self, + field_path: &str, + data_type: &DataType, + index_details: &prost_types::Any, + ) -> Result>> { + let details = index_details.to_msg::().ok(); + let Some(rows_per_zone) = details.as_ref().and_then(|d| d.rows_per_zone) else { + return Ok(None); + }; + if !details.as_ref().and_then(|d| d.use_seeds).unwrap_or(false) { + return Ok(None); + } + Ok(Some(Box::new(ZoneMapSeedWriter::new( + field_path, + rows_per_zone, + data_type.clone(), + )?))) + } - use crate::scalar::{ - SargableQuery, ScalarIndex, SearchResult, - lance_format::LanceIndexStore, - zonemap::{ - ZONEMAP_FILENAME, ZONEMAP_SIZE_META_KEY, ZoneMapIndex, ZoneMapIndexBuilderParams, - }, - }; + async fn update_from_seeds( + &self, + seeds: Vec, + reference_index: Arc, + index_details: &prost_types::Any, + dest_store: &dyn IndexStore, + ) -> Result> { + let Some(rows_per_zone) = index_details + .to_msg::() + .ok() + .and_then(|d| d.rows_per_zone) + else { + return Ok(None); + }; - // Add missing imports for the tests - use crate::Index; // Import Index trait to access calculate_included_frags - use crate::metrics::NoOpMetricsCollector; - use roaring::RoaringBitmap; // Import RoaringBitmap for the test - use std::collections::Bound; + // Validate each seed was written with the same rows_per_zone. + for seed in &seeds { + let rpz_in_seed = seed + .metadata_value + .split_once(':') + .and_then(|(_, rpz)| rpz.parse::().ok()); + if rpz_in_seed != Some(rows_per_zone) { + return Ok(None); + } + } - // Adds a _rowaddr column emulating each batch as a new fragment - fn add_row_addr(stream: SendableRecordBatchStream) -> SendableRecordBatchStream { - let schema = stream.schema(); - let schema_with_row_addr = Arc::new(Schema::new(vec![ - schema.field(0).clone(), - Field::new(ROW_ADDR, DataType::UInt64, false), - ])); - let schema = schema_with_row_addr.clone(); - let stream = stream.enumerate().map(move |(frag_id, batch)| { - let batch = batch.unwrap(); - let row_addr = Arc::new(UInt64Array::from_iter_values( - (0..batch.num_rows() as u64).map(|off| off + ((frag_id as u64) << 32)), - )); - Ok(RecordBatch::try_new( - schema_with_row_addr.clone(), - vec![batch.column(0).clone(), row_addr], - )?) - }); - Box::pin(RecordBatchStreamAdapter::new(schema, stream)) + let Some(zone_map) = reference_index.as_any().downcast_ref::() else { + return Ok(None); + }; + zone_map.try_update_with_seeds(&seeds, dest_store).await } +} - /// Build a single-column ZoneMap of primitive type `T` from `fragments` - /// (one batch -> one fragment), with small zones, then load it back. - async fn train_and_load( - fragments: Vec>>, - ) -> Arc - where - PrimitiveArray: From>>, - { - let tmpdir = TempObjDir::default(); - let test_store = Arc::new(LanceIndexStore::new( - Arc::new(ObjectStore::local()), - tmpdir.clone(), - Arc::new(LanceCache::no_cache()), - )); - let schema = Arc::new(Schema::new(vec![Field::new( - VALUE_COLUMN_NAME, - T::DATA_TYPE, - true, - )])); - let batches: Vec = fragments - .into_iter() - .map(|vals| { - RecordBatch::try_new( - schema.clone(), - vec![Arc::new(PrimitiveArray::::from(vals))], - ) - .unwrap() - }) - .collect(); - let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - stream::iter(batches.into_iter().map(Ok)), - )); - let stream = add_row_addr(stream); - - ZoneMapIndexPlugin::train_zonemap_index( - stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::new(2)), - ) - .await - .unwrap(); +/// A seed writer that observes column values during data file writes and +/// accumulates zone map statistics for later harvest during index updates. +/// +/// Zone statistics are serialized as Arrow IPC bytes and embedded in the data +/// file footer as a global buffer, keyed by `"lance.seed."`. +#[derive(Debug)] +pub struct ZoneMapSeedWriter { + column_name: String, + rows_per_zone: u64, + data_type: DataType, + completed_zones: Vec, + processor: ZoneMapProcessor, + rows_in_current_zone: u64, + next_zone_start: u64, + /// Null row offsets within this fragment (sequential, 0-indexed). + null_offsets: RoaringBitmap, +} - ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex") +impl ZoneMapSeedWriter { + /// Create a new `ZoneMapSeedWriter` for the given column. + pub fn new( + column_name: impl Into, + rows_per_zone: u64, + data_type: DataType, + ) -> Result { + if rows_per_zone == 0 { + return Err(lance_core::Error::invalid_input( + "rows_per_zone must be greater than zero", + )); + } + let processor = ZoneMapProcessor::new(data_type.clone())?; + Ok(Self { + column_name: column_name.into(), + rows_per_zone, + data_type, + completed_zones: Vec::new(), + processor, + rows_in_current_zone: 0, + next_zone_start: 0, + null_offsets: RoaringBitmap::new(), + }) } - #[tokio::test] - async fn test_value_range_spans_fragments() { - // Two fragments, multiple zones each; global min/max straddle both. - let index = train_and_load::(vec![ - vec![Some(10), Some(50), Some(30)], - vec![Some(5), Some(99), Some(42)], - ]) - .await; - assert_eq!( - index.value_range(), - Some((ScalarValue::Int64(Some(5)), ScalarValue::Int64(Some(99)))) - ); - } + fn seed_batch_from_zones( + zones: &[ZoneMapStatistics], + data_type: &DataType, + ) -> Result { + let mins = if zones.is_empty() { + arrow_array::new_empty_array(data_type) + } else { + datafusion_common::ScalarValue::iter_to_array(zones.iter().map(|s| s.min.clone()))? + }; + let maxs = if zones.is_empty() { + arrow_array::new_empty_array(data_type) + } else { + datafusion_common::ScalarValue::iter_to_array(zones.iter().map(|s| s.max.clone()))? + }; + let null_counts = + arrow_array::UInt32Array::from_iter_values(zones.iter().map(|s| s.null_count)); + let nan_counts = + arrow_array::UInt32Array::from_iter_values(zones.iter().map(|s| s.nan_count)); + let zone_lengths = + arrow_array::UInt64Array::from_iter_values(zones.iter().map(|s| s.bound.length as u64)); - #[tokio::test] - async fn test_value_range_all_null_is_none() { - let index = train_and_load::(vec![vec![None, None, None]]).await; - assert_eq!(index.value_range(), None); - } + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("min", data_type.clone(), true), + Field::new("max", data_type.clone(), true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("zone_length", DataType::UInt64, false), + ])); - #[tokio::test] - async fn test_value_range_nan_max_is_none() { - // Zones of size 2: [1.0, 2.0] then [100.0, NaN]. The NaN-bearing zone hides - // its finite max (100.0), so the only sound answer is None. - let index = train_and_load::(vec![vec![ - Some(1.0), - Some(2.0), - Some(100.0), - Some(f32::NAN), - ]]) - .await; - assert_eq!(index.value_range(), None); + let columns: Vec = vec![ + mins, + maxs, + Arc::new(null_counts) as ArrayRef, + Arc::new(nan_counts) as ArrayRef, + Arc::new(zone_lengths) as ArrayRef, + ]; + Ok(arrow_array::RecordBatch::try_new(schema, columns)?) } - #[tokio::test] - async fn test_value_range_over_folds_segments() { - // Two disjoint segments of one logical index; the global range straddles - // both (min and max from `b`), proving the fold spans segments. - let a = train_and_load::(vec![vec![Some(5), Some(9)]]).await; - let b = train_and_load::(vec![vec![Some(1), Some(20)]]).await; - assert_eq!( - ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]), - Some((ScalarValue::Int64(Some(1)), ScalarValue::Int64(Some(20)))) - ); - } + /// Deserialize zone map seed bytes (Arrow IPC) into zone statistics and a null bitmap. + /// + /// Returns zone statistics (bounds reconstructed from sequential fragment layout) and an + /// optional `RoaringBitmap` of null row offsets within the fragment. The bitmap is absent + /// for seeds written by older code that did not track null positions. + pub(crate) fn deserialize_seed( + fragment_id: u64, + bytes: &bytes::Bytes, + rows_per_zone: u64, + ) -> Result<(Vec, Option)> { + use arrow_ipc::reader::FileReader; + use std::io::Cursor; - #[tokio::test] - async fn test_value_range_over_nan_in_any_segment_is_none() { - // NaN in one segment hides that segment's finite max; the cross-segment - // fold must bail to None just as the single-segment path does. - let a = train_and_load::(vec![vec![Some(1.0), Some(2.0)]]).await; - let b = train_and_load::(vec![vec![Some(100.0), Some(f32::NAN)]]).await; - assert_eq!( - ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]), + let cursor = Cursor::new(bytes.as_ref()); + let mut reader = FileReader::try_new(cursor, None).map_err(|e| { + lance_core::Error::invalid_input(format!("failed to read zone map seed IPC: {}", e)) + })?; + + let null_bitmap = if let Some(hex) = reader.schema().metadata.get(SEED_NULL_BITMAP_META_KEY) + { + let bitmap_bytes = hex_decode(hex).map_err(|e| { + lance_core::Error::invalid_input(format!( + "failed to decode seed null bitmap hex: {}", + e + )) + })?; + let bitmap = if bitmap_bytes.is_empty() { + RoaringBitmap::default() + } else { + RoaringBitmap::deserialize_from(bitmap_bytes.as_slice()).map_err(|e| { + lance_core::Error::invalid_input(format!( + "failed to deserialize seed null bitmap: {}", + e + )) + })? + }; + Some(bitmap) + } else { None - ); - } + }; - #[tokio::test] - async fn test_value_range_over_skips_all_null_segment() { - // An all-null segment yields no finite zone; folding it with a finite - // segment returns the finite segment's range (null contributes nothing). - let a = train_and_load::(vec![vec![None, None]]).await; - let b = train_and_load::(vec![vec![Some(3), Some(7)]]).await; - assert_eq!( - ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]), - Some((ScalarValue::Int64(Some(3)), ScalarValue::Int64(Some(7)))) - ); - } + let batch = match reader.next() { + Some(Ok(batch)) => batch, + Some(Err(e)) => { + return Err(lance_core::Error::invalid_input(format!( + "failed to read zone map seed batch: {}", + e + ))); + } + None => return Ok((Vec::new(), null_bitmap)), + }; - #[tokio::test] - async fn test_empty_zonemap_index() { - let tmpdir = TempObjDir::default(); - let test_store = Arc::new(LanceIndexStore::new( - Arc::new(ObjectStore::local()), - tmpdir.clone(), - Arc::new(LanceCache::no_cache()), - )); + let min_col = batch + .column_by_name("min") + .ok_or_else(|| lance_core::Error::invalid_input("seed batch missing 'min' column"))?; + let max_col = batch + .column_by_name("max") + .ok_or_else(|| lance_core::Error::invalid_input("seed batch missing 'max' column"))?; + let null_count_col = batch + .column_by_name("null_count") + .ok_or_else(|| { + lance_core::Error::invalid_input("seed batch missing 'null_count' column") + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::invalid_input("seed 'null_count' is not UInt32"))?; + let nan_count_col = batch + .column_by_name("nan_count") + .ok_or_else(|| { + lance_core::Error::invalid_input("seed batch missing 'nan_count' column") + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::invalid_input("seed 'nan_count' is not UInt32"))?; + let zone_length_col = batch + .column_by_name("zone_length") + .ok_or_else(|| { + lance_core::Error::invalid_input("seed batch missing 'zone_length' column") + })? + .as_any() + .downcast_ref::() + .ok_or_else(|| lance_core::Error::invalid_input("seed 'zone_length' is not UInt64"))?; - let data = arrow_array::Int32Array::from(Vec::::new()); - let row_ids = arrow_array::UInt64Array::from(Vec::::new()); - let schema = Arc::new(Schema::new(vec![ - Field::new(VALUE_COLUMN_NAME, DataType::Int32, false), - Field::new(ROW_ADDR, DataType::UInt64, false), - ])); - let data = - RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); + let num_zones = batch.num_rows(); + let mut zones = Vec::with_capacity(num_zones); + for i in 0..num_zones { + let zone_start = i as u64 * rows_per_zone; + let zone_length = zone_length_col.value(i) as usize; + zones.push(ZoneMapStatistics { + min: datafusion_common::ScalarValue::try_from_array(min_col, i)?, + max: datafusion_common::ScalarValue::try_from_array(max_col, i)?, + null_count: null_count_col.value(i), + nan_count: nan_count_col.value(i), + bound: ZoneBound { + fragment_id, + start: zone_start, + length: zone_length, + }, + }); + } + Ok((zones, null_bitmap)) + } +} - let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema, - stream::once(std::future::ready(Ok(data))), - )); +impl IndexSeedWriter for ZoneMapSeedWriter { + fn column_name(&self) -> &str { + &self.column_name + } - ZoneMapIndexPlugin::train_zonemap_index(data_stream, test_store.as_ref(), None) - .await - .unwrap(); + fn observe_batch(&mut self, values: &ArrayRef) -> lance_core::Result<()> { + let mut offset = 0usize; - log::debug!("Successfully wrote the index file"); + while offset < values.len() { + let remaining_in_zone = self.rows_per_zone - self.rows_in_current_zone; + let chunk_len = ((values.len() as u64 - offset as u64).min(remaining_in_zone)) as usize; + let chunk = values.slice(offset, chunk_len); + self.processor.process_chunk(&chunk)?; - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 0); - assert_eq!(index.data_type, DataType::Int32); - assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT); + if chunk.null_count() > 0 { + let base_offset = (self.next_zone_start + self.rows_in_current_zone) as u32; + for i in 0..chunk_len { + if chunk.is_null(i) { + self.null_offsets.insert(base_offset + i as u32); + } + } + } - // Equals query: null (should match nothing, as there are no nulls) - let query = SargableQuery::Equals(ScalarValue::Int32(None)); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); - } + self.rows_in_current_zone += chunk_len as u64; + offset += chunk_len; - #[tokio::test] - // Test that a zonemap index can be created with null values from few fragments - async fn test_null_zonemap_index() { - let tmpdir = TempObjDir::default(); - let test_store = Arc::new(LanceIndexStore::new( - Arc::new(ObjectStore::local()), - tmpdir.clone(), - Arc::new(LanceCache::no_cache()), - )); + if self.rows_in_current_zone >= self.rows_per_zone { + let bound = ZoneBound { + fragment_id: 0, + start: self.next_zone_start, + length: self.rows_per_zone as usize, + }; + let stats = self.processor.finish_zone(bound)?; + self.processor.reset()?; + self.completed_zones.push(stats); + self.next_zone_start += self.rows_per_zone; + self.rows_in_current_zone = 0; + } + } + Ok(()) + } - let stream = lance_datagen::gen_batch() - .col( - VALUE_COLUMN_NAME, - array::rand::().with_nulls(&[true, false, false, false, false]), - ) - .into_df_stream(RowCount::from(5000), BatchCount::from(10)); + fn finish(&mut self) -> lance_core::Result> { + use arrow_ipc::writer::FileWriter; + use std::io::Cursor; + + // Flush partial final zone + if self.rows_in_current_zone > 0 { + let bound = ZoneBound { + fragment_id: 0, + start: self.next_zone_start, + length: self.rows_in_current_zone as usize, + }; + let stats = self.processor.finish_zone(bound)?; + self.processor.reset()?; + self.completed_zones.push(stats); + self.next_zone_start += self.rows_in_current_zone; + self.rows_in_current_zone = 0; + } - // Add _rowaddr column - let stream = add_row_addr(stream); + if self.completed_zones.is_empty() { + return Ok(None); + } - ZoneMapIndexPlugin::train_zonemap_index( - stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::new(5000)), - ) - .await - .unwrap(); + let batch = Self::seed_batch_from_zones(&self.completed_zones, &self.data_type)?; - log::debug!("Successfully wrote the index file"); + // Embed null bitmap in schema metadata so deserialize_seed can reconstruct null_rows. + let null_offsets = std::mem::take(&mut self.null_offsets); + let bitmap_bytes = if null_offsets.is_empty() { + Vec::default() + } else { + let mut bitmap_bytes = Vec::with_capacity(null_offsets.serialized_size()); + null_offsets + .serialize_into(&mut bitmap_bytes) + .map_err(|e| { + lance_core::Error::invalid_input(format!( + "failed to serialize seed null bitmap: {}", + e + )) + })?; + bitmap_bytes + }; + let mut schema = batch.schema().as_ref().clone(); + schema.metadata.insert( + SEED_NULL_BITMAP_META_KEY.to_string(), + hex_encode(&bitmap_bytes), + ); + let batch = RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec())?; - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 10); - for (i, zone) in index.zones.iter().enumerate() { - assert_eq!(zone.null_count, 1000); - assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); - assert_eq!(zone.bound.length, 5000); - assert_eq!(zone.bound.fragment_id, i as u64); + // Serialize to Arrow IPC + let mut buf = Cursor::new(Vec::new()); + { + let mut writer = FileWriter::try_new(&mut buf, batch.schema_ref()).map_err(|e| { + lance_core::Error::invalid_input(format!("failed to create IPC writer: {}", e)) + })?; + writer.write(&batch).map_err(|e| { + lance_core::Error::invalid_input(format!("failed to write IPC batch: {}", e)) + })?; + writer.finish().map_err(|e| { + lance_core::Error::invalid_input(format!("failed to finish IPC writer: {}", e)) + })?; } - // Equals query: null (should match all zones since they contain null values) - let query = SargableQuery::Equals(ScalarValue::Int32(None)); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Reset state for next fragment + self.completed_zones.clear(); + self.next_zone_start = 0; + self.processor = ZoneMapProcessor::new(self.data_type.clone())?; - // Create expected RowAddrTreeMap with all zones since they contain null values - let mut expected = RowAddrTreeMap::new(); - for fragment_id in 0..10 { - let start = (fragment_id as u64) << 32; - let end = start + 5000; - expected.insert_range(start..end); - } - assert_eq!(result, SearchResult::at_most(expected)); + Ok(Some(bytes::Bytes::from(buf.into_inner()))) + } - // Test update - add new data with Float32 values (matching the original data type) - let new_data = - arrow_array::Float32Array::from_iter_values((0..5000).map(|i| i as f32 / 1000.0)); - // Create row addresses for fragment 10 (next fragment after 0-9) - let new_row_addr = - UInt64Array::from_iter_values((0..5000).map(|i| (10u64 << 32) | (i as u64))); - let new_schema = Arc::new(Schema::new(vec![ - Field::new(VALUE_COLUMN_NAME, DataType::Float32, false), // Match original schema - Field::new(ROW_ADDR, DataType::UInt64, false), // Use _rowaddr as expected by the builder - ])); - let new_data_batch = RecordBatch::try_new( - new_schema.clone(), - vec![Arc::new(new_data), Arc::new(new_row_addr)], + fn schema_metadata_key(&self) -> String { + format!( + "{}{}", + crate::scalar::seed::SEED_META_KEY_PREFIX, + self.column_name ) - .unwrap(); - let new_data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - new_schema, - stream::once(std::future::ready(Ok(new_data_batch))), - )); + } - // Directly pass the stream with proper row addresses instead of using MockTrainingSource - // which would regenerate row addresses starting from 0 - index - .update(new_data_stream, test_store.as_ref(), None) - .await - .unwrap(); + fn schema_metadata_value(&self, buf_index: u32) -> String { + format!("{}:{}", buf_index, self.rows_per_zone) + } +} - // Verify the updated index has more zones - let updated_index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load updated ZoneMapIndex"); +fn hex_encode(data: &[u8]) -> String { + data.iter().map(|b| format!("{:02x}", b)).collect() +} - // Should have original 10 zones + 1 new zone (5000 rows with zone size 5000) - assert_eq!(updated_index.zones.len(), 11); +fn hex_decode(s: &str) -> std::result::Result, String> { + if s.is_empty() { + return Ok(Vec::default()); + } + if !s.len().is_multiple_of(2) { + return Err(format!("odd hex length: {}", s.len())); + } + (0..s.len()) + .step_by(2) + .map(|i| { + u8::from_str_radix(&s[i..i + 2], 16).map_err(|e| format!("invalid hex at {}: {}", i, e)) + }) + .collect() +} - // Verify the new zone was added - let new_zone = &updated_index.zones[10]; // Last zone should be the new one - assert_eq!(new_zone.bound.fragment_id, 10u64); // New fragment ID - assert_eq!(new_zone.bound.length, 5000); - assert_eq!(new_zone.null_count, 0); // New data has no nulls - assert_eq!(new_zone.nan_count, 0); // New data has no NaN values +#[cfg(test)] +mod tests { + use crate::scalar::registry::VALUE_COLUMN_NAME; + use crate::scalar::{IndexStore, zonemap::ROWS_PER_ZONE_DEFAULT}; + use std::sync::Arc; - // Test search on updated index - search for null values should still work - let query = SargableQuery::Equals(ScalarValue::Float32(None)); - let result = updated_index - .search(&query, &NoOpMetricsCollector) - .await - .unwrap(); + use crate::scalar::zoned::ZoneBound; + use crate::scalar::zonemap::{ZoneMapIndexPlugin, ZoneMapStatistics}; + use arrow::datatypes::{ArrowPrimitiveType, Decimal128Type, Float32Type, Int64Type}; + use arrow_array::{Array, PrimitiveArray, RecordBatch, UInt64Array, record_batch}; + use arrow_schema::{DataType, Field, Schema}; + use datafusion::execution::SendableRecordBatchStream; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use datafusion_common::ScalarValue; + use futures::{StreamExt, TryStreamExt, stream}; + use lance_core::utils::tempfile::TempObjDir; + use lance_core::{ + ROW_ADDR, + cache::{LanceCache, WeakLanceCache}, + }; + use lance_datafusion::datagen::DatafusionDatagenExt; + use lance_datagen::ArrayGeneratorExt; + use lance_datagen::{BatchCount, RowCount, array}; + use lance_io::object_store::ObjectStore; + use lance_select::RowAddrTreeMap; - // Should match original 10 zones (with nulls) but not the new zone (no nulls) - let mut expected = RowAddrTreeMap::new(); - for fragment_id in 0..10 { - let start = (fragment_id as u64) << 32; - let end = start + 5000; - expected.insert_range(start..end); - } - assert_eq!(result, SearchResult::at_most(expected)); + use crate::scalar::{ + RowIdRemapper, SargableQuery, ScalarIndex, SearchResult, + lance_format::LanceIndexStore, + zonemap::{ + ZONEMAP_FILENAME, ZONEMAP_SIZE_META_KEY, ZoneMapIndex, ZoneMapIndexBuilderParams, + merge_zonemap_indices, remap_zone, + }, + }; - // Test search for a value that should be in the new zone - let query = SargableQuery::Equals(ScalarValue::Float32(Some(2.5))); // Value 2500/1000 = 2.5 - let result = updated_index - .search(&query, &NoOpMetricsCollector) - .await - .unwrap(); + // Add missing imports for the tests + use crate::Index; // Import Index trait to access calculate_included_frags + use crate::metrics::NoOpMetricsCollector; + use roaring::RoaringBitmap; // Import RoaringBitmap for the test + use std::collections::{Bound, HashMap}; - // Should match the new zone (fragment 10) - let mut expected = RowAddrTreeMap::new(); - let start = 10u64 << 32; - let end = start + 5000; - expected.insert_range(start..end); - assert_eq!(result, SearchResult::at_most(expected)); + // Adds a _rowaddr column emulating each batch as a new fragment + fn add_row_addr(stream: SendableRecordBatchStream) -> SendableRecordBatchStream { + let schema = stream.schema(); + let schema_with_row_addr = Arc::new(Schema::new(vec![ + schema.field(0).clone(), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let schema = schema_with_row_addr.clone(); + let stream = stream.enumerate().map(move |(frag_id, batch)| { + let batch = batch.unwrap(); + let row_addr = Arc::new(UInt64Array::from_iter_values( + (0..batch.num_rows() as u64).map(|off| off + ((frag_id as u64) << 32)), + )); + Ok(RecordBatch::try_new( + schema_with_row_addr.clone(), + vec![batch.column(0).clone(), row_addr], + )?) + }); + Box::pin(RecordBatchStreamAdapter::new(schema, stream)) } - #[tokio::test] - async fn test_zonemap_null_handling_in_queries() { - // Test that zonemap index correctly returns null_list for queries + /// Build a single-column ZoneMap of primitive type `T` from `fragments` + /// (one batch -> one fragment), with small zones, then load it back. + async fn train_and_load( + fragments: Vec>>, + ) -> Arc + where + PrimitiveArray: From>>, + { let tmpdir = TempObjDir::default(); - let store = Arc::new(LanceIndexStore::new( + let test_store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), tmpdir.clone(), Arc::new(LanceCache::no_cache()), )); + let schema = Arc::new(Schema::new(vec![Field::new( + VALUE_COLUMN_NAME, + T::DATA_TYPE, + true, + )])); + let batches: Vec = fragments + .into_iter() + .map(|vals| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(PrimitiveArray::::from(vals))], + ) + .unwrap() + }) + .collect(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + stream::iter(batches.into_iter().map(Ok)), + )); + let stream = add_row_addr(stream); - // Create test data: [0, 5, null] - let batch = record_batch!( - (VALUE_COLUMN_NAME, Int64, [Some(0), Some(5), None]), - (ROW_ADDR, UInt64, [0, 1, 2]) + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(2)), ) + .await .unwrap(); - let schema = batch.schema(); - let stream = stream::once(async move { Ok(batch) }); - let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); - // Train and write the zonemap index - ZoneMapIndexPlugin::train_zonemap_index(stream, store.as_ref(), None) + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await - .unwrap(); + .expect("Failed to load ZoneMapIndex") + } - let cache = LanceCache::with_capacity(1024 * 1024); - let index = ZoneMapIndex::load(store.clone(), None, &cache) - .await - .unwrap(); + #[derive(Debug)] + struct TestRemapper { + mappings: HashMap, + } - // Test 1: Search for value 5 - zonemap should return at_most with all rows - // Since ZoneMap returns AtMost (superset), it's correct to include nulls in the result - let query = SargableQuery::Equals(ScalarValue::Int64(Some(5))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + impl TestRemapper { + fn new(mappings: impl IntoIterator) -> Self { + Self { + mappings: mappings.into_iter().collect(), + } + } + } - match result { - SearchResult::AtMost(row_ids) => { - // Zonemap can't determine exact matches, so it returns all rows in the zone - // This includes nulls because ZoneMap can't prove they don't match - let all_rows: Vec = row_ids - .true_rows() - .row_addrs() - .unwrap() - .map(u64::from) - .collect(); - assert_eq!( - all_rows, - vec![0, 1, 2], - "Should return all rows (including nulls) since ZoneMap is inexact" - ); + impl RowIdRemapper for TestRemapper { + fn remap_row_id(&self, row_id: u64) -> Option { + self.mappings.get(&row_id).copied() + } - // For AtMost results, nulls are included in the superset - // Downstream processing will handle null filtering - } - _ => panic!("Expected AtMost search result from zonemap"), + fn remap_row_addrs_tree_map(&self, _row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { + unreachable!() } - // Test 2: Range query - should also return all rows as AtMost - let query = SargableQuery::Range( - std::ops::Bound::Included(ScalarValue::Int64(Some(0))), - std::ops::Bound::Included(ScalarValue::Int64(Some(3))), - ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + fn remap_row_ids_roaring_tree_map( + &self, + _row_ids: &roaring::RoaringTreemap, + ) -> roaring::RoaringTreemap { + unreachable!() + } - match result { - SearchResult::AtMost(row_ids) => { - // Again, ZoneMap returns superset including nulls - let all_rows: Vec = row_ids - .true_rows() - .row_addrs() - .unwrap() - .map(u64::from) - .collect(); - assert_eq!( - all_rows, - vec![0, 1, 2], - "Should return all rows in zone as possible matches" - ); - } - _ => panic!("Expected AtMost search result from zonemap"), + fn remap_row_ids_record_batch( + &self, + _batch: RecordBatch, + _row_id_idx: usize, + ) -> lance_core::Result { + unreachable!() } } - #[tokio::test] - async fn test_nan_zonemap_index() { - let tmpdir = TempObjDir::default(); - let test_store = Arc::new(LanceIndexStore::new( - Arc::new(ObjectStore::local()), - tmpdir.clone(), - Arc::new(LanceCache::no_cache()), - )); + #[test] + fn test_remap_zone_splits_discontiguous_runs() { + let old_start = (2_u64 << 32) + 10; + let new_start = 3_u64 << 32; + let zone = ZoneMapStatistics { + min: ScalarValue::Int64(Some(10)), + max: ScalarValue::Int64(Some(40)), + null_count: 1, + nan_count: 0, + bound: ZoneBound { + fragment_id: 2, + start: 10, + length: 4, + }, + }; + let mut first_run = zone.clone(); + first_run.bound = ZoneBound { + fragment_id: 3, + start: 1, + length: 2, + }; + let mut second_run = zone.clone(); + second_run.bound = ZoneBound { + fragment_id: 3, + start: 7, + length: 2, + }; + second_run.null_count = 0; - // Create deterministic data with NaN values - // Pattern: [1.0, 2.0, NaN, 3.0, 4.0, 5.0, NaN, 6.0, 7.0, 8.0, ...] - let mut values = Vec::new(); - for i in 0..500 { - if i % 5 == 2 { - values.push(f32::NAN); - } else { - // Other values are sequential numbers - values.push(i as f32); - } - } + let remapper = TestRemapper::new([ + (old_start, new_start + 1), + (old_start + 1, new_start + 2), + (old_start + 2, new_start + 7), + (old_start + 3, new_start + 8), + ]); + let mut remapped_null_rows = RowAddrTreeMap::new(); + remapped_null_rows.insert(new_start + 1); - let float_data = arrow_array::Float32Array::from(values); - let row_ids = UInt64Array::from_iter_values((0..float_data.len()).map(|i| i as u64)); - let schema = Arc::new(Schema::new(vec![ - Field::new(VALUE_COLUMN_NAME, DataType::Float32, true), - Field::new(ROW_ADDR, DataType::UInt64, false), - ])); - let data = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(float_data.clone()), Arc::new(row_ids)], - ) - .unwrap(); - let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema, - stream::once(std::future::ready(Ok(data))), - )); + assert_eq!( + remap_zone(&zone, &remapper, Some(&remapped_null_rows), false).unwrap(), + vec![first_run, second_run] + ); + } - ZoneMapIndexPlugin::train_zonemap_index( - data_stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::new(100)), + #[tokio::test] + async fn test_remap_nested_mixed_null_zone_keeps_non_null_candidates() { + let index = train_and_load_fsl( + vec![ + vec![Some(0.0), Some(0.0)], + vec![Some(0.0), Some(0.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + ], + 2, ) - .await - .unwrap(); + .await; + let mut mixed_null_zone = index.zones[0].clone(); + mixed_null_zone.null_count = 2; + + let new_start = 3_u64 << 32; + let remapper = TestRemapper::new([(2, new_start), (3, new_start + 1)]); + let remapped_null_rows = RowAddrTreeMap::new(); + let zones = + remap_zone(&mixed_null_zone, &remapper, Some(&remapped_null_rows), true).unwrap(); + + assert_eq!(zones.len(), 1); + assert_eq!(zones[0].bound.length, 2); + assert_eq!(zones[0].null_count, 0); + assert!( + index + .evaluate_zone_against_query( + &zones[0], + &SargableQuery::Equals(fsl_scalar(vec![Some(3.0), Some(4.0)])), + ) + .unwrap(), + "the two surviving rows are non-null candidates" + ); + } - // Load the index - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); + #[test] + fn test_remap_nested_mixed_null_singleton_without_bitmap_is_rejected() { + let zone = ZoneMapStatistics { + min: ScalarValue::Null, + max: ScalarValue::Null, + null_count: 2, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 4, + }, + }; + let remapper = TestRemapper::new([(2, 0)]); - // Should have 5 zones since we have 500 rows and zone size is 100 - assert_eq!(index.zones.len(), 5); + let error = remap_zone(&zone, &remapper, None, true).unwrap_err(); + assert!(matches!(error, lance_core::Error::NotSupported { .. })); + assert!( + error + .to_string() + .contains("cannot safely remap a mixed-null ZoneMap zone") + ); + } - // Check that each zone has the expected NaN count - // Each zone has 100 values, and every 5th value (indices 2, 7, 12, ...) is NaN - // So each zone should have 20 NaN values (100/5 = 20) - for (i, zone) in index.zones.iter().enumerate() { - assert_eq!(zone.nan_count, 20, "Zone {} should have 20 NaN values", i); - assert_eq!( - zone.bound.length, 100, - "Zone {} should have zone_length 100", - i - ); - assert_eq!( - zone.bound.fragment_id, 0u64, - "Zone {} should have fragment_id 0", - i + #[tokio::test] + async fn test_merge_legacy_decimal_mixed_null_singleton_is_rejected() { + let mut source = train_and_load::(vec![vec![None, Some(200)]]).await; + let source_mut = Arc::get_mut(&mut source).unwrap(); + let zone = &mut source_mut.zones[0]; + zone.min = ScalarValue::Decimal128(None, 38, 10); + zone.max = ScalarValue::Decimal128(None, 38, 10); + source_mut.null_rows = None; + source_mut.fri = Some(Arc::new(TestRemapper::new([(1, 3_u64 << 32)]))); + + let candidate = ScalarValue::Decimal128(Some(200), 38, 10); + let queries = [ + SargableQuery::Equals(candidate.clone()), + SargableQuery::Range( + Bound::Included(candidate.clone()), + Bound::Included(candidate.clone()), + ), + SargableQuery::IsIn(vec![candidate]), + ]; + for query in queries { + assert!( + source + .evaluate_zone_against_query(&source.zones[0], &query) + .unwrap(), + "legacy Decimal source must retain the non-null candidate for {query:?}" ); } - let zone = &index.zones[0]; - assert!(matches!( - zone.max, - ScalarValue::Float32(Some(value)) if value.is_nan() + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), )); - let finite_target = ScalarValue::Float32(Some(1000.0)); + let error = merge_zonemap_indices( + &[source.as_ref()], + dest_store.as_ref(), + &RoaringBitmap::from_iter([3]), + ) + .await + .err() + .expect("ambiguous legacy Decimal singleton must reject consolidation"); + + assert!(matches!(error, lance_core::Error::NotSupported { .. })); assert!( - finite_target >= zone.min && finite_target <= zone.max, - "ScalarValue total ordering keeps finite values below NaN max" + error + .to_string() + .contains("cannot safely remap a mixed-null ZoneMap zone") ); + } - // Test search for NaN values using Equals with NaN - let query = SargableQuery::Equals(ScalarValue::Float32(Some(f32::NAN))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - - // Should match all zones since they all contain NaN values - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); // All rows since NaN is in every zone - assert_eq!(result, SearchResult::at_most(expected)); - - // Test search for a specific finite value that exists in the data - let query = SargableQuery::Equals(ScalarValue::Float32(Some(5.0))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - - // Should match only the first zone since 5.0 only exists in rows 0-99 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..100); - assert_eq!(result, SearchResult::at_most(expected)); - - // Test search for a value that doesn't exist - let query = SargableQuery::Equals(ScalarValue::Float32(Some(1000.0))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - - // Since zones contain NaN values, their max will be NaN, so they will be included - // as potential matches for any finite target (false positive, but acceptable for zone maps) - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); - - // Test range query that should include finite values - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Float32(Some(0.0))), - Bound::Included(ScalarValue::Float32(Some(250.0))), + #[tokio::test] + async fn test_value_range_spans_fragments() { + // Two fragments, multiple zones each; global min/max straddle both. + let index = train_and_load::(vec![ + vec![Some(10), Some(50), Some(30)], + vec![Some(5), Some(99), Some(42)], + ]) + .await; + assert_eq!( + index.value_range(), + Some((ScalarValue::Int64(Some(5)), ScalarValue::Int64(Some(99)))) ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - - // Should match the first three zones since they contain values in the range [0, 250] - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..300); - assert_eq!(result, SearchResult::at_most(expected)); - - // Test IsIn query with NaN and finite values - let query = SargableQuery::IsIn(vec![ - ScalarValue::Float32(Some(f32::NAN)), - ScalarValue::Float32(Some(5.0)), - ScalarValue::Float32(Some(150.0)), // This value exists in the second zone - ]); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - - // Should match all zones since they all contain NaN values - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); + } - // Test range query that excludes all values - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Float32(Some(1000.0))), - Bound::Included(ScalarValue::Float32(Some(2000.0))), - ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + #[tokio::test] + async fn test_value_range_all_null_is_none() { + let index = train_and_load::(vec![vec![None, None, None]]).await; + assert_eq!(index.value_range(), None); - // Since zones contain NaN values, their max will be NaN, so they will be included - // as potential matches for any range query (false positive, but acceptable for zone maps) - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); + let result = index + .search( + &SargableQuery::Equals(ScalarValue::Int64(Some(1))), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + } - // Test IsNull query (should match nothing since there are no null values) - let query = SargableQuery::IsNull(); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty())); + #[tokio::test] + async fn test_value_range_nan_max_is_none() { + // Zones of size 2: [1.0, 2.0] then [100.0, NaN]. The NaN-bearing zone hides + // its finite max (100.0), so the only sound answer is None. + let index = train_and_load::(vec![vec![ + Some(1.0), + Some(2.0), + Some(100.0), + Some(f32::NAN), + ]]) + .await; + assert_eq!(index.value_range(), None); + } - // Test range queries with NaN bounds - // Range with NaN as start bound (included) - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Float32(Some(f32::NAN))), - Bound::Unbounded, + #[tokio::test] + async fn test_value_range_over_folds_segments() { + // Two disjoint segments of one logical index; the global range straddles + // both (min and max from `b`), proving the fold spans segments. + let a = train_and_load::(vec![vec![Some(5), Some(9)]]).await; + let b = train_and_load::(vec![vec![Some(1), Some(20)]]).await; + assert_eq!( + ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]), + Some((ScalarValue::Int64(Some(1)), ScalarValue::Int64(Some(20)))) ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match all zones since they all contain NaN values - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); + } - // Range with NaN as end bound (included) - let query = SargableQuery::Range( - Bound::Unbounded, - Bound::Included(ScalarValue::Float32(Some(f32::NAN))), + #[tokio::test] + async fn test_value_range_over_nan_in_any_segment_is_none() { + // NaN in one segment hides that segment's finite max; the cross-segment + // fold must bail to None just as the single-segment path does. + let a = train_and_load::(vec![vec![Some(1.0), Some(2.0)]]).await; + let b = train_and_load::(vec![vec![Some(100.0), Some(f32::NAN)]]).await; + assert_eq!( + ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]), + None ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match all zones since they all contain NaN values - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); + } - // Range with NaN as end bound (excluded) - let query = SargableQuery::Range( - Bound::Unbounded, - Bound::Excluded(ScalarValue::Float32(Some(f32::NAN))), + #[tokio::test] + async fn test_value_range_over_missing_extrema() { + // An all-null segment yields no finite zone; folding it with a finite + // segment returns the finite segment's range (null contributes nothing). + let a = train_and_load::(vec![vec![None, None]]).await; + let b = train_and_load::(vec![vec![Some(3), Some(7)]]).await; + assert_eq!( + ZoneMapIndex::value_range_over([a.as_ref(), b.as_ref()]), + Some((ScalarValue::Int64(Some(3)), ScalarValue::Int64(Some(7)))) ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match all zones since everything is less than NaN - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); - // Range with NaN as start bound (excluded) - let query = SargableQuery::Range( - Bound::Excluded(ScalarValue::Float32(Some(f32::NAN))), - Bound::Unbounded, + // Lance v8 persisted typed-null Decimal extrema even when a zone contained + // values. Such unknown bounds cannot be skipped like an all-null segment. + let mut legacy = train_and_load::(vec![vec![Some(100), Some(200)]]).await; + for zone in &mut Arc::get_mut(&mut legacy).unwrap().zones { + zone.min = ScalarValue::Decimal128(None, 38, 10); + zone.max = ScalarValue::Decimal128(None, 38, 10); + } + let current = + train_and_load::(vec![vec![Some(10_000), Some(20_000)]]).await; + assert_eq!( + ZoneMapIndex::value_range_over([legacy.as_ref(), current.as_ref()]), + None ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match nothing since nothing is greater than NaN - assert_eq!(result, SearchResult::AtMost(NullableRowAddrSet::empty())); - - // Test IsIn query with mixed float types (Float16, Float32, Float64) - let query = SargableQuery::IsIn(vec![ - ScalarValue::Float16(Some(half::f16::NAN)), - ScalarValue::Float32(Some(f32::NAN)), - ScalarValue::Float64(Some(f64::NAN)), - ScalarValue::Float32(Some(5.0)), - ]); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match all zones since they all contain NaN values - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..500); - assert_eq!(result, SearchResult::at_most(expected)); } #[tokio::test] - // Test data that belongs to the same fragment but coming from different batches - async fn test_basic_zonemap_index() { + async fn test_empty_zonemap_index() { let tmpdir = TempObjDir::default(); let test_store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), @@ -1745,324 +2303,254 @@ mod tests { Arc::new(LanceCache::no_cache()), )); - let data = arrow_array::Int32Array::from_iter_values(0..=100); - let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64)); + let data = arrow_array::Int32Array::from(Vec::::new()); + let row_ids = arrow_array::UInt64Array::from(Vec::::new()); let schema = Arc::new(Schema::new(vec![ Field::new(VALUE_COLUMN_NAME, DataType::Int32, false), Field::new(ROW_ADDR, DataType::UInt64, false), ])); let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( schema, stream::once(std::future::ready(Ok(data))), )); + ZoneMapIndexPlugin::train_zonemap_index(data_stream, test_store.as_ref(), None) + .await + .unwrap(); + + log::debug!("Successfully wrote the index file"); + + // Read the index file back and check its contents + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); + assert_eq!(index.zones.len(), 0); + assert_eq!(index.data_type, DataType::Int32); + assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT); + + // Equals query: null (should match nothing, as there are no nulls) + let query = SargableQuery::Equals(ScalarValue::Int32(None)); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + } + + #[tokio::test] + // Test that a zonemap index can be created with null values from few fragments + async fn test_null_zonemap_index() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let stream = lance_datagen::gen_batch() + .col( + VALUE_COLUMN_NAME, + array::rand::().with_nulls(&[true, false, false, false, false]), + ) + .into_df_stream(RowCount::from(5000), BatchCount::from(10)); + + // Add _rowaddr column + let stream = add_row_addr(stream); + ZoneMapIndexPlugin::train_zonemap_index( - data_stream, + stream, test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::new(100)), + Some(ZoneMapIndexBuilderParams::new(5000)), ) .await .unwrap(); log::debug!("Successfully wrote the index file"); - // Read the raw index file back and check its contents - let index_file = test_store.open_index_file(ZONEMAP_FILENAME).await.unwrap(); - // Print the metadata from the index_file - let metadata = index_file.schema().metadata.clone(); - let record_batch = index_file - .read_record_batch(0, index_file.num_rows() as u64) - .await - .unwrap(); - assert_eq!(record_batch.num_rows(), 2); - assert_eq!( - record_batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .values(), - &[0, 100] - ); - assert_eq!( - record_batch - .column(1) - .as_any() - .downcast_ref::() - .unwrap() - .values(), - &[99, 100] - ); - assert_eq!( - record_batch - .column(2) - .as_any() - .downcast_ref::() - .unwrap() - .values(), - &[0, 0] - ); - assert_eq!(metadata.get(ZONEMAP_SIZE_META_KEY).unwrap(), "100"); - - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + // Read the index file back and check its contents + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 2); - assert_eq!( - index.zones, - vec![ - ZoneMapStatistics { - min: ScalarValue::Int32(Some(0)), - max: ScalarValue::Int32(Some(99)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int32(Some(100)), - max: ScalarValue::Int32(Some(100)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 100, - length: 1, - }, - } - ] - ); - // Verify nan_count is 0 for all zones (no NaN values in integer data) + assert_eq!(index.zones.len(), 10); for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.null_count, 1000); assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); + assert_eq!(zone.bound.length, 5000); + assert_eq!(zone.bound.fragment_id, i as u64); } - assert_eq!(index.data_type, DataType::Int32); - assert_eq!(index.rows_per_zone, 100); - assert_eq!( - index.calculate_included_frags().await.unwrap(), - RoaringBitmap::from_iter(0..1) - ); - - // Test search functionality - - // 1. Range query: (50, +inf) - let query = SargableQuery::Range( - Bound::Excluded(ScalarValue::Int32(Some(50))), - Bound::Unbounded, - ); + // Equals query: null (should match all zones since they contain null values) + let query = SargableQuery::Equals(ScalarValue::Int32(None)); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(0..=100)); - // 2. Range query: [0, 50] - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Int32(Some(0))), - Bound::Included(ScalarValue::Int32(Some(50))), - ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(0..=99)); + // Create expected RowAddrTreeMap with all zones since they contain null values + let mut expected = RowAddrTreeMap::new(); + for fragment_id in 0..10 { + let start = (fragment_id as u64) << 32; + let end = start + 5000; + expected.insert_range(start..end); + } + assert_eq!(result, SearchResult::at_most(expected)); - // 3. Range query: [101, 200] (should only match the second zone, which is row 100) - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Int32(Some(101))), - Bound::Included(ScalarValue::Int32(Some(200))), - ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Only row 100 is in the second zone, but its value is 100, so this should be empty - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + // Test update - add new data with Float32 values (matching the original data type) + let new_data = + arrow_array::Float32Array::from_iter_values((0..5000).map(|i| i as f32 / 1000.0)); + // Create row addresses for fragment 10 (next fragment after 0-9) + let new_row_addr = + UInt64Array::from_iter_values((0..5000).map(|i| (10u64 << 32) | (i as u64))); + let new_schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Float32, false), // Match original schema + Field::new(ROW_ADDR, DataType::UInt64, false), // Use _rowaddr as expected by the builder + ])); + let new_data_batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(new_data), Arc::new(new_row_addr)], + ) + .unwrap(); + let new_data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + new_schema, + stream::once(std::future::ready(Ok(new_data_batch))), + )); - // 4. Range query: [100, 100] (should match only the last row) - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Int32(Some(100))), - Bound::Included(ScalarValue::Int32(Some(100))), - ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(100..=100)); + // Directly pass the stream with proper row addresses instead of using MockTrainingSource + // which would regenerate row addresses starting from 0 + index + .update(new_data_stream, test_store.as_ref(), None) + .await + .unwrap(); - // 5. Equals query: 0 (should match first row) - let query = SargableQuery::Equals(ScalarValue::Int32(Some(0))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(0..=99)); + // Verify the updated index has more zones + let updated_index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load updated ZoneMapIndex"); - // 6. Equals query: 100 (should match only last row) - let query = SargableQuery::Equals(ScalarValue::Int32(Some(100))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(100..=100)); + // Should have original 10 zones + 1 new zone (5000 rows with zone size 5000) + assert_eq!(updated_index.zones.len(), 11); - // 7. Equals query: 101 (should match nothing) - let query = SargableQuery::Equals(ScalarValue::Int32(Some(101))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + // Verify the new zone was added + let new_zone = &updated_index.zones[10]; // Last zone should be the new one + assert_eq!(new_zone.bound.fragment_id, 10u64); // New fragment ID + assert_eq!(new_zone.bound.length, 5000); + assert_eq!(new_zone.null_count, 0); // New data has no nulls + assert_eq!(new_zone.nan_count, 0); // New data has no NaN values - // 8. IsNull query (no nulls in data, should match nothing) - let query = SargableQuery::IsNull(); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); - // 9. IsIn query: [0, 100, 101, 50] - let query = SargableQuery::IsIn(vec![ - ScalarValue::Int32(Some(0)), - ScalarValue::Int32(Some(100)), - ScalarValue::Int32(Some(101)), - ScalarValue::Int32(Some(50)), - ]); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // 0 and 50 are in the first zone, 100 in the second, 101 is not present - assert_eq!(result, SearchResult::at_most(0..=100)); + // Test search on updated index - search for null values should still work + let query = SargableQuery::Equals(ScalarValue::Float32(None)); + let result = updated_index + .search(&query, &NoOpMetricsCollector) + .await + .unwrap(); - // 10. IsIn query: [101, 102] (should match nothing) - let query = SargableQuery::IsIn(vec![ - ScalarValue::Int32(Some(101)), - ScalarValue::Int32(Some(102)), - ]); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + // Should match original 10 zones (with nulls) but not the new zone (no nulls) + let mut expected = RowAddrTreeMap::new(); + for fragment_id in 0..10 { + let start = (fragment_id as u64) << 32; + let end = start + 5000; + expected.insert_range(start..end); + } + assert_eq!(result, SearchResult::at_most(expected)); - // 11. IsIn query: [null] (should match nothing, as there are no nulls) - let query = SargableQuery::IsIn(vec![ScalarValue::Int32(None)]); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + // Test search for a value that should be in the new zone + let query = SargableQuery::Equals(ScalarValue::Float32(Some(2.5))); // Value 2500/1000 = 2.5 + let result = updated_index + .search(&query, &NoOpMetricsCollector) + .await + .unwrap(); - // 12. Equals query: null (should match nothing, as there are no nulls) - let query = SargableQuery::Equals(ScalarValue::Int32(None)); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + // Should match the new zone (fragment 10) + let mut expected = RowAddrTreeMap::new(); + let start = 10u64 << 32; + let end = start + 5000; + expected.insert_range(start..end); + assert_eq!(result, SearchResult::at_most(expected)); } #[tokio::test] - // Test zonemap with same fragment from multiple batches - async fn test_complex_zonemap_index() { + async fn test_zonemap_null_handling_in_queries() { + // Test that zonemap index correctly returns null_list for queries let tmpdir = TempObjDir::default(); - let test_store = Arc::new(LanceIndexStore::new( + let store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), tmpdir.clone(), Arc::new(LanceCache::no_cache()), )); - // Create data that will produce the expected zonemap zones - let data = - arrow_array::Int64Array::from_iter_values(0..(ROWS_PER_ZONE_DEFAULT * 2 + 42) as i64); - let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64)); - let schema = Arc::new(Schema::new(vec![ - Field::new(VALUE_COLUMN_NAME, DataType::Int64, false), - Field::new(ROW_ADDR, DataType::UInt64, false), - ])); - let data = - RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); - let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema, - stream::once(std::future::ready(Ok(data))), - )); - - ZoneMapIndexPlugin::train_zonemap_index( - data_stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::default()), + // Create test data: [0, 5, null] + let batch = record_batch!( + (VALUE_COLUMN_NAME, Int64, [Some(0), Some(5), None]), + (ROW_ADDR, UInt64, [0, 1, 2]) ) - .await .unwrap(); + let schema = batch.schema(); + let stream = stream::once(async move { Ok(batch) }); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); - log::debug!("Successfully wrote the index file"); + // Train and write the zonemap index + ZoneMapIndexPlugin::train_zonemap_index(stream, store.as_ref(), None) + .await + .unwrap(); - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) + let cache = LanceCache::with_capacity(1024 * 1024); + let index = ZoneMapIndex::load(store.clone(), None, &cache, false) .await - .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 3); - assert_eq!( - index.zones, - vec![ - ZoneMapStatistics { - min: ScalarValue::Int64(Some(0)), - max: ScalarValue::Int64(Some(8191)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 8192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(8192)), - max: ScalarValue::Int64(Some(16383)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 8192, - length: 8192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(16384)), - max: ScalarValue::Int64(Some(16425)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 16384, - length: 42, - }, - } - ] - ); - // Verify nan_count is 0 for all zones (no NaN values in integer data) - for (i, zone) in index.zones.iter().enumerate() { - assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); - } - - assert_eq!(index.data_type, DataType::Int64); - assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT); - assert_eq!( - index.calculate_included_frags().await.unwrap(), - RoaringBitmap::from_iter(0..1) - ); - - // TODO: Test search functionality - // Test search functionality + .unwrap(); - // Search for a value in the first zone - let query = SargableQuery::Equals(ScalarValue::Int64(Some(1000))); + // Test 1: Search for value 5 - zonemap should return at_most with all rows + // Since ZoneMap returns AtMost (superset), it's correct to include nulls in the result + let query = SargableQuery::Equals(ScalarValue::Int64(Some(5))); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match row 1000 in fragment 0: row address = (0 << 32) + 1000 = 1000 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..=8191); - assert_eq!(result, SearchResult::at_most(expected)); - // Search for a value in the second zone - let query = SargableQuery::Equals(ScalarValue::Int64(Some(9000))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match row 9000 in fragment 0: row address = (0 << 32) + 9000 = 9000 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(8192..=16383); - assert_eq!(result, SearchResult::at_most(expected)); + match result { + SearchResult::AtMost(row_ids) => { + // Zonemap can't determine exact matches, so it returns all rows in the zone + // This includes nulls because ZoneMap can't prove they don't match + let all_rows: Vec = row_ids + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .collect(); + assert_eq!( + all_rows, + vec![0, 1, 2], + "Should return all rows (including nulls) since ZoneMap is inexact" + ); - // Search for a value not present in any zone - let query = SargableQuery::Equals(ScalarValue::Int64(Some(20000))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + // For AtMost results, nulls are included in the superset + // Downstream processing will handle null filtering + } + _ => panic!("Expected AtMost search result from zonemap"), + } - // Search for a range that spans multiple zones + // Test 2: Range query - should also return all rows as AtMost let query = SargableQuery::Range( - Bound::Included(ScalarValue::Int64(Some(9000))), - Bound::Included(ScalarValue::Int64(Some(16400))), + std::ops::Bound::Included(ScalarValue::Int64(Some(0))), + std::ops::Bound::Included(ScalarValue::Int64(Some(3))), ); let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should match all rows from 8000 to 16400 (inclusive) - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(8192..=16425); - assert_eq!(result, SearchResult::at_most(expected)); + + match result { + SearchResult::AtMost(row_ids) => { + // Again, ZoneMap returns superset including nulls + let all_rows: Vec = row_ids + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .collect(); + assert_eq!( + all_rows, + vec![0, 1, 2], + "Should return all rows in zone as possible matches" + ); + } + _ => panic!("Expected AtMost search result from zonemap"), + } } #[tokio::test] - // Test zonemap with multiple fragments from different batches - async fn test_multiple_fragments_zonemap() { + async fn test_nan_zonemap_index() { let tmpdir = TempObjDir::default(); let test_store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), @@ -2070,737 +2558,2420 @@ mod tests { Arc::new(LanceCache::no_cache()), )); + // Create deterministic data with NaN values + // Pattern: [1.0, 2.0, NaN, 3.0, 4.0, 5.0, NaN, 6.0, 7.0, 8.0, ...] + let mut values = Vec::new(); + for i in 0..500 { + if i % 5 == 2 { + values.push(f32::NAN); + } else { + // Other values are sequential numbers + values.push(i as f32); + } + } + + let float_data = arrow_array::Float32Array::from(values); + let row_ids = UInt64Array::from_iter_values((0..float_data.len()).map(|i| i as u64)); let schema = Arc::new(Schema::new(vec![ - Field::new(VALUE_COLUMN_NAME, DataType::Int64, false), + Field::new(VALUE_COLUMN_NAME, DataType::Float32, true), Field::new(ROW_ADDR, DataType::UInt64, false), ])); - - // Create multiple fragments with data that will produce expected zones - // Fragment 0: values 0-8191 (first zone) - let fragment0_data = - arrow_array::Int64Array::from_iter_values(0..ROWS_PER_ZONE_DEFAULT as i64); - let fragment0_row_ids = UInt64Array::from_iter_values(0..ROWS_PER_ZONE_DEFAULT); - let fragment0_batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(fragment0_data), Arc::new(fragment0_row_ids)], - ) - .unwrap(); - - // Fragment 1: values 8192-16383 (second zone) - let fragment1_data = arrow_array::Int64Array::from_iter_values( - (ROWS_PER_ZONE_DEFAULT as i64)..((ROWS_PER_ZONE_DEFAULT * 2) as i64), - ); - let fragment1_row_ids = - UInt64Array::from_iter_values((0..ROWS_PER_ZONE_DEFAULT).map(|i| i + (1 << 32))); - let fragment1_batch = RecordBatch::try_new( + let data = RecordBatch::try_new( schema.clone(), - vec![Arc::new(fragment1_data), Arc::new(fragment1_row_ids)], + vec![Arc::new(float_data.clone()), Arc::new(row_ids)], ) .unwrap(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); - // Fragment 2: values 16384-16426 (third zone) - let fragment2_data = arrow_array::Int64Array::from_iter_values( - ((ROWS_PER_ZONE_DEFAULT * 2) as i64)..((ROWS_PER_ZONE_DEFAULT * 2 + 42) as i64), - ); - let fragment2_row_ids = - UInt64Array::from_iter_values((0..42).map(|i| (i as u64) + (2 << 32))); - let fragment2_batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(fragment2_data), Arc::new(fragment2_row_ids)], + ZoneMapIndexPlugin::train_zonemap_index( + data_stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(100)), ) + .await .unwrap(); - // Each fragment is broken into few batches - { - // Create a stream with multiple batches (fragments) - let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - stream::iter(vec![ - Ok(fragment0_batch.clone()), - Ok(fragment1_batch.clone()), - Ok(fragment2_batch.clone()), - ]), - )); - ZoneMapIndexPlugin::train_zonemap_index( - data_stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::new(5000)), - ) + // Load the index + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) .await - .unwrap(); + .expect("Failed to load ZoneMapIndex"); - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 5); + // Should have 5 zones since we have 500 rows and zone size is 100 + assert_eq!(index.zones.len(), 5); + + // Check that each zone has the expected NaN count + // Each zone has 100 values, and every 5th value (indices 2, 7, 12, ...) is NaN + // So each zone should have 20 NaN values (100/5 = 20) + for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.nan_count, 20, "Zone {} should have 20 NaN values", i); assert_eq!( - index.zones, - vec![ - ZoneMapStatistics { - min: ScalarValue::Int64(Some(0)), - max: ScalarValue::Int64(Some(4999)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 5000, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(5000)), - max: ScalarValue::Int64(Some(8191)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 5000, - length: 3192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(8192)), - max: ScalarValue::Int64(Some(13191)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 0, - length: 5000, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(13192)), - max: ScalarValue::Int64(Some(16383)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 5000, - length: 3192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(16384)), - max: ScalarValue::Int64(Some(16425)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 2, - start: 0, - length: 42, - }, - } - ] + zone.bound.length, 100, + "Zone {} should have zone_length 100", + i ); - // Verify nan_count is 0 for all zones (no NaN values in integer data) - for (i, zone) in index.zones.iter().enumerate() { - assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); - } - - assert_eq!(index.data_type, DataType::Int64); - assert_eq!(index.rows_per_zone, 5000); assert_eq!( - index.calculate_included_frags().await.unwrap(), - RoaringBitmap::from_iter(0..3) + zone.bound.fragment_id, 0u64, + "Zone {} should have fragment_id 0", + i ); + } - // Verify _rowaddr column values are properly assigned - let verify_data_stream: SendableRecordBatchStream = - Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - stream::iter(vec![ - Ok(fragment0_batch.clone()), - Ok(fragment1_batch.clone()), - Ok(fragment2_batch.clone()), - ]), - )); - let batches: Vec = verify_data_stream.try_collect().await.unwrap(); + let zone = &index.zones[0]; + assert!(matches!( + zone.max, + ScalarValue::Float32(Some(value)) if value.is_nan() + )); + let finite_target = ScalarValue::Float32(Some(1000.0)); + assert!( + finite_target >= zone.min && finite_target <= zone.max, + "ScalarValue total ordering keeps finite values below NaN max" + ); - assert_eq!(batches.len(), 3); + // Test search for NaN values using Equals with NaN + let query = SargableQuery::Equals(ScalarValue::Float32(Some(f32::NAN))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match all zones since they all contain NaN values + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); // All rows since NaN is in every zone + assert_eq!(result, SearchResult::at_most(expected)); + + // Test search for a specific finite value that exists in the data + let query = SargableQuery::Equals(ScalarValue::Float32(Some(5.0))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match only the first zone since 5.0 only exists in rows 0-99 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..100); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test search for a value that doesn't exist + let query = SargableQuery::Equals(ScalarValue::Float32(Some(1000.0))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Since zones contain NaN values, their max will be NaN, so they will be included + // as potential matches for any finite target (false positive, but acceptable for zone maps) + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test range query that should include finite values + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Float32(Some(0.0))), + Bound::Included(ScalarValue::Float32(Some(250.0))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match the first three zones since they contain values in the range [0, 250] + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..300); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test IsIn query with NaN and finite values + let query = SargableQuery::IsIn(vec![ + ScalarValue::Float32(Some(f32::NAN)), + ScalarValue::Float32(Some(5.0)), + ScalarValue::Float32(Some(150.0)), // This value exists in the second zone + ]); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match all zones since they all contain NaN values + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test range query that excludes all values + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Float32(Some(1000.0))), + Bound::Included(ScalarValue::Float32(Some(2000.0))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Since zones contain NaN values, their max will be NaN, so they will be included + // as potential matches for any range query (false positive, but acceptable for zone maps) + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test IsNull query (should match nothing since there are no null values) + let query = SargableQuery::IsNull(); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new())); + + // Test range queries with NaN bounds + // Range with NaN as start bound (included) + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Float32(Some(f32::NAN))), + Bound::Unbounded, + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match all zones since they all contain NaN values + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + + // Range with NaN as end bound (included) + let query = SargableQuery::Range( + Bound::Unbounded, + Bound::Included(ScalarValue::Float32(Some(f32::NAN))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match all zones since they all contain NaN values + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + + // Range with NaN as end bound (excluded) + let query = SargableQuery::Range( + Bound::Unbounded, + Bound::Excluded(ScalarValue::Float32(Some(f32::NAN))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match all zones since everything is less than NaN + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + + // Range with NaN as start bound (excluded) + let query = SargableQuery::Range( + Bound::Excluded(ScalarValue::Float32(Some(f32::NAN))), + Bound::Unbounded, + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match nothing since nothing is greater than NaN + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // Test IsIn query with mixed float types (Float16, Float32, Float64) + let query = SargableQuery::IsIn(vec![ + ScalarValue::Float16(Some(half::f16::NAN)), + ScalarValue::Float32(Some(f32::NAN)), + ScalarValue::Float64(Some(f64::NAN)), + ScalarValue::Float32(Some(5.0)), + ]); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match all zones since they all contain NaN values + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..500); + assert_eq!(result, SearchResult::at_most(expected)); + } + + #[tokio::test] + // Test data that belongs to the same fragment but coming from different batches + async fn test_basic_zonemap_index() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let data = arrow_array::Int32Array::from_iter_values(0..=100); + let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64)); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Int32, false), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let data = + RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + + ZoneMapIndexPlugin::train_zonemap_index( + data_stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(100)), + ) + .await + .unwrap(); + + log::debug!("Successfully wrote the index file"); + + // Read the raw index file back and check its contents + let index_file = test_store.open_index_file(ZONEMAP_FILENAME).await.unwrap(); + // Print the metadata from the index_file + let metadata = index_file.schema().metadata.clone(); + let record_batch = index_file + .read_record_batch(0, index_file.num_rows() as u64) + .await + .unwrap(); + assert_eq!(record_batch.num_rows(), 2); + assert_eq!( + record_batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[0, 100] + ); + assert_eq!( + record_batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[99, 100] + ); + assert_eq!( + record_batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap() + .values(), + &[0, 0] + ); + assert_eq!(metadata.get(ZONEMAP_SIZE_META_KEY).unwrap(), "100"); + + // Read the index file back and check its contents + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); + assert_eq!(index.zones.len(), 2); + assert_eq!( + index.zones, + vec![ + ZoneMapStatistics { + min: ScalarValue::Int32(Some(0)), + max: ScalarValue::Int32(Some(99)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int32(Some(100)), + max: ScalarValue::Int32(Some(100)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 100, + length: 1, + }, + } + ] + ); + // Verify nan_count is 0 for all zones (no NaN values in integer data) + for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); + } + + assert_eq!(index.data_type, DataType::Int32); + assert_eq!(index.rows_per_zone, 100); + assert_eq!( + index.calculate_included_frags().await.unwrap(), + RoaringBitmap::from_iter(0..1) + ); + + // Test search functionality + + // 1. Range query: (50, +inf) + let query = SargableQuery::Range( + Bound::Excluded(ScalarValue::Int32(Some(50))), + Bound::Unbounded, + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(0..=100)); + + // 2. Range query: [0, 50] + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Int32(Some(0))), + Bound::Included(ScalarValue::Int32(Some(50))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(0..=99)); + + // 3. Range query: [101, 200] (should only match the second zone, which is row 100) + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Int32(Some(101))), + Bound::Included(ScalarValue::Int32(Some(200))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Only row 100 is in the second zone, but its value is 100, so this should be empty + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // 4. Range query: [100, 100] (should match only the last row) + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Int32(Some(100))), + Bound::Included(ScalarValue::Int32(Some(100))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(100..=100)); + + // 5. Equals query: 0 (should match first row) + let query = SargableQuery::Equals(ScalarValue::Int32(Some(0))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(0..=99)); + + // 6. Equals query: 100 (should match only last row) + let query = SargableQuery::Equals(ScalarValue::Int32(Some(100))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(100..=100)); + + // 7. Equals query: 101 (should match nothing) + let query = SargableQuery::Equals(ScalarValue::Int32(Some(101))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // 8. IsNull query (no nulls in data, should match nothing) + let query = SargableQuery::IsNull(); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::exact(RowAddrTreeMap::new())); + // 9. IsIn query: [0, 100, 101, 50] + let query = SargableQuery::IsIn(vec![ + ScalarValue::Int32(Some(0)), + ScalarValue::Int32(Some(100)), + ScalarValue::Int32(Some(101)), + ScalarValue::Int32(Some(50)), + ]); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // 0 and 50 are in the first zone, 100 in the second, 101 is not present + assert_eq!(result, SearchResult::at_most(0..=100)); + + // 10. IsIn query: [101, 102] (should match nothing) + let query = SargableQuery::IsIn(vec![ + ScalarValue::Int32(Some(101)), + ScalarValue::Int32(Some(102)), + ]); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // 11. IsIn query: [null] (should match nothing, as there are no nulls) + let query = SargableQuery::IsIn(vec![ScalarValue::Int32(None)]); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // 12. Equals query: null (should match nothing, as there are no nulls) + let query = SargableQuery::Equals(ScalarValue::Int32(None)); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + } + + #[tokio::test] + // Test zonemap with same fragment from multiple batches + async fn test_complex_zonemap_index() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Create data that will produce the expected zonemap zones + let data = + arrow_array::Int64Array::from_iter_values(0..(ROWS_PER_ZONE_DEFAULT * 2 + 42) as i64); + let row_ids = UInt64Array::from_iter_values((0..data.len()).map(|i| i as u64)); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Int64, false), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let data = + RecordBatch::try_new(schema.clone(), vec![Arc::new(data), Arc::new(row_ids)]).unwrap(); + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(std::future::ready(Ok(data))), + )); + + ZoneMapIndexPlugin::train_zonemap_index( + data_stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::default()), + ) + .await + .unwrap(); + + log::debug!("Successfully wrote the index file"); + + // Read the index file back and check its contents + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); + assert_eq!(index.zones.len(), 3); + assert_eq!( + index.zones, + vec![ + ZoneMapStatistics { + min: ScalarValue::Int64(Some(0)), + max: ScalarValue::Int64(Some(8191)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 8192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(8192)), + max: ScalarValue::Int64(Some(16383)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 8192, + length: 8192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(16384)), + max: ScalarValue::Int64(Some(16425)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 16384, + length: 42, + }, + } + ] + ); + // Verify nan_count is 0 for all zones (no NaN values in integer data) + for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); + } + + assert_eq!(index.data_type, DataType::Int64); + assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT); + assert_eq!( + index.calculate_included_frags().await.unwrap(), + RoaringBitmap::from_iter(0..1) + ); + + // TODO: Test search functionality + // Test search functionality + + // Search for a value in the first zone + let query = SargableQuery::Equals(ScalarValue::Int64(Some(1000))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match row 1000 in fragment 0: row address = (0 << 32) + 1000 = 1000 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..=8191); + assert_eq!(result, SearchResult::at_most(expected)); + + // Search for a value in the second zone + let query = SargableQuery::Equals(ScalarValue::Int64(Some(9000))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match row 9000 in fragment 0: row address = (0 << 32) + 9000 = 9000 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(8192..=16383); + assert_eq!(result, SearchResult::at_most(expected)); + + // Search for a value not present in any zone + let query = SargableQuery::Equals(ScalarValue::Int64(Some(20000))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // Search for a range that spans multiple zones + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(9000))), + Bound::Included(ScalarValue::Int64(Some(16400))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should match all rows from 8000 to 16400 (inclusive) + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(8192..=16425); + assert_eq!(result, SearchResult::at_most(expected)); + } + + #[tokio::test] + // Test zonemap with multiple fragments from different batches + async fn test_multiple_fragments_zonemap() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, DataType::Int64, false), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // Create multiple fragments with data that will produce expected zones + // Fragment 0: values 0-8191 (first zone) + let fragment0_data = + arrow_array::Int64Array::from_iter_values(0..ROWS_PER_ZONE_DEFAULT as i64); + let fragment0_row_ids = UInt64Array::from_iter_values(0..ROWS_PER_ZONE_DEFAULT); + let fragment0_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(fragment0_data), Arc::new(fragment0_row_ids)], + ) + .unwrap(); + + // Fragment 1: values 8192-16383 (second zone) + let fragment1_data = arrow_array::Int64Array::from_iter_values( + (ROWS_PER_ZONE_DEFAULT as i64)..((ROWS_PER_ZONE_DEFAULT * 2) as i64), + ); + let fragment1_row_ids = + UInt64Array::from_iter_values((0..ROWS_PER_ZONE_DEFAULT).map(|i| i + (1 << 32))); + let fragment1_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(fragment1_data), Arc::new(fragment1_row_ids)], + ) + .unwrap(); + + // Fragment 2: values 16384-16426 (third zone) + let fragment2_data = arrow_array::Int64Array::from_iter_values( + ((ROWS_PER_ZONE_DEFAULT * 2) as i64)..((ROWS_PER_ZONE_DEFAULT * 2 + 42) as i64), + ); + let fragment2_row_ids = + UInt64Array::from_iter_values((0..42).map(|i| (i as u64) + (2 << 32))); + let fragment2_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(fragment2_data), Arc::new(fragment2_row_ids)], + ) + .unwrap(); + + // Each fragment is broken into few batches + { + // Create a stream with multiple batches (fragments) + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + stream::iter(vec![ + Ok(fragment0_batch.clone()), + Ok(fragment1_batch.clone()), + Ok(fragment2_batch.clone()), + ]), + )); + ZoneMapIndexPlugin::train_zonemap_index( + data_stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(5000)), + ) + .await + .unwrap(); + + // Read the index file back and check its contents + let index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); + assert_eq!(index.zones.len(), 5); + assert_eq!( + index.zones, + vec![ + ZoneMapStatistics { + min: ScalarValue::Int64(Some(0)), + max: ScalarValue::Int64(Some(4999)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 5000, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(5000)), + max: ScalarValue::Int64(Some(8191)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 5000, + length: 3192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(8192)), + max: ScalarValue::Int64(Some(13191)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 0, + length: 5000, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(13192)), + max: ScalarValue::Int64(Some(16383)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 5000, + length: 3192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(16384)), + max: ScalarValue::Int64(Some(16425)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 2, + start: 0, + length: 42, + }, + } + ] + ); + // Verify nan_count is 0 for all zones (no NaN values in integer data) + for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); + } + + assert_eq!(index.data_type, DataType::Int64); + assert_eq!(index.rows_per_zone, 5000); + assert_eq!( + index.calculate_included_frags().await.unwrap(), + RoaringBitmap::from_iter(0..3) + ); + + // Verify _rowaddr column values are properly assigned + let verify_data_stream: SendableRecordBatchStream = + Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + stream::iter(vec![ + Ok(fragment0_batch.clone()), + Ok(fragment1_batch.clone()), + Ok(fragment2_batch.clone()), + ]), + )); + let batches: Vec = verify_data_stream.try_collect().await.unwrap(); + + assert_eq!(batches.len(), 3); + + // Check fragment 0 _rowaddr values (should start from 0) + let fragment0_rowaddr_col = batches[0].column_by_name(ROW_ADDR).unwrap(); + let fragment0_rowaddrs = fragment0_rowaddr_col + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + fragment0_rowaddrs.values().len(), + ROWS_PER_ZONE_DEFAULT as usize + ); + assert_eq!(fragment0_rowaddrs.values()[0], 0); + assert_eq!( + fragment0_rowaddrs.values()[fragment0_rowaddrs.values().len() - 1], + 8191 + ); + + // Check fragment 1 _rowaddr values (should start from fragment_id=1) + let fragment1_rowaddr_col = batches[1].column_by_name(ROW_ADDR).unwrap(); + let fragment1_rowaddrs = fragment1_rowaddr_col + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + fragment1_rowaddrs.values().len(), + ROWS_PER_ZONE_DEFAULT as usize + ); + assert_eq!(fragment1_rowaddrs.values()[0], 1u64 << 32); // fragment_id=1, local_offset=0 + assert_eq!( + fragment1_rowaddrs.values()[fragment1_rowaddrs.values().len() - 1], + 8191 | (1u64 << 32) + ); + + // Check fragment 2 _rowaddr values (should start from fragment_id=2) + let fragment2_rowaddr_col = batches[2].column_by_name(ROW_ADDR).unwrap(); + let fragment2_rowaddrs = fragment2_rowaddr_col + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(fragment2_rowaddrs.values().len(), 42); + assert_eq!(fragment2_rowaddrs.values()[0], 2u64 << 32); // fragment_id=2, local_offset=0 + assert_eq!( + fragment2_rowaddrs.values()[fragment2_rowaddrs.values().len() - 1], + (2u64 << 32) | 41 + ); + + // Add a few tests for search functionality + + // Test range query that spans multiple fragments + let query = SargableQuery::Range( + Bound::Included(ScalarValue::Int64(Some(5000))), + Bound::Included(ScalarValue::Int64(Some(12000))), + ); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should include zones from fragments 0 and 1 since they overlap with range 5000-12000 + let mut expected = RowAddrTreeMap::new(); + // zone 1 + expected.insert_range(5000..8192); + // zone 2 + expected.insert_range((1u64 << 32)..((1u64 << 32) + 5000)); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test exact match query from zone 2 + let query = SargableQuery::Equals(ScalarValue::Int64(Some(8192))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should include zone 2 since it contains value 8192 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range((1u64 << 32)..((1u64 << 32) + 5000)); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test exact match query from zone 4 + let query = SargableQuery::Equals(ScalarValue::Int64(Some(16385))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Should include zone 4 since it contains value 16385 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(2u64 << 32..((2u64 << 32) + 42)); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test query that matches nothing + let query = SargableQuery::Equals(ScalarValue::Int64(Some(99999))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + + // Test is_in query + let query = SargableQuery::IsIn(vec![ScalarValue::Int64(Some(16385))]); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(2u64 << 32..((2u64 << 32) + 42)); + assert_eq!(result, SearchResult::at_most(expected)); + + // Test equals query with null + let query = SargableQuery::Equals(ScalarValue::Int64(None)); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..=16425); + // expected = {:?}", expected + assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + } + + // Each fragment is its own batch + { + // Create a stream with multiple batches (fragments) + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + stream::iter(vec![ + Ok(fragment0_batch.clone()), + Ok(fragment1_batch.clone()), + Ok(fragment2_batch.clone()), + ]), + )); + ZoneMapIndexPlugin::train_zonemap_index( + data_stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::default()), + ) + .await + .unwrap(); + + // Read the index file back and check its contents + let index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); + assert_eq!(index.zones.len(), 3); + assert_eq!( + index.zones, + vec![ + ZoneMapStatistics { + min: ScalarValue::Int64(Some(0)), + max: ScalarValue::Int64(Some(8191)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 8192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(8192)), + max: ScalarValue::Int64(Some(16383)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 0, + length: 8192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(16384)), + max: ScalarValue::Int64(Some(16425)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 2, + start: 0, + length: 42, + }, + } + ] + ); + // Verify nan_count is 0 for all zones (no NaN values in integer data) + for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); + } + + assert_eq!(index.data_type, DataType::Int64); + assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT); + assert_eq!( + index.calculate_included_frags().await.unwrap(), + RoaringBitmap::from_iter(0..3) + ); + } + + // All fragments are in the same batch + { + // Create a stream with multiple batches (fragments) + let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + stream::iter(vec![ + Ok(fragment0_batch.clone()), + Ok(fragment1_batch.clone()), + Ok(fragment2_batch.clone()), + ]), + )); + ZoneMapIndexPlugin::train_zonemap_index( + data_stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(ROWS_PER_ZONE_DEFAULT * 3)), + ) + .await + .unwrap(); + + // Read the index file back and check its contents + let index = + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("Failed to load ZoneMapIndex"); + assert_eq!(index.zones.len(), 3); + assert_eq!( + index.zones, + vec![ + ZoneMapStatistics { + min: ScalarValue::Int64(Some(0)), + max: ScalarValue::Int64(Some(8191)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 8192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(8192)), + max: ScalarValue::Int64(Some(16383)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 0, + length: 8192, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Int64(Some(16384)), + max: ScalarValue::Int64(Some(16425)), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 2, + start: 0, + length: 42, + }, + } + ] + ); + // Verify nan_count is 0 for all zones (no NaN values in integer data) + for (i, zone) in index.zones.iter().enumerate() { + assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); + } + + assert_eq!(index.data_type, DataType::Int64); + assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT * 3); + } + } + + #[tokio::test] + async fn test_fragment_id_assignment() { + // Test that fragment IDs are properly assigned in _rowaddr values + let schema = Arc::new(Schema::new(vec![Field::new( + VALUE_COLUMN_NAME, + DataType::Int32, + false, + )])); + + // Create multiple fragments + let fragment0_data = arrow_array::Int32Array::from_iter_values(0..5); + let fragment0_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(fragment0_data)]).unwrap(); + + let fragment1_data = arrow_array::Int32Array::from_iter_values(5..10); + let fragment1_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(fragment1_data)]).unwrap(); + + let aligned_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::iter(vec![Ok(fragment0_batch), Ok(fragment1_batch)]), + )); + + let aligned_stream = add_row_addr(aligned_stream); + + let batches: Vec = aligned_stream.try_collect().await.unwrap(); + + assert_eq!(batches.len(), 2); + + // Check fragment 0 _rowaddr values + let fragment0_rowaddr_col = batches[0].column_by_name(ROW_ADDR).unwrap(); + let fragment0_rowaddrs = fragment0_rowaddr_col + .as_any() + .downcast_ref::() + .unwrap(); + + // Fragment 0 should have _rowaddr values: 0, 1, 2, 3, 4 + assert_eq!(fragment0_rowaddrs.values(), &[0, 1, 2, 3, 4]); + + // Check fragment 1 _rowaddr values + let fragment1_rowaddr_col = batches[1].column_by_name(ROW_ADDR).unwrap(); + let fragment1_rowaddrs = fragment1_rowaddr_col + .as_any() + .downcast_ref::() + .unwrap(); + + // Fragment 1 should have _rowaddr values: (1 << 32) | 0, (1 << 32) | 1, etc. + // which is: 4294967296, 4294967297, 4294967298, 4294967299, 4294967300 + assert_eq!( + fragment1_rowaddrs.values(), + &[4294967296, 4294967297, 4294967298, 4294967299, 4294967300] + ); + } + + #[tokio::test] + async fn test_like_prefix_query() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Create zones with different string ranges + // Zone 0: ["aaa", "azz"] - should NOT match "foo%" + // Zone 1: ["bar", "baz"] - should NOT match "foo%" + // Zone 2: ["fa", "foz"] - should match "foo%" (contains potential matches) + // Zone 3: ["fop", "fzz"] - should NOT match "foo%" (all values >= "fop") + // Zone 4: ["foo", "foobar"] - should match "foo%" + // Zone 5: ["gaa", "gzz"] - should NOT match "foo%" + + let zones = vec![ + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("aaa".to_string())), + max: ScalarValue::Utf8(Some("azz".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("bar".to_string())), + max: ScalarValue::Utf8(Some("baz".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("fa".to_string())), + max: ScalarValue::Utf8(Some("foz".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 2, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("fop".to_string())), + max: ScalarValue::Utf8(Some("fzz".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 3, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("foo".to_string())), + max: ScalarValue::Utf8(Some("foobar".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 4, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("gaa".to_string())), + max: ScalarValue::Utf8(Some("gzz".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 5, + start: 0, + length: 100, + }, + }, + ]; + + let index = ZoneMapIndex { + zones, + data_type: DataType::Utf8, + rows_per_zone: ROWS_PER_ZONE_DEFAULT, + use_seeds: false, + store: test_store, + fri: None, + index_cache: WeakLanceCache::from(&LanceCache::no_cache()), + null_rows: None, + }; + + // Test LikePrefix query for "foo" + let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("foo".to_string()))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match zones 2 and 4 only + let mut expected = RowAddrTreeMap::new(); + // Zone 2: fragment 2 + expected.insert_range((2u64 << 32)..((2u64 << 32) + 100)); + // Zone 4: fragment 4 + expected.insert_range((4u64 << 32)..((4u64 << 32) + 100)); + + assert_eq!(result, SearchResult::at_most(expected)); + } + + #[tokio::test] + async fn test_like_prefix_edge_cases() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Test edge cases for LIKE prefix + let zones = vec![ + // Zone with values that contain the prefix exactly + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("test".to_string())), + max: ScalarValue::Utf8(Some("test".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 100, + }, + }, + // Zone with values that span across the prefix boundary + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("te".to_string())), + max: ScalarValue::Utf8(Some("tf".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 0, + length: 100, + }, + }, + // Zone completely before prefix + ZoneMapStatistics { + min: ScalarValue::Utf8(Some("abc".to_string())), + max: ScalarValue::Utf8(Some("def".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 2, + start: 0, + length: 100, + }, + }, + ]; + + let index = ZoneMapIndex { + zones, + data_type: DataType::Utf8, + rows_per_zone: ROWS_PER_ZONE_DEFAULT, + use_seeds: false, + store: test_store, + fri: None, + index_cache: WeakLanceCache::from(&LanceCache::no_cache()), + null_rows: None, + }; + + // Test LikePrefix "test" + let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("test".to_string()))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match zones 0 and 1 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..100); // Zone 0: fragment 0 + expected.insert_range((1u64 << 32)..((1u64 << 32) + 100)); + + assert_eq!(result, SearchResult::at_most(expected)); + + // Test empty prefix - should match all zones + let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("".to_string()))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + let mut expected = RowAddrTreeMap::new(); + expected.insert_range(0..100); // Zone 0: fragment 0 + expected.insert_range((1u64 << 32)..((1u64 << 32) + 100)); + expected.insert_range((2u64 << 32)..((2u64 << 32) + 100)); + + assert_eq!(result, SearchResult::at_most(expected)); + } + + #[tokio::test] + async fn test_like_prefix_large_utf8() { + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Test with LargeUtf8 type + let zones = vec![ + ZoneMapStatistics { + min: ScalarValue::LargeUtf8(Some("aaa".to_string())), + max: ScalarValue::LargeUtf8(Some("azz".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 0, + start: 0, + length: 100, + }, + }, + ZoneMapStatistics { + min: ScalarValue::LargeUtf8(Some("foo".to_string())), + max: ScalarValue::LargeUtf8(Some("foobar".to_string())), + null_count: 0, + nan_count: 0, + bound: ZoneBound { + fragment_id: 1, + start: 0, + length: 100, + }, + }, + ]; + + let index = ZoneMapIndex { + zones, + data_type: DataType::LargeUtf8, + rows_per_zone: ROWS_PER_ZONE_DEFAULT, + use_seeds: false, + store: test_store, + fri: None, + index_cache: WeakLanceCache::from(&LanceCache::no_cache()), + null_rows: None, + }; + + // Test LikePrefix with LargeUtf8 + let query = SargableQuery::LikePrefix(ScalarValue::LargeUtf8(Some("foo".to_string()))); + let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + + // Should match only zone 1 + let mut expected = RowAddrTreeMap::new(); + expected.insert_range((1u64 << 32)..((1u64 << 32) + 100)); + + assert_eq!(result, SearchResult::at_most(expected)); + } + + #[test] + fn test_compute_next_prefix() { + use super::compute_next_prefix; + + // Basic cases + assert_eq!(compute_next_prefix("foo"), Some("fop".to_string())); + assert_eq!(compute_next_prefix("abc"), Some("abd".to_string())); + assert_eq!(compute_next_prefix("a"), Some("b".to_string())); + assert_eq!(compute_next_prefix("z"), Some("{".to_string())); // 'z' + 1 = '{' + + // Edge case: prefix with 'z' at the end + assert_eq!(compute_next_prefix("abz"), Some("ab{".to_string())); + + // Edge case with tilde (~) which is 0x7E + assert_eq!(compute_next_prefix("ab~"), Some("ab\x7f".to_string())); + + // Empty prefix + assert_eq!(compute_next_prefix(""), None); + + // Non-ASCII: works correctly by incrementing Unicode code points + // é (U+00E9) -> ê (U+00EA) + assert_eq!(compute_next_prefix("café"), Some("cafê".to_string())); + // 中 (U+4E2D) -> 丮 (U+4E2E) + assert_eq!(compute_next_prefix("abc中"), Some("abc丮".to_string())); + // ÿ (U+00FF) -> Ā (U+0100) - crosses byte boundary but works + assert_eq!(compute_next_prefix("cafÿ"), Some("cafĀ".to_string())); + + // Edge case: character just before surrogate range + // U+D7FF -> U+E000 (skips surrogate range U+D800-U+DFFF) + assert_eq!( + compute_next_prefix("a\u{D7FF}"), + Some("a\u{E000}".to_string()) + ); + + // Edge case: max Unicode character U+10FFFF, falls back to previous char + assert_eq!(compute_next_prefix("ab\u{10FFFF}"), Some("ac".to_string())); + // All max characters + assert_eq!(compute_next_prefix("\u{10FFFF}\u{10FFFF}"), None); + } + + // When merging zone map segments, if ANY source segment has null_rows = None + // (legacy — null positions unknown), the merged result must also be None. + // The bug: any_null_bitmap is set to true as soon as one source has Some(...), + // and the None sources are silently skipped. The merged index then has + // null_rows = Some(partial_bitmap), so an IsNull search returns exact results + // that only cover the modern segment's nulls — a false negative for the legacy + // segment whose null positions were never tracked. + #[tokio::test] + async fn test_merge_with_legacy_none_segment_not_treated_as_no_nulls() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + use arrow_array::{Int32Array, UInt32Array}; + + // Index A: fragment 0, modern — has a complete null bitmap with 2 known null rows. + let schema_a = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch_a = RecordBatch::try_new( + schema_a, + vec![ + Arc::new(Int32Array::from(vec![Some(1i32)])) as _, + Arc::new(Int32Array::from(vec![Some(5i32)])) as _, + Arc::new(UInt32Array::from(vec![2u32])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let mut modern_null_rows = RowAddrTreeMap::new(); + modern_null_rows.insert(3); // frag 0 row 3 + modern_null_rows.insert(7); // frag 0 row 7 + let cache = LanceCache::no_cache(); + let index_a = Arc::new( + ZoneMapIndex::try_from_serialized( + batch_a, + store.clone(), + None, + &cache, + 10, + Some(modern_null_rows), // modern: complete bitmap + false, + ) + .unwrap(), + ); + + // Index B: fragment 1, legacy — null_rows = None despite null_count = 3. + let schema_b = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch_b = RecordBatch::try_new( + schema_b, + vec![ + Arc::new(Int32Array::from(vec![Some(10i32)])) as _, + Arc::new(Int32Array::from(vec![Some(20i32)])) as _, + Arc::new(UInt32Array::from(vec![3u32])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![1u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let index_b = Arc::new( + ZoneMapIndex::try_from_serialized( + batch_b, + store.clone(), + None, + &cache, + 10, + None, // legacy: null positions unknown + false, + ) + .unwrap(), + ); + + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + let all_frags = RoaringBitmap::from_iter([0u32, 1]); + merge_zonemap_indices( + &[index_a.as_ref(), index_b.as_ref()], + dest_store.as_ref(), + &all_frags, + ) + .await + .unwrap(); + + let merged = ZoneMapIndex::load(dest_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); + + // Index B had null_rows = None, so the merged index cannot know all null positions. + // IsNull must NOT return exact — that would be a false negative for fragment 1's nulls. + let result = merged + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + + // With the bug: any_null_bitmap=true (from A) → null_rows=Some(A's bitmap only) + // → IsNull returns exact, missing B's unknown nulls ← FALSE NEGATIVE + // With the fix: any_null_bitmap=false (because B is None) → null_rows=None + // → IsNull falls through to zone scan → AtMost + assert!( + !result.is_exact(), + "IsNull on a merged index where one source had null_rows=None must not return \ + exact; the legacy segment had null_count=3 so its nulls exist at unknown positions" + ); + } + + // Writes a zonemap file in the legacy format (no null bitmap global buffer), + // simulating an index created before the null bitmap feature was added. + async fn write_legacy_zonemap(store: &dyn IndexStore, null_count: u32) { + use arrow_array::{Int32Array, UInt32Array}; + let schema = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![Some(0)])) as _, + Arc::new(Int32Array::from(vec![Some(99)])) as _, + Arc::new(UInt32Array::from(vec![null_count])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![100u64])) as _, + ], + ) + .unwrap(); + let mut file_schema = schema.as_ref().clone(); + file_schema + .metadata + .insert(ZONEMAP_SIZE_META_KEY.to_string(), "8192".to_string()); + let mut writer = store + .new_index_file(ZONEMAP_FILENAME, Arc::new(file_schema)) + .await + .unwrap(); + writer.write_record_batch(batch).await.unwrap(); + writer.finish().await.unwrap(); + } + + #[tokio::test] + async fn test_legacy_zonemap_no_null_bitmap() { + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + + // Write a legacy index with one zone that has nulls but no null bitmap. + write_legacy_zonemap(store.as_ref(), 10).await; + + let index = ZoneMapIndex::load(store, None, &LanceCache::no_cache(), false) + .await + .expect("failed to load legacy zonemap"); + + assert!( + index.null_rows.is_none(), + "legacy index should have no null bitmap" + ); + + // IS NULL should fall back to the zone-scan path and return AtMost, not Exact. + let result = index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on a legacy index should not be exact" + ); + } + + #[tokio::test] + async fn test_zone_map_seed_writer_round_trip() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + use arrow_array::{ArrayRef, Int32Array}; + use datafusion_common::ScalarValue; + + let rows_per_zone = 4u64; + let data_type = DataType::Int32; + let mut writer = ZoneMapSeedWriter::new("test_col", rows_per_zone, data_type).unwrap(); + + // Batch 1: values 0..4 (fills exactly one zone) + let batch1: ArrayRef = Arc::new(Int32Array::from_iter_values(0..4)); + writer.observe_batch(&batch1).unwrap(); + + // Batch 2: values 10..14 (fills a second zone exactly) + let batch2: ArrayRef = Arc::new(Int32Array::from_iter_values(10..14)); + writer.observe_batch(&batch2).unwrap(); + + // Batch 3: values 20..22 (partial final zone) + let batch3: ArrayRef = Arc::new(Int32Array::from_iter_values(20..22)); + writer.observe_batch(&batch3).unwrap(); + + let bytes = writer.finish().unwrap().expect("should produce bytes"); + + // Check schema metadata key/value format + assert_eq!(writer.schema_metadata_key(), "lance.seed.test_col"); + let meta_val = writer.schema_metadata_value(3); + assert_eq!(meta_val, "3:4"); + + // Deserialize and verify + let (zones, null_bitmap) = + ZoneMapSeedWriter::deserialize_seed(42, &bytes, rows_per_zone).unwrap(); + assert_eq!(zones.len(), 3, "expected 3 zones"); + + // No nulls in the input, so null bitmap is empty + assert_eq!(null_bitmap, Some(RoaringBitmap::default())); + + // Zone 0: values 0..4 -> min=0, max=3 + assert_eq!(zones[0].bound.fragment_id, 42); + assert_eq!(zones[0].bound.start, 0); + assert_eq!(zones[0].min, ScalarValue::Int32(Some(0))); + assert_eq!(zones[0].max, ScalarValue::Int32(Some(3))); + assert_eq!(zones[0].null_count, 0); + + // Zone 1: values 10..14 -> min=10, max=13 + assert_eq!(zones[1].bound.start, 4); + assert_eq!(zones[1].min, ScalarValue::Int32(Some(10))); + assert_eq!(zones[1].max, ScalarValue::Int32(Some(13))); + + // Zone 2: values 20..22 -> min=20, max=21, partial zone of 2 rows + assert_eq!(zones[2].bound.start, 8); + assert_eq!( + zones[2].bound.length, 2, + "partial zone length must be exact" + ); + assert_eq!(zones[2].min, ScalarValue::Int32(Some(20))); + assert_eq!(zones[2].max, ScalarValue::Int32(Some(21))); - // Check fragment 0 _rowaddr values (should start from 0) - let fragment0_rowaddr_col = batches[0].column_by_name(ROW_ADDR).unwrap(); - let fragment0_rowaddrs = fragment0_rowaddr_col - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!( - fragment0_rowaddrs.values().len(), - ROWS_PER_ZONE_DEFAULT as usize - ); - assert_eq!(fragment0_rowaddrs.values()[0], 0); - assert_eq!( - fragment0_rowaddrs.values()[fragment0_rowaddrs.values().len() - 1], - 8191 - ); + // Full zones must have the full rows_per_zone length + assert_eq!(zones[0].bound.length, rows_per_zone as usize); + assert_eq!(zones[1].bound.length, rows_per_zone as usize); + } - // Check fragment 1 _rowaddr values (should start from fragment_id=1) - let fragment1_rowaddr_col = batches[1].column_by_name(ROW_ADDR).unwrap(); - let fragment1_rowaddrs = fragment1_rowaddr_col - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!( - fragment1_rowaddrs.values().len(), - ROWS_PER_ZONE_DEFAULT as usize - ); - assert_eq!(fragment1_rowaddrs.values()[0], 1u64 << 32); // fragment_id=1, local_offset=0 - assert_eq!( - fragment1_rowaddrs.values()[fragment1_rowaddrs.values().len() - 1], - 8191 | (1u64 << 32) - ); + #[tokio::test] + async fn test_zone_map_seed_writer_spanning_batches() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + use arrow_array::{ArrayRef, Int32Array}; + use datafusion_common::ScalarValue; + + let rows_per_zone = 5u64; + let data_type = DataType::Int32; + let mut writer = ZoneMapSeedWriter::new("val", rows_per_zone, data_type).unwrap(); + + // Single batch with 12 values -> should produce 2 complete zones + 1 partial + let batch: ArrayRef = Arc::new(Int32Array::from_iter_values(0..12)); + writer.observe_batch(&batch).unwrap(); + + let bytes = writer.finish().unwrap().expect("should produce bytes"); + let (zones, _null_bitmap) = + ZoneMapSeedWriter::deserialize_seed(1, &bytes, rows_per_zone).unwrap(); + assert_eq!( + zones.len(), + 3, + "expected 3 zones from 12 rows with zone size 5" + ); - // Check fragment 2 _rowaddr values (should start from fragment_id=2) - let fragment2_rowaddr_col = batches[2].column_by_name(ROW_ADDR).unwrap(); - let fragment2_rowaddrs = fragment2_rowaddr_col - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(fragment2_rowaddrs.values().len(), 42); - assert_eq!(fragment2_rowaddrs.values()[0], 2u64 << 32); // fragment_id=2, local_offset=0 - assert_eq!( - fragment2_rowaddrs.values()[fragment2_rowaddrs.values().len() - 1], - (2u64 << 32) | 41 - ); + // Zone 0: rows 0..5 + assert_eq!(zones[0].bound.length, 5); + assert_eq!(zones[0].min, ScalarValue::Int32(Some(0))); + assert_eq!(zones[0].max, ScalarValue::Int32(Some(4))); + // Zone 1: rows 5..10 + assert_eq!(zones[1].bound.length, 5); + assert_eq!(zones[1].min, ScalarValue::Int32(Some(5))); + assert_eq!(zones[1].max, ScalarValue::Int32(Some(9))); + // Zone 2: rows 10..12 (partial) + assert_eq!(zones[2].bound.length, 2, "partial zone length must be 2"); + assert_eq!(zones[2].min, ScalarValue::Int32(Some(10))); + assert_eq!(zones[2].max, ScalarValue::Int32(Some(11))); + } - // Add a few tests for search functionality + #[tokio::test] + async fn test_zone_map_seed_writer_empty() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; - // Test range query that spans multiple fragments - let query = SargableQuery::Range( - Bound::Included(ScalarValue::Int64(Some(5000))), - Bound::Included(ScalarValue::Int64(Some(12000))), - ); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should include zones from fragments 0 and 1 since they overlap with range 5000-12000 - let mut expected = RowAddrTreeMap::new(); - // zone 1 - expected.insert_range(5000..8192); - // zone 2 - expected.insert_range((1u64 << 32)..((1u64 << 32) + 5000)); - assert_eq!(result, SearchResult::at_most(expected)); + let mut writer = ZoneMapSeedWriter::new("col", 8, DataType::Int32).unwrap(); + let result = writer.finish().unwrap(); + assert!(result.is_none(), "empty fragment should return None"); + } - // Test exact match query from zone 2 - let query = SargableQuery::Equals(ScalarValue::Int64(Some(8192))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should include zone 2 since it contains value 8192 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range((1u64 << 32)..((1u64 << 32) + 5000)); - assert_eq!(result, SearchResult::at_most(expected)); + /// Seeds produced for fragments that contain null values must carry a null + /// bitmap so that `try_update_with_seeds` can reconstruct `null_rows` without + /// rescanning data. + #[tokio::test] + async fn test_seed_null_bitmap_round_trip() { + use crate::scalar::seed::IndexSeedWriter; + use crate::scalar::zonemap::ZoneMapSeedWriter; + use arrow_array::Int32Array; + use lance_select::RowSetOps; + + // rows_per_zone = 4, 10 rows: nulls at positions 1, 3, 5, 9 + // zone 0: rows 0-3 → nulls at 1, 3 + // zone 1: rows 4-7 → null at 5 + // zone 2: rows 8-9 → null at 9 + let rows_per_zone = 4u64; + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(0), + None, // offset 1 + Some(2), + None, // offset 3 + Some(4), + None, // offset 5 + Some(6), + Some(7), + Some(8), + None, // offset 9 + ])); - // Test exact match query from zone 4 - let query = SargableQuery::Equals(ScalarValue::Int64(Some(16385))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - // Should include zone 4 since it contains value 16385 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(2u64 << 32..((2u64 << 32) + 42)); - assert_eq!(result, SearchResult::at_most(expected)); + let mut writer = ZoneMapSeedWriter::new("col", rows_per_zone, DataType::Int32).unwrap(); + writer.observe_batch(&values).unwrap(); + let bytes = writer.finish().unwrap().expect("should produce bytes"); + + let (zones, null_bitmap) = + ZoneMapSeedWriter::deserialize_seed(7, &bytes, rows_per_zone).unwrap(); + assert_eq!(zones.len(), 3); + assert_eq!(zones[0].null_count, 2); + assert_eq!(zones[1].null_count, 1); + assert_eq!(zones[2].null_count, 1); + + let bitmap = null_bitmap.expect("null bitmap must be present"); + assert!(bitmap.contains(1), "offset 1 must be in bitmap"); + assert!(bitmap.contains(3), "offset 3 must be in bitmap"); + assert!(bitmap.contains(5), "offset 5 must be in bitmap"); + assert!(bitmap.contains(9), "offset 9 must be in bitmap"); + assert_eq!(bitmap.len(), 4, "exactly 4 null positions"); + + // Reconstruct a RowAddrTreeMap as try_update_with_seeds would do. + let mut null_rows = RowAddrTreeMap::new(); + null_rows.insert_bitmap(7, bitmap); + // Verify the row addresses are (fragment 7 << 32 | offset). + let frag7 = 7u64 << 32; + assert!(null_rows.contains(frag7 | 1)); + assert!(null_rows.contains(frag7 | 3)); + assert!(null_rows.contains(frag7 | 5)); + assert!(null_rows.contains(frag7 | 9)); + assert!(!null_rows.contains(frag7)); + } - // Test query that matches nothing - let query = SargableQuery::Equals(ScalarValue::Int64(Some(99999))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); + /// After updating a zone map index via seeds from a fragment that contains + /// nulls, IS NULL must return an exact result (not a zone-scan approximation). + #[tokio::test] + async fn test_update_with_seeds_propagates_null_bitmap() { + use crate::scalar::seed::{FragmentSeed, IndexSeedWriter}; + use crate::scalar::zonemap::{ZoneMapIndexBuilder, ZoneMapSeedWriter}; + use arrow_array::Int32Array; + use lance_select::RowSetOps; - // Test is_in query - let query = SargableQuery::IsIn(vec![ScalarValue::Int64(Some(16385))]); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(2u64 << 32..((2u64 << 32) + 42)); - assert_eq!(result, SearchResult::at_most(expected)); + // Build an initial zone map index with one fragment that has nulls at rows 0 and 2. + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); - // Test equals query with null - let query = SargableQuery::Equals(ScalarValue::Int64(None)); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..=16425); - // expected = {:?}", expected - assert_eq!(result, SearchResult::at_most(RowAddrTreeMap::new())); - } + // Initial training: fragment 0, rows 0-3, nulls at 0 and 2. + let rows_per_zone = 4u64; + let initial_values: ArrayRef = Arc::new(Int32Array::from(vec![ + None, // frag 0, row 0 + Some(1), + None, // frag 0, row 2 + Some(3), + ])); + let mut builder = ZoneMapIndexBuilder::try_new( + ZoneMapIndexBuilderParams::new(rows_per_zone), + DataType::Int32, + ) + .unwrap(); + let row_addrs: ArrayRef = Arc::new(UInt64Array::from(vec![0u64, 1, 2, 3])); + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("value", DataType::Int32, true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new(schema.clone(), vec![initial_values, row_addrs]).unwrap(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::once(async { Ok(batch) }), + )); + builder.train(stream).await.unwrap(); + builder.write_index(store.as_ref()).await.unwrap(); - // Each fragment is its own batch - { - // Create a stream with multiple batches (fragments) - let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - stream::iter(vec![ - Ok(fragment0_batch.clone()), - Ok(fragment1_batch.clone()), - Ok(fragment2_batch.clone()), - ]), - )); - ZoneMapIndexPlugin::train_zonemap_index( - data_stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::default()), - ) + // Load the trained index. + let index = ZoneMapIndex::load(store.clone(), None, &LanceCache::no_cache(), true) .await .unwrap(); + assert!( + index.null_rows.is_some(), + "initial index must have null bitmap" + ); - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 3); - assert_eq!( - index.zones, - vec![ - ZoneMapStatistics { - min: ScalarValue::Int64(Some(0)), - max: ScalarValue::Int64(Some(8191)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 8192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(8192)), - max: ScalarValue::Int64(Some(16383)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 0, - length: 8192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(16384)), - max: ScalarValue::Int64(Some(16425)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 2, - start: 0, - length: 42, - }, - } - ] - ); - // Verify nan_count is 0 for all zones (no NaN values in integer data) - for (i, zone) in index.zones.iter().enumerate() { - assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); - } + // Build a seed for fragment 1: rows 0-3, null at row 1. + let seed_values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(10), + None, // frag 1, row 1 + Some(12), + Some(13), + ])); + let mut seed_writer = + ZoneMapSeedWriter::new("value", rows_per_zone, DataType::Int32).unwrap(); + seed_writer.observe_batch(&seed_values).unwrap(); + let seed_bytes = seed_writer + .finish() + .unwrap() + .expect("seed must produce bytes"); - assert_eq!(index.data_type, DataType::Int64); - assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT); - assert_eq!( - index.calculate_included_frags().await.unwrap(), - RoaringBitmap::from_iter(0..3) - ); - } + let seed = FragmentSeed { + fragment_id: 1, + bytes: seed_bytes, + metadata_value: format!("0:{}", rows_per_zone), + }; - // All fragments are in the same batch - { - // Create a stream with multiple batches (fragments) - let data_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - stream::iter(vec![ - Ok(fragment0_batch.clone()), - Ok(fragment1_batch.clone()), - Ok(fragment2_batch.clone()), - ]), - )); - ZoneMapIndexPlugin::train_zonemap_index( - data_stream, - test_store.as_ref(), - Some(ZoneMapIndexBuilderParams::new(ROWS_PER_ZONE_DEFAULT * 3)), - ) + // Update the index with the seed. + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + index + .try_update_with_seeds(&[seed], dest_store.as_ref()) + .await + .unwrap() + .expect("update must produce a result"); + + let updated_index = ZoneMapIndex::load(dest_store, None, &LanceCache::no_cache(), true) .await .unwrap(); - // Read the index file back and check its contents - let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache()) - .await - .expect("Failed to load ZoneMapIndex"); - assert_eq!(index.zones.len(), 3); - assert_eq!( - index.zones, - vec![ - ZoneMapStatistics { - min: ScalarValue::Int64(Some(0)), - max: ScalarValue::Int64(Some(8191)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 8192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(8192)), - max: ScalarValue::Int64(Some(16383)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 0, - length: 8192, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Int64(Some(16384)), - max: ScalarValue::Int64(Some(16425)), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 2, - start: 0, - length: 42, - }, - } - ] - ); - // Verify nan_count is 0 for all zones (no NaN values in integer data) - for (i, zone) in index.zones.iter().enumerate() { - assert_eq!(zone.nan_count, 0, "Zone {} should have nan_count = 0", i); - } + // The updated index must have a null bitmap covering both fragments. + let null_rows = updated_index + .null_rows + .as_ref() + .expect("updated index must have null bitmap"); + + // Fragment 0: nulls at offsets 0 and 2 + assert!(null_rows.contains(0u64), "frag 0 row 0 must be null"); + assert!(null_rows.contains(2u64), "frag 0 row 2 must be null"); + assert!(!null_rows.contains(1u64), "frag 0 row 1 must not be null"); - assert_eq!(index.data_type, DataType::Int64); - assert_eq!(index.rows_per_zone, ROWS_PER_ZONE_DEFAULT * 3); - } + // Fragment 1: null at offset 1 + let frag1 = 1u64 << 32; + assert!(null_rows.contains(frag1 | 1), "frag 1 row 1 must be null"); + assert!(!null_rows.contains(frag1), "frag 1 row 0 must not be null"); + + // IS NULL must return exact results (not a zone-scan approximation). + let result = updated_index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!(result.is_exact(), "IS NULL after seed update must be exact"); } + /// A legacy index (null_rows = None) updated with a seed must still have + /// null_rows = None in the result. The seed cannot retroactively supply null + /// positions for the rows already in the old index, so the updated index must + /// stay conservative and fall back to zone-scan for IS NULL. #[tokio::test] - async fn test_fragment_id_assignment() { - // Test that fragment IDs are properly assigned in _rowaddr values - let schema = Arc::new(Schema::new(vec![Field::new( - VALUE_COLUMN_NAME, - DataType::Int32, - false, - )])); - - // Create multiple fragments - let fragment0_data = arrow_array::Int32Array::from_iter_values(0..5); - let fragment0_batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(fragment0_data)]).unwrap(); + async fn test_update_with_seeds_legacy_index_keeps_null_rows_none() { + use crate::scalar::seed::{FragmentSeed, IndexSeedWriter}; + use crate::scalar::zonemap::{ZoneMapIndex, ZoneMapSeedWriter}; + use arrow_array::{Int32Array, UInt32Array}; - let fragment1_data = arrow_array::Int32Array::from_iter_values(5..10); - let fragment1_batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(fragment1_data)]).unwrap(); + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); - let aligned_stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + // Build a legacy index (fragment 0, null_rows = None). + let schema = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( schema, - stream::iter(vec![Ok(fragment0_batch), Ok(fragment1_batch)]), - )); + vec![ + Arc::new(Int32Array::from(vec![Some(0i32)])) as _, + Arc::new(Int32Array::from(vec![Some(9i32)])) as _, + Arc::new(UInt32Array::from(vec![2u32])) as _, // 2 nulls, positions unknown + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let cache = LanceCache::no_cache(); + let legacy_index = Arc::new( + ZoneMapIndex::try_from_serialized( + batch, + store.clone(), + None, + &cache, + 10, + None, // legacy: null positions unknown + true, + ) + .unwrap(), + ); + assert!(legacy_index.null_rows.is_none()); - let aligned_stream = add_row_addr(aligned_stream); + // Build a seed for fragment 1 with known null positions. + let rows_per_zone = 10u64; + let seed_values: ArrayRef = Arc::new(Int32Array::from(vec![ + None, // frag 1, row 0 + Some(1), + ])); + let mut seed_writer = + ZoneMapSeedWriter::new("value", rows_per_zone, DataType::Int32).unwrap(); + seed_writer.observe_batch(&seed_values).unwrap(); + let seed_bytes = seed_writer + .finish() + .unwrap() + .expect("seed must produce bytes"); - let batches: Vec = aligned_stream.try_collect().await.unwrap(); + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + legacy_index + .try_update_with_seeds( + &[FragmentSeed { + fragment_id: 1, + bytes: seed_bytes, + metadata_value: format!("0:{}", rows_per_zone), + }], + dest_store.as_ref(), + ) + .await + .unwrap() + .expect("update must produce a result"); - assert_eq!(batches.len(), 2); + let updated = ZoneMapIndex::load(dest_store, None, &LanceCache::no_cache(), true) + .await + .unwrap(); - // Check fragment 0 _rowaddr values - let fragment0_rowaddr_col = batches[0].column_by_name(ROW_ADDR).unwrap(); - let fragment0_rowaddrs = fragment0_rowaddr_col - .as_any() - .downcast_ref::() + assert!( + updated.null_rows.is_none(), + "updating a legacy index must not fabricate a null bitmap: \ + the old fragment's null positions were never tracked" + ); + + let result = updated + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on an index derived from a legacy source must use zone scan, not exact lookup" + ); + } - // Fragment 0 should have _rowaddr values: 0, 1, 2, 3, 4 - assert_eq!(fragment0_rowaddrs.values(), &[0, 1, 2, 3, 4]); + /// Merging a modern index (null_rows = Some) with a legacy index (null_rows = None) + /// must produce a merged index with null_rows = None. Returning the modern + /// fragment's bitmap as-is would be a false negative for the legacy fragment. + #[tokio::test] + async fn test_merge_modern_and_legacy_index_drops_null_bitmap() { + use arrow_array::{Int32Array, UInt32Array}; - // Check fragment 1 _rowaddr values - let fragment1_rowaddr_col = batches[1].column_by_name(ROW_ADDR).unwrap(); - let fragment1_rowaddrs = fragment1_rowaddr_col - .as_any() - .downcast_ref::() + let tmpdir = TempObjDir::default(); + let store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let cache = LanceCache::no_cache(); + + // Modern index: fragment 0, null_rows = Some(bitmap with 1 known null). + let schema = Arc::new(Schema::new(vec![ + Field::new("min", DataType::Int32, true), + Field::new("max", DataType::Int32, true), + Field::new("null_count", DataType::UInt32, false), + Field::new("nan_count", DataType::UInt32, false), + Field::new("fragment_id", DataType::UInt64, false), + Field::new("zone_start", DataType::UInt64, false), + Field::new("zone_length", DataType::UInt64, false), + ])); + let modern_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![Some(1i32)])) as _, + Arc::new(Int32Array::from(vec![Some(5i32)])) as _, + Arc::new(UInt32Array::from(vec![1u32])) as _, + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let mut modern_null_rows = RowAddrTreeMap::new(); + modern_null_rows.insert(2u64); // frag 0, row 2 + let modern_index = Arc::new( + ZoneMapIndex::try_from_serialized( + modern_batch, + store.clone(), + None, + &cache, + 10, + Some(modern_null_rows), + false, + ) + .unwrap(), + ); + + // Legacy index: fragment 1, null_rows = None. + let legacy_batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![Some(10i32)])) as _, + Arc::new(Int32Array::from(vec![Some(20i32)])) as _, + Arc::new(UInt32Array::from(vec![3u32])) as _, // 3 nulls, positions unknown + Arc::new(UInt32Array::from(vec![0u32])) as _, + Arc::new(UInt64Array::from(vec![1u64])) as _, + Arc::new(UInt64Array::from(vec![0u64])) as _, + Arc::new(UInt64Array::from(vec![10u64])) as _, + ], + ) + .unwrap(); + let legacy_index = Arc::new( + ZoneMapIndex::try_from_serialized( + legacy_batch, + store.clone(), + None, + &cache, + 10, + None, // legacy: null positions unknown + false, + ) + .unwrap(), + ); + + let dest_tmpdir = TempObjDir::default(); + let dest_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + dest_tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let all_frags = RoaringBitmap::from_iter([0u32, 1]); + merge_zonemap_indices( + &[modern_index.as_ref(), legacy_index.as_ref()], + dest_store.as_ref(), + &all_frags, + ) + .await + .unwrap(); + + let merged = ZoneMapIndex::load(dest_store, None, &LanceCache::no_cache(), false) + .await .unwrap(); - // Fragment 1 should have _rowaddr values: (1 << 32) | 0, (1 << 32) | 1, etc. - // which is: 4294967296, 4294967297, 4294967298, 4294967299, 4294967300 - assert_eq!( - fragment1_rowaddrs.values(), - &[4294967296, 4294967297, 4294967298, 4294967299, 4294967300] + assert!( + merged.null_rows.is_none(), + "merging a modern index with a legacy index must drop the null bitmap: \ + the legacy fragment's null positions are unknown" + ); + + let result = merged + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + assert!( + !result.is_exact(), + "IS NULL on a merged index with a legacy source must use zone scan, not exact lookup" ); } - #[tokio::test] - async fn test_like_prefix_query() { + // ───────────────────────────── Nested type zone map tests ───────────────────────────── + + use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array}; + + /// Build a FixedSizeList zone map from `rows` (each row is a + /// `Vec>` of length `list_size`), then load and return the index. + async fn train_and_load_fsl(rows: Vec>>, list_size: i32) -> Arc { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), list_size); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + let mut fsl_builder = + arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), list_size); + for row in &rows { + assert_eq!(row.len(), list_size as usize); + for &v in row { + match v { + Some(f) => fsl_builder.values().append_value(f), + None => fsl_builder.values().append_null(), + } + } + fsl_builder.append(true); + } + let fsl_arr: ArrayRef = Arc::new(fsl_builder.finish()); + let n = rows.len() as u64; + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..n)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + let tmpdir = TempObjDir::default(); let test_store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), tmpdir.clone(), Arc::new(LanceCache::no_cache()), )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); - // Create zones with different string ranges - // Zone 0: ["aaa", "azz"] - should NOT match "foo%" - // Zone 1: ["bar", "baz"] - should NOT match "foo%" - // Zone 2: ["fa", "foz"] - should match "foo%" (contains potential matches) - // Zone 3: ["fop", "fzz"] - should NOT match "foo%" (all values >= "fop") - // Zone 4: ["foo", "foobar"] - should match "foo%" - // Zone 5: ["gaa", "gzz"] - should NOT match "foo%" + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); - let zones = vec![ - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("aaa".to_string())), - max: ScalarValue::Utf8(Some("azz".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("bar".to_string())), - max: ScalarValue::Utf8(Some("baz".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("fa".to_string())), - max: ScalarValue::Utf8(Some("foz".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 2, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("fop".to_string())), - max: ScalarValue::Utf8(Some("fzz".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 3, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("foo".to_string())), - max: ScalarValue::Utf8(Some("foobar".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 4, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("gaa".to_string())), - max: ScalarValue::Utf8(Some("gzz".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 5, - start: 0, - length: 100, - }, - }, - ]; + ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .expect("failed to load ZoneMapIndex") + } - let index = ZoneMapIndex { - zones, - data_type: DataType::Utf8, - rows_per_zone: ROWS_PER_ZONE_DEFAULT, - store: test_store, - fri: None, - index_cache: WeakLanceCache::from(&LanceCache::no_cache()), - }; + use arrow_array::builder::Float32Builder; + use datafusion_common::ScalarValue as SV; - // Test LikePrefix query for "foo" - let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("foo".to_string()))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + fn fsl_scalar(values: Vec>) -> SV { + let list_size = values.len() as i32; + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let items: ArrayRef = Arc::new(Float32Array::from(values)); + let arr = Arc::new(FixedSizeListArray::new(item_field, list_size, items, None)); + SV::FixedSizeList(arr) + } - // Should match zones 2 and 4 only - let mut expected = RowAddrTreeMap::new(); - // Zone 2: fragment 2 - expected.insert_range((2u64 << 32)..((2u64 << 32) + 100)); - // Zone 4: fragment 4 - expected.insert_range((4u64 << 32)..((4u64 << 32) + 100)); + // Zones for nested types have null min/max; all non-null queries are conservative (not pruned) + // unless every row in the zone is null. + #[tokio::test] + async fn test_fsl_zonemap_conservative_equals() { + // Two zones, 4 rows each, no null rows. + let index = train_and_load_fsl( + vec![ + vec![Some(1.0), Some(2.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + vec![Some(7.0), Some(8.0)], + vec![Some(10.0), Some(20.0)], + vec![Some(30.0), Some(40.0)], + vec![Some(50.0), Some(60.0)], + vec![Some(70.0), Some(80.0)], + ], + 2, + ) + .await; + assert_eq!(index.zones.len(), 2); + // min/max are null for nested types + assert!(index.zones[0].min.is_null()); + assert!(index.zones[0].max.is_null()); + assert_eq!(index.zones[0].null_count, 0); + assert_eq!(index.zones[0].nan_count, 0); + + // Non-null Equals: conservative — neither zone is pruned + let q = SargableQuery::Equals(fsl_scalar(vec![Some(100.0), Some(200.0)])); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "non-null Equals on nested type is conservative" + ); + assert!( + index + .evaluate_zone_against_query(&index.zones[1], &q) + .unwrap(), + "non-null Equals on nested type is conservative" + ); - assert_eq!(result, SearchResult::at_most(expected)); + // IsNull: no null rows → both zones pruned + let q_null = SargableQuery::IsNull(); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q_null) + .unwrap(), + "IsNull pruned when null_count=0" + ); + assert!( + !index + .evaluate_zone_against_query(&index.zones[1], &q_null) + .unwrap(), + "IsNull pruned when null_count=0" + ); } #[tokio::test] - async fn test_like_prefix_edge_cases() { + async fn test_fsl_zonemap_null_list() { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), 2); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // Build FSL array with some null entries + let mut builder = arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), 2); + // Row 0: null list + builder.values().append_value(0.0); + builder.values().append_value(0.0); + builder.append(false); + // Row 1: [3.0, 4.0] + builder.values().append_value(3.0); + builder.values().append_value(4.0); + builder.append(true); + let fsl_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..2)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + let tmpdir = TempObjDir::default(); let test_store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), tmpdir.clone(), Arc::new(LanceCache::no_cache()), )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); - // Test edge cases for LIKE prefix - let zones = vec![ - // Zone with values that contain the prefix exactly - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("test".to_string())), - max: ScalarValue::Utf8(Some("test".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 100, - }, - }, - // Zone with values that span across the prefix boundary - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("te".to_string())), - max: ScalarValue::Utf8(Some("tf".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 0, - length: 100, - }, - }, - // Zone completely before prefix - ZoneMapStatistics { - min: ScalarValue::Utf8(Some("abc".to_string())), - max: ScalarValue::Utf8(Some("def".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 2, - start: 0, - length: 100, - }, - }, - ]; + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 1, "one null list"); - let index = ZoneMapIndex { - zones, - data_type: DataType::Utf8, - rows_per_zone: ROWS_PER_ZONE_DEFAULT, - store: test_store, - fri: None, - index_cache: WeakLanceCache::from(&LanceCache::no_cache()), - }; + // IsNull should match + let q = SargableQuery::IsNull(); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap() + ); - // Test LikePrefix "test" - let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("test".to_string()))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // Equals null should match (null_count > 0) + let null_scalar = SV::FixedSizeList(Arc::new(FixedSizeListArray::new_null( + item_field.clone(), + 2, + 1, + ))); + let q2 = SargableQuery::Equals(null_scalar); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q2) + .unwrap() + ); + } - // Should match zones 0 and 1 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..100); // Zone 0: fragment 0 - expected.insert_range((1u64 << 32)..((1u64 << 32) + 100)); + #[tokio::test] + async fn test_fsl_zonemap_is_in_conservative() { + // Two zones (4 rows each), no null rows. + let index = train_and_load_fsl( + vec![ + vec![Some(1.0), Some(2.0)], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)], + vec![Some(7.0), Some(8.0)], + vec![Some(10.0), Some(20.0)], + vec![Some(30.0), Some(40.0)], + vec![Some(50.0), Some(60.0)], + vec![Some(70.0), Some(80.0)], + ], + 2, + ) + .await; + assert_eq!(index.zones.len(), 2); - assert_eq!(result, SearchResult::at_most(expected)); + // IsIn with non-null values: conservative — neither zone is pruned + let q = SargableQuery::IsIn(vec![ + fsl_scalar(vec![Some(4.0), Some(5.0)]), + fsl_scalar(vec![Some(100.0), Some(200.0)]), + ]); + assert!( + index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "IsIn with non-null values is conservative for nested types" + ); + assert!( + index + .evaluate_zone_against_query(&index.zones[1], &q) + .unwrap(), + "IsIn with non-null values is conservative for nested types" + ); + } - // Test empty prefix - should match all zones - let query = SargableQuery::LikePrefix(ScalarValue::Utf8(Some("".to_string()))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + #[tokio::test] + async fn test_fsl_zonemap_value_range_is_none() { + let index = train_and_load_fsl( + vec![vec![Some(1.0), Some(2.0)], vec![Some(3.0), Some(4.0)]], + 2, + ) + .await; + assert_eq!(index.value_range(), None, "FSL index has no scalar range"); + } - let mut expected = RowAddrTreeMap::new(); - expected.insert_range(0..100); // Zone 0: fragment 0 - expected.insert_range((1u64 << 32)..((1u64 << 32) + 100)); - expected.insert_range((2u64 << 32)..((2u64 << 32) + 100)); + #[tokio::test] + async fn test_fsl_zonemap_all_null_zone() { + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl_type = DataType::FixedSizeList(item_field.clone(), 2); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, fsl_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + let mut builder = arrow_array::builder::FixedSizeListBuilder::new(Float32Builder::new(), 2); + for _ in 0..2 { + builder.values().append_value(0.0); + builder.values().append_value(0.0); + builder.append(false); // null list + } + let fsl_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..2)); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl_arr, row_addr]).unwrap(); + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(4)), + ) + .await + .unwrap(); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); - assert_eq!(result, SearchResult::at_most(expected)); + assert_eq!(index.zones[0].null_count, 2); + assert!(index.zones[0].min.is_null(), "all-null zone: min is null"); + + // When every row is null (null_count == zone length), a non-null query is pruned. + let q = SargableQuery::Equals(fsl_scalar(vec![Some(1.0), Some(2.0)])); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q) + .unwrap(), + "all-null zone (null_count == length) is pruned for non-null target" + ); + + // Range and IsIn are also pruned when all rows are null. + let q_range = SargableQuery::Range(std::ops::Bound::Unbounded, std::ops::Bound::Unbounded); + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &q_range) + .unwrap(), + "all-null zone is pruned for Range query" + ); } + /// Build a zone map on a Struct column and verify training succeeds and stats are correct. #[tokio::test] - async fn test_like_prefix_large_utf8() { + async fn test_struct_zonemap_null_tracking() { + use arrow_array::StructArray; + use arrow_schema::Fields; + + let item_field = Arc::new(Field::new("x", DataType::Int32, true)); + let struct_type = DataType::Struct(Fields::from(vec![item_field.clone()])); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, struct_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); + + // 3 non-null struct rows + let x_values: ArrayRef = Arc::new(arrow_array::Int32Array::from(vec![10, 20, 30])); + let struct_arr: ArrayRef = + Arc::new(StructArray::from(vec![(item_field.clone(), x_values)])); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..3)); + let batch = RecordBatch::try_new(schema.clone(), vec![struct_arr, row_addr]).unwrap(); + let tmpdir = TempObjDir::default(); let test_store = Arc::new(LanceIndexStore::new( Arc::new(ObjectStore::local()), tmpdir.clone(), Arc::new(LanceCache::no_cache()), )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(10)), + ) + .await + .unwrap(); - // Test with LargeUtf8 type - let zones = vec![ - ZoneMapStatistics { - min: ScalarValue::LargeUtf8(Some("aaa".to_string())), - max: ScalarValue::LargeUtf8(Some("azz".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 0, - start: 0, - length: 100, - }, - }, - ZoneMapStatistics { - min: ScalarValue::LargeUtf8(Some("foo".to_string())), - max: ScalarValue::LargeUtf8(Some("foobar".to_string())), - null_count: 0, - nan_count: 0, - bound: ZoneBound { - fragment_id: 1, - start: 0, - length: 100, - }, - }, - ]; + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); - let index = ZoneMapIndex { - zones, - data_type: DataType::LargeUtf8, - rows_per_zone: ROWS_PER_ZONE_DEFAULT, - store: test_store, - fri: None, - index_cache: WeakLanceCache::from(&LanceCache::no_cache()), - }; + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 0, "no null struct rows"); + assert_eq!(index.zones[0].nan_count, 0); + // min/max are typed null ScalarValues (exact representation varies by DataFusion version) + // but IsNull correctly returns false and Equals is conservative - // Test LikePrefix with LargeUtf8 - let query = SargableQuery::LikePrefix(ScalarValue::LargeUtf8(Some("foo".to_string()))); - let result = index.search(&query, &NoOpMetricsCollector).await.unwrap(); + // IsNull: no null rows → zone is pruned + assert!( + !index + .evaluate_zone_against_query(&index.zones[0], &SargableQuery::IsNull()) + .unwrap() + ); - // Should match only zone 1 - let mut expected = RowAddrTreeMap::new(); - expected.insert_range((1u64 << 32)..((1u64 << 32) + 100)); + // Non-null Equals: conservative (not pruned) since not all rows are null + assert!( + index + .evaluate_zone_against_query( + &index.zones[0], + &SargableQuery::Equals(SV::Int32(Some(99))) + ) + .unwrap() + ); - assert_eq!(result, SearchResult::at_most(expected)); + // value_range: always None for nested types + assert_eq!(index.value_range(), None); } - #[test] - fn test_compute_next_prefix() { - use super::compute_next_prefix; - - // Basic cases - assert_eq!(compute_next_prefix("foo"), Some("fop".to_string())); - assert_eq!(compute_next_prefix("abc"), Some("abd".to_string())); - assert_eq!(compute_next_prefix("a"), Some("b".to_string())); - assert_eq!(compute_next_prefix("z"), Some("{".to_string())); // 'z' + 1 = '{' + /// Build a zone map on a List column and verify null tracking works. + #[tokio::test] + async fn test_list_zonemap_null_tracking() { + use arrow_array::builder::ListBuilder; - // Edge case: prefix with 'z' at the end - assert_eq!(compute_next_prefix("abz"), Some("ab{".to_string())); + let item_field = Arc::new(Field::new("item", DataType::Int32, true)); + let list_type = DataType::List(item_field.clone()); + let schema = Arc::new(Schema::new(vec![ + Field::new(VALUE_COLUMN_NAME, list_type.clone(), true), + Field::new(ROW_ADDR, DataType::UInt64, false), + ])); - // Edge case with tilde (~) which is 0x7E - assert_eq!(compute_next_prefix("ab~"), Some("ab\x7f".to_string())); + // Build: [null, [1,2], [3]] + let mut builder = ListBuilder::new(arrow_array::builder::Int32Builder::new()); + builder.append_null(); + builder.values().append_value(1); + builder.values().append_value(2); + builder.append(true); + builder.values().append_value(3); + builder.append(true); + let list_arr: ArrayRef = Arc::new(builder.finish()); + let row_addr: ArrayRef = Arc::new(UInt64Array::from_iter_values(0..3)); + let batch = RecordBatch::try_new(schema.clone(), vec![list_arr, row_addr]).unwrap(); - // Empty prefix - assert_eq!(compute_next_prefix(""), None); + let tmpdir = TempObjDir::default(); + let test_store = Arc::new(LanceIndexStore::new( + Arc::new(ObjectStore::local()), + tmpdir.clone(), + Arc::new(LanceCache::no_cache()), + )); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(std::future::ready(Ok(batch))), + )); + ZoneMapIndexPlugin::train_zonemap_index( + stream, + test_store.as_ref(), + Some(ZoneMapIndexBuilderParams::new(10)), + ) + .await + .unwrap(); - // Non-ASCII: works correctly by incrementing Unicode code points - // é (U+00E9) -> ê (U+00EA) - assert_eq!(compute_next_prefix("café"), Some("cafê".to_string())); - // 中 (U+4E2D) -> 丮 (U+4E2E) - assert_eq!(compute_next_prefix("abc中"), Some("abc丮".to_string())); - // ÿ (U+00FF) -> Ā (U+0100) - crosses byte boundary but works - assert_eq!(compute_next_prefix("cafÿ"), Some("cafĀ".to_string())); + let index = ZoneMapIndex::load(test_store.clone(), None, &LanceCache::no_cache(), false) + .await + .unwrap(); - // Edge case: character just before surrogate range - // U+D7FF -> U+E000 (skips surrogate range U+D800-U+DFFF) - assert_eq!( - compute_next_prefix("a\u{D7FF}"), - Some("a\u{E000}".to_string()) - ); + assert_eq!(index.zones.len(), 1); + assert_eq!(index.zones[0].null_count, 1, "one null list row"); + // min/max are typed null ScalarValues for nested types - // Edge case: max Unicode character U+10FFFF, falls back to previous char - assert_eq!(compute_next_prefix("ab\u{10FFFF}"), Some("ac".to_string())); - // All max characters - assert_eq!(compute_next_prefix("\u{10FFFF}\u{10FFFF}"), None); + // IsNull search: the null bitmap is populated during training so + // the result is an exact set containing just the null row (row address 0). + let result = index + .search(&SargableQuery::IsNull(), &NoOpMetricsCollector) + .await + .unwrap(); + let mut exact_nulls = RowAddrTreeMap::new(); + exact_nulls.insert(0); // only row 0 is null + assert_eq!(result, SearchResult::exact(exact_nulls)); } } diff --git a/rust/lance-index/src/traits.rs b/rust/lance-index/src/traits.rs index 5cb5c830666..f5441bd80f0 100644 --- a/rust/lance-index/src/traits.rs +++ b/rust/lance-index/src/traits.rs @@ -1,10 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::fmt::Display; + use lance_core::Result; use lance_table::format::IndexMetadata; +use crate::scalar::inverted::DocumentGranularity; + /// A set of criteria used to filter potential indices to use for a query #[derive(Debug, Default)] pub struct IndexCriteria<'a> { @@ -15,6 +19,8 @@ pub struct IndexCriteria<'a> { pub has_name: Option<&'a str>, /// If true, only consider indices that support FTS pub must_support_fts: bool, + /// Logical FTS document boundary. FTS lookups default to row documents. + pub fts_document_granularity: Option, /// If true, only consider indices that support exact equality pub must_support_exact_equality: bool, } @@ -39,6 +45,15 @@ impl<'a> IndexCriteria<'a> { self } + /// Select an FTS index with the requested logical document boundary. + pub fn with_fts_document_granularity( + mut self, + document_granularity: DocumentGranularity, + ) -> Self { + self.fts_document_granularity = Some(document_granularity); + self + } + /// Only consider indices that support exact equality /// /// This will disqualify, for example, the ngram and inverted indices @@ -58,6 +73,9 @@ pub type ScalarIndexCriteria<'a> = IndexCriteria<'a>; pub struct FtsPrewarmOptions { /// If true, prewarm positions along with posting lists. pub with_position: bool, + /// Controls whether prewarm requires the full requested FTS working set to + /// remain resident when the operation completes. + pub mode: FtsPrewarmMode, } impl FtsPrewarmOptions { @@ -69,6 +87,207 @@ impl FtsPrewarmOptions { self.with_position = with_position; self } + + pub fn with_mode(mut self, mode: FtsPrewarmMode) -> Self { + self.mode = mode; + self + } + + pub fn best_effort(mut self) -> Self { + self.mode = FtsPrewarmMode::BestEffort; + self + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum FtsPrewarmMode { + #[default] + Strict, + BestEffort, +} + +impl FtsPrewarmMode { + pub fn is_best_effort(self) -> bool { + matches!(self, Self::BestEffort) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmResult { + pub fully_resident: bool, + pub diagnostics: Option, +} + +impl FtsPrewarmResult { + pub fn fully_resident() -> Self { + Self { + fully_resident: true, + diagnostics: None, + } + } + + pub fn partial(diagnostics: FtsPrewarmDiagnostics) -> Self { + Self { + fully_resident: false, + diagnostics: Some(diagnostics), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmDiagnostics { + pub partition_count: usize, + pub failing_segments: Vec, + pub failing_partitions: Vec, +} + +impl FtsPrewarmDiagnostics { + pub fn fully_resident(&self) -> bool { + self.failing_segments.is_empty() && self.failing_partitions.is_empty() + } +} + +impl Display for FtsPrewarmDiagnostics { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "FTS prewarm completed without publishing query-ready scalar container, document, and posting state; \ + {} segment(s) and {} of {} partition(s) are not fully resident", + self.failing_segments.len(), + self.failing_partitions.len(), + self.partition_count + )?; + if !self.failing_segments.is_empty() || !self.failing_partitions.is_empty() { + write!(f, ": ")?; + let mut first = true; + for segment in &self.failing_segments { + if !first { + write!(f, "; ")?; + } + first = false; + write!(f, "{segment}")?; + } + for partition in &self.failing_partitions { + if !first { + write!(f, "; ")?; + } + first = false; + write!(f, "{partition}")?; + } + } + write!( + f, + ". Likely cause: index cache pressure or insufficient capacity. \ + Suggested remediation: increase index-cache capacity, reduce the number of FTS \ + segments assigned to each executor, or adjust placement/segment sizing." + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmSegmentStatus { + pub segment_id: String, + pub scalar_index_container_resident: bool, + pub scalar_index_container_matches_prewarmed: bool, +} + +impl FtsPrewarmSegmentStatus { + pub fn query_ready(&self) -> bool { + self.scalar_index_container_resident && self.scalar_index_container_matches_prewarmed + } +} + +impl Display for FtsPrewarmSegmentStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut missing = Vec::new(); + if !self.scalar_index_container_resident { + missing.push("resident scalar index container"); + } + if !self.scalar_index_container_matches_prewarmed { + missing.push("stable scalar index container identity"); + } + write!( + f, + "segment {} missing {}", + self.segment_id, + missing.join(", ") + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FtsPrewarmPartitionStatus { + pub segment_id: Option, + pub partition_id: u64, + pub documents: FtsPrewarmDocumentStatus, + pub posting_validation_ready: bool, + pub posting_resident: bool, + pub position_resident: Option, +} + +impl FtsPrewarmPartitionStatus { + pub fn query_ready(&self) -> bool { + self.documents.query_ready() + && self.posting_validation_ready + && self.posting_resident + && self.position_resident.unwrap_or(true) + } +} + +impl Display for FtsPrewarmPartitionStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut missing = Vec::new(); + if !self.documents.prewarm_complete { + missing.push("document prewarm completion"); + } + if !self.documents.scoring_ready { + missing.push("scoring lengths/norms"); + } + if !self.documents.reverse_lookup_ready { + missing.push("reverse document lookup"); + } + if !self.documents.projection_resident { + missing.push("resident row-address projection"); + } + if !self.posting_validation_ready { + missing.push("posting validation"); + } + if !self.posting_resident { + missing.push("resident posting lists"); + } + if self.position_resident == Some(false) { + missing.push("resident positions"); + } + let segment = self + .segment_id + .as_deref() + .map(|segment_id| format!("segment {segment_id} ")) + .unwrap_or_default(); + write!( + f, + "{}partition {} missing {}", + segment, + self.partition_id, + missing.join(", ") + ) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FtsPrewarmDocumentStatus { + pub prewarm_complete: bool, + pub scoring_ready: bool, + pub reverse_lookup_ready: bool, + pub projection_resident: bool, +} + +impl FtsPrewarmDocumentStatus { + pub fn query_ready(&self) -> bool { + self.prewarm_complete + && self.scoring_ready + && self.reverse_lookup_ready + && self.projection_resident + } } /// Options for prewarming an index. diff --git a/rust/lance-index/src/vector.rs b/rust/lance-index/src/vector.rs index 76b9ccc7a51..1192b2f5afb 100644 --- a/rust/lance-index/src/vector.rs +++ b/rust/lance-index/src/vector.rs @@ -85,8 +85,9 @@ pub const DEFAULT_QUERY_PARALLELISM: i32 = 0; /// Controls the speed / accuracy tradeoff for approximate vector search. /// -/// This currently only affects RQ-quantized vector indexes, such as IVF_RQ. -/// Other index types ignore this setting. +/// This currently affects RQ-quantized vector indexes (such as IVF_RQ) and +/// prefiltered search on HNSW sub-indexes, where `Fast` enables the ACORN +/// traversal. Other index types ignore this setting. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ApproxMode { /// Prefer lower query latency, which can reduce recall. @@ -354,6 +355,54 @@ pub trait VectorIndex: Send + Sync + std::fmt::Debug + Index { ))) } + /// Whether this index can search multiple query vectors in a single pass + /// via [`VectorIndex::search_partitions_batch`], reading each partition's + /// storage once and scoring every query that probes it. + /// + /// Defaults to `false`; callers should fall back to repeated single-query + /// search for indices that return `false`. + fn supports_batch_partition_search(&self) -> bool { + false + } + + /// Search a batch of query vectors against a shared set of partitions. + /// + /// `query.key` holds all query vectors concatenated (length + /// `query_count * dim`, where `query_count == partitions_per_query.len()`). + /// `partitions_per_query[i]` / `q_c_dists_per_query[i]` are the ranked + /// partition ids and query-to-centroid distances for query `i`. + /// + /// Returns one [RecordBatch] per query (in query order) with the + /// [`VECTOR_RESULT_SCHEMA`] (`_distance`, `_rowid`) and at most `query.k` + /// rows each. Implementations should read each distinct partition's storage + /// only once and score every query assigned to it against the loaded data. + /// + /// The default implementation returns an error; callers must gate on + /// [`VectorIndex::supports_batch_partition_search`]. + #[allow(clippy::too_many_arguments)] + async fn search_partitions_batch( + self: Arc, + query: Query, + partitions_per_query: Vec>, + q_c_dists_per_query: Vec>, + pre_filter: Arc, + metrics: Arc, + ) -> Result> + where + Self: 'static, + { + let _ = ( + query, + partitions_per_query, + q_c_dists_per_query, + pre_filter, + metrics, + ); + Err(Error::not_supported( + "batch partition search is not supported for this index", + )) + } + /// If the index is loadable by IVF, so it can be a sub-index that /// is loaded on demand by IVF. fn is_loadable(&self) -> bool; diff --git a/rust/lance-index/src/vector/bq.rs b/rust/lance-index/src/vector/bq.rs index 7a47fa88d54..4e3ef461379 100644 --- a/rust/lance-index/src/vector/bq.rs +++ b/rust/lance-index/src/vector/bq.rs @@ -28,6 +28,8 @@ pub mod transform; pub const RABIT_MIN_NUM_BITS: u8 = 1; pub const RABIT_MAX_NUM_BITS: u8 = 9; pub const RABIT_BINARY_NUM_BITS: u8 = 1; +/// Default number of bits per dimension for IVF_RQ indexes. +pub(crate) const RABIT_DEFAULT_NUM_BITS: u8 = 5; #[derive(Clone, Default)] pub struct BinaryQuantization {} @@ -110,6 +112,7 @@ impl FromStr for RQRotationType { #[derive(Clone, Debug)] pub struct RQBuildParams { + /// Number of bits per dimension. Defaults to 5. pub num_bits: u8, pub rotation_type: RQRotationType, /// Optional pre-built rotation to reuse instead of generating a fresh random one. @@ -193,7 +196,7 @@ impl QuantizerBuildParams for RQBuildParams { impl Default for RQBuildParams { fn default() -> Self { Self { - num_bits: 1, + num_bits: RABIT_DEFAULT_NUM_BITS, rotation_type: RQRotationType::default(), rotation: None, } @@ -237,6 +240,11 @@ mod tests { assert!("invalid".parse::().is_err()); } + #[test] + fn test_rq_build_params_default_num_bits() { + assert_eq!(RQBuildParams::default().num_bits, 5); + } + #[test] fn test_rabit_num_bits_validation() { validate_rq_num_bits(1).unwrap(); diff --git a/rust/lance-index/src/vector/bq/builder.rs b/rust/lance-index/src/vector/bq/builder.rs index 9eb7fc76903..c77c3027b8b 100644 --- a/rust/lance-index/src/vector/bq/builder.rs +++ b/rust/lance-index/src/vector/bq/builder.rs @@ -25,7 +25,7 @@ use crate::vector::bq::transform::{ SCALE_FACTORS_FIELD, }; use crate::vector::bq::{ - RQBuildParams, RQRotationType, rabit_binary_code_bytes, rabit_ex_bits, + RABIT_DEFAULT_NUM_BITS, RQBuildParams, RQRotationType, rabit_binary_code_bytes, rabit_ex_bits, rotation::{apply_fast_rotation, fast_rotation_signs_len, random_fast_rotation_signs}, validate_rq_num_bits, }; @@ -33,7 +33,7 @@ use crate::vector::quantizer::{Quantization, Quantizer, QuantizerBuildParams}; /// Build parameters for RabitQuantizer. /// -/// num_bits: the number of bits per dimension. +/// num_bits: the number of bits per dimension. Defaults to 5. pub struct RabitBuildParams { pub num_bits: u8, pub rotation_type: RQRotationType, @@ -42,7 +42,7 @@ pub struct RabitBuildParams { impl Default for RabitBuildParams { fn default() -> Self { Self { - num_bits: 1, + num_bits: RABIT_DEFAULT_NUM_BITS, rotation_type: RQRotationType::default(), } } @@ -81,6 +81,74 @@ fn pack_sign_bits(codes: &mut [u8], rotated: &[f32]) { const EX_QUANTIZATION_EPSILON: f32 = 1.0e-5; const EX_TIGHT_START: [f32; 9] = [0.0, 0.15, 0.20, 0.52, 0.59, 0.71, 0.75, 0.77, 0.81]; +/// Sort packed `(positive_f32_bits, index)` values by their floating-point key. +/// +/// All thresholds emitted by [`best_ex_rescale_factor`] are positive and finite, +/// so their IEEE-754 bit patterns have the same order as the represented values. +/// Four stable byte-wise passes avoid the comparison-heavy tuple sort on every +/// vector while preserving the exact threshold order. +fn radix_sort_positive_f32_indices(values: &mut [u64]) { + if values.len() < 2 { + return; + } + + fn pass(source: &[u64], destination: &mut [u64], shift: u32) { + let mut offsets = [0usize; 256]; + for &value in source { + offsets[((value >> shift) & 0xff) as usize] += 1; + } + let mut next_offset = 0; + for offset in &mut offsets { + let count = *offset; + *offset = next_offset; + next_offset += count; + } + for &value in source { + let bucket = ((value >> shift) & 0xff) as usize; + destination[offsets[bucket]] = value; + offsets[bucket] += 1; + } + } + + let mut scratch = vec![0u64; values.len()]; + pass(values, &mut scratch, 32); + pass(&scratch, values, 40); + pass(values, &mut scratch, 48); + pass(&scratch, values, 56); +} + +/// Sort RQ threshold events without changing the existing equal-key behavior. +/// +/// The quantizer updates its floating-point objective after every event, so the +/// order chosen by the previous unstable comparison sort remains observable when +/// thresholds tie. Radix sort the common unique-key case, but reconstruct the +/// original event order and use the previous sort when duplicate keys are found. +fn sort_ex_thresholds(values: &mut [u64]) { + radix_sort_positive_f32_indices(values); + let has_duplicate_keys = values + .windows(2) + .any(|pair| pair[0] >> u32::BITS == pair[1] >> u32::BITS); + if !has_duplicate_keys { + return; + } + + // Events were originally emitted by index, then by increasing threshold. + values.sort_unstable_by_key(|value| (*value as u32, (value >> u32::BITS) as u32)); + let mut comparison_thresholds = values + .iter() + .map(|value| { + ( + f32::from_bits((value >> u32::BITS) as u32), + *value as u32 as usize, + ) + }) + .collect::>(); + comparison_thresholds.sort_unstable_by(|(left, _), (right, _)| left.total_cmp(right)); + for (value, (threshold, idx)) in values.iter_mut().zip(comparison_thresholds) { + *value = ((threshold.to_bits() as u64) << u32::BITS) | idx as u64; + } +} + fn best_ex_rescale_factor(abs_normalized: &[f32], ex_bits: u8) -> f32 { let max_value = abs_normalized .iter() @@ -117,17 +185,20 @@ fn best_ex_rescale_factor(abs_normalized: &[f32], ex_bits: u8) -> f32 { while next <= max_code { let threshold = next as f32 / value; if threshold < t_end { - thresholds.push((threshold, idx)); + debug_assert!(u32::try_from(idx).is_ok()); + thresholds.push(((threshold.to_bits() as u64) << u32::BITS) | idx as u64); } next += 1; } } - thresholds.sort_unstable_by(|(left, _), (right, _)| left.total_cmp(right)); + sort_ex_thresholds(&mut thresholds); let mut best_inner_product = numerator / squared_denominator.sqrt(); let mut best_t = t_start; - for (threshold, idx) in thresholds { + for packed_threshold in thresholds { + let threshold = f32::from_bits((packed_threshold >> u32::BITS) as u32); + let idx = packed_threshold as u32 as usize; current_codes[idx] += 1; let updated = current_codes[idx]; squared_denominator += (2 * updated) as f32; @@ -892,6 +963,124 @@ mod tests { use crate::vector::bq::storage::RABIT_BLOCKED_EX_CODE_COLUMN; + #[test] + fn test_rabit_build_params_default_num_bits() { + assert_eq!(RabitBuildParams::default().num_bits, 5); + } + + fn reference_best_ex_rescale_factor(abs_normalized: &[f32], ex_bits: u8) -> f32 { + let max_value = abs_normalized + .iter() + .copied() + .filter(|value| value.is_finite()) + .fold(0.0f32, f32::max); + if max_value <= 0.0 { + return 0.0; + } + + let max_code = (1usize << ex_bits) - 1; + let t_end = ((max_code + 10) as f32) / max_value; + let t_start = t_end * EX_TIGHT_START[ex_bits as usize]; + let mut current_codes = Vec::with_capacity(abs_normalized.len()); + let mut squared_denominator = abs_normalized.len() as f32 * 0.25; + let mut numerator = 0.0f32; + let mut thresholds = Vec::with_capacity(abs_normalized.len() * max_code); + + for (idx, &value) in abs_normalized.iter().enumerate() { + if value <= 0.0 || !value.is_finite() { + current_codes.push(0usize); + continue; + } + let current = ((t_start * value) + EX_QUANTIZATION_EPSILON) + .floor() + .clamp(0.0, max_code as f32) as usize; + current_codes.push(current); + squared_denominator += (current * current + current) as f32; + numerator += (current as f32 + 0.5) * value; + + for next in (current + 1)..=max_code { + let threshold = next as f32 / value; + if threshold < t_end { + thresholds.push((threshold, idx)); + } + } + } + + thresholds.sort_unstable_by(|(left, _), (right, _)| left.total_cmp(right)); + let mut best_inner_product = numerator / squared_denominator.sqrt(); + let mut best_t = t_start; + for (threshold, idx) in thresholds { + current_codes[idx] += 1; + let updated = current_codes[idx]; + squared_denominator += (2 * updated) as f32; + numerator += abs_normalized[idx]; + let current_inner_product = numerator / squared_denominator.sqrt(); + if current_inner_product > best_inner_product { + best_inner_product = current_inner_product; + best_t = threshold; + } + } + best_t + } + + #[test] + fn test_radix_sort_positive_f32_indices_matches_float_order() { + let mut values = (0..4096u32) + .map(|idx| { + let key = ((idx.wrapping_mul(2654435761) % 997) + 1) as f32 / 37.0; + ((key.to_bits() as u64) << u32::BITS) | idx as u64 + }) + .collect::>(); + let mut expected = values.clone(); + expected.sort_by_key(|value| *value >> u32::BITS); + + radix_sort_positive_f32_indices(&mut values); + + assert_eq!(values, expected); + } + + #[test] + fn test_best_ex_rescale_factor_matches_comparison_sort() { + let mut values = (0..1536u32) + .map(|idx| { + let mixed = idx + .wrapping_mul(747796405) + .wrapping_add(2891336453) + .rotate_right((idx % 31) + 1); + (mixed as f32 / u32::MAX as f32) * 0.1 + }) + .collect::>(); + values[0] = 0.0; + values[1] = f32::NAN; + values[2] = f32::INFINITY; + values[3] = values[4]; + + for ex_bits in 1..=8 { + let expected = reference_best_ex_rescale_factor(&values, ex_bits); + let actual = best_ex_rescale_factor(&values, ex_bits); + assert_eq!(actual.to_bits(), expected.to_bits(), "ex_bits={ex_bits}"); + } + } + + #[test] + fn test_best_ex_rescale_factor_preserves_equal_threshold_order() { + let rotated = [0.75, 1.0, 0.0625, 0.5, 2.0 / 3.0, 0.75, 0.5, 2.0 / 3.0]; + let norm = rotated + .iter() + .map(|value| value * value) + .sum::() + .sqrt(); + let abs_normalized = rotated + .iter() + .map(|value| value.abs() / norm) + .collect::>(); + + let expected = reference_best_ex_rescale_factor(&abs_normalized, 7); + let actual = best_ex_rescale_factor(&abs_normalized, 7); + + assert_eq!(actual.to_bits(), expected.to_bits()); + } + #[rstest] #[case(8)] #[case(16)] diff --git a/rust/lance-index/src/vector/bq/storage.rs b/rust/lance-index/src/vector/bq/storage.rs index 7bcc2526b43..0f7f034b2c1 100644 --- a/rust/lance-index/src/vector/bq/storage.rs +++ b/rust/lance-index/src/vector/bq/storage.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use lance_core::utils::parse::str_is_truthy; use lance_core::utils::row_addr_remap::RowAddrRemap; use std::borrow::Cow; use std::collections::{BinaryHeap, HashMap}; @@ -22,7 +23,7 @@ use itertools::{Itertools, izip}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray, RecordBatchExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_linalg::distance::{DistanceType, Dot, dot, l2::l2}; use lance_linalg::simd::{ self, @@ -39,8 +40,9 @@ use num_traits::AsPrimitive; use prost::Message; use serde::{Deserialize, Serialize}; -use crate::frag_reuse::FragReuseIndex; +use crate::frag_reuse::{FragReuseIndex, FragReuseIndexHandle}; use crate::pb; +use crate::scalar::RowIdRemapper; use crate::vector::ApproxMode; use crate::vector::bq::dist_table_quant::{ DistTableDequant, quantize_dist_table_into, quantize_dist_table_u16_into, @@ -104,10 +106,7 @@ static RABIT_PRUNE_STATS_INTERVAL: OnceLock = OnceLock::new(); fn rabit_prune_stats_enabled() -> bool { *RABIT_PRUNE_STATS_ENABLED.get_or_init(|| match std::env::var(RABIT_PRUNE_STATS_ENV) { - Ok(value) => { - let value = value.to_ascii_lowercase(); - !matches!(value.as_str(), "" | "0" | "false" | "off" | "no") - } + Ok(value) => str_is_truthy(value.trim()), Err(_) => false, }) } @@ -464,7 +463,7 @@ impl QuantizerMetadata for RabitQuantizationMetadata { } } - async fn load(reader: &PreviousFileReader) -> Result { + async fn load(reader: &V1FileReader) -> Result { let metadata_str = reader .schema() .metadata @@ -519,8 +518,8 @@ impl RabitQuantizationStorage { fn residual_query_factor(&self, dist_q_c: f32) -> f32 { match self.distance_type { - DistanceType::L2 => dist_q_c, - DistanceType::Cosine | DistanceType::Dot => dist_q_c - 1.0, + DistanceType::L2 | DistanceType::Cosine => dist_q_c, + DistanceType::Dot => dist_q_c - 1.0, _ => unimplemented!( "RabitQ does not support distance type: {}", self.distance_type @@ -931,20 +930,25 @@ impl<'a> RabitDistCalculator<'a> { /// Fill `dists[0..n]` with exact per-row binary distances computed /// directly from the f32 dist table — the fallback when the quantized /// reconstruction scale would be non-finite ([`DistTableDequant::Exact`]). - #[allow(clippy::uninit_vec)] fn fill_exact_binary_distances(&self, n: usize, code_len: usize, dists: &mut Vec) { dists.clear(); dists.reserve(n); - // SAFETY: the loop initializes every element in [0, n). - unsafe { - dists.set_len(n); - } - dists.iter_mut().enumerate().for_each(|(id, dist)| { - *dist = compute_single_rq_distance(self.codes, id, n, code_len, &self.dist_table); - }); + dists.spare_capacity_mut()[..n] + .iter_mut() + .enumerate() + .for_each(|(id, dist)| { + dist.write(compute_single_rq_distance( + self.codes, + id, + n, + code_len, + &self.dist_table, + )); + }); + // Every reserved slot was initialized above. + unsafe { dists.set_len(n) }; } - #[allow(clippy::uninit_vec)] fn binary_distances_with_scratch( &self, n: usize, @@ -979,52 +983,50 @@ impl<'a> RabitDistCalculator<'a> { let simd_len = n - remainder; quantized_dists.clear(); quantized_dists.reserve(simd_len); - // SAFETY: sum_4bit_dist_table overwrites each element in the SIMD batch range. unsafe { + // Storage construction proves the code and table layouts, and the + // reserved output has exactly one slot per SIMD row. + simd::dist_table::sum_4bit_dist_table_uninit( + simd_len, + code_len, + self.codes, + quantized_dists_table, + &mut quantized_dists.spare_capacity_mut()[..simd_len], + ); + // The distance-table kernel initialized every SIMD output slot. quantized_dists.set_len(simd_len); } - simd::dist_table::sum_4bit_dist_table( - simd_len, - code_len, - self.codes, - quantized_dists_table, - quantized_dists, - ); let range = (qmax - qmin) / 255.0; let num_tables = quantized_dists_table.len() / SEGMENT_NUM_CODES; let sum_min = num_tables as f32 * qmin; dists.clear(); dists.reserve(n); - // SAFETY: the SIMD section below writes [0, simd_len), and the - // remainder section writes [simd_len, n). - unsafe { - dists.set_len(n); - } - let (simd_dists, remainder_dists) = dists.split_at_mut(simd_len); - simd_dists + let uninit_dists = &mut dists.spare_capacity_mut()[..n]; + uninit_dists[..simd_len] .iter_mut() .zip(quantized_dists.iter()) .for_each(|(dist, q_dist)| { - *dist = (*q_dist as f32) * range + sum_min; + dist.write((*q_dist as f32) * range + sum_min); }); - remainder_dists + uninit_dists[simd_len..] .iter_mut() .enumerate() .for_each(|(id, dist)| { - *dist = compute_single_rq_distance( + dist.write(compute_single_rq_distance( self.codes, simd_len + id, n, code_len, &self.dist_table, - ); + )); }); + // Both the SIMD reconstruction and scalar remainder initialized their slots. + unsafe { dists.set_len(n) }; simd_len } - #[allow(clippy::uninit_vec)] fn binary_distances_hacc_with_scratch( &self, n: usize, @@ -1049,48 +1051,47 @@ impl<'a> RabitDistCalculator<'a> { let simd_len = n - remainder; quantized_dists.clear(); quantized_dists.reserve(simd_len); - // SAFETY: sum_4bit_hacc_dist_table overwrites each element in the batch range. unsafe { + // Storage construction proves the code and table layouts, and the + // reserved output has exactly one slot per SIMD row. + simd::dist_table::sum_4bit_hacc_dist_table_uninit( + simd_len, + code_len, + self.codes, + hacc_dist_table, + &mut quantized_dists.spare_capacity_mut()[..simd_len], + ); + // The high-accuracy kernel initialized every SIMD output slot. quantized_dists.set_len(simd_len); } - simd::dist_table::sum_4bit_hacc_dist_table( - simd_len, - code_len, - self.codes, - hacc_dist_table, - quantized_dists, - ); let range = (qmax - qmin) / u16::MAX as f32; let num_tables = quantized_dist_table.len() / SEGMENT_NUM_CODES; let sum_min = num_tables as f32 * qmin; dists.clear(); dists.reserve(n); - // SAFETY: the batch section writes [0, simd_len), and the - // remainder section writes [simd_len, n). - unsafe { - dists.set_len(n); - } - let (simd_dists, remainder_dists) = dists.split_at_mut(simd_len); - simd_dists + let uninit_dists = &mut dists.spare_capacity_mut()[..n]; + uninit_dists[..simd_len] .iter_mut() .zip(quantized_dists.iter()) .for_each(|(dist, q_dist)| { - *dist = (*q_dist as f32) * range + sum_min; + dist.write((*q_dist as f32) * range + sum_min); }); - remainder_dists + uninit_dists[simd_len..] .iter_mut() .enumerate() .for_each(|(id, dist)| { - *dist = compute_single_rq_distance( + dist.write(compute_single_rq_distance( self.codes, simd_len + id, n, code_len, &self.dist_table, - ); + )); }); + // Both the SIMD reconstruction and scalar remainder initialized their slots. + unsafe { dists.set_len(n) }; simd_len } @@ -1102,7 +1103,6 @@ impl<'a> RabitDistCalculator<'a> { } } - #[allow(clippy::uninit_vec)] fn one_bit_distances_with_scratch( &self, n: usize, @@ -1131,7 +1131,6 @@ impl<'a> RabitDistCalculator<'a> { }); } - #[allow(clippy::uninit_vec)] fn apply_raw_query_multi_bit_distances( &self, simd_len: usize, @@ -1165,17 +1164,19 @@ impl<'a> RabitDistCalculator<'a> { ); quantized_dists.clear(); quantized_dists.reserve(fastscan_len); - // SAFETY: sum_4bit_dist_table overwrites each element in the SIMD batch range. unsafe { + // The packed ex-code layout and table size are fixed at + // construction, and the output reserves one slot per row. + simd::dist_table::sum_4bit_dist_table_uninit( + fastscan_len, + fastscan_code_len, + packed_ex_codes, + quantized_dists_table, + &mut quantized_dists.spare_capacity_mut()[..fastscan_len], + ); + // The distance-table kernel initialized every fast-scan slot. quantized_dists.set_len(fastscan_len); } - simd::dist_table::sum_4bit_dist_table( - fastscan_len, - fastscan_code_len, - packed_ex_codes, - quantized_dists_table, - quantized_dists, - ); let range = (qmax - qmin) / quantization_max; let num_tables = quantized_dists_table.len() / SEGMENT_NUM_CODES; @@ -1827,7 +1828,6 @@ impl DistCalculator for RabitDistCalculator<'_> { } #[inline(always)] - #[allow(clippy::uninit_vec)] fn distance_all_with_scratch( &self, _: usize, @@ -2392,13 +2392,10 @@ pub fn unpack_codes(codes: &FixedSizeListArray) -> FixedSizeListArray { /// to `Some(new_id)` for surviving rows or `None` for rows whose covering /// fragment was compacted away, suitable for `RabitQuantizationStorage::remap`. fn build_frag_reuse_mapping( - fri: Option<&FragReuseIndex>, + fri: Option<&dyn RowIdRemapper>, row_ids: &UInt64Array, ) -> Option>> { let fri = fri?; - if fri.row_id_maps.is_empty() { - return None; - } let mut mapping: HashMap> = HashMap::new(); for row_id in row_ids.values().iter() { match fri.remap_row_id(*row_id) { @@ -2424,6 +2421,16 @@ impl QuantizerStorage for RabitQuantizationStorage { metadata: &Self::Metadata, distance_type: DistanceType, fri: Option>, + ) -> Result { + let fri = fri.map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::try_from_batch_with_remapper(batch, metadata, distance_type, fri) + } + + fn try_from_batch_with_remapper( + batch: RecordBatch, + metadata: &Self::Metadata, + distance_type: DistanceType, + fri: Option>, ) -> Result { let distance_type = match (metadata.query_estimator, distance_type) { (RabitQueryEstimator::RawQuery, DistanceType::Cosine) => DistanceType::L2, @@ -2545,7 +2552,7 @@ impl QuantizerStorage for RabitQuantizationStorage { } async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, @@ -3930,7 +3937,7 @@ mod tests { .into_iter() .map(|(id, dist)| (id as u64, dist)) .collect::>(); - expected.sort_by(|left, right| left.0.cmp(&right.0)); + expected.sort_by_key(|left| left.0); let mut heap = BinaryHeap::with_capacity(k); let mut distances = Vec::new(); @@ -3952,7 +3959,7 @@ mod tests { .into_iter() .map(|node| (node.id, node.dist.0)) .collect::>(); - actual.sort_by(|left, right| left.0.cmp(&right.0)); + actual.sort_by_key(|left| left.0); assert_eq!(actual.len(), expected.len()); for ((actual_id, actual_dist), (expected_id, expected_dist)) in @@ -4298,20 +4305,33 @@ mod tests { } #[test] - fn test_try_from_batch_keeps_cosine_for_legacy_residual_query() { + fn test_residual_query_cosine_uses_l2_query_factor() { let original_codes = make_test_codes(50, 64); let mut metadata = make_test_metadata(original_codes.value_length() as usize * 8); metadata.query_estimator = RabitQueryEstimator::ResidualQuery; + let batch = make_test_batch(original_codes); - let storage = RabitQuantizationStorage::try_from_batch( - make_test_batch(original_codes), + let cosine_storage = RabitQuantizationStorage::try_from_batch( + batch.clone(), &metadata, DistanceType::Cosine, None, ) .unwrap(); + let l2_storage = + RabitQuantizationStorage::try_from_batch(batch, &metadata, DistanceType::L2, None) + .unwrap(); + + assert_eq!(cosine_storage.distance_type(), DistanceType::Cosine); + + let query = Arc::new(Float32Array::from(vec![0.125; 64])) as ArrayRef; + let dist_q_c = 0.25; + let cosine_distances = cosine_storage + .dist_calculator(query.clone(), dist_q_c) + .distance_all(0); + let l2_distances = l2_storage.dist_calculator(query, dist_q_c).distance_all(0); - assert_eq!(storage.distance_type(), DistanceType::Cosine); + assert_eq!(cosine_distances, l2_distances); } #[test] diff --git a/rust/lance-index/src/vector/distributed/index_merger.rs b/rust/lance-index/src/vector/distributed/index_merger.rs index 70371ad4794..370011eb8bd 100755 --- a/rust/lance-index/src/vector/distributed/index_merger.rs +++ b/rust/lance-index/src/vector/distributed/index_merger.rs @@ -9,16 +9,17 @@ use crate::vector::shared::partition_merger::{ }; use arrow::{compute::concat_batches, datatypes::Float32Type}; use arrow_array::cast::AsArray; -use arrow_array::types::UInt8Type; +use arrow_array::types::{UInt8Type, UInt64Type}; use arrow_array::{Array, FixedSizeListArray, RecordBatch}; use futures::StreamExt as _; use lance_arrow::{FixedSizeListArrayExt, RecordBatchExt}; -use lance_core::{Error, ROW_ID_FIELD, Result}; +use lance_core::{Error, ROW_ID, ROW_ID_FIELD, Result}; use std::ops::Range; use std::sync::Arc; use crate::IndexMetadata as IndexMetaSchema; use crate::pb; +use crate::scalar::OldIndexDataFilter; use crate::vector::bq::storage::{ RABIT_CODE_COLUMN, RABIT_METADATA_KEY, RabitQuantizationMetadata, RabitQueryEstimator, pack_codes, rabit_binary_code_field, rabit_ex_code_field, @@ -39,8 +40,9 @@ use crate::{INDEX_AUXILIARY_FILE_NAME, INDEX_METADATA_SCHEMA_KEY}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use bytes::Bytes; use lance_core::datatypes::Schema as LanceSchema; -use lance_encoding::version::LanceFileVersion; use lance_file::reader::{FileReader as V2Reader, FileReaderOptions as V2ReaderOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions; use lance_file::writer::{FileWriter as V2Writer, FileWriter, FileWriterOptions}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::utils::CachedFileSize; @@ -91,6 +93,11 @@ fn fixed_size_list_equal(a: &FixedSizeListArray, b: &FixedSizeListArray) -> bool let vb = b.values().as_primitive::(); va.values() == vb.values() } + (DataType::UInt8, DataType::UInt8) => { + let va = a.values().as_primitive::(); + let vb = b.values().as_primitive::(); + va.values() == vb.values() + } _ => false, } } @@ -112,6 +119,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, return false; } for i in 0..av.len() { + if av[i].is_nan() || bv[i].is_nan() { + return false; + } if (av[i] - bv[i]).abs() > tol { return false; } @@ -127,6 +137,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, return false; } for i in 0..av.len() { + if av[i].is_nan() || bv[i].is_nan() { + return false; + } if (av[i] - bv[i]).abs() > tol as f64 { return false; } @@ -144,6 +157,9 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, for i in 0..av.len() { let da = av[i].to_f32(); let db = bv[i].to_f32(); + if da.is_nan() || db.is_nan() { + return false; + } if (da - db).abs() > tol { return false; } @@ -154,6 +170,63 @@ fn fixed_size_list_almost_equal(a: &FixedSizeListArray, b: &FixedSizeListArray, } } +fn ensure_fixed_size_list_compatible( + what: &str, + reference: &FixedSizeListArray, + candidate: &FixedSizeListArray, +) -> Result<()> { + if !fixed_size_list_equal(reference, candidate) { + const TOL: f32 = 1e-5; + if !fixed_size_list_almost_equal(reference, candidate, TOL) { + return Err(Error::index(format!("{what} mismatch across shards"))); + } + log::warn!("{what} differs within tolerance; proceeding with first shard value"); + } + Ok(()) +} + +async fn try_read_ivf_proto(reader: &V2Reader) -> Result> { + let Some(ivf_idx) = reader.metadata().file_schema.metadata.get(IVF_METADATA_KEY) else { + return Ok(None); + }; + let ivf_idx = ivf_idx + .parse() + .map_err(|_| Error::index("IVF index parse error".to_string()))?; + let bytes = reader.read_global_buffer(ivf_idx).await?; + Ok(Some(pb::Ivf::decode(bytes)?)) +} + +fn ivf_centroids_from_proto(ivf: &pb::Ivf) -> Result> { + ivf.centroids_tensor + .as_ref() + .map(FixedSizeListArray::try_from) + .transpose() +} + +async fn open_sibling_index_reader( + object_store: &lance_io::object_store::ObjectStore, + sched: &Arc, + idx_path: &object_store::path::Path, +) -> Result> { + if !object_store.exists(idx_path).await? { + return Ok(None); + } + + let fh = sched + .open_file(idx_path, &CachedFileSize::unknown()) + .await?; + Ok(Some( + V2Reader::try_open( + fh, + None, + Arc::default(), + &lance_core::cache::LanceCache::no_cache(), + V2ReaderOptions::default(), + ) + .await?, + )) +} + /// Initialize schema-level metadata on a writer for a given storage. /// /// It writes the distance type and the storage metadata (as a vector payload), @@ -183,7 +256,7 @@ pub async fn init_writer_for_flat( d0: usize, item_type: &DataType, dt: DistanceType, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let arrow_schema = ArrowSchema::new(vec![ (*ROW_ID_FIELD).clone(), @@ -197,13 +270,11 @@ pub async fn init_writer_for_flat( ), ]); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let meta_json = serde_json::to_string(&FlatMetadata { dim: d0 })?; init_writer_for_storage(&mut w, dt, &meta_json, "")?; @@ -219,7 +290,7 @@ pub async fn init_writer_for_pq( aux_out: &object_store::path::Path, dt: DistanceType, pm: &ProductQuantizationMetadata, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let num_bytes = if pm.nbits == 4 { pm.num_sub_vectors / 2 @@ -238,13 +309,11 @@ pub async fn init_writer_for_pq( ), ]); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let mut pm_init = pm.clone(); let cb = pm_init @@ -266,7 +335,7 @@ pub async fn init_writer_for_sq( aux_out: &object_store::path::Path, dt: DistanceType, sq_meta: &ScalarQuantizationMetadata, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let d0 = sq_meta.dim; let arrow_schema = ArrowSchema::new(vec![ @@ -281,13 +350,11 @@ pub async fn init_writer_for_sq( ), ]); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let meta_json = serde_json::to_string(sq_meta)?; init_writer_for_storage(&mut w, dt, &meta_json, SQ_METADATA_KEY)?; @@ -300,7 +367,7 @@ pub async fn init_writer_for_rq( aux_out: &object_store::path::Path, dt: DistanceType, rq_meta: &RabitQuantizationMetadata, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, ) -> Result { let mut fields = vec![ (*ROW_ID_FIELD).clone(), @@ -318,13 +385,11 @@ pub async fn init_writer_for_rq( } let arrow_schema = ArrowSchema::new(fields); let writer = object_store.create(aux_out).await?; - let mut w = FileWriter::try_new( + let mut w = versions::create_writer( + format_version, writer, LanceSchema::try_from(&arrow_schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; let mut rq_meta_init = rq_meta.clone(); @@ -363,6 +428,35 @@ pub async fn write_partition_rows( Ok(()) } +/// Stream a partition range, retain its owned rows, and return the number written. +async fn write_filtered_partition_rows( + reader: &V2Reader, + w: &mut FileWriter, + range: Range, + row_filter: &OldIndexDataFilter, +) -> Result { + let mut stream = reader + .read_stream( + lance_io::ReadBatchParams::Range(range), + u32::MAX, + 4, + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await?; + let mut written_rows = 0usize; + while let Some(batch) = stream.next().await { + let batch = filter_batch_to_owned_rows(&batch?, row_filter)?; + if batch.num_rows() == 0 { + continue; + } + written_rows = written_rows.checked_add(batch.num_rows()).ok_or_else(|| { + Error::index("Filtered partition row count exceeds usize capacity".to_string()) + })?; + w.write_batch(&batch).await?; + } + Ok(written_rows) +} + /// Transpose the PQ code column for a batch and write it to the unified writer. /// /// This helper assumes `batch` contains a contiguous range of rows for a single @@ -460,6 +554,7 @@ struct ShardInfo { lengths: Vec, partition_offsets: Vec, total_rows: usize, + row_filter: Option>, } #[derive(Debug)] @@ -469,6 +564,7 @@ struct ShardWindowReadJob { window_total_rows: usize, start_offset: usize, end_offset: usize, + row_filter: Option>, } #[derive(Debug)] @@ -587,6 +683,7 @@ async fn read_partition_window( window_total_rows, start_offset, end_offset, + row_filter: shard.row_filter.clone(), } }) .collect(); @@ -674,7 +771,13 @@ async fn read_shard_window_partitions( } let to_take = std::cmp::min(remaining, rb.num_rows() - consumed); - per_partition_batches[rel_partition].push(rb.slice(consumed, to_take)); + let mut partition_batch = rb.slice(consumed, to_take); + if let Some(row_filter) = shard_job.row_filter.as_deref() { + partition_batch = filter_batch_to_owned_rows(&partition_batch, row_filter)?; + } + if partition_batch.num_rows() > 0 { + per_partition_batches[rel_partition].push(partition_batch); + } consumed += to_take; remaining -= to_take; } @@ -697,6 +800,19 @@ async fn read_shard_window_partitions( Ok(per_partition_batches) } +fn filter_batch_to_owned_rows( + batch: &RecordBatch, + row_filter: &OldIndexDataFilter, +) -> Result { + let row_ids = batch + .column_by_name(ROW_ID) + .ok_or_else(|| Error::index(format!("Column {ROW_ID} missing in auxiliary shard")))? + .as_primitive_opt::() + .ok_or_else(|| Error::index(format!("Column {ROW_ID} is not UInt64 in auxiliary shard")))?; + let keep = row_filter.filter_row_ids(row_ids); + Ok(arrow::compute::filter_record_batch(batch, &keep)?) +} + /// Merge the selected segment auxiliary files into `target_dir`. /// /// This is the storage merge kernel for vector segment build. Callers choose @@ -713,6 +829,42 @@ pub async fn merge_partial_vector_auxiliary_files( aux_paths: &[object_store::path::Path], target_dir: &object_store::path::Path, progress: Arc, +) -> Result { + merge_partial_vector_auxiliary_files_inner(object_store, aux_paths, target_dir, None, progress) + .await +} + +/// Merge auxiliary files while retaining only rows owned by each source segment. +pub async fn merge_partial_vector_auxiliary_files_with_row_filters( + object_store: &lance_io::object_store::ObjectStore, + aux_paths: &[object_store::path::Path], + target_dir: &object_store::path::Path, + row_filters: &[OldIndexDataFilter], + progress: Arc, +) -> Result { + if aux_paths.len() != row_filters.len() { + return Err(Error::invalid_input(format!( + "Expected one row filter per auxiliary file, got {} files and {} filters", + aux_paths.len(), + row_filters.len() + ))); + } + merge_partial_vector_auxiliary_files_inner( + object_store, + aux_paths, + target_dir, + Some(row_filters), + progress, + ) + .await +} + +async fn merge_partial_vector_auxiliary_files_inner( + object_store: &lance_io::object_store::ObjectStore, + aux_paths: &[object_store::path::Path], + target_dir: &object_store::path::Path, + row_filters: Option<&[OldIndexDataFilter]>, + progress: Arc, ) -> Result { if aux_paths.is_empty() { return Err(Error::index( @@ -728,7 +880,7 @@ pub async fn merge_partial_vector_auxiliary_files( let mut dim: Option = None; let mut detected_index_type: Option = None; // Inherit file format version from the first shard (set on first iteration) - let mut format_version: Option = None; + let mut format_version: Option = None; // Prepare output path; we'll create writer once when we know schema let aux_out = target_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); @@ -771,6 +923,12 @@ pub async fn merge_partial_vector_auxiliary_files( ) .await?; let meta = reader.metadata(); + let idx_path = aux + .parent() + .unwrap_or_default() + .join(crate::INDEX_FILE_NAME); + let mut idx_reader: Option = None; + let mut idx_reader_checked = false; // Inherit format version from the first shard file if format_version.is_none() { @@ -795,45 +953,33 @@ pub async fn merge_partial_vector_auxiliary_files( // Detect index type (first iteration only) if detected_index_type.is_none() { // Try to derive precise type from sibling partial index.idx metadata if available - let idx_path = aux - .parent() - .unwrap_or_default() - .join(crate::INDEX_FILE_NAME); - if object_store.exists(&idx_path).await.unwrap_or(false) { - let fh2 = sched - .open_file(&idx_path, &CachedFileSize::unknown()) - .await?; - let idx_reader = V2Reader::try_open( - fh2, - None, - Arc::default(), - &lance_core::cache::LanceCache::no_cache(), - V2ReaderOptions::default(), - ) - .await?; - if let Some(idx_meta_json) = idx_reader + if !idx_reader_checked { + idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?; + idx_reader_checked = true; + } + if let Some(idx_reader) = idx_reader.as_ref() + && let Some(idx_meta_json) = idx_reader .metadata() .file_schema .metadata .get(INDEX_METADATA_SCHEMA_KEY) - { - let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?; - detected_index_type = Some(match idx_meta.index_type.as_str() { - "IVF_FLAT" => SupportedIvfIndexType::IvfFlat, - "IVF_PQ" => SupportedIvfIndexType::IvfPq, - "IVF_SQ" => SupportedIvfIndexType::IvfSq, - "IVF_RQ" => SupportedIvfIndexType::IvfRq, - "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat, - "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq, - "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq, - other => { - return Err(Error::index(format!( - "Unsupported index type in shard index.idx: {}", - other - ))); - } - }); - } + { + let idx_meta: IndexMetaSchema = serde_json::from_str(idx_meta_json)?; + detected_index_type = Some(match idx_meta.index_type.as_str() { + "IVF_FLAT" => SupportedIvfIndexType::IvfFlat, + "IVF_PQ" => SupportedIvfIndexType::IvfPq, + "IVF_SQ" => SupportedIvfIndexType::IvfSq, + "IVF_RQ" => SupportedIvfIndexType::IvfRq, + "IVF_HNSW_FLAT" => SupportedIvfIndexType::IvfHnswFlat, + "IVF_HNSW_PQ" => SupportedIvfIndexType::IvfHnswPq, + "IVF_HNSW_SQ" => SupportedIvfIndexType::IvfHnswSq, + other => { + return Err(Error::index(format!( + "Unsupported index type in shard index.idx: {}", + other + ))); + } + }); } // Fallback: infer from auxiliary schema if detected_index_type.is_none() { @@ -843,43 +989,60 @@ pub async fn merge_partial_vector_auxiliary_files( } // Read IVF lengths from global buffer - let ivf_idx: u32 = reader - .metadata() - .file_schema - .metadata - .get(IVF_METADATA_KEY) - .ok_or_else(|| Error::index("IVF meta missing".to_string()))? - .parse() - .map_err(|_| Error::index("IVF index parse error".to_string()))?; - let bytes = reader.read_global_buffer(ivf_idx).await?; - let pb_ivf: pb::Ivf = prost::Message::decode(bytes)?; + let pb_ivf = try_read_ivf_proto(&reader) + .await? + .ok_or_else(|| Error::index("IVF meta missing".to_string()))?; let lengths = pb_ivf.lengths.clone(); let nlist = lengths.len(); + let mut current_centroids = ivf_centroids_from_proto(&pb_ivf)?; + if current_centroids.is_none() { + if !idx_reader_checked { + idx_reader = open_sibling_index_reader(object_store, &sched, &idx_path).await?; + } + if let Some(idx_reader) = idx_reader.as_ref() + && let Some(index_ivf) = try_read_ivf_proto(idx_reader).await? + { + current_centroids = ivf_centroids_from_proto(&index_ivf)?; + } + } if nlist_opt.is_none() { nlist_opt = Some(nlist); accumulated_lengths = vec![0; nlist]; - // Try load centroids tensor if present - if let Some(tensor) = pb_ivf.centroids_tensor.as_ref() { - let arr = FixedSizeListArray::try_from(tensor)?; - first_centroids = Some(arr.clone()); + if let Some(arr) = current_centroids { let d0 = arr.value_length() as usize; if dim.is_none() { dim = Some(d0); } + first_centroids = Some(arr); } } else if nlist_opt.as_ref().map(|v| *v != nlist).unwrap_or(false) { return Err(Error::index( "IVF partition count mismatch across shards".to_string(), )); + } else { + match (&first_centroids, ¤t_centroids) { + (Some(reference), Some(candidate)) => { + ensure_fixed_size_list_compatible("IVF centroids", reference, candidate)?; + } + (Some(_), None) => { + return Err(Error::index("IVF centroids missing from shard".to_string())); + } + (None, Some(_)) => { + return Err(Error::index( + "IVF centroids missing from first shard".to_string(), + )); + } + (None, None) => {} + } } // Handle logic based on detected index type let idx_type = detected_index_type .ok_or_else(|| Error::index("Unable to detect index type".to_string()))?; - // Compute format version once; defaults to V2_0 if no shards processed yet - let fv = format_version.unwrap_or(LanceFileVersion::V2_0); + // Preserve the historical fallback while keeping the writer boundary exact. + let fv = format_version.unwrap_or(ConcreteFileVersion::V2_0); match idx_type { SupportedIvfIndexType::IvfSq => { @@ -985,6 +1148,11 @@ pub async fn merge_partial_vector_auxiliary_files( rq_meta_parsed.parse_buffer(rotate_mat_bytes)?; } validate_rq_num_bits(rq_meta_parsed.num_bits)?; + if rq_meta_parsed.packed { + return Err(Error::index(format!( + "Distributed RQ merge: source shard {idx} stores packed RQ codes; expected row-major distributed shard" + ))); + } let d0 = rq_meta_parsed.rotated_dim(); if d0 == 0 { @@ -1001,7 +1169,9 @@ pub async fn merge_partial_vector_auxiliary_files( if let Some(existing_rq) = rq_meta.as_ref() && (existing_rq.code_dim != rq_meta_parsed.code_dim || existing_rq.num_bits != rq_meta_parsed.num_bits - || existing_rq.rotation_type != rq_meta_parsed.rotation_type) + || existing_rq.rotation_type != rq_meta_parsed.rotation_type + || existing_rq.query_estimator != rq_meta_parsed.query_estimator + || existing_rq.fast_rotation_signs != rq_meta_parsed.fast_rotation_signs) { return Err(Error::index(format!( "Distributed RQ merge: structural mismatch across shards; first(code_dim={}, num_bits={}, rotation_type={:?}), current(code_dim={}, num_bits={}, rotation_type={:?})", @@ -1013,6 +1183,24 @@ pub async fn merge_partial_vector_auxiliary_files( rq_meta_parsed.rotation_type ))); } + if let Some(existing_rq) = rq_meta.as_ref() { + match (&existing_rq.rotate_mat, &rq_meta_parsed.rotate_mat) { + (Some(reference), Some(candidate)) => { + ensure_fixed_size_list_compatible( + "RQ rotation matrix", + reference, + candidate, + )?; + } + (Some(_), None) | (None, Some(_)) => { + return Err(Error::index( + "Distributed RQ merge: rotation matrix mismatch across shards" + .to_string(), + )); + } + (None, None) => {} + } + } if rq_meta.is_none() { rq_meta = Some(rq_meta_parsed.clone()); } @@ -1061,6 +1249,11 @@ pub async fn merge_partial_vector_auxiliary_files( }; let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json) .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?; + if pm.transposed { + return Err(Error::index(format!( + "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard" + ))); + } // Load codebook from global buffer if not present if pm.codebook.is_none() { let tensor_bytes = reader @@ -1100,18 +1293,11 @@ pub async fn merge_partial_vector_auxiliary_files( .codebook .as_ref() .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?; - if !fixed_size_list_equal(existing_cb, current_cb) { - const TOL: f32 = 1e-5; - if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) { - return Err(Error::index( - "PQ codebook content mismatch across shards".to_string(), - )); - } else { - log::warn!( - "PQ codebook differs within tolerance; proceeding with first shard codebook" - ); - } - } + ensure_fixed_size_list_compatible( + "PQ codebook content", + existing_cb, + current_cb, + )?; } if pq_meta.is_none() { pq_meta = Some(pm.clone()); @@ -1222,6 +1408,11 @@ pub async fn merge_partial_vector_auxiliary_files( }; let mut pm: ProductQuantizationMetadata = serde_json::from_str(&pm_json) .map_err(|e| Error::index(format!("PQ metadata parse error: {}", e)))?; + if pm.transposed { + return Err(Error::index(format!( + "Distributed PQ merge: source shard {idx} stores transposed PQ codes; expected row-major distributed shard" + ))); + } if pm.codebook.is_none() { let tensor_bytes = reader .read_global_buffer(pm.codebook_position as u32) @@ -1260,18 +1451,11 @@ pub async fn merge_partial_vector_auxiliary_files( .codebook .as_ref() .ok_or_else(|| Error::index("PQ codebook missing in shard".to_string()))?; - if !fixed_size_list_equal(existing_cb, current_cb) { - const TOL: f32 = 1e-5; - if !fixed_size_list_almost_equal(existing_cb, current_cb, TOL) { - return Err(Error::index( - "PQ codebook content mismatch across shards".to_string(), - )); - } else { - log::warn!( - "PQ codebook differs within tolerance; proceeding with first shard codebook" - ); - } - } + ensure_fixed_size_list_compatible( + "PQ codebook content", + existing_cb, + current_cb, + )?; } if pq_meta.is_none() { pq_meta = Some(pm.clone()); @@ -1357,6 +1541,7 @@ pub async fn merge_partial_vector_auxiliary_files( lengths, partition_offsets, total_rows: running_offset, + row_filter: row_filters.map(|filters| Arc::new(filters[idx].clone())), }); progress .stage_progress("read_shard_metadata", idx as u64 + 1) @@ -1383,6 +1568,7 @@ pub async fn merge_partial_vector_auxiliary_files( .stage_start("merge_partitions", Some(total_rows), "rows") .await?; let mut merged_rows = 0u64; + let mut merged_lengths = vec![0u32; nlist]; match idx_type_final { SupportedIvfIndexType::IvfPq | SupportedIvfIndexType::IvfHnswPq => { @@ -1398,22 +1584,20 @@ pub async fn merge_partial_vector_auxiliary_files( ); while let Some((pid, batches)) = shard_merge_reader.next_partition().await? { - if accumulated_lengths[pid] == 0 { + let partition_len = batches.iter().map(RecordBatch::num_rows).sum::(); + if partition_len == 0 { continue; } - if batches.is_empty() { - return Err(Error::index(format!( - "No merged batches found for non-empty partition {}", - pid - ))); - } let schema = batches[0].schema(); let partition_batch = concat_batches(&schema, batches.iter())?; if let Some(w) = v2w_opt.as_mut() { write_partition_rows_pq_transposed(w, partition_batch).await?; } - merged_rows = merged_rows.saturating_add(accumulated_lengths[pid] as u64); + merged_lengths[pid] = u32::try_from(partition_len).map_err(|_| { + Error::index(format!("Merged partition {pid} exceeds u32 row capacity")) + })?; + merged_rows = merged_rows.saturating_add(partition_len as u64); progress .stage_progress("merge_partitions", merged_rows) .await?; @@ -1430,15 +1614,10 @@ pub async fn merge_partial_vector_auxiliary_files( ); while let Some((pid, batches)) = shard_merge_reader.next_partition().await? { - if accumulated_lengths[pid] == 0 { + let partition_len = batches.iter().map(RecordBatch::num_rows).sum::(); + if partition_len == 0 { continue; } - if batches.is_empty() { - return Err(Error::index(format!( - "No merged batches found for non-empty partition {}", - pid - ))); - } // Shards written by older lance versions carry sequential ex // codes; normalize every batch to the blocked layout before @@ -1464,30 +1643,58 @@ pub async fn merge_partial_vector_auxiliary_files( if let Some(w) = v2w_opt.as_mut() { write_partition_rows_rq_packed(w, partition_batch).await?; } - merged_rows = merged_rows.saturating_add(accumulated_lengths[pid] as u64); + merged_lengths[pid] = u32::try_from(partition_len).map_err(|_| { + Error::index(format!("Merged partition {pid} exceeds u32 row capacity")) + })?; + merged_rows = merged_rows.saturating_add(partition_len as u64); progress .stage_progress("merge_partitions", merged_rows) .await?; } } _ => { - for (pid, total_part_len) in accumulated_lengths.iter().copied().enumerate().take(nlist) - { - for shard in shard_infos.iter() { - let part_len = shard.lengths[pid] as usize; - if part_len == 0 { + // FLAT, SQ, and their HNSW variants do not need whole-partition + // transforms. Stream one shard partition at a time so filtering + // never materializes a multi-partition window in memory. + for (pid, merged_length) in merged_lengths.iter_mut().enumerate() { + let mut partition_len = 0usize; + for shard in &shard_infos { + let source_len = shard.lengths[pid] as usize; + if source_len == 0 { continue; } let offset = shard.partition_offsets[pid]; - if let Some(w) = v2w_opt.as_mut() { - write_partition_rows(shard.reader.as_ref(), w, offset..offset + part_len) - .await?; - } + let writer = v2w_opt.as_mut().ok_or_else(|| { + Error::index("Failed to initialize unified writer".to_string()) + })?; + let written = if let Some(row_filter) = shard.row_filter.as_deref() { + write_filtered_partition_rows( + shard.reader.as_ref(), + writer, + offset..offset + source_len, + row_filter, + ) + .await? + } else { + write_partition_rows( + shard.reader.as_ref(), + writer, + offset..offset + source_len, + ) + .await?; + source_len + }; + partition_len = partition_len.checked_add(written).ok_or_else(|| { + Error::index(format!("Merged partition {pid} exceeds usize row capacity")) + })?; } - if total_part_len == 0 { + if partition_len == 0 { continue; } - merged_rows = merged_rows.saturating_add(total_part_len as u64); + *merged_length = u32::try_from(partition_len).map_err(|_| { + Error::index(format!("Merged partition {pid} exceeds u32 row capacity")) + })?; + merged_rows = merged_rows.saturating_add(partition_len as u64); progress .stage_progress("merge_partitions", merged_rows) .await?; @@ -1506,7 +1713,7 @@ pub async fn merge_partial_vector_auxiliary_files( } else { IvfStorageModel::empty() }; - for len in accumulated_lengths.iter() { + for len in merged_lengths.iter() { ivf_model.add_partition(*len); } let dt2 = distance_type.ok_or_else(|| Error::index("Distance type missing".to_string()))?; @@ -1554,6 +1761,31 @@ mod tests { lance_core::Result<()> ); + #[test] + fn test_uint8_fixed_size_list_compatibility() { + let values = (0_u8..16).collect::>(); + let reference = + FixedSizeListArray::try_new_from_values(UInt8Array::from(values.clone()), 8).unwrap(); + let matching = + FixedSizeListArray::try_new_from_values(UInt8Array::from(values.clone()), 8).unwrap(); + + ensure_fixed_size_list_compatible("IVF centroids", &reference, &matching).unwrap(); + + let mut differing_values = values; + differing_values[15] = 16; + let differing = + FixedSizeListArray::try_new_from_values(UInt8Array::from(differing_values), 8).unwrap(); + let error = + ensure_fixed_size_list_compatible("IVF centroids", &reference, &differing).unwrap_err(); + + assert!(matches!(&error, Error::Index { .. })); + assert!( + error + .to_string() + .contains("IVF centroids mismatch across shards") + ); + } + async fn write_flat_partial_aux( store: &ObjectStore, aux_path: &Path, @@ -1561,6 +1793,7 @@ mod tests { lengths: &[u32], base_row_id: u64, distance_type: DistanceType, + file_version: ConcreteFileVersion, ) -> Result { let arrow_schema = ArrowSchema::new(vec![ (*ROW_ID_FIELD).clone(), @@ -1572,7 +1805,8 @@ mod tests { ]); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::create_writer( + file_version, writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -1642,7 +1876,7 @@ mod tests { ]); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -1703,12 +1937,28 @@ mod tests { let lengths1 = vec![1_u32, 2_u32]; let dim = 2_i32; - write_flat_partial_aux(&object_store, &aux0, dim, &lengths0, 0, DistanceType::L2) - .await - .unwrap(); - write_flat_partial_aux(&object_store, &aux1, dim, &lengths1, 100, DistanceType::L2) - .await - .unwrap(); + write_flat_partial_aux( + &object_store, + &aux0, + dim, + &lengths0, + 0, + DistanceType::L2, + ConcreteFileVersion::V2_2, + ) + .await + .unwrap(); + write_flat_partial_aux( + &object_store, + &aux1, + dim, + &lengths1, + 100, + DistanceType::L2, + ConcreteFileVersion::V2_1, + ) + .await + .unwrap(); let progress = Arc::new(RecordingProgress::default()); merge_partial_vector_auxiliary_files( @@ -1812,6 +2062,7 @@ mod tests { .await .unwrap(); let meta = reader.metadata(); + assert_eq!(meta.version(), ConcreteFileVersion::V2_2); // Validate IVF lengths aggregation. let ivf_idx: u32 = meta @@ -1858,6 +2109,95 @@ mod tests { assert_eq!(total_rows, expected_total); } + #[tokio::test] + async fn test_merge_ivf_flat_filters_each_source_by_ownership() { + let object_store = ObjectStore::memory(); + let index_dir = Path::from("index/uuid"); + let aux0 = index_dir + .clone() + .join("stale") + .join(INDEX_AUXILIARY_FILE_NAME); + let aux1 = index_dir + .clone() + .join("fresh") + .join(INDEX_AUXILIARY_FILE_NAME); + let lengths = vec![2_u32, 1_u32]; + + write_flat_partial_aux( + &object_store, + &aux0, + 2, + &lengths, + 0, + DistanceType::L2, + ConcreteFileVersion::V2_1, + ) + .await + .unwrap(); + write_flat_partial_aux( + &object_store, + &aux1, + 2, + &lengths, + 100, + DistanceType::L2, + ConcreteFileVersion::V2_1, + ) + .await + .unwrap(); + + merge_partial_vector_auxiliary_files_with_row_filters( + &object_store, + &[aux0, aux1], + &index_dir, + &[ + OldIndexDataFilter::Fragments { + to_keep: roaring::RoaringBitmap::new(), + to_remove: roaring::RoaringBitmap::new(), + }, + OldIndexDataFilter::RowIds(lance_select::RowAddrTreeMap::from_iter(100_u64..103)), + ], + Arc::new(RecordingProgress::default()), + ) + .await + .unwrap(); + + let aux_out = index_dir.join(INDEX_AUXILIARY_FILE_NAME); + let sched = ScanScheduler::new( + Arc::new(object_store.clone()), + SchedulerConfig::max_bandwidth(&object_store), + ); + let reader = V2Reader::try_open( + sched + .open_file(&aux_out, &CachedFileSize::unknown()) + .await + .unwrap(), + None, + Arc::default(), + &lance_core::cache::LanceCache::no_cache(), + V2ReaderOptions::default(), + ) + .await + .unwrap(); + let merged_ivf = try_read_ivf_proto(&reader).await.unwrap().unwrap(); + assert_eq!(merged_ivf.lengths, lengths); + + let mut total_rows = 0; + let mut stream = reader + .read_stream( + lance_io::ReadBatchParams::RangeFull, + u32::MAX, + 4, + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + .unwrap(); + while let Some(batch) = stream.next().await { + total_rows += batch.unwrap().num_rows(); + } + assert_eq!(total_rows, 3, "stale source rows must not be copied"); + } + #[tokio::test] async fn test_merge_distance_type_mismatch() { let object_store = ObjectStore::memory(); @@ -1871,9 +2211,17 @@ mod tests { let lengths = vec![2_u32, 2_u32]; let dim = 2_i32; - write_flat_partial_aux(&object_store, &aux0, dim, &lengths, 0, DistanceType::L2) - .await - .unwrap(); + write_flat_partial_aux( + &object_store, + &aux0, + dim, + &lengths, + 0, + DistanceType::L2, + ConcreteFileVersion::V2_1, + ) + .await + .unwrap(); write_flat_partial_aux( &object_store, &aux1, @@ -1881,6 +2229,7 @@ mod tests { &lengths, 100, DistanceType::Cosine, + ConcreteFileVersion::V2_1, ) .await .unwrap(); @@ -1976,6 +2325,7 @@ mod tests { base_row_id: u64, distance_type: DistanceType, codebook: &FixedSizeListArray, + transposed: bool, ) -> Result { let num_bytes = if nbits == 4 { // Two 4-bit codes per byte. @@ -1997,7 +2347,7 @@ mod tests { ]); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -2014,7 +2364,7 @@ mod tests { dimension, codebook: Some(codebook.clone()), codebook_tensor: Vec::new(), - transposed: true, + transposed, }; let codebook_tensor: pb::Tensor = pb::Tensor::try_from(codebook)?; @@ -2109,7 +2459,7 @@ mod tests { let arrow_schema = ArrowSchema::new(fields); let writer = store.create(aux_path).await?; - let mut v2w = V2Writer::try_new( + let mut v2w = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&arrow_schema)?, V2WriterOptions::default(), @@ -2225,6 +2575,7 @@ mod tests { 0, DistanceType::L2, &codebook, + false, ) .await .unwrap(); @@ -2239,6 +2590,7 @@ mod tests { 1_000, DistanceType::L2, &codebook, + false, ) .await .unwrap(); @@ -2319,6 +2671,66 @@ mod tests { assert!(fixed_size_list_equal(&codebook, &merged_codebook)); } + #[tokio::test] + async fn test_merge_ivf_pq_rejects_transposed_source_shard() { + let object_store = ObjectStore::memory(); + let index_dir = Path::from("index/uuid_pq_transposed"); + + let partial0 = index_dir.clone().join("partial_0"); + let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME); + let lengths = vec![2_u32, 1_u32]; + + let nbits = 4_u32; + let num_sub_vectors = 2_usize; + let dimension = 8_usize; + let num_centroids = 1_usize << nbits; + let num_codebook_vectors = num_centroids * num_sub_vectors; + let total_values = num_codebook_vectors * dimension; + let values = Float32Array::from_iter((0..total_values).map(|v| v as f32)); + let codebook = FixedSizeListArray::try_new_from_values(values, dimension as i32).unwrap(); + + write_pq_partial_aux( + &object_store, + &aux0, + nbits, + num_sub_vectors, + dimension, + &lengths, + 0, + DistanceType::L2, + &codebook, + true, + ) + .await + .unwrap(); + + let res = merge_partial_vector_auxiliary_files( + &object_store, + std::slice::from_ref(&aux0), + &index_dir, + crate::progress::noop_progress(), + ) + .await; + match res { + Err(Error::Index { message, .. }) => { + assert!( + message.contains("source shard 0"), + "unexpected message: {}", + message + ); + assert!( + message.contains("transposed PQ codes"), + "unexpected message: {}", + message + ); + } + other => panic!( + "expected Error::Index for transposed PQ source shard, got {:?}", + other + ), + } + } + #[tokio::test] async fn test_merge_ivf_rq_success() { let object_store = ObjectStore::memory(); @@ -2455,6 +2867,64 @@ mod tests { assert_eq!(total_rows, expected_total); } + #[tokio::test] + async fn test_merge_ivf_rq_rejects_packed_source_shard() { + let object_store = ObjectStore::memory(); + let index_dir = Path::from("index/uuid_rq_packed"); + + let partial0 = index_dir.clone().join("partial_0"); + let aux0 = partial0.clone().join(INDEX_AUXILIARY_FILE_NAME); + let lengths = vec![2_u32, 1_u32]; + + let rq_meta = RabitQuantizationMetadata { + rotate_mat: None, + rotate_mat_position: None, + fast_rotation_signs: Some(vec![0xAA; 2]), + rotation_type: RQRotationType::Fast, + code_dim: 16, + num_bits: 1, + packed: true, + query_estimator: RabitQueryEstimator::RawQuery, + }; + + write_rq_partial_aux( + &object_store, + &aux0, + &rq_meta, + &lengths, + 0, + DistanceType::L2, + ) + .await + .unwrap(); + + let res = merge_partial_vector_auxiliary_files( + &object_store, + std::slice::from_ref(&aux0), + &index_dir, + crate::progress::noop_progress(), + ) + .await; + match res { + Err(Error::Index { message, .. }) => { + assert!( + message.contains("source shard 0"), + "unexpected message: {}", + message + ); + assert!( + message.contains("packed RQ codes"), + "unexpected message: {}", + message + ); + } + other => panic!( + "expected Error::Index for packed RQ source shard, got {:?}", + other + ), + } + } + #[tokio::test] async fn test_merge_ivf_rq_multi_bit_preserves_split_columns() { let object_store = ObjectStore::memory(); @@ -2612,6 +3082,7 @@ mod tests { 0, DistanceType::L2, &codebook0, + false, ) .await .unwrap(); @@ -2626,6 +3097,7 @@ mod tests { 1_000, DistanceType::L2, &codebook1, + false, ) .await .unwrap(); @@ -2691,6 +3163,7 @@ mod tests { 0, DistanceType::L2, &codebook, + false, ) .await .unwrap(); @@ -2706,6 +3179,7 @@ mod tests { 1_000, DistanceType::L2, &codebook, + false, ) .await .unwrap(); diff --git a/rust/lance-index/src/vector/flat/index.rs b/rust/lance-index/src/vector/flat/index.rs index cc6f6d021eb..647d375d7ae 100644 --- a/rust/lance-index/src/vector/flat/index.rs +++ b/rust/lance-index/src/vector/flat/index.rs @@ -13,7 +13,7 @@ use arrow_array::{Array, ArrayRef, Float32Array, RecordBatch, UInt64Array}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID_FIELD, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_linalg::distance::DistanceType; use serde::{Deserialize, Serialize}; @@ -66,6 +66,11 @@ static ANN_SEARCH_SCHEMA: LazyLock = LazyLock::new(|| { .into() }); +/// Marker schema for the flat index, which stores no data of its own. +static FLAT_SCHEMA: LazyLock = LazyLock::new(|| { + Schema::new(vec![Field::new("__flat_marker", DataType::UInt64, false)]).into() +}); + #[derive(Default)] pub struct FlatQueryParams { lower_bound: Option, @@ -98,7 +103,7 @@ impl IvfSubIndex for FlatIndex { } fn schema() -> arrow_schema::SchemaRef { - Schema::new(vec![Field::new("__flat_marker", DataType::UInt64, false)]).into() + FLAT_SCHEMA.clone() } fn search( @@ -329,7 +334,7 @@ pub struct FlatMetadata { #[async_trait::async_trait] impl QuantizerMetadata for FlatMetadata { - async fn load(_: &PreviousFileReader) -> Result { + async fn load(_: &V1FileReader) -> Result { unimplemented!("Flat will be used in new index builder which doesn't require this") } } @@ -518,6 +523,15 @@ mod tests { use crate::metrics::NoOpMetricsCollector; use crate::prefilter::NoFilter; + #[test] + fn test_schema_is_initialized_once() { + // The subindex schema is requested per call, so it is shared rather + // than rebuilt. Pointer equality is what distinguishes a shared schema + // from an equal-but-freshly-allocated one. + assert!(Arc::ptr_eq(&FlatIndex::schema(), &FlatIndex::schema())); + assert_eq!(FlatIndex::schema().field(0).name(), "__flat_marker"); + } + struct MaskPreFilter { mask: Arc, } @@ -570,7 +584,7 @@ mod tests { .zip(dists.values().iter()) .map(|(row_id, dist)| (*row_id, *dist)) .collect::>(); - results.sort_by(|left, right| left.0.cmp(&right.0)); + results.sort_by_key(|left| left.0); results } @@ -579,7 +593,7 @@ mod tests { .into_iter() .map(|node| (node.id, node.dist.0)) .collect::>(); - results.sort_by(|left, right| left.0.cmp(&right.0)); + results.sort_by_key(|left| left.0); results } diff --git a/rust/lance-index/src/vector/flat/storage.rs b/rust/lance-index/src/vector/flat/storage.rs index c3ec30d5086..1de4c4387fb 100644 --- a/rust/lance-index/src/vector/flat/storage.rs +++ b/rust/lance-index/src/vector/flat/storage.rs @@ -4,7 +4,8 @@ use std::{borrow::Cow, sync::Arc}; use super::index::FlatMetadata; -use crate::frag_reuse::FragReuseIndex; +use crate::frag_reuse::{FragReuseIndex, FragReuseIndexHandle}; +use crate::scalar::RowIdRemapper; use crate::vector::quantizer::QuantizerStorage; use crate::vector::storage::{DistCalculator, VectorStore}; use crate::vector::utils::do_prefetch; @@ -13,18 +14,31 @@ use arrow::compute::concat_batches; use arrow::datatypes::{Float16Type, Float64Type, UInt8Type}; use arrow_array::ArrowPrimitiveType; use arrow_array::{ - Array, ArrayRef, FixedSizeListArray, RecordBatch, UInt64Array, + Array, ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, UInt64Array, types::{Float32Type, UInt64Type}, }; use arrow_schema::{DataType, SchemaRef}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_linalg::distance::hamming::hamming; -use lance_linalg::distance::{Cosine, DistanceType, Dot, L2}; +use lance_linalg::distance::{Cosine, DistanceType, Dot, L2, Normalize, norm_l2_fsl}; pub const FLAT_COLUMN: &str = "flat"; +/// Per-vector L2 norms cached for Cosine distance, so `cosine_with_norms` can +/// skip recomputing each stored vector's norm per comparison. `None` for other +/// metrics and for value types without a norm kernel. +fn cosine_norms_cache( + vectors: &FixedSizeListArray, + distance_type: DistanceType, +) -> Option> { + if distance_type != DistanceType::Cosine { + return None; + } + norm_l2_fsl(vectors).ok().map(Arc::new) +} + /// All data are stored in memory #[derive(Debug, Clone)] pub struct FlatFloatStorage { @@ -35,11 +49,17 @@ pub struct FlatFloatStorage { // helper fields pub(super) row_ids: Arc, vectors: Arc, + /// Per-vector L2 norms for Cosine. `None` for other metrics. + norms: Option>, } impl DeepSizeOf for FlatFloatStorage { fn deep_size_of_children(&self, _: &mut lance_core::deepsize::Context) -> usize { - self.batch.get_array_memory_size() + let mut size = self.batch.get_array_memory_size(); + if let Some(norms) = &self.norms { + size += norms.get_array_memory_size(); + } + size } } @@ -52,6 +72,17 @@ impl QuantizerStorage for FlatFloatStorage { metadata: &Self::Metadata, distance_type: DistanceType, frag_reuse_index: Option>, + ) -> Result { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::try_from_batch_with_remapper(batch, metadata, distance_type, frag_reuse_index) + } + + fn try_from_batch_with_remapper( + batch: RecordBatch, + metadata: &Self::Metadata, + distance_type: DistanceType, + frag_reuse_index: Option>, ) -> Result { let batch = if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() { frag_reuse_index_ref.remap_row_ids_record_batch(batch, 0)? @@ -73,12 +104,14 @@ impl QuantizerStorage for FlatFloatStorage { .as_fixed_size_list() .clone(), ); + let norms = cosine_norms_cache(&vectors, distance_type); Ok(Self { metadata: metadata.clone(), batch, distance_type, row_ids, vectors, + norms, }) } @@ -87,7 +120,7 @@ impl QuantizerStorage for FlatFloatStorage { } async fn load_partition( - _: &PreviousFileReader, + _: &V1FileReader, _: std::ops::Range, _: DistanceType, _: &Self::Metadata, @@ -109,6 +142,7 @@ impl FlatFloatStorage { ]) .unwrap(); + let norms = cosine_norms_cache(&vectors, distance_type); Self { metadata: FlatMetadata { dim: vectors.value_length() as usize, @@ -117,6 +151,7 @@ impl FlatFloatStorage { distance_type, row_ids, vectors, + norms, } } @@ -134,7 +169,7 @@ impl VectorStore for FlatFloatStorage { fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result { // TODO: use chunked storage - let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?; + let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?; let mut storage = self.clone(); storage.row_ids = Arc::new( new_batch @@ -150,6 +185,7 @@ impl VectorStore for FlatFloatStorage { .as_fixed_size_list() .clone(), ); + storage.norms = cosine_norms_cache(&storage.vectors, storage.distance_type); storage.batch = new_batch; Ok(storage) } @@ -179,11 +215,13 @@ impl VectorStore for FlatFloatStorage { } fn dist_calculator(&self, query: ArrayRef, _dist_q_c: f32) -> Self::DistanceCalculator<'_> { - Self::DistanceCalculator::new(self.vectors.as_ref(), query, self.distance_type) + let norms = self.norms.as_ref().map(|n| n.values().as_ref()); + Self::DistanceCalculator::new(self.vectors.as_ref(), query, self.distance_type, norms) } fn dist_calculator_from_id(&self, id: u32) -> Self::DistanceCalculator<'_> { - Self::DistanceCalculator::new_from_id(self.vectors.as_ref(), id, self.distance_type) + let norms = self.norms.as_ref().map(|n| n.values().as_ref()); + Self::DistanceCalculator::new_from_id(self.vectors.as_ref(), id, self.distance_type, norms) } } @@ -214,6 +252,17 @@ impl QuantizerStorage for FlatBinStorage { metadata: &Self::Metadata, distance_type: DistanceType, frag_reuse_index: Option>, + ) -> Result { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::try_from_batch_with_remapper(batch, metadata, distance_type, frag_reuse_index) + } + + fn try_from_batch_with_remapper( + batch: RecordBatch, + metadata: &Self::Metadata, + distance_type: DistanceType, + frag_reuse_index: Option>, ) -> Result { let batch = if let Some(frag_reuse_index_ref) = frag_reuse_index.as_ref() { frag_reuse_index_ref.remap_row_ids_record_batch(batch, 0)? @@ -249,7 +298,7 @@ impl QuantizerStorage for FlatBinStorage { } async fn load_partition( - _: &PreviousFileReader, + _: &V1FileReader, _: std::ops::Range, _: DistanceType, _: &Self::Metadata, @@ -296,7 +345,7 @@ impl VectorStore for FlatBinStorage { fn append_batch(&self, batch: RecordBatch, _vector_column: &str) -> Result { // TODO: use chunked storage - let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch].into_iter())?; + let new_batch = concat_batches(&batch.schema(), vec![&self.batch, &batch])?; let mut storage = self.clone(); storage.row_ids = Arc::new( new_batch @@ -353,6 +402,8 @@ pub struct FlatDistanceCal<'a, T: ArrowPrimitiveType> { vectors: &'a [T::Native], query: Cow<'a, [T::Native]>, dimension: usize, + query_norm: Option, + vector_norms: Option<&'a [f32]>, #[allow(clippy::type_complexity)] distance_fn: fn(&[T::Native], &[T::Native]) -> f32, } @@ -362,27 +413,60 @@ where T: ArrowPrimitiveType, T::Native: L2 + Cosine + Dot, { - fn new(vectors: &'a FixedSizeListArray, query: ArrayRef, distance_type: DistanceType) -> Self { + fn new( + vectors: &'a FixedSizeListArray, + query: ArrayRef, + distance_type: DistanceType, + vector_norms: Option<&'a [f32]>, + ) -> Self { + debug_assert!( + vector_norms.is_none_or(|norms| norms.len() == vectors.len()), + "expected one cached norm per vector" + ); // Gained significant performance improvement by using strong typed primitive slice. let flat_array = vectors.values().as_primitive::(); let dimension = vectors.value_length() as usize; + let query: Cow<'a, [T::Native]> = Cow::Owned(query.as_primitive::().values().to_vec()); + // Only cache the query norm alongside the stored norms. + let query_norm = (distance_type == DistanceType::Cosine && vector_norms.is_some()) + .then(|| T::Native::norm_l2(query.as_ref())); Self { vectors: flat_array.values(), - query: Cow::Owned(query.as_primitive::().values().to_vec()), + query, dimension, + query_norm, + vector_norms, distance_fn: distance_type.func(), } } - fn new_from_id(vectors: &'a FixedSizeListArray, id: u32, distance_type: DistanceType) -> Self { + fn new_from_id( + vectors: &'a FixedSizeListArray, + id: u32, + distance_type: DistanceType, + vector_norms: Option<&'a [f32]>, + ) -> Self { + debug_assert!( + vector_norms.is_none_or(|norms| norms.len() == vectors.len()), + "expected one cached norm per vector" + ); let flat_array = vectors.values().as_primitive::(); let dimension = vectors.value_length() as usize; let vectors = flat_array.values(); let id = id as usize; + let query: Cow<'a, [T::Native]> = + Cow::Borrowed(&vectors[dimension * id..dimension * (id + 1)]); + // The query is stored vector `id`, so reuse its cached norm. + let query_norm = match (distance_type, vector_norms) { + (DistanceType::Cosine, Some(norms)) => Some(norms[id]), + _ => None, + }; Self { vectors, - query: Cow::Borrowed(&vectors[dimension * id..dimension * (id + 1)]), + query, dimension, + query_norm, + vector_norms, distance_fn: distance_type.func(), } } @@ -402,6 +486,8 @@ impl<'a> FlatDistanceCal<'a, UInt8Type> { vectors: flat_array.values(), query: Cow::Owned(query.as_primitive::().values().to_vec()), dimension, + query_norm: None, + vector_norms: None, distance_fn: hamming, } } @@ -419,6 +505,8 @@ impl<'a> FlatDistanceCal<'a, UInt8Type> { vectors, query: Cow::Borrowed(&vectors[dimension * id..dimension * (id + 1)]), dimension, + query_norm: None, + vector_norms: None, distance_fn: hamming, } } @@ -431,20 +519,45 @@ impl FlatDistanceCal<'_, T> { } } -impl DistCalculator for FlatDistanceCal<'_, T> { +impl DistCalculator for FlatDistanceCal<'_, T> +where + T::Native: Cosine, +{ #[inline] fn distance(&self, id: u32) -> f32 { let query = self.query.as_ref(); let vector = self.get_vector(id); - (self.distance_fn)(query, vector) + match (self.query_norm, self.vector_norms) { + (Some(x_norm), Some(norms)) => { + T::Native::cosine_with_norms(query, x_norm, norms[id as usize], vector) + } + _ => (self.distance_fn)(query, vector), + } } fn distance_all(&self, _k_hint: usize) -> Vec { let query = self.query.as_ref(); - self.vectors - .chunks_exact(self.dimension) - .map(|vector| (self.distance_fn)(query, vector)) - .collect() + match (self.query_norm, self.vector_norms) { + (Some(x_norm), Some(norms)) => { + debug_assert_eq!( + norms.len(), + self.vectors.len() / self.dimension, + "cached norms must cover every vector, otherwise `zip` silently truncates" + ); + self.vectors + .chunks_exact(self.dimension) + .zip(norms) + .map(|(vector, &y_norm)| { + T::Native::cosine_with_norms(query, x_norm, y_norm, vector) + }) + .collect() + } + _ => self + .vectors + .chunks_exact(self.dimension) + .map(|vector| (self.distance_fn)(query, vector)) + .collect(), + } } #[inline] @@ -461,43 +574,59 @@ pub enum FlatFloatDistanceCalc<'a> { } impl<'a> FlatFloatDistanceCalc<'a> { - fn new(vectors: &'a FixedSizeListArray, query: ArrayRef, distance_type: DistanceType) -> Self { + fn new( + vectors: &'a FixedSizeListArray, + query: ArrayRef, + distance_type: DistanceType, + vector_norms: Option<&'a [f32]>, + ) -> Self { match vectors.value_type() { DataType::Float16 => Self::Float16(FlatDistanceCal::::new( vectors, query, distance_type, + vector_norms, )), DataType::Float32 => Self::Float32(FlatDistanceCal::::new( vectors, query, distance_type, + vector_norms, )), DataType::Float64 => Self::Float64(FlatDistanceCal::::new( vectors, query, distance_type, + vector_norms, )), dt => panic!("flat float storage does not support data type {dt}"), } } - fn new_from_id(vectors: &'a FixedSizeListArray, id: u32, distance_type: DistanceType) -> Self { + fn new_from_id( + vectors: &'a FixedSizeListArray, + id: u32, + distance_type: DistanceType, + vector_norms: Option<&'a [f32]>, + ) -> Self { match vectors.value_type() { DataType::Float16 => Self::Float16(FlatDistanceCal::::new_from_id( vectors, id, distance_type, + vector_norms, )), DataType::Float32 => Self::Float32(FlatDistanceCal::::new_from_id( vectors, id, distance_type, + vector_norms, )), DataType::Float64 => Self::Float64(FlatDistanceCal::::new_from_id( vectors, id, distance_type, + vector_norms, )), dt => panic!("flat float storage does not support data type {dt}"), } @@ -534,9 +663,10 @@ impl DistCalculator for FlatFloatDistanceCalc<'_> { mod tests { use super::*; - use arrow_array::{Float16Array, Float64Array}; + use arrow_array::{Float16Array, Float32Array, Float64Array}; use half::f16; use lance_arrow::FixedSizeListArrayExt; + use rstest::rstest; fn make_f16_storage() -> FlatFloatStorage { let values = Float16Array::from(vec![ @@ -583,4 +713,212 @@ mod tests { assert_eq!(distances[0], 0.0); assert!((distances[1] - 25.0).abs() < 1e-6); } + + fn make_flat_test_batch(vectors: FixedSizeListArray, first_row_id: u64) -> RecordBatch { + let num_rows = vectors.len() as u64; + RecordBatch::try_from_iter(vec![ + ( + ROW_ID, + Arc::new(UInt64Array::from_iter_values( + first_row_id..first_row_id + num_rows, + )) as ArrayRef, + ), + (FLAT_COLUMN, Arc::new(vectors) as ArrayRef), + ]) + .unwrap() + } + + /// Assert that the cached-norm Cosine path agrees with the uncached + /// `Cosine::cosine` reference for both `distance` and `distance_all`. + fn assert_cosine_matches_uncached(vectors: FixedSizeListArray, query: ArrayRef) + where + T: ArrowPrimitiveType, + T::Native: L2 + Cosine + Dot, + { + let dim = vectors.value_length() as usize; + let values = vectors.values().as_primitive::().values().to_vec(); + let query_values = query.as_primitive::().values().to_vec(); + + let storage = FlatFloatStorage::new(vectors, DistanceType::Cosine); + let calc = storage.dist_calculator(query, 0.0); + let all = calc.distance_all(storage.len()); + assert_eq!(all.len(), storage.len()); + + for (id, vector) in values.chunks_exact(dim).enumerate() { + let expected = T::Native::cosine(&query_values, vector); + assert!( + (all[id] - expected).abs() < 1e-5, + "distance_all[{id}]: {} vs uncached {expected}", + all[id] + ); + assert!( + (calc.distance(id as u32) - expected).abs() < 1e-5, + "distance({id}): {} vs uncached {expected}", + calc.distance(id as u32) + ); + } + } + + #[test] + fn test_cosine_cached_norms_match_uncached_cosine() { + // Caching the stored vectors' norms must not change the distances that + // `Cosine::cosine` would compute inline, for any supported value type. + let f32_vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]), + 4, + ) + .unwrap(); + assert_cosine_matches_uncached::( + f32_vectors, + Arc::new(Float32Array::from(vec![0.5, 0.5, 0.5, 0.5])), + ); + + let f16_values: Vec = [1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + .iter() + .map(|&v| f16::from_f32(v)) + .collect(); + let f16_vectors = + FixedSizeListArray::try_new_from_values(Float16Array::from(f16_values), 4).unwrap(); + assert_cosine_matches_uncached::( + f16_vectors, + Arc::new(Float16Array::from(vec![f16::from_f32(0.5); 4])), + ); + + let f64_vectors = FixedSizeListArray::try_new_from_values( + Float64Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]), + 4, + ) + .unwrap(); + assert_cosine_matches_uncached::( + f64_vectors, + Arc::new(Float64Array::from(vec![0.5, 0.5, 0.5, 0.5])), + ); + } + + #[rstest] + #[case::l2(DistanceType::L2, false)] + #[case::dot(DistanceType::Dot, false)] + #[case::cosine(DistanceType::Cosine, true)] + fn test_norms_cached_only_for_cosine( + #[case] distance_type: DistanceType, + #[case] expect_norms: bool, + ) { + let values = Float32Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); + let vectors = FixedSizeListArray::try_new_from_values(values, 4).unwrap(); + let batch = make_flat_test_batch(vectors.clone(), 0); + let metadata = FlatMetadata { dim: 4 }; + + let loaded = + FlatFloatStorage::try_from_batch(batch, &metadata, distance_type, None).unwrap(); + assert_eq!(loaded.distance_type(), distance_type); + assert_eq!(loaded.norms.is_some(), expect_norms); + + // `new` is on the build path (see `HnswBuilder::build`) and must agree. + let built = FlatFloatStorage::new(vectors, distance_type); + assert_eq!(built.norms.is_some(), expect_norms); + } + + #[test] + fn test_append_batch_recomputes_norms() { + // A stale norms cache would silently truncate `distance_all` via `zip`, + // so appending must extend it to cover the new rows. + let head = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![1.0, 2.0, 3.0, 4.0]), + 4, + ) + .unwrap(); + let storage = FlatFloatStorage::new(head, DistanceType::Cosine); + assert_eq!(storage.norms.as_ref().unwrap().len(), 1); + + let tail = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0]), + 4, + ) + .unwrap(); + let appended = storage + .append_batch(make_flat_test_batch(tail, 1), FLAT_COLUMN) + .unwrap(); + + assert_eq!(appended.len(), 3); + let norms = appended.norms.as_ref().expect("norms kept after append"); + assert_eq!(norms.len(), 3); + for (id, expected) in [ + (0, 30.0f32.sqrt()), + (1, 174.0f32.sqrt()), + (2, 446.0f32.sqrt()), + ] { + assert!( + (norms.value(id) - expected).abs() < 1e-4, + "norms[{id}]: {} vs {expected}", + norms.value(id) + ); + } + + let query: ArrayRef = Arc::new(Float32Array::from(vec![0.5, 0.5, 0.5, 0.5])); + assert_eq!( + appended.dist_calculator(query, 0.0).distance_all(3).len(), + 3 + ); + } + + #[test] + fn test_dist_calculator_from_id_reuses_cached_norms() { + // HNSW builds graphs through `dist_calculator_from_id`; the query is a + // stored vector, so its norm comes from the cache instead of `norm_l2`. + let values = Float32Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); + let vectors = FixedSizeListArray::try_new_from_values(values, 4).unwrap(); + let storage = FlatFloatStorage::new(vectors, DistanceType::Cosine); + + let calc = storage.dist_calculator_from_id(1); + // Self-distance of a vector against itself is ~0 under cosine. + assert!(calc.distance(1).abs() < 1e-6, "got {}", calc.distance(1)); + + let expected = f32::cosine(&[5.0, 6.0, 7.0, 8.0], &[1.0, 2.0, 3.0, 4.0]); + assert!( + (calc.distance(0) - expected).abs() < 1e-6, + "{} vs {expected}", + calc.distance(0) + ); + } + + #[test] + fn normalized_f16_cosine_keeps_self_match_at_zero_lower_bound() { + let values = Float16Array::from(vec![ + f16::from_f32(7.0), + f16::from_f32(47.0), + f16::from_f32(13.0), + ]); + let raw = FixedSizeListArray::try_new_from_values(values, 3).unwrap(); + let normalized = lance_linalg::kernels::normalize_fsl(&raw).unwrap(); + let query = normalized.value(0); + let batch = make_flat_test_batch(normalized, 0); + let storage = FlatFloatStorage::try_from_batch( + batch, + &FlatMetadata { dim: 3 }, + DistanceType::Cosine, + None, + ) + .unwrap(); + + let distance = storage.dist_calculator(query, 0.0).distance(0); + assert!( + distance >= 0.0, + "lower_bound=0 would drop self-match: {distance}" + ); + } + + #[test] + fn test_deep_size_accounts_for_cached_norms() { + let values = Float32Array::from(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); + let vectors = FixedSizeListArray::try_new_from_values(values, 4).unwrap(); + + let cosine = FlatFloatStorage::new(vectors.clone(), DistanceType::Cosine); + let l2 = FlatFloatStorage::new(vectors, DistanceType::L2); + assert!( + cosine.deep_size_of() > l2.deep_size_of(), + "cosine storage must report the cached norms: {} vs {}", + cosine.deep_size_of(), + l2.deep_size_of() + ); + } } diff --git a/rust/lance-index/src/vector/flat/transform.rs b/rust/lance-index/src/vector/flat/transform.rs index 75a465ce262..b2358ccd1eb 100644 --- a/rust/lance-index/src/vector/flat/transform.rs +++ b/rust/lance-index/src/vector/flat/transform.rs @@ -26,7 +26,7 @@ impl FlatTransformer { impl Transformer for FlatTransformer { #[instrument(name = "FlatTransformer::transform", level = "debug", skip_all)] - fn transform(&self, batch: &RecordBatch) -> crate::Result { + fn transform(&self, batch: &RecordBatch) -> lance_core::Result { let input_arr = batch .column_by_name(&self.input_column) .ok_or(Error::index(format!( @@ -45,3 +45,109 @@ impl Transformer for FlatTransformer { Ok(batch) } } + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::Arc; + + use arrow::buffer::NullBuffer; + use arrow_array::{Array, FixedSizeListArray, Float32Array, Int32Array}; + use arrow_schema::{DataType, Schema}; + use lance_arrow::FixedSizeListArrayExt; + use rstest::rstest; + + const DIM: i32 = 4; + const ROWS: usize = 8; + + fn batch() -> RecordBatch { + let values = Float32Array::from_iter((0..(DIM as usize * ROWS)).map(|v| v as f32)); + let vectors = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM).unwrap()); + let schema = Schema::new(vec![ + Field::new("vec", vectors.data_type().clone(), true), + Field::new("other", DataType::Int32, false), + ]); + RecordBatch::try_new( + Arc::new(schema), + vec![ + vectors, + Arc::new(Int32Array::from_iter_values(0..ROWS as i32)), + ], + ) + .unwrap() + } + + #[test] + fn test_flat_transform_renames_vector_column() { + let input = batch(); + let output = FlatTransformer::new("vec").transform(&input).unwrap(); + + assert!(output.column_by_name("vec").is_none()); + assert!( + output.column_by_name("other").is_some(), + "unrelated columns should survive" + ); + assert_eq!(output.num_rows(), ROWS); + + let flat = output.column_by_name(FLAT_COLUMN).unwrap(); + assert_eq!( + flat.data_type(), + input.column_by_name("vec").unwrap().data_type() + ); + assert_eq!(flat.as_ref(), input.column_by_name("vec").unwrap().as_ref()); + } + + /// Builds a batch whose vector column holds `nulls`, one entry per row. + fn batch_with_nulls(nulls: &[bool]) -> RecordBatch { + let rows = nulls.len(); + let values = Float32Array::from_iter((0..(DIM as usize * rows)).map(|v| v as f32)); + let item = Arc::new(Field::new("item", DataType::Float32, true)); + let validity = NullBuffer::from(nulls.iter().map(|n| !n).collect::>()); + let vectors = Arc::new( + FixedSizeListArray::try_new(item, DIM, Arc::new(values), Some(validity)).unwrap(), + ); + let schema = Schema::new(vec![ + Field::new("vec", vectors.data_type().clone(), true), + Field::new("other", DataType::Int32, false), + ]); + RecordBatch::try_new( + Arc::new(schema), + vec![ + vectors, + Arc::new(Int32Array::from_iter_values(0..rows as i32)), + ], + ) + .unwrap() + } + + /// The renamed field takes its nullability from `Array::is_nullable`, which + /// reports whether the column *currently holds* nulls rather than what the + /// source schema declared. Both directions are pinned so the flag cannot be + /// hardcoded either way. + #[rstest] + #[case::no_nulls(&[false, false], false)] + #[case::some_nulls(&[false, true], true)] + fn test_flat_transform_nullability_follows_the_data( + #[case] nulls: &[bool], + #[case] expected: bool, + ) { + let input = batch_with_nulls(nulls); + let output = FlatTransformer::new("vec").transform(&input).unwrap(); + let field = output + .schema() + .field_with_name(FLAT_COLUMN) + .unwrap() + .clone(); + assert_eq!(field.is_nullable(), expected); + } + + #[test] + fn test_flat_transform_reports_missing_column() { + let message = FlatTransformer::new("absent") + .transform(&batch()) + .unwrap_err() + .to_string(); + assert!(message.contains("column absent"), "{message}"); + } +} diff --git a/rust/lance-index/src/vector/graph.rs b/rust/lance-index/src/vector/graph.rs index 097aa064d67..9100034eaea 100644 --- a/rust/lance-index/src/vector/graph.rs +++ b/rust/lance-index/src/vector/graph.rs @@ -5,7 +5,7 @@ //! use std::cmp::Reverse; -use std::collections::BinaryHeap; +use std::collections::{BinaryHeap, VecDeque}; use std::sync::Arc; use arrow_schema::{DataType, Field}; @@ -303,7 +303,6 @@ macro_rules! beam_search_loop { $visited:ident, $k:expr, $dist_calc:expr, - $prefetch_distance:expr, $accepts_result:expr, |$current:ident, $process_neighbor:ident| $visit_neighbors:block ) => {{ @@ -321,6 +320,10 @@ macro_rules! beam_search_loop { } $visited.insert(neighbor); let dist: OrderedFloat = $dist_calc.distance(neighbor).into(); + // Algorithm 2 refreshes the furthest result after every + // update to W, including updates made by an earlier neighbor + // of this same expanded node. + let furthest = furthest_distance(&$results); if dist <= furthest || $results.len() < $k { if $accepts_result(neighbor, dist) { push_result(&mut $results, (dist, neighbor).into(), $k); @@ -338,7 +341,6 @@ macro_rules! greedy_search_loop { $current:ident, $closest_dist:ident, $dist_calc:expr, - $prefetch_distance:expr, |$process_neighbor:ident| $visit_neighbors:block ) => {{ loop { @@ -412,7 +414,6 @@ pub fn beam_search( visited, k, dist_calc, - prefetch_distance, accepts_result, |current, process_neighbor| { let neighbors = graph.neighbors(current.id); @@ -451,7 +452,6 @@ pub fn beam_search( visited, k, dist_calc, - prefetch_distance, accepts_result, |current, process_neighbor| { let neighbors = graph.neighbors(current.id); @@ -493,7 +493,6 @@ pub fn beam_search_borrowed( visited, k, dist_calc, - prefetch_distance, accepts_result, |current, process_neighbor| { let neighbors = graph.neighbors(current.id); @@ -531,7 +530,6 @@ pub fn beam_search_borrowed( visited, k, dist_calc, - prefetch_distance, accepts_result, |current, process_neighbor| { let neighbors = graph.neighbors(current.id); @@ -546,6 +544,152 @@ pub fn beam_search_borrowed( results.into_sorted_vec() } +/// Number of mask-passing nodes used to seed [beam_search_acorn]'s frontier. +const ACORN_SEED_COUNT: usize = 16; + +/// Cap on starved-frontier waypoint expansions in [beam_search_acorn], +/// as a multiple of `ef`. +const ACORN_BRIDGE_BUDGET_FACTOR: usize = 4; + +/// Beam search over the mask-passing subgraph (ACORN-1). +/// +/// Only nodes in `bitset` get distances. A filtered-out neighbor contributes +/// its own neighbors instead, expanded once via `expanded`. Deeper masked +/// chains are crossed through unscored waypoints under a budget. The frontier +/// starts from the entry point plus mask-sampled seeds. May return fewer than +/// `min(ef, passing)` results if the budget runs out, so callers needing a +/// guarantee must check the count. +#[allow(clippy::too_many_arguments)] +pub fn beam_search_acorn( + graph: &impl BorrowingGraph, + ep: &OrderedNode, + params: &HnswQueryParams, + dist_calc: &impl DistCalculator, + bitset: &Visited, + prefetch_distance: Option, + visited: &mut Visited, + expanded: &mut Visited, +) -> Vec { + let ef = params.ef; + let lower_bound: OrderedFloat = params.lower_bound.unwrap_or(f32::MIN).into(); + let upper_bound: OrderedFloat = params.upper_bound.unwrap_or(f32::MAX).into(); + let passing_total = bitset.count_ones(); + let mut candidates = BinaryHeap::with_capacity(ef); + let mut results = BinaryHeap::with_capacity(ef); + // collected per node before scoring so prefetch targets are the ids + // that actually get distances + let mut passing: Vec = Vec::with_capacity(64); + // masked nodes seen two hops out, expandable if the frontier starves, + // deduped against `expanded` at pop rather than at push + let mut waypoints: VecDeque = VecDeque::new(); + let mut bridge_budget = ACORN_BRIDGE_BUDGET_FACTOR * ef; + + // the entry point seeds the traversal even if it fails the mask + visited.insert(ep.id); + candidates.push(Reverse(ep.clone())); + if bitset.contains(ep.id) && ep.dist >= lower_bound && ep.dist < upper_bound { + results.push(ep.clone()); + } + + let stride = (passing_total / ACORN_SEED_COUNT).max(1); + for seed in bitset.iter_ones().step_by(stride).take(ACORN_SEED_COUNT) { + let seed = seed as u32; + if visited.contains(seed) { + continue; + } + visited.insert(seed); + let dist: OrderedFloat = dist_calc.distance(seed).into(); + if dist >= lower_bound && dist < upper_bound { + push_result(&mut results, (dist, seed).into(), ef); + } + candidates.push(Reverse((dist, seed).into())); + } + + loop { + let Some(Reverse(current)) = candidates.pop() else { + // frontier starved: burn bridge budget through masked waypoints + // until a new passing node is found + if results.len() >= ef.min(passing_total) { + break; + } + let mut found = false; + while let Some(waypoint) = waypoints.pop_front() { + if bridge_budget == 0 { + break; + } + if expanded.contains(waypoint) { + continue; + } + expanded.insert(waypoint); + bridge_budget -= 1; + for &neighbor in graph.neighbors(waypoint) { + if bitset.contains(neighbor) { + if !visited.contains(neighbor) { + visited.insert(neighbor); + let dist: OrderedFloat = dist_calc.distance(neighbor).into(); + if dist >= lower_bound && dist < upper_bound { + push_result(&mut results, (dist, neighbor).into(), ef); + } + candidates.push(Reverse((dist, neighbor).into())); + found = true; + } + } else if !expanded.contains(neighbor) { + waypoints.push_back(neighbor); + } + } + if found { + break; + } + } + if !found { + break; + } + continue; + }; + if current.dist > furthest_distance(&results) && results.len() == ef { + break; + } + + passing.clear(); + for &neighbor in graph.neighbors(current.id) { + if bitset.contains(neighbor) { + if !visited.contains(neighbor) { + visited.insert(neighbor); + passing.push(neighbor); + } + } else if !expanded.contains(neighbor) { + expanded.insert(neighbor); + for &second_hop in graph.neighbors(neighbor) { + if bitset.contains(second_hop) { + if !visited.contains(second_hop) { + visited.insert(second_hop); + passing.push(second_hop); + } + } else if !expanded.contains(second_hop) { + waypoints.push_back(second_hop); + } + } + } + } + + process_neighbors_with_look_ahead( + &passing, + |node| { + let dist: OrderedFloat = dist_calc.distance(node).into(); + if dist <= furthest_distance(&results) || results.len() < ef { + if dist >= lower_bound && dist < upper_bound { + push_result(&mut results, (dist, node).into(), ef); + } + candidates.push(Reverse((dist, node).into())); + } + }, + prefetch_distance, + dist_calc, + ); + } + results.into_sorted_vec() +} + /// Greedy search over a graph /// /// This searches for only one result, only used for finding the entry point @@ -572,21 +716,15 @@ pub fn greedy_search( ) -> OrderedNode { let mut current = start.id; let mut closest_dist = start.dist.0; - greedy_search_loop!( - current, - closest_dist, - dist_calc, - prefetch_distance, - |process_neighbor| { - let neighbors = graph.neighbors(current); - process_neighbors_with_look_ahead( - &neighbors, - process_neighbor, - prefetch_distance, - dist_calc, - ); - } - ); + greedy_search_loop!(current, closest_dist, dist_calc, |process_neighbor| { + let neighbors = graph.neighbors(current); + process_neighbors_with_look_ahead( + &neighbors, + process_neighbor, + prefetch_distance, + dist_calc, + ); + }); OrderedNode::new(current, closest_dist.into()) } @@ -598,23 +736,168 @@ pub fn greedy_search_borrowed( ) -> OrderedNode { let mut current = start.id; let mut closest_dist = start.dist.0; - greedy_search_loop!( - current, - closest_dist, - dist_calc, - prefetch_distance, - |process_neighbor| { - let neighbors = graph.neighbors(current); - process_neighbors_with_look_ahead( - neighbors, - process_neighbor, - prefetch_distance, - dist_calc, - ); - } - ); + greedy_search_loop!(current, closest_dist, dist_calc, |process_neighbor| { + let neighbors = graph.neighbors(current); + process_neighbors_with_look_ahead( + neighbors, + process_neighbor, + prefetch_distance, + dist_calc, + ); + }); OrderedNode::new(current, closest_dist.into()) } #[cfg(test)] -mod tests {} +mod tests { + use std::cell::Cell; + + use super::*; + + struct ChainGraph { + neighbors: Vec>, + } + + impl BorrowingGraph for ChainGraph { + fn len(&self) -> usize { + self.neighbors.len() + } + + fn neighbors(&self, key: u32) -> &[u32] { + &self.neighbors[key as usize] + } + } + + struct ZeroDistance; + + impl DistCalculator for ZeroDistance { + fn distance(&self, _id: u32) -> f32 { + 0.0 + } + + fn distance_all(&self, _k_hint: usize) -> Vec { + Vec::new() + } + } + + struct CountingDistCalculator<'a> { + distances: &'a [f32], + computations: Cell, + } + + impl DistCalculator for CountingDistCalculator<'_> { + fn distance(&self, id: u32) -> f32 { + self.computations.set(self.computations.get() + 1); + self.distances[id as usize] + } + + fn distance_all(&self, _k_hint: usize) -> Vec { + self.distances.to_vec() + } + } + + #[test] + fn test_beam_search_refreshes_furthest_neighbor_bound() { + // The entry point is closer than the first candidates. Once those + // candidates fill W, its furthest distance grows from 1 to 10. The + // next neighbor must be compared with that new bound so it can enter + // the candidate queue and lead the search to node 4. + let graph = ChainGraph { + neighbors: vec![vec![1, 2, 3], vec![], vec![], vec![4], vec![]], + }; + let distances = [1.0, 10.0, 9.0, 8.0, 7.0]; + let dist_calc = CountingDistCalculator { + distances: &distances, + computations: Cell::new(0), + }; + let params = HnswQueryParams { + ef: 3, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let entry = OrderedNode::new(0, distances[0].into()); + let mut visited_generator = VisitedGenerator::new(graph.len()); + + let results = beam_search_borrowed( + &graph, + &entry, + ¶ms, + &dist_calc, + None, + None, + &mut visited_generator.generate(graph.len()), + ); + + assert_eq!( + results.iter().map(|node| node.id).collect::>(), + vec![0, 4, 3] + ); + assert_eq!(dist_calc.computations.get(), 4); + } + + /// Passing components joined only through chains of two masked nodes + /// must still all be found (from review: without waypoint expansion + /// only the seeded nodes return). + #[test] + fn test_acorn_reaches_across_masked_chains() { + const PASSING_COUNT: usize = 20; + const FAILING_COUNT: usize = (PASSING_COUNT - 1) * 2; + let mut neighbors = vec![Vec::new(); PASSING_COUNT + FAILING_COUNT]; + for index in 0..PASSING_COUNT - 1 { + let left = index as u32; + let first_failing = (PASSING_COUNT + index * 2) as u32; + let second_failing = first_failing + 1; + let right = left + 1; + + neighbors[left as usize].push(first_failing); + neighbors[first_failing as usize].extend([left, second_failing]); + neighbors[second_failing as usize].extend([first_failing, right]); + neighbors[right as usize].push(second_failing); + } + let graph = ChainGraph { neighbors }; + let params = HnswQueryParams { + ef: 30, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let entry = OrderedNode::new(0, 0.0.into()); + + let mut mask_generator = VisitedGenerator::new(graph.len()); + let mut mask = mask_generator.generate(graph.len()); + for id in 0..PASSING_COUNT as u32 { + mask.insert(id); + } + + let mut acorn_visited_generator = VisitedGenerator::new(graph.len()); + let mut acorn_expanded_generator = VisitedGenerator::new(graph.len()); + let acorn_results = beam_search_acorn( + &graph, + &entry, + ¶ms, + &ZeroDistance, + &mask, + None, + &mut acorn_visited_generator.generate(graph.len()), + &mut acorn_expanded_generator.generate(graph.len()), + ); + + let mut basic_visited_generator = VisitedGenerator::new(graph.len()); + let basic_results = beam_search_borrowed( + &graph, + &entry, + ¶ms, + &ZeroDistance, + Some(&mask), + None, + &mut basic_visited_generator.generate(graph.len()), + ); + + assert_eq!(basic_results.len(), PASSING_COUNT); + assert_eq!(acorn_results.len(), PASSING_COUNT); + assert!(acorn_results.iter().all(|node| mask.contains(node.id))); + } +} diff --git a/rust/lance-index/src/vector/graph/builder.rs b/rust/lance-index/src/vector/graph/builder.rs index 36e42b7cf9a..2cbc8855568 100644 --- a/rust/lance-index/src/vector/graph/builder.rs +++ b/rust/lance-index/src/vector/graph/builder.rs @@ -60,25 +60,4 @@ impl GraphBuilderNode { self.bottom_neighbors = self.level_neighbors[0].clone(); } } - - pub(crate) fn cutoff(&self, level: u16, max_size: usize) -> OrderedFloat { - let neighbors = &self.level_neighbors_ranked[level as usize]; - if neighbors.len() < max_size { - OrderedFloat(f32::INFINITY) - } else { - neighbors.last().unwrap().dist - } - } -} - -#[derive(Debug)] -pub struct GraphBuilderStats { - #[allow(dead_code)] - pub num_nodes: usize, - #[allow(dead_code)] - pub max_edges: usize, - #[allow(dead_code)] - pub mean_edges: f32, - #[allow(dead_code)] - pub mean_distance: f32, } diff --git a/rust/lance-index/src/vector/hnsw.rs b/rust/lance-index/src/vector/hnsw.rs index a618f34753f..0b86f4d2bf7 100644 --- a/rust/lance-index/src/vector/hnsw.rs +++ b/rust/lance-index/src/vector/hnsw.rs @@ -48,7 +48,7 @@ pub struct HnswMetadata { impl Default for HnswMetadata { fn default() -> Self { let params = HnswBuildParams::default(); - let level_offsets = vec![0; params.max_level as usize]; + let level_offsets = vec![0; params.max_level as usize + 1]; Self { entry_point: 0, params, @@ -59,6 +59,13 @@ impl Default for HnswMetadata { /// Algorithm 4 in the HNSW paper. /// +/// This uses the paper's `extendCandidates = false` and +/// `keepPrunedConnections = true` configuration. Keeping pruned connections +/// fills the requested degree without exceeding it, reducing sparse directed +/// components. It does not guarantee that every node is reachable: lower +/// construction settings trade graph quality for size and build work. Callers +/// supply the complete candidate set that should participate in this selection. +/// /// # NOTE /// The results are not ordered. pub(crate) fn select_neighbors_heuristic( @@ -85,14 +92,56 @@ pub(crate) fn select_neighbors_heuristic_owned( candidates.sort_unstable(); let mut results: Vec = Vec::with_capacity(k); - for u in candidates.iter() { + let mut pruned = Vec::with_capacity(candidates.len()); + for candidate in candidates { if results.len() >= k { break; } - if results.is_empty() || storage.prefers_candidate(u, &results) { - results.push(u.clone()); + if results.is_empty() || storage.prefers_candidate(&candidate, &results) { + results.push(candidate); + } else { + pruned.push(candidate); } } + results.extend(pruned.into_iter().take(k - results.len())); results } + +#[cfg(test)] +mod tests { + use arrow_array::{FixedSizeListArray, Float32Array}; + use lance_arrow::FixedSizeListArrayExt; + use lance_linalg::distance::DistanceType; + + use super::select_neighbors_heuristic_owned; + use crate::vector::flat::storage::FlatFloatStorage; + use crate::vector::graph::OrderedNode; + use crate::vector::storage::VectorStore; + + /// A reciprocal candidate must join the complete old-plus-new set before + /// Algorithm 4 runs. A farther, directionally diverse connection is + /// selected first, then pruned connections refill the remaining capacity. + #[test] + fn test_selection_retains_farther_diverse_reciprocal_candidate() { + let vectors = Float32Array::from(vec![ + 0.0, 0.0, // node receiving the reciprocal connection + 1.0, 0.0, // close candidate + 2.0, 0.0, // redundant candidate + 3.0, 0.0, // redundant candidate + 4.0, 0.0, // redundant candidate + 0.0, 5.0, // farther but directionally diverse candidate + ]); + let vectors = FixedSizeListArray::try_new_from_values(vectors, 2).unwrap(); + let storage = FlatFloatStorage::new(vectors, DistanceType::L2); + let candidates = (1..=5) + .map(|id| OrderedNode::new(id, storage.dist_between(0, id).into())) + .collect(); + + let selected = select_neighbors_heuristic_owned(&storage, candidates, 4); + assert_eq!( + selected.iter().map(|node| node.id).collect::>(), + vec![1, 5, 2, 3] + ); + } +} diff --git a/rust/lance-index/src/vector/hnsw/builder.rs b/rust/lance-index/src/vector/hnsw/builder.rs index cc1ac1abf81..005c09893f0 100644 --- a/rust/lance-index/src/vector/hnsw/builder.rs +++ b/rust/lance-index/src/vector/hnsw/builder.rs @@ -6,7 +6,7 @@ use arrow::array::{AsArray, ListBuilder, UInt32Builder}; use arrow::compute::concat_batches; use arrow::datatypes::{DataType, UInt32Type}; -use arrow_array::{ArrayRef, Float32Array, ListArray, RecordBatch, UInt64Array}; +use arrow_array::{Array, ArrayRef, Float32Array, ListArray, RecordBatch, UInt64Array}; use crossbeam_queue::ArrayQueue; use itertools::Itertools; use lance_core::deepsize::DeepSizeOf; @@ -18,7 +18,6 @@ use rayon::prelude::*; use std::cmp::min; use std::collections::{BinaryHeap, HashMap, VecDeque}; use std::fmt::Debug; -use std::iter; use std::sync::Arc; use std::sync::RwLock; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -40,10 +39,12 @@ use crate::vector::graph::{ BorrowingGraph, DISTS_FIELD, Graph, NEIGHBORS_COL, NEIGHBORS_FIELD, OrderedFloat, OrderedNode, VisitedGenerator, }; -use crate::vector::graph::{Visited, beam_search_borrowed, greedy_search, greedy_search_borrowed}; +use crate::vector::graph::{ + Visited, beam_search_acorn, beam_search_borrowed, greedy_search, greedy_search_borrowed, +}; use crate::vector::storage::{DistCalculator, VectorStore}; use crate::vector::v3::subindex::IvfSubIndex; -use crate::vector::{Query, VECTOR_RESULT_SCHEMA}; +use crate::vector::{ApproxMode, Query, VECTOR_RESULT_SCHEMA}; pub const HNSW_METADATA_KEY: &str = "lance:hnsw"; @@ -56,10 +57,23 @@ pub const HNSW_METADATA_KEY: &str = "lance:hnsw"; /// ([`super::online::OnlineHnswBuilder`]) builders so both produce comparable graphs. pub(crate) const HNSW_LEVEL_RNG_SEED: u64 = 42; +/// Minimum edge count that avoids severely fragmented graphs during parallel +/// construction while still allowing deliberately small test and index sizes. +pub(crate) const MIN_HNSW_M: usize = 4; + +/// Draw a node level using the distribution from Algorithm 1. +pub(crate) fn random_level_with(params: &HnswBuildParams, rng: &mut R) -> u16 { + let ml = 1.0 / (params.m as f32).ln(); + min( + (-rng.random::().ln() * ml) as u16, + params.max_level - 1, + ) +} + /// Parameters of building HNSW index #[derive(Debug, Clone, Serialize, Deserialize, DeepSizeOf)] pub struct HnswBuildParams { - /// max level ofm + /// Maximum number of levels in the graph. pub max_level: u16, /// number of connections to establish while inserting new element @@ -95,14 +109,17 @@ impl Default for HnswBuildParams { impl HnswBuildParams { /// The maximum level of the graph. - /// The default value is `8`. + /// The default value is `7`. pub fn max_level(mut self, max_level: u16) -> Self { self.max_level = max_level; self } /// The number of connections to establish while inserting new element - /// The default value is `30`. + /// + /// Must be at least 4. Smaller values produce severely fragmented graphs + /// during parallel construction. + /// The default value is `20`. pub fn num_edges(mut self, m: usize) -> Self { self.m = m; self @@ -111,12 +128,41 @@ impl HnswBuildParams { /// Number of candidates to be considered when searching for the nearest neighbors /// during the construction of the graph. /// - /// The default value is `100`. + /// The default value is `150`. pub fn ef_construction(mut self, ef_construction: usize) -> Self { self.ef_construction = ef_construction; self } + pub(crate) fn validate(&self) -> Result<()> { + if self.max_level == 0 { + return Err(Error::invalid_input(format!( + "HnswBuildParams::max_level must be greater than 0, got {}", + self.max_level + ))); + } + if self.m < MIN_HNSW_M { + return Err(Error::invalid_input(format!( + "HnswBuildParams::m must be at least {MIN_HNSW_M} to avoid severely fragmented graphs, got {}", + self.m, + ))); + } + if self.m > usize::MAX / 2 { + return Err(Error::invalid_input(format!( + "HnswBuildParams::m must be at most {} so the level-0 reciprocal limit can be represented, got {}", + usize::MAX / 2, + self.m + ))); + } + if self.ef_construction < self.m { + return Err(Error::invalid_input(format!( + "HnswBuildParams::ef_construction must be at least m ({}), got {}", + self.m, self.ef_construction + ))); + } + Ok(()) + } + /// Build the HNSW index from the given data. /// /// # Parameters @@ -176,7 +222,10 @@ impl DeepSizeOf for HnswCore { impl HnswCore { fn max_level(&self) -> u16 { - self.params.max_level + self.level_count + .iter() + .rposition(|count| *count != 0) + .map_or(0, |level| level + 1) as u16 } fn num_nodes(&self, level: usize) -> usize { @@ -258,6 +307,35 @@ impl HNSW { } } + /// Refuse a graph whose nodes outrun the vectors behind them. + /// + /// Node ids are row numbers into `storage`, so a graph with more nodes than + /// storage has rows holds ids no vector backs. Scoring one indexes past the + /// storage buffer and panics, which takes the worker rather than the query, + /// and the entry point is scored before any traversal decision -- so this has + /// to run first. + /// + /// Only that direction is refused. Storage with rows the graph never reached + /// is safe and common: those rows are simply unreachable by traversal, and a + /// sparse-prefilter search still brute-forces them. + /// + /// Written before the export bounded the pair, such an index cannot be + /// searched at all -- the vectors are not on disk -- so it is refused with a + /// message naming it rather than left to fault. + fn ensure_storage_covers_graph(&self, storage: &impl VectorStore) -> Result<()> { + let nodes = self.len(); + let rows = storage.len(); + if nodes > rows { + return Err(Error::index(format!( + "HNSW graph has {nodes} nodes but its vector storage has {rows} \ + rows, so {} node(s) have no vector to score; the index predates \ + the export bound and has to be rebuilt", + nodes - rows + ))); + } + Ok(()) + } + #[allow(clippy::too_many_arguments)] pub fn search_inner( &self, @@ -269,6 +347,7 @@ impl HNSW { storage: &impl VectorStore, prefetch_distance: Option, ) -> Result> { + self.ensure_storage_covers_graph(storage)?; let dist_calc = storage.dist_calculator(query, params.dist_q_c); let entry = self.inner.entry_point; let ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); @@ -338,8 +417,13 @@ impl HNSW { L: BorrowingGraph, B: BorrowingGraph, { + // Greedy descent stops at level 1 (HNSW paper, Algorithm 2): level 0 + // is searched only by the ef-bounded beam below. Greedily descending + // into level 0 first collapses the entry point into a deep local + // minimum, which costs extra distance computations and measurably + // hurts recall at low ef (https://github.com/lance-format/lance/issues/5208). let mut ep = ep; - for level in (0..self.max_level()).rev() { + for level in (1..self.max_level()).rev() { let cur_level = make_level(level); ep = greedy_search_borrowed( &cur_level, @@ -397,6 +481,142 @@ impl HNSW { result } + /// Like [Self::search_basic] but the bottom level runs + /// [beam_search_acorn], which only scores mask-passing nodes. + pub fn search_acorn( + &self, + query: ArrayRef, + k: usize, + params: &HnswQueryParams, + bitset: &Visited, + storage: &impl VectorStore, + ) -> Result> { + let mut visited_generator = self + .inner + .visited_generator_queue + .pop() + .unwrap_or_else(|| VisitedGenerator::new(storage.len())); + let mut expanded_generator = self + .inner + .visited_generator_queue + .pop() + .unwrap_or_else(|| VisitedGenerator::new(storage.len())); + + let result = self.search_acorn_inner( + query, + k, + params, + bitset, + &mut visited_generator, + &mut expanded_generator, + storage, + Some(2), + ); + + // if the queue is full, we just don't push it back, so ignore the error here + let _ = self.inner.visited_generator_queue.push(visited_generator); + let _ = self.inner.visited_generator_queue.push(expanded_generator); + result + } + + #[allow(clippy::too_many_arguments)] + fn search_acorn_inner( + &self, + query: ArrayRef, + k: usize, + params: &HnswQueryParams, + bitset: &Visited, + visited_generator: &mut VisitedGenerator, + expanded_generator: &mut VisitedGenerator, + storage: &impl VectorStore, + prefetch_distance: Option, + ) -> Result> { + self.ensure_storage_covers_graph(storage)?; + let dist_calc = storage.dist_calculator(query, params.dist_q_c); + let entry = self.inner.entry_point; + let ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); + + let result = match &self.inner.graph { + HnswGraph::Built(nodes) => { + let nodes = nodes.as_slice(); + self.run_search_acorn( + ep, + params, + bitset, + visited_generator, + expanded_generator, + storage.len(), + prefetch_distance, + &dist_calc, + |level| ImmutableHnswLevelView::new(level, nodes), + ImmutableHnswBottomView::new(nodes), + ) + } + HnswGraph::Loaded(graph) => { + let graph = graph.as_ref(); + self.run_search_acorn( + ep, + params, + bitset, + visited_generator, + expanded_generator, + storage.len(), + prefetch_distance, + &dist_calc, + |level| LoadedHnswLevelView::new(level, graph), + LoadedHnswBottomView::new(graph), + ) + } + }; + Ok(result.into_iter().take(k).collect()) + } + + /// [Self::run_search] for the ACORN traversal: same level descent, but + /// the bottom level runs [beam_search_acorn]. + #[allow(clippy::too_many_arguments)] + fn run_search_acorn( + &self, + ep: OrderedNode, + params: &HnswQueryParams, + bitset: &Visited, + visited_generator: &mut VisitedGenerator, + expanded_generator: &mut VisitedGenerator, + storage_len: usize, + prefetch_distance: Option, + dist_calc: &impl DistCalculator, + make_level: impl Fn(u16) -> L, + bottom: B, + ) -> Vec + where + L: BorrowingGraph, + B: BorrowingGraph, + { + // Same level descent as [Self::run_search]: greedy stops at level 1 + // (see the comment there); level 0 is left to the beam below. + let mut ep = ep; + for level in (1..self.max_level()).rev() { + let cur_level = make_level(level); + ep = greedy_search_borrowed( + &cur_level, + ep, + dist_calc, + self.inner.params.prefetch_distance, + ); + } + let mut visited = visited_generator.generate(storage_len); + let mut expanded = expanded_generator.generate(storage_len); + beam_search_acorn( + &bottom, + &ep, + params, + dist_calc, + bitset, + prefetch_distance, + &mut visited, + &mut expanded, + ) + } + #[instrument(level = "debug", skip(self, storage, query, prefilter_bitset))] fn flat_search( &self, @@ -433,7 +653,7 @@ impl HNSW { } let dist: OrderedFloat = dist_calc.distance(node_id).into(); - if dist <= lower_bound || dist > upper_bound { + if dist < lower_bound || dist >= upper_bound { continue; } if heap.len() < k { @@ -447,7 +667,7 @@ impl HNSW { _ => { for node_id in prefilter_bitset.iter_ones().map(|i| i as u32) { let dist: OrderedFloat = dist_calc.distance(node_id).into(); - if dist <= lower_bound || dist > upper_bound { + if dist < lower_bound || dist >= upper_bound { continue; } if heap.len() < k { @@ -464,19 +684,18 @@ impl HNSW { /// Returns the metadata of this [`HNSW`]. pub fn metadata(&self) -> HnswMetadata { - // calculate the offsets of each level, - // start from 0 - let level_offsets = self - .inner - .level_count - .iter() - .chain(iter::once(&0)) - .scan(0, |state, x| { - let start = *state; - *state += *x; - Some(start) - }) - .collect(); + // Version-1 readers use params.max_level as the number of level + // batches and index them directly. Preserve that configured shape, + // padding levels above the sampled graph height with empty ranges. + let configured_levels = self.inner.params.max_level as usize; + let mut level_offsets = Vec::with_capacity(configured_levels + 1); + let mut offset = 0; + level_offsets.push(0); + for level in 0..configured_levels { + let level_count = self.inner.level_count.get(level).copied().unwrap_or(0); + offset += level_count; + level_offsets.push(offset); + } HnswMetadata { entry_point: self.inner.entry_point, @@ -508,7 +727,7 @@ impl DeepSizeOf for HnswBuilder { impl HnswBuilder { fn finish(self) -> HNSW { - let nodes = match Arc::try_unwrap(self.nodes) { + let nodes: Vec = match Arc::try_unwrap(self.nodes) { Ok(nodes) => nodes .into_iter() .map(|node| node.into_inner().expect("builder lock poisoned")) @@ -519,9 +738,14 @@ impl HnswBuilder { .collect(), }; + let actual_levels = nodes + .get(self.entry_point as usize) + .map(|node| node.level_neighbors.len()) + .unwrap_or(0); let level_count = self .level_count .into_iter() + .take(actual_levels) .map(|count| count.load(Ordering::Relaxed)) .collect(); @@ -564,34 +788,24 @@ impl HnswBuilder { } let mut nodes = Vec::with_capacity(len); - { - if len > 0 { - nodes.push(RwLock::new(GraphBuilderNode::new(0, max_level as usize))); - } - let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); - for i in 1..len { - nodes.push(RwLock::new(GraphBuilderNode::new( - i as u32, - builder.random_level(&mut level_rng) as usize + 1, - ))); + let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); + let mut highest_level = 0; + for i in 0..len { + let target_level = random_level_with(&builder.params, &mut level_rng); + if target_level > highest_level { + highest_level = target_level; + builder.entry_point = i as u32; } + nodes.push(RwLock::new(GraphBuilderNode::new( + i as u32, + target_level as usize + 1, + ))); } builder.nodes = Arc::new(nodes); builder } - /// New node's level - /// - /// See paper `Algorithm 1` - fn random_level(&self, rng: &mut R) -> u16 { - let ml = 1.0 / (self.params.m as f32).ln(); - min( - (-rng.random::().ln() * ml) as u16, - self.params.max_level - 1, - ) - } - /// Insert one node. fn insert( &self, @@ -601,6 +815,12 @@ impl HnswBuilder { ) { let nodes = &self.nodes; let target_level = nodes[node as usize].read().unwrap().level_neighbors.len() as u16 - 1; + let entry_level = nodes[self.entry_point as usize] + .read() + .unwrap() + .level_neighbors + .len() as u16 + - 1; let dist_calc = storage.dist_calculator_from_id(node); let mut ep = OrderedNode::new( self.entry_point, @@ -615,7 +835,7 @@ impl HnswBuilder { // ep = Select-Neighbors(W, 1) // } // ``` - for level in (target_level + 1..self.params.max_level).rev() { + for level in (target_level + 1..=entry_level).rev() { let cur_level = HnswLevelView::new(level, nodes); ep = greedy_search(&cur_level, ep, &dist_calc, self.params.prefetch_distance); } @@ -631,7 +851,9 @@ impl HnswBuilder { for neighbor in &neighbors { current_node.add_neighbor(neighbor.id, neighbor.dist, level); } - self.prune(storage, &mut current_node, level); + // Algorithm 1 selects M neighbors for the new node. Mmax0 is + // reserved for reciprocal edges on existing level-0 nodes. + self.prune(storage, &mut current_node, level, self.params.m); pruned_neighbors_per_level[level as usize] .clone_from(¤t_node.level_neighbors_ranked[level as usize]); @@ -639,22 +861,19 @@ impl HnswBuilder { } } for (level, pruned_neighbors) in pruned_neighbors_per_level.iter().enumerate() { - for unpruned_edge in pruned_neighbors { - let level = level as u16; - let m_max = match level { - 0 => self.params.m * 2, - _ => self.params.m, - }; - if unpruned_edge.dist - < nodes[unpruned_edge.id as usize] - .read() - .unwrap() - .cutoff(level, m_max) - { - let mut chosen_node = nodes[unpruned_edge.id as usize].write().unwrap(); - chosen_node.add_neighbor(node, unpruned_edge.dist, level); - self.prune(storage, &mut chosen_node, level); - } + let level = level as u16; + let reciprocal_limit = if level == 0 { + self.params.m * 2 + } else { + self.params.m + }; + for selected_edge in pruned_neighbors { + // Algorithm 1 adds the reciprocal candidate before applying + // SELECT-NEIGHBORS. A distance-only cutoff would incorrectly + // discard farther candidates that improve directional diversity. + let mut chosen_node = nodes[selected_edge.id as usize].write().unwrap(); + chosen_node.add_neighbor(node, selected_edge.dist, level); + self.prune(storage, &mut chosen_node, level, reciprocal_limit); } } } @@ -677,6 +896,7 @@ impl HnswBuilder { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, dist_calc, None, @@ -685,20 +905,22 @@ impl HnswBuilder { ) } - fn prune(&self, storage: &impl VectorStore, builder_node: &mut GraphBuilderNode, level: u16) { - let m_max = match level { - 0 => self.params.m * 2, - _ => self.params.m, - }; - + fn prune( + &self, + storage: &impl VectorStore, + builder_node: &mut GraphBuilderNode, + level: u16, + max_connections: usize, + ) { let neighbors_ranked = &mut builder_node.level_neighbors_ranked[level as usize]; - if neighbors_ranked.len() <= m_max { + if neighbors_ranked.len() <= max_connections { builder_node.update_from_ranked_neighbors(level); return; } let level_neighbors = std::mem::take(neighbors_ranked); - *neighbors_ranked = select_neighbors_heuristic_owned(storage, level_neighbors, m_max); + *neighbors_ranked = + select_neighbors_heuristic_owned(storage, level_neighbors, max_connections); builder_node.update_from_ranked_neighbors(level); } } @@ -798,15 +1020,69 @@ enum LevelLookup { /// `__vector_id` column. /// /// We do *not* assume the column is sorted or that the slice is aligned - /// to a true level boundary: `level_offsets`/`level_count` omit the - /// entry-point node (it is written at every level by `to_batch` but only - /// counted at level 0), so upper-level slices can be off-by-one and - /// non-monotonic. Keying by the `__vector_id` value -- exactly what the - /// old per-node `load` did -- preserves behavior bit-for-bit. Upper - /// levels shrink geometrically, so this map stays tiny. + /// to a true level boundary. Indices written before issue #5156 was fixed + /// omitted the entry point from every upper-level count, so their slices + /// start progressively earlier than the true boundaries. Keying by the + /// `__vector_id` value -- exactly what the old per-node `load` did -- + /// preserves their behavior bit-for-bit. Upper levels shrink + /// geometrically, so this map stays tiny. Sparse(HashMap), } +/// Drop neighbor ids that name no node in this graph. +/// +/// A writer that read adjacency live while snapshotting a node count could +/// persist edges past its own node count, and those indices are already on +/// disk. Traversal scores a neighbor id before it is ever looked up as a node, +/// so a guard at the lookup is too late -- the id has to be gone before search +/// begins. +/// +/// Returns the original array untouched when every id is in domain, which is +/// the only case that matters for cost: the ids stay zero-copy views of the +/// loaded batch and nothing is allocated. `to_batch()` still returns the +/// retained batch verbatim, so a filtered edge is dropped for this reader +/// without rewriting what is on disk. +fn neighbors_within_domain(neighbors: &ListArray, node_count: usize) -> (ListArray, usize) { + // Ids are `u32` on the wire, so a node count past `u32::MAX` cannot be + // addressed by one; clamping keeps every id in domain rather than wrapping. + let node_count = u32::try_from(node_count).unwrap_or(u32::MAX); + let values = neighbors.values().as_primitive::(); + // Each level is a slice of the concatenated batch and `values()` hands back + // the whole child array regardless, so bound the scan to this array's own + // offset window. Scanning all of it would count another level's ids, put a + // clean level on the rebuild path, and report a count that is not this + // level's. + let offsets = neighbors.offsets(); + let start = offsets[0] as usize; + let end = offsets[offsets.len() - 1] as usize; + let dropped = values.values()[start..end] + .iter() + .filter(|&&id| id >= node_count) + .count(); + if dropped == 0 { + return (neighbors.clone(), 0); + } + + let mut builder = ListBuilder::with_capacity(UInt32Builder::new(), neighbors.len()); + for row in 0..neighbors.len() { + if neighbors.is_null(row) { + builder.append_null(); + continue; + } + let row_ids = neighbors.value(row); + let row_ids = row_ids.as_primitive::(); + builder.append_value( + row_ids + .values() + .iter() + .copied() + .filter(|&id| id < node_count) + .map(Some), + ); + } + (builder.finish(), dropped) +} + /// A search-only HNSW graph backed directly by the Arrow buffers of the /// on-disk `RecordBatch`. /// @@ -828,6 +1104,10 @@ struct LoadedHnswGraph { level_lookup: Vec, /// Number of nodes present at each level (`level_count[0]` == total). level_count: Vec, + /// Bytes in `level_neighbors` that are *not* views into `batch`, from a + /// level rebuilt to drop out-of-domain ids. Zero for a clean index, which + /// keeps every level zero-copy. + owned_neighbor_bytes: usize, } impl DeepSizeOf for LoadedHnswGraph { @@ -837,7 +1117,11 @@ impl DeepSizeOf for LoadedHnswGraph { // `vector/flat/storage.rs`). The upper-level `level_lookup` maps are // sized to the geometrically-shrinking node counts above level 0 -- // negligible next to the batch and not separately accounted here. - self.batch.get_array_memory_size() + // + // A level rebuilt to drop out-of-domain ids owns its buffers instead, + // so those bytes are counted on top: they are real and the cache sizes + // itself from this number. + self.batch.get_array_memory_size() + self.owned_neighbor_bytes } } @@ -847,6 +1131,14 @@ impl LoadedHnswGraph { #[inline] fn neighbors_at(&self, level: usize, key: u32) -> &[u32] { let row = match &self.level_lookup[level] { + // `Dense` means row == id, so an id at or beyond the level's row + // count addresses nothing. The writers now bound what they emit to + // the prefix they publish, but indices written before that are + // already on disk, and following such an edge panicked the search + // instead of degrading it. Treat it exactly like an absent node: no + // neighbors, so greedy search stays put and descends, losing one + // edge rather than the whole query. + LevelLookup::Dense if key as usize >= self.level_count[level] => return &[], LevelLookup::Dense => key as usize, LevelLookup::Sparse(id_to_row) => match id_to_row.get(&key) { Some(&row) => row as usize, @@ -965,6 +1257,7 @@ pub struct HnswQueryParams { pub lower_bound: Option, pub upper_bound: Option, pub dist_q_c: f32, + pub use_acorn: bool, } impl From<&Query> for HnswQueryParams { @@ -975,6 +1268,7 @@ impl From<&Query> for HnswQueryParams { lower_bound: query.lower_bound, upper_bound: query.upper_bound, dist_q_c: query.dist_q_c, + use_acorn: query.approx_mode == ApproxMode::Fast, } } } @@ -1022,10 +1316,18 @@ impl IvfSubIndex for HNSW { // need it, and `to_batch()` returns the retained `data` verbatim. let mut level_neighbors = Vec::with_capacity(level_batches.len()); let mut level_lookup = Vec::with_capacity(level_batches.len()); + let mut dropped_edges = 0usize; + let mut owned_neighbor_bytes = 0usize; for (level, batch) in level_batches.iter().enumerate() { // `.clone()` on an Arrow array bumps a refcount; buffers stay // shared with `data` (zero copy). let neighbors = batch[NEIGHBORS_COL].as_list::().clone(); + let (neighbors, dropped) = neighbors_within_domain(&neighbors, level_count[0]); + if dropped > 0 { + // Rebuilt, so it no longer borrows `batch`; see `DeepSizeOf`. + owned_neighbor_bytes += neighbors.get_array_memory_size(); + } + dropped_edges += dropped; let ids = batch[VECTOR_ID_COL].as_primitive::(); if level == 0 { // `to_batch` writes every node at level 0 exactly once in @@ -1066,6 +1368,17 @@ impl IvfSubIndex for HNSW { level_neighbors.push(neighbors); } + if dropped_edges > 0 { + // Dropped, not rejected: an edge to a node this graph does not hold + // costs one edge, where refusing the batch costs every query over + // it. The entry point below is refused instead, because search + // cannot start without it. + log::warn!( + "HNSW batch carried {dropped_edges} neighbor id(s) outside its {} nodes; dropping them for this reader", + level_count[0] + ); + } + // `entry_point` is read from untrusted metadata and indexes the `Dense` // level-0 lookup directly; an out-of-range value would read past the // level-0 neighbor buffer during search. Validate it under the same @@ -1093,6 +1406,7 @@ impl IvfSubIndex for HNSW { level_neighbors, level_lookup, level_count: level_count.clone(), + owned_neighbor_bytes, }; let inner = HnswCore { params: hnsw_metadata.params, @@ -1125,6 +1439,15 @@ impl IvfSubIndex for HNSW { .into() } + // `schema()` governs the on-disk index file, and readers older than v8.0.0 + // index the distance column there unconditionally, panicking when it is + // absent, so it cannot be dropped from what we write. Skipping it on read + // costs nothing in compatibility and keeps most of the graph bytes off the + // wire. + fn read_columns() -> Option<&'static [&'static str]> { + Some(&[VECTOR_ID_COL, NEIGHBORS_COL]) + } + #[instrument(level = "debug", skip(self, query, storage, prefilter, _metrics))] fn search( &self, @@ -1151,27 +1474,40 @@ impl IvfSubIndex for HNSW { .visited_generator_queue .pop() .unwrap_or_else(|| VisitedGenerator::new(storage.len())); - let prefilter_bitset = if prefilter.is_empty() { - None + let results = if prefilter.is_empty() { + self.search_basic(query, k, ¶ms, None, storage)? } else { + // the bitset must be moved into a callee on every path so its + // borrow of `prefilter_generator` ends before the push below let indices = prefilter.filter_row_ids(Box::new(storage.row_ids())); - let mut bitset = prefilter_generator.generate(storage.len()); - for indices in indices { - bitset.insert(indices as u32); + let mut prefilter_bitset = prefilter_generator.generate(storage.len()); + for index in indices { + prefilter_bitset.insert(index as u32); + } + let remained = prefilter_bitset.count_ones(); + if remained == storage.len() { + // mask passes every row: same as unfiltered + drop(prefilter_bitset); + self.search_basic(query, k, ¶ms, None, storage)? + } else if remained < self.len() * 10 / 100 { + // few matching rows: brute force is cheaper and exact + self.flat_search(storage, query, k, prefilter_bitset, ¶ms) + } else if params.use_acorn { + let acorn_results = + self.search_acorn(query.clone(), k, ¶ms, &prefilter_bitset, storage)?; + // under-delivery means the budget ran out on a fragmented + // mask, except range-bounded queries which return short + // legitimately + let bounded = params.lower_bound.is_some() || params.upper_bound.is_some(); + if !bounded && acorn_results.len() < k.min(remained) { + self.search_basic(query, k, ¶ms, Some(prefilter_bitset), storage)? + } else { + drop(prefilter_bitset); + acorn_results + } + } else { + self.search_basic(query, k, ¶ms, Some(prefilter_bitset), storage)? } - Some(bitset) - }; - - let remained = prefilter_bitset - .as_ref() - .map(|b| b.count_ones()) - .unwrap_or(storage.len()); - let results = if remained < self.len() * 10 / 100 { - let prefilter_bitset = - prefilter_bitset.expect("the prefilter bitset must be set for flat search"); - self.flat_search(storage, query, k, prefilter_bitset, ¶ms) - } else { - self.search_basic(query, k, ¶ms, prefilter_bitset, storage)? }; // if the queue is full, we just don't push it back, so ignore the error here let _ = self.inner.visited_generator_queue.push(prefilter_generator); @@ -1193,6 +1529,7 @@ impl IvfSubIndex for HNSW { where Self: Sized, { + params.validate()?; let builder = HnswBuilder::with_params(params, storage); log::debug!( @@ -1209,13 +1546,23 @@ impl IvfSubIndex for HNSW { } let len = storage.len(); - builder.level_count[0].fetch_add(1, Ordering::Relaxed); - (1..len).into_par_iter().for_each_init( - || VisitedGenerator::new(len), - |visited_generator, node| { - builder.insert(node as u32, visited_generator, storage); - }, - ); + let entry_levels = builder.nodes[builder.entry_point as usize] + .read() + .unwrap() + .level_neighbors + .len(); + for count in builder.level_count.iter().take(entry_levels) { + count.fetch_add(1, Ordering::Relaxed); + } + (0..len) + .into_par_iter() + .filter(|node| *node as u32 != builder.entry_point) + .for_each_init( + || VisitedGenerator::new(len), + |visited_generator, node| { + builder.insert(node as u32, visited_generator, storage); + }, + ); assert_eq!(builder.level_count[0].load(Ordering::Relaxed), len); Ok(builder.finish()) @@ -1310,38 +1657,81 @@ impl IvfSubIndex for HNSW { #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; - use arrow_array::{ArrayRef, FixedSizeListArray, RecordBatch, UInt8Array, UInt32Array}; + use arrow_array::cast::AsArray; + use arrow_array::{ + ArrayRef, FixedSizeListArray, Float32Array, RecordBatch, UInt8Array, UInt32Array, + }; use arrow_schema::Schema; + use async_trait::async_trait; use lance_arrow::FixedSizeListArrayExt; - use lance_core::deepsize::DeepSizeOf; - use lance_file::previous::{ - reader::FileReader as PreviousFileReader, - writer::{ - FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions, - }, + use lance_core::{Error, Result, deepsize::DeepSizeOf}; + use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, + writer::{FileWriter as V1FileWriter, FileWriterOptions as V1FileWriterOptions}, }; use lance_io::object_store::ObjectStore; use lance_linalg::distance::DistanceType; + use lance_select::{RowAddrMask, RowAddrTreeMap}; use lance_table::format::SelfDescribingFileReader; use lance_table::io::manifest::ManifestDescribing; use lance_testing::datagen::generate_random_array; use object_store::path::Path; + use rand::{Rng, SeedableRng, rngs::SmallRng}; use rstest::rstest; - use super::HnswGraph; - use crate::scalar::IndexWriter; + use super::{ + HNSW_LEVEL_RNG_SEED, HNSW_METADATA_KEY, HnswBuilder, HnswGraph, ImmutableHnswBottomView, + ImmutableHnswLevelView, MIN_HNSW_M, random_level_with, + }; + use crate::metrics::NoOpMetricsCollector; + use crate::prefilter::PreFilter; + use crate::vector::graph::builder::GraphBuilderNode; use crate::vector::storage::{DistCalculator, VectorStore}; use crate::vector::v3::subindex::IvfSubIndex; use crate::vector::{ flat::storage::{FlatBinStorage, FlatFloatStorage}, - graph::{DISTS_FIELD, NEIGHBORS_FIELD}, + graph::{DISTS_FIELD, NEIGHBORS_FIELD, OrderedNode, VisitedGenerator}, hnsw::{ - HNSW, VECTOR_ID_FIELD, + HNSW, HnswMetadata, VECTOR_ID_FIELD, builder::{HnswBuildParams, HnswQueryParams}, }, }; + fn with_hnsw_metadata(batch: &RecordBatch, hnsw_metadata: HnswMetadata) -> RecordBatch { + let mut metadata = batch.schema_ref().metadata().clone(); + metadata.insert( + HNSW_METADATA_KEY.to_string(), + serde_json::to_string(&hnsw_metadata).unwrap(), + ); + let schema = batch.schema().as_ref().clone().with_metadata(metadata); + RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec()).unwrap() + } + + struct MaskPreFilter { + mask: Arc, + } + + #[async_trait] + impl PreFilter for MaskPreFilter { + async fn wait_for_ready(&self) -> Result<()> { + Ok(()) + } + + fn is_empty(&self) -> bool { + false + } + + fn mask(&self) -> Arc { + self.mask.clone() + } + + fn filter_row_ids<'a>(&self, row_ids: Box + 'a>) -> Vec { + self.mask.selected_indices(row_ids) + } + } + #[tokio::test] async fn test_builder_write_load() { const DIM: usize = 32; @@ -1367,18 +1757,18 @@ mod tests { DISTS_FIELD.clone(), ]); let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap(); - let mut writer = PreviousFileWriter::::with_object_writer( + let mut writer = V1FileWriter::::with_object_writer( writer, schema, - &PreviousFileWriterOptions::default(), + &V1FileWriterOptions::default(), ) .unwrap(); let batch = builder.to_batch().unwrap(); let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await.unwrap(); + writer.write(&[batch]).await.unwrap(); writer.finish_with_metadata(&metadata).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None) + let reader = V1FileReader::try_new_self_described(&object_store, &path, None) .await .unwrap(); let batch = reader @@ -1394,6 +1784,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let builder_results = builder .search_basic(query.clone(), k, ¶ms, None, store.as_ref()) @@ -1428,18 +1819,18 @@ mod tests { DISTS_FIELD.clone(), ]); let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap(); - let mut writer = PreviousFileWriter::::with_object_writer( + let mut writer = V1FileWriter::::with_object_writer( writer, schema, - &PreviousFileWriterOptions::default(), + &V1FileWriterOptions::default(), ) .unwrap(); let batch = builder.to_batch().unwrap(); let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await.unwrap(); + writer.write(&[batch]).await.unwrap(); writer.finish_with_metadata(&metadata).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None) + let reader = V1FileReader::try_new_self_described(&object_store, &path, None) .await .unwrap(); let batch = reader @@ -1455,6 +1846,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let builder_results = builder .search_basic(query.clone(), k, ¶ms, None, store.as_ref()) @@ -1475,6 +1867,165 @@ mod tests { all.into_iter().take(k).map(|(_, id)| id).collect() } + #[rstest] + #[case::zero_max_level( + HnswBuildParams::default().max_level(0), + "max_level must be greater than 0" + )] + #[case::zero_m( + HnswBuildParams::default().num_edges(0), + "m must be at least 4" + )] + #[case::one_m( + HnswBuildParams::default().num_edges(1), + "m must be at least 4" + )] + #[case::three_m( + HnswBuildParams::default().num_edges(3), + "m must be at least 4" + )] + #[case::small_ef( + HnswBuildParams::default().num_edges(20).ef_construction(19), + "ef_construction must be at least m (20)" + )] + #[case::overflowing_level_zero_limit( + HnswBuildParams::default() + .num_edges(usize::MAX) + .ef_construction(usize::MAX), + "level-0 reciprocal limit can be represented" + )] + fn test_rejects_invalid_build_params( + #[case] params: HnswBuildParams, + #[case] expected_message: &str, + ) { + let fsl = + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0, 0.0]), 2).unwrap(); + let store = FlatFloatStorage::new(fsl, DistanceType::L2); + + let error = HNSW::index_vectors(&store, params).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error.to_string().contains(expected_message), + "unexpected error: {error}" + ); + } + + /// The lowest accepted construction settings retain a useful graph, but + /// do not promise full reachability. This seeded floor makes that quality + /// trade-off explicit and guards against catastrophic fragmentation. + #[test] + fn test_minimum_params_reachability() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let mut rng = SmallRng::seed_from_u64(0); + let values = Float32Array::from( + (0..TOTAL * DIM) + .map(|_| rng.random::()) + .collect::>(), + ); + let vectors = FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap(); + let store = FlatFloatStorage::new(vectors.clone(), DistanceType::L2); + let hnsw = HNSW::index_vectors( + &store, + HnswBuildParams::default() + .num_edges(MIN_HNSW_M) + .ef_construction(MIN_HNSW_M), + ) + .unwrap(); + let results = hnsw + .search_basic( + vectors.value(0), + TOTAL, + &HnswQueryParams { + ef: TOTAL, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }, + None, + &store, + ) + .unwrap(); + + let minimum_reachable = TOTAL * 90 / 100; + assert!( + results.len() >= minimum_reachable, + "minimum HNSW construction settings reached only {} of {TOTAL} nodes; expected at least {minimum_reachable}", + results.len(), + ); + } + + /// Algorithm 1 limits a newly inserted node to M connections even on + /// level 0. The larger Mmax0 limit only applies when old nodes receive + /// reciprocal connections. + #[test] + fn test_new_node_uses_m_connections() { + // Four equidistant, mutually diverse points around the final point. + // With the old shared Mmax0 limit, node 4 retained all four. + let values = Float32Array::from(vec![ + 1.0, 0.0, // east + 0.0, 1.0, // north + -1.0, 0.0, // west + 0.0, -1.0, // south + 0.0, 0.0, // final node + ]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + let store = FlatFloatStorage::new(fsl, DistanceType::L2); + let params = HnswBuildParams::default() + .max_level(1) + .num_edges(2) + .ef_construction(5); + let builder = HnswBuilder::with_params(params, &store); + let mut visited_generator = VisitedGenerator::new(store.len()); + + for node in 1..store.len() as u32 { + builder.insert(node, &mut visited_generator, &store); + } + + let final_node = builder.nodes[4].read().unwrap(); + assert_eq!(final_node.level_neighbors_ranked[0].len(), 2); + assert!( + builder + .nodes + .iter() + .all(|node| node.read().unwrap().level_neighbors_ranked[0].len() <= 4), + "existing level-0 nodes must remain bounded by Mmax0" + ); + } + + /// Offline construction pre-assigns the same random node heights as the + /// online builder and uses the first globally highest node as the anchor + /// for parallel insertion. This matches the final entry point produced by + /// sequential dynamic promotion without forcing node 0 to full height. + #[test] + fn test_offline_entry_point_uses_random_node_levels() { + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * 2), 2).unwrap(); + let store = FlatFloatStorage::new(fsl, DistanceType::L2); + let params = HnswBuildParams::default(); + let builder = HnswBuilder::with_params(params.clone(), &store); + + let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); + let expected_levels = (0..TOTAL) + .map(|_| random_level_with(¶ms, &mut level_rng)) + .collect::>(); + let highest_level = *expected_levels.iter().max().unwrap(); + let expected_entry = expected_levels + .iter() + .position(|level| *level == highest_level) + .unwrap() as u32; + + for (node, expected_level) in builder.nodes.iter().zip(expected_levels) { + assert_eq!( + node.read().unwrap().level_neighbors.len(), + expected_level as usize + 1 + ); + } + assert_eq!(builder.entry_point, expected_entry); + } + /// The Arrow-backed loaded graph must search bit-identically to the /// in-memory build, across distance types and graph sizes (single node, /// pair, and a multi-level graph exercising the sparse upper-level @@ -1511,6 +2062,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let query = fsl.value(0); @@ -1534,17 +2086,397 @@ mod tests { assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})"); } - /// Regression guard for the `level_offsets` misalignment (issue #6746). - /// `to_batch` writes the entry-point node at *every* level, but - /// `level_count` only counts it at level 0, so the serialized batch has - /// strictly more rows than `sum(level_count)` and the upper-level - /// `level_offsets` slices are off-by-one / non-monotonic. The Arrow-backed - /// loaded graph must still search bit-identically to the in-memory build: - /// it keys upper levels by `__vector_id` value via the `Sparse` map - /// (last-write-wins), never `row == id`. A naive `row == id` - /// reimplementation would pass the small cases but break here. + /// A [`DistCalculator`] over fixed per-node distances that counts how + /// many distances it computes, so a test can assert exactly which search + /// phases ran. + struct CountingDistCalculator<'a> { + distances: &'a [f32], + calls: &'a AtomicUsize, + } + + impl DistCalculator for CountingDistCalculator<'_> { + fn distance(&self, id: u32) -> f32 { + self.calls.fetch_add(1, Ordering::Relaxed); + self.distances[id as usize] + } + + fn distance_all(&self, _k_hint: usize) -> Vec { + self.distances.to_vec() + } + } + + /// Regression test for : + /// the query-time greedy descent must stop at level 1 (HNSW paper, + /// Algorithm 2); level 0 must be searched only by the ef-bounded beam. + /// + /// Drives [`HNSW::run_search`] / [`HNSW::run_search_acorn`] directly over + /// a hand-built graph: node 0 is the entry point and the only node at + /// level 1 (with no level-1 neighbors), so the greedy descent over + /// levels >= 1 computes no distances, and with ef == N the bottom beam + /// visits every level-0 node exactly once. A greedy step at level 0 + /// would show up as 10 extra distance computations: the level-0 degrees + /// along the path are 1,2,2,2,2,1 and the strictly decreasing distances + /// make greedy walk it end to end. + #[test] + fn test_greedy_descent_stops_before_level_0() { + const N: usize = 6; + // Strictly decreasing along the path; the true top-3 is nodes 5,4,3. + let distances: Vec = (0..N).map(|id| (N - id) as f32).collect(); + + // Level 0 is the path 0-1-...-5, mirrored into both the bottom view + // and the level-0 view (only the bottom view may be used at query + // time, which is what this test asserts). + let mut nodes: Vec = (0..N as u32) + .map(|id| GraphBuilderNode::new(id, 2)) + .collect(); + for (id, node) in nodes.iter_mut().enumerate() { + let mut adjacency = Vec::new(); + if id > 0 { + adjacency.push(id as u32 - 1); + } + if id + 1 < N { + adjacency.push(id as u32 + 1); + } + let adjacency = Arc::new(adjacency); + node.bottom_neighbors = adjacency.clone(); + node.level_neighbors[0] = adjacency; + } + + let build_params = HnswBuildParams { + max_level: 2, + m: 4, + ef_construction: 10, + prefetch_distance: None, + }; + let hnsw = HNSW::from_parts(build_params, nodes.clone(), vec![N, 1], 0); + let query_params = HnswQueryParams { + ef: N, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + + let calls = AtomicUsize::new(0); + let dist_calc = CountingDistCalculator { + distances: &distances, + calls: &calls, + }; + // The entry-point distance (1 call) mirrors `search_inner`. + let ep = OrderedNode::new(0, dist_calc.distance(0).into()); + let mut visited_generator = VisitedGenerator::new(N); + + let results = hnsw.run_search( + ep.clone(), + 3, + &query_params, + None, + &mut visited_generator, + N, + None, + &dist_calc, + |level| ImmutableHnswLevelView::new(level, &nodes), + ImmutableHnswBottomView::new(&nodes), + ); + assert_eq!( + results.iter().map(|node| node.id).collect::>(), + vec![5, 4, 3] + ); + // Exactly 1 entry-point distance + 5 beam visits (one per remaining + // node); the pre-fix level-0 greedy step brought this to 16. + assert_eq!(calls.load(Ordering::Relaxed), N); + + // The ACORN traversal shares the descent. Its beam additionally seeds + // mask-passing nodes with distance computations, so only bound the + // total: a level-0 greedy step would push it past one call per node. + let mut mask_generator = VisitedGenerator::new(N); + let mut mask = mask_generator.generate(N); + for id in 0..N as u32 { + mask.insert(id); + } + let mut expanded_generator = VisitedGenerator::new(N); + calls.store(0, Ordering::Relaxed); + let results = hnsw.run_search_acorn( + ep, + &query_params, + &mask, + &mut visited_generator, + &mut expanded_generator, + N, + None, + &dist_calc, + |level| ImmutableHnswLevelView::new(level, &nodes), + ImmutableHnswBottomView::new(&nodes), + ); + assert_eq!( + results + .iter() + .take(3) + .map(|node| node.id) + .collect::>(), + vec![5, 4, 3] + ); + assert!( + calls.load(Ordering::Relaxed) <= N, + "level-0 greedy descent adds distance computations beyond the \ + beam's one per node" + ); + } + + /// Brute-force top-`k` restricted to mask-passing ids. + fn brute_force_topk_masked( + store: &FlatFloatStorage, + query: ArrayRef, + k: usize, + passes: impl Fn(u32) -> bool, + ) -> Vec { + let dist_calc = store.dist_calculator(query, 0.0); + let mut matching: Vec<(f32, u32)> = (0..store.len() as u32) + .filter(|id| passes(*id)) + .map(|id| (dist_calc.distance(id), id)) + .collect(); + matching.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap()); + matching.into_iter().take(k).map(|(_, id)| id).collect() + } + + /// ACORN returns only mask-passing nodes, searches built and loaded + /// graphs identically, and holds recall vs brute force over the mask. + #[tokio::test] + async fn test_acorn_filtered_search() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let builder = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + let loaded = HNSW::load(builder.to_batch().unwrap()).unwrap(); + + let mut mask_generator = VisitedGenerator::new(TOTAL); + let k = 10; + let params = HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let query = fsl.value(0); + let truth: std::collections::HashSet = + brute_force_topk_masked(store.as_ref(), query.clone(), k, |id| id % 2 == 0) + .into_iter() + .collect(); + + let mut all_results = vec![]; + for hnsw in [&builder, &loaded] { + let mut bitset = mask_generator.generate(TOTAL); + for id in (0..TOTAL as u32).step_by(2) { + bitset.insert(id); + } + let results = hnsw + .search_acorn(query.clone(), k, ¶ms, &bitset, store.as_ref()) + .unwrap(); + assert_eq!(results.len(), k); + assert!(results.iter().all(|node| node.id % 2 == 0)); + assert!(results.windows(2).all(|w| w[0].dist <= w[1].dist)); + let hits = results.iter().filter(|n| truth.contains(&n.id)).count(); + let recall = hits as f32 / k as f32; + assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})"); + all_results.push(results); + } + assert_eq!(all_results[0], all_results[1]); + + // default ef (k + k/2) and a deletion-style mask (all but a few rows) + let default_ef_params = HnswQueryParams { + ef: k + k / 2, + ..params + }; + for excluded_stride in [2, 400] { + let passes = |id: u32| id % excluded_stride != 1; + let mut bitset = mask_generator.generate(TOTAL); + for id in (0..TOTAL as u32).filter(|id| passes(*id)) { + bitset.insert(id); + } + let truth: std::collections::HashSet = + brute_force_topk_masked(store.as_ref(), query.clone(), k, passes) + .into_iter() + .collect(); + let results = builder + .search_acorn( + query.clone(), + k, + &default_ef_params, + &bitset, + store.as_ref(), + ) + .unwrap(); + assert_eq!(results.len(), k); + assert!(results.iter().all(|node| passes(node.id))); + let hits = results.iter().filter(|n| truth.contains(&n.id)).count(); + let recall = hits as f32 / k as f32; + assert!(recall >= 0.5, "recall {recall} below 0.5 (k={k})"); + } + } + + /// Dispatch: dense prefilters take the graph traversal, sparse ones the + /// exact flat scan, and both return only mask-passing row ids. #[tokio::test] - async fn test_loaded_level_offsets_misalignment_invariant() { + async fn test_subindex_prefilter_dispatch() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let hnsw = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + + let k = 10; + let query_key = fsl.value(0); + + let search_row_ids = |allowed: Vec, use_acorn: bool| { + let params = HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn, + }; + let filter = Arc::new(MaskPreFilter { + mask: Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allowed, + ))), + }); + let batch = hnsw + .search( + query_key.clone(), + k, + params, + store.as_ref(), + filter, + &NoOpMetricsCollector, + ) + .unwrap(); + batch[lance_core::ROW_ID] + .as_primitive::() + .values() + .to_vec() + }; + + // Dense mask (50% of rows), in both modes. + let dense: Vec = (0..TOTAL as u64).step_by(2).collect(); + for use_acorn in [false, true] { + let row_ids = search_row_ids(dense.clone(), use_acorn); + assert_eq!(row_ids.len(), k); + assert!(row_ids.iter().all(|id| id % 2 == 0)); + } + + // All-pass mask: shortcuts to the unfiltered path. + let all: Vec = (0..TOTAL as u64).collect(); + let unfiltered = hnsw + .search_basic( + query_key.clone(), + k, + &HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }, + None, + store.as_ref(), + ) + .unwrap(); + let row_ids = search_row_ids(all, true); + assert_eq!( + row_ids, + unfiltered.iter().map(|n| n.id as u64).collect::>() + ); + + // Sparse mask (< 10% of rows): the flat scan, which is exact. + let sparse: Vec = (0..TOTAL as u64).step_by(25).collect(); + let row_ids = search_row_ids(sparse.clone(), true); + assert_eq!(row_ids.len(), k); + let truth = brute_force_topk_masked(store.as_ref(), query_key.clone(), k, |id| { + sparse.contains(&(id as u64)) + }); + let mut got: Vec = row_ids.iter().map(|id| *id as u32).collect(); + got.sort_unstable(); + let mut expected = truth; + expected.sort_unstable(); + assert_eq!(got, expected); + } + + #[rstest] + #[case::prefetch(Some(2))] + #[case::no_prefetch(None)] + fn test_distance_range_prefilter_dispatch(#[case] prefetch_distance: Option) { + const DIM: usize = 32; + const TOTAL: usize = 100; + + let mut values = vec![0.0; TOTAL * DIM]; + for row in 1..TOTAL { + values[row * DIM] = row as f32; + } + let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(values), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let hnsw = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams { + prefetch_distance, + ..HnswBuildParams::default() + }, + ) + .unwrap(); + let query = fsl.value(0); + + let search_row_ids = |allowed: Vec| { + let filter = Arc::new(MaskPreFilter { + mask: Arc::new(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allowed, + ))), + }); + let batch = hnsw + .search( + query.clone(), + 10, + HnswQueryParams { + ef: TOTAL, + lower_bound: Some(0.0), + upper_bound: Some(1.0), + dist_q_c: 0.0, + use_acorn: false, + }, + store.as_ref(), + filter, + &NoOpMetricsCollector, + ) + .unwrap(); + batch[lance_core::ROW_ID] + .as_primitive::() + .values() + .to_vec() + }; + + // Sparse masks take the exact flat scan, while dense masks traverse + // the graph. Both must include the lower bound and exclude the upper. + assert_eq!(search_row_ids(vec![0, 1, 2]), vec![0]); + assert_eq!(search_row_ids((0..60).collect()), vec![0]); + } + + /// Every fresh `level_offsets` range must exactly delimit the rows emitted + /// for that HNSW level (issue #5156). + #[test] + fn test_level_offsets_match_serialized_levels() { use arrow::array::AsArray; use arrow::datatypes::UInt32Type; @@ -1553,7 +2485,7 @@ mod tests { let fsl = FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) .unwrap(); - let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2)); let builder = HNSW::index_vectors( store.as_ref(), HnswBuildParams::default().num_edges(20).ef_construction(50), @@ -1568,53 +2500,122 @@ mod tests { ); let batch = builder.to_batch().unwrap(); - let md = builder.metadata(); - let total_counted = *md.level_offsets.last().unwrap(); - - // The exact misalignment: more serialized rows than `level_count` sums - // to, because the entry-point node is written at every level yet - // counted only at level 0. - assert!( - batch.num_rows() > total_counted, - "expected serialized rows ({}) to exceed sum(level_count) ({}) -- \ - entry point should be written at every level", + let metadata = builder.metadata(); + assert_eq!( + *metadata.level_offsets.last().unwrap(), batch.num_rows(), - total_counted, + "level offsets must cover every serialized row", ); - // Level-0 slice must still be exactly `[0, N)` with - // `__vector_id == row` -- the precondition for `LevelLookup::Dense`. - let n = md.level_offsets[1]; - assert_eq!(n, TOTAL); - let level0 = batch.slice(0, n); - let ids = level0.column(0).as_primitive::(); - assert!( - ids.values() + let nodes = builder.nodes().unwrap(); + for level in 0..builder.max_level() as usize { + let start = metadata.level_offsets[level]; + let end = metadata.level_offsets[level + 1]; + let level_batch = batch.slice(start, end - start); + let ids = level_batch.column(0).as_primitive::(); + let expected_ids = nodes .iter() .enumerate() - .all(|(row, id)| *id == row as u32), - "level-0 __vector_id must equal the row index", + .filter_map(|(id, node)| (level < node.level_neighbors.len()).then_some(id as u32)) + .collect::>(); + + assert_eq!( + ids.values().as_ref(), + expected_ids.as_slice(), + "serialized ids do not match level {level}", + ); + assert_eq!(builder.num_nodes(level), expected_ids.len()); + } + } + + /// Version-1 readers use the configured max level to index serialized + /// level batches directly. Empty trailing ranges keep that persisted + /// shape while current readers use only the sampled, non-empty height. + #[test] + fn test_metadata_preserves_configured_empty_levels() { + const CONFIGURED_LEVELS: usize = 7; + let params = HnswBuildParams::default().max_level(CONFIGURED_LEVELS as u16); + let hnsw = HNSW::from_parts(params, vec![GraphBuilderNode::new(0, 1)], vec![1], 0); + + assert_eq!(hnsw.max_level(), 1); + let metadata = hnsw.metadata(); + assert_eq!(metadata.level_offsets.len(), CONFIGURED_LEVELS + 1); + assert_eq!(metadata.level_offsets[0], 0); + assert!( + metadata.level_offsets[1..] + .iter() + .all(|offset| *offset == 1) ); - // Despite the surplus rows and off-by-one upper slices, the loaded - // graph searches bit-identically to the in-memory build (old `load` - // semantics preserved via the `Sparse` last-write-wins map). - let loaded = HNSW::load(batch).unwrap(); + let loaded = HNSW::load(hnsw.to_batch().unwrap()).unwrap(); + assert_eq!(loaded.max_level(), 1); + match &loaded.inner.graph { + HnswGraph::Loaded(graph) => { + assert_eq!(graph.level_neighbors.len(), CONFIGURED_LEVELS); + assert_eq!(graph.level_count[0], 1); + assert!(graph.level_count[1..].iter().all(|count| *count == 0)); + } + HnswGraph::Built(_) => panic!("expected an Arrow-backed loaded graph"), + } + } + + /// Indices written before issue #5156 was fixed omitted the entry point + /// from every upper-level count. Loading those misaligned slices must keep + /// the previous id-keyed, last-write-wins behavior. + #[test] + fn test_load_legacy_misaligned_level_offsets() { + const DIM: usize = 32; + const TOTAL: usize = 2048; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl.clone(), DistanceType::L2)); + let builder = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + assert!(builder.max_level() >= 2); + + let batch = builder.to_batch().unwrap(); + let mut metadata = builder.metadata(); + let mut legacy_offsets = Vec::with_capacity(metadata.level_offsets.len()); + legacy_offsets.push(0); + for level in 0..builder.max_level() as usize { + let level_count = metadata.level_offsets[level + 1] - metadata.level_offsets[level]; + let legacy_level_count = level_count - usize::from(level > 0); + legacy_offsets.push(legacy_offsets.last().unwrap() + legacy_level_count); + } + metadata.level_offsets = legacy_offsets; + assert!(*metadata.level_offsets.last().unwrap() < batch.num_rows()); + + let legacy_batch = with_hnsw_metadata(&batch, metadata); + let loaded = HNSW::load(legacy_batch).unwrap(); assert!(matches!(loaded.inner.graph, HnswGraph::Loaded(_))); let params = HnswQueryParams { ef: 50, lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; - let query = fsl.value(0); - let builder_results = builder - .search_basic(query.clone(), 10, ¶ms, None, store.as_ref()) - .unwrap(); - let loaded_results = loaded - .search_basic(query, 10, ¶ms, None, store.as_ref()) - .unwrap(); - assert_eq!(builder_results, loaded_results); + let entry_point = builder.inner.entry_point as usize; + let query_indices = [0, 1, TOTAL / 3, TOTAL - 1] + .into_iter() + .filter(|query_index| *query_index != entry_point) + .take(3) + .collect::>(); + assert_eq!(query_indices.len(), 3); + for query_index in query_indices { + let query = fsl.value(query_index); + let builder_results = builder + .search_basic(query.clone(), 10, ¶ms, None, store.as_ref()) + .unwrap(); + let loaded_results = loaded + .search_basic(query, 10, ¶ms, None, store.as_ref()) + .unwrap(); + assert_eq!(builder_results, loaded_results); + } } /// `load()` must reject a batch whose level-0 `__vector_id` no longer @@ -1658,14 +2659,184 @@ mod tests { ); } + /// A graph whose nodes outrun its storage must be refused, not faulted. + /// + /// Node ids are storage row numbers, so scoring a node past the last row + /// indexes out of bounds and panics the worker rather than failing the + /// query. The entry point is scored before any traversal decision, so the + /// refusal has to come first. Storage with rows the graph never reached is + /// left alone -- that direction is safe and ordinary. + #[test] + fn search_refuses_a_graph_its_storage_cannot_cover() { + const DIM: usize = 16; + const NODES: usize = 256; + let build_store = |rows: usize| { + let fsl = FixedSizeListArray::try_new_from_values( + generate_random_array(rows * DIM), + DIM as i32, + ) + .unwrap(); + Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2)) + }; + + let full = build_store(NODES); + let hnsw = HNSW::index_vectors( + full.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + assert_eq!(hnsw.len(), NODES); + + let params = HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let query = Arc::new(generate_random_array(DIM)) as ArrayRef; + + // Storage short of the graph: refused with a message, never scored. + let short = build_store(NODES / 4); + let refused = hnsw.search_basic(query.clone(), 10, ¶ms, None, short.as_ref()); + let message = refused + .expect_err("a graph its storage cannot cover must be refused") + .to_string(); + assert!( + message.contains("no vector to score"), + "the error has to name the defect, got: {message}" + ); + + // The safe direction, and the matching one, both still search. + let over = build_store(NODES * 2); + for storage in [full.as_ref(), over.as_ref()] { + let results = hnsw + .search_basic(query.clone(), 10, ¶ms, None, storage) + .expect("storage that covers the graph must search"); + assert!(!results.is_empty()); + } + } + + /// The domain scan must see only the rows it was handed. + /// + /// Each level is a slice of the concatenated batch, and `ListArray::values()` + /// hands back the whole child array regardless of the slice, so a scan over + /// it would count another level's ids. + #[test] + fn neighbors_within_domain_counts_only_the_sliced_rows() { + use arrow::array::{ListBuilder, UInt32Builder}; + use arrow_array::Array; + + use super::neighbors_within_domain; + + const NODE_COUNT: usize = 4; + let mut builder = ListBuilder::with_capacity(UInt32Builder::new(), 4); + // Rows 0..2 stay inside the domain; rows 2..4 do not. + builder.append_value([Some(0u32), Some(1)]); + builder.append_value([Some(2u32), Some(3)]); + builder.append_value([Some(99u32)]); + builder.append_value([Some(100u32)]); + let all = builder.finish(); + + let clean = all.slice(0, 2); + let (out, dropped) = neighbors_within_domain(&clean, NODE_COUNT); + assert_eq!(dropped, 0, "a clean slice must report no dropped ids"); + assert_eq!(out.len(), 2); + assert_eq!(out.value(0).len(), 2, "a clean slice keeps its ids"); + + let dirty = all.slice(2, 2); + let (out, dropped) = neighbors_within_domain(&dirty, NODE_COUNT); + assert_eq!(dropped, 2, "both out-of-domain ids are counted"); + assert_eq!(out.value(0).len(), 0, "the bad id is gone"); + assert_eq!(out.value(1).len(), 0); + } + + /// A dangling neighbor id must be gone before search, not caught at lookup. + /// + /// Traversal scores a neighbor before it is ever looked up as a node, so a + /// guard inside the node lookup runs too late -- the id has already reached + /// the distance calculator. Indices written before the writer bounded its + /// own snapshot carry such edges, so `load()` drops them and the query + /// still answers. + #[tokio::test] + async fn test_load_drops_neighbor_ids_outside_the_graph() { + use arrow::array::{AsArray, ListBuilder, UInt32Builder}; + use arrow::datatypes::UInt32Type; + use arrow_array::Array; + + const DIM: usize = 16; + const TOTAL: usize = 256; + let fsl = + FixedSizeListArray::try_new_from_values(generate_random_array(TOTAL * DIM), DIM as i32) + .unwrap(); + let store = Arc::new(FlatFloatStorage::new(fsl, DistanceType::L2)); + let builder = HNSW::index_vectors( + store.as_ref(), + HnswBuildParams::default().num_edges(20).ef_construction(50), + ) + .unwrap(); + let batch = builder.to_batch().unwrap(); + + // Put an out-of-domain edge on every node, the shape a writer that + // snapshotted a node count while reading adjacency live would persist. + // Every node, so whichever ones traversal expands, it scores the bad id + // -- `FlatFloatStorage::dist_calculator` panics on an id past its rows. + let neighbors = batch.column(1).as_list::(); + let mut rebuilt = ListBuilder::with_capacity(UInt32Builder::new(), neighbors.len()); + for row in 0..neighbors.len() { + let ids = neighbors.value(row); + let ids = ids.as_primitive::(); + let mut ids: Vec = ids.values().to_vec(); + ids.insert(0, TOTAL as u32 + 7); + rebuilt.append_value(ids.into_iter().map(Some)); + } + let mut columns = batch.columns().to_vec(); + columns[1] = Arc::new(rebuilt.finish()); + // `__distance` is now shorter than `__neighbors` per row, which search + // does not read; the ids are what traversal follows. + let corrupted = RecordBatch::try_new(batch.schema(), columns).unwrap(); + + let corrupted_bytes = corrupted.get_array_memory_size(); + let clean_loaded = HNSW::load(batch.clone()).expect("the clean batch loads"); + let loaded = HNSW::load(corrupted).expect("a dangling edge must not fail the load"); + // A clean load borrows every level from its batch, so it charges little + // beyond it. A repaired level owns its buffers, and they have to be + // charged too or the index cache sizes itself from memory it is not + // holding. Measured: ~0.3 KiB over for clean, ~38 KiB for repaired. + let clean_over = clean_loaded.deep_size_of() - batch.get_array_memory_size(); + let repaired_over = loaded.deep_size_of() - corrupted_bytes; + assert!( + clean_over < 1024, + "a clean load keeps its levels zero-copy, but charged {clean_over} bytes over its batch" + ); + assert!( + repaired_over > 16 * 1024, + "a repaired load must charge the buffers it owns, but charged only \ + {repaired_over} bytes over its batch" + ); + + assert_eq!(loaded.len(), TOTAL); + // Searching has to answer rather than panic on the out-of-domain id. + let query = Arc::new(generate_random_array(DIM)) as ArrayRef; + let params = HnswQueryParams { + ef: 50, + lower_bound: None, + upper_bound: None, + dist_q_c: 0.0, + use_acorn: false, + }; + let results = loaded + .search_basic(query, 10, ¶ms, None, store.as_ref()) + .expect("search must survive a dropped edge"); + assert!(!results.is_empty(), "the query still returns neighbors"); + } + /// `load()` must reject metadata whose `entry_point` is out of range for /// the node count: it indexes the `Dense` level-0 lookup directly, so an /// out-of-range value would read past the level-0 neighbor buffer at search /// time. #[tokio::test] async fn test_load_rejects_out_of_range_entry_point() { - use super::{HNSW_METADATA_KEY, HnswMetadata}; - const DIM: usize = 16; const TOTAL: usize = 256; let fsl = @@ -1679,21 +2850,18 @@ mod tests { .unwrap(); let batch = builder.to_batch().unwrap(); - let mut metadata = batch.schema_ref().metadata().clone(); - let mut md: HnswMetadata = - serde_json::from_str(metadata.get(HNSW_METADATA_KEY).unwrap()).unwrap(); + let mut md: HnswMetadata = serde_json::from_str( + batch + .schema_ref() + .metadata() + .get(HNSW_METADATA_KEY) + .unwrap(), + ) + .unwrap(); // Valid entry points are `[0, N)`; `level_offsets[1]` == N is one past. let n = md.level_offsets[1]; md.entry_point = n as u32; - metadata.insert( - HNSW_METADATA_KEY.to_string(), - serde_json::to_string(&md).unwrap(), - ); - // Rebuild the batch under the rewritten metadata. `with_schema` would - // reject this: it requires the new metadata to be a superset, but we - // are changing an existing key's value, not adding one. - let schema = batch.schema().as_ref().clone().with_metadata(metadata); - let corrupted = RecordBatch::try_new(Arc::new(schema), batch.columns().to_vec()).unwrap(); + let corrupted = with_hnsw_metadata(&batch, md); assert!( HNSW::load(corrupted).is_err(), @@ -1752,6 +2920,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let query = fsl.value(7); let a = builder diff --git a/rust/lance-index/src/vector/hnsw/index.rs b/rust/lance-index/src/vector/hnsw/index.rs index dca9ea78955..4a5d23e63f2 100644 --- a/rust/lance-index/src/vector/hnsw/index.rs +++ b/rust/lance-index/src/vector/hnsw/index.rs @@ -16,7 +16,7 @@ use lance_arrow::RecordBatchExt; use lance_core::ROW_ID; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result, datatypes::Schema}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::traits::Reader; use lance_linalg::distance::DistanceType; use lance_table::format::SelfDescribingFileReader; @@ -70,8 +70,7 @@ impl HNSWIndex { aux_reader: Arc, options: HNSWIndexOptions, ) -> Result { - let reader = - PreviousFileReader::try_new_self_described_from_reader(reader.clone(), None).await?; + let reader = V1FileReader::try_new_self_described_from_reader(reader.clone(), None).await?; let partition_metadata = match reader.schema().metadata.get(IVF_PARTITION_KEY) { Some(json) => { @@ -215,7 +214,7 @@ impl VectorIndex for HNSWIndex { VECTOR_ID_FIELD.clone(), ]))?; - let reader = PreviousFileReader::try_new_from_reader( + let reader = V1FileReader::try_new_from_reader( reader.path(), reader.clone(), None, @@ -247,7 +246,7 @@ impl VectorIndex for HNSWIndex { length: usize, partition_id: usize, ) -> Result> { - let reader = PreviousFileReader::try_new_self_described_from_reader(reader, None).await?; + let reader = V1FileReader::try_new_self_described_from_reader(reader, None).await?; let metadata = self.get_partition_metadata(partition_id)?; let storage = Arc::new(self.partition_storage.load_partition(partition_id).await?); diff --git a/rust/lance-index/src/vector/hnsw/online.rs b/rust/lance-index/src/vector/hnsw/online.rs index 8fbdcbcb1c5..170884f7748 100644 --- a/rust/lance-index/src/vector/hnsw/online.rs +++ b/rust/lance-index/src/vector/hnsw/online.rs @@ -18,7 +18,7 @@ //! //! # Lifecycle //! -//! 1. `OnlineHnswBuilder::with_capacity(...)` pre-allocates fixed-size node +//! 1. `OnlineHnswBuilder::try_with_capacity(...)` pre-allocates fixed-size node //! arrays. Each slot has its target level pre-assigned so concurrent //! inserts don't need to allocate. //! 2. Writer calls `insert(id, storage)` for `id` in `0..capacity`. The vector @@ -29,22 +29,24 @@ //! 4. `finalize()` consumes the builder and returns an immutable //! [`super::HNSW`] that can be serialized via `HNSW::to_batch()`. -use std::cmp::min; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; use arc_swap::ArcSwap; use crossbeam_queue::ArrayQueue; -use rand::{Rng, SeedableRng, rngs::SmallRng}; +use rand::{SeedableRng, rngs::SmallRng}; -use super::builder::{HNSW, HNSW_LEVEL_RNG_SEED, HnswBuildParams, HnswQueryParams}; +use super::builder::{ + HNSW, HNSW_LEVEL_RNG_SEED, HnswBuildParams, HnswQueryParams, random_level_with, +}; use super::select_neighbors_heuristic; use crate::vector::graph::builder::GraphBuilderNode; use crate::vector::graph::{ Graph, OrderedFloat, OrderedNode, VisitedGenerator, beam_search, greedy_search, }; use crate::vector::storage::{DistCalculator, VectorStore}; +use lance_core::Result; use lance_core::utils::tokio::get_num_compute_intensive_cpus; /// A node in the online HNSW graph. @@ -97,22 +99,6 @@ impl OnlineGraphBuilderNode { ranked[level as usize].push(OrderedNode { dist, id: v }); } - fn cutoff(&self, level: u16, max_size: usize) -> OrderedFloat { - if !self.has_level(level) { - return OrderedFloat(f32::NEG_INFINITY); - } - let ranked = self - .level_neighbors_ranked - .lock() - .expect("level_neighbors_ranked mutex poisoned"); - let neighbors = &ranked[level as usize]; - if neighbors.len() < max_size { - OrderedFloat(f32::INFINITY) - } else { - neighbors.last().unwrap().dist - } - } - /// Rebuild `level_neighbors[level]` from the current ranked list and /// publish it via `ArcSwap`. Also updates `bottom_neighbors` for level 0. fn publish_from_ranked(&self, level: u16) { @@ -156,28 +142,48 @@ pub struct OnlineHnswBuilder { } impl OnlineHnswBuilder { - /// Create a new builder with the given capacity. Each node's target level - /// is pre-assigned at random. + /// Create a new builder with validated parameters. + /// + /// Each node's target level is pre-assigned at random. + /// + /// # Examples + /// + /// ``` + /// use lance_index::vector::hnsw::{OnlineHnswBuilder, builder::HnswBuildParams}; + /// + /// let builder = OnlineHnswBuilder::try_with_capacity( + /// 1_024, + /// HnswBuildParams::default(), + /// )?; + /// assert_eq!(builder.capacity(), 1_024); + /// # Ok::<(), lance_core::Error>(()) + /// ``` + pub fn try_with_capacity(capacity: usize, params: HnswBuildParams) -> Result { + params.validate()?; + Ok(Self::new(capacity, params)) + } + + /// Create a new builder with the given capacity. + /// + /// Use [`Self::try_with_capacity`] to handle invalid parameters without + /// panicking. + /// + /// # Panics + /// + /// Panics when `params` violates an HNSW construction precondition. + #[deprecated(note = "use OnlineHnswBuilder::try_with_capacity")] pub fn with_capacity(capacity: usize, params: HnswBuildParams) -> Self { - assert!( - params.max_level > 0, - "HnswBuildParams::max_level must be > 0" - ); + Self::try_with_capacity(capacity, params) + .unwrap_or_else(|error| panic!("invalid HNSW build parameters: {error}")) + } + + fn new(capacity: usize, params: HnswBuildParams) -> Self { let max_level = params.max_level; let level_count = (0..max_level).map(|_| AtomicUsize::new(0)).collect(); let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); let nodes: Vec<_> = (0..capacity) - .map(|i| { - let target_level = if i == 0 { - // First inserted node anchors the graph; matches offline - // builder which always starts at level 0. - 0 - } else { - Self::random_level_with(¶ms, &mut level_rng) - }; - OnlineGraphBuilderNode::new(target_level) - }) + .map(|_| OnlineGraphBuilderNode::new(random_level_with(¶ms, &mut level_rng))) .collect(); let queue_size = get_num_compute_intensive_cpus().max(1); @@ -196,14 +202,6 @@ impl OnlineHnswBuilder { } } - fn random_level_with(params: &HnswBuildParams, rng: &mut R) -> u16 { - let ml = 1.0 / (params.m as f32).ln(); - min( - (-rng.random::().ln() * ml) as u16, - params.max_level - 1, - ) - } - pub fn capacity(&self) -> usize { self.nodes.len() } @@ -258,10 +256,11 @@ impl OnlineHnswBuilder { } let entry = self.entry_point.load(Ordering::Acquire); + let entry_target_level = nodes[entry as usize].target_level(); let mut ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); // Walk down upper levels with greedy_search to refine the entry point. - for level in (target_level + 1..self.params.max_level).rev() { + for level in (target_level + 1..=entry_target_level).rev() { let cur_level = OnlineHnswLevelView::new(level, nodes); ep = greedy_search(&cur_level, ep, &dist_calc, self.params.prefetch_distance); } @@ -284,7 +283,9 @@ impl OnlineHnswBuilder { } current_node.add_neighbor(neighbor.id, neighbor.dist, level); } - self.prune(storage, current_node, level); + // New nodes select M connections at every level. The larger + // level-0 limit applies only to reciprocal edges on old nodes. + self.prune(storage, current_node, level, self.params.m); // Snapshot the pruned ranked list before publishing. let snapshot = { let ranked = current_node @@ -310,18 +311,16 @@ impl OnlineHnswBuilder { // Add reverse edges to chosen neighbors, prune them too. for (level, pruned_neighbors) in pruned_neighbors_per_level.iter().enumerate() { let level = level as u16; - let m_max = if level == 0 { + let reciprocal_limit = if level == 0 { self.params.m * 2 } else { self.params.m }; - for unpruned_edge in pruned_neighbors { - let chosen = &nodes[unpruned_edge.id as usize]; - if unpruned_edge.dist < chosen.cutoff(level, m_max) { - chosen.add_neighbor(id, unpruned_edge.dist, level); - self.prune(storage, chosen, level); - chosen.publish_from_ranked(level); - } + for selected_edge in pruned_neighbors { + let chosen = &nodes[selected_edge.id as usize]; + chosen.add_neighbor(id, selected_edge.dist, level); + self.prune(storage, chosen, level, reciprocal_limit); + chosen.publish_from_ranked(level); } } @@ -331,7 +330,6 @@ impl OnlineHnswBuilder { // plain store to keep `entry_point` updates atomic against concurrent // searches and to leave the door open if the writer ever becomes // multi-threaded. - let entry_target_level = nodes[entry as usize].target_level(); if target_level > entry_target_level { let _ = self.entry_point @@ -359,6 +357,7 @@ impl OnlineHnswBuilder { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, dist_calc, None, @@ -367,22 +366,23 @@ impl OnlineHnswBuilder { ) } - fn prune(&self, storage: &impl VectorStore, node: &OnlineGraphBuilderNode, level: u16) { - let m_max = if level == 0 { - self.params.m * 2 - } else { - self.params.m - }; - + fn prune( + &self, + storage: &impl VectorStore, + node: &OnlineGraphBuilderNode, + level: u16, + max_connections: usize, + ) { let mut ranked = node .level_neighbors_ranked .lock() .expect("level_neighbors_ranked mutex poisoned"); let level_neighbors = ranked[level as usize].clone(); - if level_neighbors.len() <= m_max { + if level_neighbors.len() <= max_connections { return; } - ranked[level as usize] = select_neighbors_heuristic(storage, &level_neighbors, m_max); + ranked[level as usize] = + select_neighbors_heuristic(storage, &level_neighbors, max_connections); } /// Search the graph for the k nearest neighbors of `query`. @@ -418,7 +418,7 @@ impl OnlineHnswBuilder { let mut ep = OrderedNode::new(entry, dist_calc.distance(entry).into()); let nodes = self.nodes.as_slice(); - for level in (1..self.params.max_level).rev() { + for level in (1..=nodes[entry as usize].target_level()).rev() { let cur_level = OnlineHnswLevelView::new(level, nodes); ep = greedy_search(&cur_level, ep, &dist_calc, self.params.prefetch_distance); } @@ -430,6 +430,7 @@ impl OnlineHnswBuilder { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let result = beam_search( &bottom, @@ -452,37 +453,79 @@ impl OnlineHnswBuilder { /// Snapshot the current graph as an immutable on-disk Lance HNSW. /// - /// Only nodes whose insert has fully completed are included. Caller must - /// ensure no concurrent inserts while this runs. + /// Only nodes whose insert has fully completed are included. + /// + /// `level_count` is recomputed from the actual per-level emissions so the + /// serialized batch and metadata stay in sync. + /// + /// # Self-containment under concurrent insert + /// + /// The node *count* is snapshotted (`inserted_len`) but each node's + /// adjacency is read live, so an insert racing this freeze can append + /// itself to an already-visited node's neighbor list. The snapshot + /// therefore drops references to ids it does not contain -- edges and the + /// entry point alike. An edge to an excluded node has no meaning in the + /// snapshot, so nothing representable is lost, and the frozen graph is + /// self-contained by construction rather than by convention. + /// + /// Ids must be dense and ascending from 0: `id` indexes the pre-allocated + /// node array and the first `inserted_len` slots are taken as the completed + /// nodes. /// - /// The entry point node is padded to full `max_level` height (with empty - /// neighbor lists at unused levels) so that search at upper levels can - /// safely traverse from it. `level_count` is recomputed from the actual - /// per-level emissions so the serialized batch and metadata stay in sync. pub fn to_hnsw(&self) -> HNSW { let inserted = self.inserted_len.load(Ordering::Acquire); - let entry_point = self.entry_point.load(Ordering::Acquire); - let max_level = self.params.max_level as usize; + // Ids are dense and ascending from 0, so the count bounds them. + let inserted_u32 = u32::try_from(inserted).unwrap_or(u32::MAX); + // The entry point is promoted before `inserted_len` is bumped, so a + // racing insert can publish itself here while this snapshot excludes + // it. An entry point outside the snapshot dangles exactly as an edge to + // one does, and search starting from an absent node finds nothing at + // all -- so fall back to the deepest node the snapshot does hold. + let published_entry = self.entry_point.load(Ordering::Acquire); + let entry_point = if published_entry < inserted_u32 { + published_entry + } else { + self.nodes + .iter() + .take(inserted) + .enumerate() + .max_by_key(|(_, node)| node.level_neighbors.len()) + .map(|(id, _)| id as u32) + .unwrap_or(0) + }; + let actual_levels = if inserted == 0 { + 0 + } else { + self.nodes[entry_point as usize].level_neighbors.len() + }; + // Retains the common case's `Arc` without copying: only a list that + // actually contains an out-of-snapshot id is rebuilt. let mut frozen_nodes: Vec = Vec::with_capacity(inserted); - for (idx, node) in self.nodes.iter().enumerate().take(inserted) { - let mut level_neighbors: Vec>> = node - .level_neighbors - .iter() - .map(|sl| sl.load_full()) - .collect(); - let mut level_neighbors_ranked = node + for node in self.nodes.iter().take(inserted) { + // Both serialized columns come from this one snapshot. Reading the + // published id lists separately pairs `__neighbors` with a + // `__distance` captured at a different moment: `level_neighbors` is + // a cache `publish_from_ranked` rebuilds, so a prune landing between + // the two reads drops ids the snapshot filter cannot restore and the + // columns disagree. + let level_neighbors_ranked: Vec> = node .level_neighbors_ranked .lock() .expect("level_neighbors_ranked mutex poisoned") - .clone(); - - if idx as u32 == entry_point { - while level_neighbors.len() < max_level { - level_neighbors.push(Arc::new(Vec::new())); - level_neighbors_ranked.push(Vec::new()); - } - } + .iter() + .map(|ranked| { + ranked + .iter() + .filter(|n| n.id < inserted_u32) + .cloned() + .collect() + }) + .collect(); + let level_neighbors: Vec>> = level_neighbors_ranked + .iter() + .map(|ranked| Arc::new(ranked.iter().map(|n| n.id).collect())) + .collect(); let bottom_neighbors = level_neighbors .first() @@ -495,9 +538,9 @@ impl OnlineHnswBuilder { )); } - let mut level_count: Vec = vec![0; max_level]; + let mut level_count: Vec = vec![0; actual_levels]; for node in &frozen_nodes { - let levels = node.level_neighbors.len().min(max_level); + let levels = node.level_neighbors.len().min(actual_levels); for count in level_count.iter_mut().take(levels) { *count += 1; } @@ -566,10 +609,15 @@ impl Graph for OnlineHnswBottomView<'_> { mod tests { use super::*; use crate::vector::flat::storage::FlatFloatStorage; - use arrow_array::FixedSizeListArray; + use std::sync::atomic::AtomicBool; + + // `to_batch` lives on the trait. + use crate::vector::v3::subindex::IvfSubIndex; + use arrow_array::{FixedSizeListArray, Float32Array}; use lance_arrow::FixedSizeListArrayExt; use lance_linalg::distance::DistanceType; use lance_testing::datagen::generate_random_array; + use rstest::rstest; use std::sync::Arc; fn build_storage(n: usize, dim: usize) -> (Arc, FixedSizeListArray) { @@ -579,6 +627,47 @@ mod tests { (storage, fsl) } + #[rstest] + #[case::zero_max_level( + HnswBuildParams::default().max_level(0), + "max_level must be greater than 0" + )] + #[case::zero_m( + HnswBuildParams::default().num_edges(0), + "m must be at least 4" + )] + #[case::one_m( + HnswBuildParams::default().num_edges(1), + "m must be at least 4" + )] + #[case::three_m( + HnswBuildParams::default().num_edges(3), + "m must be at least 4" + )] + #[case::small_ef( + HnswBuildParams::default().num_edges(20).ef_construction(19), + "ef_construction must be at least m (20)" + )] + #[case::overflowing_level_zero_limit( + HnswBuildParams::default() + .num_edges(usize::MAX) + .ef_construction(usize::MAX), + "level-0 reciprocal limit can be represented" + )] + fn test_try_with_capacity_rejects_invalid_params( + #[case] params: HnswBuildParams, + #[case] expected_message: &str, + ) { + let Err(error) = OnlineHnswBuilder::try_with_capacity(2, params) else { + panic!("expected invalid HNSW parameters to be rejected"); + }; + assert!(matches!(&error, lance_core::Error::InvalidInput { .. })); + assert!( + error.to_string().contains(expected_message), + "unexpected error: {error}" + ); + } + #[test] fn test_online_hnsw_recall() { const N: usize = 1000; @@ -588,7 +677,7 @@ mod tests { let params = HnswBuildParams::default() .num_edges(16) .ef_construction(100); - let builder = OnlineHnswBuilder::with_capacity(N, params); + let builder = OnlineHnswBuilder::try_with_capacity(N, params).unwrap(); for i in 0..N { builder.insert(i as u32, storage.as_ref()); @@ -644,7 +733,7 @@ mod tests { let params = HnswBuildParams::default() .num_edges(16) .ef_construction(100); - let builder = OnlineHnswBuilder::with_capacity(N, params); + let builder = OnlineHnswBuilder::try_with_capacity(N, params).unwrap(); for i in 0..N { builder.insert(i as u32, storage.as_ref()); } @@ -662,6 +751,7 @@ mod tests { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }, None, &mut visited, @@ -688,9 +778,147 @@ mod tests { #[test] fn test_online_hnsw_empty_search() { let params = HnswBuildParams::default(); - let builder = OnlineHnswBuilder::with_capacity(16, params); + let builder = OnlineHnswBuilder::try_with_capacity(16, params).unwrap(); let (storage, fsl) = build_storage(1, 8); let results = builder.search(fsl.value(0), 10, 32, storage.as_ref()); assert!(results.is_empty()); } + + /// Online construction uses the same Algorithm 1 distinction as the + /// offline builder: M for a new node and Mmax0 for reciprocal edges. + #[test] + fn test_online_new_node_uses_m_connections() { + let values = Float32Array::from(vec![ + 1.0, 0.0, // east + 0.0, 1.0, // north + -1.0, 0.0, // west + 0.0, -1.0, // south + 0.0, 0.0, // final node + ]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + let storage = FlatFloatStorage::new(fsl, DistanceType::L2); + let params = HnswBuildParams::default() + .max_level(1) + .num_edges(2) + .ef_construction(5); + // Bypass public validation to isolate the Algorithm 1 degree rule. + let builder = OnlineHnswBuilder::new(5, params); + + for id in 0..5 { + builder.insert(id, &storage); + } + + let final_node = builder.nodes[4].level_neighbors_ranked.lock().unwrap(); + assert_eq!(final_node[0].len(), 2); + drop(final_node); + assert!( + builder + .nodes + .iter() + .all(|node| { node.level_neighbors_ranked.lock().unwrap()[0].len() <= 4 }), + "existing level-0 nodes must remain bounded by Mmax0" + ); + } + + #[test] + fn test_online_promotes_first_highest_random_level() { + const TOTAL: usize = 128; + let (storage, _) = build_storage(TOTAL, 4); + let params = HnswBuildParams::default(); + let builder = OnlineHnswBuilder::try_with_capacity(TOTAL, params.clone()).unwrap(); + + let mut level_rng = SmallRng::seed_from_u64(HNSW_LEVEL_RNG_SEED); + let expected_levels = (0..TOTAL) + .map(|_| random_level_with(¶ms, &mut level_rng)) + .collect::>(); + for (node, expected_level) in builder.nodes.iter().zip(&expected_levels) { + assert_eq!(node.target_level(), *expected_level); + } + + for id in 0..TOTAL as u32 { + builder.insert(id, storage.as_ref()); + } + let highest_level = *expected_levels.iter().max().unwrap(); + let expected_entry = expected_levels + .iter() + .position(|level| *level == highest_level) + .unwrap() as u32; + assert_eq!(builder.entry_point.load(Ordering::Acquire), expected_entry); + } + + /// A freeze racing inserts must still produce a self-contained graph. + /// + /// `to_hnsw` snapshots the node *count* but reads adjacency live, so an + /// insert landing mid-freeze can append itself to an already-visited node's + /// neighbor list -- or promote itself to entry point. Either reference points + /// outside the snapshot; persisted, `HNSW::load` slices level 0 short and the + /// first query that walks one addresses past the level's rows. + #[test] + fn test_to_hnsw_snapshot_is_self_contained_under_concurrent_insert() { + const N: usize = 1200; + const DIM: usize = 16; + let (storage, _fsl) = build_storage(N, DIM); + let params = HnswBuildParams::default().num_edges(12).ef_construction(30); + let builder = Arc::new(OnlineHnswBuilder::new(N, params)); + + // Seed enough that a freeze has real adjacency to walk. + for id in 0..(N / 2) as u32 { + builder.insert(id, storage.as_ref()); + } + + let writing = Arc::new(AtomicBool::new(true)); + let writer = { + let builder = Arc::clone(&builder); + let storage = Arc::clone(&storage); + let writing = Arc::clone(&writing); + std::thread::spawn(move || { + for id in (N / 2) as u32..N as u32 { + builder.insert(id, storage.as_ref()); + } + writing.store(false, Ordering::Release); + }) + }; + + // Freeze for as long as the writer runs rather than a fixed count, so the + // overlap does not depend on how fast this machine inserts; the floor + // covers the writer finishing first. + let mut freezes = 0; + let mut edges_checked = 0; + while writing.load(Ordering::Acquire) || freezes < 5 { + let hnsw = builder.to_hnsw(); + let nodes = hnsw.nodes().expect("freshly built graph exposes nodes"); + let n = nodes.len() as u32; + for (id, node) in nodes.iter().enumerate() { + for (level, neighbors) in node.level_neighbors.iter().enumerate() { + for &nid in neighbors.iter() { + assert!( + nid < n, + "frozen graph has a dangling edge: node {id} level {level} \ + points at {nid}, but the snapshot holds only {n} nodes" + ); + edges_checked += 1; + } + } + } + let meta = hnsw.metadata(); + // Search starts here, so an entry point outside the snapshot finds + // nothing at all rather than merely losing one edge. + assert!( + n == 0 || meta.entry_point < n, + "frozen graph entry point {} is outside its {n} nodes", + meta.entry_point + ); + // The serialized form must agree with its own metadata, or a reader + // slices level 0 short and the dangling edge comes back. + let batch = hnsw.to_batch().unwrap(); + assert_eq!( + *meta.level_offsets.last().unwrap(), + batch.num_rows(), + "level offsets must cover exactly the serialized rows" + ); + freezes += 1; + } + writer.join().unwrap(); + assert!(edges_checked > 0, "test never inspected an edge"); + } } diff --git a/rust/lance-index/src/vector/ivf/shuffler.rs b/rust/lance-index/src/vector/ivf/shuffler.rs index 175d5ecb9fb..ca0e8f7ebe7 100644 --- a/rust/lance-index/src/vector/ivf/shuffler.rs +++ b/rust/lance-index/src/vector/ivf/shuffler.rs @@ -32,10 +32,11 @@ use lance_core::utils::futures::StreamOnDropExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::{Error, ROW_ID, Result, datatypes::Schema}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; -use lance_encoding::version::LanceFileVersion; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; use lance_file::reader::{FileReader as Lancev2FileReader, FileReaderOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions; +use lance_file::versions::v1::reader::FileReader as V1FileReader; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_file::writer::FileWriterOptions; use lance_io::ReadBatchParams; use lance_io::object_store::ObjectStore; @@ -253,9 +254,7 @@ pub async fn shuffle_dataset( let shuffler = if let Some((path, buffers)) = precomputed_shuffle_buffers { info!("Precomputed shuffle files provided, skip calculation of IVF partition."); let mut shuffler = IvfShuffler::try_new(num_partitions, Some(path), true, None)?; - unsafe { - shuffler.set_unsorted_buffers(&buffers); - } + shuffler.set_unsorted_buffers(&buffers); shuffler } else { @@ -378,9 +377,7 @@ pub async fn shuffle_vectors( Some(shuffle_output_root_filename.to_string()), )?; - unsafe { - shuffler.set_unsorted_buffers(&unsorted_filenames); - } + shuffler.set_unsorted_buffers(&unsorted_filenames); let partition_files = shuffler .write_partitioned_shuffles(shuffle_partition_batches, shuffle_partition_concurrency) @@ -406,7 +403,7 @@ pub struct IvfShuffler { shuffle_output_root_filename: String, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, } /// Represents a range of batches in a file that should be shuffled @@ -446,21 +443,17 @@ impl IvfShuffler { unsorted_buffers: vec![], is_legacy, shuffle_output_root_filename, - format_version: LanceFileVersion::V2_0, + format_version: ConcreteFileVersion::V2_0, }) } - pub fn with_format_version(mut self, format_version: LanceFileVersion) -> Self { + pub fn with_format_version(mut self, format_version: ConcreteFileVersion) -> Self { self.format_version = format_version; self } /// Set the unsorted buffers to be shuffled. - /// - /// # Safety - /// - /// user must ensure the buffers are valid. - pub unsafe fn set_unsorted_buffers(&mut self, unsorted_buffers: &[impl ToString]) { + pub fn set_unsorted_buffers(&mut self, unsorted_buffers: &[impl ToString]) { self.unsorted_buffers = unsorted_buffers.iter().map(|x| x.to_string()).collect(); } @@ -496,7 +489,7 @@ impl IvfShuffler { info!("Writing unsorted data to disk at {}", path); info!("with schema: {:?}", schema); - let mut file_writer = PreviousFileWriter::::with_object_writer( + let mut file_writer = V1FileWriter::::with_object_writer( writer, Schema::try_from(schema.as_ref())?, &Default::default(), @@ -513,9 +506,7 @@ impl IvfShuffler { file_writer.finish().await?; - unsafe { - self.set_unsorted_buffers(&[UNSORTED_BUFFER]); - } + self.set_unsorted_buffers(&[UNSORTED_BUFFER]); Ok(()) } @@ -528,7 +519,7 @@ impl IvfShuffler { if self.is_legacy { let reader = - PreviousFileReader::try_new_self_described(&object_store, &path, None).await?; + V1FileReader::try_new_self_described(&object_store, &path, None).await?; total_batches.push(reader.num_batches()); } else { let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); @@ -572,7 +563,7 @@ impl IvfShuffler { if self.is_legacy { let reader = - PreviousFileReader::try_new_self_described(&object_store, &path, None).await?; + V1FileReader::try_new_self_described(&object_store, &path, None).await?; let lance_schema = reader .schema() .project(&[PART_ID_COLUMN]) @@ -655,9 +646,8 @@ impl IvfShuffler { let mut _reader_handle = None; let mut stream = if self.is_legacy { - _reader_handle = Some( - PreviousFileReader::try_new_self_described(&object_store, &path, None).await?, - ); + _reader_handle = + Some(V1FileReader::try_new_self_described(&object_store, &path, None).await?); stream::iter(start..end) .map(|i| { @@ -806,13 +796,11 @@ impl IvfShuffler { true, )])); let lance_schema = Schema::try_from(sorted_file_schema.as_ref())?; - let mut file_writer = lance_file::writer::FileWriter::try_new( + let mut file_writer = versions::create_writer( + this.format_version, writer, lance_schema, - FileWriterOptions { - format_version: Some(this.format_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; for partition_and_idx in shuffled.into_iter().enumerate() { @@ -977,6 +965,14 @@ mod test { (stream, shuffler) } + #[tokio::test] + async fn test_missing_unsorted_buffer_returns_error() { + let mut shuffler = IvfShuffler::try_new(1, None, false, None).unwrap(); + shuffler.set_unsorted_buffers(&["missing.lance"]); + + shuffler.total_batches().await.unwrap_err(); + } + fn check_batch(batch: RecordBatch, idx: usize, num_rows: usize) { let row_ids = batch .column_by_name(ROW_ID) @@ -1089,7 +1085,7 @@ mod test { shuffler.write_unsorted_stream(stream).await.unwrap(); // set the same buffer twice we should get double the data - unsafe { shuffler.set_unsorted_buffers(&[UNSORTED_BUFFER, UNSORTED_BUFFER]) } + shuffler.set_unsorted_buffers(&[UNSORTED_BUFFER, UNSORTED_BUFFER]); let partition_files = shuffler.write_partitioned_shuffles(200, 1).await.unwrap(); @@ -1119,7 +1115,7 @@ mod test { shuffler.write_unsorted_stream(stream).await.unwrap(); // set the same buffer twice we should get double the data - unsafe { shuffler.set_unsorted_buffers(&[UNSORTED_BUFFER, UNSORTED_BUFFER]) } + shuffler.set_unsorted_buffers(&[UNSORTED_BUFFER, UNSORTED_BUFFER]); let partition_files = shuffler.write_partitioned_shuffles(1, 32).await.unwrap(); assert_eq!(partition_files.len(), 200); diff --git a/rust/lance-index/src/vector/ivf/storage.rs b/rust/lance-index/src/vector/ivf/storage.rs index 5d58401bb12..0781ebc1ea5 100644 --- a/rust/lance-index/src/vector/ivf/storage.rs +++ b/rust/lance-index/src/vector/ivf/storage.rs @@ -8,8 +8,8 @@ use itertools::Itertools; use lance_arrow::FixedSizeListArrayExt; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; -use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, +use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, writer::FileWriter as V1FileWriter, }; use lance_io::{traits::WriteExt, utils::read_message}; use lance_linalg::distance::DistanceType; @@ -146,7 +146,7 @@ impl IvfModel { start..end } - pub async fn load(reader: &PreviousFileReader) -> Result { + pub async fn load(reader: &V1FileReader) -> Result { let schema = reader.schema(); let meta_str = schema .metadata @@ -167,7 +167,7 @@ impl IvfModel { } /// Write the IVF metadata to the lance file. - pub async fn write(&self, writer: &mut PreviousFileWriter) -> Result<()> { + pub async fn write(&self, writer: &mut V1FileWriter) -> Result<()> { let pb = PbIvf::try_from(self)?; let pos = writer.object_writer.write_protobuf(&pb).await?; let ivf_metadata = IvfMetadata { pb_position: pos }; @@ -286,14 +286,10 @@ mod tests { let schema = Schema::try_from(&arrow_schema).unwrap(); { - let mut writer = PreviousFileWriter::try_new( - &object_store, - &path, - schema.clone(), - &Default::default(), - ) - .await - .unwrap(); + let mut writer = + V1FileWriter::try_new(&object_store, &path, schema.clone(), &Default::default()) + .await + .unwrap(); // Write some dummy data let batch = RecordBatch::try_new( Arc::new(arrow_schema), @@ -305,7 +301,7 @@ mod tests { writer.finish().await.unwrap(); } - let reader = PreviousFileReader::try_new_self_described(&object_store, &path, None) + let reader = V1FileReader::try_new_self_described(&object_store, &path, None) .await .unwrap(); assert!(reader.schema().metadata.contains_key(IVF_METADATA_KEY)); diff --git a/rust/lance-index/src/vector/ivf/transform.rs b/rust/lance-index/src/vector/ivf/transform.rs index b09579e46be..418f6ee96e6 100644 --- a/rust/lance-index/src/vector/ivf/transform.rs +++ b/rust/lance-index/src/vector/ivf/transform.rs @@ -176,3 +176,239 @@ impl Transformer for PartitionFilter { Ok(batch.take(&indices)?) } } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{Int32Array, cast::AsArray}; + use arrow_schema::{DataType, Field, Schema}; + use lance_arrow::FixedSizeListArrayExt; + + const VECTOR_COLUMN: &str = "v"; + + /// Two centroids far enough apart that assignment is unambiguous regardless + /// of accumulation precision. + fn centroids() -> FixedSizeListArray { + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0, 0.0, 100.0, 100.0]), 2) + .unwrap() + } + + fn transformer() -> PartitionTransformer { + PartitionTransformer::new(centroids(), DistanceType::L2, VECTOR_COLUMN) + } + + fn vector_batch(vectors: Vec>) -> RecordBatch { + let fsl = FixedSizeListArray::from_iter_primitive::( + vectors.into_iter().map(|v| Some(v.into_iter().map(Some))), + 2, + ); + let schema = Schema::new(vec![Field::new( + VECTOR_COLUMN, + fsl.data_type().clone(), + true, + )]); + RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap() + } + + fn part_ids_of(batch: &RecordBatch) -> Vec> { + batch + .column_by_name(PART_ID_COLUMN) + .unwrap() + .as_primitive::() + .iter() + .collect() + } + + fn loss_of(batch: &RecordBatch) -> f64 { + batch + .schema_ref() + .metadata() + .get(LOSS_METADATA_KEY) + .expect("loss metadata should be attached") + .parse() + .unwrap() + } + + /// A vector assigned to the wrong partition is never searched in the right + /// one, so this is the assertion the whole file exists for. + #[test] + fn test_assigns_the_nearest_centroid() { + let batch = vector_batch(vec![vec![1.0, -1.0], vec![99.0, 101.0], vec![2.0, 2.0]]); + + let output = transformer().transform(&batch).unwrap(); + + assert_eq!(part_ids_of(&output), vec![Some(0), Some(1), Some(0)]); + // The vector column survives untouched next to the new partition column. + assert_eq!( + output.column_by_name(VECTOR_COLUMN).unwrap(), + batch.column_by_name(VECTOR_COLUMN).unwrap() + ); + } + + /// Loss is the accumulated centroid distance and is read back by the + /// shuffler, so it has to be attached even when nothing is off-centroid. + #[test] + fn test_loss_is_zero_when_vectors_sit_on_centroids() { + let batch = vector_batch(vec![vec![0.0, 0.0], vec![100.0, 100.0]]); + + let output = transformer().transform(&batch).unwrap(); + + assert_eq!(loss_of(&output), 0.0); + } + + #[test] + fn test_loss_accumulates_across_rows() { + let batch = vector_batch(vec![vec![3.0, 4.0], vec![100.0, 100.0]]); + + let loss = loss_of(&transformer().transform(&batch).unwrap()); + + assert!( + loss > 0.0, + "loss should reflect the off-centroid row: {loss}" + ); + } + + #[test] + fn test_centroid_distance_is_opt_in() { + let batch = vector_batch(vec![vec![1.0, 1.0]]); + + let without = transformer().transform(&batch).unwrap(); + assert!(without.column_by_name(CENTROID_DIST_COLUMN).is_none()); + + let with = transformer().with_distance(true).transform(&batch).unwrap(); + let dists = with + .column_by_name(CENTROID_DIST_COLUMN) + .expect("distance column requested") + .as_primitive::(); + assert_eq!(dists.len(), 1); + assert!(dists.value(0) > 0.0); + } + + /// Recomputing over an already-assigned batch would waste the work and could + /// disagree with the ids the caller already wrote. + #[test] + fn test_is_noop_when_partitions_already_present() { + let assigned = transformer() + .transform(&vector_batch(vec![vec![1.0, 1.0]])) + .unwrap(); + + let output = transformer().transform(&assigned).unwrap(); + + assert_eq!(output, assigned); + } + + /// Partitions present but distances missing is the one case that still has to + /// recompute, otherwise `with_distance` silently returns a batch without the + /// column it promised. + #[test] + fn test_recomputes_when_distance_requested_but_absent() { + let assigned = transformer() + .transform(&vector_batch(vec![vec![1.0, 1.0]])) + .unwrap(); + assert!(assigned.column_by_name(CENTROID_DIST_COLUMN).is_none()); + + let output = transformer() + .with_distance(true) + .transform(&assigned) + .unwrap(); + + assert!(output.column_by_name(CENTROID_DIST_COLUMN).is_some()); + assert_eq!(part_ids_of(&output), vec![Some(0)]); + } + + #[test] + fn test_reports_missing_vector_column() { + let batch = RecordBatch::try_new( + Schema::new(vec![Field::new("other", DataType::Int32, false)]).into(), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + + let message = transformer().transform(&batch).unwrap_err().to_string(); + + assert!(message.contains(VECTOR_COLUMN), "{message}"); + assert!(message.contains("not found"), "{message}"); + } + + #[test] + fn test_reports_non_vector_column() { + let batch = RecordBatch::try_new( + Schema::new(vec![Field::new(VECTOR_COLUMN, DataType::Int32, false)]).into(), + vec![Arc::new(Int32Array::from(vec![1]))], + ) + .unwrap(); + + let message = transformer().transform(&batch).unwrap_err().to_string(); + + assert!(message.contains("is not a FixedSizeListArray"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } + + fn partitioned_batch(part_ids: Vec, tags: Vec) -> RecordBatch { + let schema = Schema::new(vec![ + Field::new(PART_ID_COLUMN, DataType::UInt32, false), + Field::new("tag", DataType::Int32, false), + ]); + RecordBatch::try_new( + schema.into(), + vec![ + Arc::new(UInt32Array::from(part_ids)), + Arc::new(Int32Array::from(tags)), + ], + ) + .unwrap() + } + + /// A sharded build only writes the partitions in its own range. Keeping a row + /// outside the range would put it in the wrong shard's file; dropping one + /// inside it loses the vector from the index entirely. + #[test] + fn test_partition_filter_keeps_only_the_requested_range() { + let batch = partitioned_batch(vec![0, 3, 1, 7, 2], vec![10, 13, 11, 17, 12]); + + let output = PartitionFilter::new(PART_ID_COLUMN, 1..3) + .transform(&batch) + .unwrap(); + + let kept: Vec = output + .column_by_name(PART_ID_COLUMN) + .unwrap() + .as_primitive::() + .values() + .to_vec(); + assert_eq!(kept, vec![1, 2]); + // The filter has to carry every other column along with it. + let tags: Vec = output + .column_by_name("tag") + .unwrap() + .as_primitive::() + .values() + .to_vec(); + assert_eq!(tags, vec![11, 12]); + } + + #[test] + fn test_partition_filter_can_keep_nothing() { + let batch = partitioned_batch(vec![0, 5], vec![1, 2]); + + let output = PartitionFilter::new(PART_ID_COLUMN, 9..10) + .transform(&batch) + .unwrap(); + + assert_eq!(output.num_rows(), 0); + assert_eq!(output.schema(), batch.schema()); + } + + #[test] + fn test_partition_filter_reports_missing_column() { + let batch = partitioned_batch(vec![0], vec![1]); + + let message = PartitionFilter::new("absent", 0..1) + .transform(&batch) + .unwrap_err() + .to_string(); + + assert!(message.contains("absent"), "{message}"); + } +} diff --git a/rust/lance-index/src/vector/kmeans.rs b/rust/lance-index/src/vector/kmeans.rs index b11fb70bed0..30016e92f8d 100644 --- a/rust/lance-index/src/vector/kmeans.rs +++ b/rust/lance-index/src/vector/kmeans.rs @@ -26,8 +26,12 @@ use arrow_array::{ArrowNumericType, UInt8Array}; use arrow_ord::sort::sort_to_indices; use arrow_schema::{ArrowError, DataType}; use bitvec::prelude::*; +use half::f16; use lance_arrow::FixedSizeListArrayExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; +use lance_linalg::distance::dot_f16::{ + PackedCentroidsF16, amx_fp16_available, amx_fp16_supported, dot_f16_batch_16, +}; use lance_linalg::distance::hamming::{hamming, hamming_distance_batch}; use lance_linalg::distance::{DistanceType, Normalize, dot_distance_batch}; use lance_linalg::kernels::{argmin_value_float, argmin_value_float_with_bias}; @@ -45,7 +49,7 @@ use { }; use crate::vector::utils::SimpleIndex; -use crate::{Error, Result}; +use lance_core::{Error, Result}; /// KMean initialization method. #[derive(Debug, PartialEq)] @@ -324,6 +328,112 @@ pub trait KMeansAlgo { ) -> KMeans; } +/// Reads a `T::Native` slice as `f16` when — and only when — that is what it is. +/// +/// The default body answers `None`, so every element type opts out until it +/// says otherwise, and [`Float16Type`] is the one that overrides it with the +/// identity. That keeps "is this f16?" a compile-time property of `T` for the +/// dot-distance kernel below, rather than a `DataType` comparison paired with a +/// transmute whose correctness the compiler cannot check. +pub(crate) trait MaybeF16: ArrowNumericType { + fn as_f16_slice(_values: &[Self::Native]) -> Option<&[f16]> { + None + } +} + +impl MaybeF16 for Float16Type { + fn as_f16_slice(values: &[f16]) -> Option<&[f16]> { + Some(values) + } +} +impl MaybeF16 for Float32Type {} +impl MaybeF16 for Float64Type {} + +/// Per-thread score-buffer budget for [`dot_membership_amx_f16`], in f32 +/// values: 256 KB, sized to stay within a typical private L2 alongside the +/// vectors and packed centroids a block streams past. +const AMX_DOT_SCRATCH_F32: usize = 64 * 1024; + +/// Assigns each row of `data` (row-major `[_, dimension]`) to its nearest +/// centroid under dot distance using the AMX-FP16 GEMM, scoring 32 vectors +/// against every centroid per tile pass instead of one vector at a time. +/// +/// `None` — the kernel is unavailable on this build or host, or the shape does +/// not suit it — means the caller must run its own per-vector path. The output +/// is otherwise identical in content and order to that path: `(centroid, +/// distance)` per row, `None` for a row whose distances are all NaN. +/// +/// Answers only "can this shape run here": the `LANCE_DISABLE_AMX` kill switch +/// is checked by the caller, so the accelerated path stays directly testable +/// while production traffic honours an operator who turned it off. +fn dot_membership_amx_f16( + centroids: &[f16], + data: &[f16], + dimension: usize, + balance_factor: f32, + cluster_sizes: Option<&[usize]>, +) -> Option>> { + let k = centroids.len() / dimension; + // Under one full 32-wide k-pass the GEMM degenerates to the kernel's scalar + // cleanup, and under one full 32-centroid block most of its work would be + // the zero padding. Neither is worth leaving the per-vector path for. + if dimension < 32 || k < 32 { + return None; + } + let packed = PackedCentroidsF16::new(centroids, k, dimension)?; + let n_padded = packed.num_centroids_padded(); + // Rows per block: as many as the scratch budget buys, rounded down to the + // kernel's 32-row granularity, and capped so a large input still splits + // into enough blocks to spread across threads. Very large `k` blows the + // budget on a single row, hence the lower clamp back to one tile pass. + let block_rows = ((AMX_DOT_SCRATCH_F32 / n_padded) & !31).clamp(32, 512); + // Precomputed once, not per row. The bias depends only on the centroid, so + // rebuilding it inside the loop would repeat `k` multiplications for every + // one of the `n` vectors -- `n * k` of them across the call, against `k` here. + let biases: Option> = cluster_sizes.map(|sizes| { + sizes + .iter() + .map(|size| balance_factor * *size as f32) + .collect() + }); + let biases = || biases.as_deref().map(|b| b.iter().copied()); + + Some( + data.par_chunks(block_rows * dimension) + .map_init( + || vec![0f32; block_rows * n_padded], + |scores, block| { + let rows = block.len() / dimension; + let tiled = rows - rows % 32; + let mut assignments = Vec::with_capacity(rows); + + packed.score(block, tiled, dimension, scores, n_padded); + for row in 0..tiled { + // Only the first `k` columns. The rest score the zero + // centroids padding `n` up to the kernel's block size, + // at distance exactly 1.0 — which beats every real + // centroid whose dot product happens to be negative. + let dots = &scores[row * n_padded..row * n_padded + k]; + assignments.push(argmin_value_float_with_bias( + dots.iter().map(|dot| 1.0 - dot), + biases(), + )); + } + // Rows past the last whole tile pass keep the per-vector path. + for vector in block[tiled * dimension..].chunks(dimension) { + assignments.push(argmin_value_float_with_bias( + dot_distance_batch(vector, centroids, dimension), + biases(), + )); + } + assignments + }, + ) + .flatten_iter() + .collect(), + ) +} + pub struct KMeansAlgoFloat where T::Native: Float + Num, @@ -331,7 +441,7 @@ where phantom_data: std::marker::PhantomData, } -impl KMeansAlgo for KMeansAlgoFloat +impl KMeansAlgo for KMeansAlgoFloat where T::Native: Float + Dot + L2 + MulAssign + DivAssign + AddAssign + FromPrimitive + Sync, PrimitiveArray: From>, @@ -368,16 +478,34 @@ where ) }) .collect::>(), - DistanceType::Dot => data - .par_chunks(dimension) - .map(|vec| { - argmin_value_float_with_bias( - dot_distance_batch(vec, centroids, dimension), - cluster_sizes - .map(|size| size.iter().map(|size| balance_factor * *size as f32)), + DistanceType::Dot => T::as_f16_slice(centroids) + .zip(T::as_f16_slice(data)) + // The kill switch is enforced here rather than inside the + // kernel wrapper: this is the one place production work is + // routed onto the GEMM, and `prefers_flat_amx_assignment` + // reads the same flag, so the two stay in lockstep. + .filter(|_| amx_fp16_available()) + .and_then(|(centroids, data)| { + dot_membership_amx_f16( + centroids, + data, + dimension, + balance_factor, + cluster_sizes, ) }) - .collect::>(), + .unwrap_or_else(|| { + data.par_chunks(dimension) + .map(|vec| { + argmin_value_float_with_bias( + dot_distance_batch(vec, centroids, dimension), + cluster_sizes.map(|size| { + size.iter().map(|size| balance_factor * *size as f32) + }), + ) + }) + .collect::>() + }), _ => { panic!( "KMeans::find_partitions: {} is not supported", @@ -1156,10 +1284,20 @@ impl KMeans { target_k ); } - debug_assert_eq!(heap.len(), target_k); + if heap.len() < target_k { + return Err(ArrowError::InvalidArgumentError(format!( + "Cannot create {target_k} IVF partitions: k-means could only form {} non-empty \ + clusters from {n} training vectors. The dataset is likely too small or has too \ + many (near-)duplicate vectors for this many partitions. Reduce num_partitions to \ + <= {} or provide more diverse data.", + heap.len(), + heap.len() + ))); + } // Construct final KMeans model with all centroids let mut all_clusters: Vec> = heap.into_vec(); + // Sort by ID to ensure consistent ordering all_clusters.sort_by_key(|c| c.id); @@ -1259,12 +1397,22 @@ pub fn kmeans_find_partitions_arrow_array( } match (centroids.value_type(), query.data_type()) { - (DataType::Float16, DataType::Float16) => Ok(kmeans_find_partitions( - centroids.values().as_primitive::().values(), - query.as_primitive::().values(), - nprobes, - distance_type, - )?), + (DataType::Float16, DataType::Float16) => { + let centroids = centroids.values().as_primitive::().values(); + let query = query.as_primitive::().values(); + if distance_type == DistanceType::Dot + && amx_fp16_available() + && let Some(dists) = dot_f16_partitions_amx(centroids, query) + { + return smallest_nprobes(dists, nprobes); + } + Ok(kmeans_find_partitions( + centroids, + query, + nprobes, + distance_type, + )?) + } (DataType::Float32, DataType::Float32) => Ok(kmeans_find_partitions( centroids.values().as_primitive::().values(), query.as_primitive::().values(), @@ -1294,6 +1442,69 @@ pub fn kmeans_find_partitions_arrow_array( /// KMeans finds N nearest partitions. /// /// Parameters: +/// The `nprobes` smallest distances and the partitions they belong to. +fn smallest_nprobes( + dists: Vec, + nprobes: usize, +) -> arrow::error::Result<(UInt32Array, Float32Array)> { + // TODO: use heap to just keep nprobes smallest values. + let dists_arr = Float32Array::from(dists); + let indices = sort_to_indices(&dists_arr, None, Some(nprobes))?; + let dists = arrow::compute::take(&dists_arr, &indices, None)? + .as_primitive::() + .clone(); + Ok((indices, dists)) +} + +/// `Dot` distances from `query` to every centroid, through the AMX-FP16 kernel, +/// or `None` when this build/CPU/shape cannot use it. +/// +/// Partition selection is one query against every centroid, so on paper it needs +/// well under 1% of this machine's arithmetic. It measured at 33% of a saturated +/// IVF_HNSW_SQ query because `dot_f16_avx512` carries no vector instruction at +/// all under GCC 13.4 -- disassembly shows 30 `vcvtsh2ss` / 15 `vmulss` / +/// 14 `vaddss` and zero `zmm` operands, since GCC has no packed `_Float16` -> +/// `float` widening pattern. The scalar loop, not the work, is the cost. +/// +/// Sixteen centroids at a time rather than the `M x N` GEMM: the GEMM steps its +/// centroid loop by 32 and would spend 31 of every 32 output columns on padding +/// for a single query (16 MAC/cycle), while this shape wastes 15 of 16 and +/// reaches 32 MAC/cycle. Those rates count tile work only; each call also pays +/// one LDTILECFG plus one TILERELEASE, which at these shapes is the larger term. +/// Beating either needs several queries scored together, which the per-query +/// search API does not offer. +fn dot_f16_partitions_amx(centroids: &[f16], query: &[f16]) -> Option> { + let dim = query.len(); + // Below one full 32-wide k-pass the kernel is all scalar cleanup, so a dim + // that short would run at a loss. Support, not the `LANCE_DISABLE_AMX` kill + // switch: the caller has already decided to use AMX, and this only declines + // shapes the kernel cannot pay for. + if dim < 32 || !amx_fp16_supported() { + return None; + } + debug_assert_eq!(centroids.len() % dim, 0); + + let mut dists = vec![0f32; centroids.len() / dim]; + let row = |i: usize| ¢roids[i * dim..(i + 1) * dim]; + for (g, out) in dists.chunks_mut(16).enumerate() { + let base = g * 16; + // `dot_f16_batch_16` requires 16 slices of the query's length even when + // only `len` of them are scored, so the tail repeats a valid row; those + // lanes are computed and discarded. + let mut group: [&[f16]; 16] = [row(base); 16]; + for (i, slot) in group.iter_mut().enumerate().take(out.len()) { + *slot = row(base + i); + } + // The kernel returns raw dot products; `Dot` distance is `1 - dot`, the + // same convention `dot_distance_batch` applies. + let dots = dot_f16_batch_16(query, &group, out.len()); + for (d, dot) in out.iter_mut().zip(dots.iter()) { + *d = 1.0 - *dot; + } + } + Some(dists) +} + /// - *centroids*: a `k * dimension` floating array. /// - *query*: a `dimension` floating array. /// - *nprobes*: the number of partitions to find. @@ -1319,13 +1530,7 @@ pub fn kmeans_find_partitions( } }; - // TODO: use heap to just keep nprobes smallest values. - let dists_arr = Float32Array::from(dists); - let indices = sort_to_indices(&dists_arr, None, Some(nprobes))?; - let dists = arrow::compute::take(&dists_arr, &indices, None)? - .as_primitive::() - .clone(); - Ok((indices, dists)) + smallest_nprobes(dists, nprobes) } pub fn kmeans_find_partitions_binary( @@ -1550,9 +1755,47 @@ mod tests { use lance_testing::datagen::generate_random_array; use super::*; + use lance_linalg::distance::dot_f16::amx_fp16_supported; use lance_linalg::distance::l2; use lance_linalg::kernels::argmin; + /// The AMX partition path must pick the same partitions as the scalar one. + /// Exact equality on the distances is not required -- the kernel accumulates + /// in a different order -- but the chosen partition ids must match, since a + /// different choice silently changes which vectors a query can ever see. + #[test] + fn test_amx_find_partitions_matches_scalar() { + if !amx_fp16_supported() { + return; + } + // (dim, k): a production shape, one with a partial 16-group tail, and one + // whose dimension is not a multiple of the kernel's 32-wide k-pass. + for (dim, k) in [(768usize, 10_000usize), (768, 37), (133, 100)] { + let mut st = 0x9E37u64; + let mut next = || { + st = st.wrapping_mul(6364136223846793005).wrapping_add(1); + f16::from_f32(((st >> 33) as f32 / (1u64 << 31) as f32) - 0.5) + }; + let centroids: Vec = (0..k * dim).map(|_| next()).collect(); + let query: Vec = (0..dim).map(|_| next()).collect(); + + let amx = dot_f16_partitions_amx(¢roids, &query) + .expect("the AMX path declined a shape it should accept"); + let scalar: Vec = dot_distance_batch(&query[..], ¢roids[..], dim).collect(); + assert_eq!(amx.len(), scalar.len(), "dim={dim} k={k}"); + + for nprobes in [1usize, 8, 32] { + let (amx_idx, _) = smallest_nprobes(amx.clone(), nprobes).unwrap(); + let (scalar_idx, _) = smallest_nprobes(scalar.clone(), nprobes).unwrap(); + assert_eq!( + amx_idx.values(), + scalar_idx.values(), + "dim={dim} k={k} nprobes={nprobes} picked different partitions" + ); + } + } + } + #[test] fn test_train_with_small_dataset() { let data = Float32Array::from(vec![1.0, 2.0, 3.0, 4.0]); @@ -1722,6 +1965,411 @@ mod tests { } } + #[tokio::test] + async fn test_hierarchical_kmeans_too_few_distinct_vectors_errors() { + // Regression test for https://github.com/lance-format/lance/issues/7867 + // + // With a small number of distinct vectors repeated many times (heavy + // near-duplication) and dot distance, hierarchical k-means cannot form + // `target_k` non-empty clusters no matter how it splits: every split of a + // cluster of identical vectors is either ineffective (`all_same`) or + // immediately hits the "<= 1 point" floor. This used to trip + // `debug_assert_eq!(heap.len(), target_k)` (panic in debug builds) or + // silently return a half-empty centroid set (release builds). It should + // now return a clear error instead. + const DIM: usize = 8; + const NUM_DISTINCT: usize = 5; + const REPEATS: usize = 200; + const TARGET_K: usize = 300; // > 256 to trigger hierarchical clustering + + let base_vectors = generate_random_array(NUM_DISTINCT * DIM); + let mut values = Vec::with_capacity(NUM_DISTINCT * REPEATS * DIM); + for _ in 0..REPEATS { + values.extend_from_slice(base_vectors.values()); + } + let values = Float32Array::from(values); + let fsl = FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap(); + + let params = KMeansParams { + max_iters: 10, + hierarchical_k: 16, + distance_type: DistanceType::Dot, + ..Default::default() + }; + + let err = KMeans::new_with_params(&fsl, TARGET_K, ¶ms) + .expect_err("training should fail rather than panic or silently under-produce"); + let msg = err.to_string(); + assert!( + msg.contains("Cannot create") && msg.contains(&TARGET_K.to_string()), + "unexpected error message: {msg}" + ); + } + + // ----------------------------------------------------------------------- + // AMX-FP16 dot-distance assignment + // ----------------------------------------------------------------------- + + /// Relative tolerance between the AMX and per-vector distances. Both + /// accumulate f32-widened products and differ only in summation order, so + /// this is far looser than what they actually differ by (~1e-4) and far + /// tighter than fp16's own representational error. + const AMX_REL_TOL: f32 = 5e-3; + + /// A vector this much nearer its best centroid than its runner-up cannot + /// change hands on summation order alone. Closer ties are allowed to + /// disagree — that is fp16 arithmetic, not a bug. + const AMX_TIE_GAP: f32 = 1e-2; + + fn random_f16(count: usize, rng: &mut SmallRng) -> Vec { + (0..count) + .map(|_| f16::from_f32(rng.random_range(-1.0f32..1.0))) + .collect() + } + + /// Assert the AMX dot path engages for this input and assigns every vector + /// where the per-vector path does. + fn assert_dot_paths_agree( + centroids: &[f16], + data: &[f16], + dimension: usize, + balance_factor: f32, + cluster_sizes: Option<&[usize]>, + ctx: &str, + ) { + let k = centroids.len() / dimension; + // The AMX path's own output, not `compute_membership_and_dist`'s: that + // entry point falls back to the per-vector path whenever this one + // declines, so going through it would silently degrade this into a + // scalar-against-scalar comparison on any host or build that lacks the + // kernel, and prove nothing about it. + let amx = dot_membership_amx_f16(centroids, data, dimension, balance_factor, cluster_sizes) + .unwrap_or_else(|| { + panic!("{ctx}: the AMX path declined this shape, so agreeing proves nothing") + }); + + for (i, vector) in data.chunks(dimension).enumerate() { + let row = dot_distance_batch(vector, centroids, dimension).collect::>(); + let want = argmin_value_float_with_bias( + row.iter().copied(), + cluster_sizes.map(|sizes| sizes.iter().map(|size| balance_factor * *size as f32)), + ); + let got = amx[i]; + let (Some((want_id, _)), Some((got_id, got_dist))) = (want, got) else { + assert_eq!( + want.is_none(), + got.is_none(), + "{ctx}: row {i} is assigned by one path only: {want:?} vs {got:?}" + ); + continue; + }; + + assert!( + (got_id as usize) < k, + "{ctx}: row {i} landed on centroid {got_id}, outside the {k} real ones" + ); + // Check the reported distance against the reported centroid's own + // rather than against the winner's: on a near-tie the paths may + // pick different centroids, and then only this identity has to hold. + let want_dist = row[got_id as usize]; + assert!( + (got_dist - want_dist).abs() <= AMX_REL_TOL * want_dist.abs() + 1e-3, + "{ctx}: row {i} centroid {got_id} distance {got_dist}, want {want_dist}" + ); + + let mut biased = row + .iter() + .enumerate() + .map(|(j, dist)| { + dist + cluster_sizes.map_or(0.0, |sizes| balance_factor * sizes[j] as f32) + }) + .collect::>(); + biased.sort_by(f32::total_cmp); + if biased[1] - biased[0] > AMX_TIE_GAP { + assert_eq!( + got_id, want_id, + "{ctx}: row {i} is not a tie ({} vs {}) but the paths disagree", + biased[0], biased[1] + ); + } + } + } + + /// The two paths across the shapes that exercise each boundary: `k` on and + /// off the kernel's 32-centroid block (so with and without zero padding), + /// `dim` with and without the kernel's scalar tail, and row counts on and + /// off the 32-row tile pass (so with and without trailing fallback rows). + #[test] + fn test_dot_amx_matches_per_vector_path() { + if !amx_fp16_supported() { + return; + } + let mut rng = SmallRng::seed_from_u64(0xD07); + for k in [32usize, 64, 100] { + for dimension in [32usize, 64, 768] { + for n in [64usize, 100, 1000] { + let centroids = random_f16(k * dimension, &mut rng); + let data = random_f16(n * dimension, &mut rng); + assert_dot_paths_agree( + ¢roids, + &data, + dimension, + 0.0, + None, + &format!("k={k} dim={dimension} n={n}"), + ); + } + } + } + } + + /// The padding columns must be unreachable by the argmin. + /// + /// `k` is not a multiple of 32, so the GEMM's `n` block is filled out with + /// zero centroids, which score a dot product of 0 — distance exactly 1.0. + /// Here every real dot product is negative, so every real distance exceeds + /// 1.0 and a reduction over the padded row width would hand *every* vector + /// a cluster id past the end of the centroid set. + #[test] + fn test_dot_amx_padding_columns_never_win() { + if !amx_fp16_supported() { + return; + } + const K: usize = 100; + const DIM: usize = 64; + const N: usize = 128; + + let mut rng = SmallRng::seed_from_u64(0xBAD5); + let negate = |v: &f16| f16::from_f32(-v.to_f32().abs() - 0.1); + let centroids = random_f16(K * DIM, &mut rng) + .iter() + .map(negate) + .collect::>(); + let data = random_f16(N * DIM, &mut rng) + .iter() + .map(|v| f16::from_f32(v.to_f32().abs() + 0.1)) + .collect::>(); + + for vector in data.chunks(DIM) { + assert!( + dot_distance_batch(vector, ¢roids, DIM).all(|dist| dist > 1.0), + "premise broken: a real centroid is nearer than the zero padding" + ); + } + assert_dot_paths_agree(¢roids, &data, DIM, 0.0, None, "padding"); + } + + /// The bias path. `argmin_value_float_with_bias` minimizes `distance + + /// bias` but reports the unbiased distance, so both halves of that have to + /// survive the AMX path; the balance factor is sized to actually move + /// assignments, which the test asserts rather than assumes. + #[test] + fn test_dot_amx_with_balance_bias() { + if !amx_fp16_supported() { + return; + } + const K: usize = 64; + const DIM: usize = 128; + const N: usize = 256; + const BALANCE_FACTOR: f32 = 0.02; + + let mut rng = SmallRng::seed_from_u64(0xB1A5); + let centroids = random_f16(K * DIM, &mut rng); + let data = random_f16(N * DIM, &mut rng); + let cluster_sizes = (0..K).map(|id| id * 4).collect::>(); + + assert_dot_paths_agree( + ¢roids, + &data, + DIM, + BALANCE_FACTOR, + Some(&cluster_sizes), + "bias", + ); + + let assign = |balance_factor, sizes| { + KMeansAlgoFloat::::compute_membership_and_dist( + ¢roids, + &data, + DIM, + DistanceType::Dot, + balance_factor, + sizes, + None, + ) + .0 + }; + assert_ne!( + assign(BALANCE_FACTOR, Some(cluster_sizes.as_slice())), + assign(0.0, None), + "the balance factor is too small to move any assignment" + ); + } + + /// A row of NaNs has no nearest centroid — `distance + bias < min` is false + /// for every centroid — and the AMX path has to reach the same `None` as + /// the per-vector one instead of defaulting to cluster 0. Covered in both + /// the tiled rows and the trailing rows that fall back per vector. + #[test] + fn test_dot_amx_all_nan_row_is_unassigned() { + if !amx_fp16_supported() { + return; + } + const K: usize = 64; + const DIM: usize = 64; + const N: usize = 100; // 3 full tile passes, then 4 fallback rows + const NAN_ROWS: [usize; 2] = [7, 98]; + + let mut rng = SmallRng::seed_from_u64(0x4A4); + let centroids = random_f16(K * DIM, &mut rng); + let mut data = random_f16(N * DIM, &mut rng); + for row in NAN_ROWS { + data[row * DIM..(row + 1) * DIM].fill(f16::NAN); + } + + assert_dot_paths_agree(¢roids, &data, DIM, 0.0, None, "nan"); + + let (membership, _) = KMeansAlgoFloat::::compute_membership_and_dist( + ¢roids, + &data, + DIM, + DistanceType::Dot, + 0.0, + None, + None, + ); + for (row, cluster_id) in membership.iter().enumerate() { + assert_eq!( + cluster_id.is_none(), + NAN_ROWS.contains(&row), + "row {row} membership {cluster_id:?}" + ); + } + } + + /// Wall-clock throughput of the dot-distance assignment the AMX path above + /// accelerates, swept over `(threads, dim, k)`. + /// + /// The path is picked inside `compute_membership_and_dist` from run-time + /// capability and the data's shape, so there is nothing to toggle per + /// iteration: run this same binary twice — once as-is for the AMX path, once + /// with `LANCE_DISABLE_AMX=1` for the per-vector path — and divide. The + /// header line reports which path the process took, so the two outputs + /// cannot be confused. + /// + /// Each point runs for a wall-clock budget rather than a fixed pass count, so + /// a 1-thread point and an all-core point take comparable time and every + /// point averages over enough work to be stable. + /// + /// `#[ignore]` -- run: + /// cargo test -p lance-index --release \ + /// kmeans_dot_f16_membership_bench -- --ignored --nocapture + /// Tune with `BENCH_N`, `BENCH_DIMS` / `BENCH_KS` / `BENCH_THREADS` + /// (comma-separated; threads default `,32,1`) and `BENCH_SECONDS` (the + /// wall-clock budget each measured point gets). + #[test] + #[ignore] + #[allow(clippy::print_stderr)] + fn kmeans_dot_f16_membership_bench() { + use std::time::{Duration, Instant}; + + let env_usize = |key: &str, default: usize| -> usize { + std::env::var(key) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(default) + }; + let env_list = |key: &str, default: &[usize]| -> Vec { + std::env::var(key) + .ok() + .map(|s| s.split(',').filter_map(|t| t.trim().parse().ok()).collect()) + .unwrap_or_else(|| default.to_vec()) + }; + + let n = env_usize("BENCH_N", 65_536); + let dims = env_list("BENCH_DIMS", &[128, 768, 1536]); + let ks = env_list("BENCH_KS", &[32, 64, 128, 256, 1024, 4096]); + let ncpu = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(8); + let thread_counts = env_list("BENCH_THREADS", &[ncpu, 32, 1]); + let budget = Duration::from_secs_f64( + std::env::var("BENCH_SECONDS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(3.0), + ); + + eprintln!( + "[kmeans_dot_f16_bench] n={n} ncpu={ncpu} budget={:.1}s amx_fp16_available={}", + budget.as_secs_f64(), + amx_fp16_available(), + ); + + let mut rng = SmallRng::seed_from_u64(0x9E37); + for &dimension in &dims { + // Random data *and* random centroids: with degenerate inputs every + // vector would reduce to the same centroid and the argmin's branches + // and the score buffer's access pattern would both be unrealistic. + let data = random_f16(n * dimension, &mut rng); + for &k in &ks { + if k >= n { + eprintln!( + "[kmeans_dot_f16_bench] dim={dimension} k={k}: skipped, k must be < n={n}" + ); + continue; + } + let centroids = random_f16(k * dimension, &mut rng); + for &nthreads in &thread_counts { + if nthreads == 0 || nthreads > ncpu { + eprintln!( + "[kmeans_dot_f16_bench] dim={dimension} k={k} threads={nthreads}: skipped, not in 1..={ncpu}" + ); + continue; + } + // A private pool so the sweep sets the width exactly, without + // reconfiguring (or being limited by) the global one. + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(nthreads) + .build() + .unwrap(); + let run_pass = || { + pool.install(|| { + KMeansAlgoFloat::::compute_membership_and_dist( + ¢roids, + &data, + dimension, + DistanceType::Dot, + 0.0, + None, + None, + ) + }) + }; + let warm = run_pass(); // page-in and thread spin-up, untimed + std::hint::black_box(&warm); + drop(warm); + + let t0 = Instant::now(); + let mut passes = 0usize; + while t0.elapsed() < budget { + let assigned = run_pass(); + std::hint::black_box(&assigned); + passes += 1; + } + let elapsed = t0.elapsed().as_secs_f64(); + let vectors = passes * n; + let vec_per_s = vectors as f64 / elapsed; + eprintln!( + "[kmeans_dot_f16_bench] dim={dimension:>5} k={k:>5} threads={nthreads:>4} passes={passes:>6} vec_per_s={vec_per_s:>12.0} us_per_vec={:>9.4} Gpair_per_s={:>8.2}", + 1e6 / vec_per_s, + vec_per_s * k as f64 / 1e9, + ); + } + } + } + } + #[tokio::test] async fn test_float16_underflow_fix() { // This test verifies the fix for float16 division underflow diff --git a/rust/lance-index/src/vector/pq.rs b/rust/lance-index/src/vector/pq.rs index 5749e56ed31..4f6ba450f7e 100644 --- a/rust/lance-index/src/vector/pq.rs +++ b/rust/lance-index/src/vector/pq.rs @@ -690,6 +690,96 @@ mod tests { }); } + #[test] + fn test_distance_with_legacy_truncated_dimension() { + const DIM: usize = 64; + const NUM_SUB_VECTORS: usize = 14; + const NUM_BITS: u32 = 8; + const NUM_CENTROIDS: usize = 1 << NUM_BITS; + const SUB_VECTOR_DIM: usize = DIM / NUM_SUB_VECTORS; + const PERSISTED_DIM: usize = NUM_SUB_VECTORS * SUB_VECTOR_DIM; + + // Older writers silently omitted the tail when the dimension was not + // divisible by the number of sub-vectors. Preserve searches over those + // indexes even though current writers reject this configuration. + let indexed_vector = (1..=DIM).map(|value| value as f32).collect::>(); + let mut codebook = Vec::with_capacity(NUM_SUB_VECTORS * NUM_CENTROIDS * SUB_VECTOR_DIM); + for sub_vector in indexed_vector[..PERSISTED_DIM].chunks_exact(SUB_VECTOR_DIM) { + for _ in 0..NUM_CENTROIDS { + codebook.extend_from_slice(sub_vector); + } + } + let query = indexed_vector + .iter() + .enumerate() + .map(|(idx, value)| value + if idx < PERSISTED_DIM { 1.0 } else { 1_000.0 }) + .collect::>(); + let code = UInt8Array::from(vec![0; NUM_SUB_VECTORS]); + + let prepared_l2 = ProductQuantizer::new( + NUM_SUB_VECTORS, + NUM_BITS, + DIM, + FixedSizeListArray::try_new_from_values( + Float32Array::from(codebook.clone()), + DIM as i32, + ) + .unwrap(), + DistanceType::L2, + ); + assert!(prepared_l2.l2_targets.is_some()); + let distances = prepared_l2 + .compute_distances(&Float32Array::from(query.clone()), &code) + .unwrap(); + assert_relative_eq!(distances.value(0), PERSISTED_DIM as f32, epsilon = 1e-4); + + let generic_l2 = ProductQuantizer::new( + NUM_SUB_VECTORS, + NUM_BITS, + DIM, + FixedSizeListArray::try_new_from_values( + PrimitiveArray::::from( + codebook + .iter() + .map(|value| *value as f64) + .collect::>(), + ), + DIM as i32, + ) + .unwrap(), + DistanceType::L2, + ); + assert!(generic_l2.l2_targets.is_none()); + let distances = generic_l2 + .compute_distances( + &PrimitiveArray::::from( + query.iter().map(|value| *value as f64).collect::>(), + ), + &code, + ) + .unwrap(); + assert_relative_eq!(distances.value(0), PERSISTED_DIM as f32, epsilon = 1e-4); + + let dot = ProductQuantizer::new( + NUM_SUB_VECTORS, + NUM_BITS, + DIM, + FixedSizeListArray::try_new_from_values(Float32Array::from(codebook), DIM as i32) + .unwrap(), + DistanceType::Dot, + ); + let expected_dot_distance = 1.0 + - indexed_vector[..PERSISTED_DIM] + .iter() + .zip(&query[..PERSISTED_DIM]) + .map(|(left, right)| left * right) + .sum::(); + let distances = dot + .compute_distances(&Float32Array::from(query), &code) + .unwrap(); + assert_relative_eq!(distances.value(0), expected_dot_distance, epsilon = 1e-4); + } + #[test] fn test_pq_transform() { const DIM: usize = 16; diff --git a/rust/lance-index/src/vector/pq/builder.rs b/rust/lance-index/src/vector/pq/builder.rs index c4dad4a6a3e..c267e7550e8 100644 --- a/rust/lance-index/src/vector/pq/builder.rs +++ b/rust/lance-index/src/vector/pq/builder.rs @@ -68,7 +68,12 @@ impl Default for PQBuildParams { impl QuantizerBuildParams for PQBuildParams { fn sample_size(&self) -> usize { - self.sample_rate * 2_usize.pow(self.num_bits as u32) + self.training_sample_size() + .expect("PQ training sample size must fit in usize") + } + + fn try_sample_size(&self) -> Result { + self.training_sample_size() } fn use_residual(distance_type: DistanceType) -> bool { @@ -94,6 +99,29 @@ impl PQBuildParams { } } + fn num_centroids(&self) -> Result { + u32::try_from(self.num_bits) + .ok() + .and_then(|num_bits| 1_usize.checked_shl(num_bits)) + .ok_or_else(|| { + Error::invalid_input(format!( + "PQ centroid count overflows: num_bits={}, usize_bits={}", + self.num_bits, + usize::BITS + )) + }) + } + + fn training_sample_size(&self) -> Result { + let num_centroids = self.num_centroids()?; + self.sample_rate.checked_mul(num_centroids).ok_or_else(|| { + Error::invalid_input(format!( + "PQ training sample size overflows: sample_rate={}, num_centroids={num_centroids}", + self.sample_rate + )) + }) + } + fn build_from_fsl( &self, data: &FixedSizeListArray, @@ -109,8 +137,10 @@ impl PQBuildParams { "PQ code does not support cosine" ); - let sub_vectors = divide_to_subvectors::(data, self.num_sub_vectors)?; - let num_centroids = 2_usize.pow(self.num_bits as u32); + let num_centroids = self.num_centroids()?; + let max_training_rows = self.try_sample_size()?; + let training_rows = data.len().min(max_training_rows); + let sub_vectors = divide_to_subvectors::(data, self.num_sub_vectors, training_rows)?; let dimension = data.value_length() as usize; let sub_vector_dimension = dimension / self.num_sub_vectors; @@ -174,7 +204,7 @@ impl PQBuildParams { data.data_type() )))?; - let num_centroids = 2_usize.pow(self.num_bits as u32); + let num_centroids = self.num_centroids()?; if data.len() < num_centroids { return Err(Error::unprocessable(format!( "Not enough rows to train PQ. Requires {num_centroids} rows but only {} available", @@ -194,3 +224,117 @@ impl PQBuildParams { } } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::Float32Array; + + #[test] + fn test_build_samples_before_materializing_subvectors() { + const N: usize = 4096; + const DIM: usize = 8; + const NUM_SUB_VECTORS: usize = 2; + const NUM_BITS: usize = 2; + const K: usize = 1 << NUM_BITS; + const SUB_DIM: usize = DIM / NUM_SUB_VECTORS; + + // The 256 * K sample cap is smaller than N. Initial centroids make + // training deterministic so the optimized and reference paths can be + // compared exactly. + let values = Float32Array::from_iter((0..N).flat_map(|row| { + let cluster = row % K; + (0..DIM).map(move |col| (cluster * 1000 + col) as f32 + row as f32 * 1e-4) + })); + let fsl = FixedSizeListArray::try_new_from_values(values.clone(), DIM as i32).unwrap(); + + let init_values: Vec = (0..NUM_SUB_VECTORS * K) + .flat_map(|i| (0..SUB_DIM).map(move |col| (i % K * 1000 + col) as f32)) + .collect(); + let init_codebook: ArrayRef = Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from(init_values.clone()), + DIM as i32, + ) + .unwrap(), + ); + + let pq = PQBuildParams::with_codebook(NUM_SUB_VECTORS, NUM_BITS, init_codebook) + .build(&fsl, DistanceType::L2) + .unwrap(); + + let mut expected = Vec::with_capacity(K * DIM); + for sub_idx in 0..NUM_SUB_VECTORS { + let mut sub_values = Vec::with_capacity(N * SUB_DIM); + for row in values.values().chunks(DIM) { + sub_values.extend_from_slice(&row[sub_idx * SUB_DIM..(sub_idx + 1) * SUB_DIM]); + } + let sub_init = FixedSizeListArray::try_new_from_values( + Float32Array::from( + init_values[sub_idx * K * SUB_DIM..(sub_idx + 1) * K * SUB_DIM].to_vec(), + ), + SUB_DIM as i32, + ) + .unwrap(); + let params = KMeansParams::new(Some(Arc::new(sub_init)), 50, 1, DistanceType::L2); + let kmeans = train_kmeans::( + &Float32Array::from(sub_values), + params, + SUB_DIM, + K, + 256, + ) + .unwrap(); + expected.extend_from_slice(kmeans.centroids.as_primitive::().values()); + } + + assert_eq!( + pq.codebook.values().as_primitive::().values(), + expected.as_slice() + ); + } + + #[test] + fn test_try_sample_size_rejects_overflow() { + let mut params = PQBuildParams::new(2, 2); + params.sample_rate = usize::MAX; + + let error = params.try_sample_size().unwrap_err(); + let expected = format!( + "PQ training sample size overflows: sample_rate={}, num_centroids=4", + usize::MAX + ); + assert!(matches!(&error, Error::InvalidInput { .. }), "{error}"); + assert!(error.to_string().contains(&expected), "{error}"); + } + + #[test] + fn test_try_sample_size_rejects_centroid_count_overflow() { + let params = PQBuildParams::new(2, usize::BITS as usize); + + let error = params.try_sample_size().unwrap_err(); + let expected = format!( + "PQ centroid count overflows: num_bits={}, usize_bits={}", + usize::BITS, + usize::BITS + ); + assert!(matches!(&error, Error::InvalidInput { .. }), "{error}"); + assert!(error.to_string().contains(&expected), "{error}"); + } + + #[test] + fn test_build_rejects_sample_size_overflow() { + let values = Float32Array::from_iter((0..4 * 8).map(|v| v as f32)); + let fsl = FixedSizeListArray::try_new_from_values(values, 8).unwrap(); + let mut params = PQBuildParams::new(2, 2); + params.sample_rate = usize::MAX; + + let error = params.build(&fsl, DistanceType::L2).unwrap_err(); + let expected = format!( + "PQ training sample size overflows: sample_rate={}, num_centroids=4", + usize::MAX + ); + assert!(matches!(&error, Error::InvalidInput { .. }), "{error}"); + assert!(error.to_string().contains(&expected), "{error}"); + } +} diff --git a/rust/lance-index/src/vector/pq/distance.rs b/rust/lance-index/src/vector/pq/distance.rs index b341ba98af7..a2798c51036 100644 --- a/rust/lance-index/src/vector/pq/distance.rs +++ b/rust/lance-index/src/vector/pq/distance.rs @@ -42,7 +42,13 @@ pub fn build_distance_table_l2_impl( let sub_vector_length = dimension / num_sub_vectors; let num_centroids = 2_usize.pow(NUM_BITS); let mut result = Vec::with_capacity(num_sub_vectors * num_centroids); - for (i, sub_vec) in query.chunks_exact(sub_vector_length).enumerate() { + // Legacy writers allowed non-divisible dimensions and truncated the tail. + // Limit iteration to the sub-vectors that were persisted by those writers. + for (i, sub_vec) in query + .chunks_exact(sub_vector_length) + .take(num_sub_vectors) + .enumerate() + { let subvec_centroids = get_sub_vector_centroids::(codebook, dimension, num_sub_vectors, i); result.extend(l2_distance_batch( @@ -63,8 +69,14 @@ pub fn build_distance_table_l2_prepared(l2_targets: &[L2Prepared], query: &[f32] let num_targets = l2_targets[0].num_targets(); let mut result = vec![0.0f32; l2_targets.len() * num_targets]; - for (i, sub_vec) in query.chunks_exact(sub_dim).enumerate() { - l2_targets[i].distances_into(sub_vec, &mut result[i * num_targets..][..num_targets]); + // The target count also bounds legacy codebooks whose writers truncated + // a non-divisible vector tail. + for (i, (target, sub_vec)) in l2_targets + .iter() + .zip(query.chunks_exact(sub_dim)) + .enumerate() + { + target.distances_into(sub_vec, &mut result[i * num_targets..][..num_targets]); } result } @@ -94,7 +106,13 @@ pub fn build_distance_table_dot_impl( let sub_vector_length = dimension / num_sub_vectors; let num_centroids = 2_usize.pow(NUM_BITS); let mut result = Vec::with_capacity(num_sub_vectors * num_centroids); - for (i, sub_vec) in query.chunks_exact(sub_vector_length).enumerate() { + // Legacy writers allowed non-divisible dimensions and truncated the tail. + // Limit iteration to the sub-vectors that were persisted by those writers. + for (i, sub_vec) in query + .chunks_exact(sub_vector_length) + .take(num_sub_vectors) + .enumerate() + { let subvec_centroids = get_sub_vector_centroids::(codebook, dimension, num_sub_vectors, i); result.extend(dot_distance_batch( @@ -377,4 +395,81 @@ mod tests { ); assert_eq!(distances, expected); } + + #[test] + fn test_compute_4bit_bulk_distance_preserves_flat_prefix_middle_and_tail() { + const NUM_VECTORS: usize = 227; + const NUM_SUB_VECTORS: usize = 4; + const NUM_PACKED_CODES: usize = NUM_SUB_VECTORS / 2; + const NUM_CENTROIDS: usize = 16; + + let distance_table = (0..NUM_SUB_VECTORS * NUM_CENTROIDS) + .map(|value| (value * value + 1) as f32) + .collect::>(); + let packed_codes = (0..NUM_VECTORS * NUM_PACKED_CODES) + .map(|value| { + let low = (value % NUM_CENTROIDS) as u8; + let high = ((value * 7 + 3) % NUM_CENTROIDS) as u8; + low | (high << 4) + }) + .collect::>(); + let packed_codes = UInt8Array::from(packed_codes); + let transposed = transpose(&packed_codes, NUM_VECTORS, NUM_PACKED_CODES); + + let actual = + compute_pq_distance(&distance_table, 4, NUM_SUB_VECTORS, transposed.values(), 10); + let expected = packed_codes + .values() + .chunks_exact(NUM_PACKED_CODES) + .map(|codes| { + codes + .iter() + .enumerate() + .map(|(byte_idx, code)| { + distance_table[byte_idx * 2 * NUM_CENTROIDS + (code & 0x0f) as usize] + + distance_table + [(byte_idx * 2 + 1) * NUM_CENTROIDS + (code >> 4) as usize] + }) + .sum::() + }) + .collect::>(); + + assert_eq!(actual.len(), NUM_VECTORS); + assert_eq!(&actual[..FLAT_NUM_4BIT_PQ], &expected[..FLAT_NUM_4BIT_PQ]); + let tail_start = NUM_VECTORS - NUM_VECTORS % NUM_CENTROIDS; + assert_eq!(&actual[tail_start..], &expected[tail_start..]); + + let qmax = expected[..FLAT_NUM_4BIT_PQ] + .iter() + .copied() + .max_by(f32::total_cmp) + .unwrap(); + let (qmin, quantized_table) = quantize_distance_table(&distance_table, qmax); + let range = (qmax - qmin) / 255.0; + for (vector_idx, actual_distance) in actual + .iter() + .enumerate() + .take(tail_start) + .skip(FLAT_NUM_4BIT_PQ) + { + let codes = &packed_codes.values() + [vector_idx * NUM_PACKED_CODES..(vector_idx + 1) * NUM_PACKED_CODES]; + let quantized_sum = codes + .iter() + .enumerate() + .fold(0_u8, |sum, (byte_idx, code)| { + sum.saturating_add( + quantized_table[byte_idx * 2 * NUM_CENTROIDS + (code & 0x0f) as usize], + ) + .saturating_add( + quantized_table[(byte_idx * 2 + 1) * NUM_CENTROIDS + (code >> 4) as usize], + ) + }); + let reference = quantized_sum as f32 * range + qmin; + assert!( + (*actual_distance - reference).abs() <= f32::EPSILON, + "4-bit bulk distance mismatch at vector {vector_idx}: actual={actual_distance}, reference={reference}" + ); + } + } } diff --git a/rust/lance-index/src/vector/pq/storage.rs b/rust/lance-index/src/vector/pq/storage.rs index 4e98df9906d..a5ee3496571 100644 --- a/rust/lance-index/src/vector/pq/storage.rs +++ b/rust/lance-index/src/vector/pq/storage.rs @@ -24,8 +24,8 @@ use bytes::{Bytes, BytesMut}; use lance_arrow::{FixedSizeListArrayExt, RecordBatchExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, +use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, writer::FileWriter as V1FileWriter, }; use lance_io::{object_store::ObjectStore, utils::read_message}; use lance_linalg::distance::{Cosine, DistanceType, Dot, L2}; @@ -37,7 +37,8 @@ use serde::{Deserialize, Serialize}; use super::ProductQuantizer; use super::distance::{build_distance_table_dot, build_distance_table_l2, compute_pq_distance}; -use crate::frag_reuse::FragReuseIndex; +use crate::frag_reuse::{FragReuseIndex, FragReuseIndexHandle}; +use crate::scalar::RowIdRemapper; use crate::vector::graph::{OrderedFloat, OrderedNode}; use crate::{ INDEX_METADATA_SCHEMA_KEY, IndexMetadata, pb, @@ -126,7 +127,7 @@ impl QuantizerMetadata for ProductQuantizationMetadata { } } - async fn load(reader: &PreviousFileReader) -> Result { + async fn load(reader: &V1FileReader) -> Result { let metadata = reader .schema() .metadata @@ -196,13 +197,38 @@ impl ProductQuantizationStorage { #[allow(clippy::too_many_arguments)] pub fn new( codebook: FixedSizeListArray, - mut batch: RecordBatch, + batch: RecordBatch, num_bits: u32, num_sub_vectors: usize, dimension: usize, distance_type: DistanceType, transposed: bool, frag_reuse_index: Option>, + ) -> Result { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::new_with_remapper( + codebook, + batch, + num_bits, + num_sub_vectors, + dimension, + distance_type, + transposed, + frag_reuse_index, + ) + } + + #[allow(clippy::too_many_arguments)] + fn new_with_remapper( + codebook: FixedSizeListArray, + mut batch: RecordBatch, + num_bits: u32, + num_sub_vectors: usize, + dimension: usize, + distance_type: DistanceType, + transposed: bool, + frag_reuse_index: Option>, ) -> Result { if batch.num_columns() != 2 { log::warn!( @@ -386,7 +412,7 @@ impl ProductQuantizationStorage { path: &Path, frag_reuse_index: Option>, ) -> Result { - let reader = PreviousFileReader::try_new_self_described(object_store, path, None).await?; + let reader = V1FileReader::try_new_self_described(object_store, path, None).await?; let schema = reader.schema(); let metadata_str = schema @@ -469,7 +495,7 @@ impl ProductQuantizationStorage { /// pub async fn write_partition( &self, - writer: &mut PreviousFileWriter, + writer: &mut V1FileWriter, ) -> Result { let batch_size: usize = 10240; // TODO: make it configurable for offset in (0..self.batch.num_rows()).step_by(batch_size) { @@ -544,6 +570,36 @@ impl QuantizerStorage for ProductQuantizationStorage { ) } + fn try_from_batch_with_remapper( + batch: RecordBatch, + metadata: &Self::Metadata, + distance_type: DistanceType, + frag_reuse_index: Option>, + ) -> Result { + let distance_type = match distance_type { + DistanceType::Cosine => DistanceType::L2, + _ => distance_type, + }; + let codebook = match &metadata.codebook { + Some(codebook) => codebook.clone(), + None => { + debug_assert!(!metadata.codebook_tensor.is_empty()); + let codebook_tensor = pb::Tensor::decode(metadata.codebook_tensor.as_slice())?; + FixedSizeListArray::try_from(&codebook_tensor)? + } + }; + Self::new_with_remapper( + codebook, + batch, + metadata.nbits, + metadata.num_sub_vectors, + metadata.dimension, + distance_type, + metadata.transposed, + frag_reuse_index, + ) + } + fn metadata(&self) -> &Self::Metadata { &self.metadata } @@ -613,9 +669,9 @@ impl QuantizerStorage for ProductQuantizationStorage { /// /// Parameters /// ---------- - /// - *reader: &PreviousFileReader + /// - *reader: &V1FileReader async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, diff --git a/rust/lance-index/src/vector/pq/utils.rs b/rust/lance-index/src/vector/pq/utils.rs index d2a9f8e8620..4203dd1f0e3 100644 --- a/rust/lance-index/src/vector/pq/utils.rs +++ b/rust/lance-index/src/vector/pq/utils.rs @@ -6,17 +6,25 @@ use arrow_array::{ }; use lance_core::{Error, Result, assume}; -/// Divide a 2D vector in [`T::Array`] to `m` sub-vectors. +/// Divide the first `num_rows` of a 2D vector in [`T::Array`] to `m` +/// sub-vectors. /// /// For example, for a `[1024x1M]` matrix, when `n = 8`, this function divides /// the matrix into `[128x1M; 8]` vector of matrix. pub(super) fn divide_to_subvectors( fsl: &FixedSizeListArray, m: usize, + num_rows: usize, ) -> Result>> where PrimitiveArray: From>, { + if num_rows > fsl.len() { + return Err(Error::invalid_input(format!( + "cannot divide {num_rows} rows from an array with {} rows", + fsl.len() + ))); + } let dim = fsl.value_length() as usize; if !dim.is_multiple_of(m) { return Err(Error::invalid_input(format!( @@ -26,15 +34,14 @@ where }; let sub_vector_length = dim / m; - let capacity = fsl.len() * sub_vector_length; + let capacity = num_rows * sub_vector_length; let mut subarrays = vec![Vec::with_capacity(capacity); m]; - // TODO: very intensive memory copy involved!!! But this is on the write path. - // Optimize for memory copy later. fsl.values() .as_primitive::() .values() .chunks(dim) + .take(num_rows) .for_each(|vec| { for i in 0..m { subarrays[i] @@ -76,21 +83,40 @@ mod tests { use super::*; use arrow_array::{FixedSizeListArray, Float32Array, types::Float32Type}; use lance_arrow::FixedSizeListArrayExt; + use rstest::rstest; - #[test] - fn test_divide_to_subvectors() { + #[rstest] + #[case::all_rows(10)] + #[case::sampled_rows(3)] + fn test_divide_to_subvectors(#[case] num_rows: usize) { let values = Float32Array::from_iter((0..320).map(|v| v as f32)); // A [10, 32] array. let mat = FixedSizeListArray::try_new_from_values(values, 32).unwrap(); - let sub_vectors = divide_to_subvectors::(&mat, 4).unwrap(); + let sub_vectors = divide_to_subvectors::(&mat, 4, num_rows).unwrap(); assert_eq!(sub_vectors.len(), 4); - assert_eq!(sub_vectors[0].len(), 10 * 8); + assert_eq!(sub_vectors[0].len(), num_rows * 8); assert_eq!( sub_vectors[0].values().to_vec(), - (0..10) + (0..num_rows) .flat_map(|i| (0..8).map(move |c| 32.0 * i as f32 + c as f32)) .collect::>() ); } + + #[test] + fn test_divide_to_subvectors_rejects_too_many_rows() { + let values = Float32Array::from_iter((0..320).map(|v| v as f32)); + let mat = FixedSizeListArray::try_new_from_values(values, 32).unwrap(); + + let error = divide_to_subvectors::(&mat, 4, 11).unwrap_err(); + + assert!(matches!(&error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("cannot divide 11 rows from an array with 10 rows"), + "unexpected error: {error}" + ); + } } diff --git a/rust/lance-index/src/vector/quantizer.rs b/rust/lance-index/src/vector/quantizer.rs index 6f4b191098b..433fa5031d4 100644 --- a/rust/lance-index/src/vector/quantizer.rs +++ b/rust/lance-index/src/vector/quantizer.rs @@ -15,7 +15,7 @@ use bytes::Bytes; use lance_arrow::RecordBatchExt; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::traits::Reader; use lance_linalg::distance::DistanceType; use lance_table::format::SelfDescribingFileReader; @@ -25,6 +25,7 @@ use super::flat::index::{FlatBinQuantizer, FlatQuantizer}; use super::pq::ProductQuantizer; use super::{ivf::storage::IvfModel, sq::ScalarQuantizer, storage::VectorStore}; use crate::frag_reuse::FragReuseIndex; +use crate::scalar::RowIdRemapper; use crate::vector::bq::builder::RabitQuantizer; use crate::{INDEX_METADATA_SCHEMA_KEY, IndexMetadata}; @@ -81,7 +82,9 @@ impl FromStr for QuantizationType { "FLATBIN" => Ok(Self::FlatBin), "PQ" => Ok(Self::Product), "SQ" => Ok(Self::Scalar), - "RABIT" => Ok(Self::Rabit), + // `Display` writes "RQ"; "RABIT" is accepted for headers written + // before this variant round-tripped. + "RQ" | "RABIT" => Ok(Self::Rabit), _ => Err(Error::index(format!("Unknown quantization type: {}", s))), } } @@ -100,7 +103,29 @@ impl std::fmt::Display for QuantizationType { } pub trait QuantizerBuildParams: Send + Sync { + /// Returns the number of rows to sample when training the quantizer. fn sample_size(&self) -> usize; + + /// Returns the number of rows to sample, rejecting parameters whose sample size + /// cannot be represented by [`usize`]. + /// + /// Implementations with fallible sample-size calculations should override this + /// method. The default preserves the behavior of existing implementations. + /// + /// # Examples + /// + /// ``` + /// use lance_index::vector::pq::PQBuildParams; + /// use lance_index::vector::quantizer::QuantizerBuildParams; + /// + /// let params = PQBuildParams::new(16, 8); + /// assert_eq!(params.try_sample_size()?, 65_536); + /// # Ok::<(), lance_core::Error>(()) + /// ``` + fn try_sample_size(&self) -> Result { + Ok(self.sample_size()) + } + fn use_residual(_: DistanceType) -> bool { false } @@ -223,7 +248,7 @@ pub trait QuantizerMetadata: Ok(None) } - async fn load(reader: &PreviousFileReader) -> Result; + async fn load(reader: &V1FileReader) -> Result; } #[async_trait::async_trait] @@ -239,6 +264,23 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore { frag_reuse_index: Option>, ) -> Result; + /// Internal entry point for compact FRI loading without changing the + /// existing concrete-type API. + #[doc(hidden)] + fn try_from_batch_with_remapper( + batch: RecordBatch, + metadata: &Self::Metadata, + distance_type: DistanceType, + frag_reuse_index: Option>, + ) -> Result { + if frag_reuse_index.is_some() { + return Err(Error::not_supported( + "this quantization storage does not support a generic row-id remapper".to_string(), + )); + } + Self::try_from_batch(batch, metadata, distance_type, None) + } + fn metadata(&self) -> &Self::Metadata; fn remap(&self, mapping: &RowAddrRemap) -> Result { @@ -277,7 +319,7 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore { } async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, @@ -287,7 +329,7 @@ pub trait QuantizerStorage: Clone + Sized + DeepSizeOf + VectorStore { /// Loader to load partitioned [VectorStore] from disk. pub struct IvfQuantizationStorage { - reader: PreviousFileReader, + reader: V1FileReader, distance_type: DistanceType, quantizer: Quantizer, @@ -322,7 +364,7 @@ impl IvfQuantizationStorage { /// /// pub async fn open(reader: Arc) -> Result { - let reader = PreviousFileReader::try_new_self_described_from_reader(reader, None).await?; + let reader = V1FileReader::try_new_self_described_from_reader(reader, None).await?; let schema = reader.schema(); let metadata_str = schema @@ -385,3 +427,34 @@ impl IvfQuantizationStorage { .await } } + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + /// `IvfIndexState` persists the quantization type with `Display` and reads + /// it back with `FromStr`, so the two must agree for every variant. + #[rstest] + #[case::flat(QuantizationType::Flat)] + #[case::flat_bin(QuantizationType::FlatBin)] + #[case::product(QuantizationType::Product)] + #[case::scalar(QuantizationType::Scalar)] + #[case::rabit(QuantizationType::Rabit)] + fn test_display_from_str_round_trip(#[case] quantization_type: QuantizationType) { + let encoded = quantization_type.to_string(); + assert_eq!( + encoded.parse::().unwrap(), + quantization_type, + "{encoded} did not round-trip" + ); + } + + #[test] + fn test_from_str_accepts_legacy_rabit_spelling() { + assert_eq!( + "RABIT".parse::().unwrap(), + QuantizationType::Rabit + ); + } +} diff --git a/rust/lance-index/src/vector/residual.rs b/rust/lance-index/src/vector/residual.rs index 6ba908ba9d1..5774d42f246 100644 --- a/rust/lance-index/src/vector/residual.rs +++ b/rust/lance-index/src/vector/residual.rs @@ -5,7 +5,7 @@ use std::ops::{AddAssign, DivAssign}; use std::sync::Arc; use std::{iter, ops::MulAssign}; -use crate::vector::kmeans::{KMeansAlgoFloat, compute_partitions}; +use crate::vector::kmeans::{KMeansAlgoFloat, MaybeF16, compute_partitions}; use arrow_array::ArrowNumericType; use arrow_array::{ Array, FixedSizeListArray, PrimitiveArray, RecordBatch, UInt32Array, @@ -53,7 +53,7 @@ impl ResidualTransform { } } -fn do_compute_residual( +fn do_compute_residual( centroids: &FixedSizeListArray, vectors: &FixedSizeListArray, distance_type: Option, @@ -190,3 +190,262 @@ impl Transformer for ResidualTransform { Ok(batch) } } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{ArrayRef, Float16Array, Float32Array, Float64Array, Int8Array, Int32Array}; + use arrow_schema::{Field, Schema}; + use half::f16; + use lance_arrow::FixedSizeListArrayExt; + + const PART_COLUMN: &str = "part_id"; + const VECTOR_COLUMN: &str = "v"; + + fn fsl(values: T, dim: i32) -> FixedSizeListArray { + FixedSizeListArray::try_new_from_values(values, dim).unwrap() + } + + fn f32_values(arr: &FixedSizeListArray) -> Vec { + arr.values().as_primitive::().values().to_vec() + } + + /// A batch holding a vector column plus the partition ids the transform reads. + fn batch(vectors: ArrayRef, part_ids: Vec) -> RecordBatch { + let schema = Schema::new(vec![ + Field::new(VECTOR_COLUMN, vectors.data_type().clone(), true), + Field::new(PART_COLUMN, DataType::UInt32, false), + ]); + RecordBatch::try_new( + schema.into(), + vec![vectors, Arc::new(UInt32Array::from(part_ids))], + ) + .unwrap() + } + + fn transform_of(centroids: FixedSizeListArray) -> ResidualTransform { + ResidualTransform::new(centroids, PART_COLUMN, VECTOR_COLUMN) + } + + /// The whole point of the file: each vector loses the centroid of *its own* + /// partition, not the first one. Two partitions with distinct centroids and + /// interleaved partition ids are what make a row/centroid mix-up visible. + #[test] + fn test_compute_residual_subtracts_the_assigned_centroid() { + let centroids = fsl(Float32Array::from(vec![0.0, 0.0, 10.0, 20.0]), 2); + let vectors = fsl(Float32Array::from(vec![1.0, 2.0, 11.0, 23.0, 3.0, 4.0]), 2); + let part_ids = UInt32Array::from(vec![0, 1, 0]); + + let residual = compute_residual(¢roids, &vectors, None, Some(&part_ids)).unwrap(); + + assert_eq!(residual.value_length(), 2); + assert_eq!(f32_values(&residual), vec![1.0, 2.0, 1.0, 3.0, 3.0, 4.0]); + } + + /// Mismatched widths would slice the centroid row out of bounds, so this is + /// rejected up front. The message has to name both widths to be actionable. + #[test] + fn test_compute_residual_rejects_dimension_mismatch() { + let centroids = fsl(Float32Array::from(vec![0.0, 0.0]), 2); + let vectors = fsl(Float32Array::from(vec![1.0, 2.0, 3.0]), 3); + + let message = compute_residual(¢roids, &vectors, None, None) + .unwrap_err() + .to_string(); + + assert!(message.contains("centroid: 2"), "{message}"); + assert!(message.contains("vector: 3"), "{message}"); + } + + /// Only the four pairs listed in `compute_residual` are dispatched; anything + /// else must report both value types rather than silently picking one. + #[test] + fn test_compute_residual_rejects_type_mismatch() { + let centroids = fsl(Float32Array::from(vec![0.0, 0.0]), 2); + let vectors = fsl(Float64Array::from(vec![1.0, 2.0]), 2); + + let message = compute_residual(¢roids, &vectors, None, None) + .unwrap_err() + .to_string(); + + assert!(message.contains("Float32"), "{message}"); + assert!(message.contains("Float64"), "{message}"); + } + + #[test] + fn test_compute_residual_float16() { + let centroids = fsl( + Float16Array::from(vec![f16::from_f32(1.0), f16::from_f32(2.0)]), + 2, + ); + let vectors = fsl( + Float16Array::from(vec![f16::from_f32(4.0), f16::from_f32(8.0)]), + 2, + ); + let part_ids = UInt32Array::from(vec![0]); + + let residual = compute_residual(¢roids, &vectors, None, Some(&part_ids)).unwrap(); + + let values = residual.values().as_primitive::().values(); + assert_eq!(values, &[f16::from_f32(3.0), f16::from_f32(6.0)]); + } + + #[test] + fn test_compute_residual_float64() { + let centroids = fsl(Float64Array::from(vec![1.0, 2.0]), 2); + let vectors = fsl(Float64Array::from(vec![4.0, 8.0]), 2); + let part_ids = UInt32Array::from(vec![0]); + + let residual = compute_residual(¢roids, &vectors, None, Some(&part_ids)).unwrap(); + + let values = residual.values().as_primitive::().values(); + assert_eq!(values, &[3.0, 6.0]); + } + + /// Int8 vectors are the one asymmetric arm: they are widened to Float32 + /// before subtraction, so the residual comes back wider than the input. + #[test] + fn test_compute_residual_widens_int8_vectors_to_float32() { + let centroids = fsl(Float32Array::from(vec![0.5, 1.5]), 2); + let vectors = fsl(Int8Array::from(vec![4i8, 9]), 2); + let part_ids = UInt32Array::from(vec![0]); + + let residual = compute_residual(¢roids, &vectors, None, Some(&part_ids)).unwrap(); + + assert_eq!(residual.value_type(), DataType::Float32); + assert_eq!(f32_values(&residual), vec![3.5, 7.5]); + } + + /// With no partition ids the transform falls back to assigning them from the + /// distance type. Centroids are far apart so the assignment is unambiguous + /// regardless of accumulation precision. + #[test] + fn test_compute_residual_assigns_partitions_when_absent() { + let centroids = fsl(Float32Array::from(vec![0.0, 0.0, 100.0, 100.0]), 2); + let vectors = fsl(Float32Array::from(vec![99.0, 101.0, 1.0, -2.0]), 2); + + let residual = + compute_residual(¢roids, &vectors, Some(DistanceType::L2), None).unwrap(); + + assert_eq!(f32_values(&residual), vec![-1.0, 1.0, 1.0, -2.0]); + } + + /// An already-quantized batch has nothing to subtract, and recomputing would + /// corrupt the codes, so the batch must pass through untouched. + #[test] + fn test_transform_is_noop_when_pq_code_present() { + let vectors = fsl(Float32Array::from(vec![1.0, 2.0]), 2); + let schema = Schema::new(vec![ + Field::new(VECTOR_COLUMN, vectors.data_type().clone(), true), + Field::new(PART_COLUMN, DataType::UInt32, false), + Field::new(PQ_CODE_COLUMN, DataType::Int32, false), + ]); + let input = RecordBatch::try_new( + schema.into(), + vec![ + Arc::new(vectors), + Arc::new(UInt32Array::from(vec![0])), + Arc::new(Int32Array::from(vec![7])), + ], + ) + .unwrap(); + + let centroids = fsl(Float32Array::from(vec![9.0, 9.0]), 2); + let output = transform_of(centroids).transform(&input).unwrap(); + + assert_eq!(output, input); + } + + #[test] + fn test_transform_reports_missing_partition_column() { + let vectors = fsl(Float32Array::from(vec![1.0, 2.0]), 2); + let schema = Schema::new(vec![Field::new( + VECTOR_COLUMN, + vectors.data_type().clone(), + true, + )]); + let input = RecordBatch::try_new(schema.into(), vec![Arc::new(vectors)]).unwrap(); + + let centroids = fsl(Float32Array::from(vec![0.0, 0.0]), 2); + let message = transform_of(centroids) + .transform(&input) + .unwrap_err() + .to_string(); + + assert!(message.contains(PART_COLUMN), "{message}"); + } + + #[test] + fn test_transform_reports_missing_vector_column() { + let input = RecordBatch::try_new( + Schema::new(vec![Field::new(PART_COLUMN, DataType::UInt32, false)]).into(), + vec![Arc::new(UInt32Array::from(vec![0]))], + ) + .unwrap(); + + let centroids = fsl(Float32Array::from(vec![0.0, 0.0]), 2); + let message = transform_of(centroids) + .transform(&input) + .unwrap_err() + .to_string(); + + assert!(message.contains(VECTOR_COLUMN), "{message}"); + } + + #[test] + fn test_transform_reports_non_vector_column() { + let input = batch(Arc::new(Int32Array::from(vec![1, 2])), vec![0, 0]); + + let centroids = fsl(Float32Array::from(vec![0.0, 0.0]), 2); + let message = transform_of(centroids) + .transform(&input) + .unwrap_err() + .to_string(); + + assert!(message.contains("is not fixed size list"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } + + /// Same-width residuals replace the column in place: the name and schema stay + /// put and only the values change, so downstream projections keep working. + #[test] + fn test_transform_replaces_vector_column_in_place() { + let vectors = fsl(Float32Array::from(vec![1.0, 2.0, 11.0, 22.0]), 2); + let input = batch(Arc::new(vectors), vec![0, 1]); + let centroids = fsl(Float32Array::from(vec![1.0, 1.0, 10.0, 20.0]), 2); + + let output = transform_of(centroids).transform(&input).unwrap(); + + assert_eq!(output.schema(), input.schema()); + let residual = output + .column_by_name(VECTOR_COLUMN) + .unwrap() + .as_fixed_size_list(); + assert_eq!(f32_values(residual), vec![0.0, 1.0, 1.0, 2.0]); + } + + /// Int8 input widens to Float32, so replacing the column alone would leave the + /// schema claiming Int8 while the data is Float32. This is the branch that has + /// to rewrite the field type as well. + #[test] + fn test_transform_rewrites_schema_when_residual_widens() { + let input = batch(Arc::new(fsl(Int8Array::from(vec![4i8, 9]), 2)), vec![0]); + let centroids = fsl(Float32Array::from(vec![0.5, 1.5]), 2); + + let output = transform_of(centroids).transform(&input).unwrap(); + + let field = output + .schema() + .field_with_name(VECTOR_COLUMN) + .unwrap() + .clone(); + let residual = output + .column_by_name(VECTOR_COLUMN) + .unwrap() + .as_fixed_size_list(); + assert_eq!(field.data_type(), residual.data_type()); + assert_eq!(residual.value_type(), DataType::Float32); + assert_eq!(f32_values(residual), vec![3.5, 7.5]); + } +} diff --git a/rust/lance-index/src/vector/sq/storage.rs b/rust/lance-index/src/vector/sq/storage.rs index 785aadc289d..6062c22242e 100644 --- a/rust/lance-index/src/vector/sq/storage.rs +++ b/rust/lance-index/src/vector/sq/storage.rs @@ -15,9 +15,9 @@ use async_trait::async_trait; use lance_arrow::ArrowFloatType; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, ROW_ID, Result}; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::object_store::ObjectStore; -use lance_linalg::distance::{DistanceType, dot_u8::dot_u8, l2_u8::l2_u8}; +use lance_linalg::distance::{DistanceType, dot_u8::dot_u8_u64, l2_u8::l2_u8_u64}; use lance_table::format::SelfDescribingFileReader; use num_traits::AsPrimitive; use object_store::path::Path; @@ -25,7 +25,8 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use super::{ScalarQuantizer, scale_to_u8}; -use crate::frag_reuse::FragReuseIndex; +use crate::frag_reuse::{FragReuseIndex, FragReuseIndexHandle}; +use crate::scalar::RowIdRemapper; use crate::{ INDEX_METADATA_SCHEMA_KEY, IndexMetadata, vector::{ @@ -53,7 +54,7 @@ impl DeepSizeOf for ScalarQuantizationMetadata { #[async_trait] impl QuantizerMetadata for ScalarQuantizationMetadata { - async fn load(reader: &PreviousFileReader) -> Result { + async fn load(reader: &V1FileReader) -> Result { let metadata_str = reader .schema() .metadata @@ -174,6 +175,18 @@ impl ScalarQuantizationStorage { bounds: Range, batches: impl IntoIterator, frag_reuse_index: Option>, + ) -> Result { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::try_new_with_remapper(num_bits, distance_type, bounds, batches, frag_reuse_index) + } + + fn try_new_with_remapper( + num_bits: u16, + distance_type: DistanceType, + bounds: Range, + batches: impl IntoIterator, + frag_reuse_index: Option>, ) -> Result { let mut chunks = Vec::with_capacity(SQ_CHUNK_CAPACITY); let mut offsets = Vec::with_capacity(SQ_CHUNK_CAPACITY + 1); @@ -215,7 +228,7 @@ impl ScalarQuantizationStorage { path: &Path, frag_reuse_index: Option>, ) -> Result { - let reader = PreviousFileReader::try_new_self_described(object_store, path, None).await?; + let reader = V1FileReader::try_new_self_described(object_store, path, None).await?; let schema = reader.schema(); let metadata_str = schema @@ -279,6 +292,21 @@ impl QuantizerStorage for ScalarQuantizationStorage { ) } + fn try_from_batch_with_remapper( + batch: RecordBatch, + metadata: &Self::Metadata, + distance_type: DistanceType, + frag_reuse_index: Option>, + ) -> Result { + Self::try_new_with_remapper( + metadata.num_bits, + distance_type, + metadata.bounds.clone(), + [batch], + frag_reuse_index, + ) + } + fn metadata(&self) -> &Self::Metadata { &self.quantizer.metadata } @@ -292,7 +320,7 @@ impl QuantizerStorage for ScalarQuantizationStorage { /// - *metric_type: metric type of the vectors /// - *metadata: scalar quantization metadata async fn load_partition( - reader: &PreviousFileReader, + reader: &V1FileReader, range: std::ops::Range, distance_type: DistanceType, metadata: &Self::Metadata, @@ -617,7 +645,7 @@ impl<'a> SQDistCalculator<'a> { sum: query_code_sum, } => { let dim = sq_code.len() as f32; - let code_dot = dot_u8(sq_code, query_sq_code) as f32; + let code_dot = dot_u8_u64(sq_code, query_sq_code) as f32; let code_sum = sq_code_sum(sq_code); dim * self.lower_bound * self.lower_bound + self.lower_bound * self.value_scale * (code_sum + *query_code_sum) @@ -635,7 +663,7 @@ impl DistCalculator for SQDistCalculator<'_> { let query_sq_code = self.query_sq_code.as_slice(); match self.storage.distance_type { DistanceType::L2 | DistanceType::Cosine => { - l2_u8(sq_code, query_sq_code) as f32 * self.scale + l2_u8_u64(sq_code, query_sq_code) as f32 * self.scale } DistanceType::Dot => self.dot_distance(sq_code), _ => panic!("We should not reach here: sq distance can only be L2 or Dot"), @@ -653,7 +681,7 @@ impl DistCalculator for SQDistCalculator<'_> { c.sq_codes .values() .chunks_exact(c.dim()) - .map(|sq_codes| l2_u8(sq_codes, query_sq_code) as f32) + .map(|sq_codes| l2_u8_u64(sq_codes, query_sq_code) as f32) }) .map(|dist| dist * self.scale) .collect(), diff --git a/rust/lance-index/src/vector/sq/transform.rs b/rust/lance-index/src/vector/sq/transform.rs index 3a81734347b..78ca2942bf6 100644 --- a/rust/lance-index/src/vector/sq/transform.rs +++ b/rust/lance-index/src/vector/sq/transform.rs @@ -56,16 +56,21 @@ impl Transformer for SQTransformer { "SQ Transform: column {} not found in batch", self.input_column )))?; - let fsl = input - .as_fixed_size_list_opt() - .ok_or(Error::index("input column is not vector type".to_string()))?; + let fsl = input.as_fixed_size_list_opt().ok_or_else(|| { + Error::index(format!( + "SQ Transform: column {} is not a fixed size list vector: {}", + self.input_column, + input.data_type() + )) + })?; let sq_code = match fsl.value_type() { DataType::Float16 => self.quantizer.transform::(input)?, DataType::Float32 => self.quantizer.transform::(input)?, DataType::Float64 => self.quantizer.transform::(input)?, _ => { return Err(Error::index(format!( - "unsupported data type: {}", + "SQ Transform: column {} has unsupported value type: {}", + self.input_column, fsl.value_type() ))); } @@ -78,3 +83,87 @@ impl Transformer for SQTransformer { Ok(batch) } } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{Array, FixedSizeListArray, Float32Array, Int32Array}; + use arrow_schema::Schema; + use lance_arrow::FixedSizeListArrayExt; + + const SQ_COLUMN: &str = "sq"; + const VECTOR_COLUMN: &str = "v"; + + fn transformer(dim: usize) -> SQTransformer { + let mut quantizer = ScalarQuantizer::new(8, dim); + quantizer.metadata.bounds = 0.0..1.0; + SQTransformer::new(quantizer, VECTOR_COLUMN.into(), SQ_COLUMN.into()) + } + + fn vector_batch(values: Float32Array, dim: i32) -> RecordBatch { + let fsl = FixedSizeListArray::try_new_from_values(values, dim).unwrap(); + let schema = Schema::new(vec![Field::new( + VECTOR_COLUMN, + fsl.data_type().clone(), + true, + )]); + RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap() + } + + #[test] + fn test_sq_transform_replaces_vector_with_code_column() { + let batch = vector_batch(Float32Array::from(vec![0.0, 0.5, 1.0, 0.25]), 2); + let output = transformer(2).transform(&batch).unwrap(); + + assert!( + output.column_by_name(VECTOR_COLUMN).is_none(), + "input column should be dropped" + ); + let codes = output + .column_by_name(SQ_COLUMN) + .unwrap() + .as_fixed_size_list(); + assert_eq!(codes.len(), 2); + assert_eq!(codes.value_length(), 2); + } + + #[test] + fn test_sq_transform_reports_missing_column() { + let batch = vector_batch(Float32Array::from(vec![0.0, 1.0]), 2); + let transformer = SQTransformer::new( + ScalarQuantizer::new(8, 2), + "absent".into(), + SQ_COLUMN.into(), + ); + let message = transformer.transform(&batch).unwrap_err().to_string(); + assert!(message.contains("column absent"), "{message}"); + } + + #[test] + fn test_sq_transform_reports_non_vector_column() { + let batch = RecordBatch::try_new( + Schema::new(vec![Field::new(VECTOR_COLUMN, DataType::Int32, false)]).into(), + vec![Arc::new(Int32Array::from(vec![1, 2]))], + ) + .unwrap(); + let message = transformer(2).transform(&batch).unwrap_err().to_string(); + assert!(message.contains("column v"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } + + #[test] + fn test_sq_transform_reports_unsupported_value_type() { + let values = Int32Array::from(vec![1, 2, 3, 4]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + let schema = Schema::new(vec![Field::new( + VECTOR_COLUMN, + fsl.data_type().clone(), + true, + )]); + let batch = RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap(); + let message = transformer(2).transform(&batch).unwrap_err().to_string(); + assert!(message.contains("column v"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } +} diff --git a/rust/lance-index/src/vector/storage.rs b/rust/lance-index/src/vector/storage.rs index a14308197ed..151be945741 100644 --- a/rust/lance-index/src/vector/storage.rs +++ b/rust/lance-index/src/vector/storage.rs @@ -28,7 +28,8 @@ use std::{ use crossbeam_queue::ArrayQueue; -use crate::frag_reuse::FragReuseIndex; +use crate::frag_reuse::{FragReuseIndex, FragReuseIndexHandle}; +use crate::scalar::RowIdRemapper; use crate::{ pb, vector::{ @@ -448,7 +449,7 @@ pub struct StorageBuilder { distance_type: DistanceType, quantizer: Q, - frag_reuse_index: Option>, + frag_reuse_index: Option>, } impl StorageBuilder { @@ -457,6 +458,18 @@ impl StorageBuilder { distance_type: DistanceType, quantizer: Q, frag_reuse_index: Option>, + ) -> Result { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::new_with_remapper(vector_column, distance_type, quantizer, frag_reuse_index) + } + + #[doc(hidden)] + pub fn new_with_remapper( + vector_column: String, + distance_type: DistanceType, + quantizer: Q, + frag_reuse_index: Option>, ) -> Result { Ok(Self { vector_column, @@ -486,7 +499,7 @@ impl StorageBuilder { debug_assert!(batch.column_by_name(ROW_ID).is_some()); debug_assert!(batch.column_by_name(self.quantizer.column()).is_some()); - Q::Storage::try_from_batch( + Q::Storage::try_from_batch_with_remapper( batch, &self.quantizer.metadata(None), self.distance_type, @@ -504,7 +517,7 @@ pub struct IvfQuantizationStorage { metadata: Q::Metadata, ivf: IvfModel, - frag_reuse_index: Option>, + frag_reuse_index: Option>, } impl DeepSizeOf for IvfQuantizationStorage { @@ -520,6 +533,16 @@ impl IvfQuantizationStorage { pub async fn try_new( reader: FileReader, frag_reuse_index: Option>, + ) -> Result { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::try_new_with_remapper(reader, frag_reuse_index).await + } + + #[doc(hidden)] + pub async fn try_new_with_remapper( + reader: FileReader, + frag_reuse_index: Option>, ) -> Result { let schema = reader.schema(); @@ -577,6 +600,19 @@ impl IvfQuantizationStorage { metadata: Q::Metadata, distance_type: DistanceType, frag_reuse_index: Option>, + ) -> Self { + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(FragReuseIndexHandle(index)) as Arc); + Self::from_cached_with_remapper(reader, ivf, metadata, distance_type, frag_reuse_index) + } + + #[doc(hidden)] + pub fn from_cached_with_remapper( + reader: FileReader, + ivf: IvfModel, + metadata: Q::Metadata, + distance_type: DistanceType, + frag_reuse_index: Option>, ) -> Self { Self { reader, @@ -660,7 +696,7 @@ impl IvfQuantizationStorage { let schema = Arc::new(self.reader.schema().as_ref().into()); concat_batches(&schema, batches.iter())? }; - Q::Storage::try_from_batch( + Q::Storage::try_from_batch_with_remapper( batch, self.metadata(), self.distance_type, diff --git a/rust/lance-index/src/vector/transform.rs b/rust/lance-index/src/vector/transform.rs index 01372a14048..2faae7b9ce6 100644 --- a/rust/lance-index/src/vector/transform.rs +++ b/rust/lance-index/src/vector/transform.rs @@ -229,6 +229,7 @@ mod tests { use super::*; use approx::assert_relative_eq; + use arrow::buffer::NullBuffer; use arrow_array::{FixedSizeListArray, Float16Array, Float32Array, Int32Array}; use arrow_schema::Schema; use half::f16; @@ -345,6 +346,92 @@ mod tests { assert!(dup_drop_result.is_ok()); } + /// Builds a 2-dim FSL column with one row per `rows` entry. `None` means a + /// null vector; otherwise the two f32 values are used as-is. + fn fsl_batch(rows: &[Option<[f32; 2]>]) -> RecordBatch { + let values = Float32Array::from( + rows.iter() + .flat_map(|row| row.unwrap_or([0.0, 0.0])) + .collect::>(), + ); + let nulls = NullBuffer::from(rows.iter().map(Option::is_some).collect::>()); + let item = Arc::new(Field::new("item", DataType::Float32, true)); + let fsl = + FixedSizeListArray::try_new(item.clone(), 2, Arc::new(values), Some(nulls)).unwrap(); + let schema = Schema::new(vec![Field::new( + "v", + DataType::FixedSizeList(item, 2), + true, + )]); + RecordBatch::try_new(schema.into(), vec![Arc::new(fsl)]).unwrap() + } + + #[test] + fn test_keep_finite_vectors_drops_null_and_non_finite_rows() { + let batch = fsl_batch(&[ + Some([1.0, 2.0]), + None, + Some([f32::NAN, 1.0]), + Some([f32::INFINITY, 1.0]), + Some([f32::NEG_INFINITY, 1.0]), + Some([1e20, -1e20]), + Some([3.0, 4.0]), + ]); + let output = KeepFiniteVectors::new("v").transform(&batch).unwrap(); + + let kept = output.column_by_name("v").unwrap().as_fixed_size_list(); + assert_eq!(kept.len(), 3, "only finite rows survive"); + assert_eq!(kept.null_count(), 0); + assert_eq!( + kept.values().as_primitive::().values(), + &[1.0, 2.0, 1e20, -1e20, 3.0, 4.0] + ); + } + + #[test] + fn test_keep_finite_vectors_on_all_null_and_empty_batches() { + let all_null = fsl_batch(&[None, None]); + assert_eq!( + KeepFiniteVectors::new("v") + .transform(&all_null) + .unwrap() + .num_rows(), + 0 + ); + + let empty = fsl_batch(&[]); + assert_eq!( + KeepFiniteVectors::new("v") + .transform(&empty) + .unwrap() + .num_rows(), + 0 + ); + } + + #[test] + fn test_keep_finite_vectors_passes_through_missing_column() { + // A batch without the configured column is returned untouched rather + // than erroring, so the transform is a no-op on unrelated batches. + let batch = fsl_batch(&[Some([1.0, 2.0])]); + let output = KeepFiniteVectors::new("other").transform(&batch).unwrap(); + assert_eq!(output.num_rows(), 1); + } + + #[test] + fn test_keep_finite_vectors_rejects_non_list_column() { + let batch = RecordBatch::try_new( + Schema::new(vec![Field::new("v", DataType::Int32, false)]).into(), + vec![Arc::new(Int32Array::from(vec![1, 2]))], + ) + .unwrap(); + let error = KeepFiniteVectors::new("v").transform(&batch).unwrap_err(); + assert!(matches!(error, Error::Index { .. }), "{error:?}"); + let message = error.to_string(); + assert!(message.contains("column v"), "{message}"); + assert!(message.contains("Int32"), "{message}"); + } + #[test] fn test_is_all_finite() { let array = Float32Array::from(vec![1.0, 2.0]); diff --git a/rust/lance-index/src/vector/utils.rs b/rust/lance-index/src/vector/utils.rs index fb4f9004c57..e343c738e85 100644 --- a/rust/lance-index/src/vector/utils.rs +++ b/rust/lance-index/src/vector/utils.rs @@ -1,14 +1,18 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use arrow::compute::cast; +use arrow::{ + array::{ArrayData, make_array}, + buffer::Buffer, + compute::cast, +}; use arrow_array::types::{Float16Type, Float32Type, Float64Type}; use arrow_array::{Array, ArrayRef, BooleanArray, FixedSizeListArray, cast::AsArray}; use arrow_schema::{DataType, Field}; -use lance_arrow::FixedSizeListArrayExt; +use lance_arrow::{BufferExt, DataTypeExt, FixedSizeListArrayExt}; use lance_core::{Error, Result}; -use lance_io::encodings::plain::bytes_to_array; use lance_linalg::distance::DistanceType; +use lance_linalg::distance::dot_f16::amx_fp16_available; use prost::bytes; use std::sync::LazyLock; use std::{ops::Range, sync::Arc}; @@ -41,6 +45,49 @@ static USE_HNSW_SPEEDUP_INDEXING: LazyLock = LazyLock::new(|| } }); +/// Whether partition assignment is better served by the exact flat path than by +/// an approximate lookup through this index. +/// +/// The index turns one `M x N` problem -- every vector against every centroid -- +/// into `M` independent top-1 graph searches. Each search walks its own path, so +/// no two vectors share a candidate set and the AMX-FP16 kernel behind them can +/// only ever score one query against a handful of neighbors: 32 MAC/cycle, one +/// of the tile's 16 output columns. Keeping the problem in its matrix shape lets +/// [`crate::vector::kmeans::compute_partitions`] reach the AMX-FP16 GEMM, which +/// fills all four accumulator tiles at 512 MAC/cycle. +/// +/// Measured on 100M x 768 fp16 (dot, node-local, m=20, ef_construction=150), the +/// flat path won on both build time and recall at every `k` tried: +/// +/// | k | flat | indexed | +/// |-------|-------------|-------------| +/// | 10000 | 1965s / .980 | 2011s / .976 | +/// | 20000 | 2494s / .964 | 2588s / .832 | +/// | 40000 | 3730s / .970 | 3976s / .940 | +/// +/// The recall gap is the larger effect: the graph lookup runs at `ef = 15`, so +/// some vectors land in a partition that is not their nearest and no `nprobes` +/// setting recovers them. Flat assignment is exact. +/// +/// The conditions below must stay in lockstep with the AMX gate in +/// `compute_membership_and_dist`; without the GEMM the flat path is ~2.7x slower +/// than the index (5361s vs 2011s at k=10000), so a mismatch here is expensive. +/// That includes the `LANCE_DISABLE_AMX` kill switch, which both consult through +/// [`amx_fp16_available`]: an operator turning AMX off has to move this decision +/// too, or the build would take the exact-assignment path with no GEMM under it. +fn prefers_flat_amx_assignment( + centroid_type: &DataType, + num_centroids: usize, + dimension: usize, + distance_type: DistanceType, +) -> bool { + centroid_type == &DataType::Float16 + && distance_type == DistanceType::Dot + && dimension >= 32 + && num_centroids >= 32 + && amx_fp16_available() +} + #[derive(Debug)] pub struct SimpleIndex { store: SimpleStore, @@ -74,6 +121,7 @@ impl SimpleIndex { // - `num_centroids * dimension >= 1_000_000` // we benchmarked that it's 2x faster in the case of 1024 centroids and 1024 dimensions, // so set the threshold to 1_000_000. + // - the exact flat assignment is not already faster, see `prefers_flat_amx_assignment` pub fn may_train_index( centroids: ArrayRef, dimension: usize, @@ -84,6 +132,14 @@ impl SimpleIndex { if centroids.len() < 1_000_000 { return Ok(None); } + if prefers_flat_amx_assignment( + centroids.data_type(), + centroids.len() / dimension, + dimension, + distance_type, + ) { + return Ok(None); + } } SimpleIndexStatus::Disabled => return Ok(None), _ => {} @@ -109,6 +165,7 @@ impl SimpleIndex { lower_bound: None, upper_bound: None, dist_q_c: 0.0, + use_acorn: false, }; let res = match &self.store { SimpleStore::Float(store) => self.index.search_basic(query, 1, ¶ms, None, store)?, @@ -212,23 +269,40 @@ impl TryFrom<&pb::Tensor> for FixedSizeListArray { } let dim = tensor.shape[1] as usize; let num_rows = tensor.shape[0] as usize; - - let data = bytes::Bytes::from(tensor.data.clone()); - let flat_array = bytes_to_array( - &DataType::from(pb::tensor::DataType::try_from(tensor.data_type).unwrap()), - data, - dim * num_rows, - 0, - )?; - - if flat_array.len() != dim * num_rows { + let num_values = dim.checked_mul(num_rows).ok_or_else(|| { + Error::index(format!( + "Tensor shape {:?} exceeds the supported size", + tensor.shape + )) + })?; + let data_type = DataType::from(pb::tensor::DataType::try_from(tensor.data_type).unwrap()); + let expected_data_len = + num_values + .checked_mul(data_type.byte_width()) + .ok_or_else(|| { + Error::index(format!( + "Tensor shape {:?} exceeds the supported byte length", + tensor.shape + )) + })?; + if tensor.data.len() != expected_data_len { return Err(Error::index(format!( - "Tensor shape {:?} does not match to data len: {}", + "Tensor shape {:?} with data type {data_type} requires {expected_data_len} bytes, got {}", tensor.shape, - flat_array.len() + tensor.data.len() ))); } + let buffer = Buffer::from_bytes_bytes( + bytes::Bytes::from(tensor.data.clone()), + data_type.byte_width() as u64, + ); + let data = ArrayData::builder(data_type) + .len(num_values) + .null_count(0) + .add_buffer(buffer) + .build()?; + let flat_array = make_array(data); let field = Field::new("item", flat_array.data_type().clone(), true); Ok(Self::try_new( Arc::new(field), @@ -277,6 +351,7 @@ mod tests { use half::f16; use lance_arrow::FixedSizeListArrayExt; use num_traits::identities::Zero; + use rayon::ThreadPoolBuilder; use arrow::compute::cast; use rstest::rstest; @@ -307,7 +382,8 @@ mod tests { (0..100).flat_map(|i| std::iter::repeat_n(i as f32, 16)).collect::>(), )) as ArrayRef, 42.0f32)] fn test_simple_index_nearest_centroid(#[case] centroids: ArrayRef, #[case] query_val: f32) { - let index = build_index(centroids, 16); + let thread_pool = ThreadPoolBuilder::new().num_threads(1).build().unwrap(); + let index = thread_pool.install(|| build_index(centroids, 16)); let query: ArrayRef = Arc::new(Float32Array::from(vec![query_val; 16])); let (id, dist) = index.search(query).unwrap(); assert_eq!(id, 42); @@ -351,6 +427,8 @@ mod tests { assert_eq!(tensor.data_type, pb::tensor::DataType::Float16 as i32); assert_eq!(tensor.shape, vec![4, 5]); assert_eq!(tensor.data.len(), 20 * 2); + let decoded = FixedSizeListArray::try_from(&tensor).unwrap(); + assert_eq!(decoded.values().to_data(), fsl.values().to_data()); let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0; 20]), 5).unwrap(); @@ -358,6 +436,8 @@ mod tests { assert_eq!(tensor.data_type, pb::tensor::DataType::Float32 as i32); assert_eq!(tensor.shape, vec![4, 5]); assert_eq!(tensor.data.len(), 20 * 4); + let decoded = FixedSizeListArray::try_from(&tensor).unwrap(); + assert_eq!(decoded.values().to_data(), fsl.values().to_data()); let fsl = FixedSizeListArray::try_new_from_values(Float64Array::from(vec![0.0; 20]), 5).unwrap(); @@ -365,5 +445,57 @@ mod tests { assert_eq!(tensor.data_type, pb::tensor::DataType::Float64 as i32); assert_eq!(tensor.shape, vec![4, 5]); assert_eq!(tensor.data.len(), 20 * 8); + let decoded = FixedSizeListArray::try_from(&tensor).unwrap(); + assert_eq!(decoded.values().to_data(), fsl.values().to_data()); + } + + #[rstest] + #[case::too_short(vec![0; 7])] + #[case::too_long(vec![0; 9])] + fn test_tensor_to_fsl_rejects_invalid_data_length(#[case] data: Vec) { + let tensor = pb::Tensor { + data_type: pb::tensor::DataType::Uint32 as i32, + shape: vec![1, 2], + data, + }; + + let error = FixedSizeListArray::try_from(&tensor).unwrap_err(); + assert!(error.to_string().contains("requires 8 bytes")); + } + + /// Every shape the AMX-FP16 GEMM cannot serve must keep the centroid index, + /// because without the GEMM the flat path it would fall back to is ~2.7x + /// slower than the index. These four are the exact complement of the gate in + /// `compute_membership_and_dist`. + #[rstest] + #[case::not_f16(&DataType::Float32, 10_000, 768, DistanceType::Dot)] + #[case::not_dot(&DataType::Float16, 10_000, 768, DistanceType::L2)] + #[case::dim_below_one_k_pass(&DataType::Float16, 10_000, 31, DistanceType::Dot)] + #[case::k_below_one_b_block(&DataType::Float16, 31, 768, DistanceType::Dot)] + fn test_flat_amx_assignment_declines_unsupported_shapes( + #[case] centroid_type: &DataType, + #[case] num_centroids: usize, + #[case] dimension: usize, + #[case] distance_type: DistanceType, + ) { + assert!(!prefers_flat_amx_assignment( + centroid_type, + num_centroids, + dimension, + distance_type + )); + } + + /// On a supported shape the decision is exactly "is AMX-FP16 usable here", + /// which is a property of the build, the CPU and the `LANCE_DISABLE_AMX` + /// kill switch, so the expectation is derived rather than hardcoded -- the + /// same assertion has to hold with the switch set, on a machine without AMX, + /// and in a build whose toolchain could not compile the kernel. + #[test] + fn test_flat_amx_assignment_follows_amx_availability() { + assert_eq!( + prefers_flat_amx_assignment(&DataType::Float16, 10_000, 768, DistanceType::Dot), + amx_fp16_available() + ); } } diff --git a/rust/lance-index/src/vector/v3.rs b/rust/lance-index/src/vector/v3.rs index b210e4a7215..07f3979c66d 100644 --- a/rust/lance-index/src/vector/v3.rs +++ b/rust/lance-index/src/vector/v3.rs @@ -1,5 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +#[doc(hidden)] +pub mod shuffle_bench; pub mod shuffler; pub mod subindex; diff --git a/rust/lance-index/src/vector/v3/shuffle_bench.rs b/rust/lance-index/src/vector/v3/shuffle_bench.rs new file mode 100644 index 00000000000..32a53ce90b3 --- /dev/null +++ b/rust/lance-index/src/vector/v3/shuffle_bench.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Benchmark-only support for reopening a frozen two-file shuffle fixture. + +use std::sync::Arc; + +use lance_core::Result; +use lance_io::object_store::ObjectStore; +use object_store::path::Path; +use serde::{Deserialize, Serialize}; + +use super::shuffler::{ShuffleReader, TwoFileShuffleReader}; +/// The path-independent metadata needed to reopen a two-file shuffle fixture. +/// +/// This is public only so the external Criterion benchmark can serialize the +/// manifest on one revision and reopen the same data from another revision. +#[doc(hidden)] +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct TwoFileShuffleFixtureManifest { + pub num_partitions: usize, + pub num_flush_groups: u64, + pub partition_counts: Vec, + pub total_loss: f64, +} + +/// Reopen a frozen local two-file shuffle fixture. +/// +/// `output_dir` is deliberately separate from the manifest so a fixture can be +/// copied without embedding a machine-specific absolute path. +#[doc(hidden)] +pub async fn open_two_file_shuffle_fixture( + output_dir: Path, + manifest: &TwoFileShuffleFixtureManifest, +) -> Result> { + TwoFileShuffleReader::try_new( + Arc::new(ObjectStore::local()), + output_dir, + manifest.num_partitions, + manifest.num_flush_groups, + manifest.partition_counts.clone(), + manifest.total_loss, + ) + .await +} diff --git a/rust/lance-index/src/vector/v3/shuffler.rs b/rust/lance-index/src/vector/v3/shuffler.rs index 4203d099d0b..ee156961d24 100644 --- a/rust/lance-index/src/vector/v3/shuffler.rs +++ b/rust/lance-index/src/vector/v3/shuffler.rs @@ -5,24 +5,24 @@ //! the corresponding IVF partitions. use std::ops::Range; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; -use arrow::compute::concat_batches; -use arrow::datatypes::UInt64Type; use arrow::{array::AsArray, compute::sort_to_indices}; -use arrow_array::{RecordBatch, UInt32Array, UInt64Array}; -use arrow_schema::{DataType, Field, Schema}; +use arrow_array::{Array, RecordBatch, UInt32Array, UInt64Array}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; use futures::{future::try_join_all, prelude::*}; -use lance_arrow::{RecordBatchExt, SchemaExt, interleave_batches}; +use lance_arrow::{DataTypeExt, RecordBatchExt, SchemaExt, interleave_batches}; use lance_core::{ Error, Result, cache::LanceCache, + utils::parse::str_is_truthy, utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}, }; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; -use lance_encoding::version::LanceFileVersion; use lance_file::reader::{FileReader, FileReaderOptions}; -use lance_file::writer::{FileWriter, FileWriterOptions}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions; +use lance_file::writer::FileWriterOptions; use lance_io::{ ReadBatchParams, object_store::ObjectStore, @@ -34,6 +34,36 @@ use object_store::path::Path; use crate::vector::{LOSS_METADATA_KEY, PART_ID_COLUMN}; +/// Target decoded size for a contiguous shuffle partition window. +pub const DEFAULT_PARTITION_WINDOW_BYTES: usize = 128 * 1024 * 1024; + +/// One partition returned by [`ShuffleReader::read_partition_window`]. +pub struct ShufflePartition { + /// Zero-based IVF partition identifier. + pub partition_id: usize, + /// Partition rows, or `None` when the partition is empty. + pub data: Option>, +} + +/// A contiguous range of shuffled partitions read as one I/O window. +pub struct ShufflePartitionWindow { + /// Half-open range covered by `partitions`. + pub partition_range: Range, + /// One entry per partition in `partition_range`, including empty ones. + pub partitions: Vec, + /// Decoded bytes already materialized by the reader, counted once per + /// backing Arrow allocation. `None` means the returned streams are lazy. + pub materialized_decoded_bytes: Option, +} + +/// Metadata-only plan for a contiguous shuffle partition window. +pub struct ShufflePartitionWindowPlan { + /// Half-open partition range that the subsequent read will return. + pub partition_range: Range, + /// Conservative decoded-memory admission charge for the read. + pub estimated_decoded_bytes: usize, +} + #[async_trait::async_trait] /// A reader that can read the shuffled partitions. pub trait ShuffleReader: Send + Sync { @@ -45,6 +75,69 @@ pub trait ShuffleReader: Send + Sync { partition_id: usize, ) -> Result>>; + /// Plan a partition window without reading or decoding partition data. + /// + /// Readers without a decoded-size estimate use an oversized admission + /// charge for non-empty partitions so the read runs exclusively. + fn plan_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + if max_decoded_bytes == 0 { + return Err(Error::invalid_input( + "max_decoded_bytes must be greater than 0", + )); + } + let end = start_partition_id.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "start_partition_id={} cannot be advanced", + start_partition_id + )) + })?; + let partition_rows = self.partition_size(start_partition_id)?; + Ok(ShufflePartitionWindowPlan { + partition_range: start_partition_id..end, + estimated_decoded_bytes: if partition_rows == 0 { 0 } else { usize::MAX }, + }) + } + + /// Read a contiguous partition window starting at `start_partition_id`. + /// + /// Readers that cannot coalesce adjacent partitions return a singleton + /// window. The byte budget is a decoded-memory target, not an encoded I/O + /// size. A partition larger than the budget is returned as a singleton. + async fn read_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + if max_decoded_bytes == 0 { + return Err(Error::invalid_input( + "max_decoded_bytes must be greater than 0", + )); + } + let end = start_partition_id.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "start_partition_id={} cannot be advanced", + start_partition_id + )) + })?; + let data = if self.partition_size(start_partition_id)? == 0 { + None + } else { + self.read_partition(start_partition_id).await? + }; + Ok(ShufflePartitionWindow { + partition_range: start_partition_id..end, + partitions: vec![ShufflePartition { + partition_id: start_partition_id, + data, + }], + materialized_decoded_bytes: None, + }) + } + /// Get the size of the partition by partition_id fn partition_size(&self, partition_id: usize) -> Result; @@ -71,7 +164,7 @@ pub struct IvfShuffler { object_store: Arc, output_dir: Path, num_partitions: usize, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, progress: Arc, } @@ -82,12 +175,12 @@ impl IvfShuffler { object_store: Arc::new(ObjectStore::local()), output_dir, num_partitions, - format_version: LanceFileVersion::V2_0, + format_version: ConcreteFileVersion::V2_0, progress: crate::progress::noop_progress(), } } - pub fn with_format_version(mut self, format_version: LanceFileVersion) -> Self { + pub fn with_format_version(mut self, format_version: ConcreteFileVersion) -> Self { self.format_version = format_version; self } @@ -107,6 +200,7 @@ impl Shuffler for IvfShuffler { let num_partitions = self.num_partitions; let mut partition_sizes = vec![0; num_partitions]; let schema = data.schema().without_column(PART_ID_COLUMN); + let estimated_row_bytes = estimate_decoded_row_bytes(&schema)?; let mut writers = stream::iter(0..num_partitions) .map(|partition_id| { let part_path = self @@ -122,13 +216,11 @@ impl Shuffler for IvfShuffler { let format_version = self.format_version; async move { let writer = object_store.create(&part_path).await?; - let file_writer = FileWriter::try_new( + let file_writer = versions::create_writer( + format_version, writer, lance_core::datatypes::Schema::try_from(&schema)?, - FileWriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + FileWriterOptions::default(), )? .with_page_metadata_spill(object_store.clone(), spill_path); Result::Ok(file_writer) @@ -202,12 +294,15 @@ impl Shuffler for IvfShuffler { writer.finish().await?; } - Ok(Box::new(IvfShufflerReader::new( - self.object_store.clone(), - self.output_dir.clone(), - partition_sizes, - total_loss, - ))) + Ok(Box::new( + IvfShufflerReader::new( + self.object_store.clone(), + self.output_dir.clone(), + partition_sizes, + total_loss, + ) + .with_estimated_row_bytes(estimated_row_bytes), + )) } } @@ -215,6 +310,7 @@ pub struct IvfShufflerReader { scheduler: Arc, output_dir: Path, partition_sizes: Vec, + estimated_row_bytes: Option, loss: f64, } @@ -231,9 +327,15 @@ impl IvfShufflerReader { scheduler, output_dir, partition_sizes, + estimated_row_bytes: None, loss, } } + + fn with_estimated_row_bytes(mut self, estimated_row_bytes: usize) -> Self { + self.estimated_row_bytes = Some(estimated_row_bytes); + self + } } #[async_trait::async_trait] @@ -276,6 +378,42 @@ impl ShuffleReader for IvfShufflerReader { )))) } + fn plan_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + if max_decoded_bytes == 0 { + return Err(Error::invalid_input( + "max_decoded_bytes must be greater than 0", + )); + } + let Some(&partition_rows) = self.partition_sizes.get(start_partition_id) else { + return Err(Error::invalid_input(format!( + "start_partition_id={} is out of range [0, {})", + start_partition_id, + self.partition_sizes.len() + ))); + }; + let end_partition_id = start_partition_id.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "start_partition_id={} cannot be advanced", + start_partition_id + )) + })?; + let estimated_decoded_bytes = match (partition_rows, self.estimated_row_bytes) { + (0, _) => 0, + (_, Some(estimated_row_bytes)) => { + conservative_partition_admission_bytes(partition_rows, estimated_row_bytes)? + } + (_, None) => usize::MAX, + }; + Ok(ShufflePartitionWindowPlan { + partition_range: start_partition_id..end_partition_id, + estimated_decoded_bytes, + }) + } + fn partition_size(&self, partition_id: usize) -> Result { Ok(self.partition_sizes.get(partition_id).copied().unwrap_or(0)) } @@ -315,11 +453,11 @@ impl ShuffleReader for EmptyReader { pub fn create_ivf_shuffler( output_dir: Path, num_partitions: usize, - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, progress: Option>, ) -> Box { let use_legacy = std::env::var("LANCE_LEGACY_SHUFFLER") - .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .map(|v| str_is_truthy(&v)) .unwrap_or(false); if use_legacy { let mut shuffler = @@ -337,8 +475,24 @@ pub fn create_ivf_shuffler( } } +/// Schema of the partition-offsets sidecar written alongside shuffled data. +static OFFSETS_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![Field::new( + "offset", + DataType::UInt64, + false, + )])) +}); + const DEFAULT_SHUFFLE_BATCH_BYTES: usize = 128 * 1024 * 1024; +/// Maximum resident size of the preloaded offsets table. +/// +/// This covers tens of millions of offsets while bounding the additional memory +/// held for unusually large shuffles. Larger tables are still validated once as +/// a stream and then read on demand. +const MAX_PRELOADED_OFFSETS_BYTES: usize = 256 * 1024 * 1024; + /// Number of rows per output batch when streaming sorted data via interleave. /// Small enough to keep the output chunk's memory footprint modest relative to /// the accumulated source data. @@ -464,18 +618,14 @@ impl Shuffler for TwoFileShuffler { let num_partitions = self.num_partitions; // No need to write partition ids since we can infer this from offsets let schema = data.schema().without_column(PART_ID_COLUMN); - let offsets_schema = Arc::new(Schema::new(vec![Field::new( - "offset", - DataType::UInt64, - false, - )])); + let offsets_schema = OFFSETS_SCHEMA.clone(); let batch_size_bytes = self.batch_size_bytes; // Create data file writer let data_path = self.output_dir.clone().join("shuffle_data.lance"); let spill_path = self.output_dir.clone().join("shuffle_data.spill"); let writer = self.object_store.create(&data_path).await?; - let mut file_writer = FileWriter::try_new( + let mut file_writer = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(&schema)?, Default::default(), @@ -486,7 +636,7 @@ impl Shuffler for TwoFileShuffler { let offsets_path = self.output_dir.clone().join("shuffle_offsets.lance"); let spill_path = self.output_dir.clone().join("shuffle_offsets.spill"); let writer = self.object_store.create(&offsets_path).await?; - let mut offsets_writer = FileWriter::try_new( + let mut offsets_writer = versions::v2_1::create_writer( writer, lance_core::datatypes::Schema::try_from(offsets_schema.as_ref())?, Default::default(), @@ -577,8 +727,8 @@ impl Shuffler for TwoFileShuffler { /// Returns `(total_rows_written, per_partition_row_counts)`. async fn flush_shuffle_batch( accumulated: Vec, - file_writer: &mut FileWriter, - offsets_writer: &mut FileWriter, + file_writer: &mut versions::v2_1::Writer, + offsets_writer: &mut versions::v2_1::Writer, offsets_schema: Arc, num_partitions: usize, global_row_count: u64, @@ -632,21 +782,48 @@ async fn flush_shuffle_batch( pub struct TwoFileShuffleReader { _scheduler: Arc, file_reader: FileReader, - offsets_reader: FileReader, num_partitions: usize, - num_batches: u64, + num_batches: usize, + offsets: ShuffleOffsets, partition_counts: Vec, + estimated_row_bytes: usize, total_loss: f64, } +enum ShuffleOffsets { + Preloaded(Vec), + OnDemand(FileReader), +} + impl TwoFileShuffleReader { - async fn try_new( + pub(super) async fn try_new( + object_store: Arc, + output_dir: Path, + num_partitions: usize, + num_batches: u64, + partition_counts: Vec, + total_loss: f64, + ) -> Result> { + Self::try_new_with_preload_limit( + object_store, + output_dir, + num_partitions, + num_batches, + partition_counts, + total_loss, + MAX_PRELOADED_OFFSETS_BYTES, + ) + .await + } + + async fn try_new_with_preload_limit( object_store: Arc, output_dir: Path, num_partitions: usize, num_batches: u64, partition_counts: Vec, total_loss: f64, + max_preloaded_offsets_bytes: usize, ) -> Result> { if num_batches == 0 { return Ok(Box::new(EmptyReader)); @@ -679,63 +856,654 @@ impl TwoFileShuffleReader { ) .await?; + if partition_counts.len() != num_partitions { + return Err(Error::invalid_input(format!( + "partition_counts has {} entries, expected num_partitions={}", + partition_counts.len(), + num_partitions + ))); + } + + let num_batches = usize::try_from(num_batches).map_err(|_| { + Error::invalid_input(format!( + "num_batches={} cannot be represented as usize", + num_batches + )) + })?; + let expected_offsets = num_batches.checked_mul(num_partitions).ok_or_else(|| { + Error::invalid_input(format!( + "num_batches={} * num_partitions={} overflows usize", + num_batches, num_partitions + )) + })?; + let expected_offsets_u64 = u64::try_from(expected_offsets).map_err(|_| { + Error::invalid_input(format!( + "expected offset count {} cannot be represented as u64", + expected_offsets + )) + })?; + if offsets_reader.num_rows() != expected_offsets_u64 { + return Err(Error::corrupt_file( + offsets_path.clone(), + format!( + "offset count is {}, expected num_batches={} * num_partitions={} = {}", + offsets_reader.num_rows(), + num_batches, + num_partitions, + expected_offsets + ), + )); + } + + let offsets_schema = offsets_reader.schema(); + let offset_field = offsets_schema.field("offset").ok_or_else(|| { + Error::corrupt_file( + offsets_path.clone(), + "required non-null UInt64 column 'offset' is missing", + ) + })?; + if offset_field.data_type() != DataType::UInt64 || offset_field.nullable { + return Err(Error::corrupt_file( + offsets_path.clone(), + format!( + "column 'offset' must be non-null UInt64, found {:?} (nullable={})", + offset_field.data_type(), + offset_field.nullable + ), + )); + } + + let should_preload_offsets = + should_preload_offsets(expected_offsets, max_preloaded_offsets_bytes)?; + let mut offsets = should_preload_offsets.then(|| Vec::with_capacity(expected_offsets)); + let mut validator = ShuffleOffsetsValidator::new( + expected_offsets, + num_partitions, + file_reader.num_rows(), + &partition_counts, + &offsets_path, + ); + let mut offsets_stream = offsets_reader + .read_stream( + ReadBatchParams::RangeFull, + 1024 * 1024, + 16, + FilterExpression::no_filter(), + ) + .await?; + while let Some(batch) = offsets_stream.try_next().await? { + let offset_column = batch + .column_by_name("offset") + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + Error::corrupt_file( + offsets_path.clone(), + "required UInt64 column 'offset' is missing from decoded batch", + ) + })?; + if offset_column.null_count() != 0 { + return Err(Error::corrupt_file( + offsets_path.clone(), + format!( + "column 'offset' contains {} null values", + offset_column.null_count() + ), + )); + } + validator.push(offset_column.values())?; + if let Some(offsets) = offsets.as_mut() { + offsets.extend_from_slice(offset_column.values()); + } + } + validator.finish()?; + let offsets = match offsets { + Some(offsets) => ShuffleOffsets::Preloaded(offsets), + None => ShuffleOffsets::OnDemand(offsets_reader), + }; + let decoded_schema: Schema = file_reader.schema().as_ref().into(); + let estimated_row_bytes = estimate_decoded_row_bytes(&decoded_schema)?; + Ok(Box::new(Self { _scheduler: scheduler, file_reader, - offsets_reader, num_partitions, num_batches, + offsets, partition_counts, + estimated_row_bytes, total_loss, })) } async fn partition_ranges(&self, partition_id: usize) -> Result>> { - let mut positions = Vec::with_capacity(self.num_batches as usize * 2); + if partition_id >= self.num_partitions { + return Err(Error::invalid_input(format!( + "partition_id={} is out of range [0, {})", + partition_id, self.num_partitions + ))); + } + + match &self.offsets { + ShuffleOffsets::Preloaded(offsets) => { + let mut ranges = Vec::with_capacity(self.num_batches); + for batch_idx in 0..self.num_batches { + let end_index = batch_idx * self.num_partitions + partition_id; + let start = if end_index == 0 { + 0 + } else { + offsets[end_index - 1] + }; + ranges.push(start..offsets[end_index]); + } + Ok(ranges) + } + ShuffleOffsets::OnDemand(offsets_reader) => { + self.read_partition_ranges(offsets_reader, partition_id) + .await + } + } + } + + async fn read_partition_ranges( + &self, + offsets_reader: &FileReader, + partition_id: usize, + ) -> Result>> { + let max_offset_values = self.num_batches.checked_mul(2).ok_or_else(|| { + Error::invalid_input(format!( + "num_batches={} overflows on-demand offset count", + self.num_batches + )) + })?; + let mut offset_ranges = Vec::with_capacity(max_offset_values); for batch_idx in 0..self.num_batches { - let end_pos = u32::try_from(batch_idx as usize * self.num_partitions + partition_id) - .map_err(|_| Error::invalid_input("There are more than 2^32 partition offsets in the spill file. Need to support 64-bit take"))?; - if end_pos != 0 { - positions.push(end_pos - 1); + let end_index = batch_idx * self.num_partitions + partition_id; + if end_index != 0 { + let start_index = u64::try_from(end_index - 1).map_err(|_| { + Error::invalid_input(format!( + "offset index {} cannot be represented as u64", + end_index - 1 + )) + })?; + offset_ranges.push(start_index..start_index + 1); } - positions.push(end_pos); + let end_index = u64::try_from(end_index).map_err(|_| { + Error::invalid_input(format!( + "offset index {} cannot be represented as u64", + end_index + )) + })?; + offset_ranges.push(end_index..end_index + 1); } - let positions = UInt32Array::from(positions); - let num_positions = positions.len() as u32; - let offsets_stream = self - .offsets_reader + + let mut offsets_stream = offsets_reader .read_stream( - ReadBatchParams::Indices(positions), - num_positions, + ReadBatchParams::Ranges(offset_ranges.into()), + u32::MAX, 1, FilterExpression::no_filter(), ) .await?; - let schema = offsets_stream.schema().clone(); - let offsets = offsets_stream.try_collect::>().await?; - let offsets = if offsets.is_empty() { - // We should not hit this path if there is no batches - unreachable!() - } else if offsets.len() == 1 { - offsets.into_iter().next().unwrap() + let expected_values = max_offset_values - usize::from(partition_id == 0); + let mut offsets = Vec::with_capacity(expected_values); + while let Some(batch) = offsets_stream.try_next().await? { + let offset_column = batch + .column_by_name("offset") + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| { + Error::corrupt_file_named( + "shuffle_offsets.lance", + "required UInt64 column 'offset' is missing from decoded batch", + ) + })?; + offsets.extend_from_slice(offset_column.values()); + } + if offsets.len() != expected_values { + return Err(Error::corrupt_file_named( + "shuffle_offsets.lance", + format!( + "decoded {} on-demand offsets for partition {}, expected {}", + offsets.len(), + partition_id, + expected_values + ), + )); + } + + let mut offsets = offsets.into_iter(); + let mut ranges = Vec::with_capacity(self.num_batches); + for batch_idx in 0..self.num_batches { + let start = if batch_idx == 0 && partition_id == 0 { + 0 + } else { + offsets.next().ok_or_else(|| { + Error::corrupt_file_named( + "shuffle_offsets.lance", + format!("missing start offset for partition {}", partition_id), + ) + })? + }; + let end = offsets.next().ok_or_else(|| { + Error::corrupt_file_named( + "shuffle_offsets.lance", + format!("missing end offset for partition {}", partition_id), + ) + })?; + ranges.push(start..end); + } + Ok(ranges) + } +} + +#[cfg(test)] +fn validate_shuffle_offsets( + offsets: &[u64], + num_batches: usize, + num_partitions: usize, + data_rows: u64, + partition_counts: &[u64], + offsets_path: &Path, +) -> Result<()> { + let expected_offsets = num_batches.checked_mul(num_partitions).ok_or_else(|| { + Error::invalid_input(format!( + "num_batches={} * num_partitions={} overflows usize", + num_batches, num_partitions + )) + })?; + let mut validator = ShuffleOffsetsValidator::new( + expected_offsets, + num_partitions, + data_rows, + partition_counts, + offsets_path, + ); + validator.push(offsets)?; + validator.finish() +} + +fn should_preload_offsets(expected_offsets: usize, max_bytes: usize) -> Result { + let offsets_bytes = expected_offsets + .checked_mul(std::mem::size_of::()) + .ok_or_else(|| { + Error::invalid_input(format!( + "expected offset count {} overflows byte-size calculation", + expected_offsets + )) + })?; + Ok(offsets_bytes <= max_bytes) +} + +struct ShuffleOffsetsValidator<'a> { + expected_offsets: usize, + num_partitions: usize, + data_rows: u64, + partition_counts: &'a [u64], + offsets_path: &'a Path, + decoded_offsets: usize, + previous_offset: u64, + decoded_partition_counts: Vec, +} + +impl<'a> ShuffleOffsetsValidator<'a> { + fn new( + expected_offsets: usize, + num_partitions: usize, + data_rows: u64, + partition_counts: &'a [u64], + offsets_path: &'a Path, + ) -> Self { + Self { + expected_offsets, + num_partitions, + data_rows, + partition_counts, + offsets_path, + decoded_offsets: 0, + previous_offset: 0, + decoded_partition_counts: vec![0; num_partitions], + } + } + + fn push(&mut self, offsets: &[u64]) -> Result<()> { + for &offset in offsets { + if self.decoded_offsets >= self.expected_offsets { + return Err(Error::corrupt_file( + self.offsets_path.clone(), + format!( + "decoded more than the expected {} offsets", + self.expected_offsets + ), + )); + } + if self.previous_offset > offset { + return Err(Error::corrupt_file( + self.offsets_path.clone(), + format!( + "offsets are not monotonic at indices {} and {}: {} > {}", + self.decoded_offsets - 1, + self.decoded_offsets, + self.previous_offset, + offset + ), + )); + } + + let partition_id = self.decoded_offsets % self.num_partitions; + self.decoded_partition_counts[partition_id] = self.decoded_partition_counts + [partition_id] + .checked_add(offset - self.previous_offset) + .ok_or_else(|| { + Error::corrupt_file( + self.offsets_path.clone(), + format!("row count for partition {} overflows u64", partition_id), + ) + })?; + self.previous_offset = offset; + self.decoded_offsets += 1; + } + Ok(()) + } + + fn finish(self) -> Result<()> { + if self.decoded_offsets != self.expected_offsets { + return Err(Error::corrupt_file( + self.offsets_path.clone(), + format!( + "decoded {} offsets, expected {}", + self.decoded_offsets, self.expected_offsets + ), + )); + } + if self.previous_offset != self.data_rows { + return Err(Error::corrupt_file( + self.offsets_path.clone(), + format!( + "final offset {} does not match shuffle data row count {}", + self.previous_offset, self.data_rows + ), + )); + } + if let Some((partition_id, (&decoded, &expected))) = self + .decoded_partition_counts + .iter() + .zip(self.partition_counts) + .enumerate() + .find(|(_, (decoded, expected))| decoded != expected) + { + return Err(Error::corrupt_file( + self.offsets_path.clone(), + format!( + "offset-derived count {} for partition {} does not match expected count {}", + decoded, partition_id, expected + ), + )); + } + Ok(()) + } +} + +/// Variable-width columns are uncommon in vector shuffle data. This fallback +/// keeps window planning bounded when one is present without claiming an exact +/// decoded size for values that have no fixed Arrow stride. +const VARIABLE_WIDTH_ROW_ESTIMATE_BYTES: usize = 64; +const WINDOW_ADMISSION_FIXED_HEADROOM_BYTES: usize = 1024 * 1024; + +fn estimate_decoded_row_bytes(schema: &Schema) -> Result { + let mut row_bytes = 0usize; + for field in schema.fields() { + let value_bytes = match field.data_type() { + DataType::Boolean => 1, + data_type => data_type + .byte_width_opt() + .unwrap_or(VARIABLE_WIDTH_ROW_ESTIMATE_BYTES), + }; + row_bytes = row_bytes.checked_add(value_bytes).ok_or_else(|| { + Error::invalid_input(format!( + "decoded row-size estimate overflows usize at field '{}'", + field.name() + )) + })?; + if field.is_nullable() { + // Arrow validity is bit-packed. One byte per row is deliberately + // conservative and also covers small-buffer alignment overhead. + row_bytes = row_bytes.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "decoded row-size estimate overflows usize at nullable field '{}'", + field.name() + )) + })?; + } + } + Ok(row_bytes.max(1)) +} + +fn plan_partition_window_end( + partition_counts: &[u64], + start_partition_id: usize, + estimated_row_bytes: usize, + max_decoded_bytes: usize, +) -> Result { + if max_decoded_bytes == 0 { + return Err(Error::invalid_input( + "max_decoded_bytes must be greater than 0", + )); + } + if start_partition_id >= partition_counts.len() { + return Err(Error::invalid_input(format!( + "start_partition_id={} is out of range [0, {})", + start_partition_id, + partition_counts.len() + ))); + } + + let mut decoded_bytes = 0usize; + let mut end_partition_id = start_partition_id; + while end_partition_id < partition_counts.len() { + let partition_rows = usize::try_from(partition_counts[end_partition_id]).map_err(|_| { + Error::invalid_input(format!( + "partition {} row count {} cannot be represented as usize", + end_partition_id, partition_counts[end_partition_id] + )) + })?; + let partition_bytes = partition_rows + .checked_mul(estimated_row_bytes) + .ok_or_else(|| { + Error::invalid_input(format!( + "decoded byte estimate overflows for partition {} with {} rows at {} bytes per row", + end_partition_id, partition_rows, estimated_row_bytes + )) + })?; + + if end_partition_id > start_partition_id + && partition_bytes > max_decoded_bytes.saturating_sub(decoded_bytes) + { + break; + } + decoded_bytes = decoded_bytes.checked_add(partition_bytes).ok_or_else(|| { + Error::invalid_input(format!( + "decoded window byte estimate overflows at partition {}", + end_partition_id + )) + })?; + end_partition_id += 1; + + // A partition that exceeds the budget must make progress as a + // singleton. Otherwise stop as soon as the target has been filled. + if decoded_bytes >= max_decoded_bytes { + break; + } + } + Ok(end_partition_id) +} + +fn conservative_window_admission_bytes( + partition_counts: &[u64], + partition_range: Range, + estimated_row_bytes: usize, +) -> Result { + let rows = partition_counts[partition_range] + .iter() + .try_fold(0usize, |total, count| { + let count = usize::try_from(*count).map_err(|_| { + Error::invalid_input(format!( + "partition row count {} cannot be represented as usize", + count + )) + })?; + total + .checked_add(count) + .ok_or_else(|| Error::invalid_input("partition window row count overflows usize")) + })?; + if rows == 0 { + return Ok(0); + } + conservative_partition_admission_bytes(rows, estimated_row_bytes) +} + +fn conservative_partition_admission_bytes( + rows: usize, + estimated_row_bytes: usize, +) -> Result { + let value_bytes = rows.checked_mul(estimated_row_bytes).ok_or_else(|| { + Error::invalid_input(format!( + "decoded byte estimate overflows for {} rows at {} bytes per row", + rows, estimated_row_bytes + )) + })?; + // Arrow buffers and batch/array allocations add a small amount beyond the + // fixed-width values. Reserve 25% plus fixed headroom before decoding; the + // charge is reconciled to the allocation-backed size immediately after. + value_bytes + .checked_add(value_bytes / 4) + .and_then(|bytes| bytes.checked_add(WINDOW_ADMISSION_FIXED_HEADROOM_BYTES)) + .ok_or_else(|| Error::invalid_input("partition window admission estimate overflows usize")) +} + +type PartitionWindowReadPlan = (Vec>, Vec>); + +fn preloaded_window_ranges( + offsets: &[u64], + num_batches: usize, + num_partitions: usize, + partition_range: Range, +) -> Result { + let window_len = partition_range.end - partition_range.start; + let mut ranges = Vec::with_capacity(num_batches); + let mut group_partition_counts = Vec::with_capacity(num_batches); + + for batch_idx in 0..num_batches { + let group_base = batch_idx * num_partitions; + let range_start_index = group_base + partition_range.start; + let range_start = if range_start_index == 0 { + 0 } else { - concat_batches(&schema, &offsets)? + offsets[range_start_index - 1] }; + let range_end = offsets[group_base + partition_range.end - 1]; + if range_start == range_end { + continue; + } - let offsets = offsets.column(0).as_primitive::(); - let mut offsets_iter = offsets.values().iter().copied(); + let mut counts = Vec::with_capacity(window_len); + let mut previous = range_start; + for partition_id in partition_range.clone() { + let end = offsets[group_base + partition_id]; + let count = usize::try_from(end - previous).map_err(|_| { + Error::corrupt_file_named( + "shuffle_offsets.lance", + format!( + "row count {} for flush group {} partition {} cannot be represented as usize", + end - previous, + batch_idx, + partition_id + ), + ) + })?; + counts.push(count); + previous = end; + } + ranges.push(range_start..range_end); + group_partition_counts.push(counts); + } + Ok((ranges, group_partition_counts)) +} - let mut ranges = Vec::with_capacity(self.num_batches as usize); - for batch_idx in 0..self.num_batches { - if batch_idx == 0 && partition_id == 0 { - // Implicit 0 for start-of-file - ranges.push(0..offsets_iter.next().unwrap()); - } else { - ranges.push(offsets_iter.next().unwrap()..offsets_iter.next().unwrap()); +async fn split_partition_window_stream( + mut stream: S, + window_len: usize, + group_partition_counts: &[Vec], +) -> Result<(Vec>, usize)> +where + S: Stream> + Unpin, +{ + let expected_rows = group_partition_counts + .iter() + .flatten() + .try_fold(0usize, |total, count| total.checked_add(*count)) + .ok_or_else(|| { + Error::corrupt_file_named("shuffle_data.lance", "window row count overflows usize") + })?; + let mut segments = group_partition_counts + .iter() + .flat_map(|counts| counts.iter().copied().enumerate()) + .filter(|(_, count)| *count != 0); + let mut current_segment = segments.next(); + let mut segment_rows_read = 0usize; + let mut actual_rows = 0usize; + let mut materialized_decoded_bytes = 0usize; + let mut partition_batches = vec![Vec::new(); window_len]; + + while let Some(batch) = stream.try_next().await? { + materialized_decoded_bytes = + batch + .columns() + .iter() + .try_fold(materialized_decoded_bytes, |total, array| { + total + .checked_add(array.get_array_memory_size()) + .ok_or_else(|| { + Error::internal("decoded partition window byte count overflows usize") + }) + })?; + let mut batch_offset = 0usize; + while batch_offset < batch.num_rows() { + let Some((partition_offset, segment_rows)) = current_segment else { + return Err(Error::corrupt_file_named( + "shuffle_data.lance", + format!( + "decoded more than the expected {} rows for partition window", + expected_rows + ), + )); + }; + let remaining_in_segment = segment_rows - segment_rows_read; + let rows_to_take = remaining_in_segment.min(batch.num_rows() - batch_offset); + partition_batches[partition_offset].push(batch.slice(batch_offset, rows_to_take)); + batch_offset += rows_to_take; + actual_rows = actual_rows.checked_add(rows_to_take).ok_or_else(|| { + Error::corrupt_file_named( + "shuffle_data.lance", + "decoded window row count overflows usize", + ) + })?; + segment_rows_read += rows_to_take; + if segment_rows_read == segment_rows { + current_segment = segments.next(); + segment_rows_read = 0; } } - Ok(ranges) } + + if current_segment.is_some() { + return Err(Error::corrupt_file_named( + "shuffle_data.lance", + format!( + "decoded {} rows for partition window, expected {}", + actual_rows, expected_rows + ), + )); + } + Ok((partition_batches, materialized_decoded_bytes)) } #[async_trait::async_trait] @@ -772,6 +1540,126 @@ impl ShuffleReader for TwoFileShuffleReader { )))) } + fn plan_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + if start_partition_id >= self.num_partitions { + return Err(Error::invalid_input(format!( + "start_partition_id={} is out of range [0, {})", + start_partition_id, self.num_partitions + ))); + } + let end_partition_id = match &self.offsets { + ShuffleOffsets::Preloaded(_) => plan_partition_window_end( + &self.partition_counts, + start_partition_id, + self.estimated_row_bytes, + max_decoded_bytes, + )?, + ShuffleOffsets::OnDemand(_) => start_partition_id.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "start_partition_id={} cannot be advanced", + start_partition_id + )) + })?, + }; + let partition_range = start_partition_id..end_partition_id; + let estimated_decoded_bytes = conservative_window_admission_bytes( + &self.partition_counts, + partition_range.clone(), + self.estimated_row_bytes, + )?; + Ok(ShufflePartitionWindowPlan { + partition_range, + estimated_decoded_bytes, + }) + } + + async fn read_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + if max_decoded_bytes == 0 { + return Err(Error::invalid_input( + "max_decoded_bytes must be greater than 0", + )); + } + + let ShuffleOffsets::Preloaded(offsets) = &self.offsets else { + // The bounded-memory offsets fallback retains the legacy singleton + // path because coalescing would otherwise re-read many offset rows. + let end = start_partition_id.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "start_partition_id={} cannot be advanced", + start_partition_id + )) + })?; + let data = self.read_partition(start_partition_id).await?; + return Ok(ShufflePartitionWindow { + partition_range: start_partition_id..end, + partitions: vec![ShufflePartition { + partition_id: start_partition_id, + data, + }], + materialized_decoded_bytes: None, + }); + }; + + let partition_range = self + .plan_partition_window(start_partition_id, max_decoded_bytes)? + .partition_range; + let (ranges, group_partition_counts) = preloaded_window_ranges( + offsets, + self.num_batches, + self.num_partitions, + partition_range.clone(), + )?; + let schema: Schema = self.file_reader.schema().as_ref().into(); + let schema = Arc::new(schema); + + let (partition_batches, materialized_decoded_bytes) = if ranges.is_empty() { + (vec![Vec::new(); partition_range.len()], 0) + } else { + let stream = self + .file_reader + .read_stream( + ReadBatchParams::Ranges(ranges.into()), + u32::MAX, + 16, + FilterExpression::no_filter(), + ) + .await?; + split_partition_window_stream(stream, partition_range.len(), &group_partition_counts) + .await? + }; + + let partitions = partition_range + .clone() + .zip(partition_batches) + .map(|(partition_id, batches)| { + let data = if batches.is_empty() { + None + } else { + let stream = futures::stream::iter(batches.into_iter().map(Ok)); + Some( + Box::new(RecordBatchStreamAdapter::new(schema.clone(), stream)) + as Box, + ) + }; + ShufflePartition { partition_id, data } + }) + .collect(); + + Ok(ShufflePartitionWindow { + partition_range, + partitions, + materialized_decoded_bytes: Some(materialized_decoded_bytes), + }) + } + fn partition_size(&self, partition_id: usize) -> Result { Ok(self .partition_counts @@ -842,6 +1730,15 @@ mod tests { Some(arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap()) } + async fn collect_values(mut stream: Box) -> Vec { + let mut values = Vec::new(); + while let Some(batch) = stream.try_next().await.unwrap() { + let batch_values: &Int32Array = batch["val"].as_primitive(); + values.extend_from_slice(batch_values.values()); + } + values + } + #[tokio::test] async fn test_two_file_shuffler_round_trip() { let dir = TempStrDir::default(); @@ -853,10 +1750,34 @@ mod tests { // Partition 2: rows with values 30 let batch = make_batch(&[0, 1, 2, 0, 1], &[10, 20, 30, 40, 50], None); - let shuffler = TwoFileShuffler::new(output_dir, num_partitions); + let shuffler = TwoFileShuffler::new(output_dir.clone(), num_partitions); let stream = batches_to_stream(vec![batch]); let reader = shuffler.shuffle(stream).await.unwrap(); + let object_store = Arc::new(ObjectStore::local()); + let scheduler = ScanScheduler::new( + object_store.clone(), + SchedulerConfig::max_bandwidth(&object_store), + ); + for filename in ["shuffle_data.lance", "shuffle_offsets.lance"] { + let file_reader = FileReader::try_open( + scheduler + .open_file( + &output_dir.clone().join(filename), + &CachedFileSize::unknown(), + ) + .await + .unwrap(), + None, + Arc::::default(), + &LanceCache::no_cache(), + FileReaderOptions::default(), + ) + .await + .unwrap(); + assert_eq!(file_reader.version(), ConcreteFileVersion::V2_1); + } + // Verify partition sizes assert_eq!(reader.partition_size(0).unwrap(), 2); assert_eq!(reader.partition_size(1).unwrap(), 2); @@ -886,6 +1807,32 @@ mod tests { assert!(reader.read_partition(3).await.unwrap().is_none()); } + #[tokio::test] + async fn test_two_file_shuffler_empty_first_batch() { + let dir = TempStrDir::default(); + let output_dir = Path::from(dir.as_ref()); + let empty_batch = make_batch(&[], &[], None); + let data_batch = make_batch(&[1, 0, 1], &[10, 20, 30], None); + + let shuffler = TwoFileShuffler::new(output_dir, 2); + let stream = batches_to_stream(vec![empty_batch, data_batch]); + let reader = shuffler.shuffle(stream).await.unwrap(); + + assert_eq!(reader.partition_size(0).unwrap(), 1); + assert_eq!(reader.partition_size(1).unwrap(), 2); + + let expected_schema = ArrowSchema::new(vec![Field::new("val", DataType::Int32, false)]); + let p0 = collect_partition(reader.as_ref(), 0).await.unwrap(); + assert_eq!(p0.schema().as_ref(), &expected_schema); + let p0_values: &Int32Array = p0["val"].as_primitive(); + assert_eq!(p0_values.values(), &[20]); + + let p1 = collect_partition(reader.as_ref(), 1).await.unwrap(); + assert_eq!(p1.schema().as_ref(), &expected_schema); + let p1_values: &Int32Array = p1["val"].as_primitive(); + assert_eq!(p1_values.values(), &[10, 30]); + } + #[tokio::test] async fn test_two_file_shuffler_empty_partitions() { let dir = TempStrDir::default(); @@ -1005,6 +1952,339 @@ mod tests { assert!((reader.total_loss().unwrap() - 6.0).abs() < 1e-10); } + #[tokio::test] + async fn test_two_file_shuffler_four_flush_groups_with_empty_partitions() { + let dir = TempStrDir::default(); + let output_dir = Path::from(dir.as_ref()); + let num_partitions = 5; + + // Each input batch is flushed independently. Partition 4 is empty in + // every group, while the other partitions exercise empty ranges at the + // beginning, middle, and end of individual groups. + let batch1 = make_batch(&[0, 2], &[10, 20], None); + let batch2 = make_batch(&[1, 3], &[30, 40], None); + let batch3 = make_batch(&[0, 3], &[50, 60], None); + let batch4 = make_batch(&[3], &[70], None); + + let shuffler = + TwoFileShuffler::new(output_dir.clone(), num_partitions).with_batch_size_bytes(1); + let reader = shuffler + .shuffle(batches_to_stream(vec![batch1, batch2, batch3, batch4])) + .await + .unwrap(); + + let expected = [vec![10, 50], vec![30], vec![20], vec![40, 60, 70]]; + for (partition_id, expected_values) in expected.iter().enumerate() { + assert_eq!( + reader.partition_size(partition_id).unwrap(), + expected_values.len() + ); + let partition = collect_partition(reader.as_ref(), partition_id) + .await + .unwrap(); + let values: &Int32Array = partition["val"].as_primitive(); + assert_eq!(values.values(), expected_values); + } + assert_eq!(reader.partition_size(4).unwrap(), 0); + assert!(reader.read_partition(4).await.unwrap().is_none()); + + // Force the bounded-memory fallback and verify its u64 range reads + // produce the same partition order and empty-partition behavior. + let fallback_reader = TwoFileShuffleReader::try_new_with_preload_limit( + Arc::new(ObjectStore::local()), + output_dir, + num_partitions, + 4, + vec![2, 1, 1, 3, 0], + 0.0, + 0, + ) + .await + .unwrap(); + for (partition_id, expected_values) in expected.iter().enumerate() { + let partition = collect_partition(fallback_reader.as_ref(), partition_id) + .await + .unwrap(); + let values: &Int32Array = partition["val"].as_primitive(); + assert_eq!(values.values(), expected_values); + } + assert!(fallback_reader.read_partition(4).await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_partition_windows_match_singletons_with_hotspot_and_empty_boundaries() { + let dir = TempStrDir::default(); + let output_dir = Path::from(dir.as_ref()); + let num_partitions = 6; + + // Four flush groups, with empty partitions at the beginning, middle, + // and end. Partition 2 is deliberately much larger than its neighbors. + let batch1 = make_batch(&[1, 2, 2, 2], &[10, 20, 21, 22], None); + let batch2 = make_batch(&[2, 2, 4], &[23, 24, 40], None); + let batch3 = make_batch(&[1, 2, 2], &[11, 25, 26], None); + let batch4 = make_batch(&[2, 2, 2], &[27, 28, 29], None); + let reader = TwoFileShuffler::new(output_dir, num_partitions) + .with_batch_size_bytes(1) + .shuffle(batches_to_stream(vec![batch1, batch2, batch3, batch4])) + .await + .unwrap(); + + let mut singleton_values = Vec::with_capacity(num_partitions); + for partition_id in 0..num_partitions { + let values = match reader.read_partition(partition_id).await.unwrap() { + Some(stream) => collect_values(stream).await, + None => Vec::new(), + }; + singleton_values.push(values); + } + + // The decoded schema is one Int32 (4 bytes). A 12-byte target fits the + // first empty + two-row partition, while the ten-row hotspot is forced + // into a singleton window. + let mut next_partition_id = 0; + let mut ranges = Vec::new(); + let mut admission_bytes = Vec::new(); + let mut window_values = vec![Vec::new(); num_partitions]; + while next_partition_id < num_partitions { + let plan = reader.plan_partition_window(next_partition_id, 12).unwrap(); + let window = reader + .read_partition_window(next_partition_id, 12) + .await + .unwrap(); + assert_eq!(window.partition_range, plan.partition_range); + assert!(window.materialized_decoded_bytes.is_some()); + assert_eq!(window.partitions.len(), window.partition_range.len()); + ranges.push(window.partition_range.clone()); + admission_bytes.push(plan.estimated_decoded_bytes); + next_partition_id = window.partition_range.end; + for partition in window.partitions { + if let Some(stream) = partition.data { + window_values[partition.partition_id] = collect_values(stream).await; + } + } + } + + assert_eq!(ranges, vec![0..2, 2..3, 3..6]); + assert!(admission_bytes[1] > admission_bytes[0]); + assert!(admission_bytes[1] > admission_bytes[2]); + assert_eq!(window_values, singleton_values); + assert_eq!(window_values[0], Vec::::new()); + assert_eq!(window_values[2].len(), 10); + assert_eq!(window_values[5], Vec::::new()); + } + + #[tokio::test] + async fn test_on_demand_offsets_window_falls_back_to_singleton() { + let dir = TempStrDir::default(); + let output_dir = Path::from(dir.as_ref()); + let reader = TwoFileShuffler::new(output_dir.clone(), 3) + .with_batch_size_bytes(1) + .shuffle(batches_to_stream(vec![ + make_batch(&[0, 1], &[10, 20], None), + make_batch(&[1, 2], &[30, 40], None), + ])) + .await + .unwrap(); + drop(reader); + + let fallback_reader = TwoFileShuffleReader::try_new_with_preload_limit( + Arc::new(ObjectStore::local()), + output_dir, + 3, + 2, + vec![1, 2, 1], + 0.0, + 0, + ) + .await + .unwrap(); + let window = fallback_reader + .read_partition_window(1, DEFAULT_PARTITION_WINDOW_BYTES) + .await + .unwrap(); + assert_eq!(window.partition_range, 1..2); + assert_eq!(window.partitions.len(), 1); + assert_eq!(window.partitions[0].partition_id, 1); + } + + #[test] + fn test_window_planning_uses_decoded_bytes_and_isolates_hotspot() { + let partition_counts = [0, 2, 10, 0, 1, 0]; + assert_eq!( + plan_partition_window_end(&partition_counts, 0, 4, 12).unwrap(), + 2 + ); + assert_eq!( + plan_partition_window_end(&partition_counts, 2, 4, 12).unwrap(), + 3 + ); + assert_eq!( + plan_partition_window_end(&partition_counts, 3, 4, 12).unwrap(), + 6 + ); + + let error = plan_partition_window_end(&partition_counts, 0, 4, 0).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("must be greater than 0")); + } + + #[tokio::test] + async fn legacy_shuffler_uses_schema_estimate_for_parallel_admission() { + let dir = TempStrDir::default(); + let output_dir = Path::from(dir.as_ref()); + let part_ids = vec![0; 32]; + let values = (0..32).collect::>(); + let reader = IvfShuffler::new(output_dir, 2) + .shuffle(batches_to_stream(vec![make_batch( + &part_ids, &values, None, + )])) + .await + .unwrap(); + + let non_empty = reader.plan_partition_window(0, 128 * 1024 * 1024).unwrap(); + assert_eq!(non_empty.partition_range, 0..1); + assert_eq!( + non_empty.estimated_decoded_bytes, + 32 * 4 + 32 * 4 / 4 + WINDOW_ADMISSION_FIXED_HEADROOM_BYTES + ); + assert_ne!(non_empty.estimated_decoded_bytes, usize::MAX); + + let empty = reader.plan_partition_window(1, 128 * 1024 * 1024).unwrap(); + assert_eq!(empty.partition_range, 1..2); + assert_eq!(empty.estimated_decoded_bytes, 0); + } + + #[test] + fn test_preloaded_window_ranges_coalesce_each_nonempty_flush_group() { + // Three groups x five partitions. Window [1, 4) is empty in the last + // group, so only two ranges are submitted. + let offsets = [1, 3, 3, 4, 4, 4, 4, 6, 7, 8, 9, 9, 9, 9, 10]; + let (ranges, counts) = preloaded_window_ranges(&offsets, 3, 5, 1..4).unwrap(); + assert_eq!(ranges, vec![1..4, 4..7]); + assert_eq!(counts, vec![vec![2, 0, 1], vec![0, 2, 1]]); + } + + #[tokio::test] + async fn test_partition_window_split_rejects_short_and_extra_rows() { + let data = make_batch(&[0, 0, 0, 0, 0], &[10, 20, 30, 40, 50], None) + .drop_column(PART_ID_COLUMN) + .unwrap(); + let group_counts = vec![vec![1, 2], vec![0, 1]]; + + let short_stream = stream::iter(vec![Ok(data.slice(0, 3))]); + let error = split_partition_window_stream(short_stream, 2, &group_counts) + .await + .unwrap_err(); + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("decoded 3 rows for partition window, expected 4"), + "unexpected error: {error}" + ); + + let extra_stream = stream::iter(vec![Ok(data)]); + let error = split_partition_window_stream(extra_stream, 2, &group_counts) + .await + .unwrap_err(); + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("decoded more than the expected 4 rows"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_partition_window_split_propagates_stream_error() { + let injected = Error::io("injected window read failure"); + let error = split_partition_window_stream(stream::iter(vec![Err(injected)]), 1, &[vec![1]]) + .await + .unwrap_err(); + assert!(matches!(error, Error::IO { .. })); + assert!(error.to_string().contains("injected window read failure")); + } + + #[test] + fn test_validate_shuffle_offsets_rejects_truncated_offsets() { + let offsets_path = Path::from("shuffle_offsets.lance"); + let offsets = [2, 2, 3, 3, 3, 6, 6, 7, 8, 8, 8]; + let error = + validate_shuffle_offsets(&offsets, 3, 4, 10, &[3, 3, 1, 3], &offsets_path).unwrap_err(); + + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("decoded 11 offsets, expected 12"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_validate_shuffle_offsets_rejects_non_monotonic_offsets() { + let offsets_path = Path::from("shuffle_offsets.lance"); + let offsets = [2, 2, 3, 3, 3, 2, 6, 7, 8, 8, 8, 10]; + let error = + validate_shuffle_offsets(&offsets, 3, 4, 10, &[3, 3, 1, 3], &offsets_path).unwrap_err(); + + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("offsets are not monotonic at indices 4 and 5: 3 > 2"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_validate_shuffle_offsets_rejects_data_boundary_mismatch() { + let offsets_path = Path::from("shuffle_offsets.lance"); + let offsets = [2, 2, 3, 3, 3, 6, 6, 7, 8, 8, 8, 11]; + let error = + validate_shuffle_offsets(&offsets, 3, 4, 10, &[3, 3, 1, 4], &offsets_path).unwrap_err(); + + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("final offset 11 does not match shuffle data row count 10"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_validate_shuffle_offsets_checks_partition_counts() { + let offsets_path = Path::from("shuffle_offsets.lance"); + let offsets = [2, 2, 3, 3, 3, 6, 6, 7, 8, 8, 8, 10]; + let error = + validate_shuffle_offsets(&offsets, 3, 4, 10, &[3, 2, 1, 4], &offsets_path).unwrap_err(); + + assert!(matches!(error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("offset-derived count 3 for partition 1 does not match expected count 2"), + "unexpected error: {error}" + ); + } + + #[test] + fn test_should_preload_offsets_enforces_byte_limit_and_checks_overflow() { + assert!(should_preload_offsets(4, 4 * std::mem::size_of::()).unwrap()); + assert!(!should_preload_offsets(5, 4 * std::mem::size_of::()).unwrap()); + + let error = should_preload_offsets(usize::MAX, usize::MAX).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("overflows byte-size calculation"), + "unexpected error: {error}" + ); + } + #[tokio::test] async fn test_two_file_shuffler_multi_batch_single_flush() { // All three batches fit within the default batch_size_bytes, so they diff --git a/rust/lance-index/src/vector/v3/subindex.rs b/rust/lance-index/src/vector/v3/subindex.rs index 9e82921347f..cf712c1b246 100644 --- a/rust/lance-index/src/vector/v3/subindex.rs +++ b/rust/lance-index/src/vector/v3/subindex.rs @@ -32,6 +32,15 @@ pub trait IvfSubIndex: Send + Sync + Debug + DeepSizeOf { /// Return the schema of the sub index fn schema() -> arrow_schema::SchemaRef; + /// The subset of [`Self::schema`] that [`Self::load`] actually reads. + /// + /// Index files always carry the full `schema()`, so narrowing the read is + /// purely a storage optimization: it keeps write-only columns from being + /// fetched, without changing what is written. `None` reads every column. + fn read_columns() -> Option<&'static [&'static str]> { + None + } + /// Search the sub index for nearest neighbors. /// # Arguments: /// * `query` - The query vector diff --git a/rust/lance-io/Cargo.toml b/rust/lance-io/Cargo.toml index 6cee04d5fd9..5785125eaf3 100644 --- a/rust/lance-io/Cargo.toml +++ b/rust/lance-io/Cargo.toml @@ -16,18 +16,11 @@ rust-version.workspace = true object_store = { workspace = true } opendal = { workspace = true, optional = true } object_store_opendal = { workspace = true, optional = true } -lance-arrow.workspace = true lance-core.workspace = true lance-namespace.workspace = true arrow = { workspace = true, features = ["ffi"] } -arrow-arith.workspace = true arrow-array.workspace = true -arrow-buffer.workspace = true -arrow-cast.workspace = true -arrow-data.workspace = true arrow-schema.workspace = true -arrow-select.workspace = true -async-recursion.workspace = true async-trait.workspace = true aws-config = { workspace = true, optional = true } aws-credential-types = { workspace = true, optional = true } @@ -37,16 +30,22 @@ chrono.workspace = true futures.workspace = true http.workspace = true log.workspace = true +metrics = { workspace = true, optional = true } moka.workspace = true pin-project.workspace = true prost.workspace = true -serde.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, optional = true } tokio.workspace = true tracing.workspace = true url.workspace = true +uuid.workspace = true path_abs.workspace = true rand.workspace = true tempfile.workspace = true +reqsign-core = { version = "3.2.0", optional = true } +reqsign-file-read-tokio = { version = "3.0.3", optional = true } +reqsign-google = { version = "3.0.3", optional = true } [target.'cfg(target_os = "linux")'.dependencies] io-uring = { workspace = true } @@ -57,9 +56,12 @@ lance-testing.workspace = true test-log.workspace = true mockall.workspace = true rstest.workspace = true +serial_test.workspace = true mock_instant.workspace = true tokio = { workspace = true, features = ["test-util"] } tracing-mock = { workspace = true } +metrics-util = { workspace = true } +wiremock.workspace = true [[bench]] name = "scheduler" @@ -67,9 +69,19 @@ harness = false [features] default = ["aws", "azure", "gcp"] +metrics = ["dep:metrics"] gcs-test = [] goosefs-test = [] -gcp = ["object_store/gcp", "dep:opendal", "opendal/services-gcs", "dep:object_store_opendal"] +gcp = [ + "object_store/gcp", + "dep:opendal", + "opendal/services-gcs", + "dep:object_store_opendal", + "dep:reqsign-core", + "dep:reqsign-file-read-tokio", + "dep:reqsign-google", + "dep:serde_json", +] aws = ["object_store/aws", "dep:aws-config", "dep:aws-credential-types", "dep:opendal", "opendal/services-s3", "dep:object_store_opendal"] azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "opendal/services-azdls", "dep:object_store_opendal"] oss = ["dep:opendal", "opendal/services-oss", "dep:object_store_opendal"] diff --git a/rust/lance-io/src/encodings.rs b/rust/lance-io/src/encodings.rs deleted file mode 100644 index de2737f2366..00000000000 --- a/rust/lance-io/src/encodings.rs +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Data encodings -//! - -use arrow_array::{Array, ArrayRef, UInt32Array}; -use async_trait::async_trait; - -pub mod binary; -pub mod dictionary; -pub mod plain; - -use crate::ReadBatchParams; -use lance_core::Result; - -/// Encoder - Write an arrow array to the file. -#[async_trait] -pub trait Encoder { - /// Write an slice of Arrays, and returns the file offset of the beginning of the batch. - async fn encode(&mut self, array: &[&dyn Array]) -> Result; -} - -/// Decoder - Read Arrow Data. -#[async_trait] -pub trait Decoder: Send + AsyncIndex { - async fn decode(&self) -> Result; - - /// Take by indices. - async fn take(&self, indices: &UInt32Array) -> Result; -} - -#[async_trait] -pub trait AsyncIndex { - type Output: Send + Sync; - - async fn get(&self, index: IndexType) -> Self::Output; -} diff --git a/rust/lance-io/src/lib.rs b/rust/lance-io/src/lib.rs index 2ef686fd551..b6bdc404e39 100644 --- a/rust/lance-io/src/lib.rs +++ b/rust/lance-io/src/lib.rs @@ -11,7 +11,6 @@ use arrow_array::{PrimitiveArray, UInt32Array}; use lance_core::{Error, Result}; -pub mod encodings; pub mod ffi; pub mod local; pub mod object_reader; diff --git a/rust/lance-io/src/local.rs b/rust/lance-io/src/local.rs index 2b8a339331a..91e82c4fd0a 100644 --- a/rust/lance-io/src/local.rs +++ b/rust/lance-io/src/local.rs @@ -3,6 +3,7 @@ //! Optimized local I/Os +use std::collections::HashSet; use std::fs::File; use std::io::{ErrorKind, Read, SeekFrom}; use std::ops::Range; @@ -16,6 +17,7 @@ use std::os::windows::fs::FileExt; use async_trait::async_trait; use bytes::{Bytes, BytesMut}; +use chrono::{DateTime, Utc}; use futures::future::BoxFuture; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; @@ -41,14 +43,134 @@ pub fn to_local_path(path: &Path) -> String { /// Recursively remove a directory, specified by [`object_store::path::Path`]. pub fn remove_dir_all(path: &Path) -> Result<()> { - let local_path = to_local_path(path); - std::fs::remove_dir_all(local_path).map_err(|err| match err.kind() { + std::fs::remove_dir_all(to_local_path(path)).map_err(|err| match err.kind() { ErrorKind::NotFound => Error::not_found(path.to_string()), _ => Error::from(err), })?; Ok(()) } +/// Remove eligible empty directories below `root` without following symbolic links. +pub(crate) fn remove_empty_dirs( + root: &Path, + retained_dirs: &HashSet, + verified_dirs: &HashSet, + unmodified_since: Option>, +) -> Result<()> { + let root_path = std::path::PathBuf::from(to_local_path(root)); + let root_metadata = match std::fs::symlink_metadata(&root_path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(Error::from(err)), + }; + if !root_metadata.file_type().is_dir() { + return Ok(()); + } + let canonical_root = std::fs::canonicalize(&root_path)?; + + let mut pending_dirs = vec![(root_path, root.clone(), None::)]; + let mut discovered_dirs = Vec::new(); + let mut file_bearing_roots = HashSet::new(); + let mut first_error = None; + while let Some((local_dir, object_store_dir, index_root)) = pending_dirs.pop() { + let entries = match std::fs::read_dir(&local_dir) { + Ok(entries) => entries, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => return Err(Error::from(err)), + }; + for entry in entries { + let entry = entry?; + // DirEntry::file_type does not follow symbolic links. Non-directories, including + // symlinks, remain in place and prevent their parent from being removed as empty. + if !entry.file_type()?.is_dir() { + if let Some(index_root) = &index_root { + file_bearing_roots.insert(index_root.clone()); + } + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + let child = object_store_dir.clone().join(name); + if retained_dirs.contains(&child) { + continue; + } + let local_child = entry.path(); + let index_root = index_root.clone().unwrap_or_else(|| child.clone()); + // Removing a child changes its parent's mtime, so capture age eligibility before + // deepest-first deletion starts. + let is_old_enough = match std::fs::symlink_metadata(&local_child) { + Ok(metadata) => unmodified_since.is_none_or(|threshold| { + metadata + .modified() + .ok() + .map(DateTime::::from) + .is_some_and(|modified| modified < threshold) + }), + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => { + first_error.get_or_insert_with(|| Error::from(err)); + false + } + }; + discovered_dirs.push(( + local_child.clone(), + child.clone(), + index_root.clone(), + is_old_enough, + )); + pending_dirs.push((local_child, child, Some(index_root))); + } + } + + discovered_dirs.sort_unstable_by_key(|(_, path, _, _)| std::cmp::Reverse(path.parts_count())); + for (local_dir, object_store_dir, index_root, is_old_enough) in discovered_dirs { + if file_bearing_roots.contains(&index_root) { + continue; + } + match std::fs::symlink_metadata(&local_dir) { + Ok(metadata) if metadata.file_type().is_dir() => {} + Ok(_) => continue, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => { + first_error.get_or_insert_with(|| Error::from(err)); + continue; + } + } + let is_verified = verified_dirs.contains(&object_store_dir); + if !is_verified && !is_old_enough { + continue; + } + + // Canonicalization rejects any directory reached through a symlink outside the supplied + // root. Removing the canonical path also avoids trusting prefixes returned by an object + // store listing. + let canonical_dir = match std::fs::canonicalize(&local_dir) { + Ok(path) if path != canonical_root && path.starts_with(&canonical_root) => path, + Ok(_) => continue, + Err(err) if err.kind() == ErrorKind::NotFound => continue, + Err(err) => { + first_error.get_or_insert_with(|| Error::from(err)); + continue; + } + }; + if let Err(err) = std::fs::remove_dir(canonical_dir) + && !matches!( + err.kind(), + ErrorKind::NotFound | ErrorKind::DirectoryNotEmpty + ) + { + first_error.get_or_insert_with(|| Error::from(err)); + } + } + + if let Some(error) = first_error { + Err(error) + } else { + Ok(()) + } +} + /// Copy a file from one location to another, supporting cross-filesystem copies. /// /// Unlike hard links, this function works across filesystem boundaries. @@ -68,6 +190,24 @@ pub fn copy_file(from: &Path, to: &Path) -> Result<()> { Ok(()) } +/// Await a filesystem operation running on a blocking thread, flattening the +/// join and IO errors into a single `object_store` error. +/// +/// Deliberately not written as `handle.await?` at the call sites: a `JoinError` +/// means the operation panicked, and short-circuiting on it would skip the +/// caller's metrics recording for exactly the failure worth counting. +pub(crate) async fn join_local_io( + handle: tokio::task::JoinHandle>, +) -> object_store::Result { + match handle.await { + Ok(result) => result.map_err(|err| object_store::Error::Generic { + store: "LocalFileSystem", + source: err.into(), + }), + Err(err) => Err(err.into()), + } +} + /// Object reader for local file system. #[derive(Debug)] pub struct LocalObjectReader { @@ -165,14 +305,13 @@ impl Reader for LocalObjectReader { let file = self.file.clone(); self.size .get_or_try_init(|| async move { - let metadata = tokio::task::spawn_blocking(move || { - file.metadata().map_err(|err| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - }) - }) - .await??; - Ok(metadata.len() as usize) + // The metadata lookup is this reader's equivalent of the HEAD + // request a cloud reader makes to learn the object size. + let metrics = self.io_tracker.begin_io("head"); + let result = + join_local_io(tokio::task::spawn_blocking(move || file.metadata())).await; + metrics.record(&result, 0); + Ok(result?.len() as usize) }) .await .cloned() @@ -189,7 +328,8 @@ impl Reader for LocalObjectReader { let range_u64 = (range.start as u64)..(range.end as u64); Box::pin(async move { - let result = tokio::task::spawn_blocking(move || { + let metrics = io_tracker.begin_io("get"); + let result = join_local_io(tokio::task::spawn_blocking(move || { let mut buf = BytesMut::with_capacity(range.len()); // Safety: `buf` is set with appropriate capacity above. It is // written to below and we check all data is initialized at that point. @@ -200,13 +340,10 @@ impl Reader for LocalObjectReader { read_exact_at(file, buf.as_mut(), range.start as u64)?; Ok(buf.freeze()) - }) - .await? - .map_err(|err: std::io::Error| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - }); + })) + .await; + metrics.record(&result, num_bytes); if result.is_ok() { io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); } @@ -223,17 +360,16 @@ impl Reader for LocalObjectReader { let io_tracker = self.io_tracker.clone(); let path = self.path.clone(); - let result = tokio::task::spawn_blocking(move || { + let metrics = io_tracker.begin_io("get"); + let result = join_local_io(tokio::task::spawn_blocking(move || { let mut buf = Vec::new(); file.read_to_end(buf.as_mut())?; Ok(Bytes::from(buf)) - }) - .await? - .map_err(|err: std::io::Error| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - }); + })) + .await; + let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64); + metrics.record(&result, num_bytes); if let Ok(bytes) = &result { io_tracker.record_read("get_all", path, bytes.len() as u64, None); } diff --git a/rust/lance-io/src/object_reader.rs b/rust/lance-io/src/object_reader.rs index 1c27800c90f..a781016756a 100644 --- a/rust/lance-io/src/object_reader.rs +++ b/rust/lance-io/src/object_reader.rs @@ -5,6 +5,7 @@ use std::fs::File; use std::ops::Range; use std::sync::Arc; +use crate::local::join_local_io; #[cfg(windows)] use crate::local::read_exact_at; #[cfg(unix)] @@ -70,6 +71,7 @@ pub struct CloudObjectReader { size: OnceCell, block_size: usize, + io_parallelism: usize, download_retry_count: usize, } @@ -94,9 +96,21 @@ impl CloudObjectReader { path, size: OnceCell::new_with(known_size), block_size, + io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM, download_retry_count, }) } + + /// Override the I/O parallelism this reader advertises. + /// + /// `ObjectStore::open` / `open_with_size` pass their normalized effective + /// parallelism (`LANCE_IO_THREADS` override applied, at least 1) so + /// consumers that size concurrency windows off the reader honor the + /// configured request limit instead of the hardcoded cloud default. + pub fn with_io_parallelism(mut self, io_parallelism: usize) -> Self { + self.io_parallelism = io_parallelism; + self + } } // Retries for the initial request are handled by object store, but @@ -169,7 +183,7 @@ impl Reader for CloudObjectReader { } fn io_parallelism(&self) -> usize { - DEFAULT_CLOUD_IO_PARALLELISM + self.io_parallelism } /// Object/File Size. @@ -419,7 +433,9 @@ pub(crate) fn stream_local_range( let next = (start + chunk_size).min(end); let file_clone = file.clone(); let path_clone = path.clone(); - let bytes = tokio::task::spawn_blocking(move || { + let num_bytes = (next - start) as u64; + let metrics = io_tracker.begin_io("get"); + let result = join_local_io(tokio::task::spawn_blocking(move || { let mut buf = bytes::BytesMut::with_capacity(next - start); // Safety: buffer capacity matches the exact number of bytes we read below. unsafe { buf.set_len(next - start) }; @@ -428,17 +444,15 @@ pub(crate) fn stream_local_range( #[cfg(windows)] read_exact_at(file_clone, buf.as_mut(), start as u64)?; Ok::<_, std::io::Error>(buf.freeze()) - }) - .await? - .map_err(|err: std::io::Error| object_store::Error::Generic { - store: "LocalFileSystem", - source: err.into(), - })?; + })) + .await; + metrics.record(&result, num_bytes); + let bytes = result?; io_tracker.record_read( "get_range_stream", path_clone, - (next - start) as u64, + num_bytes, Some(start as u64..next as u64), ); diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs index dafb5e5342f..53a9767fb66 100644 --- a/rust/lance-io/src/object_store.rs +++ b/rust/lance-io/src/object_store.rs @@ -4,13 +4,16 @@ //! Extend [object_store::ObjectStore] functionalities use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ops::Range; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; use std::pin::Pin; use std::str::FromStr; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; +use ::tracing::{Span, field::Empty, instrument}; use async_trait::async_trait; use bytes::Bytes; use chrono::{DateTime, Utc}; @@ -18,15 +21,19 @@ use futures::{FutureExt, Stream}; use futures::{StreamExt, TryStreamExt, future, stream::BoxStream}; use lance_core::deepsize::DeepSizeOf; use lance_core::error::LanceOptionExt; -use lance_core::utils::parse::str_is_truthy; +use lance_core::utils::parse::{parse_env_as_bool, str_is_truthy}; use list_retry::ListRetryStream; use object_store::DynObjectStore; use object_store::ObjectStoreExt as OSObjectStoreExt; #[cfg(feature = "aws")] use object_store::aws::AwsCredentialProvider; +use object_store::list::PaginatedListStore; #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] use object_store::{ClientOptions, HeaderMap, HeaderValue}; -use object_store::{ListResult, ObjectMeta, ObjectStore as OSObjectStore, path::Path}; +use object_store::{ + ListResult, ObjectMeta, ObjectStore as OSObjectStore, PutMode, PutOptions, PutPayload, + path::Path, +}; use providers::local::FileStoreProvider; use providers::memory::MemoryStoreProvider; use tokio::io::AsyncWriteExt; @@ -40,7 +47,21 @@ pub(crate) mod dynamic_credentials; #[cfg(any(feature = "oss", feature = "huggingface", feature = "tos"))] pub(crate) mod dynamic_opendal; mod list_retry; +#[cfg(feature = "metrics")] +pub mod metrics; +#[cfg(any( + feature = "aws", + feature = "gcp", + feature = "azure", + feature = "oss", + feature = "tencent", + feature = "huggingface", + feature = "tos", + feature = "goosefs", +))] +pub(crate) mod opendal_store; pub mod providers; +pub(crate) mod read_dir; pub mod storage_options; #[cfg(test)] pub(crate) mod test_utils; @@ -61,6 +82,8 @@ pub const DEFAULT_LOCAL_IO_PARALLELISM: usize = 8; // Cloud disks often need many many threads to saturate the network pub const DEFAULT_CLOUD_IO_PARALLELISM: usize = 64; +const SERVER_SIDE_COPY_ENABLED_ENV: &str = "LANCE_IO_SERVER_SIDE_COPY_ENABLED"; + const DEFAULT_LOCAL_BLOCK_SIZE: usize = 4 * 1024; // 4KB block size #[cfg(any( feature = "aws", @@ -82,7 +105,46 @@ pub static DEFAULT_MAX_IOP_SIZE: std::sync::LazyLock = std::sync::LazyLock: pub const DEFAULT_DOWNLOAD_RETRY_COUNT: usize = 3; +#[derive(Debug)] +struct StreamCopyError { + stage: &'static str, + source_path: String, + destination_path: String, + source: Box, +} + +impl std::fmt::Display for StreamCopyError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "multipart_stream_copy failed during {} from {} to {}: {}", + self.stage, self.source_path, self.destination_path, self.source + ) + } +} + +impl std::error::Error for StreamCopyError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +fn stream_copy_error( + stage: &'static str, + source_path: &Path, + destination_path: &Path, + source: impl std::error::Error + Send + Sync + 'static, +) -> Error { + Error::io_source(Box::new(StreamCopyError { + stage, + source_path: source_path.to_string(), + destination_path: destination_path.to_string(), + source: Box::new(source), + })) +} + pub use providers::{ObjectStoreProvider, ObjectStoreRegistry}; +pub use read_dir::ReadDirOptions; pub use storage_options::{ BASE_SCOPED_OPTION_PREFIX, BaseScopedStorageOptionsProvider, EXPIRES_AT_MILLIS_KEY, LanceNamespaceStorageOptionsProvider, REFRESH_OFFSET_MILLIS_KEY, StorageOptionsAccessor, @@ -105,6 +167,11 @@ pub trait ObjectStoreExt { ) -> BoxStream<'a, Result>; } +#[async_trait] +pub(super) trait LocalDirOperations: std::fmt::Debug + Send + Sync { + async fn remove_dir_all(&self, path: &Path) -> Result<()>; +} + #[async_trait] impl ObjectStoreExt for O { fn read_dir_all<'a, 'b>( @@ -132,10 +199,12 @@ impl ObjectStoreExt for O { } /// Wraps [ObjectStore](object_store::ObjectStore) -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct ObjectStore { // Inner object store pub inner: Arc, + // Provider-owned native directory operations for rooted local stores. + local_dir_operations: Option>, scheme: String, block_size: usize, max_iop_size: u64, @@ -154,6 +223,31 @@ pub struct ObjectStore { /// which usually cannot be found in the URL such as Azure account name. The prefix plus the /// path uniquely identifies any object inside the store. pub store_prefix: String, + /// The backend's paginated listing API, when it has one. `None` means + /// [`Self::read_dir_page`] has to list a directory in full to page through it. + pub(crate) paginated_lister: Option>, +} + +// Hand-written because `PaginatedListStore` is not `Debug`. +impl std::fmt::Debug for ObjectStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ObjectStore") + .field("inner", &self.inner) + .field("scheme", &self.scheme) + .field("block_size", &self.block_size) + .field("max_iop_size", &self.max_iop_size) + .field( + "use_constant_size_upload_parts", + &self.use_constant_size_upload_parts, + ) + .field("list_is_lexically_ordered", &self.list_is_lexically_ordered) + .field("io_parallelism", &self.io_parallelism) + .field("download_retry_count", &self.download_retry_count) + .field("io_tracker", &self.io_tracker) + .field("store_prefix", &self.store_prefix) + .field("paginated_lister", &self.paginated_lister.is_some()) + .finish() + } } impl DeepSizeOf for ObjectStore { @@ -178,6 +272,34 @@ pub trait WrappingObjectStore: std::fmt::Debug + Send + Sync { /// The store_prefix is a string which uniquely identifies the object /// store being wrapped. fn wrap(&self, store_prefix: &str, original: Arc) -> Arc; + + /// Wrap the paginated listing API that goes with the store, if it has one. + /// + /// [`ObjectStore::read_dir_page`] pushes the page size and the resume position into + /// [`PaginatedListStore`], which is a separate trait from [`OSObjectStore`] and so cannot + /// be reached through the store [`Self::wrap`] returns. A listing that is pushed down + /// therefore does not pass through [`Self::wrap`], and this is where a wrapper says what + /// should happen instead: + /// + /// - `Some(lister)` keeps the pushdown, wrapping the lister or handing back the one + /// given. Right for a wrapper that observes rather than intercepts — metering, caching, + /// mirroring writes. + /// - `None` gives up the pushdown, so listings go through [`Self::wrap`] as a full + /// directory read. Right for a wrapper that hides, rewrites or fails paths, which a + /// pushed-down listing would otherwise walk straight past. + /// + /// A wrapper that keeps the pushdown must leave the listing itself alone: setting + /// [`offset`](object_store::list::PaginatedListOptions::offset) or changing the delimiter + /// breaks paging, since `read_dir_page` reads one directory level and resumes by the token + /// it got back. + /// + /// There is deliberately no default: getting this wrong is either a silent loss of speed + /// or a silent loss of the wrapper, and neither announces itself. + fn wrap_paginated( + &self, + store_prefix: &str, + original: Arc, + ) -> Option>; } #[derive(Debug, Clone)] @@ -201,6 +323,18 @@ impl WrappingObjectStore for ChainedWrappingObjectStore { .iter() .fold(original, |acc, wrapper| wrapper.wrap(store_prefix, acc)) } + + // One wrapper giving up the pushdown gives it up for the chain: the listing has to go + // through `wrap`, which is every wrapper in the chain at once. + fn wrap_paginated( + &self, + store_prefix: &str, + original: Arc, + ) -> Option> { + self.wrappers.iter().try_fold(original, |acc, wrapper| { + wrapper.wrap_paginated(store_prefix, acc) + }) + } } /// Parameters to create an [ObjectStore] @@ -284,6 +418,12 @@ impl ObjectStoreParams { } } +fn wrapper_allocation_ptr(wrapper: &Arc) -> *const () { + // Trait object pointers include vtable metadata, which is not stable across codegen units. + // Cache identity must follow the Arc allocation instead. + Arc::as_ptr(wrapper) as *const () +} + // We implement hash for caching impl std::hash::Hash for ObjectStoreParams { #[allow(deprecated)] @@ -300,7 +440,7 @@ impl std::hash::Hash for ObjectStoreParams { Arc::as_ptr(aws_credentials).hash(state); } if let Some(wrapper) = &self.object_store_wrapper { - Arc::as_ptr(wrapper).hash(state); + wrapper_allocation_ptr(wrapper).hash(state); } if let Some(accessor) = &self.storage_options_accessor { accessor.accessor_id().hash(state); @@ -332,8 +472,14 @@ impl PartialEq for ObjectStoreParams { .as_ref() .map(|(store, url)| (Arc::as_ptr(store), url)) && self.s3_credentials_refresh_offset == other.s3_credentials_refresh_offset - && self.object_store_wrapper.as_ref().map(Arc::as_ptr) - == other.object_store_wrapper.as_ref().map(Arc::as_ptr) + && self + .object_store_wrapper + .as_ref() + .map(wrapper_allocation_ptr) + == other + .object_store_wrapper + .as_ref() + .map(wrapper_allocation_ptr) && self .storage_options_accessor .as_ref() @@ -480,22 +626,48 @@ impl ObjectStore { registry: Arc, uri: &str, params: &ObjectStoreParams, + ) -> Result<(Arc, Path)> { + Self::from_uri_and_params_impl(registry, uri, params, true).await + } + + /// Parse a URI and build a fresh object store outside the registry cache. + /// + /// The caller must retain the returned store for as long as its + /// provider-local state should be reused. + #[doc(hidden)] + pub async fn from_uri_and_params_uncached( + registry: Arc, + uri: &str, + params: &ObjectStoreParams, + ) -> Result<(Arc, Path)> { + Self::from_uri_and_params_impl(registry, uri, params, false).await + } + + async fn from_uri_and_params_impl( + registry: Arc, + uri: &str, + params: &ObjectStoreParams, + use_registry_cache: bool, ) -> Result<(Arc, Path)> { #[allow(deprecated)] if let Some((store, path)) = params.object_store.as_ref() { let mut inner = store.clone(); let store_prefix = registry.calculate_object_store_prefix(uri, params.storage_options())?; + + let mut io_tracker = IOTracker::default(); + meter_store(&mut inner, &mut io_tracker, &store_prefix); + if let Some(wrapper) = params.object_store_wrapper.as_ref() { inner = wrapper.wrap(&store_prefix, inner); } // Always wrap with IO tracking - let io_tracker = IOTracker::default(); let tracked_store = io_tracker.wrap("", inner); let store = Self { inner: tracked_store, + local_dir_operations: None, scheme: path.scheme().to_string(), block_size: params.block_size.unwrap_or(64 * 1024), max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -505,13 +677,19 @@ impl ObjectStore { download_retry_count: DEFAULT_DOWNLOAD_RETRY_COUNT, io_tracker, store_prefix, + // Type-erased on the way in, so there is no telling if it can paginate. + paginated_lister: None, }; let path = Path::parse(path.path())?; return Ok((Arc::new(store), path)); } let url = uri_to_url(uri)?; - let store = registry.get_store(url.clone(), params).await?; + let store = if use_registry_cache { + registry.get_store(url.clone(), params).await? + } else { + registry.new_store(url.clone(), params).await? + }; // We know the scheme is valid if we got a store back. let provider = registry.get_provider(url.scheme()).expect_ok()?; let path = provider.extract_path(&url)?; @@ -576,6 +754,14 @@ impl ObjectStore { self.scheme == "file" || self.scheme == "file+uring" } + /// Returns true when object paths directly encode absolute local filesystem paths. + /// + /// Local stores rooted below the filesystem root, such as UNC-backed stores, use + /// their inner object-store implementation instead of direct filesystem access. + pub fn has_direct_local_paths(&self) -> bool { + self.is_local() && self.store_prefix == self.scheme + } + pub fn is_cloud(&self) -> bool { if self.is_local() || self.scheme == "memory" || self.scheme == "shared-memory" { return false; @@ -641,13 +827,26 @@ impl ObjectStore { self.io_tracker.incremental_stats() } + /// Apply a [`WrappingObjectStore`] to both `inner` and `paginated_lister` together. + /// + /// Keeps both halves in sync: a wrapper returning `None` from + /// [`WrappingObjectStore::wrap_paginated`] clears the lister so that + /// [`Self::read_dir_page`] falls back through the (already-wrapped) `inner`. + pub fn apply_wrapper(&mut self, wrapper: &dyn WrappingObjectStore) { + self.inner = wrapper.wrap(&self.store_prefix, self.inner.clone()); + self.paginated_lister = self + .paginated_lister + .take() + .and_then(|lister| wrapper.wrap_paginated(&self.store_prefix, lister)); + } + /// Open a file for path. /// /// Parameters /// - ``path``: Absolute path to the file. pub async fn open(&self, path: &Path) -> Result> { match self.scheme.as_str() { - "file" => { + "file" if self.has_direct_local_paths() => { LocalObjectReader::open_with_tracker( path, self.block_size, @@ -681,13 +880,16 @@ impl ObjectStore { .await } } - _ => Ok(Box::new(CloudObjectReader::new( - self.inner.clone(), - path.clone(), - self.block_size, - None, - self.download_retry_count, - )?)), + _ => Ok(Box::new( + CloudObjectReader::new( + self.inner.clone(), + path.clone(), + self.block_size, + None, + self.download_retry_count, + )? + .with_io_parallelism(self.io_parallelism()), + )), } } @@ -709,7 +911,7 @@ impl ObjectStore { } match self.scheme.as_str() { - "file" => { + "file" if self.has_direct_local_paths() => { LocalObjectReader::open_with_tracker( path, self.block_size, @@ -743,13 +945,16 @@ impl ObjectStore { .await } } - _ => Ok(Box::new(CloudObjectReader::new( - self.inner.clone(), - path.clone(), - self.block_size, - Some(known_size), - self.download_retry_count, - )?)), + _ => Ok(Box::new( + CloudObjectReader::new( + self.inner.clone(), + path.clone(), + self.block_size, + Some(known_size), + self.download_retry_count, + )? + .with_io_parallelism(self.io_parallelism()), + )), } } @@ -772,7 +977,7 @@ impl ObjectStore { /// Create a new file. pub async fn create(&self, path: &Path) -> Result> { match self.scheme.as_str() { - "file" => { + "file" if self.has_direct_local_paths() => { let local_path = super::local::to_local_path(path); let local_path = std::path::PathBuf::from(&local_path); if let Some(parent) = local_path.parent() { @@ -782,10 +987,19 @@ impl ObjectStore { .parent() .expect("file path must have parent") .to_owned(); - let named_temp = - tokio::task::spawn_blocking(move || tempfile::NamedTempFile::new_in(parent)) - .await - .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))??; + let named_temp = tokio::task::spawn_blocking(move || { + #[cfg(unix)] + { + // NamedTempFile defaults to 0o600. Use ordinary file creation permissions so the published file honors the caller's umask. + tempfile::Builder::new() + .permissions(std::fs::Permissions::from_mode(0o666)) + .tempfile_in(parent) + } + #[cfg(not(unix))] + tempfile::NamedTempFile::new_in(parent) + }) + .await + .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))??; let (std_file, temp_path) = named_temp.into_parts(); let file = tokio::fs::File::from_std(std_file); Ok(Box::new(LocalWriter::new( @@ -806,6 +1020,57 @@ impl ObjectStore { Writer::shutdown(writer.as_mut()).await } + /// Atomically creates an object without replacing an existing object. + /// + /// Local stores publish a uniquely named staging object with a conditional + /// rename. Other stores use their conditional create operation. Tencent COS + /// is rejected because it can silently ignore conditional create requests. + /// + /// Returns [`object_store::Error::NotSupported`] without writing when the + /// backend cannot reliably provide put-if-absent semantics. + pub async fn put_if_absent( + &self, + path: &Path, + content: PutPayload, + ) -> object_store::Result<()> { + if self.scheme == "cos" { + return Err(object_store::Error::NotSupported { + source: "Tencent COS does not reliably enforce put-if-absent after bucket \ + versioning has ever been enabled" + .into(), + }); + } + + if self.is_local() { + let staging_path = + Path::from(format!("{}.tmp.{}", path, uuid::Uuid::new_v4().simple())); + self.inner.put(&staging_path, content).await?; + let result = self.inner.rename_if_not_exists(&staging_path, path).await; + if result.is_err() + && let Err(error) = self.inner.delete(&staging_path).await + { + log::warn!( + "Failed to remove staging object {} after atomic create failed: {}", + staging_path, + error + ); + } + result + } else { + self.inner + .put_opts( + path, + content, + PutOptions { + mode: PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| ()) + } + } + pub async fn delete(&self, path: &Path) -> Result<()> { self.inner.delete(path).await?; Ok(()) @@ -831,6 +1096,334 @@ impl ObjectStore { .await } + /// Copy an object using the policy for bulk file movement. + /// + /// Streaming is the default because it works across object stores and does + /// not require provider-native copy support. Setting + /// `LANCE_IO_SERVER_SIDE_COPY_ENABLED` to a truthy value opts same-store + /// copies into [`Self::copy`]. Cross-store and local copies continue to use + /// [`Self::copy_via_stream`]. + /// + /// ```no_run + /// # use lance_core::Result; + /// # use lance_io::object_store::ObjectStore; + /// # use object_store::path::Path; + /// # async fn copy(source: &ObjectStore, destination: &ObjectStore) -> Result<()> { + /// source + /// .copy_bulk( + /// &Path::from("staging/index.lance"), + /// destination, + /// &Path::from("index.lance"), + /// ) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn copy_bulk( + &self, + source_path: &Path, + destination_store: &Self, + destination_path: &Path, + ) -> Result { + self.copy_bulk_with_server_side_copy( + source_path, + destination_store, + destination_path, + self.uses_server_side_copy(destination_store), + ) + .await + } + + fn uses_server_side_copy(&self, destination_store: &Self) -> bool { + parse_env_as_bool(SERVER_SIDE_COPY_ENABLED_ENV, false) + && self.can_server_side_copy_to(destination_store) + } + + async fn copy_bulk_with_server_side_copy( + &self, + source_path: &Path, + destination_store: &Self, + destination_path: &Path, + server_side_copy_enabled: bool, + ) -> Result { + if !server_side_copy_enabled || !self.can_server_side_copy_to(destination_store) { + return self + .copy_via_stream(source_path, destination_store, destination_path) + .await; + } + + let source_size = self.size(source_path).await?; + let result_size = usize::try_from(source_size).map_err(|source| { + Error::io(format!( + "server-side copy source size conversion failed from {source_path} to \ + {destination_path}: source_size={source_size}, error={source}" + )) + })?; + destination_store + .copy(source_path, destination_path) + .await?; + let destination_size = destination_store.size(destination_path).await?; + if destination_size != source_size { + return Err(Error::io(format!( + "server-side copy destination size mismatch from {source_path} to \ + {destination_path}: source_size={source_size}, \ + destination_size={destination_size}" + ))); + } + + Ok(WriteResult { + size: result_size, + e_tag: None, + }) + } + + fn can_server_side_copy_to(&self, destination_store: &Self) -> bool { + // Prefixes can collide across endpoints or wrappers, where native copy could + // read or write the wrong backend. Exact client identity is required. + self.is_cloud() + && destination_store.is_cloud() + && Arc::ptr_eq(&self.inner, &destination_store.inner) + } + + /// Copy an object by streaming its bytes through Lance's multipart-aware writer. + /// + /// Unlike [`Self::copy`], this never delegates to a provider-native server-side + /// copy. The source and destination may use different object stores. The copy + /// succeeds only after the byte count reported by the writer and a destination + /// metadata lookup both match the source size. + /// + /// ```no_run + /// # use lance_core::Result; + /// # use lance_io::object_store::ObjectStore; + /// # use object_store::path::Path; + /// # async fn copy(source: &ObjectStore, destination: &ObjectStore) -> Result<()> { + /// source + /// .copy_via_stream( + /// &Path::from("staging/index.lance"), + /// destination, + /// &Path::from("index.lance"), + /// ) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + #[instrument( + name = "multipart_stream_copy", + level = "info", + skip(self, source_path, destination_store, destination_path), + fields( + source = %source_path, + destination = %destination_path, + source_size = Empty, + read_chunk_size = Empty, + multipart_part_size = crate::object_writer::initial_upload_size(), + multipart_concurrency = crate::object_writer::max_upload_parallelism(), + part_count = Empty, + bytes_transferred = Empty, + destination_size = Empty, + validation = Empty, + elapsed_ms = Empty, + ), + err + )] + pub async fn copy_via_stream( + &self, + source_path: &Path, + destination_store: &Self, + destination_path: &Path, + ) -> Result { + let started_at = Instant::now(); + if self.has_direct_local_paths() && destination_store.has_direct_local_paths() { + let source_size = std::fs::metadata(super::local::to_local_path(source_path)) + .map_err(|source| { + let source = if source.kind() == std::io::ErrorKind::NotFound { + Error::not_found(source_path.to_string()) + } else { + Error::from(source) + }; + stream_copy_error("source metadata", source_path, destination_path, source) + })? + .len(); + let source_size = usize::try_from(source_size).map_err(|source| { + stream_copy_error( + "source size conversion", + source_path, + destination_path, + source, + ) + })?; + Span::current().record("source_size", source_size as u64); + + let metrics = destination_store.io_tracker.begin_io("copy"); + let result = super::local::copy_file(source_path, destination_path); + metrics.record(&result, source_size as u64); + result.map_err(|source| { + stream_copy_error( + "local filesystem copy", + source_path, + destination_path, + source, + ) + })?; + + let destination_size = + destination_store + .size(destination_path) + .await + .map_err(|source| { + stream_copy_error( + "destination validation", + source_path, + destination_path, + source, + ) + })?; + Span::current().record("bytes_transferred", source_size as u64); + Span::current().record("destination_size", destination_size); + if destination_size != source_size as u64 { + Span::current().record("validation", "failed"); + return Err(Error::io(format!( + "multipart_stream_copy destination size mismatch from {source_path} to \ + {destination_path}: source_size={source_size}, \ + destination_size={destination_size}" + ))); + } + + Span::current().record("validation", "passed"); + Span::current().record("elapsed_ms", started_at.elapsed().as_millis() as u64); + return Ok(WriteResult { + size: source_size, + e_tag: None, + }); + } + + let reader = self.open(source_path).await.map_err(|source| { + stream_copy_error("source open", source_path, destination_path, source) + })?; + let source_size = reader.size().await.map_err(|source| { + stream_copy_error("source metadata", source_path, destination_path, source) + })?; + Span::current().record("source_size", source_size as u64); + + let mut writer = destination_store + .create(destination_path) + .await + .map_err(|source| { + stream_copy_error( + "destination writer creation", + source_path, + destination_path, + source, + ) + })?; + let read_chunk_size = usize::try_from(self.max_iop_size()) + .unwrap_or(usize::MAX) + .max(1); + Span::current().record("read_chunk_size", read_chunk_size as u64); + let mut bytes_transferred = 0usize; + if source_size > 0 { + let first_range = 0..read_chunk_size.min(source_size); + let mut current_range = first_range.clone(); + let mut current_bytes = reader.get_range(first_range).await.map_err(|source| { + stream_copy_error("source read", source_path, destination_path, source) + })?; + + loop { + let expected_bytes = current_range.len(); + if current_bytes.len() != expected_bytes { + Span::current().record("validation", "failed"); + return Err(Error::io(format!( + "multipart_stream_copy source range size mismatch from {source_path} to \ + {destination_path}: range={current_range:?}, \ + expected_bytes={expected_bytes}, actual_bytes={}", + current_bytes.len() + ))); + } + bytes_transferred = bytes_transferred + .checked_add(current_bytes.len()) + .ok_or_else(|| { + Error::io(format!( + "multipart_stream_copy byte count overflow from {source_path} to \ + {destination_path}" + )) + })?; + + if bytes_transferred == source_size { + writer.write_all(¤t_bytes).await.map_err(|source| { + stream_copy_error( + "destination write", + source_path, + destination_path, + source, + ) + })?; + break; + } + + let range_end = bytes_transferred + .checked_add(read_chunk_size) + .unwrap_or(source_size) + .min(source_size); + let next_range = bytes_transferred..range_end; + let next_read = reader.get_range(next_range.clone()); + let (write_result, next_bytes) = + tokio::join!(writer.write_all(¤t_bytes), next_read); + write_result.map_err(|source| { + stream_copy_error("destination write", source_path, destination_path, source) + })?; + current_bytes = next_bytes.map_err(|source| { + stream_copy_error("source read", source_path, destination_path, source) + })?; + current_range = next_range; + } + } + Span::current().record("bytes_transferred", bytes_transferred as u64); + + let write_result = Writer::shutdown(writer.as_mut()).await.map_err(|source| { + stream_copy_error( + "destination completion", + source_path, + destination_path, + source, + ) + })?; + if write_result.size != source_size { + Span::current().record("validation", "failed"); + return Err(Error::io(format!( + "multipart_stream_copy writer size mismatch from {source_path} to \ + {destination_path}: source_size={source_size}, \ + writer_size={}", + write_result.size + ))); + } + + let destination_size = + destination_store + .size(destination_path) + .await + .map_err(|source| { + stream_copy_error( + "destination validation", + source_path, + destination_path, + source, + ) + })?; + Span::current().record("destination_size", destination_size); + if destination_size != source_size as u64 { + Span::current().record("validation", "failed"); + return Err(Error::io(format!( + "multipart_stream_copy destination size mismatch from {source_path} to \ + {destination_path}: source_size={source_size}, \ + destination_size={destination_size}" + ))); + } + + Span::current().record("validation", "passed"); + Span::current().record("elapsed_ms", started_at.elapsed().as_millis() as u64); + Ok(write_result) + } + /// Copy `from` to `to`. When `multipart_copy_fallback` is set, a source /// larger than `max_single_copy` is streamed through a multipart write /// instead of a single-shot server-side copy. Both are parameters so tests @@ -843,9 +1436,12 @@ impl ObjectStore { multipart_copy_fallback: bool, max_single_copy: u64, ) -> Result<()> { - if self.is_local() { + if self.has_direct_local_paths() { // Use std::fs::copy for local filesystem to support cross-filesystem copies - return super::local::copy_file(from, to); + let metrics = self.io_tracker.begin_io("copy"); + let result = super::local::copy_file(from, to); + metrics.record(&result, 0); + return result; } if multipart_copy_fallback { // Reuse the reader for both the size lookup (a single cached HEAD) @@ -862,6 +1458,9 @@ impl ObjectStore { } /// Read a directory (start from base directory) and returns all sub-paths in the directory. + /// + /// This enumerates the whole prefix before it returns, however many children it holds. + /// Use [`Self::read_dir_page`] to page through a directory instead. pub async fn read_dir(&self, dir_path: impl Into) -> Result> { let path = dir_path.into(); let path = Path::parse(&path)?; @@ -907,9 +1506,20 @@ impl ObjectStore { let path = dir_path.into(); let path = Path::parse(&path)?; - if self.is_local() { + if let Some(local_dir_operations) = &self.local_dir_operations { + let metrics = self.io_tracker.begin_io("delete"); + let result = local_dir_operations.remove_dir_all(&path).await; + metrics.record(&result, 0); + return result; + } + if self.has_direct_local_paths() { // The local file system provider needs to delete both files and directories. - return super::local::remove_dir_all(&path); + // Counted as a single delete request, matching how `delete_stream` + // counts one batched request regardless of how many paths it removes. + let metrics = self.io_tracker.begin_io("delete"); + let result = super::local::remove_dir_all(&path); + metrics.record(&result, 0); + return result; } let sub_entries = self .inner @@ -928,6 +1538,53 @@ impl ObjectStore { Ok(()) } + /// Remove eligible materialized empty directories below a local root. + /// + /// This is a no-op for object stores, which do not materialize directories. + /// Traversal does not follow symbolic links. Directories in `retained_dirs` and their + /// descendants are preserved. Other directories are removed only if they are empty and + /// either appear in `verified_dirs` or predate `unmodified_since`. Passing `None` for + /// `unmodified_since` disables the age check. + /// + /// ``` + /// # use std::collections::HashSet; + /// # use chrono::Utc; + /// # use lance_core::Result; + /// # use lance_io::object_store::ObjectStore; + /// # async fn remove_stale_index_dirs(store: &ObjectStore) -> Result<()> { + /// store + /// .remove_empty_dirs( + /// "dataset/_indices", + /// HashSet::new(), + /// HashSet::new(), + /// Some(Utc::now()), + /// ) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn remove_empty_dirs( + &self, + root_path: impl Into, + retained_dirs: HashSet, + verified_dirs: HashSet, + unmodified_since: Option>, + ) -> Result<()> { + if !self.has_direct_local_paths() && self.scheme != "file-object-store" { + return Ok(()); + } + + let path = Path::parse(root_path.into())?; + let metrics = self.io_tracker.begin_io("delete"); + let result = tokio::task::spawn_blocking(move || { + super::local::remove_empty_dirs(&path, &retained_dirs, &verified_dirs, unmodified_since) + }) + .await + .map_err(|error| Error::io(format!("empty-directory cleanup task failed: {error}")))?; + metrics.record(&result, 0); + result + } + pub fn remove_stream<'a>( &'a self, locations: BoxStream<'a, Result>, @@ -1107,7 +1764,7 @@ static DEFAULT_OBJECT_STORE_REGISTRY: std::sync::LazyLock = impl ObjectStore { #[allow(clippy::too_many_arguments)] pub fn new( - store: Arc, + mut store: Arc, location: Url, block_size: Option, wrapper: Option>, @@ -1132,17 +1789,20 @@ impl ObjectStore { store_prefix } }; + let mut io_tracker = IOTracker::default(); + meter_store(&mut store, &mut io_tracker, &store_prefix); + let store = match wrapper { Some(wrapper) => wrapper.wrap(&store_prefix, store), None => store, }; // Always wrap with IO tracking - let io_tracker = IOTracker::default(); let tracked_store = io_tracker.wrap("", store); Self { inner: tracked_store, + local_dir_operations: None, scheme: scheme.into(), block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -1152,10 +1812,35 @@ impl ObjectStore { download_retry_count, io_tracker, store_prefix, + // Type-erased on the way in, so there is no telling if it can paginate. + paginated_lister: None, } } } +/// Wrap `inner` so its operations publish metrics labelled by `store_prefix`, +/// and label `io_tracker` with the same prefix so the local reads and writes +/// that bypass `inner` publish under it too. +/// +/// The two go together on purpose: a store metered on one path but not the other +/// would report a partial picture that reads like a complete one. Every +/// constructor that hands an [`ObjectStore`] to a caller must route its `inner` +/// through here, or through nothing at all. +#[cfg(feature = "metrics")] +fn meter_store(inner: &mut Arc, io_tracker: &mut IOTracker, store_prefix: &str) { + use crate::object_store::metrics::ObjectStoreMetricsExt; + io_tracker.set_metrics_base(store_prefix); + *inner = inner.clone().metered(store_prefix.to_owned()); +} + +#[cfg(not(feature = "metrics"))] +fn meter_store( + _inner: &mut Arc, + _io_tracker: &mut IOTracker, + _store_prefix: &str, +) { +} + fn infer_block_size(scheme: &str) -> usize { // Block size: On local file systems, we use 4KB block size. On cloud // object stores, we use 64KB block size. This is generally the largest @@ -1175,15 +1860,16 @@ mod tests { use object_store::memory::InMemory; use object_store::{ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, PutMultipartOptions, - PutOptions, PutPayload, PutResult, Result as OSResult, + PutOptions, PutPayload, PutResult, Result as OSResult, UploadPart, }; use rstest::rstest; + use serial_test::serial; use std::env::set_current_dir; use std::fmt::{Display, Formatter}; use std::fs::{create_dir_all, write}; use std::ops::Range; use std::path::Path as StdPath; - use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; /// Write test content to file. fn write_to_file(path_str: &str, contents: &str) -> std::io::Result<()> { @@ -1200,11 +1886,54 @@ mod tests { Ok(contents) } - #[test] - fn test_io_parallelism_clamped_to_nonzero() { + #[tokio::test] + async fn test_put_if_absent() { + let temp_dir = TempStrDir::default(); + let path = Path::from(format!("{}/atomic-create", temp_dir.as_str())); + let store = ObjectStore::local(); + store + .put_if_absent(&path, Bytes::from_static(b"first").into()) + .await + .unwrap(); + let error = store + .put_if_absent(&path, Bytes::from_static(b"second").into()) + .await + .unwrap_err(); + assert!(matches!( + error, + object_store::Error::AlreadyExists { .. } | object_store::Error::Precondition { .. } + )); + assert_eq!( + store.read_one_all(&path).await.unwrap(), + b"first".as_slice() + ); + } + + #[tokio::test] + async fn test_put_if_absent_rejects_cos() { + let mut store = ObjectStore::memory(); + store.scheme = "cos".to_string(); + let path = Path::from("atomic-create"); + + let error = store + .put_if_absent(&path, Bytes::from_static(b"value").into()) + .await + .unwrap_err(); + + assert!(matches!(error, object_store::Error::NotSupported { .. })); + assert!(!store.exists(&path).await.unwrap()); + } + + #[tokio::test] + async fn test_io_parallelism_clamped_to_nonzero() { // `io_parallelism()` feeds `buffered`/`buffer_unordered` windows; a value of 0 makes those // streams never poll, hanging callers (e.g. a metadata-only `count_rows`). It must clamp. let store = ObjectStore::local(); + // Readers opened by the store must advertise the store's normalized + // effective parallelism, not the hardcoded cloud default. + let mem_store = ObjectStore::memory(); + let path = Path::from("/io_parallelism_probe"); + mem_store.put(&path, b"x").await.unwrap(); // SAFETY: process-global env var, set and restored within this test. `io_parallelism()` // only reads it, and a concurrent reader observes a valid clamped value, never 0. @@ -1214,6 +1943,11 @@ mod tests { 1, "LANCE_IO_THREADS=0 must clamp to 1" ); + assert_eq!( + mem_store.open(&path).await.unwrap().io_parallelism(), + 1, + "an opened reader must report the store's clamped parallelism" + ); unsafe { std::env::set_var("LANCE_IO_THREADS", "8") }; assert_eq!( @@ -1221,6 +1955,20 @@ mod tests { 8, "a positive override must pass through unchanged" ); + assert_eq!( + mem_store.open(&path).await.unwrap().io_parallelism(), + 8, + "an opened reader must honor the configured request limit" + ); + assert_eq!( + mem_store + .open_with_size(&path, 1024 * 1024) + .await + .unwrap() + .io_parallelism(), + 8, + "a sized reader must honor the configured request limit" + ); unsafe { std::env::remove_var("LANCE_IO_THREADS") }; assert!( @@ -1433,6 +2181,84 @@ mod tests { assert!(!path.join("foo").exists()); } + #[rstest] + #[case("file")] + #[case("file-object-store")] + #[tokio::test] + async fn test_remove_empty_directories(#[case] scheme: &str) { + let path = TempStdDir::default(); + let stale_dir = path.join("stale"); + let nested_stale_dir = path.join("nested_stale"); + let nested_stale_child = nested_stale_dir.join("child"); + create_dir_all(&stale_dir).unwrap(); + create_dir_all(&nested_stale_child).unwrap(); + create_dir_all(path.join("retained").join("child")).unwrap(); + write_to_file( + path.join("file_bearing") + .join("test_file") + .to_str() + .unwrap(), + "keep", + ) + .unwrap(); + create_dir_all(path.join("file_bearing").join("empty_child")).unwrap(); + + let file_url = Url::from_directory_path(&path).unwrap(); + let mut url = Url::parse(&format!("{scheme}:///")).unwrap(); + url.set_path(file_url.path()); + let (store, base) = ObjectStore::from_uri(url.as_ref()).await.unwrap(); + + #[cfg(unix)] + let unmodified_since = { + let old_modified_time = + std::time::SystemTime::now() - std::time::Duration::from_secs(10 * 24 * 60 * 60); + for directory in [&stale_dir, &nested_stale_dir, &nested_stale_child] { + std::fs::File::open(directory) + .unwrap() + .set_times(std::fs::FileTimes::new().set_modified(old_modified_time)) + .unwrap(); + } + DateTime::::from(std::time::SystemTime::now()) + - chrono::TimeDelta::try_days(7).unwrap() + }; + #[cfg(not(unix))] + let unmodified_since = DateTime::::from(std::time::SystemTime::now()) + + chrono::TimeDelta::try_days(1).unwrap(); + + store + .remove_empty_dirs( + base.clone(), + HashSet::from([base.clone().join("retained")]), + HashSet::new(), + Some(unmodified_since), + ) + .await + .unwrap(); + + assert!(!path.join("stale").exists()); + assert!(!path.join("nested_stale").exists()); + assert!(path.join("retained").join("child").exists()); + assert!(path.join("file_bearing").join("empty_child").exists()); + + create_dir_all(path.join("fresh")).unwrap(); + create_dir_all(path.join("verified")).unwrap(); + store + .remove_empty_dirs( + base.clone(), + HashSet::from([base.clone().join("retained")]), + HashSet::from([base.clone().join("verified")]), + Some( + DateTime::::from(std::time::SystemTime::now()) + - chrono::TimeDelta::try_days(7).unwrap(), + ), + ) + .await + .unwrap(); + + assert!(path.join("fresh").exists()); + assert!(!path.join("verified").exists()); + } + #[derive(Debug)] struct TestWrapper { called: AtomicBool, @@ -1451,6 +2277,16 @@ mod tests { // return a mocked value so we can check if the final store is the one we expect self.return_value.clone() } + + // This one swaps the store out entirely, so a listing that went around it would be + // listing something else. + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } impl TestWrapper { @@ -1459,15 +2295,162 @@ mod tests { } } - #[tokio::test] - async fn test_wrapping_object_store_option_is_used() { - // Make a store for the inner store first - let mock_inner_store: Arc = Arc::new(InMemory::new()); - let registry = Arc::new(ObjectStoreRegistry::default()); + /// A lister that exists only to be wrapped. + #[derive(Debug)] + struct StubLister; - assert_eq!(Arc::strong_count(&mock_inner_store), 1); + #[async_trait] + impl PaginatedListStore for StubLister { + async fn list_paginated( + &self, + _prefix: Option<&str>, + _opts: object_store::list::PaginatedListOptions, + ) -> object_store::Result { + unimplemented!("this lister exists to be wrapped, not to list") + } + } - let wrapper = Arc::new(TestWrapper { + /// Records the listers it was handed, and leaves the store alone. + #[derive(Debug)] + struct PaginatedTestWrapper { + name: &'static str, + log: Arc>>, + } + + impl WrappingObjectStore for PaginatedTestWrapper { + fn wrap( + &self, + _store_prefix: &str, + original: Arc, + ) -> Arc { + original + } + + fn wrap_paginated( + &self, + store_prefix: &str, + original: Arc, + ) -> Option> { + self.log + .lock() + .unwrap() + .push(format!("{}@{store_prefix}", self.name)); + Some(original) + } + } + + /// A chain hands the lister to each of its wrappers in turn. One wrapper giving up the + /// pushdown gives it up for the chain, and the wrappers after it are never asked: the + /// listing is going through `wrap` either way, which is every wrapper at once. + #[rstest] + #[case::every_wrapper_keeps_it(false, vec!["first@memory", "second@memory"])] + #[case::one_wrapper_gives_it_up(true, vec!["first@memory"])] + fn test_a_chain_wraps_the_lister_until_one_gives_it_up( + #[case] gives_up: bool, + #[case] expected_log: Vec<&str>, + ) { + let log = Arc::new(std::sync::Mutex::new(Vec::new())); + let mut wrappers: Vec> = + vec![Arc::new(PaginatedTestWrapper { + name: "first", + log: log.clone(), + })]; + if gives_up { + wrappers.push(Arc::new(TestWrapper { + called: AtomicBool::new(false), + return_value: Arc::new(InMemory::new()), + })); + } + wrappers.push(Arc::new(PaginatedTestWrapper { + name: "second", + log: log.clone(), + })); + + let wrapped = ChainedWrappingObjectStore::new(wrappers) + .wrap_paginated("memory", Arc::new(StubLister)); + + assert_eq!(wrapped.is_none(), gives_up); + assert_eq!(*log.lock().unwrap(), expected_log); + } + + /// `apply_wrapper` keeps both halves of the store in sync. A wrapper that gives up the + /// pushdown has to clear the lister too, or `read_dir_page` would keep talking to the + /// backend behind the wrapper's back. + #[rstest] + #[case::gives_up_the_pushdown(true)] + #[case::keeps_the_pushdown(false)] + fn test_apply_wrapper_keeps_inner_and_the_lister_in_sync(#[case] gives_up: bool) { + let replacement = Arc::new(InMemory::new()); + let giving_up = TestWrapper { + called: AtomicBool::new(false), + return_value: replacement.clone(), + }; + let keeping = PaginatedTestWrapper { + name: "passthrough", + log: Arc::new(std::sync::Mutex::new(Vec::new())), + }; + let wrapper: &dyn WrappingObjectStore = match gives_up { + true => &giving_up, + false => &keeping, + }; + + let mut store = ObjectStore::memory(); + store.paginated_lister = Some(Arc::new(StubLister) as Arc); + store.apply_wrapper(wrapper); + + assert_eq!( + store.paginated_lister.is_some(), + !gives_up, + "the lister has to follow what the wrapper said" + ); + // The wrapper that gives up the pushdown is also the one that swaps the store out, so + // whether `inner` was replaced says that `wrap` ran on the same wrapper. + assert_eq!( + Arc::ptr_eq(&store.inner, &(replacement as Arc)), + gives_up + ); + } + + #[tokio::test] + async fn test_wrapper_identity_is_stable_across_tasks() { + let wrapper = Arc::new(TestWrapper { + called: AtomicBool::new(false), + return_value: Arc::new(InMemory::new()), + }); + let initial_params = ObjectStoreParams { + object_store_wrapper: Some(wrapper.clone()), + ..ObjectStoreParams::default() + }; + let task_params = tokio::spawn(async move { + ObjectStoreParams { + object_store_wrapper: Some(wrapper), + ..ObjectStoreParams::default() + } + }) + .await + .unwrap(); + + assert_eq!(initial_params, task_params); + + let mut initial_hasher = std::hash::DefaultHasher::new(); + std::hash::Hash::hash(&initial_params, &mut initial_hasher); + let mut task_hasher = std::hash::DefaultHasher::new(); + std::hash::Hash::hash(&task_params, &mut task_hasher); + assert_eq!( + std::hash::Hasher::finish(&initial_hasher), + std::hash::Hasher::finish(&task_hasher) + ); + } + + #[tokio::test] + async fn test_wrapping_object_store_option_is_used() { + // Make a store for the inner store first + let mock_inner_store: Arc = Arc::new(InMemory::new()); + let registry = Arc::new(ObjectStoreRegistry::default()); + + assert_eq!(Arc::strong_count(&mock_inner_store), 1); + + let wrapper = Arc::new(TestWrapper { called: AtomicBool::new(false), return_value: mock_inner_store.clone(), }); @@ -1504,6 +2487,29 @@ mod tests { assert_eq!(buf.as_ref(), b"LOCAL"); } + #[cfg(unix)] + #[tokio::test] + async fn test_direct_local_writer_uses_standard_file_permissions() { + let directory = TempStdDir::default(); + let reference_path = directory.join("reference"); + std::fs::File::create(&reference_path).unwrap(); + let expected_mode = std::fs::metadata(reference_path) + .unwrap() + .permissions() + .mode() + & 0o777; + + let output_path = directory.join("output"); + let object_path = Path::from_absolute_path(&output_path).unwrap(); + let store = ObjectStore::local(); + let mut writer = store.create(&object_path).await.unwrap(); + writer.write_all(b"LOCAL").await.unwrap(); + Writer::shutdown(writer.as_mut()).await.unwrap(); + + let actual_mode = std::fs::metadata(output_path).unwrap().permissions().mode() & 0o777; + assert_eq!(actual_mode, expected_mode); + } + #[tokio::test] async fn test_read_one() { let file_path = TempStdFile::default(); @@ -1644,6 +2650,131 @@ mod tests { } } + #[derive(Debug, Default)] + struct MultipartObservations { + part_count: AtomicUsize, + abort_count: AtomicUsize, + native_copy_count: AtomicUsize, + } + + #[derive(Debug)] + struct ObservedMultipartUpload { + inner: Box, + observations: Arc, + fail_parts: bool, + } + + #[async_trait] + impl MultipartUpload for ObservedMultipartUpload { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + self.observations.part_count.fetch_add(1, Ordering::SeqCst); + if self.fail_parts { + return Box::pin(async { + Err(object_store::Error::Generic { + store: "ObservedMultipartStore", + source: "injected multipart part failure".into(), + }) + }); + } + self.inner.put_part(data) + } + + async fn complete(&mut self) -> OSResult { + self.inner.complete().await + } + + async fn abort(&mut self) -> OSResult<()> { + self.observations.abort_count.fetch_add(1, Ordering::SeqCst); + self.inner.abort().await + } + } + + #[derive(Debug)] + struct ObservedMultipartStore { + inner: InMemory, + observations: Arc, + fail_parts: bool, + destination_size_adjustment: u64, + } + + impl Display for ObservedMultipartStore { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "ObservedMultipartStore") + } + } + + #[async_trait] + impl OSObjectStore for ObservedMultipartStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + self.inner.put_opts(location, bytes, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + let inner = self.inner.put_multipart_opts(location, opts).await?; + Ok(Box::new(ObservedMultipartUpload { + inner, + observations: self.observations.clone(), + fail_parts: self.fail_parts, + })) + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + let is_head = options.head; + let mut result = self.inner.get_opts(location, options).await?; + if is_head && location.filename() == Some("destination.bin") { + result.meta.size = result + .meta + .size + .checked_add(self.destination_size_adjustment) + .expect("test destination size should not overflow"); + } + Ok(result) + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + self.inner.get_ranges(location, ranges).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + self.inner.list(prefix) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, OSResult> { + self.inner.list_with_offset(prefix, offset) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> { + self.observations + .native_copy_count + .fetch_add(1, Ordering::SeqCst); + self.inner.copy_opts(from, to, opts).await + } + } + #[async_trait] impl OSObjectStore for CopyFailingStore { async fn put_opts( @@ -1729,6 +2860,373 @@ mod tests { ); } + #[tokio::test] + async fn test_copy_via_stream_never_uses_native_copy() { + let mut store = ObjectStore::memory(); + store.inner = Arc::new(CopyFailingStore { + inner: InMemory::new(), + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = b"stream raw bytes instead of issuing native copy"; + store.put(&source, contents).await.unwrap(); + + let result = store + .copy_via_stream(&source, &store, &destination) + .await + .unwrap(); + + assert_eq!(result.size, contents.len()); + assert_eq!( + store.read_one_all(&destination).await.unwrap().as_ref(), + contents + ); + } + + #[tokio::test] + async fn test_bulk_copy_streams_when_server_side_copy_is_disabled() { + let observations = Arc::new(MultipartObservations::default()); + let mut store = ObjectStore::memory(); + store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: observations.clone(), + fail_parts: false, + destination_size_adjustment: 0, + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = b"stream by default"; + store.put(&source, contents).await.unwrap(); + + let result = store + .copy_bulk_with_server_side_copy(&source, &store, &destination, false) + .await + .unwrap(); + + assert_eq!(result.size, contents.len()); + assert_eq!(observations.native_copy_count.load(Ordering::SeqCst), 0); + assert_eq!( + store.read_one_all(&destination).await.unwrap().as_ref(), + contents + ); + } + + #[test] + #[serial(server_side_copy_env)] + fn test_server_side_copy_environment_policy() { + let previous_value = std::env::var_os(SERVER_SIDE_COPY_ENABLED_ENV); + let mut store = ObjectStore::memory(); + store.scheme = "test-cloud".to_string(); + let destination_store = store.clone(); + + // SAFETY: this serialized test is the only test that mutates this task-specific + // environment variable, and it restores the original value before returning. + unsafe { std::env::remove_var(SERVER_SIDE_COPY_ENABLED_ENV) }; + assert!(!store.uses_server_side_copy(&destination_store)); + + // SAFETY: see the serialized-test guarantee above. + unsafe { std::env::set_var(SERVER_SIDE_COPY_ENABLED_ENV, "true") }; + assert!(store.uses_server_side_copy(&destination_store)); + + // SAFETY: restore the process environment before the test returns. + unsafe { + match previous_value { + Some(value) => std::env::set_var(SERVER_SIDE_COPY_ENABLED_ENV, value), + None => std::env::remove_var(SERVER_SIDE_COPY_ENABLED_ENV), + } + } + } + + #[tokio::test] + async fn test_bulk_copy_uses_server_side_copy_when_enabled_for_same_store() { + let observations = Arc::new(MultipartObservations::default()); + let mut source_store = ObjectStore::memory(); + source_store.scheme = "test-cloud".to_string(); + source_store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: observations.clone(), + fail_parts: false, + destination_size_adjustment: 0, + }); + let destination_store = source_store.clone(); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = b"use native copy when explicitly enabled"; + source_store.put(&source, contents).await.unwrap(); + + let result = source_store + .copy_bulk_with_server_side_copy(&source, &destination_store, &destination, true) + .await + .unwrap(); + + assert_eq!(result.size, contents.len()); + assert_eq!(observations.native_copy_count.load(Ordering::SeqCst), 1); + assert_eq!( + destination_store + .read_one_all(&destination) + .await + .unwrap() + .as_ref(), + contents + ); + } + + #[tokio::test] + async fn test_bulk_copy_streams_for_distinct_clients_with_same_prefix() { + let shared_inner = InMemory::new(); + let source_observations = Arc::new(MultipartObservations::default()); + let mut source_store = ObjectStore::memory(); + source_store.scheme = "test-cloud".to_string(); + source_store.store_prefix = "test-cloud$bucket".to_string(); + source_store.inner = Arc::new(ObservedMultipartStore { + inner: shared_inner.clone(), + observations: source_observations.clone(), + fail_parts: false, + destination_size_adjustment: 0, + }); + let destination_observations = Arc::new(MultipartObservations::default()); + let mut destination_store = ObjectStore::memory(); + destination_store.scheme = "test-cloud".to_string(); + destination_store.store_prefix = "test-cloud$bucket".to_string(); + destination_store.inner = Arc::new(ObservedMultipartStore { + inner: shared_inner, + observations: destination_observations.clone(), + fail_parts: false, + destination_size_adjustment: 0, + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = b"use native copy when explicitly enabled"; + source_store.put(&source, contents).await.unwrap(); + + let result = source_store + .copy_bulk_with_server_side_copy(&source, &destination_store, &destination, true) + .await + .unwrap(); + + assert_eq!(result.size, contents.len()); + assert_eq!( + source_observations.native_copy_count.load(Ordering::SeqCst), + 0 + ); + assert_eq!( + destination_observations + .native_copy_count + .load(Ordering::SeqCst), + 0 + ); + assert_eq!( + destination_store + .read_one_all(&destination) + .await + .unwrap() + .as_ref(), + contents + ); + } + + #[tokio::test] + async fn test_bulk_copy_rejects_server_side_destination_size_mismatch() { + let observations = Arc::new(MultipartObservations::default()); + let mut source_store = ObjectStore::memory(); + source_store.scheme = "test-cloud".to_string(); + source_store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: observations.clone(), + fail_parts: false, + destination_size_adjustment: 1, + }); + let destination_store = source_store.clone(); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + source_store + .put(&source, b"validate native copy") + .await + .unwrap(); + + let error = source_store + .copy_bulk_with_server_side_copy(&source, &destination_store, &destination, true) + .await + .unwrap_err(); + + assert!( + error.to_string().contains("destination size mismatch"), + "expected validation failure, got: {error}" + ); + assert_eq!(observations.native_copy_count.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn test_bulk_copy_streams_across_stores_when_server_side_copy_is_enabled() { + let source_store = ObjectStore::memory(); + let observations = Arc::new(MultipartObservations::default()); + let mut destination_store = ObjectStore::memory(); + destination_store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: observations.clone(), + fail_parts: false, + destination_size_adjustment: 0, + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = b"cross-store copies must stream"; + source_store.put(&source, contents).await.unwrap(); + + let result = source_store + .copy_bulk_with_server_side_copy(&source, &destination_store, &destination, true) + .await + .unwrap(); + + assert_eq!(result.size, contents.len()); + assert_eq!(observations.native_copy_count.load(Ordering::SeqCst), 0); + assert_eq!( + destination_store + .read_one_all(&destination) + .await + .unwrap() + .as_ref(), + contents + ); + } + + #[tokio::test] + async fn test_copy_via_stream_preserves_local_not_found() { + let directory = TempStdDir::default(); + let (store, base_path) = ObjectStore::from_uri(directory.to_str().unwrap()) + .await + .unwrap(); + let source = base_path.clone().join("missing.bin"); + let destination = base_path.join("destination.bin"); + + let error = store + .copy_via_stream(&source, &store, &destination) + .await + .unwrap_err(); + + assert!( + error.is_not_found(), + "expected not-found error, got: {error}" + ); + } + + #[tokio::test] + async fn test_copy_via_stream_uses_multiple_parts() { + let mut source_store = ObjectStore::memory(); + source_store.max_iop_size = 1024 * 1024; + let observations = Arc::new(MultipartObservations::default()); + let mut destination_store = ObjectStore::memory(); + destination_store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: observations.clone(), + fail_parts: false, + destination_size_adjustment: 0, + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = vec![42; crate::object_writer::initial_upload_size() * 2 + 1]; + source_store.put(&source, &contents).await.unwrap(); + + let result = source_store + .copy_via_stream(&source, &destination_store, &destination) + .await + .unwrap(); + + assert_eq!(result.size, contents.len()); + assert!( + observations.part_count.load(Ordering::SeqCst) >= 2, + "stream copy should split a large destination into multiple upload parts" + ); + assert_eq!( + destination_store + .read_one_all(&destination) + .await + .unwrap() + .as_ref(), + contents.as_slice() + ); + } + + #[tokio::test] + async fn test_copy_via_stream_aborts_failed_upload_and_retains_source() { + let source_store = ObjectStore::memory(); + let observations = Arc::new(MultipartObservations::default()); + let mut destination_store = ObjectStore::memory(); + destination_store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: observations.clone(), + fail_parts: true, + destination_size_adjustment: 0, + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = vec![7; crate::object_writer::initial_upload_size() * 2]; + source_store.put(&source, &contents).await.unwrap(); + + let error = source_store + .copy_via_stream(&source, &destination_store, &destination) + .await + .unwrap_err(); + let error_message = error.to_string(); + assert!( + (error_message.contains("destination write") + || error_message.contains("destination completion")) + && error_message.contains("injected multipart part failure"), + "expected upload-stage context and the underlying error, got: {error}" + ); + + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if observations.abort_count.load(Ordering::SeqCst) > 0 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("multipart abort should complete"); + assert_eq!(observations.abort_count.load(Ordering::SeqCst), 1); + assert_eq!( + source_store.read_one_all(&source).await.unwrap().as_ref(), + contents.as_slice() + ); + assert!(!destination_store.exists(&destination).await.unwrap()); + } + + #[tokio::test] + async fn test_copy_via_stream_rejects_destination_size_mismatch() { + let source_store = ObjectStore::memory(); + let mut destination_store = ObjectStore::memory(); + destination_store.inner = Arc::new(ObservedMultipartStore { + inner: InMemory::new(), + observations: Arc::new(MultipartObservations::default()), + fail_parts: false, + destination_size_adjustment: 1, + }); + + let source = Path::from("source.bin"); + let destination = Path::from("destination.bin"); + let contents = b"validate the destination after completion"; + source_store.put(&source, contents).await.unwrap(); + + let error = source_store + .copy_via_stream(&source, &destination_store, &destination) + .await + .unwrap_err(); + + assert!( + error.to_string().contains("destination size mismatch"), + "expected validation failure, got: {error}" + ); + } + #[test] #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] fn test_client_options_extracts_headers() { diff --git a/rust/lance-io/src/object_store/dynamic_opendal.rs b/rust/lance-io/src/object_store/dynamic_opendal.rs index 3367eacd507..50b15d2180b 100644 --- a/rust/lance-io/src/object_store/dynamic_opendal.rs +++ b/rust/lance-io/src/object_store/dynamic_opendal.rs @@ -14,10 +14,10 @@ use object_store::{ ObjectStore as OSObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, }; -use object_store_opendal::OpendalStore; use tokio::sync::RwLock; use crate::object_store::StorageOptionsAccessor; +use crate::object_store::opendal_store::OpendalStore; use lance_core::Result; type NormalizeConfigFn = fn(&HashMap) -> Result>; @@ -278,7 +278,7 @@ mod tests { "Failed to create memory operator: {e:?}" )) })?; - Ok(OpendalStore::new(operator.finish())) + Ok(OpendalStore::new(operator)) }, ); @@ -316,7 +316,7 @@ mod tests { "Failed to create memory operator: {e:?}" )) })?; - Ok(OpendalStore::new(operator.finish())) + Ok(OpendalStore::new(operator)) }, ) .with_protected_keys(["bucket", "root"]); diff --git a/rust/lance-io/src/object_store/metrics.rs b/rust/lance-io/src/object_store/metrics.rs new file mode 100644 index 00000000000..de00c82ffaa --- /dev/null +++ b/rust/lance-io/src/object_store/metrics.rs @@ -0,0 +1,1821 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Publishes object store metrics via the [`metrics`] crate. +//! +//! Two layers cooperate: +//! +//! * [`MeteredObjectStore`] wraps any [`object_store::ObjectStore`] and records +//! per-operation request counts, transferred bytes, latency, errors, and the +//! number of requests currently in flight. It works for every store +//! regardless of backend. +//! * [`MeteringHttpConnector`] wraps the HTTP client used by the native cloud +//! stores (S3 / GCS / Azure) and records throttle / retryable responses per +//! attempt. Because `object_store`'s retry loop re-issues each request +//! through the [`HttpService`](object_store::client::HttpService), this sees +//! every retried response, which a store-level wrapper cannot observe. +//! +//! The two layers have different coverage: every store gets the request-level +//! metrics from [`MeteredObjectStore`], but only the native cloud stores get +//! the HTTP-level throttle metrics. Opendal-backed stores (tos, oss, etc.) +//! bypass `object_store`'s HTTP client, so there is no place to install the +//! connector for them. +//! +//! Neither layer sees the optimized local reads and writes ([`LocalObjectReader`], +//! [`LocalWriter`], the io_uring readers, and the local `copy` / recursive delete +//! shortcuts), which go straight to the filesystem. Those publish the same +//! request-level metrics themselves through +//! [`IOTracker::begin_io`](crate::utils::tracking_store::IOTracker::begin_io). +//! The two are installed together, so a store either publishes for all of its +//! IO or for none of it. A store built by calling a provider's `new_store` +//! directly, bypassing both `ObjectStore` constructors — as +//! [`ObjectStore::local`](crate::object_store::ObjectStore::local) and +//! [`ObjectStore::memory`](crate::object_store::ObjectStore::memory) do — is in +//! the "none of it" case. +//! +//! [`LocalObjectReader`]: crate::local::LocalObjectReader +//! [`LocalWriter`]: crate::object_writer::LocalWriter +//! +//! Metrics carry a `base` label identifying the store. Its cardinality is +//! controlled by the `LANCE_OBJECT_STORE_METRICS_LABEL` environment variable +//! ([`BASE_LABEL_ENV_VAR`]): +//! +//! * `scheme` (default) — scheme only, e.g. `s3`; low, bounded cardinality. +//! * `full` — the full store prefix, e.g. `s3$bucket` or `az$container@account`, +//! so multiple buckets on the same cloud can be told apart. +//! * `off` — omit the `base` label entirely. +//! +//! The metric name constants ([`METRIC_REQUESTS`] etc.) and the recording +//! helpers ([`record_request`], [`record_count`], [`record_error`], +//! [`InFlightGuard`]) are public so custom object stores can emit the same +//! metrics. + +use std::ops::Range; +use std::pin::Pin; +use std::sync::{Arc, OnceLock}; +use std::task::{Context, Poll}; +use std::time::Instant; + +use bytes::Bytes; +use futures::stream::BoxStream; +use futures::{FutureExt, Stream, StreamExt}; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, GetResultPayload, ListResult, MultipartUpload, ObjectMeta, + PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult, + UploadPart, +}; + +/// Total number of object store requests, labelled by `operation` and `base`. +pub const METRIC_REQUESTS: &str = "lance_object_store_requests_total"; +/// Total bytes transferred by object store requests, labelled by `operation` and `base`. +pub const METRIC_BYTES: &str = "lance_object_store_request_bytes_total"; +/// Object store request latency in seconds, labelled by `operation` and `base`. +pub const METRIC_DURATION: &str = "lance_object_store_request_duration_seconds"; +/// Total number of failed object store requests, labelled by `operation` and `base`. +pub const METRIC_ERRORS: &str = "lance_object_store_errors_total"; +/// Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer, +/// labelled by `status` and `base`. Counts every attempt, including retries. +pub const METRIC_THROTTLE: &str = "lance_object_store_throttle_total"; +/// Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP +/// layer, labelled by `status` and `base`. Counts every attempt, including +/// retries. This is a superset of [`METRIC_THROTTLE`]; 409 (conflict) is +/// deliberately excluded so commit conflicts are not counted as retries. +pub const METRIC_RETRYABLE: &str = "lance_object_store_retryable_responses_total"; +/// Number of object store requests currently in flight, labelled by `operation` +/// and `base`. +pub const METRIC_IN_FLIGHT: &str = "lance_object_store_in_flight_requests"; + +/// Environment variable controlling the cardinality of the `base` label. +pub const BASE_LABEL_ENV_VAR: &str = "LANCE_OBJECT_STORE_METRICS_LABEL"; + +/// Controls how much of a store's identity the `base` label carries, traded off +/// against metric cardinality. Selected via [`BASE_LABEL_ENV_VAR`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BaseLabelMode { + /// Full store prefix, e.g. `s3$bucket` or `az$container@account`. Highest + /// cardinality: one series family per bucket/container. + Full, + /// Scheme only, e.g. `s3`. The default: low, bounded cardinality. + Scheme, + /// Omit the `base` label entirely. + Off, +} + +fn parse_base_label_mode(value: Option<&str>) -> BaseLabelMode { + match value { + Some("full") => BaseLabelMode::Full, + Some("off") | Some("none") => BaseLabelMode::Off, + Some("scheme") | None => BaseLabelMode::Scheme, + Some(other) => { + tracing::warn!( + "Unrecognized {BASE_LABEL_ENV_VAR}={other:?}; \ + expected one of full, scheme, off. Defaulting to scheme." + ); + BaseLabelMode::Scheme + } + } +} + +/// The label mode is read once from the environment and cached for the process. +fn base_label_mode() -> BaseLabelMode { + static MODE: OnceLock = OnceLock::new(); + *MODE.get_or_init(|| parse_base_label_mode(std::env::var(BASE_LABEL_ENV_VAR).ok().as_deref())) +} + +/// Reduce a full store prefix (`scheme$authority`, or just `scheme` for stores +/// without buckets) to the configured `base` label value, or `None` when the +/// label should be omitted. +fn scoped_base(mode: BaseLabelMode, base: &str) -> Option { + match mode { + BaseLabelMode::Full => Some(base.to_owned()), + BaseLabelMode::Scheme => Some(base.split('$').next().unwrap_or(base).to_owned()), + BaseLabelMode::Off => None, + } +} + +/// Build the `operation` (+ optional `base`) label set shared by all +/// store-level metrics, honoring the configured label mode. +fn operation_labels(base: &str, operation: &'static str) -> Vec { + let mut labels = vec![metrics::Label::new("operation", operation)]; + if let Some(base) = scoped_base(base_label_mode(), base) { + labels.push(metrics::Label::new("base", base)); + } + labels +} + +/// Recommended histogram bucket boundaries for [`METRIC_DURATION`], in seconds. +/// +/// Object store requests can take anywhere from a few milliseconds to the +/// client timeout (commonly ~120s), so the boundaries are dense below 10s and +/// keep useful resolution through the timeout band out to 5 minutes. Exporters +/// that aggregate into fixed buckets (e.g. the OpenTelemetry bridge in the +/// Python bindings) use these. +pub const REQUEST_DURATION_BOUNDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, // sub-10s + 10.0, 20.0, 30.0, 45.0, 60.0, 90.0, 120.0, 150.0, 180.0, 240.0, 300.0, // 10s–5min +]; + +/// Register descriptions (units and help text) for the object store metrics. +/// +/// This routes through whatever [`metrics::Recorder`] is currently installed, +/// so it must be called *after* the recorder is set. Exporters that build a +/// catalog of available metrics (such as the OpenTelemetry bridge) rely on +/// these descriptions to discover metric names, kinds, and units up front. +pub fn describe_metrics() { + metrics::describe_counter!( + METRIC_REQUESTS, + metrics::Unit::Count, + "Total number of object store requests, by operation and scheme." + ); + metrics::describe_counter!( + METRIC_BYTES, + metrics::Unit::Bytes, + "Total bytes transferred by object store requests, by operation and scheme." + ); + metrics::describe_histogram!( + METRIC_DURATION, + metrics::Unit::Seconds, + "Object store request latency in seconds, by operation and scheme." + ); + metrics::describe_counter!( + METRIC_ERRORS, + metrics::Unit::Count, + "Total number of failed object store requests, by operation and scheme." + ); + metrics::describe_counter!( + METRIC_THROTTLE, + metrics::Unit::Count, + "Total number of throttle responses (HTTP 429 / 503) seen at the HTTP layer, by status and scheme." + ); + metrics::describe_counter!( + METRIC_RETRYABLE, + metrics::Unit::Count, + "Total number of retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, by status and scheme." + ); + metrics::describe_gauge!( + METRIC_IN_FLIGHT, + metrics::Unit::Count, + "Number of object store requests currently in flight, by operation and scheme." + ); +} + +/// Recommended fixed bucket boundaries for the histogram metrics defined here, +/// as `(metric_name, boundaries)` pairs. Exporters that aggregate histograms +/// into fixed buckets read this to configure each histogram. +pub fn histogram_bounds() -> &'static [(&'static str, &'static [f64])] { + &[(METRIC_DURATION, REQUEST_DURATION_BOUNDS)] +} + +/// Record the outcome of a unary request: count, latency, bytes (on success), and errors. +pub fn record_request( + base: &str, + operation: &'static str, + start: Instant, + bytes: u64, + result: &OSResult, +) { + record_outcome(base, operation, start, bytes, result.is_err()); +} + +/// Record count, latency, and either transferred bytes or an error for a +/// completed request. Used both for unary requests and for streamed GETs whose +/// bytes are only known once the body finishes. +pub fn record_outcome( + base: &str, + operation: &'static str, + start: Instant, + bytes: u64, + is_error: bool, +) { + let elapsed = start.elapsed().as_secs_f64(); + let labels = operation_labels(base, operation); + metrics::counter!(METRIC_REQUESTS, labels.clone()).increment(1); + metrics::histogram!(METRIC_DURATION, labels.clone()).record(elapsed); + if is_error { + metrics::counter!(METRIC_ERRORS, labels).increment(1); + } else if bytes > 0 { + metrics::counter!(METRIC_BYTES, labels).increment(bytes); + } +} + +/// Record a single request count without latency, used for streaming operations +/// (list / delete) whose work happens lazily as the stream is polled. +pub fn record_count(base: &str, operation: &'static str) { + metrics::counter!(METRIC_REQUESTS, operation_labels(base, operation)).increment(1); +} + +/// Record a single error for an operation. +pub fn record_error(base: &str, operation: &'static str) { + metrics::counter!(METRIC_ERRORS, operation_labels(base, operation)).increment(1); +} + +/// Raises the in-flight gauge for an operation on creation and lowers it on +/// drop, so the count stays balanced even if the request future or stream is +/// cancelled or dropped before completing. +pub struct InFlightGuard { + labels: Vec, +} + +impl InFlightGuard { + pub fn new(base: &str, operation: &'static str) -> Self { + let labels = operation_labels(base, operation); + metrics::gauge!(METRIC_IN_FLIGHT, labels.clone()).increment(1.0); + Self { labels } + } +} + +impl Drop for InFlightGuard { + fn drop(&mut self) { + metrics::gauge!(METRIC_IN_FLIGHT, self.labels.clone()).decrement(1.0); + } +} + +#[derive(Debug)] +pub struct MeteredObjectStore { + target: Arc, + base: String, +} + +impl std::fmt::Display for MeteredObjectStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "MeteredObjectStore({})", self.target) + } +} + +#[async_trait::async_trait] +#[deny(clippy::missing_trait_methods)] +impl object_store::ObjectStore for MeteredObjectStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + let size = bytes.content_length() as u64; + let _in_flight = InFlightGuard::new(&self.base, "put"); + let start = Instant::now(); + let result = self.target.put_opts(location, bytes, opts).await; + record_request(&self.base, "put", start, size, &result); + result + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + let upload = self.target.put_multipart_opts(location, opts).await?; + Ok(Box::new(MeteredMultipartUpload { + target: upload, + base: self.base.clone(), + })) + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + // `head()` is implemented as a `get_opts` call with `head = true`, so we + // distinguish it here to keep HEAD and GET as separate operations. + let is_head = options.head; + let operation = if is_head { "head" } else { "get" }; + let in_flight = InFlightGuard::new(&self.base, operation); + let start = Instant::now(); + let result = self.target.get_opts(location, options).await; + + // A HEAD transfers only metadata, and errors carry no payload, so both + // are recorded immediately. `get_opts` only resolves once the response + // headers arrive; the body is streamed afterwards, so for a successful + // GET we defer recording until the body has been drained (see below). + if is_head || result.is_err() { + record_request(&self.base, operation, start, 0, &result); + return result; + } + + let result = result.expect("checked to be Ok above"); + Ok(meter_get_result( + result, + self.base.clone(), + start, + in_flight, + )) + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + let _in_flight = InFlightGuard::new(&self.base, "get"); + let start = Instant::now(); + let result = self.target.get_ranges(location, ranges).await; + let bytes = match &result { + Ok(parts) => parts.iter().map(|b| b.len() as u64).sum(), + Err(_) => 0, + }; + record_request(&self.base, "get", start, bytes, &result); + result + } + + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + let base = self.base.clone(); + // Count one logical delete request per call, matching `list`: a single + // `delete_stream` maps to one batched request on stores that support it + // (e.g. S3's `DeleteObjects`), so counting per yielded path would + // over-count. Errors are still recorded per failing path. + record_count(&self.base, "delete"); + let in_flight = InFlightGuard::new(&self.base, "delete"); + self.target + .delete_stream(locations) + .map(move |result| { + // Reference `in_flight` so this `move` closure captures (owns) + // the guard, keeping the gauge raised until the stream is + // dropped (a move closure only captures the variables it uses). + let _in_flight = &in_flight; + if result.is_err() { + record_error(&base, "delete"); + } + result + }) + .boxed() + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + record_count(&self.base, "list"); + meter_list_stream( + self.target.list(prefix), + self.base.clone(), + InFlightGuard::new(&self.base, "list"), + ) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, OSResult> { + record_count(&self.base, "list"); + meter_list_stream( + self.target.list_with_offset(prefix, offset), + self.base.clone(), + InFlightGuard::new(&self.base, "list"), + ) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + let _in_flight = InFlightGuard::new(&self.base, "list"); + let start = Instant::now(); + let result = self.target.list_with_delimiter(prefix).await; + record_request(&self.base, "list", start, 0, &result); + result + } + + async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> { + let _in_flight = InFlightGuard::new(&self.base, "copy"); + let start = Instant::now(); + let result = self.target.copy_opts(from, to, opts).await; + record_request(&self.base, "copy", start, 0, &result); + result + } + + async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> { + let _in_flight = InFlightGuard::new(&self.base, "rename"); + let start = Instant::now(); + let result = self.target.rename_opts(from, to, opts).await; + record_request(&self.base, "rename", start, 0, &result); + result + } +} + +/// Count errors yielded while draining a list stream. The request itself is +/// counted once when the stream is created (a single LIST may return many items). +fn meter_list_stream( + stream: BoxStream<'static, OSResult>, + base: String, + in_flight: InFlightGuard, +) -> BoxStream<'static, OSResult> { + stream + .map(move |result| { + // Reference `in_flight` so this `move` closure captures (owns) the + // guard: a move closure only captures the variables it uses, and + // holding it here keeps the gauge raised until the stream is dropped. + let _in_flight = &in_flight; + if result.is_err() { + record_error(&base, "list"); + } + result + }) + .boxed() +} + +/// Wrap a successful GET so the request is recorded once its body has been +/// fully read, capturing the true transfer duration and byte count rather than +/// the time-to-first-byte and declared range. For payloads without a body +/// stream (e.g. a local file handle) the request is recorded immediately. +fn meter_get_result( + mut result: GetResult, + base: String, + start: Instant, + in_flight: InFlightGuard, +) -> GetResult { + match result.payload { + GetResultPayload::Stream(stream) => { + result.payload = GetResultPayload::Stream( + MeteredGetStream { + inner: stream, + base, + start, + bytes: 0, + errored: false, + recorded: false, + _in_flight: in_flight, + } + .boxed(), + ); + result + } + // No body stream to observe (e.g. a local file), so record now. + other => { + let bytes = result.range.end - result.range.start; + record_outcome(&base, "get", start, bytes, false); + result.payload = other; + result + } + } +} + +/// Stream wrapper over a GET body that records the request (count, duration, +/// bytes, errors) once the body is fully drained or the stream is dropped. +struct MeteredGetStream { + inner: BoxStream<'static, OSResult>, + base: String, + start: Instant, + bytes: u64, + errored: bool, + recorded: bool, + _in_flight: InFlightGuard, +} + +impl MeteredGetStream { + fn record(&mut self) { + if self.recorded { + return; + } + self.recorded = true; + record_outcome(&self.base, "get", self.start, self.bytes, self.errored); + } +} + +impl Stream for MeteredGetStream { + type Item = OSResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.inner.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(chunk))) => { + self.bytes += chunk.len() as u64; + Poll::Ready(Some(Ok(chunk))) + } + Poll::Ready(Some(Err(e))) => { + self.errored = true; + Poll::Ready(Some(Err(e))) + } + Poll::Ready(None) => { + self.record(); + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } +} + +impl Drop for MeteredGetStream { + fn drop(&mut self) { + // Records the partial transfer if the body was dropped before it drained. + self.record(); + } +} + +#[derive(Debug)] +struct MeteredMultipartUpload { + target: Box, + base: String, +} + +#[async_trait::async_trait] +impl MultipartUpload for MeteredMultipartUpload { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + // Each part upload is a distinct request, recorded under the `put_part` + // operation with the same count / bytes / latency / error set as a + // unary put. + let base = self.base.clone(); + let size = data.content_length() as u64; + let inner = self.target.put_part(data); + async move { + let _in_flight = InFlightGuard::new(&base, "put_part"); + let start = Instant::now(); + let result = inner.await; + record_request(&base, "put_part", start, size, &result); + result + } + .boxed() + } + + async fn complete(&mut self) -> OSResult { + // Completing a multipart upload issues its own request that can throttle + // or fail, so it is metered like any other operation. + let _in_flight = InFlightGuard::new(&self.base, "complete_multipart"); + let start = Instant::now(); + let result = self.target.complete().await; + record_request(&self.base, "complete_multipart", start, 0, &result); + result + } + + async fn abort(&mut self) -> OSResult<()> { + let _in_flight = InFlightGuard::new(&self.base, "abort_multipart"); + let start = Instant::now(); + let result = self.target.abort().await; + record_request(&self.base, "abort_multipart", start, 0, &result); + result + } +} + +pub trait ObjectStoreMetricsExt { + /// Wrap this store so its operations publish metrics under the given `base` label. + fn metered(self, base: String) -> Arc; +} + +impl ObjectStoreMetricsExt for Arc { + fn metered(self, base: String) -> Arc { + Arc::new(MeteredObjectStore { target: self, base }) + } +} + +// --- Layer 2: HTTP-level throttle metrics for native cloud stores --- + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +mod http { + use super::*; + use object_store::client::{ + ClientOptions, HttpClient, HttpConnector, HttpError, HttpRequest, HttpResponse, + HttpService, ReqwestConnector, + }; + + /// An [`HttpConnector`] that records throttle and retryable responses + /// observed by the underlying HTTP client. Install it on the S3 / GCS / + /// Azure builders via `with_http_connector`. + #[derive(Debug)] + pub struct MeteringHttpConnector { + base: String, + inner: ReqwestConnector, + } + + impl MeteringHttpConnector { + pub fn new(base: String) -> Self { + Self { + base, + inner: ReqwestConnector::default(), + } + } + } + + impl HttpConnector for MeteringHttpConnector { + fn connect(&self, options: &ClientOptions) -> object_store::Result { + let client = self.inner.connect(options)?; + Ok(HttpClient::new(MeteringHttpService { + base: self.base.clone(), + inner: client, + })) + } + } + + #[derive(Debug)] + struct MeteringHttpService { + base: String, + inner: HttpClient, + } + + #[async_trait::async_trait] + impl HttpService for MeteringHttpService { + async fn call(&self, req: HttpRequest) -> Result { + let response = self.inner.execute(req).await?; + let status = response.status().as_u16(); + // Each attempt that object_store may retry is recorded with its + // numeric status. Throttles (429 / 503) are a distinct, narrower + // signal than the broader set of retryable responses, so they get + // their own counter. 409 (conflict) is intentionally excluded from + // the retryable set so commit conflicts are not counted as retries. + let is_throttle = status == 429 || status == 503; + let is_retryable = status == 429 || status == 408 || (500..600).contains(&status); + if is_throttle { + metrics::counter!(METRIC_THROTTLE, status_labels(&self.base, status)).increment(1); + } + if is_retryable { + metrics::counter!(METRIC_RETRYABLE, status_labels(&self.base, status)).increment(1); + } + Ok(response) + } + } + + /// Build the `status` (+ optional `base`) label set for HTTP-layer metrics, + /// honoring the configured label mode. + fn status_labels(base: &str, status: u16) -> Vec { + let mut labels = vec![metrics::Label::new("status", status.to_string())]; + if let Some(base) = scoped_base(base_label_mode(), base) { + labels.push(metrics::Label::new("base", base)); + } + labels + } + + #[cfg(test)] + mod tests { + use super::*; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + use object_store::client::{HttpRequestBody, HttpResponseBody}; + + /// A mock [`HttpService`] that always responds with a fixed status code. + #[derive(Debug)] + struct StaticStatusService { + status: u16, + } + + #[async_trait::async_trait] + impl HttpService for StaticStatusService { + async fn call(&self, _req: HttpRequest) -> Result { + Ok(::http::Response::builder() + .status(self.status) + .body(HttpResponseBody::from(Bytes::new())) + .unwrap()) + } + } + + fn request() -> HttpRequest { + ::http::Request::builder() + .method("GET") + .uri("http://example.com/obj") + .body(HttpRequestBody::empty()) + .unwrap() + } + + fn metric_count( + metrics: &[(metrics::Key, DebugValue)], + name: &str, + base: &str, + status: &str, + ) -> u64 { + for (key, value) in metrics { + if key.name() != name { + continue; + } + let labels: std::collections::HashMap<&str, &str> = + key.labels().map(|l| (l.key(), l.value())).collect(); + if labels.get("base") == Some(&base) + && labels.get("status") == Some(&status) + && let DebugValue::Counter(v) = value + { + return *v; + } + } + 0 + } + + #[test] + fn test_throttle_and_retryable_responses_counted_by_status() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + // Each attempt that object_store retries flows through `call` + // again; here we simulate that by issuing several responses. + // The base is baked into the connector, so it labels the + // metric. Bases here have no `$`, so they are unaffected by + // the label mode and this test isolates status handling. + for (base, status) in [ + ("s3", 429u16), + ("s3", 503), + ("s3", 503), + ("s3", 500), + ("s3", 408), + ("s3", 409), + ("s3", 200), + ("s3", 404), + ("gs", 429), + ] { + let service = MeteringHttpService { + base: base.into(), + inner: HttpClient::new(StaticStatusService { status }), + }; + service.call(request()).await.unwrap(); + } + }); + }); + + let recorded: Vec<_> = snapshotter + .snapshot() + .into_vec() + .into_iter() + .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value)) + .collect(); + + let throttle = |base, status| metric_count(&recorded, METRIC_THROTTLE, base, status); + let retryable = |base, status| metric_count(&recorded, METRIC_RETRYABLE, base, status); + + // Throttles are only 429 and 503. + assert_eq!(throttle("s3", "429"), 1); + assert_eq!(throttle("s3", "503"), 2); + assert_eq!(throttle("s3", "500"), 0); + assert_eq!(throttle("s3", "408"), 0); + + // Retryable is the broader set: 5xx, 429, 408 (but not 409). + assert_eq!(retryable("s3", "429"), 1); + assert_eq!(retryable("s3", "503"), 2); + assert_eq!(retryable("s3", "500"), 1); + assert_eq!(retryable("s3", "408"), 1); + // 409 conflict is excluded so commit conflicts are not counted as retries. + assert_eq!(retryable("s3", "409"), 0); + + // Success and non-retryable client errors count as neither. + assert_eq!(throttle("s3", "200"), 0); + assert_eq!(retryable("s3", "404"), 0); + + // The base label is taken from the connector, not shared across stores. + assert_eq!(throttle("gs", "429"), 1); + assert_eq!(retryable("gs", "429"), 1); + assert_eq!(throttle("gs", "503"), 0); + } + } +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +pub use http::MeteringHttpConnector; + +#[cfg(test)] +mod tests { + use super::*; + + use lance_core::utils::tempfile::TempStdDir; + use metrics_util::debugging::{DebugValue, DebuggingRecorder, Snapshotter}; + use object_store::memory::InMemory; + use object_store::{ObjectStoreExt, PutPayload}; + use tokio::io::AsyncWriteExt; + use url::Url; + + use crate::object_store::ObjectStore as LanceObjectStore; + use crate::traits::Writer; + + fn payload(data: &[u8]) -> PutPayload { + PutPayload::from_bytes(Bytes::copy_from_slice(data)) + } + + fn metered_store() -> Arc { + (Arc::new(InMemory::new()) as Arc).metered("memory".into()) + } + + /// A single materialized snapshot of recorded metrics. It must be taken + /// only once: the snapshotter *drains* histogram samples on every + /// `snapshot()` call, so a second snapshot would see empty histograms. + type Metrics = Vec<(metrics::Key, DebugValue)>; + + /// Materialize the current recorder state. Histogram samples are *drained* + /// on each call, so a metric must be read from a single snapshot. + fn snapshot(snapshotter: &Snapshotter) -> Metrics { + snapshotter + .snapshot() + .into_vec() + .into_iter() + .map(|(ck, _unit, _desc, value)| (ck.key().clone(), value)) + .collect() + } + + /// Run an async closure with a thread-local metrics recorder installed and + /// return the resulting metrics. Uses a current-thread runtime so all polls + /// happen on the thread that holds the recorder guard. + fn capture_metrics(f: F) -> Metrics + where + F: FnOnce() -> Fut, + Fut: std::future::Future, + { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(f()); + }); + snapshot(&snapshotter) + } + + fn key_matches(key: &metrics::Key, name: &str, labels: &[(&str, &str)]) -> bool { + if key.name() != name { + return false; + } + let actual: std::collections::HashSet<(&str, &str)> = + key.labels().map(|l| (l.key(), l.value())).collect(); + labels.len() == actual.len() && labels.iter().all(|l| actual.contains(l)) + } + + fn counter_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> u64 { + for (key, value) in metrics { + if key_matches(key, name, labels) + && let DebugValue::Counter(v) = value + { + return *v; + } + } + 0 + } + + fn histogram_count(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> usize { + for (key, value) in metrics { + if key_matches(key, name, labels) + && let DebugValue::Histogram(samples) = value + { + return samples.len(); + } + } + 0 + } + + fn gauge_value(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> f64 { + for (key, value) in metrics { + if key_matches(key, name, labels) + && let DebugValue::Gauge(v) = value + { + return v.0; + } + } + 0.0 + } + + fn has_metric(metrics: &Metrics, name: &str, labels: &[(&str, &str)]) -> bool { + metrics + .iter() + .any(|(key, _)| key_matches(key, name, labels)) + } + + #[test] + fn test_parse_base_label_mode() { + assert_eq!(parse_base_label_mode(None), BaseLabelMode::Scheme); + assert_eq!(parse_base_label_mode(Some("scheme")), BaseLabelMode::Scheme); + assert_eq!(parse_base_label_mode(Some("full")), BaseLabelMode::Full); + assert_eq!(parse_base_label_mode(Some("off")), BaseLabelMode::Off); + assert_eq!(parse_base_label_mode(Some("none")), BaseLabelMode::Off); + // Unrecognized values fall back to the conservative default. + assert_eq!(parse_base_label_mode(Some("bogus")), BaseLabelMode::Scheme); + } + + #[test] + fn test_scoped_base() { + assert_eq!( + scoped_base(BaseLabelMode::Full, "s3$bucket").as_deref(), + Some("s3$bucket") + ); + assert_eq!( + scoped_base(BaseLabelMode::Scheme, "s3$bucket").as_deref(), + Some("s3") + ); + // Azure keeps only the scheme even though its prefix carries the account. + assert_eq!( + scoped_base(BaseLabelMode::Scheme, "az$container@account").as_deref(), + Some("az") + ); + // A prefix without `$` (e.g. memory/file) is unchanged by scheme mode. + assert_eq!( + scoped_base(BaseLabelMode::Scheme, "memory").as_deref(), + Some("memory") + ); + assert_eq!(scoped_base(BaseLabelMode::Off, "s3$bucket"), None); + } + + #[test] + fn test_base_label_defaults_to_scheme() { + // No env var is set in the test process, so the default `scheme` mode + // applies: the full prefix collapses to just the scheme. + let recorded = capture_metrics(|| async { + let store = (Arc::new(InMemory::new()) as Arc) + .metered("s3$my-bucket".into()); + store.put(&Path::from("a"), payload(b"x")).await.unwrap(); + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "put"), ("base", "s3")] + ), + 1 + ); + // The full prefix is not emitted as the label under the default mode. + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "put"), ("base", "s3$my-bucket")] + ), + 0 + ); + } + + #[test] + fn test_put_records_count_bytes_and_latency() { + let data = b"hello world"; + let recorded = capture_metrics(|| async { + let store = metered_store(); + store + .put(&Path::from("a/b.bin"), payload(data)) + .await + .unwrap(); + }); + + let labels = [("operation", "put"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &labels), + data.len() as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_get_records_count_and_bytes() { + let data = b"hello world"; + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(data)).await.unwrap(); + // The GET is only recorded once its body has been fully drained. + store.get(&path).await.unwrap().bytes().await.unwrap(); + }); + + let labels = [("operation", "get"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &labels), + data.len() as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_get_not_recorded_until_body_drained() { + let data = b"hello world"; + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "get"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(data)).await.unwrap(); + + // Holding the result without reading the body records nothing yet. + let result = store.get(&path).await.unwrap(); + assert_eq!( + counter_value(&snapshot(&snapshotter), METRIC_REQUESTS, &labels), + 0 + ); + + // Draining the body records the request with the true byte count. + let bytes = result.bytes().await.unwrap(); + assert_eq!(bytes.len(), data.len()); + let recorded = snapshot(&snapshotter); + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &labels), + data.len() as u64 + ); + }); + }); + } + + #[test] + fn test_head_is_a_separate_operation() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(b"hello world")).await.unwrap(); + store.head(&path).await.unwrap(); + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "head"), ("base", "memory")] + ), + 1 + ); + // The head call must not be counted as a get. + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "get"), ("base", "memory")] + ), + 0 + ); + // A HEAD transfers only metadata, so it records no payload bytes. + assert_eq!( + counter_value( + &recorded, + METRIC_BYTES, + &[("operation", "head"), ("base", "memory")] + ), + 0 + ); + } + + #[test] + fn test_delete_records_one_request_per_call() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + for i in 0..3 { + store + .put(&Path::from(format!("a/{i}.bin")), payload(b"x")) + .await + .unwrap(); + } + // `delete` drives `delete_stream`; deleting three paths is still one + // logical delete request (a single batched request on real stores). + let paths = + futures::stream::iter((0..3).map(|i| Ok(Path::from(format!("a/{i}.bin"))))).boxed(); + let _: Vec<_> = store.delete_stream(paths).collect().await; + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "delete"), ("base", "memory")] + ), + 1 + ); + } + + #[test] + fn test_list_counts_one_request_not_per_item() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + for i in 0..3 { + store + .put(&Path::from(format!("a/{i}.bin")), payload(b"x")) + .await + .unwrap(); + } + let _: Vec<_> = store.list(Some(&Path::from("a"))).collect().await; + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "list"), ("base", "memory")] + ), + 1 + ); + } + + #[test] + fn test_error_is_counted() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + // Getting a missing object errors. + let _ = store.get(&Path::from("does/not/exist")).await; + }); + + let labels = [("operation", "get"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1); + // A failed request is still counted as a request, with latency recorded. + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + // No bytes are transferred on a failed get. + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + #[test] + fn test_get_ranges_sums_part_bytes_and_labels_get() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(b"hello world")).await.unwrap(); + // Two disjoint ranges of 3 bytes each. + store.get_ranges(&path, &[2..5, 6..9]).await.unwrap(); + }); + + let labels = [("operation", "get"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 6); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_copy_and_rename_record_zero_bytes() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + store + .put(&Path::from("a/src"), payload(b"x")) + .await + .unwrap(); + store + .copy(&Path::from("a/src"), &Path::from("a/copy")) + .await + .unwrap(); + store + .rename(&Path::from("a/copy"), &Path::from("a/moved")) + .await + .unwrap(); + }); + + for operation in ["copy", "rename"] { + let labels = [("operation", operation), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + } + + #[test] + fn test_list_with_delimiter_records_latency() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + store.put(&Path::from("a/b"), payload(b"x")).await.unwrap(); + store + .list_with_delimiter(Some(&Path::from("a"))) + .await + .unwrap(); + }); + + let labels = [("operation", "list"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + #[test] + fn test_list_with_offset_counts_one_request() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + for i in 0..3 { + store + .put(&Path::from(format!("a/{i}")), payload(b"x")) + .await + .unwrap(); + } + let _: Vec<_> = store + .list_with_offset(Some(&Path::from("a")), &Path::from("a/0")) + .collect() + .await; + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "list"), ("base", "memory")] + ), + 1 + ); + } + + #[test] + fn test_multipart_records_each_part_and_complete() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap(); + upload.put_part(payload(b"hello")).await.unwrap(); // 5 bytes + upload.put_part(payload(b"world!!")).await.unwrap(); // 7 bytes + upload.complete().await.unwrap(); + }); + + let part_labels = [("operation", "put_part"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &part_labels), 2); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &part_labels), 12); + // Each part records its own latency sample, like a unary put. + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &part_labels), 2); + // A successful part upload records no error. + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &part_labels), 0); + + // Completing the upload is its own metered request. + let complete_labels = [("operation", "complete_multipart"), ("base", "memory")]; + assert_eq!( + counter_value(&recorded, METRIC_REQUESTS, &complete_labels), + 1 + ); + assert_eq!( + histogram_count(&recorded, METRIC_DURATION, &complete_labels), + 1 + ); + } + + #[test] + fn test_multipart_abort_is_recorded() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap(); + upload.put_part(payload(b"hello")).await.unwrap(); + upload.abort().await.unwrap(); + }); + + let labels = [("operation", "abort_multipart"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + } + + #[test] + fn test_multipart_part_error_is_counted() { + let recorded = capture_metrics(|| async { + let store = (Arc::new(FailingStreamStore) as Arc) + .metered("memory".into()); + let mut upload = store.put_multipart(&Path::from("a/big")).await.unwrap(); + let _ = upload.put_part(payload(b"data")).await; + }); + + let labels = [("operation", "put_part"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + // A failed part transfers no counted bytes. + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + #[test] + fn test_in_flight_guard_tracks_and_releases() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "get"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let g1 = InFlightGuard::new("memory", "get"); + let g2 = InFlightGuard::new("memory", "get"); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 2.0 + ); + drop(g1); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(g2); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + } + + #[test] + fn test_in_flight_gauge_is_wired_and_balances() { + let recorded = capture_metrics(|| async { + let store = metered_store(); + let path = Path::from("a/b.bin"); + store.put(&path, payload(b"hello")).await.unwrap(); + store.get(&path).await.unwrap(); + }); + + // The gauge is emitted for each operation (guard is wired in) and, once + // the operation completes, balances back to zero. + for operation in ["put", "get"] { + let labels = [("operation", operation), ("base", "memory")]; + assert!(has_metric(&recorded, METRIC_IN_FLIGHT, &labels)); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 0.0); + } + } + + #[test] + fn test_list_stream_holds_in_flight_until_dropped() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "list"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let store = metered_store(); + store.put(&Path::from("a/x"), payload(b"x")).await.unwrap(); + + // Creating the stream raises the gauge; it stays raised until the + // stream is dropped, even before any items are drained. + let stream = store.list(Some(&Path::from("a"))); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(stream); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + }); + } + + #[test] + fn test_delete_stream_holds_in_flight_until_dropped() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "delete"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let store = metered_store(); + let locations = futures::stream::iter(vec![Ok(Path::from("a/b"))]).boxed(); + + // Like list, creating the delete stream raises the gauge and holds + // it until the stream is dropped, before any items are drained. + let stream = store.delete_stream(locations); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(stream); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + }); + } + + #[test] + fn test_in_flight_released_when_operation_future_dropped() { + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "get"), ("base", "memory")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let started = Arc::new(tokio::sync::Notify::new()); + // Never signalled: the request stays blocked mid-flight. + let release = Arc::new(tokio::sync::Notify::new()); + let store = (Arc::new(BlockingStore { + started: started.clone(), + release, + }) as Arc) + .metered("memory".into()); + + let path = Path::from("a/b"); + let mut fut = Box::pin(store.get(&path)); + // Drive the request until it is blocked inside the inner store. + tokio::select! { + _ = &mut fut => unreachable!("the blocking store never returns"), + _ = started.notified() => {} + } + + // The gauge is raised while the request is outstanding, and + // dropping the future before it completes releases it. + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 1.0 + ); + drop(fut); + assert_eq!( + gauge_value(&snapshot(&snapshotter), METRIC_IN_FLIGHT, &labels), + 0.0 + ); + }); + }); + } + + #[test] + fn test_streaming_errors_are_counted() { + let recorded = capture_metrics(|| async { + let delete_store = (Arc::new(FailingStreamStore) as Arc) + .metered("memory".into()); + let _ = delete_store.delete(&Path::from("a/b")).await; + + let list_store = (Arc::new(FailingStreamStore) as Arc) + .metered("memory".into()); + let _: Vec<_> = list_store.list(None).collect().await; + }); + + // delete_stream counts the item and records an error when it fails. + let delete_labels = [("operation", "delete"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &delete_labels), 1); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &delete_labels), 1); + + // A list request is counted once; a failure while draining records an error. + let list_labels = [("operation", "list"), ("base", "memory")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &list_labels), 1); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &list_labels), 1); + } + + /// The optimized local reads and writes talk to the filesystem directly, so + /// they never reach [`MeteredObjectStore`] and publish these metrics + /// themselves. They must land under the same `base` label as the store's + /// metered operations, which for a local store is its scheme. + #[test] + fn test_local_filesystem_io_is_metered() { + let tmp = TempStdDir::default(); + let dir = tmp.join("sub"); + let data = b"hello world"; + let recorded = capture_metrics(|| async { + // Built through the registry, like any store opened from a URI. + let (store, path) = LanceObjectStore::from_uri(dir.join("a.bin").to_str().unwrap()) + .await + .unwrap(); + // Writes go through LocalWriter. + store.put(&path, data).await.unwrap(); + + // Reads go through LocalObjectReader. + let reader = store.open(&path).await.unwrap(); + assert_eq!(reader.size().await.unwrap(), data.len()); + assert_eq!(reader.get_range(0..5).await.unwrap().len(), 5); + assert_eq!(reader.get_all().await.unwrap().len(), data.len()); + // The file is smaller than the block size, so it streams as one chunk. + let chunks: Vec<_> = reader.get_stream().await.unwrap().collect().await; + assert_eq!(chunks.len(), 1); + + // Copy and recursive delete both shortcut to the filesystem too. + store + .copy(&path, &Path::from_absolute_path(dir.join("b.bin")).unwrap()) + .await + .unwrap(); + store + .remove_dir_all(Path::from_absolute_path(&dir).unwrap()) + .await + .unwrap(); + }); + + let put_labels = [("operation", "put"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &put_labels), 1); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &put_labels), + data.len() as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &put_labels), 1); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &put_labels), 0.0); + + // One request each for the range read, the full read and the single + // streamed chunk. + let get_labels = [("operation", "get"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &get_labels), 3); + assert_eq!( + counter_value(&recorded, METRIC_BYTES, &get_labels), + (5 + 2 * data.len()) as u64 + ); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &get_labels), 3); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &get_labels), 0.0); + + // The size lookup is the local equivalent of a HEAD, and transfers no + // payload bytes. + let head_labels = [("operation", "head"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &head_labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &head_labels), 0); + + for operation in ["copy", "delete"] { + let labels = [("operation", operation), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &get_labels), 0); + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &put_labels), 0); + } + + #[test] + fn test_local_read_error_is_counted() { + let tmp = TempStdDir::default(); + let recorded = capture_metrics(|| async { + let (store, path) = LanceObjectStore::from_uri(tmp.join("a.bin").to_str().unwrap()) + .await + .unwrap(); + store.put(&path, b"hello").await.unwrap(); + + let reader = store.open(&path).await.unwrap(); + // Reading past the end of the file fails. + assert!(reader.get_range(0..100).await.is_err()); + }); + + let labels = [("operation", "get"), ("base", "file")]; + assert_eq!(counter_value(&recorded, METRIC_ERRORS, &labels), 1); + // A failed read is still counted as a request, with latency recorded. + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(histogram_count(&recorded, METRIC_DURATION, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 0); + } + + /// A local write is reported as a single `put` covering the whole file, so it + /// stays in flight until the file is persisted under its final path. + #[test] + fn test_local_write_is_in_flight_until_persisted() { + let tmp = TempStdDir::default(); + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let labels = [("operation", "put"), ("base", "file")]; + metrics::with_local_recorder(&recorder, || { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + rt.block_on(async { + let (store, path) = LanceObjectStore::from_uri(tmp.join("a.bin").to_str().unwrap()) + .await + .unwrap(); + let mut writer = store.create(&path).await.unwrap(); + writer.write_all(b"hello").await.unwrap(); + + let recorded = snapshot(&snapshotter); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 1.0); + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 0); + + Writer::shutdown(writer.as_mut()).await.unwrap(); + let recorded = snapshot(&snapshotter); + assert_eq!(gauge_value(&recorded, METRIC_IN_FLIGHT, &labels), 0.0); + assert_eq!(counter_value(&recorded, METRIC_REQUESTS, &labels), 1); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), 5); + }); + }); + } + + /// A store handed in by the caller is metered like one built by the registry. + #[test] + fn test_caller_supplied_store_is_metered() { + let recorded = capture_metrics(|| async { + #[allow(deprecated)] + let params = crate::object_store::ObjectStoreParams { + object_store: Some(( + Arc::new(InMemory::new()) as Arc, + Url::parse("memory:///").unwrap(), + )), + ..Default::default() + }; + let (store, _) = LanceObjectStore::from_uri_and_params( + Arc::new(crate::object_store::ObjectStoreRegistry::default()), + "memory:///", + ¶ms, + ) + .await + .unwrap(); + store.put(&Path::from("a"), b"hello").await.unwrap(); + }); + + assert_eq!( + counter_value( + &recorded, + METRIC_REQUESTS, + &[("operation", "put"), ("base", "memory")] + ), + 1 + ); + } + + /// `ObjectStore::new` is how `DatasetBuilder` wraps a caller-supplied store, + /// so it must meter both halves of the store: the operations that go through + /// `inner`, and the local ones that bypass it. Metering only one half would + /// report a partial picture that reads like a complete one. + #[test] + fn test_store_built_from_new_is_metered() { + let tmp = TempStdDir::default(); + let recorded = capture_metrics(|| async { + let store = LanceObjectStore::new( + Arc::new(object_store::local::LocalFileSystem::new()), + Url::parse("file:///").unwrap(), + None, + None, + false, + false, + 1, + 3, + None, + ); + let path = Path::from_absolute_path(tmp.join("a.bin")).unwrap(); + // put and open bypass `inner` and publish for themselves. + store.put(&path, b"hello").await.unwrap(); + let reader = store.open(&path).await.unwrap(); + assert_eq!(reader.get_all().await.unwrap().len(), 5); + // delete goes through `inner`, so only MeteredObjectStore can count it. + store.delete(&path).await.unwrap(); + }); + + for (operation, bytes) in [("put", 5), ("get", 5), ("delete", 0)] { + let labels = [("operation", operation), ("base", "file")]; + assert_eq!( + counter_value(&recorded, METRIC_REQUESTS, &labels), + 1, + "expected one {operation} request" + ); + assert_eq!(counter_value(&recorded, METRIC_BYTES, &labels), bytes); + } + } + + /// A store whose stream-producing operations always yield an error, used to + /// exercise the error branches of the streaming wrappers. + #[derive(Debug)] + struct FailingStreamStore; + + impl std::fmt::Display for FailingStreamStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FailingStreamStore") + } + } + + fn test_error() -> object_store::Error { + object_store::Error::Generic { + store: "FailingStreamStore", + source: "injected failure".into(), + } + } + + #[async_trait::async_trait] + impl object_store::ObjectStore for FailingStreamStore { + async fn put_opts( + &self, + _location: &Path, + _bytes: PutPayload, + _opts: PutOptions, + ) -> OSResult { + unimplemented!() + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> OSResult> { + Ok(Box::new(FailingUpload)) + } + + async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult { + unimplemented!() + } + + fn delete_stream( + &self, + _locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + futures::stream::once(async { Err(test_error()) }).boxed() + } + + fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + futures::stream::once(async { Err(test_error()) }).boxed() + } + + fn list_with_offset( + &self, + _prefix: Option<&Path>, + _offset: &Path, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult { + unimplemented!() + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> { + unimplemented!() + } + + async fn rename_opts( + &self, + _from: &Path, + _to: &Path, + _opts: RenameOptions, + ) -> OSResult<()> { + unimplemented!() + } + } + + /// A [`MultipartUpload`] whose part uploads always fail, used to exercise the + /// error branch of the metered `put_part`. + #[derive(Debug)] + struct FailingUpload; + + #[async_trait::async_trait] + impl MultipartUpload for FailingUpload { + fn put_part(&mut self, _data: PutPayload) -> UploadPart { + async { Err(test_error()) }.boxed() + } + + async fn complete(&mut self) -> OSResult { + unimplemented!() + } + + async fn abort(&mut self) -> OSResult<()> { + unimplemented!() + } + } + + /// A store whose `get_opts` blocks after signalling `started`, so a request + /// can be observed mid-flight and then dropped before it completes. + #[derive(Debug)] + struct BlockingStore { + started: Arc, + release: Arc, + } + + impl std::fmt::Display for BlockingStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "BlockingStore") + } + } + + #[async_trait::async_trait] + impl object_store::ObjectStore for BlockingStore { + async fn put_opts( + &self, + _location: &Path, + _bytes: PutPayload, + _opts: PutOptions, + ) -> OSResult { + unimplemented!() + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> OSResult> { + unimplemented!() + } + + async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult { + self.started.notify_one(); + self.release.notified().await; + unreachable!("release is never signalled in the test") + } + + fn delete_stream( + &self, + _locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + fn list_with_offset( + &self, + _prefix: Option<&Path>, + _offset: &Path, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult { + unimplemented!() + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> { + unimplemented!() + } + + async fn rename_opts( + &self, + _from: &Path, + _to: &Path, + _opts: RenameOptions, + ) -> OSResult<()> { + unimplemented!() + } + } +} diff --git a/rust/lance-io/src/object_store/opendal_store.rs b/rust/lance-io/src/object_store/opendal_store.rs new file mode 100644 index 00000000000..99523759095 --- /dev/null +++ b/rust/lance-io/src/object_store/opendal_store.rs @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::fmt; +use std::ops::Range; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::{StreamExt, TryStreamExt, future, stream::BoxStream}; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, + ObjectStore as OSObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, + RenameOptions, +}; +use object_store_opendal::OpendalStore as InnerOpendalStore; +use opendal::Operator; + +/// Adapts OpenDAL listing paths to the spelling used by the request. +/// +/// The upstream bridge builds listed locations with [`Path::from`], which +/// percent-encodes reserved characters. Lance builds dataset base paths with +/// [`Path::from_url_path`], so mismatched listed locations must be decoded. +/// Locations that already match the requested prefix retain their spelling to +/// preserve paths containing literal percent escapes. +#[derive(Debug, Clone)] +pub(super) struct OpendalStore { + inner: InnerOpendalStore, +} + +impl OpendalStore { + pub(super) fn new(operator: Operator) -> Self { + Self { + inner: InnerOpendalStore::new(operator), + } + } +} + +impl fmt::Display for OpendalStore { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + self.inner.fmt(formatter) + } +} + +fn normalize_location(location: &Path, prefix: Option<&Path>) -> object_store::Result { + if prefix.is_none_or(|prefix| location.prefix_matches(prefix)) { + return Ok(location.clone()); + } + + Path::from_url_path(location.as_ref()).map_err(|source| object_store::Error::Generic { + store: "OpendalStore", + source: Box::new(source), + }) +} + +fn normalize_object_meta( + mut meta: ObjectMeta, + prefix: Option<&Path>, +) -> object_store::Result { + meta.location = normalize_location(&meta.location, prefix)?; + Ok(meta) +} + +#[async_trait] +impl OSObjectStore for OpendalStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + options: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, options).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + options: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, options).await + } + + async fn get_opts( + &self, + location: &Path, + options: GetOptions, + ) -> object_store::Result { + self.inner.get_opts(location, options).await + } + + async fn get_ranges( + &self, + location: &Path, + ranges: &[Range], + ) -> object_store::Result> { + self.inner.get_ranges(location, ranges).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + let listed = self.inner.list(prefix); + let prefix = prefix.cloned(); + listed + .map(move |result| result.and_then(|meta| normalize_object_meta(meta, prefix.as_ref()))) + .boxed() + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, object_store::Result> { + if self.inner.info().capability().list_with_start_after { + let listed = self.inner.list_with_offset(prefix, offset); + let prefix = prefix.cloned(); + listed + .map(move |result| { + result.and_then(|meta| normalize_object_meta(meta, prefix.as_ref())) + }) + .boxed() + } else { + // The bridge's fallback compares its encoded output with the raw + // offset. Filter normalized locations so both sides use one form. + let offset = offset.clone(); + self.list(prefix) + .try_filter(move |meta| future::ready(meta.location > offset)) + .boxed() + } + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + let mut result = self.inner.list_with_delimiter(prefix).await?; + for object in &mut result.objects { + object.location = normalize_location(&object.location, prefix)?; + } + for common_prefix in &mut result.common_prefixes { + *common_prefix = normalize_location(common_prefix, prefix)?; + } + Ok(result) + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } + + async fn rename_opts( + &self, + from: &Path, + to: &Path, + options: RenameOptions, + ) -> object_store::Result<()> { + self.inner.rename_opts(from, to, options).await + } +} + +#[cfg(test)] +mod tests { + use bytes::Bytes; + use futures::TryStreamExt; + use object_store::ObjectStoreExt; + use opendal::services::Memory; + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::raw_reserved_character("tables/run~1/t.lance")] + #[case::literal_percent_escape("tables/run%25231/t.lance")] + #[tokio::test] + async fn test_list_preserves_request_path_spelling(#[case] base_url: &str) { + let operator = Operator::new(Memory::default()).unwrap(); + let store = OpendalStore::new(operator); + let base = Path::from_url_path(base_url).unwrap(); + let direct_location = base.clone().join("manifest.lance"); + let nested_location = Path::from_url_path(format!("{base_url}/data/part.lance")).unwrap(); + for location in [&direct_location, &nested_location] { + store + .put(location, Bytes::from_static(b"data").into()) + .await + .unwrap(); + } + + let listed = store + .list(Some(&base)) + .try_collect::>() + .await + .unwrap(); + let mut listed_locations = listed + .into_iter() + .map(|meta| meta.location) + .collect::>(); + listed_locations.sort(); + let mut expected_locations = vec![direct_location.clone(), nested_location.clone()]; + expected_locations.sort(); + assert_eq!(listed_locations, expected_locations); + assert!( + listed_locations + .iter() + .all(|location| location.prefix_matches(&base)) + ); + + let listed_after_nested = store + .list_with_offset(Some(&base), &nested_location) + .try_collect::>() + .await + .unwrap(); + assert_eq!(listed_after_nested.len(), 1); + assert_eq!(listed_after_nested[0].location, direct_location); + + let delimited = store.list_with_delimiter(Some(&base)).await.unwrap(); + assert_eq!(delimited.objects.len(), 1); + assert_eq!(delimited.objects[0].location, direct_location); + assert_eq!(delimited.common_prefixes, vec![base.clone().join("data")]); + } +} diff --git a/rust/lance-io/src/object_store/providers.rs b/rust/lance-io/src/object_store/providers.rs index 775c98552a8..d8d184e07da 100644 --- a/rust/lance-io/src/object_store/providers.rs +++ b/rust/lance-io/src/object_store/providers.rs @@ -205,6 +205,57 @@ impl ObjectStoreRegistry { Error::invalid_input(message) } + async fn build_store( + &self, + provider: Arc, + base_path: Url, + params: &ObjectStoreParams, + store_prefix: &str, + ) -> Result> { + let mut store = provider.new_store(base_path, params).await?; + + store.inner = store.inner.traced(); + + // Label metrics by the store's unique prefix (e.g. `s3$bucket`, + // `az$container@account`) so multiple stores on one cloud differ. + crate::object_store::meter_store(&mut store.inner, &mut store.io_tracker, store_prefix); + + if let Some(wrapper) = ¶ms.object_store_wrapper { + store.apply_wrapper(wrapper.as_ref()); + } + + // Always wrap with IO tracking + store.inner = store.io_tracker.wrap("", store.inner); + + Ok(Arc::new(store)) + } + + /// Build a fresh object store without consulting or populating the cache. + /// + /// Callers should retain the returned [`Arc`] for as long as they want to + /// reuse provider-local state such as HTTP clients and rate limiters. + #[doc(hidden)] + pub async fn new_store( + &self, + base_path: Url, + params: &ObjectStoreParams, + ) -> Result> { + // Base-scoped storage options (`base_.`) are directives for + // other registered base paths; resolve them away before building a + // store for this location. + let params = params.scoped_to_base(None); + let params = params.as_ref(); + let scheme = base_path.scheme(); + let Some(provider) = self.get_provider(scheme) else { + return Err(self.scheme_not_found_error(scheme)); + }; + let store_prefix = + provider.calculate_object_store_prefix(&base_path, params.storage_options())?; + + self.build_store(provider, base_path, params, &store_prefix) + .await + } + /// Get an object store for a given base path and parameters. /// /// If the object store is already in use, it will return a strong reference @@ -261,18 +312,9 @@ impl ObjectStoreRegistry { self.misses.fetch_add(1, Ordering::Relaxed); - let mut store = provider.new_store(base_path, params).await?; - - store.inner = store.inner.traced(); - - if let Some(wrapper) = ¶ms.object_store_wrapper { - store.inner = wrapper.wrap(&cache_path, store.inner); - } - - // Always wrap with IO tracking - store.inner = store.io_tracker.wrap("", store.inner); - - let store = Arc::new(store); + let store = self + .build_store(provider, base_path, params, &cache_path) + .await?; { // Insert the store into the cache @@ -373,8 +415,14 @@ impl ObjectStoreRegistry { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Mutex; use super::*; + use object_store::ObjectStore as OSObjectStore; + + use crate::object_store::providers::memory::MemoryStoreProvider; + use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; + use rstest::rstest; #[derive(Debug)] struct DummyProvider; @@ -390,6 +438,125 @@ mod tests { } } + /// A lister that exists only to be handed to a wrapper. + struct StubLister; + + #[async_trait::async_trait] + impl PaginatedListStore for StubLister { + async fn list_paginated( + &self, + _prefix: Option<&str>, + _opts: PaginatedListOptions, + ) -> object_store::Result { + unimplemented!("this lister exists to be wrapped, not to list") + } + } + + /// A provider whose stores come with a paginated lister, which the memory store does not. + #[derive(Debug)] + struct PaginatedProvider; + + #[async_trait::async_trait] + impl ObjectStoreProvider for PaginatedProvider { + async fn new_store( + &self, + base_path: Url, + params: &ObjectStoreParams, + ) -> Result { + let mut store = MemoryStoreProvider.new_store(base_path, params).await?; + store.paginated_lister = Some(Arc::new(StubLister)); + Ok(store) + } + + fn calculate_object_store_prefix( + &self, + _url: &Url, + _storage_options: Option<&HashMap>, + ) -> Result { + Ok("memory".to_string()) + } + } + + /// Swaps the store out for an empty one, the way a wrapper enforcing visibility would, and + /// records the prefix each call was labelled with. `keep_pushdown` is what it answers when + /// asked about the lister. + #[derive(Debug)] + struct RecordingWrapper { + keep_pushdown: bool, + prefixes: Mutex>, + } + + impl WrappingObjectStore for RecordingWrapper { + fn wrap( + &self, + store_prefix: &str, + _original: Arc, + ) -> Arc { + self.prefixes + .lock() + .unwrap() + .push(format!("wrap@{store_prefix}")); + Arc::new(object_store::memory::InMemory::new()) + } + + fn wrap_paginated( + &self, + store_prefix: &str, + original: Arc, + ) -> Option> { + self.prefixes + .lock() + .unwrap() + .push(format!("wrap_paginated@{store_prefix}")); + self.keep_pushdown.then_some(original) + } + } + + /// A decorator supplied through [`ObjectStoreParams`] has to reach the paginated lister + /// too, or `read_dir_page` would talk to the backend behind its back — and a decorator + /// that gives the pushdown up gets a store with no lister, so its listings go through the + /// wrapped `inner` and see what the wrapper allows rather than what the backend holds. + #[rstest] + #[case::keeps_the_pushdown(true)] + #[case::gives_up_the_pushdown(false)] + #[tokio::test] + async fn test_the_registry_hands_the_lister_to_the_wrapper(#[case] keep_pushdown: bool) { + let wrapper = Arc::new(RecordingWrapper { + keep_pushdown, + prefixes: Mutex::new(Vec::new()), + }); + let registry = ObjectStoreRegistry::default(); + registry.insert("pagmem", Arc::new(PaginatedProvider)); + + let store = registry + .get_store( + Url::parse("pagmem:///").unwrap(), + &ObjectStoreParams { + object_store_wrapper: Some(wrapper.clone()), + ..Default::default() + }, + ) + .await + .unwrap(); + + assert_eq!(store.paginated_lister.is_some(), keep_pushdown); + // Both halves of the store are labelled with the same prefix. + assert_eq!( + *wrapper.prefixes.lock().unwrap(), + vec!["wrap@memory", "wrap_paginated@memory"] + ); + if !keep_pushdown { + // `StubLister` panics if it is ever asked to list, so reaching a page at all is + // the other half of the assertion. + let page = store + .read_dir_page(Path::from(""), Default::default()) + .await + .unwrap(); + assert!(page.result.common_prefixes.is_empty()); + assert!(page.result.objects.is_empty()); + } + } + #[test] fn test_calculate_object_store_prefix() { let provider = DummyProvider; @@ -513,4 +680,18 @@ mod tests { // Same params returns same instance assert!(Arc::ptr_eq(&stores[0], &stores[1])); } + + #[tokio::test] + async fn test_new_store_bypasses_cache() { + let registry = ObjectStoreRegistry::default(); + let url = Url::parse("memory://test").unwrap(); + let params = ObjectStoreParams::default(); + + let first = registry.new_store(url.clone(), ¶ms).await.unwrap(); + let second = registry.new_store(url, ¶ms).await.unwrap(); + + assert!(!Arc::ptr_eq(&first, &second)); + let stats = registry.stats(); + assert_eq!((stats.hits, stats.misses, stats.active_stores), (0, 0, 0)); + } } diff --git a/rust/lance-io/src/object_store/providers/aws.rs b/rust/lance-io/src/object_store/providers/aws.rs index 9aad637bce2..c05d0714f13 100644 --- a/rust/lance-io/src/object_store/providers/aws.rs +++ b/rust/lance-io/src/object_store/providers/aws.rs @@ -10,41 +10,85 @@ use mock_instant::thread_local::{SystemTime, UNIX_EPOCH}; use std::time::{SystemTime, UNIX_EPOCH}; use object_store::ObjectStore as OSObjectStore; -use object_store_opendal::OpendalStore; +use object_store::list::PaginatedListStore; use opendal::{Operator, services::S3}; use aws_config::default_provider::credentials::DefaultCredentialsChain; +use aws_config::ecs::EcsCredentialsProvider; +use aws_config::provider_config::ProviderConfig; +use aws_config::web_identity_token::WebIdentityTokenCredentialsProvider; +use aws_config::{BehaviorVersion, Region, SdkConfig}; use aws_credential_types::provider::ProvideCredentials; use object_store::{ ClientOptions, CredentialProvider, Result as ObjectStoreResult, RetryConfig, StaticCredentialProvider, aws::{ - AmazonS3Builder, AmazonS3ConfigKey, AwsCredential as ObjectStoreAwsCredential, + AmazonS3, AmazonS3Builder, AmazonS3ConfigKey, AwsCredential as ObjectStoreAwsCredential, AwsCredentialProvider, }, }; use tokio::sync::RwLock; use url::Url; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor, dynamic_credentials::{NamespaceCredentialsProvider, build_dynamic_credential_provider}, - throttle::{AimdThrottleConfig, AimdThrottledStore}, + throttle::{AimdThrottleConfig, AimdThrottleState, cloud_http_connector, with_throttling}, }; use lance_core::error::{Error, Result}; +use lance_core::utils::parse::str_is_truthy; #[derive(Default, Debug)] pub struct AwsStoreProvider; +struct ResolvedS3StorageOptions { + options: HashMap, + profile_region: Option, +} + +impl ResolvedS3StorageOptions { + fn new( + mut options: HashMap, + profile_config: Option<&SdkConfig>, + ) -> Self { + if effective_s3_endpoint(&options).is_none() + && let Some(endpoint) = profile_config.and_then(SdkConfig::endpoint_url) + { + options.insert(AmazonS3ConfigKey::Endpoint, endpoint.to_string()); + } + let profile_region = profile_config + .and_then(SdkConfig::region) + .map(|region| region.as_ref().to_string()); + Self { + options, + profile_region, + } + } + + fn effective_endpoint(&self) -> Option<&str> { + effective_s3_endpoint(&self.options) + } + + fn requires_constant_size_upload_parts(&self) -> bool { + self.effective_endpoint() + .is_some_and(|endpoint| endpoint.contains("r2.cloudflarestorage.com")) + } +} + impl AwsStoreProvider { async fn build_amazon_s3_store( &self, base_path: &mut Url, params: &ObjectStoreParams, storage_options: &StorageOptions, + mut resolved_s3_options: ResolvedS3StorageOptions, is_s3_express: bool, - ) -> Result> { + throttle_state: Option<&AimdThrottleState>, + // Concrete rather than `dyn`, so the caller keeps the handle a paginated listing + // needs: `PaginatedListStore` is a separate trait from `ObjectStore`. + ) -> Result> { // Use a low retry count since the AIMD throttle layer handles // throttle recovery with its own retry loop. let retry_config = RetryConfig { @@ -53,26 +97,35 @@ impl AwsStoreProvider { retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()), }; - let mut s3_storage_options = storage_options.as_s3_options(); - let region = resolve_s3_region(base_path, &s3_storage_options).await?; + let region = resolve_s3_region(base_path, &resolved_s3_options).await?; // Get accessor from params let accessor = params.get_accessor(); + let provider_scheme = storage_options.aws_provider_scheme()?; + let (aws_creds, region) = build_aws_credential( params.s3_credentials_refresh_offset, params.aws_credentials.clone(), - Some(&s3_storage_options), + Some(&resolved_s3_options.options), region, accessor, + provider_scheme, ) .await?; // Set S3Express flag if detected if is_s3_express { - s3_storage_options.insert(AmazonS3ConfigKey::S3Express, true.to_string()); + resolved_s3_options + .options + .insert(AmazonS3ConfigKey::S3Express, true.to_string()); } + // Compute the metrics label before rewriting the URL below so it + // matches the prefix the registry uses to key this store. + let store_prefix = + self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?; + // before creating the OSObjectStore we need to rewrite the url to drop ddb related parts base_path.set_scheme("s3").unwrap(); base_path.set_query(None); @@ -80,7 +133,7 @@ impl AwsStoreProvider { // we can't use parse_url_opts here because we need to manually set the credentials provider let mut builder = AmazonS3Builder::new().with_client_options(storage_options.client_options()?); - for (key, value) in s3_storage_options { + for (key, value) in resolved_s3_options.options { builder = builder.with_config(key, value); } builder = builder @@ -89,7 +142,9 @@ impl AwsStoreProvider { .with_retry(retry_config) .with_region(region); - Ok(Arc::new(builder.build()?) as Arc) + builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix)); + + Ok(Arc::new(builder.build()?)) } async fn build_opendal_s3_store( @@ -108,6 +163,13 @@ impl AwsStoreProvider { // OpenDAL will handle environment variables through its default credentials chain let mut config_map: HashMap = storage_options.0.clone(); + if let Some(provider_scheme) = storage_options.aws_provider_scheme()? { + return Result::Err(Error::not_supported(format!( + "OpendalStore does not currently support an explicit provider_scheme (currently set to {:?})", + provider_scheme + ))); + } + // Set required OpenDAL configuration config_map.insert("bucket".to_string(), bucket); @@ -116,8 +178,7 @@ impl AwsStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create S3 operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create S3 operator: {:?}", e)))?; Ok(Arc::new(OpendalStore::new(operator)) as Arc) } @@ -139,36 +200,61 @@ impl ObjectStoreProvider for AwsStoreProvider { let use_opendal = storage_options .0 .get("use_opendal") - .map(|v| v == "true") + .map(|v| str_is_truthy(v)) .unwrap_or(false); + let profile_config = if std::env::var_os("AWS_PROFILE").is_some() { + Some(aws_config::load_defaults(BehaviorVersion::latest()).await) + } else { + None + }; + let resolved_s3_options = + ResolvedS3StorageOptions::new(storage_options.as_s3_options(), profile_config.as_ref()); + // Determine S3 Express and constant size upload parts before building the store let is_s3_express = check_s3_express(&base_path, &storage_options); - let use_constant_size_upload_parts = storage_options - .0 - .get("aws_endpoint") - .map(|endpoint| endpoint.contains("r2.cloudflarestorage.com")) - .unwrap_or(false); + let use_constant_size_upload_parts = + resolved_s3_options.requires_constant_size_upload_parts(); - let inner = if use_opendal { - // Use OpenDAL implementation - self.build_opendal_s3_store(&base_path, &storage_options) - .await? + let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; + let throttle_state = if throttle_config.is_disabled() { + None } else { - // Use default Amazon S3 implementation - self.build_amazon_s3_store(&mut base_path, params, &storage_options, is_s3_express) - .await? + Some(AimdThrottleState::new(throttle_config)?) }; - let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; - let inner = if throttle_config.is_disabled() { - inner + + let (inner, paginated_lister) = if use_opendal { + // Use OpenDAL implementation + // Listed in full: no paginated lister covers OpenDAL yet. + ( + self.build_opendal_s3_store(&base_path, &storage_options) + .await?, + None, + ) } else { - Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc + // Use default Amazon S3 implementation + let store = self + .build_amazon_s3_store( + &mut base_path, + params, + &storage_options, + resolved_s3_options, + is_s3_express, + throttle_state.as_ref(), + ) + .await?; + ( + store.clone() as Arc, + Some(store as Arc), + ) }; + let (inner, paginated_lister) = + with_throttling(throttle_state, !use_opendal, inner, paginated_lister); Ok(ObjectStore { inner, + local_dir_operations: None, scheme: String::from(base_path.scheme()), block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -179,6 +265,7 @@ impl ObjectStoreProvider for AwsStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + paginated_lister, }) } } @@ -188,25 +275,34 @@ fn check_s3_express(url: &Url, storage_options: &StorageOptions) -> bool { storage_options .0 .get("s3_express") - .map(|v| v == "true") + .map(|v| str_is_truthy(v)) .unwrap_or(false) || url.authority().ends_with("--x-s3") } +fn effective_s3_endpoint(storage_options: &HashMap) -> Option<&str> { + storage_options + .get(&AmazonS3ConfigKey::S3Endpoint) + .or_else(|| storage_options.get(&AmazonS3ConfigKey::Endpoint)) + .map(String::as_str) +} + /// Figure out the S3 region of the bucket. /// /// This resolves in order of precedence: /// 1. The region provided in the storage options -/// 2. (If endpoint is not set), the region returned by the S3 API for the bucket +/// 2. The selected AWS profile's region when a custom endpoint is configured +/// 3. (If endpoint is not set), the region returned by the S3 API for the bucket /// /// It can return None if no region is provided and the endpoint is set. async fn resolve_s3_region( url: &Url, - storage_options: &HashMap, + resolved_s3_options: &ResolvedS3StorageOptions, ) -> Result> { + let storage_options = &resolved_s3_options.options; if let Some(region) = storage_options.get(&AmazonS3ConfigKey::Region) { Ok(Some(region.clone())) - } else if storage_options.get(&AmazonS3ConfigKey::Endpoint).is_none() { + } else if resolved_s3_options.effective_endpoint().is_none() { // If no endpoint is set, we can assume this is AWS S3 and the region // can be resolved from the bucket. let bucket = url.host_str().ok_or_else(|| { @@ -224,18 +320,40 @@ async fn resolve_s3_region( object_store::aws::resolve_bucket_region(bucket, &client_options).await?; Ok(Some(bucket_region)) } else { - Ok(None) + Ok(resolved_s3_options.profile_region.clone()) } } +/// Selects which AWS credential provider to use for a dataset. +/// +/// When set, overrides automatic credential resolution for everything except an +/// explicitly-supplied `credentials` provider or `storage_options_accessor`. +#[derive(Debug, Clone, PartialEq)] +pub enum AwsProviderScheme { + /// Require static access-key credentials (`aws_access_key_id` + + /// `aws_secret_access_key`). Returns an error if they are absent. + Token, + /// Use the ECS/Pod Identity container credential endpoint. + /// The endpoint URI is read from the `AWS_CONTAINER_CREDENTIALS_FULL_URI` + /// or `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI` environment variables. + Ecs, + /// Use IRSA (IAM Roles for Service Accounts) web identity token credentials. + /// The token file and role ARN are read from the `AWS_WEB_IDENTITY_TOKEN_FILE` + /// and `AWS_ROLE_ARN` environment variables. + Irsa, +} + /// Build AWS credentials /// /// This resolves credentials from the following sources in order: -/// 1. An explicit `storage_options_accessor` with a provider -/// 2. An explicit `credentials` provider -/// 3. Explicit credentials in storage_options (as in `aws_access_key_id`, -/// `aws_secret_access_key`, `aws_session_token`) -/// 4. The default credential provider chain from AWS SDK. +/// 1. An explicit `credentials` provider +/// 2. An explicit `storage_options_accessor` with a provider +/// 3. If `provider_scheme` is set: +/// - [`AwsProviderScheme::Token`]: static access-key credentials (error if absent) +/// - [`AwsProviderScheme::Ecs`]: ECS container credential provider +/// - [`AwsProviderScheme::Irsa`]: web identity token (IRSA) provider +/// 4. Static access-key credentials from `storage_options`, if present +/// 5. The default AWS credential provider chain /// /// # Storage Options Accessor /// @@ -250,6 +368,7 @@ pub async fn build_aws_credential( storage_options: Option<&HashMap>, region: Option, storage_options_accessor: Option>, + provider_scheme: Option, ) -> Result<(AwsCredentialProvider, String)> { use aws_config::meta::region::RegionProviderChain; const DEFAULT_REGION: &str = "us-west-2"; @@ -265,18 +384,22 @@ pub async fn build_aws_credential( .unwrap_or(DEFAULT_REGION.to_string()) }; - let storage_options_credentials = storage_options.and_then(extract_static_s3_credentials); + // If the user supplied their own credential provider that takes top priority + if let Some(creds) = credentials { + return Ok((creds, region)); + } - // Explicit aws_credentials takes precedence over dynamic credentials. - if credentials.is_none() - && let Some(dynamic_creds) = build_dynamic_credential_provider::( - storage_options_accessor.clone(), - ) - .await? + // Otherwise, if the user provided a storage_options_accessor, try and use that + if let Some(dynamic_creds) = build_dynamic_credential_provider::( + storage_options_accessor.clone(), + ) + .await? { return Ok((dynamic_creds, region)); } + // If the user provided a storage_options_accessor, then it must not have matched AWS. + // Log a message and ignore it. if storage_options_accessor .as_ref() .is_some_and(|a| a.has_provider()) @@ -287,22 +410,61 @@ pub async fn build_aws_credential( ); } - // Fall back to existing logic for static credentials - if let Some(creds) = credentials { - Ok((creds, region)) - } else if let Some(creds) = storage_options_credentials { - Ok((Arc::new(creds), region)) - } else { - let credentials_provider = DefaultCredentialsChain::builder().build().await; + // If the caller specified an explicit provider scheme, use only that provider. + if let Some(scheme) = provider_scheme { + return match scheme { + AwsProviderScheme::Token => { + let creds = storage_options + .and_then(extract_static_s3_credentials) + .ok_or_else(|| { + Error::invalid_input( + "aws_provider_scheme=token requires aws_access_key_id \ + and aws_secret_access_key to be set", + ) + })?; + Ok((Arc::new(creds), region)) + } + AwsProviderScheme::Ecs => { + let provider = EcsCredentialsProvider::builder().build(); + Ok(( + Arc::new(AwsCredentialAdapter::new( + Arc::new(provider), + credentials_refresh_offset, + )), + region, + )) + } + AwsProviderScheme::Irsa => { + let conf = ProviderConfig::default().with_region(Some(Region::new(region.clone()))); + let provider = WebIdentityTokenCredentialsProvider::builder() + .configure(&conf) + .build(); + Ok(( + Arc::new(AwsCredentialAdapter::new( + Arc::new(provider), + credentials_refresh_offset, + )), + region, + )) + } + }; + } - Ok(( - Arc::new(AwsCredentialAdapter::new( - Arc::new(credentials_provider), - credentials_refresh_offset, - )), - region, - )) + if let Some(opts) = storage_options { + // Check for static credentials (access key & secret) + if let Some(creds) = extract_static_s3_credentials(opts) { + return Ok((Arc::new(creds), region)); + } } + + let credentials_provider = DefaultCredentialsChain::builder().build().await; + Ok(( + Arc::new(AwsCredentialAdapter::new( + Arc::new(credentials_provider), + credentials_refresh_offset, + )), + region, + )) } fn extract_static_s3_credentials( @@ -389,10 +551,12 @@ impl CredentialProvider for AwsCredentialAdapter { token: creds.session_token().map(|s| s.to_string()), })) } else { - let refreshed_creds = - Arc::new(self.inner.provide_credentials().await.map_err(|e| { - Error::internal(format!("Failed to get AWS credentials: {:?}", e)) - })?); + let refreshed_creds = Arc::new( + self.inner + .provide_credentials() + .await + .map_err(|e| Error::io(format!("Failed to get AWS credentials: {:?}", e)))?, + ); self.cache .write() @@ -409,7 +573,10 @@ impl CredentialProvider for AwsCredentialAdapter { } impl StorageOptions { - /// Add values from the environment to storage options + /// Add values from the environment to storage options. + /// + /// Only adds keys that are not already present, so explicitly-set options + /// (including empty-string sentinels) always take precedence over env vars. pub fn with_env_s3(&mut self) { for (os_key, os_value) in std::env::vars_os() { if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) @@ -432,6 +599,20 @@ impl StorageOptions { }) .collect() } + + /// Parse the `aws_provider_scheme` storage option, if set. + pub fn aws_provider_scheme(&self) -> Result> { + match self.0.get("aws_provider_scheme").map(|s| s.as_str()) { + None | Some("") => Ok(None), + Some("token") => Ok(Some(AwsProviderScheme::Token)), + Some("ecs") => Ok(Some(AwsProviderScheme::Ecs)), + Some("irsa") => Ok(Some(AwsProviderScheme::Irsa)), + Some(other) => Err(Error::invalid_input(format!( + "Invalid aws_provider_scheme '{}'. Valid values are: token, ecs, irsa", + other + ))), + } + } } impl ObjectStoreParams { @@ -460,6 +641,9 @@ pub type DynamicStorageOptionsCredentialProvider = mod tests { use crate::object_store::ObjectStoreRegistry; use crate::object_store::StorageOptionsProvider; + #[allow(deprecated)] + use aws_config::profile::profile_file::{ProfileFileKind, ProfileFiles}; + use aws_credential_types::provider::error::CredentialsError; use mock_instant::thread_local::MockClock; use object_store::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; @@ -485,6 +669,58 @@ mod tests { } } + #[allow(deprecated)] + async fn load_test_profile(config: &str) -> SdkConfig { + let profile_files = ProfileFiles::builder() + .with_contents(ProfileFileKind::Config, config) + .build(); + aws_config::defaults(BehaviorVersion::latest()) + .profile_name("selected") + .profile_files(profile_files) + .load() + .await + } + + #[derive(Debug)] + struct FailingAwsCredentialsProvider; + + impl ProvideCredentials for FailingAwsCredentialsProvider { + fn provide_credentials<'a>( + &'a self, + ) -> aws_credential_types::provider::future::ProvideCredentials<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideCredentials::new(async { + Err(CredentialsError::provider_error(Box::new( + std::io::Error::other("Glue credential endpoint unavailable"), + ))) + }) + } + } + + #[tokio::test] + async fn test_aws_credential_failure_is_io_error() { + let provider = AwsCredentialAdapter::new( + Arc::new(FailingAwsCredentialsProvider), + Duration::from_secs(60), + ); + + let error = provider.get_credential().await.unwrap_err(); + let object_store::Error::Generic { source, .. } = &error else { + panic!("expected a generic object store error, got {error}"); + }; + assert!(matches!( + source.downcast_ref::(), + Some(Error::IO { .. }) + )); + + let message = error.to_string(); + assert!(message.contains("Failed to get AWS credentials")); + assert!(message.contains("Glue credential endpoint unavailable")); + assert!(!message.contains("Encountered internal error")); + } + #[tokio::test] async fn test_injected_aws_creds_option_is_used() { let mock_provider = Arc::new(MockAwsCredentialsProvider::default()); @@ -514,6 +750,79 @@ mod tests { assert!(mock_provider.called.load(Ordering::Relaxed)); } + #[tokio::test] + async fn test_resolve_s3_region_from_aws_profile() { + let profile_config = load_test_profile( + "[profile selected]\n\ + region = us-west-004\n\ + endpoint_url = https://s3.us-west-004.backblazeb2.com\n\ + aws_access_key_id = test-key\n\ + aws_secret_access_key = test-secret", + ) + .await; + let url = Url::parse("s3://test-bucket/path").unwrap(); + + let resolved_s3_options = + ResolvedS3StorageOptions::new(HashMap::new(), Some(&profile_config)); + let region = resolve_s3_region(&url, &resolved_s3_options).await.unwrap(); + + assert_eq!(region.as_deref(), Some("us-west-004")); + assert_eq!( + resolved_s3_options + .options + .get(&AmazonS3ConfigKey::Endpoint), + Some(&"https://s3.us-west-004.backblazeb2.com".to_string()) + ); + + let explicit_options = HashMap::from([ + (AmazonS3ConfigKey::Region, "explicit-region".to_string()), + ( + AmazonS3ConfigKey::Endpoint, + "https://explicit.example.com".to_string(), + ), + ]); + let resolved_s3_options = + ResolvedS3StorageOptions::new(explicit_options, Some(&profile_config)); + let region = resolve_s3_region(&url, &resolved_s3_options).await.unwrap(); + + assert_eq!(region.as_deref(), Some("explicit-region")); + assert_eq!( + resolved_s3_options + .options + .get(&AmazonS3ConfigKey::Endpoint), + Some(&"https://explicit.example.com".to_string()) + ); + } + + #[tokio::test] + async fn test_region_only_aws_profile_preserves_bucket_discovery() { + let profile_config = load_test_profile("[profile selected]\nregion = us-east-1").await; + let resolved_s3_options = + ResolvedS3StorageOptions::new(HashMap::new(), Some(&profile_config)); + + let url = Url::parse("s3:///path").unwrap(); + let error = resolve_s3_region(&url, &resolved_s3_options) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("Could not parse bucket")); + } + + #[tokio::test] + async fn test_r2_aws_profile_requires_constant_size_upload_parts() { + let profile_config = load_test_profile( + "[profile selected]\n\ + region = auto\n\ + endpoint_url = https://account.r2.cloudflarestorage.com", + ) + .await; + let resolved_s3_options = + ResolvedS3StorageOptions::new(HashMap::new(), Some(&profile_config)); + + assert!(resolved_s3_options.requires_constant_size_upload_parts()); + } + #[test] fn test_s3_path_parsing() { let provider = AwsStoreProvider; @@ -618,6 +927,36 @@ mod tests { assert_eq!(store.scheme, "s3"); } + /// S3 Express ignores `start-after` and does not list in key order, but it does hand back + /// continuation tokens, which is all the native store resumes from — so an Express bucket + /// pages like any other. The OpenDAL arm has no lister to page with at all. + #[rstest::rstest] + #[case::native("false", true)] + #[case::opendal("true", false)] + #[tokio::test] + async fn test_s3_express_is_paged_by_continuation_token( + #[case] use_opendal: &str, + #[case] paginated: bool, + ) { + let provider = AwsStoreProvider; + // Express bucket names carry their availability zone, which the S3 client validates. + let url = Url::parse("s3://test-bucket--use1-az4--x-s3/path").unwrap(); + let params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([ + ("use_opendal".to_string(), use_opendal.to_string()), + ("region".to_string(), "us-west-2".to_string()), + ]), + ))), + ..Default::default() + }; + + let store = provider.new_store(url, ¶ms).await.unwrap(); + + assert!(!store.list_is_lexically_ordered); + assert_eq!(store.paginated_lister.is_some(), paginated); + } + #[derive(Debug)] struct MockStorageOptionsProvider { call_count: Arc>, @@ -1031,6 +1370,7 @@ mod tests { None, // no storage_options Some("us-west-2".to_string()), Some(accessor), + None, ) .await .unwrap(); @@ -1094,6 +1434,7 @@ mod tests { None, // no storage_options Some("us-west-2".to_string()), Some(accessor), + None, ) .await .unwrap(); @@ -1117,4 +1458,126 @@ mod tests { // Storage options provider should have been called once assert_eq!(mock_storage_provider.get_call_count().await, 1); } + + // Test that aws_provider_scheme=token selects static credentials. + #[tokio::test] + async fn test_provider_scheme_token() { + let opts = HashMap::from([ + (AmazonS3ConfigKey::AccessKeyId, "AKID".to_string()), + (AmazonS3ConfigKey::SecretAccessKey, "SECRET".to_string()), + ]); + + let (provider, _) = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Token), + ) + .await + .unwrap(); + + let cred = provider.get_credential().await.unwrap(); + assert_eq!(cred.key_id, "AKID"); + assert_eq!(cred.secret_key, "SECRET"); + } + + // Test that aws_provider_scheme=token errors when no static credentials are present. + #[tokio::test] + async fn test_provider_scheme_token_errors_without_credentials() { + let opts: HashMap = HashMap::new(); + + let result = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Token), + ) + .await; + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("aws_provider_scheme=token"), + "error should mention aws_provider_scheme=token" + ); + } + + // Test that aws_provider_scheme=ecs builds a provider without error. + // The ECS provider itself reads from env vars lazily; construction always succeeds. + #[tokio::test] + async fn test_provider_scheme_ecs() { + let opts: HashMap = HashMap::new(); + + let result = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Ecs), + ) + .await; + assert!(result.is_ok(), "ECS provider should build without error"); + } + + // Test that aws_provider_scheme=irsa builds a provider and attempts credential + // retrieval (which fails with a provider error, not a config error like + // "Missing Region" — confirming the region is wired through to the STS client). + #[tokio::test] + async fn test_provider_scheme_irsa() { + let opts: HashMap = HashMap::new(); + + let (provider, _) = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + Some(AwsProviderScheme::Irsa), + ) + .await + .unwrap(); + + // Credential retrieval must fail with a provider error (missing env vars or + // network), NOT a configuration error like "Invalid Configuration: Missing Region". + let err = provider.get_credential().await.unwrap_err(); + assert!( + !err.to_string().contains("Missing Region"), + "should not fail with Missing Region; region was provided. got: {err}" + ); + } + + // Test that an invalid aws_provider_scheme value produces a clear error. + #[test] + fn test_provider_scheme_invalid_value() { + let opts = StorageOptions::new(HashMap::from([( + "aws_provider_scheme".to_string(), + "magic".to_string(), + )])); + let result = opts.aws_provider_scheme(); + assert!(result.is_err()); + assert!(result.unwrap_err().to_string().contains("magic")); + } + + // Test that no aws_provider_scheme falls through to DefaultCredentialsChain without error. + #[tokio::test] + async fn test_no_provider_scheme_uses_default_chain() { + let opts: HashMap = HashMap::new(); + + let result = build_aws_credential( + Duration::from_secs(300), + None, + Some(&opts), + Some("us-east-1".to_string()), + None, + None, + ) + .await; + assert!(result.is_ok()); + } } diff --git a/rust/lance-io/src/object_store/providers/azure.rs b/rust/lance-io/src/object_store/providers/azure.rs index e61f3f3b364..2ad922fa241 100644 --- a/rust/lance-io/src/object_store/providers/azure.rs +++ b/rust/lance-io/src/object_store/providers/azure.rs @@ -9,22 +9,24 @@ use std::{ }; use object_store::ObjectStore as OSObjectStore; -use object_store_opendal::OpendalStore; +use object_store::list::PaginatedListStore; use opendal::{Operator, services::Azblob, services::Azdls}; use object_store::{ RetryConfig, - azure::{AzureConfigKey, AzureCredential, MicrosoftAzureBuilder}, + azure::{AzureConfigKey, AzureCredential, MicrosoftAzure, MicrosoftAzureBuilder}, }; use url::Url; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor, dynamic_credentials::build_dynamic_credential_provider, - throttle::{AimdThrottleConfig, AimdThrottledStore}, + throttle::{AimdThrottleConfig, AimdThrottleState, cloud_http_connector, with_throttling}, }; use lance_core::error::{Error, Result}; +use lance_core::utils::parse::str_is_truthy; #[derive(Default, Debug)] pub struct AzureBlobStoreProvider; @@ -106,14 +108,9 @@ impl AzureBlobStoreProvider { config_map.insert("root".to_string(), format!("/{}", prefix)); } - Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!( - "Failed to create Azure Blob operator: {:?}", - e - )) - }) - .map(|b| b.finish()) + Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!("Failed to create Azure Blob operator: {:?}", e)) + }) } "abfss" => { let filesystem = base_path.username(); @@ -139,14 +136,12 @@ impl AzureBlobStoreProvider { config_map.insert("root".to_string(), format!("/{}", root_path)); } - Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!( - "Failed to create Azure DFS (ADLS Gen2) operator: {:?}", - e - )) - }) - .map(|b| b.finish()) + Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!( + "Failed to create Azure DFS (ADLS Gen2) operator: {:?}", + e + )) + }) } _ => Err(Error::invalid_input(format!( "Unsupported Azure scheme: {}", @@ -169,7 +164,10 @@ impl AzureBlobStoreProvider { base_path: &Url, storage_options: &StorageOptions, accessor: Option>, - ) -> Result> { + throttle_state: Option<&AimdThrottleState>, + // Concrete rather than `dyn`, so the caller keeps the handle a paginated listing + // needs: `PaginatedListStore` is a separate trait from `ObjectStore`. + ) -> Result> { // Use a low retry count since the AIMD throttle layer handles // throttle recovery with its own retry loop. let retry_config = RetryConfig { @@ -192,7 +190,11 @@ impl AzureBlobStoreProvider { builder = builder.with_credentials(credentials); } - Ok(Arc::new(builder.build()?) as Arc) + let store_prefix = + self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?; + builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix)); + + Ok(Arc::new(builder.build()?)) } fn calculate_object_store_prefix_with_env( @@ -252,29 +254,47 @@ impl ObjectStoreProvider for AzureBlobStoreProvider { let use_opendal = storage_options .0 .get("use_opendal") - .map(|v| v.as_str() == "true") + .map(|v| str_is_truthy(v.as_str())) .unwrap_or(false); let accessor = params.get_accessor(); - let inner: Arc = if use_opendal { - // OpenDAL Azure intentionally uses static/environment-backed configuration only. - // Namespace-vended dynamic credentials are supported on the native object_store path. - self.build_opendal_azure_store(&base_path, &storage_options) - .await? + let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; + let throttle_state = if throttle_config.is_disabled() { + None } else { - self.build_microsoft_azure_store(&base_path, &storage_options, accessor) - .await? + Some(AimdThrottleState::new(throttle_config)?) }; - let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; - let inner = if throttle_config.is_disabled() { - inner + + let (inner, paginated_lister) = if use_opendal { + // OpenDAL Azure intentionally uses static/environment-backed configuration only. + // Namespace-vended dynamic credentials are supported on the native object_store path. + // Listed in full: no paginated lister covers OpenDAL yet. + ( + self.build_opendal_azure_store(&base_path, &storage_options) + .await?, + None, + ) } else { - Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc + let store = self + .build_microsoft_azure_store( + &base_path, + &storage_options, + accessor, + throttle_state.as_ref(), + ) + .await?; + ( + store.clone() as Arc, + Some(store as Arc), + ) }; + let (inner, paginated_lister) = + with_throttling(throttle_state, !use_opendal, inner, paginated_lister); Ok(ObjectStore { inner, + local_dir_operations: None, scheme, block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -285,6 +305,7 @@ impl ObjectStoreProvider for AzureBlobStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + paginated_lister, }) } @@ -565,6 +586,29 @@ mod tests { "abfss:// without use_opendal should use MicrosoftAzureBuilder, got: {}", inner_desc ); + assert!( + store.paginated_lister.is_some(), + "the native store pages an ADLS Gen2 account by continuation token" + ); + } + + #[tokio::test] + async fn test_a_blob_container_is_paged() { + use crate::object_store::StorageOptionsAccessor; + let provider = AzureBlobStoreProvider; + let url = Url::parse("az://container@testaccount.blob.core.windows.net/data").unwrap(); + let params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([ + ("account_name".to_string(), "testaccount".to_string()), + ("account_key".to_string(), "dGVzdA==".to_string()), + ]), + ))), + ..Default::default() + }; + + let store = provider.new_store(url, ¶ms).await.unwrap(); + assert!(store.paginated_lister.is_some()); } #[tokio::test] @@ -642,8 +686,8 @@ mod tests { let abfss_operator = AzureBlobStoreProvider::build_opendal_operator(&abfss_url, &common_opts).unwrap(); - let azblob_cap = az_operator.info().native_capability(); - let azdls_cap = abfss_operator.info().native_capability(); + let azblob_cap = az_operator.info().capability(); + let azdls_cap = abfss_operator.info().capability(); // Both support basic operations assert!(azblob_cap.read); diff --git a/rust/lance-io/src/object_store/providers/gcp.rs b/rust/lance-io/src/object_store/providers/gcp.rs index f7f0a7672ff..d64462a013d 100644 --- a/rust/lance-io/src/object_store/providers/gcp.rs +++ b/rust/lance-io/src/object_store/providers/gcp.rs @@ -3,27 +3,205 @@ use std::{collections::HashMap, str::FromStr, sync::Arc, time::Duration}; -use object_store::ObjectStore as OSObjectStore; -use object_store_opendal::OpendalStore; +use object_store::list::PaginatedListStore; +use object_store::{ + ClientOptions, CredentialProvider, ObjectStore as OSObjectStore, Result as ObjectStoreResult, + client::{HttpClient, HttpConnector, HttpRequestBody, ReqwestConnector}, +}; use opendal::{Operator, services::Gcs}; +use reqsign_core::{Context as ReqsignContext, HttpSend, OsEnv, ProvideCredential}; +use reqsign_file_read_tokio::TokioFileRead; +use reqsign_google::{Credential as ReqsignCredential, FileCredentialProvider}; +use tokio::sync::RwLock; use object_store::{ RetryConfig, StaticCredentialProvider, - gcp::{GcpCredential, GoogleCloudStorageBuilder, GoogleConfigKey}, + gcp::{GcpCredential, GoogleCloudStorage, GoogleCloudStorageBuilder, GoogleConfigKey}, }; use url::Url; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor, dynamic_credentials::build_dynamic_credential_provider, - throttle::{AimdThrottleConfig, AimdThrottledStore}, + throttle::{AimdThrottleConfig, AimdThrottleState, cloud_http_connector, with_throttling}, }; use lance_core::error::{Error, Result}; - +use lance_core::utils::parse::str_is_truthy; #[derive(Default, Debug)] pub struct GcsStoreProvider; +#[derive(Debug)] +struct ObjectStoreHttpSend { + client: HttpClient, +} + +impl HttpSend for ObjectStoreHttpSend { + async fn http_send( + &self, + request: http::Request, + ) -> reqsign_core::Result> { + let (parts, body) = request.into_parts(); + let request = http::Request::from_parts(parts, HttpRequestBody::from(body)); + let response = self.client.execute(request).await.map_err(|source| { + reqsign_core::Error::unexpected("failed to send Google workload identity HTTP request") + .with_source(source) + })?; + let (parts, body) = response.into_parts(); + let body = body.bytes().await.map_err(|source| { + reqsign_core::Error::unexpected("failed to read Google workload identity HTTP response") + .with_source(source) + })?; + Ok(http::Response::from_parts(parts, body)) + } +} + +#[derive(Debug)] +struct WorkloadIdentityCredentialProvider { + provider: FileCredentialProvider, + context: ReqsignContext, + cached_credential: RwLock>, +} + +impl WorkloadIdentityCredentialProvider { + fn new(application_credentials_path: String, http_client: HttpClient) -> Self { + let context = ReqsignContext::new() + .with_file_read(TokioFileRead) + .with_http_send(ObjectStoreHttpSend { + client: http_client, + }) + .with_env(OsEnv); + Self { + provider: FileCredentialProvider::new(application_credentials_path), + context, + cached_credential: RwLock::new(None), + } + } +} + +fn usable_gcp_credential(credential: &ReqsignCredential) -> Option { + credential + .token + .as_ref() + .filter(|_| credential.has_valid_token()) + .map(|token| GcpCredential { + bearer: token.access_token.clone(), + }) +} + +fn workload_identity_error( + source: impl std::error::Error + Send + Sync + 'static, +) -> object_store::Error { + object_store::Error::Generic { + store: "GCS workload identity credentials", + source: Box::new(source), + } +} + +#[async_trait::async_trait] +impl CredentialProvider for WorkloadIdentityCredentialProvider { + type Credential = GcpCredential; + + async fn get_credential(&self) -> ObjectStoreResult> { + if let Some(credential) = self + .cached_credential + .read() + .await + .as_ref() + .and_then(usable_gcp_credential) + { + return Ok(Arc::new(credential)); + } + + let mut cached_credential = self.cached_credential.write().await; + if let Some(credential) = cached_credential.as_ref().and_then(usable_gcp_credential) { + return Ok(Arc::new(credential)); + } + + let credential = self + .provider + .provide_credential(&self.context) + .await + .map_err(workload_identity_error)? + .ok_or_else(|| { + workload_identity_error(std::io::Error::other( + "application credentials did not provide a Google access token", + )) + })?; + let gcp_credential = usable_gcp_credential(&credential).ok_or_else(|| { + workload_identity_error(std::io::Error::other( + "application credentials provided an expired or unusable Google access token", + )) + })?; + *cached_credential = Some(credential); + Ok(Arc::new(gcp_credential)) + } +} + +#[derive(serde::Deserialize)] +struct ApplicationCredentialKind { + #[serde(rename = "type")] + credential_type: String, +} + +struct GcsClientOptions { + object_requests: ClientOptions, + credential_requests: ClientOptions, +} + +fn gcs_client_options(storage_options: &StorageOptions) -> Result { + let mut object_requests = storage_options.client_options()?; + // headers.* options are scoped to object requests and may contain secrets. Credential + // exchanges can target unrelated identity endpoints, so only share typed client settings. + let mut credential_requests = object_requests + .clone() + .with_default_headers(Default::default()); + for (key, value) in storage_options.as_gcs_options() { + if let GoogleConfigKey::Client(key) = key { + object_requests = object_requests.with_config(key, value.clone()); + credential_requests = credential_requests.with_config(key, value); + } + } + Ok(GcsClientOptions { + object_requests, + credential_requests, + }) +} + +fn workload_identity_credential_provider( + storage_options: &StorageOptions, + client_options: &ClientOptions, +) -> Result>>> { + let gcs_options = storage_options.as_gcs_options(); + if gcs_options.contains_key(&GoogleConfigKey::ServiceAccount) + || gcs_options.contains_key(&GoogleConfigKey::ServiceAccountKey) + { + return Ok(None); + } + + let Some(application_credentials_path) = + gcs_options.get(&GoogleConfigKey::ApplicationCredentials) + else { + return Ok(None); + }; + let Ok(contents) = std::fs::read(application_credentials_path) else { + return Ok(None); + }; + let Ok(credential_kind) = serde_json::from_slice::(&contents) else { + return Ok(None); + }; + if credential_kind.credential_type != "external_account" { + return Ok(None); + } + + let http_client = ReqwestConnector::default().connect(client_options)?; + Ok(Some(Arc::new(WorkloadIdentityCredentialProvider::new( + application_credentials_path.clone(), + http_client, + )))) +} + impl GcsStoreProvider { async fn build_opendal_gcs_store( &self, @@ -49,8 +227,7 @@ impl GcsStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))?; Ok(Arc::new(OpendalStore::new(operator)) as Arc) } @@ -60,7 +237,10 @@ impl GcsStoreProvider { base_path: &Url, storage_options: &StorageOptions, accessor: Option>, - ) -> Result> { + throttle_state: Option<&AimdThrottleState>, + // Concrete rather than `dyn`, so the caller keeps the handle a paginated listing + // needs: `PaginatedListStore` is a separate trait from `ObjectStore`. + ) -> Result> { // Use a low retry count since the AIMD throttle layer handles // throttle recovery with its own retry loop. let retry_config = RetryConfig { @@ -69,10 +249,12 @@ impl GcsStoreProvider { retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()), }; + let client_options = gcs_client_options(storage_options)?; + let mut builder = GoogleCloudStorageBuilder::new() .with_url(base_path.as_ref()) .with_retry(retry_config) - .with_client_options(storage_options.client_options()?); + .with_client_options(client_options.object_requests.clone()); for (key, value) in storage_options.as_gcs_options() { builder = builder.with_config(key, value); } @@ -87,9 +269,20 @@ impl GcsStoreProvider { }; let credential_provider = Arc::new(StaticCredentialProvider::new(credential)) as _; builder = builder.with_credentials(credential_provider); + } else if let Some(credential_provider) = workload_identity_credential_provider( + storage_options, + &client_options.credential_requests, + )? { + // object_store cannot exchange external-account ADC files, while reqsign supports + // the workload identity format emitted by google-github-actions/auth. + builder = builder.with_credentials(credential_provider); } - Ok(Arc::new(builder.build()?) as Arc) + let store_prefix = + self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?; + builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix)); + + Ok(Arc::new(builder.build()?)) } } @@ -105,29 +298,47 @@ impl ObjectStoreProvider for GcsStoreProvider { let use_opendal = storage_options .0 .get("use_opendal") - .map(|v| v.as_str() == "true") + .map(|v| str_is_truthy(v.as_str())) .unwrap_or(false); let accessor = params.get_accessor(); - let inner = if use_opendal { - // OpenDAL GCS intentionally uses static/environment-backed configuration only. - // Namespace-vended dynamic credentials are supported on the native object_store path. - self.build_opendal_gcs_store(&base_path, &storage_options) - .await? + let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; + let throttle_state = if throttle_config.is_disabled() { + None } else { - self.build_google_cloud_store(&base_path, &storage_options, accessor) - .await? + Some(AimdThrottleState::new(throttle_config)?) }; - let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?; - let inner = if throttle_config.is_disabled() { - inner + + let (inner, paginated_lister) = if use_opendal { + // OpenDAL GCS intentionally uses static/environment-backed configuration only. + // Namespace-vended dynamic credentials are supported on the native object_store path. + // Listed in full: no paginated lister covers OpenDAL yet. + ( + self.build_opendal_gcs_store(&base_path, &storage_options) + .await?, + None, + ) } else { - Arc::new(AimdThrottledStore::new(inner, throttle_config)?) as Arc + let store = self + .build_google_cloud_store( + &base_path, + &storage_options, + accessor, + throttle_state.as_ref(), + ) + .await?; + ( + store.clone() as Arc, + Some(store as Arc), + ) }; + let (inner, paginated_lister) = + with_throttling(throttle_state, !use_opendal, inner, paginated_lister); Ok(ObjectStore { inner, + local_dir_operations: None, scheme: String::from("gs"), block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -138,6 +349,7 @@ impl ObjectStoreProvider for GcsStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + paginated_lister, }) } } @@ -179,11 +391,47 @@ impl StorageOptions { #[cfg(test)] mod tests { use super::*; - use std::sync::Arc; + use std::{collections::HashMap, fs, sync::Arc}; use crate::object_store::test_utils::StaticMockStorageOptionsProvider; use crate::object_store::{ObjectStoreParams, StorageOptionsAccessor}; - use std::collections::HashMap; + use tempfile::TempDir; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }; + + fn external_account_storage_options( + temp_dir: &TempDir, + token_url: String, + ) -> HashMap { + let subject_token_path = temp_dir.path().join("oidc-token"); + fs::write(&subject_token_path, "github-oidc-token").unwrap(); + let application_credentials_path = temp_dir.path().join("credentials.json"); + fs::write( + &application_credentials_path, + serde_json::to_vec(&serde_json::json!({ + "type": "external_account", + "audience": "test-audience", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": token_url, + "credential_source": { + "file": subject_token_path.to_string_lossy(), + "format": { "type": "text" } + } + })) + .unwrap(), + ) + .unwrap(); + + HashMap::from([ + ( + "google_application_credentials".to_string(), + application_credentials_path.to_string_lossy().into_owned(), + ), + ("allow_http".to_string(), "true".to_string()), + ]) + } #[test] fn test_gcs_store_path() { @@ -240,4 +488,104 @@ mod tests { assert_eq!(credentials.bearer, "gcp-token"); } + + #[tokio::test] + async fn test_external_account_application_credentials() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "access_token": "federated-token", + "expires_in": 3600 + }))) + .expect(1) + .mount(&mock_server) + .await; + + let temp_dir = tempfile::tempdir().unwrap(); + let mut storage_options = + external_account_storage_options(&temp_dir, format!("{}/token", mock_server.uri())); + storage_options.insert( + "headers.Authorization".to_string(), + "Bearer storage-secret".to_string(), + ); + let params = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + storage_options.clone(), + ))), + ..Default::default() + }; + + let store = GcsStoreProvider + .new_store(Url::parse("gs://test-bucket/path").unwrap(), ¶ms) + .await + .expect("external account credentials should build a GCS store"); + assert_eq!(store.scheme, "gs"); + + let storage_options = StorageOptions::new(storage_options); + let client_options = gcs_client_options(&storage_options).unwrap(); + let credential_provider = workload_identity_credential_provider( + &storage_options, + &client_options.credential_requests, + ) + .expect("external account credential provider should build") + .expect("external account credentials should select the reqsign provider"); + for _ in 0..2 { + let credential = credential_provider + .get_credential() + .await + .expect("workload identity token exchange should succeed"); + assert_eq!(credential.bearer, "federated-token"); + } + mock_server.verify().await; + let requests = mock_server.received_requests().await.unwrap(); + assert!(!requests[0].headers.contains_key("authorization")); + } + + #[tokio::test] + async fn test_external_account_respects_client_timeout() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_secs(1)) + .set_body_json(serde_json::json!({ + "access_token": "federated-token", + "expires_in": 3600 + })), + ) + .expect(1) + .mount(&mock_server) + .await; + + let temp_dir = tempfile::tempdir().unwrap(); + let mut storage_options = + external_account_storage_options(&temp_dir, format!("{}/token", mock_server.uri())); + storage_options.insert("timeout".to_string(), "50ms".to_string()); + let storage_options = StorageOptions::new(storage_options); + let client_options = gcs_client_options(&storage_options).unwrap(); + let credential_provider = workload_identity_credential_provider( + &storage_options, + &client_options.credential_requests, + ) + .expect("external account credential provider should build") + .expect("external account credentials should select the reqsign provider"); + + let credential_result = tokio::time::timeout( + Duration::from_millis(200), + credential_provider.get_credential(), + ) + .await + .expect("configured client timeout should bound the credential exchange"); + let error = credential_result.expect_err("the delayed token exchange should time out"); + assert!(matches!(&error, object_store::Error::Generic { .. })); + assert!( + error + .to_string() + .contains("failed to send Google workload identity HTTP request"), + "unexpected error: {error}" + ); + mock_server.verify().await; + } } diff --git a/rust/lance-io/src/object_store/providers/goosefs.rs b/rust/lance-io/src/object_store/providers/goosefs.rs index 5d2a648522b..dcc11c8c9d7 100644 --- a/rust/lance-io/src/object_store/providers/goosefs.rs +++ b/rust/lance-io/src/object_store/providers/goosefs.rs @@ -4,10 +4,10 @@ use std::collections::HashMap; use std::sync::Arc; -use object_store_opendal::OpendalStore; use opendal::{Operator, services::GooseFs}; use url::Url; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, @@ -171,11 +171,9 @@ impl ObjectStoreProvider for GooseFsStoreProvider { } // Create OpenDAL Operator with GooseFS service - let operator = Operator::from_iter::(config_map) - .map_err(|e| { - Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e)) - })? - .finish(); + let operator = Operator::from_iter::(config_map).map_err(|e| { + Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e)) + })?; // Wrap as object_store::ObjectStore via OpendalStore bridge let opendal_store = Arc::new(OpendalStore::new(operator)); @@ -183,6 +181,7 @@ impl ObjectStoreProvider for GooseFsStoreProvider { Ok(ObjectStore { scheme: "goosefs".to_string(), inner: opendal_store, + local_dir_operations: None, block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, use_constant_size_upload_parts: params.use_constant_size_upload_parts, @@ -192,6 +191,8 @@ impl ObjectStoreProvider for GooseFsStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + // Listed in full: no paginated lister covers OpenDAL yet. + paginated_lister: None, }) } diff --git a/rust/lance-io/src/object_store/providers/huggingface.rs b/rust/lance-io/src/object_store/providers/huggingface.rs index cfef0440068..02ff7b8ae9f 100644 --- a/rust/lance-io/src/object_store/providers/huggingface.rs +++ b/rust/lance-io/src/object_store/providers/huggingface.rs @@ -6,11 +6,11 @@ use std::sync::Arc; use object_store::ObjectStore as OSObjectStore; use object_store::path::Path; -use object_store_opendal::OpendalStore; use opendal::{Operator, services::Huggingface}; use url::Url; use crate::object_store::dynamic_opendal::DynamicOpenDalStore; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::parse_hf_repo_id; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, @@ -160,11 +160,9 @@ fn build_hf_store(config_map: HashMap) -> Result { builder = builder.download_mode(download_mode); } - let operator = Operator::new(builder) - .map_err(|e| { - Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e)) - })? - .finish(); + let operator = Operator::new(builder).map_err(|e| { + Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e)) + })?; Ok(OpendalStore::new(operator)) } @@ -209,6 +207,7 @@ impl ObjectStoreProvider for HuggingfaceStoreProvider { Ok(ObjectStore { scheme: "hf".to_string(), inner, + local_dir_operations: None, block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, use_constant_size_upload_parts: params.use_constant_size_upload_parts, @@ -218,6 +217,8 @@ impl ObjectStoreProvider for HuggingfaceStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + // Listed in full: no paginated lister covers OpenDAL yet. + paginated_lister: None, }) } diff --git a/rust/lance-io/src/object_store/providers/local.rs b/rust/lance-io/src/object_store/providers/local.rs index 9f0762916f7..4dd834c9151 100644 --- a/rust/lance-io/src/object_store/providers/local.rs +++ b/rust/lance-io/src/object_store/providers/local.rs @@ -3,6 +3,8 @@ use std::{collections::HashMap, sync::Arc}; +#[cfg(any(windows, test))] +use crate::object_store::LocalDirOperations; use crate::object_store::{ DEFAULT_LOCAL_BLOCK_SIZE, DEFAULT_LOCAL_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, @@ -10,19 +12,122 @@ use crate::object_store::{ use lance_core::Error; use lance_core::error::Result; use object_store::{local::LocalFileSystem, path::Path}; +#[cfg(any(windows, test))] +use std::io::ErrorKind; use url::Url; #[derive(Default, Debug)] pub struct FileStoreProvider; +#[cfg(any(windows, test))] +#[derive(Debug)] +struct FileSystemDirOperations { + local_file_system: LocalFileSystem, +} + +#[cfg(any(windows, test))] +#[async_trait::async_trait] +impl LocalDirOperations for FileSystemDirOperations { + async fn remove_dir_all(&self, path: &Path) -> Result<()> { + let local_path = self.local_file_system.path_to_filesystem(path)?; + let object_store_path = path.to_string(); + tokio::task::spawn_blocking(move || { + std::fs::remove_dir_all(local_path).map_err(|error| match error.kind() { + ErrorKind::NotFound => Error::not_found(object_store_path), + _ => Error::from(error), + }) + }) + .await + .map_err(|error| Error::io(format!("recursive directory removal task failed: {error}")))? + } +} + +#[cfg(windows)] +mod windows { + use std::path::PathBuf; + + use super::*; + + #[derive(Debug)] + pub(super) struct UncPath { + pub(super) root: PathBuf, + pub(super) relative_path: Path, + pub(super) store_prefix: String, + } + + pub(super) fn extract_unc_path(url: &Url) -> Result> { + if url.scheme() != "file" { + return Ok(None); + } + + let Some(host) = url.host_str().filter(|host| *host != "localhost") else { + return Ok(None); + }; + let encoded_path = url.path().strip_prefix('/').unwrap_or(url.path()); + let (encoded_share, relative_path) = + encoded_path.split_once('/').unwrap_or((encoded_path, "")); + if encoded_share.is_empty() { + return Err(Error::invalid_input(format!( + "UNC URL '{}' is missing a share name", + url + ))); + } + + let share = Path::from_url_path(encoded_share).map_err(|error| { + Error::invalid_input(format!( + "Failed to parse share name from UNC URL '{}': {}", + url, error + )) + })?; + if share.parts_count() != 1 || share.as_ref().contains('\\') { + return Err(Error::invalid_input(format!( + "UNC URL '{}' has an invalid share name", + url + ))); + } + + Ok(Some(UncPath { + root: PathBuf::from(format!(r"\\{}\{}", host, share)), + relative_path: Path::from_url_path(relative_path).map_err(|error| { + Error::invalid_input(format!( + "Failed to parse path '{}' from UNC URL '{}': {}", + relative_path, url, error + )) + })?, + store_prefix: format!("{}${}/{}", url.scheme(), host, encoded_share), + })) + } +} + #[async_trait::async_trait] impl ObjectStoreProvider for FileStoreProvider { async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result { let block_size = params.block_size.unwrap_or(DEFAULT_LOCAL_BLOCK_SIZE); let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default()); let download_retry_count = storage_options.download_retry_count(); + + #[cfg(windows)] + let (inner, local_dir_operations) = match windows::extract_unc_path(&base_path)? { + Some(unc_path) => { + let inner = LocalFileSystem::new_with_prefix(unc_path.root)?; + let operations = FileSystemDirOperations { + local_file_system: inner.clone(), + }; + ( + inner, + Some(Arc::new(operations) as Arc), + ) + } + None => (LocalFileSystem::new(), None), + }; + #[cfg(not(windows))] + let inner = LocalFileSystem::new(); + #[cfg(not(windows))] + let local_dir_operations = None; + Ok(ObjectStore { - inner: Arc::new(LocalFileSystem::new()), + inner: Arc::new(inner), + local_dir_operations, scheme: base_path.scheme().to_owned(), block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -33,10 +138,17 @@ impl ObjectStoreProvider for FileStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + // Listed in full: reading a directory is one local walk whatever the page size, + // so there is no request for a page to be pushed into. + paginated_lister: None, }) } fn extract_path(&self, url: &Url) -> Result { + #[cfg(windows)] + if let Some(unc_path) = windows::extract_unc_path(url)? { + return Ok(unc_path.relative_path); + } if let Ok(file_path) = url.to_file_path() && let Ok(path) = Path::from_absolute_path(&file_path) { @@ -53,16 +165,89 @@ impl ObjectStoreProvider for FileStoreProvider { url: &Url, _storage_options: Option<&HashMap>, ) -> Result { + #[cfg(windows)] + if let Some(unc_path) = windows::extract_unc_path(url)? { + return Ok(unc_path.store_prefix); + } + Ok(url.scheme().to_string()) } } #[cfg(test)] mod tests { + use std::fs::{create_dir_all, write}; + use std::path::Path as StdPath; + use crate::object_store::uri_to_url; + #[cfg(unix)] + use std::os::unix::fs::symlink; + use tempfile::tempdir; use super::*; + fn rooted_local_store(root: &StdPath) -> ObjectStore { + let inner = LocalFileSystem::new_with_prefix(root).unwrap(); + let local_dir_operations = Arc::new(FileSystemDirOperations { + local_file_system: inner.clone(), + }); + ObjectStore { + inner: Arc::new(inner), + local_dir_operations: Some(local_dir_operations), + scheme: "file".to_owned(), + block_size: DEFAULT_LOCAL_BLOCK_SIZE, + max_iop_size: *DEFAULT_MAX_IOP_SIZE, + use_constant_size_upload_parts: false, + list_is_lexically_ordered: false, + io_parallelism: DEFAULT_LOCAL_IO_PARALLELISM, + download_retry_count: 0, + io_tracker: Default::default(), + store_prefix: "file$rooted-test".to_owned(), + paginated_lister: None, + } + } + + #[tokio::test] + async fn test_rooted_remove_dir_all_removes_tree() { + let sandbox = tempdir().unwrap(); + let root = sandbox.path().join("share"); + let dataset = root.join("dataset"); + create_dir_all(dataset.join("nested")).unwrap(); + write(dataset.join("nested/data"), "delete").unwrap(); + + rooted_local_store(&root) + .remove_dir_all(Path::from("dataset")) + .await + .unwrap(); + + assert!(!dataset.exists(), "recursive deletion must remove the tree"); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_rooted_remove_dir_all_does_not_follow_directory_symlink() { + let sandbox = tempdir().unwrap(); + let root = sandbox.path().join("share"); + let dataset = root.join("dataset"); + let outside = sandbox.path().join("outside"); + create_dir_all(&dataset).unwrap(); + create_dir_all(&outside).unwrap(); + let sentinel = outside.join("sentinel"); + write(&sentinel, "keep").unwrap(); + symlink(&outside, dataset.join("link")).unwrap(); + + rooted_local_store(&root) + .remove_dir_all(Path::from("dataset")) + .await + .unwrap(); + + assert!( + sentinel.exists(), + "recursive deletion must not follow links" + ); + assert!(!dataset.exists(), "recursive deletion must remove the tree"); + } + #[test] fn test_file_store_path() { let provider = FileStoreProvider; @@ -128,6 +313,10 @@ mod tests { "file:///C:/Users/RUNNER~1/AppData/Local/Temp/tmpm49j_w0f", "C:/Users/RUNNER~1/AppData/Local/Temp/tmpm49j_w0f", ), + ( + "file://192.168.0.1/My%20Share/data/my-dataset.lance", + "data/my-dataset.lance", + ), ]; for (uri, expected_path) in cases { @@ -136,4 +325,26 @@ mod tests { assert_eq!(path.as_ref(), expected_path); } } + + #[test] + #[cfg(windows)] + fn test_unc_share_path() { + let url = Url::parse("file://server/My%20Share/data/my-dataset.lance").unwrap(); + let unc_path = windows::extract_unc_path(&url).unwrap().unwrap(); + + assert_eq!( + unc_path.root, + std::path::PathBuf::from(r"\\server\My Share") + ); + assert_eq!(unc_path.relative_path.as_ref(), "data/my-dataset.lance"); + assert_eq!(unc_path.store_prefix, "file$server/My%20Share"); + + let object_store_url = + Url::parse("file-object-store://server/My%20Share/data/my-dataset.lance").unwrap(); + assert!( + windows::extract_unc_path(&object_store_url) + .unwrap() + .is_none() + ); + } } diff --git a/rust/lance-io/src/object_store/providers/memory.rs b/rust/lance-io/src/object_store/providers/memory.rs index dd72edc4627..f9b4a22e2cc 100644 --- a/rust/lance-io/src/object_store/providers/memory.rs +++ b/rust/lance-io/src/object_store/providers/memory.rs @@ -23,6 +23,7 @@ impl ObjectStoreProvider for MemoryStoreProvider { let download_retry_count = storage_options.download_retry_count(); Ok(ObjectStore { inner: Arc::new(InMemory::new()), + local_dir_operations: None, scheme: String::from("memory"), block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, @@ -33,6 +34,9 @@ impl ObjectStoreProvider for MemoryStoreProvider { io_tracker: Default::default(), store_prefix: self .calculate_object_store_prefix(&base_path, params.storage_options())?, + // Listed in full: the store is already in memory, so a page costs no less than + // the directory does. + paginated_lister: None, }) } diff --git a/rust/lance-io/src/object_store/providers/oss.rs b/rust/lance-io/src/object_store/providers/oss.rs index b84afb8ed1f..0adf52db0a2 100644 --- a/rust/lance-io/src/object_store/providers/oss.rs +++ b/rust/lance-io/src/object_store/providers/oss.rs @@ -5,11 +5,11 @@ use std::collections::HashMap; use std::sync::Arc; use object_store::ObjectStore as OSObjectStore; -use object_store_opendal::OpendalStore; use opendal::{Operator, services::Oss}; use url::Url; use crate::object_store::dynamic_opendal::DynamicOpenDalStore; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, @@ -95,8 +95,7 @@ impl OssStoreProvider { fn build_oss_store(config_map: HashMap) -> Result { let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))?; Ok(OpendalStore::new(operator)) } @@ -137,6 +136,7 @@ impl ObjectStoreProvider for OssStoreProvider { Ok(ObjectStore { scheme: "oss".to_string(), inner, + local_dir_operations: None, block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, use_constant_size_upload_parts: params.use_constant_size_upload_parts, @@ -145,6 +145,8 @@ impl ObjectStoreProvider for OssStoreProvider { download_retry_count: storage_options.download_retry_count(), io_tracker: Default::default(), store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?, + // Listed in full: no paginated lister covers OpenDAL yet. + paginated_lister: None, }) } } diff --git a/rust/lance-io/src/object_store/providers/tencent.rs b/rust/lance-io/src/object_store/providers/tencent.rs index 5fa885ea5a9..9ad0a91765d 100644 --- a/rust/lance-io/src/object_store/providers/tencent.rs +++ b/rust/lance-io/src/object_store/providers/tencent.rs @@ -4,10 +4,10 @@ use std::collections::HashMap; use std::sync::Arc; -use object_store_opendal::OpendalStore; use opendal::{Operator, services::Cos}; use url::Url; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, @@ -80,8 +80,7 @@ impl ObjectStoreProvider for TencentStoreProvider { } let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))?; let opendal_store = Arc::new(OpendalStore::new(operator)); @@ -93,6 +92,7 @@ impl ObjectStoreProvider for TencentStoreProvider { Ok(ObjectStore { scheme: "cos".to_string(), inner: opendal_store, + local_dir_operations: None, block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, use_constant_size_upload_parts: params.use_constant_size_upload_parts, @@ -101,6 +101,8 @@ impl ObjectStoreProvider for TencentStoreProvider { download_retry_count: storage_options.download_retry_count(), io_tracker: Default::default(), store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?, + // Listed in full: no paginated lister covers OpenDAL yet. + paginated_lister: None, }) } } diff --git a/rust/lance-io/src/object_store/providers/tos.rs b/rust/lance-io/src/object_store/providers/tos.rs index 923186484c6..9f558e92550 100644 --- a/rust/lance-io/src/object_store/providers/tos.rs +++ b/rust/lance-io/src/object_store/providers/tos.rs @@ -5,11 +5,11 @@ use std::collections::HashMap; use std::sync::Arc; use object_store::ObjectStore as OSObjectStore; -use object_store_opendal::OpendalStore; use opendal::{Operator, services::Tos}; use url::Url; use crate::object_store::dynamic_opendal::DynamicOpenDalStore; +use crate::object_store::opendal_store::OpendalStore; use crate::object_store::{ DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore, ObjectStoreParams, ObjectStoreProvider, StorageOptions, @@ -101,8 +101,7 @@ impl TosStoreProvider { fn build_tos_store(config_map: HashMap) -> Result { let operator = Operator::from_iter::(config_map) - .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))? - .finish(); + .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))?; Ok(OpendalStore::new(operator)) } @@ -143,6 +142,7 @@ impl ObjectStoreProvider for TosStoreProvider { Ok(ObjectStore { scheme: "tos".to_string(), inner, + local_dir_operations: None, block_size, max_iop_size: *DEFAULT_MAX_IOP_SIZE, use_constant_size_upload_parts: params.use_constant_size_upload_parts, @@ -151,6 +151,8 @@ impl ObjectStoreProvider for TosStoreProvider { download_retry_count: storage_options.download_retry_count(), io_tracker: Default::default(), store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?, + // Listed in full: no paginated lister covers OpenDAL yet. + paginated_lister: None, }) } } diff --git a/rust/lance-io/src/object_store/read_dir.rs b/rust/lance-io/src/object_store/read_dir.rs new file mode 100644 index 00000000000..9be5a805d62 --- /dev/null +++ b/rust/lance-io/src/object_store/read_dir.rs @@ -0,0 +1,643 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Paginated listing of a single directory level. +//! +//! [`ObjectStore::read_dir_page`] returns one page of the immediate children of a prefix, plus +//! a token that resumes after it. Where the backend implements `object_store`'s paginated list +//! API — S3, GCS and Azure do — the page size and the resume position are pushed into the list +//! request, so a caller that wants the first few children pays for the first few children +//! rather than for the whole prefix. Everything else lists the level in full and pages +//! locally, which is correct but costs what the whole directory costs. +//! +//! The token is opaque, and a caller only ever hands it back: it is the backend's own +//! continuation token where there is one, and the last key of the page where there is not. +//! That is what lets a store without key-ordered listings, such as S3 Express, be paged at +//! all — nothing outside this module compares one token to another. + +use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; +use object_store::{ListResult, ObjectMeta, ObjectStore as OSObjectStore, path::Path}; +use tracing::instrument; + +use lance_core::{Error, Result}; + +use super::ObjectStore; + +#[cfg(feature = "metrics")] +use crate::object_store::metrics::{InFlightGuard, record_outcome}; +#[cfg(feature = "metrics")] +use std::time::Instant; + +/// The path delimiter that separates directory levels. +const DELIMITER: &str = "/"; + +/// Operation label for the metrics and IO statistics a paginated listing records. +const LIST_OP: &str = "list_paginated"; + +/// Options for [`ObjectStore::read_dir_page`]. +#[derive(Debug, Clone, Default)] +pub struct ReadDirOptions { + /// Resume after the page a previous call returned, using the token it handed back. + /// + /// A token means something only to the store that minted it, and only for the directory + /// it was minted over. Handing one to a different store resumes from the wrong place + /// rather than failing. + pub page_token: Option, + /// The page size to ask the backend for. Must be at least one. `None` lets the backend + /// return as much as it will. + pub limit: Option, +} + +impl ObjectStore { + /// One page of the immediate children of `dir`, one directory level deep. + /// + /// On backends with a paginated list API — S3, GCS and Azure — the resume position and the + /// page size are pushed into the list request, so the page costs what the page holds. + /// Elsewhere the directory is listed in full and paged locally, which is correct but no + /// cheaper than [`Self::read_dir`]. + /// + /// Child directories come back as [`ListResult::common_prefixes`] and child objects as + /// [`ListResult::objects`], the same split [`Self::list_with_delimiter`] returns. + /// + /// One page is one request, so a page can hold fewer children than `limit` asked for and + /// still be followed by more: with a delimiter a backend spends its page budget on keys it + /// collapses away, and it has a cap of its own besides. Walk until + /// [`PaginatedListResult::page_token`] is `None` rather than until a page comes back short. + /// + /// ``` + /// # use lance_io::object_store::{ObjectStore, ReadDirOptions}; + /// # async fn example(store: &ObjectStore) -> lance_core::Result> { + /// let mut tables = Vec::new(); + /// let mut page_token = None; + /// loop { + /// let page = store + /// .read_dir_page("my_db", ReadDirOptions { page_token, limit: Some(10) }) + /// .await?; + /// // A table is a directory, so a loose object that happens to be named like one is not + /// // a table. + /// tables.extend(page.result.common_prefixes.iter().filter_map(|table| { + /// Some(table.filename()?.strip_suffix(".lance")?.to_string()) + /// })); + /// page_token = page.page_token; + /// if page_token.is_none() || tables.len() >= 10 { + /// break; + /// } + /// } + /// # Ok(tables) + /// # } + /// ``` + pub async fn read_dir_page( + &self, + dir: impl Into, + options: ReadDirOptions, + ) -> Result { + let dir = dir.into(); + // A page of nothing cannot advance a listing, and the two paths below would disagree + // about what it means: the pushdown path would report an empty directory while the + // full listing ignored the limit and returned everything. + if options.limit == Some(0) { + return Err(Error::invalid_input( + "read_dir_page limit must be at least 1, got 0", + )); + } + match &self.paginated_lister { + Some(lister) => self.pushdown_page(lister.as_ref(), &dir, options).await, + // Goes through `inner`, so the wrappers around it instrument the request. The + // pushdown path talks to the backend directly and instruments itself. + None => full_listing_page(self.inner.as_ref(), &dir, options).await, + } + } + + /// One page from the backend's own paginated list API. + /// + /// The pushdown path holds the backend directly, so its request never passes through the + /// wrappers around [`Self::inner`] that would otherwise record it, and it records itself. + #[instrument(level = "debug", skip_all, fields(dir = %dir))] + async fn pushdown_page( + &self, + lister: &dyn PaginatedListStore, + dir: &Path, + options: ReadDirOptions, + ) -> Result { + let prefix = list_prefix(dir); + self.io_tracker.record_read(LIST_OP, dir.clone(), 0, None); + #[cfg(feature = "metrics")] + let _in_flight = InFlightGuard::new(&self.store_prefix, LIST_OP); + #[cfg(feature = "metrics")] + let start = Instant::now(); + + let page = lister + .list_paginated( + prefix.as_deref(), + PaginatedListOptions { + delimiter: Some(DELIMITER.into()), + max_keys: options.limit, + page_token: options.page_token, + // `offset` is left unset: a continuation token is a position of its own, + // and a caller-supplied key means something different on every store — + // S3 excludes it, Azure includes it. + ..Default::default() + }, + ) + .await; + + #[cfg(feature = "metrics")] + record_outcome(&self.store_prefix, LIST_OP, start, 0, page.is_err()); + let mut page = page?; + + retain_children(&mut page.result, prefix.as_deref()); + Ok(page) + } +} + +/// The prefix to list under, carrying the trailing delimiter that the paginated API expects. +/// `None` for the root of the store, which has no prefix at all. +fn list_prefix(dir: &Path) -> Option { + let dir = dir.as_ref(); + (!dir.is_empty()).then(|| format!("{dir}{DELIMITER}")) +} + +/// One page of a directory on a store with no paginated list API: list the level in full and +/// page it locally. +/// +/// The page has to be the smallest `limit` children past the token rather than any `limit` of +/// them, since the next call lists the same directory again and keeps only what sorts after +/// the key this page hands back. That means putting the listing in key order, which +/// `list_with_delimiter` does not promise — a sort over children already in memory, costing no +/// extra request. +async fn full_listing_page( + store: &dyn OSObjectStore, + dir: &Path, + options: ReadDirOptions, +) -> Result { + let listed = store.list_with_delimiter(Some(dir)).await?; + let mut children = keyed_children(listed, list_prefix(dir).as_deref()); + if let Some(resume) = &options.page_token { + children.retain(|child| child.key > *resume); + } + let total = children.len(); + children.truncate(options.limit.unwrap_or(total).min(total)); + // The last key this page took, so a page that took nothing ends the listing rather than + // resuming from a position no page ever reached. + let page_token = match children.last() { + Some(last) if children.len() < total => Some(last.key.clone()), + _ => None, + }; + + let mut result = ListResult { + common_prefixes: Vec::new(), + objects: Vec::new(), + }; + for child in children { + match child.child { + Child::Directory(location) => result.common_prefixes.push(location), + Child::File(meta) => result.objects.push(meta), + } + } + Ok(PaginatedListResult { result, page_token }) +} + +/// Drop everything in `listed` that is not a child of the level being listed. +/// +/// This covers the marker object some stores keep for a directory: it lists as an object whose +/// location is the directory's own prefix. +fn retain_children(listed: &mut ListResult, prefix: Option<&str>) { + listed + .common_prefixes + .retain(|location| relative_key(prefix, location).is_some()); + listed + .objects + .retain(|object| relative_key(prefix, &object.location).is_some()); +} + +/// A child of the directory being listed, with the key the backend listed it under. +struct KeyedChild { + /// The key relative to the directory, which is what a full-listing token names. A child + /// directory keeps its trailing delimiter, since that is the prefix its keys share and so + /// where it sits in the listing; a child file is its name. + key: String, + child: Child, +} + +enum Child { + Directory(Path), + File(ObjectMeta), +} + +/// The children of `prefix` in `listed`, in key order, each with the key it was listed under. +/// +/// Stores report common prefixes and objects as two separate lists, so the two are put back +/// into one order here — the order a full-listing token pages through. Anything that is not a +/// child of this level is dropped, as in [`retain_children`]. +fn keyed_children(listed: ListResult, prefix: Option<&str>) -> Vec { + let ListResult { + common_prefixes, + objects, + } = listed; + let directories = common_prefixes.into_iter().filter_map(|location| { + let key = format!("{}{DELIMITER}", relative_key(prefix, &location)?); + Some(KeyedChild { + key, + child: Child::Directory(location), + }) + }); + let files = objects.into_iter().filter_map(|meta| { + let key = relative_key(prefix, &meta.location)?.to_string(); + Some(KeyedChild { + key, + child: Child::File(meta), + }) + }); + let mut children: Vec = directories.chain(files).collect(); + children.sort_unstable_by(|left, right| left.key.cmp(&right.key)); + children +} + +/// Where a listed location sits inside the directory being listed, which is the space +/// full-listing tokens live in, or `None` if it is not a child of that directory at all. +fn relative_key<'a>(prefix: Option<&str>, location: &'a Path) -> Option<&'a str> { + let location = location.as_ref(); + let relative = match prefix { + // Both halves of the prefix, so a location that merely starts with the directory's + // name — `dbx/y` against `db/` — is reported as not being under it, and so is the + // directory's own marker, whose location is `db`. + Some(prefix) => location.strip_prefix(prefix)?, + None => location, + }; + (!relative.is_empty()).then_some(relative) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use super::*; + use crate::object_store::{ObjectStoreParams, ObjectStoreRegistry}; + use chrono::Utc; + use object_store::memory::InMemory; + use object_store::{ObjectStoreExt, PutPayload}; + use rstest::rstest; + + /// How the store under test resolves a listing. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum Backend { + /// No paginated API: list the whole directory and page it locally. + FullListing, + /// A paginated API, as the native S3, GCS and Azure stores have. + Pushdown, + } + use Backend::{FullListing, Pushdown}; + + /// One list request, as the backend saw it. + #[derive(Debug, Clone)] + struct ListRequest { + prefix: Option, + opts: PaginatedListOptions, + } + + /// A stand-in for a store with a paginated list API. + /// + /// Keys are listed in the order they were given, a delimiter collapses each level, and the + /// continuation token is a position in that listing order — which is what a real token is: + /// exact, and never compared against a key. `page_bound` is the store's own cap, which is + /// why a page can come back holding less than it was asked for. + #[derive(Debug)] + struct FakeListStore { + keys: Vec, + page_bound: usize, + requests: Arc>>, + } + + #[async_trait::async_trait] + impl PaginatedListStore for FakeListStore { + async fn list_paginated( + &self, + prefix: Option<&str>, + opts: PaginatedListOptions, + ) -> object_store::Result { + self.requests.lock().unwrap().push(ListRequest { + prefix: prefix.map(String::from), + opts: opts.clone(), + }); + let prefix = prefix.unwrap_or(""); + let budget = opts + .max_keys + .unwrap_or(self.page_bound) + .min(self.page_bound); + let mut result = ListResult { + common_prefixes: Vec::new(), + objects: Vec::new(), + }; + let mut idx: usize = match &opts.page_token { + Some(token) => token.parse().expect("a token this store minted"), + None => 0, + }; + + while idx < self.keys.len() { + if result.common_prefixes.len() + result.objects.len() >= budget { + return Ok(PaginatedListResult { + result, + page_token: Some(idx.to_string()), + }); + } + // `Path::parse`, so that a key holding a character `Path::from` would encode + // is reported under the name it was stored with. This is what the S3, GCS and + // Azure clients do. + let key = self.keys[idx].clone(); + idx += 1; + let Some(rest) = key.strip_prefix(prefix) else { + continue; + }; + match rest.find(DELIMITER) { + // A collapsed prefix, and everything behind it: a store reports the child + // directory once and skips the keys it stands for. + Some(end) => { + let child = format!("{prefix}{}", &rest[..=end]); + result.common_prefixes.push(Path::parse(&child).unwrap()); + while idx < self.keys.len() && self.keys[idx].starts_with(&child) { + idx += 1; + } + } + None => result.objects.push(ObjectMeta { + location: Path::parse(&key).unwrap(), + last_modified: Utc::now(), + size: 1, + e_tag: None, + version: None, + }), + } + } + + Ok(PaginatedListResult { + result, + page_token: None, + }) + } + } + + struct TestStore { + store: ObjectStore, + requests: Arc>>, + } + + impl TestStore { + /// Every child of `dir`, taken a page at a time, which is how a caller walks a + /// directory: the token ends the walk, never a short page. + async fn walk(&self, dir: &str, limit: Option) -> Result> { + let mut names = Vec::new(); + let mut page_token = None; + for _ in 0..100 { + let page = self + .store + .read_dir_page(Path::from(dir), ReadDirOptions { page_token, limit }) + .await?; + names.extend(page_names(&page)); + page_token = page.page_token; + if page_token.is_none() { + return Ok(names); + } + } + panic!("the walk is not making progress: {names:?}") + } + + async fn names(&self, dir: &str, limit: Option) -> Vec { + self.walk(dir, limit).await.unwrap() + } + + /// The first page only, as a caller wanting a bounded number of children would take it. + async fn first_page(&self, dir: &str, limit: Option) -> PaginatedListResult { + self.store + .read_dir_page( + Path::from(dir), + ReadDirOptions { + page_token: None, + limit, + }, + ) + .await + .unwrap() + } + } + + /// The names of every child in a page, directories and files alike. + fn page_names(page: &PaginatedListResult) -> Vec { + page.result + .common_prefixes + .iter() + .chain(page.result.objects.iter().map(|object| &object.location)) + .map(|location| location.filename().unwrap().to_string()) + .collect() + } + + async fn test_store(backend: Backend, keys: &[&str]) -> TestStore { + paged_test_store(backend, keys, usize::MAX).await + } + + /// A store over `keys`, listing them in the order given. `page_bound` is the store's own + /// cap on a page, which only the pushdown backend has. + async fn paged_test_store(backend: Backend, keys: &[&str], page_bound: usize) -> TestStore { + let inner = Arc::new(InMemory::new()); + for key in keys { + // `Path::parse`, so that a key holding a character `Path::from` would encode is + // stored under the name it was given. + inner + .put(&Path::parse(key).unwrap(), PutPayload::from_static(b"x")) + .await + .unwrap(); + } + #[allow(deprecated)] + let params = ObjectStoreParams { + object_store: Some((inner, url::Url::parse("memory:///").unwrap())), + // Set because the deprecated hand-built path assumes nothing about the store it + // was given, and set conservatively: nothing on the `read_dir_page` path reads it, + // since the fallback sorts what it listed and the pushdown never compares keys. + list_is_lexically_ordered: Some(false), + ..Default::default() + }; + let (store, _) = ObjectStore::from_uri_and_params( + Arc::new(ObjectStoreRegistry::default()), + "memory:///", + ¶ms, + ) + .await + .unwrap(); + let mut store = Arc::try_unwrap(store).unwrap(); + + let requests = Arc::new(Mutex::new(Vec::new())); + if backend == Pushdown { + store.paginated_lister = Some(Arc::new(FakeListStore { + keys: keys.iter().map(|key| key.to_string()).collect(), + page_bound, + requests: requests.clone(), + })); + } + TestStore { store, requests } + } + + const TABLES: &[&str] = &[ + "db/a.lance/_versions/1.manifest", + "db/a.lance/data/1.lance", + "db/b.lance/data/1.lance", + "db/c.lance/data/1.lance", + "db/loose.txt", + "other/d.lance/data/1.lance", + ]; + + /// Walking a directory hands back every child exactly once, however the store resolves the + /// listing and however small the pages are. That it holds whatever order the store lists + /// in is [`test_an_unordered_store_is_still_paged`]. + #[rstest] + #[case::whole_directory(TABLES, "db", vec!["a.lance", "b.lance", "c.lance", "loose.txt"])] + #[case::empty_directory(TABLES, "nonexistent", vec![])] + // A directory and the sibling that follows it: `foo/` and `foo0` are adjacent in key order + // with nothing between them, so resuming past `foo`'s contents must not swallow `foo0`. + #[case::the_sibling_after_a_directory(&["db/foo/inside", "db/foo0"], "db", vec!["foo", "foo0"])] + // Siblings where one name is a prefix of another, which is where a page boundary is easiest + // to get wrong: `foo/` and `foo-bar/` differ at `/` against `-`. + #[case::a_prefix_shaped_sibling(&["db/foo/inside", "db/foo-bar/inside", "db/zzz.txt"], "db", vec!["foo", "foo-bar", "zzz.txt"])] + // A store that keeps a marker object for a directory reports the directory itself when + // that directory is listed. Dropping the marker must not also drop the progress the page + // made, or a page holding nothing but the marker reads as the end of the listing. + #[case::a_directory_marker(&["db/marked/", "db/marked/a.txt", "db/marked/b.txt"], "db/marked", vec!["a.txt", "b.txt"])] + // A name holding a character `Path::from` would percent-encode is still reported, and + // sorted, under the name it was stored with. + #[case::an_encodable_name(&["db/az", "db/a~"], "db", vec!["az", "a~"])] + #[tokio::test] + async fn test_walking_a_directory_is_complete( + #[values(FullListing, Pushdown)] backend: Backend, + #[values(None, Some(1), Some(2), Some(3))] limit: Option, + #[case] keys: &[&str], + #[case] dir: &str, + #[case] expected: Vec<&str>, + ) { + let store = test_store(backend, keys).await; + + // The order children come back in is the store's, so the walk is checked for holding + // every child once rather than for holding them in one particular order. + let mut listed = store.names(dir, limit).await; + let seen = listed.clone(); + listed.sort(); + assert_eq!(listed, expected, "from {seen:?}"); + } + + /// A store that lists in no particular order — S3 Express — is still paged, because a + /// continuation token is never compared to anything. This is the case a token spelled as a + /// key could not serve. + #[tokio::test] + async fn test_an_unordered_store_is_still_paged() { + let reversed: Vec<&str> = TABLES.iter().rev().copied().collect(); + let store = test_store(Pushdown, &reversed).await; + + let mut names = store.names("db", Some(1)).await; + names.sort(); + + assert_eq!(names, vec!["a.lance", "b.lance", "c.lance", "loose.txt"]); + assert!( + !store.requests.lock().unwrap().is_empty(), + "the paginated lister should have been used" + ); + } + + /// The point of the pushdown: a caller that wants one child of a directory makes one + /// request, for one child, one level deep. A listing that quietly fell back to reading the + /// whole directory would answer the same thing, so what tells the two apart is the request. + #[tokio::test] + async fn test_a_bounded_page_is_one_request_for_that_page() { + let store = test_store(Pushdown, TABLES).await; + + let page = store.first_page("db", Some(1)).await; + + assert_eq!(page_names(&page).len(), 1); + assert!(page.page_token.is_some()); + let requests = store.requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].prefix.as_deref(), Some("db/")); + assert_eq!(requests[0].opts.max_keys, Some(1)); + // Without a delimiter the listing would be recursive rather than one level deep. + assert_eq!(requests[0].opts.delimiter.as_deref(), Some(DELIMITER)); + // A continuation token is a position of its own, so no offset goes with it. + assert_eq!(requests[0].opts.offset, None); + } + + /// A backend that caps its pages below what was asked for hands back a short page with + /// more to come. Only the token ends a walk, so a caller that stopped at a short page + /// would report a directory as smaller than it is. + #[tokio::test] + async fn test_a_short_page_is_not_the_end_of_the_listing() { + let store = paged_test_store(Pushdown, TABLES, 1).await; + + let page = store.first_page("db", Some(3)).await; + + assert_eq!(page_names(&page).len(), 1); + assert!( + page.page_token.is_some(), + "the directory holds four children" + ); + assert_eq!( + store.names("db", Some(3)).await.len(), + 4, + "the walk should still reach every child" + ); + } + + /// A page of nothing is rejected rather than left to mean whatever the backend makes of it: + /// pushing it down reports an empty directory, and a full listing ignores it. The rejection + /// comes before the store is consulted, so one backend covers it. + #[tokio::test] + async fn test_zero_limit_is_rejected() { + let store = test_store(Pushdown, TABLES).await; + + let err = store.walk("db", Some(0)).await.unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}"); + assert!(err.to_string().contains("limit must be at least 1")); + assert!(store.requests.lock().unwrap().is_empty()); + } + + /// Child directories and child objects stay in the two lists a listing reports them in, so + /// a caller that wants only one of the two — tables are directories — can take it. + #[rstest] + #[tokio::test] + async fn test_directories_and_files_stay_apart( + #[values(FullListing, Pushdown)] backend: Backend, + ) { + let store = test_store(backend, TABLES).await; + + let page = store.first_page("db", None).await; + + let mut directories: Vec<&str> = page + .result + .common_prefixes + .iter() + .map(|location| location.filename().unwrap()) + .collect(); + directories.sort(); + assert_eq!(directories, vec!["a.lance", "b.lance", "c.lance"]); + let files: Vec<&str> = page + .result + .objects + .iter() + .map(|object| object.location.filename().unwrap()) + .collect(); + assert_eq!(files, vec!["loose.txt"]); + // The metadata a listing reports for a child object survives the page. + assert_eq!(page.result.objects[0].size, 1); + } + + /// The pushdown path holds the backend directly, so it has to record its own IO. A listing + /// invisible to `io_tracker` would also be invisible to the metrics and tracing layers that + /// sit in the same chain. + #[rstest] + #[tokio::test] + async fn test_listing_is_recorded_in_io_stats( + #[values(FullListing, Pushdown)] backend: Backend, + ) { + let store = test_store(backend, TABLES).await; + assert_eq!(store.store.io_tracker().stats().read_iops, 0); + + let _ = store.first_page("db", Some(2)).await; + + // The full listing reaches the store through its wrappers, which record it there. + assert_eq!(store.store.io_tracker().stats().read_iops, 1); + } +} diff --git a/rust/lance-io/src/object_store/storage_options.rs b/rust/lance-io/src/object_store/storage_options.rs index fa5f4995afd..355845fd9b6 100644 --- a/rust/lance-io/src/object_store/storage_options.rs +++ b/rust/lance-io/src/object_store/storage_options.rs @@ -138,6 +138,8 @@ impl StorageOptionsProvider for LanceNamespaceStorageOptionsProvider { async fn fetch_storage_options(&self) -> Result>> { let request = DescribeTableRequest { id: Some(self.table_id.clone()), + // Some server implementations may not return credentials unless explicitly requested + vend_credentials: Some(true), ..Default::default() }; diff --git a/rust/lance-io/src/object_store/throttle.rs b/rust/lance-io/src/object_store/throttle.rs index bac1d3a538e..6e0050e79f2 100644 --- a/rust/lance-io/src/object_store/throttle.rs +++ b/rust/lance-io/src/object_store/throttle.rs @@ -33,6 +33,11 @@ use lance_core::utils::aimd::{AimdConfig, AimdController, RequestOutcome}; use lance_core::utils::tracing::TRACE_OBJECT_STORE_THROTTLE; #[cfg(test)] use object_store::ObjectStoreExt; +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +use object_store::client::{ + ClientOptions, HttpClient, HttpConnector, HttpError, HttpErrorKind, HttpRequest, HttpResponse, + HttpResponseBody, HttpService, +}; use object_store::path::Path; use object_store::{ CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, @@ -43,6 +48,8 @@ use rand::Rng; use tokio::sync::Mutex; use tracing::{debug, warn}; +use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; + /// Check whether an `object_store::Error` represents a throttle response /// (HTTP 429 / 503) from a cloud object store. /// @@ -407,24 +414,44 @@ impl OperationThrottle { } Err(_) => RequestOutcome::Success, }; - let prev_rate = self.controller.current_rate(); - let new_rate = self.controller.record_outcome(outcome); - if new_rate < prev_rate - && let Err(err) = result.as_ref() - { - warn!( - target: TRACE_OBJECT_STORE_THROTTLE, - previous_rate = format!("{prev_rate:.1}"), - new_rate = format!("{new_rate:.1}"), - error = %err, - "AIMD throttle: rate reduced due to throttle errors" - ); - } + let error = result + .as_ref() + .err() + .map(|error| error as &dyn std::fmt::Display); + let new_rate = self.record_outcome(outcome, error); if let Ok(mut bucket) = self.bucket.try_lock() { bucket.rate = new_rate; } } + fn record_outcome( + &self, + outcome: RequestOutcome, + error: Option<&dyn std::fmt::Display>, + ) -> f64 { + let prev_rate = self.controller.current_rate(); + let new_rate = self.controller.record_outcome(outcome); + if new_rate < prev_rate { + if let Some(error) = error { + warn!( + target: TRACE_OBJECT_STORE_THROTTLE, + previous_rate = format!("{prev_rate:.1}"), + new_rate = format!("{new_rate:.1}"), + error = %error, + "AIMD throttle: rate reduced due to throttle errors" + ); + } else { + warn!( + target: TRACE_OBJECT_STORE_THROTTLE, + previous_rate = format!("{prev_rate:.1}"), + new_rate = format!("{new_rate:.1}"), + "AIMD throttle: rate reduced due to throttle errors" + ); + } + } + new_rate + } + /// Execute an operation with throttling: acquire token, run, classify result. /// On throttle errors, retries up to `max_retries` times with a random /// backoff between `min_backoff_ms` and `max_backoff_ms` between attempts. @@ -448,19 +475,11 @@ impl OperationThrottle { } Err(_) => RequestOutcome::Success, // Non-throttle errors don't indicate capacity problems }; - let prev_rate = self.controller.current_rate(); - let new_rate = self.controller.record_outcome(outcome); - if new_rate < prev_rate - && let Err(err) = result.as_ref() - { - warn!( - target: TRACE_OBJECT_STORE_THROTTLE, - previous_rate = format!("{prev_rate:.1}"), - new_rate = format!("{new_rate:.1}"), - error = %err, - "AIMD throttle: rate reduced due to throttle errors" - ); - } + let error = result + .as_ref() + .err() + .map(|error| error as &dyn std::fmt::Display); + let new_rate = self.record_outcome(outcome, error); self.update_bucket_rate(new_rate).await; match &result { @@ -494,12 +513,229 @@ impl Debug for OperationThrottle { } } -/// A [`MultipartUpload`] wrapper that throttles and retries `put_part`, -/// `complete`, and `abort`, feeding outcomes back to the write AIMD -/// controller. +#[derive(Clone)] +pub(crate) struct AimdThrottleState { + read: Arc, + write: Arc, + delete: Arc, + list: Arc, +} + +impl AimdThrottleState { + pub(crate) fn new(config: AimdThrottleConfig) -> lance_core::Result { + let burst_capacity = config.burst_capacity as f64; + let max_retries = config.max_retries; + let min_backoff_ms = config.min_backoff_ms; + let max_backoff_ms = config.max_backoff_ms; + Ok(Self { + read: Arc::new(OperationThrottle::new( + config.read, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + write: Arc::new(OperationThrottle::new( + config.write, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + delete: Arc::new(OperationThrottle::new( + config.delete, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + list: Arc::new(OperationThrottle::new( + config.list, + burst_capacity, + max_retries, + min_backoff_ms, + max_backoff_ms, + )?), + }) + } +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +#[derive(Debug)] +pub(crate) struct AimdMultipartUploadConnector { + inner: C, + write: Option>, +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +impl AimdMultipartUploadConnector { + fn new(inner: C, state: Option<&AimdThrottleState>) -> Self { + Self { + inner, + write: state.map(|state| Arc::clone(&state.write)), + } + } +} + +#[cfg(all( + any(feature = "aws", feature = "azure", feature = "gcp"), + feature = "metrics" +))] +pub(crate) fn cloud_http_connector( + state: Option<&AimdThrottleState>, + metrics_base: String, +) -> AimdMultipartUploadConnector { + AimdMultipartUploadConnector::new( + crate::object_store::metrics::MeteringHttpConnector::new(metrics_base), + state, + ) +} + +#[cfg(all( + any(feature = "aws", feature = "azure", feature = "gcp"), + not(feature = "metrics") +))] +pub(crate) fn cloud_http_connector( + state: Option<&AimdThrottleState>, + _metrics_base: String, +) -> AimdMultipartUploadConnector { + AimdMultipartUploadConnector::new(object_store::client::ReqwestConnector::default(), state) +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +impl HttpConnector for AimdMultipartUploadConnector { + fn connect(&self, options: &ClientOptions) -> object_store::Result { + Ok(HttpClient::new(AimdMultipartUploadService { + inner: self.inner.connect(options)?, + write: self.write.clone(), + })) + } +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +#[derive(Debug)] +struct AimdMultipartUploadService { + inner: HttpClient, + write: Option>, +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +fn is_multipart_part_request(request: &HttpRequest) -> bool { + if request.method() != ::http::Method::PUT { + return false; + } + request.uri().query().is_some_and(|query| { + url::form_urlencoded::parse(query.as_bytes()).any(|(key, value)| { + key.eq_ignore_ascii_case("partNumber") + || (key.eq_ignore_ascii_case("comp") && value.eq_ignore_ascii_case("block")) + }) + }) +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +fn is_retryable_http_error(error: &HttpError) -> bool { + matches!( + error.kind(), + HttpErrorKind::Connect + | HttpErrorKind::Request + | HttpErrorKind::Timeout + | HttpErrorKind::Interrupted + ) +} + +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +#[async_trait] +impl HttpService for AimdMultipartUploadService { + async fn call(&self, request: HttpRequest) -> Result { + let Some(write) = self.write.as_ref() else { + return self.inner.execute(request).await; + }; + if !is_multipart_part_request(&request) { + return self.inner.execute(request).await; + } + + for attempt in 0..=write.max_retries { + write.acquire_token().await; + let mut result = self.inner.execute(request.clone()).await; + let mut is_retryable = result.as_ref().err().is_some_and(is_retryable_http_error); + let mut is_throttle = false; + let mut response_status = None; + + if let Ok(response) = result { + let status = response.status(); + response_status = Some(status); + is_retryable = status == ::http::StatusCode::REQUEST_TIMEOUT + || status == ::http::StatusCode::TOO_MANY_REQUESTS + || status.is_server_error(); + is_throttle = status == ::http::StatusCode::TOO_MANY_REQUESTS + || status == ::http::StatusCode::SERVICE_UNAVAILABLE; + + let (parts, body) = response.into_parts(); + result = match body.bytes().await { + Ok(bytes) => { + let body = String::from_utf8_lossy(&bytes).to_ascii_lowercase(); + let is_throttle_body = body.contains("requesttimeout") + || body.contains("slowdown") + || body.contains("serverbusy") + || body.contains("throttl"); + is_retryable |= is_throttle_body; + is_throttle |= is_throttle_body; + Ok(HttpResponse::from_parts( + parts, + HttpResponseBody::from(bytes), + )) + } + Err(error) => { + is_retryable = is_retryable_http_error(&error); + Err(error) + } + }; + } + + let detail = response_status + .filter(|status| !status.is_success()) + .map(|status| format!("HTTP status {status}")); + let error = result + .as_ref() + .err() + .map(|error| error as &dyn std::fmt::Display) + .or_else(|| { + detail + .as_ref() + .map(|detail| detail as &dyn std::fmt::Display) + }); + let outcome = if is_throttle { + RequestOutcome::Throttled + } else { + RequestOutcome::Success + }; + let new_rate = write.record_outcome(outcome, error); + write.update_bucket_rate(new_rate).await; + + if is_retryable && attempt < write.max_retries { + let backoff_ms = + rand::rng().random_range(write.min_backoff_ms..=write.max_backoff_ms); + debug!( + target: TRACE_OBJECT_STORE_THROTTLE, + attempt = attempt + 1, + max_retries = write.max_retries, + backoff_ms, + "Retrying multipart upload part after retryable HTTP response" + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + continue; + } + return result; + } + unreachable!() + } +} + +/// A [`MultipartUpload`] wrapper that applies the write AIMD controller. struct ThrottledMultipartUpload { target: Box, write: Arc, + parts_throttled_at_http: bool, } impl Debug for ThrottledMultipartUpload { @@ -511,10 +747,13 @@ impl Debug for ThrottledMultipartUpload { #[async_trait] impl MultipartUpload for ThrottledMultipartUpload { fn put_part(&mut self, data: PutPayload) -> UploadPart { - let write = Arc::clone(&self.write); // Call put_part synchronously to preserve part ordering regardless // of which futures are awaited first. let fut = self.target.put_part(data); + if self.parts_throttled_at_http { + return fut; + } + let write = Arc::clone(&self.write); Box::pin(async move { write.acquire_token().await; let result = fut.await; @@ -585,6 +824,7 @@ pub struct AimdThrottledStore { write: Arc, delete: Arc, list: Arc, + multipart_parts_throttled_at_http: bool, } impl Debug for AimdThrottledStore { @@ -595,6 +835,10 @@ impl Debug for AimdThrottledStore { .field("write", &self.write) .field("delete", &self.delete) .field("list", &self.list) + .field( + "multipart_parts_throttled_at_http", + &self.multipart_parts_throttled_at_http, + ) .finish() } } @@ -610,44 +854,91 @@ impl AimdThrottledStore { target: Arc, config: AimdThrottleConfig, ) -> lance_core::Result { - let burst = config.burst_capacity as f64; - let max_retries = config.max_retries; - let min_backoff_ms = config.min_backoff_ms; - let max_backoff_ms = config.max_backoff_ms; - Ok(Self { + Ok(Self::new_with_state( target, - read: Arc::new(OperationThrottle::new( - config.read, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - write: Arc::new(OperationThrottle::new( - config.write, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - delete: Arc::new(OperationThrottle::new( - config.delete, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), - list: Arc::new(OperationThrottle::new( - config.list, - burst, - max_retries, - min_backoff_ms, - max_backoff_ms, - )?), + AimdThrottleState::new(config)?, + false, + )) + } + + pub(crate) fn new_with_state( + target: Arc, + state: AimdThrottleState, + multipart_parts_throttled_at_http: bool, + ) -> Self { + Self { + target, + read: state.read, + write: state.write, + delete: state.delete, + list: state.list, + multipart_parts_throttled_at_http, + } + } + + /// Put a paginated lister on the same list budget as this store. + pub fn wrap_paginated( + &self, + inner: Arc, + ) -> Arc { + Arc::new(ThrottledListStore { + inner, + throttle: self.list.clone(), }) } } +/// A store paired with the paginated lister that shares its rate limits. +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +type StoreWithLister = (Arc, Option>); + +/// Apply AIMD throttling to a store and to the lister that shares its list budget. +/// +/// [`crate::object_store::ObjectStore::read_dir_page`] goes to the lister rather than +/// through the store, so both have to be wrapped for list requests to be counted once +/// against one rate. +#[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] +pub(crate) fn with_throttling( + state: Option, + multipart_parts_throttled_at_http: bool, + store: Arc, + lister: Option>, +) -> StoreWithLister { + let Some(state) = state else { + return (store, lister); + }; + let store = Arc::new(AimdThrottledStore::new_with_state( + store, + state, + multipart_parts_throttled_at_http, + )); + let lister = lister.map(|lister| store.wrap_paginated(lister)); + (store, lister) +} + +/// A [`PaginatedListStore`] whose requests draw on a store's list token bucket. +struct ThrottledListStore { + inner: Arc, + throttle: Arc, +} + +// Throttling only adds waiting, so every semantic of the store it wraps has to reach the +// listing unchanged; the lint keeps a method added to the trait from silently falling back to +// its default here. +#[async_trait] +#[deny(clippy::missing_trait_methods)] +impl PaginatedListStore for ThrottledListStore { + async fn list_paginated( + &self, + prefix: Option<&str>, + opts: PaginatedListOptions, + ) -> OSResult { + self.throttle + .throttled(|| self.inner.list_paginated(prefix, opts.clone())) + .await + } +} + #[async_trait] #[deny(clippy::missing_trait_methods)] impl ObjectStore for AimdThrottledStore { @@ -674,6 +965,7 @@ impl ObjectStore for AimdThrottledStore { Ok(Box::new(ThrottledMultipartUpload { target, write: Arc::clone(&self.write), + parts_throttled_at_http: self.multipart_parts_throttled_at_http, })) } @@ -811,6 +1103,111 @@ mod tests { assert!(!is_throttle_error(&err)); } + #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] + #[rstest] + #[case::s3("https://bucket/object?partNumber=1&uploadId=id", true)] + #[case::azure_block("https://account/object?comp=block&blockid=id", true)] + #[case::azure_block_list("https://account/object?comp=blocklist", false)] + #[case::ordinary_put("https://bucket/object", false)] + fn test_is_multipart_part_request(#[case] uri: &str, #[case] expected: bool) { + let request = ::http::Request::builder() + .method(::http::Method::PUT) + .uri(uri) + .body(object_store::client::HttpRequestBody::empty()) + .unwrap(); + assert_eq!(is_multipart_part_request(&request), expected); + } + + /// One page of a fixed directory, counting the requests that reached it. + #[derive(Default)] + struct CountingListStore { + calls: AtomicUsize, + fail_with: Option, + } + + #[async_trait] + impl PaginatedListStore for CountingListStore { + async fn list_paginated( + &self, + _prefix: Option<&str>, + _opts: PaginatedListOptions, + ) -> OSResult { + self.calls.fetch_add(1, Ordering::SeqCst); + match &self.fail_with { + Some(message) => Err(make_generic_error(message)), + None => Ok(PaginatedListResult { + result: ListResult { + common_prefixes: vec![Path::from("prefix/child")], + objects: Vec::new(), + }, + page_token: None, + }), + } + } + } + + #[tokio::test(start_paused = true)] + async fn test_paginated_lister_acquires_a_token_before_listing() { + let lister = Arc::new(CountingListStore::default()); + let throttled = AimdThrottledStore::new( + Arc::new(InMemory::new()) as Arc, + list_start_throttle_config(), + ) + .unwrap(); + let throttled_lister = throttled.wrap_paginated(lister.clone()); + + let mut page = Box::pin( + throttled_lister.list_paginated(Some("prefix/"), PaginatedListOptions::default()), + ); + // With rate=10 tokens/s and burst_capacity=0, the token acquisition sleeps for + // 100 ms. A 50 ms timeout must expire before that. + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut page) + .await + .is_err() + ); + assert_eq!(lister.calls.load(Ordering::SeqCst), 0); + + let page = tokio::time::timeout(std::time::Duration::from_millis(300), page) + .await + .unwrap() + .unwrap(); + assert_eq!(page.result.common_prefixes.len(), 1); + assert_eq!(lister.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn test_paginated_lister_throttle_errors_decrease_rate() { + let lister = Arc::new(CountingListStore { + calls: AtomicUsize::new(0), + fail_with: Some(THROTTLE_ERROR_RESPONSE.to_string()), + }); + let mut config = AimdThrottleConfig::default().with_list_aimd( + AimdConfig::default() + .with_initial_rate(100.0) + .with_decrease_factor(0.5) + .with_window_duration(std::time::Duration::from_millis(1)), + ); + config.max_retries = 1; + config.min_backoff_ms = 0; + config.max_backoff_ms = 0; + let throttled = + AimdThrottledStore::new(Arc::new(InMemory::new()) as Arc, config) + .unwrap(); + let throttled_lister = throttled.wrap_paginated(lister.clone()); + + assert!( + throttled_lister + .list_paginated(Some("prefix/"), PaginatedListOptions::default()) + .await + .is_err() + ); + + // The request was retried once, and the throttle response pushed the rate down. + assert_eq!(lister.calls.load(Ordering::SeqCst), 2); + assert!(throttled.list.controller.current_rate() < 100.0); + } + #[tokio::test] async fn test_basic_put_get_through_wrapper() { let store = Arc::new(InMemory::new()); @@ -1682,6 +2079,145 @@ mod tests { assert_eq!(mock.get_call_count.load(Ordering::Relaxed), 4); } + #[cfg(feature = "aws")] + #[derive(Debug)] + struct MultipartRetryState { + failures_remaining: AtomicUsize, + part_uris: std::sync::Mutex>, + } + + #[cfg(feature = "aws")] + #[derive(Debug)] + struct MultipartRetryConnector { + state: Arc, + } + + #[cfg(feature = "aws")] + impl HttpConnector for MultipartRetryConnector { + fn connect(&self, _options: &ClientOptions) -> object_store::Result { + Ok(HttpClient::new(MultipartRetryService { + state: Arc::clone(&self.state), + })) + } + } + + #[cfg(feature = "aws")] + #[derive(Debug)] + struct MultipartRetryService { + state: Arc, + } + + #[cfg(feature = "aws")] + #[async_trait] + impl HttpService for MultipartRetryService { + async fn call(&self, request: HttpRequest) -> Result { + let method = request.method().clone(); + let query = request.uri().query().unwrap_or_default(); + let (status, body, e_tag) = if method == ::http::Method::POST + && query + .split('&') + .any(|part| part == "uploads" || part == "uploads=") + { + ( + ::http::StatusCode::OK, + "bucketobjectupload-id", + None, + ) + } else if method == ::http::Method::PUT && query.contains("partNumber=") { + self.state + .part_uris + .lock() + .unwrap() + .push(request.uri().to_string()); + let mut remaining = self.state.failures_remaining.load(Ordering::SeqCst); + let should_fail = loop { + let Some(next) = remaining.checked_sub(1) else { + break false; + }; + match self.state.failures_remaining.compare_exchange_weak( + remaining, + next, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => break true, + Err(actual) => remaining = actual, + } + }; + if should_fail { + ( + ::http::StatusCode::SERVICE_UNAVAILABLE, + "SlowDownPlease reduce your request rate.", + None, + ) + } else { + (::http::StatusCode::OK, "", Some("\"part-etag\"")) + } + } else if method == ::http::Method::POST && query.contains("uploadId=") { + ( + ::http::StatusCode::OK, + "https://bucket/objectbucketobject\"object-etag\"", + None, + ) + } else { + (::http::StatusCode::BAD_REQUEST, "unexpected request", None) + }; + + let mut response = ::http::Response::builder().status(status); + if let Some(e_tag) = e_tag { + response = response.header(::http::header::ETAG, e_tag); + } + Ok(response + .body(HttpResponseBody::from(body.to_string())) + .unwrap()) + } + } + + /// Retries must remain inside the original S3 `put_part` call. Re-entering + /// `MultipartUpload::put_part` would allocate a new part number and leave a + /// gap that makes `complete` fail with "Missing part". + #[cfg(feature = "aws")] + #[tokio::test(start_paused = true)] + async fn test_multipart_http_retry_reuses_part_number() { + use object_store::RetryConfig; + use object_store::aws::AmazonS3Builder; + + let retry_state = Arc::new(MultipartRetryState { + failures_remaining: AtomicUsize::new(3), + part_uris: std::sync::Mutex::new(Vec::new()), + }); + let throttle_state = AimdThrottleState::new(AimdThrottleConfig::default()).unwrap(); + let connector = AimdMultipartUploadConnector::new( + MultipartRetryConnector { + state: Arc::clone(&retry_state), + }, + Some(&throttle_state), + ); + let store = AmazonS3Builder::new() + .with_bucket_name("bucket") + .with_region("us-east-1") + .with_skip_signature(true) + .with_retry(RetryConfig { + max_retries: 0, + ..Default::default() + }) + .with_http_connector(connector) + .build() + .unwrap(); + + let mut upload = store.put_multipart(&Path::from("object")).await.unwrap(); + upload + .put_part(PutPayload::from_static(b"payload")) + .await + .unwrap(); + upload.complete().await.unwrap(); + + let part_uris = retry_state.part_uris.lock().unwrap(); + assert_eq!(part_uris.len(), 4); + assert!(part_uris.iter().all(|uri| uri == &part_uris[0])); + assert!(part_uris[0].contains("partNumber=1")); + } + #[tokio::test] async fn test_throttled_multipart_reorders_parts() { let store = Arc::new(InMemory::new()) as Arc; diff --git a/rust/lance-io/src/object_writer.rs b/rust/lance-io/src/object_writer.rs index 0fd0a30f9e7..68bfdb638bb 100644 --- a/rust/lance-io/src/object_writer.rs +++ b/rust/lance-io/src/object_writer.rs @@ -5,15 +5,15 @@ use std::io; use std::pin::Pin; use std::sync::{Arc, OnceLock}; use std::task::Poll; +use std::time::Instant; use crate::object_store::ObjectStore as LanceObjectStore; use async_trait::async_trait; use bytes::Bytes; use futures::FutureExt; use futures::future::BoxFuture; -use object_store::{Error as OSError, ObjectStore, Result as OSResult, path::Path}; use object_store::{MultipartUpload, ObjectStoreExt}; -use rand::Rng; +use object_store::{ObjectStore, path::Path}; use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio::task::JoinSet; @@ -21,13 +21,13 @@ use lance_core::{Error, Result}; use tracing::Instrument; use crate::traits::Writer; -use crate::utils::tracking_store::IOTracker; +use crate::utils::tracking_store::{IOTracker, IoMetricsGuard}; use tokio::runtime::Handle; /// Start at 5MB. const INITIAL_UPLOAD_STEP: usize = 1024 * 1024 * 5; -fn max_upload_parallelism() -> usize { +pub(crate) fn max_upload_parallelism() -> usize { static MAX_UPLOAD_PARALLELISM: OnceLock = OnceLock::new(); *MAX_UPLOAD_PARALLELISM.get_or_init(|| { std::env::var("LANCE_UPLOAD_CONCURRENCY") @@ -37,16 +37,6 @@ fn max_upload_parallelism() -> usize { }) } -fn max_conn_reset_retries() -> u16 { - static MAX_CONN_RESET_RETRIES: OnceLock = OnceLock::new(); - *MAX_CONN_RESET_RETRIES.get_or_init(|| { - std::env::var("LANCE_CONN_RESET_RETRIES") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(20) - }) -} - /// Maximum body size for a single S3 PUT: strictly less than 5 GiB. /// AWS rejects single-PUT bodies of exactly 5 GiB (= 5 * 1024^3) with /// `EntityTooLarge`, so we clamp `LANCE_INITIAL_UPLOAD_SIZE` one byte @@ -61,7 +51,7 @@ fn clamp_initial_upload_size(raw: usize) -> (usize, bool) { (clamped, clamped != raw) } -fn initial_upload_size() -> usize { +pub(crate) fn initial_upload_size() -> usize { static LANCE_INITIAL_UPLOAD_SIZE: OnceLock = OnceLock::new(); *LANCE_INITIAL_UPLOAD_SIZE.get_or_init(|| { let Some(raw) = std::env::var("LANCE_INITIAL_UPLOAD_SIZE") @@ -89,12 +79,16 @@ fn initial_upload_size() -> usize { /// PUT request. If the object is larger, the writer will create a multipart /// upload and upload parts in parallel. /// +/// Parts stay in flight across writes and flushes, so a writer can hold up to +/// `LANCE_UPLOAD_CONCURRENCY` part bodies in memory at once. With a large +/// `LANCE_INITIAL_UPLOAD_SIZE` that product is what bounds the writer's +/// footprint, not the part size alone. +/// /// This implements the `AsyncWrite` trait. pub struct ObjectWriter { state: UploadState, path: Arc, cursor: usize, - connection_resets: u16, buffer: Vec, // TODO: use constant size to support R2 use_constant_size_upload_parts: bool, @@ -106,23 +100,123 @@ pub struct WriteResult { pub e_tag: Option, } +/// An object-store upload failure, annotated with what Lance was uploading. +/// +/// `object_store` reports its own elapsed time, but its clock starts inside +/// `RetryContext::new`, which runs on the *first poll* of the request future. +/// The `elapsed` reported here is measured from the moment Lance handed the +/// request to the uploader, so the two together tell a slow request (both +/// durations agree) apart from one whose task sat unpolled before it ever +/// issued (this duration is much larger). That distinction is what identifies +/// runtime starvation as the cause of a whole-request timeout, and it is not +/// recoverable from the object-store error alone. +#[derive(Debug)] +struct UploadFailure { + context: String, + /// The kind `into_io_error` restores. Without it every contextualized + /// failure would collapse to `ErrorKind::Other`, changing what callers + /// matching on the kind observe. + kind: io::ErrorKind, + source: Box, +} + +impl UploadFailure { + /// Wraps an object store error. + /// + /// The `io::ErrorKind` is taken from `object_store`'s own conversion rather + /// than a local copy of its mapping, and the error itself is kept as the + /// source, so both callers matching on the kind and `Error::is_not_found` + /// (which downcasts along the source chain) keep working. + fn new(context: String, source: object_store::Error) -> Self { + let mapped = io::Error::from(source); + let kind = mapped.kind(); + let source: Box = + match mapped.downcast::() { + Ok(source) => Box::new(source), + Err(mapped) => Box::new(mapped), + }; + Self { + context, + kind, + source, + } + } + + /// Wraps a failure that carries no object store error to map a kind from. + fn from_task(context: String, source: tokio::task::JoinError) -> Self { + Self { + context, + kind: io::ErrorKind::Other, + source: Box::new(source), + } + } + + fn into_io_error(self) -> io::Error { + let kind = self.kind; + io::Error::new(kind, self) + } +} + +impl std::fmt::Display for UploadFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.context, self.source) + } +} + +impl std::error::Error for UploadFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + +type UploadResult = std::result::Result; + +/// Identifies a single part upload, for its failure message. +struct PartUpload { + path: Arc, + part_idx: u16, + /// Concurrent part uploads, counting this one, when it was submitted. + parts_in_flight: usize, + /// The part size in effect, which is the buffer capacity this part was + /// filled to. [`ObjectWriter::next_part_buffer`] grows the part size every + /// 100 parts so one upload can cover a very large object within the + /// 10,000-part limit, so on a long upload this is a multiple of + /// `LANCE_INITIAL_UPLOAD_SIZE` rather than equal to it. A body smaller than + /// this is the final flush. + part_size: usize, +} + +/// Describes the upload knobs in effect, for inclusion in failure messages. +/// +/// Both are process-global and read from the environment, so a failure that is +/// sensitive to either is impossible to interpret without them. Note that +/// `LANCE_INITIAL_UPLOAD_SIZE` is the starting part size, not the size in +/// effect: see [`PartUpload::part_size`]. +fn upload_settings() -> String { + format!( + "LANCE_INITIAL_UPLOAD_SIZE={} bytes, LANCE_UPLOAD_CONCURRENCY={}", + initial_upload_size(), + max_upload_parallelism() + ) +} + enum UploadState { /// The writer has been opened but no data has been written yet. Will be in /// this state until the buffer is full or the writer is shut down. Started(Arc), /// The writer is in the process of creating a multipart upload. - CreatingUpload(BoxFuture<'static, OSResult>>), + CreatingUpload(BoxFuture<'static, UploadResult>>), /// The writer is in the process of uploading parts. InProgress { part_idx: u16, upload: Box, - futures: JoinSet>, + futures: JoinSet>, }, /// The writer is in the process of uploading data in a single PUT request. /// This happens when shutdown is called before the buffer is full. - PuttingSingle(BoxFuture<'static, OSResult>), + PuttingSingle(BoxFuture<'static, UploadResult>), /// The writer is in the process of completing the multipart upload. - Completing(BoxFuture<'static, OSResult>), + Completing(BoxFuture<'static, UploadResult>), /// The writer has been shut down and all data has been written. Done(WriteResult), } @@ -134,9 +228,20 @@ impl UploadState { let this = std::mem::replace(self, Self::Done(WriteResult::default())); *self = match this { Self::Started(store) => { + tracing::Span::current().record("part_count", 1_u64); + let started_at = Instant::now(); let fut = async move { let size = buffer.len(); - let res = store.put(&path, buffer.into()).await?; + let res = store.put(&path, buffer.into()).await.map_err(|source| { + UploadFailure::new( + format!( + "single PUT of {path} failed after {:?} ({size} bytes, {})", + started_at.elapsed(), + upload_settings() + ), + source, + ) + })?; Ok(WriteResult { size, e_tag: res.e_tag, @@ -148,18 +253,30 @@ impl UploadState { } } - fn in_progress_to_completing(&mut self) { + fn in_progress_to_completing(&mut self, path: Arc, bytes_written: usize) { // To get owned self, we temporarily swap with Done. let this = std::mem::replace(self, Self::Done(WriteResult::default())); *self = match this { Self::InProgress { mut upload, futures, - .. + part_idx, } => { debug_assert!(futures.is_empty()); + tracing::Span::current().record("part_count", part_idx as u64); + let started_at = Instant::now(); let fut = async move { - let res = upload.complete().await?; + let res = upload.complete().await.map_err(|source| { + UploadFailure::new( + format!( + "completing multipart upload of {path} failed after {:?} \ + ({part_idx} parts, {bytes_written} bytes, {})", + started_at.elapsed(), + upload_settings() + ), + source, + ) + })?; Ok(WriteResult { size: 0, // This will be set properly later. e_tag: res.e_tag, @@ -178,7 +295,6 @@ impl ObjectWriter { state: UploadState::Started(object_store.inner.clone()), cursor: 0, path: Arc::new(path.clone()), - connection_resets: 0, buffer: Vec::with_capacity(initial_upload_size()), use_constant_size_upload_parts: object_store.use_constant_size_upload_parts, }) @@ -202,24 +318,33 @@ impl ObjectWriter { fn put_part( upload: &mut dyn MultipartUpload, buffer: Bytes, - part_idx: u16, - sleep: Option, - ) -> BoxFuture<'static, std::result::Result<(), UploadPutError>> { - log::debug!( - "MultipartUpload submitting part with {} bytes", - buffer.len() - ); - let fut = upload.put_part(buffer.clone().into()); + part: PartUpload, + ) -> BoxFuture<'static, UploadResult<()>> { + let body_size = buffer.len(); + log::debug!("MultipartUpload submitting part with {} bytes", body_size); + // Stamped before the future is spawned so the reported duration covers + // any time the task spent waiting to be polled, not just the request. + let queued_at = Instant::now(); + let fut = upload.put_part(buffer.into()); Box::pin(async move { - if let Some(sleep) = sleep { - tokio::time::sleep(sleep).await; - } - fut.await.map_err(|source| UploadPutError { - part_idx, - buffer, - source, - })?; - Ok(()) + fut.await.map_err(|source| { + let PartUpload { + path, + part_idx, + parts_in_flight, + part_size, + } = part; + UploadFailure::new( + format!( + "multipart upload of part {part_idx} of {path} failed after {:?} \ + ({body_size} bytes, part_size={part_size} bytes, \ + parts_in_flight={parts_in_flight} at submission, {})", + queued_at.elapsed(), + upload_settings() + ), + source, + ) + }) }) } @@ -235,12 +360,24 @@ impl ObjectWriter { Poll::Ready(Ok(mut upload)) => { let mut futures = JoinSet::new(); + // Read before the buffer is swapped out: capacity is the + // part size this body was filled to. + let part_size = mut_self.buffer.capacity(); let data = Self::next_part_buffer( &mut mut_self.buffer, 0, mut_self.use_constant_size_upload_parts, ); - futures.spawn(Self::put_part(upload.as_mut(), data, 0, None)); + futures.spawn(Self::put_part( + upload.as_mut(), + data, + PartUpload { + path: mut_self.path.clone(), + part_idx: 0, + parts_in_flight: 1, + part_size, + }, + )); mut_self.state = UploadState::InProgress { part_idx: 1, // We just used 0 @@ -248,46 +385,24 @@ impl ObjectWriter { upload, }; } - Poll::Ready(Err(e)) => return Err(std::io::Error::other(e)), + Poll::Ready(Err(err)) => return Err(err.into_io_error()), Poll::Pending => break, }, - UploadState::InProgress { - upload, futures, .. - } => { + UploadState::InProgress { futures, .. } => { while let Poll::Ready(Some(res)) = futures.poll_join_next(cx) { match res { Ok(Ok(())) => {} - Err(err) => return Err(std::io::Error::other(err)), - Ok(Err(err)) if should_retry_upload_put(&err.source) => { - if mut_self.connection_resets < max_conn_reset_retries() { - // Retry, but only up to max_conn_reset_retries of them. - mut_self.connection_resets += 1; - - // Resubmit with random jitter - let sleep_time_ms = rand::rng().random_range(2_000..8_000); - let sleep_time = - std::time::Duration::from_millis(sleep_time_ms); - - futures.spawn(Self::put_part( - upload.as_mut(), - err.buffer, - err.part_idx, - Some(sleep_time), - )); - } else { - return Err(io::Error::new( - io::ErrorKind::ConnectionReset, - Box::new(ConnectionResetError { - message: format!( - "Hit max retries ({}) for retryable upload error", - max_conn_reset_retries() - ), - source: Box::new(err.source), - }), - )); - } + Err(err) => { + return Err(UploadFailure::from_task( + format!( + "multipart upload task for {} did not complete", + mut_self.path + ), + err, + ) + .into_io_error()); } - Ok(Err(err)) => return Err(err.source.into()), + Ok(Err(err)) => return Err(err.into_io_error()), } } break; @@ -298,7 +413,7 @@ impl ObjectWriter { res.size = mut_self.cursor; mut_self.state = UploadState::Done(res) } - Poll::Ready(Err(e)) => return Err(std::io::Error::other(e)), + Poll::Ready(Err(err)) => return Err(err.into_io_error()), Poll::Pending => break, } } @@ -333,38 +448,6 @@ impl Drop for ObjectWriter { } } -/// Returned error from trying to upload a part. -/// Has the part_idx and buffer so we can pass -/// them to the retry logic. -struct UploadPutError { - part_idx: u16, - buffer: Bytes, - source: OSError, -} - -fn should_retry_upload_put(source: &OSError) -> bool { - let OSError::Generic { source, .. } = source else { - return false; - }; - - let message = source.to_string().to_ascii_lowercase(); - message.contains("connection reset by peer") || message.contains("requesttimeout") -} - -#[derive(Debug)] -struct ConnectionResetError { - message: String, - source: Box, -} - -impl std::error::Error for ConnectionResetError {} - -impl std::fmt::Display for ConnectionResetError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}: {}", self.message, self.source) - } -} - impl AsyncWrite for ObjectWriter { fn poll_write( mut self: std::pin::Pin<&mut Self>, @@ -389,28 +472,48 @@ impl AsyncWrite for ObjectWriter { UploadState::Started(store) => { let path = mut_self.path.clone(); let store = store.clone(); - let fut = Box::pin(async move { store.put_multipart(path.as_ref()).await }); + let started_at = Instant::now(); + let fut = Box::pin(async move { + store.put_multipart(path.as_ref()).await.map_err(|source| { + UploadFailure::new( + format!( + "failed to create multipart upload for {path} after {:?} ({})", + started_at.elapsed(), + upload_settings() + ), + source, + ) + }) + }); self.state = UploadState::CreatingUpload(fut); } + // TODO: Make max concurrency configurable from storage options. UploadState::InProgress { upload, part_idx, futures, .. - } => { - // TODO: Make max concurrency configurable from storage options. - if futures.len() < max_upload_parallelism() { - let data = Self::next_part_buffer( - &mut mut_self.buffer, - *part_idx, - mut_self.use_constant_size_upload_parts, - ); - futures.spawn( - Self::put_part(upload.as_mut(), data, *part_idx, None) - .instrument(tracing::Span::current()), - ); - *part_idx += 1; - } + } if futures.len() < max_upload_parallelism() => { + // Read before the buffer is swapped out: capacity is the + // part size this body was filled to, which grows as the + // upload progresses. + let part_size = mut_self.buffer.capacity(); + let data = Self::next_part_buffer( + &mut mut_self.buffer, + *part_idx, + mut_self.use_constant_size_upload_parts, + ); + let part = PartUpload { + path: mut_self.path.clone(), + part_idx: *part_idx, + parts_in_flight: futures.len() + 1, + part_size, + }; + futures.spawn( + Self::put_part(upload.as_mut(), data, part) + .instrument(tracing::Span::current()), + ); + *part_idx += 1; } _ => {} } @@ -435,13 +538,16 @@ impl AsyncWrite for ObjectWriter { UploadState::CreatingUpload(_) | UploadState::Completing(_) | UploadState::PuttingSingle(_) => Poll::Pending, - UploadState::InProgress { futures, .. } => { - if futures.is_empty() { - Poll::Ready(Ok(())) - } else { - Poll::Pending - } - } + // In-flight parts are spawned tasks, so the runtime drives them + // whether or not this writer is polled again; `poll_tasks` above + // only reaps them. Waiting for them here would serialize every part + // upload behind the caller's next batch, because callers flush once + // per batch. `poll_shutdown` still drains them before completing the + // upload, which is the only point at which the object becomes + // readable. Note this never flushed the tail buffer either, so it + // was not a "all data has reached the destination" barrier to begin + // with. + UploadState::InProgress { .. } => Poll::Ready(Ok(())), } } @@ -474,9 +580,19 @@ impl AsyncWrite for ObjectWriter { // Flush final batch if !mut_self.buffer.is_empty() && futures.len() < max_upload_parallelism() { // We can just use `take` since we don't need the buffer anymore. + let part_size = mut_self.buffer.capacity(); let data = Bytes::from(std::mem::take(&mut mut_self.buffer)); + let part = PartUpload { + path: mut_self.path.clone(), + part_idx: *part_idx, + parts_in_flight: futures.len() + 1, + part_size, + }; + // Counted like every other part so the part total + // reported when completing the upload is accurate. + *part_idx += 1; futures.spawn( - Self::put_part(upload.as_mut(), data, *part_idx, None) + Self::put_part(upload.as_mut(), data, part) .instrument(tracing::Span::current()), ); // We need to go back to beginning of loop to poll the @@ -486,7 +602,9 @@ impl AsyncWrite for ObjectWriter { // We handle the transition from in progress to completing here. if futures.is_empty() { - self.state.in_progress_to_completing(); + let path = mut_self.path.clone(); + let bytes_written = mut_self.cursor; + self.state.in_progress_to_completing(path, bytes_written); } else { return Poll::Pending; } @@ -503,12 +621,10 @@ impl Writer for ObjectWriter { } async fn shutdown(&mut self) -> Result { - AsyncWriteExt::shutdown(self).await.map_err(|e| { - Error::io(format!( - "failed to shutdown object writer for {}: {}", - self.path, e - )) - })?; + // Propagated structurally rather than formatted into a message: every + // failure from this writer already names the path, and stringifying it + // would flatten the object store error out of the source chain. + AsyncWriteExt::shutdown(self).await?; if let UploadState::Done(result) = &self.state { Ok(result.clone()) } else { @@ -524,7 +640,7 @@ pub struct LocalWriter { #[derive(Default)] enum LocalWriteState { - Writing(WritingState), + Writing(Box), Finishing { size: usize, future: BoxFuture<'static, Result>, @@ -540,6 +656,10 @@ struct WritingState { /// Temp path that auto-deletes on drop. Set to `None` after `persist()`. temp_path: tempfile::TempPath, io_tracker: Arc, + /// The whole file is reported as a single `put`, so this covers everything + /// from opening the file to it being durable under its final path. A writer + /// dropped before `persist()` records nothing, like an aborted upload. + metrics: IoMetricsGuard, } impl LocalWriter { @@ -551,12 +671,13 @@ impl LocalWriter { ) -> Self { Self { path, - state: LocalWriteState::Writing(WritingState { + state: LocalWriteState::Writing(Box::new(WritingState { writer: tokio::io::BufWriter::new(file), cursor: 0, temp_path, + metrics: io_tracker.begin_io("put"), io_tracker, - }), + })), } } @@ -576,9 +697,10 @@ impl LocalWriter { final_path: Path, size: usize, io_tracker: Arc, + metrics: IoMetricsGuard, ) -> Result { let local_path = crate::local::to_local_path(&final_path); - let e_tag = tokio::task::spawn_blocking(move || -> Result { + let persisted = tokio::task::spawn_blocking(move || -> Result { temp_path.persist(&local_path).map_err(|e| { Error::io(format!( "failed to persist temp file to {}: {}", @@ -592,7 +714,11 @@ impl LocalWriter { Ok(get_etag(&metadata)) }) .await - .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))??; + .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e))) + .and_then(|e_tag| e_tag); + + metrics.record(&persisted, size as u64); + let e_tag = persisted?; io_tracker.record_write("put", final_path, size as u64); @@ -657,6 +783,7 @@ impl AsyncWrite for LocalWriter { mut_self.path.clone(), size, state.io_tracker, + state.metrics, )), }; } @@ -730,10 +857,550 @@ fn get_inode(_metadata: &std::fs::Metadata) -> u64 { #[cfg(test)] mod tests { + use futures::stream::BoxStream; + use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, ObjectMeta, PutMultipartOptions, + PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult, UploadPart, + }; + use std::sync::Mutex; + use std::time::Duration; use tokio::io::AsyncWriteExt; + use tokio::sync::Semaphore; use super::*; + /// Which stage of an upload the mock store rejects. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum FailAt { + Nothing, + CreateMultipart, + PutPart, + Complete, + SinglePut, + } + + /// What the mock store saw, so a test can assert on uploads that are still + /// in flight as well as on the object they eventually assemble. + #[derive(Debug)] + struct UploadObservations { + /// One permit per part upload that has begun. A test waits on this + /// rather than sampling a counter: `write_all` and a non-waiting + /// `flush` can both complete without ever returning `Pending`, so on a + /// current-thread runtime the spawned upload tasks may not have run yet. + started: Semaphore, + /// `(part index, body)` in completion order. The index is recorded + /// because it, not completion order, determines the assembled object. + parts: Mutex)>>, + } + + impl Default for UploadObservations { + fn default() -> Self { + Self { + started: Semaphore::new(0), + parts: Mutex::new(Vec::new()), + } + } + } + + fn rejected(stage: &'static str) -> object_store::Error { + object_store::Error::Generic { + store: "FailingUploadStore", + source: format!("{stage} rejected by test").into(), + } + } + + #[derive(Debug)] + struct FailingUpload { + fail_at: FailAt, + /// When set, a part upload does not resolve until the gate is given a + /// permit, so a test can hold requests in flight. + gate: Option>, + observations: Arc, + next_part: usize, + } + + #[async_trait] + impl MultipartUpload for FailingUpload { + fn put_part(&mut self, data: PutPayload) -> UploadPart { + let fails = self.fail_at == FailAt::PutPart; + let part_idx = self.next_part; + self.next_part += 1; + let gate = self.gate.clone(); + let observations = self.observations.clone(); + Box::pin(async move { + observations.started.add_permits(1); + if let Some(gate) = gate { + // `forget` keeps the permit from being returned on drop, so + // adding N permits releases exactly N parts. + gate.acquire_owned().await.unwrap().forget(); + } + if fails { + return Err(rejected("part")); + } + let body = data + .iter() + .flat_map(|chunk| chunk.iter().copied()) + .collect(); + observations.parts.lock().unwrap().push((part_idx, body)); + Ok(()) + }) + } + + async fn complete(&mut self) -> OSResult { + if self.fail_at == FailAt::Complete { + Err(rejected("complete")) + } else { + Ok(PutResult { + e_tag: None, + version: None, + }) + } + } + + async fn abort(&mut self) -> OSResult<()> { + Ok(()) + } + } + + /// Rejects exactly one stage of an upload so each failure site can be + /// exercised on its own, and optionally holds part uploads open. + #[derive(Debug)] + struct FailingUploadStore { + fail_at: FailAt, + gate: Option>, + observations: Arc, + } + + impl FailingUploadStore { + fn new(fail_at: FailAt) -> Self { + Self { + fail_at, + gate: None, + observations: Arc::new(UploadObservations::default()), + } + } + + /// Builds a store whose part uploads stay in flight until the returned + /// gate is given permits. + fn gated(fail_at: FailAt) -> (Self, Arc) { + let gate = Arc::new(Semaphore::new(0)); + let store = Self { + fail_at, + gate: Some(gate.clone()), + observations: Arc::new(UploadObservations::default()), + }; + (store, gate) + } + } + + impl std::fmt::Display for FailingUploadStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FailingUploadStore") + } + } + + #[async_trait] + impl ObjectStore for FailingUploadStore { + async fn put_opts( + &self, + _location: &Path, + _bytes: PutPayload, + _opts: PutOptions, + ) -> OSResult { + if self.fail_at == FailAt::SinglePut { + Err(rejected("single put")) + } else { + Ok(PutResult { + e_tag: None, + version: None, + }) + } + } + + async fn put_multipart_opts( + &self, + _location: &Path, + _opts: PutMultipartOptions, + ) -> OSResult> { + if self.fail_at == FailAt::CreateMultipart { + Err(rejected("create multipart")) + } else { + Ok(Box::new(FailingUpload { + fail_at: self.fail_at, + gate: self.gate.clone(), + observations: self.observations.clone(), + next_part: 0, + })) + } + } + + async fn get_opts(&self, _location: &Path, _options: GetOptions) -> OSResult { + unimplemented!() + } + + fn delete_stream( + &self, + _locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + fn list_with_offset( + &self, + _prefix: Option<&Path>, + _offset: &Path, + ) -> BoxStream<'static, OSResult> { + unimplemented!() + } + + async fn list_with_delimiter(&self, _prefix: Option<&Path>) -> OSResult { + unimplemented!() + } + + async fn copy_opts(&self, _from: &Path, _to: &Path, _opts: CopyOptions) -> OSResult<()> { + unimplemented!() + } + + async fn rename_opts( + &self, + _from: &Path, + _to: &Path, + _opts: RenameOptions, + ) -> OSResult<()> { + unimplemented!() + } + } + + const FAILING_UPLOAD_PATH: &str = "part_7_invert.lance"; + + /// Enough bytes for two full multipart parts, so a failing part has a + /// sibling in flight. Derived from the configured part size rather than the + /// default, since `LANCE_INITIAL_UPLOAD_SIZE` may raise it. + fn two_parts() -> usize { + initial_upload_size() * 2 + } + + /// Drives a write against a store that rejects `fail_at`, returning the + /// error. The failure can surface either from a write or from shutdown + /// depending on when the rejected request is reaped, so both are checked. + async fn failing_upload(fail_at: FailAt, num_bytes: usize) -> io::Error { + let mut store = LanceObjectStore::memory(); + store.inner = Arc::new(FailingUploadStore::new(fail_at)); + + let mut writer = ObjectWriter::new(&store, &Path::from(FAILING_UPLOAD_PATH)) + .await + .unwrap(); + let buf = vec![0u8; num_bytes]; + match writer.write_all(buf.as_slice()).await { + Err(err) => err, + Ok(()) => AsyncWriteExt::shutdown(&mut writer) + .await + .expect_err("upload should have failed"), + } + } + + #[tokio::test] + async fn test_part_upload_failure_reports_upload_context() { + let err = failing_upload(FailAt::PutPart, two_parts()).await; + let message = err.to_string(); + + assert!( + message.contains("multipart upload of part"), + "should name the failing stage: {message}" + ); + assert!( + message.contains(FAILING_UPLOAD_PATH), + "should name the object: {message}" + ); + assert!( + message.contains(&format!("{} bytes", initial_upload_size())), + "should report the body size: {message}" + ); + assert!( + message.contains(&format!("part_size={} bytes", initial_upload_size())), + "should report the part size in effect: {message}" + ); + assert!( + message.contains("parts_in_flight="), + "should report upload concurrency in use: {message}" + ); + assert!( + message.contains("LANCE_INITIAL_UPLOAD_SIZE") + && message.contains("LANCE_UPLOAD_CONCURRENCY"), + "should report the knobs governing the request: {message}" + ); + assert!( + message.contains("part rejected by test"), + "should keep the underlying object store error: {message}" + ); + } + + // The elapsed time is the whole point of the added context: it is what tells + // a slow request apart from one whose task was never polled. + #[tokio::test] + async fn test_part_upload_failure_reports_elapsed_time() { + let err = failing_upload(FailAt::PutPart, two_parts()).await; + let message = err.to_string(); + assert!( + message.contains("failed after"), + "should report how long the request took: {message}" + ); + } + + // `part_size` in the failure message is the live buffer capacity, which is + // what makes it report the size actually in effect. The part size grows + // every 100 parts, so on a long upload that diverges from + // LANCE_INITIAL_UPLOAD_SIZE by a multiple; reporting only the configured + // value would understate a late part by that factor. Reaching part 100 + // through the writer would mean allocating hundreds of MiB, so the growth + // is asserted on the buffer the message reads from. + #[test] + fn test_part_buffer_capacity_tracks_grown_part_size() { + let mut buffer = Vec::::with_capacity(initial_upload_size()); + assert_eq!(buffer.capacity(), initial_upload_size()); + + let _ = ObjectWriter::next_part_buffer(&mut buffer, 0, false); + assert_eq!( + buffer.capacity(), + initial_upload_size(), + "early parts stay at the configured size" + ); + + let _ = ObjectWriter::next_part_buffer(&mut buffer, 100, false); + assert_eq!( + buffer.capacity(), + initial_upload_size().max(2 * INITIAL_UPLOAD_STEP), + "the part size has grown past the first step" + ); + + // A store pinned to constant part sizes never grows, so the reported + // size stays equal to the configured one. + let _ = ObjectWriter::next_part_buffer(&mut buffer, 100, true); + assert_eq!(buffer.capacity(), initial_upload_size()); + } + + #[tokio::test] + async fn test_part_upload_failure_preserves_source_chain() { + let err = failing_upload(FailAt::PutPart, two_parts()).await; + + let failure = err + .get_ref() + .expect("io error should carry the upload failure"); + let source = std::error::Error::source(failure) + .expect("upload failure should expose the object store error"); + assert!( + source.downcast_ref::().is_some(), + "source should still be the object store error, got: {source}" + ); + } + + /// Adding context must not flatten the `io::ErrorKind` that `object_store` + /// maps an error to, since that kind is observable to callers. + #[test] + fn test_upload_failure_preserves_error_kind() { + fn not_found() -> object_store::Error { + object_store::Error::NotFound { + path: FAILING_UPLOAD_PATH.to_string(), + source: "not found".into(), + } + } + + let unwrapped = io::Error::from(not_found()); + assert_eq!(unwrapped.kind(), io::ErrorKind::NotFound); + + let wrapped = + UploadFailure::new("part upload failed".to_string(), not_found()).into_io_error(); + assert_eq!(wrapped.kind(), unwrapped.kind()); + } + + /// `Writer::shutdown` is the public boundary most callers see. The object + /// store error has to remain reachable through it, not be flattened into a + /// message. + #[tokio::test] + async fn test_writer_shutdown_preserves_object_store_source() { + let mut store = LanceObjectStore::memory(); + store.inner = Arc::new(FailingUploadStore::new(FailAt::SinglePut)); + let mut writer = ObjectWriter::new(&store, &Path::from(FAILING_UPLOAD_PATH)) + .await + .unwrap(); + writer.write_all(&[0u8; 256]).await.unwrap(); + let err = Writer::shutdown(&mut writer).await.unwrap_err(); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&err); + let mut found_object_store = false; + while let Some(source) = current { + if source.downcast_ref::().is_some() { + found_object_store = true; + break; + } + current = source.source(); + } + assert!(found_object_store, "source chain was flattened: {err:?}"); + + assert!( + err.to_string().contains(FAILING_UPLOAD_PATH), + "should still name the object: {err}" + ); + } + + #[tokio::test] + async fn test_create_multipart_failure_reports_upload_context() { + let err = failing_upload(FailAt::CreateMultipart, two_parts()).await; + let message = err.to_string(); + + assert!( + message.contains("failed to create multipart upload for"), + "should name the failing stage: {message}" + ); + assert!( + message.contains(FAILING_UPLOAD_PATH), + "should name the object: {message}" + ); + assert!( + message.contains("create multipart rejected by test"), + "should keep the underlying object store error: {message}" + ); + } + + #[tokio::test] + async fn test_complete_multipart_failure_reports_upload_context() { + let num_bytes = two_parts(); + let err = failing_upload(FailAt::Complete, num_bytes).await; + let message = err.to_string(); + + assert!( + message.contains("completing multipart upload of"), + "should name the failing stage: {message}" + ); + assert!( + message.contains(&format!("{num_bytes} bytes")), + "should report how much had been written: {message}" + ); + assert!( + message.contains("complete rejected by test"), + "should keep the underlying object store error: {message}" + ); + } + + #[tokio::test] + async fn test_single_put_failure_reports_upload_context() { + // Below the multipart threshold, so shutdown takes the single-PUT path. + let err = failing_upload(FailAt::SinglePut, 256).await; + let message = err.to_string(); + + assert!( + message.contains("single PUT of"), + "should name the failing stage: {message}" + ); + assert!( + message.contains(FAILING_UPLOAD_PATH), + "should name the object: {message}" + ); + assert!( + message.contains("256 bytes"), + "should report the body size: {message}" + ); + assert!( + message.contains("single put rejected by test"), + "should keep the underlying object store error: {message}" + ); + } + + /// Released permits, comfortably above the part count of any test here so + /// no test depends on the exact number of parts a payload produces. + const GATE_RELEASE: usize = 64; + + /// How long a flush is allowed to take before it counts as waiting. A flush + /// that does not wait resolves immediately; this bound only has to be short + /// of the test harness timeout. + const FLUSH_BOUND: Duration = Duration::from_secs(10); + + /// Blocks until a part upload has begun, so the assertions that follow are + /// made against a request that is genuinely in flight. + async fn await_part_in_flight(observations: &UploadObservations) { + tokio::time::timeout(FLUSH_BOUND, observations.started.acquire()) + .await + .expect("a part upload should have started") + .unwrap() + .forget(); + } + + #[tokio::test] + async fn test_flush_does_not_wait_for_in_flight_parts() { + let (store, gate) = FailingUploadStore::gated(FailAt::Nothing); + let observations = store.observations.clone(); + let mut lance_store = LanceObjectStore::memory(); + lance_store.inner = Arc::new(store); + + let mut writer = ObjectWriter::new(&lance_store, &Path::from("gated.lance")) + .await + .unwrap(); + // Distinct bytes so a part landing out of order is detectable. + let payload = (0..two_parts()).map(|i| i as u8).collect::>(); + writer.write_all(payload.as_slice()).await.unwrap(); + await_part_in_flight(&observations).await; + + tokio::time::timeout(FLUSH_BOUND, AsyncWriteExt::flush(&mut writer)) + .await + .expect("flush must not wait for in-flight part uploads") + .unwrap(); + + assert!( + observations.parts.lock().unwrap().is_empty(), + "no gated part may have completed before the gate opened" + ); + + gate.add_permits(GATE_RELEASE); + let result = Writer::shutdown(&mut writer).await.unwrap(); + assert_eq!(result.size, payload.len()); + + let mut parts = observations.parts.lock().unwrap().clone(); + parts.sort_by_key(|(part_idx, _)| *part_idx); + let assembled = parts + .into_iter() + .flat_map(|(_, body)| body) + .collect::>(); + assert_eq!( + assembled, payload, + "parts must reassemble into the original bytes" + ); + } + + #[tokio::test] + async fn test_part_failure_after_flush_surfaces_at_shutdown() { + let (store, gate) = FailingUploadStore::gated(FailAt::PutPart); + let observations = store.observations.clone(); + let mut lance_store = LanceObjectStore::memory(); + lance_store.inner = Arc::new(store); + + let mut writer = ObjectWriter::new(&lance_store, &Path::from(FAILING_UPLOAD_PATH)) + .await + .unwrap(); + writer + .write_all(vec![0u8; two_parts()].as_slice()) + .await + .unwrap(); + await_part_in_flight(&observations).await; + // The parts are still gated, so nothing has failed yet and flush passes. + AsyncWriteExt::flush(&mut writer).await.unwrap(); + + // Now let them fail. Shutdown is the first place that can report it, so + // no longer waiting in flush must not lose the error. + gate.add_permits(GATE_RELEASE); + let err = AsyncWriteExt::shutdown(&mut writer) + .await + .expect_err("a failed part upload must still surface"); + let message = err.to_string(); + assert!( + message.contains(FAILING_UPLOAD_PATH), + "should name the object being written: {message}" + ); + } + #[tokio::test] async fn test_write() { let store = LanceObjectStore::memory(); @@ -862,35 +1529,6 @@ mod tests { assert_eq!(clamp_initial_upload_size(mid), (mid, false)); } - #[test] - fn should_retry_upload_put_detects_transient_errors() { - let request_timeout = OSError::Generic { - store: "S3", - source: Box::new(io::Error::other( - "Server returned non-2xx status code: 400 Bad Request: \ - RequestTimeoutYour socket connection to the server \ - was not read from or written to within the timeout period. Idle connections will \ - be closed.", - )), - }; - assert!(should_retry_upload_put(&request_timeout)); - - let connection_reset = OSError::Generic { - store: "S3", - source: Box::new(io::Error::new( - io::ErrorKind::ConnectionReset, - "connection reset by peer", - )), - }; - assert!(should_retry_upload_put(&connection_reset)); - - let not_retryable = OSError::Generic { - store: "S3", - source: Box::new(io::Error::other("access denied")), - }; - assert!(!should_retry_upload_put(¬_retryable)); - } - #[test] fn clamp_initial_upload_size_above_max_is_clamped_down() { assert_eq!( diff --git a/rust/lance-io/src/scheduler.rs b/rust/lance-io/src/scheduler.rs index 0c2d4dc44bd..1876fe3feba 100644 --- a/rust/lance-io/src/scheduler.rs +++ b/rust/lance-io/src/scheduler.rs @@ -3,6 +3,7 @@ use bytes::Bytes; use futures::channel::oneshot; +use futures::future::Either; use futures::{FutureExt, TryFutureExt}; use object_store::path::Path; use std::collections::BinaryHeap; @@ -29,6 +30,7 @@ mod lite; const BACKPRESSURE_MIN: u64 = 5; // Don't log backpressure warnings more than once / minute const BACKPRESSURE_DEBOUNCE: u64 = 60; +const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state"; // Global counter of how many IOPS we have issued static IOPS_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -83,11 +85,23 @@ impl PrioritiesInFlight { self.in_flight.remove(pos); } } + + fn len(&self) -> usize { + self.in_flight.len() + } + + fn is_empty(&self) -> bool { + self.in_flight.is_empty() + } } struct IoQueueState { + // The configured number of IOPS that can be issued concurrently. + io_capacity: u32, // Number of IOPS we can issue concurrently before pausing I/O iops_avail: u32, + // The configured byte budget for unread I/O. + io_buffer_size: u64, // Number of bytes we are allowed to buffer in memory before pausing I/O // // This can dip below 0 due to I/O prioritization @@ -110,7 +124,9 @@ struct IoQueueState { impl IoQueueState { fn new(io_capacity: u32, io_buffer_size: u64) -> Self { Self { + io_capacity, iops_avail: io_capacity, + io_buffer_size, bytes_avail: io_buffer_size as i64, pending_requests: BinaryHeap::new(), priorities_in_flight: PrioritiesInFlight::new(io_capacity), @@ -121,6 +137,65 @@ impl IoQueueState { } } + fn scheduler_state_event(&self) -> Option { + if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) { + return None; + } + + let pending_bytes = self + .pending_requests + .iter() + .map(IoTask::num_bytes) + .sum::(); + let head_task = self.pending_requests.peek(); + let min_in_flight_priority = if self.priorities_in_flight.is_empty() { + None + } else { + Some(self.priorities_in_flight.min_in_flight()) + }; + let head_task_priority_bypass = head_task.map(|task| { + self.no_backpressure + || task.bypass_backpressure + || task.priority <= self.priorities_in_flight.min_in_flight() + }); + let head_task_blocked_by_iops = head_task.map(|_| self.iops_avail == 0); + let head_task_blocked_by_bytes = head_task.map(|task| { + let bypasses_bytes = self.no_backpressure + || task.bypass_backpressure + || task.priority <= self.priorities_in_flight.min_in_flight(); + !bypasses_bytes && task.num_bytes() as i64 > self.bytes_avail + }); + let head_task_can_deliver = head_task.map(|task| self.can_deliver_without_warning(task)); + let head_task_bytes = head_task.map(IoTask::num_bytes); + let (head_task_priority_high, head_task_priority_low) = + split_priority(head_task.map(|task| task.priority)); + let (min_in_flight_priority_high, min_in_flight_priority_low) = + split_priority(min_in_flight_priority); + + Some(SchedulerStateEvent { + queue_kind: "standard", + io_capacity: u64::from(self.io_capacity), + iops_available: u64::from(self.iops_avail), + active_iops: u64::from(self.io_capacity.saturating_sub(self.iops_avail)), + pending_iops: self.pending_requests.len() as u64, + pending_bytes, + bytes_available: self.bytes_avail, + bytes_reserved: self.io_buffer_size as i64 - self.bytes_avail, + io_buffer_size_bytes: self.io_buffer_size, + priorities_in_flight: self.priorities_in_flight.len() as u64, + no_backpressure: self.no_backpressure, + head_task_bytes, + head_task_priority_high, + head_task_priority_low, + min_in_flight_priority_high, + min_in_flight_priority_low, + head_task_can_deliver, + head_task_priority_bypass, + head_task_blocked_by_iops, + head_task_blocked_by_bytes, + }) + } + fn warn_if_needed(&self) { let seconds_elapsed = self.start.elapsed().as_secs(); let last_warn = self.last_warn.load(Ordering::Acquire); @@ -140,6 +215,20 @@ impl IoQueueState { } fn can_deliver(&self, task: &IoTask) -> bool { + let can_deliver = self.can_deliver_without_warning(task); + if !can_deliver + && self.iops_avail > 0 + && !(self.no_backpressure + || task.bypass_backpressure + || task.priority <= self.priorities_in_flight.min_in_flight()) + && task.num_bytes() as i64 > self.bytes_avail + { + self.warn_if_needed(); + } + can_deliver + } + + fn can_deliver_without_warning(&self, task: &IoTask) -> bool { if self.iops_avail == 0 { false } else if self.no_backpressure @@ -151,11 +240,8 @@ impl IoQueueState { || self.priorities_in_flight.contains(task.priority) { true - } else if task.num_bytes() as i64 > self.bytes_avail { - self.warn_if_needed(); - false } else { - true + task.num_bytes() as i64 <= self.bytes_avail } } @@ -191,13 +277,15 @@ struct IoQueue { state: Mutex, // Used to signal new I/O requests have arrived that might potentially be runnable notify: Notify, + stats: IoStats, } impl IoQueue { - fn new(io_capacity: u32, io_buffer_size: u64) -> Self { + fn new(io_capacity: u32, io_buffer_size: u64, stats: IoStats) -> Self { Self { state: Mutex::new(IoQueueState::new(io_capacity, io_buffer_size)), notify: Notify::new(), + stats, } } @@ -208,9 +296,12 @@ impl IoQueue { task.priority >> 64, task.priority & 0xFFFFFFFFFFFFFFFF ); - let mut state = self.state.lock().unwrap(); - state.pending_requests.push(task); - drop(state); + let event = { + let mut state = self.state.lock().unwrap(); + state.pending_requests.push(task); + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); self.notify.notify_one(); } @@ -220,6 +311,9 @@ impl IoQueue { { let mut state = self.state.lock().unwrap(); if let Some(task) = state.next_task() { + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); return Some(task); } @@ -233,29 +327,39 @@ impl IoQueue { } fn on_iop_complete(&self) { - let mut state = self.state.lock().unwrap(); - state.iops_avail += 1; - drop(state); + let event = { + let mut state = self.state.lock().unwrap(); + state.iops_avail += 1; + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); self.notify.notify_one(); } fn on_bytes_consumed(&self, bytes: u64, priority: u128, num_reqs: usize) { - let mut state = self.state.lock().unwrap(); - state.bytes_avail += bytes as i64; - for _ in 0..num_reqs { - state.priorities_in_flight.remove(priority); - } - drop(state); + let event = { + let mut state = self.state.lock().unwrap(); + state.bytes_avail += bytes as i64; + for _ in 0..num_reqs { + state.priorities_in_flight.remove(priority); + } + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); self.notify.notify_one(); } fn close(&self) { - let mut state = self.state.lock().unwrap(); - state.done_scheduling = true; - let pending_requests = std::mem::take(&mut state.pending_requests); - drop(state); + let (pending_requests, event) = { + let mut state = self.state.lock().unwrap(); + state.done_scheduling = true; + let pending_requests = std::mem::take(&mut state.pending_requests); + let event = state.scheduler_state_event(); + (pending_requests, event) + }; + emit_scheduler_state_event(event, &self.stats); for request in pending_requests { request.cancel(); } @@ -274,9 +378,13 @@ struct MutableBatch { num_bytes: u64, priority: u128, num_reqs: usize, - err: Option>, + num_delivered: usize, + err: Option, // When true, report 0 bytes consumed so the backpressure budget is unaffected bypass_backpressure: bool, + // Queue the batch's backpressure reservation is refunded to once its response + // is delivered or discarded (see `Response`'s `Drop`). + io_queue: Arc, } impl MutableBatch { @@ -286,6 +394,7 @@ impl MutableBatch { priority: u128, num_reqs: usize, bypass_backpressure: bool, + io_queue: Arc, ) -> Self { Self { when_done: Some(when_done), @@ -293,8 +402,10 @@ impl MutableBatch { num_bytes: 0, priority, num_reqs, + num_delivered: 0, err: None, bypass_backpressure, + io_queue, } } } @@ -305,9 +416,16 @@ impl MutableBatch { // data. impl Drop for MutableBatch { fn drop(&mut self) { - // If we have an error, return that. Otherwise return the data - let result = if self.err.is_some() { - Err(Error::wrapped(self.err.take().unwrap())) + // If we have an error, return that. Otherwise return the data, as long as the I/O requests have been processed. + let result = if let Some(err) = self.err.take() { + Err(err) + } else if self.num_delivered < self.data_buffers.len() { + // This usually happens on tokio runtime shutdown + Err(Error::io(format!( + "I/O request was dropped before completion ({} of {} reads delivered)", + self.num_delivered, + self.data_buffers.len() + ))) } else { let mut data = Vec::new(); std::mem::swap(&mut data, &mut self.data_buffers); @@ -316,7 +434,8 @@ impl Drop for MutableBatch { // We don't really care if no one is around to receive it, just let // the result go out of scope and get cleaned up let response = Response { - data: result, + data: Some(result), + io_queue: self.io_queue.clone(), // Report 0 bytes for bypass tasks so the backpressure budget is unaffected num_bytes: if self.bypass_backpressure { 0 @@ -344,13 +463,14 @@ impl DataSink for MutableBatch { // Called by worker tasks to add data to the MutableBatch fn deliver_data(&mut self, data: DataChunk) { self.num_bytes += data.num_bytes; + self.num_delivered += 1; match data.data { Ok(data_bytes) => { self.data_buffers[data.task_idx] = data_bytes; } Err(err) => { // This keeps the original error, if present - self.err.get_or_insert(Box::new(err)); + self.err.get_or_insert(err); } } } @@ -364,6 +484,23 @@ struct IoTask { bypass_backpressure: bool, } +fn validate_read_length( + file_path: &Path, + requested_range: &Range, + bytes: Bytes, +) -> Result { + let expected_len = requested_range.end - requested_range.start; + if bytes.len() as u64 != expected_len { + return Err(Error::io(format!( + "I/O request for file {file_path} and range {}..{} returned {} bytes, expected {expected_len} bytes", + requested_range.start, + requested_range.end, + bytes.len() + ))); + } + Ok(bytes) +} + impl Eq for IoTask {} impl PartialEq for IoTask { @@ -415,6 +552,7 @@ impl IoTask { }) .await .map_err(Error::from) + .and_then(|bytes| validate_read_length(self.reader.path(), &self.to_read, bytes)) }; // Emit per-file I/O trace event only when tracing is enabled tracing::trace!( @@ -519,6 +657,84 @@ impl ScanStats { } } +fn split_priority(priority: Option) -> (Option, Option) { + priority + .map(|priority| ((priority >> 64) as u64, priority as u64)) + .unzip() +} + +#[derive(Debug, Clone, Copy)] +pub(super) struct SchedulerStateEvent { + pub(super) queue_kind: &'static str, + pub(super) io_capacity: u64, + pub(super) iops_available: u64, + pub(super) active_iops: u64, + pub(super) pending_iops: u64, + pub(super) pending_bytes: u64, + pub(super) bytes_available: i64, + pub(super) bytes_reserved: i64, + pub(super) io_buffer_size_bytes: u64, + pub(super) priorities_in_flight: u64, + pub(super) no_backpressure: bool, + pub(super) head_task_bytes: Option, + pub(super) head_task_priority_high: Option, + pub(super) head_task_priority_low: Option, + pub(super) min_in_flight_priority_high: Option, + pub(super) min_in_flight_priority_low: Option, + pub(super) head_task_can_deliver: Option, + pub(super) head_task_priority_bypass: Option, + pub(super) head_task_blocked_by_iops: Option, + pub(super) head_task_blocked_by_bytes: Option, +} + +impl SchedulerStateEvent { + fn trace(self, stats: ScanStats) { + tracing::event!( + target: SCHEDULER_STATE_EVENT_TARGET, + tracing::Level::TRACE, + queue_kind = self.queue_kind, + scheduler_iops = stats.iops, + scheduler_requests = stats.requests, + scheduler_bytes_read = stats.bytes_read, + io_capacity = self.io_capacity, + iops_available = self.iops_available, + active_iops = self.active_iops, + pending_iops = self.pending_iops, + pending_bytes = self.pending_bytes, + bytes_available = self.bytes_available, + bytes_reserved = self.bytes_reserved, + io_buffer_size_bytes = self.io_buffer_size_bytes, + priorities_in_flight = self.priorities_in_flight, + no_backpressure = self.no_backpressure, + head_task_bytes_present = self.head_task_bytes.is_some(), + head_task_bytes = self.head_task_bytes.unwrap_or_default(), + head_task_priority_high_present = self.head_task_priority_high.is_some(), + head_task_priority_high = self.head_task_priority_high.unwrap_or_default(), + head_task_priority_low_present = self.head_task_priority_low.is_some(), + head_task_priority_low = self.head_task_priority_low.unwrap_or_default(), + min_in_flight_priority_high_present = self.min_in_flight_priority_high.is_some(), + min_in_flight_priority_high = self.min_in_flight_priority_high.unwrap_or_default(), + min_in_flight_priority_low_present = self.min_in_flight_priority_low.is_some(), + min_in_flight_priority_low = self.min_in_flight_priority_low.unwrap_or_default(), + head_task_can_deliver_present = self.head_task_can_deliver.is_some(), + head_task_can_deliver = self.head_task_can_deliver.unwrap_or(false), + head_task_priority_bypass_present = self.head_task_priority_bypass.is_some(), + head_task_priority_bypass = self.head_task_priority_bypass.unwrap_or(false), + head_task_blocked_by_iops_present = self.head_task_blocked_by_iops.is_some(), + head_task_blocked_by_iops = self.head_task_blocked_by_iops.unwrap_or(false), + head_task_blocked_by_bytes_present = self.head_task_blocked_by_bytes.is_some(), + head_task_blocked_by_bytes = self.head_task_blocked_by_bytes.unwrap_or(false), + "Scheduler state" + ); + } +} + +pub(super) fn emit_scheduler_state_event(event: Option, stats: &IoStats) { + if let Some(event) = event { + event.trace(stats.snapshot()); + } +} + /// A shareable, cloneable handle to a set of cumulative I/O counters. /// /// All clones share the same underlying counters. This serves two purposes: @@ -598,12 +814,25 @@ impl Debug for ScanScheduler { } struct Response { - data: Result>, + // `Option` so the caller can take the data out while the response (and its + // backpressure refund on drop) stays intact. + data: Option>>, + io_queue: Arc, priority: u128, num_reqs: usize, num_bytes: u64, } +// Refund the batch's backpressure reservation when the response is dropped, be +// that on delivery or when a cancelled request's undelivered response is +// discarded. This releases the budget even if the caller drops the future early. +impl Drop for Response { + fn drop(&mut self) { + self.io_queue + .on_bytes_consumed(self.num_bytes, self.priority, self.num_reqs); + } +} + #[derive(Debug, Clone, Copy)] pub struct SchedulerConfig { /// the # of bytes that can be buffered but not yet requested. @@ -659,6 +888,7 @@ impl ScanScheduler { /// * config - configuration settings for the scheduler pub fn new(object_store: Arc, config: SchedulerConfig) -> Arc { let io_capacity = object_store.io_parallelism(); + let stats = IoStats::new(); let use_lite = config .use_lite_scheduler .unwrap_or_else(|| object_store.prefers_lite_scheduler()); @@ -666,12 +896,14 @@ impl ScanScheduler { let io_queue = Arc::new(lite::IoQueue::new( io_capacity as u64, config.io_buffer_size_bytes, + stats.clone(), )); IoQueueType::Lite(io_queue) } else { let io_queue = Arc::new(IoQueue::new( io_capacity as u32, config.io_buffer_size_bytes, + stats.clone(), )); let io_queue_clone = io_queue.clone(); // Best we can do here is fire and forget. If the I/O loop is still running when the scheduler is @@ -683,7 +915,7 @@ impl ScanScheduler { Arc::new(Self { object_store, io_queue, - stats: IoStats::new(), + stats, }) } @@ -782,6 +1014,7 @@ impl ScanScheduler { priority, request.len(), bypass_backpressure, + io_queue.clone(), )))); for (task_idx, iop) in request.into_iter().enumerate() { @@ -820,14 +1053,11 @@ impl ScanScheduler { self.do_submit_request(reader, request, tx, priority, io_queue, bypass_backpressure); - let io_queue_clone = io_queue.clone(); - - rx.map(move |wrapped_rsp| { - // Right now, it isn't possible for I/O to be cancelled so a cancel error should - // not occur - let rsp = wrapped_rsp.unwrap(); - io_queue_clone.on_bytes_consumed(rsp.num_bytes, rsp.priority, rsp.num_reqs); - rsp.data + rx.map(|wrapped_rsp| { + // A cancel error can't occur: the sender always sends before dropping. + // The reservation is refunded on `Response` drop, so just take the data. + let mut rsp = wrapped_rsp.unwrap(); + rsp.data.take().unwrap() }) } @@ -845,11 +1075,15 @@ impl ScanScheduler { .map(|task| { let reader = reader.clone(); let queue = io_queue.clone(); + let requested_range = task.clone(); let run_fn = Box::new(move || { - reader - .get_range(task.start as usize..task.end as usize) - .map_err(Error::from) - .boxed() + let bytes_fut = reader + .get_range(requested_range.start as usize..requested_range.end as usize); + async move { + let bytes = bytes_fut.await.map_err(Error::from)?; + validate_read_length(reader.path(), &requested_range, bytes) + } + .boxed() }); queue.submit(task, priority, run_fn, bypass_backpressure) }) @@ -957,6 +1191,8 @@ impl FileScheduler { /// Each request has a backpressure ID which controls which backpressure throttle /// is applied to the request. Requests made to the same backpressure throttle /// will be throttled together. + /// + /// Ranges must be sorted by their start offset. pub fn submit_request( &self, request: Vec>, @@ -965,6 +1201,19 @@ impl FileScheduler { // The final priority is a combination of the row offset and the file number let priority = ((self.base_priority as u128) << 64) + priority as u128; + if let Some((range_index, ranges)) = request + .windows(2) + .enumerate() + .find(|(_, ranges)| ranges[0].start > ranges[1].start) + { + return Either::Left(std::future::ready(Err(Error::invalid_input(format!( + "I/O request ranges must be sorted by start offset: range at index {range_index} is {:?}, but range at index {} is {:?}", + ranges[0], + range_index + 1, + ranges[1] + ))))); + } + let mut merged_requests = Vec::with_capacity(request.len()); if !request.is_empty() { @@ -1017,7 +1266,7 @@ impl FileScheduler { let mut updated_index = 0; let mut final_bytes = Vec::with_capacity(request.len()); - async move { + Either::Right(async move { let bytes_vec = bytes_vec_fut.await?; let mut orig_index = 0; @@ -1060,7 +1309,7 @@ impl FileScheduler { } Ok(final_bytes) - } + }) } pub fn with_priority(&self, priority: u64) -> Self { @@ -1134,6 +1383,7 @@ mod tests { use futures::poll; use lance_core::utils::tempfile::TempObjFile; use rand::RngCore; + use rstest::rstest; use object_store::{GetRange, ObjectStore as OSObjectStore, ObjectStoreExt, memory::InMemory}; use tokio::{runtime::Handle, time::timeout}; @@ -1159,6 +1409,47 @@ mod tests { } } + #[test] + fn test_scheduler_state_event_fields() { + use tracing_mock::{expect, subscriber}; + + let event = expect::event() + .with_target(SCHEDULER_STATE_EVENT_TARGET) + .at_level(tracing::Level::TRACE) + .with_fields( + expect::field("queue_kind") + .with_value(&"standard") + .and(expect::field("scheduler_iops").with_value(&7u64)) + .and(expect::field("scheduler_requests").with_value(&3u64)) + .and(expect::field("scheduler_bytes_read").with_value(&4096u64)) + .and(expect::field("io_capacity").with_value(&4u64)) + .and(expect::field("pending_iops").with_value(&1u64)) + .and(expect::field("bytes_available").with_value(&128i64)) + .and(expect::field("head_task_bytes_present").with_value(&true)) + .and(expect::field("head_task_bytes").with_value(&1u64)) + .and(expect::field("head_task_can_deliver_present").with_value(&true)) + .and(expect::field("head_task_can_deliver").with_value(&true)), + ); + let (subscriber, handle) = subscriber::mock().event(event).run_with_handle(); + + let stats = IoStats::new(); + stats.add_scan_stats(&ScanStats { + iops: 7, + requests: 3, + bytes_read: 4096, + }); + let mut state = IoQueueState::new(4, 192); + state.iops_avail = 2; + state.bytes_avail = 128; + state.pending_requests.push(make_task(1, false)); + + tracing::subscriber::with_default(subscriber, || { + emit_scheduler_state_event(state.scheduler_state_event(), &stats); + }); + + handle.assert_finished(); + } + #[test] fn test_iotask_ordering() { // Bypass tasks must come out of the heap before non-bypass tasks. @@ -1176,6 +1467,29 @@ mod tests { assert_eq!(order, vec![(5, true), (20, true), (1, false), (10, false)]); } + #[test] + fn test_batch_with_undelivered_slot_is_error() { + let response = Arc::new(Mutex::new(None)); + let response_clone = response.clone(); + let io_queue = Arc::new(IoQueue::new(1, 1024, IoStats::new())); + let batch = MutableBatch::new( + move |rsp| *response_clone.lock().unwrap() = Some(rsp), + 2, // num_data_buffers + 0, // priority + 2, // num_reqs + false, + io_queue, + ); + drop(batch); + + let mut rsp = response.lock().unwrap().take().unwrap(); + let data = rsp.data.take().unwrap(); + assert!( + data.is_err(), + "undelivered slot must yield an error, got {data:?}", + ); + } + #[tokio::test] async fn test_full_seq_read() { let tmp_file = TempObjFile::default(); @@ -1248,6 +1562,75 @@ mod tests { assert_eq!(bytes[0], some_data); } + #[derive(Debug)] + struct ShortReader { + path: Path, + } + + impl lance_core::deepsize::DeepSizeOf for ShortReader { + fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { + 0 + } + } + + impl Reader for ShortReader { + fn path(&self) -> &Path { + &self.path + } + + fn block_size(&self) -> usize { + 4096 + } + + fn io_parallelism(&self) -> usize { + 1 + } + + fn size(&self) -> futures::future::BoxFuture<'_, object_store::Result> { + Box::pin(async { Ok(0) }) + } + + fn get_range( + &self, + _range: Range, + ) -> futures::future::BoxFuture<'static, object_store::Result> { + Box::pin(async { Ok(Bytes::new()) }) + } + + fn get_all(&self) -> futures::future::BoxFuture<'_, object_store::Result> { + Box::pin(async { Ok(Bytes::new()) }) + } + } + + #[rstest] + #[case::standard(false)] + #[case::lite(true)] + #[tokio::test] + async fn test_short_read_returns_io_error(#[case] use_lite_scheduler: bool) { + let config = SchedulerConfig { + use_lite_scheduler: Some(use_lite_scheduler), + ..SchedulerConfig::default_for_testing() + }; + let scheduler = ScanScheduler::new(Arc::new(ObjectStore::memory()), config); + let reader = Arc::new(ShortReader { + path: Path::parse("short-file").unwrap(), + }); + let file_scheduler = scheduler.open_reader(reader); + + let error = file_scheduler + .submit_request(vec![0..8], 0) + .await + .unwrap_err(); + + assert!(matches!(error, Error::IO { .. }), "{error:?}"); + assert!( + error.to_string().contains( + "I/O request for file short-file and range 0..8 returned 0 bytes, expected 8 bytes" + ), + "{error}" + ); + } + #[tokio::test] async fn test_split_coalesce() { let tmp_file = TempObjFile::default(); @@ -1332,6 +1715,38 @@ mod tests { assert_eq!(11, scheduler.stats().iops); } + #[rstest] + #[case::standard(false)] + #[case::lite(true)] + #[tokio::test] + async fn test_unordered_ranges_are_rejected(#[case] use_lite_scheduler: bool) { + let path = Path::parse("unordered-ranges").unwrap(); + let source = (0_u8..64).collect::>(); + let object_store = Arc::new(ObjectStore::memory()); + object_store.put(&path, &source).await.unwrap(); + + let config = SchedulerConfig { + use_lite_scheduler: Some(use_lite_scheduler), + ..SchedulerConfig::default_for_testing() + }; + let scheduler = ScanScheduler::new(object_store, config); + let file_scheduler = scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + + let ranges = vec![9..26, 0..49]; + let error = file_scheduler.submit_request(ranges, 0).await.unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. }), "{error:?}"); + assert!( + error.to_string().contains( + "I/O request ranges must be sorted by start offset: range at index 0 is 9..26, but range at index 1 is 0..49" + ), + "{error}" + ); + } + #[tokio::test] async fn test_io_stats_sink() { let tmp_file = TempObjFile::default(); @@ -1474,6 +1889,136 @@ mod tests { assert!(second_fut.await.unwrap().unwrap().len() == 20); } + #[tokio::test] + async fn test_standard_scheduler_state_tracks_queue_state() { + let some_path = Path::parse("foo").unwrap(); + let base_store = Arc::new(InMemory::new()); + base_store + .put(&some_path, vec![0; 1000].into()) + .await + .unwrap(); + + let semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + let mut obj_store = MockObjectStore::default(); + let semaphore_copy = semaphore.clone(); + obj_store + .expect_get_opts() + .returning(move |location, options| { + let semaphore = semaphore.clone(); + let base_store = base_store.clone(); + let location = location.clone(); + async move { + semaphore.acquire().await.unwrap().forget(); + base_store.get_opts(&location, options).await + } + .boxed() + }); + let obj_store = Arc::new(ObjectStore::new( + Arc::new(obj_store), + Url::parse("mem://").unwrap(), + Some(500), + None, + false, + false, + 1, + DEFAULT_DOWNLOAD_RETRY_COUNT, + None, + )); + + let scheduler = ScanScheduler::new( + obj_store, + SchedulerConfig { + io_buffer_size_bytes: 1024 * 1024, + use_lite_scheduler: Some(false), + }, + ); + let file_scheduler = scheduler + .open_file(&Path::parse("foo").unwrap(), &CachedFileSize::new(1000)) + .await + .unwrap(); + + let first_fut = timeout( + Duration::from_secs(10), + file_scheduler.submit_single(0..10, 0), + ) + .boxed(); + let second_fut = timeout( + Duration::from_secs(10), + file_scheduler.submit_single(0..20, 100), + ) + .boxed(); + let third_fut = timeout( + Duration::from_secs(10), + file_scheduler.submit_single(0..30, 0), + ) + .boxed(); + + let io_queue = match &scheduler.io_queue { + IoQueueType::Standard(io_queue) => io_queue.clone(), + IoQueueType::Lite(_) => unreachable!("test forces the standard scheduler"), + }; + let ( + io_capacity, + iops_available, + pending_bytes, + bytes_reserved, + priorities_in_flight, + head_task_bytes, + head_task_blocked_by_iops, + head_task_blocked_by_bytes, + ) = timeout(Duration::from_secs(5), async { + loop { + let observed = { + let state = io_queue.state.lock().unwrap(); + let active_iops = state.io_capacity.saturating_sub(state.iops_avail); + if active_iops == 1 && state.pending_requests.len() == 2 { + let pending_bytes = state + .pending_requests + .iter() + .map(IoTask::num_bytes) + .sum::(); + let head_task = state.pending_requests.peek().unwrap(); + let bypasses_bytes = state.no_backpressure + || head_task.bypass_backpressure + || head_task.priority <= state.priorities_in_flight.min_in_flight(); + Some(( + state.io_capacity, + state.iops_avail, + pending_bytes, + state.io_buffer_size as i64 - state.bytes_avail, + state.priorities_in_flight.len(), + head_task.num_bytes(), + state.iops_avail == 0, + !bypasses_bytes && head_task.num_bytes() as i64 > state.bytes_avail, + )) + } else { + None + } + }; + if let Some(observed) = observed { + break observed; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + assert_eq!(io_capacity, 1); + assert_eq!(iops_available, 0); + assert_eq!(pending_bytes, 50); + assert_eq!(bytes_reserved, 10); + assert_eq!(priorities_in_flight, 1); + assert_eq!(head_task_bytes, 30); + assert!(head_task_blocked_by_iops); + assert!(!head_task_blocked_by_bytes); + + semaphore_copy.add_permits(3); + assert_eq!(first_fut.await.unwrap().unwrap().len(), 10); + assert_eq!(third_fut.await.unwrap().unwrap().len(), 30); + assert_eq!(second_fut.await.unwrap().unwrap().len(), 20); + } + #[tokio::test(flavor = "multi_thread")] async fn test_backpressure() { let some_path = Path::parse("foo").unwrap(); @@ -1609,6 +2154,7 @@ mod tests { #[derive(Debug)] struct BlockingReader { semaphore: Arc, + get_range_count: Arc, path: Path, } @@ -1639,6 +2185,7 @@ mod tests { &self, range: Range, ) -> futures::future::BoxFuture<'static, object_store::Result> { + self.get_range_count.fetch_add(1, Ordering::Release); let semaphore = self.semaphore.clone(); let num_bytes = range.end - range.start; Box::pin(async move { @@ -1675,6 +2222,7 @@ mod tests { let semaphore = Arc::new(tokio::sync::Semaphore::new(0)); let reader: Arc = Arc::new(BlockingReader { semaphore: semaphore.clone(), + get_range_count: Arc::new(AtomicU64::new(0)), path: Path::parse("test").unwrap(), }); @@ -1981,4 +2529,105 @@ mod tests { .unwrap(); assert_eq!(bytes_dispatched.load(Ordering::Acquire), 30); } + + // Against a 100-byte budget: submit fut1 (50 bytes, priority 0), drop it while + // its read is still blocked in get_range, then submit fut2 (60 bytes, priority 1). + // fut2's priority can't win the priority-bypass, so it needs 60 of the budget -- + // available only if fut1's dropped reservation was refunded. Returns whether fut2 + // completed within 2s (false = the reservation leaked and fut2 deadlocked). + async fn run_caller_drop_scenario(use_lite_scheduler: bool) -> (bool, Duration) { + let obj_store = Arc::new(ObjectStore::new( + Arc::new(InMemory::new()), + Url::parse("mem://").unwrap(), + Some(4096), + None, + false, + false, + 1, + DEFAULT_DOWNLOAD_RETRY_COUNT, + None, + )); + let scheduler = ScanScheduler::new( + obj_store, + SchedulerConfig { + io_buffer_size_bytes: 100, + use_lite_scheduler: Some(use_lite_scheduler), + }, + ); + + let semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + let get_range_count = Arc::new(AtomicU64::new(0)); + let reader: Arc = Arc::new(BlockingReader { + semaphore: semaphore.clone(), + get_range_count: get_range_count.clone(), + path: Path::parse("test").unwrap(), + }); + + // Step 1: reserve 50 of the 100 budget bytes with a read we never consume. + // Spawn it so we can cancel the caller-side future while it is still parked + // waiting for the (blocked) read to finish. + let fut1 = scheduler.submit_request(reader.clone(), vec![0..50], 0, false); + let handle = tokio::spawn(async move { + let _ = fut1.await; + }); + + // Wait until the read is genuinely in flight (blocked on the semaphore). + // This guarantees the 50-byte reservation has been taken before we drop + // the caller, closing the race between the I/O loop and the abort. + while get_range_count.load(Ordering::Acquire) == 0 { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + // Step 2: drop the caller-side future while its `rx` is still pending. + handle.abort(); + let _ = handle.await; + + // Step 3: let the in-flight read finish. The reservation should be refunded + // now that the request is done, whether or not the caller is still around. + semaphore.add_permits(1); + // Give the read time to run to completion so the refund would already have + // happened. + tokio::time::sleep(Duration::from_millis(50)).await; + + // Step 4: submit the follow-up. Add a permit up front so that, if it *is* + // admitted, its own read can complete rather than block on the semaphore. + semaphore.add_permits(1); + let fut2 = scheduler.submit_request(reader, vec![100..160], 1, false); + + let start = std::time::Instant::now(); + let outcome = timeout(Duration::from_secs(2), fut2).await; + let elapsed = start.elapsed(); + match outcome { + Ok(res) => { + assert_eq!(res.unwrap().iter().map(|b| b.len()).sum::(), 60); + (true, elapsed) + } + Err(_) => (false, elapsed), + } + } + + /// Dropping a standard-scheduler request future while its read is in flight must + /// still refund the backpressure reservation, so a later request that needs the + /// budget does not deadlock. + #[tokio::test(flavor = "multi_thread")] + async fn standard_scheduler_refunds_reservation_on_caller_drop() { + let (completed, elapsed) = run_caller_drop_scenario(false).await; + assert!( + completed, + "standard scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \ + the dropped request's reservation was not refunded" + ); + } + + /// Same guarantee for the lite scheduler: dropping a request future mid-read + /// releases its reservation via the `TaskHandle` drop path. + #[tokio::test(flavor = "multi_thread")] + async fn lite_scheduler_refunds_reservation_on_caller_drop() { + let (completed, elapsed) = run_caller_drop_scenario(true).await; + assert!( + completed, + "lite scheduler deadlocked the follow-up request (elapsed {elapsed:?}); \ + the dropped request's reservation was not refunded" + ); + } } diff --git a/rust/lance-io/src/scheduler/lite.rs b/rust/lance-io/src/scheduler/lite.rs index fc666139a05..b7a06e11bb7 100644 --- a/rust/lance-io/src/scheduler/lite.rs +++ b/rust/lance-io/src/scheduler/lite.rs @@ -33,7 +33,10 @@ use std::{ use bytes::Bytes; use lance_core::{Error, Result}; -use super::{BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN}; +use super::{ + BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN, IoStats, SCHEDULER_STATE_EVENT_TARGET, + SchedulerStateEvent, emit_scheduler_state_event, +}; type RunFn = Box Pin> + Send>> + Send>; @@ -67,6 +70,26 @@ enum TaskState { }, } +impl TaskState { + fn backpressure_reservation(&self) -> Option { + match self { + Self::Reserved { + backpressure_reservation, + .. + } + | Self::Running { + backpressure_reservation, + .. + } + | Self::Finished { + backpressure_reservation, + .. + } => Some(*backpressure_reservation), + Self::Initial { .. } | Self::Broken => None, + } + } +} + /// A custom error type that might have a backpressure reservation /// /// This is used instead of Lance's standard error type so we can ensure @@ -85,25 +108,14 @@ impl BrokenTaskError { // This will capture any backpressure reservation the task has and put it into the // error so we make sure to release it when returning the error. fn new(task_state: TaskState, message: String) -> Self { - match task_state { - TaskState::Reserved { - backpressure_reservation, - .. - } - | TaskState::Running { - backpressure_reservation, - .. - } - | TaskState::Finished { - backpressure_reservation, - .. - } => Self { + match task_state.backpressure_reservation() { + None => Self { message, - backpressure_reservation: Some(backpressure_reservation), + backpressure_reservation: None, }, - TaskState::Broken | TaskState::Initial { .. } => Self { + Some(reservation) => Self { message, - backpressure_reservation: None, + backpressure_reservation: Some(reservation), }, } } @@ -237,6 +249,7 @@ trait BackpressureThrottle: Send { /// Unconditionally acquire a zero-cost reservation, tracking only the priority. /// Used for bypass tasks that must never be blocked by backpressure. fn force_acquire(&mut self, priority: u128) -> BackpressureReservation; + fn state(&self) -> BackpressureState; } // We want to allow requests that have a lower priority than any @@ -279,9 +292,22 @@ impl PrioritiesInFlight { self.in_flight.remove(pos); } } + + fn len(&self) -> usize { + self.in_flight.len() + } +} + +#[derive(Debug, Clone, Copy)] +struct BackpressureState { + max_bytes: u64, + bytes_available: i64, + priorities_in_flight: u64, + no_backpressure: bool, } struct SimpleBackpressureThrottle { + max_bytes: u64, start: Instant, last_warn: AtomicU64, bytes_available: i64, @@ -297,6 +323,7 @@ impl SimpleBackpressureThrottle { panic!("Max bytes must be less than {}", i64::MAX); } Self { + max_bytes, start: Instant::now(), last_warn: AtomicU64::new(0), bytes_available: max_bytes as i64, @@ -358,6 +385,15 @@ impl BackpressureThrottle for SimpleBackpressureThrottle { priority, } } + + fn state(&self) -> BackpressureState { + BackpressureState { + max_bytes: self.max_bytes, + bytes_available: self.bytes_available, + priorities_in_flight: self.priorities_in_flight.len() as u64, + no_backpressure: self.no_backpressure, + } + } } struct TaskEntry { @@ -427,6 +463,48 @@ impl IoQueueState { Ok(()) } } + + fn scheduler_state_event(&self) -> Option { + if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) { + return None; + } + + let backpressure = self.backpressure_throttle.state(); + let pending_bytes = self + .pending_tasks + .iter() + .filter_map(|entry| self.tasks.get(&entry.task_id)) + .map(|task| task.num_bytes) + .sum::(); + let active_iops = self + .tasks + .values() + .filter(|task| matches!(task.state, TaskState::Running { .. })) + .count() as u64; + + Some(SchedulerStateEvent { + queue_kind: "lite", + io_capacity: 0, + iops_available: 0, + active_iops, + pending_iops: self.pending_tasks.len() as u64, + pending_bytes, + bytes_available: backpressure.bytes_available, + bytes_reserved: backpressure.max_bytes as i64 - backpressure.bytes_available, + io_buffer_size_bytes: backpressure.max_bytes, + priorities_in_flight: backpressure.priorities_in_flight, + no_backpressure: backpressure.no_backpressure, + head_task_bytes: None, + head_task_priority_high: None, + head_task_priority_low: None, + min_in_flight_priority_high: None, + min_in_flight_priority_low: None, + head_task_can_deliver: None, + head_task_priority_bypass: None, + head_task_blocked_by_iops: None, + head_task_blocked_by_bytes: None, + }) + } } /// A queue of I/O tasks to be shared between the I/O scheduler and the I/O decoder. @@ -449,12 +527,14 @@ impl IoQueueState { /// day as well) pub(super) struct IoQueue { state: Arc>, + stats: IoStats, } impl IoQueue { - pub fn new(max_concurrency: u64, max_bytes: u64) -> Self { + pub fn new(max_concurrency: u64, max_bytes: u64, stats: IoStats) -> Self { Self { state: Arc::new(Mutex::new(IoQueueState::new(max_concurrency, max_bytes))), + stats, } } @@ -471,6 +551,9 @@ impl IoQueue { state.handle_result(task.reserve(reservation))?; state.handle_result(task.start())?; state.tasks.insert(task_id, task); + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); return Ok(()); } @@ -480,6 +563,9 @@ impl IoQueue { reserved: task.is_reserved(), }); state.tasks.insert(task_id, task); + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); Ok(()) } @@ -518,34 +604,42 @@ impl IoQueue { // When a task completes we should check to see if any other tasks are now runnable fn on_task_complete(&self, mut state: MutexGuard) -> Result<()> { - let state_ref = &mut *state; - let mut task_result = TaskResult::Ok(()); - while !state_ref.pending_tasks.is_empty() { - // Unwrap safe here since we just checked the queue is not empty - let next_task = state_ref.pending_tasks.peek().unwrap(); - let Some(task) = state_ref.tasks.get_mut(&next_task.task_id) else { - log::warn!("Task with id {} was lost", next_task.task_id); - continue; - }; - if !task.is_reserved() { - let Some(reservation) = state_ref - .backpressure_throttle - .try_acquire(task.num_bytes, task.priority) - else { - break; + let result = { + let state_ref = &mut *state; + let mut task_result = TaskResult::Ok(()); + while !state_ref.pending_tasks.is_empty() { + // Unwrap safe here since we just checked the queue is not empty + let task_id = state_ref.pending_tasks.peek().unwrap().task_id; + let Some(task) = state_ref.tasks.get_mut(&task_id) else { + // The caller dropped this task's handle (see `abandon`); discard the + // stale queue entry instead of spinning on it. + state_ref.pending_tasks.pop(); + continue; }; - if let Err(e) = task.reserve(reservation) { + if !task.is_reserved() { + let Some(reservation) = state_ref + .backpressure_throttle + .try_acquire(task.num_bytes, task.priority) + else { + break; + }; + if let Err(e) = task.reserve(reservation) { + task_result = Err(e); + break; + } + } + state_ref.pending_tasks.pop(); + if let Err(e) = task.start() { task_result = Err(e); break; } } - state_ref.pending_tasks.pop(); - if let Err(e) = task.start() { - task_result = Err(e); - break; - } - } - state_ref.handle_result(task_result) + state_ref.handle_result(task_result) + }; + let event = state.scheduler_state_event(); + drop(state); + emit_scheduler_state_event(event, &self.stats); + result } fn poll(&self, task_id: u64, cx: &mut Context<'_>) -> Poll> { @@ -573,10 +667,34 @@ impl IoQueue { } pub(super) fn close(&self) { + let event = { + let mut state = self.state.lock().unwrap(); + for task in std::mem::take(&mut state.tasks).values_mut() { + task.cancel(); + } + state.scheduler_state_event() + }; + emit_scheduler_state_event(event, &self.stats); + } + + // Called when a caller drops a task's handle before the task finishes. Removes + // the task and returns any backpressure reservation it holds to the budget, then + // re-checks the queue so newly-affordable tasks can start. Unlike the standard + // release path (`poll`), this runs without the task being polled to completion, + // so a cancelled read does not leak its reservation. + fn abandon(&self, task_id: u64) { let mut state = self.state.lock().unwrap(); - for task in std::mem::take(&mut state.tasks).values_mut() { - task.cancel(); + let Some(task) = state.tasks.remove(&task_id) else { + // Already consumed by `poll`; nothing to release. + return; + }; + + if let Some(reservation) = task.state.backpressure_reservation() { + state.backpressure_throttle.release(reservation); } + // Freed budget may make queued tasks runnable; there is no caller to surface + // an error to here. + let _ = self.on_task_complete(state); } } @@ -592,6 +710,12 @@ impl Future for TaskHandle { } } +impl Drop for TaskHandle { + fn drop(&mut self) { + self.queue.abandon(self.task_id); + } +} + #[cfg(test)] mod tests { use super::*; @@ -600,7 +724,7 @@ mod tests { #[tokio::test] async fn test_priority_ordering() { // Backpressure budget of 10 bytes: only one 10-byte task runs at a time. - let queue = Arc::new(IoQueue::new(128, 10)); + let queue = Arc::new(IoQueue::new(128, 10, IoStats::default())); // Records the priority of each task when its run_fn is invoked (i.e. when // the task transitions to Running). @@ -708,7 +832,7 @@ mod tests { async fn test_zero_buffer_bypasses_backpressure() { // Budget = 0 sets no_backpressure = true, so all tasks start immediately // regardless of how many bytes are "outstanding". - let queue = Arc::new(IoQueue::new(128, 0)); + let queue = Arc::new(IoQueue::new(128, 0, IoStats::default())); let start_order: Arc>> = Arc::new(Mutex::new(Vec::new())); let make_run_fn = @@ -750,7 +874,7 @@ mod tests { async fn test_bypass_flag_proceeds_past_exhausted_budget() { // Budget of 10 bytes. A blocker task fills it. A task with bypass=true starts // immediately despite the exhausted budget; a normal task stays queued. - let queue = Arc::new(IoQueue::new(128, 10)); + let queue = Arc::new(IoQueue::new(128, 10, IoStats::default())); let start_order: Arc>> = Arc::new(Mutex::new(Vec::new())); let make_run_fn = diff --git a/rust/lance-io/src/uring/current_thread.rs b/rust/lance-io/src/uring/current_thread.rs index abac772218b..ae2e3414ee9 100644 --- a/rust/lance-io/src/uring/current_thread.rs +++ b/rust/lance-io/src/uring/current_thread.rs @@ -14,8 +14,8 @@ use crate::traits::Reader; use crate::uring::DEFAULT_URING_QUEUE_DEPTH; use crate::utils::tracking_store::IOTracker; use bytes::{Bytes, BytesMut}; +use futures::FutureExt; use futures::future::BoxFuture; -use futures::{FutureExt, TryFutureExt}; use io_uring::{IoUring, opcode, types}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; @@ -396,10 +396,14 @@ impl Reader for UringCurrentThreadReader { let num_bytes = range.len() as u64; let range_u64 = (range.start as u64)..(range.end as u64); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(range.start as u64, range.len()) - .map_ok(move |bytes| { - io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); - bytes + .map(move |result| { + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); + } + result }) .boxed() } @@ -411,10 +415,15 @@ impl Reader for UringCurrentThreadReader { let io_tracker = self.io_tracker.clone(); let path = self.handle.path.clone(); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(0, size) - .map_ok(move |bytes| { - io_tracker.record_read("get_all", path, bytes.len() as u64, None); - bytes + .map(move |result| { + let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64); + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_all", path, num_bytes, None); + } + result }) .boxed() } diff --git a/rust/lance-io/src/uring/reader.rs b/rust/lance-io/src/uring/reader.rs index a948e6c63dc..88b784165f9 100644 --- a/rust/lance-io/src/uring/reader.rs +++ b/rust/lance-io/src/uring/reader.rs @@ -5,15 +5,15 @@ use super::future::UringReadFuture; use super::requests::IoRequest; -use super::thread::{SUBMITTED_COUNTER, THREAD_SELECTOR, URING_THREADS}; +use super::thread::{QueuedRequest, THREAD_SELECTOR, URING_THREADS}; use super::{DEFAULT_URING_BLOCK_SIZE, DEFAULT_URING_IO_PARALLELISM, URING_BLOCK_SIZE}; use crate::local::to_local_path; use crate::traits::Reader; use crate::uring::requests::RequestState; use crate::utils::tracking_store::IOTracker; use bytes::{Bytes, BytesMut}; +use futures::FutureExt; use futures::future::BoxFuture; -use futures::{FutureExt, TryFutureExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; use object_store::path::Path; @@ -205,35 +205,54 @@ impl UringReader { }), }); - // Increment submitted counter before sending to channel - SUBMITTED_COUNTER.fetch_add(1, Ordering::Relaxed); + if URING_THREADS.threads.is_empty() { + let initialization_errors = if URING_THREADS.initialization_errors.is_empty() { + "LANCE_URING_THREAD_COUNT is 0".to_owned() + } else { + URING_THREADS.initialization_errors.join("; ") + }; + return Box::pin(async move { + Err(object_store::Error::Generic { + store: "UringReader", + source: Box::new(io::Error::other(format!( + "no io_uring worker threads are available: {initialization_errors}" + ))), + }) + }); + } // Select thread in round-robin fashion - let thread_idx = - (THREAD_SELECTOR.fetch_add(1, Ordering::Relaxed) as usize) % URING_THREADS.len(); + let thread_idx = (THREAD_SELECTOR.fetch_add(1, Ordering::Relaxed) as usize) + % URING_THREADS.threads.len(); + let thread = &URING_THREADS.threads[thread_idx]; + + if !thread.is_alive.load(Ordering::Acquire) { + return Box::pin(async move { + Err(object_store::Error::Generic { + store: "UringReader", + source: Box::new(io::Error::new( + io::ErrorKind::BrokenPipe, + "io_uring thread died", + )), + }) + }); + } // Send to selected thread via channel - match URING_THREADS[thread_idx] + match thread .request_tx - .send(Arc::clone(&request)) + .send(QueuedRequest::new(Arc::clone(&request))) { - Ok(()) => { - // Return future that will be woken when operation completes - Box::pin(UringReadFuture { request }) - } - Err(_) => { - // Thread died - decrement counter and return error future - SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed); - Box::pin(async move { - Err(object_store::Error::Generic { - store: "UringReader", - source: Box::new(io::Error::new( - io::ErrorKind::BrokenPipe, - "io_uring thread died", - )), - }) + Ok(()) => Box::pin(UringReadFuture { request }), + Err(_) => Box::pin(async move { + Err(object_store::Error::Generic { + store: "UringReader", + source: Box::new(io::Error::new( + io::ErrorKind::BrokenPipe, + "io_uring thread died", + )), }) - } + }), } } } @@ -267,10 +286,14 @@ impl Reader for UringReader { let num_bytes = range.len() as u64; let range_u64 = (range.start as u64)..(range.end as u64); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(range.start as u64, range.len()) - .map_ok(move |bytes| { - io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); - bytes + .map(move |result| { + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_range", path, num_bytes, Some(range_u64)); + } + result }) .boxed() } @@ -282,10 +305,15 @@ impl Reader for UringReader { let io_tracker = self.io_tracker.clone(); let path = self.handle.path.clone(); + let metrics = self.io_tracker.begin_io("get"); self.submit_read(0, size) - .map_ok(move |bytes| { - io_tracker.record_read("get_all", path, bytes.len() as u64, None); - bytes + .map(move |result| { + let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64); + metrics.record(&result, num_bytes); + if result.is_ok() { + io_tracker.record_read("get_all", path, num_bytes, None); + } + result }) .boxed() } diff --git a/rust/lance-io/src/uring/requests.rs b/rust/lance-io/src/uring/requests.rs index fa257507dc1..d47c18d631b 100644 --- a/rust/lance-io/src/uring/requests.rs +++ b/rust/lance-io/src/uring/requests.rs @@ -44,6 +44,9 @@ impl IoRequest { /// Used when a request cannot be submitted (e.g. SQ full). pub(super) fn fail(&self, err: io::Error) { let mut state = self.state.lock().unwrap(); + if state.completed { + return; + } state.err = Some(err); state.completed = true; if let Some(waker) = state.waker.take() { diff --git a/rust/lance-io/src/uring/tests.rs b/rust/lance-io/src/uring/tests.rs index 19d931da629..e1d83bba214 100644 --- a/rust/lance-io/src/uring/tests.rs +++ b/rust/lance-io/src/uring/tests.rs @@ -9,6 +9,14 @@ use std::io::Write; use std::time::Duration; use tempfile::NamedTempFile; +macro_rules! skip_if_no_uring_workers { + () => { + if super::thread::URING_THREADS.threads.is_empty() { + return Ok(()); + } + }; +} + /// Helper to create a temporary file with test data fn create_test_file(size: usize) -> Result<(NamedTempFile, Vec)> { let mut file = NamedTempFile::new()?; @@ -20,6 +28,7 @@ fn create_test_file(size: usize) -> Result<(NamedTempFile, Vec)> { #[tokio::test] async fn test_read_small_file() -> Result<()> { + skip_if_no_uring_workers!(); let (file, expected_data) = create_test_file(1024)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -36,6 +45,7 @@ async fn test_read_small_file() -> Result<()> { #[tokio::test] async fn test_read_range() -> Result<()> { + skip_if_no_uring_workers!(); let (file, expected_data) = create_test_file(4096)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -53,6 +63,7 @@ async fn test_read_range() -> Result<()> { #[tokio::test] async fn test_read_multiple_ranges() -> Result<()> { + skip_if_no_uring_workers!(); let (file, expected_data) = create_test_file(8192)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -72,6 +83,7 @@ async fn test_read_multiple_ranges() -> Result<()> { #[tokio::test] async fn test_file_size() -> Result<()> { + skip_if_no_uring_workers!(); let size = 5000; let (file, _) = create_test_file(size)?; let file_path = file.path().to_str().unwrap(); @@ -87,6 +99,7 @@ async fn test_file_size() -> Result<()> { #[tokio::test] async fn test_concurrent_reads() -> Result<()> { + skip_if_no_uring_workers!(); let (file, expected_data) = create_test_file(16384)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -115,6 +128,7 @@ async fn test_concurrent_reads() -> Result<()> { #[tokio::test] async fn test_large_file_read() -> Result<()> { + skip_if_no_uring_workers!(); // Test with a larger file (1MB) let size = 1024 * 1024; let (file, expected_data) = create_test_file(size)?; @@ -134,6 +148,7 @@ async fn test_large_file_read() -> Result<()> { #[tokio::test] async fn test_read_edge_cases() -> Result<()> { + skip_if_no_uring_workers!(); let (file, expected_data) = create_test_file(4096)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -157,17 +172,21 @@ async fn test_read_edge_cases() -> Result<()> { } #[tokio::test] -async fn test_file_not_found() { +async fn test_file_not_found() -> Result<()> { + skip_if_no_uring_workers!(); let uri = "file+uring:///nonexistent/file.dat"; let (store, path) = ObjectStore::from_uri(uri).await.unwrap(); // Should fail to open non-existent file let result = store.open(&path).await; assert!(result.is_err()); + + Ok(()) } #[tokio::test] async fn test_block_size_and_parallelism() -> Result<()> { + skip_if_no_uring_workers!(); let (file, _) = create_test_file(1024)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -184,6 +203,7 @@ async fn test_block_size_and_parallelism() -> Result<()> { #[tokio::test] async fn test_path() -> Result<()> { + skip_if_no_uring_workers!(); let (file, _) = create_test_file(1024)?; let file_path = file.path().to_str().unwrap(); let uri = format!("file+uring://{}", file_path); @@ -203,6 +223,7 @@ async fn test_path() -> Result<()> { /// than the actual file, causing io_uring to hit EOF before the full read completes. #[tokio::test] async fn test_short_read_get_all() -> Result<()> { + skip_if_no_uring_workers!(); let actual_size: usize = 8192; let (file, _expected_data) = create_test_file(actual_size)?; let file_path = file.path().to_str().unwrap(); @@ -225,6 +246,7 @@ async fn test_short_read_get_all() -> Result<()> { /// Test that a range read extending past EOF returns an error. #[tokio::test] async fn test_short_read_get_range_past_eof() -> Result<()> { + skip_if_no_uring_workers!(); let actual_size: usize = 8192; let (file, _expected_data) = create_test_file(actual_size)?; let file_path = file.path().to_str().unwrap(); @@ -257,6 +279,7 @@ async fn test_short_read_get_range_past_eof() -> Result<()> { /// future hangs and the timeout fires. #[tokio::test] async fn test_retry_sq_full_thread() -> Result<()> { + skip_if_no_uring_workers!(); use super::future::UringReadFuture; use super::requests::{IoRequest, RequestState}; use super::thread::push_to_sq; @@ -321,6 +344,7 @@ async fn test_retry_sq_full_thread() -> Result<()> { /// has already completed the request with an error. #[tokio::test(flavor = "current_thread")] async fn test_retry_sq_full_current_thread() -> Result<()> { + skip_if_no_uring_workers!(); use super::current_thread_future::UringCurrentThreadFuture; use super::requests::{IoRequest, RequestState}; use super::thread::push_to_sq; diff --git a/rust/lance-io/src/uring/thread.rs b/rust/lance-io/src/uring/thread.rs index d2ef197947d..290310477ea 100644 --- a/rust/lance-io/src/uring/thread.rs +++ b/rust/lance-io/src/uring/thread.rs @@ -12,7 +12,7 @@ use super::requests::IoRequest; use io_uring::{IoUring, opcode, types}; use std::collections::HashMap; use std::io; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, sync_channel}; use std::sync::{Arc, LazyLock}; use std::time::{Duration, Instant}; @@ -21,38 +21,106 @@ use std::time::{Duration, Instant}; /// /// This provides a channel sender for submitting read requests to the thread. pub(super) struct UringThreadHandle { - pub request_tx: SyncSender>, + pub request_tx: SyncSender, + pub is_alive: Arc, +} + +/// Owns the obligation to fail a request until a worker accepts it. +/// +/// Dropping the receiver also drops every queued item. Keeping the failure +/// obligation with the queued item guarantees that a request accepted during +/// worker shutdown cannot be abandoned without waking its future. +pub(super) struct QueuedRequest { + request: Option>, +} + +impl QueuedRequest { + pub(super) fn new(request: Arc) -> Self { + SUBMITTED_COUNTER.fetch_add(1, Ordering::Relaxed); + Self { + request: Some(request), + } + } + + fn into_request(mut self) -> Arc { + SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed); + self.request.take().unwrap() + } + + fn fail(mut self, error: io::Error) { + SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed); + self.request.take().unwrap().fail(error); + } +} + +impl Drop for QueuedRequest { + fn drop(&mut self) { + if let Some(request) = self.request.take() { + SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed); + request.fail(io::Error::new( + io::ErrorKind::BrokenPipe, + "io_uring worker stopped before accepting request", + )); + } + } +} + +pub(super) struct UringThreadPool { + pub threads: Vec, + pub initialization_errors: Vec, } /// Lazy-initialized io_uring thread pool. /// /// Multiple threads are spawned on first access and run until process exit. -pub(super) static URING_THREADS: LazyLock> = LazyLock::new(|| { +pub(super) static URING_THREADS: LazyLock = LazyLock::new(|| { let queue_depth = get_queue_depth(); let thread_count = get_thread_count(); let mut threads = Vec::with_capacity(thread_count); + let mut initialization_errors = Vec::new(); for i in 0..thread_count { - let (tx, rx) = sync_channel(queue_depth); - - std::thread::Builder::new() - .name(format!("lance-uring-{}", i)) - .spawn(move || run_uring_thread(rx, queue_depth, i)) - .expect("Failed to spawn io_uring thread"); - - threads.push(UringThreadHandle { request_tx: tx }); + match start_uring_thread(queue_depth, i) { + Ok(thread) => threads.push(thread), + Err(error) => { + let message = format!("thread {i}: {error}"); + log::error!("Failed to start io_uring {message}"); + initialization_errors.push(message); + } + } } log::info!( "io_uring thread pool spawned ({} threads, queue_depth={})", - thread_count, + threads.len(), queue_depth ); - threads + UringThreadPool { + threads, + initialization_errors, + } }); +fn start_uring_thread(queue_depth: usize, thread_id: usize) -> io::Result { + // Initialize the ring before publishing its sender so a request can never be + // accepted by a worker that subsequently fails during startup. + let ring = IoUring::builder().build(queue_depth as u32)?; + let (request_tx, request_rx) = sync_channel(queue_depth); + let is_alive = Arc::new(AtomicBool::new(true)); + let worker_is_alive = Arc::clone(&is_alive); + + std::thread::Builder::new() + .name(format!("lance-uring-{}", thread_id)) + .spawn(move || run_uring_thread(ring, request_rx, worker_is_alive, thread_id))?; + + Ok(UringThreadHandle { + request_tx, + is_alive, + }) +} + /// Atomic counter for round-robin thread selection. pub(super) static THREAD_SELECTOR: AtomicU64 = AtomicU64::new(0); @@ -114,13 +182,13 @@ fn get_thread_count() -> usize { /// 2. Submits them to io_uring /// 3. Processes completions /// 4. Wakes futures via their wakers -fn run_uring_thread(request_rx: Receiver>, queue_depth: usize, thread_id: usize) { - // Create local io_uring instance - let mut ring = IoUring::builder() - // .setup_sqpoll(100) - .build(queue_depth as u32) - .expect("Failed to create io_uring"); - +fn run_uring_thread( + mut ring: IoUring, + request_rx: Receiver, + is_alive: Arc, + thread_id: usize, +) { + let queue_depth = ring.submission().capacity(); let mut pending: HashMap> = HashMap::with_capacity(queue_depth); let poll_timeout = get_poll_timeout(); let submit_batch_size = get_submit_batch_size(); @@ -197,8 +265,7 @@ fn run_uring_thread(request_rx: Receiver>, queue_depth: usize, th match recv_result { Ok(request) => { - // Decrement submitted counter when we receive the request from channel - SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed); + let request = request.into_request(); // Push to submission queue (but don't submit yet) if let Err(e) = push_to_sq(&mut ring, &mut pending, request) { @@ -218,14 +285,19 @@ fn run_uring_thread(request_rx: Receiver>, queue_depth: usize, th } Err(std::sync::mpsc::TryRecvError::Disconnected) => { // All senders dropped - submit batch and shutdown - if batch_count > 0 - && let Err(e) = ring.submit() - { - log::error!( - "io_uring[{}]: Failed to submit io_uring batch: {}", - thread_id, - e - ); + if batch_count > 0 { + let queued = ring.submission().len(); + if let Err(error) = submit_all(queued, || ring.submit()) { + shutdown_with_error( + ring, + pending, + &request_rx, + &is_alive, + thread_id, + error, + ); + return; + } } log::info!( "io_uring thread {} shutting down (channel disconnected)", @@ -237,18 +309,69 @@ fn run_uring_thread(request_rx: Receiver>, queue_depth: usize, th } // Submit if we have any requests (from channel or retries) - if (batch_count > 0 || needs_submit) - && let Err(e) = ring.submit() - { - log::error!( - "Failed to submit io_uring batch of {} requests: {}", - batch_count, - e - ); + if batch_count > 0 || needs_submit { + let queued = ring.submission().len(); + if let Err(error) = submit_all(queued, || ring.submit()) { + shutdown_with_error(ring, pending, &request_rx, &is_alive, thread_id, error); + return; + } } } } +/// Submit every entry currently published to the submission queue. +/// +/// `io_uring_enter` may be interrupted or accept only part of a batch. The +/// remaining entries stay in the userspace submission queue and must be retried; +/// otherwise their requests remain pending without a possible completion. +fn submit_all(mut queued: usize, mut submit: impl FnMut() -> io::Result) -> io::Result<()> { + while queued > 0 { + match submit() { + Ok(0) => { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + format!("io_uring submitted 0 of {queued} queued requests"), + )); + } + Ok(submitted) if submitted <= queued => queued -= submitted, + Ok(submitted) => { + return Err(io::Error::other(format!( + "io_uring reported {submitted} submissions for {queued} queued requests" + ))); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + } + } + + Ok(()) +} + +fn shutdown_with_error( + ring: IoUring, + mut pending: HashMap>, + request_rx: &Receiver, + is_alive: &AtomicBool, + thread_id: usize, + error: io::Error, +) { + let error_kind = error.kind(); + let error_message = format!("io_uring worker {thread_id} stopped: {error}"); + log::error!("{}", error_message); + + // Closing the ring cancels in-flight operations before their request buffers + // can be released by the futures receiving the errors below. + drop(ring); + is_alive.store(false, Ordering::Release); + + for request in pending.drain().map(|(_, request)| request) { + request.fail(io::Error::new(error_kind, error_message.clone())); + } + for request in request_rx.try_iter() { + request.fail(io::Error::new(error_kind, error_message.clone())); + } +} + /// Push a read request to the io_uring submission queue (without submitting). /// /// This generates a unique user_data ID, prepares the read operation, @@ -394,3 +517,87 @@ fn process_completions( retries, }) } + +#[cfg(test)] +mod tests { + use super::{QueuedRequest, start_uring_thread, submit_all}; + use crate::uring::requests::{IoRequest, RequestState}; + use bytes::BytesMut; + use std::collections::VecDeque; + use std::io; + use std::sync::mpsc::sync_channel; + use std::sync::{Arc, Barrier, Mutex}; + use std::thread; + + #[test] + fn test_submit_all_retries_interrupted_and_partial_submissions() { + let mut results = VecDeque::from([ + Err(io::Error::from(io::ErrorKind::Interrupted)), + Ok(1), + Ok(2), + ]); + + submit_all(3, || results.pop_front().unwrap()).unwrap(); + + assert!(results.is_empty()); + } + + #[test] + fn test_submit_all_rejects_zero_progress() { + let error = submit_all(1, || Ok(0)).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::WriteZero); + assert!(error.to_string().contains("submitted 0 of 1")); + } + + #[test] + fn test_worker_is_not_published_when_ring_initialization_fails() { + let result = start_uring_thread(0, 0); + + assert!(result.is_err()); + } + + #[test] + fn test_late_queued_request_is_failed_when_worker_receiver_drops() { + let request = Arc::new(IoRequest { + fd: -1, + offset: 0, + length: 1, + thread_id: thread::current().id(), + state: Mutex::new(RequestState { + completed: false, + waker: None, + err: None, + buffer: BytesMut::zeroed(1), + bytes_read: 0, + }), + }); + let (request_tx, request_rx) = sync_channel(1); + let drained = Arc::new(Barrier::new(2)); + let request_sent = Arc::new(Barrier::new(2)); + + let sender = { + let request = Arc::clone(&request); + let drained = Arc::clone(&drained); + let request_sent = Arc::clone(&request_sent); + thread::spawn(move || { + drained.wait(); + assert!(request_tx.send(QueuedRequest::new(request)).is_ok()); + request_sent.wait(); + }) + }; + + assert!(request_rx.try_recv().is_err()); + drained.wait(); + request_sent.wait(); + drop(request_rx); + sender.join().unwrap(); + + let state = request.state.lock().unwrap(); + assert!(state.completed); + assert_eq!( + state.err.as_ref().unwrap().kind(), + io::ErrorKind::BrokenPipe + ); + } +} diff --git a/rust/lance-io/src/utils.rs b/rust/lance-io/src/utils.rs index a36aadec9d9..fbcd2c7a131 100644 --- a/rust/lance-io/src/utils.rs +++ b/rust/lance-io/src/utils.rs @@ -1,81 +1,47 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{cmp::min, num::NonZero, sync::atomic::AtomicU64}; +use std::{cmp::min, num::NonZero, ops::Range, sync::atomic::AtomicU64}; -use arrow_array::{ - ArrayRef, - types::{BinaryType, LargeBinaryType, LargeUtf8Type, Utf8Type}, -}; -use arrow_schema::DataType; use byteorder::{ByteOrder, LittleEndian}; -use bytes::Bytes; -use lance_arrow::*; +use bytes::{Bytes, BytesMut}; +use futures::{Stream, StreamExt, TryStreamExt}; use lance_core::deepsize::DeepSizeOf; use prost::Message; use serde::{Deserialize, Serialize}; -use crate::{ReadBatchParams, traits::Reader}; -use crate::{ - encodings::{AsyncIndex, Decoder, binary::BinaryDecoder, plain::PlainDecoder}, - traits::ProtoStruct, -}; +use crate::traits::{ProtoStruct, Reader}; use lance_core::{Error, Result}; pub mod tracking_store; -/// Read a binary array from a [Reader]. +/// Chunk size for splitting a large metadata read into concurrent range requests. /// -pub async fn read_binary_array( +/// A single object-store GET streams its body over one connection, so its +/// throughput is capped by the TCP window over the round-trip time; on +/// high-latency links that tops out in the tens of MB/s. Fetching the range as +/// a window of concurrent chunk requests multiplies that per-connection limit. +/// 16 MiB keeps per-request overhead negligible (a ~1 GiB manifest costs ~64 +/// GET requests) while a `Reader::io_parallelism` window of such chunks is +/// enough to saturate the link. +pub const METADATA_READ_CHUNK_SIZE: usize = 16 * 1024 * 1024; + +/// Read `range` from `reader` as `chunk_size`-sized concurrent range requests, +/// yielding the chunks in file order. Concurrency is bounded by +/// [`Reader::io_parallelism`], clamped to at least 1: a `buffered(0)` window +/// never polls its input, so an unvalidated reader value (e.g. +/// `LANCE_URING_IO_PARALLELISM=0`) would hang the read. +pub fn read_range_in_chunks( reader: &dyn Reader, - data_type: &DataType, - nullable: bool, - position: usize, - length: usize, - params: impl Into, -) -> Result { - use arrow_schema::DataType::*; - let decoder: Box> + Send> = match data_type { - Utf8 => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - Binary => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - LargeUtf8 => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - LargeBinary => Box::new(BinaryDecoder::::new( - reader, position, length, nullable, - )), - _ => { - return Err(Error::invalid_input(format!( - "Unsupported binary type: {}", - data_type - ))); - } - }; - let fut = decoder.as_ref().get(params.into()); - fut.await -} - -/// Read a fixed stride array from disk. -/// -pub async fn read_fixed_stride_array( - reader: &dyn Reader, - data_type: &DataType, - position: usize, - length: usize, - params: impl Into, -) -> Result { - if !data_type.is_fixed_stride() { - return Err(Error::schema(format!( - "{data_type} is not a fixed stride type" - ))); - } - // TODO: support more than plain encoding here. - let decoder = PlainDecoder::new(reader, data_type, position, length)?; - decoder.get(params.into()).await + range: Range, + chunk_size: usize, +) -> impl Stream> + '_ { + let end = range.end; + let chunk_ranges = range + .step_by(chunk_size) + .map(move |start| start..min(start + chunk_size, end)); + futures::stream::iter(chunk_ranges.map(|chunk| reader.get_range(chunk))) + .buffered(reader.io_parallelism().max(1)) } /// Read a protobuf message at file position 'pos'. @@ -98,12 +64,19 @@ pub async fn read_message(reader: &dyn Reader, pos: usize) if msg_len + 4 > buf.len() { let remaining_range = range.end..min(4 + pos + msg_len, file_size); - let remaining_bytes = reader.get_range(remaining_range).await?; - let buf = [buf, remaining_bytes].concat(); - if buf.len() < msg_len + 4 { + // Assemble into one pre-allocated buffer; fetching the remainder as + // concurrent chunks lifts the single-connection throughput cap on + // large messages (e.g. manifests of datasets with many fragments). + let mut full = BytesMut::with_capacity(buf.len() + remaining_range.len()); + full.extend_from_slice(&buf); + let mut chunks = read_range_in_chunks(reader, remaining_range, METADATA_READ_CHUNK_SIZE); + while let Some(chunk) = chunks.try_next().await? { + full.extend_from_slice(&chunk); + } + if full.len() < msg_len + 4 { return Err(Error::io("file size is too small".to_string())); } - Ok(M::decode(&buf[4..4 + msg_len])?) + Ok(M::decode(&full[4..4 + msg_len])?) } else { Ok(M::decode(&buf[4..4 + msg_len])?) } @@ -263,7 +236,8 @@ impl CachedFileSize { #[cfg(test)] mod tests { - use bytes::Bytes; + use bytes::{Bytes, BytesMut}; + use futures::TryStreamExt; use object_store::path::Path; use crate::{ @@ -272,7 +246,7 @@ mod tests { object_store::{DEFAULT_DOWNLOAD_RETRY_COUNT, ObjectStore}, object_writer::ObjectWriter, traits::{ProtoStruct, WriteExt, Writer}, - utils::read_struct, + utils::{METADATA_READ_CHUNK_SIZE, read_range_in_chunks, read_struct}, }; // Bytes is a prost::Message, since we don't have any .proto files in this crate we @@ -318,6 +292,73 @@ mod tests { assert_eq!(some_message, actual); } + #[tokio::test] + async fn test_read_range_in_chunks_reassembles_in_order() { + let store = ObjectStore::memory(); + let path = Path::from("/chunked"); + // Patterned data with a range that neither starts nor ends on a chunk + // boundary, so ordering or off-by-one mistakes change the bytes. + let data: Vec = (0..10 * 1024 + 37).map(|i| (i % 251) as u8).collect(); + store.put(&path, &data).await.unwrap(); + let reader = store.open(&path).await.unwrap(); + + let range = 5..data.len() - 3; + let mut assembled = BytesMut::new(); + let mut chunks = read_range_in_chunks(reader.as_ref(), range.clone(), 1024); + while let Some(chunk) = chunks.try_next().await.unwrap() { + assembled.extend_from_slice(&chunk); + } + assert_eq!(assembled.as_ref(), &data[range]); + } + + #[tokio::test] + async fn test_read_range_in_chunks_zero_parallelism_reader() { + // A reader advertising io_parallelism 0 (e.g. LANCE_URING_IO_PARALLELISM=0) + // must not hang the chunked read: the window is clamped to at least 1. + let store = ObjectStore::memory(); + let path = Path::from("/zero_parallelism"); + let data: Vec = (0..4096).map(|i| (i % 249) as u8).collect(); + store.put(&path, &data).await.unwrap(); + let reader = + CloudObjectReader::new(store.inner, path, 1024, None, DEFAULT_DOWNLOAD_RETRY_COUNT) + .unwrap() + .with_io_parallelism(0); + + let assembled = tokio::time::timeout(std::time::Duration::from_secs(5), async { + let mut buf = BytesMut::new(); + let mut chunks = read_range_in_chunks(&reader, 0..data.len(), 1024); + while let Some(chunk) = chunks.try_next().await.unwrap() { + buf.extend_from_slice(&chunk); + } + buf + }) + .await + .expect("chunked read with a zero-parallelism reader must not hang"); + assert_eq!(assembled.as_ref(), &data[..]); + } + + #[tokio::test] + async fn test_read_message_larger_than_chunk_size() { + // A message body crossing METADATA_READ_CHUNK_SIZE forces read_message + // to fetch the remainder as multiple concurrent chunks. + let store = ObjectStore::memory(); + let path = Path::from("/large_message"); + + let mut object_writer = ObjectWriter::new(&store, &path).await.unwrap(); + let payload: Vec = (0..METADATA_READ_CHUNK_SIZE + 5 * 1024 * 1024) + .map(|i| (i % 253) as u8) + .collect(); + let message = BytesWrapper(Bytes::from(payload)); + let pos = object_writer.write_struct(&message).await.unwrap(); + object_writer.shutdown().await.unwrap(); + + let object_reader = + CloudObjectReader::new(store.inner, path, 4096, None, DEFAULT_DOWNLOAD_RETRY_COUNT) + .unwrap(); + let actual: BytesWrapper = read_struct(&object_reader, pos).await.unwrap(); + assert_eq!(message, actual); + } + #[tokio::test] async fn test_copy_reader_to_writer() { let store = ObjectStore::memory(); diff --git a/rust/lance-io/src/utils/tracking_store.rs b/rust/lance-io/src/utils/tracking_store.rs index 588ff8f71b3..ffa74dcb89a 100644 --- a/rust/lance-io/src/utils/tracking_store.rs +++ b/rust/lance-io/src/utils/tracking_store.rs @@ -13,6 +13,8 @@ use std::ops::Range; #[cfg(feature = "test-util")] use std::sync::atomic::AtomicU16; use std::sync::{Arc, Mutex}; +#[cfg(feature = "metrics")] +use std::time::Instant; use bytes::Bytes; use futures::StreamExt; @@ -26,9 +28,20 @@ use object_store::{ }; use crate::object_store::WrappingObjectStore; +#[cfg(feature = "metrics")] +use crate::object_store::metrics::{InFlightGuard, record_outcome}; +use object_store::list::PaginatedListStore; #[derive(Debug, Default, Clone)] -pub struct IOTracker(Arc>); +pub struct IOTracker { + stats: Arc>, + /// The `base` label for the object store metrics published by IO that + /// bypasses the `object_store` layer (see [`Self::begin_io`]). `None` when + /// the IO cannot be attributed to a store, in which case no metrics are + /// published. + #[cfg(feature = "metrics")] + metrics_base: Option>, +} impl IOTracker { /// Get IO statistics and reset the counters (incremental pattern). @@ -36,7 +49,7 @@ impl IOTracker { /// This returns the accumulated statistics since the last call and resets /// the internal counters to zero. pub fn incremental_stats(&self) -> IoStats { - std::mem::take(&mut *self.0.lock().unwrap()) + std::mem::take(&mut *self.stats.lock().unwrap()) } /// Get a snapshot of current IO statistics without resetting counters. @@ -44,7 +57,7 @@ impl IOTracker { /// This returns a clone of the current statistics without modifying the /// internal state. Use this when you need to check stats without resetting. pub fn stats(&self) -> IoStats { - self.0.lock().unwrap().clone() + self.stats.lock().unwrap().clone() } /// Record a read operation for tracking. @@ -58,7 +71,7 @@ impl IOTracker { num_bytes: u64, #[allow(unused_variables)] range: Option>, ) { - let mut stats = self.0.lock().unwrap(); + let mut stats = self.stats.lock().unwrap(); stats.read_iops += 1; stats.read_bytes += num_bytes; #[cfg(feature = "test-util")] @@ -79,7 +92,7 @@ impl IOTracker { #[allow(unused_variables)] path: Path, num_bytes: u64, ) { - let mut stats = self.0.lock().unwrap(); + let mut stats = self.stats.lock().unwrap(); stats.write_iops += 1; stats.written_bytes += num_bytes; #[cfg(feature = "test-util")] @@ -89,11 +102,97 @@ impl IOTracker { range: None, }); } + + /// Label the metrics published through [`Self::begin_io`] with the prefix of + /// the store this tracker belongs to, so IO that bypasses the `object_store` + /// layer carries the same `base` label as the store's metered operations. + /// + /// Only `meter_store` should call this, so that labelling the tracker and + /// wrapping the store stay inseparable — see the rationale there. + #[cfg(feature = "metrics")] + pub(crate) fn set_metrics_base(&mut self, base: &str) { + self.metrics_base = Some(base.into()); + } + + /// Begin an operation that talks to storage without going through the + /// `object_store` layer, and so is invisible to the `MeteredObjectStore` + /// wrapper: the optimized local reads and writes go straight to the + /// filesystem. `operation` must be one of the labels that wrapper uses + /// (`get`, `put`, `head`, ...) so this IO aggregates with the rest. + /// + /// The returned guard keeps the in-flight gauge raised until it is dropped. + #[cfg(feature = "metrics")] + pub fn begin_io(&self, operation: &'static str) -> IoMetricsGuard { + IoMetricsGuard { + state: self.metrics_base.as_ref().map(|base| IoMetricsState { + _in_flight: InFlightGuard::new(base, operation), + base: base.clone(), + operation, + start: Instant::now(), + }), + } + } + + /// Without the `metrics` feature there is nothing to publish. + #[cfg(not(feature = "metrics"))] + pub fn begin_io(&self, _operation: &'static str) -> IoMetricsGuard { + IoMetricsGuard {} + } +} + +/// Publishes the object store metrics for a single operation that bypassed the +/// `object_store` layer (see [`IOTracker::begin_io`]). +/// +/// The operation is only counted by [`Self::record`]; one dropped before that — +/// a cancelled read, an abandoned write — counts as neither a success nor a +/// failure, and only lowers the in-flight gauge. +#[must_use = "the operation is not recorded until `record` is called"] +pub struct IoMetricsGuard { + #[cfg(feature = "metrics")] + state: Option, +} + +#[cfg(feature = "metrics")] +struct IoMetricsState { + base: Arc, + operation: &'static str, + start: Instant, + /// Lowers the in-flight gauge when the guard is dropped. + _in_flight: InFlightGuard, +} + +impl IoMetricsGuard { + /// Record the operation's count and latency, along with `num_bytes` + /// transferred if `result` is `Ok` or an error if it is not. + pub fn record(self, result: &std::result::Result, num_bytes: u64) { + #[cfg(feature = "metrics")] + if let Some(state) = self.state { + record_outcome( + &state.base, + state.operation, + state.start, + num_bytes, + result.is_err(), + ); + } + #[cfg(not(feature = "metrics"))] + let _ = (result, num_bytes); + } } impl WrappingObjectStore for IOTracker { fn wrap(&self, _store_prefix: &str, target: Arc) -> Arc { - Arc::new(IoTrackingStore::new(target, self.0.clone())) + Arc::new(IoTrackingStore::new(target, self.stats.clone())) + } + + // A pushed-down listing records itself against the store's tracker, so it is already + // counted without passing through here. + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) } } diff --git a/rust/lance-io/tests/goosefs_integration.rs b/rust/lance-io/tests/goosefs_integration.rs index fd9015bcda0..0cd18d65c3c 100644 --- a/rust/lance-io/tests/goosefs_integration.rs +++ b/rust/lance-io/tests/goosefs_integration.rs @@ -25,7 +25,7 @@ fn get_operator() -> Operator { cfg.insert("master_addr".to_string(), addr); cfg.insert("root".to_string(), "/lance-test/opendal".to_string()); cfg.insert("auth_type".to_string(), auth_type); - Operator::from_iter::(cfg).unwrap().finish() + Operator::from_iter::(cfg).unwrap() } // ============================================================ @@ -192,6 +192,17 @@ async fn test_diag_lance_io_write_modes() { .await .expect("Failed to create ObjectStore"); + // Best-effort cleanup of leftover test files. object_store_opendal maps + // relative paths to the GooseFS root, so prior runs leave files at / + // even when the ObjectStore URI uses a per-run subdirectory. The + // concurrent exactly-one-winner race now lives in its own + // `test_diag_concurrent_put_create_exactly_one_winner` test, which + // uses a per-run filename so it does not share state with this test. + for name in ["test_file.txt", "test_create.txt"].iter() { + let p = object_store::path::Path::parse(name).unwrap(); + let _ = object_store.inner.delete(&p).await; + } + // Test 1: Basic put + get let test_path = object_store::path::Path::parse("test_file.txt").unwrap(); let test_data = bytes::Bytes::from("Hello from lance-io ObjectStore!"); @@ -224,12 +235,14 @@ async fn test_diag_lance_io_write_modes() { Err(e) => eprintln!("[DIAG] Read FAILED: {:?}", e), } - // Test 2: PutMode::Create (if_not_exists) + // PutMode::Create (if_not_exists) — required by + // ConditionalPutCommitHandler for concurrent-safe manifest commits. + let create_path = object_store::path::Path::parse("test_create.txt").unwrap(); eprintln!("[DIAG] Writing with PutMode::Create (if_not_exists)..."); - match object_store + object_store .inner .put_opts( - &object_store::path::Path::parse("test_create.txt").unwrap(), + &create_path, bytes::Bytes::from("conditional write!").into(), object_store::PutOptions { mode: object_store::PutMode::Create, @@ -237,12 +250,30 @@ async fn test_diag_lance_io_write_modes() { }, ) .await - { - Ok(_) => eprintln!("[DIAG] PutMode::Create succeeded! ✅"), - Err(e) => { - eprintln!("[DIAG] PutMode::Create FAILED: {:?}", e); - } - } + .expect("PutMode::Create should succeed for a new path"); + eprintln!("[DIAG] PutMode::Create succeeded! ✅"); + + eprintln!("[DIAG] Second PutMode::Create on same path (expect AlreadyExists)..."); + let conflict = object_store + .inner + .put_opts( + &create_path, + bytes::Bytes::from("should not overwrite").into(), + object_store::PutOptions { + mode: object_store::PutMode::Create, + ..Default::default() + }, + ) + .await; + assert!( + matches!( + conflict, + Err(object_store::Error::AlreadyExists { .. }) + | Err(object_store::Error::Precondition { .. }) + ), + "second PutMode::Create must fail with AlreadyExists/Precondition, got: {conflict:?}" + ); + eprintln!("[DIAG] PutMode::Create conflict correctly rejected! ✅"); // Test 3: rename_if_not_exists eprintln!("[DIAG] Testing rename_if_not_exists..."); @@ -269,3 +300,186 @@ async fn test_diag_lance_io_write_modes() { eprintln!("[DIAG] lance-io direct write test complete ✅"); } + +/// Concurrency regression: two `PutMode::Create` writers racing on the +/// same fresh path must produce exactly one winner, and the stored bytes +/// must match the winner's payload byte-for-byte. This is the property +/// that makes `ConditionalPutCommitHandler` safe under concurrent +/// manifest commits: every writer either commits its manifest or observes +/// the loser's precondition failure and retries against a fresh version. +/// +/// Split out of `test_diag_lance_io_write_modes` so the race assertions +/// cannot be silently skipped — the basic-write `return` on error in the +/// diagnostic test no longer covers the race, and the race is now a +/// first-class ignored test with its own setup/teardown invariants: +/// +/// 1. **Per-run path.** The race filename embeds a nanosecond timestamp so +/// concurrent or back-to-back runs in the same process never share +/// state. `object_store_opendal` maps the relative filename to the +/// GooseFS root regardless of the ObjectStore URI path, so a +/// per-run URI subdirectory would not isolate runs on its own. +/// 2. **Explicit pre-cleanup.** A leftover from a prior crashed run is +/// expected (`NotFound` is the only accepted pre-cleanup error); any +/// other error fails the test rather than being silently ignored. +/// 3. **Strict setup.** Every setup step uses `expect` so a transient +/// `ObjectStore` construction or filename-parse error is surfaced +/// rather than logged and skipped. +/// 4. **Post-cleanup.** The winning path is removed on success; cleanup +/// must report `Ok` or `NotFound`. +/// +/// Requires a live GooseFS cluster. +#[tokio::test] +#[ignore = "Requires GooseFS cluster"] +async fn test_diag_concurrent_put_create_exactly_one_winner() { + let addr = std::env::var("GOOSEFS_MASTER_ADDR").unwrap_or_else(|_| "127.0.0.1:9200".into()); + // Nanosecond timestamp gives effectively unique filenames even for + // back-to-back runs in the same process. The race path is *not* put + // under the per-run URI subdirectory because `object_store_opendal` + // ignores the URL path when resolving relative keys. + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let race_filename = format!("test_create_race_{}.txt", ts); + let root = format!("goosefs://{}/lance-test", addr); + eprintln!( + "[DIAG] Creating ObjectStore at: {} (race path: {})", + root, race_filename + ); + + let params = ObjectStoreParams::default(); + let registry = Arc::new(ObjectStoreRegistry::default()); + let (object_store, _path) = ObjectStore::from_uri_and_params(registry.clone(), &root, ¶ms) + .await + .expect("Failed to create ObjectStore"); + + let race_path = object_store::path::Path::parse(&race_filename).unwrap(); + + // Pre-flight cleanup: the race path is per-run, so a leftover can + // only come from a prior run that crashed before its post-cleanup + // completed. `Ok` and `NotFound` are both expected; any other error + // is a setup failure that would otherwise silently skip the race. + match object_store.inner.delete(&race_path).await { + Ok(_) => {} + Err(object_store::Error::NotFound { .. }) => {} + Err(e) => panic!("pre-cleanup of race path {race_filename} failed: {e:?}"), + } + + let store_a = object_store.clone(); + let store_b = object_store.clone(); + let path_a = race_path.clone(); + let path_b = race_path.clone(); + let payload_a = bytes::Bytes::from("writer-A"); + let payload_b = bytes::Bytes::from("writer-B"); + + eprintln!("[DIAG] Launching two concurrent PutMode::Create on fresh path..."); + let fut_a = tokio::spawn(async move { + store_a + .inner + .put_opts( + &path_a, + payload_a.clone().into(), + object_store::PutOptions { + mode: object_store::PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| payload_a) + }); + let fut_b = tokio::spawn(async move { + store_b + .inner + .put_opts( + &path_b, + payload_b.clone().into(), + object_store::PutOptions { + mode: object_store::PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| payload_b) + }); + let (res_a, res_b) = tokio::join!(fut_a, fut_b); + let res_a = res_a.expect("writer A join"); + let res_b = res_b.expect("writer B join"); + + let mut wins = 0usize; + let mut conflicts = 0usize; + let mut other_err: Option = None; + let mut winner_payload: Option = None; + match (&res_a, &res_b) { + (Ok(p), Err(_)) => { + wins = 1; + conflicts = 1; + winner_payload = Some(p.clone()); + } + (Err(_), Ok(p)) => { + wins = 1; + conflicts = 1; + winner_payload = Some(p.clone()); + } + (Ok(_), Ok(_)) => { + other_err = Some("both writers reported success — if-not-exists violated".into()); + } + (Err(ea), Err(eb)) => { + other_err = Some(format!("both writers failed: a={ea:?} b={eb:?}")); + } + } + for r in [&res_a, &res_b] { + if let Err(e) = r { + let s = format!("{e:?}").to_lowercase(); + assert!( + s.contains("already exists") + || s.contains("precondition") + || s.contains("conditionnotmatch") + || s.contains("if_not_exists"), + "loser must report AlreadyExists/Precondition, got: {e:?}" + ); + } + } + assert!( + other_err.is_none(), + "exactly-one-winner violated: {}", + other_err.unwrap() + ); + assert_eq!(wins, 1, "exactly one writer must win (got {wins})"); + assert_eq!( + conflicts, 1, + "exactly one writer must conflict (got {conflicts})" + ); + + // Read back — stored bytes must match the winner's payload exactly. + let stored = object_store + .inner + .get(&race_path) + .await + .expect("get winning payload") + .bytes() + .await + .expect("read winning payload"); + let expected = winner_payload.expect("winner payload recorded"); + assert_eq!( + stored, expected, + "stored bytes must match the winner's payload" + ); + eprintln!( + "[DIAG] concurrent exactly-one-winner ok ✅ stored={} bytes", + stored.len() + ); + + // Post-flight cleanup: must succeed. `NotFound` after a winning write + // would mean the file never actually persisted, contradicting the + // assertion above, so we surface it as a failure rather than masking + // a real regression. + match object_store.inner.delete(&race_path).await { + Ok(_) => {} + Err(object_store::Error::NotFound { .. }) => { + panic!( + "post-cleanup of race path {race_filename} reported NotFound after a winning write" + ) + } + Err(e) => panic!("post-cleanup of race path {race_filename} failed: {e:?}"), + } +} diff --git a/rust/lance-linalg/Cargo.toml b/rust/lance-linalg/Cargo.toml index 6a188ec3c62..8a75eb077c4 100644 --- a/rust/lance-linalg/Cargo.toml +++ b/rust/lance-linalg/Cargo.toml @@ -11,20 +11,21 @@ categories = { workspace = true } [dependencies] arrow-array = { workspace = true } -arrow-buffer = { workspace = true } arrow-schema = { workspace = true } half = { workspace = true } lance-arrow = { workspace = true } lance-core = { workspace = true } num-traits = { workspace = true } -rand = { workspace = true } rayon = { workspace = true } [dev-dependencies] approx = { workspace = true } +arrow-buffer = { workspace = true } criterion = { workspace = true } -lance-testing = { path = "../lance-testing" } +lance-testing = { workspace = true } proptest.workspace = true +rand = { workspace = true } +rstest.workspace = true [build-dependencies] cc = "1.0.83" @@ -59,5 +60,9 @@ harness = false name = "dist_table" harness = false +[[bench]] +name = "batch_distance" +harness = false + [lints] workspace = true diff --git a/rust/lance-linalg/README.md b/rust/lance-linalg/README.md index 06db3e9684e..c436da15617 100644 --- a/rust/lance-linalg/README.md +++ b/rust/lance-linalg/README.md @@ -1,6 +1,6 @@ # Lance Linear Algebra Library -`lance-linalg` is a internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) -native linear algebra algorithm used the [Lance](https://github.com/lancedb/lance). +`lance-linalg` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs) +native linear algebra algorithms used by [Lance](https://github.com/lance-format/lance). diff --git a/rust/lance-linalg/benches/batch_distance.rs b/rust/lance-linalg/benches/batch_distance.rs new file mode 100644 index 00000000000..d4921b1a80c --- /dev/null +++ b/rust/lance-linalg/benches/batch_distance.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::hint::black_box; +use std::time::Duration; + +use criterion::{Criterion, criterion_group, criterion_main}; +use lance_linalg::distance::{Cosine, Dot, L2}; + +const DIMENSION: usize = 8; +const TOTAL_VALUES: usize = 1024 * 1024; + +fn bench_batch_distance(c: &mut Criterion) { + let key = (0..DIMENSION) + .map(|index| index as f32 * 0.125 + 0.25) + .collect::>(); + let batch = (0..TOTAL_VALUES) + .map(|index| (index % 31) as f32 * 0.03125 - 0.5) + .collect::>(); + + c.bench_function("Batch distance dot f32 dim 8", |b| { + b.iter(|| { + black_box( + ::dot_batch(black_box(&key), black_box(&batch), DIMENSION).sum::(), + ) + }) + }); + c.bench_function("Batch distance l2 f32 dim 8", |b| { + b.iter(|| { + black_box( + ::l2_batch(black_box(&key), black_box(&batch), DIMENSION).sum::(), + ) + }) + }); + c.bench_function("Batch distance cosine f32 dim 8", |b| { + b.iter(|| { + black_box( + ::cosine_batch(black_box(&key), black_box(&batch), DIMENSION) + .sum::(), + ) + }) + }); +} + +fn bench_time() -> Duration { + let seconds = option_env!("TARGET_TIME") + .unwrap_or("5") + .parse() + .expect("TARGET_TIME must be an integer number of seconds"); + Duration::from_secs(seconds) +} + +criterion_group!( + name = benches; + config = Criterion::default() + .significance_level(0.1) + .sample_size(10) + .measurement_time(bench_time()); + targets = bench_batch_distance +); +criterion_main!(benches); diff --git a/rust/lance-linalg/build.rs b/rust/lance-linalg/build.rs index 407f2a589ea..82987771be5 100644 --- a/rust/lance-linalg/build.rs +++ b/rust/lance-linalg/build.rs @@ -17,12 +17,14 @@ fn main() -> Result<(), String> { // Let clippy know about our custom cfg attribute println!( - "cargo::rustc-check-cfg=cfg(kernel_support, values(\"avx512_f16\", \"avx512_bf16\", \"avx512_dist_table\"))" + "cargo::rustc-check-cfg=cfg(kernel_support, values(\"avx512_f16\", \"avx512_bf16\", \"avx512_dist_table\", \"amx_fp16\"))" ); println!("cargo:rerun-if-changed=src/simd/f16.c"); println!("cargo:rerun-if-changed=src/simd/bf16.c"); println!("cargo:rerun-if-changed=src/simd/dist_table.c"); + println!("cargo:rerun-if-changed=src/simd/amx_fp16.c"); + println!("cargo:rerun-if-env-changed=LANCE_AMX_FP16_CC"); // Important: we don't use `cfg!(target_arch)` here because that is the target_arch // for the build script, not the target_arch for the library. Similar story for @@ -84,6 +86,28 @@ fn main() -> Result<(), String> { } else { println!("cargo:rustc-cfg=kernel_support=\"avx512_dist_table\""); }; + // Build the AMX-FP16 batched f16 dot-product kernel (Granite Rapids+). + // + // No cargo feature of its own: whether it can be built is a property of + // the toolchain, not a choice the user makes. `-mamx-fp16` needs clang + // >= 16 or gcc >= 13, so a capable compiler is probed for below; if none + // is found we warn, skip the kernel, and leave `kernel_support` unset. + // + // Linux only: every Rust-side cfg gate on this kernel also demands + // `target_os = "linux"`, and entering it needs XTILEDATA from + // `arch_prctl(ARCH_REQ_XCOMP_PERM)`, which is Linux-specific. + if target_os == "linux" { + if let Err(err) = + build_amx_fp16_with_flags(&["-march=sapphirerapids", "-mamx-fp16", "-mamx-tile"]) + { + println!( + "cargo:warning=Skipping build of AMX-FP16 kernels. Error: {}", + err + ); + } else { + println!("cargo:rustc-cfg=kernel_support=\"amx_fp16\""); + }; + } // Build a version with AVX // While GCC doesn't have support for _Float16 until GCC 12, clang // has support for __fp16 going back to at least clang 6. @@ -196,3 +220,96 @@ fn build_dist_table_with_flags(suffix: &str, flags: &[&str]) -> Result<(), cc::E } builder.try_compile(&format!("dist_table_{}", suffix)) } + +/// Compile the AMX-FP16 kernel. Unlike the other kernels this may need a +/// different C compiler: `_tile_dpfp16ps` / `-mamx-fp16` require clang >= 16 or +/// gcc >= 13, which is often newer than the platform default `cc`. We probe for +/// a capable compiler (respecting a `LANCE_AMX_FP16_CC` override) and use it +/// only for this one file; everything else keeps using the default toolchain. +fn build_amx_fp16_with_flags(flags: &[&str]) -> Result<(), String> { + let compiler = find_amx_fp16_compiler(flags).ok_or_else(|| { + "no C compiler supporting -mamx-fp16 found (need clang>=16 or gcc>=13; \ + set LANCE_AMX_FP16_CC to override)" + .to_string() + })?; + let mut builder = cc::Build::new(); + builder + .compiler(&compiler) + .std("c17") + .file("src/simd/amx_fp16.c") + .flag("-funroll-loops") + .flag("-O3") + .flag("-Wall") + .flag("-Wextra"); + for flag in flags { + builder.flag(flag); + } + builder.try_compile("amx_fp16").map_err(|e| e.to_string()) +} + +/// Find a C compiler that can build the AMX-FP16 kernel with `flags`, in order: +/// the `LANCE_AMX_FP16_CC` override, the default toolchain `cc`, then common +/// modern clang names. Returns the first that compiles a `_tile_dpfp16ps` probe. +fn find_amx_fp16_compiler(flags: &[&str]) -> Option { + let mut candidates: Vec = Vec::new(); + if let Ok(cc) = env::var("LANCE_AMX_FP16_CC") { + candidates.push(cc); + } + if let Ok(default_cc) = cc::Build::new() + .get_compiler() + .path() + .to_str() + .ok_or(()) + .map(str::to_string) + { + candidates.push(default_cc); + } + for c in ["clang-18", "clang-17", "clang-16", "clang"] { + candidates.push(c.to_string()); + } + candidates + .into_iter() + .find(|cc| cc_supports_amx_fp16(cc, flags)) +} + +/// True iff invoking `cc` with `flags` can compile a translation unit that uses +/// `_tile_dpfp16ps`. `-Werror=implicit-function-declaration` turns gcc's +/// "intrinsic not declared" *warning* (which otherwise still exits 0 on the +/// too-old gcc that lacks amx-fp16) into a hard failure. +fn cc_supports_amx_fp16(cc: &str, flags: &[&str]) -> bool { + use std::io::Write; + use std::process::{Command, Stdio}; + + let src = b"#include \n\ + void t(const void* a, const void* b) {\n\ + _tile_loadd(1, a, 64); _tile_loadd(2, b, 4);\n\ + _tile_dpfp16ps(0, 1, 2); _tile_release(); }\n"; + let out = env::var("OUT_DIR") + .map(|d| format!("{d}/amx_fp16_probe.o")) + .unwrap_or_else(|_| "/dev/null".to_string()); + + let mut cmd = Command::new(cc); + cmd.args(flags) + .args([ + "-Werror=implicit-function-declaration", + "-x", + "c", + "-c", + "-", + "-o", + ]) + .arg(&out) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + + let Ok(mut child) = cmd.spawn() else { + return false; + }; + if let Some(mut stdin) = child.stdin.take() + && stdin.write_all(src).is_err() + { + return false; + } + child.wait().map(|s| s.success()).unwrap_or(false) +} diff --git a/rust/lance-linalg/proptest-regressions/distance/cosine.txt b/rust/lance-linalg/proptest-regressions/distance/cosine.txt new file mode 100644 index 00000000000..04e6dcb967f --- /dev/null +++ b/rust/lance-linalg/proptest-regressions/distance/cosine.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 679c764b02e2726ec2d943b3abdc7d7d3565e168d4081b155860c1e69b616fce # shrinks to (x, y) = ([-1.2933521e-15, -1.1194652e-8, 7.447341e-32, -0.0, -14247490.0, -8.55e-43, -4576.872, 0.009181444], [4.926e-42, -9.267065e-15, -0.0, 7e-45, -0.00036999694, -0.0, -5.7810876e-8, 1.9273644e-21]) diff --git a/rust/lance-linalg/src/distance.rs b/rust/lance-linalg/src/distance.rs index 23d1cae2d63..73f4d51df55 100644 --- a/rust/lance-linalg/src/distance.rs +++ b/rust/lance-linalg/src/distance.rs @@ -19,18 +19,253 @@ use arrow_schema::{ArrowError, DataType}; pub mod cosine; pub mod cosine_u8; pub mod dot; +pub mod dot_f16; pub mod dot_u8; pub mod hamming; pub mod l2; pub mod l2_u8; pub mod norm_l2; +#[inline] +fn assert_equal_lengths(left_len: usize, right_len: usize) { + assert_eq!( + left_len, right_len, + "distance inputs must have equal lengths: left={left_len}, right={right_len}" + ); +} + +#[inline] +fn assert_batch_layout(vector_len: usize, batch_len: usize, dimension: usize) { + assert!( + dimension > 0, + "distance dimension must be greater than zero" + ); + assert_eq!( + vector_len, dimension, + "distance vector length must match dimension: vector={vector_len}, dimension={dimension}" + ); + assert_eq!( + batch_len % dimension, + 0, + "distance batch length must be divisible by dimension: batch={batch_len}, dimension={dimension}" + ); +} + +/// Largest number of maximal u8 product terms whose sum fits in a u32. +const U8_U32_ACCUMULATOR_MAX_LEN: usize = u32::MAX as usize / (u8::MAX as usize * u8::MAX as usize); + +/// Number of distances computed per call into a runtime-selected batch kernel. +/// +/// Keeping a small output buffer amortizes the `#[target_feature]` call while +/// avoiding the allocation and full-batch materialization that dominate the +/// common dimension-8 case. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +const BATCH_BUFFER_SIZE: usize = 64; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +pub(crate) type BatchKernel = unsafe fn(&[f32], &[f32], usize, &mut [f32]); + +/// Runtime-selected target-feature tier for a batch kernel. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +#[derive(Clone, Copy)] +pub(crate) enum BatchKind { + Scalar, + Avx, + AvxFma, + Avx512, +} + +/// Per-vector operations used when a batch consumer can be folded directly. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +pub(crate) trait BatchOperation { + fn fold_scalar(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; + + unsafe fn fold_avx(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; + + unsafe fn fold_avx_fma(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; + + unsafe fn fold_avx512(key: &[f32], batch: &[f32], dimension: usize, init: B, f: F) -> B + where + F: FnMut(B, f32) -> B; +} + +/// Allocation-free iterator over a runtime-selected f32 distance kernel. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +pub(crate) struct BatchIter<'a, O> { + key: &'a [f32], + batch: &'a [f32], + dimension: usize, + kernel: BatchKernel, + kind: BatchKind, + buffer: [f32; BATCH_BUFFER_SIZE], + buffer_index: usize, + buffer_len: usize, + operation: std::marker::PhantomData, +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl<'a, O> BatchIter<'a, O> { + /// Creates an iterator after the caller has verified `kernel`'s CPU feature + /// requirements. + /// + /// # Safety + /// The host must support every target feature required by `kernel`. + #[inline] + pub(crate) unsafe fn new( + key: &'a [f32], + batch: &'a [f32], + dimension: usize, + kernel: BatchKernel, + kind: BatchKind, + ) -> Self { + // Match `chunks_exact` validation before the buffered iterator performs + // division by the dimension. + let _ = batch.chunks_exact(dimension); + Self { + key, + batch, + dimension, + kernel, + kind, + buffer: [0.0; BATCH_BUFFER_SIZE], + buffer_index: 0, + buffer_len: 0, + operation: std::marker::PhantomData, + } + } + + #[inline] + fn refill(&mut self) -> bool { + let num_vectors = (self.batch.len() / self.dimension).min(BATCH_BUFFER_SIZE); + if num_vectors == 0 { + return false; + } + + let num_values = num_vectors * self.dimension; + let (input, remaining) = self.batch.split_at(num_values); + // SAFETY: `new` requires the caller to verify the selected kernel's + // target features before constructing the iterator. + unsafe { + (self.kernel)( + self.key, + input, + self.dimension, + &mut self.buffer[..num_vectors], + ); + } + self.batch = remaining; + self.buffer_index = 0; + self.buffer_len = num_vectors; + true + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl Iterator for BatchIter<'_, O> { + type Item = f32; + + #[inline] + fn next(&mut self) -> Option { + if self.buffer_index == self.buffer_len && !self.refill() { + return None; + } + let value = self.buffer[self.buffer_index]; + self.buffer_index += 1; + Some(value) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } + + /// Processes each filled buffer directly so consumers such as `sum` do not + /// pay a refill check for every distance. + #[inline] + fn fold(self, init: B, mut f: F) -> B + where + F: FnMut(B, Self::Item) -> B, + { + let accumulator = self.buffer[self.buffer_index..self.buffer_len] + .iter() + .copied() + .fold(init, &mut f); + + // SAFETY: `new` requires the caller to verify the selected tier's + // target features. Each helper runs the complete remaining loop in + // that target-feature context, avoiding intermediate output writes. + match self.kind { + BatchKind::Scalar => { + O::fold_scalar(self.key, self.batch, self.dimension, accumulator, f) + } + BatchKind::Avx => unsafe { + O::fold_avx(self.key, self.batch, self.dimension, accumulator, f) + }, + BatchKind::AvxFma => unsafe { + O::fold_avx_fma(self.key, self.batch, self.dimension, accumulator, f) + }, + BatchKind::Avx512 => unsafe { + O::fold_avx512(self.key, self.batch, self.dimension, accumulator, f) + }, + } + } + + #[inline] + fn for_each(self, mut f: F) + where + F: FnMut(Self::Item), + { + self.fold((), |(), value| f(value)); + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl ExactSizeIterator for BatchIter<'_, O> { + #[inline] + fn len(&self) -> usize { + self.buffer_len - self.buffer_index + self.batch.len() / self.dimension + } +} + pub use cosine::*; pub use dot::*; pub use hamming::{ - Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges, cluster_pairwise_result, - extract_hashes_from_fixed_list, hamming_distance_arrow_batch, hamming_u64, - pairwise_hamming_distance, pairwise_hamming_distance_parallel, + BinaryHashValues, Cluster, ClusteringResult, PairwiseResult, UnionFind, cluster_edges, + cluster_pairwise_result, extract_binary_hashes_from_fixed_list, extract_hashes_from_fixed_list, + hamming_distance_arrow_batch, hamming_u64, pairwise_hamming_distance, + pairwise_hamming_distance_binary, pairwise_hamming_distance_binary_parallel, + pairwise_hamming_distance_parallel, }; pub use l2::*; use lance_core::deepsize::DeepSizeOf; @@ -111,28 +346,79 @@ impl TryFrom<&str> for DistanceType { } } +/// Computes the additive late-interaction distance from a multivector query. +/// +/// For each query sub-vector, this finds the minimum distance to any stored +/// sub-vector in the row, then sums those minimum distances. Null or empty +/// stored rows produce `NaN`. pub fn multivec_distance( query: &dyn Array, vectors: &ListArray, distance_type: DistanceType, ) -> Result> { - let dim = if let DataType::FixedSizeList(_, dim) = vectors.value_type() { - dim as usize - } else { - return Err(ArrowError::InvalidArgumentError( - "vectors must be a list of fixed size list".to_string(), - )); - }; - - // check the query vectors type first - // because we don't want to check the vectors type for each vector - match query.data_type() { - DataType::Float16 | DataType::Float32 | DataType::Float64 | DataType::UInt8 => {} + let (element_type, dim) = match vectors.value_type() { + DataType::FixedSizeList(field, dim) => (field.data_type().clone(), dim as usize), _ => { return Err(ArrowError::InvalidArgumentError( - "query must be a float array or binary array".to_string(), + "vectors must be a list of fixed size list".to_string(), )); } + }; + + // Validate the query once, up front, rather than per vector. The type and + // metric checks below prevent an arrow downcast panic or the `unreachable!` + // dispatch arm — the dispatch picks its kernel type from the query's dtype + // and then downcasts the *stored* values to that same type. The dim, null + // and length checks prevent a `chunks_exact` panic and, worse, silently + // wrong results: a short query yields no sub-vectors and scores every row + // `0.0`, and a null slot is scored from whatever the values buffer holds. + let query_type = query.data_type(); + // Which element types have a kernel here at all. `Int8` is a valid vector + // element type elsewhere in the stack (`l2_distance_arrow_batch` and its + // siblings have an `Int8` arm) but has no multivector kernel, so it is + // rejected for the type, not the metric. + let type_supported = matches!( + query_type, + DataType::UInt8 | DataType::Float16 | DataType::Float32 | DataType::Float64 + ); + if !type_supported { + return Err(ArrowError::InvalidArgumentError(format!( + "multivec_distance: unsupported vector element type {query_type}" + ))); + } + let metric_supported = match query_type { + DataType::UInt8 => distance_type == DistanceType::Hamming, + _ => matches!( + distance_type, + DistanceType::L2 | DistanceType::Cosine | DistanceType::Dot + ), + }; + if !metric_supported { + return Err(ArrowError::InvalidArgumentError(format!( + "multivec_distance: distance type {distance_type} does not support query type {query_type}" + ))); + } + if *query_type != element_type { + return Err(ArrowError::InvalidArgumentError(format!( + "multivec_distance: query type {query_type} does not match the stored vector type {element_type}" + ))); + } + if dim == 0 { + return Err(ArrowError::InvalidArgumentError( + "multivec_distance: stored vectors have dimension 0".to_string(), + )); + } + if query.null_count() > 0 { + return Err(ArrowError::InvalidArgumentError(format!( + "multivec_distance: query must not contain nulls, got {} null(s)", + query.null_count() + ))); + } + if query.is_empty() || !query.len().is_multiple_of(dim) { + return Err(ArrowError::InvalidArgumentError(format!( + "multivec_distance: query length {} must be a positive multiple of the vector dimension {dim}", + query.len() + ))); } let mut dists = Vec::with_capacity(vectors.len()); @@ -146,47 +432,37 @@ pub fn multivec_distance( continue; } - let sim = match distance_type { - DistanceType::Hamming => { - let query = query.as_primitive::().values(); - query - .chunks_exact(dim) - .map(|q| { - multivector - .values() - .as_primitive::() - .values() - .chunks_exact(dim) - .map(|v| hamming::hamming(q, v)) - .min_by(|a, b| a.partial_cmp(b).unwrap()) - .unwrap() - }) - .sum() - } + let distance = match distance_type { + DistanceType::Hamming => multivec_distance_impl::( + query, + multivector, + dim, + hamming::hamming, + ), _ => match query.data_type() { DataType::Float16 => multivec_distance_impl::( query, multivector, dim, - distance_type, + distance_type.func(), ), DataType::Float32 => multivec_distance_impl::( query, multivector, dim, - distance_type, + distance_type.func(), ), DataType::Float64 => multivec_distance_impl::( query, multivector, dim, - distance_type, + distance_type.func(), ), _ => unreachable!("missed to check query type"), }, }; - dists.push(1.0 - sim); + dists.push(distance); } } } @@ -197,11 +473,8 @@ fn multivec_distance_impl( query: &dyn Array, multivector: &FixedSizeListArray, dim: usize, - distance_type: DistanceType, -) -> f32 -where - T::Native: L2 + Cosine + Dot, -{ + distance_func: DistanceFunc, +) -> f32 { let query = query.as_primitive::().values(); query .chunks_exact(dim) @@ -211,8 +484,8 @@ where .as_primitive::() .values() .chunks_exact(dim) - .map(|v| 1.0 - distance_type.func()(q, v)) - .max_by(|a, b| a.total_cmp(b)) + .map(|v| distance_func(q, v)) + .min_by(|a, b| a.total_cmp(b)) .unwrap() }) .sum() @@ -222,12 +495,284 @@ where mod tests { use super::*; + #[cfg(target_arch = "x86_64")] + use std::io::Write; use std::sync::Arc; - use arrow_array::types::Float32Type; - use arrow_array::{Float32Array, ListArray}; - use arrow_buffer::OffsetBuffer; + use arrow_array::types::{Float16Type, Float32Type, Int8Type}; + use arrow_array::{ + Float32Array, Float64Array, Int8Array, Int32Array, ListArray, PrimitiveArray, UInt8Array, + }; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; use arrow_schema::Field; + use half::f16; + use lance_arrow::FixedSizeListArrayExt; + + #[cfg(target_arch = "x86_64")] + #[test] + fn test_x86_runtime_feature_report() { + // Write directly to stderr so this remains visible when libtest captures + // ordinary output from passing tests. + writeln!( + std::io::stderr().lock(), + "lance-linalg x86 runtime features: avx={}, fma={}, avx2={}, avx512f={}, avx512bw={}, avx512vnni={}, avx512vpopcntdq={}", + std::is_x86_feature_detected!("avx"), + std::is_x86_feature_detected!("fma"), + std::is_x86_feature_detected!("avx2"), + std::is_x86_feature_detected!("avx512f"), + std::is_x86_feature_detected!("avx512bw"), + std::is_x86_feature_detected!("avx512vnni"), + std::is_x86_feature_detected!("avx512vpopcntdq"), + ) + .expect("write x86 runtime feature report"); + } + + #[test] + fn test_arrow_batch_type_errors_identify_the_argument() { + let float32_targets = + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![1.0, 2.0]), 2).unwrap(); + let unsupported_query = Int32Array::from(vec![1, 2]); + + for distance_type in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] { + let error = + distance_type.arrow_batch_func()(&unsupported_query, &float32_targets).unwrap_err(); + assert!( + matches!(error, ArrowError::InvalidArgumentError(_)), + "{distance_type} returned a different error variant: {error}" + ); + } + + let unsupported_from_error = + cosine_distance_arrow_batch(&unsupported_query, &float32_targets).unwrap_err(); + assert!( + matches!(&unsupported_from_error, ArrowError::InvalidArgumentError(message) + if message == "`from` has unsupported data type Int32"), + "unexpected unsupported `from` error: {unsupported_from_error}" + ); + + let float32_query = Float32Array::from(vec![1.0, 2.0]); + let float64_targets = + FixedSizeListArray::try_new_from_values(Float64Array::from(vec![1.0, 2.0]), 2).unwrap(); + let mismatched_to_error = + cosine_distance_arrow_batch(&float32_query, &float64_targets).unwrap_err(); + assert!( + matches!(&mismatched_to_error, ArrowError::InvalidArgumentError(message) + if message == "`to` values have data type Float64, expected Float32 to match `from`"), + "unexpected mismatched `to` error: {mismatched_to_error}" + ); + } + + /// Build `List>` rows from flattened sub-vector values. + fn multivecs_of(rows: Vec>, dim: i32) -> ListArray { + let lengths = rows + .iter() + .map(|row| { + assert_eq!(row.len() % dim as usize, 0); + row.len() / dim as usize + }) + .collect::>(); + let values = ScalarBuffer::from(rows.into_iter().flatten().collect::>()); + let inner = PrimitiveArray::::new(values, None); + let fsl = FixedSizeListArray::try_new( + Arc::new(Field::new("item", T::DATA_TYPE, true)), + dim, + Arc::new(inner), + None, + ) + .unwrap(); + let offsets = OffsetBuffer::from_lengths(lengths); + let field = Arc::new(Field::new("item", fsl.data_type().clone(), true)); + ListArray::try_new(field, offsets, Arc::new(fsl), None).unwrap() + } + + /// Build one `List>` row. + fn multivec_of(values: Vec, dim: i32) -> ListArray { + multivecs_of::(vec![values], dim) + } + + /// The `(query dtype, distance type)` pre-check and the dispatch must agree. + /// `UInt8` is only valid with Hamming, and the float types only with the + /// float metrics; a mismatch must be an error rather than a panic in the + /// dispatch arm or inside an arrow downcast. + #[test] + fn test_multivec_distance_rejects_dtype_metric_mismatch() { + let f32_vectors = multivec_of::(vec![1.0, 2.0], 2); + let u8_vectors = multivec_of::(vec![1, 2], 2); + + let u8_query: Arc = Arc::new(UInt8Array::from(vec![1_u8, 2])); + let f32_query: Arc = Arc::new(Float32Array::from(vec![1.0_f32, 2.0])); + + // Query and stored types MATCH in each case, so only the metric is wrong + // — otherwise the element-type check would reject these first and this + // test would pass with the metric guard deleted. + for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] { + let err = multivec_distance(u8_query.as_ref(), &u8_vectors, dt).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) if m.contains("does not support query type")), + "UInt8 query with {dt} must be rejected for the metric, got: {err}" + ); + } + + let err = + multivec_distance(f32_query.as_ref(), &f32_vectors, DistanceType::Hamming).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) if m.contains("does not support query type")), + "Float32 query with hamming must be rejected for the metric, got: {err}" + ); + } + + /// `Int8` is a valid vector element type elsewhere in the crate but has no + /// multivector kernel, so it must be rejected for the type, not the metric. + #[test] + fn test_multivec_distance_rejects_unsupported_element_type() { + let i8_vectors = multivec_of::(vec![1, 2], 2); + let i8_query: Arc = Arc::new(Int8Array::from(vec![1_i8, 2])); + + let err = multivec_distance(i8_query.as_ref(), &i8_vectors, DistanceType::L2).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) if m.contains("unsupported vector element type")), + "Int8 must be rejected for the element type, got: {err}" + ); + } + + /// The query's element type must match the stored vectors': the dispatch + /// picks `T` from the query and then downcasts the stored array to the same + /// `T` without checking it. + #[test] + fn test_multivec_distance_rejects_element_type_mismatch() { + let f16_vectors = + multivec_of::(vec![f16::from_f32(1.0), f16::from_f32(2.0)], 2); + let f32_query: Arc = Arc::new(Float32Array::from(vec![1.0_f32, 2.0])); + + let err = + multivec_distance(f32_query.as_ref(), &f16_vectors, DistanceType::L2).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) if m.contains("does not match the stored vector type")), + "Float32 query against a Float16 column must be rejected, got: {err}" + ); + } + + /// A query length that is not a positive multiple of `dim` is structurally + /// invalid: `chunks_exact` would silently drop the tail, and a query shorter + /// than `dim` would yield no sub-vectors at all and score every row `0.0`. + #[test] + fn test_multivec_distance_rejects_bad_query_length() { + let vectors = multivec_of::(vec![1.0, 2.0], 2); + + for bad in [vec![7.0_f32], vec![7.0, 7.0, 999.0], vec![]] { + let len = bad.len(); + let query: Arc = Arc::new(Float32Array::from(bad)); + let err = multivec_distance(query.as_ref(), &vectors, DistanceType::L2).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) if m.contains("must be a positive multiple")), + "query of length {len} against dim 2 must be rejected, got: {err}" + ); + } + } + + /// A zero-dimension column would panic in `chunks_exact(0)`; it gets its own + /// message rather than blaming the query's length. + #[test] + fn test_multivec_distance_rejects_zero_dim() { + let values = Float32Array::from(Vec::::new()); + let fsl = FixedSizeListArray::try_new_with_length( + Arc::new(Field::new("item", DataType::Float32, true)), + 0, + Arc::new(values), + None, + 1, + ) + .unwrap(); + let field = Arc::new(Field::new("item", fsl.data_type().clone(), true)); + let vectors = ListArray::try_new( + field, + OffsetBuffer::from_lengths([1_usize]), + Arc::new(fsl), + None, + ) + .unwrap(); + let query: Arc = Arc::new(Float32Array::from(vec![1.0_f32, 2.0])); + + let err = multivec_distance(query.as_ref(), &vectors, DistanceType::L2).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) + if m.contains("stored vectors have dimension 0") + && !m.contains("positive multiple")), + "a zero-dim column must be rejected on its own terms, got: {err}" + ); + } + + /// A null query slot is read from the raw values buffer, so it would be + /// silently scored as whatever the buffer holds. + #[test] + fn test_multivec_distance_rejects_null_query() { + let vectors = multivec_of::(vec![1.0, 2.0], 2); + let query: Arc = Arc::new(Float32Array::from(vec![Some(1.0_f32), None])); + + let err = multivec_distance(query.as_ref(), &vectors, DistanceType::L2).unwrap_err(); + assert!( + matches!(&err, ArrowError::InvalidArgumentError(m) if m.contains("must not contain nulls")), + "a query with nulls must be rejected, got: {err}" + ); + } + + /// Each query sub-vector contributes its minimum Hamming distance to the + /// row total. + #[test] + fn test_multivec_distance_hamming() { + let vectors = + multivecs_of::(vec![vec![0b0000_0000, 0b0000_1111], vec![0b0000_0011]], 1); + let query: Arc = Arc::new(UInt8Array::from(vec![0b0000_0000_u8, 0b0000_1111])); + + let dists = multivec_distance(query.as_ref(), &vectors, DistanceType::Hamming).unwrap(); + + assert_eq!(dists, vec![0.0, 4.0]); + } + + #[rstest::rstest] + #[case::l2_perfect( + DistanceType::L2, + vec![1.0, 0.0, 0.0, 1.0], + vec![1.0, 0.0, 0.0, 1.0], + 0.0 + )] + #[case::cosine_perfect( + DistanceType::Cosine, + vec![1.0, 0.0, 0.0, 1.0], + vec![1.0, 0.0, 0.0, 1.0], + 0.0 + )] + #[case::dot_perfect( + DistanceType::Dot, + vec![1.0, 0.0, 0.0, 1.0], + vec![1.0, 0.0, 0.0, 1.0], + 0.0 + )] + #[case::cosine_repeated_query( + DistanceType::Cosine, + vec![0.6, 0.8], + vec![1.0, 0.0, 1.0, 0.0], + 0.8 + )] + #[case::cosine_single_query( + DistanceType::Cosine, + vec![0.0, 1.0], + vec![1.0, 0.0], + 1.0 + )] + fn test_multivec_distance_float( + #[case] distance_type: DistanceType, + #[case] vectors: Vec, + #[case] query: Vec, + #[case] expected: f32, + ) { + let vectors = multivec_of::(vectors, 2); + let query: Arc = Arc::new(Float32Array::from(query)); + + let dists = multivec_distance(query.as_ref(), &vectors, distance_type).unwrap(); + + assert!((dists[0] - expected).abs() < 1e-6); + } #[test] fn test_multivec_distance_empty_row_is_nan() { diff --git a/rust/lance-linalg/src/distance/cosine.rs b/rust/lance-linalg/src/distance/cosine.rs index 995191b77eb..e3580dc4e12 100644 --- a/rust/lance-linalg/src/distance/cosine.rs +++ b/rust/lance-linalg/src/distance/cosine.rs @@ -7,6 +7,10 @@ //! //! `bf16, f16, f32, f64` types are supported. +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +use std::arch::x86_64::{ + _mm_add_ps, _mm_add_ss, _mm_cvtss_f32, _mm_loadu_ps, _mm_movehl_ps, _mm_mul_ps, _mm_shuffle_ps, +}; use std::sync::Arc; use arrow_array::{ @@ -17,12 +21,14 @@ use arrow_array::{ use arrow_schema::DataType; use half::{bf16, f16}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray}; -use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] -use lance_core::utils::cpu::SimdSupport; +#[allow(unused_imports)] +use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; use super::{Dot, norm_l2::norm_l2}; use super::{Normalize, dot::dot}; +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +use crate::distance::BatchKind; +#[allow(unused_imports)] use crate::simd::{ FloatSimd, SIMD, f32::{f32x8, f32x16}, @@ -127,6 +133,10 @@ impl Cosine for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::cosine_bf16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels in `bf16_kernel::*` are compiled with + // `-march=haswell` minimum (which requires AVX2), so they cannot + // run on AVX-only or AVX+FMA hosts. Scalar is the correct route. _ => cosine_scalar(x, x_norm, y), } } @@ -168,7 +178,7 @@ impl Cosine for f16 { kernel::cosine_f16_avx512(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::cosine_f16_avx2(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -179,86 +189,394 @@ impl Cosine for f16 { SimdSupport::Lsx => unsafe { kernel::cosine_f16_lsx(x.as_ptr(), x_norm, y.as_ptr(), y.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => cosine_scalar(x, x_norm, y), } } } -/// f32 kernels for Cosine +/// f32 single-vector cosine helpers used by `cosine_batch` for fixed +/// dimensions 8 and 16. +/// +/// These were previously a single generic `cosine_once` but the +/// monomorphizations have to dispatch on `SIMD_SUPPORT` for the SIMD path +/// to stay correct under any compile baseline. Splitting them into two +/// concrete entry points keeps the dispatch site flat and lets each width +/// route to a `#[target_feature]` AVX2 inner function. mod f32 { use super::*; - // TODO: how can we explicitly infer N? + #[cfg(any(test, not(target_arch = "x86_64")))] #[inline] - pub(super) fn cosine_once, const N: usize>( - x: &[f32], - x_norm: f32, - y: &[f32], - ) -> f32 { - let x = unsafe { S::load_unaligned(x.as_ptr()) }; - let y = unsafe { S::load_unaligned(y.as_ptr()) }; - let y2 = y * y; - let xy = x * y; + pub(super) fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + cosine_once_x86::cosine_once_8_avx512(x, x_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + cosine_once_x86::cosine_once_8_avx_fma(x, x_norm, y) + }, + SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_8_avx(x, x_norm, y) }, + _ => cosine_once_8_scalar(x, x_norm, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_once_8_other(x, x_norm, y) + } + } + + #[cfg(any(test, not(target_arch = "x86_64")))] + #[inline] + pub(super) fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + cosine_once_x86::cosine_once_16_avx512(x, x_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y) + }, + SimdSupport::Avx => unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) }, + _ => cosine_once_16_scalar(x, x_norm, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_once_16_other(x, x_norm, y) + } + } + + /// Portable scalar `cosine_once` for length-8 vectors. Matches the SIMD + /// path modulo summation order. + #[cfg(all(target_arch = "x86_64", test))] + #[inline] + pub(super) fn cosine_once_8_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let mut xy = 0.0f32; + let mut y2 = 0.0f32; + for i in 0..8 { + xy += x[i] * y[i]; + y2 += y[i] * y[i]; + } + 1.0 - xy / x_norm / y2.sqrt() + } + + #[cfg(all(target_arch = "x86_64", any(test, not(target_feature = "avx2"))))] + #[inline] + pub(super) fn cosine_once_16_scalar(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let mut xy = 0.0f32; + let mut y2 = 0.0f32; + for i in 0..16 { + xy += x[i] * y[i]; + y2 += y[i] * y[i]; + } + 1.0 - xy / x_norm / y2.sqrt() + } + + #[cfg(target_arch = "x86_64")] + pub(super) mod cosine_once_x86 { + use std::arch::x86_64::*; + + #[cfg(test)] + use super::f32x8; + #[cfg(any(test, not(target_feature = "avx2")))] + use super::f32x16; + #[cfg(any(test, not(target_feature = "avx2")))] + use crate::simd::SIMD; + + /// AVX + FMA path for 8-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[cfg(test)] + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_once_8_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x8::load_unaligned(x.as_ptr()); + let yv = f32x8::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX + FMA path for 16-lane cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[cfg(any(test, not(target_feature = "avx2")))] + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_once_16_avx_fma(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x16::load_unaligned(x.as_ptr()); + let yv = f32x16::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX-only path for 8-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge. + #[cfg(test)] + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn cosine_once_8_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x8::load_unaligned(x.as_ptr()); + let yv = f32x8::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX-only path for 16-lane cosine (no FMA): body unchanged from AVX2 path; gated on Sandy/Ivy Bridge. + #[cfg(any(test, not(target_feature = "avx2")))] + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn cosine_once_16_avx(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = f32x16::load_unaligned(x.as_ptr()); + let yv = f32x16::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// AVX-512 path for 8-lane cosine: masked load into a `__m512` lower half, reduce. + #[cfg(any(test, target_feature = "avx2"))] + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_once_8_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + // mask 0x00FF: load the lower 8 f32 lanes, zero the upper 8. + let mask: __mmask16 = 0x00FF; + let xv = _mm512_maskz_loadu_ps(mask, x.as_ptr()); + let yv = _mm512_maskz_loadu_ps(mask, y.as_ptr()); + let xy = _mm512_mul_ps(xv, yv); + let y2 = _mm512_mul_ps(yv, yv); + let xy_sum = _mm512_reduce_add_ps(xy); + let y2_sum = _mm512_reduce_add_ps(y2); + 1.0 - xy_sum / x_norm / y2_sum.sqrt() + } + + /// AVX-512 path for 16-lane cosine: single full-width `__m512` load (16 f32 fits one `zmm`). + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_once_16_avx512(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = _mm512_loadu_ps(x.as_ptr()); + let yv = _mm512_loadu_ps(y.as_ptr()); + let xy = _mm512_mul_ps(xv, yv); + let y2 = _mm512_mul_ps(yv, yv); + let xy_sum = _mm512_reduce_add_ps(xy); + let y2_sum = _mm512_reduce_add_ps(y2); + 1.0 - xy_sum / x_norm / y2_sum.sqrt() + } + } + + #[cfg(not(target_arch = "x86_64"))] + #[inline] + fn cosine_once_8_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = unsafe { f32x8::load_unaligned(x.as_ptr()) }; + let yv = unsafe { f32x8::load_unaligned(y.as_ptr()) }; + let y2 = yv * yv; + let xy = xv * yv; 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() } -} -impl Cosine for f32 { + #[cfg(not(target_arch = "x86_64"))] #[inline] - fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 { - let dim = x.len(); - let unrolled_len = dim / 16 * 16; - let mut y_norm16 = f32x16::zeros(); - let mut xy16 = f32x16::zeros(); - for i in (0..unrolled_len).step_by(16) { - unsafe { - let x = f32x16::load_unaligned(x.as_ptr().add(i)); - let y = f32x16::load_unaligned(other.as_ptr().add(i)); - xy16.multiply_add(x, y); - y_norm16.multiply_add(y, y); + fn cosine_once_16_other(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + let xv = unsafe { f32x16::load_unaligned(x.as_ptr()) }; + let yv = unsafe { f32x16::load_unaligned(y.as_ptr()) }; + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + + /// Batch-level SIMD helper used for direct AVX/FMA parity tests. + #[cfg(all(target_arch = "x86_64", test))] + #[target_feature(enable = "avx,fma")] + pub(super) unsafe fn cosine_batch_avx_fma( + x: &[f32], + x_norm: f32, + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + match dimension { + 8 => { + let x_values = unsafe { f32x8::load_unaligned(x.as_ptr()) }; + output + .iter_mut() + .zip(batch.chunks_exact(8)) + .for_each(|(distance, y)| { + let y_values = unsafe { f32x8::load_unaligned(y.as_ptr()) }; + let y2 = y_values * y_values; + let xy = x_values * y_values; + *distance = 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt(); + }); } + 16 => output + .iter_mut() + .zip(batch.chunks_exact(16)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_16_avx_fma(x, x_norm, y) }; + }), + _ => output + .iter_mut() + .zip(batch.chunks_exact(dimension)) + .for_each(|(distance, y)| { + *distance = unsafe { super::f32_x86::cosine_fast_avx_fma(x, x_norm, y) }; + }), } - let aligned_len = dim / 8 * 8; - let mut y_norm8 = f32x8::zeros(); - let mut xy8 = f32x8::zeros(); - for i in (unrolled_len..aligned_len).step_by(8) { - unsafe { - let x = f32x8::load_unaligned(x.as_ptr().add(i)); - let y = f32x8::load_unaligned(other.as_ptr().add(i)); - xy8.multiply_add(x, y); - y_norm8.multiply_add(y, y); + } + + #[cfg(all(target_arch = "x86_64", any(test, target_feature = "avx2")))] + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn cosine_batch_avx512( + x: &[f32], + x_norm: f32, + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + match dimension { + 8 => output + .iter_mut() + .zip(batch.chunks_exact(8)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_8_avx512(x, x_norm, y) }; + }), + 16 => output + .iter_mut() + .zip(batch.chunks_exact(16)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_16_avx512(x, x_norm, y) }; + }), + _ => output + .iter_mut() + .zip(batch.chunks_exact(dimension)) + .for_each(|(distance, y)| { + *distance = unsafe { super::f32_x86::cosine_fast_avx512(x, x_norm, y) }; + }), + } + } + + #[cfg(all(target_arch = "x86_64", test))] + #[target_feature(enable = "avx")] + pub(super) unsafe fn cosine_batch_avx( + x: &[f32], + x_norm: f32, + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + match dimension { + 8 => { + let x_values = unsafe { f32x8::load_unaligned(x.as_ptr()) }; + output + .iter_mut() + .zip(batch.chunks_exact(8)) + .for_each(|(distance, y)| { + let y_values = unsafe { f32x8::load_unaligned(y.as_ptr()) }; + let y2 = y_values * y_values; + let xy = x_values * y_values; + *distance = 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt(); + }); } + 16 => output + .iter_mut() + .zip(batch.chunks_exact(16)) + .for_each(|(distance, y)| { + *distance = unsafe { cosine_once_x86::cosine_once_16_avx(x, x_norm, y) }; + }), + _ => output + .iter_mut() + .zip(batch.chunks_exact(dimension)) + .for_each(|(distance, y)| { + *distance = unsafe { super::f32_x86::cosine_fast_avx(x, x_norm, y) }; + }), } - let y_norm = - y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2); - let xy = - xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]); - 1.0 - xy / x_norm / y_norm.sqrt() } +} + +/// Inlined f32 cosine kernels for builds whose baseline already guarantees +/// AVX2. No `#[target_feature]`, no runtime dispatch: under +/// `target-feature=+avx2,+fma` these compile to AVX2 and inline into the batch +/// loop, so explicitly tuned builds are not taxed by the runtime-dispatch +/// machinery needed below the AVX2 baseline. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +mod f32_baseline { + use super::{dot, f32x8, f32x16, norm_l2}; + use crate::simd::{FloatSimd, SIMD}; #[inline] - fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self { - let dim = x.len(); - let unrolled_len = dim / 16 * 16; - let mut xy16 = f32x16::zeros(); - for i in (0..unrolled_len).step_by(16) { - unsafe { - let x = f32x16::load_unaligned(x.as_ptr().add(i)); - let y = f32x16::load_unaligned(y.as_ptr().add(i)); - xy16.multiply_add(x, y); - } + pub fn cosine_once_8(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + unsafe { + let xv = f32x8::load_unaligned(x.as_ptr()); + let yv = f32x8::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() } - let aligned_len = dim / 8 * 8; - let mut xy8 = f32x8::zeros(); - for i in (unrolled_len..aligned_len).step_by(8) { - unsafe { - let x = f32x8::load_unaligned(x.as_ptr().add(i)); - let y = f32x8::load_unaligned(y.as_ptr().add(i)); - xy8.multiply_add(x, y); + } + + #[inline] + pub fn cosine_once_16(x: &[f32], x_norm: f32, y: &[f32]) -> f32 { + unsafe { + let xv = f32x16::load_unaligned(x.as_ptr()); + let yv = f32x16::load_unaligned(y.as_ptr()); + let y2 = yv * yv; + let xy = xv * yv; + 1.0 - xy.reduce_sum() / x_norm / y2.reduce_sum().sqrt() + } + } + + #[inline] + pub fn cosine_fast(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + unsafe { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut y_norm16 = f32x16::zeros(); + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(other.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + y_norm16.multiply_add(yv, yv); } + let aligned_len = dim / 8 * 8; + let mut y_norm8 = f32x8::zeros(); + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(other.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + let y_norm = y_norm16.reduce_sum() + + y_norm8.reduce_sum() + + norm_l2(&other[aligned_len..]).powi(2); + let xy = xy16.reduce_sum() + + xy8.reduce_sum() + + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() } - let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]); - 1.0 - xy / x_norm / y_norm + } +} + +impl Cosine for f32 { + #[inline] + fn cosine_fast(x: &[Self], x_norm: Self, other: &[Self]) -> f32 { + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback. + cosine_fast_f32_dispatched(x, x_norm, other) + } + + #[inline] + fn cosine_with_norms(x: &[Self], x_norm: Self, y_norm: Self, y: &[Self]) -> Self { + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback. + cosine_with_norms_f32_dispatched(x, x_norm, y_norm, y) } fn cosine_batch<'a>( @@ -266,59 +584,362 @@ impl Cosine for f32 { batch: &'a [Self], dimension: usize, ) -> Box + 'a> { + // Preserve the original `chunks_exact` validation before constructing + // a specialized iterator. + let _ = batch.chunks_exact(dimension); let x_norm = norm_l2(x); - match dimension { - 8 => Box::new( - batch - .chunks_exact(dimension) - .map(move |y| f32::cosine_once::(x, x_norm, y)), - ), - 16 => Box::new( - batch - .chunks_exact(dimension) - .map(move |y| f32::cosine_once::(x, x_norm, y)), - ), - _ => Box::new( - batch - .chunks_exact(dimension) - .map(move |y| Self::cosine_fast(x, x_norm, y)), - ), + // On a build whose baseline already guarantees AVX2, avoid the + // per-vector runtime dispatch + `#[target_feature]` wrapping that taxes + // the tuned path. Dispatch ONCE per batch: AVX-512 hosts get the wide + // kernel; everyone else uses the inlined AVX2 baseline path. The + // runtime-dispatch path below is only compiled/reached when the baseline + // is below AVX2. + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + { + // dim 8/16 always use the inlined AVX2 baseline: AVX-512 gives no + // benefit for such tiny vectors (a masked 512-bit load is slower than + // a plain AVX2 load) and only adds dispatch + eager-collect overhead. + // Only the larger-dim path routes to AVX-512 on capable hosts — that's + // where the wider lanes actually pay off. + match dimension { + 8 => Box::new( + batch + .chunks_exact(8) + .map(move |y| f32_baseline::cosine_once_8(x, x_norm, y)), + ), + 16 => Box::new( + batch + .chunks_exact(16) + .map(move |y| f32_baseline::cosine_once_16(x, x_norm, y)), + ), + _ => { + if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) { + let mut distances = vec![0.0; batch.len() / dimension]; + unsafe { + f32::cosine_batch_avx512(x, x_norm, batch, dimension, &mut distances); + } + Box::new(distances.into_iter()) + } else { + Box::new( + batch + .chunks_exact(dimension) + .map(move |y| f32_baseline::cosine_fast(x, x_norm, y)), + ) + } + } + } + } + + // Sub-AVX2 build: select once. Dimension 8 uses an inlined SSE + // iterator; wider dimensions enter the selected target-feature kernel + // once per vector without materializing the full batch. + #[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] + { + if dimension == 8 && x.len() >= 8 { + // SAFETY: both loads read from the verified eight-element key, + // and x86_64 guarantees SSE. + return Box::new(unsafe { CosineBatch8Iter::new(x, x_norm, batch) }); + } + + let kind = match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if dimension > 16 => { + BatchKind::Avx512 + } + SimdSupport::Avx512 | SimdSupport::Avx512FP16 + if std::is_x86_feature_detected!("fma") => + { + BatchKind::AvxFma + } + SimdSupport::Avx2 | SimdSupport::AvxFma => BatchKind::AvxFma, + SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx => BatchKind::Avx, + _ => BatchKind::Scalar, + }; + Box::new(CosineBatchIter { + key: x, + key_norm: x_norm, + batch, + dimension, + offset: 0, + kind, + }) + } + + // Scalar / non-x86 fallback. + #[cfg(not(target_arch = "x86_64"))] + { + match dimension { + 8 => Box::new( + batch + .chunks_exact(dimension) + .map(move |y| f32::cosine_once_8(x, x_norm, y)), + ), + 16 => Box::new( + batch + .chunks_exact(dimension) + .map(move |y| f32::cosine_once_16(x, x_norm, y)), + ), + _ => Box::new( + batch + .chunks_exact(dimension) + .map(move |y| Self::cosine_fast(x, x_norm, y)), + ), + } + } + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +struct CosineBatchIter<'a> { + key: &'a [f32], + key_norm: f32, + batch: &'a [f32], + dimension: usize, + offset: usize, + kind: BatchKind, +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +struct CosineBatch8Iter<'a> { + key_lo: std::arch::x86_64::__m128, + key_hi: std::arch::x86_64::__m128, + key_norm: f32, + batch: &'a [f32], + offset: usize, +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl<'a> CosineBatch8Iter<'a> { + /// # Safety + /// `key` must contain at least eight elements. + #[inline] + unsafe fn new(key: &[f32], key_norm: f32, batch: &'a [f32]) -> Self { + Self { + key_lo: unsafe { _mm_loadu_ps(key.as_ptr()) }, + key_hi: unsafe { _mm_loadu_ps(key.as_ptr().add(4)) }, + key_norm, + batch, + offset: 0, } } } +/// SSE implementation for the dimension-8 boxed iterator. SSE is guaranteed +/// on x86_64, so this inlines into `Iterator::next` without a target-feature +/// call boundary while wider kernels remain runtime-dispatched. +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +#[inline] +unsafe fn cosine_once_8_sse( + key_lo: std::arch::x86_64::__m128, + key_hi: std::arch::x86_64::__m128, + key_norm: f32, + vector: &[f32], +) -> f32 { + let vector_lo = unsafe { _mm_loadu_ps(vector.as_ptr()) }; + let vector_hi = unsafe { _mm_loadu_ps(vector.as_ptr().add(4)) }; + let xy = unsafe { _mm_add_ps(_mm_mul_ps(key_lo, vector_lo), _mm_mul_ps(key_hi, vector_hi)) }; + let y2 = unsafe { + _mm_add_ps( + _mm_mul_ps(vector_lo, vector_lo), + _mm_mul_ps(vector_hi, vector_hi), + ) + }; + 1.0 - unsafe { hsum128_ps(xy) } / key_norm / unsafe { hsum128_ps(y2) }.sqrt() +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl Iterator for CosineBatch8Iter<'_> { + type Item = f32; + + #[inline] + fn next(&mut self) -> Option { + let end = self.offset + 8; + let vector = self.batch.get(self.offset..end)?; + self.offset = end; + // SAFETY: the constructor preloads a valid key and `vector` contains + // exactly eight elements. x86_64 guarantees SSE. + Some(unsafe { cosine_once_8_sse(self.key_lo, self.key_hi, self.key_norm, vector) }) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl ExactSizeIterator for CosineBatch8Iter<'_> { + #[inline] + fn len(&self) -> usize { + (self.batch.len() - self.offset) / 8 + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +#[inline] +unsafe fn hsum128_ps(values: std::arch::x86_64::__m128) -> f32 { + let sum64 = unsafe { _mm_add_ps(values, _mm_movehl_ps(values, values)) }; + let sum32 = unsafe { _mm_add_ss(sum64, _mm_shuffle_ps(sum64, sum64, 0x55)) }; + unsafe { _mm_cvtss_f32(sum32) } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl Iterator for CosineBatchIter<'_> { + type Item = f32; + + #[inline] + fn next(&mut self) -> Option { + let remaining = &self.batch[self.offset..]; + if remaining.len() < self.dimension { + return None; + } + let vector = &remaining[..self.dimension]; + self.offset += self.dimension; + + let distance = match self.kind { + BatchKind::Scalar => match self.dimension { + 16 => f32::cosine_once_16_scalar(self.key, self.key_norm, vector), + _ => cosine_scalar(self.key, self.key_norm, vector), + }, + // SAFETY: each kind is selected only after runtime detection of + // the kernel's required target features. + BatchKind::Avx => match self.dimension { + 16 => unsafe { + f32::cosine_once_x86::cosine_once_16_avx(self.key, self.key_norm, vector) + }, + _ => unsafe { f32_x86::cosine_fast_avx(self.key, self.key_norm, vector) }, + }, + BatchKind::AvxFma => match self.dimension { + 16 => unsafe { + f32::cosine_once_x86::cosine_once_16_avx_fma(self.key, self.key_norm, vector) + }, + _ => unsafe { f32_x86::cosine_fast_avx_fma(self.key, self.key_norm, vector) }, + }, + BatchKind::Avx512 => match self.dimension { + 16 => unsafe { + f32::cosine_once_x86::cosine_once_16_avx512(self.key, self.key_norm, vector) + }, + _ => unsafe { f32_x86::cosine_fast_avx512(self.key, self.key_norm, vector) }, + }, + }; + Some(distance) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + let len = self.len(); + (len, Some(len)) + } +} + +#[cfg(all(target_arch = "x86_64", not(target_feature = "avx2")))] +impl ExactSizeIterator for CosineBatchIter<'_> { + #[inline] + fn len(&self) -> usize { + (self.batch.len() - self.offset) / self.dimension + } +} + impl Cosine for f64 { #[inline] fn cosine_fast(x: &[Self], x_norm: f32, y: &[Self]) -> f32 { - use crate::simd::f64::{f64x4, f64x8}; - use crate::simd::{FloatSimd, SIMD}; + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 inner kernel on capable hosts, or a portable scalar fallback. + cosine_fast_f64_dispatched(x, x_norm, y) + } +} + +/// Fast cosine for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD +/// primitives in `crate::simd::f64`. +#[inline] +fn cosine_fast_f64_dispatched(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + f64_x86::cosine_fast_avx512(x, x_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + f64_x86::cosine_fast_avx_fma(x, x_norm, y) + }, + SimdSupport::Avx => unsafe { f64_x86::cosine_fast_avx(x, x_norm, y) }, + _ => cosine_scalar(x, x_norm, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_fast_f64_simd_other(x, x_norm, y) + } +} +/// AVX2 + FMA implementation of the f64 cosine_fast kernel. +/// +/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives +/// in `crate::simd::f64` (which use raw AVX intrinsics) inline correctly +/// even when the compile baseline does not have AVX2 enabled. Caller must +/// ensure the host supports AVX2 + FMA. +#[cfg(target_arch = "x86_64")] +mod f64_x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_pd; + use crate::simd::{FloatSimd, SIMD}; + + /// AVX-512 path for f64 fast cosine: 8-wide `__m512d` xy/yy with `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_fast_avx512(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc_xy = _mm512_setzero_pd(); + let mut acc_yy = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let xv = _mm512_loadu_pd(x.as_ptr().add(i)); + let yv = _mm512_loadu_pd(y.as_ptr().add(i)); + acc_xy = _mm512_fmadd_pd(xv, yv, acc_xy); + acc_yy = _mm512_fmadd_pd(yv, yv, acc_yy); + } + + let mut xy = _mm512_reduce_add_pd(acc_xy); + let mut yy = _mm512_reduce_add_pd(acc_yy); + for i in unrolled_len..dim { + xy += x[i] * y[i]; + yy += y[i] * y[i]; + } + + let y_norm_sq = yy as f32; + let xy_f32 = xy as f32; + 1.0 - xy_f32 / x_norm / y_norm_sq.sqrt() + } + + /// AVX + FMA path for f64 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_fast_avx_fma(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { let dim = x.len(); let unrolled_len = dim / 8 * 8; let mut y_norm8 = f64x8::zeros(); let mut xy8 = f64x8::zeros(); for i in (0..unrolled_len).step_by(8) { - unsafe { - let xv = f64x8::load_unaligned(x.as_ptr().add(i)); - let yv = f64x8::load_unaligned(y.as_ptr().add(i)); - xy8.multiply_add(xv, yv); - y_norm8.multiply_add(yv, yv); - } + let xv = f64x8::load_unaligned(x.as_ptr().add(i)); + let yv = f64x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); } let aligned_len = dim / 4 * 4; let mut y_norm4 = f64x4::zeros(); let mut xy4 = f64x4::zeros(); for i in (unrolled_len..aligned_len).step_by(4) { - unsafe { - let xv = f64x4::load_unaligned(x.as_ptr().add(i)); - let yv = f64x4::load_unaligned(y.as_ptr().add(i)); - xy4.multiply_add(xv, yv); - y_norm4.multiply_add(yv, yv); - } + let xv = f64x4::load_unaligned(x.as_ptr().add(i)); + let yv = f64x4::load_unaligned(y.as_ptr().add(i)); + xy4.multiply_add(xv, yv); + y_norm4.multiply_add(yv, yv); } - let tail_y_norm: Self = y[aligned_len..].iter().map(|&v| v * v).sum(); - let tail_xy: Self = x[aligned_len..] + let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum(); + let tail_xy: f64 = x[aligned_len..] .iter() .zip(y[aligned_len..].iter()) .map(|(&a, &b)| a * b) @@ -328,6 +949,336 @@ impl Cosine for f64 { let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32; 1.0 - xy / x_norm / y_norm_sq.sqrt() } + + /// AVX-only path for f64 fast cosine (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration; tail handled inline. + #[target_feature(enable = "avx")] + pub unsafe fn cosine_fast_avx(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + let dim = x.len(); + let aligned_len = dim / 4 * 4; + + let mut acc_xy = _mm256_setzero_pd(); + let mut acc_yy = _mm256_setzero_pd(); + for i in (0..aligned_len).step_by(4) { + let xv = _mm256_loadu_pd(x.as_ptr().add(i)); + let yv = _mm256_loadu_pd(y.as_ptr().add(i)); + acc_xy = _mm256_add_pd(acc_xy, _mm256_mul_pd(xv, yv)); + acc_yy = _mm256_add_pd(acc_yy, _mm256_mul_pd(yv, yv)); + } + + let xy_main = hsum256_pd(acc_xy); + let yy_main = hsum256_pd(acc_yy); + + let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum(); + let tail_xy: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + let y_norm_sq = (yy_main + tail_y_norm) as f32; + let xy = (xy_main + tail_xy) as f32; + 1.0 - xy / x_norm / y_norm_sq.sqrt() + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn cosine_fast_f64_simd_other(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::{FloatSimd, SIMD}; + + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + let mut y_norm8 = f64x8::zeros(); + let mut xy8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + unsafe { + let xv = f64x8::load_unaligned(x.as_ptr().add(i)); + let yv = f64x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + } + let aligned_len = dim / 4 * 4; + let mut y_norm4 = f64x4::zeros(); + let mut xy4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + unsafe { + let xv = f64x4::load_unaligned(x.as_ptr().add(i)); + let yv = f64x4::load_unaligned(y.as_ptr().add(i)); + xy4.multiply_add(xv, yv); + y_norm4.multiply_add(yv, yv); + } + } + let tail_y_norm: f64 = y[aligned_len..].iter().map(|&v| v * v).sum(); + let tail_xy: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + let y_norm_sq = (y_norm8.reduce_sum() + y_norm4.reduce_sum() + tail_y_norm) as f32; + let xy = (xy8.reduce_sum() + xy4.reduce_sum() + tail_xy) as f32; + 1.0 - xy / x_norm / y_norm_sq.sqrt() +} + +/// Cosine for f32 with known norms, runtime-dispatched via `SIMD_SUPPORT` +/// on x86_64 (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses +/// the auto-vectorised scalar loop. +#[inline] +fn cosine_with_norms_f32_dispatched(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + f32_x86::cosine_with_norms_avx512(x, x_norm, y_norm, y) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + f32_x86::cosine_with_norms_avx_fma(x, x_norm, y_norm, y) + }, + SimdSupport::Avx => unsafe { f32_x86::cosine_with_norms_avx(x, x_norm, y_norm, y) }, + _ => cosine_scalar_fast(x, x_norm, y, y_norm), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_with_norms_f32_simd_other(x, x_norm, y_norm, y) + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn cosine_with_norms_f32_simd_other(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + unsafe { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(y.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + } + } + let aligned_len = dim / 8 * 8; + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + unsafe { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + } + } + let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]); + 1.0 - xy / x_norm / y_norm +} + +/// Fast cosine for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// `simd::f32` primitives, unconditionally backed by NEON / LSX. +#[inline] +fn cosine_fast_f32_dispatched(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + f32_x86::cosine_fast_avx512(x, x_norm, other) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { + f32_x86::cosine_fast_avx_fma(x, x_norm, other) + }, + SimdSupport::Avx => unsafe { f32_x86::cosine_fast_avx(x, x_norm, other) }, + _ => cosine_scalar(x, x_norm, other), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + cosine_fast_f32_simd_other(x, x_norm, other) + } +} + +/// AVX2 + FMA implementation of the f32 fast cosine kernel. +/// +/// Lives in a `#[target_feature]`-annotated function so the SIMD primitives +/// in `crate::simd::f32` (which use raw AVX intrinsics) inline correctly +/// even when the compile baseline does not have AVX2 enabled. Caller must +/// ensure the host supports AVX2 + FMA. +#[cfg(target_arch = "x86_64")] +mod f32_x86 { + use std::arch::x86_64::*; + + use super::{dot, f32x8, f32x16, norm_l2}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// AVX + FMA path for f32 fast cosine. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_fast_avx_fma(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut y_norm16 = f32x16::zeros(); + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(other.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + y_norm16.multiply_add(yv, yv); + } + let aligned_len = dim / 8 * 8; + let mut y_norm8 = f32x8::zeros(); + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(other.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + let y_norm = + y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2); + let xy = + xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() + } + + /// AVX-only path for f32 fast cosine (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`/`norm_l2`. + #[target_feature(enable = "avx")] + pub unsafe fn cosine_fast_avx(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let aligned_len = dim / 8 * 8; + + let mut acc_xy = _mm256_setzero_ps(); + let mut acc_yy = _mm256_setzero_ps(); + for i in (0..aligned_len).step_by(8) { + let xv = _mm256_loadu_ps(x.as_ptr().add(i)); + let yv = _mm256_loadu_ps(other.as_ptr().add(i)); + acc_xy = _mm256_add_ps(acc_xy, _mm256_mul_ps(xv, yv)); + acc_yy = _mm256_add_ps(acc_yy, _mm256_mul_ps(yv, yv)); + } + + let xy_main = hsum256_ps(acc_xy); + let yy_main = hsum256_ps(acc_yy); + + let y_norm = yy_main + norm_l2(&other[aligned_len..]).powi(2); + let xy = xy_main + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() + } + + /// AVX-512 path for f32 fast cosine: 16-wide `__m512` xy/yy with `vfmadd231ps` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_fast_avx512(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc_xy = _mm512_setzero_ps(); + let mut acc_yy = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let xv = _mm512_loadu_ps(x.as_ptr().add(i)); + let yv = _mm512_loadu_ps(other.as_ptr().add(i)); + acc_xy = _mm512_fmadd_ps(xv, yv, acc_xy); + acc_yy = _mm512_fmadd_ps(yv, yv, acc_yy); + } + + let mut xy = _mm512_reduce_add_ps(acc_xy); + let mut yy = _mm512_reduce_add_ps(acc_yy); + for i in unrolled_len..dim { + xy += x[i] * other[i]; + yy += other[i] * other[i]; + } + + 1.0 - xy / x_norm / yy.sqrt() + } + + /// AVX-512 path for f32 cosine with known norms: 16-wide `__m512` with `vfmadd231ps` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn cosine_with_norms_avx512(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let xv = _mm512_loadu_ps(x.as_ptr().add(i)); + let yv = _mm512_loadu_ps(y.as_ptr().add(i)); + acc = _mm512_fmadd_ps(xv, yv, acc); + } + + let mut xy = _mm512_reduce_add_ps(acc); + for i in unrolled_len..dim { + xy += x[i] * y[i]; + } + + 1.0 - xy / x_norm / y_norm + } + + /// AVX + FMA path for f32 cosine with known norms. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn cosine_with_norms_avx_fma(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(y.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + } + let aligned_len = dim / 8 * 8; + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(y.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + } + let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &y[aligned_len..]); + 1.0 - xy / x_norm / y_norm + } + + /// AVX-only path for f32 cosine with known norms (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration; tail via trait-routed `dot`. + #[target_feature(enable = "avx")] + pub unsafe fn cosine_with_norms_avx(x: &[f32], x_norm: f32, y_norm: f32, y: &[f32]) -> f32 { + let dim = x.len(); + let aligned_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..aligned_len).step_by(8) { + let xv = _mm256_loadu_ps(x.as_ptr().add(i)); + let yv = _mm256_loadu_ps(y.as_ptr().add(i)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(xv, yv)); + } + + let xy_main = hsum256_ps(acc); + let xy = xy_main + dot(&x[aligned_len..], &y[aligned_len..]); + 1.0 - xy / x_norm / y_norm + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn cosine_fast_f32_simd_other(x: &[f32], x_norm: f32, other: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + let mut y_norm16 = f32x16::zeros(); + let mut xy16 = f32x16::zeros(); + for i in (0..unrolled_len).step_by(16) { + unsafe { + let xv = f32x16::load_unaligned(x.as_ptr().add(i)); + let yv = f32x16::load_unaligned(other.as_ptr().add(i)); + xy16.multiply_add(xv, yv); + y_norm16.multiply_add(yv, yv); + } + } + let aligned_len = dim / 8 * 8; + let mut y_norm8 = f32x8::zeros(); + let mut xy8 = f32x8::zeros(); + for i in (unrolled_len..aligned_len).step_by(8) { + unsafe { + let xv = f32x8::load_unaligned(x.as_ptr().add(i)); + let yv = f32x8::load_unaligned(other.as_ptr().add(i)); + xy8.multiply_add(xv, yv); + y_norm8.multiply_add(yv, yv); + } + } + let y_norm = + y_norm16.reduce_sum() + y_norm8.reduce_sum() + norm_l2(&other[aligned_len..]).powi(2); + let xy = xy16.reduce_sum() + xy8.reduce_sum() + dot(&x[aligned_len..], &other[aligned_len..]); + 1.0 - xy / x_norm / y_norm.sqrt() } /// Fallback non-SIMD implementation @@ -391,8 +1342,9 @@ where .as_any() .downcast_ref::() .ok_or(Error::InvalidArgumentError(format!( - "Unsupported data type {:?}", - to.values().data_type() + "`to` values have data type {}, expected {} to match `from`", + to.values().data_type(), + from.data_type() )))?; let dists = cosine_distance_batch(from.as_slice(), to_values.as_slice(), dimension); @@ -431,18 +1383,38 @@ pub fn cosine_distance_arrow_batch( &to.convert_to_floating_point()?, ), _ => Err(Error::InvalidArgumentError(format!( - "Unsupported data type {:?}", + "`from` has unsupported data type {}", from.data_type() ))), } } +/// Portable scalar reference cosine over f64 inputs. Used by parity tests +/// to compare against every dispatched per-tier inner kernel. Computes +/// `1 - xy / (x_norm * y_norm_sq.sqrt())` in f64 then casts to f32, matching +/// the reduction order of the dispatched kernels. +#[cfg(test)] +fn cosine_fast_scalar(x: &[f64], x_norm: f32, y: &[f64]) -> f32 { + let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum(); + let y_norm_sq: f64 = y.iter().map(|&v| v * v).sum(); + 1.0 - (xy as f32) / x_norm / (y_norm_sq as f32).sqrt() +} + +/// Portable scalar reference cosine when both norms are known. Mirrors +/// `cosine_with_norms_f32_dispatched` for parity testing. +#[cfg(test)] +fn cosine_with_norms_scalar(x: &[f64], x_norm: f32, y_norm: f32, y: &[f64]) -> f32 { + let xy: f64 = x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum(); + 1.0 - (xy as f32) / x_norm / y_norm +} + #[cfg(test)] mod tests { use super::*; use crate::test_utils::{ arbitrary_bf16, arbitrary_f16, arbitrary_f32, arbitrary_f64, arbitrary_vector_pair, + dimension_shard, run_vector_pair_proptest, }; use approx::assert_relative_eq; use num_traits::AsPrimitive; @@ -530,6 +1502,78 @@ mod tests { Ok(()) } + #[rstest::rstest] + fn test_cosine_f32( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + do_cosine_test(&x, &y) + }); + } + + #[rstest::rstest] + fn test_cosine_f64( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f64, dimension_shard(shard), |x, y| { + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + do_cosine_test(&x, &y) + }); + } + + #[rstest::rstest] + fn test_cosine_fast_f32_scalar_simd_parity( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + let simd = ::cosine_fast(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + Ok(()) + }); + } + + #[rstest::rstest] + fn test_cosine_with_norms_f32_scalar_simd_parity( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_with_norms_scalar(&x_f64, x_norm, y_norm, &y_f64); + let simd = ::cosine_with_norms(&x, x_norm, y_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + Ok(()) + }); + } + + #[rstest::rstest] + fn test_cosine_fast_f64_scalar_simd_parity( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f64, dimension_shard(shard), |x, y| { + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let simd = ::cosine_fast(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + Ok(()) + }); + } + proptest::proptest! { #[test] fn test_cosine_f16((x, y) in arbitrary_vector_pair(arbitrary_f16, 4..4048)) { @@ -546,18 +1590,423 @@ mod tests { do_cosine_test(&x, &y)?; } + /// AVX-512-direct parity for the f32 cosine_fast kernel. Early-returns + /// on hosts without AVX-512F so the test stays portable. + #[cfg(target_arch = "x86_64")] #[test] - fn test_cosine_f32((x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)){ + fn test_cosine_fast_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } prop_assume!(norm_l2(&x) > 1e-10); prop_assume!(norm_l2(&y) > 1e-10); - do_cosine_test(&x, &y)?; + let x_norm = norm_l2(&x); + let scalar = cosine_scalar(&x, x_norm, &y); + let avx512 = unsafe { f32_x86::cosine_fast_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); } + /// AVX + FMA-direct parity for the f32 cosine_fast kernel. Covers + /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns + /// on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] #[test] - fn test_cosine_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){ + fn test_cosine_fast_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = cosine_scalar(&x, x_norm, &y); + let avx_fma = unsafe { f32_x86::cosine_fast_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the f32 cosine_fast kernel. Covers + /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on + /// hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = cosine_scalar(&x, x_norm, &y); + let avx = unsafe { f32_x86::cosine_fast_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// AVX-512-direct parity for the f32 cosine_with_norms kernel. + /// Early-returns on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_with_norms_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm); + let avx512 = unsafe { f32_x86::cosine_with_norms_avx512(&x, x_norm, y_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the f32 cosine_with_norms kernel. + /// Covers the AMD Piledriver / Steamroller / FX-7500 tier. + /// Early-returns on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_with_norms_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm); + let avx_fma = unsafe { f32_x86::cosine_with_norms_avx_fma(&x, x_norm, y_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the f32 cosine_with_norms kernel. + /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_with_norms_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let y_norm = norm_l2(&y); + let scalar = cosine_scalar_fast(&x, x_norm, &y, y_norm); + let avx = unsafe { f32_x86::cosine_with_norms_avx(&x, x_norm, y_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// AVX-512-direct parity for the f64 cosine_fast kernel. Early-returns + /// on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f64_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } prop_assume!(norm_l2(&x) > 1e-20); prop_assume!(norm_l2(&y) > 1e-20); - do_cosine_test(&x, &y)?; + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let avx512 = unsafe { f64_x86::cosine_fast_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the f64 cosine_fast kernel. Covers + /// the AMD Piledriver / Steamroller / FX-7500 tier. Early-returns + /// on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f64_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let avx_fma = unsafe { f64_x86::cosine_fast_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the f64 cosine_fast kernel. Covers + /// the Intel Sandy Bridge / Ivy Bridge tier. Early-returns on + /// hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_fast_f64_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-20); + prop_assume!(norm_l2(&y) > 1e-20); + let x_norm = norm_l2(&x); + let scalar = cosine_fast_scalar(&x, x_norm, &y); + let avx = unsafe { f64_x86::cosine_fast_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// Parity check for `cosine_once_8` (despecialised 8-lane width). + /// + /// The `epsilon = 1e-6` clause handles the case where the proptest + /// generator produces inputs with extreme dynamic range (e.g., mixing + /// `1e-43` with `1e7` in the same vector). When the dot product is + /// dominated by one large term and the cosine result is near zero, + /// the f32-precision SIMD path and the f64-precision scalar reference + /// can legitimately differ by more than `max_relative = 1e-3` of the + /// (near-zero) result. The absolute epsilon catches these without + /// masking real bugs (where the absolute error would be > 1e-6). + #[test] + fn test_cosine_once_8_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + let simd = f32::cosine_once_8(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6)); + } + + /// Parity check for `cosine_once_16` (despecialised 16-lane width). + /// See `test_cosine_once_8_scalar_simd_parity` for `epsilon` rationale. + #[test] + fn test_cosine_once_16_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + let simd = f32::cosine_once_16(&x, x_norm, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3, epsilon = 1e-6)); + } + + /// AVX-512-direct parity for the 8-lane cosine_once kernel. Verifies + /// the masked-load (mask 0x00FF) AVX-512 implementation produces + /// the same result as the scalar reference. Early-returns on hosts + /// without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_8_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y); + let avx512 = + unsafe { super::f32::cosine_once_x86::cosine_once_8_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX-512-direct parity for the 16-lane cosine_once kernel. Verifies + /// the full-width `__m512` load implementation produces the same + /// result as the scalar reference. Early-returns on hosts without + /// AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_16_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y); + let avx512 = + unsafe { super::f32::cosine_once_x86::cosine_once_16_avx512(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the 8-lane cosine_once kernel. + /// Covers the AMD Piledriver / Steamroller / FX-7500 tier. + /// Early-returns on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_8_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y); + let avx_fma = + unsafe { super::f32::cosine_once_x86::cosine_once_8_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX + FMA-direct parity for the 16-lane cosine_once kernel. + /// Covers the AMD Piledriver / Steamroller / FX-7500 tier. + /// Early-returns on hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_16_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y); + let avx_fma = + unsafe { super::f32::cosine_once_x86::cosine_once_16_avx_fma(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the 8-lane cosine_once kernel. + /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_8_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 8..9) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_8_scalar(&x, x_norm, &y); + let avx = unsafe { super::f32::cosine_once_x86::cosine_once_8_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + + /// AVX-only-direct parity for the 16-lane cosine_once kernel. + /// Covers the Intel Sandy Bridge / Ivy Bridge tier. Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_once_16_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 16..17) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + prop_assume!(norm_l2(&x) > 1e-10); + prop_assume!(norm_l2(&y) > 1e-10); + let x_norm = norm_l2(&x); + let scalar = super::f32::cosine_once_16_scalar(&x, x_norm, &y); + let avx = unsafe { super::f32::cosine_once_x86::cosine_once_16_avx(&x, x_norm, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-5)); + } + } + + #[rstest::rstest] + #[case::dim_8(8)] + #[case::dim_16(16)] + #[case::dim_40(40)] + fn test_cosine_batch_matches_per_vector(#[case] dimension: usize) { + let num_vectors = 5; + let x: Vec = (0..dimension) + .map(|index| (index % 13) as f32 * 0.25 + 1.0) + .collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|index| (index % 11) as f32 * 0.5 - 2.0) + .collect(); + + let boxed: Vec = f32::cosine_batch(&x, &batch, dimension).collect(); + + assert_eq!(boxed.len(), num_vectors); + for (boxed_distance, vector) in boxed.iter().zip(batch.chunks_exact(dimension)) { + let expected = f32::cosine(&x, vector); + assert_relative_eq!( + *boxed_distance, + expected, + max_relative = 1e-5, + epsilon = 1e-6 + ); + } + } + + /// Asserts a batch-level f32 cosine SIMD kernel matches the scalar + /// `cosine_fast` reference for every vector in a multi-vector batch. Runs + /// each of the kernel's three internal dimension arms (8, 16, and the + /// general `chunks_exact` path). The AVX/FMA helpers are test-only, while + /// the AVX-512 helper is selected only by AVX2-baseline builds, so direct + /// calls cover them under the default x86-64-v2 baseline. + #[cfg(target_arch = "x86_64")] + fn check_cosine_batch_kernel(kernel: unsafe fn(&[f32], f32, &[f32], usize, &mut [f32])) { + for dimension in [8_usize, 16, 40] { + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect(); + let x_norm = norm_l2(&x); + let num_vectors = 3; + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 7) as f32) + 1.0) + .collect(); + + let mut got = vec![0.0; num_vectors]; + unsafe { kernel(&x, x_norm, &batch, dimension, &mut got) }; + assert_eq!(got.len(), num_vectors); + + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + let y_f64: Vec = chunk.iter().map(|&v| v as f64).collect(); + let expected = cosine_fast_scalar(&x_f64, x_norm, &y_f64); + assert_relative_eq!(g, expected, max_relative = 1e-3, epsilon = 1e-6); + } + } + } + + /// AVX + FMA batch kernel parity (AVX2 / AVX+FMA tiers). Runs on any + /// Haswell-or-newer host; early-returns without AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_batch_avx_fma_matches_scalar() { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return; + } + check_cosine_batch_kernel(super::f32::cosine_batch_avx_fma); + } + + /// AVX-only batch kernel parity (Sandy Bridge / Ivy Bridge tier). + /// Early-returns on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_batch_avx_matches_scalar() { + if !std::is_x86_feature_detected!("avx") { + return; + } + check_cosine_batch_kernel(super::f32::cosine_batch_avx); + } + + /// AVX-512 batch kernel parity. Early-returns on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_cosine_batch_avx512_matches_scalar() { + if !std::is_x86_feature_detected!("avx512f") { + return; } + check_cosine_batch_kernel(super::f32::cosine_batch_avx512); } } diff --git a/rust/lance-linalg/src/distance/cosine_u8.rs b/rust/lance-linalg/src/distance/cosine_u8.rs index b2d06d35c31..40b6867b1c3 100644 --- a/rust/lance-linalg/src/distance/cosine_u8.rs +++ b/rust/lance-linalg/src/distance/cosine_u8.rs @@ -15,6 +15,8 @@ use std::sync::OnceLock; +use super::assert_equal_lengths; + /// Intermediate results from the fused u8 cosine kernel: (dot_ab, norm_a², norm_b²). /// /// Separated from the final normalization so SIMD backends can be tested @@ -29,7 +31,7 @@ pub struct CosineAccumulators { /// Portable scalar fused cosine accumulation. #[inline] pub fn cosine_u8_accum_scalar(a: &[u8], b: &[u8]) -> CosineAccumulators { - debug_assert_eq!(a.len(), b.len()); + assert_equal_lengths(a.len(), b.len()); let (mut dot_ab, mut norm_a_sq, mut norm_b_sq) = (0u32, 0u32, 0u32); for (&x, &y) in a.iter().zip(b.iter()) { let (xu, yu) = (x as u32, y as u32); @@ -199,6 +201,10 @@ fn select_backend() -> CosineU8AccumFn { if is_x86_feature_detected!("avx2") { return |a, b| unsafe { x86::cosine_u8_accum_avx2(a, b) }; } + // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy + // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses + // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor + // AVX+FMA provides. } cosine_u8_accum_scalar @@ -207,6 +213,7 @@ fn select_backend() -> CosineU8AccumFn { /// Dispatched fused u8 cosine accumulation. #[inline] fn cosine_u8_accum(a: &[u8], b: &[u8]) -> CosineAccumulators { + assert_equal_lengths(a.len(), b.len()); (DISPATCH.get_or_init(select_backend))(a, b) } @@ -222,6 +229,18 @@ pub fn cosine_u8(a: &[u8], b: &[u8]) -> f32 { mod tests { use super::*; + #[rstest::rstest] + #[case::shorter_right(64, 1)] + #[case::longer_right(1, 64)] + fn rejects_mismatched_lengths(#[case] a_len: usize, #[case] b_len: usize) { + let a = vec![1; a_len]; + let b = vec![1; b_len]; + + assert!(std::panic::catch_unwind(|| cosine_u8_accum_scalar(&a, &b)).is_err()); + assert!(std::panic::catch_unwind(|| cosine_u8_scalar(&a, &b)).is_err()); + assert!(std::panic::catch_unwind(|| cosine_u8(&a, &b)).is_err()); + } + fn fill_random(buf: &mut [u8], seed: &mut u32) { for slot in buf.iter_mut() { *seed = seed.wrapping_mul(1103515245).wrapping_add(12345); diff --git a/rust/lance-linalg/src/distance/dot.rs b/rust/lance-linalg/src/distance/dot.rs index 5903d24e0e5..1f5be08b38b 100644 --- a/rust/lance-linalg/src/distance/dot.rs +++ b/rust/lance-linalg/src/distance/dot.rs @@ -3,6 +3,11 @@ //! Dot product. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use std::arch::x86_64::{_mm256_loadu_ps, _mm256_mul_ps}; use std::iter::Sum; use std::ops::AddAssign; use std::sync::Arc; @@ -13,13 +18,22 @@ use arrow_array::{Array, FixedSizeListArray, Float32Array, cast::AsArray, types: use arrow_schema::DataType; use half::{bf16, f16}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray}; -use lance_core::assume_eq; -use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] -use lance_core::utils::cpu::SimdSupport; +#[allow(unused_imports)] +use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; use num_traits::{AsPrimitive, Num, real::Real}; use crate::Result; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::distance::{BatchIter, BatchKernel, BatchKind, BatchOperation}; +use crate::distance::{assert_batch_layout, assert_equal_lengths}; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::simd::x86::hsum256_ps; /// Default implementation of dot product. /// @@ -68,37 +82,7 @@ pub fn dot(from: &[T], to: &[T]) -> f32 { /// needed on top of the generic [`dot`]. #[inline] pub fn dot_f32(x: &[f32], y: &[f32]) -> f32 { - #[cfg(target_arch = "x86_64")] - { - use lance_core::utils::cpu::SimdSupport; - if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) { - // SAFETY: guarded by the runtime AVX-512 detection above. - return unsafe { dot_f32_avx512(x, y) }; - } - } - dot(x, y) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx512f")] -unsafe fn dot_f32_avx512(x: &[f32], y: &[f32]) -> f32 { - use std::arch::x86_64::*; - debug_assert_eq!(x.len(), y.len()); - let n = x.len(); - let mut acc = _mm512_setzero_ps(); - let mut i = 0usize; - while i + 16 <= n { - let a = _mm512_loadu_ps(x.as_ptr().add(i)); - let b = _mm512_loadu_ps(y.as_ptr().add(i)); - acc = _mm512_fmadd_ps(a, b, acc); - i += 16; - } - let mut sum = _mm512_reduce_add_ps(acc); - while i < n { - sum += x[i] * y[i]; - i += 1; - } - sum + f32::dot(x, y) } /// Negative [Dot] distance. @@ -111,6 +95,25 @@ pub fn dot_distance(from: &[T], to: &[T]) -> f32 { pub trait Dot: Num { /// Dot product. fn dot(x: &[Self], y: &[Self]) -> f32; + + /// Dot product of `x` against each `dimension`-sized vector in `batch`. + /// + /// The default calls [`Dot::dot`] per vector. `f32` overrides it so the + /// SIMD tier is chosen once for the whole batch instead of once per + /// vector — on a build whose baseline already implies AVX2, per-vector + /// dispatch costs more than the kernel it selects. + /// + /// Returns `impl Iterator` rather than a trait object: hot consumers drive + /// this one element at a time, so a `Box` would cost a + /// virtual call per element and an allocation per batch. + fn dot_batch<'a>( + x: &'a [Self], + batch: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + assert_batch_layout(x.len(), batch.len(), dimension); + batch.chunks_exact(dimension).map(move |y| Self::dot(x, y)) + } } #[cfg(feature = "fp16kernels")] @@ -136,6 +139,7 @@ mod bf16_kernel { impl Dot for bf16 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { + assert_equal_lengths(x.len(), y.len()); match *SIMD_SUPPORT { #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))] SimdSupport::Neon => unsafe { @@ -161,6 +165,9 @@ impl Dot for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::dot_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => dot_scalar::(x, y), } } @@ -189,6 +196,7 @@ mod kernel { impl Dot for f16 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { + assert_equal_lengths(x.len(), y.len()); match *SIMD_SUPPORT { #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))] SimdSupport::Neon => unsafe { @@ -203,7 +211,7 @@ impl Dot for f16 { kernel::dot_f16_avx512(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::dot_f16_avx2(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -214,6 +222,9 @@ impl Dot for f16 { SimdSupport::Lsx => unsafe { kernel::dot_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => dot_scalar::(x, y), } } @@ -222,20 +233,486 @@ impl Dot for f16 { impl Dot for f32 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { - dot_scalar::(x, y) + assert_equal_lengths(x.len(), y.len()); + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable + // scalar fallback. Same shape as the f64 sibling and the existing + // u8 distance kernels in `dot_u8.rs`. + dot_f32_dispatched(x, y) + } + + fn dot_batch<'a>( + x: &'a [Self], + batch: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + assert_batch_layout(x.len(), batch.len(), dimension); + // Exactly one arm compiles. Keeping each a tail expression (rather than + // an early `return` guarded by `cfg`) mirrors `dot_f32_dispatched` and + // avoids an unreachable tail on AVX2-baseline builds. + // On an AVX2-baseline build, hoist the tier choice out of the loop, but + // keep the SIMD kernel: the baseline already guarantees avx2+fma, so + // call the AVX+FMA kernel directly rather than re-checking per vector. + // Falling back to the scalar kernel here would lose ~4x at small + // dimensions, which is where batch calls live (PQ sub-vectors are 8 + // wide). + // + // The iterator is a bare `Map`: `Map` is `TrustedLen`, + // so `.collect()` preallocates, and `Map::fold` drives `ChunksExact` in + // one inlined loop. Any wrapper — trait object or enum — loses both. + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "fma" + ))] + { + // See `L2::l2_batch` for f32: below 16 lanes `dot_scalar`'s chunking + // degenerates to a scalar remainder loop, so the explicit AVX kernel + // wins big; above it the autovectorizer is already good and the + // 8-wide kernel can lose, so keep the pre-dispatch kernel exactly. + // + // SAFETY: avx2+fma are enabled for the whole crate by the build + // baseline, so the kernel's `#[target_feature]` contract holds + // statically. + let narrow = dimension <= 16; + batch.chunks_exact(dimension).map(move |y| { + if narrow { + unsafe { x86::dot_f32_avx_fma(x, y) } + } else { + dot_f32_scalar(x, y) + } + }) + } + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + { + dot_batch_f32_runtime_dispatch(x, batch, dimension) + } + #[cfg(not(target_arch = "x86_64"))] + { + // `assert_batch_layout` proves every chunk has the same length as + // `x`, so call the private kernel directly instead of repeating + // the public `Dot::dot` validation for every vector. + batch + .chunks_exact(dimension) + .map(move |y| dot_f32_dispatched(x, y)) + } + } +} + +/// Sub-AVX2 builds: the scalar kernel cannot reach the wide registers, so pick +/// a `#[target_feature]` kernel — once for the batch, not once per vector. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +#[inline] +fn dot_batch_f32_runtime_dispatch<'a>( + x: &'a [f32], + batch: &'a [f32], + dimension: usize, +) -> impl Iterator + 'a { + let (kernel, kind): (BatchKernel, BatchKind) = match *SIMD_SUPPORT { + // AVX-512 has no useful work for an eight-element vector. Retain the + // AVX/FMA kernel used by the former Haswell baseline for PQ dimensions. + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if dimension > 16 => { + (x86::dot_batch_f32_avx512, BatchKind::Avx512) + } + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if std::is_x86_feature_detected!("fma") => { + (x86::dot_batch_f32_avx_fma, BatchKind::AvxFma) + } + SimdSupport::Avx2 | SimdSupport::AvxFma => (x86::dot_batch_f32_avx_fma, BatchKind::AvxFma), + SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx => { + (x86::dot_batch_f32_avx, BatchKind::Avx) + } + _ => (dot_batch_f32_scalar, BatchKind::Scalar), + }; + + // SAFETY: the runtime tier and the explicit FMA check above establish the + // selected kernel's target-feature contract. + unsafe { BatchIter::::new(x, batch, dimension, kernel, kind) } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +struct DotBatch; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl BatchOperation for DotBatch { + #[inline] + fn fold_scalar(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + batch + .chunks_exact(dimension) + .fold(init, |acc, vector| f(acc, dot_f32_scalar(key, vector))) + } + + #[target_feature(enable = "avx")] + unsafe fn fold_avx(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let product = _mm256_mul_ps(key_values, vector_values); + f(acc, unsafe { hsum256_ps(product) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::dot_f32_avx(key, vector) }) + }) + } + + #[target_feature(enable = "avx,fma")] + unsafe fn fold_avx_fma( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let product = _mm256_mul_ps(key_values, vector_values); + f(acc, unsafe { hsum256_ps(product) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::dot_f32_avx_fma(key, vector) }) + }) + } + + #[target_feature(enable = "avx512f")] + unsafe fn fold_avx512( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::dot_f32_avx512(key, vector) }) + }) + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +unsafe fn dot_batch_f32_scalar(x: &[f32], batch: &[f32], dimension: usize, output: &mut [f32]) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = dot_f32_scalar(x, y); + } +} + +/// Dot product for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// auto-vectorised scalar loop. +#[inline] +fn dot_f32_dispatched(x: &[f32], y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f32_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f32_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::dot_f32_avx(x, y) }, + _ => dot_f32_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + dot_f32_scalar(x, y) } } +/// Portable scalar dot product for f32. Used as the x86_64 fallback when no +/// AVX2 is detected, and as the only path on non-x86 architectures. The +/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above. +#[inline] +fn dot_f32_scalar(x: &[f32], y: &[f32]) -> f32 { + dot_scalar::(x, y) +} + impl Dot for f64 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { + assert_equal_lengths(x.len(), y.len()); dot_f64_simd(x, y) } } -/// Explicit SIMD dot product for f64. +/// Dot product for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD +/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX. #[inline] fn dot_f64_simd(x: &[f64], y: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::dot_f64_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::dot_f64_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::dot_f64_avx(x, y) }, + _ => dot_f64_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + dot_f64_simd_other(x, y) + } +} + +/// Portable scalar dot product for f64. Used as the x86_64 fallback when no +/// AVX2 is detected, and exposed for cross-backend parity testing. +#[cfg(target_arch = "x86_64")] +#[inline] +fn dot_f64_scalar(x: &[f64], y: &[f64]) -> f32 { + x.iter().zip(y.iter()).map(|(&a, &b)| a * b).sum::() as f32 +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// Dot product of `x` against every `dimension`-sized vector in `batch`, + /// entering the AVX-512 tier once for the whole batch rather than once per + /// vector. + /// + /// # Safety + /// The host must support AVX-512F. + /// + /// Only compiled for builds whose baseline is below avx2+fma; at or above + /// that baseline `dot_batch` inlines the kernel directly and never runtime- + /// dispatches, so this wrapper would be dead code (see `dot_batch`). + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn dot_batch_f32_avx512( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { dot_f32_avx512(x, y) }; + } + } + + /// As [`dot_batch_f32_avx512`], for the AVX2 and AVX+FMA tiers. + /// + /// # Safety + /// The host must support AVX and FMA. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx,fma")] + pub(super) unsafe fn dot_batch_f32_avx_fma( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { dot_f32_avx_fma(x, y) }; + } + } + + /// As [`dot_batch_f32_avx512`], for the AVX-without-FMA tier. + /// + /// # Safety + /// The host must support AVX. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx")] + pub(super) unsafe fn dot_batch_f32_avx( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { dot_f32_avx(x, y) }; + } + } + + /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn dot_f64_avx512(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm512_loadu_pd(x.as_ptr().add(i)); + let b = _mm512_loadu_pd(y.as_ptr().add(i)); + acc = _mm512_fmadd_pd(a, b, acc); + } + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + (_mm512_reduce_add_pd(acc) + tail) as f32 + } + + /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn dot_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + let a = f64x8::load_unaligned(x.as_ptr().add(i)); + let b = f64x8::load_unaligned(y.as_ptr().add(i)); + acc8.multiply_add(a, b); + } + + let aligned_len = dim / 4 * 4; + let mut acc4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + let a = f64x4::load_unaligned(x.as_ptr().add(i)); + let b = f64x4::load_unaligned(y.as_ptr().add(i)); + acc4.multiply_add(a, b); + } + + let tail: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32 + } + + /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn dot_f64_avx(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 4 * 4; + + let mut acc = _mm256_setzero_pd(); + for i in (0..unrolled_len).step_by(4) { + let a = _mm256_loadu_pd(x.as_ptr().add(i)); + let b = _mm256_loadu_pd(y.as_ptr().add(i)); + acc = _mm256_add_pd(acc, _mm256_mul_pd(a, b)); + } + + // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes. + let lo = _mm256_castpd256_pd128(acc); + let hi = _mm256_extractf128_pd(acc, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + let acc_sum = _mm_cvtsd_f64(sum64); + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + (acc_sum + tail) as f32 + } + + /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration. + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn dot_f32_avx512(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let a = _mm512_loadu_ps(x.as_ptr().add(i)); + let b = _mm512_loadu_ps(y.as_ptr().add(i)); + acc = _mm512_fmadd_ps(a, b, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + _mm512_reduce_add_ps(acc) + tail + } + + /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn dot_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + acc = _mm256_fmadd_ps(a, b, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + hsum256_ps(acc) + tail + } + + /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge. + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn dot_f32_avx(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(a, b)); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| a * b) + .sum(); + + hsum256_ps(acc) + tail + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn dot_f64_simd_other(x: &[f64], y: &[f64]) -> f32 { use crate::simd::f64::{f64x4, f64x8}; use crate::simd::{FloatSimd, SIMD}; @@ -273,7 +750,8 @@ fn dot_f64_simd(x: &[f64], y: &[f64]) -> f32 { impl Dot for u8 { #[inline] fn dot(x: &[Self], y: &[Self]) -> f32 { - super::dot_u8::dot_u8(x, y) as f32 + assert_equal_lengths(x.len(), y.len()); + super::dot_u8::dot_u8_u64(x, y) as f32 } } @@ -283,9 +761,7 @@ pub fn dot_distance_batch<'a, T: Dot>( to: &'a [T], dimension: usize, ) -> Box + 'a> { - assume_eq!(from.len(), dimension); - assume_eq!(to.len() % dimension, 0); - Box::new(to.chunks_exact(dimension).map(|v| dot_distance(from, v))) + Box::new(T::dot_batch(from, to, dimension).map(|d| 1.0 - d)) } fn do_dot_distance_arrow_batch( @@ -309,10 +785,9 @@ where to.value_type() )))?; - let dists = to_values - .as_slice() - .chunks_exact(dimension) - .map(|v| dot_distance(from.as_slice(), v)); + // Route through `dot_distance_batch` rather than mapping `dot_distance` per + // vector, so this entry point gets the same hoisted dispatch. + let dists = dot_distance_batch(from.as_slice(), to_values.as_slice(), dimension); Ok(Arc::new(Float32Array::new( dists.collect(), @@ -363,10 +838,25 @@ mod tests { use super::*; use crate::test_utils::{ arbitrary_bf16, arbitrary_f16, arbitrary_f32, arbitrary_f64, arbitrary_vector_pair, + dimension_shard, run_vector_pair_proptest, }; use num_traits::{Float, FromPrimitive}; use proptest::prelude::*; + #[test] + fn test_dot_rejects_mismatched_lengths() { + let short = [1.0_f32]; + let long = [1.0_f32, 2.0]; + + assert!(std::panic::catch_unwind(|| dot(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| dot_f32(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| f32::dot(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| dot_distance(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| dot_distance_batch(&short, &long, 2)).is_err()); + assert!(std::panic::catch_unwind(|| dot_distance_batch(&long, &[1.0_f32; 3], 2)).is_err()); + assert!(std::panic::catch_unwind(|| dot_distance_batch::(&[], &[], 0)).is_err()); + } + #[test] fn test_dot_f32_dispatch_matches_scalar() { use approx::assert_relative_eq; @@ -460,6 +950,39 @@ mod tests { Ok(()) } + #[rstest::rstest] + fn test_dot_f32(#[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + do_dot_test(&x, &y) + }); + } + + #[rstest::rstest] + fn test_dot_f64(#[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize) { + run_vector_pair_proptest(arbitrary_f64, dimension_shard(shard), |x, y| { + do_dot_test(&x, &y) + }); + } + + #[rstest::rstest] + fn test_dot_f32_scalar_simd_parity( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let scalar = x_f64 + .iter() + .zip(y_f64.iter()) + .map(|(&a, &b)| a * b) + .sum::() as f32; + let simd = ::dot(&x, &y); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error)); + Ok(()) + }); + } + proptest::proptest! { #[test] fn test_dot_f16((x, y) in arbitrary_vector_pair(arbitrary_f16, 4..4048)) { @@ -471,14 +994,242 @@ mod tests { do_dot_test(&x, &y)?; } + /// Cross-backend parity: scalar fallback must match the dispatched + /// SIMD path within numerical tolerance. Exercises `dot_f64_scalar` + /// directly so the runtime fallback is exercised even on AVX2-capable + /// CI hosts. + #[cfg(target_arch = "x86_64")] #[test] - fn test_dot_f32((x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)){ - do_dot_test(&x, &y)?; + fn test_dot_f64_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + let scalar = dot_f64_scalar(&x, &y); + let simd = dot_f64_simd(&x, &y); + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, simd, epsilon = max_error)); } + /// AVX-512-direct parity for f32: explicitly compares the scalar + /// fallback against the native f32 AVX-512 inner kernel on + /// AVX-512F-capable hosts. Early-returns on hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] #[test] - fn test_dot_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){ - do_dot_test(&x, &y)?; + fn test_dot_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = dot_f32_scalar(&x, &y); + let avx512 = unsafe { x86::dot_f32_avx512(&x, &y) }; + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error)); + } + + /// AVX + FMA-direct parity for the f32 dot kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = dot_f32_scalar(&x, &y); + let avx_fma = unsafe { x86::dot_f32_avx_fma(&x, &y) }; + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error)); + } + + /// AVX-only-direct parity for the f32 dot kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without + /// AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = dot_f32_scalar(&x, &y); + let avx = unsafe { x86::dot_f32_avx(&x, &y) }; + let x_f64: Vec = x.iter().map(|&v| v as f64).collect(); + let y_f64: Vec = y.iter().map(|&v| v as f64).collect(); + let max_error = max_error::(&x_f64, &y_f64); + prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error)); + } + + /// AVX-512-direct parity: explicitly compares the scalar fallback + /// against the native AVX-512 inner kernel on AVX-512F-capable hosts + /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on + /// hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = dot_f64_scalar(&x, &y); + let avx512 = unsafe { x86::dot_f64_avx512(&x, &y) }; + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, avx512, epsilon = max_error)); + } + + /// AVX + FMA-direct parity for the f64 dot kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = dot_f64_scalar(&x, &y); + let avx_fma = unsafe { x86::dot_f64_avx_fma(&x, &y) }; + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, avx_fma, epsilon = max_error)); + } + + /// AVX-only-direct parity for the f64 dot kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier (AVX without FMA). Early-returns + /// on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_dot_f64_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = dot_f64_scalar(&x, &y); + let avx = unsafe { x86::dot_f64_avx(&x, &y) }; + let max_error = max_error::(&x, &y); + prop_assert!(approx::relative_eq!(scalar, avx, epsilon = max_error)); + } + } + + /// `dot_batch` must agree with the per-vector `dot` it replaced, on every + /// build: AVX2-baseline, hoisted-dispatch, and portable fallback all + /// funnel through here. + #[rstest::rstest] + #[case::dim_8(8)] + #[case::dim_16(16)] + #[case::dim_32(32)] + #[case::dim_1024(1024)] + fn test_dot_batch_f32_matches_per_vector_dot(#[case] dimension: usize) { + let num_vectors = 5; + let x: Vec = (0..dimension) + .map(|i| ((i % 13) as f32) * 0.25 + 1.0) + .collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 11) as f32) * 0.5 - 2.0) + .collect(); + + let got: Vec = f32::dot_batch(&x, &batch, dimension).collect(); + let want: Vec = batch + .chunks_exact(dimension) + .map(|y| f32::dot(&x, y)) + .collect(); + + assert_eq!(got.len(), num_vectors); + for (g, w) in got.iter().zip(want.iter()) { + assert!( + approx::relative_eq!(g, w, epsilon = 1e-4), + "dim {dimension}: batch {g} != per-vector {w}" + ); + } + } + + /// `dot_distance_batch` still yields `1.0 - dot`, unchanged by the hoist. + #[test] + fn test_dot_distance_batch_preserves_distance_semantics() { + let dimension = 32; + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.1).collect(); + let batch: Vec = (0..dimension * 3).map(|i| (i as f32) * 0.05).collect(); + + let got: Vec = dot_distance_batch(&x, &batch, dimension).collect(); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + assert!(approx::relative_eq!( + g, + 1.0 - f32::dot(&x, chunk), + epsilon = 1e-5 + )); + } + } + + /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2 + /// builds or AVX-512 hosts, so call them directly to cover them. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + fn check_dot_batch_kernel(kernel: BatchKernel) { + for dimension in [8_usize, 16, 40] { + let num_vectors = 3; + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 7) as f32) + 1.0) + .collect(); + + let mut got = vec![0.0; num_vectors]; + unsafe { kernel(&x, &batch, dimension, &mut got) }; + assert_eq!(got.len(), num_vectors); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + let want = dot_scalar::(&x, chunk); + assert!( + approx::relative_eq!(g, want, epsilon = 1e-4), + "dim {dimension}: kernel {g} != scalar {want}" + ); + } + } + } + + // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds + // (see `x86::dot_batch_f32_avx512`), so gate their tests the same way. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_dot_batch_avx_fma_matches_scalar() { + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } + check_dot_batch_kernel(x86::dot_batch_f32_avx_fma); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_dot_batch_avx_matches_scalar() { + if !std::is_x86_feature_detected!("avx") { + return; + } + check_dot_batch_kernel(x86::dot_batch_f32_avx); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_dot_batch_avx512_matches_scalar() { + if !std::is_x86_feature_detected!("avx512f") { + return; } + check_dot_batch_kernel(x86::dot_batch_f32_avx512); } } diff --git a/rust/lance-linalg/src/distance/dot_f16.rs b/rust/lance-linalg/src/distance/dot_f16.rs new file mode 100644 index 00000000000..0fad2c27f88 --- /dev/null +++ b/rust/lance-linalg/src/distance/dot_f16.rs @@ -0,0 +1,1299 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Batched f16 dot product with an optional AMX-FP16 tile backend. +//! +//! Used by IVF over an fp16 column under dot distance: [`dot_f16_batch_16`] +//! scores one query against 16 centroids when choosing the partitions to probe, +//! and the GEMM behind [`PackedCentroidsF16`] assigns vectors to partitions +//! while an index is built. +//! +//! [`dot_f16_batch_16`] returns the 16 raw dot products `Σ query·candidate` +//! (same value convention as [`crate::distance::dot()`]); the caller applies the +//! `1.0 - dot` distance wrapping. On Linux/x86_64 hosts with AMX-FP16 it +//! dispatches to a single tile pass; everywhere else (and on any AMX +//! unavailability) it falls back to 16 independent [`crate::distance::dot()`] +//! calls, which are bit-identical to the per-vector scalar path. +//! +//! Two gates cover all of this, and they are deliberately separate: +//! [`amx_fp16_supported`] answers whether the tile instructions can run here at +//! all, and [`amx_fp16_available`] adds the `LANCE_DISABLE_AMX` kill switch on +//! top. The kernels here are guarded by the former so tests can always reach +//! them; callers routing production work consult the latter. AMX is on by +//! default — the switch exists for A/B measurement and for getting the previous +//! path back without a rebuild. +//! +//! Unlike integer AMX kernels this is floating point: the AMX and fallback paths +//! are **not** bit-for-bit identical (tile accumulation order rounds +//! differently), but both accumulate products in f32 and agree to within fp16 +//! precision — a relative error on the order of 1e-4, far below fp16's own +//! representational error, so recall is unaffected. + +use half::f16; + +use crate::distance::dot::dot; + +/// Batched f16 dot product: the raw dot products of one `query` against the +/// first `len` of 16 `candidates`, in order. Every candidate slice must have the +/// same length as `query`, and `len` must be in `1..=16`. +/// +/// A batch is shorter than 16 only in the last group of a sweep, when the +/// centroid count is not a multiple of 16. `len` keeps the `16 - len` padding +/// rows out of the kernel's staging copy; with at most one short group per +/// sweep that saves much less than it would for a caller whose batches were +/// usually partial. Lanes `len..16` are returned as `0` rather than left +/// unspecified, so both the AMX and fallback paths agree exactly on what a +/// caller that reads past `len` sees. +/// +/// Safe to call unconditionally: the caller never needs to know whether AMX is +/// present. The function panics if any candidate has a different length from +/// `query`, or if `len` is out of range, rather than allowing a malformed batch +/// to reach the FFI kernel. See the module docs for the accuracy contract. +#[inline] +pub fn dot_f16_batch_16(query: &[f16], candidates: &[&[f16]; 16], len: usize) -> [f32; 16] { + assert!( + candidates + .iter() + .all(|candidate| candidate.len() == query.len()), + "all candidate vectors must have the same length as query" + ); + assert!( + (1..=16).contains(&len), + "batch length must be in 1..=16, got {len}" + ); + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + { + // AMX only earns its tile-config overhead once there is at least one + // full 32-wide pass; for tiny dims the fallback is cheaper anyway. + if query.len() >= 32 && crate::simd::amx_fp16::amx_supported() { + return unsafe { crate::simd::amx_fp16::dot_f16_batch_16_amx(query, candidates, len) }; + } + } + dot_f16_batch_16_fallback(query, candidates, len) +} + +/// Fallback for [`dot_f16_batch_16`]: `len` independent [`crate::distance::dot()`] +/// calls — bit-identical to the per-vector scalar path (both go through +/// `f16::dot`) — and `0` for the remaining lanes, matching what the kernel +/// leaves there. Exposed separately so tests can exercise it regardless of host. +#[inline] +pub(crate) fn dot_f16_batch_16_fallback( + query: &[f16], + candidates: &[&[f16]; 16], + len: usize, +) -> [f32; 16] { + std::array::from_fn(|i| { + if i < len { + dot(query, candidates[i]) + } else { + 0.0 + } + }) +} + +/// Centroids pre-packed into the layout the AMX-FP16 GEMM reads its B operand +/// in, together with the scoring entry point that consumes them. +/// +/// This type exists unconditionally, and construction is the only gate: +/// [`PackedCentroidsF16::new`] returns `None` on a build or host without the +/// kernel. Callers in other crates cannot see lance-linalg's `kernel_support` +/// cfg, so an `Option` at run time is the only form of the gate they can branch +/// on to keep their own fallback path. +/// +/// Packing costs `O(k * dim)` and is done once here rather than per block of +/// vectors, where it would outweigh the GEMM it feeds. +pub struct PackedCentroidsF16(Packed); + +/// [`PackedCentroidsF16`]'s payload — and the reason that type needs no `cfg` +/// of its own: without the kernel this is uninhabited, so `PackedCentroidsF16` +/// is too. `new` provably cannot return `Some`, which is what lets the methods +/// discharge their bodies against a value that cannot exist instead of carrying +/// a fallback implementation that could never run. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +struct Packed { + /// Zero-padded `[n_padded, dim]` centroids, row-major and tight. Held + /// because the kernel reads the unpacked centroids directly for the + /// `dim % 32` tail dims, which are not part of the packed layout. + centroids: Vec, + /// `centroids` in the kernel's VNNI B-tile order. + packed: Vec, + n_padded: usize, + dim: usize, +} + +#[cfg(not(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +)))] +enum Packed {} + +/// How many elements a buffer must hold for `m` rows of `row_len`, `stride` +/// apart — `(m - 1) * stride + row_len` — or `None` if that overflows `usize`. +/// +/// The checked arithmetic is the point. Written inline, the expression wraps +/// silently in release builds, and a wrapped requirement is *small*, so it +/// satisfies the very length check it was computed for: `m = 32`, +/// `stride = 595_056_260_442_243_601`, `row_len = 32` wraps to 47, admitting a +/// 47-element slice into a kernel that then strides `data + i * stride` past its +/// end. Every caller here is a safe function guarding an FFI boundary, so an +/// overflow has to be rejected rather than folded into a comparison. +/// +/// Zero rows need zero elements. Handling that here rather than leaving it to +/// the caller keeps the function total: `m - 1` would underflow, which panics in +/// debug builds and wraps to `usize::MAX` in release ones — the same class of +/// silent wrap this function exists to prevent. +pub(crate) fn strided_len(m: usize, stride: usize, row_len: usize) -> Option { + let Some(last_row) = m.checked_sub(1) else { + return Some(0); + }; + last_row.checked_mul(stride)?.checked_add(row_len) +} + +impl PackedCentroidsF16 { + /// Packs `n` row-major `dim`-dimensional `centroids` for repeated scoring. + /// + /// `None` means the GEMM is unavailable — this build has no kernel, this + /// host cannot run it (both are [`amx_fp16_supported`]), or the shape is + /// empty — and the caller must keep using its own path. There is no partial + /// mode. Whether an operator has taken the AMX paths out of service is a + /// separate question, answered by [`amx_fp16_available`] at the caller's + /// routing decision, so that tests can build one of these regardless. + /// + /// `n` is rounded up to a multiple of 32 with zero centroids, since the + /// kernel blocks its `n` loop by 32 and has no partial-tile path. The + /// padding is visible to [`score`](Self::score)'s output and callers must + /// account for it; see [`num_centroids_padded`](Self::num_centroids_padded). + /// + /// # Panics + /// If `centroids` does not hold exactly `n * dim` values. + pub fn new(centroids: &[f16], n: usize, dim: usize) -> Option { + let expected = n + .checked_mul(dim) + .unwrap_or_else(|| panic!("centroid shape n = {n} x dim = {dim} overflows usize")); + assert_eq!( + centroids.len(), + expected, + "centroids must hold n*dim = {expected} values, got {}", + centroids.len() + ); + if n == 0 || dim == 0 || !amx_fp16_supported() { + return None; + } + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + { + // Same checked-size contract as the length guards below: this + // allocation is what the kernel later reads through a raw pointer, + // so a shape whose padded size is not representable is rejected + // here rather than wrapped into an allocation smaller than the rows + // the kernel will address. `packed_centroids_len` needs no separate + // check -- `(dim / 32) * (n_padded / 16) * 512 <= n_padded * dim` + // for every input, so it cannot overflow once this one holds. + let n_padded = n.checked_next_multiple_of(32).unwrap_or_else(|| { + panic!("padding n = {n} up to a multiple of 32 overflows usize") + }); + let padded_len = n_padded.checked_mul(dim).unwrap_or_else(|| { + panic!("padded centroid shape {n_padded} x dim = {dim} overflows usize") + }); + let mut padded = vec![f16::ZERO; padded_len]; + padded[..centroids.len()].copy_from_slice(centroids); + let mut packed = + Vec::with_capacity(crate::simd::amx_fp16::packed_centroids_len(n_padded, dim)); + crate::simd::amx_fp16::pack_centroids_vnni(&padded, n_padded, dim, &mut packed); + return Some(Self(Packed { + centroids: padded, + packed, + n_padded, + dim, + })); + } + #[allow(unreachable_code)] + None + } + + /// The centroid count [`score`](Self::score) actually writes per row: the + /// `n` given to [`new`](Self::new) rounded up to a multiple of 32. + pub fn num_centroids_padded(&self) -> usize { + self.shape().0 + } + + /// `(padded centroid count, dim)`. The single place the uninhabited-payload + /// build is discharged, so the methods above it read as ordinary code. + fn shape(&self) -> (usize, usize) { + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + { + (self.0.n_padded, self.0.dim) + } + #[cfg(not(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + )))] + { + match self.0 {} + } + } + + /// Scores `m` vectors against every centroid: `out[i * out_stride + j]` is + /// the raw dot product `Σ data_i·centroid_j` — the same value convention as + /// [`crate::distance::dot()`], with no `1.0 - d` distance wrapping. + /// + /// Row `i` of `data` starts at `i * data_stride`, so both buffers may be + /// windows of larger ones. Columns `n..num_centroids_padded()` are the zero + /// padding centroids' scores; they are `0.0` for any finite input, which + /// *beats* a real centroid whose dot product is negative. A caller reducing + /// across a row must therefore stop at its own centroid count. + /// + /// See the module docs for the accuracy contract against the scalar path. + /// + /// # Panics + /// If `m` is not a multiple of 32 (the kernel blocks its `m` loop by 32 and + /// has no partial-tile path), if either stride is too small, or if either + /// slice is too short for the last row its stride reaches. + pub fn score( + &self, + data: &[f16], + m: usize, + data_stride: usize, + out: &mut [f32], + out_stride: usize, + ) { + let (n_padded, dim) = self.shape(); + assert_eq!(m % 32, 0, "m ({m}) must be a multiple of 32"); + assert!( + data_stride >= dim, + "data_stride ({data_stride}) is below dim ({dim})" + ); + assert!( + out_stride >= n_padded, + "out_stride ({out_stride}) is below the padded centroid count ({n_padded})" + ); + if m == 0 { + return; + } + let data_needed = strided_len(m, data_stride, dim).unwrap_or_else(|| { + panic!("m = {m} rows of dim {dim} at stride {data_stride} overflow usize") + }); + assert!( + data.len() >= data_needed, + "data ({}) holds fewer than m = {m} rows of dim {dim} at stride {data_stride}", + data.len() + ); + let out_needed = strided_len(m, out_stride, n_padded).unwrap_or_else(|| { + panic!("m = {m} rows of {n_padded} at stride {out_stride} overflow usize") + }); + assert!( + out.len() >= out_needed, + "out ({}) holds fewer than m = {m} rows of {n_padded} at stride {out_stride}", + out.len() + ); + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + // SAFETY: the kernel's preconditions are exactly the asserts above plus + // AMX availability. `n_padded % 32 == 0` and the packing / length + // agreement between `packed`, `centroids`, `n_padded` and `dim` hold by + // construction in `new`, which also proved `amx_fp16_supported()` — a + // process-wide, monotonic property (CPUID plus a one-time, idempotent + // XTILEDATA grant), so it cannot have lapsed since. + unsafe { + crate::simd::amx_fp16::dot_f16_gemm_amx( + data, + m, + data_stride, + &self.0.packed, + &self.0.centroids, + n_padded, + dim, + out, + out_stride, + ); + } + } +} + +/// Whether this process *can* execute the AMX-FP16 kernels at all. True when +/// all of the following hold: +/// +/// - the kernel was compiled in, i.e. `build.rs` found a C compiler accepting +/// `-mamx-fp16` (clang >= 16 or gcc >= 13) and set `kernel_support`; +/// - the target is Linux/x86_64; +/// - the CPU reports both the amx-tile and the amx-fp16 CPUID bits; +/// - the one-time XTILEDATA `arch_prctl` grant succeeded. +/// +/// This is the safety question — without the `arch_prctl` grant the first tile +/// instruction would SIGILL — so no tile instruction in this module runs until +/// it holds. It deliberately ignores [`LANCE_DISABLE_AMX`][amx_fp16_available], +/// so kernel-level tests can exercise the kernels on any host that can run them +/// even when an operator has turned the production paths off. +/// +/// Callers choosing between the AMX path and their fallback want +/// [`amx_fp16_available`] instead. +/// +/// Evaluated once and cached by the hardware probe underneath; nothing is +/// recomputed per call. +pub fn amx_fp16_supported() -> bool { + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + { + return crate::simd::amx_fp16::amx_supported(); + } + #[allow(unreachable_code)] + false +} + +/// Whether production work should be routed onto the AMX-FP16 kernels: this +/// host can run them ([`amx_fp16_supported`]) and no operator has turned them +/// off. +/// +/// **AMX is on by default.** Dispatch is a run-time decision made from CPU +/// capability alone, so a host with the silicon uses it with nothing to enable — +/// there is no Cargo feature and no opt-in variable. `LANCE_DISABLE_AMX` is the +/// escape hatch for the cases where a capability probe is not the whole story: +/// A/B measurement, and an operator who needs the previous code path back +/// without rebuilding. Set it to `1`, `true` or `on` (case-insensitive, +/// surrounding whitespace ignored) to take the AMX paths out of service; every +/// other value, and an unset variable, leave them in. +/// +/// Note this changes *which algorithm* an index build uses, not just how fast it +/// runs: without the GEMM, partition assignment falls back to an approximate +/// graph lookup (see `lance_index`'s `prefers_flat_amx_assignment`). Two indexes +/// built on either side of this variable are not interchangeable. +pub fn amx_fp16_available() -> bool { + !amx_fp16_disabled() && amx_fp16_supported() +} + +/// The `LANCE_DISABLE_AMX` kill switch on its own, read once and cached. Cached +/// because the routing decisions that consult it run per block of vectors, and +/// because a build that flipped behaviour halfway through would be far harder to +/// reason about than one that reads the environment at startup. +fn amx_fp16_disabled() -> bool { + static DISABLED: std::sync::OnceLock = std::sync::OnceLock::new(); + *DISABLED.get_or_init(|| { + std::env::var("LANCE_DISABLE_AMX").is_ok_and(|value| is_amx_disable_value(&value)) + }) +} + +/// The accepted spellings of "off". Anything else — including `0`, `false` and +/// the empty string — leaves AMX enabled, because an unrecognised value must not +/// silently disable a path the operator did not clearly ask to disable. +fn is_amx_disable_value(value: &str) -> bool { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "on" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + /// Dims covering: < 32 (no tile pass, pure fallback), exactly 32 (one pass, + /// no tail), non-multiples of 32 (exercise the AMX tail), and larger dims + /// (multiple passes). + const BATCH_DIMS: &[usize] = &[ + 1, 7, 31, 32, 33, 47, 64, 96, 100, 127, 128, 200, 256, 384, 768, 1000, 1536, + ]; + + fn make_batch(dim: usize, rng: &mut StdRng) -> (Vec, Vec>) { + let gen_vec = |rng: &mut StdRng| -> Vec { + (0..dim) + .map(|_| f16::from_f32(rng.random_range(-1.0f32..1.0))) + .collect() + }; + let query = gen_vec(rng); + let candidates = (0..16).map(|_| gen_vec(rng)).collect(); + (query, candidates) + } + + /// f32-accumulated reference dot product — the semantic contract both the + /// AMX and fallback paths approximate. `f16::dot` itself accumulates in f32. + fn ref_dot_f32(query: &[f16], cand: &[f16]) -> f32 { + query + .iter() + .zip(cand.iter()) + .map(|(&q, &c)| q.to_f32() * c.to_f32()) + .sum() + } + + /// Relative-error tolerance justification: fp16 carries ~11 bits of mantissa + /// (~3 decimal digits). The AMX path and the f32 reference differ only in + /// summation order of f32-widened products, so the error is many orders + /// tighter than fp16's own representational error; 5e-3 relative is a very + /// safe bound (observed worst case is ~2e-4). + const REL_TOL: f32 = 5e-3; + + fn assert_close(got: f32, want: f32, ctx: &str) { + let rel = (got - want).abs() / (want.abs() + 1e-6); + assert!( + rel <= REL_TOL || (got - want).abs() <= 1e-3, + "{ctx}: got {got} want {want} rel_err {rel}" + ); + } + + #[test] + fn fallback_matches_reference() { + let mut rng = StdRng::seed_from_u64(0xF16); + for &dim in BATCH_DIMS { + let (query, cands) = make_batch(dim, &mut rng); + let candidates: [&[f16]; 16] = std::array::from_fn(|i| cands[i].as_slice()); + let got = dot_f16_batch_16_fallback(&query, &candidates, 16); + for i in 0..16 { + assert_close( + got[i], + ref_dot_f32(&query, &cands[i]), + &format!("fb dim={dim} i={i}"), + ); + } + } + } + + #[test] + fn dispatch_matches_reference() { + let mut rng = StdRng::seed_from_u64(0xBEEF); + for &dim in BATCH_DIMS { + let (query, cands) = make_batch(dim, &mut rng); + let candidates: [&[f16]; 16] = std::array::from_fn(|i| cands[i].as_slice()); + let got = dot_f16_batch_16(&query, &candidates, 16); + for i in 0..16 { + assert_close( + got[i], + ref_dot_f32(&query, &cands[i]), + &format!("disp dim={dim} i={i}"), + ); + } + } + } + + #[test] + #[should_panic(expected = "all candidate vectors must have the same length")] + fn rejects_mismatched_candidate_length() { + let query = vec![f16::from_f32(1.0); 32]; + let short = vec![f16::from_f32(1.0); 31]; + let candidates: [&[f16]; 16] = std::array::from_fn(|i| { + if i == 0 { + short.as_slice() + } else { + query.as_slice() + } + }); + let _ = dot_f16_batch_16(&query, &candidates, 16); + } + + /// A live lane must be bit-identical whatever `len` the batch was issued at. + /// Shortening the batch changes only how many rows are gathered, never a + /// row's contents nor the order the tile passes accumulate them, so anything + /// less than bit-exact here would mean a staging offset moved with `len` — + /// exactly the bug a tolerance would hide. Lanes past `len` must read 0 on + /// both paths: dispatch is host-dependent, so a caller must not be able to + /// tell which one ran. + #[test] + fn partial_len_matches_full_batch_and_zeroes_the_rest() { + let mut rng = StdRng::seed_from_u64(0x1EE); + for &dim in BATCH_DIMS { + let (query, cands) = make_batch(dim, &mut rng); + let candidates: [&[f16]; 16] = std::array::from_fn(|i| cands[i].as_slice()); + let full = dot_f16_batch_16(&query, &candidates, 16); + let full_fb = dot_f16_batch_16_fallback(&query, &candidates, 16); + for len in 1..=16 { + let got = dot_f16_batch_16(&query, &candidates, len); + let got_fb = dot_f16_batch_16_fallback(&query, &candidates, len); + for i in 0..len { + assert_eq!( + got[i].to_bits(), + full[i].to_bits(), + "dim={dim} len={len} i={i}: {} vs {}", + got[i], + full[i] + ); + assert_eq!( + got_fb[i].to_bits(), + full_fb[i].to_bits(), + "fb dim={dim} i={i}" + ); + } + for i in len..16 { + assert_eq!(got[i], 0.0, "dim={dim} len={len}: lane {i} must be 0"); + assert_eq!(got_fb[i], 0.0, "fb dim={dim} len={len}: lane {i} must be 0"); + } + } + } + } + + /// `len` is rejected, never clamped: the kernel indexes its staging buffer + /// by it, and a caller that meant 16 and passed 17 should hear about it. + #[rstest::rstest] + #[case::zero(0)] + #[case::seventeen(17)] + #[should_panic(expected = "batch length must be in 1..=16")] + fn rejects_out_of_range_len(#[case] len: usize) { + let query = vec![f16::from_f32(1.0); 32]; + let candidates: [&[f16]; 16] = std::array::from_fn(|_| query.as_slice()); + let _ = dot_f16_batch_16(&query, &candidates, len); + } + + /// The `LANCE_DISABLE_AMX` truth table, pinned in one place. + /// + /// Which spellings mean "off" is an operator-facing contract, and both ways + /// of getting it wrong are silent. A typo accepted as a kill switch — + /// `LANCE_DISABLE_AMX=disable` — would quietly change which algorithm an + /// index build uses, and the only visible trace would be a recall number + /// nobody was comparing. A deliberate `1` rejected would leave an operator + /// who needs the old path back believing they have it. + /// + /// The asymmetry with the enable direction is deliberate: an unrecognised + /// value leaves AMX **on**, because the default is on and an unparsable + /// request to deviate from it should not be honoured by halves. + #[test] + fn amx_disable_flag_accepts_only_explicit_on() { + for value in ["1", "true", "on", "TRUE", "On", " 1 ", "true\n"] { + assert!(is_amx_disable_value(value), "{value:?} should disable AMX"); + } + for value in ["", " ", "0", "false", "off", "no", "yes", "2", "disable"] { + assert!( + !is_amx_disable_value(value), + "{value:?} should not disable AMX" + ); + } + } + + /// With the kill switch unset, availability is exactly hardware support. + /// + /// This is the "on by default" contract itself: the assertion that would + /// fail if a Cargo feature or an opt-in variable ever crept back in front of + /// the dispatch decision. It is derived rather than hardcoded so it holds + /// identically on a host without AMX and in a build whose toolchain could + /// not compile the kernel. + #[test] + fn amx_is_available_by_default_wherever_it_is_supported() { + if std::env::var_os("LANCE_DISABLE_AMX").is_some() { + return; // the switch is under test elsewhere; respect it here + } + assert_eq!(amx_fp16_available(), amx_fp16_supported()); + } + + /// The length arithmetic guarding the GEMM's FFI boundary must reject a + /// shape it cannot represent, not wrap it into a small number. + /// + /// The first case is the one that made this necessary: unchecked, + /// `(32 - 1) * 595_056_260_442_243_601 + 32` wraps to **47**, so a + /// 47-element slice satisfied a check that meant to demand ~1.8e19 + /// elements — and the kernel then strode `data + i * stride` far past the + /// end of it. A wrapped requirement is dangerous precisely because it comes + /// out *small* enough to pass. + /// + /// Deliberately a plain-function test: it runs on every host, including + /// those where `PackedCentroidsF16` cannot be constructed at all. + #[test] + fn strided_len_rejects_shapes_it_cannot_represent() { + assert_eq!(strided_len(32, 595_056_260_442_243_601, 32), None); + assert_eq!(strided_len(2, usize::MAX, 1), None); + assert_eq!(strided_len(usize::MAX, 2, 0), None); + // Representable shapes still come through, including the degenerate + // single-row case where the stride is never applied. + assert_eq!(strided_len(32, 768, 768), Some(31 * 768 + 768)); + assert_eq!(strided_len(1, usize::MAX, 5), Some(5)); + // Zero rows need zero elements, and must not underflow `m - 1`. + assert_eq!(strided_len(0, usize::MAX, usize::MAX), Some(0)); + } + + /// End to end: the safe `score` wrapper must panic on the overflowing + /// shape rather than hand the undersized slice to the C kernel. + /// + /// `strided_len_rejects_shapes_it_cannot_represent` pins the arithmetic; + /// this pins that `score` actually consults it before the `unsafe` block. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn score_rejects_overflowing_stride_before_ffi() { + use std::panic::{AssertUnwindSafe, catch_unwind}; + + let centroids = vec![f16::ONE; 32 * 32]; + let Some(packed) = PackedCentroidsF16::new(¢roids, 32, 32) else { + return; // no AMX on this host; the arithmetic test above still ran + }; + let data = vec![f16::ZERO; 47]; + let mut out = vec![0f32; 32 * 32]; + + let hook = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); // the panic is the expected result + let result = catch_unwind(AssertUnwindSafe(|| { + packed.score(&data, 32, 595_056_260_442_243_601, &mut out, 32); + })); + std::panic::set_hook(hook); + + assert!( + result.is_err(), + "score accepted a 47-element slice for a stride whose row count overflows usize" + ); + } + + /// On AMX-FP16 hardware, assert the AMX branch is actually selected (not a + /// silent fallback) and agrees with the f32 reference within tolerance. + /// Reaching the end without a SIGILL is itself proof the tile path executed. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn amx_path_is_active_and_close() { + if !crate::simd::amx_fp16::amx_supported() { + return; + } + let mut rng = StdRng::seed_from_u64(0xA11); + let mut worst = 0f32; + for &dim in BATCH_DIMS.iter().filter(|&&d| d >= 32) { + let (query, cands) = make_batch(dim, &mut rng); + let candidates: [&[f16]; 16] = std::array::from_fn(|i| cands[i].as_slice()); + let amx = + unsafe { crate::simd::amx_fp16::dot_f16_batch_16_amx(&query, &candidates, 16) }; + for i in 0..16 { + let want = ref_dot_f32(&query, &cands[i]); + assert_close(amx[i], want, &format!("amx dim={dim} i={i}")); + let rel = (amx[i] - want).abs() / (want.abs() + 1e-6); + worst = worst.max(rel); + } + } + assert!(worst <= REL_TOL, "worst AMX relative error: {worst:.2e}"); + } + + /// Another AMX user on this thread retiring the tile configuration must not + /// break the next kernel call. + /// + /// Regression for a stale-cache bug: the kernels used to remember which + /// configuration they had loaded and skip LDTILECFG when it matched, but that + /// record was private to Lance while LDTILECFG and TILERELEASE are + /// architectural per-logical-processor state. After a foreign TILERELEASE the + /// record still said "SEARCH is live" and the hardware was back in INIT, so + /// the kernel skipped the load and its first tile op raised #UD. Reloading on + /// every entry is the fix; this is what would fail if a cache came back. + /// + /// Both halves of that design are checked here, and only one of them has any + /// other symptom. The reload shows up as the kernel surviving a clobber; the + /// release on exit shows up nowhere but in the tile unit itself, read back at + /// the end. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn kernel_reconfigures_after_foreign_tile_release() { + use crate::simd::amx_fp16::{ + amx_supported, clobber_tile_state_for_test, dot_f16_batch_16_amx, + tile_config_is_live_for_test, + }; + + if !amx_supported() { + return; + } + let mut rng = StdRng::seed_from_u64(0xC10B); + let (query, cands) = make_batch(256, &mut rng); + let candidates: [&[f16]; 16] = std::array::from_fn(|i| cands[i].as_slice()); + + let before = unsafe { dot_f16_batch_16_amx(&query, &candidates, 16) }; + // SAFETY: the `amx_supported()` check above granted XTILEDATA, so + // TILERELEASE is legal here. + unsafe { clobber_tile_state_for_test() }; + // Reaching the end of this call at all is the assertion that matters: + // under the bug it raised #UD and killed the test process. Comparing the + // results additionally catches the quieter variant, where a foreign + // configuration supplies the wrong shapes instead of none. + let after = unsafe { dot_f16_batch_16_amx(&query, &candidates, 16) }; + assert_eq!( + before, after, + "kernel output changed after a foreign TILERELEASE" + ); + + // Everything above only asks "did it crash". That is too weak on its own: + // `lance_amx_tile_ensure` reloading on entry is by itself enough to keep + // results right, so deleting `lance_amx_tile_done`'s TILERELEASE would + // leave every assertion so far green while the tile unit stayed held + // against the next AMX user on this thread. Reading the hardware is what + // catches that. + assert!( + !tile_config_is_live_for_test(), + "a kernel left its tile configuration loaded after returning" + ); + } + + /// The batch-16 kernel's tile shape, pinned byte for byte. + /// + /// The shape is the one thing a refactor of `amx_fp16.c` can change with no + /// visible symptom until it runs on AMX hardware, where a wrong shape is a + /// #UD or wrong results rather than a clean failure. Reading the + /// configuration image back is a way to check it without executing a single + /// tile instruction. + /// + /// It still needs the `amx_supported()` guard, which is easy to mistake for + /// redundant: `lance_amx_tilecfg_image` only fills a 64-byte struct. But + /// `amx_fp16.c` is compiled as one translation unit with + /// `-march=sapphirerapids`, so the compiler may use instructions from that + /// baseline anywhere in the file — entering *any* function in it faults on + /// an older CPU. Under `qemu -cpu Nehalem` that is a SIGILL, which is + /// exactly what the `pre-Haswell SIGILL check` in CI runs. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn search_tile_config_image_is_pinned() { + use crate::simd::amx_fp16::{AMX_CFG_SEARCH, amx_supported, tilecfg_image}; + + if !amx_supported() { + return; + } + + // Image layout per Intel SDM: palette_id, start_row, 14 reserved bytes, + // a u16 colsb[16] array, then a u8 rows[16] array. + const COLSB: usize = 16; + const ROWS: usize = 48; + + let mut want = [0u8; 64]; + want[0] = 1; // palette_id + // C = tmm0: 16 x 1 fp32, fed by three independent (A, B) pairs so three + // TDPFP16PS can be in flight at once: A = tmm1/3/5, each 16 x 32 fp16; + // B = tmm2/4/6, each 16 x 2 fp16. tmm7 is left unconfigured. + for (tmm, colsb) in [ + (0usize, 4u16), + (1, 64), + (2, 4), + (3, 64), + (4, 4), + (5, 64), + (6, 4), + ] { + want[COLSB + tmm * 2..COLSB + tmm * 2 + 2].copy_from_slice(&colsb.to_le_bytes()); + want[ROWS + tmm] = 16; + } + + let got = tilecfg_image(AMX_CFG_SEARCH).expect("search config kind must be known"); + assert_eq!(got, want, "batch-16 tile configuration changed"); + } + + /// An unknown config kind must be rejected rather than answered with a + /// zeroed image: an all-zero (unconfigured) shape #UDs on the first tile op. + /// + /// Guarded for the same reason as the test above: reaching the C side at + /// all requires a CPU that can run its `-march=sapphirerapids` code. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn unknown_tile_config_kind_is_rejected() { + if !crate::simd::amx_fp16::amx_supported() { + return; + } + assert!(crate::simd::amx_fp16::tilecfg_image(-1).is_none()); + } + + // ----------------------------------------------------------------------- + // AMX-FP16 m x n GEMM + // ----------------------------------------------------------------------- + + /// `[m, dim]` vectors and `[n, dim]` centroids, both row-major and tightly + /// packed. Separate rngs per role would make a transposition bug harder to + /// spot, so both come off one stream. + fn make_gemm(m: usize, n: usize, dim: usize, rng: &mut StdRng) -> (Vec, Vec) { + let mut sample = |count: usize| -> Vec { + (0..count) + .map(|_| f16::from_f32(rng.random_range(-1.0f32..1.0))) + .collect() + }; + (sample(m * dim), sample(n * dim)) + } + + /// f32-accumulated reference GEMM — the same semantic contract as + /// [`ref_dot_f32`], extended to every (vector, centroid) pair. + fn ref_gemm( + data: &[f16], + m: usize, + data_stride: usize, + centroids: &[f16], + n: usize, + dim: usize, + ) -> Vec { + let mut out = vec![0f32; m * n]; + for i in 0..m { + let row = &data[i * data_stride..i * data_stride + dim]; + for j in 0..n { + out[i * n + j] = ref_dot_f32(row, ¢roids[j * dim..j * dim + dim]); + } + } + out + } + + /// Pack + run the GEMM kernel, returning an `[m, out_stride]` buffer. + /// + /// The destination starts as NaN rather than 0 so a tile store that never + /// lands (wrong stride, wrong tile index) fails the comparison instead of + /// passing on a coincidentally-correct zero. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + fn run_gemm( + data: &[f16], + m: usize, + data_stride: usize, + centroids: &[f16], + n: usize, + dim: usize, + out_stride: usize, + ) -> Vec { + use crate::simd::amx_fp16::{dot_f16_gemm_amx, pack_centroids_vnni}; + let mut packed = Vec::new(); + pack_centroids_vnni(centroids, n, dim, &mut packed); + let mut out = vec![f32::NAN; m * out_stride]; + unsafe { + dot_f16_gemm_amx( + data, + m, + data_stride, + &packed, + centroids, + n, + dim, + &mut out, + out_stride, + ); + } + out + } + + /// Compare a `[m, out_stride]` kernel result against a tightly-packed + /// `[m, n]` reference. + fn assert_gemm_close( + got: &[f32], + want: &[f32], + m: usize, + n: usize, + out_stride: usize, + ctx: &str, + ) { + for i in 0..m { + for j in 0..n { + assert_close( + got[i * out_stride + j], + want[i * n + j], + &format!("{ctx} [{i}][{j}]"), + ); + } + } + } + + /// The VNNI interleave, checked against the identity it exists to satisfy. + /// + /// This is the one part of the GEMM with no cheap sanity signal: a wrong + /// permutation still produces plausible-looking finite numbers, and on a + /// host without AMX nothing else here can run at all. Asserting the + /// element-for-element mapping keeps the layout pinned everywhere. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn packed_centroid_layout_matches_tile_operand_order() { + use crate::simd::amx_fp16::{pack_centroids_vnni, packed_centroids_len}; + + // Reused across shapes to also pin that packing clears rather than + // appends — a stale prefix would silently offset every B tile. + let mut packed = Vec::new(); + let mut rng = StdRng::seed_from_u64(0x9EC7); + for (n, dim) in [(16usize, 32usize), (32, 64), (48, 100), (32, 31), (32, 768)] { + let centroids: Vec = (0..n * dim) + .map(|_| f16::from_f32(rng.random_range(-1.0f32..1.0))) + .collect(); + pack_centroids_vnni(¢roids, n, dim, &mut packed); + assert_eq!( + packed.len(), + packed_centroids_len(n, dim), + "n={n} dim={dim}" + ); + + for kb in 0..dim / 32 { + for jb in 0..n / 16 { + for k in 0..16 { + for nn in 0..16 { + for p in 0..2 { + let at = ((kb * (n / 16)) + jb) * 512 + k * 32 + nn * 2 + p; + let from = (jb * 16 + nn) * dim + kb * 32 + 2 * k + p; + assert_eq!( + packed[at], centroids[from], + "n={n} dim={dim} kb={kb} jb={jb} k={k} nn={nn} p={p}" + ); + } + } + } + } + } + } + } + + /// GEMM results against the f32 reference across the shapes that exercise + /// each loop boundary: one vs. several 32-row/32-column register blocks, + /// one vs. many k-passes, dims with and without a scalar tail, dims too + /// short for any tile pass at all, and a case with padded row strides on + /// both the input and the output. + /// + /// Dims below 32 matter disproportionately: the kernel skips the tile loop + /// entirely, so `out` is never written by a tile store and the scalar tail + /// has to zero it first. Accumulating onto uninitialized memory instead + /// would still look plausible on a freshly-allocated buffer. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn gemm_matches_reference() { + if !crate::simd::amx_fp16::amx_supported() { + return; + } + let mut rng = StdRng::seed_from_u64(0x6E33); + for &m in &[32usize, 64] { + for &n in &[32usize, 64] { + for &dim in &[1usize, 16, 31, 32, 33, 64, 100, 768, 1000, 1536] { + let (data, centroids) = make_gemm(m, n, dim, &mut rng); + let want = ref_gemm(&data, m, dim, ¢roids, n, dim); + let got = run_gemm(&data, m, dim, ¢roids, n, dim, n); + assert_gemm_close(&got, &want, m, n, n, &format!("gemm m={m} n={n} dim={dim}")); + } + } + } + + // Padded strides: the kernel must address rows by the caller's stride, + // not by `dim` / `n`, so it can score a window of a larger buffer. + let (m, n, dim) = (64usize, 32usize, 100usize); + let (data_stride, out_stride) = (dim + 7, n + 5); + let mut data: Vec = vec![f16::from_f32(f32::MAX); m * data_stride]; + let (tight, centroids) = make_gemm(m, n, dim, &mut rng); + for i in 0..m { + data[i * data_stride..i * data_stride + dim] + .copy_from_slice(&tight[i * dim..(i + 1) * dim]); + } + let want = ref_gemm(&data, m, data_stride, ¢roids, n, dim); + let got = run_gemm(&data, m, data_stride, ¢roids, n, dim, out_stride); + assert_gemm_close(&got, &want, m, n, out_stride, "gemm padded strides"); + } + + /// The safe wrapper, over centroid counts that do and do not divide 32. + /// + /// Padding is the part a caller cannot see and must still reason about: the + /// real centroids have to score exactly as they would unpadded, and the + /// columns beyond them have to be the zeros that make a caller's argmin + /// bound (`< n`) load-bearing rather than decorative. + #[test] + fn packed_centroids_pad_to_the_kernel_block() { + let mut rng = StdRng::seed_from_u64(0x9AD); + for (n, dim) in [(32usize, 64usize), (100, 100), (48, 768)] { + let m = 64; + let (data, centroids) = make_gemm(m, n, dim, &mut rng); + let Some(packed) = PackedCentroidsF16::new(¢roids, n, dim) else { + return; // no AMX-FP16 on this build or host + }; + let n_padded = packed.num_centroids_padded(); + assert_eq!(n_padded, n.next_multiple_of(32), "n={n}"); + + // A wider output stride than the padded width, so a row's tail is + // untouched memory rather than the next row's scores. + let out_stride = n_padded + 3; + let mut out = vec![f32::NAN; m * out_stride]; + packed.score(&data, m, dim, &mut out, out_stride); + + let want = ref_gemm(&data, m, dim, ¢roids, n, dim); + assert_gemm_close(&out, &want, m, n, out_stride, &format!("packed n={n}")); + for i in 0..m { + for j in n..n_padded { + assert_eq!(out[i * out_stride + j], 0.0, "padding n={n} [{i}][{j}]"); + } + } + } + } + + /// The GEMM kernel's tile shape, pinned byte for byte, for the same reason + /// as [`search_tile_config_image_is_pinned`]. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn gemm_tile_config_image_is_pinned() { + use crate::simd::amx_fp16::{AMX_CFG_GEMM, amx_supported, tilecfg_image}; + + if !amx_supported() { + return; + } + + const COLSB: usize = 16; + const ROWS: usize = 48; + + let mut want = [0u8; 64]; + want[0] = 1; // palette_id + // All eight tiles at the architectural maximum 16 x 64 B: four fp32 + // accumulators, two A panels, two VNNI-packed B panels. + for tmm in 0..8usize { + want[COLSB + tmm * 2..COLSB + tmm * 2 + 2].copy_from_slice(&64u16.to_le_bytes()); + want[ROWS + tmm] = 16; + } + + let got = tilecfg_image(AMX_CFG_GEMM).expect("gemm config kind must be known"); + assert_eq!(got, want, "gemm tile configuration changed"); + } + + /// Alternate the two kernels on one thread and check both stay correct. + /// + /// They ask for incompatible tile shapes (7 tiles, four of them 16x4, vs. 8 + /// tiles all 16x64), so each call has to install its own shape over the one + /// the previous call left. That is what this pins: a kernel running against + /// the other one's tile shape reads garbage or #UDs, and only alternating + /// the two shapes can expose it. It says nothing about how the configuration + /// got there — `kernel_reconfigures_after_foreign_tile_release` is the test + /// that pins the reload itself. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn interleaved_search_and_gemm_stay_correct() { + if !crate::simd::amx_fp16::amx_supported() { + return; + } + let mut rng = StdRng::seed_from_u64(0x11E12EA5); + let (m, n, dim) = (32usize, 32usize, 96usize); + for round in 0..20 { + let (query, cands) = make_batch(dim, &mut rng); + let candidates: [&[f16]; 16] = std::array::from_fn(|i| cands[i].as_slice()); + let batch = + unsafe { crate::simd::amx_fp16::dot_f16_batch_16_amx(&query, &candidates, 16) }; + for i in 0..16 { + assert_close( + batch[i], + ref_dot_f32(&query, &cands[i]), + &format!("interleaved batch round={round} i={i}"), + ); + } + + let (data, centroids) = make_gemm(m, n, dim, &mut rng); + let want = ref_gemm(&data, m, dim, ¢roids, n, dim); + let got = run_gemm(&data, m, dim, ¢roids, n, dim, n); + assert_gemm_close( + &got, + &want, + m, + n, + n, + &format!("interleaved gemm round={round}"), + ); + } + } + + /// Both kernels running concurrently on different threads. + /// + /// LDTILECFG is per-logical-processor state, so correctness here rests on + /// each call building its configuration image on its own stack. A shared or + /// `static` image would let one thread's shape reach another's tile ops, + /// which this catches and a single-threaded test cannot. + #[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" + ))] + #[test] + fn concurrent_search_and_gemm_stay_correct() { + if !crate::simd::amx_fp16::amx_supported() { + return; + } + const THREADS: u64 = 8; + std::thread::scope(|scope| { + for t in 0..THREADS { + scope.spawn(move || { + let mut rng = StdRng::seed_from_u64(0xC0FFEE + t); + let dim = 128; + for round in 0..25 { + if t % 2 == 0 { + let (query, cands) = make_batch(dim, &mut rng); + let candidates: [&[f16]; 16] = + std::array::from_fn(|i| cands[i].as_slice()); + let batch = unsafe { + crate::simd::amx_fp16::dot_f16_batch_16_amx(&query, &candidates, 16) + }; + for i in 0..16 { + assert_close( + batch[i], + ref_dot_f32(&query, &cands[i]), + &format!("concurrent batch t={t} round={round} i={i}"), + ); + } + } else { + let (m, n) = (32usize, 32usize); + let (data, centroids) = make_gemm(m, n, dim, &mut rng); + let want = ref_gemm(&data, m, dim, ¢roids, n, dim); + let got = run_gemm(&data, m, dim, ¢roids, n, dim, n); + assert_gemm_close( + &got, + &want, + m, + n, + n, + &format!("concurrent gemm t={t} round={round}"), + ); + } + } + }); + } + }); + } + + /// The two costs a membership-level benchmark cannot separate: packing the + /// centroids, which happens once per call and is amortized over every block + /// of vectors, and the GEMM's throughput as a function of the block height + /// `m` its caller chooses. + /// + /// `m` sets how much f32 scratch one block writes (`m * n_padded * 4` bytes, + /// reported per row), which is what decides whether the reduction that + /// follows reads it out of L2 or out of memory. A caller sizing its blocks + /// from a scratch budget is making exactly this trade, so sweeping `m` here + /// says whether it lands on the right side of it. No knob is needed in the + /// production path for that — `score` takes `m` directly. + /// + /// `#[ignore]` -- run: + /// cargo test -p lance-linalg --release \ + /// packed_centroids_gemm_shape_bench -- --ignored --nocapture + /// Tune with `BENCH_DIMS` / `BENCH_KS` (comma-separated) and `BENCH_SECONDS` + /// (the wall-clock budget each measured point gets). + #[test] + #[ignore] + #[allow(clippy::print_stderr)] + // Without the kernel `PackedCentroidsF16` is uninhabited, so the first + // `expect` below has type `!` and everything after it is provably dead. That + // is the property the type is designed to have; it is not a sign the bench is + // wrong. + #[allow(unreachable_code, unused_variables)] + fn packed_centroids_gemm_shape_bench() { + use std::time::{Duration, Instant}; + + // Multiples of 32 (the kernel's `m` granularity) spanning scratch that + // fits a private L2 up to scratch that cannot. + const BLOCK_ROWS: &[usize] = &[32, 64, 128, 256, 512, 1024, 2048]; + + if !amx_fp16_supported() { + eprintln!("[gemm_shape_bench] skipped: amx_fp16_supported=false on this build or host"); + return; + } + + let env_list = |key: &str, default: &[usize]| -> Vec { + std::env::var(key) + .ok() + .map(|s| s.split(',').filter_map(|t| t.trim().parse().ok()).collect()) + .unwrap_or_else(|| default.to_vec()) + }; + let dims = env_list("BENCH_DIMS", &[768]); + let ks = env_list("BENCH_KS", &[256, 4096]); + let budget = Duration::from_secs_f64( + std::env::var("BENCH_SECONDS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(3.0), + ); + + let mut rng = StdRng::seed_from_u64(0x6E33); + let mut random_f16 = |count: usize| -> Vec { + (0..count) + .map(|_| f16::from_f32(rng.random_range(-1.0f32..1.0))) + .collect() + }; + + eprintln!( + "[gemm_shape_bench] budget={:.1}s amx_fp16_supported=true", + budget.as_secs_f64() + ); + for &dim in &dims { + for &k in &ks { + let centroids = random_f16(k * dim); + + let mut packs = 0usize; + let t0 = Instant::now(); + while t0.elapsed() < budget { + let packed = PackedCentroidsF16::new(¢roids, k, dim); + std::hint::black_box(&packed); + packs += 1; + } + let pack_us = t0.elapsed().as_secs_f64() * 1e6 / packs as f64; + + let packed = PackedCentroidsF16::new(¢roids, k, dim) + .expect("availability was just checked"); + let n_padded = packed.num_centroids_padded(); + eprintln!( + "[gemm_shape_bench] dim={dim} k={k} n_padded={n_padded} pack_calls={packs} pack_us={pack_us:.1}" + ); + + let mut best_vec_per_s = 0f64; + for &m in BLOCK_ROWS { + let data = random_f16(m * dim); + let mut out = vec![0f32; m * n_padded]; + packed.score(&data, m, dim, &mut out, n_padded); // untimed warm-up + + let t1 = Instant::now(); + let mut iters = 0usize; + while t1.elapsed() < budget { + packed.score(&data, m, dim, &mut out, n_padded); + iters += 1; + } + let elapsed = t1.elapsed().as_secs_f64(); + std::hint::black_box(&out); + + let vec_per_s = (iters * m) as f64 / elapsed; + best_vec_per_s = best_vec_per_s.max(vec_per_s); + eprintln!( + // Pairs count the padding columns, since the kernel does + // the work either way — this is its true rate, not the + // caller's useful fraction of it. + "[gemm_shape_bench] m={m:>5} scratch_kb={:>7} iters={iters:>8} vec_per_s={vec_per_s:>12.0} us_per_vec={:>8.4} Gpair_per_s={:>8.2}", + m * n_padded * 4 / 1024, + 1e6 / vec_per_s, + vec_per_s * n_padded as f64 / 1e9, + ); + } + eprintln!( + "[gemm_shape_bench] pack_us={pack_us:.1} buys {:.0} vectors of scoring at the best m: packing is amortized above that", + pack_us * 1e-6 * best_vec_per_s, + ); + } + } + } +} diff --git a/rust/lance-linalg/src/distance/dot_u8.rs b/rust/lance-linalg/src/distance/dot_u8.rs index de5522cddfe..32033f336e4 100644 --- a/rust/lance-linalg/src/distance/dot_u8.rs +++ b/rust/lance-linalg/src/distance/dot_u8.rs @@ -7,6 +7,8 @@ //! vector dimension as a u8 after linearly mapping [min, max] → [0, 255]. //! Distance computation between SQ-encoded vectors reduces to a u8 × u8 //! dot product plus precomputed per-vector scalar terms. +//! The u32 entry points return the low 32 bits, while [`dot_u8_u64`] chunks +//! those kernels to produce the full result used by SQ. //! //! Backends (selected at runtime, best available wins): //! 1. scalar — portable reference, also used for tails @@ -28,14 +30,18 @@ use std::sync::OnceLock; +use super::{U8_U32_ACCUMULATOR_MAX_LEN, assert_equal_lengths}; + /// Portable scalar u8 dot product, also used for SIMD tail elements. +/// +/// The result is the low 32 bits of the exact dot product. Use +/// [`dot_u8_u64`] when the full result is required. #[inline] pub fn dot_u8_scalar(a: &[u8], b: &[u8]) -> u32 { - debug_assert_eq!(a.len(), b.len()); + assert_equal_lengths(a.len(), b.len()); a.iter() .zip(b.iter()) - .map(|(&x, &y)| x as u32 * y as u32) - .sum() + .fold(0, |sum, (&x, &y)| sum.wrapping_add(x as u32 * y as u32)) } #[cfg(target_arch = "x86_64")] @@ -74,7 +80,7 @@ mod x86 { let mut result = _mm_cvtsi128_si32(sum128) as u32; while i < n { - result += a[i] as u32 * b[i] as u32; + result = result.wrapping_add(a[i] as u32 * b[i] as u32); i += 1; } result @@ -110,7 +116,7 @@ mod x86 { let mut result = (biased_dot as i64 + 128 * sum_a) as u32; while i < n { - result += a[i] as u32 * b[i] as u32; + result = result.wrapping_add(a[i] as u32 * b[i] as u32); i += 1; } result @@ -134,21 +140,59 @@ fn select_backend() -> DotU8Fn { if is_x86_feature_detected!("avx2") { return |a, b| unsafe { x86::dot_u8_avx2(a, b) }; } + // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy + // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses + // `vpmaddubsw` / `vpmaddwd` integer ops which neither AVX nor + // AVX+FMA provides. } dot_u8_scalar } /// Dispatched u8 dot product, selecting the best available SIMD backend. +/// +/// The result is the low 32 bits of the exact dot product. Use +/// [`dot_u8_u64`] when the full result is required. #[inline] pub fn dot_u8(a: &[u8], b: &[u8]) -> u32 { + assert_equal_lengths(a.len(), b.len()); (DISPATCH.get_or_init(select_backend))(a, b) } +/// Calculates the exact u8 dot product with a u64 accumulator. +/// +/// This retains the runtime-selected SIMD kernel and widens its result between +/// chunks that are guaranteed to fit in a u32. +/// +/// # Example +/// +/// ``` +/// use lance_linalg::distance::dot_u8::dot_u8_u64; +/// +/// assert_eq!(dot_u8_u64(&[2, 3], &[4, 5]), 23); +/// ``` +#[inline] +pub fn dot_u8_u64(a: &[u8], b: &[u8]) -> u64 { + assert_equal_lengths(a.len(), b.len()); + if a.len() <= U8_U32_ACCUMULATOR_MAX_LEN { + return dot_u8(a, b) as u64; + } + a.chunks(U8_U32_ACCUMULATOR_MAX_LEN) + .zip(b.chunks(U8_U32_ACCUMULATOR_MAX_LEN)) + .map(|(a, b)| dot_u8(a, b) as u64) + .sum() +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn rejects_mismatched_lengths() { + assert!(std::panic::catch_unwind(|| dot_u8_scalar(&[1, 2], &[1])).is_err()); + assert!(std::panic::catch_unwind(|| dot_u8(&[1, 2], &[1])).is_err()); + } + fn fill_random(buf: &mut [u8], seed: &mut u32) { for slot in buf.iter_mut() { *seed = seed.wrapping_mul(1103515245).wrapping_add(12345); @@ -252,4 +296,17 @@ mod tests { assert_eq!(dot_u8_scalar(&a[..n], &b[..n]), n as u32); } } + + #[test] + fn overflow_is_backend_independent_and_wide_result_is_exact() { + let len = U8_U32_ACCUMULATOR_MAX_LEN + 1; + let a = vec![u8::MAX; len]; + let b = vec![u8::MAX; len]; + let exact = u8::MAX as u64 * u8::MAX as u64 * len as u64; + + check_all_backends(&a, &b, "u32 overflow"); + assert_eq!(dot_u8_scalar(&a, &b), exact as u32); + assert_eq!(dot_u8_u64(&a, &b), exact); + assert_eq!(crate::distance::dot::dot::(&a, &b), exact as f32); + } } diff --git a/rust/lance-linalg/src/distance/hamming.rs b/rust/lance-linalg/src/distance/hamming.rs index a6f4b038195..b1a52cd7fbe 100644 --- a/rust/lance-linalg/src/distance/hamming.rs +++ b/rust/lance-linalg/src/distance/hamming.rs @@ -4,10 +4,10 @@ //! Hamming distance. //! //! This module provides hamming distance computation for binary vectors, -//! including SIMD-accelerated pairwise hamming distance for 64-bit hashes. +//! including SIMD-accelerated pairwise hamming distance for binary hashes. use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use arrow_array::builder::{ListBuilder, UInt64Builder}; use arrow_array::cast::AsArray; @@ -21,6 +21,27 @@ use rayon::prelude::*; use crate::{Error, Result}; +/// Schema of a Hamming distance-pair batch. +static DISTANCE_PAIR_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![ + Field::new("row_id_a", DataType::UInt64, false), + Field::new("row_id_b", DataType::UInt64, false), + Field::new("distance", DataType::UInt32, false), + ])) +}); + +/// Schema of a Hamming clustering-result batch. +static CLUSTER_SCHEMA: LazyLock = LazyLock::new(|| { + Arc::new(Schema::new(vec![ + Field::new("representative", DataType::UInt64, false), + Field::new( + "duplicates", + DataType::List(Arc::new(Field::new("item", DataType::UInt64, true))), + false, + ), + ])) +}); + pub trait Hamming { /// Hamming distance between two vectors. fn hamming(x: &[u8], y: &[u8]) -> f32; @@ -102,6 +123,196 @@ pub fn hamming_u64(a: u64, b: u64) -> u32 { (a ^ b).count_ones() } +/// Binary hash values stored as 64-bit lanes in lane-major order. +/// +/// For a hash width of `N` bytes, `N` must be a positive multiple of 8 and the +/// number of lanes is `N / 8`. Lane-major layout keeps each lane contiguous for +/// all rows, which allows the pairwise path to reuse the SIMD `u64` batch +/// implementation for wider hashes. +#[derive(Debug, Clone)] +pub struct BinaryHashValues { + lane_values: Vec, + num_rows: usize, + byte_width: usize, +} + +impl BinaryHashValues { + /// Create hash values from lane-major `u64` values. + pub fn try_new(lane_values: Vec, num_rows: usize, byte_width: usize) -> Result { + let num_lanes = validate_hash_byte_width(byte_width)?; + let expected_values = checked_lane_value_count(num_rows, num_lanes)?; + if lane_values.len() != expected_values { + return Err(Error::InvalidArgumentError(format!( + "Expected {} lane values for {} rows and {} byte hashes, got {}", + expected_values, + num_rows, + byte_width, + lane_values.len() + ))); + } + Ok(Self { + lane_values, + num_rows, + byte_width, + }) + } + + /// Extract binary hash values from a `FixedSizeList` Arrow array. + pub fn from_fixed_size_list(array: &FixedSizeListArray) -> Result { + let byte_width = usize::try_from(array.value_length()).map_err(|_| { + Error::InvalidArgumentError(format!( + "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}", + array.value_length() + )) + })?; + let num_lanes = validate_hash_byte_width(byte_width)?; + + let values = array + .values() + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::InvalidArgumentError( + "Expected UInt8Array values in FixedSizeList".to_string(), + ) + })?; + + let num_rows = array.len(); + let value_count = checked_lane_value_count(num_rows, num_lanes)?; + let expected_bytes = checked_hash_byte_count(num_rows, byte_width)?; + let bytes = values.values(); + if bytes.len() != expected_bytes { + return Err(Error::InvalidArgumentError(format!( + "Expected {} bytes for {} rows and {} byte hashes, got {}", + expected_bytes, + num_rows, + byte_width, + bytes.len() + ))); + } + + let mut lane_values = vec![0u64; value_count]; + for row in 0..num_rows { + let row_start = row * byte_width; + for lane in 0..num_lanes { + let start = row_start + lane * 8; + let mut arr = [0u8; 8]; + arr.copy_from_slice(&bytes[start..start + 8]); + lane_values[lane * num_rows + row] = u64::from_le_bytes(arr); + } + } + + Ok(Self { + lane_values, + num_rows, + byte_width, + }) + } + + /// Concatenate chunks with the same hash width into a single lane-major set. + pub fn concat(chunks: &[Self]) -> Result { + let Some(first) = chunks.first() else { + return Err(Error::InvalidArgumentError( + "Cannot concatenate zero binary hash chunks".to_string(), + )); + }; + + let byte_width = first.byte_width; + let num_lanes = first.num_lanes(); + let mut num_rows = 0usize; + for chunk in chunks { + if chunk.byte_width != byte_width { + return Err(Error::InvalidArgumentError(format!( + "Cannot concatenate binary hash chunks with different widths: {} and {} bytes", + byte_width, chunk.byte_width + ))); + } + num_rows = num_rows.checked_add(chunk.num_rows).ok_or_else(|| { + Error::InvalidArgumentError( + "Binary hash row count overflowed while concatenating chunks".to_string(), + ) + })?; + } + + let value_count = checked_lane_value_count(num_rows, num_lanes)?; + let mut lane_values = vec![0u64; value_count]; + let mut row_offset = 0; + for chunk in chunks { + for lane in 0..num_lanes { + let dest_start = lane * num_rows + row_offset; + let dest_end = dest_start + chunk.num_rows; + lane_values[dest_start..dest_end].copy_from_slice(chunk.lane(lane)); + } + row_offset += chunk.num_rows; + } + + Ok(Self { + lane_values, + num_rows, + byte_width, + }) + } + + pub fn len(&self) -> usize { + self.num_rows + } + + pub fn is_empty(&self) -> bool { + self.num_rows == 0 + } + + pub fn byte_width(&self) -> usize { + self.byte_width + } + + pub fn num_lanes(&self) -> usize { + self.byte_width / 8 + } + + pub fn lane(&self, lane: usize) -> &[u64] { + let start = lane * self.num_rows; + &self.lane_values[start..start + self.num_rows] + } + + pub fn into_u64_values(self) -> Result> { + if self.num_lanes() != 1 { + return Err(Error::InvalidArgumentError(format!( + "Expected 8-byte binary hashes, got {} byte hashes", + self.byte_width + ))); + } + Ok(self.lane_values) + } +} + +fn checked_lane_value_count(num_rows: usize, num_lanes: usize) -> Result { + num_rows.checked_mul(num_lanes).ok_or_else(|| { + Error::InvalidArgumentError(format!( + "Binary hash lane value count overflowed for {} rows and {} lanes", + num_rows, num_lanes + )) + }) +} + +fn checked_hash_byte_count(num_rows: usize, byte_width: usize) -> Result { + num_rows.checked_mul(byte_width).ok_or_else(|| { + Error::InvalidArgumentError(format!( + "Binary hash byte count overflowed for {} rows and {} byte hashes", + num_rows, byte_width + )) + }) +} + +fn validate_hash_byte_width(byte_width: usize) -> Result { + if byte_width == 0 || !byte_width.is_multiple_of(8) { + return Err(Error::InvalidArgumentError(format!( + "Expected FixedSizeList with a positive size that is a multiple of 8 bytes, got size {}", + byte_width + ))); + } + Ok(byte_width / 8) +} + /// Result of pairwise hamming distance computation. #[derive(Debug, Clone)] pub struct PairwiseResult { @@ -149,11 +360,7 @@ impl PairwiseResult { /// Convert to Arrow RecordBatch, consuming self. pub fn into_record_batch(self) -> RecordBatch { - let schema = Arc::new(Schema::new(vec![ - Field::new("row_id_a", DataType::UInt64, false), - Field::new("row_id_b", DataType::UInt64, false), - Field::new("distance", DataType::UInt32, false), - ])); + let schema = DISTANCE_PAIR_SCHEMA.clone(); let row_id_a = Arc::new(UInt64Array::from(self.row_id_a)); let row_id_b = Arc::new(UInt64Array::from(self.row_id_b)); @@ -172,24 +379,57 @@ impl Default for PairwiseResult { /// Compute hamming distances for a query against multiple targets. /// Uses SIMD acceleration when available. +/// +/// # Panics +/// +/// Panics if `results` is not the same length as `targets`. #[inline] pub fn hamming_batch_u64(query: u64, targets: &[u64], results: &mut [u32]) { - debug_assert_eq!(targets.len(), results.len()); + // Rejected in either direction, and before the dispatch. A shorter `results` the + // dispatcher's reslice would refuse on its own, and that reslice is what keeps the + // kernels' raw stores in bounds; this check just gets there first, with a message + // that names both lengths. An over-long one the reslice would instead truncate, + // handing the surplus tail back unwritten for the caller to read as distances. + // That is wrong output rather than a crash, which is why this is an `assert_eq!` + // and not a `debug_assert_eq!`. + let num_targets = targets.len(); + let num_slots = results.len(); + assert_eq!( + num_targets, num_slots, + "hamming_batch_u64 needs one result slot per target, \ + got {num_targets} target(s) and {num_slots} slot(s)" + ); hamming_batch_simd(query, targets, results); } /// SIMD-accelerated batch hamming distance computation. +/// +/// # Panics +/// +/// Panics if `results` is shorter than `targets`. `results.len()` is required to equal +/// `targets.len()`: an over-long `results` is truncated here rather than rejected, and +/// the surplus tail goes back to the caller unwritten. `hamming_batch_u64` asserts the +/// equality before dispatching. #[inline] fn hamming_batch_simd(query: u64, targets: &[u64], results: &mut [u32]) { + // Both x86 kernels store through raw pointers over a range bounded by + // `targets.len()`, and this reslice makes `results` exactly that long, so it has to + // stay above the dispatch. + let results = &mut results[..targets.len()]; + #[cfg(target_arch = "x86_64")] { if is_x86_feature_detected!("avx512vpopcntdq") && is_x86_feature_detected!("avx512f") { + // SAFETY: both required features were just detected, and the reslice + // above makes `results.len() == targets.len()`. unsafe { hamming_batch_avx512(query, targets, results); } return; } if is_x86_feature_detected!("avx2") { + // SAFETY: AVX2 was just detected, and the reslice above makes + // `results.len() == targets.len()`. unsafe { hamming_batch_avx2(query, targets, results); } @@ -229,6 +469,15 @@ fn hamming_batch_scalar(query: u64, targets: &[u64], results: &mut [u32]) { } /// AVX-512 VPOPCNTDQ: Process 8 x 64-bit values at once. +/// +/// The chunk loop reaches only the first `targets.len() / 8 * 8` slots through a +/// raw pointer, 8 x u32 per chunk with no bounds check; the trailing slots go +/// through bounds-checked indexing. +/// +/// # Safety +/// +/// The host must support AVX-512F and AVX512VPOPCNTDQ, and `results.len()` must +/// be at least `targets.len()`. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx512f", enable = "avx512vpopcntdq")] unsafe fn hamming_batch_avx512(query: u64, targets: &[u64], results: &mut [u32]) { @@ -264,6 +513,15 @@ unsafe fn hamming_batch_avx512(query: u64, targets: &[u64], results: &mut [u32]) } /// AVX2 popcount using lookup table (Harley-Seal / PSHUFB method). +/// +/// The chunk loop reaches only the first `targets.len() / 4 * 4` slots through a +/// raw pointer, 4 x u32 per chunk with no bounds check; the trailing slots go +/// through bounds-checked indexing. +/// +/// # Safety +/// +/// The host must support AVX2, and `results.len()` must be at least +/// `targets.len()`. #[cfg(target_arch = "x86_64")] #[target_feature(enable = "avx2")] unsafe fn hamming_batch_avx2(query: u64, targets: &[u64], results: &mut [u32]) { @@ -386,6 +644,88 @@ pub fn pairwise_hamming_distance_parallel( combined } +/// Compute pairwise hamming distances for all pairs of fixed-width binary hashes. +/// +/// This supports any hash width that is a positive multiple of 8 bytes. For +/// 8-byte hashes this delegates to the existing `u64` implementation. +pub fn pairwise_hamming_distance_binary( + hashes: &BinaryHashValues, + row_ids: Option<&[u64]>, + threshold: Option, +) -> PairwiseResult { + let n = hashes.len(); + if n < 2 { + return PairwiseResult::new(); + } + + if hashes.num_lanes() == 1 { + return pairwise_hamming_distance(hashes.lane(0), row_ids, threshold); + } + + let threshold = threshold.unwrap_or(u32::MAX); + let num_pairs = n * (n - 1) / 2; + let mut result = PairwiseResult::with_capacity(num_pairs.min(1_000_000)); + + for i in 0..n { + for j in (i + 1)..n { + let mut dist = 0; + for lane in 0..hashes.num_lanes() { + let lane_values = hashes.lane(lane); + dist += hamming_u64(lane_values[i], lane_values[j]); + } + if dist <= threshold { + let id_a = row_ids.map_or(i as u64, |ids| ids[i]); + let id_b = row_ids.map_or(j as u64, |ids| ids[j]); + result.push(id_a, id_b, dist); + } + } + } + + result +} + +/// Compute pairwise hamming distances in parallel for fixed-width binary hashes. +/// +/// Wider hashes reuse the SIMD `u64` batch implementation lane-by-lane. +pub fn pairwise_hamming_distance_binary_parallel( + hashes: &BinaryHashValues, + row_ids: Option<&[u64]>, + threshold: Option, +) -> PairwiseResult { + let n = hashes.len(); + if n < 2 { + return PairwiseResult::new(); + } + + if hashes.num_lanes() == 1 { + return pairwise_hamming_distance_parallel(hashes.lane(0), row_ids, threshold); + } + + let threshold = threshold.unwrap_or(u32::MAX); + let total_pairs = n * (n - 1) / 2; + + if total_pairs < 10_000 { + return pairwise_hamming_distance_binary(hashes, row_ids, Some(threshold)); + } + + let threads = rayon::current_num_threads(); + let pairs_per_chunk = total_pairs.div_ceil(threads); + let chunks = compute_balanced_chunks(n, pairs_per_chunk); + + let results: Vec = chunks + .into_par_iter() + .map(|(start_row, end_row)| { + process_row_range_binary(hashes, row_ids, threshold, start_row, end_row) + }) + .collect(); + + let mut combined = PairwiseResult::new(); + for r in results { + combined.extend(r); + } + combined +} + /// Compute balanced chunks for parallel processing. fn compute_balanced_chunks(n: usize, target_pairs_per_chunk: usize) -> Vec<(usize, usize)> { let mut chunks = Vec::new(); @@ -439,36 +779,80 @@ fn process_row_range( result } +/// Process a range of rows for pairwise comparison of wider binary hashes. +fn process_row_range_binary( + hashes: &BinaryHashValues, + row_ids: Option<&[u64]>, + threshold: u32, + start_row: usize, + end_row: usize, +) -> PairwiseResult { + let n = hashes.len(); + let mut result = PairwiseResult::new(); + let mut distances = Vec::new(); + let mut lane_distances = Vec::new(); + + for i in start_row..end_row { + let remaining = n - i - 1; + if remaining == 0 { + continue; + } + + distances.clear(); + distances.resize(remaining, 0); + + let first_lane = hashes.lane(0); + hamming_batch_u64( + first_lane[i], + &first_lane[i + 1..], + distances.as_mut_slice(), + ); + + for lane in 1..hashes.num_lanes() { + lane_distances.clear(); + lane_distances.resize(remaining, 0); + let lane_values = hashes.lane(lane); + hamming_batch_u64( + lane_values[i], + &lane_values[i + 1..], + lane_distances.as_mut_slice(), + ); + for (distance, lane_distance) in distances.iter_mut().zip(&lane_distances) { + *distance += *lane_distance; + } + } + + let id_a = row_ids.map_or(i as u64, |ids| ids[i]); + for (j_offset, &dist) in distances.iter().enumerate() { + if dist <= threshold { + let j = i + 1 + j_offset; + let id_b = row_ids.map_or(j as u64, |ids| ids[j]); + result.push(id_a, id_b, dist); + } + } + } + + result +} + /// Extract u64 hashes from a FixedSizeList Arrow array. pub fn extract_hashes_from_fixed_list(array: &FixedSizeListArray) -> Result> { - let list_size = array.value_length(); - if list_size != 8 { + let hashes = extract_binary_hashes_from_fixed_list(array)?; + if hashes.byte_width() != 8 { return Err(Error::InvalidArgumentError(format!( "Expected FixedSizeList with size 8, got size {}", - list_size + hashes.byte_width() ))); } + hashes.into_u64_values() +} - let values = array - .values() - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::InvalidArgumentError("Expected UInt8Array values in FixedSizeList".to_string()) - })?; - - let n = array.len(); - let mut hashes = Vec::with_capacity(n); - - for i in 0..n { - let start = i * 8; - let bytes = &values.values()[start..start + 8]; - let mut arr = [0u8; 8]; - arr.copy_from_slice(bytes); - hashes.push(u64::from_le_bytes(arr)); - } - - Ok(hashes) +/// Extract binary hashes from a `FixedSizeList` Arrow array where +/// `N` is a positive multiple of 8 bytes. +pub fn extract_binary_hashes_from_fixed_list( + array: &FixedSizeListArray, +) -> Result { + BinaryHashValues::from_fixed_size_list(array) } /// Union-Find data structure with path compression for clustering. @@ -599,14 +983,7 @@ impl ClusteringResult { /// Get the schema for clustering result batches. pub fn schema() -> SchemaRef { - Arc::new(Schema::new(vec![ - Field::new("representative", DataType::UInt64, false), - Field::new( - "duplicates", - DataType::List(Arc::new(Field::new("item", DataType::UInt64, true))), - false, - ), - ])) + CLUSTER_SCHEMA.clone() } /// Convert to Arrow RecordBatch with columns: @@ -733,6 +1110,22 @@ pub fn cluster_pairwise_result(result: &PairwiseResult) -> ClusteringResult { #[cfg(test)] mod tests { use super::*; + use lance_arrow::FixedSizeListArrayExt; + use rstest::rstest; + + #[test] + fn test_result_schemas_are_initialized_once() { + // These schemas are handed out per call and per batch, so they are + // shared rather than rebuilt. Pointer equality is what distinguishes a + // shared schema from an equal-but-freshly-allocated one. + assert!(Arc::ptr_eq( + &ClusteringResult::schema(), + &ClusteringResult::schema() + )); + + let batch = PairwiseResult::default().into_record_batch(); + assert!(Arc::ptr_eq(&batch.schema(), &DISTANCE_PAIR_SCHEMA)); + } #[test] fn test_hamming() { @@ -770,6 +1163,85 @@ mod tests { assert_eq!(results[7], 3); // 0b111 has 3 bits set } + #[rstest] + // Fewer slots than targets is the case that used to write out of bounds: on a + // host that takes the AVX2 path, `hamming_batch_avx2` wrote 28 bytes past a + // one-slot `results`. + #[case::eight_targets_one_slot(8, 1)] + // On the same path this one overran by 4 bytes instead of 28. + #[case::eight_targets_seven_slots(8, 7)] + // More slots than targets left the tail unwritten rather than overrunning, but + // the contract is one slot per target either way. This and the zero-target case + // below are the two that catch a guard weakened to a one-sided + // `results.len() >= targets.len()`. + #[case::eight_targets_nine_slots(8, 9)] + // The target count varies too, so that the two shortcuts most likely to appear + // here cannot slip past every case: an `if targets.is_empty() { return; }` above + // the check, or a `targets.len() < 8` fast path, would otherwise leave every + // eight-target case green. Four is the smallest target count that still reaches + // the AVX2 chunk loop. + #[case::four_targets_one_slot(4, 1)] + #[case::no_targets_one_slot(0, 1)] + // Zero slots is the case with no in-bounds slot at all, and the only one here + // that a `results.is_empty()` early return above the check would not survive. + // Its sentinel assertion is vacuous, so it pins the message and nothing else. + #[case::eight_targets_no_slots(8, 0)] + fn test_hamming_batch_u64_rejects_mismatched_lengths( + #[case] num_targets: usize, + #[case] num_slots: usize, + ) { + // No CPU feature gate: the length check precedes every kernel, so the panic + // happens on each host and no case reaches a kernel at all. Every case here is + // a mismatch; matching lengths are covered by `test_hamming_batch_u64` above, + // which calls `hamming_batch_u64` directly, and by + // `test_pairwise_correctness_1000_deterministic` below, which reaches it + // through `pairwise_hamming_distance_parallel`'s chunked path and checks the + // result against `reference_pairwise`. + // + // `catch_unwind` rather than `#[should_panic]`, because the property worth + // protecting is that the check runs before the dispatch, and + // `#[should_panic]` cannot observe order: a check moved below it can still + // panic with this same message, after a kernel has already written through + // `results`. The sentinel assertion at the end is what rules that out. It is + // also why the nine-slot case is here: a kernel's writes all land in bounds + // there, so nothing else would notice that it ran. + // + // A `debug_assert_eq!` carrying the same message still passes here when + // debug assertions are on, which is the profile `--profile ci` gives this + // crate; `cargo test --release -p lance-linalg --lib` (rust.yml) is what + // catches that revert. + const SENTINEL: u32 = 0xDEAD_BEEF; + let targets = vec![u64::MAX; num_targets]; + let mut results = vec![SENTINEL; num_slots]; + + let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + hamming_batch_u64(0, &targets, &mut results); + })) + .expect_err("a length mismatch must panic"); + + let message = payload + .downcast_ref::() + .map(String::as_str) + .or_else(|| payload.downcast_ref::<&'static str>().copied()) + .expect("panic payload should be a string"); + // Matching the whole message, not the two numbers separately, is deliberate: + // it pins the function name and the order the two lengths are reported in, + // which is what a message copied from a sibling guard would get wrong. + let expected = format!( + "hamming_batch_u64 needs one result slot per target, \ + got {num_targets} target(s) and {num_slots} slot(s)" + ); + assert!( + message.contains(&expected), + "expected {expected:?} in panic message, got {message:?}" + ); + assert_eq!( + results.iter().position(|&slot| slot != SENTINEL), + None, + "a kernel reached `results`, so the length check no longer precedes it" + ); + } + #[test] fn test_pairwise_basic() { let hashes = vec![0b0000u64, 0b0001, 0b0011, 0b0111]; @@ -912,6 +1384,121 @@ mod tests { v } + #[test] + fn test_extract_binary_hashes_from_fixed_list_128() { + use arrow_array::UInt8Array; + + let rows = [ + (0x0102_0304_0506_0708u64, 0x1112_1314_1516_1718u64), + (0x2122_2324_2526_2728u64, 0x3132_3334_3536_3738u64), + ]; + let bytes: Vec = rows + .iter() + .flat_map(|(lo, hi)| lo.to_le_bytes().into_iter().chain(hi.to_le_bytes())) + .collect(); + let array = FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 16).unwrap(); + + let hashes = extract_binary_hashes_from_fixed_list(&array).unwrap(); + assert_eq!(hashes.len(), 2); + assert_eq!(hashes.byte_width(), 16); + assert_eq!(hashes.num_lanes(), 2); + assert_eq!(hashes.lane(0), &[rows[0].0, rows[1].0]); + assert_eq!(hashes.lane(1), &[rows[0].1, rows[1].1]); + } + + #[test] + fn test_extract_binary_hashes_rejects_non_u64_multiple() { + use arrow_array::UInt8Array; + + let array = + FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![0u8; 24]), 12).unwrap(); + let err = extract_binary_hashes_from_fixed_list(&array).unwrap_err(); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + } + + #[test] + fn test_binary_hash_values_rejects_width_not_divisible_by_8() { + let err = BinaryHashValues::try_new(Vec::new(), 0, 12).unwrap_err(); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + } + + #[test] + fn test_pairwise_binary_hashes_128() { + let hashes = BinaryHashValues::try_new( + vec![ + 0, + 0, + 1, + u64::MAX, // lane 0 + 0, + 1, + 1, + u64::MAX, // lane 1 + ], + 4, + 16, + ) + .unwrap(); + + let seq = pairwise_hamming_distance_binary(&hashes, None, Some(1)); + let par = pairwise_hamming_distance_binary_parallel(&hashes, None, Some(1)); + let expected = vec![(0, 1, 1), (1, 2, 1)]; + assert_eq!(result_to_sorted_vec(&seq), expected); + assert_eq!(result_to_sorted_vec(&par), expected); + } + + #[test] + fn test_pairwise_binary_hashes_parallel_128_matches_sequential() { + let rows: Vec<(u64, u64)> = (0..80) + .flat_map(|i| { + let lo = (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15); + let hi = !lo.rotate_left(17); + [(lo, hi), (lo ^ 1, hi)] + }) + .collect(); + let mut lane_values = Vec::with_capacity(rows.len() * 2); + lane_values.extend(rows.iter().map(|(lo, _)| *lo)); + lane_values.extend(rows.iter().map(|(_, hi)| *hi)); + let hashes = BinaryHashValues::try_new(lane_values, rows.len(), 16).unwrap(); + let row_ids: Vec = (0..rows.len()).map(|i| 10_000 + i as u64).collect(); + + let seq = pairwise_hamming_distance_binary(&hashes, Some(&row_ids), Some(1)); + let par = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(1)); + + assert_eq!(result_to_sorted_vec(&par), result_to_sorted_vec(&seq)); + assert!(par.len() >= 80); + } + + #[test] + fn test_pairwise_binary_hashes_32_with_row_ids() { + let hashes = BinaryHashValues::try_new( + vec![ + 0, + 0, + 1, // lane 0 + 0, + 1, + 1, // lane 1 + 7, + 7, + 7, // lane 2 + u64::MAX, + u64::MAX, + u64::MAX - 1, // lane 3 + ], + 3, + 32, + ) + .unwrap(); + let row_ids = [10, 20, 30]; + + let result = pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(2)); + assert_eq!( + result_to_sorted_vec(&result), + vec![(10, 20, 1), (20, 30, 2)] + ); + } + #[test] fn test_pairwise_correctness_small() { // Deterministic hashes with known distances diff --git a/rust/lance-linalg/src/distance/l2.rs b/rust/lance-linalg/src/distance/l2.rs index c47aedd749f..16f4ef194c8 100644 --- a/rust/lance-linalg/src/distance/l2.rs +++ b/rust/lance-linalg/src/distance/l2.rs @@ -4,6 +4,11 @@ //! L2 (Euclidean) distance. //! +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use std::arch::x86_64::{_mm256_loadu_ps, _mm256_mul_ps, _mm256_sub_ps}; use std::iter::Sum; use std::ops::AddAssign; use std::sync::Arc; @@ -17,21 +22,51 @@ use arrow_array::{ use arrow_schema::DataType; use half::{bf16, f16}; use lance_arrow::{ArrowFloatType, FixedSizeListArrayExt, FloatArray}; -use lance_core::assume_eq; use lance_core::deepsize::DeepSizeOf; use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] +// Named tiers are only matched on x86_64, or by the fp16 kernels on the other +// architectures; without either, nothing below names a `SimdSupport` variant. +#[cfg(any(feature = "fp16kernels", target_arch = "x86_64"))] use lance_core::utils::cpu::SimdSupport; use num_traits::{AsPrimitive, Num}; +use crate::distance::{assert_batch_layout, assert_equal_lengths}; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::distance::{BatchIter, BatchKernel, BatchKind, BatchOperation}; +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +use crate::simd::x86::hsum256_ps; + /// Calculate the L2 distance between two vectors. /// pub trait L2: Num { /// Calculate the L2 distance between two vectors. fn l2(x: &[Self], y: &[Self]) -> f32; - fn l2_batch(x: &[Self], y: &[Self], dimension: usize) -> impl Iterator { - y.chunks_exact(dimension).map(|v| Self::l2(x, v)) + /// L2 distance from `x` to each `dimension`-sized vector in `y`. + /// + /// The default calls [`L2::l2`] per vector. `f32` overrides it so the SIMD + /// tier is chosen once for the whole batch instead of once per vector — + /// on a build whose baseline already implies AVX2, per-vector dispatch + /// costs more than the kernel it selects. + /// + /// Returns `impl Iterator` rather than a trait object: the k-means + /// assignment loop drives this one element at a time, so a + /// `Box` would cost a virtual call per element and an + /// allocation per batch. + fn l2_batch<'a>( + x: &'a [Self], + y: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + assert_batch_layout(x.len(), y.len(), dimension); + y.chunks_exact(dimension).map(move |v| Self::l2(x, v)) } } @@ -44,54 +79,22 @@ pub fn l2(from: &[T], to: &[T]) -> f32 { /// available at runtime. /// /// On x86_64 with AVX-512 this uses 16-wide f32 lanes; otherwise it falls back -/// to [`l2`], which auto-vectorizes to the compiled target (AVX2 on the default -/// `haswell` build). Lance ships an AVX2-baseline binary, so the generic -/// [`l2`] never emits AVX-512 even on capable CPUs — this dispatcher recovers -/// that throughput for callers in the hot path (e.g. the in-memory HNSW index). +/// to [`l2`], whose x86_64 implementation selects the best runtime-supported +/// kernel. This entry point gives hot-path callers such as the in-memory HNSW +/// index an explicit f32 API. #[inline] pub fn l2_f32(x: &[f32], y: &[f32]) -> f32 { - #[cfg(target_arch = "x86_64")] - { - use lance_core::utils::cpu::SimdSupport; - if matches!(*SIMD_SUPPORT, SimdSupport::Avx512 | SimdSupport::Avx512FP16) { - // SAFETY: guarded by the runtime AVX-512 detection above. - return unsafe { l2_f32_avx512(x, y) }; - } - } - l2(x, y) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx512f")] -unsafe fn l2_f32_avx512(x: &[f32], y: &[f32]) -> f32 { - use std::arch::x86_64::*; - debug_assert_eq!(x.len(), y.len()); - let n = x.len(); - let mut acc = _mm512_setzero_ps(); - let mut i = 0usize; - while i + 16 <= n { - let a = _mm512_loadu_ps(x.as_ptr().add(i)); - let b = _mm512_loadu_ps(y.as_ptr().add(i)); - let diff = _mm512_sub_ps(a, b); - acc = _mm512_fmadd_ps(diff, diff, acc); - i += 16; - } - let mut sum = _mm512_reduce_add_ps(acc); - while i < n { - let diff = x[i] - y[i]; - sum += diff * diff; - i += 1; - } - sum + f32::l2(x, y) } /// Calculate L2 distance between two uint8 slices. #[inline] pub fn l2_distance_uint_scalar(key: &[u8], target: &[u8]) -> f32 { + assert_equal_lengths(key.len(), target.len()); key.iter() .zip(target.iter()) - .map(|(&x, &y)| (x.abs_diff(y) as u32).pow(2)) - .sum::() as f32 + .map(|(&x, &y)| (x.abs_diff(y) as u64).pow(2)) + .sum::() as f32 } /// Calculate the L2 distance between two vectors, using scalar operations. @@ -108,6 +111,7 @@ pub fn l2_scalar< from: &[T], to: &[T], ) -> Output { + assert_equal_lengths(from.len(), to.len()); let x_chunks = from.chunks_exact(LANES); let y_chunks = to.chunks_exact(LANES); @@ -139,7 +143,8 @@ pub fn l2_scalar< impl L2 for u8 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { - super::l2_u8::l2_u8(x, y) as f32 + assert_equal_lengths(x.len(), y.len()); + super::l2_u8::l2_u8_u64(x, y) as f32 } } @@ -166,6 +171,7 @@ mod bf16_kernel { impl L2 for bf16 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { + assert_equal_lengths(x.len(), y.len()); match *SIMD_SUPPORT { #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))] SimdSupport::Neon => unsafe { @@ -191,6 +197,9 @@ impl L2 for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::l2_bf16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => l2_scalar::(x, y), } } @@ -219,6 +228,7 @@ mod kernel { impl L2 for f16 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { + assert_equal_lengths(x.len(), y.len()); match *SIMD_SUPPORT { #[cfg(all(feature = "fp16kernels", target_arch = "aarch64"))] SimdSupport::Neon => unsafe { @@ -233,7 +243,7 @@ impl L2 for f16 { kernel::l2_f16_avx512(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::l2_f16_avx2(x.as_ptr(), y.as_ptr(), x.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -244,6 +254,9 @@ impl L2 for f16 { SimdSupport::Lsx => unsafe { kernel::l2_f16_lsx(x.as_ptr(), y.as_ptr(), x.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => l2_scalar::(x, y), } } @@ -252,22 +265,507 @@ impl L2 for f16 { impl L2 for f32 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { - // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32)) - // See https://github.com/lance-format/lance/pull/2450. - l2_scalar::(x, y) + assert_equal_lengths(x.len(), y.len()); + // Trait methods cannot carry `#[target_feature]` attributes, so the body + // lives in a free function that runtime-dispatches via `*SIMD_SUPPORT` + // to an AVX2 or AVX-512 inner kernel on capable hosts, or a portable + // scalar fallback. + l2_f32_dispatched(x, y) + } + + fn l2_batch<'a>( + x: &'a [Self], + y: &'a [Self], + dimension: usize, + ) -> impl Iterator + 'a { + assert_batch_layout(x.len(), y.len(), dimension); + // Exactly one arm compiles; see `Dot::dot_batch` for f32. + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "fma" + ))] + { + // `l2_scalar::<_, _, 16>` chunks the vector by 16 lanes. At or below + // that width the chunking degenerates to its scalar remainder loop + // and vectorizes nothing, so the explicit AVX kernel is worth ~40%. + // Above it the autovectorizer already does well and the 8-wide + // kernel can lose, so keep the exact kernel the pre-dispatch code + // used and stay non-regressing by construction. + // + // SAFETY: the build baseline enables avx2+fma, which imply avx+fma, + // so the kernel's `#[target_feature]` contract is met statically. + let narrow = dimension <= 16; + y.chunks_exact(dimension).map(move |v| { + if narrow { + unsafe { x86::l2_f32_avx_fma(x, v) } + } else { + l2_f32_scalar(x, v) + } + }) + } + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + { + l2_batch_f32_runtime_dispatch(x, y, dimension) + } + #[cfg(not(target_arch = "x86_64"))] + { + // `assert_batch_layout` proves every chunk has the same length as + // `x`, so call the private kernel directly instead of repeating + // the public `L2::l2` validation for every vector. + y.chunks_exact(dimension) + .map(move |v| l2_f32_dispatched(x, v)) + } + } +} + +/// Sub-AVX2 builds: pick a `#[target_feature]` kernel once for the batch. +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +#[inline] +fn l2_batch_f32_runtime_dispatch<'a>( + x: &'a [f32], + y: &'a [f32], + dimension: usize, +) -> impl Iterator + 'a { + let (kernel, kind): (BatchKernel, BatchKind) = match *SIMD_SUPPORT { + // AVX-512 has no useful work for an eight-element vector. Retain the + // AVX/FMA kernel used by the former Haswell baseline for PQ dimensions. + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if dimension > 16 => { + (x86::l2_batch_f32_avx512, BatchKind::Avx512) + } + SimdSupport::Avx512 | SimdSupport::Avx512FP16 if std::is_x86_feature_detected!("fma") => { + (x86::l2_batch_f32_avx_fma, BatchKind::AvxFma) + } + SimdSupport::Avx2 | SimdSupport::AvxFma => (x86::l2_batch_f32_avx_fma, BatchKind::AvxFma), + SimdSupport::Avx512 | SimdSupport::Avx512FP16 | SimdSupport::Avx => { + (x86::l2_batch_f32_avx, BatchKind::Avx) + } + _ => (l2_batch_f32_scalar, BatchKind::Scalar), + }; + + // SAFETY: the runtime tier and the explicit FMA check above establish the + // selected kernel's target-feature contract. + unsafe { BatchIter::::new(x, y, dimension, kernel, kind) } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +struct L2Batch; + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +impl BatchOperation for L2Batch { + #[inline] + fn fold_scalar(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + batch + .chunks_exact(dimension) + .fold(init, |acc, vector| f(acc, l2_f32_scalar(key, vector))) + } + + #[target_feature(enable = "avx")] + unsafe fn fold_avx(key: &[f32], batch: &[f32], dimension: usize, init: B, mut f: F) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let difference = _mm256_sub_ps(key_values, vector_values); + let squared = _mm256_mul_ps(difference, difference); + f(acc, unsafe { hsum256_ps(squared) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::l2_f32_avx(key, vector) }) + }) + } + + #[target_feature(enable = "avx,fma")] + unsafe fn fold_avx_fma( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + if dimension == 8 { + let key_values = unsafe { _mm256_loadu_ps(key.as_ptr()) }; + return batch.chunks_exact(8).fold(init, |acc, vector| { + let vector_values = unsafe { _mm256_loadu_ps(vector.as_ptr()) }; + let difference = _mm256_sub_ps(key_values, vector_values); + let squared = _mm256_mul_ps(difference, difference); + f(acc, unsafe { hsum256_ps(squared) }) + }); + } + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::l2_f32_avx_fma(key, vector) }) + }) + } + + #[target_feature(enable = "avx512f")] + unsafe fn fold_avx512( + key: &[f32], + batch: &[f32], + dimension: usize, + init: B, + mut f: F, + ) -> B + where + F: FnMut(B, f32) -> B, + { + batch.chunks_exact(dimension).fold(init, |acc, vector| { + f(acc, unsafe { x86::l2_f32_avx512(key, vector) }) + }) + } +} + +#[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) +))] +unsafe fn l2_batch_f32_scalar(x: &[f32], batch: &[f32], dimension: usize, output: &mut [f32]) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = l2_f32_scalar(x, y); } } +/// L2 distance for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// auto-vectorised scalar loop. +#[inline] +fn l2_f32_dispatched(x: &[f32], y: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f32_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f32_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::l2_f32_avx(x, y) }, + _ => l2_f32_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + l2_f32_scalar(x, y) + } +} + +/// Portable scalar L2 distance for f32. Used as the x86_64 fallback when no +/// AVX2 is detected, and as the only path on non-x86 architectures. The +/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above. +#[inline] +fn l2_f32_scalar(x: &[f32], y: &[f32]) -> f32 { + // 16 = 512 (avx512) / 8 bits / 4 (sizeof(f32)) + // See https://github.com/lance-format/lance/pull/2450. + l2_scalar::(x, y) +} + impl L2 for f64 { #[inline] fn l2(x: &[Self], y: &[Self]) -> f32 { + assert_equal_lengths(x.len(), y.len()); l2_f64_simd(x, y) } } -/// Explicit SIMD L2 distance for f64. +/// L2 distance for f64, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the SIMD +/// primitives in `crate::simd::f64`, unconditionally backed by NEON / LSX-LASX. #[inline] fn l2_f64_simd(x: &[f64], y: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { x86::l2_f64_avx512(x, y) }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::l2_f64_avx_fma(x, y) }, + SimdSupport::Avx => unsafe { x86::l2_f64_avx(x, y) }, + _ => l2_f64_scalar(x, y), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + l2_f64_simd_other(x, y) + } +} + +/// Portable scalar L2 distance for f64. Used as the x86_64 fallback when no +/// AVX2 is detected, and exposed for cross-backend parity testing. +#[cfg(target_arch = "x86_64")] +#[inline] +fn l2_f64_scalar(x: &[f64], y: &[f64]) -> f32 { + x.iter() + .zip(y.iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum::() as f32 +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// L2 distance from `x` to every `dimension`-sized vector in `batch`, with + /// the AVX-512 tier entered once for the whole batch rather than once per + /// vector. + /// + /// # Safety + /// The host must support AVX-512F. + /// + /// Only compiled for builds whose baseline is below avx2+fma; at or above + /// that baseline `l2_batch` inlines the kernel directly and never runtime- + /// dispatches, so this wrapper would be dead code (see `l2_batch`). + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx512f")] + pub(super) unsafe fn l2_batch_f32_avx512( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { l2_f32_avx512(x, y) }; + } + } + + /// As [`l2_batch_f32_avx512`], for the AVX+FMA and AVX2 tiers. + /// + /// # Safety + /// The host must support AVX and FMA. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx,fma")] + pub(super) unsafe fn l2_batch_f32_avx_fma( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { l2_f32_avx_fma(x, y) }; + } + } + + /// As [`l2_batch_f32_avx512`], for the AVX-without-FMA tier. + /// + /// # Safety + /// The host must support AVX. + #[cfg(not(all(target_feature = "avx2", target_feature = "fma")))] + #[target_feature(enable = "avx")] + pub(super) unsafe fn l2_batch_f32_avx( + x: &[f32], + batch: &[f32], + dimension: usize, + output: &mut [f32], + ) { + debug_assert_eq!(output.len(), batch.len() / dimension); + for (distance, y) in output.iter_mut().zip(batch.chunks_exact(dimension)) { + *distance = unsafe { l2_f32_avx(x, y) }; + } + } + + /// AVX-512 path for f64: 8-wide `__m512d` with `vsubpd` + `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn l2_f64_avx512(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm512_loadu_pd(x.as_ptr().add(i)); + let b = _mm512_loadu_pd(y.as_ptr().add(i)); + let diff = _mm512_sub_pd(a, b); + acc = _mm512_fmadd_pd(diff, diff, acc); + } + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + (_mm512_reduce_add_pd(acc) + tail) as f32 + } + + /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn l2_f64_avx_fma(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + let a = f64x8::load_unaligned(x.as_ptr().add(i)); + let b = f64x8::load_unaligned(y.as_ptr().add(i)); + let diff = a - b; + acc8.multiply_add(diff, diff); + } + + let aligned_len = dim / 4 * 4; + let mut acc4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + let a = f64x4::load_unaligned(x.as_ptr().add(i)); + let b = f64x4::load_unaligned(y.as_ptr().add(i)); + let diff = a - b; + acc4.multiply_add(diff, diff); + } + + let tail: f64 = x[aligned_len..] + .iter() + .zip(y[aligned_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + (acc8.reduce_sum() + acc4.reduce_sum() + tail) as f32 + } + + /// AVX-only path for f64 (no FMA): squared diff via `_mm256_mul_pd` + `_mm256_add_pd` for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn l2_f64_avx(x: &[f64], y: &[f64]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 4 * 4; + + let mut acc = _mm256_setzero_pd(); + for i in (0..unrolled_len).step_by(4) { + let a = _mm256_loadu_pd(x.as_ptr().add(i)); + let b = _mm256_loadu_pd(y.as_ptr().add(i)); + let diff = _mm256_sub_pd(a, b); + acc = _mm256_add_pd(acc, _mm256_mul_pd(diff, diff)); + } + + // Horizontal sum of __m256d -> f64. + let lo = _mm256_castpd256_pd128(acc); + let hi = _mm256_extractf128_pd(acc, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + let acc_sum = _mm_cvtsd_f64(sum64); + + let tail: f64 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + (acc_sum + tail) as f32 + } + + /// AVX-512 path for f32: 16-wide `__m512` with `vsubps` + `vfmadd231ps` per iteration. + #[inline] + #[target_feature(enable = "avx512f")] + pub unsafe fn l2_f32_avx512(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let a = _mm512_loadu_ps(x.as_ptr().add(i)); + let b = _mm512_loadu_ps(y.as_ptr().add(i)); + let diff = _mm512_sub_ps(a, b); + acc = _mm512_fmadd_ps(diff, diff, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + _mm512_reduce_add_ps(acc) + tail + } + + /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[inline] + #[target_feature(enable = "avx,fma")] + pub unsafe fn l2_f32_avx_fma(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + let diff = _mm256_sub_ps(a, b); + acc = _mm256_fmadd_ps(diff, diff, acc); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + hsum256_ps(acc) + tail + } + + /// AVX-only path for f32 (no FMA): squared diff via `_mm256_mul_ps` + `_mm256_add_ps` for Sandy/Ivy Bridge. + #[inline] + #[target_feature(enable = "avx")] + pub unsafe fn l2_f32_avx(x: &[f32], y: &[f32]) -> f32 { + let dim = x.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let a = _mm256_loadu_ps(x.as_ptr().add(i)); + let b = _mm256_loadu_ps(y.as_ptr().add(i)); + let diff = _mm256_sub_ps(a, b); + acc = _mm256_add_ps(acc, _mm256_mul_ps(diff, diff)); + } + + let tail: f32 = x[unrolled_len..] + .iter() + .zip(y[unrolled_len..].iter()) + .map(|(&a, &b)| { + let diff = a - b; + diff * diff + }) + .sum(); + + hsum256_ps(acc) + tail + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn l2_f64_simd_other(x: &[f64], y: &[f64]) -> f32 { use crate::simd::f64::{f64x4, f64x8}; use crate::simd::{FloatSimd, SIMD}; @@ -431,9 +929,6 @@ pub fn l2_distance_batch<'a, T: L2>( to: &'a [T], dimension: usize, ) -> impl Iterator + 'a { - assume_eq!(from.len(), dimension); - assume_eq!(to.len() % dimension, 0); - T::l2_batch(from, to, dimension) } @@ -493,7 +988,7 @@ pub fn l2_distance_arrow_batch( .collect(), &to.convert_to_floating_point()?, ), - _ => Err(Error::ComputeError(format!( + _ => Err(Error::InvalidArgumentError(format!( "Unsupported data type: {}", from.data_type() ))), @@ -508,8 +1003,25 @@ mod tests { use num_traits::ToPrimitive; use proptest::prelude::*; + #[test] + fn test_l2_rejects_mismatched_lengths() { + let short = [1.0_f32]; + let long = [1.0_f32, 2.0]; + + assert!(std::panic::catch_unwind(|| l2(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| l2_f32(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| f32::l2(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| l2_distance(&short, &long)).is_err()); + assert!(std::panic::catch_unwind(|| l2_distance_batch(&short, &long, 2)).is_err()); + assert!(std::panic::catch_unwind(|| l2_distance_batch(&long, &[1.0_f32; 3], 2)).is_err()); + assert!(std::panic::catch_unwind(|| l2_distance_batch::(&[], &[], 0)).is_err()); + assert!(std::panic::catch_unwind(|| l2_distance_uint_scalar(&[1, 2], &[1])).is_err()); + assert!(std::panic::catch_unwind(|| l2_scalar::(&[1, 2], &[1])).is_err()); + } + use crate::test_utils::{ arbitrary_bf16, arbitrary_f16, arbitrary_f32, arbitrary_f64, arbitrary_vector_pair, + dimension_shard, run_vector_pair_proptest, }; #[test] @@ -647,6 +1159,40 @@ mod tests { do_l2_test(&x, &y).unwrap(); } + #[rstest::rstest] + fn test_l2_distance_f32( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + do_l2_test(&x, &y) + }); + } + + #[rstest::rstest] + fn test_l2_distance_f64( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f64, dimension_shard(shard), |x, y| { + do_l2_test(&x, &y) + }); + } + + #[rstest::rstest] + fn test_l2_f32_scalar_simd_parity( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_pair_proptest(arbitrary_f32, dimension_shard(shard), |x, y| { + let scalar = x + .iter() + .zip(y.iter()) + .map(|(&a, &b)| ((a as f64) - (b as f64)).powi(2)) + .sum::() as f32; + let simd = ::l2(&x, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + Ok(()) + }); + } + // Test L2 distance over different types. // * L2 is valid over the entire range of f16. // * L2 is valid over f32 and bf16 in the range of +-1e12. @@ -662,14 +1208,115 @@ mod tests { do_l2_test(&x, &y)?; } + /// Cross-backend parity: scalar fallback must match the dispatched + /// SIMD path within numerical tolerance. Exercises `l2_f64_scalar` + /// directly so the runtime fallback is exercised even on AVX2-capable + /// CI hosts. + #[cfg(target_arch = "x86_64")] #[test] - fn test_l2_distance_f32((x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048)){ - do_l2_test(&x, &y)?; + fn test_l2_f64_scalar_simd_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + let scalar = l2_f64_scalar(&x, &y); + let simd = l2_f64_simd(&x, &y); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6)); } + /// AVX-512-direct parity: explicitly compares the scalar fallback + /// against the native AVX-512 inner kernel on AVX-512F-capable hosts + /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on + /// hosts without AVX-512F. + #[cfg(target_arch = "x86_64")] #[test] - fn test_l2_distance_f64((x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048)){ - do_l2_test(&x, &y)?; + fn test_l2_f64_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = l2_f64_scalar(&x, &y); + let avx512 = unsafe { x86::l2_f64_avx512(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6)); + } + + /// AVX + FMA-direct parity for the f64 L2 kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f64_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = l2_f64_scalar(&x, &y); + let avx_fma = unsafe { x86::l2_f64_avx_fma(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6)); + } + + /// AVX-only-direct parity for the f64 L2 kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without + /// AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f64_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f64, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = l2_f64_scalar(&x, &y); + let avx = unsafe { x86::l2_f64_avx(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6)); + } + + /// AVX-512-direct parity for f32: explicitly compares the scalar + /// fallback against the native f32 AVX-512 inner kernel on + /// AVX-512F-capable hosts. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f32_scalar_vs_avx512_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = l2_f32_scalar(&x, &y); + let avx512 = unsafe { x86::l2_f32_avx512(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3)); + } + + /// AVX + FMA-direct parity for the f32 L2 kernel. Covers the AMD + /// Piledriver / Steamroller / FX-7500 tier. Early-returns on hosts + /// without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f32_scalar_vs_avx_fma_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = l2_f32_scalar(&x, &y); + let avx_fma = unsafe { x86::l2_f32_avx_fma(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3)); + } + + /// AVX-only-direct parity for the f32 L2 kernel. Covers the Intel + /// Sandy Bridge / Ivy Bridge tier. Early-returns on hosts without + /// AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_f32_scalar_vs_avx_parity( + (x, y) in arbitrary_vector_pair(arbitrary_f32, 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = l2_f32_scalar(&x, &y); + let avx = unsafe { x86::l2_f32_avx(&x, &y) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3)); } } @@ -787,4 +1434,101 @@ mod tests { assert_relative_eq!(d1[0], 0.0); // q1 == target[0] assert_relative_eq!(d2[1], 0.0); // q2 == target[1] } + + /// `l2_batch` must agree with the per-vector `l2` it replaced, on every + /// build: the AVX2-baseline path, the hoisted-dispatch path, and the + /// portable fallback all funnel through here. + #[rstest::rstest] + #[case::dim_8(8)] + #[case::dim_16(16)] + #[case::dim_32(32)] + #[case::dim_1024(1024)] + fn test_l2_batch_f32_matches_per_vector_l2(#[case] dimension: usize) { + let num_vectors = 5; + let x: Vec = (0..dimension) + .map(|i| ((i % 13) as f32) * 0.25 + 1.0) + .collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 11) as f32) * 0.5 - 2.0) + .collect(); + + let got: Vec = f32::l2_batch(&x, &batch, dimension).collect(); + let want: Vec = batch + .chunks_exact(dimension) + .map(|y| f32::l2(&x, y)) + .collect(); + + assert_eq!(got.len(), num_vectors); + for (g, w) in got.iter().zip(want.iter()) { + assert!( + approx::relative_eq!(g, w, epsilon = 1e-4), + "dim {dimension}: batch {g} != per-vector {w}" + ); + } + } + + /// The per-batch `#[target_feature]` kernels are only reached on sub-AVX2 + /// builds or AVX-512 hosts, so call them directly to cover them. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + fn check_l2_batch_kernel(kernel: BatchKernel) { + for dimension in [8_usize, 16, 40] { + let num_vectors = 3; + let x: Vec = (0..dimension).map(|i| (i as f32) * 0.5 + 1.0).collect(); + let batch: Vec = (0..dimension * num_vectors) + .map(|i| ((i % 7) as f32) + 1.0) + .collect(); + + let mut got = vec![0.0; num_vectors]; + unsafe { kernel(&x, &batch, dimension, &mut got) }; + assert_eq!(got.len(), num_vectors); + for (chunk, &g) in batch.chunks_exact(dimension).zip(got.iter()) { + let want = l2_scalar::(&x, chunk); + assert!( + approx::relative_eq!(g, want, epsilon = 1e-4), + "dim {dimension}: kernel {g} != scalar {want}" + ); + } + } + } + + // The runtime-dispatch batch kernels only exist in sub-avx2+fma builds + // (see `x86::l2_batch_f32_avx512`), so gate their tests the same way. + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_l2_batch_avx_fma_matches_scalar() { + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } + check_l2_batch_kernel(x86::l2_batch_f32_avx_fma); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_l2_batch_avx_matches_scalar() { + if !std::is_x86_feature_detected!("avx") { + return; + } + check_l2_batch_kernel(x86::l2_batch_f32_avx); + } + + #[cfg(all( + target_arch = "x86_64", + not(all(target_feature = "avx2", target_feature = "fma")) + ))] + #[test] + fn test_l2_batch_avx512_matches_scalar() { + if !std::is_x86_feature_detected!("avx512f") { + return; + } + check_l2_batch_kernel(x86::l2_batch_f32_avx512); + } } diff --git a/rust/lance-linalg/src/distance/l2_u8.rs b/rust/lance-linalg/src/distance/l2_u8.rs index 1b111f91338..efcff21dcb4 100644 --- a/rust/lance-linalg/src/distance/l2_u8.rs +++ b/rust/lance-linalg/src/distance/l2_u8.rs @@ -3,9 +3,10 @@ //! Unsigned int8 squared L2 distance with runtime-dispatched SIMD backends. //! -//! Computes `Σ(a[i] - b[i])²` for u8 slices, returning a u32 result. -//! Used by Scalar Quantization (SQ) distance computation where both L2 -//! and Cosine metric types operate on quantized u8 codes. +//! Computes `Σ(a[i] - b[i])²` for u8 slices. The u32 entry points return the +//! low 32 bits, while [`l2_u8_u64`] chunks those kernels to produce the full +//! result used by Scalar Quantization (SQ), where both L2 and Cosine metric +//! types operate on quantized u8 codes. //! //! Backends (selected at runtime, best available wins): //! 1. scalar — portable reference, also used for tails @@ -22,14 +23,18 @@ use std::sync::OnceLock; +use super::{U8_U32_ACCUMULATOR_MAX_LEN, assert_equal_lengths}; + /// Portable scalar u8 squared L2 distance, also used for SIMD tail elements. +/// +/// The result is the low 32 bits of the exact squared distance. Use +/// [`l2_u8_u64`] when the full result is required. #[inline] pub fn l2_u8_scalar(a: &[u8], b: &[u8]) -> u32 { - debug_assert_eq!(a.len(), b.len()); - a.iter() - .zip(b.iter()) - .map(|(&x, &y)| (x.abs_diff(y) as u32).pow(2)) - .sum() + assert_equal_lengths(a.len(), b.len()); + a.iter().zip(b.iter()).fold(0, |sum, (&x, &y)| { + sum.wrapping_add((x.abs_diff(y) as u32).pow(2)) + }) } #[cfg(target_arch = "x86_64")] @@ -80,7 +85,7 @@ mod x86 { // Scalar tail while i < n { let d = a[i].abs_diff(b[i]) as u32; - result += d * d; + result = result.wrapping_add(d * d); i += 1; } result @@ -119,7 +124,7 @@ mod x86 { // Scalar tail while i < n { let d = a[i].abs_diff(b[i]) as u32; - result += d * d; + result = result.wrapping_add(d * d); i += 1; } result @@ -143,21 +148,59 @@ fn select_backend() -> L2U8Fn { if is_x86_feature_detected!("avx2") { return |a, b| unsafe { x86::l2_u8_avx2(a, b) }; } + // AvxFma and Avx hosts (AMD Piledriver / Steamroller, Intel Sandy + // Bridge / Ivy Bridge) fall through to scalar: the AVX2 inner uses + // AVX2 integer ops (`vpsubusb` / `vpmaddwd`) which neither AVX nor + // AVX+FMA provides. } l2_u8_scalar } /// Dispatched u8 squared L2 distance, selecting the best available SIMD backend. +/// +/// The result is the low 32 bits of the exact squared distance. Use +/// [`l2_u8_u64`] when the full result is required. #[inline] pub fn l2_u8(a: &[u8], b: &[u8]) -> u32 { + assert_equal_lengths(a.len(), b.len()); (DISPATCH.get_or_init(select_backend))(a, b) } +/// Calculates the exact u8 squared L2 distance with a u64 accumulator. +/// +/// This retains the runtime-selected SIMD kernel and widens its result between +/// chunks that are guaranteed to fit in a u32. +/// +/// # Example +/// +/// ``` +/// use lance_linalg::distance::l2_u8::l2_u8_u64; +/// +/// assert_eq!(l2_u8_u64(&[10, 20], &[7, 21]), 10); +/// ``` +#[inline] +pub fn l2_u8_u64(a: &[u8], b: &[u8]) -> u64 { + assert_equal_lengths(a.len(), b.len()); + if a.len() <= U8_U32_ACCUMULATOR_MAX_LEN { + return l2_u8(a, b) as u64; + } + a.chunks(U8_U32_ACCUMULATOR_MAX_LEN) + .zip(b.chunks(U8_U32_ACCUMULATOR_MAX_LEN)) + .map(|(a, b)| l2_u8(a, b) as u64) + .sum() +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn rejects_mismatched_lengths() { + assert!(std::panic::catch_unwind(|| l2_u8_scalar(&[1, 2], &[1])).is_err()); + assert!(std::panic::catch_unwind(|| l2_u8(&[1, 2], &[1])).is_err()); + } + fn fill_random(buf: &mut [u8], seed: &mut u32) { for slot in buf.iter_mut() { *seed = seed.wrapping_mul(1103515245).wrapping_add(12345); @@ -269,4 +312,21 @@ mod tests { assert_eq!(l2_u8(&[0], &[255]), 65025); assert_eq!(l2_u8(&[255], &[0]), 65025); } + + #[test] + fn overflow_is_backend_independent_and_wide_result_is_exact() { + let len = U8_U32_ACCUMULATOR_MAX_LEN + 1; + let a = vec![u8::MAX; len]; + let b = vec![0; len]; + let exact = u8::MAX as u64 * u8::MAX as u64 * len as u64; + + check_all_backends(&a, &b, "u32 overflow"); + assert_eq!(l2_u8_scalar(&a, &b), exact as u32); + assert_eq!(l2_u8_u64(&a, &b), exact); + assert_eq!( + crate::distance::l2::l2_distance_uint_scalar(&a, &b), + exact as f32 + ); + assert_eq!(crate::distance::l2::l2::(&a, &b), exact as f32); + } } diff --git a/rust/lance-linalg/src/distance/norm_l2.rs b/rust/lance-linalg/src/distance/norm_l2.rs index b1daf85ab3b..91294dda8c4 100644 --- a/rust/lance-linalg/src/distance/norm_l2.rs +++ b/rust/lance-linalg/src/distance/norm_l2.rs @@ -3,15 +3,13 @@ use std::{iter::Sum, ops::AddAssign}; -use arrow_array::FixedSizeListArray; use arrow_array::cast::AsArray; use arrow_array::types::{Float16Type, Float32Type, Float64Type}; -use arrow_schema::DataType; +use arrow_array::{FixedSizeListArray, Float32Array}; +use arrow_schema::{ArrowError, DataType}; use half::{bf16, f16}; #[allow(unused_imports)] -use lance_core::utils::cpu::SIMD_SUPPORT; -#[cfg(feature = "fp16kernels")] -use lance_core::utils::cpu::SimdSupport; +use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; use num_traits::{AsPrimitive, Float, Num}; /// L2 normalization @@ -64,7 +62,7 @@ impl Normalize for f16 { kernel::norm_l2_f16_avx512(vector.as_ptr(), vector.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "x86_64"))] - SimdSupport::Avx2 => unsafe { + SimdSupport::Avx2 | SimdSupport::Avx512 => unsafe { kernel::norm_l2_f16_avx2(vector.as_ptr(), vector.len() as u32) }, #[cfg(all(feature = "fp16kernels", target_arch = "loongarch64"))] @@ -75,6 +73,9 @@ impl Normalize for f16 { SimdSupport::Lsx => unsafe { kernel::norm_l2_f16_lsx(vector.as_ptr(), vector.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the f16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => norm_l2_impl::(vector), } } @@ -126,6 +127,9 @@ impl Normalize for bf16 { SimdSupport::Lsx => unsafe { bf16_kernel::norm_l2_bf16_lsx(vector.as_ptr(), vector.len() as u32) }, + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the bf16 C kernels are compiled with `-march=haswell` minimum + // (AVX2), so they cannot run on AVX-only or AVX+FMA hosts. _ => norm_l2_impl::(vector), } } @@ -134,10 +138,40 @@ impl Normalize for bf16 { impl Normalize for f32 { #[inline] fn norm_l2(vector: &[Self]) -> f32 { - norm_l2_impl::(vector) + norm_l2_f32_dispatched(vector) + } +} + +/// L2 norm for f32, runtime-dispatched via `SIMD_SUPPORT` on x86_64 +/// (AVX-512 / AVX2+FMA / AVX+FMA / AVX / scalar). Non-x86 uses the +/// auto-vectorised scalar loop. +#[inline] +fn norm_l2_f32_dispatched(vector: &[f32]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + x86::norm_l2_f32_avx512(vector) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f32_avx_fma(vector) }, + SimdSupport::Avx => unsafe { x86::norm_l2_f32_avx(vector) }, + _ => norm_l2_f32_scalar(vector), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + norm_l2_f32_scalar(vector) } } +/// Portable scalar L2 norm for f32. Used as the x86_64 fallback when no +/// AVX2 is detected, and as the only path on non-x86 architectures. The +/// `LANES = 16` chunking matches the explicit-SIMD inner kernels above. +#[inline] +fn norm_l2_f32_scalar(vector: &[f32]) -> f32 { + norm_l2_impl::(vector) +} + impl Normalize for f64 { #[inline] fn norm_l2(vector: &[Self]) -> f32 { @@ -145,11 +179,166 @@ impl Normalize for f64 { } } -/// Explicit SIMD implementation of L2 norm for f64. +/// L2 norm for f64. Runtime-dispatched to the best available backend. /// -/// Two-level unrolling: f64x8 main loop, f64x4 remainder, scalar tail. +/// On x86_64, dispatches via `SIMD_SUPPORT` to a native AVX-512 inner kernel +/// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4), an AVX2 + FMA kernel +/// (Haswell+), an AVX + FMA kernel (AMD Piledriver / Steamroller), an +/// AVX-only kernel (Intel Sandy Bridge / Ivy Bridge), or a portable scalar +/// fallback. The per-tier inner functions each carry their own +/// `#[target_feature]` so they stay correct under any compile baseline. +/// On aarch64 and loongarch64, the SIMD primitives in `crate::simd::f64` +/// are unconditionally backed by NEON / LSX-LASX respectively, so no +/// runtime gate is required. #[inline] pub fn norm_l2_f64_simd(vector: &[f64]) -> f32 { + #[cfg(target_arch = "x86_64")] + { + match *SIMD_SUPPORT { + SimdSupport::Avx512 | SimdSupport::Avx512FP16 => unsafe { + x86::norm_l2_f64_avx512(vector) + }, + SimdSupport::Avx2 | SimdSupport::AvxFma => unsafe { x86::norm_l2_f64_avx_fma(vector) }, + SimdSupport::Avx => unsafe { x86::norm_l2_f64_avx(vector) }, + _ => norm_l2_f64_scalar(vector), + } + } + #[cfg(not(target_arch = "x86_64"))] + { + norm_l2_f64_simd_other(vector) + } +} + +/// Portable scalar L2 norm. Used as the x86_64 fallback when no AVX2 is +/// detected, and exposed for cross-backend parity testing. +#[cfg(target_arch = "x86_64")] +#[inline] +fn norm_l2_f64_scalar(vector: &[f64]) -> f32 { + vector.iter().map(|v| v * v).sum::().sqrt() as f32 +} + +#[cfg(target_arch = "x86_64")] +mod x86 { + use std::arch::x86_64::*; + + use crate::simd::f64::{f64x4, f64x8}; + use crate::simd::x86::hsum256_ps; + use crate::simd::{FloatSimd, SIMD}; + + /// AVX-512 path for f64: 8-wide `__m512d` with `vfmadd231pd` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn norm_l2_f64_avx512(vector: &[f64]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm512_setzero_pd(); + for i in (0..unrolled_len).step_by(8) { + let v = _mm512_loadu_pd(vector.as_ptr().add(i)); + acc = _mm512_fmadd_pd(v, v, acc); + } + + let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (_mm512_reduce_add_pd(acc) + tail).sqrt() as f32 + } + + /// AVX + FMA path for f64. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn norm_l2_f64_avx_fma(vector: &[f64]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc8 = f64x8::zeros(); + for i in (0..unrolled_len).step_by(8) { + let v = f64x8::load_unaligned(vector.as_ptr().add(i)); + acc8.multiply_add(v, v); + } + + let aligned_len = dim / 4 * 4; + let mut acc4 = f64x4::zeros(); + for i in (unrolled_len..aligned_len).step_by(4) { + let v = f64x4::load_unaligned(vector.as_ptr().add(i)); + acc4.multiply_add(v, v); + } + + let tail: f64 = vector[aligned_len..].iter().map(|&v| v * v).sum(); + (acc8.reduce_sum() + acc4.reduce_sum() + tail).sqrt() as f32 + } + + /// AVX-only path for f64 (no FMA): `_mm256_mul_pd` + `_mm256_add_pd` per iteration for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn norm_l2_f64_avx(vector: &[f64]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 4 * 4; + + let mut acc = _mm256_setzero_pd(); + for i in (0..unrolled_len).step_by(4) { + let v = _mm256_loadu_pd(vector.as_ptr().add(i)); + acc = _mm256_add_pd(acc, _mm256_mul_pd(v, v)); + } + + // Horizontal sum of __m256d -> f64. Two pairwise adds across lanes. + let lo = _mm256_castpd256_pd128(acc); + let hi = _mm256_extractf128_pd(acc, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + let acc_sum = _mm_cvtsd_f64(sum64); + + let tail: f64 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (acc_sum + tail).sqrt() as f32 + } + + /// AVX-512 path for f32: 16-wide `__m512` with `vfmadd231ps` per iteration. + #[target_feature(enable = "avx512f")] + pub unsafe fn norm_l2_f32_avx512(vector: &[f32]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 16 * 16; + + let mut acc = _mm512_setzero_ps(); + for i in (0..unrolled_len).step_by(16) { + let v = _mm512_loadu_ps(vector.as_ptr().add(i)); + acc = _mm512_fmadd_ps(v, v, acc); + } + + let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (_mm512_reduce_add_ps(acc) + tail).sqrt() + } + + /// AVX + FMA path for f32. Covers both AvxFma and AVX2 dispatch (body uses no AVX2-specific intrinsics). + #[target_feature(enable = "avx,fma")] + pub unsafe fn norm_l2_f32_avx_fma(vector: &[f32]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let v = _mm256_loadu_ps(vector.as_ptr().add(i)); + acc = _mm256_fmadd_ps(v, v, acc); + } + + let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (hsum256_ps(acc) + tail).sqrt() + } + + /// AVX-only path for f32 (no FMA): `_mm256_mul_ps` + `_mm256_add_ps` per iteration for Sandy/Ivy Bridge. + #[target_feature(enable = "avx")] + pub unsafe fn norm_l2_f32_avx(vector: &[f32]) -> f32 { + let dim = vector.len(); + let unrolled_len = dim / 8 * 8; + + let mut acc = _mm256_setzero_ps(); + for i in (0..unrolled_len).step_by(8) { + let v = _mm256_loadu_ps(vector.as_ptr().add(i)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(v, v)); + } + + let tail: f32 = vector[unrolled_len..].iter().map(|&v| v * v).sum(); + (hsum256_ps(acc) + tail).sqrt() + } +} + +#[cfg(not(target_arch = "x86_64"))] +#[inline] +fn norm_l2_f64_simd_other(vector: &[f64]) -> f32 { use crate::simd::f64::{f64x4, f64x8}; use crate::simd::{FloatSimd, SIMD}; @@ -214,6 +403,47 @@ pub fn norm_l2(vector: &[T]) -> f32 { T::norm_l2(vector) } +/// L2 norm of every vector in a [FixedSizeListArray], Returns one norm per row. +pub fn norm_l2_fsl(fsl: &FixedSizeListArray) -> crate::Result { + let dim = fsl.value_length() as usize; + if dim == 0 { + return Err(ArrowError::InvalidArgumentError( + "cannot compute L2 norms of a FixedSizeListArray with value_length 0".into(), + )); + } + let values = fsl.values(); + Ok(match fsl.value_type() { + DataType::Float16 => values + .as_primitive::() + .values() + .chunks_exact(dim) + .map(::norm_l2) + .collect(), + DataType::Float32 => values + .as_primitive::() + .values() + .chunks_exact(dim) + .map(::norm_l2) + .collect(), + DataType::Float64 => values + .as_primitive::() + .values() + .chunks_exact(dim) + .map(::norm_l2) + .collect(), + value_type => { + return Err(ArrowError::SchemaError(format!( + "norm_l2_fsl only supports float16/float32/float64 vectors, got: {value_type}" + ))); + } + }) +} + +/// Squared L2 norm of every vector in a [FixedSizeListArray]. +/// +/// Each square is accumulated in `f32` (or wider) rather than in the element +/// type: squaring an `f16` saturates to `inf` at `|x| >= 256` and to zero at +/// `|x| <= 1.726e-4`. pub fn norm_squared_fsl(fsl: &FixedSizeListArray) -> Vec { let dim = fsl.value_length() as usize; match fsl.value_type() { @@ -222,7 +452,14 @@ pub fn norm_squared_fsl(fsl: &FixedSizeListArray) -> Vec { .as_primitive::() .values() .chunks_exact(dim) - .map(|v| v.iter().map(|v| v * v).sum::().to_f32()) + .map(|v| { + v.iter() + .map(|v| { + let v = v.to_f32(); + v * v + }) + .sum::() + }) .collect::>(), DataType::Float32 => fsl .values() @@ -247,7 +484,12 @@ pub fn norm_squared_fsl(fsl: &FixedSizeListArray) -> Vec { #[cfg(test)] mod tests { use super::*; - use crate::test_utils::{arbitrary_bf16, arbitrary_f16, arbitrary_f32, arbitrary_f64}; + use crate::test_utils::{ + arbitrary_bf16, arbitrary_f16, arbitrary_f32, arbitrary_f64, dimension_shard, + run_vector_proptest, + }; + use arrow_array::{Float16Array, Float64Array, UInt8Array}; + use lance_arrow::FixedSizeListArrayExt; use num_traits::ToPrimitive; use proptest::prelude::*; @@ -271,6 +513,36 @@ mod tests { Ok(()) } + #[rstest::rstest] + fn test_l2_norm_f32( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_proptest(arbitrary_f32, dimension_shard(shard), |data| { + do_norm_l2_test(&data) + }); + } + + #[rstest::rstest] + fn test_l2_norm_f64( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_proptest(arbitrary_f64, dimension_shard(shard), |data| { + do_norm_l2_test(&data) + }); + } + + #[rstest::rstest] + fn test_l2_norm_f32_scalar_simd_parity( + #[values(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)] shard: usize, + ) { + run_vector_proptest(arbitrary_f32, dimension_shard(shard), |data| { + let scalar = data.iter().map(|&v| (v as f64).powi(2)).sum::().sqrt() as f32; + let simd = ::norm_l2(&data); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-3)); + Ok(()) + }); + } + proptest::proptest! { #[test] fn test_l2_norm_f16(data in prop::collection::vec(arbitrary_f16(), 4..4048)) { @@ -282,14 +554,212 @@ mod tests { do_norm_l2_test(&data)?; } + /// Cross-backend parity: scalar fallback must match the dispatched + /// SIMD path within numerical tolerance. Exercises `norm_l2_f64_scalar` + /// directly so the runtime fallback is exercised even on AVX2-capable + /// CI hosts. + #[cfg(target_arch = "x86_64")] #[test] - fn test_l2_norm_f32(data in prop::collection::vec(arbitrary_f32(), 4..4048)){ - do_norm_l2_test(&data)?; + fn test_l2_norm_f64_scalar_simd_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + let scalar = norm_l2_f64_scalar(&data); + let simd = norm_l2_f64_simd(&data); + prop_assert!(approx::relative_eq!(scalar, simd, max_relative = 1e-6)); } + /// AVX-512-direct parity: explicitly compares the scalar fallback + /// against the native AVX-512 inner kernel on AVX-512F-capable hosts + /// (Skylake-X+, Ice Lake, Sapphire Rapids, Zen 4). Early-returns on + /// hosts without AVX-512F so the test stays portable; CI runners with + /// AVX-512F exercise the `_mm512_*` path. + #[cfg(target_arch = "x86_64")] #[test] - fn test_l2_norm_f64(data in prop::collection::vec(arbitrary_f64(), 4..4048)){ - do_norm_l2_test(&data)?; + fn test_l2_norm_f64_scalar_vs_avx512_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = norm_l2_f64_scalar(&data); + let avx512 = unsafe { x86::norm_l2_f64_avx512(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-6)); + } + + /// AVX + FMA-direct parity for the f64 L2-norm kernel. Covers the + /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on + /// hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_norm_f64_scalar_vs_avx_fma_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = norm_l2_f64_scalar(&data); + let avx_fma = unsafe { x86::norm_l2_f64_avx_fma(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-6)); + } + + /// AVX-only-direct parity for the f64 L2-norm kernel. Covers the + /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA). + /// Early-returns on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_l2_norm_f64_scalar_vs_avx_parity( + data in prop::collection::vec(arbitrary_f64(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = norm_l2_f64_scalar(&data); + let avx = unsafe { x86::norm_l2_f64_avx(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-6)); + } + + /// AVX-512-direct parity for the f32 L2-norm kernel. Explicitly + /// compares the scalar fallback against the native AVX-512 inner + /// kernel on AVX-512F-capable hosts. Early-returns on hosts without + /// AVX-512F; CI runners with AVX-512F exercise the `_mm512_*` path. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_norm_l2_f32_scalar_vs_avx512_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx512f") { + return Ok(()); + } + let scalar = norm_l2_f32_scalar(&data); + let avx512 = unsafe { x86::norm_l2_f32_avx512(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx512, max_relative = 1e-3)); + } + + /// AVX + FMA-direct parity for the f32 L2-norm kernel. Covers the + /// AMD Piledriver / Steamroller / FX-7500 tier. Early-returns on + /// hosts without both AVX and FMA. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_norm_l2_f32_scalar_vs_avx_fma_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + if !(std::is_x86_feature_detected!("avx") && std::is_x86_feature_detected!("fma")) { + return Ok(()); + } + let scalar = norm_l2_f32_scalar(&data); + let avx_fma = unsafe { x86::norm_l2_f32_avx_fma(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx_fma, max_relative = 1e-3)); + } + + /// AVX-only-direct parity for the f32 L2-norm kernel. Covers the + /// Intel Sandy Bridge / Ivy Bridge tier (AVX without FMA). + /// Early-returns on hosts without AVX. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_norm_l2_f32_scalar_vs_avx_parity( + data in prop::collection::vec(arbitrary_f32(), 4..4048) + ) { + if !std::is_x86_feature_detected!("avx") { + return Ok(()); + } + let scalar = norm_l2_f32_scalar(&data); + let avx = unsafe { x86::norm_l2_f32_avx(&data) }; + prop_assert!(approx::relative_eq!(scalar, avx, max_relative = 1e-3)); + } + } + + #[test] + fn test_norm_l2_fsl_f32() { + let values = Float32Array::from(vec![3.0, 4.0, 6.0, 8.0]); + let fsl = FixedSizeListArray::try_new_from_values(values, 2).unwrap(); + + let norms = norm_l2_fsl(&fsl).unwrap(); + assert_eq!(norms.len(), 2); + assert!(approx::relative_eq!( + norms.value(0), + 5.0, + max_relative = 1e-6 + )); + assert!(approx::relative_eq!( + norms.value(1), + 10.0, + max_relative = 1e-6 + )); + } + + #[test] + fn test_norm_l2_fsl_f16_and_f64_match_norm_l2() { + // The FSL helper must agree with the per-vector `norm_l2` kernel for + // every supported value type, since callers cache these norms and feed + // them to `cosine_with_norms`. + let f16_vals: Vec = [3.0f32, 4.0, 6.0, 8.0] + .iter() + .map(|&v| f16::from_f32(v)) + .collect(); + let f16_fsl = + FixedSizeListArray::try_new_from_values(Float16Array::from(f16_vals), 2).unwrap(); + let f16_norms = norm_l2_fsl(&f16_fsl).unwrap(); + assert_eq!( + f16_norms.value(0), + norm_l2(&[f16::from_f32(3.0), f16::from_f32(4.0)]) + ); + assert_eq!( + f16_norms.value(1), + norm_l2(&[f16::from_f32(6.0), f16::from_f32(8.0)]) + ); + + let f64_fsl = FixedSizeListArray::try_new_from_values( + Float64Array::from(vec![3.0, 4.0, 6.0, 8.0]), + 2, + ) + .unwrap(); + let f64_norms = norm_l2_fsl(&f64_fsl).unwrap(); + assert_eq!(f64_norms.value(0), norm_l2(&[3.0f64, 4.0])); + assert_eq!(f64_norms.value(1), norm_l2(&[6.0f64, 8.0])); + } + + #[test] + fn test_norm_l2_fsl_rejects_unsupported_value_type() { + let fsl = FixedSizeListArray::try_new_from_values(UInt8Array::from(vec![1u8, 2, 3, 4]), 2) + .unwrap(); + let err = norm_l2_fsl(&fsl).unwrap_err().to_string(); + assert!(err.contains("float16/float32/float64"), "got: {err}"); + } + + /// `norm_squared_fsl` must accumulate in a type wider than the element type. + /// `f16 * f16` rounds each square back to `f16`, which saturates to `inf` at + /// `|x| >= 256` and to zero at `|x| <= 1.726e-4`. + #[test] + fn test_norm_squared_fsl_f16_accumulates_wide() { + use arrow_array::Float16Array; + use arrow_schema::Field; + use std::sync::Arc; + + // dim 2: row 0 overflows at the square, row 1 underflows at the square. + let raw = [256.0f32, 0.0, 1e-4, 1e-4]; + let values = Float16Array::from_iter_values(raw.map(f16::from_f32)); + let field = Arc::new(Field::new("item", DataType::Float16, true)); + let fsl = FixedSizeListArray::try_new(field, 2, Arc::new(values), None).unwrap(); + + let got = norm_squared_fsl(&fsl); + // Independent reference: square and sum the same f16 inputs in f64. + let expected = raw + .chunks(2) + .map(|c| { + c.iter() + .map(|&x| { + let x = f16::from_f32(x).to_f64(); + x * x + }) + .sum::() + }) + .collect::>(); + + for (row, (&got, &want)) in got.iter().zip(expected.iter()).enumerate() { + assert!( + approx::relative_eq!(got as f64, want, max_relative = 1e-3), + "row {row}: got {got}, want {want}" + ); } } } diff --git a/rust/lance-linalg/src/kernels.rs b/rust/lance-linalg/src/kernels.rs index 1fe485c7157..ad46b7bac4b 100644 --- a/rust/lance-linalg/src/kernels.rs +++ b/rust/lance-linalg/src/kernels.rs @@ -16,6 +16,7 @@ use arrow_array::{ }, }; use arrow_schema::{ArrowError, DataType}; +use half::{bf16, f16}; use num_traits::AsPrimitive; use num_traits::{Float, Num, bounds::Bounded}; @@ -135,19 +136,72 @@ pub fn argmin_opt( argmin_value_opt(iter).map(|(idx, _)| idx) } +/// The accumulator used to sum squares when normalizing a `T` vector. +/// +/// A type narrow enough that its squares leave its own range, and for which a +/// wider float exists, accumulates in that wider type: squaring an `f16` +/// saturates to `inf` at `|x| >= 256` and to zero at `|x| <= 1.726e-4`, so an +/// ordinary `f16` vector would otherwise normalize to all-zero or all-`inf`. +/// `f32` and `f64` accumulate in themselves — the same saturation still exists at +/// the extremes of their own range (an `f32` square overflows above `1.8447e19`), +/// but widening `f32` would perturb the output of existing f32 vectors by a few +/// ulp (about 10 at dimension 768, growing as sqrt(dim)), so it is deliberately +/// left alone; `f64` has nothing wider to widen to. +/// +/// The width relation is a contract on the implementor, not something the bounds +/// can express. Tying the accumulator to the element type here still buys two +/// things over passing it in: the accumulator cannot drift between the three +/// dispatch sites, and no call site can pick the wrong one. +pub trait Normalizable: Float + AsPrimitive { + /// Must be at least as wide as `Self` in both exponent and mantissa. + type Acc: Float + Sum + AsPrimitive + AsPrimitive; +} + +/// `f16` squares leave its own range, so it accumulates in `f32` — which has +/// 2^96 of headroom over `f16::MAX` squared. +impl Normalizable for f16 { + type Acc = f32; +} + +/// `bf16` has the same exponent range as `f32` (both 8 bits), so widening to +/// `f32` would buy no headroom for squaring — `bf16(1e20)` squared already +/// overflows `f32`. It needs `f64`. +impl Normalizable for bf16 { + type Acc = f64; +} + +impl Normalizable for f32 { + type Acc = Self; +} + +impl Normalizable for f64 { + type Acc = Self; +} + /// L2 normalize a vector. /// -/// Returns an iterator of normalized values. -pub fn normalize>( - v: &[T], -) -> (impl Iterator + '_, f32) { - let l2_norm = v.iter().map(|x| x.powi(2)).sum::().sqrt(); - (v.iter().map(move |&x| x / l2_norm), l2_norm.as_()) +/// Returns an iterator of normalized values, and the norm as `f32`. +/// +/// The sum of squares is accumulated in [`Normalizable::Acc`], which is wider +/// than `T` where `T` alone would overflow. +pub fn normalize(v: &[T]) -> (impl Iterator + '_, f32) { + let l2_norm = v + .iter() + .map(|x| { + let x: T::Acc = x.as_(); + x * x + }) + .sum::() + .sqrt(); + ( + v.iter().map(move |&x| (x.as_() / l2_norm).as_()), + l2_norm.as_(), + ) } fn do_normalize_arrow(arr: &dyn Array) -> Result<(ArrayRef, f32)> where - ::Native: Float + Sum + AsPrimitive, + T::Native: Normalizable, { let v = arr.as_primitive::(); let (iter, l2_norm) = normalize(v.values()); @@ -171,7 +225,7 @@ pub fn normalize_arrow(v: &dyn Array) -> Result<(ArrayRef, f32)> { fn do_normalize_fsl(fsl: &FixedSizeListArray) -> Result where - T::Native: Float + Sum + AsPrimitive, + T::Native: Normalizable, { let dim = fsl.value_length() as usize; let norm_arr = PrimitiveArray::::from_iter_values( @@ -214,7 +268,7 @@ fn do_normalize_fsl_inplace( fsl: FixedSizeListArray, ) -> Result where - T::Native: Float + Sum + AsPrimitive, + T::Native: Normalizable, { let dim = fsl.value_length() as usize; let (field, size, values_array, nulls) = fsl.into_parts(); @@ -233,9 +287,17 @@ where match prim.into_builder() { Ok(mut builder) => { for chunk in builder.values_slice_mut().chunks_mut(dim) { - let l2_norm = chunk.iter().map(|x| x.powi(2)).sum::().sqrt(); + // Accumulate in the wider type; see [`Normalizable`]. + let l2_norm = chunk + .iter() + .map(|x| { + let x: ::Acc = x.as_(); + x * x + }) + .sum::<::Acc>() + .sqrt(); for x in chunk.iter_mut() { - *x = *x / l2_norm; + *x = (x.as_() / l2_norm).as_(); } } FixedSizeListArray::try_new(field, size, Arc::new(builder.finish()), nulls) @@ -323,10 +385,12 @@ mod tests { use approx::assert_relative_eq; use arrow_array::{ - Float32Array, Int8Array, Int16Array, LargeStringArray, StringArray, UInt8Array, UInt32Array, + Float16Array, Float32Array, Float64Array, Int8Array, Int16Array, LargeStringArray, + StringArray, UInt8Array, UInt32Array, }; use arrow_buffer::NullBuffer; use arrow_schema::Field; + use half::f16; #[test] fn test_argmax() { @@ -433,6 +497,204 @@ mod tests { assert_relative_eq!(1.0, normalized.iter().map(|&x| x.powi(2)).sum::()); } + /// The accumulator must not be *narrower* than the element type either. + /// Accumulating f64 in f32 overflows above `|x| = 1.8447e19` (where f64 has + /// headroom to 1.34e154), collapses to a zero norm at or below `|x| = 2^-75` + /// (2.647e-23), and costs ~29 bits of mantissa on every ordinary vector. + #[test] + fn test_normalize_f64_accumulates_wide() { + // Range: each case is finite and correctly normalizable in f64, but + // overflows or underflows an f32 accumulator. + let range_cases: &[(&str, Vec)] = &[ + ("square_overflows", vec![1e20, 0.0]), + ("sum_overflows", vec![1e19; 12]), + ("square_underflows", vec![1e-25, 1e-25]), + ("element_underflows", vec![1e-100, 1e-100]), + ]; + for (name, v) in range_cases { + // Cover all three public entry points: each has its own `match` over + // the element type, so a dispatch cell could regress on its own. + for out in [ + ("normalize_arrow", normalize_f64(v)), + ("normalize_fsl", normalize_f64_fsl(v, false)), + ("normalize_fsl_owned", normalize_f64_fsl(v, true)), + ] { + let (entry, out) = out; + let norm = out.iter().map(|x| x * x).sum::().sqrt(); + assert!( + approx::relative_eq!(norm, 1.0, max_relative = 1e-9), + "{entry} / {name}: normalized norm {norm} != 1, output {out:?}" + ); + } + } + + // Precision: an f32 accumulator would round the output to f32, leaving + // a relative error around f32::EPSILON (~1.2e-7). + let v = vec![1.0_f64, 2.0, 3.0]; + let expected_norm = 14.0_f64.sqrt(); + let out = normalize_f64(&v); + for (i, (&got, &raw)) in out.iter().zip(v.iter()).enumerate() { + let want = raw / expected_norm; + assert!( + approx::relative_eq!(got, want, max_relative = 1e-15), + "element {i}: got {got:.17}, want {want:.17}" + ); + } + } + + /// Normalize an `f64` slice through the public Arrow entry point, so the + /// test exercises the accumulator `normalize_arrow` actually selects. + fn normalize_f64(v: &[f64]) -> Vec { + let arr = Float64Array::from(v.to_vec()); + let (out, _) = normalize_arrow(&arr).unwrap(); + out.as_primitive::().values().to_vec() + } + + /// Same, through the FSL entry points. `owned` selects + /// [`normalize_fsl_owned`], whose freshly built array takes the in-place + /// branch of `do_normalize_fsl_inplace`. + fn normalize_f64_fsl(v: &[f64], owned: bool) -> Vec { + let values = Float64Array::from(v.to_vec()); + let field = Arc::new(Field::new("item", DataType::Float64, true)); + let fsl = + FixedSizeListArray::try_new(field, v.len() as i32, Arc::new(values), None).unwrap(); + let out = if owned { + normalize_fsl_owned(fsl).unwrap() + } else { + normalize_fsl(&fsl).unwrap() + }; + out.values().as_primitive::().values().to_vec() + } + + /// `normalize` must accumulate the sum of squares in a type wider than the + /// element type. `f16::powi` rounds each square back to `f16`, which + /// saturates to `inf` at `|x| >= 256` and to zero at `|x| <= 1.726e-4`, so an + /// ordinary vector normalizes to all-zero or all-`inf`. + #[test] + fn test_normalize_f16_accumulates_wide() { + let cases: &[(&str, &[f32])] = &[ + // A single element whose square leaves the f16 range. + ("square_overflows", &[256.0, 0.0]), + // No element overflows, but the sum of squares does. + ("sum_overflows", &[100.0; 7]), + // Every square rounds to zero, so the norm is zero and x/0 is inf. + ("square_underflows", &[1e-4; 8]), + ]; + for (name, input) in cases { + let v = input.iter().map(|&x| f16::from_f32(x)).collect::>(); + // Independent reference: accumulate the same f16 inputs in f64. + let expected = v + .iter() + .map(|x| x.to_f64() * x.to_f64()) + .sum::() + .sqrt(); + let (normalized, norm) = normalize(&v); + assert!( + approx::relative_eq!(norm, expected as f32, max_relative = 1e-3), + "{name}: norm {norm} != expected {expected}" + ); + let normalized = normalized.collect::>(); + assert!( + normalized.iter().all(|x| x.is_finite()), + "{name}: non-finite output {normalized:?}" + ); + + // The output must be a unit vector. This is the assertion that pins + // the division: an independent f64 sum of the squares, not a + // comparison against another call into the same code. + let unit = normalized + .iter() + .map(|x| x.to_f64() * x.to_f64()) + .sum::(); + assert!( + approx::relative_eq!(unit, 1.0, max_relative = 1e-2), + "{name}: output is not a unit vector, sum of squares {unit}" + ); + + // Also drive the `normalize_arrow` Float16 arm, so the dispatch cell + // is covered. Equality against the generic path only proves the two + // agree — the unit-norm check above is what proves either is right. + let (out, arrow_norm) = normalize_arrow(&Float16Array::from(v)).unwrap(); + assert!( + approx::relative_eq!(arrow_norm, expected as f32, max_relative = 1e-3), + "{name}: normalize_arrow norm {arrow_norm} != expected {expected}" + ); + let out = out.as_primitive::(); + assert_eq!( + out.values().as_ref(), + normalized.as_slice(), + "{name}: normalize_arrow values differ from the generic path" + ); + } + } + + /// `bf16` shares f32's exponent range, so it needs an `f64` accumulator — + /// `f32` would leave the same overflow the f16 case exists to fix. + #[test] + fn test_normalize_bf16_accumulates_wide() { + let cases: &[(&str, &[f32])] = &[ + ("square_overflows", &[1e20, 0.0]), + ("sum_overflows", &[1e19; 12]), + ("square_underflows", &[1e-25, 1e-25]), + ]; + for (name, input) in cases { + let v = input.iter().map(|&x| bf16::from_f32(x)).collect::>(); + let expected = v + .iter() + .map(|x| x.to_f64() * x.to_f64()) + .sum::() + .sqrt(); + let (normalized, norm) = normalize(&v); + let normalized = normalized.collect::>(); + assert!( + approx::relative_eq!(norm as f64, expected, max_relative = 1e-2), + "{name}: norm {norm} != expected {expected}" + ); + let unit = normalized + .iter() + .map(|x| x.to_f64() * x.to_f64()) + .sum::(); + assert!( + approx::relative_eq!(unit, 1.0, max_relative = 1e-2), + "{name}: output is not a unit vector, sum of squares {unit}" + ); + } + } + + /// Both FSL entry points share the defect, including the in-place path in + /// [`do_normalize_fsl_inplace`], which has its own copy of the expression. + #[test] + fn test_normalize_fsl_f16_accumulates_wide() { + // dim 2, row 0 overflows at the square, row 1 underflows at the square. + let make = || { + let values = + Float16Array::from_iter_values([256.0f32, 0.0, 1e-4, 1e-4].map(f16::from_f32)); + let field = Arc::new(Field::new("item", DataType::Float16, true)); + FixedSizeListArray::try_new(field, 2, Arc::new(values), None).unwrap() + }; + + // `normalize_fsl_owned` gets a freshly built array so the buffer is + // uniquely owned and the in-place branch is the one exercised. + let outputs = [ + ("normalize_fsl", normalize_fsl(&make()).unwrap()), + ("normalize_fsl_owned", normalize_fsl_owned(make()).unwrap()), + ]; + for (label, out) in outputs { + let got = out.values().as_primitive::(); + for (row, chunk) in got.values().chunks(2).enumerate() { + let norm = chunk + .iter() + .map(|x| x.to_f64() * x.to_f64()) + .sum::() + .sqrt(); + assert!( + approx::relative_eq!(norm, 1.0, max_relative = 1e-2), + "{label} row {row}: normalized norm {norm} != 1, values {chunk:?}" + ); + } + } + } + #[test] fn test_normalize_fsl_with_nulls() { // Create test data with nulls diff --git a/rust/lance-linalg/src/simd.rs b/rust/lance-linalg/src/simd.rs index 91dc1c6959d..23298694f75 100644 --- a/rust/lance-linalg/src/simd.rs +++ b/rust/lance-linalg/src/simd.rs @@ -14,11 +14,14 @@ use std::ops::{Add, AddAssign, Mul, Sub, SubAssign}; +pub mod amx_fp16; pub mod dist_table; pub mod f32; pub mod f64; pub mod i32; pub mod u8; +#[cfg(target_arch = "x86_64")] +pub(crate) mod x86; use num_traits::{Float, Num}; use u8::u8x16; diff --git a/rust/lance-linalg/src/simd/amx_fp16.c b/rust/lance-linalg/src/simd/amx_fp16.c new file mode 100644 index 00000000000..7d97844d3f7 --- /dev/null +++ b/rust/lance-linalg/src/simd/amx_fp16.c @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +// AMX-FP16 tile kernels, and the tile-configuration plumbing they share. +// +// Every kernel here computes fp16 x fp16 dot products with TDPFP16PS +// (fp16 x fp16 -> fp32 accumulate). The tile *shapes* differ per kernel and are +// declared next to each one; the code that turns a shape into a loaded +// LDTILECFG image is shared, because the one subtle step in it (the +// dead-store barrier below) fails silently and only on some compilers, so it +// must exist exactly once. +// +// Two kernels: +// * `lance_amx_dot_f16_batch_16` -- one query against 16 candidates, for +// choosing the IVF partitions a query probes (16 centroids per call). +// Ported in spirit from FAISS PR +// facebookresearch/faiss#5235's AMX-BF16 kernel: fp16 and bf16 are both +// 2-byte tile elements consumed by the same `_tile_dp*ps` shape, so only the +// instruction (`_tile_dpbf16ps` -> `_tile_dpfp16ps`) and the element -> +// float conversion (bf16 bit-shift -> real IEEE fp16 via F16C `_cvtsh_ss`) +// differ. Its C tile is fed by three independent (A, B) tile pairs so three +// TDPFP16PS can be in flight at once; see the tile roles below. +// * `lance_amx_dot_f16_gemm` -- an m x n GEMM, for scoring many vectors +// against many centroids at once (k-means assignment). Uses all eight tiles +// as a 2x2 register-blocked accumulator. +// +// ## Tile configuration is reloaded on every call +// +// Each kernel has one compile-time tile shape (`SEARCH_TILES` / `GEMM_TILES`), +// so caching its LDTILECFG is tempting: that reconfiguration costs a few +// hundred cycles, against the ~64 cycles of useful tile work a dim-128 search +// call performs. It is still wrong, because Lance does not own the tile unit: +// LDTILECFG and TILERELEASE are architectural per-logical-processor state, so +// another AMX user on the thread (oneDNN under PyTorch, ONNX Runtime, same +// Python process) can retire or reshape a configuration Lance believes is live +// -- a foreign TILERELEASE leaves the tiles in INIT and the next tile op raises +// #UD, a foreign LDTILECFG silently substitutes wrong shapes. Neither is +// observable from here, and a kernel reached from arbitrary Rust and C cannot +// bound what runs between two of its own calls. +// +// So nothing is cached: every kernel configures the tiles on entry and releases +// them on exit, and pays that per call. Against a GEMM over a whole block of +// vectors it is noise; against a batch-16 search it is most of the call, and is +// spent anyway, because the alternative is a configuration whose validity this +// file has no way to establish. +// +// ## Thread safety +// +// LDTILECFG sets per-logical-processor state, and nothing here is shared +// mutably: the spec tables are `static const` and the 64-byte config *image* is +// built on the calling thread's stack, so one thread's shape can never reach +// another's tile ops -- as with the XTILECFG it is loaded into, which rides in +// the thread's own XSAVE area. That says nothing about what *other* libraries +// on this thread have done to the tile unit, which is what reconfiguring on +// every entry is for. +// +// SAFETY: executing any AMX tile instruction without first (a) confirming the +// amx-tile + amx-fp16 CPUID bits and (b) obtaining XTILEDATA permission from +// the kernel raises SIGILL. Both are the Rust caller's responsibility (see +// `simd/amx_fp16.rs`); `lance_amx_fp16_request_perm` below performs (b). + +// Must precede all includes: exposes the glibc `syscall()` prototype from +// , which -std=c17 otherwise hides. +#define _GNU_SOURCE + +#include +#include +#include +#include + +#ifdef __linux__ +#include +#include +#endif + +// --------------------------------------------------------------------------- +// XTILEDATA permission +// --------------------------------------------------------------------------- + +// Ask the kernel to enable AMX tile data state for this process, via +// arch_prctl(ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA). Returns 0 on success, +// non-zero otherwise. Requesting XTILEDATA (18) implicitly also grants +// XTILECFG (17). Constants per Linux Documentation/arch/x86/xstate.rst. +// +// XTILEDATA is the single dynamically-enabled XSAVE state component backing the +// physical TMM tile registers, shared by every AMX compute instruction +// (TDPBUUD / TDPBF16PS / TDPFP16PS ...). The syscall is idempotent, so +// requesting an already-granted permission is harmless. +int lance_amx_fp16_request_perm(void) { +#ifdef __linux__ + const unsigned long ARCH_REQ_XCOMP_PERM = 0x1023; + const unsigned long XFEATURE_XTILEDATA = 18; + return (int)syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA); +#else + return -1; +#endif +} + +// --------------------------------------------------------------------------- +// Shared tile configuration +// --------------------------------------------------------------------------- + +// Tile configuration kinds. One per distinct tile shape, and the selector +// `lance_amx_tilecfg_image` exposes to the Rust tests so they can pin a shape +// without executing a tile instruction. Kept in sync with the `AMX_CFG_*` +// constants in `simd/amx_fp16.rs`. +#define LANCE_AMX_CFG_SEARCH 0 +#define LANCE_AMX_CFG_GEMM 1 + +// The 64-byte tile configuration image loaded by LDTILECFG +// (`_tile_loadconfig`). Layout per Intel SDM: palette_id, start_row, 14 +// reserved bytes, then a u16 colsb[16] (bytes-per-row) array and a u8 rows[16] +// array. Slots not named by a kernel's spec table stay zero. +// +// 64-byte aligned so the configuration load does not straddle a cache line. +typedef struct __attribute__((packed, aligned(64))) { + uint8_t palette_id; + uint8_t start_row; + uint8_t reserved[14]; + uint16_t colsb[16]; + uint8_t rows[16]; +} lance_amx_tilecfg; + +// The image LDTILECFG reads is exactly 64 bytes; a layout change that altered +// the size would silently feed the instruction garbage. +_Static_assert(sizeof(lance_amx_tilecfg) == 64, + "LDTILECFG image must be exactly 64 bytes"); + +// One tile register's shape. `tmm` is the register index (0..7), `colsb` its +// bytes per row, `rows` its row count. +typedef struct { + uint8_t tmm; + uint8_t rows; + uint16_t colsb; +} lance_amx_tile_spec; + +#define LANCE_AMX_TILE_COUNT(specs) (sizeof(specs) / sizeof((specs)[0])) + +// Fill `cfg` with the LDTILECFG image described by `specs`, without loading it. +// Split from the load so `lance_amx_tilecfg_image` can hand the Rust tests the +// exact bytes a kernel would configure. +static void lance_amx_tilecfg_build(lance_amx_tilecfg *cfg, + const lance_amx_tile_spec *specs, + size_t n) { + memset(cfg, 0, sizeof(*cfg)); + cfg->palette_id = 1; + for (size_t i = 0; i < n; i++) { + cfg->rows[specs[i].tmm] = specs[i].rows; + cfg->colsb[specs[i].tmm] = specs[i].colsb; + } +} + +// Load the tile configuration described by `specs`. Unconditional: see the file +// header for why no state is kept about what is already loaded. +// +// The barrier is not optional: some GCC versions do not model +// _tile_loadconfig as reading the 64-byte cfg image, dead-store-eliminate the +// rows/colsb writes, and load an all-zero (unconfigured) tile shape -> #UD on +// the first tile op (documented in FAISS #5235). Keeping it here, in the one +// function that owns the image, is why kernels do not build configurations +// themselves. Harmless under clang. +static inline void lance_amx_tile_ensure(const lance_amx_tile_spec *specs, + size_t n) { + lance_amx_tilecfg cfg; + lance_amx_tilecfg_build(&cfg, specs, n); + __asm__ volatile("" : : "m"(cfg) : "memory"); + _tile_loadconfig(&cfg); +} + +// Hand the tile unit back at the end of a kernel. Pairs with +// `lance_amx_tile_ensure`, and must run on every exit path of a kernel that +// configured tiles. +// +// Not for Lance's own benefit -- `lance_amx_tile_ensure` reloads on every entry, +// so a stale configuration could never reach one of these kernels either way. It +// is for everyone else on the thread: a live tile configuration keeps 8 KB of +// XTILEDATA in this thread's XSAVE area across every context switch, and leaves +// a shape another AMX user did not ask for sitting on hardware it also uses. +// +// Because it is nobody's correctness but the neighbours', deleting it leaves +// every result right and every "did it crash?" test green. Only +// `lance_amx_tilecfg_current_for_test`, which reads the hardware, notices. +static inline void lance_amx_tile_done(void) { _tile_release(); } + +// Test hook: retire the tile configuration the way a *foreign* AMX user on this +// thread would, so a test can put the tile unit in INIT under a kernel that is +// about to run and check the kernel reconfigures instead of assuming. +// +// Hidden visibility because this manufactures the state the rest of this file +// exists to prevent: it is linked into the crate for its regression test, but is +// not something a shipped shared object should offer to whatever else is in the +// process. +// +// TILERELEASE needs no XTILEDATA grant of its own -- XFD traps only instructions +// that touch a TMM register -- but a test calling this is about to call a kernel +// that does, so it should have gone through `amx_supported()` anyway. +__attribute__((visibility("hidden"))) void lance_amx_tile_clobber_for_test(void) { + _tile_release(); +} + +// Test hook: copy this logical processor's *live* tile configuration into the +// 64 bytes at `out`, via STTILECFG. A `palette_id` of 0 means the tile unit is +// in INIT state -- nothing configured. +// +// This reads the hardware rather than any record Lance keeps, which is what +// makes it able to catch the half of this design that has no other symptom: +// `lance_amx_tile_ensure` reloading on every entry is what keeps results +// correct, so dropping `lance_amx_tile_done`'s release leaves behaviour right +// and every "did it crash?" test passing, and only a live `palette_id` reported +// back here says the tile unit was never handed over. +// +// STTILECFG does not touch a TMM register, so unlike the kernels it is legal +// without the XTILEDATA grant. Hidden for the same reason as the clobber hook. +__attribute__((visibility("hidden"))) void lance_amx_tilecfg_current_for_test( + uint8_t *out) { + lance_amx_tilecfg cfg; + _tile_storeconfig(&cfg); + memcpy(out, &cfg, sizeof(cfg)); +} + +// --------------------------------------------------------------------------- +// Kernel: batch-16 search (one query x 16 candidates) +// --------------------------------------------------------------------------- + +// Tile roles. These are immediate operands of the tile intrinsics, so they must +// be compile-time constants; `#define` rather than `enum` avoids relying on how +// strictly a compiler treats enum constants as immediates. +// +// One TDPFP16PS pass covers K = 32 fp16 dims. With N = 1 the query needs no +// VNNI repacking: B.row[k].fp16[i] = query[k*2 + i] is just the contiguous +// query halfwords, obtained by loading with a 4-byte row stride. The candidates +// form the A tile's rows; they live at unrelated addresses, so this kernel +// stages them a few k-blocks at a time into a fixed-stride scratch buffer that +// the tile load can read (see `lance_amx_stage_rows`). +// +// Three (A, B) pairs, not one. A single pair would make the loop strictly +// serial -- every TDPFP16PS waiting on the two loads that just overwrote its +// own operands -- so the tile unit would idle through each load's latency. +// Three independent pairs let three k-blocks' loads issue before the first +// TDPFP16PS needs its result, which is enough to keep the dp ops back to back. +// Seven tiles is what that costs; tmm7 is left unconfigured. +#define SEARCH_TMM_C 0 // 16 results x 1 fp32 +#define SEARCH_TMM_A0 1 // 16 candidate rows x 32 fp16, k-block 3t +#define SEARCH_TMM_B0 2 // query, VNNI-packed at N = 1, k-block 3t +#define SEARCH_TMM_A1 3 // k-block 3t + 1 +#define SEARCH_TMM_B1 4 +#define SEARCH_TMM_A2 5 // k-block 3t + 2 +#define SEARCH_TMM_B2 6 + +static const lance_amx_tile_spec SEARCH_TILES[] = { + {SEARCH_TMM_C, 16, 4}, {SEARCH_TMM_A0, 16, 64}, {SEARCH_TMM_B0, 16, 4}, + {SEARCH_TMM_A1, 16, 64}, {SEARCH_TMM_B1, 16, 4}, {SEARCH_TMM_A2, 16, 64}, + {SEARCH_TMM_B2, 16, 4}, +}; + +// Halfwords of one k-block: 32 fp16 dims, the K a single TDPFP16PS covers. +#define SEARCH_K_BLOCK 32 + +// Bytes one tile row spans in one k-block: 32 fp16, the A tile's full row width. +#define SEARCH_ROW_BYTES (SEARCH_K_BLOCK * (int)sizeof(uint16_t)) + +// K-blocks gathered per staging step. Three, so one staged buffer feeds exactly +// the three (A, B) pairs the main loop issues together. +#define SEARCH_STAGE_BLOCKS 3 +#define SEARCH_STAGE_ROW_BYTES (SEARCH_STAGE_BLOCKS * SEARCH_ROW_BYTES) + +// One staging buffer: the 16 rows a tile load always reads, whatever the batch +// actually holds. +#define SEARCH_STAGE_BYTES (16 * SEARCH_STAGE_ROW_BYTES) + +// The furthest-reaching tile load is A2: 64 bytes at offset 128 of the last of +// 16 rows, so the last byte it touches is at 128 + 15*192 + 63 and the span it +// covers is exactly SEARCH_STAGE_BYTES. Asserted because an overrun here would +// be a stack smash with no other symptom. +// +// What this actually pins is that the three A tiles tile one staging row with +// no gap and no overlap, i.e. SEARCH_STAGE_BLOCKS == 3; it does not check +// SEARCH_K_BLOCK, nor the 64-byte A-tile width, which is hardcoded separately +// in SEARCH_TILES. +_Static_assert(2 * SEARCH_ROW_BYTES + 15 * SEARCH_STAGE_ROW_BYTES + + SEARCH_ROW_BYTES == + SEARCH_STAGE_BYTES, + "the three A tiles must tile a staging row exactly"); + +// Gather `row_bytes` starting at k-block `kb` out of each of the first `count` +// candidates into `dst`, one candidate per row. +// +// Rows are SEARCH_STAGE_ROW_BYTES apart even when `row_bytes` is smaller: an A +// tile only reads 64 bytes at its own offset within a row, so a wider stride +// simply leaves the trailing bytes unread. Keeping the stride fixed across +// every staging step is what lets rows [count, 16) be zeroed once per call -- +// each step then rewrites the same rows at the same addresses. +// +// The k-blocks a row covers are adjacent inside the candidate vector, so each +// candidate costs exactly one straight-line copy no matter how many k-blocks the +// step covers. `row_bytes` should stay a compile-time constant at every call +// site: a constant size expands to inline wide moves, while a variable one +// becomes a libc `memcpy` call that `-funroll-loops` then multiplies (five PLT +// calls for the one remainder loop, measured under clang-16). +// +// Zeroing the padding rows is deliberately *not* routed through here: it writes +// rows [count, 16) rather than [0, count), and when the rows are full width it +// is one contiguous `memset` rather than a per-row loop. +static inline void lance_amx_stage_rows(uint8_t *dst, + const uint16_t *const *candidates, + size_t count, size_t kb, + size_t row_bytes) { + for (size_t n = 0; n < count; n++) { + memcpy(dst + n * SEARCH_STAGE_ROW_BYTES, + candidates[n] + kb * SEARCH_K_BLOCK, row_bytes); + } +} + +// out[i] = sum_{d in 0..dim} f32(query[d]) * f32(candidates[i][d]), i < count. +// +// `query` -- IEEE-754 binary16 values as raw uint16_t bit patterns +// (half::f16 has identical layout); `dim` valid halfwords. +// `candidates` -- pointers to `dim` halfwords each; only the first `count` are +// read. The vectors live wherever the storage put them; this +// kernel owns the gather. +// `count` -- candidates carrying a real vector. **Precondition: +// 1 <= count <= 16**, rejected at the Rust boundary +// (`dot_f16_batch_16`) rather than clamped here; a larger value +// would run the gather off the end of a staging buffer. +// `dim` -- vector dimension. +// `out` -- destination for 16 fp32 dot products. Lanes [count, 16) are +// written as 0, not left untouched. +// +// ## Why `count` rather than always 16 +// +// The caller sweeps centroids 16 at a time, so when their count is not a +// multiple of 16 the last group is short and it fills the spare slots by +// repeating a row it already holds. Staging all 16 rows would copy that row an +// extra 16 - count times; skipping them saves that much of a memcpy on at most +// one group per sweep, so `count` buys far less here than it did for a caller +// whose batches were usually partial. Only rows [0, count) are gathered, so the +// copying scales with the vectors actually scored while the tile work stays one +// fixed-cost pass. The padded rows still have to exist, since a tile load reads +// 16 rows unconditionally; they are zeroed once per call, and an all-zero A row +// yields a zero dot product. +// +// ## Why the gather is here and not in the caller +// +// A tile load reads 16 rows at one fixed stride from one base pointer, and no +// stride is guaranteed between this kernel's 16 candidate pointers, so their +// bytes have to be brought together somewhere. Doing it in the caller means +// copying 16 * dim * 2 bytes -- 24 KB at dim 768, 32 KB at dim 1024 -- and every +// one of those bytes has to land before the first TDPFP16PS can issue. Against a +// 48 KB L1D that buffer alone is half the cache, and the copy is pure exposed +// latency: nothing overlaps it. +// +// Staging inside the k-block loop instead keeps the working buffer at 3 KB and +// lets the copies for one triple run underneath the tile ops of the previous +// one. The bytes moved are identical; what changes is that they move while the +// tile unit is busy rather than before it starts. (Measured on the caller-side +// version at dim 1024: __memmove 3.7M cycles/query against 0.5M for the scalar +// kernel, IPC 1.70 -> 1.17, and a critical path 18% longer even though total CPU +// work per query was 6.5% lower. The same structure is what +// epeshared/hnswlib-amx uses for its AMX-BF16 kernel.) +// +// For dim > 32 we accumulate across floor(dim/32) tile passes into the same C +// tile, three k-blocks at a time. The tail (dim % 32 dims) is computed in +// scalar fp32 afterwards. The result is NOT bit-exact against a sequential +// scalar loop: floating-point tile-order accumulation rounds differently. It +// matches an f32-accumulated reference dot product to within fp16 precision, +// which is all the fp16 distance path requires (see amx_fp16.rs / the Rust-side +// tests). +void lance_amx_dot_f16_batch_16(const uint16_t *query, + const uint16_t *const *candidates, size_t count, + size_t dim, float *out) { + const size_t blocks = dim / SEARCH_K_BLOCK; // whole 32-wide tile passes + const size_t full = blocks * SEARCH_K_BLOCK; // dims they cover + + if (blocks > 0) { + lance_amx_tile_ensure(SEARCH_TILES, LANCE_AMX_TILE_COUNT(SEARCH_TILES)); + + // Two staging buffers, 16 rows x 192 bytes each. Double buffered so a + // triple's gather can be issued a whole triple before the tile loads that + // read it: a tile load cannot take its data from the store buffer, so the + // gather's stores have to reach L1 first, and with one buffer there is + // nothing to overlap that drain with. 3 KB each, so both sit in L1D + // alongside the candidate rows streaming through it. + __attribute__((aligned(64))) uint8_t stage[2][SEARCH_STAGE_BYTES]; + + const size_t triples = blocks / SEARCH_STAGE_BLOCKS; + const size_t rem = blocks % SEARCH_STAGE_BLOCKS; + + // Rows [count, 16) are padding that a tile load reads but no candidate + // fills, so they are zeroed here and an all-zero A row then contributes a + // zero dot product. Once per call is enough: every staging step below + // writes rows [0, count) only, always at the same row stride, so nothing + // disturbs the padding again. + // + // Only bytes some tile load actually reads are zeroed. This cost is the one + // part of the call that does not shrink with `dim`, so zeroing the full + // 2 x 15 x 192 bytes unconditionally would dominate short vectors: + // * `stage[1]` is tile-loaded only if the main loop reaches a second + // iteration, or the remainder lands on it (an odd `triples`). A single + // triple with no remainder never reads it at all. + // * A padding row is read to its full width only by the main loop. With + // `triples == 0` the remainder is the only reader, and it loads A0 -- + // plus A1 when `rem == 2` -- so only the first `rem` k-block slots of + // each row are ever seen. + if (count < 16) { + const size_t pad_off = count * SEARCH_STAGE_ROW_BYTES; + if (triples > 0) { + // Full-width padding rows are contiguous: one memset covers them all. + memset(stage[0] + pad_off, 0, SEARCH_STAGE_BYTES - pad_off); + if (triples > 1 || rem > 0) { + memset(stage[1] + pad_off, 0, SEARCH_STAGE_BYTES - pad_off); + } + } else { + for (size_t n = count; n < 16; n++) { + memset(stage[0] + n * SEARCH_STAGE_ROW_BYTES, 0, + rem * (size_t)SEARCH_ROW_BYTES); + } + } + } + + _tile_zero(SEARCH_TMM_C); + + // Take ownership of the destination line now (PREFETCHW), so the RFO + // overlaps the tile work instead of stalling TILESTORED at the end. The + // store is 64 bytes issued as one instruction, and waiting on the RFO there + // backs up the store queue: measured on the FAISS #5235 BF16 kernel as + // SQ_Full 28.9% of cycles, with XQ.FULL_CYCLES 56x an AVX-512 baseline. + _mm_prefetch((const char *)out, _MM_HINT_ET0); + + // Prologue of the software pipeline below, and the one gather in the call + // with no tile work ahead of it to hide behind. + if (triples > 0) { + lance_amx_stage_rows(stage[0], candidates, count, 0, + SEARCH_STAGE_ROW_BYTES); + + // The in-loop prefetch below runs two triples ahead, so k-blocks + // [3, 6) -- read by the very first iteration's gather -- would otherwise + // be touched by nothing. Guarded on the address staying inside the + // vectors, which also covers a remainder that follows a single triple. + if (SEARCH_STAGE_BLOCKS < blocks) { + for (size_t n = 0; n < count; n++) { + _mm_prefetch( + (const char *)(candidates[n] + + SEARCH_STAGE_BLOCKS * (size_t)SEARCH_K_BLOCK), + _MM_HINT_T0); + } + } + } + + size_t kb = 0; + for (size_t t = 0; t < triples; t++, kb += SEARCH_STAGE_BLOCKS) { + const uint8_t *st = stage[t & 1]; + + // Gather the *next* triple before issuing this one's tile ops, into the + // buffer the previous iteration's tile loads have already consumed. Those + // stores then have a full triple of tile work to commit to L1 under, + // instead of the tile load immediately below them waiting on the drain. + if (t + 1 < triples) { + // Open the candidate streams the iteration after this one will copy + // from, so that copy finds them resident. With the rows at 16 unrelated + // addresses there is no single stride for a hardware prefetcher to + // latch onto; what it can do is run each candidate forward once that + // candidate has been touched, and these touches are what start it. + // (Measured on the FAISS #5235 BF16 kernel without any prefetch: L1D MPI + // 2.6x and DTLB load MPI 14.6x an AVX-512 baseline.) Guarded so the last + // iterations stay inside the vectors. + if (kb + 2 * SEARCH_STAGE_BLOCKS < blocks) { + const size_t ahead = (kb + 2 * SEARCH_STAGE_BLOCKS) * SEARCH_K_BLOCK; + for (size_t n = 0; n < count; n++) { + _mm_prefetch((const char *)(candidates[n] + ahead), _MM_HINT_T0); + } + } + lance_amx_stage_rows(stage[(t + 1) & 1], candidates, count, + kb + SEARCH_STAGE_BLOCKS, SEARCH_STAGE_ROW_BYTES); + } + + // All six loads first: they are independent, so the three dp ops below + // issue back to back rather than each waiting on its own operands. + _tile_loadd(SEARCH_TMM_A0, st + 0 * SEARCH_ROW_BYTES, + SEARCH_STAGE_ROW_BYTES); + _tile_loadd(SEARCH_TMM_B0, query + kb * SEARCH_K_BLOCK, 4); + _tile_loadd(SEARCH_TMM_A1, st + 1 * SEARCH_ROW_BYTES, + SEARCH_STAGE_ROW_BYTES); + _tile_loadd(SEARCH_TMM_B1, query + (kb + 1) * SEARCH_K_BLOCK, 4); + _tile_loadd(SEARCH_TMM_A2, st + 2 * SEARCH_ROW_BYTES, + SEARCH_STAGE_ROW_BYTES); + _tile_loadd(SEARCH_TMM_B2, query + (kb + 2) * SEARCH_K_BLOCK, 4); + + // C += A * B (fp16 x fp16 -> fp32) + _tile_dpfp16ps(SEARCH_TMM_C, SEARCH_TMM_A0, SEARCH_TMM_B0); + _tile_dpfp16ps(SEARCH_TMM_C, SEARCH_TMM_A1, SEARCH_TMM_B1); + _tile_dpfp16ps(SEARCH_TMM_C, SEARCH_TMM_A2, SEARCH_TMM_B2); + } + + // The 1 or 2 k-blocks left over when `blocks` is not a multiple of three. + // `stage[triples & 1]` is the buffer the loop above neither wrote nor read + // last, so filling it now cannot collide with a tile load still in flight. + if (rem > 0) { + uint8_t *st = stage[triples & 1]; + // Two constant-size cases rather than one `rem * 64` copy, because a + // variable-length memcpy here does not stay one instruction: clang-16 + // emits a libc call and `-funroll-loops` then multiplies it, measured as + // five `memcpy@PLT` calls for this one loop (the pre-`count` kernel, + // whose loop ran to 16, paid sixteen). The main loop's gather is + // unaffected either way -- it keeps its inline wide moves -- so this is + // about the remainder alone. The branch costs one predictable compare. + if (rem == 2) { + lance_amx_stage_rows(st, candidates, count, kb, 2 * SEARCH_ROW_BYTES); + } else { + lance_amx_stage_rows(st, candidates, count, kb, SEARCH_ROW_BYTES); + } + + // Loaded at the main loop's row stride even though only `rem` k-blocks are + // live: A0 and A1 read their own 64 bytes at offsets 0 and 64, which is + // what the staging step just filled, and the padding rows are already zero + // at exactly this stride. + _tile_loadd(SEARCH_TMM_A0, st, SEARCH_STAGE_ROW_BYTES); + _tile_loadd(SEARCH_TMM_B0, query + kb * SEARCH_K_BLOCK, 4); + if (rem == 2) { + _tile_loadd(SEARCH_TMM_A1, st + SEARCH_ROW_BYTES, + SEARCH_STAGE_ROW_BYTES); + _tile_loadd(SEARCH_TMM_B1, query + (kb + 1) * SEARCH_K_BLOCK, 4); + } + _tile_dpfp16ps(SEARCH_TMM_C, SEARCH_TMM_A0, SEARCH_TMM_B0); + if (rem == 2) { + _tile_dpfp16ps(SEARCH_TMM_C, SEARCH_TMM_A1, SEARCH_TMM_B1); + } + } + + _tile_stored(SEARCH_TMM_C, out, 4); // 16 fp32, row stride 4 bytes + // Last tile op in this call; the tail below is scalar. Hands the tile unit + // back so an interleaved AMX user on this thread cannot be surprised by a + // configuration it did not ask for. + lance_amx_tile_done(); + } else { + for (int i = 0; i < 16; i++) out[i] = 0.0f; + } + + // Tail: the dims not covered by a full 32-wide tile, in scalar fp32. F16C + // `_cvtsh_ss` is an exact (lossless) binary16 -> binary32 widening. Lanes at + // or past `count` are skipped rather than accumulated onto: they are already + // 0 (zeroed staging rows, or the no-tile-pass branch above) and must stay so. + const size_t tail = dim - full; + if (tail > 0) { + for (size_t i = 0; i < count; i++) { + float acc = 0.0f; + const uint16_t *row = candidates[i]; + for (size_t d = full; d < dim; d++) { + acc += _cvtsh_ss(row[d]) * _cvtsh_ss(query[d]); + } + out[i] += acc; + } + } +} + +// --------------------------------------------------------------------------- +// Kernel: M x N GEMM (many vectors x many centroids) +// --------------------------------------------------------------------------- + +// Tile roles for the 2x2-register-blocked GEMM. Two A tiles (32 vectors) and +// two B tiles (32 centroids) feed four C accumulators, so one k-pass issues 4 +// TDPFP16PS against 4 tile loads -- the highest compute-per-load ratio the 8 +// physical tiles allow, and the reason all 8 are claimed here. +#define GEMM_TMM_C00 0 // 16 vectors x 16 centroids, fp32 +#define GEMM_TMM_C01 1 +#define GEMM_TMM_C10 2 +#define GEMM_TMM_C11 3 +#define GEMM_TMM_A0 4 // 16 vector rows x 32 fp16 dims +#define GEMM_TMM_A1 5 +#define GEMM_TMM_B0 6 // 32 dims x 16 centroids, VNNI-interleaved +#define GEMM_TMM_B1 7 + +// All eight at the architectural maximum (16 rows x 64 bytes = 1 KB), which is +// exactly the 8 KB of tile state AMX provides. +static const lance_amx_tile_spec GEMM_TILES[] = { + {GEMM_TMM_C00, 16, 64}, {GEMM_TMM_C01, 16, 64}, {GEMM_TMM_C10, 16, 64}, + {GEMM_TMM_C11, 16, 64}, {GEMM_TMM_A0, 16, 64}, {GEMM_TMM_A1, 16, 64}, + {GEMM_TMM_B0, 16, 64}, {GEMM_TMM_B1, 16, 64}, +}; + +// Halfwords per packed B block: 16 tile rows x 32 halfwords per row. +#define GEMM_B_BLOCK 512 + +// out[i*out_stride + j] = sum_{d in 0..dim} f32(data[i*data_stride + d]) * +// f32(centroids[j*dim + d]). +// +// `data` -- [m, dim] row-major fp16 bit patterns, rows `data_stride` +// halfwords apart (`data_stride >= dim`). +// `m` -- number of vectors; **must be a multiple of 32**. +// `packed_b` -- centroids pre-interleaved by `pack_centroids_vnni` (see +// `amx_fp16.rs`), holding only the floor(dim/32) whole +// 32-dim k-blocks. +// `centroids` -- the same [n, dim] row-major centroids `packed_b` was built +// from. Read only for the `dim % 32` tail dims, which are not +// worth a tile pass and so are never packed; still required +// when dim % 32 == 0, where it goes unread. +// `n` -- number of centroids; **must be a multiple of 32**. +// `dim` -- vector dimension; any value, the tail runs scalar. +// `out` -- [m, n] row-major fp32, rows `out_stride` floats apart +// (`out_stride >= n`). +// +// The m and n multiple-of-32 requirements are preconditions, not something this +// kernel checks or works around: they let the register-blocked loop run with no +// edge cases, and the Rust caller is the layer that knows how to pad or split. +// +// The k dimension carries no such requirement. TDPFP16PS accumulation rounds +// differently from a sequential scalar loop, so results match an f32-accumulated +// reference to fp16 precision rather than bit-exactly -- same contract as +// `lance_amx_dot_f16_batch_16`. +// +// B's VNNI interleave is what makes A loadable straight out of `data`: with +// packed_b[((kb*(n/16) + jb)*512) + k*32 + nn*2 + p] +// == centroids[(jb*16 + nn)*dim + kb*32 + 2*k + p] +// TDPFP16PS's b.row[k].fp16[2*nn+p] lands on centroid (jb*16+nn) dim +// (kb*32+2*k+p), pairing it with a.row[mm].fp16[2*k+p] = the same dim of vector +// (i+mm). k-blocks are the outer index so one k-pass reads the two B tiles it +// needs from adjacent memory. +void lance_amx_dot_f16_gemm(const uint16_t *data, size_t m, size_t data_stride, + const uint16_t *packed_b, const uint16_t *centroids, + size_t n, size_t dim, float *out, + size_t out_stride) { + const size_t full = (dim / 32) * 32; // dims covered by full 32-wide passes + + if (full > 0) { + lance_amx_tile_ensure(GEMM_TILES, LANCE_AMX_TILE_COUNT(GEMM_TILES)); + + const size_t a_stride_bytes = data_stride * sizeof(uint16_t); + const size_t c_stride_bytes = out_stride * sizeof(float); + const size_t b_blocks_per_k = n / 16; + + for (size_t i = 0; i < m; i += 32) { + const uint16_t *a0 = data + i * data_stride; + const uint16_t *a1 = a0 + 16 * data_stride; + float *c0 = out + i * out_stride; + float *c1 = c0 + 16 * out_stride; + + for (size_t j = 0; j < n; j += 32) { + _tile_zero(GEMM_TMM_C00); + _tile_zero(GEMM_TMM_C01); + _tile_zero(GEMM_TMM_C10); + _tile_zero(GEMM_TMM_C11); + + for (size_t kbase = 0; kbase < full; kbase += 32) { + // The B tiles for centroid blocks j/16 and j/16+1 are adjacent + // because jb is the inner index of the packed layout. + const uint16_t *b = + packed_b + ((kbase / 32) * b_blocks_per_k + j / 16) * GEMM_B_BLOCK; + _tile_loadd(GEMM_TMM_A0, a0 + kbase, a_stride_bytes); + _tile_loadd(GEMM_TMM_A1, a1 + kbase, a_stride_bytes); + _tile_loadd(GEMM_TMM_B0, b, 64); + _tile_loadd(GEMM_TMM_B1, b + GEMM_B_BLOCK, 64); + _tile_dpfp16ps(GEMM_TMM_C00, GEMM_TMM_A0, GEMM_TMM_B0); + _tile_dpfp16ps(GEMM_TMM_C01, GEMM_TMM_A0, GEMM_TMM_B1); + _tile_dpfp16ps(GEMM_TMM_C10, GEMM_TMM_A1, GEMM_TMM_B0); + _tile_dpfp16ps(GEMM_TMM_C11, GEMM_TMM_A1, GEMM_TMM_B1); + } + + _tile_stored(GEMM_TMM_C00, c0 + j, c_stride_bytes); + _tile_stored(GEMM_TMM_C01, c0 + j + 16, c_stride_bytes); + _tile_stored(GEMM_TMM_C10, c1 + j, c_stride_bytes); + _tile_stored(GEMM_TMM_C11, c1 + j + 16, c_stride_bytes); + } + } + // Last tile op in this call; the tail below is scalar. One LDTILECFG plus + // one TILERELEASE against an m x n GEMM is why the per-call reconfiguration + // the file header argues for costs this kernel nothing. + lance_amx_tile_done(); + } else { + // dim < 32: no tile ever stores to `out`, so the scalar tail below has to + // accumulate onto a known-zero destination rather than whatever was there. + for (size_t i = 0; i < m; i++) { + memset(out + i * out_stride, 0, n * sizeof(float)); + } + } + + const size_t tail = dim - full; + if (tail > 0) { + for (size_t i = 0; i < m; i++) { + // Widen this vector's tail once per row instead of once per (row, + // centroid) pair; `tail` is at most 31 so the buffer is a fixed 32. + float vec_tail[32]; + const uint16_t *row = data + i * data_stride + full; + for (size_t d = 0; d < tail; d++) vec_tail[d] = _cvtsh_ss(row[d]); + + float *out_row = out + i * out_stride; + for (size_t j = 0; j < n; j++) { + const uint16_t *cent = centroids + j * dim + full; + float acc = 0.0f; + for (size_t d = 0; d < tail; d++) acc += vec_tail[d] * _cvtsh_ss(cent[d]); + out_row[j] += acc; + } + } + } +} + +// --------------------------------------------------------------------------- +// Configuration introspection +// --------------------------------------------------------------------------- + +// Write the 64-byte LDTILECFG image for `cfg_kind` (a `LANCE_AMX_CFG_*` +// constant) into `out`, without loading it. Returns 0 on success, -1 for an +// unknown `cfg_kind`. +// +// Exposed so the Rust tests can pin each kernel's tile shape byte for byte. A +// wrong shape does not fail cleanly — it is a #UD or silently wrong results — +// so the shape is asserted directly rather than inferred from kernel output. +int lance_amx_tilecfg_image(int cfg_kind, uint8_t *out) { + const lance_amx_tile_spec *specs; + size_t n; + + switch (cfg_kind) { + case LANCE_AMX_CFG_SEARCH: + specs = SEARCH_TILES; + n = LANCE_AMX_TILE_COUNT(SEARCH_TILES); + break; + case LANCE_AMX_CFG_GEMM: + specs = GEMM_TILES; + n = LANCE_AMX_TILE_COUNT(GEMM_TILES); + break; + default: + return -1; + } + + lance_amx_tilecfg cfg; + lance_amx_tilecfg_build(&cfg, specs, n); + memcpy(out, &cfg, sizeof(cfg)); + return 0; +} diff --git a/rust/lance-linalg/src/simd/amx_fp16.rs b/rust/lance-linalg/src/simd/amx_fp16.rs new file mode 100644 index 00000000000..82a19e399b5 --- /dev/null +++ b/rust/lance-linalg/src/simd/amx_fp16.rs @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! AMX-FP16 accelerated f16 x f16 dot products. +//! +//! Two shapes: `dot_f16_batch_16_amx` (one query against 16 candidates, for +//! choosing the partitions a query probes) and `dot_f16_gemm_amx` (an m x n +//! GEMM, for scoring many vectors against many centroids). Both are named in +//! plain code spans rather than intra-doc links: they are +//! `kernel_support = "amx_fp16"`-gated, so a link from these unconditional +//! module docs is unresolved — and hence a rustdoc error under `-D warnings` — +//! on any build without the kernel. +//! +//! The tile math lives in `amx_fp16.c` (compiled by `build.rs` with a compiler +//! new enough for `-mamx-fp16`, which sets `kernel_support = "amx_fp16"`). This +//! module holds the FFI declarations, the B-operand packing the GEMM's tile +//! layout requires, and the runtime safety gate. Everything here is +//! crate-internal; the safe public entry points are +//! [`crate::distance::dot_f16::dot_f16_batch_16`] for the batch-16 shape and +//! `PackedCentroidsF16` for the GEMM (named, not linked, for the same reason as +//! the kernels above). The latter owns what this layer deliberately does not: +//! padding `n` up to a multiple of 32, holding the packed B operand across +//! calls, and turning the absence of AMX into an `Option` its caller — k-means +//! assignment, in another crate that cannot see `kernel_support` — can branch +//! on. Every kernel here is guarded by +//! [`crate::distance::dot_f16::amx_fp16_supported`], which is also what callers +//! consult before routing work here: one gate, decided by run-time capability +//! alone. +//! +//! ## Safety gate +//! +//! A set CPUID bit is not sufficient to run AMX tile instructions on Linux: +//! the OS must first grant the extended (XTILEDATA) +//! state via `arch_prctl(ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA)`; skipping +//! that SIGILLs the first tile instruction. `amx_supported` requires, and +//! caches process-wide, all of: +//! 1. `target_arch = "x86_64"` and `target_os = "linux"` (compile-time cfg), +//! 2. the amx-tile CPUID bit (leaf 7, sub-leaf 0, EDX bit 24) **and** the +//! amx-fp16 CPUID bit (leaf 7, sub-leaf 1, EAX bit 21), +//! 3. a successful one-time `arch_prctl` permission request. +//! +//! On any failure the caller falls back to the existing AVX-512-FP16 / scalar +//! `f16::dot` path, so results are unchanged (both accumulate in f32; only the +//! summation order — hence fp16-level rounding — differs). +//! +//! ## XTILEDATA is a shared AMX state permission +//! +//! XTILEDATA (component 18) is the single dynamically-enabled XSAVE state +//! backing the physical TMM tile registers; it is requested per-*state*, not +//! per-*instruction*, so one grant covers every AMX compute instruction — see +//! Linux `Documentation/arch/x86/xstate.rst` ("Dynamically Enabled XSAVE +//! Features", AMX example) and Intel SDM Vol.1 §13.3. The syscall is +//! idempotent, so requesting an already-granted permission again is harmless. + +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +use crate::distance::dot_f16::strided_len; +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +use half::f16; + +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +unsafe extern "C" { + /// arch_prctl(ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA); 0 on success. + fn lance_amx_fp16_request_perm() -> i32; + + /// out[i] = sum_d f32(query[d]) * f32(candidates[i][d]), i in 0..count; + /// out[count..16] = 0. `query` is `dim` IEEE binary16 bit patterns; + /// `candidates` is 16 pointers to `dim` of them each, of which only the + /// first `count` (1..=16) are read; `out` holds 16 f32. The kernel gathers + /// the rows itself, k-block by k-block. See `amx_fp16.c`. + fn lance_amx_dot_f16_batch_16( + query: *const u16, + candidates: *const *const u16, + count: usize, + dim: usize, + out: *mut f32, + ); + + /// out[i*out_stride + j] = sum_d f32(data[i*data_stride + d]) * + /// f32(centroids[j*dim + d]), for i in 0..m, j in 0..n. `packed_b` is + /// [`pack_centroids_vnni`]'s output for the same centroids; `centroids` + /// itself is read only for the `dim % 32` unpacked tail dims. `m` and `n` + /// must both be multiples of 32. See `amx_fp16.c`. + fn lance_amx_dot_f16_gemm( + data: *const u16, + m: usize, + data_stride: usize, + packed_b: *const u16, + centroids: *const u16, + n: usize, + dim: usize, + out: *mut f32, + out_stride: usize, + ); + + /// Writes the 64-byte LDTILECFG image for `cfg_kind` into `out` without + /// loading it; 0 on success, -1 for an unknown kind. See `amx_fp16.c`. + #[cfg(test)] + fn lance_amx_tilecfg_image(cfg_kind: i32, out: *mut u8) -> i32; + + /// Retires the tile configuration the way a foreign AMX user sharing this + /// thread would. See `amx_fp16.c`. + #[cfg(test)] + fn lance_amx_tile_clobber_for_test(); + + /// Writes this logical processor's live 64-byte tile configuration to `out` + /// via STTILECFG; `palette_id` (byte 0) is 0 when nothing is configured. + /// See `amx_fp16.c`. + #[cfg(test)] + fn lance_amx_tilecfg_current_for_test(out: *mut u8); +} + +/// Test-only: retire this thread's tile configuration the way another AMX user +/// on the same thread would, leaving the tile unit in INIT under a kernel that +/// is about to run. +/// +/// # Safety +/// Executes TILERELEASE, so [`amx_supported`] must have returned `true` first. +#[cfg(all( + test, + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) unsafe fn clobber_tile_state_for_test() { + unsafe { lance_amx_tile_clobber_for_test() }; +} + +/// Test-only: `true` when a tile configuration is currently live on this logical +/// processor, read straight off the hardware with STTILECFG. +/// +/// This is what distinguishes a working release path from a deleted one. +/// `lance_amx_tile_ensure` reloads on every kernel entry, so results stay right +/// with `lance_amx_tile_done`'s TILERELEASE gone and no "did it crash?" test +/// notices; what is lost is only the promise made to whoever shares the thread, +/// and that is visible nowhere but here. +#[cfg(all( + test, + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) fn tile_config_is_live_for_test() -> bool { + let mut image = [0u8; 64]; + // SAFETY: the C side writes exactly 64 bytes, and STTILECFG touches no TMM + // register so it is legal without the XTILEDATA grant. + unsafe { lance_amx_tilecfg_current_for_test(image.as_mut_ptr()) }; + image[0] != 0 +} + +/// Config kind for [`tilecfg_image`]: the batch-16 search kernel's tile shape. +/// Must match `LANCE_AMX_CFG_SEARCH` in `amx_fp16.c`. +#[cfg(all( + test, + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) const AMX_CFG_SEARCH: i32 = 0; + +/// Config kind for [`tilecfg_image`]: the GEMM kernel's tile shape. +/// Must match `LANCE_AMX_CFG_GEMM` in `amx_fp16.c`. +#[cfg(all( + test, + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) const AMX_CFG_GEMM: i32 = 1; + +/// The 64-byte LDTILECFG image a kernel would configure, without loading it. +/// `None` if `cfg_kind` is not one of the `AMX_CFG_*` constants. +/// +/// Exists for the tests: a wrong tile shape never surfaces as a clean error — +/// it is a #UD or silently wrong results — so the shape is pinned directly +/// rather than inferred from kernel output. +#[cfg(all( + test, + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) fn tilecfg_image(cfg_kind: i32) -> Option<[u8; 64]> { + let mut image = [0u8; 64]; + // SAFETY: the C side writes exactly `sizeof(lance_amx_tilecfg)` bytes, which + // a `_Static_assert` there pins to 64 — the length of `image`. + let rc = unsafe { lance_amx_tilecfg_image(cfg_kind, image.as_mut_ptr()) }; + (rc == 0).then_some(image) +} + +/// True iff AMX-FP16 tile instructions can be executed safely in this process. +/// Evaluated once and cached; the `arch_prctl` permission request (a syscall) +/// happens at most once, process-wide. +/// +/// This is a hardware question only — a pure "would a tile instruction fault +/// here?" — so that kernel-level tests can exercise the kernels on any host +/// that can run them. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) fn amx_supported() -> bool { + static SUPPORTED: std::sync::OnceLock = std::sync::OnceLock::new(); + *SUPPORTED.get_or_init(|| { + if !detect_amx_fp16() { + return false; + } + // Request XTILEDATA permission; without a 0 return, any tile instruction + // would SIGILL, so treat anything else as unavailable. + unsafe { lance_amx_fp16_request_perm() == 0 } + }) +} + +/// AMX-TILE = CPUID leaf 7, sub-leaf 0, EDX bit 24. AMX-FP16 = CPUID leaf 7, +/// sub-leaf 1, EAX bit 21. Both required. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +fn detect_amx_fp16() -> bool { + use std::arch::x86_64::__cpuid_count; + // `__cpuid_count` is safe on nightly but `unsafe` on stable; allow both. + #[allow(unused_unsafe)] + let leaf7_0 = unsafe { __cpuid_count(7, 0) }; + let amx_tile = (leaf7_0.edx & (1 << 24)) != 0; + #[allow(unused_unsafe)] + let leaf7_1 = unsafe { __cpuid_count(7, 1) }; + let amx_fp16 = (leaf7_1.eax & (1 << 21)) != 0; + amx_tile && amx_fp16 +} + +/// Batched AMX-FP16 dot product: one query against the first `len` of 16 +/// candidates. Returns the 16 raw dot products (`Σ query·candidate`, no `1.0 -` +/// distance wrapping); lanes `len..16` are 0. +/// +/// Hands the kernel 16 pointers rather than a packed `16 x dim` buffer. The +/// candidates still have to be brought together for a tile load, but the kernel +/// does it one k-block at a time into 3 KB of stack, which overlaps the copies +/// with the tile ops; packing all `16 * dim * 2` bytes here first could not +/// overlap with anything, and at dim 1024 that is 32 KB against a 48 KB L1D. +/// See the kernel comment in `amx_fp16.c` for the measurements behind this. +/// +/// `len` is what keeps a partial batch cheap: the tile pass is a fixed cost for +/// 16 lanes either way, but only `len` rows are gathered. +/// +/// # Safety +/// `amx_supported` must have returned `true`. `len` must be in `1..=16` — the +/// kernel does not clamp it, and a larger value walks its staging buffer off the +/// end; [`crate::distance::dot_f16::dot_f16_batch_16`] is where that is +/// rejected. Every candidate slice must have length `query.len()`, and must +/// outlive the call. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) unsafe fn dot_f16_batch_16_amx( + query: &[f16], + candidates: &[&[f16]; 16], + len: usize, +) -> [f32; 16] { + debug_assert!((1..=16).contains(&len), "len ({len}) must be in 1..=16"); + let dim = query.len(); + // half::f16 is #[repr(transparent)] over u16, so the casts below reinterpret + // the identical IEEE binary16 bit patterns the kernel expects. All 16 slots + // are filled even though the kernel reads only `len` of them: the caller + // already holds 16 valid slices, so there is nothing to gain from leaving + // the tail of the array undefined. + let mut rows = [std::ptr::null::(); 16]; + for (i, cand) in candidates.iter().enumerate() { + debug_assert_eq!( + cand.len(), + dim, + "candidate {i} length must equal query length" + ); + rows[i] = cand.as_ptr() as *const u16; + } + let mut out = [0f32; 16]; + unsafe { + lance_amx_dot_f16_batch_16( + query.as_ptr() as *const u16, + rows.as_ptr(), + len, + dim, + out.as_mut_ptr(), + ); + } + out +} + +/// Halfwords in one packed B block: 16 tile rows x 32 halfwords per row. +/// Mirrors `GEMM_B_BLOCK` in `amx_fp16.c`. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +const GEMM_B_BLOCK: usize = 512; + +/// Number of `f16` [`pack_centroids_vnni`] writes for `n` centroids of `dim` +/// dims. Only whole 32-dim k-blocks are packed; the `dim % 32` tail is left to +/// the kernel's scalar cleanup, which reads the unpacked centroids directly. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) fn packed_centroids_len(n: usize, dim: usize) -> usize { + (dim / 32) * (n / 16) * GEMM_B_BLOCK +} + +/// Interleave `[n, dim]` row-major `centroids` into the VNNI order +/// [`dot_f16_gemm_amx`]'s B tiles are loaded in, replacing `out`'s contents. +/// +/// The layout is dictated by TDPFP16PS, which reads its B operand as +/// `b.row[k].fp16[2*nn + p]` and pairs it with `a.row[mm].fp16[2*k + p]`. Since +/// A is loaded straight out of the vector buffer — `a.row[mm].fp16[2*k+p]` is +/// dim `kb*32 + 2*k + p` of vector `mm` — B must satisfy +/// +/// ```text +/// out[((kb * (n/16)) + jb) * 512 + k*32 + nn*2 + p] +/// == centroids[(jb*16 + nn) * dim + kb*32 + 2*k + p] +/// ``` +/// +/// with `kb` the 32-dim k-block and `jb` the 16-centroid block. `jb` is the +/// inner index so that the two B tiles a single k-pass consumes are adjacent. +/// +/// `n` must be a multiple of 16 (one B tile covers exactly 16 centroids); +/// `centroids` must hold `n * dim` values. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +pub(crate) fn pack_centroids_vnni(centroids: &[f16], n: usize, dim: usize, out: &mut Vec) { + debug_assert_eq!(n % 16, 0, "n ({n}) must be a multiple of 16"); + debug_assert_eq!( + centroids.len(), + n * dim, + "centroids must hold n*dim = {} values", + n * dim + ); + out.clear(); + out.reserve(packed_centroids_len(n, dim)); + for kb in 0..dim / 32 { + for jb in 0..n / 16 { + for k in 0..16 { + for nn in 0..16 { + for p in 0..2 { + out.push(centroids[(jb * 16 + nn) * dim + kb * 32 + 2 * k + p]); + } + } + } + } + } +} + +/// AMX-FP16 `[m, dim] x [n, dim]^T -> [m, n]` dot-product GEMM: scores every +/// row of `data` against every centroid, writing raw dot products (no distance +/// wrapping) into `out`. +/// +/// `packed_b` must come from [`pack_centroids_vnni`] over the same `centroids`, +/// `n` and `dim`; `centroids` is additionally passed through because the +/// `dim % 32` tail dims are not packed and the kernel finishes them in scalar +/// fp32. Rows of `data` are `data_stride` halfwords apart and rows of `out` are +/// `out_stride` floats apart, so a caller can hand over a window of a larger +/// buffer without copying. +/// +/// Accuracy matches [`dot_f16_batch_16_amx`]'s contract: f32-accumulated to +/// within fp16 precision, not bit-exact against a sequential scalar loop. +/// +/// # Safety +/// * [`crate::distance::dot_f16::amx_fp16_supported`] must have returned `true`. +/// * `m % 32 == 0` and `n % 32 == 0`. The kernel has no edge-case path for +/// partial tiles and would read and write past the ends of its buffers. +/// * `data_stride >= dim` and `out_stride >= n`, and the slices must be long +/// enough for the last row those strides reach. +#[cfg(all( + kernel_support = "amx_fp16", + target_arch = "x86_64", + target_os = "linux" +))] +#[allow(clippy::too_many_arguments)] +pub(crate) unsafe fn dot_f16_gemm_amx( + data: &[f16], + m: usize, + data_stride: usize, + packed_b: &[f16], + centroids: &[f16], + n: usize, + dim: usize, + out: &mut [f32], + out_stride: usize, +) { + debug_assert_eq!(m % 32, 0, "m ({m}) must be a multiple of 32"); + debug_assert_eq!(n % 32, 0, "n ({n}) must be a multiple of 32"); + debug_assert!( + data_stride >= dim, + "data_stride ({data_stride}) < dim ({dim})" + ); + debug_assert!(out_stride >= n, "out_stride ({out_stride}) < n ({n})"); + // Through `strided_len` rather than inline arithmetic, for the same reason + // the safe caller uses it: `(m - 1) * stride + row_len` wraps in release + // builds, and a wrapped requirement is small enough to satisfy the very + // check it was computed for. These are `debug_assert`s restating a contract + // the caller already enforced, but they should fail loudly on the + // overflowing shape rather than quietly agree with it. + debug_assert!( + strided_len(m, data_stride, dim).is_some_and(|need| data.len() >= need), + "data too short" + ); + debug_assert!( + strided_len(m, out_stride, n).is_some_and(|need| out.len() >= need), + "out too short" + ); + debug_assert_eq!( + Some(centroids.len()), + n.checked_mul(dim), + "centroids must hold n*dim values" + ); + debug_assert_eq!( + packed_b.len(), + packed_centroids_len(n, dim), + "packed_b must be pack_centroids_vnni's output for this n and dim" + ); + // half::f16 is #[repr(transparent)] over u16, so these casts reinterpret the + // identical IEEE binary16 bit patterns the kernel expects. + unsafe { + lance_amx_dot_f16_gemm( + data.as_ptr() as *const u16, + m, + data_stride, + packed_b.as_ptr() as *const u16, + centroids.as_ptr() as *const u16, + n, + dim, + out.as_mut_ptr(), + out_stride, + ); + } +} diff --git a/rust/lance-linalg/src/simd/dist_table.rs b/rust/lance-linalg/src/simd/dist_table.rs index 626c1581b15..00bc9143cf0 100644 --- a/rust/lance-linalg/src/simd/dist_table.rs +++ b/rust/lance-linalg/src/simd/dist_table.rs @@ -5,6 +5,7 @@ use std::arch::aarch64::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; +use std::mem::MaybeUninit; #[allow(unused_imports)] use lance_core::utils::cpu::{SIMD_SUPPORT, SimdSupport}; @@ -33,8 +34,39 @@ pub fn sum_4bit_dist_table( codes: &[u8], dist_table: &[u8], dists: &mut [u16], +) { + assert!(n.is_multiple_of(BATCH_SIZE)); + assert!(dists.len() >= n); + assert!(codes.len() >= n * code_len); + assert!(dist_table.len() >= BATCH_SIZE * code_len); + // A `u16` slice is also a valid `MaybeUninit` slice. The dispatched + // kernels overwrite every output slot. + let dists = unsafe { + std::slice::from_raw_parts_mut(dists.as_mut_ptr().cast::>(), dists.len()) + }; + unsafe { sum_4bit_dist_table_uninit(n, code_len, codes, dist_table, dists) }; +} + +/// Sum a 4-bit distance table into potentially uninitialized output storage. +/// +/// Every element in `dists[..n]` is initialized before this function returns. +/// +/// # Safety +/// +/// `n` must be a multiple of [`BATCH_SIZE`], `codes` must contain at least +/// `n * code_len` bytes, `dist_table` must contain at least +/// `BATCH_SIZE * code_len` bytes, and `dists` must contain at least `n` slots. +#[inline] +pub unsafe fn sum_4bit_dist_table_uninit( + n: usize, + code_len: usize, + codes: &[u8], + dist_table: &[u8], + dists: &mut [MaybeUninit], ) { debug_assert!(n.is_multiple_of(BATCH_SIZE)); + debug_assert!(dists.len() >= n); + debug_assert!(codes.len() >= n * code_len); match *SIMD_SUPPORT { #[cfg(all(kernel_support = "avx512_dist_table", target_arch = "x86_64"))] @@ -48,7 +80,7 @@ pub fn sum_4bit_dist_table( codes.as_ptr(), codes.len(), dist_table.as_ptr(), - dists[i..i + BATCH_SIZE].as_mut_ptr(), + dists[i..i + BATCH_SIZE].as_mut_ptr().cast::(), ) } } @@ -73,7 +105,17 @@ pub fn sum_4bit_dist_table( ) } }, - _ => sum_4bit_dist_table_scalar(code_len, codes, dist_table, dists), + // SimdSupport::AvxFma and SimdSupport::Avx fall through here: + // the AVX2 inner uses `_mm256_shuffle_epi8` / `_mm256_and_si256` / + // `_mm256_srli_epi16` / `_mm256_add_epi16` integer ops which + // neither AVX nor AVX+FMA provides. Scalar is the correct route. + _ => { + dists[..n].fill(MaybeUninit::new(0)); + // Every slot was initialized immediately above. + let dists = + unsafe { std::slice::from_raw_parts_mut(dists.as_mut_ptr().cast::(), n) }; + sum_4bit_dist_table_scalar(code_len, &codes[..n * code_len], dist_table, dists); + } } } @@ -159,6 +201,35 @@ pub fn sum_4bit_hacc_dist_table( codes: &[u8], hacc_dist_table: &[u8], dists: &mut [u32], +) { + assert!(n.is_multiple_of(BATCH_SIZE)); + assert!(dists.len() >= n); + assert!(codes.len() >= n * code_len); + assert!(hacc_dist_table.len() >= code_len * 64); + // A `u32` slice is also a valid `MaybeUninit` slice. The dispatched + // kernels overwrite every output slot. + let dists = unsafe { + std::slice::from_raw_parts_mut(dists.as_mut_ptr().cast::>(), dists.len()) + }; + unsafe { sum_4bit_hacc_dist_table_uninit(n, code_len, codes, hacc_dist_table, dists) }; +} + +/// Sum a high-accuracy 4-bit distance table into uninitialized output storage. +/// +/// Every element in `dists[..n]` is initialized before this function returns. +/// +/// # Safety +/// +/// `n` must be a multiple of [`BATCH_SIZE`], `codes` must contain at least +/// `n * code_len` bytes, `hacc_dist_table` must contain at least +/// `code_len * 64` bytes, and `dists` must contain at least `n` slots. +#[inline] +pub unsafe fn sum_4bit_hacc_dist_table_uninit( + n: usize, + code_len: usize, + codes: &[u8], + hacc_dist_table: &[u8], + dists: &mut [MaybeUninit], ) { debug_assert!(n.is_multiple_of(BATCH_SIZE)); debug_assert!(dists.len() >= n); @@ -172,7 +243,18 @@ pub fn sum_4bit_hacc_dist_table( { sum_4bit_hacc_dist_table_avx2(n, code_len, codes, hacc_dist_table, dists); } - _ => sum_4bit_hacc_dist_table_scalar(code_len, codes, hacc_dist_table, dists), + _ => { + dists[..n].fill(MaybeUninit::new(0)); + // Every slot was initialized immediately above. + let dists = + unsafe { std::slice::from_raw_parts_mut(dists.as_mut_ptr().cast::(), n) }; + sum_4bit_hacc_dist_table_scalar( + code_len, + &codes[..n * code_len], + hacc_dist_table, + dists, + ); + } } } @@ -257,20 +339,24 @@ fn sum_4bit_hacc_dist_table_avx2( code_len: usize, codes: &[u8], hacc_dist_table: &[u8], - dists: &mut [u32], + dists: &mut [MaybeUninit], ) { const SAFE_CODE_LEN: usize = 128; for i in (0..n).step_by(BATCH_SIZE) { let batch_codes = &codes[i * code_len..(i + BATCH_SIZE) * code_len]; let batch_dists = &mut dists[i..i + BATCH_SIZE]; - batch_dists.fill(0); + + if code_len == 0 { + batch_dists.fill(MaybeUninit::new(0)); + continue; + } for code_start in (0..code_len).step_by(SAFE_CODE_LEN) { let code_end = (code_start + SAFE_CODE_LEN).min(code_len); let code_range = code_start * BATCH_SIZE..code_end * BATCH_SIZE; let table_range = code_start * 64..code_end * 64; - if code_start == 0 && code_end == code_len { + if code_start == 0 { unsafe { sum_hacc_dist_table_32bytes_batch_avx2( &batch_codes[code_range], @@ -279,7 +365,7 @@ fn sum_4bit_hacc_dist_table_avx2( ); } } else { - let mut chunk_dists = [0u32; BATCH_SIZE]; + let mut chunk_dists = [MaybeUninit::::uninit(); BATCH_SIZE]; unsafe { sum_hacc_dist_table_32bytes_batch_avx2( &batch_codes[code_range], @@ -287,6 +373,17 @@ fn sum_4bit_hacc_dist_table_avx2( &mut chunk_dists, ); } + // The kernel above initializes every temporary output slot. + let chunk_dists = unsafe { + std::slice::from_raw_parts(chunk_dists.as_ptr().cast::(), BATCH_SIZE) + }; + // The first code chunk initialized every output slot. + let batch_dists = unsafe { + std::slice::from_raw_parts_mut( + batch_dists.as_mut_ptr().cast::(), + BATCH_SIZE, + ) + }; batch_dists .iter_mut() .zip(chunk_dists.iter()) @@ -303,7 +400,7 @@ fn sum_4bit_hacc_dist_table_avx2( unsafe fn sum_hacc_dist_table_32bytes_batch_avx2( codes: &[u8], hacc_dist_table: &[u8], - dists: &mut [u32], + dists: &mut [MaybeUninit], ) { let low_mask = _mm256_set1_epi8(0x0f); let mut low_accu0 = _mm256_setzero_si256(); @@ -385,7 +482,11 @@ unsafe fn sum_hacc_dist_table_32bytes_batch_avx2( #[target_feature(enable = "avx2")] #[inline] #[allow(unused)] -unsafe fn sum_dist_table_32bytes_batch_avx2(codes: &[u8], dist_table: &[u8], dists: &mut [u16]) { +unsafe fn sum_dist_table_32bytes_batch_avx2( + codes: &[u8], + dist_table: &[u8], + dists: &mut [MaybeUninit], +) { let mut c = _mm256_undefined_si256(); let mut lo = _mm256_undefined_si256(); let mut hi = _mm256_undefined_si256(); @@ -457,7 +558,11 @@ unsafe fn sum_dist_table_32bytes_batch_avx2(codes: &[u8], dist_table: &[u8], dis #[cfg(target_arch = "aarch64")] #[inline] -unsafe fn sum_dist_table_32bytes_batch_neon(codes: &[u8], dist_table: &[u8], dists: &mut [u16]) { +unsafe fn sum_dist_table_32bytes_batch_neon( + codes: &[u8], + dist_table: &[u8], + dists: &mut [MaybeUninit], +) { let low_mask = vdupq_n_u8(0x0f); // 8 accumulators: 4 per 128-bit "lane" (lo = bytes 0..16, hi = bytes 16..32 of each block) @@ -513,8 +618,8 @@ unsafe fn sum_dist_table_32bytes_batch_neon(codes: &[u8], dist_table: &[u8], dis // This is the NEON equivalent of AVX2's permute2f128 + blend + add let dis0_even = vaddq_u16(accu0_lo, accu0_hi); let dis0_odd = vaddq_u16(accu1_lo, accu1_hi); - vst1q_u16(dists.as_mut_ptr(), dis0_even); - vst1q_u16(dists.as_mut_ptr().add(8), dis0_odd); + vst1q_u16(dists.as_mut_ptr().cast::(), dis0_even); + vst1q_u16(dists.as_mut_ptr().add(8).cast::(), dis0_odd); // Same for hi-nibble accumulators (vectors 16..31) accu2_lo = vsubq_u16(accu2_lo, vshlq_n_u16::<8>(accu3_lo)); @@ -522,8 +627,8 @@ unsafe fn sum_dist_table_32bytes_batch_neon(codes: &[u8], dist_table: &[u8], dis let dis1_even = vaddq_u16(accu2_lo, accu2_hi); let dis1_odd = vaddq_u16(accu3_lo, accu3_hi); - vst1q_u16(dists.as_mut_ptr().add(16), dis1_even); - vst1q_u16(dists.as_mut_ptr().add(24), dis1_odd); + vst1q_u16(dists.as_mut_ptr().add(16).cast::(), dis1_even); + vst1q_u16(dists.as_mut_ptr().add(24).cast::(), dis1_odd); } // We implement the AVX512 version in C because AVX512 is not stable yet in Rust, diff --git a/rust/lance-linalg/src/simd/f32.rs b/rust/lance-linalg/src/simd/f32.rs index 78042997121..434a1ef9f18 100644 --- a/rust/lance-linalg/src/simd/f32.rs +++ b/rust/lance-linalg/src/simd/f32.rs @@ -46,14 +46,42 @@ impl std::fmt::Debug for f32x8 { } impl f32x8 { + /// Gather 8 f32 values from `slice` at the offsets in `indices`. + /// + /// On x86_64 this uses the AVX2 `vgatherdps` instruction when the host + /// supports it (gated at runtime via `is_x86_feature_detected!`). On + /// other architectures (and on x86_64 hosts without AVX2) the function + /// falls back to a per-index scalar load followed by `Self::from(&out)`, + /// which goes through `load_unaligned` (NEON / LASX / `_mm256_loadu_ps` + /// depending on platform). Per-tier macro stamping (e.g., the + /// `multiversion` crate) was considered but doesn't fit here: the function + /// returns `Self` and `_mm256_i32gather_ps::<4>` requires the const-generic + /// stride to be a compile-time literal — neither composes with the macro. + /// + /// # Panics + /// + /// If any index is negative or lands outside `slice`. #[inline] pub fn gather(slice: &[f32], indices: &[i32; 8]) -> Self { - #[cfg(target_arch = "x86_64")] - unsafe { - use super::i32::i32x8; + // Every backend below reads without bounds checking: `vgatherdps` does + // none, and the NEON / LASX arms offset a raw pointer. Check once here + // so an out-of-range index panics on every host rather than reading out + // of bounds on some and panicking on others. + for &i in indices { + assert!( + (i as usize) < slice.len(), + "gather index {i} is out of bounds for a slice of length {}", + slice.len() + ); + } - let idx = i32x8::from(indices); - Self(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0)) + #[cfg(target_arch = "x86_64")] + { + if is_x86_feature_detected!("avx2") { + unsafe { gather_avx2(slice, indices) } + } else { + gather_scalar_x86(slice, indices) + } } #[cfg(target_arch = "aarch64")] @@ -94,8 +122,37 @@ impl f32x8 { } } +/// AVX2 gather. Caller must ensure the host supports AVX2 (gated by +/// the `is_x86_feature_detected!("avx2")` check in `f32x8::gather`). +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn gather_avx2(slice: &[f32], indices: &[i32; 8]) -> f32x8 { + use super::i32::i32x8; + + let idx = i32x8::from(indices); + f32x8(_mm256_i32gather_ps::<4>(slice.as_ptr(), idx.0)) +} + +/// Portable scalar gather for x86_64 hosts without AVX2. +/// +/// Indexes the slice rather than offsetting a raw pointer: this is the slow +/// path already, so an out-of-range index should panic instead of reading out +/// of bounds. +#[cfg(target_arch = "x86_64")] +#[inline] +fn gather_scalar_x86(slice: &[f32], indices: &[i32; 8]) -> f32x8 { + let values = indices.map(|i| slice[i as usize]); + // SAFETY: `values` is eight contiguous, initialized `f32`. + unsafe { f32x8::load_unaligned(values.as_ptr()) } +} + impl From<&[f32]> for f32x8 { fn from(value: &[f32]) -> Self { + assert!( + value.len() >= 8, + "f32x8 requires at least 8 values, got {}", + value.len() + ); unsafe { Self::load_unaligned(value.as_ptr()) } } } @@ -439,15 +496,18 @@ impl Mul for f32x8 { } } -/// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible. +/// 16 of 32-bit `f32` values. Stored as a pair of 256-bit AVX vectors on +/// x86_64. Originally there was a sibling AVX-512 variant gated on +/// `target_feature = "avx512f"`, but no project CI configuration enables +/// `+avx512f` globally (and one of the avx512 arms still contained `todo!()`), +/// so the variant was dead code. Removed in the runtime-SIMD-dispatch +/// retrofit; per-tier dispatch happens in the kernel functions in +/// `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier +/// `#[target_feature(enable = "...")]` inner functions. #[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] +#[cfg(target_arch = "x86_64")] #[derive(Clone, Copy)] pub struct f32x16(__m256, __m256); -#[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -#[derive(Clone, Copy)] -pub struct f32x16(__m512); /// 16 of 32-bit `f32` values. Use 512-bit SIMD if possible. #[allow(non_camel_case_types)] @@ -473,6 +533,11 @@ impl std::fmt::Debug for f32x16 { impl From<&[f32]> for f32x16 { fn from(value: &[f32]) -> Self { + assert!( + value.len() >= 16, + "f32x16 requires at least 16 values, got {}", + value.len() + ); unsafe { Self::load_unaligned(value.as_ptr()) } } } @@ -486,11 +551,7 @@ impl<'a> From<&'a [f32; 16]> for f32x16 { impl SIMD for f32x16 { #[inline] fn splat(val: f32) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_set1_ps(val)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_set1_ps(val), _mm256_set1_ps(val)) } @@ -514,11 +575,7 @@ impl SIMD for f32x16 { #[inline] fn zeros() -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_setzero_ps()) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_setzero_ps(), _mm256_setzero_ps()) } @@ -534,14 +591,10 @@ impl SIMD for f32x16 { #[inline] unsafe fn load(ptr: *const f32) -> Self { - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_load_ps(ptr), _mm256_load_ps(ptr.add(8))) } - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_load_ps(ptr)) - } #[cfg(target_arch = "aarch64")] { Self::load_unaligned(ptr) @@ -557,14 +610,10 @@ impl SIMD for f32x16 { #[inline] unsafe fn load_unaligned(ptr: *const f32) -> Self { - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_loadu_ps(ptr), _mm256_loadu_ps(ptr.add(8))) } - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_loadu_ps(ptr)) - } #[cfg(target_arch = "aarch64")] { Self(vld1q_f32_x4(ptr)) @@ -580,11 +629,7 @@ impl SIMD for f32x16 { #[inline] unsafe fn store(&self, ptr: *mut f32) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_store_ps(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_store_ps(ptr, self.0); _mm256_store_ps(ptr.add(8), self.1); @@ -602,11 +647,7 @@ impl SIMD for f32x16 { #[inline] unsafe fn store_unaligned(&self, ptr: *mut f32) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_storeu_ps(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_storeu_ps(ptr, self.0); _mm256_storeu_ps(ptr.add(8), self.1); @@ -622,12 +663,9 @@ impl SIMD for f32x16 { } } + #[inline] fn reduce_sum(&self) -> f32 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_add_ps(0xFFFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let mut sum = _mm256_add_ps(self.0, self.1); // Shift and add vector, until only 1 value left. @@ -657,11 +695,7 @@ impl SIMD for f32x16 { #[inline] fn reduce_min(&self) -> f32 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_min_ps(0xFFFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let mut m1 = _mm256_min_ps(self.0, self.1); let mut m2 = _mm256_permute2f128_ps(m1, m1, 1); @@ -695,11 +729,7 @@ impl SIMD for f32x16 { #[inline] fn min(&self, rhs: &Self) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_min_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_min_ps(self.0, rhs.0), _mm256_min_ps(self.1, rhs.1)) } @@ -718,21 +748,15 @@ impl SIMD for f32x16 { } } + #[inline] fn find(&self, val: f32) -> Option { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - // let tgt = _mm512_set1_ps(val); - // let mask = _mm512_cmpeq_ps_mask(self.0, tgt); - // if mask != 0 { - // return Some(mask.trailing_zeros() as i32); - // } - todo!() - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { - // _mm256_cmpeq_ps_mask requires "avx512l". + // _mm256_cmpeq_ps_mask requires AVX-512 (avx512f); use a scalar scan here + // since we only require AVX2. + let arr = self.as_array(); for i in 0..16 { - if self.as_array().get_unchecked(i) == &val { + if arr.get_unchecked(i) == &val { return Some(i as i32); } } @@ -774,11 +798,7 @@ impl SIMD for f32x16 { impl FloatSimd for f32x16 { #[inline] fn multiply_add(&mut self, a: Self, b: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_fmadd_ps(a.0, b.0, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_fmadd_ps(a.0, b.0, self.0); self.1 = _mm256_fmadd_ps(a.1, b.1, self.1); @@ -803,11 +823,7 @@ impl Add for f32x16 { #[inline] fn add(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_add_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_add_ps(self.0, rhs.0), _mm256_add_ps(self.1, rhs.1)) } @@ -830,11 +846,7 @@ impl Add for f32x16 { impl AddAssign for f32x16 { #[inline] fn add_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_add_ps(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_add_ps(self.0, rhs.0); self.1 = _mm256_add_ps(self.1, rhs.1); @@ -859,11 +871,7 @@ impl Mul for f32x16 { #[inline] fn mul(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_mul_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_mul_ps(self.0, rhs.0), _mm256_mul_ps(self.1, rhs.1)) } @@ -888,11 +896,7 @@ impl Sub for f32x16 { #[inline] fn sub(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_sub_ps(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_sub_ps(self.0, rhs.0), _mm256_sub_ps(self.1, rhs.1)) } @@ -915,11 +919,7 @@ impl Sub for f32x16 { impl SubAssign for f32x16 { #[inline] fn sub_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_sub_ps(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_sub_ps(self.0, rhs.0); self.1 = _mm256_sub_ps(self.1, rhs.1); @@ -943,9 +943,23 @@ impl SubAssign for f32x16 { mod tests { use super::*; + use rstest::rstest; + + #[test] + fn test_slice_conversion_rejects_short_input() { + assert!(std::panic::catch_unwind(|| f32x8::from(&[0.0; 7][..])).is_err()); + assert!(std::panic::catch_unwind(|| f32x16::from(&[0.0; 15][..])).is_err()); + } #[test] fn test_basic_ops() { + // Load / store / arithmetic on `f32x8` lower to AVX intrinsics, and + // `multiply_add` lowers to `_mm256_fmadd_ps`, which needs FMA. Both + // are present from the AvxFma tier up. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a = (0..8).map(|f| f as f32).collect::>(); let b = (10..18).map(|f| f as f32).collect::>(); @@ -983,6 +997,11 @@ mod tests { #[test] fn test_f32x8_cmp_ops() { + // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0]; let b = [2.0_f32, 1.0, 4.0, 5.0, 9.0, 5.0, 6.0, 2.0]; let c = [2.0_f32, 1.0, 4.0, 5.0, 7.0, 3.0, 2.0, 1.0]; @@ -1007,6 +1026,11 @@ mod tests { #[test] fn test_basic_f32x16_ops() { + // `f32x16` is a pair of `__m256`; `multiply_add` needs FMA. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a = (0..16).map(|f| f as f32).collect::>(); let b = (10..26).map(|f| f as f32).collect::>(); @@ -1041,6 +1065,11 @@ mod tests { #[test] fn test_f32x16_cmp_ops() { + // `min` / `reduce_min` are AVX intrinsics; `find` is a scalar scan. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [ 1.0_f32, 2.0, 5.0, 6.0, 7.0, 3.0, 2.0, 1.0, -0.5, 5.0, 6.0, 7.0, 8.0, 9.0, 1.0, 2.0, ]; @@ -1074,9 +1103,59 @@ mod tests { #[test] fn test_f32x8_gather() { + // `f32x8::gather` does its own runtime AVX2 detection and falls back + // to a scalar gather, so this test only needs whatever reading the + // `__m256`-backed result costs: AVX, for `reduce_sum`. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = (0..256).map(|f| f as f32).collect::>(); let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29]; let v = f32x8::gather(&a, &idx); assert_eq!(v.reduce_sum(), 113.0); } + + /// Directly exercises `gather_scalar_x86`, the per-index scalar fallback + /// `f32x8::gather` takes on x86_64 hosts without AVX2. Runtime AVX2 hosts + /// route through `gather_avx2` instead, so the fallback is otherwise never + /// hit under coverage. Reading the `__m256`-backed result needs AVX, so + /// skip on hosts without it. + #[cfg(target_arch = "x86_64")] + #[test] + fn test_gather_scalar_x86() { + if !std::is_x86_feature_detected!("avx") { + return; + } + let a = (0..256).map(|f| f as f32).collect::>(); + let idx = [0_i32, 4, 8, 12, 16, 20, 24, 29]; + let v = gather_scalar_x86(&a, &idx); + let expected = idx.map(|i| a[i as usize]); + assert_eq!(v.as_array(), expected); + } + + /// An index past the end of the slice panics rather than reading out of + /// bounds. The bounds check fires before any AVX instruction, so this + /// case runs on every x86_64 host. + #[cfg(target_arch = "x86_64")] + #[test] + #[should_panic(expected = "index out of bounds")] + fn test_gather_scalar_x86_rejects_out_of_range_index() { + let a = (0..8).map(|f| f as f32).collect::>(); + let idx = [0_i32, 1, 2, 3, 4, 5, 6, 99]; + let _ = gather_scalar_x86(&a, &idx); + } + + /// `gather` validates before dispatching, so every backend — `vgatherdps`, + /// the x86 scalar fallback, and the NEON / LASX raw-pointer arms — rejects + /// a bad index identically instead of reading out of bounds. + #[rstest] + #[case::past_end(99)] + #[case::negative(-1)] + #[should_panic(expected = "out of bounds")] + fn test_gather_rejects_invalid_index(#[case] bad_index: i32) { + let a = (0..8).map(|f| f as f32).collect::>(); + let idx = [0_i32, 1, 2, 3, 4, 5, 6, bad_index]; + let _ = f32x8::gather(&a, &idx); + } } diff --git a/rust/lance-linalg/src/simd/f64.rs b/rust/lance-linalg/src/simd/f64.rs index 32c0d389e5b..129b2f088ec 100644 --- a/rust/lance-linalg/src/simd/f64.rs +++ b/rust/lance-linalg/src/simd/f64.rs @@ -45,6 +45,11 @@ impl std::fmt::Debug for f64x4 { impl From<&[f64]> for f64x4 { fn from(value: &[f64]) -> Self { + assert!( + value.len() >= 4, + "f64x4 requires at least 4 values, got {}", + value.len() + ); unsafe { Self::load_unaligned(value.as_ptr()) } } } @@ -355,14 +360,15 @@ impl Mul for f64x4 { // f64x8: 8 × f64 values (512-bit SIMD or 2 × 256-bit) // --------------------------------------------------------------------------- -/// 8 of 64-bit `f64` values. Uses 512-bit SIMD if possible. +/// 8 of 64-bit `f64` values. Stored as a pair of 256-bit AVX vectors on +/// x86_64. Originally there was a sibling AVX-512 variant gated on +/// `target_feature = "avx512f"`, but no project CI configuration enables +/// `+avx512f` globally, so the variant was dead code. Removed in the +/// runtime-SIMD-dispatch retrofit; per-tier dispatch happens in the kernel +/// functions in `crate::distance::*` via `match *SIMD_SUPPORT` + per-tier +/// `#[target_feature(enable = "...")]` inner functions. #[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] -#[derive(Clone, Copy)] -pub struct f64x8(__m512d); - -#[allow(non_camel_case_types)] -#[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] +#[cfg(target_arch = "x86_64")] #[derive(Clone, Copy)] pub struct f64x8(__m256d, __m256d); @@ -388,6 +394,11 @@ impl std::fmt::Debug for f64x8 { impl From<&[f64]> for f64x8 { fn from(value: &[f64]) -> Self { + assert!( + value.len() >= 8, + "f64x8 requires at least 8 values, got {}", + value.len() + ); unsafe { Self::load_unaligned(value.as_ptr()) } } } @@ -401,11 +412,7 @@ impl<'a> From<&'a [f64; 8]> for f64x8 { impl SIMD for f64x8 { #[inline] fn splat(val: f64) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_set1_pd(val)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_set1_pd(val), _mm256_set1_pd(val)) } @@ -423,11 +430,7 @@ impl SIMD for f64x8 { #[inline] fn zeros() -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_setzero_pd()) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_setzero_pd(), _mm256_setzero_pd()) } @@ -443,11 +446,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn load(ptr: *const f64) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_load_pd(ptr)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_load_pd(ptr), _mm256_load_pd(ptr.add(4))) } @@ -466,11 +465,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn load_unaligned(ptr: *const f64) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_loadu_pd(ptr)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_loadu_pd(ptr), _mm256_loadu_pd(ptr.add(4))) } @@ -489,11 +484,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn store(&self, ptr: *mut f64) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_store_pd(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_store_pd(ptr, self.0); _mm256_store_pd(ptr.add(4), self.1); @@ -512,11 +503,7 @@ impl SIMD for f64x8 { #[inline] unsafe fn store_unaligned(&self, ptr: *mut f64) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_storeu_pd(ptr, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { _mm256_storeu_pd(ptr, self.0); _mm256_storeu_pd(ptr.add(4), self.1); @@ -533,12 +520,9 @@ impl SIMD for f64x8 { } } + #[inline] fn reduce_sum(&self) -> f64 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_add_pd(0xFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let sum = _mm256_add_pd(self.0, self.1); let hi = _mm256_permute2f128_pd(sum, sum, 1); @@ -561,11 +545,7 @@ impl SIMD for f64x8 { #[inline] fn reduce_min(&self) -> f64 { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - _mm512_mask_reduce_min_pd(0xFF, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { let m = _mm256_min_pd(self.0, self.1); let hi = _mm256_permute2f128_pd(m, m, 1); @@ -592,11 +572,7 @@ impl SIMD for f64x8 { #[inline] fn min(&self, rhs: &Self) -> Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_min_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_min_pd(self.0, rhs.0), _mm256_min_pd(self.1, rhs.1)) } @@ -613,6 +589,7 @@ impl SIMD for f64x8 { } } + #[inline] fn find(&self, val: f64) -> Option { unsafe { for i in 0..8 { @@ -628,11 +605,7 @@ impl SIMD for f64x8 { impl FloatSimd for f64x8 { #[inline] fn multiply_add(&mut self, a: Self, b: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_fmadd_pd(a.0, b.0, self.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_fmadd_pd(a.0, b.0, self.0); self.1 = _mm256_fmadd_pd(a.1, b.1, self.1); @@ -657,11 +630,7 @@ impl Add for f64x8 { #[inline] fn add(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_add_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_add_pd(self.0, rhs.0), _mm256_add_pd(self.1, rhs.1)) } @@ -682,11 +651,7 @@ impl Add for f64x8 { impl AddAssign for f64x8 { #[inline] fn add_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_add_pd(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_add_pd(self.0, rhs.0); self.1 = _mm256_add_pd(self.1, rhs.1); @@ -711,11 +676,7 @@ impl Mul for f64x8 { #[inline] fn mul(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_mul_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_mul_pd(self.0, rhs.0), _mm256_mul_pd(self.1, rhs.1)) } @@ -738,11 +699,7 @@ impl Sub for f64x8 { #[inline] fn sub(self, rhs: Self) -> Self::Output { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - Self(_mm512_sub_pd(self.0, rhs.0)) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { Self(_mm256_sub_pd(self.0, rhs.0), _mm256_sub_pd(self.1, rhs.1)) } @@ -763,11 +720,7 @@ impl Sub for f64x8 { impl SubAssign for f64x8 { #[inline] fn sub_assign(&mut self, rhs: Self) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] - unsafe { - self.0 = _mm512_sub_pd(self.0, rhs.0) - } - #[cfg(all(target_arch = "x86_64", not(target_feature = "avx512f")))] + #[cfg(target_arch = "x86_64")] unsafe { self.0 = _mm256_sub_pd(self.0, rhs.0); self.1 = _mm256_sub_pd(self.1, rhs.1); @@ -791,8 +744,20 @@ impl SubAssign for f64x8 { mod tests { use super::*; + #[test] + fn test_slice_conversion_rejects_short_input() { + assert!(std::panic::catch_unwind(|| f64x4::from(&[0.0; 3][..])).is_err()); + assert!(std::panic::catch_unwind(|| f64x8::from(&[0.0; 7][..])).is_err()); + } + #[test] fn test_f64x4_basic_ops() { + // The `f64x4` constructor / load / store / arithmetic paths all lower + // to AVX intrinsics on x86_64; none of them need AVX2. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [1.0_f64, 2.0, 3.0, 4.0]; let b = [5.0_f64, 6.0, 7.0, 8.0]; @@ -814,6 +779,11 @@ mod tests { #[test] fn test_f64x4_fma() { + // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a = [1.0_f64, 2.0, 3.0, 4.0]; let b = [2.0_f64, 3.0, 4.0, 5.0]; @@ -826,6 +796,11 @@ mod tests { #[test] fn test_f64x4_min() { + // `min` / `reduce_min` are AVX intrinsics. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a = [1.0_f64, 5.0, 2.0, 8.0]; let b = [3.0_f64, 2.0, 4.0, 1.0]; let simd_a: f64x4 = (&a).into(); @@ -838,6 +813,11 @@ mod tests { #[test] fn test_f64x8_basic_ops() { + // `f64x8` is a pair of `__m256d`; add / reduce are AVX intrinsics. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let b: [f64; 8] = [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0]; @@ -856,6 +836,11 @@ mod tests { #[test] fn test_f64x8_fma() { + // `multiply_add` lowers to `_mm256_fmadd_pd`, which needs FMA. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") || !std::is_x86_feature_detected!("fma") { + return; + } let a: [f64; 8] = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]; let b: [f64; 8] = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]; @@ -869,6 +854,11 @@ mod tests { #[test] fn test_f64x8_min() { + // `min` / `reduce_min` are AVX intrinsics. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx") { + return; + } let a: [f64; 8] = [5.0, 1.0, 8.0, 3.0, 9.0, 2.0, 7.0, 4.0]; let b: [f64; 8] = [2.0, 6.0, 3.0, 7.0, 1.0, 8.0, 4.0, 9.0]; let simd_a: f64x8 = (&a).into(); diff --git a/rust/lance-linalg/src/simd/i32.rs b/rust/lance-linalg/src/simd/i32.rs index fa8cdafe6e7..6e0812928db 100644 --- a/rust/lance-linalg/src/simd/i32.rs +++ b/rust/lance-linalg/src/simd/i32.rs @@ -15,16 +15,24 @@ use std::mem::transmute; use super::SIMD; +/// 8 of 32-bit `i32` values. Use 256-bit SIMD if possible. +/// +/// The x86_64 arm reaches AVX and AVX2 intrinsics with no `#[target_feature]` +/// gate of its own, so callers must already be inside an AVX2-checked context. +/// `x86_64-unknown-linux-gnu` is pinned to `target-cpu=x86-64-v2` +/// (`.cargo/config.toml`), which is below AVX. #[allow(non_camel_case_types)] #[cfg(target_arch = "x86_64")] #[derive(Clone, Copy)] pub struct i32x8(pub(crate) __m256i); +/// 8 of 32-bit `i32` values. Use 256-bit SIMD if possible. #[allow(non_camel_case_types)] #[cfg(target_arch = "aarch64")] #[derive(Clone, Copy)] pub struct i32x8(int32x4x2_t); +/// 8 of 32-bit `i32` values. Use 256-bit SIMD if possible. #[allow(non_camel_case_types)] #[cfg(target_arch = "loongarch64")] #[derive(Clone, Copy)] @@ -42,6 +50,11 @@ impl std::fmt::Debug for i32x8 { impl From<&[i32]> for i32x8 { fn from(value: &[i32]) -> Self { + assert!( + value.len() >= 8, + "i32x8 requires at least 8 values, got {}", + value.len() + ); unsafe { Self::load_unaligned(value.as_ptr()) } } } @@ -297,11 +310,24 @@ impl SubAssign for i32x8 { impl Mul for i32x8 { type Output = Self; + /// Lane-wise product, keeping the low 32 bits of each result. + /// + /// `mul` wraps on overflow rather than panicking the way scalar `i32 * i32` + /// does in a debug build, and all three arms agree on that: `vpmulld`, + /// `vmulq_s32` and `lasx_xvmul_w` each discard the high half. This is a + /// statement about `mul` alone — `reduce_sum` sums in scalar `i32` on x86_64 + /// and loongarch64 (so it panics on overflow in a debug build) but reduces + /// in-register on aarch64, where it wraps. + /// + /// Picking a widening variant here is a silent wrong answer, not a compile + /// error: `_mm256_mul_epi32` (`vpmuldq`) multiplies only the even 32-bit + /// lanes and writes four 64-bit results, so `[1, 2, ..., 8]` squared came + /// back as `[1, 0, 9, 0, 25, 0, 49, 0]`. #[inline] fn mul(self, rhs: Self) -> Self::Output { #[cfg(target_arch = "x86_64")] unsafe { - Self(_mm256_mul_epi32(self.0, rhs.0)) + Self(_mm256_mullo_epi32(self.0, rhs.0)) } #[cfg(target_arch = "aarch64")] unsafe { @@ -318,4 +344,37 @@ impl Mul for i32x8 { } #[cfg(test)] -mod tests {} +mod tests { + use super::*; + use rstest::rstest; + + #[test] + fn test_slice_conversion_rejects_short_input() { + assert!(std::panic::catch_unwind(|| i32x8::from(&[0; 7][..])).is_err()); + } + + /// Lane-wise, low-32-bits multiplication is what all three arms promise, so + /// this runs everywhere: only the x86 feature check is arch-gated, matching + /// `f32.rs`'s and `f64.rs`'s test modules. + /// + /// Every case below has to produce a different answer under the widening + /// `vpmuldq` this file used to call. All-zero *inputs* would not: `vpmuldq` + /// returns zeros for those too. + #[rstest] + #[case::squares([1, 2, 3, 4, 5, 6, 7, 8], [1, 2, 3, 4, 5, 6, 7, 8], [1, 4, 9, 16, 25, 36, 49, 64])] + #[case::mixed_signs([-3, 7, -3, 7, -3, 7, -3, 7], [7, -3, 7, -3, 7, -3, 7, -3], [-21; 8])] + #[case::wraps_to_low_32_bits([65536; 8], [65536; 8], [0; 8])] + fn mul_is_lane_wise(#[case] lhs: [i32; 8], #[case] rhs: [i32; 8], #[case] expected: [i32; 8]) { + // `load_unaligned` / `store_unaligned` are AVX and `mul` is AVX2, and + // none of them is `#[target_feature]`-gated, so a pre-Haswell host would + // SIGILL. The `qemu-pre-haswell` CI job runs exactly that. + #[cfg(target_arch = "x86_64")] + if !std::is_x86_feature_detected!("avx2") { + return; + } + + let product = i32x8::from(&lhs) * i32x8::from(&rhs); + + assert_eq!(product.as_array(), expected); + } +} diff --git a/rust/lance-linalg/src/simd/u8.rs b/rust/lance-linalg/src/simd/u8.rs index 357a02a94ae..8720cd86e8c 100644 --- a/rust/lance-linalg/src/simd/u8.rs +++ b/rust/lance-linalg/src/simd/u8.rs @@ -85,6 +85,11 @@ impl std::fmt::Debug for u8x16 { impl From<&[u8]> for u8x16 { fn from(value: &[u8]) -> Self { + assert!( + value.len() >= 16, + "u8x16 requires at least 16 values, got {}", + value.len() + ); unsafe { Self::load_unaligned(value.as_ptr()) } } } @@ -403,6 +408,11 @@ mod tests { use super::*; + #[test] + fn test_slice_conversion_rejects_short_input() { + assert!(std::panic::catch_unwind(|| u8x16::from(&[0; 15][..])).is_err()); + } + #[test] fn test_basic_u8x16_ops() { let a = (0..16).map(|f| f as u8).collect::>(); diff --git a/rust/lance-linalg/src/simd/x86.rs b/rust/lance-linalg/src/simd/x86.rs new file mode 100644 index 00000000000..437018669e2 --- /dev/null +++ b/rust/lance-linalg/src/simd/x86.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Reduction helpers shared by the x86_64 AVX kernels in [`crate::distance`]. +//! +//! Each distance kernel accumulates into a 256-bit register and folds it down +//! to a scalar once, after the main loop. The fold is identical across +//! `cosine`, `dot`, `l2` and `norm_l2`, so it lives here instead of being +//! copied into each kernel's private `mod x86`. +//! +//! The module itself is `pub(crate)`, which is what keeps these helpers off the +//! public API; the items are `pub` rather than `pub(crate)` only because +//! `clippy::redundant_pub_crate` fires on the narrower visibility. + +use std::arch::x86_64::*; + +/// Horizontal sum of the eight `f32` lanes of an `__m256`. +/// +/// Folds the upper 128-bit lane into the lower one, then reduces the +/// remaining four lanes pairwise. Uses `movehl`/`shuffle` plus scalar adds +/// rather than two `vhaddps`, which is one fewer uop on most cores. +/// +/// # Safety +/// +/// The host must support AVX. Callers are `#[target_feature]`-annotated +/// kernels that the runtime dispatcher only selects after checking. +#[inline] +#[target_feature(enable = "avx")] +pub unsafe fn hsum256_ps(v: __m256) -> f32 { + let lo = _mm256_castps256_ps128(v); + let hi = _mm256_extractf128_ps(v, 1); + let sum128 = _mm_add_ps(lo, hi); + let sum64 = _mm_add_ps(sum128, _mm_movehl_ps(sum128, sum128)); + // 0x55 broadcasts lane 1 into lane 0, so the scalar add below lands the + // last of the four partial sums. + let sum32 = _mm_add_ss(sum64, _mm_shuffle_ps(sum64, sum64, 0x55)); + _mm_cvtss_f32(sum32) +} + +/// Horizontal sum of the four `f64` lanes of an `__m256d`. +/// +/// Folds the upper 128-bit lane into the lower one, then adds the remaining +/// pair. +/// +/// # Safety +/// +/// The host must support AVX. Callers are `#[target_feature]`-annotated +/// kernels that the runtime dispatcher only selects after checking. +#[inline] +#[target_feature(enable = "avx")] +pub unsafe fn hsum256_pd(v: __m256d) -> f64 { + let lo = _mm256_castpd256_pd128(v); + let hi = _mm256_extractf128_pd(v, 1); + let sum128 = _mm_add_pd(lo, hi); + let sum64 = _mm_add_pd(sum128, _mm_unpackhi_pd(sum128, sum128)); + _mm_cvtsd_f64(sum64) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case::ascending([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], 36.0)] + #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5, 0.0, -0.5, 1.0, 2.5], 5.0)] + #[case::zeros([0.0; 8], 0.0)] + #[case::cancelling([1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0, -4.0], 0.0)] + fn hsum256_ps_sums_every_lane(#[case] lanes: [f32; 8], #[case] expected: f32) { + if !std::is_x86_feature_detected!("avx") { + return; + } + let sum = unsafe { hsum256_ps(_mm256_loadu_ps(lanes.as_ptr())) }; + assert_eq!(sum, expected); + } + + #[rstest] + #[case::ascending([1.0, 2.0, 3.0, 4.0], 10.0)] + #[case::negative_lanes([-1.5, 2.0, -3.0, 4.5], 2.0)] + #[case::zeros([0.0; 4], 0.0)] + #[case::cancelling([1.0, -1.0, 2.0, -2.0], 0.0)] + fn hsum256_pd_sums_every_lane(#[case] lanes: [f64; 4], #[case] expected: f64) { + if !std::is_x86_feature_detected!("avx") { + return; + } + let sum = unsafe { hsum256_pd(_mm256_loadu_pd(lanes.as_ptr())) }; + assert_eq!(sum, expected); + } +} diff --git a/rust/lance-linalg/src/test_utils.rs b/rust/lance-linalg/src/test_utils.rs index afe173010fc..3779c6743a3 100644 --- a/rust/lance-linalg/src/test_utils.rs +++ b/rust/lance-linalg/src/test_utils.rs @@ -3,6 +3,24 @@ use half::{bf16, f16}; use proptest::prelude::*; +use proptest::test_runner::{Config, TestCaseResult, TestRunner}; +use std::ops::Range; + +const CASES_PER_DIMENSION_SHARD: u32 = 16; +const MIN_TEST_DIMENSION: usize = 4; +const MAX_TEST_DIMENSION: usize = 4048; +const NUM_DIMENSION_SHARDS: usize = 16; + +pub fn dimension_shard(shard: usize) -> Range { + let dimensions_per_shard = (MAX_TEST_DIMENSION - MIN_TEST_DIMENSION) / NUM_DIMENSION_SHARDS; + let start = MIN_TEST_DIMENSION + shard * dimensions_per_shard; + let end = if shard + 1 == NUM_DIMENSION_SHARDS { + MAX_TEST_DIMENSION + } else { + start + dimensions_per_shard + }; + start..end +} /// Arbitrary finite f16 value. pub fn arbitrary_f16() -> impl Strategy { @@ -79,3 +97,31 @@ where (x, y) }) } + +pub fn run_vector_pair_proptest(values: fn() -> S, dim_range: Range, property: F) +where + T: std::fmt::Debug, + S: Strategy + 'static, + F: Fn(Vec, Vec) -> TestCaseResult, +{ + let strategy = arbitrary_vector_pair(values, dim_range); + let mut runner = TestRunner::new(Config { + cases: CASES_PER_DIMENSION_SHARD, + ..Config::default() + }); + runner.run(&strategy, |(x, y)| property(x, y)).unwrap(); +} + +pub fn run_vector_proptest(values: fn() -> S, dim_range: Range, property: F) +where + T: std::fmt::Debug, + S: Strategy, + F: Fn(Vec) -> TestCaseResult, +{ + let strategy = prop::collection::vec(values(), dim_range); + let mut runner = TestRunner::new(Config { + cases: CASES_PER_DIMENSION_SHARD, + ..Config::default() + }); + runner.run(&strategy, property).unwrap(); +} diff --git a/rust/lance-namespace-datafusion/Cargo.toml b/rust/lance-namespace-datafusion/Cargo.toml index 28be0bd18f0..2a98ce4ef63 100755 --- a/rust/lance-namespace-datafusion/Cargo.toml +++ b/rust/lance-namespace-datafusion/Cargo.toml @@ -12,17 +12,17 @@ rust-version.workspace = true [dependencies] async-trait.workspace = true -dashmap = "6" +dashmap.workspace = true datafusion.workspace = true lance.workspace = true lance-namespace.workspace = true -tokio.workspace = true [dev-dependencies] arrow-array.workspace = true arrow-schema.workspace = true lance-namespace-impls.workspace = true tempfile.workspace = true +tokio.workspace = true [lints] workspace = true diff --git a/rust/lance-namespace-datafusion/src/catalog.rs b/rust/lance-namespace-datafusion/src/catalog.rs index 4fe57f63c9b..ce699037ba0 100755 --- a/rust/lance-namespace-datafusion/src/catalog.rs +++ b/rust/lance-namespace-datafusion/src/catalog.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::collections::HashSet; use std::sync::Arc; @@ -53,10 +52,6 @@ impl LanceCatalogProviderList { } impl CatalogProviderList for LanceCatalogProviderList { - fn as_any(&self) -> &dyn Any { - self - } - /// Adds a new catalog to this catalog list. /// If a catalog of the same name existed before, it is replaced in the list and returned. fn register_catalog( @@ -116,10 +111,6 @@ impl LanceCatalogProvider { } impl CatalogProvider for LanceCatalogProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema_names(&self) -> Vec { self.schemas .iter() diff --git a/rust/lance-namespace-datafusion/src/schema.rs b/rust/lance-namespace-datafusion/src/schema.rs index 9acf30a97bf..194346001c3 100755 --- a/rust/lance-namespace-datafusion/src/schema.rs +++ b/rust/lance-namespace-datafusion/src/schema.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::sync::Arc; use async_trait::async_trait; @@ -51,10 +50,6 @@ impl LanceSchemaProvider { #[async_trait] impl SchemaProvider for LanceSchemaProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn table_names(&self) -> Vec { self.tables .iter() diff --git a/rust/lance-namespace-impls/Cargo.toml b/rust/lance-namespace-impls/Cargo.toml index 27b9a4bc0e2..5a60cd07d09 100644 --- a/rust/lance-namespace-impls/Cargo.toml +++ b/rust/lance-namespace-impls/Cargo.toml @@ -32,8 +32,7 @@ lance-namespace.workspace = true lance-core.workspace = true # REST implementation dependencies (optional, enabled by "rest" feature) -reqwest = { version = "0.12", optional = true, default-features = false, features = [ - "json", +reqwest = { workspace = true, optional = true, features = [ "charset", "gzip", "http2", @@ -79,11 +78,6 @@ base64 = { version = "0.22", optional = true } aws-sdk-sts = { version = "1.38.0", optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] } aws-config = { workspace = true, optional = true } -# Pin: time 0.3.48 conflicts with aws-smithy-types (E0119: conflicting `From` impls), which this -# crate pulls in via the AWS credential vendor. Capping time here forces the workspace resolver to -# 0.3.47 even for no-lock builds. Not used directly; remove once the upstream conflict is resolved. -time = "=0.3.47" - # GCP credential vending dependencies (optional, enabled by "credential-vendor-gcp" feature) ring = { version = "0.17", optional = true } rustls-pki-types = { version = "1", optional = true } @@ -91,20 +85,12 @@ rustls-pki-types = { version = "1", optional = true } # Azure credential vending dependencies (optional, enabled by "credential-vendor-azure" feature) chrono = { workspace = true, optional = true } hmac = { version = "0.12", optional = true } -quick-xml = { version = "0.38", optional = true } +quick-xml = { version = "0.40", optional = true } [dev-dependencies] -opendal = { workspace = true, features = ["services-goosefs"] } -tokio = { workspace = true, features = ["full"] } tempfile.workspace = true wiremock.workspace = true -arrow = { workspace = true } -arrow-array = { workspace = true } -arrow-ipc = { workspace = true } rstest.workspace = true -lance-table.workspace = true -lance-arrow = { workspace = true } -lance = { workspace = true } serde = { workspace = true, features = ["derive"] } [[example]] diff --git a/rust/lance-namespace-impls/src/credentials.rs b/rust/lance-namespace-impls/src/credentials.rs index e841ac620f6..900745ceb3a 100644 --- a/rust/lance-namespace-impls/src/credentials.rs +++ b/rust/lance-namespace-impls/src/credentials.rs @@ -223,6 +223,14 @@ pub mod aws_props { /// AWS credential duration in milliseconds. /// Default: 3600000 (1 hour). Range: 900000 (15 min) to 43200000 (12 hours). pub const DURATION_MILLIS: &str = "aws_duration_millis"; + + /// When "true", the scoped assume is performed via `AssumeRoleWithWebIdentity` + /// using the pod's projected service-account OIDC token + pub const ASSUME_VIA_POD_WEB_IDENTITY: &str = "aws_assume_via_pod_web_identity"; + + /// Explicit path to the pod's projected SA OIDC token file. Overrides + /// `AWS_WEB_IDENTITY_TOKEN_FILE` when set. + pub const POD_WEB_IDENTITY_TOKEN_FILE: &str = "aws_pod_web_identity_token_file"; } /// GCP-specific property keys (short form, without prefix) @@ -475,6 +483,7 @@ async fn create_aws_vendor( properties: &HashMap, ) -> Result>> { use aws::{AwsCredentialVendor, AwsCredentialVendorConfig}; + use lance_core::utils::parse::str_is_truthy; use lance_namespace::error::NamespaceError; // AWS requires role_arn to be configured @@ -503,6 +512,43 @@ async fn create_aws_vendor( config = config.with_role_session_name(session_name); } + // Direct (non-chained) web-identity assume for the pod, when enabled. An + // explicit token-file path wins; otherwise, if opted in, resolve the + // EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`. Falling back to the chained + // AssumeRole path when neither is present keeps existing behavior. + let assume_via_pod = properties + .get(aws_props::ASSUME_VIA_POD_WEB_IDENTITY) + .map(|v| str_is_truthy(v)) + .unwrap_or(false); + let pod_token_file = properties + .get(aws_props::POD_WEB_IDENTITY_TOKEN_FILE) + .cloned() + .or_else(|| { + assume_via_pod + .then(|| std::env::var("AWS_WEB_IDENTITY_TOKEN_FILE").ok()) + .flatten() + }); + // Log the resolved assume path once at vendor init so a deployment can + // confirm at runtime which branch `assume_scoped` will take. + match &pod_token_file { + Some(path) => log::info!( + "AWS credential vendor (role {role_arn}): direct AssumeRoleWithWebIdentity \ + via pod token file '{path}'" + ), + None if assume_via_pod => log::warn!( + "AWS credential vendor (role {role_arn}): aws_assume_via_pod_web_identity=true \ + but no token file resolved (aws_pod_web_identity_token_file unset and \ + AWS_WEB_IDENTITY_TOKEN_FILE not in env); falling back to chained AssumeRole" + ), + None => log::info!( + "AWS credential vendor (role {role_arn}): chained AssumeRole \ + (pod web-identity not enabled)" + ), + } + if let Some(path) = pod_token_file { + config = config.with_pod_web_identity_token_file(path); + } + let vendor = AwsCredentialVendor::new(config).await?; Ok(Some(Box::new(vendor))) } diff --git a/rust/lance-namespace-impls/src/credentials/aws.rs b/rust/lance-namespace-impls/src/credentials/aws.rs index 7dedaa6e108..56dc9a54c2c 100644 --- a/rust/lance-namespace-impls/src/credentials/aws.rs +++ b/rust/lance-namespace-impls/src/credentials/aws.rs @@ -24,6 +24,18 @@ use super::{ redact_credential, }; +/// Render an error together with its full `source()` chain. +fn full_error_chain(err: &dyn std::error::Error) -> String { + let mut out = err.to_string(); + let mut source = err.source(); + while let Some(cause) = source { + out.push_str(": "); + out.push_str(&cause.to_string()); + source = cause.source(); + } + out +} + /// Configuration for AWS credential vending. #[derive(Debug, Clone)] pub struct AwsCredentialVendorConfig { @@ -60,6 +72,17 @@ pub struct AwsCredentialVendorConfig { /// When an API key is provided, its hash is looked up in this map. /// If found, the mapped permission is used instead of the default permission. pub api_key_hash_permissions: HashMap, + + /// Optional path to the pod's projected service-account OIDC token file (typically the + /// EKS-injected `AWS_WEB_IDENTITY_TOKEN_FILE`). + /// This is the recommended method when running in Kubernetes. + /// When set, the scoped assume is performed with `AssumeRoleWithWebIdentity` + /// using this token instead of a role-chained `AssumeRole` from the process's + /// ambient credentials. A web-identity assumption is *not* role chaining, so + /// STS honors the role's `MaxSessionDuration` (up to 12h) and the returned + /// expiration is accurate. When `None`, the `AssumeRole` path + /// is used and `duration_millis` is subject to the source role duration and may be invalid + pub pod_web_identity_token_file: Option, } impl AwsCredentialVendorConfig { @@ -74,6 +97,7 @@ impl AwsCredentialVendorConfig { permission: VendedPermission::default(), api_key_salt: None, api_key_hash_permissions: HashMap::new(), + pod_web_identity_token_file: None, } } @@ -101,6 +125,14 @@ impl AwsCredentialVendorConfig { self } + /// Set the pod web-identity token file, enabling direct (non-chained) + /// `AssumeRoleWithWebIdentity` for the scoped assume. See + /// [`AwsCredentialVendorConfig::pod_web_identity_token_file`]. + pub fn with_pod_web_identity_token_file(mut self, path: impl Into) -> Self { + self.pod_web_identity_token_file = Some(path.into()); + self + } + /// Set the permission level for vended credentials. pub fn with_permission(mut self, permission: VendedPermission) -> Self { self.permission = permission; @@ -407,7 +439,8 @@ impl AwsCredentialVendor { lance_core::Error::from(NamespaceError::Internal { message: format!( "AssumeRoleWithWebIdentity failed for role '{}': {}", - self.config.role_arn, e + self.config.role_arn, + full_error_chain(&e) ), }) })?; @@ -421,6 +454,95 @@ impl AwsCredentialVendor { } /// Vend credentials using AssumeRole with API key validation. + /// Perform the scoped assume for `(bucket, prefix, permission)`, attaching the + /// per-table session policy. + /// + /// When [`AwsCredentialVendorConfig::pod_web_identity_token_file`] is set this + /// uses `AssumeRoleWithWebIdentity` with the pod's projected SA OIDC token -- a + /// *direct*, non-chained assumption that honors the role's `MaxSessionDuration` + /// (so `expires_at_millis` is accurate). Otherwise it falls back to the legacy + /// role-chained `AssumeRole` from ambient credentials (STS-capped at 1h). + /// + /// `external_id` is only applied on the chained `AssumeRole` path; + /// `AssumeRoleWithWebIdentity` has no external-id parameter (the OIDC + /// `sub`/`aud` trust condition is the binding instead). + async fn assume_scoped( + &self, + bucket: &str, + prefix: &str, + permission: VendedPermission, + session_name: &str, + external_id: Option<&str>, + ) -> Result { + let policy = Self::build_policy(bucket, prefix, permission); + let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32; + + let credentials = if let Some(token_file) = &self.config.pod_web_identity_token_file { + // DIRECT (non-chained): AssumeRoleWithWebIdentity with the pod's SA + // OIDC token. Re-read the file every vend -- kubelet rotates it. + let token = tokio::fs::read_to_string(token_file).await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "failed to read pod web identity token '{}': {}", + token_file, e + ), + }) + })?; + debug!( + "AWS AssumeRoleWithWebIdentity (pod): role={}, session={}, permission={}", + self.config.role_arn, session_name, permission + ); + let response = self + .sts_client + .assume_role_with_web_identity() + .role_arn(&self.config.role_arn) + .web_identity_token(token.trim()) + .role_session_name(session_name) + .policy(&policy) + .duration_seconds(duration_secs) + .send() + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "AssumeRoleWithWebIdentity (pod) failed for role '{}': {}", + self.config.role_arn, + full_error_chain(&e) + ), + }) + })?; + response.credentials().cloned() + } else { + // LEGACY chained path: AssumeRole from ambient credentials (1h cap). + debug!( + "AWS AssumeRole (chained): role={}, session={}, permission={}", + self.config.role_arn, session_name, permission + ); + let mut request = self + .sts_client + .assume_role() + .role_arn(&self.config.role_arn) + .role_session_name(session_name) + .policy(&policy) + .duration_seconds(duration_secs); + if let Some(external_id) = external_id { + request = request.external_id(external_id); + } + let response = request.send().await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "AssumeRole failed for role '{}': {}", + self.config.role_arn, + full_error_chain(&e) + ), + }) + })?; + response.credentials().cloned() + }; + + self.extract_credentials(credentials.as_ref(), bucket, prefix, permission) + } + async fn vend_with_api_key( &self, bucket: &str, @@ -452,42 +574,19 @@ impl AwsCredentialVendor { }) })?; - let policy = Self::build_policy(bucket, prefix, permission); + // The api_key authorizes the client and picks the permission; the AWS + // assume itself goes through `assume_scoped` (pod web-identity when + // configured, else chained AssumeRole with the key hash as external_id). let session_name = Self::cap_session_name(&format!("lance-api-{}", &key_hash[..16])); - let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32; - - debug!( - "AWS AssumeRole with API key: role={}, session={}, permission={}", - self.config.role_arn, session_name, permission - ); - - let request = self - .sts_client - .assume_role() - .role_arn(&self.config.role_arn) - .role_session_name(&session_name) - .policy(&policy) - .duration_seconds(duration_secs) - .external_id(&key_hash); // Use hash as external_id - - let response = request.send().await.map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "AssumeRole with API key failed for role '{}': {}", - self.config.role_arn, e - ), - }) - })?; - - self.extract_credentials(response.credentials(), bucket, prefix, permission) + self.assume_scoped(bucket, prefix, permission, &session_name, Some(&key_hash)) + .await } - /// Vend credentials using AssumeRole with static configuration. + /// Vend credentials using the vendor's static (default) permission. async fn vend_with_static_config( &self, bucket: &str, prefix: &str, - policy: &str, ) -> Result { let role_session_name = self .config @@ -496,40 +595,14 @@ impl AwsCredentialVendor { .unwrap_or_else(|| "lance-credential-vending".to_string()); let role_session_name = Self::cap_session_name(&role_session_name); - let duration_secs = self.config.duration_millis.div_ceil(1000).clamp(900, 43200) as i32; - - debug!( - "AWS AssumeRole (static): role={}, session={}, permission={}", - self.config.role_arn, role_session_name, self.config.permission - ); - - let mut request = self - .sts_client - .assume_role() - .role_arn(&self.config.role_arn) - .role_session_name(&role_session_name) - .policy(policy) - .duration_seconds(duration_secs); - - if let Some(ref external_id) = self.config.external_id { - request = request.external_id(external_id); - } - - let response = request.send().await.map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "AssumeRole failed for role '{}': {}", - self.config.role_arn, e - ), - }) - })?; - - self.extract_credentials( - response.credentials(), + self.assume_scoped( bucket, prefix, self.config.permission, + &role_session_name, + self.config.external_id.as_deref(), ) + .await } } @@ -575,10 +648,8 @@ impl CredentialVendor for AwsCredentialVendor { .into()) } None => { - // Use AssumeRole with static configuration - let policy = Self::build_policy(&bucket, &prefix, self.config.permission); - self.vend_with_static_config(&bucket, &prefix, &policy) - .await + // Use the vendor's static (default) permission + self.vend_with_static_config(&bucket, &prefix).await } } } @@ -743,6 +814,41 @@ mod tests { assert_eq!(config.duration_millis, 7200000); assert_eq!(config.role_session_name, Some("my-session".to_string())); assert_eq!(config.region, Some("us-west-2".to_string())); + // Defaults to the legacy chained AssumeRole path. + assert_eq!(config.pod_web_identity_token_file, None); + + let pod_config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole") + .with_pod_web_identity_token_file("/var/run/secrets/.../token"); + assert_eq!( + pod_config.pod_web_identity_token_file, + Some("/var/run/secrets/.../token".to_string()) + ); + } + + #[tokio::test] + async fn test_pod_web_identity_path_reads_token_file() { + // When pod_web_identity_token_file is set, the scoped assume takes the + // AssumeRoleWithWebIdentity branch and reads the token file first. Point + // it at a missing file so we deterministically hit the read error without + // needing a live STS -- this proves the branch selection + file read. + let sdk_config = aws_config::SdkConfig::builder() + .behavior_version(aws_config::BehaviorVersion::latest()) + .region(aws_config::Region::new("us-east-2")) + .build(); + let sts_client = StsClient::new(&sdk_config); + let config = AwsCredentialVendorConfig::new("arn:aws:iam::123456789012:role/MyRole") + .with_pod_web_identity_token_file("/nonexistent/pod/web-identity-token"); + let vendor = AwsCredentialVendor::with_sts_client(config, sts_client); + + let err = vendor + .vend_credentials("s3://bucket/prefix", None) + .await + .expect_err("missing token file must fail before any STS call"); + assert!( + err.to_string() + .contains("failed to read pod web identity token"), + "unexpected error: {err}" + ); } // ============================================================================ diff --git a/rust/lance-namespace-impls/src/dir.rs b/rust/lance-namespace-impls/src/dir.rs index dc2d83cf278..80c800d05e0 100644 --- a/rust/lance-namespace-impls/src/dir.rs +++ b/rust/lance-namespace-impls/src/dir.rs @@ -30,21 +30,29 @@ use lance_index::scalar::{ BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams, }; use lance_index::vector::{ - bq::RQBuildParams, hnsw::builder::HnswBuildParams, ivf::IvfBuildParams, pq::PQBuildParams, + bq::{RABIT_MAX_NUM_BITS, RABIT_MIN_NUM_BITS, RQBuildParams, validate_supported_rq_num_bits}, + hnsw::builder::HnswBuildParams, + ivf::IvfBuildParams, + pq::PQBuildParams, sq::builder::SQBuildParams, }; use lance_index::{IndexType, is_system_index}; +use lance_io::object_store::throttle::is_throttle_error; use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_linalg::distance::MetricType; use lance_table::io::commit::{ManifestNamingScheme, VERSIONS_DIR}; use object_store::ObjectStoreExt; use object_store::path::Path; -use object_store::{Error as ObjectStoreError, ObjectStore as OSObjectStore, PutMode, PutOptions}; +use object_store::{ + Error as ObjectStoreError, ObjectMeta, ObjectStore as OSObjectStore, PutMode, PutOptions, +}; use std::collections::HashMap; use std::io::Cursor; use std::sync::{Arc, Mutex}; +use tokio::sync::OnceCell; use crate::context::DynamicContextProvider; +use crate::merge_insert_on_columns; use lance_namespace::models::{ AlterTableAddColumnsRequest, AlterTableAddColumnsResponse, AlterTableAlterColumnsRequest, AlterTableAlterColumnsResponse, AlterTableDropColumnsRequest, AlterTableDropColumnsResponse, @@ -76,7 +84,11 @@ use lance_namespace::models::{ UpdateTableSchemaMetadataResponse, UpdateTableTagRequest, UpdateTableTagResponse, }; +use lance_core::utils::parse::str_to_bool; use lance_core::{Error, Result, box_error}; +use lance_index::scalar::inverted::query::{ + BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Occur, Operator, PhraseQuery, +}; use lance_namespace::LanceNamespace; use lance_namespace::error::NamespaceError; use lance_namespace::schema::arrow_schema_to_json; @@ -491,31 +503,31 @@ impl DirectoryNamespaceBuilder { // Extract manifest_enabled (default: true) let manifest_enabled = properties .get("manifest_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(true); // Extract dir_listing_enabled (default: true) let dir_listing_enabled = properties .get("dir_listing_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(true); // Extract inline_optimization_enabled (default: true) let inline_optimization_enabled = properties .get("inline_optimization_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(true); // Extract table_version_tracking_enabled (default: false) let table_version_tracking_enabled = properties .get("table_version_tracking_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(false); // Extract dir_listing_to_manifest_migration_enabled (default: false) let dir_listing_to_manifest_migration_enabled = properties .get("dir_listing_to_manifest_migration_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(false); // Extract credential vendor properties (properties prefixed with "credential_vendor.") @@ -536,7 +548,7 @@ impl DirectoryNamespaceBuilder { // Extract vend_input_storage_options (default: false) let vend_input_storage_options = properties .get("vend_input_storage_options") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(false); // Extract vend_input_storage_options_refresh_interval_millis (optional) @@ -547,7 +559,7 @@ impl DirectoryNamespaceBuilder { // Extract ops_metrics_enabled (default: false) let ops_metrics_enabled = properties .get("ops_metrics_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(false); Ok(Self { @@ -738,7 +750,7 @@ impl DirectoryNamespaceBuilder { Self::initialize_object_store(&self.root, &self.storage_options, &self.session).await?; let manifest_ns = if self.manifest_enabled { - match manifest::ManifestNamespace::from_directory( + match manifest::ManifestNamespace::open_from_directory( self.root.clone(), self.storage_options.clone(), self.session.clone(), @@ -757,18 +769,19 @@ impl DirectoryNamespaceBuilder { // degrading to a directory-listing view that ignores it. return Err(e); } - Err(e) => { - // Failed to initialize manifest namespace, fall back to directory listing only - log::warn!( - "Failed to initialize manifest namespace, falling back to directory listing only: {}", - e - ); + Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => { + log::debug!("Manifest namespace does not exist yet: {}", e); None } + Err(e) => return Err(e), } } else { None }; + let manifest_cell = OnceCell::new(); + if let Some(manifest_ns) = manifest_ns { + let _ = manifest_cell.set(manifest_ns); + } // Create credential vendor once during initialization if enabled let credential_vendor = if has_credential_vendor_config(&self.credential_vendor_properties) @@ -792,8 +805,12 @@ impl DirectoryNamespaceBuilder { session: self.session, object_store, base_path, - manifest_ns, + manifest_ns: manifest_cell, + write_manifest_ns: OnceCell::new(), + manifest_enabled: self.manifest_enabled, dir_listing_enabled: self.dir_listing_enabled, + inline_optimization_enabled: self.inline_optimization_enabled, + commit_retries: self.commit_retries, dir_listing_to_manifest_migration_enabled: self .dir_listing_to_manifest_migration_enabled, table_version_tracking_enabled: self.table_version_tracking_enabled, @@ -870,8 +887,12 @@ pub struct DirectoryNamespace { session: Option>, object_store: Arc, base_path: Path, - manifest_ns: Option>, + manifest_ns: OnceCell>, + write_manifest_ns: OnceCell>, + manifest_enabled: bool, dir_listing_enabled: bool, + inline_optimization_enabled: bool, + commit_retries: Option, /// When true, root-level table operations check the manifest first before /// falling back to directory listing. When false, root-level tables skip /// the manifest check and use directory listing directly. @@ -907,6 +928,16 @@ impl std::fmt::Display for DirectoryNamespace { } } +/// Inputs for resolving an already-published `create_table_version` target. +struct ExistingTableVersionResolve<'a> { + staging_path: &'a Path, + final_path: &'a Path, + version: u64, + table_uri: &'a str, + final_meta: &'a ObjectMeta, + request_manifest_size: Option, +} + /// Describes the version ranges to delete for a single table. /// Used by `batch_delete_table_versions` and `delete_physical_version_files`. struct TableDeleteEntry { @@ -983,6 +1014,98 @@ impl TransactionAlteration { } impl DirectoryNamespace { + fn manifest_ns_for_read(&self) -> Option<&Arc> { + self.write_manifest_ns + .get() + .or_else(|| self.manifest_ns.get()) + } + + async fn manifest_ns_for_write(&self) -> Result>> { + if !self.manifest_enabled { + return Ok(None); + } + + let manifest_ns = self + .write_manifest_ns + .get_or_try_init(|| async { + manifest::ManifestNamespace::from_directory( + self.root.clone(), + self.storage_options.clone(), + self.session.clone(), + self.object_store.clone(), + self.base_path.clone(), + self.dir_listing_enabled, + self.inline_optimization_enabled, + self.commit_retries, + ) + .await + .map(Arc::new) + }) + .await?; + Ok(Some(manifest_ns.clone())) + } + + /// Lazily open the `__manifest` dataset (read-only) into the read cell. + /// + /// `manifest_ns` is populated at construction only if `__manifest` already + /// existed then. A manifest created afterwards -- e.g. by another + /// connection's write, since Phalanx caches a connection per db and the + /// first op on a fresh db is usually a read -- would otherwise stay + /// invisible to this connection's reads forever, so describe/list/exists + /// report "not found" for a table that is in fact registered. Re-open on + /// demand so reads self-heal once the manifest exists. Idempotent and cheap + /// once the cell is populated; unlike `manifest_ns_for_write` it never + /// creates the manifest. + async fn ensure_read_manifest(&self) -> Result<()> { + if !self.manifest_enabled + || self.manifest_ns.get().is_some() + || self.write_manifest_ns.get().is_some() + { + return Ok(()); + } + match self + .manifest_ns + .get_or_try_init(|| async { + manifest::ManifestNamespace::open_from_directory( + self.root.clone(), + self.storage_options.clone(), + self.session.clone(), + self.object_store.clone(), + self.base_path.clone(), + self.dir_listing_enabled, + self.inline_optimization_enabled, + self.commit_retries, + ) + .await + .map(Arc::new) + }) + .await + { + Ok(_) => Ok(()), + // Manifest still doesn't exist: leave the cell empty so a later read + // retries once it does. A genuinely absent table is still reported + // not-found by the callers' existing `manifest_ns_for_read() == None` + // branch, exactly as before. + Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => Ok(()), + Err(e) => Err(e), + } + } + + fn child_namespace_requires_manifest_error(&self) -> Error { + if self.manifest_enabled { + NamespaceError::NamespaceNotFound { + message: "Child namespace reads require an existing __manifest dataset".to_string(), + } + .into() + } else { + NamespaceError::Unsupported { + message: "Child namespaces are only supported when manifest mode is enabled" + .to_string(), + } + .into() + } + } + /// Apply pagination to a list of table names /// /// Sorts the list alphabetically and applies pagination using page_token (start_after) and limit. @@ -1056,7 +1179,7 @@ impl DirectoryNamespace { let table_name = &path[..path.len() - 6]; // Use atomic check to skip deregistered tables. - let status = self.check_table_status(table_name).await; + let status = self.check_table_status(table_name).await?; if status.is_deregistered { continue; } @@ -1407,17 +1530,19 @@ impl DirectoryNamespace { .load() .await .map_err(|e| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!( - "branch '{}' not found for table at '{}': {}", - branch, table_uri, e - ), - }) + let message = format!( + "branch '{}' not found for table at '{}': {}", + branch, table_uri, e + ); + Self::map_open_error(e, NamespaceError::TableNotFound { message }) })?; - dataset.branches().get(branch).await.map_err(|_| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!("branch '{}' not found for table at '{}'", branch, table_uri), - }) + dataset.branches().get(branch).await.map_err(|e| { + Self::map_open_error( + e, + NamespaceError::TableNotFound { + message: format!("branch '{}' not found for table at '{}'", branch, table_uri), + }, + ) })?; Ok(dataset) } @@ -1430,29 +1555,35 @@ impl DirectoryNamespace { .uri) } - /// Resolves a branch to its `(uri, object-store path)` for `create_table_version`. + /// Resolves a branch to its `(uri, object-store path, parent_version)` for + /// `create_table_version`. /// /// `BranchContents` is the source of truth, so check the ref first: a - /// registered branch commits directly. With no ref, accept the commit only on - /// an empty chain (the `create_branch` bootstrap, whose first commit precedes - /// its ref); reject a chain that already holds committed versions as a zombie. + /// registered branch commits directly and returns its `parent_version` for + /// empty-chain CAS. With no ref, accept the commit only on an empty chain + /// (the `create_branch` bootstrap, whose first commit precedes its ref) and + /// return `parent_version = None`; reject a chain that already holds + /// committed versions as a zombie. async fn resolve_branch_for_commit( &self, table_uri: &str, branch: &str, - ) -> Result<(String, Path)> { + ) -> Result<(String, Path, Option)> { let main = self .configured_builder(table_uri) .load() .await .map_err(|e| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!("table at '{}' not found: {}", table_uri, e), - }) + let message = format!("table at '{}' not found: {}", table_uri, e); + Self::map_open_error(e, NamespaceError::TableNotFound { message }) })?; let branch_location = main.branch_location().find_branch(Some(branch))?; match main.branches().get(branch).await { - Ok(_) => Ok((branch_location.uri, branch_location.path)), + Ok(contents) => Ok(( + branch_location.uri, + branch_location.path, + Some(contents.parent_version), + )), Err(lance_core::Error::RefNotFound { .. }) => { if self .branch_has_committed_versions(&branch_location.path) @@ -1466,7 +1597,7 @@ impl DirectoryNamespace { } .into()); } - Ok((branch_location.uri, branch_location.path)) + Ok((branch_location.uri, branch_location.path, None)) } Err(e) => Err(e), } @@ -1543,6 +1674,236 @@ impl DirectoryNamespace { ManifestNamingScheme::detect_scheme(filename)?.parse_version(filename) } + /// Build a successful `CreateTableVersionResponse` from an existing final manifest. + fn create_table_version_response( + version: u64, + final_path: &Path, + final_meta: &ObjectMeta, + ) -> CreateTableVersionResponse { + CreateTableVersionResponse { + transaction_id: None, + version: Some(Box::new(TableVersion { + version: version as i64, + manifest_path: final_path.to_string(), + manifest_size: Some(final_meta.size as i64), + e_tag: final_meta.e_tag.clone(), + timestamp_millis: None, + metadata: None, + })), + ..Default::default() + } + } + + /// Whether the staging blob matches the already-published version blob. + /// + /// Used for idempotent retries of `create_table_version`. Object-store + /// `e_tag` is opaque metadata (not a validated content hash) and may also + /// change across Create/rename materialize, so it is never used for + /// identity. Size mismatch is a cheap negative check; byte equality is the + /// durable success condition. + async fn staging_matches_final_manifest( + &self, + staging_path: &Path, + final_path: &Path, + final_meta: &ObjectMeta, + request_manifest_size: Option, + ) -> Result { + if let Some(size) = request_manifest_size + && size != final_meta.size as i64 + { + return Ok(false); + } + + let staging_bytes = match self.object_store.inner.get(staging_path).await { + Ok(r) => r.bytes().await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read staging manifest at '{}': {}", + staging_path, e + ), + }) + })?, + Err(ObjectStoreError::NotFound { .. }) => return Ok(false), + Err(e) => { + return Err(lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read staging manifest at '{}': {}", + staging_path, e + ), + })); + } + }; + + let final_bytes = self + .object_store + .inner + .get(final_path) + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read existing version manifest at '{}': {}", + final_path, e + ), + }) + })? + .bytes() + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to read existing version manifest bytes at '{}': {}", + final_path, e + ), + }) + })?; + + Ok(staging_bytes.as_ref() == final_bytes.as_ref()) + } + + /// Idempotent success or conflict when the target version path already exists. + async fn resolve_existing_table_version( + &self, + args: ExistingTableVersionResolve<'_>, + ) -> Result { + if self + .staging_matches_final_manifest( + args.staging_path, + args.final_path, + args.final_meta, + args.request_manifest_size, + ) + .await? + { + // Best-effort cleanup of a retry's staging blob. + if let Err(e) = self.object_store.inner.delete(args.staging_path).await { + log::warn!( + "Failed to delete staging manifest at '{}': {:?}", + args.staging_path, + e + ); + } + return Ok(Self::create_table_version_response( + args.version, + args.final_path, + args.final_meta, + )); + } + + Err(lance_core::Error::from( + NamespaceError::ConcurrentModification { + message: format!( + "Version {} already exists for table at '{}' with different content", + args.version, args.table_uri + ), + }, + )) + } + + /// Enforce version CAS: requested version must be `latest + 1` (or bootstrap). + /// + /// Empty-chain bootstrap: + /// - main must start at v1 + /// - a registered branch must start at `BranchContents.parent_version` (the + /// shallow-clone fork version, which may be > 1) + /// - an unregistered branch (create_branch phase-1, ref not written yet) + /// accepts the requested version because `parent_version` is not known yet + async fn enforce_create_table_version_cas( + &self, + table_path: &Path, + version: u64, + table_uri: &str, + is_branch: bool, + branch_parent_version: Option, + ) -> Result<()> { + let latest = self.list_versions_under(table_path, true, Some(1)).await?; + let expected = match latest.first() { + Some(v) => (v.version as u64).checked_add(1).ok_or_else(|| { + lance_core::Error::from(NamespaceError::ConcurrentModification { + message: format!( + "Version overflow computing next version for table at '{}': \ + latest version {} cannot advance", + table_uri, v.version + ), + }) + })?, + None => { + if is_branch { + // Prefer BranchContents.parent_version when the ref exists so a + // branch forked at v5 cannot bootstrap at an arbitrary version. + match branch_parent_version { + Some(parent_version) => parent_version, + None => version, + } + } else { + 1 + } + } + }; + if version != expected { + let latest_display = latest + .first() + .map(|v| v.version.to_string()) + .unwrap_or_else(|| "none".to_string()); + return Err(lance_core::Error::from( + NamespaceError::ConcurrentModification { + message: format!( + "Version CAS failed for table at '{}': requested {}, expected {} (latest {})", + table_uri, version, expected, latest_display + ), + }, + )); + } + Ok(()) + } + + /// Materialize staging → final with Create semantics only (never overwrite). + async fn materialize_version_manifest_create( + &self, + staging_path: &Path, + final_path: &Path, + staging_manifest_path: &str, + ) -> std::result::Result<(), ObjectStoreError> { + match self + .object_store + .inner + .copy_if_not_exists(staging_path, final_path) + .await + { + Ok(()) => Ok(()), + Err(ObjectStoreError::NotImplemented { .. }) + | Err(ObjectStoreError::NotSupported { .. }) => { + let manifest_data = self + .object_store + .inner + .get(staging_path) + .await? + .bytes() + .await + .map_err(|e| ObjectStoreError::Generic { + store: "DirectoryNamespace", + source: Box::new(std::io::Error::other(format!( + "Failed to read staging manifest bytes at '{}': {}", + staging_manifest_path, e + ))), + })?; + self.object_store + .inner + .put_opts( + final_path, + manifest_data.into(), + PutOptions { + mode: PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| ()) + } + Err(e) => Err(e), + } + } + async fn list_table_versions_from_storage( &self, table_uri: &str, @@ -1564,63 +1925,92 @@ impl DirectoryNamespace { limit: Option, ) -> Result> { let versions_dir = table_path.clone().join(VERSIONS_DIR); - let manifest_metas: Vec<_> = self - .object_store - .read_dir_all(&versions_dir, None) - .try_collect() - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to list manifest files under '{}': {}", - versions_dir, e - ), - }) - })?; - - let is_v2_naming = manifest_metas - .first() - .is_some_and(|meta| meta.location.filename().is_some_and(|f| f.len() == 29)); - - let mut table_versions: Vec = manifest_metas - .into_iter() - .filter_map(|meta| { - let filename = meta.location.filename()?; - let actual_version = Self::manifest_version_from_filename(filename)?; - - Some(TableVersion { - version: actual_version as i64, - manifest_path: meta.location.to_string(), - manifest_size: Some(meta.size as i64), - e_tag: meta.e_tag, - timestamp_millis: Some(meta.last_modified.timestamp_millis()), - metadata: None, - }) + let mut stream = self.object_store.read_dir_all(&versions_dir, None); + let list_err = |e: lance_core::Error| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to list manifest files under '{}': {}", + versions_dir, e + ), }) - .collect(); + }; - let list_is_ordered = self.object_store.list_is_lexically_ordered; + let limit = limit + .filter(|limit| *limit >= 0) + .map(|limit| limit as usize); - let needs_sort = if list_is_ordered { - if is_v2_naming { - !descending - } else { - descending - } - } else { + let mut table_versions: Vec = Vec::new(); + let push_meta = |meta: ObjectMeta, out: &mut Vec| -> bool { + let Some(filename) = meta.location.filename() else { + return false; + }; + let Some(actual_version) = Self::manifest_version_from_filename(filename) else { + return false; + }; + out.push(TableVersion { + version: actual_version as i64, + manifest_path: meta.location.to_string(), + manifest_size: Some(meta.size as i64), + e_tag: meta.e_tag, + timestamp_millis: Some(meta.last_modified.timestamp_millis()), + metadata: None, + }); true }; - if needs_sort { - if descending { - table_versions.sort_by(|a, b| b.version.cmp(&a.version)); + // Detect the naming scheme from the first committed manifest, not the + // first raw entry: retained staging blobs (`{manifest}-`) sort + // ahead of it and would misclassify the stream as non-V2. V2 filenames + // are a fixed 29 chars (`{u64::MAX - version:020}.manifest`). + let mut first_manifest_filename_len = None; + while first_manifest_filename_len.is_none() { + match stream.try_next().await.map_err(list_err)? { + Some(meta) => { + let filename_len = meta.location.filename().map(|f| f.len()); + if push_meta(meta, &mut table_versions) { + first_manifest_filename_len = filename_len; + } + } + None => break, + } + } + let is_v2_naming = first_manifest_filename_len == Some(29); + + // V2 filenames invert the version, so a lexically-ordered stream + // arrives newest-first; when that matches the requested order, stop + // after `limit` manifests instead of paginating the whole directory + // (the `get_latest_version` hot path: descending, limit 1). + let list_is_ordered = self.object_store.list_is_lexically_ordered; + let stream_matches_request = list_is_ordered + && if is_v2_naming { + descending } else { - table_versions.sort_by(|a, b| a.version.cmp(&b.version)); + !descending + }; + let early_stop_at = limit.filter(|_| stream_matches_request); + + while early_stop_at.is_none_or(|n| table_versions.len() < n) { + match stream.try_next().await.map_err(list_err)? { + Some(meta) => { + push_meta(meta, &mut table_versions); + } + None => break, } } - if let Some(limit) = limit { - table_versions.truncate(limit as usize); + // Scheme detection pushes the first manifest regardless of the limit, + // so re-enforce the limit on both paths (covers limit=0). + if let Some(n) = early_stop_at { + table_versions.truncate(n); + } else { + if descending { + table_versions.sort_by_key(|v| std::cmp::Reverse(v.version)); + } else { + table_versions.sort_by_key(|v| v.version); + } + if let Some(limit) = limit { + table_versions.truncate(limit); + } } Ok(table_versions) @@ -1634,10 +2024,24 @@ impl DirectoryNamespace { request: DescribeTableRequest, ) -> Result { let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1); + let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1); let skip_manifest_for_root = self.dir_listing_enabled && is_root_level && !self.dir_listing_to_manifest_migration_enabled; - if let Some(ref manifest_ns) = self.manifest_ns + // Self-heal the manifest wherever it can be authoritative: a child table + // (no dir-listing fallback), a manifest-only namespace, or migration mode + // (manifest-first -- it can hold registered_table -> external .lance + // aliases that dir-listing cannot resolve, so a reader built before + // __manifest must re-probe to see them). The bypass -- skipping the probe + // -- applies ONLY to migration-disabled directory-backed root reads, + // which are served entirely from the directory listing. + if is_child_table + || !self.dir_listing_enabled + || self.dir_listing_to_manifest_migration_enabled + { + self.ensure_read_manifest().await?; + } + if let Some(manifest_ns) = self.manifest_ns_for_read() && !skip_manifest_for_root { match manifest_ns.describe_table(request.clone()).await { @@ -1661,19 +2065,31 @@ impl DirectoryNamespace { // rather than degrading to a directory-listing view. return Err(e); } - Err(_) if self.dir_listing_enabled && is_root_level => { - // Fall through to directory check only for single-level IDs + Err(e) if self.dir_listing_enabled && is_root_level => { + // Only a genuinely-absent table (e.g. an unmigrated on-disk + // table) may fall through to the directory check; any other + // manifest error must propagate rather than be read as missing. + if !Self::is_manifest_table_absent_error(&e) { + return Err(Self::classify_storage_error(e)); + } } Err(e) => return Err(e), } } + if is_child_table { + return Err(self.child_namespace_requires_manifest_error()); + } let table_name = Self::table_name_from_id(&request.id)?; let table_id = Self::format_table_id_from_request(&request.id); + if !self.dir_listing_enabled { + return Err(NamespaceError::TableNotFound { message: table_id }.into()); + } + let table_uri = self.table_full_uri(&table_name); // Atomically check table existence and deregistration status - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { @@ -1774,12 +2190,14 @@ impl DirectoryNamespace { .checkout_version(requested_version as u64) .await .map_err(|e| { - lance_core::Error::from(NamespaceError::TableVersionNotFound { - message: format!( - "Version {} not found for table '{}': {}", - requested_version, table_name, e - ), - }) + let message = format!( + "Version {} not found for table '{}': {}", + requested_version, table_name, e + ); + Self::map_open_error( + e, + NamespaceError::TableVersionNotFound { message }, + ) })?; } @@ -1893,22 +2311,20 @@ impl DirectoryNamespace { let builder = self.configured_builder(table_uri); let dataset = builder.load().await.map_err(|e| { - lance_core::Error::from(NamespaceError::TableNotFound { - message: format!( - "Failed to open table at '{}' for {}: {}", - table_uri, operation, e - ), - }) + let message = format!( + "Failed to open table at '{}' for {}: {}", + table_uri, operation, e + ); + Self::map_open_error(e, NamespaceError::TableNotFound { message }) })?; if let Some(version) = version { return dataset.checkout_version(version as u64).await.map_err(|e| { - lance_core::Error::from(NamespaceError::TableVersionNotFound { - message: format!( - "Failed to checkout version {} for table at '{}' during {}: {}", - version, table_uri, operation, e - ), - }) + let message = format!( + "Failed to checkout version {} for table at '{}' during {}: {}", + version, table_uri, operation, e + ); + Self::map_open_error(e, NamespaceError::TableVersionNotFound { message }) }); } @@ -2042,14 +2458,30 @@ impl DirectoryNamespace { SQBuildParams::default(), ), }, - IndexType::IvfRq => DirectoryIndexParams::Vector { - index_type, - params: VectorIndexParams::with_ivf_rq_params( - Self::parse_metric_type(request.distance_type.as_deref())?, - IvfBuildParams::default(), - RQBuildParams::default(), - ), - }, + IndexType::IvfRq => { + let rq_params = if let Some(requested_num_bits) = request.num_bits { + let invalid_num_bits = || NamespaceError::InvalidInput { + message: format!( + "IVF_RQ num_bits must be in {}..={}, got {}", + RABIT_MIN_NUM_BITS, RABIT_MAX_NUM_BITS, requested_num_bits + ), + }; + let num_bits = + u8::try_from(requested_num_bits).map_err(|_| invalid_num_bits())?; + validate_supported_rq_num_bits(num_bits).map_err(|_| invalid_num_bits())?; + RQBuildParams::new(num_bits) + } else { + RQBuildParams::default() + }; + DirectoryIndexParams::Vector { + index_type, + params: VectorIndexParams::with_ivf_rq_params( + Self::parse_metric_type(request.distance_type.as_deref())?, + IvfBuildParams::default(), + rq_params, + ), + } + } IndexType::IvfHnswFlat => DirectoryIndexParams::Vector { index_type, params: VectorIndexParams::ivf_hnsw( @@ -2125,6 +2557,7 @@ impl DirectoryNamespace { Operation::CreateIndex { new_indices, removed_indices, + .. } if new_indices.is_empty() && !removed_indices.is_empty() => "DropIndex".to_string(), _ => transaction.operation.to_string(), } @@ -2173,6 +2606,7 @@ impl DirectoryNamespace { DescribeTransactionResponse { status: effective_status, properties: Some(properties), + ..Default::default() } } @@ -2199,6 +2633,7 @@ impl DirectoryNamespace { num_indexed_rows: get_i64("num_indexed_rows"), num_unindexed_rows: get_i64("num_unindexed_rows"), num_indices: get_i64("num_indices").and_then(|value| i32::try_from(value).ok()), + ..Default::default() } } @@ -2339,7 +2774,7 @@ impl DirectoryNamespace { } fn table_full_uri(&self, table_name: &str) -> String { - format!("{}/{}.lance", &self.root, table_name) + format!("{}/{}.lance", self.root, table_name) } /// Get the object store path for a table (relative to base_path) @@ -2370,50 +2805,105 @@ impl DirectoryNamespace { /// This performs a single directory listing to get a consistent snapshot of the /// table's state, avoiding race conditions between checking existence and /// checking deregistration status. - pub(crate) async fn check_table_status(&self, table_name: &str) -> TableStatus { + pub(crate) async fn check_table_status(&self, table_name: &str) -> Result { let table_path = self.table_path(table_name); match self.object_store.read_dir(table_path).await { Ok(entries) => { let exists = !entries.is_empty(); let is_deregistered = entries.iter().any(|e| e.ends_with(".lance-deregistered")); let has_reserved_file = entries.iter().any(|e| e.ends_with(".lance-reserved")); - TableStatus { + Ok(TableStatus { exists, is_deregistered, has_reserved_file, - } + }) } - Err(_) => TableStatus { + // Local filesystems error on a missing directory where object stores + // return an empty listing; both mean the table does not exist. + Err(e) if manifest::ManifestNamespace::is_not_found_load_error(&e) => Ok(TableStatus { exists: false, is_deregistered: false, has_reserved_file: false, - }, + }), + // Any other failure must propagate: collapsing it to "does not exist" + // lets a transient error overwrite a live table via create/exist-ok + // callers and destroys the retry evidence classifiers depend on. + Err(e) => Err(Self::classify_storage_error(e)), } } - async fn put_marker_file_atomic( - &self, - path: &Path, - file_description: &str, - ) -> std::result::Result<(), String> { - let put_opts = PutOptions { - mode: PutMode::Create, - ..Default::default() - }; - - match self - .object_store - .inner - .put_opts(path, bytes::Bytes::new().into(), put_opts) - .await + /// Classify a storage error into a typed [`NamespaceError`]. The full source + /// text is embedded in the message because the pyo3 layer flattens namespace + /// errors to message-only (no `__cause__`), so that is the only place the + /// 429/503 evidence survives to Python. + fn classify_storage_error(err: Error) -> Error { + if matches!(&err, Error::Namespace { .. }) { + return err; + } + let detail = err.to_string(); + if let Error::IO { source, .. } = &err + && let Some(os_err) = source.downcast_ref::() { - Ok(_) => Ok(()), - Err(ObjectStoreError::AlreadyExists { .. }) - | Err(ObjectStoreError::Precondition { .. }) => { - Err(format!("{} already exists", file_description)) + if is_throttle_error(os_err) { + return NamespaceError::Throttling { + message: format!( + "Storage request was throttled while resolving table: {detail}" + ), + } + .into(); + } + if Self::is_service_unavailable_error(os_err) { + return NamespaceError::ServiceUnavailable { + message: format!("Storage service unavailable while resolving table: {detail}"), + } + .into(); } - Err(e) => Err(format!("Failed to create {}: {:?}", file_description, e)), } + NamespaceError::Internal { + message: format!("Storage error while resolving table: {detail}"), + } + .into() + } + + /// Detect a clearly-transient 5xx not already caught by [`is_throttle_error`]. + /// `object_store` does not expose HTTP status codes, so match the (deliberately + /// narrow) canonical status phrases in the message. + fn is_service_unavailable_error(err: &ObjectStoreError) -> bool { + if let ObjectStoreError::Generic { source, .. } = err { + let message = source.to_string().to_ascii_lowercase(); + message.contains("503 service unavailable") + || message.contains("502 bad gateway") + || message.contains("504 gateway timeout") + } else { + false + } + } + + /// Whether a manifest error means the table is genuinely absent (rather than a + /// storage failure while consulting the manifest). Only such errors may fall + /// through to the directory listing; anything else must propagate. + fn is_manifest_table_absent_error(err: &Error) -> bool { + if manifest::ManifestNamespace::is_not_found_load_error(err) { + return true; + } + if let Error::Namespace { source, .. } = err + && let Some(ns_err) = source.downcast_ref::() + { + return matches!(ns_err, NamespaceError::TableNotFound { .. }); + } + false + } + + /// Map a dataset/version/branch open error: a transient IO error propagates + /// typed via [`classify_storage_error`], while a genuine not-found (missing + /// dataset, version, or ref) keeps the caller's `not_found` variant. + fn map_open_error(err: Error, not_found: NamespaceError) -> Error { + if matches!(&err, Error::IO { .. }) + && !manifest::ManifestNamespace::is_not_found_load_error(&err) + { + return Self::classify_storage_error(err); + } + not_found.into() } /// Get storage options for a table, using credential vending if configured. @@ -2525,7 +3015,7 @@ impl DirectoryNamespace { /// - Manifest registration fails pub async fn migrate(&self) -> Result { // We only care about tables in the root namespace - let Some(ref manifest_ns) = self.manifest_ns else { + let Some(manifest_ns) = self.manifest_ns_for_write().await? else { return Ok(0); // No manifest, nothing to migrate }; @@ -2797,10 +3287,14 @@ impl LanceNamespace for DirectoryNamespace { request: ListNamespacesRequest, ) -> Result { self.record_op("list_namespaces"); - if let Some(ref manifest_ns) = self.manifest_ns { + self.ensure_read_manifest().await?; + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.list_namespaces(request).await; } + if request.id.as_ref().is_some_and(|id| !id.is_empty()) { + return Err(self.child_namespace_requires_manifest_error()); + } Self::validate_root_namespace_id(&request.id)?; Ok(ListNamespacesResponse::new(vec![])) } @@ -2810,10 +3304,14 @@ impl LanceNamespace for DirectoryNamespace { request: DescribeNamespaceRequest, ) -> Result { self.record_op("describe_namespace"); - if let Some(ref manifest_ns) = self.manifest_ns { + self.ensure_read_manifest().await?; + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.describe_namespace(request).await; } + if request.id.as_ref().is_some_and(|id| !id.is_empty()) { + return Err(self.child_namespace_requires_manifest_error()); + } Self::validate_root_namespace_id(&request.id)?; #[allow(clippy::needless_update)] Ok(DescribeNamespaceResponse { @@ -2827,7 +3325,7 @@ impl LanceNamespace for DirectoryNamespace { request: CreateNamespaceRequest, ) -> Result { self.record_op("create_namespace"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.create_namespace(request).await; } @@ -2847,7 +3345,7 @@ impl LanceNamespace for DirectoryNamespace { async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result { self.record_op("drop_namespace"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.drop_namespace(request).await; } @@ -2867,7 +3365,8 @@ impl LanceNamespace for DirectoryNamespace { async fn namespace_exists(&self, request: NamespaceExistsRequest) -> Result<()> { self.record_op("namespace_exists"); - if let Some(ref manifest_ns) = self.manifest_ns { + self.ensure_read_manifest().await?; + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.namespace_exists(request).await; } @@ -2875,11 +3374,7 @@ impl LanceNamespace for DirectoryNamespace { return Ok(()); } - Err(NamespaceError::NamespaceNotFound { - message: "Child namespaces are only supported when manifest mode is enabled" - .to_string(), - } - .into()) + Err(self.child_namespace_requires_manifest_error()) } async fn list_tables(&self, request: ListTablesRequest) -> Result { @@ -2891,33 +3386,44 @@ impl LanceNamespace for DirectoryNamespace { }) })?; + // Self-heal the manifest wherever it can be authoritative: a child + // namespace, a manifest-only namespace, or migration mode (which merges + // manifest entries -- including registered aliases -- into the root + // listing). The bypass applies only to migration-disabled directory-backed + // root lists. + if !namespace_id.is_empty() + || !self.dir_listing_enabled + || self.dir_listing_to_manifest_migration_enabled + { + self.ensure_read_manifest().await?; + } + // For child namespaces, always delegate to manifest (if enabled) if !namespace_id.is_empty() { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_read() { return manifest_ns.list_tables(request).await; } - return Err(NamespaceError::Unsupported { - message: "Child namespaces are only supported when manifest mode is enabled" - .to_string(), - } - .into()); + return Err(self.child_namespace_requires_manifest_error()); } // When only manifest is enabled (no directory listing), delegate directly to manifest - if let Some(ref manifest_ns) = self.manifest_ns + if let Some(manifest_ns) = self.manifest_ns_for_read() && !self.dir_listing_enabled { return manifest_ns.list_tables(request).await; } + if !self.dir_listing_enabled { + return Ok(ListTablesResponse::new(vec![])); + } // When both manifest and directory listing are enabled with migration mode, // we need to merge and deduplicate - let mut tables = if self.manifest_ns.is_some() + let mut tables = if self.manifest_ns_for_read().is_some() && self.dir_listing_enabled && self.dir_listing_to_manifest_migration_enabled { // Get all manifest table locations (for deduplication) - let manifest_locations = if let Some(ref manifest_ns) = self.manifest_ns { + let manifest_locations = if let Some(manifest_ns) = self.manifest_ns_for_read() { manifest_ns.list_manifest_table_locations().await? } else { std::collections::HashSet::new() @@ -2927,7 +3433,7 @@ impl LanceNamespace for DirectoryNamespace { let mut manifest_request = request.clone(); manifest_request.limit = None; manifest_request.page_token = None; - let manifest_tables = if let Some(ref manifest_ns) = self.manifest_ns { + let manifest_tables = if let Some(manifest_ns) = self.manifest_ns_for_read() { let manifest_response = manifest_ns.list_tables(manifest_request).await?; manifest_response.tables } else { @@ -2975,10 +3481,19 @@ impl LanceNamespace for DirectoryNamespace { async fn table_exists(&self, request: TableExistsRequest) -> Result<()> { self.record_op("table_exists"); let is_root_level = request.id.as_ref().is_some_and(|id| id.len() == 1); + let is_child_table = request.id.as_ref().is_some_and(|id| id.len() > 1); let skip_manifest_for_root = self.dir_listing_enabled && is_root_level && !self.dir_listing_to_manifest_migration_enabled; - if let Some(ref manifest_ns) = self.manifest_ns + // Child table, manifest-only, or migration mode (see describe_table_impl). + // Only a migration-disabled directory-backed root read bypasses the probe. + if is_child_table + || !self.dir_listing_enabled + || self.dir_listing_to_manifest_migration_enabled + { + self.ensure_read_manifest().await?; + } + if let Some(manifest_ns) = self.manifest_ns_for_read() && !skip_manifest_for_root { match manifest_ns.table_exists(request.clone()).await { @@ -2988,18 +3503,29 @@ impl LanceNamespace for DirectoryNamespace { // rather than degrading to a directory-listing view. return Err(e); } - Err(_) if self.dir_listing_enabled && is_root_level => { - // Fall through to directory check only for single-level IDs + Err(e) if self.dir_listing_enabled && is_root_level => { + // Only a genuinely-absent table (e.g. an unmigrated on-disk + // table) may fall through to the directory check; any other + // manifest error must propagate rather than be read as missing. + if !Self::is_manifest_table_absent_error(&e) { + return Err(Self::classify_storage_error(e)); + } } Err(e) => return Err(e), } } + if is_child_table { + return Err(self.child_namespace_requires_manifest_error()); + } let table_name = Self::table_name_from_id(&request.id)?; let table_id = Self::format_table_id_from_request(&request.id); + if !self.dir_listing_enabled { + return Err(NamespaceError::TableNotFound { message: table_id }.into()); + } // Atomically check table existence and deregistration status - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { @@ -3020,7 +3546,7 @@ impl LanceNamespace for DirectoryNamespace { async fn drop_table(&self, request: DropTableRequest) -> Result { self.record_op("drop_table"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.drop_table(request).await; } @@ -3050,7 +3576,7 @@ impl LanceNamespace for DirectoryNamespace { request_data: Bytes, ) -> Result { self.record_op("create_table"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.create_table(request, request_data).await; } @@ -3058,7 +3584,7 @@ impl LanceNamespace for DirectoryNamespace { let table_name = Self::table_name_from_id(&request.id)?; let table_uri = self.table_full_uri(&table_name); - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; let (reader, _num_rows) = Self::ipc_reader_from_request_data(&request_data, "create_table")?; @@ -3097,7 +3623,7 @@ impl LanceNamespace for DirectoryNamespace { async fn declare_table(&self, request: DeclareTableRequest) -> Result { self.record_op("declare_table"); - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { let mut response = manifest_ns.declare_table(request.clone()).await?; if let Some(ref location) = response.location { // For backwards compatibility, only skip vending credentials when explicitly set to false @@ -3136,7 +3662,7 @@ impl LanceNamespace for DirectoryNamespace { // Check if table already has data (created via create_table). // The atomic put only prevents races between concurrent declare_table calls, // not between declare_table and existing data. - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if status.exists && !status.has_reserved_file { // Table has data but no reserved file - it was created with data return Err(NamespaceError::TableAlreadyExists { @@ -3150,17 +3676,22 @@ impl LanceNamespace for DirectoryNamespace { // concurrent declare_table calls. let reserved_file_path = self.table_reserved_file_path(&table_name); - self.put_marker_file_atomic(&reserved_file_path, &format!("table {}", table_name)) - .await - .map_err(|e| { - if e.contains("already exists") { - lance_core::Error::from(NamespaceError::TableAlreadyExists { - message: table_name.to_string(), - }) - } else { - lance_core::Error::from(NamespaceError::Internal { message: e }) - } - })?; + put_marker_file_atomic( + &self.object_store, + &reserved_file_path, + &format!("table {}", table_name), + ) + .await + .map_err(|e| match e { + MarkerFileError::AlreadyExists { .. } => { + lance_core::Error::from(NamespaceError::TableAlreadyExists { + message: table_name.to_string(), + }) + } + MarkerFileError::Other { message } => { + lance_core::Error::from(NamespaceError::Internal { message }) + } + })?; // For backwards compatibility, only skip vending credentials when explicitly set to false let vend_credentials = request.vend_credentials.unwrap_or(true); @@ -3188,7 +3719,7 @@ impl LanceNamespace for DirectoryNamespace { ) -> Result { self.record_op("register_table"); // If manifest is enabled, delegate to manifest namespace - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return LanceNamespace::register_table(manifest_ns.as_ref(), request).await; } @@ -3205,7 +3736,7 @@ impl LanceNamespace for DirectoryNamespace { ) -> Result { self.record_op("deregister_table"); // If manifest is enabled, delegate to manifest namespace - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return LanceNamespace::deregister_table(manifest_ns.as_ref(), request).await; } @@ -3215,7 +3746,7 @@ impl LanceNamespace for DirectoryNamespace { // Check table existence and deregistration status. // This provides better error messages for common cases. - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { @@ -3237,18 +3768,20 @@ impl LanceNamespace for DirectoryNamespace { // If a race occurs and another process already created the file, // we'll get an AlreadyExists error which we convert to a proper message. let deregistered_path = self.table_deregistered_file_path(&table_name); - self.put_marker_file_atomic( + put_marker_file_atomic( + &self.object_store, &deregistered_path, &format!("deregistration marker for table {}", table_name), ) .await - .map_err(|e| { - if e.contains("already exists") { + .map_err(|e| match e { + MarkerFileError::AlreadyExists { .. } => { lance_core::Error::from(NamespaceError::InvalidTableState { message: format!("Table is already deregistered: {}", table_name), }) - } else { - lance_core::Error::from(NamespaceError::Internal { message: e }) + } + MarkerFileError::Other { message } => { + lance_core::Error::from(NamespaceError::Internal { message }) } })?; @@ -3263,7 +3796,7 @@ impl LanceNamespace for DirectoryNamespace { &self, request: AlterTableAddColumnsRequest, ) -> Result { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.alter_table_add_columns(request).await; } @@ -3272,7 +3805,7 @@ impl LanceNamespace for DirectoryNamespace { let table_uri = self.table_full_uri(&table_name); // Check table existence and deregistration status before opening the dataset - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { message: table_name, @@ -3321,7 +3854,7 @@ impl LanceNamespace for DirectoryNamespace { &self, request: AlterTableAlterColumnsRequest, ) -> Result { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.alter_table_alter_columns(request).await; } @@ -3329,7 +3862,7 @@ impl LanceNamespace for DirectoryNamespace { let table_uri = self.table_full_uri(&table_name); // Check table existence and deregistration status before opening the dataset - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { message: table_name, @@ -3371,7 +3904,7 @@ impl LanceNamespace for DirectoryNamespace { &self, request: AlterTableDropColumnsRequest, ) -> Result { - if let Some(ref manifest_ns) = self.manifest_ns { + if let Some(manifest_ns) = self.manifest_ns_for_write().await? { return manifest_ns.alter_table_drop_columns(request).await; } @@ -3379,7 +3912,7 @@ impl LanceNamespace for DirectoryNamespace { let table_uri = self.table_full_uri(&table_name); // Check table existence and deregistration status before opening the dataset - let status = self.check_table_status(&table_name).await; + let status = self.check_table_status(&table_name).await?; if !status.exists { return Err(NamespaceError::TableNotFound { message: table_name, @@ -3435,6 +3968,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(ListTableVersionsResponse { versions: table_versions, page_token: None, + ..Default::default() }) } @@ -3445,11 +3979,11 @@ impl LanceNamespace for DirectoryNamespace { self.record_op("create_table_version"); let branch = Self::normalized_branch(request.branch.as_deref())?; let table_uri = self.resolve_table_location(&request.id).await?; - let (table_uri, table_path) = match branch { + let (table_uri, table_path, branch_parent_version) = match branch { Some(b) => self.resolve_branch_for_commit(&table_uri, b).await?, None => { let table_path = self.object_store_path_from_uri(&table_uri)?; - (table_uri, table_path) + (table_uri, table_path, None) } }; @@ -3474,66 +4008,79 @@ impl LanceNamespace for DirectoryNamespace { }) })?; - let copy_result = match self - .object_store - .inner - .copy_if_not_exists(&staging_path, &final_path) - .await - { - Ok(()) => Ok(()), - Err(ObjectStoreError::NotImplemented { .. }) - | Err(ObjectStoreError::NotSupported { .. }) => { - let manifest_data = self - .object_store - .inner - .get(&staging_path) - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to read staging manifest at '{}': {}", - staging_manifest_path, e - ), - }) - })? - .bytes() - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to read staging manifest bytes at '{}': {}", - staging_manifest_path, e - ), - }) - })?; - self.object_store - .inner - .put_opts( - &final_path, - manifest_data.into(), - PutOptions { - mode: PutMode::Create, - ..Default::default() - }, - ) - .await - .map(|_| ()) + // Idempotent retry: version path already published with the same content. + match self.object_store.inner.head(&final_path).await { + Ok(existing_meta) => { + return self + .resolve_existing_table_version(ExistingTableVersionResolve { + staging_path: &staging_path, + final_path: &final_path, + version, + table_uri: &table_uri, + final_meta: &existing_meta, + request_manifest_size: request.manifest_size, + }) + .await; } - Err(e) => Err(e), - }; + Err(ObjectStoreError::NotFound { .. }) => {} + Err(e) => { + return Err(lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to stat version {} for table at '{}': {}", + version, table_uri, e + ), + })); + } + } + + // Strict CAS: only allow appending latest+1 (or the empty-chain bootstrap + // version: v1 on main, BranchContents.parent_version on a registered branch). + let is_branch = branch.is_some(); + self.enforce_create_table_version_cas( + &table_path, + version, + &table_uri, + is_branch, + branch_parent_version, + ) + .await?; + + // Materialize with Create / copy_if_not_exists only — never overwrite. + let copy_result = self + .materialize_version_manifest_create(&staging_path, &final_path, staging_manifest_path) + .await; match copy_result { Ok(()) => {} Err(ObjectStoreError::AlreadyExists { .. }) | Err(ObjectStoreError::Precondition { .. }) => { - return Err(lance_core::Error::from( - NamespaceError::ConcurrentModification { + // Lost a Create race: succeed only if the winner published identical bytes. + let existing_meta = self.object_store.inner.head(&final_path).await.map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { message: format!( - "Version {} already exists for table at '{}'", - version, table_uri + "Version {} conflict for table at '{}' but failed to stat winner: {}", + version, table_uri, e ), - }, - )); + }) + })?; + return self + .resolve_existing_table_version(ExistingTableVersionResolve { + staging_path: &staging_path, + final_path: &final_path, + version, + table_uri: &table_uri, + final_meta: &existing_meta, + request_manifest_size: request.manifest_size, + }) + .await; + } + Err(ObjectStoreError::NotFound { .. }) => { + return Err(lance_core::Error::from(NamespaceError::InvalidInput { + message: format!( + "Staging manifest not found at '{}' for version {} of table at '{}'", + staging_manifest_path, version, table_uri + ), + })); } Err(e) => { return Err(lance_core::Error::from(NamespaceError::Internal { @@ -3558,7 +4105,6 @@ impl LanceNamespace for DirectoryNamespace { ), }) })?; - let manifest_size = final_meta.size as i64; // Delete the staging manifest after successful copy if let Err(e) = self.object_store.inner.delete(&staging_path).await { @@ -3569,17 +4115,11 @@ impl LanceNamespace for DirectoryNamespace { ); } - Ok(CreateTableVersionResponse { - transaction_id: None, - version: Some(Box::new(TableVersion { - version: version as i64, - manifest_path: final_path.to_string(), - manifest_size: Some(manifest_size), - e_tag: final_meta.e_tag, - timestamp_millis: None, - metadata: None, - })), - }) + Ok(Self::create_table_version_response( + version, + &final_path, + &final_meta, + )) } async fn describe_table_version( @@ -3622,6 +4162,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(DescribeTableVersionResponse { version: Box::new(table_version), + ..Default::default() }) } @@ -3675,6 +4216,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(BatchDeleteTableVersionsResponse { deleted_count: Some(total_deleted_count), transaction_id: None, + ..Default::default() }) } @@ -3744,7 +4286,10 @@ impl LanceNamespace for DirectoryNamespace { })? .map(|transaction| transaction.uuid); - Ok(CreateTableIndexResponse { transaction_id }) + Ok(CreateTableIndexResponse { + transaction_id, + ..Default::default() + }) } async fn list_table_indices( @@ -3784,7 +4329,7 @@ impl LanceNamespace for DirectoryNamespace { .map(|field_id| { dataset .schema() - .field_path(i32::try_from(*field_id).map_err(|e| { + .field_path_minimal(i32::try_from(*field_id).map_err(|e| { lance_core::Error::from(NamespaceError::Internal { message: format!( "Field id {} does not fit in i32 for table '{}': {}", @@ -3839,6 +4384,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(ListTableIndicesResponse { indexes: indices, page_token, + ..Default::default() }) } @@ -4130,6 +4676,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(AlterTransactionResponse { status: final_status, properties: response.properties, + ..Default::default() }) } @@ -4152,6 +4699,7 @@ impl LanceNamespace for DirectoryNamespace { let response = self.create_table_index(request).await?; Ok(CreateTableScalarIndexResponse { transaction_id: response.transaction_id, + ..Default::default() }) } @@ -4212,7 +4760,10 @@ impl LanceNamespace for DirectoryNamespace { })? .map(|transaction| transaction.uuid); - Ok(DropTableIndexResponse { transaction_id }) + Ok(DropTableIndexResponse { + transaction_id, + ..Default::default() + }) } async fn list_all_tables(&self, request: ListTablesRequest) -> Result { @@ -4282,7 +4833,10 @@ impl LanceNamespace for DirectoryNamespace { })? .map(|t| t.uuid); - Ok(RestoreTableResponse { transaction_id }) + Ok(RestoreTableResponse { + transaction_id, + ..Default::default() + }) } async fn update_table_schema_metadata( @@ -4325,6 +4879,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(UpdateTableSchemaMetadataResponse { metadata: Some(updated_metadata), transaction_id, + ..Default::default() }) } @@ -4550,6 +5105,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(InsertIntoTableResponse { transaction_id: None, + ..Default::default() }) } @@ -4560,11 +5116,7 @@ impl LanceNamespace for DirectoryNamespace { ) -> Result { self.record_op("merge_insert_into_table"); let table_uri = self.resolve_table_location(&request.id).await?; - let on = request.on.as_ref().ok_or_else(|| { - lance_core::Error::from(NamespaceError::InvalidInput { - message: "'on' field is required for merge_insert_into_table".to_string(), - }) - })?; + let on = merge_insert_on_columns(request.on.as_deref(), "merge_insert_into_table")?; let table_has_manifests = self.table_uri_has_actual_manifests(&table_uri).await?; let (reader, num_rows) = @@ -4581,6 +5133,7 @@ impl LanceNamespace for DirectoryNamespace { num_inserted_rows: Some(num_rows as i64), num_deleted_rows: Some(0), version: Some(version), + ..Default::default() }); } @@ -4589,8 +5142,8 @@ impl LanceNamespace for DirectoryNamespace { .await?, ); - let mut merge_builder = MergeInsertBuilder::try_new(dataset.clone(), vec![on.clone()]) - .map_err(|e| { + let mut merge_builder = + MergeInsertBuilder::try_new(dataset.clone(), on.to_vec()).map_err(|e| { lance_core::Error::from(NamespaceError::InvalidInput { message: format!("Failed to create merge_insert_into_table builder: {}", e), }) @@ -4651,6 +5204,7 @@ impl LanceNamespace for DirectoryNamespace { num_inserted_rows: Some(stats.num_inserted_rows as i64), num_deleted_rows: Some(stats.num_deleted_rows as i64), version: Some(dataset.version().version as i64), + ..Default::default() }) } @@ -4735,6 +5289,7 @@ impl LanceNamespace for DirectoryNamespace { updated_rows: result.rows_updated as i64, version, properties: None, + ..Default::default() }) } @@ -4764,6 +5319,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(DeleteFromTableResponse { transaction_id: None, version: Some(result.new_dataset.version().version as i64), + ..Default::default() }) } @@ -4896,14 +5452,21 @@ impl LanceNamespace for DirectoryNamespace { })?; } + scanner + .full_text_search(fts) + .map_err(|e| NamespaceError::InvalidInput { + message: format!("Invalid full text search: {:?}", e), + })?; + } else if let Some(ref structured_query) = fts_query.structured_query { + // Structured FTS: map the namespace query model into the engine FtsQuery. + let engine_query = build_engine_fts_query(&structured_query.query)?; + let fts = FullTextSearchQuery::new_query(engine_query); scanner .full_text_search(fts) .map_err(|e| NamespaceError::InvalidInput { message: format!("Invalid full text search: {:?}", e), })?; } - // Note: structured_query would require more complex parsing - // For now, we only support string_query } // Apply column projection if specified @@ -5030,6 +5593,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(ListTableTagsResponse { tags, page_token: None, + ..Default::default() }) } @@ -5059,6 +5623,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(GetTableTagVersionResponse { version: contents.version as i64, branch: contents.branch, + ..Default::default() }) } @@ -5096,6 +5661,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(CreateTableTagResponse { transaction_id: None, + ..Default::default() }) } @@ -5124,6 +5690,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(DeleteTableTagResponse { transaction_id: None, + ..Default::default() }) } @@ -5161,6 +5728,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(UpdateTableTagResponse { transaction_id: None, + ..Default::default() }) } @@ -5230,6 +5798,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(CreateTableBranchResponse { transaction_id: None, + ..Default::default() }) } @@ -5275,6 +5844,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(ListTableBranchesResponse { branches, page_token: None, + ..Default::default() }) } @@ -5311,6 +5881,7 @@ impl LanceNamespace for DirectoryNamespace { Ok(DeleteTableBranchResponse { transaction_id: None, + ..Default::default() }) } @@ -5319,74 +5890,485 @@ impl LanceNamespace for DirectoryNamespace { } } -#[cfg(test)] -mod tests { - use super::*; - use arrow_ipc::reader::{FileReader, StreamReader}; - use lance::dataset::Dataset; - use lance::index::DatasetIndexExt; - use lance_core::utils::tempfile::{TempStdDir, TempStrDir}; - use lance_core::utils::testing::CountingObjectStore; - use lance_io::object_store::{providers::local::FileStoreProvider, uri_to_url}; - use lance_namespace::error::ErrorCode; - use lance_namespace::models::{ - CreateTableRequest, JsonArrowDataType, JsonArrowField, JsonArrowSchema, ListTablesRequest, - QueryTableRequestColumns, - }; - use lance_namespace::schema::convert_json_arrow_schema; - use std::io::Cursor; - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - use url::Url; +/// Error from [`put_marker_file_atomic`]. +#[derive(Debug)] +pub(crate) enum MarkerFileError { + /// The final marker path is already present (Create / rename race). + AlreadyExists { description: String }, + /// Staging or publish failed for a non-conflict reason. + Other { message: String }, +} - fn assert_plan_contains_all(plan: &str, expected_fragments: &[&str], context: &str) { - for expected_fragment in expected_fragments { - assert!( - plan.contains(expected_fragment), - "{}. Missing fragment: '{}'. Plan:\n{}", - context, - expected_fragment, - plan - ); +impl std::fmt::Display for MarkerFileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::AlreadyExists { description } => { + write!(f, "{} already exists", description) + } + Self::Other { message } => write!(f, "{}", message), } } +} - fn mutation_error_code(err: lance_core::Error) -> ErrorCode { - match err { - lance_core::Error::Namespace { source, .. } => source - .downcast_ref::() - .expect("mutation error should wrap a NamespaceError") - .code(), - other => panic!("expected Namespace error, got: {other:?}"), - } - } +/// Atomically create a marker file (e.g. `.lance-reserved`) with Create semantics. +/// +/// Some object stores implement `PutMode::Create` via temp+rename that reuses the +/// final basename. Dotfile targets such as `.lance-reserved` therefore produce +/// temp names containing `..`, which these stores reject. Stage under a non-dot +/// sibling, then claim the final path with `rename_if_not_exists`. +/// +/// When `rename_if_not_exists` is unavailable, fall back to +/// `copy_if_not_exists(staging → target)`, then `PutMode::Create` on the target. +/// That Create path is only for stores whose Create is a true conditional PUT +/// (not basename-derived temp+rename); such stores are exactly the ones that +/// typically omit rename/copy conditionals. +/// +/// Some object stores also fail to flush empty objects, so the conditional rename +/// can fail with NotFound. Use a tiny non-empty payload. +/// +/// Staging cleanup is best-effort with a few short retries. A delete that still +/// fails after retries leaves a tiny `lance-marker.staging.*` orphan. Async Drop +/// cannot await object-store I/O, so RAII is not used here. Each call uses a +/// unique staging UUID, so concurrent callers never contend on the same cleanup. +pub(crate) async fn put_marker_file_atomic( + object_store: &ObjectStore, + path: &Path, + file_description: &str, +) -> std::result::Result<(), MarkerFileError> { + let staging_name = format!("lance-marker.staging.{}", uuid::Uuid::new_v4().simple()); + let path_str = path.as_ref(); + let staging_path = match path_str.rfind('/') { + Some(idx) => Path::from(format!("{}/{}", &path_str[..idx], staging_name)), + None => Path::from(staging_name.as_str()), + }; - /// `map_mutation_error` must classify commit-conflict variants the same way as - /// `convert_lance_commit_error` in `manifest.rs`: `CommitConflict` is a retries-exhausted - /// version collision that is safe to retry (`Throttling`), while the semantic-conflict variants - /// map to `ConcurrentModification`. - #[test] - fn test_map_mutation_error_commit_conflict_alignment() { - let boxed = || -> Box { - Box::::from("inner conflict") - }; + object_store + .inner + .put(&staging_path, bytes::Bytes::from_static(b"reserved").into()) + .await + .map_err(|e| MarkerFileError::Other { + message: format!("Failed to stage {}: {:?}", file_description, e), + })?; - let throttling_cases = vec![lance_core::Error::commit_conflict_source(1, boxed())]; - for err in throttling_cases { - let code = mutation_error_code(DirectoryNamespace::map_mutation_error( - err, - "update", - "memory://t", - )); - assert_eq!(code, ErrorCode::Throttling); + // Successful rename consumes the staging object; every other path must + // delete it (best-effort) so conflict/fallback races do not accumulate. + let mut staging_consumed = false; + let publish_result = match object_store + .inner + .rename_if_not_exists(&staging_path, path) + .await + { + Ok(()) => { + staging_consumed = true; + Ok(()) + } + Err(ObjectStoreError::NotImplemented { .. }) + | Err(ObjectStoreError::NotSupported { .. }) => { + match object_store + .inner + .copy_if_not_exists(&staging_path, path) + .await + { + Ok(()) => Ok(()), + Err(ObjectStoreError::NotImplemented { .. }) + | Err(ObjectStoreError::NotSupported { .. }) => object_store + .inner + .put_opts( + path, + bytes::Bytes::from_static(b"reserved").into(), + PutOptions { + mode: PutMode::Create, + ..Default::default() + }, + ) + .await + .map(|_| ()), + Err(e) => Err(e), + } } + Err(e) => Err(e), + }; - let concurrent_cases = vec![ - lance_core::Error::too_much_write_contention("contention"), - lance_core::Error::retryable_commit_conflict_source(1, boxed()), - lance_core::Error::incompatible_transaction_source(boxed()), + if !staging_consumed { + delete_staging_marker_best_effort(object_store, &staging_path).await; + } + + match publish_result { + Ok(()) => Ok(()), + Err(ObjectStoreError::AlreadyExists { .. }) + | Err(ObjectStoreError::Precondition { .. }) => Err(MarkerFileError::AlreadyExists { + description: file_description.to_string(), + }), + Err(e) => Err(MarkerFileError::Other { + message: format!("Failed to create {}: {:?}", file_description, e), + }), + } +} + +/// Best-effort delete of a per-call staging marker, with short retries for +/// transient store errors. `NotFound` is treated as success (delete may have +/// succeeded despite an earlier ambiguous failure). +async fn delete_staging_marker_best_effort(object_store: &ObjectStore, staging_path: &Path) { + const MAX_ATTEMPTS: u32 = 3; + const BACKOFF_MS: [u64; 2] = [20, 50]; + + let mut last_err: Option = None; + for attempt in 0..MAX_ATTEMPTS { + match object_store.inner.delete(staging_path).await { + Ok(()) => return, + Err(ObjectStoreError::NotFound { .. }) => return, + Err(e) => { + last_err = Some(e); + if let Some(&delay_ms) = BACKOFF_MS.get(attempt as usize) { + tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await; + } + } + } + } + if let Some(del_err) = last_err { + log::warn!( + "Failed to delete staging marker at '{}' after {} attempts: {:?}", + staging_path, + MAX_ATTEMPTS, + del_err + ); + } +} + +/// Maps a namespace structured `FtsQuery` model into the engine `FtsQuery`. Mirrors the mapping the +/// JNI scanner performs, so the local `queryTable` path honors `structured_query` the same way a +/// `fragment.newScan(fullTextQuery)` does. +fn build_engine_fts_query( + query: &lance_namespace::models::FtsQuery, +) -> std::result::Result { + if let Some(ref m) = query.r#match { + Ok(FtsQuery::Match(build_engine_match_query(m)?)) + } else if let Some(ref p) = query.phrase { + let mut phrase = PhraseQuery::new(p.terms.clone()); + if let Some(ref column) = p.column { + phrase = phrase.with_column(Some(column.clone())); + } + if let Some(slop) = p.slop { + phrase = phrase.with_slop(slop as u32); + } + Ok(FtsQuery::Phrase(phrase)) + } else if let Some(ref mm) = query.multi_match { + let match_queries = mm + .match_queries + .iter() + .map(build_engine_match_query) + .collect::, _>>()?; + Ok(FtsQuery::MultiMatch(MultiMatchQuery { match_queries })) + } else if let Some(ref b) = query.boolean { + let mut clauses: Vec<(Occur, FtsQuery)> = Vec::new(); + for clause in &b.must { + clauses.push((Occur::Must, build_engine_fts_query(clause)?)); + } + for clause in &b.should { + clauses.push((Occur::Should, build_engine_fts_query(clause)?)); + } + for clause in &b.must_not { + clauses.push((Occur::MustNot, build_engine_fts_query(clause)?)); + } + Ok(FtsQuery::Boolean(BooleanQuery::new(clauses))) + } else if let Some(ref boost) = query.boost { + let positive = build_engine_fts_query(&boost.positive)?; + let negative = build_engine_fts_query(&boost.negative)?; + Ok(FtsQuery::Boost(BoostQuery::new( + positive, + negative, + boost.negative_boost, + ))) + } else { + Err(NamespaceError::InvalidInput { + message: "structured_query.query must set exactly one of match, phrase, multi_match, \ + boolean, or boost" + .to_string(), + }) + } +} + +fn build_engine_match_query( + m: &lance_namespace::models::MatchQuery, +) -> std::result::Result { + let mut match_query = MatchQuery::new(m.terms.clone()); + if let Some(ref column) = m.column { + match_query = match_query.with_column(Some(column.clone())); + } + if let Some(boost) = m.boost { + match_query = match_query.with_boost(boost); + } + if let Some(fuzziness) = m.fuzziness { + match_query = match_query.with_fuzziness(Some(fuzziness as u32)); + } + if let Some(max_expansions) = m.max_expansions { + match_query = match_query.with_max_expansions(max_expansions as usize); + } + if let Some(ref operator) = m.operator { + let op = + Operator::try_from(operator.as_str()).map_err(|e| NamespaceError::InvalidInput { + message: format!("Invalid FTS operator: {:?}", e), + })?; + match_query = match_query.with_operator(op); + } + if let Some(prefix_length) = m.prefix_length { + match_query = match_query.with_prefix_length(prefix_length as u32); + } + Ok(match_query) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_ipc::reader::{FileReader, StreamReader}; + use lance::index::vector::StageParams; + use rstest::rstest; + + fn build_ivf_rq_num_bits(num_bits: Option) -> Result { + let mut request = CreateTableIndexRequest::new("vector".to_string(), "IVF_RQ".to_string()); + request.num_bits = num_bits; + + let DirectoryIndexParams::Vector { + index_type: IndexType::IvfRq, + params, + } = DirectoryNamespace::build_index_params(&request)? + else { + panic!("expected IVF_RQ vector index params"); + }; + match params.stages.as_slice() { + [StageParams::Ivf(_), StageParams::RQ(rq)] => Ok(rq.num_bits), + stages => panic!("expected IVF and RQ stages, got {stages:?}"), + } + } + + #[rstest] + #[case::omitted(None, 5)] + #[case::explicit_one(Some(1), 1)] + #[case::explicit_max(Some(9), 9)] + fn test_build_index_params_ivf_rq_num_bits( + #[case] requested: Option, + #[case] expected: u8, + ) { + assert_eq!(build_ivf_rq_num_bits(requested).unwrap(), expected); + } + + #[rstest] + #[case::negative(-1)] + #[case::zero(0)] + #[case::above_max(10)] + #[case::conversion_overflow(i32::MAX)] + fn test_build_index_params_rejects_invalid_ivf_rq_num_bits(#[case] requested: i32) { + let error = build_ivf_rq_num_bits(Some(requested)) + .expect_err("invalid IVF_RQ num_bits should fail"); + let message = error.to_string(); + + assert_eq!(mutation_error_code(error), ErrorCode::InvalidInput); + assert!( + message.contains(&format!( + "IVF_RQ num_bits must be in 1..=9, got {requested}" + )), + "unexpected error message: {message}" + ); + } + + #[test] + fn test_build_engine_fts_query_match() { + let mut ns_match = lance_namespace::models::MatchQuery::new("hello world".to_string()); + ns_match.column = Some("body".to_string()); + ns_match.operator = Some("AND".to_string()); + ns_match.fuzziness = Some(1); + ns_match.max_expansions = Some(30); + ns_match.boost = Some(2.0); + ns_match.prefix_length = Some(2); + + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.r#match = Some(Box::new(ns_match)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Match(m) => { + assert_eq!(m.terms, "hello world"); + assert_eq!(m.column, Some("body".to_string())); + assert_eq!(m.operator, Operator::And); + assert_eq!(m.fuzziness, Some(1)); + assert_eq!(m.max_expansions, 30); + assert_eq!(m.boost, 2.0); + assert_eq!(m.prefix_length, 2); + } + other => panic!("expected Match, got {:?}", other), + } + } + + /// Wraps a namespace `MatchQuery` (with a column) as an `FtsQuery` for use as a clause in + /// compound queries (boolean / boost). + fn ns_match_query(terms: &str, column: &str) -> lance_namespace::models::FtsQuery { + let mut m = lance_namespace::models::MatchQuery::new(terms.to_string()); + m.column = Some(column.to_string()); + let mut q = lance_namespace::models::FtsQuery::new(); + q.r#match = Some(Box::new(m)); + q + } + + #[test] + fn test_build_engine_fts_query_phrase() { + let mut ns_phrase = lance_namespace::models::PhraseQuery::new("hello world".to_string()); + ns_phrase.column = Some("body".to_string()); + ns_phrase.slop = Some(2); + + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.phrase = Some(Box::new(ns_phrase)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Phrase(p) => { + assert_eq!(p.terms, "hello world"); + assert_eq!(p.column, Some("body".to_string())); + assert_eq!(p.slop, 2); + } + other => panic!("expected Phrase, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_multi_match() { + let mut m1 = lance_namespace::models::MatchQuery::new("hello".to_string()); + m1.column = Some("title".to_string()); + let mut m2 = lance_namespace::models::MatchQuery::new("hello".to_string()); + m2.column = Some("body".to_string()); + m2.boost = Some(2.0); + + let ns_multi = lance_namespace::models::MultiMatchQuery::new(vec![m1, m2]); + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.multi_match = Some(Box::new(ns_multi)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::MultiMatch(mm) => { + assert_eq!(mm.match_queries.len(), 2); + assert_eq!(mm.match_queries[0].terms, "hello"); + assert_eq!(mm.match_queries[0].column, Some("title".to_string())); + assert_eq!(mm.match_queries[1].column, Some("body".to_string())); + assert_eq!(mm.match_queries[1].boost, 2.0); + } + other => panic!("expected MultiMatch, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_boolean() { + // BooleanQuery::new(must, must_not, should) + let ns_boolean = lance_namespace::models::BooleanQuery::new( + vec![ns_match_query("must-term", "body")], + vec![ns_match_query("must-not-term", "body")], + vec![ns_match_query("should-term", "body")], + ); + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.boolean = Some(Box::new(ns_boolean)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Boolean(b) => { + assert!(matches!(&b.must[..], [FtsQuery::Match(m)] if m.terms == "must-term")); + assert!( + matches!(&b.must_not[..], [FtsQuery::Match(m)] if m.terms == "must-not-term") + ); + assert!(matches!(&b.should[..], [FtsQuery::Match(m)] if m.terms == "should-term")); + } + other => panic!("expected Boolean, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_boost() { + let mut ns_boost = lance_namespace::models::BoostQuery::new( + ns_match_query("positive-term", "body"), + ns_match_query("negative-term", "body"), + ); + ns_boost.negative_boost = Some(0.25); + + let mut ns_query = lance_namespace::models::FtsQuery::new(); + ns_query.boost = Some(Box::new(ns_boost)); + + match build_engine_fts_query(&ns_query).unwrap() { + FtsQuery::Boost(b) => { + assert!( + matches!(b.positive.as_ref(), FtsQuery::Match(m) if m.terms == "positive-term") + ); + assert!( + matches!(b.negative.as_ref(), FtsQuery::Match(m) if m.terms == "negative-term") + ); + assert_eq!(b.negative_boost, 0.25); + } + other => panic!("expected Boost, got {:?}", other), + } + } + + #[test] + fn test_build_engine_fts_query_requires_a_variant() { + // An FtsQuery with no variant set is rejected rather than silently ignored. + let empty = lance_namespace::models::FtsQuery::new(); + assert!(build_engine_fts_query(&empty).is_err()); + } + use lance::dataset::Dataset; + use lance::index::DatasetIndexExt; + use lance_core::utils::tempfile::{TempStdDir, TempStrDir}; + use lance_core::utils::testing::CountingObjectStore; + use lance_io::object_store::{providers::local::FileStoreProvider, uri_to_url}; + use lance_namespace::error::ErrorCode; + use lance_namespace::models::{ + CreateTableRequest, JsonArrowDataType, JsonArrowField, JsonArrowSchema, ListTablesRequest, + QueryTableRequestColumns, + }; + use lance_namespace::schema::convert_json_arrow_schema; + use std::io::Cursor; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + use url::Url; + + fn assert_plan_contains_all(plan: &str, expected_fragments: &[&str], context: &str) { + for expected_fragment in expected_fragments { + assert!( + plan.contains(expected_fragment), + "{}. Missing fragment: '{}'. Plan:\n{}", + context, + expected_fragment, + plan + ); + } + } + + fn mutation_error_code(err: lance_core::Error) -> ErrorCode { + match err { + lance_core::Error::Namespace { source, .. } => source + .downcast_ref::() + .expect("mutation error should wrap a NamespaceError") + .code(), + other => panic!("expected Namespace error, got: {other:?}"), + } + } + + /// `map_mutation_error` must classify commit-conflict variants the same way as + /// `convert_lance_commit_error` in `manifest.rs`: `CommitConflict` is a retries-exhausted + /// version collision that is safe to retry (`Throttling`), while the semantic-conflict variants + /// map to `ConcurrentModification`. + #[test] + fn test_map_mutation_error_commit_conflict_alignment() { + let boxed = || -> Box { + Box::::from("inner conflict") + }; + + let throttling_cases = vec![lance_core::Error::commit_conflict_source(1, boxed())]; + for err in throttling_cases { + let code = mutation_error_code(DirectoryNamespace::map_mutation_error( + err, + "update", + "memory://t", + )); + assert_eq!(code, ErrorCode::Throttling); + } + + let concurrent_cases = vec![ + lance_core::Error::too_much_write_contention("contention"), + lance_core::Error::retryable_commit_conflict_source(1, boxed()), + lance_core::Error::incompatible_transaction_source(boxed()), lance_core::Error::version_conflict("conflict", 0, 3), ]; for err in concurrent_cases { @@ -5410,6 +6392,250 @@ mod tests { (namespace, temp_dir) } + /// The early-stop path (ordered stores) and the collect-then-sort path + /// must return the same results for every descending/limit combination. + #[tokio::test] + async fn test_list_versions_under_ordering_and_limit() { + use lance_table::io::commit::ManifestNamingScheme; + + async fn seed_and_check(ns: &DirectoryNamespace) { + let table_path = ns.base_path.clone().join("lv_test.lance"); + for v in 1..=7u64 { + let p = ManifestNamingScheme::V2.manifest_path(&table_path, v); + ns.object_store.put(&p, b"m".as_slice()).await.unwrap(); + } + // A retained staging blob (sorts ahead of every committed + // manifest) and a detached manifest (sorts after) must be excluded + // without breaking naming-scheme detection. + let staging = Path::parse(format!( + "{}-cee4fbbb-eb19-4ea3-8ca7-54f5ec33dedc", + ManifestNamingScheme::V2.manifest_path(&table_path, 8) + )) + .unwrap(); + ns.object_store + .put(&staging, b"s".as_slice()) + .await + .unwrap(); + let detached = table_path.clone().join(VERSIONS_DIR).join("d123.manifest"); + ns.object_store + .put(&detached, b"d".as_slice()) + .await + .unwrap(); + fn versions(r: &[TableVersion]) -> Vec { + r.iter().map(|t| t.version).collect() + } + + let got = ns + .list_versions_under(&table_path, true, Some(1)) + .await + .unwrap(); + assert_eq!(versions(&got), vec![7]); + let got = ns + .list_versions_under(&table_path, true, Some(3)) + .await + .unwrap(); + assert_eq!(versions(&got), vec![7, 6, 5]); + + let got = ns + .list_versions_under(&table_path, false, Some(2)) + .await + .unwrap(); + assert_eq!(versions(&got), vec![1, 2]); + + let got = ns + .list_versions_under(&table_path, true, None) + .await + .unwrap(); + assert_eq!(versions(&got), vec![7, 6, 5, 4, 3, 2, 1]); + let got = ns + .list_versions_under(&table_path, false, None) + .await + .unwrap(); + assert_eq!(versions(&got), vec![1, 2, 3, 4, 5, 6, 7]); + + let got = ns + .list_versions_under(&table_path, true, Some(0)) + .await + .unwrap(); + assert!(got.is_empty()); + let got = ns + .list_versions_under(&table_path, true, Some(100)) + .await + .unwrap(); + assert_eq!(versions(&got), vec![7, 6, 5, 4, 3, 2, 1]); + + // Negative limits are ignored, matching `apply_pagination`. + let got = ns + .list_versions_under(&table_path, true, Some(-1)) + .await + .unwrap(); + assert_eq!(versions(&got), vec![7, 6, 5, 4, 3, 2, 1]); + } + + let ns_mem = DirectoryNamespaceBuilder::new("memory://lv-test") + .build() + .await + .unwrap(); + assert!(ns_mem.object_store.list_is_lexically_ordered); + seed_and_check(&ns_mem).await; + + let (ns_fs, _tmp) = create_test_namespace().await; + assert!(!ns_fs.object_store.list_is_lexically_ordered); + seed_and_check(&ns_fs).await; + } + + /// A retained staging blob sorts ahead of the newest committed manifest; + /// if scheme detection reads it, the `descending, limit=1` hot path falls + /// back to consuming the whole directory. Asserts the consumption bound. + #[tokio::test] + async fn test_list_versions_under_early_stop_bounded_consumption() { + use lance_io::object_store::providers::memory::MemoryStoreProvider; + use lance_table::io::commit::ManifestNamingScheme; + + #[derive(Debug)] + struct EntryCountingStore { + target: Arc, + entries_listed: Arc, + } + + impl std::fmt::Display for EntryCountingStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "EntryCountingStore({})", self.target) + } + } + + #[async_trait] + impl OSObjectStore for EntryCountingStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + self.target.put_opts(location, bytes, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + self.target.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + self.target.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + self.target.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + let entries_listed = self.entries_listed.clone(); + self.target + .list(prefix) + .inspect(move |_| { + entries_listed.fetch_add(1, Ordering::SeqCst); + }) + .boxed() + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + self.target.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> { + self.target.copy_opts(from, to, opts).await + } + } + + #[derive(Debug)] + struct EntryCountingMemoryProvider { + entries_listed: Arc, + } + + #[async_trait] + impl lance_io::object_store::ObjectStoreProvider for EntryCountingMemoryProvider { + async fn new_store( + &self, + base_path: Url, + params: &ObjectStoreParams, + ) -> Result { + let mut store = MemoryStoreProvider.new_store(base_path, params).await?; + store.inner = Arc::new(EntryCountingStore { + target: store.inner.clone(), + entries_listed: self.entries_listed.clone(), + }); + Ok(store) + } + + fn extract_path(&self, url: &Url) -> Result { + MemoryStoreProvider.extract_path(url) + } + + fn calculate_object_store_prefix( + &self, + url: &Url, + storage_options: Option<&HashMap>, + ) -> Result { + MemoryStoreProvider.calculate_object_store_prefix(url, storage_options) + } + } + + let entries_listed = Arc::new(AtomicUsize::new(0)); + let registry = Arc::new(ObjectStoreRegistry::default()); + registry.insert( + "memory-object-store", + Arc::new(EntryCountingMemoryProvider { + entries_listed: entries_listed.clone(), + }), + ); + let session = Arc::new(Session::new(0, 0, registry)); + let ns = DirectoryNamespaceBuilder::new("memory-object-store://lv-count") + .session(session) + .build() + .await + .unwrap(); + assert!(ns.object_store.list_is_lexically_ordered); + + let table_path = ns.base_path.clone().join("lv_count.lance"); + for v in 1..=100u64 { + let p = ManifestNamingScheme::V2.manifest_path(&table_path, v); + ns.object_store.put(&p, b"m".as_slice()).await.unwrap(); + } + // Sorts ahead of every committed manifest: the first raw entry. + let staging = Path::parse(format!( + "{}-cee4fbbb-eb19-4ea3-8ca7-54f5ec33dedc", + ManifestNamingScheme::V2.manifest_path(&table_path, 101) + )) + .unwrap(); + ns.object_store + .put(&staging, b"s".as_slice()) + .await + .unwrap(); + + let consumed_before = entries_listed.load(Ordering::SeqCst); + let got = ns + .list_versions_under(&table_path, true, Some(1)) + .await + .unwrap(); + let consumed = entries_listed.load(Ordering::SeqCst) - consumed_before; + + assert_eq!(got.len(), 1); + assert_eq!(got[0].version, 100); + assert_eq!( + consumed, 2, + "latest-version query must consume only the staging entry plus the \ + first committed manifest, not the whole directory (consumed {} of \ + 101 entries)", + consumed + ); + } + #[derive(Debug)] #[allow(dead_code)] struct CountingFileStoreProvider { @@ -5462,7 +6688,515 @@ mod tests { "file-object-store", Arc::new(CountingFileStoreProvider { listing_count }), ); - Arc::new(Session::new(0, 0, registry)) + Arc::new(Session::new(0, 0, registry)) + } + + // Fault-injection store: returns a runtime-toggleable result from + // `list_with_delimiter` (the call `check_table_status` makes) and delegates + // everything else, so a table can be created before failures are injected. + use futures::stream::BoxStream; + use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, + PutMultipartOptions, PutPayload, PutResult, Result as OSResult, + }; + use std::ops::Range; + + #[derive(Debug, Clone, Copy)] + enum ListBehavior { + Throttle, + ServiceUnavailable, + Internal, + NotFound, + EmptyListing, + } + + #[derive(Debug)] + struct FailingListStore { + target: Arc, + behavior: Arc>>, + } + + impl std::fmt::Display for FailingListStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FailingListStore({})", self.target) + } + } + + #[async_trait] + impl OSObjectStore for FailingListStore { + async fn put_opts( + &self, + location: &Path, + bytes: PutPayload, + opts: PutOptions, + ) -> OSResult { + self.target.put_opts(location, bytes, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> OSResult> { + self.target.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + self.target.get_opts(location, options).await + } + + async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + self.target.get_ranges(location, ranges).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, OSResult>, + ) -> BoxStream<'static, OSResult> { + self.target.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult> { + self.target.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult { + let behavior = *self.behavior.lock().unwrap(); + match behavior { + None => self.target.list_with_delimiter(prefix).await, + Some(ListBehavior::EmptyListing) => Ok(ListResult { + common_prefixes: Vec::new(), + objects: Vec::new(), + }), + // Mirrors the object_store retry-exhaustion message shape for an + // Azure ServerBusy response, which is what the incident produced. + Some(ListBehavior::Throttle) => Err(ObjectStoreError::Generic { + store: "test", + source: "Error performing list request: response error, after 3 retries, \ + max_retries: 3, retry_timeout: 180s - HTTP status server error \ + (503 Service Unavailable): ServerBusy: The server is busy" + .into(), + }), + Some(ListBehavior::ServiceUnavailable) => Err(ObjectStoreError::Generic { + store: "test", + source: "Error performing list request: 503 Service Unavailable".into(), + }), + Some(ListBehavior::Internal) => Err(ObjectStoreError::Generic { + store: "test", + source: "Error performing list request: catastrophic unclassified failure" + .into(), + }), + Some(ListBehavior::NotFound) => Err(ObjectStoreError::NotFound { + path: "test_table.lance".to_string(), + source: "entity not found".into(), + }), + } + } + + async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> { + self.target.copy_opts(from, to, opts).await + } + } + + #[derive(Debug)] + struct FailingListStoreProvider { + behavior: Arc>>, + } + + #[async_trait] + impl lance_io::object_store::ObjectStoreProvider for FailingListStoreProvider { + async fn new_store( + &self, + base_path: Url, + params: &ObjectStoreParams, + ) -> Result { + let mut store = FileStoreProvider.new_store(base_path, params).await?; + store.inner = Arc::new(FailingListStore { + target: store.inner.clone(), + behavior: self.behavior.clone(), + }); + Ok(store) + } + + fn extract_path(&self, url: &Url) -> Result { + FileStoreProvider.extract_path(url) + } + + fn calculate_object_store_prefix( + &self, + url: &Url, + storage_options: Option<&HashMap>, + ) -> Result { + FileStoreProvider.calculate_object_store_prefix(url, storage_options) + } + } + + fn build_failing_list_session(behavior: Arc>>) -> Arc { + let registry = Arc::new(ObjectStoreRegistry::default()); + registry.insert( + "file-object-store", + Arc::new(FailingListStoreProvider { behavior }), + ); + Arc::new(Session::new(0, 0, registry)) + } + + /// Build a dir-listing namespace whose object store's listing calls follow a + /// shared, runtime-toggleable behavior. Returns the namespace, the temp dir + /// (kept alive for the store), and the behavior toggle. + async fn failing_list_namespace() -> ( + DirectoryNamespace, + TempStdDir, + Arc>>, + ) { + let temp_dir = TempStdDir::default(); + let root_uri = file_object_store_uri(temp_dir.to_str().unwrap()); + let behavior = Arc::new(Mutex::new(None)); + let session = build_failing_list_session(behavior.clone()); + let namespace = DirectoryNamespaceBuilder::new(root_uri) + .session(session) + .manifest_enabled(false) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + (namespace, temp_dir, behavior) + } + + async fn create_named_dir_table(namespace: &DirectoryNamespace, name: &str) { + let schema = create_test_schema(); + let ipc_data = create_test_ipc_data(&schema); + let mut create_req = CreateTableRequest::new(); + create_req.id = Some(vec![name.to_string()]); + namespace + .create_table(create_req, Bytes::from(ipc_data)) + .await + .unwrap(); + } + + /// Regression test for the throttling-induced TableNotFound bug: a storage + /// error while resolving a table must surface as a typed storage error + /// (Throttling / ServiceUnavailable / Internal) carrying the underlying + /// evidence in its message — never as TableNotFound. + #[tokio::test] + async fn test_table_resolution_propagates_storage_errors_not_table_not_found() { + for (behavior, expected_code, evidence) in [ + (ListBehavior::Throttle, ErrorCode::Throttling, "serverbusy"), + ( + ListBehavior::ServiceUnavailable, + ErrorCode::ServiceUnavailable, + "503 service unavailable", + ), + (ListBehavior::Internal, ErrorCode::Internal, "catastrophic"), + ] { + let (namespace, _temp_dir, toggle) = failing_list_namespace().await; + create_named_dir_table(&namespace, "checkpoint").await; + *toggle.lock().unwrap() = Some(behavior); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["checkpoint".to_string()]); + let err = namespace.describe_table(describe_req).await.unwrap_err(); + let msg = err.to_string(); + assert_eq!( + mutation_error_code(err), + expected_code, + "describe_table under {behavior:?}; msg: {msg}" + ); + assert!( + msg.to_ascii_lowercase().contains(evidence), + "describe_table message must carry storage evidence '{evidence}', got: {msg}" + ); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["checkpoint".to_string()]); + let err = namespace.table_exists(exists_req).await.unwrap_err(); + let msg = err.to_string(); + assert_eq!( + mutation_error_code(err), + expected_code, + "table_exists under {behavior:?}; msg: {msg}" + ); + assert!( + msg.to_ascii_lowercase().contains(evidence), + "table_exists message must carry storage evidence '{evidence}', got: {msg}" + ); + } + } + + /// A genuine not-found error and an empty listing must both still resolve to + /// TableNotFound (the local-FS and object-store representations of "missing"). + #[tokio::test] + async fn test_table_resolution_missing_table_yields_table_not_found() { + for behavior in [ListBehavior::NotFound, ListBehavior::EmptyListing] { + let (namespace, _temp_dir, toggle) = failing_list_namespace().await; + *toggle.lock().unwrap() = Some(behavior); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["missing".to_string()]); + let err = namespace.describe_table(describe_req).await.unwrap_err(); + assert_eq!( + mutation_error_code(err), + ErrorCode::TableNotFound, + "describe_table under {behavior:?} should be TableNotFound" + ); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["missing".to_string()]); + let err = namespace.table_exists(exists_req).await.unwrap_err(); + assert_eq!( + mutation_error_code(err), + ErrorCode::TableNotFound, + "table_exists under {behavior:?} should be TableNotFound" + ); + } + } + + /// Hybrid (manifest + directory) resolution must exercise the + /// manifest→directory fall-through for a table that exists on disk but is not + /// registered in the manifest: the fall-through must succeed normally, and + /// must not degrade a storage error into TableNotFound. + /// + /// A `__manifest` table must actually exist for the manifest branch to run; + /// otherwise `manifest_ns_for_read()` is None and the manifest branch (and its + /// fall-through arm) is skipped entirely. We therefore create a *separate* + /// table through a manifest-enabled namespace first so `__manifest` exists. + #[tokio::test] + async fn test_hybrid_resolution_falls_through_and_does_not_mask_throttle() { + let temp_dir = TempStdDir::default(); + let root_uri = file_object_store_uri(temp_dir.to_str().unwrap()); + let behavior = Arc::new(Mutex::new(None)); + let session = build_failing_list_session(behavior.clone()); + + // Seed table via a manifest-enabled namespace so `__manifest` exists. + let manifest_ns = DirectoryNamespaceBuilder::new(root_uri.clone()) + .session(session.clone()) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + create_named_dir_table(&manifest_ns, "seed").await; + + // The table under test: on disk but never registered in the manifest. + let dir_ns = DirectoryNamespaceBuilder::new(root_uri.clone()) + .session(session.clone()) + .manifest_enabled(false) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + create_named_dir_table(&dir_ns, "checkpoint").await; + + // Migration enabled so root-level reads consult the manifest and fall + // through to the directory check on a manifest miss. + let hybrid_ns = DirectoryNamespaceBuilder::new(root_uri) + .session(session) + .manifest_enabled(true) + .dir_listing_enabled(true) + .dir_listing_to_manifest_migration_enabled(true) + .build() + .await + .unwrap(); + + // (a) Healthy fall-through: the manifest reports "checkpoint" absent and it + // resolves via the directory listing (guards the migration lookup). + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["checkpoint".to_string()]); + hybrid_ns + .describe_table(describe_req) + .await + .expect("unregistered on-disk table should resolve via the manifest fall-through"); + + // (b) The throttle here surfaces from the directory check after the manifest + // reports absent; the fall-through arm's own storage-error guard is covered + // by the classify_storage_error / is_manifest_table_absent_error unit tests. + *behavior.lock().unwrap() = Some(ListBehavior::Throttle); + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["checkpoint".to_string()]); + let err = hybrid_ns.describe_table(describe_req).await.unwrap_err(); + let code = mutation_error_code(err); + assert_ne!( + code, + ErrorCode::TableNotFound, + "hybrid resolution masked a throttle as TableNotFound" + ); + assert!( + matches!( + code, + ErrorCode::Throttling | ErrorCode::ServiceUnavailable | ErrorCode::Internal + ), + "hybrid resolution should surface a storage error, got {code:?}" + ); + } + + #[test] + fn test_classify_storage_error_maps_variants_and_preserves_evidence() { + let throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "list request failed, after 3 retries, max_retries: 3 - 503 ServerBusy".into(), + } + .into(); + assert!(matches!(throttle, Error::IO { .. })); + let classified = DirectoryNamespace::classify_storage_error(throttle); + let msg = classified.to_string(); + assert_eq!(mutation_error_code(classified), ErrorCode::Throttling); + assert!( + msg.to_ascii_lowercase().contains("serverbusy"), + "throttle evidence lost: {msg}" + ); + + let service: Error = ObjectStoreError::Generic { + store: "test", + source: "504 Gateway Timeout".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::classify_storage_error(service)), + ErrorCode::ServiceUnavailable + ); + + let internal: Error = ObjectStoreError::Generic { + store: "test", + source: "disk caught fire".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::classify_storage_error(internal)), + ErrorCode::Internal + ); + + // A pre-existing namespace error keeps its own code rather than being reclassified. + let preexisting: Error = NamespaceError::TableAlreadyExists { + message: "t".to_string(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::classify_storage_error(preexisting)), + ErrorCode::TableAlreadyExists + ); + } + + #[test] + fn test_is_manifest_table_absent_error() { + let table_not_found: Error = NamespaceError::TableNotFound { + message: "t".to_string(), + } + .into(); + assert!(DirectoryNamespace::is_manifest_table_absent_error( + &table_not_found + )); + let raw_not_found: Error = ObjectStoreError::NotFound { + path: "t".to_string(), + source: "x".into(), + } + .into(); + assert!(DirectoryNamespace::is_manifest_table_absent_error( + &raw_not_found + )); + + let throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "after 3 retries, max_retries: 3 ServerBusy".into(), + } + .into(); + assert!(!DirectoryNamespace::is_manifest_table_absent_error( + &throttle + )); + let internal: Error = NamespaceError::Internal { + message: "boom".to_string(), + } + .into(); + assert!(!DirectoryNamespace::is_manifest_table_absent_error( + &internal + )); + } + + #[test] + fn test_map_open_error() { + let not_found = || NamespaceError::TableNotFound { + message: "table at 'x' not found: ...".to_string(), + }; + + let throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "after 3 retries, max_retries: 3 - 503 ServerBusy".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error(throttle, not_found())), + ErrorCode::Throttling + ); + + let generic_io: Error = ObjectStoreError::Generic { + store: "test", + source: "connection reset".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error(generic_io, not_found())), + ErrorCode::Internal + ); + + let io_not_found: Error = ObjectStoreError::NotFound { + path: "x".to_string(), + source: "missing".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + io_not_found, + not_found() + )), + ErrorCode::TableNotFound + ); + + let dataset_not_found = Error::dataset_not_found("x".to_string(), "missing".into()); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + dataset_not_found, + not_found() + )), + ErrorCode::TableNotFound + ); + + // RefNotFound is not an IO error, so it is not reclassified as a storage error. + let ref_not_found = Error::RefNotFound { + message: "branch 'b' does not exist".to_string(), + }; + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + ref_not_found, + not_found() + )), + ErrorCode::TableNotFound + ); + + // The caller's not-found variant is honored, but a throttle still propagates. + let version_miss = Error::RefNotFound { + message: "version 5 does not exist".to_string(), + }; + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + version_miss, + NamespaceError::TableVersionNotFound { + message: "version 5 not found".to_string(), + }, + )), + ErrorCode::TableVersionNotFound + ); + let version_throttle: Error = ObjectStoreError::Generic { + store: "test", + source: "after 3 retries, max_retries: 3 - 503 ServerBusy".into(), + } + .into(); + assert_eq!( + mutation_error_code(DirectoryNamespace::map_open_error( + version_throttle, + NamespaceError::TableVersionNotFound { + message: "version 5 not found".to_string(), + }, + )), + ErrorCode::Throttling + ); } /// Helper to create test IPC data from a schema @@ -5574,6 +7308,43 @@ mod tests { create_ipc_data_from_batches(schema, vec![batch]) } + async fn create_legacy_manifest_without_primary_key_metadata(root: &str) { + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use arrow::record_batch::{RecordBatch, RecordBatchIterator}; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("object_id", DataType::Utf8, false), + Field::new("object_type", DataType::Utf8, false), + Field::new("location", DataType::Utf8, true), + Field::new("metadata", DataType::Utf8, true), + Field::new( + "base_objects", + DataType::List(Arc::new(Field::new("object_id", DataType::Utf8, true))), + true, + ), + ])); + let batch = RecordBatch::new_empty(schema.clone()); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(Box::new(reader), &format!("{}/__manifest", root), None) + .await + .unwrap(); + } + + async fn manifest_has_primary_key_metadata(root: &str) -> bool { + let dataset = Dataset::open(&format!("{}/__manifest", root)) + .await + .unwrap(); + dataset + .schema() + .field("object_id") + .map(|field| { + field + .metadata + .contains_key(lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION) + }) + .unwrap_or(false) + } + fn create_vector_table_ipc_data() -> Vec { use arrow::array::{FixedSizeListArray, Float32Array, Int32Array}; use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; @@ -8233,6 +10004,139 @@ mod tests { assert!(response.location.is_some()); } + /// Regression: a connection built before `__manifest` exists must still + /// resolve a child-namespaced table that a *different* connection registers + /// afterwards. Phalanx caches one DirectoryNamespace per db and the first op + /// on a fresh db is usually a read, so without a self-healing read path the + /// cached reader pins an empty manifest cell and every describe/exists/list + /// on the table reports "not found" forever -- even though `create_table` + /// reports it already exists. This is the geneva `__system$geneva_jobs` + /// open->create->open livelock. + #[tokio::test] + async fn test_read_self_heals_after_manifest_created_by_other_connection() { + let temp_dir = TempStdDir::default(); + let root = temp_dir.to_str().unwrap(); + + // Reader is built while no `__manifest` exists yet -> its read cell is + // empty and, before the fix, stays empty forever. + let reader = DirectoryNamespaceBuilder::new(root).build().await.unwrap(); + + // A *separate* connection creates the child namespace + table, which + // lazily creates `__manifest` and registers the entry. + let writer = DirectoryNamespaceBuilder::new(root).build().await.unwrap(); + let mut create_ns_req = CreateNamespaceRequest::new(); + create_ns_req.id = Some(vec!["test_ns".to_string()]); + writer.create_namespace(create_ns_req).await.unwrap(); + let ipc_data = create_test_ipc_data(&create_test_schema()); + let mut create_table_req = CreateTableRequest::new(); + create_table_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + writer + .create_table(create_table_req, bytes::Bytes::from(ipc_data)) + .await + .unwrap(); + + // The reader, though built before the manifest existed, must now resolve + // the table on every read path (was TableNotFound before the fix). + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + let resp = reader + .describe_table(describe_req) + .await + .expect("describe_table must resolve a table registered after build"); + assert!(resp.location.is_some()); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["test_ns".to_string(), "table1".to_string()]); + reader + .table_exists(exists_req) + .await + .expect("table_exists must resolve a table registered after build"); + + let list_req = ListTablesRequest { + id: Some(vec!["test_ns".to_string()]), + ..Default::default() + }; + let tables = reader.list_tables(list_req).await.unwrap().tables; + assert_eq!(tables, vec!["table1".to_string()]); + } + + /// Migration mode promises manifest-first lookup even at the root, so a + /// reader built before `__manifest` existed must still resolve a + /// manifest-only alias (`registered_table` -> `external_table.lance`) that + /// dir-listing cannot produce. Before the read path self-healed in migration + /// mode, the root gate bypassed the manifest probe and fell back to + /// dir-listing, which sees `external_table` but never `registered_table` -> + /// permanent TableNotFound for every registration made after build. + #[tokio::test] + async fn test_migration_root_read_self_heals_registered_alias() { + use lance_namespace::models::RegisterTableRequest; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + + // Reader built while the root is empty -> no `__manifest`, read cell + // empty (and, before the fix, frozen empty forever). + let reader = DirectoryNamespaceBuilder::new(temp_path) + .dir_listing_enabled(true) + .dir_listing_to_manifest_migration_enabled(true) + .build() + .await + .unwrap(); + + // A separate connection writes an external dataset and registers it in + // the manifest under a *different* logical name -- an alias dir-listing + // cannot resolve. This is what lazily creates `__manifest`. + let writer = DirectoryNamespaceBuilder::new(temp_path) + .dir_listing_enabled(true) + .dir_listing_to_manifest_migration_enabled(true) + .build() + .await + .unwrap(); + let ipc_data = create_test_ipc_data(&create_test_schema()); + let table_uri = format!("{}/external_table.lance", temp_path); + let cursor = Cursor::new(ipc_data); + let stream_reader = StreamReader::try_new(cursor, None).unwrap(); + let batches: Vec<_> = stream_reader + .collect::, _>>() + .unwrap(); + let schema = batches[0].schema(); + let batch_results: Vec<_> = batches.into_iter().map(Ok).collect(); + let batch_reader = RecordBatchIterator::new(batch_results, schema); + Dataset::write(Box::new(batch_reader), &table_uri, None) + .await + .unwrap(); + let mut register_req = RegisterTableRequest::new("external_table.lance".to_string()); + register_req.id = Some(vec!["registered_table".to_string()]); + writer.register_table(register_req).await.unwrap(); + + // The reader, built before `__manifest` existed, must now resolve the + // manifest-only alias on every read path. + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["registered_table".to_string()]); + reader + .describe_table(describe_req) + .await + .expect("describe_table must resolve a manifest alias registered after build"); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["registered_table".to_string()]); + reader + .table_exists(exists_req) + .await + .expect("table_exists must resolve a manifest alias registered after build"); + + let list_req = ListTablesRequest { + id: Some(vec![]), + ..Default::default() + }; + let tables = reader.list_tables(list_req).await.unwrap().tables; + assert!( + tables.contains(&"registered_table".to_string()), + "list_tables must include the manifest alias registered after build, got {:?}", + tables + ); + } + #[tokio::test] async fn test_multiple_tables_in_child_namespace() { let (namespace, _temp_dir) = create_test_namespace().await; @@ -9477,7 +11381,7 @@ mod tests { let mut merge_req = MergeInsertIntoTableRequest::new(); merge_req.id = Some(vec!["test_table".to_string()]); - merge_req.on = Some("id".to_string()); + merge_req.on = Some(vec!["id".to_string()]); let response = namespace .merge_insert_into_table( merge_req, @@ -9528,31 +11432,269 @@ mod tests { let mut merge_req = MergeInsertIntoTableRequest::new(); merge_req.id = Some(vec!["test_table".to_string()]); - merge_req.on = Some("id".to_string()); + merge_req.on = Some(vec!["id".to_string()]); let response = namespace .merge_insert_into_table( merge_req, bytes::Bytes::from(create_non_empty_test_ipc_data()), ) .await - .unwrap(); - - assert_eq!(response.num_inserted_rows, Some(2)); - assert_eq!(response.num_updated_rows, Some(0)); - - let mut describe_req = DescribeTableRequest::new(); - describe_req.id = Some(vec!["test_table".to_string()]); - describe_req.load_detailed_metadata = Some(true); - let describe_response = namespace.describe_table(describe_req).await.unwrap(); - assert_eq!(describe_response.is_only_declared, Some(false)); - assert_eq!(describe_response.version, Some(1)); + .unwrap(); + + assert_eq!(response.num_inserted_rows, Some(2)); + assert_eq!(response.num_updated_rows, Some(0)); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["test_table".to_string()]); + describe_req.load_detailed_metadata = Some(true); + let describe_response = namespace.describe_table(describe_req).await.unwrap(); + assert_eq!(describe_response.is_only_declared, Some(false)); + assert_eq!(describe_response.version, Some(1)); + + let mut list_req = ListTablesRequest::new(); + list_req.id = Some(vec![]); + list_req.include_declared = Some(false); + assert_eq!( + namespace.list_tables(list_req).await.unwrap().tables, + vec!["test_table".to_string()] + ); + } + + /// `(region, id, value)` rows, for merge inserts keyed on `region` + `id`. + /// + /// `region` is nullable so tests can cover a NULL in one half of the key. + fn create_composite_key_ipc_data(rows: &[(Option<&str>, i32, &str)]) -> Vec { + use arrow::array::{Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema as ArrowSchema}; + use arrow::record_batch::RecordBatch; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("region", DataType::Utf8, true), + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from_iter( + rows.iter().map(|(region, _, _)| *region), + )), + Arc::new(Int32Array::from_iter_values( + rows.iter().map(|(_, id, _)| *id), + )), + Arc::new(StringArray::from_iter_values( + rows.iter().map(|(_, _, value)| *value), + )), + ], + ) + .unwrap(); + create_ipc_data_from_batches(schema, vec![batch]) + } + + /// `test_table`'s rows as `(region, id, value)`, sorted for a stable comparison. + async fn read_composite_key_rows(root: &str) -> Vec<(Option, i32, String)> { + use arrow::array::Array; + + let dataset = Dataset::open(&format!("{}/test_table.lance", root)) + .await + .unwrap(); + let batch = dataset.scan().try_into_batch().await.unwrap(); + let regions = batch["region"] + .as_any() + .downcast_ref::() + .unwrap(); + let ids = batch["id"] + .as_any() + .downcast_ref::() + .unwrap(); + let values = batch["value"] + .as_any() + .downcast_ref::() + .unwrap(); + + let mut rows: Vec<_> = (0..batch.num_rows()) + .map(|row| { + ( + regions + .is_valid(row) + .then(|| regions.value(row).to_string()), + ids.value(row), + values.value(row).to_string(), + ) + }) + .collect(); + rows.sort_unstable(); + rows + } + + #[tokio::test] + async fn test_merge_insert_matches_on_every_column_of_a_composite_key() { + use lance_namespace::models::{DeclareTableRequest, MergeInsertIntoTableRequest}; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(false) + .build() + .await + .unwrap(); + + let mut declare_req = DeclareTableRequest::new(); + declare_req.id = Some(vec!["test_table".to_string()]); + namespace.declare_table(declare_req).await.unwrap(); + + let seed = create_composite_key_ipc_data(&[ + (Some("us"), 1, "a"), + (Some("us"), 2, "b"), + (Some("eu"), 1, "c"), + ]); + let mut merge_req = MergeInsertIntoTableRequest::new(); + merge_req.id = Some(vec!["test_table".to_string()]); + merge_req.on = Some(vec!["region".to_string(), "id".to_string()]); + namespace + .merge_insert_into_table(merge_req, bytes::Bytes::from(seed)) + .await + .unwrap(); + + // ("us", 1) matches an existing row; ("eu", 2) matches nothing even though a row + // with region "eu" and a row with id 2 both exist. + let mut merge_req = MergeInsertIntoTableRequest::new(); + merge_req.id = Some(vec!["test_table".to_string()]); + merge_req.on = Some(vec!["region".to_string(), "id".to_string()]); + merge_req.when_matched_update_all = Some(true); + let response = namespace + .merge_insert_into_table( + merge_req, + bytes::Bytes::from(create_composite_key_ipc_data(&[ + (Some("us"), 1, "updated"), + (Some("eu"), 2, "inserted"), + ])), + ) + .await + .unwrap(); + + assert_eq!(response.num_updated_rows, Some(1)); + assert_eq!(response.num_inserted_rows, Some(1)); + + assert_eq!( + read_composite_key_rows(temp_path).await, + vec![ + // ("eu", 1) keeps its value: matching on `id` alone would have clobbered it. + (Some("eu".into()), 1, "c".into()), + (Some("eu".into()), 2, "inserted".into()), + (Some("us".into()), 1, "updated".into()), + (Some("us".into()), 2, "b".into()), + ] + ); + } + + /// Core switches NULL join semantics on the arity of the match key + /// (`merge_insert.rs`, `NullEquality`): a single-column key treats NULL as equal to + /// NULL, while a composite key uses standard SQL equality, under which it is not. So + /// adding a second key column changes whether NULL-keyed rows match at all. + #[tokio::test] + async fn test_merge_insert_composite_key_never_matches_a_null_key_column() { + use lance_namespace::models::{DeclareTableRequest, MergeInsertIntoTableRequest}; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(false) + .build() + .await + .unwrap(); + + let mut declare_req = DeclareTableRequest::new(); + declare_req.id = Some(vec!["test_table".to_string()]); + namespace.declare_table(declare_req).await.unwrap(); + + let mut merge_req = MergeInsertIntoTableRequest::new(); + merge_req.id = Some(vec!["test_table".to_string()]); + merge_req.on = Some(vec!["region".to_string(), "id".to_string()]); + namespace + .merge_insert_into_table( + merge_req, + bytes::Bytes::from(create_composite_key_ipc_data(&[ + (None, 1, "seeded"), + (Some("us"), 1, "us-seeded"), + ])), + ) + .await + .unwrap(); + + let mut merge_req = MergeInsertIntoTableRequest::new(); + merge_req.id = Some(vec!["test_table".to_string()]); + merge_req.on = Some(vec!["region".to_string(), "id".to_string()]); + merge_req.when_matched_update_all = Some(true); + let response = namespace + .merge_insert_into_table( + merge_req, + bytes::Bytes::from(create_composite_key_ipc_data(&[(None, 1, "not-a-match")])), + ) + .await + .unwrap(); + + // The incoming row is byte-identical to the seeded one, and still does not match. + assert_eq!(response.num_updated_rows, Some(0)); + assert_eq!(response.num_inserted_rows, Some(1)); + + assert_eq!( + read_composite_key_rows(temp_path).await, + vec![ + (None, 1, "not-a-match".into()), + (None, 1, "seeded".into()), + (Some("us".into()), 1, "us-seeded".into()), + ] + ); + } + + #[rstest::rstest] + #[case::missing(None, "'on' field is required")] + #[case::empty(Some(vec![]), "must name at least one column")] + #[case::duplicate( + Some(vec!["region".to_string(), "region".to_string()]), + "names column 'region' more than once" + )] + #[tokio::test] + async fn test_merge_insert_rejects_an_invalid_on_key( + #[case] on: Option>, + #[case] expected_message: &str, + ) { + use lance_namespace::models::{DeclareTableRequest, MergeInsertIntoTableRequest}; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(false) + .build() + .await + .unwrap(); + + let mut declare_req = DeclareTableRequest::new(); + declare_req.id = Some(vec!["test_table".to_string()]); + namespace.declare_table(declare_req).await.unwrap(); + + let mut merge_req = MergeInsertIntoTableRequest::new(); + merge_req.id = Some(vec!["test_table".to_string()]); + merge_req.on = on; + let error = namespace + .merge_insert_into_table( + merge_req, + bytes::Bytes::from(create_composite_key_ipc_data(&[(Some("us"), 1, "a")])), + ) + .await + .unwrap_err(); - let mut list_req = ListTablesRequest::new(); - list_req.id = Some(vec![]); - list_req.include_declared = Some(false); - assert_eq!( - namespace.list_tables(list_req).await.unwrap().tables, - vec!["test_table".to_string()] + let lance_core::Error::Namespace { source, .. } = &error else { + panic!("expected a Namespace error, got: {}", error); + }; + let ns_err = source + .downcast_ref::() + .expect("expected a NamespaceError source"); + assert_eq!(ns_err.code(), lance_namespace::ErrorCode::InvalidInput); + assert!( + error.to_string().contains(expected_message), + "unexpected error message: {error}" ); } @@ -9633,6 +11775,45 @@ mod tests { ); } + #[tokio::test] + async fn test_declare_table_with_manifest_marker_already_exists() { + // Pre-existing .lance-reserved (concurrent/incomplete declare) must map to + // TableAlreadyExists, not Internal. + use lance_namespace::error::ErrorCode; + use lance_namespace::models::DeclareTableRequest; + + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + + let table_dir = temp_dir.join("test_table.lance"); + std::fs::create_dir_all(&table_dir).unwrap(); + std::fs::write(table_dir.join(".lance-reserved"), b"reserved").unwrap(); + + let mut declare_req = DeclareTableRequest::new(); + declare_req.id = Some(vec!["test_table".to_string()]); + let err = namespace + .declare_table(declare_req) + .await + .expect_err("declare with existing marker must fail"); + let msg = err.to_string(); + assert!( + msg.contains("already exists") || msg.contains("TableAlreadyExists"), + "expected TableAlreadyExists, got: {msg}" + ); + assert_eq!( + mutation_error_code(err), + ErrorCode::TableAlreadyExists, + "expected TableAlreadyExists error code" + ); + } + #[tokio::test] async fn test_declare_table_when_table_exists() { use lance_namespace::models::DeclareTableRequest; @@ -9932,7 +12113,7 @@ mod tests { .unwrap(); // Table status should show exists=true, is_deregistered=false - let status = namespace.check_table_status("test_table").await; + let status = namespace.check_table_status("test_table").await.unwrap(); assert!(status.exists); assert!(!status.is_deregistered); assert!(!status.has_reserved_file); @@ -10319,12 +12500,257 @@ mod tests { .await .unwrap(); - // Should return version 3 as it's the latest - assert_eq!(describe_resp.version.version, 3); + // Should return version 3 as it's the latest + assert_eq!(describe_resp.version.version, 3); + } + + #[tokio::test] + async fn test_create_table_version() { + use futures::TryStreamExt; + use lance::dataset::builder::DatasetBuilder; + use lance_namespace::models::CreateTableVersionRequest; + + let temp_dir = TempStrDir::default(); + let temp_path: &str = &temp_dir; + + let namespace: Arc = Arc::new( + DirectoryNamespaceBuilder::new(temp_path) + .table_version_tracking_enabled(true) + .build() + .await + .unwrap(), + ); + + // Create a table + let schema = create_test_schema(); + let ipc_data = create_test_ipc_data(&schema); + let mut create_req = CreateTableRequest::new(); + create_req.id = Some(vec!["test_table".to_string()]); + namespace + .create_table(create_req, bytes::Bytes::from(ipc_data)) + .await + .unwrap(); + + // Open the dataset using from_namespace to get proper object_store and paths + let table_id = vec!["test_table".to_string()]; + let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone()) + .await + .unwrap() + .load() + .await + .unwrap(); + + // Use dataset's object_store to find and copy the manifest + let versions_path = dataset.versions_dir(); + let manifest_metas: Vec<_> = dataset + .object_store(None) + .await + .unwrap() + .inner + .list(Some(&versions_path)) + .try_collect() + .await + .unwrap(); + + let manifest_meta = manifest_metas + .iter() + .find(|m| { + m.location + .filename() + .map(|f| f.ends_with(".manifest")) + .unwrap_or(false) + }) + .expect("No manifest file found"); + + // Read the existing manifest data + let manifest_data = dataset + .object_store(None) + .await + .unwrap() + .inner + .get(&manifest_meta.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + + // Write to a staging location using the dataset's object_store + let staging_path = dataset.versions_dir().join("staging_manifest"); + dataset + .object_store(None) + .await + .unwrap() + .inner + .put(&staging_path, manifest_data.into()) + .await + .unwrap(); + + // Create version 2 from staging manifest + // Use the same naming scheme as the existing dataset (V2) + let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); + create_version_req.id = Some(table_id.clone()); + create_version_req.naming_scheme = Some("V2".to_string()); + + let result = namespace.create_table_version(create_version_req).await; + assert!( + result.is_ok(), + "create_table_version should succeed: {:?}", + result + ); + + // Verify version 2 was created at the path returned in the response + let response = result.unwrap(); + let version_info = response + .version + .expect("response should contain version info"); + let version_2_path = Path::parse(&version_info.manifest_path).unwrap(); + let head_result = dataset + .object_store(None) + .await + .unwrap() + .inner + .head(&version_2_path) + .await; + assert!( + head_result.is_ok(), + "Version 2 manifest should exist at {}", + version_2_path + ); + + // Verify the staging file has been deleted + let staging_head_result = dataset + .object_store(None) + .await + .unwrap() + .inner + .head(&staging_path) + .await; + assert!( + staging_head_result.is_err(), + "Staging manifest should have been deleted after create_table_version" + ); + } + + #[tokio::test] + async fn test_create_table_version_idempotent() { + // A network retry of create_table_version with the same staging content + // must succeed (not ConcurrentModification) once the version is published. + use futures::TryStreamExt; + use lance::dataset::builder::DatasetBuilder; + use lance_namespace::models::CreateTableVersionRequest; + + let temp_dir = TempStrDir::default(); + let temp_path: &str = &temp_dir; + + let namespace: Arc = Arc::new( + DirectoryNamespaceBuilder::new(temp_path) + .table_version_tracking_enabled(true) + .build() + .await + .unwrap(), + ); + + let schema = create_test_schema(); + let ipc_data = create_test_ipc_data(&schema); + let mut create_req = CreateTableRequest::new(); + create_req.id = Some(vec!["test_table".to_string()]); + namespace + .create_table(create_req, bytes::Bytes::from(ipc_data)) + .await + .unwrap(); + + let table_id = vec!["test_table".to_string()]; + let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone()) + .await + .unwrap() + .load() + .await + .unwrap(); + + let versions_path = dataset.versions_dir(); + let manifest_metas: Vec<_> = dataset + .object_store(None) + .await + .unwrap() + .inner + .list(Some(&versions_path)) + .try_collect() + .await + .unwrap(); + + let manifest_meta = manifest_metas + .iter() + .find(|m| { + m.location + .filename() + .map(|f| f.ends_with(".manifest")) + .unwrap_or(false) + }) + .expect("No manifest file found"); + + let manifest_data = dataset + .object_store(None) + .await + .unwrap() + .inner + .get(&manifest_meta.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + + let staging_path = dataset.versions_dir().join("staging_manifest"); + dataset + .object_store(None) + .await + .unwrap() + .inner + .put(&staging_path, manifest_data.clone().into()) + .await + .unwrap(); + + let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); + create_version_req.id = Some(table_id.clone()); + create_version_req.naming_scheme = Some("V2".to_string()); + let first = namespace + .create_table_version(create_version_req) + .await + .expect("first create_table_version should succeed"); + + // Re-stage identical bytes (simulates Lance commit retry rewriting staging). + let retry_staging = dataset.versions_dir().join("staging_manifest_retry"); + dataset + .object_store(None) + .await + .unwrap() + .inner + .put(&retry_staging, manifest_data.into()) + .await + .unwrap(); + + let mut retry_req = CreateTableVersionRequest::new(2, retry_staging.to_string()); + retry_req.id = Some(table_id.clone()); + retry_req.naming_scheme = Some("V2".to_string()); + let second = namespace + .create_table_version(retry_req) + .await + .expect("idempotent retry must succeed"); + + assert_eq!( + first.version.as_ref().map(|v| v.version), + second.version.as_ref().map(|v| v.version) + ); + assert_eq!( + first.version.as_ref().map(|v| &v.manifest_path), + second.version.as_ref().map(|v| &v.manifest_path) + ); } #[tokio::test] - async fn test_create_table_version() { + async fn test_create_table_version_conflict() { + // Same version with different content must fail ConcurrentModification. use futures::TryStreamExt; use lance::dataset::builder::DatasetBuilder; use lance_namespace::models::CreateTableVersionRequest; @@ -10405,56 +12831,75 @@ mod tests { .await .unwrap(); - // Create version 2 from staging manifest - // Use the same naming scheme as the existing dataset (V2) + // First create version 2 (should succeed) let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); create_version_req.id = Some(table_id.clone()); create_version_req.naming_scheme = Some("V2".to_string()); - - let result = namespace.create_table_version(create_version_req).await; + let first_result = namespace.create_table_version(create_version_req).await; assert!( - result.is_ok(), - "create_table_version should succeed: {:?}", - result + first_result.is_ok(), + "First create_table_version for version 2 should succeed: {:?}", + first_result ); - // Verify version 2 was created at the path returned in the response - let response = result.unwrap(); - let version_info = response - .version - .expect("response should contain version info"); - let version_2_path = Path::parse(&version_info.manifest_path).unwrap(); - let head_result = dataset + // Get the path from the response for verification + let version_2_path = Path::parse( + &first_result + .unwrap() + .version + .expect("response should contain version info") + .manifest_path, + ) + .unwrap(); + + // Different content for the same version number must conflict. + let conflict_staging = dataset.versions_dir().join("staging_manifest_conflict"); + dataset .object_store(None) .await .unwrap() .inner - .head(&version_2_path) - .await; + .put( + &conflict_staging, + bytes::Bytes::from_static(b"not-a-real-manifest").into(), + ) + .await + .unwrap(); + + let mut create_version_req = + CreateTableVersionRequest::new(2, conflict_staging.to_string()); + create_version_req.id = Some(table_id.clone()); + create_version_req.naming_scheme = Some("V2".to_string()); + + let result = namespace.create_table_version(create_version_req).await; assert!( - head_result.is_ok(), - "Version 2 manifest should exist at {}", - version_2_path + result.is_err(), + "create_table_version should fail for existing version with different content" + ); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("already exists") || err.contains("ConcurrentModification"), + "expected ConcurrentModification, got: {err}" ); - // Verify the staging file has been deleted - let staging_head_result = dataset + // Verify version 2 still exists using the dataset's object_store + let head_result = dataset .object_store(None) .await .unwrap() .inner - .head(&staging_path) + .head(&version_2_path) .await; assert!( - staging_head_result.is_err(), - "Staging manifest should have been deleted after create_table_version" + head_result.is_ok(), + "Version 2 manifest should still exist at {}", + version_2_path ); } #[tokio::test] - async fn test_create_table_version_conflict() { - // create_table_version should fail if the version already exists. - // Each version always writes to a new file location. + async fn test_create_table_version_cas_rejects_gap() { + // Strict CAS: version must be latest+1; skipping ahead is ConcurrentModification. use futures::TryStreamExt; use lance::dataset::builder::DatasetBuilder; use lance_namespace::models::CreateTableVersionRequest; @@ -10470,7 +12915,6 @@ mod tests { .unwrap(), ); - // Create a table let schema = create_test_schema(); let ipc_data = create_test_ipc_data(&schema); let mut create_req = CreateTableRequest::new(); @@ -10480,7 +12924,6 @@ mod tests { .await .unwrap(); - // Open the dataset using from_namespace to get proper object_store and paths let table_id = vec!["test_table".to_string()]; let dataset = DatasetBuilder::from_namespace(namespace.clone(), table_id.clone()) .await @@ -10489,7 +12932,6 @@ mod tests { .await .unwrap(); - // Use dataset's object_store to find and copy the manifest let versions_path = dataset.versions_dir(); let manifest_metas: Vec<_> = dataset .object_store(None) @@ -10500,7 +12942,6 @@ mod tests { .try_collect() .await .unwrap(); - let manifest_meta = manifest_metas .iter() .find(|m| { @@ -10510,8 +12951,6 @@ mod tests { .unwrap_or(false) }) .expect("No manifest file found"); - - // Read the existing manifest data let manifest_data = dataset .object_store(None) .await @@ -10524,8 +12963,7 @@ mod tests { .await .unwrap(); - // Write to a staging location using the dataset's object_store - let staging_path = dataset.versions_dir().join("staging_manifest"); + let staging_path = dataset.versions_dir().join("staging_gap"); dataset .object_store(None) .await @@ -10535,51 +12973,148 @@ mod tests { .await .unwrap(); - // First create version 2 (should succeed) - let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); - create_version_req.id = Some(table_id.clone()); - create_version_req.naming_scheme = Some("V2".to_string()); - let first_result = namespace.create_table_version(create_version_req).await; + // After create_table, latest is 1; requesting 5 must fail CAS. + let mut req = CreateTableVersionRequest::new(5, staging_path.to_string()); + req.id = Some(table_id); + req.naming_scheme = Some("V2".to_string()); + let err = namespace + .create_table_version(req) + .await + .expect_err("gap create must fail CAS"); + let msg = err.to_string(); assert!( - first_result.is_ok(), - "First create_table_version for version 2 should succeed: {:?}", - first_result + msg.contains("CAS") || msg.contains("ConcurrentModification"), + "expected CAS ConcurrentModification, got: {msg}" ); + } - // Get the path from the response for verification - let version_2_path = Path::parse( - &first_result - .unwrap() - .version - .expect("response should contain version info") - .manifest_path, - ) - .unwrap(); + #[tokio::test] + async fn test_create_table_version_branch_cas_requires_parent_version() { + // Empty branch chain with BranchContents must bootstrap at parent_version, + // not an arbitrary version (e.g. 1 when forked from v2). + use futures::TryStreamExt; + use lance_namespace::models::CreateTableVersionRequest; - // Create version 2 again (should fail - conflict) - let mut create_version_req = CreateTableVersionRequest::new(2, staging_path.to_string()); - create_version_req.id = Some(table_id.clone()); - create_version_req.naming_scheme = Some("V2".to_string()); + let (namespace, _temp_dir) = create_test_namespace().await; + create_scalar_table(&namespace, "users").await; + let main_uri = open_dataset(&namespace, "users").await.uri().to_string(); + append_scalar_version(&main_uri, 10).await; // main -> v2 - let result = namespace.create_table_version(create_version_req).await; - assert!( - result.is_err(), - "create_table_version should fail for existing version" + let mut main = open_dataset(&namespace, "users").await; + let fork_version = main.version().version; + assert_eq!(fork_version, 2); + let branch_uri = main + .create_branch("exp", fork_version, None) + .await + .unwrap() + .uri() + .to_string(); + + let branch_ds = Dataset::open(&branch_uri).await.unwrap(); + let versions_dir = branch_ds.versions_dir(); + let store = branch_ds.object_store(None).await.unwrap(); + let manifests: Vec<_> = store + .inner + .list(Some(&versions_dir)) + .try_collect() + .await + .unwrap(); + for meta in &manifests { + if meta + .location + .filename() + .is_some_and(|f| f.ends_with(".manifest")) + { + store.inner.delete(&meta.location).await.unwrap(); + } + } + // Confirm the branch object-store chain is empty (do not open the dataset: + // with no manifests, Dataset::open would fail). + let remaining_manifests = store + .inner + .list(Some(&versions_dir)) + .try_collect::>() + .await + .unwrap() + .into_iter() + .filter(|m| { + m.location + .filename() + .is_some_and(|f| f.ends_with(".manifest")) + }) + .count(); + assert_eq!( + remaining_manifests, 0, + "branch version chain should be empty after deleting manifests" ); - // Verify version 2 still exists using the dataset's object_store - let head_result = dataset - .object_store(None) + // Stage bytes from a main manifest. + let main_ds = open_dataset(&namespace, "users").await; + let main_versions = main_ds.versions_dir(); + let main_store = main_ds.object_store(None).await.unwrap(); + let source_meta = main_store + .inner + .list(Some(&main_versions)) + .try_collect::>() .await .unwrap() + .into_iter() + .find(|m| { + m.location + .filename() + .is_some_and(|f| f.ends_with(".manifest")) + }) + .expect("main should have a manifest"); + let source_bytes = main_store .inner - .head(&version_2_path) - .await; + .get(&source_meta.location) + .await + .unwrap() + .bytes() + .await + .unwrap(); + + let staging_wrong = versions_dir.clone().join("staging_wrong"); + store + .inner + .put(&staging_wrong, source_bytes.clone().into()) + .await + .unwrap(); + let err = namespace + .create_table_version(CreateTableVersionRequest { + id: Some(vec!["users".to_string()]), + version: 1, + manifest_path: staging_wrong.to_string(), + naming_scheme: Some("V2".to_string()), + branch: Some("exp".to_string()), + ..Default::default() + }) + .await + .expect_err("bootstrap at v1 must fail when parent_version is 2"); + let msg = err.to_string(); assert!( - head_result.is_ok(), - "Version 2 manifest should still exist at {}", - version_2_path + msg.contains("CAS") || msg.contains("ConcurrentModification"), + "expected CAS ConcurrentModification, got: {msg}" ); + + let staging_ok = versions_dir.join("staging_ok"); + store + .inner + .put(&staging_ok, source_bytes.into()) + .await + .unwrap(); + let resp = namespace + .create_table_version(CreateTableVersionRequest { + id: Some(vec!["users".to_string()]), + version: 2, + manifest_path: staging_ok.to_string(), + naming_scheme: Some("V2".to_string()), + branch: Some("exp".to_string()), + ..Default::default() + }) + .await + .expect("bootstrap at parent_version must succeed"); + assert_eq!(resp.version.as_ref().map(|v| v.version), Some(2)); } #[tokio::test] @@ -12256,7 +14791,7 @@ mod tests { &plan_str, &[ "ProjectionExec: expr=[id@0 as id, name@2 as name", - "Take: columns=\"id, _rowid, (name)\"", + "projection=[name], source=stream(_rowid)", "LanceRead: uri=", "projection=[id]", "row_id=true, row_addr=false", @@ -12353,9 +14888,7 @@ mod tests { "AnalyzeExec verbose=true", "ProjectionExec: elapsed=", "expr=[id@0 as id, name@2 as name", - "Take: elapsed=", - "columns=\"id, _rowid, (name)\"", - "CoalesceBatchesExec: elapsed=", + "projection=[name], source=stream(_rowid)", "LanceRead: elapsed=", "projection=[id]", "row_id=true, row_addr=false", @@ -12435,6 +14968,109 @@ mod tests { ); } + #[tokio::test] + async fn test_build_and_root_reads_do_not_create_manifest() { + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + let manifest_path = std::path::Path::new(temp_path).join("__manifest"); + + let dir_only_ns = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(false) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + create_scalar_table(&dir_only_ns, "catalog").await; + assert!(!manifest_path.exists()); + + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + assert!(!manifest_path.exists()); + + let mut exists_req = TableExistsRequest::new(); + exists_req.id = Some(vec!["catalog".to_string()]); + namespace.table_exists(exists_req).await.unwrap(); + assert!(!manifest_path.exists()); + + let mut describe_req = DescribeTableRequest::new(); + describe_req.id = Some(vec!["catalog".to_string()]); + namespace.describe_table(describe_req).await.unwrap(); + assert!(!manifest_path.exists()); + + let list_response = namespace + .list_tables(ListTablesRequest { + id: Some(vec![]), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(list_response.tables, vec!["catalog".to_string()]); + assert!(!manifest_path.exists()); + + let mut list_namespaces_req = ListNamespacesRequest::new(); + list_namespaces_req.id = Some(vec!["workspace".to_string()]); + let err = namespace + .list_namespaces(list_namespaces_req) + .await + .unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let err = namespace + .list_tables(ListTablesRequest { + id: Some(vec!["workspace".to_string()]), + ..Default::default() + }) + .await + .unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let mut child_describe_req = DescribeTableRequest::new(); + child_describe_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]); + let err = namespace + .describe_table(child_describe_req) + .await + .unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let mut child_exists_req = TableExistsRequest::new(); + child_exists_req.id = Some(vec!["workspace".to_string(), "catalog".to_string()]); + let err = namespace.table_exists(child_exists_req).await.unwrap_err(); + assert!(err.to_string().contains("__manifest")); + assert!(!manifest_path.exists()); + + let mut create_ns_req = CreateNamespaceRequest::new(); + create_ns_req.id = Some(vec!["workspace".to_string()]); + namespace.create_namespace(create_ns_req).await.unwrap(); + assert!(manifest_path.exists()); + } + + #[tokio::test] + async fn test_migrate_updates_read_opened_legacy_manifest() { + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + create_legacy_manifest_without_primary_key_metadata(temp_path).await; + assert!(!manifest_has_primary_key_metadata(temp_path).await); + + let namespace = DirectoryNamespaceBuilder::new(temp_path) + .manifest_enabled(true) + .dir_listing_enabled(true) + .build() + .await + .unwrap(); + assert!(!manifest_has_primary_key_metadata(temp_path).await); + + let migrated = namespace.migrate().await.unwrap(); + assert_eq!(migrated, 0); + assert!(manifest_has_primary_key_metadata(temp_path).await); + } + #[tokio::test] async fn test_describe_declared_table_checks_versions_only_when_requested() { let temp_dir = TempStdDir::default(); @@ -12518,9 +15154,13 @@ mod tests { .await .unwrap(); - // table_exists first checks __manifest (which on local FS uses the - // version hint and does no list call), then falls back to the table - // directory (one list_with_delimiter on test_table.lance). + // In migration mode the manifest is authoritative, so table_exists first + // probes __manifest via ensure_read_manifest() to self-heal any + // manifest-registered aliases (why the gate now admits this mode). Here + // the table is dir-only and __manifest does not exist yet, so that probe + // costs one list to confirm absence; the table-directory fallback is the + // second. (When __manifest exists the probe uses the version hint and + // adds no list, so a real self-heal is free.) listing_count.store(0, Ordering::SeqCst); let mut exists_req = TableExistsRequest::new(); @@ -12529,13 +15169,14 @@ mod tests { let count = listing_count.load(Ordering::SeqCst); assert_eq!( - count, 1, - "Expected exactly 1 listing call for table_exists with migration mode \ - (table directory fallback; manifest reload uses the version hint), but got {}", + count, 2, + "Expected 2 listing calls for table_exists with migration mode \ + (absent-__manifest probe + table directory fallback), but got {}", count ); - // describe_table follows the same path when the table is not yet registered in __manifest. + // describe_table follows the same path: an ensure_read_manifest() probe + // of the (absent) __manifest, then the table-directory fallback. listing_count.store(0, Ordering::SeqCst); let mut describe_req = DescribeTableRequest::new(); @@ -12544,9 +15185,9 @@ mod tests { let count = listing_count.load(Ordering::SeqCst); assert_eq!( - count, 1, - "Expected exactly 1 listing call for describe_table with migration mode \ - (table directory fallback; manifest reload uses the version hint), but got {}", + count, 2, + "Expected 2 listing calls for describe_table with migration mode \ + (absent-__manifest probe + table directory fallback), but got {}", count ); } diff --git a/rust/lance-namespace-impls/src/dir/manifest.rs b/rust/lance-namespace-impls/src/dir/manifest.rs index bca7408369e..123f7e314fe 100644 --- a/rust/lance-namespace-impls/src/dir/manifest.rs +++ b/rust/lance-namespace-impls/src/dir/manifest.rs @@ -36,7 +36,9 @@ use lance_index::progress::noop_progress; use lance_index::registry::IndexPluginRegistry; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::VALUE_COLUMN_NAME; -use lance_index::scalar::{BuiltinIndexType, CreatedIndex, ScalarIndexParams}; +use lance_index::scalar::{ + BuiltinIndexType, CreatedIndex, ScalarIndexParams, index_files_to_table, +}; use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_io::stream::RecordBatchStream as LanceRecordBatchStream; use lance_namespace::LanceNamespace; @@ -53,7 +55,7 @@ use lance_namespace::models::{ TableExistsRequest, }; use lance_namespace::schema::arrow_schema_to_json; -use lance_table::feature_flags::apply_feature_flags; +use lance_table::feature_flags::{apply_feature_flags, ensure_can_write_manifest}; use lance_table::format::{Fragment, IndexMetadata, Manifest}; use lance_table::io::commit::{ CommitError, CommitHandler, commit_handler_from_url, write_manifest_file_to_path, @@ -66,7 +68,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, hash::{DefaultHasher, Hash, Hasher}, ops::{Deref, DerefMut}, - sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard}, + sync::{Arc, LazyLock, Mutex as StdMutex, MutexGuard as StdMutexGuard}, }; use tokio::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; use uuid::Uuid; @@ -86,6 +88,15 @@ const OBJECT_ID_INDEX_NAME: &str = "object_id_btree"; const OBJECT_TYPE_INDEX_NAME: &str = "object_type_bitmap"; /// LabelList index on the base_objects column for view dependencies const BASE_OBJECTS_INDEX_NAME: &str = "base_objects_label_list"; +/// Value field of the base_objects index, whose nested `List` type would +/// otherwise allocate an inner field per use. +static BASE_OBJECTS_VALUE_FIELD: LazyLock = LazyLock::new(|| { + Field::new( + VALUE_COLUMN_NAME, + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ) +}); // Each retry reloads and rewrites the full manifest. Match the regular Lance // commit retry budget so multi-process namespace writes can make progress. const DEFAULT_MANIFEST_REWRITE_COMMIT_RETRIES: u32 = 20; @@ -852,7 +863,60 @@ impl ManifestNamespace { Self::ensure_manifest_table_up_to_date(&root, &storage_options, session.clone()) .await?; - Ok(Self { + Ok(Self::new( + root, + storage_options, + session, + object_store, + base_path, + manifest_dataset, + dir_listing_enabled, + inline_optimization_enabled, + commit_retries, + )) + } + + /// Open an existing manifest dataset without creating or migrating it. + #[allow(clippy::too_many_arguments)] + pub async fn open_from_directory( + root: String, + storage_options: Option>, + session: Option>, + object_store: Arc, + base_path: Path, + dir_listing_enabled: bool, + inline_optimization_enabled: bool, + commit_retries: Option, + ) -> Result { + let manifest_dataset = + Self::open_manifest_table(&root, &storage_options, session.clone()).await?; + + Ok(Self::new( + root, + storage_options, + session, + object_store, + base_path, + manifest_dataset, + dir_listing_enabled, + inline_optimization_enabled, + commit_retries, + )) + } + + #[allow(clippy::too_many_arguments)] + fn new( + root: String, + storage_options: Option>, + session: Option>, + object_store: Arc, + base_path: Path, + manifest_dataset: DatasetConsistencyWrapper, + dir_listing_enabled: bool, + inline_optimization_enabled: bool, + commit_retries: Option, + ) -> Self { + Self { root, storage_options, session, @@ -863,7 +927,7 @@ impl ManifestNamespace { inline_optimization_enabled, commit_retries, manifest_mutation_lock: Arc::new(Mutex::new(())), - }) + } } /// Build object ID from namespace path and name @@ -1129,11 +1193,7 @@ impl ManifestNamespace { base_objects_values: Vec>>, base_objects_row_ids: Vec, ) -> SendableRecordBatchStream { - let schema = Self::value_row_id_schema(Field::new( - VALUE_COLUMN_NAME, - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - )); + let schema = Self::value_row_id_schema(BASE_OBJECTS_VALUE_FIELD.clone()); let stream_schema = schema.clone(); let stream = stream::unfold( ( @@ -1219,6 +1279,7 @@ impl ManifestNamespace { Ok(IndexMetadata { uuid: trained_index.uuid, fields: vec![lance_schema.field_id(trained_index.column_name)?], + covering_fields: vec![], name: trained_index.index_name.to_string(), dataset_version, fragment_bitmap: Some(fragment_bitmap.clone()), @@ -1226,7 +1287,7 @@ impl ManifestNamespace { index_version: trained_index.created_index.index_version as i32, created_at: None, base_id: None, - files: Some(trained_index.created_index.files), + files: Some(index_files_to_table(trained_index.created_index.files)), }) } @@ -1321,11 +1382,7 @@ impl ManifestNamespace { index_name: BASE_OBJECTS_INDEX_NAME, column_name: "base_objects", params: ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList), - field: Field::new( - VALUE_COLUMN_NAME, - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - true, - ), + field: BASE_OBJECTS_VALUE_FIELD.clone(), stream: Self::base_objects_index_stream(base_objects_values, base_objects_row_ids), }, &fragment_bitmap, @@ -1783,6 +1840,7 @@ impl ManifestNamespace { indices: Option>, transaction: Transaction, ) -> std::result::Result<(), CommitError> { + ensure_can_write_manifest(manifest).map_err(CommitError::from)?; apply_feature_flags(manifest, false, false).map_err(CommitError::from)?; let timestamp_nanos = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -1875,6 +1933,7 @@ impl ManifestNamespace { /// concurrent upgrade in between is still caught. async fn ensure_manifest_writable(&self) -> Result<()> { let dataset_guard = self.manifest_dataset.get().await?; + ensure_can_write_manifest(dataset_guard.manifest())?; ensure_writable(dataset_guard.metadata()) } @@ -1895,10 +1954,11 @@ impl ManifestNamespace { loop { let dataset_guard = self.manifest_dataset.get_refreshed().await?; + ensure_can_write_manifest(dataset_guard.manifest())?; let dataset = Arc::new(dataset_guard.clone()); drop(dataset_guard); - // Refuse to mutate a manifest written with a writer feature flag this - // build does not understand. + // The namespace format has its own capabilities in table metadata, + // separate from the Lance manifest capabilities checked above. ensure_writable(dataset.metadata())?; // Staged files, indices, the commit, and cleanup must all use the dataset's // own object store (see `commit_manifest_overwrite`). @@ -2411,6 +2471,37 @@ impl ManifestNamespace { Ok(found_result) } + /// Load an existing manifest dataset without creating or migrating it. + async fn open_manifest_table( + root: &str, + storage_options: &Option>, + session: Option>, + ) -> Result { + let manifest_path = format!("{}/{}", root, MANIFEST_TABLE_NAME); + log::debug!("Attempting to load manifest from {}", manifest_path); + let store_options = ObjectStoreParams { + storage_options_accessor: storage_options.as_ref().map(|opts| { + Arc::new( + lance_io::object_store::StorageOptionsAccessor::with_static_options( + opts.clone(), + ), + ) + }), + ..Default::default() + }; + let read_params = ReadParams { + session, + store_options: Some(store_options), + ..Default::default() + }; + let dataset = DatasetBuilder::from_uri(&manifest_path) + .with_read_params(read_params) + .load() + .await?; + ensure_readable(dataset.metadata())?; + Ok(DatasetConsistencyWrapper::new(dataset)) + } + /// Create or load the manifest dataset, ensuring it has the latest schema setup. /// /// This function will: @@ -2443,129 +2534,135 @@ impl ManifestNamespace { .with_read_params(read_params) .load() .await; - if let Ok(mut dataset) = dataset_result { - // Reject a manifest written with a reader feature flag this build - // does not understand before touching it. - ensure_readable(dataset.metadata())?; - - // Check if the object_id field has primary key metadata, migrate if not - let needs_pk_migration = dataset - .schema() - .field("object_id") - .map(|f| { - !f.metadata - .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) - }) - .unwrap_or(false); - - if needs_pk_migration { - // This legacy migration writes to the manifest, so confirm this - // build is allowed to write the current format first. - ensure_writable(dataset.metadata())?; - log::info!("Migrating __manifest table to add primary key metadata on object_id"); - dataset - .update_field_metadata() - .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")]) - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to find object_id field for migration: {:?}", - e - ), - }) - })? - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!("Failed to migrate primary key metadata: {:?}", e), - }) - })?; - } - - Ok(DatasetConsistencyWrapper::new(dataset)) - } else { - log::info!("Creating new manifest table at {}", manifest_path); - let schema = Self::manifest_schema(); - let empty_batch = RecordBatch::new_empty(schema.clone()); - let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone()); - - let store_params = ObjectStoreParams { - storage_options_accessor: storage_options.as_ref().map(|opts| { - Arc::new( - lance_io::object_store::StorageOptionsAccessor::with_static_options( - opts.clone(), - ), - ) - }), - ..Default::default() - }; - let write_params = WriteParams { - session: session.clone(), - store_params: Some(store_params), - ..Default::default() - }; - - let dataset = - Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await; + match dataset_result { + Ok(mut dataset) => { + // Reject a manifest written with a reader feature flag this build + // does not understand before touching it. + ensure_readable(dataset.metadata())?; + + // Check if the object_id field has primary key metadata, migrate if not + let needs_pk_migration = dataset + .schema() + .field("object_id") + .map(|f| { + !f.metadata + .contains_key(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) + }) + .unwrap_or(false); - // Handle race condition where another process created the manifest concurrently - match dataset { - Ok(dataset) => { + if needs_pk_migration { + // This legacy migration writes to the manifest, so confirm this + // build is allowed to write the current format first. + ensure_writable(dataset.metadata())?; log::info!( - "Successfully created manifest table at {}, version={}, uri={}", - manifest_path, - dataset.version().version, - dataset.uri() + "Migrating __manifest table to add primary key metadata on object_id" ); - Ok(DatasetConsistencyWrapper::new(dataset)) - } - Err(ref e) - if matches!( - e, - LanceError::DatasetAlreadyExists { .. } - | LanceError::CommitConflict { .. } - | LanceError::IncompatibleTransaction { .. } - | LanceError::RetryableCommitConflict { .. } - ) => - { - // Another process created the manifest concurrently, try to load it - log::info!( - "Manifest table was created by another process, loading it: {}", - manifest_path - ); - let recovery_store_options = ObjectStoreParams { - storage_options_accessor: storage_options.as_ref().map(|opts| { - Arc::new( - lance_io::object_store::StorageOptionsAccessor::with_static_options( - opts.clone(), - ), - ) - }), - ..Default::default() - }; - let recovery_read_params = ReadParams { - session, - store_options: Some(recovery_store_options), - ..Default::default() - }; - let dataset = DatasetBuilder::from_uri(&manifest_path) - .with_read_params(recovery_read_params) - .load() - .await + dataset + .update_field_metadata() + .update("object_id", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "0")]) .map_err(|e| { lance_core::Error::from(NamespaceError::Internal { message: format!( - "Failed to load manifest dataset after creation conflict: {}", + "Failed to find object_id field for migration: {:?}", e ), }) + })? + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!("Failed to migrate primary key metadata: {:?}", e), + }) })?; - Ok(DatasetConsistencyWrapper::new(dataset)) } - Err(e) => Err(lance_core::Error::from(NamespaceError::Internal { - message: format!("Failed to create manifest dataset: {:?}", e), - })), + + Ok(DatasetConsistencyWrapper::new(dataset)) } + Err(err) if Self::is_not_found_load_error(&err) => { + log::info!("Creating new manifest table at {}", manifest_path); + let schema = Self::manifest_schema(); + let empty_batch = RecordBatch::new_empty(schema.clone()); + let reader = RecordBatchIterator::new(vec![Ok(empty_batch)], schema.clone()); + + let store_params = ObjectStoreParams { + storage_options_accessor: storage_options.as_ref().map(|opts| { + Arc::new( + lance_io::object_store::StorageOptionsAccessor::with_static_options( + opts.clone(), + ), + ) + }), + ..Default::default() + }; + let write_params = WriteParams { + session: session.clone(), + store_params: Some(store_params), + ..Default::default() + }; + + let dataset = + Dataset::write(Box::new(reader), &manifest_path, Some(write_params)).await; + + // Handle race condition where another process created the manifest concurrently + match dataset { + Ok(dataset) => { + log::info!( + "Successfully created manifest table at {}, version={}, uri={}", + manifest_path, + dataset.version().version, + dataset.uri() + ); + Ok(DatasetConsistencyWrapper::new(dataset)) + } + Err(ref e) + if matches!( + e, + LanceError::DatasetAlreadyExists { .. } + | LanceError::CommitConflict { .. } + | LanceError::IncompatibleTransaction { .. } + | LanceError::RetryableCommitConflict { .. } + ) => + { + // Another process created the manifest concurrently, try to load it + log::info!( + "Manifest table was created by another process, loading it: {}", + manifest_path + ); + let recovery_store_options = ObjectStoreParams { + storage_options_accessor: storage_options.as_ref().map(|opts| { + Arc::new( + lance_io::object_store::StorageOptionsAccessor::with_static_options( + opts.clone(), + ), + ) + }), + ..Default::default() + }; + let recovery_read_params = ReadParams { + session, + store_options: Some(recovery_store_options), + ..Default::default() + }; + let dataset = DatasetBuilder::from_uri(&manifest_path) + .with_read_params(recovery_read_params) + .load() + .await + .map_err(|e| { + lance_core::Error::from(NamespaceError::Internal { + message: format!( + "Failed to load manifest dataset after creation conflict: {}", + e + ), + }) + })?; + Ok(DatasetConsistencyWrapper::new(dataset)) + } + Err(e) => Err(lance_core::Error::from(NamespaceError::Internal { + message: format!("Failed to create manifest dataset: {:?}", e), + })), + } + } + Err(err) => Err(err), } } @@ -3398,30 +3495,28 @@ impl LanceNamespace for ManifestNamespace { } } - // Create the .lance-reserved file to mark the table as existing - let reserved_file_path = table_path.clone().join(".lance-reserved"); + self.ensure_manifest_writable().await?; - self.object_store - .create(&reserved_file_path) - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to create .lance-reserved file for table {}: {}", - table_name, e - ), - }) - })? - .shutdown() - .await - .map_err(|e| { - lance_core::Error::from(NamespaceError::Internal { - message: format!( - "Failed to finalize .lance-reserved file for table {}: {}", - table_name, e - ), + // Atomically create the .lance-reserved file to mark the table as declared. + // Shared with DirectoryNamespace via put_marker_file_atomic (dotfile-safe + // staging + MarkerFileError::AlreadyExists → TableAlreadyExists). + let reserved_file_path = table_path.clone().join(".lance-reserved"); + super::put_marker_file_atomic( + &self.object_store, + &reserved_file_path, + &format!("table {}", table_name), + ) + .await + .map_err(|e| match e { + super::MarkerFileError::AlreadyExists { .. } => { + lance_core::Error::from(NamespaceError::TableAlreadyExists { + message: table_name.to_string(), }) - })?; + } + super::MarkerFileError::Other { message } => { + lance_core::Error::from(NamespaceError::Internal { message }) + } + })?; let metadata = Self::serialize_metadata(request.properties.as_ref(), "table", &object_id)?; @@ -3765,9 +3860,10 @@ mod tests { use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_namespace::LanceNamespace; use lance_namespace::models::{ - CreateNamespaceRequest, CreateTableRequest, DescribeTableRequest, DropTableRequest, - ListTablesRequest, TableExistsRequest, + CreateNamespaceRequest, CreateTableRequest, DeclareTableRequest, DescribeTableRequest, + DropTableRequest, ListTablesRequest, TableExistsRequest, }; + use lance_table::feature_flags::FLAG_UNKNOWN; use lance_table::format::Fragment; use rstest::rstest; use std::collections::{HashMap, HashSet}; @@ -4300,6 +4396,76 @@ mod tests { ); } + #[tokio::test] + async fn test_manifest_writes_reject_unknown_writer_flag_before_staging() { + let temp_dir = TempStdDir::default(); + let temp_path = temp_dir.to_str().unwrap(); + let manifest_ns = create_manifest_namespace(temp_path, false).await; + let data_paths_before = manifest_data_paths(&manifest_ns).await; + let original_version = { + let mut dataset = manifest_ns.manifest_dataset.get_mut().await.unwrap(); + let mut manifest = dataset.manifest().clone(); + manifest.writer_feature_flags |= FLAG_UNKNOWN << 1; + let version = manifest.version; + dataset.manifest = Arc::new(manifest); + version + }; + + let entries_before = dir_entry_names(temp_path); + let mut declare_request = DeclareTableRequest::new(); + declare_request.id = Some(vec!["declared_table".to_string()]); + let error = manifest_ns + .declare_table(declare_request) + .await + .unwrap_err(); + assert!( + error.to_string().to_lowercase().contains("upgrade"), + "expected an upgrade error, got: {error}" + ); + assert_eq!(dir_entry_names(temp_path), entries_before); + + let mut create_request = CreateTableRequest::new(); + create_request.id = Some(vec!["new_table".to_string()]); + let error = manifest_ns + .create_table(create_request, Bytes::from(create_test_ipc_data())) + .await + .unwrap_err(); + assert!( + error.to_string().to_lowercase().contains("upgrade"), + "expected an upgrade error, got: {error}" + ); + assert_eq!(dir_entry_names(temp_path), entries_before); + + let error = manifest_ns + .insert_into_manifest_with_metadata( + vec![ManifestEntry { + object_id: "table".to_string(), + object_type: ObjectType::Table, + location: Some("table.lance".to_string()), + metadata: None, + }], + None, + ) + .await + .unwrap_err(); + + assert!( + error.to_string().to_lowercase().contains("upgrade"), + "expected an upgrade error, got: {error}" + ); + assert_eq!( + manifest_ns + .manifest_dataset + .get() + .await + .unwrap() + .version() + .version, + original_version + ); + assert_eq!(manifest_data_paths(&manifest_ns).await, data_paths_before); + } + #[tokio::test] async fn test_manifest_noop_delete_uses_latest_snapshot() { let temp_dir = TempStdDir::default(); diff --git a/rust/lance-namespace-impls/src/lib.rs b/rust/lance-namespace-impls/src/lib.rs index 58e29aca5ef..2fbfc5fe452 100644 --- a/rust/lance-namespace-impls/src/lib.rs +++ b/rust/lance-namespace-impls/src/lib.rs @@ -71,6 +71,10 @@ //! # } //! ``` +use std::collections::HashSet; + +use lance_namespace::NamespaceError; + pub mod connect; pub mod context; pub mod credentials; @@ -116,3 +120,38 @@ pub use rest::{RestNamespace, RestNamespaceBuilder}; #[cfg(feature = "rest-adapter")] pub use rest_adapter::{RestAdapter, RestAdapterConfig, RestAdapterHandle}; + +/// Validate the `on` match key of a merge insert request. +/// +/// The columns form a composite key, so an empty list matches nothing and a repeated +/// column adds a redundant equality to the join. +pub(crate) fn merge_insert_on_columns<'a>( + on: Option<&'a [String]>, + operation: &str, +) -> lance_core::Result<&'a [String]> { + let on = on.ok_or_else(|| { + lance_core::Error::from(NamespaceError::InvalidInput { + message: format!("'on' field is required for {}", operation), + }) + })?; + + if on.is_empty() { + return Err(NamespaceError::InvalidInput { + message: format!("'on' field must name at least one column for {}", operation), + } + .into()); + } + + let mut seen = HashSet::with_capacity(on.len()); + if let Some(duplicate) = on.iter().find(|column| !seen.insert(*column)) { + return Err(NamespaceError::InvalidInput { + message: format!( + "'on' field for {} names column '{}' more than once: {:?}", + operation, duplicate, on + ), + } + .into()); + } + + Ok(on) +} diff --git a/rust/lance-namespace-impls/src/rest.rs b/rust/lance-namespace-impls/src/rest.rs index c245a1e6dc1..c8501a2867b 100644 --- a/rust/lance-namespace-impls/src/rest.rs +++ b/rust/lance-namespace-impls/src/rest.rs @@ -8,6 +8,7 @@ use std::str::FromStr; use std::sync::Arc; use crate::OpsMetrics; +use crate::merge_insert_on_columns; use async_trait::async_trait; use bytes::Bytes; @@ -50,6 +51,7 @@ use lance_namespace::models::{ }; use serde::{Serialize, de::DeserializeOwned}; +use lance_core::utils::parse::str_to_bool; use lance_core::{Error, Result}; use lance_namespace::LanceNamespace; @@ -290,13 +292,13 @@ impl RestNamespaceBuilder { let ssl_ca_cert = properties.get("tls.ssl_ca_cert").cloned(); let assert_hostname = properties .get("tls.assert_hostname") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(true); // Extract ops_metrics_enabled (default: false) let ops_metrics_enabled = properties .get("ops_metrics_enabled") - .and_then(|v| v.parse::().ok()) + .and_then(|v| str_to_bool(v)) .unwrap_or(false); Ok(Self { @@ -1010,14 +1012,13 @@ impl LanceNamespace for RestNamespace { let id = object_id_str(&request.id, &self.delimiter)?; let encoded_id = urlencode(&id); - let on = request.on.as_deref().ok_or_else(|| { - lance_core::Error::from(NamespaceError::InvalidInput { - message: "'on' field is required for merge insert".to_string(), - }) - })?; + let on = merge_insert_on_columns(request.on.as_deref(), "merge_insert_into_table")?; let path = format!("/v1/table/{}/merge_insert", encoded_id); - let mut query = vec![("delimiter", self.delimiter.as_str()), ("on", on)]; + // The `on` query parameter uses `style: form, explode: true`, so a composite key + // repeats the parameter once per column. + let mut query = vec![("delimiter", self.delimiter.as_str())]; + query.extend(on.iter().map(|column| ("on", column.as_str()))); let when_matched_update_all_str; if let Some(v) = request.when_matched_update_all { diff --git a/rust/lance-namespace-impls/src/rest_adapter.rs b/rust/lance-namespace-impls/src/rest_adapter.rs index 44ebd866810..10dcdf0a925 100644 --- a/rust/lance-namespace-impls/src/rest_adapter.rs +++ b/rust/lance-namespace-impls/src/rest_adapter.rs @@ -12,7 +12,7 @@ use std::sync::Arc; use axum::{ Json, Router, ServiceExt, body::Bytes, - extract::{FromRequest, Path, Query, Request, State}, + extract::{FromRequest, Path, Query, RawQuery, Request, State}, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, routing::{get, post}, @@ -22,6 +22,7 @@ use tokio::sync::watch; use tower::Layer; use tower_http::normalize_path::NormalizePathLayer; use tower_http::trace::TraceLayer; +use url::form_urlencoded; use lance_core::{Error, Result}; use lance_namespace::LanceNamespace; @@ -699,10 +700,12 @@ async fn insert_into_table( } } +/// `on` is absent here on purpose: it repeats once per column of a composite match key, +/// and `serde_urlencoded` (what axum's `Query` is built on) cannot deserialize a sequence. +/// It is collected from the raw query string by [`merge_insert_on_params`] instead. #[derive(Debug, Deserialize)] struct MergeInsertQuery { delimiter: Option, - on: Option, when_matched_update_all: Option, when_matched_update_all_filt: Option, when_not_matched_insert_all: Option, @@ -712,16 +715,25 @@ struct MergeInsertQuery { use_index: Option, } +fn merge_insert_on_params(raw_query: Option<&str>) -> Option> { + let on: Vec = form_urlencoded::parse(raw_query?.as_bytes()) + .filter(|(key, _)| key == "on") + .map(|(_, value)| value.into_owned()) + .collect(); + (!on.is_empty()).then_some(on) +} + async fn merge_insert_into_table( State(backend): State>, headers: HeaderMap, Path(id): Path, Query(params): Query, + RawQuery(raw_query): RawQuery, body: Bytes, ) -> Response { let request = MergeInsertIntoTableRequest { id: Some(parse_id(&id, params.delimiter.as_deref())), - on: params.on, + on: merge_insert_on_params(raw_query.as_deref()), when_matched_update_all: params.when_matched_update_all, when_matched_update_all_filt: params.when_matched_update_all_filt, when_not_matched_insert_all: params.when_not_matched_insert_all, @@ -1503,6 +1515,25 @@ mod tests { assert_eq!(id, vec!["table"]); } + #[test] + fn test_merge_insert_on_params() { + assert_eq!(merge_insert_on_params(None), None); + assert_eq!(merge_insert_on_params(Some("delimiter=%24")), None); + assert_eq!( + merge_insert_on_params(Some("on=id&use_index=true")), + Some(vec!["id".to_string()]) + ); + assert_eq!( + merge_insert_on_params(Some("on=region&use_index=true&on=id")), + Some(vec!["region".to_string(), "id".to_string()]) + ); + // A backtick-quoted field path arrives percent-encoded. + assert_eq!( + merge_insert_on_params(Some("on=%60a.b%60&on=nested.leaf")), + Some(vec!["`a.b`".to_string(), "nested.leaf".to_string()]) + ); + } + // ============================================================================ // Integration Tests // ============================================================================ @@ -3061,6 +3092,92 @@ mod tests { assert_eq!(a_col.values(), &[100, 200]); } + /// `(region, id, value)` rows, for merge inserts keyed on `region` + `id`. + fn create_composite_key_arrow_data(rows: &[(&str, i32, &str)]) -> Bytes { + use arrow::array::{Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow::ipc::writer::StreamWriter; + use arrow::record_batch::RecordBatch; + + let schema = Arc::new(Schema::new(vec![ + Field::new("region", DataType::Utf8, false), + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from_iter_values( + rows.iter().map(|(region, _, _)| *region), + )), + Arc::new(Int32Array::from_iter_values( + rows.iter().map(|(_, id, _)| *id), + )), + Arc::new(StringArray::from_iter_values( + rows.iter().map(|(_, _, value)| *value), + )), + ], + ) + .unwrap(); + + let mut buffer = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut buffer, &schema).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + Bytes::from(buffer) + } + + /// A composite match key survives the round trip through the REST client and the + /// adapter's query string, which repeats `on` once per column. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_merge_insert_composite_key_round_trip() { + use lance_namespace::LanceNamespace; + + let fixture = RestServerFixture::new().await; + let table_id = vec!["merge_ns".to_string(), "merge_table".to_string()]; + let on = vec!["region".to_string(), "id".to_string()]; + + let mut create_ns = CreateNamespaceRequest::new(); + create_ns.id = Some(vec!["merge_ns".to_string()]); + fixture.namespace.create_namespace(create_ns).await.unwrap(); + + let create_table_req = CreateTableRequest { + id: Some(table_id.clone()), + mode: Some("Create".to_string()), + ..Default::default() + }; + fixture + .namespace + .create_table( + create_table_req, + create_composite_key_arrow_data(&[ + ("us", 1, "a"), + ("us", 2, "b"), + ("eu", 1, "c"), + ]), + ) + .await + .unwrap(); + + let mut merge_req = MergeInsertIntoTableRequest::new(); + merge_req.id = Some(table_id); + merge_req.on = Some(on); + merge_req.when_matched_update_all = Some(true); + let response = fixture + .namespace + .merge_insert_into_table( + merge_req, + create_composite_key_arrow_data(&[("us", 1, "updated"), ("eu", 2, "inserted")]), + ) + .await + .unwrap(); + + assert_eq!(response.num_updated_rows, Some(1)); + assert_eq!(response.num_inserted_rows, Some(1)); + } + // ============================================================================ // DynamicContextProvider Integration Test // ============================================================================ diff --git a/rust/lance-namespace/Cargo.toml b/rust/lance-namespace/Cargo.toml index cd32c8f611e..ceac8ffa501 100644 --- a/rust/lance-namespace/Cargo.toml +++ b/rust/lance-namespace/Cargo.toml @@ -16,6 +16,8 @@ async-trait.workspace = true bytes.workspace = true arrow.workspace = true lance-core.workspace = true +serde.workspace = true +serde_json.workspace = true snafu.workspace = true lance-namespace-reqwest-client.workspace = true diff --git a/rust/lance-namespace/src/compat.rs b/rust/lance-namespace/src/compat.rs new file mode 100644 index 00000000000..57c381831f8 --- /dev/null +++ b/rust/lance-namespace/src/compat.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Deserialization shims that keep older SDK builds working against current native code. + +use crate::error::NamespaceError; +use crate::models::MergeInsertIntoTableRequest; + +/// Deserialize a [`MergeInsertIntoTableRequest`] whose `on` field may be a bare string. +/// +/// Java and Python SDK requests reach the Rust implementations as JSON across JNI and +/// pyo3, so a jar or wheel built against lance-namespace 0.11 or earlier sends +/// `"on": "id"` where the current model expects `"on": ["id"]`. A scalar is promoted to +/// a one-element list so those callers keep working. +/// +/// This is inbound only. A Java namespace implementation called *from* Rust still needs +/// a jar matching the current model. +pub fn merge_insert_request_from_json( + mut value: serde_json::Value, +) -> crate::Result { + if let Some(on) = value.get_mut("on") + && let Some(column) = on.as_str() + { + *on = serde_json::json!([column]); + } + + serde_json::from_value(value).map_err(|e| { + NamespaceError::InvalidInput { + message: format!("Failed to parse merge_insert_into_table request: {}", e), + } + .into() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(json: &str) -> MergeInsertIntoTableRequest { + merge_insert_request_from_json(serde_json::from_str(json).unwrap()).unwrap() + } + + #[test] + fn scalar_on_is_promoted_to_a_single_column_key() { + let request = parse(r#"{"id": ["t"], "on": "id"}"#); + assert_eq!(request.on, Some(vec!["id".to_string()])); + assert_eq!(request.id, Some(vec!["t".to_string()])); + } + + #[test] + fn list_on_is_preserved() { + let request = parse(r#"{"id": ["t"], "on": ["a", "b"], "use_index": true}"#); + assert_eq!(request.on, Some(vec!["a".to_string(), "b".to_string()])); + assert_eq!(request.use_index, Some(true)); + } + + #[test] + fn absent_and_null_on_stay_absent() { + assert_eq!(parse(r#"{"id": ["t"]}"#).on, None); + assert_eq!(parse(r#"{"id": ["t"], "on": null}"#).on, None); + } + + #[test] + fn a_non_string_non_list_on_is_still_rejected() { + let error = + merge_insert_request_from_json(serde_json::json!({"id": ["t"], "on": 7})).unwrap_err(); + assert!( + error.to_string().contains("invalid type"), + "unexpected error: {error}" + ); + } +} diff --git a/rust/lance-namespace/src/lib.rs b/rust/lance-namespace/src/lib.rs index 6fd9a9b7ab2..2fc34e03549 100644 --- a/rust/lance-namespace/src/lib.rs +++ b/rust/lance-namespace/src/lib.rs @@ -15,6 +15,7 @@ //! See [`error::ErrorCode`] for the list of error codes and //! [`error::NamespaceError`] for the error types. +pub mod compat; pub mod error; pub mod namespace; pub mod schema; diff --git a/rust/lance-select/Cargo.toml b/rust/lance-select/Cargo.toml index 4cba7f082a8..678521dfd12 100644 --- a/rust/lance-select/Cargo.toml +++ b/rust/lance-select/Cargo.toml @@ -17,7 +17,6 @@ arrow-buffer = { workspace = true } arrow-schema = { workspace = true } byteorder = { workspace = true } tracing = { workspace = true } -bytes = { workspace = true } itertools = { workspace = true } lance-core = { workspace = true } roaring = { workspace = true } @@ -25,7 +24,6 @@ roaring = { workspace = true } [dev-dependencies] criterion = { workspace = true } proptest = { workspace = true } -rstest = { workspace = true } [[bench]] name = "index_expr_result" diff --git a/rust/lance-select/benches/row_addr_mask.rs b/rust/lance-select/benches/row_addr_mask.rs index c5d6c44dd85..850a0bd084f 100644 --- a/rust/lance-select/benches/row_addr_mask.rs +++ b/rust/lance-select/benches/row_addr_mask.rs @@ -69,8 +69,7 @@ fn bench_iter_addrs(c: &mut Criterion) { group.throughput(Throughput::Elements(n)); group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { b.iter(|| { - // SAFETY: the map only contains Partial selections; no Full entries. - let count: u64 = unsafe { map.clone().into_addr_iter() }.count() as u64; + let count: u64 = map.clone().into_addr_iter().count() as u64; std::hint::black_box(count); }); }); @@ -121,9 +120,8 @@ fn bench_iter_runs_partial(c: &mut Criterion) { group.throughput(Throughput::Elements(n)); group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { b.iter(|| { - // SAFETY: map only contains Partial selections. let mut runs: u64 = 0; - for _ in unsafe { map.iter_runs() } { + for _ in map.iter_runs() { runs += 1; } std::hint::black_box(runs); @@ -181,7 +179,7 @@ fn bench_range_to_ranges_round_trip(c: &mut Criterion) { // consumer actually does — e.g. GroupingIterator). let mut ids = RowAddrTreeMap::from(src.clone()); ids.mask(&mask); - let count = unsafe { ids.into_addr_iter() }.count(); + let count = ids.into_addr_iter().count(); std::hint::black_box(count); }); }); @@ -204,8 +202,8 @@ fn bench_range_to_ranges_round_trip_runs(c: &mut Criterion) { b.iter(|| { let mut ids = RowAddrTreeMap::from(src.clone()); ids.mask(&mask); - // SAFETY: only Partial selections in play. - let count: u64 = unsafe { ids.iter_runs() } + let count: u64 = ids + .iter_runs() .map(|(_, r)| (*r.end() as u64) - (*r.start() as u64) + 1) .sum(); std::hint::black_box(count); diff --git a/rust/lance-select/src/mask.rs b/rust/lance-select/src/mask.rs index f9df7720441..a39e923810e 100644 --- a/rust/lance-select/src/mask.rs +++ b/rust/lance-select/src/mask.rs @@ -85,6 +85,14 @@ impl RowAddrMask { matches!(self, Self::BlockList(b) if b.is_empty()) } + /// Returns whether this mask selects every row in `rows`. + pub fn selects_all(&self, rows: &RowAddrTreeMap) -> bool { + match self { + Self::AllowList(allow_list) => (rows.clone() - allow_list).is_empty(), + Self::BlockList(block_list) => (rows.clone() & block_list).is_empty(), + } + } + /// Return the indices of the input row ids that were valid pub fn selected_indices<'a>(&self, row_ids: impl Iterator + 'a) -> Vec { row_ids @@ -107,6 +115,51 @@ impl RowAddrMask { } } + /// Build a mask from serialized [`RowAddrTreeMap`] payloads. + /// + /// `allow` selects rows, `block` excludes them; each is the output of + /// [`RowAddrTreeMap::serialize_into`]. Returns `None` when neither is given, + /// which callers read as "no mask" rather than "select nothing". + /// + /// Bytes rather than treemaps on purpose: a caller living in a different + /// dynamically-linked extension module has its own copy of these Rust types + /// and cannot hand one over, but both sides agree on this encoding. + pub fn from_serialized_parts( + allow: Option<&[u8]>, + block: Option<&[u8]>, + ) -> Result> { + // Name the offending side: the underlying failure is a bare "failed to + // fill whole buffer", which tells a caller holding two blobs nothing. + fn decode(bytes: &[u8], which: &str) -> Result { + RowAddrTreeMap::deserialize_from(bytes).map_err(|e| { + Error::invalid_input(format!( + "row address {which} is not a serialized RowAddrTreeMap: {e}" + )) + }) + } + let allow = allow.map(|b| decode(b, "allowlist")).transpose()?; + let block = block.map(|b| decode(b, "blocklist")).transpose()?; + Ok(match (allow, block) { + (Some(allow), Some(block)) => Some(Self::from_allowed(allow).also_block(block)), + (Some(allow), None) => Some(Self::from_allowed(allow)), + (None, Some(block)) => Some(Self::from_block(block)), + (None, None) => None, + }) + } + + /// Intersect two masks: a row survives only if both select it. + /// + /// Lets a planner apply a caller-supplied mask at one boundary rather than + /// at every branch that produces rows, which is how branches get missed. + pub fn intersect(self, other: Self) -> Self { + match (self, other) { + (Self::AllowList(a), Self::AllowList(b)) => Self::AllowList(a & b), + (Self::AllowList(a), Self::BlockList(b)) => Self::AllowList(a).also_block(b), + (Self::BlockList(a), Self::AllowList(b)) => Self::AllowList(b).also_block(a), + (Self::BlockList(a), Self::BlockList(b)) => Self::BlockList(a | b), + } + } + /// Also allow the given addrs pub fn also_allow(self, allow_list: RowAddrTreeMap) -> Self { match self { @@ -623,8 +676,21 @@ impl RowAddrTreeMap { if bitmap_size == 0 { inner.insert(fragment, RowAddrSelection::Full); } else { - let mut buffer = vec![0; bitmap_size as usize]; - reader.read_exact(&mut buffer)?; + // Grow with the bytes that actually arrive instead of trusting the + // declared size. This is reachable from a public byte boundary, so + // a 12-byte payload could otherwise declare 4 GiB and abort the + // process on the allocation before any read fails. + let mut buffer = Vec::new(); + let read = reader + .by_ref() + .take(u64::from(bitmap_size)) + .read_to_end(&mut buffer)?; + if read != bitmap_size as usize { + return Err(Error::invalid_input(format!( + "row addr treemap declares a {bitmap_size} byte bitmap for \ + fragment {fragment} but only {read} bytes remain" + ))); + } let set = RoaringBitmap::deserialize_from(&buffer[..])?; inner.insert(fragment, RowAddrSelection::Partial(set)); } @@ -649,12 +715,10 @@ impl RowAddrTreeMap { /// Convert the set into an iterator of row addrs /// - /// # Safety + /// # Panics /// - /// This is unsafe because if any of the inner RowAddrSelection elements - /// is not a Partial then the iterator will panic because we don't know - /// the size of the bitmap. - pub unsafe fn into_addr_iter(self) -> impl Iterator { + /// Panics if any selection is `Full` because the fragment size is unknown. + pub fn into_addr_iter(self) -> impl Iterator { self.inner .into_iter() .flat_map(|(fragment, selection)| match selection { @@ -675,10 +739,10 @@ impl RowAddrTreeMap { /// rather than its individual bits, so dense ranges cost /// O(num_containers) (roughly num_rows / 65536) instead of O(num_rows). /// - /// # Safety - /// Same contract as [`Self::into_addr_iter`]: panics if any entry is - /// `Full`, since the fragment size is unknown at this layer. - pub unsafe fn iter_runs(&self) -> impl Iterator)> + '_ { + /// # Panics + /// + /// Panics if any selection is `Full` because the fragment size is unknown. + pub fn iter_runs(&self) -> impl Iterator)> + '_ { self.inner .iter() .flat_map(|(&fragment, selection)| match selection { @@ -1239,6 +1303,23 @@ mod tests { assert!(allow_list.iter_addrs().is_none()); } + #[test] + fn test_row_addr_mask_selects_all_known_rows() { + let partition_rows = rows(&[10, 20, 2_u64 << 32 | 3]); + + assert!(RowAddrMask::all_rows().selects_all(&partition_rows)); + assert!( + RowAddrMask::from_allowed(rows(&[10, 20, 30, 2_u64 << 32 | 3])) + .selects_all(&partition_rows) + ); + assert!( + !RowAddrMask::from_allowed(rows(&[10, 2_u64 << 32 | 3])).selects_all(&partition_rows) + ); + assert!(RowAddrMask::from_block(rows(&[30])).selects_all(&partition_rows)); + assert!(!RowAddrMask::from_block(rows(&[20, 30])).selects_all(&partition_rows)); + assert!(RowAddrMask::allow_nothing().selects_all(&RowAddrTreeMap::new())); + } + #[test] fn test_selected_indices() { // Allow list @@ -1296,6 +1377,102 @@ mod tests { assert!(mask.iter_addrs().is_none()); } + #[test] + fn test_row_addr_mask_intersect() { + let a = rows(&[1, 2, 3]); + let b = rows(&[3, 4]); + + // allow & allow -> only rows in both + assert_mask_selects( + &RowAddrMask::from_allowed(a.clone()).intersect(RowAddrMask::from_allowed(b.clone())), + &[3], + &[1, 2, 4, 100], + ); + // allow & block -> allowed minus blocked + assert_mask_selects( + &RowAddrMask::from_allowed(a.clone()).intersect(RowAddrMask::from_block(b.clone())), + &[1, 2], + &[3, 4, 100], + ); + // block & allow -> same, order independent + assert_mask_selects( + &RowAddrMask::from_block(b.clone()).intersect(RowAddrMask::from_allowed(a.clone())), + &[1, 2], + &[3, 4, 100], + ); + // block & block -> both exclusions apply + assert_mask_selects( + &RowAddrMask::from_block(a.clone()).intersect(RowAddrMask::from_block(b)), + &[100], + &[1, 2, 3, 4], + ); + // all_rows is the identity, and intersecting with itself changes nothing + let allow_a = RowAddrMask::from_allowed(a.clone()); + assert_eq!(allow_a.clone().intersect(RowAddrMask::all_rows()), allow_a); + assert_eq!(allow_a.clone().intersect(allow_a.clone()), allow_a); + // allow_nothing absorbs + assert_mask_selects( + &RowAddrMask::allow_nothing().intersect(RowAddrMask::from_allowed(a)), + &[], + &[1, 2, 3, 100], + ); + } + + #[test] + fn test_row_addr_mask_from_serialized_parts() { + fn ser(tm: &RowAddrTreeMap) -> Vec { + let mut buf = Vec::new(); + tm.serialize_into(&mut buf).unwrap(); + buf + } + let allow = ser(&rows(&[1, 2, 3])); + let block = ser(&rows(&[3, 4])); + + // Neither part means "no mask", which is not the same as "select nothing". + assert!( + RowAddrMask::from_serialized_parts(None, None) + .unwrap() + .is_none() + ); + + let m = RowAddrMask::from_serialized_parts(Some(&allow), None) + .unwrap() + .unwrap(); + assert_mask_selects(&m, &[1, 2, 3], &[4, 100]); + + let m = RowAddrMask::from_serialized_parts(None, Some(&block)) + .unwrap() + .unwrap(); + assert_mask_selects(&m, &[1, 2, 100], &[3, 4]); + + // Block wins on the overlap. + let m = RowAddrMask::from_serialized_parts(Some(&allow), Some(&block)) + .unwrap() + .unwrap(); + assert_mask_selects(&m, &[1, 2], &[3, 4, 100]); + + // Round trips through the same encoding the caller used. + let again = RowAddrMask::from_serialized_parts(Some(&ser(m.allow_list().unwrap())), None) + .unwrap() + .unwrap(); + assert_mask_selects(&again, &[1, 2], &[3, 4]); + + assert!(RowAddrMask::from_serialized_parts(Some(b"not a treemap"), None).is_err()); + + // A declared bitmap size must not be allocated before the bytes are + // known to exist: this 12-byte payload claims ~4 GiB. + let bomb = [ + 1u8, 0, 0, 0, // one entry + 0, 0, 0, 0, // fragment zero + 0xff, 0xff, 0xff, 0xff, // declared bitmap size + ]; + let err = RowAddrMask::from_serialized_parts(Some(&bomb), None).unwrap_err(); + assert!( + err.to_string().contains("only 0 bytes remain"), + "expected a length complaint, got: {err}" + ); + } + #[test] fn test_row_addr_mask_not() { let allow_list = RowAddrMask::from_allowed(rows(&[1, 2, 3])); @@ -1582,9 +1759,7 @@ mod tests { let mut mask = RowAddrTreeMap::default(); mask.insert_fragment(0); - unsafe { - let _ = mask.into_addr_iter().collect::>(); - } + let _ = mask.into_addr_iter().collect::>(); } #[test] @@ -1596,7 +1771,7 @@ mod tests { mask.insert(2 << 32 | 10); let expected = vec![0u64, 1, 1 << 32 | 5, 2 << 32 | 10]; - let actual: Vec = unsafe { mask.into_addr_iter().collect() }; + let actual: Vec = mask.into_addr_iter().collect(); assert_eq!(actual, expected); } @@ -1608,8 +1783,7 @@ mod tests { mask.insert_range(10..15); mask.insert_range((1u64 << 32) + 100..(1u64 << 32) + 103); - // SAFETY: only Partial entries. - let runs: Vec<(u32, RangeInclusive)> = unsafe { mask.iter_runs().collect() }; + let runs: Vec<(u32, RangeInclusive)> = mask.iter_runs().collect(); assert_eq!(runs, vec![(0, 0..=2), (0, 10..=14), (1, 100..=102)]); } @@ -1622,13 +1796,14 @@ mod tests { mask.insert_range(20..25); mask.insert_range((1u64 << 32)..(1u64 << 32) + 3); - let from_runs: Vec = unsafe { mask.iter_runs() } + let from_runs: Vec = mask + .iter_runs() .flat_map(|(frag, run)| { let frag = u64::from(frag); (*run.start()..=*run.end()).map(move |v| (frag << 32) | u64::from(v)) }) .collect(); - let from_bits: Vec = unsafe { mask.clone().into_addr_iter() }.collect(); + let from_bits: Vec = mask.clone().into_addr_iter().collect(); assert_eq!(from_runs, from_bits); } diff --git a/rust/lance-table/Cargo.toml b/rust/lance-table/Cargo.toml index 042ae92c618..4fb4da70ba9 100644 --- a/rust/lance-table/Cargo.toml +++ b/rust/lance-table/Cargo.toml @@ -24,6 +24,7 @@ arrow-ipc.workspace = true arrow-schema.workspace = true async-trait.workspace = true aws-credential-types = { workspace = true, optional = true } +blake3.workspace = true aws-sdk-dynamodb = { workspace = true, optional = true, default-features = false, features = ["default-https-client", "rt-tokio"] } byteorder.workspace = true bytes.workspace = true @@ -56,7 +57,7 @@ rstest.workspace = true [build-dependencies] prost-build.workspace = true -protobuf-src = { version = "2.1", optional = true } +protobuf-src = { workspace = true, optional = true } [features] dynamodb = ["dep:aws-sdk-dynamodb", "dep:aws-credential-types", "lance-io/aws"] @@ -74,5 +75,9 @@ harness = false name = "manifest_intern" harness = false +[[bench]] +name = "system_columns" +harness = false + [lints] workspace = true diff --git a/rust/lance-table/benches/manifest_intern.rs b/rust/lance-table/benches/manifest_intern.rs index 78b7e352207..81bd57c1a22 100644 --- a/rust/lance-table/benches/manifest_intern.rs +++ b/rust/lance-table/benches/manifest_intern.rs @@ -59,6 +59,7 @@ fn make_uniform_pb_fragments(n: u64, num_fields: usize) -> Vec file_size_bytes: 0, base_id: None, }], + overlays: vec![], deletion_file: None, row_id_sequence: None, physical_rows: 1000, @@ -135,6 +136,7 @@ fn make_diverse_pb_fragments( file_size_bytes: 0, base_id: None, }], + overlays: vec![], deletion_file: None, row_id_sequence: None, physical_rows: 1000, diff --git a/rust/lance-table/benches/system_columns.rs b/rust/lance-table/benches/system_columns.rs new file mode 100644 index 00000000000..e04f1b1f3ab --- /dev/null +++ b/rust/lance-table/benches/system_columns.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{hint::black_box, sync::Arc, time::Duration}; + +use arrow_array::{RecordBatch, RecordBatchOptions, UInt64Array}; +use arrow_schema::{DataType, Field, Schema}; +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; +use futures::{FutureExt, StreamExt, TryStreamExt, future, stream}; +use lance_io::ReadBatchParams; +use lance_table::{ + rowids::RowIdSequence, + utils::stream::{ + ReadBatchTask, ReadBatchTaskStream, RowIdAndDeletesConfig, wrap_with_row_id_and_delete, + }, +}; + +fn make_batch(batch_size: usize, has_payload: bool) -> RecordBatch { + if has_payload { + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "value", + DataType::UInt64, + false, + )])), + vec![Arc::new(UInt64Array::from(vec![0; batch_size]))], + ) + .unwrap() + } else { + RecordBatch::try_new_with_options( + Arc::new(Schema::empty()), + Vec::new(), + &RecordBatchOptions::new().with_row_count(Some(batch_size)), + ) + .unwrap() + } +} + +fn make_tasks(batch: RecordBatch, total_rows: usize, batch_size: usize) -> ReadBatchTaskStream { + let tasks = (0..total_rows).step_by(batch_size).map(move |offset| { + let num_rows = batch_size.min(total_rows - offset); + let batch = if num_rows == batch.num_rows() { + batch.clone() + } else { + batch.slice(0, num_rows) + }; + ReadBatchTask { + task: future::ready(Ok(batch)).boxed(), + num_rows: num_rows as u32, + } + }); + stream::iter(tasks).boxed() +} + +fn make_config(total_rows: usize, sequence: Arc) -> RowIdAndDeletesConfig { + RowIdAndDeletesConfig { + params: ReadBatchParams::RangeFull, + with_row_id: true, + with_row_addr: false, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + deletion_vector: None, + row_id_sequence: Some(sequence), + last_updated_at_sequence: None, + created_at_sequence: None, + make_deletions_null: false, + total_num_rows: total_rows as u32, + } +} + +fn bench_stream_row_ids(c: &mut Criterion) { + let total_rows = std::env::var("BENCH_SYSTEM_ROWS") + .map(|value| value.parse().unwrap()) + .unwrap_or(100_000_usize); + let batch_size = std::env::var("BENCH_SYSTEM_BATCH_SIZE") + .map(|value| value.parse().unwrap()) + .unwrap_or(1_024_usize) + .min(total_rows); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + + let mut group = c.benchmark_group("stream_row_ids"); + group.sample_size(10); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(3)); + for hole_stride in [2_u64, 17] { + let sequence = Arc::new( + RowIdSequence::try_from_iter( + (0_u64..) + .filter(|value| value % hole_stride != 0) + .take(total_rows), + ) + .unwrap(), + ); + for has_payload in [false, true] { + let batch = make_batch(batch_size, has_payload); + let parameter = format!("holes_{hole_stride}/payload_{has_payload}"); + group.bench_with_input( + BenchmarkId::new("shape", parameter), + &has_payload, + |b, _| { + b.iter_batched( + || { + ( + make_tasks(batch.clone(), total_rows, batch_size), + make_config(total_rows, sequence.clone()), + ) + }, + |(tasks, config)| { + let batches = runtime + .block_on( + wrap_with_row_id_and_delete(tasks, 0, config) + .buffered(8) + .try_collect::>(), + ) + .unwrap(); + black_box(batches); + }, + BatchSize::SmallInput, + ); + }, + ); + } + } + group.finish(); +} + +criterion_group!(benches, bench_stream_row_ids); +criterion_main!(benches); diff --git a/rust/lance-table/src/feature_flags.rs b/rust/lance-table/src/feature_flags.rs index 096f0da79e5..d5a7ec6e31c 100644 --- a/rust/lance-table/src/feature_flags.rs +++ b/rust/lance-table/src/feature_flags.rs @@ -8,20 +8,67 @@ use lance_core::{Error, Result}; /// Fragments may contain deletion files, which record the tombstones of /// soft-deleted rows. -pub const FLAG_DELETION_FILES: u64 = 1; +pub const FLAG_DELETION_FILES: u64 = 1 << 0; /// Row ids are stable for both moves and updates. Fragments contain an index /// mapping row ids to row addresses. -pub const FLAG_STABLE_ROW_IDS: u64 = 2; +pub const FLAG_STABLE_ROW_IDS: u64 = 1 << 1; /// Files are written with the new v2 format (this flag is no longer used) -pub const FLAG_USE_V2_FORMAT_DEPRECATED: u64 = 4; +pub const FLAG_USE_V2_FORMAT_DEPRECATED: u64 = 1 << 2; /// Table config is present -pub const FLAG_TABLE_CONFIG: u64 = 8; +pub const FLAG_TABLE_CONFIG: u64 = 1 << 3; /// Dataset uses multiple base paths (for shallow clones or multi-base datasets) -pub const FLAG_BASE_PATHS: u64 = 16; +pub const FLAG_BASE_PATHS: u64 = 1 << 4; /// Disable writing transaction file under _transaction/, this flag is set when we only want to write inline transaction in manifest -pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 32; +pub const FLAG_DISABLE_TRANSACTION_FILE: u64 = 1 << 5; +/// Fragments contain data overlay files, which supply new values for a subset of +/// cells without rewriting base data files. A reader that does not understand +/// overlays must refuse the dataset, since ignoring an overlay would silently +/// return stale base values. +/// +/// Data overlay files are not yet a released feature: in release builds this flag +/// is treated as unknown (so a release reader/writer refuses an overlay dataset) +/// unless [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set, which lets benchmarks opt in. +/// Debug builds always understand it so tests exercise the path. +pub const FLAG_UNSTABLE_DATA_OVERLAY_FILES: u64 = 1 << 6; +/// Some index declares covering columns: `IndexMetadata.covering_fields` names +/// columns the index carries values for but is not keyed on. +/// +/// Covering makes `fields` mean "keyed columns followed by carried columns" +/// rather than "the columns this index is searched on". A reader without this +/// bit still selects a vector index by testing membership of `fields`, so it +/// would answer a query on a merely-carried column with an index keyed on a +/// different column and return wrong neighbours with no error. A writer without +/// it would maintain the index as though every entry of `fields` were keyed. +/// Both must refuse the table. +/// +/// This takes the bit reclaimed from the retired MemWAL index-catchup flag +/// (), which is the boundary the +/// current released build treats as unknown -- so that build refuses a covering +/// dataset without needing a change of its own. Builds from the window where the +/// bit was allocated to index catch-up (v11.0.0-beta.4 through beta.17) still +/// count it as supported and will open a covering dataset rather than refuse it; +/// that exposure comes with the reclamation and is inherited by whichever flag +/// takes the bit. +pub const FLAG_COVERED_INDEX_METADATA: u64 = 1 << 7; +/// Reserved for datasets that reference recognized V2 data files with +/// different exact versions. +pub const FLAG_MIXED_DATA_FILE_VERSIONS: u64 = 1 << 8; /// The first bit that is unknown as a feature flag -pub const FLAG_UNKNOWN: u64 = 64; +pub const FLAG_UNKNOWN: u64 = 1 << 8; + +// Supported flags stay below the unknown boundary; the mixed-version bit is +// reserved at the boundary until its storage contract lands. +const _: () = assert!(FLAG_COVERED_INDEX_METADATA < FLAG_UNKNOWN); +// The fence needs a bit the current released build already refuses, which means +// at or above the boundary that build shipped with (bit 7). +const _: () = assert!(FLAG_COVERED_INDEX_METADATA >= 1 << 7); +const _: () = assert!(FLAG_MIXED_DATA_FILE_VERSIONS == FLAG_UNKNOWN); + +pub(crate) const STICKY_PAIRED_FLAGS: u64 = FLAG_MIXED_DATA_FILE_VERSIONS; + +/// Environment variable that opts a release build into reading and writing data +/// overlay files before the feature is generally released. +pub const ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV: &str = "LANCE_ENABLE_UNSTABLE_DATA_OVERLAY_FILES"; /// Set the reader and writer feature flags in the manifest based on the contents of the manifest. pub fn apply_feature_flags( @@ -29,6 +76,15 @@ pub fn apply_feature_flags( enable_stable_row_id: bool, disable_transaction_file: bool, ) -> Result<()> { + // Carried across the reset: a `Manifest` only points at its index section, + // so whether any index declares covering columns is not visible here. `build_manifest` decides it from the index list it is + // committing and sets the bit after calling this; without the carry the + // second call, from `write_manifest_file`, would clear that decision + // immediately before the write. + let covered_index_metadata = (manifest.reader_feature_flags | manifest.writer_feature_flags) + & FLAG_COVERED_INDEX_METADATA; + let sticky_paired_flags = validated_sticky_paired_flags(manifest)?; + // Reset flags manifest.reader_feature_flags = 0; manifest.writer_feature_flags = 0; @@ -71,26 +127,171 @@ pub fn apply_feature_flags( manifest.writer_feature_flags |= FLAG_BASE_PATHS; } + // Overlay files change cell values on read, so a reader that ignores them + // would return stale base values. Both readers and writers must understand + // them. + let has_overlays = manifest + .fragments + .iter() + .any(|frag| !frag.overlays.is_empty()); + if has_overlays { + manifest.reader_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES; + manifest.writer_feature_flags |= FLAG_UNSTABLE_DATA_OVERLAY_FILES; + } + if disable_transaction_file { manifest.writer_feature_flags |= FLAG_DISABLE_TRANSACTION_FILE; } + + manifest.reader_feature_flags |= covered_index_metadata; + manifest.writer_feature_flags |= covered_index_metadata; + manifest.reader_feature_flags |= sticky_paired_flags; + manifest.writer_feature_flags |= sticky_paired_flags; + Ok(()) } +/// Carry sticky paired capabilities from the manifest a new one is derived +/// from. +/// +/// [`apply_feature_flags`] carries these bits across its own reset, but it only +/// ever sees one manifest. Constructors preserve these flags, and this helper +/// also validates that the source is not half-set before a derived manifest is +/// committed. +/// +/// A half-set state is refused rather than normalized: one bit set means a +/// legacy reader or a legacy writer is still permitted, which is neither mode. +pub fn inherit_sticky_feature_flags(destination: &mut Manifest, source: &Manifest) -> Result<()> { + let sticky_flags = validated_sticky_paired_flags(source)?; + destination.reader_feature_flags |= sticky_flags; + destination.writer_feature_flags |= sticky_flags; + Ok(()) +} + +/// Whether this build understands data overlay files: always in debug builds, +/// and in release builds only when [`ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV`] is set. +fn data_overlay_files_enabled() -> bool { + cfg!(debug_assertions) || std::env::var_os(ENABLE_UNSTABLE_DATA_OVERLAY_FILES_ENV).is_some() +} + +/// Clear `flag` from `flags` when its gating feature is not enabled in this +/// build; leave it set otherwise. One call per unstable flag, so support for +/// several unstable features chains cleanly. +fn mark_supported(flags: &mut u64, flag: u64, feature_enabled: bool) { + if !feature_enabled { + *flags &= !flag; + } +} + +/// The feature-flag bits this build understands, given whether overlay support +/// is enabled. Split out from [`supported_flags`] so the policy is testable +/// without toggling the build profile or environment. +fn supported_flags_when(overlay_enabled: bool) -> u64 { + let mut supported = FLAG_UNKNOWN - 1; + mark_supported( + &mut supported, + FLAG_UNSTABLE_DATA_OVERLAY_FILES, + overlay_enabled, + ); + supported +} + +fn supported_flags() -> u64 { + supported_flags_when(data_overlay_files_enabled()) +} + pub fn can_read_dataset(reader_flags: u64) -> bool { - reader_flags < FLAG_UNKNOWN + reader_flags & !supported_flags() == 0 } pub fn can_write_dataset(writer_flags: u64) -> bool { - writer_flags < FLAG_UNKNOWN + writer_flags & !supported_flags() == 0 +} + +/// Refuse reads from manifests whose required reader features this build does +/// not support or whose paired capabilities are inconsistent. +pub fn ensure_can_read_manifest(manifest: &Manifest) -> Result<()> { + validate_paired_feature_flags(manifest)?; + if !can_read_dataset(manifest.reader_feature_flags) { + return Err(Error::not_supported_source( + format!( + "This dataset cannot be read by this version of Lance. Please upgrade \ + Lance to read this dataset. Flags: {}", + manifest.reader_feature_flags + ) + .into(), + )); + } + Ok(()) +} + +/// Refuse writes to manifests whose required writer features this build does +/// not support or whose paired capabilities are inconsistent. +pub fn ensure_can_write_manifest(manifest: &Manifest) -> Result<()> { + validate_paired_feature_flags(manifest)?; + if !can_write_dataset(manifest.writer_feature_flags) { + return Err(Error::not_supported_source( + format!( + "This dataset cannot be written by this version of Lance. Please upgrade \ + Lance to write this dataset. Flags: {}", + manifest.writer_feature_flags + ) + .into(), + )); + } + Ok(()) } pub fn has_deprecated_v2_feature_flag(writer_flags: u64) -> bool { writer_flags & FLAG_USE_V2_FORMAT_DEPRECATED != 0 } +/// Refuse a manifest whose paired reader and writer capability bits disagree. +/// +/// One word set and the other not is neither mode: it would let a legacy reader +/// or a legacy writer through on a table where the other half is enforcing. The +/// commit path refuses to *produce* this, so seeing it on read means the +/// manifest was written by something that did not. +pub fn validate_paired_feature_flags(manifest: &Manifest) -> Result<()> { + let reader = manifest.reader_feature_flags & FLAG_MIXED_DATA_FILE_VERSIONS != 0; + let writer = manifest.writer_feature_flags & FLAG_MIXED_DATA_FILE_VERSIONS != 0; + if reader != writer { + return Err(Error::corrupt_file_named( + "manifest", + "Manifest has only one of the mixed data-file-version reader and writer feature bits set, \ + so its semantics are undefined", + )); + } + Ok(()) +} + +fn validated_sticky_paired_flags(manifest: &Manifest) -> Result { + validate_paired_feature_flags(manifest)?; + Ok(manifest.reader_feature_flags & STICKY_PAIRED_FLAGS) +} + #[cfg(test)] mod tests { + /// The covering fence only works if the bit is one the current released + /// build already rejects. That build's unknown boundary is 128, so the bit + /// has to be 128 and this build has to have moved its own boundary past it + /// -- otherwise either that build accepts a covering dataset, or we refuse + /// our own. + #[test] + fn test_covered_index_metadata_fences_older_builds_only() { + assert_eq!( + FLAG_COVERED_INDEX_METADATA, 128, + "the fence must sit on the boundary the released build shipped with" + ); + assert!( + can_read_dataset(FLAG_COVERED_INDEX_METADATA), + "this build implements covering, so it must accept its own datasets" + ); + assert!(can_write_dataset(FLAG_COVERED_INDEX_METADATA)); + // A build whose boundary is still 128 refuses the bit, which is the fence; + // the module-level `const _` assertion keeps it at or above that boundary. + } + use super::*; use crate::format::BasePath; @@ -103,6 +304,13 @@ mod tests { assert!(can_read_dataset(super::FLAG_TABLE_CONFIG)); assert!(can_read_dataset(super::FLAG_BASE_PATHS)); assert!(can_read_dataset(super::FLAG_DISABLE_TRANSACTION_FILE)); + // Overlay support is gated on the build profile / env opt-in, so the + // flag is readable exactly when overlays are enabled (see + // test_data_overlay_flag_release_gating for the full policy). + assert_eq!( + can_read_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES), + data_overlay_files_enabled() + ); assert!(can_read_dataset( super::FLAG_DELETION_FILES | super::FLAG_STABLE_ROW_IDS @@ -111,6 +319,58 @@ mod tests { assert!(!can_read_dataset(super::FLAG_UNKNOWN)); } + #[test] + fn test_data_overlay_flag_release_gating() { + // Release default (overlays disabled): the overlay flag is treated as + // unknown so the dataset is refused, while other known flags still pass. + let supported = supported_flags_when(false); + assert_eq!(supported & FLAG_UNSTABLE_DATA_OVERLAY_FILES, 0); + assert_eq!(FLAG_DELETION_FILES & !supported, 0); + assert_ne!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0); + // Enabled (debug or env opt-in): the overlay flag is understood. + let supported = supported_flags_when(true); + assert_eq!(FLAG_UNSTABLE_DATA_OVERLAY_FILES & !supported, 0); + } + + #[test] + fn test_apply_feature_flags_sets_overlay_flag() { + use crate::format::overlay::{DataOverlayFile, OverlayCoverage}; + use crate::format::{DataFile, DataStorageFormat, Fragment}; + use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use roaring::RoaringBitmap; + use std::collections::HashMap; + use std::sync::Arc; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "id", + arrow_schema::DataType::Int64, + false, + )]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + let mut fragment = Fragment::new(0); + fragment.overlays = vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }]; + let mut manifest = Manifest::new( + schema, + Arc::new(vec![fragment]), + DataStorageFormat::default(), + HashMap::new(), + ); + apply_feature_flags(&mut manifest, false, false).unwrap(); + assert_ne!( + manifest.reader_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES, + 0 + ); + assert_ne!( + manifest.writer_feature_flags & FLAG_UNSTABLE_DATA_OVERLAY_FILES, + 0 + ); + } + #[test] fn test_write_check() { assert!(can_write_dataset(0)); @@ -120,6 +380,13 @@ mod tests { assert!(can_write_dataset(super::FLAG_TABLE_CONFIG)); assert!(can_write_dataset(super::FLAG_BASE_PATHS)); assert!(can_write_dataset(super::FLAG_DISABLE_TRANSACTION_FILE)); + // Overlay support is gated on the build profile / env opt-in, so the + // flag is writable exactly when overlays are enabled (see + // test_data_overlay_flag_release_gating for the full policy). + assert_eq!( + can_write_dataset(super::FLAG_UNSTABLE_DATA_OVERLAY_FILES), + data_overlay_files_enabled() + ); assert!(can_write_dataset( super::FLAG_DELETION_FILES | super::FLAG_STABLE_ROW_IDS @@ -181,4 +448,107 @@ mod tests { 0 ); } + #[test] + fn inheriting_carries_sticky_paired_bits_from_the_source() { + let mut source = empty_manifest(); + source.reader_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + source.writer_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + // A fresh destination models any derived manifest before inheritance. + let mut destination = empty_manifest(); + + inherit_sticky_feature_flags(&mut destination, &source).unwrap(); + + assert_ne!( + destination.reader_feature_flags & FLAG_MIXED_DATA_FILE_VERSIONS, + 0 + ); + assert_ne!( + destination.writer_feature_flags & FLAG_MIXED_DATA_FILE_VERSIONS, + 0 + ); + } + + #[test] + fn inheriting_refuses_a_half_set_source() { + for (reader, writer) in [ + (FLAG_MIXED_DATA_FILE_VERSIONS, 0), + (0, FLAG_MIXED_DATA_FILE_VERSIONS), + ] { + let mut source = empty_manifest(); + source.reader_feature_flags = reader; + source.writer_feature_flags = writer; + let mut destination = empty_manifest(); + + let err = inherit_sticky_feature_flags(&mut destination, &source).unwrap_err(); + + assert!(err.to_string().contains("only one of"), "{err}"); + } + } + + #[test] + fn apply_feature_flags_carries_sticky_paired_bits_across_its_reset() { + let mut manifest = empty_manifest(); + manifest.reader_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + manifest.writer_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + + apply_feature_flags(&mut manifest, false, false).unwrap(); + + assert_ne!( + manifest.reader_feature_flags & FLAG_MIXED_DATA_FILE_VERSIONS, + 0 + ); + assert_ne!( + manifest.writer_feature_flags & FLAG_MIXED_DATA_FILE_VERSIONS, + 0 + ); + } + + #[test] + fn apply_feature_flags_rejects_half_set_sticky_bits() { + let mut manifest = empty_manifest(); + manifest.reader_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + + let err = apply_feature_flags(&mut manifest, false, false).unwrap_err(); + + assert!(matches!(err, Error::CorruptFile { .. })); + assert!(err.to_string().contains("only one of"), "{err}"); + } + + #[test] + fn writer_gate_rejects_reserved_mixed_capability() { + let mut manifest = empty_manifest(); + manifest.reader_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + manifest.writer_feature_flags = FLAG_MIXED_DATA_FILE_VERSIONS; + + let err = ensure_can_write_manifest(&manifest).unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + assert!(err.to_string().contains("cannot be written"), "{err}"); + } + + fn empty_manifest() -> Manifest { + use crate::format::DataStorageFormat; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use std::collections::HashMap; + use std::sync::Arc; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); + Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::default(), + HashMap::new(), + ) + } + + /// A build that does not know the bit must refuse the table rather than + /// continue with legacy semantics. + #[test] + fn mixed_capability_remains_at_the_unknown_boundary() { + assert!(can_read_dataset(FLAG_COVERED_INDEX_METADATA)); + assert!(can_write_dataset(FLAG_COVERED_INDEX_METADATA)); + assert!(!can_read_dataset(FLAG_MIXED_DATA_FILE_VERSIONS)); + assert!(!can_write_dataset(FLAG_MIXED_DATA_FILE_VERSIONS)); + assert_eq!(FLAG_MIXED_DATA_FILE_VERSIONS, FLAG_UNKNOWN); + } } diff --git a/rust/lance-table/src/format.rs b/rust/lance-table/src/format.rs index 842c76f1e58..1c9e0e37c8c 100644 --- a/rust/lance-table/src/format.rs +++ b/rust/lance-table/src/format.rs @@ -6,7 +6,10 @@ use uuid::Uuid; mod fragment; mod index; +pub mod key_existence; mod manifest; +pub mod overlay; +mod row_ids; mod transaction; pub use crate::rowids::version::{ @@ -16,10 +19,12 @@ pub use fragment::*; pub use index::{IndexFile, IndexMetadata, index_metadata_codec, list_index_files_with_sizes}; pub use manifest::{ - BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, SelfDescribingFileReader, - WriterVersion, is_detached_version, + BasePath, DETACHED_VERSION_MASK, DataStorageFormat, Manifest, ManifestBuildConfig, + SelfDescribingFileReader, WriterVersion, is_detached_version, + populate_manifest_schema_dictionaries, }; -pub use transaction::Transaction; +pub use row_ids::{ExternalFile, InlineRowIds, RowIdMeta}; +pub use transaction::{Transaction, operation_may_change_schema}; use lance_core::{Error, Result}; diff --git a/rust/lance-table/src/format/fragment.rs b/rust/lance-table/src/format/fragment.rs index 431e466dbd4..82afee22c1d 100644 --- a/rust/lance-table/src/format/fragment.rs +++ b/rust/lance-table/src/format/fragment.rs @@ -7,12 +7,13 @@ use std::sync::Arc; use lance_core::Error; use lance_core::deepsize::DeepSizeOf; -use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; -use lance_file::version::LanceFileVersion; +use lance_file::version::ConcreteFileVersion; use lance_io::utils::CachedFileSize; use object_store::path::Path; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use super::overlay::{DataOverlayFile, TOMBSTONE_FIELD_ID, sort_overlays_newest_last}; +use super::row_ids::{ExternalFile, RowIdMeta}; use crate::format::pb; use crate::rowids::version::{ @@ -36,7 +37,7 @@ pub struct DataFile { pub fields: Arc<[i32]>, /// The offsets of the fields listed in `fields`, empty in v1 files /// - /// Note that -1 is a possibility and it indices that the field has + /// Note that -1 is a possibility and it indicates that the field has /// no top-level column in the file. /// /// Columns that lack a field id may still exist as extra entries in @@ -103,15 +104,16 @@ impl<'de> Deserialize<'de> for DataFile { } impl DataFile { + /// Create a `DataFile` and encode its exact format version for manifest storage. pub fn new( path: impl Into, fields: Vec, column_indices: Vec, - file_major_version: u32, - file_minor_version: u32, + file_version: ConcreteFileVersion, file_size_bytes: Option>, base_id: Option, ) -> Self { + let (file_major_version, file_minor_version) = file_version.to_data_file_numbers(); Self { path: path.into(), fields: Arc::from(fields), @@ -123,12 +125,9 @@ impl DataFile { } } - /// Create a new `DataFile` with the expectation that fields and column_indices will be set later - pub fn new_unstarted( - path: impl Into, - file_major_version: u32, - file_minor_version: u32, - ) -> Self { + /// Create a new `DataFile` whose fields and column indices will be set later. + pub fn new_unstarted(path: impl Into, file_version: ConcreteFileVersion) -> Self { + let (file_major_version, file_minor_version) = file_version.to_data_file_numbers(); Self { path: path.into(), fields: Arc::from([]), @@ -145,15 +144,7 @@ impl DataFile { fields: Vec, base_id: Option, ) -> Self { - Self::new( - path, - fields, - vec![], - MAJOR_VERSION as u32, - MINOR_VERSION as u32, - None, - base_id, - ) + Self::new(path, fields, vec![], ConcreteFileVersion::V1, None, base_id) } pub fn new_legacy( @@ -168,8 +159,7 @@ impl DataFile { path, field_ids, vec![], - MAJOR_VERSION as u32, - MINOR_VERSION as u32, + ConcreteFileVersion::V1, file_size_bytes, base_id, ) @@ -179,13 +169,30 @@ impl DataFile { full_schema.project_by_ids(&self.fields, false) } - pub fn is_legacy_file(&self) -> bool { + fn uses_v1_data_file_encoding(&self) -> bool { self.file_major_version == 0 && self.file_minor_version < 3 } + /// Decode the exact file version stored in this `DataFile` metadata. + pub fn file_version(&self) -> Result { + ConcreteFileVersion::from_data_file_numbers( + self.file_major_version, + self.file_minor_version, + ) + } + pub fn validate(&self, base_path: &Path) -> Result<()> { - if self.is_legacy_file() { - if !self.fields.windows(2).all(|w| w[0] < w[1]) { + if self.uses_v1_data_file_encoding() { + // A tombstone marks a field superseded by a later data file. It is + // not a field id, so it carries no ordering; the live ids around it + // must still be sorted and distinct. + let live: Vec = self + .fields + .iter() + .copied() + .filter(|field| *field != TOMBSTONE_FIELD_ID) + .collect(); + if !live.windows(2).all(|w| w[0] < w[1]) { return Err(Error::corrupt_file( base_path.clone().join(self.path.clone()), "contained unsorted or duplicate field ids", @@ -375,6 +382,15 @@ impl DataFileFieldInterner { .into_iter() .map(|f| self.intern_data_file(f)) .collect::>()?, + overlays: { + let mut overlays = p + .overlays + .into_iter() + .map(DataOverlayFile::try_from) + .collect::>>()?; + sort_overlays_newest_last(&mut overlays); + overlays + }, deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?, row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?, physical_rows, @@ -439,38 +455,6 @@ impl TryFrom for DeletionFile { } } -/// A reference to a part of a file. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct ExternalFile { - pub path: String, - pub offset: u64, - pub size: u64, -} - -/// Metadata about location of the row id sequence. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub enum RowIdMeta { - Inline(Vec), - External(ExternalFile), -} - -impl TryFrom for RowIdMeta { - type Error = Error; - - fn try_from(value: pb::data_fragment::RowIdSequence) -> Result { - match value { - pb::data_fragment::RowIdSequence::InlineRowIds(data) => Ok(Self::Inline(data)), - pb::data_fragment::RowIdSequence::ExternalRowIds(file) => { - Ok(Self::External(ExternalFile { - path: file.path.clone(), - offset: file.offset, - size: file.size, - })) - } - } - } -} - /// Data fragment. /// /// A fragment is a set of files which represent the different columns of the same rows. @@ -483,6 +467,12 @@ pub struct Fragment { /// Files within the fragment. pub files: Vec, + /// Overlay files supplying new values for a subset of cells without + /// rewriting the base data files. Order is significant: a later entry is + /// newer than an earlier one. See [`DataOverlayFile`] for resolution rules. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub overlays: Vec, + /// Optional file with deleted local row offsets. #[serde(skip_serializing_if = "Option::is_none")] pub deletion_file: Option, @@ -510,6 +500,7 @@ impl Fragment { Self { id, files: vec![], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -534,6 +525,52 @@ impl Fragment { } } + /// Every Lance-format file this fragment references: the base data files + /// plus the data file of each overlay. The fragment's other referenced + /// files are not in this format: a deletion file is `.arrow` or `.bin`, and + /// external row-id or row-version metadata is a raw byte range. + /// + /// Prefer this over `files`, which is the base data files only and so omits + /// overlays. + pub fn referenced_lance_files(&self) -> impl Iterator + '_ { + // Destructured on purpose: a new field here fails to compile until + // someone decides whether it references files. + let Self { + id: _, + files, + overlays, + deletion_file: _, + row_id_meta: _, + physical_rows: _, + last_updated_at_version_meta: _, + created_at_version_meta: _, + } = self; + files + .iter() + .chain(overlays.iter().map(|overlay| &overlay.data_file)) + } + + /// Mutable counterpart of [`Self::referenced_lance_files`], for rewriting + /// the fields a clone has to normalize (`base_id`) across base and overlay + /// files alike. + pub fn referenced_lance_files_mut(&mut self) -> impl Iterator + '_ { + // Destructured for the same reason as `referenced_lance_files`, and so + // the two disjoint field borrows are visible to the borrow checker. + let Self { + id: _, + files, + overlays, + deletion_file: _, + row_id_meta: _, + physical_rows: _, + last_updated_at_version_meta: _, + created_at_version_meta: _, + } = self; + files + .iter_mut() + .chain(overlays.iter_mut().map(|overlay| &mut overlay.data_file)) + } + pub fn from_json(json: &str) -> Result { let fragment: Self = serde_json::from_str(json)?; Ok(fragment) @@ -549,6 +586,7 @@ impl Fragment { Self { id, files: vec![DataFile::new_legacy(path, schema, None, None)], + overlays: vec![], deletion_file: None, physical_rows, row_id_meta: None, @@ -562,16 +600,14 @@ impl Fragment { path: impl Into, field_ids: Vec, column_indices: Vec, - version: &LanceFileVersion, + version: ConcreteFileVersion, file_size_bytes: Option>, ) -> Self { - let (major, minor) = version.to_numbers(); let data_file = DataFile::new( path, field_ids, column_indices, - major, - minor, + version, file_size_bytes, None, ); @@ -589,16 +625,14 @@ impl Fragment { path: impl Into, field_ids: Vec, column_indices: Vec, - version: &LanceFileVersion, + version: ConcreteFileVersion, file_size_bytes: Option>, ) { - let (major, minor) = version.to_numbers(); self.files.push(DataFile::new( path, field_ids, column_indices, - major, - minor, + version, file_size_bytes, None, )); @@ -610,17 +644,11 @@ impl Fragment { .push(DataFile::new_legacy(path, schema, None, None)); } - // True if this fragment is made up of legacy v1 files, false otherwise - pub fn has_legacy_files(&self) -> bool { - // If any file in a fragment is legacy then all files in the fragment must be - self.files[0].is_legacy_file() - } - // Helper method to infer the Lance version from a set of fragments // // Returns None if there are no data files // Returns an error if the data files have different versions - pub fn try_infer_version(fragments: &[Self]) -> Result> { + pub fn try_infer_version(fragments: &[Self]) -> Result> { // Otherwise we need to check the actual file versions // Determine version from first file let Some(sample_file) = fragments @@ -630,17 +658,11 @@ impl Fragment { else { return Ok(None); }; - let file_version = LanceFileVersion::try_from_major_minor( - sample_file.file_major_version, - sample_file.file_minor_version, - )?; + let file_version = sample_file.file_version()?; // Ensure all files match for frag in fragments { for file in &frag.files { - let this_file_version = LanceFileVersion::try_from_major_minor( - file.file_major_version, - file.file_minor_version, - )?; + let this_file_version = file.file_version()?; if file_version != this_file_version { return Err(Error::invalid_input(format!( "All data files must have the same version. Detected both {} and {}", @@ -669,6 +691,15 @@ impl TryFrom for Fragment { .into_iter() .map(DataFile::try_from) .collect::>()?, + overlays: { + let mut overlays = p + .overlays + .into_iter() + .map(DataOverlayFile::try_from) + .collect::>>()?; + sort_overlays_newest_last(&mut overlays); + overlays + }, deletion_file: p.deletion_file.map(DeletionFile::try_from).transpose()?, row_id_meta: p.row_id_sequence.map(RowIdMeta::try_from).transpose()?, physical_rows, @@ -701,7 +732,9 @@ impl From<&Fragment> for pb::DataFragment { }); let row_id_sequence = f.row_id_meta.as_ref().map(|m| match m { - RowIdMeta::Inline(data) => pb::data_fragment::RowIdSequence::InlineRowIds(data.clone()), + RowIdMeta::Inline(data) => { + pb::data_fragment::RowIdSequence::InlineRowIds(data.to_vec()) + } RowIdMeta::External(file) => { pb::data_fragment::RowIdSequence::ExternalRowIds(pb::ExternalFile { path: file.path.clone(), @@ -716,6 +749,7 @@ impl From<&Fragment> for pb::DataFragment { Self { id: f.id, files: f.files.iter().map(pb::DataFile::from).collect(), + overlays: f.overlays.iter().map(pb::DataOverlayFile::from).collect(), deletion_file, row_id_sequence, physical_rows: f.physical_rows.unwrap_or_default() as u64, @@ -728,12 +762,109 @@ impl From<&Fragment> for pb::DataFragment { #[cfg(test)] mod tests { use super::*; + use crate::format::overlay::OverlayCoverage; use arrow_schema::{ DataType, Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema, }; + use lance_file::format::{MAJOR_VERSION, MINOR_VERSION}; use object_store::path::Path; + use roaring::RoaringBitmap; use serde_json::{Value, json}; + #[test] + fn test_data_overlay_roundtrip() { + // A fragment carrying a dense overlay round-trips through protobuf and + // back, and the parsed coverage bitmap is recovered per field. + let mut bitmap = RoaringBitmap::new(); + bitmap.insert(1); + bitmap.insert(3); + + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None), + coverage: OverlayCoverage::dense(bitmap.clone()), + committed_version: 7, + }; + let mut fragment = Fragment::new(0); + fragment.files = vec![DataFile::new_legacy_from_fields( + "base.lance", + vec![1, 3], + None, + )]; + fragment.overlays = vec![overlay]; + + let proto = pb::DataFragment::from(&fragment); + assert_eq!(proto.overlays.len(), 1); + let round_tripped = Fragment::try_from(proto).unwrap(); + assert_eq!(round_tripped, fragment); + + // Dense coverage applies to every field. + let recovered = round_tripped.overlays[0].coverage_for_field(0).unwrap(); + assert_eq!(*recovered, bitmap); + assert_eq!( + *round_tripped.overlays[0].coverage_for_field(5).unwrap(), + bitmap + ); + } + + #[test] + fn test_data_overlay_sparse_per_field_coverage() { + // A sparse overlay carries one bitmap per field, recovered by position. + let name_coverage = RoaringBitmap::from_iter([2u32, 3]); + let embedding_coverage = RoaringBitmap::from_iter([1u32]); + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay-1.lance", vec![2, 4], None), + coverage: OverlayCoverage::sparse(vec![ + name_coverage.clone(), + embedding_coverage.clone(), + ]), + committed_version: 3, + }; + let mut fragment = Fragment::new(1); + fragment.overlays = vec![overlay]; + + let round_tripped = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap(); + assert_eq!( + *round_tripped.overlays[0].coverage_for_field(0).unwrap(), + name_coverage + ); + assert_eq!( + *round_tripped.overlays[0].coverage_for_field(1).unwrap(), + embedding_coverage + ); + } + + #[test] + fn test_overlays_sorted_newest_last_on_load() { + // Overlays load stable-sorted by committed_version (newest last), with + // list position preserved as the tiebreak for equal versions. + let mk = |version: u64, field: i32| DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: version, + }; + let mut fragment = Fragment::new(0); + // Written out of order: v5, v2, v2 (second), v3. + fragment.overlays = vec![mk(5, 1), mk(2, 2), mk(2, 3), mk(3, 4)]; + + let loaded = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap(); + let versions: Vec = loaded + .overlays + .iter() + .map(|o| o.committed_version) + .collect(); + assert_eq!(versions, vec![2, 2, 3, 5]); + // Stable: the two v2 overlays keep their original relative order (field 2 + // before field 3). + assert_eq!( + loaded.overlays[0].data_file.fields.as_ref(), + [2i32].as_slice() + ); + assert_eq!( + loaded.overlays[1].data_file.fields.as_ref(), + [3i32].as_slice() + ); + } + #[test] fn test_new_fragment() { let path = "foobar.lance"; @@ -786,6 +917,56 @@ mod tests { assert_eq!(fragment, fragment2); } + #[test] + fn infer_exact_file_version_and_reject_mixed_fragments() { + assert_eq!(Fragment::try_infer_version(&[]).unwrap(), None); + + let v2_0 = Fragment::new(0).with_file( + "v2_0.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + ); + assert_eq!( + ( + v2_0.files[0].file_major_version, + v2_0.files[0].file_minor_version + ), + (2, 0) + ); + let unstarted = DataFile::new_unstarted("unstarted.lance", ConcreteFileVersion::V2_0); + assert_eq!( + (unstarted.file_major_version, unstarted.file_minor_version), + (2, 0) + ); + let v2_0_second = Fragment::new(1).with_file( + "v2_0_second.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + ); + assert_eq!( + Fragment::try_infer_version(&[v2_0.clone(), v2_0_second]).unwrap(), + Some(ConcreteFileVersion::V2_0) + ); + + let v2_1 = Fragment::new(2).with_file( + "v2_1.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_1, + None, + ); + let error = Fragment::try_infer_version(&[v2_0, v2_1]).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + let message = error.to_string(); + assert!(message.contains("All data files must have the same version")); + assert!(message.contains("2.0")); + assert!(message.contains("2.1")); + } + #[test] fn test_to_json() { let mut fragment = Fragment::new(123); diff --git a/rust/lance-table/src/format/index.rs b/rust/lance-table/src/format/index.rs index f603536a3eb..87ae62ddea8 100644 --- a/rust/lance-table/src/format/index.rs +++ b/rust/lance-table/src/format/index.rs @@ -3,7 +3,7 @@ //! Metadata for index -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use chrono::{DateTime, Utc}; @@ -34,8 +34,22 @@ pub struct IndexMetadata { pub uuid: Uuid, /// Fields to build the index. + /// + /// `fields[0]` is always a column the index is keyed on. Trailing entries + /// may instead be merely carried, not keyed on -- see [`Self::covering_fields`]. pub fields: Vec, + /// Fields whose values this index carries but is not keyed on. + /// + /// Always a suffix of [`Self::fields`], and never all of it, so + /// `fields[0]` is always a column the index is keyed on. Empty for an + /// index that carries no extra columns. + /// + /// These ids also appear in [`Self::fields`]. That is deliberate: every + /// consumer that reads `fields` as the index's dependency set then covers + /// them with no change. + pub covering_fields: Vec, + /// Human readable index name pub name: String, @@ -119,12 +133,113 @@ impl IndexMetadata { let fragment_bitmap = self.fragment_bitmap.as_ref()?; Some(fragment_bitmap - existing_fragments) } + + /// True when the index reports matches as physical row addresses rather than row ids + /// (`ScalarIndex::results_are_row_addresses`). + /// + /// Such an index cannot follow its data through a rewrite: the addresses it stores + /// name fragments and offsets, and neither kind supports remap. + pub fn results_are_row_addrs(&self) -> bool { + self.index_details.as_ref().is_some_and(|details| { + details.type_url.ends_with("ZoneMapIndexDetails") + || details.type_url.ends_with("BloomFilterIndexDetails") + }) + } + + /// The prefix of [`Self::fields`] this index is keyed on, with the carried + /// columns of [`Self::covering_fields`] removed. + /// + /// Only this prefix decides which column an index answers for; the full + /// `fields` vector answers what invalidates it. Empty for a system index + /// that declares no fields, and empty for a declaration longer than + /// `fields`: decoding validates, but metadata built by a caller this build + /// never validated does not, and failing closed beats an underflow. + pub fn keyed_fields(&self) -> &[i32] { + let keyed = self.fields.len().saturating_sub(self.covering_fields.len()); + &self.fields[..keyed] + } + + /// The single column this index is keyed on, or `None` when it is keyed on + /// several -- a genuinely composite index -- or on none at all. + /// + /// Most selection paths are only defined for one keyed column, so they can + /// compare this against the column they are resolving. + pub fn keyed_field(&self) -> Option { + match self.keyed_fields() { + [only] => Some(*only), + _ => None, + } + } + + /// Check the covering declaration against [`Self::fields`]. + /// + /// Carried columns must be a suffix of `fields` and must not consume all of + /// it, so `fields[0]` is always a column the index is keyed on. An empty + /// declaration is always valid, which is what keeps the system indices -- + /// `mem_wal` and `frag_reuse`, both of which commit no fields at all -- + /// passing this check. + /// + /// The rules are checked from most to least specific, because one bad + /// declaration usually trips several: an id that is not a field at all is + /// reported ahead of the length and ordering rules, which would otherwise + /// name a consequence instead of the cause. + pub fn validate_covering_fields(&self) -> Result<()> { + if self.covering_fields.is_empty() { + return Ok(()); + } + + let missing: Vec = self + .covering_fields + .iter() + .copied() + .filter(|f| !self.fields.contains(f)) + .collect(); + if !missing.is_empty() { + return Err(Error::invalid_input(format!( + "index '{}' declares covering fields {:?} but {:?} are not \ + among its fields {:?}", + self.name, self.covering_fields, missing, self.fields, + ))); + } + + // A column carried twice would be projected twice. The suffix check + // below cannot stand in for this, because `fields` may repeat the id in + // the same positions -- `fields = [7, 11, 11]` has `[11, 11]` as a + // genuine tail. + let mut seen = HashSet::with_capacity(self.covering_fields.len()); + if let Some(duplicate) = self.covering_fields.iter().find(|f| !seen.insert(**f)) { + return Err(Error::invalid_input(format!( + "index '{}' declares covering field {} more than once in {:?}", + self.name, duplicate, self.covering_fields, + ))); + } + + if self.covering_fields.len() >= self.fields.len() { + return Err(Error::invalid_input(format!( + "index '{}' declares covering fields {:?} but its fields are {:?}; \ + at least one field must remain indexed", + self.name, self.covering_fields, self.fields, + ))); + } + + let suffix_start = self.fields.len() - self.covering_fields.len(); + if self.fields[suffix_start..] != self.covering_fields[..] { + return Err(Error::invalid_input(format!( + "index '{}' declares covering fields {:?} which are not the trailing \ + entries of {:?}; covering fields must come last", + self.name, self.covering_fields, self.fields, + ))); + } + + Ok(()) + } } impl DeepSizeOf for IndexMetadata { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { self.uuid.as_bytes().deep_size_of_children(context) + self.fields.deep_size_of_children(context) + + self.covering_fields.deep_size_of_children(context) + self.name.deep_size_of_children(context) + self.dataset_version.deep_size_of_children(context) + self @@ -163,12 +278,13 @@ impl TryFrom for IndexMetadata { ) }; - Ok(Self { + let metadata = Self { uuid: proto.uuid.as_ref().map(Uuid::try_from).ok_or_else(|| { Error::invalid_input("uuid field does not exist in Index metadata".to_string()) })??, name: proto.name, fields: proto.fields, + covering_fields: proto.covering_fields, dataset_version: proto.dataset_version, fragment_bitmap, index_details: proto.index_details.map(Arc::new), @@ -179,20 +295,37 @@ impl TryFrom for IndexMetadata { }), base_id: proto.base_id, files, - }) + }; + + // This is the single boundary between manifest bytes and + // `IndexMetadata`, so validating once here is what lets every reader + // treat the declaration as a trailing slice of `fields`. A manifest + // that fails this was written by something that did not follow the + // format contract; refuse it rather than let each use site quietly + // ignore the index. + metadata.validate_covering_fields()?; + + Ok(metadata) } } impl From<&IndexMetadata> for pb::IndexMetadata { fn from(idx: &IndexMetadata) -> Self { let mut fragment_bitmap = Vec::new(); - if let Some(bitmap) = &idx.fragment_bitmap - && let Err(e) = bitmap.serialize_into(&mut fragment_bitmap) - { - // In theory, this should never error. But if we do, just - // recover gracefully. - log::error!("Failed to serialize fragment bitmap: {}", e); - fragment_bitmap.clear(); + if let Some(bitmap) = &idx.fragment_bitmap { + // Fragment ids are allocated monotonically, so index coverage is + // highly contiguous. Run containers are part of the standard + // roaring serialization format, so converting eligible containers + // to runs before writing shrinks the bitmap from O(fragments) to + // O(runs) bytes. + let mut bitmap = bitmap.clone(); + bitmap.optimize(); + if let Err(e) = bitmap.serialize_into(&mut fragment_bitmap) { + // In theory, this should never error. But if we do, just + // recover gracefully. + log::error!("Failed to serialize fragment bitmap: {}", e); + fragment_bitmap.clear(); + } } let files = idx @@ -213,6 +346,7 @@ impl From<&IndexMetadata> for pb::IndexMetadata { uuid: Some((&idx.uuid).into()), name: idx.name.clone(), fields: idx.fields.clone(), + covering_fields: idx.covering_fields.clone(), dataset_version: idx.dataset_version, fragment_bitmap, index_details: idx @@ -303,8 +437,41 @@ pub async fn list_index_files_with_sizes( #[cfg(test)] mod tests { use super::*; + use rstest::rstest; use std::collections::HashMap; + #[test] + fn test_fragment_bitmap_serialized_run_optimized() { + let bitmap = RoaringBitmap::from_sorted_iter(0..1_000_000).unwrap(); + let unoptimized_size = bitmap.serialized_size(); + + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + name: "my_index".to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap: Some(bitmap.clone()), + index_details: None, + index_version: 1, + created_at: None, + base_id: None, + files: None, + }; + + let proto = pb::IndexMetadata::from(&metadata); + assert!( + proto.fragment_bitmap.len() < unoptimized_size / 100, + "expected run-optimized bitmap ({} bytes) to be <1% of the \ + unoptimized serialization ({} bytes)", + proto.fragment_bitmap.len(), + unoptimized_size + ); + + let recovered = IndexMetadata::try_from(proto).unwrap(); + assert_eq!(recovered.fragment_bitmap, Some(bitmap)); + } + /// Demonstrates the pattern a disk-backed cache backend would use: /// serialize entries to bytes, store in a key-value map, then /// deserialize on retrieval. @@ -317,6 +484,7 @@ mod tests { uuid: Uuid::new_v4(), name: "my_index".to_string(), fields: vec![0, 1], + covering_fields: vec![], dataset_version: 42, fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2, 3])), index_details: None, @@ -332,6 +500,7 @@ mod tests { uuid: Uuid::new_v4(), name: "second_index".to_string(), fields: vec![2], + covering_fields: vec![], dataset_version: 43, fragment_bitmap: None, index_details: None, @@ -374,4 +543,141 @@ mod tests { assert_eq!(orig.files, rec.files); } } + + /// The covering declaration must survive both conversion directions. + /// A dropped `covering_fields` would leave index files holding carried + /// columns the manifest no longer names. + #[test] + fn test_covering_fields_survives_proto_roundtrip() { + let original = IndexMetadata { + uuid: Uuid::new_v4(), + name: "covered".to_string(), + fields: vec![7, 11, 13], + covering_fields: vec![11, 13], + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + + let proto = pb::IndexMetadata::from(&original); + assert_eq!(proto.covering_fields, vec![11, 13]); + + let recovered = IndexMetadata::try_from(proto).unwrap(); + assert_eq!(recovered, original); + } + + /// `TryFrom` is the only path from manifest bytes to + /// `IndexMetadata`, so validating here is what lets every reader downstream + /// assume the declaration really is a trailing slice of `fields`. Without + /// it, a malformed declaration reaches each use site instead, where the + /// keyed count saturates to zero and the index is silently ignored. + #[test] + fn test_try_from_proto_rejects_a_malformed_covering_declaration() { + let mut proto = pb::IndexMetadata::from(&index_metadata_with(vec![7, 11], vec![11])); + // The leading entry, not the trailing one: claims the keyed column is + // carried. + proto.covering_fields = vec![7]; + + let err = IndexMetadata::try_from(proto) + .expect_err("a malformed covering declaration must not decode"); + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected InvalidInput, got {:?}", + err + ); + assert!( + err.to_string().contains("must come last"), + "unexpected message: {err}" + ); + } + + /// Only the keyed prefix decides which column an index answers for, and + /// nearly every selection path needs exactly one such column. Metadata read + /// from a manifest this build never wrote can still be malformed, so both + /// accessors must fail closed -- no keyed field -- rather than underflow. + #[rstest] + #[case::not_covered(vec![7], vec![], vec![7], Some(7))] + #[case::covered(vec![7, 11], vec![11], vec![7], Some(7))] + #[case::covered_multi(vec![7, 11, 13], vec![11, 13], vec![7], Some(7))] + #[case::covered_composite(vec![7, 11, 13], vec![13], vec![7, 11], None)] + #[case::composite(vec![7, 11], vec![], vec![7, 11], None)] + #[case::system_index_no_fields(vec![], vec![], vec![], None)] + #[case::malformed_longer_than_fields(vec![7], vec![11, 13], vec![], None)] + fn test_keyed_fields( + #[case] fields: Vec, + #[case] covering_fields: Vec, + #[case] expected_keyed: Vec, + #[case] expected_single: Option, + ) { + let metadata = index_metadata_with(fields, covering_fields); + + assert_eq!(metadata.keyed_fields(), expected_keyed.as_slice()); + assert_eq!(metadata.keyed_field(), expected_single); + } + + fn index_metadata_with(fields: Vec, covering_fields: Vec) -> IndexMetadata { + IndexMetadata { + uuid: Uuid::new_v4(), + name: "idx".to_string(), + fields, + covering_fields, + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + #[rstest] + #[case::empty_is_valid(vec![7], vec![], None)] + // mem_wal and frag_reuse both commit no fields at all; a bare + // `covering_fields.len() < fields.len()` check would reject them. + #[case::system_index_no_fields(vec![], vec![], None)] + #[case::valid_single_covered(vec![7, 11], vec![11], None)] + #[case::valid_suffix(vec![7, 11, 13], vec![11, 13], None)] + #[case::not_a_suffix(vec![7, 11, 13], vec![11], Some("must come last"))] + #[case::not_a_subset(vec![7, 11], vec![99], Some("are not among its fields"))] + #[case::wrong_order(vec![7, 11, 13], vec![13, 11], Some("must come last"))] + #[case::all_fields_covered(vec![7], vec![7], Some("at least one field must remain indexed"))] + #[case::covers_the_search_key(vec![7, 11], vec![7, 11], Some("at least one field must remain indexed"))] + // `fields` repeats the id in the same positions, so `[11, 11]` is a genuine + // tail of it -- only the duplicate check rejects this. + #[case::duplicate_covered(vec![7, 11, 11], vec![11, 11], Some("more than once"))] + // Both over-long and naming an unknown id: the unknown id is the cause, so + // it must be reported ahead of "at least one field must remain indexed". + #[case::unknown_id_reported_before_length(vec![7, 11], vec![99, 11], Some("are not among its fields"))] + fn test_validate_covering_fields( + #[case] fields: Vec, + #[case] covering_fields: Vec, + #[case] expected_error: Option<&str>, + ) { + let metadata = index_metadata_with(fields, covering_fields); + let result = metadata.validate_covering_fields(); + + match expected_error { + None => assert!(result.is_ok(), "expected valid, got {:?}", result), + Some(fragment) => { + let err = result.expect_err("expected a validation error"); + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected InvalidInput, got {:?}", + err + ); + let message = err.to_string(); + assert!( + message.contains(fragment), + "expected message to contain {:?}, got {:?}", + fragment, + message + ); + } + } + } } diff --git a/rust/lance-table/src/format/key_existence.rs b/rust/lance-table/src/format/key_existence.rs new file mode 100644 index 00000000000..210ef5f3836 --- /dev/null +++ b/rust/lance-table/src/format/key_existence.rs @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Key existence tracking for merge insert conflict detection. +//! +//! A merge insert records the join keys it inserted into a bloom filter, which +//! is carried in the transaction so a concurrent commit can detect whether it +//! inserted any of the same keys. The filter is serialized into the transaction +//! protobuf, so it lives at the table layer next to [`crate::format::pb`]. + +use std::collections::HashSet; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; + +use crate::format::pb; +use arrow_array::cast::AsArray; +use arrow_array::{ + Array, BinaryArray, LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, RecordBatch, + StringArray, StructArray, +}; +use arrow_schema::DataType; +use lance_core::Result; +use lance_core::deepsize::DeepSizeOf; +use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder}; + +// Default bloom filter config: 8192 items @ 0.00057 fpp -> 16KiB filter +pub const BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS: u64 = 8192; +pub const BLOOM_FILTER_DEFAULT_PROBABILITY: f64 = 0.00057; + +/// Key value for conflict detection. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum KeyValue { + String(String), + Int64(i64), + UInt64(u64), + Binary(Vec), + List(Vec), + Struct(Vec), + Composite(Vec), +} + +impl KeyValue { + pub fn to_bytes(&self) -> Vec { + match self { + Self::String(s) => s.as_bytes().to_vec(), + Self::Int64(i) => i.to_le_bytes().to_vec(), + Self::UInt64(u) => u.to_le_bytes().to_vec(), + Self::Binary(b) => b.clone(), + Self::List(values) | Self::Struct(values) | Self::Composite(values) => { + let mut result = Vec::new(); + for value in values { + result.extend_from_slice(&value.to_bytes()); + result.push(0); + } + result + } + } + } + + pub fn hash_value(&self) -> u64 { + let mut hasher = DefaultHasher::new(); + self.to_bytes().hash(&mut hasher); + hasher.finish() + } +} + +/// Builder for KeyExistenceFilter using Split Block Bloom Filter. +#[derive(Debug, Clone)] +pub struct KeyExistenceFilterBuilder { + sbbf: Sbbf, + field_ids: Vec, + item_count: usize, +} + +impl KeyExistenceFilterBuilder { + pub fn new(field_ids: Vec) -> Self { + let sbbf = SbbfBuilder::new() + .expected_items(BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS) + .false_positive_probability(BLOOM_FILTER_DEFAULT_PROBABILITY) + .build() + .expect("Failed to build SBBF"); + Self { + sbbf, + field_ids, + item_count: 0, + } + } + + pub fn insert(&mut self, key: KeyValue) -> Result<()> { + self.sbbf.insert(&key.to_bytes()[..]); + self.item_count += 1; + Ok(()) + } + + pub fn contains(&self, key: &KeyValue) -> bool { + self.sbbf.check(&key.to_bytes()[..]) + } + + pub fn might_intersect(&self, other: &Self) -> Result { + self.sbbf + .might_intersect(&other.sbbf) + .map_err(|e| lance_core::Error::invalid_input(e.to_string())) + } + + pub fn field_ids(&self) -> &[i32] { + &self.field_ids + } + + pub fn estimated_size_bytes(&self) -> usize { + self.sbbf.size_bytes() + } + + pub fn len(&self) -> usize { + self.item_count + } + + pub fn is_empty(&self) -> bool { + self.item_count == 0 + } + + pub fn build(&self) -> KeyExistenceFilter { + KeyExistenceFilter { + field_ids: self.field_ids.clone(), + filter: FilterType::Bloom { + bitmap: self.sbbf.to_bytes(), + num_bits: (self.sbbf.size_bytes() as u32) * 8, + number_of_items: BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, + probability: BLOOM_FILTER_DEFAULT_PROBABILITY, + }, + } + } +} + +impl From<&KeyExistenceFilterBuilder> for pb::transaction::KeyExistenceFilter { + fn from(builder: &KeyExistenceFilterBuilder) -> Self { + Self { + field_ids: builder.field_ids.clone(), + data: Some(pb::transaction::key_existence_filter::Data::Bloom( + pb::transaction::BloomFilter { + bitmap: builder.sbbf.to_bytes(), + num_bits: (builder.sbbf.size_bytes() as u32) * 8, + number_of_items: BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, + probability: BLOOM_FILTER_DEFAULT_PROBABILITY, + }, + )), + } + } +} + +/// Filter type for key existence data. +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub enum FilterType { + ExactSet(HashSet), + Bloom { + bitmap: Vec, + num_bits: u32, + number_of_items: u64, + probability: f64, + }, +} + +/// Tracks keys of inserted rows for conflict detection. +/// Only created when ON columns match the schema's unenforced primary key. +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct KeyExistenceFilter { + pub field_ids: Vec, + pub filter: FilterType, +} + +impl KeyExistenceFilter { + pub fn from_bloom_filter(bloom: &KeyExistenceFilterBuilder) -> Self { + bloom.build() + } + + /// Check if two filters intersect. Returns (has_intersection, might_be_false_positive). + /// Errors if bloom filter configs don't match. + pub fn intersects(&self, other: &Self) -> Result<(bool, bool)> { + match (&self.filter, &other.filter) { + (FilterType::ExactSet(a), FilterType::ExactSet(b)) => { + Ok((a.iter().any(|h| b.contains(h)), false)) + } + (FilterType::ExactSet(_), FilterType::Bloom { .. }) + | (FilterType::Bloom { .. }, FilterType::ExactSet(_)) => { + // Can't compare different hash schemes, assume intersection + Ok((true, true)) + } + ( + FilterType::Bloom { + bitmap: a_bits, + number_of_items: a_num_items, + probability: a_prob, + .. + }, + FilterType::Bloom { + bitmap: b_bits, + number_of_items: b_num_items, + probability: b_prob, + .. + }, + ) => { + if a_num_items != b_num_items || (a_prob - b_prob).abs() > f64::EPSILON { + return Err(lance_core::Error::invalid_input(format!( + "Bloom filter config mismatch: ({}, {}) vs ({}, {})", + a_num_items, a_prob, b_num_items, b_prob + ))); + } + let has = Sbbf::bytes_might_intersect(a_bits, b_bits) + .map_err(|e| lance_core::Error::invalid_input(e.to_string()))?; + Ok((has, has)) + } + } + } +} + +impl From<&KeyExistenceFilter> for pb::transaction::KeyExistenceFilter { + fn from(filter: &KeyExistenceFilter) -> Self { + match &filter.filter { + FilterType::ExactSet(hashes) => Self { + field_ids: filter.field_ids.clone(), + data: Some(pb::transaction::key_existence_filter::Data::Exact( + pb::transaction::ExactKeySetFilter { + key_hashes: hashes.iter().copied().collect(), + }, + )), + }, + FilterType::Bloom { + bitmap, + num_bits, + number_of_items, + probability, + } => Self { + field_ids: filter.field_ids.clone(), + data: Some(pb::transaction::key_existence_filter::Data::Bloom( + pb::transaction::BloomFilter { + bitmap: bitmap.clone(), + num_bits: *num_bits, + number_of_items: *number_of_items, + probability: *probability, + }, + )), + }, + } + } +} + +impl TryFrom<&pb::transaction::KeyExistenceFilter> for KeyExistenceFilter { + type Error = lance_core::Error; + + fn try_from(message: &pb::transaction::KeyExistenceFilter) -> Result { + let filter = match message.data.as_ref() { + Some(pb::transaction::key_existence_filter::Data::Exact(exact)) => { + FilterType::ExactSet(exact.key_hashes.iter().copied().collect()) + } + Some(pb::transaction::key_existence_filter::Data::Bloom(b)) => { + // Use defaults for backwards compatibility + let number_of_items = if b.number_of_items == 0 { + BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS + } else { + b.number_of_items + }; + let probability = if b.probability == 0.0 { + BLOOM_FILTER_DEFAULT_PROBABILITY + } else { + b.probability + }; + FilterType::Bloom { + bitmap: b.bitmap.clone(), + num_bits: b.num_bits, + number_of_items, + probability, + } + } + None => FilterType::ExactSet(HashSet::new()), + }; + Ok(Self { + field_ids: message.field_ids.clone(), + filter, + }) + } +} + +/// Extract key value from a batch row. Returns None if null or unsupported type. +pub fn extract_key_value_from_batch( + batch: &RecordBatch, + row_idx: usize, + on_columns: &[String], +) -> Option { + let mut parts: Vec = Vec::with_capacity(on_columns.len()); + + for col_name in on_columns { + let (col_idx, _) = batch.schema().column_with_name(col_name)?; + let column = batch.column(col_idx); + + if column.is_null(row_idx) { + return None; + } + + let key_part = extract_key_value(column, row_idx)?; + parts.push(key_part); + } + + if parts.is_empty() { + None + } else if parts.len() == 1 { + Some(parts.into_iter().next().unwrap()) + } else { + Some(KeyValue::Composite(parts)) + } +} + +fn extract_key_value(array: &dyn Array, row_idx: usize) -> Option { + let v = match array.data_type() { + DataType::Utf8 => { + let arr = array.as_any().downcast_ref::()?; + KeyValue::String(arr.value(row_idx).to_string()) + } + DataType::LargeUtf8 => { + let arr = array.as_any().downcast_ref::()?; + KeyValue::String(arr.value(row_idx).to_string()) + } + DataType::UInt64 => { + let arr = array.as_primitive::(); + KeyValue::UInt64(arr.value(row_idx)) + } + DataType::Int64 => { + let arr = array.as_primitive::(); + KeyValue::Int64(arr.value(row_idx)) + } + DataType::UInt32 => { + let arr = array.as_primitive::(); + KeyValue::UInt64(arr.value(row_idx) as u64) + } + DataType::Int32 => { + let arr = array.as_primitive::(); + KeyValue::Int64(arr.value(row_idx) as i64) + } + DataType::Binary => { + let arr = array.as_any().downcast_ref::()?; + KeyValue::Binary(arr.value(row_idx).to_vec()) + } + DataType::LargeBinary => { + let arr = array.as_any().downcast_ref::()?; + KeyValue::Binary(arr.value(row_idx).to_vec()) + } + DataType::List(_) => { + let list_array = array.as_any().downcast_ref::().unwrap(); + let values = list_array.value(row_idx); + + let mut elements = Vec::with_capacity(values.len()); + for i in 0..values.len() { + if values.is_null(i) { + return None; + } + let element = extract_key_value(&values, i)?; + elements.push(element); + } + KeyValue::List(elements) + } + DataType::LargeList(_) => { + let list_array = array.as_any().downcast_ref::().unwrap(); + let values = list_array.value(row_idx); + + let mut elements = Vec::with_capacity(values.len()); + for i in 0..values.len() { + if values.is_null(i) { + return None; + } + let element = extract_key_value(&values, i)?; + elements.push(element); + } + KeyValue::List(elements) + } + DataType::Struct(_) => { + let struct_array = array.as_any().downcast_ref::()?; + let mut elements = Vec::with_capacity(struct_array.num_columns()); + for i in 0..struct_array.num_columns() { + let child = struct_array.column(i); + if child.is_null(row_idx) { + return None; + } + let field_value = extract_key_value(child.as_ref(), row_idx)?; + elements.push(field_value); + } + KeyValue::Struct(elements) + } + _ => return None, + }; + Some(v) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use arrow_array::builder::{Int32Builder, ListBuilder, StringBuilder}; + use arrow_array::{Int32Array, RecordBatch, StringArray, StructArray}; + use arrow_schema::{Field, Schema}; + + #[test] + fn test_extract_key_value_from_batch_list_int() { + let values_builder = Int32Builder::new(); + let mut list_builder = ListBuilder::new(values_builder); + + list_builder.append_value([Some(1), Some(2)]); + list_builder.append_value([Some(3), Some(4), Some(5)]); + + let list_array = list_builder.finish(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + list_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) + .expect("batch should be valid"); + + let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("first row should produce a key"); + let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) + .expect("second row should produce a key"); + + match &key0 { + KeyValue::List(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::Int64(1)); + assert_eq!(values[1], KeyValue::Int64(2)); + } + other => panic!("expected list key, got {:?}", other), + } + + match &key1 { + KeyValue::List(values) => { + assert_eq!(values.len(), 3); + assert_eq!(values[0], KeyValue::Int64(3)); + assert_eq!(values[1], KeyValue::Int64(4)); + assert_eq!(values[2], KeyValue::Int64(5)); + } + other => panic!("expected list key, got {:?}", other), + } + + assert_ne!( + key0.hash_value(), + key1.hash_value(), + "different list values should hash differently", + ); + } + + #[test] + fn test_extract_key_value_from_batch_empty_list() { + let values_builder = Int32Builder::new(); + let mut list_builder = ListBuilder::new(values_builder); + + list_builder.append_value(std::iter::empty::>()); + + let list_array = list_builder.finish(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + list_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) + .expect("batch should be valid"); + + let key = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("empty list should still produce a key"); + + match key { + KeyValue::List(values) => { + assert!(values.is_empty(), "expected empty list"); + } + other => panic!("expected list key, got {:?}", other), + } + } + + #[test] + fn test_extract_key_value_from_batch_list_utf8() { + let values_builder = StringBuilder::new(); + let mut list_builder = ListBuilder::new(values_builder); + + list_builder.append_value([Some("a"), Some("bc")]); + list_builder.append_value([Some("de")]); + + let list_array = list_builder.finish(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + list_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) + .expect("batch should be valid"); + + let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("first row should produce a key"); + let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) + .expect("second row should produce a key"); + + match &key0 { + KeyValue::List(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::String("a".to_string())); + assert_eq!(values[1], KeyValue::String("bc".to_string())); + } + other => panic!("expected list key, got {:?}", other), + } + + match &key1 { + KeyValue::List(values) => { + assert_eq!(values.len(), 1); + assert_eq!(values[0], KeyValue::String("de".to_string())); + } + other => panic!("expected list key, got {:?}", other), + } + + assert_ne!( + key0.hash_value(), + key1.hash_value(), + "different list values should hash differently", + ); + } + + #[test] + fn test_extract_key_value_from_batch_list_with_null_child() { + let values_builder = Int32Builder::new(); + let mut list_builder = ListBuilder::new(values_builder); + + list_builder.append_value([Some(1), Some(2)]); + list_builder.append_value([Some(3), None]); + + let list_array = list_builder.finish(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + list_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) + .expect("batch should be valid"); + + let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("first row should produce a key"); + let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]); + + match &key0 { + KeyValue::List(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::Int64(1)); + assert_eq!(values[1], KeyValue::Int64(2)); + } + other => panic!("expected list key, got {:?}", other), + } + + assert!( + key1.is_none(), + "list row with a null child should not produce a key", + ); + } + + #[test] + fn test_extract_key_value_from_batch_struct_int() { + let a_values = Int32Array::from(vec![1, 3]); + let b_values = Int32Array::from(vec![2, 4]); + + let struct_array = StructArray::from(vec![ + ( + Arc::new(Field::new("a", arrow_schema::DataType::Int32, false)), + Arc::new(a_values) as Arc, + ), + ( + Arc::new(Field::new("b", arrow_schema::DataType::Int32, false)), + Arc::new(b_values) as Arc, + ), + ]); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + struct_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]) + .expect("batch should be valid"); + + let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("first row should produce a key"); + let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) + .expect("second row should produce a key"); + + match &key0 { + KeyValue::Struct(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::Int64(1)); + assert_eq!(values[1], KeyValue::Int64(2)); + } + other => panic!("expected struct key, got {:?}", other), + } + + match &key1 { + KeyValue::Struct(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::Int64(3)); + assert_eq!(values[1], KeyValue::Int64(4)); + } + other => panic!("expected struct key, got {:?}", other), + } + + assert_ne!( + key0.hash_value(), + key1.hash_value(), + "different struct values should hash differently", + ); + } + + #[test] + fn test_extract_key_value_from_batch_struct_utf8() { + let first_names = StringArray::from(vec!["alice", "bob"]); + let last_names = StringArray::from(vec!["smith", "jones"]); + + let struct_array = StructArray::from(vec![ + ( + Arc::new(Field::new("first", arrow_schema::DataType::Utf8, false)), + Arc::new(first_names) as Arc, + ), + ( + Arc::new(Field::new("last", arrow_schema::DataType::Utf8, false)), + Arc::new(last_names) as Arc, + ), + ]); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + struct_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]) + .expect("batch should be valid"); + + let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("first row should produce a key"); + let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) + .expect("second row should produce a key"); + + match &key0 { + KeyValue::Struct(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::String("alice".to_string())); + assert_eq!(values[1], KeyValue::String("smith".to_string())); + } + other => panic!("expected struct key, got {:?}", other), + } + + match &key1 { + KeyValue::Struct(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::String("bob".to_string())); + assert_eq!(values[1], KeyValue::String("jones".to_string())); + } + other => panic!("expected struct key, got {:?}", other), + } + + assert_ne!( + key0.hash_value(), + key1.hash_value(), + "different struct values should hash differently", + ); + } + + #[test] + fn test_extract_key_value_from_batch_struct_with_null_child() { + let a_values = Int32Array::from(vec![Some(1), None]); + let b_values = Int32Array::from(vec![Some(2), Some(3)]); + + let struct_array = StructArray::from(vec![ + ( + Arc::new(Field::new("a", arrow_schema::DataType::Int32, true)), + Arc::new(a_values) as Arc, + ), + ( + Arc::new(Field::new("b", arrow_schema::DataType::Int32, true)), + Arc::new(b_values) as Arc, + ), + ]); + + let schema = Arc::new(Schema::new(vec![Field::new( + "id", + struct_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]) + .expect("batch should be valid"); + + let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) + .expect("first row should produce a key"); + let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]); + + match &key0 { + KeyValue::Struct(values) => { + assert_eq!(values.len(), 2); + assert_eq!(values[0], KeyValue::Int64(1)); + assert_eq!(values[1], KeyValue::Int64(2)); + } + other => panic!("expected struct key, got {:?}", other), + } + + assert!( + key1.is_none(), + "struct row with a null child should not produce a key", + ); + } +} diff --git a/rust/lance-table/src/format/manifest.rs b/rust/lance-table/src/format/manifest.rs index 9845061b7e4..628e313a9a7 100644 --- a/rust/lance-table/src/format/manifest.rs +++ b/rust/lance-table/src/format/manifest.rs @@ -4,9 +4,11 @@ use async_trait::async_trait; use chrono::prelude::*; use lance_core::deepsize::DeepSizeOf; -use lance_file::datatypes::{Fields, FieldsWithMeta, populate_schema_dictionary}; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::version::{LEGACY_FORMAT_VERSION, LanceFileVersion}; +use lance_file::datatypes::{Fields, FieldsWithMeta}; +use lance_file::version::{ConcreteFileVersion, stable_file_version}; +use lance_file::versions::v1::{ + encoding::populate_schema_dictionaries, reader::FileReader as V1FileReader, +}; use lance_io::traits::{ProtoStruct, Reader}; use object_store::path::Path; use prost::Message; @@ -16,6 +18,7 @@ use std::ops::Range; use std::sync::Arc; use super::Fragment; +use crate::feature_flags::{FLAG_COVERED_INDEX_METADATA, STICKY_PAIRED_FLAGS}; use crate::feature_flags::{FLAG_STABLE_ROW_IDS, has_deprecated_v2_feature_flag}; use crate::format::fragment::DataFileFieldInterner; use crate::format::pb; @@ -185,8 +188,8 @@ impl Manifest { index_section: None, timestamp_nanos: 0, tag: None, - reader_feature_flags: 0, - writer_feature_flags: 0, + reader_feature_flags: 0, // These will be set on commit + writer_feature_flags: 0, // These will be set on commit max_fragment_id: None, transaction_file: None, transaction_section: None, @@ -216,8 +219,8 @@ impl Manifest { index_section: None, // Caller should update index if they want to keep them. timestamp_nanos: 0, // This will be set on commit tag: None, - reader_feature_flags: 0, // These will be set on commit - writer_feature_flags: 0, // These will be set on commit + reader_feature_flags: previous.reader_feature_flags & STICKY_PAIRED_FLAGS, + writer_feature_flags: previous.writer_feature_flags & STICKY_PAIRED_FLAGS, max_fragment_id: previous.max_fragment_id, transaction_file: None, transaction_section: None, @@ -248,7 +251,7 @@ impl Manifest { .iter() .map(|fragment| { let mut cloned_fragment = fragment.clone(); - for file in &mut cloned_fragment.files { + for file in cloned_fragment.referenced_lance_files_mut() { if file.base_id.is_none() { file.base_id = Some(ref_base_id); } @@ -273,8 +276,19 @@ impl Manifest { index_section: None, // These will be set on commit timestamp_nanos: self.timestamp_nanos, tag: None, - reader_feature_flags: 0, // These will be set on commit - writer_feature_flags: 0, // These will be set on commit + // Not derivable from the manifest, so it would be lost like any other + // zeroed word: a clone of a table with covering indexes would come + // back unfenced, and since the clone copies the index metadata + // wholesale -- `covering_fields` included -- a build that predates + // covering could then open it and read carried columns as keyed ones. + // Kept unconditionally rather than derived from the cloned indexes: + // over-fencing a clone is harmless, under-fencing one is not. + // Sticky capabilities are also retained because the clone keeps the + // source file identities that require them. + reader_feature_flags: self.reader_feature_flags + & (FLAG_COVERED_INDEX_METADATA | STICKY_PAIRED_FLAGS), + writer_feature_flags: self.writer_feature_flags + & (FLAG_COVERED_INDEX_METADATA | STICKY_PAIRED_FLAGS), max_fragment_id: self.max_fragment_id, transaction_file: Some(transaction_file), transaction_section: None, @@ -421,16 +435,21 @@ impl Manifest { /// Get the max used field id /// /// This is different than [Schema::max_field_id] because it also considers - /// the field ids in the data files that have been dropped from the schema. + /// the field ids in the data files that have been dropped from the schema, + /// including overlay files referenced by fragments. pub fn max_field_id(&self) -> i32 { let schema_max_id = self.schema.max_field_id().unwrap_or(-1); let fragment_max_id = self .fragments .iter() - .flat_map(|f| f.files.iter().flat_map(|file| file.fields.iter())) + .flat_map(|fragment| { + fragment + .referenced_lance_files() + .flat_map(|file| file.fields.iter()) + }) + .copied() .max() - .copied(); - let fragment_max_id = fragment_max_id.unwrap_or(-1); + .unwrap_or(-1); schema_max_id.max(fragment_max_id) } @@ -501,10 +520,6 @@ impl Manifest { pb_manifest.encode_to_vec() } - pub fn should_use_legacy_format(&self) -> bool { - self.data_storage_format.version == LEGACY_FORMAT_VERSION - } - /// Get the summary information of a manifest. /// /// This function calculates various statistics about the manifest, including: @@ -552,6 +567,40 @@ impl Manifest { } } +/// Populate dictionary values stored outside a V1 manifest. +/// +/// Other exact file versions store their schema dictionaries inline, so this +/// is a no-op for those manifests. +/// +/// # Examples +/// +/// ``` +/// # use lance_core::Result; +/// # use lance_io::traits::Reader; +/// # use lance_table::format::{Manifest, populate_manifest_schema_dictionaries}; +/// # async fn hydrate_v1_manifest( +/// # manifest: &mut Manifest, +/// # reader: &dyn Reader, +/// # ) -> Result<()> { +/// populate_manifest_schema_dictionaries(manifest, reader).await +/// # } +/// ``` +pub async fn populate_manifest_schema_dictionaries( + manifest: &mut Manifest, + reader: &dyn Reader, +) -> Result<()> { + match manifest.data_storage_format.version { + ConcreteFileVersion::V1 => { + populate_schema_dictionaries(&mut manifest.schema, reader).await?; + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => {} + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq)] pub struct BasePath { pub id: u32, @@ -606,39 +655,72 @@ pub struct WriterVersion { #[derive(Debug, Clone, PartialEq, DeepSizeOf)] pub struct DataStorageFormat { pub file_format: String, - pub version: String, + pub version: ConcreteFileVersion, } const LANCE_FORMAT_NAME: &str = "lance"; impl DataStorageFormat { - pub fn new(version: LanceFileVersion) -> Self { + pub fn new(version: ConcreteFileVersion) -> Self { Self { file_format: LANCE_FORMAT_NAME.to_string(), - version: version.resolve().to_string(), + version, } } - pub fn lance_file_version(&self) -> Result { - self.version.parse::() + /// Return the exact file format version persisted by this manifest. + pub fn lance_file_format(&self) -> ConcreteFileVersion { + self.version } } impl Default for DataStorageFormat { fn default() -> Self { - Self::new(LanceFileVersion::default()) + Self::new(stable_file_version()) } } -impl From for DataStorageFormat { - fn from(pb: pb::manifest::DataStorageFormat) -> Self { - Self { +impl TryFrom for DataStorageFormat { + type Error = Error; + + fn try_from(pb: pb::manifest::DataStorageFormat) -> Result { + Ok(Self { file_format: pb.file_format, - version: pb.version, - } + version: ConcreteFileVersion::from_manifest_string(&pb.version)?, + }) } } +/// Options controlling how a new [`Manifest`] is assembled from a transaction. +/// +/// The timestamp arrives already resolved to nanoseconds since the Unix epoch. +/// Callers own the clock so that a caller wanting a mockable one keeps it: the +/// `lance` crate mocks `SystemTime` under `cfg(test)`, which only takes effect in +/// that crate. +#[derive(Debug, Clone)] +pub struct ManifestBuildConfig { + /// Recompute the manifest's feature flags from the fragments and settings + /// below. False leaves whatever flags the previous manifest carried. + pub auto_set_feature_flags: bool, + /// Value for the new manifest's timestamp, in nanoseconds since the Unix epoch. + pub timestamp_nanos: u128, + /// Request the stable row id feature. The flag is also inherited from the + /// previous manifest, so false does not turn it off for a dataset that has it. + pub use_stable_row_ids: bool, + /// Overwrite only: force the legacy (true) or v2 (false) file format. `None` + /// keeps the format the dataset already had. + pub use_legacy_format: Option, + /// Overwrite only: force this storage format, taking precedence over + /// `use_legacy_format`. `None` keeps the format the dataset already had. + pub storage_format: Option, + /// Skip writing a detached transaction file for this commit. + pub disable_transaction_file: bool, + /// When `Some`, this commit is the second step of `migrate_to_stable_row_ids`. + /// It bypasses the "cannot enable stable row ids on existing dataset" guard and + /// sets `manifest.next_row_id` to the provided value before activating the flag. + pub migration_next_row_id: Option, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VersionPart { Major, @@ -891,16 +973,16 @@ impl TryFrom for Manifest { } else { // No fragments to inspect, best we can do is look at writer flags if has_deprecated_v2_feature_flag(p.writer_feature_flags) { - DataStorageFormat::new(LanceFileVersion::Stable) + DataStorageFormat::new(stable_file_version()) } else { - DataStorageFormat::new(LanceFileVersion::Legacy) + DataStorageFormat::new(ConcreteFileVersion::V1) } } } - Some(format) => DataStorageFormat::from(format), + Some(format) => DataStorageFormat::try_from(format)?, }; - let schema = Schema::from(fields_with_meta); + let schema = Schema::try_from(fields_with_meta)?; Ok(Self { schema, @@ -980,7 +1062,11 @@ impl From<&Manifest> for pb::Manifest { next_row_id: m.next_row_id, data_format: Some(pb::manifest::DataStorageFormat { file_format: m.data_storage_format.file_format.clone(), - version: m.data_storage_format.version.clone(), + version: m + .data_storage_format + .version + .to_manifest_string() + .to_string(), }), config: m.config.clone(), base_paths: m @@ -1028,7 +1114,7 @@ pub trait SelfDescribingFileReader { } #[async_trait] -impl SelfDescribingFileReader for PreviousFileReader { +impl SelfDescribingFileReader for V1FileReader { async fn try_new_self_described_from_reader( reader: Arc, cache: Option<&LanceCache>, @@ -1039,9 +1125,7 @@ impl SelfDescribingFileReader for PreviousFileReader { reader.path(), )))?; let mut manifest: Manifest = read_struct(reader.as_ref(), manifest_position).await?; - if manifest.should_use_legacy_format() { - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; - } + populate_manifest_schema_dictionaries(&mut manifest, reader.as_ref()).await?; let schema = manifest.schema; let max_field_id = schema.max_field_id().unwrap_or_default(); Self::try_new_from_reader( @@ -1060,6 +1144,8 @@ impl SelfDescribingFileReader for PreviousFileReader { #[cfg(test)] mod tests { + use crate::feature_flags::FLAG_USE_V2_FORMAT_DEPRECATED; + use crate::format::overlay::{DataOverlayFile, OverlayCoverage}; use crate::format::{DataFile, DeletionFile, DeletionFileType}; use std::num::NonZero; @@ -1067,6 +1153,144 @@ mod tests { use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; use lance_core::datatypes::Field; + use roaring::RoaringBitmap; + + /// A shallow clone points every local file at the parent through `base_id`. + /// An overlay's data file lives in the parent too, so it needs the same + /// stamp; without it the clone looks for the overlay under its own root. + #[test] + fn shallow_clone_stamps_base_id_on_overlay_files() { + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "a", + arrow_schema::DataType::Int64, + false, + )]); + let schema = Schema::try_from(&arrow_schema).unwrap(); + + let mut fragment = Fragment::with_file_legacy(0, "base.lance", &schema, Some(10)); + fragment.overlays = vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: 1, + }]; + let manifest = Manifest::new( + schema, + Arc::new(vec![fragment]), + DataStorageFormat::default(), + HashMap::new(), + ); + + let cloned = manifest.shallow_clone( + Some("parent".to_string()), + "memory://parent".to_string(), + 7, + None, + String::new(), + ); + + let fragment = &cloned.fragments[0]; + assert_eq!(fragment.files[0].base_id, Some(7)); + assert_eq!( + fragment.overlays[0].data_file.base_id, + Some(7), + "the overlay's data file resolves against the parent as well" + ); + } + + #[test] + fn old_empty_manifest_recovers_v1_or_current_stable() { + let old_manifest = pb::Manifest { + data_format: None, + ..Default::default() + }; + let recovered_v1 = Manifest::try_from(old_manifest.clone()).unwrap(); + assert_eq!( + recovered_v1.data_storage_format.lance_file_format(), + ConcreteFileVersion::V1 + ); + + let recovered_stable = Manifest::try_from(pb::Manifest { + writer_feature_flags: FLAG_USE_V2_FORMAT_DEPRECATED, + ..old_manifest + }) + .unwrap(); + assert_eq!( + recovered_stable.data_storage_format.lance_file_format(), + stable_file_version() + ); + } + + #[test] + fn manifest_persistence_rejects_selectors_and_public_aliases() { + for version in ["stable", "next", "legacy", "0.3"] { + let manifest = pb::Manifest { + data_format: Some(pb::manifest::DataStorageFormat { + file_format: LANCE_FORMAT_NAME.to_string(), + version: version.to_string(), + }), + ..Default::default() + }; + assert!(Manifest::try_from(manifest).is_err(), "accepted {version}"); + } + } + + #[test] + fn manifest_codec_writes_canonical_exact_string() { + let manifest = Manifest::new( + Schema::default(), + Arc::new(Vec::new()), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + let encoded = pb::Manifest::from(&manifest); + assert_eq!(encoded.data_format.unwrap().version, "2.0"); + } + + #[test] + fn missing_format_infers_exact_version_and_rejects_mixed_files() { + let v2_0 = Fragment::new(0).with_file( + "v2_0.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_0, + None, + ); + let manifest = Manifest::new( + Schema::default(), + Arc::new(vec![v2_0.clone()]), + DataStorageFormat::new(ConcreteFileVersion::V1), + HashMap::new(), + ); + let mut encoded = pb::Manifest::from(&manifest); + encoded.data_format = None; + let recovered = Manifest::try_from(encoded).unwrap(); + assert_eq!( + recovered.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_0 + ); + + let v2_1 = Fragment::new(1).with_file( + "v2_1.lance", + vec![0], + vec![0], + ConcreteFileVersion::V2_1, + None, + ); + let mixed_manifest = Manifest::new( + Schema::default(), + Arc::new(vec![v2_0, v2_1]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + let mut encoded = pb::Manifest::from(&mixed_manifest); + encoded.data_format = None; + let error = Manifest::try_from(encoded).unwrap_err(); + assert!( + error + .to_string() + .contains("All data files must have the same version") + ); + } #[test] fn test_writer_version() { @@ -1316,6 +1540,7 @@ mod tests { vec![0, 1, 2], None, )], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1328,6 +1553,7 @@ mod tests { DataFile::new_legacy_from_fields("path2", vec![0, 1, 43], None), DataFile::new_legacy_from_fields("path3", vec![2], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1346,6 +1572,42 @@ mod tests { assert_eq!(manifest.max_field_id(), 43); } + #[test] + fn test_max_field_id_includes_overlay_files() { + let mut field0 = + Field::try_from(ArrowField::new("a", arrow_schema::DataType::Int64, false)).unwrap(); + field0.set_id(-1, &mut 0); + let schema = Schema { + fields: vec![field0], + metadata: Default::default(), + }; + + let mut fragment = Fragment { + id: 0, + files: vec![DataFile::new_legacy_from_fields("path1", vec![0], None)], + overlays: vec![], + deletion_file: None, + row_id_meta: None, + physical_rows: None, + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + fragment.overlays = vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![43], None), + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: 1, + }]; + + let manifest = Manifest::new( + schema, + Arc::new(vec![fragment]), + DataStorageFormat::default(), + HashMap::new(), + ); + + assert_eq!(manifest.max_field_id(), 43); + } + #[test] fn test_config() { let arrow_schema = ArrowSchema::new(vec![ArrowField::new( @@ -1448,14 +1710,13 @@ mod tests { assert_eq!(real_data_summary.total_data_file_rows, 425); assert_eq!(real_data_summary.total_deletion_files, 0); - let file_version = LanceFileVersion::default(); // Step 4: write deletion files and verify summary let mut fragment_with_deletion = Fragment::new(0) .with_file( "data_with_deletion.lance", vec![0, 1], vec![0, 1], - &file_version, + stable_file_version(), NonZero::new(1000), ) .with_physical_rows(50); diff --git a/rust/lance-table/src/format/overlay.rs b/rust/lance-table/src/format/overlay.rs new file mode 100644 index 00000000000..38014534a25 --- /dev/null +++ b/rust/lance-table/src/format/overlay.rs @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Data overlay files. +//! +//! An overlay file supplies new values for a subset of `(physical offset, field)` +//! cells within a fragment, without rewriting the fragment's base data files. See +//! the Data Overlay Files specification for the full rules; the invariants this +//! module relies on are: +//! +//! - **Physical-offset coverage.** Coverage bitmaps index *physical* row offsets +//! (positions in the base data files, counting deleted rows), so they are stable +//! across deletions, like deletion vectors. +//! - **Rank-based values.** The overlay's `data_file` stores one value column per +//! field, with no row-offset key column. Within a value column, a covered +//! offset's value sits at its **rank** — the 0-based count of set bits below it +//! in that field's coverage bitmap. +//! - **Dense vs. sparse coverage.** A dense overlay shares one bitmap across every +//! field ([`OverlayCoverage::Shared`]); a sparse overlay carries one bitmap per +//! field ([`OverlayCoverage::PerField`]). +//! - **Parse once.** Bitmaps are parsed from their 32-bit Roaring encoding a single +//! time when the fragment loads and held behind an `Arc`, so cloning a fragment +//! is cheap. +//! - **Newest-last ordering.** A fragment's overlays are stored newest-last and +//! stable-sorted by `committed_version` on load (see [`sort_overlays_newest_last`]), +//! with list position breaking ties for equal versions. When two overlays cover +//! the same `(offset, field)`, the higher `committed_version` wins. +//! - **Field tombstones.** When new base values are written for a field (a +//! DataReplacement, or an in-place column rewrite), any overlay value for that +//! field is stale and must stop shadowing the fresh base. The field is marked +//! obsolete in the overlay's `data_file.fields` with [`TOMBSTONE_FIELD_ID`] +//! (the same sentinel used for obsolete base columns) rather than physically +//! removed, so the overlay's other fields — and its coverage positions — stay +//! intact (see [`tombstone_overlay_fields`]). + +pub mod staleness; + +use std::sync::Arc; + +use lance_core::Error; +use lance_core::deepsize::DeepSizeOf; +use lance_core::error::Result; +use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; + +use object_store::path::Path; + +use super::DataFile; +use crate::format::pb; + +/// Field-id sentinel marking a tombstoned (obsolete) field within an overlay's +/// `data_file.fields`. Matches the tombstone convention for obsolete columns in +/// base data files; a tombstoned field's values are ignored on read. +pub const TOMBSTONE_FIELD_ID: i32 = -2; + +/// Which `(physical offset, field)` cells a [`DataOverlayFile`] provides values +/// for. +/// +/// Bitmaps are parsed from their 32-bit Roaring encoding once when the fragment +/// is loaded and held behind an `Arc` so cloning a fragment is cheap; use +/// [`DataOverlayFile::coverage_for_field`] to obtain the one that applies to a +/// given field. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(into = "OverlayCoverageBytes", try_from = "OverlayCoverageBytes")] +pub enum OverlayCoverage { + /// A single bitmap that applies to every field in the overlay's + /// `data_file.fields` (a dense / rectangular overlay): every covered offset + /// has a value for every field. + Shared(Arc), + /// One bitmap per field, in the same order as the overlay's + /// `data_file.fields` (a sparse overlay): different fields may cover + /// different offset sets. + PerField(Vec>), +} + +/// Serialized form of [`OverlayCoverage`] — each bitmap as its 32-bit Roaring +/// byte encoding. The in-memory form parses these once at load. +#[derive(Debug, Clone, Serialize, Deserialize)] +enum OverlayCoverageBytes { + Shared(Vec), + PerField(Vec>), +} + +// The bytes come from a persisted overlay (the protobuf manifest or a +// serialized fragment), so a decode failure is on-disk corruption, not caller +// input. `path` locates the overlay's data file when known (empty on the serde +// path, which deserializes coverage in isolation). +fn deserialize_roaring(bytes: &[u8], path: &Path) -> Result { + RoaringBitmap::deserialize_from(bytes).map_err(|e| { + Error::corrupt_file( + path.clone(), + format!("failed to deserialize overlay coverage bitmap: {e}"), + ) + }) +} + +fn serialize_roaring(bitmap: &RoaringBitmap) -> Vec { + let mut bitmap = bitmap.clone(); + bitmap.optimize(); + let mut bytes = Vec::with_capacity(bitmap.serialized_size()); + // Writing to a Vec is infallible. + bitmap.serialize_into(&mut bytes).unwrap(); + bytes +} + +impl From for OverlayCoverageBytes { + fn from(coverage: OverlayCoverage) -> Self { + match coverage { + OverlayCoverage::Shared(bitmap) => Self::Shared(serialize_roaring(&bitmap)), + OverlayCoverage::PerField(bitmaps) => { + Self::PerField(bitmaps.iter().map(|b| serialize_roaring(b)).collect()) + } + } + } +} + +impl TryFrom for OverlayCoverage { + type Error = Error; + + fn try_from(bytes: OverlayCoverageBytes) -> Result { + // Serde deserializes the coverage in isolation, so the owning data + // file's path is not available here. + let path = Path::default(); + Ok(match bytes { + OverlayCoverageBytes::Shared(b) => { + Self::Shared(Arc::new(deserialize_roaring(&b, &path)?)) + } + OverlayCoverageBytes::PerField(bs) => Self::PerField( + bs.iter() + .map(|b| deserialize_roaring(b, &path).map(Arc::new)) + .collect::>()?, + ), + }) + } +} + +impl DeepSizeOf for OverlayCoverage { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + // The same `Arc` is shared across every clone of a + // fragment, so mark each Arc's pointer and count its heap only the first + // time it is seen — otherwise walking many fragments double-counts the + // shared bitmaps. RoaringBitmap does not expose its allocation size; its + // serialized size is a cheap, close proxy for the heap it holds. + let bitmap_heap = |bitmap: &Arc, + context: &mut lance_core::deepsize::Context| { + if context.mark_seen(Arc::as_ptr(bitmap) as usize) { + std::mem::size_of::() + bitmap.serialized_size() + } else { + 0 + } + }; + match self { + Self::Shared(bitmap) => bitmap_heap(bitmap, context), + Self::PerField(bitmaps) => { + bitmaps.capacity() * std::mem::size_of::>() + + bitmaps + .iter() + .map(|b| bitmap_heap(b, context)) + .sum::() + } + } + } +} + +impl OverlayCoverage { + /// Build a dense coverage from a single bitmap shared across every field. + pub fn dense(bitmap: RoaringBitmap) -> Self { + Self::Shared(Arc::new(bitmap)) + } + + /// Build a sparse coverage from one bitmap per field. + pub fn sparse(bitmaps: Vec) -> Self { + Self::PerField(bitmaps.into_iter().map(Arc::new).collect()) + } +} + +/// An overlay file supplies new values for a subset of `(physical offset, field)` +/// cells within a fragment, without rewriting the fragment's base data files. See +/// the [module documentation](self) for the coverage, rank, and versioning rules. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] +pub struct DataOverlayFile { + /// The data file storing the overlay's new cell values. + pub data_file: DataFile, + /// Which cells this overlay provides values for. + pub coverage: OverlayCoverage, + /// The dataset version at which this overlay became effective (the version of + /// the commit that introduced it, stamped at commit time and re-stamped on + /// retry). Higher wins when two overlays cover the same `(offset, field)`. + pub committed_version: u64, +} + +impl DataOverlayFile { + /// The parsed coverage bitmap that applies to the field stored at + /// `field_pos` within `data_file.fields`. + /// + /// For a dense overlay the same shared bitmap is returned for every field; + /// for a sparse overlay the per-field bitmap at `field_pos` is returned. The + /// bitmap is already parsed, so this is a cheap `Arc` clone. + pub fn coverage_for_field(&self, field_pos: usize) -> Result> { + match &self.coverage { + OverlayCoverage::Shared(bitmap) => Ok(bitmap.clone()), + OverlayCoverage::PerField(bitmaps) => { + bitmaps.get(field_pos).cloned().ok_or_else(|| { + Error::invalid_input(format!( + "overlay per-field coverage has {} bitmaps but field position {} was requested", + bitmaps.len(), + field_pos + )) + }) + } + } + } +} + +/// Stable-sort a fragment's overlays newest-last by `committed_version`. The +/// stable sort preserves list position as the tiebreak for equal versions, so +/// resolution can rely on the ordering without re-checking. See the [module +/// documentation](self) for the ordering invariant. +pub fn sort_overlays_newest_last(overlays: &mut [DataOverlayFile]) { + overlays.sort_by_key(|overlay| overlay.committed_version); +} + +/// Verify a fragment's overlays are stored newest-last (non-decreasing +/// `committed_version`), the ordering invariant readers rely on for +/// resolution. Returns an error identifying the first out-of-order pair. +/// +/// [`sort_overlays_newest_last`] normalizes on load; this is the write-side +/// guard that rejects any commit path that assembled overlays out of order. See +/// the [module documentation](self) for the ordering invariant. +pub fn verify_overlays_newest_last(overlays: &[DataOverlayFile]) -> Result<()> { + for pair in overlays.windows(2) { + if pair[0].committed_version > pair[1].committed_version { + return Err(Error::invalid_input(format!( + "overlay files must be stored newest-last, but committed_version {} precedes {}", + pair[0].committed_version, pair[1].committed_version + ))); + } + } + Ok(()) +} + +/// Tombstone `fields` across a fragment's `overlays`, dropping any overlay left +/// with no live fields. +/// +/// Called when new base values are written for those fields (a DataReplacement, +/// or an in-place column rewrite): the stale overlay values must stop shadowing +/// the fresh base. Each matching field id is replaced with [`TOMBSTONE_FIELD_ID`] +/// in place, preserving the overlay's remaining fields and its coverage positions +/// (a per-field coverage bitmap stays aligned with `data_file.fields`). An overlay +/// whose fields are now all tombstoned is removed entirely. See the [module +/// documentation](self) for the tombstone invariant. +pub fn tombstone_overlay_fields(overlays: &mut Vec, fields: &[u32]) { + for overlay in overlays.iter_mut() { + let tombstoned: Vec = overlay + .data_file + .fields + .iter() + .map(|&field| { + if field >= 0 && fields.contains(&(field as u32)) { + TOMBSTONE_FIELD_ID + } else { + field + } + }) + .collect(); + overlay.data_file.fields = tombstoned.into(); + } + overlays.retain(|overlay| { + overlay + .data_file + .fields + .iter() + .any(|&field| field != TOMBSTONE_FIELD_ID) + }); +} + +impl From<&DataOverlayFile> for pb::DataOverlayFile { + fn from(overlay: &DataOverlayFile) -> Self { + let coverage = match &overlay.coverage { + OverlayCoverage::Shared(bitmap) => { + pb::data_overlay_file::Coverage::SharedOffsetBitmap(serialize_roaring(bitmap)) + } + OverlayCoverage::PerField(bitmaps) => { + pb::data_overlay_file::Coverage::FieldCoverage(pb::FieldCoverage { + offset_bitmaps: bitmaps.iter().map(|b| serialize_roaring(b)).collect(), + }) + } + }; + Self { + data_file: Some(pb::DataFile::from(&overlay.data_file)), + coverage: Some(coverage), + committed_version: overlay.committed_version, + } + } +} + +impl TryFrom for DataOverlayFile { + type Error = Error; + + fn try_from(proto: pb::DataOverlayFile) -> Result { + let data_file = proto + .data_file + .ok_or_else(|| Error::invalid_input("DataOverlayFile is missing its data_file"))?; + let path = Path::from(data_file.path.as_str()); + let coverage = match proto.coverage { + Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(bytes)) => { + OverlayCoverage::Shared(Arc::new(deserialize_roaring(&bytes, &path)?)) + } + Some(pb::data_overlay_file::Coverage::FieldCoverage(fc)) => OverlayCoverage::PerField( + fc.offset_bitmaps + .iter() + .map(|b| deserialize_roaring(b, &path).map(Arc::new)) + .collect::>()?, + ), + None => { + return Err(Error::invalid_input( + "DataOverlayFile is missing its coverage", + )); + } + }; + Ok(Self { + data_file: DataFile::try_from(data_file)?, + coverage, + committed_version: proto.committed_version, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_data_overlay_missing_fields_error() { + // A DataOverlayFile proto missing its coverage or data_file is rejected. + let no_coverage = pb::DataOverlayFile { + data_file: Some(pb::DataFile::from(&DataFile::new_legacy_from_fields( + "overlay.lance", + vec![3], + None, + ))), + coverage: None, + committed_version: 1, + }; + let err = DataOverlayFile::try_from(no_coverage).unwrap_err(); + assert!(err.to_string().contains("missing its coverage"), "{err}"); + + let no_data_file = pb::DataOverlayFile { + data_file: None, + coverage: Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap( + serialize_roaring(&RoaringBitmap::from_iter([0u32])), + )), + committed_version: 1, + }; + let err = DataOverlayFile::try_from(no_data_file).unwrap_err(); + assert!(err.to_string().contains("missing its data_file"), "{err}"); + } + + #[test] + fn test_coverage_bitmap_serialized_run_optimized() { + let bitmap = RoaringBitmap::from_sorted_iter(0..1_000_000).unwrap(); + let unoptimized_size = bitmap.serialized_size(); + + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![3], None), + coverage: OverlayCoverage::dense(bitmap.clone()), + committed_version: 1, + }; + + let proto = pb::DataOverlayFile::from(&overlay); + let Some(pb::data_overlay_file::Coverage::SharedOffsetBitmap(bytes)) = &proto.coverage + else { + panic!("dense coverage must serialize as a shared offset bitmap"); + }; + assert!( + bytes.len() < unoptimized_size / 100, + "expected run-optimized coverage ({} bytes) to be <1% of the \ + unoptimized serialization ({} bytes)", + bytes.len(), + unoptimized_size + ); + + let recovered = DataOverlayFile::try_from(proto).unwrap(); + assert_eq!(recovered.coverage, OverlayCoverage::dense(bitmap)); + } + + #[test] + fn test_overlay_coverage_serde_json_roundtrip() { + // The custom serde impl round-trips through JSON for dense/sparse, + // including empty bitmaps and a zero-bitmap sparse coverage. + for coverage in [ + OverlayCoverage::dense(RoaringBitmap::from_iter([1u32, 5, 100])), + OverlayCoverage::dense(RoaringBitmap::new()), + OverlayCoverage::sparse(vec![ + RoaringBitmap::from_iter([2u32, 3]), + RoaringBitmap::new(), + ]), + OverlayCoverage::sparse(vec![]), + ] { + let json = serde_json::to_string(&coverage).unwrap(); + let back: OverlayCoverage = serde_json::from_str(&json).unwrap(); + assert_eq!(back, coverage); + } + } + + #[test] + fn test_tombstone_overlay_fields() { + // An overlay covering fields [3, 5]: replacing field 5 tombstones just + // field 5's slot and keeps field 3. An overlay covering only field 5 is + // dropped entirely. An overlay touching no replaced field is untouched. + let mut overlays = vec![ + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("a.lance", vec![3, 5], None), + coverage: OverlayCoverage::sparse(vec![ + RoaringBitmap::from_iter([0u32]), + RoaringBitmap::from_iter([1u32]), + ]), + committed_version: 1, + }, + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("b.lance", vec![5], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }, + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("c.lance", vec![7], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }, + ]; + + tombstone_overlay_fields(&mut overlays, &[5]); + + // The single-field overlay on field 5 is gone; the others remain. + assert_eq!(overlays.len(), 2); + // Field 3 preserved, field 5 tombstoned in place (coverage stays aligned). + assert_eq!( + overlays[0].data_file.fields.as_ref(), + &[3, TOMBSTONE_FIELD_ID] + ); + // The untouched overlay keeps its field. + assert_eq!(overlays[1].data_file.fields.as_ref(), &[7]); + } + + #[test] + fn test_verify_overlays_newest_last() { + let mk = |version: u64| DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![3], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: version, + }; + // Non-decreasing (including equal versions) is accepted. + assert!(verify_overlays_newest_last(&[]).is_ok()); + assert!(verify_overlays_newest_last(&[mk(1), mk(2), mk(2), mk(5)]).is_ok()); + // A newer version before an older one is rejected. + let err = verify_overlays_newest_last(&[mk(2), mk(1)]).unwrap_err(); + assert!(err.to_string().contains("newest-last"), "{err}"); + } + + #[test] + fn test_coverage_for_field_out_of_bounds() { + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None), + coverage: OverlayCoverage::sparse(vec![ + RoaringBitmap::from_iter([1u32]), + RoaringBitmap::from_iter([2u32]), + ]), + committed_version: 1, + }; + assert!(overlay.coverage_for_field(0).is_ok()); + assert!(overlay.coverage_for_field(1).is_ok()); + let err = overlay.coverage_for_field(5).unwrap_err(); + assert!(err.to_string().contains("field position"), "{err}"); + } +} diff --git a/rust/lance-table/src/format/overlay/staleness.rs b/rust/lance-table/src/format/overlay/staleness.rs new file mode 100644 index 00000000000..a6eebe61254 --- /dev/null +++ b/rust/lance-table/src/format/overlay/staleness.rs @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Which rows an overlay makes stale with respect to an index. +//! +//! An overlay supplies replacement values for some `(row, field)` cells without +//! rewriting the base data. An index built before that overlay was committed still +//! reflects the old values, so those rows must be excluded from the index's results +//! and re-evaluated against current values on the flat path. +//! +//! Deciding which rows are affected needs only fragment and index metadata — the +//! overlay coverage bitmaps, the overlay `committed_version`, and the indexed field +//! ids — so it lives here rather than in the read path that consumes it. + +use std::collections::HashMap; + +use lance_core::Result; +use lance_core::datatypes::Schema; +use roaring::RoaringBitmap; + +use crate::format::overlay::DataOverlayFile; +use crate::format::{Fragment, IndexMetadata}; + +/// The physical offsets within a fragment whose value for an indexed field may be +/// stale relative to an index built at `index_version`, and so must be excluded +/// from that index's results and re-evaluated against current values on the flat +/// path. +/// +/// The set is the union, over every overlay whose `committed_version` is newer +/// than `index_version`, of that overlay's coverage **restricted to the indexed +/// fields**. The restriction makes exclusion field-aware: an overlay that touches +/// only non-indexed fields contributes nothing. An overlay whose +/// `committed_version <= index_version` is already incorporated by the index and +/// is ignored. +pub fn overlay_exclusion_offsets( + overlays: &[DataOverlayFile], + indexed_field_ids: &[i32], + index_version: u64, + schema: &Schema, +) -> Result { + let mut excluded = RoaringBitmap::new(); + for overlay in overlays { + if overlay.committed_version <= index_version { + continue; + } + for (field_pos, field_id) in overlay.data_file.fields.iter().enumerate() { + let overlay_ancestry = schema.field_ancestry_by_id(*field_id); + let affects_index = indexed_field_ids.iter().any(|indexed_field_id| { + indexed_field_id == field_id + || overlay_ancestry.as_ref().is_some_and(|ancestry| { + ancestry + .iter() + .any(|ancestor| ancestor.id == *indexed_field_id) + }) + || schema + .field_ancestry_by_id(*indexed_field_id) + .is_some_and(|ancestry| { + ancestry.iter().any(|ancestor| ancestor.id == *field_id) + }) + }); + if affects_index { + excluded |= &*overlay.coverage_for_field(field_pos)?; + } + } + } + Ok(excluded) +} + +// Stale row offsets contributed by one fragment's overlays for a given index version. +// Applies a cheap version gate first: if every overlay predates the segment it is already +// incorporated by the index, so there is nothing stale and the field/bitmap work is skipped. +fn stale_offsets_for_fragment( + fragment: &Fragment, + fields: &[i32], + index_version: u64, + schema: &Schema, +) -> Result { + if fragment + .overlays + .iter() + .all(|o| o.committed_version <= index_version) + { + return Ok(RoaringBitmap::new()); + } + overlay_exclusion_offsets(&fragment.overlays, fields, index_version, schema) +} + +// A missing `fragment_bitmap` means the index predates fragment-bitmap tracking; treat it as +// covering every fragment (matching `lance::index::prefilter::DatasetPreFilter::new`) so +// overlay-stale rows can't slip through unmasked. Only skip fragments explicitly absent from a +// present bitmap. +fn covers_fragment(coverage: Option<&RoaringBitmap>, frag_id: u32) -> bool { + coverage.is_none_or(|c| c.contains(frag_id)) +} + +/// Index by fragment id the fragments that carry at least one overlay. Overlays are rare, so +/// this is empty on the common path, letting callers skip index loading entirely; when non-empty +/// it bounds the stale-collection loops to `O(overlaid fragments)`. +pub fn overlaid_fragments(fragments: &[Fragment]) -> HashMap { + fragments + .iter() + .filter(|f| !f.overlays.is_empty()) + .map(|f| (f.id as u32, f)) + .collect() +} + +/// Insert into `stale` the ids of fragments covered by `segment` whose index entries may be +/// stale because an overlay committed after the segment was built touches a field the segment +/// indexes. Field-aware and version-gated via [`overlay_exclusion_offsets`]. +/// +/// `overlaid_frags` holds only the fragments that actually carry overlays (rare), so the loop is +/// `O(overlaid_frags)` rather than `O(fragments the segment covers)`. +pub fn collect_overlay_stale_frags( + segment: &IndexMetadata, + overlaid_frags: &HashMap, + stale: &mut RoaringBitmap, + schema: &Schema, +) -> Result<()> { + let coverage = segment.fragment_bitmap.as_ref(); + for (&frag_id, fragment) in overlaid_frags { + if stale.contains(frag_id) || !covers_fragment(coverage, frag_id) { + continue; + } + if !stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)? + .is_empty() + { + stale.insert(frag_id); + } + } + Ok(()) +} + +/// Like [`collect_overlay_stale_frags`] but with row-level granularity: instead of marking the +/// whole fragment stale, it computes exactly which row offsets within each covered fragment are +/// stale and accumulates them into `stale` (fragment_id → stale row offsets). +/// +/// Used by the scalar and vector paths to block only the affected rows from index results and +/// re-evaluate only those rows on the flat path, keeping overhead proportional to the number of +/// overlaid rows rather than the whole fragment size. +pub fn collect_overlay_stale_rows_for_segment( + segment: &IndexMetadata, + overlaid_frags: &HashMap, + stale: &mut HashMap, + schema: &Schema, +) -> Result<()> { + let coverage = segment.fragment_bitmap.as_ref(); + for (&frag_id, fragment) in overlaid_frags { + if !covers_fragment(coverage, frag_id) { + continue; + } + let excluded = + stale_offsets_for_fragment(fragment, &segment.fields, segment.dataset_version, schema)?; + if !excluded.is_empty() { + *stale.entry(frag_id).or_default() |= &excluded; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::format::overlay::OverlayCoverage; + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + fn flat_test_schema() -> Schema { + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + + let mut schema = Schema::try_from(&ArrowSchema::new( + (0..5) + .map(|id| ArrowField::new(format!("field_{id}"), DataType::Int32, true)) + .collect::>(), + )) + .unwrap(); + schema.set_field_id(None); + schema + } + + /// `outer: struct>`, for the ancestry checks. + fn nested_struct_schema() -> Schema { + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + + let mid = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let outer_fields = + Fields::from(vec![ArrowField::new("middle", DataType::Struct(mid), true)]); + let mut schema = Schema::try_from(&ArrowSchema::new(vec![ArrowField::new( + "outer", + DataType::Struct(outer_fields), + true, + )])) + .unwrap(); + schema.set_field_id(None); + schema + } + + /// A dense overlay covering `offsets` for `field_ids`, committed at `version`. + fn dense_overlay( + field_ids: Vec, + offsets: impl IntoIterator, + version: u64, + ) -> DataOverlayFile { + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", field_ids, None), + coverage: OverlayCoverage::dense(bitmap(offsets)), + committed_version: version, + } + } + + #[test] + fn test_exclusion_offsets_version_gate() { + let schema = flat_test_schema(); + // index built at version 5; only overlays committed > 5 are excluded. + let overlays = vec![ + dense_overlay(vec![3], [0, 1], 4), + dense_overlay(vec![3], [2, 7], 6), + ]; + let excluded = overlay_exclusion_offsets(&overlays, &[3], 5, &schema).unwrap(); + assert_eq!(excluded, bitmap([2, 7])); + // An overlay exactly at the index version is already incorporated. + let overlays = vec![dense_overlay(vec![3], [9], 5)]; + assert!( + overlay_exclusion_offsets(&overlays, &[3], 5, &schema) + .unwrap() + .is_empty() + ); + } + + #[test] + fn test_exclusion_offsets_is_field_aware() { + let schema = flat_test_schema(); + // An overlay touching only an unrelated field excludes nothing. + let overlays = vec![dense_overlay(vec![2], [0, 1, 2], 9)]; + assert!( + overlay_exclusion_offsets(&overlays, &[3], 1, &schema) + .unwrap() + .is_empty() + ); + // The union spans only the indexed fields the overlay actually carries. + let overlays = vec![dense_overlay(vec![2, 3], [4], 9)]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[3], 1, &schema).unwrap(), + bitmap([4]) + ); + } + + #[test] + fn test_exclusion_offsets_matches_nested_fields() { + let schema = nested_struct_schema(); + let outer = &schema.fields[0]; + let middle = &outer.children[0]; + let a = &middle.children[0]; + let b = &middle.children[1]; + + let overlays = vec![dense_overlay(vec![a.id], [1], 9)]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[outer.id], 1, &schema).unwrap(), + bitmap([1]) + ); + assert!( + overlay_exclusion_offsets(&overlays, &[b.id], 1, &schema) + .unwrap() + .is_empty() + ); + + let overlays = vec![dense_overlay(vec![middle.id], [2], 9)]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[a.id], 1, &schema).unwrap(), + bitmap([2]) + ); + } + + #[test] + fn test_exclusion_offsets_sparse_per_field() { + let schema = flat_test_schema(); + // Sparse overlay: field 2 covers {2,3}, field 4 covers {1}. + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![2, 4], None), + coverage: OverlayCoverage::sparse(vec![bitmap([2, 3]), bitmap([1])]), + committed_version: 9, + }; + let overlays = vec![overlay]; + // Only the bitmap for the indexed field (4) contributes. + assert_eq!( + overlay_exclusion_offsets(&overlays, &[4], 1, &schema).unwrap(), + bitmap([1]) + ); + assert_eq!( + overlay_exclusion_offsets(&overlays, &[2], 1, &schema).unwrap(), + bitmap([2, 3]) + ); + } + + #[test] + fn test_exclusion_offsets_unions_multiple_overlays() { + let schema = flat_test_schema(); + let overlays = vec![ + dense_overlay(vec![3], [1], 6), + dense_overlay(vec![3], [4, 5], 7), + ]; + assert_eq!( + overlay_exclusion_offsets(&overlays, &[3], 1, &schema).unwrap(), + bitmap([1, 4, 5]) + ); + } + + /// An index segment covering `fields`, built at `dataset_version`, with the given + /// fragment coverage (`None` = legacy index predating fragment-bitmap tracking). + fn segment( + fields: Vec, + dataset_version: u64, + fragment_bitmap: Option, + ) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: "idx".into(), + fields, + covering_fields: vec![], + dataset_version, + fragment_bitmap, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + fn fragment_with_overlay(id: u64, overlay: DataOverlayFile) -> Fragment { + let mut fragment = Fragment::new(id); + fragment.overlays.push(overlay); + fragment + } + + #[test] + fn test_collect_frags_missing_bitmap_covers_all() { + let schema = flat_test_schema(); + // A segment with no fragment_bitmap (legacy index predating bitmap tracking) must treat + // every overlaid fragment as covered so stale rows can't leak past the index unmasked. + let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9)); + let overlaid: HashMap = HashMap::from([(3u32, &fragment)]); + + let mut stale = RoaringBitmap::new(); + collect_overlay_stale_frags(&segment(vec![3], 1, None), &overlaid, &mut stale, &schema) + .unwrap(); + assert_eq!(stale, bitmap([3]), "missing bitmap must cover fragment 3"); + + // A present bitmap that excludes fragment 3 leaves it untouched. + let mut stale = RoaringBitmap::new(); + collect_overlay_stale_frags( + &segment(vec![3], 1, Some(bitmap([0]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert!( + stale.is_empty(), + "fragment absent from bitmap is not covered" + ); + + // A present bitmap that includes fragment 3 marks it stale. + let mut stale = RoaringBitmap::new(); + collect_overlay_stale_frags( + &segment(vec![3], 1, Some(bitmap([3]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert_eq!(stale, bitmap([3])); + } + + #[test] + fn test_collect_rows_missing_bitmap_covers_all() { + let schema = flat_test_schema(); + // Same covers-all guarantee at row-level granularity. + let fragment = fragment_with_overlay(3, dense_overlay(vec![3], [1, 2], 9)); + let overlaid: HashMap = HashMap::from([(3u32, &fragment)]); + + let mut stale = HashMap::new(); + collect_overlay_stale_rows_for_segment( + &segment(vec![3], 1, None), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert_eq!( + stale.get(&3), + Some(&bitmap([1, 2])), + "missing bitmap must cover fragment 3" + ); + + // A present bitmap that excludes fragment 3 yields no stale rows. + let mut stale = HashMap::new(); + collect_overlay_stale_rows_for_segment( + &segment(vec![3], 1, Some(bitmap([0]))), + &overlaid, + &mut stale, + &schema, + ) + .unwrap(); + assert!( + stale.is_empty(), + "fragment absent from bitmap contributes no rows" + ); + } +} diff --git a/rust/lance-table/src/format/row_ids.rs b/rust/lance-table/src/format/row_ids.rs new file mode 100644 index 00000000000..4cac0f8c7a0 --- /dev/null +++ b/rust/lance-table/src/format/row_ids.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::ops::Deref; +use std::sync::{Arc, OnceLock}; + +use lance_core::deepsize::{Context, DeepSizeOf}; +use lance_core::{Error, Result}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use super::pb; + +/// A reference to a part of a file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] +pub struct ExternalFile { + pub path: String, + pub offset: u64, + pub size: u64, +} + +/// A fragment's row id sequence, encoded inline in the manifest. +/// +/// Carries a memoized [`digest`](Self::digest) of the encoded bytes. The digest +/// identifies *which* sequence these bytes are, which is what the row id +/// sequence cache keys on: a fragment id alone does not identify a sequence, +/// because fragment ids are reused across dataset generations. +/// +/// The digest is memoized because computing it is proportional to the encoded +/// size. A run-encoded sequence is a handful of bytes per run, but a heavily +/// fragmented one is array-encoded at 8 bytes per row, and the cache is +/// consulted on every scan, count, prefilter and index load. +/// +/// Bytes and memo share one immutable allocation, so cloning shares both. That +/// is what makes the memo worth having: callers clone `Fragment` before loading +/// its sequence (see `count_from_mask`), and a memo held per clone would be +/// filled and dropped by each scan, rehashing the whole sequence every time. +/// Sharing also keeps a cloned fragment from duplicating the encoded bytes. +/// +/// The digest lives *with* the bytes rather than beside them so the two cannot +/// drift: several write paths replace a fragment's `row_id_meta` after the +/// fragment is built, and a digest that outlived its bytes would silently +/// resolve to another generation's sequence. +#[derive(Clone)] +pub struct InlineRowIds { + inner: Arc, +} + +struct InlineRowIdsInner { + data: Vec, + digest: OnceLock<[u8; 32]>, +} + +impl InlineRowIds { + /// Digest of the encoded bytes, computed on first use and shared by clones. + pub fn digest(&self) -> &[u8; 32] { + self.inner + .digest + .get_or_init(|| blake3::hash(&self.inner.data).into()) + } +} + +impl From> for InlineRowIds { + fn from(data: Vec) -> Self { + Self { + inner: Arc::new(InlineRowIdsInner { + data, + digest: OnceLock::new(), + }), + } + } +} + +impl Deref for InlineRowIds { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + &self.inner.data + } +} + +// Debug, equality and serialization all present the bytes alone: the memo is a +// derived value and must not show up in output, comparisons or the manifest. +impl std::fmt::Debug for InlineRowIds { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.inner.data.fmt(f) + } +} + +impl PartialEq for InlineRowIds { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) || self.inner.data == other.inner.data + } +} + +impl Eq for InlineRowIds {} + +impl Serialize for InlineRowIds { + fn serialize(&self, serializer: S) -> std::result::Result { + self.inner.data.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for InlineRowIds { + fn deserialize>(deserializer: D) -> std::result::Result { + Vec::::deserialize(deserializer).map(Self::from) + } +} + +impl DeepSizeOf for InlineRowIds { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + // Delegate to the `Arc` so clones sharing one allocation are counted once. + self.inner.deep_size_of_children(context) + } +} + +impl DeepSizeOf for InlineRowIdsInner { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.data.deep_size_of_children(context) + } +} + +/// Metadata about location of the row id sequence. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] +pub enum RowIdMeta { + Inline(InlineRowIds), + External(ExternalFile), +} + +impl TryFrom for RowIdMeta { + type Error = Error; + + fn try_from(value: pb::data_fragment::RowIdSequence) -> Result { + match value { + pb::data_fragment::RowIdSequence::InlineRowIds(data) => Ok(Self::Inline(data.into())), + pb::data_fragment::RowIdSequence::ExternalRowIds(file) => { + Ok(Self::External(ExternalFile { + path: file.path.clone(), + offset: file.offset, + size: file.size, + })) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::Fragment; + + #[test] + fn inline_row_ids_digest_identifies_contents() { + let first = InlineRowIds::from(vec![1, 2, 3]); + let same = InlineRowIds::from(vec![1, 2, 3]); + let other = InlineRowIds::from(vec![1, 2, 4]); + + // The digest is what the row id sequence cache keys on, so it must + // follow the bytes exactly: same bytes, same key; any change, new key. + assert_eq!(first.digest(), same.digest()); + assert_ne!(first.digest(), other.digest()); + assert_eq!(first, same); + assert_ne!(first, other); + + // Memoized on first use, so repeat use must return the same digest. + let memoized = *first.digest(); + assert_eq!(first.digest(), &memoized); + } + + #[test] + fn inline_row_ids_clones_share_bytes_and_memo() { + let first = InlineRowIds::from(vec![1, 2, 3]); + let cloned = first.clone(); + + // Callers clone `Fragment` before loading its row id sequence, so a memo + // held per clone would be filled and dropped by each scan and rehash the + // whole sequence every time. Sharing one allocation is what makes the + // memo pay off, and it keeps clones from duplicating the bytes. + assert!(std::ptr::eq(first.as_ptr(), cloned.as_ptr())); + assert!(std::ptr::eq(first.digest(), cloned.digest())); + + // Guard against the assertions above passing vacuously: only clones + // share storage, equal bytes built separately do not. + let separate = InlineRowIds::from(vec![1, 2, 3]); + assert_eq!(first, separate); + assert!(!std::ptr::eq(first.as_ptr(), separate.as_ptr())); + + // Computing through one clone must be visible from the other. + let fresh = InlineRowIds::from(vec![4, 5, 6]); + let fresh_clone = fresh.clone(); + let via_clone = *fresh_clone.digest(); + assert_eq!(fresh.digest(), &via_clone); + } + + #[test] + fn inline_row_ids_serializes_as_bare_bytes() { + // The manifest is a stable format: the memo must not reach the wire. + let meta = RowIdMeta::Inline(InlineRowIds::from(vec![7, 8, 9])); + let json = serde_json::to_string(&meta).unwrap(); + assert_eq!(json, r#"{"Inline":[7,8,9]}"#); + assert_eq!(serde_json::from_str::(&json).unwrap(), meta); + + // ...and round-trips through protobuf unchanged. + let fragment = Fragment { + row_id_meta: Some(meta.clone()), + ..Fragment::new(0) + }; + let restored = Fragment::try_from(pb::DataFragment::from(&fragment)).unwrap(); + assert_eq!(restored.row_id_meta, Some(meta)); + } +} diff --git a/rust/lance-table/src/format/transaction.rs b/rust/lance-table/src/format/transaction.rs index e9d0bf42129..c8084e7435f 100755 --- a/rust/lance-table/src/format/transaction.rs +++ b/rust/lance-table/src/format/transaction.rs @@ -26,6 +26,44 @@ impl Transaction { pub fn as_pb(&self) -> &pb::Transaction { &self.inner } + + /// Whether this transaction can change the schema, and so can introduce or + /// worsen an invalid primary key. + /// + /// The rest leave the key exactly as they found it, so a table that already + /// carries an invalid one stays writable through them -- including the + /// deletes needed to repair it. An unrecognized operation counts as + /// schema-changing: an unknown write is not a safe one to exempt. + pub fn may_change_schema(&self) -> bool { + operation_may_change_schema(&self.inner) + } +} + +/// The same classification for a protobuf that has not been wrapped yet. +/// +/// The commit path has to classify the operation before it knows whether the +/// encoded bytes are small enough to inline into the manifest. Reading the +/// disposition off the inline copy instead would tie it to the payload size, +/// so the identical operation would be classified one way under the inline +/// limit and the other way above it. +pub fn operation_may_change_schema(transaction: &pb::Transaction) -> bool { + use pb::transaction::Operation; + !matches!( + transaction.operation.as_ref(), + Some( + Operation::Append(_) + | Operation::Delete(_) + | Operation::CreateIndex(_) + | Operation::Rewrite(_) + | Operation::DataReplacement(_) + | Operation::ReserveFragments(_) + | Operation::Update(_) + | Operation::UpdateConfig(_) + | Operation::UpdateMemWalState(_) + | Operation::UpdateBases(_) + | Operation::DataOverlay(_) + ) + ) } /// Write-boundary conversion: serialize using protobuf at the last step. @@ -40,3 +78,62 @@ impl From for Transaction { Self { inner: pb_tx } } } + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message; + + /// The classification that must never depend on payload size. A MemWAL + /// table's transactions carry mem-table state and routinely outgrow the + /// inline limit, so an exempt operation has to stay exempt while large -- + /// otherwise the deletes that repair an invalid key are blocked on exactly + /// the tables most likely to have one. + #[test] + fn an_exempt_operation_is_classified_the_same_at_any_size() { + let small = pb::Transaction { + operation: Some(pb::transaction::Operation::Delete( + pb::transaction::Delete::default(), + )), + ..Default::default() + }; + let mut large = small.clone(); + large.tag = "x".repeat(4 * 1024 * 1024); + + assert!(large.encoded_len() > small.encoded_len() * 100); + assert!(!operation_may_change_schema(&small)); + assert!(!operation_may_change_schema(&large)); + } + + /// An overlay attaches files to existing fragments and supplies new cell + /// values; it carries no schema. Omitting it left a legacy nullable-key + /// dataset unable to commit one, which is the upgrade path this exemption + /// exists to keep open. + #[test] + fn a_data_overlay_is_exempt() { + let overlay = pb::Transaction { + operation: Some(pb::transaction::Operation::DataOverlay( + pb::transaction::DataOverlay::default(), + )), + ..Default::default() + }; + assert!(!operation_may_change_schema(&overlay)); + } + + /// And the converse, so the exemption cannot silently widen to everything. + #[test] + fn a_schema_carrying_operation_is_never_exempt() { + let overwrite = pb::Transaction { + operation: Some(pb::transaction::Operation::Overwrite( + pb::transaction::Overwrite::default(), + )), + ..Default::default() + }; + assert!(operation_may_change_schema(&overwrite)); + + // An operation this build does not recognise must not be exempt + // either: an unknown write is not a safe one to skip. + let unknown = pb::Transaction::default(); + assert!(operation_may_change_schema(&unknown)); + } +} diff --git a/rust/lance-table/src/io/commit.rs b/rust/lance-table/src/io/commit.rs index e1a4086730b..2d6486442b6 100644 --- a/rust/lance-table/src/io/commit.rs +++ b/rust/lance-table/src/io/commit.rs @@ -239,10 +239,28 @@ pub struct ManifestLocation { pub size: Option, /// Naming scheme of the manifest file. pub naming_scheme: ManifestNamingScheme, - /// Optional e-tag, used for integrity checks. Manifests should be immutable, so - /// if we detect a change in the e-tag, it means the manifest was tampered with. - /// This might happen if the dataset was deleted and then re-created. + /// Optional opaque object generation token observed at `path`. + /// + /// An ETag is not necessarily a content checksum and may change when an + /// object is rewritten with identical bytes. In particular, S3 Express + /// returns an object-specific opaque value. Callers must not treat it as a + /// content checksum, logical manifest identity, or dataset-incarnation + /// identity. The generic + /// [`ExternalManifestStore`](crate::io::commit::external_manifest::ExternalManifestStore) + /// workflow therefore neither persists nor validates it: COPY and external + /// index publication are not atomic, so an otherwise correct equivalent + /// materialization can make a stored token stale before it is published. + /// + /// When present, the token still distinguishes the physical object + /// generation observed by this caller and can prevent reuse of an older + /// cached Dataset at the same URI and version. Conversely, `None` must not + /// be interpreted as proof that two observations belong to the same dataset + /// incarnation. pub e_tag: Option, + /// A token unique to this manifest record in the commit handler's store, + /// where it keeps one (`ExternalManifestStore::get_identity`). A dataset + /// recreated at the same version has a different one. + pub identity: Option, } impl TryFrom for ManifestLocation { @@ -263,6 +281,7 @@ impl TryFrom for ManifestLocation { size: Some(meta.size), naming_scheme: scheme, e_tag: meta.e_tag, + identity: None, }) } } @@ -280,7 +299,7 @@ async fn current_manifest_path( object_store: &ObjectStore, base: &Path, ) -> Result { - if object_store.is_local() { + if object_store.has_direct_local_paths() { if let Ok(Some(location)) = current_manifest_local(base) { return Ok(location); } @@ -386,6 +405,7 @@ async fn read_version_hint_and_probe( size: Some(meta.size), naming_scheme: scheme, e_tag: meta.e_tag, + identity: None, }) } @@ -514,6 +534,7 @@ async fn list_manifests_since_version_with_hint( size: Some(meta.size), naming_scheme: scheme, e_tag: meta.e_tag, + identity: None, }) .collect(); @@ -535,6 +556,7 @@ async fn list_manifests_since_version_with_hint( size: Some(meta.size), naming_scheme: scheme, e_tag: meta.e_tag, + identity: None, }) }) .buffer_unordered(object_store.io_parallelism()) @@ -608,6 +630,7 @@ async fn resolve_version_from_listing( size: Some(meta.size), naming_scheme: scheme, e_tag: meta.e_tag, + identity: None, }) } // If the list is not lexically ordered, we need to iterate all manifests @@ -641,6 +664,7 @@ async fn resolve_version_from_listing( size: Some(current_meta.size), naming_scheme: scheme, e_tag: current_meta.e_tag, + identity: None, }) } (None, _) => Err(Error::not_found( @@ -656,7 +680,7 @@ fn current_manifest_local(base: &Path) -> std::io::Result = None; + let mut latest_entry: Option<(u64, DirEntry, ManifestNamingScheme)> = None; let mut scheme: Option = None; @@ -689,25 +713,24 @@ fn current_manifest_local(base: &Path) -> std::io::Result *latest_version { - latest_entry = Some((version, entry)); + latest_entry = Some((version, entry, entry_scheme)); } } else { - latest_entry = Some((version, entry)); + latest_entry = Some((version, entry, entry_scheme)); } } - if let Some((version, entry)) = latest_entry { - let path = Path::from_filesystem_path(entry.path()) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.to_string()))?; + if let Some((version, entry, naming_scheme)) = latest_entry { let metadata = entry.metadata()?; Ok(Some(ManifestLocation { version, - path, + path: naming_scheme.manifest_path(base, version), size: Some(metadata.len()), - naming_scheme: scheme.unwrap(), + naming_scheme, e_tag: Some(get_etag(&metadata)), + identity: None, })) } else { Ok(None) @@ -742,6 +765,7 @@ fn detached_manifest_location_from_meta( size: Some(meta.size), naming_scheme: ManifestNamingScheme::V2, e_tag: meta.e_tag, + identity: None, }) } @@ -762,7 +786,7 @@ pub fn list_detached_manifests<'a>( .boxed() } -fn make_staging_manifest_path(base: &Path) -> Result { +pub(crate) fn make_staging_manifest_path(base: &Path) -> Result { let id = uuid::Uuid::new_v4().to_string(); Path::parse(format!("{base}-{id}")).map_err(|e| Error::io_source(Box::new(e))) } @@ -770,6 +794,94 @@ fn make_staging_manifest_path(base: &Path) -> Result { #[cfg(feature = "dynamodb")] const DDB_URL_QUERY_KEY: &str = "ddbTableName"; +/// Object-store listing of `_versions/`; the `CommitHandler` defaults. +pub(crate) fn default_list_manifest_locations<'a>( + base_path: &Path, + object_store: &'a ObjectStore, + sorted_descending: bool, +) -> BoxStream<'a, Result> { + let underlying_stream = list_manifests(base_path, &object_store.inner); + + if !sorted_descending { + return underlying_stream.boxed(); + } + + async fn sort_stream( + input_stream: impl futures::Stream> + Unpin, + ) -> Result> + Unpin> { + let mut locations = input_stream.try_collect::>().await?; + locations.sort_by_key(|m| std::cmp::Reverse(m.version)); + Ok(futures::stream::iter(locations.into_iter().map(Ok))) + } + + // If the object store supports lexicographically ordered lists and + // the naming scheme is V2, we can use an optimized list operation. + if object_store.list_is_lexically_ordered { + // We don't know the naming scheme until we see the first manifest. + let mut peekable = underlying_stream.peekable(); + + futures::stream::once(async move { + let naming_scheme = match Pin::new(&mut peekable).peek().await { + Some(Ok(m)) => m.naming_scheme, + // If we get an error or no manifests are found, we default + // to V2 naming scheme, since it doesn't matter. + Some(Err(_)) => ManifestNamingScheme::V2, + None => ManifestNamingScheme::V2, + }; + + if naming_scheme == ManifestNamingScheme::V2 { + // If the first manifest is V2, we can use the optimized list operation. + Ok(Either::Left(peekable)) + } else { + sort_stream(peekable).await.map(Either::Right) + } + }) + .try_flatten() + .boxed() + } else { + // If the object store does not support lexicographically ordered lists, + // we need to sort the manifests in memory. Systems where this isn't + // supported (local fs, S3 express) are typically fast enough + // that this is not a problem. + futures::stream::once(sort_stream(underlying_stream)) + .try_flatten() + .boxed() + } +} + +pub(crate) fn default_list_manifest_locations_since<'a>( + base_path: &Path, + object_store: &'a ObjectStore, + since_version: u64, +) -> BoxStream<'a, Result> { + if !uses_version_hint(object_store) { + return default_list_manifest_locations(base_path, object_store, true) + .try_take_while(move |loc| future::ready(Ok(loc.version > since_version))) + .boxed(); + } + + let base_path = base_path.clone(); + futures::stream::once(async move { + let locations = + match list_manifests_since_version_with_hint(object_store, &base_path, since_version) + .await + { + Some(locations) => locations, + None => { + let mut locations = list_manifests(&base_path, &object_store.inner) + .try_collect::>() + .await?; + locations.retain(|loc| loc.version > since_version); + locations.sort_by_key(|loc| std::cmp::Reverse(loc.version)); + locations + } + }; + Ok::<_, Error>(futures::stream::iter(locations.into_iter().map(Ok))) + }) + .try_flatten() + .boxed() +} + /// Handle commits that prevent conflicting writes. /// /// Commit implementations ensure that if there are multiple concurrent writers @@ -781,6 +893,27 @@ const DDB_URL_QUERY_KEY: &str = "ddbTableName"; #[async_trait::async_trait] #[allow(clippy::too_many_arguments)] pub trait CommitHandler: Debug + Send + Sync { + /// Whether a not-found result from [`Self::resolve_version_location`] is + /// definitive immediately after a commit attempt. + /// + /// Handlers backed by an eventually consistent or external source of + /// truth should keep the conservative default. This prevents callers from + /// deleting files that a newly committed manifest may reference while the + /// manifest is not yet visible through the resolver. + fn is_version_not_found_definitive(&self) -> bool { + false + } + + /// Whether an error should still be returned after readback proves that + /// the manifest from the current commit attempt landed. + /// + /// The conservative default preserves errors from custom handlers. Built-in + /// object-store handlers override this because their commit errors may be + /// ambiguous transport failures whose successful outcome is authoritative. + fn propagate_commit_error_after_success(&self) -> bool { + true + } + async fn resolve_latest_location( &self, base_path: &Path, @@ -841,53 +974,7 @@ pub trait CommitHandler: Debug + Send + Sync { object_store: &'a ObjectStore, sorted_descending: bool, ) -> BoxStream<'a, Result> { - let underlying_stream = list_manifests(base_path, &object_store.inner); - - if !sorted_descending { - return underlying_stream.boxed(); - } - - async fn sort_stream( - input_stream: impl futures::Stream> + Unpin, - ) -> Result> + Unpin> { - let mut locations = input_stream.try_collect::>().await?; - locations.sort_by_key(|m| std::cmp::Reverse(m.version)); - Ok(futures::stream::iter(locations.into_iter().map(Ok))) - } - - // If the object store supports lexicographically ordered lists and - // the naming scheme is V2, we can use an optimized list operation. - if object_store.list_is_lexically_ordered { - // We don't know the naming scheme until we see the first manifest. - let mut peekable = underlying_stream.peekable(); - - futures::stream::once(async move { - let naming_scheme = match Pin::new(&mut peekable).peek().await { - Some(Ok(m)) => m.naming_scheme, - // If we get an error or no manifests are found, we default - // to V2 naming scheme, since it doesn't matter. - Some(Err(_)) => ManifestNamingScheme::V2, - None => ManifestNamingScheme::V2, - }; - - if naming_scheme == ManifestNamingScheme::V2 { - // If the first manifest is V2, we can use the optimized list operation. - Ok(Either::Left(peekable)) - } else { - sort_stream(peekable).await.map(Either::Right) - } - }) - .try_flatten() - .boxed() - } else { - // If the object store does not support lexicographically ordered lists, - // we need to sort the manifests in memory. Systems where this isn't - // supported (local fs, S3 express) are typically fast enough - // that this is not a problem. - futures::stream::once(sort_stream(underlying_stream)) - .try_flatten() - .boxed() - } + default_list_manifest_locations(base_path, object_store, sorted_descending) } /// List manifest locations with version `> since_version`, in descending @@ -903,36 +990,7 @@ pub trait CommitHandler: Debug + Send + Sync { object_store: &'a ObjectStore, since_version: u64, ) -> BoxStream<'a, Result> { - if !uses_version_hint(object_store) { - return self - .list_manifest_locations(base_path, object_store, true) - .try_take_while(move |loc| future::ready(Ok(loc.version > since_version))) - .boxed(); - } - - let base_path = base_path.clone(); - futures::stream::once(async move { - let locations = match list_manifests_since_version_with_hint( - object_store, - &base_path, - since_version, - ) - .await - { - Some(locations) => locations, - None => { - let mut locations = list_manifests(&base_path, &object_store.inner) - .try_collect::>() - .await?; - locations.retain(|loc| loc.version > since_version); - locations.sort_by_key(|loc| std::cmp::Reverse(loc.version)); - locations - } - }; - Ok::<_, Error>(futures::stream::iter(locations.into_iter().map(Ok))) - }) - .try_flatten() - .boxed() + default_list_manifest_locations_since(base_path, object_store, since_version) } /// Commit a manifest. @@ -950,6 +1008,63 @@ pub trait CommitHandler: Debug + Send + Sync { transaction: Option, ) -> std::result::Result; + /// Whether [`Self::commit_after`] is available. + fn supports_predecessor_condition(&self) -> bool { + false + } + + /// The identity of the latest manifest, for [`Self::commit_after`]. + /// `None` where the handler cannot condition on it. + async fn resolve_latest_identity( + &self, + _base_path: &Path, + _object_store: &ObjectStore, + ) -> Result> { + Ok(None) + } + + /// The identity of the manifest at `version` as the handler's store + /// records it now; `None` where it keeps none or has no record. + async fn resolve_identity( + &self, + _base_path: &Path, + _object_store: &ObjectStore, + _version: u64, + ) -> Result> { + Ok(None) + } + + /// Commit only if `predecessor` is still the manifest at its version, + /// decided with the reservation; otherwise [`Error::PrerequisiteFailed`], + /// never a conflict. + #[allow(clippy::too_many_arguments)] + async fn commit_after( + &self, + _manifest: &mut Manifest, + _indices: Option>, + _base_path: &Path, + _object_store: &ObjectStore, + _manifest_writer: ManifestWriter, + _naming_scheme: ManifestNamingScheme, + _transaction: Option, + _predecessor: &PredecessorIdentity, + ) -> std::result::Result { + Err(CommitError::OtherError(Error::not_supported( + "this commit handler cannot condition publication on the predecessor manifest", + ))) + } + + /// Retire the record for `version` after its manifest was removed, only + /// while the record still carries `identity`; a no-op otherwise. + async fn forget_version( + &self, + _base_path: &Path, + _version: u64, + _identity: &str, + ) -> Result<()> { + Ok(()) + } + /// Delete the recorded manifest information for a dataset at the base_path async fn delete(&self, _base_path: &Path) -> Result<()> { Ok(()) @@ -971,6 +1086,7 @@ async fn default_resolve_version( path: ManifestNamingScheme::V2.manifest_path(base_path, version), size: None, e_tag: None, + identity: None, }); } @@ -984,6 +1100,7 @@ async fn default_resolve_version( size: Some(meta.size), naming_scheme: scheme, e_tag: meta.e_tag, + identity: None, }), Err(ObjectStoreError::NotFound { .. }) => { // fallback to V1 @@ -994,6 +1111,7 @@ async fn default_resolve_version( size: None, naming_scheme: scheme, e_tag: None, + identity: None, }) } Err(e) => Err(e.into()), @@ -1091,9 +1209,10 @@ pub async fn commit_handler_from_url( match url.scheme() { "file" | "file-object-store" => Ok(local_handler), - "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "cos" | "shared-memory" => { + "s3" | "gs" | "az" | "abfss" | "memory" | "oss" | "tos" | "shared-memory" | "goosefs" => { Ok(Arc::new(ConditionalPutCommitHandler)) } + "cos" => Ok(Arc::new(TencentCosCommitHandler)), #[cfg(not(feature = "dynamodb"))] "s3+ddb" => Err(Error::invalid_input_source( "`s3+ddb://` scheme requires `dynamodb` feature to be enabled".into(), @@ -1133,12 +1252,15 @@ pub async fn commit_handler_from_url( // Get accessor from the options let accessor = options.get_accessor(); + let provider_scheme = storage_options_raw.aws_provider_scheme()?; + let (aws_creds, region) = build_aws_credential( options.s3_credentials_refresh_offset, options.aws_credentials.clone(), Some(&storage_options), region, accessor, + provider_scheme, ) .await?; @@ -1201,6 +1323,14 @@ pub struct UnsafeCommitHandler; #[async_trait::async_trait] #[allow(clippy::too_many_arguments)] impl CommitHandler for UnsafeCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + async fn commit( &self, manifest: &mut Manifest, @@ -1232,6 +1362,7 @@ impl CommitHandler for UnsafeCommitHandler { naming_scheme, path: version_path, e_tag: res.e_tag, + identity: None, }) } } @@ -1328,6 +1459,10 @@ impl CommitHandler for T where T::Lease: 'static, { + fn is_version_not_found_definitive(&self) -> bool { + true + } + async fn commit( &self, manifest: &mut Manifest, @@ -1378,6 +1513,7 @@ where naming_scheme, path, e_tag: res.e_tag, + identity: None, }) } } @@ -1387,6 +1523,14 @@ impl CommitHandler for Arc where T::Lease: 'static, { + fn is_version_not_found_definitive(&self) -> bool { + self.as_ref().is_version_not_found_definitive() + } + + fn propagate_commit_error_after_success(&self) -> bool { + self.as_ref().propagate_commit_error_after_success() + } + async fn commit( &self, manifest: &mut Manifest, @@ -1418,6 +1562,14 @@ pub struct RenameCommitHandler; #[async_trait::async_trait] impl CommitHandler for RenameCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + async fn commit( &self, manifest: &mut Manifest, @@ -1449,7 +1601,8 @@ impl CommitHandler for RenameCommitHandler { path, size: Some(res.size as u64), naming_scheme, - e_tag: None, // Re-name can change e-tag. + e_tag: None, // Re-name can change e-tag., + identity: None, }) } Err(ObjectStoreError::AlreadyExists { .. }) => { @@ -1477,6 +1630,14 @@ pub struct ConditionalPutCommitHandler; #[async_trait::async_trait] impl CommitHandler for ConditionalPutCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + async fn commit( &self, manifest: &mut Manifest, @@ -1527,6 +1688,7 @@ impl CommitHandler for ConditionalPutCommitHandler { size: Some(size), naming_scheme, e_tag: res.e_tag, + identity: None, }) } } @@ -1537,6 +1699,54 @@ impl Debug for ConditionalPutCommitHandler { } } +/// A read-capable handler that prevents unsafe default commits to Tencent COS. +/// +/// COS silently ignores its put-if-not-exists header on buckets that have ever +/// had versioning enabled. Since that bucket history cannot be inferred from +/// the URI or storage options, using [`ConditionalPutCommitHandler`] here can +/// let concurrent writers overwrite the same manifest without reporting a +/// conflict. +struct TencentCosCommitHandler; + +#[async_trait::async_trait] +impl CommitHandler for TencentCosCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + async fn commit( + &self, + _manifest: &mut Manifest, + _indices: Option>, + _base_path: &Path, + _object_store: &ObjectStore, + _manifest_writer: ManifestWriter, + _naming_scheme: ManifestNamingScheme, + _transaction: Option, + ) -> std::result::Result { + Err(CommitError::OtherError(Error::not_supported( + "Default writes to Tencent COS are disabled because COS does not reliably enforce \ + put-if-not-exists after bucket versioning has ever been enabled. Provide a \ + distributed commit_lock in Python or a custom CommitHandler in Rust.", + ))) + } +} + +impl Debug for TencentCosCommitHandler { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TencentCosCommitHandler").finish() + } +} + +/// A manifest as a commit handler identifies it: its version and a token +/// unique to that physical manifest, so a dataset recreated at the same +/// version is told apart. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PredecessorIdentity { + pub version: u64, + pub identity: String, +} + #[derive(Debug, Clone)] pub struct CommitConfig { pub num_retries: u32, @@ -1966,19 +2176,27 @@ mod tests { } #[tokio::test] - async fn test_commit_handler_from_url_memory_schemes() { - // Both `memory://` and `shared-memory://` must route to - // ConditionalPutCommitHandler — otherwise concurrent writers fall - // through to UnsafeCommitHandler and silently clobber each other's - // manifests. - for url in ["memory://bucket-a/ds", "shared-memory://bucket-a/ds"] { - let handler = commit_handler_from_url(url, &None).await.unwrap(); - assert_eq!( - format!("{:?}", handler), - "ConditionalPutCommitHandler", - "{url} should route to ConditionalPutCommitHandler", - ); - } + #[rstest::rstest] + #[case::memory("memory://bucket-a/ds")] + #[case::shared_memory("shared-memory://bucket-a/ds")] + #[case::s3("s3://bucket-a/ds")] + #[case::gs("gs://bucket-a/ds")] + #[case::az("az://bucket-a/ds")] + #[case::abfss("abfss://bucket-a/ds")] + #[case::oss("oss://bucket-a/ds")] + #[case::tos("tos://bucket-a/ds")] + #[case::goosefs("goosefs://bucket-a/ds")] + async fn test_commit_handler_from_url_conditional_put_schemes(#[case] url: &str) { + // Every scheme whose store supports atomic put-if-not-exists must + // route to ConditionalPutCommitHandler — otherwise concurrent writers + // fall through to UnsafeCommitHandler and silently clobber each + // other's manifests. + let handler = commit_handler_from_url(url, &None).await.unwrap(); + assert_eq!( + format!("{:?}", handler), + "ConditionalPutCommitHandler", + "{url} should route to ConditionalPutCommitHandler", + ); } /// A [CommitLock] whose lease records whether it was released, so we can @@ -2069,6 +2287,51 @@ mod tests { Box::pin(async move { Ok(WriteResult::default()) }) } + fn test_manifest() -> Manifest { + use std::collections::HashMap; + + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use lance_file::version::LanceFileVersion; + + use crate::format::DataStorageFormat; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); + Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::new(LanceFileVersion::Stable.resolve()), + HashMap::new(), + ) + } + + #[tokio::test] + async fn test_cos_commit_requires_custom_handler() { + let handler = commit_handler_from_url("cos://bucket-a/ds", &None) + .await + .unwrap(); + assert_eq!(format!("{:?}", handler), "TencentCosCommitHandler"); + + let mut manifest = test_manifest(); + let error = handler + .commit( + &mut manifest, + None, + &Path::from("test"), + &ObjectStore::memory(), + succeeding_manifest_writer, + ManifestNamingScheme::V2, + None, + ) + .await + .unwrap_err(); + let CommitError::OtherError(error) = error else { + panic!("expected a not-supported commit error"); + }; + assert!(matches!(error, Error::NotSupported { .. })); + assert!(error.to_string().contains("distributed commit_lock")); + } + /// A manifest writer that never completes, simulating a hung object store. fn hanging_manifest_writer<'a>( _object_store: &'a ObjectStore, @@ -2087,16 +2350,9 @@ mod tests { /// still release the lock; otherwise it leaks until the lease's TTL expires. #[tokio::test] async fn test_commit_lock_released_on_cancellation() { - use std::collections::HashMap; use std::sync::atomic::Ordering; use std::time::Duration; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Schema; - use lance_file::version::LanceFileVersion; - - use crate::format::DataStorageFormat; - let released = Arc::new(AtomicBool::new(false)); let lock = TrackingLock { released: released.clone(), @@ -2104,13 +2360,7 @@ mod tests { let object_store = ObjectStore::memory(); let base_path = Path::from("test"); - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); - let mut manifest = Manifest::new( - Schema::try_from(&arrow_schema).unwrap(), - Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Stable), - HashMap::new(), - ); + let mut manifest = test_manifest(); // The commit will hang on the manifest writer while holding the lock. // Cancel it the same way a commit timeout would: drop the future. @@ -2144,16 +2394,9 @@ mod tests { /// lock via the drop-path best-effort release. #[tokio::test] async fn test_commit_lock_released_on_cancellation_during_release() { - use std::collections::HashMap; use std::sync::atomic::Ordering; use std::time::Duration; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Schema; - use lance_file::version::LanceFileVersion; - - use crate::format::DataStorageFormat; - let release_calls = Arc::new(AtomicUsize::new(0)); let released = Arc::new(AtomicBool::new(false)); let lock = HangingReleaseLock { @@ -2163,13 +2406,7 @@ mod tests { let object_store = ObjectStore::memory(); let base_path = Path::from("test"); - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("i", DataType::Int32, false)]); - let mut manifest = Manifest::new( - Schema::try_from(&arrow_schema).unwrap(), - Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Stable), - HashMap::new(), - ); + let mut manifest = test_manifest(); // The manifest writer succeeds, so the commit reaches the explicit // release, which hangs. Cancel it the same way a commit timeout would. diff --git a/rust/lance-table/src/io/commit/dynamodb.rs b/rust/lance-table/src/io/commit/dynamodb.rs index d4dab02f504..8a96d070ecf 100644 --- a/rust/lance-table/src/io/commit/dynamodb.rs +++ b/rust/lance-table/src/io/commit/dynamodb.rs @@ -305,8 +305,6 @@ impl ExternalManifestStore for DynamoDBExternalManifestStore { .get("size") .and_then(|attr| attr.as_n().ok().and_then(|v| v.parse().ok())); - let e_tag = item.get("e_tag").and_then(|attr| attr.as_s().ok().cloned()); - let naming_scheme = detect_naming_scheme_from_path(&path)?; Ok(ManifestLocation { @@ -314,7 +312,12 @@ impl ExternalManifestStore for DynamoDBExternalManifestStore { path, size, naming_scheme, - e_tag, + // DynamoDB coordinates the logical version but does not own an + // object generation. Older rows may still contain `e_tag`; ignore + // it and let the commit handler obtain the current token from the + // authoritative object store when it validates the final path. + e_tag: None, + identity: None, }) } @@ -372,8 +375,6 @@ impl ExternalManifestStore for DynamoDBExternalManifestStore { _ => None, }); - let e_tag = item.get("e_tag").and_then(|attr| attr.as_s().ok().cloned()); - match (version_attribute, path_attribute) { (AttributeValue::N(version), AttributeValue::S(path)) => { let version = version.parse().map_err(|e| Error::invalid_input(format!("dynamodb error: could not parse the version number returned {}, error: {}", version, e)))?; @@ -384,7 +385,11 @@ impl ExternalManifestStore for DynamoDBExternalManifestStore { path, size, naming_scheme, - e_tag, + // See `get_manifest_location`: legacy DDB ETags + // are physical-generation observations, not + // version identity, and are intentionally ignored. + e_tag: None, + identity: None, }; Ok(Some(location)) } @@ -404,21 +409,20 @@ impl ExternalManifestStore for DynamoDBExternalManifestStore { version: u64, path: &str, size: u64, - e_tag: Option, + _e_tag: Option, ) -> Result<()> { - let mut put_item = self - .ddb_put() + // Do not persist an object-store ETag. Staging paths are immutable and + // uniquely selected by this conditional write; finalized paths are + // validated against object storage. Persisting an ETag adds no DDB + // concurrency or content-integrity guarantee and can make the row stale + // after an identical copy. The commit handler returns the destination + // ETag separately as an ephemeral runtime cache discriminator. + self.ddb_put() .item(base_uri!(), AttributeValue::S(base_uri.into())) .item(version!(), AttributeValue::N(version.to_string())) .item(path!(), AttributeValue::S(path.to_string())) .item(committer!(), AttributeValue::S(self.committer_name.clone())) - .item("size", AttributeValue::N(size.to_string())); - - if let Some(e_tag) = e_tag { - put_item = put_item.item("e_tag", AttributeValue::S(e_tag)); - } - - put_item + .item("size", AttributeValue::N(size.to_string())) .condition_expression(format!( "attribute_not_exists({}) AND attribute_not_exists({})", base_uri!(), @@ -438,21 +442,17 @@ impl ExternalManifestStore for DynamoDBExternalManifestStore { version: u64, path: &str, size: u64, - e_tag: Option, + _e_tag: Option, ) -> Result<()> { - let mut put_item = self - .ddb_put() + // Replacing the staging pointer with the canonical path publishes the + // same generation-independent `(path, size)` tuple from every helper. + // Each helper still returns the canonical ETag it observed to its caller. + self.ddb_put() .item(base_uri!(), AttributeValue::S(base_uri.into())) .item(version!(), AttributeValue::N(version.to_string())) .item(path!(), AttributeValue::S(path.to_string())) .item(committer!(), AttributeValue::S(self.committer_name.clone())) - .item("size", AttributeValue::N(size.to_string())); - - if let Some(e_tag) = e_tag { - put_item = put_item.item("e_tag", AttributeValue::S(e_tag)); - } - - put_item + .item("size", AttributeValue::N(size.to_string())) .condition_expression(format!( "attribute_exists({}) AND attribute_exists({})", base_uri!(), diff --git a/rust/lance-table/src/io/commit/external_manifest.rs b/rust/lance-table/src/io/commit/external_manifest.rs index 22ebaa10b4a..20ec3a619b8 100644 --- a/rust/lance-table/src/io/commit/external_manifest.rs +++ b/rust/lance-table/src/io/commit/external_manifest.rs @@ -9,7 +9,8 @@ use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; -use futures::StreamExt; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; use lance_core::utils::tracing::{ AUDIT_MODE_CREATE, AUDIT_MODE_DELETE, AUDIT_TYPE_MANIFEST, TRACE_FILE_AUDIT, }; @@ -26,20 +27,138 @@ use super::{ default_resolve_version, make_staging_manifest_path, write_version_hint, }; use crate::format::{IndexMetadata, Manifest, Transaction}; -use crate::io::commit::{CommitError, CommitHandler}; +use crate::io::commit::{ + CommitError, CommitHandler, PredecessorIdentity, default_list_manifest_locations, + default_list_manifest_locations_since, +}; + +/// Copy `staging_path` to the canonical manifest path for `version`, point +/// the store's record at it, and drop the staging object. +#[allow(clippy::too_many_arguments)] +pub async fn finalize_staged( + store: &S, + base_path: &Path, + version: u64, + staging_path: &Path, + size: u64, + object_store: &dyn OSObjectStore, + naming_scheme: ManifestNamingScheme, +) -> Result { + // Step 2: Copy staging to final path + let final_path = naming_scheme.manifest_path(base_path, version); + let final_e_tag = + copy_or_verify_final_manifest(object_store, staging_path, &final_path, version, size) + .await?; + + let location = ManifestLocation { + version, + path: final_path.clone(), + size: Some(size), + naming_scheme, + e_tag: final_e_tag, + identity: None, + }; + + // Step 3: Update the external index to the final path. + // + // Publish only generation-independent metadata. COPY and this update + // are not one atomic operation, so an ETag observed above can already + // be stale when this call linearizes. `location` still carries that + // observation to the current caller for cache separation. + let published = store + .put_if_exists(base_path.as_ref(), version, final_path.as_ref(), size, None) + .await; + + if let Err(error) = published { + // The canonical object is already durable and is the commit point. + // Keep staging so an old or new reader that still observes the + // reservation can retry this cache/index update. A DDB failure must + // not turn an S3-committed transaction into a reported conflict. + warn!( + "Final manifest '{}' is committed, but the external manifest index could not be updated; retaining staging manifest '{}' for repair: {}", + final_path, staging_path, error + ); + return Ok(location); + } + + // Step 4: Delete staging manifest + match object_store.delete(staging_path).await { + Ok(_) => {} + Err(ObjectStoreError::NotFound { .. }) => {} + Err(error) => { + // Staging is no longer authoritative after the canonical + // object and final index entry exist. Its deletion is garbage + // collection and cannot roll back the commit. + warn!( + "Failed to delete finalized staging manifest '{}': {}", + staging_path, error + ); + return Ok(location); + } + } + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); + + Ok(location) +} + +/// Outcome of [`ExternalManifestStore::put_if_predecessor`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Reservation { + /// The version is recorded at the given path, under the identity the + /// store minted for it. + Reserved { identity: String }, + /// The version was already recorded; nothing was written. + Taken, + /// The predecessor is no longer the manifest it was judged as; nothing + /// was written. + PredecessorChanged, +} /// External manifest store /// -/// This trait abstracts an external storage for source of truth for manifests. -/// The storage is expected to remember (uri, version) -> manifest_path -/// and able to run transactions on the manifest_path. +/// This trait abstracts a concurrency coordinator and lookup index for +/// manifests. The store is expected to remember +/// `(uri, version) -> manifest_path` and to atomically select one staging path +/// for each version. The manifest bytes in object storage remain authoritative. /// /// This trait is called an **External** manifest store because the store is /// expected to work in tandem with the object store. We are only leveraging /// the external store for concurrent commit. Any manifest committed thru this /// trait should ultimately be materialized in the object store. +/// +/// # Correctness model +/// +/// 1. Writers first upload immutable manifests to unique staging paths. +/// 2. `put_if_not_exists` linearizes `(dataset, version)` and records exactly +/// one winning staging path. A writer that loses this operation must never +/// materialize its own staging object at the final path. +/// 3. The winner, or any helping reader, copies the recorded staging object to +/// the deterministic final path. Successful final-path materialization is +/// the durable commit point. Repeating this step is content-idempotent +/// because every helper reads the same immutable source selected in step 2. +/// 4. The external row is then compacted from staging to final path and staging +/// is deleted. These are repair and garbage-collection operations: failures +/// leave enough information for another helper and cannot undo step 3. +/// +/// Object-store overwrites can assign a new ETag to identical bytes. An ETag is +/// therefore neither logical manifest identity nor dataset-incarnation identity. +/// The generic protocol never persists or validates ETags in the external index: +/// a finalizer can observe generation E1, another finalizer can replace it with +/// the same selected bytes as E2, and then the first finalizer can publish after +/// the second. Persisting E1 would make a correct canonical object look corrupt. +/// +/// A canonical HEAD still returns the generation observed by the current caller +/// in [`ManifestLocation`]. That ephemeral token keeps runtime caches from +/// treating a newly materialized object as the same observation as an older +/// object at the same `(uri, version)`, without turning the external index into +/// a second authority for physical object generations. The generic external +/// index stores only stable `(path, size)` metadata and readers ignore any legacy +/// stored ETag. This protocol assumes one dataset incarnation owns the physical +/// prefix; a separate incarnation identity is required to make arbitrary prefix +/// reuse unconditionally safe. /// For a visual explanation of the commit loop see /// + #[async_trait] pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { /// Get the manifest path for a given base_uri and version @@ -59,6 +178,7 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { size: None, naming_scheme, e_tag: None, + identity: None, }) } @@ -86,6 +206,7 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { size: None, naming_scheme, e_tag: None, + identity: None, }) }) .transpose() @@ -107,75 +228,118 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { version: u64, staging_path: &Path, size: u64, - e_tag: Option, + _e_tag: Option, object_store: &dyn OSObjectStore, naming_scheme: ManifestNamingScheme, ) -> Result { // Default implementation: staging-based workflow // Step 1: Record staging path atomically + // The external index owns version reservation, not object identity. + // Staging paths are immutable and unique, so path and size are enough + // to identify the selected source. Keeping ETags out of every generic + // write also makes rolling upgrades converge naturally: new readers + // ignore legacy values and every new publication removes them. self.put_if_not_exists( base_path.as_ref(), version, staging_path.as_ref(), size, - e_tag.clone(), + None, ) .await?; - // Step 2: Copy staging to final path - let final_path = naming_scheme.manifest_path(base_path, version); - let copied = match copy_size_aware(object_store, staging_path, &final_path, size).await { - Ok(_) => true, - Err(ObjectStoreError::NotFound { .. }) => false, - Err(e) => return Err(e.into()), - }; - if copied { - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_path.as_ref()); - } - - // Get final e_tag (may change after copy for large files) - let e_tag = if copied && size < 5 * 1024 * 1024 { - e_tag - } else { - let meta = object_store.head(&final_path).await?; - meta.e_tag - }; - - let location = ManifestLocation { + self.finalize( + base_path, version, - path: final_path.clone(), - size: Some(size), + staging_path, + size, + object_store, naming_scheme, - e_tag: e_tag.clone(), - }; - - if !copied { - return Ok(location); - } + ) + .await + } - // Step 3: Update external store to final path - self.put_if_exists( - base_path.as_ref(), + /// Steps 2-4 of [`Self::put`], once `version` is recorded at + /// `staging_path`; see [`finalize_staged`]. + async fn finalize( + &self, + base_path: &Path, + version: u64, + staging_path: &Path, + size: u64, + object_store: &dyn OSObjectStore, + naming_scheme: ManifestNamingScheme, + ) -> Result { + finalize_staged( + self, + base_path, version, - final_path.as_ref(), + staging_path, size, - e_tag, + object_store, + naming_scheme, ) - .await?; + .await + } - // Step 4: Delete staging manifest - match object_store.delete(staging_path).await { - Ok(_) => {} - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(e.into()), - } - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); + /// Whether [`Self::put_if_predecessor`] is implemented. Such a store also + /// fills [`ManifestLocation::identity`] on every location it returns. + fn supports_predecessor_condition(&self) -> bool { + false + } - Ok(location) + /// A token unique to the record at `version`, minted when the record is + /// first written and never reused, so a recreated dataset's record at the + /// same version is told apart. `None` where the store keeps none. + async fn get_identity(&self, _base_uri: &str, _version: u64) -> Result> { + Ok(None) + } + + /// Every committed record with version `> since` (all of them for `None`), + /// each a final location carrying its identity. A store that supports + /// predecessor conditions must implement this: its conditioned manifests + /// are not discoverable by listing the object store. `None` otherwise. + async fn list_versions( + &self, + _base_uri: &str, + _since: Option, + ) -> Result>> { + Ok(None) + } + + /// Remove the record for `version` if it still carries `identity`, so a + /// recreated dataset's record at that version is left alone. Idempotent. + /// Only identity-bearing records are ever retired, so a store that mints + /// identities must implement this; the default refuses. + async fn forget_version(&self, _base_uri: &str, _version: u64, _identity: &str) -> Result<()> { + Err(Error::not_supported( + "this external manifest store cannot retire a version record", + )) + } + + /// [`Self::put_if_not_exists`], applied only if the record at + /// `predecessor.version` still carries `predecessor.identity`, decided + /// atomically with the version reservation. + async fn put_if_predecessor( + &self, + _base_uri: &str, + _version: u64, + _path: &str, + _size: u64, + _predecessor: &PredecessorIdentity, + ) -> Result { + Err(Error::not_supported( + "this external manifest store cannot condition a reservation on its predecessor", + )) } - /// Put the manifest path for a given base_uri and version, should fail if the version already exists + /// Put the manifest path for a given base_uri and version, should fail if the version already exists. + /// + /// The generic staging workflow always passes `None` for `e_tag`. The + /// parameter remains part of the trait for compatibility with stores that + /// override the full [`Self::put`] protocol. Generic implementations must + /// not retain a previous ETag when `None` is supplied. async fn put_if_not_exists( &self, base_uri: &str, @@ -185,7 +349,9 @@ pub trait ExternalManifestStore: std::fmt::Debug + Send + Sync { e_tag: Option, ) -> Result<()>; - /// Put the manifest path for a given base_uri and version, should fail if the version **does not** already exist + /// Put the manifest path for a given base_uri and version, should fail if the version **does not** already exist. + /// + /// See [`Self::put_if_not_exists`] for the `e_tag` contract. async fn put_if_exists( &self, base_uri: &str, @@ -265,6 +431,58 @@ async fn copy_size_aware( } } +/// Copy the selected staging manifest to its canonical path. +/// +/// A successful copy is the object store's acknowledgement that the known +/// immutable bytes were materialized. We then HEAD the destination for two +/// separate reasons: validate that the materialized size matches the selected +/// staging object, and return the physical-generation token observed by this +/// caller. The token is not content identity, but downstream caches currently +/// use it to avoid reusing an older object at the same `(uri, version)`. +/// +/// `NotFound` is different: the selected staging object may have disappeared +/// because another helper finalized and deleted it, or because the commit is +/// unrecoverable. Only in that ambiguous recovery path do we HEAD the canonical +/// object and require its size to match the external-store-selected staging +/// manifest. Any ETag returned by that required HEAD is merely the current +/// object's opaque generation metadata. +async fn copy_or_verify_final_manifest( + object_store: &dyn OSObjectStore, + staging_path: &Path, + final_path: &Path, + version: u64, + selected_size: u64, +) -> Result> { + match copy_size_aware(object_store, staging_path, final_path, selected_size).await { + Ok(()) => { + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_path.as_ref()); + let final_meta = object_store.head(final_path).await?; + if final_meta.size != selected_size { + return Err(Error::corrupt_file( + final_path.clone(), + format!( + "Manifest size mismatch for version {}: selected staging manifest had {}, object store returned {}", + version, selected_size, final_meta.size + ), + )); + } + Ok(final_meta.e_tag) + } + Err(ObjectStoreError::NotFound { .. }) => match object_store.head(final_path).await { + Ok(final_meta) if final_meta.size == selected_size => Ok(final_meta.e_tag), + Ok(final_meta) => Err(Error::corrupt_file( + final_path.clone(), + format!( + "Manifest size mismatch for version {}: selected staging manifest had {}, object store returned {}", + version, selected_size, final_meta.size + ), + )), + Err(error) => Err(error.into()), + }, + Err(error) => Err(error.into()), + } +} + // NOTE: parts are uploaded sequentially. This could be parallelized (a // bounded JoinSet, like lance-io/src/object_writer.rs's // LANCE_UPLOAD_CONCURRENCY) or sidestepped entirely by switching to @@ -347,15 +565,68 @@ pub struct ExternalManifestCommitHandler { } impl ExternalManifestCommitHandler { - /// The manifest is considered committed once the staging manifest is written - /// to object store and that path is committed to the external store. - /// - /// However, to fully complete this, the staging manifest should be materialized - /// into the final path, the final path should be committed to the external store - /// and the staging manifest should be deleted. These steps may be completed - /// by any number of readers or writers, so care should be taken to ensure - /// that the manifest is not lost nor any errors occur due to duplicate - /// operations. + async fn verify_finalized_manifest_location( + &self, + base_path: &Path, + location: ManifestLocation, + object_store: &dyn OSObjectStore, + ) -> std::result::Result { + match object_store.head(&location.path).await { + Ok(ObjectMeta { size, e_tag, .. }) => { + let ManifestLocation { + version, + path, + size: expected_size, + naming_scheme, + e_tag: _, + identity, + } = location; + + let size = match expected_size { + Some(expected_size) if expected_size != size => { + return Err(Error::corrupt_file( + path, + format!( + "Manifest size mismatch for version {}: external store expected {}, object store returned {}", + version, expected_size, size + ), + )); + } + Some(expected_size) => Some(expected_size), + None => Some(size), + }; + + // Ignore any ETag returned by the external index. It may be a + // legacy value published after a later equivalent COPY and is + // therefore neither a safe generation fence nor content proof. + // The HEAD result is the canonical object's current generation + // and is returned only as an ephemeral cache discriminator. + + Ok(ManifestLocation { + version, + path, + size, + naming_scheme, + e_tag, + identity, + }) + } + Err(ObjectStoreError::NotFound { .. }) => { + // The external store may hold a stale finalized V2 path while + // the object store still has the manifest at the V1 location. + default_resolve_version(base_path, location.version, object_store).await + } + Err(e) => Err(e.into()), + } + } + + /// Recording the staging path in the external store reserves the version + /// for one immutable manifest. The commit becomes authoritative when those + /// bytes are materialized at the deterministic final object-store path. + /// Updating the external row to that final path and deleting staging are + /// repair and garbage-collection steps. They may be completed by any number + /// of readers or writers and must not roll back an already materialized + /// canonical manifest. #[allow(clippy::too_many_arguments)] async fn finalize_manifest( &self, @@ -363,64 +634,69 @@ impl ExternalManifestCommitHandler { staging_manifest_path: &Path, version: u64, size: u64, - e_tag: Option, store: &dyn OSObjectStore, naming_scheme: ManifestNamingScheme, ) -> std::result::Result { // step 1: copy the manifest to the final location let final_manifest_path = naming_scheme.manifest_path(base_path, version); - let copied = - match copy_size_aware(store, staging_manifest_path, &final_manifest_path, size).await { - Ok(_) => true, - Err(ObjectStoreError::NotFound { .. }) => false, // Another writer beat us to it. - Err(e) => return Err(e.into()), - }; - if copied { - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_MANIFEST, path = final_manifest_path.as_ref()); - } - - // On S3, the etag can change if originally was MultipartUpload and later was Copy - // https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html#AmazonS3-Type-Object-ETag - // We only do MultipartUpload for > 5MB files, so we can skip this check - // if size < 5MB. However, we need to double check the final_manifest_path - // exists before we change the external store, otherwise we may point to a - // non-existing manifest. - let e_tag = if copied && size < 5 * 1024 * 1024 { - e_tag - } else { - let meta = store.head(&final_manifest_path).await?; - meta.e_tag - }; + let final_e_tag = copy_or_verify_final_manifest( + store, + staging_manifest_path, + &final_manifest_path, + version, + size, + ) + .await?; let location = ManifestLocation { version, path: final_manifest_path, size: Some(size), naming_scheme, - e_tag, + e_tag: final_e_tag, + identity: None, }; - if !copied { - return Ok(location); - } - - // step 2: flip the external store to point to the final location - self.external_manifest_store + // Step 2: point the external index at the final location without an + // ETag. A direct writer and any number of helping readers can perform + // the same immutable COPY concurrently. Since COPY and index update + // are not atomic, persisting a helper's observed generation would let + // an older helper overwrite a newer token. `location` retains the + // current helper's observation for runtime cache separation only. + let published = self + .external_manifest_store .put_if_exists( base_path.as_ref(), version, location.path.as_ref(), size, - location.e_tag.clone(), + None, ) - .await?; + .await; + + if let Err(error) = published { + // The canonical object is the data authority. Retaining staging + // lets another helper repair the external index without making + // this successfully materialized commit appear to have failed. + warn!( + "Final manifest '{}' is committed, but the external manifest index could not be updated; retaining staging manifest '{}' for repair: {}", + location.path, staging_manifest_path, error + ); + return Ok(location); + } // step 3: delete the staging manifest match store.delete(staging_manifest_path).await { Ok(_) => {} Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(e.into()), + Err(error) => { + warn!( + "Failed to delete finalized staging manifest '{}': {}", + staging_manifest_path, error + ); + return Ok(location); + } } info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_manifest_path.as_ref()); @@ -430,6 +706,31 @@ impl ExternalManifestCommitHandler { #[async_trait] impl CommitHandler for ExternalManifestCommitHandler { + async fn version_exists( + &self, + base_path: &Path, + version: u64, + object_store: &dyn OSObjectStore, + naming_scheme: ManifestNamingScheme, + ) -> Result { + match self + .external_manifest_store + .get_manifest_location(base_path.as_ref(), version) + .await + { + Ok(_) => Ok(true), + Err(Error::NotFound { .. }) => { + let path = naming_scheme.manifest_path(base_path, version); + match object_store.head(&path).await { + Ok(_) => Ok(true), + Err(ObjectStoreError::NotFound { .. }) => Ok(false), + Err(e) => Err(e.into()), + } + } + Err(e) => Err(e), + } + } + async fn resolve_latest_location( &self, base_path: &Path, @@ -441,29 +742,34 @@ impl CommitHandler for ExternalManifestCommitHandler { .await?; match location { - Some(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }) => { - // The path is finalized, no need to check object store - if path.extension() == Some(MANIFEST_EXTENSION) { - return Ok(ManifestLocation { - version, - path, - size, - naming_scheme, - e_tag, - }); + Some(location) => { + if location.identity.is_some() { + return recorded_as_final(location, object_store.inner.as_ref()).await; } + if location.path.extension() == Some(MANIFEST_EXTENSION) { + return self + .verify_finalized_manifest_location( + base_path, + location, + object_store.inner.as_ref(), + ) + .await; + } + + let ManifestLocation { + version, + path, + size, + naming_scheme, + e_tag: _, + identity, + } = location; - let (size, e_tag) = if let Some(size) = size { - (size, e_tag) + let size = if let Some(size) = size { + size } else { match object_store.inner.head(&path).await { - Ok(meta) => (meta.size, meta.e_tag), + Ok(meta) => meta.size, Err(ObjectStoreError::NotFound { .. }) => { // there may be other threads that have finished executing finalize_manifest. let new_location = self @@ -476,18 +782,17 @@ impl CommitHandler for ExternalManifestCommitHandler { } }; - let final_location = self + let mut final_location = self .finalize_manifest( base_path, &path, version, size, - e_tag.clone(), &object_store.inner, naming_scheme, ) .await?; - + final_location.identity = identity; Ok(final_location) } // Dataset not found in the external store, this could be because the dataset did not @@ -524,7 +829,7 @@ impl CommitHandler for ExternalManifestCommitHandler { version, path.as_ref(), size, - e_tag.clone(), + None, ) .await; if let Err(e) = res { @@ -541,6 +846,7 @@ impl CommitHandler for ExternalManifestCommitHandler { size: Some(size), naming_scheme, e_tag, + identity: None, }); } Err(ObjectStoreError::NotFound { .. }) => { @@ -552,56 +858,106 @@ impl CommitHandler for ExternalManifestCommitHandler { Err(e) => return Err(e), }; - // finalized path, just return + if location.identity.is_some() { + return recorded_as_final(location, object_store).await; + } if location.path.extension() == Some(MANIFEST_EXTENSION) { - return Ok(location); + return self + .verify_finalized_manifest_location(base_path, location, object_store) + .await; } let naming_scheme = ManifestNamingScheme::detect_scheme_staging(location.path.filename().unwrap()); - let (size, e_tag) = if let Some(size) = location.size { - (size, location.e_tag.clone()) + let size = if let Some(size) = location.size { + size } else { let meta = object_store.head(&location.path).await?; - (meta.size as u64, meta.e_tag) + meta.size }; - self.finalize_manifest( - base_path, - &location.path, - version, - size, - e_tag, - object_store, - naming_scheme, - ) - .await + let mut final_location = self + .finalize_manifest( + base_path, + &location.path, + version, + size, + object_store, + naming_scheme, + ) + .await?; + final_location.identity = location.identity; + Ok(final_location) } - async fn version_exists( + async fn resolve_identity( &self, base_path: &Path, + _object_store: &ObjectStore, version: u64, - object_store: &dyn OSObjectStore, - naming_scheme: ManifestNamingScheme, - ) -> Result { - match self + ) -> Result> { + Ok(self .external_manifest_store - .get_manifest_location(base_path.as_ref(), version) - .await - { - Ok(_) => Ok(true), - Err(Error::NotFound { .. }) => { - let path = naming_scheme.manifest_path(base_path, version); - match object_store.head(&path).await { - Ok(_) => Ok(true), - Err(ObjectStoreError::NotFound { .. }) => Ok(false), - Err(e) => Err(e.into()), + .get_identity(base_path.as_ref(), version) + .await? + .map(|identity| PredecessorIdentity { version, identity })) + } + + fn list_manifest_locations<'a>( + &self, + base_path: &Path, + object_store: &'a ObjectStore, + sorted_descending: bool, + ) -> BoxStream<'a, Result> { + let store = self.external_manifest_store.clone(); + let base_path = base_path.clone(); + futures::stream::once(async move { + match store.list_versions(base_path.as_ref(), None).await? { + Some(mut locations) => { + if sorted_descending { + locations.sort_by_key(|l| std::cmp::Reverse(l.version)); + } + Ok::<_, Error>(futures::stream::iter(locations.into_iter().map(Ok)).boxed()) } + None => Ok(default_list_manifest_locations( + &base_path, + object_store, + sorted_descending, + )), } - Err(e) => Err(e), - } + }) + .try_flatten() + .boxed() + } + + fn list_manifest_locations_since<'a>( + &self, + base_path: &Path, + object_store: &'a ObjectStore, + since_version: u64, + ) -> BoxStream<'a, Result> { + let store = self.external_manifest_store.clone(); + let base_path = base_path.clone(); + futures::stream::once(async move { + match store + .list_versions(base_path.as_ref(), Some(since_version)) + .await? + { + Some(mut locations) => { + locations.retain(|l| l.version > since_version); + locations.sort_by_key(|l| std::cmp::Reverse(l.version)); + Ok::<_, Error>(futures::stream::iter(locations.into_iter().map(Ok)).boxed()) + } + None => Ok(default_list_manifest_locations_since( + &base_path, + object_store, + since_version, + )), + } + }) + .try_flatten() + .boxed() } async fn commit( @@ -642,16 +998,15 @@ impl CommitHandler for ExternalManifestCommitHandler { write_version_hint(object_store, base_path, manifest.version).await; Ok(location) } - Err(_) => { - // delete the staging manifest - match object_store.inner.delete(&staging_path).await { - Ok(_) => {} - Err(ObjectStoreError::NotFound { .. }) => {} - Err(e) => return Err(CommitError::OtherError(e.into())), - } - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); - Err(CommitError::CommitConflict {}) - } + Err(error) => Err(self + .lose_or_retain( + base_path, + manifest.version, + &staging_path, + object_store, + error, + ) + .await), } } @@ -660,4 +1015,1539 @@ impl CommitHandler for ExternalManifestCommitHandler { .delete(base_path.as_ref()) .await } + + async fn forget_version(&self, base_path: &Path, version: u64, identity: &str) -> Result<()> { + self.external_manifest_store + .forget_version(base_path.as_ref(), version, identity) + .await + } + + fn supports_predecessor_condition(&self) -> bool { + self.external_manifest_store + .supports_predecessor_condition() + } + + async fn resolve_latest_identity( + &self, + base_path: &Path, + _object_store: &ObjectStore, + ) -> Result> { + let Some((version, _)) = self + .external_manifest_store + .get_latest_version(base_path.as_ref()) + .await? + else { + return Ok(None); + }; + Ok(self + .external_manifest_store + .get_identity(base_path.as_ref(), version) + .await? + .map(|identity| PredecessorIdentity { version, identity })) + } + + async fn commit_after( + &self, + manifest: &mut Manifest, + indices: Option>, + base_path: &Path, + object_store: &ObjectStore, + manifest_writer: super::ManifestWriter, + naming_scheme: ManifestNamingScheme, + transaction: Option, + predecessor: &PredecessorIdentity, + ) -> std::result::Result { + // Written once at a staging path, which listing never discovers, and + // recorded as final by the reservation itself; the canonical path a + // recreated dataset would share is never written. + let path = + make_staging_manifest_path(&naming_scheme.manifest_path(base_path, manifest.version))?; + let write_res = + manifest_writer(object_store, manifest, indices, &path, transaction).await?; + let size = write_res.size as u64; + + let reserved = self + .external_manifest_store + .put_if_predecessor( + base_path.as_ref(), + manifest.version, + path.as_ref(), + size, + predecessor, + ) + .await; + match reserved { + Ok(Reservation::Reserved { identity }) => { + write_version_hint(object_store, base_path, manifest.version).await; + Ok(ManifestLocation { + version: manifest.version, + path, + size: Some(size), + naming_scheme, + e_tag: write_res.e_tag, + identity: Some(identity), + }) + } + Ok(Reservation::PredecessorChanged) => { + // Nothing was recorded, so the object is ours to drop. + delete_staging(object_store, &path, "refused").await; + Err(CommitError::OtherError( + lance_core::error::PrerequisiteFailedSnafu { + message: format!( + "manifest {} is no longer the predecessor this commit was judged against", + predecessor.version + ), + } + .build(), + )) + } + Ok(Reservation::Taken) => Err(self + .lose_or_retain( + base_path, + manifest.version, + &path, + object_store, + Error::commit_conflict_source( + manifest.version, + "manifest already exists".into(), + ), + ) + .await), + Err(error) => Err(self + .lose_or_retain(base_path, manifest.version, &path, object_store, error) + .await), + } + } +} + +impl ExternalManifestCommitHandler { + /// A different recorded path proves the staging manifest lost, so it is + /// removed; otherwise it is retained for outcome verification. + async fn lose_or_retain( + &self, + base_path: &Path, + version: u64, + staging_path: &Path, + object_store: &ObjectStore, + error: Error, + ) -> CommitError { + let recorded_location = self + .external_manifest_store + .get_manifest_location(base_path.as_ref(), version) + .await; + if matches!(&recorded_location, Ok(location) if location.path != *staging_path) { + delete_staging(object_store, staging_path, "losing").await; + return CommitError::CommitConflict; + } + warn!( + "External manifest commit for version {} failed; retaining staging manifest \ + '{}' until the commit outcome is resolved: {}", + version, staging_path, error + ); + CommitError::CommitConflict + } +} + +/// A record from a store that keeps identities is final as recorded and is +/// never repaired onto the canonical path. +async fn recorded_as_final( + mut location: ManifestLocation, + object_store: &dyn OSObjectStore, +) -> Result { + if location.size.is_none() { + location.size = Some(object_store.head(&location.path).await?.size); + } + Ok(location) +} + +async fn delete_staging(object_store: &ObjectStore, staging_path: &Path, why: &str) { + match object_store.inner.delete(staging_path).await { + Ok(()) => { + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_MANIFEST, path = staging_path.as_ref()); + } + Err(ObjectStoreError::NotFound { .. }) => {} + Err(delete_error) => { + warn!( + "Failed to delete {} staging manifest '{}': {}", + why, staging_path, delete_error + ); + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use lance_core::utils::testing::{ProxyObjectStore, ProxyObjectStorePolicy}; + use lance_file::version::LanceFileVersion; + use tokio::sync::Notify; + + use super::*; + use crate::format::DataStorageFormat; + use crate::io::commit::{VERSIONS_DIR, write_manifest_file_to_path}; + use futures::TryStreamExt; + + #[derive(Debug, Clone)] + struct StoredManifest { + path: String, + size: u64, + e_tag: Option, + } + + #[derive(Debug)] + struct TestExternalManifestStore { + manifests: Mutex>, + fail_next_put_response: AtomicBool, + fail_next_final_publish: AtomicBool, + block_first_final_publish: bool, + final_publish_calls: AtomicUsize, + first_final_publish_started: Notify, + release_first_final_publish: Notify, + } + + impl TestExternalManifestStore { + fn new(fail_next_put_response: bool) -> Self { + Self { + manifests: Mutex::new(HashMap::new()), + fail_next_put_response: AtomicBool::new(fail_next_put_response), + fail_next_final_publish: AtomicBool::new(false), + block_first_final_publish: false, + final_publish_calls: AtomicUsize::new(0), + first_final_publish_started: Notify::new(), + release_first_final_publish: Notify::new(), + } + } + + fn failing_final_publish_once() -> Self { + Self { + fail_next_final_publish: AtomicBool::new(true), + ..Self::new(false) + } + } + + fn blocking_first_final_publish() -> Self { + Self { + block_first_final_publish: true, + ..Self::new(false) + } + } + } + + #[async_trait] + impl ExternalManifestStore for TestExternalManifestStore { + async fn get(&self, base_uri: &str, version: u64) -> Result { + self.manifests + .lock() + .unwrap() + .get(&(base_uri.to_string(), version)) + .map(|manifest| manifest.path.clone()) + .ok_or_else(|| Error::not_found(format!("{base_uri}@{version}"))) + } + + async fn get_manifest_location( + &self, + base_uri: &str, + version: u64, + ) -> Result { + let stored = self + .manifests + .lock() + .unwrap() + .get(&(base_uri.to_string(), version)) + .cloned() + .ok_or_else(|| Error::not_found(format!("{base_uri}@{version}")))?; + let path = Path::from(stored.path); + Ok(ManifestLocation { + version, + naming_scheme: detect_naming_scheme_from_path(&path)?, + path, + size: Some(stored.size), + e_tag: stored.e_tag, + identity: None, + }) + } + + async fn get_latest_version(&self, base_uri: &str) -> Result> { + Ok(self + .manifests + .lock() + .unwrap() + .iter() + .filter(|((stored_base, _), _)| stored_base == base_uri) + .max_by_key(|((_, version), _)| *version) + .map(|((_, version), manifest)| (*version, manifest.path.clone()))) + } + + async fn put_if_not_exists( + &self, + base_uri: &str, + version: u64, + path: &str, + size: u64, + e_tag: Option, + ) -> Result<()> { + let key = (base_uri.to_string(), version); + let mut manifests = self.manifests.lock().unwrap(); + if manifests.contains_key(&key) { + return Err(Error::commit_conflict_source( + version, + "manifest already exists".to_string().into(), + )); + } + manifests.insert( + key, + StoredManifest { + path: path.to_string(), + size, + e_tag, + }, + ); + drop(manifests); + if self.fail_next_put_response.swap(false, Ordering::SeqCst) { + Err(Error::io("simulated lost external-store response")) + } else { + Ok(()) + } + } + + async fn put_if_exists( + &self, + base_uri: &str, + version: u64, + path: &str, + size: u64, + e_tag: Option, + ) -> Result<()> { + if self.block_first_final_publish + && self.final_publish_calls.fetch_add(1, Ordering::SeqCst) == 0 + { + self.first_final_publish_started.notify_one(); + self.release_first_final_publish.notified().await; + } + if self.fail_next_final_publish.swap(false, Ordering::SeqCst) { + return Err(Error::io("simulated final index update failure")); + } + let key = (base_uri.to_string(), version); + let mut manifests = self.manifests.lock().unwrap(); + let manifest = manifests + .get_mut(&key) + .ok_or_else(|| Error::not_found(format!("{base_uri}@{version}")))?; + *manifest = StoredManifest { + path: path.to_string(), + size, + e_tag, + }; + Ok(()) + } + } + + fn test_manifest() -> Manifest { + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + Manifest::new( + Schema::try_from(&arrow_schema).unwrap(), + Arc::new(vec![]), + DataStorageFormat::new(LanceFileVersion::Stable.resolve()), + HashMap::new(), + ) + } + + #[tokio::test] + async fn test_finalized_manifest_ignores_legacy_external_store_etag() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, 1); + + object_store + .inner + .put( + &final_path, + object_store::PutPayload::from_static(b"manifest"), + ) + .await + .unwrap(); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + + external_store + .put_if_not_exists( + base_path.as_ref(), + 1, + final_path.as_ref(), + final_meta.size, + Some("expected-generation".to_string()), + ) + .await + .unwrap(); + + let resolved = handler + .resolve_version_location(&base_path, 1, object_store.inner.as_ref()) + .await + .expect("a legacy external-store ETag must not override object storage"); + assert_eq!(resolved.path, final_path); + assert_eq!(resolved.size, Some(final_meta.size)); + assert_eq!(resolved.e_tag, final_meta.e_tag); + } + + #[tokio::test] + async fn test_finalized_manifest_without_external_store_etag_uses_current_etag() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, 1); + + object_store + .inner + .put( + &final_path, + object_store::PutPayload::from_static(b"manifest"), + ) + .await + .unwrap(); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + external_store + .put_if_not_exists( + base_path.as_ref(), + 1, + final_path.as_ref(), + final_meta.size, + None, + ) + .await + .unwrap(); + + let resolved = handler + .resolve_version_location(&base_path, 1, object_store.inner.as_ref()) + .await + .expect("an absent external-store ETag must opt out of comparison"); + assert_eq!(resolved.path, final_path); + assert_eq!(resolved.size, Some(final_meta.size)); + assert_eq!(resolved.e_tag, final_meta.e_tag); + } + + #[tokio::test] + async fn test_default_store_returns_but_does_not_persist_etag() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + + let committed = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect("the default store should finalize the selected manifest"); + let original = object_store.inner.head(&committed.path).await.unwrap(); + assert_eq!(committed.e_tag, original.e_tag); + + let indexed = external_store + .get_manifest_location(base_path.as_ref(), committed.version) + .await + .unwrap(); + assert_eq!(indexed.e_tag, None); + + object_store + .inner + .put( + &committed.path, + object_store::PutPayload::from(vec![0_u8; original.size as usize]), + ) + .await + .unwrap(); + + let replacement = object_store.inner.head(&committed.path).await.unwrap(); + assert_ne!(replacement.e_tag, original.e_tag); + + let resolved = handler + .resolve_version_location(&base_path, committed.version, object_store.inner.as_ref()) + .await + .expect("the external index must not reject a new physical generation"); + assert_eq!(resolved.e_tag, replacement.e_tag); + } + + #[tokio::test] + async fn test_helping_finalizer_returns_but_does_not_persist_etag() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let version = 1; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let staging_path = make_staging_manifest_path(&final_path).unwrap(); + let manifest_bytes = Bytes::from_static(b"immutable manifest bytes"); + + object_store + .inner + .put(&staging_path, manifest_bytes.clone().into()) + .await + .unwrap(); + let staging_meta = object_store.inner.head(&staging_path).await.unwrap(); + external_store + .put_if_not_exists( + base_path.as_ref(), + version, + staging_path.as_ref(), + staging_meta.size, + staging_meta.e_tag, + ) + .await + .unwrap(); + + let finalized = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("a reader should finalize the selected staging manifest"); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + assert_eq!(finalized.e_tag, final_meta.e_tag); + + let indexed = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .unwrap(); + assert_eq!(indexed.e_tag, None); + } + + #[tokio::test] + async fn test_onboarding_returns_but_does_not_persist_etag() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let version = 1; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + + object_store + .inner + .put( + &final_path, + object_store::PutPayload::from_static(b"manifest"), + ) + .await + .unwrap(); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + + let resolved = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("an existing manifest should be indexed during onboarding"); + assert_eq!(resolved.e_tag, final_meta.e_tag); + + let indexed = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .unwrap(); + assert_eq!(indexed.e_tag, None); + } + + #[tokio::test] + async fn test_finalized_manifest_size_mismatch_remains_corruption() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, 1); + + object_store + .inner + .put( + &final_path, + object_store::PutPayload::from_static(b"manifest"), + ) + .await + .unwrap(); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + external_store + .put_if_not_exists( + base_path.as_ref(), + 1, + final_path.as_ref(), + final_meta.size + 1, + None, + ) + .await + .unwrap(); + + let error = handler + .resolve_version_location(&base_path, 1, object_store.inner.as_ref()) + .await + .expect_err("copies of the selected staging object must preserve its size"); + assert!(matches!(error, Error::CorruptFile { .. })); + assert!(error.to_string().contains("Manifest size mismatch")); + } + + #[tokio::test] + async fn test_canonical_manifest_commits_before_index_repair() { + let external_store = Arc::new(TestExternalManifestStore::failing_final_publish_once()); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + let version = manifest.version; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + + let committed = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect("a failed index update must not overturn a canonical S3 commit"); + assert_eq!(committed.path, final_path); + assert!( + committed.e_tag.is_some(), + "the caller must retain the canonical generation even when index repair fails" + ); + object_store + .inner + .head(&final_path) + .await + .expect("the canonical manifest is the durable commit point"); + + let pending = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .unwrap(); + assert_ne!(pending.path, final_path); + object_store + .inner + .head(&pending.path) + .await + .expect("staging must remain until the external index is repaired"); + + let repaired = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("a reader must be able to repair the pending external index"); + assert_eq!(repaired.path, final_path); + assert!( + repaired.e_tag.is_some(), + "a helping reader must receive the generation it observed" + ); + let indexed = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .unwrap(); + assert_eq!(indexed.path, final_path); + assert_eq!(indexed.size, repaired.size); + assert_eq!( + indexed.e_tag, None, + "the repaired index must not retain a physical object generation" + ); + let staging_error = object_store + .inner + .head(&pending.path) + .await + .expect_err("repair should garbage-collect the retained staging object"); + assert!(matches!(staging_error, ObjectStoreError::NotFound { .. })); + } + + #[tokio::test] + async fn test_concurrent_finalizers_return_but_do_not_persist_generations() { + let external_store = Arc::new(TestExternalManifestStore::blocking_first_final_publish()); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let version = 1; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let staging_path = make_staging_manifest_path(&final_path).unwrap(); + let manifest_bytes = Bytes::from_static(b"immutable manifest bytes"); + + object_store + .inner + .put(&staging_path, manifest_bytes.clone().into()) + .await + .unwrap(); + let staging_meta = object_store.inner.head(&staging_path).await.unwrap(); + + let writer_store = object_store.inner.clone(); + let writer_external_store = external_store.clone(); + let writer_base_path = base_path.clone(); + let writer_staging_path = staging_path.clone(); + let writer_e_tag = staging_meta.e_tag.clone(); + let writer = tokio::spawn(async move { + writer_external_store + .put( + &writer_base_path, + version, + &writer_staging_path, + staging_meta.size, + writer_e_tag, + writer_store.as_ref(), + ManifestNamingScheme::V2, + ) + .await + }); + + tokio::time::timeout( + std::time::Duration::from_secs(5), + external_store.first_final_publish_started.notified(), + ) + .await + .expect("the direct finalizer should pause after COPY"); + + let first_generation = object_store.inner.head(&final_path).await.unwrap(); + let reservation = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .unwrap(); + assert_eq!(reservation.path, staging_path); + assert_eq!(reservation.e_tag, None); + + // The writer created generation E1. While its final index update is + // paused, a reader observes the DDB-selected staging path and performs + // the same immutable copy, producing generation E2. Each helper HEADs + // the canonical object after its copy and returns the generation it + // observed, but neither persists that race-prone token in the external + // index. Both copies have exactly the same bytes; only their physical + // object generations differ. + let reader_location = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .unwrap(); + + external_store.release_first_final_publish.notify_one(); + let writer_location = writer.await.unwrap().unwrap(); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + let final_bytes = object_store + .inner + .get(&final_path) + .await + .unwrap() + .bytes() + .await + .unwrap(); + let indexed = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .unwrap(); + + assert_eq!(final_bytes, manifest_bytes); + assert_ne!( + first_generation.e_tag, final_meta.e_tag, + "the deterministic race must create a new physical generation" + ); + assert_eq!(writer_location.e_tag, first_generation.e_tag); + assert_eq!(reader_location.e_tag, final_meta.e_tag); + assert_eq!(indexed.path, final_path); + assert_eq!(indexed.size, Some(final_meta.size)); + assert_eq!( + indexed.e_tag, None, + "all finalizers must publish the same generation-independent tuple" + ); + + let resolved = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("the finalized manifest must remain readable after the race"); + assert_eq!(resolved.e_tag, final_meta.e_tag); + } + + #[tokio::test] + async fn test_lost_external_store_response_retains_staging_manifest() { + let external_store = Arc::new(TestExternalManifestStore::new(true)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + + let commit_error = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect_err("the simulated response loss must be surfaced"); + assert!(matches!(commit_error, CommitError::CommitConflict)); + + let staging_path = Path::from(external_store.get("dataset", 1).await.unwrap()); + object_store.inner.head(&staging_path).await.unwrap(); + + let resolved = handler + .resolve_version_location(&base_path, 1, object_store.inner.as_ref()) + .await + .expect("the retained staging manifest must allow finalization"); + assert_eq!( + resolved.path, + ManifestNamingScheme::V2.manifest_path(&base_path, 1) + ); + object_store.inner.head(&resolved.path).await.unwrap(); + } + + #[tokio::test] + async fn test_finalization_returns_etag_without_persisting_it() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + let version = manifest.version; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + + let committed = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect("the generic workflow should commit the canonical manifest"); + assert_eq!(committed.path, final_path); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + assert_eq!( + committed.e_tag, final_meta.e_tag, + "the freshly committed Dataset needs the observed generation for cache separation" + ); + + let indexed = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .expect("the external index must advance after the canonical copy"); + assert_eq!(indexed.path, final_path); + assert_eq!( + indexed.e_tag, None, + "the external index must remain independent of physical generations" + ); + } + + #[tokio::test] + async fn test_missing_staging_verifies_existing_final_manifest() { + let object_store = ObjectStore::memory(); + let staging_path = Path::from("dataset/_versions/1.manifest-missing"); + let final_path = Path::from("dataset/_versions/1.manifest"); + let manifest_bytes = Bytes::from_static(b"immutable manifest bytes"); + object_store + .inner + .put(&final_path, manifest_bytes.clone().into()) + .await + .unwrap(); + let final_meta = object_store.inner.head(&final_path).await.unwrap(); + + let recovered_e_tag = copy_or_verify_final_manifest( + object_store.inner.as_ref(), + &staging_path, + &final_path, + 1, + manifest_bytes.len() as u64, + ) + .await + .expect("an existing canonical manifest should prove another helper finalized it"); + + assert_eq!(recovered_e_tag, final_meta.e_tag); + } + + #[tokio::test] + async fn test_missing_staging_rejects_missing_final_manifest() { + let object_store = ObjectStore::memory(); + let staging_path = Path::from("dataset/_versions/1.manifest-missing"); + let final_path = Path::from("dataset/_versions/1.manifest"); + + let error = copy_or_verify_final_manifest( + object_store.inner.as_ref(), + &staging_path, + &final_path, + 1, + 42, + ) + .await + .expect_err("missing staging and canonical objects cannot establish a commit"); + + assert!(matches!(error, Error::NotFound { .. }), "{error:?}"); + assert!(error.to_string().contains(final_path.as_ref()), "{error}"); + } + + #[tokio::test] + async fn test_missing_staging_rejects_wrong_final_size() { + let object_store = ObjectStore::memory(); + let staging_path = Path::from("dataset/_versions/1.manifest-missing"); + let final_path = Path::from("dataset/_versions/1.manifest"); + object_store + .inner + .put(&final_path, Bytes::from_static(b"wrong size").into()) + .await + .unwrap(); + + let error = copy_or_verify_final_manifest( + object_store.inner.as_ref(), + &staging_path, + &final_path, + 1, + 42, + ) + .await + .expect_err("a same-path object with the wrong size is not the selected manifest"); + + assert!(matches!(error, Error::CorruptFile { .. }), "{error:?}"); + assert!( + error.to_string().contains("Manifest size mismatch"), + "{error}" + ); + } + + #[tokio::test] + async fn test_copy_failure_after_external_store_commit_retains_staging_manifest() { + let external_store = Arc::new(TestExternalManifestStore::new(false)); + let handler = ExternalManifestCommitHandler { + external_manifest_store: external_store.clone(), + }; + + let mut object_store = ObjectStore::memory(); + let fail_next_copy = Arc::new(AtomicBool::new(true)); + let failed_copy_source = Arc::new(Mutex::new(None)); + let mut policy = ProxyObjectStorePolicy::new(); + let policy_fail_next_copy = fail_next_copy.clone(); + let policy_failed_copy_source = failed_copy_source.clone(); + policy.set_before_policy( + "fail-copy-once", + Arc::new(move |method, location| { + if method == "copy" && policy_fail_next_copy.swap(false, Ordering::SeqCst) { + *policy_failed_copy_source.lock().unwrap() = Some(location.clone()); + return Err(Error::io("simulated copy failure")); + } + Ok(()) + }), + ); + let policy = Arc::new(Mutex::new(policy)); + object_store.inner = Arc::new(ProxyObjectStore::new( + object_store.inner.clone(), + policy.clone(), + )); + + let base_path = Path::from("dataset"); + let mut manifest = test_manifest(); + let version = manifest.version; + let canonical_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + + let commit_error = handler + .commit( + &mut manifest, + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .expect_err("the simulated copy failure must be surfaced"); + assert!(matches!(commit_error, CommitError::CommitConflict)); + assert!( + !fail_next_copy.load(Ordering::SeqCst), + "the one-shot copy failure must be consumed" + ); + + let recorded_location = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .expect("the external store must retain the committed staging location"); + let staging_path = failed_copy_source + .lock() + .unwrap() + .clone() + .expect("the failure must be injected at copy(staging, canonical)"); + assert_eq!(recorded_location.path, staging_path); + object_store + .inner + .head(&staging_path) + .await + .expect("the winning staging manifest must be retained"); + + let canonical_error = object_store + .inner + .head(&canonical_path) + .await + .expect_err("copy failed before creating the canonical manifest"); + assert!( + matches!(canonical_error, ObjectStoreError::NotFound { .. }), + "unexpected canonical manifest error: {canonical_error}" + ); + + policy.lock().unwrap().clear_before_policy("fail-copy-once"); + let resolved = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("the retained staging manifest must allow finalization"); + assert_eq!(resolved.path, canonical_path); + + let finalized_location = external_store + .get_manifest_location(base_path.as_ref(), version) + .await + .expect("the external store must publish the canonical location"); + assert_eq!(finalized_location.path, canonical_path); + object_store + .inner + .head(&canonical_path) + .await + .expect("the canonical manifest must exist after finalization"); + + let staging_error = object_store + .inner + .head(&staging_path) + .await + .expect_err("successful finalization must clean up the staging manifest"); + assert!( + matches!(staging_error, ObjectStoreError::NotFound { .. }), + "unexpected staging manifest error: {staging_error}" + ); + } + + /// `(path, size, identity)` per version; identities are minted per record + /// and never reused. + #[derive(Debug, Default)] + struct IdentifiedStore { + rows: Mutex>, + next_identity: AtomicUsize, + hold_next_reservation: AtomicBool, + reservation_held: Notify, + release_reservation: Notify, + } + + impl IdentifiedStore { + fn mint(&self) -> String { + format!( + "identity-{}", + self.next_identity.fetch_add(1, Ordering::SeqCst) + ) + } + + fn handler(self: &Arc) -> ExternalManifestCommitHandler { + ExternalManifestCommitHandler { + external_manifest_store: self.clone(), + } + } + + /// Drop every record and write a replacement dataset's records at the + /// same versions. + fn recreate(&self) { + let mut rows = self.rows.lock().unwrap(); + let versions: Vec = rows.keys().copied().collect(); + rows.clear(); + for version in versions { + rows.insert(version, (v2_path(version), 1, self.mint())); + } + } + + fn identity_of(&self, version: u64) -> Option { + self.rows + .lock() + .unwrap() + .get(&version) + .map(|row| row.2.clone()) + } + } + + #[async_trait] + impl ExternalManifestStore for IdentifiedStore { + async fn get(&self, _base_uri: &str, version: u64) -> Result { + self.rows + .lock() + .unwrap() + .get(&version) + .map(|row| row.0.clone()) + .ok_or_else(|| Error::not_found(format!("@{version}"))) + } + + async fn get_manifest_location( + &self, + _base_uri: &str, + version: u64, + ) -> Result { + let row = self + .rows + .lock() + .unwrap() + .get(&version) + .cloned() + .ok_or_else(|| Error::not_found(format!("@{version}")))?; + let path = Path::parse(&row.0).unwrap(); + Ok(ManifestLocation { + version, + naming_scheme: detect_naming_scheme_from_path(&path)?, + path, + size: Some(row.1), + e_tag: None, + identity: Some(row.2), + }) + } + + async fn get_latest_version(&self, _base_uri: &str) -> Result> { + Ok(self + .rows + .lock() + .unwrap() + .iter() + .max_by_key(|(version, _)| **version) + .map(|(version, row)| (*version, row.0.clone()))) + } + + async fn get_latest_manifest_location( + &self, + base_uri: &str, + ) -> Result> { + match self.get_latest_version(base_uri).await? { + Some((version, _)) => self + .get_manifest_location(base_uri, version) + .await + .map(Some), + None => Ok(None), + } + } + + async fn put_if_not_exists( + &self, + _base_uri: &str, + version: u64, + path: &str, + size: u64, + _e_tag: Option, + ) -> Result<()> { + let identity = self.mint(); + let mut rows = self.rows.lock().unwrap(); + if rows.contains_key(&version) { + return Err(Error::commit_conflict_source(version, "exists".into())); + } + rows.insert(version, (path.to_string(), size, identity)); + Ok(()) + } + + async fn put_if_exists( + &self, + _base_uri: &str, + version: u64, + path: &str, + size: u64, + _e_tag: Option, + ) -> Result<()> { + let mut rows = self.rows.lock().unwrap(); + let row = rows + .get_mut(&version) + .ok_or_else(|| Error::not_found(format!("@{version}")))?; + row.0 = path.to_string(); + row.1 = size; + Ok(()) + } + + fn supports_predecessor_condition(&self) -> bool { + true + } + + async fn get_identity(&self, _base_uri: &str, version: u64) -> Result> { + Ok(self.identity_of(version)) + } + + async fn forget_version( + &self, + _base_uri: &str, + version: u64, + identity: &str, + ) -> Result<()> { + let mut rows = self.rows.lock().unwrap(); + if rows.get(&version).is_some_and(|row| row.2 == identity) { + rows.remove(&version); + } + Ok(()) + } + + async fn list_versions( + &self, + base_uri: &str, + since: Option, + ) -> Result>> { + let versions: Vec = self.rows.lock().unwrap().keys().copied().collect(); + let mut locations = Vec::new(); + for version in versions { + if since.is_none_or(|since| version > since) { + locations.push(self.get_manifest_location(base_uri, version).await?); + } + } + Ok(Some(locations)) + } + + async fn put_if_predecessor( + &self, + _base_uri: &str, + version: u64, + path: &str, + size: u64, + predecessor: &PredecessorIdentity, + ) -> Result { + if self.hold_next_reservation.swap(false, Ordering::SeqCst) { + self.reservation_held.notify_one(); + self.release_reservation.notified().await; + } + let identity = self.mint(); + let mut rows = self.rows.lock().unwrap(); + let held = rows + .get(&predecessor.version) + .is_some_and(|row| row.2 == predecessor.identity); + if !held { + return Ok(Reservation::PredecessorChanged); + } + if rows.contains_key(&version) { + return Ok(Reservation::Taken); + } + rows.insert(version, (path.to_string(), size, identity.clone())); + Ok(Reservation::Reserved { identity }) + } + } + + fn v2_path(version: u64) -> String { + ManifestNamingScheme::V2 + .manifest_path(&Path::from("dataset"), version) + .to_string() + } + + fn v2_names(versions: &[u64]) -> Vec { + let mut names: Vec = versions + .iter() + .map(|v| Path::from(v2_path(*v)).filename().unwrap().to_string()) + .collect(); + names.sort(); + names + } + + /// Version 1 committed through `store`, plus what a conditioned commit of + /// version 2 needs. + async fn identified_fixture( + store: &Arc, + ) -> ( + ExternalManifestCommitHandler, + ObjectStore, + Path, + PredecessorIdentity, + ) { + let handler = store.handler(); + let object_store = ObjectStore::memory(); + let base_path = Path::from("dataset"); + handler + .commit( + &mut test_manifest(), + None, + &base_path, + &object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + ) + .await + .unwrap(); + let predecessor = handler + .resolve_latest_identity(&base_path, &object_store) + .await + .unwrap() + .unwrap(); + assert_eq!(predecessor.version, 1); + (handler, object_store, base_path, predecessor) + } + + async fn commit_after_v2( + handler: &ExternalManifestCommitHandler, + object_store: &ObjectStore, + base_path: &Path, + predecessor: &PredecessorIdentity, + ) -> std::result::Result { + let mut manifest = test_manifest(); + manifest.version = 2; + handler + .commit_after( + &mut manifest, + None, + base_path, + object_store, + write_manifest_file_to_path, + ManifestNamingScheme::V2, + None, + predecessor, + ) + .await + } + + async fn versions_dir_files(object_store: &ObjectStore, base_path: &Path) -> Vec { + let mut files: Vec = object_store + .inner + .list(Some(&base_path.clone().join(VERSIONS_DIR))) + .map_ok(|meta| meta.location.filename().unwrap().to_string()) + .try_collect() + .await + .unwrap(); + files.sort(); + files + } + + #[tokio::test] + async fn test_a_conditioned_commit_lands_under_its_minted_identity() { + let store = Arc::new(IdentifiedStore::default()); + let (handler, object_store, base_path, predecessor) = identified_fixture(&store).await; + let location = commit_after_v2(&handler, &object_store, &base_path, &predecessor) + .await + .unwrap(); + // Published at a staging name: invisible to object-store listing, + // final only through the store's record. + let name = location.path.filename().unwrap(); + assert!(name.contains(".manifest-"), "{name}"); + assert_eq!(ManifestNamingScheme::detect_scheme(name), None); + + assert!(location.identity.is_some()); + assert_eq!(location.identity, store.identity_of(2)); + let resolved = handler + .resolve_latest_location(&base_path, &object_store) + .await + .unwrap(); + assert_eq!(resolved.path, location.path); + assert_eq!(resolved.identity, location.identity); + // The store, not the object store, is the history. + assert_eq!( + listed_versions(&handler, &object_store, &base_path).await, + vec![2, 1] + ); + let since: Vec = handler + .list_manifest_locations_since(&base_path, &object_store, 1) + .map_ok(|l| l.version) + .try_collect() + .await + .unwrap(); + assert_eq!(since, vec![2]); + assert_eq!(versions_dir_files(&object_store, &base_path).await.len(), 2); + } + + #[tokio::test] + async fn test_a_changed_predecessor_is_refused_without_publishing() { + let store = Arc::new(IdentifiedStore::default()); + let (handler, object_store, base_path, _) = identified_fixture(&store).await; + let stale = PredecessorIdentity { + version: 1, + identity: "identity-from-a-dropped-dataset".to_string(), + }; + let err = commit_after_v2(&handler, &object_store, &base_path, &stale) + .await + .unwrap_err(); + assert!( + matches!( + err, + CommitError::OtherError(Error::PrerequisiteFailed { .. }) + ), + "{err:?}" + ); + assert!(store.identity_of(2).is_none()); + assert_eq!( + versions_dir_files(&object_store, &base_path).await, + v2_names(&[1]) + ); + } + + #[tokio::test] + async fn test_a_taken_version_is_a_conflict() { + let store = Arc::new(IdentifiedStore::default()); + let (handler, object_store, base_path, predecessor) = identified_fixture(&store).await; + store + .put_if_not_exists("dataset", 2, &v2_path(2), 1, None) + .await + .unwrap(); + let err = commit_after_v2(&handler, &object_store, &base_path, &predecessor) + .await + .unwrap_err(); + assert!(matches!(err, CommitError::CommitConflict), "{err:?}"); + assert_eq!( + versions_dir_files(&object_store, &base_path).await, + v2_names(&[1]) + ); + } + + /// A recreated dataset's records never carry the observed identity, so + /// the reservation refuses and nothing is published. + #[tokio::test] + async fn test_a_recreation_before_publication_is_refused() { + let store = Arc::new(IdentifiedStore::default()); + let (handler, object_store, base_path, predecessor) = identified_fixture(&store).await; + store.recreate(); + let err = commit_after_v2(&handler, &object_store, &base_path, &predecessor) + .await + .unwrap_err(); + assert!( + matches!( + err, + CommitError::OtherError(Error::PrerequisiteFailed { .. }) + ), + "{err:?}" + ); + assert_eq!( + versions_dir_files(&object_store, &base_path).await, + v2_names(&[1]) + ); + } + async fn listed_versions( + handler: &ExternalManifestCommitHandler, + object_store: &ObjectStore, + base_path: &Path, + ) -> Vec { + handler + .list_manifest_locations(base_path, object_store, true) + .map_ok(|l| l.version) + .try_collect() + .await + .unwrap() + } + + /// A commit cancelled after its write but before the reservation leaves + /// an object nothing discovers: no record, and no listed version. + #[tokio::test(flavor = "multi_thread")] + async fn test_a_cancelled_reservation_publishes_nothing() { + let store = Arc::new(IdentifiedStore::default()); + let (handler, object_store, base_path, predecessor) = identified_fixture(&store).await; + store.hold_next_reservation.store(true, Ordering::SeqCst); + let task = { + let (handler, object_store, base_path) = + (store.handler(), object_store.clone(), base_path.clone()); + tokio::spawn(async move { + commit_after_v2(&handler, &object_store, &base_path, &predecessor).await + }) + }; + tokio::time::timeout( + std::time::Duration::from_secs(30), + store.reservation_held.notified(), + ) + .await + .expect("the commit never reached its reservation"); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + + assert!(store.identity_of(2).is_none()); + assert_eq!( + listed_versions(&handler, &object_store, &base_path).await, + vec![1] + ); + // The orphaned object is on the object store, but not as a version. + assert_eq!(versions_dir_files(&object_store, &base_path).await.len(), 2); + let raw: Vec = default_list_manifest_locations(&base_path, &object_store, true) + .map_ok(|l| l.version) + .try_collect() + .await + .unwrap(); + assert_eq!(raw, vec![1]); + } + /// Forgetting retires exactly the record cleanup removed: a stale identity + /// leaves a recreated dataset's record alone, and repeats are no-ops. + #[tokio::test] + async fn test_forgetting_a_version_retires_only_that_record() { + let store = Arc::new(IdentifiedStore::default()); + let (handler, object_store, base_path, predecessor) = identified_fixture(&store).await; + commit_after_v2(&handler, &object_store, &base_path, &predecessor) + .await + .unwrap(); + handler + .forget_version(&base_path, 1, "identity-from-a-dropped-dataset") + .await + .unwrap(); + assert_eq!( + listed_versions(&handler, &object_store, &base_path).await, + vec![2, 1] + ); + let identity = store.identity_of(1).unwrap(); + handler + .forget_version(&base_path, 1, &identity) + .await + .unwrap(); + handler + .forget_version(&base_path, 1, &identity) + .await + .unwrap(); + assert_eq!( + listed_versions(&handler, &object_store, &base_path).await, + vec![2] + ); + } + /// A store that mints identities but cannot retire records fails cleanup + /// loudly instead of leaving rows behind. + #[tokio::test] + async fn test_retirement_is_refused_where_the_store_cannot_forget() { + #[derive(Debug)] + struct NoForget(Arc); + #[async_trait] + impl ExternalManifestStore for NoForget { + async fn get(&self, b: &str, v: u64) -> Result { + self.0.get(b, v).await + } + async fn get_latest_version(&self, b: &str) -> Result> { + self.0.get_latest_version(b).await + } + async fn put_if_not_exists( + &self, + b: &str, + v: u64, + p: &str, + s: u64, + e: Option, + ) -> Result<()> { + self.0.put_if_not_exists(b, v, p, s, e).await + } + async fn put_if_exists( + &self, + b: &str, + v: u64, + p: &str, + s: u64, + e: Option, + ) -> Result<()> { + self.0.put_if_exists(b, v, p, s, e).await + } + fn supports_predecessor_condition(&self) -> bool { + true + } + } + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(NoForget(Arc::new(IdentifiedStore::default()))), + }; + let err = handler + .forget_version(&Path::from("dataset"), 1, "identity-0") + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. }), "{err}"); + } } diff --git a/rust/lance-table/src/io/deletion.rs b/rust/lance-table/src/io/deletion.rs index 01bc6d3ba18..a26a8ddd7ad 100644 --- a/rust/lance-table/src/io/deletion.rs +++ b/rust/lance-table/src/io/deletion.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{collections::HashSet, sync::Arc}; +use std::{collections::HashSet, sync::Arc, sync::LazyLock}; use arrow_array::{RecordBatch, UInt32Array}; use arrow_ipc::CompressionType; use arrow_ipc::reader::FileReader as ArrowFileReader; use arrow_ipc::writer::{FileWriter as ArrowFileWriter, IpcWriteOptions}; -use arrow_schema::{ArrowError, DataType, Field, Schema}; +use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef}; use bytes::Buf; use lance_core::error::{CorruptFileSnafu, box_error}; use lance_core::utils::deletion::DeletionVector; @@ -24,14 +24,14 @@ use crate::format::{DeletionFile, DeletionFileType}; pub const DELETIONS_DIR: &str = "_deletions"; -/// Get the Arrow schema for an Arrow deletion file. -fn deletion_arrow_schema() -> Arc { +/// The Arrow schema for an Arrow deletion file. +static DELETION_ARROW_SCHEMA: LazyLock = LazyLock::new(|| { Arc::new(Schema::new(vec![Field::new( "row_id", DataType::UInt32, false, )])) -} +}); /// Get the file path for a deletion file. This is relative to the dataset root. pub fn deletion_file_path(base: &Path, fragment_id: u64, deletion_file: &DeletionFile) -> Path { @@ -85,7 +85,7 @@ pub async fn write_deletion_file( let array = UInt32Array::from_iter(set.iter().copied()); let array = Arc::new(array); - let schema = deletion_arrow_schema(); + let schema = DELETION_ARROW_SCHEMA.clone(); let batch = RecordBatch::try_new(schema.clone(), vec![array])?; let mut out: Vec = Vec::new(); @@ -170,12 +170,12 @@ pub async fn read_deletion_file( } let batch = batches.pop().unwrap(); - if batch.schema() != deletion_arrow_schema() { + if batch.schema().as_ref() != DELETION_ARROW_SCHEMA.as_ref() { return Err(Error::corrupt_file( path, format!( "Expected schema {:?} in deletion file, got {:?}", - deletion_arrow_schema(), + DELETION_ARROW_SCHEMA.as_ref(), batch.schema() ), )); @@ -279,7 +279,7 @@ mod test { assert_eq!(batches.len(), 1); let batch = batches.pop().unwrap(); - assert_eq!(batch.schema(), deletion_arrow_schema()); + assert_eq!(batch.schema(), *DELETION_ARROW_SCHEMA); let array = batch["row_id"] .as_any() .downcast_ref::() diff --git a/rust/lance-table/src/io/manifest.rs b/rust/lance-table/src/io/manifest.rs index 6a6bfd2724c..f3e6dc0a7bf 100644 --- a/rust/lance-table/src/io/manifest.rs +++ b/rust/lance-table/src/io/manifest.rs @@ -4,9 +4,12 @@ use async_trait::async_trait; use byteorder::{ByteOrder, LittleEndian}; use bytes::{Bytes, BytesMut}; -use lance_arrow::DataTypeExt; +use futures::TryStreamExt; use lance_file::{ - previous::writer::ManifestProvider as PreviousManifestProvider, version::LanceFileVersion, + version::ConcreteFileVersion, + versions::v1::{ + encoding::write_schema_dictionaries, writer::ManifestProvider as V1ManifestProvider, + }, }; use object_store::ObjectStoreExt; use object_store::path::Path; @@ -17,10 +20,9 @@ use tracing::instrument; use lance_core::{Error, Result, datatypes::Schema}; use lance_io::{ - encodings::{Encoder, binary::BinaryEncoder, plain::PlainEncoder}, object_store::ObjectStore, traits::{WriteExt, Writer}, - utils::read_message, + utils::{METADATA_READ_CHUNK_SIZE, read_message, read_range_in_chunks}, }; use crate::format::{DataStorageFormat, IndexMetadata, MAGIC, Manifest, Transaction, pb}; @@ -74,20 +76,22 @@ pub async fn read_manifest( // The prefetch captured the entire manifest. We just need to trim the buffer. buf.slice(buf.len() - manifest_len..buf.len()) } else { - // The prefetch only captured part of the manifest. We need to make an - // additional range request to read the remainder. - let mut buf2: BytesMut = object_store - .inner - .get_range( - path, - Range { - start: manifest_pos as u64, - end: file_size - PREFETCH_SIZE, - }, - ) - .await? - .into_iter() - .collect(); + // The prefetch only captured part of the manifest. Fetch the remainder + // as concurrent chunked range requests: a single GET is limited to one + // connection's throughput, which dominates load time for manifests of + // datasets with many fragments. + let reader = object_store + .open_with_size(path, file_size as usize) + .await?; + let mut buf2 = BytesMut::with_capacity(manifest_len); + let mut chunks = read_range_in_chunks( + reader.as_ref(), + manifest_pos..(file_size - PREFETCH_SIZE) as usize, + METADATA_READ_CHUNK_SIZE, + ); + while let Some(chunk) = chunks.try_next().await? { + buf2.extend_from_slice(&chunk); + } buf2.extend_from_slice(&buf); buf2.freeze() }; @@ -168,6 +172,10 @@ async fn do_write_manifest( }; let pos = writer.write_protobuf(§ion).await?; manifest.index_section = Some(pos); + } else { + // No index section is written to this file, so an inherited offset + // would point at unrelated bytes in the new manifest file. + manifest.index_section = None; } // Write inline transaction if presented. @@ -176,6 +184,11 @@ async fn do_write_manifest( let pb_tx: pb::Transaction = tx.into(); let pos = writer.write_protobuf(&pb_tx).await?; manifest.transaction_section = Some(pos); + } else { + // No inline copy is written to this file. Clear any offset inherited + // from a previous manifest (e.g. via restore or clone), which would + // otherwise point at arbitrary bytes of the file being written. + manifest.transaction_section = None; } writer.write_struct(manifest).await @@ -188,45 +201,14 @@ pub async fn write_manifest( indices: Option>, transaction: Option, ) -> Result { - // Write dictionary values. - let max_field_id = manifest.schema.max_field_id().unwrap_or(-1); - let is_legacy_storage = manifest.should_use_legacy_format(); - for field_id in 0..max_field_id + 1 { - if let Some(field) = manifest.schema.mut_field_by_id(field_id) - && field.data_type().is_dictionary() - && is_legacy_storage - { - let dict_info = field.dictionary.as_mut().ok_or_else(|| { - Error::io(format!("Lance field {} misses dictionary info", field.name)) - })?; - - let value_arr = dict_info.values.as_ref().ok_or_else(|| { - Error::io(format!( - "Lance field {} is dictionary type, but misses the dictionary value array", - field.name - )) - })?; - - let data_type = value_arr.data_type(); - let pos = match data_type { - dt if dt.is_numeric() => { - let mut encoder = PlainEncoder::new(writer, dt); - encoder.encode(&[value_arr]).await? - } - dt if dt.is_binary_like() => { - let mut encoder = BinaryEncoder::new(writer); - encoder.encode(&[value_arr]).await? - } - _ => { - return Err(Error::schema(format!( - "Does not support {} as dictionary value type", - value_arr.data_type() - ))); - } - }; - dict_info.offset = pos; - dict_info.length = value_arr.len(); + match manifest.data_storage_format.version { + ConcreteFileVersion::V1 => { + write_schema_dictionaries(writer, &mut manifest.schema).await?; } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => {} } do_write_manifest(writer, manifest, indices, transaction).await @@ -237,7 +219,7 @@ pub async fn write_manifest( pub struct ManifestDescribing {} #[async_trait] -impl PreviousManifestProvider for ManifestDescribing { +impl V1ManifestProvider for ManifestDescribing { async fn store_schema( object_writer: &mut dyn Writer, schema: &Schema, @@ -245,7 +227,7 @@ impl PreviousManifestProvider for ManifestDescribing { let mut manifest = Manifest::new( schema.clone(), Arc::new(vec![]), - DataStorageFormat::new(LanceFileVersion::Legacy), + DataStorageFormat::new(ConcreteFileVersion::V1), HashMap::new(), ); let pos = do_write_manifest(object_writer, &mut manifest, None, None).await?; @@ -261,8 +243,8 @@ mod test { use crate::format::SelfDescribingFileReader; use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use lance_file::format::{MAGIC, MAJOR_VERSION, MINOR_VERSION}; - use lance_file::previous::{ - reader::FileReader as PreviousFileReader, writer::FileWriter as PreviousFileWriter, + use lance_file::versions::v1::{ + reader::FileReader as V1FileReader, writer::FileWriter as V1FileWriter, }; use rand::{Rng, distr::Alphanumeric}; use tokio::io::AsyncWriteExt; @@ -282,11 +264,8 @@ mod test { .collect(); writer.write_all(&prefix).await.unwrap(); - let long_name: String = rand::rng() - .sample_iter(&Alphanumeric) - .take(manifest_min_size) - .map(char::from) - .collect(); + // A cheap deterministic filler; only the size matters for these tests. + let long_name: String = "a".repeat(manifest_min_size); let arrow_schema = ArrowSchema::new(vec![ArrowField::new(long_name, DataType::Int64, false)]); @@ -324,6 +303,40 @@ mod test { test_roundtrip_manifest(1000, 1000).await; } + #[tokio::test] + async fn test_read_manifest_larger_than_read_chunk() { + // Crosses METADATA_READ_CHUNK_SIZE so the manifest body is fetched as + // multiple concurrent chunks and reassembled with the prefetched tail. + test_roundtrip_manifest(1000, METADATA_READ_CHUNK_SIZE + 4 * 1024 * 1024).await; + } + + #[tokio::test] + async fn test_write_manifest_clears_unwritten_index_section() { + let store = ObjectStore::memory(); + let path = Path::from("/clear_unwritten_index_section"); + let mut writer = store.create(&path).await.unwrap(); + let mut manifest = Manifest::new( + Schema::default(), + Arc::new(vec![]), + DataStorageFormat::default(), + HashMap::new(), + ); + manifest.index_section = Some(42); + + let pos = write_manifest(writer.as_mut(), &mut manifest, None, None) + .await + .unwrap(); + writer + .write_magics(pos, MAJOR_VERSION, MINOR_VERSION, MAGIC) + .await + .unwrap(); + Writer::shutdown(writer.as_mut()).await.unwrap(); + + assert!(manifest.index_section.is_none()); + let roundtripped_manifest = read_manifest(&store, &path, None).await.unwrap(); + assert!(roundtripped_manifest.index_section.is_none()); + } + #[tokio::test] async fn test_update_schema_metadata() { let store = ObjectStore::memory(); @@ -335,7 +348,7 @@ mod test { false, )])); let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut file_writer = PreviousFileWriter::::try_new( + let mut file_writer = V1FileWriter::::try_new( &store, &path, schema.clone(), @@ -355,7 +368,7 @@ mod test { file_writer.finish_with_metadata(&metadata).await.unwrap(); let reader = store.open(&path).await.unwrap(); - let reader = PreviousFileReader::try_new_self_described_from_reader(reader.into(), None) + let reader = V1FileReader::try_new_self_described_from_reader(reader.into(), None) .await .unwrap(); let schema = ArrowSchema::from(reader.schema()); diff --git a/rust/lance-table/src/lib.rs b/rust/lance-table/src/lib.rs index 89b424adc61..1a008740439 100644 --- a/rust/lance-table/src/lib.rs +++ b/rust/lance-table/src/lib.rs @@ -6,4 +6,5 @@ pub mod format; pub mod io; pub mod rowids; pub mod system_index; +pub mod transaction; pub mod utils; diff --git a/rust/lance-table/src/rowids.rs b/rust/lance-table/src/rowids.rs index ab5dac72b48..4cadaf67b76 100644 --- a/rust/lance-table/src/rowids.rs +++ b/rust/lance-table/src/rowids.rs @@ -32,7 +32,7 @@ use lance_select::{RowAddrMask, RowAddrTreeMap, RowSetOps}; pub use serde::{read_row_ids, write_row_ids}; use crate::utils::LanceIteratorExtension; -use segment::U64Segment; +use segment::{SegmentCursorState, U64Segment}; use tracing::instrument; /// A sequence of row ids. @@ -50,6 +50,125 @@ use tracing::instrument; #[derive(Debug, Clone, DeepSizeOf, PartialEq, Eq, Default)] pub struct RowIdSequence(Vec); +/// Stateful reader for selections that usually advance through a sequence. +/// +/// Streaming readers reuse this cursor across record batches. If a later +/// selection moves backwards then the cursor rewinds before continuing. +#[derive(Debug, Default)] +pub(crate) struct RowIdSequenceCursor { + segment_idx: usize, + rows_passed: usize, + segment_len: Option, + segment_cursor: SegmentCursorState, + last_index: Option, +} + +impl RowIdSequenceCursor { + fn advance_segment(&mut self) { + self.rows_passed += self.segment_len.unwrap_or_default(); + self.segment_idx += 1; + self.segment_len = None; + self.segment_cursor = SegmentCursorState::default(); + } + + fn get(&mut self, sequence: &RowIdSequence, index: usize) -> Option { + if index < self.rows_passed || self.last_index.is_some_and(|last| index < last) { + *self = Self::default(); + } + self.last_index = Some(index); + + loop { + let segment = sequence.0.get(self.segment_idx)?; + let segment_len = *self.segment_len.get_or_insert_with(|| segment.len()); + let local_index = index - self.rows_passed; + if local_index < segment_len { + return self.segment_cursor.get(segment, local_index); + } + self.advance_segment(); + } + } + + fn extend_range( + &mut self, + sequence: &RowIdSequence, + selection: Range, + row_ids: &mut Vec, + ) { + if selection.is_empty() { + return; + } + if selection.start < self.rows_passed + || self.last_index.is_some_and(|last| selection.start < last) + { + *self = Self::default(); + } + self.last_index = Some(selection.end - 1); + + let mut index = selection.start; + while index < selection.end { + let Some(segment) = sequence.0.get(self.segment_idx) else { + break; + }; + let segment_len = *self.segment_len.get_or_insert_with(|| segment.len()); + let local_start = index - self.rows_passed; + if local_start >= segment_len { + self.advance_segment(); + continue; + } + + let count = (selection.end - index).min(segment_len - local_start); + let local_end = local_start + count; + self.segment_cursor + .extend_range(segment, local_start..local_end, row_ids); + index += count; + if local_end == segment_len { + self.advance_segment(); + } + } + } + + // Keep the sparse loop in `extend_range` unchanged. Sharing this loop with + // the dense decoder measurably slows sparse system-only scans. + fn extend_dense_range( + &mut self, + sequence: &RowIdSequence, + selection: Range, + row_ids: &mut Vec, + ) { + if selection.is_empty() { + return; + } + if selection.start < self.rows_passed + || self.last_index.is_some_and(|last| selection.start < last) + { + *self = Self::default(); + } + self.last_index = Some(selection.end - 1); + + let mut index = selection.start; + while index < selection.end { + let Some(segment) = sequence.0.get(self.segment_idx) else { + break; + }; + let segment_len = *self.segment_len.get_or_insert_with(|| segment.len()); + let local_start = index - self.rows_passed; + if local_start >= segment_len { + self.advance_segment(); + continue; + } + + let count = (selection.end - index).min(segment_len - local_start); + let local_end = local_start + count; + self.segment_cursor + .extend_dense_range(segment, local_start..local_end, row_ids); + index += count; + if local_end == segment_len { + self.advance_segment(); + } + } + } +} + impl std::fmt::Display for RowIdSequence { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let mut iter = self.iter(); @@ -98,11 +217,52 @@ impl From<&[u64]> for RowIdSequence { } } +/// Return some value that appears more than once in `row_ids`, if any. +/// +/// The already-sorted case is the common one for row id sequences, and is +/// checked in a single pass without allocating. +fn find_duplicate(row_ids: &[u64]) -> Option { + if row_ids.windows(2).all(|pair| pair[0] < pair[1]) { + return None; + } + let mut sorted = row_ids.to_vec(); + sorted.sort_unstable(); + sorted + .windows(2) + .find(|pair| pair[0] == pair[1]) + .map(|pair| pair[0]) +} + impl RowIdSequence { pub fn new() -> Self { Self::default() } + /// Build a sequence from row ids, rejecting duplicates within the sequence. + /// + /// The segment encodings represent a sorted run as a range plus its holes, + /// so a repeated value would be silently encoded as a shorter sequence with + /// a spurious hole. Callers assembling a sequence from untrusted input + /// should use this instead of the infallible `From` conversions, which + /// assume uniqueness. + /// + /// Row ids must also be unique across the dataset. That is not checked + /// here, and commit does not re-check it either. + pub fn try_from_iter(row_ids: impl IntoIterator) -> Result { + let row_ids: Vec = row_ids.into_iter().collect(); + if row_ids.is_empty() { + return Ok(Self::new()); + } + if let Some(duplicate) = find_duplicate(&row_ids) { + return Err(Error::invalid_input(format!( + "Row ids must be unique, but row id {} appears more than once in the sequence of {} row ids", + duplicate, + row_ids.len() + ))); + } + Ok(Self(vec![U64Segment::from_iter(row_ids)])) + } + pub fn iter(&self) -> impl DoubleEndedIterator + '_ { self.0.iter().flat_map(|segment| segment.iter()) } @@ -316,6 +476,11 @@ impl RowIdSequence { /// Get the row id at the given index. /// /// If the index is out of bounds, this will return None. + /// The segments backing the sequence, in offset order. + pub fn segments(&self) -> &[U64Segment] { + &self.0 + } + pub fn get(&self, index: usize) -> Option { let mut offset = 0; for segment in &self.0 { @@ -339,31 +504,71 @@ impl RowIdSequence { &'a self, selection: impl Iterator + 'a, ) -> impl Iterator + 'a { - let mut seg_iter = self.0.iter(); - let mut cur_seg = seg_iter.next(); - let mut rows_passed = 0; - let mut cur_seg_len = cur_seg.map(|seg| seg.len()).unwrap_or(0); - let mut last_index = 0; + let mut cursor = RowIdSequenceCursor::default(); + let mut last_index = None; selection.filter_map(move |index| { - if index < last_index { + if last_index.is_some_and(|last| index < last) { panic!("Selection is not sorted"); } - last_index = index; + last_index = Some(index); + cursor.get(self, index) + }) + } - cur_seg?; + pub(crate) fn cursor(&self) -> RowIdSequenceCursor { + RowIdSequenceCursor::default() + } - while (index - rows_passed) >= cur_seg_len { - rows_passed += cur_seg_len; - cur_seg = seg_iter.next(); - if let Some(cur_seg) = cur_seg { - cur_seg_len = cur_seg.len(); - } else { - return None; - } - } + /// Choose the dense decoder once for a stream and reuse its cardinality. + /// + /// A stream uses one decoder for its lifetime, so multi-segment sequences + /// conservatively retain the sparse path. For a single bitmap segment, the + /// cardinality computed for the density decision seeds the cursor instead + /// of scanning the bitmap again on the first batch. + pub(crate) fn cursor_with_dense_range_expansion(&self) -> (RowIdSequenceCursor, bool) { + let mut cursor = self.cursor(); + let [segment @ U64Segment::RangeWithBitmap { .. }] = self.0.as_slice() else { + return (cursor, false); + }; + let segment_len = segment.len(); + cursor.segment_len = Some(segment_len); + let use_dense_range_expansion = segment.use_dense_range_expansion(segment_len); + (cursor, use_dense_range_expansion) + } - Some(cur_seg.unwrap().get(index - rows_passed).unwrap()) - }) + /// Get a contiguous range of row ids while preserving scan state from a + /// previous call. + pub(crate) fn select_range_with_cursor( + &self, + cursor: &mut RowIdSequenceCursor, + selection: Range, + ) -> Vec { + let mut row_ids = Vec::with_capacity(selection.len()); + cursor.extend_range(self, selection, &mut row_ids); + row_ids + } + + /// Get a contiguous range from a sequence whose bitmap segments are dense. + pub(crate) fn select_dense_range_with_cursor( + &self, + cursor: &mut RowIdSequenceCursor, + selection: Range, + ) -> Vec { + let mut row_ids = Vec::with_capacity(selection.len()); + cursor.extend_dense_range(self, selection, &mut row_ids); + row_ids + } + + /// Get row ids while preserving scan state from a previous call. + /// + /// Decreasing offsets are supported by rewinding the cursor. This matters + /// for take requests, whose indices are not required to be sorted. + pub(crate) fn select_with_cursor<'a>( + &'a self, + cursor: &'a mut RowIdSequenceCursor, + selection: impl Iterator + 'a, + ) -> impl Iterator + 'a { + selection.filter_map(move |index| cursor.get(self, index)) } /// Given a mask of row ids, calculate the offset ranges of the row ids that are present @@ -391,9 +596,8 @@ impl RowIdSequence { ids.mask(mask); // Range-aware path: walk the bitmap's runs directly via // iter_runs so the per-row cost collapses to per-run cost. - // SAFETY: built from a u64 range; no Full entries possible. let mut cur: Option> = None; - for (fragment, run) in unsafe { ids.iter_runs() } { + for (fragment, run) in ids.iter_runs() { let frag = u64::from(fragment); let run_start = (frag << 32) | u64::from(*run.start()); let run_end_excl = (frag << 32) | (u64::from(*run.end()) + 1); @@ -428,19 +632,17 @@ impl RowIdSequence { sorted_holes.sort_unstable(); let mut next_holes_iter = sorted_holes.into_iter().peekable(); let mut holes_passed = 0; - ranges.extend(GroupingIterator::new(unsafe { ids.into_addr_iter() }.map( - |addr| { - while let Some(next_hole) = next_holes_iter.peek() { - if *next_hole < addr { - next_holes_iter.next(); - holes_passed += 1; - } else { - break; - } + ranges.extend(GroupingIterator::new(ids.into_addr_iter().map(|addr| { + while let Some(next_hole) = next_holes_iter.peek() { + if *next_hole < addr { + next_holes_iter.next(); + holes_passed += 1; + } else { + break; } - addr - range.start + offset_start - holes_passed - }, - ))); + } + addr - range.start + offset_start - holes_passed + }))); } U64Segment::RangeWithBitmap { range, bitmap } => { let mut ids = RowAddrTreeMap::from(range.clone()); @@ -455,18 +657,16 @@ impl RowIdSequence { let mut bitmap_iter = bitmap.iter(); let mut bitmap_iter_pos = 0; let mut holes_passed = 0; - ranges.extend(GroupingIterator::new(unsafe { ids.into_addr_iter() }.map( - |addr| { - let position_in_range = addr - range.start; - while bitmap_iter_pos < position_in_range { - if !bitmap_iter.next().unwrap() { - holes_passed += 1; - } - bitmap_iter_pos += 1; + ranges.extend(GroupingIterator::new(ids.into_addr_iter().map(|addr| { + let position_in_range = addr - range.start; + while bitmap_iter_pos < position_in_range { + if !bitmap_iter.next().unwrap() { + holes_passed += 1; } - offset_start + position_in_range - holes_passed - }, - ))); + bitmap_iter_pos += 1; + } + offset_start + position_in_range - holes_passed + }))); } U64Segment::SortedArray(array) | U64Segment::Array(array) => { // TODO: Could probably optimize the sorted array case to be O(N) instead of O(N log N) @@ -721,16 +921,28 @@ pub fn select_row_ids<'a>( }; match offsets { - // TODO: Optimize this if indices are sorted, which is a common case. - ReadBatchParams::Indices(indices) => indices - .values() - .iter() - .map(|index| { - sequence - .get(*index as usize) - .ok_or_else(|| out_of_bounds_err(*index)) - }) - .collect(), + ReadBatchParams::Indices(indices) => { + let indices = indices.values(); + if indices.windows(2).all(|pair| pair[0] <= pair[1]) { + // `select` drops out-of-bounds indices instead of erroring. + if let Some(&last) = indices.last() + && last as u64 >= sequence.len() + { + return Err(out_of_bounds_err(last)); + } + return Ok(sequence + .select(indices.iter().map(|&index| index as usize)) + .collect()); + } + indices + .iter() + .map(|index| { + sequence + .get(*index as usize) + .ok_or_else(|| out_of_bounds_err(*index)) + }) + .collect() + } ReadBatchParams::Range(range) => { if range.end > sequence.len() as usize { return Err(out_of_bounds_err(range.end as u32)); @@ -788,6 +1000,50 @@ mod test { assert_eq!(iter.collect::>(), (0..10).collect::>()); } + #[rstest::rstest] + #[case::sorted_contiguous(vec![0, 1, 2, 3])] + #[case::sorted_with_gaps(vec![0, 2, 4])] + #[case::sparse(vec![0, 1_000_000])] + #[case::unsorted(vec![12, 11, 10])] + fn test_row_id_sequence_try_from_iter(#[case] row_ids: Vec) { + let sequence = RowIdSequence::try_from_iter(row_ids.clone()).unwrap(); + assert_eq!(sequence.len(), row_ids.len() as u64); + assert_eq!(sequence.iter().collect::>(), row_ids); + } + + #[test] + fn test_row_id_sequence_try_from_iter_contiguous_is_a_range() { + let sequence = RowIdSequence::try_from_iter(0..10).unwrap(); + assert_eq!(sequence.0, vec![U64Segment::Range(0..10)]); + } + + #[test] + fn test_row_id_sequence_try_from_iter_empty() { + let sequence = RowIdSequence::try_from_iter(std::iter::empty()).unwrap(); + assert_eq!(sequence.len(), 0); + assert!(sequence.is_empty()); + } + + #[rstest::rstest] + #[case::adjacent(vec![1, 1, 2])] + #[case::separated(vec![1, 2, 3, 1])] + #[case::unsorted(vec![5, 3, 5])] + fn test_row_id_sequence_try_from_iter_rejects_duplicates(#[case] row_ids: Vec) { + // Without validation these encode to a shorter sequence with a spurious + // hole rather than failing, so assert the error rather than the output. + let error = RowIdSequence::try_from_iter(row_ids).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {:?}", + error + ); + assert!( + error.to_string().contains("must be unique"), + "unexpected message: {}", + error + ); + } + #[test] fn test_row_id_sequence_extend() { let mut sequence = RowIdSequence::from(0..10); @@ -937,6 +1193,7 @@ mod test { // All forms of offsets let offsets = [ ReadBatchParams::Indices(vec![1, 3, 9, 5, 7, 6].into()), + ReadBatchParams::Indices(vec![1, 3, 5, 6, 7, 9].into()), ReadBatchParams::Range(2..8), ReadBatchParams::RangeFull, ReadBatchParams::RangeTo(..5), @@ -1002,6 +1259,7 @@ mod test { fn test_select_row_ids_out_of_bounds() { let offsets = [ ReadBatchParams::Indices(vec![1, 1000, 4].into()), + ReadBatchParams::Indices(vec![1, 4, 1000].into()), ReadBatchParams::Range(2..1000), ReadBatchParams::RangeTo(..1000), ]; @@ -1134,15 +1392,187 @@ mod test { fn test_selection() { let sequence = RowIdSequence(vec![ U64Segment::Range(0..5), - U64Segment::Range(10..15), - U64Segment::Range(20..25), + U64Segment::RangeWithHoles { + range: 10..16, + holes: vec![12].into(), + }, + U64Segment::RangeWithBitmap { + range: 20..28, + bitmap: [true, false, true, true, false, true, false, true] + .as_slice() + .into(), + }, + U64Segment::SortedArray(vec![40, 42, 45].into()), + U64Segment::Array(vec![60, 50, 70].into()), ]); + let live = sequence.iter().collect::>(); let selection = sequence.select(vec![2, 4, 13, 14, 57].into_iter()); - assert_eq!(selection.collect::>(), vec![2, 4, 23, 24]); + assert_eq!( + selection.collect::>(), + vec![live[2], live[4], live[13], live[14]] + ); + + for chunk_size in [1, 3, 7, 16] { + let mut cursor = sequence.cursor(); + let mut chunked = Vec::new(); + for start in (0..live.len()).step_by(chunk_size) { + let end = (start + chunk_size).min(live.len()); + chunked.extend(sequence.select_range_with_cursor(&mut cursor, start..end)); + } + assert_eq!(chunked, live); + } + + let mut cursor = sequence.cursor(); + assert_eq!( + sequence.select_range_with_cursor(&mut cursor, 6..19), + live[6..19] + ); + assert_eq!( + sequence.select_range_with_cursor(&mut cursor, 1..8), + live[1..8] + ); + assert_eq!( + sequence.select_range_with_cursor(&mut cursor, live.len() - 2..live.len() + 5), + live[live.len() - 2..] + ); + } + + #[test] + fn test_selection_over_bitmap_segments() { + let mut bitmap = Bitmap::new_full(40); + for hole in [3, 4, 17, 39] { + bitmap.clear(hole); + } + let sequence = RowIdSequence(vec![ + U64Segment::RangeWithBitmap { + range: 100..140, + bitmap, + }, + U64Segment::Range(200..205), + ]); + let live: Vec = sequence.iter().collect(); + assert_eq!(live.len(), 41); + + // Every index, one cursor pass. + let all = sequence.select(0..live.len()).collect::>(); + assert_eq!(all, live); + // Sparse, repeated, and past-the-end indices agree with the full pass. + let picks = vec![0, 2, 3, 3, 15, 16, 35, 36, 40, 99]; + let got = sequence.select(picks.iter().copied()).collect::>(); + let want: Vec = picks.iter().filter_map(|&i| live.get(i).copied()).collect(); + assert_eq!(got, want); + + let mut cursor = sequence.cursor(); + let mut chunked = Vec::new(); + for range in [0..7, 7..30, 30..live.len()] { + chunked.extend(sequence.select_range_with_cursor(&mut cursor, range)); + } + assert_eq!(chunked, live); + assert_eq!( + sequence.select_range_with_cursor(&mut cursor, 2..6), + live[2..6] + ); + } + + #[test] + fn test_dense_range_cursor_selection() { + let mut bitmap = Bitmap::new_full(40); + for hole in [3, 4, 17, 39] { + bitmap.clear(hole); + } + let sequence = RowIdSequence(vec![U64Segment::RangeWithBitmap { + range: 100..140, + bitmap, + }]); + let expected = sequence.iter().collect::>(); + let (mut cursor, use_dense_range_expansion) = sequence.cursor_with_dense_range_expansion(); + assert!(use_dense_range_expansion); + assert_eq!(cursor.segment_len, Some(expected.len())); + + let mut actual = Vec::new(); + for selection in [0..7, 7..8, 8..31, 31..expected.len() + 5] { + actual.extend(sequence.select_dense_range_with_cursor(&mut cursor, selection)); + } + assert_eq!(actual, expected); + assert_eq!( + sequence.select_dense_range_with_cursor(&mut cursor, 2..9), + expected[2..9] + ); + + let mut sparse_bitmap = Bitmap::new_empty(40); + for value in (0..40).step_by(2) { + sparse_bitmap.set(value); + } + let sparse = RowIdSequence(vec![U64Segment::RangeWithBitmap { + range: 0..40, + bitmap: sparse_bitmap, + }]); + let (sparse_cursor, use_dense_range_expansion) = sparse.cursor_with_dense_range_expansion(); + assert!(!use_dense_range_expansion); + assert_eq!(sparse_cursor.segment_len, Some(20)); + + let mut multiple_segments = sequence.clone(); + multiple_segments.extend(RowIdSequence::from(200..205)); + let (multiple_cursor, use_dense_range_expansion) = + multiple_segments.cursor_with_dense_range_expansion(); + assert!(!use_dense_range_expansion); + assert_eq!(multiple_cursor.segment_len, None); + } + + #[test] + fn test_selection_over_a_large_bitmap_segment() { + // A restart-per-index scan of this segment takes tens of seconds, so a + // regression to that shows up as a test that no longer finishes quickly. + const ROWS: usize = 1_000_000; + let mut bitmap = Bitmap::new_full(ROWS); + for hole in (0..ROWS).step_by(17) { + bitmap.clear(hole); + } + let sequence = RowIdSequence(vec![ + U64Segment::Range(0..8), + U64Segment::RangeWithBitmap { + range: 1_000..(1_000 + ROWS as u64), + bitmap, + }, + ]); + let live: Vec = sequence.iter().collect(); + + let all = sequence.select(0..live.len()).collect::>(); + assert_eq!(all, live); + + // Byte-boundary and tail indices, read through one cursor. + let mut picks: Vec = [0, 7, 8, 9, 15, 16, 63, 64, 65] + .into_iter() + .chain((0..live.len()).step_by(9973)) + .chain([live.len() - 1, live.len()]) + .collect(); + picks.sort_unstable(); + let got = sequence.select(picks.iter().copied()).collect::>(); + let want: Vec = picks.iter().filter_map(|&i| live.get(i).copied()).collect(); + assert_eq!(got, want); + + let tail_start = live.len() - 100_000; + let mut cursor = sequence.cursor(); + assert_eq!( + sequence.select_range_with_cursor(&mut cursor, tail_start..live.len()), + live[tail_start..] + ); + + for chunk_size in [1, 7, 8, 9, 1_024, 4_097] { + let mut cursor = sequence.cursor(); + let mut chunked = Vec::with_capacity(live.len() - tail_start); + let mut start = tail_start; + while start < live.len() { + let end = (start + chunk_size).min(live.len()); + chunked.extend(sequence.select_range_with_cursor(&mut cursor, start..end)); + start = end; + } + assert_eq!(chunked, live[tail_start..]); + } } #[test] - #[should_panic] + #[should_panic(expected = "Selection is not sorted")] fn test_selection_unsorted() { let sequence = RowIdSequence(vec![ U64Segment::Range(0..5), diff --git a/rust/lance-table/src/rowids/bitmap.rs b/rust/lance-table/src/rowids/bitmap.rs index ce7eadd5634..664679c6521 100644 --- a/rust/lance-table/src/rowids/bitmap.rs +++ b/rust/lance-table/src/rowids/bitmap.rs @@ -9,6 +9,21 @@ pub struct Bitmap { pub len: usize, } +/// Set bits in `data`, counted a word at a time. +fn count_ones(data: &[u8]) -> usize { + let mut words = data.chunks_exact(8); + let full: usize = words + .by_ref() + .map(|word| u64::from_le_bytes(word.try_into().unwrap()).count_ones() as usize) + .sum(); + let tail: usize = words + .remainder() + .iter() + .map(|byte| byte.count_ones() as usize) + .sum(); + full + tail +} + impl std::fmt::Debug for Bitmap { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "Bitmap {{ data: ")?; @@ -22,7 +37,7 @@ impl std::fmt::Debug for Bitmap { impl Bitmap { pub fn new_empty(len: usize) -> Self { let data = vec![0; len.div_ceil(8)]; - Self { data, len } + Self::from_parts(data, len) } pub fn new_full(len: usize) -> Self { @@ -37,9 +52,22 @@ impl Bitmap { *last_byte &= !(1 << i); } } + Self::from_parts(data, len) + } + + pub(crate) fn from_parts(data: Vec, len: usize) -> Self { Self { data, len } } + #[inline] + pub(crate) fn bytes(&self) -> &[u8] { + &self.data + } + + pub(crate) fn into_bytes(self) -> Vec { + self.data + } + pub fn set(&mut self, i: usize) { self.data[i / 8] |= 1 << (i % 8); } @@ -65,7 +93,7 @@ impl Bitmap { } pub fn count_ones(&self) -> usize { - self.data.iter().map(|&x| x.count_ones() as usize).sum() + count_ones(&self.data) } pub fn count_zeros(&self) -> usize { @@ -132,10 +160,7 @@ impl BitmapSlice<'_> { } // Middle bytes can just use count_ones - count += self.bitmap.data[first_byte + 1..last_byte] - .iter() - .map(|&x| x.count_ones() as usize) - .sum::(); + count += count_ones(&self.bitmap.data[first_byte + 1..last_byte]); count } } @@ -191,6 +216,29 @@ mod tests { assert_eq!(bitmap_slice.count_ones(), 2); } + #[test] + fn test_count_ones_spans_words_and_tail() { + for len in [1_usize, 7, 8, 63, 64, 65, 130] { + let mut bitmap = Bitmap::new_empty(len); + for i in (0..len).step_by(3) { + bitmap.set(i); + } + assert_eq!(bitmap.count_ones(), len.div_ceil(3), "len {len}"); + } + } + + #[test] + fn test_count_ones_tracks_direct_data_mutation() { + let mut bitmap = Bitmap::new_empty(16); + assert_eq!(bitmap.count_ones(), 0); + + bitmap.data[0] = 0b1010_0101; + assert_eq!(bitmap.count_ones(), 4); + + bitmap.data[1] = 0xff; + assert_eq!(bitmap.count_ones(), 12); + } + #[test] fn test_equality() { for len in 48..56 { diff --git a/rust/lance-table/src/rowids/index.rs b/rust/lance-table/src/rowids/index.rs index 4fed0e651c5..738e2ec6bcc 100644 --- a/rust/lance-table/src/rowids/index.rs +++ b/rust/lance-table/src/rowids/index.rs @@ -11,6 +11,15 @@ use lance_core::utils::deletion::DeletionVector; use lance_core::{Error, Result}; use rangemap::RangeInclusiveMap; +/// Fragments one lookup may have to probe before the merged map is worth its +/// build, whatever that build costs. A compacted table interleaves its +/// fragments, and one measured at 46. +const MAX_PROBE_DEPTH: u64 = 64; + +/// Row ids the merged build may read before probing is worth its per-lookup +/// cost instead. +const MERGE_ROWS_BUDGET: u64 = 1 << 20; + /// An index of row ids /// /// This index is used to map row ids to their corresponding addresses. These @@ -20,11 +29,23 @@ use rangemap::RangeInclusiveMap; /// map to addresses that have been tombstoned. A separate tombstone index is /// used to track tombstoned rows. // (Implementation) -// Disjoint ranges of row ids are stored as the keys of the map. The values are -// a pair of segments. The first segment is the row ids, and the second segment -// is the addresses. +// Two representations answer the same lookups, chosen once by `new`. The merged +// map keys disjoint ranges of row ids to a pair of segments, the row ids and +// the addresses, and reads every row id to build. A probe instead reads each +// segment's bounds and asks the covering segment for the position of the id; +// `new` takes that when few fragments cover one id and the merged build would +// read a lot of them. #[derive(Debug)] -pub struct RowIdIndex(RangeInclusiveMap); +pub struct RowIdIndex { + /// Fragments that hold at least one row id, sorted by their lowest row id. + fragments: Vec, + /// Max-`end` heap over `fragments`: `end_tree[1]` is the root and leaf `i` + /// sits at `end_tree[len() / 2 + i]`. + end_tree: Vec, + merged: Option, +} + +type MergedIndex = RangeInclusiveMap; pub struct FragmentRowIdIndex { pub fragment_id: u32, @@ -35,7 +56,34 @@ pub struct FragmentRowIdIndex { impl RowIdIndex { /// Create a new index from a list of fragment ids and their corresponding row id sequences. pub fn new(fragment_indices: &[FragmentRowIdIndex]) -> Result { - let chunks = fragment_indices + let mut fragments: Vec = fragment_indices + .iter() + .filter_map(FragmentEntry::new) + .collect(); + fragments.sort_unstable_by_key(|entry| entry.start); + + let mut index = Self { + end_tree: build_end_tree(&fragments), + fragments, + merged: None, + }; + if !probing_beats_merging(&index.fragments) { + index.merged = Some(index.build_merged()?); + } + Ok(index) + } + + fn build_merged(&self) -> Result { + let sources: Vec = self + .fragments + .iter() + .map(|entry| FragmentRowIdIndex { + fragment_id: entry.fragment_id, + row_id_sequence: entry.sequence.clone(), + deletion_vector: entry.deletion_vector.clone(), + }) + .collect(); + let chunks = sources .iter() .flat_map(decompose_sequence) .collect::>(); @@ -56,17 +104,22 @@ impl RowIdIndex { } } - Ok(Self(RangeInclusiveMap::from_iter(final_chunks))) + Ok(RangeInclusiveMap::from_iter(final_chunks)) } /// Get the address for a given row id. /// /// Will return None if the row id does not exist in the index. - pub fn get(&self, row_id: u64) -> Option { - let (row_id_segment, address_segment) = self.0.get(&row_id)?; - let pos = row_id_segment.position(row_id)?; - let address = address_segment.get(pos)?; - Some(RowAddress::from(address)) + /// + /// # Errors + /// + /// Returns an error if the row id is live in more than one fragment, + /// which means the stable row ids are corrupt. + pub fn get(&self, row_id: u64) -> Result> { + if let Some(merged) = &self.merged { + return Ok(merged_get(merged, row_id)); + } + self.probe(row_id) } /// Get addresses for many row ids in one pass over the index. @@ -75,17 +128,30 @@ impl RowIdIndex { /// Sorts a working copy of the input internally so the chunk iterator /// is advanced at most once per chunk, amortizing the per-id tree walk /// from O(N · log F) to O(F + N). - pub fn get_many(&self, row_ids: &[u64]) -> Vec> { + /// + /// # Errors + /// + /// Returns an error if any requested row id is live in more than one + /// fragment, which means the stable row ids are corrupt. + pub fn get_many(&self, row_ids: &[u64]) -> Result>> { let n = row_ids.len(); let mut out = vec![None; n]; if n == 0 { - return out; + return Ok(out); } let mut sorted: Vec<(u64, usize)> = row_ids.iter().copied().zip(0..n).collect(); sorted.sort_unstable_by_key(|&(id, _)| id); - let mut chunks = self.0.iter().peekable(); + let Some(merged) = &self.merged else { + // Sorted ids keep one fragment and its segments warm across the run. + for (id, orig_idx) in sorted { + out[orig_idx] = self.probe(id)?; + } + return Ok(out); + }; + + let mut chunks = merged.iter().peekable(); for (id, orig_idx) in sorted { // Advance past chunks that end before this id. while let Some((range, _)) = chunks.peek() { @@ -107,21 +173,262 @@ impl RowIdIndex { out[orig_idx] = Some(RowAddress::from(addr)); } } - out + Ok(out) + } + + /// Address of `row_id`, from the fragment that holds it live. Descends the + /// max-`end` tree, so a fragment out of reach of the id costs nothing. + /// + /// Visits every candidate rather than stopping at the first hit, and + /// errors when a second fragment holds the id live. + fn probe(&self, row_id: u64) -> Result> { + let fragments = self.fragments.len(); + if fragments == 0 { + return Ok(None); + } + // Only a fragment that starts at or below the id can hold it. + let upper = self + .fragments + .partition_point(|entry| entry.start <= row_id); + if upper == 0 { + return Ok(None); + } + let leaves = self.end_tree.len() / 2; + // Depth is log2(leaves), at most 64, and each level leaves one sibling. + let mut stack = [(0usize, 0usize, 0usize); 64]; + stack[0] = (1, 0, leaves); + let mut depth = 1; + let mut found: Option = None; + while depth > 0 { + depth -= 1; + let (node, lo, hi) = stack[depth]; + if lo >= upper || self.end_tree[node] < row_id { + continue; + } + if hi - lo == 1 { + if lo < fragments + && let Some(candidate) = self.fragments[lo].resolve(row_id) + { + if found.is_some() { + return Err(Error::internal(format!( + "row id index corrupt: stable row id {row_id} is \ + live in multiple fragments", + ))); + } + found = Some(candidate); + } + continue; + } + let mid = (lo + hi) / 2; + // Push the left half first so the right half pops first: candidates + // arrive in descending slot order. + stack[depth] = (2 * node, lo, mid); + stack[depth + 1] = (2 * node + 1, mid, hi); + depth += 2; + } + Ok(found) + } +} + +fn merged_get(merged: &MergedIndex, row_id: u64) -> Option { + let (row_id_segment, address_segment) = merged.get(&row_id)?; + let pos = row_id_segment.position(row_id)?; + let address = address_segment.get(pos)?; + Some(RowAddress::from(address)) +} + +/// One segment of a sequence, and the offset its first row sits at. +#[derive(Debug)] +struct SegmentEntry { + seq_idx: usize, + range: RangeInclusive, + start_offset: u32, + /// Row id to position for an unsorted [`U64Segment::Array`], whose own + /// `position` scans. `None` for the encodings that search themselves. + positions: Option>, +} + +impl SegmentEntry { + /// Position of `row_id` in this segment, or `None` if it holds no such id. + fn position(&self, sequence: &RowIdSequence, row_id: u64) -> Option { + match &self.positions { + None => sequence.0[self.seq_idx].position(row_id), + Some(positions) => positions + .binary_search_by_key(&row_id, |(id, _)| *id) + .ok() + .map(|found| positions[found].1 as usize), + } + } +} + +/// Row id to position for a segment, sorted by row id. The first position of a +/// repeated id wins, which is what `position` returns. +fn build_positions(segment: &U64Segment) -> Option> { + if !matches!(segment, U64Segment::Array(_)) { + return None; + } + let mut positions: Vec<(u64, u32)> = segment + .iter() + .enumerate() + .map(|(position, row_id)| (row_id, position as u32)) + .collect(); + positions.sort_unstable(); + positions.dedup_by_key(|(row_id, _)| *row_id); + Some(positions) +} + +#[derive(Debug)] +struct FragmentEntry { + fragment_id: u32, + sequence: Arc, + deletion_vector: Arc, + segments: Vec, + start: u64, + end: u64, + /// Row ids the merged build reads one by one. + merge_rows: u64, +} + +impl FragmentEntry { + fn new(source: &FragmentRowIdIndex) -> Option { + let mut segments: Vec = Vec::new(); + let mut start_offset: u32 = 0; + let mut merge_rows: u64 = 0; + let deleted = !source.deletion_vector.is_empty(); + for (seq_idx, segment) in source.row_id_sequence.0.iter().enumerate() { + let len = segment.len(); + // A `Range` without deletions decomposes in constant time. + if deleted || !matches!(segment, U64Segment::Range(_)) { + merge_rows += len as u64; + } + // `range()` reports the span of a holed encoding, so ask `len` which + // ids the segment actually holds before trusting those bounds. + if len > 0 + && let Some(range) = segment.range() + { + segments.push(SegmentEntry { + seq_idx, + range, + start_offset, + positions: build_positions(segment), + }); + } + start_offset += len as u32; + } + let start = segments.iter().map(|entry| *entry.range.start()).min()?; + let end = segments.iter().map(|entry| *entry.range.end()).max()?; + Some(Self { + fragment_id: source.fragment_id, + sequence: source.row_id_sequence.clone(), + deletion_vector: source.deletion_vector.clone(), + segments, + start, + end, + merge_rows, + }) + } + + /// Address of `row_id` here, or `None` when the fragment lacks it or holds + /// it deleted. + fn resolve(&self, row_id: u64) -> Option { + for entry in &self.segments { + if !entry.range.contains(&row_id) { + continue; + } + let Some(position) = entry.position(&self.sequence, row_id) else { + continue; + }; + let row_offset = entry.start_offset + position as u32; + if self.deletion_vector.contains(row_offset) { + continue; + } + return Some(RowAddress::new_from_parts(self.fragment_id, row_offset)); + } + None + } +} + +/// Whether to answer lookups by probing the fragments rather than by merging +/// every row id. +/// +/// Probing costs the fragments that cover one id, per lookup; merging costs the +/// row ids it reads, once. So probe only when both stay on the right side of +/// [`MAX_PROBE_DEPTH`] and [`MERGE_ROWS_BUDGET`]. +fn probing_beats_merging(fragments: &[FragmentEntry]) -> bool { + let merge_rows: u64 = fragments.iter().map(|entry| entry.merge_rows).sum(); + merge_rows > MERGE_ROWS_BUDGET && max_overlap_depth(fragments) <= MAX_PROBE_DEPTH +} + +/// Most fragments that cover any one row id. +fn max_overlap_depth(fragments: &[FragmentEntry]) -> u64 { + let mut ends: Vec = fragments.iter().map(|entry| entry.end).collect(); + ends.sort_unstable(); + let mut closed = 0; + let mut depth: u64 = 0; + for (opened, entry) in fragments.iter().enumerate() { + while closed < ends.len() && ends[closed] < entry.start { + closed += 1; + } + depth = depth.max((opened + 1 - closed) as u64); + } + depth +} + +/// Implicit max-`end` heap over `fragments`, padded to a power of two. Padding +/// leaves hold 0, which prunes for every id above 0 and is filtered by slot. +fn build_end_tree(fragments: &[FragmentEntry]) -> Vec { + if fragments.is_empty() { + return Vec::new(); + } + let leaves = fragments.len().next_power_of_two(); + let mut tree = vec![0_u64; 2 * leaves]; + for (slot, entry) in fragments.iter().enumerate() { + tree[leaves + slot] = entry.end; } + for node in (1..leaves).rev() { + tree[node] = tree[2 * node].max(tree[2 * node + 1]); + } + tree } impl DeepSizeOf for RowIdIndex { + /// Charges the sequences and deletion vectors the `Arc`s keep alive, which + /// a sequence cached under its own key is charged for as well. fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.0 + let fragment_bytes: usize = self + .fragments .iter() - .map(|(_, (row_id_segment, address_segment))| { - (2 * std::mem::size_of::()) - + std::mem::size_of::<(U64Segment, U64Segment)>() - + row_id_segment.deep_size_of_children(context) - + address_segment.deep_size_of_children(context) + .map(|entry| { + entry.sequence.deep_size_of_children(context) + + entry.deletion_vector.deep_size_of_children(context) + + entry.segments.capacity() * std::mem::size_of::() + + entry + .segments + .iter() + .filter_map(|segment| segment.positions.as_ref()) + .map(|positions| positions.capacity() * std::mem::size_of::<(u64, u32)>()) + .sum::() + }) + .sum(); + let merged_bytes: usize = self + .merged + .as_ref() + .map(|merged| { + merged + .iter() + .map(|(_, (row_id_segment, address_segment))| { + (2 * std::mem::size_of::()) + + std::mem::size_of::<(U64Segment, U64Segment)>() + + row_id_segment.deep_size_of_children(context) + + address_segment.deep_size_of_children(context) + }) + .sum() }) - .sum() + .unwrap_or(0); + fragment_bytes + + merged_bytes + + self.fragments.capacity() * std::mem::size_of::() + + self.end_tree.capacity() * std::mem::size_of::() } } @@ -159,10 +466,13 @@ fn decompose_sequence( } /// Build an IndexChunk from a list of (row_id, address) pairs. -fn build_chunk_from_pairs(pairs: Vec<(u64, u64)>) -> Option { +fn build_chunk_from_pairs(mut pairs: Vec<(u64, u64)>) -> Option { if pairs.is_empty() { return None; } + // Sorted, so the row id segment encodes as one a lookup can search rather + // than an `Array` it has to scan. The address segment follows the pairing. + pairs.sort_unstable_by_key(|(row_id, _)| *row_id); let (row_ids, addresses): (Vec, Vec) = pairs.into_iter().unzip(); let row_id_segment = U64Segment::from_iter(row_ids); let address_segment = U64Segment::from_iter(addresses); @@ -365,10 +675,138 @@ fn merge_overlapping_chunks(overlapping_chunks: Vec) -> Result Result { + let mut fragments: Vec = fragment_indices + .iter() + .filter_map(FragmentEntry::new) + .collect(); + fragments.sort_unstable_by_key(|entry| entry.start); + Ok(Self { + end_tree: build_end_tree(&fragments), + fragments, + merged: None, + }) + } +} + #[cfg(test)] mod tests { use super::*; - use proptest::{prelude::Strategy, prop_assert_eq}; + use proptest::{ + prelude::{Just, Strategy, any}, + prop_assert, prop_assert_eq, + }; + + /// Sequence of `len` even row ids, held as a sorted array. + fn sparse_sequence(len: u64) -> RowIdSequence { + RowIdSequence(vec![U64Segment::SortedArray( + (0..len).map(|value| value * 2).collect::>().into(), + )]) + } + + fn fragment(fragment_id: u32, sequence: RowIdSequence) -> FragmentRowIdIndex { + FragmentRowIdIndex { + fragment_id, + row_id_sequence: Arc::new(sequence), + deletion_vector: Arc::new(DeletionVector::default()), + } + } + + #[test] + fn test_new_builds_the_merged_map_unless_probing_wins() { + // Ranges decompose in constant time, and a small sequence is cheap to + // read whatever its encoding. + let ranges = fragment(1, RowIdSequence(vec![U64Segment::Range(0..1_000_000)])); + assert!(RowIdIndex::new(&[ranges]).unwrap().merged.is_some()); + let small = fragment(1, sparse_sequence(16)); + assert!(RowIdIndex::new(&[small]).unwrap().merged.is_some()); + + // Past the row budget, with one fragment covering any id. + let wide = fragment(1, sparse_sequence(MERGE_ROWS_BUDGET + 1)); + let index = RowIdIndex::new(&[wide]).unwrap(); + assert!(index.merged.is_none()); + assert_eq!( + index.get(6).unwrap(), + Some(RowAddress::new_from_parts(1, 3)) + ); + } + + #[test] + fn test_deep_overlap_merges_however_many_rows_it_reads() { + // Just past the row budget in total, interleaved so every fragment + // covers every id: the depth alone forces the merged build. + let fragments = MAX_PROBE_DEPTH + 1; + let rows_per_fragment = MERGE_ROWS_BUDGET / fragments + 1; + let deep: Vec = (0..fragments as u32) + .map(|id| { + let ids: Vec = (0..rows_per_fragment) + .map(|value| value * fragments + id as u64) + .collect(); + fragment(id, RowIdSequence(vec![U64Segment::SortedArray(ids.into())])) + }) + .collect(); + + assert!(RowIdIndex::new(&deep).unwrap().merged.is_some()); + } + + #[test] + fn test_probe_resolves_a_row_id_the_merged_map_rejects() { + let sources = [ + fragment(1, RowIdSequence::from(&[0, 2][..])), + fragment(2, RowIdSequence::from(&[1, 2][..])), + ]; + assert!(RowIdIndex::new(&sources).is_err()); + + let index = RowIdIndex::probing(&sources[..1]).unwrap(); + assert_eq!( + index.get(2).unwrap(), + Some(RowAddress::new_from_parts(1, 1)) + ); + } + + #[test] + fn test_probe_errors_when_two_fragments_hold_an_id_live() { + let sources = [ + fragment(1, RowIdSequence::from(&[0, 2][..])), + fragment(2, RowIdSequence::from(&[1, 2][..])), + ]; + let index = RowIdIndex::probing(&sources).unwrap(); + assert_eq!( + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(1, 0)) + ); + + let error = index.get(2).unwrap_err(); + assert!(matches!(&error, Error::Internal { .. })); + assert!( + error + .to_string() + .contains("stable row id 2 is live in multiple fragments") + ); + + let error = index.get_many(&[0, 2]).unwrap_err(); + assert!(matches!(&error, Error::Internal { .. })); + } + + #[test] + fn test_probe_finds_every_position_of_an_unsorted_array() { + let row_ids: Vec = (0..2048).map(|value| (value * 7919) % 2048).collect(); + let index = RowIdIndex::probing(&[fragment( + 3, + RowIdSequence(vec![U64Segment::Array(row_ids.clone().into())]), + )]) + .unwrap(); + for (offset, row_id) in row_ids.iter().enumerate() { + assert_eq!( + index.get(*row_id).unwrap(), + Some(RowAddress::new_from_parts(3, offset as u32)) + ); + } + assert!(index.merged.is_none()); + } #[test] fn test_new_index() { @@ -401,14 +839,32 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); // Check various queries. - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(15), None); - assert_eq!(index.get(16), Some(RowAddress::new_from_parts(10, 14))); - assert_eq!(index.get(17), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(25), Some(RowAddress::new_from_parts(10, 16))); - assert_eq!(index.get(40), Some(RowAddress::new_from_parts(20, 2))); - assert_eq!(index.get(60), Some(RowAddress::new_from_parts(20, 4))); - assert_eq!(index.get(61), None); + assert_eq!( + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(10, 0)) + ); + assert_eq!(index.get(15).unwrap(), None); + assert_eq!( + index.get(16).unwrap(), + Some(RowAddress::new_from_parts(10, 14)) + ); + assert_eq!( + index.get(17).unwrap(), + Some(RowAddress::new_from_parts(20, 0)) + ); + assert_eq!( + index.get(25).unwrap(), + Some(RowAddress::new_from_parts(10, 16)) + ); + assert_eq!( + index.get(40).unwrap(), + Some(RowAddress::new_from_parts(20, 2)) + ); + assert_eq!( + index.get(60).unwrap(), + Some(RowAddress::new_from_parts(20, 4)) + ); + assert_eq!(index.get(61).unwrap(), None); } #[test] @@ -440,15 +896,42 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); // Check various queries. - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(42, 0))); - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(23, 0))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(42, 1))); - assert_eq!(index.get(6), Some(RowAddress::new_from_parts(23, 1))); - assert_eq!(index.get(7), Some(RowAddress::new_from_parts(10, 2))); - assert_eq!(index.get(8), Some(RowAddress::new_from_parts(42, 2))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(23, 2))); + assert_eq!( + index.get(1).unwrap(), + Some(RowAddress::new_from_parts(10, 0)) + ); + assert_eq!( + index.get(2).unwrap(), + Some(RowAddress::new_from_parts(42, 0)) + ); + assert_eq!( + index.get(3).unwrap(), + Some(RowAddress::new_from_parts(23, 0)) + ); + assert_eq!( + index.get(4).unwrap(), + Some(RowAddress::new_from_parts(10, 1)) + ); + assert_eq!( + index.get(5).unwrap(), + Some(RowAddress::new_from_parts(42, 1)) + ); + assert_eq!( + index.get(6).unwrap(), + Some(RowAddress::new_from_parts(23, 1)) + ); + assert_eq!( + index.get(7).unwrap(), + Some(RowAddress::new_from_parts(10, 2)) + ); + assert_eq!( + index.get(8).unwrap(), + Some(RowAddress::new_from_parts(42, 2)) + ); + assert_eq!( + index.get(9).unwrap(), + Some(RowAddress::new_from_parts(23, 2)) + ); } #[test] @@ -481,19 +964,46 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); // Check that all row ids can be found regardless of their order in the segments - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(30, 1))); - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(20, 1))); - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(30, 2))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(20, 2))); - assert_eq!(index.get(6), Some(RowAddress::new_from_parts(10, 2))); - assert_eq!(index.get(7), Some(RowAddress::new_from_parts(30, 0))); - assert_eq!(index.get(8), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(10, 0))); + assert_eq!( + index.get(1).unwrap(), + Some(RowAddress::new_from_parts(30, 1)) + ); + assert_eq!( + index.get(2).unwrap(), + Some(RowAddress::new_from_parts(20, 1)) + ); + assert_eq!( + index.get(3).unwrap(), + Some(RowAddress::new_from_parts(10, 1)) + ); + assert_eq!( + index.get(4).unwrap(), + Some(RowAddress::new_from_parts(30, 2)) + ); + assert_eq!( + index.get(5).unwrap(), + Some(RowAddress::new_from_parts(20, 2)) + ); + assert_eq!( + index.get(6).unwrap(), + Some(RowAddress::new_from_parts(10, 2)) + ); + assert_eq!( + index.get(7).unwrap(), + Some(RowAddress::new_from_parts(30, 0)) + ); + assert_eq!( + index.get(8).unwrap(), + Some(RowAddress::new_from_parts(20, 0)) + ); + assert_eq!( + index.get(9).unwrap(), + Some(RowAddress::new_from_parts(10, 0)) + ); // Check that non-existent row ids return None - assert_eq!(index.get(0), None); - assert_eq!(index.get(10), None); + assert_eq!(index.get(0).unwrap(), None); + assert_eq!(index.get(10).unwrap(), None); } #[test] @@ -517,11 +1027,26 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); // Check various queries. - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(0, 0))); - assert_eq!(index.get(49), Some(RowAddress::new_from_parts(0, 49))); - assert_eq!(index.get(50), Some(RowAddress::new_from_parts(1, 0))); - assert_eq!(index.get(51), Some(RowAddress::new_from_parts(0, 50))); - assert_eq!(index.get(99), Some(RowAddress::new_from_parts(0, 98))); + assert_eq!( + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(0, 0)) + ); + assert_eq!( + index.get(49).unwrap(), + Some(RowAddress::new_from_parts(0, 49)) + ); + assert_eq!( + index.get(50).unwrap(), + Some(RowAddress::new_from_parts(1, 0)) + ); + assert_eq!( + index.get(51).unwrap(), + Some(RowAddress::new_from_parts(0, 50)) + ); + assert_eq!( + index.get(99).unwrap(), + Some(RowAddress::new_from_parts(0, 98)) + ); } #[test] @@ -548,15 +1073,36 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(20, 1))); - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(4), None); + assert_eq!( + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(20, 0)) + ); + assert_eq!( + index.get(1).unwrap(), + Some(RowAddress::new_from_parts(10, 0)) + ); + assert_eq!( + index.get(2).unwrap(), + Some(RowAddress::new_from_parts(20, 1)) + ); + assert_eq!( + index.get(3).unwrap(), + Some(RowAddress::new_from_parts(10, 1)) + ); + assert_eq!(index.get(4).unwrap(), None); // Surviving ids keep their original offsets (the hole is not compacted). - assert_eq!(index.get(6), Some(RowAddress::new_from_parts(20, 3))); - assert_eq!(index.get(8), Some(RowAddress::new_from_parts(20, 4))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(10, 4))); + assert_eq!( + index.get(6).unwrap(), + Some(RowAddress::new_from_parts(20, 3)) + ); + assert_eq!( + index.get(8).unwrap(), + Some(RowAddress::new_from_parts(20, 4)) + ); + assert_eq!( + index.get(9).unwrap(), + Some(RowAddress::new_from_parts(10, 4)) + ); } #[test] @@ -571,13 +1117,25 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(10, 1))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(10, 4))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(10, 5))); + assert_eq!( + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(10, 0)) + ); + assert_eq!( + index.get(1).unwrap(), + Some(RowAddress::new_from_parts(10, 1)) + ); + assert_eq!( + index.get(4).unwrap(), + Some(RowAddress::new_from_parts(10, 4)) + ); + assert_eq!( + index.get(5).unwrap(), + Some(RowAddress::new_from_parts(10, 5)) + ); - assert_eq!(index.get(2), None); - assert_eq!(index.get(3), None); + assert_eq!(index.get(2).unwrap(), None); + assert_eq!(index.get(3).unwrap(), None); } #[test] @@ -597,9 +1155,15 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(7), Some(RowAddress::new_from_parts(20, 2))); - assert_eq!(index.get(4), None); + assert_eq!( + index.get(5).unwrap(), + Some(RowAddress::new_from_parts(20, 0)) + ); + assert_eq!( + index.get(7).unwrap(), + Some(RowAddress::new_from_parts(20, 2)) + ); + assert_eq!(index.get(4).unwrap(), None); } #[test] @@ -607,8 +1171,8 @@ mod tests { let fragment_indices = vec![]; let index = RowIdIndex::new(&fragment_indices).unwrap(); - assert_eq!(index.get(0), None); - assert_eq!(index.get(100), None); + assert_eq!(index.get(0).unwrap(), None); + assert_eq!(index.get(100).unwrap(), None); } #[test] @@ -633,12 +1197,30 @@ mod tests { let index = RowIdIndex::new(&fragment_indices).unwrap(); - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(10, 0))); - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(10, 4))); - assert_eq!(index.get(5), Some(RowAddress::new_from_parts(20, 0))); - assert_eq!(index.get(9), Some(RowAddress::new_from_parts(20, 4))); - assert_eq!(index.get(10), Some(RowAddress::new_from_parts(30, 0))); - assert_eq!(index.get(14), Some(RowAddress::new_from_parts(30, 4))); + assert_eq!( + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(10, 0)) + ); + assert_eq!( + index.get(4).unwrap(), + Some(RowAddress::new_from_parts(10, 4)) + ); + assert_eq!( + index.get(5).unwrap(), + Some(RowAddress::new_from_parts(20, 0)) + ); + assert_eq!( + index.get(9).unwrap(), + Some(RowAddress::new_from_parts(20, 4)) + ); + assert_eq!( + index.get(10).unwrap(), + Some(RowAddress::new_from_parts(30, 0)) + ); + assert_eq!( + index.get(14).unwrap(), + Some(RowAddress::new_from_parts(30, 4)) + ); } fn arbitrary_row_ids( @@ -666,6 +1248,42 @@ mod tests { }) } + fn arbitrary_row_ids_with_deletions( + num_fragments_range: std::ops::Range, + frag_size_range: std::ops::Range, + ) -> impl Strategy, Arc)>> { + arbitrary_row_ids(num_fragments_range, frag_size_range) + .prop_flat_map(|row_ids| { + let num_rows = row_ids + .iter() + .map(|(_, sequence)| sequence.len() as usize) + .sum::(); + ( + Just(row_ids), + proptest::collection::vec(any::(), num_rows), + ) + }) + .prop_map(|(row_ids, deleted_rows)| { + let mut deleted_rows = deleted_rows.into_iter(); + row_ids + .into_iter() + .map(|(fragment_id, sequence)| { + let mut deletion_bitmap = roaring::RoaringBitmap::new(); + for offset in 0..sequence.len() as u32 { + if deleted_rows.next().unwrap() { + deletion_bitmap.insert(offset); + } + } + ( + fragment_id, + sequence, + Arc::new(DeletionVector::Bitmap(deletion_bitmap)), + ) + }) + .collect() + }) + } + #[test] fn test_large_range_segments_no_deletions() { // Simulates a real-world scenario: many fragments with large Range segments @@ -694,24 +1312,27 @@ mod tests { let elapsed = start.elapsed(); // Verify correctness at boundaries - assert_eq!(index.get(0), Some(RowAddress::new_from_parts(0, 0))); assert_eq!( - index.get(rows_per_fragment - 1), + index.get(0).unwrap(), + Some(RowAddress::new_from_parts(0, 0)) + ); + assert_eq!( + index.get(rows_per_fragment - 1).unwrap(), Some(RowAddress::new_from_parts(0, rows_per_fragment as u32 - 1)) ); assert_eq!( - index.get(rows_per_fragment), + index.get(rows_per_fragment).unwrap(), Some(RowAddress::new_from_parts(1, 0)) ); let last_row = num_fragments as u64 * rows_per_fragment - 1; assert_eq!( - index.get(last_row), + index.get(last_row).unwrap(), Some(RowAddress::new_from_parts( num_fragments - 1, rows_per_fragment as u32 - 1 )) ); - assert_eq!(index.get(last_row + 1), None); + assert_eq!(index.get(last_row + 1).unwrap(), None); // With the optimization, building an index for 25M rows across 100 fragments // should complete in well under 1 second (typically < 1ms). @@ -757,66 +1378,143 @@ mod tests { // Deleted rows (offset 0, 3, 6, ...) should not be found. // Row ID 0 has offset 0 in fragment 0 -> deleted. - assert_eq!(index.get(0), None); + assert_eq!(index.get(0).unwrap(), None); // Row ID 3 has offset 3 in fragment 0 -> deleted. - assert_eq!(index.get(3), None); + assert_eq!(index.get(3).unwrap(), None); // Non-deleted rows should resolve correctly. // Row ID 1 has offset 1 in fragment 0 -> address (frag=0, row=1). - assert_eq!(index.get(1), Some(RowAddress::new_from_parts(0, 1))); + assert_eq!( + index.get(1).unwrap(), + Some(RowAddress::new_from_parts(0, 1)) + ); // Row ID 2 has offset 2 in fragment 0 -> address (frag=0, row=2). - assert_eq!(index.get(2), Some(RowAddress::new_from_parts(0, 2))); + assert_eq!( + index.get(2).unwrap(), + Some(RowAddress::new_from_parts(0, 2)) + ); // Row ID 4 has offset 4 in fragment 0 -> address (frag=0, row=4). - assert_eq!(index.get(4), Some(RowAddress::new_from_parts(0, 4))); + assert_eq!( + index.get(4).unwrap(), + Some(RowAddress::new_from_parts(0, 4)) + ); // Check second fragment: row IDs start at 1000. // Row ID 1000 has offset 0 in fragment 1 -> deleted. - assert_eq!(index.get(rows_per_fragment), None); + assert_eq!(index.get(rows_per_fragment).unwrap(), None); // Row ID 1001 has offset 1 in fragment 1 -> address (frag=1, row=1). assert_eq!( - index.get(rows_per_fragment + 1), + index.get(rows_per_fragment + 1).unwrap(), Some(RowAddress::new_from_parts(1, 1)) ); // Last fragment, last non-deleted row. // Row ID 9999 has offset 999 in fragment 9 -> 999 % 3 == 0 -> deleted. let last_row = num_fragments as u64 * rows_per_fragment - 1; - assert_eq!(index.get(last_row), None); + assert_eq!(index.get(last_row).unwrap(), None); // Row ID 9998 has offset 998 -> 998 % 3 == 2 -> not deleted. assert_eq!( - index.get(last_row - 1), + index.get(last_row - 1).unwrap(), Some(RowAddress::new_from_parts(num_fragments - 1, 998)) ); // Out of range. - assert_eq!(index.get(last_row + 1), None); + assert_eq!(index.get(last_row + 1).unwrap(), None); } proptest::proptest! { #[test] - fn test_new_index_robustness(row_ids in arbitrary_row_ids(0..5, 0..32)) { + fn test_new_index_robustness( + row_ids in arbitrary_row_ids_with_deletions(0..5, 0..32) + ) { let fragment_indices: Vec = row_ids .iter() - .map(|(frag_id, sequence)| FragmentRowIdIndex { + .map(|(frag_id, sequence, deletion_vector)| FragmentRowIdIndex { fragment_id: *frag_id, row_id_sequence: sequence.clone(), - deletion_vector: Arc::new(DeletionVector::default()), + deletion_vector: deletion_vector.clone(), }) .collect(); - let index = RowIdIndex::new(&fragment_indices).unwrap(); - for (frag_id, sequence) in row_ids.iter() { - for (local_offset, row_id) in sequence.iter().enumerate() { - prop_assert_eq!( - index.get(row_id), - Some(RowAddress::new_from_parts(*frag_id, local_offset as u32)), - "Row id {} in sequence {:?} not found in index {:?}", - row_id, - sequence, - index - ); + let merged = RowIdIndex::new(&fragment_indices).unwrap(); + let probing = RowIdIndex::probing(&fragment_indices).unwrap(); + for index in [&merged, &probing] { + for (frag_id, sequence, deletion_vector) in row_ids.iter() { + for (local_offset, row_id) in sequence.iter().enumerate() { + let expected = if deletion_vector.contains(local_offset as u32) { + None + } else { + Some(RowAddress::new_from_parts(*frag_id, local_offset as u32)) + }; + prop_assert_eq!( + index.get(row_id).unwrap(), + expected, + "Row id {} in sequence {:?} not found in index {:?}", + row_id, + sequence, + index + ); + } } } } + + #[test] + fn test_new_index_moved_row_id( + row_id in any::(), + source_fragment in 0u32..1024, + fragment_delta in 1u32..1024, + ) { + let target_fragment = source_fragment + fragment_delta; + let fragment_indices = [ + FragmentRowIdIndex { + fragment_id: source_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::Bitmap( + roaring::RoaringBitmap::from_iter([0]), + )), + }, + FragmentRowIdIndex { + fragment_id: target_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::default()), + }, + ]; + + let index = RowIdIndex::new(&fragment_indices).unwrap(); + prop_assert_eq!( + index.get(row_id).unwrap(), + Some(RowAddress::new_from_parts(target_fragment, 0)) + ); + } + + #[test] + fn test_new_index_rejects_duplicate_live_row_id( + row_id in any::(), + first_fragment in 0u32..1024, + fragment_delta in 1u32..1024, + ) { + let second_fragment = first_fragment + fragment_delta; + let fragment_indices = [ + FragmentRowIdIndex { + fragment_id: first_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::default()), + }, + FragmentRowIdIndex { + fragment_id: second_fragment, + row_id_sequence: Arc::new(RowIdSequence::from(&[row_id][..])), + deletion_vector: Arc::new(DeletionVector::default()), + }, + ]; + + let error = RowIdIndex::new(&fragment_indices).unwrap_err(); + let is_internal = matches!(&error, Error::Internal { .. }); + let expected_message = + format!("stable row id {row_id} is live in multiple fragments"); + let error_message = error.to_string(); + prop_assert!(is_internal); + prop_assert!(error_message.contains(&expected_message)); + } } } diff --git a/rust/lance-table/src/rowids/segment.rs b/rust/lance-table/src/rowids/segment.rs index 6fba8599016..fe9c8232342 100644 --- a/rust/lance-table/src/rowids/segment.rs +++ b/rust/lance-table/src/rowids/segment.rs @@ -277,28 +277,34 @@ impl U64Segment { } } + pub(crate) fn use_dense_range_expansion(&self, segment_len: usize) -> bool { + let Self::RangeWithBitmap { bitmap, .. } = self else { + return false; + }; + // Keep sparse segments on the compact per-bit decoder. Contiguous-run + // expansion pays off when dense bytes dominate the segment. + let dense_threshold = bitmap.len().saturating_sub(bitmap.len() / 4); + segment_len >= dense_threshold + } + pub fn is_empty(&self) -> bool { self.len() == 0 } /// Get the min and max value of the segment, excluding tombstones. + /// + /// Returns `None` for an empty segment, which has no extrema. Decoding accepts an + /// empty encoding of every variant, so no arm here may assume it holds a value. pub fn range(&self) -> Option> { match self { - Self::Range(range) if range.is_empty() => None, Self::Range(range) | Self::RangeWithBitmap { range, .. } - | Self::RangeWithHoles { range, .. } => Some(range.start..=(range.end - 1)), - Self::SortedArray(array) => { - // We can assume that the array is sorted. - let min_value = array.first().unwrap(); - let max_value = array.last().unwrap(); - Some(min_value..=max_value) - } - Self::Array(array) => { - let min_value = array.min().unwrap(); - let max_value = array.max().unwrap(); - Some(min_value..=max_value) + | Self::RangeWithHoles { range, .. } => { + (!range.is_empty()).then(|| range.start..=(range.end - 1)) } + // We can assume that the array is sorted. + Self::SortedArray(array) => Some(array.first()?..=array.last()?), + Self::Array(array) => Some(array.min()?..=array.max()?), } } @@ -379,30 +385,20 @@ impl U64Segment { } Some(range.start + i as u64 + lo as u64) } - Self::RangeWithBitmap { range, bitmap } => { - // Find the i-th set bit (a "select1") via byte-wise popcount. - // Bytes past `bitmap.len()` are zero-padded by construction - // (Bitmap::new_full), so popcount counts only valid positions. - let mut remaining = i; - for (byte_idx, &byte) in bitmap.data.iter().enumerate() { - let ones = byte.count_ones() as usize; - if remaining < ones { - let mut b = byte; - for _ in 0..remaining { - b &= b - 1; // clear lowest set bit - } - let bit = b.trailing_zeros() as usize; - return Some(range.start + (byte_idx * 8 + bit) as u64); - } - remaining -= ones; - } - None - } + Self::RangeWithBitmap { .. } => self.cursor().get(i), Self::SortedArray(array) => array.get(i), Self::Array(array) => array.get(i), } } + /// Reads values at non-decreasing indices in one pass. + pub fn cursor(&self) -> SegmentCursor<'_> { + SegmentCursor { + segment: self, + state: SegmentCursorState::default(), + } + } + /// Check if a value is contained in the segment pub fn contains(&self, val: u64) -> bool { match self { @@ -660,10 +656,260 @@ impl U64Segment { } } +/// Segment reader that keeps its scan position across calls. +pub struct SegmentCursor<'a> { + segment: &'a U64Segment, + state: SegmentCursorState, +} + +#[derive(Debug, Default)] +pub(crate) struct SegmentCursorState { + /// Byte the next select1 scan resumes at. + byte_idx: usize, + /// Set bits in the bitmap bytes before `byte_idx`. + ones_before: usize, +} + +impl SegmentCursor<'_> { + /// The value at index `i`. A decreasing index rewinds the scan. + pub fn get(&mut self, i: usize) -> Option { + self.state.get(self.segment, i) + } +} + +impl SegmentCursorState { + /// Append a contiguous range of values while preserving the bitmap scan + /// position for the next call. + pub(crate) fn extend_range( + &mut self, + segment: &U64Segment, + selection: Range, + values: &mut Vec, + ) { + let U64Segment::RangeWithBitmap { range, bitmap } = segment else { + match segment { + U64Segment::Range(range) => { + let segment_len = (range.end - range.start) as usize; + let end = selection.end.min(segment_len); + if selection.start < end { + values.extend( + (range.start + selection.start as u64)..(range.start + end as u64), + ); + } + } + _ => values.extend(selection.filter_map(|index| segment.get(index))), + } + return; + }; + + if selection.start < self.ones_before { + self.byte_idx = 0; + self.ones_before = 0; + } + + while let Some(&byte) = bitmap.data.get(self.byte_idx) { + let ones = byte.count_ones() as usize; + let ones_after_byte = self.ones_before + ones; + if selection.start >= ones_after_byte { + self.ones_before = ones_after_byte; + self.byte_idx += 1; + continue; + } + + let mut remaining_bits = byte; + let mut rank = self.ones_before; + while remaining_bits != 0 { + if rank >= selection.end { + return; + } + let bit = remaining_bits.trailing_zeros() as usize; + if rank >= selection.start { + values.push(range.start + (self.byte_idx * 8 + bit) as u64); + } + remaining_bits &= remaining_bits - 1; + rank += 1; + } + + self.ones_before = ones_after_byte; + self.byte_idx += 1; + if self.ones_before >= selection.end { + return; + } + } + } + + pub(crate) fn extend_dense_range( + &mut self, + segment: &U64Segment, + selection: Range, + values: &mut Vec, + ) { + let U64Segment::RangeWithBitmap { range, bitmap } = segment else { + self.extend_range(segment, selection, values); + return; + }; + if selection.start < self.ones_before { + self.byte_idx = 0; + self.ones_before = 0; + } + self.extend_dense_bitmap_range(range.start, bitmap.bytes(), selection, values); + } + + #[inline] + fn extend_dense_bitmap_range( + &mut self, + range_start: u64, + bitmap_bytes: &[u8], + selection: Range, + values: &mut Vec, + ) { + while let Some(&byte) = bitmap_bytes.get(self.byte_idx) { + let ones = byte.count_ones() as usize; + let ones_after_byte = self.ones_before + ones; + if selection.start >= ones_after_byte { + self.ones_before = ones_after_byte; + self.byte_idx += 1; + continue; + } + + let includes_entire_byte = + selection.start <= self.ones_before && selection.end >= ones_after_byte; + if includes_entire_byte && ones >= 6 { + let byte_start = range_start + (self.byte_idx * 8) as u64; + if byte == u8::MAX { + values.extend(byte_start..byte_start + 8); + } else { + let mut remaining_bits = byte; + let mut bit_offset = 0_u64; + while remaining_bits != 0 { + let zeros = remaining_bits.trailing_zeros(); + remaining_bits >>= zeros; + bit_offset += u64::from(zeros); + let run = remaining_bits.trailing_ones(); + values.extend( + (byte_start + bit_offset)..(byte_start + bit_offset + u64::from(run)), + ); + remaining_bits >>= run; + bit_offset += u64::from(run); + } + } + self.ones_before = ones_after_byte; + self.byte_idx += 1; + if self.ones_before >= selection.end { + return; + } + continue; + } + + let mut remaining_bits = byte; + let mut rank = self.ones_before; + while remaining_bits != 0 { + if rank >= selection.end { + return; + } + let bit = remaining_bits.trailing_zeros() as usize; + if rank >= selection.start { + values.push(range_start + (self.byte_idx * 8 + bit) as u64); + } + remaining_bits &= remaining_bits - 1; + rank += 1; + } + + self.ones_before = ones_after_byte; + self.byte_idx += 1; + if self.ones_before >= selection.end { + return; + } + } + } + + /// The value at index `i`. A decreasing index rewinds the scan. + pub(crate) fn get(&mut self, segment: &U64Segment, i: usize) -> Option { + let U64Segment::RangeWithBitmap { range, bitmap } = segment else { + return segment.get(i); + }; + if i < self.ones_before { + self.byte_idx = 0; + self.ones_before = 0; + } + // Deserialization rejects a bitmap whose padding bits are set, so + // popcount counts only valid positions. + let mut remaining = i - self.ones_before; + let range_start = range.start; + let bitmap_bytes = bitmap.bytes(); + while let Some(&byte) = bitmap_bytes.get(self.byte_idx) { + let ones = byte.count_ones() as usize; + if remaining < ones { + let mut b = byte; + for _ in 0..remaining { + b &= b - 1; // clear lowest set bit + } + let bit = b.trailing_zeros() as usize; + return Some(range_start + (self.byte_idx * 8 + bit) as u64); + } + remaining -= ones; + self.ones_before += ones; + self.byte_idx += 1; + } + None + } +} + #[cfg(test)] mod test { use super::*; + #[test] + fn test_range_with_bitmap_data_remains_publicly_mutable() { + let mut segment = U64Segment::RangeWithBitmap { + range: 0..8, + bitmap: Bitmap::new_empty(8), + }; + let U64Segment::RangeWithBitmap { bitmap, .. } = &mut segment else { + unreachable!(); + }; + + bitmap.data[0] = 0b1010_0101; + assert_eq!(bitmap.len, 8); + assert_eq!(bitmap.count_ones(), 4); + } + + #[test] + fn test_extend_range_over_full_and_near_dense_bitmap_bytes() { + let mut bitmap = Bitmap::new_full(24); + bitmap.clear(10); + bitmap.clear(17); + bitmap.clear(22); + let segment = U64Segment::RangeWithBitmap { + range: 100..124, + bitmap, + }; + assert!(segment.use_dense_range_expansion(segment.len())); + + let mut sparse_bitmap = Bitmap::new_empty(24); + for i in [0, 8, 16] { + sparse_bitmap.set(i); + } + let sparse_segment = U64Segment::RangeWithBitmap { + range: 0..24, + bitmap: sparse_bitmap, + }; + assert!(!sparse_segment.use_dense_range_expansion(sparse_segment.len())); + let expected = segment.iter().collect::>(); + + let mut state = SegmentCursorState::default(); + let mut actual = Vec::new(); + for selection in [0..8, 8..15, 15..21] { + state.extend_dense_range(&segment, selection, &mut actual); + } + assert_eq!(actual, expected); + + let mut state = SegmentCursorState::default(); + let mut partial = Vec::new(); + state.extend_dense_range(&segment, 9..20, &mut partial); + assert_eq!(partial, expected[9..20]); + } + #[test] fn test_segments() { fn check_segment(values: &[u64], expected: &U64Segment) { @@ -765,6 +1011,30 @@ mod test { ); } + /// Decoding accepts an empty encoding of every variant, so `range()` must report the + /// absence of extrema for all of them rather than unwrapping a value or computing + /// `end - 1` on a zero-length range. + #[test] + fn test_empty_segments_have_no_range() { + let empty: Vec = Vec::new(); + let segments = [ + U64Segment::Range(5..5), + U64Segment::RangeWithHoles { + range: 0..0, + holes: empty.clone().into(), + }, + U64Segment::RangeWithBitmap { + range: 0..0, + bitmap: Bitmap::new_empty(0), + }, + U64Segment::SortedArray(empty.clone().into()), + U64Segment::Array(empty.into()), + ]; + for segment in segments { + assert_eq!(segment.range(), None, "{segment:?} should have no range"); + } + } + #[test] fn test_segment_overflow_boundary() { // Sparse range spanning i64::MAX — the original overflow reproducer. diff --git a/rust/lance-table/src/rowids/serde.rs b/rust/lance-table/src/rowids/serde.rs index c087fa603dc..6d44088875e 100644 --- a/rust/lance-table/src/rowids/serde.rs +++ b/rust/lance-table/src/rowids/serde.rs @@ -7,16 +7,87 @@ use lance_core::{Error, Result}; use super::{RowIdSequence, U64Segment, encoded_array::EncodedU64Array}; use prost::Message; +const ROW_ID_METADATA: &str = "row ID metadata"; + +fn corrupt_row_id_metadata(message: impl Into) -> Error { + Error::corrupt_file_named(ROW_ID_METADATA, message) +} + +fn validate_range(segment_type: &str, start: u64, end: u64) -> Result { + let len = end.checked_sub(start).ok_or_else(|| { + corrupt_row_id_metadata(format!( + "{segment_type} range start {start} exceeds end {end}" + )) + })?; + usize::try_from(len).map_err(|_| { + corrupt_row_id_metadata(format!( + "{segment_type} range length {len} for start {start} and end {end} exceeds usize::MAX" + )) + }) +} + +fn validate_packed_array_length(array_type: &str, byte_len: usize, width: usize) -> Result<()> { + if !byte_len.is_multiple_of(width) { + return Err(corrupt_row_id_metadata(format!( + "encoded {array_type} array byte length {byte_len} is not a multiple of element width {width}" + ))); + } + Ok(()) +} + +fn first_descending_pair(array: &EncodedU64Array) -> Option<(usize, u64, u64)> { + match array { + EncodedU64Array::U16 { offsets, .. } => offsets + .windows(2) + .position(|pair| pair[0] > pair[1]) + .map(|index| (index, offsets[index] as u64, offsets[index + 1] as u64)), + EncodedU64Array::U32 { offsets, .. } => offsets + .windows(2) + .position(|pair| pair[0] > pair[1]) + .map(|index| (index, offsets[index] as u64, offsets[index + 1] as u64)), + EncodedU64Array::U64(values) => values + .windows(2) + .position(|pair| pair[0] > pair[1]) + .map(|index| (index, values[index], values[index + 1])), + } +} + +fn first_non_increasing_pair(array: &EncodedU64Array) -> Option<(usize, u64, u64)> { + let mut values = array.iter(); + let previous = values.next()?; + values + .scan(previous, |previous, value| { + let pair = (*previous, value); + *previous = value; + Some(pair) + }) + .enumerate() + .find_map(|(index, (previous, next))| (previous >= next).then_some((index, previous, next))) +} + impl TryFrom for RowIdSequence { type Error = Error; fn try_from(pb: pb::RowIdSequence) -> Result { - Ok(Self( - pb.segments - .into_iter() - .map(U64Segment::try_from) - .collect::>>()?, - )) + let segments = pb + .segments + .into_iter() + .map(U64Segment::try_from) + .collect::>>()?; + // Each segment length fits a usize on its own, but the total need not fit a u64. + // Reject that here so `RowIdSequence::len()` stays total for anything decoded. + segments + .iter() + .try_fold(0_u64, |total, segment| { + total.checked_add(segment.len() as u64) + }) + .ok_or_else(|| { + corrupt_row_id_metadata(format!( + "row ID sequence of {} segments has a total length exceeding u64::MAX", + segments.len() + )) + })?; + Ok(Self(segments)) } } @@ -27,30 +98,71 @@ impl TryFrom for U64Segment { use pb::u64_segment as pb_seg; use pb::u64_segment::Segment::*; match pb.segment { - Some(Range(pb_seg::Range { start, end })) => Ok(Self::Range(start..end)), + Some(Range(pb_seg::Range { start, end })) => { + validate_range("Range", start, end)?; + Ok(Self::Range(start..end)) + } Some(RangeWithHoles(pb_seg::RangeWithHoles { start, end, holes })) => { + validate_range("RangeWithHoles", start, end)?; let holes = holes - .ok_or_else(|| Error::invalid_input("missing hole"))? + .ok_or_else(|| { + corrupt_row_id_metadata("RangeWithHoles is missing its holes array") + })? .try_into()?; + if let Some((index, previous, next)) = first_non_increasing_pair(&holes) { + return Err(corrupt_row_id_metadata(format!( + "RangeWithHoles values are not strictly increasing at indices {index} and {}: {previous} is not less than {next}", + index + 1 + ))); + } + if let Some(hole) = holes.iter().find(|hole| *hole < start || *hole >= end) { + return Err(corrupt_row_id_metadata(format!( + "RangeWithHoles hole {hole} is outside the range {start}..{end}" + ))); + } Ok(Self::RangeWithHoles { range: start..end, holes, }) } Some(RangeWithBitmap(pb_seg::RangeWithBitmap { start, end, bitmap })) => { + let range_len = validate_range("RangeWithBitmap", start, end)?; + let expected_bitmap_len = range_len.div_ceil(8); + if bitmap.len() != expected_bitmap_len { + return Err(corrupt_row_id_metadata(format!( + "RangeWithBitmap byte length {} does not match expected {expected_bitmap_len} for range start {start}, end {end}, and length {range_len}", + bitmap.len() + ))); + } + let remainder = range_len % 8; + if remainder != 0 { + let padding_mask = !((1_u8 << remainder) - 1); + let last_byte = bitmap[expected_bitmap_len - 1]; + if last_byte & padding_mask != 0 { + return Err(corrupt_row_id_metadata(format!( + "RangeWithBitmap padding bits must be zero for range start {start}, end {end}, and length {range_len}: last byte {last_byte:#04x} has padding mask {padding_mask:#04x} set" + ))); + } + } Ok(Self::RangeWithBitmap { range: start..end, - bitmap: Bitmap { - data: bitmap, - len: (end - start) as usize, - }, + bitmap: Bitmap::from_parts(bitmap, range_len), }) } - Some(SortedArray(array)) => Ok(Self::SortedArray(EncodedU64Array::try_from(array)?)), + Some(SortedArray(array)) => { + let array = EncodedU64Array::try_from(array)?; + if let Some((index, previous, next)) = first_descending_pair(&array) { + return Err(corrupt_row_id_metadata(format!( + "SortedArray values are not sorted at indices {index} and {}: {previous} exceeds {next}", + index + 1 + ))); + } + Ok(Self::SortedArray(array)) + } Some(Array(array)) => Ok(Self::Array(EncodedU64Array::try_from(array)?)), // TODO: why non-exhaustive? // Some(_) => Err(Error::invalid_input("unknown segment type")), - None => Err(Error::invalid_input("missing segment type")), + None => Err(corrupt_row_id_metadata("missing row ID segment type")), } } } @@ -63,32 +175,37 @@ impl TryFrom for EncodedU64Array { use pb::encoded_u64_array::Array::*; match pb.array { Some(U16Array(pb_arr::U16Array { base, offsets })) => { - assert!( - offsets.len() % 2 == 0, - "Must have even number of bytes to store u16 array" - ); + validate_packed_array_length("u16", offsets.len(), 2)?; let offsets = offsets .chunks_exact(2) .map(|chunk| u16::from_le_bytes([chunk[0], chunk[1]])) - .collect(); + .collect::>(); + if let Some(max_offset) = offsets.iter().copied().max() + && base.checked_add(u64::from(max_offset)).is_none() + { + return Err(corrupt_row_id_metadata(format!( + "U16Array base {base} plus maximum offset {max_offset} overflows u64" + ))); + } Ok(Self::U16 { base, offsets }) } Some(U32Array(pb_arr::U32Array { base, offsets })) => { - assert!( - offsets.len() % 4 == 0, - "Must have even number of bytes to store u32 array" - ); + validate_packed_array_length("u32", offsets.len(), 4)?; let offsets = offsets .chunks_exact(4) .map(|chunk| u32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])) - .collect(); + .collect::>(); + if let Some(max_offset) = offsets.iter().copied().max() + && base.checked_add(u64::from(max_offset)).is_none() + { + return Err(corrupt_row_id_metadata(format!( + "U32Array base {base} plus maximum offset {max_offset} overflows u64" + ))); + } Ok(Self::U32 { base, offsets }) } Some(U64Array(pb_arr::U64Array { values })) => { - assert!( - values.len() % 8 == 0, - "Must have even number of bytes to store u64 array" - ); + validate_packed_array_length("u64", values.len(), 8)?; let values = values .chunks_exact(8) .map(|chunk| { @@ -102,7 +219,7 @@ impl TryFrom for EncodedU64Array { } // TODO: shouldn't this enum be non-exhaustive? // Some(_) => Err(Error::invalid_input("unknown array type")), - None => Err(Error::invalid_input("missing array type")), + None => Err(corrupt_row_id_metadata("missing encoded row ID array type")), } } } @@ -138,7 +255,7 @@ impl From for pb::U64Segment { pb::u64_segment::RangeWithBitmap { start: range.start, end: range.end, - bitmap: bitmap.data, + bitmap: bitmap.into_bytes(), }, )), }, @@ -199,14 +316,79 @@ pub fn write_row_ids(sequence: &RowIdSequence) -> Vec { /// Deserialize a rowid sequence from some bytes. pub fn read_row_ids(reader: &[u8]) -> Result { - let pb_sequence = pb::RowIdSequence::decode(reader)?; + let pb_sequence = pb::RowIdSequence::decode(reader).map_err(|error| { + corrupt_row_id_metadata(format!("failed to decode row ID sequence: {error}")) + })?; RowIdSequence::try_from(pb_sequence) } #[cfg(test)] mod test { - use super::*; use pretty_assertions::assert_eq; + use proptest::prelude::*; + use rstest::rstest; + + use super::*; + + #[test] + fn test_bitmap_serialization_is_byte_exact() { + let mut bitmap = Bitmap::new_full(10); + bitmap.clear(2); + let segment = U64Segment::RangeWithBitmap { + range: 100..110, + bitmap, + }; + assert_eq!(segment.len(), 9); + + let serialized = pb::U64Segment::from(segment.clone()); + let Some(pb::u64_segment::Segment::RangeWithBitmap(encoded)) = &serialized.segment else { + panic!("expected bitmap segment"); + }; + assert_eq!(encoded.bitmap, vec![0xfb, 0x03]); + assert_eq!(U64Segment::try_from(serialized).unwrap(), segment); + } + fn read_segment(segment: pb::u64_segment::Segment) -> Result { + let sequence = pb::RowIdSequence { + segments: vec![pb::U64Segment { + segment: Some(segment), + }], + }; + read_row_ids(&sequence.encode_to_vec()) + } + + fn assert_corrupt_segment(segment: pb::u64_segment::Segment, expected_message: &str) { + let error = read_segment(segment).unwrap_err(); + assert!(matches!(&error, Error::CorruptFile { .. })); + assert!( + error.to_string().contains(expected_message), + "expected error containing {expected_message:?}, got {error}" + ); + } + + /// Each segment length fits a usize, but the aggregate does not fit a u64. Accepting + /// this would leave `RowIdSequence::len()` overflowing for callers such as + /// `Dataset::validate()`. + #[test] + fn test_reject_sequence_length_overflow() { + let segment = || pb::U64Segment { + segment: Some(pb::u64_segment::Segment::Range(pb::u64_segment::Range { + start: 0, + end: u64::MAX, + })), + }; + let sequence = pb::RowIdSequence { + segments: vec![segment(), segment()], + }; + + let error = read_row_ids(&sequence.encode_to_vec()).unwrap_err(); + assert!(matches!(&error, Error::CorruptFile { .. })); + assert!( + error + .to_string() + .contains("total length exceeding u64::MAX"), + "got {error}" + ); + } #[test] fn test_write_read_row_ids() { @@ -216,9 +398,11 @@ mod test { range: 100..200, holes: EncodedU64Array::U64(vec![104, 108, 150]), }); + let mut bitmap = Bitmap::new_empty(100); + bitmap.set(99); sequence.0.push(U64Segment::RangeWithBitmap { range: 200..300, - bitmap: Bitmap::new_empty(100), + bitmap, }); sequence .0 @@ -228,7 +412,7 @@ mod test { })); sequence .0 - .push(U64Segment::Array(EncodedU64Array::U64(vec![1, 2, 3]))); + .push(U64Segment::Array(EncodedU64Array::U64(vec![3, 1, 2]))); let serialized = write_row_ids(&sequence); @@ -236,4 +420,275 @@ mod test { assert_eq!(sequence.0, sequence2.0); } + + proptest! { + #[test] + fn test_row_id_sequence_len_round_trips( + values in proptest::collection::btree_set(any::(), 0..128) + ) { + let values = values.into_iter().collect::>(); + let sequence = RowIdSequence::from(values.as_slice()); + let deserialized = read_row_ids(&write_row_ids(&sequence)).unwrap(); + + prop_assert_eq!(deserialized.len(), sequence.len()); + prop_assert_eq!(deserialized.iter().collect::>(), values); + } + + #[test] + fn test_rejects_wrong_range_with_bitmap_length( + range_len in 1usize..512, + ) { + let expected_len = range_len.div_ceil(8); + for actual_len in [expected_len - 1, expected_len + 1] { + let segment = pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start: 0, + end: range_len as u64, + bitmap: vec![0; actual_len], + }, + ); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("byte length")); + } + } + + #[test] + fn test_rejects_range_with_bitmap_padding_bits( + full_bytes in 0usize..64, + valid_bits in 1usize..8, + ) { + let range_len = full_bytes * 8 + valid_bits; + let mut bitmap = vec![0; full_bytes + 1]; + bitmap[full_bytes] = 1 << valid_bits; + let segment = pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start: 0, + end: range_len as u64, + bitmap, + }, + ); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("padding bits must be zero")); + } + + #[test] + fn test_rejects_reversed_range_with_bitmap( + start in 1u64..u64::MAX, + ) { + let segment = pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start, + end: start - 1, + bitmap: Vec::new(), + }, + ); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("range start")); + } + + #[test] + fn test_rejects_misaligned_encoded_array_bytes( + encoding in 0u8..3, + element_count in 0usize..16, + ) { + let width = match encoding { + 0 => 2, + 1 => 4, + _ => 8, + }; + let bytes = vec![0; element_count * width + 1]; + let array = match encoding { + 0 => pb::encoded_u64_array::Array::U16Array( + pb::encoded_u64_array::U16Array { base: 0, offsets: bytes }, + ), + 1 => pb::encoded_u64_array::Array::U32Array( + pb::encoded_u64_array::U32Array { base: 0, offsets: bytes }, + ), + _ => pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values: bytes }, + ), + }; + let segment = pb::u64_segment::Segment::Array(pb::EncodedU64Array { + array: Some(array), + }); + + let error = read_segment(segment).unwrap_err(); + let is_corrupt_file = matches!(&error, Error::CorruptFile { .. }); + prop_assert!(is_corrupt_file); + prop_assert!(error.to_string().contains("byte length")); + } + } + + #[test] + fn test_rejects_encoded_offset_overflow() { + use pb::encoded_u64_array as pb_array; + + let arrays = [ + pb_array::Array::U16Array(pb_array::U16Array { + base: u64::MAX, + offsets: 1u16.to_le_bytes().to_vec(), + }), + pb_array::Array::U32Array(pb_array::U32Array { + base: u64::MAX, + offsets: 1u32.to_le_bytes().to_vec(), + }), + ]; + for array in arrays { + assert_corrupt_segment( + pb::u64_segment::Segment::Array(pb::EncodedU64Array { array: Some(array) }), + "overflows u64", + ); + } + } + + #[rstest] + #[case::descending(vec![6, 5], "not strictly increasing")] + #[case::duplicate(vec![5, 5], "not strictly increasing")] + #[case::below_range(vec![4], "outside the range")] + #[case::at_end(vec![7], "outside the range")] + fn test_rejects_invalid_range_with_holes(#[case] values: Vec, #[case] message: &str) { + let values = values.into_iter().flat_map(u64::to_le_bytes).collect(); + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithHoles(pb::u64_segment::RangeWithHoles { + start: 5, + end: 7, + holes: Some(pb::EncodedU64Array { + array: Some(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values }, + )), + }), + }), + message, + ); + } + + #[test] + fn test_rejects_missing_range_with_holes_array() { + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithHoles(pb::u64_segment::RangeWithHoles { + start: 5, + end: 7, + holes: None, + }), + "missing its holes array", + ); + } + + #[rstest] + #[case::u16( + pb::encoded_u64_array::Array::U16Array(pb::encoded_u64_array::U16Array { + base: 0, + offsets: vec![1], + }), + "encoded u16 array byte length 1 is not a multiple of element width 2" + )] + #[case::u32( + pb::encoded_u64_array::Array::U32Array(pb::encoded_u64_array::U32Array { + base: 0, + offsets: vec![1, 2, 3], + }), + "encoded u32 array byte length 3 is not a multiple of element width 4" + )] + #[case::u64( + pb::encoded_u64_array::Array::U64Array(pb::encoded_u64_array::U64Array { + values: vec![1, 2, 3, 4, 5, 6, 7], + }), + "encoded u64 array byte length 7 is not a multiple of element width 8" + )] + fn test_rejects_misaligned_encoded_array( + #[case] array: pb::encoded_u64_array::Array, + #[case] message: &str, + ) { + assert_corrupt_segment( + pb::u64_segment::Segment::Array(pb::EncodedU64Array { array: Some(array) }), + message, + ); + } + + #[rstest] + #[case::range(pb::u64_segment::Segment::Range(pb::u64_segment::Range { + start: 10, + end: 9, + }))] + #[case::range_with_holes(pb::u64_segment::Segment::RangeWithHoles( + pb::u64_segment::RangeWithHoles { + start: 10, + end: 9, + holes: Some(pb::EncodedU64Array { + array: Some(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values: Vec::new() }, + )), + }), + } + ))] + #[case::range_with_bitmap(pb::u64_segment::Segment::RangeWithBitmap( + pb::u64_segment::RangeWithBitmap { + start: 10, + end: 9, + bitmap: Vec::new(), + } + ))] + fn test_rejects_reversed_range(#[case] segment: pb::u64_segment::Segment) { + assert_corrupt_segment(segment, "range start 10 exceeds end 9"); + } + + #[rstest] + #[case::short(vec![0])] + #[case::long(vec![0, 0, 0])] + fn test_rejects_incorrect_bitmap_length(#[case] bitmap: Vec) { + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithBitmap(pb::u64_segment::RangeWithBitmap { + start: 5, + end: 14, + bitmap, + }), + "does not match expected 2 for range start 5, end 14, and length 9", + ); + } + + #[test] + fn test_rejects_set_bitmap_padding_bits() { + assert_corrupt_segment( + pb::u64_segment::Segment::RangeWithBitmap(pb::u64_segment::RangeWithBitmap { + start: 5, + end: 14, + bitmap: vec![0xff, 0x03], + }), + "padding bits must be zero", + ); + } + + #[rstest] + #[case::u16(pb::encoded_u64_array::Array::U16Array( + pb::encoded_u64_array::U16Array { + base: 100, + offsets: vec![2, 0, 1, 0], + } + ))] + #[case::u32(pb::encoded_u64_array::Array::U32Array( + pb::encoded_u64_array::U32Array { + base: 100, + offsets: vec![2, 0, 0, 0, 1, 0, 0, 0], + } + ))] + #[case::u64(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { + values: vec![2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0], + } + ))] + fn test_rejects_unsorted_sorted_array(#[case] array: pb::encoded_u64_array::Array) { + assert_corrupt_segment( + pb::u64_segment::Segment::SortedArray(pb::EncodedU64Array { array: Some(array) }), + "SortedArray values are not sorted at indices 0 and 1: 2 exceeds 1", + ); + } } diff --git a/rust/lance-table/src/system_index.rs b/rust/lance-table/src/system_index.rs index 021c01a5e52..c315b1af326 100644 --- a/rust/lance-table/src/system_index.rs +++ b/rust/lance-table/src/system_index.rs @@ -13,3 +13,12 @@ pub mod frag_reuse; pub mod mem_wal; + +use crate::format::IndexMetadata; +use frag_reuse::FRAG_REUSE_INDEX_NAME; +use mem_wal::MEM_WAL_INDEX_NAME; + +/// Whether `index_meta` describes one of the system indices defined in this module. +pub fn is_system_index(index_meta: &IndexMetadata) -> bool { + index_meta.name == FRAG_REUSE_INDEX_NAME || index_meta.name == MEM_WAL_INDEX_NAME +} diff --git a/rust/lance-table/src/system_index/frag_reuse.rs b/rust/lance-table/src/system_index/frag_reuse.rs index 40bbc4f58b6..401cbde7a87 100644 --- a/rust/lance-table/src/system_index/frag_reuse.rs +++ b/rust/lance-table/src/system_index/frag_reuse.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, io::Cursor, sync::Arc}; use arrow_array::cast::AsArray; use arrow_array::types::UInt64Type; use arrow_array::{Array, ArrayRef, PrimitiveArray, RecordBatch, UInt64Array}; use lance_core::deepsize::{Context, DeepSizeOf}; +use lance_core::utils::row_addr_remap::{GroupInputWithLayout, RowAddrRemap}; use lance_core::{Error, Result}; use lance_select::RowAddrTreeMap; use roaring::{RoaringBitmap, RoaringTreemap}; @@ -196,9 +197,11 @@ impl FragReuseIndexDetails { } } -/// An index that stores row ID maps. -/// A row ID map describes the mapping from old row address to new address after compactions. -/// Each version contains the mapping for one round of compaction. +/// An index that stores materialized row ID maps. +/// +/// This type is retained for API and serde compatibility. Dataset loading uses +/// [`CompactFragReuseIndex`] so persisted FRI details are not expanded into a +/// hash-map entry for every affected row. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FragReuseIndex { pub uuid: Uuid, @@ -225,18 +228,162 @@ impl FragReuseIndex { } } + pub fn is_empty(&self) -> bool { + self.row_id_maps.iter().all(HashMap::is_empty) + } + pub fn remap_row_id(&self, row_id: u64) -> Option { - let mut mapped_value = Some(row_id); - for row_id_map in self.row_id_maps.iter() { - if mapped_value.is_some() { - mapped_value = row_id_map - .get(&mapped_value.unwrap()) - .copied() - .unwrap_or(mapped_value); + let mut mapped = Some(row_id); + for row_id_map in &self.row_id_maps { + if let Some(current) = mapped { + mapped = row_id_map.get(¤t).copied().unwrap_or(mapped); } } + mapped + } - mapped_value + pub fn remap_row_ids_in_place(&self, row_ids: &mut [Option]) { + for row_id_map in &self.row_id_maps { + for row_id in row_ids.iter_mut() { + if let Some(current) = *row_id + && let Some(mapped) = row_id_map.get(¤t) + { + *row_id = *mapped; + } + } + } + } + + pub fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { + RowAddrTreeMap::from_iter( + row_addrs + .row_addrs() + .unwrap() + .filter_map(|addr| self.remap_row_id(u64::from(addr))), + ) + } + + pub fn remap_row_ids_roaring_tree_map(&self, row_ids: &RoaringTreemap) -> RoaringTreemap { + RoaringTreemap::from_iter(row_ids.iter().filter_map(|addr| self.remap_row_id(addr))) + } + + pub fn remap_row_ids_record_batch( + &self, + batch: RecordBatch, + row_id_idx: usize, + ) -> Result { + remap_row_ids_record_batch(batch, row_id_idx, |row_ids| { + self.remap_row_ids_in_place(row_ids) + }) + } + + pub fn remap_row_ids_array(&self, array: ArrayRef) -> PrimitiveArray { + remap_row_ids_array(array, |row_ids| self.remap_row_ids_in_place(row_ids)) + } + + pub fn remap_fragment_bitmap(&self, fragment_bitmap: &mut RoaringBitmap) -> Result<()> { + remap_fragment_bitmap(&self.details, fragment_bitmap) + } +} + +/// A compact row-address remap chain for deferred compactions. +/// +/// Each FRI version retains rewritten-row bitmaps and fragment layouts. Queries +/// use bitmap rank plus the ordered new-fragment ranges instead of storing one +/// hash-map entry per affected row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompactFragReuseIndex { + pub uuid: Uuid, + row_addr_remap: RowAddrRemap, + pub details: FragReuseIndexDetails, +} + +impl DeepSizeOf for CompactFragReuseIndex { + fn deep_size_of_children(&self, cx: &mut Context) -> usize { + self.row_addr_remap.deep_size_of_children(cx) + self.details.deep_size_of_children(cx) + } +} + +impl CompactFragReuseIndex { + #[doc(hidden)] + pub fn from_row_id_maps( + uuid: Uuid, + row_id_maps: Vec>>, + details: FragReuseIndexDetails, + ) -> Self { + Self { + uuid, + row_addr_remap: RowAddrRemap::chained( + row_id_maps.into_iter().map(RowAddrRemap::direct), + ), + details, + } + } + + /// Build a queryable index directly from serialized FRI details without + /// expanding each affected row into a hash map. + pub fn try_new(uuid: Uuid, details: FragReuseIndexDetails) -> Result { + let mut version_remaps = Vec::with_capacity(details.versions.len()); + for (version_idx, version) in details.versions.iter().enumerate() { + let mut groups = Vec::with_capacity(version.groups.len()); + for (group_idx, group) in version.groups.iter().enumerate() { + let changed_row_addrs = RoaringTreemap::deserialize_from(Cursor::new( + &group.changed_row_addrs, + )) + .map_err(|error| { + Error::index(format!( + "failed to deserialize changed row addresses for FRI version {version_idx}, group {group_idx}: {error}" + )) + })?; + let old_frags = group + .old_frags + .iter() + .map(|frag| fragment_layout(frag, "old", version_idx, group_idx)) + .collect::>>()?; + let new_frags = group + .new_frags + .iter() + .map(|frag| fragment_layout(frag, "new", version_idx, group_idx)) + .collect::>>()?; + groups.push(GroupInputWithLayout { + rewritten_old_row_addrs: changed_row_addrs, + old_frags, + new_frags, + }); + } + let remap = RowAddrRemap::compact_with_layout(groups).map_err(|error| { + Error::index(format!( + "failed to build compact remap for FRI version {version_idx}: {error}" + )) + })?; + version_remaps.push(remap); + } + + Ok(Self { + uuid, + row_addr_remap: RowAddrRemap::chained(version_remaps), + details, + }) + } + + /// The ordered remap chain used by index and transaction remapping paths. + pub fn row_addr_remap(&self) -> &RowAddrRemap { + &self.row_addr_remap + } + + /// Returns whether the index contains no row-address remapping. + pub fn is_empty(&self) -> bool { + self.row_addr_remap.is_empty() + } + + pub fn remap_row_id(&self, row_id: u64) -> Option { + self.row_addr_remap.get(row_id).unwrap_or(Some(row_id)) + } + + /// Apply all FRI versions to row addresses in place. `None` values remain + /// deleted and missing mappings pass through unchanged. + pub fn remap_row_ids_in_place(&self, row_ids: &mut [Option]) { + self.row_addr_remap.remap_in_place(row_ids); } pub fn remap_row_addrs_tree_map(&self, row_addrs: &RowAddrTreeMap) -> RowAddrTreeMap { @@ -260,98 +407,304 @@ impl FragReuseIndex { batch: RecordBatch, row_id_idx: usize, ) -> Result { - assert_eq!(batch.schema().fields().len(), 2); - let other_column_idx = 1 - row_id_idx; - let row_ids = batch.column(row_id_idx).as_primitive::(); - let (val_indices, new_row_ids): (Vec, Vec) = row_ids - .values() - .iter() - .enumerate() - .filter_map(|(idx, old_id)| { - self.remap_row_id(*old_id) - .map(|new_id| (idx as u64, new_id)) - }) - .unzip(); - let new_val_indices = UInt64Array::from_iter_values(val_indices); - let new_vals = - arrow::compute::take(batch.column(other_column_idx), &new_val_indices, None)?; - - let mut batch_data: Vec<(usize, ArrayRef)> = vec![ - ( - row_id_idx, - Arc::new(UInt64Array::from_iter_values(new_row_ids)) as ArrayRef, - ), - (other_column_idx, Arc::new(new_vals)), - ]; - batch_data.sort_by_key(|(i, _)| *i); - Ok(RecordBatch::try_new( - batch.schema(), - batch_data.into_iter().map(|(_, item)| item).collect(), - )?) + remap_row_ids_record_batch(batch, row_id_idx, |row_ids| { + self.remap_row_ids_in_place(row_ids) + }) } pub fn remap_row_ids_array(&self, array: ArrayRef) -> PrimitiveArray { - let primitive_array = array - .as_any() - .downcast_ref::>() - .expect("expected row IDs to be uint64 array"); - (0..primitive_array.len()) - .map(|i| { - if primitive_array.is_null(i) { - None - } else { - self.remap_row_id(primitive_array.value(i)) - } - }) - .collect() + remap_row_ids_array(array, |row_ids| self.remap_row_ids_in_place(row_ids)) } pub fn remap_fragment_bitmap(&self, fragment_bitmap: &mut RoaringBitmap) -> Result<()> { - for version in self.details.versions.iter() { - for group in version.groups.iter() { - let mut removed = 0; - for old_frag in group.old_frags.iter() { - if fragment_bitmap.remove(old_frag.id as u32) { - removed += 1; - } + remap_fragment_bitmap(&self.details, fragment_bitmap) + } +} + +fn remap_row_ids_record_batch( + batch: RecordBatch, + row_id_idx: usize, + remap: impl FnOnce(&mut [Option]), +) -> Result { + assert_eq!(batch.schema().fields().len(), 2); + let other_column_idx = 1 - row_id_idx; + let row_ids = batch.column(row_id_idx).as_primitive::(); + let mut remapped_row_ids = row_ids + .values() + .iter() + .copied() + .map(Some) + .collect::>(); + remap(&mut remapped_row_ids); + let (val_indices, new_row_ids): (Vec, Vec) = remapped_row_ids + .iter() + .enumerate() + .filter_map(|(idx, new_id)| new_id.map(|new_id| (idx as u64, new_id))) + .unzip(); + let new_val_indices = UInt64Array::from_iter_values(val_indices); + let new_vals = arrow::compute::take(batch.column(other_column_idx), &new_val_indices, None)?; + + let mut batch_data: Vec<(usize, ArrayRef)> = vec![ + ( + row_id_idx, + Arc::new(UInt64Array::from_iter_values(new_row_ids)) as ArrayRef, + ), + (other_column_idx, Arc::new(new_vals)), + ]; + batch_data.sort_by_key(|(i, _)| *i); + Ok(RecordBatch::try_new( + batch.schema(), + batch_data.into_iter().map(|(_, item)| item).collect(), + )?) +} + +fn remap_row_ids_array( + array: ArrayRef, + remap: impl FnOnce(&mut [Option]), +) -> PrimitiveArray { + let primitive_array = array + .as_any() + .downcast_ref::>() + .expect("expected row IDs to be uint64 array"); + let mut remapped = (0..primitive_array.len()) + .map(|i| { + if primitive_array.is_null(i) { + None + } else { + Some(primitive_array.value(i)) + } + }) + .collect::>(); + remap(&mut remapped); + PrimitiveArray::from(remapped) +} + +fn remap_fragment_bitmap( + details: &FragReuseIndexDetails, + fragment_bitmap: &mut RoaringBitmap, +) -> Result<()> { + for version in details.versions.iter() { + for group in version.groups.iter() { + let mut removed = 0; + for old_frag in group.old_frags.iter() { + if fragment_bitmap.remove(old_frag.id as u32) { + removed += 1; } + } - if removed > 0 { - if removed != group.old_frags.len() { - // Straddle: the index covered only part of this rewrite - // group. Caused by the bug fixed in - // . - // We've already removed the indexed old_frags from the - // bitmap above; deliberately do NOT insert new_frags, - // since the merged fragment also contains rows that - // were never indexed. Affected rows fall through to - // flat scan until the next optimize_indices. The fix - // is persisted on the next write via build_manifest. - tracing::warn!( - "Healing straddling fragment-reuse rewrite group in index bitmap: \ + if removed > 0 { + if removed != group.old_frags.len() { + // Straddle: the index covered only part of this rewrite + // group. Caused by the bug fixed in + // . + // We've already removed the indexed old_frags from the + // bitmap above; deliberately do NOT insert new_frags, + // since the merged fragment also contains rows that + // were never indexed. Affected rows fall through to + // flat scan until the next optimize_indices. The fix + // is persisted on the next write via build_manifest. + tracing::warn!( + "Healing straddling fragment-reuse rewrite group in index bitmap: \ group {:?} was only partially indexed ({} of {} old fragments). \ Affected rows will use flat scan until the next optimize_indices.", - group.old_frags, - removed, - group.old_frags.len(), - ); - continue; - } - - for new_frag in group.new_frags.iter() { - fragment_bitmap.insert(new_frag.id as u32); - } + group.old_frags, + removed, + group.old_frags.len(), + ); + continue; + } + + for new_frag in group.new_frags.iter() { + fragment_bitmap.insert(new_frag.id as u32); } } } - Ok(()) } + Ok(()) +} + +fn fragment_layout( + frag: &FragDigest, + role: &str, + version_idx: usize, + group_idx: usize, +) -> Result<(u32, u32)> { + let fragment_id = u32::try_from(frag.id).map_err(|_| { + Error::index(format!( + "FRI version {version_idx}, group {group_idx} has {role} fragment id {} outside the row-address range", + frag.id + )) + })?; + let physical_rows = u32::try_from(frag.physical_rows).map_err(|_| { + Error::index(format!( + "FRI version {version_idx}, group {group_idx} has {role} fragment {fragment_id} with physical_rows={} outside the row-address range", + frag.physical_rows + )) + })?; + Ok((fragment_id, physical_rows)) } #[cfg(test)] mod tests { use super::*; + use rstest::rstest; + + fn addr(fragment_id: u32, offset: u32) -> u64 { + u64::from(lance_core::utils::address::RowAddress::new_from_parts( + fragment_id, + offset, + )) + } + + fn serialize_changed(addrs: impl IntoIterator) -> Vec { + let changed = RoaringTreemap::from_iter(addrs); + let mut bytes = Vec::with_capacity(changed.serialized_size()); + changed.serialize_into(&mut bytes).unwrap(); + bytes + } + + fn digest(id: u64, physical_rows: usize) -> FragDigest { + FragDigest { + id, + physical_rows, + num_deleted_rows: 0, + } + } + + #[test] + fn test_compact_fri_tristate_one_to_many_and_chain() { + let details = FragReuseIndexDetails { + versions: vec![ + FragReuseVersion { + dataset_version: 1, + groups: vec![ + // One old fragment is split into two new fragments. + FragReuseGroup { + changed_row_addrs: serialize_changed([ + addr(1, 0), + addr(1, 2), + addr(1, 3), + ]), + old_frags: vec![digest(1, 4)], + new_frags: vec![digest(10, 1), digest(11, 2)], + }, + // A separate rewrite group deletes an entire fragment. + FragReuseGroup { + changed_row_addrs: serialize_changed([]), + old_frags: vec![digest(3, 2)], + new_frags: vec![], + }, + ], + }, + FragReuseVersion { + dataset_version: 2, + groups: vec![FragReuseGroup { + changed_row_addrs: serialize_changed([addr(10, 0), addr(11, 1)]), + old_frags: vec![digest(10, 1), digest(11, 2)], + new_frags: vec![digest(20, 2)], + }], + }, + ], + }; + let details = FragReuseIndexDetails::try_from(InlineContent::from(&details)).unwrap(); + let fri = CompactFragReuseIndex::try_new(Uuid::new_v4(), details).unwrap(); + + // Surviving rows follow both versions in oldest-to-newest order. + assert_eq!(fri.remap_row_id(addr(1, 0)), Some(addr(20, 0))); + assert_eq!(fri.remap_row_id(addr(1, 3)), Some(addr(20, 1))); + // Deletes can happen in either the first or a later version. + assert_eq!(fri.remap_row_id(addr(1, 1)), None); + assert_eq!(fri.remap_row_id(addr(1, 2)), None); + assert_eq!(fri.remap_row_id(addr(3, 0)), None); + // Uncovered fragments and out-of-range offsets retain the existing + // missing-map pass-through semantics. + assert_eq!(fri.remap_row_id(addr(2, 0)), Some(addr(2, 0))); + assert_eq!(fri.remap_row_id(addr(1, 4)), Some(addr(1, 4))); + + let mut batch = vec![ + Some(addr(1, 0)), + Some(addr(1, 1)), + Some(addr(1, 2)), + Some(addr(1, 3)), + Some(addr(2, 0)), + None, + ]; + fri.remap_row_ids_in_place(&mut batch); + assert_eq!( + batch, + vec![ + Some(addr(20, 0)), + None, + None, + Some(addr(20, 1)), + Some(addr(2, 0)), + None, + ] + ); + } + + #[test] + fn test_compact_fri_rejects_invalid_changed_row_bitmap() { + let details = FragReuseIndexDetails { + versions: vec![FragReuseVersion { + dataset_version: 1, + groups: vec![FragReuseGroup { + changed_row_addrs: vec![1, 2, 3], + old_frags: vec![digest(1, 1)], + new_frags: vec![digest(2, 1)], + }], + }], + }; + let error = CompactFragReuseIndex::try_new(Uuid::new_v4(), details).unwrap_err(); + assert!(matches!(error, Error::Index { .. })); + assert!( + error + .to_string() + .contains("failed to deserialize changed row addresses for FRI version 0, group 0"), + "{error}" + ); + } + + #[rstest] + #[case::unknown_fragment( + vec![addr(2, 0)], + vec![digest(1, 1)], + "from fragments [2] not in its old fragments" + )] + #[case::offset_out_of_range( + vec![addr(1, 1)], + vec![digest(1, 1)], + "row offset outside old fragment 1 with physical_rows=1" + )] + #[case::duplicate_old_fragment( + vec![addr(1, 0)], + vec![digest(1, 1), digest(1, 1)], + "old fragment 1 more than once" + )] + fn test_compact_fri_preserves_layout_validation( + #[case] changed_addrs: Vec, + #[case] old_frags: Vec, + #[case] expected_message: &str, + ) { + let details = FragReuseIndexDetails { + versions: vec![FragReuseVersion { + dataset_version: 1, + groups: vec![FragReuseGroup { + changed_row_addrs: serialize_changed(changed_addrs), + old_frags, + new_frags: vec![digest(10, 1)], + }], + }], + }; + + let error = CompactFragReuseIndex::try_new(Uuid::new_v4(), details).unwrap_err(); + assert!(matches!(error, Error::Index { .. })); + let message = error.to_string(); + assert!(message.contains("FRI version 0"), "{message}"); + assert!(message.contains("rewrite group 0"), "{message}"); + assert!(message.contains(expected_message), "{message}"); + } #[tokio::test] async fn test_serialize_deserialize_index_details() { diff --git a/rust/lance-table/src/system_index/mem_wal.rs b/rust/lance-table/src/system_index/mem_wal.rs index 1d82fd9e44f..7db96ed8356 100644 --- a/rust/lance-table/src/system_index/mem_wal.rs +++ b/rust/lance-table/src/system_index/mem_wal.rs @@ -1,59 +1,72 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashMap; +//! MemWAL index data structures and metadata helpers. +//! +//! The MemWAL Index stores: +//! - Configuration (sharding_specs, maintained_indexes) +//! - SSTable compaction progress +//! - Shard state snapshots (eventually consistent) +//! +//! Writers no longer update the index on every write. Instead, they update +//! shard manifests directly. This module provides functions to: +//! - Load the MemWAL index +//! - Update compacted SSTables (called during merge-insert commits) + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; -use lance_core::Error; use lance_core::deepsize::DeepSizeOf; +use lance_core::{Error, Result}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::format::pb; +use crate::format::{IndexMetadata, pb}; pub const MEM_WAL_INDEX_NAME: &str = "__lance_mem_wal"; /// Type alias for shard identifier (UUID v4). pub type ShardId = Uuid; -/// A flushed MemTable generation and its storage location. +/// An SSTable: the immutable result of flushing a MemTable, stored as a Lance dataset. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] -pub struct FlushedGeneration { +pub struct SsTable { pub generation: u64, pub path: String, } -impl From<&FlushedGeneration> for pb::FlushedGeneration { - fn from(fg: &FlushedGeneration) -> Self { +impl From<&SsTable> for pb::SsTable { + fn from(sstable: &SsTable) -> Self { Self { - generation: fg.generation, - path: fg.path.clone(), + generation: sstable.generation, + path: sstable.path.clone(), } } } -impl From for FlushedGeneration { - fn from(fg: pb::FlushedGeneration) -> Self { +impl From for SsTable { + fn from(sstable: pb::SsTable) -> Self { Self { - generation: fg.generation, - path: fg.path, + generation: sstable.generation, + path: sstable.path, } } } -/// A shard's merged generation, used in MemWalIndexDetails. +/// A pointer to the latest SSTable compacted for a shard. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash, Serialize, Deserialize)] -pub struct MergedGeneration { +pub struct CompactedSsTable { pub shard_id: Uuid, pub generation: u64, } -impl DeepSizeOf for MergedGeneration { +impl DeepSizeOf for CompactedSsTable { fn deep_size_of_children(&self, _context: &mut lance_core::deepsize::Context) -> usize { 0 // UUID is 16 bytes fixed size, no heap allocations } } -impl MergedGeneration { +impl CompactedSsTable { pub fn new(shard_id: Uuid, generation: u64) -> Self { Self { shard_id, @@ -62,41 +75,41 @@ impl MergedGeneration { } } -impl From<&MergedGeneration> for pb::MergedGeneration { - fn from(mg: &MergedGeneration) -> Self { +impl From<&CompactedSsTable> for pb::CompactedSsTable { + fn from(sstable: &CompactedSsTable) -> Self { Self { - shard_id: Some((&mg.shard_id).into()), - generation: mg.generation, + shard_id: Some((&sstable.shard_id).into()), + generation: sstable.generation, } } } -impl TryFrom for MergedGeneration { +impl TryFrom for CompactedSsTable { type Error = Error; - fn try_from(mg: pb::MergedGeneration) -> lance_core::Result { - let shard_id = mg + fn try_from(sstable: pb::CompactedSsTable) -> lance_core::Result { + let shard_id = sstable .shard_id .as_ref() .map(Uuid::try_from) - .ok_or_else(|| Error::invalid_input("Missing shard_id in MergedGeneration"))??; + .ok_or_else(|| Error::invalid_input("Missing shard_id in CompactedSsTable"))??; Ok(Self { shard_id, - generation: mg.generation, + generation: sstable.generation, }) } } -/// Tracks which merged generation a base table index has been rebuilt to cover. -/// Used to determine whether to read from flushed MemTable indexes or base table. +/// Tracks which compacted SSTable generation a base table index covers. +/// Used to determine whether to read from SSTable indexes or base table. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, DeepSizeOf)] pub struct IndexCatchupProgress { pub index_name: String, - pub caught_up_generations: Vec, + pub caught_up_generations: Vec, } impl IndexCatchupProgress { - pub fn new(index_name: String, caught_up_generations: Vec) -> Self { + pub fn new(index_name: String, caught_up_generations: Vec) -> Self { Self { index_name, caught_up_generations, @@ -108,8 +121,8 @@ impl IndexCatchupProgress { pub fn caught_up_generation_for_shard(&self, shard_id: &Uuid) -> Option { self.caught_up_generations .iter() - .find(|mg| &mg.shard_id == shard_id) - .map(|mg| mg.generation) + .find(|sstable| &sstable.shard_id == shard_id) + .map(|sstable| sstable.generation) } } @@ -120,7 +133,7 @@ impl From<&IndexCatchupProgress> for pb::IndexCatchupProgress { caught_up_generations: icp .caught_up_generations .iter() - .map(|mg| mg.into()) + .map(|sstable| sstable.into()) .collect(), } } @@ -135,7 +148,7 @@ impl TryFrom for IndexCatchupProgress { caught_up_generations: icp .caught_up_generations .into_iter() - .map(MergedGeneration::try_from) + .map(CompactedSsTable::try_from) .collect::>()?, }) } @@ -198,16 +211,27 @@ pub struct ShardManifest { /// 1-based. pub wal_entry_position_last_seen: u64, pub current_generation: u64, - pub flushed_generations: Vec, + pub sstables: Vec, /// Lifecycle status (drop-table 2PC). Defaults to `Active`; preserved /// across claims via `..base` so only fresh constructions set it. pub status: ShardStatus, } +impl ShardManifest { + /// The version a manifest built on this one must carry. + /// + /// Manifest versions are CAS-allocated and must stay gap-free: a reader + /// scans forward and stops at the first version it cannot find, so a gap + /// hides everything past it. + pub fn next_version(&self) -> u64 { + self.version + 1 + } +} + impl DeepSizeOf for ShardManifest { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { self.shard_field_values.deep_size_of_children(context) - + self.flushed_generations.deep_size_of_children(context) + + self.sstables.deep_size_of_children(context) } } @@ -229,7 +253,7 @@ impl From<&ShardManifest> for pb::ShardManifest { replay_after_wal_entry_position: rm.replay_after_wal_entry_position, wal_entry_position_last_seen: rm.wal_entry_position_last_seen, current_generation: rm.current_generation, - flushed_generations: rm.flushed_generations.iter().map(|fg| fg.into()).collect(), + sstables: rm.sstables.iter().map(|sstable| sstable.into()).collect(), status: rm.status.to_i32(), } } @@ -258,11 +282,7 @@ impl TryFrom for ShardManifest { replay_after_wal_entry_position: rm.replay_after_wal_entry_position, wal_entry_position_last_seen: rm.wal_entry_position_last_seen, current_generation: rm.current_generation, - flushed_generations: rm - .flushed_generations - .into_iter() - .map(FlushedGeneration::from) - .collect(), + sstables: rm.sstables.into_iter().map(SsTable::from).collect(), status: ShardStatus::from_i32(rm.status), }) } @@ -338,7 +358,7 @@ pub struct MemWalIndexDetails { pub inline_snapshots: Option>, pub sharding_specs: Vec, pub maintained_indexes: Vec, - pub merged_generations: Vec, + pub compacted_sstables: Vec, pub index_catchup: Vec, /// Default `ShardWriter` configuration values for this MemWAL index. /// @@ -357,10 +377,10 @@ impl From<&MemWalIndexDetails> for pb::MemWalIndexDetails { inline_snapshots: details.inline_snapshots.clone(), sharding_specs: details.sharding_specs.iter().map(|rs| rs.into()).collect(), maintained_indexes: details.maintained_indexes.clone(), - merged_generations: details - .merged_generations + compacted_sstables: details + .compacted_sstables .iter() - .map(|mg| mg.into()) + .map(|sstable| sstable.into()) .collect(), index_catchup: details.index_catchup.iter().map(|icp| icp.into()).collect(), writer_config_defaults: details.writer_config_defaults.clone(), @@ -382,10 +402,10 @@ impl TryFrom for MemWalIndexDetails { .map(ShardingSpec::from) .collect(), maintained_indexes: details.maintained_indexes, - merged_generations: details - .merged_generations + compacted_sstables: details + .compacted_sstables .into_iter() - .map(MergedGeneration::try_from) + .map(CompactedSsTable::try_from) .collect::>()?, index_catchup: details .index_catchup @@ -408,12 +428,12 @@ impl MemWalIndex { Self { details } } - pub fn merged_generation_for_shard(&self, shard_id: &Uuid) -> Option { + pub fn compacted_generation_for_shard(&self, shard_id: &Uuid) -> Option { self.details - .merged_generations + .compacted_sstables .iter() - .find(|mg| &mg.shard_id == shard_id) - .map(|mg| mg.generation) + .find(|sstable| &sstable.shard_id == shard_id) + .map(|sstable| sstable.generation) } /// Get the caught up generation for a specific index and shard. @@ -425,14 +445,133 @@ impl MemWalIndex { .find(|icp| icp.index_name == index_name) .and_then(|icp| icp.caught_up_generation_for_shard(shard_id)) } +} - /// Check if an index is fully caught up for a shard. - /// Returns true if the index covers all merged data for the shard. - pub fn is_index_caught_up(&self, index_name: &str, shard_id: &Uuid) -> bool { - let merged_gen = self.merged_generation_for_shard(shard_id).unwrap_or(0); - let caught_up_gen = self.index_caught_up_generation(index_name, shard_id); +// Reading and updating the `IndexMetadata` entry that carries the details above. - // If not tracked in index_catchup, assumed fully caught up - caught_up_gen.is_none_or(|generation| generation >= merged_gen) +/// Load MemWalIndexDetails from an IndexMetadata. +pub fn load_mem_wal_index_details(index: IndexMetadata) -> Result { + if let Some(details_any) = index.index_details.as_ref() { + if !details_any.type_url.ends_with("MemWalIndexDetails") { + return Err(Error::index(format!( + "Index details is not for the MemWAL index, but {}", + details_any.type_url + ))); + } + + Ok(MemWalIndexDetails::try_from( + details_any.to_msg::()?, + )?) + } else { + Err(Error::index("Index details not found for the MemWAL index")) } } + +/// Open the MemWAL index from its metadata. +pub fn open_mem_wal_index(index: IndexMetadata) -> Result> { + Ok(Arc::new(MemWalIndex::new(load_mem_wal_index_details( + index, + )?))) +} + +/// Update `compacted_sstables` in the MemWAL index. +/// +/// Called from the final data-changing merge-insert commit for a compaction +/// target, so the rows and the generation that describes them publish +/// together. +/// +/// A proposed generation must be **strictly greater** than the one the latest +/// state records for that shard, and a stale one fails the whole transaction. +/// Accepting it while keeping the larger marker would publish that worker's row +/// mutations under a generation it did not produce, and anything reading only +/// the marker could then stop serving SSTables whose rows were never inserted. +/// +/// Every other `MemWalIndexDetails` field is carried through untouched. +pub fn update_mem_wal_index_compacted_sstables( + indices: &mut [IndexMetadata], + dataset_version: u64, + new_compacted_sstables: Vec, +) -> Result<()> { + if new_compacted_sstables.is_empty() { + return Ok(()); + } + + let mut seen_shards = HashSet::with_capacity(new_compacted_sstables.len()); + for sstable in &new_compacted_sstables { + if !seen_shards.insert(sstable.shard_id) { + return Err(Error::invalid_input(format!( + "Duplicate shard {} in one SSTable compaction update; each shard \ + may advance at most once per transaction", + sstable.shard_id + ))); + } + } + + // Default details would describe a table with no MemWAL shards at all, so + // the recorded generation would name a shard nothing can corroborate. + // Refuse instead of inventing metadata. + let pos = indices + .iter() + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + .ok_or_else(|| { + Error::invalid_input(format!( + "Cannot record SSTable compaction progress: the {} system index \ + does not exist on this table", + MEM_WAL_INDEX_NAME + )) + })?; + + // Validated against a copy so a rejected update leaves `indices` exactly as + // the caller passed it. + let mut details = load_mem_wal_index_details(indices[pos].clone())?; + + for new_sstable in new_compacted_sstables { + match details + .compacted_sstables + .iter_mut() + .find(|sstable| sstable.shard_id == new_sstable.shard_id) + { + Some(existing) if new_sstable.generation <= existing.generation => { + return Err(Error::invalid_input(format!( + "Stale SSTable compaction for shard {}: proposed generation {} \ + is not greater than the recorded generation {}", + new_sstable.shard_id, new_sstable.generation, existing.generation + ))); + } + Some(existing) => existing.generation = new_sstable.generation, + None => details.compacted_sstables.push(new_sstable), + } + } + + // Replaced in place so the index list keeps its order. + indices[pos] = new_mem_wal_index_meta(dataset_version, details)?; + Ok(()) +} + +/// Create a new MemWAL index metadata entry. +/// +/// A fresh UUID is minted on every rewrite, including metadata-only updates. +/// The decoded-details cache is keyed on that UUID, so the change of identity +/// is what invalidates it; holding the UUID steady would leave a warmed reader +/// answering with the state from before the update. +pub fn new_mem_wal_index_meta( + dataset_version: u64, + details: MemWalIndexDetails, +) -> Result { + Ok(IndexMetadata { + uuid: Uuid::new_v4(), + name: MEM_WAL_INDEX_NAME.to_string(), + fields: vec![], + covering_fields: vec![], + dataset_version, + fragment_bitmap: None, + index_details: Some(Arc::new(prost_types::Any::from_msg( + &pb::MemWalIndexDetails::from(&details), + )?)), + index_version: 0, + created_at: Some(chrono::Utc::now()), + base_id: None, + // Memory WAL index is inline (no files) + files: None, + }) +} diff --git a/rust/lance-table/src/transaction.rs b/rust/lance-table/src/transaction.rs new file mode 100644 index 00000000000..74fba1b33ba --- /dev/null +++ b/rust/lance-table/src/transaction.rs @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Transaction definitions for updating datasets +//! +//! Prior to creating a new manifest, a transaction must be created representing +//! the changes being made to the dataset. By representing them as incremental +//! changes, we can detect whether concurrent operations are compatible with +//! one another. We can also rebuild manifests when retrying committing a +//! manifest. +//! +//! For more details please refer to the +//! [Transaction Specification](https://lance.org/format/table/transaction/#transaction-types). +//! +//! The work splits along these lines: +//! +//! ```text +//! builder Transaction: an operation plus the version it was based on +//! operation the vocabulary of changes an operation can describe +//! update_map incremental edits to the manifest's string maps +//! validate pre-commit checks against the manifest being replaced +//! manifest_build applying an operation to produce the next manifest +//! index_maintenance how that narrows or drops index metadata +//! row_version how it assigns row ids and per-row version metadata +//! conflicts whether two operations collide, for the commit retry path +//! proto the persisted protobuf encoding of all of the above +//! ``` + +mod builder; +mod conflicts; +mod index_maintenance; +mod manifest_build; +mod operation; +mod proto; +mod row_version; +mod update_map; +mod validate; + +#[cfg(test)] +pub(crate) mod test_support; + +pub use builder::{Transaction, TransactionBuilder}; +pub use operation::{ + DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, UpdateMode, + UpdatedFragmentOffsets, +}; +pub use update_map::{ + UpdateMap, UpdateMapEntry, translate_config_updates, translate_schema_metadata_updates, +}; +pub use validate::validate_operation; + +use crate::format::{IndexMetadata, Manifest}; +use roaring::RoaringBitmap; +use std::collections::BTreeMap; +use uuid::Uuid; + +/// Non-system logical index name -> its physical segments, ordered by UUID. +/// +/// Whole segment metadata rather than UUIDs alone: operations such as `Rewrite` +/// prune a segment's fragment bitmap while keeping its UUID, so a UUID-only +/// comparison would keep coverage for an index that no longer spans the same +/// base fragments. +pub type LogicalIndexSegments = BTreeMap>; + +/// What one physical index segment contributes to coverage. +/// +/// Deliberately not the whole [`IndexMetadata`]. It rests on one contract: +/// changing an index's physical contents mints a new UUID. Of the mutations +/// sanctioned under an existing UUID, only the fragment bitmap changes which +/// rows the index answers for -- an `Update` prunes it in place, and +/// `migrate_indices` recalculates it -- so the UUID alone is not enough and the +/// bitmap has to be compared too. The rest of the metadata, file lists and +/// timestamps and inferred details, is filled in by migrations routinely; +/// comparing it would withdraw coverage for no reason. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CoverageIdentity { + uuid: Uuid, + fragment_bitmap: Option, +} + +/// The version a transaction read, as the coverage derivation needs it. +/// +/// An index covering every fragment live at this version holds every row +/// compaction had copied into the base table by then, so it is caught up to +/// that version's `compacted_sstables`. That is the only proof available: +/// nothing maps a compaction generation to the fragments its rows landed in. +/// +/// `read_version` is fixed for the life of a transaction and survives rebase, +/// so the credit a commit can prove is stable across attempts. The recorded +/// result may still differ between attempts, because a rebased attempt sees a +/// different head: other commits move the compacted generations and the +/// positions already recorded. +#[derive(Debug, Clone, Copy)] +pub struct ReadVersionState<'a> { + pub manifest: &'a Manifest, + pub indices: &'a [IndexMetadata], +} diff --git a/rust/lance-table/src/transaction/builder.rs b/rust/lance-table/src/transaction/builder.rs new file mode 100644 index 00000000000..1240de4d882 --- /dev/null +++ b/rust/lance-table/src/transaction/builder.rs @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! The transaction itself: an operation plus the version it was based on. + +use crate::transaction::Operation; +use lance_core::deepsize::DeepSizeOf; +use std::collections::HashMap; +use std::sync::Arc; +use uuid::Uuid; + +/// A change to a dataset that can be retried +/// +/// This contains enough information to be able to build the next manifest, +/// given the current manifest. +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct Transaction { + /// The version of the table this transaction is based off of. If this is + /// the first transaction, this should be 0. + pub read_version: u64, + pub uuid: String, + pub operation: Operation, + pub tag: Option, + pub transaction_properties: Option>>, +} + +/// Add TransactionBuilder for flexibly setting option without using `mut` +pub struct TransactionBuilder { + read_version: u64, + // uuid is optional for builder since it can autogenerate + uuid: Option, + operation: Operation, + tag: Option, + transaction_properties: Option>>, +} + +impl TransactionBuilder { + pub fn new(read_version: u64, operation: Operation) -> Self { + Self { + read_version, + uuid: None, + operation, + tag: None, + transaction_properties: None, + } + } + + pub fn uuid(mut self, uuid: String) -> Self { + self.uuid = Some(uuid); + self + } + + pub fn tag(mut self, tag: Option) -> Self { + self.tag = tag; + self + } + + pub fn transaction_properties( + mut self, + transaction_properties: Option>>, + ) -> Self { + self.transaction_properties = transaction_properties; + self + } + + pub fn build(self) -> Transaction { + let uuid = self + .uuid + .unwrap_or_else(|| Uuid::new_v4().hyphenated().to_string()); + Transaction { + read_version: self.read_version, + uuid, + operation: self.operation, + tag: self.tag, + transaction_properties: self.transaction_properties, + } + } +} + +impl Transaction { + pub fn new_from_version(read_version: u64, operation: Operation) -> Self { + TransactionBuilder::new(read_version, operation).build() + } + + pub fn new(read_version: u64, operation: Operation, tag: Option) -> Self { + TransactionBuilder::new(read_version, operation) + .tag(tag) + .build() + } +} diff --git a/rust/lance-table/src/transaction/conflicts.rs b/rust/lance-table/src/transaction/conflicts.rs new file mode 100644 index 00000000000..cba1b5fb547 --- /dev/null +++ b/rust/lance-table/src/transaction/conflicts.rs @@ -0,0 +1,1042 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Deciding whether two operations describe the same change or touch the same +//! metadata. +//! +//! The commit path uses these when it retries against a newer version: equality +//! tells it whether the operation it is holding is the one already committed, and +//! the metadata checks tell it whether a concurrent operation wrote keys it +//! depends on. +//! +//! `PartialEq` is hand-written rather than derived because several operations +//! carry `Vec` fields whose order is not meaningful. + +use crate::transaction::{Operation, UpdateMap}; +use std::collections::HashSet; + +impl PartialEq for Operation { + fn eq(&self, other: &Self) -> bool { + // Many of the operations contain `Vec` where the order of the + // elements don't matter. So we need to compare them in a way that + // ignores the order of the elements. + // TODO: we can make it so the vecs are always constructed in order. + // Then we can use `==` instead of `compare_vec`. + fn compare_vec(a: &[T], b: &[T]) -> bool { + a.len() == b.len() && a.iter().all(|f| b.contains(f)) + } + match (self, other) { + (Self::Append { fragments: a }, Self::Append { fragments: b }) => compare_vec(a, b), + ( + Self::Clone { + is_shallow: a_is_shallow, + ref_name: a_ref_name, + ref_version: a_ref_version, + ref_path: a_source_path, + branch_name: a_branch_name, + }, + Self::Clone { + is_shallow: b_is_shallow, + ref_name: b_ref_name, + ref_version: b_ref_version, + ref_path: b_source_path, + branch_name: b_branch_name, + }, + ) => { + a_is_shallow == b_is_shallow + && a_ref_name == b_ref_name + && a_ref_version == b_ref_version + && a_source_path == b_source_path + && a_branch_name == b_branch_name + } + ( + Self::Delete { + updated_fragments: a_updated, + deleted_fragment_ids: a_deleted, + predicate: a_predicate, + }, + Self::Delete { + updated_fragments: b_updated, + deleted_fragment_ids: b_deleted, + predicate: b_predicate, + }, + ) => { + compare_vec(a_updated, b_updated) + && compare_vec(a_deleted, b_deleted) + && a_predicate == b_predicate + } + ( + Self::Overwrite { + fragments: a_fragments, + schema: a_schema, + config_upsert_values: a_config, + initial_bases: a_initial, + }, + Self::Overwrite { + fragments: b_fragments, + schema: b_schema, + config_upsert_values: b_config, + initial_bases: b_initial, + }, + ) => { + compare_vec(a_fragments, b_fragments) + && a_schema == b_schema + && a_config == b_config + && a_initial == b_initial + } + ( + Self::CreateIndex { + new_indices: a_new, + removed_indices: a_removed, + }, + Self::CreateIndex { + new_indices: b_new, + removed_indices: b_removed, + }, + ) => compare_vec(a_new, b_new) && compare_vec(a_removed, b_removed), + ( + Self::Rewrite { + groups: a_groups, + rewritten_indices: a_indices, + frag_reuse_index: a_frag_reuse_index, + }, + Self::Rewrite { + groups: b_groups, + rewritten_indices: b_indices, + frag_reuse_index: b_frag_reuse_index, + }, + ) => { + compare_vec(a_groups, b_groups) + && compare_vec(a_indices, b_indices) + && a_frag_reuse_index == b_frag_reuse_index + } + ( + Self::Merge { + fragments: a_fragments, + schema: a_schema, + preserves_nullability: a_preserves, + }, + Self::Merge { + fragments: b_fragments, + schema: b_schema, + preserves_nullability: b_preserves, + }, + ) => { + compare_vec(a_fragments, b_fragments) + && a_schema == b_schema + && a_preserves == b_preserves + } + (Self::Restore { version: a }, Self::Restore { version: b }) => a == b, + ( + Self::ReserveFragments { num_fragments: a }, + Self::ReserveFragments { num_fragments: b }, + ) => a == b, + ( + Self::Update { + removed_fragment_ids: a_removed, + updated_fragments: a_updated, + new_fragments: a_new, + fields_modified: a_fields, + compacted_sstables: a_compacted_sstables, + fields_for_preserving_frag_bitmap: a_fields_for_preserving_frag_bitmap, + update_mode: a_update_mode, + inserted_rows_filter: a_inserted_rows_filter, + updated_fragment_offsets: a_updated_fragment_offsets, + }, + Self::Update { + removed_fragment_ids: b_removed, + updated_fragments: b_updated, + new_fragments: b_new, + fields_modified: b_fields, + compacted_sstables: b_compacted_sstables, + fields_for_preserving_frag_bitmap: b_fields_for_preserving_frag_bitmap, + update_mode: b_update_mode, + inserted_rows_filter: b_inserted_rows_filter, + updated_fragment_offsets: b_updated_fragment_offsets, + }, + ) => { + compare_vec(a_removed, b_removed) + && compare_vec(a_updated, b_updated) + && compare_vec(a_new, b_new) + && compare_vec(a_fields, b_fields) + && compare_vec(a_compacted_sstables, b_compacted_sstables) + && compare_vec( + a_fields_for_preserving_frag_bitmap, + b_fields_for_preserving_frag_bitmap, + ) + && a_update_mode == b_update_mode + && a_inserted_rows_filter == b_inserted_rows_filter + && a_updated_fragment_offsets == b_updated_fragment_offsets + } + ( + Self::Project { + schema: a, + preserves_nullability: a_preserves, + }, + Self::Project { + schema: b, + preserves_nullability: b_preserves, + }, + ) => a == b && a_preserves == b_preserves, + ( + Self::UpdateConfig { + config_updates: a_config, + table_metadata_updates: a_table_metadata, + schema_metadata_updates: a_schema, + field_metadata_updates: a_field, + }, + Self::UpdateConfig { + config_updates: b_config, + table_metadata_updates: b_table_metadata, + schema_metadata_updates: b_schema, + field_metadata_updates: b_field, + }, + ) => { + a_config == b_config + && a_table_metadata == b_table_metadata + && a_schema == b_schema + && a_field == b_field + } + ( + Self::DataReplacement { replacements: a }, + Self::DataReplacement { replacements: b }, + ) => a.len() == b.len() && a.iter().all(|r| b.contains(r)), + // Handle all remaining combinations. + // We spell out all combinations explicitly to prevent + // us accidentally handling a new case in the wrong way. + (Self::Append { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Append { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Delete { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Overwrite { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::CreateIndex { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Rewrite { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Merge { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Restore { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::ReserveFragments { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Update { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Project { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::UpdateConfig { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::DataReplacement { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::UpdateMemWalState { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + ( + Self::UpdateMemWalState { + compacted_sstables: a_compacted, + }, + Self::UpdateMemWalState { + compacted_sstables: b_compacted, + }, + ) => compare_vec(a_compacted, b_compacted), + (Self::Clone { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::UpdateBases { new_bases: a }, Self::UpdateBases { new_bases: b }) => { + compare_vec(a, b) + } + + (Self::UpdateBases { .. }, Self::Append { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Delete { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Overwrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::CreateIndex { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Rewrite { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Merge { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Restore { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::ReserveFragments { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Update { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Project { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::UpdateConfig { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::DataReplacement { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::UpdateMemWalState { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateBases { .. }, Self::Clone { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + + (Self::Append { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Delete { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Overwrite { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::CreateIndex { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Rewrite { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Merge { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Restore { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::ReserveFragments { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Update { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Project { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateConfig { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataReplacement { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::UpdateMemWalState { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::Clone { .. }, Self::UpdateBases { .. }) => { + std::mem::discriminant(self) == std::mem::discriminant(other) + } + (Self::DataOverlay { groups: a }, Self::DataOverlay { groups: b }) => compare_vec(a, b), + (Self::DataOverlay { .. }, _) | (_, Self::DataOverlay { .. }) => false, + } + } +} + +impl Operation { + /// Returns the config keys that have been upserted by this operation. + fn get_upsert_config_keys(&self) -> Vec { + match self { + Self::Overwrite { + config_upsert_values: Some(upsert_values), + .. + } => { + let vec: Vec = upsert_values.keys().cloned().collect(); + vec + } + Self::UpdateConfig { + config_updates: Some(config_updates), + .. + } => config_updates + .update_entries + .iter() + .filter_map(|entry| { + if entry.value.is_some() { + Some(entry.key.clone()) + } else { + None + } + }) + .collect(), + _ => Vec::::new(), + } + } + + /// Returns the config keys that have been deleted by this operation. + fn get_delete_config_keys(&self) -> Vec { + match self { + Self::UpdateConfig { + config_updates: Some(config_updates), + .. + } => config_updates + .update_entries + .iter() + .filter_map(|entry| { + if entry.value.is_none() { + Some(entry.key.clone()) + } else { + None + } + }) + .collect(), + _ => Vec::::new(), + } + } + + pub fn modifies_same_metadata(&self, other: &Self) -> bool { + match (self, other) { + ( + Self::UpdateConfig { + table_metadata_updates, + schema_metadata_updates, + field_metadata_updates, + .. + }, + Self::UpdateConfig { + table_metadata_updates: other_table_metadata, + schema_metadata_updates: other_schema_metadata, + field_metadata_updates: other_field_metadata, + .. + }, + ) => { + if Self::update_maps_conflict( + table_metadata_updates.as_ref(), + other_table_metadata.as_ref(), + ) { + return true; + } + if schema_metadata_updates.is_some() && other_schema_metadata.is_some() { + return true; + } + if !field_metadata_updates.is_empty() && !other_field_metadata.is_empty() { + for field in field_metadata_updates.keys() { + if other_field_metadata.contains_key(field) { + return true; + } + } + } + false + } + _ => false, + } + } + + fn update_maps_conflict(left: Option<&UpdateMap>, right: Option<&UpdateMap>) -> bool { + let (Some(left), Some(right)) = (left, right) else { + return false; + }; + if left.replace || right.replace { + return true; + } + let left_keys = left + .update_entries + .iter() + .map(|entry| entry.key.as_str()) + .collect::>(); + right + .update_entries + .iter() + .any(|entry| left_keys.contains(entry.key.as_str())) + } + + /// Check whether another operation upserts a key that is referenced by another operation + pub fn upsert_key_conflict(&self, other: &Self) -> bool { + let self_upsert_keys = self.get_upsert_config_keys(); + let other_upsert_keys = other.get_upsert_config_keys(); + + let self_delete_keys = self.get_delete_config_keys(); + let other_delete_keys = other.get_delete_config_keys(); + + self_upsert_keys + .iter() + .any(|x| other_upsert_keys.contains(x) || other_delete_keys.contains(x)) + || other_upsert_keys + .iter() + .any(|x| self_upsert_keys.contains(x) || self_delete_keys.contains(x)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::test_support::overlay_with_field; + use crate::transaction::{DataOverlayGroup, UpdateMapEntry}; + use std::collections::HashMap; + + fn table_metadata_update(entries: Vec<(&str, Option<&str>)>, replace: bool) -> Operation { + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: Some(UpdateMap { + update_entries: entries.into_iter().map(UpdateMapEntry::from).collect(), + replace, + }), + schema_metadata_updates: None, + field_metadata_updates: HashMap::new(), + } + } + + #[test] + fn test_table_metadata_conflicts_on_same_key() { + let left = table_metadata_update(vec![("key", Some("1"))], false); + let same_key = table_metadata_update(vec![("key", Some("2"))], false); + let different_key = table_metadata_update(vec![("other", Some("2"))], false); + let replace = table_metadata_update(vec![("other", Some("2"))], true); + + assert!(left.modifies_same_metadata(&same_key)); + assert!(!left.modifies_same_metadata(&different_key)); + assert!(left.modifies_same_metadata(&replace)); + } + + #[test] + fn test_data_overlay_operation_eq() { + let overlay = |field: i32| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(field, 1)], + }], + }; + // Reflexive and value-based (the arm previously returned false for self). + assert_eq!(overlay(1), overlay(1)); + assert_ne!(overlay(1), overlay(2)); + // Not equal to a different operation kind (previously returned true vs Rewrite). + let rewrite = Operation::Rewrite { + groups: vec![], + rewritten_indices: vec![], + frag_reuse_index: None, + }; + assert_ne!(overlay(1), rewrite); + } +} diff --git a/rust/lance-table/src/transaction/index_maintenance.rs b/rust/lance-table/src/transaction/index_maintenance.rs new file mode 100644 index 00000000000..06cf9648a88 --- /dev/null +++ b/rust/lance-table/src/transaction/index_maintenance.rs @@ -0,0 +1,1050 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Keeping index metadata honest about what the new fragment list contains. +//! +//! An index entry claims coverage of a set of fragments and fields. Any operation +//! that rewrites data can invalidate part of that claim, so a commit has to either +//! narrow the entry's fragment bitmap, drop the fields it no longer describes, or +//! drop the index. Getting this wrong does not fail the commit -- it silently +//! returns stale rows from the index -- so each rule here is paired with a test. + +use crate::format::overlay::staleness::collect_overlay_stale_frags; +use crate::format::{Fragment, IndexMetadata}; +use crate::system_index::frag_reuse::FRAG_REUSE_INDEX_NAME; +use crate::system_index::is_system_index; +use crate::transaction::{RewriteGroup, RewrittenIndex, Transaction}; +use lance_core::datatypes::Schema; +use lance_core::{Error, Result}; +use roaring::RoaringBitmap; +use std::collections::{HashMap, HashSet}; + +impl Transaction { + pub(super) fn register_pure_rewrite_rows_update_frags_in_indices( + indices: &mut [IndexMetadata], + pure_update_frag_ids: &[u64], + original_fragment_ids: &[u64], + fields_for_preserving_frag_bitmap: &[u32], + original_overlaid_frags: &HashMap, + schema: &Schema, + ) -> Result<()> { + if pure_update_frag_ids.is_empty() { + return Ok(()); + } + + let value_updated_field_set = fields_for_preserving_frag_bitmap + .iter() + .collect::>(); + + for index in indices.iter_mut() { + // Physical row addresses cannot follow moved rows into a new fragment. + // Leave that fragment uncovered so the scanner reads it directly. + if index.results_are_row_addrs() { + continue; + } + let index_covers_modified_field = index.fields.iter().any(|field_id| { + value_updated_field_set.contains(&u32::try_from(*field_id).unwrap()) + }); + if index_covers_modified_field { + continue; + } + let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() else { + continue; + }; + + // Check that all the original fragments containing the updated rows are covered by + // the index. If not, some updated rows were not indexed, so we cannot index them. + let index_covers_all_original_fragments = original_fragment_ids + .iter() + .all(|&fragment_id| fragment_bitmap.contains(fragment_id as u32)); + if !index_covers_all_original_fragments { + continue; + } + + // A rewrite materializes overlays. If any of those overlays touched the + // column being indexed then the rewrite will modify that column. As a + // result, that index will no longer cover the fragment and it does not + // count as a pure rewrite and we must exclude it from the index's fragment + // bitmap. + let mut overlay_stale = RoaringBitmap::new(); + collect_overlay_stale_frags( + index, + original_overlaid_frags, + &mut overlay_stale, + schema, + )?; + if !overlay_stale.is_empty() { + continue; + } + + if let Some(fragment_bitmap) = index.fragment_bitmap.as_mut() { + for fragment_id in pure_update_frag_ids.iter().map(|f| *f as u32) { + fragment_bitmap.insert(fragment_id); + } + } + } + Ok(()) + } + + /// If an operation modifies one or more fields in a fragment then we need to remove + /// that fragment from any indices that cover one of the modified fields. + pub fn prune_updated_fields_from_indices( + indices: &mut [IndexMetadata], + updated_fragments: &[Fragment], + fields_modified: &[u32], + ) { + if fields_modified.is_empty() { + return; + } + + // If we modified any fields in the fragments then we need to remove those fragments + // from the index if the index covers one of those modified fields. + let fields_modified_set = fields_modified.iter().collect::>(); + for index in indices.iter_mut() { + if index + .fields + .iter() + .any(|field_id| fields_modified_set.contains(&u32::try_from(*field_id).unwrap())) + && let Some(fragment_bitmap) = &mut index.fragment_bitmap + { + for fragment_id in updated_fragments.iter().map(|f| f.id as u32) { + fragment_bitmap.remove(fragment_id); + } + } + } + } + + /// Map each (non-tombstoned) field id in a fragment to the path of the data + /// file that backs it. + fn fragment_field_paths(frag: &Fragment) -> HashMap { + let mut map = HashMap::new(); + for file in &frag.files { + for &field_id in file.fields.iter() { + if field_id >= 0 { + map.insert(field_id, file.path.as_str()); + } + } + } + map + } + + /// A `Merge` can rewrite a column's data *in place* -- the field stays in the + /// schema but its backing data file changes (the overlay fragment carries a new + /// file for the field and tombstones its old field id). `retain_relevant_indices` + /// only drops indices for *removed* fields, so without this the index keeps + /// covering the rewritten fragments with stale entries. Remove each such fragment + /// from any index covering a field whose backing data file changed. + pub(super) fn prune_merge_rewritten_fields_from_indices( + indices: &mut [IndexMetadata], + prev_fragments: &[Fragment], + new_fragments: &[Fragment], + ) { + let prev_by_id: HashMap = + prev_fragments.iter().map(|f| (f.id, f)).collect(); + for new_frag in new_fragments { + let Some(prev) = prev_by_id.get(&new_frag.id) else { + continue; // brand-new fragment: nothing stale to prune + }; + let prev_paths = Self::fragment_field_paths(prev); + let new_paths = Self::fragment_field_paths(new_frag); + // Fields still present whose backing file path changed == rewritten data. + let changed: Vec = prev_paths + .iter() + .filter(|(field_id, prev_path)| { + new_paths + .get(*field_id) + .is_some_and(|new_path| new_path != *prev_path) + }) + .map(|(field_id, _)| *field_id as u32) + .collect(); + if changed.is_empty() { + continue; + } + Self::prune_updated_fields_from_indices( + indices, + std::slice::from_ref(new_frag), + &changed, + ); + } + } + + /// After a `Rewrite` fully compacts a fragment, its data overlays are baked + /// into the new fragment's base data. An index built *before* one of those + /// overlays (`overlay.committed_version > index.dataset_version`) indexed the + /// stale pre-overlay values -- and unlike a live overlay, the compacted + /// fragment no longer signals that staleness to the query path. Drop each + /// rewritten (new) fragment from the coverage of any index covering a field + /// such an overlay supplied, so those rows fall back to a flat scan. + pub(super) fn prune_overlay_stale_fields_from_indices( + indices: &mut [IndexMetadata], + groups: &[RewriteGroup], + ) { + for group in groups { + // field id -> newest overlay committed_version supplying that field + let mut overlaid_field_versions: HashMap = HashMap::new(); + for old_frag in &group.old_fragments { + for overlay in &old_frag.overlays { + for &field_id in overlay.data_file.fields.iter() { + if field_id < 0 { + // Tombstoned (obsolete) overlay field: supplies nothing. + continue; + } + let entry = overlaid_field_versions.entry(field_id).or_insert(0); + *entry = (*entry).max(overlay.committed_version); + } + } + } + if overlaid_field_versions.is_empty() { + continue; + } + + let new_fragment_ids = group + .new_fragments + .iter() + .map(|f| f.id as u32) + .collect::>(); + for index in indices.iter_mut() { + let is_stale = index.fields.iter().any(|field_id| { + overlaid_field_versions + .get(field_id) + .is_some_and(|&overlay_version| overlay_version > index.dataset_version) + }); + if is_stale && let Some(fragment_bitmap) = &mut index.fragment_bitmap { + for new_id in &new_fragment_ids { + fragment_bitmap.remove(*new_id); + } + } + } + } + } + + pub(crate) fn retain_relevant_indices( + indices: &mut Vec, + schema: &Schema, + fragments: &[Fragment], + ) { + let field_ids = schema + .fields_pre_order() + .map(|f| f.id) + .collect::>(); + + // Remove indices for fields no longer in schema + indices.retain(|existing_index| { + existing_index + .fields + .iter() + .all(|field_id| field_ids.contains(field_id)) + || is_system_index(existing_index) + }); + + let mut indices_by_name: std::collections::HashMap> = + std::collections::HashMap::new(); + + for index in indices.iter() { + if index.name != FRAG_REUSE_INDEX_NAME { + indices_by_name + .entry(index.name.clone()) + .or_default() + .push(index); + } + } + + let mut uuids_to_keep = std::collections::HashSet::new(); + + let existing_fragments = fragments + .iter() + .map(|f| f.id as u32) + .collect::(); + + for (_, same_name_indices) in indices_by_name { + // Unknown coverage is not empty coverage: a segment whose bitmap is + // missing has never been measured, and dropping it deletes an index + // that migration could not open yet. + let (unknown_coverage, same_name_indices): (Vec<_>, Vec<_>) = same_name_indices + .into_iter() + .partition(|index| index.fragment_bitmap.is_none()); + for index in unknown_coverage { + uuids_to_keep.insert(index.uuid); + } + + if same_name_indices.len() > 1 { + let (empty_indices, non_empty_indices): (Vec<_>, Vec<_>) = + same_name_indices.iter().partition(|index| { + index + .effective_fragment_bitmap(&existing_fragments) + .as_ref() + .is_none_or(|bitmap| bitmap.is_empty()) + }); + + if non_empty_indices.is_empty() { + // All indices are empty -- keep only the oldest definition. + // + // An empty index definition is still correct: the scanner + // falls back to scanning unindexed fragments, and normal + // index maintenance rebuilds coverage once rows accrue. + // Dropping the definition instead would silently lose the + // index whenever an operation replaces every fragment it + // covered (e.g. a full table rewrite), leaving the dataset + // without its declared index. + let mut sorted_indices = empty_indices; + sorted_indices.sort_by_key(|index: &&IndexMetadata| index.dataset_version); + + if let Some(oldest) = sorted_indices.first() { + uuids_to_keep.insert(oldest.uuid); + } + } else { + for index in non_empty_indices { + uuids_to_keep.insert(index.uuid); + } + } + } else { + // Single index whose column is still in schema: keep it, even + // when its coverage is empty (see the all-empty note above). + if let Some(index) = same_name_indices.first() { + uuids_to_keep.insert(index.uuid); + } + } + } + + indices.retain(|index| { + index.name == FRAG_REUSE_INDEX_NAME || uuids_to_keep.contains(&index.uuid) + }); + } + + pub(super) fn recalculate_fragment_bitmap( + old: &RoaringBitmap, + groups: &[RewriteGroup], + ) -> Result { + let mut new_bitmap = old.clone(); + for group in groups { + let any_in_index = group + .old_fragments + .iter() + .any(|frag| old.contains(frag.id as u32)); + let all_in_index = group + .old_fragments + .iter() + .all(|frag| old.contains(frag.id as u32)); + // Any rewrite group may or may not be covered by the index. However, if any fragment + // in a rewrite group was previously covered by the index then all fragments in the rewrite + // group must have been previously covered by the index. plan_compaction takes care of + // this for us so this should be safe to assume. + if any_in_index { + if all_in_index { + for frag_id in group.old_fragments.iter().map(|frag| frag.id as u32) { + new_bitmap.remove(frag_id); + } + new_bitmap.extend(group.new_fragments.iter().map(|frag| frag.id as u32)); + } else { + return Err(Error::invalid_input( + "The compaction plan included a rewrite group that was a split of indexed and non-indexed data", + )); + } + } + } + Ok(new_bitmap) + } + + pub(super) fn handle_rewrite_indices( + indices: &mut [IndexMetadata], + rewritten_indices: &[RewrittenIndex], + groups: &[RewriteGroup], + ) -> Result<()> { + let mut modified_indices = HashSet::new(); + + for rewritten_index in rewritten_indices { + if !modified_indices.insert(rewritten_index.old_id) { + return Err(Error::invalid_input(format!( + "An invalid compaction plan must have been generated because multiple tasks modified the same index: {}", + rewritten_index.old_id + ))); + } + + // Skip indices that no longer exist (may have been removed by concurrent operation) + let Some(index) = indices + .iter_mut() + .find(|idx| idx.uuid == rewritten_index.old_id) + else { + continue; + }; + + index.fragment_bitmap = Some(Self::recalculate_fragment_bitmap( + index.fragment_bitmap.as_ref().ok_or_else(|| { + Error::invalid_input(format!( + "Cannot rewrite index {} which did not store fragment bitmap", + index.uuid + )) + })?, + groups, + )?); + index.uuid = rewritten_index.new_id; + // Update file sizes to match the new index files. When not available + // (e.g., from older writers), clear the old file sizes to avoid + // using stale sizes from the pre-remap index. + index.files = rewritten_index.new_index_files.clone(); + } + Ok(()) + } + + pub(super) fn handle_rewrite_fragments( + final_fragments: &mut Vec, + groups: &[RewriteGroup], + fragment_id: &mut u64, + version: u64, + _next_row_id: Option<&u64>, + ) -> Result<()> { + for group in groups { + // If the old fragments are contiguous, find the range + let replace_range = { + let start = final_fragments + .iter() + .enumerate() + .find(|(_, f)| f.id == group.old_fragments[0].id) + .ok_or_else(|| { + Error::commit_conflict_source( + version, + format!( + "dataset does not contain a fragment a rewrite operation wants to replace: id={}", + group.old_fragments[0].id + ) + .into(), + ) + })? + .0; + + // Verify old_fragments matches contiguous range + let mut i = 1; + loop { + if i == group.old_fragments.len() { + break Some(start..start + i); + } + if final_fragments[start + i].id != group.old_fragments[i].id { + break None; + } + i += 1; + } + }; + + let new_fragments = Self::fragments_with_ids(group.new_fragments.clone(), fragment_id) + .collect::>(); + + // Version metadata for rewritten fragments is handled by the compaction code + // (recalc_versions_for_rewritten_fragments) which preserves version information + // from the original fragments. We don't modify it here. + + if let Some(replace_range) = replace_range { + // Efficiently path using slice + final_fragments.splice(replace_range, new_fragments); + } else { + // Slower path for non-contiguous ranges + for fragment in group.old_fragments.iter() { + final_fragments.retain(|f| f.id != fragment.id); + } + final_fragments.extend(new_fragments); + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::test_support::overlay_with_field; + use uuid::Uuid; + + #[test] + fn test_rewrite_fragments() { + let existing_fragments: Vec = (0..10).map(Fragment::new).collect(); + + let mut final_fragments = existing_fragments; + let rewrite_groups = vec![ + // Since these are contiguous, they will be put in the same location + // as 1 and 2. + RewriteGroup { + old_fragments: vec![Fragment::new(1), Fragment::new(2)], + // These two fragments were previously reserved + new_fragments: vec![Fragment::new(15), Fragment::new(16)], + }, + // These are not contiguous, so they will be inserted at the end. + RewriteGroup { + old_fragments: vec![Fragment::new(5), Fragment::new(8)], + // We pretend this id was not reserved. Does not happen in practice today + // but we want to leave the door open. + new_fragments: vec![Fragment::new(0)], + }, + ]; + + let mut fragment_id = 20; + let version = 0; + + Transaction::handle_rewrite_fragments( + &mut final_fragments, + &rewrite_groups, + &mut fragment_id, + version, + None, + ) + .unwrap(); + + assert_eq!(fragment_id, 21); + + let expected_fragments: Vec = vec![ + Fragment::new(0), + Fragment::new(15), + Fragment::new(16), + Fragment::new(3), + Fragment::new(4), + Fragment::new(6), + Fragment::new(7), + Fragment::new(9), + Fragment::new(20), + ]; + + assert_eq!(final_fragments, expected_fragments); + } + + #[test] + fn test_retain_indices_removes_missing_fields() { + let schema = create_test_schema(&[1, 2]); + let fragments = vec![Fragment::new(1), Fragment::new(2)]; + + let mut indices = vec![ + create_test_index("idx1", 1, 1, Some(RoaringBitmap::from_iter([1])), false), + create_test_index("idx2", 2, 1, Some(RoaringBitmap::from_iter([1])), false), + create_test_index("idx3", 99, 1, Some(RoaringBitmap::from_iter([1])), false), // Field doesn't exist + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + assert_eq!(indices.len(), 2); + assert!(indices.iter().all(|idx| idx.fields[0] != 99)); + } + + #[test] + fn test_retain_indices_keeps_system_indices() { + use crate::system_index::mem_wal::MEM_WAL_INDEX_NAME; + + let schema = create_test_schema(&[1, 2]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_system_index(FRAG_REUSE_INDEX_NAME, 99), // Field doesn't exist but should be kept + create_system_index(MEM_WAL_INDEX_NAME, 99), // Field doesn't exist but should be kept + create_test_index("regular_idx", 99, 1, Some(RoaringBitmap::new()), false), // Should be removed + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + assert_eq!(indices.len(), 2); + assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); + assert!(indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME)); + } + + #[test] + fn test_retain_indices_keeps_fragment_reuse_index() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_system_index(FRAG_REUSE_INDEX_NAME, 1), + create_test_index("other_idx", 1, 1, Some(RoaringBitmap::new()), false), + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Fragment reuse index should always be kept + assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); + } + + #[test] + fn test_retain_single_empty_scalar_index() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![create_test_index( + "scalar_idx", + 1, + 1, + Some(RoaringBitmap::new()), // Empty bitmap + false, + )]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Single empty scalar index should be kept + assert_eq!(indices.len(), 1); + } + + #[test] + fn test_retain_single_empty_vector_index_is_kept() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![create_test_index( + "vector_idx", + 1, + 1, + Some(RoaringBitmap::new()), // Empty bitmap + true, + )]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // The empty definition is retained: coverage is empty but the index + // declaration must survive operations that replace every fragment. + assert_eq!(indices.len(), 1); + } + + #[test] + fn test_retain_single_nonempty_index() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut scalar_indices = vec![create_test_index( + "scalar_idx", + 1, + 1, + Some(RoaringBitmap::from_iter([1])), + false, + )]; + + let mut vector_indices = vec![create_test_index( + "vector_idx", + 1, + 1, + Some(RoaringBitmap::from_iter([1])), + true, + )]; + + Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); + Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); + + // Both should be kept + assert_eq!(scalar_indices.len(), 1); + assert_eq!(vector_indices.len(), 1); + } + + #[test] + fn test_retain_single_index_with_none_bitmap() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut scalar_indices = vec![create_test_index("scalar_idx", 1, 1, None, false)]; + let mut vector_indices = vec![create_test_index("vector_idx", 1, 1, None, true)]; + + Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); + Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); + + // Both kept: a None bitmap is unknown coverage, not empty coverage, and + // an unmeasured segment is retained regardless of index type. + assert_eq!(scalar_indices.len(), 1); + assert_eq!(vector_indices.len(), 1); + } + + #[test] + fn test_retain_unknown_coverage_alongside_nonempty_sibling() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1), Fragment::new(2)]; + + let mut indices = vec![ + create_test_index("idx", 1, 1, None, false), // Coverage never measured + create_test_index("idx", 1, 2, Some(RoaringBitmap::from_iter([2])), false), + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // The unmeasured segment must survive its non-empty sibling: its bitmap + // is missing because migration could not open the index, and deleting + // the segment would take the only record of it with it. + assert_eq!(indices.len(), 2); + assert!(indices.iter().any(|idx| idx.fragment_bitmap.is_none())); + } + + #[test] + fn test_retain_multiple_empty_scalar_indices_keeps_oldest() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_test_index("idx", 1, 3, Some(RoaringBitmap::new()), false), + create_test_index("idx", 1, 1, Some(RoaringBitmap::new()), false), // Oldest + create_test_index("idx", 1, 2, Some(RoaringBitmap::new()), false), + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Should keep only the oldest (dataset_version = 1) + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].dataset_version, 1); + } + + #[test] + fn test_retain_multiple_empty_vector_indices_keeps_oldest() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_test_index("vec_idx", 1, 1, Some(RoaringBitmap::new()), true), + create_test_index("vec_idx", 1, 2, Some(RoaringBitmap::new()), true), + create_test_index("vec_idx", 1, 3, Some(RoaringBitmap::new()), true), + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Same as the scalar case: all deltas are empty, so only the oldest + // definition survives. + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].dataset_version, 1); + } + + #[test] + fn test_retain_mixed_empty_nonempty_keeps_nonempty() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_test_index("idx", 1, 1, Some(RoaringBitmap::new()), false), // Empty + create_test_index("idx", 1, 2, Some(RoaringBitmap::from_iter([1])), false), // Non-empty + create_test_index("idx", 1, 3, Some(RoaringBitmap::new()), false), // Empty + create_test_index("idx", 1, 4, Some(RoaringBitmap::from_iter([1])), false), // Non-empty + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Should keep only non-empty indices + assert_eq!(indices.len(), 2); + assert!( + indices + .iter() + .all(|idx| idx.dataset_version == 2 || idx.dataset_version == 4) + ); + } + + #[test] + fn test_retain_mixed_empty_nonempty_vector_keeps_nonempty() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_test_index("vec_idx", 1, 1, Some(RoaringBitmap::new()), true), // Empty + create_test_index("vec_idx", 1, 2, Some(RoaringBitmap::from_iter([1])), true), // Non-empty + create_test_index("vec_idx", 1, 3, Some(RoaringBitmap::new()), true), // Empty + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Should keep only non-empty index + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].dataset_version, 2); + } + + #[test] + fn test_retain_fragment_bitmap_with_nonexistent_fragments() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1), Fragment::new(2)]; // Only fragments 1 and 2 exist + + let mut indices = vec![create_test_index( + "idx", + 1, + 1, + Some(RoaringBitmap::from_iter([1, 2, 3, 4])), // References non-existent fragments 3, 4 + false, + )]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Should still keep the index (effective bitmap will be intersection with existing) + assert_eq!(indices.len(), 1); + // Original bitmap should be unchanged + assert_eq!( + indices[0].fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter([1, 2, 3, 4]) + ); + } + + #[test] + fn test_retain_effective_empty_bitmap_single_index() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(5), Fragment::new(6)]; + + // Bitmap references fragments that don't exist, so effective bitmap is empty + let mut scalar_indices = vec![create_test_index( + "scalar_idx", + 1, + 1, + Some(RoaringBitmap::from_iter([1, 2, 3])), + false, + )]; + + let mut vector_indices = vec![create_test_index( + "vector_idx", + 1, + 1, + Some(RoaringBitmap::from_iter([1, 2, 3])), + true, + )]; + + Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); + Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); + + // Both kept: a single index whose column is still in schema is + // retained even when its effective coverage is empty. + assert_eq!(scalar_indices.len(), 1); + assert_eq!(vector_indices.len(), 1); + } + + #[test] + fn test_retain_different_index_names() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_test_index("idx_a", 1, 1, Some(RoaringBitmap::new()), false), + create_test_index("idx_b", 1, 1, Some(RoaringBitmap::new()), true), + create_test_index("idx_c", 1, 1, Some(RoaringBitmap::from_iter([1])), false), + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // All three kept: empty definitions are retained for scalar and + // vector indexes alike. + assert_eq!(indices.len(), 3); + assert!(indices.iter().any(|idx| idx.name == "idx_a")); + assert!(indices.iter().any(|idx| idx.name == "idx_b")); + assert!(indices.iter().any(|idx| idx.name == "idx_c")); + } + + #[test] + fn test_retain_empty_indices_vec() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices: Vec = vec![]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + assert_eq!(indices.len(), 0); + } + + #[test] + fn test_retain_all_indices_removed() { + let schema = create_test_schema(&[1]); + let fragments = vec![Fragment::new(1)]; + + let mut indices = vec![ + create_test_index("vec1", 1, 1, Some(RoaringBitmap::new()), true), + create_test_index("vec2", 1, 1, Some(RoaringBitmap::new()), true), + create_test_index("idx3", 99, 1, Some(RoaringBitmap::from_iter([1])), false), // Bad field + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Only the bad-field index is dropped; the empty vector definitions + // are retained. + assert_eq!(indices.len(), 2); + assert!(!indices.iter().any(|idx| idx.name == "idx3")); + } + + #[test] + fn test_retain_complex_scenario() { + let schema = create_test_schema(&[1, 2]); + let fragments = vec![Fragment::new(1), Fragment::new(2)]; + + let mut indices = vec![ + // System index - should always be kept + create_system_index(FRAG_REUSE_INDEX_NAME, 1), + // Group "idx_a" - all empty scalars, keep oldest + create_test_index("idx_a", 1, 3, Some(RoaringBitmap::new()), false), + create_test_index("idx_a", 1, 1, Some(RoaringBitmap::new()), false), // Oldest + create_test_index("idx_a", 1, 2, Some(RoaringBitmap::new()), false), + // Group "vec_b" - all empty vectors, keep oldest definition + create_test_index("vec_b", 1, 1, Some(RoaringBitmap::new()), true), + create_test_index("vec_b", 1, 2, Some(RoaringBitmap::new()), true), + // Group "idx_c" - mixed empty/non-empty, keep non-empty + create_test_index("idx_c", 2, 1, Some(RoaringBitmap::new()), false), + create_test_index("idx_c", 2, 2, Some(RoaringBitmap::from_iter([1])), false), // Keep + create_test_index("idx_c", 2, 3, Some(RoaringBitmap::from_iter([2])), false), // Keep + // Single non-empty - keep + create_test_index("idx_d", 1, 1, Some(RoaringBitmap::from_iter([1, 2])), false), + // Index with bad field - remove + create_test_index("idx_e", 99, 1, Some(RoaringBitmap::from_iter([1])), false), + ]; + + Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); + + // Expected: frag_reuse, idx_a (oldest), vec_b (oldest), idx_c (2 + // non-empty), idx_d = 6 total + assert_eq!(indices.len(), 6); + + // Verify system index kept + assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); + + // Verify idx_a kept oldest only + let idx_a_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "idx_a").collect(); + assert_eq!(idx_a_indices.len(), 1); + assert_eq!(idx_a_indices[0].dataset_version, 1); + + // Verify vec_b kept oldest definition only + let vec_b_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "vec_b").collect(); + assert_eq!(vec_b_indices.len(), 1); + assert_eq!(vec_b_indices[0].dataset_version, 1); + + // Verify idx_c kept non-empty only + let idx_c_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "idx_c").collect(); + assert_eq!(idx_c_indices.len(), 2); + assert!( + idx_c_indices + .iter() + .all(|idx| idx.dataset_version == 2 || idx.dataset_version == 3) + ); + + // Verify idx_d kept + assert!(indices.iter().any(|idx| idx.name == "idx_d")); + + // Verify idx_e removed (bad field) + assert!(!indices.iter().any(|idx| idx.name == "idx_e")); + } + + #[test] + fn test_handle_rewrite_indices_skips_missing_index() { + // Create an empty indices list + let mut indices = vec![]; + + // Create rewritten_indices referring to a non-existent index + let rewritten_indices = vec![RewrittenIndex { + old_id: Uuid::new_v4(), + new_id: Uuid::new_v4(), + new_index_details: prost_types::Any { + type_url: String::new(), + value: vec![], + }, + new_index_version: 1, + new_index_files: None, + }]; + + // Should succeed (skip missing index) instead of error + let result = Transaction::handle_rewrite_indices(&mut indices, &rewritten_indices, &[]); + assert!(result.is_ok()); + assert!(indices.is_empty()); + } + + #[test] + fn test_prune_overlay_stale_fields_from_indices() { + // Fragment 0 carried an overlay on field 1 committed at v5, and was + // fully compacted into new fragment 7. + let mut old_frag = Fragment::new(0); + old_frag.overlays = vec![overlay_with_field(1, 5)]; + let groups = vec![RewriteGroup { + old_fragments: vec![old_frag], + new_fragments: vec![Fragment::new(7)], + }]; + + // Post-remap state: every index already covers the new fragment (7). + let covering = || Some(RoaringBitmap::from_iter([7u32])); + let mut indices = vec![ + // Stale: covers the overlaid field 1, built (v2) before the overlay. + create_test_index("stale", 1, 2, covering(), false), + // Not stale: covers field 1 but built at the overlay's version (v5); + // `committed_version > dataset_version` is false at equality. + create_test_index("fresh", 1, 5, covering(), false), + // Unrelated: covers field 2, which the overlay never touched. + create_test_index("unrelated", 2, 2, covering(), false), + ]; + + Transaction::prune_overlay_stale_fields_from_indices(&mut indices, &groups); + + assert!( + !indices[0].fragment_bitmap.as_ref().unwrap().contains(7), + "stale index must drop the rewritten fragment from its coverage" + ); + assert!( + indices[1].fragment_bitmap.as_ref().unwrap().contains(7), + "an index built at/after the overlay is not stale" + ); + assert!( + indices[2].fragment_bitmap.as_ref().unwrap().contains(7), + "an index on an un-overlaid field is unaffected" + ); + } + + // Helper functions for retain_relevant_indices tests + fn create_test_index( + name: &str, + field_id: i32, + dataset_version: u64, + fragment_bitmap: Option, + is_vector: bool, + ) -> IndexMetadata { + use prost_types::Any; + use std::sync::Arc; + + let index_details = if is_vector { + Some(Arc::new(Any { + type_url: "type.googleapis.com/lance.index.VectorIndexDetails".to_string(), + value: vec![], + })) + } else { + Some(Arc::new(Any { + type_url: "type.googleapis.com/lance.index.ScalarIndexDetails".to_string(), + value: vec![], + })) + }; + + IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![field_id], + covering_fields: vec![], + name: name.to_string(), + dataset_version, + fragment_bitmap, + index_details, + index_version: 1, + created_at: None, + base_id: None, + files: None, + } + } + + fn create_system_index(name: &str, field_id: i32) -> IndexMetadata { + use prost_types::Any; + use std::sync::Arc; + + IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![field_id], + covering_fields: vec![], + name: name.to_string(), + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2])), + index_details: Some(Arc::new(Any { + type_url: "type.googleapis.com/lance.index.SystemIndexDetails".to_string(), + value: vec![], + })), + index_version: 1, + created_at: None, + base_id: None, + files: None, + } + } + + fn create_test_schema(field_ids: &[i32]) -> Schema { + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema as LanceSchema; + + let fields: Vec = field_ids + .iter() + .map(|id| ArrowField::new(format!("field_{}", id), DataType::Int32, false)) + .collect(); + + let arrow_schema = ArrowSchema::new(fields); + let mut lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // Assign field IDs + for (i, field_id) in field_ids.iter().enumerate() { + lance_schema.mut_field_by_id(i as i32).unwrap().id = *field_id; + } + + lance_schema + } +} diff --git a/rust/lance-table/src/transaction/manifest_build.rs b/rust/lance-table/src/transaction/manifest_build.rs new file mode 100644 index 00000000000..6f216f78c06 --- /dev/null +++ b/rust/lance-table/src/transaction/manifest_build.rs @@ -0,0 +1,3506 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Applying an operation to produce the next manifest. +//! +//! [`Transaction::build_manifest`] is the centre of this module and of the +//! transaction machinery generally: given the current manifest and index list, it +//! decides the new fragment list, the surviving indices and the next row id, then +//! assembles the manifest. Everything else in `super` exists to serve it -- the +//! operation vocabulary it matches on, the index rules it applies, the row version +//! metadata it stamps, the validation that runs before it. + +use crate::feature_flags::{ + FLAG_COVERED_INDEX_METADATA, FLAG_STABLE_ROW_IDS, apply_feature_flags, + ensure_can_read_manifest, ensure_can_write_manifest, inherit_sticky_feature_flags, +}; +use crate::format::overlay::TOMBSTONE_FIELD_ID; +use crate::format::{ + DataFile, DataStorageFormat, Fragment, IndexMetadata, Manifest, ManifestBuildConfig, + overlay::DataOverlayFile, +}; +use crate::io::{ + commit::CommitHandler, + manifest::{read_manifest, read_manifest_indexes}, +}; +use crate::rowids::version::build_version_meta; +use crate::system_index::is_system_index; +use crate::system_index::mem_wal::{ + CompactedSsTable, IndexCatchupProgress, MEM_WAL_INDEX_NAME, load_mem_wal_index_details, + new_mem_wal_index_meta, update_mem_wal_index_compacted_sstables, +}; +use crate::transaction::UpdateMode::{RewriteColumns, RewriteRows}; +use crate::transaction::row_version::resolve_update_version_metadata; +use crate::transaction::update_map::apply_update_map; +use crate::transaction::validate::merge_fragment_physically_rewritten; +use crate::transaction::{ + CoverageIdentity, DataReplacementGroup, LogicalIndexSegments, Operation, ReadVersionState, + RewriteGroup, Transaction, UpdatedFragmentOffsets, +}; +use lance_core::datatypes::{ + LANCE_UNENFORCED_CLUSTERING_KEY_POSITION, LANCE_UNENFORCED_PRIMARY_KEY, + LANCE_UNENFORCED_PRIMARY_KEY_POSITION, +}; +use lance_core::utils::parse::str_is_truthy; +use lance_core::{Error, Result}; +use lance_file::version::ConcreteFileVersion; +use lance_io::object_store::ObjectStore; +use object_store::path::Path; +use roaring::RoaringBitmap; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; +use uuid::Uuid; + +impl Transaction { + pub(super) fn fragments_with_ids<'a, T>( + new_fragments: T, + fragment_id: &'a mut u64, + ) -> impl Iterator + 'a + where + T: IntoIterator + 'a, + { + new_fragments.into_iter().map(move |mut f| { + if f.id == 0 { + f.id = *fragment_id; + *fragment_id += 1; + } + f + }) + } + + fn data_storage_format_from_files( + fragments: &[Fragment], + user_requested: Option, + ) -> Result { + if let Some(file_version) = Fragment::try_infer_version(fragments)? { + // Ensure user-requested matches data files + if let Some(user_requested) = user_requested + && user_requested != file_version + { + return Err(Error::invalid_input(format!( + "User requested data storage version ({}) does not match version in data files ({})", + user_requested, file_version + ))); + } + Ok(DataStorageFormat::new(file_version)) + } else { + // If no files use user-requested or default + Ok(user_requested + .map(DataStorageFormat::new) + .unwrap_or_default()) + } + } + + pub async fn restore_old_manifest( + object_store: &ObjectStore, + commit_handler: &dyn CommitHandler, + base_path: &Path, + version: u64, + config: &ManifestBuildConfig, + tx_path: &str, + current_manifest: &Manifest, + ) -> Result<(Manifest, Vec)> { + let location = commit_handler + .resolve_version_location(base_path, version, &object_store.inner) + .await?; + let mut manifest = read_manifest(object_store, &location.path, location.size).await?; + // This read bypasses Dataset's feature gates. Refuse unsupported target + // manifests before apply_feature_flags can clear their unknown bits and + // republish the referenced files as legacy-compatible. + ensure_can_read_manifest(&manifest)?; + ensure_can_write_manifest(&manifest)?; + manifest.set_timestamp(config.timestamp_nanos); + manifest.transaction_file = Some(tx_path.to_string()); + let indices = read_manifest_indexes(object_store, &location, &manifest).await?; + manifest.max_fragment_id = manifest + .max_fragment_id + .max(current_manifest.max_fragment_id); + // Row ids are a high-water mark like fragment ids: rewinding hands old ids to new rows. + manifest.next_row_id = manifest.next_row_id.max(current_manifest.next_row_id); + // Turning stable row ids off would revert `_rowid` to row addresses, whose + // namespace overlaps the ids this table has already handed out. + if current_manifest.uses_stable_row_ids() && !manifest.uses_stable_row_ids() { + return Err(Error::invalid_input(format!( + "Cannot restore version {version}: stable row ids were enabled \ + after it, and turning them back off would let row addresses \ + collide with ids this table has already used" + ))); + } + inherit_sticky_feature_flags(&mut manifest, current_manifest)?; + Ok((manifest, indices)) + } + + /// Every non-system logical index, mapped to what determines its coverage. + /// + /// A logical index may be backed by several physical segments, so "did this + /// index change" is a question about the whole set. Sorted by UUID so the + /// two sides compare positionally. + pub fn logical_index_segments(indices: &[IndexMetadata]) -> LogicalIndexSegments { + let mut by_name: LogicalIndexSegments = BTreeMap::new(); + for idx in indices.iter().filter(|idx| !is_system_index(idx)) { + by_name + .entry(idx.name.clone()) + .or_default() + .push(CoverageIdentity { + uuid: idx.uuid, + fragment_bitmap: idx.fragment_bitmap.clone(), + }); + } + for segments in by_name.values_mut() { + segments.sort_unstable_by_key(|segment| segment.uuid); + } + by_name + } + + /// Apply MemWAL index-coverage rules once the final index list is known. + /// + /// Coverage records that a base-table index contains the rows a compaction + /// copied in, and the WAL pod retires SSTables against it. + /// + /// It is derived, not reported. An index covering every fragment live at the + /// transaction's read version holds every row compaction had copied in by + /// then, so it is caught up to that version's `compacted_sstables`. That is + /// the only proof available: nothing maps a generation to the fragments its + /// rows landed in, so covering the table as the transaction read it is how + /// an index shows it covered those rows. Fragments appended since are a + /// later gap. + /// + /// Deriving rather than transmitting means no claim can go stale between + /// inspection and commit, the answer survives rebase (`read_version` is + /// fixed for a transaction's life), and any operation can earn coverage -- + /// an ordinary reindex that fully covers no longer has to throw its work + /// away and wait for a repair. + /// + /// Only meaningful once catch-up is required, where a missing entry means + /// "not caught up" and the SSTables stay. A legacy table reads a missing + /// entry as "fully caught up", so this leaves it untouched rather than + /// making the table look more covered than it is. + pub fn apply_mem_wal_index_coverage( + final_indices: &mut [IndexMetadata], + segments_before: &LogicalIndexSegments, + read_version_state: Option>, + new_version: u64, + ) -> Result<()> { + let Some(pos) = final_indices + .iter() + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + else { + // The system index went away with this transaction (MemWAL disable, + // or an overwrite). There is no coverage left to maintain. + return Ok(()); + }; + + let mut details = load_mem_wal_index_details(final_indices[pos].clone())?; + + // Nothing has ever been compacted, so no index can be behind and there + // is no coverage to invalidate. + if details.compacted_sstables.is_empty() && details.index_catchup.is_empty() { + return Ok(()); + } + + let segments_after = Self::logical_index_segments(final_indices); + let catchup_before = std::mem::take(&mut details.index_catchup); + + // Per shard: what this commit records as compacted, and the most the + // read version may credit. Generations compacted after that read landed + // in fragments no index under consideration has seen; the committed + // value caps it in turn, so a read version since rolled back cannot + // retire SSTables no live commit copied in. + let read_details = read_version_state + .map(|state| { + state + .indices + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .cloned() + .map(load_mem_wal_index_details) + .transpose() + }) + .transpose()? + .flatten(); + let shards: Vec<(Uuid, u64, u64)> = details + .compacted_sstables + .iter() + .map(|committed| { + let at_read = read_details + .as_ref() + .and_then(|read| { + read.compacted_sstables + .iter() + .find(|s| s.shard_id == committed.shard_id) + }) + .map_or(0, |s| s.generation); + ( + committed.shard_id, + committed.generation, + at_read.min(committed.generation), + ) + }) + .collect(); + + // Every fragment live when the transaction read the table. An index + // spanning all of them holds every row compacted by then. + let read_fragments: Option = read_version_state.map(|state| { + state + .manifest + .fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect() + }); + + let covers_read_version = |segments: &[CoverageIdentity]| -> bool { + let Some(required) = read_fragments.as_ref() else { + return false; + }; + if required.is_empty() { + // Subset-of-empty is trivially true, so this would credit every + // index on a table with no fragments. Refused because an empty + // fragment list is not only what an emptied table looks like: + // it is also what a manifest written before #8438 looks like, + // where UpdateMemWalState published no fragments at all. On + // such a table the SSTables are the last copy of those rows, + // and crediting coverage would retire them. The cost is that a + // genuinely emptied table keeps its SSTables. + return false; + } + let mut covered = RoaringBitmap::new(); + for segment in segments { + match segment.fragment_bitmap.as_ref() { + Some(bitmap) => covered |= bitmap, + // An unknown bitmap cannot be shown to cover anything. + None => return false, + } + } + required.is_subset(&covered) + }; + + let mut rebuilt: Vec = Vec::new(); + for (name, after) in segments_after.iter() { + // Compared by [`CoverageIdentity`], not segment UUID: an Update + // that touches an indexed field prunes a segment's fragment bitmap + // in place while keeping its UUID, so a UUID-only comparison would + // carry a position forward that the index no longer earns. + let unchanged = segments_before.get(name) == Some(after); + let carried = unchanged + .then(|| catchup_before.iter().find(|e| e.index_name == *name)) + .flatten(); + let proven = covers_read_version(after); + + if carried.is_none() && !proven { + // Changed, and nothing shows the new index covers the read + // version. No entry: a missing one reads as "not caught up". + continue; + } + + let generations = shards + .iter() + .map(|&(shard_id, committed, creditable)| { + let prior = carried + .and_then(|entry| entry.caught_up_generation_for_shard(&shard_id)) + .unwrap_or(0); + let credited = if proven { creditable } else { 0 }; + // Takes the better of what this commit proves and what an + // unchanged index already held, so a commit reading an older + // version does not lower a position it cannot re-prove. The + // clamp is the exception: a position above what this commit + // records as compacted describes rows no live commit copied + // in. + CompactedSsTable::new(shard_id, prior.max(credited).min(committed)) + }) + .collect::>(); + if generations.iter().all(|g| g.generation == 0) { + continue; + } + rebuilt.push(IndexCatchupProgress::new(name.clone(), generations)); + } + rebuilt.sort_by(|a, b| a.index_name.cmp(&b.index_name)); + + let mut before_sorted = catchup_before; + before_sorted.sort_by(|a, b| a.index_name.cmp(&b.index_name)); + if rebuilt == before_sorted { + return Ok(()); + } + + let dropped: Vec<&str> = before_sorted + .iter() + .map(|e| e.index_name.as_str()) + .filter(|name| !rebuilt.iter().any(|kept| kept.index_name == *name)) + .collect(); + if !dropped.is_empty() { + // The first thing to check when SSTables stop becoming trimmable. + log::info!( + "MemWAL index catch-up invalidated at version {new_version} for {dropped:?}: \ + these indices changed and no longer cover the version this commit read" + ); + } + + details.index_catchup = rebuilt; + final_indices[pos] = new_mem_wal_index_meta(new_version, details)?; + Ok(()) + } + + /// Drop coverage for indices a post-`build_manifest` step narrowed. + /// + /// The derivation runs while the manifest is being built, but the index list + /// is not final there: `migrate_indices` can recalculate a segment's + /// fragment bitmap and keep its UUID, so an index can narrow after its + /// position was decided. It reports which ones it touched rather than the + /// caller re-snapshotting every bitmap to find out. Only ever removes. + pub fn withdraw_coverage_invalidated_after_build( + indices: &mut [IndexMetadata], + changed: &[String], + new_version: u64, + ) -> Result<()> { + if changed.is_empty() { + return Ok(()); + } + let Some(pos) = indices + .iter() + .position(|idx| idx.name == MEM_WAL_INDEX_NAME) + else { + return Ok(()); + }; + let mut details = load_mem_wal_index_details(indices[pos].clone())?; + let before = details.index_catchup.len(); + details + .index_catchup + .retain(|entry| !changed.contains(&entry.index_name)); + if details.index_catchup.len() == before { + return Ok(()); + } + log::info!( + "MemWAL index catch-up withdrawn at version {new_version} for {changed:?}: \ + these indices were recalculated after their coverage was derived" + ); + indices[pos] = new_mem_wal_index_meta(new_version, details)?; + Ok(()) + } + + /// Create a new manifest from the current manifest and the transaction. + /// + /// `current_manifest` should only be None if the dataset does not yet exist. + pub fn build_manifest( + &self, + current_manifest: Option<&Manifest>, + current_indices: Vec, + transaction_file_path: &str, + config: &ManifestBuildConfig, + ) -> Result<(Manifest, Vec)> { + self.build_manifest_with_read_version( + current_manifest, + current_indices, + transaction_file_path, + config, + None, + ) + } + + /// [`Self::build_manifest`] with the version this transaction read. + /// + /// Supplied by the commit path, which already materializes that version. + /// `None` where there is none to read -- dataset creation and detached + /// commits -- in which case no index can be shown to cover it and coverage + /// is left as the invalidation rules put it. + pub fn build_manifest_with_read_version( + &self, + current_manifest: Option<&Manifest>, + current_indices: Vec, + transaction_file_path: &str, + config: &ManifestBuildConfig, + read_version_state: Option>, + ) -> Result<(Manifest, Vec)> { + if config.use_stable_row_ids + && config.migration_next_row_id.is_none() + && current_manifest + .map(|m| !m.uses_stable_row_ids()) + .unwrap_or_default() + { + return Err(Error::not_supported_source( + "This dataset was not created with the stable row ids feature. Please run `migrate_to_stable_row_ids` before attempting to use stable row ids".into(), + )); + } + + if config.migration_next_row_id.is_some() && !current_indices.is_empty() { + let names: Vec<&str> = current_indices + .iter() + .map(|idx| idx.name.as_str()) + .collect(); + return Err(Error::invalid_input(format!( + "Cannot migrate to stable row IDs while indexes exist on the dataset. \ + Drop the following indexes first, then re-run the migration, and \ + recreate them afterwards: {}", + names.join(", ") + ))); + } + let mut reference_paths = match current_manifest { + Some(m) => m.base_paths.clone(), + None => HashMap::new(), + }; + + if let Operation::Overwrite { + initial_bases: Some(initial_bases), + .. + } = &self.operation + { + if current_manifest.is_none() { + // CREATE mode: registering base paths + // Base IDs should have been assigned during write operation + // Validate uniqueness and insert them into the manifest + for base_path in initial_bases.iter() { + if reference_paths.contains_key(&base_path.id) { + return Err(Error::invalid_input(format!( + "Duplicate base path ID {} detected. Base path IDs must be unique.", + base_path.id + ))); + } + reference_paths.insert(base_path.id, base_path.clone()); + } + } else { + // OVERWRITE mode with initial_bases should have been rejected by validation + // This branch should never be reached + return Err(Error::invalid_input( + "OVERWRITE mode cannot register new bases. This should have been caught by validation.", + )); + } + } + + // Get the schema and the final fragment list + let schema = match self.operation { + Operation::Overwrite { ref schema, .. } => schema.clone(), + Operation::Merge { ref schema, .. } => schema.clone(), + Operation::Project { ref schema, .. } => schema.clone(), + _ => { + if let Some(current_manifest) = current_manifest { + current_manifest.schema.clone() + } else { + return Err(Error::internal( + "Cannot create a new dataset without a schema".to_string(), + )); + } + } + }; + + // Fragment ids are a high water mark for the whole dataset history: an id + // must never name two different sets of rows, or per-fragment state keyed + // by id (caches, deletion files, row addresses) can be attributed to the + // wrong rows. + let mut fragment_id = current_manifest + .and_then(|m| m.max_fragment_id()) + .map(|id| id + 1) + .unwrap_or(0); + let mut final_fragments = Vec::new(); + let mut final_indices = current_indices; + + // Snapshot taken before the operation rewrites the list, so coverage can + // be compared against what each logical index looked like going in. Only + // tables with a MemWAL index maintain coverage, so every other commit -- + // and the segment clones this costs -- pays nothing. + let mem_wal_segments_before = final_indices + .iter() + .any(|idx| idx.name == MEM_WAL_INDEX_NAME) + .then(|| Self::logical_index_segments(&final_indices)); + + let mut next_row_id = { + // Only use row ids if the feature flag is set already, or this is + // a migration activation that explicitly provides the next_row_id. + match (current_manifest, config.use_stable_row_ids) { + (Some(manifest), _) if manifest.reader_feature_flags & FLAG_STABLE_ROW_IDS != 0 => { + Some(manifest.next_row_id) + } + (None, true) => Some(0), + (_, false) => None, + (Some(_), true) => { + // Migration activation: use the provided next_row_id. + if let Some(migration_nri) = config.migration_next_row_id { + Some(migration_nri) + } else { + return Err(Error::not_supported_source( + "This dataset was not created with the stable row ids feature. Please run `migrate_to_stable_row_ids` before attempting to use stable row ids".into(), + )); + } + } + } + }; + + let maybe_existing_fragments = + current_manifest + .map(|m| m.fragments.as_ref()) + .ok_or_else(|| { + Error::internal(format!( + "No current manifest was provided while building manifest for operation {}", + self.operation.name() + )) + }); + + let new_version = current_manifest.map_or(1, |m| m.version + 1); + + match &self.operation { + Operation::Clone { .. } => { + return Err(Error::internal( + "Clone operation should not enter build_manifest.".to_string(), + )); + } + Operation::Append { fragments } => { + final_fragments.extend(maybe_existing_fragments?.clone()); + let mut new_fragments = + Self::fragments_with_ids(fragments.clone(), &mut fragment_id) + .collect::>(); + if let Some(next_row_id) = &mut next_row_id { + Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; + // Add version metadata for all new fragments + for fragment in new_fragments.iter_mut() { + let version_meta = build_version_meta(fragment, new_version); + fragment.last_updated_at_version_meta = version_meta.clone(); + fragment.created_at_version_meta = version_meta; + } + } + final_fragments.extend(new_fragments); + } + Operation::Delete { + updated_fragments, + deleted_fragment_ids, + .. + } => { + // Remove the deleted fragments + // Hash lookups keep this linear on tables with many fragments. + let deleted_ids: HashSet = deleted_fragment_ids.iter().copied().collect(); + let updated_by_id: HashMap = + updated_fragments.iter().map(|f| (f.id, f)).collect(); + final_fragments.extend(maybe_existing_fragments?.clone()); + final_fragments.retain(|f| !deleted_ids.contains(&f.id)); + final_fragments.iter_mut().for_each(|f| { + if let Some(updated) = updated_by_id.get(&f.id) { + *f = (*updated).clone(); + } + }); + Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) + } + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + fields_modified, + compacted_sstables, + fields_for_preserving_frag_bitmap, + update_mode, + updated_fragment_offsets, + .. + } => { + // Extract existing fragments once for reuse + let existing_fragments = maybe_existing_fragments?; + + // Apply updates to existing fragments + // Hash lookups keep this linear on tables with many fragments. + let removed_ids: HashSet = removed_fragment_ids.iter().copied().collect(); + let mut updated_by_id: HashMap = + HashMap::with_capacity(updated_fragments.len()); + for fragment in updated_fragments { + updated_by_id.entry(fragment.id).or_insert(fragment); + } + let updated_frags: Vec = existing_fragments + .iter() + .filter_map(|f| { + if removed_ids.contains(&f.id) { + return None; + } + if let Some(&updated) = updated_by_id.get(&f.id) { + let mut updated = updated.clone(); + // Carry forward the fragment's current overlays (which + // may include ones added by a concurrent commit). An + // in-place column rewrite then tombstones the overlaid + // fields it rewrote, since the fresh base values + // supersede them. + updated.overlays = f.overlays.clone(); + if matches!(update_mode, Some(RewriteColumns)) { + crate::format::overlay::tombstone_overlay_fields( + &mut updated.overlays, + fields_modified, + ); + } + Some(updated) + } else { + Some(f.clone()) + } + }) + .collect(); + + // Update version metadata for updated fragments if stable row IDs are enabled + // Note: We don't update version metadata for fragments with deletion vectors + // because the version sequences are indexed by physical row position, not logical position. + // Version metadata for deleted rows will be filtered out during scan using the deletion vector. + if next_row_id.is_some() { + // Version metadata will be properly set during compaction when deletions are materialized + } + + final_fragments.extend(updated_frags); + + if next_row_id.is_some() + && matches!(update_mode, Some(RewriteColumns)) + && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets + && !off_map.is_empty() + { + let prev_version = current_manifest.map(|m| m.version).unwrap_or(0); + for fragment in final_fragments.iter_mut() { + let Some(bitmap) = off_map.get(&fragment.id) else { + continue; + }; + // Defense-in-depth: only stamp fragments that were actually + // rewritten. validate_operation enforces this invariant before + // build_manifest is called; this guard catches any path that + // bypasses validation. + if !updated_by_id.contains_key(&fragment.id) { + continue; + } + if bitmap.is_empty() { + continue; + } + // Skip fragments with no existing version metadata: the helper + // would fill unmatched rows with prev_version, fabricating a + // last_updated stamp for rows that never had one. + if fragment.last_updated_at_version_meta.is_none() { + continue; + } + let max_allowed = existing_fragments + .iter() + .find(|f| f.id == fragment.id) + .and_then(|f| f.physical_rows) + .unwrap_or(1 << 24); + if bitmap.len() as usize > max_allowed { + return Err(Error::invalid_input(format!( + "updatedFragmentOffsets cardinality {} exceeds fragment {} limit {}", + bitmap.len(), + fragment.id, + max_allowed + ))); + } + if let Some(max_off) = bitmap.max() + && max_off as usize >= max_allowed + { + return Err(Error::invalid_input(format!( + "updatedFragmentOffsets max offset {} exceeds fragment {} limit {}", + max_off, fragment.id, max_allowed + ))); + } + let offsets: Vec = bitmap.iter().map(|o| o as usize).collect(); + crate::rowids::version::refresh_row_latest_update_meta_for_partial_frag_rewrite_cols( + fragment, + &offsets, + new_version, + prev_version, + )?; + } + } + + // If we updated any fields, remove those fragments from indices covering those fields + Self::prune_updated_fields_from_indices( + &mut final_indices, + updated_fragments, + fields_modified, + ); + + let mut new_fragments = + Self::fragments_with_ids(new_fragments.clone(), &mut fragment_id) + .collect::>(); + + // Assign row IDs to any fragments that don't have them yet + // (e.g., inserted rows from merge_insert operations) + if let Some(next_row_id) = &mut next_row_id { + Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; + } + + if next_row_id.is_some() { + resolve_update_version_metadata( + existing_fragments, + new_fragments.as_mut_slice(), + new_version, + )?; + } + + if config.use_stable_row_ids + && update_mode.is_some() + && *update_mode == Some(RewriteRows) + { + let pure_updated_frag_ids = + Self::collect_pure_rewrite_row_update_frags_ids(&new_fragments)?; + + // collect all the original frag ids that contains the updated rows + let original_fragment_ids: Vec = removed_fragment_ids + .iter() + .chain(updated_fragments.iter().map(|f| &f.id)) + .copied() + .collect(); + + // The original fragments that carried an overlay: their moved rows may have a + // stale index entry (see `register_pure_rewrite_rows_update_frags_in_indices`). + // Reuse the hash lookups built above instead of scanning + // `original_fragment_ids` per fragment. + let original_overlaid_frags: HashMap = existing_fragments + .iter() + .filter(|f| { + (removed_ids.contains(&f.id) || updated_by_id.contains_key(&f.id)) + && !f.overlays.is_empty() + }) + .map(|f| (f.id as u32, f)) + .collect(); + + Self::register_pure_rewrite_rows_update_frags_in_indices( + &mut final_indices, + &pure_updated_frag_ids, + &original_fragment_ids, + fields_for_preserving_frag_bitmap, + &original_overlaid_frags, + &schema, + )?; + } + + if let Some(next_row_id) = &mut next_row_id { + Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; + // Note: Version metadata is already set above (lines 1627-1755) + // for Update operations, preserving created_at from original fragments. + // Don't overwrite it here. + } + // Identify fragments that were updated or newly created in this update + let mut target_ids: HashSet = HashSet::new(); + target_ids.extend(new_fragments.iter().map(|f| f.id)); + final_fragments.extend(new_fragments); + Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments); + + if !compacted_sstables.is_empty() { + update_mem_wal_index_compacted_sstables( + &mut final_indices, + new_version, + compacted_sstables.clone(), + )?; + } + } + Operation::Overwrite { fragments, .. } => { + // Every fragment in an overwrite is newly written, so all of them + // take fresh ids regardless of the id they arrive with. Fragments + // carried over from the dataset being replaced are rejected by + // `validate_operation`, which is what makes ignoring the incoming + // id safe here. + let mut new_fragments = fragments.clone(); + for fragment in new_fragments.iter_mut() { + fragment.id = fragment_id; + fragment_id += 1; + } + if let Some(next_row_id) = &mut next_row_id { + Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; + // Add version metadata for all new fragments + for fragment in new_fragments.iter_mut() { + let version_meta = build_version_meta(fragment, new_version); + fragment.last_updated_at_version_meta = version_meta.clone(); + fragment.created_at_version_meta = version_meta; + } + } + final_fragments.extend(new_fragments); + final_indices = Vec::new(); + } + Operation::Rewrite { + groups, + rewritten_indices, + frag_reuse_index, + } => { + final_fragments.extend(maybe_existing_fragments?.clone()); + let current_version = current_manifest.map(|m| m.version).unwrap_or_default(); + Self::handle_rewrite_fragments( + &mut final_fragments, + groups, + &mut fragment_id, + current_version, + next_row_id.as_ref(), + )?; + + if next_row_id.is_some() { + // We can re-use indices, but need to rewrite the fragment bitmaps + debug_assert!(rewritten_indices.is_empty()); + for index in final_indices.iter_mut() { + let results_are_row_addrs = index.results_are_row_addrs(); + if let Some(fragment_bitmap) = &mut index.fragment_bitmap { + *fragment_bitmap = if results_are_row_addrs { + // Stable row ids survive a rewrite, so a row-id-domain index + // can simply follow its data to the new fragments. An + // address-domain index cannot: its stored addresses point into + // the fragments the rewrite dropped. Claiming coverage of the + // new fragments would make it answer queries with addresses + // that no longer resolve, so drop the rewritten fragments from + // its coverage instead and let the scanner fall back to a full + // scan for them. + Self::drop_rewritten_fragments(fragment_bitmap, groups) + } else { + Self::recalculate_fragment_bitmap(fragment_bitmap, groups)? + }; + } + } + } else { + Self::handle_rewrite_indices(&mut final_indices, rewritten_indices, groups)?; + } + + // A full compaction materializes a fragment's overlays into fresh + // base data. Any index older than one of those overlays was built on + // the pre-overlay values, so drop the rewritten fragment from its + // coverage to keep it from serving stale values. + Self::prune_overlay_stale_fields_from_indices(&mut final_indices, groups); + + if let Some(frag_reuse_index) = frag_reuse_index { + final_indices.retain(|idx| idx.name != frag_reuse_index.name); + final_indices.push(frag_reuse_index.clone()); + } + } + Operation::CreateIndex { + new_indices, + removed_indices, + .. + } => { + final_fragments.extend(maybe_existing_fragments?.clone()); + let removed_uuids = removed_indices + .iter() + .map(|old_index| old_index.uuid) + .collect::>(); + let new_uuids = new_indices + .iter() + .map(|new_index| new_index.uuid) + .collect::>(); + final_indices.retain(|existing_index| { + !removed_uuids.contains(&existing_index.uuid) + && !new_uuids.contains(&existing_index.uuid) + }); + for new_index in new_indices { + new_index.validate_covering_fields()?; + } + final_indices.extend(new_indices.clone()); + } + Operation::ReserveFragments { .. } | Operation::UpdateConfig { .. } => { + final_fragments.extend(maybe_existing_fragments?.clone()); + } + Operation::Merge { fragments, .. } => { + let existing_fragments = maybe_existing_fragments?; + let mut merged_fragments = fragments.clone(); + if next_row_id.is_some() { + let prev_by_id: HashMap = + existing_fragments.iter().map(|f| (f.id, f)).collect(); + for fragment in merged_fragments.iter_mut() { + match prev_by_id.get(&fragment.id) { + Some(prev) => { + if merge_fragment_physically_rewritten(prev, fragment) { + crate::rowids::version::refresh_row_latest_update_meta_for_full_frag_rewrite_cols( + fragment, + new_version, + )?; + } + } + None => { + // Brand-new fragment ID not present in the previous manifest. + // Set both last_updated and created version meta, consistent + // with Append/Overwrite for genuinely new fragments. + crate::rowids::version::refresh_row_latest_update_meta_for_full_frag_rewrite_cols( + fragment, + new_version, + )?; + fragment.created_at_version_meta = + fragment.last_updated_at_version_meta.clone(); + } + } + } + } + final_fragments.extend(merged_fragments); + + // A Merge can rewrite a column's data file in place; the field stays + // in the schema, so the index is retained -- prune its now-stale + // entries for the rewritten fragments. + Self::prune_merge_rewritten_fields_from_indices( + &mut final_indices, + existing_fragments, + fragments, + ); + + // Some fields that have indices may have been removed, so we should + // remove those indices as well. + Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) + } + Operation::Project { .. } => { + final_fragments.extend(maybe_existing_fragments?.clone()); + + // We might have removed all fields for certain data files, so + // we should remove the data files that are no longer relevant. + let remaining_field_ids = schema + .fields_pre_order() + .map(|f| f.id) + .collect::>(); + for fragment in final_fragments.iter_mut() { + fragment.files.retain(|file| { + file.fields + .iter() + .any(|field_id| remaining_field_ids.contains(field_id)) + }); + } + + // Some fields that have indices may have been removed, so we should + // remove those indices as well. + Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) + } + Operation::Restore { .. } => { + unreachable!() + } + Operation::DataReplacement { replacements } => { + log::warn!( + "Building manifest with DataReplacement operation. This operation is not stable yet, please use with caution." + ); + + let (old_fragment_ids, new_datafiles): (Vec<&u64>, Vec<&DataFile>) = replacements + .iter() + .map(|DataReplacementGroup(fragment_id, new_file)| (fragment_id, new_file)) + .unzip(); + + // 1. make sure the new files all have the same fields / or empty + // NOTE: arguably this requirement could be relaxed in the future + // for the sake of simplicity, we require the new files to have the same fields + if new_datafiles + .iter() + .map(|f| f.fields.clone()) + .collect::>() + .len() + > 1 + { + let field_info = new_datafiles + .iter() + .enumerate() + .map(|(id, f)| (id, f.fields.clone())) + .fold("".to_string(), |acc, (id, fields)| { + format!("{}File {}: {:?}\n", acc, id, fields) + }); + + return Err(Error::invalid_input(format!( + "All new data files must have the same fields, but found different fields:\n{field_info}" + ))); + } + + let existing_fragments = maybe_existing_fragments?; + + // Collect replaced field IDs before consuming new_datafiles + let replaced_fields: Vec = new_datafiles + .first() + .map(|f| { + f.fields + .iter() + .filter(|&&id| id >= 0) + .map(|&id| id as u32) + .collect() + }) + .unwrap_or_default(); + + // 2. check that the fragments being modified have isomorphic layouts along the columns being replaced + // 3. add modified fragments to final_fragments + for (frag_id, new_file) in old_fragment_ids.iter().zip(new_datafiles) { + let frag = existing_fragments + .iter() + .find(|f| f.id == **frag_id) + .ok_or_else(|| { + Error::invalid_input( + "Fragment being replaced not found in existing fragments", + ) + })?; + let mut new_frag = frag.clone(); + + // TODO(rmeng): check new file and fragment are the same length + + let mut columns_covered = HashSet::new(); + // Set when an existing file covers exactly the replaced + // fields, so the whole file swaps rather than part of it. + let mut replaced_in_place = false; + for file in &mut new_frag.files { + if file.fields == new_file.fields + && file.file_major_version == new_file.file_major_version + && file.file_minor_version == new_file.file_minor_version + { + // assign the new file path / size / base to the fragment + file.path = new_file.path.clone(); + file.file_size_bytes = new_file.file_size_bytes.clone(); + file.base_id = new_file.base_id; + replaced_in_place = true; + } + columns_covered.extend(file.fields.iter()); + } + // Reject a file whose version does not decode before any + // arm publishes it. + new_file.file_version()?; + + // SPECIAL CASE: if the column(s) being replaced are not covered by the fragment + // Then it means it's a all-NULL column that is being replaced with real data + // just add it to the final fragments. Push the DataFile as + // given so every field (including base_id) is preserved. + if columns_covered.is_disjoint(&new_file.fields.iter().collect()) { + new_frag.files.push(new_file.clone()); + } else if !replaced_in_place + && new_file.fields.iter().all(|field| { + let mut covering = new_frag + .files + .iter() + .filter(|file| file.fields.contains(field)) + .peekable(); + // Covered by something, and by nothing we cannot + // tombstone. A field no file covers leaves the + // mixed layout the error below reports. + covering.peek().is_some() + && covering.all(|file| { + file.file_version() + .is_ok_and(|version| version != ConcreteFileVersion::V1) + }) + }) + { + // Tombstone the replaced fields where they live and + // append the new file to answer for them, the idiom + // `update_columns` uses. Compaction decides that layout, + // so the fields may sit in one wider file or span + // several. + // + // Legacy V1 is excluded: its reader derives the page table + // offset from the first field in the metadata, so + // tombstoning one field leaves its siblings decoding from + // the wrong pages. A field a V1 file covers keeps + // exact-match replacement. + for file in &mut new_frag.files { + // Same reason as the guard above. + if file.file_version()? == ConcreteFileVersion::V1 { + continue; + } + file.fields = file + .fields + .iter() + .map(|field| { + if new_file.fields.contains(field) { + TOMBSTONE_FIELD_ID + } else { + *field + } + }) + .collect::>() + .into(); + } + // Every data file must share at least one field with + // the dataset schema: a file kept alive only by + // tombstones or by ids the schema no longer defines is + // unreachable to readers, uncollectable by cleanup, + // and reported corrupt by validate(). + let live_ids = schema + .fields_pre_order() + .map(|field| field.id) + .collect::>(); + new_frag + .files + .retain(|file| file.fields.iter().any(|f| live_ids.contains(f))); + new_frag.files.push(new_file.clone()); + } + + // Nothing changed in the current fragment, which is not expected -- error out + if &new_frag == frag { + return Err(Error::invalid_input( + "Expected to modify the fragment but no changes were made. This means the new data files does not align with any exiting datafiles. Please check if the schema of the new data files matches the schema of the old data files including the file major and minor versions", + )); + } + + // New base values supersede any overlay still shadowing + // them, so tombstone the overlaid fields. An overlay + // committed after this transaction's snapshot is the newer + // value though -- the conflict resolver rebases these two + // precisely because the overlay wins -- so it stays, and + // being newer it stays last, preserving the ordering. + let (mut superseded, newer): (Vec<_>, Vec<_>) = new_frag + .overlays + .drain(..) + .partition(|overlay| overlay.committed_version <= self.read_version); + crate::format::overlay::tombstone_overlay_fields( + &mut superseded, + &replaced_fields, + ); + superseded.extend(newer); + new_frag.overlays = superseded; + + final_fragments.push(new_frag); + } + + let fragments_changed = old_fragment_ids + .iter() + .cloned() + .cloned() + .collect::>(); + + // 4. push fragments that didn't change back to final_fragments + let unmodified_fragments = existing_fragments + .iter() + .filter(|f| !fragments_changed.contains(&f.id)) + .cloned() + .collect::>(); + + final_fragments.extend(unmodified_fragments); + + // 5. Invalidate index bitmaps for replaced fields + let modified_fragments: Vec = final_fragments + .iter() + .filter(|f| fragments_changed.contains(&f.id)) + .cloned() + .collect(); + + // A replacement changes what its rows read as, so stamp them + // updated. Without this, get_updated_rows never reports them and + // an incremental consumer skips them for good. + if next_row_id.is_some() { + let new_version = current_manifest.map_or(1, |m| m.version + 1); + for fragment in final_fragments + .iter_mut() + .filter(|f| fragments_changed.contains(&f.id)) + { + crate::rowids::version::refresh_row_latest_update_meta_for_full_frag_rewrite_cols( + fragment, + new_version, + )?; + } + } + + Self::prune_updated_fields_from_indices( + &mut final_indices, + &modified_fragments, + &replaced_fields, + ); + } + Operation::DataOverlay { groups } => { + // Stamp each overlay with the version this commit is producing. + // build_manifest re-runs on every retry with an updated + // current_manifest, so this is naturally re-stamped on retry. + let new_version = current_manifest.map_or(1, |m| m.version + 1); + + let existing_fragments = maybe_existing_fragments?; + // Multiple groups may target the same fragment; merge them in + // order rather than letting a HashMap collapse drop all but the + // last group's overlays. + let mut overlays_by_fragment: HashMap> = HashMap::new(); + for group in groups { + overlays_by_fragment + .entry(group.fragment_id) + .or_default() + .extend(group.overlays.iter()); + } + + // Every group must target an existing fragment. Build a set of + // existing ids once so this is O(groups + fragments) rather than + // O(groups * fragments). + let existing_fragment_ids: HashSet = + existing_fragments.iter().map(|f| f.id).collect(); + for fragment_id in overlays_by_fragment.keys() { + if !existing_fragment_ids.contains(fragment_id) { + return Err(Error::invalid_input(format!( + "DataOverlay targets fragment {fragment_id}, which does not exist" + ))); + } + } + + for fragment in existing_fragments { + let mut fragment = fragment.clone(); + if let Some(new_overlays) = overlays_by_fragment.get(&fragment.id) { + // Appended (not replaced) so concurrently-written overlays + // survive; later entries are newer. + fragment + .overlays + .extend(new_overlays.iter().map(|&overlay| { + let mut overlay = overlay.clone(); + overlay.committed_version = new_version; + overlay + })); + } + final_fragments.push(fragment); + } + } + Operation::UpdateMemWalState { + compacted_sstables, .. + } => { + // Updates the MemWAL index only; the fragments are unchanged. + final_fragments.extend(maybe_existing_fragments?.clone()); + update_mem_wal_index_compacted_sstables( + &mut final_indices, + new_version, + compacted_sstables.clone(), + )?; + } + Operation::UpdateBases { .. } => { + // UpdateBases operation doesn't modify fragments or indices + // Base paths are handled in the manifest creation section below + final_fragments.extend(maybe_existing_fragments?.clone()); + } + }; + + // If a fragment was reserved then it may not belong at the end of the fragments list. + final_fragments.sort_by_key(|frag| frag.id); + + // Clean up data files that only contain tombstoned fields + Self::remove_tombstoned_data_files(&mut final_fragments); + + // Enforce the newest-last overlay ordering invariant at the write + // boundary. Load normalizes with a sort; this rejects any commit path + // that assembled a fragment's overlays out of order. + for fragment in &final_fragments { + if !fragment.overlays.is_empty() { + crate::format::overlay::verify_overlays_newest_last(&fragment.overlays)?; + } + } + + let user_requested_version = match (&config.storage_format, config.use_legacy_format) { + (Some(storage_format), _) => Some(storage_format.lance_file_format()), + (None, Some(true)) => Some(ConcreteFileVersion::V1), + (None, Some(false)) => Some(ConcreteFileVersion::V2_0), + (None, None) => None, + }; + + // Applied once the final index list is known, so it sees exactly the + // indices this commit publishes rather than what any one operation arm + // intended. + if let Some(segments_before) = mem_wal_segments_before.as_ref() { + Self::apply_mem_wal_index_coverage( + &mut final_indices, + segments_before, + read_version_state, + new_version, + )?; + } + + let mut manifest = if let Some(current_manifest) = current_manifest { + // OVERWRITE with initial_bases on existing dataset is not allowed (caught by validation) + // So we always use new_from_previous which preserves base_paths + let mut prev_manifest = + Manifest::new_from_previous(current_manifest, schema, Arc::new(final_fragments)); + + if let (Some(user_requested_version), Operation::Overwrite { .. }) = + (user_requested_version, &self.operation) + { + // If this is an overwrite operation and the user has requested a specific version + // then overwrite with that version. Otherwise, if the user didn't request a specific + // version, then overwrite with whatever version we had before. + prev_manifest.data_storage_format = DataStorageFormat::new(user_requested_version); + } + + prev_manifest + } else { + let data_storage_format = + Self::data_storage_format_from_files(&final_fragments, user_requested_version)?; + Manifest::new( + schema, + Arc::new(final_fragments), + data_storage_format, + reference_paths, + ) + }; + + manifest.tag.clone_from(&self.tag); + + if config.auto_set_feature_flags { + // Internal operations (e.g. CreateIndex) build with the default config, + // which has use_stable_row_ids = false. Without inheriting from the previous + // manifest, apply_feature_flags would clear FLAG_STABLE_ROW_IDS. + let inherited = current_manifest + .map(|m| m.uses_stable_row_ids()) + .unwrap_or(false); + let use_stable_row_ids = config.use_stable_row_ids || inherited; + apply_feature_flags( + &mut manifest, + use_stable_row_ids, + config.disable_transaction_file, + )?; + } + // Set after apply_feature_flags, which resets both flag words -- and a + // `Manifest` only points at its index section, so the flag cannot be + // derived there. + // + // Derived fresh from `final_indices` on every commit, never inherited. + // Every manifest this reaches starts without the covering bit, so there + // is no stale bit to clear. Dropping the last covering index lifts the + // fence by simply not setting it again. Inheriting it from the previous + // manifest instead would make the fence permanent. + // + // Both words: a reader that selects a vector index by membership of + // `fields` would answer a query on a merely-carried column with an index + // keyed on another one, and a writer that treats every entry of `fields` + // as keyed would mismaintain it. + if final_indices + .iter() + .any(|index| !index.covering_fields.is_empty()) + { + manifest.reader_feature_flags |= FLAG_COVERED_INDEX_METADATA; + manifest.writer_feature_flags |= FLAG_COVERED_INDEX_METADATA; + } + + if let Some(current_manifest) = current_manifest { + inherit_sticky_feature_flags(&mut manifest, current_manifest)?; + } + + manifest.set_timestamp(config.timestamp_nanos); + + manifest.update_max_fragment_id(); + + match &self.operation { + Operation::Overwrite { + config_upsert_values: Some(tm), + .. + } => { + manifest.config_mut().extend(tm.clone()); + } + Operation::UpdateConfig { + config_updates, + table_metadata_updates, + schema_metadata_updates, + field_metadata_updates, + } => { + if let Some(config_updates) = config_updates { + let mut config = manifest.config.clone(); + apply_update_map(&mut config, config_updates); + manifest.config = config; + } + if let Some(table_metadata_updates) = table_metadata_updates { + let mut table_metadata = manifest.table_metadata.clone(); + apply_update_map(&mut table_metadata, table_metadata_updates); + manifest.table_metadata = table_metadata; + } + if let Some(schema_metadata_updates) = schema_metadata_updates { + let mut schema_metadata = manifest.schema.metadata.clone(); + apply_update_map(&mut schema_metadata, schema_metadata_updates); + manifest.schema.metadata = schema_metadata; + } + // The unenforced primary and clustering keys are reserved + // schema properties: each is immutable once set, and its + // reserved metadata keys cannot be written with an invalid + // value. Capture the prior keys, and whether this transaction + // writes a reserved key, before applying the updates so + // violations can be rejected below. This runs on every apply, + // including conflict-rebase, so it also rejects the + // concurrent-writer race. + let primary_key_before: Vec = manifest + .schema + .unenforced_primary_key() + .iter() + .map(|field| field.id) + .collect(); + let writes_primary_key = field_metadata_updates.values().any(|update| { + update.update_entries.iter().any(|entry| { + entry.key == LANCE_UNENFORCED_PRIMARY_KEY + || entry.key == LANCE_UNENFORCED_PRIMARY_KEY_POSITION + }) + }); + let clustering_key_before: Vec = manifest + .schema + .unenforced_clustering_key() + .iter() + .map(|field| field.id) + .collect(); + let writes_clustering_key = field_metadata_updates.values().any(|update| { + update + .update_entries + .iter() + .any(|entry| entry.key == LANCE_UNENFORCED_CLUSTERING_KEY_POSITION) + }); + for (field_id, field_metadata_update) in field_metadata_updates { + if let Some(field) = manifest.schema.field_by_id_mut(*field_id) { + apply_update_map(&mut field.metadata, field_metadata_update); + // Also set unenforced primary key based on updated field metadata. + field.unenforced_primary_key_position = field + .metadata + .get(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) + .and_then(|s| s.parse::().ok()) + .or_else(|| { + field + .metadata + .get(LANCE_UNENFORCED_PRIMARY_KEY) + .filter(|s| str_is_truthy(s)) + .map(|_| 0) + }); + // Also set unenforced clustering key based on updated + // field metadata. + field.unenforced_clustering_key_position = field + .metadata + .get(LANCE_UNENFORCED_CLUSTERING_KEY_POSITION) + .and_then(|s| s.parse::().ok()); + } else { + return Err(Error::invalid_input_source( + format!("Field with id {} does not exist", field_id).into(), + )); + } + } + let primary_key_after: Vec = manifest + .schema + .unenforced_primary_key() + .iter() + .map(|field| field.id) + .collect(); + if !primary_key_before.is_empty() { + // The primary key is already set: reject any change to it, + // and any write that touches a reserved primary key. + if writes_primary_key || primary_key_after != primary_key_before { + return Err(Error::invalid_input( + "the unenforced primary key is a reserved key and cannot be changed once set", + )); + } + } else if writes_primary_key && primary_key_after.is_empty() { + // A reserved primary key was written but did not install a + // valid primary key (e.g. a non-marker flag value or a + // non-numeric position). + return Err(Error::invalid_input( + "the unenforced primary key is a reserved key and cannot be set to an invalid value", + )); + } + if writes_primary_key { + // Installing by field metadata skips the Arrow-schema + // conversion that would otherwise validate the key. + manifest.schema.verify_primary_key()?; + } + let clustering_key_after: Vec = manifest + .schema + .unenforced_clustering_key() + .iter() + .map(|field| field.id) + .collect(); + if !clustering_key_before.is_empty() { + // The clustering key is already set: reject any change to + // it, and any write that touches the reserved key. + if writes_clustering_key || clustering_key_after != clustering_key_before { + return Err(Error::invalid_input( + "the unenforced clustering key is a reserved key and cannot be changed once set", + )); + } + } else if writes_clustering_key && clustering_key_after.is_empty() { + // The reserved clustering key was written but did not + // install a valid clustering key (e.g. a non-numeric + // position value). + return Err(Error::invalid_input( + "the unenforced clustering key is a reserved key and cannot be set to an invalid value", + )); + } + } + _ => {} + } + + // Handle UpdateBases operation to update manifest base_paths + if let Operation::UpdateBases { new_bases } = &self.operation { + // Validate and add new base paths to the manifest + for new_base in new_bases { + // Check for conflicts with existing base paths + if let Some(existing_base) = manifest + .base_paths + .values() + .find(|bp| bp.name == new_base.name || bp.path == new_base.path) + { + return Err(Error::invalid_input(format!( + "Conflict detected: Base path with name '{:?}' or path '{}' already exists. Existing: name='{:?}', path='{}'", + new_base.name, new_base.path, existing_base.name, existing_base.path + ))); + } + + // Assign a new ID if not already assigned + let mut base_to_add = new_base.clone(); + if base_to_add.id == 0 { + let next_id = manifest + .base_paths + .keys() + .max() + .map(|&id| id + 1) + .unwrap_or(1); + base_to_add.id = next_id; + } + + manifest.base_paths.insert(base_to_add.id, base_to_add); + } + } + + if let Operation::ReserveFragments { num_fragments } = self.operation { + manifest.max_fragment_id = Some(manifest.max_fragment_id.unwrap_or(0) + num_fragments); + } + + manifest.transaction_file = Some(transaction_file_path.to_string()); + + if let Some(next_row_id) = next_row_id { + manifest.next_row_id = next_row_id; + } + + Ok((manifest, final_indices)) + } + + /// Remove data files that only contain tombstoned fields (-2) + /// These files no longer contain any live data and can be safely dropped + fn remove_tombstoned_data_files(fragments: &mut [Fragment]) { + for fragment in fragments { + fragment.files.retain(|file| { + // Keep file if it has at least one non-tombstoned field + file.fields.iter().any(|&field_id| field_id != -2) + }); + } + } + /// Coverage of an index that a rewrite invalidates: the rewritten fragments are + /// removed and the fragments they became are *not* added. + fn drop_rewritten_fragments(old: &RoaringBitmap, groups: &[RewriteGroup]) -> RoaringBitmap { + let mut new_bitmap = old.clone(); + for group in groups { + for old_fragment in &group.old_fragments { + new_bitmap.remove(old_fragment.id as u32); + } + } + new_bitmap + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::overlay::OverlayCoverage; + use crate::format::pb; + use crate::format::{RowDatasetVersionMeta, RowDatasetVersionSequence, RowIdMeta}; + use crate::rowids::{RowIdSequence, write_row_ids}; + use crate::transaction::test_support::{ + default_build_config, make_stable_row_id_manifest, overlay_with_field, + sample_index_metadata, sample_manifest, + }; + use crate::transaction::{DataOverlayGroup, UpdateMode, validate_operation}; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::Schema as LanceSchema; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + use lance_io::utils::CachedFileSize; + use std::collections::HashMap; + use std::sync::Arc; + + fn sample_manifest_with_fragments(ids: std::ops::Range) -> Manifest { + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(ids.map(Fragment::new).collect()), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ) + } + + #[test] + fn test_create_index_build_manifest_keeps_unremoved_same_name_indices() { + let manifest = sample_manifest(); + let first_index = sample_index_metadata("vector_idx"); + let second_index = sample_index_metadata("vector_idx"); + let third_index = sample_index_metadata("vector_idx"); + + let transaction = Transaction::new( + manifest.version, + Operation::CreateIndex { + new_indices: vec![third_index.clone()], + removed_indices: vec![second_index.clone()], + }, + None, + ); + + let (_, final_indices) = transaction + .build_manifest( + Some(&manifest), + vec![first_index.clone(), second_index.clone()], + "txn", + &default_build_config(), + ) + .unwrap(); + + assert_eq!(final_indices.len(), 2); + assert!(final_indices.iter().any(|idx| idx.uuid == first_index.uuid)); + assert!(final_indices.iter().any(|idx| idx.uuid == third_index.uuid)); + assert!( + !final_indices + .iter() + .any(|idx| idx.uuid == second_index.uuid) + ); + } + + #[test] + fn test_create_index_build_manifest_deduplicates_relisted_indices_by_uuid() { + let manifest = sample_manifest(); + let first_index = sample_index_metadata("vector_idx"); + let second_index = sample_index_metadata("vector_idx"); + let third_index = sample_index_metadata("vector_idx"); + + let transaction = Transaction::new( + manifest.version, + Operation::CreateIndex { + new_indices: vec![first_index.clone(), third_index.clone()], + removed_indices: vec![second_index.clone()], + }, + None, + ); + + let (_, final_indices) = transaction + .build_manifest( + Some(&manifest), + vec![first_index.clone(), second_index.clone()], + "txn", + &default_build_config(), + ) + .unwrap(); + + assert_eq!(final_indices.len(), 2); + assert_eq!( + final_indices + .iter() + .filter(|idx| idx.uuid == first_index.uuid) + .count(), + 1 + ); + assert!(final_indices.iter().any(|idx| idx.uuid == third_index.uuid)); + assert!( + !final_indices + .iter() + .any(|idx| idx.uuid == second_index.uuid) + ); + } + + #[test] + fn test_update_build_manifest_replaces_and_removes_fragments() { + let manifest = sample_manifest_with_fragments(0..5); + + let mut updated2 = Fragment::new(2); + updated2.physical_rows = Some(42); + let mut updated4 = Fragment::new(4); + updated4.physical_rows = Some(43); + + let transaction = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![1], + // Fragment 99 does not exist in the dataset; it must be ignored, + // not appended. + updated_fragments: vec![updated2, updated4, Fragment::new(99)], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ); + + let (new_manifest, _) = transaction + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let ids: Vec = new_manifest.fragments.iter().map(|f| f.id).collect(); + assert_eq!(ids, vec![0, 2, 3, 4]); + let rows: Vec> = new_manifest + .fragments + .iter() + .map(|f| f.physical_rows) + .collect(); + assert_eq!(rows, vec![None, Some(42), None, Some(43)]); + } + + #[test] + fn test_delete_build_manifest_replaces_and_removes_fragments() { + let manifest = sample_manifest_with_fragments(0..5); + + let mut updated2 = Fragment::new(2); + updated2.physical_rows = Some(42); + + let transaction = Transaction::new( + manifest.version, + Operation::Delete { + updated_fragments: vec![updated2], + deleted_fragment_ids: vec![1, 3], + predicate: "id > 0".to_string(), + }, + None, + ); + + let (new_manifest, _) = transaction + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let ids: Vec = new_manifest.fragments.iter().map(|f| f.id).collect(); + assert_eq!(ids, vec![0, 2, 4]); + let rows: Vec> = new_manifest + .fragments + .iter() + .map(|f| f.physical_rows) + .collect(); + assert_eq!(rows, vec![None, Some(42), None]); + } + + #[test] + fn test_remove_tombstoned_data_files() { + // Create a fragment with mixed data files: some normal, some fully tombstoned + let mut fragment = Fragment::new(1); + + // Add a normal data file with valid field IDs + fragment.files.push(DataFile { + path: "normal.lance".to_string(), + fields: Arc::from([1, 2, 3]), + column_indices: Arc::from([]), + file_major_version: 2, + file_minor_version: 0, + file_size_bytes: CachedFileSize::new(1000), + base_id: None, + }); + + // Add a data file with all fields tombstoned + fragment.files.push(DataFile { + path: "all_tombstoned.lance".to_string(), + fields: Arc::from([-2, -2, -2]), + column_indices: Arc::from([]), + file_major_version: 2, + file_minor_version: 0, + file_size_bytes: CachedFileSize::new(500), + base_id: None, + }); + + // Add a data file with mixed tombstoned and valid fields + fragment.files.push(DataFile { + path: "mixed.lance".to_string(), + fields: Arc::from([4, -2, 5]), + column_indices: Arc::from([]), + file_major_version: 2, + file_minor_version: 0, + file_size_bytes: CachedFileSize::new(750), + base_id: None, + }); + + // Add another fully tombstoned file + fragment.files.push(DataFile { + path: "another_tombstoned.lance".to_string(), + fields: Arc::from([-2_i32]), + column_indices: Arc::from([]), + file_major_version: 2, + file_minor_version: 0, + file_size_bytes: CachedFileSize::new(250), + base_id: None, + }); + + let mut fragments = vec![fragment]; + + // Apply the cleanup + Transaction::remove_tombstoned_data_files(&mut fragments); + + // Should have removed the two fully tombstoned files + assert_eq!(fragments[0].files.len(), 2); + assert_eq!(fragments[0].files[0].path, "normal.lance"); + assert_eq!(fragments[0].files[1].path, "mixed.lance"); + } + + /// When a fragment has no existing last_updated_at_version_meta (None), a + /// partial RewriteColumns refresh must leave it as None rather than fabricating + /// prev_version for unmatched rows. + #[test] + fn test_partial_rewrite_skips_fragment_with_no_version_meta() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); + + // Simulate a RewriteColumns update that matched offsets 1 and 3 + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter([1u32, 3]))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, + ); + + let (out, _) = tx + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + assert!( + out.fragments[0].last_updated_at_version_meta.is_none(), + "fragment with no prior version metadata must not have fabricated prev_version stamped on unmatched rows" + ); + } + + #[test] + fn test_bitmap_cardinality_exceeds_physical_rows() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let version_seq = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq).unwrap(); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(version_meta.clone()), + created_at_version_meta: Some(version_meta), + }; + + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); + + // Bitmap with 10 offsets but fragment only has 5 physical rows. + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter(0u32..10))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, + ); + + let result = tx.build_manifest(Some(&manifest), vec![], "txn", &default_build_config()); + assert!(result.is_err()); + let msg = result.unwrap_err().to_string(); + assert!( + msg.contains("cardinality"), + "expected cardinality error, got: {msg}" + ); + } + + #[test] + fn test_bitmap_max_offset_exceeds_physical_rows() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let version_seq = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq).unwrap(); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(version_meta.clone()), + created_at_version_meta: Some(version_meta), + }; + + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); + + // Only 2 offsets (within cardinality) but max offset 100 exceeds physical_rows 5. + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter([0u32, 100]))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, + ); + + let result = tx.build_manifest(Some(&manifest), vec![], "txn", &default_build_config()); + assert!(result.is_err()); + let msg = result.unwrap_err().to_string(); + assert!( + msg.contains("max offset"), + "expected max offset error, got: {msg}" + ); + } + + #[test] + fn test_bitmap_at_exact_physical_rows_boundary_succeeds() { + let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let version_seq = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let version_meta = RowDatasetVersionMeta::from_sequence(&version_seq).unwrap(); + + let fragment = Fragment { + id: 1, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(version_meta.clone()), + created_at_version_meta: Some(version_meta), + }; + + let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); + + // All 5 offsets on a 5-row fragment — exactly at the boundary, should succeed. + let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter(0u32..5))]); + let tx = Transaction::new( + manifest.version, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![fragment], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }, + None, + ); + + tx.build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .expect("bitmap at exact physical_rows boundary should succeed"); + } + + #[test] + fn test_updated_fragment_offsets_key_not_in_updated_fragments_is_rejected() { + // Fragment A is being rewritten; fragment B exists in the manifest but is + // NOT in updated_fragments. Supplying an offset key for B must be rejected + // so that B's version metadata cannot be stamped by an unrelated commit. + let make_fragment = |id: u64| { + let row_ids = RowIdSequence::from([id * 10].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + Fragment { + id, + files: vec![DataFile::new( + format!("{id}.lance"), + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + )], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: None, + created_at_version_meta: None, + } + }; + + let frag_a = make_fragment(1); + let frag_b = make_fragment(2); + let manifest = make_stable_row_id_manifest(vec![frag_a.clone(), frag_b.clone()]); + + // updated_fragments contains only A; offsets are keyed to B — must fail. + let off_map = HashMap::from([(frag_b.id, RoaringBitmap::from_iter([0u32, 1, 2]))]); + let operation = Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![frag_a], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), + }; + + let err = validate_operation(Some(&manifest), &operation).unwrap_err(); + assert!( + err.to_string().contains("not in updated_fragments"), + "expected key-presence error, got: {err}" + ); + } + + #[test] + fn test_proto_round_trip_field_10() { + let off_map = HashMap::from([ + (1u64, RoaringBitmap::from_iter([1u32, 3, 5])), + (2u64, RoaringBitmap::from_iter([0u32, 2, 4, 6])), + ]); + let tx = Transaction::new( + 1, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map.clone())), + }, + None, + ); + + let pb_tx: pb::Transaction = pb::Transaction::from(&tx); + + // Field 9 must be empty; field 10 must be populated. + if let Some(pb::transaction::Operation::Update(ref update)) = pb_tx.operation { + assert!( + update.updated_fragment_offsets.is_empty(), + "field 9 should be empty" + ); + assert_eq!(update.updated_fragment_offset_bitmaps.len(), 2); + } else { + panic!("expected Update operation"); + } + + let tx2 = Transaction::try_from(pb_tx).unwrap(); + if let Operation::Update { + updated_fragment_offsets: Some(UpdatedFragmentOffsets(m)), + .. + } = &tx2.operation + { + assert_eq!(m.len(), 2); + assert_eq!(*m.get(&1).unwrap(), off_map[&1]); + assert_eq!(*m.get(&2).unwrap(), off_map[&2]); + } else { + panic!("expected Update with offsets"); + } + } + + #[test] + fn test_proto_legacy_field_9_read() { + // Simulate a manifest written by old Lance: only field 9, no field 10. + let pb_tx = pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + tag: String::new(), + transaction_properties: HashMap::new(), + operation: Some(pb::transaction::Operation::Update( + pb::transaction::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: 1, + inserted_rows: None, + updated_fragment_offsets: HashMap::from([( + 1u64, + pb::transaction::UInt32List { + values: vec![1, 3, 5], + }, + )]), + updated_fragment_offset_bitmaps: HashMap::new(), + }, + )), + }; + + let tx = Transaction::try_from(pb_tx).unwrap(); + if let Operation::Update { + updated_fragment_offsets: Some(UpdatedFragmentOffsets(m)), + .. + } = &tx.operation + { + assert_eq!(m.len(), 1); + let bitmap = m.get(&1).unwrap(); + let offsets: Vec = bitmap.iter().collect(); + assert_eq!(offsets, vec![1, 3, 5]); + } else { + panic!("expected Update with offsets from legacy field 9"); + } + } + + #[test] + fn test_proto_field_10_takes_precedence_over_field_9() { + // When both fields present, field 10 wins. + let mut bitmap_bytes = Vec::new(); + RoaringBitmap::from_iter([10u32, 20, 30]) + .serialize_into(&mut bitmap_bytes) + .unwrap(); + + let pb_tx = pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + tag: String::new(), + transaction_properties: HashMap::new(), + operation: Some(pb::transaction::Operation::Update( + pb::transaction::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: 1, + inserted_rows: None, + // Field 9 has different values than field 10. + updated_fragment_offsets: HashMap::from([( + 1u64, + pb::transaction::UInt32List { + values: vec![99, 100], + }, + )]), + updated_fragment_offset_bitmaps: HashMap::from([(1u64, bitmap_bytes)]), + }, + )), + }; + + let tx = Transaction::try_from(pb_tx).unwrap(); + if let Operation::Update { + updated_fragment_offsets: Some(UpdatedFragmentOffsets(m)), + .. + } = &tx.operation + { + let offsets: Vec = m.get(&1).unwrap().iter().collect(); + assert_eq!(offsets, vec![10, 20, 30], "field 10 should take precedence"); + } else { + panic!("expected Update with offsets from field 10"); + } + } + + #[test] + fn merge_build_manifest_refreshes_last_updated_when_data_files_change_stable_row_ids() { + use crate::feature_flags::FLAG_STABLE_ROW_IDS; + use lance_file::version::LanceFileVersion; + + let mk_file = |path: &str| { + DataFile::new( + path, + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ) + }; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + let row_ids = RowIdSequence::from([100u64, 101, 102, 103, 104].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let prev_fragment = Fragment { + id: 0, + files: vec![mk_file("before.lance")], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + + let mut manifest = Manifest::new( + lance_schema.clone(), + Arc::new(vec![prev_fragment.clone()]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; + manifest.next_row_id = 100; + + let merged_fragment = Fragment { + files: vec![mk_file("after.lance")], + ..prev_fragment + }; + + let tx = Transaction::new( + manifest.version, + Operation::Merge { + fragments: vec![merged_fragment], + schema: lance_schema, + preserves_nullability: true, + }, + None, + ); + + let (out, _) = tx + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + assert_eq!(out.version, 2); + let frag = &out.fragments[0]; + let seq = frag + .last_updated_at_version_meta + .as_ref() + .unwrap() + .load_sequence() + .unwrap(); + assert_eq!(seq.version_at(0).unwrap(), 2); + assert_eq!(seq.version_at(4).unwrap(), 2); + } + + #[test] + fn merge_build_manifest_skips_refresh_when_carry_forward_stable_row_ids() { + use crate::feature_flags::FLAG_STABLE_ROW_IDS; + use crate::rowids::version::{RowDatasetVersionMeta, RowDatasetVersionSequence}; + use lance_file::version::LanceFileVersion; + + let data_file = DataFile::new( + "same.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ); + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + let row_ids = RowIdSequence::from([200u64, 201, 202, 203, 204].as_slice()); + let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids).into())); + + let uniform_v1 = RowDatasetVersionSequence::from_uniform_row_count(5, 1); + let meta_v1 = RowDatasetVersionMeta::from_sequence(&uniform_v1).unwrap(); + + let prev_fragment = Fragment { + id: 0, + files: vec![data_file.clone()], + overlays: vec![], + deletion_file: None, + row_id_meta: row_id_meta.clone(), + physical_rows: Some(5), + last_updated_at_version_meta: Some(meta_v1.clone()), + created_at_version_meta: None, + }; + + let mut manifest = Manifest::new( + lance_schema.clone(), + Arc::new(vec![prev_fragment]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; + manifest.next_row_id = 100; + + let merged_fragment = Fragment { + id: 0, + files: vec![data_file], + overlays: vec![], + deletion_file: None, + row_id_meta, + physical_rows: Some(5), + last_updated_at_version_meta: Some(meta_v1), + created_at_version_meta: None, + }; + + let tx = Transaction::new( + manifest.version, + Operation::Merge { + fragments: vec![merged_fragment], + schema: lance_schema, + preserves_nullability: true, + }, + None, + ); + + let (out, _) = tx + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let seq = out.fragments[0] + .last_updated_at_version_meta + .as_ref() + .unwrap() + .load_sequence() + .unwrap(); + assert_eq!(seq.version_at(0).unwrap(), 1); + assert_eq!(seq.version_at(4).unwrap(), 1); + } + + #[test] + fn merge_build_manifest_no_last_updated_refresh_without_stable_row_ids() { + use crate::feature_flags::FLAG_STABLE_ROW_IDS; + use lance_file::version::LanceFileVersion; + + let mk_file = |path: &str| { + DataFile::new( + path, + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ) + }; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + let prev_fragment = Fragment { + id: 0, + files: vec![mk_file("before.lance")], + overlays: vec![], + deletion_file: None, + row_id_meta: None, + physical_rows: Some(5), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + + let manifest = Manifest::new( + lance_schema.clone(), + Arc::new(vec![prev_fragment.clone()]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + assert_eq!( + manifest.reader_feature_flags & FLAG_STABLE_ROW_IDS, + 0, + "manifest must not use stable row IDs for this guard test" + ); + + let merged_fragment = Fragment { + files: vec![mk_file("after.lance")], + ..prev_fragment + }; + + let tx = Transaction::new( + manifest.version, + Operation::Merge { + fragments: vec![merged_fragment], + schema: lance_schema, + preserves_nullability: true, + }, + None, + ); + + let (out, _) = tx + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + assert!( + out.fragments[0].last_updated_at_version_meta.is_none(), + "without stable row IDs, Merge must not populate per-row last_updated metadata" + ); + } + + #[test] + fn merge_build_manifest_sets_both_version_meta_for_new_fragment_id_stable_row_ids() { + use crate::feature_flags::FLAG_STABLE_ROW_IDS; + use lance_file::version::LanceFileVersion; + + let mk_file = |path: &str| { + DataFile::new( + path, + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + None, + None, + ) + }; + + let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // Existing fragment (id=0) with stable row IDs + let row_ids_0 = RowIdSequence::from([10u64, 11, 12].as_slice()); + let existing_fragment = Fragment { + id: 0, + files: vec![mk_file("existing.lance")], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_0).into())), + physical_rows: Some(3), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + + let mut manifest = Manifest::new( + lance_schema.clone(), + Arc::new(vec![existing_fragment.clone()]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; + manifest.next_row_id = 100; + manifest.version = 1; + + // New fragment (id=1) not present in prev manifest — exercises the None branch + let row_ids_1 = RowIdSequence::from([20u64, 21, 22, 23].as_slice()); + let new_fragment = Fragment { + id: 1, + files: vec![mk_file("new.lance")], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_1).into())), + physical_rows: Some(4), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + + let tx = Transaction::new( + manifest.version, + Operation::Merge { + fragments: vec![existing_fragment, new_fragment], + schema: lance_schema, + preserves_nullability: true, + }, + None, + ); + + let (out, _) = tx + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + assert_eq!(out.version, 2); + + let new_frag = out.fragments.iter().find(|f| f.id == 1).unwrap(); + + // last_updated_at_version must be set to the commit version + let last_updated_seq = new_frag + .last_updated_at_version_meta + .as_ref() + .expect("new fragment must have last_updated_at_version_meta") + .load_sequence() + .unwrap(); + assert_eq!(last_updated_seq.version_at(0).unwrap(), 2); + assert_eq!(last_updated_seq.version_at(3).unwrap(), 2); + + // created_at_version must also be set — must not be None + let created_seq = new_frag + .created_at_version_meta + .as_ref() + .expect("new fragment must have created_at_version_meta") + .load_sequence() + .unwrap(); + assert_eq!(created_seq.version_at(0).unwrap(), 2); + assert_eq!(created_seq.version_at(3).unwrap(), 2); + } + + #[test] + fn test_data_overlay_build_manifest_multi_fragment() { + // Overlays targeting two distinct fragments are each applied and stamped. + // A targeted fragment already carrying an overlay (committed at v3) gets + // the new overlay appended and stamped while its existing overlay is + // preserved, and a fragment the operation does not target is passed + // through with its existing overlays untouched. + let mut frag0 = Fragment::new(0); + frag0.overlays = vec![overlay_with_field(5, 3)]; // targeted, pre-existing at v3 + let frag1 = Fragment::new(1); + let mut frag2 = Fragment::new(2); + frag2.overlays = vec![overlay_with_field(9, 3)]; // untargeted, committed at v3 + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let mut manifest = Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(vec![frag0, frag1, frag2]), + crate::format::DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + // The pre-existing overlays were committed at v3, so the current + // manifest must be at least that version; the new commit then stamps + // its overlay at v4, keeping the fragment's overlays newest-last. + manifest.version = 3; + + let txn = Transaction::new( + manifest.version, + Operation::DataOverlay { + groups: vec![ + DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(1, 0)], + }, + DataOverlayGroup { + fragment_id: 1, + overlays: vec![overlay_with_field(2, 0)], + }, + ], + }, + None, + ); + + let (result, _) = txn + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let frag = |id: u64| { + result + .fragments + .iter() + .find(|f| f.id == id) + .unwrap_or_else(|| panic!("fragment {id} missing from result")) + }; + // The already-overlaid target keeps its v3 overlay and appends the new + // one, stamped to the new version. + assert_eq!(frag(0).overlays.len(), 2); + assert_eq!(frag(0).overlays[0].committed_version, 3); + assert_eq!(frag(0).overlays[1].committed_version, result.version); + // The fresh target gets its overlay, stamped to the new version. + assert_eq!(frag(1).overlays.len(), 1); + assert_eq!(frag(1).overlays[0].committed_version, result.version); + // The untargeted fragment is unchanged: same overlay, original version. + assert_eq!(frag(2).overlays.len(), 1); + assert_eq!(frag(2).overlays[0].committed_version, 3); + assert!(result.version > manifest.version); + } + + #[test] + fn test_data_replacement_tombstones_overlaid_fields() { + // A DataReplacement writing new base values for field 5 must stop any + // overlay already shadowing those cells: field 5 is tombstoned in place + // (preserving the overlay's field 3), and an overlay covering only field + // 5 is dropped entirely. Both overlays predate the transaction's read + // version, which is what makes the replacement the newer value. + let mut fragment = Fragment::new(0); + fragment.files = vec![ + DataFile::new_legacy_from_fields("f3.lance", vec![3], None), + DataFile::new_legacy_from_fields("f5.lance", vec![5], None), + ]; + fragment.overlays = vec![ + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o35.lance", vec![3, 5], None), + coverage: OverlayCoverage::sparse(vec![ + roaring::RoaringBitmap::from_iter([0u32]), + roaring::RoaringBitmap::from_iter([0u32]), + ]), + committed_version: 1, + }, + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o5.lance", vec![5], None), + coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])), + committed_version: 1, + }, + ]; + + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let manifest = Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(vec![fragment]), + crate::format::DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + + let txn = Transaction::new( + manifest.version, + Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 0, + DataFile::new_legacy_from_fields("f5-new.lance", vec![5], None), + )], + }, + None, + ); + + let (result, _) = txn + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let frag = &result.fragments[0]; + // The base data file for field 5 was swapped in. + assert!(frag.files.iter().any(|f| f.path == "f5-new.lance")); + // The [3, 5] overlay keeps field 3 and tombstones field 5; the [5]-only + // overlay is dropped. + assert_eq!(frag.overlays.len(), 1); + assert_eq!(frag.overlays[0].data_file.fields.as_ref(), &[3, -2]); + } + + /// Replace `fields` in `fragment` at `read_version`, against a manifest + /// at `manifest_version` whose schema declares field ids 3 ("x"), 4 ("a"), + /// 5 ("v") and 6 ("y"). + fn replace_fields( + fragment: Fragment, + fields: Vec, + manifest_version: u64, + read_version: u64, + ) -> Result { + let schema = ArrowSchema::new(vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("v", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ]); + let mut lance_schema = LanceSchema::try_from(&schema).unwrap(); + lance_schema.fields[0].id = 3; + lance_schema.fields[1].id = 4; + lance_schema.fields[2].id = 5; + lance_schema.fields[3].id = 6; + let mut manifest = Manifest::new( + lance_schema, + Arc::new(vec![fragment]), + crate::format::DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + manifest.version = manifest_version; + + let column_indices = (0..fields.len() as i32).collect(); + let txn = Transaction::new( + read_version, + Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 0, + DataFile::new( + "v-new.lance", + fields, + column_indices, + ConcreteFileVersion::V2_0, + None, + None, + ), + )], + }, + None, + ); + txn.build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .map(|(manifest, _)| manifest.fragments[0].clone()) + } + + /// Replace field 5 in `fragment` at `read_version`, against a manifest at + /// `manifest_version`. + fn replace_field_5( + fragment: Fragment, + manifest_version: u64, + read_version: u64, + ) -> Result { + replace_fields(fragment, vec![5], manifest_version, read_version) + } + + #[test] + fn test_data_replacement_rejects_subset_of_legacy_file() { + // The V1 reader derives its page table offset from the first field in + // the file metadata, so turning `[4, 5]` into `[-2, 5]` would leave + // field 4 decoding from field 5's pages. With no exact match to swap, + // the replacement must be rejected rather than corrupting the sibling. + let mut fragment = Fragment::new(0); + fragment.files = vec![DataFile::new_legacy_from_fields( + "wide.lance", + vec![4, 5], + None, + )]; + + let result = replace_field_5(fragment, 1, 1); + assert!( + result.is_err(), + "legacy subset replacement must be rejected, got: {:?}", + result.map(|fragment| fragment.files) + ); + } + + #[test] + fn test_data_replacement_tombstones_fields_spanning_files() { + // The replaced fields sit in two different wider files. Each file is + // tombstoned for the field it holds and survives on its remaining + // live one, with the new file answering for both. + let mut fragment = Fragment::new(0); + fragment.files = vec![ + DataFile::new( + "ab.lance", + vec![3, 4], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ), + DataFile::new( + "cd.lance", + vec![5, 6], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ), + ]; + + let fragment = replace_fields(fragment, vec![4, 5], 1, 1).unwrap(); + let file = |path| { + fragment + .files + .iter() + .find(|file| file.path == path) + .unwrap_or_else(|| panic!("{path} survives on its live field")) + }; + assert_eq!(file("ab.lance").fields.as_ref(), &[3, TOMBSTONE_FIELD_ID]); + assert_eq!(file("cd.lance").fields.as_ref(), &[TOMBSTONE_FIELD_ID, 6]); + assert!(fragment.files.iter().any(|file| file.path == "v-new.lance")); + } + + #[test] + fn test_data_replacement_rejects_fields_spanning_a_legacy_file() { + // Spanning is only resolvable while every covering file can be + // tombstoned. A V1 file holding one of the replaced fields cannot, + // so the replacement must be rejected rather than half applied. + let mut fragment = Fragment::new(0); + fragment.files = vec![ + DataFile::new( + "ab.lance", + vec![3, 4], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ), + DataFile::new_legacy_from_fields("cd.lance", vec![5, 6], None), + ]; + + let result = replace_fields(fragment, vec![4, 5], 1, 1); + assert!( + result.is_err(), + "spanning a legacy file must be rejected, got: {:?}", + result.map(|fragment| fragment.files) + ); + } + + #[test] + fn test_data_replacement_retombstones_wider_file() { + // A wider file carrying a tombstone from an earlier round is + // tombstoned again for the newly replaced field and survives on its + // remaining live field. + let mut fragment = Fragment::new(0); + fragment.files = vec![DataFile::new( + "wide.lance", + vec![4, TOMBSTONE_FIELD_ID, 5], + vec![0, 1, 2], + ConcreteFileVersion::V2_0, + None, + None, + )]; + + let fragment = replace_fields(fragment, vec![5], 1, 1).unwrap(); + let wide = fragment + .files + .iter() + .find(|file| file.path == "wide.lance") + .expect("wider file survives on its live field"); + assert_eq!( + wide.fields.as_ref(), + &[4, TOMBSTONE_FIELD_ID, TOMBSTONE_FIELD_ID] + ); + assert!(fragment.files.iter().any(|file| file.path == "v-new.lance")); + } + + #[test] + fn test_data_replacement_preserves_overlay_newer_than_snapshot() { + // An overlay committed after this transaction read its snapshot holds + // the newer value; the conflict resolver rebases the two precisely + // because the overlay wins. Tombstoning it would discard a committed + // write, so only overlays the transaction could have seen are superseded. + let mut fragment = Fragment::new(0); + // One wider file, so the replacement takes the tombstone-and-append path. + fragment.files = vec![DataFile::new( + "wide.lance", + vec![4, 5], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + )]; + fragment.overlays = vec![DataOverlayFile { + data_file: DataFile::new( + "newer.lance", + vec![5], + vec![0], + ConcreteFileVersion::V2_0, + None, + None, + ), + coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])), + committed_version: 7, + }]; + + // Staged against version 6, i.e. before the overlay landed. + let fragment = replace_field_5(fragment, 7, 6).unwrap(); + assert!(fragment.files.iter().any(|f| f.path == "v-new.lance")); + assert_eq!( + fragment.overlays.len(), + 1, + "overlay committed after the snapshot must survive" + ); + assert_eq!(fragment.overlays[0].data_file.fields.as_ref(), &[5]); + } + + #[test] + fn test_data_overlay_build_manifest_merges_duplicate_groups() { + // Two groups targeting the same fragment must both survive (a HashMap + // collapse would have dropped the first). + let manifest = sample_manifest(); + let txn = Transaction::new( + manifest.version, + Operation::DataOverlay { + groups: vec![ + DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(1, 0)], + }, + DataOverlayGroup { + fragment_id: 0, + overlays: vec![overlay_with_field(2, 0)], + }, + ], + }, + None, + ); + + let (result, _) = txn + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let overlays = &result.fragments[0].overlays; + assert_eq!(overlays.len(), 2); + assert_eq!(overlays[0].data_file.fields.as_ref(), [1i32].as_slice()); + assert_eq!(overlays[1].data_file.fields.as_ref(), [2i32].as_slice()); + } + + #[test] + fn test_data_overlay_build_manifest_rejects_unknown_fragment() { + let manifest = sample_manifest(); + let txn = Transaction::new( + manifest.version, + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 99, + overlays: vec![overlay_with_field(1, 0)], + }], + }, + None, + ); + let err = txn + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap_err(); + assert!(err.to_string().contains("does not exist"), "{err}"); + } + + #[test] + fn test_nullability_assertion_defaults_conservative() { + // A writer that predates the field encodes nothing, which decodes as + // false: no assertion, so a legacy tightening or required-field merge + // still conflicts. Only an explicit true skips the barrier. + for encoded in [false, true] { + let txn = Transaction::try_from(pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + operation: Some(pb::transaction::Operation::Project( + pb::transaction::Project { + schema: vec![], + preserves_nullability: encoded, + }, + )), + ..Default::default() + }) + .unwrap(); + assert!( + matches!(txn.operation, Operation::Project { preserves_nullability, .. } if preserves_nullability == encoded), + "encoded={encoded:?}" + ); + + let txn = Transaction::try_from(pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + operation: Some(pb::transaction::Operation::Merge(pb::transaction::Merge { + fragments: vec![], + schema: vec![], + schema_metadata: Default::default(), + preserves_nullability: encoded, + })), + ..Default::default() + }) + .unwrap(); + assert!( + matches!(txn.operation, Operation::Merge { preserves_nullability, .. } if preserves_nullability == encoded), + "encoded={encoded:?}" + ); + } + } + + mod mem_wal_index_coverage { + use super::*; + use crate::system_index::mem_wal::{ + CompactedSsTable, IndexCatchupProgress, MEM_WAL_INDEX_NAME, MemWalIndexDetails, + }; + + fn user_index(name: &str, uuid: Uuid, frags: &[u32]) -> IndexMetadata { + IndexMetadata { + uuid, + name: name.to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter(frags.iter().copied())), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + fn mem_wal_index(details: MemWalIndexDetails) -> IndexMetadata { + crate::system_index::mem_wal::new_mem_wal_index_meta(1, details).unwrap() + } + + fn coverage_for(indices: &[IndexMetadata], name: &str) -> Option> { + let meta = indices + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .expect("mem wal index present"); + load_mem_wal_index_details(meta.clone()) + .unwrap() + .index_catchup + .into_iter() + .find(|entry| entry.index_name == name) + .map(|entry| entry.caught_up_generations) + } + + fn compacted(shard: Uuid, generation: u64) -> Vec { + vec![CompactedSsTable::new(shard, generation)] + } + + /// A manifest carrying exactly `frags`, standing in for the version a + /// transaction read. + fn manifest_with(frags: &[u32]) -> Manifest { + let fragments: Vec = + frags.iter().map(|id| Fragment::new(*id as u64)).collect(); + Manifest::new( + LanceSchema::default(), + Arc::new(fragments), + DataStorageFormat::default(), + Default::default(), + ) + } + + /// Drives the production path, so these exercise the real derivation. + fn apply( + after: &mut [IndexMetadata], + before: &[IndexMetadata], + read_frags: &[u32], + read_indices: &[IndexMetadata], + ) -> Result<()> { + let manifest = manifest_with(read_frags); + let segments_before = Transaction::logical_index_segments(before); + Transaction::apply_mem_wal_index_coverage( + after, + &segments_before, + Some(ReadVersionState { + manifest: &manifest, + indices: read_indices, + }), + 2, + ) + } + + fn table(idx_frags: &[u32], uuid: Uuid, details: MemWalIndexDetails) -> Vec { + vec![user_index("idx", uuid, idx_frags), mem_wal_index(details)] + } + + fn progress(shard: Uuid, generation: u64) -> MemWalIndexDetails { + MemWalIndexDetails { + compacted_sstables: compacted(shard, generation), + ..Default::default() + } + } + + fn progress_with_catchup(shard: Uuid, generation: u64, caught: u64) -> MemWalIndexDetails { + MemWalIndexDetails { + compacted_sstables: compacted(shard, generation), + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + compacted(shard, caught), + )], + ..Default::default() + } + } + + /// An index spanning every fragment the transaction read is credited + /// with what that version had compacted. + #[test] + fn an_index_covering_the_read_version_is_credited() { + let shard = Uuid::new_v4(); + let read = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + let mut after = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + apply(&mut after, &read, &[0, 1], &read).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// An index short of the read version proves nothing, so it gets no + /// entry -- absence reads as "not caught up". + #[test] + fn an_index_short_of_the_read_version_is_not_credited() { + let shard = Uuid::new_v4(); + let read = table(&[0], Uuid::new_v4(), progress(shard, 5)); + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 5)); + apply(&mut after, &read, &[0, 1], &read).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// The hazard that makes the comparison use whole metadata. + /// + /// `Operation::Update` prunes a segment's fragment bitmap in place when + /// it touches an indexed field, keeping the same UUID. A UUID-only + /// "unchanged" test carries the old position forward while the index + /// covers fewer fragments, and the WAL pod then trims on a position the + /// index no longer earns. Reachable from the ordinary SSTable merge. + #[test] + fn a_bitmap_pruned_in_place_does_not_keep_its_position() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0, 1], uuid, progress_with_catchup(shard, 5, 5)); + // Same UUID, fragment 1 pruned away. + let mut after = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + apply(&mut after, &before, &[0, 1], &before).unwrap(); + assert_eq!( + coverage_for(&after, "idx"), + None, + "a shrunken index kept a position it no longer earns" + ); + } + + /// Carrying a position forward is not the same as extending it. An + /// index that has not moved still only holds the generations it caught + /// up to; the compaction that has landed since is in fragments it does + /// not span. + #[test] + fn an_unchanged_index_is_not_raised_beyond_what_it_proves() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + // Recorded at generation 2; generation 5 has since been folded in. + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 2)); + let mut after = before.clone(); + // Fragment 1 arrived with that compaction and this index lacks it. + apply(&mut after, &before, &[0, 1], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 2))); + } + + /// A recorded position above what this commit says was compacted is + /// clamped down. Nothing should produce one, but a position the base + /// table cannot back would retire SSTables whose rows are nowhere. + #[test] + fn a_carried_position_cannot_exceed_the_committed_progress() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 3, 9)); + let mut after = before.clone(); + apply(&mut after, &before, &[0], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 3))); + } + + /// An unchanged index keeps what it recorded even when this commit's + /// own snapshot cannot prove as much. + #[test] + fn an_unchanged_index_is_never_lowered() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 9, 9)); + let mut after = before.clone(); + apply(&mut after, &before, &[0, 1], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 9))); + } + + /// Credit never exceeds what this commit records as compacted, so a + /// read version since rolled back cannot retire SSTables no live commit + /// copied in. + #[test] + fn credit_is_capped_by_the_committed_progress() { + let shard = Uuid::new_v4(); + let read = table(&[0], Uuid::new_v4(), progress(shard, 9)); + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 3)); + apply(&mut after, &read, &[0], &read).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 3))); + } + + /// The cap is the read version's progress, not this commit's. A + /// compaction that landed while the index was being built put its rows + /// in fragments this transaction never inspected, so covering + /// everything it *did* read earns only what had been folded in by then. + #[test] + fn credit_never_reaches_past_the_read_version() { + let shard = Uuid::new_v4(); + // Read at generation 2; generation 5 landed while this ran. + let read = table(&[0], Uuid::new_v4(), progress(shard, 2)); + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 5)); + apply(&mut after, &read, &[0], &read).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 2))); + } + + /// One segment with an unknown bitmap makes the whole index unproven, + /// even when its siblings happen to span everything. Coverage that + /// cannot be read is not coverage that can be relied on. + #[test] + fn an_index_with_an_unknown_segment_is_not_credited() { + let shard = Uuid::new_v4(); + let mut unknown = user_index("idx", Uuid::new_v4(), &[]); + unknown.fragment_bitmap = None; + let read = vec![ + user_index("idx", Uuid::new_v4(), &[0, 1]), + unknown, + mem_wal_index(progress(shard, 5)), + ]; + let mut after = read.clone(); + apply(&mut after, &read, &[0, 1], &read).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// A dropped index has no coverage left to gate anything. + #[test] + fn a_dropped_index_loses_its_entry() { + let shard = Uuid::new_v4(); + let before = table(&[0], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + let mut after = vec![mem_wal_index(progress_with_catchup(shard, 5, 5))]; + apply(&mut after, &before, &[0], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// An index created by this commit is credited if it spans the read + /// version -- it was built over those fragments, so it holds their + /// rows. This is what the advance model could not express: an ordinary + /// build that fully covers had to throw its work away and wait. + #[test] + fn a_new_index_covering_the_read_version_is_credited() { + let shard = Uuid::new_v4(); + let before = vec![mem_wal_index(progress(shard, 5))]; + let mut after = table(&[0], Uuid::new_v4(), progress(shard, 5)); + // Covers the read version, but was not there when it was read. + apply(&mut after, &before, &[0], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// A table carrying compaction progress but no catch-up entry earns one + /// from an ordinary commit. This is how a table written before catch-up + /// was maintained heals itself: nothing has to be run against it. + #[test] + fn a_table_with_no_catchup_entry_earns_one() { + let shard = Uuid::new_v4(); + let before = table(&[0], Uuid::new_v4(), progress(shard, 5)); + let mut after = before.clone(); + apply(&mut after, &before, &[0], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// Two shards, only one of them compacted. + #[test] + fn each_shard_is_credited_independently() { + let merged = Uuid::new_v4(); + let idle = Uuid::new_v4(); + let details = MemWalIndexDetails { + compacted_sstables: vec![ + CompactedSsTable::new(merged, 4), + CompactedSsTable::new(idle, 0), + ], + ..Default::default() + }; + let read = table(&[0], Uuid::new_v4(), details.clone()); + let mut after = table(&[0], Uuid::new_v4(), details); + apply(&mut after, &read, &[0], &read).unwrap(); + let coverage = coverage_for(&after, "idx").expect("credited"); + assert_eq!( + coverage + .iter() + .find(|g| g.shard_id == merged) + .map(|g| g.generation), + Some(4) + ); + assert_eq!( + coverage + .iter() + .find(|g| g.shard_id == idle) + .map(|g| g.generation), + Some(0) + ); + } + + /// Two indexes advance independently: one covering, one behind. + #[test] + fn indexes_are_credited_independently() { + let shard = Uuid::new_v4(); + let read = vec![ + user_index("fast", Uuid::new_v4(), &[0, 1]), + user_index("slow", Uuid::new_v4(), &[0]), + mem_wal_index(progress(shard, 6)), + ]; + let mut after = read.clone(); + apply(&mut after, &read, &[0, 1], &read).unwrap(); + assert_eq!(coverage_for(&after, "fast"), Some(compacted(shard, 6))); + assert_eq!(coverage_for(&after, "slow"), None); + } + + /// An index whose coverage is unknown cannot be shown to cover anything. + #[test] + fn an_index_without_a_bitmap_is_not_credited() { + let shard = Uuid::new_v4(); + let mut idx = user_index("idx", Uuid::new_v4(), &[0]); + idx.fragment_bitmap = None; + let read = vec![idx, mem_wal_index(progress(shard, 5))]; + let mut after = read.clone(); + apply(&mut after, &read, &[0], &read).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// Nothing compacted means nothing to be behind on. + #[test] + fn no_compaction_progress_writes_no_entries() { + let before = table(&[0], Uuid::new_v4(), MemWalIndexDetails::default()); + let mut after = before.clone(); + let untouched = after.clone(); + apply(&mut after, &before, &[0], &before).unwrap(); + assert_eq!(after, untouched); + } + + /// No MemWAL system index: nothing to maintain, and no error. + #[test] + fn a_table_without_mem_wal_is_a_no_op() { + let before = vec![user_index("idx", Uuid::new_v4(), &[0])]; + let mut after = before.clone(); + let untouched = after.clone(); + apply(&mut after, &before, &[0], &before).unwrap(); + assert_eq!(after, untouched); + } + + /// No read version -- dataset creation, detached commits -- credits + /// nothing and lowers nothing. + #[test] + fn without_a_read_version_nothing_changes() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + let mut after = before.clone(); + let segments_before = Transaction::logical_index_segments(&before); + Transaction::apply_mem_wal_index_coverage(&mut after, &segments_before, None, 2) + .unwrap(); + assert_eq!(coverage_for(&after, "idx"), Some(compacted(shard, 5))); + } + + /// An untrained index covers nothing that exists, so a sibling's work + /// is no evidence for it. + #[test] + fn an_untrained_index_earns_nothing() { + let shard = Uuid::new_v4(); + let read = vec![ + user_index("untrained", Uuid::new_v4(), &[]), + user_index("trained", Uuid::new_v4(), &[0]), + mem_wal_index(progress(shard, 10)), + ]; + let mut after = read.clone(); + apply(&mut after, &read, &[0], &read).unwrap(); + assert_eq!(coverage_for(&after, "untrained"), None); + assert_eq!(coverage_for(&after, "trained"), Some(compacted(shard, 10))); + } + + /// Shards move independently within one index: one advances on this + /// commit's proof while another keeps the position it already had. + #[test] + fn a_shard_keeps_its_position_while_another_advances() { + let (advancing, quiet) = (Uuid::new_v4(), Uuid::new_v4()); + let uuid = Uuid::new_v4(); + let details = |advancing_gen: u64| MemWalIndexDetails { + compacted_sstables: vec![ + CompactedSsTable::new(advancing, advancing_gen), + CompactedSsTable::new(quiet, 10), + ], + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + vec![CompactedSsTable::new(quiet, 7)], + )], + ..Default::default() + }; + // The quiet shard was never compacted as of the read, so nothing + // this commit proves reaches it -- it keeps its recorded 7. + let read = vec![ + user_index("idx", uuid, &[0]), + mem_wal_index(MemWalIndexDetails { + compacted_sstables: vec![CompactedSsTable::new(advancing, 9)], + ..details(9) + }), + ]; + let mut after = vec![user_index("idx", uuid, &[0]), mem_wal_index(details(10))]; + apply(&mut after, &read, &[0], &read).unwrap(); + + let mut coverage = coverage_for(&after, "idx").expect("credited"); + coverage.sort_unstable_by_key(|sstable| sstable.shard_id); + let mut expected = vec![ + CompactedSsTable::new(advancing, 9), + CompactedSsTable::new(quiet, 7), + ]; + expected.sort_unstable_by_key(|sstable| sstable.shard_id); + assert_eq!(coverage, expected); + } + + /// The derivation drops coverage an index no longer earns, but it never + /// rejects the commit -- an ordinary index job must not be blocked by + /// a protocol it knows nothing about. + #[test] + fn an_ordinary_index_job_is_never_blocked() { + let shard = Uuid::new_v4(); + let before = table(&[0, 1], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + // Rebuilt over a subset -- the shape a partial reindex leaves. + let mut after = table(&[0], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + apply(&mut after, &before, &[0, 1], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// A reader's rule is that a missing entry means "not caught up", so an + /// index caught up to nothing must be absent rather than present at + /// generation zero -- otherwise it reads as known-and-covered. + #[test] + fn an_index_caught_up_to_nothing_gets_no_entry() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 0)); + let mut after = before.clone(); + // Does not span the read version, so nothing lifts it off zero. + apply(&mut after, &before, &[0, 1], &before).unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// Each shard carries its own position. Collapsing them to one value + /// would credit a lagging shard with a busier shard's progress. + #[test] + fn carried_positions_do_not_leak_between_shards() { + let (ahead, behind) = (Uuid::new_v4(), Uuid::new_v4()); + let uuid = Uuid::new_v4(); + let details = MemWalIndexDetails { + compacted_sstables: vec![ + CompactedSsTable::new(ahead, 10), + CompactedSsTable::new(behind, 10), + ], + index_catchup: vec![IndexCatchupProgress::new( + "idx".to_string(), + vec![ + CompactedSsTable::new(ahead, 8), + CompactedSsTable::new(behind, 2), + ], + )], + ..Default::default() + }; + let before = vec![user_index("idx", uuid, &[0]), mem_wal_index(details)]; + let mut after = before.clone(); + // Unchanged and unproven: both shards keep exactly what they had. + apply(&mut after, &before, &[0, 1], &before).unwrap(); + + let mut coverage = coverage_for(&after, "idx").expect("carried"); + coverage.sort_unstable_by_key(|sstable| sstable.shard_id); + let mut expected = vec![ + CompactedSsTable::new(ahead, 8), + CompactedSsTable::new(behind, 2), + ]; + expected.sort_unstable_by_key(|sstable| sstable.shard_id); + assert_eq!(coverage, expected); + } + + /// The derivation runs while the manifest is being built, but the + /// index list is not final there: `migrate_indices` recalculates a + /// segment's fragment bitmap and keeps its UUID. A position decided + /// before that must not survive the narrowing, or the WAL pod trims + /// against an index that no longer covers those rows. + #[test] + fn a_bitmap_narrowed_after_the_build_loses_its_position() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + // What migrate_indices leaves behind: same UUID, fewer fragments, + // and it says so. + let mut migrated = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + + Transaction::withdraw_coverage_invalidated_after_build( + &mut migrated, + &["idx".to_string()], + 3, + ) + .unwrap(); + + assert_eq!(coverage_for(&migrated, "idx"), None); + } + + /// Migration routinely fills in file lists and inferred details. Those + /// do not change which rows an index answers for, so withdrawing on + /// them would drop coverage every commit for no reason. + #[test] + fn metadata_migration_that_does_not_narrow_keeps_its_position() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let mut migrated = table(&[0, 1], uuid, progress_with_catchup(shard, 5, 5)); + migrated[0].files = Some(Vec::new()); + migrated[0].created_at = Some(chrono::Utc::now()); + + // Nothing narrowed, so migration reports nothing. + Transaction::withdraw_coverage_invalidated_after_build(&mut migrated, &[], 3).unwrap(); + + assert_eq!(coverage_for(&migrated, "idx"), Some(compacted(shard, 5))); + } + + /// A commit that changes nothing must not churn the system index: a new + /// UUID on every append would invalidate its cache entry fleet-wide. + #[test] + fn an_unchanged_commit_does_not_rewrite_the_system_index() { + let shard = Uuid::new_v4(); + let uuid = Uuid::new_v4(); + let before = table(&[0], uuid, progress_with_catchup(shard, 5, 5)); + let mut after = before.clone(); + apply(&mut after, &before, &[0], &before).unwrap(); + + let system_uuid = |indices: &[IndexMetadata]| { + indices + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .unwrap() + .uuid + }; + assert_eq!(system_uuid(&after), system_uuid(&before)); + } + + /// A commit with no read version still withdraws. It can prove nothing, + /// so an index it changed keeps no position -- the alternative leaves a + /// position describing an index that no longer exists. + #[test] + fn without_a_read_version_a_changed_index_still_loses_its_position() { + let shard = Uuid::new_v4(); + let before = table(&[0, 1], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + let mut after = table(&[0], Uuid::new_v4(), progress_with_catchup(shard, 5, 5)); + let segments_before = Transaction::logical_index_segments(&before); + Transaction::apply_mem_wal_index_coverage(&mut after, &segments_before, None, 2) + .unwrap(); + assert_eq!(coverage_for(&after, "idx"), None); + } + + /// Two attempts against the same read version agree, which is what makes + /// a rebase safe: `read_version` is fixed for a transaction's life. + #[test] + fn the_derivation_is_stable_across_attempts() { + let shard = Uuid::new_v4(); + let read = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + let mut first = table(&[0, 1], Uuid::new_v4(), progress(shard, 5)); + let mut second = first.clone(); + apply(&mut first, &read, &[0, 1], &read).unwrap(); + apply(&mut second, &read, &[0, 1], &read).unwrap(); + assert_eq!(coverage_for(&first, "idx"), coverage_for(&second, "idx")); + } + } +} diff --git a/rust/lance-table/src/transaction/operation.rs b/rust/lance-table/src/transaction/operation.rs new file mode 100644 index 00000000000..7612464b3a0 --- /dev/null +++ b/rust/lance-table/src/transaction/operation.rs @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! The vocabulary of changes a transaction can describe. +//! +//! Each [`Operation`] variant names one kind of change and carries exactly the +//! inputs needed to apply it: the fragments to add, the fields that were +//! rewritten, the indices that were rebuilt. Applying them is +//! [`super::manifest_build`]; deciding whether two of them collide is +//! [`super::conflicts`]. + +use crate::format::key_existence::KeyExistenceFilter; +use crate::format::overlay::DataOverlayFile; +use crate::format::{BasePath, DataFile, Fragment, IndexFile, IndexMetadata}; +use crate::system_index::mem_wal::CompactedSsTable; +use crate::transaction::UpdateMap; +use lance_core::datatypes::Schema; +use lance_core::deepsize::DeepSizeOf; +use roaring::RoaringBitmap; +use std::collections::HashMap; +use uuid::Uuid; + +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct DataReplacementGroup(pub u64, pub DataFile); + +/// Overlay files to append to a single fragment, in order (the last entry is +/// newest). The overlays are appended to the fragment's existing `overlays` +/// list rather than replacing it, so overlays written by concurrent commits are +/// preserved. Each overlay's `committed_version` is stamped to the new dataset +/// version at commit time (re-stamped on retry). +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct DataOverlayGroup { + pub fragment_id: u64, + pub overlays: Vec, +} + +/// An operation on a dataset. +#[derive(Debug, Clone, DeepSizeOf)] +pub enum Operation { + /// Adding new fragments to the dataset. The fragments contained within + /// haven't yet been assigned a final ID. + Append { fragments: Vec }, + /// Updated fragments contain those that have been modified with new deletion + /// files. The deleted fragment IDs are those that should be removed from + /// the manifest. + Delete { + updated_fragments: Vec, + deleted_fragment_ids: Vec, + predicate: String, + }, + /// Overwrite the entire dataset with the given fragments. This is also + /// used when initially creating a table. + /// + /// The fragments are newly written ones and are assigned fresh ids at commit + /// time, continuing from the dataset's highest id ever used; the ids they + /// arrive with are ignored. + /// + /// A fragment carrying a deletion file is rejected. A deletion file's path + /// embeds the fragment id, so it cannot follow its fragment to the new id: + /// minting a fragment and giving it a deletion file are mutually exclusive in + /// one transaction. Use [`Self::Delete`] to commit deletions against existing + /// fragments, or [`Self::Merge`] to change their schema. + Overwrite { + fragments: Vec, + schema: Schema, + config_upsert_values: Option>, + initial_bases: Option>, + }, + /// A new index has been created. + CreateIndex { + /// The new secondary indices, + /// any existing indices with the same name will be replaced. + new_indices: Vec, + /// The indices that have been modified. + removed_indices: Vec, + }, + /// Data is rewritten but *not* modified. This is used for things like + /// compaction or re-ordering. Contains the old fragments and the new + /// ones that have been replaced. + /// + /// This operation will modify the row addresses of existing rows and + /// so any existing index covering a rewritten fragment will need to be + /// remapped. + Rewrite { + /// Groups of fragments that have been modified + groups: Vec, + /// Indices that have been updated with the new row addresses + rewritten_indices: Vec, + /// The fragment reuse index to be created or updated to + frag_reuse_index: Option, + }, + /// Replace data in a column in the dataset with new data. This is used for + /// null column population where we replace an entirely null column with a + /// new column that has data. + /// + /// This operation will only allow replacing files that contain the same schema + /// e.g. if the original files contain columns A, B, C and the new files contain + /// only columns A, B then the operation is not allowed. As we would need to split + /// the original files into two files, one with column A, B and the other with column C. + /// + /// Corollary to the above: the operation will also not allow replacing files unless the + /// affected columns all have the same datafile layout across the fragments being replaced. + /// + /// e.g. if fragments being replaced contain files with different schema layouts on + /// the column being replaced, the operation is not allowed. + /// say `frag_1: [A] [B, C]` and `frag_2: [A, B] [C]` and we are trying to replace column A + /// with a new column A, the operation is not allowed. + DataReplacement { + replacements: Vec, + }, + /// Attach overlay files to fragments, supplying new values for a subset of + /// `(physical offset, field)` cells without rewriting the fragments' base + /// data files. See [`DataOverlayFile`] and the Data Overlay Files + /// specification for resolution, coverage, and versioning rules. + DataOverlay { groups: Vec }, + /// Merge a new column in + /// 'fragments' is the final fragments include all data files, the new fragments must align with old ones at rows. + /// 'schema' is not forced to include existed columns, which means we could use Merge to drop column data + Merge { + fragments: Vec, + schema: Schema, + /// Set when this merge makes no nullability-affecting schema change: + /// it introduces no field that data staged against an earlier schema + /// could not safely omit. Without the assertion the merge conflicts + /// with concurrent appends in either commit order, since a stale + /// append omits new columns entirely and its rows read as null. + preserves_nullability: bool, + }, + /// Restore an old version of the database + Restore { version: u64 }, + /// Reserves fragment ids for future use + /// This can be used when row ids need to be known before a transaction + /// has been committed. It is used during a rewrite operation to allow + /// indices to be remapped to the new row ids as part of the operation. + ReserveFragments { num_fragments: u32 }, + + /// Update values in the dataset. + /// + /// Updates are generally vertical or horizontal. + /// + /// A vertical update adds new rows. In this case, the updated_fragments + /// will only have existing rows deleted and will not have any new fields added. + /// All new data will be contained in new_fragments. + /// This is what is used by a merge_insert that matches the whole schema and what + /// is used by the dataset updater. + /// + /// A horizontal update adds new columns. In this case, the updated fragments + /// may have fields removed or added. It is even possible for a field to be tombstoned + /// and then added back in the same update. (which is a field modification). If any + /// fields are modified in this way then they need to be added to the fields_modified list. + /// This way we can correctly update the indices. + /// This is what is used by a merge insert that does not match the whole schema. + Update { + /// Ids of fragments that have been moved + removed_fragment_ids: Vec, + /// Fragments that have been updated + updated_fragments: Vec, + /// Fragments that have been added + new_fragments: Vec, + /// The fields that have been modified + fields_modified: Vec, + /// MemWAL SSTables to mark as compacted after this transaction. + compacted_sstables: Vec, + /// The fields that used to judge whether to preserve the new frag's id into + /// the frag bitmap of the specified indices. + fields_for_preserving_frag_bitmap: Vec, + /// The mode of update + update_mode: Option, + /// Optional filter for detecting conflicts on inserted row keys. + /// Only tracks keys from INSERT operations during merge insert, not updates. + inserted_rows_filter: Option, + /// Physical row offsets (per fragment) that matched `update_columns` for RewriteColumns. + /// `None` means callers did not supply offsets; `build_manifest` skips partial refresh then. + updated_fragment_offsets: Option, + }, + + /// Project to a new schema. + Project { + schema: Schema, + /// Set when this projection makes no nullability-affecting schema + /// change, as a rename or a drop does not. A nullability tightening + /// must not set this: its producer proved the claim by scanning at its + /// read version, so a concurrent write can falsify it and the + /// projection conflicts with value-writes in either commit order. + preserves_nullability: bool, + }, + + /// Update the dataset configuration and metadata. + /// + /// Schema or field metadata updates conflict with a concurrent + /// [`Self::Merge`] in either commit order. A merge carries complete schema + /// state from its read version, so rebasing the operations could discard + /// metadata installed by the other transaction. + UpdateConfig { + config_updates: Option, + table_metadata_updates: Option, + schema_metadata_updates: Option, + field_metadata_updates: HashMap, + }, + /// Update SSTable compaction progress in the MemWAL index. + /// + /// This is used during merge-insert to atomically record which + /// SSTables have been compacted into the base table. + UpdateMemWalState { + compacted_sstables: Vec, + }, + + /// Clone a dataset. + Clone { + is_shallow: bool, + ref_name: Option, + ref_version: u64, + ref_path: String, + branch_name: Option, + }, + + // Update base paths in the dataset (currently only supports adding new bases). + UpdateBases { + /// The new base paths to add to the manifest. + new_bases: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, DeepSizeOf)] +pub enum UpdateMode { + /// rows are deleted in current fragments and rewritten in new fragments. + /// This is most optimal when the majority of columns are being rewritten + /// or only a few rows are being updated. + RewriteRows, + + /// within each fragment, columns are fully rewritten and inserted as new data files. + /// Old versions of columns are tombstoned. This is most optimal when most rows are affected + /// but a small subset of columns are affected. + RewriteColumns, +} + +/// Matched physical row offsets per fragment for a partial [`UpdateMode::RewriteColumns`] update. +/// +/// Used with stable row IDs so `build_manifest` can refresh row-level version +/// metadata only for rows that were rewritten. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct UpdatedFragmentOffsets(pub HashMap); + +impl DeepSizeOf for UpdatedFragmentOffsets { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.0.iter().fold(0_usize, |acc, (frag_id, bitmap)| { + acc + frag_id.deep_size_of_children(context) + + (bitmap.len() as usize).saturating_mul(std::mem::size_of::()) + }) + } +} + +impl std::fmt::Display for Operation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Append { .. } => write!(f, "Append"), + Self::Delete { .. } => write!(f, "Delete"), + Self::Overwrite { .. } => write!(f, "Overwrite"), + Self::CreateIndex { .. } => write!(f, "CreateIndex"), + Self::Rewrite { .. } => write!(f, "Rewrite"), + Self::Merge { .. } => write!(f, "Merge"), + Self::Restore { .. } => write!(f, "Restore"), + Self::ReserveFragments { .. } => write!(f, "ReserveFragments"), + Self::Update { .. } => write!(f, "Update"), + Self::Project { .. } => write!(f, "Project"), + Self::UpdateConfig { .. } => write!(f, "UpdateConfig"), + Self::DataReplacement { .. } => write!(f, "DataReplacement"), + Self::DataOverlay { .. } => write!(f, "DataOverlay"), + Self::Clone { .. } => write!(f, "Clone"), + Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"), + Self::UpdateBases { .. } => write!(f, "UpdateBases"), + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RewrittenIndex { + pub old_id: Uuid, + pub new_id: Uuid, + pub new_index_details: prost_types::Any, + pub new_index_version: u32, + /// Files in the new index with their sizes. + /// Empty list from older writers that didn't persist this field. + pub new_index_files: Option>, +} + +impl DeepSizeOf for RewrittenIndex { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.new_index_details + .type_url + .deep_size_of_children(context) + + self.new_index_details.value.deep_size_of_children(context) + } +} + +#[derive(Debug, Clone, DeepSizeOf)] +pub struct RewriteGroup { + pub old_fragments: Vec, + pub new_fragments: Vec, +} + +impl PartialEq for RewriteGroup { + fn eq(&self, other: &Self) -> bool { + fn compare_vec(a: &[T], b: &[T]) -> bool { + a.len() == b.len() && a.iter().all(|f| b.contains(f)) + } + compare_vec(&self.old_fragments, &other.old_fragments) + && compare_vec(&self.new_fragments, &other.new_fragments) + } +} + +impl Operation { + pub fn name(&self) -> &str { + match self { + Self::Append { .. } => "Append", + Self::Delete { .. } => "Delete", + Self::Overwrite { .. } => "Overwrite", + Self::CreateIndex { .. } => "CreateIndex", + Self::Rewrite { .. } => "Rewrite", + Self::Merge { .. } => "Merge", + Self::ReserveFragments { .. } => "ReserveFragments", + Self::Restore { .. } => "Restore", + Self::Update { .. } => "Update", + Self::Project { .. } => "Project", + Self::UpdateConfig { .. } => "UpdateConfig", + Self::DataReplacement { .. } => "DataReplacement", + Self::DataOverlay { .. } => "DataOverlay", + Self::UpdateMemWalState { .. } => "UpdateMemWalState", + Self::Clone { .. } => "Clone", + Self::UpdateBases { .. } => "UpdateBases", + } + } +} diff --git a/rust/lance-table/src/transaction/proto.rs b/rust/lance-table/src/transaction/proto.rs new file mode 100644 index 00000000000..c51c8f16719 --- /dev/null +++ b/rust/lance-table/src/transaction/proto.rs @@ -0,0 +1,857 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Conversions between the transaction types and their protobuf encoding. +//! +//! A transaction is persisted as a `pb::Transaction` alongside the manifest it +//! produced, so these conversions are the format contract for everything in this +//! module: a field added to an `Operation` is only durable once it round-trips +//! here. + +use crate::format::key_existence::KeyExistenceFilter; +use crate::format::pb; +use crate::format::{BasePath, Fragment, IndexFile, IndexMetadata, overlay::DataOverlayFile}; +use crate::system_index::mem_wal::CompactedSsTable; +use crate::transaction::{ + DataOverlayGroup, DataReplacementGroup, Operation, RewriteGroup, RewrittenIndex, Transaction, + UpdateMap, UpdateMapEntry, UpdateMode, UpdatedFragmentOffsets, translate_config_updates, + translate_schema_metadata_updates, +}; +use lance_core::datatypes::Schema; +use lance_core::{Error, Result}; +use lance_file::datatypes::Fields; +use roaring::RoaringBitmap; +use std::collections::HashMap; +use std::sync::Arc; +use uuid::Uuid; + +impl From<&DataReplacementGroup> for pb::transaction::DataReplacementGroup { + fn from(DataReplacementGroup(fragment_id, new_file): &DataReplacementGroup) -> Self { + Self { + fragment_id: *fragment_id, + new_file: Some(new_file.into()), + } + } +} + +/// Convert a protobug DataReplacementGroup to a rust native DataReplacementGroup +/// this is unfortunately TryFrom instead of From because of the Option in the pb::DataReplacementGroup +impl TryFrom for DataReplacementGroup { + type Error = Error; + + fn try_from(message: pb::transaction::DataReplacementGroup) -> Result { + Ok(Self( + message.fragment_id, + message + .new_file + .ok_or(Error::invalid_input( + "DataReplacementGroup must have a new_file", + ))? + .try_into()?, + )) + } +} + +impl From<&DataOverlayGroup> for pb::transaction::DataOverlayGroup { + fn from(group: &DataOverlayGroup) -> Self { + Self { + fragment_id: group.fragment_id, + overlays: group + .overlays + .iter() + .map(pb::DataOverlayFile::from) + .collect(), + } + } +} + +impl TryFrom for DataOverlayGroup { + type Error = Error; + + fn try_from(message: pb::transaction::DataOverlayGroup) -> Result { + Ok(Self { + fragment_id: message.fragment_id, + overlays: message + .overlays + .into_iter() + .map(DataOverlayFile::try_from) + .collect::>>()?, + }) + } +} + +impl TryFrom for Transaction { + type Error = Error; + + fn try_from(message: pb::Transaction) -> Result { + let operation = match message.operation { + Some(pb::transaction::Operation::Append(pb::transaction::Append { fragments })) => { + Operation::Append { + fragments: fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + } + } + Some(pb::transaction::Operation::Clone(pb::transaction::Clone { + is_shallow, + ref_name, + ref_version, + ref_path, + branch_name, + })) => Operation::Clone { + is_shallow, + ref_name, + ref_version, + ref_path, + branch_name, + }, + Some(pb::transaction::Operation::Delete(pb::transaction::Delete { + updated_fragments, + deleted_fragment_ids, + predicate, + })) => Operation::Delete { + updated_fragments: updated_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + deleted_fragment_ids, + predicate, + }, + Some(pb::transaction::Operation::Overwrite(pb::transaction::Overwrite { + fragments, + schema, + schema_metadata: _schema_metadata, // TODO: handle metadata + config_upsert_values, + initial_bases, + })) => { + let config_upsert_option = if config_upsert_values.is_empty() { + None + } else { + Some(config_upsert_values) + }; + + Operation::Overwrite { + fragments: fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + schema: Schema::try_from(&Fields(schema))?, + config_upsert_values: config_upsert_option, + initial_bases: if initial_bases.is_empty() { + None + } else { + Some(initial_bases.into_iter().map(BasePath::from).collect()) + }, + } + } + Some(pb::transaction::Operation::ReserveFragments( + pb::transaction::ReserveFragments { num_fragments }, + )) => Operation::ReserveFragments { num_fragments }, + Some(pb::transaction::Operation::Rewrite(pb::transaction::Rewrite { + old_fragments, + new_fragments, + groups, + rewritten_indices, + })) => { + let groups = if !groups.is_empty() { + groups + .into_iter() + .map(RewriteGroup::try_from) + .collect::>()? + } else { + vec![RewriteGroup { + old_fragments: old_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + new_fragments: new_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + }] + }; + let rewritten_indices = rewritten_indices + .iter() + .map(RewrittenIndex::try_from) + .collect::>()?; + + Operation::Rewrite { + groups, + rewritten_indices, + frag_reuse_index: None, + } + } + Some(pb::transaction::Operation::CreateIndex(pb::transaction::CreateIndex { + new_indices, + removed_indices, + })) => Operation::CreateIndex { + new_indices: new_indices + .into_iter() + .map(IndexMetadata::try_from) + .collect::>()?, + removed_indices: removed_indices + .into_iter() + .map(IndexMetadata::try_from) + .collect::>()?, + }, + Some(pb::transaction::Operation::Merge(pb::transaction::Merge { + fragments, + schema, + schema_metadata: _schema_metadata, // TODO: handle metadata + preserves_nullability, + })) => Operation::Merge { + fragments: fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + schema: Schema::try_from(&Fields(schema))?, + // False for a writer that predates the field: no assertion, so + // a legacy required-field merge still conflicts and a legacy + // nullable merge over-conflicts, which only retries. + preserves_nullability, + }, + Some(pb::transaction::Operation::Restore(pb::transaction::Restore { version })) => { + Operation::Restore { version } + } + Some(pb::transaction::Operation::Update(pb::transaction::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + fields_modified, + compacted_sstables, + fields_for_preserving_frag_bitmap, + update_mode, + inserted_rows, + updated_fragment_offsets, + updated_fragment_offset_bitmaps, + })) => Operation::Update { + removed_fragment_ids, + updated_fragments: updated_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + new_fragments: new_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + fields_modified, + compacted_sstables: compacted_sstables + .into_iter() + .map(|m| CompactedSsTable::try_from(m).unwrap()) + .collect(), + fields_for_preserving_frag_bitmap, + update_mode: match update_mode { + 0 => Some(UpdateMode::RewriteRows), + 1 => Some(UpdateMode::RewriteColumns), + _ => Some(UpdateMode::RewriteRows), + }, + inserted_rows_filter: inserted_rows + .map(|ik| KeyExistenceFilter::try_from(&ik)) + .transpose()?, + updated_fragment_offsets: { + // Prefer field 10 (RoaringBitmap bytes); fall back to field 9 (UInt32List) + // for manifests written before this change. + let m: HashMap = + if !updated_fragment_offset_bitmaps.is_empty() { + updated_fragment_offset_bitmaps + .into_iter() + .filter(|(_, bytes)| !bytes.is_empty()) + .map(|(id, bytes)| { + let bitmap = RoaringBitmap::deserialize_from(bytes.as_slice()) + .map_err(|e| { + Error::invalid_input(format!( + "invalid updated_fragment_offset_bitmaps \ + for fragment {id}: {e}" + )) + })?; + Ok((id, bitmap)) + }) + .collect::>>()? + } else { + updated_fragment_offsets + .into_iter() + .filter(|(_, list)| !list.values.is_empty()) + .map(|(id, list)| (id, RoaringBitmap::from_iter(list.values))) + .collect() + }; + if m.is_empty() { + None + } else { + Some(UpdatedFragmentOffsets(m)) + } + }, + }, + Some(pb::transaction::Operation::Project(pb::transaction::Project { + schema, + preserves_nullability, + })) => Operation::Project { + schema: Schema::try_from(&Fields(schema))?, + // False for a writer that predates the field: no assertion, so + // a legacy tightening still conflicts and a legacy rename + // over-conflicts, which only retries. + preserves_nullability, + }, + Some(pb::transaction::Operation::UpdateConfig(update_config)) => { + // Check if new-style fields are present + let has_new_fields = update_config.config_updates.is_some() + || update_config.table_metadata_updates.is_some() + || update_config.schema_metadata_updates.is_some() + || !update_config.field_metadata_updates.is_empty(); + + // Check if old-style fields are present + let has_old_fields = !update_config.upsert_values.is_empty() + || !update_config.delete_keys.is_empty() + || !update_config.schema_metadata.is_empty() + || !update_config.field_metadata.is_empty(); + + // Error if both are present + if has_new_fields && has_old_fields { + return Err(Error::invalid_input_source( + "Cannot mix old and new style UpdateConfig fields".into(), + )); + } + + if has_old_fields { + // Translate old-style to new-style + let config_updates = if !update_config.upsert_values.is_empty() + || !update_config.delete_keys.is_empty() + { + Some(translate_config_updates( + &update_config.upsert_values, + &update_config.delete_keys, + )) + } else { + None + }; + + let schema_metadata_updates = if !update_config.schema_metadata.is_empty() { + Some(translate_schema_metadata_updates( + &update_config.schema_metadata, + )) + } else { + None + }; + + let field_metadata_updates = update_config + .field_metadata + .into_iter() + .map(|(field_id, field_meta_update)| { + ( + field_id as i32, + translate_schema_metadata_updates(&field_meta_update.metadata), + ) + }) + .collect(); + + Operation::UpdateConfig { + config_updates, + table_metadata_updates: None, + schema_metadata_updates, + field_metadata_updates, + } + } else { + // Use new-style fields directly (convert from protobuf) + Operation::UpdateConfig { + config_updates: update_config.config_updates.as_ref().map(UpdateMap::from), + table_metadata_updates: update_config + .table_metadata_updates + .as_ref() + .map(UpdateMap::from), + schema_metadata_updates: update_config + .schema_metadata_updates + .as_ref() + .map(UpdateMap::from), + field_metadata_updates: update_config + .field_metadata_updates + .iter() + .map(|(field_id, pb_update_map)| { + (*field_id, UpdateMap::from(pb_update_map)) + }) + .collect(), + } + } + } + Some(pb::transaction::Operation::DataReplacement( + pb::transaction::DataReplacement { replacements }, + )) => Operation::DataReplacement { + replacements: replacements + .into_iter() + .map(DataReplacementGroup::try_from) + .collect::>>()?, + }, + Some(pb::transaction::Operation::UpdateMemWalState( + pb::transaction::UpdateMemWalState { compacted_sstables }, + )) => Operation::UpdateMemWalState { + compacted_sstables: compacted_sstables + .into_iter() + .map(CompactedSsTable::try_from) + .collect::>()?, + }, + Some(pb::transaction::Operation::UpdateBases(pb::transaction::UpdateBases { + new_bases, + })) => Operation::UpdateBases { + new_bases: new_bases.into_iter().map(BasePath::from).collect(), + }, + Some(pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay { + groups, + })) => Operation::DataOverlay { + groups: groups + .into_iter() + .map(DataOverlayGroup::try_from) + .collect::>>()?, + }, + None => { + return Err(Error::internal( + "Transaction message did not contain an operation".to_string(), + )); + } + }; + Ok(Self { + read_version: message.read_version, + uuid: message.uuid.clone(), + operation, + tag: if message.tag.is_empty() { + None + } else { + Some(message.tag.clone()) + }, + transaction_properties: if message.transaction_properties.is_empty() { + None + } else { + Some(Arc::new(message.transaction_properties)) + }, + }) + } +} + +impl TryFrom<&pb::transaction::rewrite::RewrittenIndex> for RewrittenIndex { + type Error = Error; + + fn try_from(message: &pb::transaction::rewrite::RewrittenIndex) -> Result { + Ok(Self { + old_id: message + .old_id + .as_ref() + .map(Uuid::try_from) + .ok_or_else(|| { + Error::invalid_input("required field (old_id) missing from message".to_string()) + })??, + new_id: message + .new_id + .as_ref() + .map(Uuid::try_from) + .ok_or_else(|| { + Error::invalid_input("required field (new_id) missing from message".to_string()) + })??, + new_index_details: message + .new_index_details + .as_ref() + .ok_or_else(|| { + Error::invalid_input("new_index_details is a required field".to_string()) + })? + .clone(), + new_index_version: message.new_index_version, + new_index_files: if message.new_index_files.is_empty() { + None + } else { + Some( + message + .new_index_files + .iter() + .map(|f| IndexFile { + path: f.path.clone(), + size_bytes: f.size_bytes, + }) + .collect(), + ) + }, + }) + } +} + +impl TryFrom for RewriteGroup { + type Error = Error; + + fn try_from(message: pb::transaction::rewrite::RewriteGroup) -> Result { + Ok(Self { + old_fragments: message + .old_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + new_fragments: message + .new_fragments + .into_iter() + .map(Fragment::try_from) + .collect::>>()?, + }) + } +} + +impl From<&Transaction> for pb::Transaction { + fn from(value: &Transaction) -> Self { + let operation = match &value.operation { + Operation::Append { fragments } => { + pb::transaction::Operation::Append(pb::transaction::Append { + fragments: fragments.iter().map(pb::DataFragment::from).collect(), + }) + } + Operation::Clone { + is_shallow, + ref_name, + ref_version, + ref_path, + branch_name, + } => pb::transaction::Operation::Clone(pb::transaction::Clone { + is_shallow: *is_shallow, + ref_name: ref_name.clone(), + ref_version: *ref_version, + ref_path: ref_path.clone(), + branch_name: branch_name.clone(), + }), + Operation::Delete { + updated_fragments, + deleted_fragment_ids, + predicate, + } => pb::transaction::Operation::Delete(pb::transaction::Delete { + updated_fragments: updated_fragments + .iter() + .map(pb::DataFragment::from) + .collect(), + deleted_fragment_ids: deleted_fragment_ids.clone(), + predicate: predicate.clone(), + }), + Operation::Overwrite { + fragments, + schema, + config_upsert_values, + initial_bases, + } => { + pb::transaction::Operation::Overwrite(pb::transaction::Overwrite { + fragments: fragments.iter().map(pb::DataFragment::from).collect(), + schema: Fields::from(schema).0, + schema_metadata: Default::default(), // TODO: handle metadata + config_upsert_values: config_upsert_values + .clone() + .unwrap_or(Default::default()), + initial_bases: initial_bases + .as_ref() + .map(|paths| { + paths + .iter() + .cloned() + .map(|bp: BasePath| -> pb::BasePath { bp.into() }) + .collect::>() + }) + .unwrap_or_default(), + }) + } + Operation::ReserveFragments { num_fragments } => { + pb::transaction::Operation::ReserveFragments(pb::transaction::ReserveFragments { + num_fragments: *num_fragments, + }) + } + Operation::Rewrite { + groups, + rewritten_indices, + frag_reuse_index: _, + } => pb::transaction::Operation::Rewrite(pb::transaction::Rewrite { + groups: groups + .iter() + .map(pb::transaction::rewrite::RewriteGroup::from) + .collect(), + rewritten_indices: rewritten_indices + .iter() + .map(|rewritten| rewritten.into()) + .collect(), + ..Default::default() + }), + Operation::CreateIndex { + new_indices, + removed_indices, + } => pb::transaction::Operation::CreateIndex(pb::transaction::CreateIndex { + new_indices: new_indices.iter().map(pb::IndexMetadata::from).collect(), + removed_indices: removed_indices + .iter() + .map(pb::IndexMetadata::from) + .collect(), + }), + Operation::Merge { + fragments, + schema, + preserves_nullability, + } => pb::transaction::Operation::Merge(pb::transaction::Merge { + fragments: fragments.iter().map(pb::DataFragment::from).collect(), + schema: Fields::from(schema).0, + schema_metadata: Default::default(), // TODO: handle metadata + preserves_nullability: *preserves_nullability, + }), + Operation::Restore { version } => { + pb::transaction::Operation::Restore(pb::transaction::Restore { version: *version }) + } + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + fields_modified, + compacted_sstables, + fields_for_preserving_frag_bitmap, + update_mode, + inserted_rows_filter, + updated_fragment_offsets, + } => pb::transaction::Operation::Update(pb::transaction::Update { + removed_fragment_ids: removed_fragment_ids.clone(), + updated_fragments: updated_fragments + .iter() + .map(pb::DataFragment::from) + .collect(), + new_fragments: new_fragments.iter().map(pb::DataFragment::from).collect(), + fields_modified: fields_modified.clone(), + compacted_sstables: compacted_sstables + .iter() + .map(pb::CompactedSsTable::from) + .collect(), + fields_for_preserving_frag_bitmap: fields_for_preserving_frag_bitmap.clone(), + update_mode: update_mode + .as_ref() + .map(|mode| match mode { + UpdateMode::RewriteRows => 0, + UpdateMode::RewriteColumns => 1, + }) + .unwrap_or(0), + inserted_rows: inserted_rows_filter.as_ref().map(|ik| ik.into()), + // Field 9: no longer written; kept empty for forward compat. + updated_fragment_offsets: HashMap::new(), + // Field 10: RoaringBitmap bytes. + updated_fragment_offset_bitmaps: updated_fragment_offsets + .as_ref() + .map(|UpdatedFragmentOffsets(m)| { + m.iter() + .filter(|(_, b)| !b.is_empty()) + .map(|(frag_id, b)| { + let mut buf = Vec::new(); + b.serialize_into(&mut buf) + .expect("RoaringBitmap serialization cannot fail"); + (*frag_id, buf) + }) + .collect::>() + }) + .unwrap_or_default(), + }), + Operation::Project { + schema, + preserves_nullability, + } => pb::transaction::Operation::Project(pb::transaction::Project { + schema: Fields::from(schema).0, + preserves_nullability: *preserves_nullability, + }), + Operation::UpdateConfig { + config_updates, + table_metadata_updates, + schema_metadata_updates, + field_metadata_updates, + } => pb::transaction::Operation::UpdateConfig(pb::transaction::UpdateConfig { + config_updates: config_updates + .as_ref() + .map(pb::transaction::UpdateMap::from), + table_metadata_updates: table_metadata_updates + .as_ref() + .map(pb::transaction::UpdateMap::from), + schema_metadata_updates: schema_metadata_updates + .as_ref() + .map(pb::transaction::UpdateMap::from), + field_metadata_updates: field_metadata_updates + .iter() + .map(|(field_id, update_map)| { + (*field_id, pb::transaction::UpdateMap::from(update_map)) + }) + .collect(), + // Leave old fields empty - we only write new-style fields + upsert_values: Default::default(), + delete_keys: Default::default(), + schema_metadata: Default::default(), + field_metadata: Default::default(), + }), + Operation::DataReplacement { replacements } => { + pb::transaction::Operation::DataReplacement(pb::transaction::DataReplacement { + replacements: replacements + .iter() + .map(pb::transaction::DataReplacementGroup::from) + .collect(), + }) + } + Operation::DataOverlay { groups } => { + pb::transaction::Operation::DataOverlay(pb::transaction::DataOverlay { + groups: groups + .iter() + .map(pb::transaction::DataOverlayGroup::from) + .collect(), + }) + } + Operation::UpdateMemWalState { compacted_sstables } => { + pb::transaction::Operation::UpdateMemWalState(pb::transaction::UpdateMemWalState { + compacted_sstables: compacted_sstables + .iter() + .map(pb::CompactedSsTable::from) + .collect::>(), + }) + } + Operation::UpdateBases { new_bases } => { + pb::transaction::Operation::UpdateBases(pb::transaction::UpdateBases { + new_bases: new_bases + .iter() + .cloned() + .map(|bp: BasePath| -> pb::BasePath { bp.into() }) + .collect::>(), + }) + } + }; + + let transaction_properties = value + .transaction_properties + .as_ref() + .map(|arc| arc.as_ref().clone()) + .unwrap_or_default(); + Self { + read_version: value.read_version, + uuid: value.uuid.clone(), + operation: Some(operation), + tag: value.tag.clone().unwrap_or("".to_string()), + transaction_properties, + } + } +} + +impl From<&RewrittenIndex> for pb::transaction::rewrite::RewrittenIndex { + fn from(value: &RewrittenIndex) -> Self { + Self { + old_id: Some((&value.old_id).into()), + new_id: Some((&value.new_id).into()), + new_index_details: Some(value.new_index_details.clone()), + new_index_version: value.new_index_version, + new_index_files: value + .new_index_files + .as_ref() + .map(|files| { + files + .iter() + .map(|f| pb::IndexFile { + path: f.path.clone(), + size_bytes: f.size_bytes, + }) + .collect() + }) + .unwrap_or_default(), + } + } +} + +impl From<&RewriteGroup> for pb::transaction::rewrite::RewriteGroup { + fn from(value: &RewriteGroup) -> Self { + Self { + old_fragments: value + .old_fragments + .iter() + .map(pb::DataFragment::from) + .collect(), + new_fragments: value + .new_fragments + .iter() + .map(pb::DataFragment::from) + .collect(), + } + } +} + +impl From<&UpdateMap> for pb::transaction::UpdateMap { + fn from(update_map: &UpdateMap) -> Self { + Self { + update_entries: update_map + .update_entries + .iter() + .map(|entry| pb::transaction::UpdateMapEntry { + key: entry.key.clone(), + value: entry.value.clone(), + }) + .collect(), + replace: update_map.replace, + } + } +} + +impl From<&pb::transaction::UpdateMap> for UpdateMap { + fn from(pb_update_map: &pb::transaction::UpdateMap) -> Self { + Self { + update_entries: pb_update_map + .update_entries + .iter() + .map(|entry| UpdateMapEntry { + key: entry.key.clone(), + value: entry.value.clone(), + }) + .collect(), + replace: pb_update_map.replace, + } + } +} + +impl From<&Transaction> for crate::format::Transaction { + fn from(value: &Transaction) -> Self { + let pb_transaction: pb::Transaction = value.into(); + Self { + inner: pb_transaction, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::DataFile; + use crate::format::overlay::OverlayCoverage; + + #[test] + fn test_data_overlay_operation_roundtrips() { + // A DataOverlay operation survives the protobuf round-trip, preserving + // the target fragment, the overlay's coverage, and its committed_version. + let mut bitmap = roaring::RoaringBitmap::new(); + bitmap.insert(1); + bitmap.insert(4); + let overlay = DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay-0.lance", vec![3], None), + coverage: OverlayCoverage::dense(bitmap.clone()), + committed_version: 6, + }; + let pb_overlay = pb::DataOverlayFile::from(&overlay); + + let message = pb::Transaction { + read_version: 1, + uuid: Uuid::new_v4().to_string(), + operation: Some(pb::transaction::Operation::DataOverlay( + pb::transaction::DataOverlay { + groups: vec![pb::transaction::DataOverlayGroup { + fragment_id: 7, + overlays: vec![pb_overlay], + }], + }, + )), + ..Default::default() + }; + + let txn = Transaction::try_from(message).unwrap(); + match txn.operation { + Operation::DataOverlay { groups } => { + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].fragment_id, 7); + assert_eq!(groups[0].overlays.len(), 1); + assert_eq!(groups[0].overlays[0].committed_version, 6); + assert_eq!( + *groups[0].overlays[0].coverage_for_field(0).unwrap(), + bitmap + ); + } + other => panic!("expected DataOverlay, got {other:?}"), + } + } +} diff --git a/rust/lance-table/src/transaction/row_version.rs b/rust/lance-table/src/transaction/row_version.rs new file mode 100644 index 00000000000..71c6229aa46 --- /dev/null +++ b/rust/lance-table/src/transaction/row_version.rs @@ -0,0 +1,1139 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Row ids and the per-row version metadata that travels with them. +//! +//! Under stable row ids each fragment carries two run-length encoded sequences: +//! `created_at_version`, stamped once when a row first appears, and +//! `last_updated_at_version`, refreshed whenever a row's values change. Keeping +//! `created_at` correct across an update means tracing each new row back to the +//! fragment and offset it came from, which is what most of this module does. + +use crate::format::{ + Fragment, RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, +}; +use crate::rowids::segment::U64Segment; +use crate::rowids::version::build_version_meta; +use crate::rowids::{RowIdSequence, read_row_ids, write_row_ids}; +use crate::transaction::Transaction; +use lance_core::{Error, Result}; +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; + +/// Fallback version for rows whose original creation version cannot be determined. +/// Version 1 is the initial dataset version in the Lance format. +const UNKNOWN_CREATED_AT_VERSION: u64 = 1; + +/// Look up the `created_at` version for a single UPDATE-branch row ID. +/// +/// Callers must only call this for row IDs that are confirmed to be present in +/// `row_id_to_source` (i.e. UPDATE branch rows whose source exists in an existing +/// fragment). INSERT branch rows (no source) must use `new_version` directly and +/// must not call this function. +/// +/// Uses `row_id_to_source` to find the originating fragment and row offset, then +/// performs a O(K) random-access lookup via [`RowDatasetVersionSequence::version_at`] +/// on the pre-decoded sequence in `version_cache` (keyed by fragment ID). +/// +/// Returns [`UNKNOWN_CREATED_AT_VERSION`] if the source fragment has no +/// `created_at_version_meta` (missing or failed to decode) or the offset is +/// out of range. +fn resolve_created_at_version( + row_id: u64, + row_id_to_source: &HashMap, + version_cache: &HashMap, +) -> u64 { + let Some((orig_frag, row_offset)) = row_id_to_source.get(&row_id) else { + return UNKNOWN_CREATED_AT_VERSION; + }; + let Some(seq) = version_cache.get(&orig_frag.id) else { + return UNKNOWN_CREATED_AT_VERSION; + }; + seq.version_at(*row_offset) + .unwrap_or(UNKNOWN_CREATED_AT_VERSION) +} + +/// For each new fragment produced by an update, set `created_at_version_meta` +/// (preserved from the original rows) and `last_updated_at_version_meta`. +pub(super) fn resolve_update_version_metadata( + existing_fragments: &[Fragment], + new_fragments: &mut [Fragment], + new_version: u64, +) -> Result<()> { + // Collect only the row IDs we actually need to resolve, those appearing in new_fragments + // with inline metadata. This bounds the lookup map to O(updated rows) instead of O(all dataset rows) + let needed_row_ids: HashSet = new_fragments + .iter() + .filter_map(|f| match &f.row_id_meta { + Some(RowIdMeta::Inline(data)) => read_row_ids(data).ok(), + _ => None, + }) + .flat_map(|seq| seq.iter().collect::>()) + .collect(); + + let mut row_id_to_source: HashMap = HashMap::new(); + + if !needed_row_ids.is_empty() { + // Compute the bounding range of the needed set once. Any fragment whose + // entire row-id range lies outside [needed_min, needed_max] cannot contain + // any needed ID and can be skipped before the inner per-row loop. + let needed_min = *needed_row_ids.iter().min().unwrap(); + let needed_max = *needed_row_ids.iter().max().unwrap(); + + // Stable row IDs must be globally unique among *live* rows, but after a rewrite-style + // update the same stable ID can appear twice in `existing_fragments`: once in an older + // fragment's inline `row_id_meta` at the original row offset (rows may be soft-deleted + // via a deletion vector) and again in a newer fragment holding rewritten data. For + // `created_at` we need the mapping from the original fragment/offset; that is always the + // first occurrence when fragments are processed in ascending `id` order. + let mut sorted_frags: Vec<&Fragment> = existing_fragments.iter().collect(); + sorted_frags.sort_by_key(|f| f.id); + for frag in sorted_frags { + if let Some(RowIdMeta::Inline(data)) = &frag.row_id_meta + && let Ok(seq) = read_row_ids(data) + { + // Range pre-filter: skip the per-row inner loop when the fragment's + // bounding row-id range has no overlap with [needed_min, needed_max]. + // row_id_range() returns None for empty sequences, which are also skipped. + // This is a conservative check (may produce false positives for sparse + // segments) but never skips a fragment that actually contains a needed ID. + if seq + .row_id_range() + .is_none_or(|r| *r.end() < needed_min || *r.start() > needed_max) + { + continue; + } + + for (offset, rid) in seq.iter().enumerate() { + if needed_row_ids.contains(&rid) { + row_id_to_source.entry(rid).or_insert((frag, offset)); + } + } + } + } + } + + // Pre-decode the `created_at` version sequence for each source fragment exactly + // once. Without this cache, resolve_created_at_version would call load_sequence() + // (a protobuf decode) for every single updated row, even when many rows originate + // from the same fragment. + let source_frag_ids: HashSet = row_id_to_source.values().map(|(f, _)| f.id).collect(); + let version_cache: HashMap = existing_fragments + .iter() + .filter(|f| source_frag_ids.contains(&f.id)) + .filter_map(|frag| { + let seq = frag + .created_at_version_meta + .as_ref()? + .load_sequence() + .ok()?; + Some((frag.id, seq)) + }) + .collect(); + + for fragment in new_fragments.iter_mut() { + let row_ids = match &fragment.row_id_meta { + Some(RowIdMeta::Inline(data)) => read_row_ids(data).ok(), + Some(RowIdMeta::External(_)) => { + log::warn!( + "Fragment {} has external row ID metadata; \ + version tracking will use defaults", + fragment.id, + ); + None + } + None => None, + }; + + if let Some(row_ids) = row_ids { + let physical_rows = fragment.physical_rows.unwrap_or(0); + let created_at_versions: Vec = row_ids + .iter() + .map(|rid| { + if row_id_to_source.contains_key(&rid) { + // UPDATE branch: stable row ID resolves to a source row in an + // existing fragment. Copy created_at from the original row so + // the row's first-appearance version is preserved across rewrites. + resolve_created_at_version(rid, &row_id_to_source, &version_cache) + } else { + // INSERT branch: stable row ID has no source in existing fragments + // (e.g. NOT MATCHED arm of MERGE INTO). The row first appears in + // this commit, so created_at equals the new commit version. + new_version + } + }) + .collect(); + debug_assert_eq!(created_at_versions.len(), physical_rows); + + let runs = encode_version_runs(&created_at_versions); + let created_at_seq = RowDatasetVersionSequence { runs }; + fragment.created_at_version_meta = Some( + RowDatasetVersionMeta::from_sequence(&created_at_seq).map_err(|e| { + Error::internal(format!( + "Failed to create created_at version metadata: {}", + e + )) + })?, + ); + + fragment.last_updated_at_version_meta = build_version_meta(fragment, new_version); + } else { + let version_meta = build_version_meta(fragment, new_version); + fragment.last_updated_at_version_meta = version_meta.clone(); + fragment.created_at_version_meta = version_meta; + } + } + Ok(()) +} + +/// Run-length encode a sequence of per-row versions into [`RowDatasetVersionRun`]s. +fn encode_version_runs(versions: &[u64]) -> Vec { + if versions.is_empty() { + return Vec::new(); + } + let mut runs = Vec::new(); + let mut current_version = versions[0]; + let mut run_start = 0u64; + for (i, &version) in versions.iter().enumerate().skip(1) { + if version != current_version { + runs.push(RowDatasetVersionRun { + span: U64Segment::Range(run_start..i as u64), + version: current_version, + }); + current_version = version; + run_start = i as u64; + } + } + runs.push(RowDatasetVersionRun { + span: U64Segment::Range(run_start..versions.len() as u64), + version: current_version, + }); + runs +} + +impl Transaction { + /// collect the pure(the num of row IDs are equal to the physical rows) "rewrite rows" updated fragment ids + pub(super) fn collect_pure_rewrite_row_update_frags_ids( + fragments: &[Fragment], + ) -> Result> { + let mut pure_update_frag_ids = Vec::new(); + + for fragment in fragments { + let physical_rows = fragment + .physical_rows + .ok_or_else(|| Error::internal("Fragment does not have physical rows"))? + as u64; + + if let Some(row_id_meta) = &fragment.row_id_meta { + let existing_row_count = match row_id_meta { + RowIdMeta::Inline(data) => { + let sequence = read_row_ids(data)?; + sequence.len() as u64 + } + _ => 0, + }; + + // only filter the fragments that match: all the rows have row id, + // which means it does not contain inserted rows in this fragment + if existing_row_count == physical_rows { + pure_update_frag_ids.push(fragment.id); + } + } + } + + Ok(pure_update_frag_ids) + } + + pub(super) fn assign_row_ids(next_row_id: &mut u64, fragments: &mut [Fragment]) -> Result<()> { + for fragment in fragments { + let physical_rows = fragment + .physical_rows + .ok_or_else(|| Error::internal("Fragment does not have physical rows"))? + as u64; + + if fragment.row_id_meta.is_some() { + // we may meet merge insert case, it only has partial row ids. + // so here, we need to check if the row ids match the physical rows + // if yes, continue + // if not, fill the remaining row ids to the physical rows, then update row_id_meta + + // Check if existing row IDs match the physical rows count + let existing_row_count = match &fragment.row_id_meta { + Some(RowIdMeta::Inline(data)) => { + // Parse the serialized row ID sequence to get the count + let sequence = read_row_ids(data)?; + sequence.len() as u64 + } + _ => 0, + }; + + match existing_row_count.cmp(&physical_rows) { + Ordering::Equal => { + // Row IDs already match physical rows, continue to next fragment + continue; + } + Ordering::Less => { + // Partial row IDs - need to fill the remaining ones + let remaining_rows = physical_rows - existing_row_count; + let new_row_ids = *next_row_id..(*next_row_id + remaining_rows); + + // Merge existing and new row IDs + let combined_sequence = match &fragment.row_id_meta { + Some(RowIdMeta::Inline(data)) => read_row_ids(data)?, + _ => { + return Err(Error::internal( + "Failed to deserialize existing row ID sequence", + )); + } + }; + + let mut row_ids: Vec = combined_sequence.iter().collect(); + for row_id in new_row_ids { + row_ids.push(row_id); + } + let combined_sequence = RowIdSequence::from(row_ids.as_slice()); + + let serialized = write_row_ids(&combined_sequence); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); + *next_row_id += remaining_rows; + } + Ordering::Greater => { + // More row IDs than physical rows - this shouldn't happen + return Err(Error::internal(format!( + "Fragment has more row IDs ({}) than physical rows ({})", + existing_row_count, physical_rows + ))); + } + } + } else { + let row_ids = *next_row_id..(*next_row_id + physical_rows); + let sequence = RowIdSequence::from(row_ids); + // TODO: write to a separate file if large. Possibly share a file with other fragments. + let serialized = write_row_ids(&sequence); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); + *next_row_id += physical_rows; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::transaction::test_support::{ + created_at_versions, default_build_config, last_updated_at_versions, + make_stable_row_id_manifest, update_txn, + }; + use std::sync::Arc; + + #[test] + fn test_assign_row_ids_new_fragment() { + // Test assigning row IDs to a fragment without existing row IDs + let mut fragments = vec![Fragment { + id: 1, + physical_rows: Some(100), + row_id_meta: None, + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }]; + let mut next_row_id = 0; + + Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); + + assert_eq!(next_row_id, 100); + assert!(fragments[0].row_id_meta.is_some()); + + if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { + let sequence = read_row_ids(data).unwrap(); + assert_eq!(sequence.len(), 100); + let row_ids: Vec = sequence.iter().collect(); + assert_eq!(row_ids, (0..100).collect::>()); + } else { + panic!("Expected inline row ID metadata"); + } + } + + #[test] + fn test_assign_row_ids_existing_complete() { + // Test with fragment that already has complete row IDs + let existing_sequence = RowIdSequence::from(0..50); + let serialized = write_row_ids(&existing_sequence); + + let mut fragments = vec![Fragment { + id: 1, + physical_rows: Some(50), + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }]; + let mut next_row_id = 100; + + Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); + + // next_row_id should not change + assert_eq!(next_row_id, 100); + + if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { + let sequence = read_row_ids(data).unwrap(); + assert_eq!(sequence.len(), 50); + let row_ids: Vec = sequence.iter().collect(); + assert_eq!(row_ids, (0..50).collect::>()); + } else { + panic!("Expected inline row ID metadata"); + } + } + + #[test] + fn test_assign_row_ids_partial_existing() { + // Test with fragment that has partial row IDs (merge insert case) + let existing_sequence = RowIdSequence::from(0..30); + let serialized = write_row_ids(&existing_sequence); + + let mut fragments = vec![Fragment { + id: 1, + physical_rows: Some(50), // More physical rows than existing row IDs + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }]; + let mut next_row_id = 100; + + Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); + + // next_row_id should advance by 20 (50 - 30) + assert_eq!(next_row_id, 120); + + if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { + let sequence = read_row_ids(data).unwrap(); + assert_eq!(sequence.len(), 50); + let row_ids: Vec = sequence.iter().collect(); + // Should contain original 0-29 plus new 100-119 + let mut expected = (0..30).collect::>(); + expected.extend(100..120); + assert_eq!(row_ids, expected); + } else { + panic!("Expected inline row ID metadata"); + } + } + + #[test] + fn test_assign_row_ids_excess_row_ids() { + // Test error case where fragment has more row IDs than physical rows + let existing_sequence = RowIdSequence::from(0..60); + let serialized = write_row_ids(&existing_sequence); + + let mut fragments = vec![Fragment { + id: 1, + physical_rows: Some(50), // Less physical rows than existing row IDs + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }]; + let mut next_row_id = 100; + + let result = Transaction::assign_row_ids(&mut next_row_id, &mut fragments); + + assert!(result.is_err()); + if let Err(Error::Internal { message, .. }) = result { + assert!(message.contains("more row IDs (60) than physical rows (50)")); + } else { + panic!("Expected Internal error about excess row IDs"); + } + } + + #[test] + fn test_assign_row_ids_multiple_fragments() { + // Test with multiple fragments, some with existing row IDs, some without + let existing_sequence = RowIdSequence::from(500..520); + let serialized = write_row_ids(&existing_sequence); + + let mut fragments = vec![ + Fragment { + id: 1, + physical_rows: Some(30), // No existing row IDs + row_id_meta: None, + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }, + Fragment { + id: 2, + physical_rows: Some(25), // Partial existing row IDs + row_id_meta: Some(RowIdMeta::Inline(serialized.into())), + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }, + ]; + let mut next_row_id = 1000; + + Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); + + // Should advance by 30 (first fragment) + 5 (second fragment partial) + assert_eq!(next_row_id, 1035); + + // Check first fragment + if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { + let sequence = read_row_ids(data).unwrap(); + assert_eq!(sequence.len(), 30); + let row_ids: Vec = sequence.iter().collect(); + assert_eq!(row_ids, (1000..1030).collect::>()); + } else { + panic!("Expected inline row ID metadata for first fragment"); + } + + // Check second fragment + if let Some(RowIdMeta::Inline(data)) = &fragments[1].row_id_meta { + let sequence = read_row_ids(data).unwrap(); + assert_eq!(sequence.len(), 25); + let row_ids: Vec = sequence.iter().collect(); + // Should contain original 500-519 plus new 1030-1034 + let mut expected = (500..520).collect::>(); + expected.extend(1030..1035); + assert_eq!(row_ids, expected); + } else { + panic!("Expected inline row ID metadata for second fragment"); + } + } + + #[test] + fn test_assign_row_ids_missing_physical_rows() { + // Test error case where fragment doesn't have physical_rows set + let mut fragments = vec![Fragment { + id: 1, + physical_rows: None, + row_id_meta: None, + files: vec![], + overlays: vec![], + deletion_file: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }]; + let mut next_row_id = 0; + + let result = Transaction::assign_row_ids(&mut next_row_id, &mut fragments); + + assert!(result.is_err()); + if let Err(Error::Internal { message, .. }) = result { + assert!(message.contains("Fragment does not have physical rows")); + } else { + panic!("Expected Internal error about missing physical rows"); + } + } + + #[test] + fn test_update_version_tracking_preserves_created_at() { + let existing_seq = RowIdSequence::from([100u64, 101, 102].as_slice()); + let created_at_seq = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..3), + version: 5, + }], + }; + let existing_fragment = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), + physical_rows: Some(3), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&created_at_seq).unwrap(), + ), + last_updated_at_version_meta: None, + }; + + let new_seq = RowIdSequence::from([100u64, 102].as_slice()); + let new_fragment = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![existing_fragment]); + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + assert_eq!(created_at_versions(&result, 10), vec![5, 5]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); + } + + #[test] + fn test_update_version_tracking_mixed_origins() { + let frag_a_seq = RowIdSequence::from([10u64, 11].as_slice()); + let frag_a_created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..2), + version: 2, + }], + }; + let frag_b_seq = RowIdSequence::from([20u64, 21, 22].as_slice()); + let frag_b_created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..3), + version: 3, + }], + }; + + let manifest = make_stable_row_id_manifest(vec![ + Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_a_seq).into())), + physical_rows: Some(2), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&frag_a_created).unwrap(), + ), + last_updated_at_version_meta: None, + }, + Fragment { + id: 2, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_b_seq).into())), + physical_rows: Some(3), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&frag_b_created).unwrap(), + ), + last_updated_at_version_meta: None, + }, + ]); + + // New fragment has rows from both original fragments: row 11 from frag_a, row 20 from frag_b + let new_seq = RowIdSequence::from([11u64, 20].as_slice()); + let new_fragment = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Row 11 came from frag_a (offset 1, version 2), row 20 came from frag_b (offset 0, version 3) + assert_eq!(created_at_versions(&result, 10), vec![2, 3]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); + } + + #[test] + fn test_update_version_tracking_insert_branch_gets_new_version() { + // Simulates the INSERT branch (NOT MATCHED) of a MERGE INTO commit: + // the new fragment contains a mix of rewritten rows (UPDATE branch, row ID + // present in existing fragments) and freshly inserted rows (INSERT branch, + // row ID not present in any existing fragment). + // + // UPDATE branch row (10): created_at must be copied from the source fragment. + // INSERT branch row (999): created_at must equal new_version (the merge commit + // version), because the row first appeared in this commit. + let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); + let existing_created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..2), + version: 5, + }], + }; + let existing_fragment = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), + physical_rows: Some(2), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&existing_created).unwrap(), + ), + last_updated_at_version_meta: None, + }; + + // New fragment has row 10 (UPDATE branch) and row 999 (INSERT branch) + let new_seq = RowIdSequence::from([10u64, 999].as_slice()); + let new_fragment = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + // update_txn uses read_version 4 → new_version is 5 + let manifest = make_stable_row_id_manifest(vec![existing_fragment]); + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Row 10 (UPDATE branch): created_at copied from source (version 5). + // Row 999 (INSERT branch): created_at == new_version (5). + assert_eq!(created_at_versions(&result, 10), vec![5, 5]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); + } + + #[test] + fn test_update_version_tracking_merge_into_distinguishes_insert_and_update_branch() { + // Verifies the MERGE INTO correctness contract when UPDATE branch rows and INSERT + // branch rows have *different* source created_at values, so we can distinguish + // which row got which value. + // + // Existing fragment (id=1): row IDs [10, 11], created_at = version 3. + // New fragment (id=20): row IDs [10, 500, 11, 501]. + // - Rows 10 and 11: UPDATE branch (present in existing fragment) → created_at = 3. + // - Rows 500 and 501: INSERT branch (no source) → created_at = new_version = 5. + let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); + let existing_created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..2), + version: 3, + }], + }; + let existing_fragment = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), + physical_rows: Some(2), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&existing_created).unwrap(), + ), + last_updated_at_version_meta: None, + }; + + let new_seq = RowIdSequence::from([10u64, 500, 11, 501].as_slice()); + let new_fragment = Fragment { + id: 20, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(4), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + // update_txn uses read_version 4 → new_version is 5 + let manifest = make_stable_row_id_manifest(vec![existing_fragment]); + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // UPDATE branch rows (10, 11): created_at preserved from source (version 3). + // INSERT branch rows (500, 501): created_at == new_version (5). + assert_eq!(created_at_versions(&result, 20), vec![3, 5, 3, 5]); + // All rows in the new fragment get last_updated == new_version. + assert_eq!(last_updated_at_versions(&result, 20), vec![5, 5, 5, 5]); + } + + #[test] + fn test_update_version_tracking_source_fragment_no_created_at_defaults_to_1() { + // Source fragment has row_id_meta but no created_at_version_meta. + // The row IS found in the lookup, but the version defaults to 1. + let existing_seq = RowIdSequence::from([50u64, 51].as_slice()); + let existing_fragment = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let new_seq = RowIdSequence::from([50u64].as_slice()); + let new_fragment = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(1), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![existing_fragment]); + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Row 50 is found in source but source has no created_at_version_meta → default 1 + assert_eq!(created_at_versions(&result, 10), vec![1]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5]); + } + + #[test] + fn test_update_version_tracking_no_row_id_meta_fallback() { + let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); + let existing_fragment = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let new_fragment = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: None, + physical_rows: Some(3), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![existing_fragment]); + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Fragment starts with no row_id_meta → assign_row_ids gives it fresh IDs → + // those IDs have no source in existing fragments (INSERT branch) → + // created_at == new_version (5) for each row. + assert_eq!(created_at_versions(&result, 10), vec![5, 5, 5]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5, 5]); + } + + #[test] + fn test_update_version_tracking_corrupt_created_at_defaults_to_1() { + let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); + let existing_fragment = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq).into())), + physical_rows: Some(2), + created_at_version_meta: Some(RowDatasetVersionMeta::Inline(Arc::from( + vec![0xFFu8; 8].as_slice(), + ))), + last_updated_at_version_meta: None, + }; + + let new_seq = RowIdSequence::from([10u64].as_slice()); + let new_fragment = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(1), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![existing_fragment]); + let (result, _) = update_txn(vec![new_fragment]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Corrupt metadata causes decode to fail → falls back to UNKNOWN_CREATED_AT_VERSION (1) + assert_eq!(created_at_versions(&result, 10), vec![1]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5]); + } + + /// Fragments whose row-ID range lies entirely outside the needed set must not + /// affect the result. Here fragment 1 has IDs [1000, 1001] which are far above + /// the needed range [10, 11]; it is skipped by the range pre-filter and its + /// created_at version (version 99) must never appear in the output. + #[test] + fn test_update_version_tracking_range_filter_skips_non_overlapping_fragment() { + // Fragment in range – IDs [10, 11], created_at = 5 + let in_range_seq = RowIdSequence::from([10u64, 11].as_slice()); + let in_range_created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..2), + version: 5, + }], + }; + let in_range_frag = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&in_range_seq).into())), + physical_rows: Some(2), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&in_range_created).unwrap(), + ), + last_updated_at_version_meta: None, + }; + + // Fragment outside range – IDs [1000, 1001], created_at = 99 (must never appear) + let out_of_range_seq = RowIdSequence::from([1000u64, 1001].as_slice()); + let out_of_range_created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..2), + version: 99, + }], + }; + let out_of_range_frag = Fragment { + id: 2, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&out_of_range_seq).into())), + physical_rows: Some(2), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&out_of_range_created).unwrap(), + ), + last_updated_at_version_meta: None, + }; + + // New fragment rewrites both rows from the in-range fragment + let new_seq = RowIdSequence::from([10u64, 11].as_slice()); + let new_frag = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![in_range_frag, out_of_range_frag]); + let (result, _) = update_txn(vec![new_frag]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Both rows originate from the in-range fragment (version 5). + // The out-of-range fragment's version 99 must not appear. + assert_eq!(created_at_versions(&result, 10), vec![5, 5]); + assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); + } + + /// When the needed row IDs fall exactly at the boundary of a fragment's range, + /// the range pre-filter must NOT skip the fragment (boundary values are inclusive). + #[test] + fn test_update_version_tracking_range_filter_boundary_inclusive() { + // Fragment IDs [10, 11, 12], created_at = 7 + let seq = RowIdSequence::from([10u64, 11, 12].as_slice()); + let created = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..3), + version: 7, + }], + }; + let existing = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq).into())), + physical_rows: Some(3), + created_at_version_meta: Some(RowDatasetVersionMeta::from_sequence(&created).unwrap()), + last_updated_at_version_meta: None, + }; + + // New fragment takes the boundary IDs: 10 (min) and 12 (max) + let new_seq = RowIdSequence::from([10u64, 12].as_slice()); + let new_frag = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![existing]); + let (result, _) = update_txn(vec![new_frag]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Boundary IDs must be found and resolved correctly + assert_eq!(created_at_versions(&result, 10), vec![7, 7]); + } + + /// When multiple updated rows all originate from the same source fragment, + /// the created_at version sequence for that fragment must be decoded exactly + /// once (not once per row). The observable correctness requirement is that + /// all rows get the right version regardless of how many there are. + #[test] + fn test_update_version_tracking_many_rows_same_source_fragment() { + // Source fragment: 100 rows with IDs 0..100, mixed versions (2 runs). + // First 50 rows at version 3, next 50 rows at version 4. + let src_ids: Vec = (0u64..100).collect(); + let src_seq = RowIdSequence::from(src_ids.as_slice()); + let src_created = RowDatasetVersionSequence { + runs: vec![ + RowDatasetVersionRun { + span: U64Segment::Range(0..50), + version: 3, + }, + RowDatasetVersionRun { + span: U64Segment::Range(0..50), + version: 4, + }, + ], + }; + let src_frag = Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&src_seq).into())), + physical_rows: Some(100), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&src_created).unwrap(), + ), + last_updated_at_version_meta: None, + }; + + // New fragment rewrites all 100 rows preserving their stable IDs. + let new_seq = RowIdSequence::from(src_ids.as_slice()); + let new_frag = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(100), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let manifest = make_stable_row_id_manifest(vec![src_frag]); + let (result, _) = update_txn(vec![new_frag]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + let versions = created_at_versions(&result, 10); + assert_eq!(versions.len(), 100); + // First 50 rows came from version 3, next 50 from version 4 + assert!(versions[..50].iter().all(|&v| v == 3)); + assert!(versions[50..].iter().all(|&v| v == 4)); + } + + /// Rows originating from multiple distinct source fragments must each get + /// the version from their own source, even when all cached together. + #[test] + fn test_update_version_tracking_cache_multiple_source_fragments() { + let seq_a = RowIdSequence::from([10u64, 11, 12].as_slice()); + let created_a = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..3), + version: 2, + }], + }; + let seq_b = RowIdSequence::from([20u64, 21, 22].as_slice()); + let created_b = RowDatasetVersionSequence { + runs: vec![RowDatasetVersionRun { + span: U64Segment::Range(0..3), + version: 8, + }], + }; + + let manifest = make_stable_row_id_manifest(vec![ + Fragment { + id: 1, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_a).into())), + physical_rows: Some(3), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&created_a).unwrap(), + ), + last_updated_at_version_meta: None, + }, + Fragment { + id: 2, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_b).into())), + physical_rows: Some(3), + created_at_version_meta: Some( + RowDatasetVersionMeta::from_sequence(&created_b).unwrap(), + ), + last_updated_at_version_meta: None, + }, + ]); + + // New fragment takes rows from both sources: 12 (frag A, offset 2) and 20 (frag B, offset 0) + let new_seq = RowIdSequence::from([12u64, 20].as_slice()); + let new_frag = Fragment { + id: 10, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq).into())), + physical_rows: Some(2), + created_at_version_meta: None, + last_updated_at_version_meta: None, + }; + + let (result, _) = update_txn(vec![new_frag]) + .build_manifest(Some(&manifest), vec![], "txn", &default_build_config()) + .unwrap(); + + // Row 12 → frag A offset 2 → version 2; row 20 → frag B offset 0 → version 8 + assert_eq!(created_at_versions(&result, 10), vec![2, 8]); + } + + #[test] + fn test_encode_version_runs_empty() { + let runs = encode_version_runs(&[]); + assert!(runs.is_empty()); + } + + #[test] + fn test_encode_version_runs_single_run() { + let runs = encode_version_runs(&[3, 3, 3]); + assert_eq!(runs.len(), 1); + assert_eq!(runs[0].version, 3); + } + + #[test] + fn test_encode_version_runs_alternating() { + let runs = encode_version_runs(&[1, 2, 1, 2]); + assert_eq!(runs.len(), 4); + assert_eq!(runs[0].version, 1); + assert_eq!(runs[1].version, 2); + assert_eq!(runs[2].version, 1); + assert_eq!(runs[3].version, 2); + } +} diff --git a/rust/lance-table/src/transaction/test_support.rs b/rust/lance-table/src/transaction/test_support.rs new file mode 100644 index 00000000000..895abfbb746 --- /dev/null +++ b/rust/lance-table/src/transaction/test_support.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Fixtures shared between the tests of several submodules. + +use crate::feature_flags::FLAG_STABLE_ROW_IDS; +use crate::format::overlay::{DataOverlayFile, OverlayCoverage}; +use crate::format::{ + DataFile, DataStorageFormat, Fragment, IndexMetadata, Manifest, ManifestBuildConfig, +}; +use crate::transaction::{Operation, Transaction}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use chrono::Utc; +use lance_core::datatypes::Schema as LanceSchema; +use lance_file::version::ConcreteFileVersion; +use std::collections::HashMap; +use std::sync::Arc; +use uuid::Uuid; + +/// The build config that `lance`'s `ManifestWriteConfig::default()` resolves to. +pub fn default_build_config() -> ManifestBuildConfig { + ManifestBuildConfig { + auto_set_feature_flags: true, + timestamp_nanos: std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos(), + use_stable_row_ids: false, + use_legacy_format: None, + storage_format: None, + disable_transaction_file: false, + migration_next_row_id: None, + } +} + +pub fn sample_manifest() -> Manifest { + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(vec![Fragment::new(0)]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ) +} + +pub fn sample_index_metadata(name: &str) -> IndexMetadata { + IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![0], + covering_fields: vec![], + name: name.to_string(), + dataset_version: 0, + fragment_bitmap: Some([0].into_iter().collect()), + index_details: None, + index_version: 1, + created_at: Some(Utc::now()), + base_id: None, + files: None, + } +} + +pub fn overlay_with_field(field: i32, committed_version: u64) -> DataOverlayFile { + DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("o.lance", vec![field], None), + coverage: OverlayCoverage::dense(roaring::RoaringBitmap::from_iter([0u32])), + committed_version, + } +} + +/// Existing fragments use id >= 1 to avoid collision with `Fragment::new(0)` +/// used by `sample_manifest`. New (updated) fragments use id = 10. +pub fn make_stable_row_id_manifest(fragments: Vec) -> Manifest { + let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); + let mut manifest = Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(fragments), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + manifest.reader_feature_flags = FLAG_STABLE_ROW_IDS; + manifest.next_row_id = 1000; + manifest.version = 4; + manifest +} + +pub fn update_txn(new_fragments: Vec) -> Transaction { + Transaction::new( + 4, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments, + fields_modified: vec![], + compacted_sstables: vec![], + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ) +} + +pub fn created_at_versions(manifest: &Manifest, frag_id: u64) -> Vec { + let frag = manifest.fragments.iter().find(|f| f.id == frag_id).unwrap(); + let seq = frag + .created_at_version_meta + .as_ref() + .unwrap() + .load_sequence() + .unwrap(); + seq.versions().collect() +} + +pub fn last_updated_at_versions(manifest: &Manifest, frag_id: u64) -> Vec { + let frag = manifest.fragments.iter().find(|f| f.id == frag_id).unwrap(); + let seq = frag + .last_updated_at_version_meta + .as_ref() + .unwrap() + .load_sequence() + .unwrap(); + seq.versions().collect() +} diff --git a/rust/lance-table/src/transaction/update_map.rs b/rust/lance-table/src/transaction/update_map.rs new file mode 100644 index 00000000000..b7d09609bc4 --- /dev/null +++ b/rust/lance-table/src/transaction/update_map.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Incremental edits to the string maps a manifest carries. +//! +//! Dataset config, table metadata, schema metadata and per-field metadata are all +//! `HashMap`, and all four are updated the same way: a list of +//! entries where a `None` value means delete the key, plus a flag choosing between +//! merging into the existing map and replacing it outright. + +use lance_core::deepsize::DeepSizeOf; + +/// An entry for a map update. If value is None, the key will be removed from the map. +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct UpdateMapEntry { + /// The key of the map entry to update. + pub key: String, + /// The value to set for the key. + pub value: Option, +} + +impl From<(String, Option)> for UpdateMapEntry { + fn from((key, value): (String, Option)) -> Self { + Self { key, value } + } +} + +impl From<(String, String)> for UpdateMapEntry { + fn from((key, value): (String, String)) -> Self { + Self::from((key, Some(value))) + } +} + +impl From<(&str, Option<&str>)> for UpdateMapEntry { + fn from((key, value): (&str, Option<&str>)) -> Self { + Self { + key: key.to_string(), + value: value.map(str::to_owned), + } + } +} + +impl From<(&str, &str)> for UpdateMapEntry { + fn from((key, value): (&str, &str)) -> Self { + Self::from((key, Some(value))) + } +} + +/// Represents updates to a map (either incremental or replacement) +#[derive(Debug, Clone, DeepSizeOf, PartialEq)] +pub struct UpdateMap { + pub update_entries: Vec, + /// If true, the map will be replaced entirely with the new entries. + /// If false, the new entries will be merged with the existing map. + pub replace: bool, +} + +/// Helper function to apply UpdateMap changes to a HashMap +pub(super) fn apply_update_map( + target: &mut std::collections::HashMap, + update_map: &UpdateMap, +) { + if update_map.replace { + // Full replacement - clear existing and replace with new entries that have values + target.clear(); + for entry in &update_map.update_entries { + if let Some(value) = &entry.value { + target.insert(entry.key.clone(), value.clone()); + } + } + } else { + // Incremental update - merge entries + for entry in &update_map.update_entries { + if let Some(value) = &entry.value { + target.insert(entry.key.clone(), value.clone()); + } else { + target.remove(&entry.key); + } + } + } +} + +/// Helper function to translate old-style config updates to new UpdateMap format +pub fn translate_config_updates( + upsert_values: &std::collections::HashMap, + delete_keys: &[String], +) -> UpdateMap { + let mut update_entries = Vec::new(); + + // Add upsert entries (with values) + for (key, value) in upsert_values { + update_entries.push(UpdateMapEntry { + key: key.clone(), + value: Some(value.clone()), + }); + } + + // Add delete entries (without values) + for key in delete_keys { + update_entries.push(UpdateMapEntry { + key: key.clone(), + value: None, + }); + } + + UpdateMap { + update_entries, + replace: false, // Old style was always incremental + } +} + +/// Helper function to translate old-style schema metadata to new UpdateMap format +pub fn translate_schema_metadata_updates( + schema_metadata: &std::collections::HashMap, +) -> UpdateMap { + let update_entries = schema_metadata + .iter() + .map(|(key, value)| UpdateMapEntry { + key: key.clone(), + value: Some(value.clone()), + }) + .collect(); + + UpdateMap { + update_entries, + replace: true, // Old style schema metadata was full replacement + } +} diff --git a/rust/lance-table/src/transaction/validate.rs b/rust/lance-table/src/transaction/validate.rs new file mode 100644 index 00000000000..505fbe89664 --- /dev/null +++ b/rust/lance-table/src/transaction/validate.rs @@ -0,0 +1,700 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Pre-commit validation of an operation against the manifest it applies to. +//! +//! These checks reject transactions that could not produce a coherent manifest — +//! a fragment list that disagrees with the schema, a merge that silently dropped +//! or rewrote data files — before any manifest is written. + +use crate::format::{Fragment, Manifest}; +use crate::io::deletion::relative_deletion_file_path; +use crate::transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}; +use lance_core::datatypes::{Field, Schema}; +use lance_core::{Error, Result}; +use lance_file::version::ConcreteFileVersion; +use std::collections::{HashMap, HashSet}; + +/// Validate the operation is valid for the given manifest. +pub fn validate_operation(manifest: Option<&Manifest>, operation: &Operation) -> Result<()> { + let manifest = match (manifest, operation) { + ( + None, + Operation::Overwrite { + fragments, schema, .. + }, + ) => { + // Validate here because we are going to return early. + overwrite_fragments_valid(fragments)?; + schema_fragments_valid(None, schema, fragments)?; + + return Ok(()); + } + (None, Operation::Clone { .. }) => return Ok(()), + (Some(manifest), _) => manifest, + (None, _) => { + return Err(Error::invalid_input(format!( + "Cannot apply operation {} to non-existent dataset", + operation.name() + ))); + } + }; + + match operation { + Operation::Append { fragments } => { + // Fragments must contain all fields in the schema + schema_fragments_valid(Some(manifest), &manifest.schema, fragments) + } + Operation::Project { schema, .. } => { + schema_fragments_valid(Some(manifest), schema, manifest.fragments.as_ref()) + } + Operation::Merge { + fragments, schema, .. + } => { + merge_fragments_valid(manifest, fragments)?; + merge_schema_valid(manifest, schema, fragments)?; + schema_fragments_valid(Some(manifest), schema, fragments) + } + Operation::Overwrite { + fragments, schema, .. + } => { + overwrite_fragments_valid(fragments)?; + // Pass None for manifest because Overwrite replaces all fragments. + // The old manifest's storage format is irrelevant for validating + // the new fragments (e.g., LEGACY→STABLE transitions). + schema_fragments_valid(None, schema, fragments) + } + Operation::Update { + updated_fragments, + new_fragments, + updated_fragment_offsets, + update_mode, + .. + } => { + schema_fragments_valid(Some(manifest), &manifest.schema, updated_fragments)?; + schema_fragments_valid(Some(manifest), &manifest.schema, new_fragments)?; + // Key-presence check only applies to RewriteColumns: that is the only + // mode where build_manifest stamps version metadata using off_map keys, + // so a stray key can corrupt an unrelated fragment's metadata. + // Other modes (e.g. rewrite_rows) may supply offsets for fragments + // outside updated_fragments for their own purposes. + if matches!(update_mode, Some(UpdateMode::RewriteColumns)) + && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets + { + let updated_ids: HashSet = updated_fragments.iter().map(|f| f.id).collect(); + for &frag_id in off_map.keys() { + if !updated_ids.contains(&frag_id) { + return Err(Error::invalid_input(format!( + "updatedFragmentOffsets key {} is not in updated_fragments; \ + offsets must reference only fragments being rewritten", + frag_id + ))); + } + } + } + Ok(()) + } + _ => Ok(()), + } +} + +// An overwrite's fragments are newly written, so they are given fresh ids at +// commit time. A deletion file cannot come along for that ride: its path embeds +// the fragment id, so renumbering the fragment would orphan the deletion vector +// and silently resurrect deleted rows. +fn overwrite_fragments_valid(fragments: &[Fragment]) -> Result<()> { + for fragment in fragments { + if let Some(deletion_file) = &fragment.deletion_file { + return Err(Error::invalid_input(format!( + "Overwrite fragments must be newly written, but fragment {} carries \ + deletion file {}. Use Delete to commit deletions against existing \ + fragments, or Merge to change their schema.", + fragment.id, + relative_deletion_file_path(fragment.id, deletion_file) + ))); + } + } + Ok(()) +} + +fn schema_fragments_valid( + manifest: Option<&Manifest>, + schema: &Schema, + fragments: &[Fragment], +) -> Result<()> { + if let Some(manifest) = manifest { + return match manifest.data_storage_format.lance_file_format() { + ConcreteFileVersion::V1 => schema_fragments_legacy_valid(schema, fragments), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => schema_fragments_modern_valid(schema, fragments), + }; + } + schema_fragments_modern_valid(schema, fragments) +} + +pub fn schema_fragments_modern_valid(_schema: &Schema, fragments: &[Fragment]) -> Result<()> { + // validate that each data file at least contains one field. + for fragment in fragments { + for data_file in &fragment.files { + if data_file.fields.iter().len() == 0 { + return Err(Error::invalid_input(format!( + "Datafile {} does not contain any fields", + data_file.path + ))); + } + } + } + Ok(()) +} + +/// Check that each fragment contains all fields in the schema. +/// It is not required that the schema contains all fields in the fragment. +/// There may be masked fields. +pub fn schema_fragments_legacy_valid(schema: &Schema, fragments: &[Fragment]) -> Result<()> { + // TODO: add additional validation. Consider consolidating with various + // validate() methods in the codebase. + for fragment in fragments { + for field in schema.fields_pre_order() { + if !fragment + .files + .iter() + .flat_map(|f| f.fields.iter()) + .any(|f_id| f_id == &field.id) + { + return Err(Error::invalid_input(format!( + "Fragment {} does not contain field {:?}", + fragment.id, field + ))); + } + } + } + Ok(()) +} + +/// Returns true if Operation::Merge rewrote this fragment's column data files (Fragment::files +/// changed versus the previous manifest). Used to bump last_updated_at_version_meta only when +/// new column values were materialized to disk. +/// +/// Deletion file changes alone are not treated as rewrites: tombstones remove rows but +/// survivors did not receive new column bytes; stamping last_updated for those rows would be +/// incorrect for CDF. +#[inline] +pub(super) fn merge_fragment_physically_rewritten(prev: &Fragment, merged: &Fragment) -> bool { + debug_assert_eq!(prev.id, merged.id); + if prev.files.len() != merged.files.len() { + return true; + } + // Compare identity fields only. file_size_bytes is an AtomicU64 cache that + // concurrent scans can populate in place on the manifest's DataFile, so it + // must not be part of the rewrite check. + prev.files.iter().zip(merged.files.iter()).any(|(p, m)| { + p.path != m.path + || p.fields != m.fields + || p.column_indices != m.column_indices + || p.file_major_version != m.file_major_version + || p.file_minor_version != m.file_minor_version + || p.base_id != m.base_id + }) +} + +/// Validate that Merge operations preserve all original fragments. +/// Merge operations should only add columns or rows, not reduce fragments. +/// This ensures fragments correspond at one-to-one with the original fragment list. +fn merge_fragments_valid(manifest: &Manifest, new_fragments: &[Fragment]) -> Result<()> { + let original_fragments = manifest.fragments.as_ref(); + + // Additional validation: ensure we're not accidentally reducing the fragment count + if new_fragments.len() < original_fragments.len() { + return Err(Error::invalid_input(format!( + "Merge operation reduced fragment count from {} to {}. \ + Merge operations should only add columns, not reduce fragments.", + original_fragments.len(), + new_fragments.len() + ))); + } + + // Collect new fragment IDs + let new_fragment_map: HashMap = + new_fragments.iter().map(|f| (f.id, f)).collect(); + + // Check that all original fragments are preserved in the new fragments list + // Validate that each original fragment's metadata is preserved + let mut missing_fragments: Vec = Vec::new(); + for original_fragment in original_fragments { + if let Some(new_fragment) = new_fragment_map.get(&original_fragment.id) { + // Validate physical_rows (row count) hasn't changed + if original_fragment.physical_rows != new_fragment.physical_rows { + return Err(Error::invalid_input(format!( + "Merge operation changed row count for fragment {}. \ + Original: {:?}, New: {:?}. \ + Merge operations should preserve fragment row counts and only add new columns.", + original_fragment.id, + original_fragment.physical_rows, + new_fragment.physical_rows + ))); + } + } else { + missing_fragments.push(original_fragment.id); + } + } + + if !missing_fragments.is_empty() { + return Err(Error::invalid_input(format!( + "Merge operation is missing original fragments: {:?}. \ + Merge operations should preserve all original fragments and only add new columns. \ + Expected fragments: {:?}, but got: {:?}", + missing_fragments, + original_fragments.iter().map(|f| f.id).collect::>(), + new_fragment_map.keys().copied().collect::>() + ))); + } + + Ok(()) +} + +/// Validate that a Merge schema preserves the dataset's field id bindings. +/// +/// Readers resolve columns by field id (name -> schema id -> DataFile::fields +/// position), so renumbered ids silently rebind live columns to other columns' +/// bytes. Shared ids must keep their field path. Their logical type, +/// nullability, storage encoding, and dictionary may change only when every +/// existing base or overlay file carrying the id is replaced and every +/// proposed fragment materializes the id in a base data file. New ids must +/// exceed the manifest's max so a dropped field's id is never reused. An +/// existing path may move to a fresh id only when every proposed fragment +/// materializes that id in a base data file (the `alter_columns` cast path). +/// Omitting a field (dropping it) and updating field metadata remain legal. +fn merge_schema_valid( + manifest: &Manifest, + new_schema: &Schema, + fragments: &[Fragment], +) -> Result<()> { + let prior_schema = &manifest.schema; + let new_fragment_map: HashMap = fragments + .iter() + .map(|fragment| (fragment.id, fragment)) + .collect(); + + // Remap and semantic errors first: a renumbered schema usually violates + // both the shared-id and new-id clauses. + for field in new_schema.fields_pre_order() { + let Some(prior_field) = prior_schema.field_by_id(field.id) else { + continue; + }; + let prior_path = prior_schema.field_path(field.id)?; + let new_path = new_schema.field_path(field.id)?; + if prior_path != new_path { + return Err(Error::invalid_input(format!( + "Merge operation remaps field id {} from \"{}\" to \"{}\". \ + Merge must preserve the dataset's field ids: derive the new schema \ + from the dataset's current schema instead of renumbering fields.", + field.id, prior_path, new_path + ))); + } + if let Some(changes) = shared_field_binding_changes(prior_field, field) + && !is_field_binding_fully_rewritten(manifest, &new_fragment_map, field.id) + { + return Err(Error::invalid_input(format!( + "Merge operation changes field id {} (\"{}\") without rewriting it in \ + every existing fragment: {}. Merge must preserve each existing field's \ + logical type, nullability, storage encoding, and dictionary unless all \ + existing base and overlay files carrying that field are replaced.", + field.id, new_path, changes + ))); + } + } + + let max_field_id = manifest.max_field_id(); + for field in new_schema.fields_pre_order() { + if prior_schema.field_by_id(field.id).is_none() && field.id <= max_field_id { + let next_id_msg = match max_field_id.checked_add(1) { + Some(next_id) => format!("New fields must use ids of at least {}.", next_id), + None => { + "No further field id can be allocated because ids are exhausted.".to_string() + } + }; + return Err(Error::invalid_input(format!( + "Merge operation assigns id {} to new field \"{}\", but ids up to {} are \ + already used by current or dropped fields. {}", + field.id, + new_schema.field_path(field.id)?, + max_field_id, + next_id_msg + ))); + } + } + + let mut prior_paths = HashMap::with_capacity(prior_schema.fields_pre_order().count()); + for field in prior_schema.fields_pre_order() { + prior_paths.insert(prior_schema.field_path(field.id)?, field); + } + for field in new_schema.fields_pre_order() { + if prior_schema.field_by_id(field.id).is_some() { + continue; + } + let new_path = new_schema.field_path(field.id)?; + let Some(prior_field) = prior_paths.get(&new_path) else { + continue; + }; + let materialized = fragments.iter().all(|fragment| { + fragment + .files + .iter() + .any(|file| file.fields.contains(&field.id)) + }); + if !materialized { + return Err(Error::invalid_input(format!( + "Merge operation remaps existing field \"{}\" from id {} to id {} without \ + rewriting its data. Every proposed fragment must materialize the new field \ + id in a base data file.", + new_path, prior_field.id, field.id + ))); + } + } + + Ok(()) +} + +fn is_field_binding_fully_rewritten( + manifest: &Manifest, + new_fragment_map: &HashMap, + field_id: i32, +) -> bool { + manifest.fragments.iter().all(|prior_fragment| { + let Some(new_fragment) = new_fragment_map.get(&prior_fragment.id) else { + return false; + }; + + let is_materialized = new_fragment + .files + .iter() + .any(|file| file.fields.contains(&field_id)); + if !is_materialized { + return false; + } + + prior_fragment + .referenced_lance_files() + .filter(|file| file.fields.contains(&field_id)) + .all(|prior_file| { + !new_fragment.referenced_lance_files().any(|new_file| { + new_file.fields.contains(&field_id) + && new_file.base_id == prior_file.base_id + && new_file.path == prior_file.path + }) + }) + }) +} + +fn shared_field_binding_changes(prior: &Field, new: &Field) -> Option { + let mut changes = Vec::with_capacity(4); + if prior.logical_type != new.logical_type { + changes.push(format!( + "logical type {} -> {}", + prior.logical_type, new.logical_type + )); + } + if prior.nullable != new.nullable { + changes.push(format!("nullable {} -> {}", prior.nullable, new.nullable)); + } + if prior.encoding != new.encoding { + changes.push(format!( + "storage encoding {:?} -> {:?}", + prior.encoding, new.encoding + )); + } + if prior.dictionary != new.dictionary { + changes.push("dictionary".to_string()); + } + if changes.is_empty() { + None + } else { + Some(changes.join(", ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::overlay::{DataOverlayFile, OverlayCoverage}; + use crate::format::{DataFile, DataStorageFormat}; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::datatypes::{Field as LanceCoreField, LogicalType, Schema as LanceSchema}; + use roaring::RoaringBitmap; + use std::collections::HashMap; + use std::sync::Arc; + + #[test] + fn test_merge_fragments_valid() { + // Create a simple schema for testing + let schema = ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("name", DataType::Utf8, false), + ]); + + // Create original fragments + let original_fragments = vec![Fragment::new(1), Fragment::new(2), Fragment::new(3)]; + + // Create a manifest with original fragments + let manifest = Manifest::new( + LanceSchema::try_from(&schema).unwrap(), + Arc::new(original_fragments), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + + // Test 1: Empty fragments should fail + let empty_fragments = vec![]; + let result = merge_fragments_valid(&manifest, &empty_fragments); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("reduced fragment count") + ); + + // Test 2: Missing original fragments should fail + let missing_fragments = vec![ + Fragment::new(1), + Fragment::new(2), + // Fragment 3 is missing + Fragment::new(4), // New fragment + ]; + let result = merge_fragments_valid(&manifest, &missing_fragments); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("missing original fragments") + ); + + // Test 3: Reduced fragment count should fail + let reduced_fragments = vec![ + Fragment::new(1), + Fragment::new(2), + // Fragment 3 is missing, no new fragments added + ]; + let result = merge_fragments_valid(&manifest, &reduced_fragments); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("reduced fragment count") + ); + + // Test 4: Valid merge with all original fragments plus new ones should succeed + let valid_fragments = vec![ + Fragment::new(1), + Fragment::new(2), + Fragment::new(3), + Fragment::new(4), // New fragment + Fragment::new(5), // Another new fragment + ]; + let result = merge_fragments_valid(&manifest, &valid_fragments); + assert!(result.is_ok()); + + // Test 5: Same fragments (no new ones) should succeed + let same_fragments = vec![Fragment::new(1), Fragment::new(2), Fragment::new(3)]; + let result = merge_fragments_valid(&manifest, &same_fragments); + assert!(result.is_ok()); + } + + fn one_field_schema() -> LanceSchema { + LanceSchema::try_from(&ArrowSchema::new(vec![ArrowField::new( + "a", + DataType::Int32, + true, + )])) + .unwrap() + } + + fn fragment_with_file_fields(id: u64, path: &str, fields: Vec) -> Fragment { + let mut fragment = Fragment::new(id); + fragment + .files + .push(DataFile::new_legacy_from_fields(path, fields, None)); + fragment + } + + fn manifest_with_file_fields(schema: LanceSchema, fields: Vec) -> Manifest { + Manifest::new( + schema, + Arc::new(vec![fragment_with_file_fields(0, "f.lance", fields)]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ) + } + + #[rstest::rstest] + #[case::logical_type(DataType::Float32, true)] + #[case::nullability(DataType::Int32, false)] + #[test] + fn test_merge_shared_id_change_requires_full_rewrite( + #[case] data_type: DataType, + #[case] nullable: bool, + ) { + let schema = one_field_schema(); + let prior_fragments = vec![ + fragment_with_file_fields(0, "old-0.lance", vec![0]), + fragment_with_file_fields(1, "old-1.lance", vec![0]), + ]; + let manifest = Manifest::new( + schema.clone(), + Arc::new(prior_fragments.clone()), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + let mut new_schema = schema; + new_schema.fields[0].logical_type = LogicalType::try_from(&data_type).unwrap(); + new_schema.fields[0].nullable = nullable; + + let rewritten_fragments = vec![ + fragment_with_file_fields(0, "new-0.lance", vec![0]), + fragment_with_file_fields(1, "new-1.lance", vec![0]), + ]; + merge_schema_valid(&manifest, &new_schema, &rewritten_fragments).unwrap(); + + let partially_rewritten = vec![rewritten_fragments[0].clone(), prior_fragments[1].clone()]; + let err = merge_schema_valid(&manifest, &new_schema, &partially_rewritten).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + assert!( + err.to_string() + .contains("without rewriting it in every existing fragment"), + "unexpected error: {}", + err + ); + } + + #[test] + fn test_merge_shared_id_change_rejects_retained_overlay() { + let schema = one_field_schema(); + let mut prior_fragment = fragment_with_file_fields(0, "old.lance", vec![0]); + prior_fragment.overlays.push(DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("old-overlay.lance", vec![0], None), + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: 1, + }); + let manifest = Manifest::new( + schema.clone(), + Arc::new(vec![prior_fragment.clone()]), + DataStorageFormat::new(ConcreteFileVersion::V2_0), + HashMap::new(), + ); + let mut new_schema = schema; + new_schema.fields[0].nullable = false; + + let mut rewritten = fragment_with_file_fields(0, "new.lance", vec![0]); + rewritten.overlays = prior_fragment.overlays.clone(); + let err = merge_schema_valid(&manifest, &new_schema, &[rewritten]).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + assert!( + err.to_string() + .contains("without rewriting it in every existing fragment"), + "unexpected error: {}", + err + ); + } + + #[test] + fn test_merge_allows_rewritten_fresh_field_id() { + let schema = one_field_schema(); + let manifest = manifest_with_file_fields(schema.clone(), vec![0]); + let mut rewritten_schema = schema; + rewritten_schema.fields[0].id = 1; + let mut rewritten = manifest.fragments[0].clone(); + rewritten.files[0] = DataFile::new_legacy_from_fields("rewritten.lance", vec![1], None); + merge_schema_valid(&manifest, &rewritten_schema, &[rewritten]).unwrap(); + } + + #[test] + fn test_merge_rejects_max_field_id_overflow() { + let schema = one_field_schema(); + let manifest = manifest_with_file_fields(schema.clone(), vec![0, i32::MAX]); + assert_eq!(manifest.max_field_id(), i32::MAX); + + let mut new_schema = schema; + let mut extra = + LanceCoreField::try_from(&ArrowField::new("b", DataType::Int32, true)).unwrap(); + extra.id = 1; + new_schema.fields.push(extra); + + let err = merge_schema_valid(&manifest, &new_schema, &manifest.fragments).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + let message = err.to_string(); + assert!( + message.contains("assigns id 1 to new field \"b\"") && message.contains("exhausted"), + "unexpected error: {}", + message + ); + } + + /// Regression test for https://github.com/lance-format/lance/issues/6417 + /// + /// When overwriting a LEGACY dataset with STABLE-format fragments, the + /// validation should not use the old manifest's format. STABLE fragments + /// omit struct parent fields, which the strict legacy check rejects. + #[test] + fn test_overwrite_legacy_to_stable_with_struct_fields() { + use arrow_schema::Fields; + + // Schema: id (field 0), name (field 1), address (field 2, struct parent), + // city (field 3), country (field 4) + let arrow_schema = ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("name", DataType::Utf8, false), + ArrowField::new( + "address", + DataType::Struct(Fields::from(vec![ + ArrowField::new("city", DataType::Utf8, false), + ArrowField::new("country", DataType::Utf8, false), + ])), + false, + ), + ]); + let schema = LanceSchema::try_from(&arrow_schema).unwrap(); + + // Old manifest is LEGACY format + let legacy_manifest = Manifest::new( + schema.clone(), + Arc::new(vec![Fragment::new(0)]), + DataStorageFormat::new(ConcreteFileVersion::V1), + HashMap::new(), + ); + + // New fragments in STABLE format omit struct parent field (id=2), + // only including leaf fields: id=0, name=1, city=3, country=4 + let stable_fragment = Fragment { + id: 0, + files: vec![DataFile::new( + "data.lance", + vec![0, 1, 3, 4], // no field 2 (struct parent) + vec![0, 1, 2, 3], + ConcreteFileVersion::V1, + None, + None, + )], + physical_rows: Some(10), + overlays: vec![], + deletion_file: None, + row_id_meta: None, + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + + let operation = Operation::Overwrite { + fragments: vec![stable_fragment], + schema, + config_upsert_values: None, + initial_bases: None, + }; + + // This should succeed — the old manifest's LEGACY format should not + // cause strict validation of the new STABLE fragments. + validate_operation(Some(&legacy_manifest), &operation).unwrap(); + } +} diff --git a/rust/lance-table/src/utils/stream.rs b/rust/lance-table/src/utils/stream.rs index f6fbbd45a61..6a958f3332d 100644 --- a/rust/lance-table/src/utils/stream.rs +++ b/rust/lance-table/src/utils/stream.rs @@ -1,25 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::Arc; +use std::{fmt, sync::Arc}; use arrow_array::{BooleanArray, RecordBatch, RecordBatchOptions, UInt64Array, make_array}; use arrow_buffer::NullBuffer; use futures::{ FutureExt, Stream, StreamExt, - future::BoxFuture, + future::{BoxFuture, Shared}, stream::{BoxStream, FuturesOrdered}, }; use lance_arrow::RecordBatchExt; use lance_core::{ - ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, + Error, ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, ROW_LAST_UPDATED_AT_VERSION_FIELD, Result, utils::{address::RowAddress, deletion::DeletionVector}, }; use lance_io::ReadBatchParams; use tracing::instrument; -use crate::rowids::RowIdSequence; +use crate::rowids::{RowIdSequence, RowIdSequenceCursor}; pub type ReadBatchFut = BoxFuture<'static, Result>; /// A task, emitted by a file reader, that will produce a batch (of the @@ -31,27 +31,124 @@ pub struct ReadBatchTask { pub type ReadBatchTaskStream = BoxStream<'static, ReadBatchTask>; pub type ReadBatchFutStream = BoxStream<'static, ReadBatchFut>; +type SharedReadBatchFut = Shared>>>; + +#[derive(Debug)] +struct SharedReadError(Arc); + +impl fmt::Display for SharedReadError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for SharedReadError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.0.as_ref()) + } +} + +struct PendingReadBatch { + task: Option, + shared_task: Option, + offset: u32, + num_rows: u32, +} + +impl PendingReadBatch { + fn new(task: ReadBatchTask) -> Self { + Self { + task: Some(task.task), + shared_task: None, + offset: 0, + num_rows: task.num_rows, + } + } + + fn take(&mut self, num_rows: u32) -> ReadBatchFut { + debug_assert!(num_rows <= self.num_rows); + + if self.offset == 0 && num_rows == self.num_rows && self.shared_task.is_none() { + self.num_rows = 0; + let Some(task) = self.task.take() else { + return async { + Err(Error::internal( + "missing read task while merging aligned streams".to_string(), + )) + } + .boxed(); + }; + return task; + } + + let shared_task = self + .shared_task + .get_or_insert_with(|| { + let task = self.task.take(); + async move { + let Some(task) = task else { + return Err(Arc::new(Error::internal( + "missing read task while splitting a merged stream".to_string(), + ))); + }; + task.await.map_err(Arc::new) + } + .boxed() + .shared() + }) + .clone(); + let offset = self.offset; + self.offset += num_rows; + self.num_rows -= num_rows; + + async move { + match shared_task.await { + Ok(batch) => Ok(batch.slice(offset as usize, num_rows as usize)), + Err(error) => Err(Error::wrapped(Box::new(SharedReadError(error)))), + } + } + .boxed() + } +} + struct MergeStream { streams: Vec, - next_batch: FuturesOrdered, - next_num_rows: u32, + pending: Vec>, index: usize, } impl MergeStream { fn emit(&mut self) -> ReadBatchTask { - let mut iter = std::mem::take(&mut self.next_batch); + let num_rows = self + .pending + .iter() + .filter_map(|pending| pending.as_ref().map(|pending| pending.num_rows)) + .min() + .unwrap_or_default(); + let mut batches = FuturesOrdered::new(); + for pending in &mut self.pending { + let Some(pending_batch) = pending.as_mut() else { + continue; + }; + batches.push_back(pending_batch.take(num_rows)); + if pending_batch.num_rows == 0 { + *pending = None; + } + } let task = async move { - let mut batch = iter.next().await.unwrap()?; - while let Some(next) = iter.next().await { + let Some(first) = batches.next().await else { + return Err(Error::internal( + "cannot merge an empty set of read batches".to_string(), + )); + }; + let mut batch = first?; + while let Some(next) = batches.next().await { let next = next?; batch = batch.merge(&next)?; } Ok(batch) } .boxed(); - let num_rows = self.next_num_rows; - self.next_num_rows = 0; ReadBatchTask { task, num_rows } } } @@ -64,21 +161,19 @@ impl Stream for MergeStream { cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { loop { + if self.pending.iter().all(Option::is_some) { + return std::task::Poll::Ready(Some(self.emit())); + } + let index = self.index; + if self.pending[index].is_some() { + self.index = (index + 1) % self.streams.len(); + continue; + } match self.streams[index].poll_next_unpin(cx) { std::task::Poll::Ready(Some(batch_task)) => { - if self.index == 0 { - self.next_num_rows = batch_task.num_rows; - } else { - debug_assert_eq!(self.next_num_rows, batch_task.num_rows); - } - self.next_batch.push_back(batch_task.task); - self.index += 1; - if self.index == self.streams.len() { - self.index = 0; - let next_batch = self.emit(); - return std::task::Poll::Ready(Some(next_batch)); - } + self.pending[index] = Some(PendingReadBatch::new(batch_task)); + self.index = (index + 1) % self.streams.len(); } std::task::Poll::Ready(None) => { return std::task::Poll::Ready(None); @@ -95,19 +190,20 @@ impl Stream for MergeStream { /// /// This pulls one batch from each stream and then combines the columns from /// all of the batches into a single batch. The order of the batches in the -/// streams is maintained and the merged batch columns will be in order from -/// first to last stream. +/// streams is maintained and the merged batch columns will be in order from first +/// to last stream. If the streams use different batch boundaries then batches are +/// sliced so each merged output remains row-aligned. /// /// This stream ends as soon as any of the input streams ends (we do not /// verify that the other input streams are finished as well) -/// -/// This will panic if any of the input streams return a batch with a different -/// number of rows than the first stream. pub fn merge_streams(streams: Vec) -> ReadBatchTaskStream { + if streams.is_empty() { + return futures::stream::empty().boxed(); + } + let pending = (0..streams.len()).map(|_| None).collect(); MergeStream { streams, - next_batch: FuturesOrdered::new(), - next_num_rows: 0, + pending, index: 0, } .boxed() @@ -249,12 +345,22 @@ impl RowIdAndDeletesConfig { } } -#[instrument(level = "debug", skip_all)] pub fn apply_row_id_and_deletes( batch: RecordBatch, batch_offset: u32, fragment_id: u32, config: &RowIdAndDeletesConfig, +) -> Result { + apply_row_id_and_deletes_with_row_ids(batch, batch_offset, fragment_id, config, None) +} + +#[instrument(name = "apply_row_id_and_deletes", level = "debug", skip_all)] +fn apply_row_id_and_deletes_with_row_ids( + batch: RecordBatch, + batch_offset: u32, + fragment_id: u32, + config: &RowIdAndDeletesConfig, + precomputed_row_ids: Option>, ) -> Result { let mut deletion_vector = config.deletion_vector.as_ref(); // Convert Some(NoDeletions) into None to simplify logic below @@ -295,7 +401,10 @@ pub fn apply_row_id_and_deletes( let row_ids = if config.with_row_id { let _rowids = tracing::span!(tracing::Level::DEBUG, "fetch_row_ids").entered(); - if let Some(row_id_sequence) = &config.row_id_sequence { + if let Some(row_ids) = precomputed_row_ids { + debug_assert_eq!(row_ids.len(), num_rows as usize); + Some(row_ids) + } else if let Some(row_id_sequence) = &config.row_id_sequence { let selection = config .params .slice(batch_offset as usize, num_rows as usize) @@ -396,6 +505,28 @@ pub fn wrap_with_row_id_and_delete( stream: ReadBatchTaskStream, fragment_id: u32, config: RowIdAndDeletesConfig, +) -> ReadBatchFutStream { + let (row_id_cursor, use_dense_row_id_expansion) = config + .row_id_sequence + .as_ref() + .filter(|_| config.with_row_id) + .map(|sequence| { + let (cursor, use_dense_range_expansion) = sequence.cursor_with_dense_range_expansion(); + (Some(cursor), use_dense_range_expansion) + }) + .unwrap_or((None, false)); + if use_dense_row_id_expansion { + wrap_with_row_id_and_delete_impl::(stream, fragment_id, config, row_id_cursor) + } else { + wrap_with_row_id_and_delete_impl::(stream, fragment_id, config, row_id_cursor) + } +} + +fn wrap_with_row_id_and_delete_impl( + stream: ReadBatchTaskStream, + fragment_id: u32, + config: RowIdAndDeletesConfig, + mut row_id_cursor: Option, ) -> ReadBatchFutStream { let config = Arc::new(config); let mut offset = 0; @@ -405,10 +536,62 @@ pub fn wrap_with_row_id_and_delete( let this_offset = offset; let num_rows = batch_task.num_rows; offset += num_rows; + // Materialize row ids while polling the ordered task stream. Batch + // futures may complete concurrently, so doing this inside the + // future would require locking the shared cursor or would reorder + // its accesses. + let row_ids = config.row_id_sequence.as_ref().and_then(|sequence| { + row_id_cursor.as_mut().map(|cursor| { + let selection = config + .params + .slice(this_offset as usize, num_rows as usize) + .unwrap() + .to_ranges() + .unwrap(); + let values = match selection.as_slice() { + [range] if USE_DENSE_ROW_ID_EXPANSION => { + UInt64Array::from(sequence.select_dense_range_with_cursor( + cursor, + range.start as usize..range.end as usize, + )) + } + [range] => UInt64Array::from(sequence.select_range_with_cursor( + cursor, + range.start as usize..range.end as usize, + )), + _ => UInt64Array::from( + sequence + .select_with_cursor( + cursor, + selection + .iter() + .flat_map(|range| range.start as usize..range.end as usize), + ) + .collect::>(), + ), + }; + if values.len() != num_rows as usize { + return Err(Error::corrupt_file_named( + "row ID metadata", + format!( + "decoded row IDs at selected offset {this_offset} contain {} rows, but the current batch requires {num_rows} rows", + values.len() + ), + )); + } + Ok(Arc::new(values)) + }) + }); batch_task .task .map(move |batch| { - apply_row_id_and_deletes(batch?, this_offset, fragment_id, config.as_ref()) + apply_row_id_and_deletes_with_row_ids( + batch?, + this_offset, + fragment_id, + config.as_ref(), + row_ids.transpose()?, + ) }) .boxed() }) @@ -422,7 +605,10 @@ mod tests { use arrow::{array::AsArray, datatypes::UInt64Type}; use arrow_array::{RecordBatch, UInt32Array, types::Int32Type}; use arrow_schema::ArrowError; - use futures::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream}; + use futures::{ + FutureExt, StreamExt, TryStreamExt, + stream::{self, BoxStream}, + }; use lance_core::{ ROW_ID, utils::{address::RowAddress, deletion::DeletionVector}, @@ -431,7 +617,7 @@ mod tests { use lance_io::{ReadBatchParams, stream::arrow_stream_to_lance_stream}; use roaring::RoaringBitmap; - use crate::utils::stream::ReadBatchTask; + use crate::{rowids::RowIdSequence, utils::stream::ReadBatchTask}; use super::RowIdAndDeletesConfig; @@ -477,6 +663,247 @@ mod tests { assert_eq!(merged, expected); } + #[tokio::test] + async fn test_stable_row_ids_across_concurrent_batches_and_deletes() { + let expected = (10_000..120_000) + .filter(|row_id| row_id % 13 != 0) + .collect::>(); + let row_id_sequence = Arc::new(RowIdSequence::try_from_iter(expected.clone()).unwrap()); + let deletion_offsets = (0..expected.len() as u32).step_by(997).collect::>(); + let deletion_vector = Some(Arc::new(DeletionVector::Bitmap( + deletion_offsets.iter().copied().collect(), + ))); + + let batches = expected + .chunks(257) + .map(|chunk| arrow_array::record_batch!(("x", Int32, vec![0; chunk.len()])).unwrap()) + .map(Ok) + .collect::>>(); + let data = batch_task_stream(stream::iter(batches).boxed()); + let config = RowIdAndDeletesConfig { + params: ReadBatchParams::RangeFull, + with_row_id: true, + with_row_addr: true, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + deletion_vector, + row_id_sequence: Some(row_id_sequence), + last_updated_at_sequence: None, + created_at_sequence: None, + make_deletions_null: false, + total_num_rows: expected.len() as u32, + }; + + let batches = super::wrap_with_row_id_and_delete(data, 7, config) + .buffered(8) + .try_collect::>() + .await + .unwrap(); + let actual_row_ids = batches + .iter() + .flat_map(|batch| batch[ROW_ID].as_primitive::().values()) + .copied() + .collect::>(); + let actual_row_addrs = batches + .iter() + .flat_map(|batch| { + batch[lance_core::ROW_ADDR] + .as_primitive::() + .values() + }) + .copied() + .collect::>(); + let expected_survivors = expected + .iter() + .enumerate() + .filter(|(offset, _)| deletion_offsets.binary_search(&(*offset as u32)).is_err()) + .map(|(offset, row_id)| { + ( + *row_id, + u64::from(RowAddress::new_from_parts(7, offset as u32)), + ) + }) + .collect::>(); + + assert_eq!( + actual_row_ids, + expected_survivors + .iter() + .map(|(row_id, _)| *row_id) + .collect::>() + ); + assert_eq!( + actual_row_addrs, + expected_survivors + .iter() + .map(|(_, row_addr)| *row_addr) + .collect::>() + ); + } + + #[tokio::test] + async fn test_stable_row_ids_with_unsorted_indices() { + let expected = (100..140) + .filter(|row_id| row_id % 3 != 0) + .collect::>(); + let indices = UInt32Array::from(vec![8, 2, 9, 1, 6]); + let batches = [2, 2, 1].into_iter().map(|num_rows| ReadBatchTask { + num_rows, + task: std::future::ready(Ok(arrow_array::record_batch!(( + "x", + Int32, + vec![0; num_rows as usize] + )) + .unwrap())) + .boxed(), + }); + let config = RowIdAndDeletesConfig { + params: ReadBatchParams::Indices(indices.clone()), + with_row_id: true, + with_row_addr: false, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + deletion_vector: None, + row_id_sequence: Some(Arc::new( + RowIdSequence::try_from_iter(expected.clone()).unwrap(), + )), + last_updated_at_sequence: None, + created_at_sequence: None, + make_deletions_null: false, + total_num_rows: expected.len() as u32, + }; + + let actual = super::wrap_with_row_id_and_delete(stream::iter(batches).boxed(), 7, config) + .buffered(3) + .try_collect::>() + .await + .unwrap() + .iter() + .flat_map(|batch| batch[ROW_ID].as_primitive::().values()) + .copied() + .collect::>(); + let expected = indices + .values() + .iter() + .map(|index| expected[*index as usize]) + .collect::>(); + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn test_repeated_row_id_after_bulk_segment_boundary() { + let mut row_ids = RowIdSequence::from(0..5); + row_ids.extend(RowIdSequence::from(10..20)); + let batches = [1_u32, 2].into_iter().map(|num_rows| ReadBatchTask { + num_rows, + task: std::future::ready(Ok(arrow_array::record_batch!(( + "x", + Int32, + vec![0; num_rows as usize] + )) + .unwrap())) + .boxed(), + }); + let config = RowIdAndDeletesConfig { + params: ReadBatchParams::Indices(UInt32Array::from(vec![4, 4, 5])), + with_row_id: true, + with_row_addr: false, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + deletion_vector: None, + row_id_sequence: Some(Arc::new(row_ids)), + last_updated_at_sequence: None, + created_at_sequence: None, + make_deletions_null: false, + total_num_rows: 15, + }; + + let actual = super::wrap_with_row_id_and_delete(stream::iter(batches).boxed(), 0, config) + .buffered(1) + .try_collect::>() + .await + .unwrap() + .iter() + .flat_map(|batch| batch[ROW_ID].as_primitive::().values()) + .copied() + .collect::>(); + assert_eq!(actual, vec![4, 4, 10]); + } + + #[tokio::test] + async fn test_truncated_stable_row_ids_returns_error() { + let task = ReadBatchTask { + num_rows: 10, + task: std::future::ready(Ok( + arrow_array::record_batch!(("x", Int32, vec![0; 10])).unwrap() + )) + .boxed(), + }; + let config = RowIdAndDeletesConfig { + params: ReadBatchParams::RangeFull, + with_row_id: true, + with_row_addr: false, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + deletion_vector: None, + row_id_sequence: Some(Arc::new(RowIdSequence::try_from_iter(0_u64..5).unwrap())), + last_updated_at_sequence: None, + created_at_sequence: None, + make_deletions_null: false, + total_num_rows: 10, + }; + + let error = super::wrap_with_row_id_and_delete(stream::iter([task]).boxed(), 0, config) + .buffered(1) + .try_collect::>() + .await + .unwrap_err(); + assert!(matches!(error, lance_core::Error::CorruptFile { .. })); + assert!(error.to_string().contains( + "decoded row IDs at selected offset 0 contain 5 rows, but the current batch requires 10 rows" + )); + } + + #[tokio::test] + async fn test_zip_with_different_batch_boundaries() { + let left_batch = + arrow_array::record_batch!(("x", Int32, (0..10).collect::>())).unwrap(); + let right_batch = + arrow_array::record_batch!(("y", Int32, (10..20).collect::>())).unwrap(); + let left = batch_task_stream( + stream::iter([Ok(left_batch.slice(0, 6)), Ok(left_batch.slice(6, 4))]).boxed(), + ); + let right = batch_task_stream( + stream::iter([Ok(right_batch.slice(0, 4)), Ok(right_batch.slice(4, 6))]).boxed(), + ); + + let merged = super::merge_streams(vec![left, right]) + .map(|batch_task| batch_task.task) + .buffered(3) + .try_collect::>() + .await + .unwrap(); + + let expected = vec![ + arrow_array::record_batch!( + ("x", Int32, (0..4).collect::>()), + ("y", Int32, (10..14).collect::>()) + ) + .unwrap(), + arrow_array::record_batch!( + ("x", Int32, (4..6).collect::>()), + ("y", Int32, (14..16).collect::>()) + ) + .unwrap(), + arrow_array::record_batch!( + ("x", Int32, (6..10).collect::>()), + ("y", Int32, (16..20).collect::>()) + ) + .unwrap(), + ]; + assert_eq!(merged, expected); + } + async fn check_row_id(params: ReadBatchParams, expected: impl IntoIterator) { let expected = Vec::from_iter(expected); diff --git a/rust/lance-test-macros/Cargo.toml b/rust/lance-test-macros/Cargo.toml index e63be927765..028b824ef47 100644 --- a/rust/lance-test-macros/Cargo.toml +++ b/rust/lance-test-macros/Cargo.toml @@ -14,9 +14,9 @@ categories.workspace = true proc-macro = true [dependencies] -proc-macro2 = "1.0.67" -quote = "1.0.33" -syn = { version = "2.0.37", features = ["full"] } +proc-macro2.workspace = true +quote.workspace = true +syn.workspace = true [lints] workspace = true diff --git a/rust/lance-tokenizer/Cargo.toml b/rust/lance-tokenizer/Cargo.toml index e1006cd93c7..ebb7b0060f8 100644 --- a/rust/lance-tokenizer/Cargo.toml +++ b/rust/lance-tokenizer/Cargo.toml @@ -15,7 +15,26 @@ rust-version.workspace = true icu_segmenter = { workspace = true } jieba-rs = { workspace = true, optional = true } lindera = { workspace = true, optional = true } -rust-stemmers = "1.2.0" +frostem = { version = "1.20260804.0", default-features = false, features = [ + "arabic", + "danish", + "dutch", + "english", + "finnish", + "french", + "german", + "greek", + "hungarian", + "italian", + "norwegian", + "portuguese", + "romanian", + "russian", + "spanish", + "swedish", + "tamil", + "turkish", +] } serde = { workspace = true, features = ["derive"] } stop-words = { version = "0.10.0", default-features = false, features = ["iso", "nltk"] } unicode-normalization = "0.1.25" diff --git a/rust/lance-tokenizer/src/code_tokenizer.rs b/rust/lance-tokenizer/src/code_tokenizer.rs new file mode 100644 index 00000000000..0d00aa2b94c --- /dev/null +++ b/rust/lance-tokenizer/src/code_tokenizer.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{iter::Peekable, str::CharIndices}; + +use crate::{Token, TokenStream, Tokenizer}; + +/// Tokenizer for code-like text. +/// +/// Identifiers are Unicode alphanumeric characters plus `_`. Other characters +/// are lexical boundaries. When operator indexing is enabled, recognized +/// multi-character operators use longest-match tokenization and remaining +/// operator characters are emitted individually. +/// +/// # Examples +/// +/// ``` +/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream}; +/// +/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(true)).build(); +/// let mut stream = analyzer.token_stream("a::b"); +/// +/// assert!(stream.advance()); +/// assert_eq!(stream.token().text, "a"); +/// assert!(stream.advance()); +/// assert_eq!(stream.token().text, "::"); +/// ``` +#[derive(Clone, Default)] +pub struct CodeLexTokenizer { + index_operators: bool, + token: Token, +} + +impl CodeLexTokenizer { + pub fn new(index_operators: bool) -> Self { + Self { + index_operators, + token: Token::default(), + } + } +} + +/// Token stream produced by [`CodeLexTokenizer`]. +pub struct CodeLexTokenStream<'a> { + text: &'a str, + chars: Peekable>, + token: &'a mut Token, + index_operators: bool, +} + +impl Tokenizer for CodeLexTokenizer { + type TokenStream<'a> = CodeLexTokenStream<'a>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + self.token.reset(); + CodeLexTokenStream { + text, + chars: text.char_indices().peekable(), + token: &mut self.token, + index_operators: self.index_operators, + } + } +} + +fn is_identifier_char(ch: char) -> bool { + ch == '_' || ch.is_alphanumeric() +} + +fn is_operator_char(ch: char) -> bool { + matches!( + ch, + '!' | '%' | '&' | '*' | '+' | '-' | '/' | ':' | '<' | '=' | '>' | '?' | '^' | '|' | '~' + ) +} + +const MULTI_CHAR_OPERATORS: &[&str] = &[ + ">>>=", "<<=", ">>=", "&&=", "||=", "??=", "**=", "//=", "===", "!==", ">>>", "<=>", "::", + "->", "=>", "==", "!=", "<=", ">=", "&&", "||", "++", "--", "+=", "-=", "*=", "/=", "%=", "&=", + "|=", "^=", "<<", ">>", "**", "//", "??", ":=", "<-", "|>", "~=", +]; + +impl CodeLexTokenStream<'_> { + fn search_token_end(&mut self, predicate: impl Fn(char) -> bool) -> usize { + while let Some((_, ch)) = self.chars.peek() { + if !predicate(*ch) { + break; + } + self.chars.next(); + } + self.chars + .peek() + .map(|(offset, _)| *offset) + .unwrap_or(self.text.len()) + } + + fn operator_token_end(&mut self, offset_from: usize) -> usize { + let remaining = &self.text[offset_from..]; + let operator_len = MULTI_CHAR_OPERATORS + .iter() + .filter(|operator| remaining.starts_with(**operator)) + .map(|operator| operator.len()) + .max() + .unwrap_or(1); + let token_end = offset_from + operator_len; + while self + .chars + .peek() + .is_some_and(|(offset, _)| *offset < token_end) + { + self.chars.next(); + } + token_end + } +} + +impl TokenStream for CodeLexTokenStream<'_> { + fn advance(&mut self) -> bool { + self.token.text.clear(); + while let Some((offset_from, ch)) = self.chars.next() { + let token_end = if is_identifier_char(ch) { + self.search_token_end(is_identifier_char) + } else if self.index_operators && is_operator_char(ch) { + self.operator_token_end(offset_from) + } else { + continue; + }; + + self.token.position = self.token.position.wrapping_add(1); + self.token.position_length = 1; + self.token.offset_from = offset_from; + self.token.offset_to = token_end; + self.token.text.push_str(&self.text[offset_from..token_end]); + return true; + } + false + } + + fn token(&self) -> &Token { + self.token + } + + fn token_mut(&mut self) -> &mut Token { + self.token + } +} + +#[cfg(test)] +mod tests { + use crate::{CodeLexTokenizer, TextAnalyzer, Token}; + + fn collect_tokens(text: &str, index_operators: bool) -> Vec { + let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(index_operators)).build(); + let mut stream = analyzer.token_stream(text); + let mut tokens = Vec::new(); + stream.process(&mut |token| tokens.push(token.clone())); + tokens + } + + #[test] + fn test_code_lex_tokenizer_identifiers() { + let tokens = collect_tokens("std::vector user-name parse.HTML2JSON", false); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!( + texts, + vec!["std", "vector", "user", "name", "parse", "HTML2JSON"] + ); + } + + #[test] + fn test_code_lex_tokenizer_operators() { + let tokens = collect_tokens("a::b != c->d", true); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!(texts, vec!["a", "::", "b", "!=", "c", "->", "d"]); + } + + #[test] + fn test_code_lex_tokenizer_splits_adjacent_operators() { + let tokens = collect_tokens("value.parse::()", true); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!(texts, vec!["value", "parse", "::", "<", "usize", ">"]); + } +} diff --git a/rust/lance-tokenizer/src/lib.rs b/rust/lance-tokenizer/src/lib.rs index 0708022b384..fc1d9512cfd 100644 --- a/rust/lance-tokenizer/src/lib.rs +++ b/rust/lance-tokenizer/src/lib.rs @@ -4,6 +4,7 @@ mod alphanum_only; mod analyzer; mod ascii_folding_filter; +mod code_tokenizer; mod icu; #[cfg(feature = "tokenizer-jieba")] mod jieba; @@ -16,6 +17,7 @@ mod stemmer; mod stop_word_filter; mod tokenizer_api; mod whitespace_tokenizer; +mod word_delimiter_filter; #[cfg(feature = "tokenizer-lindera")] mod lindera; @@ -23,6 +25,7 @@ mod lindera; pub use alphanum_only::AlphaNumOnlyFilter; pub use analyzer::{TextAnalyzer, TextAnalyzerBuilder}; pub use ascii_folding_filter::AsciiFoldingFilter; +pub use code_tokenizer::CodeLexTokenizer; pub use icu::IcuTokenizer; #[cfg(feature = "tokenizer-jieba")] pub use jieba::JiebaTokenizer; @@ -37,3 +40,4 @@ pub use stemmer::{Language, Stemmer}; pub use stop_word_filter::StopWordFilter; pub use tokenizer_api::{BoxTokenStream, Token, TokenFilter, TokenStream, Tokenizer}; pub use whitespace_tokenizer::WhitespaceTokenizer; +pub use word_delimiter_filter::WordDelimiterFilter; diff --git a/rust/lance-tokenizer/src/stemmer.rs b/rust/lance-tokenizer/src/stemmer.rs index 03fcf118019..a79dd20d5f5 100644 --- a/rust/lance-tokenizer/src/stemmer.rs +++ b/rust/lance-tokenizer/src/stemmer.rs @@ -7,7 +7,7 @@ use std::borrow::Cow; use std::mem; -use rust_stemmers::Algorithm; +use frostem::Algorithm; use serde::{Deserialize, Serialize}; use crate::{Token, TokenFilter, TokenStream, Tokenizer}; @@ -101,7 +101,7 @@ impl Tokenizer for StemmerFilter { fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { StemmerTokenStream { tail: self.inner.token_stream(text), - stemmer: rust_stemmers::Stemmer::create(self.stemmer_algorithm), + stemmer: frostem::Stemmer::new(self.stemmer_algorithm), buffer: String::new(), } } @@ -109,7 +109,7 @@ impl Tokenizer for StemmerFilter { pub struct StemmerTokenStream { tail: T, - stemmer: rust_stemmers::Stemmer, + stemmer: frostem::Stemmer, buffer: String, } @@ -139,3 +139,19 @@ impl TokenStream for StemmerTokenStream { self.tail.token_mut() } } + +#[cfg(test)] +mod tests { + use crate::{Language, RawTokenizer, Stemmer, TextAnalyzer, TokenStream}; + + #[test] + fn test_greek_stemmer_handles_multibyte_suffixes() { + let mut analyzer = TextAnalyzer::builder(RawTokenizer::default()) + .filter(Stemmer::new(Language::Greek)) + .build(); + let mut stream = analyzer.token_stream("αντιθετε"); + + assert!(stream.advance()); + assert_eq!(stream.token().text, "ανετ"); + } +} diff --git a/rust/lance-tokenizer/src/stop_word_filter.rs b/rust/lance-tokenizer/src/stop_word_filter.rs index 2acf0b3dbd5..9a690b0ec06 100644 --- a/rust/lance-tokenizer/src/stop_word_filter.rs +++ b/rust/lance-tokenizer/src/stop_word_filter.rs @@ -17,7 +17,12 @@ fn all_stop_words() -> impl Iterator { stop_words::get("ar"), stopwords::DANISH, stopwords::DUTCH, - stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words) rather + // than the local Tantivy-style list (~33 words), which omits extremely + // common pronouns/function words (you, my, your, we, she, what, ...). + // Those omissions let the highest-frequency English tokens through the + // ICU stop-word path and build pathologically large posting lists. + stop_words::get("en"), stopwords::FINNISH, stopwords::FRENCH, stopwords::GERMAN, @@ -51,7 +56,11 @@ impl StopWordFilter { Language::Arabic => stop_words::get("ar"), Language::Danish => stopwords::DANISH, Language::Dutch => stopwords::DUTCH, - Language::English => stopwords::ENGLISH, + // Use the fuller `stop-words` crate English list (~198 words); the + // local Tantivy-style list (~33 words) omits common pronouns/function + // words (you, my, your, we, ...) that would otherwise leak through + // stop-word removal and build pathologically large posting lists. + Language::English => stop_words::get("en"), Language::Finnish => stopwords::FINNISH, Language::French => stopwords::FRENCH, Language::German => stopwords::GERMAN, diff --git a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs index 227556ba527..2ac3f4a28aa 100644 --- a/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs +++ b/rust/lance-tokenizer/src/stop_word_filter/stopwords.rs @@ -37,12 +37,6 @@ ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -pub const ENGLISH: &[&str] = &[ - "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", - "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", - "they", "this", "to", "was", "will", "with", -]; - pub const DANISH: &[&str] = &[ "og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som", "på", "de", "med", "han", "af", "for", "ikke", "der", "var", "mig", "sig", "men", "et", "har", "om", "vi", "min", "havde", diff --git a/rust/lance-tokenizer/src/word_delimiter_filter.rs b/rust/lance-tokenizer/src/word_delimiter_filter.rs new file mode 100644 index 00000000000..70138529efe --- /dev/null +++ b/rust/lance-tokenizer/src/word_delimiter_filter.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::VecDeque; + +use crate::{Token, TokenFilter, TokenStream, Tokenizer}; + +/// Splits code identifiers into subword tokens. +/// +/// The original identifier can be preserved at the first subword position. If +/// a token is not a compound identifier then it is passed through unchanged. +/// +/// # Examples +/// +/// ``` +/// use lance_tokenizer::{CodeLexTokenizer, TextAnalyzer, TokenStream, WordDelimiterFilter}; +/// +/// let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false)) +/// .filter(WordDelimiterFilter::new(true, true)) +/// .build(); +/// let mut stream = analyzer.token_stream("parseHTML2JSON"); +/// +/// let mut tokens = Vec::new(); +/// stream.process(&mut |token| tokens.push(token.text.clone())); +/// assert_eq!(tokens, vec!["parseHTML2JSON", "parse", "HTML", "2", "JSON"]); +/// ``` +#[derive(Clone)] +pub struct WordDelimiterFilter { + preserve_original: bool, + split_on_numerics: bool, +} + +impl WordDelimiterFilter { + pub fn new(preserve_original: bool, split_on_numerics: bool) -> Self { + Self { + preserve_original, + split_on_numerics, + } + } +} + +impl TokenFilter for WordDelimiterFilter { + type Tokenizer = WordDelimiterFilterWrapper; + + fn transform(self, tokenizer: T) -> Self::Tokenizer { + WordDelimiterFilterWrapper { + tokenizer, + preserve_original: self.preserve_original, + split_on_numerics: self.split_on_numerics, + } + } +} + +#[derive(Clone)] +/// Tokenizer wrapper produced by [`WordDelimiterFilter`]. +pub struct WordDelimiterFilterWrapper { + tokenizer: T, + preserve_original: bool, + split_on_numerics: bool, +} + +impl Tokenizer for WordDelimiterFilterWrapper { + type TokenStream<'a> = WordDelimiterTokenStream>; + + fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> { + WordDelimiterTokenStream { + tail: self.tokenizer.token_stream(text), + current: Token::default(), + pending: VecDeque::new(), + preserve_original: self.preserve_original, + split_on_numerics: self.split_on_numerics, + position_offset: 0, + } + } +} + +/// Token stream produced by [`WordDelimiterFilterWrapper`]. +pub struct WordDelimiterTokenStream { + tail: T, + current: Token, + pending: VecDeque, + preserve_original: bool, + split_on_numerics: bool, + position_offset: usize, +} + +#[derive(Clone, Copy)] +struct CharInfo { + offset: usize, + ch: char, +} + +fn is_identifier_text(text: &str) -> bool { + text.chars().any(|ch| ch == '_' || ch.is_alphanumeric()) +} + +fn split_identifier(text: &str, split_on_numerics: bool) -> Vec<(usize, usize)> { + let mut pieces = Vec::new(); + let mut segment_start = None; + for (offset, ch) in text.char_indices() { + if ch == '_' { + if let Some(start) = segment_start.take() { + split_segment(text, start, offset, split_on_numerics, &mut pieces); + } + } else if segment_start.is_none() { + segment_start = Some(offset); + } + } + if let Some(start) = segment_start { + split_segment(text, start, text.len(), split_on_numerics, &mut pieces); + } + pieces +} + +fn split_segment( + text: &str, + start: usize, + end: usize, + split_on_numerics: bool, + pieces: &mut Vec<(usize, usize)>, +) { + if start == end { + return; + } + let chars = text[start..end] + .char_indices() + .map(|(offset, ch)| CharInfo { + offset: start + offset, + ch, + }) + .collect::>(); + let mut piece_start = start; + for idx in 1..chars.len() { + if is_boundary(&chars, idx, split_on_numerics) { + pieces.push((piece_start, chars[idx].offset)); + piece_start = chars[idx].offset; + } + } + pieces.push((piece_start, end)); +} + +fn is_boundary(chars: &[CharInfo], idx: usize, split_on_numerics: bool) -> bool { + let prev = chars[idx - 1].ch; + let cur = chars[idx].ch; + if split_on_numerics && prev.is_ascii_digit() != cur.is_ascii_digit() { + return true; + } + if prev.is_lowercase() && cur.is_uppercase() { + return true; + } + if prev.is_uppercase() + && cur.is_uppercase() + && chars + .get(idx + 1) + .is_some_and(|next| next.ch.is_lowercase()) + { + return true; + } + false +} + +impl WordDelimiterTokenStream { + fn refill(&mut self) -> bool { + while self.pending.is_empty() { + if !self.tail.advance() { + return false; + } + self.enqueue_current_tail_token(); + } + true + } + + fn enqueue_current_tail_token(&mut self) { + let token = self.tail.token(); + let adjusted_position = token.position.saturating_add(self.position_offset); + if !is_identifier_text(&token.text) { + let mut token = token.clone(); + token.position = adjusted_position; + self.pending.push_back(token); + return; + } + + let parts = split_identifier(&token.text, self.split_on_numerics); + if parts.len() <= 1 { + let mut token = token.clone(); + token.position = adjusted_position; + self.pending.push_back(token); + return; + } + + if self.preserve_original { + let mut original = token.clone(); + original.position = adjusted_position; + original.position_length = parts.len(); + self.pending.push_back(original); + } + + for (part_idx, (start, end)) in parts.iter().copied().enumerate() { + let mut part = token.clone(); + part.offset_from = token.offset_from + start; + part.offset_to = token.offset_from + end; + part.position = adjusted_position + part_idx; + part.position_length = 1; + part.text.clear(); + part.text.push_str(&token.text[start..end]); + self.pending.push_back(part); + } + self.position_offset += parts.len() - 1; + } +} + +impl TokenStream for WordDelimiterTokenStream { + fn advance(&mut self) -> bool { + if !self.refill() { + return false; + } + let Some(token) = self.pending.pop_front() else { + debug_assert!(false, "pending token should be available after refill"); + return false; + }; + self.current = token; + true + } + + fn token(&self) -> &Token { + &self.current + } + + fn token_mut(&mut self) -> &mut Token { + &mut self.current + } +} + +#[cfg(test)] +mod tests { + use crate::{CodeLexTokenizer, TextAnalyzer, Token, WordDelimiterFilter}; + + fn collect_tokens(text: &str) -> Vec { + let mut analyzer = TextAnalyzer::builder(CodeLexTokenizer::new(false)) + .filter(WordDelimiterFilter::new(true, true)) + .build(); + let mut stream = analyzer.token_stream(text); + let mut tokens = Vec::new(); + stream.process(&mut |token| tokens.push(token.clone())); + tokens + } + + #[test] + fn test_word_delimiter_code_identifiers() { + let tokens = collect_tokens("getUserName XMLHttpRequest parseHTML2JSON utf8_reader"); + let texts = tokens + .iter() + .map(|token| token.text.as_str()) + .collect::>(); + assert_eq!( + texts, + vec![ + "getUserName", + "get", + "User", + "Name", + "XMLHttpRequest", + "XML", + "Http", + "Request", + "parseHTML2JSON", + "parse", + "HTML", + "2", + "JSON", + "utf8_reader", + "utf", + "8", + "reader", + ] + ); + } + + #[test] + fn test_word_delimiter_positions_span_compounds() { + let tokens = collect_tokens("getUserName next"); + let positions = tokens + .iter() + .map(|token| (token.text.as_str(), token.position, token.position_length)) + .collect::>(); + assert_eq!( + positions, + vec![ + ("getUserName", 0, 3), + ("get", 0, 1), + ("User", 1, 1), + ("Name", 2, 1), + ("next", 3, 1), + ] + ); + } +} diff --git a/rust/lance/Cargo.toml b/rust/lance/Cargo.toml index 36ea5facc29..417d41a5fe1 100644 --- a/rust/lance/Cargo.toml +++ b/rust/lance/Cargo.toml @@ -12,7 +12,8 @@ keywords.workspace = true categories.workspace = true [package.metadata.docs.rs] -features = [] +# Publish the feature-gated metrics catalogue; keep cloud backends off. +features = ["metrics"] no-default-features = true [package.metadata.cargo-machete] @@ -47,7 +48,7 @@ async-trait.workspace = true byteorder.workspace = true bytes.workspace = true chrono.workspace = true -clap = { version = "4.1.1", features = ["derive"], optional = true } +clap = { workspace = true, optional = true } # Only used by the (disabled) `mem_wal_kv_point_lookup` benchmark's RocksDB arm. # Commented out so CI's all-features build never compiles the bundled librocksdb # C++ sources (needs libclang). Uncomment with the bench target + feature to run. @@ -55,18 +56,14 @@ clap = { version = "4.1.1", features = ["derive"], optional = true } crossbeam-queue = { workspace = true } crossbeam-skiplist.workspace = true # This is already used by datafusion -dashmap = "6" -# matches arrow-rs use -half.workspace = true +dashmap.workspace = true # Fast non-cryptographic hasher for the hot FTS mem-index insert path. rustc-hash = "2.1" # Compact FST term dictionary for the FTS mem-index partitions. -fst = "0.4" +fst.workspace = true itertools.workspace = true moka.workspace = true object_store = { workspace = true } -aws-credential-types.workspace = true -aws-credential-types.optional = true pin-project.workspace = true prost.workspace = true prost-types.workspace = true @@ -100,7 +97,7 @@ tokio-util = { workspace = true } [build-dependencies] prost-build.workspace = true -protobuf-src = { version = "2.1", optional = true } +protobuf-src = { workspace = true, optional = true } [target.'cfg(target_os = "linux")'.dev-dependencies] pprof.workspace = true @@ -110,17 +107,18 @@ lzma-sys = { version = "0.1" } [dev-dependencies] lance-test-macros = { workspace = true } lance-datagen = { workspace = true } +lance-datafusion = { workspace = true, features = ["datagen"] } pretty_assertions = { workspace = true } libc = { workspace = true } clap = { workspace = true, features = ["derive"] } criterion = { workspace = true } approx.workspace = true -all_asserts = "2.3.1" +all_asserts.workspace = true mock_instant.workspace = true lance-testing = { workspace = true } lance-io = { workspace = true, features = ["test-util"] } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -env_logger = "0.11.7" +env_logger.workspace = true tempfile.workspace = true test-log.workspace = true tracing-chrome = "0.7.1" @@ -129,16 +127,19 @@ serial_test = { workspace = true } tracking-allocator = { version = "0.4", features = ["tracing-compat"] } # For S3 / DynamoDB tests aws-config = { workspace = true } +aws-credential-types = { workspace = true } aws-sdk-s3 = { workspace = true, default-features = false, features = ["default-https-client", "http-1x", "rt-tokio"] } +half.workspace = true geoarrow-array = { workspace = true } geoarrow-schema = { workspace = true } geo-types = { workspace = true } datafusion-substrait = { workspace = true } -parquet = { version = "58", default-features = false, features = ["arrow", "async"] } -reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json"] } +parquet = { workspace = true } +reqwest = { workspace = true, features = ["rustls-tls"] } [features] default = ["aws", "azure", "gcp", "oss", "huggingface", "tencent", "tos", "goosefs", "geo"] +backtrace = ["lance-core/backtrace"] fp16kernels = ["lance-linalg/fp16kernels"] # Prevent dynamic linking of lzma, which comes from datafusion cli = ["dep:clap", "lzma-sys/static"] @@ -152,7 +153,7 @@ protoc = [ "lance-index/protoc", "lance-table/protoc", ] -aws = ["lance-io/aws", "dep:aws-credential-types"] +aws = ["lance-io/aws"] gcp = ["lance-io/gcp"] azure = ["lance-io/azure"] oss = ["lance-io/oss"] @@ -160,6 +161,8 @@ tencent = ["lance-io/tencent"] goosefs = ["lance-io/goosefs"] tos = ["lance-io/tos"] huggingface = ["lance-io/huggingface"] +# Publish object store metrics via the `metrics` crate. +metrics = ["lance-io/metrics"] geo = ["lance-datafusion/geo", "lance-index/geo"] # Enable slow integration tests (disabled by default in CI) slow_tests = [] @@ -195,6 +198,10 @@ harness = false name = "scan" harness = false +[[bench]] +name = "s3_file_reader_diagnostics" +harness = false + [[bench]] name = "count_pushdown" harness = false @@ -235,6 +242,10 @@ harness = false name = "distributed_vector_build" harness = false +[[bench]] +name = "frag_reuse" +harness = false + [[bench]] name = "mem_wal_write" path = "benches/mem_wal/write/mem_wal_write.rs" diff --git a/rust/lance/benches/concurrent_append.rs b/rust/lance/benches/concurrent_append.rs index ac7cf3f610f..f0103626739 100644 --- a/rust/lance/benches/concurrent_append.rs +++ b/rust/lance/benches/concurrent_append.rs @@ -47,6 +47,7 @@ use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use criterion::{Criterion, criterion_group, criterion_main}; use lance::dataset::{Dataset, InsertBuilder, WriteMode, WriteParams, builder::DatasetBuilder}; use lance::session::Session; +use lance_core::utils::parse::str_is_truthy; use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry, StorageOptionsAccessor}; use std::collections::HashMap; use std::env; @@ -67,9 +68,7 @@ fn env_usize(key: &str, default: usize) -> usize { } fn env_bool(key: &str) -> bool { - env::var(key) - .map(|s| s.eq_ignore_ascii_case("true")) - .unwrap_or(false) + env::var(key).map(|s| str_is_truthy(&s)).unwrap_or(false) } fn storage_label(uri: &str) -> &'static str { diff --git a/rust/lance/benches/frag_reuse.rs b/rust/lance/benches/frag_reuse.rs new file mode 100644 index 00000000000..de4adc01c01 --- /dev/null +++ b/rust/lance/benches/frag_reuse.rs @@ -0,0 +1,826 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Reproducible comparison of the legacy per-row FRI maps and the compact +//! bitmap/rank representation. +//! +//! This benchmark starts from the same decoded `FragReuseIndexDetails` for +//! both implementations. The open measurement includes Roaring deserialization +//! and construction of the queryable runtime representation, but excludes +//! object-store I/O and protobuf decoding. Pass `--storage-uri` to additionally +//! measure external FRI detail fetch, protobuf decode, and runtime open on local +//! storage or an object store such as S3. + +#![allow(clippy::print_stdout)] + +use std::collections::HashMap; +use std::hint::black_box; +use std::io::Cursor; +use std::sync::Arc; +use std::time::Instant; + +use lance::dataset::optimize::remapping::transpose_row_ids_from_digest; +use lance_core::deepsize::{Context, DeepSizeOf}; +use lance_core::utils::address::RowAddress; +use lance_index::frag_reuse::{ + CompactFragReuseIndex, FragDigest, FragReuseGroup, FragReuseIndexDetails, FragReuseVersion, +}; +use lance_io::object_store::ObjectStore as LanceObjectStore; +use lance_table::format::pb::fragment_reuse_index_details::InlineContent; +use object_store::path::Path; +use prost::Message; +use roaring::RoaringTreemap; +use serde_json::json; +use tokio::io::AsyncWriteExt; +use uuid::Uuid; + +const EXTERNAL_DETAILS_THRESHOLD: usize = 204_800; + +#[derive(Clone, Copy)] +struct Case { + name: &'static str, + rows: usize, + changed_basis_points: u32, + chain_len: usize, + old_fragment_count: usize, + groups_per_version: usize, +} + +struct Config { + repeats: usize, + lookups: usize, + batch_size: usize, + storage_repeats: usize, + storage_uri: Option, + quick: bool, +} + +struct StorageTarget { + kind: &'static str, + store: Arc, + base_path: Path, +} + +struct LegacyFragReuseIndex { + row_id_maps: Vec>>, + details: FragReuseIndexDetails, +} + +impl LegacyFragReuseIndex { + fn open(details: &FragReuseIndexDetails) -> Self { + let mut row_id_maps = Vec::with_capacity(details.versions.len()); + for version in &details.versions { + let mut row_id_map = HashMap::new(); + for group in &version.groups { + let changed_row_addrs = + RoaringTreemap::deserialize_from(Cursor::new(&group.changed_row_addrs)) + .unwrap(); + row_id_map.extend(transpose_row_ids_from_digest( + changed_row_addrs, + &group.old_frags, + &group.new_frags, + )); + } + row_id_maps.push(row_id_map); + } + Self { + row_id_maps, + details: details.clone(), + } + } + + fn remap_row_id(&self, row_id: u64) -> Option { + let mut mapped = Some(row_id); + for row_id_map in &self.row_id_maps { + if let Some(current) = mapped { + mapped = row_id_map.get(¤t).copied().unwrap_or(mapped); + } + } + mapped + } + + fn remap_row_ids_in_place(&self, row_ids: &mut [Option]) { + for row_id in row_ids { + if let Some(current) = *row_id { + *row_id = self.remap_row_id(current); + } + } + } +} + +impl DeepSizeOf for LegacyFragReuseIndex { + fn deep_size_of_children(&self, context: &mut Context) -> usize { + self.row_id_maps.deep_size_of_children(context) + + self.details.deep_size_of_children(context) + } +} + +#[tokio::main] +async fn main() { + let config = parse_config(); + let cases = if config.quick { + vec![Case { + name: "quick", + rows: 100_000, + changed_basis_points: 5_000, + chain_len: 2, + old_fragment_count: 16, + groups_per_version: 2, + }] + } else { + vec![ + Case { + name: "small_very_sparse", + rows: 100_000, + changed_basis_points: 10, + chain_len: 1, + old_fragment_count: 16, + groups_per_version: 1, + }, + Case { + name: "small_sparse", + rows: 100_000, + changed_basis_points: 100, + chain_len: 1, + old_fragment_count: 16, + groups_per_version: 1, + }, + Case { + name: "medium_density_5", + rows: 1_000_000, + changed_basis_points: 500, + chain_len: 1, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "medium_sparse_chain", + rows: 1_000_000, + changed_basis_points: 1_000, + chain_len: 4, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "medium_density_25", + rows: 1_000_000, + changed_basis_points: 2_500, + chain_len: 1, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "medium_balanced", + rows: 1_000_000, + changed_basis_points: 5_000, + chain_len: 1, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "medium_density_75", + rows: 1_000_000, + changed_basis_points: 7_500, + chain_len: 1, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "medium_dense", + rows: 1_000_000, + changed_basis_points: 9_000, + chain_len: 1, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "medium_density_99", + rows: 1_000_000, + changed_basis_points: 9_900, + chain_len: 1, + old_fragment_count: 64, + groups_per_version: 8, + }, + Case { + name: "fragmented_sparse", + rows: 1_000_000, + changed_basis_points: 100, + chain_len: 1, + old_fragment_count: 4_096, + groups_per_version: 256, + }, + Case { + name: "fragmented_dense_chain", + rows: 500_000, + changed_basis_points: 9_000, + chain_len: 4, + old_fragment_count: 1_024, + groups_per_version: 128, + }, + Case { + name: "long_dense_chain", + rows: 250_000, + changed_basis_points: 9_000, + chain_len: 8, + old_fragment_count: 128, + groups_per_version: 16, + }, + Case { + name: "large_sparse", + rows: 10_000_000, + changed_basis_points: 100, + chain_len: 1, + old_fragment_count: 128, + groups_per_version: 16, + }, + Case { + name: "large_balanced_chain", + rows: 5_000_000, + changed_basis_points: 5_000, + chain_len: 4, + old_fragment_count: 256, + groups_per_version: 32, + }, + Case { + name: "large_dense", + rows: 5_000_000, + changed_basis_points: 9_000, + chain_len: 1, + old_fragment_count: 128, + groups_per_version: 16, + }, + ] + }; + + let storage = match config.storage_uri.as_deref() { + Some(uri) => Some(StorageTarget::open(uri).await), + None => None, + }; + + println!( + "{}", + json!({ + "type": "environment", + "os": std::env::consts::OS, + "arch": std::env::consts::ARCH, + "repeats": config.repeats, + "lookups": config.lookups, + "batch_size": config.batch_size, + "storage_repeats": config.storage_repeats, + "storage_kind": storage.as_ref().map(|target| target.kind), + "unaffected_query_percent": 10, + "profile": "cargo bench (release)", + "memory_metric": "DeepSizeOf retained-bytes proxy; retained Roaring containers use serialized_size", + }) + ); + + for case in cases { + run_case(case, &config, storage.as_ref()).await; + } +} + +fn parse_config() -> Config { + let mut config = Config { + repeats: 30, + lookups: 200_000, + batch_size: 65_536, + storage_repeats: 10, + storage_uri: None, + quick: false, + }; + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + // `cargo bench` passes this libtest-compatible marker even for a + // custom harness. + "--bench" => {} + "--quick" => { + config.quick = true; + config.repeats = 5; + config.lookups = 20_000; + config.batch_size = 8_192; + config.storage_repeats = 3; + } + "--repeats" => config.repeats = parse_value(&mut args, "--repeats"), + "--lookups" => config.lookups = parse_value(&mut args, "--lookups"), + "--batch-size" => config.batch_size = parse_value(&mut args, "--batch-size"), + "--storage-repeats" => { + config.storage_repeats = parse_value(&mut args, "--storage-repeats") + } + "--storage-uri" => { + config.storage_uri = Some( + args.next() + .unwrap_or_else(|| panic!("--storage-uri requires a URI")), + ) + } + other => panic!("unknown argument: {other}"), + } + } + assert!(config.repeats > 0, "--repeats must be greater than zero"); + assert!(config.lookups > 0, "--lookups must be greater than zero"); + assert!( + config.batch_size > 0, + "--batch-size must be greater than zero" + ); + assert!( + config.storage_repeats > 0, + "--storage-repeats must be greater than zero" + ); + config +} + +fn parse_value(args: &mut impl Iterator, name: &str) -> usize { + args.next() + .unwrap_or_else(|| panic!("{name} requires a value")) + .parse() + .unwrap_or_else(|_| panic!("{name} requires a positive integer")) +} + +async fn run_case(case: Case, config: &Config, storage: Option<&StorageTarget>) { + let (details, baseline_frags) = generate_details(case); + let queries = sample_queries(&baseline_frags, config.lookups); + let random_batch = queries + .iter() + .take(config.batch_size) + .copied() + .map(Some) + .collect::>(); + let mut fragment_grouped_batch = random_batch.clone(); + fragment_grouped_batch.sort_by_key(|row_id| row_id.map(|row_id| row_id >> 32)); + let mut monotonic_batch = random_batch.clone(); + monotonic_batch.sort_unstable(); + + let legacy = LegacyFragReuseIndex::open(&details); + let compact = + CompactFragReuseIndex::try_new(Uuid::nil(), details.clone()).expect("valid benchmark FRI"); + for row_id in &queries { + assert_eq!( + legacy.remap_row_id(*row_id), + compact.remap_row_id(*row_id), + "legacy and compact semantics differ for case {} at row address {}", + case.name, + row_id + ); + } + + emit_memory(case, "legacy_hash_map", legacy.deep_size_of()); + emit_memory(case, "compact_rank", compact.deep_size_of()); + + // Warm allocator and code paths before collecting samples. + black_box(LegacyFragReuseIndex::open(&details)); + black_box(CompactFragReuseIndex::try_new(Uuid::nil(), details.clone()).unwrap()); + + let legacy_open = measure(config.repeats, || { + black_box(LegacyFragReuseIndex::open(black_box(&details))); + }); + let compact_open = measure(config.repeats, || { + black_box(CompactFragReuseIndex::try_new(Uuid::nil(), black_box(details.clone())).unwrap()); + }); + emit_timing(case, "open_ns", "legacy_hash_map", 1, legacy_open); + emit_timing(case, "open_ns", "compact_rank", 1, compact_open); + + let legacy_lookup = measure(config.repeats, || { + for row_id in &queries { + black_box(legacy.remap_row_id(black_box(*row_id))); + } + }); + let compact_lookup = measure(config.repeats, || { + for row_id in &queries { + black_box(compact.remap_row_id(black_box(*row_id))); + } + }); + emit_timing( + case, + "single_lookup_ns_per_row", + "legacy_hash_map", + queries.len(), + legacy_lookup, + ); + emit_timing( + case, + "single_lookup_ns_per_row", + "compact_rank", + queries.len(), + compact_lookup, + ); + + measure_batch_order( + case, + config.repeats, + "batch_random_ns_per_row", + &random_batch, + &legacy, + &compact, + ); + measure_batch_order( + case, + config.repeats, + "batch_fragment_grouped_ns_per_row", + &fragment_grouped_batch, + &legacy, + &compact, + ); + measure_batch_order( + case, + config.repeats, + "batch_monotonic_ns_per_row", + &monotonic_batch, + &legacy, + &compact, + ); + + if let Some(storage) = storage { + storage + .benchmark_external_open(case, &details, config.storage_repeats) + .await; + } +} + +fn measure_batch_order( + case: Case, + repeats: usize, + metric: &str, + batch_source: &[Option], + legacy: &LegacyFragReuseIndex, + compact: &CompactFragReuseIndex, +) { + let legacy_batch = measure(repeats, || { + let mut batch = batch_source.to_vec(); + legacy.remap_row_ids_in_place(black_box(&mut batch)); + black_box(batch); + }); + let compact_batch = measure(repeats, || { + let mut batch = batch_source.to_vec(); + compact.remap_row_ids_in_place(black_box(&mut batch)); + black_box(batch); + }); + emit_timing( + case, + metric, + "legacy_hash_map", + batch_source.len(), + legacy_batch, + ); + emit_timing( + case, + metric, + "compact_rank", + batch_source.len(), + compact_batch, + ); +} + +impl StorageTarget { + async fn open(uri: &str) -> Self { + let (store, base_path) = LanceObjectStore::from_uri(uri) + .await + .unwrap_or_else(|error| panic!("failed to open benchmark storage URI {uri}: {error}")); + let kind = if uri.starts_with("s3://") { + "s3" + } else if uri.starts_with("file://") || !uri.contains("://") { + "local" + } else { + "object_store" + }; + Self { + kind, + store, + base_path, + } + } + + async fn benchmark_external_open( + &self, + case: Case, + details: &FragReuseIndexDetails, + repeats: usize, + ) { + let encoded = InlineContent::from(details).encode_to_vec(); + if encoded.len() <= EXTERNAL_DETAILS_THRESHOLD { + println!( + "{}", + json!({ + "type": "storage_skip", + "case": case.name, + "storage": self.kind, + "encoded_bytes": encoded.len(), + "reason": "FRI details remain inline at the production external-file threshold", + }) + ); + return; + } + + let path = self + .base_path + .clone() + .join(format!("{}.details.binpb", case.name)); + let mut writer = self.store.create(&path).await.unwrap_or_else(|error| { + panic!( + "failed to create {} benchmark object {}: {error}", + self.kind, path + ) + }); + writer.write_all(&encoded).await.unwrap_or_else(|error| { + panic!( + "failed to write {} benchmark object {}: {error}", + self.kind, path + ) + }); + writer.shutdown().await.unwrap_or_else(|error| { + panic!( + "failed to finish {} benchmark object {}: {error}", + self.kind, path + ) + }); + + let loaded = self.load_details(&path, encoded.len()).await; + assert_eq!( + &loaded, details, + "{} storage roundtrip changed FRI details for case {}", + self.kind, case.name + ); + + let mut legacy_samples = Vec::with_capacity(repeats); + let mut compact_samples = Vec::with_capacity(repeats); + for repeat in 0..repeats { + if repeat % 2 == 0 { + legacy_samples.push(self.measure_legacy_open(&path, encoded.len()).await); + compact_samples.push(self.measure_compact_open(&path, encoded.len()).await); + } else { + compact_samples.push(self.measure_compact_open(&path, encoded.len()).await); + legacy_samples.push(self.measure_legacy_open(&path, encoded.len()).await); + } + } + emit_storage_timing( + case, + self.kind, + encoded.len(), + "legacy_hash_map", + legacy_samples, + ); + emit_storage_timing( + case, + self.kind, + encoded.len(), + "compact_rank", + compact_samples, + ); + } + + async fn load_details(&self, path: &Path, encoded_len: usize) -> FragReuseIndexDetails { + let data = self + .store + .open(path) + .await + .unwrap_or_else(|error| panic!("failed to open {} object {path}: {error}", self.kind)) + .get_range(0..encoded_len) + .await + .unwrap_or_else(|error| panic!("failed to read {} object {path}: {error}", self.kind)); + let content = InlineContent::decode(data).unwrap_or_else(|error| { + panic!("failed to decode {} FRI details {path}: {error}", self.kind) + }); + FragReuseIndexDetails::try_from(content).unwrap_or_else(|error| { + panic!( + "failed to convert {} FRI details {path}: {error}", + self.kind + ) + }) + } + + async fn measure_legacy_open(&self, path: &Path, encoded_len: usize) -> u128 { + let start = Instant::now(); + let details = self.load_details(path, encoded_len).await; + black_box(LegacyFragReuseIndex::open(&details)); + start.elapsed().as_nanos() + } + + async fn measure_compact_open(&self, path: &Path, encoded_len: usize) -> u128 { + let start = Instant::now(); + let details = self.load_details(path, encoded_len).await; + black_box( + CompactFragReuseIndex::try_new(Uuid::nil(), details) + .expect("valid stored benchmark FRI"), + ); + start.elapsed().as_nanos() + } +} + +fn measure(mut repeats: usize, mut operation: impl FnMut()) -> Vec { + let mut samples = Vec::with_capacity(repeats); + while repeats > 0 { + let start = Instant::now(); + operation(); + samples.push(start.elapsed().as_nanos()); + repeats -= 1; + } + samples +} + +fn emit_memory(case: Case, implementation: &str, bytes: usize) { + println!( + "{}", + json!({ + "type": "memory", + "case": case.name, + "rows": case.rows, + "changed_basis_points": case.changed_basis_points, + "chain_len": case.chain_len, + "old_fragment_count": case.old_fragment_count, + "groups_per_version": case.groups_per_version, + "implementation": implementation, + "retained_bytes_proxy": bytes, + }) + ); +} + +fn emit_timing( + case: Case, + metric: &str, + implementation: &str, + operations: usize, + samples_ns: Vec, +) { + let mut normalized = samples_ns + .iter() + .map(|sample| *sample as f64 / operations as f64) + .collect::>(); + normalized.sort_by(f64::total_cmp); + println!( + "{}", + json!({ + "type": "timing", + "case": case.name, + "rows": case.rows, + "changed_basis_points": case.changed_basis_points, + "chain_len": case.chain_len, + "old_fragment_count": case.old_fragment_count, + "groups_per_version": case.groups_per_version, + "implementation": implementation, + "metric": metric, + "operations_per_sample": operations, + "repeats": samples_ns.len(), + "p50": percentile(&normalized, 0.50), + "p99": percentile(&normalized, 0.99), + "raw_total_ns": samples_ns, + }) + ); +} + +fn emit_storage_timing( + case: Case, + storage: &str, + encoded_bytes: usize, + implementation: &str, + samples_ns: Vec, +) { + let mut normalized = samples_ns + .iter() + .map(|sample| *sample as f64) + .collect::>(); + normalized.sort_by(f64::total_cmp); + println!( + "{}", + json!({ + "type": "storage_timing", + "case": case.name, + "rows": case.rows, + "changed_basis_points": case.changed_basis_points, + "chain_len": case.chain_len, + "old_fragment_count": case.old_fragment_count, + "groups_per_version": case.groups_per_version, + "storage": storage, + "encoded_bytes": encoded_bytes, + "implementation": implementation, + "metric": "external_details_fetch_decode_open_ns", + "operations_per_sample": 1, + "repeats": samples_ns.len(), + "p50": percentile(&normalized, 0.50), + "p99": percentile(&normalized, 0.99), + "raw_total_ns": samples_ns, + }) + ); +} + +fn percentile(sorted: &[f64], percentile: f64) -> f64 { + let index = ((sorted.len() as f64 * percentile).ceil() as usize) + .saturating_sub(1) + .min(sorted.len() - 1); + sorted[index] +} + +fn generate_details(case: Case) -> (FragReuseIndexDetails, Vec) { + let mut next_fragment_id = 1u64; + let mut old_frags = distribute_rows(case.rows, case.old_fragment_count, &mut next_fragment_id); + let baseline_frags = old_frags.clone(); + let mut versions = Vec::with_capacity(case.chain_len); + + for version_idx in 0..case.chain_len { + let mut groups = Vec::new(); + let mut next_old_frags = Vec::new(); + let mut ordinal = 0u64; + let group_size = old_frags.len().div_ceil(case.groups_per_version).max(1); + for old_group in old_frags.chunks(group_size) { + let mut changed = RoaringTreemap::new(); + let mut old_with_deletions = Vec::with_capacity(old_group.len()); + for frag in old_group { + let mut num_changed = 0usize; + for offset in 0..frag.physical_rows as u32 { + let hash = ordinal + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add((version_idx as u64 + 1) * 1_442_695_040_888_963_407); + if (hash >> 32) % 10_000 < case.changed_basis_points as u64 { + changed.insert(u64::from(RowAddress::new_from_parts( + frag.id as u32, + offset, + ))); + num_changed += 1; + } + ordinal += 1; + } + old_with_deletions.push(FragDigest { + id: frag.id, + physical_rows: frag.physical_rows, + num_deleted_rows: frag.physical_rows - num_changed, + }); + } + + let num_changed = changed.len() as usize; + let new_fragment_count = if num_changed == 0 { + 0 + } else { + (old_group.len() + old_group.len() / 2) + .max(1) + .min(num_changed) + }; + let new_frags = distribute_rows(num_changed, new_fragment_count, &mut next_fragment_id); + let mut changed_row_addrs = Vec::with_capacity(changed.serialized_size()); + changed.serialize_into(&mut changed_row_addrs).unwrap(); + groups.push(FragReuseGroup { + changed_row_addrs, + old_frags: old_with_deletions, + new_frags: new_frags.clone(), + }); + next_old_frags.extend(new_frags); + } + + versions.push(FragReuseVersion { + dataset_version: version_idx as u64 + 1, + groups, + }); + old_frags = next_old_frags; + } + + (FragReuseIndexDetails { versions }, baseline_frags) +} + +fn distribute_rows(total_rows: usize, count: usize, next_id: &mut u64) -> Vec { + if count == 0 { + return Vec::new(); + } + let base = total_rows / count; + let remainder = total_rows % count; + (0..count) + .map(|index| { + let digest = FragDigest { + id: *next_id, + physical_rows: base + usize::from(index < remainder), + num_deleted_rows: 0, + }; + *next_id += 1; + digest + }) + .collect() +} + +fn sample_queries(fragments: &[FragDigest], count: usize) -> Vec { + let total_rows = fragments + .iter() + .map(|frag| frag.physical_rows) + .sum::(); + let unaffected_fragment = u32::MAX - 1; + let mut state = 0x4d595df4d0f33173u64; + (0..count) + .map(|index| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + if index % 10 == 0 { + return u64::from(RowAddress::new_from_parts( + unaffected_fragment, + state as u32, + )); + } + let mut logical_row = state as usize % total_rows; + for frag in fragments { + if logical_row < frag.physical_rows { + return u64::from(RowAddress::new_from_parts( + frag.id as u32, + logical_row as u32, + )); + } + logical_row -= frag.physical_rows; + } + unreachable!("logical row is bounded by total_rows") + }) + .collect() +} diff --git a/rust/lance/benches/manifest_commit.rs b/rust/lance/benches/manifest_commit.rs index 2a98a37a498..f657f81dc95 100644 --- a/rust/lance/benches/manifest_commit.rs +++ b/rust/lance/benches/manifest_commit.rs @@ -46,6 +46,7 @@ use criterion::{Criterion, criterion_group, criterion_main}; use lance::dataset::builder::DatasetBuilder; use lance::dataset::{CommitBuilder, Dataset, InsertBuilder, WriteMode, WriteParams}; use lance::session::Session; +use lance_core::utils::parse::str_is_truthy; use lance_io::object_store::ObjectStoreRegistry; use std::sync::Arc; use std::time::Instant; @@ -71,13 +72,13 @@ fn get_num_iterations() -> usize { fn get_delete_dataset() -> bool { std::env::var("DELETE_DATASET") - .map(|s| s.to_lowercase() == "true") + .map(|s| str_is_truthy(&s)) .unwrap_or(false) } fn get_enable_cache() -> bool { std::env::var("ENABLE_CACHE") - .map(|s| s.to_lowercase() == "true") + .map(|s| str_is_truthy(&s)) .unwrap_or(false) } diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs index 63f9368c3c5..188e2c73ce2 100644 --- a/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs +++ b/rust/lance/benches/mem_wal/fts/mem_wal_fineweb_fts.rs @@ -111,11 +111,6 @@ impl Mode { fn durable_write(self) -> bool { matches!(self, Self::SyncNoIndex | Self::SyncIndexed) } - - /// Index update happens inline in `put` (only meaningful when indexed). - fn sync_indexed_write(self) -> bool { - matches!(self, Self::SyncIndexed) - } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -428,7 +423,6 @@ fn shard_writer_config(args: &Args, shard_id: Uuid, disable_auto_flush: bool) -> }; let mut config = ShardWriterConfig::new(shard_id) .with_durable_write(args.mode.durable_write()) - .with_sync_indexed_write(args.mode.sync_indexed_write()) .with_max_memtable_size(args.max_memtable_size) .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes) .with_max_memtable_rows(max_rows) @@ -638,7 +632,7 @@ async fn run_read(args: &Args, uri: &str, corpus: &[String]) -> Result Result Result<()> { qps_nt, term_recall_v, phrase_recall_v, - index.memory_usage() as f64 / 1.0e6, + index.resident_bytes_exact() as f64 / 1.0e6, ); println!( "{{\"impl\":\"lance_fts\",\"run\":\"{}\",\"docs\":{},\"queries\":{},\"k\":{},\ @@ -738,7 +738,7 @@ fn run_bench(args: &BenchArgs) -> Result<()> { term_recall_v, phrase_recall_v, or_recall_v, - index.memory_usage(), + index.resident_bytes_exact(), ); Ok(()) } diff --git a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs index 08fdb76b95f..09812fff1fb 100644 --- a/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs +++ b/rust/lance/benches/mem_wal/fts/mem_wal_fts_read_bench.rs @@ -5,11 +5,11 @@ //! //! Sibling of `mem_wal_vector_bench.rs` / `mem_wal_point_lookup_bench.rs`: //! same `--phase prepare|search` shape, same `ShardWriter`-based ingestion -//! of flushed generations + an active memtable, same `--uri` cloud/local +//! of SSTables + an active memtable, same `--uri` cloud/local //! detection, and the same JSON output contract. The payload is real //! HuggingFace FineWeb `text` and the query path is -//! [`LsmFtsSearchPlanner`] (local scoring) over the base table + flushed -//! generations + active memtable. +//! [`LsmFtsSearchPlanner`] (local scoring) over the base table + SSTables +//! + active memtable. //! //! Each `search` invocation times a query set against the LSM hierarchy //! and reports latency percentiles. With `--with-baseline`, it also builds @@ -108,7 +108,7 @@ struct Args { uri: String, base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, batch_rows: usize, queries: usize, k: usize, @@ -126,7 +126,7 @@ impl Default for Args { uri: String::new(), base_rows: 1_000_000, max_memtable_rows: 100_000, - flushed_generations: 2, + sstables: 2, batch_rows: 1_000, queries: 200, k: 10, @@ -175,7 +175,7 @@ fn parse_args() -> Result { } "--base-rows" => args.base_rows = parse_val(&flag, &value)?, "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?, - "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?, + "--sstables" => args.sstables = parse_val(&flag, &value)?, "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?, "--queries" => args.queries = parse_val(&flag, &value)?, "--k" => args.k = parse_val(&flag, &value)?, @@ -664,7 +664,7 @@ async fn run_search(args: &Args) -> Result { // covering both the memtable payload and the query-term sample, instead // of re-reading the whole base corpus from parquet. let active_rows = args.max_memtable_rows / 2; - let total_memtable_rows = args.flushed_generations * args.max_memtable_rows + active_rows; + let total_memtable_rows = args.sstables * args.max_memtable_rows + active_rows; let sample_rows = args.base_rows.min(50_000); let load_rows = total_memtable_rows.max(sample_rows); println!("loading {load_rows} FineWeb rows for memtable payload + query sample ..."); @@ -673,14 +673,9 @@ async fn run_search(args: &Args) -> Result { let shard_id = Uuid::new_v4(); let row_bytes = 2048; // rough FineWeb text row size - // The memtable flush trigger is `estimated_size >= max_memtable_size || - // batch_store_full`. FineWeb text rows vary in size, so a byte threshold - // is an unreliable way to flush exactly one generation per - // `max_memtable_rows`. Instead make the *batch-count* cap the trigger: - // set `max_memtable_batches` to one generation's worth of batches so the - // store fills (and flushes) precisely at each generation boundary, - // independent of text length. Keep `max_memtable_size` high so it never - // pre-empts the batch-count trigger. + // The memtable seals at `max_memtable_rows`, giving one generation per cap. + // `max_memtable_batches` is sized to that same boundary, and + // `max_memtable_size` kept high so the byte threshold never pre-empts it. let batches_per_gen = (args.max_memtable_rows / args.batch_rows).max(1); let config = ShardWriterConfig { shard_id, @@ -688,7 +683,6 @@ async fn run_search(args: &Args) -> Result { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_memtable_size: args.max_memtable_rows * row_bytes * 100, max_memtable_rows: args.max_memtable_rows, max_memtable_batches: batches_per_gen, @@ -704,10 +698,8 @@ async fn run_search(args: &Args) -> Result { Duration::from_millis(500) }; - // Ingest flushed generations + 1 active (50% full). - let mut gen_sizes: Vec = (0..args.flushed_generations) - .map(|_| args.max_memtable_rows) - .collect(); + // Ingest SSTables + 1 active (50% full). + let mut gen_sizes: Vec = (0..args.sstables).map(|_| args.max_memtable_rows).collect(); gen_sizes.push(active_rows); let id_base = args.base_rows as i64; @@ -724,19 +716,19 @@ async fn run_search(args: &Args) -> Result { cursor += chunk; written += chunk; } - let is_flushed = gen_idx < args.flushed_generations; + let is_sstable = gen_idx < args.sstables; println!( " gen {}: wrote {} rows ({})", gen_idx + 1, gen_rows, - if is_flushed { "flushed" } else { "active" } + if is_sstable { "sstable" } else { "active" } ); - if is_flushed { + if is_sstable { tokio::time::sleep(flush_wait).await; } } // Wait for any triggered (sealed) memtable flushes to commit to the - // manifest before we snapshot it — otherwise the flushed generations + // manifest before we snapshot it — otherwise the SSTables // race the read and may not all be visible yet. writer.wait_for_flush_drain().await?; println!( @@ -750,17 +742,14 @@ async fn run_search(args: &Args) -> Result { let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } - let num_flushed = manifest - .as_ref() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); - println!("manifest: {num_flushed} flushed generations"); + let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0); + println!("manifest: {num_sstables} SSTables"); - // Flushed generations carry the same maintained secondary indexes as + // SSTables carry the same maintained secondary indexes as // the active memtable: the flush handler builds them during flush // (lance #6901), so each generation already has the FTS index and // both scoring modes use the fast indexed path. No manual indexing @@ -827,7 +816,7 @@ async fn run_search(args: &Args) -> Result { "uri_kind": if is_cloud_uri(&args.uri) { "cloud" } else { "local" }, "base_rows": args.base_rows, "max_memtable_rows": args.max_memtable_rows, - "flushed_generations": num_flushed, + "sstables": num_sstables, "active_rows": active_rows, "k": args.k, "queries": queries.len(), @@ -848,12 +837,12 @@ async fn run_search(args: &Args) -> Result { async fn run(args: Args) -> Result<()> { println!( - "bench=mem_wal_fts_read phase={} uri={} base_rows={} max_memtable_rows={} flushed_generations={} queries={} k={} with_baseline={}", + "bench=mem_wal_fts_read phase={} uri={} base_rows={} max_memtable_rows={} sstables={} queries={} k={} with_baseline={}", args.phase.as_str(), args.uri, args.base_rows, args.max_memtable_rows, - args.flushed_generations, + args.sstables, args.queries, args.k, args.with_baseline, diff --git a/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh b/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh index 7ea692ee0b1..31ededa04e2 100755 --- a/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh +++ b/rust/lance/benches/mem_wal/fts/run_fts_read_sweep.sh @@ -8,7 +8,7 @@ # # For each (backend, base_rows) the bench's `prepare` phase runs once to # write the base dataset + FTS index + MemWAL; then for each k the `search` -# phase ingests flushed generations + an active memtable through ShardWriter +# phase ingests SSTables + an active memtable through ShardWriter # and times the FTS query panel under both Local and LocalWithGlobalRescore # scoring modes. # @@ -23,8 +23,8 @@ # CACHE_DIR FineWeb shard download cache (default /lance-fineweb-cache) # BASE_ROWS_LIST space-separated base sizes (default "100000 1000000") # K_LIST space-separated top-k values (default "10 100") -# MAX_MEMTABLE_ROWS active/flushed memtable cap (default 100000) -# GENS_LIST space-separated flushed-generation counts (default "1 2 5") +# MAX_MEMTABLE_ROWS active/SSTable cap (default 100000) +# GENS_LIST space-separated SSTable counts (default "1 2 5") # QUERIES queries per config (default 200) # WITH_BASELINE "1" to also build the merged-index accuracy baseline # and report local-vs-merged Jaccard (default off) @@ -117,7 +117,7 @@ for backend in $BACKENDS; do --base-rows "$base_rows" --batch-rows 1000 \ --cache-dir "$CACHE_DIR" || continue - # search for each (flushed-generations, k) + # search for each (SSTables, k) for gens in $GENS_LIST; do for k in $K_LIST; do name="search_${backend}_${btag}_g${gens}_k${k}" @@ -136,7 +136,7 @@ for backend in $BACKENDS; do --phase search --uri "$uri" \ --base-rows "$base_rows" \ --max-memtable-rows "$MAX_MEMTABLE_ROWS" \ - --flushed-generations "$gens" \ + --sstables "$gens" \ --batch-rows 1000 \ --queries "$QUERIES" --k "$k" \ "${baseline_flag[@]}" \ diff --git a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs index 757c2539642..335a41d3bbd 100644 --- a/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs +++ b/rust/lance/benches/mem_wal/kv/mem_wal_kv_point_lookup.rs @@ -43,8 +43,7 @@ use datafusion::common::ScalarValue; use datafusion::prelude::SessionContext; use futures::TryStreamExt; use lance::dataset::mem_wal::scanner::{ - FlushedMemTableCache, InMemoryMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner, - ShardSnapshot, + InMemoryMemTableRef, LsmDataSourceCollector, LsmPointLookupPlanner, ShardSnapshot, SsTableCache, }; use lance::dataset::mem_wal::{DatasetMemWalExt, ShardWriterConfig}; use lance::dataset::{Dataset, WriteParams}; @@ -422,7 +421,7 @@ enum LanceReadMode { Plan, /// Probe the active MemTable's BTree index directly and materialize the /// row from the BatchStore, bypassing DataFusion. Single-active-memtable - /// fast path (no flushed generations); misses fall through as "not found". + /// fast path (no SSTables); misses fall through as "not found". Fast, /// Call the production `LsmPointLookupPlanner::lookup` API, which uses the /// direct BTree fast path internally and falls back to the plan path for @@ -492,27 +491,27 @@ impl KeyType { } /// Where the data under test lives. `Active` = the in-memory active MemTable -/// (never flushed). `Flushed` = an on-disk flushed generation (a Lance data +/// (never flushed). `SsTable` = an on-disk SSTable (a Lance data /// file + on-disk BTree index, read via the indexed-scan path) vs a single /// RocksDB SST on disk. #[derive(Debug, Clone, Copy, PartialEq)] enum Storage { Active, - Flushed, + SsTable, } impl Storage { fn parse(v: &str) -> std::result::Result { match v { "active" => Ok(Self::Active), - "flushed" => Ok(Self::Flushed), - _ => Err(format!("unknown storage '{v}', expected active|flushed")), + "sstable" => Ok(Self::SsTable), + _ => Err(format!("unknown storage '{v}', expected active|sstable")), } } fn as_str(self) -> &'static str { match self { Self::Active => "active", - Self::Flushed => "flushed", + Self::SsTable => "sstable", } } } @@ -553,7 +552,7 @@ struct Args { engine: Engine, key_type: KeyType, storage: Storage, - /// Number of flushed generations below the single active MemTable (Lance) / + /// Number of SSTables below the single active MemTable (Lance) / /// immutable SSTs below the active memtable (RocksDB). 0 = the existing /// single-tier behavior. >0 builds a full LSM: rows are split into /// `generations+1` parts, the first `generations` are flushed to on-disk @@ -574,12 +573,12 @@ struct Args { /// the caches and hit NVMe. Caps the RocksDB write buffer + uses a small /// block cache + compacts to one SST, and drops the OS page cache before /// the read phase (both engines). Use with a `--rows`×`--value-size` larger - /// than RAM. Only affects the `--storage flushed` path. + /// than RAM. Only affects the `--storage sstable` path. cold: bool, - /// Prewarm all flushed generations (open + warm indexes) into the dataset + /// Prewarm all SSTables (open + warm indexes) into the dataset /// session before the read phase, via `DatasetMemWalExt::prewarm_mem_wal`. /// Default on. `--prewarm false` disables it to measure the lazy-warm - /// baseline (the flushed cache is still set, so each generation is opened + /// baseline (the SSTable cache is still set, so each generation is opened /// on its first gen-key lookup instead of up front). Only affects the Lance /// `--storage active` LSM path. prewarm: bool, @@ -792,7 +791,6 @@ async fn run_lance( wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: true, - sync_indexed_write: true, max_memtable_size: big, max_memtable_rows: args.rows * 4 + 1_000_000, max_memtable_batches: args.rows / args.batch_rows + 1_000_000, @@ -834,7 +832,7 @@ async fn run_lance( let n = writer .manifest() .await? - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0); if n > g { break; @@ -850,10 +848,10 @@ async fn run_lance( let n_gens = writer .manifest() .await? - .map(|m| m.flushed_generations.len()) + .map(|m| m.sstables.len()) .unwrap_or(0); println!( - "[lance] wrote {} rows in {:.2}s = {:.0} rows/s (cpu {:.2}s, flushed_gens={n_gens}+active)", + "[lance] wrote {} rows in {:.2}s = {:.0} rows/s (cpu {:.2}s, sstables={n_gens}+active)", args.rows, write_s, write_rows_per_s, write_cpu_s ); @@ -863,8 +861,8 @@ async fn run_lance( let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } // Keep a handle to the active MemTable for the direct fast path before @@ -872,22 +870,22 @@ async fn run_lance( let active = Arc::new(in_memory_refs.active.clone()); let collector = LsmDataSourceCollector::new(dataset.clone(), vec![shard_snapshot.clone()]) .with_in_memory_memtables(shard_id, in_memory_refs); - // Thread the dataset session + a flushed-dataset cache into the planner, and - // prewarm every flushed generation (open + warm its indexes) up front via + // Thread the dataset session + an SSTable cache into the planner, and + // prewarm every SSTable (open + warm its indexes) up front via // the general MemWAL API, so gen-key lookups never re-open a generation per // query (the equivalent of RocksDB keeping its DB + SSTs resident). Without // this, each plan-path lookup pays a fresh manifest read + Dataset open — a // fixed per-lookup cost independent of generation count. - let flushed_cache = Arc::new(FlushedMemTableCache::new((gens as u64).max(1))); + let sstable_cache = Arc::new(SsTableCache::new((gens as u64).max(1))); if args.prewarm { dataset - .prewarm_mem_wal(std::slice::from_ref(&shard_snapshot), Some(&flushed_cache)) + .prewarm_mem_wal(std::slice::from_ref(&shard_snapshot), Some(&sstable_cache)) .await?; } let planner = Arc::new( LsmPointLookupPlanner::new(collector, vec![KEY_COL.to_string()], arrow_schema) .with_session(dataset.session()) - .with_flushed_cache(flushed_cache), + .with_sstable_cache(sstable_cache), ); // Warmup + correctness: a hit key must resolve to exactly one row under @@ -1206,7 +1204,7 @@ async fn run_lance( } // ---------------------------------------------------------------------- -// Lance flushed (on-disk) engine +// Lance SSTable (on-disk) engine // ---------------------------------------------------------------------- /// Drop the OS page cache so subsequent reads hit storage (cold). Best-effort: @@ -1221,7 +1219,7 @@ fn drop_page_cache() { /// One indexed point lookup via the **DataFusion** path: `scan().filter("id = /// key")` parses + plans + executes a query per lookup (uses the on-disk BTree /// index). Returns the matched row count. -async fn flushed_probe(dataset: &Dataset, key: i64) -> Result { +async fn sstable_probe(dataset: &Dataset, key: i64) -> Result { use futures::StreamExt; let mut scanner = dataset.scan(); scanner.filter(&format!("{KEY_COL} = {key}"))?; @@ -1235,8 +1233,8 @@ async fn flushed_probe(dataset: &Dataset, key: i64) -> Result { /// One point lookup via the **direct** path: search the on-disk BTree scalar /// index for the row id, then `take` that row — bypassing DataFusion plan -/// construction. Diagnostic for how much of the flushed read cost is the plan. -async fn flushed_probe_direct( +/// construction. Diagnostic for how much of the SSTable read cost is the plan. +async fn sstable_probe_direct( dataset: &Dataset, scalar_index: &Arc, key: i64, @@ -1257,25 +1255,25 @@ async fn flushed_probe_direct( Ok(batch.num_rows()) } -/// Dispatch a flushed point lookup: `direct` = on-disk BTree index search + take +/// Dispatch an SSTable point lookup: `direct` = on-disk BTree index search + take /// (no DataFusion); otherwise the DataFusion `scan().filter()` path. -async fn flushed_lookup( +async fn sstable_lookup( dataset: &Dataset, scalar_index: &Arc, key: i64, direct: bool, ) -> Result { if direct { - flushed_probe_direct(dataset, scalar_index, key).await + sstable_probe_direct(dataset, scalar_index, key).await } else { - flushed_probe(dataset, key).await + sstable_probe(dataset, key).await } } -/// Batched flushed lookup over a chunk of keys: `direct` searches the index for +/// Batched SSTable lookup over a chunk of keys: `direct` searches the index for /// each key then issues one `take_rows` for all; otherwise one DataFusion scan /// with `id IN (...)`. Returns the total matched row count. -async fn flushed_batch( +async fn sstable_batch( dataset: &Dataset, scalar_index: &Arc, keys: &[i64], @@ -1321,11 +1319,11 @@ async fn flushed_batch( } } -/// Flushed Lance: write all rows as one on-disk Lance dataset with a BTree +/// SSTable Lance: write all rows as one on-disk Lance dataset with a BTree /// scalar index — the exact artifact a MemTable flush emits (forward-written /// data file + on-disk BTree index) — then point-lookup through the indexed /// scan path. Int keys only (the SQL filter literal is the integer). -async fn run_lance_flushed( +async fn run_lance_sstable( args: &Args, insert_order: &[i64], queries: &[(i64, bool)], @@ -1333,12 +1331,12 @@ async fn run_lance_flushed( assert_eq!( args.key_type, KeyType::Int, - "flushed mode currently supports --key-type int only" + "sstable mode currently supports --key-type int only" ); let sampler = RssSampler::start(); let key_type = args.key_type; let schema = make_schema(key_type); - let uri = format!("{}/lance_flushed", args.uri.trim_end_matches('/')); + let uri = format!("{}/lance_sstable", args.uri.trim_end_matches('/')); let _ = std::fs::remove_dir_all(&uri); // --- write + flush: build the on-disk data file + BTree index --- @@ -1379,13 +1377,13 @@ async fn run_lance_flushed( .iter() .find(|i| i.name == BTREE_INDEX_NAME) .map(|i| i.uuid.to_string()) - .ok_or_else(|| lance_core::Error::internal("flushed: btree index not found"))?; + .ok_or_else(|| lance_core::Error::internal("sstable: btree index not found"))?; dataset .open_scalar_index(KEY_COL, &uuid, &NoOpMetricsCollector) .await? }; println!( - "[lance] flushed read path = {}", + "[lance] sstable read path = {}", if direct { "direct btree-index search + take" } else { @@ -1395,8 +1393,8 @@ async fn run_lance_flushed( // warmup + correctness: a hit resolves to exactly one row via the index. if let Some((probe, _)) = queries.iter().find(|(_, h)| *h) { - let n = flushed_lookup(&dataset, &scalar_index, *probe, direct).await?; - assert_eq!(n, 1, "flushed warmup lookup for key {probe} returned {n}"); + let n = sstable_lookup(&dataset, &scalar_index, *probe, direct).await?; + assert_eq!(n, 1, "sstable warmup lookup for key {probe} returned {n}"); } // Cold mode: drop the OS page cache so reads hit NVMe (data > RAM assumed). @@ -1405,7 +1403,7 @@ async fn run_lance_flushed( println!("[lance] dropped page cache (cold reads from NVMe)"); } - // --- batch-get path: gather `batch_get` keys per call from the flushed gen --- + // --- batch-get path: gather `batch_get` keys per call from the SSTable --- if args.batch_get > 0 { let bg = args.batch_get; let hit_keys: Vec = queries @@ -1419,7 +1417,7 @@ async fn run_lance_flushed( let t = Instant::now(); for chunk in hit_keys.chunks(bg) { let t0 = Instant::now(); - found_total += flushed_batch(&dataset, &scalar_index, chunk, direct).await?; + found_total += sstable_batch(&dataset, &scalar_index, chunk, direct).await?; latencies_us.push(t0.elapsed().as_nanos() as f64 / 1000.0); } let read_qps_1t = hit_keys.len() as f64 / t.elapsed().as_secs_f64().max(1e-9); @@ -1439,7 +1437,7 @@ async fn run_lance_flushed( let chunks: Vec<&[i64]> = keys.chunks(bg).collect(); let mut i = shard; while i < chunks.len() { - let _ = flushed_batch(&dataset, &si, chunks[i], direct).await; + let _ = sstable_batch(&dataset, &si, chunks[i], direct).await; i += threads; } })); @@ -1456,7 +1454,7 @@ async fn run_lance_flushed( args.threads, stats.p50_us, stats.p99_us ); return Ok(EngineResult { - engine: "lance-flushed-batch", + engine: "lance-sstable-batch", write_rows_per_s, write_cpu_s, read_p50_us: stats.p50_us, @@ -1481,7 +1479,7 @@ async fn run_lance_flushed( let t_read = Instant::now(); for &(key, expect_hit) in queries { let t0 = Instant::now(); - let n = flushed_lookup(&dataset, &scalar_index, key, direct).await?; + let n = sstable_lookup(&dataset, &scalar_index, key, direct).await?; latencies_us.push(t0.elapsed().as_nanos() as f64 / 1000.0); if expect_hit { assert_eq!(n, 1, "expected hit for key {key}, got {n}"); @@ -1511,7 +1509,7 @@ async fn run_lance_flushed( handles.push(tokio::spawn(async move { let mut i = shard; while i < keys.len() { - let _ = flushed_lookup(&dataset, &scalar_index, keys[i], direct).await; + let _ = sstable_lookup(&dataset, &scalar_index, keys[i], direct).await; i += threads; } })); @@ -1536,7 +1534,7 @@ async fn run_lance_flushed( ); Ok(EngineResult { - engine: "lance-flushed", + engine: "lance-sstable", write_rows_per_s, write_cpu_s, read_p50_us: stats.p50_us, @@ -1585,7 +1583,7 @@ fn run_rocksdb(args: &Args, insert_order: &[i64], queries: &[(i64, bool)]) -> Re opts.set_min_write_buffer_number_to_merge(2); opts.set_disable_auto_compactions(true); opts.set_db_write_buffer_size(write_buf); - // Block cache for the `--storage flushed` SST reads. Warm: large enough to + // Block cache for the `--storage sstable` SST reads. Warm: large enough to // hold the SST index/filter + hot data blocks. Cold: small (128MB) so data // blocks miss the cache and reads go to NVMe (index/filter stay in memory). { @@ -1654,17 +1652,17 @@ fn run_rocksdb(args: &Args, insert_order: &[i64], queries: &[(i64, bool)]) -> Re write_buf >> 20 ); - // Single-tier `--storage flushed` (no extra generations): flush the active + // Single-tier `--storage sstable` (no extra generations): flush the active // to one SST (compact to one if cold). With generations>0 the per-chunk // flushes already produced N separate L0 SSTs + the active memtable. - if gens == 0 && args.storage == Storage::Flushed { + if gens == 0 && args.storage == Storage::SsTable { db.flush() .map_err(|e| lance_core::Error::io(format!("rocksdb flush: {e}")))?; if args.cold { db.compact_range::<&[u8], &[u8]>(None, None); } } - if args.storage == Storage::Flushed || gens > 0 { + if args.storage == Storage::SsTable || gens > 0 { let n_sst = db .property_int_value("rocksdb.num-files-at-level0") .ok() @@ -1939,7 +1937,7 @@ async fn run(args: Args) -> Result<()> { if matches!(args.engine, Engine::Lance | Engine::Both) { let res = match args.storage { Storage::Active => run_lance(&args, &insert_order, &queries).await?, - Storage::Flushed => run_lance_flushed(&args, &insert_order, &queries).await?, + Storage::SsTable => run_lance_sstable(&args, &insert_order, &queries).await?, }; results.push(res); } diff --git a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs index cb9e6413ac1..0b92d1d9c73 100644 --- a/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs +++ b/rust/lance/benches/mem_wal/point_lookup/mem_wal_point_lookup_bench.rs @@ -4,8 +4,8 @@ //! Standalone CLI benchmark for PK-based point lookups across LSM levels. //! //! Measures lookup latency against three tiers of the LSM tree: -//! - Base table (on-disk, merged data) -//! - Flushed MemTable generations (on-disk L0) +//! - Base table (on-disk, compacted data) +//! - SSTables (on-disk L0) //! - Active MemTable (in-memory write buffer) //! //! Two phases, selected with `--phase`: @@ -151,7 +151,7 @@ struct Args { uri: String, base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, batch_rows: usize, queries: usize, output: Option, @@ -164,7 +164,7 @@ impl Default for Args { uri: String::new(), base_rows: 1_000_000, max_memtable_rows: 100_000, - flushed_generations: 2, + sstables: 2, batch_rows: 1_000, queries: 500, output: None, @@ -205,7 +205,7 @@ fn parse_args() -> Result { } "--base-rows" => args.base_rows = parse_val(&flag, &value)?, "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?, - "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?, + "--sstables" => args.sstables = parse_val(&flag, &value)?, "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?, "--queries" => args.queries = parse_val(&flag, &value)?, "--output" => args.output = Some(PathBuf::from(value)), @@ -276,11 +276,11 @@ fn is_cloud_uri(uri: &str) -> bool { fn generate_lookup_ids( base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, queries: usize, ) -> (Vec>, Vec<&'static str>) { - let flushed_total = flushed_generations * max_memtable_rows; - let active_start = base_rows + flushed_total; + let sstable_total = sstables * max_memtable_rows; + let active_start = base_rows + sstable_total; let active_end = active_start + max_memtable_rows / 2; let mut groups = Vec::new(); @@ -296,19 +296,19 @@ fn generate_lookup_ids( groups.push(base_ids); names.push("base"); - // Flushed IDs (only if there are flushed generations) - if flushed_generations > 0 { - let flushed_start = base_rows; - let flushed_end = base_rows + flushed_total; - let flushed_ids: Vec = (0..queries) + // SSTable IDs (only if there are SSTables) + if sstables > 0 { + let sstable_start = base_rows; + let sstable_end = base_rows + sstable_total; + let sstable_ids: Vec = (0..queries) .map(|i| { - let range = flushed_end - flushed_start; + let range = sstable_end - sstable_start; let step = range.max(1) / queries.max(1); - (flushed_start + (i * step) % range) as i64 + (sstable_start + (i * step) % range) as i64 }) .collect(); - groups.push(flushed_ids); - names.push("flushed"); + groups.push(sstable_ids); + names.push("sstable"); } // Active memtable IDs @@ -338,7 +338,6 @@ async fn run_lookup(args: &Args) -> Result { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_memtable_size: max_memtable_rows * 200, max_memtable_rows, max_wal_flush_interval: Some(Duration::from_secs(60)), @@ -354,11 +353,11 @@ async fn run_lookup(args: &Args) -> Result { }; let id_base = args.base_rows as i64; - let num_flushed = args.flushed_generations; + let num_sstables = args.sstables; let active_rows = max_memtable_rows / 2; - // Ingest flushed generations (each triggers a flush) + 1 active (50% full) - let mut gen_sizes: Vec = (0..num_flushed).map(|_| max_memtable_rows).collect(); + // Ingest SSTables (each triggers a flush) + 1 active (50% full) + let mut gen_sizes: Vec = (0..num_sstables).map(|_| max_memtable_rows).collect(); gen_sizes.push(active_rows); let mut cursor = 0usize; @@ -371,22 +370,22 @@ async fn run_lookup(args: &Args) -> Result { writer.put(vec![batch]).await?; cursor += rows; } - let is_flushed = gen_idx < num_flushed; + let is_sstable = gen_idx < num_sstables; println!( " gen {}: wrote {} rows ({}) cursor={}", gen_idx + 1, gen_rows, - if is_flushed { "flushed" } else { "active" }, + if is_sstable { "sstable" } else { "active" }, cursor, ); - if is_flushed { + if is_sstable { tokio::time::sleep(flush_wait).await; } } println!( - "ingested {} rows total ({} flushed gens + active)", - cursor, num_flushed + "ingested {} rows total ({} SSTables + active)", + cursor, num_sstables ); let manifest = writer.manifest().await.unwrap(); @@ -396,18 +395,15 @@ async fn run_lookup(args: &Args) -> Result { let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } - let num_flushed = manifest - .as_ref() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); + let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0); println!( - "manifest: {} flushed generations, current_generation={}", - num_flushed, + "manifest: {} SSTables, current_generation={}", + num_sstables, manifest.as_ref().map(|m| m.current_generation).unwrap_or(0) ); @@ -418,8 +414,12 @@ async fn run_lookup(args: &Args) -> Result { let planner = LsmPointLookupPlanner::new(collector, pk_columns, arrow_schema); // Generate lookup IDs for each category - let (id_groups, category_names) = - generate_lookup_ids(args.base_rows, max_memtable_rows, num_flushed, args.queries); + let (id_groups, category_names) = generate_lookup_ids( + args.base_rows, + max_memtable_rows, + num_sstables, + args.queries, + ); let session_ctx = SessionContext::new(); let task_ctx = session_ctx.task_ctx(); @@ -476,7 +476,7 @@ async fn run_lookup(args: &Args) -> Result { output.insert("phase".into(), json!("lookup")); output.insert("base_rows".into(), json!(args.base_rows)); output.insert("max_memtable_rows".into(), json!(max_memtable_rows)); - output.insert("flushed_generations".into(), json!(num_flushed)); + output.insert("sstables".into(), json!(num_sstables)); output.insert("active_rows".into(), json!(active_rows)); output.insert("queries_per_category".into(), json!(args.queries)); for (key, val) in &results { @@ -491,12 +491,12 @@ async fn run_lookup(args: &Args) -> Result { async fn run(args: Args) -> Result<()> { println!( - "bench=mem_wal_point_lookup phase={} uri={} base_rows={} max_memtable_rows={} flushed_generations={} batch_rows={} queries={}", + "bench=mem_wal_point_lookup phase={} uri={} base_rows={} max_memtable_rows={} sstables={} batch_rows={} queries={}", args.phase.as_str(), args.uri, args.base_rows, args.max_memtable_rows, - args.flushed_generations, + args.sstables, args.batch_rows, args.queries, ); diff --git a/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py b/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py index e3b81123ece..f93a06e5188 100644 --- a/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py +++ b/rust/lance/benches/mem_wal/vector/hnsw/disk_ann_compare.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -"""Disk-ANN comparison: Lance on-disk IVF_HNSW_SQ (the flushed-memtable index) +"""Disk-ANN comparison: Lance on-disk IVF_HNSW_SQ (the SSTable index) vs DiskANN vs FAISS, all backed by local NVMe. faiss/diskannpy/lance bundle conflicting tcmalloc/MKL/OpenMP and crash if @@ -18,7 +18,12 @@ recall@10 vs p50/p99 latency and QPS. The Lance index is served fully cached (large index_cache_size_bytes). """ -import argparse, json, os, time + +import argparse +import json +import os +import time + import numpy as np K = 10 @@ -27,7 +32,9 @@ DIM = 1536 EF_SWEEP = [16, 32, 64, 128, 256] HF_TREE = "https://huggingface.co/api/datasets/KShivendu/dbpedia-entities-openai-1M/tree/main/data" -HF_BASE = "https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M/resolve/main/" +HF_BASE = ( + "https://huggingface.co/datasets/KShivendu/dbpedia-entities-openai-1M/resolve/main/" +) def data_dir(base, rows): @@ -42,10 +49,13 @@ def normalize(x): # ---------------- prepare ---------------- def load_corpus(cache_dir, needed): - import requests, pyarrow.parquet as pq + import pyarrow.parquet as pq + import requests + os.makedirs(cache_dir, exist_ok=True) shards = sorted( - e["path"] for e in requests.get(HF_TREE, timeout=60).json() + e["path"] + for e in requests.get(HF_TREE, timeout=60).json() if e["type"] == "file" and e["path"].endswith(".parquet") ) out = np.empty((needed, DIM), dtype=np.float32) @@ -62,7 +72,7 @@ def load_corpus(cache_dir, needed): col = pq.read_table(local, columns=["openai"]).column("openai") arr = np.stack(col.to_pylist()).astype(np.float32) take = min(len(arr), needed - n) - out[n:n + take] = arr[:take] + out[n : n + take] = arr[:take] n += take print(f" shard {os.path.basename(rel)} -> {take} (cum {n})", flush=True) assert n == needed, f"only got {n}/{needed}" @@ -73,7 +83,6 @@ def numpy_ground_truth(corpus, queries): gt = np.empty((len(queries), K), dtype=np.int64) # corpus is normalized -> cosine == inner product; chunk over corpus. chunk = 200_000 - sims_top = None # Compute full similarity in query-major chunks to bound memory. sim = np.zeros((len(queries), len(corpus)), dtype=np.float32) for s in range(0, len(corpus), chunk): @@ -98,10 +107,13 @@ def cmd_prepare(args): rng = np.random.default_rng(SEED) qidx = rng.choice(args.rows, size=NUM_QUERIES, replace=False) queries = corpus[qidx].copy() - print(f"corpus={len(corpus)} queries={len(queries)} dim={DIM}; computing GT...", flush=True) + print( + f"corpus={len(corpus)} queries={len(queries)} dim={DIM}; computing GT...", + flush=True, + ) t = time.perf_counter() gt = numpy_ground_truth(corpus, queries) - print(f" GT in {time.perf_counter()-t:.1f}s", flush=True) + print(f" GT in {time.perf_counter() - t:.1f}s", flush=True) np.save(os.path.join(d, "corpus.npy"), corpus) np.save(os.path.join(d, "queries.npy"), queries) np.save(os.path.join(d, "gt.npy"), gt) @@ -110,7 +122,9 @@ def cmd_prepare(args): # ---------------- shared run helpers ---------------- def recall_at_k(gt, got): - return sum(len(set(g.tolist()) & set(r.tolist())) for g, r in zip(gt, got)) / (len(gt) * K) + return sum(len(set(g.tolist()) & set(r.tolist())) for g, r in zip(gt, got)) / ( + len(gt) * K + ) def latency_qps(query_fn, queries, repeats=3): @@ -133,56 +147,97 @@ def sweep(name, make_q, params, queries, gt): got = np.stack([qf(v) for v in queries]) rec = recall_at_k(gt, got) p50, p99, qps = latency_qps(qf, queries) - rows.append({"param": p, "recall": rec, "p50_us": p50, "p99_us": p99, "qps": qps}) - print(f" {name} param={p} recall={rec:.4f} p50={p50:.0f}us p99={p99:.0f}us qps={qps:.0f}", flush=True) + rows.append( + {"param": p, "recall": rec, "p50_us": p50, "p99_us": p99, "qps": qps} + ) + print( + f" {name} param={p} recall={rec:.4f} " + f"p50={p50:.0f}us p99={p99:.0f}us qps={qps:.0f}", + flush=True, + ) return rows # ---------------- systems ---------------- def run_lance(base, rows, corpus, queries, gt): - import lance, pyarrow as pa, shutil + import shutil + + import lance + import pyarrow as pa + uri = os.path.join(base, f"lance_{rows}") shutil.rmtree(uri, ignore_errors=True) - vecs = pa.FixedSizeListArray.from_arrays(pa.array(corpus.reshape(-1), type=pa.float32()), DIM) + vecs = pa.FixedSizeListArray.from_arrays( + pa.array(corpus.reshape(-1), type=pa.float32()), DIM + ) tbl = pa.table({"id": pa.array(np.arange(rows, dtype=np.int64)), "vec": vecs}) ds = lance.write_dataset(tbl, uri, mode="overwrite") - # The flushed memtable index is a SINGLE-partition HNSW+SQ, so model it with + # The SSTable index is a SINGLE-partition HNSW+SQ, so model it with # num_partitions=1 (nprobes=1); ef is the search knob, like DiskANN/FAISS. t = time.perf_counter() - ds.create_index("vec", "IVF_HNSW_SQ", metric="cosine", num_partitions=1, - m=20, ef_construction=150) + ds.create_index( + "vec", + "IVF_HNSW_SQ", + metric="cosine", + num_partitions=1, + m=20, + ef_construction=150, + ) build_s = time.perf_counter() - t ds = lance.dataset(uri, index_cache_size_bytes=48 * 1024**3) def make_q(ef): def q(v): - return ds.to_table(nearest={"column": "vec", "q": v, "k": K, - "nprobes": 1, "ef": ef}, - columns=["id"]).column("id").to_numpy() + return ( + ds.to_table( + nearest={"column": "vec", "q": v, "k": K, "nprobes": 1, "ef": ef}, + columns=["id"], + ) + .column("id") + .to_numpy() + ) + return q - return {"build_s": build_s, "nlist": 1, "sweep": sweep("lance", make_q, None, queries, gt)} + + return { + "build_s": build_s, + "nlist": 1, + "sweep": sweep("lance", make_q, None, queries, gt), + } -def run_lance_flushed(base, rows, corpus, queries, gt, lance_path, id_offset, column): - # Open a flushed MemTable generation directly from its dataset path and +def run_lance_sstable(base, rows, corpus, queries, gt, lance_path, id_offset, column): + # Open an SSTable generation directly from its dataset path and # benchmark its on-disk IVF_HNSW_SQ index (single partition), fully cached. import lance + ds = lance.dataset(lance_path, index_cache_size_bytes=48 * 1024**3) def make_q(ef): def q(v): - ids = ds.to_table(nearest={"column": column, "q": v, "k": K, - "nprobes": 1, "ef": ef}, - columns=["id"]).column("id").to_numpy() - return ids - id_offset # map flushed-gen id -> corpus index + ids = ( + ds.to_table( + nearest={"column": column, "q": v, "k": K, "nprobes": 1, "ef": ef}, + columns=["id"], + ) + .column("id") + .to_numpy() + ) + return ids - id_offset # map SSTable id -> corpus index + return q - return {"lance_path": lance_path, "id_offset": id_offset, - "sweep": sweep("lance_flushed", make_q, None, queries, gt)} + + return { + "lance_path": lance_path, + "id_offset": id_offset, + "sweep": sweep("lance_sstable", make_q, None, queries, gt), + } def run_faiss(base, rows, corpus, queries, gt): # Full-precision HNSW reference (shows what no quantization buys). import faiss + index = faiss.IndexHNSWFlat(DIM, 32, faiss.METRIC_INNER_PRODUCT) index.hnsw.efConstruction = 200 t = time.perf_counter() @@ -194,16 +249,20 @@ def make_q(ef): def q(v): index.hnsw.efSearch = ef return index.search(v.reshape(1, -1), K)[1][0] + return q + return {"build_s": build_s, "sweep": sweep("faiss", make_q, None, queries, gt)} def run_faiss_sq(base, rows, corpus, queries, gt): # HNSW + 8-bit scalar quantization — apples-to-apples with Lance IVF_HNSW_SQ. import faiss + try: - index = faiss.IndexHNSWSQ(DIM, faiss.ScalarQuantizer.QT_8bit, 32, - faiss.METRIC_INNER_PRODUCT) + index = faiss.IndexHNSWSQ( + DIM, faiss.ScalarQuantizer.QT_8bit, 32, faiss.METRIC_INNER_PRODUCT + ) except Exception: # Fall back to L2; on unit-normalized vectors L2 ranking == cosine. index = faiss.IndexHNSWSQ(DIM, faiss.ScalarQuantizer.QT_8bit, 32) @@ -218,28 +277,44 @@ def make_q(ef): def q(v): index.hnsw.efSearch = ef return index.search(v.reshape(1, -1), K)[1][0] + return q + return {"build_s": build_s, "sweep": sweep("faiss_sq", make_q, None, queries, gt)} def run_diskann(base, rows, corpus, queries, gt): import diskannpy as dap + idx_dir = os.path.join(base, f"diskann_{rows}") os.makedirs(idx_dir, exist_ok=True) t = time.perf_counter() dap.build_memory_index( - data=corpus, distance_metric="cosine", index_directory=idx_dir, - index_prefix="ann", complexity=150, graph_degree=64, - num_threads=0, alpha=1.2, use_pq_build=False, num_pq_bytes=0, + data=corpus, + distance_metric="cosine", + index_directory=idx_dir, + index_prefix="ann", + complexity=150, + graph_degree=64, + num_threads=0, + alpha=1.2, + use_pq_build=False, + num_pq_bytes=0, ) build_s = time.perf_counter() - t - idx = dap.StaticMemoryIndex(index_directory=idx_dir, index_prefix="ann", - num_threads=0, initial_search_complexity=256) + idx = dap.StaticMemoryIndex( + index_directory=idx_dir, + index_prefix="ann", + num_threads=0, + initial_search_complexity=256, + ) def make_q(L): def q(v): return idx.search(v, k_neighbors=K, complexity=max(L, K)).identifiers + return q + return {"build_s": build_s, "sweep": sweep("diskann", make_q, None, queries, gt)} @@ -249,12 +324,24 @@ def cmd_run(args): queries = np.load(os.path.join(d, "queries.npy")) gt = np.load(os.path.join(d, "gt.npy")) print(f"=== {args.system} rows={args.rows} corpus={len(corpus)} ===", flush=True) - if args.system == "lance_flushed": - res = run_lance_flushed(args.base, args.rows, corpus, queries, gt, - args.lance_path, args.id_offset, args.column) + if args.system == "lance_sstable": + res = run_lance_sstable( + args.base, + args.rows, + corpus, + queries, + gt, + args.lance_path, + args.id_offset, + args.column, + ) else: - fn = {"lance": run_lance, "faiss": run_faiss, "faiss_sq": run_faiss_sq, - "diskann": run_diskann}[args.system] + fn = { + "lance": run_lance, + "faiss": run_faiss, + "faiss_sq": run_faiss_sq, + "diskann": run_diskann, + }[args.system] res = fn(args.base, args.rows, corpus, queries, gt) res["rows"] = args.rows res["system"] = args.system @@ -267,9 +354,16 @@ def cmd_run(args): def main(): ap = argparse.ArgumentParser() sub = ap.add_subparsers(dest="cmd", required=True) - p = sub.add_parser("prepare"); p.add_argument("--rows", type=int, required=True); p.add_argument("--base", required=True) - r = sub.add_parser("run"); r.add_argument("--rows", type=int, required=True); r.add_argument("--base", required=True); r.add_argument("--system", required=True) - r.add_argument("--lance-path", default=None); r.add_argument("--id-offset", type=int, default=0); r.add_argument("--column", default="vector") + p = sub.add_parser("prepare") + p.add_argument("--rows", type=int, required=True) + p.add_argument("--base", required=True) + r = sub.add_parser("run") + r.add_argument("--rows", type=int, required=True) + r.add_argument("--base", required=True) + r.add_argument("--system", required=True) + r.add_argument("--lance-path", default=None) + r.add_argument("--id-offset", type=int, default=0) + r.add_argument("--column", default="vector") args = ap.parse_args() (cmd_prepare if args.cmd == "prepare" else cmd_run)(args) diff --git a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs index 7b9f35b73c9..625900cf834 100644 --- a/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs +++ b/rust/lance/benches/mem_wal/vector/hnsw/mem_wal_recall_hnsw.rs @@ -355,7 +355,7 @@ async fn run_checkpoint( let temp = tempfile::tempdir().map_err(|e| lance_core::Error::io(format!("tempdir: {}", e)))?; // When BENCH_URI_BASE is set, write to a persistent path and flush the // MemTable to an on-disk generation (instead of querying the active - // MemTable), printing the flushed generation's dataset path for a + // MemTable), printing the SSTable's dataset path for a // downstream direct-read benchmark. let flush_base = std::env::var("BENCH_URI_BASE").ok(); let local_dir = flush_base.as_ref().map(|b| format!("{}/cp_{}", b, cp)); @@ -389,7 +389,6 @@ async fn run_checkpoint( wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: true, max_memtable_size: cp.saturating_mul(row_size_estimate).saturating_mul(4), max_memtable_rows: cp.saturating_mul(2), max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000), @@ -416,11 +415,11 @@ async fn run_checkpoint( writer.put(vec![batch]).await?; } - let target_batch_pos = total_batches.saturating_sub(1); + let target_indexed_count = total_batches; let mut spins = 0u64; loop { let active = writer.active_memtable_ref().await?; - if active.index_store.max_visible_batch_position() >= target_batch_pos { + if active.index_store.indexed_count() >= target_indexed_count { break; } drop(active); @@ -450,9 +449,14 @@ async fn run_checkpoint( let mut waited = 0u64; let gen_path = loop { if let Some(m) = writer.manifest().await? - && let Some(fg) = m.flushed_generations.last() + && let Some(sstable) = m.sstables.last() { - break format!("{}/_mem_wal/{}/{}", dir, shard_id.as_hyphenated(), fg.path); + break format!( + "{}/_mem_wal/{}/{}", + dir, + shard_id.as_hyphenated(), + sstable.path + ); } tokio::time::sleep(Duration::from_millis(100)).await; waited += 1; @@ -467,7 +471,7 @@ async fn run_checkpoint( } }; println!( - "FLUSHED_OK cp={} id_offset={} flush_s={:.2} path={}", + "SSTABLE_OK cp={} id_offset={} flush_s={:.2} path={}", cp, id_offset, seal_start.elapsed().as_secs_f64(), @@ -476,7 +480,7 @@ async fn run_checkpoint( std::io::stdout().flush().ok(); writer.close().await?; - // Item 4: open the flushed generation directly from its dataset path and + // Item 4: open the SSTable directly from its dataset path and // benchmark its on-disk IVF_HNSW_SQ read in Rust (single-thread per-query // latency + recall vs brute force), sweeping ef. nprobes=1 (single part). let gen_uri = format!("file://{}", gen_path); @@ -526,7 +530,7 @@ async fn run_checkpoint( let p99_q = lat[lat.len() * 99 / 100]; let mean_recall = recall_sum / num_queries as f64; println!( - "FLUSHED_READ cp={} ef={} mean_recall={:.4} median_us={} p99_us={}", + "SSTABLE_READ cp={} ef={} mean_recall={:.4} median_us={} p99_us={}", cp, ef, mean_recall, median_q, p99_q ); std::io::stdout().flush().ok(); @@ -537,13 +541,13 @@ async fn run_checkpoint( // Raw-index path: call VectorIndex::search() directly (partition-find + // HNSW+SQ search, returns _rowid/_distance) bypassing the DataFusion - // scanner/take/projection. The RAW_READ vs FLUSHED_READ latency delta at + // scanner/take/projection. The RAW_READ vs SSTABLE_READ latency delta at // matched ef isolates the DataFusion per-query overhead from the actual // index search cost. let idx_metas = gen_ds.load_indices_by_name(VECTOR_INDEX_NAME).await?; let uuid = idx_metas .first() - .ok_or_else(|| lance_core::Error::io("flushed gen has no vector index".to_string()))? + .ok_or_else(|| lance_core::Error::io("SSTable has no vector index".to_string()))? .uuid; let vidx = gen_ds .open_vector_index(VECTOR_COL, &uuid, &NoOpMetricsCollector) @@ -595,7 +599,7 @@ async fn run_checkpoint( }; // IVFIndex::search is intentionally unimplemented (top-level does // partition-aware search); replicate the ANN exec node: pick the - // closest partition then search it. Single-partition flushed gen. + // closest partition then search it. Single-partition SSTable. let t = Instant::now(); let (parts, _) = vidx.find_partitions(&query)?; let pid = parts.value(0) as usize; diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs index 8bff64dfcca..c6fb42834a3 100644 --- a/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs +++ b/rust/lance/benches/mem_wal/vector/mem_wal_index_micro.rs @@ -10,7 +10,9 @@ //! index maintained, opens `mem_wal_writer` with a `ShardWriterConfig` //! sized to hold the largest checkpoint without flushing, and times //! `writer.put(batch)` calls. Index updates therefore go through the -//! parallel `IndexStore::insert_batches_parallel` path. At each +//! `IndexStore::insert_batches` path, which indexes inline at or below +//! `PARALLEL_INDEX_MIN_ROWS` rows and spawns a thread per index above it — +//! so the checkpoint sizes here straddle that crossover. At each //! checkpoint, queries are issued against `active_memtable_ref()` via //! `MemTableScanner::nearest`. //! @@ -212,7 +214,6 @@ async fn main() -> lance_core::Result<()> { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write, - sync_indexed_write: true, max_memtable_size: max_rows.saturating_mul(row_size_estimate).saturating_mul(4), max_memtable_rows: max_rows.saturating_mul(2), max_memtable_batches: total_batches_max.saturating_mul(2).max(8_000), @@ -253,7 +254,7 @@ async fn main() -> lance_core::Result<()> { while next_cp_idx < checkpoints.len() && total_inserted >= checkpoints[next_cp_idx] { let cp = checkpoints[next_cp_idx]; - let target_batch_pos = (cp / batch_size).saturating_sub(1); + let target_indexed_count = cp / batch_size; // The WAL flush handler only updates the index watermark when a // flush is triggered, and the time-based trigger inside the // writer runs only when `put()` is called. After the final put @@ -265,7 +266,7 @@ async fn main() -> lance_core::Result<()> { let mut spins = 0u64; loop { let active = writer.active_memtable_ref().await?; - if active.index_store.max_visible_batch_position() >= target_batch_pos { + if active.index_store.indexed_count() >= target_indexed_count { break; } drop(active); @@ -375,12 +376,11 @@ async fn measure_flush( memtable.set_indexes(registry); let total_batches = cp.div_ceil(batch_size); - for (wal_pos, i) in (0_u64..).zip(0..total_batches) { + for i in 0..total_batches { let start = (i * batch_size) as i64; let rows = batch_size.min(cp - i * batch_size); let batch = make_batch(start, rows, dim); - let frag_id = memtable.insert(batch).await?; - memtable.mark_wal_flushed(&[frag_id], wal_pos + 1, &[i]); + let _frag_id = memtable.insert(batch).await?; } let temp_dir = @@ -406,11 +406,19 @@ async fn measure_flush( let flusher = MemTableFlusher::new(store, base_path, uri, shard_id, manifest_store); // total_batches WAL entries were stamped at positions 1..=total_batches - // by the mark_wal_flushed loop above (1-based positions). + // by the loop above (1-based positions). let covered_wal_entry_position = total_batches as u64; + // Every batch was appended above, so the writer-global durable count is all of them. + let durable = total_batches; let t = Instant::now(); let _result = flusher - .flush_with_indexes(&memtable, epoch, index_configs, covered_wal_entry_position) + .flush_with_indexes( + &memtable, + epoch, + index_configs, + covered_wal_entry_position, + durable, + ) .await?; let elapsed = t.elapsed(); diff --git a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs index 632bd37626c..e5bd050fd65 100644 --- a/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs +++ b/rust/lance/benches/mem_wal/vector/mem_wal_vector_bench.rs @@ -6,7 +6,7 @@ //! //! Uses real embeddings from the `lance-format/fineweb-edu` HuggingFace //! dataset (384-dim) with an IVF-RQ index on the base table, then ingests -//! additional rows through ShardWriter to populate flushed generations and +//! additional rows through ShardWriter to populate SSTables and //! an active memtable. //! //! Three phases, selected with `--phase`: @@ -102,7 +102,7 @@ struct Args { uri: String, base_rows: usize, max_memtable_rows: usize, - flushed_generations: usize, + sstables: usize, batch_rows: usize, queries: usize, k: usize, @@ -120,7 +120,7 @@ impl Default for Args { uri: String::new(), base_rows: 1_000_000, max_memtable_rows: 100_000, - flushed_generations: 2, + sstables: 2, batch_rows: 1_000, queries: 100, k: 10, @@ -166,7 +166,7 @@ fn parse_args() -> Result { } "--base-rows" => args.base_rows = parse_val(&flag, &value)?, "--max-memtable-rows" => args.max_memtable_rows = parse_val(&flag, &value)?, - "--flushed-generations" => args.flushed_generations = parse_val(&flag, &value)?, + "--sstables" => args.sstables = parse_val(&flag, &value)?, "--batch-rows" => args.batch_rows = parse_val(&flag, &value)?, "--queries" => args.queries = parse_val(&flag, &value)?, "--k" => args.k = parse_val(&flag, &value)?, @@ -520,7 +520,6 @@ async fn run_search(args: &Args) -> Result { wal_persist_retry_base_delay: std::time::Duration::from_millis(50), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_memtable_size: args.max_memtable_rows * row_bytes * 2, max_memtable_rows: args.max_memtable_rows, max_unflushed_memtable_bytes: args.max_memtable_rows * row_bytes * 6, @@ -535,11 +534,11 @@ async fn run_search(args: &Args) -> Result { Duration::from_millis(500) }; - // Ingest N flushed generations + 1 active (50% full) - let num_flushed_target = args.flushed_generations; + // Ingest N SSTables + 1 active (50% full) + let num_sstable_target = args.sstables; let active_rows = args.max_memtable_rows / 2; - let total_memtable_rows = num_flushed_target * args.max_memtable_rows + active_rows; - let mut gen_sizes: Vec = (0..num_flushed_target) + let total_memtable_rows = num_sstable_target * args.max_memtable_rows + active_rows; + let mut gen_sizes: Vec = (0..num_sstable_target) .map(|_| args.max_memtable_rows) .collect(); gen_sizes.push(active_rows); @@ -616,14 +615,14 @@ async fn run_search(args: &Args) -> Result { gen_rows, ingest_start.elapsed().as_secs_f64(), ); - if gen_idx < num_flushed_target { + if gen_idx < num_sstable_target { tokio::time::sleep(flush_wait).await; } } println!( - "ingested {} total memtable rows ({} flushed + active) in {:.1}s", + "ingested {} total memtable rows ({} SSTables + active) in {:.1}s", total_memtable_rows, - num_flushed_target, + num_sstable_target, ingest_start.elapsed().as_secs_f64(), ); @@ -633,17 +632,14 @@ async fn run_search(args: &Args) -> Result { let mut shard_snapshot = ShardSnapshot::new(shard_id); if let Some(ref m) = manifest { shard_snapshot = shard_snapshot.with_current_generation(m.current_generation); - for fg in &m.flushed_generations { - shard_snapshot = shard_snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &m.sstables { + shard_snapshot = shard_snapshot.with_sstable(sstable.generation, sstable.path.clone()); } } - let num_flushed = manifest - .as_ref() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); + let num_sstables = manifest.as_ref().map(|m| m.sstables.len()).unwrap_or(0); println!( - "manifest: {} flushed generations, current_generation={}", - num_flushed, + "manifest: {} SSTables, current_generation={}", + num_sstables, manifest.as_ref().map(|m| m.current_generation).unwrap_or(0) ); @@ -761,7 +757,7 @@ async fn run_search(args: &Args) -> Result { "phase": "search", "base_rows": args.base_rows, "max_memtable_rows": args.max_memtable_rows, - "flushed_generations": num_flushed, + "sstables": num_sstables, "active_rows": active_rows, "vector_dim": VECTOR_DIM, "k": args.k, diff --git a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs index 14ec406e5e5..6fda0748006 100644 --- a/rust/lance/benches/mem_wal/write/mem_wal_replay.rs +++ b/rust/lance/benches/mem_wal/write/mem_wal_replay.rs @@ -179,7 +179,6 @@ async fn populate_shard_wal( let dataset = Dataset::open(dataset_uri).await.unwrap(); let mut config = ShardWriterConfig::new(shard_id); config.durable_write = true; - config.sync_indexed_write = false; let writer = dataset.mem_wal_writer(shard_id, config).await.unwrap(); for i in 0..num_entries { diff --git a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs index f4e4fb47395..14eea778481 100644 --- a/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs +++ b/rust/lance/benches/mem_wal/write/mem_wal_shard_writer_backpressure.rs @@ -99,10 +99,6 @@ impl Mode { fn durable_write(self) -> bool { matches!(self, Self::SyncNoIndex | Self::SyncIndexed) } - - fn sync_indexed_write(self) -> bool { - matches!(self, Self::SyncIndexed) - } } /// Which index the MemTable maintains in the indexed (`*_idx`) modes. @@ -192,7 +188,6 @@ struct Args { max_memtable_batches: Option, max_wal_buffer_size: usize, max_wal_flush_interval_ms: u64, - async_index_buffer_rows: usize, sample_interval_ms: u64, target_rows_per_sec: Option, num_partitions: usize, @@ -223,7 +218,6 @@ impl Default for Args { max_memtable_batches: None, max_wal_buffer_size: 10 * 1024 * 1024, max_wal_flush_interval_ms: 100, - async_index_buffer_rows: 10_000, sample_interval_ms: 500, target_rows_per_sec: None, num_partitions: 1, @@ -343,11 +337,9 @@ async fn run(args: Args) -> Result<()> { let shard_id = Uuid::new_v4(); let mut config = ShardWriterConfig::new(shard_id) .with_durable_write(args.mode.durable_write()) - .with_sync_indexed_write(args.mode.sync_indexed_write()) .with_max_memtable_size(args.max_memtable_size) .with_max_unflushed_memtable_bytes(args.max_unflushed_memtable_bytes) .with_max_wal_buffer_size(args.max_wal_buffer_size) - .with_async_index_buffer_rows(args.async_index_buffer_rows) .with_max_memtable_rows(memtable_limits.rows) .with_max_memtable_batches(memtable_limits.batches); if args.max_wal_flush_interval_ms == 0 { @@ -406,6 +398,7 @@ async fn run(args: Args) -> Result<()> { elapsed, &stats_handle.snapshot(), writer.memtable_stats().await.ok(), + writer.memory().active_bytes(), ); while next_sample_at <= elapsed { next_sample_at += interval; @@ -424,12 +417,14 @@ async fn run(args: Args) -> Result<()> { } let elapsed_puts_s = puts_start.elapsed().as_secs_f64(); let final_memtable_stats = writer.memtable_stats().await.ok(); + let final_resident_bytes = writer.memory().active_bytes(); push_sample( &mut samples, "puts_done", puts_start.elapsed(), &stats_handle.snapshot(), final_memtable_stats.clone(), + final_resident_bytes, ); let (elapsed_drain_s, elapsed_total_s, stats) = if args.skip_close { @@ -440,6 +435,7 @@ async fn run(args: Args) -> Result<()> { puts_start.elapsed(), &stats, final_memtable_stats.clone(), + final_resident_bytes, ); (0.0, elapsed_puts_s, stats) } else { @@ -448,7 +444,14 @@ async fn run(args: Args) -> Result<()> { let elapsed_drain_s = close_start.elapsed().as_secs_f64(); let elapsed_total_s = puts_start.elapsed().as_secs_f64(); let stats = stats_handle.snapshot(); - push_sample(&mut samples, "closed", puts_start.elapsed(), &stats, None); + push_sample( + &mut samples, + "closed", + puts_start.elapsed(), + &stats, + None, + 0, + ); (elapsed_drain_s, elapsed_total_s, stats) }; @@ -544,7 +547,6 @@ async fn run(args: Args) -> Result<()> { "max_memtable_batches": memtable_limits.batches, "max_wal_buffer_size": args.max_wal_buffer_size, "max_wal_flush_interval_ms": args.max_wal_flush_interval_ms, - "async_index_buffer_rows": args.async_index_buffer_rows, "sample_interval_ms": args.sample_interval_ms, "skip_close": args.skip_close, "setup_seconds": setup_s, @@ -564,7 +566,7 @@ async fn run(args: Args) -> Result<()> { "p99_ms": p99_ms, "slow_puts_1s": slow_puts_1s, "slow_puts_10s": slow_puts_10s, - "final_memtable_stats": memtable_stats_json(final_memtable_stats.as_ref()), + "final_memtable_stats": memtable_stats_json(final_memtable_stats.as_ref(), final_resident_bytes), "puts": puts, "samples": samples, "write_stats": { @@ -631,6 +633,7 @@ fn push_sample( elapsed: Duration, stats: &WriteStatsSnapshot, memtable: Option, + resident_bytes: usize, ) { samples.push(json!({ "phase": phase, @@ -647,10 +650,10 @@ fn push_sample( "memtable_flush_rows": stats.memtable_flush_rows, "active_memtable_rows": memtable.as_ref().map(|stats| stats.row_count), "active_memtable_batches": memtable.as_ref().map(|stats| stats.batch_count), - "active_memtable_bytes": memtable.as_ref().map(|stats| stats.estimated_size), + "active_memtable_bytes": resident_bytes, "active_memtable_generation": memtable.as_ref().map(|stats| stats.generation), "active_memtable_max_buffered_batch_position": memtable.as_ref().and_then(|stats| stats.max_buffered_batch_position), - "active_memtable_max_flushed_batch_position": memtable.as_ref().and_then(|stats| stats.max_flushed_batch_position), + "active_memtable_durable_batch_count": memtable.as_ref().map(|stats| stats.durable_batch_count), "wal_queue_pending_batches": memtable.as_ref().map(|stats| stats.pending_wal_batch_count), "wal_queue_pending_rows": memtable.as_ref().map(|stats| stats.pending_wal_row_count), "wal_queue_pending_bytes": memtable.as_ref().map(|stats| stats.pending_wal_estimated_bytes), @@ -659,15 +662,18 @@ fn push_sample( })); } -fn memtable_stats_json(memtable: Option<&MemTableStats>) -> serde_json::Value { +fn memtable_stats_json( + memtable: Option<&MemTableStats>, + resident_bytes: usize, +) -> serde_json::Value { match memtable { Some(stats) => json!({ "row_count": stats.row_count, "batch_count": stats.batch_count, - "estimated_size": stats.estimated_size, + "resident_bytes": resident_bytes, "generation": stats.generation, "max_buffered_batch_position": stats.max_buffered_batch_position, - "max_flushed_batch_position": stats.max_flushed_batch_position, + "durable_batch_count": stats.durable_batch_count, "wal_queue_pending_start_batch_position": stats.pending_wal_start_batch_position, "wal_queue_pending_end_batch_position": stats.pending_wal_end_batch_position, "wal_queue_pending_batches": stats.pending_wal_batch_count, @@ -980,7 +986,6 @@ fn parse_args() -> Result { "--max-wal-flush-interval-ms" => { args.max_wal_flush_interval_ms = parse(&flag, &value)?; } - "--async-index-buffer-rows" => args.async_index_buffer_rows = parse(&flag, &value)?, "--sample-interval-ms" => args.sample_interval_ms = parse(&flag, &value)?, "--target-rows-per-sec" => args.target_rows_per_sec = Some(parse(&flag, &value)?), "--num-partitions" => args.num_partitions = parse(&flag, &value)?, diff --git a/rust/lance/benches/mem_wal/write/mem_wal_write.rs b/rust/lance/benches/mem_wal/write/mem_wal_write.rs index 6d3a31c4011..df21c145299 100644 --- a/rust/lance/benches/mem_wal/write/mem_wal_write.rs +++ b/rust/lance/benches/mem_wal/write/mem_wal_write.rs @@ -30,7 +30,6 @@ //! - `BATCH_SIZE`: Number of rows per write batch (default: 20) //! - `NUM_BATCHES`: Total number of batches to write (default: 1000) //! - `DURABLE_WRITE`: yes/no/both (default: no) - whether writes wait for WAL flush -//! - `INDEXED_WRITE`: yes/no/both (default: no) - whether writes update indexes synchronously //! - `MAX_WAL_BUFFER_SIZE`: WAL buffer size in bytes (default: 1MB from ShardWriterConfig) //! - `MAX_FLUSH_INTERVAL_MS`: WAL flush interval in milliseconds, 0 to disable (default: 1000ms) //! - `MAX_MEMTABLE_SIZE`: MemTable size threshold in bytes (default: 64MB from ShardWriterConfig) @@ -44,8 +43,7 @@ //! `no` uses WAL-only mode (no MemTable, no indexes, no Lance flushes; pure WAL throughput). //! `both` runs each combination twice, once per mode, side-by-side. //! When `no` or `both`, the WAL-only branch always runs with -//! `MEMWAL_MAINTAINED_INDEXES=none` and skips `INDEXED_WRITE=yes` -//! (sync-indexed writes require a MemTable). +//! `MEMWAL_MAINTAINED_INDEXES=none`. //! - `SAMPLE_SIZE`: Number of benchmark iterations (default: 10, minimum: 10) #![allow(clippy::print_stdout, clippy::print_stderr)] @@ -119,11 +117,6 @@ fn get_durable_write_options() -> Vec { parse_yes_no_both("DURABLE_WRITE", "no") } -/// Get indexed write settings from environment. -fn get_indexed_write_options() -> Vec { - parse_yes_no_both("INDEXED_WRITE", "no") -} - /// Get enable_memtable settings from environment. Default `yes` keeps /// existing benchmark behavior; `no` runs WAL-only mode; `both` runs both /// modes side-by-side for comparison. @@ -442,31 +435,26 @@ fn build_label( num_batches: usize, batch_size: usize, durable: bool, - indexed: bool, enable_memtable: bool, storage: &str, ) -> String { let durable_str = if durable { "durable" } else { "nondurable" }; - // sync_indexed_write controls sync vs async index updates - let indexed_str = if indexed { "sync_idx" } else { "async_idx" }; let mode_str = if enable_memtable { "memtable" } else { "wal_only" }; format!( - "{}x{} {} {} {} ({})", - num_batches, batch_size, mode_str, durable_str, indexed_str, storage + "{}x{} {} {} ({})", + num_batches, batch_size, mode_str, durable_str, storage ) } /// Build dataset name prefix from config options. -fn build_name_prefix(durable: bool, indexed: bool, enable_memtable: bool) -> String { +fn build_name_prefix(durable: bool, enable_memtable: bool) -> String { let d = if durable { "d" } else { "nd" }; - // sync_indexed_write: sync (si) vs async (ai) - let i = if indexed { "si" } else { "ai" }; let m = if enable_memtable { "mt" } else { "wo" }; - format!("{}_{}_{}", m, d, i) + format!("{}_{}", m, d) } /// Benchmark Lance MemWAL write throughput. @@ -490,7 +478,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) { let maintained_indexes = get_maintained_indexes(); let durable_options = get_durable_write_options(); - let indexed_options = get_indexed_write_options(); let enable_memtable_options = get_enable_memtable_options(); let max_wal_buffer_size = get_max_wal_buffer_size(); let max_flush_interval = get_max_flush_interval(); @@ -550,70 +537,56 @@ fn bench_lance_memwal_write(c: &mut Criterion) { // Generate benchmarks for all combinations for &enable_memtable in &enable_memtable_options { for &durable in &durable_options { - for &indexed in &indexed_options { - if !enable_memtable && indexed { - eprintln!( - "Skipping wal_only + sync_idx (sync_indexed_write requires a MemTable)" - ); - continue; - } - - let label = build_label( - num_batches, - batch_size, - durable, - indexed, - enable_memtable, - storage_label, - ); - let name_prefix = build_name_prefix(durable, indexed, enable_memtable); - - // WAL-only mode never uses indexes; force the dataset - // setup to skip the MemWAL index list. - let effective_indexes: Vec = if enable_memtable { - maintained_indexes.clone() - } else { - Vec::new() - }; - - // Create dataset ONCE before benchmark iterations - // Each iteration will use a different shard on the same dataset - let dataset = rt.block_on(create_dataset( - &schema, - &name_prefix, - vector_dim, - &effective_indexes, - &dataset_prefix, - )); - let dataset_uri = dataset.uri().to_string(); - - // Pre-generate all batches before timing (outside iter_custom) - let batches: Arc> = Arc::new( - (0..num_batches) - .map(|i| { - create_test_batch( - &schema, - (i * batch_size) as i64, - batch_size, - vector_dim, - ) - }) - .collect(), - ); - - println!("Running: {}", label); - - // Track if we've printed stats (only print once across all samples) - let stats_printed = Arc::new(AtomicBool::new(false)); - - group.bench_with_input( - BenchmarkId::new("Lance MemWAL", &label), - &(batch_size, num_batches, durable, indexed, row_size_bytes), - |b, &(_batch_size, _num_batches, durable, indexed, row_size_bytes)| { - let dataset_uri = dataset_uri.clone(); - let batches = batches.clone(); - let stats_printed = stats_printed.clone(); - b.to_async(&rt).iter_custom(|iters| { + let label = build_label( + num_batches, + batch_size, + durable, + enable_memtable, + storage_label, + ); + let name_prefix = build_name_prefix(durable, enable_memtable); + + // WAL-only mode never uses indexes; force the dataset + // setup to skip the MemWAL index list. + let effective_indexes: Vec = if enable_memtable { + maintained_indexes.clone() + } else { + Vec::new() + }; + + // Create dataset ONCE before benchmark iterations + // Each iteration will use a different shard on the same dataset + let dataset = rt.block_on(create_dataset( + &schema, + &name_prefix, + vector_dim, + &effective_indexes, + &dataset_prefix, + )); + let dataset_uri = dataset.uri().to_string(); + + // Pre-generate all batches before timing (outside iter_custom) + let batches: Arc> = Arc::new( + (0..num_batches) + .map(|i| { + create_test_batch(&schema, (i * batch_size) as i64, batch_size, vector_dim) + }) + .collect(), + ); + + println!("Running: {}", label); + + // Track if we've printed stats (only print once across all samples) + let stats_printed = Arc::new(AtomicBool::new(false)); + + group.bench_with_input( + BenchmarkId::new("Lance MemWAL", &label), + &(batch_size, num_batches, durable, row_size_bytes), + |b, &(_batch_size, _num_batches, durable, row_size_bytes)| { + let dataset_uri = dataset_uri.clone(); + let batches = batches.clone(); + let stats_printed = stats_printed.clone(); + b.to_async(&rt).iter_custom(|iters| { let dataset_uri = dataset_uri.clone(); let batches = batches.clone(); let stats_printed = stats_printed.clone(); @@ -631,10 +604,10 @@ fn bench_lance_memwal_write(c: &mut Criterion) { shard_id, shard_spec_id: 0, max_wal_persist_retries: 3, - wal_persist_retry_base_delay: - std::time::Duration::from_millis(50), + wal_persist_retry_base_delay: std::time::Duration::from_millis( + 50, + ), durable_write: durable, - sync_indexed_write: indexed, max_wal_buffer_size: max_wal_buffer_size .unwrap_or(default_config.max_wal_buffer_size), max_wal_flush_interval: max_flush_interval @@ -643,8 +616,6 @@ fn bench_lance_memwal_write(c: &mut Criterion) { .unwrap_or(default_config.max_memtable_size), max_memtable_rows: default_config.max_memtable_rows, max_memtable_batches: default_config.max_memtable_batches, - async_index_buffer_rows: default_config.async_index_buffer_rows, - async_index_interval: default_config.async_index_interval, manifest_scan_batch_size: default_config .manifest_scan_batch_size, max_unflushed_memtable_bytes: default_config @@ -656,6 +627,12 @@ fn bench_lance_memwal_write(c: &mut Criterion) { enable_memtable, hnsw_params: default_config.hnsw_params, warmer: None, + observer: None, + store_params: default_config.store_params, + session: default_config.session, + // Measure the built-in per-shard valve, not + // an injected policy. + backpressure: None, }; // Get writer through Dataset API (index configs loaded automatically) @@ -704,9 +681,8 @@ fn bench_lance_memwal_write(c: &mut Criterion) { total_duration } }) - }, - ); - } + }, + ); } } diff --git a/rust/lance/benches/random_access.rs b/rust/lance/benches/random_access.rs index ef86f812ea4..6fb76f0783b 100644 --- a/rust/lance/benches/random_access.rs +++ b/rust/lance/benches/random_access.rs @@ -7,7 +7,7 @@ use arrow_array::{Float64Array, Int64Array, RecordBatch, RecordBatchIterator, St use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use criterion::{Criterion, criterion_group, criterion_main}; use lance::dataset::{Dataset, ProjectionRequest, WriteParams}; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use std::collections::HashMap; use tokio::runtime::Runtime; use uuid::Uuid; @@ -85,7 +85,12 @@ fn utf8_field_without_fsst(name: &str) -> Field { } fn utf8_field_for(version: LanceFileVersion, enable_fsst: bool, name: &str) -> Field { - if enable_fsst && version >= LanceFileVersion::V2_1 { + if enable_fsst + && matches!( + version.resolve(), + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 + ) + { Field::new(name, DataType::Utf8, false) } else { utf8_field_without_fsst(name) diff --git a/rust/lance/benches/s3_file_reader_diagnostics.rs b/rust/lance/benches/s3_file_reader_diagnostics.rs new file mode 100644 index 00000000000..1762048d495 --- /dev/null +++ b/rust/lance/benches/s3_file_reader_diagnostics.rs @@ -0,0 +1,2357 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +#![allow(clippy::print_stdout)] +#![recursion_limit = "256"] + +use std::collections::BTreeMap; +use std::env; +use std::fs; +use std::hint::black_box; +use std::ops::Range; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use arrow_array::RecordBatch; +use futures::future::BoxFuture; +use futures::stream::{FuturesOrdered, FuturesUnordered}; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use lance::dataset::ProjectionRequest; +use lance::dataset::builder::DatasetBuilder; +use lance::dataset::fragment::{FileFragment, FragReadConfig}; +use lance::dataset::scanner::{ExecutionStatsCallback, ExecutionSummaryCounts}; +use lance_core::datatypes::Schema; +use lance_encoding::decoder::PageEncoding; +use lance_encoding::format::pb21; +use lance_file::reader::{ + DEFAULT_READ_CHUNK_SIZE, FileReader as LanceFileReader, FileReaderOptions, +}; +use lance_io::object_store::ObjectStore as LanceObjectStore; +use lance_io::scheduler::{FileScheduler, ScanScheduler, ScanStats, SchedulerConfig}; +use lance_io::utils::CachedFileSize; +use serde_json::{Value, json}; +use tracing::field::{Field, Visit}; +use tracing::subscriber::Interest; +use tracing::{Event, Metadata, Subscriber}; +use tracing_subscriber::layer::{Context, Layer}; +use tracing_subscriber::prelude::*; + +type Error = Box; +type Result = std::result::Result; + +const GIB: u64 = 1024 * 1024 * 1024; +const SCHEDULER_STATE_EVENT_TARGET: &str = "lance_io::scheduler::state"; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum SchedulerQueueKind { + Standard, + Lite, +} + +#[derive(Debug, Clone, Copy)] +struct SchedulerDiagnostics { + kind: SchedulerQueueKind, + stats: ScanStats, + io_capacity: u64, + iops_available: u64, + active_iops: u64, + pending_iops: u64, + pending_bytes: u64, + bytes_available: i64, + bytes_reserved: i64, + io_buffer_size_bytes: u64, + priorities_in_flight: u64, + no_backpressure: bool, + head_task_bytes: Option, + head_task_priority_high: Option, + head_task_priority_low: Option, + min_in_flight_priority_high: Option, + min_in_flight_priority_low: Option, + head_task_can_deliver: Option, + head_task_priority_bypass: Option, + head_task_blocked_by_iops: Option, + head_task_blocked_by_bytes: Option, +} + +#[derive(Debug, Clone)] +struct Config { + backend: Backend, + uri: String, + dataset_version: u64, + columns: Option>, + limit_rows: u64, + target_bytes: Option, + raw_range_size_bytes: u64, + raw_range_mode: RawRangeMode, + raw_column_indices: Option>, + raw_submit_mode: RawSubmitMode, + raw_completion_mode: RawCompletionMode, + take_repetitions: u64, + io_buffer_gib: Vec>, + batch_size: u32, + batch_size_bytes: Option, + skip_batch_byte_accounting: bool, + read_chunk_size: Option, + fragment_concurrency: usize, + batch_concurrency: usize, + sample_ms: u64, + out_dir: String, + case_name: String, + describe_layout: bool, + detach_fragment_streams: bool, + drop_read_tasks: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum Backend { + FileReader, + Scanner, + SchedulerRaw, + DatasetTake, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RawSubmitMode { + Single, + SplitNoConcat, + SplitConcat, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RawRangeMode { + FileSequential, + MetadataPages, + MetadataPagesRoundRobin, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +enum RawCompletionMode { + Unordered, + Ordered, +} + +impl RawRangeMode { + fn name(self) -> &'static str { + match self { + Self::FileSequential => "file-sequential", + Self::MetadataPages => "metadata-pages", + Self::MetadataPagesRoundRobin => "metadata-pages-round-robin", + } + } +} + +impl RawSubmitMode { + fn name(self) -> &'static str { + match self { + Self::Single => "single", + Self::SplitNoConcat => "split-no-concat", + Self::SplitConcat => "split-concat", + } + } +} + +impl RawCompletionMode { + fn name(self) -> &'static str { + match self { + Self::Unordered => "unordered", + Self::Ordered => "ordered", + } + } +} + +impl Backend { + fn name(self) -> &'static str { + match self { + Self::FileReader => "lance-file-reader", + Self::Scanner => "lance-scanner", + Self::SchedulerRaw => "lance-scheduler-raw", + Self::DatasetTake => "lance-dataset-take", + } + } + + fn layer(self) -> &'static str { + match self { + Self::FileReader => "file-reader", + Self::Scanner => "scanner", + Self::SchedulerRaw => "scheduler", + Self::DatasetTake => "dataset-take", + } + } +} + +#[derive(Debug, Clone, Copy)] +struct CpuSample { + idle: u64, + total: u64, +} + +#[derive(Debug, Default)] +struct SharedCounters { + fragments_started: AtomicU64, + fragments_completed: AtomicU64, + batch_futures_emitted: AtomicU64, + batch_futures_received: AtomicU64, + batches_completed: AtomicU64, + rows_completed: AtomicU64, + arrow_bytes: AtomicU64, + open_reader_ns: AtomicU64, + read_stream_create_ns: AtomicU64, + next_batch_poll_ns: AtomicU64, + channel_send_wait_ns: AtomicU64, + decode_ns: AtomicU64, + raw_reassemble_ns: AtomicU64, +} + +#[derive(Debug)] +struct CaseStats { + rows: u64, + batches: u64, + arrow_bytes: u64, + planned_fragments: usize, + planned_rows: u64, + elapsed: Duration, + producer_finished_at: Option, + peak_decode_in_flight: usize, + cpu_avg: Option, + scheduler_diagnostics: SchedulerDiagnostics, + counters: Arc, + samples: Vec, +} + +#[derive(Debug)] +struct LastSample { + elapsed: Duration, + scheduler_stats: ScanStats, + rows: u64, + batches: u64, + arrow_bytes: u64, +} + +fn usage() -> &'static str { + "usage: s3_file_reader_diagnostics --uri \ + [--backend ] \ + [--dataset-version ] [--columns ] \ + [--limit-rows ] [--target-bytes ] [--raw-range-size-bytes ] \ + [--raw-range-mode ] \ + [--raw-column-indices ] \ + [--raw-submit-mode ] \ + [--raw-completion-mode ] \ + [--take-repetitions ] \ + [--io-buffer-gib ] \ + [--batch-size ] [--batch-size-bytes ] [--read-chunk-size ] \ + [--skip-batch-byte-accounting] \ + [--fragment-concurrency ] [--batch-concurrency ] \ + [--sample-ms ] [--out-dir ] [--case ] \ + [--detach-fragment-streams] [--drop-read-tasks] [--describe-layout]" +} + +fn parse_args() -> Result { + let mut backend = Backend::FileReader; + let mut uri = None; + let mut dataset_version = 1u64; + let mut columns = Some(vec!["vector".to_string()]); + let mut limit_rows = 67_108_864u64; + let mut target_bytes = None; + let mut raw_range_size_bytes = 16 * 1024 * 1024; + let mut raw_range_mode = RawRangeMode::FileSequential; + let mut raw_column_indices = None; + let mut raw_submit_mode = RawSubmitMode::Single; + let mut raw_completion_mode = RawCompletionMode::Unordered; + let mut take_repetitions = 100u64; + let mut io_buffer_gib = vec![Some(8)]; + let mut batch_size = 8192u32; + let mut batch_size_bytes = None; + let mut skip_batch_byte_accounting = false; + let mut read_chunk_size = None; + let mut fragment_concurrency = 256usize; + let mut batch_concurrency = 256usize; + let mut sample_ms = 1000u64; + let mut out_dir = "/tmp/lance-s3-bottleneck-results".to_string(); + let mut case_name = "lance-file-reader-diagnostics".to_string(); + let mut describe_layout = false; + let mut detach_fragment_streams = false; + let mut drop_read_tasks = false; + + let mut args = env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--backend" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --backend. {}", usage()))?; + backend = parse_backend(&value)?; + } + "--uri" => uri = args.next(), + "--dataset-version" => { + dataset_version = parse_required_value(&mut args, "--dataset-version")?; + } + "--columns" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --columns. {}", usage()))?; + columns = parse_columns(&value)?; + } + "--limit-rows" => { + limit_rows = parse_required_value(&mut args, "--limit-rows")?; + } + "--target-bytes" => { + target_bytes = Some(parse_required_value(&mut args, "--target-bytes")?); + } + "--raw-range-size-bytes" => { + raw_range_size_bytes = parse_required_value(&mut args, "--raw-range-size-bytes")?; + } + "--raw-range-mode" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --raw-range-mode. {}", usage()))?; + raw_range_mode = parse_raw_range_mode(&value)?; + } + "--raw-column-indices" => { + let value = args.next().ok_or_else(|| { + format!("missing value for --raw-column-indices. {}", usage()) + })?; + raw_column_indices = parse_raw_column_indices(&value)?; + } + "--raw-submit-mode" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --raw-submit-mode. {}", usage()))?; + raw_submit_mode = parse_raw_submit_mode(&value)?; + } + "--raw-completion-mode" => { + let value = args.next().ok_or_else(|| { + format!("missing value for --raw-completion-mode. {}", usage()) + })?; + raw_completion_mode = parse_raw_completion_mode(&value)?; + } + "--take-repetitions" => { + take_repetitions = parse_required_value(&mut args, "--take-repetitions")?; + } + "--io-buffer-gib" => { + let value = args + .next() + .ok_or_else(|| format!("missing value for --io-buffer-gib. {}", usage()))?; + io_buffer_gib = parse_io_buffer_gib(&value)?; + } + "--batch-size" => { + batch_size = parse_required_value(&mut args, "--batch-size")?; + } + "--batch-size-bytes" => { + batch_size_bytes = Some(parse_required_value(&mut args, "--batch-size-bytes")?); + } + "--skip-batch-byte-accounting" => { + skip_batch_byte_accounting = true; + } + "--read-chunk-size" => { + read_chunk_size = Some(parse_required_value(&mut args, "--read-chunk-size")?); + } + "--fragment-concurrency" => { + fragment_concurrency = parse_required_value(&mut args, "--fragment-concurrency")?; + } + "--batch-concurrency" => { + batch_concurrency = parse_required_value(&mut args, "--batch-concurrency")?; + } + "--sample-ms" => { + sample_ms = parse_required_value(&mut args, "--sample-ms")?; + } + "--out-dir" => { + out_dir = args + .next() + .ok_or_else(|| format!("missing value for --out-dir. {}", usage()))?; + } + "--case" => { + case_name = args + .next() + .ok_or_else(|| format!("missing value for --case. {}", usage()))?; + } + "--describe-layout" => { + describe_layout = true; + } + "--detach-fragment-streams" => { + detach_fragment_streams = true; + } + "--drop-read-tasks" => { + drop_read_tasks = true; + } + "--help" | "-h" => { + println!("{}", usage()); + std::process::exit(0); + } + "--bench" => { + // Cargo appends this flag when running harness-free benches. + } + other => { + return Err(format!("unknown argument {other}. {}", usage()).into()); + } + } + } + + let uri = uri.ok_or_else(|| format!("missing required --uri. {}", usage()))?; + if limit_rows == 0 { + return Err("--limit-rows must be greater than zero".into()); + } + if matches!(target_bytes, Some(0)) { + return Err("--target-bytes must be greater than zero".into()); + } + if raw_range_size_bytes == 0 { + return Err("--raw-range-size-bytes must be greater than zero".into()); + } + if take_repetitions == 0 { + return Err("--take-repetitions must be greater than zero".into()); + } + if io_buffer_gib.is_empty() { + return Err("--io-buffer-gib must not be empty".into()); + } + if batch_size == 0 { + return Err("--batch-size must be greater than zero".into()); + } + if matches!(batch_size_bytes, Some(0)) { + return Err("--batch-size-bytes must be greater than zero".into()); + } + if matches!(read_chunk_size, Some(0)) { + return Err("--read-chunk-size must be greater than zero".into()); + } + if fragment_concurrency == 0 && !matches!(backend, Backend::Scanner) { + return Err("--fragment-concurrency must be greater than zero".into()); + } + if batch_concurrency == 0 && !matches!(backend, Backend::Scanner) { + return Err("--batch-concurrency must be greater than zero".into()); + } + if sample_ms == 0 { + return Err("--sample-ms must be greater than zero".into()); + } + + Ok(Config { + backend, + uri, + dataset_version, + columns, + limit_rows, + target_bytes, + raw_range_size_bytes, + raw_range_mode, + raw_column_indices, + raw_submit_mode, + raw_completion_mode, + take_repetitions, + io_buffer_gib, + batch_size, + batch_size_bytes, + skip_batch_byte_accounting, + read_chunk_size, + fragment_concurrency, + batch_concurrency, + sample_ms, + out_dir, + case_name, + describe_layout, + detach_fragment_streams, + drop_read_tasks, + }) +} + +fn parse_backend(value: &str) -> Result { + match value { + "file-reader" | "lance-file-reader" => Ok(Backend::FileReader), + "scanner" | "lance-scanner" => Ok(Backend::Scanner), + "scheduler-raw" | "lance-scheduler-raw" => Ok(Backend::SchedulerRaw), + "dataset-take" | "take" | "lance-dataset-take" => Ok(Backend::DatasetTake), + other => Err(format!( + "invalid --backend value {other}; expected file-reader, scanner, scheduler-raw, or dataset-take" + ) + .into()), + } +} + +fn parse_raw_submit_mode(value: &str) -> Result { + match value { + "single" => Ok(RawSubmitMode::Single), + "split-no-concat" => Ok(RawSubmitMode::SplitNoConcat), + "split-concat" => Ok(RawSubmitMode::SplitConcat), + other => Err(format!( + "invalid --raw-submit-mode value {other}; expected single, split-no-concat, or split-concat" + ) + .into()), + } +} + +fn parse_raw_range_mode(value: &str) -> Result { + match value { + "file-sequential" => Ok(RawRangeMode::FileSequential), + "metadata-pages" => Ok(RawRangeMode::MetadataPages), + "metadata-pages-round-robin" => Ok(RawRangeMode::MetadataPagesRoundRobin), + other => Err(format!( + "invalid --raw-range-mode value {other}; expected file-sequential, metadata-pages, or metadata-pages-round-robin" + ) + .into()), + } +} + +fn parse_raw_column_indices(value: &str) -> Result>> { + if value == "all" { + return Ok(None); + } + + let indices = value + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(|part| { + part.parse::() + .map_err(|err| format!("invalid raw column index {part}: {err}").into()) + }) + .collect::>>()?; + if indices.is_empty() { + return Err("--raw-column-indices must specify at least one column index or all".into()); + } + Ok(Some(indices)) +} + +fn parse_raw_completion_mode(value: &str) -> Result { + match value { + "unordered" => Ok(RawCompletionMode::Unordered), + "ordered" => Ok(RawCompletionMode::Ordered), + other => Err(format!( + "invalid --raw-completion-mode value {other}; expected unordered or ordered" + ) + .into()), + } +} + +fn parse_required_value(args: &mut impl Iterator, name: &str) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display + Send + Sync + 'static, +{ + let value = args + .next() + .ok_or_else(|| format!("missing value for {name}. {}", usage()))?; + value + .parse() + .map_err(|err| format!("invalid {name} value {value}: {err}").into()) +} + +fn parse_columns(value: &str) -> Result>> { + match value { + "all" => Ok(None), + "empty" => Err("FileReader benchmark requires at least one data column".into()), + _ => Ok(Some( + value + .split(',') + .map(str::trim) + .filter(|column| !column.is_empty()) + .map(ToString::to_string) + .collect(), + )), + } +} + +fn parse_io_buffer_gib(value: &str) -> Result>> { + value + .split(',') + .map(|part| { + let part = part.trim(); + if part == "auto" { + Ok(None) + } else { + part.parse::() + .map(Some) + .map_err(|err| format!("invalid --io-buffer-gib value {part}: {err}").into()) + } + }) + .collect() +} + +fn projection_name(columns: &Option>) -> String { + match columns { + None => "all".to_string(), + Some(columns) => columns.join(","), + } +} + +fn page_layout_kind(encoding: &PageEncoding) -> &'static str { + match encoding { + PageEncoding::Legacy(_) => "legacy", + PageEncoding::Structural(layout) => match layout.layout.as_ref() { + Some(pb21::page_layout::Layout::MiniBlockLayout(_)) => "miniblock", + Some(pb21::page_layout::Layout::ConstantLayout(_)) => "constant", + Some(pb21::page_layout::Layout::FullZipLayout(_)) => "fullzip", + Some(pb21::page_layout::Layout::BlobLayout(_)) => "blob", + Some(pb21::page_layout::Layout::SparseLayout(_)) => "sparse", + None => "missing", + }, + } +} + +fn summarize_u64(values: &[u64]) -> Value { + if values.is_empty() { + return json!({ + "count": 0, + "min": null, + "p50": null, + "p90": null, + "max": null, + "sum": 0, + }); + } + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + let percentile = |p: f64| { + let idx = ((sorted.len() - 1) as f64 * p).round() as usize; + sorted[idx] + }; + json!({ + "count": sorted.len(), + "min": sorted[0], + "p50": percentile(0.5), + "p90": percentile(0.9), + "max": *sorted.last().unwrap(), + "sum": values.iter().sum::(), + }) +} + +async fn describe_layout(config: &Config) -> Result<()> { + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + let fragment = dataset + .fragments() + .first() + .ok_or("dataset has no fragments")?; + let data_file = fragment + .files + .first() + .ok_or("first fragment has no data files")?; + if data_file.base_id.is_some() { + return Err("layout diagnostics do not support external base data files yet".into()); + } + + let data_path = dataset.data_dir().join(data_file.path.as_str()); + let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?; + let scheduler = ScanScheduler::new(object_store, SchedulerConfig::new(8 * GIB)); + let file_scheduler = scheduler + .open_file(&data_path, &CachedFileSize::unknown()) + .await?; + let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?; + + let columns = metadata + .column_infos + .iter() + .map(|column| { + let mut layout_counts = BTreeMap::new(); + let mut page_rows = Vec::with_capacity(column.page_infos.len()); + let mut page_bytes = Vec::with_capacity(column.page_infos.len()); + for page in column.page_infos.iter() { + *layout_counts + .entry(page_layout_kind(&page.encoding)) + .or_insert(0usize) += 1; + page_rows.push(page.num_rows); + page_bytes.push( + page.buffer_offsets_and_sizes + .iter() + .map(|(_, size)| *size) + .sum::(), + ); + } + json!({ + "column_index": column.index, + "num_pages": column.page_infos.len(), + "layout_counts": layout_counts, + "page_rows": summarize_u64(&page_rows), + "page_bytes": summarize_u64(&page_bytes), + }) + }) + .collect::>(); + + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "dataset_uri": config.uri, + "dataset_version": config.dataset_version, + "fragment_id": fragment.id, + "data_file_path": data_file.path, + "resolved_data_path": data_path.to_string(), + "file_version": metadata.version().to_string(), + "num_rows": metadata.num_rows, + "num_data_bytes": metadata.num_data_bytes, + "columns": columns, + }))? + ); + Ok(()) +} + +fn projected_schema(dataset_schema: &Schema, columns: &Option>) -> Result { + Ok(match columns { + None => dataset_schema.clone(), + Some(columns) => dataset_schema.project(columns)?, + }) +} + +fn file_reader_options(config: &Config) -> Option { + if config.batch_size_bytes.is_none() && config.read_chunk_size.is_none() { + return None; + } + Some(FileReaderOptions { + batch_size_bytes: config.batch_size_bytes, + read_chunk_size: config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE), + ..Default::default() + }) +} + +fn add_duration(counter: &AtomicU64, duration: Duration) { + let nanos = duration.as_nanos().min(u128::from(u64::MAX)) as u64; + counter.fetch_add(nanos, Ordering::Relaxed); +} + +fn ns_to_seconds(ns: u64) -> f64 { + ns as f64 / 1_000_000_000.0 +} + +fn diff_u64(current: u64, previous: u64) -> u64 { + current.saturating_sub(previous) +} + +fn scheduler_kind_name(kind: SchedulerQueueKind) -> &'static str { + match kind { + SchedulerQueueKind::Standard => "standard", + SchedulerQueueKind::Lite => "lite", + } +} + +fn diagnostics_json(diagnostics: SchedulerDiagnostics) -> Value { + json!({ + "queue_kind": scheduler_kind_name(diagnostics.kind), + "scheduler_iops": diagnostics.stats.iops, + "scheduler_requests": diagnostics.stats.requests, + "scheduler_bytes_read": diagnostics.stats.bytes_read, + "io_capacity": diagnostics.io_capacity, + "iops_available": diagnostics.iops_available, + "active_iops": diagnostics.active_iops, + "pending_iops": diagnostics.pending_iops, + "pending_bytes": diagnostics.pending_bytes, + "bytes_available": diagnostics.bytes_available, + "bytes_reserved": diagnostics.bytes_reserved, + "io_buffer_size_bytes": diagnostics.io_buffer_size_bytes, + "priorities_in_flight": diagnostics.priorities_in_flight, + "no_backpressure": diagnostics.no_backpressure, + "head_task_bytes": diagnostics.head_task_bytes, + "head_task_priority_high": diagnostics.head_task_priority_high, + "head_task_priority_low": diagnostics.head_task_priority_low, + "min_in_flight_priority_high": diagnostics.min_in_flight_priority_high, + "min_in_flight_priority_low": diagnostics.min_in_flight_priority_low, + "head_task_can_deliver": diagnostics.head_task_can_deliver, + "head_task_priority_bypass": diagnostics.head_task_priority_bypass, + "head_task_blocked_by_iops": diagnostics.head_task_blocked_by_iops, + "head_task_blocked_by_bytes": diagnostics.head_task_blocked_by_bytes, + }) +} + +#[derive(Debug, Default)] +struct ExecutionStatsHolder { + collected_stats: Arc>>, +} + +impl ExecutionStatsHolder { + fn get_setter(&self) -> ExecutionStatsCallback { + let collected_stats = self.collected_stats.clone(); + Arc::new(move |stats| { + *collected_stats.lock().unwrap() = Some(stats.clone()); + }) + } + + fn consume(self) -> Option { + self.collected_stats.lock().unwrap().take() + } +} + +#[derive(Debug, Clone, Default)] +struct SchedulerDiagnosticsCollector { + latest: Arc>>, +} + +impl SchedulerDiagnosticsCollector { + fn clear(&self) { + *self.latest.lock().unwrap() = None; + } + + fn observe(&self, diagnostics: SchedulerDiagnostics) { + *self.latest.lock().unwrap() = Some(diagnostics); + } + + fn snapshot(&self, io_buffer_gib: Option) -> SchedulerDiagnostics { + self.latest + .lock() + .unwrap() + .as_ref() + .copied() + .unwrap_or_else(|| diagnostics_from_scan_stats(ScanStats::default(), io_buffer_gib)) + } +} + +#[derive(Debug, Clone)] +struct SchedulerDiagnosticsLayer { + collector: SchedulerDiagnosticsCollector, +} + +impl SchedulerDiagnosticsLayer { + fn new(collector: SchedulerDiagnosticsCollector) -> Self { + Self { collector } + } +} + +fn is_scheduler_state_metadata(metadata: &Metadata<'_>) -> bool { + // The scheduler uses `tracing::enabled!` before constructing the event; + // that guard registers a HINT callsite, not an EVENT callsite. + metadata.target() == SCHEDULER_STATE_EVENT_TARGET && *metadata.level() == tracing::Level::TRACE +} + +impl Layer for SchedulerDiagnosticsLayer +where + S: Subscriber, +{ + fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest { + if is_scheduler_state_metadata(metadata) { + Interest::always() + } else { + Interest::never() + } + } + + fn enabled(&self, metadata: &Metadata<'_>, _ctx: Context<'_, S>) -> bool { + is_scheduler_state_metadata(metadata) + } + + fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { + if !is_scheduler_state_metadata(event.metadata()) { + return; + } + let mut visitor = SchedulerDiagnosticsVisitor::default(); + event.record(&mut visitor); + if let Some(diagnostics) = visitor.into_diagnostics() { + self.collector.observe(diagnostics); + } + } +} + +#[derive(Debug, Default)] +struct SchedulerDiagnosticsVisitor { + kind: Option, + scheduler_iops: Option, + scheduler_requests: Option, + scheduler_bytes_read: Option, + io_capacity: Option, + iops_available: Option, + active_iops: Option, + pending_iops: Option, + pending_bytes: Option, + bytes_available: Option, + bytes_reserved: Option, + io_buffer_size_bytes: Option, + priorities_in_flight: Option, + no_backpressure: Option, + head_task_bytes_present: bool, + head_task_bytes: Option, + head_task_priority_high_present: bool, + head_task_priority_high: Option, + head_task_priority_low_present: bool, + head_task_priority_low: Option, + min_in_flight_priority_high_present: bool, + min_in_flight_priority_high: Option, + min_in_flight_priority_low_present: bool, + min_in_flight_priority_low: Option, + head_task_can_deliver_present: bool, + head_task_can_deliver: Option, + head_task_priority_bypass_present: bool, + head_task_priority_bypass: Option, + head_task_blocked_by_iops_present: bool, + head_task_blocked_by_iops: Option, + head_task_blocked_by_bytes_present: bool, + head_task_blocked_by_bytes: Option, +} + +impl SchedulerDiagnosticsVisitor { + fn into_diagnostics(self) -> Option { + Some(SchedulerDiagnostics { + kind: self.kind?, + stats: ScanStats { + iops: self.scheduler_iops.unwrap_or_default(), + requests: self.scheduler_requests.unwrap_or_default(), + bytes_read: self.scheduler_bytes_read.unwrap_or_default(), + }, + io_capacity: self.io_capacity.unwrap_or_default(), + iops_available: self.iops_available.unwrap_or_default(), + active_iops: self.active_iops.unwrap_or_default(), + pending_iops: self.pending_iops.unwrap_or_default(), + pending_bytes: self.pending_bytes.unwrap_or_default(), + bytes_available: self.bytes_available.unwrap_or_default(), + bytes_reserved: self.bytes_reserved.unwrap_or_default(), + io_buffer_size_bytes: self.io_buffer_size_bytes.unwrap_or_default(), + priorities_in_flight: self.priorities_in_flight.unwrap_or_default(), + no_backpressure: self.no_backpressure.unwrap_or(false), + head_task_bytes: optional_u64(self.head_task_bytes_present, self.head_task_bytes), + head_task_priority_high: optional_u64( + self.head_task_priority_high_present, + self.head_task_priority_high, + ), + head_task_priority_low: optional_u64( + self.head_task_priority_low_present, + self.head_task_priority_low, + ), + min_in_flight_priority_high: optional_u64( + self.min_in_flight_priority_high_present, + self.min_in_flight_priority_high, + ), + min_in_flight_priority_low: optional_u64( + self.min_in_flight_priority_low_present, + self.min_in_flight_priority_low, + ), + head_task_can_deliver: optional_bool( + self.head_task_can_deliver_present, + self.head_task_can_deliver, + ), + head_task_priority_bypass: optional_bool( + self.head_task_priority_bypass_present, + self.head_task_priority_bypass, + ), + head_task_blocked_by_iops: optional_bool( + self.head_task_blocked_by_iops_present, + self.head_task_blocked_by_iops, + ), + head_task_blocked_by_bytes: optional_bool( + self.head_task_blocked_by_bytes_present, + self.head_task_blocked_by_bytes, + ), + }) + } +} + +impl Visit for SchedulerDiagnosticsVisitor { + fn record_bool(&mut self, field: &Field, value: bool) { + match field.name() { + "no_backpressure" => self.no_backpressure = Some(value), + "head_task_bytes_present" => self.head_task_bytes_present = value, + "head_task_priority_high_present" => self.head_task_priority_high_present = value, + "head_task_priority_low_present" => self.head_task_priority_low_present = value, + "min_in_flight_priority_high_present" => { + self.min_in_flight_priority_high_present = value; + } + "min_in_flight_priority_low_present" => { + self.min_in_flight_priority_low_present = value; + } + "head_task_can_deliver_present" => self.head_task_can_deliver_present = value, + "head_task_can_deliver" => self.head_task_can_deliver = Some(value), + "head_task_priority_bypass_present" => { + self.head_task_priority_bypass_present = value; + } + "head_task_priority_bypass" => self.head_task_priority_bypass = Some(value), + "head_task_blocked_by_iops_present" => { + self.head_task_blocked_by_iops_present = value; + } + "head_task_blocked_by_iops" => self.head_task_blocked_by_iops = Some(value), + "head_task_blocked_by_bytes_present" => { + self.head_task_blocked_by_bytes_present = value; + } + "head_task_blocked_by_bytes" => self.head_task_blocked_by_bytes = Some(value), + _ => {} + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + match field.name() { + "bytes_available" => self.bytes_available = Some(value), + "bytes_reserved" => self.bytes_reserved = Some(value), + _ => {} + } + } + + fn record_u64(&mut self, field: &Field, value: u64) { + match field.name() { + "scheduler_iops" => self.scheduler_iops = Some(value), + "scheduler_requests" => self.scheduler_requests = Some(value), + "scheduler_bytes_read" => self.scheduler_bytes_read = Some(value), + "io_capacity" => self.io_capacity = Some(value), + "iops_available" => self.iops_available = Some(value), + "active_iops" => self.active_iops = Some(value), + "pending_iops" => self.pending_iops = Some(value), + "pending_bytes" => self.pending_bytes = Some(value), + "io_buffer_size_bytes" => self.io_buffer_size_bytes = Some(value), + "priorities_in_flight" => self.priorities_in_flight = Some(value), + "head_task_bytes" => self.head_task_bytes = Some(value), + "head_task_priority_high" => self.head_task_priority_high = Some(value), + "head_task_priority_low" => self.head_task_priority_low = Some(value), + "min_in_flight_priority_high" => self.min_in_flight_priority_high = Some(value), + "min_in_flight_priority_low" => self.min_in_flight_priority_low = Some(value), + _ => {} + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "queue_kind" { + self.kind = match value { + "standard" => Some(SchedulerQueueKind::Standard), + "lite" => Some(SchedulerQueueKind::Lite), + _ => None, + }; + } + } + + fn record_debug(&mut self, _field: &Field, _value: &dyn std::fmt::Debug) {} +} + +fn optional_u64(present: bool, value: Option) -> Option { + present.then(|| value.unwrap_or_default()) +} + +fn optional_bool(present: bool, value: Option) -> Option { + present.then(|| value.unwrap_or(false)) +} + +fn diagnostics_from_scan_stats( + stats: ScanStats, + io_buffer_gib: Option, +) -> SchedulerDiagnostics { + SchedulerDiagnostics { + kind: SchedulerQueueKind::Standard, + stats, + io_capacity: 0, + iops_available: 0, + active_iops: 0, + pending_iops: 0, + pending_bytes: 0, + bytes_available: 0, + bytes_reserved: 0, + io_buffer_size_bytes: io_buffer_gib.map(|value| value * GIB).unwrap_or_default(), + priorities_in_flight: 0, + no_backpressure: false, + head_task_bytes: None, + head_task_priority_high: None, + head_task_priority_low: None, + min_in_flight_priority_high: None, + min_in_flight_priority_low: None, + head_task_can_deliver: None, + head_task_priority_bypass: None, + head_task_blocked_by_iops: None, + head_task_blocked_by_bytes: None, + } +} + +fn scan_stats_from_execution_summary(summary: &ExecutionSummaryCounts) -> ScanStats { + ScanStats { + iops: summary.iops as u64, + requests: summary.requests as u64, + bytes_read: summary.bytes_read as u64, + } +} + +fn sample_json( + started: Instant, + counters: &SharedCounters, + diagnostics: SchedulerDiagnostics, + decode_in_flight: usize, + channel_buffered: usize, + last: &mut LastSample, +) -> Value { + let elapsed = started.elapsed(); + let interval = elapsed.saturating_sub(last.elapsed); + let interval_secs = interval.as_secs_f64(); + let rows = counters.rows_completed.load(Ordering::Relaxed); + let batches = counters.batches_completed.load(Ordering::Relaxed); + let arrow_bytes = counters.arrow_bytes.load(Ordering::Relaxed); + let scheduler_stats = diagnostics.stats; + let delta_scheduler_bytes = + diff_u64(scheduler_stats.bytes_read, last.scheduler_stats.bytes_read); + let delta_rows = diff_u64(rows, last.rows); + let delta_arrow_bytes = diff_u64(arrow_bytes, last.arrow_bytes); + let physical_gbps = if interval_secs > 0.0 { + delta_scheduler_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let logical_gbps = if interval_secs > 0.0 { + delta_arrow_bytes as f64 * 8.0 / interval_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let rows_per_second = if interval_secs > 0.0 { + delta_rows as f64 / interval_secs + } else { + 0.0 + }; + + last.elapsed = elapsed; + last.scheduler_stats = scheduler_stats; + last.rows = rows; + last.batches = batches; + last.arrow_bytes = arrow_bytes; + + json!({ + "elapsed_seconds": elapsed.as_secs_f64(), + "interval_seconds": interval_secs, + "physical_gbps": physical_gbps, + "logical_gbps": logical_gbps, + "rows_per_second": rows_per_second, + "rows": rows, + "batches": batches, + "arrow_bytes": arrow_bytes, + "delta_rows": delta_rows, + "delta_arrow_bytes": delta_arrow_bytes, + "delta_scheduler_bytes": delta_scheduler_bytes, + "fragments_started": counters.fragments_started.load(Ordering::Relaxed), + "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed), + "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed), + "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed), + "batches_completed": counters.batches_completed.load(Ordering::Relaxed), + "decode_in_flight": decode_in_flight, + "channel_buffered": channel_buffered, + "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)), + "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)), + "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)), + "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)), + "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)), + "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)), + "scheduler": diagnostics_json(diagnostics), + }) +} + +async fn run_scanner_case( + config: &Config, + io_buffer_gib: Option, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + + let mut remaining_rows = config.limit_rows; + let mut planned_fragments = 0usize; + let mut planned_rows = 0u64; + for fragment in dataset.fragments().iter() { + if remaining_rows == 0 { + break; + } + let fragment_rows = fragment + .num_rows() + .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))? + as u64; + let rows = fragment_rows.min(remaining_rows); + planned_fragments += 1; + planned_rows += rows; + remaining_rows -= rows; + } + if planned_fragments == 0 { + return Err("no fragments selected".into()); + } + + let counters = Arc::new(SharedCounters::default()); + let stats_holder = ExecutionStatsHolder::default(); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + + let mut scanner = dataset.scan(); + if let Some(columns) = config.columns.as_ref() { + scanner.project(columns)?; + } + scanner + .batch_size(config.batch_size as usize) + .scan_in_order(false) + .scan_stats_callback(stats_holder.get_setter()); + if config.batch_concurrency > 0 { + scanner + .batch_readahead(config.batch_concurrency) + .target_parallelism(config.batch_concurrency); + } + if config.fragment_concurrency > 0 { + scanner.fragment_readahead(config.fragment_concurrency); + } + if let Some(file_reader_options) = file_reader_options(config) { + scanner.with_file_reader_options(file_reader_options); + } + if let Some(batch_size_bytes) = config.batch_size_bytes { + scanner.batch_size_bytes(batch_size_bytes); + } + if let Some(io_buffer_gib) = io_buffer_gib { + scanner.io_buffer_size(io_buffer_gib * GIB); + } + let limit_rows = i64::try_from(config.limit_rows) + .map_err(|_| "--limit-rows is too large for scanner limit")?; + scanner.limit(Some(limit_rows), None)?; + + let mut stream = scanner.try_into_stream().await?; + let mut rows = 0u64; + let mut batches = 0u64; + let mut arrow_bytes = 0u64; + let mut samples = Vec::new(); + let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms)); + sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_sample = LastSample { + elapsed: Duration::default(), + scheduler_stats: ScanStats::default(), + rows: 0, + batches: 0, + arrow_bytes: 0, + }; + loop { + tokio::select! { + maybe_batch = stream.next() => { + let Some(batch) = maybe_batch else { + break; + }; + let batch = batch?; + let batch_bytes = if config.skip_batch_byte_accounting { + 0 + } else { + batch.get_array_memory_size() as u64 + }; + rows += batch.num_rows() as u64; + batches += 1; + arrow_bytes += batch_bytes; + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + counters + .rows_completed + .fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + counters.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed); + } + _ = sample_interval.tick() => { + samples.push(sample_json( + started, + counters.as_ref(), + scheduler_diagnostics.snapshot(io_buffer_gib), + 0, + 0, + &mut last_sample, + )); + } + } + } + drop(stream); + + let elapsed = started.elapsed(); + let summary = stats_holder + .consume() + .ok_or("scanner execution stats callback did not run")?; + let scheduler_stats = scan_stats_from_execution_summary(&summary); + let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib); + final_diagnostics.stats = scheduler_stats; + let cpu_after = read_cpu_sample(); + samples.push(sample_json( + started, + counters.as_ref(), + final_diagnostics, + 0, + 0, + &mut last_sample, + )); + + Ok(CaseStats { + rows, + batches, + arrow_bytes, + planned_fragments, + planned_rows, + elapsed, + producer_finished_at: Some(elapsed), + peak_decode_in_flight: 0, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: final_diagnostics, + counters, + samples, + }) +} + +async fn run_dataset_take_case( + config: &Config, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let dataset = DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?; + let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?); + let total_rows = dataset + .fragments() + .iter() + .map(|fragment| { + fragment + .num_rows() + .map(|rows| rows as u64) + .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id)) + }) + .collect::, _>>()? + .into_iter() + .sum::(); + if total_rows == 0 { + return Err("dataset has no rows".into()); + } + + let counters = Arc::new(SharedCounters::default()); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + let mut rows = 0u64; + let mut batches = 0u64; + let mut arrow_bytes = 0u64; + const STRIDE: u64 = 104_729; + + for repetition in 0..config.take_repetitions { + let row_ids = (0..config.limit_rows) + .map(|offset| { + repetition + .wrapping_mul(STRIDE) + .wrapping_add(offset.wrapping_mul(STRIDE)) + % total_rows + }) + .collect::>(); + let batch = dataset + .take(&row_ids, ProjectionRequest::Schema(projection.clone())) + .await?; + if batch.num_rows() as u64 != config.limit_rows { + return Err(format!( + "take_rows returned {} rows, expected {}", + batch.num_rows(), + config.limit_rows + ) + .into()); + } + black_box(&batch); + rows += batch.num_rows() as u64; + batches += 1; + let batch_bytes = if config.skip_batch_byte_accounting { + 0 + } else { + batch.get_array_memory_size() as u64 + }; + arrow_bytes += batch_bytes; + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + counters + .rows_completed + .fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + counters + .arrow_bytes + .fetch_add(batch_bytes, Ordering::Relaxed); + } + + let elapsed = started.elapsed(); + let cpu_after = read_cpu_sample(); + Ok(CaseStats { + rows, + batches, + arrow_bytes, + planned_fragments: dataset.fragments().len(), + planned_rows: total_rows, + elapsed, + producer_finished_at: Some(elapsed), + peak_decode_in_flight: 0, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: scheduler_diagnostics.snapshot(None), + counters, + samples: Vec::new(), + }) +} + +enum RawInFlight { + Unordered(FuturesUnordered>>), + Ordered(FuturesOrdered>>), +} + +impl RawInFlight { + fn new(mode: RawCompletionMode) -> Self { + match mode { + RawCompletionMode::Unordered => Self::Unordered(FuturesUnordered::new()), + RawCompletionMode::Ordered => Self::Ordered(FuturesOrdered::new()), + } + } + + fn len(&self) -> usize { + match self { + Self::Unordered(in_flight) => in_flight.len(), + Self::Ordered(in_flight) => in_flight.len(), + } + } + + fn is_empty(&self) -> bool { + match self { + Self::Unordered(in_flight) => in_flight.is_empty(), + Self::Ordered(in_flight) => in_flight.is_empty(), + } + } + + fn push(&mut self, future: BoxFuture<'static, lance_core::Result>) { + match self { + Self::Unordered(in_flight) => in_flight.push(future), + Self::Ordered(in_flight) => in_flight.push_back(future), + } + } + + async fn next(&mut self) -> Option> { + match self { + Self::Unordered(in_flight) => in_flight.next().await, + Self::Ordered(in_flight) => in_flight.next().await, + } + } +} + +fn raw_read_future( + file_scheduler: FileScheduler, + range: Range, + priority: u64, + raw_submit_mode: RawSubmitMode, + read_chunk_size: u64, + counters: Arc, +) -> BoxFuture<'static, lance_core::Result> { + async move { + match raw_submit_mode { + RawSubmitMode::Single => { + let bytes = file_scheduler.submit_single(range, priority).await?; + Ok(bytes.len()) + } + RawSubmitMode::SplitNoConcat => { + let ranges = split_range_by_size(range, read_chunk_size); + let bytes = file_scheduler.submit_request(ranges, priority).await?; + Ok(bytes.iter().map(bytes::Bytes::len).sum()) + } + RawSubmitMode::SplitConcat => { + let ranges = split_range_by_size(range, read_chunk_size); + let bytes = file_scheduler.submit_request(ranges, priority).await?; + let reassemble_started = Instant::now(); + let total_size = bytes.iter().map(bytes::Bytes::len).sum(); + let mut combined = Vec::with_capacity(total_size); + for chunk in bytes { + combined.extend_from_slice(&chunk); + } + add_duration(&counters.raw_reassemble_ns, reassemble_started.elapsed()); + let len = combined.len(); + black_box(&combined); + Ok(len) + } + } + } + .boxed() +} + +fn split_range_by_size(range: Range, chunk_size: u64) -> Vec> { + let range_size = range.end - range.start; + if range_size <= chunk_size { + return vec![range]; + } + + let num_chunks = range_size.div_ceil(chunk_size); + let per_chunk = range_size / num_chunks; + let mut ranges = Vec::with_capacity(num_chunks as usize); + for idx in 0..num_chunks { + let start = range.start + idx * per_chunk; + let end = if idx == num_chunks - 1 { + range.end + } else { + start + per_chunk + }; + ranges.push(start..end); + } + ranges +} + +fn push_split_planned_ranges( + planned: &mut Vec<(usize, Range)>, + file_idx: usize, + range: Range, + chunk_size: u64, + remaining: &mut u64, +) { + let mut start = range.start; + while start < range.end && *remaining > 0 { + let bytes_to_read = chunk_size.min(range.end - start).min(*remaining); + if bytes_to_read == 0 { + break; + } + let end = start + bytes_to_read; + planned.push((file_idx, start..end)); + *remaining -= bytes_to_read; + start = end; + } +} + +fn push_split_ranges(ranges: &mut Vec>, range: Range, chunk_size: u64) { + let mut start = range.start; + while start < range.end { + let bytes_to_read = chunk_size.min(range.end - start); + if bytes_to_read == 0 { + break; + } + let end = start + bytes_to_read; + ranges.push(start..end); + start = end; + } +} + +async fn run_scheduler_raw_case( + config: &Config, + io_buffer_gib: Option, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let target_bytes = config + .target_bytes + .ok_or("--target-bytes is required for --backend scheduler-raw")?; + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?; + let scheduler_config = io_buffer_gib + .map(|gib| SchedulerConfig::new(gib * GIB)) + .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref())); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + + let (selected_files, planned) = match config.raw_range_mode { + RawRangeMode::FileSequential => { + let mut selected_files = Vec::new(); + let mut selected_file_bytes = 0u64; + for fragment in dataset.fragments().iter() { + for data_file in &fragment.files { + if data_file.base_id.is_some() { + continue; + } + let Some(file_size) = data_file.file_size_bytes.get() else { + continue; + }; + let path = dataset.data_dir().join(data_file.path.as_str()); + let file_scheduler = scheduler + .open_file_with_priority(&path, 0, &data_file.file_size_bytes) + .await?; + selected_file_bytes += file_size.get(); + selected_files.push((file_scheduler, file_size.get())); + if selected_file_bytes >= target_bytes { + break; + } + } + if selected_file_bytes >= target_bytes { + break; + } + } + if selected_files.is_empty() { + return Err("scheduler-raw found no data files with known sizes".into()); + } + + let mut offsets = vec![0u64; selected_files.len()]; + let mut planned = Vec::new(); + let mut remaining = target_bytes; + let mut file_idx = 0usize; + while remaining > 0 { + let idx = file_idx % selected_files.len(); + let file_size = selected_files[idx].1; + if offsets[idx] >= file_size { + offsets[idx] = 0; + } + let available = file_size - offsets[idx]; + let bytes_to_read = config.raw_range_size_bytes.min(available).min(remaining); + let start = offsets[idx]; + let end = start + bytes_to_read; + planned.push((idx, start..end)); + offsets[idx] = end; + remaining -= bytes_to_read; + file_idx += 1; + } + (selected_files, planned) + } + RawRangeMode::MetadataPages | RawRangeMode::MetadataPagesRoundRobin => { + let mut selected_files = Vec::new(); + let mut per_file_ranges = Vec::>>::new(); + let mut candidate_bytes = 0u64; + + 'fragments: for fragment in dataset.fragments().iter() { + for data_file in &fragment.files { + if data_file.base_id.is_some() { + continue; + } + if data_file.file_size_bytes.get().is_none() { + continue; + } + let path = dataset.data_dir().join(data_file.path.as_str()); + let file_scheduler = scheduler + .open_file_with_priority(&path, 0, &data_file.file_size_bytes) + .await?; + let metadata = LanceFileReader::read_all_metadata(&file_scheduler).await?; + let mut file_ranges = Vec::new(); + + let raw_column_indices = config + .raw_column_indices + .clone() + .unwrap_or_else(|| (0..metadata.column_infos.len() as u32).collect()); + for column_index in raw_column_indices { + let column_info = metadata + .column_infos + .get(column_index as usize) + .ok_or_else(|| { + format!( + "raw metadata-pages requested column index {column_index} but file has {} columns", + metadata.column_infos.len() + ) + })?; + for page in column_info.page_infos.iter() { + for (offset, size) in page.buffer_offsets_and_sizes.iter() { + if *size == 0 { + continue; + } + push_split_ranges( + &mut file_ranges, + *offset..(*offset + *size), + config.raw_range_size_bytes, + ); + } + } + } + + if !file_ranges.is_empty() { + candidate_bytes += file_ranges + .iter() + .map(|range| range.end - range.start) + .sum::(); + selected_files.push(( + file_scheduler, + data_file.file_size_bytes.get().unwrap().get(), + )); + per_file_ranges.push(file_ranges); + if candidate_bytes >= target_bytes { + break 'fragments; + } + } + } + } + if selected_files.is_empty() || per_file_ranges.is_empty() { + return Err("scheduler-raw metadata-pages found no readable page buffers".into()); + } + + let mut planned = Vec::new(); + let mut remaining = target_bytes; + match config.raw_range_mode { + RawRangeMode::MetadataPages => { + 'ranges: for (file_idx, ranges) in per_file_ranges.iter().enumerate() { + for range in ranges { + push_split_planned_ranges( + &mut planned, + file_idx, + range.clone(), + config.raw_range_size_bytes, + &mut remaining, + ); + if remaining == 0 { + break 'ranges; + } + } + } + } + RawRangeMode::MetadataPagesRoundRobin => { + let mut positions = vec![0usize; per_file_ranges.len()]; + while remaining > 0 { + let mut made_progress = false; + for (file_idx, ranges) in per_file_ranges.iter().enumerate() { + if positions[file_idx] >= ranges.len() { + continue; + } + let range = ranges[positions[file_idx]].clone(); + positions[file_idx] += 1; + made_progress = true; + push_split_planned_ranges( + &mut planned, + file_idx, + range, + config.raw_range_size_bytes, + &mut remaining, + ); + if remaining == 0 { + break; + } + } + if !made_progress { + break; + } + } + } + RawRangeMode::FileSequential => unreachable!(), + } + + if remaining > 0 { + return Err(format!( + "scheduler-raw metadata-pages planned {} bytes but target is {target_bytes}", + target_bytes - remaining + ) + .into()); + } + (selected_files, planned) + } + }; + let planned_bytes = planned + .iter() + .map(|(_, range)| range.end - range.start) + .sum::(); + + let counters = Arc::new(SharedCounters::default()); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + let mut samples = Vec::new(); + let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms)); + sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_sample = LastSample { + elapsed: Duration::default(), + scheduler_stats: ScanStats::default(), + rows: 0, + batches: 0, + arrow_bytes: 0, + }; + let mut in_flight = RawInFlight::new(config.raw_completion_mode); + let mut next_range = 0usize; + let read_chunk_size = config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE); + while next_range < planned.len() && in_flight.len() < config.batch_concurrency { + let (idx, range) = planned[next_range].clone(); + in_flight.push(raw_read_future( + selected_files[idx].0.clone(), + range, + next_range as u64, + config.raw_submit_mode, + read_chunk_size, + counters.clone(), + )); + next_range += 1; + } + + let mut bytes_read = 0u64; + let mut requests_completed = 0u64; + while !in_flight.is_empty() { + tokio::select! { + maybe_bytes = in_flight.next() => { + let bytes = maybe_bytes.expect("raw read future disappeared")?; + let bytes = bytes as u64; + bytes_read += bytes; + requests_completed += 1; + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + counters.arrow_bytes.fetch_add(bytes, Ordering::Relaxed); + if next_range < planned.len() { + let (idx, range) = planned[next_range].clone(); + in_flight.push(raw_read_future( + selected_files[idx].0.clone(), + range, + next_range as u64, + config.raw_submit_mode, + read_chunk_size, + counters.clone(), + )); + next_range += 1; + } + } + _ = sample_interval.tick() => { + samples.push(sample_json( + started, + counters.as_ref(), + scheduler_diagnostics.snapshot(io_buffer_gib), + in_flight.len(), + planned.len().saturating_sub(next_range), + &mut last_sample, + )); + } + } + } + counters.arrow_bytes.store(bytes_read, Ordering::Relaxed); + let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib); + final_diagnostics.stats = scheduler.stats(); + samples.push(sample_json( + started, + counters.as_ref(), + final_diagnostics, + in_flight.len(), + 0, + &mut last_sample, + )); + let elapsed = started.elapsed(); + let cpu_after = read_cpu_sample(); + + Ok(CaseStats { + rows: 0, + batches: requests_completed, + arrow_bytes: bytes_read, + planned_fragments: selected_files.len(), + planned_rows: planned_bytes, + elapsed, + producer_finished_at: Some(elapsed), + peak_decode_in_flight: config.batch_concurrency, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: final_diagnostics, + counters, + samples, + }) +} + +async fn run_case( + config: &Config, + io_buffer_gib: Option, + scheduler_diagnostics: &SchedulerDiagnosticsCollector, +) -> Result { + scheduler_diagnostics.clear(); + let dataset = Arc::new( + DatasetBuilder::from_uri(&config.uri) + .with_version(config.dataset_version) + .load() + .await?, + ); + let projection = Arc::new(projected_schema(dataset.schema(), &config.columns)?); + let (object_store, _) = LanceObjectStore::from_uri(&config.uri).await?; + let scheduler_config = io_buffer_gib + .map(|gib| SchedulerConfig::new(gib * GIB)) + .unwrap_or_else(|| SchedulerConfig::max_bandwidth(object_store.as_ref())); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + + let mut planned = Vec::new(); + let mut remaining_rows = config.limit_rows; + for fragment in dataset.fragments().iter() { + if remaining_rows == 0 { + break; + } + let fragment_rows = fragment + .num_rows() + .ok_or_else(|| format!("fragment {} is missing num_rows", fragment.id))? + as u64; + let rows = fragment_rows.min(remaining_rows); + planned.push((fragment.clone(), rows)); + remaining_rows -= rows; + } + if planned.is_empty() { + return Err("no fragments selected".into()); + } + let planned_rows: u64 = planned.iter().map(|(_, rows)| *rows).sum(); + let planned_fragments = planned.len(); + + let counters = Arc::new(SharedCounters::default()); + let cpu_before = read_cpu_sample(); + let started = Instant::now(); + + let (tx, mut rx) = tokio::sync::mpsc::channel::< + BoxFuture<'static, lance_core::Result>, + >(config.batch_concurrency * 2); + let producer = if config.detach_fragment_streams { + tokio::spawn({ + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let counters = counters.clone(); + let batch_size = config.batch_size; + let file_reader_options = file_reader_options(config); + let fragment_concurrency = config.fragment_concurrency; + async move { + let drainers = futures::stream::iter(planned.into_iter().enumerate()) + .map({ + move |(priority, (fragment, rows))| { + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let tx = tx.clone(); + let counters = counters.clone(); + let file_reader_options = file_reader_options.clone(); + async move { + counters.fragments_started.fetch_add(1, Ordering::Relaxed); + let file_fragment = FileFragment::new(dataset, fragment); + let read_config = FragReadConfig::default() + .with_scan_scheduler(scheduler) + .with_reader_priority(priority as u32); + let read_config = if let Some(file_reader_options) = + file_reader_options.clone() + { + read_config.with_file_reader_options(file_reader_options) + } else { + read_config + }; + + let open_started = Instant::now(); + let reader = + file_fragment.open(projection.as_ref(), read_config).await?; + add_duration(&counters.open_reader_ns, open_started.elapsed()); + + let create_stream_started = Instant::now(); + let mut read_stream = + reader.read_ranges(vec![0..rows].into(), batch_size).await?; + add_duration( + &counters.read_stream_create_ns, + create_stream_started.elapsed(), + ); + + let drainer = tokio::spawn(async move { + loop { + let next_started = Instant::now(); + let maybe_batch_fut = read_stream.next().await; + add_duration( + &counters.next_batch_poll_ns, + next_started.elapsed(), + ); + let Some(batch_fut) = maybe_batch_fut else { + break; + }; + counters + .batch_futures_emitted + .fetch_add(1, Ordering::Relaxed); + let send_started = Instant::now(); + tx.send(batch_fut) + .await + .map_err(|_| "batch consumer dropped")?; + add_duration( + &counters.channel_send_wait_ns, + send_started.elapsed(), + ); + } + counters.fragments_completed.fetch_add(1, Ordering::Relaxed); + Ok::<_, Error>(()) + }); + Ok::<_, Error>(drainer) + } + } + }) + .buffer_unordered(fragment_concurrency) + .try_collect::>() + .await?; + + for drainer in drainers { + drainer.await.map_err(Error::from)??; + } + Ok::<_, Error>(()) + } + }) + } else { + tokio::spawn({ + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let counters = counters.clone(); + let batch_size = config.batch_size; + let file_reader_options = file_reader_options(config); + let fragment_concurrency = config.fragment_concurrency; + async move { + futures::stream::iter(planned.into_iter().enumerate()) + .map({ + move |(priority, (fragment, rows))| { + let dataset = dataset.clone(); + let projection = projection.clone(); + let scheduler = scheduler.clone(); + let tx = tx.clone(); + let counters = counters.clone(); + let file_reader_options = file_reader_options.clone(); + async move { + counters.fragments_started.fetch_add(1, Ordering::Relaxed); + let file_fragment = FileFragment::new(dataset, fragment); + let read_config = FragReadConfig::default() + .with_scan_scheduler(scheduler) + .with_reader_priority(priority as u32); + let read_config = if let Some(file_reader_options) = + file_reader_options.clone() + { + read_config.with_file_reader_options(file_reader_options) + } else { + read_config + }; + + let open_started = Instant::now(); + let reader = + file_fragment.open(projection.as_ref(), read_config).await?; + add_duration(&counters.open_reader_ns, open_started.elapsed()); + + let create_stream_started = Instant::now(); + let mut read_stream = + reader.read_ranges(vec![0..rows].into(), batch_size).await?; + add_duration( + &counters.read_stream_create_ns, + create_stream_started.elapsed(), + ); + + loop { + let next_started = Instant::now(); + let maybe_batch_fut = read_stream.next().await; + add_duration( + &counters.next_batch_poll_ns, + next_started.elapsed(), + ); + let Some(batch_fut) = maybe_batch_fut else { + break; + }; + counters + .batch_futures_emitted + .fetch_add(1, Ordering::Relaxed); + let send_started = Instant::now(); + tx.send(batch_fut) + .await + .map_err(|_| "batch consumer dropped")?; + add_duration( + &counters.channel_send_wait_ns, + send_started.elapsed(), + ); + } + counters.fragments_completed.fetch_add(1, Ordering::Relaxed); + Ok::<_, Error>(()) + } + } + }) + .buffer_unordered(fragment_concurrency) + .try_collect::>() + .await?; + Ok::<_, Error>(()) + } + }) + }; + + let mut in_flight = FuturesUnordered::new(); + let skip_batch_byte_accounting = config.skip_batch_byte_accounting; + let drop_read_tasks = config.drop_read_tasks; + let mut producer_done = false; + let mut producer_finished_at = None; + let mut rows = 0u64; + let mut batches = 0u64; + let mut arrow_bytes = 0u64; + let mut peak_decode_in_flight = 0usize; + let mut samples = Vec::new(); + let mut sample_interval = tokio::time::interval(Duration::from_millis(config.sample_ms)); + sample_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut last_sample = LastSample { + elapsed: Duration::default(), + scheduler_stats: ScanStats::default(), + rows: 0, + batches: 0, + arrow_bytes: 0, + }; + + loop { + if producer_done && in_flight.is_empty() { + break; + } + tokio::select! { + maybe_batch_fut = rx.recv(), if !producer_done && in_flight.len() < config.batch_concurrency => { + if let Some(batch_fut) = maybe_batch_fut { + counters.batch_futures_received.fetch_add(1, Ordering::Relaxed); + if drop_read_tasks { + drop(batch_fut); + counters.batches_completed.fetch_add(1, Ordering::Relaxed); + batches += 1; + continue; + } + let counters_for_task = counters.clone(); + in_flight.push(async move { + let decode_started = Instant::now(); + let batch = batch_fut.await?; + let batch_bytes = if skip_batch_byte_accounting { + 0 + } else { + batch.get_array_memory_size() as u64 + }; + add_duration(&counters_for_task.decode_ns, decode_started.elapsed()); + counters_for_task.batches_completed.fetch_add(1, Ordering::Relaxed); + counters_for_task.rows_completed.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + counters_for_task.arrow_bytes.fetch_add(batch_bytes, Ordering::Relaxed); + Ok::<_, lance_core::Error>((batch, batch_bytes)) + }); + peak_decode_in_flight = peak_decode_in_flight.max(in_flight.len()); + } else { + producer_done = true; + producer_finished_at = Some(started.elapsed()); + } + } + maybe_batch = in_flight.next(), if !in_flight.is_empty() => { + let (batch, batch_bytes) = maybe_batch.expect("in-flight batch future disappeared")?; + rows += batch.num_rows() as u64; + batches += 1; + arrow_bytes += batch_bytes; + } + _ = sample_interval.tick() => { + samples.push(sample_json( + started, + counters.as_ref(), + scheduler_diagnostics.snapshot(io_buffer_gib), + in_flight.len(), + rx.len(), + &mut last_sample, + )); + } + } + } + producer.await??; + + let mut final_diagnostics = scheduler_diagnostics.snapshot(io_buffer_gib); + final_diagnostics.stats = scheduler.stats(); + samples.push(sample_json( + started, + counters.as_ref(), + final_diagnostics, + in_flight.len(), + rx.len(), + &mut last_sample, + )); + + let elapsed = started.elapsed(); + let cpu_after = read_cpu_sample(); + + Ok(CaseStats { + rows, + batches, + arrow_bytes, + planned_fragments, + planned_rows, + elapsed, + producer_finished_at, + peak_decode_in_flight, + cpu_avg: cpu_before.zip(cpu_after).and_then(|(before, after)| { + let total = after.total.checked_sub(before.total)?; + let idle = after.idle.checked_sub(before.idle)?; + if total == 0 { + return None; + } + Some((total - idle) as f64 / total as f64 * 100.0) + }), + scheduler_diagnostics: final_diagnostics, + counters, + samples, + }) +} + +fn read_cpu_sample() -> Option { + let contents = fs::read_to_string("/proc/stat").ok()?; + let line = contents.lines().next()?; + let values = line + .split_whitespace() + .skip(1) + .map(|value| value.parse::()) + .collect::, _>>() + .ok()?; + if values.len() < 5 { + return None; + } + + let idle = values[3] + values[4]; + let total = values.iter().sum(); + Some(CpuSample { idle, total }) +} + +fn now_unix_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or_default() +} + +fn current_commit() -> String { + option_env!("LANCE_BENCH_COMMIT") + .or_else(|| option_env!("GIT_COMMIT")) + .unwrap_or("unknown") + .to_string() +} + +fn env_var(name: &str) -> Option { + env::var(name).ok() +} + +#[tokio::main] +async fn main() -> Result<()> { + let config = parse_args()?; + if config.describe_layout { + return describe_layout(&config).await; + } + let scheduler_diagnostics = SchedulerDiagnosticsCollector::default(); + let subscriber = tracing_subscriber::registry().with(SchedulerDiagnosticsLayer::new( + scheduler_diagnostics.clone(), + )); + tracing::subscriber::set_global_default(subscriber).map_err(|error| { + std::io::Error::other(format!( + "failed to install scheduler diagnostics subscriber: {error}" + )) + })?; + + fs::create_dir_all(&config.out_dir)?; + let output_path = format!( + "{}/s3_file_reader_diagnostics_{}.jsonl", + config.out_dir, + now_unix_secs() + ); + let mut jsonl = String::new(); + let commit = current_commit(); + let instance = env_var("EC2_INSTANCE_TYPE").unwrap_or_else(|| "unknown".to_string()); + let region = env_var("AWS_REGION") + .or_else(|| env_var("AWS_DEFAULT_REGION")) + .unwrap_or_else(|| "unknown".to_string()); + let projection = projection_name(&config.columns); + + for io_buffer_gib in &config.io_buffer_gib { + println!( + "running case={} backend={} projection={} limit_rows={} io_buffer_gib={} batch_size={} fragment_concurrency={} batch_concurrency={} sample_ms={}", + config.case_name, + config.backend.name(), + projection, + config.limit_rows, + io_buffer_gib + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".to_string()), + config.batch_size, + config.fragment_concurrency, + config.batch_concurrency, + config.sample_ms + ); + let stats = match config.backend { + Backend::FileReader => { + run_case(&config, *io_buffer_gib, &scheduler_diagnostics).await? + } + Backend::Scanner => { + run_scanner_case(&config, *io_buffer_gib, &scheduler_diagnostics).await? + } + Backend::SchedulerRaw => { + run_scheduler_raw_case(&config, *io_buffer_gib, &scheduler_diagnostics).await? + } + Backend::DatasetTake => run_dataset_take_case(&config, &scheduler_diagnostics).await?, + }; + let elapsed_secs = stats.elapsed.as_secs_f64(); + let scheduler_stats = stats.scheduler_diagnostics.stats; + let logical_gbps = if elapsed_secs > 0.0 { + stats.arrow_bytes as f64 * 8.0 / elapsed_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let physical_gbps = if elapsed_secs > 0.0 { + scheduler_stats.bytes_read as f64 * 8.0 / elapsed_secs / 1_000_000_000.0 + } else { + 0.0 + }; + let rows_per_second = if elapsed_secs > 0.0 { + stats.rows as f64 / elapsed_secs + } else { + 0.0 + }; + let bytes_per_row = stats.arrow_bytes.checked_div(stats.rows).unwrap_or(0); + let avg_bytes_per_scheduler_request = scheduler_stats + .bytes_read + .checked_div(scheduler_stats.requests) + .unwrap_or(0); + let avg_bytes_per_scheduler_iop = scheduler_stats + .bytes_read + .checked_div(scheduler_stats.iops) + .unwrap_or(0); + let counters = stats.counters.as_ref(); + let record = json!({ + "case": config.case_name, + "instance": instance, + "region": region, + "layer": config.backend.layer(), + "backend": config.backend.name(), + "dataset_uri": config.uri, + "dataset_version": config.dataset_version, + "lance_commit": commit, + "projection": projection, + "limit_rows": config.limit_rows, + "target_bytes": config.target_bytes, + "raw_range_size_bytes": config.raw_range_size_bytes, + "raw_range_mode": config.raw_range_mode.name(), + "raw_column_indices": config.raw_column_indices.clone(), + "raw_submit_mode": config.raw_submit_mode.name(), + "raw_completion_mode": config.raw_completion_mode.name(), + "take_repetitions": config.take_repetitions, + "raw_read_chunk_size_bytes": config.read_chunk_size.unwrap_or(DEFAULT_READ_CHUNK_SIZE), + "planned_rows": stats.planned_rows, + "planned_fragments": stats.planned_fragments, + "rows": stats.rows, + "batches": stats.batches, + "batch_size": config.batch_size, + "batch_size_bytes": config.batch_size_bytes, + "skip_batch_byte_accounting": config.skip_batch_byte_accounting, + "read_chunk_size": config.read_chunk_size, + "fragment_concurrency": config.fragment_concurrency, + "batch_concurrency": config.batch_concurrency, + "detach_fragment_streams": config.detach_fragment_streams, + "drop_read_tasks": config.drop_read_tasks, + "sample_ms": config.sample_ms, + "io_buffer_bytes": io_buffer_gib.map(|value| value * GIB), + "io_buffer_mode": if io_buffer_gib.is_some() { "explicit" } else { "auto" }, + "lance_io_threads": env_var("LANCE_IO_THREADS").or_else(|| env_var("IO_THREADS")), + "lance_default_io_buffer_size": env_var("LANCE_DEFAULT_IO_BUFFER_SIZE"), + "lance_max_iop_size": env_var("LANCE_MAX_IOP_SIZE"), + "lance_use_lite_scheduler": env_var("LANCE_USE_LITE_SCHEDULER"), + "lance_inline_scheduling_threshold": env_var("LANCE_INLINE_SCHEDULING_THRESHOLD"), + "elapsed_seconds": elapsed_secs, + "producer_finished_seconds": stats.producer_finished_at.map(|duration| duration.as_secs_f64()), + "logical_gbps": logical_gbps, + "physical_gbps": physical_gbps, + "rows_per_second": rows_per_second, + "arrow_bytes": stats.arrow_bytes, + "bytes_per_row": bytes_per_row, + "avg_bytes_per_scheduler_request": avg_bytes_per_scheduler_request, + "avg_bytes_per_scheduler_iop": avg_bytes_per_scheduler_iop, + "scheduler_iops": scheduler_stats.iops, + "scheduler_requests": scheduler_stats.requests, + "scheduler_bytes_read": scheduler_stats.bytes_read, + "scheduler_diagnostics": diagnostics_json(stats.scheduler_diagnostics), + "fragments_started": counters.fragments_started.load(Ordering::Relaxed), + "fragments_completed": counters.fragments_completed.load(Ordering::Relaxed), + "batch_futures_emitted": counters.batch_futures_emitted.load(Ordering::Relaxed), + "batch_futures_received": counters.batch_futures_received.load(Ordering::Relaxed), + "batches_completed": counters.batches_completed.load(Ordering::Relaxed), + "peak_decode_in_flight": stats.peak_decode_in_flight, + "open_reader_seconds_total": ns_to_seconds(counters.open_reader_ns.load(Ordering::Relaxed)), + "read_stream_create_seconds_total": ns_to_seconds(counters.read_stream_create_ns.load(Ordering::Relaxed)), + "next_batch_poll_seconds_total": ns_to_seconds(counters.next_batch_poll_ns.load(Ordering::Relaxed)), + "channel_send_wait_seconds_total": ns_to_seconds(counters.channel_send_wait_ns.load(Ordering::Relaxed)), + "decode_seconds_total": ns_to_seconds(counters.decode_ns.load(Ordering::Relaxed)), + "raw_reassemble_seconds_total": ns_to_seconds(counters.raw_reassemble_ns.load(Ordering::Relaxed)), + "cpu_avg": stats.cpu_avg, + "samples": stats.samples, + }); + println!( + "case={} backend={} projection={} io_buffer_gib={} elapsed={:.3}s logical_gbps={:.2} physical_gbps={:.2} rows={} batches={} scheduler_bytes_read={} scheduler_iops={} scheduler_requests={} active_iops={} pending_iops={} cpu_avg={}", + config.case_name, + config.backend.name(), + projection, + io_buffer_gib + .map(|value| value.to_string()) + .unwrap_or_else(|| "auto".to_string()), + elapsed_secs, + logical_gbps, + physical_gbps, + stats.rows, + stats.batches, + scheduler_stats.bytes_read, + scheduler_stats.iops, + scheduler_stats.requests, + stats.scheduler_diagnostics.active_iops, + stats.scheduler_diagnostics.pending_iops, + stats + .cpu_avg + .map(|value| format!("{value:.1}%")) + .unwrap_or_else(|| "unknown".to_string()) + ); + jsonl.push_str(&serde_json::to_string(&record)?); + jsonl.push('\n'); + fs::write(&output_path, &jsonl)?; + } + + println!("wrote {output_path}"); + Ok(()) +} diff --git a/rust/lance/benches/take_blob.rs b/rust/lance/benches/take_blob.rs index 7f3483bffda..0da30d9f277 100644 --- a/rust/lance/benches/take_blob.rs +++ b/rust/lance/benches/take_blob.rs @@ -15,7 +15,7 @@ use lance::dataset::{Dataset, ProjectionRequest, ReadParams, WriteParams}; use lance_arrow::BLOB_META_KEY; use lance_encoding::decoder::DecoderConfig; use lance_file::reader::FileReaderOptions; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; #[cfg(target_os = "linux")] use lance_testing::pprof::{Output, PProfProfiler}; use tokio::runtime::Runtime; @@ -209,7 +209,10 @@ async fn write_blob_dataset( version: LanceFileVersion, cache_repetition_index: bool, ) -> Dataset { - let batches = if version >= LanceFileVersion::V2_2 { + let batches = if matches!( + version.resolve(), + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 + ) { make_blob_v2_batches() } else { make_legacy_blob_batches() diff --git a/rust/lance/src/blob.rs b/rust/lance/src/blob.rs index b112f011419..f3a6aa745cd 100644 --- a/rust/lance/src/blob.rs +++ b/rust/lance/src/blob.rs @@ -3,8 +3,11 @@ //! Builders and file-level writer helpers for Lance blob v2 columns. //! -//! Logical blob input uses `Struct`. File-level blob -//! descriptors use a physical writer-side struct with `kind`, `blob_id`, and range fields. +//! Logical blob input uses either `Struct` or the complete +//! `Struct` shape. In the +//! complete shape, `position` and `size` select a non-empty range within an external `uri` and +//! must be set together. Every non-null row must set exactly one of `data` and `uri`. File-level +//! blob preparation produces a kind-aware writer intermediate with `blob_id` and range fields. use std::collections::{HashMap, HashSet}; use std::num::NonZeroUsize; @@ -21,14 +24,17 @@ use arrow_array::{ types::{UInt8Type, UInt32Type, UInt64Type}, }; use arrow_buffer::NullBufferBuilder; -use arrow_schema::{DataType, Field, Fields}; +use arrow_schema::{DataType, Field}; use bytes::Bytes; use lance_arrow::{ ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, FieldExt, }; use lance_core::{ - datatypes::{BlobKind, Field as LanceField, Schema as LanceSchema}, + datatypes::{ + BLOB_V2_LOGICAL_MINIMAL_FIELDS, BLOB_V2_PREPARED_FIELDS, BLOB_V2_PREPARED_TYPE, BlobKind, + BlobV2Layout, Field as LanceField, Schema as LanceSchema, + }, utils::blob::blob_path, }; use lance_io::{ @@ -42,8 +48,10 @@ use crate::{Error, Result}; /// Construct the Arrow field for a blob v2 column. /// -/// Blob v2 expects a column shaped as `Struct` and -/// tagged with `ARROW:extension:name = "lance.blob.v2"`. +/// This helper constructs the minimal logical shape +/// `Struct`, tagged with +/// `ARROW:extension:name = "lance.blob.v2"`. Writers also accept the complete logical shape +/// with trailing `position: UInt64?` and `size: UInt64?` fields for external URI ranges. pub fn blob_field(name: &str, nullable: bool) -> Field { blob_field_with_options(name, nullable, BlobFieldOptions::default()) } @@ -76,8 +84,10 @@ impl BlobFieldOptions { /// Construct the Arrow field for a blob v2 column with storage layout options. /// -/// Blob v2 expects a column shaped as `Struct` and -/// tagged with `ARROW:extension:name = "lance.blob.v2"`. +/// This helper constructs the minimal logical shape +/// `Struct`, tagged with +/// `ARROW:extension:name = "lance.blob.v2"`. Writers also accept the complete logical shape +/// with trailing `position: UInt64?` and `size: UInt64?` fields for external URI ranges. /// /// ``` /// # use lance::{BlobFieldOptions, blob_field_with_options}; @@ -112,125 +122,87 @@ pub fn blob_field_with_options(name: &str, nullable: bool, options: BlobFieldOpt } Field::new( name, - DataType::Struct( - vec![ - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - ] - .into(), - ), + DataType::Struct(BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone()), nullable, ) .with_metadata(metadata) } -fn prepared_blob_child_fields() -> Fields { - Fields::from(vec![ - Field::new("kind", DataType::UInt8, true), - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - Field::new("blob_id", DataType::UInt32, true), - Field::new("blob_size", DataType::UInt64, true), - Field::new("position", DataType::UInt64, true), - ]) -} - fn prepared_blob_field_with_metadata( name: &str, nullable: bool, metadata: HashMap, ) -> Field { - Field::new( - name, - DataType::Struct(prepared_blob_child_fields()), - nullable, - ) - .with_metadata(metadata) + Field::new(name, BLOB_V2_PREPARED_TYPE.clone(), nullable).with_metadata(metadata) } fn logical_blob_lance_children() -> Result> { - [ - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - ] - .iter() - .map(LanceField::try_from) - .collect() -} - -fn field_matches(field: &Field, name: &str, data_type: &DataType, nullable: bool) -> bool { - field.name() == name && field.data_type() == data_type && field.is_nullable() == nullable -} - -fn blob_v2_shape_error(field: &Field) -> Error { - Error::invalid_input(format!( - "Blob v2 field '{}' must use either logical struct \ - with optional position/size UInt64 fields or prepared struct", - field.name() - )) + BLOB_V2_LOGICAL_MINIMAL_FIELDS + .iter() + .map(|field| LanceField::try_from(field.as_ref())) + .collect() } -/// Returns true when `field` is the writer-side prepared blob v2 struct. -pub(crate) fn is_prepared_blob_v2_field(field: &Field) -> bool { +pub(crate) fn blob_v2_layout(field: &Field) -> Option { if !field.is_blob_v2() { - return false; + return None; } let DataType::Struct(fields) = field.data_type() else { - return false; + return None; }; - let expected = prepared_blob_child_fields(); - fields.len() == expected.len() - && fields - .iter() - .zip(expected.iter()) - .all(|(actual, expected)| actual.as_ref() == expected.as_ref()) + BlobV2Layout::classify(fields) } -/// Returns true when `field` is the logical blob v2 input struct. -pub(crate) fn is_logical_blob_v2_field(field: &Field) -> bool { - if !field.is_blob_v2() { - return false; - } - let DataType::Struct(fields) = field.data_type() else { - return false; +pub(crate) fn blob_v2_shape_error(field: &Field, expected: &[BlobV2Layout]) -> Error { + let expected = expected + .iter() + .map(ToString::to_string) + .collect::>() + .join(" or "); + let actual = match field.data_type() { + DataType::Struct(fields) => BlobV2Layout::classify(fields) + .map(|layout| format!("{layout} layout")) + .unwrap_or_else(|| format!("unrecognized layout {fields:?}")), + data_type => format!("non-struct type {data_type}"), }; - match fields.len() { - 2 => { - field_matches(fields[0].as_ref(), "data", &DataType::LargeBinary, true) - && field_matches(fields[1].as_ref(), "uri", &DataType::Utf8, true) - } - 4 => { - field_matches(fields[0].as_ref(), "data", &DataType::LargeBinary, true) - && field_matches(fields[1].as_ref(), "uri", &DataType::Utf8, true) - && fields[2].name() == "position" - && fields[2].data_type() == &DataType::UInt64 - && fields[3].name() == "size" - && fields[3].data_type() == &DataType::UInt64 - } - _ => false, - } + Error::invalid_input(format!( + "Blob v2 field '{}' has {actual}; expected {expected} layout", + field.name(), + )) } -fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result { +fn prepared_to_logical_blob_lance_field(field: &LanceField) -> Result { if field.is_blob_v2() { let arrow_field = Field::from(field); - if is_prepared_blob_v2_field(&arrow_field) { - let mut normalized = field.clone(); - let mut logical_children = logical_blob_lance_children()?; - for (logical_child, prepared_child) in - logical_children.iter_mut().zip(field.children.iter()) - { - logical_child.id = prepared_child.id; - logical_child.parent_id = field.id; + match blob_v2_layout(&arrow_field) { + Some(BlobV2Layout::Prepared) => { + let mut normalized = field.clone(); + let mut logical_children = logical_blob_lance_children()?; + for logical_child in &mut logical_children { + let prepared_child = field + .children + .iter() + .find(|prepared_child| prepared_child.name == logical_child.name) + .ok_or_else(|| { + Error::internal(format!( + "Prepared blob v2 field '{}' is missing logical child '{}'", + field.name, logical_child.name + )) + })?; + logical_child.id = prepared_child.id; + logical_child.parent_id = field.id; + } + normalized.children = logical_children; + return Ok(normalized); + } + Some(BlobV2Layout::Logical) => return Ok(field.clone()), + _ => { + return Err(blob_v2_shape_error( + &arrow_field, + &[BlobV2Layout::Logical, BlobV2Layout::Prepared], + )); } - normalized.children = logical_children; - return Ok(normalized); - } - if is_logical_blob_v2_field(&arrow_field) { - return Ok(field.clone()); } - return Err(blob_v2_shape_error(&arrow_field)); } if field.children.is_empty() { @@ -240,7 +212,7 @@ fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result let normalized_children = field .children .iter() - .map(normalize_prepared_blob_lance_field) + .map(prepared_to_logical_blob_lance_field) .collect::>>()?; Ok(LanceField { @@ -249,11 +221,11 @@ fn normalize_prepared_blob_lance_field(field: &LanceField) -> Result }) } -pub(crate) fn normalize_prepared_blob_schema(schema: &LanceSchema) -> Result { +pub(crate) fn prepared_to_logical_blob_schema(schema: &LanceSchema) -> Result { let fields = schema .fields .iter() - .map(normalize_prepared_blob_lance_field) + .map(prepared_to_logical_blob_lance_field) .collect::>>()?; Ok(LanceSchema { fields, @@ -268,27 +240,57 @@ pub(crate) struct BlobIdAllocator { #[derive(Debug)] struct BlobIdAllocatorInner { + start_inclusive: u32, next: AtomicU32, - used: Mutex>, + end_exclusive: Option, + state: Mutex, +} + +#[derive(Debug, Default)] +struct BlobIdAllocatorState { + used: HashSet, + allocated: HashSet, } impl BlobIdAllocator { pub(crate) fn new(start: u32) -> Self { Self { inner: Arc::new(BlobIdAllocatorInner { + start_inclusive: start, next: AtomicU32::new(start), - used: Mutex::new(HashSet::new()), + end_exclusive: None, + state: Mutex::new(BlobIdAllocatorState::default()), }), } } + pub(crate) fn from_range(range: Range) -> Result { + if range.start == 0 || range.start >= range.end { + return Err(Error::invalid_input(format!( + "Blob ID range must be non-empty and start at 1 or greater, got {}..{}", + range.start, range.end + ))); + } + Ok(Self { + inner: Arc::new(BlobIdAllocatorInner { + start_inclusive: range.start, + next: AtomicU32::new(range.start), + end_exclusive: Some(range.end), + state: Mutex::new(BlobIdAllocatorState::default()), + }), + }) + } + pub(crate) fn next(&self) -> Result { loop { let id = self.inner.next.load(Ordering::Relaxed); - if id == u32::MAX { - return Err(Error::invalid_input( - "Blob id allocator exhausted u32 id space", - )); + if id == u32::MAX || self.inner.end_exclusive.is_some_and(|end| id >= end) { + return Err(Error::invalid_input(match self.inner.end_exclusive { + Some(end) => format!( + "Blob ID range exhausted before allocating another sidecar; range ends at {end}" + ), + None => "Blob id allocator exhausted u32 id space".to_string(), + })); } if self .inner @@ -298,15 +300,59 @@ impl BlobIdAllocator { { continue; } - let mut used = - self.inner.used.lock().map_err(|_| { + let mut state = + self.inner.state.lock().map_err(|_| { Error::internal("Blob id allocator mutex was poisoned".to_string()) })?; - if used.insert(id) { + if state.used.insert(id) { + state.allocated.insert(id); return Ok(id); } } } + + pub(crate) fn reserve(&self, id: u32) -> Result<()> { + if id < self.inner.start_inclusive || self.inner.end_exclusive.is_some_and(|end| id >= end) + { + return Err(Error::invalid_input(match self.inner.end_exclusive { + Some(end) => format!( + "Blob ID {id} is outside allocator range {}..{end}", + self.inner.start_inclusive + ), + None => format!( + "Blob ID {id} is below allocator start {}", + self.inner.start_inclusive + ), + })); + } + let mut state = self + .inner + .state + .lock() + .map_err(|_| Error::internal("Blob id allocator mutex was poisoned".to_string()))?; + if state.allocated.contains(&id) { + return Err(Error::invalid_input(format!( + "Blob ID {id} was already allocated for a generated sidecar" + ))); + } + state.used.insert(id); + Ok(()) + } + + #[cfg(test)] + pub(crate) fn allocated_ids(&self) -> Result> { + let mut ids = self + .inner + .state + .lock() + .map_err(|_| Error::internal("Blob id allocator mutex was poisoned".to_string()))? + .allocated + .iter() + .copied() + .collect::>(); + ids.sort_unstable(); + Ok(ids) + } } fn validate_blob_id(blob_id: u32) -> Result<()> { @@ -331,14 +377,23 @@ fn validate_range(offset: u64, size: u64, object_size: u64, label: &str) -> Resu } fn validate_prepared_blob_value_array(field: &Field, array: &ArrayRef) -> Result<()> { - if !is_prepared_blob_v2_field(field) { - return Err(blob_v2_shape_error(field)); + if blob_v2_layout(field) != Some(BlobV2Layout::Prepared) { + return Err(blob_v2_shape_error(field, &[BlobV2Layout::Prepared])); } let struct_arr = array .as_any() .downcast_ref::() .ok_or_else(|| Error::invalid_input("Prepared blob column was not a struct array"))?; + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Prepared) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input(format!( + "Prepared blob column '{}' has {actual} array layout; expected prepared layout", + field.name() + ))); + } let kind_col = struct_arr .column_by_name("kind") .ok_or_else(|| Error::invalid_input("Prepared blob struct missing `kind` field"))? @@ -465,7 +520,7 @@ pub struct BlobRange { pub size: u64, } -/// A physical blob descriptor row. +/// A kind-aware row used to build the writer-prepared blob representation. #[derive(Clone, Debug, PartialEq, Eq)] pub enum BlobDescriptor { /// A null blob row. @@ -489,19 +544,22 @@ pub enum BlobDescriptor { }, } -/// A physical blob descriptor column ready to be included in a [`RecordBatch`](arrow_array::RecordBatch). +/// A writer-prepared blob column ready to be included in a [`RecordBatch`](arrow_array::RecordBatch). +/// +/// Despite the legacy type name, its Arrow representation is +/// [`BlobV2Layout::Prepared`], not the descriptor stored in a Lance file. pub struct BlobDescriptorColumn { field: Field, array: ArrayRef, } impl BlobDescriptorColumn { - /// Return the Arrow field for the descriptor column. + /// Return the Arrow field for the prepared column. pub fn field(&self) -> &Field { &self.field } - /// Return the Arrow array for the descriptor column. + /// Return the Arrow array for the prepared column. pub fn array(&self) -> &ArrayRef { &self.array } @@ -512,9 +570,9 @@ impl BlobDescriptorColumn { } } -/// Builds physical blob descriptors for one blob v2 column. +/// Builds the writer-prepared representation for one blob v2 column. /// -/// This builder only produces the writer-side descriptor struct array. It does not allocate blob ids, +/// This builder only produces the writer-prepared struct array. It does not allocate blob ids, /// choose sidecar paths, write blob objects, or commit data files. pub struct BlobDescriptorArrayBuilder { field: Field, @@ -606,12 +664,12 @@ impl BlobDescriptorArrayBuilder { self.push(BlobDescriptor::Null) } - /// Return the descriptor Arrow field for this blob column. + /// Return the prepared Arrow field for this blob column. pub fn field(&self) -> &Field { &self.field } - /// Finish this column and return the writer-side descriptor struct array. + /// Finish this column and return the writer-prepared struct array. pub fn finish(self) -> Result { let mut kind_builder = PrimitiveBuilder::::with_capacity(self.values.len()); let mut data_builder = LargeBinaryBuilder::with_capacity(self.values.len(), 0); @@ -682,7 +740,7 @@ impl BlobDescriptorArrayBuilder { } let array = Arc::new(StructArray::try_new( - prepared_blob_child_fields(), + BLOB_V2_PREPARED_FIELDS.clone(), vec![ Arc::new(kind_builder.finish()), Arc::new(data_builder.finish()), @@ -736,12 +794,28 @@ fn validate_blob_descriptor(value: &BlobDescriptor) -> Result<()> { } } +fn packed_descriptor(blob_id: u32, offset: u64, size: u64) -> Result<(BlobDescriptor, u64)> { + let next_offset = offset.checked_add(size).ok_or_else(|| { + Error::invalid_input(format!( + "Packed blob writer offset overflowed: offset={offset}, size={size}" + )) + })?; + Ok(( + BlobDescriptor::Packed { + blob_id, + offset, + size, + }, + next_offset, + )) +} + /// Writes a Lance-owned packed sidecar blob for one data file and returns descriptors. pub struct PackedBlobWriter { object_store: ObjectStore, path: Path, blob_id: u32, - writer: Box, + writer: Option>, offset: u64, values: Vec, } @@ -759,7 +833,7 @@ impl PackedBlobWriter { object_store, path, blob_id, - writer, + writer: Some(writer), offset: 0, values: Vec::new(), }) @@ -782,10 +856,60 @@ impl PackedBlobWriter { Ok(()) } + /// Append multiple logical blobs, one per iterator item. + /// + /// Each `Some(bytes)` is appended to the sidecar and records a packed + /// descriptor; an empty slice records a valid zero-length blob. Each `None` + /// records a [`BlobDescriptor::Null`] without writing any bytes, so the + /// descriptors returned by [`Self::finish`] stay row-aligned with the input. + /// + /// If writing fails or the future is cancelled after a partial write, no + /// descriptors from this call are recorded, the active writer is dropped, + /// and this instance cannot be reused. + /// + /// ``` + /// # use lance::{PackedBlobWriter, Result}; + /// # async fn write(mut writer: PackedBlobWriter) -> Result<()> { + /// writer + /// .write_packed_blobs([Some(b"first".as_slice()), None, Some(b"second".as_slice())]) + /// .await?; + /// let descriptors = writer.finish().await?; + /// assert_eq!(descriptors.len(), 3); + /// # Ok(()) + /// # } + /// ``` + pub async fn write_packed_blobs<'a>( + &mut self, + blobs: impl IntoIterator>, + ) -> Result<()> { + let mut writer = self.take_writer()?; + let mut descriptors = Vec::new(); + let mut next_offset = self.offset; + for blob in blobs { + let Some(blob) = blob else { + descriptors.push(BlobDescriptor::Null); + continue; + }; + let (descriptor, following_offset) = + packed_descriptor(self.blob_id, next_offset, blob.len() as u64)?; + if !blob.is_empty() { + writer.write_all(blob).await?; + } + descriptors.push(descriptor); + next_offset = following_offset; + } + self.writer = Some(writer); + self.offset = next_offset; + self.values.extend(descriptors); + Ok(()) + } + pub(crate) async fn write_blob_bytes(&mut self, bytes: &[u8]) -> Result { let size = bytes.len() as u64; let offset = self.offset; - self.writer.write_all(bytes).await?; + let mut writer = self.take_writer()?; + writer.write_all(bytes).await?; + self.writer = Some(writer); self.record_written_blob(offset, size) } @@ -796,28 +920,32 @@ impl PackedBlobWriter { ) -> Result { let size = range.len() as u64; let offset = self.offset; - self.writer.copy_range_from_reader(reader, range).await?; + let mut writer = self.take_writer()?; + writer.copy_range_from_reader(reader, range).await?; + self.writer = Some(writer); self.record_written_blob(offset, size) } fn record_written_blob(&mut self, offset: u64, size: u64) -> Result { - self.offset = self.offset.checked_add(size).ok_or_else(|| { - Error::invalid_input(format!( - "Packed blob writer offset overflowed: offset={offset}, size={size}" - )) - })?; - let value = BlobDescriptor::Packed { - blob_id: self.blob_id, - offset, - size, - }; + let (value, next_offset) = packed_descriptor(self.blob_id, offset, size)?; + self.offset = next_offset; self.values.push(value.clone()); Ok(value) } + fn take_writer(&mut self) -> Result> { + self.writer.take().ok_or_else(|| { + Error::io(format!( + "Packed blob writer for '{}' has no active upload", + self.path + )) + }) + } + /// Finish the packed sidecar and return descriptors in write order. pub async fn finish(mut self) -> Result> { - Writer::shutdown(self.writer.as_mut()).await?; + let mut writer = self.take_writer()?; + Writer::shutdown(writer.as_mut()).await?; let object_size = self.object_store.size(&self.path).await?; validate_range(0, self.offset, object_size, "Packed blob")?; Ok(self.values) @@ -987,11 +1115,7 @@ impl BlobArrayBuilder { let validity = self.validity.finish(); let struct_array = StructArray::try_new( - vec![ - Field::new("data", DataType::LargeBinary, true), - Field::new("uri", DataType::Utf8, true), - ] - .into(), + BLOB_V2_LOGICAL_MINIMAL_FIELDS.clone(), vec![data as ArrayRef, uri as ArrayRef], validity, )?; @@ -1010,13 +1134,132 @@ impl BlobArrayBuilder { #[cfg(test)] mod tests { + use std::future::Future; + use std::io; use std::num::NonZeroUsize; + use std::pin::Pin; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::task::{Context, Poll}; use super::*; use arrow_array::cast::AsArray; use arrow_array::{Array, StringArray}; - use arrow_schema::Schema as ArrowSchema; + use arrow_schema::{Fields, Schema as ArrowSchema}; + use async_trait::async_trait; + use futures::task::noop_waker; + use lance_core::datatypes::{BLOB_V2_DESC_FIELDS, BLOB_V2_LOGICAL_FIELDS}; use lance_core::utils::tempfile::TempDir; + use lance_io::object_writer::WriteResult; + use rstest::rstest; + use tokio::io::AsyncWrite; + + #[derive(Clone, Copy)] + enum WriteTerminal { + Error, + Pending, + } + + struct PartialWriter { + bytes_before_terminal: usize, + terminal: WriteTerminal, + bytes_written: Arc, + dropped: Arc, + } + + impl AsyncWrite for PartialWriter { + fn poll_write( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + bytes: &[u8], + ) -> Poll> { + if self.bytes_before_terminal > 0 { + let written = self.bytes_before_terminal.min(bytes.len()); + self.bytes_before_terminal -= written; + self.bytes_written.fetch_add(written, Ordering::SeqCst); + return Poll::Ready(Ok(written)); + } + match self.terminal { + WriteTerminal::Error => { + Poll::Ready(Err(io::Error::other("injected write failure"))) + } + WriteTerminal::Pending => Poll::Pending, + } + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + + #[async_trait] + impl Writer for PartialWriter { + async fn tell(&mut self) -> Result { + Ok(self.bytes_written.load(Ordering::SeqCst)) + } + + async fn shutdown(&mut self) -> Result { + Ok(WriteResult::default()) + } + } + + impl Drop for PartialWriter { + fn drop(&mut self) { + self.dropped.store(true, Ordering::SeqCst); + } + } + + fn partial_writer( + terminal: WriteTerminal, + ) -> (Box, Arc, Arc) { + let bytes_written = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicBool::new(false)); + ( + Box::new(PartialWriter { + bytes_before_terminal: 2, + terminal, + bytes_written: bytes_written.clone(), + dropped: dropped.clone(), + }), + bytes_written, + dropped, + ) + } + + fn cancel_pending(future: F) { + let mut future = Box::pin(future); + let waker = noop_waker(); + let mut context = Context::from_waker(&waker); + assert!(future.as_mut().poll(&mut context).is_pending()); + } + + #[test] + fn part_blob_id_allocator_stops_at_lease_end() { + let allocator = BlobIdAllocator::from_range(7..8).unwrap(); + assert_eq!(allocator.next().unwrap(), 7); + let error = allocator.next().unwrap_err(); + assert!(error.to_string().contains("range ends at 8"), "{error}"); + assert_eq!(allocator.allocated_ids().unwrap(), vec![7]); + + let allocator = BlobIdAllocator::from_range(7..9).unwrap(); + allocator.reserve(7).unwrap(); + assert_eq!(allocator.next().unwrap(), 8); + assert_eq!(allocator.allocated_ids().unwrap(), vec![8]); + } + + #[test] + fn part_blob_id_allocator_rejects_generated_id_reservations() { + let allocator = BlobIdAllocator::from_range(7..9).unwrap(); + assert_eq!(allocator.next().unwrap(), 7); + let error = allocator.reserve(7).unwrap_err(); + assert!(error.to_string().contains("already allocated"), "{error}"); + + allocator.reserve(8).unwrap(); + allocator.reserve(8).unwrap(); + } #[test] fn test_field_metadata() { @@ -1103,7 +1346,7 @@ mod tests { writer.push_null().unwrap(); let column = writer.finish().unwrap(); - assert!(is_prepared_blob_v2_field(column.field())); + assert_eq!(blob_v2_layout(column.field()), Some(BlobV2Layout::Prepared)); let struct_arr = column.array().as_struct(); let kinds = struct_arr .column_by_name("kind") @@ -1145,7 +1388,7 @@ mod tests { ] { let array = Arc::new( StructArray::try_new( - prepared_blob_child_fields(), + BLOB_V2_PREPARED_FIELDS.clone(), vec![ Arc::new(arrow_array::UInt8Array::from(vec![kind as u8])) as ArrayRef, Arc::new(arrow_array::LargeBinaryArray::from_iter([None::<&[u8]>])), @@ -1164,8 +1407,97 @@ mod tests { } } + #[derive(Clone, Copy)] + enum LogicalBlobShape { + Minimal, + CompleteNullableRange, + CompleteRequiredRange, + } + + fn blob_v2_field_with_children(name: &str, children: Fields, nullable: bool) -> Field { + let mut metadata = HashMap::new(); + metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string()); + metadata.insert("blob-root-metadata".to_string(), "preserved".to_string()); + Field::new(name, DataType::Struct(children), nullable).with_metadata(metadata) + } + + fn logical_blob_fields(shape: LogicalBlobShape) -> Fields { + let source = match shape { + LogicalBlobShape::Minimal => &*BLOB_V2_LOGICAL_MINIMAL_FIELDS, + LogicalBlobShape::CompleteNullableRange | LogicalBlobShape::CompleteRequiredRange => { + &*BLOB_V2_LOGICAL_FIELDS + } + }; + source + .iter() + .enumerate() + .map(|(index, field)| { + let nullable = !matches!(shape, LogicalBlobShape::CompleteRequiredRange) + || index < BLOB_V2_LOGICAL_MINIMAL_FIELDS.len(); + Arc::new( + field + .as_ref() + .clone() + .with_nullable(nullable) + .with_metadata(HashMap::from([( + "blob-child-metadata".to_string(), + field.name().to_string(), + )])), + ) + }) + .collect::>() + .into() + } + + fn lance_schema_with_metadata(fields: Vec) -> LanceSchema { + let arrow_schema = ArrowSchema::new_with_metadata( + fields, + HashMap::from([("schema-metadata".to_string(), "preserved".to_string())]), + ); + let mut schema = LanceSchema::try_from(&arrow_schema).unwrap(); + schema.set_field_id(None); + schema + } + + #[rstest] + #[case::minimal(LogicalBlobShape::Minimal)] + #[case::complete_nullable_range(LogicalBlobShape::CompleteNullableRange)] + #[case::complete_required_range(LogicalBlobShape::CompleteRequiredRange)] + fn test_logical_blob_schema_normalization_is_identity(#[case] shape: LogicalBlobShape) { + let blob_field = blob_v2_field_with_children("blob", logical_blob_fields(shape), false); + let schema = lance_schema_with_metadata(vec![blob_field]); + + let normalized = prepared_to_logical_blob_schema(&schema).unwrap(); + + assert_eq!(normalized.fields, schema.fields); + assert_eq!(normalized.metadata, schema.metadata); + } + #[test] - fn test_normalize_prepared_blob_schema_preserves_non_blob_fields() { + fn test_nested_logical_blob_schema_normalization_is_identity() { + let struct_blob = blob_v2_field_with_children( + "struct_blob", + logical_blob_fields(LogicalBlobShape::CompleteNullableRange), + true, + ); + let list_blob = blob_v2_field_with_children( + "item", + logical_blob_fields(LogicalBlobShape::CompleteRequiredRange), + false, + ); + let schema = lance_schema_with_metadata(vec![ + Field::new("payload", DataType::Struct(vec![struct_blob].into()), true), + Field::new("items", DataType::List(Arc::new(list_blob)), false), + ]); + + let normalized = prepared_to_logical_blob_schema(&schema).unwrap(); + + assert_eq!(normalized.fields, schema.fields); + assert_eq!(normalized.metadata, schema.metadata); + } + + #[test] + fn test_prepared_to_logical_blob_schema_preserves_non_blob_fields() { let mut metadata = HashMap::new(); metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string()); let prepared_field = prepared_blob_field_with_metadata("blob", true, metadata); @@ -1182,7 +1514,7 @@ mod tests { let dictionary_values = Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef; schema.fields[0].set_dictionary_values(&dictionary_values); - let normalized = normalize_prepared_blob_schema(&schema).unwrap(); + let normalized = prepared_to_logical_blob_schema(&schema).unwrap(); assert_eq!(normalized.fields[0].id, 42); assert_eq!( @@ -1201,6 +1533,72 @@ mod tests { assert!(normalized.fields[1].children[1].id >= 0); } + #[test] + fn test_prepared_blob_schema_normalizes_by_semantic_child_name() { + let mut metadata = HashMap::new(); + metadata.insert(ARROW_EXT_NAME_KEY.to_string(), BLOB_V2_EXT_NAME.to_string()); + metadata.insert("blob-root-metadata".to_string(), "preserved".to_string()); + let prepared_field = prepared_blob_field_with_metadata("blob", false, metadata); + let dict_field = Field::new( + "dict", + DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)), + true, + ); + let mut schema = lance_schema_with_metadata(vec![dict_field, prepared_field]); + schema.fields[0].id = 42; + schema.fields[1].id = 7; + for (child, id) in schema.fields[1].children.iter_mut().zip(70..76) { + child.id = id; + child.parent_id = 7; + } + + let dictionary_values = Arc::new(StringArray::from(vec!["a", "b"])) as ArrayRef; + schema.fields[0].set_dictionary_values(&dictionary_values); + + let normalized = prepared_to_logical_blob_schema(&schema).unwrap(); + + assert_eq!(normalized.metadata, schema.metadata); + assert_eq!(normalized.fields[0], schema.fields[0]); + assert_eq!(normalized.fields[1].id, 7); + assert!(!normalized.fields[1].nullable); + assert_eq!(normalized.fields[1].metadata, schema.fields[1].metadata); + assert_eq!(normalized.fields[1].children.len(), 2); + assert_eq!(normalized.fields[1].children[0].name, "data"); + assert_eq!(normalized.fields[1].children[0].id, 71); + assert_eq!(normalized.fields[1].children[0].parent_id, 7); + assert_eq!(normalized.fields[1].children[1].name, "uri"); + assert_eq!(normalized.fields[1].children[1].id, 72); + assert_eq!(normalized.fields[1].children[1].parent_id, 7); + } + + #[rstest] + #[case::descriptor(BLOB_V2_DESC_FIELDS.clone(), "descriptor layout")] + #[case::malformed( + vec![ + Field::new("data", DataType::LargeBinary, true), + Field::new("uri", DataType::Utf8, true), + Field::new("size", DataType::UInt64, true), + ].into(), + "unrecognized layout" + )] + fn test_non_logical_blob_schema_normalization_is_rejected( + #[case] fields: Fields, + #[case] actual_layout: &str, + ) { + let field = blob_v2_field_with_children("blob", fields, true); + let schema = lance_schema_with_metadata(vec![field]); + + let error = prepared_to_logical_blob_schema(&schema).unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains(actual_layout)); + assert!( + error + .to_string() + .contains("expected logical or prepared layout") + ); + } + #[tokio::test] async fn test_sidecar_writers_return_prepared_values() { let temp_dir = TempDir::default(); @@ -1265,4 +1663,106 @@ mod tests { let column = builder.finish().unwrap(); assert_eq!(column.array().len(), 3); } + + #[tokio::test] + async fn test_packed_blob_writer_bulk_bytes() { + let temp_dir = TempDir::default(); + let data_dir = Path::from_absolute_path(temp_dir.std_path().join("data")).unwrap(); + let data_file_path = data_dir.join("data-file.lance"); + let mut writer = PackedBlobWriter::try_new(ObjectStore::local(), data_file_path, 7) + .await + .unwrap(); + + writer + .write_packed_blobs([ + Some(b"a".as_slice()), + Some(b"".as_slice()), + None, + Some(b"bc".as_slice()), + ]) + .await + .unwrap(); + + assert_eq!( + writer.finish().await.unwrap(), + vec![ + BlobDescriptor::Packed { + blob_id: 7, + offset: 0, + size: 1, + }, + BlobDescriptor::Packed { + blob_id: 7, + offset: 1, + size: 0, + }, + BlobDescriptor::Null, + BlobDescriptor::Packed { + blob_id: 7, + offset: 1, + size: 2, + }, + ] + ); + } + + #[tokio::test] + async fn test_packed_blob_writer_bulk_drops_after_partial_write_error() { + let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Error); + let previous_descriptor = BlobDescriptor::Packed { + blob_id: 7, + offset: 0, + size: 3, + }; + let mut writer = PackedBlobWriter { + object_store: ObjectStore::local(), + path: Path::from("packed.blob"), + blob_id: 7, + writer: Some(partial_writer), + offset: 3, + values: vec![previous_descriptor.clone()], + }; + + let error = writer + .write_packed_blobs([Some(b"abcdef".as_slice())]) + .await + .unwrap_err(); + + assert!(matches!(error, Error::IO { .. })); + assert!(error.to_string().contains("injected write failure")); + assert_eq!(bytes_written.load(Ordering::SeqCst), 2); + assert!(dropped.load(Ordering::SeqCst)); + assert!(writer.writer.is_none()); + assert_eq!(writer.offset, 3); + assert_eq!(writer.values, vec![previous_descriptor]); + let retry_error = writer.write_blob(b"retry").await.unwrap_err(); + assert!(matches!(retry_error, Error::IO { .. })); + assert!(retry_error.to_string().contains("no active upload")); + } + + #[test] + fn test_packed_blob_writer_bulk_drops_if_cancelled() { + let (partial_writer, bytes_written, dropped) = partial_writer(WriteTerminal::Pending); + let previous_descriptor = BlobDescriptor::Packed { + blob_id: 7, + offset: 0, + size: 3, + }; + let mut writer = PackedBlobWriter { + object_store: ObjectStore::local(), + path: Path::from("packed.blob"), + blob_id: 7, + writer: Some(partial_writer), + offset: 3, + values: vec![previous_descriptor.clone()], + }; + + cancel_pending(writer.write_packed_blobs([Some(b"abcdef".as_slice())])); + + assert_eq!(bytes_written.load(Ordering::SeqCst), 2); + assert!(dropped.load(Ordering::SeqCst)); + assert!(writer.writer.is_none()); + assert_eq!(writer.offset, 3); + assert_eq!(writer.values, vec![previous_descriptor]); + } } diff --git a/rust/lance/src/datafusion/dataframe.rs b/rust/lance/src/datafusion/dataframe.rs index 00db9920bf9..87113242a7c 100644 --- a/rust/lance/src/datafusion/dataframe.rs +++ b/rust/lance/src/datafusion/dataframe.rs @@ -1,10 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{ - any::Any, - sync::{Arc, Mutex}, -}; +use std::sync::{Arc, Mutex}; use arrow_schema::{Schema, SchemaRef}; use async_trait::async_trait; @@ -42,6 +39,9 @@ pub struct LanceTableProvider { row_id_idx: Option, row_addr_idx: Option, ordered: bool, + blob_handling: Option, + batch_size: Option, + batch_size_bytes: Option, } impl LanceTableProvider { @@ -72,7 +72,46 @@ impl LanceTableProvider { row_id_idx, row_addr_idx, ordered, + blob_handling: None, + batch_size: None, + batch_size_bytes: None, + } + } + + /// Overrides how blob columns are read during [`TableProvider::scan`]. + /// When unset, the underlying dataset scan uses its default + /// [`BlobHandling`](lance_core::datatypes::BlobHandling) policy. + pub fn with_blob_handling(mut self, handling: lance_core::datatypes::BlobHandling) -> Self { + let converted = self + .dataset + .full_projection() + .with_blob_handling(handling.clone()) + .to_bare_schema(); + let mut full_schema = Schema::from(&converted); + if self.row_id_idx.is_some() { + full_schema = full_schema.try_with_column(ROW_ID_FIELD.clone()).unwrap(); + } + if self.row_addr_idx.is_some() { + full_schema = full_schema.try_with_column(ROW_ADDR_FIELD.clone()).unwrap(); } + self.full_schema = Arc::new(full_schema); + self.blob_handling = Some(handling); + self + } + + /// Overrides the maximum number of rows produced by each dataset scan batch. + /// + /// The batch size must be between 1 and [`u32::MAX`], inclusive. Invalid + /// values are rejected when DataFusion creates the scan plan. + pub fn with_batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = Some(batch_size); + self + } + + /// Overrides the approximate maximum bytes produced by each dataset scan batch. + pub fn with_batch_size_bytes(mut self, batch_size_bytes: u64) -> Self { + self.batch_size_bytes = Some(batch_size_bytes); + self } pub fn dataset(&self) -> Arc { @@ -82,10 +121,6 @@ impl LanceTableProvider { #[async_trait] impl TableProvider for LanceTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.full_schema.clone() } @@ -96,32 +131,34 @@ impl TableProvider for LanceTableProvider { async fn scan( &self, - _state: &dyn Session, + state: &dyn Session, projection: Option<&Vec>, filters: &[Expr], limit: Option, ) -> datafusion::common::Result> { let mut scan = self.dataset.scan(); - match projection { - Some(projection) if projection.is_empty() => { - scan.empty_project()?; - } - Some(projection) => { - let mut columns = Vec::with_capacity(projection.len()); - for field_idx in projection { - if Some(*field_idx) == self.row_id_idx { - scan.with_row_id(); - } else if Some(*field_idx) == self.row_addr_idx { - scan.with_row_address(); - } else { - columns.push(self.full_schema.field(*field_idx).name()); - } - } - if !columns.is_empty() { - scan.project(&columns)?; + if let Some(handling) = self.blob_handling.clone() { + scan.blob_handling(handling); + } + if let Some(batch_size) = self.batch_size { + scan.batch_size(batch_size); + } + if let Some(batch_size_bytes) = self.batch_size_bytes { + scan.batch_size_bytes(batch_size_bytes); + } + + if let Some(projection) = projection { + let mut columns = Vec::with_capacity(projection.len()); + for field_idx in projection { + if Some(*field_idx) == self.row_id_idx { + scan.with_row_id(); + } else if Some(*field_idx) == self.row_addr_idx { + scan.with_row_address(); + } else { + columns.push(self.full_schema.field(*field_idx).name()); } } - _ => {} + scan.project(&columns)?; } let combined_filter = match filters.len() { @@ -141,7 +178,9 @@ impl TableProvider for LanceTableProvider { scan.limit(limit.map(|l| l as i64), None)?; scan.scan_in_order(self.ordered); - scan.create_plan().await.map_err(DataFusionError::from) + scan.create_plan_with_session(state) + .await + .map_err(DataFusionError::from) } // Since we are using datafusion itself to apply the filters it should @@ -166,13 +205,7 @@ pub trait SessionContextExt { with_row_id: bool, with_row_addr: bool, ) -> datafusion::common::Result; - /// Creates a DataFrame for reading a Lance dataset without ordering - fn read_lance_unordered( - &self, - dataset: Arc, - with_row_id: bool, - with_row_addr: bool, - ) -> datafusion::common::Result; + /// Creates a DataFrame for reading a stream of data /// /// This dataframe may only be queried once, future queries will fail @@ -232,20 +265,6 @@ impl SessionContextExt for SessionContext { ))) } - fn read_lance_unordered( - &self, - dataset: Arc, - with_row_id: bool, - with_row_addr: bool, - ) -> datafusion::common::Result { - self.read_table(Arc::new(LanceTableProvider::new_with_ordering( - dataset, - with_row_id, - with_row_addr, - false, - ))) - } - fn read_one_shot( &self, data: SendableRecordBatchStream, @@ -309,4 +328,39 @@ mod tests { // SUM(0..100) - SUM(0..50) = 3675 assert_eq!(results.column(0).as_primitive::().value(0), 3675); } + + #[tokio::test] + async fn test_table_provider_rejects_invalid_batch_size() { + let data = Arc::new( + lance_datagen::gen_batch() + .col("x", array::step::()) + .into_dataset( + "memory://test_table_provider_rejects_invalid_batch_size", + FragmentCount::from(1), + FragmentRowCount::from(3), + ) + .await + .unwrap(), + ); + + for batch_size in [0, u32::MAX as usize + 1] { + let provider = + LanceTableProvider::new(data.clone(), false, false).with_batch_size(batch_size); + let ctx = SessionContext::new(); + ctx.register_table("dataset", Arc::new(provider)).unwrap(); + + let error = ctx + .sql("SELECT x FROM dataset") + .await + .unwrap() + .collect() + .await + .expect_err("invalid batch size should be rejected"); + assert!( + error + .to_string() + .contains(&format!("batch_size must be between 1 and {}", u32::MAX)) + ); + } + } } diff --git a/rust/lance/src/datafusion/logical_plan.rs b/rust/lance/src/datafusion/logical_plan.rs index a9fe0ed7750..039aa75864f 100644 --- a/rust/lance/src/datafusion/logical_plan.rs +++ b/rust/lance/src/datafusion/logical_plan.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::{any::Any, borrow::Cow, sync::Arc}; +use std::{borrow::Cow, sync::Arc}; use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; @@ -19,10 +19,6 @@ use crate::Dataset; #[async_trait] impl TableProvider for Dataset { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> Arc { Arc::new(self.schema().into()) } @@ -167,17 +163,11 @@ mod tests { // DataFusion will create a cooperative execution plan, so we need to get its inner plan let physical_plan = physical_plan - .as_any() .downcast_ref::() .unwrap() .children()[0]; - assert!( - physical_plan - .as_any() - .downcast_ref::() - .is_some() - ); + assert!(physical_plan.downcast_ref::().is_some()); let expected_fields = schema .fields() diff --git a/rust/lance/src/dataset.rs b/rust/lance/src/dataset.rs index 470c6873dc7..ef025388d9d 100644 --- a/rust/lance/src/dataset.rs +++ b/rust/lance/src/dataset.rs @@ -5,7 +5,6 @@ //! use arrow_array::{RecordBatch, RecordBatchReader}; -use arrow_schema::DataType; use byteorder::{ByteOrder, LittleEndian}; use chrono::{Duration, prelude::*}; use futures::future::BoxFuture; @@ -15,7 +14,6 @@ use lance_core::deepsize::DeepSizeOf; use crate::dataset::metadata::UpdateFieldMetadataBuilder; use crate::dataset::transaction::translate_schema_metadata_updates; -use crate::index::DatasetIndexExt; use crate::session::caches::{DSMetadataCache, ManifestKey, TransactionKey}; use crate::session::index_caches::DSIndexCache; use itertools::Itertools; @@ -27,9 +25,8 @@ use lance_core::utils::tracing::{ DATASET_DELETING_EVENT, DATASET_DROPPING_COLUMN_EVENT, TRACE_DATASET_EVENTS, }; use lance_datafusion::projection::ProjectionPlan; -use lance_file::datatypes::populate_schema_dictionary; use lance_file::reader::{FileReader, FileReaderOptions}; -use lance_file::version::LanceFileVersion; +use lance_file::versions as file_versions; use lance_index::{IndexType, progress::IndexBuildProgress}; use lance_io::object_store::{ ChainedWrappingObjectStore, LanceNamespaceStorageOptionsProvider, ObjectStore, @@ -42,8 +39,8 @@ use lance_io::utils::{ }; use lance_namespace::LanceNamespace; use lance_table::format::{ - DataFile, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, MAGIC, Manifest, RowIdMeta, - pb, + DataFile, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, MAGIC, Manifest, + ManifestBuildConfig, RowIdMeta, pb, populate_manifest_schema_dictionaries, }; use lance_table::io::commit::{ CommitConfig, CommitError, CommitHandler, CommitLock, ManifestLocation, ManifestNamingScheme, @@ -60,18 +57,19 @@ use roaring::RoaringBitmap; use rowids::get_row_id_index; use serde::{Deserialize, Serialize}; use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fmt::Debug; use std::num::NonZero; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; -use tracing::{info, instrument}; +use tracing::{info, instrument, warn}; pub(crate) mod blob; pub(crate) mod branch_location; pub mod builder; pub mod cleanup; +mod data_file; pub mod delta; pub mod files; pub mod fragment; @@ -80,20 +78,45 @@ pub mod index; pub mod mem_wal; mod metadata; pub mod optimize; +pub(crate) mod overlay; pub mod progress; pub mod refs; -pub(crate) mod rowids; +pub mod rowids; pub mod scanner; mod schema_evolution; pub mod sql; pub mod statistics; mod take; -pub mod transaction; +/// Transaction definitions for updating datasets +/// +/// Prior to creating a new manifest, a transaction must be created representing +/// the changes being made to the dataset. By representing them as incremental +/// changes, we can detect whether concurrent operations are compatible with +/// one another. We can also rebuild manifests when retrying committing a +/// manifest. +/// +/// The definitions live in [`lance_table::transaction`]: building a manifest from +/// a transaction reads and writes only table metadata, so it belongs at the table +/// layer. This module re-exports them at the path callers have always used. +/// +/// For more details please refer to the +/// [Transaction Specification](https://lance.org/format/table/transaction/#transaction-types). +pub mod transaction { + pub use lance_table::transaction::{ + DataOverlayGroup, DataReplacementGroup, Operation, ReadVersionState, RewriteGroup, + RewrittenIndex, Transaction, TransactionBuilder, UpdateMap, UpdateMapEntry, UpdateMode, + UpdatedFragmentOffsets, translate_config_updates, translate_schema_metadata_updates, + validate_operation, + }; +} pub mod udtf; pub mod updater; mod utils; +pub(crate) mod versions; pub mod write; +pub use data_file::{DataFilePart, DataFileTarget}; + pub(crate) use take::row_offsets_to_row_addresses; use self::builder::DatasetBuilder; @@ -103,36 +126,42 @@ use self::refs::Refs; use self::scanner::{DatasetRecordBatchStream, Scanner}; use self::statistics::DatasetStatistics; use self::transaction::{Operation, Transaction, TransactionBuilder, UpdateMapEntry}; -use self::write::{cleanup_data_fragments, write_fragments_internal}; +use self::write::cleanup_data_fragments; use crate::dataset::branch_location::BranchLocation; use crate::dataset::cleanup::{CleanupOperation, CleanupPolicy, CleanupPolicyBuilder}; use crate::dataset::refs::{BranchContents, BranchIdentifier, Branches, Tags}; use crate::dataset::sql::SqlQueryBuilder; use crate::datatypes::Schema; -use crate::index::retain_supported_indices; use crate::io::commit::{ - commit_detached_transaction, commit_new_dataset, commit_transaction, - detect_overlapping_fragments, + DEFAULT_COMMIT_RETRY_TIMEOUT, commit_detached_transaction, commit_new_dataset, + commit_transaction, detect_overlapping_fragments, }; use crate::session::Session; use crate::utils::temporal::{SystemTime, timestamp_to_nanos, utc_now}; use crate::{Error, Result}; -pub use blob::{BlobFile, ReadBlob, ReadBlobsBuilder, ReadBlobsStream}; +pub use blob::{ + BlobFile, BlobRangeRequest, BlobReadRange, ReadBlob, ReadBlobRange, ReadBlobRangesBuilder, + ReadBlobRangesStream, ReadBlobsBuilder, ReadBlobsStream, +}; use hash_joiner::HashJoiner; pub use lance_core::ROW_ID; use lance_core::box_error; use lance_index::scalar::lance_format::LanceIndexStore; use lance_namespace::models::{DeclareTableRequest, DescribeTableRequest}; -use lance_table::feature_flags::{apply_feature_flags, can_read_dataset}; +use lance_table::feature_flags::{ + apply_feature_flags, ensure_can_read_manifest, ensure_can_write_manifest, + validate_paired_feature_flags, +}; use lance_table::io::deletion::{DELETIONS_DIR, relative_deletion_file_path}; +use lance_table::rowids::{RowIdSequence, write_row_ids}; pub use schema_evolution::{ BatchInfo, BatchUDF, ColumnAlteration, NewColumnTransform, UDFCheckpointStore, }; pub use take::TakeBuilder; use uuid::Uuid; pub use write::merge_insert::{ - MergeInsertBuilder, MergeInsertJob, MergeStats, UncommittedMergeInsert, WhenMatched, - WhenNotMatched, WhenNotMatchedBySource, + MergeInsertBuilder, MergeInsertJob, MergeInsertWriteMode, MergeStats, UncommittedMergeInsert, + WhenMatched, WhenNotMatched, WhenNotMatchedBySource, }; use crate::dataset::index::LanceIndexStoreExt; @@ -147,6 +176,32 @@ pub use write::{ pub(crate) const INDICES_DIR: &str = "_indices"; pub(crate) const DATA_DIR: &str = "data"; pub(crate) const TRANSACTIONS_DIR: &str = "_transactions"; +const DEFAULT_MAX_STREAM_COPY_PARALLELISM: usize = 4; + +fn parse_deep_clone_stream_concurrency(value: &str) -> Result { + value + .parse::>() + .map(NonZero::get) + .map_err(|_| { + Error::invalid_input(format!( + "LANCE_DEEP_CLONE_STREAM_CONCURRENCY must be a positive integer, got {value:?}" + )) + }) +} + +fn deep_clone_copy_parallelism( + configured_io_parallelism: usize, + uses_streaming_copy: bool, + stream_copy_parallelism: Option, +) -> usize { + if !uses_streaming_copy { + configured_io_parallelism + } else if let Some(value) = stream_copy_parallelism { + value + } else { + configured_io_parallelism.min(DEFAULT_MAX_STREAM_COPY_PARALLELISM) + } +} // We default to 6GB for the index cache, since indices are often large but // worth caching. @@ -191,8 +246,15 @@ pub struct Dataset { pub(crate) store_params: Option>, /// Optional runtime-only object store parameters keyed by base path URI. pub(crate) base_store_params: Option>>, + /// Object stores for additional base paths, normally shared across clones. + /// Applying new object store wrappers starts a fresh cache scope. + pub(crate) base_object_stores: BaseObjectStores, } +/// The `OnceCell` coalesces concurrent first resolutions into one build. +pub(crate) type BaseObjectStores = + Arc>>>>>; + impl std::fmt::Debug for Dataset { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Dataset") @@ -218,6 +280,14 @@ pub struct Version { pub metadata: BTreeMap, } +/// A lightweight reference to an attached dataset version, which could be used to uniquely identify a version. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct VersionRef { + /// Version number within the current branch's history. + pub version: u64, +} + /// Convert Manifest to Data Version. impl From<&Manifest> for Version { fn from(m: &Manifest) -> Self { @@ -229,6 +299,23 @@ impl From<&Manifest> for Version { } } +/// The transaction that produced a version of the dataset, along with the +/// version's commit timestamp. +/// +/// Returned by [`Dataset::read_version_transaction`], which reads this +/// information directly from storage without checking out the version. +#[derive(Debug, Clone)] +pub struct VersionTransaction { + /// Version number. + pub version: u64, + + /// Timestamp the version was committed, in UTC. + pub timestamp: DateTime, + + /// The transaction that produced this version, if one was recorded. + pub transaction: Option, +} + /// Customize read behavior of a dataset. #[derive(Clone, Debug)] pub struct ReadParams { @@ -468,6 +555,16 @@ impl Dataset { /// Check out the latest version of the dataset pub async fn checkout_latest(&mut self) -> Result<()> { let (manifest, manifest_location) = self.latest_manifest().await?; + self.set_manifest(manifest, manifest_location); + Ok(()) + } + + /// Replace the manifest, refreshing derived state. Base stores are kept + /// when `base_paths` is unchanged. + fn set_manifest(&mut self, manifest: Arc, manifest_location: ManifestLocation) { + if manifest.base_paths != self.manifest.base_paths { + self.base_object_stores = Default::default(); + } self.manifest = manifest; self.manifest_location = manifest_location; self.fragment_bitmap = Arc::new( @@ -477,7 +574,6 @@ impl Dataset { .map(|f| f.id as u32) .collect(), ); - Ok(()) } /// Check out the latest version of the branch @@ -527,7 +623,7 @@ impl Dataset { ) .with_object_store(Arc::new(self.object_store.as_ref().clone())) .with_commit_handler(self.commit_handler.clone()) - .with_storage_format(self.manifest.data_storage_format.lance_file_version()?); + .with_exact_storage_format(self.manifest.data_storage_format.lance_file_format()); let dataset = builder.execute(transaction).await?; // Create BranchContents after shallow_clone @@ -552,8 +648,10 @@ impl Dataset { } fn already_checked_out(&self, location: &ManifestLocation, branch_name: Option<&str>) -> bool { - // We check the e_tag here just in case it has been overwritten. This can - // happen if the table has been dropped then re-created recently. + // The ETag is an opaque object-generation token, not a content hash. + // Comparing the token still prevents reusing this Dataset's manifest + // after the physical object was replaced, for example by a recent + // drop/recreate at the same URI and version. self.manifest.branch.as_deref() == branch_name && self.manifest.version == location.version && self.manifest_location.naming_scheme == location.naming_scheme @@ -599,7 +697,7 @@ impl Dataset { return Ok(self.clone()); } - let manifest = Self::load_manifest( + let manifest = Self::get_manifest( self.object_store.as_ref(), &manifest_location, &new_location.uri, @@ -625,7 +723,7 @@ impl Dataset { self.object_store.clone(), new_location.path, new_location.uri, - Arc::new(manifest), + manifest, manifest_location, self.session.clone(), self.commit_handler.clone(), @@ -695,14 +793,7 @@ impl Dataset { read_struct(object_reader.as_ref(), offset).await }?; - if !can_read_dataset(manifest.reader_feature_flags) { - let message = format!( - "This dataset cannot be read by this version of Lance. \ - Please upgrade Lance to read this dataset.\n Flags: {}", - manifest.reader_feature_flags - ); - return Err(Error::not_supported_source(message.into())); - } + ensure_can_read_manifest(&manifest)?; // If indices were also in the last block, we can take the opportunity to // decode them now and cache them. @@ -714,15 +805,21 @@ impl Dataset { LittleEndian::read_u32(&last_block[offset_in_block..offset_in_block + 4]) as usize; let message_data = &last_block[offset_in_block + 4..offset_in_block + 4 + message_len]; let section = lance_table::format::pb::IndexSection::decode(message_data)?; - let mut indices: Vec = section + // Cached unfiltered: this is the same cache the commit path reads + // from, and an index this build cannot decode still has to survive + // into the next manifest. Version filtering happens on the way out, + // in `DatasetIndexExt::load_indices`. + let indices: Vec = section .indices .into_iter() .map(IndexMetadata::try_from) .collect::>>()?; - retain_supported_indices(&mut indices); + crate::index::warn_about_unsupported_indices(&indices); let ds_index_cache = session.index_cache.for_dataset(uri); let metadata_key = crate::session::index_caches::IndexMetadataKey { version: manifest_location.version, + store_identity: &object_store.store_prefix, + e_tag: manifest_location.e_tag.as_deref(), }; ds_index_cache .insert_with_key(&metadata_key, Arc::new(indices)) @@ -738,21 +835,20 @@ impl Dataset { let message_len = LittleEndian::read_u32(&last_block[offset_in_block..offset_in_block + 4]) as usize; let message_data = &last_block[offset_in_block + 4..offset_in_block + 4 + message_len]; - let transaction: Transaction = - lance_table::format::pb::Transaction::decode(message_data)?.try_into()?; - - let metadata_cache = session.metadata_cache.for_dataset(uri); - let metadata_key = TransactionKey { - version: manifest_location.version, - }; - metadata_cache - .insert_with_key(&metadata_key, Arc::new(transaction)) - .await; + if let Some(transaction) = + decode_inline_transaction(message_data, manifest_location.version) + { + let metadata_cache = session.metadata_cache.for_dataset(uri); + let metadata_key = TransactionKey { + version: manifest_location.version, + }; + metadata_cache + .insert_with_key(&metadata_key, Arc::new(transaction)) + .await; + } } - if manifest.should_use_legacy_format() { - populate_schema_dictionary(&mut manifest.schema, object_reader.as_ref()).await?; - } + populate_manifest_schema_dictionaries(&mut manifest, object_reader.as_ref()).await?; Ok(manifest) } @@ -765,12 +861,18 @@ impl Dataset { uri: &str, session: &Session, ) -> Result> { + if manifest_location.size.is_none() { + return Ok(Arc::new( + Self::load_manifest(object_store, manifest_location, uri, session).await?, + )); + } let metadata_cache = session.metadata_cache.for_dataset(uri); let manifest_key = ManifestKey { version: manifest_location.version, e_tag: manifest_location.e_tag.as_deref(), }; if let Some(cached) = metadata_cache.get_with_key(&manifest_key).await { + ensure_can_read_manifest(&cached)?; return Ok(cached); } let loaded = @@ -826,6 +928,7 @@ impl Dataset { file_reader_options, store_params: store_params.map(Box::new), base_store_params, + base_object_stores: Default::default(), }) } @@ -1123,50 +1226,19 @@ impl Dataset { delta::DatasetDeltaBuilder::new(self.clone()) } - // TODO: Cache this - pub(crate) fn is_legacy_storage(&self) -> bool { - self.manifest - .data_storage_format - .lance_file_version() - .unwrap() - == LanceFileVersion::Legacy - } - pub async fn latest_manifest(&self) -> Result<(Arc, ManifestLocation)> { let location = self .commit_handler .resolve_latest_location(&self.base, &self.object_store) .await?; - // Check if manifest is in cache before reading from storage - let manifest_key = ManifestKey { - version: location.version, - e_tag: location.e_tag.as_deref(), - }; - let cached_manifest = self.metadata_cache.get_with_key(&manifest_key).await; - if let Some(cached_manifest) = cached_manifest { - return Ok((cached_manifest, location)); - } - if self.already_checked_out(&location, self.manifest.branch.as_deref()) { + ensure_can_read_manifest(&self.manifest)?; return Ok((self.manifest.clone(), self.manifest_location.clone())); } - let mut manifest = read_manifest(&self.object_store, &location.path, location.size).await?; - if manifest.schema.has_dictionary_types() && manifest.should_use_legacy_format() { - let reader = if let Some(size) = location.size { - self.object_store - .open_with_size(&location.path, size as usize) - .await? - } else { - self.object_store.open(&location.path).await? - }; - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; - } - let manifest_arc = Arc::new(manifest); - self.metadata_cache - .insert_with_key(&manifest_key, manifest_arc.clone()) - .await; - Ok((manifest_arc, location)) + let manifest = + Self::get_manifest(&self.object_store, &location, &self.uri, &self.session).await?; + Ok((manifest, location)) } /// Read the transaction file for this version of the dataset. @@ -1181,43 +1253,146 @@ impl Dataset { return Ok(Some((*transaction).clone())); } + let transaction = self + .read_transaction_from_storage(&self.manifest, &self.manifest_location) + .await?; + + if let Some(tx) = transaction.as_ref() { + self.metadata_cache + .insert_with_key(&transaction_key, Arc::new(tx.clone())) + .await; + } + Ok(transaction) + } + + /// Read the transaction recorded by `manifest` directly from storage, + /// without consulting or populating any session cache. + async fn read_transaction_from_storage( + &self, + manifest: &Manifest, + manifest_location: &ManifestLocation, + ) -> Result> { // Prefer inline transaction from manifest when available - let transaction = if let Some(pos) = self.manifest.transaction_section { - let reader = if let Some(size) = self.manifest_location.size { - self.object_store - .open_with_size(&self.manifest_location.path, size as usize) - .await? - } else { - self.object_store.open(&self.manifest_location.path).await? + if let Some(pos) = manifest.transaction_section { + let reader = match manifest_location.size { + Some(size) => { + self.object_store + .open_with_size(&manifest_location.path, size as usize) + .await? + } + None => self.object_store.open(&manifest_location.path).await?, }; - let tx: pb::Transaction = read_message(reader.as_ref(), pos).await?; - Transaction::try_from(tx).map(Some)? - } else if let Some(path) = &self.manifest.transaction_file { + // A concurrent overwrite can leave the listed size too small; retry + // once with the true size. + let tx: pb::Transaction = match read_message(reader.as_ref(), pos).await { + Err(e) + if manifest_location.size.is_some() + && e.to_string().contains("file size is too small") => + { + let reader = self.object_store.open(&manifest_location.path).await?; + read_message(reader.as_ref(), pos).await? + } + other => other?, + }; + Transaction::try_from(tx).map(Some) + } else if let Some(path) = &manifest.transaction_file { // Fallback: read external transaction file if present let path = self.transactions_dir().join(path.as_str()); let data = self.object_store.inner.get(&path).await?.bytes().await?; let transaction = lance_table::format::pb::Transaction::decode(data)?; - Transaction::try_from(transaction).map(Some)? + Transaction::try_from(transaction).map(Some) } else { - None - }; + Ok(None) + } + } - if let Some(tx) = transaction.as_ref() { - self.metadata_cache - .insert_with_key(&transaction_key, Arc::new(tx.clone())) - .await; + /// Read the transaction (if any) and commit timestamp of a version of the + /// dataset. `version` is a version number on this dataset's current branch. + /// + /// Reads the version's manifest transiently: no historical `Dataset` is + /// constructed, no `IndexSection` is decoded, and no session cache is read + /// or written, so scanning many historical versions does not fill the + /// shared caches. + /// + /// Returns an error if the version does not exist (for example, if it has + /// been cleaned up). + /// + /// # Example + /// + /// ``` + /// # use lance::{Dataset, Result}; + /// # async fn example(dataset: &Dataset) -> Result<()> { + /// let record = dataset.read_version_transaction(5).await?; + /// let committed_at = record.timestamp; + /// let operation = record.transaction.as_ref().map(|t| t.operation.name()); + /// # Ok(()) + /// # } + /// ``` + pub async fn read_version_transaction(&self, version: u64) -> Result { + // Resolve against this dataset's current branch. + let manifest_location = self + .commit_handler + .resolve_version_location(&self.base, version, &self.object_store.inner) + .await?; + + // Keep the DatasetNotFound variant callers expect for a missing version. + let manifest = read_manifest( + &self.object_store, + &manifest_location.path, + manifest_location.size, + ) + .await + .map_err(|e| match &e { + Error::NotFound { uri, .. } => Error::dataset_not_found(uri.clone(), box_error(e)), + _ => e, + })?; + + // The resolved manifest must belong to this dataset's branch. A + // mismatch means the commit handler resolved against a different chain + // (for example an external manifest store that ignores + // branch-qualified paths); error loudly rather than hand back another + // branch's transaction. + if manifest.branch != self.manifest.branch { + return Err(Error::internal(format!( + "reading version {} on branch '{}' resolved a manifest belonging to branch '{}'", + version, + refs::normalize_branch(self.manifest.branch.as_deref()), + refs::normalize_branch(manifest.branch.as_deref()), + ))); } - Ok(transaction) + + let transaction = self + .read_transaction_from_storage(&manifest, &manifest_location) + .await?; + + Ok(VersionTransaction { + version: manifest.version, + timestamp: manifest.timestamp(), + transaction, + }) } /// Read the transaction file for this version of the dataset. /// /// If there was no transaction file written for this version of the dataset /// then this will return None. + /// + /// Does not populate the session caches; see + /// [`Self::read_version_transaction`]. + /// + /// # Example + /// + /// ``` + /// # use lance::{Dataset, Result}; + /// # async fn example(dataset: &Dataset) -> Result<()> { + /// let transaction = dataset.read_transaction_by_version(5).await?; + /// let operation = transaction.as_ref().map(|t| t.operation.name()); + /// # Ok(()) + /// # } + /// ``` pub async fn read_transaction_by_version(&self, version: u64) -> Result> { - let dataset_version = self.checkout_version(version).await?; - dataset_version.read_transaction().await + Ok(self.read_version_transaction(version).await?.transaction) } /// List transactions for the dataset, up to a maximum number. @@ -1489,20 +1664,13 @@ impl Dataset { &transaction, write_config, commit_config, + DEFAULT_COMMIT_RETRY_TIMEOUT, self.manifest_location.naming_scheme, None, ) .await?; - self.manifest = Arc::new(manifest); - self.manifest_location = manifest_location; - self.fragment_bitmap = Arc::new( - self.manifest - .fragments - .iter() - .map(|f| f.id as u32) - .collect(), - ); + self.set_manifest(Arc::new(manifest), manifest_location); Ok(()) } @@ -1604,11 +1772,30 @@ impl Dataset { } /// Take [BlobFile] by row IDs. + /// + /// The returned vector has one element per row ID. Null blob values are + /// represented as `None`; valid empty blobs return a `BlobFile` with size + /// zero. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lance::dataset::Dataset; + /// # use lance::Result; + /// # async fn example(dataset: Arc) -> Result<()> { + /// let blobs = dataset.take_blobs(&[42], "images").await?; + /// match &blobs[0] { + /// None => { /* The selected blob is null. */ } + /// Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ } + /// Some(blob) => { let _size = blob.size(); } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn take_blobs( self: &Arc, row_ids: &[u64], column: impl AsRef, - ) -> Result> { + ) -> Result>> { blob::take_blobs(self, row_ids, column.as_ref()).await } @@ -1618,21 +1805,57 @@ impl Dataset { /// Use this method when you already have row addresses, for example from /// a scan with `with_row_address()`. For row IDs (stable identifiers), use /// [`Self::take_blobs`]. For row indices (offsets), use - /// [`Self::take_blobs_by_indices`]. + /// [`Self::take_blobs_by_indices`]. The result has the same null and empty + /// blob representation as [`Self::take_blobs`]. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lance::dataset::Dataset; + /// # use lance::Result; + /// # async fn example(dataset: Arc, row_address: u64) -> Result<()> { + /// let blobs = dataset + /// .take_blobs_by_addresses(&[row_address], "images") + /// .await?; + /// match &blobs[0] { + /// None => { /* The selected blob is null. */ } + /// Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ } + /// Some(blob) => { let _size = blob.size(); } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn take_blobs_by_addresses( self: &Arc, row_addrs: &[u64], column: impl AsRef, - ) -> Result> { + ) -> Result>> { blob::take_blobs_by_addresses(self, row_addrs, column.as_ref()).await } /// Take [BlobFile] by row indices (offsets in the dataset). + /// + /// The result has the same null and empty blob representation as + /// [`Self::take_blobs`]. + /// + /// ``` + /// # use std::sync::Arc; + /// # use lance::dataset::Dataset; + /// # use lance::Result; + /// # async fn example(dataset: Arc) -> Result<()> { + /// let blobs = dataset.take_blobs_by_indices(&[0], "images").await?; + /// match &blobs[0] { + /// None => { /* The selected blob is null. */ } + /// Some(blob) if blob.size() == 0 => { /* The selected blob is valid but empty. */ } + /// Some(blob) => { let _size = blob.size(); } + /// } + /// # Ok(()) + /// # } + /// ``` pub async fn take_blobs_by_indices( self: &Arc, row_indices: &[u64], column: impl AsRef, - ) -> Result> { + ) -> Result>> { let fragments = self.get_fragments(); let row_addrs = row_offsets_to_row_addresses(&fragments, row_indices).await?; blob::take_blobs_by_addresses(self, &row_addrs, column.as_ref()).await @@ -1643,7 +1866,9 @@ impl Dataset { /// This API complements [`Self::take_blobs`]. `take_blobs` returns /// [`BlobFile`] handles for caller-driven random access, while /// `read_blobs` builds a streaming read plan for sequential or batched blob - /// retrieval. + /// retrieval. Every selected row produces one result: null blob values have + /// `ReadBlob::data` set to `None`, while valid empty blobs contain an empty + /// buffer. /// /// ```rust /// # use std::sync::Arc; @@ -1670,6 +1895,38 @@ impl Dataset { )) } + /// Create a planned reader for row-specific blob-local byte ranges. + /// + /// Each [`BlobRangeRequest`] contains both its row selector and byte range, + /// so requests can be repeated or reordered without coordinating parallel + /// selector and range lists. Every request produces one result. A null blob + /// has `ReadBlobRange::data` set to `None`; an empty range on a non-null blob + /// contains an empty buffer. + /// + /// ```rust + /// # use std::sync::Arc; + /// # use lance::dataset::{BlobRangeRequest, Dataset}; + /// # use lance::Result; + /// # async fn example(dataset: Arc) -> Result<()> { + /// let ranges = dataset + /// .read_blob_ranges("images")? + /// .with_row_indices([ + /// BlobRangeRequest::new(7, 0, 1024), + /// BlobRangeRequest::new(7, 4096, 1024), + /// ]) + /// .execute() + /// .await?; + /// # let _ = ranges; + /// # Ok(()) + /// # } + /// ``` + pub fn read_blob_ranges( + self: &Arc, + column: impl AsRef, + ) -> Result { + Ok(ReadBlobRangesBuilder::new(self.read_blobs(column)?)) + } + /// Get a stream of batches based on iterator of ranges of row numbers. /// /// This is an experimental API. It may change at any time. @@ -1711,26 +1968,7 @@ impl Dataset { )); } - let selected_fragment_ids = fragment_ids.iter().copied().collect::>(); - let selected_fragments = self - .get_fragments() - .into_iter() - .filter(|fragment| selected_fragment_ids.contains(&(fragment.id() as u32))) - .collect::>(); - - if selected_fragments.len() != selected_fragment_ids.len() { - let present_fragment_ids = selected_fragments - .iter() - .map(|fragment| fragment.id() as u32) - .collect::>(); - let missing_fragment_ids = selected_fragment_ids - .into_iter() - .filter(|fragment_id| !present_fragment_ids.contains(fragment_id)) - .collect::>(); - return Err(Error::invalid_input(format!( - "Dataset::sample received fragment ids that are not part of the current dataset version: {missing_fragment_ids:?}", - ))); - } + let selected_fragments = self.get_fragments_from_ids(fragment_ids)?; let num_rows = stream::iter(selected_fragments.iter().cloned()) .map(|fragment| async move { fragment.count_rows(None).await }) @@ -1818,6 +2056,7 @@ impl Dataset { ) -> Self { let mut cloned = self.clone(); cloned.object_store = object_store; + cloned.base_object_stores = Default::default(); if let Some(store_params) = store_params { cloned.store_params = Some(Box::new(store_params)); } @@ -1839,10 +2078,13 @@ impl Dataset { } let mut cloned = self.clone(); + // Each wrapper application defines a new store lifetime. Keep base + // stores alive within the derived dataset without sharing stateful + // provider layers (such as an AIMD throttle) with other scopes. + cloned.base_object_stores = Default::default(); let mut object_store = self.object_store.as_ref().clone(); for wrapper in &wrappers { - object_store.inner = - wrapper.wrap(&object_store.store_prefix, object_store.inner.clone()); + object_store.apply_wrapper(wrapper.as_ref()); } cloned.object_store = Arc::new(object_store); cloned.refs = Refs::new( @@ -2029,12 +2271,7 @@ impl Dataset { .await?; let file_metadata = FileReader::read_all_metadata(&file).await?; - let file_version = LanceFileVersion::try_from_major_minor( - file_metadata.major_version as u32, - file_metadata.minor_version as u32, - )?; - - let is_structural = file_version >= LanceFileVersion::V2_1; + let lance_file_format = file_metadata.version; let physical_columns = file_metadata.column_metadatas.len(); let has_footer_orphans = file_metadata.file_schema.fields.len() > physical_columns; let dataset_schema = self.schema(); @@ -2042,29 +2279,6 @@ impl Dataset { let mut column_names = Vec::new(); let mut consumed_top_level_fields = 0usize; - fn physical_column_count( - field: &lance_core::datatypes::Field, - is_structural: bool, - ) -> usize { - if !is_structural { - return 1 + field - .children - .iter() - .map(|child| physical_column_count(child, is_structural)) - .sum::(); - } - - if field.children.is_empty() || field.is_blob() || field.is_packed_struct() { - 1 - } else { - field - .children - .iter() - .map(|child| physical_column_count(child, is_structural)) - .sum() - } - } - fn field_contains_blob(field: &lance_core::datatypes::Field) -> bool { field.is_blob() || field.children.iter().any(field_contains_blob) } @@ -2072,15 +2286,15 @@ impl Dataset { fn field_names_match( fields: &[lance_core::datatypes::Field], start: usize, - names: &[&str], + expected: &arrow_schema::Fields, ) -> bool { fields - .get(start..start + names.len()) + .get(start..start + expected.len()) .is_some_and(|candidate| { candidate .iter() - .zip(names) - .all(|(field, name)| field.name == *name) + .zip(expected.iter()) + .all(|(field, expected)| field.name == expected.name().as_str()) }) } @@ -2088,51 +2302,15 @@ impl Dataset { fields: &[lance_core::datatypes::Field], start: usize, ) -> usize { - const BLOB_V2_DESCRIPTOR_FIELDS: &[&str] = - &["kind", "position", "size", "blob_id", "blob_uri"]; - const BLOB_V1_DESCRIPTOR_FIELDS: &[&str] = &["position", "size"]; - - if field_names_match(fields, start, BLOB_V2_DESCRIPTOR_FIELDS) { - BLOB_V2_DESCRIPTOR_FIELDS.len() - } else if field_names_match(fields, start, BLOB_V1_DESCRIPTOR_FIELDS) { - BLOB_V1_DESCRIPTOR_FIELDS.len() + if field_names_match(fields, start, &lance_core::datatypes::BLOB_V2_DESC_FIELDS) { + lance_core::datatypes::BLOB_V2_DESC_FIELDS.len() + } else if field_names_match(fields, start, &lance_core::datatypes::BLOB_DESC_FIELDS) { + lance_core::datatypes::BLOB_DESC_FIELDS.len() } else { 0 } } - fn collect_columns( - field: &lance_core::datatypes::Field, - is_structural: bool, - fields: &mut Vec, - column_indices: &mut Vec, - curr_column_idx: &mut i32, - ) { - let contributes = !is_structural - || field.children.is_empty() - || field.is_blob() - || field.is_packed_struct(); - let recurse = !is_structural || (!field.is_blob() && !field.is_packed_struct()); - - if contributes { - fields.push(field.id); - column_indices.push(*curr_column_idx); - *curr_column_idx += 1; - } - - if recurse { - for child in &field.children { - collect_columns( - child, - is_structural, - fields, - column_indices, - curr_column_idx, - ); - } - } - } - fn validate_file_field_matches_dataset( dataset_field: &lance_core::datatypes::Field, file_field: &lance_core::datatypes::Field, @@ -2186,7 +2364,7 @@ impl Dataset { }; validate_file_field_matches_dataset(dataset_field, field, &field.name)?; - represented_columns += physical_column_count(field, is_structural); + represented_columns += file_versions::physical_column_count(lance_file_format, field); column_names.push(field.name.as_str()); consumed_top_level_fields = idx + 1; idx += 1; @@ -2219,23 +2397,17 @@ impl Dataset { let projected_ds_schema = self.schema().project(&column_names)?; - let mut fields = Vec::new(); - let mut column_indices = Vec::new(); - let mut curr_column_idx: i32 = 0; - for field in &projected_ds_schema.fields { - collect_columns( - field, - is_structural, - &mut fields, - &mut column_indices, - &mut curr_column_idx, - ); - } + let (fields, column_indices) = + file_versions::data_file_columns(lance_file_format, &projected_ds_schema); + let represented_dataset_columns = column_indices + .iter() + .filter(|column_index| **column_index >= 0) + .count(); - if curr_column_idx as usize != physical_columns { + if represented_dataset_columns != physical_columns { return Err(Error::invalid_input(format!( "Schema mismatch: dataset projection maps to {} physical columns but file has {} columns", - curr_column_idx, physical_columns + represented_dataset_columns, physical_columns ))); } @@ -2250,8 +2422,7 @@ impl Dataset { path, fields, column_indices, - file_metadata.major_version as u32, - file_metadata.minor_version as u32, + lance_file_format, file_size_nz, base_id, )) @@ -2283,14 +2454,35 @@ impl Dataset { })?; let store_params = self.store_params_for_base(Some(base_path)); - let (store, _) = ObjectStore::from_uri_and_params( - self.session.store_registry(), - &base_path.path, - &store_params, - ) - .await?; - - Ok(store) + let cell = { + let mut stores = self.base_object_stores.lock().unwrap(); + stores.entry(base_id).or_default().clone() + }; + let store = cell + .get_or_try_init(|| async { + // Wrappers define a request or execution scope. Keep the + // fully resolved store in this dataset's OnceCell, but do not + // also put it in the global registry: provider-local state + // such as GCS AIMD token buckets must not cross that scope. + let (store, _) = if store_params.object_store_wrapper.is_some() { + ObjectStore::from_uri_and_params_uncached( + self.session.store_registry(), + &base_path.path, + &store_params, + ) + .await? + } else { + ObjectStore::from_uri_and_params( + self.session.store_registry(), + &base_path.path, + &store_params, + ) + .await? + }; + Ok::<_, Error>(store) + }) + .await?; + Ok(store.clone()) } /// Resolve the object store for the primary dataset or an additional base. @@ -2304,6 +2496,13 @@ impl Dataset { } } + /// The `ObjectStoreParams` this dataset was opened with, or `None` when + /// opened without explicit params. Lets a caller re-open a derived path + /// (e.g. a MemWAL SSTable) with the same store this dataset used. + pub fn store_params(&self) -> Option<&ObjectStoreParams> { + self.store_params.as_deref() + } + pub(crate) async fn object_store_for_data_file( &self, data_file: &DataFile, @@ -2426,6 +2625,37 @@ impl Dataset { Ok(versions) } + /// Get the number of versions in the current version history. + /// + /// Unlike [`Self::versions`], this only enumerates manifest locations and does not read or + /// deserialize every manifest. + pub async fn count_versions(&self) -> Result { + self.commit_handler + .list_manifest_locations(&self.base, &self.object_store, false) + .try_fold(0_u64, |count, _| async move { Ok(count + 1) }) + .await + } + + /// List lightweight references to all attached versions in the current branch's history. + /// + /// Unlike [`Self::versions`], this only enumerates manifest locations and does not read or + /// deserialize every manifest. The references are sorted by version in ascending order. + /// Detached manifests are excluded; see [`Self::list_detached_manifests`]. + /// + /// Use [`Self::latest_version_id`] instead when only the latest version is needed. + pub async fn version_refs(&self) -> Result> { + let mut versions: Vec<_> = self + .commit_handler + .list_manifest_locations(&self.base, &self.object_store, false) + .map_ok(|location| VersionRef { + version: location.version, + }) + .try_collect() + .await?; + versions.sort_unstable_by_key(|version| version.version); + Ok(versions) + } + /// List all detached manifest locations. /// /// Detached manifests are versions that are not part of the main version history. @@ -2527,54 +2757,150 @@ impl Dataset { } pub fn get_fragment(&self, fragment_id: usize) -> Option { - let dataset = Arc::new(self.clone()); - let fragment = self - .manifest - .fragments - .iter() - .find(|f| f.id == fragment_id as u64)?; - Some(FileFragment::new(dataset, fragment.clone())) + let metadata = self.find_fragment(fragment_id as u64)?.clone(); + Some(FileFragment::new(Arc::new(self.clone()), metadata)) } pub fn fragments(&self) -> &Arc> { &self.manifest.fragments } - // Gets a filtered list of fragments from ids in O(N) time instead of using - // `get_fragment` which would require O(N^2) time. + pub(crate) fn normalize_fragment_ids(fragment_ids: &[u32]) -> Vec { + let mut ids = fragment_ids.to_vec(); + ids.sort_unstable(); + ids.dedup(); + ids + } + + pub(crate) fn get_fragments_from_ids(&self, fragment_ids: &[u32]) -> Result> { + let ordered_ids = Self::normalize_fragment_ids(fragment_ids); + let fragments = self.get_frags_from_ordered_ids(&ordered_ids); + if let Some(missing_id) = fragments + .iter() + .zip(ordered_ids.iter()) + .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id)) + { + return Err(Error::invalid_input(format!( + "Unknown fragment id {missing_id} in fragment filter; not part of the current dataset version" + ))); + } + + Ok(fragments.into_iter().flatten().collect()) + } + + pub(crate) fn get_existing_fragments_from_ids( + &self, + fragment_ids: &[u32], + ) -> Vec { + let ordered_ids = Self::normalize_fragment_ids(fragment_ids); + self.get_frags_from_ordered_ids(&ordered_ids) + .into_iter() + .flatten() + .collect() + } + + pub(crate) fn get_fragment_metadata_from_ids( + &self, + fragment_ids: &[u32], + ) -> Result> { + Ok(self + .get_fragments_from_ids(fragment_ids)? + .into_iter() + .map(|fragment| fragment.metadata().clone()) + .collect()) + } + + pub(crate) fn get_existing_fragment_metadata_from_ids( + &self, + fragment_ids: &[u32], + ) -> Vec { + self.get_existing_fragments_from_ids(fragment_ids) + .into_iter() + .map(|fragment| fragment.metadata().clone()) + .collect() + } + + pub(crate) async fn count_rows_in_fragments(&self, fragment_ids: &[u32]) -> Result { + let fragments = self.get_fragments_from_ids(fragment_ids)?; + self.count_rows_in_resolved_fragments(fragments).await + } + + pub(crate) async fn count_rows_in_existing_fragments( + &self, + fragment_ids: &[u32], + ) -> Result { + let fragments = self.get_existing_fragments_from_ids(fragment_ids); + self.count_rows_in_resolved_fragments(fragments).await + } + + async fn count_rows_in_resolved_fragments( + &self, + fragments: Vec, + ) -> Result { + let counts = stream::iter(fragments) + .map(|fragment| async move { fragment.count_rows(None).await }) + .buffer_unordered(16) + .try_collect::>() + .await?; + Ok(counts.iter().sum()) + } + + /// Resolves fragments for the given ids without scanning the manifest. + /// + /// The ids do not need to be sorted or deduplicated. Each id is resolved + /// independently via the fragment bitmap. pub fn get_frags_from_ordered_ids(&self, ordered_ids: &[u32]) -> Vec> { - let mut fragments = Vec::with_capacity(ordered_ids.len()); - let mut id_iter = ordered_ids.iter(); - let mut id = id_iter.next(); - // This field is just used to assert the ids are in order - let mut last_id: i64 = -1; - for frag in self.manifest.fragments.iter() { - let mut the_id = if let Some(id) = id { *id } else { break }; - // Assert the given ids are, in fact, in order - assert!(the_id as i64 > last_id); - // For any IDs we've passed we can assume that no fragment exists any longer - // with that ID. - while the_id < frag.id as u32 { - fragments.push(None); - last_id = the_id as i64; - id = id_iter.next(); - the_id = if let Some(id) = id { *id } else { break }; - } + let dataset = Arc::new(self.clone()); + ordered_ids + .iter() + .map(|id| { + if !self.fragment_bitmap.contains(*id) { + return None; + } + let fragment_index = self.fragment_bitmap.rank(*id) as usize - 1; + let fragment = self.manifest.fragments.get(fragment_index)?; + debug_assert_eq!( + fragment.id, *id as u64, + "fragment_bitmap rank({id}) resolved to fragment {}, but fragment_bitmap and manifest.fragments are expected to stay in sync", + fragment.id + ); + Some(FileFragment::new(dataset.clone(), fragment.clone())) + }) + .collect() + } - if the_id == frag.id as u32 { - fragments.push(Some(FileFragment::new( - Arc::new(self.clone()), - frag.clone(), - ))); - last_id = the_id as i64; - id = id_iter.next(); - } + /// Look up the fragment with `id` in the manifest. + /// + /// `Manifest::fragments` is kept sorted by id, so this binary searches + /// rather than scanning. Two kinds of manifest predate that invariant and + /// are still readable: those written before fragments were forced into id + /// order (Lance 0.10 and earlier), and those with duplicate fragment ids + /// (Lance 0.16 and earlier). Neither is rejected on read, so the search + /// result is checked and a scan takes over when it does not match -- + /// returning some other fragment's data would be silent corruption. + fn find_fragment(&self, id: u64) -> Option<&Fragment> { + if !u32::try_from(id).is_ok_and(|id| self.fragment_bitmap.contains(id)) { + return None; + } + let fragments = self.manifest.fragments.as_slice(); + let index = fragments.partition_point(|fragment| fragment.id < id); + match fragments.get(index) { + Some(fragment) if fragment.id == id => Some(fragment), + _ => fragments.iter().find(|fragment| fragment.id == id), } - fragments } // This method filters deleted items from `addr_or_ids` using `addrs` as a reference async fn filter_addr_or_ids(&self, addr_or_ids: &[u64], addrs: &[u64]) -> Result> { + // The final zip pairs these positionally; misalignment must fail + // loud rather than truncate. + if addr_or_ids.len() != addrs.len() { + return Err(Error::internal(format!( + "filter_addr_or_ids: addr_or_ids has {} entries but addrs has {}", + addr_or_ids.len(), + addrs.len() + ))); + } if addrs.is_empty() { return Ok(Vec::new()); } @@ -2686,17 +3012,24 @@ impl Dataset { } pub(crate) async fn filter_deleted_ids(&self, ids: &[u64]) -> Result> { - let addresses = if let Some(row_id_index) = get_row_id_index(self).await? { - let addresses = ids - .iter() - .filter_map(|id| row_id_index.get(*id).map(|address| address.into())) - .collect::>(); - Cow::Owned(addresses) + let (ids, addresses) = if let Some(row_id_index) = get_row_id_index(self).await? { + // Ids absent from the deletion-aware index are deleted; drop + // them from both lists to keep the zip aligned. ids.len() is an + // upper bound on the output size, so allocate once up front. + let mut live_ids = Vec::with_capacity(ids.len()); + let mut addresses = Vec::with_capacity(ids.len()); + for id in ids { + if let Some(address) = row_id_index.get(*id)? { + live_ids.push(*id); + addresses.push(u64::from(address)); + } + } + (Cow::Owned(live_ids), Cow::Owned(addresses)) } else { - Cow::Borrowed(ids) + (Cow::Borrowed(ids), Cow::Borrowed(ids)) }; - self.filter_addr_or_ids(ids, &addresses).await + self.filter_addr_or_ids(&ids, &addresses).await } /// Gets the number of files that are so small they don't even have a full @@ -2758,8 +3091,14 @@ impl Dataset { .try_collect::>() .await?; - // Validate indices - let indices = self.load_indices().await?; + rowids::validate_stable_row_ids(self).await?; + + // Validate indices. Over the complete list: these checks are about what + // the manifest says, not about what this build can use, and duplicate + // uuids or overlapping coverage are no less corrupt for involving an + // index this build has no reader for. `migrate_indices` already runs the + // same overlap check over the complete list on every commit. + let indices = crate::index::load_all_indices(self).await?; self.validate_indices(&indices)?; Ok(()) @@ -2774,7 +3113,7 @@ impl Dataset { self.manifest_location.path.clone(), format!( "Duplicate index id {} found in dataset {:?}", - &index.uuid, self.base + index.uuid, self.base ), )); } @@ -2840,6 +3179,80 @@ impl Dataset { Ok(()) } + /// Assign stable row ID sequences to fragments that do not yet have them, + /// contiguously from `start`, and return the resulting `next_row_id` + /// high-water mark. + fn assign_stable_row_ids_for_migration(fragments: &mut [Fragment], start: u64) -> Result { + let mut next_row_id = start; + for fragment in fragments.iter_mut() { + let physical_rows = fragment.physical_rows.ok_or_else(|| { + Error::internal(format!( + "Fragment {} is missing physical_rows; cannot assign stable row IDs", + fragment.id + )) + })? as u64; + let end = next_row_id + .checked_add(physical_rows) + .ok_or_else(|| Error::internal("Row ID overflow during stable row ID migration"))?; + let sequence = RowIdSequence::from(next_row_id..end); + fragment.row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&sequence).into())); + next_row_id = end; + } + Ok(next_row_id) + } + + /// Migrate a table to use stable row IDs. + /// + /// Stable row IDs assign a persistent identifier to each row that remains + /// stable across compaction operations. This enables more efficient updates + /// to secondary indices. + /// + /// A single Merge commit assigns row ID sequences to all fragments and + /// activates the stable row ID feature flag atomically. Because `Merge` + /// conflicts with all data-modifying operations, a successful commit + /// guarantees no concurrent write occurred — no separate validation step + /// is needed. + /// + /// **No retries are attempted.** Callers should quiesce concurrent writes + /// before running this migration. If a conflicting write is detected, this + /// method returns an error and the caller must retry. + /// + /// This method is idempotent: if the table already uses stable row IDs, + /// it returns `Ok(())` immediately. + pub async fn migrate_to_stable_row_ids(&mut self) -> Result<()> { + if self.manifest.uses_stable_row_ids() { + return Ok(()); + } + + let mut fragments = self.manifest.fragments.as_ref().clone(); + // Restore carries the high-water mark forward across a version that + // predates activation, so a re-migration must allocate above it rather + // than reissue ids the earlier versions still hold. + let next_row_id = + Self::assign_stable_row_ids_for_migration(&mut fragments, self.manifest.next_row_id)?; + let schema = self.manifest.schema.clone(); + let read_version = self.manifest.version; + + let transaction = Transaction::new( + read_version, + Operation::Merge { + fragments, + schema, + preserves_nullability: true, + }, + None, + ); + + let new_ds = CommitBuilder::new(Arc::new(self.clone())) + .with_max_retries(0) + .with_stable_row_id_migration_activation(next_row_id) + .execute(transaction) + .await?; + + *self = new_ds; + Ok(()) + } + /// Shallow clone the target version into a new dataset at target_path. /// 'target_path': the uri string to clone the dataset into. /// 'version': the version cloned from, could be a version number or tag. @@ -2867,19 +3280,33 @@ impl Dataset { ) .with_object_store(Arc::new(self.object_store.as_ref().clone())) .with_commit_handler(self.commit_handler.clone()) - .with_storage_format(self.manifest.data_storage_format.lance_file_version()?); + .with_exact_storage_format(self.manifest.data_storage_format.lance_file_format()); builder.execute(transaction).await } /// Deep clone the target version into a new dataset at target_path. - /// This performs a server-side copy of all relevant dataset files (data files, - /// deletion files, and any external row-id files) into the target dataset - /// without loading data into memory. + /// This copies all relevant dataset files (data files, deletion files, and + /// index files) into the target dataset with bounded memory use. + /// + /// The source files are read through this dataset's own object store while the + /// copies are written through the target object store built from `store_params`. + /// This makes the clone work across accounts/stores (e.g. between two abfss + /// accounts). Object-store files are streamed through this process by default; + /// `LANCE_IO_SERVER_SIDE_COPY_ENABLED` opts same-store copies into + /// provider-native copy operations. Cross-store copies continue to stream, and + /// local files retain their filesystem copy path. /// /// Parameters: /// - `target_path`: the URI string to clone the dataset into. /// - `version`: the version cloned from, could be a version number, branch head, or tag. - /// - `store_params`: the object store params to use for the new dataset. + /// - `store_params`: the object store params for the target dataset (e.g. the + /// credentials of the target account). + /// + /// Note: external `base_paths` referenced by the source manifest are read through + /// this dataset's object store; per-base distinct source credentials are not yet + /// supported (see ). + /// Object-store streaming defaults to at most four concurrent file copies; + /// `LANCE_DEEP_CLONE_STREAM_CONCURRENCY` overrides that limit for this operation. pub async fn deep_clone( &mut self, target_path: &str, @@ -2890,6 +3317,7 @@ impl Dataset { // Resolve source dataset and its manifest using checkout_version let src_ds = self.checkout_version(version).await?; + ensure_can_write_manifest(&src_ds.manifest)?; let src_paths = src_ds.collect_paths().await?; // Prepare target object store and base path @@ -2920,20 +3348,41 @@ impl Dataset { path }; - // TODO: Leverage object store bulk copy for efficient deep_clone - // - // All cloud storage providers support batch copy APIs that would provide significant - // performance improvements. We use single file copy before we have upstream support. - // - // Tracked by: https://github.com/lance-format/lance/issues/5435 - let io_parallelism = self.object_store.io_parallelism(); + let configured_io_parallelism = src_ds.object_store.io_parallelism(); + // Provider-native copy can fall back to streaming for large objects, so every + // non-direct-local transfer stays within the bounded file-copy window. + let uses_streaming_copy = !(src_ds.object_store.has_direct_local_paths() + && target_store.has_direct_local_paths()); + let stream_copy_parallelism = match std::env::var("LANCE_DEEP_CLONE_STREAM_CONCURRENCY") { + Ok(value) => Some(parse_deep_clone_stream_concurrency(&value)?), + Err(std::env::VarError::NotPresent) => None, + Err(std::env::VarError::NotUnicode(value)) => { + return Err(Error::invalid_input(format!( + "LANCE_DEEP_CLONE_STREAM_CONCURRENCY must be valid UTF-8 and a positive \ + integer, got {value:?}" + ))); + } + }; + // Limit the number of concurrently buffered transfers by default while + // preserving efficient local copies and the operation-specific override. + let io_parallelism = deep_clone_copy_parallelism( + configured_io_parallelism, + uses_streaming_copy, + stream_copy_parallelism, + ); let copy_futures = src_paths .iter() .map(|(relative_path, base)| { - let store = Arc::clone(&target_store); + let source_store = Arc::clone(&src_ds.object_store); + let target_store = Arc::clone(&target_store); let src_path = build_absolute_path(relative_path, base); let target_path = build_absolute_path(relative_path, &target_base); - async move { store.copy(&src_path, &target_path).await.map(|_| ()) } + async move { + source_store + .copy_bulk(&src_path, &target_store, &target_path) + .await?; + Result::Ok(()) + } }) .collect::>(); @@ -2958,8 +3407,9 @@ impl Dataset { let builder = CommitBuilder::new(WriteDestination::Uri(target_path)) .with_store_params(store_params.clone().unwrap_or_default()) .with_object_store(target_store.clone()) + .with_source_store(src_ds.object_store.clone()) .with_commit_handler(self.commit_handler.clone()) - .with_storage_format(self.manifest.data_storage_format.lance_file_version()?); + .with_exact_storage_format(self.manifest.data_storage_format.lance_file_format()); let new_ds = builder.execute(txn).await?; Ok(new_ds) } @@ -2999,7 +3449,7 @@ impl Dataset { external_file.path ))); } - for data_file in fragment.files.iter() { + for data_file in fragment.referenced_lance_files() { let base_root = if let Some(base_id) = data_file.base_id { let base_path = self.manifest.base_paths.get(&base_id).ok_or_else(|| { @@ -3072,29 +3522,6 @@ impl Dataset { pub fn sql(&self, sql: &str) -> SqlQueryBuilder { SqlQueryBuilder::new(self.clone(), sql) } - - /// Returns true if Lance supports writing this datatype with nulls. - pub(crate) fn lance_supports_nulls(&self, datatype: &DataType) -> bool { - match self - .manifest() - .data_storage_format - .lance_file_version() - .unwrap_or(LanceFileVersion::Legacy) - .resolve() - { - LanceFileVersion::Legacy => matches!( - datatype, - DataType::Utf8 - | DataType::LargeUtf8 - | DataType::Binary - | DataType::List(_) - | DataType::FixedSizeBinary(_) - | DataType::FixedSizeList(_, _) - ), - LanceFileVersion::V2_0 => !matches!(datatype, DataType::Struct(..)), - _ => true, - } - } } pub(crate) struct NewTransactionResult<'a> { @@ -3319,11 +3746,14 @@ impl Dataset { .try_collect::>() .await?; + let preserves_nullability = + !schema_evolution::merge_introduces_required_field(self.schema(), &new_schema); let transaction = Transaction::new( self.manifest.version, Operation::Merge { fragments: updated_fragments, schema: new_schema, + preserves_nullability, }, None, ); @@ -3636,6 +4066,10 @@ pub(crate) struct ManifestWriteConfig { use_legacy_format: Option, // default None storage_format: Option, // default None disable_transaction_file: bool, // default false + /// When `Some`, this commit is the second step of `migrate_to_stable_row_ids`. + /// It bypasses the "cannot enable stable row ids on existing dataset" guard and + /// sets `manifest.next_row_id` to the provided value before activating the flag. + migration_next_row_id: Option, // default None } impl Default for ManifestWriteConfig { @@ -3647,6 +4081,7 @@ impl Default for ManifestWriteConfig { disable_transaction_file: false, use_legacy_format: None, storage_format: None, + migration_next_row_id: None, } } } @@ -3655,6 +4090,53 @@ impl ManifestWriteConfig { pub fn disable_transaction_file(&self) -> bool { self.disable_transaction_file } + + #[cfg(test)] + pub(crate) fn with_transaction_file_disabled(mut self) -> Self { + self.disable_transaction_file = true; + self + } + + /// Resolve into the config `Transaction::build_manifest` consumes. + /// + /// The timestamp is resolved here rather than during the build so it goes + /// through this crate's mockable `SystemTime`. + pub(crate) fn to_build_config(&self) -> ManifestBuildConfig { + ManifestBuildConfig { + auto_set_feature_flags: self.auto_set_feature_flags, + timestamp_nanos: timestamp_to_nanos(self.timestamp), + use_stable_row_ids: self.use_stable_row_ids, + use_legacy_format: self.use_legacy_format, + storage_format: self.storage_format.clone(), + disable_transaction_file: self.disable_transaction_file, + migration_next_row_id: self.migration_next_row_id, + } + } +} + +/// Decode an inline transaction section for opportunistic caching. +/// +/// Returns `None` instead of failing when the transaction cannot be decoded: +/// the section may have been written by a newer version of Lance with an +/// operation type this version does not know, and that must not prevent +/// opening the dataset. Paths that need the transaction contents surface the +/// error at their call sites instead. +fn decode_inline_transaction(message_data: &[u8], version: u64) -> Option { + match lance_table::format::pb::Transaction::decode(message_data) + .map_err(Error::from) + .and_then(Transaction::try_from) + { + Ok(transaction) => Some(transaction), + Err(err) => { + log::warn!( + "Failed to decode the inline transaction of version {}; \ + it may have been written by a newer version of Lance: {}", + version, + err + ); + None + } + } } /// Commit a manifest file and create a copy at the latest manifest path. @@ -3667,8 +4149,35 @@ pub(crate) async fn write_manifest_file( indices: Option>, config: &ManifestWriteConfig, naming_scheme: ManifestNamingScheme, - mut transaction: Option<&Transaction>, + transaction: Option, + may_change_schema: bool, ) -> std::result::Result { + validate_paired_feature_flags(manifest)?; + // Every manifest write funnels through here, including restore and clone, + // which rebuild a manifest from a stored one rather than from an Arrow + // schema, so this is where the invariant holds for a schema that never + // passed through that conversion. + // + // Only for transactions that can change the schema. Released versions could + // install a key on a nullable column through the metadata path, and + // validating every write would make such a table read-only on upgrade -- + // including through the delete that removes the offending rows, which is + // the first step of repairing it. A repair still has to pass: it changes + // the schema, and the schema it produces is valid. + // + // The caller classifies the operation, rather than this reading it off + // `transaction`, which is None whenever the encoded bytes were too large + // to inline. Deriving it here would make the verdict depend on payload + // size, so the same operation would be exempt while small and validated + // once it spilled -- and a MemWAL table spills routinely, since its + // transactions carry mem-table state. + if may_change_schema { + manifest + .schema + .verify_primary_key() + .map_err(CommitError::OtherError)?; + } + if config.auto_set_feature_flags { // build_manifest may have already set FLAG_STABLE_ROW_IDS on the manifest. // Preserve it here so this second apply_feature_flags call does not clear it @@ -3693,7 +4202,7 @@ pub(crate) async fn write_manifest_file( object_store, write_manifest_file_to_path, naming_scheme, - transaction.take().map(|tx| tx.into()), + transaction, ) .await } @@ -3704,5 +4213,130 @@ impl Projectable for Dataset { } } +/// Marker files that `DirectoryNamespace` writes for a table that is declared or +/// deregistered but was never materialized. Spelled out here because +/// `lance-namespace-impls` depends on this crate, not the other way around. +const NAMESPACE_TABLE_MARKERS: &[&str] = &[".lance-reserved", ".lance-deregistered"]; + +/// Check that `base` is a Lance dataset root before deleting it recursively. +/// +/// Dropping a dataset removes whatever the caller pointed at, so a mistyped or +/// misconfigured URI — a warehouse root, a bucket root, a home directory — destroys +/// unrelated data with no way back. Requiring the target to actually be a dataset +/// turns that class of mistake into an error instead of silent data loss. +/// +/// A path qualifies on positive evidence only, which is one of: +/// +/// * a file under `_versions/` that both parses as a manifest location and deserializes +/// as a manifest, attached or detached. Every dataset that has ever committed has one, +/// whatever naming scheme or commit handler produced it. +/// * a `DirectoryNamespace` declare or deregister marker, for a table that a namespace +/// reserved but never wrote. +/// +/// Nothing weaker qualifies. A non-empty `_versions/` is not evidence, because any file +/// can be put there; neither are data files, which look identical to a storage root whose +/// only prefix happens to be `data/`. Leftovers from a write that never committed, +/// manifests that are corrupt, and the staging manifest an external store writes before +/// it materializes the canonical path therefore need an explicit storage-level delete +/// rather than a weaker default guard here. That costs little: leftovers do not block +/// re-creating the dataset, because creation only refuses a path that already holds a +/// manifest, and [`Dataset::cleanup_old_versions`] removes data files no manifest +/// references. +/// +/// Unmanaged files that a user keeps next to a committed dataset do not change the +/// answer, matching the way cleanup leaves them alone. Note that the recursive delete +/// this guards still removes them. +/// +/// A missing or empty path also qualifies, so callers keep whatever not-found behavior +/// they have today rather than seeing a new error kind. +/// +/// This cannot protect files that another dataset references through `base_paths`; +/// shallow-clone sources still need the reference tracking discussed in +/// [#7514](https://github.com/lance-format/lance/issues/7514). +pub async fn validate_dataset_root_for_drop(object_store: &ObjectStore, base: &Path) -> Result<()> { + if holds_readable_manifest(object_store, base).await? { + return Ok(()); + } + + for marker in NAMESPACE_TABLE_MARKERS { + if object_store.exists(&base.clone().join(*marker)).await? { + return Ok(()); + } + } + + // Rejecting a path that holds nothing would replace the not-found error callers + // already handle, and `ignore_not_found` relies on, with a different error kind. + if !has_any_entry(object_store, base).await? { + return Ok(()); + } + + Err(Error::invalid_input(format!( + "Refusing to drop '{base}': no readable Lance manifest was found under \ + '{VERSIONS_DIR}', so this is not a dataset root. Check that the path points at a \ + dataset and not at a parent directory, and check the logs for manifests that \ + could not be read. A path holding only data files, or only manifests that cannot \ + be read, needs an explicit storage-level delete instead: such leftovers neither \ + block re-creating the dataset nor survive cleanup." + ))) +} + +/// Whether `base` holds a manifest that actually deserializes, which is the only proof +/// that a dataset was ever committed here. +/// +/// Returns on the first manifest that reads, so a real dataset costs one listing plus one +/// manifest read no matter how many versions it has. +async fn holds_readable_manifest(object_store: &ObjectStore, base: &Path) -> Result { + let mut entries = object_store.list(Some(base.clone().join(VERSIONS_DIR))); + loop { + let meta = match entries.try_next().await { + Ok(Some(meta)) => meta, + Ok(None) => return Ok(false), + // Local filesystems report a missing directory as an error where object stores + // return an empty listing. Neither holds a manifest. + Err(e) if e.is_not_found() => return Ok(false), + Err(e) => return Err(e), + }; + + if !is_manifest_location(&meta) { + continue; + } + + match read_manifest(object_store, &meta.location, Some(meta.size)).await { + Ok(_) => return Ok(true), + // A file that only looks like a manifest proves nothing, so keep looking + // rather than authorizing the delete. The reason is logged because a read + // that failed for an unrelated cause, such as a transient storage error, + // otherwise leaves no trace of why the path was refused. + Err(e) => warn!( + "Ignoring '{}' while checking whether '{base}' is a dataset root: {e}", + meta.location + ), + } + } +} + +/// Whether `meta` names a manifest, using the same parsing that manifest discovery uses. +fn is_manifest_location(meta: &object_store::ObjectMeta) -> bool { + if ManifestLocation::try_from(meta.clone()).is_ok() { + return true; + } + meta.location + .filename() + .and_then(ManifestNamingScheme::parse_detached_version) + .is_some() +} + +/// Whether anything at all lives under `prefix`. Stops at the first entry, so this stays +/// cheap even on a storage root holding millions of objects. +async fn has_any_entry(object_store: &ObjectStore, prefix: &Path) -> Result { + match object_store.list(Some(prefix.clone())).try_next().await { + Ok(entry) => Ok(entry.is_some()), + // Local filesystems report a missing directory as an error where object stores + // return an empty listing. Neither has anything to protect. + Err(e) if e.is_not_found() => Ok(false), + Err(e) => Err(e), + } +} + #[cfg(test)] mod tests; diff --git a/rust/lance/src/dataset/blob.rs b/rust/lance/src/dataset/blob.rs index 84c1b8bdcad..1b17ce28920 100644 --- a/rust/lance/src/dataset/blob.rs +++ b/rust/lance/src/dataset/blob.rs @@ -2,44 +2,51 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::{ - collections::{BTreeMap, HashMap, VecDeque}, + collections::{BTreeMap, HashMap, HashSet}, future::Future, ops::{DerefMut, Range}, panic::AssertUnwindSafe, sync::Arc, - task::Poll, }; use arrow::array::AsArray; use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type}; -use arrow_array::RecordBatch; -use arrow_array::{Array, ArrayRef}; -use arrow_schema::{DataType as ArrowDataType, Field as ArrowField}; +use arrow_array::{ + Array, ArrayRef, GenericListArray, OffsetSizeTrait, RecordBatch, builder::LargeBinaryBuilder, +}; +use arrow_buffer::{ArrowNativeType, OffsetBuffer, ScalarBuffer}; +use arrow_schema::{ + DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef, +}; use bytes::Bytes; use futures::future::BoxFuture; use futures::stream::BoxStream; use futures::{FutureExt, StreamExt, TryStreamExt, stream}; use lance_arrow::{ - BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, - BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt, r#struct::StructArrayExt, + ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, + BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, FieldExt, + list::ListArrayExt, r#struct::StructArrayExt, }; use lance_io::object_store::{ObjectStore, ObjectStoreParams, ObjectStoreRegistry}; use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig}; use object_store::path::Path; -use tokio::sync::{Mutex, OnceCell, oneshot}; +use tokio::sync::{Mutex, Notify, OnceCell, oneshot}; use url::Url; -use super::take::TakeBuilder; +use super::take::{MissingRowPolicy, TakeBuilder}; use super::write::ExternalBlobMode; use super::{Dataset, ProjectionRequest}; use crate::blob::{ BlobDescriptor, BlobDescriptorArrayBuilder, BlobIdAllocator, BlobRange, PackedBlobWriter, - is_logical_blob_v2_field, is_prepared_blob_v2_field, validate_prepared_blob_array, + blob_v2_layout, blob_v2_shape_error, validate_prepared_blob_array, }; use arrow_array::StructArray; -use lance_core::datatypes::{BlobKind, BlobVersion, parse_field_path}; +use lance_core::datatypes::{ + BLOB_DESC_FIELDS, BlobKind, BlobV2Layout, BlobVersion, Field as LanceField, Schema, + parse_field_path, +}; use lance_core::utils::blob::blob_path; -use lance_core::{Error, Result, utils::address::RowAddress}; +use lance_core::{Error, ROW_ADDR, Result, utils::address::RowAddress}; use lance_io::traits::Reader; use lance_io::utils::CachedFileSize; @@ -183,6 +190,158 @@ impl ExternalBaseResolver { } } +fn arrow_field_contains_blob_v2(field: &ArrowField) -> bool { + if field.is_blob_v2() { + return true; + } + match field.data_type() { + ArrowDataType::Struct(children) => children + .iter() + .any(|child| arrow_field_contains_blob_v2(child)), + ArrowDataType::List(child) | ArrowDataType::LargeList(child) => { + arrow_field_contains_blob_v2(child) + } + _ => false, + } +} + +fn collect_external_blob_uris( + field: &ArrowField, + array: &ArrayRef, + selected_rows: &[bool], + field_path: &str, + external_uris: &mut Vec<(String, String)>, +) -> Result<()> { + if !arrow_field_contains_blob_v2(field) { + return Ok(()); + } + if array.len() != selected_rows.len() { + return Err(Error::internal(format!( + "Blob field '{}' row count {} did not match selection length {}", + field_path, + array.len(), + selected_rows.len() + ))); + } + + if field.is_blob_v2() { + let struct_array = array.as_struct(); + if BlobV2Layout::classify(struct_array.fields()) != Some(BlobV2Layout::Logical) { + return Err(blob_v2_shape_error(field, &[BlobV2Layout::Logical])); + } + let uri_column = struct_array + .column_by_name("uri") + .ok_or_else(|| Error::invalid_input("Blob struct missing `uri` field"))? + .as_string::(); + for (row_idx, is_selected) in selected_rows.iter().copied().enumerate() { + if is_selected && struct_array.is_valid(row_idx) && uri_column.is_valid(row_idx) { + external_uris.push(( + field_path.to_string(), + uri_column.value(row_idx).to_string(), + )); + } + } + return Ok(()); + } + + match field.data_type() { + ArrowDataType::Struct(children) => { + let struct_array = array.as_struct(); + let child_selection = selected_rows + .iter() + .copied() + .enumerate() + .map(|(row_idx, is_selected)| is_selected && struct_array.is_valid(row_idx)) + .collect::>(); + for (child_field, child_array) in children.iter().zip(struct_array.columns()) { + let child_path = format!("{}.{}", field_path, child_field.name()); + collect_external_blob_uris( + child_field, + child_array, + &child_selection, + &child_path, + external_uris, + )?; + } + } + ArrowDataType::List(child) => { + let list_array = array.as_list::(); + let mut child_selection = vec![false; list_array.values().len()]; + for (row_idx, is_selected) in selected_rows.iter().copied().enumerate() { + if is_selected && list_array.is_valid(row_idx) { + let start = list_array.value_offsets()[row_idx].as_usize(); + let end = list_array.value_offsets()[row_idx + 1].as_usize(); + child_selection[start..end].fill(true); + } + } + let child_path = format!("{}.{}", field_path, child.name()); + collect_external_blob_uris( + child, + list_array.values(), + &child_selection, + &child_path, + external_uris, + )?; + } + ArrowDataType::LargeList(child) => { + let list_array = array.as_list::(); + let mut child_selection = vec![false; list_array.values().len()]; + for (row_idx, is_selected) in selected_rows.iter().copied().enumerate() { + if is_selected && list_array.is_valid(row_idx) { + let start = list_array.value_offsets()[row_idx].as_usize(); + let end = list_array.value_offsets()[row_idx + 1].as_usize(); + child_selection[start..end].fill(true); + } + } + let child_path = format!("{}.{}", field_path, child.name()); + collect_external_blob_uris( + child, + list_array.values(), + &child_selection, + &child_path, + external_uris, + )?; + } + _ => {} + } + Ok(()) +} + +/// Validate external blob references supplied by selected input rows. +/// +/// Existing rows can contain trusted absolute references that were accepted by an earlier write. +/// Update paths use this check before allowing those fallback values through the writer, so newly +/// matched values must still resolve beneath a registered external base. +pub(super) async fn validate_external_blob_references( + resolver: &ExternalBaseResolver, + batch: &RecordBatch, + selected_rows: &[bool], +) -> Result<()> { + let mut external_uris = Vec::new(); + for (field, array) in batch.schema().fields().iter().zip(batch.columns()) { + collect_external_blob_uris( + field, + array, + selected_rows, + field.name(), + &mut external_uris, + )?; + } + + let mut validated_uris = HashSet::new(); + for (field_path, uri) in external_uris { + if validated_uris.insert(uri.clone()) + && resolver.resolve_external_uri(&uri).await?.is_none() + { + return Err(Error::invalid_input(format!( + "External blob URI '{}' in field '{}' is outside registered external bases (dataset root is not allowed)", + uri, field_path + ))); + } + } + Ok(()) +} + struct RollingPackedBlobWriter { current: Option, current_size: usize, @@ -265,6 +424,12 @@ impl RollingPackedBlobWriter { self.current_max_pack_size = None; Ok(()) } + + fn abort(&mut self) { + self.current.take(); + self.current_size = 0; + self.current_max_pack_size = None; + } } /// Preprocesses blob v2 columns on the write path so the encoder only sees lightweight descriptors: @@ -277,6 +442,7 @@ pub struct BlobPreprocessor { data_dir: Path, data_file_key: String, blob_id_allocator: BlobIdAllocator, + part_blob_ids: Option>, pack_writer: RollingPackedBlobWriter, /// Write-param override for the pack-file roll size. When set, it takes /// precedence over each field's `blob-pack-file-size-threshold` metadata for @@ -323,43 +489,41 @@ enum BlobPreprocessFieldKind { Struct { children: Vec, }, + List { + child: Box, + }, Passthrough, } impl BlobPreprocessField { fn new(field: &ArrowField) -> Result { if field.is_blob_v2() { - if is_prepared_blob_v2_field(field) { - return Ok(Self { + return match blob_v2_layout(field) { + Some(BlobV2Layout::Prepared) => Ok(Self { kind: BlobPreprocessFieldKind::Passthrough, - }); - } - if !is_logical_blob_v2_field(field) { - return Err(Error::invalid_input(format!( - "Blob v2 field '{}' must use either logical struct with optional position/size UInt64 fields or prepared \ - struct", - field.name() - ))); - } - return Ok(Self { - kind: BlobPreprocessFieldKind::BlobV2 { - inline_threshold: blob_inline_threshold_from_metadata( - field.metadata(), - field.name(), - )?, - dedicated_threshold: blob_dedicated_threshold_from_metadata( - field.metadata(), - field.name(), - )?, - pack_file_threshold: blob_pack_file_threshold_from_metadata( - field.metadata(), - field.name(), - )?, - writer_metadata: field.metadata().clone(), - }, - }); + }), + Some(BlobV2Layout::Logical) => Ok(Self { + kind: BlobPreprocessFieldKind::BlobV2 { + inline_threshold: blob_inline_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + dedicated_threshold: blob_dedicated_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + pack_file_threshold: blob_pack_file_threshold_from_metadata( + field.metadata(), + field.name(), + )?, + writer_metadata: field.metadata().clone(), + }, + }), + _ => Err(blob_v2_shape_error( + field, + &[BlobV2Layout::Logical, BlobV2Layout::Prepared], + )), + }; } if let ArrowDataType::Struct(children) = field.data_type() { @@ -374,6 +538,17 @@ impl BlobPreprocessField { } } + if let ArrowDataType::List(child) | ArrowDataType::LargeList(child) = field.data_type() { + let child = Self::new(child.as_ref())?; + if child.requires_preprocessing() { + return Ok(Self { + kind: BlobPreprocessFieldKind::List { + child: Box::new(child), + }, + }); + } + } + Ok(Self { kind: BlobPreprocessFieldKind::Passthrough, }) @@ -382,6 +557,23 @@ impl BlobPreprocessField { fn requires_preprocessing(&self) -> bool { !matches!(self.kind, BlobPreprocessFieldKind::Passthrough) } + + fn force_non_empty_inline_to_sidecar(&mut self) { + match &mut self.kind { + BlobPreprocessFieldKind::BlobV2 { + inline_threshold, .. + } => *inline_threshold = 0, + BlobPreprocessFieldKind::Struct { children } => { + for child in children { + child.force_non_empty_inline_to_sidecar(); + } + } + BlobPreprocessFieldKind::List { child } => { + child.force_non_empty_inline_to_sidecar(); + } + BlobPreprocessFieldKind::Passthrough => {} + } + } } impl ExternalBlobSource { @@ -415,6 +607,9 @@ impl ExternalBlobSource { /// Materialize the slice into memory for the inline blob path. async fn read_all(&self) -> Result { + if self.size == 0 { + return Ok(bytes::Bytes::new()); + } let range = self.reader_range()?; self.reader.get_range(range).await.map_err(Into::into) } @@ -459,6 +654,7 @@ impl BlobPreprocessor { data_dir, data_file_key, blob_id_allocator: BlobIdAllocator::new(1), + part_blob_ids: None, pack_writer, pack_file_size_override, field_processors, @@ -470,6 +666,15 @@ impl BlobPreprocessor { }) } + pub(super) fn with_part_blob_ids(mut self, blob_ids: Range) -> Result { + self.blob_id_allocator = BlobIdAllocator::from_range(blob_ids.clone())?; + self.part_blob_ids = Some(blob_ids); + for processor in &mut self.field_processors { + processor.force_non_empty_inline_to_sidecar(); + } + Ok(self) + } + fn blob_writer_with_metadata( &self, field: &ArrowField, @@ -521,6 +726,102 @@ impl BlobPreprocessor { .await } + async fn prepare_blob_for_part( + &mut self, + array: ArrayRef, + field: &ArrowField, + pack_file_threshold: usize, + writer_metadata: &HashMap, + ) -> Result<(ArrayRef, Arc)> { + validate_prepared_blob_array(field, &array)?; + let values = array.as_struct(); + let kinds = values + .column_by_name("kind") + .expect("validated prepared Blob has kind") + .as_primitive::(); + let data = values + .column_by_name("data") + .expect("validated prepared Blob has data") + .as_binary::(); + let uris = values + .column_by_name("uri") + .expect("validated prepared Blob has uri") + .as_string::(); + let blob_ids = values + .column_by_name("blob_id") + .expect("validated prepared Blob has blob_id") + .as_primitive::(); + let sizes = values + .column_by_name("blob_size") + .expect("validated prepared Blob has blob_size") + .as_primitive::(); + let positions = values + .column_by_name("position") + .expect("validated prepared Blob has position") + .as_primitive::(); + let mut output = self.blob_writer_with_metadata(field, writer_metadata.clone()); + + for row in 0..values.len() { + if values.is_null(row) { + continue; + } + match BlobKind::try_from(kinds.value(row))? { + BlobKind::Packed | BlobKind::Dedicated => { + let blob_id = blob_ids.value(row); + self.blob_id_allocator.reserve(blob_id).map_err(|error| { + Error::invalid_input(format!( + "Prepared Blob v2 field '{}' row {row} uses invalid managed Blob ID {blob_id}: {error}", + field.name() + )) + })?; + } + BlobKind::Inline | BlobKind::External => {} + } + } + + for row in 0..values.len() { + if values.is_null(row) { + output.push_null()?; + continue; + } + match BlobKind::try_from(kinds.value(row))? { + BlobKind::Inline => { + let value = data.value(row); + if value.is_empty() { + output.push_inline(Bytes::new())?; + } else { + let descriptor = self + .write_packed(pack_file_threshold, BlobWriteSource::Bytes(value)) + .await?; + output.push(descriptor)?; + } + } + BlobKind::Packed => { + output.push_packed( + blob_ids.value(row), + BlobRange { + offset: positions.value(row), + size: sizes.value(row), + }, + )?; + } + BlobKind::Dedicated => { + output.push_dedicated(blob_ids.value(row), sizes.value(row))?; + } + BlobKind::External => { + output.push(BlobDescriptor::External { + base_id: blob_ids.value(row), + uri: uris.value(row).to_string(), + offset: positions.value(row), + size: sizes.value(row), + })?; + } + } + } + let (field, array) = output.finish()?.into_parts(); + Ok((array, Arc::new(field))) + } + async fn resolve_external_reference(&mut self, uri: &str) -> Result<(u32, String)> { let mapped = if let Some(resolver) = &self.external_base_resolver { resolver.resolve_external_uri(uri).await? @@ -632,7 +933,23 @@ impl BlobPreprocessor { field: &'a Arc, ) -> BoxFuture<'a, Result<(ArrayRef, Arc)>> { async move { - if is_prepared_blob_v2_field(field.as_ref()) { + if blob_v2_layout(field.as_ref()) == Some(BlobV2Layout::Prepared) { + if self.part_blob_ids.is_some() + && let BlobPreprocessFieldKind::BlobV2 { + pack_file_threshold, + writer_metadata, + .. + } = &processor.kind + { + return self + .prepare_blob_for_part( + array, + field.as_ref(), + *pack_file_threshold, + writer_metadata, + ) + .await; + } validate_prepared_blob_array(field.as_ref(), &array)?; return Ok((array, field.clone())); } @@ -645,7 +962,7 @@ impl BlobPreprocessor { pack_file_threshold, writer_metadata, } => { - self.preprocess_blob_array( + self.logical_to_prepared_blob_array( array, field.as_ref(), *inline_threshold, @@ -659,6 +976,20 @@ impl BlobPreprocessor { self.preprocess_struct_array(array, field.as_ref(), children) .await } + BlobPreprocessFieldKind::List { child } => match field.data_type() { + ArrowDataType::List(_) => { + self.preprocess_list_array::(array, field.as_ref(), child) + .await + } + ArrowDataType::LargeList(_) => { + self.preprocess_list_array::(array, field.as_ref(), child) + .await + } + _ => Err(Error::internal(format!( + "Blob list preprocessor received non-list field '{}'", + field.name() + ))), + }, } } .boxed() @@ -691,10 +1022,8 @@ impl BlobPreprocessor { let mut new_columns = Vec::with_capacity(children.len()); let mut new_fields = Vec::with_capacity(children.len()); - for ((child_processor, child_array), child_field) in children - .iter() - .zip(child_columns.into_iter()) - .zip(child_fields.iter()) + for ((child_processor, child_array), child_field) in + children.iter().zip(child_columns).zip(child_fields.iter()) { let (new_column, new_field) = self .preprocess_field(child_processor, child_array, child_field) @@ -716,7 +1045,80 @@ impl BlobPreprocessor { Ok((Arc::new(struct_array), field)) } - async fn preprocess_blob_array( + async fn preprocess_list_array( + &mut self, + array: ArrayRef, + field: &ArrowField, + child: &BlobPreprocessField, + ) -> Result<(ArrayRef, Arc)> { + let list_arr = array.as_list::(); + let list_arr = if list_arr.null_count() > 0 { + list_arr.filter_garbage_nulls() + } else { + list_arr.clone() + }; + + let first_offset = *list_arr + .offsets() + .first() + .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?; + let last_offset = *list_arr + .offsets() + .last() + .ok_or_else(|| Error::invalid_input("List offsets cannot be empty"))?; + let values_len = list_arr.values().len(); + let needs_trim = first_offset != O::zero() + || last_offset.to_usize().ok_or_else(|| { + Error::invalid_input(format!( + "List field '{}' offset does not fit into usize", + field.name() + )) + })? != values_len; + + let (offsets, values) = if needs_trim { + let values = list_arr.trimmed_values(); + let offsets = list_arr + .offsets() + .iter() + .map(|offset| *offset - first_offset) + .collect::>(); + (OffsetBuffer::new(ScalarBuffer::from(offsets)), values) + } else { + (list_arr.offsets().clone(), list_arr.values().clone()) + }; + + let child_field = match field.data_type() { + ArrowDataType::List(child_field) | ArrowDataType::LargeList(child_field) => { + child_field.clone() + } + other => { + return Err(Error::invalid_input(format!( + "Blob list preprocessor expected list field '{}', got {other}", + field.name() + ))); + } + }; + let (new_values, new_child_field) = + self.preprocess_field(child, values, &child_field).await?; + + let list_array = GenericListArray::::try_new( + new_child_field, + offsets, + new_values, + list_arr.nulls().cloned(), + )?; + let field = Arc::new( + ArrowField::new( + field.name(), + list_array.data_type().clone(), + field.is_nullable(), + ) + .with_metadata(field.metadata().clone()), + ); + Ok((Arc::new(list_array), field)) + } + + async fn logical_to_prepared_blob_array( &mut self, array: ArrayRef, field: &ArrowField, @@ -729,6 +1131,15 @@ impl BlobPreprocessor { .as_any() .downcast_ref::() .ok_or_else(|| Error::invalid_input("Blob column was not a struct array"))?; + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Logical) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has {actual} array layout; expected logical layout before preparation", + field.name() + ))); + } let data_col = struct_arr .column_by_name("data") @@ -763,6 +1174,32 @@ impl BlobPreprocessor { .as_ref() .map(|col| !col.is_null(i)) .unwrap_or(false); + + if has_position != has_size { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' row {i} must set both `position` and `size`, or neither", + field.name() + ))); + } + if has_position && !has_uri { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' row {i} sets `position` and `size` but `uri` is null", + field.name() + ))); + } + if has_data == has_uri { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' row {i} must set exactly one of `data` and `uri`", + field.name() + ))); + } + if has_size && size_col.as_ref().is_some_and(|col| col.value(i) == 0) { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' row {i} external range `size` must be greater than zero", + field.name() + ))); + } + let data_len = if has_data { data_col.value(i).len() } else { 0 }; if has_data && data_len > dedicated_threshold { @@ -873,6 +1310,10 @@ impl BlobPreprocessor { pub(crate) async fn finish(&mut self) -> Result<()> { self.pack_writer.finish().await } + + pub(super) fn abort(&mut self) { + self.pack_writer.abort(); + } } pub async fn preprocess_blob_batches( @@ -987,7 +1428,9 @@ impl BlobSource { /// Drain currently queued requests and submit them as scheduler batches. /// /// Each loop iteration grabs the queued requests with a short mutex hold and - /// immediately releases the lock before any I/O is awaited. + /// dispatches them without waiting for earlier batches to finish. Awaiting a + /// batch here would hold later, naturally staggered callers behind its I/O. + /// [`FileScheduler`] owns the concurrency and backpressure for dispatched I/O. async fn drain_pending_reads(self: Arc, scheduler: FileScheduler) { loop { let batch = { @@ -998,7 +1441,10 @@ impl BlobSource { } std::mem::take(&mut pending_reads.requests) }; - fulfill_pending_blob_reads(&scheduler, batch).await; + let scheduler = scheduler.clone(); + tokio::spawn(async move { + fulfill_pending_blob_reads(&scheduler, batch).await; + }); } } } @@ -1468,13 +1914,80 @@ impl BlobFile { pub struct ReadBlob { /// Row address of the blob that was read. pub row_address: u64, - /// Blob payload bytes. - pub data: Bytes, + /// Blob payload bytes, or `None` when the selected blob value is null. + /// + /// A valid empty blob is represented as `Some(Bytes::new())`. + pub data: Option, +} + +/// A byte range relative to the beginning of one logical blob value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobReadRange { + /// Byte offset from the beginning of the blob value. + pub offset: u64, + /// Number of bytes to read. + pub length: u64, +} + +impl BlobReadRange { + /// Create a blob-local byte range from an offset and length. + pub const fn new(offset: u64, length: u64) -> Self { + Self { offset, length } + } + + fn checked_range(self, request_index: usize) -> Result> { + let end = self.offset.checked_add(self.length).ok_or_else(|| { + Error::invalid_input(format!( + "Blob range request {request_index} offset + length overflowed u64: offset={}, length={}", + self.offset, self.length + )) + })?; + Ok(self.offset..end) + } +} + +/// One row-specific blob range read request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BlobRangeRequest { + /// Row selector interpreted by the selection method on + /// [`ReadBlobRangesBuilder`]. + pub row: u64, + /// Blob-local byte range to read from the selected row. + pub range: BlobReadRange, +} + +impl BlobRangeRequest { + /// Create a request for `length` bytes at `offset` in the selected row. + pub const fn new(row: u64, offset: u64, length: u64) -> Self { + Self { + row, + range: BlobReadRange::new(offset, length), + } + } +} + +/// Bytes materialized for one request submitted through [`ReadBlobRangesBuilder`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReadBlobRange { + /// Zero-based position of this request in the caller's request list. + pub request_index: usize, + /// Physical row address of the selected blob value. + pub row_address: u64, + /// Blob-local range supplied for this request. + pub range: BlobReadRange, + /// Bytes in `range`, or `None` when the selected blob value is null. + /// + /// An empty range on a non-null blob is represented as + /// `Some(Bytes::new())`. + pub data: Option, } /// Stream returned by [`ReadBlobsBuilder::try_into_stream`]. pub type ReadBlobsStream = BoxStream<'static, Result>; +/// Stream returned by [`ReadBlobRangesBuilder::try_into_stream`]. +pub type ReadBlobRangesStream = BoxStream<'static, Result>; + /// Row selector configured on [`ReadBlobsBuilder`]. #[derive(Debug, Clone)] enum ReadBlobsSelection { @@ -1484,6 +1997,43 @@ enum ReadBlobsSelection { RowAddresses(Vec), } +#[derive(Debug, Clone)] +enum ReadBlobRangesSelection { + None, + RowIds(Vec), + RowIndices(Vec), + RowAddresses(Vec), +} + +impl ReadBlobRangesSelection { + fn requests(&self) -> Option<&[BlobRangeRequest]> { + match self { + Self::None => None, + Self::RowIds(requests) | Self::RowIndices(requests) | Self::RowAddresses(requests) => { + Some(requests) + } + } + } + + fn into_row_selection(self) -> Option<(ReadBlobsSelection, Vec)> { + match self { + Self::None => None, + Self::RowIds(requests) => { + let rows = requests.iter().map(|request| request.row).collect(); + Some((ReadBlobsSelection::RowIds(rows), requests)) + } + Self::RowIndices(requests) => { + let rows = requests.iter().map(|request| request.row).collect(); + Some((ReadBlobsSelection::RowIndices(rows), requests)) + } + Self::RowAddresses(requests) => { + let rows = requests.iter().map(|request| request.row).collect(); + Some((ReadBlobsSelection::RowAddresses(rows), requests)) + } + } + } +} + /// Planner knobs for [`ReadBlobsBuilder`]. /// /// Options that shape how `read_blobs` uses Lance's existing schedulers. @@ -1559,77 +2109,51 @@ impl ReadBlobsBuilder { /// Execute the planned blob read and return a stream of blob payloads. /// - /// The stream yields one [`ReadBlob`] per selected non-null blob row. + /// The stream yields one [`ReadBlob`] per selected row. Null blob values + /// have `data` set to `None`; valid empty blobs contain an empty buffer. pub async fn try_into_stream(self) -> Result { self.validate()?; - let entries = collect_blob_entries_for_selection( + let collected = collect_blob_selection_for_selection( &self.dataset, self.blob_field_id, &self.column, &self.selection, ) .await?; - let expected_selection_indices = entries - .iter() - .map(|entry| entry.selection_index) - .collect::>(); - let plans = plan_blob_read_plans(entries); - let execution = Arc::new(ReadBlobsExecution::new(self.options.io_buffer_size_bytes)); - if plans.is_empty() { - return Ok(stream::empty().boxed()); - } - - let plan_stream = stream::iter(plans.into_iter().map(move |plan| { - let execution = execution.clone(); - execute_blob_read_plan(plan, execution) - })) - .buffer_unordered(self.dataset.object_store.io_parallelism().max(1)); - - if !self.options.preserve_order { - return Ok(plan_stream - .map_ok(|blobs| { - stream::iter(blobs.into_iter().map(|blob| Ok(into_read_blob(blob)))) - }) - .try_flatten() - .boxed()); - } - - let mut plan_stream = plan_stream.boxed(); - let mut expected_selection_indices = expected_selection_indices; - let mut ready = BTreeMap::::new(); - - Ok(stream::poll_fn(move |cx| { - loop { - let Some(next_selection_index) = expected_selection_indices.front().copied() else { - return Poll::Ready(None); - }; - - if let Some(blob) = ready.remove(&next_selection_index) { - expected_selection_indices.pop_front(); - return Poll::Ready(Some(Ok(blob))); - } - - match plan_stream.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(blobs))) => { - for blob in blobs { - ready.insert(blob.selection_index, into_read_blob(blob)); - } - } - Poll::Ready(Some(Err(err))) => { - return Poll::Ready(Some(Err(err))); - } - Poll::Ready(None) => { - let err = Error::internal(format!( - "planned blob read stream completed before selection index {} was produced", - next_selection_index - )); - return Poll::Ready(Some(Err(err))); - } - Poll::Pending => return Poll::Pending, + let partitioned = + partition_blob_selections(collected, |selection_index, row_address, range| { + if range.is_some() { + return Err(Error::internal(format!( + "Whole-blob selection {} unexpectedly carried range metadata", + selection_index + ))); } - } + Ok(ReadBlob { + row_address, + data: None, + }) + })?; + let physical_buffer_size = self.options.io_buffer_size_bytes.unwrap_or_else(|| { + SchedulerConfig::max_bandwidth(self.dataset.object_store.as_ref()).io_buffer_size_bytes + }); + let batches = plan_blob_read_batches(partitioned.reads, physical_buffer_size)?; + let execution = Arc::new(ReadBlobsExecution::new(self.options.io_buffer_size_bytes)); + let non_null_blobs = execute_blob_read_batches_stream( + batches, + execution, + self.dataset.object_store.io_parallelism(), + ) + .map_ok(|blob| { + let selection_index = blob.selection_index; + (selection_index, into_read_blob(blob)) }) - .boxed()) + .boxed(); + Ok(totalize_blob_selection_stream( + non_null_blobs, + partitioned.ready_values, + partitioned.selection_count, + self.options.preserve_order, + )) } /// Execute the planned blob read and collect the full result in memory. @@ -1650,12 +2174,177 @@ impl ReadBlobsBuilder { } } -/// One logical blob selected for planned reading. -#[derive(Debug)] -struct BlobEntry { - selection_index: usize, - row_address: u64, - file: BlobFile, +/// Builder for planned blob-local range reads. +/// +/// Planning retains `O(request_count)` metadata. [`Self::try_into_stream`] +/// bounds each scheduler-visible physical batch by the configured I/O buffer +/// size after accounting for range coalescing, except that one request larger +/// than the buffer must be materialized by itself. The existing scheduler +/// separately applies byte backpressure to physical I/O. [`Self::execute`] +/// additionally retains all returned payload bytes. +#[derive(Debug, Clone)] +pub struct ReadBlobRangesBuilder { + inner: ReadBlobsBuilder, + selection: ReadBlobRangesSelection, +} + +impl ReadBlobRangesBuilder { + pub(crate) fn new(inner: ReadBlobsBuilder) -> Self { + Self { + inner, + selection: ReadBlobRangesSelection::None, + } + } + + /// Read requests whose row values are stable row ids. + pub fn with_row_ids(mut self, requests: impl IntoIterator) -> Self { + self.selection = ReadBlobRangesSelection::RowIds(requests.into_iter().collect()); + self + } + + /// Read requests whose row values are offsets in dataset order. + pub fn with_row_indices( + mut self, + requests: impl IntoIterator, + ) -> Self { + self.selection = ReadBlobRangesSelection::RowIndices(requests.into_iter().collect()); + self + } + + /// Read requests whose row values are physical row addresses. + pub fn with_row_addresses( + mut self, + requests: impl IntoIterator, + ) -> Self { + self.selection = ReadBlobRangesSelection::RowAddresses(requests.into_iter().collect()); + self + } + + /// Set the scheduler I/O buffer size used while materializing ranges. + pub fn with_io_buffer_size_bytes(mut self, bytes: u64) -> Self { + self.inner.options.io_buffer_size_bytes = Some(bytes); + self + } + + /// Whether results must follow the caller's request order. + pub fn preserve_order(mut self, preserve: bool) -> Self { + self.inner.options.preserve_order = preserve; + self + } + + /// Execute the planned range read and return one result per request. + /// + /// Null blob requests have `data` set to `None`. Empty ranges on non-null + /// blobs contain an empty buffer without issuing payload I/O. By default, + /// results follow input order; `request_index` identifies the original + /// request even when ordering is disabled. Blob-local bounds are not + /// evaluated for null values because they have no logical payload length. + pub async fn try_into_stream(self) -> Result { + self.validate()?; + let (row_selection, requests) = self.selection.into_row_selection().ok_or_else(|| { + Error::internal("Validated blob range selection was missing".to_string()) + })?; + let mut selections = collect_blob_selection_for_selection( + &self.inner.dataset, + self.inner.blob_field_id, + &self.inner.column, + &row_selection, + ) + .await?; + if selections.len() != requests.len() { + return Err(Error::internal(format!( + "Resolved blob selection count {} did not match range request count {}", + selections.len(), + requests.len() + ))); + } + for (selection, request) in selections.iter_mut().zip(requests) { + selection.requested_range = Some(request.range); + } + let partitioned = partition_blob_selections( + selections, + |request_index, row_address, requested_range| { + let range = requested_range.ok_or_else(|| { + Error::internal(format!( + "Blob range request {} was missing range metadata", + request_index + )) + })?; + Ok(ReadBlobRange { + request_index, + row_address, + range, + data: None, + }) + }, + )?; + let physical_buffer_size = self.inner.options.io_buffer_size_bytes.unwrap_or_else(|| { + SchedulerConfig::max_bandwidth(self.inner.dataset.object_store.as_ref()) + .io_buffer_size_bytes + }); + let batches = plan_blob_read_batches(partitioned.reads, physical_buffer_size)?; + let execution = Arc::new(ReadBlobsExecution::new( + self.inner.options.io_buffer_size_bytes, + )); + let non_null_ranges = execute_blob_read_batches_stream( + batches, + execution, + self.inner.dataset.object_store.io_parallelism(), + ) + .map(|result| { + result.and_then(|blob| { + let selection_index = blob.selection_index; + into_read_blob_range(blob).map(|range| (selection_index, range)) + }) + }) + .boxed(); + Ok(totalize_blob_selection_stream( + non_null_ranges, + partitioned.ready_values, + partitioned.selection_count, + self.inner.options.preserve_order, + )) + } + + /// Execute the planned range read and collect all returned bytes in memory. + pub async fn execute(self) -> Result> { + self.try_into_stream().await?.try_collect().await + } + + fn validate(&self) -> Result<()> { + let requests = self.selection.requests().ok_or_else(|| { + Error::invalid_input( + "ReadBlobRangesBuilder requires requests; call one of with_row_ids, with_row_indices, or with_row_addresses".to_string(), + ) + })?; + if self.inner.options.io_buffer_size_bytes == Some(0) { + return Err(Error::invalid_input( + "ReadBlobRangesBuilder io_buffer_size must be greater than 0".to_string(), + )); + } + for (request_index, request) in requests.iter().enumerate() { + request.range.checked_range(request_index)?; + } + Ok(()) + } +} + +/// One logical blob selected for planned reading. +#[derive(Debug)] +struct BlobEntry { + selection_index: usize, + row_address: u64, + file: BlobFile, + requested_range: Option, +} + +/// One resolved logical selection. Its position in the containing vector is +/// the caller's selection index. +#[derive(Debug)] +struct ResolvedBlobSelection { + row_address: u64, + file: Option, + requested_range: Option, } /// Physical read input derived from one [`BlobEntry`]. @@ -1663,9 +2352,26 @@ struct BlobEntry { struct PlannedBlobRead { selection_index: usize, row_address: u64, + requested_range: Option, physical_range: Range, } +/// A slice of one disjoint physical range submitted to the file scheduler. +#[derive(Debug)] +struct PlannedBlobReadSlice { + read_index: usize, + physical_range_index: usize, + relative_range: Range, +} + +/// One physical read paired with the backing source used to batch it. +#[derive(Debug)] +struct SourcePlannedBlobRead { + source_key: BlobSourceKey, + source: Arc, + read: PlannedBlobRead, +} + /// One per-source read plan emitted by `read_blobs`. #[derive(Debug)] struct BlobReadPlan { @@ -1674,6 +2380,126 @@ struct BlobReadPlan { reads: Vec, } +/// A payload-bounded group of per-source plans. +#[derive(Debug)] +struct BlobReadBatch { + plans: Vec, +} + +#[derive(Debug)] +struct CoalescedRangeInsertion { + merged: Range, + replaced_starts: Vec, + replaced_bytes: u64, +} + +impl CoalescedRangeInsertion { + fn additional_bytes(&self) -> Result { + let merged_bytes = self.merged.end - self.merged.start; + merged_bytes.checked_sub(self.replaced_bytes).ok_or_else(|| { + Error::internal(format!( + "Coalesced blob range {:?} was smaller than its replaced ranges totaling {} bytes", + self.merged, self.replaced_bytes + )) + }) + } +} + +/// Physical ranges already charged to one source in the current batch. +#[derive(Debug, Default)] +struct CoalescedPhysicalRanges { + ranges: BTreeMap, +} + +impl CoalescedPhysicalRanges { + fn plan_insertion( + &self, + range: &Range, + block_size: u64, + ) -> Result> { + if range.is_empty() { + return Ok(None); + } + + let scan_start = self + .ranges + .range(..=range.start) + .next_back() + .map(|(start, _)| *start) + .unwrap_or(range.start); + let mut merged = range.clone(); + let mut replaced_starts = Vec::new(); + let mut replaced_bytes = 0_u64; + for (&start, &end) in self.ranges.range(scan_start..) { + if end.saturating_add(block_size) < merged.start { + continue; + } + if start > merged.end.saturating_add(block_size) { + break; + } + merged.start = merged.start.min(start); + merged.end = merged.end.max(end); + replaced_starts.push(start); + replaced_bytes = replaced_bytes + .checked_add(end - start) + .ok_or_else(|| Error::internal("Coalesced blob range size overflow".to_string()))?; + } + + Ok(Some(CoalescedRangeInsertion { + merged, + replaced_starts, + replaced_bytes, + })) + } + + fn insert(&mut self, insertion: CoalescedRangeInsertion) { + for start in insertion.replaced_starts { + self.ranges.remove(&start); + } + self.ranges + .insert(insertion.merged.start, insertion.merged.end); + } +} + +/// Exact scheduler-visible physical footprint of the current batch. +#[derive(Debug, Default)] +struct BlobReadBatchFootprint { + sources: HashMap, + physical_bytes: u64, +} + +impl BlobReadBatchFootprint { + fn try_insert( + &mut self, + planned: &SourcePlannedBlobRead, + physical_buffer_size: u64, + allow_oversized: bool, + ) -> Result { + let remaining = physical_buffer_size.saturating_sub(self.physical_bytes); + let additional_bytes = { + let source_ranges = self.sources.entry(planned.source_key.clone()).or_default(); + let insertion = source_ranges.plan_insertion( + &planned.read.physical_range, + planned.source.object_store.block_size() as u64, + )?; + let Some(insertion) = insertion else { + return Ok(true); + }; + let additional_bytes = insertion.additional_bytes()?; + if !allow_oversized && additional_bytes > remaining { + return Ok(false); + } + source_ranges.insert(insertion); + additional_bytes + }; + self.physical_bytes = self + .physical_bytes + .checked_add(additional_bytes) + .ok_or_else(|| Error::internal("Blob read batch size overflow".to_string()))?; + Ok(true) + } +} + /// Operation-scoped scheduler cache for one [`ReadBlobsBuilder`] execution. /// /// We reuse one [`ScanScheduler`] per object store during a single `read_blobs` @@ -1684,6 +2510,213 @@ struct ReadBlobsExecution { schedulers: std::sync::Mutex>>, } +#[derive(Debug)] +struct BlobMaterializationBudget { + limit: u64, + state: std::sync::Mutex, + notify: Notify, +} + +#[derive(Debug, Default)] +struct BlobMaterializationBudgetState { + reserved: u64, + next_ticket: u64, + serving_ticket: u64, + cancelled_tickets: HashSet, + #[cfg(test)] + peak_reserved: u64, +} + +impl BlobMaterializationBudgetState { + fn skip_cancelled(&mut self) { + while self.cancelled_tickets.remove(&self.serving_ticket) { + self.serving_ticket = self.serving_ticket.wrapping_add(1); + } + } +} + +impl BlobMaterializationBudget { + fn admission(self: &Arc) -> BlobMaterializationAdmission { + let ticket = { + let mut state = self.state.lock().unwrap(); + let ticket = state.next_ticket; + state.next_ticket = state.next_ticket.wrapping_add(1); + ticket + }; + BlobMaterializationAdmission { + budget: Some(self.clone()), + ticket, + acquired: false, + } + } + + #[cfg(test)] + async fn reserve(self: &Arc, bytes: u64) -> BlobMaterializationReservation { + self.admission().reserve(bytes).await.unwrap() + } +} + +pub struct BlobMaterializationAdmission { + budget: Option>, + ticket: u64, + acquired: bool, +} + +impl BlobMaterializationAdmission { + async fn reserve(mut self, bytes: u64) -> Option { + let Some(budget) = self.budget.clone() else { + self.acquired = true; + return None; + }; + loop { + let notified = budget.notify.notified(); + { + let mut state = budget.state.lock().unwrap(); + let fits = bytes <= budget.limit.saturating_sub(state.reserved); + let oversized_and_idle = state.reserved == 0 && bytes > budget.limit; + if self.ticket == state.serving_ticket && (fits || oversized_and_idle) { + state.reserved = state.reserved.saturating_add(bytes); + #[cfg(test)] + { + state.peak_reserved = state.peak_reserved.max(state.reserved); + } + state.serving_ticket = state.serving_ticket.wrapping_add(1); + state.skip_cancelled(); + self.acquired = true; + let reservation = BlobMaterializationReservation { + budget: budget.clone(), + bytes, + }; + drop(state); + budget.notify.notify_waiters(); + return Some(reservation); + } + } + notified.await; + } + } +} + +impl Drop for BlobMaterializationAdmission { + fn drop(&mut self) { + let Some(budget) = &self.budget else { + return; + }; + if self.acquired { + return; + } + let mut state = budget.state.lock().unwrap(); + if self.ticket >= state.serving_ticket { + state.cancelled_tickets.insert(self.ticket); + state.skip_cancelled(); + } + drop(state); + budget.notify.notify_waiters(); + } +} + +#[derive(Debug)] +struct BlobMaterializationReservation { + budget: Arc, + bytes: u64, +} + +impl Drop for BlobMaterializationReservation { + fn drop(&mut self) { + let mut state = self.budget.state.lock().unwrap(); + state.reserved = state.reserved.saturating_sub(self.bytes); + drop(state); + self.budget.notify.notify_waiters(); + } +} + +/// Shared state for asynchronously materializing blob v2 descriptor batches. +#[derive(Debug)] +pub struct BlobMaterializationContext { + execution: Arc, + budget: Option>, +} + +impl BlobMaterializationContext { + pub(crate) fn new( + io_buffer_size_bytes: Option, + materialization_readahead_bytes: Option, + ) -> Arc { + Arc::new(Self { + execution: Arc::new(ReadBlobsExecution::new(io_buffer_size_bytes)), + budget: materialization_readahead_bytes.map(|limit| { + Arc::new(BlobMaterializationBudget { + limit, + state: std::sync::Mutex::new(BlobMaterializationBudgetState::default()), + notify: Notify::new(), + }) + }), + }) + } + + pub(crate) fn admission(&self) -> BlobMaterializationAdmission { + match &self.budget { + Some(budget) => budget.admission(), + None => BlobMaterializationAdmission { + budget: None, + ticket: 0, + acquired: false, + }, + } + } + + #[cfg(test)] + pub(crate) fn peak_reserved_bytes(&self) -> u64 { + self.budget + .as_ref() + .map(|budget| budget.state.lock().unwrap().peak_reserved) + .unwrap_or(0) + } +} + +/// A materialized batch that retains its byte-budget reservation until yielded. +pub struct MaterializedBlobBatch { + batch: RecordBatch, + _reservations: Vec, +} + +impl MaterializedBlobBatch { + pub(crate) fn unreserved(batch: RecordBatch) -> Self { + Self { + batch, + _reservations: Vec::new(), + } + } + + pub(crate) fn batch(&self) -> &RecordBatch { + &self.batch + } + + pub(crate) fn with_batch(self, batch: RecordBatch) -> Self { + Self { + batch, + _reservations: self._reservations, + } + } + + pub(crate) fn concat(schema: &SchemaRef, batches: Vec) -> Result { + let mut record_batches = Vec::with_capacity(batches.len()); + let mut reservations = Vec::new(); + for batch in batches { + record_batches.push(batch.batch); + reservations.extend(batch._reservations); + } + Ok(Self { + batch: arrow::compute::concat_batches(schema, record_batches.iter())?, + _reservations: reservations, + }) + } + + pub(crate) fn into_batch(self) -> RecordBatch { + self.batch + } +} + impl ReadBlobsExecution { fn new(io_buffer_size_bytes: Option) -> Self { Self { @@ -1715,109 +2748,516 @@ impl ReadBlobsExecution { struct IndexedReadBlob { selection_index: usize, row_address: u64, + requested_range: Option, data: Bytes, } fn into_read_blob(blob: IndexedReadBlob) -> ReadBlob { + debug_assert!( + blob.requested_range.is_none(), + "whole-blob reads must not carry a requested range" + ); ReadBlob { row_address: blob.row_address, - data: blob.data, + data: Some(blob.data), } } -/// Group selected blobs by physical source and sort each group's ranges by -/// physical offset before handing them to the file scheduler. -fn plan_blob_read_plans(entries: Vec) -> Vec { - let mut plan_indices = HashMap::::new(); - let mut plans = Vec::::new(); - - for entry in entries { - let source_key = BlobSourceKey::new(&entry.file.source); - let plan_index = if let Some(plan_index) = plan_indices.get(&source_key) { - *plan_index - } else { - let plan_index = plans.len(); - plans.push(BlobReadPlan { - source_key: source_key.clone(), - source: entry.file.source.clone(), - reads: Vec::new(), - }); - plan_indices.insert(source_key.clone(), plan_index); - plan_index - }; +struct PartitionedBlobSelection { + selection_count: usize, + reads: Vec, + ready_values: Vec<(usize, T)>, +} - plans[plan_index].reads.push(PlannedBlobRead { - selection_index: entry.selection_index, - row_address: entry.row_address, - physical_range: entry.file.position..(entry.file.position + entry.file.size), - }); +fn partition_blob_selections( + selections: Vec, + mut make_ready_value: impl FnMut(usize, u64, Option) -> Result, +) -> Result> { + let selection_count = selections.len(); + let read_count = selections + .iter() + .filter(|selection| selection.file.is_some()) + .count(); + let mut reads = Vec::with_capacity(read_count); + let mut ready_values = Vec::with_capacity(selection_count - read_count); + for (selection_index, selection) in selections.into_iter().enumerate() { + match selection.file { + Some(file) => reads.push(BlobEntry { + selection_index, + row_address: selection.row_address, + file, + requested_range: selection.requested_range, + }), + None => { + let value = make_ready_value( + selection_index, + selection.row_address, + selection.requested_range, + )?; + ready_values.push((selection_index, value)); + } + } } + Ok(PartitionedBlobSelection { + selection_count, + reads, + ready_values, + }) +} - plans.sort_by(|left, right| { - left.source_key - .store_prefix - .cmp(&right.source_key.store_prefix) - .then_with(|| left.source_key.path.cmp(&right.source_key.path)) - }); - - for plan in &mut plans { - plan.reads.sort_by(|left, right| { - left.physical_range - .start - .cmp(&right.physical_range.start) - .then_with(|| left.physical_range.end.cmp(&right.physical_range.end)) - .then_with(|| left.selection_index.cmp(&right.selection_index)) - }); +fn totalize_blob_selection_stream( + read_values: BoxStream<'static, Result<(usize, T)>>, + ready_values: Vec<(usize, T)>, + selection_count: usize, + preserve_order: bool, +) -> BoxStream<'static, Result> { + if preserve_order { + return totalize_ordered_blob_selection_stream(read_values, ready_values, selection_count); } - plans + let ready_values = stream::iter(ready_values.into_iter().map(Ok::<(usize, T), Error>)); + let values = stream::select(ready_values, read_values).boxed(); + totalize_unordered_blob_selection_stream(values, selection_count) } -/// Execute one per-source blob read plan with a single scheduler submission. -async fn execute_blob_read_plan( - task: BlobReadPlan, - execution: Arc, -) -> Result> { - let ranges = task - .reads - .iter() - .map(|read| read.physical_range.clone()) - .collect::>(); - let scheduler = execution.scheduler_for(&task.source); - let file_scheduler = scheduler - .open_file(&task.source.path, &task.source.file_size) - .await?; - let priority = ranges[0].start; - let bytes = file_scheduler.submit_request(ranges, priority).await?; - - Ok(task - .reads - .into_iter() - .zip(bytes) - .map(|(read, data)| IndexedReadBlob { - selection_index: read.selection_index, - row_address: read.row_address, - data, - }) - .collect()) +struct UnorderedBlobSelectionState { + values: BoxStream<'static, Result<(usize, T)>>, + seen_selection_indices: Vec, + remaining: usize, +} + +fn totalize_unordered_blob_selection_stream( + values: BoxStream<'static, Result<(usize, T)>>, + selection_count: usize, +) -> BoxStream<'static, Result> { + let state = UnorderedBlobSelectionState { + values, + seen_selection_indices: vec![false; selection_count], + remaining: selection_count, + }; + stream::try_unfold(state, move |mut state| async move { + let Some((selection_index, value)) = state.values.try_next().await? else { + return if state.remaining == 0 { + Ok(None) + } else { + Err(Error::internal(format!( + "planned blob read stream completed after producing {} of {} selections", + selection_count - state.remaining, + selection_count + ))) + }; + }; + + if selection_index >= selection_count { + return Err(Error::internal(format!( + "Blob selection index {} exceeded selected row count {}", + selection_index, selection_count + ))); + } + if state.seen_selection_indices[selection_index] { + return Err(Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + ))); + } + state.seen_selection_indices[selection_index] = true; + state.remaining -= 1; + Ok(Some((value, state))) + }) + .boxed() +} + +struct OrderedBlobSelectionState { + read_values: BoxStream<'static, Result<(usize, T)>>, + ready: BTreeMap, + next_selection_index: usize, +} + +fn totalize_ordered_blob_selection_stream( + read_values: BoxStream<'static, Result<(usize, T)>>, + ready_values: Vec<(usize, T)>, + selection_count: usize, +) -> BoxStream<'static, Result> { + // Seed the reorder buffer so ordered consumers do not start later physical + // I/O while the next logical selection is already available. + let mut ready = BTreeMap::new(); + for (selection_index, value) in ready_values { + if selection_index >= selection_count { + let error = Error::internal(format!( + "Blob selection index {} exceeded selected row count {}", + selection_index, selection_count + )); + return stream::once(async move { Err(error) }).boxed(); + } + if ready.insert(selection_index, value).is_some() { + let error = Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + )); + return stream::once(async move { Err(error) }).boxed(); + } + } + let state = OrderedBlobSelectionState { + read_values, + ready, + next_selection_index: 0, + }; + stream::try_unfold(state, move |mut state| async move { + loop { + if let Some(value) = state.ready.remove(&state.next_selection_index) { + state.next_selection_index += 1; + return Ok(Some((value, state))); + } + + match state.read_values.try_next().await? { + Some((selection_index, value)) => { + if selection_index >= selection_count { + return Err(Error::internal(format!( + "Blob selection index {} exceeded selected row count {}", + selection_index, selection_count + ))); + } + if selection_index < state.next_selection_index { + return Err(Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + ))); + } + if state.ready.insert(selection_index, value).is_some() { + return Err(Error::internal(format!( + "Blob selection index {} was produced more than once", + selection_index + ))); + } + } + None if state.next_selection_index == selection_count => return Ok(None), + None => { + return Err(Error::internal(format!( + "planned blob read stream completed before selection index {} was produced", + state.next_selection_index + ))); + } + } + } + }) + .boxed() +} + +fn into_read_blob_range(blob: IndexedReadBlob) -> Result { + let range = blob.requested_range.ok_or_else(|| { + Error::internal(format!( + "Blob range request {} completed without range metadata", + blob.selection_index + )) + })?; + Ok(ReadBlobRange { + request_index: blob.selection_index, + row_address: blob.row_address, + range, + data: Some(blob.data), + }) +} + +/// Split selected reads into request-order batches before grouping each batch +/// by physical source. Each batch accounts for the physical spans that the file +/// scheduler will read after coalescing nearby ranges. +fn plan_blob_read_batches( + mut entries: Vec, + physical_buffer_size: u64, +) -> Result> { + debug_assert!(physical_buffer_size > 0); + entries.sort_by_key(|entry| entry.selection_index); + let planned_reads = plan_blob_reads(entries)?; + + let mut batches = Vec::new(); + let mut current_reads = Vec::new(); + let mut footprint = BlobReadBatchFootprint::default(); + for planned in planned_reads { + if !footprint.try_insert(&planned, physical_buffer_size, current_reads.is_empty())? { + batches.push(into_blob_read_batch(std::mem::take(&mut current_reads))); + footprint = BlobReadBatchFootprint::default(); + let inserted = footprint.try_insert(&planned, physical_buffer_size, true)?; + debug_assert!(inserted, "an empty batch must accept one physical read"); + } + current_reads.push(planned); + } + if !current_reads.is_empty() { + batches.push(into_blob_read_batch(current_reads)); + } + + Ok(batches) +} + +fn plan_blob_reads(entries: Vec) -> Result> { + let mut planned_reads = Vec::with_capacity(entries.len()); + for entry in entries { + let logical_range = match entry.requested_range { + Some(range) => range.checked_range(entry.selection_index)?, + None => 0..entry.file.size, + }; + let physical_range = entry.file.read_phys_range(logical_range).map_err(|err| { + Error::invalid_input(format!( + "Blob range request {} for row address {} is invalid: {}", + entry.selection_index, entry.row_address, err + )) + })?; + let source_key = BlobSourceKey::new(&entry.file.source); + planned_reads.push(SourcePlannedBlobRead { + source_key, + source: entry.file.source.clone(), + read: PlannedBlobRead { + selection_index: entry.selection_index, + row_address: entry.row_address, + requested_range: entry.requested_range, + physical_range, + }, + }); + } + + Ok(planned_reads) +} + +fn into_blob_read_batch(planned_reads: Vec) -> BlobReadBatch { + BlobReadBatch { + plans: group_blob_read_plans(planned_reads), + } +} + +/// Group selected blobs by physical source and sort each group's ranges by +/// physical offset before handing them to the file scheduler. +fn group_blob_read_plans(planned_reads: Vec) -> Vec { + let mut plan_indices = HashMap::::new(); + let mut plans = Vec::::new(); + + for planned in planned_reads { + let plan_index = if let Some(plan_index) = plan_indices.get(&planned.source_key) { + *plan_index + } else { + let plan_index = plans.len(); + plans.push(BlobReadPlan { + source_key: planned.source_key.clone(), + source: planned.source, + reads: Vec::new(), + }); + plan_indices.insert(planned.source_key, plan_index); + plan_index + }; + plans[plan_index].reads.push(planned.read); + } + + plans.sort_by(|left, right| { + left.source_key + .store_prefix + .cmp(&right.source_key.store_prefix) + .then_with(|| left.source_key.path.cmp(&right.source_key.path)) + }); + + for plan in &mut plans { + plan.reads.sort_by(|left, right| { + left.physical_range + .start + .cmp(&right.physical_range.start) + .then_with(|| left.physical_range.end.cmp(&right.physical_range.end)) + .then_with(|| left.selection_index.cmp(&right.selection_index)) + }); + } + + plans +} + +fn plan_blob_read_plans(entries: Vec) -> Result> { + Ok(group_blob_read_plans(plan_blob_reads(entries)?)) +} + +/// Merge overlapping physical reads before submitting them to [`FileScheduler`]. +/// +/// `FileScheduler` can safely coalesce and split disjoint ranges. Original +/// overlapping ranges need to be mapped onto their union first so a split does +/// not advance past the start of a later nested range while reconstructing the +/// caller's buffers. +fn plan_disjoint_blob_reads( + reads: &[PlannedBlobRead], +) -> (Vec>, Vec) { + let mut non_empty_ranges = reads + .iter() + .enumerate() + .filter(|(_, read)| !read.physical_range.is_empty()) + .map(|(read_index, read)| (read_index, read.physical_range.clone())) + .collect::>(); + non_empty_ranges.sort_by_key(|(read_index, range)| (range.start, range.end, *read_index)); + + let mut physical_ranges = Vec::>::with_capacity(non_empty_ranges.len()); + let mut slices = Vec::with_capacity(non_empty_ranges.len()); + for (read_index, range) in non_empty_ranges { + let physical_range_index = match physical_ranges.last_mut() { + Some(physical_range) if range.start <= physical_range.end => { + physical_range.end = physical_range.end.max(range.end); + physical_ranges.len() - 1 + } + _ => { + physical_ranges.push(range.clone()); + physical_ranges.len() - 1 + } + }; + let physical_start = physical_ranges[physical_range_index].start; + slices.push(PlannedBlobReadSlice { + read_index, + physical_range_index, + relative_range: range.start - physical_start..range.end - physical_start, + }); + } + + (physical_ranges, slices) +} + +/// Execute one per-source blob read plan with a single scheduler submission. +async fn execute_blob_read_plan( + task: BlobReadPlan, + execution: Arc, +) -> Result> { + let (physical_ranges, slices) = plan_disjoint_blob_reads(&task.reads); + let mut bytes = vec![Bytes::new(); task.reads.len()]; + if let Some(first_range) = physical_ranges.first() { + let scheduler = execution.scheduler_for(&task.source); + let file_scheduler = scheduler + .open_file(&task.source.path, &task.source.file_size) + .await?; + let priority = first_range.start; + let physical_range_count = physical_ranges.len(); + let returned = file_scheduler + .submit_request(physical_ranges, priority) + .await?; + if returned.len() != physical_range_count { + return Err(Error::internal(format!( + "Blob read scheduler returned {} ranges for {} disjoint physical ranges from {}", + returned.len(), + physical_range_count, + task.source.path + ))); + } + for slice in slices { + let start = usize::try_from(slice.relative_range.start).map_err(|_| { + Error::internal(format!( + "Blob read slice start {} does not fit into usize for {}", + slice.relative_range.start, task.source.path + )) + })?; + let end = usize::try_from(slice.relative_range.end).map_err(|_| { + Error::internal(format!( + "Blob read slice end {} does not fit into usize for {}", + slice.relative_range.end, task.source.path + )) + })?; + let data = &returned[slice.physical_range_index]; + if end > data.len() { + return Err(Error::internal(format!( + "Blob read slice {:?} exceeds the {} bytes returned for physical range {} from {}", + slice.relative_range, + data.len(), + slice.physical_range_index, + task.source.path + ))); + } + bytes[slice.read_index] = data.slice(start..end); + } + } + + Ok(task + .reads + .into_iter() + .zip(bytes) + .map(|(read, data)| IndexedReadBlob { + selection_index: read.selection_index, + row_address: read.row_address, + requested_range: read.requested_range, + data, + }) + .collect()) +} + +fn execute_blob_read_batches_stream( + batches: Vec, + execution: Arc, + io_parallelism: usize, +) -> BoxStream<'static, Result> { + let streams = batches.into_iter().map(move |batch| { + execute_blob_read_plans_stream(batch.plans, execution.clone(), io_parallelism) + }); + stream::iter(streams).flatten().boxed() +} + +fn execute_blob_read_plans_stream( + plans: Vec, + execution: Arc, + io_parallelism: usize, +) -> BoxStream<'static, Result> { + if plans.is_empty() { + return stream::empty().boxed(); + } + + let plan_stream = stream::iter(plans.into_iter().map(move |plan| { + let execution = execution.clone(); + execute_blob_read_plan(plan, execution) + })) + .buffer_unordered(io_parallelism.max(1)); + + plan_stream + .map_ok(|blobs| stream::iter(blobs.into_iter().map(Ok))) + .try_flatten() + .boxed() +} + +#[cfg(test)] +async fn execute_blob_entries( + entries: Vec, + io_parallelism: usize, + io_buffer_size_bytes: Option, +) -> Result> { + execute_blob_entries_with_execution( + entries, + io_parallelism, + Arc::new(ReadBlobsExecution::new(io_buffer_size_bytes)), + ) + .await +} + +async fn execute_blob_entries_with_execution( + entries: Vec, + io_parallelism: usize, + execution: Arc, +) -> Result> { + let plans = plan_blob_read_plans(entries)?; + if plans.is_empty() { + return Ok(Vec::new()); + } + + let batches = stream::iter(plans.into_iter().map(move |plan| { + let execution = execution.clone(); + execute_blob_read_plan(plan, execution) + })) + .buffer_unordered(io_parallelism.max(1)) + .try_collect::>() + .await?; + Ok(batches.into_iter().flatten().collect()) } pub(super) async fn take_blobs( dataset: &Arc, row_ids: &[u64], column: &str, -) -> Result> { +) -> Result>> { let blob_field_id = validate_blob_column(dataset, column)?; - Ok(collect_blob_entries_for_selection( + let collected = collect_blob_selection_for_selection( dataset, blob_field_id, column, &ReadBlobsSelection::RowIds(row_ids.to_vec()), ) - .await? - .into_iter() - .map(|entry| entry.file) - .collect()) + .await?; + Ok(collected + .into_iter() + .map(|selection| selection.file) + .collect()) } /// Take [BlobFile] by row addresses. @@ -1831,18 +3271,19 @@ pub async fn take_blobs_by_addresses( dataset: &Arc, row_addrs: &[u64], column: &str, -) -> Result> { +) -> Result>> { let blob_field_id = validate_blob_column(dataset, column)?; - Ok(collect_blob_entries_for_selection( + let collected = collect_blob_selection_for_selection( dataset, blob_field_id, column, &ReadBlobsSelection::RowAddresses(row_addrs.to_vec()), ) - .await? - .into_iter() - .map(|entry| entry.file) - .collect()) + .await?; + Ok(collected + .into_iter() + .map(|selection| selection.file) + .collect()) } /// Validate that `column` exists and is a blob column, returning its field id. @@ -1868,6 +3309,7 @@ async fn take_blob_descriptions_by_row_ids( let projection = dataset.schema().project(&[column])?; dataset .take_builder(row_ids, projection)? + .with_missing_row_policy(MissingRowPolicy::Error) .with_row_address(true) .execute() .await @@ -1888,14 +3330,13 @@ async fn take_blob_descriptions_by_row_addresses( .await } -/// Resolve a caller selection into [`BlobEntry`] values that share `BlobSource` -/// instances by physical backing object. -async fn collect_blob_entries_for_selection( +/// Resolve every selected row address and the non-null blob entries among them. +async fn collect_blob_selection_for_selection( dataset: &Arc, blob_field_id: u32, column: &str, selection: &ReadBlobsSelection, -) -> Result> { +) -> Result> { let description_and_addr = match selection { ReadBlobsSelection::None => { return Err(Error::invalid_input( @@ -1923,256 +3364,994 @@ async fn collect_blob_entries_for_selection( let descriptions = leaf_descriptor_struct(&description_and_addr, column)?; let row_addrs = description_and_addr.column(1).as_primitive::(); - match blob_version_from_descriptions(descriptions)? { - BlobVersion::V1 => collect_blob_entries_v1(dataset, blob_field_id, descriptions, row_addrs), + let files = match blob_version_from_descriptions(descriptions)? { + BlobVersion::V1 => collect_blob_files_v1(dataset, blob_field_id, descriptions, row_addrs), BlobVersion::V2 => { - collect_blob_entries_v2(dataset, blob_field_id, descriptions, row_addrs).await + collect_blob_files_v2(dataset, blob_field_id, descriptions, row_addrs).await } + }?; + if files.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Resolved blob file count {} did not match row address count {}", + files.len(), + row_addrs.len() + ))); } + Ok(row_addrs + .values() + .iter() + .copied() + .zip(files) + .map(|(row_address, file)| ResolvedBlobSelection { + row_address, + file, + requested_range: None, + }) + .collect()) } /// Walk into the descriptor `RecordBatch` at `column` and return the leaf /// descriptor `StructArray`, descending through nested struct children for /// dotted paths. fn leaf_descriptor_struct<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a StructArray> { + let current = leaf_descriptor_array(batch, column)?; + current + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Blob column '{}' expected descriptor struct but got {}", + column, + current.data_type() + ) + .into(), + ) + }) +} + +fn leaf_descriptor_array<'a>(batch: &'a RecordBatch, column: &str) -> Result<&'a dyn Array> { let path = parse_field_path(column)?; - let mut current = batch + let mut current: &dyn Array = batch .column_by_name(&path[0]) - .expect("validate_blob_column ensured column exists") - .as_struct(); + .ok_or_else(|| { + Error::invalid_input(format!( + "Blob column '{}' was not found in descriptor batch", + column + )) + })? + .as_ref(); for segment in &path[1..] { - current = current + let struct_array = current + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input_source( + format!( + "Blob column path '{}' expected struct before segment '{}' but got {}", + column, + segment, + current.data_type() + ) + .into(), + ) + })?; + current = struct_array .column_by_name(segment) - .expect("validate_blob_column ensured all path segments exist") - .as_struct(); + .ok_or_else(|| { + Error::invalid_input(format!( + "Blob column path '{}' missing segment '{}'", + column, segment + )) + })? + .as_ref(); } Ok(current) } +fn blob_descriptor_fields_match( + actual: &arrow_schema::Fields, + expected: &arrow_schema::Fields, +) -> bool { + actual.len() == expected.len() + && actual + .iter() + .zip(expected.iter()) + .all(|(actual, expected)| { + actual.name() == expected.name() && actual.data_type() == expected.data_type() + }) +} + fn blob_version_from_descriptions(descriptions: &StructArray) -> Result { let fields = descriptions.fields(); - if fields.len() == 2 && fields[0].name() == "position" && fields[1].name() == "size" { + if blob_descriptor_fields_match(fields, &BLOB_DESC_FIELDS) { return Ok(BlobVersion::V1); } - if fields.len() == 5 - && fields[0].name() == "kind" - && fields[1].name() == "position" - && fields[2].name() == "size" - && fields[3].name() == "blob_id" - && fields[4].name() == "blob_uri" - { + if BlobV2Layout::classify(fields) == Some(BlobV2Layout::Descriptor) { return Ok(BlobVersion::V2); } Err(Error::invalid_input_source(format!( - "Unrecognized blob descriptions schema: expected v1 (position,size) or v2 (kind,position,size,blob_id,blob_uri) but got {:?}", - fields.iter().map(|f| f.name().as_str()).collect::>(), + "Unrecognized blob descriptions schema: expected v1 descriptor or v2 descriptor layout but got {fields:?}", ) .into())) } -/// Convert blob v1 descriptors into logical blob entries. -fn collect_blob_entries_v1( +struct BlobV2DescriptorColumns<'a> { + descriptions: &'a StructArray, + kinds: &'a arrow::array::PrimitiveArray, + positions: &'a arrow::array::PrimitiveArray, + sizes: &'a arrow::array::PrimitiveArray, + blob_ids: &'a arrow::array::PrimitiveArray, + blob_uris: &'a arrow::array::GenericStringArray, +} + +impl<'a> BlobV2DescriptorColumns<'a> { + fn new(descriptions: &'a StructArray) -> Self { + Self { + descriptions, + kinds: descriptions.column(0).as_primitive::(), + positions: descriptions.column(1).as_primitive::(), + sizes: descriptions.column(2).as_primitive::(), + blob_ids: descriptions.column(3).as_primitive::(), + blob_uris: descriptions.column(4).as_string::(), + } + } + + fn is_null_blob(&self, idx: usize) -> bool { + self.descriptions.is_null(idx) || self.kinds.is_null(idx) + } +} + +/// Resolve blob v1 descriptors without dropping null selection slots. +fn collect_blob_files_v1( dataset: &Arc, blob_field_id: u32, descriptions: &StructArray, row_addrs: &arrow::array::PrimitiveArray, -) -> Result> { +) -> Result>> { + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Blob descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); + } let positions = descriptions.column(0).as_primitive::(); let sizes = descriptions.column(1).as_primitive::(); let mut source_cache = HashMap::>::new(); - row_addrs - .values() - .iter() - .zip(positions.iter()) - .zip(sizes.iter()) - .enumerate() - .filter_map(|(selection_index, ((row_addr, position), size))| { - let position = position?; - let size = size?; - Some((selection_index, *row_addr, position, size)) - }) - .map(|(selection_index, row_addr, position, size)| { - let frag_id = RowAddress::from(row_addr).fragment_id(); - let frag = dataset.get_fragment(frag_id as usize).ok_or_else(|| { - Error::invalid_input(format!( - "Blob row address {} references missing fragment {}", - row_addr, frag_id - )) - })?; - let data_file = frag.data_file_for_field(blob_field_id).ok_or_else(|| { - Error::invalid_input(format!( - "Blob field {} has no data file in fragment {} for row address {}", - blob_field_id, frag_id, row_addr - )) - })?; - let data_file_path = dataset.data_dir().join(data_file.path.as_str()); - Ok(BlobEntry { - selection_index, - row_address: row_addr, - file: BlobFile::with_source( - shared_blob_source( - &mut source_cache, - dataset.object_store.clone(), - &data_file_path, - ), - position, - size, - BlobKind::Inline, - None, - ), - }) - }) - .collect() + let mut files = Vec::with_capacity(row_addrs.len()); + for selection_index in 0..row_addrs.len() { + if descriptions.is_null(selection_index) + || positions.is_null(selection_index) + || sizes.is_null(selection_index) + { + files.push(None); + continue; + } + + let position = positions.value(selection_index); + let size = sizes.value(selection_index); + // V1 encodes valid empty blobs as (0, 0) and smuggles null + // repetition/definition levels through a non-zero zero-sized position. + if size == 0 && position != 0 { + files.push(None); + continue; + } + + let row_addr = row_addrs.value(selection_index); + let frag_id = RowAddress::from(row_addr).fragment_id(); + let frag = dataset.get_fragment(frag_id as usize).ok_or_else(|| { + Error::invalid_input(format!( + "Blob row address {} references missing fragment {}", + row_addr, frag_id + )) + })?; + let data_file = frag.data_file_for_field(blob_field_id).ok_or_else(|| { + Error::invalid_input(format!( + "Blob field {} has no data file in fragment {} for row address {}", + blob_field_id, frag_id, row_addr + )) + })?; + let data_file_path = dataset.data_dir().join(data_file.path.as_str()); + files.push(Some(BlobFile::with_source( + shared_blob_source( + &mut source_cache, + dataset.object_store.clone(), + &data_file_path, + ), + position, + size, + BlobKind::Inline, + None, + ))); + } + Ok(files) } -/// Convert blob v2 descriptors into logical blob entries. -async fn collect_blob_entries_v2( +/// Resolve blob v2 descriptors without dropping null selection slots. +async fn collect_blob_files_v2( dataset: &Arc, blob_field_id: u32, descriptions: &StructArray, row_addrs: &arrow::array::PrimitiveArray, -) -> Result> { - let kinds = descriptions.column(0).as_primitive::(); - let positions = descriptions.column(1).as_primitive::(); - let sizes = descriptions.column(2).as_primitive::(); - let blob_ids = descriptions.column(3).as_primitive::(); - let blob_uris = descriptions.column(4).as_string::(); +) -> Result>> { + collect_blob_v2_descriptor_files(dataset, blob_field_id, descriptions, row_addrs.values()).await +} +/// Resolve blob v2 descriptors to lazy handles without materializing their payloads. +pub(super) async fn collect_blob_v2_descriptor_files( + dataset: &Arc, + blob_field_id: u32, + descriptions: &StructArray, + row_addrs: &[u64], +) -> Result>> { + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Blob descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); + } + let columns = BlobV2DescriptorColumns::new(descriptions); let mut files = Vec::with_capacity(row_addrs.len()); - let mut fragment_cache = HashMap::::new(); - let mut store_cache = HashMap::>::new(); - let mut external_base_path_cache = HashMap::::new(); - let mut source_cache = HashMap::>::new(); - for (selection_index, row_addr) in row_addrs.values().iter().enumerate() { - let idx = selection_index; - let kind = BlobKind::try_from(kinds.value(idx))?; + let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id); + for (selection_index, row_addr) in row_addrs.iter().enumerate() { + files.push( + read_context + .collect_file(&columns, selection_index, *row_addr) + .await?, + ); + } - // Struct is non-nullable; null rows are encoded as inline with zero position/size and empty uri - if matches!(kind, BlobKind::Inline) && positions.value(idx) == 0 && sizes.value(idx) == 0 { - continue; - } + Ok(files) +} - match kind { - BlobKind::Inline => { - let position = positions.value(idx); - let size = sizes.value(idx); - let location = resolve_blob_read_location( - dataset, - blob_field_id, - *row_addr, - &mut fragment_cache, - &mut store_cache, - ) - .await?; - let source = shared_blob_source( - &mut source_cache, - location.object_store, - &location.data_file_path, - ); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source(source, position, size, BlobKind::Inline, None), - }); +fn is_blob_v2_binary_view(field: &LanceField) -> bool { + field.is_blob_v2() && matches!(field.data_type(), ArrowDataType::LargeBinary) +} + +fn public_blob_v2_binary_output_field(mut field: LanceField) -> LanceField { + if is_blob_v2_binary_view(&field) { + field.metadata.remove(ARROW_EXT_NAME_KEY); + } + field.children = field + .children + .into_iter() + .map(public_blob_v2_binary_output_field) + .collect(); + field +} + +/// Return the public Arrow-facing schema for a blob v2 binary scan. +/// +/// Scan planning uses a blob v2 extension marker on `LargeBinary` leaves to +/// identify payloads that need descriptor-based materialization. This helper +/// removes that internal marker before the schema is exposed to callers. +pub fn public_blob_v2_binary_output_schema(schema: &Schema) -> Schema { + Schema { + fields: schema + .fields + .iter() + .cloned() + .map(public_blob_v2_binary_output_field) + .collect(), + metadata: schema.metadata.clone(), + } +} + +fn field_has_blob_v2_binary_view(field: &LanceField) -> bool { + is_blob_v2_binary_view(field) || field.children.iter().any(field_has_blob_v2_binary_view) +} + +/// Return true if the schema contains a blob v2 leaf in binary payload view. +/// +/// This detects the internal `LargeBinary` view created by +/// [`BlobHandling::AllBinary`](lance_core::datatypes::BlobHandling::AllBinary) +/// or selective binary blob handling. +pub fn schema_has_blob_v2_binary_view(schema: &Schema) -> bool { + schema.fields.iter().any(field_has_blob_v2_binary_view) +} + +fn blob_v2_descriptor_field(mut field: LanceField) -> LanceField { + if is_blob_v2_binary_view(&field) { + field.unloaded_mut(); + return field; + } + + field.children = field + .children + .into_iter() + .map(blob_v2_descriptor_field) + .collect(); + field +} + +/// Convert blob v2 binary-view leaves back to descriptor-view leaves. +/// +/// Readers use this schema to fetch stored blob descriptors first. The scan +/// layer then materializes those descriptors into the caller's binary payload +/// view after row addresses are available. +pub fn blob_v2_descriptor_schema(schema: &Schema) -> Schema { + Schema { + fields: schema + .fields + .iter() + .cloned() + .map(blob_v2_descriptor_field) + .collect(), + metadata: schema.metadata.clone(), + } +} + +/// Materialize blob v2 descriptor arrays in a decoded batch into binary arrays. +/// +/// The input batch must include `_rowaddr`, which is used to resolve packed, +/// dedicated, inline, and external blob payload locations. `output_schema` +/// defines the exact returned columns, including requested system columns, with +/// blob v2 binary leaves exposed as plain `LargeBinary` fields. +pub async fn materialize_blob_v2_binary_batch( + dataset: &Arc, + output_schema: &Schema, + batch: RecordBatch, +) -> Result { + let context = BlobMaterializationContext::new(None, None); + Ok( + materialize_blob_v2_binary_batch_with_context(dataset, output_schema, batch, &context) + .await? + .into_batch(), + ) +} + +pub fn materialize_blob_v2_binary_batch_with_context<'a>( + dataset: &'a Arc, + output_schema: &'a Schema, + batch: RecordBatch, + context: &'a Arc, +) -> BoxFuture<'a, Result> { + let admission = context.admission(); + materialize_blob_v2_binary_batch_with_admission( + dataset, + output_schema, + batch, + context, + admission, + ) +} + +pub fn materialize_blob_v2_binary_batch_with_admission<'a>( + dataset: &'a Arc, + output_schema: &'a Schema, + batch: RecordBatch, + context: &'a Arc, + admission: BlobMaterializationAdmission, +) -> BoxFuture<'a, Result> { + async move { + let materialized_bytes = + estimate_blob_v2_materialized_batch_bytes(dataset, output_schema, &batch).await?; + let reservation = admission.reserve(materialized_bytes).await; + let row_addr_idx = batch + .schema() + .column_with_name(ROW_ADDR) + .ok_or_else(|| { + Error::internal(format!( + "_rowaddr column missing from blob v2 binary scan batch, columns: {:?}", + batch + .schema() + .fields() + .iter() + .map(|field| field.name()) + .collect::>() + )) + })? + .0; + let row_addrs = batch + .column(row_addr_idx) + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let row_addrs: Arc<[u64]> = row_addrs.into(); + + let mut columns = Vec::with_capacity(output_schema.fields.len()); + let mut fields = Vec::with_capacity(output_schema.fields.len()); + + for field in &output_schema.fields { + let input = batch + .column_by_name(&field.name) + .ok_or_else(|| { + Error::internal(format!( + "blob v2 binary scan batch missing projected column '{}'", + field.name + )) + })? + .clone(); + let materialized = + materialize_blob_v2_binary_array(dataset, field, input, row_addrs.clone(), context) + .await?; + columns.push(materialized); + let output_field = public_blob_v2_binary_output_field(field.clone()); + fields.push(ArrowField::from(&output_field)); + } + + Ok(MaterializedBlobBatch { + batch: RecordBatch::try_new( + Arc::new(ArrowSchema::new_with_metadata( + fields, + batch.schema().metadata().clone(), + )), + columns, + )?, + _reservations: reservation.into_iter().collect(), + }) + } + .boxed() +} + +async fn estimate_blob_v2_materialized_batch_bytes( + dataset: &Arc, + output_schema: &Schema, + batch: &RecordBatch, +) -> Result { + let mut bytes = u64::try_from(batch.get_array_memory_size()).unwrap_or(u64::MAX); + let row_addr_idx = batch + .schema() + .column_with_name(ROW_ADDR) + .ok_or_else(|| Error::internal("_rowaddr missing while estimating blob materialization"))? + .0; + let row_addrs = batch + .column(row_addr_idx) + .as_primitive::() + .values(); + for field in &output_schema.fields { + let input = batch.column_by_name(&field.name).ok_or_else(|| { + Error::internal(format!( + "blob v2 binary scan batch missing projected column '{}'", + field.name + )) + })?; + bytes = bytes.saturating_add( + estimate_blob_v2_materialized_array_bytes(dataset, field, input, row_addrs.as_ref()) + .await?, + ); + } + Ok(bytes) +} + +fn estimate_blob_v2_materialized_array_bytes<'a>( + dataset: &'a Arc, + field: &'a LanceField, + array: &'a ArrayRef, + row_addrs: &'a [u64], +) -> BoxFuture<'a, Result> { + async move { + if is_blob_v2_binary_view(field) { + let descriptions = array.as_struct(); + match blob_version_from_descriptions(descriptions)? { + BlobVersion::V1 => { + return Err(Error::not_supported( + "Blob v2 binary materialization received a legacy blob descriptor" + .to_string(), + )); + } + BlobVersion::V2 => {} } - BlobKind::Dedicated => { - let blob_id = blob_ids.value(idx); - let size = sizes.value(idx); - let location = resolve_blob_read_location( - dataset, - blob_field_id, - *row_addr, - &mut fragment_cache, - &mut store_cache, - ) - .await?; - let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); - let source = shared_blob_source(&mut source_cache, location.object_store, &path); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source(source, 0, size, BlobKind::Dedicated, None), - }); + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "blob v2 descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); } - BlobKind::Packed => { - let blob_id = blob_ids.value(idx); - let size = sizes.value(idx); - let position = positions.value(idx); - let location = resolve_blob_read_location( - dataset, - blob_field_id, - *row_addr, - &mut fragment_cache, - &mut store_cache, - ) - .await?; - let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); - let source = shared_blob_source(&mut source_cache, location.object_store, &path); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source(source, position, size, BlobKind::Packed, None), - }); + let columns = BlobV2DescriptorColumns::new(descriptions); + let mut read_context = BlobV2ReadContext::new(dataset, field.id as u32); + let mut payload_bytes = 0_u64; + for (idx, row_addr) in row_addrs.iter().copied().enumerate() { + if columns.is_null_blob(idx) { + continue; + } + let kind = BlobKind::try_from(columns.kinds.value(idx))?; + if matches!(kind, BlobKind::Inline) + && columns.positions.value(idx) == 0 + && columns.sizes.value(idx) == 0 + { + continue; + } + let file = read_context + .collect_file(&columns, idx, row_addr) + .await? + .ok_or_else(|| { + Error::internal(format!( + "blob v2 descriptor at index {idx} unexpectedly resolved to null" + )) + })?; + payload_bytes = payload_bytes.saturating_add(file.size); } - BlobKind::External => { - let uri_or_path = blob_uris.value(idx).to_string(); - let position = positions.value(idx); - let size = sizes.value(idx); - let base_id = blob_ids.value(idx); - let (object_store, path) = if base_id == 0 { - let registry = dataset.session.store_registry(); - let params = dataset - .store_params - .as_ref() - .map(|p| Arc::new((**p).clone())) - .unwrap_or_else(|| Arc::new(ObjectStoreParams::default())); - ObjectStore::from_uri_and_params(registry, &uri_or_path, ¶ms).await? - } else { - let object_store = if let Some(store) = store_cache.get(&base_id) { - store.clone() - } else { - let store = dataset.object_store(Some(base_id)).await?; - store_cache.insert(base_id, store.clone()); - store - }; - let base_root = if let Some(path) = external_base_path_cache.get(&base_id) { - path.clone() - } else { - let base = dataset.manifest.base_paths.get(&base_id).ok_or_else(|| { - Error::invalid_input(format!( - "External blob references unknown base_id {}", - base_id - )) - })?; - let path = base.extract_path(dataset.session.store_registry())?; - external_base_path_cache.insert(base_id, path.clone()); - path - }; - let path = join_base_and_relative_path(&base_root, &uri_or_path)?; - (object_store, path) - }; - let size = if size > 0 { - size - } else { - object_store.size(&path).await? + let offsets_bytes = + u64::try_from((descriptions.len() + 1).saturating_mul(std::mem::size_of::())) + .unwrap_or(u64::MAX); + return Ok(payload_bytes.saturating_add(offsets_bytes)); + } + + match field.data_type() { + ArrowDataType::Struct(_) => { + let array = array.as_struct(); + let mut total = 0_u64; + for (child, array) in field.children.iter().zip(array.columns()) { + total = total.saturating_add( + estimate_blob_v2_materialized_array_bytes(dataset, child, array, row_addrs) + .await?, + ); + } + Ok(total) + } + ArrowDataType::List(_) => { + let array = array.as_list::(); + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "List field '{}' missing child while estimating blob v2 materialization", + field.name + )) + })?; + let (values_start, child_row_addrs) = + list_child_row_addrs(array.value_offsets(), row_addrs)?; + let values = array.values().slice(values_start, child_row_addrs.len()); + estimate_blob_v2_materialized_array_bytes(dataset, child, &values, &child_row_addrs) + .await + } + ArrowDataType::LargeList(_) => { + let array = array.as_list::(); + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "List field '{}' missing child while estimating blob v2 materialization", + field.name + )) + })?; + let (values_start, child_row_addrs) = + list_child_row_addrs(array.value_offsets(), row_addrs)?; + let values = array.values().slice(values_start, child_row_addrs.len()); + estimate_blob_v2_materialized_array_bytes(dataset, child, &values, &child_row_addrs) + .await + } + _ => Ok(0), + } + } + .boxed() +} + +fn list_child_row_addrs( + offsets: &[O], + row_addrs: &[u64], +) -> Result<(usize, Vec)> { + if offsets.len() != row_addrs.len() + 1 { + return Err(Error::internal( + "list offsets did not match row addresses while estimating blob materialization" + .to_string(), + )); + } + let values_start = offsets[0].as_usize(); + let values_end = offsets[row_addrs.len()].as_usize(); + let mut child_row_addrs = Vec::with_capacity(values_end.saturating_sub(values_start)); + for (row_idx, row_addr) in row_addrs.iter().copied().enumerate() { + let start = offsets[row_idx].as_usize(); + let end = offsets[row_idx + 1].as_usize(); + if end < start { + return Err(Error::internal( + "list offsets decreased while estimating blob materialization".to_string(), + )); + } + child_row_addrs.extend(std::iter::repeat_n(row_addr, end - start)); + } + Ok((values_start, child_row_addrs)) +} + +fn materialize_blob_v2_binary_array<'a>( + dataset: &'a Arc, + field: &'a LanceField, + array: ArrayRef, + row_addrs: Arc<[u64]>, + context: &'a Arc, +) -> BoxFuture<'a, Result> { + async move { + if is_blob_v2_binary_view(field) { + let descriptions = array.as_struct(); + return materialize_blob_v2_descriptors( + dataset, + field.id as u32, + descriptions, + row_addrs.as_ref(), + context, + ) + .await; + } + + match field.data_type() { + ArrowDataType::Struct(_) => { + let struct_array = array.as_struct(); + let mut children = Vec::with_capacity(field.children.len()); + for (child_field, child_array) in + field.children.iter().zip(struct_array.columns().iter()) + { + children.push( + materialize_blob_v2_binary_array( + dataset, + child_field, + child_array.clone(), + row_addrs.clone(), + context, + ) + .await?, + ); + } + let public_field = public_blob_v2_binary_output_field(field.clone()); + let ArrowDataType::Struct(fields) = public_field.data_type() else { + unreachable!("public output field preserved struct type") }; - let source = shared_blob_source(&mut source_cache, object_store, &path); - files.push(BlobEntry { - selection_index, - row_address: *row_addr, - file: BlobFile::with_source( - source, - position, - size, - BlobKind::External, - Some(uri_or_path), - ), - }); + Ok(Arc::new(StructArray::try_new( + fields, + children, + struct_array.nulls().cloned(), + )?) as ArrayRef) + } + ArrowDataType::List(_) => { + let list_array = array.as_list::(); + materialize_blob_v2_list_array::( + dataset, field, list_array, row_addrs, context, + ) + .await + } + ArrowDataType::LargeList(_) => { + let list_array = array.as_list::(); + materialize_blob_v2_list_array::( + dataset, field, list_array, row_addrs, context, + ) + .await } + _ => Ok(array), } } + .boxed() +} - Ok(files) +async fn materialize_blob_v2_list_array( + dataset: &Arc, + field: &LanceField, + list_array: &GenericListArray, + row_addrs: Arc<[u64]>, + context: &Arc, +) -> Result { + let offsets = list_array.value_offsets(); + let values_start = offsets[0].as_usize(); + let values_end = offsets[list_array.len()].as_usize(); + if values_end < values_start { + return Err(Error::internal(format!( + "List field '{}' has invalid offsets while materializing blob v2 binary scan", + field.name + ))); + } + + let values_len = values_end - values_start; + let mut normalized_offsets = Vec::with_capacity(list_array.len() + 1); + normalized_offsets.push(O::usize_as(0)); + let mut child_row_addrs = Vec::with_capacity(values_len); + for row_idx in 0..list_array.len() { + let start = offsets[row_idx].as_usize(); + let end = offsets[row_idx + 1].as_usize(); + if end < start { + return Err(Error::internal(format!( + "List field '{}' has decreasing offsets while materializing blob v2 binary scan", + field.name + ))); + } + let row_addr = row_addrs.get(row_idx).copied().ok_or_else(|| { + Error::internal(format!( + "List field '{}' row address count {} did not match row count {}", + field.name, + row_addrs.len(), + list_array.len() + )) + })?; + for _ in start..end { + child_row_addrs.push(row_addr); + } + normalized_offsets.push(O::usize_as(end - values_start)); + } + let child_row_addrs: Arc<[u64]> = child_row_addrs.into(); + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "List field '{}' missing child while materializing blob v2 binary scan", + field.name + )) + })?; + let values = list_array.values().slice(values_start, values_len); + let values = + materialize_blob_v2_binary_array(dataset, child, values, child_row_addrs, context).await?; + let child_field = public_blob_v2_binary_output_field(child.clone()); + let list_array = GenericListArray::::try_new( + Arc::new(ArrowField::from(&child_field)), + OffsetBuffer::new(ScalarBuffer::from(normalized_offsets)), + values, + list_array.nulls().cloned(), + )?; + Ok(Arc::new(list_array)) +} + +async fn materialize_blob_v2_descriptors( + dataset: &Arc, + blob_field_id: u32, + descriptions: &StructArray, + row_addrs: &[u64], + context: &Arc, +) -> Result { + if descriptions.len() != row_addrs.len() { + return Err(Error::internal(format!( + "blob v2 descriptor count {} did not match row address count {}", + descriptions.len(), + row_addrs.len() + ))); + } + match blob_version_from_descriptions(descriptions)? { + BlobVersion::V1 => { + return Err(Error::not_supported( + "Blob v2 binary materialization received a legacy blob descriptor".to_string(), + )); + } + BlobVersion::V2 => {} + } + + let columns = BlobV2DescriptorColumns::new(descriptions); + let mut read_context = BlobV2ReadContext::new(dataset, blob_field_id); + let mut entries = Vec::with_capacity(descriptions.len()); + let mut payloads = vec![None; descriptions.len()]; + + for (idx, row_addr) in row_addrs.iter().copied().enumerate() { + if descriptions.is_null(idx) || columns.kinds.is_null(idx) { + continue; + } + + let kind = BlobKind::try_from(columns.kinds.value(idx))?; + if matches!(kind, BlobKind::Inline) + && columns.positions.value(idx) == 0 + && columns.sizes.value(idx) == 0 + { + payloads[idx] = Some(Bytes::new()); + continue; + } + + let file = read_context + .collect_file(&columns, idx, row_addr) + .await? + .ok_or_else(|| { + Error::internal(format!( + "blob v2 descriptor at index {idx} unexpectedly resolved to null" + )) + })?; + entries.push(BlobEntry { + selection_index: idx, + row_address: row_addr, + file, + requested_range: None, + }); + } + + let blobs = execute_blob_entries_with_execution( + entries, + dataset.object_store.io_parallelism(), + context.execution.clone(), + ) + .await?; + for blob in blobs { + let payload = payloads.get_mut(blob.selection_index).ok_or_else(|| { + Error::internal(format!( + "blob result selection index {} exceeded descriptor count {}", + blob.selection_index, + descriptions.len() + )) + })?; + if payload.replace(blob.data).is_some() { + return Err(Error::internal(format!( + "blob result selection index {} was produced more than once", + blob.selection_index + ))); + } + } + + let payload_capacity = payloads.iter().flatten().map(Bytes::len).sum::(); + let mut builder = LargeBinaryBuilder::with_capacity(descriptions.len(), payload_capacity); + for (idx, payload) in payloads.into_iter().enumerate() { + if descriptions.is_null(idx) || columns.kinds.is_null(idx) { + builder.append_null(); + } else { + let payload = payload.ok_or_else(|| { + Error::internal(format!( + "blob v2 descriptor at index {idx} did not produce a payload" + )) + })?; + builder.append_value(payload); + } + } + + Ok(Arc::new(builder.finish())) +} + +struct BlobV2ReadContext<'a> { + dataset: &'a Arc, + blob_field_id: u32, + fragment_cache: HashMap, + store_cache: HashMap>, + external_base_path_cache: HashMap, + source_cache: HashMap>, +} + +impl<'a> BlobV2ReadContext<'a> { + fn new(dataset: &'a Arc, blob_field_id: u32) -> Self { + Self { + dataset, + blob_field_id, + fragment_cache: HashMap::new(), + store_cache: HashMap::new(), + external_base_path_cache: HashMap::new(), + source_cache: HashMap::new(), + } + } + + async fn collect_file( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result> { + if columns.is_null_blob(idx) { + return Ok(None); + } + + let kind = BlobKind::try_from(columns.kinds.value(idx))?; + let file = match kind { + BlobKind::Inline => self.collect_inline(columns, idx, row_addr).await?, + BlobKind::Dedicated => self.collect_dedicated(columns, idx, row_addr).await?, + BlobKind::Packed => self.collect_packed(columns, idx, row_addr).await?, + BlobKind::External => self.collect_external(columns, idx).await?, + }; + + Ok(Some(file)) + } + + async fn blob_read_location(&mut self, row_addr: u64) -> Result { + resolve_blob_read_location( + self.dataset, + self.blob_field_id, + row_addr, + &mut self.fragment_cache, + &mut self.store_cache, + ) + .await + } + + async fn collect_inline( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result { + let position = columns.positions.value(idx); + let size = columns.sizes.value(idx); + let location = self.blob_read_location(row_addr).await?; + let source = shared_blob_source( + &mut self.source_cache, + location.object_store, + &location.data_file_path, + ); + Ok(BlobFile::with_source( + source, + position, + size, + BlobKind::Inline, + None, + )) + } + + async fn collect_dedicated( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result { + let blob_id = columns.blob_ids.value(idx); + let size = columns.sizes.value(idx); + let location = self.blob_read_location(row_addr).await?; + let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); + let source = shared_blob_source(&mut self.source_cache, location.object_store, &path); + Ok(BlobFile::with_source( + source, + 0, + size, + BlobKind::Dedicated, + None, + )) + } + + async fn collect_packed( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + row_addr: u64, + ) -> Result { + let blob_id = columns.blob_ids.value(idx); + let size = columns.sizes.value(idx); + let position = columns.positions.value(idx); + let location = self.blob_read_location(row_addr).await?; + let path = blob_path(&location.data_file_dir, &location.data_file_key, blob_id); + let source = shared_blob_source(&mut self.source_cache, location.object_store, &path); + Ok(BlobFile::with_source( + source, + position, + size, + BlobKind::Packed, + None, + )) + } + + async fn collect_external( + &mut self, + columns: &BlobV2DescriptorColumns<'_>, + idx: usize, + ) -> Result { + let uri_or_path = columns.blob_uris.value(idx).to_string(); + let position = columns.positions.value(idx); + let size = columns.sizes.value(idx); + let base_id = columns.blob_ids.value(idx); + let (object_store, path) = if base_id == 0 { + let registry = self.dataset.session.store_registry(); + let params = self + .dataset + .store_params + .as_ref() + .map(|p| Arc::new((**p).clone())) + .unwrap_or_else(|| Arc::new(ObjectStoreParams::default())); + ObjectStore::from_uri_and_params(registry, &uri_or_path, ¶ms).await? + } else { + let object_store = if let Some(store) = self.store_cache.get(&base_id) { + store.clone() + } else { + let store = self.dataset.object_store(Some(base_id)).await?; + self.store_cache.insert(base_id, store.clone()); + store + }; + let base_root = if let Some(path) = self.external_base_path_cache.get(&base_id) { + path.clone() + } else { + let base = self + .dataset + .manifest + .base_paths + .get(&base_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "External blob references unknown base_id {}", + base_id + )) + })?; + let path = base.extract_path(self.dataset.session.store_registry())?; + self.external_base_path_cache.insert(base_id, path.clone()); + path + }; + let path = join_base_and_relative_path(&base_root, &uri_or_path)?; + (object_store, path) + }; + let size = if size > 0 { + size + } else { + object_store.size(&path).await? + }; + let source = shared_blob_source(&mut self.source_cache, object_store, &path); + Ok(BlobFile::with_source( + source, + position, + size, + BlobKind::External, + Some(uri_or_path), + )) + } } fn normalize_external_absolute_uri(uri: &str) -> Result { @@ -2259,7 +4438,10 @@ fn data_file_key_from_path(path: &str) -> &str { mod tests { use std::collections::HashMap; use std::ops::Range; - use std::sync::Arc; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use std::time::Duration; use arrow::{ @@ -2271,18 +4453,19 @@ mod tests { Array, ArrayRef, Int32Array, LargeBinaryArray, RecordBatchIterator, StringArray, StructArray, UInt8Array, UInt32Array, UInt64Array, }; + use arrow_buffer::NullBuffer; use arrow_schema::{DataType, Field, Schema}; use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; - use futures::{StreamExt, TryStreamExt, future::try_join_all}; + use futures::{StreamExt, TryStreamExt}; use lance_arrow::{ ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, DataTypeExt, }; use lance_core::{ - datatypes::{BlobHandling, BlobKind}, + datatypes::{BLOB_V2_LOGICAL_FIELDS, BlobHandling, BlobKind, OnMissing}, utils::blob::blob_path, }; use lance_io::object_store::{ @@ -2295,30 +4478,32 @@ mod tests { MultipartUpload, ObjectMeta, PutMultipartOptions, PutOptions, PutPayload, PutResult, path::Path, }; - use tokio::sync::Notify; + use rstest::rstest; + use tokio::sync::{Notify, Semaphore}; use url::Url; use lance_core::{ - Error, Result, + Error, ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION, Result, utils::tempfile::{TempDir, TempStrDir}, }; use lance_datagen::{BatchCount, RowCount, array}; - use lance_file::{ - version::LanceFileVersion, - writer::{FileWriter, FileWriterOptions}, - }; + use lance_file::{version::LanceFileVersion, writer::FileWriterOptions}; use uuid::Uuid; use super::{ - BlobEntry, BlobFile, BlobSource, ExternalBaseCandidate, ExternalBaseResolver, - ReadBlobsExecution, collect_blob_entries_v1, data_file_key_from_path, - execute_blob_read_plan, plan_blob_read_plans, + BlobEntry, BlobFile, BlobMaterializationBudget, BlobMaterializationBudgetState, + BlobRangeRequest, BlobReadRange, BlobSource, ExternalBaseCandidate, ExternalBaseResolver, + ExternalBlobSource, ReadBlobsExecution, blob_version_from_descriptions, + collect_blob_files_v1, data_file_key_from_path, execute_blob_entries, + execute_blob_read_batches_stream, execute_blob_read_plan, plan_blob_read_batches, + plan_blob_read_plans, }; use crate::{ Dataset, blob::{BlobArrayBuilder, BlobDescriptorArrayBuilder, PackedBlobWriter, blob_field}, dataset::{ CommitBuilder, ExternalBlobMode, WriteMode, WriteParams, + scanner::MaterializationStyle, transaction::{DataReplacementGroup, Operation, Transaction}, }, utils::test::TestDatasetGenerator, @@ -2336,16 +4521,87 @@ mod tests { expected: Vec, } - fn nested_blob_v2_batch(blob_array: ArrayRef) -> (Arc, RecordBatch) { - let blob_field = blob_field("blob", true); - let info_fields = vec![Field::new("name", DataType::Utf8, false), blob_field]; - let info_array: ArrayRef = Arc::new( - StructArray::try_new( - info_fields.clone().into(), - vec![ - Arc::new(StringArray::from_iter_values( - (0..blob_array.len()).map(|idx| format!("name-{idx}")), - )) as ArrayRef, + #[test] + fn test_blob_version_rejects_malformed_v2_descriptor_layout() { + let descriptions = StructArray::try_new( + vec![ + Field::new("kind", DataType::UInt8, false), + Field::new("position", DataType::UInt64, false), + Field::new("size", DataType::UInt32, false), + Field::new("blob_id", DataType::UInt32, false), + Field::new("blob_uri", DataType::Utf8, false), + ] + .into(), + vec![ + Arc::new(UInt8Array::from(vec![BlobKind::Inline as u8])), + Arc::new(UInt64Array::from(vec![0])), + Arc::new(UInt32Array::from(vec![0])), + Arc::new(UInt32Array::from(vec![0])), + Arc::new(StringArray::from(vec![""])), + ], + None, + ) + .unwrap(); + + let error = blob_version_from_descriptions(&descriptions).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("expected v1 descriptor or v2 descriptor layout") + ); + } + + fn complete_blob_v2_field(name: &str, nullable: bool) -> Field { + Field::new( + name, + DataType::Struct(BLOB_V2_LOGICAL_FIELDS.clone()), + nullable, + ) + .with_metadata(HashMap::from([( + ARROW_EXT_NAME_KEY.to_string(), + BLOB_V2_EXT_NAME.to_string(), + )])) + } + + fn complete_blob_v2_array( + data: Vec>>, + uris: Vec>, + positions: Vec>, + sizes: Vec>, + validity: Option, + ) -> ArrayRef { + Arc::new( + StructArray::try_new( + BLOB_V2_LOGICAL_FIELDS.clone(), + vec![ + Arc::new(LargeBinaryArray::from_iter( + data.iter().map(|value| value.as_deref()), + )) as ArrayRef, + Arc::new(StringArray::from_iter( + uris.iter().map(|value| value.as_deref()), + )) as ArrayRef, + Arc::new(UInt64Array::from(positions)) as ArrayRef, + Arc::new(UInt64Array::from(sizes)) as ArrayRef, + ], + validity, + ) + .unwrap(), + ) + } + + fn nested_blob_v2_batch_with_field( + blob_field: Field, + blob_array: ArrayRef, + ) -> (Arc, RecordBatch) { + let info_fields = vec![Field::new("name", DataType::Utf8, false), blob_field]; + let info_array: ArrayRef = Arc::new( + StructArray::try_new( + info_fields.clone().into(), + vec![ + Arc::new(StringArray::from_iter_values( + (0..blob_array.len()).map(|idx| format!("name-{idx}")), + )) as ArrayRef, blob_array, ], None, @@ -2362,6 +4618,10 @@ mod tests { (schema, batch) } + fn nested_blob_v2_batch(blob_array: ArrayRef) -> (Arc, RecordBatch) { + nested_blob_v2_batch_with_field(blob_field("blob", true), blob_array) + } + #[cfg(feature = "azure")] fn azure_store_params(account_name: &str) -> ObjectStoreParams { ObjectStoreParams { @@ -2538,8 +4798,12 @@ mod tests { #[derive(Debug)] struct RecordingRangeObjectStore { data: Bytes, - gate: Option>, + gate: Option>, requested_ranges: std::sync::Mutex>>, + started_blob_requests: AtomicUsize, + active_blob_requests: AtomicUsize, + peak_active_blob_requests: AtomicUsize, + request_started: Notify, } impl RecordingRangeObjectStore { @@ -2548,17 +4812,39 @@ mod tests { data, gate: None, requested_ranges: std::sync::Mutex::new(Vec::new()), + started_blob_requests: AtomicUsize::new(0), + active_blob_requests: AtomicUsize::new(0), + peak_active_blob_requests: AtomicUsize::new(0), + request_started: Notify::new(), } } - fn with_gate(data: Bytes, gate: Arc) -> Self { + fn with_gate(data: Bytes, gate: Arc) -> Self { Self { data, gate: Some(gate), requested_ranges: std::sync::Mutex::new(Vec::new()), + started_blob_requests: AtomicUsize::new(0), + active_blob_requests: AtomicUsize::new(0), + peak_active_blob_requests: AtomicUsize::new(0), + request_started: Notify::new(), + } + } + + async fn wait_for_blob_requests(&self, expected: usize) { + loop { + let request_started = self.request_started.notified(); + if self.started_blob_requests.load(Ordering::Acquire) >= expected { + return; + } + request_started.await; } } + fn peak_active_blob_requests(&self) -> usize { + self.peak_active_blob_requests.load(Ordering::Acquire) + } + fn requested_ranges(&self) -> Vec> { self.requested_ranges.lock().unwrap().clone() } @@ -2622,10 +4908,21 @@ mod tests { } }; let is_full_object_probe = range.start == 0 && range.end == self.data.len() as u64; - if !is_full_object_probe && let Some(gate) = &self.gate { - gate.notified().await; - } self.requested_ranges.lock().unwrap().push(range.clone()); + if !is_full_object_probe { + let active = self.active_blob_requests.fetch_add(1, Ordering::AcqRel) + 1; + self.peak_active_blob_requests + .fetch_max(active, Ordering::AcqRel); + self.started_blob_requests.fetch_add(1, Ordering::AcqRel); + self.request_started.notify_waiters(); + if let Some(gate) = &self.gate { + gate.acquire() + .await + .expect("test gate should remain open") + .forget(); + } + self.active_blob_requests.fetch_sub(1, Ordering::AcqRel); + } let bytes = self.data.slice(range.start as usize..range.end as usize); Ok(GetResult { payload: GetResultPayload::Stream( @@ -2668,9 +4965,10 @@ mod tests { } } - fn recording_range_store_with_url( + fn recording_range_store_with_url_and_block_size( data: Bytes, url: &str, + block_size: Option, ) -> (Arc, Arc) { const TEST_RANGE_STORE_SIZE: usize = 128 * 1024; let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())]; @@ -2679,7 +4977,7 @@ mod tests { let store = Arc::new(ObjectStore::new( inner.clone() as Arc, Url::parse(url).unwrap(), - None, + block_size, None, false, true, @@ -2690,22 +4988,40 @@ mod tests { (store, inner) } + fn recording_range_store_with_url( + data: Bytes, + url: &str, + ) -> (Arc, Arc) { + recording_range_store_with_url_and_block_size(data, url, None) + } + fn recording_range_store(data: Bytes) -> (Arc, Arc) { recording_range_store_with_url(data, "mock://recording/blob-range-tests") } + fn recording_range_store_with_block_size( + data: Bytes, + block_size: usize, + ) -> (Arc, Arc) { + recording_range_store_with_url_and_block_size( + data, + "mock://recording/blob-range-tests", + Some(block_size), + ) + } + fn gated_range_store( data: Bytes, url: &str, ) -> ( Arc, Arc, - Arc, + Arc, ) { const TEST_RANGE_STORE_SIZE: usize = 128 * 1024; let mut padded = vec![0; TEST_RANGE_STORE_SIZE.max(data.len())]; padded[..data.len()].copy_from_slice(data.as_ref()); - let gate = Arc::new(Notify::new()); + let gate = Arc::new(Semaphore::new(0)); let inner = Arc::new(RecordingRangeObjectStore::with_gate( Bytes::from(padded), gate.clone(), @@ -2731,7 +5047,7 @@ mod tests { let data = lance_datagen::gen_batch() .col("filterme", array::step::()) .col("blobs", array::blob()) - .into_reader_rows(RowCount::from(10), BatchCount::from(10)) + .into_reader_rows(RowCount::from(10), BatchCount::from(4)) .map(|batch| Ok(batch?)) .collect::>>() .unwrap(); @@ -2750,6 +5066,55 @@ mod tests { } } + async fn stable_row_id_blob_dataset_with_deleted_row() -> (TempStrDir, Arc, Vec) { + let test_dir = TempStrDir::default(); + let mut blob_builder = BlobArrayBuilder::new(3); + blob_builder.push_bytes(b"AAaa").unwrap(); + blob_builder.push_bytes(b"BBbb").unwrap(); + blob_builder.push_bytes(b"CCcc").unwrap(); + let schema = Arc::new(Schema::new(vec![ + blob_field("blob", true), + Field::new("idx", DataType::UInt64, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + blob_builder.finish().unwrap(), + Arc::new(UInt64Array::from(vec![0, 1, 2])), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let with_row_ids = dataset + .scan() + .project(&["idx"]) + .unwrap() + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let indices = with_row_ids["idx"].as_primitive::(); + let ids = with_row_ids[ROW_ID].as_primitive::(); + let mut row_ids = vec![0; with_row_ids.num_rows()]; + for (index, row_id) in indices.values().iter().zip(ids.values()) { + row_ids[*index as usize] = *row_id; + } + + dataset.delete("idx = 1").await.unwrap(); + (test_dir, Arc::new(dataset), row_ids) + } + async fn create_multi_base_blob_v2_fixture( payload: Vec, dedicated_threshold: Option, @@ -2829,21 +5194,21 @@ mod tests { .scan() .project::(&[]) .unwrap() - .filter("filterme >= 50") + .filter("filterme >= 10") .unwrap() .with_row_id() .try_into_batch() .await .unwrap(); let row_ids = row_ids.column(0).as_primitive::().values(); - let row_ids = vec![row_ids[5], row_ids[17], row_ids[33]]; + let row_ids = vec![row_ids[5], row_ids[17], row_ids[23]]; let blobs = fixture.dataset.take_blobs(&row_ids, "blobs").await.unwrap(); for (actual_idx, (expected_batch_idx, expected_row_idx)) in - [(5, 5), (6, 7), (8, 3)].iter().enumerate() + [(1, 5), (2, 7), (3, 3)].iter().enumerate() { - let val = blobs[actual_idx].read().await.unwrap(); + let val = blobs[actual_idx].as_ref().unwrap().read().await.unwrap(); let expected = fixture.data[*expected_batch_idx] .column(1) .as_binary::() @@ -2889,7 +5254,7 @@ mod tests { .column(1) .as_binary::() .value(*expected_row_idx); - assert_eq!(blobs[actual_idx].data.as_ref(), expected); + assert_eq!(blobs[actual_idx].data.as_deref(), Some(expected)); } } @@ -2909,7 +5274,7 @@ mod tests { indices.pop(); // Row indices - assert_eq!(indices, [2, 12, 22, 32, 42, 52, 62, 72, 82]); + assert_eq!(indices, [2, 12, 22]); let blobs = fixture .dataset .take_blobs_by_indices(&indices, "blobs") @@ -2924,10 +5289,84 @@ mod tests { let blobs2 = fixture.dataset.take_blobs(&row_ids, "blobs").await.unwrap(); for (blob1, blob2) in blobs.iter().zip(blobs2.iter()) { + let blob1 = blob1.as_ref().unwrap(); + let blob2 = blob2.as_ref().unwrap(); assert_eq!(blob1.position(), blob2.position()); assert_eq!(blob1.size(), blob2.size()); assert_eq!(blob1.data_path(), blob2.data_path()); } + + // Unsorted indices spanning fragments use the take remapping path, which + // carries _rowaddr internally and must still preserve the requested order. + let indices = [33_u64, 17, 5, 28, 12, 39]; + let blobs = fixture + .dataset + .take_blobs_by_indices(&indices, "blobs") + .await + .unwrap(); + for (blob, index) in blobs.iter().zip(indices) { + let actual = blob.as_ref().unwrap().read().await.unwrap(); + let index = index as usize; + let expected = fixture.data[index / 10] + .column(1) + .as_binary::() + .value(index % 10); + assert_eq!(actual.as_ref(), expected); + } + } + + #[rstest] + #[case::all_valid_first(false)] + #[case::nullable_first(true)] + #[tokio::test] + async fn test_write_blob_batches_with_mixed_nullability(#[case] nulls_first: bool) { + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("blob", DataType::LargeBinary, true).with_metadata(HashMap::from([( + BLOB_META_KEY.to_string(), + "true".to_string(), + )])), + ])); + let batch = |values: Vec>| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(LargeBinaryArray::from(values))], + ) + .unwrap() + }; + + // Definition-level semantics come from each batch's validity bitmap. Both + // transitions must start a new descriptor page so nulls remain distinct + // from valid empty blobs. + let all_valid = batch(vec![Some(b"a".as_slice()), Some(b"".as_slice())]); + let with_null = batch(vec![Some(b"c".as_slice()), None]); + let (batches, expected) = if nulls_first { + ( + vec![with_null, all_valid], + vec![Some(b"c".as_slice()), None, Some(b"a"), Some(b"")], + ) + } else { + ( + vec![all_valid, with_null], + vec![Some(b"a".as_slice()), Some(b""), Some(b"c"), None], + ) + }; + + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); + let dataset = Arc::new(Dataset::write(reader, &test_dir, None).await.unwrap()); + + let blobs = dataset + .take_blobs_by_indices(&[0, 1, 2, 3], "blob") + .await + .unwrap(); + assert_eq!(blobs.len(), expected.len()); + for (row_idx, (blob, expected)) in blobs.into_iter().zip(expected).enumerate() { + let actual = match blob { + Some(blob) => Some(blob.read().await.unwrap()), + None => None, + }; + assert_eq!(actual.as_deref(), expected, "row {row_idx}"); + } } #[tokio::test] @@ -2940,7 +5379,63 @@ mod tests { } #[tokio::test] - async fn test_collect_blob_entries_v1_rejects_missing_fragment() { + async fn test_take_blobs_by_ids_rejects_deleted_stable_row_id() { + let (_test_dir, dataset, row_ids) = stable_row_id_blob_dataset_with_deleted_row().await; + + let err = dataset.take_blobs(&row_ids, "blob").await.unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("Could not resolve all requested row IDs") + ); + } + + #[tokio::test] + async fn test_read_blobs_by_ids_rejects_deleted_stable_row_id() { + let (_test_dir, dataset, row_ids) = stable_row_id_blob_dataset_with_deleted_row().await; + + let err = dataset + .read_blobs("blob") + .unwrap() + .with_row_ids(row_ids) + .execute() + .await + .unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("Could not resolve all requested row IDs") + ); + } + + #[tokio::test] + async fn test_read_blob_ranges_by_ids_rejects_deleted_stable_row_id() { + let (_test_dir, dataset, row_ids) = stable_row_id_blob_dataset_with_deleted_row().await; + let requests = [ + BlobRangeRequest::new(row_ids[0], 0, 2), + BlobRangeRequest::new(row_ids[1], 0, 2), + BlobRangeRequest::new(row_ids[2], 2, 2), + ]; + + let err = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_ids(requests) + .execute() + .await + .unwrap_err(); + + assert!(matches!(err, Error::InvalidInput { .. })); + assert!( + err.to_string() + .contains("Could not resolve all requested row IDs") + ); + } + + #[tokio::test] + async fn test_collect_blob_files_v1_rejects_missing_fragment() { let fixture = BlobTestFixture::new().await; let blob_field_id = fixture.dataset.schema().project(&["blobs"]).unwrap().fields[0].id as u32; @@ -2956,9 +5451,8 @@ mod tests { ]); let row_addrs = UInt64Array::from(vec![(999_u64 << 32) | 7]); - let err = - collect_blob_entries_v1(&fixture.dataset, blob_field_id, &descriptions, &row_addrs) - .unwrap_err(); + let err = collect_blob_files_v1(&fixture.dataset, blob_field_id, &descriptions, &row_addrs) + .unwrap_err(); assert!(err.to_string().contains("references missing fragment")); } @@ -2993,7 +5487,7 @@ mod tests { let batches = batches.try_collect::>().await.unwrap(); - assert_eq!(batches.len(), 10); + assert_eq!(batches.len(), 4); for batch in batches.iter() { assert_eq!(batch.num_columns(), 1); assert!(batch.column(0).data_type().is_struct()); @@ -3005,7 +5499,7 @@ mod tests { .scan() .project(&["blobs"]) .unwrap() - .filter("filterme = 50") + .filter("filterme = 30") .unwrap() .try_into_stream() .await @@ -3079,7 +5573,7 @@ mod tests { // Verify we can read the blob content for blob in &blobs { - let content = blob.read().await.unwrap(); + let content = blob.as_ref().unwrap().read().await.unwrap(); assert!(!content.is_empty(), "Blob content should not be empty"); } @@ -3092,7 +5586,7 @@ mod tests { // Verify we can read the blob content from second fragment for blob in &blobs { - let content = blob.read().await.unwrap(); + let content = blob.as_ref().unwrap().read().await.unwrap(); assert!(!content.is_empty(), "Blob content should not be empty"); } @@ -3280,7 +5774,10 @@ mod tests { .unwrap(); assert_eq!(blob_files.len(), payloads.len()); for (blob_file, expected) in blob_files.iter().zip(payloads) { - assert_eq!(blob_file.read().await.unwrap().as_ref(), expected); + assert_eq!( + blob_file.as_ref().unwrap().read().await.unwrap().as_ref(), + expected + ); } let read_blobs = dataset @@ -3292,7 +5789,7 @@ mod tests { .unwrap(); assert_eq!(read_blobs.len(), payloads.len()); for (read_blob, expected) in read_blobs.iter().zip(payloads) { - assert_eq!(read_blob.data.as_ref(), expected); + assert_eq!(read_blob.data.as_deref(), Some(expected)); } } @@ -3338,8 +5835,8 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 2); - let first = blobs[0].read().await.unwrap(); - let second = blobs[1].read().await.unwrap(); + let first = blobs[0].as_ref().unwrap().read().await.unwrap(); + let second = blobs[1].as_ref().unwrap().read().await.unwrap(); assert_eq!(first.as_ref(), b"hello"); assert_eq!(second.as_ref(), b"world"); } @@ -3389,7 +5886,10 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-inline"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"prepared-inline" + ); } #[tokio::test] @@ -3455,13 +5955,10 @@ mod tests { .unwrap(); let object_writer = dataset.object_store.create(&data_file_path).await.unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(append_schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); file_writer.write_batch(&replacement_batch).await.unwrap(); @@ -3489,8 +5986,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"prepared-packed"); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.read().await.unwrap().as_ref(), b"prepared-packed"); + assert_eq!(blob.kind(), BlobKind::Packed); } #[tokio::test] @@ -3571,8 +6069,14 @@ mod tests { .await .unwrap(); assert_eq!(blobs.len(), 2); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"initial"); - assert_eq!(blobs[1].read().await.unwrap().as_ref(), b"append"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"initial" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"append" + ); } #[tokio::test] @@ -3634,13 +6138,10 @@ mod tests { RecordBatch::try_new(replacement_schema.clone(), vec![info_array]).unwrap(); let object_writer = dataset.object_store.create(&data_file_path).await.unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(replacement_schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); file_writer.write_batch(&replacement_batch).await.unwrap(); @@ -3672,10 +6173,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), b"nested-replacement" ); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + assert_eq!(blobs[0].as_ref().unwrap().kind(), BlobKind::Packed); } #[tokio::test] @@ -3741,13 +6242,10 @@ mod tests { RecordBatch::try_new(replacement_schema.clone(), vec![info_array]).unwrap(); let object_writer = dataset.object_store.create(&data_file_path).await.unwrap(); - let mut file_writer = FileWriter::try_new( + let mut file_writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(replacement_schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); file_writer.write_batch(&replacement_batch).await.unwrap(); @@ -3824,9 +6322,12 @@ mod tests { .await .unwrap(); assert_eq!(blobs.len(), 2); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"hello"); assert_eq!( - blobs[1].read().await.unwrap().as_ref(), + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"hello" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), packed_payload.as_slice() ); @@ -3834,54 +6335,431 @@ mod tests { .take_blobs_by_indices(&[2], "info.blob") .await .unwrap(); - assert!(null_blobs.is_empty()); + assert_eq!(null_blobs.len(), 1); + assert!(null_blobs[0].is_none()); + + let filtered = dataset + .scan() + .project(&["info"]) + .unwrap() + .filter("info.blob IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(filtered.num_rows(), 2); } #[tokio::test] - async fn test_nested_blob_v2_requires_v2_2() { + async fn test_write_and_take_nested_complete_blob_v2() { let test_dir = TempStrDir::default(); + let packed_payload = vec![0x4A; super::INLINE_MAX + 1024]; - let mut blob_builder = BlobArrayBuilder::new(1); - blob_builder.push_bytes(b"hello").unwrap(); - let blob_array: ArrayRef = blob_builder.finish().unwrap(); + let blob_array = complete_blob_v2_array( + vec![Some(b"hello".to_vec()), Some(packed_payload.clone()), None], + vec![None, None, None], + vec![None, None, None], + vec![None, None, None], + Some(NullBuffer::from(vec![true, true, false])), + ); - let (schema, batch) = nested_blob_v2_batch(blob_array); + let (schema, batch) = + nested_blob_v2_batch_with_field(complete_blob_v2_field("blob", true), blob_array); let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); - let result = Dataset::write( - reader, - &test_dir, - Some(WriteParams { - data_storage_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }), - ) - .await; + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); - assert!( - result.is_err(), - "Nested blob v2 should be rejected for file version 2.1" + let info_batch = dataset + .scan() + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let blob_desc = info_batch + .column(0) + .as_struct() + .column_by_name("blob") + .unwrap() + .as_struct(); + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(0), + BlobKind::Inline as u8 ); - assert!( - result - .unwrap_err() - .to_string() - .contains("Blob v2 requires file version >= 2.2") + assert_eq!( + blob_desc + .column_by_name("kind") + .unwrap() + .as_primitive::() + .value(1), + BlobKind::Packed as u8 ); - } - - #[tokio::test] - async fn test_blob_file_read_empty_range_returns_empty_bytes() { - let store = reject_empty_range_store(); - let path = Path::from("blobs/test.bin"); - - let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0); - assert!(empty_blob.read().await.unwrap().is_empty()); - assert!(empty_blob.read_up_to(16).await.unwrap().is_empty()); - } - #[tokio::test] - async fn test_blob_file_read_tracks_relative_cursor() { + let blobs = dataset + .take_blobs_by_indices(&[0, 1], "info.blob") + .await + .unwrap(); + assert_eq!(blobs.len(), 2); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"hello" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + packed_payload.as_slice() + ); + + let null_blobs = dataset + .take_blobs_by_indices(&[2], "info.blob") + .await + .unwrap(); + assert_eq!(null_blobs.len(), 1); + assert!(null_blobs[0].is_none()); + + let filtered = dataset + .scan() + .project(&["info"]) + .unwrap() + .filter("info.blob IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(filtered.num_rows(), 2); + } + + #[tokio::test] + async fn test_write_and_scan_list_blob_v2_descriptions() { + let test_dir = TempStrDir::default(); + let packed_payload = vec![0x4B; super::INLINE_MAX + 1024]; + + let mut blob_builder = BlobArrayBuilder::new(4); + blob_builder.push_bytes(b"hello").unwrap(); + blob_builder.push_null().unwrap(); + blob_builder.push_bytes(&packed_payload).unwrap(); + blob_builder.push_bytes(b"tail").unwrap(); + let blob_values = blob_builder.finish().unwrap(); + + let item_field = Arc::new(blob_field("item", true)); + let list_array: ArrayRef = Arc::new( + arrow_array::ListArray::try_new( + item_field.clone(), + arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![ + 0i32, 3, 3, 3, 4, + ])), + blob_values, + Some(arrow_buffer::NullBuffer::from(vec![ + true, true, false, true, + ])), + ) + .unwrap(), + ); + + let schema = Arc::new(Schema::new(vec![ + Field::new("blobs", DataType::List(item_field), true), + Field::new("id", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![list_array, Arc::new(Int32Array::from(vec![0, 1, 2, 3]))], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let descriptions = dataset + .scan() + .project(&["blobs"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let lists = descriptions.column(0).as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]); + assert!(lists.is_valid(0)); + assert!(lists.is_valid(1)); + assert!(lists.is_null(2)); + assert!(lists.is_valid(3)); + + let DataType::List(descriptor_field) = lists.data_type() else { + panic!("unexpected list type: {}", lists.data_type()); + }; + assert!(matches!(descriptor_field.data_type(), DataType::Struct(_))); + assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let descriptors = lists.values().as_struct(); + assert_eq!(descriptors.fields().len(), 5); + assert_eq!(descriptors.fields()[0].name(), "kind"); + assert!(descriptors.is_valid(0)); + assert!(descriptors.is_null(1)); + assert!(descriptors.is_valid(2)); + assert!(descriptors.is_valid(3)); + let kinds = descriptors + .column_by_name("kind") + .unwrap() + .as_primitive::(); + assert_eq!(kinds.value(0), BlobKind::Inline as u8); + assert_eq!(kinds.value(2), BlobKind::Packed as u8); + assert_eq!(kinds.value(3), BlobKind::Inline as u8); + + let filtered = dataset + .scan() + .project(&["blobs"]) + .unwrap() + .filter("blobs IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(filtered.num_rows(), 3); + + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let bytes = scanner + .project(&["blobs"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let lists = bytes.column(0).as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 3, 3, 3, 4]); + assert!(lists.is_valid(0)); + assert!(lists.is_valid(1)); + assert!(lists.is_null(2)); + assert!(lists.is_valid(3)); + let DataType::List(value_field) = lists.data_type() else { + panic!("unexpected list type: {}", lists.data_type()); + }; + assert_eq!(value_field.data_type(), &DataType::LargeBinary); + assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let values = lists.values().as_binary::(); + assert_eq!(values.value(0), b"hello"); + assert!(values.is_null(1)); + assert_eq!(values.value(2), packed_payload.as_slice()); + assert_eq!(values.value(3), b"tail"); + + for (filter, materialization_style) in [ + (None, MaterializationStyle::Heuristic), + (Some("id >= 2"), MaterializationStyle::Heuristic), + (Some("id >= 2"), MaterializationStyle::AllEarly), + ] { + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + scanner.materialization_style(materialization_style); + scanner + .project(&["blobs", ROW_LAST_UPDATED_AT_VERSION, ROW_CREATED_AT_VERSION]) + .unwrap() + .with_row_id() + .with_row_address(); + if let Some(filter) = filter { + scanner.filter(filter).unwrap(); + } + + let expected_schema = scanner.schema().await.unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + assert_eq!(batch.schema().as_ref(), expected_schema.as_ref()); + assert_eq!(batch.num_rows(), if filter.is_some() { 2 } else { 4 }); + for column in [ + ROW_ID, + ROW_ADDR, + ROW_LAST_UPDATED_AT_VERSION, + ROW_CREATED_AT_VERSION, + ] { + assert!( + batch.column_by_name(column).is_some(), + "requested system column {column} was missing" + ); + } + } + } + + #[tokio::test] + async fn test_write_and_scan_struct_nested_list_blob_v2() { + let test_dir = TempStrDir::default(); + + let mut blob_builder = BlobArrayBuilder::new(2); + blob_builder.push_bytes(b"nested").unwrap(); + blob_builder.push_null().unwrap(); + let blob_values = blob_builder.finish().unwrap(); + + let item_field = Arc::new(blob_field("item", true)); + let list_field = Field::new("blobs", DataType::List(item_field.clone()), true); + let list_array: ArrayRef = Arc::new( + arrow_array::ListArray::try_new( + item_field, + arrow_buffer::OffsetBuffer::new(arrow_buffer::ScalarBuffer::from(vec![0i32, 2, 2])), + blob_values, + None, + ) + .unwrap(), + ); + let info_fields = vec![Field::new("name", DataType::Utf8, false), list_field]; + let info_array: ArrayRef = Arc::new( + StructArray::try_new( + info_fields.clone().into(), + vec![ + Arc::new(StringArray::from(vec!["row-0", "row-1"])) as ArrayRef, + list_array, + ], + None, + ) + .unwrap(), + ); + + let schema = Arc::new(Schema::new(vec![Field::new( + "info", + DataType::Struct(info_fields.into()), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![info_array]).unwrap(); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let dataset = Arc::new( + Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let descriptions = dataset + .scan() + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let info = descriptions.column(0).as_struct(); + assert_eq!( + info.column_by_name("name") + .unwrap() + .as_string::() + .value(0), + "row-0" + ); + let lists = info.column_by_name("blobs").unwrap().as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]); + let DataType::List(descriptor_field) = lists.data_type() else { + panic!("unexpected nested list type: {}", lists.data_type()); + }; + assert!(matches!(descriptor_field.data_type(), DataType::Struct(_))); + assert!(!descriptor_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let descriptors = lists.values().as_struct(); + assert_eq!(descriptors.fields().len(), 5); + assert!(descriptors.is_valid(0)); + assert!(descriptors.is_null(1)); + + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let bytes = scanner + .project(&["info"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let info = bytes.column(0).as_struct(); + let lists = info.column_by_name("blobs").unwrap().as_list::(); + assert_eq!(lists.offsets().inner().as_ref(), &[0, 2, 2]); + let DataType::List(value_field) = lists.data_type() else { + panic!("unexpected nested list type: {}", lists.data_type()); + }; + assert_eq!(value_field.data_type(), &DataType::LargeBinary); + assert!(!value_field.metadata().contains_key(ARROW_EXT_NAME_KEY)); + let values = lists.values().as_binary::(); + assert_eq!(values.value(0), b"nested"); + assert!(values.is_null(1)); + } + + #[tokio::test] + async fn test_nested_blob_v2_requires_v2_2() { + let test_dir = TempStrDir::default(); + + let mut blob_builder = BlobArrayBuilder::new(1); + blob_builder.push_bytes(b"hello").unwrap(); + let blob_array: ArrayRef = blob_builder.finish().unwrap(); + + let (schema, batch) = nested_blob_v2_batch(blob_array); + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + + let result = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await; + + assert!( + result.is_err(), + "Nested blob v2 should be rejected for file version 2.1" + ); + assert!( + result + .unwrap_err() + .to_string() + .contains("Blob v2 requires file version >= 2.2") + ); + } + + #[tokio::test] + async fn test_blob_file_read_empty_range_returns_empty_bytes() { + let store = reject_empty_range_store(); + let path = Path::from("blobs/test.bin"); + + let empty_blob = BlobFile::new_packed(store.clone(), path.clone(), 1, 0); + assert!(empty_blob.read().await.unwrap().is_empty()); + assert!(empty_blob.read_up_to(16).await.unwrap().is_empty()); + } + + #[tokio::test] + async fn test_external_blob_source_read_all_empty_range_returns_empty_bytes() { + let store = reject_empty_range_store(); + let reader = store.open(&Path::from("blobs/test.bin")).await.unwrap(); + let source = ExternalBlobSource { + reader, + start: 0, + size: 0, + }; + + assert!(source.read_all().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn test_blob_file_read_tracks_relative_cursor() { let test_dir = TempDir::default(); let file_path = test_dir.std_path().join("blob.bin"); std::fs::write(&file_path, b"abcd").unwrap(); @@ -3922,61 +6800,858 @@ mod tests { } #[tokio::test] - async fn test_blob_file_read_range_rejects_out_of_bounds() { - let (store, _) = recording_range_store(Bytes::from_static(b"abcdef")); - let path = Path::from("blobs/test.bin"); - let blob = BlobFile::new_packed(store, path, 0, 4); + async fn test_blob_file_read_range_rejects_out_of_bounds() { + let (store, _) = recording_range_store(Bytes::from_static(b"abcdef")); + let path = Path::from("blobs/test.bin"); + let blob = BlobFile::new_packed(store, path, 0, 4); + + let err = blob.read_range(1..5).await.unwrap_err(); + assert!(err.to_string().contains("exceeds blob size")); + } + + #[tokio::test] + async fn test_blob_file_staggered_same_source_ranges_run_concurrently() { + let (store, inner, gate) = gated_range_store( + Bytes::from_static(b"abcdefgh"), + "mock://same-source/blob-range-tests", + ); + let blob = Arc::new(BlobFile::new_dedicated( + store, + Path::from("blobs/test.bin"), + 8, + )); + + let first_blob = blob.clone(); + let first = tokio::spawn(async move { first_blob.read_range(0..3).await }); + inner.wait_for_blob_requests(1).await; + + let second_blob = blob.clone(); + let second = tokio::spawn(async move { second_blob.read_range(4..7).await }); + tokio::time::timeout(Duration::from_secs(1), inner.wait_for_blob_requests(2)) + .await + .expect("the second same-source range should start while the first is in flight"); + + assert_eq!(inner.peak_active_blob_requests(), 2); + gate.add_permits(2); + assert_eq!(first.await.unwrap().unwrap().as_ref(), b"abc"); + assert_eq!(second.await.unwrap().unwrap().as_ref(), b"efg"); + } + + #[tokio::test] + async fn test_blob_file_staggered_multiple_source_ranges_run_concurrently() { + let (first_store, first_inner, first_gate) = gated_range_store( + Bytes::from_static(b"abcdefgh"), + "mock://first-source/blob-range-tests", + ); + let (second_store, second_inner, second_gate) = gated_range_store( + Bytes::from_static(b"ijklmnop"), + "mock://second-source/blob-range-tests", + ); + let first_blob = Arc::new(BlobFile::new_dedicated( + first_store, + Path::from("blobs/first.bin"), + 8, + )); + let second_blob = Arc::new(BlobFile::new_dedicated( + second_store, + Path::from("blobs/second.bin"), + 8, + )); + + let first = tokio::spawn(async move { first_blob.read_range(0..3).await }); + first_inner.wait_for_blob_requests(1).await; + + let second = tokio::spawn(async move { second_blob.read_range(4..7).await }); + tokio::time::timeout( + Duration::from_secs(1), + second_inner.wait_for_blob_requests(1), + ) + .await + .expect("a read from another source should start while the first is in flight"); + + first_gate.add_permits(1); + second_gate.add_permits(1); + assert_eq!(first.await.unwrap().unwrap().as_ref(), b"abc"); + assert_eq!(second.await.unwrap().unwrap().as_ref(), b"mno"); + } + + #[tokio::test] + async fn test_blob_files_share_source_and_coalesce() { + let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); + let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None); + let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None); + + let (data1, data2) = tokio::join!(blob1.read(), blob2.read()); + assert_eq!(data1.unwrap().as_ref(), b"bcd"); + assert_eq!(data2.unwrap().as_ref(), b"efg"); + assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + } + + #[tokio::test] + async fn test_execute_blob_entries_preserves_order_and_coalesces() { + let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); + let entries = vec![ + BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None), + requested_range: None, + }, + BlobEntry { + selection_index: 1, + row_address: 11, + file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None), + requested_range: None, + }, + ]; + let mut blobs = execute_blob_entries(entries, 2, None).await.unwrap(); + blobs.sort_by_key(|blob| blob.selection_index); + + assert_eq!(blobs.len(), 2); + assert_eq!(blobs[0].row_address, 10); + assert_eq!(blobs[0].data.as_ref(), b"efg"); + assert_eq!(blobs[1].row_address, 11); + assert_eq!(blobs[1].data.as_ref(), b"bcd"); + assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + } + + #[tokio::test] + async fn test_blob_materialization_budget_blocks_and_admits_oversized_batch() { + let budget = Arc::new(BlobMaterializationBudget { + limit: 10, + state: std::sync::Mutex::new(BlobMaterializationBudgetState::default()), + notify: Notify::new(), + }); + let first = budget.reserve(8).await; + let waiting_budget = budget.clone(); + let waiting = tokio::spawn(async move { waiting_budget.reserve(4).await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), waiting_budget_wait(&waiting)) + .await + .is_err() + ); + drop(first); + let second = waiting.await.unwrap(); + drop(second); + + let oversized = budget.reserve(11).await; + assert_eq!(budget.state.lock().unwrap().reserved, 11); + drop(oversized); + assert_eq!(budget.state.lock().unwrap().reserved, 0); + } + + #[tokio::test] + async fn test_blob_materialization_admission_cannot_invert_output_order() { + let budget = Arc::new(BlobMaterializationBudget { + limit: 100, + state: std::sync::Mutex::new(BlobMaterializationBudgetState::default()), + notify: Notify::new(), + }); + let first = budget.admission(); + let second = budget.admission(); + let later = tokio::spawn(async move { second.reserve(60).await.unwrap() }); + assert!( + tokio::time::timeout(Duration::from_millis(20), waiting_budget_wait(&later)) + .await + .is_err() + ); + assert_eq!(budget.state.lock().unwrap().reserved, 0); + + let earlier = first.reserve(80).await.unwrap(); + assert_eq!(budget.state.lock().unwrap().reserved, 80); + drop(earlier); + let later = later.await.unwrap(); + assert_eq!(budget.state.lock().unwrap().reserved, 60); + drop(later); + assert_eq!(budget.state.lock().unwrap().reserved, 0); + } + + async fn waiting_budget_wait( + task: &tokio::task::JoinHandle, + ) { + while !task.is_finished() { + tokio::task::yield_now().await; + } + } + + #[test] + fn test_blob_read_batches_bound_physical_bytes() { + let (store, _) = recording_range_store(Bytes::from_static(b"abcdefghij")); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); + let entries = vec![ + BlobEntry { + selection_index: 1, + row_address: 11, + file: BlobFile::with_source(source.clone(), 4, 6, BlobKind::Packed, None), + requested_range: Some(BlobReadRange::new(0, 5)), + }, + BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None), + requested_range: Some(BlobReadRange::new(0, 3)), + }, + ]; + + let batches = plan_blob_read_batches(entries, 4).unwrap(); + + assert_eq!(batches.len(), 2); + let selection_indices = batches + .iter() + .map(|batch| { + batch + .plans + .iter() + .flat_map(|plan| plan.reads.iter()) + .map(|read| read.selection_index) + .collect::>() + }) + .collect::>(); + assert_eq!(selection_indices, vec![vec![0], vec![1]]); + } + + #[tokio::test] + async fn test_blob_read_batches_bound_coalesced_physical_bytes() { + const BLOCK_SIZE: usize = 64 * 1024; + const RANGE_GAP: u64 = 32 * 1024; + const REQUEST_COUNT: usize = 4; + + let mut data = vec![0_u8; 128 * 1024]; + for request_index in 0..REQUEST_COUNT { + data[request_index * RANGE_GAP as usize] = request_index as u8 + 1; + } + let (store, inner) = recording_range_store_with_block_size(Bytes::from(data), BLOCK_SIZE); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/dense.pack"))); + let entries = (0..REQUEST_COUNT) + .map(|request_index| BlobEntry { + selection_index: request_index, + row_address: request_index as u64, + file: BlobFile::with_source( + source.clone(), + request_index as u64 * RANGE_GAP, + 1, + BlobKind::Packed, + None, + ), + requested_range: Some(BlobReadRange::new(0, 1)), + }) + .collect(); + + let batches = plan_blob_read_batches(entries, REQUEST_COUNT as u64).unwrap(); + assert_eq!(batches.len(), REQUEST_COUNT); + + let results = execute_blob_read_batches_stream( + batches, + Arc::new(ReadBlobsExecution::new(Some(REQUEST_COUNT as u64))), + REQUEST_COUNT, + ) + .try_collect::>() + .await + .unwrap(); + + assert_eq!(results.len(), REQUEST_COUNT); + for (request_index, result) in results.iter().enumerate() { + assert_eq!(result.selection_index, request_index); + assert_eq!(result.data.as_ref(), &[request_index as u8 + 1]); + } + assert_eq!( + inner.requested_blob_ranges(), + (0..REQUEST_COUNT) + .map(|request_index| { + let start = request_index as u64 * RANGE_GAP; + start..start + 1 + }) + .collect::>() + ); + } + + #[tokio::test] + async fn test_read_blob_ranges_preserves_request_order_across_fragments() { + let fixture = BlobTestFixture::new().await; + let selections = [ + (22_u64, 2_usize, 2_usize), + (2, 0, 2), + (12, 1, 2), + (12, 1, 2), + ]; + let ranges = vec![ + BlobReadRange::new(2, 3), + BlobReadRange::new(0, 2), + BlobReadRange::new(3, 2), + BlobReadRange::new(0, 3), + ]; + + let results = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .with_row_indices( + selections + .iter() + .zip(&ranges) + .map(|((row_index, _, _), range)| { + BlobRangeRequest::new(*row_index, range.offset, range.length) + }) + .collect::>(), + ) + .execute() + .await + .unwrap(); + + assert_eq!(results.len(), selections.len()); + for (request_index, (((_, batch_index, row_index), range), result)) in + selections.iter().zip(&ranges).zip(&results).enumerate() + { + let payload = fixture.data[*batch_index] + .column(1) + .as_binary::() + .value(*row_index); + let start = range.offset as usize; + let end = start + range.length as usize; + assert_eq!(result.request_index, request_index); + assert_eq!(result.range, *range); + assert_eq!(result.data.as_deref(), Some(&payload[start..end])); + } + } + + #[tokio::test] + async fn test_blob_selection_apis_preserve_v1_nulls_and_empty_values() { + let test_dir = TempStrDir::default(); + let blob_metadata = HashMap::from([(BLOB_META_KEY.to_string(), "true".to_string())]); + let schema = Arc::new(Schema::new(vec![ + Field::new("blob", DataType::LargeBinary, true).with_metadata(blob_metadata), + ])); + let blobs = + LargeBinaryArray::from(vec![None, Some(b"".as_slice()), Some(b"abc".as_slice())]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(blobs)]).unwrap(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_0), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let blob_files = dataset + .take_blobs_by_indices(&[2, 1, 0], "blob") + .await + .unwrap(); + assert_eq!(blob_files.len(), 3); + assert_eq!( + blob_files[0] + .as_ref() + .unwrap() + .read() + .await + .unwrap() + .as_ref(), + b"abc" + ); + assert!( + blob_files[1] + .as_ref() + .unwrap() + .read() + .await + .unwrap() + .is_empty() + ); + assert!(blob_files[2].is_none()); + + let blobs = dataset + .read_blobs("blob") + .unwrap() + .with_row_indices([2, 1, 0]) + .execute() + .await + .unwrap(); + assert_eq!( + blobs + .iter() + .map(|blob| (blob.row_address, blob.data.as_deref())) + .collect::>(), + vec![ + (2, Some(b"abc".as_slice())), + (1, Some(b"".as_slice())), + (0, None), + ] + ); + + let results = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_indices([ + BlobRangeRequest::new(0, 0, 0), + BlobRangeRequest::new(0, 0, 1), + BlobRangeRequest::new(1, 0, 0), + BlobRangeRequest::new(2, 1, 1), + ]) + .execute() + .await + .unwrap(); + + assert_eq!( + results + .iter() + .map(|result| (result.request_index, result.data.as_deref())) + .collect::>(), + vec![ + (0, None), + (1, None), + (2, Some(b"".as_slice())), + (3, Some(b"b".as_slice())), + ] + ); + + let descriptions = StructArray::try_new( + vec![ + Arc::new(Field::new("position", DataType::UInt64, false)), + Arc::new(Field::new("size", DataType::UInt64, false)), + ] + .into(), + vec![ + Arc::new(UInt64Array::from(vec![0])) as ArrayRef, + Arc::new(UInt64Array::from(vec![0])) as ArrayRef, + ], + Some(NullBuffer::from(vec![false])), + ) + .unwrap(); + let row_addrs = UInt64Array::from(vec![u64::MAX]); + let files = collect_blob_files_v1(&dataset, u32::MAX, &descriptions, &row_addrs).unwrap(); + assert_eq!(files.len(), 1); + assert!(files[0].is_none()); + } + + #[tokio::test] + async fn test_blob_selection_apis_preserve_v2_nulls_and_empty_values() { + let test_dir = TempStrDir::default(); + let mut blob_builder = BlobArrayBuilder::new(3); + blob_builder.push_empty().unwrap(); + blob_builder.push_null().unwrap(); + blob_builder.push_bytes(b"abc").unwrap(); + let schema = Arc::new(Schema::new(vec![blob_field("blob", true)])); + let batch = + RecordBatch::try_new(schema.clone(), vec![blob_builder.finish().unwrap()]).unwrap(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let blob_files = dataset + .take_blobs_by_indices(&[2, 1, 0], "blob") + .await + .unwrap(); + assert_eq!(blob_files.len(), 3); + assert_eq!( + blob_files[0] + .as_ref() + .unwrap() + .read() + .await + .unwrap() + .as_ref(), + b"abc" + ); + assert!(blob_files[1].is_none()); + assert!( + blob_files[2] + .as_ref() + .unwrap() + .read() + .await + .unwrap() + .is_empty() + ); + + let blobs = dataset + .read_blobs("blob") + .unwrap() + .with_row_indices([2, 1, 0]) + .execute() + .await + .unwrap(); + assert_eq!( + blobs + .iter() + .map(|blob| (blob.row_address, blob.data.as_deref())) + .collect::>(), + vec![ + (2, Some(b"abc".as_slice())), + (1, None), + (0, Some(b"".as_slice())), + ] + ); + + let results = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_indices([ + BlobRangeRequest::new(0, 0, 0), + BlobRangeRequest::new(1, 0, 0), + BlobRangeRequest::new(1, 0, 1), + BlobRangeRequest::new(2, 1, 1), + ]) + .execute() + .await + .unwrap(); + + assert_eq!(results.len(), 4); + assert_eq!(results[0].request_index, 0); + assert_eq!(results[0].data.as_deref(), Some(b"".as_slice())); + assert_eq!(results[1].request_index, 1); + assert!(results[1].data.is_none()); + assert_eq!(results[2].request_index, 2); + assert!(results[2].data.is_none()); + assert_eq!(results[3].request_index, 3); + assert_eq!(results[3].data.as_deref(), Some(b"b".as_slice())); + } + + #[tokio::test] + async fn test_read_blob_ranges_handles_overlaps_across_scheduler_splits() { + let max_iop_size = *lance_io::object_store::DEFAULT_MAX_IOP_SIZE; + let value_size = max_iop_size + 12; + let nested_start = value_size / 2 - 1; + let nested_end = value_size; + let (store, inner) = recording_range_store(Bytes::from(vec![7; value_size as usize])); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/overlapping.bin"))); + let entries = vec![ + BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source( + source.clone(), + 0, + value_size, + BlobKind::Dedicated, + None, + ), + requested_range: Some(BlobReadRange::new(0, value_size)), + }, + BlobEntry { + selection_index: 1, + row_address: 10, + file: BlobFile::with_source(source, 0, value_size, BlobKind::Dedicated, None), + requested_range: Some(BlobReadRange::new(nested_start, nested_end - nested_start)), + }, + ]; + + let mut plans = plan_blob_read_plans(entries).unwrap(); + let results = execute_blob_read_plan( + plans.pop().unwrap(), + Arc::new(ReadBlobsExecution::new(None)), + ) + .await + .unwrap(); + + assert_eq!(results.len(), 2); + assert_eq!(results[0].selection_index, 0); + assert_eq!(results[0].data.len(), value_size as usize); + assert!(results[0].data.iter().all(|byte| *byte == 7)); + assert_eq!(results[1].selection_index, 1); + assert_eq!(results[1].data.len(), (nested_end - nested_start) as usize); + assert!(results[1].data.iter().all(|byte| *byte == 7)); + + let mut physical_ranges = inner.requested_blob_ranges(); + physical_ranges.sort_by_key(|range| range.start); + assert_eq!( + physical_ranges.len(), + value_size.div_ceil(max_iop_size) as usize + ); + assert_eq!(physical_ranges[0].start, 0); + assert_eq!(physical_ranges.last().unwrap().end, value_size); + assert!( + physical_ranges + .windows(2) + .all(|ranges| ranges[0].end == ranges[1].start) + ); + } + + #[tokio::test] + async fn test_read_blob_ranges_rejects_overflow_and_out_of_bounds() { + let fixture = BlobTestFixture::new().await; + + let err = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("requires requests")); + + let err = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .with_row_indices([BlobRangeRequest::new(0, u64::MAX, 1)]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("offset + length overflowed")); + + let err = fixture + .dataset + .read_blob_ranges("blobs") + .unwrap() + .with_row_indices([BlobRangeRequest::new(0, 0, u64::MAX)]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("exceeds blob size")); + assert!(err.to_string().contains("request 0")); + } + + #[derive(Clone, Copy, Debug)] + enum BlobSelectionStreamCase { + Normal, + Empty, + Missing, + Duplicate, + DuplicateReplacesMissing, + OutOfRange, + UpstreamError, + } + + #[rstest] + #[case::normal(BlobSelectionStreamCase::Normal)] + #[case::empty(BlobSelectionStreamCase::Empty)] + #[case::missing(BlobSelectionStreamCase::Missing)] + #[case::duplicate(BlobSelectionStreamCase::Duplicate)] + #[case::duplicate_replaces_missing(BlobSelectionStreamCase::DuplicateReplacesMissing)] + #[case::out_of_range(BlobSelectionStreamCase::OutOfRange)] + #[case::upstream_error(BlobSelectionStreamCase::UpstreamError)] + #[tokio::test] + async fn test_blob_selection_stream( + #[case] case: BlobSelectionStreamCase, + #[values(true, false)] preserve_order: bool, + ) { + let (non_null, null_values, selection_count, mut expected_values, expected_error) = + match case { + BlobSelectionStreamCase::Normal => ( + vec![Ok((2, "row2")), Ok((0, "row0"))], + vec![(1, "null1")], + 3, + vec!["row0", "null1", "row2"], + None, + ), + BlobSelectionStreamCase::Empty => (Vec::new(), Vec::new(), 0, Vec::new(), None), + BlobSelectionStreamCase::Missing => ( + vec![Ok((0, "row0"))], + vec![(1, "null1")], + 3, + vec!["row0", "null1"], + Some(if preserve_order { + "completed before selection index 2 was produced" + } else { + "completed after producing 2 of 3 selections" + }), + ), + BlobSelectionStreamCase::Duplicate => ( + vec![Ok((0, "row0")), Ok((0, "duplicate"))], + Vec::new(), + 1, + vec!["row0"], + Some("selection index 0 was produced more than once"), + ), + BlobSelectionStreamCase::DuplicateReplacesMissing => ( + vec![Ok((0, "row0")), Ok((0, "duplicate"))], + Vec::new(), + 2, + vec!["row0"], + Some("selection index 0 was produced more than once"), + ), + BlobSelectionStreamCase::OutOfRange => ( + vec![Ok((1, "out-of-range"))], + Vec::new(), + 1, + Vec::new(), + Some("selection index 1 exceeded selected row count 1"), + ), + BlobSelectionStreamCase::UpstreamError => ( + vec![ + Ok((0, "row0")), + Err(Error::internal("planned blob read failed".to_string())), + ], + Vec::new(), + 1, + vec!["row0"], + Some("planned blob read failed"), + ), + }; + let non_null = futures::stream::iter(non_null).boxed(); + let results = super::totalize_blob_selection_stream( + non_null, + null_values, + selection_count, + preserve_order, + ) + .take(8) + .collect::>() + .await; + + let mut actual_values = results + .iter() + .filter_map(|result| result.as_ref().ok().copied()) + .collect::>(); + if !preserve_order { + actual_values.sort_unstable(); + expected_values.sort_unstable(); + } + assert_eq!(actual_values, expected_values); + + let errors = results + .iter() + .filter_map(|result| result.as_ref().err()) + .collect::>(); + assert_eq!( + errors.len(), + if expected_error.is_some() { 1 } else { 0 }, + "stream must terminate after its first error: {errors:?}" + ); + if let Some(expected_error) = expected_error { + assert!(errors[0].to_string().contains(expected_error)); + } + } + + #[tokio::test] + async fn test_ordered_blob_selection_stream_drains_ready_prefix_before_reads() { + let read_poll_count = Arc::new(AtomicUsize::new(0)); + let read_poll_count_for_stream = read_poll_count.clone(); + let read_values = futures::stream::once(async move { + read_poll_count_for_stream.fetch_add(1, Ordering::SeqCst); + Ok::<_, Error>((2, "row2")) + }) + .boxed(); + let mut values = super::totalize_blob_selection_stream( + read_values, + vec![(0, "null0"), (1, "null1")], + 3, + true, + ); - let err = blob.read_range(1..5).await.unwrap_err(); - assert!(err.to_string().contains("exceeds blob size")); + assert_eq!(values.try_next().await.unwrap(), Some("null0")); + assert_eq!(read_poll_count.load(Ordering::SeqCst), 0); + assert_eq!(values.try_next().await.unwrap(), Some("null1")); + assert_eq!(read_poll_count.load(Ordering::SeqCst), 0); + assert_eq!(values.try_next().await.unwrap(), Some("row2")); + assert_eq!(read_poll_count.load(Ordering::SeqCst), 1); + assert_eq!(values.try_next().await.unwrap(), None); } #[tokio::test] - async fn test_blob_files_share_source_and_coalesce() { - let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); - let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); - let blob1 = BlobFile::with_source(source.clone(), 1, 3, BlobKind::Packed, None); - let blob2 = BlobFile::with_source(source, 4, 3, BlobKind::Packed, None); + async fn test_planned_blob_range_avoids_whole_value_read_amplification() { + const VALUE_SIZE: u64 = 500 * 1024 * 1024; + const WINDOW_SIZE: u64 = 100 * 1024; + + let (store, inner) = recording_range_store(Bytes::from(vec![7; WINDOW_SIZE as usize])); + let source = Arc::new(BlobSource::new(store, Path::from("blobs/large.bin"))); + source + .file_size + .set(std::num::NonZeroU64::new(VALUE_SIZE).unwrap()); + let entries = vec![BlobEntry { + selection_index: 0, + row_address: 10, + file: BlobFile::with_source(source, 0, VALUE_SIZE, BlobKind::Dedicated, None), + requested_range: Some(BlobReadRange::new(0, WINDOW_SIZE)), + }]; + let mut plans = plan_blob_read_plans(entries).unwrap(); + let results = execute_blob_read_plan( + plans.pop().unwrap(), + Arc::new(ReadBlobsExecution::new(None)), + ) + .await + .unwrap(); - let (data1, data2) = tokio::join!(blob1.read(), blob2.read()); - assert_eq!(data1.unwrap().as_ref(), b"bcd"); - assert_eq!(data2.unwrap().as_ref(), b"efg"); - assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + assert_eq!(results.len(), 1); + assert_eq!(results[0].data.len(), WINDOW_SIZE as usize); + let physical_ranges = inner.requested_blob_ranges(); + assert_eq!(physical_ranges, vec![0..WINDOW_SIZE]); + assert_eq!( + physical_ranges + .iter() + .map(|range| range.end - range.start) + .sum::(), + WINDOW_SIZE + ); + assert!(!physical_ranges.contains(&(0..VALUE_SIZE))); } #[tokio::test] - async fn test_read_blobs_plan_preserves_order_and_coalesces() { - let (store, inner) = recording_range_store(Bytes::from_static(b"abcdefghij")); - let source = Arc::new(BlobSource::new(store, Path::from("blobs/test.bin"))); - let entries = vec![ - BlobEntry { - selection_index: 0, - row_address: 10, - file: BlobFile::with_source(source.clone(), 4, 3, BlobKind::Packed, None), - }, - BlobEntry { - selection_index: 1, - row_address: 11, - file: BlobFile::with_source(source, 1, 3, BlobKind::Packed, None), - }, - ]; - let execution = Arc::new(ReadBlobsExecution::new(None)); - let blobs = try_join_all( - plan_blob_read_plans(entries) - .into_iter() - .map(|plan| execute_blob_read_plan(plan, execution.clone())), + async fn test_read_blob_ranges_avoids_whole_value_read_amplification_end_to_end() { + const VALUE_SIZE: usize = 8 * 1024 * 1024; + const WINDOW_SIZE: u64 = 100 * 1024; + + let test_dir = TempStrDir::default(); + let mut blob_builder = BlobArrayBuilder::new(1); + blob_builder.push_bytes(vec![0xA5; VALUE_SIZE]).unwrap(); + let mut blob_field = blob_field("blob", false); + let mut metadata = blob_field.metadata().clone(); + metadata.insert( + BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY.to_string(), + "1".to_string(), + ); + blob_field = blob_field.with_metadata(metadata); + let schema = Arc::new(Schema::new(vec![blob_field])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(blob_builder.finish().unwrap())], ) - .await .unwrap(); - let mut blobs = blobs.into_iter().flatten().collect::>(); - blobs.sort_by_key(|blob| blob.selection_index); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); - assert_eq!(blobs.len(), 2); - assert_eq!(blobs[0].row_address, 10); - assert_eq!(blobs[0].data.as_ref(), b"efg"); - assert_eq!(blobs[1].row_address, 11); - assert_eq!(blobs[1].data.as_ref(), b"bcd"); - assert_eq!(inner.requested_blob_ranges(), vec![1..7]); + let _ = dataset.object_store.io_stats_incremental(); + let results = dataset + .read_blob_ranges("blob") + .unwrap() + .with_row_indices([BlobRangeRequest::new(0, 4 * 1024 * 1024, WINDOW_SIZE)]) + .execute() + .await + .unwrap(); + let stats = dataset.object_store.io_stats_incremental(); + + assert_eq!(results.len(), 1); + let data = results[0].data.as_ref().unwrap(); + assert_eq!(data.len(), WINDOW_SIZE as usize); + assert!(data.iter().all(|byte| *byte == 0xA5)); + let blob_payload_ranges = stats + .requests + .iter() + .filter(|request| request.path.as_ref().ends_with(".blob")) + .filter_map(|request| request.range.clone()) + .collect::>(); + assert_eq!( + blob_payload_ranges, + vec![4 * 1024 * 1024..4 * 1024 * 1024 + WINDOW_SIZE] + ); } #[tokio::test] @@ -4000,6 +7675,7 @@ mod tests { BlobKind::Packed, None, ), + requested_range: None, }, BlobEntry { selection_index: 1, @@ -4011,11 +7687,13 @@ mod tests { BlobKind::Packed, None, ), + requested_range: None, }, ]; let execution = Arc::new(ReadBlobsExecution::new(None)); let mut stream: super::ReadBlobsStream = futures::stream::iter( plan_blob_read_plans(entries) + .unwrap() .into_iter() .map(move |plan| execute_blob_read_plan(plan, execution.clone())), ) @@ -4036,16 +7714,16 @@ mod tests { .unwrap() .unwrap(); assert_eq!(first.row_address, 11); - assert_eq!(first.data.as_ref(), b"uvw"); + assert_eq!(first.data.as_deref(), Some(b"uvw".as_slice())); - slow_gate.notify_one(); + slow_gate.add_permits(1); let second = tokio::time::timeout(Duration::from_secs(1), stream.next()) .await .unwrap() .unwrap() .unwrap(); assert_eq!(second.row_address, 10); - assert_eq!(second.data.as_ref(), b"abc"); + assert_eq!(second.data.as_deref(), Some(b"abc".as_slice())); } #[tokio::test] @@ -4059,9 +7737,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Inline); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Inline); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4079,9 +7758,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4097,9 +7777,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Dedicated); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Dedicated); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4117,9 +7798,10 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); assert_eq!( - blobs[0].read().await.unwrap().as_ref(), + blob.read().await.unwrap().as_ref(), fixture.expected.as_slice() ); } @@ -4220,27 +7902,83 @@ mod tests { .unwrap(), ); - let desc = dataset + let descriptor_batch = dataset .scan() .project(&["blob"]) .unwrap() + .with_row_address() .try_into_batch() .await + .unwrap(); + { + let desc = descriptor_batch.column_by_name("blob").unwrap().as_struct(); + assert_eq!( + desc.column(0).as_primitive::().value(0), + BlobKind::External as u8 + ); + assert_eq!(desc.column(2).as_primitive::().value(0), 0); + assert_eq!(desc.column(3).as_primitive::().value(0), 0); + let expected_uri = super::normalize_external_absolute_uri(&external_uri).unwrap(); + assert_eq!(desc.column(4).as_string::().value(0), expected_uri); + } + + let output_schema = dataset + .empty_projection() + .union_columns(["blob"], OnMissing::Error) .unwrap() - .column(0) - .as_struct() - .to_owned(); + .with_blob_handling(BlobHandling::AllBinary) + .to_schema(); + let descriptor_bytes = + u64::try_from(descriptor_batch.get_array_memory_size()).unwrap_or(u64::MAX); + let context = super::BlobMaterializationContext::new(None, Some(1)); + let materialized = super::materialize_blob_v2_binary_batch_with_context( + &dataset, + &output_schema, + descriptor_batch, + &context, + ) + .await + .unwrap(); + let reserved = context + .budget + .as_ref() + .unwrap() + .state + .lock() + .unwrap() + .reserved; assert_eq!( - desc.column(0).as_primitive::().value(0), - BlobKind::External as u8 + reserved, + descriptor_bytes + b"outside".len() as u64 + 2 * std::mem::size_of::() as u64 + ); + assert_eq!( + materialized + .batch() + .column_by_name("blob") + .unwrap() + .as_binary::() + .value(0), + b"outside" + ); + drop(materialized); + assert_eq!( + context + .budget + .as_ref() + .unwrap() + .state + .lock() + .unwrap() + .reserved, + 0 ); - assert_eq!(desc.column(3).as_primitive::().value(0), 0); - let expected_uri = super::normalize_external_absolute_uri(&external_uri).unwrap(); - assert_eq!(desc.column(4).as_string::().value(0), expected_uri); let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"outside"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"outside" + ); } #[tokio::test] @@ -4303,7 +8041,10 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"mapped"); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"mapped" + ); } #[tokio::test] @@ -4388,8 +8129,288 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Inline); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), b"inline"); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Inline); + assert_eq!(blob.read().await.unwrap().as_ref(), b"inline"); + } + + #[tokio::test] + async fn test_complete_blob_v2_schema_survives_create() { + let dataset_dir = TempDir::default(); + let schema = Arc::new(Schema::new(vec![complete_blob_v2_field("blob", true)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![complete_blob_v2_array( + vec![Some(b"created".to_vec())], + vec![None], + vec![None], + vec![None], + None, + )], + ) + .unwrap(); + + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &dataset_dir.path_str(), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let dataset_schema = Schema::from(dataset.schema()); + let DataType::Struct(fields) = dataset_schema.field_with_name("blob").unwrap().data_type() + else { + panic!("expected complete logical blob struct after create"); + }; + assert_eq!(fields.as_ref(), BLOB_V2_LOGICAL_FIELDS.as_ref()); + + let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"created" + ); + } + + #[tokio::test] + async fn test_complete_blob_v2_schema_survives_append() { + let dataset_dir = TempDir::default(); + let schema = Arc::new(Schema::new(vec![complete_blob_v2_field("blob", true)])); + let initial_batch = RecordBatch::try_new( + schema.clone(), + vec![complete_blob_v2_array( + vec![Some(b"initial".to_vec())], + vec![None], + vec![None], + vec![None], + None, + )], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), + &dataset_dir.path_str(), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + + let append_batch = RecordBatch::try_new( + schema.clone(), + vec![complete_blob_v2_array( + vec![Some(b"appended".to_vec())], + vec![None], + vec![None], + vec![None], + None, + )], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(append_batch)], schema), + None, + ) + .await + .unwrap(); + + let dataset = Arc::new(dataset); + let dataset_schema = Schema::from(dataset.schema()); + let DataType::Struct(fields) = dataset_schema.field_with_name("blob").unwrap().data_type() + else { + panic!("expected complete logical blob struct after append"); + }; + assert_eq!(fields.as_ref(), BLOB_V2_LOGICAL_FIELDS.as_ref()); + + let blobs = dataset + .take_blobs_by_indices(&[0, 1], "blob") + .await + .unwrap(); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"initial" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"appended" + ); + } + + #[rstest] + #[case::reference_missing_size( + ExternalBlobMode::Reference, + Some("file:///source.bin"), + Some(3), + None, + "both `position` and `size`" + )] + #[case::reference_missing_position( + ExternalBlobMode::Reference, + Some("file:///source.bin"), + None, + Some(2), + "both `position` and `size`" + )] + #[case::reference_range_without_uri( + ExternalBlobMode::Reference, + None, + Some(3), + Some(2), + "`uri` is null" + )] + #[case::ingest_missing_size( + ExternalBlobMode::Ingest, + Some("file:///source.bin"), + Some(3), + None, + "both `position` and `size`" + )] + #[case::ingest_missing_position( + ExternalBlobMode::Ingest, + Some("file:///source.bin"), + None, + Some(2), + "both `position` and `size`" + )] + #[case::ingest_range_without_uri( + ExternalBlobMode::Ingest, + None, + Some(3), + Some(2), + "`uri` is null" + )] + #[tokio::test] + async fn test_complete_blob_v2_rejects_invalid_ranges( + #[case] external_blob_mode: ExternalBlobMode, + #[case] uri: Option<&str>, + #[case] position: Option, + #[case] size: Option, + #[case] expected_message: &str, + ) { + let dataset_dir = TempDir::default(); + let schema = Arc::new(Schema::new(vec![complete_blob_v2_field("blob", true)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![complete_blob_v2_array( + vec![None], + vec![uri.map(str::to_string)], + vec![position], + vec![size], + None, + )], + ) + .unwrap(); + + let error = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &dataset_dir.path_str(), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + allow_external_blob_outside_bases: matches!( + external_blob_mode, + ExternalBlobMode::Reference + ), + external_blob_mode, + ..Default::default() + }), + ) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains(expected_message)); + } + + #[rstest] + #[case::reference(ExternalBlobMode::Reference)] + #[case::ingest(ExternalBlobMode::Ingest)] + #[tokio::test] + async fn test_complete_blob_v2_rejects_zero_size_range( + #[case] external_blob_mode: ExternalBlobMode, + ) { + let dataset_dir = TempDir::default(); + let schema = Arc::new(Schema::new(vec![complete_blob_v2_field("blob", true)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![complete_blob_v2_array( + vec![None], + vec![Some("file:///source.bin".to_string())], + vec![Some(3)], + vec![Some(0)], + None, + )], + ) + .unwrap(); + + let error = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &dataset_dir.path_str(), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + allow_external_blob_outside_bases: matches!( + external_blob_mode, + ExternalBlobMode::Reference + ), + external_blob_mode, + ..Default::default() + }), + ) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("greater than zero")); + } + + #[rstest] + #[case::both_small(Some(5), true)] + #[case::both_packed(Some(crate::dataset::blob::INLINE_MAX + 1), true)] + #[case::neither(None, false)] + #[tokio::test] + async fn test_complete_blob_v2_rejects_invalid_representation( + #[case] data_size: Option, + #[case] has_uri: bool, + ) { + let dataset_dir = TempDir::default(); + let schema = Arc::new(Schema::new(vec![complete_blob_v2_field("blob", true)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![complete_blob_v2_array( + vec![data_size.map(|size| vec![0x41; size])], + vec![has_uri.then(|| "file:///source.bin".to_string())], + vec![None], + vec![None], + None, + )], + ) + .unwrap(); + + let error = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &dataset_dir.path_str(), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + allow_external_blob_outside_bases: true, + ..Default::default() + }), + ) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("must set exactly one of `data` and `uri`") + ); } #[tokio::test] @@ -4442,8 +8463,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice()); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); + assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice()); } #[tokio::test] @@ -4486,8 +8508,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Packed); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice()); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Packed); + assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice()); } #[tokio::test] @@ -4540,8 +8563,9 @@ mod tests { let blobs = dataset.take_blobs_by_indices(&[0], "blob").await.unwrap(); assert_eq!(blobs.len(), 1); - assert_eq!(blobs[0].kind(), BlobKind::Dedicated); - assert_eq!(blobs[0].read().await.unwrap().as_ref(), payload.as_slice()); + let blob = blobs[0].as_ref().unwrap(); + assert_eq!(blob.kind(), BlobKind::Dedicated); + assert_eq!(blob.read().await.unwrap().as_ref(), payload.as_slice()); } #[tokio::test] diff --git a/rust/lance/src/dataset/builder.rs b/rust/lance/src/dataset/builder.rs index aecbf92f0d8..f7619a1ceb4 100644 --- a/rust/lance/src/dataset/builder.rs +++ b/rust/lance/src/dataset/builder.rs @@ -9,9 +9,8 @@ use super::{DEFAULT_INDEX_CACHE_SIZE, DEFAULT_METADATA_CACHE_SIZE, ReadParams, W use crate::dataset::branch_location::BranchLocation; use crate::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore; use crate::{Dataset, Error, Result, session::Session}; -use futures::FutureExt; +use futures::{FutureExt, TryStreamExt}; use lance_core::utils::tracing::{DATASET_LOADING_EVENT, TRACE_DATASET_EVENTS}; -use lance_file::datatypes::populate_schema_dictionary; use lance_file::reader::FileReaderOptions; use lance_io::object_store::{ DEFAULT_CLOUD_IO_PARALLELISM, LanceNamespaceStorageOptionsProvider, ObjectStore, @@ -20,9 +19,10 @@ use lance_io::object_store::{ use lance_namespace::LanceNamespace; use lance_namespace::models::DescribeTableRequest; use lance_table::{ - format::Manifest, + feature_flags::ensure_can_read_manifest, + format::{Manifest, populate_manifest_schema_dictionaries}, io::commit::external_manifest::ExternalManifestCommitHandler, - io::commit::{CommitHandler, commit_handler_from_url}, + io::commit::{CommitHandler, ManifestLocation, commit_handler_from_url}, }; #[cfg(feature = "aws")] use object_store::aws::AwsCredentialProvider; @@ -600,6 +600,36 @@ impl DatasetBuilder { Ok((object_store, base_path, commit_handler)) } + /// List manifest locations without reading the manifest contents. + /// + /// The returned locations are not guaranteed to be ordered. This operation may list and + /// materialize the full manifest history. Explicit version, branch, and tag targets are not + /// supported. Custom commit handlers and externally managed version stores are also not + /// supported because listing physical manifest objects may omit committed versions whose + /// authoritative locations are held outside the object store. + pub async fn list_manifest_locations(mut self) -> Result> { + if self.version.is_some() { + return Err(Error::invalid_input( + "list_manifest_locations does not support an explicit version, branch, or tag", + )); + } + let uses_external_or_custom_commit_handler = self.commit_handler.is_some() + || self.namespace_managed.is_some() + || Url::parse(&self.table_uri).is_ok_and(|url| url.scheme() == "s3+ddb"); + if uses_external_or_custom_commit_handler { + return Err(Error::not_supported( + "list_manifest_locations does not support external or custom commit handlers; \ + object-store listing may omit committed manifest locations", + )); + } + self.apply_storage_options_override(); + let (object_store, base_path, commit_handler) = self.build_object_store().await?; + commit_handler + .list_manifest_locations(&base_path, object_store.as_ref(), false) + .try_collect() + .await + } + #[instrument(skip_all)] pub async fn load(self) -> Result { let uri = self.table_uri.clone(); @@ -646,12 +676,16 @@ impl DatasetBuilder { merged_params } - async fn load_impl(mut self) -> Result { - // Apply storage_options_override to merge namespace client options with any existing accessor - if let Some(override_opts) = self.storage_options_override.take() { + fn apply_storage_options_override(&mut self) { + if let Some(override_options) = self.storage_options_override.take() { self.options = - Self::merge_store_params_with_storage_options(&self.options, &override_opts); + Self::merge_store_params_with_storage_options(&self.options, &override_options); } + } + + async fn load_impl(mut self) -> Result { + // Apply storage_options_override to merge namespace client options with any existing accessor + self.apply_storage_options_override(); let index_cache_backend = self.index_cache_backend.take(); let session = match self.session.as_ref() { @@ -835,12 +869,13 @@ impl DatasetBuilder { base_store_params: Option>>, ) -> Result { let (manifest, location) = if let Some(mut manifest) = manifest { + ensure_can_read_manifest(&manifest)?; let location = commit_handler .resolve_version_location(&base_path, manifest.version, &object_store.inner) .await?; - if manifest.schema.has_dictionary_types() && manifest.should_use_legacy_format() { + if manifest.schema.has_dictionary_types() { let reader = object_store.open(&location.path).await?; - populate_schema_dictionary(&mut manifest.schema, reader.as_ref()).await?; + populate_manifest_schema_dictionaries(&mut manifest, reader.as_ref()).await?; } (Arc::new(manifest), location) } else { @@ -900,3 +935,97 @@ impl DatasetBuilder { ) } } + +#[cfg(test)] +mod tests { + use async_trait::async_trait; + use lance_io::object_store::StorageOptionsProvider; + use lance_table::io::commit::UnsafeCommitHandler; + + use super::*; + + #[derive(Debug)] + struct TestStorageOptionsProvider; + + #[derive(Debug)] + struct TestNamespace; + + #[async_trait] + impl LanceNamespace for TestNamespace { + fn namespace_id(&self) -> String { + "test-namespace".to_string() + } + } + + #[async_trait] + impl StorageOptionsProvider for TestStorageOptionsProvider { + async fn fetch_storage_options(&self) -> Result>> { + Ok(None) + } + + fn provider_id(&self) -> String { + "test-storage-options-provider".to_string() + } + } + + #[test] + fn test_storage_options_override_wins_and_preserves_provider() { + let caller_options = HashMap::from([ + ("endpoint".to_string(), "caller".to_string()), + ("caller-only".to_string(), "caller-value".to_string()), + ]); + let provider: Arc = Arc::new(TestStorageOptionsProvider); + let options = ObjectStoreParams { + storage_options_accessor: Some(Arc::new( + StorageOptionsAccessor::with_initial_and_provider(caller_options, provider), + )), + ..Default::default() + }; + let mut builder = DatasetBuilder::from_uri("memory://table").with_store_params(options); + builder.storage_options_override = Some(HashMap::from([ + ("endpoint".to_string(), "namespace".to_string()), + ("namespace-only".to_string(), "namespace-value".to_string()), + ])); + + builder.apply_storage_options_override(); + + let merged = builder.options.storage_options().unwrap(); + assert_eq!(merged.get("endpoint").unwrap(), "namespace"); + assert_eq!(merged.get("caller-only").unwrap(), "caller-value"); + assert_eq!(merged.get("namespace-only").unwrap(), "namespace-value"); + assert_eq!( + builder + .options + .get_accessor() + .unwrap() + .provider() + .unwrap() + .provider_id(), + "test-storage-options-provider" + ); + assert!(builder.storage_options_override.is_none()); + } + + #[tokio::test] + async fn test_list_manifest_locations_rejects_external_and_custom_commit_handlers() { + let mut namespace_builder = DatasetBuilder::from_uri("memory://namespace-table"); + namespace_builder.namespace_managed = + Some((Arc::new(TestNamespace), vec!["namespace-table".to_string()])); + + let builders = [ + DatasetBuilder::from_uri("memory://custom-handler-table") + .with_commit_handler(Arc::new(UnsafeCommitHandler)), + DatasetBuilder::from_uri("s3+ddb://bucket/table.lance?ddbTableName=manifest-table"), + namespace_builder, + ]; + + for builder in builders { + let err = builder.list_manifest_locations().await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + assert!( + err.to_string() + .contains("does not support external or custom commit handlers") + ); + } + } +} diff --git a/rust/lance/src/dataset/cleanup.rs b/rust/lance/src/dataset/cleanup.rs index 65928038cea..3d4fe59f2aa 100644 --- a/rust/lance/src/dataset/cleanup.rs +++ b/rust/lance/src/dataset/cleanup.rs @@ -58,8 +58,8 @@ use lance_table::{ manifest::{read_manifest, read_manifest_indexes}, }, }; +use object_store::ObjectMeta; use object_store::path::Path; -use object_store::{Error as ObjectStoreError, ObjectMeta}; use std::fmt::Debug; use std::{ collections::{HashMap, HashSet}, @@ -69,7 +69,7 @@ use std::{ }; use tokio::time::{MissedTickBehavior, interval}; use tokio_stream::wrappers::IntervalStream; -use tracing::{Span, debug, info, instrument}; +use tracing::{Span, debug, info, instrument, warn}; #[derive(Clone, Debug, Default)] struct ReferencedFiles { @@ -302,6 +302,9 @@ struct CleanupTask<'a> { #[derive(Clone, Debug, Default)] struct CleanupInspection { old_manifests: HashMap, + /// Store records to retire once their manifests are gone, by version; + /// see `CommitHandler::forget_version`. + retired_records: HashMap, /// Referenced files are part of our working set referenced_files: ReferencedFiles, /// Verified files may or may not be part of the working set but they are @@ -312,6 +315,32 @@ struct CleanupInspection { tagged_old_versions: HashSet, /// The earliest timestamp of all retained manifests. earliest_retained_manifest_time: Option>, + /// The latest timestamp of all manifests that will be removed. + latest_deleted_manifest_time: Option>, +} + +impl CleanupInspection { + /// Cutoff for `read_dir_all(..., unmodified_since)`. + /// + /// Listing only files with `last_modified <= earliest_retained` is valid + /// when the working set is a time suffix: every retained version is newer + /// than every deleted one. A tagged old version (or any other sparse + /// retain) pulls that cutoff backwards, so files from newer deleted + /// versions are never listed. Their manifests are still removed, which + /// permanently orphans the data files ([#8705](https://github.com/lance-format/lance/issues/8705)). + /// + /// When a deleted manifest is newer than the earliest retained one, drop + /// the cutoff and scan the whole subtree — the same approach already used + /// for `_indices/`. + fn listing_unmodified_since(&self) -> Option> { + match ( + self.earliest_retained_manifest_time, + self.latest_deleted_manifest_time, + ) { + (Some(retained), Some(deleted)) if deleted > retained => None, + (retained, _) => retained, + } + } } /// If a file cannot be verified then it will only be deleted if it is at least @@ -523,17 +552,43 @@ impl<'a> CleanupTask<'a> { // ignore it then we might delete valid data files thinking they are not // referenced. - let manifest = - read_manifest(&self.dataset.object_store, &location.path, location.size).await?; + let manifest_and_indexes = async { + let manifest = + read_manifest(&self.dataset.object_store, &location.path, location.size).await?; + let indexes = + read_manifest_indexes(&self.dataset.object_store, &location, &manifest).await?; + Ok::<_, Error>((manifest, indexes)) + } + .await; + let (manifest, indexes) = match manifest_and_indexes { + Ok(manifest_and_indexes) => manifest_and_indexes, + Err(error) if location.version < self.read_version && error.is_not_found() => { + // Another cleanup may remove an old manifest after this cleanup lists it. + // The current manifest is never safe to skip because it anchors our snapshot. + debug!( + manifest_version = location.version, + read_version = self.read_version, + manifest_path = %location.path, + "Skipping old manifest removed by concurrent cleanup" + ); + // Its record may still be there if that cleanup stopped early. + if let Some(identity) = location.identity { + inspection + .lock() + .unwrap() + .retired_records + .insert(location.version, identity); + } + return Ok(()); + } + Err(error) => return Err(error), + }; // Don't delete the latest version, even if it is old. Don't delete tagged versions, // regardless of age. Don't delete manifests if their version is newer than the dataset // version. These are either in-progress or newly added since we started. let is_latest = self.read_version <= manifest.version; let is_tagged = tagged_versions.contains(&manifest.version); let in_working_set = is_latest || !self.policy.should_clean(&manifest) || is_tagged; - let indexes = - read_manifest_indexes(&self.dataset.object_store, &location, &manifest).await?; - let mut inspection = inspection.lock().unwrap(); // Track tagged old versions in case we want to return a `CleanupError` later. @@ -543,18 +598,24 @@ impl<'a> CleanupTask<'a> { } self.process_manifest(&manifest, &indexes, in_working_set, &mut inspection)?; + let commit_ts = manifest.timestamp(); if !in_working_set { inspection .old_manifests .insert(location.path.clone(), manifest.version); + if let Some(identity) = location.identity.clone() { + inspection + .retired_records + .insert(manifest.version, identity); + } + match inspection.latest_deleted_manifest_time { + Some(ts) if commit_ts <= ts => {} + _ => inspection.latest_deleted_manifest_time = Some(commit_ts), + } } else { - let commit_ts = manifest.timestamp(); - if let Some(ts) = inspection.earliest_retained_manifest_time { - if commit_ts < ts { - inspection.earliest_retained_manifest_time = Some(commit_ts); - } - } else { - inspection.earliest_retained_manifest_time = Some(commit_ts); + match inspection.earliest_retained_manifest_time { + Some(ts) if commit_ts >= ts => {} + _ => inspection.earliest_retained_manifest_time = Some(commit_ts), } } Ok(()) @@ -576,7 +637,7 @@ impl<'a> CleanupTask<'a> { }; for fragment in manifest.fragments.iter() { - for file in fragment.files.iter() { + for file in fragment.referenced_lance_files() { let full_data_path = self.dataset.data_dir().clone().join(file.path.as_str()); let relative_data_path = remove_prefix(&full_data_path, &self.dataset.base); referenced_files.data_paths.insert(relative_data_path); @@ -621,26 +682,29 @@ impl<'a> CleanupTask<'a> { ) -> Result { let cleanup_result = Mutex::new(CleanupRunResult::default()); let deletes_files = self.action.deletes_files(); + let removes_empty_dirs = matches!( + self.dataset.object_store.scheme(), + "file" | "file+uring" | "file-object-store" + ); + let indices_dir = self.dataset.indices_dir(); + let retained_index_dirs = inspection + .referenced_files + .index_uuids + .iter() + .map(|uuid| indices_dir.clone().join(uuid.as_str())) + .collect::>(); + let index_dirs_to_remove = Mutex::new(HashSet::new()); let candidate_file_limit = self.action.candidate_file_limit(); let verification_threshold = utc_now() - TimeDelta::try_days(UNVERIFIED_THRESHOLD_DAYS).expect("TimeDelta::try_days"); - let is_not_found_err = |e: &Error| { - matches!( - e, - Error::IO { source,.. } - if source - .downcast_ref::() - .map(|os_err| matches!(os_err, ObjectStoreError::NotFound {.. })) - .unwrap_or(false) - ) - }; + let is_not_found_err = |e: &Error| matches!(e, Error::NotFound { .. }); // Build stream for a managed subtree - let build_listing_stream = |dir: Path| { + let build_listing_stream = |dir: Path, unmodified_since| { let inspection_ref = &inspection; self.dataset .object_store - .read_dir_all(&dir, inspection.earliest_retained_manifest_time) + .read_dir_all(&dir, unmodified_since) .map_ok(|obj| stream::once(future::ready(Ok(obj))).boxed()) .or_else(|e| { // If the directory doesn't exist then we can just return an empty stream. @@ -667,12 +731,20 @@ impl<'a> CleanupTask<'a> { }; // Restrict scanning to Lance-managed subtrees for safety and performance. + // Drop the retained-manifest cutoff when a sparse retain (e.g. a tag) + // would hide files that belong to newer deleted versions. See + // [`CleanupInspection::listing_unmodified_since`]. + let unmodified_since = inspection.listing_unmodified_since(); let streams = vec![ - build_listing_stream(self.dataset.versions_dir()), - build_listing_stream(self.dataset.transactions_dir()), - build_listing_stream(self.dataset.data_dir()), - build_listing_stream(self.dataset.indices_dir()), - build_listing_stream(self.dataset.deletions_dir()), + build_listing_stream(self.dataset.versions_dir(), unmodified_since), + build_listing_stream(self.dataset.transactions_dir(), unmodified_since), + build_listing_stream(self.dataset.data_dir(), unmodified_since), + // Index UUIDs from manifests being removed are proof that their files are + // safe to delete. Scan every index artifact while that proof is available; + // a retained-manifest cutoff can otherwise skip newer artifacts and lose + // the proof when the old manifests are removed by this cleanup pass. + build_listing_stream(self.dataset.indices_dir(), None), + build_listing_stream(self.dataset.deletions_dir(), unmodified_since), ]; let unreferenced_files = stream::iter(streams).flatten().boxed(); @@ -720,6 +792,17 @@ impl<'a> CleanupTask<'a> { .lock() .unwrap() .record_file(&file, candidate_file_limit, self.track_removed_manifests); + if deletes_files && removes_empty_dirs && matches!(file.kind, CleanupFileKind::Index) { + let mut parent = file.path.parent(); + let mut index_dirs = index_dirs_to_remove.lock().unwrap(); + while let Some(dir_path) = parent { + if dir_path == indices_dir || !dir_path.prefix_matches(&indices_dir) { + break; + } + index_dirs.insert(dir_path.clone()); + parent = dir_path.parent(); + } + } Ok(file.path) }); @@ -743,6 +826,35 @@ impl<'a> CleanupTask<'a> { .remove_stream(paths_to_delete) .try_for_each(|_| future::ready(Ok(()))) .await?; + + // Only after the objects are gone: a record that outlives its + // manifest is retired by the next cleanup, the reverse is a lost + // version. + for (version, identity) in &inspection.retired_records { + self.dataset + .commit_handler + .forget_version(&self.dataset.base, *version, identity) + .await?; + } + + if removes_empty_dirs + && let Err(error) = self + .dataset + .object_store + .remove_empty_dirs( + indices_dir.clone(), + retained_index_dirs, + index_dirs_to_remove.into_inner().unwrap(), + (!self.policy.delete_unverified).then_some(verification_threshold), + ) + .await + { + warn!( + path = indices_dir.as_ref(), + error = %error, + "Failed to remove empty index directories" + ); + } } else { // Drain the stream to populate stats, but do not call remove_stream. all_paths_to_remove @@ -1138,7 +1250,7 @@ impl<'a> CleanupTask<'a> { let mut is_referenced = false; for fragment in manifest.fragments.iter() { - for file in fragment.files.iter() { + for file in fragment.referenced_lance_files() { if let Some(base_id) = file.base_id { let base_path = manifest.base_paths.get(&base_id); if let Some(base_path) = base_path @@ -1202,6 +1314,8 @@ impl<'a> CleanupTask<'a> { inspection .old_manifests .retain(|_path, version_number| *version_number != referenced_version); + // Kept on disk, so its record stays too. + inspection.retired_records.remove(&referenced_version); } Ok(()) @@ -1233,6 +1347,8 @@ pub struct CleanupPolicy { pub before_timestamp: Option>, /// If not none, cleanup all versions before the specified version. pub before_version: Option, + /// If not none, cleanup only the specified versions. + pub versions: Option>, /// If true, delete unverified data files even if they are recent pub delete_unverified: bool, /// If true, return an Error if a tagged version is old @@ -1256,6 +1372,9 @@ impl CleanupPolicy { if let Some(before_version) = self.before_version { should_clean &= manifest.version < before_version; } + if let Some(versions) = self.versions.as_ref() { + should_clean &= versions.contains(&manifest.version); + } should_clean } } @@ -1265,6 +1384,7 @@ impl Default for CleanupPolicy { Self { before_timestamp: None, before_version: None, + versions: None, delete_unverified: false, error_if_tagged_old_versions: true, clean_referenced_branches: false, @@ -1291,8 +1411,35 @@ impl CleanupPolicyBuilder { self } + /// Cleanup only the specified dataset versions. + /// + /// This is an exact-version filter. If other policy filters are also + /// configured, a manifest is removed only when it satisfies all of them. + /// + /// # Errors + /// + /// Returns an error if `versions` is empty. + pub fn versions(mut self, versions: Vec) -> Result { + if versions.is_empty() { + return Err(Error::invalid_input( + "versions must not be empty when specified", + )); + } + self.policy.versions = Some(versions.into_iter().collect()); + Ok(self) + } + /// Cleanup all versions except the last `n` versions of the dataset. + /// + /// # Errors + /// + /// Returns an error if `n` is zero. pub async fn retain_n_versions(mut self, dataset: &Dataset, n: usize) -> Result { + if n == 0 { + return Err(Error::invalid_input(format!( + "retain_versions must be greater than 0, got {n}" + ))); + } let versions = dataset.versions().await?; self.policy.before_version = if versions.len() <= n { Some(versions[0].version) @@ -1567,6 +1714,7 @@ mod tests { use lance_table::io::commit::RenameCommitHandler; use lance_testing::datagen::{BatchGenerator, IncrementingInt32, RandomVector, some_batch}; use mock_instant::thread_local::MockClock; + use rstest::rstest; use uuid::Uuid; #[derive(Debug)] @@ -1583,6 +1731,15 @@ mod tests { ) -> Arc { Arc::new(ProxyObjectStore::new(original, self.policy.clone())) } + + // Injects behaviour into every request, so a listing must not go around it. + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } impl MockObjectStore { @@ -1633,7 +1790,7 @@ mod tests { struct MockDatasetFixture { // This is a temporary directory that will be deleted when the fixture // is dropped - _tmpdir: TempStrDir, + tmpdir: TempStrDir, dataset_path: String, mock_store: Arc, } @@ -1653,12 +1810,19 @@ mod tests { }; let dataset_path = format!("file-object-store://{path_prefix}{tmpdir_path}/my_db"); Ok(Self { - _tmpdir: tmpdir, + tmpdir, dataset_path, mock_store: Arc::new(MockObjectStore::new()), }) } + fn local_index_dir(&self, uuid: Uuid) -> std::path::PathBuf { + std::path::Path::new(self.tmpdir.as_str()) + .join("my_db") + .join(crate::dataset::INDICES_DIR) + .join(uuid.to_string()) + } + fn os_params(&self) -> ObjectStoreParams { ObjectStoreParams { object_store_wrapper: Some(self.mock_store.clone()), @@ -1969,6 +2133,7 @@ mod tests { uuid, name: "some_index".to_string(), fields: vec![field_id], + covering_fields: vec![], dataset_version: dataset.version().version, fragment_bitmap: Some(fragment_bitmap.into_iter().collect()), index_details: None, @@ -2041,6 +2206,42 @@ mod tests { assert_gt!(after_count.num_tx_files, 0); } + #[tokio::test] + async fn cleanup_ignores_old_manifest_removed_after_listing() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + let dataset = fixture.open().await.unwrap(); + + let old_manifest = dataset + .commit_handler + .list_manifest_locations(&dataset.base, &dataset.object_store, false) + .try_filter(|location| future::ready(location.version == 1)) + .try_next() + .await + .unwrap() + .unwrap(); + dataset + .object_store + .delete(&old_manifest.path) + .await + .unwrap(); + + let cleanup = CleanupTask::new( + &dataset, + CleanupPolicyBuilder::default().build(), + CleanupAction::Execute, + ); + cleanup + .process_manifest_file( + old_manifest, + &Mutex::new(CleanupInspection::default()), + &HashSet::new(), + ) + .await + .unwrap(); + } + #[tokio::test] async fn explain_cleanup_does_not_delete_files() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2320,6 +2521,49 @@ mod tests { assert_eq!(removed.old_versions, 1); } + #[tokio::test] + async fn cleanup_deletes_data_files_newer_than_tagged_version() { + // A tag on an old version must not prevent cleanup from deleting data + // files that belong only to newer, untagged versions. The listing + // cutoff used to be the earliest retained manifest time; with a tag + // that pulled the cutoff backwards and skipped those newer files. + // After their manifests were removed they became permanent orphans + // (https://github.com/lance-format/lance/issues/8705). + MockClock::set_system_time(std::time::Duration::from_secs(0)); + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + MockClock::set_system_time(TimeDelta::try_days(1).unwrap().to_std().unwrap()); + fixture.overwrite_some_data().await.unwrap(); + MockClock::set_system_time(TimeDelta::try_days(2).unwrap().to_std().unwrap()); + fixture.overwrite_some_data().await.unwrap(); + + let dataset = *(fixture.open().await.unwrap()); + dataset.tags().create("keep-v1", 1).await.unwrap(); + + MockClock::set_system_time(TimeDelta::try_days(10).unwrap().to_std().unwrap()); + + let before_count = fixture.count_files().await.unwrap(); + assert_eq!(before_count.num_data_files, 3); + assert_eq!(before_count.num_manifest_files, 3); + + let removed = fixture + .run_cleanup_with_override( + utc_now() - TimeDelta::try_days(8).unwrap(), + None, + Some(false), + ) + .await + .unwrap(); + + assert_eq!(removed.old_versions, 1); + assert_eq!(removed.data_files_removed, 1); + + let after_count = fixture.count_files().await.unwrap(); + assert_eq!(after_count.num_manifest_files, 2); + assert_eq!(after_count.num_data_files, 2); + assert_eq!(after_count.num_tx_files, 2); + } + // Helper function to check that the number of files is correct. async fn check_num_files(fixture: &MockDatasetFixture, num_expected_files: usize) { let file_count = fixture.count_files().await.unwrap(); @@ -2655,6 +2899,178 @@ mod tests { assert_gt!(removed.deletion_files_removed, 0); } + /// A branch reaches its parent's files through `base_id`, and + /// `retain_branch_lineage_files` promotes those into the parent's keep set. An + /// overlay inherited that way must be promoted too, or the parent deletes a + /// file the branch still reads. + #[tokio::test] + async fn lineage_retention_covers_inherited_overlay_files() { + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + // Give the parent an overlay, then branch from it: `shallow_clone` stamps + // the overlay's `base_id` so the branch resolves it against the parent. + let mut dataset = fixture.open().await.unwrap(); + let mut fragments: Vec<_> = dataset + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + let mut overlay_file = fragments[0].files[0].clone(); + overlay_file.path = "overlay.lance".to_string(); + fragments[0].overlays = vec![DataOverlayFile { + data_file: overlay_file, + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: dataset.manifest.version, + }]; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::Overwrite { + fragments, + schema: dataset.schema().clone(), + config_upsert_values: None, + initial_bases: None, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let root_version = dataset.manifest.version; + let branch = fixture + .create_branch_and_load(&mut dataset, "child", (None, None)) + .await + .unwrap(); + let branch_fragments = branch.get_fragments(); + let inherited = &branch_fragments[0].metadata().overlays[0].data_file; + assert!( + inherited.base_id.is_some(), + "the branch must reach the parent's overlay through base_id" + ); + + // The parent's cleanup walks the branch's manifest and must promote that + // overlay out of `verified_files`. + let task = CleanupTask::new( + &dataset, + CleanupPolicyBuilder::default() + .before_timestamp(utc_now()) + .build(), + CleanupAction::Execute, + ); + let inspection = task.process_manifests(&HashSet::new()).await.unwrap(); + // Queue the branch root for removal on both sides; rescuing the + // manifest must rescue its store record with it. + let inspection = Mutex::new(inspection); + { + let mut queued = inspection.lock().unwrap(); + queued + .old_manifests + .insert(Path::from("_versions/root.manifest"), root_version); + queued + .retired_records + .insert(root_version, "root-identity".to_string()); + } + task.process_branch_referenced_manifests( + branch.manifest_location.clone(), + root_version, + &inspection, + ) + .await + .unwrap(); + let inspection = inspection.into_inner().unwrap(); + assert!( + !inspection + .old_manifests + .values() + .any(|v| *v == root_version) + ); + assert!( + !inspection.retired_records.contains_key(&root_version), + "a retained branch root must not be retired from authoritative history" + ); + let referenced_branches = task.find_referenced_branches().await.unwrap(); + let inspection = task + .retain_branch_lineage_files(inspection, &referenced_branches, &HashSet::new()) + .await + .unwrap(); + + let overlay_path = Path::from("data/overlay.lance"); + assert!( + inspection + .referenced_files + .data_paths + .contains(&overlay_path), + "the inherited overlay must be promoted into the parent's keep set" + ); + } + + /// A keep set built from `fragment.files` alone omits overlay data files, so + /// cleanup would delete live data. + #[tokio::test] + async fn keep_set_covers_referenced_overlay_files() { + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + // The overlay file need not exist: the keep set comes from manifest + // metadata alone. + let mut dataset = fixture.open().await.unwrap(); + let mut fragments: Vec<_> = dataset + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + let mut overlay_file = fragments[0].files[0].clone(); + overlay_file.path = "overlay.lance".to_string(); + fragments[0].overlays = vec![DataOverlayFile { + data_file: overlay_file, + coverage: OverlayCoverage::Shared(Arc::new(RoaringBitmap::from_iter([0_u32]))), + committed_version: dataset.manifest.version, + }]; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::Overwrite { + fragments, + schema: dataset.schema().clone(), + config_upsert_values: None, + initial_bases: None, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let task = CleanupTask::new( + &dataset, + CleanupPolicyBuilder::default() + .before_timestamp(utc_now()) + .build(), + CleanupAction::Execute, + ); + let inspection = task.process_manifests(&HashSet::new()).await.unwrap(); + let kept: HashSet<&Path> = inspection + .referenced_files + .data_paths + .iter() + .chain(inspection.verified_files.data_paths.iter()) + .collect(); + + let overlay_path = Path::from("data/overlay.lance"); + assert!( + kept.contains(&overlay_path), + "the overlay's data file must be in the keep set, got {kept:?}" + ); + } + #[tokio::test] async fn dont_clean_index_data_files() { // Indexes have .lance files in them that are not referenced @@ -2676,6 +3092,119 @@ mod tests { assert_eq!(before_count, after_count); } + #[tokio::test] + async fn cleanup_removes_preexisting_empty_index_directories() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + let mut dataset = fixture.open().await.unwrap(); + let field_id = dataset.schema().field("indexable").unwrap().id; + let stale_uuid = Uuid::new_v4(); + let nested_stale_uuid = Uuid::new_v4(); + let referenced_uuid = Uuid::new_v4(); + + std::fs::create_dir_all(fixture.local_index_dir(stale_uuid)).unwrap(); + std::fs::create_dir_all( + fixture + .local_index_dir(nested_stale_uuid) + .join("empty_nested_dir"), + ) + .unwrap(); + std::fs::create_dir_all(fixture.local_index_dir(referenced_uuid)).unwrap(); + + let referenced_index = dummy_index_metadata(&dataset, field_id, referenced_uuid, [0_u32]); + let create_index_tx = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![referenced_index], + removed_indices: vec![], + }, + None, + ); + dataset + .apply_commit(create_index_tx, &Default::default(), &Default::default()) + .await + .unwrap(); + + let real_now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + MockClock::set_system_time(real_now + TimeDelta::try_days(10).unwrap().to_std().unwrap()); + let in_progress_uuid = Uuid::new_v4(); + write_dummy_index_artifact(&dataset, in_progress_uuid) + .await + .unwrap(); + let in_progress_empty_dir = fixture + .local_index_dir(in_progress_uuid) + .join("empty_in_progress_dir"); + std::fs::create_dir_all(&in_progress_empty_dir).unwrap(); + + let removed = fixture + .run_cleanup(utc_now() - TimeDelta::try_days(7).unwrap()) + .await + .unwrap(); + + assert_eq!(removed.index_files_removed, 0); + assert!(!fixture.local_index_dir(stale_uuid).exists()); + assert!(!fixture.local_index_dir(nested_stale_uuid).exists()); + assert!(fixture.local_index_dir(referenced_uuid).exists()); + assert!(fixture.local_index_dir(in_progress_uuid).exists()); + assert!(in_progress_empty_dir.exists()); + } + + #[rstest] + #[case::default_policy(false, true)] + #[case::delete_unverified(true, false)] + #[tokio::test] + async fn cleanup_applies_unverified_policy_to_fresh_empty_index_directory( + #[case] delete_unverified: bool, + #[case] should_preserve: bool, + ) { + let real_now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(); + MockClock::set_system_time(real_now); + + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + let in_progress_dir = fixture.local_index_dir(Uuid::new_v4()); + std::fs::create_dir_all(&in_progress_dir).unwrap(); + + fixture + .run_cleanup_with_override( + utc_now() - TimeDelta::try_days(7).unwrap(), + Some(delete_unverified), + None, + ) + .await + .unwrap(); + + assert_eq!(in_progress_dir.exists(), should_preserve); + } + + #[cfg(unix)] + #[tokio::test] + async fn cleanup_does_not_remove_empty_directory_through_index_symlink() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + let outside = tempfile::tempdir().unwrap(); + let outside_empty_dir = outside.path().join("must_remain"); + std::fs::create_dir_all(&outside_empty_dir).unwrap(); + + let link_path = fixture.local_index_dir(Uuid::new_v4()); + std::fs::create_dir_all(link_path.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink(outside.path(), &link_path).unwrap(); + + fixture + .run_cleanup_with_override(utc_now(), Some(true), None) + .await + .unwrap(); + + assert!(outside_empty_dir.exists()); + assert!(link_path.is_symlink()); + } + #[tokio::test] async fn cleanup_old_replaced_segment_keeps_still_referenced_segments() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2728,6 +3257,7 @@ mod tests { .unwrap(); assert_eq!(removed.index_files_removed, 2); + assert!(!fixture.local_index_dir(seg_a).exists()); assert!( !dataset .object_store @@ -2772,6 +3302,96 @@ mod tests { ); } + #[tokio::test] + async fn cleanup_recent_replaced_index_with_short_retention() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + fixture.append_some_data().await.unwrap(); + + let mut dataset = fixture.open().await.unwrap(); + let field_id = dataset.schema().field("indexable").unwrap().id; + let old_uuid = Uuid::new_v4(); + let current_uuid = Uuid::new_v4(); + + let old_index = dummy_index_metadata(&dataset, field_id, old_uuid, [0_u32, 1]); + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![old_index.clone()], + removed_indices: vec![], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + MockClock::set_system_time(TimeDelta::try_minutes(1).unwrap().to_std().unwrap()); + let current_index = dummy_index_metadata(&dataset, field_id, current_uuid, [0_u32, 1]); + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![current_index], + removed_indices: vec![old_index], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + // Model index artifacts whose storage timestamp is newer than the retained + // manifest. UUID verification must not be hidden by the manifest cutoff. + MockClock::set_system_time(TimeDelta::try_minutes(2).unwrap().to_std().unwrap()); + write_dummy_index_artifact(&dataset, old_uuid) + .await + .unwrap(); + write_dummy_index_artifact(&dataset, current_uuid) + .await + .unwrap(); + + let short_retention = TimeDelta::try_seconds(30).unwrap(); + let removed = fixture + .run_cleanup(utc_now() - short_retention) + .await + .unwrap(); + assert_eq!(removed.old_versions, 3); + assert_eq!(removed.index_files_removed, 2); + + let old_index_file = dataset + .indices_dir() + .join(old_uuid.to_string()) + .join("index.idx"); + let current_index_file = dataset + .indices_dir() + .join(current_uuid.to_string()) + .join("index.idx"); + assert!( + !dataset + .object_store + .as_ref() + .exists(&old_index_file) + .await + .unwrap() + ); + assert!( + dataset + .object_store + .as_ref() + .exists(¤t_index_file) + .await + .unwrap() + ); + } + #[tokio::test] async fn cleanup_old_uncommitted_index_artifacts() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2798,6 +3418,8 @@ mod tests { assert_eq!(removed.old_versions, 0); assert_eq!(removed.index_files_removed, 4); + assert!(!fixture.local_index_dir(staging_uuid).exists()); + assert!(!fixture.local_index_dir(built_segment_uuid).exists()); assert!( !dataset .object_store @@ -2949,6 +3571,26 @@ mod tests { assert_eq!(after_count.num_manifest_files, 1); } + #[tokio::test] + async fn cleanup_rejects_retain_zero_versions() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + + let error = CleanupPolicyBuilder::default() + .retain_n_versions(&fixture.open().await.unwrap(), 0) + .await + .err() + .expect("retaining zero versions should return an error"); + + assert!(matches!(&error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("retain_versions must be greater than 0, got 0"), + "unexpected error: {error}" + ); + } + #[tokio::test] async fn cleanup_and_retain_3_recent_versions() { let fixture = MockDatasetFixture::try_new().unwrap(); @@ -2981,16 +3623,58 @@ mod tests { assert_eq!(after_count.num_data_files, 3); assert_eq!(after_count.num_manifest_files, 3); + assert_eq!( + fixture + .open() + .await + .unwrap() + .version_refs() + .await + .unwrap() + .iter() + .map(|version| version.version) + .collect::>(), + vec![3, 4, 5] + ); + } + + #[tokio::test] + async fn cleanup_specific_versions_only() { + let fixture = MockDatasetFixture::try_new().unwrap(); + fixture.create_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + fixture.overwrite_some_data().await.unwrap(); + + let before_count = fixture.count_files().await.unwrap(); + assert_eq!(before_count.num_manifest_files, 3); + + let policy = CleanupPolicyBuilder::default() + .versions(vec![2]) + .unwrap() + .build(); + let removed = fixture.run_cleanup_with_policy(policy).await.unwrap(); + + assert_eq!(removed.old_versions, 1); + + let versions = fixture + .open() + .await + .unwrap() + .version_refs() + .await + .unwrap() + .iter() + .map(|version| version.version) + .collect::>(); + assert_eq!(versions, vec![1, 3]); } #[tokio::test] async fn cleanup_before_ts_and_retain_n_recent_versions() { let fixture = MockDatasetFixture::try_new().unwrap(); fixture.create_some_data().await.unwrap(); - let mut time = 1i64; - for _ in 0..4 { + for time in (1i64..).take(4) { MockClock::set_system_time(TimeDelta::try_days(time).unwrap().to_std().unwrap()); - time += 1i64; fixture.overwrite_some_data().await.unwrap(); } @@ -4305,7 +4989,7 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_cleanup_with_rate_limit() { // Create multiple versions with data files that will be deleted. let fixture = MockDatasetFixture::try_new().unwrap(); @@ -4324,7 +5008,7 @@ mod tests { .unwrap() .build(); - let start = std::time::Instant::now(); + let start = tokio::time::Instant::now(); let db = fixture.open().await.unwrap(); let stats = cleanup_old_versions(&db, policy).await.unwrap(); let elapsed = start.elapsed(); @@ -4342,4 +5026,206 @@ mod tests { elapsed ); } + + /// Cleanup retires the store record of every manifest it removes, one + /// whose object was already gone included, so store-backed history + /// matches what is on disk. + #[tokio::test] + async fn test_cleanup_forgets_removed_versions_in_the_external_store() { + use crate::dataset::{InsertBuilder, WriteDestination}; + use lance_table::io::commit::external_manifest::{ + ExternalManifestCommitHandler, ExternalManifestStore, + }; + use lance_table::io::commit::{CommitHandler, ManifestLocation, ManifestNamingScheme}; + + /// `(path, size, identity)` per version. + #[derive(Debug, Default)] + struct IdentifiedStore { + rows: Mutex>, + next_identity: std::sync::atomic::AtomicU64, + } + + #[async_trait::async_trait] + impl ExternalManifestStore for IdentifiedStore { + async fn get(&self, _base_uri: &str, version: u64) -> Result { + self.rows + .lock() + .unwrap() + .get(&version) + .map(|row| row.0.clone()) + .ok_or_else(|| Error::not_found(format!("@{version}"))) + } + + async fn get_manifest_location( + &self, + _base_uri: &str, + version: u64, + ) -> Result { + let row = self + .rows + .lock() + .unwrap() + .get(&version) + .cloned() + .ok_or_else(|| Error::not_found(format!("@{version}")))?; + Ok(ManifestLocation { + version, + path: Path::parse(&row.0).unwrap(), + size: Some(row.1), + naming_scheme: ManifestNamingScheme::V2, + e_tag: None, + identity: Some(row.2), + }) + } + + async fn get_latest_version(&self, _base_uri: &str) -> Result> { + Ok(self + .rows + .lock() + .unwrap() + .iter() + .max_by_key(|(version, _)| **version) + .map(|(version, row)| (*version, row.0.clone()))) + } + + async fn get_latest_manifest_location( + &self, + base_uri: &str, + ) -> Result> { + match self.get_latest_version(base_uri).await? { + Some((version, _)) => self + .get_manifest_location(base_uri, version) + .await + .map(Some), + None => Ok(None), + } + } + + async fn put_if_not_exists( + &self, + _base_uri: &str, + version: u64, + path: &str, + size: u64, + _e_tag: Option, + ) -> Result<()> { + let identity = format!( + "identity-{}", + self.next_identity + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + ); + let mut rows = self.rows.lock().unwrap(); + if rows.contains_key(&version) { + return Err(Error::commit_conflict_source(version, "exists".into())); + } + rows.insert(version, (path.to_string(), size, identity)); + Ok(()) + } + + async fn put_if_exists( + &self, + _base_uri: &str, + version: u64, + path: &str, + size: u64, + _e_tag: Option, + ) -> Result<()> { + let mut rows = self.rows.lock().unwrap(); + let row = rows + .get_mut(&version) + .ok_or_else(|| Error::not_found(format!("@{version}")))?; + row.0 = path.to_string(); + row.1 = size; + Ok(()) + } + + fn supports_predecessor_condition(&self) -> bool { + true + } + + async fn get_identity(&self, _base_uri: &str, version: u64) -> Result> { + Ok(self + .rows + .lock() + .unwrap() + .get(&version) + .map(|row| row.2.clone())) + } + + async fn list_versions( + &self, + base_uri: &str, + since: Option, + ) -> Result>> { + let versions: Vec = self.rows.lock().unwrap().keys().copied().collect(); + let mut locations = Vec::new(); + for version in versions { + if since.is_none_or(|since| version > since) { + locations.push(self.get_manifest_location(base_uri, version).await?); + } + } + Ok(Some(locations)) + } + + async fn forget_version( + &self, + _base_uri: &str, + version: u64, + identity: &str, + ) -> Result<()> { + let mut rows = self.rows.lock().unwrap(); + if rows.get(&version).is_some_and(|row| row.2 == identity) { + rows.remove(&version); + } + Ok(()) + } + } + + let store = Arc::new(IdentifiedStore::default()); + let handler: Arc = Arc::new(ExternalManifestCommitHandler { + external_manifest_store: store.clone(), + }); + let uri = TempStrDir::default(); + let batch = || arrow_array::record_batch!(("i", Int32, [1, 2, 3])).unwrap(); + let mut dataset = InsertBuilder::new(uri.as_str()) + .with_params(&WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }) + .execute(vec![batch()]) + .await + .unwrap(); + for _ in 0..2 { + dataset = InsertBuilder::new(WriteDestination::Dataset(Arc::new(dataset))) + .with_params(&WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }) + .execute(vec![batch()]) + .await + .unwrap(); + } + assert_eq!(dataset.count_versions().await.unwrap(), 3); + + // Version 1's object is already gone, as after a cleanup that stopped + // before retiring records. + let v1 = Path::parse(store.get("", 1).await.unwrap()).unwrap(); + dataset.object_store.delete(&v1).await.unwrap(); + + cleanup_old_versions( + &dataset, + CleanupPolicyBuilder::default() + .before_timestamp(chrono::Utc::now()) + .build(), + ) + .await + .unwrap(); + + let mut remaining: Vec = store.rows.lock().unwrap().keys().copied().collect(); + remaining.sort(); + assert_eq!(remaining, vec![3]); + assert_eq!(dataset.count_versions().await.unwrap(), 1); + assert_eq!(dataset.versions().await.unwrap().len(), 1); + } } diff --git a/rust/lance/src/dataset/data_file.rs b/rust/lance/src/dataset/data_file.rs new file mode 100644 index 00000000000..c9939cdd6b0 --- /dev/null +++ b/rust/lance/src/dataset/data_file.rs @@ -0,0 +1,476 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Stateless writing and concatenation of complete encoded data-file parts. + +use std::{collections::HashSet, num::NonZeroU64, ops::Range, sync::Arc}; + +use arrow_array::RecordBatch; +use futures::{Stream, StreamExt}; +use lance_core::{Error, Result, datatypes::Schema}; +use lance_file::{ + concat::{ + BlobTargetId, EncodedFileInput, FileConcatOptions, FileConcatReason, FileConcatResult, + FileConcatTarget, concat_data_file_parts as concat_parts, + }, + version::ConcreteFileVersion, + versions as file_versions, + writer::{FileWriteSummary, FileWriterOptions}, +}; +use lance_io::traits::Writer; +use lance_table::format::DataFile; +use object_store::path::Path; + +pub use lance_file::concat::DataFilePart; + +use super::{ + Dataset, + fragment::{FileFragment, write::generate_random_filename}, + transaction::DataReplacementGroup, +}; +use crate::{ + blob::prepared_to_logical_blob_schema, + dataset::{ + blob::BlobPreprocessor, + write::{ + ExternalBlobMode, WriteParams, blob_v2_external_base_resolver, + validate_blob_v2_write_schema, + }, + }, +}; + +/// Runtime identity and logical schema of a final concatenated data file. +/// +/// Reuse the same live value for every part write and final concatenation. Lance +/// defines no serialization or recovery contract for this type. The caller must +/// keep every use associated with the same dataset and resolved base; Lance does +/// not validate that association across [`Dataset`] instances. +#[derive(Debug, Clone)] +pub struct DataFileTarget { + file_name: String, + base_id: Option, + schema: Arc, + version: ConcreteFileVersion, + blob_target_id: Option, +} + +impl DataFileTarget { + /// Create a final data-file target with Lance's ordinary random file naming. + /// + /// This only creates a runtime identity; it does not create, reserve, or + /// register an object. The caller owns the target lifetime, part storage, + /// cleanup, and commit state. Prepared Blob v2 schemas are normalized to + /// their caller-visible logical form; the persisted descriptor schema remains + /// an internal writer detail. + /// + /// # Example + /// + /// ``` + /// use std::sync::Arc; + /// use lance::dataset::DataFileTarget; + /// use lance_core::datatypes::Schema; + /// use lance_file::version::ConcreteFileVersion; + /// + /// # fn target(schema: Arc) -> lance_core::Result { + /// DataFileTarget::new( + /// None, + /// schema, + /// ConcreteFileVersion::V2_2, + /// ) + /// # } + /// ``` + pub fn new( + base_id: Option, + schema: Arc, + version: ConcreteFileVersion, + ) -> Result { + if version == ConcreteFileVersion::V1 { + return Err(Error::not_supported( + "data-file part concatenation does not support Lance v1".to_string(), + )); + } + if base_id == Some(0) { + return Err(Error::invalid_input( + "DataFileTarget.base_id must not use reserved ID 0", + )); + } + if schema.fields.is_empty() { + return Err(Error::invalid_input( + "DataFileTarget.schema must contain at least one top-level field", + )); + } + let mut field_ids = HashSet::with_capacity(schema.fields.len()); + for field in &schema.fields { + if !field_ids.insert(field.id) { + return Err(Error::invalid_input(format!( + "DataFileTarget.schema contains duplicate top-level field ID {}", + field.id + ))); + } + } + let schema = Arc::new(prepared_to_logical_blob_schema(schema.as_ref())?); + let has_blob_v2 = schema.fields_pre_order().any(|field| field.is_blob_v2()); + if schema + .fields_pre_order() + .any(|field| field.is_blob() && !field.is_blob_v2()) + { + return Err(Error::not_supported( + "DataFileTarget does not support legacy Blob v1 fields", + )); + } + let file_name = format!("{}.lance", generate_random_filename()); + let blob_target_id = has_blob_v2.then(|| { + let base = base_id + .map(|id| format!("base:{id}")) + .unwrap_or_else(|| "primary".to_string()); + BlobTargetId::new(format!("{base}/{file_name}")) + }); + Ok(Self { + file_name, + base_id, + schema, + version, + blob_target_id, + }) + } + + /// Relative path of the final data file within its selected base. + pub fn file_name(&self) -> &str { + &self.file_name + } + + /// Optional registered dataset base that owns the final data file. + pub fn base_id(&self) -> Option { + self.base_id + } + + /// Caller-visible logical schema encoded by every part. + pub fn schema(&self) -> &Arc { + &self.schema + } + + /// Exact Lance file grammar used by parts and final output. + pub fn version(&self) -> ConcreteFileVersion { + self.version + } + + /// Open one caller-provided part and associate its managed Blob descriptors + /// with this runtime target. + /// + /// The caller must ensure that Blob payloads were written through this target + /// using the same dataset and resolved base that will assemble the part. + pub async fn open_part( + &self, + input: EncodedFileInput, + blob_ids: Option>, + ) -> Result { + DataFilePart::open(input, blob_ids, self.blob_target_id.clone()).await + } + + fn object_path(&self, data_dir: &Path) -> Path { + data_dir.clone().join(self.file_name.as_str()) + } +} + +impl Dataset { + fn validate_data_file_target(&self, target: &DataFileTarget) -> Result<()> { + let dataset_version = self.manifest.data_storage_format.lance_file_format(); + if target.version != dataset_version { + return Err(Error::invalid_input(format!( + "DataFileTarget.version is {}, but dataset version {} uses {}", + target.version, + self.version_id(), + dataset_version + ))); + } + self.data_file_dir_for_base(target.base_id)?; + + if target.schema.metadata != self.schema().metadata { + return Err(Error::invalid_input( + "DataFileTarget.schema metadata differs from the dataset schema metadata", + )); + } + for target_field in &target.schema.fields { + let Some(dataset_field) = self + .schema() + .fields + .iter() + .find(|field| field.id == target_field.id) + else { + return Err(Error::invalid_input(format!( + "DataFileTarget.schema field ID {} is not a top-level dataset field", + target_field.id + ))); + }; + if dataset_field != target_field { + return Err(Error::invalid_input(format!( + "DataFileTarget.schema field ID {} differs from the current dataset field", + target_field.id + ))); + } + } + + Ok(()) + } + + /// Encode one independently persisted part for a future data file. + /// + /// The caller owns `output` and its storage path. Managed Blob payloads are + /// written directly beneath the sidecar directory selected by the final + /// target using IDs from `blob_ids`; every non-empty logical Inline value is + /// spilled to Packed or Dedicated storage so final concatenation never copies + /// Blob payload bytes. + /// Every use of `target` must refer to the same dataset and resolved base; + /// associating a runtime target with that storage context is the caller's + /// responsibility. + /// + /// # Example + /// + /// ``` + /// use arrow_array::RecordBatch; + /// use futures::stream; + /// use lance::{Dataset, dataset::DataFileTarget}; + /// use lance_io::traits::Writer; + /// + /// # async fn write_part( + /// # dataset: &Dataset, + /// # target: &DataFileTarget, + /// # output: Box, + /// # batch: RecordBatch, + /// # ) -> lance_core::Result<()> { + /// dataset + /// .write_data_file_part(target, output, None, stream::iter([Ok(batch)])) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn write_data_file_part( + &self, + target: &DataFileTarget, + output: Box, + blob_ids: Option>, + data: impl Stream> + Send, + ) -> Result { + self.validate_data_file_target(target)?; + validate_blob_v2_write_schema(target.schema.as_ref())?; + let has_blob = target + .schema + .fields_pre_order() + .any(|field| field.is_blob_v2()); + if has_blob && blob_ids.is_none() { + return Err(Error::invalid_input( + "write_data_file_part requires a non-empty Blob ID range for a schema containing Blob v2 fields", + )); + } + + let mut preprocessor = if let Some(blob_ids) = blob_ids { + let data_dir = self.data_file_dir_for_base(target.base_id)?; + let data_file_key = target.file_name.strip_suffix(".lance").ok_or_else(|| { + Error::invalid_input("DataFileTarget.file_name must end in '.lance'") + })?; + let object_store = self.object_store(target.base_id).await?; + let external_base_resolver = blob_v2_external_base_resolver( + Some(self), + &WriteParams::default(), + target.schema.as_ref(), + ) + .await?; + Some( + BlobPreprocessor::new( + object_store.as_ref().clone(), + data_dir, + data_file_key.to_string(), + target.schema.as_ref(), + external_base_resolver, + false, + ExternalBlobMode::Reference, + self.session().store_registry(), + self.store_params().cloned().unwrap_or_default(), + None, + )? + .with_part_blob_ids(blob_ids)?, + ) + } else { + None + }; + + let mut writer = file_versions::create_writer( + target.version, + output, + target.schema.as_ref().clone(), + FileWriterOptions::default(), + )?; + let mut data = Box::pin(data); + let write_result = async { + while let Some(batch) = data.next().await { + let batch = batch?; + if let Some(preprocessor) = preprocessor.as_mut() { + let batch = preprocessor.preprocess_batch(&batch).await?; + writer.write_batch(&batch).await?; + } else { + writer.write_batch(&batch).await?; + } + } + if let Some(preprocessor) = preprocessor.as_mut() { + preprocessor.finish().await?; + } + writer.finish().await + } + .await; + + match write_result { + Ok(summary) => Ok(summary), + Err(error) => { + writer.abort().await; + if let Some(preprocessor) = preprocessor.as_mut() { + preprocessor.abort(); + } + Err(error) + } + } + } + + /// Concatenate validated parts into the Lance-generated final data file. + /// + /// Part order is the final physical row order. The operation copies + /// encoded page buffers and regenerates metadata and the footer; incompatible + /// inputs fail without a decode/re-encode fallback or dataset commit. The + /// caller owns cleanup of all durable part, Blob, and final-file objects. + /// The caller must also assemble the target through the same dataset and + /// resolved base used to write managed Blob payloads. + /// + /// # Example + /// + /// ``` + /// use lance::{Dataset, dataset::{DataFilePart, DataFileTarget}}; + /// + /// # async fn concat( + /// # dataset: &Dataset, + /// # target: &DataFileTarget, + /// # ordered_parts: &[DataFilePart], + /// # ) -> lance_core::Result<()> { + /// let data_file = dataset.concat_data_file_parts(target, ordered_parts).await?; + /// // The caller decides when and how to commit `data_file`. + /// # let _ = data_file; + /// # Ok(()) + /// # } + /// ``` + pub async fn concat_data_file_parts( + &self, + target: &DataFileTarget, + ordered_parts: &[DataFilePart], + ) -> Result { + self.validate_data_file_target(target)?; + if ordered_parts.is_empty() { + return Err(Error::invalid_input( + "concat_data_file_parts requires at least one part", + )); + } + let data_dir = self.data_file_dir_for_base(target.base_id)?; + let output_path = target.object_path(&data_dir); + let object_store = self.object_store(target.base_id).await?; + let mut concat_target = FileConcatTarget::new(target.version, target.schema.clone()); + if let Some(blob_target_id) = target.blob_target_id.clone() { + concat_target = concat_target.with_blob_target_id(blob_target_id); + } + let result = concat_parts( + &concat_target, + ordered_parts, + { + let object_store = object_store.clone(); + let output_path = output_path.clone(); + move || async move { object_store.create(&output_path).await } + }, + FileConcatOptions::default(), + ) + .await; + + let output = match result { + Ok(FileConcatResult::Written(output)) => output, + Ok(FileConcatResult::Reused(_, _)) => { + return Err(Error::internal( + "data-file part concatenation unexpectedly reused an input".to_string(), + )); + } + Ok(FileConcatResult::Unsupported(reason)) => { + let message = format!( + "parts cannot be concatenated into target {:?}: {reason}", + target.file_name + ); + return Err(match reason { + FileConcatReason::VersionMismatch { actual, .. } => { + let (major, minor) = actual.to_standard_footer_numbers(); + Error::version_conflict(message, major, minor) + } + FileConcatReason::SchemaMismatch { .. } => Error::schema_mismatch(message), + FileConcatReason::LegacyVersion + | FileConcatReason::ColumnLayoutMismatch { .. } + | FileConcatReason::ColumnEncodingMismatch { .. } + | FileConcatReason::ColumnBuffers { .. } + | FileConcatReason::ExtraGlobalBuffers { .. } + | FileConcatReason::BlobColumns => Error::not_supported(message), + }); + } + Err(error) => return Err(error), + }; + let (fields, column_indices) = + file_versions::data_file_columns(target.version, target.schema.as_ref()); + Ok(DataFile::new( + target.file_name.clone(), + fields, + column_indices, + target.version, + NonZeroU64::new(output.size_bytes), + target.base_id, + )) + } +} + +impl FileFragment { + /// Write parts as a complete replacement for existing top-level columns. + /// + /// The target schema must name current top-level fields, and the sum of + /// part footer row counts must equal this fragment's physical row count. + /// The returned group is uncommitted; the caller retains snapshot fencing. + /// + /// # Example + /// + /// ``` + /// use lance::dataset::{DataFilePart, DataFileTarget}; + /// use lance::dataset::fragment::FileFragment; + /// + /// # async fn replace( + /// # fragment: &FileFragment, + /// # target: &DataFileTarget, + /// # ordered_parts: &[DataFilePart], + /// # ) -> lance_core::Result<()> { + /// let replacement = fragment.write_columns_from_parts(target, ordered_parts).await?; + /// // The caller includes `replacement` in its fenced transaction. + /// # let _ = replacement; + /// # Ok(()) + /// # } + /// ``` + pub async fn write_columns_from_parts( + &self, + target: &DataFileTarget, + ordered_parts: &[DataFilePart], + ) -> Result { + let expected_rows = self.physical_rows().await? as u64; + let actual_rows = ordered_parts.iter().try_fold(0u64, |total, part| { + total + .checked_add(part.num_rows()) + .ok_or_else(|| Error::invalid_input("part physical row count overflows u64")) + })?; + if actual_rows != expected_rows { + return Err(Error::invalid_input(format!( + "parts contain {actual_rows} physical rows, but fragment {} contains {expected_rows}", + self.id() + ))); + } + let data_file = self + .dataset() + .concat_data_file_parts(target, ordered_parts) + .await?; + Ok(DataReplacementGroup(self.id() as u64, data_file)) + } +} diff --git a/rust/lance/src/dataset/delta.rs b/rust/lance/src/dataset/delta.rs index 96c7364223c..b9e12da2482 100644 --- a/rust/lance/src/dataset/delta.rs +++ b/rust/lance/src/dataset/delta.rs @@ -4,15 +4,61 @@ use super::transaction::Transaction; use crate::Dataset; use crate::Result; -use crate::dataset::scanner::DatasetRecordBatchStream; +use crate::dataset::fragment::FileFragment; +use crate::dataset::rowids::load_row_id_sequence; +use crate::dataset::scanner::{ + BATCH_SIZE_FALLBACK, DatasetRecordBatchStream, get_default_batch_size, +}; +use arrow_array::{ArrayRef, RecordBatch, UInt64Array}; +use arrow_schema::Schema as ArrowSchema; +use arrow_schema::SortOptions; use chrono::{DateTime, Utc}; +use datafusion::common::NullEquality; +use datafusion::error::DataFusionError; +use datafusion::logical_expr::JoinType; +use datafusion::physical_expr::{LexOrdering, PhysicalSortExpr}; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::joins::SortMergeJoinExec; +use datafusion::physical_plan::sorts::sort::SortExec; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion_physical_expr::expressions::Column; +use futures::Stream; use futures::stream::{self, StreamExt, TryStreamExt}; use lance_core::Error; use lance_core::ROW_CREATED_AT_VERSION; use lance_core::ROW_ID; +use lance_core::ROW_ID_FIELD; use lance_core::ROW_LAST_UPDATED_AT_VERSION; use lance_core::WILDCARD; +use lance_core::utils::deletion::DeletionVector; use lance_core::utils::tokio::get_num_compute_intensive_cpus; +use lance_datafusion::exec::{LanceExecutionOptions, OneShotExec, execute_plan}; +use lance_table::format::Fragment; +use lance_table::rowids::RowIdSequence; +use lance_table::rowids::segment::U64Segment; +use std::collections::HashMap; +use std::sync::Arc; + +/// Rows per batch of [`DatasetDelta::get_deleted_row_ids`], taken from the +/// scanner so it matches the sibling readers. +fn deleted_row_id_batch_rows() -> usize { + batch_rows(get_default_batch_size()) +} + +/// The largest batch this reader will emit whatever the configuration says: +/// the batch is the unit of buffering, so an unbounded setting would defeat +/// the chunking. +const DELETED_ROW_ID_BATCH_CAP: usize = 64 * 1024; + +/// A configured size of zero would mean no bound at all, so it is refused in +/// favour of the default; an oversized one is clamped to the cap. +fn batch_rows(configured: Option) -> usize { + configured + .filter(|rows| *rows > 0) + .unwrap_or(BATCH_SIZE_FALLBACK) + .min(DELETED_ROW_ID_BATCH_CAP) +} /// Builder for creating a [`DatasetDelta`] to explore changes between dataset versions. /// @@ -270,6 +316,107 @@ impl DatasetDelta { .await } + /// The stable row ids live at the begin version and absent at the end + /// version, as a stream of batches carrying a single [`ROW_ID`] column. + /// Rows in a fragment the range removed outright count as deleted. + /// + /// Requires stable row ids at both endpoints and an ordered range; + /// version 0 is the empty snapshot. Runs in bounded memory: subtracting + /// the still-live ids is a sort-merge anti join that spills past the + /// session memory pool. + /// + /// # Example + /// + /// ``` + /// # use lance::{Dataset, Result}; + /// # use futures::TryStreamExt; + /// # async fn example(dataset: &Dataset, previous_version: u64) -> Result<()> { + /// let delta = dataset + /// .delta() + /// .compared_against_version(previous_version) + /// .build()?; + /// let mut deleted = delta.get_deleted_row_ids().await?; + /// while let Some(batch) = deleted.try_next().await? { + /// // Each batch holds a `_rowid` column of deleted ids. + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn get_deleted_row_ids(&self) -> Result { + let (begin_version, end_version) = self.resolve_range().await?; + if begin_version > end_version { + // A reversed range would report the rows the range added as + // deleted. + return Err(Error::invalid_input(format!( + "begin version {begin_version} is newer than end version {end_version}" + ))); + } + let schema = Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()])); + // Version 0 is the empty snapshot: nothing is live at it, so nothing + // is deleted relative to it. + if begin_version == 0 { + return Ok(DatasetRecordBatchStream::new(Box::pin( + RecordBatchStreamAdapter::new(schema, stream::empty()), + ))); + } + let begin = Arc::new(self.base_dataset.checkout_version(begin_version).await?); + let end = Arc::new(self.base_dataset.checkout_version(end_version).await?); + // Both endpoints: a restore can leave later versions without them. + for endpoint in [&begin, &end] { + if !endpoint.manifest.uses_stable_row_ids() { + return Err(Error::invalid_input(format!( + "deleted row ids require stable row ids, version {} does not use them", + endpoint.manifest.version + ))); + } + } + + let begin_frags = begin.get_fragments(); + let end_frags = end.get_fragments(); + let delta = fragment_delta( + begin_frags.iter().map(|f| f.metadata()), + end_frags.iter().map(|f| f.metadata()), + ); + let out_schema = schema.clone(); + let candidate_end = end.clone(); + let candidate_begin = begin.clone(); + let batches = stream::iter(delta.candidates) + .map(move |(before, after)| { + deleted_batches_in_fragment( + candidate_begin.clone(), + candidate_end.clone(), + before, + after, + schema.clone(), + ) + }) + .buffered(get_num_compute_intensive_cpus()) + .try_flatten() + .map_err(DataFusionError::from) + .try_filter(|batch| std::future::ready(batch.num_rows() > 0)); + let candidates: SendableRecordBatchStream = + Box::pin(RecordBatchStreamAdapter::new(out_schema.clone(), batches)); + if delta.added.is_empty() && delta.changed.is_empty() { + return Ok(DatasetRecordBatchStream::new(candidates)); + } + + // A candidate is only deleted if it is live nowhere at end: a moved + // row lands in a fragment the range created, a restored one where a + // deletion vector shrank. Subtracting those newly live ids is an + // anti join, run the way merge_insert runs its joins: sorted with + // spilling past the memory pool, so a delta of any size is bounded. + let live = live_id_batches(begin.clone(), end, delta.added, delta.changed, out_schema); + let stream = anti_join( + candidates, + live, + LanceExecutionOptions { + use_spilling: true, + ..Default::default() + }, + )?; + Ok(DatasetRecordBatchStream::new(stream)) + } + /// Get inserted rows between the two versions. /// /// This returns rows where `_row_created_at_version` is greater than `begin_version` @@ -450,9 +597,358 @@ impl DatasetDelta { } } +/// A fragment's deletion vector at this version, empty where it has none. +async fn deletion_offsets( + dataset: Arc, + fragment: &Fragment, +) -> Result> { + let fragment = FileFragment::new(dataset, fragment.clone()); + Ok(fragment.get_deletion_vector().await?.unwrap_or_default()) +} + +/// The fragment-level shape of a version range, from metadata alone: a +/// shared fragment whose deletion file is unchanged has neither lost nor +/// regained a row, so nothing else costs any I/O. Each entry carries the +/// fragment metadata it names, so readers never look ids up in a manifest. +struct FragmentDelta { + /// Fragments only the end version holds: every live row is newly live. + added: Vec, + /// Shared fragments whose deletion vector changed, as (begin, end) + /// metadata: only rows a shrink revived are newly live. + changed: Vec<(Fragment, Fragment)>, + /// Begin fragments that can have lost rows, with their end-version + /// metadata where they survive: vanished, or a changed deletion vector. + candidates: Vec<(Fragment, Option)>, +} + +fn fragment_delta<'a>( + begin: impl Iterator, + end: impl Iterator, +) -> FragmentDelta { + let begin_meta: HashMap = begin.map(|f| (f.id, f)).collect(); + let mut added = Vec::new(); + let mut changed = Vec::new(); + let mut end_meta = HashMap::new(); + for fragment in end { + end_meta.insert(fragment.id, fragment); + match begin_meta.get(&fragment.id) { + None => added.push(fragment.clone()), + Some(before) if before.deletion_file != fragment.deletion_file => { + changed.push(((*before).clone(), fragment.clone())); + } + Some(_) => {} + } + } + let candidates = begin_meta + .into_values() + .filter_map(|before| match end_meta.get(&before.id) { + None => Some((before.clone(), None)), + Some(after) if after.deletion_file != before.deletion_file => { + Some((before.clone(), Some((*after).clone()))) + } + Some(_) => None, + }) + .collect(); + FragmentDelta { + added, + changed, + candidates, + } +} + +/// The ids one begin-version fragment lost by the end version, a batch at a +/// time. The offsets are iterated straight off the deletion vectors, so a +/// fragment's deletions are never held whole. +async fn deleted_batches_in_fragment( + begin: Arc, + end: Arc, + before: Fragment, + after: Option, + schema: Arc, +) -> Result> + Send> { + let before_dv = deletion_offsets(begin.clone(), &before).await?; + let emit = if let Some(after) = after { + // The rows it lost are the offsets its deletion vector gained. + let after_dv = deletion_offsets(end, &after).await?; + let before_dv = before_dv.clone(); + let gained: Box + Send> = Box::new( + DeletionVector::clone(&after_dv) + .into_sorted_iter() + .filter(move |offset| !before_dv.contains(*offset)), + ); + Emit::At(gained.peekable()) + } else { + // Gone: every row it still held at the begin version left with it. + Emit::Skipping(before_dv) + }; + + let sequence = load_row_id_sequence(&begin, &before).await?; + Ok(id_batches(SequenceCursor::new(sequence, emit), schema)) +} + +/// Batches of the ids newly live at the end version: every live row of a +/// fragment the range created, and the rows a shrunk deletion vector +/// revived in a shared one. A growth-only change revives nothing and feeds +/// nothing. +fn live_id_batches( + begin: Arc, + end: Arc, + added: Vec, + changed: Vec<(Fragment, Fragment)>, + schema: Arc, +) -> SendableRecordBatchStream { + // Paired with the begin-version metadata for a shared fragment; a + // fragment the range created has none. + let fragments: Vec<(Fragment, Option)> = added + .into_iter() + .map(|f| (f, None)) + .chain( + changed + .into_iter() + .map(|(before, after)| (after, Some(before))), + ) + .collect(); + let batches = stream::iter(fragments) + .map(move |(fragment, before)| { + let (begin, end, schema) = (begin.clone(), end.clone(), schema.clone()); + async move { + let end_dv = deletion_offsets(end.clone(), &fragment).await?; + let sequence = load_row_id_sequence(&end, &fragment).await?; + let emit = match before { + None => Emit::Skipping(end_dv), + Some(before) => { + let begin_dv = deletion_offsets(begin.clone(), &before).await?; + // Lazy: a mass restore revives offsets without ever + // holding them whole. + let revived: Box + Send> = Box::new( + DeletionVector::clone(&begin_dv) + .into_sorted_iter() + .filter(move |offset| !end_dv.contains(*offset)), + ); + Emit::At(revived.peekable()) + } + }; + Ok::<_, Error>(id_batches(SequenceCursor::new(sequence, emit), schema)) + } + }) + .buffered(get_num_compute_intensive_cpus()) + .try_flatten() + .map_err(DataFusionError::from); + let schema = Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()])); + Box::pin(RecordBatchStreamAdapter::new(schema, batches)) +} + +/// One forward traversal of a row id sequence, resumable across batches. +/// The cursor keeps only positions and reads storage through the shared +/// sequence each round, so nothing is cloned, and resuming re-walks no +/// prefix. +struct SequenceCursor { + sequence: Arc, + segment: usize, + /// Length of the current segment, computed once on entry: encoded + /// cardinality is not constant-time. + segment_len: Option, + /// Rows of the current segment already consumed. + consumed: usize, + /// Global offset of the next unconsumed row. + offset: u32, + /// Value resume point for the sorted range-backed encodings; the + /// array-backed ones resume by element through `consumed`. + next_value: u64, + emit: Emit, +} + +/// Which of the traversed ids to emit. +enum Emit { + /// Every offset the deletion vector does not hold. + Skipping(Arc), + /// Exactly these offsets, ascending. + At(std::iter::Peekable + Send>>), +} + +/// A segment's ids from a resume point, without cloning storage or +/// re-walking what came before. +fn segment_ids<'a>( + segment: &'a U64Segment, + consumed: usize, + next_value: u64, +) -> Box + 'a> { + match segment { + U64Segment::Range(range) => Box::new(next_value.max(range.start)..range.end), + U64Segment::RangeWithHoles { range, holes } => { + let start = next_value.max(range.start); + Box::new((start..range.end).filter(move |&v| holes.binary_search(v).is_err())) + } + U64Segment::RangeWithBitmap { range, bitmap } => { + let (base, start) = (range.start, next_value.max(range.start)); + Box::new((start..range.end).filter(move |&v| bitmap.get((v - base) as usize))) + } + U64Segment::SortedArray(array) | U64Segment::Array(array) => { + Box::new((consumed..array.len()).filter_map(move |i| array.get(i))) + } + } +} + +impl SequenceCursor { + fn new(sequence: Arc, emit: Emit) -> Self { + Self { + sequence, + segment: 0, + segment_len: None, + consumed: 0, + offset: 0, + next_value: 0, + emit, + } + } + + /// Append up to `cap` emitted ids to `out`, stopping early when the + /// traversal is exhausted. + fn fill(&mut self, out: &mut Vec, cap: usize) { + while out.len() < cap { + let Some(segment) = self.sequence.segments().get(self.segment) else { + return; + }; + let segment_len = *self.segment_len.get_or_insert_with(|| segment.len()); + let remaining = segment_len - self.consumed; + if remaining == 0 { + self.segment += 1; + self.segment_len = None; + self.consumed = 0; + self.next_value = 0; + continue; + } + // Hop the rest of a segment with no wanted offset in it without + // touching its encoding. + if let Emit::At(wanted) = &mut self.emit { + let Some(target) = wanted.peek().copied() else { + return; + }; + if (target - self.offset) as usize >= remaining { + self.offset += remaining as u32; + self.segment += 1; + self.segment_len = None; + self.consumed = 0; + self.next_value = 0; + continue; + } + } + let mut ids = segment_ids(segment, self.consumed, self.next_value); + match &mut self.emit { + Emit::Skipping(dv) => { + let take = remaining.min(cap - out.len()); + for _ in 0..take { + let Some(id) = ids.next() else { + debug_assert!(false, "sequence shorter than segment lengths"); + return; + }; + if !dv.contains(self.offset) { + out.push(id); + } + self.offset += 1; + self.consumed += 1; + self.next_value = id.saturating_add(1); + } + } + Emit::At(wanted) => { + while out.len() < cap { + let Some(target) = wanted.peek().copied() else { + return; + }; + let skip = (target - self.offset) as usize; + if skip >= segment_len - self.consumed { + break; + } + let Some(id) = ids.nth(skip) else { + debug_assert!(false, "sequence shorter than segment lengths"); + return; + }; + out.push(id); + wanted.next(); + self.consumed += skip + 1; + self.offset = target + 1; + self.next_value = id.saturating_add(1); + } + } + } + } + } +} + +/// The cursor's ids, batched. +fn id_batches( + cursor: SequenceCursor, + schema: Arc, +) -> impl Stream> + Send { + let rows = deleted_row_id_batch_rows(); + stream::try_unfold(cursor, move |mut cursor| { + let schema = schema.clone(); + async move { + let mut ids: Vec = Vec::with_capacity(rows); + cursor.fill(&mut ids, rows); + if ids.is_empty() { + return Ok(None); + } + let batch = + RecordBatch::try_new(schema, vec![Arc::new(UInt64Array::from(ids)) as ArrayRef])?; + Ok(Some((batch, cursor))) + } + }) +} + +/// Candidates minus the live ids, streamed. Sort-merge rather than hash: +/// the sorts spill past the memory pool where a hash build cannot, so a +/// delta of any size runs in bounded memory. +fn anti_join( + candidates: SendableRecordBatchStream, + live: SendableRecordBatchStream, + options: LanceExecutionOptions, +) -> Result { + let sorted = |stream: SendableRecordBatchStream| -> Result> { + let key = Column::new_with_schema(ROW_ID, stream.schema().as_ref())?; + let ordering = LexOrdering::new(vec![PhysicalSortExpr::new( + Arc::new(key), + SortOptions::default(), + )]) + .expect("one sort key"); + Ok(Arc::new(SortExec::new( + ordering, + Arc::new(OneShotExec::new(stream)), + ))) + }; + let candidate_key = Column::new_with_schema(ROW_ID, candidates.schema().as_ref())?; + let live_key = Column::new_with_schema(ROW_ID, live.schema().as_ref())?; + let joined = Arc::new(SortMergeJoinExec::try_new( + sorted(candidates)?, + sorted(live)?, + vec![(Arc::new(candidate_key), Arc::new(live_key))], + None, + JoinType::LeftAnti, + vec![SortOptions::default()], + NullEquality::NullEqualsNothing, + )?); + execute_plan(joined, options) +} + #[cfg(test)] mod tests { + async fn collect_deleted(delta: &super::DatasetDelta) -> Vec { + let mut ids = Vec::new(); + let mut stream = delta.get_deleted_row_ids().await.unwrap(); + while let Some(batch) = stream.try_next().await.unwrap() { + ids.extend( + batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied(), + ); + } + ids.sort_unstable(); + ids + } + use crate::dataset::transaction::Operation; use crate::dataset::{Dataset, WriteParams}; use arrow_array::cast::AsArray; @@ -1371,6 +1867,548 @@ mod tests { } } + /// One deleted row must not drag the fragment's survivors through the + /// join: a growth-only change revives nothing. + #[tokio::test] + async fn test_one_row_deletion_on_a_large_fragment() { + let mut dataset = create_test_dataset(200_000, 1, "value", true).await; + let begin = dataset.manifest.version; + dataset.delete("key = 123456").await.unwrap(); + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + assert_eq!(collect_deleted(&delta).await, vec![123456]); + } + + /// The cursor walks a many-segment sequence once, with either a skip + /// set or a wanted list; a naive per-offset read is its oracle. The + /// sequence covers every segment encoding, asserted below. + #[test] + fn test_sequence_cursor_matches_naive_reads() { + use lance_core::utils::deletion::DeletionVector; + use lance_table::rowids::RowIdSequence; + use lance_table::rowids::segment::U64Segment; + + let mut sequence = RowIdSequence::from(100..200); + sequence.extend(RowIdSequence::try_from_iter([5, 900, 42]).unwrap()); + sequence.extend(RowIdSequence::from(300..350)); + sequence.extend(RowIdSequence::try_from_iter((20_000..26_000).step_by(2)).unwrap()); + sequence.extend( + RowIdSequence::try_from_iter((50_000..53_000).filter(|v| v % 997 != 0)).unwrap(), + ); + // Large enough to span many bounded batches during the resume loops. + sequence.extend(RowIdSequence::try_from_iter((100_000..500_000).step_by(4)).unwrap()); + sequence.extend( + RowIdSequence::try_from_iter((600_000..700_000).filter(|v| v % 9973 != 0)).unwrap(), + ); + let sequence = Arc::new(sequence); + for expected in [ + |s: &U64Segment| matches!(s, U64Segment::Range(_)), + |s: &U64Segment| matches!(s, U64Segment::Array(_) | U64Segment::SortedArray(_)), + |s: &U64Segment| matches!(s, U64Segment::RangeWithBitmap { .. }), + |s: &U64Segment| matches!(s, U64Segment::RangeWithHoles { .. }), + ] { + assert!(sequence.segments().iter().any(expected), "encoding missing"); + } + let len = sequence.len() as u32; + let dv = Arc::new(DeletionVector::from_iter( + (0..len).step_by(97).chain([3u32, 101, 152]), + )); + + let mut skipped: Vec = Vec::new(); + super::SequenceCursor::new(sequence.clone(), super::Emit::Skipping(dv.clone())) + .fill(&mut skipped, usize::MAX); + let naive: Vec = sequence + .iter() + .enumerate() + .filter(|(offset, _)| !dv.contains(*offset as u32)) + .map(|(_, id)| id) + .collect(); + assert_eq!(skipped, naive); + + // A tiny cap forces many resumes, covering the position keeping + // across batches. + let mut resumed: Vec = Vec::new(); + let mut cursor = super::SequenceCursor::new(sequence.clone(), super::Emit::Skipping(dv)); + loop { + let before = resumed.len(); + cursor.fill(&mut resumed, before + 3); + if resumed.len() == before { + break; + } + } + assert_eq!(resumed, naive); + + // The second list leaves whole segments and a consumed tail + // unwanted, covering the hops. + for wanted in [ + vec![ + 0u32, 99, 100, 102, 152, 153, 154, 500, 3152, 3153, 4000, 6149, + ], + vec![5u32, 200, 6000, 6150, 106_000, 206_139], + ] { + let lazy: Box + Send> = Box::new(wanted.clone().into_iter()); + let mut at: Vec = Vec::new(); + let mut cursor = + super::SequenceCursor::new(sequence.clone(), super::Emit::At(lazy.peekable())); + loop { + let before = at.len(); + cursor.fill(&mut at, before + 1); + if at.len() == before { + break; + } + } + let naive: Vec = wanted + .iter() + .filter_map(|offset| sequence.get(*offset as usize)) + .collect(); + assert_eq!(at, naive); + } + } + + /// A range spanning the stable-id migration has a bare begin endpoint + /// and is rejected, naming the offending version. + #[tokio::test] + async fn test_mixed_stable_id_endpoints_are_rejected() { + let dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = write_dataset_temp(&dir, 0, 10, 1, "v1", false, false).await; + dataset.migrate_to_stable_row_ids().await.unwrap(); + + let delta = dataset.delta().compared_against_version(1).build().unwrap(); + let err = delta.get_deleted_row_ids().await.err().unwrap(); + assert!( + err.to_string().contains("stable row ids") && err.to_string().contains("version 1"), + "{err}" + ); + } + + /// One deletion per fragment across many fragments. + #[tokio::test] + async fn test_deletes_across_many_fragments_are_reported() { + let data = lance_datagen::gen_batch() + .col("key", array::step::()) + .into_reader_rows(RowCount::from(160), BatchCount::from(1)); + let params = WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 10, + ..Default::default() + }; + let mut dataset = Dataset::write(data, "memory://", Some(params)) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 16); + let begin = dataset.manifest.version; + + dataset.delete("key % 10 = 3").await.unwrap(); + + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let expected: Vec = (0..16).map(|i| i * 10 + 3).collect(); + assert_eq!(collect_deleted(&delta).await, expected); + } + + /// An append neither removes nor moves a row: the stream is empty. + #[tokio::test] + async fn test_appends_report_no_deleted_row_ids() { + let dir = lance_core::utils::tempfile::TempStrDir::default(); + write_dataset_temp(&dir, 0, 10, 1, "v1", true, false).await; + let ds = write_dataset_temp(&dir, 10, 10, 1, "v2", true, true).await; + let delta = ds.delta().compared_against_version(1).build().unwrap(); + let deleted = collect_deleted(&delta).await; + assert!(deleted.is_empty(), "an append deletes nothing: {deleted:?}"); + } + + /// Deletes on both sides of a merging compaction are all reported. + #[tokio::test] + async fn test_deletes_after_a_merging_compaction_are_reported() { + use crate::dataset::optimize::{CompactionOptions, compact_files}; + + let dir = lance_core::utils::tempfile::TempStrDir::default(); + write_dataset_temp(&dir, 0, 100, 1, "v1", true, false).await; + let mut dataset = write_dataset_temp(&dir, 100, 100, 1, "v2", true, true).await; + let begin = dataset.manifest.version; + + dataset.delete("key = 0 OR key = 100").await.unwrap(); + let options = CompactionOptions { + materialize_deletions_threshold: 0.0, + ..Default::default() + }; + compact_files(&mut dataset, options, None).await.unwrap(); + dataset.delete("key = 150").await.unwrap(); + + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert_eq!(deleted, vec![0, 100, 150], "one id per deleted row"); + } + + /// Repeated partial updates leave fully tombstoned outputs; the result + /// stays exact regardless. + #[tokio::test] + async fn test_repeated_updates_report_no_deleted_row_ids() { + let mut dataset = create_test_dataset(100, 1, "value", true).await; + for round in 0..4 { + dataset = update_where(dataset, "key < 75", &format!("round {round}")).await; + } + let delta = dataset.delta().compared_against_version(1).build().unwrap(); + let deleted = collect_deleted(&delta).await; + assert!(deleted.is_empty(), "updates delete nothing: {deleted:?}"); + } + + /// The anti join must stay correct when its build side exceeds the + /// memory pool and spills. + #[tokio::test] + async fn test_anti_join_is_exact_under_a_tiny_memory_pool() { + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use lance_core::ROW_ID_FIELD; + use lance_datafusion::exec::LanceExecutionOptions; + + let schema = Arc::new(arrow_schema::Schema::new(vec![ROW_ID_FIELD.clone()])); + // ~8 MB of candidates against a 2 MB pool: the sorts must spill, + // and the pool still clears DataFusion's fixed merge reservations. + let candidate_ids: Vec = (0..1_000_000).collect(); + let live_ids: Vec = (0..1_000_000).filter(|id| id % 3 == 0).collect(); + let expected = candidate_ids.len() - live_ids.len(); + let as_stream = |ids: Vec| -> datafusion::physical_plan::SendableRecordBatchStream { + let batches: Vec<_> = ids + .chunks(8192) + .map(|chunk| { + Ok(arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::UInt64Array::from(chunk.to_vec())) as _], + ) + .unwrap()) + }) + .collect(); + Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(batches), + )) + }; + let stream = super::anti_join( + as_stream(candidate_ids), + as_stream(live_ids), + LanceExecutionOptions { + use_spilling: true, + mem_pool_size: Some(2 * 1024 * 1024), + ..Default::default() + }, + ) + .unwrap(); + let batches: Vec<_> = stream.try_collect().await.unwrap(); + let total: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total, expected, "anti join dropped or kept the wrong ids"); + } + + /// Interleaved updates leave every row live; none may read as deleted. + #[tokio::test] + async fn test_interleaved_updates_are_not_reported_as_deleted() { + let dataset = create_test_dataset(100, 2, "value", true).await; + let begin = dataset.manifest.version; + let dataset = update_where(dataset, "key % 2 = 0", "even").await; + let dataset = update_where(dataset, "key % 2 = 1", "odd").await; + + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert!( + deleted.is_empty(), + "every updated row is live at end: {deleted:?}" + ); + } + + /// A restore must not rewind the row-id high-water mark, or the next + /// append reuses old ids. + #[tokio::test] + async fn test_restore_preserves_the_row_id_high_water_mark() { + let dir = lance_core::utils::tempfile::TempStrDir::default(); + write_dataset_temp(&dir, 0, 1, 1, "v1", true, false).await; + // v2: append row A, taking the next stable id. + let a = write_dataset_temp(&dir, 1, 1, 1, "v2", true, true).await; + let begin = a.manifest.version; + // v3: restore v1, dropping row A. + let mut dataset = a.checkout_version(1).await.unwrap(); + dataset.restore().await.unwrap(); + // v4: append row B, which must not reuse A's id. + let dataset = write_dataset_temp(&dir, 2, 1, 1, "v4", true, true).await; + + let delta = dataset + .delta() + .with_begin_version(begin) + .with_end_version(dataset.manifest.version) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert_eq!( + deleted, + vec![1], + "row A's id is gone, not reused: {deleted:?}" + ); + } + + /// A mass delete-and-restore revives every row; the revived offsets are + /// streamed, and the result is exact at scale. + #[tokio::test] + async fn test_mass_restore_reports_no_deleted_row_ids() { + let mut dataset = create_test_dataset(50_000, 1, "value", true).await; + dataset.delete("key >= 0").await.unwrap(); + let begin = dataset.manifest.version; + let mut restored = dataset.checkout_version(1).await.unwrap(); + restored.restore().await.unwrap(); + + let delta = restored + .delta() + .with_begin_version(begin) + .with_end_version(restored.manifest.version) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert!( + deleted.is_empty(), + "every row revived: {} ids", + deleted.len() + ); + } + + /// A restore drops the deletion vector an update left behind, so the + /// updated row is live at both endpoints in a fragment both hold. + #[tokio::test] + async fn test_restored_updated_rows_are_not_reported_as_deleted() { + let dataset = create_test_dataset(100, 2, "value", true).await; + let updated = update_where(dataset, "key = 0", "changed").await; + + let mut restored = updated.checkout_version(1).await.unwrap(); + restored.restore().await.unwrap(); + let delta = restored + .delta() + .with_begin_version(2) + .with_end_version(3) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert!( + deleted.is_empty(), + "a restored row is live at both endpoints: {deleted:?}" + ); + } + + /// A reversed range would report the rows the range added as deleted. + #[tokio::test] + async fn test_deleted_row_ids_rejects_a_reversed_range() { + let dir = lance_core::utils::tempfile::TempStrDir::default(); + write_dataset_temp(&dir, 0, 10, 1, "v1", true, false).await; + let ds = write_dataset_temp(&dir, 10, 10, 1, "v2", true, true).await; + let delta = ds + .delta() + .with_begin_version(2) + .with_end_version(1) + .build() + .unwrap(); + let Err(err) = delta.get_deleted_row_ids().await else { + panic!("a reversed range must be rejected") + }; + assert!(err.to_string().contains("newer than end version"), "{err}"); + } + + /// A window opening before v1 resolves begin to the version-0 sentinel: + /// the empty snapshot, relative to which nothing is deleted. + #[tokio::test] + async fn test_deleted_row_ids_accepts_the_zero_version_sentinel() { + MockClock::set_system_time(std::time::Duration::from_secs(100)); + let mut dataset = create_test_dataset(10, 1, "v1", true).await; + MockClock::set_system_time(std::time::Duration::from_secs(200)); + dataset.delete("key = 0").await.unwrap(); + + let delta = dataset + .delta() + .with_begin_date(chrono::DateTime::::from_timestamp(50, 0).unwrap()) + .with_end_date(chrono::DateTime::::from_timestamp(250, 0).unwrap()) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert!( + deleted.is_empty(), + "nothing is deleted relative to the empty snapshot: {deleted:?}" + ); + } + + /// Unchanged shared fragments appear nowhere; changed ones on both + /// sides; vanished as candidates; added as probed. + #[test] + fn test_fragment_delta_classifies_by_metadata() { + use lance_table::format::{DeletionFile, DeletionFileType, Fragment}; + + let deletion_file = |read_version| DeletionFile { + read_version, + id: 7, + file_type: DeletionFileType::Bitmap, + num_deleted_rows: Some(1), + base_id: None, + }; + let dv_version = |f: &Fragment| f.deletion_file.as_ref().unwrap().read_version; + let unchanged = Fragment::new(1); + let mut changed_before = Fragment::new(2); + changed_before.deletion_file = Some(deletion_file(1)); + let mut changed_after = changed_before.clone(); + changed_after.deletion_file = Some(deletion_file(2)); + let vanished = Fragment::new(3); + let added = Fragment::new(4); + + let begin = [unchanged.clone(), changed_before, vanished]; + let end = [unchanged, changed_after, added]; + let delta = super::fragment_delta(begin.iter(), end.iter()); + + let added: Vec = delta.added.iter().map(|f| f.id).collect(); + assert_eq!(added, vec![4], "only the new fragment is added"); + let changed: Vec<(u64, u64, u64)> = delta + .changed + .iter() + .map(|(b, a)| (b.id, dv_version(b), dv_version(a))) + .collect(); + assert_eq!( + changed, + vec![(2, 1, 2)], + "the changed pair carries each side's metadata" + ); + let mut candidates: Vec<(u64, Option)> = delta + .candidates + .iter() + .map(|(b, a)| (b.id, a.as_ref().map(dv_version))) + .collect(); + candidates.sort_unstable(); + assert_eq!( + candidates, + vec![(2, Some(2)), (3, None)], + "changed and vanished bear candidates, with end metadata where it survives" + ); + } + + /// A configured batch size of zero would leave the stream unbounded. + #[test] + fn test_batch_rows_refuses_a_nonpositive_configuration() { + use super::{BATCH_SIZE_FALLBACK, DELETED_ROW_ID_BATCH_CAP, batch_rows}; + assert_eq!(batch_rows(Some(0)), BATCH_SIZE_FALLBACK); + assert_eq!(batch_rows(None), BATCH_SIZE_FALLBACK); + assert_eq!(batch_rows(Some(64)), 64); + assert_eq!(batch_rows(Some(usize::MAX)), DELETED_ROW_ID_BATCH_CAP); + } + + /// An update rewrites a row under the same stable id, so the old + /// fragment gains a deletion offset for a row that still exists. + #[tokio::test] + async fn test_updated_rows_are_not_reported_as_deleted() { + let dataset = create_test_dataset(100, 2, "value", true).await; + let begin = dataset.manifest.version; + + let dataset = update_where(dataset, "key >= 10 AND key < 20", "changed").await; + + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert!( + deleted.is_empty(), + "an update is not a deletion: {deleted:?}" + ); + } + + /// A fragment's deletions can outnumber one batch. + #[tokio::test] + async fn test_deleted_row_ids_arrive_in_bounded_batches() { + let rows = super::deleted_row_id_batch_rows() * 2 + 100; + let mut dataset = create_test_dataset(rows, 1, "value", true).await; + let begin = dataset.manifest.version; + dataset.delete("true").await.unwrap(); + + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let mut stream = delta.get_deleted_row_ids().await.unwrap(); + let mut sizes = Vec::new(); + while let Some(batch) = stream.try_next().await.unwrap() { + sizes.push(batch.num_rows()); + } + assert_eq!(sizes.iter().sum::(), rows, "{sizes:?}"); + assert!( + sizes + .iter() + .all(|n| *n <= super::deleted_row_id_batch_rows()), + "a batch exceeded the bound: {sizes:?}" + ); + } + + /// Deleted ids are recoverable even though the rows cannot be scanned, + /// and a compaction in the range is not mistaken for deletion. + #[tokio::test] + async fn test_get_deleted_row_ids() { + use crate::dataset::optimize::{CompactionOptions, compact_files}; + + let mut dataset = create_test_dataset(100, 2, "value", true).await; + let begin = dataset.manifest.version; + + dataset.delete("key >= 10 AND key < 20").await.unwrap(); + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let deleted = collect_deleted(&delta).await; + assert_eq!(deleted.len(), 10, "one id per deleted row: {deleted:?}"); + + // Compaction rewrites the surviving rows into new fragments; their + // ids are unchanged, so the deleted set must not grow. + // Materializing the deletions rewrites the fragment under a new id. + let options = CompactionOptions { + materialize_deletions_threshold: 0.0, + ..Default::default() + }; + let metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert!( + metrics.fragments_removed > 0, + "the compaction case is vacuous unless fragments were actually rewritten" + ); + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let after_compaction = collect_deleted(&delta).await; + assert_eq!( + after_compaction, deleted, + "compaction moved live rows; only the deleted ids may be reported" + ); + + // A row deleted after its fragment was compacted away is still + // addressable, so surviving an address lookup does not prove it lives. + dataset.delete("key >= 20 AND key < 30").await.unwrap(); + let delta = dataset + .delta() + .compared_against_version(begin) + .build() + .unwrap(); + let after_delete = collect_deleted(&delta).await; + assert_eq!( + after_delete.len(), + 20, + "deletes on both sides of the compaction must be reported: {after_delete:?}" + ); + } + #[tokio::test] async fn test_get_updated_rows() { // Create initial dataset (version 1) diff --git a/rust/lance/src/dataset/files.rs b/rust/lance/src/dataset/files.rs index 848add7e4a8..cd9c05d46c7 100644 --- a/rust/lance/src/dataset/files.rs +++ b/rust/lance/src/dataset/files.rs @@ -6,7 +6,6 @@ use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; use arrow_array::RecordBatch; use arrow_array::builder::{ @@ -16,8 +15,7 @@ use arrow_array::types::Int32Type; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use either::Either; -use futures::stream::FuturesUnordered; -use futures::{Future, StreamExt, TryStreamExt}; +use futures::{StreamExt, TryStreamExt}; use lance_table::format::IndexMetadata; use lance_table::utils::LanceIteratorExtension; use object_store::path::Path; @@ -26,21 +24,16 @@ use uuid::Uuid; use crate::Dataset; use crate::dataset::files::arrow::{TRACKED_FILES_SCHEMA, TrackedFileBatch}; use crate::dataset::files::file_types::FileType; +use crate::dataset::files::scan::{ManifestScan, scan_manifests}; use crate::dataset::{DATA_DIR, INDICES_DIR, TRANSACTIONS_DIR}; use lance_core::Result; use lance_table::io::deletion::relative_deletion_file_path; -use lance_table::io::manifest::{read_manifest, read_manifest_indexes}; mod arrow; mod file_types; +pub(crate) mod scan; const BATCH_SIZE: usize = 4096; -/// Memory budget for in-flight manifests (estimated in-memory size). -const MANIFEST_MEMORY_BUDGET: usize = 1024 * 1024 * 1024; // 1 GB -/// Estimated ratio of in-memory size to on-disk size for manifests. Found -/// empirically; manifests are protobuf with significant decompression and -/// allocator overhead once parsed. -const MANIFEST_DECOMPRESSION_RATIO: usize = 4; fn remove_prefix(path: &Path, prefix: &Path) -> Path { match path.prefix_match(prefix) { @@ -57,17 +50,29 @@ struct FileRow<'a> { file_type: FileType, } -/// Resolve the base URI a file lives under. Files referenced from a shallow -/// clone carry a `base_id` pointing into `manifest.base_paths`; otherwise they -/// live under this dataset's own `base_uri`. -fn resolve_base_uri<'a>( +struct ResolvedFileBase<'a> { + uri: &'a str, + is_dataset_root: bool, +} + +/// Resolve the base a file lives under. Files referenced from a shallow clone +/// carry a `base_id` pointing into `manifest.base_paths`; otherwise they live +/// under this dataset's own root. +fn resolve_file_base<'a>( manifest: &'a lance_table::format::Manifest, base_id: Option, base_uri: &'a str, -) -> &'a str { +) -> ResolvedFileBase<'a> { base_id - .and_then(|id| manifest.base_paths.get(&id).map(|bp| bp.path.as_str())) - .unwrap_or(base_uri) + .and_then(|id| manifest.base_paths.get(&id)) + .map(|base_path| ResolvedFileBase { + uri: base_path.path.as_str(), + is_dataset_root: base_path.is_dataset_root, + }) + .unwrap_or(ResolvedFileBase { + uri: base_uri, + is_dataset_root: true, + }) } fn manifest_file_rows<'a>( @@ -98,7 +103,9 @@ fn manifest_file_rows<'a>( }; for fragment in manifest.fragments.iter() { - files += fragment.files.len(); + // Precount with the same accessor as the iterator below, or `exact_size` + // drifts. + files += fragment.referenced_lance_files().count(); if fragment.deletion_file.is_some() { files += 1; @@ -106,12 +113,17 @@ fn manifest_file_rows<'a>( } let data_files = manifest.fragments.iter().flat_map(move |fragment| { - fragment.files.iter().map(move |data_file| { - let effective_base_uri = resolve_base_uri(manifest, data_file.base_id, base_uri); + fragment.referenced_lance_files().map(move |data_file| { + let resolved_base = resolve_file_base(manifest, data_file.base_id, base_uri); + let path = if resolved_base.is_dataset_root { + Cow::Owned(format!("{}/{}", DATA_DIR, data_file.path)) + } else { + Cow::Borrowed(data_file.path.as_str()) + }; FileRow { version: manifest.version, - base_uri: Cow::Borrowed(effective_base_uri), - path: Cow::Owned(format!("{}/{}", DATA_DIR, data_file.path)), + base_uri: Cow::Borrowed(resolved_base.uri), + path, file_type: FileType::DataFile, } }) @@ -120,7 +132,7 @@ fn manifest_file_rows<'a>( let deletion_files = manifest.fragments.iter().filter_map(|fragment| { fragment.deletion_file.as_ref().map(|del_file| FileRow { version: manifest.version, - base_uri: Cow::Borrowed(resolve_base_uri(manifest, del_file.base_id, base_uri)), + base_uri: Cow::Borrowed(resolve_file_base(manifest, del_file.base_id, base_uri).uri), path: Cow::Owned(relative_deletion_file_path(fragment.id, del_file)), file_type: FileType::DeletionFile, }) @@ -257,12 +269,6 @@ pub struct TrackedFilesOptions { pub progress: Option>, } -// A `ManifestLocation` is ~100 bytes, so a 50k-slot mpsc channel costs ~5 MB -// in the worst case. That's enough headroom for the lister to run well ahead -// of the reader on datasets with hundreds of thousands of manifests, while -// still bounding memory. -const MAX_BUFFERED_LOCATIONS: usize = 50_000; - impl Dataset { /// Returns one row per (version, file) for every file referenced in any manifest. /// @@ -290,201 +296,72 @@ impl Dataset { &self, options: TrackedFilesOptions, ) -> SendableRecordBatchStream { - use lance_table::io::commit::ManifestLocation; - - let base = self.base.clone(); let uri = self.uri().to_string(); let object_store = self.object_store.clone(); - let commit_handler = self.commit_handler.clone(); + let base = self.base.clone(); // Pipeline architecture: // - // Lister ──► tx_locations ──► Reader ──┬──► tx_manifest ──► Emitter ──► tx (output) - // └──► tx_indexes ──► IndexLister ──► tx (output) + // scan_manifests ──► Emitter ──┬──► tx (output) + // └──► tx_indexes ──► IndexLister ──► tx (output) + // + // The Lister and Reader stages live in `scan::scan_manifests`, shared + // with the keep-set walk. + let ManifestScan { stream, total, .. } = scan_manifests(self, options.min_version); // Output channel: Emitter and IndexLister both send batches here. let (tx, rx) = tokio::sync::mpsc::channel::>(4); - // Location channel: Lister -> Reader. Large buffer since locations are - // small (~100 bytes each) and we want the lister to run ahead. - let (tx_locations, mut rx_locations) = - tokio::sync::mpsc::channel::(MAX_BUFFERED_LOCATIONS); - // Manifest channel: Reader -> Emitter (small buffer for backpressure - // since manifests can be large). - let (tx_manifest, mut rx_manifest) = - tokio::sync::mpsc::channel::<(Arc, String, usize)>(2); - // Index channel: Reader -> IndexLister. + // Index channel: Emitter -> IndexLister. let (tx_indexes, mut rx_indexes) = tokio::sync::mpsc::channel::<(u64, Vec)>(8); - // Tracks estimated in-memory size of in-flight manifests. Reader adds - // before sending; Emitter subtracts after processing. - let inflight_mem = Arc::new(AtomicUsize::new(0)); - let mem_notify = Arc::new(tokio::sync::Notify::new()); - - // Progress: total is set by Lister once listing finishes, read by Emitter. - let total_manifests: Arc> = Arc::new(std::sync::OnceLock::new()); - - // --- Lister task --- - // Lists manifest locations, applies min_version filter, and counts the - // total. Locations are lightweight so we buffer up to MAX_BUFFERED_LOCATIONS. - let tx_err_lister = tx.clone(); - let os_lister = object_store.clone(); - let base_lister = base.clone(); - let total_manifests_lister = total_manifests.clone(); - let min_version = options.min_version; - tokio::spawn(async move { - let result: lance_core::Result<()> = async { - let mut locations = - commit_handler.list_manifest_locations(&base_lister, &os_lister, false); - let mut count = 0usize; - while let Some(loc) = locations.next().await { - let loc = loc?; - if let Some(min_v) = min_version - && loc.version < min_v - { - continue; - } - count += 1; - if tx_locations.send(loc).await.is_err() { - return Ok(()); - } - } - let _ = total_manifests_lister.set(count); - Ok(()) - } - .await; - if let Err(e) = result { - let _ = tx_err_lister - .send(Err(datafusion::error::DataFusionError::from(e))) - .await; - } - }); - - // --- Reader task --- - // Reads manifests with memory-aware parallelism and fans out to - // Emitter (file batches) and IndexLister (index metadata). - let tx_err_reader = tx.clone(); - let os_reader = object_store.clone(); - let base_reader = base.clone(); - let inflight_mem_reader = inflight_mem.clone(); - let mem_notify_reader = mem_notify.clone(); - tokio::spawn(async move { - let result: lance_core::Result<()> = async { - let max_parallelism = os_reader.io_parallelism(); - - type ManifestResult = lance_core::Result<( - Arc, - String, - Vec, - usize, - )>; - let mut in_flight: FuturesUnordered< - std::pin::Pin + Send>>, - > = FuturesUnordered::new(); - let mut locations_exhausted = false; - - loop { - let can_launch = !locations_exhausted - && in_flight.len() < max_parallelism - && (in_flight.is_empty() - || inflight_mem_reader.load(Ordering::Acquire) - < MANIFEST_MEMORY_BUDGET); - - if in_flight.is_empty() && !can_launch { - break; - } - - tokio::select! { - biased; - // Always drain completed reads first. - Some(item) = in_flight.next(), if !in_flight.is_empty() => { - let (manifest, manifest_path, indexes, estimated) = item?; - let version = manifest.version; - if tx_manifest - .send((manifest, manifest_path, estimated)) - .await - .is_err() - { - return Ok(()); - } - if !indexes.is_empty() - && tx_indexes.send((version, indexes)).await.is_err() - { - return Ok(()); - } - } - // Receive next location and start a read. - loc = rx_locations.recv(), if can_launch => { - match loc { - Some(loc) => { - let estimated = - loc.size.unwrap_or(0) as usize - * MANIFEST_DECOMPRESSION_RATIO; - inflight_mem_reader.fetch_add(estimated, Ordering::AcqRel); - - let os = os_reader.clone(); - let base = base_reader.clone(); - in_flight.push(Box::pin(async move { - let manifest = - read_manifest(&os, &loc.path, loc.size).await?; - let indexes = - read_manifest_indexes(&os, &loc, &manifest).await?; - let manifest_path = - remove_prefix(&loc.path, &base).to_string(); - lance_core::Result::Ok(( - Arc::new(manifest), - manifest_path, - indexes, - estimated, - )) - })); - } - None => { - locations_exhausted = true; - } - } - } - // Wake up when Emitter frees memory. - _ = mem_notify_reader.notified(), - if !can_launch && !in_flight.is_empty() => {} - } - } - Ok(()) - } - .await; - - if let Err(e) = result { - let _ = tx_err_reader - .send(Err(datafusion::error::DataFusionError::from(e))) - .await; - } - }); - // --- Emitter task --- - // Converts manifests into file-row batches, releases memory budget, - // and reports progress. + // Converts scanned manifests into file-row batches, forwards index + // metadata to the IndexLister, and reports progress. Dropping each + // `ScannedManifest` releases its share of the scan's memory budget. let tx_emitter = tx.clone(); let uri_emitter = uri.clone(); let progress_cb = options.progress; tokio::spawn(async move { + let mut stream = stream; let mut processed = 0usize; - while let Some((manifest, manifest_path, estimated)) = rx_manifest.recv().await { - let batches = manifest_file_batches(&manifest, &uri_emitter, &manifest_path); + while let Some(scanned) = stream.next().await { + let mut scanned = match scanned { + Ok(scanned) => scanned, + Err(e) => { + let _ = tx_emitter + .send(Err(datafusion::error::DataFusionError::from(e))) + .await; + return; + } + }; + + let batches = + manifest_file_batches(&scanned.manifest, &uri_emitter, &scanned.manifest_path); for batch_result in batches { let df_result = batch_result.map_err(datafusion::error::DataFusionError::from); if tx_emitter.send(df_result).await.is_err() { return; } } - drop(manifest); - inflight_mem.fetch_sub(estimated, Ordering::AcqRel); - mem_notify.notify_one(); + + // Fan out to the index lister only after this manifest's rows + // are out and its memory is released. Doing it earlier would + // block file-row output on a full index channel while still + // holding the manifest's share of the scan's memory budget. + let version = scanned.manifest.version; + let indexes = std::mem::take(&mut scanned.indexes); + drop(scanned); + + if !indexes.is_empty() && tx_indexes.send((version, indexes)).await.is_err() { + return; + } processed += 1; if let Some(ref cb) = progress_cb { cb(TrackedFilesProgress { manifests_processed: processed, - manifests_total: total_manifests.get().copied(), + manifests_total: total.get().copied(), }); } } @@ -1002,12 +879,13 @@ mod tests { ); } - /// Each `DataFile` inside a fragment carries its own `base_id`; the - /// emitted `base_uri` must be looked up per file, not per fragment. + /// Each `DataFile` inside a fragment carries its own `base_id`; both the + /// emitted `base_uri` and relative path must honor that base's layout. #[test] fn test_manifest_file_rows_per_file_base_id() { use lance_core::datatypes::{Field as LanceField, Schema as LanceSchema}; use lance_io::utils::CachedFileSize; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; use lance_table::format::{ BasePath, DataFile, DataStorageFormat, DeletionFile, DeletionFileType, Fragment, Manifest, @@ -1036,6 +914,13 @@ mod tests { // No base_id -> falls back to the dataset base_uri. mk_file("c.lance", None), ], + // An overlay's data file is reported like any other, and resolves + // its own base_id. + overlays: vec![DataOverlayFile { + data_file: mk_file("d.lance", Some(1)), + coverage: OverlayCoverage::Shared(Arc::new(Default::default())), + committed_version: 1, + }], // Deletion files also carry a base_id when they originate from a // shallow clone, and must resolve against base_paths too. deletion_file: Some(DeletionFile { @@ -1058,7 +943,7 @@ mod tests { ); base_paths.insert( 2, - BasePath::new(2, "s3://bucket-b/root".to_string(), None, false), + BasePath::new(2, "s3://bucket-b/root".to_string(), None, true), ); let manifest = Manifest::new( @@ -1076,9 +961,10 @@ mod tests { .map(|r| (r.path.as_ref(), r.base_uri.as_ref())) .collect(); - assert_eq!(by_path.get("data/a.lance"), Some(&"s3://bucket-a/root")); + assert_eq!(by_path.get("a.lance"), Some(&"s3://bucket-a/root")); assert_eq!(by_path.get("data/b.lance"), Some(&"s3://bucket-b/root")); assert_eq!(by_path.get("data/c.lance"), Some(&"memory://main")); + assert_eq!(by_path.get("d.lance"), Some(&"s3://bucket-a/root")); let deletion = rows .iter() diff --git a/rust/lance/src/dataset/files/file_types.rs b/rust/lance/src/dataset/files/file_types.rs index 7c20a81eae5..6734c5169aa 100644 --- a/rust/lance/src/dataset/files/file_types.rs +++ b/rust/lance/src/dataset/files/file_types.rs @@ -31,3 +31,47 @@ impl From for i8 { file_type as Self } } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::{Array, StringArray, cast::AsArray}; + + use crate::dataset::files::arrow::FILE_TYPE_DICT_ARRAY; + + const ALL: [FileType; 5] = [ + FileType::Manifest, + FileType::DataFile, + FileType::DeletionFile, + FileType::TransactionFile, + FileType::IndexFile, + ]; + + /// The discriminants double as dictionary keys for the `tracked_files` + /// output, so reordering either list would silently mislabel every row. + #[test] + fn test_discriminants_index_into_the_dictionary() { + let dict: &StringArray = FILE_TYPE_DICT_ARRAY.as_string(); + assert_eq!( + dict.len(), + ALL.len(), + "every variant needs a dictionary slot" + ); + + for file_type in ALL { + let key = i8::from(file_type); + assert_eq!( + dict.value(key as usize), + file_type.to_string(), + "{file_type:?} has key {key}" + ); + } + } + + #[test] + fn test_discriminants_are_contiguous_from_zero() { + let keys: Vec = ALL.iter().copied().map(i8::from).collect(); + assert_eq!(keys, (0..ALL.len() as i8).collect::>()); + } +} diff --git a/rust/lance/src/dataset/files/scan.rs b/rust/lance/src/dataset/files/scan.rs new file mode 100644 index 00000000000..47d4e762b78 --- /dev/null +++ b/rust/lance/src/dataset/files/scan.rs @@ -0,0 +1,498 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! The shared manifest walk behind [`Dataset::tracked_files`] and +//! [`Dataset::referenced_files`]. +//! +//! Both need the same thing: every present manifest, read with bounded memory +//! and bounded parallelism, together with the index metadata stored alongside +//! it. They differ only in what they build from it, so the walk lives here and +//! each caller materializes its own result. +//! +//! ```text +//! Lister ──► tx_locations ──► Reader ──► tx_manifest ──► caller's stream +//! ``` +//! +//! The reader keeps several manifests in flight but stops launching reads once +//! the estimated in-flight size reaches [`MANIFEST_MEMORY_BUDGET`]. The budget +//! is charged for as long as the consumer holds a [`ScannedManifest`], so +//! dropping each one after use is what keeps the pipeline moving. +//! +//! The bound is on the reader's prefetch, not on what a consumer chooses to +//! retain: one read is always allowed when nothing is in flight, so a consumer +//! that holds every manifest degrades the walk to serial reads rather than +//! stopping it. That escape hatch is what keeps a manifest larger than the whole +//! budget from deadlocking the walk. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use futures::stream::{BoxStream, FuturesUnordered}; +use futures::{Future, StreamExt}; +use lance_core::Result; +use lance_table::format::{IndexMetadata, Manifest}; +use lance_table::io::commit::ManifestLocation; +use lance_table::io::manifest::{read_manifest, read_manifest_indexes}; +use object_store::path::Path; + +use super::remove_prefix; +use crate::Dataset; + +/// Memory budget for in-flight manifests (estimated in-memory size). +const MANIFEST_MEMORY_BUDGET: usize = 1024 * 1024 * 1024; // 1 GB +/// Estimated ratio of in-memory size to on-disk size for manifests. Found +/// empirically; manifests are protobuf with significant decompression and +/// allocator overhead once parsed. +const MANIFEST_DECOMPRESSION_RATIO: usize = 4; + +// A `ManifestLocation` is ~100 bytes, so a 50k-slot mpsc channel costs ~5 MB +// in the worst case. That's enough headroom for the lister to run well ahead +// of the reader on datasets with hundreds of thousands of manifests, while +// still bounding memory. +const MAX_BUFFERED_LOCATIONS: usize = 50_000; + +/// Releases a manifest's share of the memory budget and wakes the reader. +/// +/// Held by [`ScannedManifest`] so the budget is returned when the caller drops +/// it, whether or not the caller remembers to. +struct MemoryPermit { + bytes: usize, + inflight: Arc, + notify: Arc, +} + +impl Drop for MemoryPermit { + fn drop(&mut self) { + self.inflight.fetch_sub(self.bytes, Ordering::AcqRel); + self.notify.notify_one(); + } +} + +/// One manifest produced by [`scan_manifests`], with what was read alongside it. +pub struct ScannedManifest { + pub manifest: Arc, + /// The manifest's own path, relative to the dataset root. + pub manifest_path: String, + /// Index metadata from this manifest's index section. Empty when it has none. + pub indexes: Vec, + // Order matters: dropping the permit last means the budget is returned only + // after `manifest` is freed. + _permit: MemoryPermit, +} + +/// A running manifest walk. +pub struct ManifestScan { + /// Manifests in completion order, which is not version order. + pub stream: BoxStream<'static, Result>, + /// Number of manifests the walk will yield. Set once listing finishes, so a + /// consumer reading it mid-walk may still see `None`. + pub total: Arc>, + /// Estimated in-memory bytes the reader has read and the consumer has not + /// yet dropped. Test-only: production consumers rely on the budget + /// implicitly, by dropping each manifest after use, so keeping this in + /// release builds would be a field nothing reads. + #[cfg(test)] + inflight_bytes: Arc, +} + +#[cfg(test)] +impl ManifestScan { + /// Estimated in-memory bytes currently charged against the budget. + fn inflight_bytes(&self) -> usize { + self.inflight_bytes.load(Ordering::Acquire) + } + + /// The budget counter itself, for assertions that outlive `stream`. + fn inflight_handle(&self) -> Arc { + self.inflight_bytes.clone() + } +} + +/// Walk every present manifest of `dataset`. +/// +/// `min_version`, when set, skips manifests older than that version. Note that +/// this makes the result an incomplete view of what the dataset references, so +/// a caller building a deletion predicate must leave it unset. +pub fn scan_manifests(dataset: &Dataset, min_version: Option) -> ManifestScan { + let base = dataset.base.clone(); + let object_store = dataset.object_store.clone(); + let commit_handler = dataset.commit_handler.clone(); + + let (tx_manifest, rx_manifest) = tokio::sync::mpsc::channel::>(2); + let (tx_locations, rx_locations) = + tokio::sync::mpsc::channel::(MAX_BUFFERED_LOCATIONS); + + let inflight_mem = Arc::new(AtomicUsize::new(0)); + let mem_notify = Arc::new(tokio::sync::Notify::new()); + let total: Arc> = Arc::new(std::sync::OnceLock::new()); + + spawn_lister( + commit_handler, + object_store.clone(), + base.clone(), + min_version, + tx_locations, + total.clone(), + tx_manifest.clone(), + ); + spawn_reader( + object_store, + base, + rx_locations, + tx_manifest, + &inflight_mem, + mem_notify, + ); + + ManifestScan { + stream: tokio_stream::wrappers::ReceiverStream::new(rx_manifest).boxed(), + total, + #[cfg(test)] + inflight_bytes: inflight_mem, + } +} + +/// Lists manifest locations, applies `min_version`, and records the total. +/// +/// Locations are small, so they are buffered generously to let the lister run +/// ahead of the reader. +fn spawn_lister( + commit_handler: Arc, + object_store: Arc, + base: Path, + min_version: Option, + tx_locations: tokio::sync::mpsc::Sender, + total: Arc>, + tx_err: tokio::sync::mpsc::Sender>, +) { + tokio::spawn(async move { + let result: Result<()> = async { + let mut locations = commit_handler.list_manifest_locations(&base, &object_store, false); + let mut count = 0usize; + while let Some(location) = locations.next().await { + let location = location?; + if let Some(min_version) = min_version + && location.version < min_version + { + continue; + } + count += 1; + if tx_locations.send(location).await.is_err() { + // The consumer went away; stop listing. + return Ok(()); + } + } + let _ = total.set(count); + Ok(()) + } + .await; + if let Err(error) = result { + let _ = tx_err.send(Err(error)).await; + } + }); +} + +/// Reads manifests with memory-aware parallelism. +/// +/// Read failures travel as `Err` items in the stream rather than ending the +/// walk, so this task itself is infallible. +fn spawn_reader( + object_store: Arc, + base: Path, + mut rx_locations: tokio::sync::mpsc::Receiver, + tx_manifest: tokio::sync::mpsc::Sender>, + inflight_mem: &Arc, + mem_notify: Arc, +) { + let inflight_mem = inflight_mem.clone(); + tokio::spawn(async move { + let max_parallelism = object_store.io_parallelism(); + type ScanResult = Result; + let mut in_flight: FuturesUnordered< + std::pin::Pin + Send>>, + > = FuturesUnordered::new(); + let mut locations_exhausted = false; + + loop { + // Always allow one read even when over budget, or a single + // manifest larger than the budget would deadlock the walk. + let can_launch = !locations_exhausted + && in_flight.len() < max_parallelism + && (in_flight.is_empty() + || inflight_mem.load(Ordering::Acquire) < MANIFEST_MEMORY_BUDGET); + + if in_flight.is_empty() && !can_launch { + break; + } + + tokio::select! { + biased; + // Always drain completed reads first. + Some(scanned) = in_flight.next(), if !in_flight.is_empty() => { + // The consumer went away; stop reading. + if tx_manifest.send(scanned).await.is_err() { + return; + } + } + location = rx_locations.recv(), if can_launch => { + match location { + Some(location) => { + let estimated = location.size.unwrap_or(0) as usize + * MANIFEST_DECOMPRESSION_RATIO; + inflight_mem.fetch_add(estimated, Ordering::AcqRel); + let permit = MemoryPermit { + bytes: estimated, + inflight: inflight_mem.clone(), + notify: mem_notify.clone(), + }; + + let object_store = object_store.clone(); + let base = base.clone(); + in_flight.push(Box::pin(async move { + let manifest = read_manifest( + &object_store, + &location.path, + location.size, + ) + .await?; + let indexes = read_manifest_indexes( + &object_store, + &location, + &manifest, + ) + .await?; + Ok(ScannedManifest { + manifest: Arc::new(manifest), + manifest_path: remove_prefix(&location.path, &base) + .to_string(), + indexes, + _permit: permit, + }) + })); + } + None => locations_exhausted = true, + } + } + // Wake up when a consumer frees budget by dropping a manifest. + _ = mem_notify.notified(), if !can_launch && !in_flight.is_empty() => {} + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator}; + use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + + fn simple_batch() -> impl arrow_array::RecordBatchReader { + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + RecordBatchIterator::new(vec![Ok(batch)], schema) + } + + async fn dataset_with_three_versions(uri: &str) -> Dataset { + let mut dataset = Dataset::write(simple_batch(), uri, None).await.unwrap(); + dataset.append(simple_batch(), None).await.unwrap(); + dataset.append(simple_batch(), None).await.unwrap(); + dataset + } + + /// The budget must return to zero once the consumer drops every manifest. + /// A leaked permit would leave it charged and eventually stall the reader. + #[tokio::test] + async fn budget_returns_to_zero_after_consuming() { + let dataset = dataset_with_three_versions("memory://scan_budget_zero").await; + let mut scan = scan_manifests(&dataset, None); + + let mut seen = 0usize; + while let Some(scanned) = scan.stream.next().await { + scanned.unwrap(); + seen += 1; + } + + assert_eq!(seen, 3, "expected every present manifest"); + assert_eq!( + scan.inflight_bytes(), + 0, + "dropping every ScannedManifest must return the whole budget" + ); + } + + /// Holding manifests keeps the budget charged, which is the signal the + /// reader throttles on. It does not stop the walk: one read is always + /// allowed when nothing is in flight, so a hoarding consumer gets serial + /// reads rather than a stall. + #[tokio::test] + async fn holding_manifests_keeps_budget_charged() { + let dataset = dataset_with_three_versions("memory://scan_budget_held").await; + let mut scan = scan_manifests(&dataset, None); + + let mut held = Vec::new(); + while let Some(scanned) = scan.stream.next().await { + held.push(scanned.unwrap()); + } + assert!( + scan.inflight_bytes() > 0, + "held manifests must still be charged against the budget" + ); + + drop(held); + assert_eq!( + scan.inflight_bytes(), + 0, + "the budget must come back when the consumer lets go" + ); + } + + /// `min_version` really does skip manifests, which is why a keep-set must + /// leave it unset. + #[tokio::test] + async fn min_version_skips_older_manifests() { + let dataset = dataset_with_three_versions("memory://scan_min_version").await; + + let mut versions = Vec::new(); + let ManifestScan { mut stream, .. } = scan_manifests(&dataset, Some(3)); + while let Some(scanned) = stream.next().await { + versions.push(scanned.unwrap().manifest.version); + } + + assert_eq!(versions, vec![3], "min_version must drop versions 1 and 2"); + } + + /// Dropping the stream early must not leave the reader running: the closed + /// channel is what tells it to stop. Observed through the budget returning + /// to zero, which happens only once every in-flight permit is released. + #[tokio::test] + async fn dropping_the_stream_releases_every_permit() { + let dataset = dataset_with_three_versions("memory://scan_drop_early").await; + let scan = scan_manifests(&dataset, None); + // Keep the budget handle after the stream goes away. + let inflight = scan.inflight_handle(); + let mut stream = scan.stream; + + // Hold the first manifest so the budget is provably charged. Without + // this the assertion below could pass on a walk that never charged + // anything. + let first = stream.next().await.expect("at least one manifest").unwrap(); + assert!( + inflight.load(Ordering::Acquire) > 0, + "holding a manifest must charge the budget" + ); + + // Drop the stream while the walk may still have reads in flight, then + // release our own manifest. + drop(stream); + drop(first); + + // The reader unwinds asynchronously, so poll rather than assume it has + // already observed the closed channel. Ten seconds is far longer than + // this needs locally and is only here so a loaded machine reports a + // real failure instead of a flake. + let mut released = false; + for _ in 0..1000 { + if inflight.load(Ordering::Acquire) == 0 { + released = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + released, + "dropping the stream must release every in-flight permit; \ + timed out waiting for the reader to unwind" + ); + } + + /// A manifest that cannot be read surfaces as an `Err` item in the stream + /// rather than being skipped. A skipped manifest would make the walk + /// silently incomplete, which for a deletion predicate means authorizing the + /// deletion of files only that manifest still references. The reader keeps + /// going after a failure; it is the consumer that decides whether to stop. + #[tokio::test] + async fn read_failure_surfaces_as_an_error_item() { + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::{ObjectStoreParams, ReadParams}; + use crate::utils::test::FailingProxyStore; + + // A real store, not `memory://`: the failing proxy wraps the store, and + // re-opening with a wrapper changes the registry cache key, so an + // in-memory reopen would land on a fresh empty store and lose the three + // versions this test needs. + let dir = tempfile::tempdir().unwrap(); + let uri = dir.path().to_str().unwrap(); + drop(dataset_with_three_versions(uri).await); + + // Install the proxy at open time but arm it only afterwards: opening + // reads the latest manifest itself, so failing that read would break the + // open rather than the walk under test. + let failing = Arc::new(FailingProxyStore::new()); + let dataset = DatasetBuilder::from_uri(uri) + .with_read_params(ReadParams { + store_options: Some(ObjectStoreParams { + object_store_wrapper: Some(failing.clone()), + ..Default::default() + }), + ..Default::default() + }) + .load() + .await + .unwrap(); + failing.fail_when("get_opts", "_versions", "injected manifest read failure"); + + let mut scan = scan_manifests(&dataset, None); + let mut errors = 0usize; + let mut successes = 0usize; + while let Some(scanned) = scan.stream.next().await { + match scanned { + Ok(_) => successes += 1, + Err(_) => errors += 1, + } + } + + // One Err per manifest, not one for the whole walk: that is what pins + // the reader continuing after a failure. A listing failure would give a + // single Err instead, so this also proves the failure came from the + // reads rather than from listing. + assert_eq!( + successes, 0, + "no manifest read can succeed while every `_versions` read fails" + ); + assert_eq!( + errors, 3, + "each of the three manifests must surface its own read error" + ); + assert_eq!( + scan.inflight_bytes(), + 0, + "a failed read must return its share of the budget" + ); + } + + /// The total is the number of manifests the walk will yield, available once + /// listing finishes. + #[tokio::test] + async fn total_counts_every_yielded_manifest() { + let dataset = dataset_with_three_versions("memory://scan_total").await; + let ManifestScan { + mut stream, total, .. + } = scan_manifests(&dataset, None); + + let mut seen = 0usize; + while let Some(scanned) = stream.next().await { + scanned.unwrap(); + seen += 1; + } + + assert_eq!(total.get().copied(), Some(seen)); + } +} diff --git a/rust/lance/src/dataset/fragment.rs b/rust/lance/src/dataset/fragment.rs index 175b1d7d2de..034f81ec5a6 100644 --- a/rust/lance/src/dataset/fragment.rs +++ b/rust/lance/src/dataset/fragment.rs @@ -17,34 +17,41 @@ use arrow_array::types::UInt64Type; use arrow_array::{ Array, RecordBatch, RecordBatchReader, StructArray, UInt32Array, UInt64Array, new_null_array, }; -use arrow_schema::Schema as ArrowSchema; +use arrow_schema::{DataType, Field as ArrowField, Fields as ArrowFields, Schema as ArrowSchema}; use datafusion::logical_expr::Expr; use datafusion::scalar::ScalarValue; use futures::future::{BoxFuture, try_join_all}; -use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, join, stream}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, join, stream}; +use lance_arrow::json::{convert_json_columns, has_json_fields, is_arrow_json_field}; use lance_arrow::{RecordBatchExt, SchemaExt}; -use lance_core::datatypes::{OnMissing, OnTypeMismatch, SchemaCompareOptions}; +use lance_core::datatypes::{ + BlobHandling, NullabilityComparison, OnMissing, OnTypeMismatch, SchemaCompareOptions, +}; use lance_core::utils::address::RowAddress; use lance_core::utils::deletion::DeletionVector; use lance_core::utils::tokio::get_num_compute_intensive_cpus; -use lance_core::{Error, Result, cache::CacheKey, datatypes::Schema}; +use lance_core::{ + Error, Result, + cache::{CacheKey, CacheKeySchema, KeyBuilder}, + datatypes::{Schema, Schema as LanceSchema}, +}; use lance_core::{ ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID, ROW_ID_FIELD, ROW_LAST_UPDATED_AT_VERSION_FIELD, }; use lance_datafusion::utils::StreamingWriteSource; use lance_encoding::decoder::DecoderPlugins; -use lance_file::previous::reader::{ - FileReader as PreviousFileReader, read_batch as previous_read_batch, -}; use lance_file::reader::{ - CachedFileMetadata, FileMetadataIndex, FileReaderOptions, ProjectedFileReader, ReaderProjection, + CachedFileMetadata, FileMetadataIndex, FileReaderOptions, ProjectedFileReader, }; -use lance_file::version::LanceFileVersion; -use lance_file::{LanceEncodingsIo, determine_file_version}; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions::v1::reader::{FileReader as V1FileReader, read_batch as v1_read_batch}; +use lance_file::{LanceEncodingsIo, determine_file_version, versions as file_versions}; use lance_io::ReadBatchParams; use lance_io::scheduler::{FileScheduler, ScanScheduler, SchedulerConfig}; +use lance_io::stream::RecordBatchStream; use lance_io::utils::CachedFileSize; +use lance_table::format::overlay::TOMBSTONE_FIELD_ID; use lance_table::format::{DataFile, DeletionFile, Fragment}; use lance_table::io::deletion::{deletion_file_path, write_deletion_file}; use lance_table::rowids::RowIdSequence; @@ -52,6 +59,7 @@ use lance_table::utils::stream::{ ReadBatchFutStream, ReadBatchTask, ReadBatchTaskStream, RowIdAndDeletesConfig, wrap_with_row_id_and_delete, }; +use object_store::path::Path; use roaring::RoaringBitmap; use self::write::FragmentCreateBuilder; @@ -61,9 +69,12 @@ use super::rowids::load_row_id_sequence; use super::scanner::Scanner; use super::updater::Updater; -use super::{NewColumnTransform, WriteParams, schema_evolution}; +use super::{NewColumnTransform, WriteParams, schema_evolution, versions}; use crate::dataset::Dataset; use crate::dataset::fragment::session::FragmentSession; +use crate::dataset::overlay::{ + OverlayReadPlanner, merge_overlay_batch, plan_overlays, resolve_overlays, +}; use crate::io::deletion::read_dataset_deletion_file; /// Result of [`FileFragment::update_columns_with_offsets`]: updated fragment metadata, modified field ids, @@ -137,30 +148,13 @@ pub trait GenericFileReader: std::fmt::Debug + Send + Sync { /// Get storage statistics for this file (ignored by v1 reader) fn storage_stats(&self) -> Result>; - // Helper functions to fallback to the legacy implementation while we - // slowly migrate functionality over to the generic reader - // Clone the reader, this is needed because Box doesn't // implement Clone fn clone_box(&self) -> Box; - // Return true if the reader is a v1 reader - fn is_legacy(&self) -> bool; - // Return a reference to the legacy reader, panics if called on a v2 - // file. - fn as_legacy(&self) -> &PreviousFileReader { - self.as_legacy_opt() - .expect("legacy function called on v2 file") - } - // Return a reference to the legacy reader if this is a v1 reader and - // return None otherwise - fn as_legacy_opt(&self) -> Option<&PreviousFileReader>; - // Return a mutable reference to the legacy reader if this is a v1 reader - // and return None otherwise - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader>; } fn ranges_to_tasks( - reader: &PreviousFileReader, + reader: &V1FileReader, ranges: Vec<(i32, Range)>, projection: Arc, ) -> ReadBatchTaskStream { @@ -171,7 +165,7 @@ fn ranges_to_tasks( let reader = reader.clone(); let projection = projection.clone(); let task = tokio::task::spawn(async move { - previous_read_batch( + v1_read_batch( &reader, &ReadBatchParams::Range(range.clone()), &projection, @@ -191,12 +185,12 @@ fn ranges_to_tasks( #[derive(Clone, Debug)] struct V1Reader { - reader: PreviousFileReader, + reader: V1FileReader, projection: Arc, } impl V1Reader { - fn new(reader: PreviousFileReader, projection: Arc) -> Self { + fn new(reader: V1FileReader, projection: Arc) -> Self { Self { reader, projection } } } @@ -309,18 +303,6 @@ impl GenericFileReader for V1Reader { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn is_legacy(&self) -> bool { - true - } - - fn as_legacy_opt(&self) -> Option<&PreviousFileReader> { - Some(&self.reader) - } - - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader> { - Some(&mut self.reader) - } } mod v2_adapter { @@ -364,7 +346,7 @@ mod v2_adapter { projection: Arc, ) -> BoxFuture<'_, Result> { async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -394,7 +376,7 @@ mod v2_adapter { projection: Arc, ) -> BoxFuture<'_, Result> { async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -423,7 +405,7 @@ mod v2_adapter { projection: Arc, ) -> BoxFuture<'_, Result> { async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -455,7 +437,7 @@ mod v2_adapter { ) -> BoxFuture<'_, Result> { let indices = UInt32Array::from(indices.to_vec()); async move { - let projection = ReaderProjection::from_field_ids( + let projection = file_versions::reader_projection_from_field_ids( self.reader.version(), projection.as_ref(), self.field_id_to_column_idx.as_ref(), @@ -524,18 +506,6 @@ mod v2_adapter { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn is_legacy(&self) -> bool { - false - } - - fn as_legacy_opt(&self) -> Option<&PreviousFileReader> { - None - } - - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader> { - None - } } } @@ -634,21 +604,9 @@ impl GenericFileReader for NullReader { fn clone_box(&self) -> Box { Box::new(self.clone()) } - - fn is_legacy(&self) -> bool { - false - } - - fn as_legacy_opt(&self) -> Option<&PreviousFileReader> { - None - } - - fn as_legacy_opt_mut(&mut self) -> Option<&mut PreviousFileReader> { - None - } } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct FragReadConfig { // Add the row id column pub with_row_id: bool, @@ -720,11 +678,75 @@ impl FragReadConfig { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum MetadataMode { +pub(crate) enum MetadataMode { LazyAllowed, Full, } +/// The first path in `fields` that names a sibling twice. Projection picks +/// children by name, so a duplicate makes that choice arbitrary, and the +/// name-set comparison the schema check uses cannot see one at all. +fn duplicate_field_path(fields: &ArrowFields, path: &str) -> Option { + let mut seen = HashSet::new(); + for field in fields { + let qualified = if path.is_empty() { + field.name().clone() + } else { + format!("{path}.{}", field.name()) + }; + if !seen.insert(field.name()) { + return Some(qualified); + } + if let Some(nested) = duplicate_nested_path(field.data_type(), &qualified) { + return Some(nested); + } + } + None +} + +fn duplicate_nested_path(data_type: &DataType, path: &str) -> Option { + match data_type { + DataType::Struct(children) => duplicate_field_path(children, path), + DataType::List(item) + | DataType::LargeList(item) + | DataType::FixedSizeList(item, _) + | DataType::Map(item, _) => { + duplicate_nested_path(item.data_type(), &format!("{path}.item")) + } + _ => None, + } +} + +/// `field` with nullability dropped at every level: the projector rebuilds +/// arrays against its target and panics rather than reports on a constraint, +/// so it gets a shape that cannot fail and the writer objects instead. +fn relax_nullability(field: &ArrowField) -> ArrowField { + let relax = |field: &Arc| Arc::new(relax_nullability(field)); + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct(children.iter().map(relax).collect()), + DataType::List(item) => DataType::List(relax(item)), + DataType::LargeList(item) => DataType::LargeList(relax(item)), + DataType::FixedSizeList(item, width) => DataType::FixedSizeList(relax(item), *width), + // A Map's entries struct and its key stay required -- Arrow rejects a + // map whose entries or keys are nullable -- so only the value relaxes. + DataType::Map(entries, sorted) => match entries.data_type() { + DataType::Struct(kv) if kv.len() == 2 => { + let value = Arc::new(relax_nullability(&kv[1])); + let entries = ArrowField::new( + entries.name(), + DataType::Struct(vec![kv[0].clone(), value].into()), + false, + ) + .with_metadata(entries.metadata().clone()); + DataType::Map(Arc::new(entries), *sorted) + } + _ => field.data_type().clone(), + }, + other => other.clone(), + }; + ArrowField::new(field.name(), data_type, true).with_metadata(field.metadata().clone()) +} + impl FileFragment { /// Creates a new FileFragment. pub fn new(dataset: Arc, metadata: Fragment) -> Self { @@ -777,68 +799,78 @@ impl FileFragment { let file_version = determine_file_version(dataset.object_store.as_ref(), &filepath, None).await?; - if file_version != dataset.manifest.data_storage_format.lance_file_version()? { - return Err(Error::invalid_input(format!( - "File version mismatch. Dataset version: {:?} Fragment version: {:?}", - dataset.manifest.data_storage_format.lance_file_version()?, - file_version - ))); - } + super::versions::create_fragment_from_file( + file_version, + dataset.manifest.data_storage_format.lance_file_format(), + filename, + dataset, + fragment_id, + physical_rows, + ) + .await + } - if file_version == LanceFileVersion::Legacy { - let fragment = Fragment::with_file_legacy( - fragment_id as u64, - filename, - dataset.schema(), - physical_rows, - ); - Ok(fragment) - } else { - // Load the file metadata, confirm the schema is compatible, and - // determine the column offsets - let mut frag = Fragment::new(fragment_id as u64); - let scheduler = ScanScheduler::new( - dataset.object_store.clone(), - SchedulerConfig::max_bandwidth(&dataset.object_store), - ); - let file_scheduler = scheduler - .open_file(&filepath, &CachedFileSize::unknown()) - .await?; - let reader = lance_file::reader::FileReader::try_open( - file_scheduler, - None, - Arc::::default(), - &dataset.metadata_cache.file_metadata_cache(&filepath), - dataset.file_reader_options.clone().unwrap_or_default(), - ) - .await?; - // If the schemas are not compatible we can't calculate field id offsets - reader - .schema() - .check_compatible(dataset.schema(), &SchemaCompareOptions::default())?; - let projection = lance_file::reader::ReaderProjection::from_whole_schema( - dataset.schema(), - reader.metadata().version(), - ); - let physical_rows = reader.metadata().num_rows as usize; - frag.physical_rows = Some(physical_rows); - frag.id = fragment_id as u64; + pub(crate) async fn create_from_v1_file( + filename: &str, + dataset: &Dataset, + fragment_id: usize, + physical_rows: Option, + ) -> Result { + Ok(Fragment::with_file_legacy( + fragment_id as u64, + filename, + dataset.schema(), + physical_rows, + )) + } - let column_indices = projection - .column_indices - .into_iter() - .map(|c| c as i32) - .collect(); + pub(crate) async fn create_from_current_file( + filename: &str, + dataset: &Dataset, + fragment_id: usize, + ) -> Result { + let filepath = dataset.data_dir().join(filename); + // Load the file metadata, confirm the schema is compatible, and + // determine the column offsets + let mut frag = Fragment::new(fragment_id as u64); + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + let file_scheduler = scheduler + .open_file(&filepath, &CachedFileSize::unknown()) + .await?; + let reader = lance_file::reader::FileReader::try_open( + file_scheduler, + None, + Arc::::default(), + &dataset.metadata_cache.file_metadata_cache(&filepath), + dataset.file_reader_options.clone().unwrap_or_default(), + ) + .await?; + // If the schemas are not compatible we can't calculate field id offsets + reader + .schema() + .check_compatible(dataset.schema(), &SchemaCompareOptions::default())?; + let projection = file_versions::reader_projection_from_whole_schema( + dataset.schema(), + reader.metadata().version(), + ); + frag.physical_rows = Some(reader.metadata().num_rows as usize); - frag.add_file( - filename, - dataset.schema().field_ids(), - column_indices, - &file_version, - None, - ); - Ok(frag) - } + let column_indices = projection + .column_indices + .into_iter() + .map(|column| column as i32) + .collect(); + frag.add_file( + filename, + dataset.schema().field_ids(), + column_indices, + reader.metadata().version(), + None, + ); + Ok(frag) } /// Returns storage stats as `(field_id, bytes_on_disk)` pairs for this fragment. @@ -948,6 +980,19 @@ impl FileFragment { Arc::new(self.metadata.clone()), )?; + // Plan overlay resolution from coverage metadata (no files opened here); the + // readers are opened lazily on read, pruned to the rows each read touches. + if !self.metadata.overlays.is_empty() { + let planner = plan_overlays(self, projection)?; + if !planner.is_empty() { + reader.overlay = Some(OverlayReadState { + planner: Arc::new(planner), + fragment: Arc::new(self.clone()), + read_config: Arc::new(read_config.clone()), + }); + } + } + if read_config.with_row_id { reader.with_row_id(); } @@ -964,28 +1009,46 @@ impl FileFragment { Ok(reader) } - fn get_field_id_offset(data_file: &DataFile) -> u32 { - data_file.fields.first().copied().unwrap_or(0) as u32 + pub(crate) async fn open_v1_fragment_reader( + &self, + projection: &Schema, + read_config: &FragReadConfig, + ) -> Result { + let open_readers = async { + let mut readers = Vec::new(); + for data_file in &self.metadata.files { + if let Some(reader) = self.open_v1_reader(data_file, Some(projection)).await? { + readers.push(reader); + } + } + Result::Ok(readers) + }; + let deletion_vec_load = self.get_deletion_vector(); + let row_id_load = if self.dataset.manifest.uses_stable_row_ids() { + futures::future::Either::Left( + load_row_id_sequence(&self.dataset, &self.metadata).map_ok(Some), + ) + } else { + futures::future::Either::Right(futures::future::ready(Ok(None))) + }; + let (readers, deletion_vec, row_id_sequence) = + join!(open_readers, deletion_vec_load, row_id_load); + let mut reader = + V1FragmentReader::try_new(readers?, deletion_vec?, row_id_sequence?, self.id())?; + if read_config.with_row_id { + reader.with_row_id(); + } + if read_config.with_row_address { + reader.with_row_address(); + } + Ok(reader) } - fn should_try_indexed_metadata( - data_file: &DataFile, - projection: &ReaderProjection, - file_version: LanceFileVersion, - ) -> bool { - if !ProjectedFileReader::supports_projection(projection, file_version) { - return false; - } - let total_columns = data_file - .column_indices - .iter() - .filter(|column_index| **column_index >= 0) - .count(); - let selected_columns = projection.column_indices.len(); - selected_columns.saturating_mul(4) < total_columns + fn get_field_id_offset(data_file: &DataFile) -> u32 { + data_file.fields.first().copied().unwrap_or(0) as u32 } - async fn open_reader( + pub(super) async fn open_reader( &self, data_file: &DataFile, projection: Option<&Schema>, @@ -1018,140 +1081,172 @@ impl FileFragment { metadata_mode: MetadataMode, ) -> BoxFuture<'a, Result>>> { async move { - let full_schema = self.dataset.schema(); - // The data file may contain fields that are not part of the dataset any longer, remove those - let data_file_schema = Arc::new(data_file.schema(full_schema)); - let projection = projection.unwrap_or(full_schema); - // Also remove any fields that are not part of the user's provided projection - let schema_per_file = - Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?); - - if data_file.is_legacy_file() { - let max_field_id = data_file.fields.iter().max().unwrap(); - if !schema_per_file.fields.is_empty() { - let path = self - .dataset - .data_file_dir(data_file)? - .join(data_file.path.as_str()); - let object_store = self.dataset.object_store_for_data_file(data_file).await?; - let field_id_offset = Self::get_field_id_offset(data_file); - let reader = PreviousFileReader::try_new_with_fragment_id( - &object_store, - &path, - self.schema().clone(), - self.id() as u32, - field_id_offset as i32, - *max_field_id, - Some(&self.dataset.metadata_cache.file_metadata_cache(&path)), - ) + super::versions::open_file_reader( + data_file.file_version()?, + self, + data_file, + projection, + read_config, + metadata_mode, + ) + .await + } + .boxed() + } + + pub(crate) async fn open_v1_file_reader( + &self, + data_file: &DataFile, + projection: Option<&Schema>, + ) -> Result>> { + Ok(self + .open_v1_reader(data_file, projection) + .await? + .map(|reader| Box::new(reader) as Box)) + } + + async fn open_v1_reader( + &self, + data_file: &DataFile, + projection: Option<&Schema>, + ) -> Result> { + let full_schema = self.dataset.schema(); + let data_file_schema = Arc::new(data_file.schema(full_schema)); + let projection = projection.unwrap_or(full_schema); + let schema_per_file = + Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?); + if schema_per_file.fields.is_empty() { + return Ok(None); + } + + let max_field_id = data_file.fields.iter().max().ok_or_else(|| { + Error::invalid_input(format!( + "Legacy data file {} does not contain any fields", + data_file.path + )) + })?; + let path = self + .dataset + .data_file_dir(data_file)? + .join(data_file.path.as_str()); + let object_store = self.dataset.object_store_for_data_file(data_file).await?; + let field_id_offset = Self::get_field_id_offset(data_file); + let reader = V1FileReader::try_new_with_fragment_id( + &object_store, + &path, + self.schema().clone(), + self.id() as u32, + field_id_offset as i32, + *max_field_id, + Some(&self.dataset.metadata_cache.file_metadata_cache(&path)), + ) + .await?; + let initialized_schema = reader.schema().project_by_schema( + schema_per_file.as_ref(), + OnMissing::Error, + OnTypeMismatch::Error, + )?; + Ok(Some(V1Reader::new(reader, Arc::new(initialized_schema)))) + } + + pub(crate) async fn open_current_file_reader( + &self, + data_file: &DataFile, + projection: Option<&Schema>, + read_config: &FragReadConfig, + metadata_mode: MetadataMode, + ) -> Result>> { + let full_schema = self.dataset.schema(); + let data_file_schema = Arc::new(data_file.schema(full_schema)); + let projection = projection.unwrap_or(full_schema); + let schema_per_file = + Arc::new(projection.intersection_ignore_types(data_file_schema.as_ref())?); + if schema_per_file.fields.is_empty() { + return Ok(None); + } + + let path = self + .dataset + .data_file_dir(data_file)? + .join(data_file.path.as_str()); + let (store_scheduler, reader_priority) = if let Some(base_id) = data_file.base_id { + // TODO: reuse the same scan scheduler for non-default bases + let object_store = self.dataset.object_store(Some(base_id)).await?; + let config = SchedulerConfig::max_bandwidth(&object_store); + ( + ScanScheduler::new(object_store, config), + read_config.reader_priority.unwrap_or(0), + ) + } else if let Some(scan_scheduler) = read_config.scan_scheduler.as_ref() { + ( + scan_scheduler.clone(), + read_config.reader_priority.unwrap_or(0), + ) + } else { + ( + ScanScheduler::new( + self.dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&self.dataset.object_store), + ), + 0, + ) + }; + let file_scheduler = store_scheduler + .open_file_with_priority(&path, reader_priority as u64, &data_file.file_size_bytes) + .await?; + let path = file_scheduler.reader().path().clone(); + let metadata_cache = self.dataset.metadata_cache.file_metadata_cache(&path); + let field_id_to_column_idx = Arc::new(BTreeMap::from_iter( + data_file + .fields + .iter() + .copied() + .zip(data_file.column_indices.iter().copied()) + .filter_map(|(field_id, column_index)| { + (column_index >= 0).then_some((field_id as u32, column_index as u32)) + }), + )); + let file_version = data_file.file_version()?; + let reader_projection = file_versions::reader_projection_from_field_ids( + file_version, + schema_per_file.as_ref(), + field_id_to_column_idx.as_ref(), + )?; + let file_reader_options = read_config + .file_reader_options + .clone() + .or_else(|| self.dataset.file_reader_options.clone()) + .unwrap_or_default(); + let prefer_indexed = metadata_mode == MetadataMode::LazyAllowed + && reader_projection.column_indices.len().saturating_mul(4) + < data_file + .column_indices + .iter() + .filter(|column_index| **column_index >= 0) + .count(); + let known_schema = self + .metadata + .physical_rows + .map(|num_rows| (data_file_schema.clone(), num_rows as u64)); + + let encodings_io = Arc::new( + LanceEncodingsIo::new(file_scheduler.clone()) + .with_read_chunk_size(file_reader_options.read_chunk_size), + ); + let reader = file_versions::open_projected_reader( + file_version, + &reader_projection, + prefer_indexed, + || async { + let metadata_index = self + .get_file_metadata_index(&file_scheduler, known_schema.clone()) .await?; - let initialized_schema = reader.schema().project_by_schema( - schema_per_file.as_ref(), - OnMissing::Error, - OnTypeMismatch::Error, - )?; - let reader = V1Reader::new(reader, Arc::new(initialized_schema)); - let reader: Box = Box::new(reader); - Ok(Some(reader)) - } else { - Ok(None) + if (reader_projection.column_indices.len() as u32).saturating_mul(4) + >= metadata_index.num_columns() + { + return Ok(None); } - } else if schema_per_file.fields.is_empty() { - Ok(None) - } else { - let path = self - .dataset - .data_file_dir(data_file)? - .join(data_file.path.as_str()); - let (store_scheduler, reader_priority) = if let Some(base_id) = data_file.base_id { - // TODO: make object stores for non-default bases reuse the same scan scheduler - // currently we always create a new one - let object_store = self.dataset.object_store(Some(base_id)).await?; - let config = SchedulerConfig::max_bandwidth(&object_store); - ( - ScanScheduler::new(object_store, config), - read_config.reader_priority.unwrap_or(0), - ) - } else if let Some(scan_scheduler) = read_config.scan_scheduler.as_ref() { - ( - scan_scheduler.clone(), - read_config.reader_priority.unwrap_or(0), - ) - } else { - ( - ScanScheduler::new( - self.dataset.object_store.clone(), - SchedulerConfig::max_bandwidth(&self.dataset.object_store), - ), - 0, - ) - }; - let file_scheduler = store_scheduler - .open_file_with_priority( - &path, - reader_priority as u64, - &data_file.file_size_bytes, - ) - .await?; - let path = file_scheduler.reader().path().clone(); - let metadata_cache = self.dataset.metadata_cache.file_metadata_cache(&path); - let field_id_to_column_idx = Arc::new(BTreeMap::from_iter( - data_file - .fields - .iter() - .copied() - .zip(data_file.column_indices.iter().copied()) - .filter_map(|(field_id, column_index)| { - if column_index < 0 { - None - } else { - Some((field_id as u32, column_index as u32)) - } - }), - )); - let file_version = LanceFileVersion::try_from_major_minor( - data_file.file_major_version, - data_file.file_minor_version, - )?; - let reader_projection = ReaderProjection::from_field_ids( - file_version, - schema_per_file.as_ref(), - field_id_to_column_idx.as_ref(), - )?; - let file_reader_options = read_config - .file_reader_options - .clone() - .or_else(|| self.dataset.file_reader_options.clone()) - .unwrap_or_default(); - let metadata_index = if metadata_mode == MetadataMode::LazyAllowed - && Self::should_try_indexed_metadata( - data_file, - &reader_projection, - file_version, - ) { - let known_schema = self - .metadata - .physical_rows - .map(|num_rows| (data_file_schema.clone(), num_rows as u64)); - let metadata_index = self - .get_file_metadata_index(&file_scheduler, known_schema) - .await?; - if (reader_projection.column_indices.len() as u32).saturating_mul(4) - < metadata_index.num_columns() - { - Some(metadata_index) - } else { - None - } - } else { - None - }; - - let encodings_io = Arc::new( - LanceEncodingsIo::new(file_scheduler.clone()) - .with_read_chunk_size(file_reader_options.read_chunk_size), - ); - let reader = if let Some(metadata_index) = metadata_index { + Ok(Some( ProjectedFileReader::try_open_with_metadata_index( encodings_io.clone(), path.clone(), @@ -1161,32 +1256,31 @@ impl FileFragment { &metadata_cache, file_reader_options.clone(), ) - .await? - } else { - let file_metadata = self.get_file_metadata(&file_scheduler).await?; - ProjectedFileReader::try_open_with_file_metadata( - encodings_io, - path.clone(), - None, - Arc::::default(), - file_metadata, - &metadata_cache, - file_reader_options, - ) - .await? - }; - let reader = v2_adapter::Reader::new( - Arc::new(reader), - schema_per_file, - field_id_to_column_idx, - reader_priority, - file_scheduler, - ); - let reader: Box = Box::new(reader); - Ok(Some(reader)) - } - } - .boxed() + .await?, + )) + }, + || async { + let file_metadata = self.get_file_metadata(&file_scheduler).await?; + ProjectedFileReader::try_open_with_file_metadata( + encodings_io.clone(), + path.clone(), + None, + Arc::::default(), + file_metadata, + &metadata_cache, + file_reader_options.clone(), + ) + .await + }, + ) + .await?; + Ok(Some(Box::new(v2_adapter::Reader::new( + Arc::new(reader), + schema_per_file, + field_id_to_column_idx, + reader_priority, + file_scheduler, + )))) } async fn open_readers( @@ -1402,6 +1496,11 @@ impl FileFragment { for data_file in &self.metadata.files { let last = -1; for field_id in data_file.fields.iter() { + // A tombstone marks a field superseded by a later data file. + // It is not a field id: it has no ordering and can repeat. + if *field_id == TOMBSTONE_FIELD_ID { + continue; + } if *field_id <= last { return Err(Error::corrupt_file( self.dataset @@ -1428,14 +1527,15 @@ impl FileFragment { } } - if self.metadata.files.iter().any(|f| f.is_legacy_file()) - != self.metadata.files.iter().all(|f| f.is_legacy_file()) - { + if let Err(error) = Fragment::try_infer_version(std::slice::from_ref(&self.metadata)) { + let first_file = self.metadata.files.first().ok_or_else(|| { + Error::internal("mixed file versions reported for an empty fragment") + })?; return Err(Error::corrupt_file( self.dataset - .data_file_dir(&self.metadata.files[0])? - .join(self.metadata.files[0].path.as_str()), - "Fragment contains a mix of v1 and v2 data files".to_string(), + .data_file_dir(first_file)? + .join(first_file.path.as_str()), + format!("Fragment contains mixed file versions: {error}"), )); } @@ -1590,37 +1690,139 @@ impl FileFragment { } } - /// Get the deletion vector for this fragment, using the cache if available. - pub async fn get_deletion_vector(&self) -> Result>> { - let Some(deletion_file) = self.metadata.deletion_file.as_ref() else { - return Ok(None); - }; - - let deletion_vector = - read_dataset_deletion_file(&self.dataset, self.id() as u64, deletion_file).await?; - - Ok(Some(deletion_vector)) - } - - /// Get the file metadata for this fragment, using the cache if available. - async fn get_file_metadata( + /// Read a fragment-local half-open physical row interval without applying deletions. + /// + /// Unlike logical range reads, offsets address the immutable rows stored in + /// the fragment's files. Deleted positions remain present with their stored + /// column values. Callers can stream the batches into + /// [`Dataset::write_data_file_part`](super::Dataset::write_data_file_part) + /// when independently computing a physical-row part. + /// + /// ``` + /// # use lance::{dataset::fragment::FileFragment, Result}; + /// # use lance_core::datatypes::Schema; + /// # async fn read(fragment: &FileFragment, schema: &Schema) -> Result<()> { + /// let batches = fragment.read_physical_slice(0..100, schema, 1024).await?; + /// # let _ = batches; + /// # Ok(()) + /// # } + /// ``` + pub async fn read_physical_slice( &self, - file_scheduler: &FileScheduler, - ) -> Result> { - let path = file_scheduler.reader().path(); - let cache = self.dataset.metadata_cache.file_metadata_cache(path); - - let file_metadata = cache - .get_or_insert_with_key(FileMetadataCacheKey, || async { - let file_metadata: CachedFileMetadata = - lance_file::reader::FileReader::read_all_metadata(file_scheduler).await?; - Ok(file_metadata) - }) + rows: Range, + projection: &Schema, + batch_size: u32, + ) -> Result { + if batch_size == 0 { + return Err(Error::invalid_input( + "read_physical_slice batch_size must be greater than zero", + )); + } + let physical_rows = self.physical_rows().await? as u64; + if rows.start > rows.end || rows.end > physical_rows { + return Err(Error::invalid_input(format!( + "physical slice {}..{} is outside fragment {} with {} physical rows", + rows.start, + rows.end, + self.id(), + physical_rows + ))); + } + let offset = i64::try_from(rows.start).map_err(|_| { + Error::invalid_input(format!( + "physical slice start {} exceeds the supported scan offset range", + rows.start + )) + })?; + let limit = i64::try_from(rows.end - rows.start).map_err(|_| { + Error::invalid_input(format!( + "physical slice length {} exceeds the supported scan limit range", + rows.end - rows.start + )) + })?; + + // Build a read-only view of this exact fragment without its deletion + // file. The normal scanner can then apply overlays and Blob descriptor + // materialization while offsets still address immutable physical rows. + let mut physical_metadata = self.metadata.clone(); + physical_metadata.deletion_file = None; + let mut physical_dataset = self.dataset.as_ref().clone(); + let mut physical_manifest = self.dataset.manifest.as_ref().clone(); + physical_manifest.fragments = Arc::new(vec![physical_metadata.clone()]); + physical_dataset.manifest = Arc::new(physical_manifest); + let physical_dataset = Arc::new(physical_dataset); + let fragment = Self::new(physical_dataset.clone(), physical_metadata); + let mut scanner = fragment.scan(); + let columns = projection + .fields + .iter() + .map(|field| field.name.as_str()) + .collect::>(); + scanner.project(&columns)?; + scanner.batch_size(batch_size as usize); + scanner.limit(Some(limit), Some(offset))?; + + let has_blob_columns = projection.fields_pre_order().any(|field| field.is_blob()); + if has_blob_columns { + scanner.with_row_address(); + } + let stream = scanner.try_into_stream().await?; + if has_blob_columns { + let rewrite_plan = Arc::new(super::optimize::BlobV2BatchRewritePlan::try_new( + projection, + stream.schema().as_ref(), + false, + )?); + Ok(stream + .map(move |batch_result| { + let physical_dataset = physical_dataset.clone(); + let rewrite_plan = rewrite_plan.clone(); + async move { + rewrite_plan + .transform_batch(&physical_dataset, batch_result?) + .await + } + .boxed() + }) + .boxed()) + } else { + Ok(stream + .map(|batch_result| async move { batch_result }.boxed()) + .boxed()) + } + } + + /// Get the deletion vector for this fragment, using the cache if available. + pub async fn get_deletion_vector(&self) -> Result>> { + let Some(deletion_file) = self.metadata.deletion_file.as_ref() else { + return Ok(None); + }; + + let deletion_vector = + read_dataset_deletion_file(&self.dataset, self.id() as u64, deletion_file).await?; + + Ok(Some(deletion_vector)) + } + + /// Get the file metadata for this fragment, using the cache if available. + pub async fn get_file_metadata( + &self, + file_scheduler: &FileScheduler, + ) -> Result> { + let path = file_scheduler.reader().path(); + let cache = self.dataset.metadata_cache.file_metadata_cache(path); + + let file_metadata = cache + .get_or_insert_with_key(FileMetadataCacheKey, || async { + let file_metadata: CachedFileMetadata = + lance_file::reader::FileReader::read_all_metadata(file_scheduler).await?; + Ok(file_metadata) + }) .await?; Ok(file_metadata) } - async fn get_file_metadata_index( + pub async fn get_file_metadata_index( &self, file_scheduler: &FileScheduler, known_schema: Option<(Arc, u64)>, @@ -1678,7 +1880,7 @@ impl FileFragment { if row_offsets.len() > 1 && Self::row_ids_contiguous(row_offsets) { let range = (row_offsets[0] as usize)..(row_offsets[row_offsets.len() - 1] as usize + 1); - reader.legacy_read_range_as_batch(range).await + reader.read_range_as_batch(range).await } else { // FIXME, change this method to streams reader.take_as_batch(row_offsets, None).await @@ -1726,11 +1928,15 @@ impl FileFragment { /// at a time. This can be useful to control memory usage when processing very large /// fields. The batch_size will only be used if the dataset is a v2 dataset. It will /// be ignored for v1 datasets. + /// + /// The `blob_handling` parameter controls the in-memory representation of blob + /// columns read by the updater. If unset, the dataset schema is used unchanged. pub(crate) async fn updater>( &self, columns: Option<&[T]>, schemas: Option<(Schema, Schema)>, batch_size: Option, + blob_handling: Option, ) -> Result { let mut schema = self.dataset.schema().clone(); @@ -1750,6 +1956,14 @@ impl FileFragment { schema = schema.project(&projection)?; } + if let Some(blob_handling) = blob_handling { + schema.fields = schema + .fields + .into_iter() + .map(|field| blob_handling.unload_if_needed(field)) + .collect(); + } + // If there is no projection, we at least need to read the row addresses with_row_addr |= !with_row_id && schema.fields.is_empty(); @@ -1819,7 +2033,7 @@ impl FileFragment { } pub(crate) async fn merge(mut self, join_column: &str, joiner: &HashJoiner) -> Result { - let mut updater = self.updater(Some(&[join_column]), None, None).await?; + let mut updater = self.updater(Some(&[join_column]), None, None, None).await?; while let Some(batch) = updater.next().await? { let batch = joiner @@ -1898,14 +2112,62 @@ impl FileFragment { if !read_columns.iter().any(|n| n.as_str() == ROW_ADDR) { read_columns.push(ROW_ADDR.to_string()); } + let selected_field_ids = read_columns + .iter() + .filter_map(|column| self.schema().field(column)) + .map(|field| field.id) + .collect::>(); + let descriptor_blob_ids = self + .schema() + .project_by_ids(&selected_field_ids, true) + .fields_pre_order() + .filter(|field| field.is_blob_v2()) + .filter_map(|field| u32::try_from(field.id).ok()) + .collect::>(); + let has_blob_v2 = !descriptor_blob_ids.is_empty(); + let blob_handling = has_blob_v2.then(|| { + let materialized_blob_ids = self + .schema() + .fields_pre_order() + .filter(|field| field.is_blob()) + .filter_map(|field| u32::try_from(field.id).ok()) + .filter(|field_id| !descriptor_blob_ids.contains(field_id)) + .collect(); + BlobHandling::SomeBlobsBinary(materialized_blob_ids) + }); let mut updater = self .updater( Some(&read_columns), Some((write_schema.clone(), self.schema().clone())), None, + blob_handling, ) .await?; + if has_blob_v2 { + updater.allow_external_blob_outside_bases(); + } + let external_base_resolver = if has_blob_v2 { + super::write::blob_v2_external_base_resolver( + Some(self.dataset()), + &WriteParams::default(), + &write_schema, + ) + .await? + } else { + None + }; // Hash join: rows matched on the right-hand stream rewrite columns; track physical offsets via `_rowaddr`. + // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) in the right stream + // so they match the physical storage format read from the fragment's left batch. + let right_stream: Box = if right_schema + .fields() + .iter() + .any(|f| is_arrow_json_field(f) || has_json_fields(f)) + { + Box::new(JsonConvertingReader::new(right_stream)) + } else { + right_stream + }; let joiner = Arc::new(HashJoiner::try_new(right_stream, right_on).await?); let mut matched_offsets = RoaringBitmap::new(); let frag_id_u32 = u32::try_from(self.metadata.id).map_err(|_| { @@ -1915,6 +2177,17 @@ impl FileFragment { )) })?; while let Some(batch) = updater.next().await? { + let batch = if has_blob_v2 { + crate::dataset::optimize::transform_blob_v2_batch( + &self.dataset, + self.schema(), + batch.clone(), + true, + ) + .await? + } else { + batch.clone() + }; let index_column = batch[left_on].clone(); let matched = joiner.matched_join_rows(index_column.clone())?; if let Some(addr_col) = batch.column_by_name(ROW_ADDR) { @@ -1930,8 +2203,12 @@ impl FileFragment { } } let updated_batch = joiner - .collect_with_fallback(batch, index_column, self.dataset()) + .collect_with_fallback(&batch, index_column, self.dataset()) .await?; + if let Some(resolver) = external_base_resolver.as_deref() { + super::blob::validate_external_blob_references(resolver, &updated_batch, &matched) + .await?; + } updater.update(updated_batch).await?; } @@ -1977,7 +2254,7 @@ impl FileFragment { read_columns: Option>, batch_size: Option, ) -> Result<(Fragment, Schema)> { - let (fragments, schema, _) = schema_evolution::add_columns_to_fragments( + let (fragments, schema, _, _) = schema_evolution::add_columns_to_fragments( self.dataset.as_ref(), transforms, read_columns, @@ -1989,6 +2266,239 @@ impl FileFragment { Ok((fragments.into_iter().next().unwrap(), schema)) } + fn schema_mismatch(&self, detail: impl std::fmt::Display) -> Error { + Error::invalid_input(format!( + "column data for fragment {} does not match the requested schema: {detail}", + self.id() + )) + } + + /// Remove a staged file that will not be returned. Best effort: it is + /// unreachable either way, and must not mask the error that caused it. + async fn discard_staged_file(&self, path: &Path) { + // Blob v2 spills sidecars into data// beside the file, and + // those are the large ones; leaving them is what makes a routine + // rejection expensive. + if let Some(stem) = path + .filename() + .and_then(|name| name.strip_suffix(".lance")) + .map(|stem| self.dataset.data_dir().join(stem)) + && let Err(delete_error) = self.dataset.object_store.remove_dir_all(stem.clone()).await + { + log::warn!("failed to delete staged blob sidecars '{stem}': {delete_error}"); + } + if let Err(delete_error) = self.dataset.object_store.delete(path).await { + log::warn!("failed to delete staged column file '{path}': {delete_error}"); + } + } + + /// Write new data for columns of this fragment as a standalone data file, + /// without committing it, and return the + /// [`DataReplacementGroup`](super::transaction::DataReplacementGroup) + /// describing it. + /// + /// Unlike [`Self::add_columns`], the staged file answers for a field that + /// already exists, so this recomputes a column rather than appending one. + /// + /// `schema` names the fields being written. Each must be a top-level + /// column the dataset schema already defines, matching its manifest + /// definition; a column is staged whole, so a nested field cannot be + /// staged on its own. To recompute a new column, declare it first with an + /// all-null [`Self::add_columns`], then stage its data. Physical layout + /// comes from the manifest, so staging cannot change a field's storage + /// encoding. Batch columns are matched by name at every level, so struct + /// children may arrive in any order, but a batch whose fields are not + /// exactly the target's, at every level, is rejected. + /// + /// `data` must produce exactly the fragment's physical row count, nulls + /// included: the file is positionally aligned with the fragment and no + /// deletion vector is applied on the way in. Batches are pulled one at a + /// time, so the full column need not be held in memory. + /// + /// Callers should take care to set the read version correctly. If this is + /// not done then multiple replacements to the same field will not be + /// detected as a conflict. + pub async fn write_columns( + &self, + data: impl Stream> + Send, + schema: &Schema, + ) -> Result { + let expected_rows = self.physical_rows().await? as u64; + + // Readers take everything but the field id from the manifest, so a + // staged field reusing an id is decoded as the manifest's version rather + // than rejected. Compare full identity, not just the storage type. + let compare_options = SchemaCompareOptions { + compare_field_ids: true, + ..Default::default() + }; + // Top-level requests match top-level manifest fields only: resolving an + // id from anywhere lets a caller reuse a field at a path the dataset + // never gave it, staging a file covering the borrowed field. Layout then + // comes from the manifest, since the metadata the identity check ignores + // -- packed structs, blob encoding -- decides physical field coverage. + let dataset_schema = self.dataset.schema(); + let mut writer_fields = Vec::with_capacity(schema.fields.len()); + let mut requested = HashSet::with_capacity(schema.fields.len()); + for field in &schema.fields { + // The per-field identity check cannot see the request naming an + // id twice, and the set-based batch comparison downstream would + // match one batch column against both copies. + if !requested.insert(field.id) { + return Err(Error::invalid_input(format!( + "column data for fragment {} names field id {} ('{}') more than once", + self.id(), + field.id, + field.name + ))); + } + if lance_core::is_system_column(&field.name) { + return Err(Error::invalid_input(format!( + "column data for fragment {} names reserved column '{}'", + self.id(), + field.name + ))); + } + let Some(existing) = dataset_schema + .fields + .iter() + .find(|existing| existing.id == field.id) + else { + // The commit path publishes data files, never schema, so a + // field the manifest does not define would commit as a file no + // live field answers for -- and a concurrent schema change + // could never be checked against it. + return Err(Error::invalid_input(format!( + "column data for fragment {} names field id {} ('{}') that the dataset schema \ + does not define; declare the column with add_columns before staging its data", + self.id(), + field.id, + field.name + ))); + }; + // `explain_difference` recurses, covering the whole subtree. + if let Some(difference) = field.explain_difference(existing, &compare_options) { + return Err(Error::invalid_input(format!( + "column data for fragment {} does not match dataset field id {}: {}", + self.id(), + field.id, + difference + ))); + } + writer_fields.push(existing.clone()); + } + let writer_schema = Schema { + fields: writer_fields, + metadata: schema.metadata.clone(), + }; + let batch_schema = ArrowSchema::from(&writer_schema); + let projection_schema = ArrowSchema::new( + batch_schema + .fields() + .iter() + .map(|field| relax_nullability(field)) + .collect::>(), + ); + + let file_version = self + .dataset + .manifest + .data_storage_format + .lance_file_format(); + + if file_version == ConcreteFileVersion::V1 { + // The legacy reader pairs a fragment's files by batch boundary, so a + // staged file chunked to the caller's batches leaves the fragment + // unreadable. Rechunking is the legacy update path's job, not this + // one's. + return Err(Error::not_supported(format!( + "write_columns is not supported for fragment {} in the legacy file format", + self.id() + ))); + } + + // The update writer, not a raw file writer: that boundary carries the + // version's write policies (blob v2 columns arrive logical and must be + // prepared for the encoders) and returns a populated `DataFile`. + // Blob v2 descriptors land under the dataset root, outside any + // registered external base, as on the other update paths. + let has_blob_v2 = writer_schema + .fields_pre_order() + .any(|field| field.is_blob_v2()); + let mut writer = versions::open_update_writer( + file_version, + self.dataset.as_ref(), + &writer_schema, + has_blob_v2, + ) + .await?; + let staged_path = { + let (file_name, _) = writer.data_file_path(); + self.dataset.data_dir().join(file_name) + }; + + // From here every failure -- a stream error, a rejected batch, a write + // or finish error, a row-count mismatch -- owns the same staged + // artifacts: the data file and any Blob sidecars already finalized + // beside it. One exit cleans them all. + let mut data = std::pin::pin!(data); + let staged: Result<_> = async { + while let Some(batch_result) = data.next().await { + let batch = batch_result?; + // Struct encoders consume children positionally, so a batch + // ordered differently from the manifest lands under the wrong + // field ids. Projection fixes that by name, but it downcasts by + // shape, so the whole tree is compared first. Nullability is the + // writer's to enforce, against the data rather than the + // declared schema. + if let Some(duplicate) = duplicate_field_path(batch.schema_ref().fields(), "") { + return Err(self.schema_mismatch(format!("column '{duplicate}' appears twice"))); + } + LanceSchema::try_from(batch.schema_ref().as_ref()) + .and_then(|staged| { + staged.check_compatible( + &writer_schema, + &SchemaCompareOptions { + compare_nullability: NullabilityComparison::Ignore, + ignore_field_order: true, + ..Default::default() + }, + ) + }) + .map_err(|mismatch| self.schema_mismatch(mismatch))?; + let batch = batch + .project_by_schema(&projection_schema) + .map_err(|err| self.schema_mismatch(err))?; + writer.write(std::slice::from_ref(&batch)).await?; + } + let (num_rows, data_file) = writer.finish().await?; + if num_rows as u64 != expected_rows { + return Err(Error::invalid_input(format!( + "column data for fragment {} has {} rows but the fragment has {} physical rows", + self.id(), + num_rows, + expected_rows + ))); + } + Ok(data_file) + } + .await; + + match staged { + Ok(data_file) => Ok(super::transaction::DataReplacementGroup( + self.id() as u64, + data_file, + )), + Err(err) => { + // The writer may still hold the file open (a buffered upload, + // an unflushed local handle); release it before deleting. + drop(writer); + self.discard_staged_file(&staged_path).await; + Err(err) + } + } + } + /// Delete rows from the fragment. /// /// If all rows are deleted, returns `Ok(None)`. Otherwise, returns a new @@ -2166,6 +2676,12 @@ impl CacheKey for FileMetadataCacheKey { fn type_name() -> &'static str { "FileMetadata" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.fragment-file-metadata-key", 1) + } + + fn write_key(&self, _builder: &mut KeyBuilder) {} } #[derive(Debug, Clone)] @@ -2181,6 +2697,12 @@ impl CacheKey for FileMetadataIndexCacheKey { fn type_name() -> &'static str { "FileMetadataIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.fragment-file-metadata-index-key", 1) + } + + fn write_key(&self, _builder: &mut KeyBuilder) {} } impl From for Fragment { @@ -2189,173 +2711,61 @@ impl From for Fragment { } } -/// [`FragmentReader`] is an abstract reader for a [`FileFragment`]. +/// Typed v1-only read operations used by the legacy pushdown path. /// -/// It opens the data files that contains the columns of the projection schema, and -/// reconstruct the RecordBatch from columns read from each data file. -#[derive(Debug)] -pub struct FragmentReader { - /// Readers and schema of each opened data file. - readers: Vec>, - - /// The output schema. The defines the order in which the columns are returned. - output_schema: ArrowSchema, - - /// The deleted row IDs +/// Keeping the previous readers here avoids exposing legacy downcasts through +/// [`GenericFileReader`]. Modern readers never implement or simulate these +/// row-group and page-statistics operations. +#[derive(Clone, Debug)] +pub(crate) struct V1FragmentReader { + readers: Vec, deletion_vec: Option>, - - /// The row id sequence - /// - /// Only populated if the stable row id feature is enabled. row_id_sequence: Option>, - - /// ID of the fragment fragment_id: usize, - - /// True if we should generate a row id for the output with_row_id: bool, - - /// True if we should generate a row address column in output with_row_addr: bool, - - /// True if we should generate a last updated at version column in output - with_row_last_updated_at_version: bool, - - /// True if we should generate a created at version column in output - with_row_created_at_version: bool, - - /// If true, deleted rows will be set to null, which is fast - /// If false, deleted rows will be removed from the batch, requiring a copy make_deletions_null: bool, - - /// The fragment metadata (needed for version columns) - fragment: Arc, - - /// The last_updated_at version sequence (loaded from fragment metadata) - last_updated_at_sequence: Option>, - - /// The created_at version sequence (loaded from fragment metadata) - created_at_sequence: Option>, - - // total number of real rows in the fragment (num_physical_rows - num_deleted_rows) - num_rows: usize, - - // total number of physical rows in the fragment (all rows, ignoring deletions) - num_physical_rows: usize, } -// Custom clone impl needed because it is not easy to clone Box -// -// We currently need FragmentReader to be Clone because the pushdown scan clones it -// to reuse the fragment reader for both "scan with row id" and "scan without row id" -impl Clone for FragmentReader { - fn clone(&self) -> Self { - Self { - readers: self - .readers - .iter() - .map(|reader| reader.clone_box()) - .collect::>(), - output_schema: self.output_schema.clone(), - deletion_vec: self.deletion_vec.clone(), - row_id_sequence: self.row_id_sequence.clone(), - fragment_id: self.fragment_id, - with_row_id: self.with_row_id, - with_row_addr: self.with_row_addr, - with_row_last_updated_at_version: self.with_row_last_updated_at_version, - with_row_created_at_version: self.with_row_created_at_version, - make_deletions_null: self.make_deletions_null, - fragment: self.fragment.clone(), - last_updated_at_sequence: self.last_updated_at_sequence.clone(), - created_at_sequence: self.created_at_sequence.clone(), - num_rows: self.num_rows, - num_physical_rows: self.num_physical_rows, +impl V1FragmentReader { + fn try_new( + readers: Vec, + deletion_vec: Option>, + row_id_sequence: Option>, + fragment_id: usize, + ) -> Result { + let first_reader = readers.first().ok_or_else(|| { + Error::invalid_input("Cannot create a v1 fragment reader without data files") + })?; + let num_batches = first_reader.reader.num_batches(); + if readers + .iter() + .any(|reader| reader.reader.num_batches() != num_batches) + { + return Err(Error::invalid_input( + "Cannot create a v1 fragment reader from data files with different numbers of batches" + .to_string(), + )); } + Ok(Self { + readers, + deletion_vec, + row_id_sequence, + fragment_id, + with_row_id: false, + with_row_addr: false, + make_deletions_null: false, + }) } -} -impl std::fmt::Display for FragmentReader { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "FragmentReader(id={})", self.fragment_id) + pub(crate) fn with_row_id(&mut self) -> &mut Self { + self.with_row_id = true; + self } -} -fn merge_batches(batches: &[RecordBatch]) -> Result { - if batches.is_empty() { - return Err(Error::invalid_input( - "Cannot merge empty batches".to_string(), - )); - } - - let mut merged = batches[0].clone(); - for batch in batches.iter().skip(1) { - merged = merged.merge(batch)?; - } - Ok(merged) -} - -impl FragmentReader { - #[allow(clippy::too_many_arguments)] - fn try_new( - fragment_id: usize, - deletion_vec: Option>, - row_id_sequence: Option>, - readers: Vec>, - output_schema: ArrowSchema, - num_rows: usize, - num_physical_rows: usize, - fragment: Arc, - ) -> Result { - if let Some(legacy_reader) = readers.first().and_then(|reader| reader.as_legacy_opt()) { - let num_batches = legacy_reader.num_batches(); - for reader in readers.iter().skip(1) { - if let Some(other_legacy) = reader.as_legacy_opt() { - if other_legacy.num_batches() != num_batches { - return Err(Error::invalid_input("Cannot create FragmentReader from data files with different number of batches" - .to_string())); - } - } else { - return Err(Error::invalid_input( - "Cannot mix legacy and non-legacy readers".to_string(), - )); - } - } - } - Ok(Self { - readers, - output_schema, - deletion_vec, - row_id_sequence, - fragment_id, - with_row_id: false, - with_row_addr: false, - with_row_last_updated_at_version: false, - with_row_created_at_version: false, - make_deletions_null: false, - fragment, - last_updated_at_sequence: None, - created_at_sequence: None, - num_rows, - num_physical_rows, - }) - } - - pub(crate) fn with_row_id(&mut self) -> &mut Self { - self.with_row_id = true; - self.output_schema = self - .output_schema - .try_with_column(ROW_ID_FIELD.clone()) - .expect("Table already has a column named _rowid"); - self - } - - pub(crate) fn with_row_address(&mut self) -> &mut Self { - self.with_row_addr = true; - self.output_schema = self - .output_schema - .try_with_column(ROW_ADDR_FIELD.clone()) - .expect("Table already has a column named _rowaddr"); - self + pub(crate) fn with_row_address(&mut self) -> &mut Self { + self.with_row_addr = true; + self } pub(crate) fn with_make_deletions_null(&mut self) -> &mut Self { @@ -2363,99 +2773,27 @@ impl FragmentReader { self } - pub(crate) fn with_row_last_updated_at_version(&mut self) -> &mut Self { - self.with_row_last_updated_at_version = true; - - // Load the version sequence if not already loaded - if self.last_updated_at_sequence.is_none() - && let Some(meta) = &self.fragment.last_updated_at_version_meta - && let Ok(sequence) = meta.load_sequence() - { - self.last_updated_at_sequence = Some(Arc::new(sequence)); - } - // If no metadata or load fails, sequence remains None (will default to version 1) - - // Add the version column to the output schema - self.output_schema = self - .output_schema - .try_with_column(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone()) - .expect("Table already has a column named _row_last_updated_at_version"); - - self - } - - pub(crate) fn with_row_created_at_version(&mut self) -> &mut Self { - self.with_row_created_at_version = true; - - // Load the version sequence if not already loaded - if self.created_at_sequence.is_none() - && let Some(meta) = &self.fragment.created_at_version_meta - && let Ok(sequence) = meta.load_sequence() - { - self.created_at_sequence = Some(Arc::new(sequence)); - } - // If no metadata or load fails, sequence remains None (will default to version 1) - - // Add the version column to the output schema - self.output_schema = self - .output_schema - .try_with_column(ROW_CREATED_AT_VERSION_FIELD.clone()) - .expect("Table already has a column named _row_created_at_version"); - - self - } - - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - pub(crate) fn legacy_num_batches(&self) -> usize { - let legacy_reader = self.readers[0].as_legacy(); - let num_batches = legacy_reader.num_batches(); - assert!( - self.readers - .iter() - .all(|r| r.as_legacy().num_batches() == num_batches), - "Data files have varying number of batches, which is not yet supported." - ); - num_batches + pub(crate) fn num_batches(&self) -> usize { + self.readers[0].reader.num_batches() } - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - /// - /// This method is also used by the updater. Even though the updater has been updated to - /// use streams, the updater still needs to know the batch size in v1 so that it can create - /// files with the same batch size. - pub(crate) fn legacy_num_rows_in_batch(&self, batch_id: u32) -> Option { - if let Some(legacy_reader) = self.readers.first().and_then(|r| r.as_legacy_opt()) { - if batch_id < legacy_reader.num_batches() as u32 { - Some(legacy_reader.num_rows_in_batch(batch_id as i32) as u32) - } else { - None - } - } else { - None - } + pub(crate) fn num_rows_in_batch(&self, batch_id: u32) -> Option { + let reader = &self.readers[0].reader; + (batch_id < reader.num_batches() as u32) + .then(|| reader.num_rows_in_batch(batch_id as i32) as u32) } - /// Read the page statistics of the fragment for the specified fields. - /// - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - pub(crate) async fn legacy_read_page_stats( + pub(crate) async fn read_page_stats( &self, projection: Option<&Schema>, ) -> Result> { - let mut stats_batches = vec![]; - for reader in self.readers.iter() { + let mut stats_batches = Vec::new(); + for reader in &self.readers { let schema = match projection { - Some(projection) => Arc::new(reader.projection().intersection(projection)?), - None => reader.projection().clone(), + Some(projection) => Arc::new(reader.projection.intersection(projection)?), + None => reader.projection.clone(), }; - let reader = reader.as_legacy(); - if let Some(stats_batch) = reader.read_page_stats(&schema.field_ids()).await? { + if let Some(stats_batch) = reader.reader.read_page_stats(&schema.field_ids()).await? { stats_batches.push(stats_batch); } } @@ -2467,39 +2805,26 @@ impl FragmentReader { } } - /// Read a batch of rows from the fragment, with a subset of columns. - /// - /// Note: the projection must be a subset of the schema the reader was created with. - /// Otherwise incorrect data will be returned. - /// - /// TODO: This method is relied upon by the v1 pushdown mechanism and will need to stay - /// in place until v1 is removed. v2 uses a different mechanism for pushdown and so there - /// is little benefit in updating the v1 pushdown node. - pub(crate) async fn legacy_read_batch_projected( + pub(crate) async fn read_batch_projected( &self, batch_id: usize, params: impl Into + Clone, projection: &Schema, ) -> Result { - let first_reader = self.readers[0].as_legacy(); + let first_reader = &self.readers[0].reader; // All batches have the same size in v1, except for the last one. let batch_offset = batch_id * first_reader.num_rows_in_batch(0); let rows_in_batch = first_reader.num_rows_in_batch(batch_id as i32); let batches = if !projection.fields.is_empty() { let read_tasks = self.readers.iter().map(|reader| { - let projection = reader.projection().intersection(projection); + let projection = reader.projection.intersection(projection); let params = params.clone(); - - let reader = reader.as_legacy(); + let reader = &reader.reader; async move { - // Apply ? inside the task to keep read_tasks a simple iter of futures - // for try_join_all let projection = projection?; if projection.fields.is_empty() { - // The projection caused one of the data files to become - // irrelevant and so we can skip it Result::Ok(None) } else { Ok(Some( @@ -2510,12 +2835,12 @@ impl FragmentReader { } } }); - let results = try_join_all(read_tasks).await?; - results.into_iter().flatten().collect::>() + try_join_all(read_tasks) + .await? + .into_iter() + .flatten() + .collect::>() } else { - // If we are selecting no columns, we can assume we are just getting - // the row ids. If this is the case, we need to generate an empty - // batch with the correct number of rows. let expected_rows = params .clone() .into() @@ -2531,10 +2856,6 @@ impl FragmentReader { let params = params.into(); let result = merge_batches(&batches)?; - - // Need to apply deletions and row ids. - // In order to apply deletions we need to change the parameters to be - // relative to the file, not the batch. let file_params = match params { ReadBatchParams::Indices(indices) => ReadBatchParams::Indices( indices @@ -2571,472 +2892,2482 @@ impl FragmentReader { row_id_sequence: self.row_id_sequence.clone(), with_row_id: self.with_row_id, with_row_addr: self.with_row_addr, - with_row_last_updated_at_version: self.with_row_last_updated_at_version, - with_row_created_at_version: self.with_row_created_at_version, - last_updated_at_sequence: self.last_updated_at_sequence.clone(), - created_at_sequence: self.created_at_sequence.clone(), + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + last_updated_at_sequence: None, + created_at_sequence: None, make_deletions_null: self.make_deletions_null, total_num_rows: first_reader.len() as u32, }, )?; - let output_schema = { - let mut output_schema = ArrowSchema::from(projection); - if self.with_row_id { - output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone())?; - } - if self.with_row_addr { - output_schema = output_schema.try_with_column(ROW_ADDR_FIELD.clone())?; - } - output_schema - }; - + let mut output_schema = ArrowSchema::from(projection); + if self.with_row_id { + output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone())?; + } + if self.with_row_addr { + output_schema = output_schema.try_with_column(ROW_ADDR_FIELD.clone())?; + } Ok(result.project_by_schema(&output_schema)?) } +} - async fn new_read_impl<'a, F>( - &'a self, - params: ReadBatchParams, - batch_size: u32, - read_fn: F, - ) -> Result - where - F: Fn(&'a dyn GenericFileReader) -> BoxFuture<'a, Result>, - { - let total_num_rows = self.num_physical_rows as u32; - // Note that the fragment length might be considerably smaller if there are deleted rows. - // E.g. if a fragment has 100 rows but rows 0..10 are deleted we still need to make - // sure it is valid to read / take 0..100 - if !params.valid_given_len(total_num_rows as usize) { - return Err(Error::invalid_input(format!( - "Invalid read params {} for fragment with {} addressable rows", - params, total_num_rows - ))); - } - // If just the row id or address there is no need to actually read any data - // and we don't need to involve the readers at all. - // - // The v1 reader does not support reading batches with zero columns, so - // we need this as a separate code path. - // In these cases, we can just emit batches with zero columns and rely - // on `wrap_with_row_id_and_delete` to add the row id or address column. - // - // We could potentially delete the support for no-columns in the wrap function or - // we can delete this path once we migrate away from any support of v1. - let merged = if self.num_system_cols() == self.output_schema.fields.len() { - let selected_rows = params.to_offsets_total(total_num_rows).len(); - let tasks = (0..selected_rows) - .step_by(batch_size as usize) - .map(move |offset| { - let num_rows = (batch_size as usize).min(selected_rows - offset); - let batch = RecordBatch::from(StructArray::new_empty_fields(num_rows, None)); - ReadBatchTask { - task: std::future::ready(Ok(batch)).boxed(), - num_rows: num_rows as u32, - } - }); - stream::iter(tasks).boxed() - } else { - // Read each data file, these reads should produce streams of equal sized - // tasks. In other words, if we get 3 tasks of 20 rows and then a task - // of 10 rows from one data file we should get the same from the other. - // - // We launch all readers' scheduling work concurrently — for v2 files - // this is where the decode scheduler's `initialize` I/O happens, so - // running them in parallel keeps the per-file scheduling I/Os from - // serializing. - let read_futs = self.readers.iter().filter_map(|reader| { - // Normally we filter out empty readers in the open_readers method - // However, we will keep the first empty reader to use for row id - // purposes on some legacy paths and so we need to filter that out - // here. - if reader.projection().fields.is_empty() { - None - } else { - Some(read_fn(reader.as_ref())) - } - }); - let read_streams = futures::future::try_join_all(read_futs).await?; - // Merge the streams, this merges the generated batches - lance_table::utils::stream::merge_streams(read_streams) - }; +/// [`FragmentReader`] is an abstract reader for a [`FileFragment`]. +/// +/// It opens the data files that contains the columns of the projection schema, and +/// reconstruct the RecordBatch from columns read from each data file. +#[derive(Debug)] +pub struct FragmentReader { + /// Readers and schema of each opened data file. + readers: Vec>, - // Add the row id column (if needed) and delete rows (if a deletion - // vector is present). - let config = RowIdAndDeletesConfig { - deletion_vector: self.deletion_vec.clone(), - row_id_sequence: self.row_id_sequence.clone(), - make_deletions_null: self.make_deletions_null, - with_row_id: self.with_row_id, - with_row_addr: self.with_row_addr, - with_row_last_updated_at_version: self.with_row_last_updated_at_version, - with_row_created_at_version: self.with_row_created_at_version, - last_updated_at_sequence: self.last_updated_at_sequence.clone(), - created_at_sequence: self.created_at_sequence.clone(), - params, - total_num_rows, - }; - let output_schema = Arc::new(self.output_schema.clone()); - Ok( - wrap_with_row_id_and_delete(merged, self.fragment_id as u32, config) - // Finally, reorder the columns to match the order specified in the projection - .map(move |batch_fut| { - let output_schema = output_schema.clone(); - batch_fut - .map(move |batch| { - batch? - .project_by_schema(&output_schema) - .map_err(Error::from) - }) - .boxed() - }) - .boxed(), - ) - } + /// The output schema. The defines the order in which the columns are returned. + output_schema: ArrowSchema, - fn patch_range_for_deletions(&self, range: Range, dv: &DeletionVector) -> Range { - let mut start = range.start; - let mut end = range.end; - for val in dv.to_sorted_iter() { - if val <= start { - start += 1; - end += 1; - } else if val < end { - end += 1; - } else { - break; - } - } - start..end - } + /// The deleted row IDs + deletion_vec: Option>, - async fn do_read_range( - &self, - mut range: Range, - batch_size: u32, - skip_deleted_rows: bool, - ) -> Result { - if skip_deleted_rows && let Some(deletion_vector) = self.deletion_vec.as_ref() { - range = self.patch_range_for_deletions(range, deletion_vector.as_ref()); - } - self.new_read_impl( - ReadBatchParams::Range(range.start as usize..range.end as usize), - batch_size, - move |reader| { - reader.read_range_tasks( - range.start as u64..range.end as u64, - batch_size, - reader.projection().clone(), - ) - }, - ) - .await - } + /// The row id sequence + /// + /// Only populated if the stable row id feature is enabled. + row_id_sequence: Option>, - fn num_system_cols(&self) -> usize { - self.with_row_id as usize - + self.with_row_addr as usize - + self.with_row_created_at_version as usize - + self.with_row_last_updated_at_version as usize - } + /// ID of the fragment + fragment_id: usize, - /// Reads a range of rows from the fragment - /// - /// This function interprets the request as the Xth to the Nth row of the fragment (after deletions) - /// and will always return range.len().min(self.num_rows()) rows. - /// - /// This is async because it drives the per-data-file decode scheduler - /// `initialize` work before returning the stream — see - /// [`GenericFileReader`]. - pub async fn read_range( - &self, - range: Range, - batch_size: u32, - ) -> Result { - self.do_read_range(range, batch_size, true).await - } + /// True if we should generate a row id for the output + with_row_id: bool, - /// Takes a range of rows from the fragment - /// - /// Unlike [`Self::read_range`], this function will NOT skip deleted rows. If rows are deleted they will - /// be filtered or set to null. This function may return less than range.len() rows as a result. - /// - /// This is async for the same reason as [`Self::read_range`]. - pub async fn take_range( - &self, - range: Range, - batch_size: u32, - ) -> Result { - self.do_read_range(range, batch_size, false).await - } + /// True if we should generate a row address column in output + with_row_addr: bool, - /// Reads all rows from the fragment. - /// - /// This is async for the same reason as [`Self::read_range`]. - pub async fn read_all(&self, batch_size: u32) -> Result { - self.new_read_impl(ReadBatchParams::RangeFull, batch_size, move |reader| { - reader.read_all_tasks(batch_size, reader.projection().clone()) - }) - .await - } + /// True if we should generate a last updated at version column in output + with_row_last_updated_at_version: bool, - // This method is a clone of new_read_impl but returns tasks instead of batches - // - // It also only supports v2 files - /// - /// This is async for the same reason as [`Self::read_range`]. - pub async fn read_ranges( - &self, - ranges: Arc<[Range]>, - batch_size: u32, - ) -> Result { - let total_num_rows = self.num_physical_rows as u32; - let mut num_requested_rows = 0; - // Note that row ranges at this point are physical and not logical. - for range in ranges.as_ref() { - if range.end > total_num_rows as u64 { - return Err(Error::internal(format!( - "Invalid read of range {:?} for fragment {} with {} addressable rows", - range, self.fragment_id, total_num_rows - ))); - } - num_requested_rows += range.end - range.start; - } + /// True if we should generate a created at version column in output + with_row_created_at_version: bool, - let merged_stream = if self.num_system_cols() == self.output_schema.fields.len() { - let tasks = (0..num_requested_rows) - .step_by(batch_size as usize) - .map(move |offset| { - let num_rows = (batch_size as u64).min(num_requested_rows - offset); - let batch = - RecordBatch::from(StructArray::new_empty_fields(num_rows as usize, None)); - ReadBatchTask { - task: std::future::ready(Ok(batch)).boxed(), - num_rows: num_rows as u32, - } - }); - stream::iter(tasks).boxed() - } else { - // Read each data file, these reads should produce streams of equal sized - // tasks. In other words, if we get 3 tasks of 20 rows and then a task - // of 10 rows from one data file we should get the same from the other. - // - // Run all readers' scheduling concurrently so the per-file - // `initialize` I/Os overlap. - let read_futs = self.readers.iter().map(|reader| { - reader.read_ranges_tasks(ranges.clone(), batch_size, reader.projection().clone()) - }); - let read_streams = futures::future::try_join_all(read_futs).await?; - // Merge the streams, this merges the generated batches - lance_table::utils::stream::merge_streams(read_streams) - }; + /// If true, deleted rows will be set to null, which is fast + /// If false, deleted rows will be removed from the batch, requiring a copy + make_deletions_null: bool, - // Add the row id column (if needed) and delete rows (if a deletion - // vector is present). - let config = RowIdAndDeletesConfig { - deletion_vector: self.deletion_vec.clone(), + /// The fragment metadata (needed for version columns) + fragment: Arc, + + /// The last_updated_at version sequence (loaded from fragment metadata) + last_updated_at_sequence: Option>, + + /// The created_at version sequence (loaded from fragment metadata) + created_at_sequence: Option>, + + // total number of real rows in the fragment (num_physical_rows - num_deleted_rows) + num_rows: usize, + + // total number of physical rows in the fragment (all rows, ignoring deletions) + num_physical_rows: usize, + + /// Read-time state for resolving data overlay files: the coverage plan plus + /// what is needed to open overlay readers. `None` when the fragment has no + /// overlays. Overlays are merged into base batches (by `offset_in_frag`) before + /// deletion filtering, opening only the files each read's rows touch. + overlay: Option, +} + +/// What [`FragmentReader`] needs to resolve overlays at read time: the coverage +/// plan (from metadata, cheap to build), and the fragment + config needed to open +/// overlay readers once the read's rows — and therefore which files it touches — +/// are known. All `Arc` so cloning a reader stays cheap. +#[derive(Clone, Debug)] +struct OverlayReadState { + planner: Arc, + fragment: Arc, + read_config: Arc, +} + +// Custom clone impl needed because it is not easy to clone Box +// +// We currently need FragmentReader to be Clone because the pushdown scan clones it +// to reuse the fragment reader for both "scan with row id" and "scan without row id" +impl Clone for FragmentReader { + fn clone(&self) -> Self { + Self { + readers: self + .readers + .iter() + .map(|reader| reader.clone_box()) + .collect::>(), + output_schema: self.output_schema.clone(), + deletion_vec: self.deletion_vec.clone(), row_id_sequence: self.row_id_sequence.clone(), - make_deletions_null: self.make_deletions_null, + fragment_id: self.fragment_id, with_row_id: self.with_row_id, with_row_addr: self.with_row_addr, with_row_last_updated_at_version: self.with_row_last_updated_at_version, with_row_created_at_version: self.with_row_created_at_version, + make_deletions_null: self.make_deletions_null, + fragment: self.fragment.clone(), last_updated_at_sequence: self.last_updated_at_sequence.clone(), created_at_sequence: self.created_at_sequence.clone(), - params: ReadBatchParams::Ranges(ranges), - total_num_rows, - }; - let output_schema = Arc::new(self.output_schema.clone()); - Ok( - wrap_with_row_id_and_delete(merged_stream, self.fragment_id as u32, config) - // Finally, reorder the columns to match the order specified in the projection - .map(move |batch_fut| { - let output_schema = output_schema.clone(); - batch_fut - .map(move |batch| { - batch? - .project_by_schema(&output_schema) - .map_err(Error::from) - }) - .boxed() - }) - .boxed(), - ) + num_rows: self.num_rows, + num_physical_rows: self.num_physical_rows, + overlay: self.overlay.clone(), + } } +} - // Legacy function that reads a range of data and concatenates the results - // into a single batch - // - // TODO: Move away from this by changing callers to support consuming a stream - pub async fn legacy_read_range_as_batch(&self, range: Range) -> Result { - let batches = self - .take_range( - range.start as u32..range.end as u32, - DEFAULT_BATCH_READ_SIZE, - ) - .await? - .buffered(get_num_compute_intensive_cpus()) - .try_collect::>() - .await?; - concat_batches(&Arc::new(self.output_schema.clone()), batches.iter()).map_err(Error::from) +impl std::fmt::Display for FragmentReader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "FragmentReader(id={})", self.fragment_id) } +} - /// Take rows from this fragment. - pub async fn take( - &self, - indices: &[u32], - batch_size: u32, - take_priority: Option, - ) -> Result { - let indices_arr = UInt32Array::from(indices.to_vec()); - self.new_read_impl( - ReadBatchParams::Indices(indices_arr), - batch_size, - move |reader| { - reader.take_all_tasks( - indices, - batch_size, - reader.projection().clone(), - take_priority, - ) - }, - ) - .await +fn merge_batches(batches: &[RecordBatch]) -> Result { + if batches.is_empty() { + return Err(Error::invalid_input( + "Cannot merge empty batches".to_string(), + )); } - /// Take rows from this fragment, will perform a copy if the underlying reader returns multiple - /// batches. May return an error if the taken rows do not fit into a single batch. - /// - /// Duplicate indices are allowed and will produce duplicate rows in the output. - pub async fn take_as_batch( - &self, - indices: &[u32], - take_priority: Option, - ) -> Result { - // The v2 encoding layer requires strictly increasing indices. Deduplicate - // here so callers (e.g. FTS with duplicate row matches) don't need to. - let has_duplicates = indices.windows(2).any(|w| w[0] == w[1]); - let (unique_indices, expand_map) = if has_duplicates { - let mut unique: Vec = Vec::with_capacity(indices.len()); - let mut mapping: Vec = Vec::with_capacity(indices.len()); - for &idx in indices { - if unique.last() != Some(&idx) { - unique.push(idx); - } - mapping.push((unique.len() - 1) as u32); - } - (Cow::Owned(unique), Some(UInt32Array::from(mapping))) - } else { - (Cow::Borrowed(indices), None) - }; + let mut merged = batches[0].clone(); + for batch in batches.iter().skip(1) { + merged = merged.merge(batch)?; + } + Ok(merged) +} + +impl FragmentReader { + #[allow(clippy::too_many_arguments)] + fn try_new( + fragment_id: usize, + deletion_vec: Option>, + row_id_sequence: Option>, + readers: Vec>, + output_schema: ArrowSchema, + num_rows: usize, + num_physical_rows: usize, + fragment: Arc, + ) -> Result { + Ok(Self { + readers, + output_schema, + deletion_vec, + row_id_sequence, + fragment_id, + with_row_id: false, + with_row_addr: false, + with_row_last_updated_at_version: false, + with_row_created_at_version: false, + make_deletions_null: false, + fragment, + last_updated_at_sequence: None, + created_at_sequence: None, + num_rows, + num_physical_rows, + overlay: None, + }) + } + + pub(crate) fn with_row_id(&mut self) -> &mut Self { + self.with_row_id = true; + self.output_schema = self + .output_schema + .try_with_column(ROW_ID_FIELD.clone()) + .expect("Table already has a column named _rowid"); + self + } + + pub(crate) fn with_row_address(&mut self) -> &mut Self { + self.with_row_addr = true; + self.output_schema = self + .output_schema + .try_with_column(ROW_ADDR_FIELD.clone()) + .expect("Table already has a column named _rowaddr"); + self + } + + pub(crate) fn with_make_deletions_null(&mut self) -> &mut Self { + self.make_deletions_null = true; + self + } + + pub(crate) fn with_row_last_updated_at_version(&mut self) -> &mut Self { + self.with_row_last_updated_at_version = true; + + // Load the version sequence if not already loaded + if self.last_updated_at_sequence.is_none() + && let Some(meta) = &self.fragment.last_updated_at_version_meta + && let Ok(sequence) = meta.load_sequence() + { + self.last_updated_at_sequence = Some(Arc::new(sequence)); + } + // If no metadata or load fails, sequence remains None (will default to version 1) + + // Add the version column to the output schema + self.output_schema = self + .output_schema + .try_with_column(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone()) + .expect("Table already has a column named _row_last_updated_at_version"); + + self + } + + pub(crate) fn with_row_created_at_version(&mut self) -> &mut Self { + self.with_row_created_at_version = true; + + // Load the version sequence if not already loaded + if self.created_at_sequence.is_none() + && let Some(meta) = &self.fragment.created_at_version_meta + && let Ok(sequence) = meta.load_sequence() + { + self.created_at_sequence = Some(Arc::new(sequence)); + } + // If no metadata or load fails, sequence remains None (will default to version 1) + + // Add the version column to the output schema + self.output_schema = self + .output_schema + .try_with_column(ROW_CREATED_AT_VERSION_FIELD.clone()) + .expect("Table already has a column named _row_created_at_version"); + + self + } + + /// Merge data overlay values onto a stream of base batches. + /// + /// Runs on physical rows in read order, *before* deletion filtering, so each + /// row can be addressed by its position in the fragment (its `offset_in_frag`, + /// derived from `params`) and deletions take precedence naturally: an overlay + /// value for a deleted row is dropped along with the row downstream. A no-op + /// when the fragment has no overlays. + /// + /// The read's `offset_in_frag` values are known from `params` up front, so + /// overlays are resolved here to just the files this read's rows touch — an + /// overlay whose cells fall outside the read is not opened at all. Within each + /// batch, the overlay reads (only the values that batch needs) are then issued + /// concurrently with the base read rather than after it. + async fn merge_overlays( + &self, + merged: ReadBatchTaskStream, + params: &ReadBatchParams, + total_num_rows: u32, + ) -> Result { + let Some(overlay) = &self.overlay else { + return Ok(merged); + }; + // The offset_in_frag of every row this read will return, materialized once. + // Cost is one u32 per output row (a whole-fragment scan is 4 bytes/row), and + // it lets us both prune overlays to the read and slice each batch's offsets + // below without reading any data. Only paid when the fragment has overlays. + // + // TODO(overlay perf): this could be avoided by teaching `ReadBatchParams` to + // yield a coverage bitmap directly (for pruning) and to slice per batch (for + // the routing below), or by moving `ReadBatchParams` to a roaring bitmap + // wholesale — a larger refactor tracked separately. + let offsets_in_frag: Arc> = + Arc::new(params.to_offsets_total(total_num_rows).values().to_vec()); + + // Open only the overlay readers this read touches (pruned by row selection). + let plans = resolve_overlays( + &overlay.planner, + &offsets_in_frag, + &overlay.fragment, + &overlay.read_config, + ) + .await?; + if plans.is_empty() { + return Ok(merged); + } + let plans = Arc::new(plans); + + // Batches arrive in physical read order, so a running total of the rows seen + // so far gives each batch its starting offset_in_batch into `offsets_in_frag`. + let mut rows_seen = 0usize; + let stream = merged + .map(move |task| { + let num_rows = task.num_rows; + let start = rows_seen; + rows_seen += num_rows as usize; + let offsets_in_frag = offsets_in_frag.clone(); + let plans = plans.clone(); + let inner = task.task; + ReadBatchTask { + num_rows, + task: async move { + let batch_offsets = &offsets_in_frag[start..start + num_rows as usize]; + merge_overlay_batch(inner, batch_offsets, &plans).await + } + .boxed(), + } + }) + .boxed(); + Ok(stream) + } + + async fn new_read_impl<'a, F>( + &'a self, + params: ReadBatchParams, + batch_size: u32, + read_fn: F, + ) -> Result + where + F: Fn(&'a dyn GenericFileReader) -> BoxFuture<'a, Result>, + { + let total_num_rows = self.num_physical_rows as u32; + // Note that the fragment length might be considerably smaller if there are deleted rows. + // E.g. if a fragment has 100 rows but rows 0..10 are deleted we still need to make + // sure it is valid to read / take 0..100 + if !params.valid_given_len(total_num_rows as usize) { + return Err(Error::invalid_input(format!( + "Invalid read params {} for fragment with {} addressable rows", + params, total_num_rows + ))); + } + // If just the row id or address there is no need to actually read any data + // and we don't need to involve the readers at all. + // + // The v1 reader does not support reading batches with zero columns, so + // we need this as a separate code path. + // In these cases, we can just emit batches with zero columns and rely + // on `wrap_with_row_id_and_delete` to add the row id or address column. + // + // We could potentially delete the support for no-columns in the wrap function or + // we can delete this path once we migrate away from any support of v1. + let merged = if self.num_system_cols() == self.output_schema.fields.len() { + let selected_rows = params.to_offsets_total(total_num_rows).len(); + let tasks = (0..selected_rows) + .step_by(batch_size as usize) + .map(move |offset| { + let num_rows = (batch_size as usize).min(selected_rows - offset); + let batch = RecordBatch::from(StructArray::new_empty_fields(num_rows, None)); + ReadBatchTask { + task: std::future::ready(Ok(batch)).boxed(), + num_rows: num_rows as u32, + } + }); + stream::iter(tasks).boxed() + } else { + // Read each data file, these reads should produce streams of equal sized + // tasks. In other words, if we get 3 tasks of 20 rows and then a task + // of 10 rows from one data file we should get the same from the other. + // + // We launch all readers' scheduling work concurrently — for v2 files + // this is where the decode scheduler's `initialize` I/O happens, so + // running them in parallel keeps the per-file scheduling I/Os from + // serializing. + let read_futs = self.readers.iter().filter_map(|reader| { + // Normally we filter out empty readers in the open_readers method + // However, we will keep the first empty reader to use for row id + // purposes on some legacy paths and so we need to filter that out + // here. + if reader.projection().fields.is_empty() { + None + } else { + Some(read_fn(reader.as_ref())) + } + }); + let read_streams = futures::future::try_join_all(read_futs).await?; + // Merge the streams, this merges the generated batches + lance_table::utils::stream::merge_streams(read_streams) + }; + + let merged = self.merge_overlays(merged, ¶ms, total_num_rows).await?; + + // Add the row id column (if needed) and delete rows (if a deletion + // vector is present). + let config = RowIdAndDeletesConfig { + deletion_vector: self.deletion_vec.clone(), + row_id_sequence: self.row_id_sequence.clone(), + make_deletions_null: self.make_deletions_null, + with_row_id: self.with_row_id, + with_row_addr: self.with_row_addr, + with_row_last_updated_at_version: self.with_row_last_updated_at_version, + with_row_created_at_version: self.with_row_created_at_version, + last_updated_at_sequence: self.last_updated_at_sequence.clone(), + created_at_sequence: self.created_at_sequence.clone(), + params, + total_num_rows, + }; + let output_schema = Arc::new(self.output_schema.clone()); + Ok( + wrap_with_row_id_and_delete(merged, self.fragment_id as u32, config) + // Finally, reorder the columns to match the order specified in the projection + .map(move |batch_fut| { + let output_schema = output_schema.clone(); + batch_fut + .map(move |batch| { + batch? + .project_by_schema(&output_schema) + .map_err(Error::from) + }) + .boxed() + }) + .boxed(), + ) + } + + fn patch_range_for_deletions(&self, range: Range, dv: &DeletionVector) -> Range { + let mut start = range.start; + let mut end = range.end; + for val in dv.to_sorted_iter() { + if val <= start { + start += 1; + end += 1; + } else if val < end { + end += 1; + } else { + break; + } + } + start..end + } + + async fn do_read_range( + &self, + mut range: Range, + batch_size: u32, + skip_deleted_rows: bool, + ) -> Result { + if skip_deleted_rows && let Some(deletion_vector) = self.deletion_vec.as_ref() { + range = self.patch_range_for_deletions(range, deletion_vector.as_ref()); + } + self.new_read_impl( + ReadBatchParams::Range(range.start as usize..range.end as usize), + batch_size, + move |reader| { + reader.read_range_tasks( + range.start as u64..range.end as u64, + batch_size, + reader.projection().clone(), + ) + }, + ) + .await + } + + fn num_system_cols(&self) -> usize { + self.with_row_id as usize + + self.with_row_addr as usize + + self.with_row_created_at_version as usize + + self.with_row_last_updated_at_version as usize + } + + /// Reads a range of rows from the fragment + /// + /// This function interprets the request as the Xth to the Nth row of the fragment (after deletions) + /// and will always return range.len().min(self.num_rows()) rows. + /// + /// This is async because it drives the per-data-file decode scheduler + /// `initialize` work before returning the stream — see + /// [`GenericFileReader`]. + pub async fn read_range( + &self, + range: Range, + batch_size: u32, + ) -> Result { + self.do_read_range(range, batch_size, true).await + } + + /// Takes a range of rows from the fragment + /// + /// Unlike [`Self::read_range`], this function will NOT skip deleted rows. If rows are deleted they will + /// be filtered or set to null. This function may return less than range.len() rows as a result. + /// + /// This is async for the same reason as [`Self::read_range`]. + pub async fn take_range( + &self, + range: Range, + batch_size: u32, + ) -> Result { + self.do_read_range(range, batch_size, false).await + } + + /// Reads all rows from the fragment. + /// + /// This is async for the same reason as [`Self::read_range`]. + pub async fn read_all(&self, batch_size: u32) -> Result { + self.new_read_impl(ReadBatchParams::RangeFull, batch_size, move |reader| { + reader.read_all_tasks(batch_size, reader.projection().clone()) + }) + .await + } + + // This method is a clone of new_read_impl but returns tasks instead of batches + // + // It also only supports v2 files + /// + /// This is async for the same reason as [`Self::read_range`]. + pub async fn read_ranges( + &self, + ranges: Arc<[Range]>, + batch_size: u32, + ) -> Result { + let total_num_rows = self.num_physical_rows as u32; + let mut num_requested_rows = 0; + // Note that row ranges at this point are physical and not logical. + for range in ranges.as_ref() { + if range.end > total_num_rows as u64 { + return Err(Error::internal(format!( + "Invalid read of range {:?} for fragment {} with {} addressable rows", + range, self.fragment_id, total_num_rows + ))); + } + num_requested_rows += range.end - range.start; + } + + let merged_stream = if self.num_system_cols() == self.output_schema.fields.len() { + let tasks = (0..num_requested_rows) + .step_by(batch_size as usize) + .map(move |offset| { + let num_rows = (batch_size as u64).min(num_requested_rows - offset); + let batch = + RecordBatch::from(StructArray::new_empty_fields(num_rows as usize, None)); + ReadBatchTask { + task: std::future::ready(Ok(batch)).boxed(), + num_rows: num_rows as u32, + } + }); + stream::iter(tasks).boxed() + } else { + // Read each data file, these reads should produce streams of equal sized + // tasks. In other words, if we get 3 tasks of 20 rows and then a task + // of 10 rows from one data file we should get the same from the other. + // + // Run all readers' scheduling concurrently so the per-file + // `initialize` I/Os overlap. + let read_futs = self.readers.iter().map(|reader| { + reader.read_ranges_tasks(ranges.clone(), batch_size, reader.projection().clone()) + }); + let read_streams = futures::future::try_join_all(read_futs).await?; + // Merge the streams, this merges the generated batches + lance_table::utils::stream::merge_streams(read_streams) + }; + + let params = ReadBatchParams::Ranges(ranges); + let merged_stream = self + .merge_overlays(merged_stream, ¶ms, total_num_rows) + .await?; + + // Add the row id column (if needed) and delete rows (if a deletion + // vector is present). + let config = RowIdAndDeletesConfig { + deletion_vector: self.deletion_vec.clone(), + row_id_sequence: self.row_id_sequence.clone(), + make_deletions_null: self.make_deletions_null, + with_row_id: self.with_row_id, + with_row_addr: self.with_row_addr, + with_row_last_updated_at_version: self.with_row_last_updated_at_version, + with_row_created_at_version: self.with_row_created_at_version, + last_updated_at_sequence: self.last_updated_at_sequence.clone(), + created_at_sequence: self.created_at_sequence.clone(), + params, + total_num_rows, + }; + let output_schema = Arc::new(self.output_schema.clone()); + Ok( + wrap_with_row_id_and_delete(merged_stream, self.fragment_id as u32, config) + // Finally, reorder the columns to match the order specified in the projection + .map(move |batch_fut| { + let output_schema = output_schema.clone(); + batch_fut + .map(move |batch| { + batch? + .project_by_schema(&output_schema) + .map_err(Error::from) + }) + .boxed() + }) + .boxed(), + ) + } + + /// Reads a range and concatenates the result into one batch. + pub async fn read_range_as_batch(&self, range: Range) -> Result { + let batches = self + .take_range( + range.start as u32..range.end as u32, + DEFAULT_BATCH_READ_SIZE, + ) + .await? + .buffered(get_num_compute_intensive_cpus()) + .try_collect::>() + .await?; + concat_batches(&Arc::new(self.output_schema.clone()), batches.iter()).map_err(Error::from) + } + + /// Take rows from this fragment. + pub async fn take( + &self, + indices: &[u32], + batch_size: u32, + take_priority: Option, + ) -> Result { + let indices_arr = UInt32Array::from(indices.to_vec()); + self.new_read_impl( + ReadBatchParams::Indices(indices_arr), + batch_size, + move |reader| { + reader.take_all_tasks( + indices, + batch_size, + reader.projection().clone(), + take_priority, + ) + }, + ) + .await + } + + /// Take rows from this fragment, will perform a copy if the underlying reader returns multiple + /// batches. May return an error if the taken rows do not fit into a single batch. + /// + /// Duplicate indices are allowed and will produce duplicate rows in the output. + pub async fn take_as_batch( + &self, + indices: &[u32], + take_priority: Option, + ) -> Result { + // The v2 encoding layer requires strictly increasing indices. Deduplicate + // here so callers (e.g. FTS with duplicate row matches) don't need to. + let has_duplicates = indices.windows(2).any(|w| w[0] == w[1]); + let (unique_indices, expand_map) = if has_duplicates { + let mut unique: Vec = Vec::with_capacity(indices.len()); + let mut mapping: Vec = Vec::with_capacity(indices.len()); + for &idx in indices { + if unique.last() != Some(&idx) { + unique.push(idx); + } + mapping.push((unique.len() - 1) as u32); + } + (Cow::Owned(unique), Some(UInt32Array::from(mapping))) + } else { + (Cow::Borrowed(indices), None) + }; + + let batches = self + .take(&unique_indices, u32::MAX, take_priority) + .await? + .buffered(get_num_compute_intensive_cpus()) + .try_collect::>() + .await?; + let mut batch = concat_batches(&Arc::new(self.output_schema.clone()), batches.iter())?; + + if let Some(expand_map) = expand_map { + batch = arrow_select::take::take_record_batch(&batch, &expand_map)?; + } + + Ok(batch) + } +} + +/// A wrapper around a `RecordBatchReader` that converts Arrow JSON columns +/// (Utf8/LargeUtf8 with `arrow.json` extension) to Lance JSON columns +/// (LargeBinary with `lance.json` extension / JSONB format). +/// +/// This is needed when user-provided data contains Arrow JSON fields but the +/// dataset stores them in Lance's JSONB binary format. +struct JsonConvertingReader { + inner: Box, + schema: arrow_schema::SchemaRef, +} + +impl JsonConvertingReader { + fn new(inner: Box) -> Self { + use lance_arrow::json::arrow_json_to_lance_json; + + // Build the converted schema (Arrow JSON fields → Lance JSON fields) + let orig_schema = inner.schema(); + let new_fields: Vec = orig_schema + .fields() + .iter() + .map(|f| { + if is_arrow_json_field(f) || has_json_fields(f) { + Arc::new(arrow_json_to_lance_json(f)) + } else { + Arc::clone(f) + } + }) + .collect(); + let schema = Arc::new(arrow_schema::Schema::new_with_metadata( + new_fields, + orig_schema.metadata().clone(), + )); + + Self { inner, schema } + } +} + +impl Iterator for JsonConvertingReader { + type Item = std::result::Result; + + fn next(&mut self) -> Option { + self.inner + .next() + .map(|result| result.and_then(|batch| convert_json_columns(&batch))) + } +} + +impl RecordBatchReader for JsonConvertingReader { + fn schema(&self) -> arrow_schema::SchemaRef { + self.schema.clone() + } +} + +#[cfg(test)] +mod tests { + use arrow_arith::numeric::mul; + use arrow_array::{ + ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatchIterator, StringArray, + }; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance_core::ROW_ID; + use lance_core::utils::tempfile::TempStrDir; + use lance_datagen::{RowCount, array, gen_batch}; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + use lance_file::writer::FileWriterOptions; + use lance_io::{assert_io_eq, assert_io_lt, object_store::ObjectStore}; + use pretty_assertions::assert_eq; + use rstest::rstest; + use std::collections::HashMap; + + use super::*; + use crate::{ + dataset::{ + InsertBuilder, + transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}, + }, + session::Session, + utils::test::TestDatasetGenerator, + }; + + async fn create_dataset(test_uri: &str, data_storage_version: LanceFileVersion) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, true), + ArrowField::new("s", DataType::Utf8, true), + ])); + + let batches: Vec = (0..10) + .map(|i| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20)), + Arc::new(StringArray::from_iter_values( + (i * 20..(i + 1) * 20).map(|v| format!("s-{}", v)), + )), + ], + ) + .unwrap() + }) + .collect(); + + let write_params = WriteParams { + max_rows_per_file: 40, + max_rows_per_group: 10, + data_storage_version: Some(data_storage_version), + ..Default::default() + }; + let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + Dataset::write(batches, test_uri, Some(write_params)) + .await + .unwrap(); + + Dataset::open(test_uri).await.unwrap() + } + + async fn create_dataset_v2(test_uri: &str) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + true, + )])); + + let batches: Vec = (0..10) + .map(|i| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20))], + ) + .unwrap() + }) + .collect(); + + let write_params = WriteParams { + max_rows_per_file: 40, + max_rows_per_group: 10, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }; + let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + Dataset::write(batches, test_uri, Some(write_params)) + .await + .unwrap(); + + Dataset::open(test_uri).await.unwrap() + } + + /// End-to-end tests for reading data overlay files (OSS-1324): overlays are + /// written, committed via the `DataOverlay` transaction, and then resolved on + /// the `take` and scan read paths. + mod overlay_read { + use std::sync::Arc; + + use arrow_array::{ + Array, ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StructArray, UInt64Array, + }; + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + use lance_core::datatypes::Schema; + use lance_file::version::LanceFileVersion; + use lance_file::writer::FileWriterOptions; + use lance_io::utils::CachedFileSize; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use object_store::path::Path; + use roaring::RoaringBitmap; + use rstest::rstest; + + use crate::dataset::transaction::{DataOverlayGroup, Operation}; + use crate::dataset::{Dataset, WriteDestination, WriteParams}; + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) + } + + /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1) + /// = id * 10, written 6 rows per file (fragments 0 and 1). + /// + async fn create_base_dataset(version: LanceFileVersion) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap() + } + + /// Write an overlay file covering `fields` (dataset field ids) of + /// `fragment_id` with the given coverage and per-field value columns, then + /// commit it as a `DataOverlay` transaction. `name` makes the file unique. + #[allow(clippy::too_many_arguments)] + async fn commit_overlay( + dataset: Dataset, + name: &str, + fragment_id: u64, + fields: &[i32], + coverage: OverlayCoverage, + columns: Vec, + version: LanceFileVersion, + ) -> Dataset { + let read_version = dataset.version().version; + let overlay_schema = dataset.schema().project_by_ids(fields, true); + + let filename = format!("{name}.lance"); + // The manifest records the bare filename; only the physical write is + // data-dir qualified. + let path = dataset.data_dir().join(filename.as_str()); + let obj_writer = dataset.object_store.create(&path).await.unwrap(); + let file_version = version.resolve(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + for (column_index, array) in columns.into_iter().enumerate() { + writer.write_column(column_index, array).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(field_id, _)| *field_id as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, column_index)| *column_index as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + let overlay = DataOverlayFile { + data_file, + coverage, + committed_version: 0, + }; + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![overlay], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap() + } + + /// `collect_paths` feeds `deep_clone`'s copy loop, so an overlay data file + /// it omits is referenced by the clone's manifest but never copied. The + /// clone then fails to read the overlaid values. + #[tokio::test] + async fn deep_clone_copies_overlay_files() { + use lance_core::utils::tempfile::TempStdDir; + + let version = LanceFileVersion::Stable; + let test_dir = TempStdDir::default(); + let source_uri = test_dir.join("source").to_str().unwrap().to_string(); + let clone_uri = test_dir.join("clone").to_str().unwrap().to_string(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 10))), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write( + reader, + &source_uri, + Some(WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Overlay `val` on offsets 0 and 1 of fragment 0. + let mut dataset = commit_overlay( + dataset, + "overlay0", + 0, + &[1], + OverlayCoverage::Shared(Arc::new(bitmap([0, 1]))), + vec![i32_array([Some(700), Some(701)])], + version, + ) + .await; + + let source_version = dataset.version().version; + dataset + .tags() + .create("clone-me", source_version) + .await + .unwrap(); + let cloned = dataset + .deep_clone(&clone_uri, "clone-me", None) + .await + .unwrap(); + + let batch = cloned + .scan() + .try_into_batch() + .await + .expect("the clone must be readable, including its overlay files"); + let val = col(&batch, "val"); + assert_eq!( + val.values()[..2], + [700, 701], + "the clone must return the overlaid values, not the base ones" + ); + } + + /// Deep-cloning a shallow clone has to drop every `base_id`, since the + /// result owns its files. An overlay whose `base_id` survives points at a + /// base the new manifest no longer lists. + #[tokio::test] + async fn deep_clone_of_shallow_clone_clears_overlay_base_id() { + use lance_core::utils::tempfile::TempStdDir; + + let version = LanceFileVersion::Stable; + let test_dir = TempStdDir::default(); + let source_uri = test_dir.join("source").to_str().unwrap().to_string(); + let shallow_uri = test_dir.join("shallow").to_str().unwrap().to_string(); + let deep_uri = test_dir.join("deep").to_str().unwrap().to_string(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 10))), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write( + reader, + &source_uri, + Some(WriteParams { + max_rows_per_file: 6, + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap(); + + let mut dataset = commit_overlay( + dataset, + "overlay0", + 0, + &[1], + OverlayCoverage::Shared(Arc::new(bitmap([0, 1]))), + vec![i32_array([Some(700), Some(701)])], + version, + ) + .await; + + let source_version = dataset.version().version; + dataset.tags().create("v", source_version).await.unwrap(); + let mut shallow = dataset + .shallow_clone(&shallow_uri, "v", None) + .await + .unwrap(); + // The shallow clone reaches the overlay through the parent. + assert!( + shallow.get_fragments()[0].metadata().overlays[0] + .data_file + .base_id + .is_some(), + "the shallow clone must reference the parent's overlay by base_id" + ); + + let shallow_version = shallow.version().version; + shallow.tags().create("v", shallow_version).await.unwrap(); + let deep = shallow.deep_clone(&deep_uri, "v", None).await.unwrap(); + + assert_eq!( + deep.get_fragments()[0].metadata().overlays[0] + .data_file + .base_id, + None, + "a deep clone owns its files, so the overlay must carry no base_id" + ); + assert!( + deep.manifest.base_paths.is_empty(), + "a deep clone lists no external bases" + ); + } + + fn full_schema(dataset: &Dataset) -> Schema { + dataset.schema().clone() + } + + fn col(batch: &RecordBatch, name: &str) -> Int32Array { + let idx = batch.schema().index_of(name).unwrap(); + batch + .column(idx) + .as_any() + .downcast_ref::() + .unwrap() + .clone() + } + + #[rstest] + #[tokio::test] + async fn test_take_covered_and_uncovered( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay fragment 0's `val` at physical offsets {1, 4}. + let dataset = commit_overlay( + dataset, + "ov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag + .take(&[0, 1, 2, 4], &full_schema(&dataset)) + .await + .unwrap(); + // Offsets 1 and 4 take overlay values; 0 and 2 fall through to base. + assert_eq!(col(&batch, "val").values(), &[0, 111, 20, 444]); + // The unrelated `id` column is untouched. + assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 4]); + } + + #[rstest] + #[tokio::test] + async fn test_take_newest_overlay_wins( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "older", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + // A newer overlay (later commit -> higher committed_version) re-covers + // offset 1. + let dataset = commit_overlay( + dataset, + "newer", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([Some(999)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 4], &full_schema(&dataset)).await.unwrap(); + // Offset 1 -> newest overlay (999); offset 4 -> only older covers it. + assert_eq!(col(&batch, "val").values(), &[999, 444]); + } + + #[rstest] + #[tokio::test] + async fn test_take_per_field_coverage( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Sparse overlay: `id` covers {2}, `val` covers {2, 3} — different + // offset sets and therefore unequal-length value columns. + let dataset = commit_overlay( + dataset, + "sparse", + 0, + &[0, 1], + OverlayCoverage::sparse(vec![bitmap([2]), bitmap([2, 3])]), + vec![i32_array([Some(777)]), i32_array([Some(220), Some(330)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap(); + // id: offset 2 covered (777), offset 3 falls through (3). + assert_eq!(col(&batch, "id").values(), &[777, 3]); + // val: both offsets covered (220, 330). + assert_eq!(col(&batch, "val").values(), &[220, 330]); + } + + #[rstest] + #[tokio::test] + async fn test_take_null_override( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "nullov", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([None])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[0, 1], &full_schema(&dataset)).await.unwrap(); + let val = col(&batch, "val"); + // Offset 0 is covered with a NULL value -> resolves to NULL; offset 1 + // falls through to the base value. + assert!(val.is_null(0)); + assert_eq!(val.value(1), 10); + } + + /// Overlays interact correctly with NULL *base* cells (distinct from a NULL + /// overlay value): a covered row whose base value is NULL is overridden to the + /// overlay's non-null value, while an uncovered NULL base cell falls through + /// and stays NULL. + #[rstest] + #[tokio::test] + async fn test_take_null_base_cell( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + // `val` is NULL at offsets 1 and 3. + Arc::new(Int32Array::from_iter([ + Some(0), + None, + Some(20), + None, + Some(40), + Some(50), + ])), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Cover offset 1 (NULL base) and offset 4 (non-null base); leave offset + // 3's NULL base uncovered. + let dataset = commit_overlay( + dataset, + "nullbase", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 3, 4], &full_schema(&dataset)).await.unwrap(); + let val = col(&batch, "val"); + // Offset 1: NULL base overridden to 111. Offset 3: uncovered NULL base + // stays NULL. Offset 4: non-null base overridden to 444. + assert_eq!(val.value(0), 111); + assert!(val.is_null(1)); + assert_eq!(val.value(2), 444); + } + + #[rstest] + #[tokio::test] + async fn test_overlay_on_deleted_row_is_inert( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let mut dataset = create_base_dataset(version).await; + // Delete global row 1 (fragment 0, physical offset 1). + dataset.delete("id = 1").await.unwrap(); + // Overlay covers the deleted offset 1 and the live offset 4. + let dataset = commit_overlay( + dataset, + "delov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + // Scan fragment 0: row 1 is gone, and offset 4's overlay value survives + // even though the deletion shifts logical positions — coverage is keyed + // by physical offset. + let frag = dataset.get_fragment(0).unwrap(); + let mut scanner = frag.scan(); + let batch = scanner + .project(&["id", "val"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 2, 3, 4, 5]); + assert_eq!(col(&batch, "val").values(), &[0, 20, 30, 444, 50]); + } + + #[rstest] + #[tokio::test] + async fn test_scan_multi_fragment_overlays( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay fragment 0 at offset 0 and fragment 1 at offset 0 (global + // row 6). Each fragment's coverage is independent. + let dataset = commit_overlay( + dataset, + "frag0", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(1000)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "frag1", + 1, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(6000)])], + version, + ) + .await; + + let batch = dataset + .scan() + .project(&["id", "val"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch.num_rows(), 12); + let expected: Vec = (0..12) + .map(|i| match i { + 0 => 1000, + 6 => 6000, + other => other * 10, + }) + .collect(); + assert_eq!(col(&batch, "val").values(), &expected); + } + + /// A `take` of a few rows must read only the overlay values those rows + /// touch — not the whole column. Uses v2.1 (which slices pages on read) and + /// an incompressible, all-covering overlay, so reading the full column would + /// be far more bytes than reading a couple of values. This is the regression + /// guard for the lazy, value-pushdown overlay read. + #[tokio::test] + async fn test_take_reads_only_needed_overlay_values() { + let version = LanceFileVersion::V2_1; + const N: usize = 100_000; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let base = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..N as i32)), + Arc::new(Int32Array::from_iter_values((0..N as i32).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: N, + max_rows_per_group: N, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(base)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay `val` over ALL N offsets with incompressible values, so the + // value column is ~N*4 bytes on disk. + let values: Vec = (0..N as u64) + .map(|i| { + let mut x = i; + x ^= x >> 33; + x = x.wrapping_mul(0xff51_afd7_ed55_8ccd); + x ^= x >> 33; + x as i32 + }) + .collect(); + let dataset = commit_overlay( + dataset, + "big", + 0, + &[1], + OverlayCoverage::dense(bitmap(0..N as u32)), + vec![Arc::new(Int32Array::from(values.clone())) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let val_only = dataset.schema().project_by_ids(&[1], true); + + // Measure only the reads that resolve the take. + dataset.object_store.io_stats_incremental(); + let batch = frag.take(&[0, 1], &val_only).await.unwrap(); + let io = dataset.object_store.io_stats_incremental(); + + // The overlay's `val` column alone is N*4 bytes; resolving two adjacent + // offsets must read only a small fraction of it. + let full_column_bytes = (N * std::mem::size_of::()) as u64; + assert!( + io.read_bytes > 0 && io.read_bytes < full_column_bytes / 4, + "take read {} bytes; expected far less than the {}-byte overlay \ + column (a take must not read the whole value column)", + io.read_bytes, + full_column_bytes, + ); + + // ...and it still resolves correctly. + let val = col(&batch, "val"); + assert_eq!(val.value(0), values[0]); + assert_eq!(val.value(1), values[1]); + } + + /// Row-selection pruning: an overlay whose coverage is disjoint from the + /// requested rows must not be opened at all. Proven by deleting the overlay's + /// data file — a `take` that misses its coverage still succeeds (the file is + /// never touched), while a `take` that hits it then fails because the file is + /// genuinely needed. + #[rstest] + #[tokio::test] + async fn test_take_prunes_overlays_outside_row_selection( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay on fragment 0 (offsets 0..6) covering only offset_in_frag 5. + let dataset = commit_overlay( + dataset, + "miss", + 0, + &[1], + OverlayCoverage::dense(bitmap([5])), + vec![i32_array([Some(5000)])], + version, + ) + .await; + + // Delete the overlay's data file: opening it now fails. + dataset + .object_store + .delete(&Path::from("data/miss.lance")) + .await + .unwrap(); + + let frag = dataset.get_fragment(0).unwrap(); + let val_only = dataset.schema().project_by_ids(&[1], true); + + // A take that misses the overlay's coverage must not open it, so it + // succeeds and returns base values (val = offset * 10). + let batch = frag.take(&[0, 1], &val_only).await.unwrap(); + assert_eq!(col(&batch, "val").values(), &[0, 10]); + + // A take that hits the coverage does need the file, so it now fails with + // a not-found error naming the missing overlay file. + let err = frag.take(&[5], &val_only).await.unwrap_err(); + let message = format!("{err:?}"); + assert!( + err.is_not_found() && message.contains("miss.lance"), + "take hitting the overlay's coverage should fail with a not-found error \ + for its missing file, got: {message}", + ); + } + + /// The overlay merge runs before `wrap_with_row_id_and_delete`, so the + /// `_rowid` system column must coexist with overlay-resolved data columns: + /// the row ids are unaffected by the merge and the overlay value still wins. + #[rstest] + #[tokio::test] + async fn test_scan_with_row_id_alongside_overlay( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "rowidov", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(1000)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag + .scan() + .with_row_id() + .project(&["id", "val"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + // Overlay value resolves... + assert_eq!(col(&batch, "val").values()[0], 1000); + assert_eq!(&col(&batch, "val").values()[1..], &[10, 20, 30, 40, 50]); + // ...and the row ids for fragment 0 are the untouched physical offsets. + let row_ids = batch + .column(batch.schema().index_of("_rowid").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(row_ids.values(), &[0, 1, 2, 3, 4, 5]); + } + + /// When the newest overlay covers every requested offset, an older overlay + /// in the same plan needs zero values and its value column must not be read + /// (the empty-input branch of `fetch_overlay_values`). The result still + /// resolves to the newest overlay. + #[rstest] + #[tokio::test] + async fn test_take_older_overlay_contributes_no_values( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Older covers {1, 4}; newer re-covers {1}. A take of only offset 1 + // routes entirely to the newer overlay, leaving the older one with no + // values to fetch even though it is part of the field's plan. + let dataset = commit_overlay( + dataset, + "older", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "newer", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([Some(999)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap(); + assert_eq!(col(&batch, "val").values(), &[999]); + } + + /// A newest overlay whose value is NULL must shadow an older overlay's + /// non-null value at the same offset — the merge resolves to NULL, it does + /// not fall back to the older overlay. + #[rstest] + #[tokio::test] + async fn test_take_newest_null_shadows_older( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "older", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([Some(111)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "newer_null", + 0, + &[1], + OverlayCoverage::dense(bitmap([1])), + vec![i32_array([None])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1], &full_schema(&dataset)).await.unwrap(); + let val = col(&batch, "val"); + assert!(val.is_null(0), "newest NULL must win over older 111"); + } + + /// Newest-wins is resolved independently per field across multiple sparse + /// overlays: for the same offset, `id` can resolve to one overlay while + /// `val` resolves to the other, depending on which overlay newly covers + /// that field at that offset. + #[rstest] + #[tokio::test] + async fn test_take_multi_sparse_per_field_newest_wins( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Older: id covers {3}, val covers {2}. + let dataset = commit_overlay( + dataset, + "older", + 0, + &[0, 1], + OverlayCoverage::sparse(vec![bitmap([3]), bitmap([2])]), + vec![i32_array([Some(7773)]), i32_array([Some(2772)])], + version, + ) + .await; + // Newer: id covers {2}, val covers {3} — the mirror image. + let dataset = commit_overlay( + dataset, + "newer", + 0, + &[0, 1], + OverlayCoverage::sparse(vec![bitmap([2]), bitmap([3])]), + vec![i32_array([Some(9992)]), i32_array([Some(9993)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[2, 3], &full_schema(&dataset)).await.unwrap(); + // id: offset 2 -> newer (9992), offset 3 -> older (7773). + assert_eq!(col(&batch, "id").values(), &[9992, 7773]); + // val: offset 2 -> older (2772), offset 3 -> newer (9993). + assert_eq!(col(&batch, "val").values(), &[2772, 9993]); + } + + /// A fragment with an overlay plan, but a take that touches only uncovered + /// offsets, must fall entirely through to the base values (the + /// `!routing.any_overlay` early-return with a plan present). + #[rstest] + #[tokio::test] + async fn test_take_plan_present_all_offsets_uncovered( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "ov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + // None of {0, 2, 5} are covered: the plan exists but contributes nothing. + let batch = frag.take(&[0, 2, 5], &full_schema(&dataset)).await.unwrap(); + assert_eq!(col(&batch, "val").values(), &[0, 20, 50]); + assert_eq!(col(&batch, "id").values(), &[0, 2, 5]); + } + + /// A dataset-level `take` spanning multiple fragments, each with its own + /// overlay, routes every global row index to the right fragment's overlay. + #[rstest] + #[tokio::test] + async fn test_dataset_take_multi_fragment_overlays( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "frag0", + 0, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(1000)])], + version, + ) + .await; + let dataset = commit_overlay( + dataset, + "frag1", + 1, + &[1], + OverlayCoverage::dense(bitmap([0])), + vec![i32_array([Some(6000)])], + version, + ) + .await; + + // Global rows 0 and 6 are the overlaid offset-0 rows of fragments 0 and + // 1; rows 1 and 7 fall through to base. + let batch = dataset + .take(&[0, 1, 6, 7], full_schema(&dataset)) + .await + .unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 1, 6, 7]); + assert_eq!(col(&batch, "val").values(), &[1000, 10, 6000, 70]); + } + + /// A scan whose read splits into multiple batches must slice + /// `offsets_in_frag` per batch correctly — the running `rows_seen` + /// accumulator in `merge_overlays` gives each batch its start. Every other + /// scan test uses single-batch fragments, so this is the only guard for the + /// cross-batch (`start > 0`) path. + #[rstest] + #[tokio::test] + async fn test_scan_multi_batch_overlay_slicing( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + use futures::TryStreamExt; + + // One fragment of 10 rows so the read can be chunked below. + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("val", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..10)), + Arc::new(Int32Array::from_iter_values((0..10).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 100, + max_rows_per_group: 100, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay one offset in each batch that batch_size 4 produces (batches + // [0,4), [4,8), [8,10)): offsets 1, 5, 9 with distinct values. A wrong + // per-batch slice would misalign these. + let dataset = commit_overlay( + dataset, + "multibatch", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 5, 9])), + vec![i32_array([Some(111), Some(555), Some(999)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let mut scanner = frag.scan(); + scanner.batch_size(4).project(&["val"]).unwrap(); + let batches: Vec = scanner + .try_into_stream() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + // Guard the guard: the read must actually span multiple batches, else + // this would not exercise the cross-batch slice at all. + assert!( + batches.len() > 1, + "expected a multi-batch scan, got {} batch(es)", + batches.len() + ); + + let merged = + arrow_select::concat::concat_batches(&batches[0].schema(), &batches).unwrap(); + let expected: Vec = (0..10) + .map(|i| match i { + 1 => 111, + 5 => 555, + 9 => 999, + other => other * 10, + }) + .collect(); + assert_eq!(col(&merged, "val").values(), &expected); + } + + /// An empty selection must not trip over the overlay path: the plan exists + /// but there are no offsets to route, so the result is an empty batch. + #[rstest] + #[tokio::test] + async fn test_take_empty_selection( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + let dataset = commit_overlay( + dataset, + "ov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![i32_array([Some(111), Some(444)])], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[], &full_schema(&dataset)).await.unwrap(); + assert_eq!(batch.num_rows(), 0); + } + + /// Overlays resolve variable-width columns end-to-end, not just fixed-width + /// ones: the value column is fetched through the real file reader (a + /// different value-pushdown path than the fixed-width case) and assembled. + #[rstest] + #[tokio::test] + async fn test_string_overlay_end_to_end( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + use arrow_array::StringArray; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("name", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e", "f"])), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay `name` at offsets {1, 4}, one of the values NULL. + let dataset = commit_overlay( + dataset, + "strov", + 0, + &[1], + OverlayCoverage::dense(bitmap([1, 4])), + vec![Arc::new(StringArray::from(vec![Some("B"), None])) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[0, 1, 4], &full_schema(&dataset)).await.unwrap(); + let name = batch + .column(batch.schema().index_of("name").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name.value(0), "a"); // falls through to base + assert_eq!(name.value(1), "B"); // overlay value + assert!(name.is_null(2)); // overlay NULL wins + } + + /// Projection pruning must do NO IO to overlay files whose fields are not + /// projected. Proven the same way as row-selection pruning: delete the + /// overlay's data file, then read projecting only the *unrelated* `id` + /// column — it must succeed (the `val` overlay file is never opened), while + /// projecting the overlaid `val` column then fails because its file is gone. + #[rstest] + #[tokio::test] + async fn test_projection_prunes_overlay_files_no_io( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let dataset = create_base_dataset(version).await; + // Overlay covers `val` (field 1) only. + let dataset = commit_overlay( + dataset, + "valov", + 0, + &[1], + OverlayCoverage::dense(bitmap([0, 1])), + vec![i32_array([Some(1000), Some(1010)])], + version, + ) + .await; + + // Delete the overlay's data file: opening it now fails. + dataset + .object_store + .delete(&Path::from("data/valov.lance")) + .await + .unwrap(); + + let frag = dataset.get_fragment(0).unwrap(); + let id_only = dataset.schema().project_by_ids(&[0], true); + let val_only = dataset.schema().project_by_ids(&[1], true); + + // Projecting only `id` must not open the `val` overlay file, so it + // succeeds and returns untouched base values. + let batch = frag.take(&[0, 1], &id_only).await.unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 1]); + // A scan projecting only `id` must likewise never touch the file. + let batch = frag + .scan() + .project(&["id"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(col(&batch, "id").values(), &[0, 1, 2, 3, 4, 5]); + + // Projecting the overlaid `val` column does need the file, so it fails + // with a not-found error naming the missing overlay file. + let err = frag.take(&[0], &val_only).await.unwrap_err(); + let message = format!("{err:?}"); + assert!( + err.is_not_found() && message.contains("valov.lance"), + "projecting the overlaid column should fail with a not-found error \ + for its missing file, got: {message}", + ); + } + + /// A top-level struct column resolves through overlays: the overlay stores + /// the struct's leaf columns (under V2_1 those are the only ids in + /// `data_file.fields`), and `plan_overlays` maps them back to the top-level + /// struct so the whole value is fetched and replaced as a unit. + #[rstest] + #[tokio::test] + async fn test_struct_overlay_end_to_end( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let struct_fields = Fields::from(vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("info", DataType::Struct(struct_fields.clone()), true), + ])); + let info = Arc::new(StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))), + ], + None, + )); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..6)), info], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay the whole `info` struct (top-level field id 1) at offset 2. + let overlay_info = Arc::new(StructArray::new( + struct_fields, + vec![ + Arc::new(Int32Array::from(vec![777])), + Arc::new(Int32Array::from(vec![888])), + ], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "structov", + 0, + &[1], + OverlayCoverage::dense(bitmap([2])), + vec![overlay_info], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let info = batch + .column(batch.schema().index_of("info").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + let x = info + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let y = info + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + // Offset 1 falls through to base {1, 100}; offset 2 takes the overlay. + assert_eq!(x.values(), &[1, 777]); + assert_eq!(y.values(), &[100, 888]); + } + + /// A top-level list column resolves through overlays the same way — the + /// overlay's leaf (item) id maps back to the top-level list, and the whole + /// list value at a covered offset is replaced. + #[rstest] + #[tokio::test] + async fn test_list_overlay_end_to_end( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + use arrow_array::ListArray; + use arrow_array::types::Int32Type; + + let item = Arc::new(ArrowField::new("item", DataType::Int32, true)); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("tags", DataType::List(item.clone()), true), + ])); + let base_tags = ListArray::from_iter_primitive::( + (0..6i32).map(|i| Some(vec![Some(i), Some(i * 10)])), + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(base_tags), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay `tags` (top-level field id 1) at offset 2 with a new list. + let overlay_tags = + ListArray::from_iter_primitive::(std::iter::once(Some(vec![ + Some(77), + Some(88), + Some(99), + ]))); + let dataset = commit_overlay( + dataset, + "listov", + 0, + &[1], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(overlay_tags) as ArrayRef], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let tags = batch + .column(batch.schema().index_of("tags").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); + let row1 = tags.value(0); + let row1 = row1.as_any().downcast_ref::().unwrap(); + let row2 = tags.value(1); + let row2 = row2.as_any().downcast_ref::().unwrap(); + // Offset 1 falls through to base [1, 10]; offset 2 takes the overlay. + assert_eq!(row1.values(), &[1, 10]); + assert_eq!(row2.values(), &[77, 88, 99]); + } + + /// A top-level Map column resolves as a single atomic field even though its + /// value spans two leaves (key and value): both leaf ids map back to the one + /// Map atomic field, and the whole map value at a covered offset is replaced. + /// Maps require + /// the 2.2+ file format, so this runs only at V2_2 (unlike the V2_0/V2_1 + /// parametrized tests). + #[tokio::test] + async fn test_map_overlay_end_to_end() { + use arrow_array::MapArray; + use arrow_array::builder::{Int32Builder, MapBuilder}; + + let version = LanceFileVersion::V2_2; + + // Base row i holds the single entry {i: i * 10}. + let mut builder = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new()); + for i in 0..6i32 { + builder.keys().append_value(i); + builder.values().append_value(i * 10); + builder.append(true).unwrap(); + } + let base_attrs = builder.finish(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("attrs", base_attrs.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(base_attrs), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Overlay `attrs` (top-level field id 1) at offset 2 with a two-entry map. + let mut ov = MapBuilder::new(None, Int32Builder::new(), Int32Builder::new()); + ov.keys().append_value(7); + ov.values().append_value(77); + ov.keys().append_value(8); + ov.values().append_value(88); + ov.append(true).unwrap(); + let overlay_attrs = ov.finish(); + let dataset = commit_overlay( + dataset, + "mapov", + 0, + &[1], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(overlay_attrs) as ArrayRef], + version, + ) + .await; - let batches = self - .take(&unique_indices, u32::MAX, take_priority) - .await? - .buffered(get_num_compute_intensive_cpus()) - .try_collect::>() - .await?; - let mut batch = concat_batches(&Arc::new(self.output_schema.clone()), batches.iter())?; + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let attrs = batch + .column(batch.schema().index_of("attrs").unwrap()) + .as_any() + .downcast_ref::() + .unwrap(); - if let Some(expand_map) = expand_map { - batch = arrow_select::take::take_record_batch(&batch, &expand_map)?; + let entries = |i: usize| -> (Vec, Vec) { + let row = attrs.value(i); + let keys = row.column(0).as_any().downcast_ref::().unwrap(); + let vals = row.column(1).as_any().downcast_ref::().unwrap(); + (keys.values().to_vec(), vals.values().to_vec()) + }; + // Offset 1 falls through to the base entry {1: 10}; offset 2 takes the + // overlay map {7: 77, 8: 88}. + assert_eq!(entries(0), (vec![1], vec![10])); + assert_eq!(entries(1), (vec![7, 8], vec![77, 88])); } - Ok(batch) - } -} + /// Base `id` + a struct `s { a, b }` (6 rows). Field ids: s=1, a=2, b=3. + async fn create_struct_dataset(version: LanceFileVersion) -> (Dataset, Fields) { + let s_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("s", DataType::Struct(s_fields.clone()), true), + ])); + let s = Arc::new(StructArray::new( + s_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))), + ], + None, + )); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..6)), s], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + (dataset, s_fields) + } -#[cfg(test)] -mod tests { - use arrow_arith::numeric::mul; - use arrow_array::{ - ArrayRef, BooleanArray, Int32Array, Int64Array, RecordBatchIterator, StringArray, - }; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::ROW_ID; - use lance_core::utils::tempfile::TempStrDir; - use lance_datagen::{RowCount, array, gen_batch}; - use lance_file::version::LanceFileVersion; - use lance_file::writer::FileWriterOptions; - use lance_io::{assert_io_eq, assert_io_lt, object_store::ObjectStore}; - use pretty_assertions::assert_eq; - use rstest::rstest; - use std::collections::HashMap; + fn struct_col<'a>(batch: &'a RecordBatch, name: &str) -> &'a StructArray { + batch + .column(batch.schema().index_of(name).unwrap()) + .as_any() + .downcast_ref::() + .unwrap() + } - use super::*; - use crate::{ - dataset::{ - InsertBuilder, - transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}, - }, - session::Session, - utils::test::TestDatasetGenerator, - }; + fn i32_child(s: &StructArray, i: usize) -> Int32Array { + s.column(i) + .as_any() + .downcast_ref::() + .unwrap() + .clone() + } - async fn create_dataset(test_uri: &str, data_storage_version: LanceFileVersion) -> Dataset { - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, true), - ArrowField::new("s", DataType::Utf8, true), - ])); + /// The reviewer's core case (r3553495147): an overlay stores only sub-field + /// `s.a`, but the read projects the whole struct `s`. The overlay must splice + /// into `a` and leave `b` untouched (previously this panicked because the merge + /// fetched the whole `s` from an overlay file holding only `a`). + #[rstest] + #[tokio::test] + async fn test_overlay_subfield_projecting_parent_struct( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let (dataset, _) = create_struct_dataset(version).await; + // Overlay ONLY `s.a` (field id 2) at offset 2. + let a_only = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]); + let overlay = Arc::new(StructArray::new( + a_only, + vec![Arc::new(Int32Array::from(vec![777]))], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "aov", + 0, + &[2], + OverlayCoverage::dense(bitmap([2])), + vec![overlay], + version, + ) + .await; - let batches: Vec = (0..10) - .map(|i| { - RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20)), - Arc::new(StringArray::from_iter_values( - (i * 20..(i + 1) * 20).map(|v| format!("s-{}", v)), - )), - ], - ) - .unwrap() - }) - .collect(); + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let s = struct_col(&batch, "s"); + // a: offset 1 base (1), offset 2 overlaid (777). + assert_eq!(i32_child(s, 0).values(), &[1, 777]); + // b: untouched base (100, 200). + assert_eq!(i32_child(s, 1).values(), &[100, 200]); + } - let write_params = WriteParams { - max_rows_per_file: 40, - max_rows_per_group: 10, - data_storage_version: Some(data_storage_version), - ..Default::default() - }; - let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); - Dataset::write(batches, test_uri, Some(write_params)) - .await - .unwrap(); + /// An overlay on a non-projected sibling leaf must be skipped and its file + /// never opened: overlay covers `s.b`, but the read projects only `s.a`. + #[rstest] + #[tokio::test] + async fn test_overlay_nonprojected_sibling_skipped( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let (dataset, _) = create_struct_dataset(version).await; + let b_only = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]); + let overlay = Arc::new(StructArray::new( + b_only, + vec![Arc::new(Int32Array::from(vec![888]))], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "bov", + 0, + &[3], + OverlayCoverage::dense(bitmap([2])), + vec![overlay], + version, + ) + .await; + // Delete the overlay file: if projecting only `s.a` opened it, this fails. + dataset + .object_store + .delete(&Path::from("data/bov.lance")) + .await + .unwrap(); - Dataset::open(test_uri).await.unwrap() - } + let frag = dataset.get_fragment(0).unwrap(); + let a_only = dataset.schema().project_by_ids(&[2], true); + let batch = frag.take(&[1, 2], &a_only).await.unwrap(); + let s = struct_col(&batch, "s"); + // Only `a` is projected, unchanged base values. + assert_eq!(i32_child(s, 0).values(), &[1, 2]); + } - async fn create_dataset_v2(test_uri: &str) -> Dataset { - let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( - "i", - DataType::Int32, - true, - )])); + /// Two overlays target different sub-fields of the same struct, and a third + /// re-overlays `s.a`. Each leaf resolves independently and newest wins on `a`. + #[rstest] + #[tokio::test] + async fn test_overlay_multiple_subfields_newest_wins( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let (dataset, _) = create_struct_dataset(version).await; + let a_field = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]); + let b_field = Fields::from(vec![ArrowField::new("b", DataType::Int32, true)]); + // Older: a := 700 at offset 2. + let dataset = commit_overlay( + dataset, + "a_old", + 0, + &[2], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(StructArray::new( + a_field.clone(), + vec![Arc::new(Int32Array::from(vec![700]))], + None, + )) as ArrayRef], + version, + ) + .await; + // b := 800 at offset 2. + let dataset = commit_overlay( + dataset, + "b_ov", + 0, + &[3], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(StructArray::new( + b_field, + vec![Arc::new(Int32Array::from(vec![800]))], + None, + )) as ArrayRef], + version, + ) + .await; + // Newest: a := 999 at offset 2 (shadows the older `a` overlay). + let dataset = commit_overlay( + dataset, + "a_new", + 0, + &[2], + OverlayCoverage::dense(bitmap([2])), + vec![Arc::new(StructArray::new( + a_field, + vec![Arc::new(Int32Array::from(vec![999]))], + None, + )) as ArrayRef], + version, + ) + .await; - let batches: Vec = (0..10) - .map(|i| { - RecordBatch::try_new( - schema.clone(), - vec![Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20))], - ) - .unwrap() - }) - .collect(); + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[2], &full_schema(&dataset)).await.unwrap(); + let s = struct_col(&batch, "s"); + assert_eq!(i32_child(s, 0).values(), &[999]); // newest `a` wins + assert_eq!(i32_child(s, 1).values(), &[800]); // `b` from its own overlay + } - let write_params = WriteParams { - max_rows_per_file: 40, - max_rows_per_group: 10, - data_storage_version: Some(LanceFileVersion::Stable), - ..Default::default() - }; - let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); - Dataset::write(batches, test_uri, Some(write_params)) - .await + /// Three levels of nesting: `outer { middle { a, b } }`. An overlay on the + /// deep leaf `outer.middle.a` splices correctly when the whole `outer` is read. + #[rstest] + #[tokio::test] + async fn test_overlay_deeply_nested_subfield( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, + ) { + let mid_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let outer_fields = Fields::from(vec![ArrowField::new( + "middle", + DataType::Struct(mid_fields.clone()), + true, + )]); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("outer", DataType::Struct(outer_fields.clone()), true), + ])); + // Field ids: outer=1, middle=2, a=3, b=4. + let middle = Arc::new(StructArray::new( + mid_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..6)), + Arc::new(Int32Array::from_iter_values((0..6).map(|v| v * 100))), + ], + None, + )); + let outer = Arc::new(StructArray::new(outer_fields, vec![middle], None)); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..6)), outer], + ) .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); - Dataset::open(test_uri).await.unwrap() + // Overlay the deep leaf `outer.middle.a` (field id 3) at offset 2. + let a_leaf = Fields::from(vec![ArrowField::new("a", DataType::Int32, true)]); + let mid_a = Fields::from(vec![ArrowField::new( + "middle", + DataType::Struct(a_leaf.clone()), + true, + )]); + let overlay = Arc::new(StructArray::new( + mid_a, + vec![Arc::new(StructArray::new( + a_leaf, + vec![Arc::new(Int32Array::from(vec![777]))], + None, + ))], + None, + )) as ArrayRef; + let dataset = commit_overlay( + dataset, + "deepov", + 0, + &[3], + OverlayCoverage::dense(bitmap([2])), + vec![overlay], + version, + ) + .await; + + let frag = dataset.get_fragment(0).unwrap(); + let batch = frag.take(&[1, 2], &full_schema(&dataset)).await.unwrap(); + let outer = struct_col(&batch, "outer"); + let middle = outer + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + // a: offset 1 base (1), offset 2 overlaid (777); b untouched. + assert_eq!(i32_child(middle, 0).values(), &[1, 777]); + assert_eq!(i32_child(middle, 1).values(), &[100, 200]); + + // Projecting the *intermediate* struct `outer.middle` (field id 2) while + // the overlay targets a deeper field (id 3) must still apply: the + // overlay's leaf id falls inside the projected subtree, so it maps to a + // projected atomic field. (This is the case wjones127/westonpace flagged where a + // top-level-only mapping would miss the overlay.) + let middle_only = dataset.schema().project_by_ids(&[2], true); + let batch = frag.take(&[2], &middle_only).await.unwrap(); + let middle = struct_col(&batch, "outer") + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(i32_child(middle, 0).values(), &[777]); + } } #[rstest] @@ -3182,7 +5513,7 @@ mod tests { updated_fragments: vec![u1.fragment], new_fragments: vec![], fields_modified: u1.fields_modified, - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(UpdateMode::RewriteColumns), inserted_rows_filter: None, @@ -3263,7 +5594,7 @@ mod tests { updated_fragments: vec![u2.fragment], new_fragments: vec![], fields_modified: u2.fields_modified, - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(UpdateMode::RewriteColumns), inserted_rows_filter: None, @@ -3447,19 +5778,20 @@ mod tests { dataset.delete("i >= 0 and i < 15").await.unwrap(); let fragment = &dataset.get_fragments()[0]; - let mut reader = fragment - .open( + let read_config = FragReadConfig::default().with_row_id(true); + + if data_storage_version == LanceFileVersion::Legacy { + let mut reader = crate::dataset::versions::open_v1_fragment_reader( + fragment, dataset.schema(), - FragReadConfig::default().with_row_id(true), + &read_config, ) .await .unwrap(); - reader.with_make_deletions_null(); - - if data_storage_version == LanceFileVersion::Legacy { + reader.with_make_deletions_null(); // The first batch is entirely deleted, deleted rows will be marked null with null row ids. let batch1 = reader - .legacy_read_batch_projected(0, .., dataset.schema()) + .read_batch_projected(0, .., dataset.schema()) .await .unwrap(); assert_eq!( @@ -3470,7 +5802,7 @@ mod tests { // The second batch is partially deleted, so the deleted rows will be // marked null with null row ids. let batch2 = reader - .legacy_read_batch_projected(1, .., dataset.schema()) + .read_batch_projected(1, .., dataset.schema()) .await .unwrap(); assert_eq!( @@ -3480,7 +5812,7 @@ mod tests { // The final batch is not deleted, so it will be returned as-is. let batch3 = reader - .legacy_read_batch_projected(2, .., dataset.schema()) + .read_batch_projected(2, .., dataset.schema()) .await .unwrap(); assert_eq!( @@ -3488,6 +5820,8 @@ mod tests { &UInt64Array::from_iter_values(20..30) ); } else { + let mut reader = fragment.open(dataset.schema(), read_config).await.unwrap(); + reader.with_make_deletions_null(); let to_batches = |range: Range| { let batch_size = range.len() as u32; let fut = reader.take_range(range, batch_size); @@ -3837,48 +6171,68 @@ mod tests { assert_eq!(dataset.count_rows(None).await.unwrap(), 195); } - let fragment = &mut dataset.get_fragment(0).unwrap(); - let mut updater = fragment.updater(Some(&["i"]), None, None).await.unwrap(); let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( "double_i", DataType::Int32, true, )])); - while let Some(batch) = updater.next().await.unwrap() { - let input_col = batch.column_by_name("i").unwrap(); - let result_col = mul(input_col, &Int32Array::new_scalar(2)).unwrap(); - let batch = RecordBatch::try_new( - new_schema.clone(), - vec![Arc::new(result_col) as ArrayRef], - ) - .unwrap(); - updater.update(batch).await.unwrap(); - } - let new_fragment = updater.finish().await.unwrap(); + // Merge keeps the fragment list intact, so every fragment gets the new + // column. Fragment 0 is the one carrying the deletions. + let fragment_ids = dataset + .manifest + .fragments + .iter() + .map(|f| f.id as usize) + .collect::>(); + let mut merged_fragments = Vec::new(); + for fragment_id in fragment_ids { + let fragment = &mut dataset.get_fragment(fragment_id).unwrap(); + let mut updater = fragment + .updater(Some(&["i"]), None, None, None) + .await + .unwrap(); + while let Some(batch) = updater.next().await.unwrap() { + let input_col = batch.column_by_name("i").unwrap(); + let result_col = mul(input_col, &Int32Array::new_scalar(2)).unwrap(); + let batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(result_col) as ArrayRef], + ) + .unwrap(); + updater.update(batch).await.unwrap(); + } + let new_fragment = updater.finish().await.unwrap(); - assert_eq!(new_fragment.files.len(), 2); + assert_eq!(new_fragment.files.len(), 2); + merged_fragments.push(new_fragment); + } // Scan again let mut full_schema = dataset.schema().merge(new_schema.as_ref()).unwrap(); full_schema.set_field_id(None); let before_version = dataset.version().version; - let op = Operation::Overwrite { - fragments: vec![new_fragment], + let op = Operation::Merge { + fragments: merged_fragments, schema: full_schema.clone(), - config_upsert_values: None, - initial_bases: None, + preserves_nullability: true, }; - let dataset = - Dataset::commit(test_uri, op, None, None, None, Default::default(), false) - .await - .unwrap(); + let dataset = Dataset::commit( + test_uri, + op, + Some(before_version), + None, + None, + Default::default(), + false, + ) + .await + .unwrap(); - // We only kept the first fragment of 40 rows assert_eq!( dataset.count_rows(None).await.unwrap(), - if with_delete { 35 } else { 40 } + if with_delete { 195 } else { 200 } ); assert_eq!(dataset.version().version, before_version + 1); dataset.validate().await.unwrap(); @@ -3920,6 +6274,82 @@ mod tests { } } + /// A deletion vector naming a row the fragment does not have leaves the restorer + /// with rows it can never account for, so `Updater::next` has to refuse at the end + /// of the stream rather than let a data file short of those rows be written. + /// + /// `write_deletions` rejects an over-long vector, so the file is written directly + /// to get a fragment into this state. + #[tokio::test] + async fn test_updater_rejects_deletion_vector_past_end_of_fragment() { + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + let mut dataset = create_dataset(test_uri, LanceFileVersion::Stable).await; + + // Point a fragment's deletion file at a row it does not have. 200 rows are + // spread over several 40-row fragments, so 10_000 is past the end of any of + // them. Pick a fragment whose id is not zero, so the assertion below cannot + // pass on a message that dropped the id entirely. + let deletion_vector: DeletionVector = [10_000].into_iter().collect(); + let fragment_index = 1; + let fragment_id = dataset.manifest.fragments[fragment_index].id; + assert_ne!(fragment_id, 0, "need a non-zero fragment id"); + let deletion_file = write_deletion_file( + &dataset.base, + fragment_id, + dataset.version().version, + &deletion_vector, + dataset.object_store.as_ref(), + ) + .await + .unwrap(); + let mut fragments = dataset.manifest.fragments.as_ref().clone(); + fragments[fragment_index].deletion_file = deletion_file; + let mut manifest = dataset.manifest.as_ref().clone(); + manifest.fragments = Arc::new(fragments); + dataset.manifest = Arc::new(manifest); + + let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "double_i", + DataType::Int32, + true, + )])); + let fragment = dataset.get_fragment(fragment_id as usize).unwrap(); + let mut updater = fragment + .updater(Some(&["i"]), None, None, None) + .await + .unwrap(); + + // Every live row is handed back, so the loop only ends when next() gives up. + let err = loop { + match updater.next().await { + Ok(Some(batch)) => { + let input_col = batch.column_by_name("i").unwrap(); + let result_col = mul(input_col, &Int32Array::new_scalar(2)).unwrap(); + let batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(result_col) as ArrayRef], + ) + .unwrap(); + updater.update(batch).await.unwrap(); + } + Ok(None) => panic!("expected next() to refuse the unaccounted-for row"), + Err(err) => break err, + } + }; + + assert!(matches!(err, Error::NotSupported { .. }), "{err:?}"); + let message = err.to_string(); + assert!( + message.contains("unaccounted for"), + "expected the stream-ended wording, got: {message}" + ); + assert!( + message.contains(&format!("fragment {fragment_id}")), + "message should name the fragment: {message}" + ); + } + #[rstest] #[tokio::test] async fn test_merge_fragment( @@ -4029,7 +6459,7 @@ mod tests { .unwrap(); let (object_store, base_path) = ObjectStore::from_uri(test_uri).await.unwrap(); - let file_reader = PreviousFileReader::try_new_with_fragment_id( + let file_reader = V1FileReader::try_new_with_fragment_id( &object_store, &base_path .clone() @@ -4092,7 +6522,7 @@ mod tests { let fragment = dataset.get_fragments().pop().unwrap(); // Write batch_s using add_columns - let mut updater = fragment.updater(Some(&["i"]), None, None).await?; + let mut updater = fragment.updater(Some(&["i"]), None, None, None).await?; updater.next().await?; updater.update(batch_s.clone()).await?; let frag = updater.finish().await?; @@ -4105,6 +6535,7 @@ mod tests { Operation::Merge { schema, fragments: vec![frag], + preserves_nullability: true, }, Some(dataset.manifest.version), None, @@ -4180,7 +6611,7 @@ mod tests { FragReadConfig::default().with_row_id(true), ) .await?; - let batch = reader.legacy_read_range_as_batch(0..20).await?; + let batch = reader.read_range_as_batch(0..20).await?; let expected_data = RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()])), @@ -4220,8 +6651,12 @@ mod tests { let store = ObjectStore::local(); let file_path = dataset.data_dir().join("some_file.lance"); let object_writer = store.create(&file_path).await.unwrap(); - let mut file_writer = - lance_file::writer::FileWriter::new_lazy(object_writer, FileWriterOptions::default()); + let mut file_writer = lance_file::versions::create_lazy_writer( + LanceFileVersion::Stable.resolve(), + object_writer, + FileWriterOptions::default(), + ) + .unwrap(); file_writer.write_batch(&new_data).await.unwrap(); file_writer.finish().await.unwrap(); @@ -4236,6 +6671,23 @@ mod tests { LanceFileVersion::Stable.resolve() ); + let mismatched_path = dataset.data_dir().join("mismatched_file.lance"); + let object_writer = store.create(&mismatched_path).await.unwrap(); + let mut mismatched_writer = lance_file::versions::create_lazy_writer( + lance_file::version::ConcreteFileVersion::V2_0, + object_writer, + FileWriterOptions::default(), + ) + .unwrap(); + mismatched_writer.write_batch(&new_data).await.unwrap(); + mismatched_writer.finish().await.unwrap(); + + let err = FileFragment::create_from_file("mismatched_file.lance", &dataset, 1, Some(128)) + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("File version mismatch")); + let op = Operation::Append { fragments: vec![frag], }; @@ -4359,6 +6811,59 @@ mod tests { ); } + #[test] + fn test_indexed_metadata_heuristic_counts_selected_physical_columns() { + let schema = Schema::try_from(&ArrowSchema::new(vec![ + ArrowField::new( + "s", + DataType::Struct( + vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ] + .into(), + ), + true, + ), + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])) + .unwrap(); + let data_file = DataFile { + path: "wide.lance".to_string(), + fields: Arc::from([0, 1, 2, 3, 4, 5]), + column_indices: Arc::from([-1, 0, 1, 2, 3, 4]), + file_major_version: 2, + file_minor_version: 1, + file_size_bytes: CachedFileSize::unknown(), + base_id: None, + }; + + let full_struct = file_versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &schema, + &["s"], + ) + .unwrap(); + assert_eq!(full_struct.column_indices.len(), 2); + let valid_column_count = data_file + .column_indices + .iter() + .filter(|column_index| **column_index >= 0) + .count(); + assert!(full_struct.column_indices.len().saturating_mul(4) >= valid_column_count); + + let partial_struct = file_versions::reader_projection_from_column_names( + ConcreteFileVersion::V2_1, + &schema, + &["s.x"], + ) + .unwrap(); + assert_eq!(partial_struct.column_indices.len(), 1); + assert!(partial_struct.column_indices.len().saturating_mul(4) < valid_column_count); + } + #[tokio::test] async fn test_iops_read_small() { // Create a file that has 8 columns. @@ -4420,4 +6925,80 @@ mod tests { assert_io_eq!(stats, read_iops, 1); assert_io_lt!(stats, read_bytes, 4096); } + + #[tokio::test] + async fn test_update_columns_with_json_extension_type() { + use arrow_array::UInt64Array; + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::ARROW_JSON_EXT_NAME; + use lance_core::ROW_ID; + use std::collections::HashMap; + + // Create a dataset with an Arrow JSON extension column + let test_dir = TempStrDir::default(); + let mut json_metadata = HashMap::new(); + json_metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int64, false), + ArrowField::new("name", DataType::Utf8, true), + ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])), + Arc::new(StringArray::from(vec![ + r#"{"x":1}"#, + r#"{"x":2}"#, + r#"{"x":3}"#, + r#"{"x":4}"#, + r#"{"x":5}"#, + ])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write(reader, test_dir.as_ref(), None) + .await + .unwrap(); + + // Build the right stream with Arrow JSON column (Utf8 + arrow.json extension) + // Only update rows with row_id 1 and 3 + let update_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new(ROW_ID, DataType::UInt64, false), + ArrowField::new("meta", DataType::Utf8, true).with_metadata(json_metadata), + ])); + let update_batch = RecordBatch::try_new( + update_schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![1, 3])), + Arc::new(StringArray::from(vec![ + r#"{"updated":true,"id":2}"#, + r#"{"updated":true,"id":4}"#, + ])), + ], + ) + .unwrap(); + let right_stream: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + + // Perform update_columns - this should NOT fail with type mismatch + // Previously this would error with: + // "It is not possible to interleave arrays of different data types (Utf8 and LargeBinary)" + let mut fragment = dataset.get_fragment(0).unwrap(); + let (updated_fragment, fields_modified) = fragment + .update_columns(right_stream, ROW_ID, ROW_ID) + .await + .unwrap(); + + // Verify the operation produced valid results + assert!(!fields_modified.is_empty()); + assert!(!updated_fragment.files.is_empty()); + } } diff --git a/rust/lance/src/dataset/fragment/session.rs b/rust/lance/src/dataset/fragment/session.rs index de50255bb72..77e4125bbc4 100644 --- a/rust/lance/src/dataset/fragment/session.rs +++ b/rust/lance/src/dataset/fragment/session.rs @@ -74,7 +74,7 @@ impl FragmentSession { if row_offsets.len() > 1 && FileFragment::row_ids_contiguous(row_offsets) { let range = (row_offsets[0] as usize)..(row_offsets[row_offsets.len() - 1] as usize + 1); - self.reader.legacy_read_range_as_batch(range).await + self.reader.read_range_as_batch(range).await } else { self.reader.take_as_batch(row_offsets, None).await } @@ -89,7 +89,7 @@ mod tests { use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use lance_core::ROW_ADDR; use lance_core::utils::tempfile::TempStrDir; - use lance_encoding::version::LanceFileVersion; + use lance_file::version::LanceFileVersion; use rstest::rstest; use std::sync::Arc; diff --git a/rust/lance/src/dataset/fragment/write.rs b/rust/lance/src/dataset/fragment/write.rs index a61f0e0c46a..b641ee16cc5 100644 --- a/rust/lance/src/dataset/fragment/write.rs +++ b/rust/lance/src/dataset/fragment/write.rs @@ -8,10 +8,13 @@ use lance_core::Error; use lance_core::datatypes::Schema; use lance_datafusion::chunker::{break_stream, chunk_stream}; use lance_datafusion::utils::StreamingWriteSource; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; +#[cfg(test)] use lance_file::version::LanceFileVersion; -use lance_file::writer::FileWriterOptions; +use lance_file::version::stable_file_version; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; +use lance_file::writer::FileWriter; use lance_io::object_store::ObjectStore; +use lance_io::traits::Writer; use lance_io::utils::CachedFileSize; use lance_table::format::{DataFile, Fragment}; use lance_table::io::manifest::ManifestDescribing; @@ -21,7 +24,8 @@ use uuid::Uuid; use crate::Result; use crate::dataset::builder::DatasetBuilder; -use crate::dataset::write::{do_write_fragments, validate_and_resolve_target_bases_with_primary}; +use crate::dataset::utils::SchemaAdapter; +use crate::dataset::write::validate_and_resolve_target_bases_with_primary; use crate::dataset::{DATA_DIR, Dataset, ReadParams, WriteMode, WriteParams}; /// Generates a filename optimized for S3 throughput using a UUID-based approach. @@ -106,7 +110,25 @@ impl<'a> FragmentCreateBuilder<'a> { id: Option, ) -> Result { let (stream, schema) = self.get_stream_and_schema(Box::new(source)).await?; - self.write_impl(stream, schema, id).await + // Convert Arrow JSON columns (`arrow.json`, stored as Utf8) into Lance JSON + // (`lance.json`, stored as JSONB-encoded LargeBinary) before writing. The + // multi-fragment and dataset write paths perform this through + // `versions::write_fragments_direct`; + // the single-fragment create path must do the same or the raw UTF-8 string bytes + // would be written into a column whose schema declares JSONB, corrupting reads. + let stream = SchemaAdapter::new(stream.schema()).to_physical_stream(stream); + let version = self + .write_params + .map(|params| params.storage_version_or_default()) + .unwrap_or_else(stable_file_version); + crate::dataset::versions::write_fragment( + version, + self, + stream, + schema, + id.unwrap_or_default(), + ) + .await } /// Write multi fragment which separated by max_rows_per_file. @@ -118,12 +140,16 @@ impl<'a> FragmentCreateBuilder<'a> { self.write_fragments_v2_impl(stream, schema).await } - async fn write_v2_impl( + pub(crate) async fn write_current_impl( &self, + create_writer: F, stream: SendableRecordBatchStream, schema: Schema, id: u64, - ) -> Result { + ) -> Result + where + F: FnOnce(Box, Schema, String) -> Result<(FileWriter, DataFile)>, + { let params = self.write_params.map(Cow::Borrowed).unwrap_or_default(); let progress = params.progress.as_ref(); @@ -140,18 +166,7 @@ impl<'a> FragmentCreateBuilder<'a> { let mut fragment = Fragment::new(id); let full_path = base_path.clone().join(DATA_DIR).join(filename.clone()); let obj_writer = object_store.create(&full_path).await?; - let mut writer = lance_file::writer::FileWriter::try_new( - obj_writer, - schema, - FileWriterOptions { - format_version: params.data_storage_version, - ..Default::default() - }, - )?; - - let (major, minor) = writer.version().to_numbers(); - - let data_file = DataFile::new_unstarted(filename, major, minor); + let (mut writer, data_file) = create_writer(obj_writer, schema, filename)?; fragment.files.push(data_file); progress.begin(&fragment).await?; @@ -203,7 +218,7 @@ impl<'a> FragmentCreateBuilder<'a> { Self::validate_schema(&schema, stream.schema().as_ref())?; - let version = params.data_storage_version.unwrap_or_default(); + let version = params.storage_version_or_default(); let needs_existing_dataset = params.target_base_names_or_paths.is_some() || params.target_bases.is_some() || params.target_all_bases.is_some() @@ -234,34 +249,28 @@ impl<'a> FragmentCreateBuilder<'a> { } else { None }; - do_write_fragments( + crate::dataset::versions::write_fragments_direct( + version, existing_dataset.as_ref(), object_store, &base_path, &schema, stream, params, - version, target_bases_info, + Vec::new(), + None, ) .await } - async fn write_impl( + pub(crate) async fn write_v1_impl( &self, stream: SendableRecordBatchStream, schema: Schema, - id: Option, + id: u64, ) -> Result { - let id = id.unwrap_or_default(); - let params = self.write_params.map(Cow::Borrowed).unwrap_or_default(); - - let storage_version = params.storage_version_or_default(); - - if storage_version != LanceFileVersion::Legacy { - return self.write_v2_impl(stream, schema, id).await; - } let progress = params.progress.as_ref(); Self::validate_schema(&schema, stream.schema().as_ref())?; @@ -275,7 +284,7 @@ impl<'a> FragmentCreateBuilder<'a> { let filename = format!("{}.lance", generate_random_filename()); let mut fragment = Fragment::with_file_legacy(id, &filename, &schema, None); let full_path = base_path.clone().join(DATA_DIR).join(filename.clone()); - let mut writer = PreviousFileWriter::::try_new( + let mut writer = V1FileWriter::::try_new( &object_store, &full_path, schema, @@ -317,30 +326,27 @@ impl<'a> FragmentCreateBuilder<'a> { } async fn existing_dataset_schema(&self) -> Result> { - let mut builder = DatasetBuilder::from_uri(self.dataset_uri); - let accessor = self - .write_params - .and_then(|p| p.store_params.as_ref()) - .and_then(|p| p.storage_options_accessor.clone()); - if let Some(accessor) = accessor { - builder = builder.with_storage_options_accessor(accessor); - } - match builder.load().await { - Ok(dataset) => { - // Use the schema from the dataset, because it has the correct - // field ids. - Ok(Some(dataset.schema().clone())) - } - Err(Error::DatasetNotFound { .. }) => { - // If the dataset does not exist, we can use the schema from - // the reader. - Ok(None) - } + let params = self.write_params.map(Cow::Borrowed).unwrap_or_default(); + match self.load_existing_dataset(¶ms).await { + // Use the schema from the dataset, because it has the correct + // field ids. + Ok(dataset) => Ok(Some(dataset.schema().clone())), + // If the dataset does not exist, we can use the schema from + // the reader. + Err(Error::DatasetNotFound { .. }) => Ok(None), Err(e) => Err(e), } } async fn existing_dataset(&self, params: &WriteParams) -> Result> { + match self.load_existing_dataset(params).await { + Ok(dataset) => Ok(Some(dataset)), + Err(Error::DatasetNotFound { .. } | Error::NotFound { .. }) => Ok(None), + Err(e) => Err(e), + } + } + + async fn load_existing_dataset(&self, params: &WriteParams) -> Result { let mut builder = DatasetBuilder::from_uri(self.dataset_uri).with_read_params(ReadParams { store_options: params.store_params.clone(), commit_handler: params.commit_handler.clone(), @@ -352,11 +358,7 @@ impl<'a> FragmentCreateBuilder<'a> { builder = builder.with_base_store_params(base_path, store_params.clone()); } } - match builder.load().await { - Ok(dataset) => Ok(Some(dataset)), - Err(Error::DatasetNotFound { .. } | Error::NotFound { .. }) => Ok(None), - Err(e) => Err(e), - } + builder.load().await } fn validate_schema(expected: &Schema, actual: &ArrowSchema) -> Result<()> { @@ -375,7 +377,7 @@ mod tests { use std::sync::Arc; use arrow_array::{ - Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, + Int64Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, record_batch, }; use arrow_schema::{DataType, Field as ArrowField}; use lance_arrow::SchemaExt; @@ -415,7 +417,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. } if source.to_string().contains("Cannot write with an empty schema.")), "{:?}", - &result + result ); // Writing empty reader produces an error @@ -429,7 +431,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. } if source.to_string().contains("Input data was empty.")), "{:?}", - &result + result ); // Writing with incorrect schema produces an error. @@ -447,7 +449,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. } if difference.contains("fields did not match")), "{:?}", - &result + result ); } @@ -492,6 +494,52 @@ mod tests { assert_eq!(fragment.files[0].column_indices.as_ref(), &[0, 1]); } + #[tokio::test] + async fn test_fragment_create_with_session() { + let session = Arc::new(crate::session::Session::new( + 0, + 1024 * 1024, + Default::default(), + )); + let write_params = WriteParams { + session: Some(session.clone()), + ..Default::default() + }; + // Keep the dataset alive: the registry only holds weak references, so + // the in-memory store survives through the dataset's strong reference. + let initial_batch = + record_batch!(("a", Int64, [1, 2, 3]), ("b", Utf8, ["a", "b", "c"])).unwrap(); + let mut dataset = InsertBuilder::new("memory://") + .with_params(&write_params) + .execute(vec![initial_batch]) + .await + .unwrap(); + // Drop a column so the surviving field id is non-trivial (!= 0). + dataset.drop_columns(&["a"]).await.unwrap(); + let field_id = dataset.schema().field("b").unwrap().id; + assert_ne!(field_id, 0); + + let append_batch = record_batch!(("b", Utf8, ["d", "e"])).unwrap(); + let append_data = + RecordBatchIterator::new([Ok(append_batch.clone())], append_batch.schema()); + + let append_params = WriteParams { + session: Some(session.clone()), + mode: WriteMode::Append, + ..Default::default() + }; + let fragment = FragmentCreateBuilder::new(dataset.uri()) + .write_params(&append_params) + .write(append_data, None) + .await + .unwrap(); + + assert_eq!(fragment.files[0].fields.as_ref(), &[field_id]); + // The manifest load for schema inference went through the shared + // session's metadata cache. + assert!(session.metadata_cache_stats().await.num_entries > 0); + } + #[tokio::test] async fn test_write_fragments_validation() { // Writing with empty schema produces an error @@ -506,7 +554,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::InvalidInput { source, .. } if source.to_string().contains("Cannot write with an empty schema.")), "{:?}", - &result + result ); // Writing empty reader produces an error @@ -533,7 +581,7 @@ mod tests { matches!(result.as_ref().unwrap_err(), Error::SchemaMismatch { difference, .. } if difference.contains("fields did not match")), "{:?}", - &result + result ); } @@ -685,7 +733,7 @@ mod tests { assert!(!fragment.files.is_empty()); fragment.files.iter().for_each(|f| { - let (major_version, minor_version) = file_version.to_numbers(); + let (major_version, minor_version) = file_version.resolve().to_data_file_numbers(); assert_eq!(f.file_major_version, major_version); assert_eq!(f.file_minor_version, minor_version); }) @@ -717,7 +765,7 @@ mod tests { assert!(!fragment.is_empty()); fragment[0].files.iter().for_each(|f| { - let (major_version, minor_version) = file_version.to_numbers(); + let (major_version, minor_version) = file_version.resolve().to_data_file_numbers(); assert_eq!(f.file_major_version, major_version); assert_eq!(f.file_minor_version, minor_version); }) diff --git a/rust/lance/src/dataset/hash_joiner.rs b/rust/lance/src/dataset/hash_joiner.rs index 92b67599134..d5410d9ae3a 100644 --- a/rust/lance/src/dataset/hash_joiner.rs +++ b/rust/lance/src/dataset/hash_joiner.rs @@ -194,21 +194,11 @@ impl HashJoiner { } pub fn check_lance_support_null(array: &ArrayRef, dataset: &Dataset) -> Result<()> { - if array.null_count() > 0 && !dataset.lance_supports_nulls(array.data_type()) { - return Err(Error::invalid_input(format!( - "Join produced null values for type: {:?}, but storing \ - nulls for this data type is not supported by the \ - dataset's current Lance file format version: {:?}. This \ - can be caused by an explicit null in the new data.", - array.data_type(), - dataset - .manifest() - .data_storage_format - .lance_file_version() - .unwrap() - ))); - } - Ok(()) + super::versions::validate_nulls( + dataset.manifest().data_storage_format.lance_file_format(), + array.data_type(), + array.null_count() > 0, + ) } /// Collecting the data using the index column from left table, @@ -307,6 +297,48 @@ mod tests { Dataset::open(&uri).await.unwrap() } + #[test] + fn test_null_validation_is_selected_by_exact_version() { + use lance_file::version::ConcreteFileVersion; + + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V1, + &DataType::Int32, + true, + ) + .is_err() + ); + assert!( + super::super::versions::validate_nulls(ConcreteFileVersion::V1, &DataType::Utf8, true,) + .is_ok() + ); + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V2_0, + &DataType::Struct(arrow_schema::Fields::empty()), + true, + ) + .is_err() + ); + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V2_1, + &DataType::Struct(arrow_schema::Fields::empty()), + true, + ) + .is_ok() + ); + assert!( + super::super::versions::validate_nulls( + ConcreteFileVersion::V1, + &DataType::Int32, + false, + ) + .is_ok() + ); + } + #[tokio::test] async fn test_joiner_collect() { let schema = Arc::new(Schema::new(vec![ diff --git a/rust/lance/src/dataset/index.rs b/rust/lance/src/dataset/index.rs index 7cac7815125..a3e0917d432 100644 --- a/rust/lance/src/dataset/index.rs +++ b/rust/lance/src/dataset/index.rs @@ -16,8 +16,8 @@ use crate::index::scalar::infer_scalar_index_details; use arrow_schema::DataType; use async_trait::async_trait; use lance_core::{Error, Result}; -use lance_encoding::version::LanceFileVersion; -use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; +use lance_file::version::ConcreteFileVersion; +use lance_index::is_system_index; use lance_index::pb::VectorIndexDetails; use lance_index::scalar::lance_format::LanceIndexStore; use lance_table::format::IndexMetadata; @@ -25,23 +25,45 @@ use serde::{Deserialize, Serialize}; use uuid::Uuid; use super::optimize::{IndexRemapper, IndexRemapperOptions}; +use super::versions; #[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct DatasetIndexRemapperOptions {} +/// Loads index metadata when compaction has at least one index to remap. +/// +/// Returns all usable index metadata, including system indices, so the remapper +/// uses a consistent snapshot. Returns `None` when there are no usable +/// non-system indices. +pub(crate) async fn load_indices_for_remapping( + dataset: &Dataset, +) -> Result>>> { + if dataset.manifest.index_section.is_none() { + return Ok(None); + } + + let indices = dataset.load_indices().await?; + let has_remappable_index = indices.iter().any(|index| !is_system_index(index)); + Ok(has_remappable_index.then_some(indices)) +} + +#[async_trait] impl IndexRemapperOptions for DatasetIndexRemapperOptions { - fn create_remapper( - &self, - dataset: &Dataset, - ) -> crate::Result> { - Ok(Box::new(DatasetIndexRemapper { + async fn create_remapper(&self, dataset: &Dataset) -> Result>> { + let Some(indices) = load_indices_for_remapping(dataset).await? else { + return Ok(None); + }; + + Ok(Some(Box::new(DatasetIndexRemapper { dataset: Arc::new(dataset.clone()), - })) + indices, + }))) } } struct DatasetIndexRemapper { dataset: Arc, + indices: Arc>, } impl DatasetIndexRemapper { @@ -62,10 +84,9 @@ impl IndexRemapper for DatasetIndexRemapper { affected_fragment_ids: &[u64], ) -> Result> { let affected_frag_ids = HashSet::::from_iter(affected_fragment_ids.iter().copied()); - let indices = self.dataset.load_indices().await?; - let mut remapped = Vec::with_capacity(indices.len()); - for index in indices.iter() { - let needs_remapped = index.name != FRAG_REUSE_INDEX_NAME + let mut remapped = Vec::with_capacity(self.indices.len()); + for index in self.indices.iter() { + let needs_remapped = !is_system_index(index) && match &index.fragment_bitmap { None => true, Some(fragment_bitmap) => fragment_bitmap @@ -73,19 +94,31 @@ impl IndexRemapper for DatasetIndexRemapper { .any(|frag_idx| affected_frag_ids.contains(&(frag_idx as u64))), }; if needs_remapped { - let remap_result = self.remap_index(index, &mapping).await?; + // Box the remap future at the call site: inlining `remap_index` into this + // loop's async layout otherwise exceeds rustc's depth limit. It has to be + // boxed here, not inside `remap_index` — boxing internally turns the + // future's `Send` check into a `Box: Send` trait obligation that + // overflows the solver through the cache types (E0275 downstream). + let remap_result = Box::pin(self.remap_index(index, &mapping)).await?; match remap_result { RemapResult::Drop => continue, RemapResult::Keep(id) => { let index_details = match &index.index_details { Some(index_details) => index_details.as_ref().clone(), None => { - // Migration path, if we didn't store details before then use the default - // details. - assert!(index.fields.len() == 1); - let field = index.fields.first().unwrap(); + // Migration path, if we didn't store details before then use the + // default details. This only supports a single keyed field, not a + // composite index. + let Some(field) = index.keyed_field() else { + return Err(Error::index(format!( + "Index {} has fields {:?} (carried fields {:?}); the \ + legacy index-details migration path only supports a \ + single keyed field", + index.uuid, index.fields, index.covering_fields + ))); + }; let field = - self.dataset.schema().field_by_id(*field).ok_or_else(|| { + self.dataset.schema().field_by_id(field).ok_or_else(|| { Error::internal(format!( "Index {} references field {} which does not exist", index.uuid, field @@ -133,18 +166,12 @@ pub trait LanceIndexStoreExt { Self: Sized; } -/// Extract the lance file version from a dataset, floored at V2_0. +/// Select the exact file version used for index files in this dataset version. /// /// Index files should never use the legacy format. If the dataset uses legacy -/// format or doesn't have a version set, V2_0 is used as the minimum. -pub(crate) fn dataset_format_version(dataset: &Dataset) -> LanceFileVersion { - dataset - .manifest - .data_storage_format - .lance_file_version() - .ok() - .map(|v| v.resolve().max(LanceFileVersion::V2_0)) - .unwrap_or(LanceFileVersion::V2_0) +/// format, V2_0 is selected explicitly by the dataset composition table. +pub(crate) fn dataset_format_version(dataset: &Dataset) -> ConcreteFileVersion { + versions::index_file_version(dataset.manifest.data_storage_format.lance_file_format()) } #[async_trait] @@ -178,13 +205,107 @@ impl LanceIndexStoreExt for LanceIndexStore { mod tests { use super::*; use crate::dataset::WriteParams; - use crate::index::DatasetIndexExt; + use crate::dataset::transaction::{Operation, Transaction}; + use crate::index::frag_reuse::build_frag_reuse_index_metadata; use crate::index::vector::VectorIndexParams; + use crate::index::{DatasetIndexExt, IntoIndexSegment}; use lance_datagen::{BatchCount, RowCount, array}; use lance_index::IndexType; + use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndexDetails}; + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use lance_linalg::distance::MetricType; + use std::collections::HashMap; use uuid::Uuid; + #[tokio::test] + async fn test_remapper_not_created_without_remappable_indices() { + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + let options = DatasetIndexRemapperOptions::default(); + + assert!(options.create_remapper(&dataset).await.unwrap().is_none()); + + let frag_reuse_index = build_frag_reuse_index_metadata( + &dataset, + None, + FragReuseIndexDetails { + versions: Vec::new(), + }, + Default::default(), + ) + .await + .unwrap(); + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![frag_reuse_index], + removed_indices: Vec::new(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].name, FRAG_REUSE_INDEX_NAME); + assert!(options.create_remapper(&dataset).await.unwrap().is_none()); + } + + #[tokio::test] + async fn test_remapper_not_created_for_unknown_index_type() { + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_string()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + let current = dataset.load_indices().await.unwrap(); + let unknown = IndexMetadata { + index_details: Some(Arc::new(prost_types::Any { + type_url: "type.googleapis.com/example.ForeignIndexDetails".to_string(), + value: Vec::new(), + })), + fragment_bitmap: None, + ..current[0].clone() + }; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![unknown], + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + assert!(dataset.load_indices().await.unwrap().is_empty()); + assert!( + DatasetIndexRemapperOptions::default() + .create_remapper(&dataset) + .await + .unwrap() + .is_none(), + "compaction must not migrate an index type this build cannot open" + ); + } + #[tokio::test] async fn test_remapper_only_touches_segments_with_affected_fragments() { let test_dir = tempfile::tempdir().unwrap(); @@ -257,20 +378,10 @@ mod tests { let segments = segments .iter() .map(|segment| { - crate::index::IndexSegment::new( - segment.uuid, - segment - .fragment_bitmap - .as_ref() - .expect("test segment metadata should have fragment coverage") - .iter(), - segment - .index_details - .as_ref() - .expect("test segment metadata should have index details") - .clone(), - segment.index_version, - ) + segment + .clone() + .into_index_segment() + .expect("test segment metadata should convert to an index segment") }) .collect::>(); @@ -296,7 +407,9 @@ mod tests { let remapper = DatasetIndexRemapperOptions::default() .create_remapper(&dataset) - .unwrap(); + .await + .unwrap() + .expect("vector index should require a remapper"); let remapped = remapper .remap_indices(RowAddrRemap::empty(), &[target_fragments[0].id() as u64]) .await @@ -307,4 +420,154 @@ mod tests { assert_ne!(remapped[0].old_id, unaffected_segment_id); assert_ne!(remapped[0].new_id, unaffected_segment_id); } + + /// A covered index must be withdrawn from remapping rather than remapped. + /// No index type carries the declared payload through a remap, so a + /// replacement would republish a covering claim its storage does not back. + /// Withdrawal also means the legacy `index_details: None` migration path is + /// never reached for such an index, so it cannot panic there either. + #[tokio::test] + async fn test_remapper_migration_path_withdraws_covered_index() { + let reader = lance_datagen::gen_batch() + .col("a", array::step::()) + .col("b", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + dataset + .create_index( + &["a"], + IndexType::BTree, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + let a_id = dataset.schema().field("a").unwrap().id; + let b_id = dataset.schema().field("b").unwrap().id; + let current = dataset.load_indices().await.unwrap(); + let mut legacy_covered = current[0].clone(); + legacy_covered.fields = vec![a_id, b_id]; + legacy_covered.covering_fields = vec![b_id]; + // Force the legacy migration path this fix touches. + legacy_covered.index_details = None; + + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![legacy_covered], + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let index_uuid = dataset.load_indices().await.unwrap()[0].uuid; + + // Fully delete every row so `remap_index` returns `RemapResult::Keep`, + // landing in the `index_details: None` migration branch this fix touches. + let remap_to_empty = (0..dataset.count_all_rows().await.unwrap()) + .map(|i| (i as u64, None)) + .collect::>(); + let remapper = DatasetIndexRemapperOptions::default() + .create_remapper(&dataset) + .await + .unwrap() + .expect("a real index should require a remapper"); + let remapped = remapper + .remap_indices(RowAddrRemap::direct(remap_to_empty), &[0]) + .await + .unwrap(); + + // Withdrawn, not remapped: no index type carries the declared payload + // through a remap, so producing a replacement would republish a covering + // claim its storage does not back. The original entry stays in the + // manifest and simply stops covering the rewritten fragments. + assert!( + remapped.is_empty(), + "a covered index must be withdrawn from remapping, got {remapped:?}" + ); + let _ = index_uuid; + } + + /// The same migration path must reject -- cleanly, not with a panic -- a + /// malformed `covering_fields` longer than `fields`. + /// + /// Note this is a genuinely synthetic scenario, not one a real commit can + /// produce: `crate::index::remap_index`'s own `keyed > 1` guard (a sibling + /// fix in this same phase) already rejects every organically-committable + /// composite index *before* this migration path ever runs, since it is + /// only reached from that function's `RemapResult::Keep` arm. And + /// `validate_covering_fields` rejects a fully-consumed `covering_fields` + /// (`keyed == 0` with non-empty `fields`) at `Operation::CreateIndex` + /// commit time, and `TryFrom` rejects it again when a + /// manifest is decoded. The only way left to reach this branch's rejection + /// is metadata that passed through neither, which is how this test reaches + /// it: seeding the index cache directly rather than committing through a + /// `Transaction`. + #[tokio::test] + async fn test_remapper_migration_path_rejects_malformed_covering_fields() { + let reader = lance_datagen::gen_batch() + .col("a", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + dataset + .create_index( + &["a"], + IndexType::BTree, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + let index_uuid = dataset.load_indices().await.unwrap()[0].uuid; + let mut indices = dataset.load_indices().await.unwrap().as_ref().clone(); + for idx in &mut indices { + if idx.uuid == index_uuid { + // Force the legacy migration path, and malform `covering_fields` + // to be longer than `fields` -- more carried fields than fields + // at all. No normal commit can produce this. + idx.index_details = None; + idx.covering_fields = idx + .fields + .iter() + .copied() + .chain(std::iter::once(999)) + .collect(); + } + } + let metadata_key = crate::session::index_caches::IndexMetadataKey { + version: dataset.version().version, + store_identity: &dataset.object_store.store_prefix, + e_tag: dataset.manifest_location.e_tag.as_deref(), + }; + dataset + .index_cache + .insert_with_key(&metadata_key, Arc::new(indices)) + .await; + + let remap_to_empty = (0..dataset.count_all_rows().await.unwrap()) + .map(|i| (i as u64, None)) + .collect::>(); + let remapper = DatasetIndexRemapperOptions::default() + .create_remapper(&dataset) + .await + .unwrap() + .expect("a real index should require a remapper"); + let error = remapper + .remap_indices(RowAddrRemap::direct(remap_to_empty), &[0]) + .await + .unwrap_err(); + assert!( + error.to_string().contains("are not among its fields"), + "malformed covering must fail closed via the validator, not be \ + withdrawn as an ordinary covered index; got: {error}" + ); + } } diff --git a/rust/lance/src/dataset/index/frag_reuse.rs b/rust/lance/src/dataset/index/frag_reuse.rs index ceebe456bbf..8c1795aa8ad 100644 --- a/rust/lance/src/dataset/index/frag_reuse.rs +++ b/rust/lance/src/dataset/index/frag_reuse.rs @@ -16,15 +16,34 @@ use roaring::RoaringBitmap; /// If all the indices currently available are already caught up to as a specific reuse version, /// all older reuse versions (inclusive) can be cleaned up. /// -/// An index is considered caught up against a specific reuse version if -/// 1. the index is created after or at the same dataset version as the reuse version -/// 2. there is no old fragment in the version that is covered by the index and can be remapped. -/// If an index's fragment bitmap is missing, we will consider it as caught up. -/// Otherwise, we will never be able to clean up the reuse version. +/// An index is considered caught up against a specific reuse version if either: +/// 1. its coverage is disjoint from the fragments the reuse chain touches, so it +/// holds nothing the FRI would remap (the common multi-index case: a +/// compaction rewrote a sibling index's fragments, not this one); or +/// 2. it is at or past the reuse version's dataset version and no old fragment +/// in the version is still in its bitmap. A missing bitmap counts as caught +/// up, else the version could never be cleaned up. /// /// Note that there could be a race condition that an index is being added during the cleanup, /// This will make that specific index not efficient until the next reindex, /// but it will not cause any correctness problem. +/// +/// Typically run after [`compact_files`] with deferred remap and per-index +/// [`remap_column_index`] have caught the indexes up. +/// +/// # Example +/// +/// ```no_run +/// # use lance::dataset::index::frag_reuse::cleanup_frag_reuse_index; +/// # async fn example(dataset: &mut lance::Dataset) -> lance::Result<()> { +/// // Trim the fragment-reuse index to the versions still needed by some index. +/// cleanup_frag_reuse_index(dataset).await?; +/// # Ok(()) +/// # } +/// ``` +/// +/// [`compact_files`]: crate::dataset::optimize::compact_files +/// [`remap_column_index`]: crate::dataset::optimize::remapping::remap_column_index pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Result<()> { // check against index metadata before auto-remap let indices = read_manifest_indexes( @@ -42,12 +61,14 @@ pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Resu .await .unwrap(); + let chain_frag_bitmap = reuse_chain_frag_bitmap(&frag_reuse_details.versions); + let mut retained_versions = Vec::new(); let mut fragment_bitmaps = RoaringBitmap::new(); for version in frag_reuse_details.versions.iter() { let check_results = indices .iter() - .map(|idx| is_index_remap_caught_up(version, idx)) + .map(|idx| is_index_remap_caught_up(version, idx, &chain_frag_bitmap)) .collect::>(); if check_results @@ -97,14 +118,38 @@ pub async fn cleanup_frag_reuse_index(dataset: &mut Dataset) -> lance_core::Resu Ok(()) } +/// Every fragment the reuse chain touches (old + new) across all versions. An +/// index disjoint from this set holds no row address the FRI remaps, so trimming +/// can never strand it (fragment ids are never reused). +fn reuse_chain_frag_bitmap(versions: &[FragReuseVersion]) -> RoaringBitmap { + let mut bitmap = RoaringBitmap::new(); + for version in versions { + bitmap.extend(version.old_frag_ids().iter().map(|&id| id as u32)); + bitmap.extend(version.new_frag_ids().iter().map(|&id| id as u32)); + } + bitmap +} + fn is_index_remap_caught_up( frag_reuse_version: &FragReuseVersion, index_meta: &IndexMetadata, + chain_frag_bitmap: &RoaringBitmap, ) -> lance_core::Result { if is_system_index(index_meta) { return Ok(true); } + // Disjoint coverage => caught up regardless of dataset_version, bypassing the + // stale-version gate below (see fn docs). The chain includes NEW fragments + // deliberately: a deferred-remap commit advances a covering index's bitmap + // onto them before its data is remapped, so an old-frag-only check would + // clear a still-stale index and trim a version it needs. + if let Some(index_frag_bitmap) = &index_meta.fragment_bitmap + && index_frag_bitmap.is_disjoint(chain_frag_bitmap) + { + return Ok(true); + } + if index_meta.dataset_version < frag_reuse_version.dataset_version { return Ok(false); } @@ -157,6 +202,106 @@ mod tests { use lance_index::IndexType; use lance_index::scalar::ScalarIndexParams; + fn frag_digest(id: u64) -> lance_index::frag_reuse::FragDigest { + lance_index::frag_reuse::FragDigest { + id, + physical_rows: 100, + num_deleted_rows: 0, + } + } + + fn reuse_version(dataset_version: u64, old: &[u64], new: &[u64]) -> FragReuseVersion { + FragReuseVersion { + dataset_version, + groups: vec![lance_index::frag_reuse::FragReuseGroup { + changed_row_addrs: Vec::new(), + old_frags: old.iter().copied().map(frag_digest).collect(), + new_frags: new.iter().copied().map(frag_digest).collect(), + }], + } + } + + fn index_covering(dataset_version: u64, covered: &[u32]) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![0], + covering_fields: vec![], + name: "test_idx".into(), + dataset_version, + fragment_bitmap: Some(RoaringBitmap::from_iter(covered.iter().copied())), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + /// The catch-up determination must not pin the FRI on an index that is + /// simply unrelated to the compaction, while still retaining versions that a + /// covering-but-not-yet-remapped index needs. + #[test] + fn test_caught_up_uses_fragment_coverage_not_only_version() { + // A reuse version at dataset_version 10 rewrote fragments [4, 5] -> [6]. + let version = reuse_version(10, &[4, 5], &[6]); + let chain = reuse_chain_frag_bitmap(std::slice::from_ref(&version)); + + // Non-covering, stale version: touches none of the rewritten frags, so + // caught up despite version 5 < 10 (the case the old gate got wrong). + assert_true!( + is_index_remap_caught_up(&version, &index_covering(5, &[1, 2, 3]), &chain).unwrap() + ); + + // Still holds an old fragment: not caught up. + assert_false!( + is_index_remap_caught_up(&version, &index_covering(5, &[1, 4, 5]), &chain).unwrap() + ); + + // Bitmap advanced onto the new fragment but data not yet remapped: not + // caught up (why the chain must include new frags). + assert_false!( + is_index_remap_caught_up(&version, &index_covering(5, &[1, 6]), &chain).unwrap() + ); + + // Once remapped (version advanced): caught up. + assert_true!( + is_index_remap_caught_up(&version, &index_covering(11, &[1, 6]), &chain).unwrap() + ); + } + + /// The chain spans every reuse version, not just the one being checked: a + /// stale index touching only a *later* version's fragment must still fall to + /// the version gate (a per-version chain would wrongly clear it). + #[test] + fn test_caught_up_uses_whole_reuse_chain() { + let v1 = reuse_version(10, &[4, 5], &[6]); // 4,5 -> 6 + let v2 = reuse_version(11, &[6], &[7]); // 6 -> 7 + let chain = reuse_chain_frag_bitmap(&[v1.clone(), v2]); + + // Stale index (version 5) covering only v2's new fragment [7]: not + // disjoint from the chain, so not caught up on v1. + assert_false!(is_index_remap_caught_up(&v1, &index_covering(5, &[1, 7]), &chain).unwrap()); + } + + /// Whole-fragment removal (every row deleted, no replacement): an index + /// emptied by the deletion has an empty bitmap and must count as caught up -- + /// it holds only dead rows -- else its stale version pins the removed-fragment + /// version forever (remap hits the drop-everything path, never advancing it). + #[test] + fn test_caught_up_handles_fragment_removal() { + // Reuse version 20 removed fragment [7] outright (no replacement). + let version = reuse_version(20, &[7], &[]); + let chain = reuse_chain_frag_bitmap(std::slice::from_ref(&version)); + + // Index emptied by the deletion (empty bitmap): caught up. + assert_true!(is_index_remap_caught_up(&version, &index_covering(5, &[]), &chain).unwrap()); + + // Bitmap still lists the removed fragment (not yet updated): retained. + assert_false!( + is_index_remap_caught_up(&version, &index_covering(5, &[7]), &chain).unwrap() + ); + } + #[tokio::test] async fn test_cleanup_frag_reuse_index() { let mut dataset = lance_datagen::gen_batch() @@ -209,7 +354,12 @@ mod tests { let scalar_index = indices.iter().find(|idx| idx.name == "scalar").unwrap(); // Should not be considered caught up because index was created at an old dataset version assert_false!( - is_index_remap_caught_up(&frag_reuse_details.versions[0], scalar_index).unwrap() + is_index_remap_caught_up( + &frag_reuse_details.versions[0], + scalar_index, + &reuse_chain_frag_bitmap(&frag_reuse_details.versions), + ) + .unwrap() ); // Remap and check index is caught up @@ -219,7 +369,12 @@ mod tests { let indices = dataset.load_indices().await.unwrap(); let scalar_index = indices.iter().find(|idx| idx.name == "scalar").unwrap(); assert_true!( - is_index_remap_caught_up(&frag_reuse_details.versions[0], scalar_index).unwrap() + is_index_remap_caught_up( + &frag_reuse_details.versions[0], + scalar_index, + &reuse_chain_frag_bitmap(&frag_reuse_details.versions), + ) + .unwrap() ); // Cleanup frag reuse index and check there is no reuse version @@ -313,7 +468,12 @@ mod tests { .find(|idx| idx.name == format!("{col}_idx")) .unwrap(); assert!( - is_index_remap_caught_up(&frag_reuse_details.versions[0], index).unwrap(), + is_index_remap_caught_up( + &frag_reuse_details.versions[0], + index, + &reuse_chain_frag_bitmap(&frag_reuse_details.versions), + ) + .unwrap(), "index {col}_idx was not caught up after remap" ); } diff --git a/rust/lance/src/dataset/mem_wal.rs b/rust/lance/src/dataset/mem_wal.rs index f5b89d06ff4..79c7fac8272 100644 --- a/rust/lance/src/dataset/mem_wal.rs +++ b/rust/lance/src/dataset/mem_wal.rs @@ -37,6 +37,7 @@ mod hnsw; pub mod index; mod manifest; pub mod memtable; +pub mod observer; pub mod scanner; pub mod sharding; #[cfg(test)] @@ -52,19 +53,19 @@ use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; /// Column name for the mem_wal tombstone (delete sentinel) marker. /// /// `_tombstone` is a *physical* column present only in mem_wal memtables and -/// flushed generations — it is deliberately kept out of the base table (hard +/// SSTables — it is deliberately kept out of the base table (hard /// delete), so it is **not** a virtual [`is_system_column`](lance_core::is_system_column). /// A row with `_tombstone = true` is a delete sentinel: the newest value for /// its primary key, carrying null in every non-PK column, that wins /// newest-per-PK resolution and is then silently dropped from query results. /// -/// The column is owned end-to-end by lance: callers pass the base schema and +/// The column is owned end-to-end by lance: callers pass the logical schema and /// lance injects the column on the write path ([`write::ShardWriter::put`] / /// [`write::ShardWriter::delete`]), so no caller ever constructs or names it. pub const TOMBSTONE: &str = "_tombstone"; -/// The mem_wal tombstone field appended to the base schema to form the -/// memtable/generation schema. +/// The mem_wal tombstone field appended to the logical schema on the way to the +/// storage schema. /// /// Non-nullable: the write path always populates it (`false` for normal rows, /// `true` for tombstones). Non-nullability also lets the point-lookup base arm @@ -74,8 +75,46 @@ pub fn tombstone_field() -> ArrowField { ArrowField::new(TOMBSTONE, DataType::Boolean, false) } -/// Extend a base schema with the trailing `_tombstone` column to form the -/// mem_wal memtable/generation schema. +/// Derive a shard's *storage* schema from its *logical* (base table) schema by +/// widening every top-level field to nullable except the primary key and +/// `_tombstone`. +/// +/// A tombstone carries the primary key and null everywhere else, so storage +/// must permit a null wherever the base table does not. The logical schema +/// stays the caller's contract — validated at [`write::ShardWriter::put`], +/// restored at the scan's egress. +/// +/// Top-level only: Arrow validates nullability only there, so a vector column's +/// item field is untouched. The primary key is excluded because +/// [`lance_core::datatypes::Schema`] requires it non-nullable, `_tombstone` +/// because the write path always populates it. Idempotent. +pub fn relax_non_pk_nullability( + logical_schema: &ArrowSchema, + pk_columns: &[String], +) -> Arc { + let fields: Vec = logical_schema + .fields() + .iter() + .map(|field| { + let keep = field.is_nullable() + || field.name() == TOMBSTONE + || pk_columns.iter().any(|c| c == field.name()); + let field = field.as_ref().clone(); + if keep { + field + } else { + field.with_nullable(true) + } + }) + .collect(); + Arc::new(ArrowSchema::new_with_metadata( + fields, + logical_schema.metadata().clone(), + )) +} + +/// Extend the logical schema with the trailing `_tombstone` column — the +/// intermediate [`relax_non_pk_nullability`] widens into the storage schema. /// /// Idempotent: a schema that already carries `_tombstone` (a reopen/replay /// path) is returned unchanged. Schema-level metadata and per-field metadata @@ -92,7 +131,8 @@ pub fn schema_with_tombstone(base: &ArrowSchema) -> Arc { )) } -pub use api::{DatasetMemWalExt, InitializeMemWalBuilder}; +pub use api::{DatasetMemWalExt, InitializeMemWalBuilder, validate_maintained_indexes}; +pub use index::{MemIndexKind, MemTableVisibility}; pub use manifest::ShardManifestStore; pub use memtable::scanner::MemTableScanner; pub use scanner::{LsmDataSource, LsmGeneration, LsmScanner, ShardSnapshot}; @@ -101,6 +141,98 @@ pub use sharding::{ evaluate_sharding_spec_with_source_columns, }; pub use wal::{BatchDurableWatcher, WalAppendResult, WalAppender, WalReadEntry, WalTailer}; +pub use write::SealFence; pub use write::ShardWriter; pub use write::ShardWriterConfig; pub use write::WriteResult; + +#[cfg(test)] +mod tests { + use super::*; + use arrow_schema::Fields; + + fn logical() -> ArrowSchema { + ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("count", DataType::Int64, false), + ArrowField::new("note", DataType::Utf8, true), + ]) + } + + #[test] + fn relax_widens_every_non_pk_field_and_leaves_the_key_alone() { + let relaxed = relax_non_pk_nullability(&logical(), &["id".to_string()]); + + assert!( + !relaxed.field(0).is_nullable(), + "the primary key stays strict" + ); + assert!( + relaxed.field(1).is_nullable(), + "`count` must accept a tombstone null" + ); + assert!( + relaxed.field(2).is_nullable(), + "already-nullable is untouched" + ); + } + + #[test] + fn relax_leaves_nested_fields_exactly_as_declared() { + // Arrow validates nullability only at the top level, and a vector + // column's item field must not gain a validity layer. + let item = Arc::new(ArrowField::new("item", DataType::Float32, false)); + let child = ArrowField::new("a", DataType::Int32, false); + let schema = ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("vector", DataType::FixedSizeList(item, 4), false), + ArrowField::new("s", DataType::Struct(Fields::from(vec![child])), false), + ]); + + let relaxed = relax_non_pk_nullability(&schema, &["id".to_string()]); + + assert!(relaxed.field(1).is_nullable()); + match relaxed.field(1).data_type() { + DataType::FixedSizeList(f, _) => assert!(!f.is_nullable(), "item field untouched"), + other => panic!("expected FixedSizeList, got {other:?}"), + } + match relaxed.field(2).data_type() { + DataType::Struct(fields) => assert!(!fields[0].is_nullable(), "child field untouched"), + other => panic!("expected Struct, got {other:?}"), + } + } + + #[test] + fn relax_keeps_tombstone_non_nullable_and_is_idempotent() { + let pk = ["id".to_string()]; + let once = relax_non_pk_nullability(&schema_with_tombstone(&logical()), &pk); + let twice = relax_non_pk_nullability(&once, &pk); + + let tombstone = once.field_with_name(TOMBSTONE).unwrap(); + assert!( + !tombstone.is_nullable(), + "the write path always populates _tombstone" + ); + assert_eq!(once, twice); + } + + #[test] + fn relax_preserves_schema_and_field_metadata() { + // The `lance-schema:unenforced-primary-key` marker rides on field + // metadata, so losing it here would silently drop the shard's PK. + let marked = ArrowField::new("count", DataType::Int64, false) + .with_metadata([("k".to_string(), "v".to_string())].into()); + let schema = ArrowSchema::new_with_metadata( + vec![ArrowField::new("id", DataType::Int32, false), marked], + [("s".to_string(), "m".to_string())].into(), + ); + + let relaxed = relax_non_pk_nullability(&schema, &["id".to_string()]); + + assert_eq!(relaxed.metadata().get("s").map(String::as_str), Some("m")); + assert_eq!( + relaxed.field(1).metadata().get("k").map(String::as_str), + Some("v") + ); + } +} diff --git a/rust/lance/src/dataset/mem_wal/api.rs b/rust/lance/src/dataset/mem_wal/api.rs index 597c65dce83..30efea72057 100644 --- a/rust/lance/src/dataset/mem_wal/api.rs +++ b/rust/lance/src/dataset/mem_wal/api.rs @@ -5,16 +5,24 @@ //! //! This module provides the user-facing API for initializing and using MemWAL //! on a Dataset. +//! +//! # Limitations +//! +//! MemWAL does not track dataset changes made after it is initialized: dropping +//! or replacing a maintained index, or projecting away its column, leaves +//! `maintained_indexes` naming something the writer cannot build. A change that +//! races the initialization commit lands the same way. Both surface as a failing +//! `mem_wal_writer`; handling them is follow-up work. use std::collections::HashMap; use std::sync::Arc; -use arrow_schema::DataType; +use arrow_schema::{DataType, Schema as ArrowSchema}; use async_trait::async_trait; +use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndexDetails, ShardingField, ShardingSpec}; use lance_index::vector::hnsw::builder::HnswBuildParams; -use lance_io::object_store::ObjectStore; use uuid::Uuid; use crate::Dataset; @@ -25,8 +33,11 @@ use crate::index::DatasetIndexInternalExt; use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta}; use super::ShardWriterConfig; -use super::scanner::flushed_cache::open_flushed_dataset; +use super::index::{MemIndexKind, unsupported_index_type, validate_index_configs}; +use super::scanner::sstable_cache::open_sstable; use super::scanner::{DatasetCache, ShardSnapshot}; +use super::schema_with_tombstone; +use super::util::derived_store_params; use super::write::MemIndexConfig; use super::write::ShardWriter; @@ -143,9 +154,9 @@ impl<'a> InitializeMemWalBuilder<'a> { /// Set the base-table indexes to maintain in MemTables, replacing any /// previously set list. /// - /// Each name must reference an index that already exists on the dataset. - /// The primary key btree, when present, is maintained implicitly and must - /// not be listed. + /// Each name must reference an existing index the MemWAL can maintain; + /// [`execute`](Self::execute) enforces both. The primary key btree, when + /// present, is maintained implicitly and must not be listed. pub fn maintained_indexes(mut self, indexes: I) -> Self where I: IntoIterator, @@ -187,8 +198,12 @@ impl<'a> InitializeMemWalBuilder<'a> { /// Initialize MemWAL on the dataset, committing the MemWAL system index. /// - /// Fails if any maintained index does not exist, if the selected sharding - /// configuration is invalid, or if MemWAL is already initialized. + /// Fails if any maintained index does not exist or cannot be maintained by + /// the MemWAL, if the selected sharding configuration is invalid, or if + /// MemWAL is already initialized. + /// + /// Validated against the dataset as it stands here; see the module-level + /// limitations for changes made afterwards. pub async fn execute(self) -> Result<()> { let Self { dataset, @@ -200,21 +215,19 @@ impl<'a> InitializeMemWalBuilder<'a> { // Resolve (and validate) the sharding choice before any I/O. let (sharding_specs, num_shards) = resolve_sharding(dataset, sharding)?; + dataset.schema().verify_primary_key()?; + let indices = dataset.load_indices().await?; - for index_name in &maintained_indexes { - if !indices.iter().any(|idx| &idx.name == index_name) { - return Err(Error::invalid_input(format!( - "Index '{}' not found on dataset. maintained_indexes must reference existing indexes.", - index_name - ))); - } - } if indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { return Err(Error::invalid_input( "MemWAL is already initialized on this dataset.", )); } + // Gate the commit, not just a preflight a caller may skip: a set the + // writer cannot open leaves the table unwritable. + validate_maintained_indexes(dataset, &maintained_indexes).await?; + let details = MemWalIndexDetails { num_shards, sharding_specs, @@ -392,10 +405,6 @@ fn writer_config_to_defaults(config: &ShardWriterConfig) -> HashMap HashMap`s) and prewarms each of its /// indexes. Opens run concurrently. @@ -597,7 +598,9 @@ impl DatasetMemWalExt for Dataset { cache: Option<&Arc>, ) -> Result<()> { let session = self.session(); - // Resolve flushed paths exactly as the LSM collector does, so the + // Every open below targets a generation URI, never the base's own. + let store_params = self.store_params().map(derived_store_params); + // Resolve SSTable paths exactly as the LSM collector does, so the // session/cache entries we warm key-match the paths later lookups open. let base_path = self.uri().trim_end_matches('/').to_string(); let opens = snapshots @@ -606,11 +609,13 @@ impl DatasetMemWalExt for Dataset { let shard_id = snapshot.shard_id; let base_path = &base_path; let session = &session; - snapshot.flushed_generations.iter().map(move |flushed| { - let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, flushed.path); + let store_params = &store_params; + snapshot.sstables.iter().map(move |sstable| { + let path = format!("{}/_mem_wal/{}/{}", base_path, shard_id, sstable.path); async move { let dataset = - open_flushed_dataset(&path, Some(session), cache, None).await?; + open_sstable(&path, Some(session), store_params.as_ref(), cache, None) + .await?; prewarm_all_indexes(&dataset).await } }) @@ -640,69 +645,23 @@ impl DatasetMemWalExt for Dataset { // Get maintained_indexes from the MemWalIndex details let maintained_indexes = &mem_wal_index.details.maintained_indexes; - // Load index configs for each maintained index - let mut index_configs = Vec::new(); - for index_name in maintained_indexes { - // A maintained index can split into multiple physical segments - // (e.g. `optimize_indices(append)` deltas), which the singular - // `load_index_by_name` rejects. Every segment carries the same - // type and params, so take the first match. - let index_meta = self - .load_indices_by_name(index_name) - .await? - .into_iter() - .next() - .ok_or_else(|| { - Error::invalid_input(format!( - "Index '{}' from maintained_indexes not found on dataset", - index_name - )) - })?; - - // Detect index type and create appropriate config - let type_url = index_meta - .index_details - .as_ref() - .map(|d| d.type_url.as_str()) - .unwrap_or(""); - - let index_type = MemIndexConfig::detect_index_type(type_url)?; - - match index_type { - "btree" => { - index_configs.push(MemIndexConfig::btree_from_metadata( - &index_meta, - self.schema(), - )?); - } - "fts" => { - index_configs.push(MemIndexConfig::fts_from_metadata( - &index_meta, - self.schema(), - )?); - } - "vector" => { - let hnsw_params = config.hnsw_params.get(index_name).cloned(); - let vector_config = - load_vector_index_config(self, index_name, &index_meta, hnsw_params) - .await?; - index_configs.push(vector_config); - } - _ => { - return Err(Error::invalid_input(format!( - "Unknown index type: {}", - index_type - ))); - } - }; - } + let index_configs = + build_index_configs(self, maintained_indexes, &config.hnsw_params).await?; // Set shard_id in config config.shard_id = shard_id; - // Get object store and base path + // Inject the dataset's store params + session so the flusher opens the + // base + generations with the same store the base was resolved with. + config.store_params = self.store_params().cloned(); + config.session = Some(self.session()); + + // Reuse the dataset's own object store + base path; `ObjectStore::from_uri` + // would discard the store params the dataset was opened with, signing WAL + // writes with the ambient identity. Mirrors `list_mem_wal_latest_shard_ids`. let base_uri = self.uri(); - let (store, base_path) = ObjectStore::from_uri(base_uri).await?; + let store = self.object_store(None).await?; + let base_path = self.branch_location().path; // Create ShardWriter ShardWriter::open( @@ -717,6 +676,95 @@ impl DatasetMemWalExt for Dataset { } } +/// Build the in-memory index configurations for `index_names`. +/// +/// Shared by [`DatasetMemWalExt::mem_wal_writer`] and +/// [`validate_maintained_indexes`], so a set that validates is one the writer +/// can build. +async fn build_index_configs( + dataset: &Dataset, + index_names: &[String], + hnsw_params: &HashMap, +) -> Result> { + let mut index_configs = Vec::with_capacity(index_names.len()); + for index_name in index_names { + // A maintained index can split into multiple physical segments + // (e.g. `optimize_indices(append)` deltas), which the singular + // `load_index_by_name` rejects. Every segment carries the same + // type and params, so take the first match. + let index_meta = dataset + .load_indices_by_name(index_name) + .await? + .into_iter() + .next() + .ok_or_else(|| { + Error::invalid_input(format!( + "Index '{}' from maintained_indexes not found on dataset", + index_name + )) + })?; + + // Detect index kind and create appropriate config + let type_url = index_meta + .index_details + .as_ref() + .map(|d| d.type_url.as_str()) + .unwrap_or(""); + + let kind = MemIndexKind::from_type_url(type_url) + .ok_or_else(|| unsupported_index_type(index_name, type_url))?; + + // Exhaustive: a new kind must be built here, or a maintained set could + // name an index this writer cannot open, failing every memtable claim. + index_configs.push(match kind { + MemIndexKind::BTree => { + MemIndexConfig::btree_from_metadata(&index_meta, dataset.schema())? + } + MemIndexKind::Fts => MemIndexConfig::fts_from_metadata(&index_meta, dataset.schema())?, + MemIndexKind::Hnsw => { + let hnsw_params = hnsw_params.get(index_name).cloned(); + load_vector_index_config(dataset, index_name, &index_meta, hnsw_params).await? + } + }); + } + Ok(index_configs) +} + +/// Whether the MemWAL can maintain `index_names` on `dataset`. +/// +/// Applies the same rules [`ShardWriter::open`] does, so a set that passes here +/// is a set the writer can open. [`InitializeMemWalBuilder::execute`] runs it +/// before committing; it is public so a caller inferring a set can ask the same +/// question first. A type url alone cannot decide this — every vector sub-type +/// maps to [`MemIndexKind::Hnsw`], but the memtable's HNSW needs a +/// `FixedSizeList` column. +/// +/// All-or-nothing: it reports the first index it cannot maintain rather than +/// returning a usable subset, so a caller inferring a set surfaces the error +/// instead of dropping an index it believes is maintained. +/// +/// Judges `dataset` as given; see the module-level limitations. +/// +/// Opens each vector index to inherit its distance type. +pub async fn validate_maintained_indexes(dataset: &Dataset, index_names: &[String]) -> Result<()> { + // Validation reads an index's name, column, and field id, never its HNSW + // tuning, so the writer's build params are not needed here. + let index_configs = build_index_configs(dataset, index_names, &HashMap::new()).await?; + + // The shard schema is base + `_tombstone`, as `ShardWriter::open` extends + // it; field ids and the primary key resolve against that, not the base. + let base_schema: ArrowSchema = dataset.schema().into(); + let schema = schema_with_tombstone(&base_schema); + let lance_schema = LanceSchema::try_from(schema.as_ref())?; + let pk_columns: Vec = lance_schema + .unenforced_primary_key() + .iter() + .map(|field| field.name.clone()) + .collect(); + + validate_index_configs(&index_configs, schema.as_ref(), &lance_schema, &pk_columns) +} + /// Build an in-memory HNSW vector index configuration from a base-table /// vector index entry. /// @@ -771,7 +819,7 @@ async fn load_vector_index_config( #[cfg(test)] mod tests { - use super::super::scanner::FlushedMemTableCache; + use super::super::scanner::SsTableCache; use super::*; use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator}; @@ -788,6 +836,76 @@ mod tests { ])) } + /// A dataset of 256 rows with an IVF vector index `vector_idx` over a + /// `FixedSizeList` column. + async fn dataset_with_vector_index(uri: &str, item_type: DataType) -> Dataset { + use crate::index::vector::VectorIndexParams; + use arrow_array::ArrayRef; + use arrow_array::builder::{FixedSizeListBuilder, Float32Builder, Float64Builder}; + use lance_linalg::distance::DistanceType; + + const ROWS: i32 = 256; + const DIM: i32 = 4; + + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", item_type.clone(), true)), DIM), + true, + ), + ])); + + let vectors: ArrayRef = match item_type { + DataType::Float32 => { + let mut builder = FixedSizeListBuilder::new(Float32Builder::new(), DIM); + for row in 0..ROWS { + for d in 0..DIM { + builder.values().append_value((row * DIM + d) as f32); + } + builder.append(true); + } + Arc::new(builder.finish()) + } + DataType::Float64 => { + let mut builder = FixedSizeListBuilder::new(Float64Builder::new(), DIM); + for row in 0..ROWS { + for d in 0..DIM { + builder.values().append_value((row * DIM + d) as f64); + } + builder.append(true); + } + Arc::new(builder.finish()) + } + other => panic!("unhandled vector item type {other:?}"), + }; + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from((0..ROWS).collect::>())), + vectors, + ], + ) + .unwrap(); + + let reader = RecordBatchIterator::new([Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, Some(WriteParams::default())) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(1, DistanceType::L2), + true, + ) + .await + .unwrap(); + dataset + } + fn id_v_batch(schema: &Arc, ids: &[i32]) -> RecordBatch { let vs: Vec = ids.iter().map(|i| i * 10).collect(); RecordBatch::try_new( @@ -800,11 +918,183 @@ mod tests { .unwrap() } + #[tokio::test] + async fn test_validate_maintained_indexes_rejects_non_f32_vector_column() { + // A `FixedSizeList` vector index is a valid durable index whose + // type url resolves to `Hnsw` like any other, but the memtable HNSW needs + // Float32 — so committing it would leave the table unwritable. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let dataset = dataset_with_vector_index(&uri, DataType::Float64).await; + + let index_meta = dataset + .load_indices_by_name("vector_idx") + .await + .unwrap() + .into_iter() + .next() + .unwrap(); + assert_eq!( + MemIndexKind::from_type_url( + index_meta.index_details.as_ref().unwrap().type_url.as_str() + ), + Some(MemIndexKind::Hnsw), + "the type url cannot see the column type" + ); + + let error = validate_maintained_indexes(&dataset, &["vector_idx".to_string()]) + .await + .expect_err("a Float64 vector column is not maintainable"); + assert!( + error.to_string().contains("FixedSizeList"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_accepts_f32_vector_column() { + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let dataset = dataset_with_vector_index(&uri, DataType::Float32).await; + + validate_maintained_indexes(&dataset, &["vector_idx".to_string()]) + .await + .expect("a Float32 vector column is maintainable"); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_accepts_btree() { + // Guards the shard-schema plumbing: validation resolves field ids against + // base + `_tombstone`, so a scalar index on an ordinary column must pass. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = + RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1, 2, 3]))], schema.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + validate_maintained_indexes(&dataset, &["id_idx".to_string()]) + .await + .expect("a BTree index on an Int32 column is maintainable"); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_rejects_unmaintainable_kind() { + // A bitmap index is a valid durable index the memtable cannot build. + // The error names it, so a caller validating a set knows which to drop. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = + RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1, 2, 3]))], schema.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + dataset + .create_index( + &["v"], + IndexType::Bitmap, + Some("v_bitmap".to_string()), + &ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::Bitmap), + true, + ) + .await + .unwrap(); + + let error = validate_maintained_indexes(&dataset, &["v_bitmap".to_string()]) + .await + .expect_err("the memtable cannot build a bitmap index"); + assert!( + error.to_string().contains("v_bitmap"), + "the error must name the index: {error}" + ); + } + + #[tokio::test] + async fn test_validate_maintained_indexes_rejects_unknown_name() { + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1]))], schema.clone()); + let dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + + let error = validate_maintained_indexes(&dataset, &["nope".to_string()]) + .await + .expect_err("an index that does not exist cannot be maintained"); + assert!( + error.to_string().contains("not found"), + "unexpected: {error}" + ); + } + + #[tokio::test] + async fn test_initialize_mem_wal_rejects_unmaintainable_index() { + // Initialization persists the set, so it must apply the writer's rules + // itself: a Float64 vector index committed here leaves the table unwritable. + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let mut dataset = dataset_with_vector_index(&uri, DataType::Float64).await; + + let error = dataset + .initialize_mem_wal() + .unsharded() + .maintained_indexes(["vector_idx"]) + .execute() + .await + .expect_err("a Float64 vector column is not maintainable"); + assert!( + error.to_string().contains("FixedSizeList"), + "unexpected error: {error}" + ); + assert!( + dataset.mem_wal_index_details().await.unwrap().is_none(), + "a rejected maintained set must not be committed" + ); + } + + #[tokio::test] + async fn test_initialize_mem_wal_rejects_unknown_index_name() { + let tmp = tempfile::tempdir().unwrap(); + let uri = format!("{}/base", tmp.path().to_str().unwrap()); + let schema = id_v_schema(); + let reader = RecordBatchIterator::new([Ok(id_v_batch(&schema, &[1]))], schema.clone()); + let mut dataset = Dataset::write(reader, &uri, Some(WriteParams::default())) + .await + .unwrap(); + + let error = dataset + .initialize_mem_wal() + .unsharded() + .maintained_indexes(["nope"]) + .execute() + .await + .expect_err("maintained_indexes must reference existing indexes"); + assert!( + error.to_string().contains("nope") && error.to_string().contains("not found"), + "unexpected error: {error}" + ); + assert!(dataset.mem_wal_index_details().await.unwrap().is_none()); + } + #[tokio::test] async fn test_prewarm_mem_wal_opens_and_warms_indexes() { - // `prewarm_mem_wal` opens each flushed generation (into the base + // `prewarm_mem_wal` opens each SSTable (into the base // dataset's session + the supplied cache) and warms its indexes. We - // place a flushed-generation dataset with a BTree index at the + // place an SSTable dataset with a BTree index at the // canonical `{base}/_mem_wal/{shard}/{folder}` path, prewarm it via a // snapshot, and assert the generation is cached and its index loadable. let tmp = tempfile::tempdir().unwrap(); @@ -817,7 +1107,7 @@ mod tests { .await .unwrap(); - // Flushed generation with a BTree index on `id`. + // SSTable with a BTree index on `id`. let shard_id = Uuid::new_v4(); let folder = "deadbeef_gen_1"; let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, folder); @@ -839,9 +1129,9 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, folder.to_string()); + .with_sstable(1, folder.to_string()); - let cache: Arc = Arc::new(FlushedMemTableCache::new(4)); + let cache: Arc = Arc::new(SsTableCache::new(4)); base.prewarm_mem_wal(std::slice::from_ref(&snapshot), Some(&cache)) .await .expect("prewarm must open the generation and warm its index"); @@ -849,7 +1139,7 @@ mod tests { // The generation is resident in the cache (same session), with its // index loadable — a later lookup that opens this path is a pure hit. let warmed = cache - .get_or_open(&gen_uri, Some(base.session())) + .get_or_open(&gen_uri, Some(base.session()), base.store_params().cloned()) .await .unwrap(); assert_eq!(warmed.load_indices().await.unwrap().len(), 1); @@ -857,7 +1147,7 @@ mod tests { #[tokio::test] async fn test_prewarm_mem_wal_empty_is_noop() { - // No snapshots / no flushed generations: prewarm is a clean no-op. + // No snapshots / no SSTables: prewarm is a clean no-op. let tmp = tempfile::tempdir().unwrap(); let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); let schema = id_v_schema(); diff --git a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs index 4009557702c..a48058d02b5 100644 --- a/rust/lance/src/dataset/mem_wal/hnsw/graph.rs +++ b/rust/lance/src/dataset/mem_wal/hnsw/graph.rs @@ -231,6 +231,17 @@ impl LevelLinks { } } + /// Heap bytes for one level, counting `published` at its full width even + /// while empty — the build fills it, and the caller budgets against a + /// ceiling where over-counting is the safe direction. + fn allocated_bytes(max_neighbors: usize) -> usize { + // Arc>: strong + weak refcounts, the Vec header, then the ids. + let published = 2 * std::mem::size_of::() + + std::mem::size_of::>() + + max_neighbors * std::mem::size_of::(); + published + max_neighbors * std::mem::size_of::() + } + fn publish_from_ranked(&self, ranked: &[ScoredPoint]) { self.published.store(Arc::new( ranked.iter().map(|point| point.id).collect::>(), @@ -259,6 +270,16 @@ impl Node { } } + /// Heap bytes held by a node of `target_level`, excluding the `Node` itself + /// (which lives inline in the graph's node arena). + fn allocated_bytes(target_level: u16, m: usize) -> usize { + let levels = target_level as usize + 1; + levels * std::mem::size_of::() + + (0..=target_level) + .map(|level| LevelLinks::allocated_bytes(max_neighbors(m, level))) + .sum::() + } + fn has_level(&self, level: u16) -> bool { (level as usize) < self.levels.len() } @@ -292,6 +313,12 @@ pub struct HnswGraph { visible_len: AtomicUsize, visited_pool: ArrayQueue, packed_level0: ArcSwap, + /// Heap bytes of the node arena and visited pool. Fixed at construction: + /// both are sized from `capacity`, not from `len()`. + base_bytes: usize, + /// Heap bytes of the current `packed_level0` snapshot, which is rebuilt + /// wholesale on each level-0 publish rather than grown. + packed_bytes: AtomicUsize, } impl HnswGraph { @@ -309,16 +336,18 @@ impl HnswGraph { let mut rng = SmallRng::seed_from_u64(params.seed); let mut nodes = Vec::with_capacity(capacity); + let mut node_bytes = 0; for id in 0..capacity { let target_level = if id == 0 { 0 } else { random_level(¶ms, &mut rng) }; + node_bytes += Node::allocated_bytes(target_level, params.m); nodes.push(Node::new(target_level, params.m)); } - let pool_size = rayon::current_num_threads().max(1) * 2; + let pool_size = visited_pool_size(); let visited_pool = ArrayQueue::new(pool_size); for _ in 0..pool_size { let _ = visited_pool.push(VisitedList::new(0)); @@ -326,6 +355,9 @@ impl HnswGraph { Ok(Self { params, + base_bytes: capacity * std::mem::size_of::() + + node_bytes + + visited_pool_bytes(capacity), nodes, build_entry_point: AtomicU32::new(0), build_max_level: AtomicU16::new(0), @@ -335,9 +367,52 @@ impl HnswGraph { visible_len: AtomicUsize::new(0), visited_pool, packed_level0: ArcSwap::from_pointee(PackedLevel::empty()), + packed_bytes: AtomicUsize::new(0), }) } + /// Upper bound on the graph's dominant heap allocations. + /// + /// Near-constant from the first insert rather than proportional to `len()`: + /// the node arena is allocated in full at construction, sized by `capacity`. + /// Callers budgeting memtable memory must account for this the moment a + /// vector memtable takes its first row. + pub(crate) fn resident_bytes(&self) -> usize { + self.base_bytes + self.packed_bytes.load(Ordering::Relaxed) + } + + /// What [`Self::resident_bytes`] will report for a graph of this shape, + /// answerable before one is built. + /// + /// Everything `try_new` allocates is sized from `capacity`; the only random + /// input is how nodes divide across levels, and that division is a + /// geometric ladder — every node holds level 0, and the share reaching each + /// level above it falls by a factor of `m`. Walking that ladder lands close + /// to the built graph instead of sampling it, and level 0 — which dominates + /// — is not an estimate at all. + /// + /// Exists because the allocation is committed well before it happens: the + /// first vector row into a memtable materializes the whole graph. Charging + /// it only from that row on would put the largest single allocation in a + /// vector memtable beyond the reach of admission control. + pub(crate) fn reserved_bytes(capacity: usize, params: &BuildParams) -> usize { + // Guard the ladder's divisor rather than `params.m` itself: `validate` + // rejects m < 2, but this is reachable before that runs. + let ratio = params.m.max(2); + let mut reaching = capacity; + let mut links = 0; + for level in 0..params.max_level { + links += reaching + * (std::mem::size_of::() + + LevelLinks::allocated_bytes(max_neighbors(params.m, level))); + reaching /= ratio; + if reaching == 0 { + break; + } + } + capacity * std::mem::size_of::() + links + visited_pool_bytes(capacity) + } + /// Number of nodes visible to readers. pub fn len(&self) -> usize { self.visible_len.load(Ordering::Acquire) @@ -526,11 +601,19 @@ impl HnswGraph { /// The resulting batch uses the same schema and `lance:hnsw` metadata /// expected by `lance-index`'s `HNSW::load`. /// - /// Call this when no writer batch is in flight. Ordinary search readers - /// can run concurrently with insertion, but flush export should snapshot a - /// completed graph prefix. - pub fn to_lance_hnsw_batch(&self) -> Result { + /// `max_nodes` caps the prefix, for a caller that has already captured a + /// companion artifact and needs this one to agree with it: vector storage + /// is materialized separately, and a graph that advanced past it would name + /// rows the storage batch has no vector for. + /// + /// Ordinary search readers can run concurrently with insertion; a flush + /// export snapshots a completed prefix. + pub fn to_lance_hnsw_batch(&self, max_nodes: Option) -> Result { let visible_len = self.visible_len.load(Ordering::Acquire); + let visible_len = match max_nodes { + Some(max_nodes) => visible_len.min(max_nodes), + None => visible_len, + }; let max_level = self.params.max_level as usize; let mut level_counts = vec![0usize; max_level]; for id in 0..visible_len { @@ -556,13 +639,45 @@ impl HnswGraph { } let ranked = node.ranked(level as u16)?; vector_id_builder.append_value(id as u32); - neighbors_builder.append_value(ranked.iter().map(|point| Some(point.id))); - distances_builder.append_value(ranked.iter().map(|point| Some(point.distance))); + // `visible_len` is snapshotted but adjacency is read live, so a + // batch landing mid-export can append itself to a node already + // emitted. Those ids are not rows in this batch: `HNSW::load` + // slices level 0 to the rows below, and `neighbors_at` would + // address past them. `search` caps in-memory traversal the same + // way; the export has to persist the bound. Both columns filter + // together so a reader pairing them sees equal lengths. + neighbors_builder.append_value( + ranked + .iter() + .filter(|point| (point.id as usize) < visible_len) + .map(|point| Some(point.id)), + ); + distances_builder.append_value( + ranked + .iter() + .filter(|point| (point.id as usize) < visible_len) + .map(|point| Some(point.distance)), + ); } } + // `publish_visible` stores the entry point before `visible_len`, so it + // can name a node outside this prefix. `search` returns no results in + // that case; an exported index cannot, so fall back to the deepest node + // the prefix does hold. + let entry_point = { + let published = self.visible_entry_point.load(Ordering::Acquire); + if (published as usize) < visible_len { + published + } else { + (0..visible_len) + .max_by_key(|&id| self.nodes[id].levels.len()) + .map(|id| id as u32) + .unwrap_or(0) + } + }; let metadata = LanceHnswMetadata { - entry_point: self.visible_entry_point.load(Ordering::Acquire), + entry_point, params: self.params.clone(), level_offsets: level_counts .iter() @@ -594,9 +709,12 @@ impl HnswGraph { } fn validate_source(&self, vectors: &impl VectorSource, needed_len: usize) -> Result<()> { + // Not caller input: the graph was sized below what the memtable holds. + // See the matching note in `storage.rs::append_batch`. if needed_len > self.nodes.len() { - return Err(Error::invalid_input(format!( - "graph capacity {} exhausted: need {needed_len}", + return Err(Error::internal(format!( + "HNSW graph capacity {} exhausted: need {needed_len}; \ + the graph is sized below the memtable's row capacity", self.nodes.len() ))); } @@ -1051,9 +1169,13 @@ impl HnswGraph { offsets.push(neighbors.len()); } + let packed_bytes = offsets.capacity() * std::mem::size_of::() + + neighbors.capacity() * std::mem::size_of::(); + // ArcSwap reclaims the prior snapshot once no reader guard holds it. self.packed_level0 .store(Arc::new(PackedLevel { offsets, neighbors })); + self.packed_bytes.store(packed_bytes, Ordering::Relaxed); Ok(()) } } @@ -1083,6 +1205,20 @@ fn max_neighbors(m: usize, level: u16) -> usize { if level == 0 { m * 2 } else { m } } +/// One list per worker, doubled so a searcher never blocks on the queue. +fn visited_pool_size() -> usize { + rayon::current_num_threads().max(1) * 2 +} + +/// Heap the visited pool settles at for a graph of `capacity` nodes. The lists +/// are pushed empty but `VisitedList::reset` resizes each to one bit per node +/// on first use, so the pool is charged at its grown size from the start. +fn visited_pool_bytes(capacity: usize) -> usize { + visited_pool_size() + * (std::mem::size_of::() + + capacity.div_ceil(WORD_BITS) * std::mem::size_of::()) +} + #[derive(Debug)] struct VisitedList { words: Vec, @@ -1125,6 +1261,7 @@ impl VisitedList { #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::AtomicBool; use arrow_array::{ArrayRef, FixedSizeListArray, Float32Array}; use arrow_schema::{DataType, Field}; @@ -1182,6 +1319,159 @@ mod tests { assert!(result.iter().any(|point| point.id == 42)); } + /// The graph must be exportable to a boundary its caller chose, because the + /// companion vector storage is captured separately: a graph that advanced + /// past it would name rows the storage batch has no vector for, and a search + /// would score them. + #[test] + fn to_lance_hnsw_batch_honors_a_caller_supplied_prefix() { + const ROWS: usize = 256; + const DIM: usize = 8; + const PREFIX: usize = 64; + let store = Arc::new( + ArrowFixedSizeListVectorStore::try_new(512, 4, DIM, DistanceType::L2).unwrap(), + ); + let ids = store.append_batch(fsl(ROWS, DIM), 0).unwrap(); + let snapshot = store.snapshot(); + let graph = HnswGraph::try_new( + 512, + BuildParams::mem_wal_default() + .num_edges(8) + .ef_construction(32) + .seed(17), + ) + .unwrap(); + graph.insert_batch(ids, &snapshot).unwrap(); + + let full = graph.to_lance_hnsw_batch(None).unwrap(); + let bounded = graph.to_lance_hnsw_batch(Some(PREFIX)).unwrap(); + assert!( + bounded.num_rows() < full.num_rows(), + "the cap has to actually bound the export" + ); + assert_eq!(HNSW::load(bounded.clone()).unwrap().len(), PREFIX); + + // Every edge must stay inside the requested prefix, not merely inside + // whatever the graph had published. + let neighbors = bounded + .column(1) + .as_any() + .downcast_ref::() + .expect("neighbors column is a list"); + for row in 0..bounded.num_rows() { + let ids = neighbors.value(row); + let ids = ids + .as_any() + .downcast_ref::() + .expect("neighbor ids are u32"); + for i in 0..ids.len() { + assert!( + (ids.value(i) as usize) < PREFIX, + "row {row} points at {} outside the {PREFIX}-node prefix", + ids.value(i) + ); + } + } + } + + /// An export racing inserts must persist only edges inside the prefix it + /// publishes. + /// + /// `to_lance_hnsw_batch` snapshots `visible_len` but reads each node's ranked + /// list live, so a batch landing mid-export can append itself to a node + /// already emitted. `HNSW::load` then slices level 0 to the exported rows and + /// a walk over that edge addresses past them. `search` bounds in-memory + /// traversal by `visible_len` for the same reason. + #[test] + fn test_lance_hnsw_batch_edges_stay_inside_the_exported_prefix() { + const ROWS: usize = 1024; + const DIM: usize = 16; + const CHUNK: usize = 32; + let store = Arc::new( + ArrowFixedSizeListVectorStore::try_new(2048, 8, DIM, DistanceType::L2).unwrap(), + ); + let ids = store.append_batch(fsl(ROWS, DIM), 0).unwrap(); + let snapshot = store.snapshot(); + let graph = Arc::new( + HnswGraph::try_new( + 2048, + BuildParams::mem_wal_default() + .num_edges(8) + .ef_construction(32) + .seed(13), + ) + .unwrap(), + ); + + // Seed a prefix so an export has real adjacency to walk. + let chunk = CHUNK as u32; + graph + .insert_batch(ids.start..ids.start + chunk, &snapshot) + .unwrap(); + + let writing = Arc::new(AtomicBool::new(true)); + let writer = { + let graph = Arc::clone(&graph); + let writing = Arc::clone(&writing); + std::thread::spawn(move || { + let mut next = ids.start + chunk; + while next < ids.end { + let stop = (next + chunk).min(ids.end); + graph.insert_batch(next..stop, &snapshot).unwrap(); + next = stop; + } + writing.store(false, Ordering::Release); + }) + }; + + // Export while the writer runs rather than a fixed count, so the overlap + // does not depend on how fast this machine inserts. + let mut exports = 0; + let mut edges_checked = 0; + while writing.load(Ordering::Acquire) || exports < 5 { + let batch = graph.to_lance_hnsw_batch(None).unwrap(); + let rows = batch.num_rows(); + let neighbors = batch + .column(1) + .as_any() + .downcast_ref::() + .expect("neighbors column is a list"); + let distances = batch + .column(2) + .as_any() + .downcast_ref::() + .expect("distances column is a list"); + // Level 0 holds every visible node, so its row count is the prefix. + let prefix = HNSW::load(batch.clone()).unwrap().len() as u32; + for row in 0..rows { + let ids = neighbors.value(row); + let ids = ids + .as_any() + .downcast_ref::() + .expect("neighbor ids are u32"); + assert_eq!( + ids.len(), + distances.value(row).len(), + "row {row} pairs {} ids with {} distances", + ids.len(), + distances.value(row).len() + ); + for i in 0..ids.len() { + let nid = ids.value(i); + assert!( + nid < prefix, + "exported row {row} points at {nid}, outside the \ + {prefix}-node prefix it was exported with" + ); + edges_checked += 1; + } + } + exports += 1; + } + writer.join().unwrap(); + assert!(edges_checked > 0, "test never inspected an edge"); + } + #[test] fn test_lance_hnsw_batch_loads_with_lance_index() { let rows = 64; @@ -1201,7 +1491,7 @@ mod tests { .unwrap(); graph.insert_batch(ids, &snapshot).unwrap(); - let batch = graph.to_lance_hnsw_batch().unwrap(); + let batch = graph.to_lance_hnsw_batch(None).unwrap(); let loaded = HNSW::load(batch).unwrap(); assert_eq!(loaded.len(), rows); } diff --git a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs index bbeb57a5fe2..bbd745ab338 100644 --- a/rust/lance/src/dataset/mem_wal/hnsw/storage.rs +++ b/rust/lance/src/dataset/mem_wal/hnsw/storage.rs @@ -196,6 +196,25 @@ impl ArrowFixedSizeListVectorStore { }) } + /// Heap bytes of the store's own slabs, all sized from `capacity` and + /// `max_batches` at construction. + /// + /// Excludes the vectors themselves: batches are held by reference, so their + /// bytes belong to the MemTable's batch store and counting them here would + /// double-count. That also makes this independent of `dim`. + pub(crate) fn resident_bytes(&self) -> usize { + Self::reserved_bytes(self.capacity, self.max_batches) + } + + /// What [`Self::resident_bytes`] will report for a store of this shape, + /// answerable before one exists — the slabs are sized from these two + /// numbers alone, and `dim` never enters. Lets a memory ceiling charge for + /// the store ahead of the first insert that allocates it. + pub(crate) fn reserved_bytes(capacity: usize, max_batches: usize) -> usize { + max_batches * std::mem::size_of::() + + capacity * (std::mem::size_of::() + std::mem::size_of::()) + } + /// Number of committed vectors. pub fn committed_len(&self) -> usize { self.committed_len.load(Ordering::Acquire) @@ -269,25 +288,30 @@ impl ArrowFixedSizeListVectorStore { ))); }; + // Exhaustion is a shard-construction bug (store sized below the memtable), + // not caller input — hence `internal`, not `invalid_input`. let start = self.committed_len.load(Ordering::Relaxed); let end = start.checked_add(num_rows).ok_or_else(|| { - Error::invalid_input(format!( + Error::internal(format!( "vector count overflow: start={}, batch_len={}", start, num_rows )) })?; if end > self.capacity { - return Err(Error::invalid_input(format!( - "capacity {} exhausted: inserting rows [{}..{})", + return Err(Error::internal(format!( + "HNSW vector store capacity {} exhausted: inserting rows [{}..{}); \ + the store is sized below the memtable's row capacity", self.capacity, start, end ))); } let batch_idx = self.committed_batches.load(Ordering::Relaxed); if batch_idx >= self.max_batches { - return Err(Error::invalid_input(format!( - "max_batches {} exhausted", - self.max_batches + return Err(Error::internal(format!( + "HNSW vector store max_batches {} exhausted at batch_idx {} \ + (inserting rows [{}..{})); the store is sized below the \ + memtable's batch capacity", + self.max_batches, batch_idx, start, end ))); } @@ -327,6 +351,19 @@ impl ArrowFixedSizeListVectorStore { /// Capture a stable visible prefix of the store. pub fn snapshot(self: &Arc) -> VectorStoreSnapshot { + self.snapshot_after_visible_len(|| {}) + } + + fn snapshot_after_visible_len( + self: &Arc, + after_visible_len: impl FnOnce(), + ) -> VectorStoreSnapshot { + // Read the length first. If it observes a newly committed batch, the + // Acquire load also makes the preceding committed_batches publication + // visible to the later load below. If it observes the old length, the + // snapshot remains a valid prefix even if a writer commits meanwhile. + let visible_len = self.committed_len(); + after_visible_len(); let committed_batches = self.committed_batches.load(Ordering::Acquire); let contiguous_values_addr = if committed_batches == 1 { // SAFETY: batch slot 0 is initialized before committed_batches is @@ -337,7 +374,7 @@ impl ArrowFixedSizeListVectorStore { }; VectorStoreSnapshot { store: self.clone(), - visible_len: self.committed_len(), + visible_len, contiguous_values_addr, } } @@ -436,12 +473,22 @@ impl VectorSource for VectorStoreSnapshot { } fn row_id(&self, id: u32) -> u64 { - debug_assert!((id as usize) < self.visible_len); + // HNSW only requests ids from its own graph, which is built from this + // snapshot's visible prefix. Keep this as a debug-only contract check. + debug_assert!( + (id as usize) < self.visible_len, + "vector id {id} is outside snapshot length {}", + self.visible_len + ); self.store.row_id_at(id) } fn vector(&self, id: u32) -> &[f32] { - debug_assert!((id as usize) < self.visible_len); + debug_assert!( + (id as usize) < self.visible_len, + "vector id {id} is outside snapshot length {}", + self.visible_len + ); if self.contiguous_values_addr != 0 { // SAFETY: this snapshot holds the store Arc, which retains the // Arrow batch backing this pointer. The id was checked above. @@ -465,6 +512,7 @@ fn uninit_boxed_slice(len: usize) -> Box<[MaybeUninit]> { #[cfg(test)] mod tests { use super::*; + use std::sync::Barrier; fn fsl(values: Vec, dim: usize) -> Arc { let values = Arc::new(Float32Array::from(values)) as ArrayRef; @@ -499,6 +547,36 @@ mod tests { ); } + #[test] + fn test_snapshot_stays_with_visible_prefix_during_commit() { + let store = + Arc::new(ArrowFixedSizeListVectorStore::try_new(8, 2, 2, DistanceType::L2).unwrap()); + store + .append_batch(fsl(vec![1.0, 2.0, 3.0, 4.0], 2), 10) + .unwrap(); + + let visible_len_loaded = Arc::new(Barrier::new(2)); + let continue_snapshot = Arc::new(Barrier::new(2)); + let snapshot_store = store.clone(); + let snapshot_visible_len_loaded = visible_len_loaded.clone(); + let snapshot_continue = continue_snapshot.clone(); + let snapshot_thread = std::thread::spawn(move || { + snapshot_store.snapshot_after_visible_len(|| { + snapshot_visible_len_loaded.wait(); + snapshot_continue.wait(); + }) + }); + + visible_len_loaded.wait(); + store.append_batch(fsl(vec![5.0, 6.0], 2), 12).unwrap(); + continue_snapshot.wait(); + + let snapshot = snapshot_thread.join().unwrap(); + assert_eq!(snapshot.len(), 2); + assert_eq!(snapshot.row_id(1), 11); + assert_eq!(snapshot.vector(1), &[3.0, 4.0]); + } + /// Build a `FixedSizeList` where `None` rows are null at the list /// level (the representation a tombstone / embedding-less row produces). fn fsl_opt(rows: &[Option>], dim: usize) -> Arc { diff --git a/rust/lance/src/dataset/mem_wal/index.rs b/rust/lance/src/dataset/mem_wal/index.rs index d16d3105551..50bf36ee1f0 100644 --- a/rust/lance/src/dataset/mem_wal/index.rs +++ b/rust/lance/src/dataset/mem_wal/index.rs @@ -23,16 +23,18 @@ mod pk_key; use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Instant; use datafusion::common::ScalarValue; use super::memtable::batch_store::StoredBatch; +use super::wal::WriterCursors; use arrow_array::RecordBatch; +use arrow_schema::{DataType, Schema as ArrowSchema}; use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::pbold; use lance_index::scalar::InvertedIndexParams; -use lance_index::scalar::inverted::InvertedListFormatVersion; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_linalg::distance::DistanceType; use lance_table::format::IndexMetadata; @@ -48,6 +50,7 @@ pub type RowPosition = u64; // Re-export public types used externally pub use btree::{BTreeIndexConfig, BTreeMemIndex}; pub use fts::{FtsIndexConfig, FtsMemIndex, FtsQueryExpr, SearchOptions}; +pub(crate) use fts::{QueryLocalFtsIndex, QueryLocalFtsStats}; pub use hnsw::{HnswIndexConfig, HnswMemIndex}; pub use pk_key::encode_pk_tuple; @@ -58,6 +61,19 @@ use pk_key::encode_pk_batch; /// [`BTreeMemIndex`]'s byte backend indexes it directly. const PK_KEY_COLUMN: &str = "__pk_key__"; +/// Row count at or below which [`IndexStore::insert_batches`] indexes inline +/// rather than spawning a thread per index. +/// +/// The spawn is one OS thread *per index* — tens of microseconds each, and a table can +/// carry several BTrees alongside its HNSW and FTS — so for a small batch it costs more +/// than the indexing it parallelizes. Small batches are not the exceptional case: a +/// durable put triggers a WAL flush covering only the batch it just inserted, so this +/// path is routinely called with a single short batch. +/// +/// The crossover depends on per-row HNSW cost, which varies with dimension and +/// `ef_construction`; tune against `benches/mem_wal/vector/mem_wal_index_micro.rs`. +const PARALLEL_INDEX_MIN_ROWS: usize = 64; + /// The memtable's primary-key index, used to answer "newest visible version of /// this key" for dedup. Single-column PKs reuse the column's compact typed /// [`BTreeMemIndex`] (no second copy); composite PKs key a `BTreeMemIndex` on @@ -80,9 +96,232 @@ enum PkIndex { // Index Store // ============================================================================ -/// Configuration for an index in MemWAL. +/// Validate every configured in-memory index, and the composite primary key, +/// against the shard schema. Call once at shard open, before any write can land. +/// +/// This is what makes poison-and-replay *terminating*. An index insert that +/// fails deterministically on a row that is already WAL-durable cannot be +/// recovered from: the writer poisons, the operator reopens, replay re-reads the +/// same WAL rows, the same insert fails again, and `open()` propagates it — a +/// shard that never comes back. Every such failure is an index *config* +/// disagreeing with the schema, never a property of the data, so one pass here +/// closes the whole class before a single row is accepted. +/// +/// The data-dependent errors inside the index layer are already unreachable +/// through `put`: `MemTable::insert_batches_only` does a full `Arc` +/// equality check, so a batch that would trip one is rejected before it reaches +/// the batch store, let alone the WAL. +/// +/// It also rejects a config whose `field_id` names a different column than its +/// `column`. Index *selection* keys off `field_id` — a single-column PK reuses +/// the BTree whose `field_id` matches its key — so a config resolved only by name +/// could be bound under the wrong identity, serving stale reads and flushing the +/// wrong column into the durable PK sidecar. `lance_schema` supplies the +/// authoritative name→id mapping. +pub fn validate_index_configs( + configs: &[MemIndexConfig], + schema: &ArrowSchema, + lance_schema: &LanceSchema, + pk_columns: &[String], +) -> Result<()> { + for config in configs { + let column = config.column(); + if let MemIndexConfig::Fts(config) = config { + let resolved = crate::index::scalar::inverted::resolve_fts_field( + lance_schema, + column, + config.params.get_document_granularity(), + ) + .map_err(|error| { + Error::invalid_input(format!( + "FTS index '{}' is invalid for field path '{}': {error}", + config.name, column + )) + })?; + if resolved.final_field_id != config.field_id { + return Err(Error::invalid_input(format!( + "index '{}' is configured with field_id {} but its field path '{}' has \ + final field_id {} in the shard schema", + config.name, config.field_id, column, resolved.final_field_id, + ))); + } + continue; + } + + let field = schema.field_with_name(column).map_err(|_| { + Error::invalid_input(format!( + "index '{}' is configured on column '{}', which is not in the shard schema; \ + available columns: [{}]", + config.name(), + column, + schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>() + .join(", ") + )) + })?; + + match config { + // BTree falls back to per-row `ScalarValue` extraction, so it + // accepts any column type the schema can hold. Existence is the + // only precondition. + MemIndexConfig::BTree(_) => {} + MemIndexConfig::Fts(_) => unreachable!("FTS configs are validated by schema path"), + MemIndexConfig::Hnsw(_) => match field.data_type() { + DataType::FixedSizeList(item, dim) => { + if item.data_type() != &DataType::Float32 { + return Err(Error::invalid_input(format!( + "HNSW index '{}' requires a FixedSizeList column; \ + column '{}' has item type {:?}", + config.name(), + column, + item.data_type() + ))); + } + // `HnswMemIndex.dim` is a placeholder until the first batch + // pins it (`hnsw.rs`), so a zero-width vector would only + // surface at insert time — i.e. on already-durable data. + if *dim <= 0 { + return Err(Error::invalid_input(format!( + "HNSW index '{}' requires a vector dimension > 0; column '{}' has \ + dimension {dim}", + config.name(), + column, + ))); + } + } + other => { + return Err(Error::invalid_input(format!( + "HNSW index '{}' requires a FixedSizeList column; \ + column '{}' is {:?}", + config.name(), + column, + other + ))); + } + }, + } + + // The column resolves, but index selection keys off `field_id`, not name. + // A config whose `field_id` identifies a *different* column would be bound + // under the wrong identity (e.g. reused as the single-column PK index), so + // reject any `field_id` that does not name the resolved column. + let resolved_field_id = lance_schema + .field(column) + .ok_or_else(|| { + Error::invalid_input(format!( + "index '{}' is configured on column '{}', which is present in the Arrow \ + schema but absent from the Lance schema", + config.name(), + column, + )) + })? + .id; + if resolved_field_id != config.field_id() { + return Err(Error::invalid_input(format!( + "index '{}' is configured with field_id {} but its column '{}' has field_id {} \ + in the shard schema", + config.name(), + config.field_id(), + column, + resolved_field_id, + ))); + } + } + + // Every PK column must exist in the schema. A single-column PK aliases a + // BTree entry (any type); only a *composite* PK builds an order-preserving + // encoded key, and only some types encode. + for column in pk_columns { + let field = schema.field_with_name(column).map_err(|_| { + Error::invalid_input(format!( + "primary-key column '{column}' is not in the shard schema" + )) + })?; + if pk_columns.len() > 1 && !is_encodable_pk_type(field.data_type()) { + return Err(Error::invalid_input(format!( + "composite primary-key column '{column}' has type {:?}, which has no \ + order-preserving key encoding", + field.data_type() + ))); + } + } + + Ok(()) +} + +/// Types `pk_key::encode_value` can encode into an order-preserving composite key. +fn is_encodable_pk_type(data_type: &DataType) -> bool { + matches!( + data_type, + DataType::Int8 + | DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::UInt8 + | DataType::UInt16 + | DataType::UInt32 + | DataType::UInt64 + | DataType::Date32 + | DataType::Date64 + | DataType::Boolean + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Binary + | DataType::LargeBinary + | DataType::FixedSizeBinary(_) + ) +} + +/// The index kinds a MemTable can maintain — the registry of MemWAL index +/// support. Data-free because indexes are identified by type url before any +/// [`MemIndexConfig`] exists. +/// +/// Adding a variant is a compile error in [`details_suffix`](Self::details_suffix), +/// `MemIndexConfig::kind`, and `Dataset::mem_wal_writer` until each handles it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MemIndexKind { + /// BTree index for scalar fields (point lookups, range queries). + BTree, + /// HNSW vector index built incrementally, queryable while building. + Hnsw, + /// Full-text search index. + Fts, +} + +impl MemIndexKind { + /// Every maintainable kind. A kind missing here is never detected, so it + /// goes unmaintained rather than reaching a memtable that cannot build it. + pub const ALL: &'static [Self] = &[Self::BTree, Self::Hnsw, Self::Fts]; + + /// Suffix of the protobuf details message identifying this kind. + /// + /// Only the suffix: the prefix varies by dataset version + /// (`/lance.table.`, `/lance.index.pb.`, and the `type.googleapis.com/` + /// form MemWAL flush once wrote), and all must resolve. + pub const fn details_suffix(self) -> &'static str { + match self { + Self::BTree => "BTreeIndexDetails", + Self::Hnsw => "VectorIndexDetails", + Self::Fts => "InvertedIndexDetails", + } + } + + /// The kind a base-table index of this protobuf type maps to, or `None` + /// when a memtable cannot maintain it. + pub fn from_type_url(type_url: &str) -> Option { + Self::ALL + .iter() + .copied() + .find(|kind| type_url.ends_with(kind.details_suffix())) + } +} + +/// Configuration for an index in MemWAL. Pairs 1:1 with [`MemIndexKind`] via +/// [`kind`](Self::kind). /// -/// Each variant contains all the configuration needed for that index type. /// `Hnsw` is boxed because `HnswBuildParams` is small but the variant may /// grow with future config (e.g. shard-specific tuning). #[derive(Debug, Clone)] @@ -96,6 +335,16 @@ pub enum MemIndexConfig { } impl MemIndexConfig { + /// The kind this config builds. Links the config enum to the registry, so + /// a new variant must declare its kind. + pub const fn kind(&self) -> MemIndexKind { + match self { + Self::BTree(_) => MemIndexKind::BTree, + Self::Hnsw(_) => MemIndexKind::Hnsw, + Self::Fts(_) => MemIndexKind::Fts, + } + } + /// Get the index name. pub fn name(&self) -> &str { match self { @@ -135,26 +384,37 @@ impl MemIndexConfig { /// Create an FTS index config from base table IndexMetadata. pub fn fts_from_metadata(index_meta: &IndexMetadata, schema: &LanceSchema) -> Result { - let (field_id, column) = Self::extract_field_info(index_meta, schema)?; + let (field_id, _) = Self::extract_field_info(index_meta, schema)?; // Extract InvertedIndexParams from index_details if available - let params = if let Some(details_any) = &index_meta.index_details { - if let Ok(details) = pbold::InvertedIndexDetails::decode(details_any.value.as_slice()) { - InvertedIndexParams::try_from(&details)? - } else { - InvertedIndexParams::default() - } + let details = if let Some(details_any) = &index_meta.index_details { + pbold::InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails for MemWAL FTS index '{}': {}", + index_meta.name, err + )) + })? } else { - InvertedIndexParams::default() + pbold::InvertedIndexDetails::default() }; - let params = params.format_version(Self::fts_format_version_from_metadata(index_meta)?); - - Ok(Self::Fts(FtsIndexConfig::with_params( - index_meta.name.clone(), + let details = + crate::index::scalar::inverted::normalize_inverted_details(index_meta, details)?; + let params = InvertedIndexParams::try_from(&details)?; + let resolved = crate::index::scalar::inverted::resolve_fts_field_by_id( + schema, field_id, - column, - params, - ))) + params.get_document_granularity(), + )?; + + Ok(Self::Fts( + FtsIndexConfig::try_with_params( + index_meta.name.clone(), + field_id, + resolved.canonical_path.clone(), + params, + )? + .with_resolved_field(resolved), + )) } /// Create an HNSW vector index config. @@ -181,37 +441,6 @@ impl MemIndexConfig { )) } - /// Detect index type from protobuf type_url. - pub fn detect_index_type(type_url: &str) -> Result<&'static str> { - if type_url.ends_with("BTreeIndexDetails") { - Ok("btree") - } else if type_url.ends_with("InvertedIndexDetails") { - Ok("fts") - } else if type_url.ends_with("VectorIndexDetails") { - Ok("vector") - } else { - Err(Error::invalid_input(format!( - "Unsupported index type for MemWAL: {}. Supported: BTree, Inverted, Vector", - type_url - ))) - } - } - - fn fts_format_version_from_metadata( - index_meta: &IndexMetadata, - ) -> Result { - match index_meta.index_version { - // Legacy Arrow FTS indexes did not use the v1/v2 metadata values, but - // the maintained-index path can only write the modern format. - 0 | 1 => Ok(InvertedListFormatVersion::V1), - 2 => Ok(InvertedListFormatVersion::V2), - version => Err(Error::invalid_input(format!( - "FTS index '{}' has unsupported index_version {}; expected 0, 1, or 2", - index_meta.name, version - ))), - } - } - /// Extract field ID and column name from index metadata. fn extract_field_info( index_meta: &IndexMetadata, @@ -232,14 +461,40 @@ impl MemIndexConfig { } } +/// Names the index, not just its type: a caller validating a maintained set +/// needs to know which one to drop. +pub(crate) fn unsupported_index_type(index_name: &str, type_url: &str) -> Error { + Error::invalid_input(format!( + "index '{}' has type {}, which the MemWAL cannot maintain. Supported: BTree, Inverted, Vector", + index_name, type_url + )) +} + +/// Which prefix of a MemTable a reader may see. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MemTableVisibility { + /// [`IndexStore::visible_count`]. Required for every reader but the writer + /// itself: a row past this bound can still fail its append and never exist. + #[default] + Published, + /// [`IndexStore::indexed_count`], which also covers writes whose append is + /// outstanding. + /// + /// Sound only for a writer reading its own prefix under the lock that makes + /// it the sole writer. Both cursors advance over contiguous prefixes, so a + /// row derived from `p` cannot be published before `p`, and a failed append + /// poisons the writer before either is acknowledged. + Indexed, +} + /// Registry managing all in-memory indexes for a MemTable. /// /// Indexes are keyed by index name. Each index stores its field_id for /// stable column-to-index resolution (column name → field_id → index). /// -/// The store also carries the MemTable's `max_visible_batch_position` -/// watermark — the highest batch position that is durable in the WAL and -/// therefore safe for scanners to read. Scanners snapshot this at plan +/// The store also carries the MemTable's two cursors: `indexed_count` (what the +/// index layer has ingested) and `visible_count` (what is indexed *and* durable, +/// and therefore safe for scanners to read). Scanners snapshot the latter at plan /// construction time so every plan keys on a stable MVCC cursor. pub struct IndexStore { /// BTree indexes keyed by index name. `Arc` so the primary-key BTrees can be @@ -253,10 +508,23 @@ pub struct IndexStore { /// primary key. Queried via [`Self::pk_newest_visible`] (see /// [`Self::enable_pk_index`]). pk_index: Option, - /// Maximum batch position that is durable in the WAL and therefore - /// visible to scanners. Advanced unconditionally after a WAL append - /// succeeds; not gated on whether any indexes are configured. - max_visible_batch_position: AtomicUsize, + /// How many batches of this memtable have been fully indexed. An exclusive + /// count: 0 means none. + /// + /// This has only ever been an *indexed* cursor — it is advanced at the end of + /// `insert_batches`, once every index insert for the batch has completed, and + /// never before. It was named `max_visible_batch_position` and treated as a + /// visibility cursor by five read sites, which is how rows became readable + /// before they were durable. Publishing is a separate step, and it is the + /// writer's to make — see `visible_count`. + indexed_count: AtomicUsize, + + /// The writer's cursors, and this memtable's coordinate within them. `None` + /// for a bare `IndexStore` (tests, benches), where visibility is just the + /// indexed prefix. + /// + /// Visibility is **derived, never stored**: see `visible_count`. + durability: Option<(Arc, usize)>, /// Conservative flag set once this memtable has observed any primary-key /// rewrite while maintaining a search index. Search planners can push top-k /// into HNSW/FTS for append-only PK data, but must switch to @@ -271,7 +539,8 @@ impl Default for IndexStore { hnsw_indexes: HashMap::new(), fts_indexes: HashMap::new(), pk_index: None, - max_visible_batch_position: AtomicUsize::new(0), + indexed_count: AtomicUsize::new(0), + durability: None, pk_has_overrides: AtomicBool::new(false), } } @@ -299,10 +568,7 @@ impl std::fmt::Debug for IndexStore { } }, ) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position.load(Ordering::Acquire), - ) + .field("indexed_count", &self.indexed_count.load(Ordering::Acquire)) .field( "pk_has_overrides", &self.pk_has_overrides.load(Ordering::Acquire), @@ -352,8 +618,19 @@ impl IndexStore { registry.hnsw_indexes.insert(c.name.clone(), index); } MemIndexConfig::Fts(c) => { - let index = - FtsMemIndex::with_params(c.field_id, c.column.clone(), c.params.clone()); + let index = match c.resolved_field.as_deref() { + Some(resolved) => FtsMemIndex::try_with_resolved_field( + c.field_id, + c.column.clone(), + c.params.clone(), + resolved.clone(), + )?, + None => FtsMemIndex::try_with_params( + c.field_id, + c.column.clone(), + c.params.clone(), + )?, + }; registry.fts_indexes.insert(c.name.clone(), index); } } @@ -452,13 +729,16 @@ impl IndexStore { field_id: i32, column: String, params: InvertedIndexParams, - ) { + ) -> Result<()> { assert!( self.pk_index.is_none() || self.pk_is_empty(), "FTS indexes must be configured before inserting rows into a PK memtable" ); - self.fts_indexes - .insert(name, FtsMemIndex::with_params(field_id, column, params)); + self.fts_indexes.insert( + name, + FtsMemIndex::try_with_params(field_id, column, params)?, + ); + Ok(()) } /// Maintain a primary-key index so the memtable can answer "newest visible @@ -516,7 +796,7 @@ impl IndexStore { /// BTree (the sidecar dedup index). Single-column emits the typed PK value; /// composite emits the order-preserving `Binary` encoded tuple. Empty when /// there is no primary key. Row positions line up 1:1 with the forward- - /// written data file, so they are the flushed row ids directly. + /// written data file, so they are the SSTable row ids directly. pub fn pk_training_batches(&self, batch_size: usize) -> Result> { match &self.pk_index { None => Ok(Vec::new()), @@ -688,26 +968,25 @@ impl IndexStore { let had_existing = self.insert_composite_pk(batch, row_offset, track_pk_overrides)?; self.mark_pk_overrides_if_needed(had_existing); - // Update global watermark after all indexes have been updated + // Update the indexed prefix after every index has been updated. if let Some(bp) = batch_position { - self.advance_max_visible_batch_position(bp); + self.advance_indexed_count(bp + 1); } Ok(()) } - /// Advance the visibility watermark to at least `batch_pos`. + /// Advance the indexed prefix to at least `count` batches. /// - /// The watermark only ever moves forward (idempotent max). The vector - /// planner relies on the insert paths setting `pk_has_overrides` before - /// calling this method, so any snapshot that can see a PK rewrite also - /// observes `pk_has_overrides == true`. - pub(crate) fn advance_max_visible_batch_position(&self, batch_pos: usize) { - let mut current = self.max_visible_batch_position.load(Ordering::Acquire); - while batch_pos > current { - match self.max_visible_batch_position.compare_exchange_weak( + /// Only ever moves forward (idempotent max). The vector planner relies on the + /// insert paths setting `pk_has_overrides` before this is called, so any + /// snapshot that can see a PK rewrite also observes `pk_has_overrides == true`. + pub(crate) fn advance_indexed_count(&self, count: usize) { + let mut current = self.indexed_count.load(Ordering::Acquire); + while count > current { + match self.indexed_count.compare_exchange_weak( current, - batch_pos, + count, Ordering::Release, Ordering::Acquire, ) { @@ -717,43 +996,134 @@ impl IndexStore { } } - /// Insert multiple batches into all indexes with cross-batch optimization. + /// Insert multiple batches into every index. + /// + /// Above `PARALLEL_INDEX_MIN_ROWS` rows each index runs on its own thread, which + /// maximizes parallelism when several indexes are maintained. At or below it they run + /// inline on the calling thread: the spawn is one OS thread *per index*, and for a + /// handful of rows that costs more than the indexing itself. + /// + /// Returns a map of index names to their update durations for performance tracking. #[instrument(name = "idx_insert_batches", level = "debug", skip_all, fields(batch_count = batches.len()))] - pub fn insert_batches(&self, batches: &[StoredBatch]) -> Result<()> { + pub fn insert_batches( + &self, + batches: &[StoredBatch], + ) -> Result> { if batches.is_empty() { - return Ok(()); + return Ok(std::collections::HashMap::new()); } let track_pk_overrides = self.should_track_pk_overrides(); - // BTree indexes: iterate batches (no cross-batch optimization benefit) - for index in self.btree_indexes.values() { + + // One task per index, boxed so the inline and the threaded path drive the very + // same closures. Each reports whether it saw an already-present PK. + type IndexTask<'a> = Box Result + Send + Sync + 'a>; + let mut tasks: Vec<(&str, IndexTask<'_>)> = Vec::new(); + + for (name, index) in &self.btree_indexes { let track_this_index = track_pk_overrides && self.is_single_pk_btree(index); - let mut had_existing = false; - for stored in batches { - if track_this_index { - had_existing |= - index.insert_and_report_existing(&stored.data, stored.row_offset)?; - } else { - index.insert(&stored.data, stored.row_offset)?; - } - } - self.mark_pk_overrides_if_needed(had_existing); + tasks.push(( + name.as_str(), + Box::new(move || { + let mut had_existing = false; + for stored in batches { + if track_this_index { + had_existing |= index + .insert_and_report_existing(&stored.data, stored.row_offset)?; + } else { + index.insert(&stored.data, stored.row_offset)?; + } + } + Ok(had_existing) + }), + )); } - // HNSW indexes: use batched insert - for index in self.hnsw_indexes.values() { - index.insert_batches(batches)?; + for (name, index) in &self.hnsw_indexes { + tasks.push(( + name.as_str(), + Box::new(move || index.insert_batches(batches).map(|_| false)), + )); } - // FTS indexes: iterate batches (potential future optimization) - for index in self.fts_indexes.values() { - for stored in batches { - index.insert(&stored.data, stored.row_offset)?; + for (name, index) in &self.fts_indexes { + tasks.push(( + name.as_str(), + Box::new(move || { + for stored in batches { + index.insert(&stored.data, stored.row_offset)?; + } + Ok(false) + }), + )); + } + + // Keep the raw `Duration` so sub-millisecond timings (the steady state for BTree + // updates) survive instead of truncating to 0. + let total_rows: usize = batches.iter().map(|b| b.num_rows).sum(); + let results: Vec<(&str, std::time::Duration, Result)> = + if tasks.len() < 2 || total_rows <= PARALLEL_INDEX_MIN_ROWS { + tasks + .iter() + .map(|(name, task)| { + let start = Instant::now(); + let result = task(); + (*name, start.elapsed(), result) + }) + .collect() + } else { + std::thread::scope(|scope| { + let handles: Vec<_> = tasks + .iter() + .map(|(name, task)| { + let handle = scope.spawn(move || { + let start = Instant::now(); + let result = task(); + (start.elapsed(), result) + }); + (*name, handle) + }) + .collect(); + + handles + .into_iter() + .map(|(name, handle)| match handle.join() { + Ok((duration, result)) => (name, duration, result), + Err(_) => ( + name, + std::time::Duration::ZERO, + Err(Error::internal(format!("Index '{}' thread panicked", name))), + ), + }) + .collect() + }) + }; + + // Every task ran to completion whether or not a peer failed (the threaded path + // joins all handles unconditionally). Keep the first error; there is no rollback, + // so a failure here is terminal for the writer. + let mut first_error: Option = None; + let mut had_existing_pk = false; + let mut duration_map = + std::collections::HashMap::::with_capacity(results.len()); + + for (name, duration, result) in results { + duration_map.insert(name.to_string(), duration); + match result { + Ok(had_existing) => had_existing_pk |= had_existing, + Err(e) if first_error.is_none() => first_error = Some(e), + Err(_) => {} } } - // Single-column PK aliases a `btree_indexes` entry (maintained above); - // a composite PK has its own index, maintained here. + if let Some(e) = first_error { + return Err(e); + } + self.mark_pk_overrides_if_needed(had_existing_pk); + + // Single-column PK aliases a `btree_indexes` entry — its task above already + // maintained it. A composite PK has its own index; maintain it here before the + // watermark advances so the visible prefix is fully indexed. let mut had_existing = false; for stored in batches { had_existing |= @@ -761,146 +1131,12 @@ impl IndexStore { } self.mark_pk_overrides_if_needed(had_existing); - // Update global watermark to the max batch position + // The indexed prefix now covers every batch up to and including the + // highest position in this call, so the count is that position plus one. let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap(); - self.advance_max_visible_batch_position(max_bp); - - Ok(()) - } - - /// Insert multiple batches into all indexes in parallel. - /// - /// Each individual index runs in its own thread, regardless of type. - /// This maximizes parallelism when multiple indexes are maintained. - /// - /// This is used during WAL flush to parallelize index updates with WAL I/O. - /// Insert batches into all indexes in parallel. - /// - /// Returns a map of index names to their update durations for performance tracking. - #[allow(clippy::print_stderr)] - #[instrument(name = "idx_insert_batches_parallel", level = "debug", skip_all, fields(batch_count = batches.len()))] - pub fn insert_batches_parallel( - &self, - batches: &[StoredBatch], - ) -> Result> { - use std::time::Instant; - - if batches.is_empty() { - return Ok(std::collections::HashMap::new()); - } + self.advance_indexed_count(max_bp + 1); - let track_pk_overrides = self.should_track_pk_overrides(); - // Use std::thread::scope for parallel CPU-bound work - std::thread::scope(|scope| { - // Each handle returns (index_name, index_type, duration, Result) - let mut handles: Vec<( - &str, - &str, - std::thread::ScopedJoinHandle<'_, (std::time::Duration, Result)>, - )> = Vec::new(); - - // Spawn a thread for each BTree index - for (name, index) in &self.btree_indexes { - let track_this_index = track_pk_overrides && self.is_single_pk_btree(index); - let handle = scope.spawn(move || -> (std::time::Duration, Result) { - let start = Instant::now(); - let result = (|| { - let mut had_existing = false; - for stored in batches { - if track_this_index { - had_existing |= index - .insert_and_report_existing(&stored.data, stored.row_offset)?; - } else { - index.insert(&stored.data, stored.row_offset)?; - } - } - Ok(had_existing) - })(); - (start.elapsed(), result) - }); - handles.push((name.as_str(), "btree", handle)); - } - - // Spawn a thread for each HNSW index - for (name, index) in &self.hnsw_indexes { - let handle = scope.spawn(move || -> (std::time::Duration, Result) { - let start = Instant::now(); - let result = index.insert_batches(batches).map(|_| false); - (start.elapsed(), result) - }); - handles.push((name.as_str(), "hnsw", handle)); - } - - // Spawn a thread for each FTS index - for (name, index) in &self.fts_indexes { - let handle = scope.spawn(move || -> (std::time::Duration, Result) { - let start = Instant::now(); - let result = (|| { - for stored in batches { - index.insert(&stored.data, stored.row_offset)?; - } - Ok(false) - })(); - (start.elapsed(), result) - }); - handles.push((name.as_str(), "fts", handle)); - } - - // Collect results, log timing, and check for errors. Keep the raw - // `Duration` so sub-millisecond timings (the steady-state case for - // BTree updates) are preserved instead of getting truncated to 0. - let mut first_error: Option = None; - let mut timings: Vec<(&str, &str, std::time::Duration)> = Vec::new(); - let mut had_existing_pk = false; - - for (name, idx_type, handle) in handles { - match handle.join() { - Ok((duration, Ok(had_existing))) => { - timings.push((name, idx_type, duration)); - had_existing_pk |= had_existing; - } - Ok((duration, Err(e))) => { - timings.push((name, idx_type, duration)); - if first_error.is_none() { - first_error = Some(e); - } - } - Err(_) => { - if first_error.is_none() { - first_error = - Some(Error::internal(format!("Index '{}' thread panicked", name))); - } - } - } - } - - if let Some(e) = first_error { - return Err(e); - } - self.mark_pk_overrides_if_needed(had_existing_pk); - - let duration_map: std::collections::HashMap = timings - .into_iter() - .map(|(name, _idx_type, duration)| (name.to_string(), duration)) - .collect(); - - // Single-column PK aliases a `btree_indexes` entry — its thread above - // already maintained it (and joined). A composite PK has its own - // index; maintain it here before the watermark advances so the - // visible prefix is fully indexed. - let mut had_existing = false; - for stored in batches { - had_existing |= - self.insert_composite_pk(&stored.data, stored.row_offset, track_pk_overrides)?; - } - self.mark_pk_overrides_if_needed(had_existing); - - // Update global watermark to the max batch position - let max_bp = batches.iter().map(|b| b.batch_position).max().unwrap(); - self.advance_max_visible_batch_position(max_bp); - - Ok(duration_map) - }) + Ok(duration_map) } /// Get a BTree index by name. @@ -941,9 +1177,20 @@ impl IndexStore { /// Searches through all FTS indexes to find one matching the field_id. /// Use this for column-to-index resolution (column → field_id → index). pub fn get_fts_by_field_id(&self, field_id: i32) -> Option<&FtsMemIndex> { - self.fts_indexes - .values() - .find(|idx| idx.field_id() == field_id) + self.get_fts_by_field_id_and_granularity( + field_id, + lance_index::scalar::inverted::DocumentGranularity::Row, + ) + } + + pub fn get_fts_by_field_id_and_granularity( + &self, + field_id: i32, + document_granularity: lance_index::scalar::inverted::DocumentGranularity, + ) -> Option<&FtsMemIndex> { + self.fts_indexes.values().find(|idx| { + idx.field_id() == field_id && idx.document_granularity() == document_granularity + }) } /// Get a BTree index by column name. @@ -963,9 +1210,40 @@ impl IndexStore { /// Get an FTS index by column name. pub fn get_fts_by_column(&self, column: &str) -> Option<&FtsMemIndex> { - self.fts_indexes + self.get_fts_by_column_and_granularity( + column, + lance_index::scalar::inverted::DocumentGranularity::Row, + ) + } + + pub fn get_fts_by_column_and_granularity( + &self, + column: &str, + document_granularity: lance_index::scalar::inverted::DocumentGranularity, + ) -> Option<&FtsMemIndex> { + self.fts_indexes.values().find(|idx| { + idx.column_name() == column && idx.document_granularity() == document_granularity + }) + } + + /// Return the distinct persisted document granularities for FTS indexes on + /// `column`, ordered from row to list-element. + pub fn fts_document_granularities_by_column( + &self, + column: &str, + ) -> Vec { + let mut granularities = self + .fts_indexes .values() - .find(|idx| idx.column_name() == column) + .filter(|index| index.column_name() == column) + .map(|index| index.document_granularity()) + .collect::>(); + granularities.sort_by_key(|document_granularity| match document_granularity { + lance_index::scalar::inverted::DocumentGranularity::Row => 0, + lance_index::scalar::inverted::DocumentGranularity::ListElement => 1, + }); + granularities.dedup(); + granularities } /// Check if the registry has any indexes. @@ -973,20 +1251,92 @@ impl IndexStore { self.btree_indexes.is_empty() && self.hnsw_indexes.is_empty() && self.fts_indexes.is_empty() } + /// Name every index this memtable carries, for diagnostics. + /// + /// Answers "is my fresh-tier vector search brute-force" — an absent + /// name is the whole explanation, and there is no other way to see it + /// from outside. Sorted so repeated calls compare cleanly; `HashMap` + /// iteration order alone would not. + pub fn index_names(&self) -> Vec { + let mut out: Vec = self + .btree_indexes + .keys() + .chain(self.hnsw_indexes.keys()) + .chain(self.fts_indexes.keys()) + .cloned() + .collect(); + out.sort(); + out + } + /// Get the total number of indexes. pub fn len(&self) -> usize { self.btree_indexes.len() + self.hnsw_indexes.len() + self.fts_indexes.len() } - /// Get the visibility watermark (max batch position safe to read). + /// Heap bytes held by every index in the registry. /// - /// Returns the highest batch position whose data is durable in the WAL - /// and therefore visible to scanners. Scanners snapshot this at plan - /// construction time so every plan runs against a stable cursor. + /// `MemTable::row_bytes` deliberately omits this — it sizes the flush + /// unit, which is row data. Callers budgeting *resident* memory must add it: + /// a configured HNSW index pre-allocates its whole graph on the first insert + /// and is charged for it from the moment it is configured (see + /// `HnswMemIndex::resident_bytes`), so it can dwarf a memtable's row bytes + /// while `row_bytes` still reads zero. + pub fn resident_bytes(&self) -> usize { + let btrees: usize = self + .btree_indexes + .values() + .map(|b| b.resident_bytes()) + .sum(); + let hnsw: usize = self.hnsw_indexes.values().map(|h| h.resident_bytes()).sum(); + let fts: usize = self.fts_indexes.values().map(|f| f.resident_bytes()).sum(); + // A `Single` PK aliases a `btree_indexes` entry, already counted above. + // A composite PK's index is held only here. + let pk = match &self.pk_index { + Some(PkIndex::Composite { index, .. }) => index.resident_bytes(), + Some(PkIndex::Single(_)) | None => 0, + }; + btrees + hnsw + fts + pk + } + + /// How many batches of this memtable have been fully indexed (exclusive + /// count; 0 before any batch is indexed). + /// + /// This is the *indexed* cursor, not the visibility watermark: it advances + /// once every index insert for a batch completes, regardless of WAL + /// durability. Readers must snapshot [`Self::visible_count`], which derives + /// what is safe to read from this cursor and the writer's durability cursor. + pub fn indexed_count(&self) -> usize { + self.indexed_count.load(Ordering::Acquire) + } + + /// The prefix of this memtable that readers may see. Snapshot this, never + /// `indexed_count`. /// - /// Returns 0 before any WAL flush has advanced the watermark. - pub fn max_visible_batch_position(&self) -> usize { - self.max_visible_batch_position.load(Ordering::Acquire) + /// Derived on every call from the two cursors rather than cached, so there is + /// no published value that can be left stale by a race between the two tasks + /// that advance them. A bare `IndexStore` has no writer, so its visible + /// prefix is simply what has been indexed. + pub fn visible_count(&self) -> usize { + let indexed = self.indexed_count(); + match &self.durability { + Some((cursors, global_offset)) => cursors.visible_count(indexed, *global_offset), + None => indexed, + } + } + + /// The prefix readable under `visibility`. + pub fn prefix_count(&self, visibility: MemTableVisibility) -> usize { + match visibility { + MemTableVisibility::Published => self.visible_count(), + MemTableVisibility::Indexed => self.indexed_count(), + } + } + + /// Bind this memtable's indexes to the writer's cursors. Called once at + /// construction, before the memtable is published. + pub(crate) fn set_durability(&mut self, cursors: Arc, global_offset: usize) { + self.durability = Some((cursors, global_offset)); } } @@ -994,26 +1344,60 @@ impl IndexStore { mod tests { use super::*; use arrow_array::{Int32Array, StringArray}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use log::warn; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + use lance_index::scalar::inverted::InvertedListFormatVersion; + use rstest::rstest; use std::sync::Arc; use uuid::Uuid; - /// Check if an index type is supported and log warning if not. - fn check_index_type_supported(index_type: &str) -> bool { - match index_type.to_lowercase().as_str() { - "btree" | "scalar" => true, - "hnsw" | "vector" => true, - "fts" | "inverted" | "fulltext" => true, - _ => { - warn!( - "Index type '{}' is not supported for MemWAL. \ - Supported types: btree, hnsw, fts. Skipping.", - index_type - ); - false - } + /// Matching is on the message-name suffix, not the whole url: `Any::from_msg` + /// emits the package (`/lance.table.`, `/lance.index.pb.`), while MemWAL flush + /// used to hand-write a `type.googleapis.com/` url that existing datasets + /// still carry. + #[rstest] + #[case::btree("/lance.table.BTreeIndexDetails", Some(MemIndexKind::BTree))] + #[case::fts("/lance.table.InvertedIndexDetails", Some(MemIndexKind::Fts))] + #[case::fts_legacy("/lance.index.pb.InvertedIndexDetails", Some(MemIndexKind::Fts))] + #[case::vector("/lance.index.pb.VectorIndexDetails", Some(MemIndexKind::Hnsw))] + // What MemWAL flush wrote before it switched to `Any::from_msg`. + #[case::vector_legacy_flush( + "type.googleapis.com/lance.index.VectorIndexDetails", + Some(MemIndexKind::Hnsw) + )] + #[case::bitmap("/lance.table.BitmapIndexDetails", None)] + #[case::label_list("/lance.table.LabelListIndexDetails", None)] + #[case::ngram("/lance.table.NGramIndexDetails", None)] + #[case::zone_map("/lance.table.ZoneMapIndexDetails", None)] + #[case::bloom_filter("/lance.index.pb.BloomFilterIndexDetails", None)] + #[case::json("/lance.index.pb.JsonIndexDetails", None)] + #[case::fm("/lance.index.pb.FMIndexDetails", None)] + #[case::absent("", None)] + fn type_urls_resolve_to_the_kind_the_writer_builds( + #[case] type_url: &str, + #[case] expected: Option, + ) { + assert_eq!(MemIndexKind::from_type_url(type_url), expected); + } + + /// `ALL` is hand-maintained, so a kind left out of it stops resolving. + #[test] + fn every_kind_is_registered_and_uniquely_identified() { + for kind in MemIndexKind::ALL { + assert_eq!( + MemIndexKind::from_type_url(&format!("/lance.table.{}", kind.details_suffix())), + Some(*kind), + "{kind:?} does not resolve from its own suffix", + ); } + let suffixes: std::collections::HashSet<_> = MemIndexKind::ALL + .iter() + .map(|k| k.details_suffix()) + .collect(); + assert_eq!( + suffixes.len(), + MemIndexKind::ALL.len(), + "two kinds share a details suffix, so one can never be resolved", + ); } fn create_test_schema() -> Arc { @@ -1040,22 +1424,46 @@ mod tests { .unwrap() } + fn create_sized_batch(schema: &ArrowSchema, start_id: i32, num_rows: usize) -> RecordBatch { + let ids: Vec = (0..num_rows as i32).map(|i| start_id + i).collect(); + let names: Vec = ids.iter().map(|id| format!("name-{id}")).collect(); + let descriptions: Vec = ids.iter().map(|id| format!("hello world {id}")).collect(); + RecordBatch::try_new( + Arc::new(schema.clone()), + vec![ + Arc::new(Int32Array::from(ids)), + Arc::new(StringArray::from(names)), + Arc::new(StringArray::from(descriptions)), + ], + ) + .unwrap() + } + fn fts_index_metadata(index_version: i32) -> IndexMetadata { - let details = - pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); - let mut value = Vec::new(); - details.encode(&mut value).unwrap(); + fts_index_metadata_with_details(index_version, None) + } + + fn fts_index_metadata_with_details( + index_version: i32, + details: Option, + ) -> IndexMetadata { + let index_details = details.map(|details| { + let mut value = Vec::new(); + details.encode(&mut value).unwrap(); + Arc::new(prost_types::Any { + type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(), + value, + }) + }); IndexMetadata { uuid: Uuid::new_v4(), fields: vec![2], + covering_fields: vec![], name: "desc_idx".to_string(), dataset_version: 1, fragment_bitmap: None, - index_details: Some(Arc::new(prost_types::Any { - type_url: "type.googleapis.com/lance.index.InvertedIndexDetails".to_string(), - value, - })), + index_details, index_version, created_at: None, base_id: None, @@ -1344,15 +1752,41 @@ mod tests { } #[test] - fn test_check_index_type_supported() { - assert!(check_index_type_supported("btree")); - assert!(check_index_type_supported("BTree")); - assert!(check_index_type_supported("hnsw")); - assert!(check_index_type_supported("vector")); - assert!(check_index_type_supported("fts")); - assert!(check_index_type_supported("inverted")); + fn fts_registry_routes_row_and_element_targets_independently() { + let mut registry = IndexStore::new(); + registry + .add_fts_with_params( + "tags_idx".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default(), + ) + .unwrap(); + registry + .add_fts_with_params( + "tags_element_idx".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default().document_granularity( + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ), + ) + .unwrap(); - assert!(!check_index_type_supported("unknown")); + assert_eq!( + registry.get_fts_by_column("tags").unwrap().column_name(), + "tags" + ); + assert_eq!( + registry + .get_fts_by_column_and_granularity( + "tags", + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ) + .unwrap() + .column_name(), + "tags" + ); } #[test] @@ -1364,6 +1798,7 @@ mod tests { (0, InvertedListFormatVersion::V1), (1, InvertedListFormatVersion::V1), (2, InvertedListFormatVersion::V2), + (3, InvertedListFormatVersion::V3), ] { let config = MemIndexConfig::fts_from_metadata(&fts_index_metadata(index_version), &schema) @@ -1386,13 +1821,106 @@ mod tests { let arrow_schema = create_test_schema(); let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); - let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(3), &schema).unwrap_err(); + let err = MemIndexConfig::fts_from_metadata(&fts_index_metadata(4), &schema).unwrap_err(); assert!( - err.to_string().contains("unsupported index_version 3"), + err.to_string().contains("unsupported index_version 4"), "{err}" ); } + #[test] + fn fts_from_metadata_accepts_element_document_v3_capability() { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + true, + ), + ])); + let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let tags = schema.field("tags").unwrap(); + for (block_size, expected_format_version) in [ + (128, InvertedListFormatVersion::V2), + (256, InvertedListFormatVersion::V3), + ] { + let params = InvertedIndexParams::default() + .block_size(block_size) + .unwrap() + .document_granularity( + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ); + let details = pbold::InvertedIndexDetails::try_from(¶ms).unwrap(); + let mut metadata = fts_index_metadata_with_details(3, Some(details)); + metadata.fields = vec![tags.id]; + let config = MemIndexConfig::fts_from_metadata(&metadata, &schema).unwrap(); + + let MemIndexConfig::Fts(config) = config else { + unreachable!("fts metadata should create an FTS config") + }; + assert_eq!(config.field_id, tags.id); + assert_eq!(config.column, "tags"); + assert_eq!( + config.params.get_document_granularity(), + lance_index::scalar::inverted::DocumentGranularity::ListElement + ); + assert_eq!( + config.params.resolved_format_version(), + expected_format_version + ); + } + } + + #[test] + fn fts_from_metadata_accepts_v3_with_legacy_block_size() { + let arrow_schema = create_test_schema(); + let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let mut legacy_details = + pbold::InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); + legacy_details.posting_format_version = None; + + for metadata in [ + fts_index_metadata(3), + fts_index_metadata_with_details(3, Some(legacy_details)), + ] { + let config = MemIndexConfig::fts_from_metadata(&metadata, &schema).unwrap(); + let MemIndexConfig::Fts(config) = config else { + unreachable!("FTS metadata should create an FTS config"); + }; + assert_eq!( + config.params.resolved_format_version(), + InvertedListFormatVersion::V3 + ); + assert_eq!(config.params.posting_block_size(), 128); + } + } + + #[test] + fn fts_from_metadata_accepts_v3_with_256_block_size() { + let arrow_schema = create_test_schema(); + let schema = LanceSchema::try_from(arrow_schema.as_ref()).unwrap(); + let params = InvertedIndexParams::default().block_size(256).unwrap(); + let details = pbold::InvertedIndexDetails::try_from(¶ms).unwrap(); + + let config = MemIndexConfig::fts_from_metadata( + &fts_index_metadata_with_details(3, Some(details)), + &schema, + ) + .unwrap(); + + match config { + MemIndexConfig::Fts(config) => { + assert_eq!( + config.params.resolved_format_version(), + InvertedListFormatVersion::V3 + ); + assert_eq!(config.params.posting_block_size(), 256); + } + _ => unreachable!("fts metadata should create an FTS config"), + } + } + #[test] fn test_from_configs() { let configs = vec![ @@ -1417,8 +1945,241 @@ mod tests { assert!(registry.get_fts_by_field_id(2).is_some()); } + /// The admission controller reads `IndexStore::resident_bytes` *before* the + /// insert that would allocate an HNSW graph, so a configured-but-untouched + /// vector index reporting zero would put the largest allocation in a vector + /// memtable outside its reach. It must be charged from configuration. + #[test] + fn test_resident_bytes_charges_hnsw_before_first_insert() { + let max_rows = 100_000; + let btree_only = IndexStore::from_configs( + &[MemIndexConfig::BTree(BTreeIndexConfig { + name: "pk_idx".to_string(), + field_id: 0, + column: "id".to_string(), + })], + max_rows, + 1_000, + ) + .unwrap(); + assert_eq!( + btree_only.resident_bytes(), + 0, + "a BTree index allocates per row, so an untouched one holds nothing" + ); + + let with_hnsw = IndexStore::from_configs( + &[MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "vec_idx".to_string(), + 2, + "vector".to_string(), + DistanceType::L2, + )))], + max_rows, + 1_000, + ) + .unwrap(); + assert!( + with_hnsw.resident_bytes() > max_rows * 128, + "the graph is sized from capacity and owed from configuration, got {}", + with_hnsw.resident_bytes() + ); + } + + fn vector_schema() -> Arc { + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("description", DataType::Utf8, true), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + Field::new( + "f64_vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float64, true)), 4), + true, + ), + ])) + } + + /// Every index config that would fail *deterministically* on insert must be + /// rejected at open instead. Such a config also fails on WAL replay, so once + /// a row is durable the shard could never reopen — poison-and-replay would + /// not terminate. + #[rstest] + #[case::btree_ok(MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), field_id: 0, column: "id".into(), + }), None)] + #[case::btree_missing_column(MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), field_id: 9, column: "nope".into(), + }), Some("not in the shard schema"))] + // Column exists, but its field_id names a *different* column ("id" is 0, not 1). + #[case::btree_field_id_column_mismatch(MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), field_id: 1, column: "id".into(), + }), Some("has field_id 0"))] + #[case::fts_ok(MemIndexConfig::Fts(FtsIndexConfig::new( + "idx".into(), 1, "description".into(), + )), None)] + #[case::fts_non_utf8(MemIndexConfig::Fts(FtsIndexConfig::new( + "idx".into(), 0, "id".into(), + )), Some("must resolve to Utf8, LargeUtf8, Utf8View, or JSON"))] + #[case::fts_missing_column(MemIndexConfig::Fts(FtsIndexConfig::new( + "idx".into(), 9, "nope".into(), + )), Some("does not exist in the dataset schema"))] + #[case::hnsw_ok(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 2, "vector".into(), DistanceType::L2, + ))), None)] + #[case::hnsw_not_a_vector(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 0, "id".into(), DistanceType::L2, + ))), Some("requires a FixedSizeList column"))] + #[case::hnsw_wrong_item_type(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 3, "f64_vector".into(), DistanceType::L2, + ))), Some("item type Float64"))] + #[case::hnsw_missing_column(MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "idx".into(), 9, "nope".into(), DistanceType::L2, + ))), Some("not in the shard schema"))] + fn test_validate_index_configs( + #[case] config: MemIndexConfig, + #[case] expected_error: Option<&str>, + ) { + let schema = vector_schema(); + let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap(); + let result = validate_index_configs(&[config], &schema, &lance_schema, &[]); + match expected_error { + None => result.expect("valid config must pass validation"), + Some(fragment) => { + let message = result + .expect_err("invalid config must be rejected") + .to_string(); + assert!( + message.contains(fragment), + "error must explain the mismatch; wanted {fragment:?}, got {message:?}" + ); + } + } + } + + #[test] + fn test_validate_nested_fts_index_config() { + let content_fields = Fields::from(vec![Field::new("content", DataType::Utf8, true)]); + let doc_item = Arc::new(Field::new("item", DataType::Struct(content_fields), true)); + let group_fields = Fields::from(vec![Field::new("docs", DataType::List(doc_item), true)]); + let group_item = Arc::new(Field::new("item", DataType::Struct(group_fields), true)); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "groups", + DataType::List(group_item), + true, + )])); + let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap(); + let resolved = crate::index::scalar::inverted::resolve_fts_field( + &lance_schema, + "groups.docs.content", + lance_index::scalar::inverted::DocumentGranularity::ListElement, + ) + .unwrap(); + + let params = InvertedIndexParams::default() + .document_granularity(lance_index::scalar::inverted::DocumentGranularity::ListElement); + let config = MemIndexConfig::Fts(FtsIndexConfig::with_params( + "idx".into(), + resolved.final_field_id, + "groups.docs.content".into(), + params.clone(), + )); + validate_index_configs(&[config], &schema, &lance_schema, &[]).unwrap(); + + let wrong_field_id = MemIndexConfig::Fts(FtsIndexConfig::with_params( + "idx".into(), + resolved.final_field_id + 1, + "groups.docs.content".into(), + params, + )); + let error = + validate_index_configs(&[wrong_field_id], &schema, &lance_schema, &[]).unwrap_err(); + assert!(error.to_string().contains("final field_id"), "{error}"); + } + + #[test] + fn test_validate_index_configs_rejects_diverged_lance_schema() { + let arrow_schema = ArrowSchema::new(vec![Field::new("id", DataType::Int32, false)]); + let lance_schema = LanceSchema::try_from(&ArrowSchema::new(vec![Field::new( + "other", + DataType::Int32, + false, + )])) + .expect("test Lance schema must be valid"); + let config = MemIndexConfig::BTree(BTreeIndexConfig { + name: "idx".into(), + field_id: 0, + column: "id".into(), + }); + + let error = validate_index_configs(&[config], &arrow_schema, &lance_schema, &[]) + .expect_err("diverged Arrow and Lance schemas must be rejected"); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + let message = error.to_string(); + assert!( + message.contains("index 'idx'"), + "error must name the index: {message}" + ); + assert!( + message.contains("column 'id'"), + "error must name the column: {message}" + ); + assert!( + message.contains("absent from the Lance schema"), + "error must explain the schema divergence: {message}" + ); + } + + /// A composite PK builds an order-preserving encoded key, so its columns must + /// be encodable. A single-column PK aliases a BTree entry, which accepts any + /// type — so it must *not* be rejected here. #[test] - fn test_index_store_max_visible_batch_position() { + fn test_validate_composite_pk_column_types() { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + Field::new( + "coords", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2), + true, + ), + ])); + let lance_schema = LanceSchema::try_from(schema.as_ref()).unwrap(); + + validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "name".into()]) + .expect("Int32 + Utf8 composite PK must be encodable"); + + let err = + validate_index_configs(&[], &schema, &lance_schema, &["id".into(), "coords".into()]) + .expect_err("a FixedSizeList PK column has no order-preserving encoding"); + assert!( + err.to_string().contains("order-preserving key encoding"), + "error must name the reason, got {err}" + ); + + // A single-column PK of the same type is fine: it aliases a BTree. + validate_index_configs(&[], &schema, &lance_schema, &["coords".into()]) + .expect("single-column PK aliases a BTree and accepts any type"); + + // But every PK column must exist. A single-column PK naming an absent + // column is rejected here, not left to fail deterministically on every + // later index build and WAL replay. + let err = validate_index_configs(&[], &schema, &lance_schema, &["missing".into()]) + .expect_err("a single-column PK on an absent column must be rejected"); + assert!( + err.to_string().contains("not in the shard schema"), + "error must name the missing column, got {err}" + ); + } + + #[test] + fn test_index_store_indexed_count() { let schema = create_test_schema(); let mut registry = IndexStore::new(); @@ -1427,7 +2188,7 @@ mod tests { registry.add_fts("desc_idx".to_string(), 2, "description".to_string()); // Initial watermark should be 0 (no data indexed yet) - assert_eq!(registry.max_visible_batch_position(), 0); + assert_eq!(registry.indexed_count(), 0); // Insert with batch position tracking let batch = create_test_batch(&schema, 0); @@ -1435,20 +2196,54 @@ mod tests { .insert_with_batch_position(&batch, 0, Some(5)) .unwrap(); - // Now watermark should be 5 - assert_eq!(registry.max_visible_batch_position(), 5); + // Indexing batch position 5 means the prefix [0, 6) is indexed. + assert_eq!(registry.indexed_count(), 6); // Insert with higher batch position registry .insert_with_batch_position(&batch, 3, Some(10)) .unwrap(); - // Watermark should advance to 10 - assert_eq!(registry.max_visible_batch_position(), 10); + // Advances to cover batch position 10. + assert_eq!(registry.indexed_count(), 11); - // Insert without batch position shouldn't change watermark + // Insert without batch position shouldn't change the cursor registry.insert(&batch, 6).unwrap(); - assert_eq!(registry.max_visible_batch_position(), 10); + assert_eq!(registry.indexed_count(), 11); + } + + /// `insert_batches` picks the inline or the threaded path by row count, so + /// exercise both and assert they leave the same index state: every row indexed + /// exactly once, in every index, with a timing reported for each. + #[rstest] + #[case::inline(8)] + #[case::threaded(PARALLEL_INDEX_MIN_ROWS + 64)] + fn test_insert_batches_indexes_every_row_once(#[case] num_rows: usize) { + let schema = create_test_schema(); + let mut registry = IndexStore::new(); + registry.add_btree("id_idx".to_string(), 0, "id".to_string()); + registry.add_fts("desc_idx".to_string(), 2, "description".to_string()); + + let batch = create_sized_batch(&schema, 0, num_rows); + let durations = registry + .insert_batches(&[StoredBatch::new(batch, 0, 2)]) + .unwrap(); + + assert_eq!(durations.len(), 2, "expected one timing per index"); + assert!(durations.contains_key("id_idx")); + assert!(durations.contains_key("desc_idx")); + + let btree = registry.get_btree("id_idx").unwrap(); + for id in 0..num_rows as i32 { + let positions = btree.get(&ScalarValue::Int32(Some(id))); + assert_eq!( + positions.len(), + 1, + "id={id} should be indexed exactly once, got {positions:?}" + ); + } + assert_eq!(registry.get_fts("desc_idx").unwrap().doc_count(), num_rows); + assert_eq!(registry.indexed_count(), 3); } #[test] diff --git a/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs b/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs index 6b7361e9f1b..ababd152340 100644 --- a/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs +++ b/rust/lance/src/dataset/mem_wal/index/arena_skiplist.rs @@ -103,11 +103,12 @@ impl Arena { } /// Bump-allocate `layout`. Caller must have exclusive access (single writer). - unsafe fn alloc(&mut self, layout: Layout) -> *mut u8 { + /// `allocated` accumulates chunk bytes; only the cold `grow` path touches it. + unsafe fn alloc(&mut self, layout: Layout, allocated: &AtomicUsize) -> *mut u8 { let align = layout.align(); let mut aligned = (self.cursor as usize).wrapping_add(align - 1) & !(align - 1); if self.cursor.is_null() || aligned + layout.size() > self.end as usize { - self.grow(layout); + self.grow(layout, allocated); aligned = (self.cursor as usize + align - 1) & !(align - 1); } self.cursor = (aligned + layout.size()) as *mut u8; @@ -116,7 +117,7 @@ impl Arena { /// Allocate a fresh chunk large enough for `layout` and make it current. #[cold] - unsafe fn grow(&mut self, layout: Layout) { + unsafe fn grow(&mut self, layout: Layout, allocated: &AtomicUsize) { let align = layout.align().max(64); let size = CHUNK_SIZE.max(layout.size().next_power_of_two()); let chunk_layout = Layout::from_size_align(size, align).expect("valid chunk layout"); @@ -126,6 +127,7 @@ impl Arena { } self.chunks .push((NonNull::new_unchecked(ptr), chunk_layout)); + allocated.fetch_add(size, Ordering::Relaxed); self.cursor = ptr; self.end = ptr.add(size); } @@ -154,6 +156,10 @@ struct SkipListCore { height: AtomicUsize, /// Number of entries. len: AtomicUsize, + /// Bytes of arena chunks allocated so far. Maintained here rather than read + /// off `arena.chunks` because the arena is writer-only; this is readable by + /// anyone. Only `Arena::grow` touches it, so it costs nothing per insert. + arena_bytes: AtomicUsize, } // SAFETY: `arena` (the only non-Sync field) is mutated exclusively by the single @@ -174,6 +180,7 @@ impl SkipListCore { arena: UnsafeCell::new(Arena::new()), height: AtomicUsize::new(1), len: AtomicUsize::new(0), + arena_bytes: AtomicUsize::new(0), } } @@ -303,7 +310,8 @@ impl SkipListWriter { // only mutator, so no link changes between read and publish. let layout = node_layout::(height); // SAFETY: single-writer exclusive access to the arena. - let node = unsafe { (*self.core.arena.get()).alloc(layout) } as *mut Node; + let node = unsafe { (*self.core.arena.get()).alloc(layout, &self.core.arena_bytes) } + as *mut Node; // SAFETY: `node` points to a fresh, uninitialized, correctly-sized and // -aligned block; we write the key then `height` tower slots. unsafe { @@ -342,6 +350,17 @@ pub struct SkipListReader { } impl SkipListReader { + /// Bytes of arena chunks backing this skiplist's nodes. + /// + /// Counts chunks, not entries, so it steps by `CHUNK_SIZE` and overshoots + /// the live nodes by at most one partly-filled chunk. Excludes any bytes a + /// key owns outside its node (e.g. a long `Box<[u8]>` key) — the arena + /// never sees those, so whoever built the key charges them; see + /// `BytesBackend::key_heap_bytes`. + pub(crate) fn resident_bytes(&self) -> usize { + self.core.arena_bytes.load(Ordering::Relaxed) + } + /// Greatest node with `key <= target`, mapped through `f` while it is alive. /// Equivalent to crossbeam's `upper_bound(Included(target))`. `None` if no /// such node. The closure avoids cloning the key on the hot path. diff --git a/rust/lance/src/dataset/mem_wal/index/btree.rs b/rust/lance/src/dataset/mem_wal/index/btree.rs index ca4dd178548..e4b49b03838 100644 --- a/rust/lance/src/dataset/mem_wal/index/btree.rs +++ b/rust/lance/src/dataset/mem_wal/index/btree.rs @@ -23,6 +23,7 @@ //! - [`ScalarBackend`] for everything else: the original `OrderableScalarValue` //! key (fat node, but handles arbitrary scalar types). +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, OnceLock}; use arrow_array::types::*; @@ -186,6 +187,16 @@ impl InlineBytes { Self::Heap(b) => b, } } + + /// Bytes this key owns outside its node. Inline keys own none; a spilled + /// key owns exactly its payload, since `Box<[u8]>` allocates no slack. + #[inline] + fn heap_bytes(&self) -> usize { + match self { + Self::Inline { .. } => 0, + Self::Heap(b) => b.len(), + } + } } impl PartialEq for InlineBytes { @@ -291,6 +302,9 @@ struct FixedIntBackend { writer: Mutex>, /// Row positions whose value is null (rare; not on the hot path). null_positions: Mutex>, + /// `null_positions`' heap, kept alongside it so a memory poll never has to + /// take that lock. See [`Backend::resident_bytes`]. + null_bytes: AtomicUsize, data_type: DataType, } @@ -301,6 +315,7 @@ impl FixedIntBackend { reader, writer: Mutex::new(writer), null_positions: Mutex::new(Vec::new()), + null_bytes: AtomicUsize::new(0), data_type, } } @@ -339,7 +354,15 @@ impl FixedIntBackend { } drop(writer); if !nulls.is_empty() { - self.null_positions.lock().unwrap().extend(nulls); + let mut positions = self.null_positions.lock().unwrap(); + // Reserve and charge before the extend, so the counter is + // never behind the positions a concurrent poll can reach. + positions.reserve(nulls.len()); + self.null_bytes.store( + positions.capacity() * std::mem::size_of::(), + Ordering::Relaxed, + ); + positions.extend(nulls); } }}; } @@ -457,7 +480,14 @@ struct BytesBackend { reader: SkipListReader, writer: Mutex>, null_positions: Mutex>, + /// `null_positions`' heap, kept alongside it so a memory poll never has to + /// take that lock. See [`Backend::resident_bytes`]. + null_bytes: AtomicUsize, data_type: DataType, + /// Payload of keys too long to live inline in their node. The skiplist's + /// own counter measures arena chunks only, so without this a column of long + /// strings would duplicate its whole payload uncharged. + key_heap_bytes: AtomicUsize, } impl BytesBackend { @@ -467,7 +497,9 @@ impl BytesBackend { reader, writer: Mutex::new(writer), null_positions: Mutex::new(Vec::new()), + null_bytes: AtomicUsize::new(0), data_type, + key_heap_bytes: AtomicUsize::new(0), } } @@ -498,6 +530,18 @@ impl BytesBackend { bytes: InlineBytes::new(bytes), position, }; + // Charge before publishing, the way the arena charges + // a chunk before the node that lives in it: the insert + // below splices the node in with `Release`, so a reader + // that can reach the key can also see its payload. A + // per-batch total added afterwards would leave a whole + // in-flight batch of keys visible but uncharged, and an + // admission sample landing there reads low. Inline keys + // own nothing, so they skip the atomic entirely. + let spilled = key.bytes.heap_bytes(); + if spilled > 0 { + self.key_heap_bytes.fetch_add(spilled, Ordering::Relaxed); + } had_existing |= writer.insert_and_check_neighbors(key, |prev, next| { prev.is_some_and(|key| key.bytes.as_slice() == bytes) || next.is_some_and(|key| key.bytes.as_slice() == bytes) @@ -506,7 +550,15 @@ impl BytesBackend { } drop(writer); if !nulls.is_empty() { - self.null_positions.lock().unwrap().extend(nulls); + let mut positions = self.null_positions.lock().unwrap(); + // Reserve and charge before the extend, so the counter is + // never behind the positions a concurrent poll can reach. + positions.reserve(nulls.len()); + self.null_bytes.store( + positions.capacity() * std::mem::size_of::(), + Ordering::Relaxed, + ); + positions.extend(nulls); } }}; } @@ -833,6 +885,22 @@ impl Backend { } } + /// Lock-free by construction: admission reads this on every put and on every + /// `DRAIN_POLL_INTERVAL` tick while a writer is parked, so taking the + /// `null_positions` mutex here would park a memory poll behind an in-flight + /// insert. + fn resident_bytes(&self) -> usize { + match self { + Self::FixedInt(b) => b.reader.resident_bytes() + b.null_bytes.load(Ordering::Relaxed), + Self::Bytes(b) => { + b.reader.resident_bytes() + + b.null_bytes.load(Ordering::Relaxed) + + b.key_heap_bytes.load(Ordering::Relaxed) + } + Self::Scalar(b) => b.reader.resident_bytes(), + } + } + fn data_type(&self) -> Option { match self { Self::FixedInt(b) => Some(b.data_type()), @@ -937,6 +1005,15 @@ impl BTreeMemIndex { self.backend.get().map(|b| b.len()).unwrap_or(0) } + /// Heap bytes held by this index; zero before the first insert. + /// + /// Grows with rows (unlike the pre-allocated HNSW index). Arena-chunk + /// granular, so it steps rather than climbs smoothly — plus the exact + /// payload of any key too long to live inline in its node. + pub(crate) fn resident_bytes(&self) -> usize { + self.backend.get().map(|b| b.resident_bytes()).unwrap_or(0) + } + /// Check if the index is empty. pub fn is_empty(&self) -> bool { self.len() == 0 @@ -1032,8 +1109,9 @@ pub struct BTreeIndexConfig { #[cfg(test)] mod tests { use super::*; - use arrow_array::{Int32Array, Int64Array, StringArray, UInt32Array}; + use arrow_array::{ArrayRef, Int32Array, Int64Array, StringArray, UInt32Array}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use rstest::rstest; use std::sync::Arc; fn create_test_schema() -> Arc { @@ -1270,6 +1348,124 @@ mod tests { assert_eq!(snapshot[2].0.0, ScalarValue::Int32(Some(2))); } + /// Keys longer than `INLINE_CAP` spill to a `Box<[u8]>` outside the + /// skiplist's arena, so the arena counter alone would leave an arbitrarily + /// large duplicate of the column uncharged. + #[test] + fn test_resident_bytes_counts_spilled_keys() { + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "s", + DataType::Utf8, + true, + )])); + let index = BTreeMemIndex::new(0, "s".to_string()); + + let rows = 256; + let width = 16 * 1024; + let values: Vec = (0..rows) + .map(|i| format!("{i:07}{}", "z".repeat(width - 7))) + .collect(); + let key_bytes = rows * width; + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from( + values.iter().map(|v| Some(v.as_str())).collect::>(), + ))], + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + assert!( + index.resident_bytes() >= key_bytes, + "resident {} must cover the {key_bytes} bytes of spilled key payload", + index.resident_bytes() + ); + } + + /// Null positions live behind a mutex the memory poll must never take, so + /// their heap is mirrored into an atomic. That mirror has to actually track + /// the vector, or a column of nulls goes uncharged against the ceiling. + #[rstest] + #[case::fixed_int(DataType::Int32)] + #[case::bytes(DataType::Utf8)] + fn test_resident_bytes_counts_null_positions(#[case] data_type: DataType) { + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "c", + data_type.clone(), + true, + )])); + let index = BTreeMemIndex::new(0, "c".to_string()); + + let rows = 1_024; + let column: ArrayRef = match data_type { + DataType::Int32 => Arc::new(Int32Array::from(vec![None::; rows])), + DataType::Utf8 => Arc::new(StringArray::from(vec![None::<&str>; rows])), + other => unreachable!("unhandled case {other:?}"), + }; + let batch = RecordBatch::try_new(schema, vec![column]).unwrap(); + index.insert(&batch, 0).unwrap(); + + let expected = rows * std::mem::size_of::(); + assert!( + index.resident_bytes() >= expected, + "resident {} must cover the {expected} bytes of null positions", + index.resident_bytes() + ); + } + + /// Charging the batch's total after the loop is not enough: each key is + /// reachable to lock-free readers the moment it is spliced in, so an + /// admission sample landing mid-batch would see a growing index against a + /// stale byte total and admit a write it should have refused. The charge + /// has to land before the key it pays for. + #[test] + fn test_resident_bytes_covers_keys_already_published() { + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "s", + DataType::Utf8, + false, + )])); + let rows = 2_000usize; + let width = 8 * 1024usize; + let values: Vec = (0..rows) + .map(|i| format!("{i:07}{}", "z".repeat(width - 7))) + .collect(); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(StringArray::from(values)) as Arc], + ) + .unwrap(); + + let index = Arc::new(BTreeMemIndex::new(0, "s".to_string())); + let inserting = Arc::clone(&index); + let handle = std::thread::spawn(move || inserting.insert(&batch, 0).unwrap()); + + // Sample until the insert is partway through. Every sample that catches + // it there must already account for the keys it can see; the loop is + // only about *reaching* that state, so the assertion is inside it. + let mut sampled_mid_insert = false; + while !handle.is_finished() { + let published = index.len(); + if published == 0 || published >= rows { + std::hint::spin_loop(); + continue; + } + sampled_mid_insert = true; + let charged = index.resident_bytes(); + assert!( + charged >= published * width, + "{published} keys are visible but only {charged} bytes are charged" + ); + } + handle.join().unwrap(); + + assert!( + sampled_mid_insert, + "the insert never became observable partway through, so nothing was proven" + ); + assert!(index.resident_bytes() >= rows * width); + } + #[test] fn test_bytes_backend_strings() { let schema = Arc::new(ArrowSchema::new(vec![Field::new( diff --git a/rust/lance/src/dataset/mem_wal/index/fts.rs b/rust/lance/src/dataset/mem_wal/index/fts.rs index 61c797db041..50ab8cc3b36 100644 --- a/rust/lance/src/dataset/mem_wal/index/fts.rs +++ b/rust/lance/src/dataset/mem_wal/index/fts.rs @@ -47,26 +47,26 @@ use std::cmp::Reverse; use std::collections::{BinaryHeap, HashMap, HashSet}; -use std::sync::Arc; -use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; use arc_swap::ArcSwap; -use arrow_array::{Array, LargeStringArray, RecordBatch, StringArray, StringViewArray}; -use arrow_schema::DataType; +use arrow_array::{Array, RecordBatch, UInt64Array}; use crossbeam_skiplist::SkipMap; use fst::{Map, Streamer}; use lance_bitpacking::{BitPacker, BitPacker4x}; +use lance_core::datatypes::Schema as LanceSchema; use lance_core::{Error, Result}; use lance_index::scalar::InvertedIndexParams; -use lance_index::scalar::inverted::query::Operator; -use lance_index::scalar::inverted::tokenizer::document_tokenizer::LanceTokenizer; +use lance_index::scalar::inverted::query::{FtsQuery, Operator, Tokens}; +use lance_index::scalar::inverted::tokenizer::document_tokenizer::{DocType, LanceTokenizer}; use lance_index::scalar::inverted::{DocSet, MemBM25Scorer, Scorer, TokenSet}; use lance_tokenizer::TokenStream; use rayon::prelude::*; -use rustc_hash::FxHashMap; +use rustc_hash::{FxHashMap, FxHashSet}; use super::RowPosition; +use crate::index::scalar::inverted::{ResolvedFtsField, resolve_fts_field}; // ============================================================================ // Public types preserved from previous API @@ -77,10 +77,46 @@ use super::RowPosition; pub struct FtsEntry { /// Row position in MemTable. pub row_position: RowPosition, + /// Root-to-leaf physical list ordinals for a ListElement document. + pub doc_index: Option>, /// BM25 score for this document. pub score: f32, } +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct DocumentKey { + row_position: RowPosition, + doc_index: Vec, +} + +impl FtsEntry { + fn key(&self) -> DocumentKey { + DocumentKey { + row_position: self.row_position, + doc_index: self.doc_index.clone().unwrap_or_default(), + } + } +} + +#[derive(Debug, Clone)] +struct DocumentMetadata { + key: DocumentKey, + num_tokens: u32, +} + +fn doc_set_key(docs: &DocSet, doc_id: u32) -> DocumentKey { + DocumentKey { + row_position: docs.row_id(doc_id), + doc_index: (0..docs.coordinate_rank()) + .map(|rank| docs.coordinate(doc_id, rank)) + .collect(), + } +} + +fn public_doc_index(coordinates: &[u32]) -> Option> { + (!coordinates.is_empty()).then(|| coordinates.to_vec()) +} + /// Full-text search query expression for composable queries. #[derive(Debug, Clone)] pub enum FtsQueryExpr { @@ -118,7 +154,7 @@ pub enum FtsQueryExpr { }, /// Boolean combination of queries. Boolean { - /// All MUST clauses must match for a document to be included. + /// All MUST clauses must match and contribute to the score. must: Vec, /// At least one SHOULD clause should match (adds to score). should: Vec, @@ -371,6 +407,41 @@ fn char_prefix(term: &str, prefix_length: u32) -> &str { .unwrap_or(term) } +fn query_tokens_to_vec(tokens: &Tokens) -> Vec { + (0..tokens.len()) + .map(|idx| tokens.get_token(idx).to_string()) + .collect() +} + +fn has_grouped_positions(tokens: &Tokens) -> bool { + let mut seen = HashSet::new(); + (0..tokens.len()).any(|idx| !seen.insert(tokens.position(idx))) +} + +fn query_position_groups(tokens: &Tokens) -> Vec> { + let mut groups = Vec::new(); + let mut current_position = None; + for idx in 0..tokens.len() { + let position = tokens.position(idx); + if current_position != Some(position) { + current_position = Some(position); + groups.push(Vec::new()); + } + let group = groups + .last_mut() + .expect("a group should exist after pushing for position"); + let token = tokens.get_token(idx).to_string(); + if !group.contains(&token) { + group.push(token); + } + } + groups +} + +fn position_groups_to_tokens(groups: &[Vec]) -> Vec { + groups.iter().flatten().cloned().collect() +} + /// Builder for constructing Boolean queries. #[derive(Debug, Clone, Default)] pub struct BooleanQueryBuilder { @@ -429,7 +500,7 @@ impl Positions { &self.data[start..end] } - fn memory_size(&self) -> usize { + fn resident_bytes(&self) -> usize { self.offsets.capacity() * std::mem::size_of::() + self.data.capacity() * std::mem::size_of::() } @@ -461,11 +532,11 @@ impl TermChunk { self.row_positions.len() } - fn memory_size(&self) -> usize { + fn resident_bytes(&self) -> usize { let base = std::mem::size_of::() + self.row_positions.capacity() * std::mem::size_of::() + self.frequencies.capacity() * std::mem::size_of::(); - base + self.positions.as_ref().map_or(0, Positions::memory_size) + base + self.positions.as_ref().map_or(0, Positions::resident_bytes) } } @@ -508,10 +579,10 @@ impl TermSlice { TermChunkIter { cur: Some(self) } } - fn memory_size(&self) -> usize { + fn resident_bytes(&self) -> usize { // Each node: the struct itself plus its chunk's payload. self.chunks() - .map(|c| std::mem::size_of::() + c.memory_size()) + .map(|c| std::mem::size_of::() + c.resident_bytes()) .sum::() + std::mem::size_of::() // empty root node } @@ -537,30 +608,38 @@ impl<'a> Iterator for TermChunkIter<'a> { } } -/// Per-batch row metadata. +/// Per-batch document metadata. #[derive(Debug)] struct BatchMeta { batch_position: usize, - row_offset: u64, - /// `doc_lengths[i]` is the token count of the row at `row_offset + i`. - doc_lengths: Vec, - rows: u32, + /// Dense tail-local document position of `documents[0]`. + document_position_start: u64, + documents: Vec, } impl BatchMeta { - fn dl(&self, row_position: u64) -> Option { - if row_position < self.row_offset { + fn document(&self, document_position: u64) -> Option<&DocumentMetadata> { + if document_position < self.document_position_start { return None; } - let idx = (row_position - self.row_offset) as usize; - self.doc_lengths.get(idx).copied() + let idx = (document_position - self.document_position_start) as usize; + self.documents.get(idx) + } + + fn dl(&self, document_position: u64) -> Option { + self.document(document_position).map(|doc| doc.num_tokens) } - fn memory_size(&self) -> usize { - std::mem::size_of::() + self.doc_lengths.capacity() * std::mem::size_of::() + fn resident_bytes(&self) -> usize { + std::mem::size_of::() + + self.documents.capacity() * std::mem::size_of::() } } +/// Per-entry overhead charged for a `SkipMap` node (tower + links) when sizing +/// the tail's term map. An estimate — the node layout is crossbeam-internal. +const SKIPMAP_ENTRY_OVERHEAD: usize = 32; + /// Size of a sealed batch block. Small enough that copying the partial tail /// block on append stays cheap; large enough that sealing (which clones the /// block-pointer vec) is rare. @@ -639,9 +718,9 @@ struct Snapshot { /// visible_count` for any snapshot the writer has stored (each publish /// appends one entry and bumps `visible_count`). batches: BatchLog, - /// `Σ batches[i].rows` for `i < visible_count`. + /// `Σ batches[i].documents.len()` for `i < visible_count`. cumulative_doc_count: u64, - /// `Σ batches[i].doc_lengths.iter().sum()` for `i < visible_count`. + /// `Σ batches[i].documents[*].num_tokens` for `i < visible_count`. cumulative_total_tokens: u64, } @@ -691,12 +770,15 @@ impl std::fmt::Debug for TokenizerPool { impl TokenizerPool { fn new(params: &InvertedIndexParams, cap: usize) -> Result { - let template = params.build()?; - Ok(Self { + Ok(Self::from_template(params.build()?, cap)) + } + + fn from_template(template: Box, cap: usize) -> Self { + Self { template, free: Mutex::new(Vec::new()), cap: cap.max(1), - }) + } } /// Acquire a tokenizer. Pops from the free list, otherwise clones the @@ -770,6 +852,11 @@ struct TailIndex { /// hash probe instead of a skiplist search. Reset implicitly when the tail /// is replaced on freeze. Uncontended — the single writer holds it briefly. writer_term_cache: Mutex, Arc>>>, + /// Running total mirroring [`Self::resident_bytes`], maintained by + /// `append_batch`. Exists so the write path can budget memtable memory + /// without the O(terms) walk. `test_tail_bytes_tracks_memory_size` pins the + /// two together. + bytes: AtomicUsize, } impl TailIndex { @@ -779,9 +866,15 @@ impl TailIndex { snapshot: ArcSwap::from(Snapshot::empty()), next_batch_position: AtomicUsize::new(0), writer_term_cache: Mutex::new(FxHashMap::default()), + bytes: AtomicUsize::new(std::mem::size_of::()), }) } + /// [`Self::resident_bytes`] without the walk. See [`Self::bytes`]. + fn resident_bytes_cached(&self) -> usize { + self.bytes.load(Ordering::Relaxed) + } + fn snapshot(&self) -> Arc { self.snapshot.load_full() } @@ -804,9 +897,8 @@ impl TailIndex { fn append_batch( &self, batch_position: usize, - row_offset: u64, - rows: u32, - doc_lengths: Vec, + document_position_start: u64, + documents: Vec, total_tokens: u64, term_builders: FxHashMap, BatchTermBuilder>, with_position: bool, @@ -815,8 +907,19 @@ impl TailIndex { .writer_term_cache .lock() .expect("writer term cache poisoned — single-writer invariant violated"); + // Mirrors `memory_size`'s per-term arithmetic; keep the two in step. + let mut added = 0; for (term, builder) in term_builders { let chunk = builder.build(batch_position, with_position); + added += std::mem::size_of::() + chunk.resident_bytes(); + if !cache.contains_key(&term) { + // First sight this generation: the SkipMap entry plus the + // slice's empty root node. + added += std::mem::size_of::>() + + term.len() + + SKIPMAP_ENTRY_OVERHEAD + + std::mem::size_of::(); + } // First sight of the term this generation populates the SkipMap // (so readers can find it) and caches the slot; later batches hit // only the cache. @@ -830,34 +933,37 @@ impl TailIndex { slot.store(TermSlice::push(cur, chunk)); } drop(cache); + let document_count = documents.len() as u64; let new_meta = Arc::new(BatchMeta { batch_position, - row_offset, - doc_lengths, - rows, + document_position_start, + documents, }); + added += new_meta.resident_bytes(); + self.bytes.fetch_add(added, Ordering::Relaxed); let cur = self.snapshot.load(); + debug_assert_eq!(document_position_start, cur.cumulative_doc_count); self.snapshot.store(Arc::new(Snapshot { visible_count: cur.visible_count + 1, batches: cur.batches.pushed(new_meta), - cumulative_doc_count: cur.cumulative_doc_count + rows as u64, + cumulative_doc_count: cur.cumulative_doc_count + document_count, cumulative_total_tokens: cur.cumulative_total_tokens + total_tokens, })); } - fn memory_size(&self) -> usize { + fn resident_bytes(&self) -> usize { let mut total = std::mem::size_of::(); for entry in self.terms.iter() { let term: &Arc = entry.key(); - total += std::mem::size_of::>() + term.len() + 32; - total += entry.value().load().memory_size(); + total += std::mem::size_of::>() + term.len() + SKIPMAP_ENTRY_OVERHEAD; + total += entry.value().load().resident_bytes(); } total += self .snapshot .load() .batches .iter() - .map(|b| b.memory_size()) + .map(|b| b.resident_bytes()) .sum::(); total } @@ -884,8 +990,9 @@ impl IndexState { /// model and visibility contract. pub struct FtsMemIndex { field_id: i32, - column_name: String, + source_column_name: String, params: InvertedIndexParams, + resolved_field: OnceLock, tokenizer_pool: Arc, /// Writer-only tokenizer slot. Held under a Mutex purely so `insert` @@ -899,6 +1006,11 @@ pub struct FtsMemIndex { /// The tail freezes into a partition once it reaches this many docs. freeze_threshold_rows: usize, + /// Query-local materializations disable freezes and tiered merges. Their + /// lifetime is bounded by one query, so background maintenance would only + /// outlive cancellation without providing reuse. + background_maintenance: bool, + /// Background tiered-merge slot. `None` = idle; `Some` with `result: None` /// = a merge is running on a worker thread; `Some` with `result: Some` = /// the merged partition is ready for the writer to install. Only the @@ -907,6 +1019,154 @@ pub struct FtsMemIndex { merge: Arc>>, } +/// Query-owned term-only postings for one residual scan. +/// +/// This deliberately exposes only the immutable feature-materialization API +/// needed by hybrid execution. Unlike [`FtsMemIndex`], it never freezes or +/// starts a detached tiered merge; dropping the query drops all residual +/// postings. +#[derive(Debug)] +pub struct QueryLocalFtsIndex { + inner: FtsMemIndex, +} + +#[derive(Debug, Default)] +pub struct QueryLocalFtsStats { + doc_count: usize, + total_tokens: u64, + token_docs: FxHashMap, +} + +impl QueryLocalFtsStats { + pub(crate) fn checked_add_assign(&mut self, other: Self) -> Result<()> { + self.doc_count = self + .doc_count + .checked_add(other.doc_count) + .ok_or_else(|| Error::internal("query-local FTS document count overflow"))?; + self.total_tokens = self + .total_tokens + .checked_add(other.total_tokens) + .ok_or_else(|| Error::internal("query-local FTS total token count overflow"))?; + for (token, df) in other.token_docs { + let current = self.token_docs.entry(token).or_default(); + *current = current + .checked_add(df) + .ok_or_else(|| Error::internal("query-local FTS term document count overflow"))?; + } + Ok(()) + } + + pub(crate) fn add_to_scorer(&self, scorer: &mut MemBM25Scorer) -> Result<()> { + scorer.num_docs = scorer + .num_docs + .checked_add(self.doc_count) + .ok_or_else(|| Error::internal("residual BM25 document count overflow"))?; + scorer.total_tokens = scorer + .total_tokens + .checked_add(self.total_tokens) + .ok_or_else(|| Error::internal("residual BM25 total token count overflow"))?; + for (token, df) in &self.token_docs { + let current = scorer.token_docs.entry(token.clone()).or_default(); + *current = current + .checked_add(*df) + .ok_or_else(|| Error::internal("residual BM25 term document count overflow"))?; + } + Ok(()) + } +} + +impl QueryLocalFtsIndex { + #[cfg(test)] + pub(crate) fn try_with_params( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + ) -> Result { + Ok(Self { + inner: FtsMemIndex::try_with_params_and_maintenance( + field_id, + column_name, + params, + false, + )?, + }) + } + + pub(crate) fn try_with_loaded_tokenizer( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + tokenizer: Box, + ) -> Result { + params.validate_format_version()?; + let pool = TokenizerPool::from_template(tokenizer, FtsMemIndex::DEFAULT_TOKENIZER_POOL_CAP); + Ok(Self { + inner: FtsMemIndex::with_tokenizer_pool_and_maintenance( + field_id, + column_name, + params, + pool, + false, + ), + }) + } + + /// Create an empty query-local shard without rebuilding tokenizer assets. + /// + /// The tokenizer pool and its loaded template are shared with the seed; + /// each shard only clones a writer tokenizer from that in-memory template. + pub(crate) fn empty_sibling(&self) -> Self { + let resolved_field = OnceLock::new(); + if let Some(resolved) = self.inner.resolved_field.get() { + resolved_field + .set(resolved.clone()) + .expect("new query-local shard traversal is empty"); + } + + Self { + inner: FtsMemIndex { + field_id: self.inner.field_id, + source_column_name: self.inner.source_column_name.clone(), + params: self.inner.params.clone(), + resolved_field, + tokenizer_pool: self.inner.tokenizer_pool.clone(), + writer_tokenizer: Mutex::new(self.inner.tokenizer_pool.acquire()), + state: ArcSwap::from(IndexState::empty()), + freeze_threshold_rows: self.inner.freeze_threshold_rows, + background_maintenance: false, + merge: Arc::new(Mutex::new(None)), + }, + } + } + + pub(crate) fn exact_query_terms(&self, query: &FtsQuery) -> Result> { + self.inner.exact_query_terms(query) + } + + pub(crate) fn insert_with_row_ids_for_terms( + &self, + batch: &RecordBatch, + row_ids: &UInt64Array, + terms: &FxHashSet, + ) -> Result { + self.inner + .insert_with_row_ids_for_terms(batch, row_ids, terms) + } + + #[cfg(test)] + fn doc_count(&self) -> usize { + self.inner.doc_count() + } + + pub(crate) fn exact_leaf_results( + &self, + query: &FtsQuery, + scorer: &MemBM25Scorer, + ) -> Result>> { + self.inner.exact_leaf_results(query, scorer) + } +} + /// A tiered merge dispatched to a background worker. struct PendingMerge { /// `Arc::as_ptr` of each source partition, for identity-matching the @@ -916,12 +1176,29 @@ struct PendingMerge { result: Option>, } +fn publish_pending_merge(slot: &Mutex>, merged: Result) { + let Ok(mut guard) = slot.lock() else { + return; + }; + match merged { + Ok(merged) => { + if let Some(pending) = guard.as_mut() { + pending.result = Some(Arc::new(merged)); + } + } + Err(error) => { + tracing::error!(?error, "background FTS tiered merge failed"); + *guard = None; + } + } +} + impl std::fmt::Debug for FtsMemIndex { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let st = self.state.load(); f.debug_struct("FtsMemIndex") .field("field_id", &self.field_id) - .field("column_name", &self.column_name) + .field("source_column_name", &self.source_column_name) .field("doc_count", &self.doc_count()) .field("partitions", &st.partitions.len()) .field("params", &self.params) @@ -960,21 +1237,71 @@ impl FtsMemIndex { /// Create a new FTS index with custom tokenizer parameters. pub fn with_params(field_id: i32, column_name: String, params: InvertedIndexParams) -> Self { - let pool = TokenizerPool::new(¶ms, Self::DEFAULT_TOKENIZER_POOL_CAP) - .expect("Failed to build tokenizer"); + Self::try_with_params(field_id, column_name, params) + .expect("invalid MemWAL FTS index parameters") + } + + /// Try to create a new FTS index with custom tokenizer parameters. + pub fn try_with_params( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + ) -> Result { + Self::try_with_params_and_maintenance(field_id, column_name, params, true) + } + + fn try_with_params_and_maintenance( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + background_maintenance: bool, + ) -> Result { + params.validate_format_version()?; + let pool = TokenizerPool::new(¶ms, Self::DEFAULT_TOKENIZER_POOL_CAP)?; + Ok(Self::with_tokenizer_pool_and_maintenance( + field_id, + column_name, + params, + pool, + background_maintenance, + )) + } + + fn with_tokenizer_pool_and_maintenance( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + pool: TokenizerPool, + background_maintenance: bool, + ) -> Self { let writer_tokenizer = pool.template.box_clone(); Self { field_id, - column_name, + source_column_name: column_name, params, + resolved_field: OnceLock::new(), tokenizer_pool: Arc::new(pool), writer_tokenizer: Mutex::new(writer_tokenizer), state: ArcSwap::from(IndexState::empty()), freeze_threshold_rows: Self::DEFAULT_FREEZE_THRESHOLD_ROWS, + background_maintenance, merge: Arc::new(Mutex::new(None)), } } + pub(crate) fn try_with_resolved_field( + field_id: i32, + column_name: String, + params: InvertedIndexParams, + resolved_field: ResolvedFtsField, + ) -> Result { + let index = Self::try_with_params(field_id, column_name, params)?; + index.resolved_field.set(resolved_field).map_err(|_| { + Error::internal("MemWAL FTS traversal was initialized twice".to_string()) + })?; + Ok(index) + } + /// Override the tail freeze threshold (docs) — the analogue of Lucene's /// `ramBufferSizeMB`. Larger keeps more rows in the un-indexed mutable tail /// (cheaper writes, costlier read-your-writes scans); smaller freezes into @@ -989,13 +1316,21 @@ impl FtsMemIndex { } pub fn column_name(&self) -> &str { - &self.column_name + &self.source_column_name + } + + pub fn source_column_name(&self) -> &str { + &self.source_column_name } pub fn params(&self) -> &InvertedIndexParams { &self.params } + pub fn document_granularity(&self) -> lance_index::scalar::inverted::DocumentGranularity { + self.params.get_document_granularity() + } + /// Number of visible documents across all partitions and the tail. pub fn doc_count(&self) -> usize { let st = self.state.load(); @@ -1005,7 +1340,7 @@ impl FtsMemIndex { /// Whether there are any visible documents. pub fn is_empty(&self) -> bool { let st = self.state.load(); - st.partitions.is_empty() && st.tail.visible_count() == 0 + st.partitions.is_empty() && st.tail.doc_count() == 0 } /// Total number of visible (term, doc) postings. @@ -1030,14 +1365,33 @@ impl FtsMemIndex { } /// Estimated bytes of heap memory held by this index. - pub fn memory_usage(&self) -> usize { + /// + /// Walks every tail term. Prefer `resident_bytes` on the write path. + pub fn resident_bytes_exact(&self) -> usize { let st = self.state.load_full(); let mut total = std::mem::size_of::(); - total += st.partitions.iter().map(|p| p.memory_size()).sum::(); - total += st.tail.memory_size(); + total += st + .partitions + .iter() + .map(|p| p.resident_bytes()) + .sum::(); + total += st.tail.resident_bytes(); total } + /// [`Self::resident_bytes_exact`] without the per-term walk: partitions are capped + /// at `MAX_PARTITIONS` and size themselves in O(1), and the tail keeps a + /// running total. Cheap enough for the write path. + pub(crate) fn resident_bytes(&self) -> usize { + let st = self.state.load(); + std::mem::size_of::() + + st.partitions + .iter() + .map(|p| p.resident_bytes()) + .sum::() + + st.tail.resident_bytes_cached() + } + /// Component memory breakdown (bytes), for diagnostics: /// `(num_partitions, term_strings, postings_meta, block_meta, doc_freq, pos, docs, tail)`. pub fn memory_breakdown(&self) -> (usize, usize, usize, usize, usize, usize, usize, usize) { @@ -1059,7 +1413,7 @@ impl FtsMemIndex { df, pos, docs, - st.tail.memory_size(), + st.tail.resident_bytes(), ) } @@ -1084,32 +1438,61 @@ impl FtsMemIndex { self.insert_batch(batch, row_offset) } - fn insert_batch(&self, batch: &RecordBatch, row_offset: u64) -> Result<()> { - let st = self.state.load_full(); - let batch_position = st.tail.next_position(); + /// Insert explicit, potentially non-contiguous rows while retaining + /// postings only for query terms. + /// The tokenizer still visits the complete document so BM25 document + /// lengths remain accurate when scoring with committed-index statistics. + pub(crate) fn insert_with_row_ids_for_terms( + &self, + batch: &RecordBatch, + row_ids: &UInt64Array, + terms: &FxHashSet, + ) -> Result { + if row_ids.len() != batch.num_rows() || row_ids.null_count() != 0 { + return Err(Error::invalid_input(format!( + "MemWAL FTS explicit row ids require {} non-null values, got len={} nulls={}", + batch.num_rows(), + row_ids.len(), + row_ids.null_count() + ))); + } + self.insert_batch_with_keys(batch, |row_index| Ok(row_ids.value(row_index)), Some(terms)) + } - let Some(col_idx) = batch - .schema() - .column_with_name(&self.column_name) - .map(|(idx, _)| idx) - else { - // Column missing: nothing to index, but publish an empty batch so - // the tail's visibility counters keep up with the writer. - st.tail.append_batch( - batch_position, - row_offset, - batch.num_rows() as u32, - vec![0; batch.num_rows()], - 0, - FxHashMap::default(), - self.params.has_positions(), - ); - return Ok(()); - }; + fn insert_batch(&self, batch: &RecordBatch, row_offset: u64) -> Result<()> { + self.insert_batch_with_keys( + batch, + |row_index| { + row_offset + .checked_add(row_index as u64) + .ok_or_else(|| Error::invalid_input("MemWAL FTS row position overflow")) + }, + None, + ) + .map(|_| ()) + } - let column = batch.column(col_idx); - let texts = extract_texts(column.as_ref())?; - debug_assert_eq!(texts.len(), batch.num_rows()); + fn insert_batch_with_keys( + &self, + batch: &RecordBatch, + row_position: impl Fn(usize) -> Result, + allowed_terms: Option<&FxHashSet>, + ) -> Result { + let st = self.state.load_full(); + let document_position_start = st.tail.doc_count(); + if self.resolved_field.get().is_none() { + let schema = LanceSchema::try_from(batch.schema().as_ref())?; + let resolved = resolve_fts_field( + &schema, + &self.source_column_name, + self.params.get_document_granularity(), + )?; + let _ = self.resolved_field.set(resolved); + } + let resolved = self.resolved_field.get().ok_or_else(|| { + Error::internal("MemWAL FTS traversal was not initialized".to_string()) + })?; + let extracted_documents = resolved.documents_from_batch(batch)?; let mut tok_guard = self .writer_tokenizer @@ -1122,93 +1505,252 @@ impl FtsMemIndex { // per-document map and per-`(term, doc)` `Vec` allocation that // dominated insert cost. `FxHashMap` skips SipHash on the hot lookup. let mut term_builders: FxHashMap, BatchTermBuilder> = FxHashMap::default(); - let mut doc_lengths: Vec = Vec::with_capacity(batch.num_rows()); + let mut documents: Vec = if allowed_terms.is_some() { + Vec::new() + } else { + Vec::with_capacity(batch.num_rows()) + }; let mut total_tokens: u64 = 0; + let mut query_local_corpus_doc_count = 0usize; + let mut query_local_corpus_total_tokens = 0u64; + let preserve_zero_token_documents = + self.params.get_document_granularity().is_list_element(); + let mut index_document = |key: DocumentKey, text: &str| -> Result<()> { + let document_position = document_position_start + documents.len() as u64; + let (num_tokens, retained_term) = match allowed_terms { + Some(allowed_terms) => index_text_filtered( + text, + document_position, + tokenizer, + &mut term_builders, + allowed_terms, + )?, + None => ( + index_text(text, document_position, tokenizer, &mut term_builders)?, + false, + ), + }; + let belongs_in_corpus = preserve_zero_token_documents || num_tokens > 0; + if allowed_terms.is_some() && belongs_in_corpus { + query_local_corpus_doc_count = query_local_corpus_doc_count + .checked_add(1) + .ok_or_else(|| Error::internal("query-local FTS document count overflow"))?; + query_local_corpus_total_tokens = query_local_corpus_total_tokens + .checked_add(num_tokens as u64) + .ok_or_else(|| Error::internal("query-local FTS total token count overflow"))?; + } + let retain_document = if allowed_terms.is_some() { + retained_term + } else { + belongs_in_corpus + }; + if retain_document { + documents.push(DocumentMetadata { key, num_tokens }); + total_tokens += num_tokens as u64; + } + Ok(()) + }; - for (local_doc_idx, text_opt) in texts.iter().enumerate() { - // Track each doc's token count even for null/missing rows so the - // dense `doc_lengths` array stays aligned with `row_offset + i`. - let mut doc_token_count: u32 = 0; - let row_position = row_offset + local_doc_idx as u64; - - if let Some(text) = text_opt { - let mut stream = tokenizer.token_stream_for_doc(text); - let mut position: u32 = 0; - while let Some(tok) = stream.next() { - let term = tok.text.as_str(); - // One hash lookup per token: extend the term's builder, or - // intern its `Arc` once on first sight this batch. - if let Some(builder) = term_builders.get_mut(term) { - builder.observe(row_position, position); - } else { - term_builders.insert( - Arc::::from(term), - BatchTermBuilder::with_first(row_position, position), - ); - } - position += 1; - doc_token_count += 1; - } + for document in extracted_documents { + index_document( + DocumentKey { + row_position: row_position(document.row_index)?, + doc_index: document.doc_index, + }, + &document.text, + )?; + } + + let query_local_stats = if allowed_terms.is_some() { + QueryLocalFtsStats { + doc_count: query_local_corpus_doc_count, + total_tokens: query_local_corpus_total_tokens, + token_docs: term_builders + .iter() + .map(|(term, builder)| (term.to_string(), builder.row_positions.len())) + .collect(), } + } else { + QueryLocalFtsStats::default() + }; - doc_lengths.push(doc_token_count); - total_tokens += doc_token_count as u64; + if documents.is_empty() { + return Ok(query_local_stats); } // Drop the tokenizer guard before publishing so we don't hold it // across the snapshot install. drop(tok_guard); + let batch_position = st.tail.next_position(); st.tail.append_batch( batch_position, - row_offset, - batch.num_rows() as u32, - doc_lengths, + document_position_start, + documents, total_tokens, term_builders, self.params.has_positions(), ); - if st.tail.doc_count() >= self.freeze_threshold_rows as u64 { - self.freeze(&st); + if self.background_maintenance && st.tail.doc_count() >= self.freeze_threshold_rows as u64 { + self.freeze(&st)?; } - Ok(()) + Ok(query_local_stats) + } + + /// Analyze every exact leaf and return the deduplicated query terms in + /// canonical leaf traversal order. + pub(crate) fn exact_query_terms(&self, query: &FtsQuery) -> Result> { + fn visit(index: &FtsMemIndex, query: &FtsQuery, terms: &mut Vec) -> Result<()> { + match query { + FtsQuery::Match(query) => { + if query.fuzziness != Some(0) { + return Err(Error::invalid_input( + "residual compound FTS only supports exact Match leaves", + )); + } + terms.extend(index.analyze_for_search(&query.terms)); + } + FtsQuery::Phrase(query) => { + terms.extend(index.analyze_for_search(&query.terms)); + } + FtsQuery::Boost(query) => { + visit(index, &query.positive, terms)?; + visit(index, &query.negative, terms)?; + } + FtsQuery::MultiMatch(query) => { + for query in &query.match_queries { + visit(index, &FtsQuery::Match(query.clone()), terms)?; + } + } + FtsQuery::Boolean(query) => { + for query in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(index, query, terms)?; + } + } + } + Ok(()) + } + + let mut terms = Vec::new(); + visit(self, query, &mut terms)?; + let mut seen = HashSet::with_capacity(terms.len()); + terms.retain(|term| seen.insert(term.clone())); + Ok(terms) + } + + /// Materialize each exact leaf with a caller-supplied scorer. Compound + /// semantics are deliberately evaluated by the canonical lance-index + /// scorer instead of being duplicated here. + pub(crate) fn exact_leaf_results( + &self, + query: &FtsQuery, + scorer: &MemBM25Scorer, + ) -> Result>> { + fn visit( + index: &FtsMemIndex, + query: &FtsQuery, + scorer: &MemBM25Scorer, + leaves: &mut Vec>, + ) -> Result<()> { + match query { + FtsQuery::Match(query) => { + if query.fuzziness != Some(0) { + return Err(Error::invalid_input( + "residual compound FTS only supports exact Match leaves", + )); + } + let st = index.state.load_full(); + let tokens = index.analyze_for_search(&query.terms); + let rows = index + .search_match_with_scorer(&st, &tokens, query.operator, scorer) + .into_iter() + .map(|entry| (entry.row_position, entry.score)) + .collect(); + leaves.push(rows); + } + FtsQuery::Phrase(query) => { + let st = index.state.load_full(); + let tokens = index.analyze_for_search(&query.terms); + let rows = index + .search_phrase_with_scorer(&st, &tokens, query.slop, scorer) + .into_iter() + .map(|entry| (entry.row_position, entry.score)) + .collect(); + leaves.push(rows); + } + FtsQuery::Boost(query) => { + visit(index, &query.positive, scorer, leaves)?; + visit(index, &query.negative, scorer, leaves)?; + } + FtsQuery::MultiMatch(query) => { + for query in &query.match_queries { + visit(index, &FtsQuery::Match(query.clone()), scorer, leaves)?; + } + } + FtsQuery::Boolean(query) => { + for query in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(index, query, scorer, leaves)?; + } + } + } + Ok(()) + } + + let mut leaves = Vec::new(); + visit(self, query, scorer, &mut leaves)?; + Ok(leaves) } /// Freeze the current tail into a new immutable partition and publish a /// fresh empty tail. Only the writer calls this; readers snapshotting the /// old `IndexState` keep a consistent view across the freeze. - fn freeze(&self, st: &IndexState) { - let Some(partition) = Partition::from_tail(&st.tail) else { - return; + fn freeze(&self, st: &IndexState) -> Result<()> { + let Some(partition) = Partition::from_tail(&st.tail)? else { + return Ok(()); }; let mut partitions: Vec> = st.partitions.iter().cloned().collect(); partitions.push(Arc::new(partition)); // Fold in any completed background merge before re-evaluating tiers. - self.install_pending_merge(&mut partitions); + self.install_pending_merge(&mut partitions)?; if partitions.len() > Self::MAX_PARTITIONS { - partitions = vec![Arc::new(Partition::merge(&partitions))]; + partitions = vec![Arc::new(Partition::merge(&partitions)?)]; } self.state.store(Arc::new(IndexState { partitions: Arc::from(partitions.into_boxed_slice()), tail: TailIndex::new(), })); // Kick off a background merge if a size tier is now over-full. - self.maybe_start_merge(); + self.maybe_start_merge()?; + Ok(()) } /// Install a completed background merge into `partitions` (in place): /// drop the merged-away source partitions and append the merged one. /// No-op while a merge is still running or none is pending. - fn install_pending_merge(&self, partitions: &mut Vec>) { - let mut guard = self.merge.lock().expect("merge slot poisoned"); - let Some(pending) = guard.as_ref() else { - return; + fn install_pending_merge(&self, partitions: &mut Vec>) -> Result<()> { + let mut guard = self + .merge + .lock() + .map_err(|_| Error::internal("FTS merge slot mutex poisoned"))?; + let Some(pending) = guard.as_mut() else { + return Ok(()); }; - let Some(merged) = pending.result.clone() else { - return; // still running + let Some(merged) = pending.result.take() else { + return Ok(()); // still running }; let sources: HashSet = pending.sources.iter().copied().collect(); + *guard = None; // Install only if every source is still live. If a synchronous // `MAX_PARTITIONS` collapse merged the sources away while this merge // ran, the merged docs are already present — appending it would @@ -1221,19 +1763,22 @@ impl FtsMemIndex { partitions.retain(|p| !sources.contains(&(Arc::as_ptr(p) as usize))); partitions.push(merged); } - *guard = None; + Ok(()) } /// If no merge is in flight and some size tier holds at least /// `MERGE_FACTOR` partitions, dispatch their merge to a background thread. - fn maybe_start_merge(&self) { - let mut guard = self.merge.lock().expect("merge slot poisoned"); + fn maybe_start_merge(&self) -> Result<()> { + let mut guard = self + .merge + .lock() + .map_err(|_| Error::internal("FTS merge slot mutex poisoned"))?; if guard.is_some() { - return; // one merge at a time + return Ok(()); // one merge at a time } let partitions = self.state.load(); let Some(group) = select_merge_group(&partitions.partitions, Self::MERGE_FACTOR) else { - return; + return Ok(()); }; let sources: Vec = group.iter().map(|p| Arc::as_ptr(p) as usize).collect(); *guard = Some(PendingMerge { @@ -1247,13 +1792,9 @@ impl FtsMemIndex { // `group`, the source partitions), so it is safe even if the index is // dropped mid-merge. std::thread::spawn(move || { - let merged = Arc::new(Partition::merge(&group)); - if let Ok(mut g) = slot.lock() - && let Some(p) = g.as_mut() - { - p.result = Some(merged); - } + publish_pending_merge(slot.as_ref(), Partition::merge(&group)); }); + Ok(()) } // ------------------------------------------------------------------ @@ -1267,7 +1808,7 @@ impl FtsMemIndex { /// use `search_with_options` for sorted/limited output. pub fn search(&self, term: &str) -> Vec { let st = self.state.load_full(); - let tokens = self.tokenize_for_search(term); + let tokens = self.analyze_for_search(term); self.search_match(&st, &tokens, Operator::Or, None, true, true) } @@ -1275,7 +1816,7 @@ impl FtsMemIndex { /// `slop` intervening tokens between consecutive query tokens. pub fn search_phrase(&self, phrase: &str, slop: u32) -> Vec { let st = self.state.load_full(); - let tokens = self.tokenize_for_search(phrase); + let tokens = self.analyze_for_search(phrase); self.search_phrase_tokens(&st, &tokens, slop, true) } @@ -1285,8 +1826,10 @@ impl FtsMemIndex { /// when the tail is empty. Writer-side: callers hold the single-writer role. pub fn flush(&self) { let st = self.state.load_full(); - if st.tail.visible_count() > 0 { - self.freeze(&st); + if st.tail.visible_count() > 0 + && let Err(error) = self.freeze(&st) + { + tracing::error!(?error, "failed to freeze the FTS MemWAL tail"); } } @@ -1322,6 +1865,22 @@ impl FtsMemIndex { /// shared rising threshold (instead of every partition cold-starting). /// Without a limit, an exact O(matches) scan across partitions + tail. fn search_match( + &self, + st: &IndexState, + query_tokens: &Tokens, + operator: Operator, + limit: Option, + include_tail: bool, + tail_skip: bool, + ) -> Vec { + if operator == Operator::And && has_grouped_positions(query_tokens) { + return self.search_grouped_and(st, query_tokens, limit, include_tail, tail_skip); + } + let tokens = query_tokens_to_vec(query_tokens); + self.search_match_strings(st, &tokens, operator, limit, include_tail, tail_skip) + } + + fn search_match_strings( &self, st: &IndexState, tokens: &[String], @@ -1364,8 +1923,9 @@ impl FtsMemIndex { Operator::Or, &scorer, theta, + false, ) { - topk.offer(e.score, e.row_position); + topk.offer(e.score, e.key()); } } topk.into_entries() @@ -1383,6 +1943,7 @@ impl FtsMemIndex { operator, &scorer, f32::NEG_INFINITY, + false, )); } results @@ -1390,19 +1951,153 @@ impl FtsMemIndex { } } - fn search_phrase_tokens( + fn search_match_with_scorer( + &self, + st: &IndexState, + query_tokens: &Tokens, + operator: Operator, + scorer: &MemBM25Scorer, + ) -> Vec { + if operator == Operator::And && has_grouped_positions(query_tokens) { + let mut result_map: Option> = None; + for group in query_position_groups(query_tokens) { + let group_results = + self.search_match_strings_with_scorer(st, &group, Operator::Or, scorer); + let group_map = group_results + .into_iter() + .map(|entry| (entry.key(), entry.score)) + .collect::>(); + let Some(current) = result_map.as_mut() else { + result_map = Some(group_map); + continue; + }; + current.retain(|key, score| { + if let Some(group_score) = group_map.get(key) { + *score += group_score; + true + } else { + false + } + }); + } + return result_map + .unwrap_or_default() + .into_iter() + .map(|(key, score)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), + score, + }) + .collect(); + } + let tokens = query_tokens_to_vec(query_tokens); + self.search_match_strings_with_scorer(st, &tokens, operator, scorer) + } + + fn search_match_strings_with_scorer( &self, st: &IndexState, tokens: &[String], + operator: Operator, + scorer: &MemBM25Scorer, + ) -> Vec { + if tokens.is_empty() { + return Vec::new(); + } + let tail = st.tail.snapshot(); + let mut results = Vec::new(); + for partition in st.partitions.iter() { + results.extend(partition.search_match(tokens, operator, scorer)); + } + results.extend(score_terms( + &tail, + &st.tail.terms, + tokens, + operator, + scorer, + f32::NEG_INFINITY, + true, + )); + results + } + + fn search_grouped_and( + &self, + st: &IndexState, + query_tokens: &Tokens, + limit: Option, + include_tail: bool, + tail_skip: bool, + ) -> Vec { + let mut result_map: Option> = None; + for group in query_position_groups(query_tokens) { + let group_results = + self.search_match_strings(st, &group, Operator::Or, None, include_tail, tail_skip); + let group_map = group_results + .into_iter() + .map(|entry| (entry.key(), entry.score)) + .collect::>(); + let Some(current) = result_map.as_mut() else { + result_map = Some(group_map); + continue; + }; + current.retain(|key, score| { + if let Some(group_score) = group_map.get(key) { + *score += group_score; + true + } else { + false + } + }); + if current.is_empty() { + return Vec::new(); + } + } + + let mut results = result_map + .unwrap_or_default() + .into_iter() + .map(|(key, score)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), + score, + }) + .collect::>(); + if let Some(limit) = limit { + results.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + results.truncate(limit); + } + results + } + + fn search_phrase_tokens( + &self, + st: &IndexState, + query_tokens: &Tokens, slop: u32, include_tail: bool, ) -> Vec { - if tokens.is_empty() { + if query_tokens.is_empty() { return Vec::new(); } - if tokens.len() == 1 { + let groups = query_position_groups(query_tokens); + if groups.is_empty() { + return Vec::new(); + } + if groups.len() == 1 { // A single-token phrase reduces to a regular term search. - return self.search_match(st, tokens, Operator::Or, None, include_tail, true); + return self.search_match_strings( + st, + &groups[0], + Operator::Or, + None, + include_tail, + true, + ); } // A multi-token phrase needs token positions; without them (the index // was built `with_position = false`) phrase search is unsupported, as @@ -1410,23 +2105,90 @@ impl FtsMemIndex { if !self.params.has_positions() { return Vec::new(); } + let has_grouped_terms = groups.iter().any(|group| group.len() > 1); + let tokens = position_groups_to_tokens(&groups); let tail_snap = st.tail.snapshot(); let scan_tail = include_tail && tail_snap.visible_count > 0; - let scorer = build_scorer(st, &tail_snap, tokens, include_tail); + let scorer = build_scorer(st, &tail_snap, &tokens, include_tail); if scorer.num_docs() == 0 { return Vec::new(); } let mut results = Vec::new(); for p in st.partitions.iter() { - results.extend(p.search_phrase(tokens, slop, &scorer)); + if has_grouped_terms { + results.extend(p.search_phrase_groups(&groups, slop, &scorer)); + } else { + results.extend(p.search_phrase(&tokens, slop, &scorer)); + } } if scan_tail { + if has_grouped_terms { + results.extend(phrase_search_tail_groups( + &tail_snap, + &st.tail.terms, + &groups, + slop, + &scorer, + )); + } else { + results.extend(phrase_search_tail( + &tail_snap, + &st.tail.terms, + &tokens, + slop, + &scorer, + )); + } + } + results + } + + fn search_phrase_with_scorer( + &self, + st: &IndexState, + query_tokens: &Tokens, + slop: u32, + scorer: &MemBM25Scorer, + ) -> Vec { + if query_tokens.is_empty() || scorer.num_docs() == 0 { + return Vec::new(); + } + let groups = query_position_groups(query_tokens); + if groups.is_empty() { + return Vec::new(); + } + if groups.len() == 1 { + return self.search_match_strings_with_scorer(st, &groups[0], Operator::Or, scorer); + } + if !self.params.has_positions() { + return Vec::new(); + } + let has_grouped_terms = groups.iter().any(|group| group.len() > 1); + let tokens = position_groups_to_tokens(&groups); + let tail = st.tail.snapshot(); + let mut results = Vec::new(); + for partition in st.partitions.iter() { + if has_grouped_terms { + results.extend(partition.search_phrase_groups(&groups, slop, scorer)); + } else { + results.extend(partition.search_phrase(&tokens, slop, scorer)); + } + } + if has_grouped_terms { + results.extend(phrase_search_tail_groups( + &tail, + &st.tail.terms, + &groups, + slop, + scorer, + )); + } else { results.extend(phrase_search_tail( - &tail_snap, + &tail, &st.tail.terms, - tokens, + &tokens, slop, - &scorer, + scorer, )); } results @@ -1464,7 +2226,7 @@ impl FtsMemIndex { if expanded.is_empty() { return Vec::new(); } - self.search_match(st, &expanded, Operator::Or, None, include_tail, true) + self.search_match_strings(st, &expanded, Operator::Or, None, include_tail, true) } /// Expand `term` against the term dictionaries of every partition (and the @@ -1556,14 +2318,14 @@ impl FtsMemIndex { operator, boost, } => { - let tokens = self.tokenize_for_search(query); + let tokens = self.analyze_for_search(query); let mut results = self.search_match(st, &tokens, *operator, limit, include_tail, tail_skip); apply_boost(&mut results, *boost); results } FtsQueryExpr::Phrase { query, slop, boost } => { - let tokens = self.tokenize_for_search(query); + let tokens = self.analyze_for_search(query); let mut results = self.search_phrase_tokens(st, &tokens, *slop, include_tail); apply_boost(&mut results, *boost); results @@ -1660,12 +2422,10 @@ impl FtsMemIndex { return results; }; let negative_results = self.search_query_with_state(neg, st, None, include_tail, true); - let negative_set: HashSet = negative_results - .into_iter() - .map(|e| e.row_position) - .collect(); + let negative_set: HashSet = + negative_results.iter().map(FtsEntry::key).collect(); for entry in &mut results { - if negative_set.contains(&entry.row_position) { + if negative_set.contains(&entry.key()) { entry.score *= negative_boost; } } @@ -1680,32 +2440,32 @@ impl FtsMemIndex { st: &IndexState, include_tail: bool, ) -> Vec { - let excluded: HashSet = must_not + let excluded: HashSet = must_not .iter() .flat_map(|q| self.search_query_with_state(q, st, None, include_tail, true)) - .map(|e| e.row_position) + .map(|entry| entry.key()) .collect(); - let mut result_map: HashMap = if must.is_empty() { - let mut map: HashMap = HashMap::new(); + let mut result_map: HashMap = if must.is_empty() { + let mut map: HashMap = HashMap::new(); for q in should { for entry in self.search_query_with_state(q, st, None, include_tail, true) { - *map.entry(entry.row_position).or_default() += entry.score; + *map.entry(entry.key()).or_default() += entry.score; } } map } else { let first_results = self.search_query_with_state(&must[0], st, None, include_tail, true); - let mut map: HashMap = first_results + let mut map: HashMap = first_results .into_iter() - .map(|e| (e.row_position, e.score)) + .map(|entry| (entry.key(), entry.score)) .collect(); for q in must.iter().skip(1) { let results = self.search_query_with_state(q, st, None, include_tail, true); - let result_set: HashMap = results + let result_set: HashMap = results .into_iter() - .map(|e| (e.row_position, e.score)) + .map(|entry| (entry.key(), entry.score)) .collect(); map = map .into_iter() @@ -1714,7 +2474,7 @@ impl FtsMemIndex { } for q in should { for entry in self.search_query_with_state(q, st, None, include_tail, true) { - if let Some(score) = map.get_mut(&entry.row_position) { + if let Some(score) = map.get_mut(&entry.key()) { *score += entry.score; } } @@ -1728,21 +2488,28 @@ impl FtsMemIndex { result_map .into_iter() - .map(|(row_position, score)| FtsEntry { - row_position, + .map(|(key, score)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), score, }) .collect() } fn tokenize_for_search(&self, text: &str) -> Vec { + query_tokens_to_vec(&self.analyze_for_search(text)) + } + + fn analyze_for_search(&self, text: &str) -> Tokens { let mut tok = PooledTokenizer::new(&self.tokenizer_pool); let mut stream = tok.get_mut().token_stream_for_search(text); - let mut out = Vec::new(); + let mut tokens = Vec::new(); + let mut positions = Vec::new(); while let Some(t) = stream.next() { - out.push(t.text.clone()); + tokens.push(t.text.clone()); + positions.push(t.position as u32); } - out + Tokens::with_positions(tokens, positions, DocType::Text) } // ------------------------------------------------------------------ @@ -1752,8 +2519,9 @@ impl FtsMemIndex { /// Export the in-memory FTS index to an `InnerBuilder` ready to be /// written to disk. /// - /// Doc row positions are kept in insert order to match the forward-written - /// flush data file 1:1. `total_rows` is used only to validate positions. + /// Stored documents are kept in insert order and retain their positions in + /// the forward-written flush data file. `total_rows` is used only to + /// validate positions. pub fn to_index_builder( &self, partition_id: u64, @@ -1764,51 +2532,58 @@ impl FtsMemIndex { let st = self.state.load_full(); let with_position = self.params.has_positions(); + let block_size = self.params.posting_block_size(); let format_version = self.params.resolved_format_version(); let posting_tail_codec = format_version.posting_tail_codec(); let total_rows_u64 = total_rows as u64; - // Step 1: collect (original_pos, num_tokens) for every doc across all + // Step 1: collect (document key, num_tokens) for every doc across all // immutable partitions and the visible tail. - let mut all_docs: Vec<(u64, u32)> = Vec::new(); + let mut all_docs: Vec<(DocumentKey, u32)> = Vec::new(); for p in st.partitions.iter() { - for (row_pos, num_tokens) in p.docs.iter() { - all_docs.push((*row_pos, *num_tokens)); + for (doc_id, (_, num_tokens)) in p.docs.iter().enumerate() { + all_docs.push((doc_set_key(&p.docs, doc_id as u32), *num_tokens)); } } let tail_snap = st.tail.snapshot(); for batch in tail_snap.batches.iter().take(tail_snap.visible_count) { - for i in 0..batch.rows as usize { - all_docs.push((batch.row_offset + i as u64, batch.doc_lengths[i])); + for document in &batch.documents { + all_docs.push((document.key.clone(), document.num_tokens)); } } if all_docs.is_empty() { - return Ok(InnerBuilder::new_with_format_version( + return Ok(InnerBuilder::new_with_format_version_and_block_size( partition_id, with_position, Default::default(), format_version, + block_size, )); } - // Step 2: assign doc_ids in ascending insert-position order, so the - // stored row positions line up 1:1 with the forward-written data file. - let mut entries: Vec<(u64, u32)> = Vec::with_capacity(all_docs.len()); - for (original, num_tokens) in &all_docs { - if *original >= total_rows_u64 { + // Step 2: assign doc_ids in ascending insert-position order while + // preserving each document's position in the forward-written data file. + let mut entries: Vec<(DocumentKey, u32)> = Vec::with_capacity(all_docs.len()); + for (key, num_tokens) in &all_docs { + if key.row_position >= total_rows_u64 { return Err(Error::io(format!( "FTS flush: row position {} >= total_rows {}", - original, total_rows + key.row_position, total_rows ))); } - entries.push((*original, *num_tokens)); + entries.push((key.clone(), *num_tokens)); } - entries.sort_by_key(|(original, _)| *original); + entries.sort_by(|left, right| left.0.cmp(&right.0)); let mut docs = DocSet::default(); - let mut original_to_doc_id: HashMap = HashMap::with_capacity(entries.len()); - for (original, num_tokens) in &entries { - let doc_id = docs.append(*original, *num_tokens); - original_to_doc_id.insert(*original, doc_id); + let mut original_to_doc_id: HashMap = + HashMap::with_capacity(entries.len()); + for (key, num_tokens) in &entries { + let doc_id = if !key.doc_index.is_empty() { + docs.append_with_doc_index(key.row_position, *num_tokens, &key.doc_index)? + } else { + docs.append(key.row_position, *num_tokens) + }; + original_to_doc_id.insert(key.clone(), doc_id); } // Step 3: merge per-term postings across every partition and the tail. @@ -1819,8 +2594,8 @@ impl FtsMemIndex { let bucket = term_postings.entry(term.to_string()).or_default(); let mut cursor = PostingCursor::new(p, term_id); while let Some(local_doc) = cursor.doc() { - let row_pos = p.docs.row_id(local_doc); - if let Some(&doc_id) = original_to_doc_id.get(&row_pos) { + let key = doc_set_key(&p.docs, local_doc); + if let Some(&doc_id) = original_to_doc_id.get(&key) { let pos = if with_position { Some(cursor.positions().to_vec()) } else { @@ -1840,8 +2615,11 @@ impl FtsMemIndex { if chunk.batch_position >= tail_snap.visible_count { continue; } - for (i, row_position) in chunk.row_positions.iter().enumerate() { - let Some(&doc_id) = original_to_doc_id.get(row_position) else { + for (i, document_position) in chunk.row_positions.iter().enumerate() { + let Some(document) = lookup_document(&tail_snap, *document_position) else { + continue; + }; + let Some(&doc_id) = original_to_doc_id.get(&document.key) else { continue; }; let pos = if with_position { @@ -1873,10 +2651,13 @@ impl FtsMemIndex { docs_for_term.sort_by_key(|(doc_id, _, _)| *doc_id); let token_id = tokens.add(token) as usize; debug_assert_eq!(token_id, posting_lists.len()); - posting_lists.push(PostingListBuilder::new_with_posting_tail_codec( - with_position, - posting_tail_codec, - )); + posting_lists.push( + PostingListBuilder::new_with_posting_tail_codec_and_block_size( + with_position, + posting_tail_codec, + block_size, + ), + ); let plb = &mut posting_lists[token_id]; for (doc_id, freq, pos) in docs_for_term { let recorder = if with_position { @@ -1888,11 +2669,12 @@ impl FtsMemIndex { } } - let mut builder = InnerBuilder::new_with_format_version( + let mut builder = InnerBuilder::new_with_format_version_and_block_size( partition_id, with_position, Default::default(), format_version, + block_size, ); builder.set_tokens(tokens); builder.set_docs(docs); @@ -1979,58 +2761,87 @@ impl BatchTermBuilder { } } -/// Borrowed text for a row, or `None` for null/missing. -type TextOpt<'a> = Option<&'a str>; - -fn extract_texts(column: &dyn Array) -> Result>> { - match column.data_type() { - DataType::Utf8 => { - let array = column - .as_any() - .downcast_ref::() - .expect("Utf8 array"); - Ok((0..array.len()) - .map(|i| (!array.is_null(i)).then(|| array.value(i))) - .collect()) - } - DataType::LargeUtf8 => { - let array = column - .as_any() - .downcast_ref::() - .expect("LargeUtf8 array"); - Ok((0..array.len()) - .map(|i| (!array.is_null(i)).then(|| array.value(i))) - .collect()) - } - DataType::Utf8View => { - let array = column - .as_any() - .downcast_ref::() - .expect("Utf8View array"); - Ok((0..array.len()) - .map(|i| (!array.is_null(i)).then(|| array.value(i))) - .collect()) - } - other => Err(Error::invalid_input(format!( - "FTS index only supports Utf8, LargeUtf8, and Utf8View columns; got {other:?}" - ))), +fn index_text( + text: &str, + document_position: u64, + tokenizer: &mut dyn LanceTokenizer, + term_builders: &mut FxHashMap, BatchTermBuilder>, +) -> Result { + index_text_with_predicate(text, document_position, tokenizer, term_builders, |_| true) + .map(|(num_tokens, _)| num_tokens) +} + +fn index_text_filtered( + text: &str, + document_position: u64, + tokenizer: &mut dyn LanceTokenizer, + term_builders: &mut FxHashMap, BatchTermBuilder>, + allowed_terms: &FxHashSet, +) -> Result<(u32, bool)> { + index_text_with_predicate(text, document_position, tokenizer, term_builders, |term| { + allowed_terms.contains(term) + }) +} + +#[inline] +fn index_text_with_predicate( + text: &str, + document_position: u64, + tokenizer: &mut dyn LanceTokenizer, + term_builders: &mut FxHashMap, BatchTermBuilder>, + mut retain_term: impl FnMut(&str) -> bool, +) -> Result<(u32, bool)> { + let mut stream = tokenizer.token_stream_for_doc(text); + let mut num_tokens = 0u32; + let mut retained_term = false; + while let Some(token) = stream.next() { + let position = u32::try_from(token.position).map_err(|_| { + Error::invalid_input(format!( + "token position overflow for document_position={document_position}: token_position={}", + token.position + )) + })?; + let term = token.text.as_str(); + if retain_term(term) { + retained_term = true; + if let Some(builder) = term_builders.get_mut(term) { + builder.observe(document_position, position); + } else { + term_builders.insert( + Arc::::from(term), + BatchTermBuilder::with_first(document_position, position), + ); + } + } + num_tokens = num_tokens.checked_add(1).ok_or_else(|| { + Error::invalid_input(format!( + "token count overflow for document_position={document_position}" + )) + })?; } + Ok((num_tokens, retained_term)) } fn has_visible_chunk(slice: &TermSlice, visible_count: usize) -> bool { slice.chunks().any(|c| c.batch_position < visible_count) } -fn lookup_dl(snap: &Snapshot, row_position: u64) -> Option { - snap.batches.iter().find_map(|b| b.dl(row_position)) +fn lookup_document(snap: &Snapshot, document_position: u64) -> Option { + snap.batches + .iter() + .find_map(|batch| batch.document(document_position).cloned()) +} + +fn lookup_dl(snap: &Snapshot, document_position: u64) -> Option { + lookup_document(snap, document_position).map(|document| document.num_tokens) } fn find_doc_in_chunks( chunks: &[Arc], - row_position: u64, + document_position: u64, ) -> Option<(&Arc, usize)> { for chunk in chunks { - if let Ok(idx) = chunk.row_positions.binary_search(&row_position) { + if let Ok(idx) = chunk.row_positions.binary_search(&document_position) { return Some((chunk, idx)); } } @@ -2099,6 +2910,11 @@ fn tail_token_df( /// Score `tokens` against the visible tail, summing each token's BM25 /// contribution per document. Uses the shared corpus-wide `scorer`. +/// +/// `retain_zero_weight_matches` is reserved for query-local residual postings +/// scored with committed-index statistics. A term absent from the committed +/// corpus has zero BM25 weight, but its fresh matching rows must remain visible +/// to compound membership and MUST_NOT evaluation. fn score_terms( snap: &Snapshot, terms: &SkipMap, Arc>>, @@ -2106,6 +2922,7 @@ fn score_terms( operator: Operator, scorer: &MemBM25Scorer, theta: f32, + retain_zero_weight_matches: bool, ) -> Vec { // Per-token tail data + its score upper bound (max freq over visible chunks, // scored at the most generous doc length of 1). If even the sum of those @@ -2121,7 +2938,7 @@ fn score_terms( continue; }; let qw = scorer.query_weight(token); - if qw == 0.0 { + if qw == 0.0 && !retain_zero_weight_matches { continue; } let slice = entry.value().load_full(); @@ -2131,15 +2948,16 @@ fn score_terms( .map(|c| c.max_freq) .max() .unwrap_or(0); - tail_ub += qw * scorer.doc_weight(max_freq, 1); + if qw != 0.0 { + tail_ub += qw * scorer.doc_weight(max_freq, 1); + } tail_terms.push((qw, slice)); } if tail_ub <= theta { return Vec::new(); } - let mut doc_scores: HashMap = HashMap::new(); - let mut doc_hits: Option> = - (operator == Operator::And).then(HashMap::new); + let mut doc_scores: HashMap = HashMap::new(); + let mut doc_hits: Option> = (operator == Operator::And).then(HashMap::new); for (qw, slice) in tail_terms { for chunk in slice.chunks() { if chunk.batch_position >= snap.visible_count { @@ -2148,28 +2966,35 @@ fn score_terms( let Some(meta) = snap.batch_for(chunk.batch_position) else { continue; }; - for (i, &row_position) in chunk.row_positions.iter().enumerate() { - let dl = meta.dl(row_position).unwrap_or(1); - let score = qw * scorer.doc_weight(chunk.frequencies[i], dl); - *doc_scores.entry(row_position).or_default() += score; + for (i, &document_position) in chunk.row_positions.iter().enumerate() { + let score = if qw == 0.0 { + 0.0 + } else { + let dl = meta.dl(document_position).unwrap_or(1); + qw * scorer.doc_weight(chunk.frequencies[i], dl) + }; + *doc_scores.entry(document_position).or_default() += score; if let Some(doc_hits) = &mut doc_hits { - *doc_hits.entry(row_position).or_default() += 1; + *doc_hits.entry(document_position).or_default() += 1; } } } } doc_scores .into_iter() - .filter(|(row_position, _)| { + .filter(|(document_position, _)| { operator == Operator::Or || doc_hits .as_ref() - .and_then(|doc_hits| doc_hits.get(row_position)) + .and_then(|doc_hits| doc_hits.get(document_position)) .is_some_and(|hits| *hits >= tokens.len() as u32) }) - .map(|(row_position, score)| FtsEntry { - row_position, - score, + .filter_map(|(document_position, score)| { + lookup_document(snap, document_position).map(|document| FtsEntry { + row_position: document.key.row_position, + doc_index: public_doc_index(&document.key.doc_index), + score, + }) }) .collect() } @@ -2214,7 +3039,7 @@ fn phrase_search_tail( let mut results = Vec::new(); for chunk in &per_token_chunks[smallest_idx] { - for (doc_idx, &row_position) in chunk.row_positions.iter().enumerate() { + for (doc_idx, &document_position) in chunk.row_positions.iter().enumerate() { let Some(pos) = chunk .positions .as_ref() @@ -2231,7 +3056,7 @@ fn phrase_search_tail( if ti == smallest_idx { continue; } - match find_doc_in_chunks(chunks, row_position) { + match find_doc_in_chunks(chunks, document_position) { Some((c, other_idx)) => { frequencies[ti] = c.frequencies[other_idx]; all_positions[ti] = c @@ -2249,21 +3074,148 @@ fn phrase_search_tail( if !all_present || !phrase_matches(&all_positions, slop) { continue; } - let dl = lookup_dl(snap, row_position).unwrap_or(1); + let dl = lookup_dl(snap, document_position).unwrap_or(1); let score: f32 = tokens .iter() .enumerate() .map(|(ti, tok)| scorer.query_weight(tok) * scorer.doc_weight(frequencies[ti], dl)) .sum(); - results.push(FtsEntry { - row_position, - score, - }); + if let Some(document) = lookup_document(snap, document_position) { + results.push(FtsEntry { + row_position: document.key.row_position, + doc_index: public_doc_index(&document.key.doc_index), + score, + }); + } } } results } +#[derive(Default, Clone)] +struct PhraseGroupDoc { + positions: Vec, + score: f32, +} + +struct PhraseCandidate { + positions_by_group: Vec>, + score: f32, +} + +fn merge_phrase_group( + candidates: &mut Option>, + group_idx: usize, + group_count: usize, + group_docs: HashMap, +) -> bool +where + K: Eq + std::hash::Hash, +{ + let Some(current) = candidates.as_mut() else { + *candidates = Some( + group_docs + .into_iter() + .map(|(doc, group_doc)| { + let mut positions_by_group = vec![Vec::new(); group_count]; + positions_by_group[group_idx] = group_doc.positions; + ( + doc, + PhraseCandidate { + positions_by_group, + score: group_doc.score, + }, + ) + }) + .collect(), + ); + return true; + }; + + current.retain(|doc, candidate| { + if let Some(group_doc) = group_docs.get(doc) { + candidate.positions_by_group[group_idx] = group_doc.positions.clone(); + candidate.score += group_doc.score; + true + } else { + false + } + }); + !current.is_empty() +} + +fn phrase_search_tail_groups( + snap: &Snapshot, + terms: &SkipMap, Arc>>, + groups: &[Vec], + slop: u32, + scorer: &MemBM25Scorer, +) -> Vec { + let mut candidates: Option> = None; + for (group_idx, group) in groups.iter().enumerate() { + let group_docs = tail_phrase_group_docs(snap, terms, group, scorer); + if group_docs.is_empty() + || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs) + { + return Vec::new(); + } + } + + candidates + .unwrap_or_default() + .into_iter() + .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop)) + .map(|(key, candidate)| FtsEntry { + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), + score: candidate.score, + }) + .collect() +} + +fn tail_phrase_group_docs( + snap: &Snapshot, + terms: &SkipMap, Arc>>, + group: &[String], + scorer: &MemBM25Scorer, +) -> HashMap { + let mut docs: HashMap = HashMap::new(); + for token in group { + let Some(entry) = terms.get(token.as_str()) else { + continue; + }; + let qw = scorer.query_weight(token); + let slice = entry.value().load_full(); + for chunk in slice.chunks() { + if chunk.batch_position >= snap.visible_count { + continue; + } + let Some(meta) = snap.batch_for(chunk.batch_position) else { + continue; + }; + let Some(positions) = &chunk.positions else { + continue; + }; + for (i, &document_position) in chunk.row_positions.iter().enumerate() { + let Some(document) = meta.document(document_position) else { + continue; + }; + let entry = docs.entry(document.key.clone()).or_default(); + entry + .positions + .extend_from_slice(positions.doc_positions(i)); + let dl = document.num_tokens; + entry.score += qw * scorer.doc_weight(chunk.frequencies[i], dl); + } + } + } + for doc in docs.values_mut() { + doc.positions.sort_unstable(); + doc.positions.dedup(); + } + docs +} + fn phrase_matches>(positions: &[T], slop: u32) -> bool { if positions.is_empty() { return false; @@ -2314,6 +3266,7 @@ pub struct FtsIndexConfig { pub field_id: i32, pub column: String, pub params: InvertedIndexParams, + pub(crate) resolved_field: Option>, } impl FtsIndexConfig { @@ -2323,6 +3276,7 @@ impl FtsIndexConfig { field_id, column, params: InvertedIndexParams::default(), + resolved_field: None, } } @@ -2332,12 +3286,29 @@ impl FtsIndexConfig { column: String, params: InvertedIndexParams, ) -> Self { - Self { + Self::try_with_params(name, field_id, column, params) + .expect("invalid MemWAL FTS index config parameters") + } + + pub fn try_with_params( + name: String, + field_id: i32, + column: String, + params: InvertedIndexParams, + ) -> Result { + params.validate_format_version()?; + Ok(Self { name, field_id, column, params, - } + resolved_field: None, + }) + } + + pub(crate) fn with_resolved_field(mut self, resolved_field: ResolvedFtsField) -> Self { + self.resolved_field = Some(Arc::new(resolved_field)); + self } } @@ -2947,7 +3918,7 @@ impl Partition { .unwrap_or(0) } - fn memory_size(&self) -> usize { + fn resident_bytes(&self) -> usize { std::mem::size_of::() + self.term_fst.as_fst().as_bytes().len() + self.postings.len() * std::mem::size_of::() @@ -2959,21 +3930,31 @@ impl Partition { /// Freeze the visible contents of `tail` into a new partition. Returns /// `None` if the tail has no visible docs. - fn from_tail(tail: &TailIndex) -> Option { + fn from_tail(tail: &TailIndex) -> Result> { let snap = tail.snapshot(); if snap.visible_count == 0 { - return None; + return Ok(None); } // Assign dense local doc ids in row-position order. let mut docs = DocSet::default(); let mut pos_to_doc: HashMap = HashMap::new(); for batch in snap.batches.iter().take(snap.visible_count) { - for i in 0..batch.rows as usize { - let rp = batch.row_offset + i as u64; - let doc_id = docs.append(rp, batch.doc_lengths[i]); - pos_to_doc.insert(rp, doc_id); + for (offset, document) in batch.documents.iter().enumerate() { + let doc_id = if !document.key.doc_index.is_empty() { + docs.append_with_doc_index( + document.key.row_position, + document.num_tokens, + &document.key.doc_index, + )? + } else { + docs.append(document.key.row_position, document.num_tokens) + }; + pos_to_doc.insert(batch.document_position_start + offset as u64, doc_id); } } + if docs.is_empty() { + return Ok(None); + } // Snapshot the term slices (a cheap sequential skip-list walk of Arc // clones), then build each term's sorted posting list in parallel — the // per-term work (chunk traversal + sort) dominates the freeze and is @@ -3010,18 +3991,35 @@ impl Partition { Some((key, docs_for_term)) }) .collect(); - Some(build_partition(entries, docs)) + Ok(Some(build_partition(entries, docs))) } /// Merge several partitions into one. Local doc ids are reassigned by /// concatenation, which keeps each merged per-term posting list sorted. - fn merge(parts: &[Arc]) -> Self { + fn merge(parts: &[Arc]) -> Result { let mut merged: HashMap, Vec<(u32, u32, Vec)>> = HashMap::new(); let mut docs = DocSet::default(); let mut doc_offset: u32 = 0; + let coordinate_rank = parts + .first() + .map_or(0, |partition| partition.docs.coordinate_rank()); for p in parts { - for (rp, nt) in p.docs.iter() { - docs.append(*rp, *nt); + if p.docs.coordinate_rank() != coordinate_rank { + return Err(Error::index(format!( + "cannot merge MemWAL FTS partitions with coordinate ranks {coordinate_rank} and {}", + p.docs.coordinate_rank() + ))); + } + for (doc_id, (row_position, num_tokens)) in p.docs.iter().enumerate() { + if coordinate_rank > 0 { + docs.append_with_doc_index( + *row_position, + *num_tokens, + &p.docs.doc_index(doc_id as u32), + )?; + } else { + docs.append(*row_position, *num_tokens); + } } let terms = p.collect_terms(); for (term_id, term) in terms.into_iter().enumerate() { @@ -3036,7 +4034,7 @@ impl Partition { } doc_offset += p.docs.len() as u32; } - build_partition(merged.into_iter().collect(), docs) + Ok(build_partition(merged.into_iter().collect(), docs)) } /// Exact O(matches) BM25 OR/AND-search of the partition by direct posting @@ -3091,8 +4089,10 @@ impl Partition { if hits[doc] == 0 || (operator == Operator::And && hits[doc] < need) { continue; } + let key = doc_set_key(&self.docs, doc as u32); results.push(FtsEntry { - row_position: self.docs.row_id(doc as u32), + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), score: scores[doc], }); } @@ -3141,7 +4141,7 @@ impl Partition { let doc = docs[i]; let dl = self.docs.num_tokens(doc); let score = qw * scorer.doc_weight(freqs[i], dl); - topk.offer(score, self.docs.row_id(doc)); + topk.offer(score, doc_set_key(&self.docs, doc)); } } } @@ -3242,7 +4242,7 @@ impl Partition { } } if alive { - topk.offer(score, self.docs.row_id(cand)); + topk.offer(score, doc_set_key(&self.docs, cand)); } // Advance the essential lanes that were positioned at the candidate. for l in lanes[ne..].iter_mut() { @@ -3304,8 +4304,10 @@ impl Partition { .zip(&freqs) .map(|(t, &f)| scorer.query_weight(t) * scorer.doc_weight(f, dl)) .sum(); + let key = doc_set_key(&self.docs, doc); results.push(FtsEntry { - row_position: self.docs.row_id(doc), + row_position: key.row_position, + doc_index: public_doc_index(&key.doc_index), score, }); } @@ -3313,13 +4315,70 @@ impl Partition { } results } + + fn search_phrase_groups( + &self, + groups: &[Vec], + slop: u32, + scorer: &MemBM25Scorer, + ) -> Vec { + let mut candidates: Option> = None; + for (group_idx, group) in groups.iter().enumerate() { + let group_docs = self.phrase_group_docs(group, scorer); + if group_docs.is_empty() + || !merge_phrase_group(&mut candidates, group_idx, groups.len(), group_docs) + { + return Vec::new(); + } + } + + candidates + .unwrap_or_default() + .into_iter() + .filter(|(_, candidate)| phrase_matches(&candidate.positions_by_group, slop)) + .map(|(doc, candidate)| FtsEntry { + row_position: self.docs.row_id(doc), + doc_index: public_doc_index(&self.docs.doc_index(doc)), + score: candidate.score, + }) + .collect() + } + + fn phrase_group_docs( + &self, + group: &[String], + scorer: &MemBM25Scorer, + ) -> HashMap { + let mut docs: HashMap = HashMap::new(); + for token in group { + let Some(term_id) = self.term_id(token) else { + continue; + }; + let qw = scorer.query_weight(token); + let mut cursor = PostingCursor::new(self, term_id); + while let Some(doc) = cursor.cursor_doc() { + let positions = cursor.positions().to_vec(); + let freq = cursor.freq(); + let dl = self.docs.num_tokens(doc); + let entry = docs.entry(doc).or_default(); + entry.positions.extend_from_slice(&positions); + entry.score += qw * scorer.doc_weight(freq, dl); + cursor.advance(); + } + } + for doc in docs.values_mut() { + doc.positions.sort_unstable(); + doc.positions.dedup(); + } + docs + } } /// A scored MemTable row, ordered by score then row position (`total_cmp`, /// so a stray non-finite score cannot panic the heap). struct ScoredEntry { score: f32, - row_position: u64, + key: DocumentKey, } impl PartialEq for ScoredEntry { @@ -3337,7 +4396,7 @@ impl Ord for ScoredEntry { fn cmp(&self, other: &Self) -> std::cmp::Ordering { self.score .total_cmp(&other.score) - .then(self.row_position.cmp(&other.row_position)) + .then(self.key.cmp(&other.key)) } } @@ -3366,18 +4425,12 @@ impl TopK { } } - fn offer(&mut self, score: f32, row_position: u64) { + fn offer(&mut self, score: f32, key: DocumentKey) { if self.heap.len() < self.k { - self.heap.push(Reverse(ScoredEntry { - score, - row_position, - })); + self.heap.push(Reverse(ScoredEntry { score, key })); } else if score > self.heap.peek().unwrap().0.score { self.heap.pop(); - self.heap.push(Reverse(ScoredEntry { - score, - row_position, - })); + self.heap.push(Reverse(ScoredEntry { score, key })); } } @@ -3385,7 +4438,8 @@ impl TopK { self.heap .into_iter() .map(|Reverse(e)| FtsEntry { - row_position: e.row_position, + row_position: e.key.row_position, + doc_index: public_doc_index(&e.key.doc_index), score: e.score, }) .collect() @@ -3590,8 +4644,11 @@ impl<'a> PostingCursor<'a> { #[cfg(test)] mod tests { use super::*; - use arrow_array::{Int32Array, StringArray}; - use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{Array, Int32Array, ListArray, StringArray, StructArray}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + use arrow_schema::{DataType, Field, Fields, Schema as ArrowSchema}; + use lance_index::scalar::inverted::DocumentGranularity; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -3628,6 +4685,344 @@ mod tests { .unwrap() } + #[test] + fn query_term_allowlist_preserves_document_lengths_with_external_scorer() { + let schema = create_test_schema(); + let batch = create_test_batch(schema.as_ref()); + let row_ids = UInt64Array::from(vec![900, 42, 777]); + let terms = FxHashSet::from_iter(["hello".to_string()]); + let index = QueryLocalFtsIndex::try_with_params( + 1, + "description".to_string(), + InvertedIndexParams::default(), + ) + .unwrap(); + let full_index = FtsMemIndex::new(1, "description".to_string()); + + let stats = index + .insert_with_row_ids_for_terms(&batch, &row_ids, &terms) + .unwrap(); + full_index.insert(&batch, 0).unwrap(); + + // The unmatched nonempty row (row id 42) contributes no postings or + // metadata, but remains part of the approximate residual BM25 corpus. + assert_eq!(index.doc_count(), 2); + assert_eq!(index.inner.entry_count(), 2); + assert_eq!(stats.doc_count, 3); + assert_eq!(stats.total_tokens, 5); + assert_eq!(stats.token_docs.get("hello"), Some(&2)); + let committed_scorer = MemBM25Scorer::new(6, 3, HashMap::from([("hello".to_string(), 2)])); + let mut residual_scorer = committed_scorer.clone(); + stats.add_to_scorer(&mut residual_scorer).unwrap(); + assert_eq!(residual_scorer.num_docs, 6); + assert_eq!(residual_scorer.total_tokens, 11); + assert_eq!(residual_scorer.token_docs.get("hello"), Some(&4)); + + let query = FtsQuery::Match( + lance_index::scalar::inverted::query::MatchQuery::new("hello".to_string()) + .with_column(Some("description".to_string())), + ); + let leaves = index.exact_leaf_results(&query, &committed_scorer).unwrap(); + let full_leaves = full_index + .exact_leaf_results(&query, &committed_scorer) + .unwrap(); + let mut actual = leaves[0] + .iter() + .map(|(row_id, _)| *row_id) + .collect::>(); + actual.sort_unstable(); + assert_eq!(actual, vec![777, 900]); + let mut actual_scores = leaves[0] + .iter() + .map(|(_, score)| score.to_bits()) + .collect::>(); + let mut full_scores = full_leaves[0] + .iter() + .map(|(_, score)| score.to_bits()) + .collect::>(); + actual_scores.sort_unstable(); + full_scores.sort_unstable(); + assert_eq!(actual_scores, full_scores); + } + + #[test] + fn query_local_external_empty_scorer_retains_zero_score_membership() { + let schema = create_test_schema(); + let batch = create_test_batch(schema.as_ref()); + let row_ids = UInt64Array::from(vec![900, 42, 777]); + let terms = FxHashSet::from_iter(["hello".to_string()]); + let index = QueryLocalFtsIndex::try_with_params( + 1, + "description".to_string(), + InvertedIndexParams::default(), + ) + .unwrap(); + index + .insert_with_row_ids_for_terms(&batch, &row_ids, &terms) + .unwrap(); + + let committed_scorer = MemBM25Scorer::new(0, 0, HashMap::from([("hello".to_string(), 0)])); + let query = FtsQuery::Match( + lance_index::scalar::inverted::query::MatchQuery::new("hello".to_string()) + .with_column(Some("description".to_string())), + ); + let leaves = index.exact_leaf_results(&query, &committed_scorer).unwrap(); + + let mut actual = leaves[0].clone(); + actual.sort_unstable_by_key(|(row_id, _)| *row_id); + assert_eq!(actual.len(), 2); + assert_eq!(actual[0].0, 777); + assert_eq!(actual[1].0, 900); + assert!(actual.iter().all(|(_, score)| score.to_bits() == 0)); + } + + #[test] + fn query_local_materialization_never_starts_background_maintenance() { + let schema = create_test_schema(); + let batch = create_test_batch(schema.as_ref()); + let row_ids = UInt64Array::from(vec![900, 42, 777]); + let terms = FxHashSet::from_iter(["hello".to_string()]); + let params = InvertedIndexParams::default(); + let tokenizer = params.build().unwrap(); + let mut index = QueryLocalFtsIndex::try_with_loaded_tokenizer( + 1, + "description".to_string(), + params, + tokenizer, + ) + .unwrap(); + // Crossing the normal freeze threshold would create a partition and + // may launch a detached tiered merge. Query-local materialization must + // remain entirely in its query-owned tail instead. + index.inner.freeze_threshold_rows = 1; + index + .insert_with_row_ids_for_terms(&batch, &row_ids, &terms) + .unwrap(); + + assert!(index.inner.state.load().partitions.is_empty()); + assert!(index.inner.merge.lock().unwrap().is_none()); + assert_eq!(index.doc_count(), 2); + + let sibling = index.empty_sibling(); + assert!(Arc::ptr_eq( + &index.inner.tokenizer_pool, + &sibling.inner.tokenizer_pool + )); + assert_eq!(sibling.doc_count(), 0); + sibling + .insert_with_row_ids_for_terms(&batch, &UInt64Array::from(vec![901, 43, 778]), &terms) + .unwrap(); + assert_eq!(index.doc_count(), 2); + assert_eq!(sibling.doc_count(), 2); + assert!(sibling.inner.state.load().partitions.is_empty()); + assert!(sibling.inner.merge.lock().unwrap().is_none()); + } + + fn create_element_test_batch() -> RecordBatch { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("alpha beta"); + tags.values().append_null(); + tags.values().append_value(""); + tags.values().append_value("beta gamma"); + tags.append(true); + tags.values().append_value("alpha"); + tags.values().append_value("beta"); + tags.append(true); + let tags = tags.finish(); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("tags", tags.data_type().clone(), true), + ])); + RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(tags)], + ) + .unwrap() + } + + #[test] + fn test_element_document_search_and_flush_identity() { + let params = InvertedIndexParams::default() + .with_position(true) + .document_granularity(DocumentGranularity::ListElement); + let index = FtsMemIndex::try_with_params(1, "tags".to_string(), params) + .unwrap() + .with_freeze_threshold_rows(1); + assert_eq!(index.column_name(), "tags"); + assert_eq!(index.source_column_name(), "tags"); + + index.insert(&create_element_test_batch(), 0).unwrap(); + assert_eq!(index.doc_count(), 6); + + let mut beta = index.search("beta"); + beta.sort_by_key(|entry| (entry.row_position, entry.doc_index.clone())); + assert_eq!( + beta.iter() + .map(|entry| (entry.row_position, entry.doc_index.clone())) + .collect::>(), + vec![(0, Some(vec![0])), (0, Some(vec![3])), (1, Some(vec![1]))] + ); + + let phrase = index.search_phrase("beta gamma", 0); + assert_eq!(phrase.len(), 1); + assert_eq!( + (phrase[0].row_position, phrase[0].doc_index.clone()), + (0, Some(vec![3])) + ); + + let conjunctive = FtsQueryExpr::boolean() + .must(FtsQueryExpr::match_query("alpha")) + .must(FtsQueryExpr::match_query("beta")) + .build(); + let results = index.search_query(&conjunctive); + assert_eq!(results.len(), 1); + assert_eq!( + (results[0].row_position, results[0].doc_index.clone()), + (0, Some(vec![0])) + ); + + index.to_index_builder(7, 2).unwrap(); + } + + #[test] + fn test_nested_element_documents_keep_all_list_ordinals() { + let doc_fields = Fields::from(vec![Field::new("content", DataType::Utf8, true)]); + let doc_values = StructArray::new( + doc_fields.clone(), + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "alpha"]))], + None, + ); + let doc_item = Arc::new(Field::new("item", DataType::Struct(doc_fields), true)); + let docs_type = DataType::List(doc_item.clone()); + let docs = ListArray::new( + doc_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 1, 3])), + Arc::new(doc_values), + None, + ); + let group_fields = Fields::from(vec![Field::new("docs", docs_type, true)]); + let group_values = StructArray::new(group_fields.clone(), vec![Arc::new(docs)], None); + let group_item = Arc::new(Field::new("item", DataType::Struct(group_fields), true)); + let groups = ListArray::new( + group_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2])), + Arc::new(group_values), + None, + ); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "groups", + groups.data_type().clone(), + true, + )])), + vec![Arc::new(groups)], + ) + .unwrap(); + let index = FtsMemIndex::try_with_params( + 1, + "groups.docs.content".to_string(), + InvertedIndexParams::default().document_granularity(DocumentGranularity::ListElement), + ) + .unwrap() + .with_freeze_threshold_rows(1); + + index.insert(&batch, 7).unwrap(); + let mut entries = index.search("alpha"); + entries.sort_by_key(|entry| entry.doc_index.clone()); + assert_eq!( + entries + .iter() + .map(|entry| (entry.row_position, entry.doc_index.clone())) + .collect::>(), + vec![(7, Some(vec![0, 0])), (7, Some(vec![1, 1]))] + ); + + index.insert(&batch, 8).unwrap(); + let state = index.state.load_full(); + assert_eq!(state.partitions.len(), 2); + let merged = Partition::merge(&state.partitions).unwrap(); + assert_eq!(merged.docs.coordinate_rank(), 2); + assert_eq!( + (0..merged.docs.len() as u32) + .map(|doc_id| doc_set_key(&merged.docs, doc_id)) + .collect::>(), + vec![ + DocumentKey { + row_position: 7, + doc_index: vec![0, 0], + }, + DocumentKey { + row_position: 7, + doc_index: vec![1, 0], + }, + DocumentKey { + row_position: 7, + doc_index: vec![1, 1], + }, + DocumentKey { + row_position: 8, + doc_index: vec![0, 0], + }, + DocumentKey { + row_position: 8, + doc_index: vec![1, 0], + }, + DocumentKey { + row_position: 8, + doc_index: vec![1, 1], + }, + ] + ); + index.to_index_builder(8, 9).unwrap(); + } + + #[test] + fn test_failed_background_merge_releases_pending_slot() { + let slot = Mutex::new(Some(PendingMerge { + sources: vec![1], + result: None, + })); + + publish_pending_merge( + &slot, + Err(Error::index("inconsistent test partitions".to_string())), + ); + + assert!(slot.lock().unwrap().is_none()); + } + + #[test] + fn test_zero_token_element_documents_remain_addressable_after_flush() { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_null(); + tags.values().append_value(""); + tags.values().append_value("!!!"); + tags.append(true); + let tags = tags.finish(); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "tags", + tags.data_type().clone(), + true, + )])), + vec![Arc::new(tags)], + ) + .unwrap(); + let index = FtsMemIndex::try_with_params( + 1, + "tags".to_string(), + InvertedIndexParams::default().document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + + index.insert(&batch, 0).unwrap(); + index.flush(); + + assert!(!index.is_empty()); + assert_eq!(index.doc_count(), 3); + assert!(index.search("missing").is_empty()); + } + #[test] fn test_fts_index_insert_and_search() { let schema = create_test_schema(); @@ -3652,6 +5047,234 @@ mod tests { assert!(entries.is_empty()); } + #[test] + fn test_code_analyzer_and_query_uses_position_alternatives() { + let schema = create_test_schema(); + let index = FtsMemIndex::with_params( + 1, + "description".to_string(), + InvertedIndexParams::code().split_identifiers(true), + ); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(StringArray::from(vec!["get user name", "getUserName"])), + ], + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + let query = FtsQueryExpr::match_query_with_operator("getUserName", Operator::And); + let mut rows = index + .search_query(&query) + .into_iter() + .map(|entry| entry.row_position) + .collect::>(); + rows.sort_unstable(); + assert_eq!(rows, vec![0, 1]); + } + + #[test] + fn test_code_analyzer_queries_keep_element_document_identity() { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("get"); + tags.values().append_value("x user name"); + tags.append(true); + tags.values().append_value("getUserName"); + tags.append(true); + let tags = tags.finish(); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "tags", + tags.data_type().clone(), + true, + )])), + vec![Arc::new(tags)], + ) + .unwrap(); + let index = FtsMemIndex::try_with_params( + 1, + "tags".to_string(), + InvertedIndexParams::code() + .with_position(true) + .split_identifiers(true) + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + let and_query = FtsQueryExpr::match_query_with_operator("getUserName", Operator::And); + let phrase_query = FtsQueryExpr::phrase("getUserName"); + for entries in [ + index.search_query(&and_query), + index.search_query(&phrase_query), + ] { + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].row_position, 1); + assert_eq!(entries[0].doc_index, Some(vec![0])); + } + + index.flush(); + let partition_only = SearchOptions::new().with_include_tail(false); + for entries in [ + index.search_with_options(&and_query, partition_only.clone()), + index.search_with_options(&phrase_query, partition_only), + ] { + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].row_position, 1); + assert_eq!(entries[0].doc_index, Some(vec![0])); + } + } + + fn tail_positions_for(index: &FtsMemIndex, term: &str, row_position: RowPosition) -> Vec { + let st = index.state.load_full(); + let snap = st.tail.snapshot(); + let entry = st.tail.terms.get(term).expect("term should be indexed"); + let slice = entry.value().load(); + for chunk in slice.chunks() { + if chunk.batch_position >= snap.visible_count { + continue; + } + if let Ok(doc_idx) = chunk.row_positions.binary_search(&row_position) { + return chunk + .positions + .as_ref() + .expect("test index stores positions") + .doc_positions(doc_idx) + .to_vec(); + } + } + panic!("term {term} should be present in row {row_position}"); + } + + #[test] + fn test_code_analyzer_phrase_uses_token_positions_and_alternatives() { + let schema = create_test_schema(); + let index = FtsMemIndex::with_params( + 1, + "description".to_string(), + InvertedIndexParams::code() + .with_position(true) + .split_identifiers(true), + ); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![0, 1, 2, 3])), + Arc::new(StringArray::from(vec![ + "get user name", + "getUserName", + "get user", + "get fast user name", + ])), + ], + ) + .unwrap(); + index.insert(&batch, 0).unwrap(); + + assert_eq!(tail_positions_for(&index, "getusername", 1), vec![0]); + assert_eq!(tail_positions_for(&index, "get", 1), vec![0]); + assert_eq!(tail_positions_for(&index, "user", 1), vec![1]); + assert_eq!(tail_positions_for(&index, "name", 1), vec![2]); + + let query = FtsQueryExpr::phrase("getUserName"); + assert_eq!(rows(index.search_phrase("getUserName", 0)), vec![0, 1]); + assert_eq!(rows(index.search_query(&query)), vec![0, 1]); + + index.flush(); + let partition_only = SearchOptions::new().with_include_tail(false); + assert_eq!( + rows(index.search_with_options(&query, partition_only)), + vec![0, 1] + ); + } + + #[test] + fn test_zero_token_documents_are_skipped_across_memwal_paths() { + let params = + InvertedIndexParams::new("whitespace".to_string(), lance_tokenizer::Language::English) + .remove_stop_words(true) + .stem(false) + .max_token_length(Some(6)); + let schema = create_test_schema(); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec![ + Some(""), + Some(" "), + Some("the"), + Some("overlength"), + None, + Some("hello"), + ])), + ], + ) + .unwrap(); + let index = FtsMemIndex::with_params(1, "description".to_string(), params.clone()); + index.insert(&batch, 0).unwrap(); + + assert_eq!(index.doc_count(), 1); + let st = index.state.load_full(); + let tail_snap = st.tail.snapshot(); + let tokens = vec!["hello".to_string()]; + let tail_scorer = build_scorer(&st, &tail_snap, &tokens, true); + let expected_scorer = MemBM25Scorer::new(1, 1, HashMap::from([("hello".to_string(), 1)])); + assert_eq!(tail_scorer.total_tokens, 1); + assert_eq!(tail_scorer.num_docs(), 1); + assert_eq!(tail_scorer.num_docs_containing_token("hello"), 1); + assert_eq!( + tail_scorer.avg_doc_length(), + expected_scorer.avg_doc_length() + ); + assert_eq!( + tail_scorer.query_weight("hello"), + expected_scorer.query_weight("hello") + ); + let tail_results = index.search("hello"); + assert_eq!(rows(tail_results.clone()), vec![5]); + let tail_score = tail_results[0].score; + assert!(!index.to_index_builder(0, 6).unwrap().is_empty()); + + index.flush(); + let st = index.state.load_full(); + assert_eq!(st.partitions.len(), 1); + assert_eq!( + st.partitions[0] + .docs + .iter() + .map(|(row_id, num_tokens)| (*row_id, *num_tokens)) + .collect::>(), + vec![(5, 1)] + ); + let frozen_scorer = build_scorer(&st, &st.tail.snapshot(), &tokens, true); + assert_eq!(frozen_scorer.total_tokens, 1); + assert_eq!(frozen_scorer.num_docs(), 1); + assert_eq!(frozen_scorer.num_docs_containing_token("hello"), 1); + assert_eq!( + frozen_scorer.avg_doc_length(), + expected_scorer.avg_doc_length() + ); + assert_eq!( + frozen_scorer.query_weight("hello"), + expected_scorer.query_weight("hello") + ); + let frozen_results = index.search("hello"); + assert_eq!(rows(frozen_results.clone()), vec![5]); + assert!((frozen_results[0].score - tail_score).abs() < f32::EPSILON); + + let all_zero_batch = batch.slice(0, 5); + let all_zero_index = FtsMemIndex::with_params(1, "description".to_string(), params); + all_zero_index.insert(&all_zero_batch, 0).unwrap(); + assert!(all_zero_index.is_empty()); + assert_eq!(all_zero_index.doc_count(), 0); + assert!(all_zero_index.to_index_builder(0, 5).unwrap().is_empty()); + all_zero_index.flush(); + assert!(all_zero_index.state.load().partitions.is_empty()); + } + fn create_phrase_test_batch(schema: &ArrowSchema) -> RecordBatch { RecordBatch::try_new( Arc::new(schema.clone()), @@ -3820,14 +5443,35 @@ mod tests { let batch = create_boolean_test_batch(&schema); index.insert(&batch, 0).unwrap(); + let rust = FtsQueryExpr::match_query("rust").with_boost(2.0); + let programming = FtsQueryExpr::match_query("programming").with_boost(3.0); + let rust_score = index + .search_query(&rust) + .into_iter() + .find(|entry| entry.row_position == 0) + .unwrap() + .score; + let programming_score = index + .search_query(&programming) + .into_iter() + .find(|entry| entry.row_position == 0) + .unwrap() + .score; let query = FtsQueryExpr::boolean() - .must(FtsQueryExpr::match_query("rust")) - .must(FtsQueryExpr::match_query("programming")) + .must(rust.clone()) + .must(programming.clone()) .build(); let entries = index.search_query(&query); assert_eq!(entries.len(), 1); assert_eq!(entries[0].row_position, 0); + let expected_score = rust_score + programming_score; + assert!((entries[0].score - expected_score).abs() < 1e-6); + + let reversed = FtsQueryExpr::boolean().must(programming).must(rust).build(); + let reversed_entries = index.search_query(&reversed); + assert_eq!(reversed_entries.len(), 1); + assert!((reversed_entries[0].score - expected_score).abs() < 1e-6); } #[test] @@ -4545,13 +6189,13 @@ mod tests { let schema = create_test_schema(); let index = FtsMemIndex::new(1, "description".to_string()); - let empty = index.memory_usage(); + let empty = index.resident_bytes_exact(); index.insert(&create_test_batch(&schema), 0).unwrap(); - let after_one = index.memory_usage(); + let after_one = index.resident_bytes_exact(); index .insert(&create_phrase_test_batch(&schema), 100) .unwrap(); - let after_two = index.memory_usage(); + let after_two = index.resident_bytes_exact(); assert!(after_one > empty, "memory should grow after first insert"); assert!( @@ -4560,6 +6204,37 @@ mod tests { ); } + /// The tail's running byte counter must stay exactly in step with the walk + /// it replaces, including across a freeze (which swaps in a fresh tail). + #[test] + fn test_tail_bytes_tracks_resident_bytes() { + let schema = create_test_schema(); + // Freeze partway through so the counter is checked on both a live tail + // and a post-freeze one. + let index = FtsMemIndex::new(1, "description".to_string()).with_freeze_threshold_rows(4); + + for round in 0..6 { + let batch = if round % 2 == 0 { + create_test_batch(&schema) + } else { + create_phrase_test_batch(&schema) + }; + index.insert(&batch, round * 100).unwrap(); + + let st = index.state.load(); + assert_eq!( + st.tail.resident_bytes_cached(), + st.tail.resident_bytes(), + "tail byte counter drifted from the walk at round {round}" + ); + assert_eq!( + index.resident_bytes(), + index.resident_bytes_exact(), + "index memory_size drifted from memory_usage at round {round}" + ); + } + } + #[test] fn test_partial_doc_never_visible_phrase() { // A phrase query inside a single document must either match fully @@ -4666,6 +6341,18 @@ mod tests { assert!(builder.id() > 0 || builder.id() == 42); } + #[test] + fn test_to_index_builder_supports_block_size_256() { + let schema = create_test_schema(); + let params = InvertedIndexParams::default().block_size(256).unwrap(); + let index = FtsMemIndex::try_with_params(1, "description".to_string(), params).unwrap(); + let batch = create_test_batch(&schema); + index.insert(&batch, 0).unwrap(); + + let builder = index.to_index_builder(42, 3).unwrap(); + assert_eq!(builder.id(), 42); + } + #[test] fn test_unsupported_column_type_errors() { let schema = Arc::new(ArrowSchema::new(vec![ @@ -4684,7 +6371,12 @@ mod tests { .unwrap(); let err = index.insert(&batch, 0).unwrap_err(); - assert!(err.to_string().contains("only supports"), "{err}"); + let message = err.to_string(); + assert!( + message.contains("must resolve to Utf8, LargeUtf8, Utf8View, or JSON") + && message.contains("got Int32"), + "{err}" + ); } // ===== Partition-structured redesign ===== diff --git a/rust/lance/src/dataset/mem_wal/index/hnsw.rs b/rust/lance/src/dataset/mem_wal/index/hnsw.rs index 7c7993bb298..2a2eb5afe73 100644 --- a/rust/lance/src/dataset/mem_wal/index/hnsw.rs +++ b/rust/lance/src/dataset/mem_wal/index/hnsw.rs @@ -156,6 +156,29 @@ impl HnswMemIndex { self.len() == 0 } + /// Upper bound on heap bytes held — or already committed — by this index. + /// + /// Sized by `capacity` (the writer's `max_memtable_rows`) rather than by + /// rows inserted: the graph and lookup slabs are pre-allocated in full on + /// the first insert, so an idle vector memtable costs the same as a full + /// one. + /// + /// Non-zero *before* that first insert too. The allocation is settled the + /// moment the index exists — only `dim` is still unknown, and no term + /// depends on it — so reporting zero until the row that triggers it would + /// hide the largest allocation in a vector memtable from the admission + /// controller that runs just ahead of it. Until then this is the reserved + /// estimate; from the first insert on it is the graph's own measurement. + pub(crate) fn resident_bytes(&self) -> usize { + match self.state.get() { + Some(s) => s.graph.resident_bytes() + s.storage.resident_bytes(), + None => { + HnswGraph::reserved_bytes(self.capacity, &build_params_of(&self.build_params)) + + ArrowFixedSizeListVectorStore::reserved_bytes(self.capacity, self.max_batches) + } + } + } + fn ensure_state(&self, dim: usize) -> Result<&HnswState> { if let Some(state) = self.state.get() { if state.storage.dim() != dim { @@ -409,25 +432,41 @@ impl HnswMemIndex { if state.graph.is_empty() { return Ok(None); } + // Bound the graph by storage, and only in that direction. A graph past + // storage names rows the batch has no vector for, which is the defect + // this fixes. Storage past the graph is left whole on purpose: those + // rows are unreachable by traversal either way, but `HNSW::search` + // brute-forces the storage domain under a narrow prefilter + // (`flat_search`), so dropping them would lose results this export + // previously returned. Closing that gap means finishing index + // application before export, not trimming storage to match. let storage_batch = state.storage.to_record_batch(total_rows)?; - let hnsw_batch = state.graph.to_lance_hnsw_batch()?; + let hnsw_batch = state + .graph + .to_lance_hnsw_batch(Some(storage_batch.num_rows()))?; let hnsw = HNSW::load(hnsw_batch)?; Ok(Some((hnsw, storage_batch))) } } fn to_lance_hnsw_params(params: &HnswBuildParams) -> Result { - let params = BuildParams { + let params = build_params_of(params); + // Validate by constructing a tiny graph with these params. This keeps + // invalid builder options as boundary errors instead of delayed panics. + HnswGraph::try_new(1, params.clone())?; + Ok(params) +} + +/// The same field-for-field translation without the validating build, so sizing +/// questions can be answered off a config that has not been accepted yet. +fn build_params_of(params: &HnswBuildParams) -> BuildParams { + BuildParams { max_level: params.max_level, m: params.m, ef_construction: params.ef_construction, prefetch_distance: params.prefetch_distance, ..BuildParams::default() - }; - // Validate by constructing a tiny graph with these params. This keeps - // invalid builder options as boundary errors instead of delayed panics. - HnswGraph::try_new(1, params.clone())?; - Ok(params) + } } #[cfg(test)] @@ -462,6 +501,56 @@ mod tests { RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(ids)), Arc::new(fsl)]).unwrap() } + /// The graph is pre-allocated from `capacity`, so its footprint is settled + /// before any row arrives and barely moves as rows do. A memory budget that + /// samples only row bytes would miss all of it, and one that waited for the + /// first insert would miss the allocation that insert triggers. + #[test] + fn test_resident_bytes_is_preallocated_not_proportional_to_rows() { + let dim = 8; + let capacity = 4_000; + let index = || { + HnswMemIndex::with_capacity( + 1, + "vector".to_string(), + DistanceType::L2, + HnswBuildParams::default().num_edges(16).ef_construction(64), + capacity, + 64, + ) + }; + + let untouched = index().resident_bytes(); + + let sparse = index(); + sparse.insert(&make_batch(0, 1, dim), 0).unwrap(); + let one_row = sparse.resident_bytes(); + + let full = index(); + full.insert(&make_batch(0, capacity, dim), 0).unwrap(); + let all_rows = full.resident_bytes(); + + // One row already pays for the whole graph: well over a KB per slot of + // capacity, and within a small factor of the fully-populated index. + assert!( + one_row > capacity * 128, + "one row should commit the pre-allocated graph, got {one_row} for capacity {capacity}" + ); + assert!( + all_rows < one_row * 2, + "a full index ({all_rows}) should not dwarf a one-row index ({one_row})" + ); + + // The charge is visible before the row that commits it, and close + // enough to the real thing to admit against. The reservation walks the + // level ladder in expectation where the graph samples it, so allow a + // 25% band either way rather than demanding equality. + assert!( + untouched.abs_diff(one_row) * 4 < one_row, + "reserved {untouched} should track the built graph {one_row} before the first insert" + ); + } + #[test] fn test_index_insert_and_search() { let dim = 8; @@ -580,6 +669,61 @@ mod tests { assert!(results.is_empty()); } + /// Storage leading the graph must keep its rows. + /// + /// `insert_batches` appends storage before it builds and publishes the + /// graph, so storage can lead. Those rows are unreachable by traversal + /// either way, but `HNSW::search` brute-forces the storage domain under a + /// narrow prefilter, so trimming storage to the graph would drop results + /// this export used to return. The graph still may not exceed storage. + #[test] + fn to_lance_hnsw_keeps_storage_rows_the_graph_has_not_reached() { + let dim = 8; + let n = 32; + let index = HnswMemIndex::with_capacity( + 1, + "vector".to_string(), + DistanceType::L2, + HnswBuildParams::default().num_edges(8).ef_construction(32), + n * 2, + 4, + ); + index.insert(&make_batch(0, n, dim), 0).unwrap(); + + // Reproduce the interval: storage takes the next batch, the graph does + // not see it yet. + let state = index.state.get().expect("state is initialized"); + let extra = make_batch(n as i32, n, dim); + let vectors = extra + .column_by_name("vector") + .unwrap() + .as_fixed_size_list_opt() + .unwrap() + .clone(); + state + .storage + .append_batch(Arc::new(vectors), n as u64) + .unwrap(); + assert!( + state.storage.committed_len() > state.graph.len(), + "the test needs storage ahead of the graph" + ); + + let Some((hnsw, storage_batch)) = index.to_lance_hnsw(None).unwrap() else { + panic!("expected HNSW snapshot"); + }; + assert_eq!( + storage_batch.num_rows(), + n * 2, + "storage keeps every committed row; a narrow prefilter scans them" + ); + assert_eq!(hnsw.len(), n, "the graph covers only what it indexed"); + assert!( + hnsw.len() <= storage_batch.num_rows(), + "the graph must never name a row storage has no vector for" + ); + } + #[test] fn test_to_lance_hnsw_reverses_row_ids() { let dim = 8; diff --git a/rust/lance/src/dataset/mem_wal/manifest.rs b/rust/lance/src/dataset/mem_wal/manifest.rs index 9c2a3aa2163..a2099e97d44 100644 --- a/rust/lance/src/dataset/mem_wal/manifest.rs +++ b/rust/lance/src/dataset/mem_wal/manifest.rs @@ -29,7 +29,7 @@ use object_store::ObjectStoreExt; use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use bytes::Bytes; use futures::StreamExt; @@ -39,8 +39,6 @@ use lance_index::mem_wal::{ShardManifest, ShardStatus}; use lance_io::object_store::ObjectStore; use lance_table::format::pb; use log::{info, warn}; -use object_store::PutMode; -use object_store::PutOptions; use object_store::path::Path; use prost::Message; use serde::{Deserialize, Serialize}; @@ -65,6 +63,15 @@ pub struct ShardManifestStore { shard_id: Uuid, manifest_dir: Path, manifest_scan_batch_size: usize, + /// This store's position: the version it may build its next write on, and + /// what [`Self::latest`] serves. + /// + /// Set by a landed write and by [`Self::refresh_latest`] — the epoch holder + /// is the sole permitted writer, so what it wrote or last refreshed to + /// stays latest until a PUT-IF-NOT-EXISTS collision proves otherwise. A + /// plain [`Self::latest`] scan never sets it, so a reader that polls keeps + /// observing the writer instead of pinning the first manifest it saw. + latest: RwLock>, } impl ShardManifestStore { @@ -88,14 +95,69 @@ impl ShardManifestStore { shard_id, manifest_dir, manifest_scan_batch_size, + latest: RwLock::new(None), + } + } + + /// The cached manifest, if this store has written one. + fn cached(&self) -> Option { + self.latest.read().expect("manifest cache lock").clone() + } + + /// Publish `manifest` as the latest. Only ever called after a durable write. + /// + /// Never regresses: the flush task and the tailer's cursor updates share one + /// handle, so two writes can win their CAS in one order and return to their + /// callers in the other. + fn cache(&self, manifest: &ShardManifest) { + let mut latest = self.latest.write().expect("manifest cache lock"); + if latest.as_ref().is_none_or(|c| manifest.version > c.version) { + *latest = Some(manifest.clone()); + } + } + + /// Drop the cache on a write collision — the one signal that another + /// writer may have moved the shard past us. + fn invalidate(&self) { + *self.latest.write().expect("manifest cache lock") = None; + } + + /// The latest manifest as far as this store knows: its own position when it + /// has one, otherwise a scan of storage. + /// + /// Cheap, and deliberately not authoritative — it can sit behind a peer's + /// commit, and a scan here does *not* become this store's position, so a + /// reader that polls keeps observing the writer. To observe a peer, or to + /// take a position to write from, use [`Self::refresh_latest`]. + /// + /// Returns `None` if no manifest exists (new shard). + pub async fn latest(&self) -> Result> { + match self.cached() { + Some(cached) => Ok(Some(cached)), + None => self.scan_latest().await, } } - /// Read the latest manifest version. + /// Read the latest manifest from storage and adopt it as this store's + /// position. + /// + /// The adopting half matters: a claim reads uncached precisely because it + /// must see another process, and what it finds is the version its own write + /// then builds on. Callers that only want to *look* want [`Self::latest`], + /// which leaves this store's position alone. /// /// Returns `None` if no manifest exists (new shard). - #[instrument(name = "manifest_read_latest", level = "debug", skip_all, fields(shard_id = %self.shard_id))] - pub async fn read_latest(&self) -> Result> { + #[instrument(name = "manifest_refresh_latest", level = "debug", skip_all, fields(shard_id = %self.shard_id))] + pub async fn refresh_latest(&self) -> Result> { + let latest = self.scan_latest().await?; + if let Some(manifest) = &latest { + self.cache(manifest); + } + Ok(latest) + } + + /// Scan storage for the latest manifest, touching no local state. + async fn scan_latest(&self) -> Result> { let version = self.find_latest_version().await?; if version == 0 { return Ok(None); @@ -146,13 +208,13 @@ impl ShardManifestStore { replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, current_generation: 1, - flushed_generations: vec![], + sstables: vec![], status: ShardStatus::Active, }; match self.write(&manifest).await { Ok(_) => Ok(manifest), - Err(error) => match self.read_latest().await? { + Err(error) => match self.refresh_latest().await? { Some(existing) if existing.shard_spec_id == manifest.shard_spec_id && existing.shard_field_values == manifest.shard_field_values => @@ -172,80 +234,55 @@ impl ShardManifestStore { /// /// Returns the version that was written. /// + /// Callers derive `manifest.version` from a manifest they just read, which + /// is what keeps the sequence gap-free — the cache treats a landed write as + /// proof of the tip, and `find_latest_version` stops at the first absent + /// batch, so a gap hides every version past it. Whoever holds that + /// predecessor checks the successor; see [`Self::commit_update`]. + /// + /// A version at or below this store's position is reported as the collision + /// it is, so callers retry. + /// /// # Errors /// - /// Returns `Error::AlreadyExists` if another writer already wrote this version. + /// Returns [`Error::RetryableCommitConflict`] if another writer already + /// holds this version. #[instrument(name = "manifest_write", level = "debug", skip_all, fields(shard_id = %self.shard_id, version = manifest.version, epoch = manifest.writer_epoch))] - pub async fn write(&self, manifest: &ShardManifest) -> Result { + pub(crate) async fn write(&self, manifest: &ShardManifest) -> Result { let version = manifest.version; + if self.cached().is_some_and(|c| version <= c.version) { + // Someone already took it — our own position proves it exists. + // Report the collision so callers retry rather than fail. + self.invalidate(); + return Err(self.version_taken(version)); + } let filename = manifest_filename(version); let path = self.manifest_dir.clone().join(filename.as_str()); let pb_manifest = pb::ShardManifest::from(manifest); let bytes = pb_manifest.encode_to_vec(); - if self.object_store.is_local() { - // Local storage: Use temp file + atomic rename for fencing - let temp_filename = format!("{}.tmp.{}", filename, uuid::Uuid::new_v4()); - let temp_path = self.manifest_dir.clone().join(temp_filename.as_str()); - - // Write to temp file - self.object_store - .inner - .put(&temp_path, Bytes::from(bytes).into()) - .await - .map_err(|e| Error::io(format!("Failed to write temp manifest: {}", e)))?; - - // Atomically rename to final path - match self - .object_store - .inner - .rename_if_not_exists(&temp_path, &path) - .await - { - Ok(()) => {} - Err(object_store::Error::AlreadyExists { .. }) => { - // Clean up temp file - let _ = self.object_store.delete(&temp_path).await; - return Err(Error::io(format!( - "Manifest version {} already exists for shard {}", - version, self.shard_id - ))); - } - Err(e) => { - // Clean up temp file - let _ = self.object_store.delete(&temp_path).await; - return Err(Error::io(format!( + self.object_store + .put_if_absent(&path, Bytes::from(bytes).into()) + .await + .inspect_err(|_| self.invalidate()) + .map_err(|error| { + if matches!( + error, + object_store::Error::AlreadyExists { .. } + | object_store::Error::Precondition { .. } + ) { + self.version_taken(version) + } else { + Error::io(format!( "Failed to write manifest version {} for shard {}: {}", - version, self.shard_id, e - ))); + version, self.shard_id, error + )) } - } - } else { - // Cloud storage: Use PUT-IF-NOT-EXISTS - let put_opts = PutOptions { - mode: PutMode::Create, - ..Default::default() - }; + })?; - self.object_store - .inner - .put_opts(&path, Bytes::from(bytes).into(), put_opts) - .await - .map_err(|e| { - if matches!(e, object_store::Error::AlreadyExists { .. }) { - Error::io(format!( - "Manifest version {} already exists for shard {}", - version, self.shard_id - )) - } else { - Error::io(format!( - "Failed to write manifest version {} for shard {}: {}", - version, self.shard_id, e - )) - } - })?; - } + // The write landed, so this is now the latest. + self.cache(manifest); // Best-effort update version hint (failures are logged as warnings) self.write_version_hint(version).await; @@ -253,6 +290,19 @@ impl ShardManifestStore { Ok(version) } + /// The error for a version another writer already holds. `commit_update` + /// matches on the variant to decide whether to retry. + fn version_taken(&self, version: u64) -> Error { + Error::retryable_commit_conflict_source( + version, + format!( + "Manifest version {} already exists for shard {}", + version, self.shard_id + ) + .into(), + ) + } + /// Find the latest manifest version. /// /// Uses HEAD requests starting from version hint, scanning forward @@ -285,9 +335,7 @@ impl ShardManifestStore { let mut found_any = false; while let Some((version, result)) = futures.next().await { - if let Ok(true) = result - && version > latest_found - { + if result? && version > latest_found { latest_found = version; found_any = true; } @@ -424,7 +472,9 @@ impl ShardManifestStore { const MAX_CLAIM_RETRIES: usize = 16; let mut last_write_err: Option = None; for _ in 0..MAX_CLAIM_RETRIES { - let current = self.read_latest().await?; + // Refreshing, not reading: a claim exists to discover another + // writer's epoch, and the tip it finds is what our write builds on. + let current = self.refresh_latest().await?; // A sealed shard is mid-drop (drop-table 2PC). Refuse the claim // with a distinguishable error rather than minting a new epoch, @@ -442,7 +492,7 @@ impl ShardManifestStore { } let (next_version, next_epoch, base_manifest) = match current { - Some(m) => (m.version + 1, m.writer_epoch + 1, Some(m)), + Some(m) => (m.next_version(), m.writer_epoch + 1, Some(m)), None => (1, 1, None), }; @@ -462,7 +512,7 @@ impl ShardManifestStore { replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, current_generation: 1, - flushed_generations: vec![], + sstables: vec![], status: ShardStatus::Active, } }; @@ -477,7 +527,7 @@ impl ShardManifestStore { } Err(write_err) => { let latest_epoch = self - .read_latest() + .refresh_latest() .await? .map(|m| m.writer_epoch) .unwrap_or(0); @@ -508,7 +558,9 @@ impl ShardManifestStore { /// is higher than the local epoch, the writer has been fenced. #[instrument(name = "manifest_check_fenced", level = "debug", skip_all, fields(shard_id = %self.shard_id, local_epoch))] pub async fn check_fenced(&self, local_epoch: u64) -> Result<()> { - let current = self.read_latest().await?; + // Refreshed: a fence is another process's write, which our own + // position can never show us. + let current = self.refresh_latest().await?; Self::check_fenced_against(¤t, local_epoch, self.shard_id) } @@ -539,11 +591,28 @@ impl ShardManifestStore { /// # Arguments /// /// * `local_epoch` - The writer's epoch (for fencing check) - /// * `prepare_fn` - Function that takes current manifest and returns new manifest + /// * `prepare_fn` - Function that takes current manifest and returns new + /// manifest. Its `version` must be `current.next_version()`; anything + /// else is rejected, so the sequence cannot develop a gap. /// /// # Returns /// /// The successfully written manifest. + /// + /// # Concurrency + /// + /// Each losing CAS clears the store's shared position, so commits that + /// overlap within one CAS round-trip all fall back to a scan and retry — + /// roughly `n^2/2` scans for `n` of them. Commits spaced further apart than + /// that window cost nothing: the winner leaves its position warm for the + /// next one. + /// + /// `MAX_RETRIES` therefore bounds how many commits can overlap on one + /// handle: the unluckiest loses every round, so past ten concurrent commits + /// it exhausts its budget and returns the conflict instead of landing. + /// Reaching that needs eleven commit sources inside a single CAS, which no + /// current caller comes close to. Worth revisiting if one funnels many + /// independent writers through a single [`Self`]. #[instrument(name = "manifest_commit_update", level = "debug", skip_all, fields(shard_id = %self.shard_id, local_epoch))] pub async fn commit_update(&self, local_epoch: u64, prepare_fn: F) -> Result where @@ -552,11 +621,16 @@ impl ShardManifestStore { const MAX_RETRIES: usize = 10; for attempt in 0..MAX_RETRIES { - // Step 1: Read latest - let current = self - .read_latest() - .await? - .ok_or_else(|| Error::io("Shard manifest not found"))?; + // Step 1: take a position to build on. A cold cache — a fresh + // store, or a retry after losing a race — must go to storage and + // adopt what it finds, or the write below has no baseline. + let current = match self.cached() { + Some(cached) => cached, + None => self + .refresh_latest() + .await? + .ok_or_else(|| Error::io("Shard manifest not found"))?, + }; // Step 2: Check fencing Self::check_fenced_against(&Some(current.clone()), local_epoch, self.shard_id)?; @@ -564,6 +638,17 @@ impl ShardManifestStore { // Step 3: Prepare new manifest let new_manifest = prepare_fn(¤t); + // Check the successor against `current`, the manifest the closure + // actually built on. The store's position is shared and moves + // under concurrent commits — a peer's failed CAS can clear it + // between here and the write — so it cannot judge this. + if new_manifest.version != current.next_version() { + return Err(Error::invalid_input(format!( + "manifest version {} is not the successor of {} for shard {}: the version sequence must stay gap-free", + new_manifest.version, current.version, self.shard_id + ))); + } + // Validate epoch matches if new_manifest.writer_epoch != local_epoch { return Err(Error::invalid_input(format!( @@ -579,7 +664,7 @@ impl ShardManifestStore { } Err(e) => { // Check if it's a version conflict (can retry) vs other error - let is_version_conflict = e.to_string().contains("already exists"); + let is_version_conflict = matches!(e, Error::RetryableCommitConflict { .. }); if is_version_conflict && attempt < MAX_RETRIES - 1 { continue; @@ -600,6 +685,8 @@ impl ShardManifestStore { #[cfg(test)] mod tests { use super::*; + use lance_core::utils::testing::{ProxyObjectStore, ProxyObjectStorePolicy}; + use std::sync::Mutex; use tempfile::TempDir; async fn create_local_store() -> (Arc, Path, TempDir) { @@ -619,18 +706,155 @@ mod tests { replay_after_wal_entry_position: 0, wal_entry_position_last_seen: 0, current_generation: 1, - flushed_generations: vec![], + sstables: vec![], status: ShardStatus::Active, } } + /// A warm cache must not hide a successor's claim from `check_fenced`. + #[tokio::test] + async fn check_fenced_sees_a_peer_through_a_warm_cache() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let incumbent = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + let successor = ShardManifestStore::new(store, &base_path, shard_id, 2); + + // Claiming writes a manifest, warming the cache. + let (epoch, _) = incumbent.claim_epoch(0).await.unwrap(); + assert!(incumbent.cached().is_some(), "the claim write must cache"); + + successor.claim_epoch(0).await.unwrap(); + + assert!( + incumbent.check_fenced(epoch).await.is_err(), + "a cached manifest must not hide a successor's epoch" + ); + } + + /// Reads must not cache, or a reader-only handle (a WAL tailer, a + /// drop-reconcile probe) would never observe the writer. + #[tokio::test] + async fn a_reader_only_store_never_caches() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let writer = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + let reader = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let (epoch, _) = writer.claim_epoch(0).await.unwrap(); + assert_eq!( + reader.latest().await.unwrap().unwrap().current_generation, + 1 + ); + assert!( + reader.cached().is_none(), + "a read must not populate the cache" + ); + + writer + .commit_update(epoch, |c| ShardManifest { + version: c.version + 1, + current_generation: 5, + ..c.clone() + }) + .await + .unwrap(); + + assert_eq!( + reader.latest().await.unwrap().unwrap().current_generation, + 5, + "a reader must see the writer's later commits" + ); + } + + /// A losing `commit_update` re-reads from storage, so it converges instead + /// of spinning on the version it lost on. #[tokio::test] - async fn test_read_latest_empty() { + async fn commit_update_recovers_from_a_stale_cache() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let ours = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + let peer = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let (epoch, _) = ours.claim_epoch(0).await.unwrap(); + ours.latest().await.unwrap().unwrap(); + + // A same-epoch commit through another handle stales our cache. + peer.commit_update(epoch, |c| ShardManifest { + version: c.version + 1, + current_generation: 7, + ..c.clone() + }) + .await + .unwrap(); + + let updated = ours + .commit_update(epoch, |c| ShardManifest { + version: c.version + 1, + wal_entry_position_last_seen: 42, + ..c.clone() + }) + .await + .unwrap(); + + // Built on the peer's version, not on the stale cached one. + assert_eq!(updated.current_generation, 7); + assert_eq!(updated.wal_entry_position_last_seen, 42); + assert_eq!( + ours.refresh_latest().await.unwrap().unwrap().version, + updated.version + ); + } + + /// A write whose CAS won earlier but returned later must not publish its + /// older manifest over the newer one. + #[tokio::test] + async fn a_late_write_never_regresses_the_cache() { let (store, base_path, _temp_dir) = create_local_store().await; let shard_id = Uuid::new_v4(); let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2); - let result = manifest_store.read_latest().await.unwrap(); + let older = create_test_manifest(shard_id, 1, 1); + let newer = create_test_manifest(shard_id, 2, 1); + manifest_store.write(&older).await.unwrap(); + manifest_store.write(&newer).await.unwrap(); + + // The straggler resolving after the newer write already cached. + manifest_store.cache(&older); + + assert_eq!( + manifest_store.latest().await.unwrap().unwrap().version, + 2, + "the cache must hold the newest version this store wrote" + ); + } + + /// The cached read and the storage read agree after a write. + #[tokio::test] + async fn latest_serves_the_written_manifest() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let mut manifest = create_test_manifest(shard_id, 1, 1); + manifest_store.write(&manifest).await.unwrap(); + manifest.version = 2; + manifest.current_generation = 9; + manifest_store.write(&manifest).await.unwrap(); + + let cached = manifest_store.latest().await.unwrap().unwrap(); + let durable = manifest_store.refresh_latest().await.unwrap().unwrap(); + assert_eq!(cached.version, 2); + assert_eq!(cached.current_generation, 9); + assert_eq!(cached, durable); + } + + #[tokio::test] + async fn test_latest_empty() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let result = manifest_store.latest().await.unwrap(); assert!(result.is_none()); } @@ -643,7 +867,7 @@ mod tests { let manifest = create_test_manifest(shard_id, 1, 1); manifest_store.write(&manifest).await.unwrap(); - let loaded = manifest_store.read_latest().await.unwrap().unwrap(); + let loaded = manifest_store.latest().await.unwrap().unwrap(); assert_eq!(loaded.version, 1); assert_eq!(loaded.writer_epoch, 1); assert_eq!(loaded.shard_id, shard_id); @@ -662,7 +886,7 @@ mod tests { } // Should find latest - let loaded = manifest_store.read_latest().await.unwrap().unwrap(); + let loaded = manifest_store.latest().await.unwrap().unwrap(); assert_eq!(loaded.version, 5); assert_eq!(loaded.writer_epoch, 5); @@ -721,7 +945,7 @@ mod tests { assert_eq!(manifest.shard_spec_id, 3); assert_eq!(manifest.shard_field_values, field_values); - let loaded = manifest_store.read_latest().await.unwrap().unwrap(); + let loaded = manifest_store.latest().await.unwrap().unwrap(); assert_eq!(loaded, manifest); } @@ -799,7 +1023,7 @@ mod tests { err.to_string().contains("sealed"), "expected a distinguishable sealed-refusal error, got: {err}" ); - let after = manifest_store.read_latest().await.unwrap().unwrap(); + let after = manifest_store.latest().await.unwrap().unwrap(); assert_eq!(after.writer_epoch, sealed.writer_epoch, "no epoch minted"); assert_eq!(after.status, ShardStatus::Sealed); @@ -834,4 +1058,216 @@ mod tests { "second initialize_shard with different fields must fail" ); } + + /// A commit closure that names the wrong version fails loudly instead of + /// having its intent rewritten underneath it. + #[tokio::test] + async fn commit_update_rejects_a_closure_that_skips_a_version() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let ours = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let (epoch, claimed) = ours.claim_epoch(0).await.unwrap(); + + let error = ours + .commit_update(epoch, |c| ShardManifest { + version: 99, + current_generation: 7, + ..c.clone() + }) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("is not the successor of"), "{}", error); + assert_eq!( + ours.refresh_latest().await.unwrap().unwrap().version, + claimed.version, + "the rejected commit left the shard alone" + ); + + // The same edit with the right version commits. + let committed = ours + .commit_update(epoch, |c| ShardManifest { + version: c.next_version(), + current_generation: 7, + ..c.clone() + }) + .await + .unwrap(); + assert_eq!(committed.version, claimed.next_version()); + assert_eq!(committed.current_generation, 7); + } + + /// The two reads differ in one thing that matters: `refresh_latest` adopts + /// what it finds as this store's position, `latest` does not. Getting that + /// backwards either pins pollers or rejects valid writes. + #[tokio::test] + async fn only_refresh_latest_adopts_a_position() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let writer = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + let observer = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let (epoch, _) = writer.claim_epoch(0).await.unwrap(); + + // A plain read leaves the observer positionless, so it keeps going to + // storage and keeps seeing the writer. + assert!(observer.latest().await.unwrap().is_some()); + assert!( + observer.cached().is_none(), + "`latest` must not take a position" + ); + + writer + .commit_update(epoch, |c| ShardManifest { + version: c.next_version(), + current_generation: 9, + ..c.clone() + }) + .await + .unwrap(); + assert_eq!( + observer.latest().await.unwrap().unwrap().current_generation, + 9, + "a poller must observe the writer's later commits" + ); + + // Refreshing takes a position, which is what lets a claim write from it. + let refreshed = observer.refresh_latest().await.unwrap().unwrap(); + assert_eq!( + observer.cached().map(|c| c.version), + Some(refreshed.version), + "`refresh_latest` must take a position" + ); + } + + /// The store's position moves under concurrent commits — a peer's failed + /// CAS clears it — so it cannot judge a closure's output. Every commit + /// must land, and none may be lost. + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn concurrent_commits_on_one_handle_all_land() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let shared = Arc::new(ShardManifestStore::new(store, &base_path, shard_id, 2)); + let (epoch, claimed) = shared.claim_epoch(0).await.unwrap(); + + const COMMITS: u64 = 8; + let mut tasks = Vec::new(); + for _ in 0..COMMITS { + let shared = shared.clone(); + tasks.push(tokio::spawn(async move { + shared + .commit_update(epoch, |c| ShardManifest { + version: c.next_version(), + current_generation: c.current_generation + 1, + ..c.clone() + }) + .await + })); + } + + let mut failures = Vec::new(); + for task in tasks { + if let Err(error) = task.await.unwrap() { + failures.push(error.to_string()); + } + } + assert!(failures.is_empty(), "commits failed: {:#?}", failures); + + let tip = shared.refresh_latest().await.unwrap().unwrap(); + assert_eq!( + tip.current_generation, + claimed.current_generation + COMMITS, + "every commit must be reflected; a lost update means one was \ + built on stale state and overwrote an intervening one" + ); + assert_eq!(tip.version, claimed.version + COMMITS); + } + /// A version this store's own position proves is taken must read as a + /// collision, so `commit_update` retries instead of failing. + #[tokio::test] + async fn write_reports_a_taken_version_as_a_collision() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let ours = ShardManifestStore::new(store, &base_path, shard_id, 2); + + let (_, claimed) = ours.claim_epoch(0).await.unwrap(); + + let mut replay = claimed.clone(); + replay.current_generation = 42; + let error = ours.write(&replay).await.unwrap_err(); + assert!( + matches!(error, Error::RetryableCommitConflict { .. }), + "the variant is what commit_update retries on: {:?}", + error + ); + assert!( + ours.cached().is_none(), + "a collision must drop the position so the retry re-reads" + ); + } + + /// A HEAD that fails is not a version that is absent, and the scan must not + /// read it as the end of the sequence: the answer becomes a position. + #[tokio::test] + async fn a_failed_head_is_not_read_as_the_end_of_the_sequence() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + + // The durable tip is v3, written by a peer that claimed epoch 2. + let peer = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + for version in 1..=3u64 { + let epoch = if version == 3 { 2 } else { 1 }; + peer.write(&create_test_manifest(shard_id, version, epoch)) + .await + .unwrap(); + } + + // The hint is written after the manifest and is best-effort, so lagging + // by one is the ordinary state during any commit. + let hint_path = shard_manifest_path(&base_path, &shard_id).join("version_hint.json"); + store + .inner + .put( + &hint_path, + Bytes::from(serde_json::to_vec(&VersionHint { version: 2 }).unwrap()).into(), + ) + .await + .unwrap(); + + // A store whose HEAD on v3 gets a transient 503. + let policy = Arc::new(Mutex::new(ProxyObjectStorePolicy::new())); + let v3_file = manifest_filename(3); + policy.lock().unwrap().set_before_policy( + "503", + Arc::new(move |method: &str, path: &Path| { + if method == "get_opts" && path.as_ref().ends_with(v3_file.as_str()) { + return Err(object_store::Error::Generic { + store: "test", + source: "503 slow down".into(), + } + .into()); + } + Ok(()) + }), + ); + let mut proxied = (*store).clone(); + proxied.inner = Arc::new(ProxyObjectStore::new(store.inner.clone(), policy.clone())); + let ours = ShardManifestStore::new(Arc::new(proxied), &base_path, shard_id, 2); + + let err = ours.refresh_latest().await.unwrap_err(); + assert!( + err.to_string().contains("503"), + "the scan must surface the HEAD failure, got: {err}" + ); + assert!(ours.cached().is_none(), "a failed scan takes no position"); + assert!( + ours.check_fenced(1).await.is_err(), + "a fence check that could not read the tip must not report clear" + ); + + // Once the blip clears, the scan sees the durable tip. + policy.lock().unwrap().clear_before_policy("503"); + assert_eq!(ours.latest().await.unwrap().unwrap().version, 3); + } } diff --git a/rust/lance/src/dataset/mem_wal/memtable.rs b/rust/lance/src/dataset/mem_wal/memtable.rs index 77611fd7c47..151294c035e 100644 --- a/rust/lance/src/dataset/mem_wal/memtable.rs +++ b/rust/lance/src/dataset/mem_wal/memtable.rs @@ -7,7 +7,6 @@ pub mod batch_store; pub mod flush; pub mod scanner; -use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -79,13 +78,6 @@ pub struct MemTable { /// Generation number (incremented on flush). generation: u64, - /// WAL batch mapping: batch_position -> (wal_entry_position, position within WAL entry). - wal_batch_mapping: HashMap, - /// Last WAL entry position that has been flushed. - last_flushed_wal_entry_position: u64, - /// Set of batch IDs that have been flushed to WAL. - flushed_batch_positions: HashSet, - /// Primary key bloom filter for staleness detection. pk_bloom_filter: Sbbf, /// Primary key field IDs (for bloom filter updates). @@ -131,6 +123,24 @@ const PK_BLOOM_FILTER_EXPECTED_ITEMS: u64 = 8192; /// Consistent with lance-index scalar bloomfilter defaults (≈ 1 in 1754). const PK_BLOOM_FILTER_FPP: f64 = 0.00057; +/// Heap one memtable's PK bloom filter holds. +/// +/// The same for every memtable, because it is sized from two constants rather +/// than from the rows in it — so this is a property of the build, not a +/// measurement, and a memory view need not carry it per memtable. +/// +/// Counted as index memory rather than row data: it is an auxiliary lookup +/// structure, and a fixed term in the `max_memtable_size` seal trigger would +/// make every memtable seal a constant early. +pub fn pk_bloom_filter_bytes() -> usize { + static BYTES: std::sync::OnceLock = std::sync::OnceLock::new(); + *BYTES.get_or_init(|| { + Sbbf::with_ndv_fpp(PK_BLOOM_FILTER_EXPECTED_ITEMS, PK_BLOOM_FILTER_FPP) + .map(|f| f.estimated_memory_size()) + .unwrap_or(0) + }) +} + impl MemTable { /// Create a new MemTable with default capacity. /// @@ -187,6 +197,29 @@ impl MemTable { pk_field_ids: Vec, cache_config: CacheConfig, batch_capacity: usize, + ) -> Result { + Self::with_capacity_at( + schema, + generation, + pk_field_ids, + cache_config, + batch_capacity, + 0, + ) + } + + /// Create a memtable whose batch 0 sits at `global_offset` in the writer's + /// batch sequence. Every memtable after the writer's first is rotated in by + /// `freeze_memtable`, which stamps the outgoing memtable's `global_end()` + /// here so writer-global cursors stay mappable onto local batch positions. + #[allow(clippy::too_many_arguments)] + pub fn with_capacity_at( + schema: Arc, + generation: u64, + pk_field_ids: Vec, + cache_config: CacheConfig, + batch_capacity: usize, + global_offset: usize, ) -> Result { let lance_schema = Schema::try_from(schema.as_ref())?; @@ -205,7 +238,7 @@ impl MemTable { let dataset_uri = format!("memory://{}", Uuid::new_v4()); // Create lock-free batch store - let batch_store = Arc::new(BatchStore::with_capacity(batch_capacity)); + let batch_store = Arc::new(BatchStore::with_capacity_at(batch_capacity, global_offset)); // Create memtable_flush_completion cell immediately so backpressure can // wait on it even before the memtable is frozen. Every memtable will @@ -220,9 +253,6 @@ impl MemTable { cache_config, cached_dataset: RwLock::new(None), generation, - wal_batch_mapping: HashMap::new(), - last_flushed_wal_entry_position: 0, - flushed_batch_positions: HashSet::new(), pk_bloom_filter, pk_field_ids, // Initialize with an empty IndexStore so the visibility cursor has @@ -478,56 +508,44 @@ impl MemTable { /// /// Returns true if the batch store is full or estimated size exceeds threshold. pub fn should_flush(&self, max_bytes: usize) -> bool { - self.batch_store.is_full() || self.batch_store.estimated_bytes() >= max_bytes + self.batch_store.is_full() || self.batch_store.row_bytes() >= max_bytes } - /// Get batches visible up to a specific batch position (inclusive). + /// Get the batches in the visible prefix. /// - /// A batch at position `i` is visible if `i <= max_visible_batch_position`. + /// A batch at position `i` is visible if `i < visible_count`. /// /// # Arguments /// - /// * `max_visible_batch_position` - The maximum batch position to include (inclusive) + /// * `visible_count` - Exclusive count of batch positions to include /// /// # Returns /// /// Vector of visible batches. - pub async fn get_visible_batches(&self, max_visible_batch_position: usize) -> Vec { - self.batch_store - .visible_record_batches(max_visible_batch_position) + pub async fn get_visible_batches(&self, visible_count: usize) -> Vec { + self.batch_store.visible_record_batches(visible_count) } - /// Get batch positions visible up to a specific batch position (inclusive). + /// Get the batch positions in the visible prefix. /// /// This is useful for filtering index results by visibility. - pub async fn get_max_visible_batch_positions( - &self, - max_visible_batch_position: usize, - ) -> Vec { - self.batch_store - .max_visible_batch_positions(max_visible_batch_position) + pub async fn get_visible_batch_positions(&self, visible_count: usize) -> Vec { + self.batch_store.visible_batch_positions(visible_count) } /// Check if a specific batch is visible at a given visibility position. /// /// Returns true if the batch is visible, false if not visible or doesn't exist. - pub async fn is_batch_visible( - &self, - batch_position: usize, - max_visible_batch_position: usize, - ) -> bool { + pub async fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool { self.batch_store - .is_batch_visible(batch_position, max_visible_batch_position) + .is_batch_visible(batch_position, visible_count) } /// Scan batches visible up to a specific batch position. /// /// This combines `get_visible_batches` with the scan interface. - pub async fn scan_batches_at_position( - &self, - max_visible_batch_position: usize, - ) -> Result> { - Ok(self.get_visible_batches(max_visible_batch_position).await) + pub async fn scan_batches_at_position(&self, visible_count: usize) -> Result> { + Ok(self.get_visible_batches(visible_count).await) } /// Update the bloom filter with primary keys from a batch. @@ -561,30 +579,6 @@ impl MemTable { Ok(()) } - /// Mark batches as flushed to WAL. - /// - /// Updates the WAL batch mapping for use during MemTable flush. - /// Also updates the batch_store's watermark to the highest flushed batch_position. - pub fn mark_wal_flushed( - &mut self, - batch_positions: &[usize], - wal_entry_position: u64, - positions: &[usize], - ) { - for (idx, &batch_position) in batch_positions.iter().enumerate() { - self.wal_batch_mapping - .insert(batch_position, (wal_entry_position, positions[idx])); - self.flushed_batch_positions.insert(batch_position); - } - self.last_flushed_wal_entry_position = wal_entry_position; - - // Update batch_store watermark to the highest batch_position flushed (inclusive) - if let Some(&max_batch_position) = batch_positions.iter().max() { - self.batch_store - .set_max_flushed_batch_position(max_batch_position); - } - } - /// Get or create a Dataset for reading. /// /// Uses caching based on the configured eventual consistency strategy: @@ -654,7 +648,7 @@ impl MemTable { /// /// This is used when flushing MemTable to persistent storage to ensure /// the flushed data is ordered from newest to oldest. This enables more - /// efficient K-way merge during LSM scan because flushed generations + /// efficient K-way merge during LSM scan because SSTables /// will be pre-sorted in the order needed for deduplication. /// /// The total number of rows in the MemTable is also returned to allow @@ -721,21 +715,6 @@ impl MemTable { self.batch_count() } - /// Get estimated size in bytes. - pub fn estimated_size(&self) -> usize { - self.batch_store.estimated_bytes() + self.pk_bloom_filter.estimated_memory_size() - } - - /// Get the WAL batch mapping. - pub fn wal_batch_mapping(&self) -> &HashMap { - &self.wal_batch_mapping - } - - /// Get the last flushed WAL entry position. - pub fn last_flushed_wal_entry_position(&self) -> u64 { - self.last_flushed_wal_entry_position - } - /// Get the bloom filter for serialization. pub fn bloom_filter(&self) -> &Sbbf { &self.pk_bloom_filter @@ -757,17 +736,15 @@ impl MemTable { self.indexes.take() } - /// Check if all batches have been flushed to WAL. - pub fn all_flushed_to_wal(&self) -> bool { - self.batch_store.pending_wal_flush_count() == 0 + /// Whether every committed batch in this memtable is WAL-durable, given the + /// writer-global durability cursor. The L0 flush's precondition. + pub fn all_flushed_to_wal(&self, durable: usize) -> bool { + self.batch_store.pending_wal_flush_count(durable) == 0 } - /// Get unflushed batch IDs. - pub fn unflushed_batch_positions(&self) -> Vec { - let batch_count = self.batch_count(); - (0..batch_count) - .filter(|id| !self.flushed_batch_positions.contains(id)) - .collect() + /// Writer-global coordinate one past this memtable's last committed batch. + pub fn global_end(&self) -> usize { + self.batch_store.global_end() } /// Get cache configuration. @@ -785,18 +762,13 @@ impl MemTable { self.batch_store.remaining_capacity() } - /// Check if batch store is full. - pub fn is_batch_store_full(&self) -> bool { - self.batch_store.is_full() - } - /// Create a scanner for querying this MemTable. /// /// # Arguments /// - /// * `max_visible_batch_position` - Maximum batch position visible (inclusive) + /// * `visible_count` - Maximum batch position visible (inclusive) /// - /// The scanner captures the current `max_visible_batch_position` from the + /// The scanner captures the current `visible_count` from the /// `IndexStore` at construction time to ensure consistent visibility. /// /// # Panics @@ -931,29 +903,11 @@ mod tests { assert_eq!(total_rows, 15); } + /// `all_flushed_to_wal(durable)` is the L0 flush's precondition (`flush.rs:171`): + /// false while any committed batch is still un-appended, true once the + /// durability watermark covers every one of them. #[tokio::test] - async fn test_memtable_wal_mapping() { - let schema = create_test_schema(); - let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap(); - - let batch_position = memtable - .insert(create_test_batch(&schema, 10)) - .await - .unwrap(); - assert!(!memtable.all_flushed_to_wal()); - - memtable.mark_wal_flushed(&[batch_position], 5, &[0]); - - assert!(memtable.all_flushed_to_wal()); - assert_eq!( - memtable.wal_batch_mapping().get(&batch_position), - Some(&(5, 0)) - ); - assert_eq!(memtable.last_flushed_wal_entry_position(), 5); - } - - #[tokio::test] - async fn test_memtable_unflushed_batches() { + async fn test_all_flushed_to_wal_tracks_the_durability_watermark() { let schema = create_test_schema(); let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap(); @@ -965,12 +919,16 @@ mod tests { .insert(create_test_batch(&schema, 5)) .await .unwrap(); + assert!(!memtable.all_flushed_to_wal(0), "nothing is durable yet"); - assert_eq!(memtable.unflushed_batch_positions(), vec![batch1, batch2]); - - memtable.mark_wal_flushed(&[batch1], 1, &[0]); + let durable = batch1 + 1; + assert!( + !memtable.all_flushed_to_wal(durable), + "batch2 is still waiting on its WAL append" + ); - assert_eq!(memtable.unflushed_batch_positions(), vec![batch2]); + let durable = batch2 + 1; + assert!(memtable.all_flushed_to_wal(durable)); } #[tokio::test] @@ -992,23 +950,21 @@ mod tests { .await .unwrap(); - // max_visible_batch_position=1 means positions 0 and 1 are visible - let visible = memtable.get_visible_batches(1).await; + // A count of N exposes the prefix [0, N). + let visible = memtable.get_visible_batches(2).await; assert_eq!(visible.len(), 2); let total_rows: usize = visible.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 15); // 10 + 5 - // max_visible_batch_position=2 means all batches are visible - let visible = memtable.get_visible_batches(2).await; + let visible = memtable.get_visible_batches(3).await; assert_eq!(visible.len(), 3); - // max_visible_batch_position=0 means only position 0 is visible - let visible = memtable.get_visible_batches(0).await; - assert_eq!(visible.len(), 1); + // A count of 0 exposes nothing — not "batch 0". + assert!(memtable.get_visible_batches(0).await.is_empty()); } #[tokio::test] - async fn test_memtable_get_max_visible_batch_positions() { + async fn test_memtable_get_visible_batch_positions() { let schema = create_test_schema(); let mut memtable = MemTable::new(schema.clone(), 1, vec![]).unwrap(); @@ -1026,17 +982,15 @@ mod tests { .await .unwrap(); - // max_visible_batch_position=1 means positions 0 and 1 visible - let visible_ids = memtable.get_max_visible_batch_positions(1).await; + // A count of N exposes the prefix [0, N). + let visible_ids = memtable.get_visible_batch_positions(2).await; assert_eq!(visible_ids, vec![0, 1]); - // max_visible_batch_position=2 means all positions visible - let visible_ids = memtable.get_max_visible_batch_positions(2).await; + let visible_ids = memtable.get_visible_batch_positions(3).await; assert_eq!(visible_ids, vec![0, 1, 2]); - // max_visible_batch_position=0 means only position 0 visible - let visible_ids = memtable.get_max_visible_batch_positions(0).await; - assert_eq!(visible_ids, vec![0]); + // A count of 0 exposes nothing. + assert!(memtable.get_visible_batch_positions(0).await.is_empty()); } #[tokio::test] @@ -1057,14 +1011,14 @@ mod tests { .await .unwrap(); // position 2 - // batch_position 0 is visible when max_visible_batch_position >= 0 - assert!(memtable.is_batch_visible(0, 0).await); + // A count of 0 means nothing is visible, batch 0 included. + assert!(!memtable.is_batch_visible(0, 0).await); + + // Batch i is visible once the count exceeds i. assert!(memtable.is_batch_visible(0, 1).await); assert!(memtable.is_batch_visible(0, 2).await); - - // batch_position 2 is only visible when max_visible_batch_position >= 2 assert!(!memtable.is_batch_visible(2, 1).await); - assert!(memtable.is_batch_visible(2, 2).await); + assert!(!memtable.is_batch_visible(2, 2).await); assert!(memtable.is_batch_visible(2, 3).await); // Non-existent batch @@ -1085,12 +1039,21 @@ mod tests { .await .unwrap(); // position 1 - let batches = memtable.scan_batches_at_position(0).await.unwrap(); + let batches = memtable.scan_batches_at_position(1).await.unwrap(); assert_eq!(batches.len(), 1); assert_eq!(batches[0].num_rows(), 10); - let batches = memtable.scan_batches_at_position(1).await.unwrap(); + let batches = memtable.scan_batches_at_position(2).await.unwrap(); assert_eq!(batches.len(), 2); + + // Nothing indexed yet => nothing scannable. + assert!( + memtable + .scan_batches_at_position(0) + .await + .unwrap() + .is_empty() + ); } #[tokio::test] @@ -1101,7 +1064,7 @@ mod tests { assert_eq!(memtable.batch_capacity(), 3); assert_eq!(memtable.remaining_batch_capacity(), 3); - assert!(!memtable.is_batch_store_full()); + assert!(!memtable.batch_store().is_full()); // Fill up the store memtable @@ -1117,7 +1080,7 @@ mod tests { .await .unwrap(); - assert!(memtable.is_batch_store_full()); + assert!(memtable.batch_store().is_full()); assert_eq!(memtable.remaining_batch_capacity(), 0); // Next insert should fail diff --git a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs index 054d9b1630e..c8607f8fd06 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/batch_store.rs @@ -1,22 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Lock-free append-only batch storage for MemTable. +//! Append-only batch storage with lock-free readers for MemTable. //! -//! This module provides a high-performance, lock-free storage structure for -//! RecordBatches in the MemTable. It is designed for a single-writer, -//! multiple-reader scenario where: +//! This module provides high-performance storage for RecordBatches in the +//! MemTable. Reads remain lock-free, while appends are serialized so the +//! single-writer invariant is also upheld for safe callers. //! -//! - A single writer task (WriteBatchHandler) appends batches +//! - A writer task (WriteBatchHandler) appends batches //! - Multiple reader tasks concurrently read batches -//! - No locks are needed for either reads or writes +//! - Accidental concurrent appends are serialized //! //! # Safety Model //! //! The lock-free design relies on these invariants: //! -//! 1. **Single Writer**: Only one thread calls `append()` at a time. -//! Enforced by the WriteBatchHandler architecture. +//! 1. **Serialized Writers**: Only one thread mutates slots at a time. +//! Enforced by an internal writer guard in addition to the +//! WriteBatchHandler architecture. //! //! 2. **Append-Only**: Once written, slots are never modified or removed //! until the entire store is dropped. @@ -40,10 +41,15 @@ //! ``` use std::cell::UnsafeCell; +use std::collections::HashSet; use std::mem::MaybeUninit; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use arrow::array::ArrayData; use arrow_array::RecordBatch; +use arrow_buffer::Buffer; +use arrow_schema::DataType; /// A batch stored in the lock-free store. #[derive(Clone)] @@ -75,14 +81,56 @@ impl StoredBatch { } /// Estimate the memory size of a RecordBatch. + /// + /// Sums each column's slice-aware buffer size (see + /// [`Self::estimate_array_size`]) plus the struct overhead, so a column that + /// is a zero-copy slice of a larger parent contributes only its own window + /// rather than the whole shared buffer. fn estimate_batch_size(batch: &RecordBatch) -> usize { batch .columns() .iter() - .map(|col| col.get_array_memory_size()) + .map(|col| Self::estimate_array_size(&col.to_data())) .sum::() + std::mem::size_of::() } + + /// Slice-aware buffer size of a single array. + /// + /// [`ArrayData::get_slice_memory_size`] reports each buffer's own window + /// (not the whole shared buffer), but omits the variadic data buffers of + /// `Utf8View`/`BinaryView` (values > 12 bytes) while still returning `Ok`, so + /// [`Self::view_data_buffers_size`] adds them. Those buffers are shared across + /// zero-copy slices and are counted at full capacity for each slice — an + /// over-count in the safe direction. + fn estimate_array_size(data: &ArrayData) -> usize { + match data.get_slice_memory_size() { + Ok(size) => size + Self::view_data_buffers_size(data), + // Fall back to the full-buffer sum for layouts the slice-aware call + // cannot handle. + Err(_) => data.get_array_memory_size(), + } + } + + /// Capacity of the variadic `Utf8View`/`BinaryView` data buffers that + /// [`ArrayData::get_slice_memory_size`] omits, summed recursively over children. + fn view_data_buffers_size(data: &ArrayData) -> usize { + let mut size = 0; + if matches!(data.data_type(), DataType::Utf8View | DataType::BinaryView) { + // buffers()[0] is the 16-byte view array that get_slice_memory_size + // already counts; [1..] are the data buffers it skips. + size += data + .buffers() + .iter() + .skip(1) + .map(|b| b.capacity()) + .sum::(); + } + for child in data.child_data() { + size += Self::view_data_buffers_size(child); + } + size + } } /// Snapshot of the active batches that have not yet been flushed to WAL. @@ -112,10 +160,9 @@ impl std::fmt::Display for StoreFull { impl std::error::Error for StoreFull {} -/// Lock-free append-only storage for memtable batches. +/// Append-only storage with lock-free readers for memtable batches. /// -/// This structure provides O(1) lock-free appends and reads for a -/// single-writer, multiple-reader scenario. +/// This structure provides O(1) serialized appends and lock-free reads. /// /// # Example /// @@ -142,6 +189,9 @@ pub struct BatchStore { /// Invariant: all slots [0, committed_len) contain valid data. committed_len: AtomicUsize, + /// Serializes slot initialization for safe callers. + writer_active: AtomicBool, + /// Total capacity (fixed at creation). capacity: usize, @@ -151,20 +201,46 @@ pub struct BatchStore { /// Estimated size in bytes (for flush threshold). estimated_bytes: AtomicUsize, - /// WAL flush watermark: the last batch ID that has been flushed to WAL (inclusive). - /// Uses usize::MAX as sentinel for "nothing flushed yet". - /// This is per-memtable tracking, not global. - max_flushed_batch_position: AtomicUsize, + /// Sum of [`Buffer::capacity`] over the distinct allocations the stored + /// batches keep alive. See [`Self::retained_bytes`]. + retained_bytes: AtomicUsize, + + /// Addresses of the allocations already counted into `retained_bytes`, so + /// batches sharing a parent buffer charge it once. + /// + /// Only `append`/`append_batches` touch it, already serialized by the + /// writer guard; the `Mutex` is what makes that sound for a `Sync` type, + /// not a second layer of exclusion. + retained_buffers: Mutex>, + + /// Writer-global coordinate of this store's batch 0. + /// + /// A *coordinate*, not a cursor: stamped once at construction and never + /// moved. `global_position = global_offset + local_position`. Batch + /// positions restart at 0 in every memtable, so this is the only thing that + /// lets a writer-global cursor (the WAL durability count) be mapped onto a + /// particular store. + global_offset: usize, } // SAFETY: Safe to share across threads because: -// - Single writer guarantee (architectural invariant) +// - writer_active serializes all slot initialization // - Readers only access committed slots (index < committed_len) // - Atomic operations provide proper synchronization // - Slots are never modified after being written unsafe impl Sync for BatchStore {} unsafe impl Send for BatchStore {} +struct BatchStoreWriterGuard<'a> { + writer_active: &'a AtomicBool, +} + +impl Drop for BatchStoreWriterGuard<'_> { + fn drop(&mut self) { + self.writer_active.store(false, Ordering::Release); + } +} + impl BatchStore { /// Create a new store with the given capacity. /// @@ -177,6 +253,13 @@ impl BatchStore { /// /// Panics if capacity is 0. pub fn with_capacity(capacity: usize) -> Self { + Self::with_capacity_at(capacity, 0) + } + + /// Create a store whose batch 0 sits at `global_offset` in the writer's + /// batch sequence. Used by `freeze_memtable` for every memtable after the + /// first; the first starts at 0. + pub fn with_capacity_at(capacity: usize, global_offset: usize) -> Self { assert!(capacity > 0, "capacity must be > 0"); // Allocate uninitialized storage @@ -188,10 +271,13 @@ impl BatchStore { Self { slots: slots.into_boxed_slice(), committed_len: AtomicUsize::new(0), + writer_active: AtomicBool::new(false), capacity, total_rows: AtomicUsize::new(0), estimated_bytes: AtomicUsize::new(0), - max_flushed_batch_position: AtomicUsize::new(usize::MAX), // Nothing flushed yet + retained_bytes: AtomicUsize::new(0), + retained_buffers: Mutex::new(HashSet::new()), + global_offset, } } @@ -227,22 +313,19 @@ impl BatchStore { } // ========================================================================= - // Writer API (Single Writer Only) + // Writer API // ========================================================================= /// Append a batch to the store. /// - /// # Safety Requirements - /// - /// This method MUST only be called from the single writer task. - /// Concurrent calls from multiple threads cause undefined behavior. - /// /// # Returns /// /// - `Ok((batch_position, row_offset, estimated_size))` - The index, row offset, and size of the appended batch /// - `Err(StoreFull)` - The store is at capacity, needs flush pub fn append(&self, batch: RecordBatch) -> Result<(usize, u64, usize), StoreFull> { - // Load current length (Relaxed is fine - we're the only writer) + let _writer_guard = self.acquire_writer(); + + // The writer guard makes Relaxed sufficient for writer-owned state. let idx = self.committed_len.load(Ordering::Relaxed); if idx >= self.capacity { @@ -252,13 +335,14 @@ impl BatchStore { // Row offset is the total rows BEFORE this batch let row_offset = self.total_rows.load(Ordering::Relaxed) as u64; + let retained = self.charge_retained(&batch); let stored = StoredBatch::new(batch, row_offset, idx); let num_rows = stored.num_rows; let estimated_size = stored.estimated_size; // SAFETY: // 1. idx < capacity, so slot exists - // 2. Single writer guarantee - no concurrent writes to this slot + // 2. The writer guard prevents concurrent writes to this slot // 3. Slot at idx is uninitialized (never written before, append-only) unsafe { let slot_ptr = self.slots[idx].get(); @@ -269,6 +353,7 @@ impl BatchStore { self.total_rows.fetch_add(num_rows, Ordering::Relaxed); self.estimated_bytes .fetch_add(estimated_size, Ordering::Relaxed); + self.retained_bytes.fetch_add(retained, Ordering::Relaxed); // CRITICAL: Publish with Release ordering. // This ensures all writes above are visible to readers @@ -283,11 +368,6 @@ impl BatchStore { /// All batches are written before publishing, so readers see either /// none of the batches or all of them (atomic visibility). /// - /// # Safety Requirements - /// - /// This method MUST only be called from the single writer task. - /// Concurrent calls from multiple threads cause undefined behavior. - /// /// # Returns /// /// - `Ok(Vec<(batch_position, row_offset, estimated_size)>)` - Info for each appended batch @@ -300,7 +380,9 @@ impl BatchStore { return Ok(vec![]); } - // Load current length (Relaxed is fine - we're the only writer) + let _writer_guard = self.acquire_writer(); + + // The writer guard makes Relaxed sufficient for writer-owned state. let start_idx = self.committed_len.load(Ordering::Relaxed); let count = batches.len(); @@ -312,18 +394,20 @@ impl BatchStore { let mut results = Vec::with_capacity(count); let mut total_rows_added = 0usize; let mut total_bytes_added = 0usize; + let mut total_retained_added = 0usize; let mut row_offset = self.total_rows.load(Ordering::Relaxed) as u64; // Write all batches to slots (not yet visible to readers) for (i, batch) in batches.into_iter().enumerate() { let idx = start_idx + i; + total_retained_added += self.charge_retained(&batch); let stored = StoredBatch::new(batch, row_offset, idx); let num_rows = stored.num_rows; let estimated_size = stored.estimated_size; // SAFETY: // 1. idx < capacity (checked above) - // 2. Single writer guarantee - no concurrent writes to this slot + // 2. The writer guard prevents concurrent writes to this slot // 3. Slot at idx is uninitialized (never written before, append-only) unsafe { let slot_ptr = self.slots[idx].get(); @@ -341,6 +425,8 @@ impl BatchStore { .fetch_add(total_rows_added, Ordering::Relaxed); self.estimated_bytes .fetch_add(total_bytes_added, Ordering::Relaxed); + self.retained_bytes + .fetch_add(total_retained_added, Ordering::Relaxed); // CRITICAL: Publish ALL batches at once with Release ordering. // This ensures all writes above are visible to readers @@ -351,6 +437,70 @@ impl BatchStore { Ok(results) } + /// Charge the allocations `batch` retains that this store has not counted + /// yet, and return how much that added. + /// + /// The unit is the allocation, not the window a batch reads through it: a + /// one-row zero-copy slice pins its whole parent buffer, so measuring the + /// window would let an unbounded footprint in under a small number. + /// + /// Charged once per *distinct buffer view*, not strictly once per + /// allocation. `ArrayData::slice` advances the offset and leaves the buffer + /// pointer alone, so ordinary slices of one parent do dedupe; a buffer that + /// came back re-sliced from a kernel (`Buffer::slice_with_length`, concat or + /// take output) presents a different `data_ptr` for the same allocation and + /// is charged again in full. That over-counts, which is the safe direction + /// for a ceiling. + /// + /// `retained_buffers` is never pruned: it grows with every batch this store + /// accepts, bounded only by the store being dropped at flush. The walk plus + /// `to_data`, the mutex and a hash insert run per column per append — fine + /// at current batch rates, and the thing to look at first if that changes. + /// + /// Call under the writer guard, before the batch is moved into its slot. + fn charge_retained(&self, batch: &RecordBatch) -> usize { + let mut seen = self.retained_buffers.lock().unwrap(); + let mut added = 0; + for column in batch.columns() { + Self::walk_buffers(&column.to_data(), &mut |buffer| { + if seen.insert(buffer.data_ptr().as_ptr() as usize) { + // `capacity` reads 0 for a foreign allocation whose size + // arrow was not told; the window is the only figure left. + added += buffer.capacity().max(buffer.len()); + } + }); + } + added + } + + /// Every buffer reachable from `data`, validity and nested children + /// included — `ArrayData::buffers` alone omits both, and the variadic + /// `Utf8View`/`BinaryView` data buffers hang off it as ordinary entries. + fn walk_buffers(data: &ArrayData, visit: &mut impl FnMut(&Buffer)) { + for buffer in data.buffers() { + visit(buffer); + } + if let Some(nulls) = data.nulls() { + visit(nulls.buffer()); + } + for child in data.child_data() { + Self::walk_buffers(child, visit); + } + } + + fn acquire_writer(&self) -> BatchStoreWriterGuard<'_> { + while self + .writer_active + .compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_err() + { + std::hint::spin_loop(); + } + BatchStoreWriterGuard { + writer_active: &self.writer_active, + } + } + // ========================================================================= // Reader API (Multiple Concurrent Readers) // ========================================================================= @@ -385,82 +535,79 @@ impl BatchStore { /// Get estimated size in bytes. #[inline] - pub fn estimated_bytes(&self) -> usize { + pub fn row_bytes(&self) -> usize { self.estimated_bytes.load(Ordering::Relaxed) } + /// Heap this store actually keeps alive: every distinct allocation its + /// batches reference, counted once at its full capacity. + /// + /// Differs from [`Self::row_bytes`] wherever a batch is a zero-copy slice. + /// `row_bytes` measures the window, because it drives the flush threshold + /// and a flush writes only the rows in that window. This measures what the + /// allocator cannot hand back until the memtable is dropped — which is the + /// question a memory ceiling is asking. Sixteen one-row slices of sixteen + /// large parents are megabytes here and a few hundred bytes there. + /// + /// Deduplicated within a store, not across them: two memtables slicing one + /// parent each charge it in full, which errs toward refusing writes. + #[inline] + pub fn retained_bytes(&self) -> usize { + self.retained_bytes.load(Ordering::Relaxed) + } + // ========================================================================= // WAL Flush Tracking API // ========================================================================= - /// Get the WAL flush watermark (the last batch ID that was flushed, inclusive). - /// Returns None if nothing has been flushed yet. + /// Writer-global coordinate one past this store's last committed batch. #[inline] - pub fn max_flushed_batch_position(&self) -> Option { - let watermark = self.max_flushed_batch_position.load(Ordering::Acquire); - if watermark == usize::MAX { - None - } else { - Some(watermark) - } + pub fn global_end(&self) -> usize { + self.global_offset + self.committed_len.load(Ordering::Acquire) } - /// Update the WAL flush watermark after successful WAL flush. - /// - /// # Arguments - /// - /// * `batch_position` - The last batch ID that was flushed (inclusive) + /// This store's writer-global coordinate for batch 0. #[inline] - pub fn set_max_flushed_batch_position(&self, batch_position: usize) { - debug_assert!( - batch_position != usize::MAX, - "batch_position cannot be usize::MAX (reserved as sentinel)" - ); - self.max_flushed_batch_position - .store(batch_position, Ordering::Release); + pub fn global_offset(&self) -> usize { + self.global_offset } - /// Get the number of batches pending WAL flush. + /// The local exclusive end of this store covered by a writer-global cursor. + /// + /// Saturating in both directions, and both directions are reachable in + /// normal operation: a cursor *below* this store's offset means "nothing + /// here yet" (the store was rotated in after the cursor last advanced — + /// the ordinary state of a fresh memtable), and a cursor beyond its end + /// clamps to what is committed. + /// + /// This is the **only** place the global-to-local subtraction is written. + /// Open-coding it underflows on every memtable rotation, which in release + /// wraps to a huge end and makes the whole new memtable instantly visible. #[inline] - pub fn pending_wal_flush_count(&self) -> usize { - let committed = self.committed_len.load(Ordering::Acquire); - let watermark = self.max_flushed_batch_position.load(Ordering::Acquire); - if watermark == usize::MAX { - // Nothing flushed yet, all committed batches are pending - committed - } else { - // Batches [0, watermark] are flushed, so pending = committed - (watermark + 1) - committed.saturating_sub(watermark + 1) - } + pub fn local_end(&self, global_cursor: usize) -> usize { + global_cursor + .saturating_sub(self.global_offset) + .min(self.committed_len.load(Ordering::Acquire)) } - /// Check if all committed batches have been WAL-flushed. + /// Batches in this store still waiting on their WAL append. #[inline] - pub fn is_wal_flush_complete(&self) -> bool { - self.pending_wal_flush_count() == 0 + pub fn pending_wal_flush_count(&self, durable: usize) -> usize { + self.committed_len.load(Ordering::Acquire) - self.local_end(durable) } - /// Get the range of batch IDs pending WAL flush: [start, end). - /// Returns None if nothing pending. + /// Local range `[start, end)` of batches still waiting on their WAL append, + /// or `None` when the store is fully durable. #[inline] - pub fn pending_wal_flush_range(&self) -> Option<(usize, usize)> { - let committed = self.committed_len.load(Ordering::Acquire); - let watermark = self.max_flushed_batch_position.load(Ordering::Acquire); - let start = if watermark == usize::MAX { - 0 - } else { - watermark + 1 - }; - if committed > start { - Some((start, committed)) - } else { - None - } + pub fn pending_wal_flush_range(&self, durable: usize) -> Option<(usize, usize)> { + let start = self.local_end(durable); + let end = self.committed_len.load(Ordering::Acquire); + (end > start).then_some((start, end)) } /// Get a point-in-time summary of batches pending WAL flush. - pub fn pending_wal_flush_stats(&self) -> PendingWalFlushStats { - let Some((start, end)) = self.pending_wal_flush_range() else { + pub fn pending_wal_flush_stats(&self, durable: usize) -> PendingWalFlushStats { + let Some((start, end)) = self.pending_wal_flush_range(durable) else { return PendingWalFlushStats::default(); }; @@ -599,66 +746,53 @@ impl BatchStore { // Visibility API // ========================================================================= - /// Get batches visible up to a specific batch position (inclusive). + /// Batches in the visible prefix `[0, visible_count)`. /// - /// A batch at position `i` is visible if `i <= max_visible_batch_position`. - pub fn visible_batches(&self, max_visible_batch_position: usize) -> Vec<&StoredBatch> { - let len = self.committed_len.load(Ordering::Acquire); - let end = (max_visible_batch_position + 1).min(len); + /// `visible_count` is an **exclusive count**, not an inclusive position: 0 + /// means nothing is visible. As an inclusive position, 0 meant *both* + /// "nothing visible" and "batch 0 is visible", so a batch that was committed + /// to the store but not yet indexed or WAL-durable was readable for a full + /// PUT round-trip. The count makes that off-by-one inexpressible. + pub fn visible_batches(&self, visible_count: usize) -> Vec<&StoredBatch> { + let end = visible_count.min(self.committed_len.load(Ordering::Acquire)); (0..end).filter_map(|i| self.get(i)).collect() } - /// Get batch positions visible up to a specific batch position (inclusive). - pub fn max_visible_batch_positions(&self, max_visible_batch_position: usize) -> Vec { - let len = self.committed_len.load(Ordering::Acquire); - let end = (max_visible_batch_position + 1).min(len); + /// Positions of the batches in the visible prefix. + pub fn visible_batch_positions(&self, visible_count: usize) -> Vec { + let end = visible_count.min(self.committed_len.load(Ordering::Acquire)); (0..end).collect() } - /// The inclusive maximum visible *row* position at `max_visible_batch_position`, - /// or `None` when no rows are visible. The visible batches are the committed - /// prefix `[0, last_visible_idx]`; each batch carries its cumulative - /// `row_offset`, so this is the end of the last visible batch minus one. - /// Used to bound MVCC seeks against the maintained PK-position index. - pub fn max_visible_row(&self, max_visible_batch_position: usize) -> Option { - let len = self.committed_len.load(Ordering::Acquire); - if len == 0 { - return None; - } - let last_visible_idx = max_visible_batch_position.min(len - 1); - let last = self.get(last_visible_idx)?; + /// The inclusive maximum visible *row* position, or `None` when no rows are + /// visible. Each batch carries its cumulative `row_offset`, so this is the + /// end of the last visible batch minus one. Bounds MVCC seeks against the + /// maintained PK-position index. + pub fn max_visible_row(&self, visible_count: usize) -> Option { + let end = visible_count.min(self.committed_len.load(Ordering::Acquire)); + let last = self.get(end.checked_sub(1)?)?; let visible_end = last.row_offset + last.num_rows as u64; // exclusive visible_end.checked_sub(1) } - /// Check if a specific batch is visible at a given visibility position. + /// Whether a batch falls inside the visible prefix. #[inline] - pub fn is_batch_visible( - &self, - batch_position: usize, - max_visible_batch_position: usize, - ) -> bool { + pub fn is_batch_visible(&self, batch_position: usize, visible_count: usize) -> bool { let len = self.committed_len.load(Ordering::Acquire); - batch_position < len && batch_position <= max_visible_batch_position + batch_position < len && batch_position < visible_count } - /// Get visible RecordBatches (clones the data). - pub fn visible_record_batches(&self, max_visible_batch_position: usize) -> Vec { - self.visible_batches(max_visible_batch_position) + /// Visible RecordBatches (clones the data). + pub fn visible_record_batches(&self, visible_count: usize) -> Vec { + self.visible_batches(visible_count) .into_iter() .map(|b| b.data.clone()) .collect() } - /// Get visible RecordBatches with their row offsets. - /// - /// Returns tuples of (batch, row_offset) for each visible batch. - /// The row_offset is the starting row position for that batch. - pub fn visible_batches_with_offsets( - &self, - max_visible_batch_position: usize, - ) -> Vec<(RecordBatch, u64)> { - self.visible_batches(max_visible_batch_position) + /// Visible RecordBatches paired with the row position each one starts at. + pub fn visible_batches_with_offsets(&self, visible_count: usize) -> Vec<(RecordBatch, u64)> { + self.visible_batches(visible_count) .into_iter() .map(|b| (b.data.clone(), b.row_offset)) .collect() @@ -766,7 +900,7 @@ mod tests { use super::*; use arrow_array::Int32Array; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; - use std::sync::Arc; + use std::sync::{Arc, Barrier}; fn create_test_schema() -> Arc { Arc::new(ArrowSchema::new(vec![ @@ -891,17 +1025,52 @@ mod tests { store.append(create_test_batch(10)).unwrap(); // position 3 store.append(create_test_batch(10)).unwrap(); // position 4 - // max_visible_batch_position=2 means positions 0, 1, 2 are visible - let visible = store.max_visible_batch_positions(2); - assert_eq!(visible, vec![0, 1, 2]); + // A count of N exposes the prefix [0, N). + assert_eq!(store.visible_batch_positions(3), vec![0, 1, 2]); + assert_eq!(store.visible_batch_positions(5), vec![0, 1, 2, 3, 4]); + + // A count of 0 exposes nothing. Under the old inclusive cursor this + // case was indistinguishable from "batch 0 is visible", so every + // memtable leaked its first batch before it was indexed or durable. + assert!(store.visible_batch_positions(0).is_empty()); + + // Beyond the committed range, clamp. + assert_eq!(store.visible_batch_positions(99), vec![0, 1, 2, 3, 4]); + } + + /// The zero of the visibility cursor must be unambiguous. + /// + /// `BatchStore::append` publishes `committed_len` on the put path, under the + /// state lock, *before* the WAL flush that indexes the batch is even + /// triggered — and that flush is a ~100ms S3 PUT on another task. So batch 0 + /// sits committed and readable for a full round-trip before it is indexed or + /// durable. As an inclusive position, a cursor of 0 meant both "nothing is + /// visible" and "batch 0 is visible", so every read arm backed by the batch + /// store served that batch while the index-backed arms did not — the tiers + /// actively disagreed. An exclusive count makes the state inexpressible. + #[test] + fn test_zero_cursor_hides_the_committed_but_unindexed_prefix() { + let store = BatchStore::with_capacity(4); + store.append(create_test_batch(10)).unwrap(); + store.append(create_test_batch(10)).unwrap(); - // max_visible_batch_position=4 means all visible - let visible = store.max_visible_batch_positions(4); - assert_eq!(visible, vec![0, 1, 2, 3, 4]); + // Committed, but nothing indexed yet: every visibility query must agree + // that there is nothing to read. + assert!(store.visible_batches(0).is_empty()); + assert!(store.visible_batch_positions(0).is_empty()); + assert!(store.visible_record_batches(0).is_empty()); + assert!(store.visible_batches_with_offsets(0).is_empty()); + assert!(!store.is_batch_visible(0, 0)); + assert_eq!(store.max_visible_row(0), None); + + // The batches are there — they are simply not yet published. + assert_eq!(store.len(), 2); - // max_visible_batch_position=0 means only position 0 visible - let visible = store.max_visible_batch_positions(0); - assert_eq!(visible, vec![0]); + // Indexing batch 0 publishes exactly batch 0. + assert_eq!(store.visible_batches(1).len(), 1); + assert!(store.is_batch_visible(0, 1)); + assert!(!store.is_batch_visible(1, 1)); + assert_eq!(store.max_visible_row(1), Some(9)); } #[test] @@ -912,14 +1081,17 @@ mod tests { store.append(create_test_batch(10)).unwrap(); // position 1 store.append(create_test_batch(10)).unwrap(); // position 2 - // Batch at position 0 is visible when max_visible_batch_position >= 0 - assert!(store.is_batch_visible(0, 0)); + // A count of 0 means *nothing* is visible — including batch 0. As an + // inclusive position this case was indistinguishable from "batch 0 is + // visible", so a batch that was committed to the store but not yet + // indexed or WAL-durable was readable for a full PUT round-trip. + assert!(!store.is_batch_visible(0, 0)); + + // Batch i is visible once the count exceeds i. assert!(store.is_batch_visible(0, 1)); assert!(store.is_batch_visible(0, 2)); - - // Batch at position 2 is only visible when max_visible_batch_position >= 2 assert!(!store.is_batch_visible(2, 1)); - assert!(store.is_batch_visible(2, 2)); + assert!(!store.is_batch_visible(2, 2)); assert!(store.is_batch_visible(2, 3)); // Batch 3 doesn't exist @@ -928,7 +1100,7 @@ mod tests { #[test] fn test_max_visible_row() { - // (1) Empty store: no rows are visible at any position. + // (1) Empty store: no rows are visible at any count. let store = BatchStore::with_capacity(10); assert_eq!(store.max_visible_row(0), None); assert_eq!(store.max_visible_row(100), None); @@ -938,23 +1110,24 @@ mod tests { store.append(create_test_batch(20)).unwrap(); // position 1 store.append(create_test_batch(30)).unwrap(); // position 2 - // (2) A position within range yields the inclusive end of that prefix. - assert_eq!(store.max_visible_row(0), Some(9)); // batch 0: 0..10 - assert_eq!(store.max_visible_row(1), Some(29)); // batch 1: 10..30 - assert_eq!(store.max_visible_row(2), Some(59)); // batch 2: 30..60 + // (2) A count of 0 means nothing is visible — not "batch 0 is visible". + assert_eq!(store.max_visible_row(0), None); + + // (3) A count of N yields the inclusive last row of the prefix [0, N). + assert_eq!(store.max_visible_row(1), Some(9)); // batch 0: 0..10 + assert_eq!(store.max_visible_row(2), Some(29)); // + batch 1: 10..30 + assert_eq!(store.max_visible_row(3), Some(59)); // + batch 2: 30..60 - // (3) A position beyond the committed range clamps to the last batch, - // i.e. the inclusive max over all rows. + // (4) A count beyond the committed range clamps to the last batch. assert_eq!(store.max_visible_row(100), Some(59)); - // (4) An empty leading batch contributes no rows: at its own position - // the inclusive end underflows to None, while a later non-empty batch - // is reported correctly. + // (5) An empty leading batch contributes no rows, so a prefix covering + // only it still yields None, while a later non-empty batch is reported. let store = BatchStore::with_capacity(10); store.append(create_test_batch(0)).unwrap(); // position 0: rows [0,0) store.append(create_test_batch(5)).unwrap(); // position 1: rows [0,5) - assert_eq!(store.max_visible_row(0), None); // empty prefix → no rows - assert_eq!(store.max_visible_row(1), Some(4)); // through batch 1 + assert_eq!(store.max_visible_row(1), None); // empty prefix → no rows + assert_eq!(store.max_visible_row(2), Some(4)); // through batch 1 } #[test] @@ -972,6 +1145,170 @@ mod tests { assert_eq!(cap, 16); // minimum } + #[test] + fn test_estimated_size_is_slice_aware() { + // A batch that is a zero-copy slice of a larger parent must contribute + // only its own window to the estimate, not the whole shared buffer. + // `get_array_memory_size` counts every buffer's full capacity regardless + // of offset/length, so N slices tiling one parent each report the + // parent's size and inflate the memtable estimate ~N×, tripping the + // flush threshold far below the configured size. + let chunk = 1_000; + let num_slices = 100; + let parent = create_test_batch(chunk * num_slices); + + // One window vs an equivalently-sized owned batch should track each + // other; the buggy per-slice estimate would be ~num_slices× larger. + let slice_est = StoredBatch::estimate_batch_size(&parent.slice(0, chunk)); + let owned_est = StoredBatch::estimate_batch_size(&create_test_batch(chunk)); + assert!( + slice_est <= owned_est * 2, + "slice estimate {slice_est} should track its own window (~{owned_est}), not the parent" + ); + + // End-to-end: tiling the parent with zero-copy slices must not multiply + // the store's running estimate. Track what the old full-buffer behavior + // would have summed to for contrast. + let store = BatchStore::with_capacity(num_slices); + let mut over_counting_sum = 0usize; + for k in 0..num_slices { + let s = parent.slice(k * chunk, chunk); + over_counting_sum += s + .columns() + .iter() + .map(|col| col.get_array_memory_size()) + .sum::() + + std::mem::size_of::(); + store.append(s).unwrap(); + } + + // Two non-nullable Int32 columns → exactly 4 bytes/row/col of payload. + let payload_bytes = num_slices * chunk * 2 * std::mem::size_of::(); + let estimated = store.row_bytes(); + assert!( + estimated >= payload_bytes, + "estimate {estimated} should cover the actual payload {payload_bytes}" + ); + // The old behavior over-counts by ~num_slices×; the fix must be far + // below it (generous 10× margin against struct/alignment overhead). + assert!( + estimated * 10 < over_counting_sum, + "estimate {estimated} should be far below the over-counting sum {over_counting_sum}" + ); + } + + /// `row_bytes` and `retained_bytes` answer different questions about the + /// same slices, and a memory ceiling needs the second one. + #[test] + fn test_retained_bytes_counts_pinned_parents_once() { + let chunk = 100_000; + + // Sixteen one-row slices, each off its own parent. The windows are + // trivial, but every parent stays alive in full for as long as the + // store does — this is the shape that lets a window-based ledger admit + // an unbounded footprint. + let distinct = BatchStore::with_capacity(16); + for _ in 0..16 { + distinct + .append(create_test_batch(chunk).slice(0, 1)) + .unwrap(); + } + // Two non-nullable Int32 columns. + let parent_payload = 16 * chunk * 2 * std::mem::size_of::(); + assert!( + distinct.retained_bytes() >= parent_payload, + "retained {} must cover the {parent_payload} bytes of pinned parents", + distinct.retained_bytes() + ); + assert!( + distinct.row_bytes() * 1_000 < distinct.retained_bytes(), + "row_bytes {} measures the windows and is nowhere near the retained {}", + distinct.row_bytes(), + distinct.retained_bytes() + ); + + // Sixteen slices of *one* parent pin one allocation, so the ledger must + // charge it once — the failure this shares with a naive full-capacity + // sum, which would report ~16×. + let parent = create_test_batch(chunk); + let shared = BatchStore::with_capacity(16); + for k in 0..16 { + shared + .append(parent.slice(k * (chunk / 16), chunk / 16)) + .unwrap(); + } + assert!( + shared.retained_bytes() * 8 < distinct.retained_bytes(), + "one shared parent ({}) must not be charged like sixteen ({})", + shared.retained_bytes(), + distinct.retained_bytes() + ); + } + + #[test] + fn test_estimated_size_counts_view_data_buffers() { + // Long Utf8View/BinaryView values live in variadic data buffers that + // `get_slice_memory_size` ignores (returning ~16 * rows). The estimate + // must include them, both for a top-level view column and for a view + // array nested in a container, which is only reached via child_data + // recursion. + use arrow_array::{Array, ArrayRef, StringViewArray, StructArray}; + + let num_rows = 1_000; + // Each value exceeds the 12-byte inline limit, so it spills to a data buffer. + let long_value = "x".repeat(64); + let payload_bytes = num_rows * long_value.len(); + // What the slice-aware call alone reports: just the 16-byte view entries. + let view_entries_only = num_rows * 16; + + let make_views = || { + StringViewArray::from( + (0..num_rows) + .map(|_| Some(long_value.as_str())) + .collect::>(), + ) + }; + let assert_covers = |batch: &RecordBatch| { + let estimated = StoredBatch::estimate_batch_size(batch); + assert!( + estimated >= payload_bytes, + "estimate {estimated} should cover the view data-buffer payload {payload_bytes}" + ); + assert!( + estimated > view_entries_only * 2, + "estimate {estimated} must exceed the ~{view_entries_only}-byte view-entry-only undercount" + ); + }; + + // Top-level view column. + let flat = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "s", + DataType::Utf8View, + false, + )])), + vec![Arc::new(make_views())], + ) + .unwrap(); + assert_covers(&flat); + + // View nested inside a struct — reachable only through child_data recursion. + let nested = StructArray::from(vec![( + Arc::new(Field::new("s", DataType::Utf8View, false)), + Arc::new(make_views()) as ArrayRef, + )]); + let nested = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "st", + nested.data_type().clone(), + false, + )])), + vec![Arc::new(nested)], + ) + .unwrap(); + assert_covers(&nested); + } + #[test] fn test_to_vec() { let store = BatchStore::with_capacity(10); @@ -1115,6 +1452,42 @@ mod tests { } } + #[test] + fn test_concurrent_writers_are_serialized() { + const NUM_WRITERS: usize = 8; + const BATCHES_PER_WRITER: usize = 50; + let expected_batches = NUM_WRITERS * BATCHES_PER_WRITER; + let store = Arc::new(BatchStore::with_capacity(expected_batches)); + let start = Arc::new(Barrier::new(NUM_WRITERS)); + + let writers: Vec<_> = (0..NUM_WRITERS) + .map(|_| { + let writer_store = store.clone(); + let writer_start = start.clone(); + std::thread::spawn(move || { + writer_start.wait(); + for _ in 0..BATCHES_PER_WRITER { + writer_store.append(create_test_batch(1)).unwrap(); + std::thread::yield_now(); + } + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap(); + } + + assert_eq!(store.len(), expected_batches); + assert_eq!(store.total_rows(), expected_batches); + let mut expected_row_offset = 0; + for (batch_position, batch) in store.iter().enumerate() { + assert_eq!(batch.batch_position, batch_position); + assert_eq!(batch.row_offset, expected_row_offset); + expected_row_offset += batch.num_rows as u64; + } + } + #[test] fn test_append_batches() { let store = BatchStore::with_capacity(10); diff --git a/rust/lance/src/dataset/mem_wal/memtable/flush.rs b/rust/lance/src/dataset/mem_wal/memtable/flush.rs index 410823c31db..c9c4c0e4dd4 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/flush.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/flush.rs @@ -12,9 +12,9 @@ use lance_core::cache::LanceCache; use lance_core::utils::deletion::DeletionVector; use lance_core::{Error, Result}; use lance_index::IndexType; -use lance_index::mem_wal::{FlushedGeneration, ShardManifest}; +use lance_index::mem_wal::{ShardManifest, SsTable}; use lance_index::scalar::{IndexStore, ScalarIndexParams}; -use lance_io::object_store::ObjectStore; +use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_table::format::IndexMetadata; use lance_table::io::commit::write_manifest_file_to_path; use lance_table::io::deletion::write_deletion_file; @@ -28,14 +28,17 @@ use uuid::Uuid; use super::super::index::MemIndexConfig; use super::super::memtable::MemTable; use crate::Dataset; +use crate::dataset::builder::DatasetBuilder; use crate::dataset::mem_wal::manifest::ShardManifestStore; -use crate::dataset::mem_wal::scanner::GenerationWarmer; +use crate::dataset::mem_wal::scanner::SsTableWarmer; use crate::dataset::mem_wal::scanner::exec::{compute_pk_hash, validate_pk_types}; -use crate::dataset::mem_wal::util::{flushed_memtable_path, generate_random_hash}; +use crate::dataset::mem_wal::util::{derived_store_params, generate_random_hash, sstable_path}; +use crate::index::vector::details::vector_index_details_default; +use crate::session::Session; #[derive(Debug, Clone)] pub struct FlushResult { - pub generation: FlushedGeneration, + pub sstable: SsTable, pub rows_flushed: usize, pub covered_wal_entry_position: u64, } @@ -71,7 +74,14 @@ pub struct MemTableFlusher { manifest_store: Arc, /// When present, each new generation is warmed before it is committed, so /// the first query sees zero cold reads. `None` => no warming. - warmer: Option>, + warmer: Option>, + /// Store params the base dataset was opened with, reused for the flusher's + /// own opens + writes. Used verbatim only for the base's own URI; generation + /// URIs go through [`derived_store_params`]. `None` opens by URI alone. + store_params: Option, + /// Session for those opens, sharing the base's store registry. `None` opens + /// with a fresh session. + session: Option>, } impl MemTableFlusher { @@ -89,15 +99,62 @@ impl MemTableFlusher { shard_id, manifest_store, warmer: None, + store_params: None, + session: None, } } /// Attach the warmer fired pre-commit for each new generation. - pub fn with_warmer(mut self, warmer: Option>) -> Self { + pub fn with_warmer(mut self, warmer: Option>) -> Self { self.warmer = warmer; self } + /// Set the store params + session used for derived-URI opens. Injected by + /// `mem_wal_writer` from the base `Dataset`. + pub fn with_storage_context( + mut self, + store_params: Option, + session: Option>, + ) -> Self { + self.store_params = store_params; + self.session = session; + self + } + + /// Open the base table, reusing the injected store params verbatim — they + /// were resolved for exactly this URI, so a path-bound `object_store` + /// binding still points where it should. + async fn open_base(&self) -> Result { + self.open_uri(&self.base_uri, self.store_params.clone()) + .await + } + + /// Open an SSTable under `_mem_wal/`. The params must be adapted + /// first: a path-bound store binding would redirect the open at the base + /// table (see [`derived_store_params`]). + async fn open_generation(&self, uri: &str) -> Result { + self.open_uri(uri, self.store_params.as_ref().map(derived_store_params)) + .await + } + + /// Open `uri` with the injected session, or by URI alone when nothing was + /// injected. + async fn open_uri( + &self, + uri: &str, + store_params: Option, + ) -> Result { + let mut builder = DatasetBuilder::from_uri(uri); + if let Some(params) = store_params { + builder = builder.with_store_params(params); + } + if let Some(session) = &self.session { + builder = builder.with_session(session.clone()); + } + builder.load().await + } + /// Warm a just-written generation before it is committed. Best-effort: a /// failure is logged and the flush proceeds — warming is never a commit /// gate. No-op without a warmer. `uri` must be the resolved reader path @@ -130,20 +187,18 @@ impl MemTableFlusher { } } - /// Storage file version of the shard's base dataset. Flushed generations + /// Storage file version of the shard's base dataset. SSTables /// (data fragments and index files) are written at this same version so the - /// whole shard stays on one format (e.g. a 2.2 base => 2.2 flushed gens). + /// whole shard stays on one format (e.g. a 2.2 base => 2.2 SSTables). /// - /// Falls back to [`LanceFileVersion::default`] when no base dataset exists at + /// Falls back to the default selector's exact version when no base dataset exists at /// `base_uri` (e.g. flusher unit tests that run without a committed base). /// In production MemWAL is always initialized on a real dataset, so the base /// version is inherited; other open errors are propagated. - async fn base_storage_version(&self) -> Result { - match Dataset::open(&self.base_uri).await { - Ok(dataset) => dataset.manifest().data_storage_format.lance_file_version(), - Err(Error::DatasetNotFound { .. }) => { - Ok(lance_file::version::LanceFileVersion::default()) - } + async fn base_storage_version(&self) -> Result { + match self.open_base().await { + Ok(dataset) => Ok(dataset.manifest().data_storage_format.lance_file_format()), + Err(Error::DatasetNotFound { .. }) => Ok(lance_file::version::stable_file_version()), Err(e) => Err(e), } } @@ -161,6 +216,7 @@ impl MemTableFlusher { memtable: &MemTable, epoch: u64, covered_wal_entry_position: u64, + durable: usize, ) -> Result { self.manifest_store.check_fenced(epoch).await?; @@ -168,7 +224,7 @@ impl MemTableFlusher { return Err(Error::invalid_input("Cannot flush empty MemTable")); } - if !memtable.all_flushed_to_wal() { + if !memtable.all_flushed_to_wal(durable) { return Err(Error::invalid_input( "MemTable has unflushed fragments - WAL flush required first", )); @@ -177,8 +233,7 @@ impl MemTableFlusher { let random_hash = generate_random_hash(); let generation = memtable.generation(); let gen_folder_name = format!("{}_gen_{}", random_hash, generation); - let gen_path = - flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation); + let gen_path = sstable_path(&self.base_path, &self.shard_id, &random_hash, generation); info!( "Flushing MemTable generation {} to {} ({} rows, {} batches)", @@ -190,11 +245,11 @@ impl MemTableFlusher { let (rows_flushed, deleted) = self.write_data_file(&gen_path, memtable).await?; - // Persist the within-generation deletion vector so the flushed - // generation exposes newest-per-PK on every read path. + // Persist the within-generation deletion vector so the + // SSTable exposes newest-per-PK on every read path. if !deleted.is_empty() { let uri = self.path_to_uri(&gen_path); - let dataset = Dataset::open(&uri).await?; + let dataset = self.open_generation(&uri).await?; self.finalize_generation(&dataset, &deleted, None).await?; } @@ -222,12 +277,12 @@ impl MemTableFlusher { .await?; info!( - "Flushed generation {} for shard {} (manifest version {})", + "Flushed SSTable {} for shard {} (manifest version {})", generation, self.shard_id, new_manifest.version ); Ok(FlushResult { - generation: FlushedGeneration { + sstable: SsTable { generation, path: gen_folder_name, }, @@ -296,13 +351,19 @@ impl MemTableFlusher { let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), memtable.schema().clone()); - // Use very large max_rows_per_file to ensure 1 fragment per flushed memtable. - // Inherit the base dataset's storage version so the flushed generation + // Use very large max_rows_per_file to ensure 1 fragment per SSTable. + // Inherit the base dataset's storage version so the SSTable // matches it (a 2.2 base also fixes the v2.1 miniblock 32 KiB chunk cap // that the dense HNSW graph List columns overflow at scale). let write_params = WriteParams { max_rows_per_file: usize::MAX, - data_storage_version: Some(self.base_storage_version().await?), + data_storage_version: Some(self.base_storage_version().await?.to_selector()), + // Write the generation through the base's store params + session so it + // uses the same store the base was opened with. Adapted for the + // generation URI: a path-bound store binding would send this write at + // the base table's own path (see [`derived_store_params`]). + store_params: self.store_params.as_ref().map(derived_store_params), + session: self.session.clone(), ..Default::default() }; Dataset::write(reader, &uri, Some(write_params)).await?; @@ -334,7 +395,7 @@ impl MemTableFlusher { let dv = DeletionVector::from(deleted.clone()); let deletion_file = write_deletion_file( &dataset.base, - 0, // 1 fragment per flushed generation + 0, // 1 fragment per SSTable dataset.version().version, &dv, dataset.object_store.as_ref(), @@ -388,6 +449,7 @@ impl MemTableFlusher { epoch: u64, index_configs: &[MemIndexConfig], covered_wal_entry_position: u64, + durable: usize, ) -> Result { self.manifest_store.check_fenced(epoch).await?; @@ -395,7 +457,7 @@ impl MemTableFlusher { return Err(Error::invalid_input("Cannot flush empty MemTable")); } - if !memtable.all_flushed_to_wal() { + if !memtable.all_flushed_to_wal(durable) { return Err(Error::invalid_input( "MemTable has unflushed fragments - WAL flush required first", )); @@ -404,8 +466,7 @@ impl MemTableFlusher { let random_hash = generate_random_hash(); let generation = memtable.generation(); let gen_folder_name = format!("{}_gen_{}", random_hash, generation); - let gen_path = - flushed_memtable_path(&self.base_path, &self.shard_id, &random_hash, generation); + let gen_path = sstable_path(&self.base_path, &self.shard_id, &random_hash, generation); info!( "Flushing MemTable generation {} with indexes to {} ({} rows, {} batches)", @@ -420,7 +481,7 @@ impl MemTableFlusher { // Open the dataset once for all index building. Dataset::write already // created a v1 manifest with the fragment data. let uri = self.path_to_uri(&gen_path); - let mut dataset = Dataset::open(&uri).await?; + let mut dataset = self.open_generation(&uri).await?; // Collect all index metadata without committing individually. // We write a single manifest containing both data and all indexes. @@ -431,7 +492,7 @@ impl MemTableFlusher { .await?; if !btree_indexes.is_empty() { info!( - "Created {} BTree indexes on flushed generation {}", + "Created {} BTree indexes on SSTable {}", btree_indexes.len(), generation ); @@ -450,7 +511,7 @@ impl MemTableFlusher { .await? else { info!( - "Skipped empty HNSW index '{}' on flushed generation {} (no vectors)", + "Skipped empty HNSW index '{}' on SSTable {} (no vectors)", hnsw_config.name, generation ); continue; @@ -474,7 +535,7 @@ impl MemTableFlusher { all_indexes.push(index_meta); info!( - "Created HNSW index '{}' on flushed generation {}", + "Created HNSW index '{}' on SSTable {}", hnsw_config.name, generation ); } @@ -520,12 +581,12 @@ impl MemTableFlusher { .await?; info!( - "Flushed generation {} for shard {} (manifest version {})", + "Flushed SSTable {} for shard {} (manifest version {})", generation, self.shard_id, new_manifest.version ); Ok(FlushResult { - generation: FlushedGeneration { + sstable: SsTable { generation, path: gen_folder_name, }, @@ -534,7 +595,7 @@ impl MemTableFlusher { }) } - /// Create BTree indexes on the flushed dataset (uncommitted). + /// Create BTree indexes on the SSTable dataset (uncommitted). /// /// Returns index metadata without committing to the dataset manifest. /// The caller is responsible for writing a single manifest with all indexes. @@ -602,7 +663,7 @@ impl MemTableFlusher { /// keys index the typed value; composite keys index the order-preserving /// `Binary` encoded tuple (see [`super::super::index::encode_pk_tuple`]). /// Row positions line up 1:1 with the forward-written data file, so they are - /// the flushed row ids directly. No-op without a primary-key index. + /// the SSTable row ids directly. No-op without a primary-key index. async fn create_pk_index( &self, gen_path: &Path, @@ -703,25 +764,25 @@ impl MemTableFlusher { let index_details = prost_types::Any::from_msg(&details) .map_err(|e| Error::io(format!("Failed to serialize index details: {}", e)))?; - let schema = dataset.schema(); - let field_idx = schema.field(&fts_cfg.column).map(|f| f.id).ok_or_else(|| { - Error::invalid_input(format!( - "FTS index '{}' references column '{}' which is not in the dataset schema", - fts_cfg.name, fts_cfg.column - )) - })?; + let field_idx = fts_cfg.field_id; let fragment_ids: roaring::RoaringBitmap = dataset.fragment_bitmap.as_ref().clone(); let format_version = fts_cfg.params.resolved_format_version(); + let index_version = if fts_cfg.params.get_document_granularity().is_list_element() { + lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V3 + } else { + format_version.index_version() + }; let index_meta = IndexMetadata { uuid: index_uuid, name: fts_cfg.name.clone(), fields: vec![field_idx], + covering_fields: vec![], dataset_version: dataset.version().version, fragment_bitmap: Some(fragment_ids), index_details: Some(Arc::new(index_details)), - index_version: format_version.index_version() as i32, + index_version: index_version as i32, created_at: None, base_id: None, files: None, @@ -749,8 +810,9 @@ impl MemTableFlusher { use std::sync::Arc; use lance_index::scalar::inverted::{ - POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1, POSITIONS_LAYOUT_KEY, - POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_TAIL_CODEC_KEY, TokenSetFormat, + FTS_FORMAT_VERSION_KEY, POSITIONS_CODEC_KEY, POSITIONS_CODEC_PACKED_DELTA_V1, + POSITIONS_LAYOUT_KEY, POSITIONS_LAYOUT_SHARED_STREAM_V2, POSTING_BLOCK_SIZE_KEY, + POSTING_TAIL_CODEC_KEY, TokenSetFormat, }; // Create metadata with params and partitions in schema metadata (this is what InvertedIndex expects) @@ -766,6 +828,14 @@ impl MemTableFlusher { POSTING_TAIL_CODEC_KEY.to_string(), format_version.posting_tail_codec().as_str().to_string(), ), + ( + FTS_FORMAT_VERSION_KEY.to_string(), + format_version.index_version().to_string(), + ), + ( + POSTING_BLOCK_SIZE_KEY.to_string(), + config.params.posting_block_size().to_string(), + ), ] .into_iter() .collect::>(); @@ -807,7 +877,7 @@ impl MemTableFlusher { /// the existing Lance `IVF_HNSW_SQ` reader path. /// /// # Arguments - /// * `gen_path` - Path to the flushed generation folder + /// * `gen_path` - Path to the SSTable folder /// * `config` - HNSW index configuration /// * `mem_index` - In-memory HNSW index (snapshotted, not consumed) /// @@ -831,7 +901,8 @@ impl MemTableFlusher { use arrow_schema::Schema as ArrowSchema; use lance_arrow::FixedSizeListArrayExt; use lance_core::ROW_ID; - use lance_file::writer::{FileWriter, FileWriterOptions}; + use lance_file::versions as file_versions; + use lance_file::writer::FileWriterOptions; use lance_index::pb; use lance_index::vector::DISTANCE_TYPE_KEY; use lance_index::vector::SQ_CODE_COLUMN; @@ -850,7 +921,8 @@ impl MemTableFlusher { // Write the index files at the base dataset's storage version (matches // the flushed data fragments; 2.2 avoids the v2.1 miniblock chunk cap). - let storage_version = self.base_storage_version().await?; + let storage_version = + crate::dataset::versions::index_file_version(self.base_storage_version().await?); let index_uuid = uuid::Uuid::new_v4(); let index_dir = gen_path @@ -922,13 +994,11 @@ impl MemTableFlusher { storage_ivf.add_partition(storage_batch.num_rows() as u32); let storage_path = index_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); - let mut storage_writer = FileWriter::try_new( + let mut storage_writer = file_versions::create_writer( + storage_version, self.object_store.create(&storage_path).await?, (&storage_schema).try_into()?, - FileWriterOptions { - format_version: Some(storage_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; storage_writer.write_batch(&storage_batch).await?; @@ -997,13 +1067,11 @@ impl MemTableFlusher { ArrowSchema::new(fields) }; let index_path = index_dir.clone().join(INDEX_FILE_NAME); - let mut index_writer = FileWriter::try_new( + let mut index_writer = file_versions::create_writer( + storage_version, self.object_store.create(&index_path).await?, (&index_schema).try_into()?, - FileWriterOptions { - format_version: Some(storage_version), - ..Default::default() - }, + FileWriterOptions::default(), )?; index_writer.write_batch(&hnsw_batch).await?; @@ -1034,14 +1102,14 @@ impl MemTableFlusher { ); index_writer.finish().await?; - let index_details = Some(Arc::new(prost_types::Any { - type_url: "type.googleapis.com/lance.index.VectorIndexDetails".to_string(), - value: vec![], - })); + // Packed the same way index creation does; hand-building the `Any` here + // produced a `type.googleapis.com/` url no other writer in lance emits. + let index_details = Some(Arc::new(vector_index_details_default())); let index_meta = IndexMetadata { uuid: index_uuid, name: config.name.clone(), fields: vec![0], // updated by caller + covering_fields: vec![], dataset_version: 0, fragment_bitmap: None, index_details, @@ -1054,7 +1122,7 @@ impl MemTableFlusher { Ok(Some(index_meta)) } - /// Update the shard manifest with the new flushed generation. + /// Update the shard manifest with the new SSTable. async fn update_manifest( &self, epoch: u64, @@ -1066,20 +1134,20 @@ impl MemTableFlusher { self.manifest_store .commit_update(epoch, |current| { - let mut flushed_generations = current.flushed_generations.clone(); - flushed_generations.push(FlushedGeneration { + let mut sstables = current.sstables.clone(); + sstables.push(SsTable { generation, path: gen_path.clone(), }); ShardManifest { - version: current.version + 1, + version: current.next_version(), replay_after_wal_entry_position: covered_wal_entry_position, wal_entry_position_last_seen: current .wal_entry_position_last_seen .max(covered_wal_entry_position), current_generation: generation + 1, - flushed_generations, + sstables, ..current.clone() } }) @@ -1119,6 +1187,7 @@ mod tests { use super::*; use arrow_array::{Int32Array, RecordBatch, StringArray}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; + use lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V2; use std::sync::Arc; use tempfile::TempDir; @@ -1185,11 +1254,12 @@ mod tests { .await .unwrap(); - // Not flushed to WAL yet - assert!(!memtable.all_flushed_to_wal()); + // Nothing is durable yet, so the L0 flush must refuse. + let durable = 0; + assert!(!memtable.all_flushed_to_wal(durable)); let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store); - let result = flusher.flush(&memtable, epoch, 0).await; + let result = flusher.flush(&memtable, epoch, 0, 0).await; assert!(result.is_err()); assert!( @@ -1218,7 +1288,7 @@ mod tests { let memtable = MemTable::new(schema, 1, vec![]).unwrap(); let flusher = MemTableFlusher::new(store, base_path, base_uri, shard_id, manifest_store); - let result = flusher.flush(&memtable, epoch, 0).await; + let result = flusher.flush(&memtable, epoch, 0, 0).await; assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("empty MemTable")); @@ -1246,8 +1316,8 @@ mod tests { .unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); - assert!(memtable.all_flushed_to_wal()); + let durable = frag_id + 1; + assert!(memtable.all_flushed_to_wal(durable)); let flusher = MemTableFlusher::new( store.clone(), @@ -1256,21 +1326,21 @@ mod tests { shard_id, manifest_store.clone(), ); - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, 10); assert_eq!(result.covered_wal_entry_position, 1); // Verify manifest was updated - let updated_manifest = manifest_store.read_latest().await.unwrap().unwrap(); + let updated_manifest = manifest_store.latest().await.unwrap().unwrap(); assert_eq!(updated_manifest.version, 2); assert_eq!(updated_manifest.replay_after_wal_entry_position, 1); assert_eq!(updated_manifest.current_generation, 2); - assert_eq!(updated_manifest.flushed_generations.len(), 1); + assert_eq!(updated_manifest.sstables.len(), 1); } - /// A `GenerationWarmer` that counts calls and optionally fails. + /// A `SsTableWarmer` that counts calls and optionally fails. #[derive(Debug)] struct CountingWarmer { calls: Arc, @@ -1278,7 +1348,7 @@ mod tests { } #[async_trait::async_trait] - impl GenerationWarmer for CountingWarmer { + impl SsTableWarmer for CountingWarmer { async fn warm(&self, _path: &str) -> Result<()> { self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); if self.fail { @@ -1310,10 +1380,10 @@ mod tests { .insert(create_test_batch(&schema, 10)) .await .unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - let warmer: Arc = Arc::new(CountingWarmer { + let warmer: Arc = Arc::new(CountingWarmer { calls: calls.clone(), fail: true, }); @@ -1327,24 +1397,24 @@ mod tests { ) .with_warmer(Some(warmer)); // Flush must succeed despite the warmer erroring. - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!( calls.load(std::sync::atomic::Ordering::SeqCst), 1, "pre-commit warm fires exactly once" ); - let updated = manifest_store.read_latest().await.unwrap().unwrap(); + let updated = manifest_store.latest().await.unwrap().unwrap(); assert_eq!( - updated.flushed_generations.len(), + updated.sstables.len(), 1, "generation still committed after a failed warm" ); } /// Flushing a generation with within-generation duplicate PKs writes a - /// deletion vector so the flushed dataset exposes newest-per-PK on scan. + /// deletion vector so the SSTable dataset exposes newest-per-PK on scan. #[tokio::test] async fn test_flush_writes_dedup_deletion_vector() { use futures::TryStreamExt; @@ -1371,7 +1441,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1380,16 +1450,16 @@ mod tests { shard_id, manifest_store, ); - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); assert_eq!(result.rows_flushed, 5, "all physical rows are written"); - // Scanning the flushed generation must honor the deletion vector and + // Scanning the SSTable must honor the deletion vector and // return only the newest version of each PK. let gen_uri = format!( "{}/_mem_wal/{}/{}", base_uri.trim_end_matches('/'), shard_id, - result.generation.path + result.sstable.path ); let dataset = Dataset::open(&gen_uri).await.unwrap(); let batches: Vec = dataset @@ -1436,7 +1506,7 @@ mod tests { /// probe by value — including for a within-gen-superseded PK (existence, /// not visibility). #[tokio::test] - async fn flushed_pk_index_sidecar_is_probeable() { + async fn sstable_pk_index_sidecar_is_probeable() { use lance_core::cache::LanceCache; use lance_index::metrics::NoOpMetricsCollector; use lance_index::registry::IndexPluginRegistry; @@ -1474,7 +1544,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1484,7 +1554,7 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &[], 1) + .flush_with_indexes(&memtable, epoch, &[], 1, durable) .await .unwrap(); @@ -1493,7 +1563,7 @@ mod tests { .clone() .join("_mem_wal") .join(shard_id.to_string()) - .join(result.generation.path.as_str()); + .join(result.sstable.path.as_str()); let index_store = Arc::new(LanceIndexStore::new( store.clone(), pk_index_path(&gen_path), @@ -1574,7 +1644,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1584,13 +1654,13 @@ mod tests { manifest_store.clone(), ); // The plain-flush path — what the writer dispatches to with no indexes. - let result = flusher.flush(&memtable, epoch, 1).await.unwrap(); + let result = flusher.flush(&memtable, epoch, 1, durable).await.unwrap(); let gen_path = base_path .clone() .join("_mem_wal") .join(shard_id.to_string()) - .join(result.generation.path.as_str()); + .join(result.sstable.path.as_str()); let index_store = Arc::new(LanceIndexStore::new( store.clone(), pk_index_path(&gen_path), @@ -1668,7 +1738,7 @@ mod tests { ) .unwrap(); let frag_id = memtable.insert(batch).await.unwrap(); - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1678,7 +1748,7 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); assert_eq!(result.rows_flushed, 5, "all physical rows are written"); @@ -1687,13 +1757,13 @@ mod tests { "{}/_mem_wal/{}/{}", base_uri.trim_end_matches('/'), shard_id, - result.generation.path + result.sstable.path ); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); // Index half of the combined manifest. @@ -1800,7 +1870,7 @@ mod tests { .unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1810,23 +1880,20 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, 10); - // Verify the flushed dataset is a single-version dataset with the BTree index - let gen_uri = format!( - "{}/_mem_wal/{}/{}", - base_uri, shard_id, result.generation.path - ); + // Verify the SSTable dataset is a single-version dataset with the BTree index + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); let indices = dataset.load_indices().await.unwrap(); @@ -1937,7 +2004,7 @@ mod tests { let frag_id = memtable.insert(batch).await.unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -1947,30 +2014,27 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, num_vectors); - // Verify the flushed dataset is a single-version dataset with the HNSW index - let gen_uri = format!( - "{}/_mem_wal/{}/{}", - base_uri, shard_id, result.generation.path - ); + // Verify the SSTable dataset is a single-version dataset with the HNSW index + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); let indices = dataset.load_indices().await.unwrap(); assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "vector_hnsw"); - // End-to-end query: pick a row from the flushed dataset, query for + // End-to-end query: pick a row from the SSTable dataset, query for // it, and verify the index path returns it as the nearest neighbor. // This exercises the on-disk HNSW + SQ8 format including the IVF // partition routing and the storage_metadata ScalarQuantizationMetadata @@ -2087,7 +2151,7 @@ mod tests { let frag_id = memtable.insert(batch).await.unwrap(); // Simulate WAL flush - memtable.mark_wal_flushed(&[frag_id], 1, &[0]); + let durable = frag_id + 1; let flusher = MemTableFlusher::new( store.clone(), @@ -2097,28 +2161,26 @@ mod tests { manifest_store.clone(), ); let result = flusher - .flush_with_indexes(&memtable, epoch, &index_configs, 1) + .flush_with_indexes(&memtable, epoch, &index_configs, 1, durable) .await .unwrap(); - assert_eq!(result.generation.generation, 1); + assert_eq!(result.sstable.generation, 1); assert_eq!(result.rows_flushed, 3); - // Verify the flushed dataset is a single-version dataset with the FTS index - let gen_uri = format!( - "{}/_mem_wal/{}/{}", - base_uri, shard_id, result.generation.path - ); + // Verify the SSTable dataset is a single-version dataset with the FTS index + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, result.sstable.path); let dataset = Dataset::open(&gen_uri).await.unwrap(); assert_eq!( dataset.version().version, 1, - "flushed dataset must be a single-version dataset" + "SSTable dataset must be a single-version dataset" ); let indices = dataset.load_indices().await.unwrap(); assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "text_fts"); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V2 as i32); // Verify FTS query returns correct results // Searching for "hello" should find the first document @@ -2172,9 +2234,8 @@ mod tests { crate::utils::test::assert_plan_node_equals( plan, "ProjectionExec: expr=[id@2 as id, text@3 as text, _score@1 as _score] - Take: ... - CoalesceBatchesExec: ... - MatchQuery: column=text, query=hello", + LanceRead: ..., source=stream(_rowid) + MatchQuery: column=text, query=[hello]", ) .await .unwrap(); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs index e1f0d6e689c..ce229a580dd 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/builder.rs @@ -18,12 +18,14 @@ use lance_datafusion::expr::safe_coerce_scalar; use lance_datafusion::planner::Planner; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::query::{FtsQuery as IndexFtsQuery, Operator}; +use lance_index::scalar::inverted::{DOC_INDEX_FIELD, DocumentGranularity}; use lance_linalg::distance::DistanceType; use super::exec::{ BTreeIndexExec, FtsIndexExec, MemTableBruteForceVectorExec, MemTableDedupScanExec, MemTableScanExec, SCORE_COLUMN, VectorIndexExec, }; +use crate::dataset::mem_wal::index::MemTableVisibility; use crate::dataset::mem_wal::scanner::{exec::validate_pk_types, parse_filter_expr}; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; @@ -75,7 +77,7 @@ pub enum FtsQueryType { }, /// Boolean query with MUST/SHOULD/MUST_NOT. Boolean { - /// Terms that must match. + /// Terms that must match and contribute to the score. must: Vec, /// Terms that should match (adds to score). should: Vec, @@ -105,6 +107,8 @@ pub struct FtsQuery { pub column: String, /// Query type. pub query_type: FtsQueryType, + /// Logical document unit. Defaults to one document per dataset row. + pub document_granularity: DocumentGranularity, /// WAND factor for early termination (0.0 to 1.0). /// 1.0 = full recall (default), <1.0 = faster but may miss low-scoring results. pub wand_factor: f32, @@ -141,6 +145,7 @@ impl FtsQuery { operator, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -155,6 +160,7 @@ impl FtsQuery { query: query.into(), slop, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -175,6 +181,7 @@ impl FtsQuery { should, must_not, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -197,6 +204,7 @@ impl FtsQuery { max_expansions: DEFAULT_MAX_EXPANSIONS, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -218,6 +226,7 @@ impl FtsQuery { max_expansions: DEFAULT_MAX_EXPANSIONS, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -241,6 +250,7 @@ impl FtsQuery { max_expansions, boost: 1.0, }, + document_granularity: DocumentGranularity::Row, wand_factor: DEFAULT_WAND_FACTOR, limit: None, include_tail: true, @@ -269,6 +279,11 @@ impl FtsQuery { self } + pub fn with_document_granularity(mut self, document_granularity: DocumentGranularity) -> Self { + self.document_granularity = document_granularity; + self + } + fn with_boost(mut self, boost: f32) -> Self { match &mut self.query_type { FtsQueryType::Match { boost: b, .. } | FtsQueryType::Fuzzy { boost: b, .. } => { @@ -286,7 +301,30 @@ impl FtsQuery { /// phrase leaf queries; the column must be bound on the query. Compound queries /// (boolean / boost / multi-match) cannot be modeled by the MemTable path and /// return a `not_supported` error rather than failing deep in planning. -fn local_fts_query(query: FullTextSearchQuery) -> Result { +fn resolve_memtable_document_granularity( + column: &str, + requested: Option, + indexes: Option<&IndexStore>, +) -> Result { + let available = indexes + .map(|indexes| indexes.fts_document_granularities_by_column(column)) + .unwrap_or_default(); + match requested { + Some(requested) if available.is_empty() || available.contains(&requested) => Ok(requested), + Some(requested) => Err(Error::invalid_input(format!( + "FTS query for field '{column}' requested {requested:?} document granularity, but \ + the active MemTable FTS index uses a different granularity: {available:?}" + ))), + None if available.is_empty() => Ok(DocumentGranularity::Row), + None if available.len() == 1 => Ok(available[0]), + None => Err(Error::invalid_input(format!( + "FTS query for field '{column}' is ambiguous because Row and ListElement active \ + MemTable indexes coexist; specify document_granularity" + ))), + } +} + +fn local_fts_query(query: FullTextSearchQuery, indexes: Option<&IndexStore>) -> Result { let wand_factor = query.wand_factor.unwrap_or(DEFAULT_WAND_FACTOR); let limit = query .limit @@ -312,7 +350,9 @@ fn local_fts_query(query: FullTextSearchQuery) -> Result { let local = match query.query { IndexFtsQuery::Match(m) => { let column = require_column(m.column)?; - match m.fuzziness { + let document_granularity = + resolve_memtable_document_granularity(&column, m.document_granularity, indexes)?; + let local = match m.fuzziness { // Some(0) is an exact match in the index model. Some(0) => FtsQuery::match_query_with_operator(column, m.terms, m.operator) .with_boost(m.boost), @@ -330,9 +370,16 @@ fn local_fts_query(query: FullTextSearchQuery) -> Result { m.max_expansions, ) .with_boost(m.boost), - } + }; + local.with_document_granularity(document_granularity) + } + IndexFtsQuery::Phrase(p) => { + let column = require_column(p.column)?; + let document_granularity = + resolve_memtable_document_granularity(&column, p.document_granularity, indexes)?; + FtsQuery::phrase(column, p.terms, p.slop) + .with_document_granularity(document_granularity) } - IndexFtsQuery::Phrase(p) => FtsQuery::phrase(require_column(p.column)?, p.terms, p.slop), other => { return Err(Error::not_supported(format!( "MemTable full-text search supports match and phrase queries, got: {other}" @@ -376,11 +423,11 @@ impl ScalarPredicate { /// Provides a builder pattern similar to Lance's Scanner interface /// for constructing DataFusion execution plans over in-memory data. /// -/// # Index Visibility Model +/// # Readable Prefix /// -/// The scanner captures `max_visible_batch_position` from the `IndexStore` at -/// construction time. This frozen visibility ensures queries only see data -/// that has been indexed, providing consistent results. +/// The scanner snapshots one readable prefix at construction, so every plan it +/// builds cuts at the same bound. [`MemTableVisibility`] selects which prefix: +/// published, or the writer's own indexed prefix. /// /// # Example /// @@ -400,9 +447,9 @@ pub struct MemTableScanner { batch_store: Arc, indexes: Arc, schema: SchemaRef, - /// Frozen visibility captured at scanner construction time. - /// This is the `max_visible_batch_position` from the IndexStore. - max_visible_batch_position: usize, + /// Readable prefix frozen at scanner construction. Which `IndexStore` + /// cursor it came from is this scanner's [`MemTableVisibility`]. + readable_count: usize, projection: Option>, filter: Option, limit: Option, @@ -425,27 +472,32 @@ pub struct MemTableScanner { } impl MemTableScanner { - /// Create a new scanner. - /// - /// Captures `max_visible_batch_position` from the `IndexStore` at construction - /// time to ensure consistent query visibility. + /// Create a new scanner over the published prefix. /// /// # Arguments /// /// * `batch_store` - Lock-free batch store containing the data - /// * `indexes` - Index registry (carries the visibility watermark) + /// * `indexes` - Index registry (carries the visibility cursors) /// * `schema` - Schema of the data pub fn new(batch_store: Arc, indexes: Arc, schema: SchemaRef) -> Self { - // Snapshot the visibility cursor at construction time. The cursor is - // advanced by `flush_from_batch_store` after the WAL append succeeds, - // so this snapshot reflects WAL-durable data. - let max_visible_batch_position = indexes.max_visible_batch_position(); + Self::new_at_visibility(batch_store, indexes, schema, MemTableVisibility::Published) + } + + /// As [`Self::new`], bounded by `visibility`. Snapshotted at construction, + /// so every plan this scanner builds keys on one stable cursor. + pub fn new_at_visibility( + batch_store: Arc, + indexes: Arc, + schema: SchemaRef, + visibility: MemTableVisibility, + ) -> Self { + let readable_count = indexes.prefix_count(visibility); Self { batch_store, indexes, schema, - max_visible_batch_position, + readable_count, projection: None, filter: None, limit: None, @@ -504,12 +556,12 @@ impl MemTableScanner { self } - /// The `max_visible_batch_position` snapshot this scanner latched at - /// construction. A downstream recency filter must key on this same snapshot - /// (not a fresh read of the IndexStore watermark, which a concurrent append - /// could have advanced) so it stays consistent with the rows the search saw. - pub fn max_visible_batch_position(&self) -> usize { - self.max_visible_batch_position + /// The readable-prefix snapshot this scanner latched at construction. A + /// downstream recency filter must key on this same snapshot (not a fresh + /// read of the IndexStore cursor, which a concurrent append could have + /// advanced) so it stays consistent with the rows the search saw. + pub fn readable_count(&self) -> usize { + self.readable_count } /// Include the _rowaddr column in output. @@ -688,7 +740,7 @@ impl MemTableScanner { /// queries are supported; compound queries (boolean/boost/multi-match) are /// not yet supported by the MemTable path and return an error. pub fn full_text_search(&mut self, query: FullTextSearchQuery) -> Result<&mut Self> { - self.full_text_query = Some(local_fts_query(query)?); + self.full_text_query = Some(local_fts_query(query, Some(self.indexes.as_ref()))?); Ok(self) } @@ -971,7 +1023,7 @@ impl MemTableScanner { let scan = MemTableScanExec::with_filter( self.batch_store.clone(), - self.max_visible_batch_position, + self.readable_count, projection_indices, self.output_schema(), self.schema.clone(), @@ -1033,7 +1085,7 @@ impl MemTableScanner { Ok(Arc::new(MemTableDedupScanExec::new( self.batch_store.clone(), - self.max_visible_batch_position, + self.readable_count, projection_indices, self.output_schema(), pk_indices, @@ -1046,7 +1098,7 @@ impl MemTableScanner { /// Plan a BTree index query. /// - /// Uses the effective visibility (min of max_visible and max_indexed) to ensure + /// Uses the effective visibility (min of max_readable and max_indexed) to ensure /// queries only see indexed data. Falls back to full scan if no index exists. async fn plan_btree_query( &self, @@ -1056,14 +1108,14 @@ impl MemTableScanner { return self.plan_full_scan().await; } - let max_visible = self.max_visible_batch_position; + let max_readable = self.readable_count; let projection_indices = self.compute_projection_indices()?; let index_exec = BTreeIndexExec::new( self.batch_store.clone(), self.indexes.clone(), predicate.clone(), - max_visible, + max_readable, projection_indices, self.output_schema(), self.with_row_id, @@ -1095,7 +1147,7 @@ impl MemTableScanner { } async fn plan_vector_search(&self, query: &VectorQuery) -> Result> { - let max_visible = self.max_visible_batch_position; + let max_readable = self.readable_count; let projection_indices = self.compute_projection_indices()?; let base_schema = self.base_output_schema(); let filter_predicate = self.filter_predicate()?; @@ -1115,15 +1167,23 @@ impl MemTableScanner { .as_ref() .map(|_| self.indexes.has_pk_index() && !self.indexes.pk_has_overrides()) .unwrap_or(true); + // A distance lower bound excludes the *nearest* rows, and + // `VectorIndexExec` can only drop them after the graph search has + // already cut to k — leaving fewer than k in-range rows, or none. + // Brute force filters the complete candidate set before its cut, so it + // is the only correct arm here. An upper bound is safe on HNSW: it + // trims the far tail, which the top-k would have dropped anyway. + let hnsw_safe_with_bounds = query.distance_lower_bound.is_none(); let exec: Arc = if filter_predicate.is_none() && hnsw_safe_with_pk + && hnsw_safe_with_bounds && self.has_vector_index(&query.column) { Arc::new(VectorIndexExec::new( self.batch_store.clone(), self.indexes.clone(), query.clone(), - max_visible, + max_readable, projection_indices, base_schema, self.with_row_id, @@ -1133,7 +1193,7 @@ impl MemTableScanner { MemTableBruteForceVectorExec::new( self.batch_store.clone(), query.clone(), - max_visible, + max_readable, projection_indices, base_schema, self.with_row_id, @@ -1147,14 +1207,14 @@ impl MemTableScanner { /// Plan a full-text search. /// - /// Uses the effective visibility (min of max_visible and max_indexed) to ensure + /// Uses the effective visibility (min of max_readable and max_indexed) to ensure /// queries only see indexed data. async fn plan_fts_search(&self, query: &FtsQuery) -> Result> { - if !self.has_fts_index(&query.column) { - return self.empty_fts_plan(); + if !self.has_fts_index(&query.column, query.document_granularity) { + return self.empty_fts_plan(query.document_granularity); } - let max_visible = self.max_visible_batch_position; + let max_readable = self.readable_count; let projection_indices = self.compute_projection_indices()?; let filter_predicate = self.filter_predicate()?; if let Some(pk_columns) = &self.pk_columns { @@ -1165,7 +1225,7 @@ impl MemTableScanner { self.batch_store.clone(), self.indexes.clone(), query.clone(), - max_visible, + max_readable, projection_indices, self.base_output_schema(), self.with_row_id, @@ -1175,7 +1235,10 @@ impl MemTableScanner { self.apply_post_index_ops(Arc::new(index_exec)).await } - fn empty_fts_plan(&self) -> Result> { + fn empty_fts_plan( + &self, + document_granularity: DocumentGranularity, + ) -> Result> { use datafusion::physical_plan::empty::EmptyExec; let mut fields: Vec = self @@ -1184,6 +1247,9 @@ impl MemTableScanner { .iter() .map(|f| f.as_ref().clone()) .collect(); + if document_granularity.is_list_element() { + fields.push(DOC_INDEX_FIELD.clone()); + } fields.push(Field::new(SCORE_COLUMN, DataType::Float32, true)); if self.with_row_id { fields.push(Field::new(ROW_ID, DataType::UInt64, true)); @@ -1230,15 +1296,111 @@ impl MemTableScanner { } } + /// Collect `col = lit OR col IN (lit, ..) OR ..` over one column into its + /// values, or return false and leave the caller to fall back to a full scan. + fn collect_or_equalities( + &self, + expr: &Expr, + column: &mut Option, + values: &mut Vec, + ) -> bool { + let mut same_column = |name: &str| match column { + Some(existing) => existing == name, + None => { + *column = Some(name.to_string()); + true + } + }; + // The exec answers `In` by concatenating a lookup per value, so a value + // listed twice would emit its rows twice. Two disjuncts can easily name + // the same value: the signed-zero rewrite turns both sides of + // `x = -0.0 OR x = 0.0` into the same two-element list. + fn push_once(values: &mut Vec, value: ScalarValue) { + if !values.contains(&value) { + values.push(value); + } + } + match expr { + Expr::BinaryExpr(binary) if binary.op == datafusion::logical_expr::Operator::Or => { + self.collect_or_equalities(&binary.left, column, values) + && self.collect_or_equalities(&binary.right, column, values) + } + Expr::BinaryExpr(binary) if binary.op == datafusion::logical_expr::Operator::Eq => { + let (Expr::Column(col), Expr::Literal(lit, _)) = + (binary.left.as_ref(), binary.right.as_ref()) + else { + return false; + }; + let Some(value) = self.coerce_literal_to_column(&col.name, lit) else { + return false; + }; + if !same_column(&col.name) { + return false; + } + push_once(values, value); + true + } + Expr::InList(in_list) if !in_list.negated => { + let Expr::Column(col) = in_list.expr.as_ref() else { + return false; + }; + if !same_column(&col.name) { + return false; + } + for item in &in_list.list { + let Expr::Literal(lit, _) = item else { + return false; + }; + // A NULL among the values makes `IN` return NULL rather than + // false, which a key lookup does not reproduce; fall back. + if lit.is_null() { + return false; + } + let Some(value) = self.coerce_literal_to_column(&col.name, lit) else { + return false; + }; + push_once(values, value); + } + true + } + _ => false, + } + } + /// Extract a BTree-compatible predicate from the filter. /// /// This method also coerces literal values to match the column's data type /// (e.g., Int64 literal -> Int32 when the column is Int32). fn extract_btree_predicate(&self) -> Option { - let filter = self.filter.as_ref()?; + // `filter()` stores the parsed expression without running `optimize_expr`, + // so run it here to pick the plan from the same expression the full scan + // would evaluate. Coercion has to happen before the signed-zero rewrite + // inside it, otherwise `value = 0` keeps its integer literal and gets a + // bit-exact lookup while the scan beside it answers per IEEE 754. An + // expression `optimize_expr` rejects is reported by `plan_full_scan`, + // which runs the same pass, so there is nothing to report here. + let planner = Planner::new(self.schema.clone()); + let filter = planner + .optimize_expr(self.filter.clone()?) + .inspect_err(|error| { + log::debug!("memtable index fast path skipped: {error}"); + }) + .ok()?; // Simple pattern matching for common predicates - match filter { + match &filter { + // `simplify` turns an `IN` list of three or fewer values back into an + // OR chain of equalities, and the signed-zero rewrite then turns any + // zero among them into a two-element list of its own, so the fast path + // has to accept the chain to keep covering `IN`. + Expr::BinaryExpr(binary) if binary.op == datafusion::logical_expr::Operator::Or => { + let mut column = None; + let mut values = Vec::new(); + if self.collect_or_equalities(&filter, &mut column, &mut values) { + debug_assert!(column.is_some(), "a true return always names the column"); + return column.map(|column| ScalarPredicate::In { column, values }); + } + } Expr::BinaryExpr(binary) => { if let (Expr::Column(col), Expr::Literal(lit, _)) = (binary.left.as_ref(), binary.right.as_ref()) @@ -1325,8 +1487,10 @@ impl MemTableScanner { } /// Check if an FTS index exists for a column. - fn has_fts_index(&self, column: &str) -> bool { - self.indexes.get_fts_by_column(column).is_some() + fn has_fts_index(&self, column: &str, document_granularity: DocumentGranularity) -> bool { + self.indexes + .get_fts_by_column_and_granularity(column, document_granularity) + .is_some() } } @@ -1426,7 +1590,7 @@ mod tests { let indexes = Arc::new(index_store); let scanner = MemTableScanner::new(batch_store, indexes, schema.clone()); let result = scanner.try_into_batch().await.unwrap(); - // max_visible_batch_position is 1, so we see batches 0 and 1 (20 rows) + // readable_count is 1, so we see batches 0 and 1 (20 rows) assert_eq!(result.num_rows(), 20); } @@ -1445,6 +1609,105 @@ mod tests { assert_eq!(result.schema().field(0).name(), "id"); } + /// The index fast path is chosen from the filter the caller set, which has not + /// been through `optimize_expr`. Running it there is what keeps a float zero + /// from getting a bit-exact lookup while the full scan beside it answers per + /// IEEE 754. The integer spelling matters too: the rewrite only fires once + /// coercion has given the literal the column's type. + #[rstest::rstest] + #[case::float_literal("value = 0.0")] + #[case::integer_literal("value = 0")] + fn test_extract_btree_predicate_covers_both_zero_encodings(#[case] equality: &str) { + let schema = Arc::new(Schema::new(vec![Field::new( + "value", + DataType::Float64, + true, + )])); + let batch_store = Arc::new(BatchStore::with_capacity(8)); + let mut scanner = MemTableScanner::new( + batch_store, + Arc::new(IndexStore::new()), + schema as SchemaRef, + ); + + scanner.filter(equality).unwrap(); + match scanner.extract_btree_predicate() { + Some(ScalarPredicate::In { column, values }) => { + assert_eq!(column, "value"); + assert_eq!( + values, + vec![ + ScalarValue::Float64(Some(-0.0)), + ScalarValue::Float64(Some(0.0)), + ] + ); + } + other => panic!("expected an In predicate over both encodings, got {other:?}"), + } + + // `simplify` shortens a two-value `IN` list into an OR chain, and the + // rewrite then replaces the zero with a list of its own. Both spellings + // still have to reach the index. + scanner.filter("value IN (0.0, 1.0)").unwrap(); + match scanner.extract_btree_predicate() { + Some(ScalarPredicate::In { column, values }) => { + assert_eq!(column, "value"); + assert_eq!( + values, + vec![ + ScalarValue::Float64(Some(-0.0)), + ScalarValue::Float64(Some(0.0)), + ScalarValue::Float64(Some(1.0)), + ] + ); + } + other => panic!("expected an In predicate covering the list, got {other:?}"), + } + + // A short list with no zero in it is shortened just the same, so this is + // what keeps the pre-existing `IN` fast path from being lost. + scanner.filter("value IN (1.0, 2.0)").unwrap(); + match scanner.extract_btree_predicate() { + Some(ScalarPredicate::In { values, .. }) => { + assert_eq!( + values, + vec![ + ScalarValue::Float64(Some(1.0)), + ScalarValue::Float64(Some(2.0)), + ] + ); + } + other => panic!("expected an In predicate, got {other:?}"), + } + + // Both disjuncts rewrite to the same two-element list. The exec answers + // `In` with one lookup per value and concatenates, so a value listed twice + // would return its rows twice. + scanner.filter("value = -0.0 OR value = 0.0").unwrap(); + match scanner.extract_btree_predicate() { + Some(ScalarPredicate::In { values, .. }) => { + assert_eq!( + values, + vec![ + ScalarValue::Float64(Some(-0.0)), + ScalarValue::Float64(Some(0.0)), + ] + ); + } + other => panic!("expected a deduplicated In predicate, got {other:?}"), + } + + // `<` has to compare against the negative encoding, or the lookup admits a + // row the predicate excludes. + scanner.filter("value < 0.0").unwrap(); + match scanner.extract_btree_predicate() { + Some(ScalarPredicate::Range { upper, .. }) => { + assert_eq!(upper, Some(ScalarValue::Float64(Some(-0.0)))); + } + other => panic!("expected a Range predicate, got {other:?}"), + } + } + #[tokio::test] async fn test_scanner_limit() { let schema = create_test_schema(); @@ -1549,20 +1812,67 @@ mod tests { let q = FullTextSearchQuery::new("hello".to_string()) .with_column("text".to_string()) .unwrap(); - let local = local_fts_query(q).unwrap(); + let local = local_fts_query(q, None).unwrap(); assert_eq!(local.column, "text"); assert!( matches!(local.query_type, FtsQueryType::Match { query, operator, .. } if query == "hello" && operator == Operator::Or) ); + let mut indexes = IndexStore::new(); + indexes + .add_fts_with_params( + "tags_list_element_idx".to_string(), + 1, + "tags".to_string(), + lance_index::scalar::inverted::InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let inferred = FullTextSearchQuery::new("hello".to_string()) + .with_column("tags".to_string()) + .unwrap(); + let inferred = local_fts_query(inferred, Some(&indexes)).unwrap(); + assert_eq!( + inferred.document_granularity, + DocumentGranularity::ListElement + ); + let conflicting = FullTextSearchQuery::new_query(IndexFtsQuery::Match( + MatchQuery::new("hello".to_string()) + .with_column(Some("tags".to_string())) + .with_document_granularity(DocumentGranularity::Row), + )); + assert!( + local_fts_query(conflicting, Some(&indexes)) + .unwrap_err() + .to_string() + .contains("different granularity") + ); + indexes + .add_fts_with_params( + "tags_idx".to_string(), + 1, + "tags".to_string(), + lance_index::scalar::inverted::InvertedIndexParams::default(), + ) + .unwrap(); + let ambiguous = FullTextSearchQuery::new("hello".to_string()) + .with_column("tags".to_string()) + .unwrap(); + assert!( + local_fts_query(ambiguous, Some(&indexes)) + .unwrap_err() + .to_string() + .contains("ambiguous") + ); + let exact_and = FullTextSearchQuery::new_query(IndexFtsQuery::Match( MatchQuery::new("hello world".to_string()) .with_operator(Operator::And) .with_boost(3.0) .with_column(Some("text".to_string())), )); - let local = local_fts_query(exact_and).unwrap(); + let local = local_fts_query(exact_and, None).unwrap(); assert!( matches!(local.query_type, FtsQueryType::Match { query, operator, boost } if query == "hello world" && operator == Operator::And && boost == 3.0) @@ -1576,7 +1886,7 @@ mod tests { .with_boost(2.5) .with_column(Some("text".to_string())), )); - let local = local_fts_query(fuzzy).unwrap(); + let local = local_fts_query(fuzzy, None).unwrap(); assert!( matches!(local.query_type, FtsQueryType::Fuzzy { fuzziness, prefix_length, boost, .. } if fuzziness == Some(2) && prefix_length == 2 && boost == 2.5) @@ -1589,7 +1899,7 @@ mod tests { .with_column(Some("text".to_string())), )); assert!( - local_fts_query(fuzzy_and).is_err(), + local_fts_query(fuzzy_and, None).is_err(), "fuzzy AND cannot be represented by the local memtable query" ); @@ -1597,7 +1907,7 @@ mod tests { let phrase = FullTextSearchQuery::new_query(IndexFtsQuery::Phrase( PhraseQuery::new("quick fox".to_string()).with_column(Some("text".to_string())), )); - let local = local_fts_query(phrase).unwrap(); + let local = local_fts_query(phrase, None).unwrap(); assert!(matches!(local.query_type, FtsQueryType::Phrase { .. })); // Compound (boolean) -> not supported. @@ -1609,14 +1919,14 @@ mod tests { ), )]))); assert!( - local_fts_query(boolean).is_err(), + local_fts_query(boolean, None).is_err(), "boolean must be rejected" ); // Missing column -> error. let no_col = FullTextSearchQuery::new("hi".to_string()); assert!( - local_fts_query(no_col).is_err(), + local_fts_query(no_col, None).is_err(), "missing column must error" ); } diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs index 2a593a0f215..3144e65595b 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec.rs @@ -36,8 +36,8 @@ pub use vector::VectorIndexExec; pub(super) fn newest_pk_positions( batch_store: &BatchStore, pk_columns: &[String], - max_visible_batch_position: usize, - max_visible_row: u64, + readable_count: usize, + max_readable_row: u64, ) -> DataFusionResult> { let mut newest: HashMap, u64> = HashMap::new(); let mut current_row: u64 = 0; @@ -46,14 +46,14 @@ pub(super) fn newest_pk_positions( if n == 0 { continue; } - if batch_position > max_visible_batch_position { + if batch_position >= readable_count { current_row += n as u64; continue; } let pk_indices = resolve_pk_indices(&stored_batch.data, pk_columns)?; for row in 0..n { let pos = current_row + row as u64; - if pos > max_visible_row { + if pos > max_readable_row { break; } let key = pk_key(&stored_batch.data, &pk_indices, row)?; diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs index 8a239605b50..634eeba5846 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/brute_force_vector.rs @@ -11,7 +11,6 @@ //! or new rows in the window between commit and next memtable rotation), this //! exec keeps KNN correct by computing exact distances row-by-row. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -40,7 +39,7 @@ use crate::dataset::mem_wal::write::BatchStore; /// Distance metric used when [`VectorQuery::distance_type`] is `None`. The /// indexed path defers to the index's own metric, but with no index there is /// no inherent default — L2 matches what most callers configure and what the -/// flushed/base arms use when re-ranking unindexed candidates. +/// SSTable/base arms use when re-ranking unindexed candidates. const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2; /// Brute-force KNN over an active memtable without an HNSW. Produces the same @@ -48,7 +47,7 @@ const DEFAULT_DISTANCE_TYPE: DistanceType = DistanceType::L2; pub struct MemTableBruteForceVectorExec { batch_store: Arc, query: VectorQuery, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, @@ -69,10 +68,7 @@ impl Debug for MemTableBruteForceVectorExec { f.debug_struct("MemTableBruteForceVectorExec") .field("column", &self.query.column) .field("k", &self.query.k) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("readable_count", &self.readable_count) .field("with_row_id", &self.with_row_id) .finish() } @@ -85,7 +81,7 @@ impl MemTableBruteForceVectorExec { pub fn new( batch_store: Arc, query: VectorQuery, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, base_schema: SchemaRef, with_row_id: bool, @@ -111,7 +107,7 @@ impl MemTableBruteForceVectorExec { Ok(Self { batch_store, query, - max_visible_batch_position, + readable_count, projection, output_schema, properties, @@ -160,23 +156,23 @@ impl MemTableBruteForceVectorExec { Ok(Some(mask)) } - /// Last row position visible under `max_visible_batch_position`, or `None` - /// if no batches are visible. Identical to `VectorIndexExec`'s helper so - /// both arms cut at the same MVCC boundary. - fn compute_max_visible_row(&self) -> Option { - let mut max_visible_row_exclusive: u64 = 0; + /// Last row position within `readable_count`, or `None` if nothing is + /// readable. Identical to `VectorIndexExec`'s helper so both arms cut at + /// the same bound. + fn compute_max_readable_row(&self) -> Option { + let mut max_readable_row_exclusive: u64 = 0; let mut current_row: u64 = 0; for (batch_position, stored_batch) in self.batch_store.iter().enumerate() { let batch_end = current_row + stored_batch.num_rows as u64; - if batch_position <= self.max_visible_batch_position { - max_visible_row_exclusive = batch_end; + if batch_position < self.readable_count { + max_readable_row_exclusive = batch_end; } current_row = batch_end; } - if max_visible_row_exclusive > 0 { - Some(max_visible_row_exclusive - 1) + if max_readable_row_exclusive > 0 { + Some(max_readable_row_exclusive - 1) } else { None } @@ -208,7 +204,7 @@ impl MemTableBruteForceVectorExec { if self.query.k == 0 { return Ok(Vec::new()); } - let Some(max_visible_row) = self.compute_max_visible_row() else { + let Some(max_readable_row) = self.compute_max_readable_row() else { return Ok(Vec::new()); }; let query_flat = self.query_as_flat()?; @@ -225,8 +221,8 @@ impl MemTableBruteForceVectorExec { newest_pk_positions( &self.batch_store, pk_columns, - self.max_visible_batch_position, - max_visible_row, + self.readable_count, + max_readable_row, ) .map_err(|e| Error::invalid_input(e.to_string()))?, ) @@ -235,7 +231,7 @@ impl MemTableBruteForceVectorExec { }; // Walk batches in append order. `current_row` is the global row offset - // of the *next* row about to be visited; rows past `max_visible_row` + // of the *next* row about to be visited; rows past `max_readable_row` // are dropped before they reach the heap. let mut current_row: u64 = 0; let mut candidates: Vec<(f32, u64)> = Vec::new(); @@ -245,7 +241,7 @@ impl MemTableBruteForceVectorExec { if n == 0 { continue; } - if batch_position > self.max_visible_batch_position { + if batch_position >= self.readable_count { current_row += n as u64; continue; } @@ -280,7 +276,7 @@ impl MemTableBruteForceVectorExec { for row in 0..n { let pos = current_row + row as u64; - if pos > max_visible_row { + if pos > max_readable_row { break; } // Skip superseded versions: only the newest version of each PK is @@ -426,10 +422,6 @@ impl ExecutionPlan for MemTableBruteForceVectorExec { "MemTableBruteForceVectorExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -466,12 +458,12 @@ impl ExecutionPlan for MemTableBruteForceVectorExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact(self.query.k), total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -615,7 +607,7 @@ mod tests { MemTableBruteForceVectorExec::new( store, query, - /* max_visible_batch_position = */ usize::MAX, + /* readable_count = */ usize::MAX, None, schema, false, @@ -668,7 +660,7 @@ mod tests { } #[tokio::test] - async fn respects_max_visible_batch_position() { + async fn respects_indexed_count() { // Two batches of two rows. Freeze at batch 0 — only ids 0,1 are // visible candidates; the (closer) ids 2,3 in batch 1 are excluded. let schema = make_schema(); @@ -678,7 +670,7 @@ mod tests { let query = query_for([0.0, 0.0], 4); let exec = Arc::new( MemTableBruteForceVectorExec::new( - store, query, /* max_visible_batch_position = */ 0, None, schema, false, + store, query, /* readable_count = */ 1, None, schema, false, ) .expect("ctor"), ); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs index fed61698fab..315c1d7cb03 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/btree.rs @@ -3,7 +3,6 @@ //! BTreeIndexExec - BTree index queries with MVCC visibility. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -31,7 +30,7 @@ pub struct BTreeIndexExec { batch_store: Arc, indexes: Arc, predicate: ScalarPredicate, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, @@ -48,10 +47,7 @@ impl Debug for BTreeIndexExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("BTreeIndexExec") .field("predicate", &self.predicate) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("readable_count", &self.readable_count) .field("with_row_id", &self.with_row_id) .field("with_row_address", &self.with_row_address) .field("column", &self.column) @@ -67,7 +63,7 @@ impl BTreeIndexExec { /// * `batch_store` - Lock-free batch store containing data /// * `indexes` - Index registry with BTree indexes /// * `predicate` - Scalar predicate to apply - /// * `max_visible_batch_position` - MVCC visibility sequence number + /// * `readable_count` - Exclusive count of batch positions this scan may read /// * `projection` - Optional column indices to project /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested) /// * `with_row_id` - Whether to include _rowid column (row position) @@ -77,7 +73,7 @@ impl BTreeIndexExec { batch_store: Arc, indexes: Arc, predicate: ScalarPredicate, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, with_row_id: bool, @@ -103,7 +99,7 @@ impl BTreeIndexExec { batch_store, indexes, predicate, - max_visible_batch_position, + readable_count, projection, output_schema, properties, @@ -114,22 +110,22 @@ impl BTreeIndexExec { }) } - /// Compute the maximum visible row position based on max_visible_batch_position. - /// Returns None if no batches are visible. - fn compute_max_visible_row(&self) -> Option { - let mut max_visible_row_exclusive: u64 = 0; + /// Last row position within `readable_count`, or None if nothing is + /// readable. + fn compute_max_readable_row(&self) -> Option { + let mut max_readable_row_exclusive: u64 = 0; let mut current_row: u64 = 0; for (batch_position, stored_batch) in self.batch_store.iter().enumerate() { let batch_end = current_row + stored_batch.num_rows as u64; - if batch_position <= self.max_visible_batch_position { - max_visible_row_exclusive = batch_end; + if batch_position < self.readable_count { + max_readable_row_exclusive = batch_end; } current_row = batch_end; } - if max_visible_row_exclusive > 0 { - Some(max_visible_row_exclusive - 1) + if max_readable_row_exclusive > 0 { + Some(max_readable_row_exclusive - 1) } else { None } @@ -141,7 +137,7 @@ impl BTreeIndexExec { return vec![]; }; - let Some(max_visible_row) = self.compute_max_visible_row() else { + let Some(max_readable_row) = self.compute_max_readable_row() else { return vec![]; }; @@ -179,7 +175,7 @@ impl BTreeIndexExec { // Filter by visibility positions .into_iter() - .filter(|&pos| pos <= max_visible_row) + .filter(|&pos| pos <= max_readable_row) .collect() } @@ -312,10 +308,6 @@ impl ExecutionPlan for BTreeIndexExec { "BTreeIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -358,13 +350,13 @@ impl ExecutionPlan for BTreeIndexExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { // We can't know the exact count without querying the index - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -434,7 +426,7 @@ mod tests { batch_store, indexes, predicate, - 0, // max_visible_batch_position (batch at position 0) + 1, // readable_count (batch at position 0) None, schema, false, @@ -478,7 +470,7 @@ mod tests { batch_store, indexes, predicate, - 0, + 1, None, schema, false, @@ -518,12 +510,12 @@ mod tests { value: ScalarValue::Int32(Some(15)), }; - // Query with max_visible=0 should not see batch at position 1 + // Query with max_readable=0 should not see batch at position 1 let exec = BTreeIndexExec::new( batch_store.clone(), indexes.clone(), predicate.clone(), - 0, + 1, None, schema.clone(), false, @@ -538,12 +530,12 @@ mod tests { let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 0); - // Query with max_visible=1 should see both batches + // Query with max_readable=1 should see both batches let exec = BTreeIndexExec::new( batch_store, indexes, predicate, - 1, + 2, None, schema, false, @@ -592,7 +584,7 @@ mod tests { batch_store, indexes, predicate, - 0, + 1, None, schema_with_rowid.clone(), true, @@ -656,7 +648,7 @@ mod tests { batch_store.clone(), indexes.clone(), predicate.clone(), - 0, + 1, None, schema.clone(), false, @@ -684,7 +676,7 @@ mod tests { batch_store, indexes, predicate, - 0, + 1, None, schema_with_rowid, true, diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs index ba5947e4b12..c170053d9b7 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/dedup_scan.rs @@ -16,7 +16,6 @@ //! forward-aligned mask. A single `filter_record_batch` over the original //! batch then emits the survivors with no per-column reverse copy. -use std::any::Any; use std::collections::HashSet; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -44,7 +43,7 @@ use crate::dataset::mem_wal::write::BatchStore; /// that satisfy the (optional) predicate. See the module doc. pub struct MemTableDedupScanExec { batch_store: Arc, - max_visible_batch_position: usize, + readable_count: usize, /// Column indices to project (into the source schema). projection: Option>, output_schema: SchemaRef, @@ -62,10 +61,7 @@ pub struct MemTableDedupScanExec { impl Debug for MemTableDedupScanExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("MemTableDedupScanExec") - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("readable_count", &self.readable_count) .field("projection", &self.projection) .field("pk_indices", &self.pk_indices) .field("with_row_address", &self.with_row_address) @@ -79,7 +75,7 @@ impl MemTableDedupScanExec { #[allow(clippy::too_many_arguments)] pub fn new( batch_store: Arc, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, pk_indices: Vec, @@ -97,7 +93,7 @@ impl MemTableDedupScanExec { Self { batch_store, - max_visible_batch_position, + readable_count, projection, output_schema, pk_indices, @@ -155,10 +151,6 @@ impl ExecutionPlan for MemTableDedupScanExec { "MemTableDedupScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -188,7 +180,7 @@ impl ExecutionPlan for MemTableDedupScanExec { // back-to-front below. let mut batches = self .batch_store - .visible_batches_with_offsets(self.max_visible_batch_position); + .visible_batches_with_offsets(self.readable_count); batches.reverse(); let projection = self.projection.clone(); @@ -280,12 +272,12 @@ impl ExecutionPlan for MemTableDedupScanExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -347,7 +339,7 @@ mod tests { /// Run the exec and collect (id -> (value, rowaddr)). async fn run( store: Arc, - max_visible: usize, + readable_count: usize, filter: Option, ) -> HashMap, u64)> { let filter_predicate = filter.map(|expr| { @@ -358,7 +350,7 @@ mod tests { let filter_expr = None; let exec = MemTableDedupScanExec::new( store, - max_visible, + readable_count, None, output_schema(), vec![0], @@ -392,11 +384,11 @@ mod tests { // id=10 inserted (100) then updated to NULL, all in one batch. store.append(batch(&[(10, Some(100)), (10, None)])).unwrap(); - let no_filter = run(store.clone(), 0, None).await; + let no_filter = run(store.clone(), 1, None).await; assert_eq!(no_filter.len(), 1); assert_eq!(no_filter[&10].0, None, "newest version of id=10 is NULL"); - let not_null = run(store, 0, Some(col("value").is_not_null())).await; + let not_null = run(store, 1, Some(col("value").is_not_null())).await; assert!( !not_null.contains_key(&10), "id=10 newest is NULL; the stale value=100 must not leak under value IS NOT NULL" @@ -413,7 +405,7 @@ mod tests { store.append(batch(&[(20, Some(999)), (30, None)])).unwrap(); // No filter: newest per PK = {10:NULL@2, 20:999@3, 30:NULL@4}. - let all = run(store.clone(), 1, None).await; + let all = run(store.clone(), 2, None).await; assert_eq!(all.len(), 3); assert_eq!(all[&10], (None, 2)); assert_eq!(all[&20], (Some(999), 3)); @@ -421,7 +413,7 @@ mod tests { // value IS NOT NULL: only id=20 (newest 999) survives; 10 and 30 are // newest-NULL so they must be absent (no stale leak). - let not_null = run(store, 1, Some(col("value").is_not_null())).await; + let not_null = run(store, 2, Some(col("value").is_not_null())).await; assert_eq!(not_null.len(), 1); assert_eq!(not_null[&20], (Some(999), 3)); } @@ -434,7 +426,7 @@ mod tests { // id=40 inserted NULL then updated to 400 (newest non-NULL). store.append(batch(&[(40, None), (40, Some(400))])).unwrap(); - let is_null = run(store, 0, Some(col("value").is_null())).await; + let is_null = run(store, 1, Some(col("value").is_null())).await; assert!( !is_null.contains_key(&40), "id=40 newest is 400; the stale NULL must not leak under value IS NULL" diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs index 364261c3276..53ca9596622 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/fts.rs @@ -3,10 +3,10 @@ //! FtsIndexExec - Full-text search with MVCC visibility. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use arrow_array::builder::{ListBuilder, UInt32Builder}; use arrow_array::{BooleanArray, Float32Array, RecordBatch, UInt32Array, UInt64Array}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use datafusion::common::ScalarValue; @@ -23,6 +23,7 @@ use datafusion::physical_plan::{ use datafusion_physical_expr::{EquivalenceProperties, PhysicalExprRef}; use futures::stream::{self, StreamExt}; use lance_core::{Error, Result}; +use lance_index::scalar::inverted::DOC_INDEX_FIELD; use super::super::builder::{FtsQuery, FtsQueryType}; use super::newest_pk_positions; @@ -41,24 +42,31 @@ struct BatchRange { batch_id: usize, } -type MaterializedFtsRows = (Vec>, Vec, Vec); +type MaterializedFtsRows = ( + Vec>, + Vec, + Vec, + Vec>>, +); /// ExecutionPlan node that queries FTS index with MVCC visibility. pub struct FtsIndexExec { batch_store: Arc, indexes: Arc, query: FtsQuery, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, /// Pre-computed batch ranges for O(log n) lookup. batch_ranges: Vec, - /// Maximum visible row position based on max_visible_batch_position (None if nothing visible). - max_visible_row: Option, + /// Last row position within `readable_count` (None if nothing is readable). + max_readable_row: Option, /// Whether to include _rowid column (row position) in output. with_row_id: bool, + /// Whether results identify element documents with `_doc_index`. + with_doc_index: bool, /// Optional prefilter predicate, compiled against the memtable schema. /// Applied to the materialized full-schema hits before projection so the /// FTS arm only returns rows matching the predicate. @@ -73,10 +81,7 @@ impl Debug for FtsIndexExec { f.debug_struct("FtsIndexExec") .field("column", &self.query.column) .field("query_type", &self.query.query_type) - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("readable_count", &self.readable_count) .field("with_row_id", &self.with_row_id) .finish() } @@ -90,7 +95,7 @@ impl FtsIndexExec { /// * `batch_store` - Lock-free batch store containing data /// * `indexes` - Index registry with FTS indexes /// * `query` - FTS query parameters - /// * `max_visible_batch_position` - MVCC visibility sequence number + /// * `readable_count` - Exclusive count of batch positions this scan may read /// * `projection` - Optional column indices to project /// * `base_schema` - Schema before adding score column (and _rowid if with_row_id) /// * `with_row_id` - Whether to include _rowid column (row position) @@ -98,30 +103,36 @@ impl FtsIndexExec { batch_store: Arc, indexes: Arc, query: FtsQuery, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, base_schema: SchemaRef, with_row_id: bool, ) -> Result { // Verify the index exists for this column let column = &query.column; - if indexes.get_fts_by_column(column).is_none() { + let Some(_index) = + indexes.get_fts_by_column_and_granularity(column, query.document_granularity) + else { return Err(Error::invalid_input(format!( "No FTS index found for column '{}'", column ))); - } + }; + let with_doc_index = query.document_granularity.is_list_element(); - // Build output schema: base fields + _score + optional _rowid + // Build output schema: base fields + optional _doc_index + _score + optional _rowid let mut fields: Vec = base_schema .fields() .iter() .map(|f| f.as_ref().clone()) .collect(); + if with_doc_index { + fields.push(DOC_INDEX_FIELD.clone()); + } // `_score` is nullable here to stay schema-compatible with - // `lance_index::scalar::inverted::FTS_SCHEMA` (the schema base/flushed + // `lance_index::scalar::inverted::FTS_SCHEMA` (the schema base/SSTable // FTS exec nodes emit). The LSM `full_text_search` planner unions the - // active arm with base/flushed arms; UnionExec requires schema equality + // active arm with base/SSTable arms; UnionExec requires schema equality // including nullability. The actual emitted column is always populated. fields.push(Field::new(SCORE_COLUMN, DataType::Float32, true)); if with_row_id { @@ -136,10 +147,10 @@ impl FtsIndexExec { Boundedness::Bounded, )); - // Pre-compute batch ranges for O(log n) lookup and max visible row + // Pre-compute batch ranges for O(log n) lookup and max readable row let mut batch_ranges = Vec::new(); let mut current_row = 0usize; - let mut max_visible_row_exclusive: u64 = 0; + let mut max_readable_row_exclusive: u64 = 0; for (batch_id, stored_batch) in batch_store.iter().enumerate() { let batch_start = current_row; @@ -149,15 +160,15 @@ impl FtsIndexExec { end: batch_end, batch_id, }); - if batch_id <= max_visible_batch_position { - max_visible_row_exclusive = batch_end as u64; + if batch_id < readable_count { + max_readable_row_exclusive = batch_end as u64; } current_row = batch_end; } - // Convert exclusive end to inclusive last position, or None if nothing visible - let max_visible_row = if max_visible_row_exclusive > 0 { - Some(max_visible_row_exclusive - 1) + // Convert exclusive end to inclusive last position, or None if nothing readable + let max_readable_row = if max_readable_row_exclusive > 0 { + Some(max_readable_row_exclusive - 1) } else { None }; @@ -166,14 +177,15 @@ impl FtsIndexExec { batch_store, indexes, query, - max_visible_batch_position, + readable_count, projection, output_schema, properties, metrics: ExecutionPlanMetricsSet::new(), batch_ranges, - max_visible_row, + max_readable_row, with_row_id, + with_doc_index, filter: None, pk_columns: None, }) @@ -203,8 +215,11 @@ impl FtsIndexExec { } /// Query the index and return matching rows with BM25 scores. - fn query_index(&self) -> Vec<(u64, f32)> { - let Some(index) = self.indexes.get_fts_by_column(&self.query.column) else { + fn query_index(&self) -> Vec<(u64, Option>, f32)> { + let Some(index) = self + .indexes + .get_fts_by_column_and_granularity(&self.query.column, self.query.document_granularity) + else { return vec![]; }; @@ -246,8 +261,8 @@ impl FtsIndexExec { }; let all_rows_visible = self.batch_ranges.last().is_none_or(|last| { - self.max_visible_row - .map(|max_visible| max_visible + 1 >= last.end as u64) + self.max_readable_row + .map(|max_readable| max_readable + 1 >= last.end as u64) .unwrap_or(last.end == 0) }); let pk_recency_is_noop = self.pk_columns.is_none() @@ -265,21 +280,24 @@ impl FtsIndexExec { } let entries = index.search_with_options(&query_expr, options); - // Convert to (row_position, score) pairs + // Convert to (row_position, element ordinal, score) tuples. entries .into_iter() - .map(|entry| (entry.row_position, entry.score)) + .map(|entry| (entry.row_position, entry.doc_index, entry.score)) .collect() } /// Filter results by MVCC visibility using max_row_position. O(n). - fn filter_by_visibility(&self, results: Vec<(u64, f32)>) -> Vec<(u64, f32)> { - let Some(max_visible) = self.max_visible_row else { + fn filter_by_visibility( + &self, + results: Vec<(u64, Option>, f32)>, + ) -> Vec<(u64, Option>, f32)> { + let Some(max_readable) = self.max_readable_row else { return vec![]; }; results .into_iter() - .filter(|&(pos, _)| pos <= max_visible) + .filter(|(pos, _, _)| *pos <= max_readable) .collect() } @@ -289,7 +307,7 @@ impl FtsIndexExec { /// then combines them into a single batch. fn materialize_rows_sorted( &self, - results: &[(u64, f32)], + results: &[(u64, Option>, f32)], ) -> DataFusionResult> { if results.is_empty() { return Ok(vec![]); @@ -299,6 +317,7 @@ impl FtsIndexExec { let mut all_rows: Vec = Vec::with_capacity(results.len()); let mut all_scores: Vec = Vec::with_capacity(results.len()); let mut all_row_positions: Vec = Vec::with_capacity(results.len()); + let mut all_doc_indices: Vec>> = Vec::with_capacity(results.len()); let mut all_columns: Vec>> = Vec::new(); // Initialize column vectors based on first batch's schema @@ -309,7 +328,9 @@ impl FtsIndexExec { } } - for &(pos, score) in results { + for (pos, doc_index, score) in results { + let pos = *pos; + let score = *score; if let Some(batch_range) = self.find_batch(pos as usize) && let Some(stored) = self.batch_store.get(batch_range.batch_id) { @@ -328,6 +349,7 @@ impl FtsIndexExec { all_rows.push(row_in_batch); all_scores.push(score); all_row_positions.push(pos); + all_doc_indices.push(doc_index.clone()); } } @@ -352,7 +374,7 @@ impl FtsIndexExec { // NULL result excludes the row, matching SQL). When a predicate exists, // query_index deliberately avoids pushing the limit into the index so // this remains an exact prefilter, not a lossy post-filter. - let (final_columns, all_scores, all_row_positions) = + let (final_columns, all_scores, all_row_positions, all_doc_indices) = if let Some(ref predicate) = self.filter { let Some(first) = self.batch_store.get(0) else { return Ok(vec![]); @@ -384,13 +406,34 @@ impl FtsIndexExec { .zip(mask.iter()) .filter_map(|(p, keep)| keep.unwrap_or(false).then_some(*p)) .collect(); - (filtered_columns, filtered_scores, filtered_positions) + let filtered_doc_indices: Vec>> = all_doc_indices + .iter() + .zip(mask.iter()) + .filter(|(_, keep)| keep.unwrap_or(false)) + .map(|(index, _)| index.clone()) + .collect(); + ( + filtered_columns, + filtered_scores, + filtered_positions, + filtered_doc_indices, + ) } else { - (final_columns, all_scores, all_row_positions) + ( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + ) }; - let (mut final_columns, mut all_scores, mut all_row_positions) = - self.filter_to_newest_pk(final_columns, all_scores, all_row_positions)?; + let (mut final_columns, mut all_scores, mut all_row_positions, mut all_doc_indices) = self + .filter_to_newest_pk( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )?; if all_scores.is_empty() { return Ok(vec![]); @@ -405,6 +448,26 @@ impl FtsIndexExec { .collect(); all_scores.truncate(limit); all_row_positions.truncate(limit); + all_doc_indices.truncate(limit); + } + + if self.with_doc_index { + let mut builder = ListBuilder::new(UInt32Builder::new()).with_field(Field::new( + "item", + DataType::UInt32, + false, + )); + for doc_index in all_doc_indices { + let doc_index = doc_index.ok_or_else(|| { + datafusion::error::DataFusionError::Internal( + "element-document FTS result is missing its document coordinate" + .to_string(), + ) + })?; + builder.values().append_slice(&doc_index); + builder.append(true); + } + final_columns.push(Arc::new(builder.finish())); } // Add score column @@ -416,6 +479,9 @@ impl FtsIndexExec { .iter() .map(|&i| final_columns[i].clone()) .collect(); + if self.with_doc_index { + projected.push(final_columns[final_columns.len() - 2].clone()); + } // Always include score as last column projected.push(final_columns.last().unwrap().clone()); projected @@ -437,21 +503,47 @@ impl FtsIndexExec { final_columns: Vec>, all_scores: Vec, all_row_positions: Vec, + all_doc_indices: Vec>>, ) -> DataFusionResult { let Some(pk_columns) = &self.pk_columns else { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); }; if pk_columns.is_empty() || all_scores.is_empty() { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); } - let Some(max_visible_row) = self.max_visible_row else { - return Ok((final_columns, all_scores, all_row_positions)); + let Some(max_readable_row) = self.max_readable_row else { + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); }; if self.indexes.has_pk_index() && !self.indexes.pk_has_overrides() { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); } let Some(first) = self.batch_store.get(0) else { - return Ok((final_columns, all_scores, all_row_positions)); + return Ok(( + final_columns, + all_scores, + all_row_positions, + all_doc_indices, + )); }; let newest_positions = if self.indexes.has_pk_index() { None @@ -459,8 +551,8 @@ impl FtsIndexExec { Some(newest_pk_positions( &self.batch_store, pk_columns, - self.max_visible_batch_position, - max_visible_row, + self.readable_count, + max_readable_row, )?) }; @@ -476,7 +568,7 @@ impl FtsIndexExec { .map(|&col| ScalarValue::try_from_array(data_batch.column(col), row)) .collect::>()?; self.indexes - .pk_is_newest(&values, all_row_positions[row], max_visible_row) + .pk_is_newest(&values, all_row_positions[row], max_readable_row) } }) }) @@ -498,8 +590,18 @@ impl FtsIndexExec { .zip(keep.iter()) .filter_map(|(p, keep)| keep.then_some(p)) .collect(); + let filtered_doc_indices = all_doc_indices + .into_iter() + .zip(keep.iter()) + .filter_map(|(index, keep)| keep.then_some(index)) + .collect(); - Ok((filtered_columns, filtered_scores, filtered_positions)) + Ok(( + filtered_columns, + filtered_scores, + filtered_positions, + filtered_doc_indices, + )) } } @@ -529,10 +631,6 @@ impl ExecutionPlan for FtsIndexExec { "FtsIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -565,7 +663,7 @@ impl ExecutionPlan for FtsIndexExec { let mut visible_results = self.filter_by_visibility(results); // Sort by score descending (best matches first) - visible_results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + visible_results.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal)); // Materialize the rows (preserving sort order) let batches = self.materialize_rows_sorted(&visible_results)?; @@ -578,12 +676,12 @@ impl ExecutionPlan for FtsIndexExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -602,9 +700,13 @@ impl ExecutionPlan for FtsIndexExec { #[cfg(test)] mod tests { use super::*; - use arrow_array::{Int32Array, StringArray}; + use arrow::array::AsArray; + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{Array, Int32Array, StringArray}; use arrow_schema::{DataType, Field, Schema}; use futures::TryStreamExt; + use lance_index::scalar::InvertedIndexParams; + use lance_index::scalar::inverted::{DOC_INDEX_COL, DocumentGranularity}; fn create_test_schema() -> Arc { Arc::new(Schema::new(vec![ @@ -646,7 +748,7 @@ mod tests { let query = FtsQuery::match_query("text", "hello"); - let exec = FtsIndexExec::new(batch_store, indexes, query, 0, None, schema, false).unwrap(); + let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap(); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -661,6 +763,67 @@ mod tests { assert!(result_schema.field_with_name(SCORE_COLUMN).is_ok()); } + #[tokio::test] + async fn test_element_document_fts_index_search() { + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("alpha beta"); + tags.values().append_value("beta gamma"); + tags.append(true); + tags.values().append_value("beta"); + tags.append(true); + let tags = tags.finish(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("tags", tags.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0, 1])), Arc::new(tags)], + ) + .unwrap(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + let mut registry = IndexStore::new(); + registry + .add_fts_with_params( + "tags_element_idx".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + registry.insert(&batch, 0).unwrap(); + batch_store.append(batch).unwrap(); + + let exec = FtsIndexExec::new( + batch_store, + Arc::new(registry), + FtsQuery::match_query("tags", "beta") + .with_document_granularity(DocumentGranularity::ListElement), + 1, + None, + schema, + false, + ) + .unwrap(); + let stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + let mut hits = Vec::new(); + for batch in batches { + let ids = batch["id"].as_primitive::(); + let coordinates = batch[DOC_INDEX_COL].as_list::(); + for row in 0..batch.num_rows() { + let coordinate = coordinates + .value(row) + .as_primitive::() + .value(0); + hits.push((ids.value(row), coordinate)); + } + } + hits.sort_unstable(); + assert_eq!(hits, vec![(0, 0), (0, 1), (1, 0)]); + } + #[tokio::test] async fn test_fts_index_visibility() { let schema = create_test_schema(); @@ -682,12 +845,12 @@ mod tests { let query = FtsQuery::match_query("text", "hello"); - // Query with max_visible=0 should only see first batch + // Query with max_readable=0 should only see first batch let exec = FtsIndexExec::new( batch_store.clone(), indexes.clone(), query.clone(), - 0, + 1, None, schema.clone(), false, @@ -701,8 +864,8 @@ mod tests { let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); assert_eq!(total_rows, 2); // "hello" in batch1 docs 0 and 2 - // Query with max_visible=1 should see both batches - let exec = FtsIndexExec::new(batch_store, indexes, query, 1, None, schema, false).unwrap(); + // Query with max_readable=1 should see both batches + let exec = FtsIndexExec::new(batch_store, indexes, query, 2, None, schema, false).unwrap(); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs index c56e960048d..c48a2698518 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/scan.rs @@ -3,7 +3,6 @@ //! MemTableScanExec - Full table scan with MVCC visibility filtering. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -28,15 +27,14 @@ use crate::dataset::mem_wal::write::BatchStore; /// Column name for row address (consistent with base table scanner). pub const ROW_ADDRESS_COLUMN: &str = "_rowaddr"; -/// ExecutionPlan node that scans all visible batches from a MemTable. +/// ExecutionPlan node that scans the readable prefix of a MemTable. /// -/// This node implements visibility filtering, returning only batches -/// where `batch_position <= max_visible_batch_position`. +/// Returns only the batches at `batch_position < readable_count`. /// /// Supports filter pushdown for efficient predicate evaluation during scan. pub struct MemTableScanExec { batch_store: Arc, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, /// Schema of the source data (before projection), used for filter evaluation. @@ -56,10 +54,7 @@ pub struct MemTableScanExec { impl Debug for MemTableScanExec { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("MemTableScanExec") - .field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ) + .field("readable_count", &self.readable_count) .field("projection", &self.projection) .field("with_row_id", &self.with_row_id) .field("with_row_address", &self.with_row_address) @@ -74,20 +69,20 @@ impl MemTableScanExec { /// # Arguments /// /// * `batch_store` - Lock-free batch store containing data - /// * `max_visible_batch_position` - Maximum batch position visible (inclusive) + /// * `readable_count` - Exclusive count of batch positions this scan may read /// * `projection` - Optional column indices to project /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested) /// * `with_row_id` - Whether to include _rowid column (row position) pub fn new( batch_store: Arc, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, with_row_id: bool, ) -> Self { Self::with_filter( batch_store, - max_visible_batch_position, + readable_count, projection, output_schema.clone(), output_schema, @@ -103,7 +98,7 @@ impl MemTableScanExec { /// # Arguments /// /// * `batch_store` - Lock-free batch store containing data - /// * `max_visible_batch_position` - Maximum batch position visible (inclusive) + /// * `readable_count` - Exclusive count of batch positions this scan may read /// * `projection` - Optional column indices to project /// * `output_schema` - Schema after projection (should include _rowid/_rowaddr if requested) /// * `source_schema` - Schema of source data (before projection), used for filter evaluation @@ -114,7 +109,7 @@ impl MemTableScanExec { #[allow(clippy::too_many_arguments)] pub fn with_filter( batch_store: Arc, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, source_schema: SchemaRef, @@ -132,7 +127,7 @@ impl MemTableScanExec { Self { batch_store, - max_visible_batch_position, + readable_count, projection, output_schema, source_schema, @@ -194,10 +189,6 @@ impl ExecutionPlan for MemTableScanExec { "MemTableScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -226,7 +217,7 @@ impl ExecutionPlan for MemTableScanExec { // Get visible batches with their row offsets let batches_with_offsets = self .batch_store - .visible_batches_with_offsets(self.max_visible_batch_position); + .visible_batches_with_offsets(self.readable_count); let projection = self.projection.clone(); let schema = self.output_schema.clone(); @@ -339,14 +330,14 @@ impl ExecutionPlan for MemTableScanExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { // Report statistics as Absent to avoid DataFusion analysis bugs // with selectivity calculation on in-memory tables. - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: Precision::Absent, total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { @@ -398,8 +389,8 @@ mod tests { let batch = create_test_batch(&schema, 0, 10); batch_store.append(batch).unwrap(); - // Batch is at position 0, max_visible=0 means position 0 is visible - let exec = MemTableScanExec::new(batch_store, 0, None, schema, false); + // Batch is at position 0, max_readable=0 means position 0 is visible + let exec = MemTableScanExec::new(batch_store, 1, None, schema, false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -425,8 +416,8 @@ mod tests { .append(create_test_batch(&schema, 20, 10)) .unwrap(); - // max_visible_batch_position=1 means positions 0 and 1 are visible (2 batches) - let exec = MemTableScanExec::new(batch_store.clone(), 1, None, schema.clone(), false); + // readable_count=1 means positions 0 and 1 are visible (2 batches) + let exec = MemTableScanExec::new(batch_store.clone(), 2, None, schema.clone(), false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); let batches: Vec = stream.try_collect().await.unwrap(); @@ -447,7 +438,7 @@ mod tests { // Project only "id" column (index 0) let projected_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); - let exec = MemTableScanExec::new(batch_store, 0, Some(vec![0]), projected_schema, false); + let exec = MemTableScanExec::new(batch_store, 1, Some(vec![0]), projected_schema, false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -463,8 +454,8 @@ mod tests { let schema = create_test_schema(); let batch_store = Arc::new(BatchStore::with_capacity(100)); - // Empty store with max_visible=0 should return no batches - let exec = MemTableScanExec::new(batch_store, 0, None, schema, false); + // Empty store with max_readable=0 should return no batches + let exec = MemTableScanExec::new(batch_store, 1, None, schema, false); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); @@ -485,8 +476,8 @@ mod tests { .append(create_test_batch(&schema, 10, 20)) .unwrap(); - // max_visible=1 means positions 0 and 1 are visible - let exec = MemTableScanExec::new(batch_store, 1, None, schema, false); + // max_readable=1 means positions 0 and 1 are visible + let exec = MemTableScanExec::new(batch_store, 2, None, schema, false); let stats = exec.partition_statistics(None).unwrap(); // Statistics are Absent to avoid DataFusion analysis bugs @@ -513,7 +504,7 @@ mod tests { Field::new("_rowid", DataType::UInt64, true), ])); - let exec = MemTableScanExec::new(batch_store, 1, None, schema_with_rowid, true); + let exec = MemTableScanExec::new(batch_store, 2, None, schema_with_rowid, true); let ctx = Arc::new(TaskContext::default()); let stream = exec.execute(0, ctx).unwrap(); diff --git a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs index c3453db68a2..2bb579ded21 100644 --- a/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs +++ b/rust/lance/src/dataset/mem_wal/memtable/scanner/exec/vector.rs @@ -3,7 +3,6 @@ //! VectorIndexExec - HNSW vector search with MVCC visibility. -use std::any::Any; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -35,7 +34,7 @@ pub struct VectorIndexExec { batch_store: Arc, indexes: Arc, query: VectorQuery, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, output_schema: SchemaRef, properties: Arc, @@ -59,10 +58,7 @@ impl Debug for VectorIndexExec { if let Some(metric) = &self.query.distance_type { debug.field("distance_type", metric); } - debug.field( - "max_visible_batch_position", - &self.max_visible_batch_position, - ); + debug.field("readable_count", &self.readable_count); debug.field("with_row_id", &self.with_row_id); debug.finish() } @@ -76,7 +72,7 @@ impl VectorIndexExec { /// * `batch_store` - Lock-free batch store containing data /// * `indexes` - Index registry with HNSW vector indexes /// * `query` - Vector query parameters - /// * `max_visible_batch_position` - MVCC visibility sequence number + /// * `readable_count` - Exclusive count of batch positions this scan may read /// * `projection` - Optional column indices to project /// * `base_schema` - Schema after projection (will add _distance column, and _rowid if with_row_id) /// * `with_row_id` - Whether to include _rowid column (row position) @@ -84,7 +80,7 @@ impl VectorIndexExec { batch_store: Arc, indexes: Arc, query: VectorQuery, - max_visible_batch_position: usize, + readable_count: usize, projection: Option>, base_schema: SchemaRef, with_row_id: bool, @@ -120,7 +116,7 @@ impl VectorIndexExec { batch_store, indexes, query, - max_visible_batch_position, + readable_count, projection, output_schema, properties, @@ -129,24 +125,22 @@ impl VectorIndexExec { }) } - /// Compute the maximum visible row position based on max_visible_batch_position. - /// - /// Returns the last row position that is visible at the given max_visible_batch_position, - /// or None if no batches are visible. - fn compute_max_visible_row(&self) -> Option { - let mut max_visible_row_exclusive: u64 = 0; + /// Last row position within `readable_count`, or None if nothing is + /// readable. + fn compute_max_readable_row(&self) -> Option { + let mut max_readable_row_exclusive: u64 = 0; let mut current_row: u64 = 0; for (batch_position, stored_batch) in self.batch_store.iter().enumerate() { let batch_end = current_row + stored_batch.num_rows as u64; - if batch_position <= self.max_visible_batch_position { - max_visible_row_exclusive = batch_end; + if batch_position < self.readable_count { + max_readable_row_exclusive = batch_end; } current_row = batch_end; } - if max_visible_row_exclusive > 0 { - Some(max_visible_row_exclusive - 1) + if max_readable_row_exclusive > 0 { + Some(max_readable_row_exclusive - 1) } else { None } @@ -161,7 +155,7 @@ impl VectorIndexExec { return Ok(vec![]); }; - let Some(max_visible_row) = self.compute_max_visible_row() else { + let Some(max_readable_row) = self.compute_max_readable_row() else { return Ok(vec![]); }; @@ -181,7 +175,7 @@ impl VectorIndexExec { })? }; - let mut results = index.search(&fsl, self.query.k, self.query.ef, max_visible_row)?; + let mut results = index.search(&fsl, self.query.k, self.query.ef, max_readable_row)?; if self.query.distance_lower_bound.is_some() || self.query.distance_upper_bound.is_some() { results.retain(|&(dist, _)| { @@ -310,10 +304,6 @@ impl ExecutionPlan for VectorIndexExec { "VectorIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -355,12 +345,12 @@ impl ExecutionPlan for VectorIndexExec { ))) } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact(self.query.k), total_byte_size: Precision::Absent, column_statistics: vec![], - }) + })) } fn metrics(&self) -> Option { diff --git a/rust/lance/src/dataset/mem_wal/observer.rs b/rust/lance/src/dataset/mem_wal/observer.rs new file mode 100644 index 00000000000..a567e953e8e --- /dev/null +++ b/rust/lance/src/dataset/mem_wal/observer.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Consumer-supplied sink for MemWAL write-path events. + +use std::fmt::Debug; +use std::time::Duration; + +/// Sink for individual write-path events, supplied by the consumer via +/// [`ShardWriterConfig::observer`](super::write::ShardWriterConfig::observer). +/// +/// Cumulative counts stay on +/// [`WriteStats`](super::write::WriteStats), which an embedder polls: a total +/// loses nothing to aggregation. A duration does — an average reconstructed +/// from a running total cannot show a tail — so each flush is reported here as +/// it completes and the consumer decides how to aggregate it. +/// +/// Observers run inline on the flush task. Do the aggregation, not the export. +/// +/// Every method defaults to a no-op, so adding an event is not a breaking +/// change for existing implementors. +pub trait WalObserver: Send + Sync + Debug { + /// A WAL buffer flush landed in object storage. This is the latency a + /// `durable_write` put waits on. + fn on_wal_flush(&self, _duration: Duration, _bytes: usize) {} + + /// A frozen memtable became an L0 SSTable. Orders of magnitude longer + /// than a WAL flush. + fn on_memtable_flush(&self, _duration: Duration, _rows: usize) {} +} diff --git a/rust/lance/src/dataset/mem_wal/scanner.rs b/rust/lance/src/dataset/mem_wal/scanner.rs index f1d84611e04..907250a1cb1 100644 --- a/rust/lance/src/dataset/mem_wal/scanner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner.rs @@ -5,8 +5,8 @@ //! //! This module provides scanners that read from multiple data sources //! in an LSM tree architecture: -//! - Base table (merged data) -//! - Flushed MemTables (persisted but not yet merged) +//! - Base table (compacted data) +//! - SSTables (persisted but not yet compacted) //! - Active MemTable (in-memory buffer) //! //! The scanner handles deduplication by primary key, keeping the newest @@ -41,11 +41,11 @@ mod builder; mod collector; mod data_source; pub mod exec; -pub(crate) mod flushed_cache; mod fts_search; mod planner; mod point_lookup; mod projection; +pub(crate) mod sstable_cache; mod vector_search; pub use block_list::write_pk_sidecar; @@ -53,13 +53,11 @@ pub use builder::LsmScanner; pub use collector::{ ActiveMemTableRef, InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector, }; -pub use data_source::{ - FlushedGeneration, FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot, -}; -pub use flushed_cache::{DatasetCache, FlushedMemTableCache, GenerationWarmer}; +pub use data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration, ShardSnapshot, SsTable}; pub use fts_search::{LsmFtsSearchPlanner, SCORE_COLUMN}; pub use point_lookup::LsmPointLookupPlanner; pub use projection::DISTANCE_COLUMN; +pub use sstable_cache::{DatasetCache, SsTableCache, SsTableWarmer}; pub use vector_search::LsmVectorSearchPlanner; /// Parse a SQL filter expression against a MemWAL source schema. diff --git a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs index 69d16930888..91fdda8c65f 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/block_list.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/block_list.rs @@ -5,7 +5,7 @@ //! //! A generation's membership is a [`GenMembership`]: in-memory generations //! (active / frozen) are probed by value against their maintained primary-key -//! index (no per-query set), while flushed generations are probed against their +//! index (no per-query set), while SSTables are probed against their //! standalone on-disk PK BTree (the sidecar written at flush, opened by path). //! Probing is batched — [`GenMembership::contains_keys`] tests a whole batch of //! keys per generation in one pass. Each source gets a `Vec` of @@ -32,11 +32,12 @@ use lance_index::scalar::{ use uuid::Uuid; use super::data_source::{FreshTierWatermark, LsmDataSource, LsmGeneration}; -use super::flushed_cache::{DatasetCache, open_flushed_dataset}; +use super::sstable_cache::{DatasetCache, open_sstable}; use crate::dataset::mem_wal::index::encode_pk_tuple; use crate::dataset::mem_wal::util::PK_INDEX_DIR; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Default-plugin registry, used only to load the standalone PK BTree by its /// `BTreeIndexDetails` type. Built once. @@ -54,7 +55,7 @@ pub enum GenMembership { /// Inclusive visible row watermark; `None` when no rows are visible. max_visible_row: Option, }, - /// Probe the flushed generation's standalone on-disk PK BTree. + /// Probe the SSTable's standalone on-disk PK BTree. OnDisk(Arc), } @@ -110,7 +111,7 @@ impl GenMembership { } /// Whether this generation has no (visible) membership — used to skip adding - /// an empty blocked set. A flushed generation always has rows (flush rejects + /// an empty blocked set. An SSTable always has rows (flush rejects /// an empty memtable), so it is never empty. fn is_empty(&self) -> bool { match self { @@ -157,7 +158,8 @@ type ShardGenSets = HashMap>; pub async fn compute_source_block_lists( sources: &[LsmDataSource], session: Option<&Arc>, - flushed_cache: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, + sstable_cache: Option<&Arc>, ) -> Result { // Membership per non-base source, grouped by shard (generations are // per-shard, so supersession is within-shard only). @@ -165,7 +167,7 @@ pub async fn compute_source_block_lists( let mut has_base = false; // Flushed PK-BTree opens are cold S3 reads; overlap them with // `try_join_all`. Order is irrelevant — gens are sorted per-shard below. - let mut flushed_loads = Vec::new(); + let mut sstable_loads = Vec::new(); for source in sources { match source { LsmDataSource::BaseTable { .. } => has_base = true, @@ -182,18 +184,18 @@ pub async fn compute_source_block_lists( .or_default() .push((*generation, membership)); } - LsmDataSource::FlushedMemTable { + LsmDataSource::SsTable { path, shard_id, generation, .. - } => flushed_loads.push(async move { - let index = open_pk_index(path, session, flushed_cache).await?; + } => sstable_loads.push(async move { + let index = open_pk_index(path, session, store_params, sstable_cache).await?; Ok::<_, Error>((*shard_id, *generation, GenMembership::OnDisk(index))) }), } } - for (shard_id, generation, membership) in futures::future::try_join_all(flushed_loads).await? { + for (shard_id, generation, membership) in futures::future::try_join_all(sstable_loads).await? { by_shard .entry(shard_id) .or_default() @@ -225,7 +227,7 @@ pub async fn compute_source_block_lists( /// The fresh-tier block-list: one [`GenMembership`] per generation that shadows /// the base table — active + frozen memtables (probed against their index) and -/// flushed generations (probed against their on-disk PK BTree). A base/external +/// SSTables (probed against their on-disk PK BTree). A base/external /// reader can test any PK against these (via [`GenMembership::contains`]) to /// decide whether the fresh tier shadows it. The base source, if present, is /// skipped (it is what gets shadowed). @@ -238,14 +240,15 @@ pub async fn compute_source_block_lists( pub async fn fresh_tier_block_list( sources: &[LsmDataSource], session: Option<&Arc>, - flushed_cache: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, + sstable_cache: Option<&Arc>, watermarks: Option<&HashMap>, ) -> Result> { // Membership per source, in source order (`None` = skipped). Flushed // PK-BTree opens are cold S3 reads, so collect them tagged with their slot // and overlap with `try_join_all` rather than opening one at a time. let mut slots: Vec> = Vec::with_capacity(sources.len()); - let mut flushed_loads = Vec::new(); + let mut sstable_loads = Vec::new(); for source in sources { match source { LsmDataSource::BaseTable { .. } => slots.push(None), @@ -278,7 +281,7 @@ pub async fn fresh_tier_block_list( }; slots.push(membership); } - LsmDataSource::FlushedMemTable { + LsmDataSource::SsTable { path, shard_id, generation, @@ -298,15 +301,16 @@ pub async fn fresh_tier_block_list( } else { let slot = slots.len(); slots.push(None); - flushed_loads.push(async move { - let index = open_pk_index(path, session, flushed_cache).await?; + sstable_loads.push(async move { + let index = + open_pk_index(path, session, store_params, sstable_cache).await?; Ok::<_, Error>((slot, GenMembership::OnDisk(index))) }); } } } } - for (slot, membership) in futures::future::try_join_all(flushed_loads).await? { + for (slot, membership) in futures::future::try_join_all(sstable_loads).await? { slots[slot] = Some(membership); } Ok(slots @@ -324,7 +328,7 @@ fn in_memory_membership( batch_store: &Arc, index_store: &Arc, ) -> GenMembership { - let max_visible_row = batch_store.max_visible_row(index_store.max_visible_batch_position()); + let max_visible_row = batch_store.max_visible_row(index_store.visible_count()); GenMembership::InMemory { index_store: index_store.clone(), max_visible_row, @@ -342,20 +346,21 @@ fn bounded_in_memory_membership( index_store: &Arc, batch_count: u64, ) -> GenMembership { - let max_visible_row = batch_count - .checked_sub(1) - .and_then(|last_batch| batch_store.max_visible_row(last_batch as usize)); + // `batch_count` is already an exclusive count, and so is what + // `max_visible_row` takes, so it passes straight through: no + // count-to-inclusive-position conversion, and `0` yields `None` on its own. + let max_visible_row = batch_store.max_visible_row(batch_count as usize); GenMembership::InMemory { index_store: index_store.clone(), max_visible_row, } } -/// Open the standalone PK BTree at `{flushed gen}/_pk_index` for one flushed -/// generation. Reuses the flushed dataset's (session-configured) object store +/// Open the standalone PK BTree at `{SSTable gen}/_pk_index` for one +/// SSTable. Reuses the SSTable dataset's (session-configured) object store /// and **its index cache**, then loads the sidecar directly by path through the /// BTree plugin — it is not a manifest index. The opened index and its pages -/// are cached in the session's index cache (keyed by the immutable flushed +/// are cached in the session's index cache (keyed by the immutable SSTable /// path), so repeated probes reuse them with no separate cache path and no /// upfront scan; concurrent first-opens may each load before the cache fills. /// A stable cache UUID for a non-manifest index identified only by its path. @@ -379,12 +384,13 @@ fn path_cache_uuid(path: &str) -> Uuid { async fn open_pk_index( path: &str, session: Option<&Arc>, - flushed_cache: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, + sstable_cache: Option<&Arc>, ) -> Result> { - let dataset = open_flushed_dataset(path, session, flushed_cache, None).await?; - // Namespace the session index cache by the (immutable) flushed path so this + let dataset = open_sstable(path, session, store_params, sstable_cache, None).await?; + // Namespace the session index cache by the (immutable) SSTable path so this // sidecar's pages live alongside every other index instead of a bespoke - // cache. `fri_uuid` is None — flushed generations carry no fragment-reuse. + // cache. `fri_uuid` is None — SSTables carry no fragment-reuse. let index_cache = dataset.index_cache.for_index(&path_cache_uuid(path), None); let index_dir = dataset.base.clone().join(PK_INDEX_DIR); let store: Arc = Arc::new(LanceIndexStore::new( @@ -410,13 +416,13 @@ async fn open_pk_index( Ok(index) } -/// Write a flushed generation's standalone PK sidecar at `{uri}/_pk_index` from +/// Write an SSTable's standalone PK sidecar at `{uri}/_pk_index` from /// `batches`, mirroring what flush does in production. `pk_columns` are the /// primary-key column names (field ids are synthesized by position — `insert` /// resolves columns by name). A no-op when no batch carries the PK columns. /// /// Used by Rust scanner tests and by the Python test-support binding to stage -/// faithful flushed generations (a flushed dataset alone, with no sidecar, is +/// faithful SSTables (an SSTable dataset alone, with no sidecar, is /// not a state production ever produces). pub async fn write_pk_sidecar( uri: &str, @@ -534,7 +540,7 @@ mod tests { active_source(shard, 1, &[3]), ]; - let memberships = fresh_tier_block_list(&sources, None, None, None) + let memberships = fresh_tier_block_list(&sources, None, None, None, None) .await .unwrap(); @@ -555,7 +561,7 @@ mod tests { active_source(shard, 2, &[1, 2]), ]; - let blocked = Box::pin(compute_source_block_lists(&sources, None, None)) + let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None)) .await .unwrap(); @@ -591,7 +597,7 @@ mod tests { active_source(Uuid::new_v4(), 1, &[1, 2]), ]; - let blocked = Box::pin(compute_source_block_lists(&sources, None, None)) + let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None)) .await .unwrap(); @@ -619,7 +625,7 @@ mod tests { active_source(b, 2, &[2]), ]; - let blocked = Box::pin(compute_source_block_lists(&sources, None, None)) + let blocked = Box::pin(compute_source_block_lists(&sources, None, None, None)) .await .unwrap(); @@ -670,7 +676,7 @@ mod tests { generation: LsmGeneration::memtable(2), }; - let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None)) + let blocked = Box::pin(compute_source_block_lists(&[g1, g2], None, None, None)) .await .unwrap(); @@ -708,7 +714,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&watermarks)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks)) .await .unwrap(); assert!(blocks(&sets, 1).await); @@ -716,7 +722,7 @@ mod tests { assert!(!blocks(&sets, 3).await); // No watermark → live tier: all three are members. - let sets = fresh_tier_block_list(&sources, None, None, None) + let sets = fresh_tier_block_list(&sources, None, None, None, None) .await .unwrap(); for id in [1, 2, 3] { @@ -750,7 +756,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&watermarks)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&watermarks)) .await .unwrap(); assert!(blocks(&sets, 1).await); // gen 1, whole @@ -760,32 +766,32 @@ mod tests { assert!(!blocks(&sets, 100).await); // gen 3 — after the snapshot } - /// A flushed generation at or above the active generation was produced by a + /// An SSTable at or above the active generation was produced by a /// flush after the snapshot and is excluded; one strictly below it is /// immutable and included. #[tokio::test] - async fn fresh_tier_watermark_excludes_flushed_at_or_above_active() { + async fn fresh_tier_watermark_excludes_sstables_at_or_above_active() { use crate::dataset::mem_wal::scanner::data_source::FreshTierWatermark; use crate::dataset::{Dataset, WriteParams}; use arrow_array::RecordBatchIterator; use std::collections::HashMap; - // A flushed generation 2 holding pk=5, staged as a flushed dataset with + // An SSTable 2 holding pk=5, staged as an SSTable dataset with // its standalone PK sidecar (what the on-disk membership probes). - let flushed_batch = id_batch(&[5]); - let schema = flushed_batch.schema(); + let sstable_batch = id_batch(&[5]); + let schema = sstable_batch.schema(); let tmp = tempfile::tempdir().unwrap(); let path = format!("{}/gen2", tmp.path().to_str().unwrap()); - let reader = RecordBatchIterator::new(vec![Ok(flushed_batch.clone())], schema.clone()); + let reader = RecordBatchIterator::new(vec![Ok(sstable_batch.clone())], schema.clone()); Dataset::write(reader, &path, Some(WriteParams::default())) .await .unwrap(); - write_pk_sidecar(&path, &[flushed_batch], &["id"]) + write_pk_sidecar(&path, &[sstable_batch], &["id"]) .await .unwrap(); let shard = Uuid::new_v4(); - let sources = vec![LsmDataSource::FlushedMemTable { + let sources = vec![LsmDataSource::SsTable { path, shard_id: shard, generation: LsmGeneration::memtable(2), @@ -801,7 +807,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&at)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&at)) .await .unwrap(); assert!(!blocks(&sets, 5).await); @@ -816,7 +822,7 @@ mod tests { )] .into_iter() .collect(); - let sets = fresh_tier_block_list(&sources, None, None, Some(&above)) + let sets = fresh_tier_block_list(&sources, None, None, None, Some(&above)) .await .unwrap(); assert!(blocks(&sets, 5).await); diff --git a/rust/lance/src/dataset/mem_wal/scanner/builder.rs b/rust/lance/src/dataset/mem_wal/scanner/builder.rs index 2947ef1464f..9da723886f1 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/builder.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/builder.rs @@ -3,8 +3,8 @@ //! LSM Scanner builder. -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use arrow_array::builder::{FixedSizeListBuilder, Float32Builder}; @@ -19,18 +19,21 @@ use datafusion::physical_plan::{ExecutionPlan, SendableRecordBatchStream}; use datafusion::prelude::{Expr, SessionContext}; use futures::TryStreamExt; use lance_core::{Error, Result, is_system_column}; +use lance_datafusion::expr::safe_coerce_scalar; use lance_index::scalar::FullTextSearchQuery; use lance_linalg::distance::DistanceType; use uuid::Uuid; use super::collector::{InMemoryMemTableRef, InMemoryMemTables, LsmDataSourceCollector}; use super::data_source::{FreshTierWatermark, ShardSnapshot}; -use super::flushed_cache::{DatasetCache, GenerationWarmer}; use super::planner::LsmScanPlanner; use super::point_lookup::LsmPointLookupPlanner; use super::projection::validate_projection_names; +use super::sstable_cache::{DatasetCache, SsTableWarmer}; use crate::dataset::Dataset; +use crate::dataset::mem_wal::util::derived_store_params; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Vector (KNN) search state, set by [`LsmScanner::nearest`] and friends. Mirrors /// the subset of `lance::dataset::scanner::Query` the LSM vector planner honors. @@ -54,9 +57,15 @@ struct LsmVectorQuery { /// If `filter` is a point-lookup shape on `pk_col` — `pk = lit` (either /// operand order) or `pk IN (lit, …)` — return the literal key values. Any /// other shape returns `None`, so the scanner falls through to the general -/// scan plan. Type coercion is left to the lookup path (an exact-type literal -/// takes the fast BTree path; a coercible one falls back internally). -fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option> { +/// scan plan. `IN` keys are normalized to `pk_type` only for deduplication; +/// the first original literal is retained so the lookup path still chooses +/// between its exact-type fast path and coercing fallback. A literal that +/// cannot be normalized routes the expression through the general scan. +fn extract_pk_point_keys( + filter: &Expr, + pk_col: &str, + pk_type: &DataType, +) -> Option> { match filter { Expr::BinaryExpr(b) if matches!(b.op, Operator::Eq) => { match (b.left.as_ref(), b.right.as_ref()) { @@ -77,11 +86,15 @@ fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option return None; } let mut vals = Vec::with_capacity(in_list.list.len()); + let mut seen = HashSet::with_capacity(in_list.list.len()); for e in &in_list.list { let Expr::Literal(lit, _) = e else { return None; // a non-literal IN element → not a point lookup }; - vals.push(lit.clone()); + let identity = safe_coerce_scalar(lit, pk_type)?; + if seen.insert(identity) { + vals.push(lit.clone()); + } } (!vals.is_empty()).then_some(vals) } @@ -90,7 +103,7 @@ fn extract_pk_point_keys(filter: &Expr, pk_col: &str) -> Option } /// Either a base Lance table, or an explicit base path used to resolve -/// flushed-generation directories when no base dataset is configured. +/// SSTable directories when no base dataset is configured. enum BaseSource { Table(Arc), PathOnly(String), @@ -150,12 +163,12 @@ fn key_to_fsl(key: &dyn Array, dim: i32) -> Result { Ok(builder.finish()) } -/// Scanner for LSM tree data spanning base table, flushed MemTables, and active MemTable. +/// Scanner for LSM tree data spanning base table, SSTables, and active MemTable. /// /// This scanner provides a unified interface for querying data across multiple /// LSM tree levels: -/// - Base table (merged data, generation = 0) -/// - Flushed MemTables (persisted but not yet merged, generation = 1, 2, ...) +/// - Base table (compacted data, generation = 0) +/// - SSTables (persisted but not yet compacted, generation = 1, 2, ...) /// - Active MemTable (in-memory buffer, highest generation) /// /// The scanner automatically handles deduplication by primary key, keeping @@ -206,15 +219,17 @@ pub struct LsmScanner { // Primary key columns (required for deduplication) pk_columns: Vec, - /// Session threaded into flushed-generation opens so the first open of - /// each generation populates the shared index / file-metadata caches. - /// Defaults to the base table's session when one is present. + /// Session for opening SSTables (shares the base's caches). + /// Defaults to the base table's session. session: Option>, - /// Cache of opened flushed-generation datasets. When set, repeated + /// Store params for opening SSTables, reusing the base dataset's + /// store. Defaults to the base table's params. + store_params: Option, + /// Cache of opened SSTable datasets. When set, repeated /// queries against the same generation skip the manifest read entirely. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Over-fetch multiple for block-listed sources in search plans /// (see [`super::LsmFtsSearchPlanner::with_overfetch_factor`]). overfetch_factor: Option, @@ -225,7 +240,7 @@ impl LsmScanner { /// /// # Arguments /// - /// * `base_table` - The base Lance table (merged data) + /// * `base_table` - The base Lance table (compacted data) /// * `shard_snapshots` - Snapshots of shard states from MemWAL index /// * `pk_columns` - Primary key column names for deduplication pub fn new( @@ -239,6 +254,10 @@ impl LsmScanner { // the shared index / metadata caches without extra wiring. An // explicit `with_session` still overrides this. let session = Some(base_table.session()); + // The scanner only ever opens SSTables with these — the base + // table is already open and handed in — so they must not carry a + // path-bound store binding. + let store_params = base_table.store_params().map(derived_store_params); Self { base: BaseSource::Table(base_table), schema: Arc::new(arrow_schema), @@ -254,25 +273,26 @@ impl LsmScanner { with_memtable_gen: false, pk_columns, session, - flushed_cache: None, + store_params, + sstable_cache: None, warmer: None, overfetch_factor: None, } } /// Create a scanner that reads only the fresh tier (active memtable and - /// flushed generations) without including a base Lance table. + /// SSTables) without including a base Lance table. /// /// This is useful when the caller owns the base read path separately and - /// only needs the WAL's contribution: active memtable ∪ L0 flushed - /// generations. Deduplication semantics are unchanged — newer generations + /// only needs the WAL's contribution: active memtable ∪ L0 SSTables. + /// Deduplication semantics are unchanged — newer generations /// still win on PK conflicts. /// /// # Arguments /// /// * `schema` - Schema used for projection, filter parsing, and empty plans. - /// Should match the schema flushed generations were written with. - /// * `base_path` - Table-root URI used to resolve relative flushed paths. + /// Should match the schema SSTables were written with. + /// * `base_path` - Table-root URI used to resolve relative SSTable paths. /// * `shard_snapshots` - Snapshots of shard states from MemWAL index. /// * `pk_columns` - Primary key column names for deduplication. pub fn without_base_table( @@ -296,7 +316,8 @@ impl LsmScanner { with_memtable_gen: false, pk_columns, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, overfetch_factor: None, } @@ -331,33 +352,40 @@ impl LsmScanner { self } - /// Thread an existing session into flushed-generation opens. - /// - /// The first open of each flushed generation then populates the shared - /// index / file-metadata caches, so later queries skip re-decoding them. - /// When a base table is configured this defaults to its session; call - /// this to override (e.g. on a fresh-tier-only scanner that owns its own - /// long-lived session). + /// Set the session used to open SSTables. Defaults to the base + /// table's; set explicitly on a fresh-tier-only scanner (no base table). pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets. + /// Set the store params used to open SSTables. Defaults to the + /// base table's; set explicitly on a fresh-tier-only scanner (no base table). + /// + /// Pass the params the *base* was opened with. As in [`Self::new`], they are + /// adapted for generation URIs: a path-bound `object_store` binding would + /// redirect every generation open at the base table itself, so it is dropped + /// while storage options, wrapper, and credentials carry over. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(derived_store_params(&store_params)); + self + } + + /// Inject a cache of opened SSTable datasets. /// /// With a cache, repeated queries against the same generation become a /// pure `Arc::clone` with no manifest read or object-store I/O. The cache /// is owned and sized by the caller (any [`DatasetCache`] impl, e.g. - /// [`FlushedMemTableCache`](super::FlushedMemTableCache)); not set by + /// [`SsTableCache`](super::SsTableCache)); not set by /// default, so behavior is unchanged unless opted in. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. Not set by + /// Inject the warmer fired on first open of an SSTable. Not set by /// default, so behavior is unchanged unless opted in. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -419,7 +447,7 @@ impl LsmScanner { } /// Find the `k` nearest neighbors of `key` in `column`. Routes `create_plan` - /// through the LSM vector planner (base ∪ flushed ∪ in-memory). Mirrors + /// through the LSM vector planner (base ∪ SSTables ∪ in-memory). Mirrors /// [`crate::dataset::scanner::Scanner::nearest`]; the LSM path supports a /// single Float32 query vector. When combined with an offset, the LSM path /// fetches `k + offset` per source before applying the final page. Tune with @@ -537,7 +565,7 @@ impl LsmScanner { Arc::new(GlobalLimitExec::new(plan, skip, self.limit)) } - /// Vector (KNN) search across base ∪ flushed ∪ in-memory, via the LSM vector + /// Vector (KNN) search across base ∪ SSTables ∪ in-memory, via the LSM vector /// planner. Honors the builder filter as a prefilter. async fn plan_vector(&self) -> Result> { let nearest = self @@ -564,8 +592,11 @@ impl LsmScanner { if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); @@ -588,7 +619,7 @@ impl LsmScanner { Ok(self.apply_limit_offset(plan)) } - /// Full-text search across base ∪ flushed ∪ in-memory, via the LSM FTS + /// Full-text search across base ∪ SSTables ∪ in-memory, via the LSM FTS /// planner. Query/scanner limits bound per-source fetches when present; /// otherwise the search remains unbounded and any offset is applied above. async fn plan_fts(&self) -> Result> { @@ -609,9 +640,6 @@ impl LsmScanner { ) })?; let base_schema = self.schema(); - base_schema.field_with_name(&column).map_err(|_| { - Error::invalid_input(format!("Column '{}' not found in schema", column)) - })?; let query_limit = query .limit .map(|limit| { @@ -640,8 +668,11 @@ impl LsmScanner { if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); @@ -660,7 +691,7 @@ impl LsmScanner { Ok(self.apply_limit_offset(plan)) } - /// Plain (filter / projection / limit) scan over base ∪ flushed ∪ in-memory. + /// Plain (filter / projection / limit) scan over base ∪ SSTables ∪ in-memory. async fn plan_scan(&self) -> Result> { let collector = self.build_collector(); let base_schema = self.schema(); @@ -678,15 +709,20 @@ impl LsmScanner { && !self.with_row_address && !self.projection_has_system_columns() && let Some(filter) = &self.filter - && let Some(keys) = extract_pk_point_keys(filter, &self.pk_columns[0]) + && let Ok(pk_field) = base_schema.field_with_name(&self.pk_columns[0]) + && let Some(keys) = + extract_pk_point_keys(filter, &self.pk_columns[0], pk_field.data_type()) { let mut planner = LsmPointLookupPlanner::new(collector, self.pk_columns.clone(), base_schema); if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); @@ -704,15 +740,15 @@ impl LsmScanner { if let Some(session) = &self.session { planner = planner.with_session(session.clone()); } - if let Some(cache) = &self.flushed_cache { - planner = planner.with_flushed_cache(cache.clone()); + if let Some(store_params) = &self.store_params { + planner = planner.with_store_params(store_params.clone()); + } + if let Some(cache) = &self.sstable_cache { + planner = planner.with_sstable_cache(cache.clone()); } if let Some(warmer) = &self.warmer { planner = planner.with_warmer(warmer.clone()); } - if let Some(factor) = self.overfetch_factor { - planner = planner.with_overfetch_factor(factor); - } planner .plan_scan( @@ -727,7 +763,7 @@ impl LsmScanner { } /// Find rows matching a full-text query. Routes `create_plan` through the - /// LSM FTS planner (base ∪ flushed ∪ in-memory), local-scored by BM25 and + /// LSM FTS planner (base ∪ SSTables ∪ in-memory), local-scored by BM25 and /// merged by `_score` DESC. Mirrors /// [`crate::dataset::scanner::Scanner::full_text_search`]: the searched /// column(s) come from the query (set via `FullTextSearchQuery::with_column`); @@ -778,7 +814,7 @@ impl LsmScanner { } /// Test which `pks` have been (re)written in the WAL fresh tier — the active - /// and frozen memtables and flushed generations this scanner spans — i.e. + /// and frozen memtables and SSTables this scanner spans — i.e. /// are shadowed above the base table. `pks` is a batch whose columns include /// the primary-key columns; the returned `Vec` is aligned with its /// rows. Hashing matches the scanner's internal dedup, so the caller never @@ -802,7 +838,8 @@ impl LsmScanner { let memberships = super::block_list::fresh_tier_block_list( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), watermarks, ) .await?; @@ -892,6 +929,12 @@ impl std::fmt::Debug for LsmScanner { #[cfg(test)] mod tests { use super::*; + use arrow_array::{Int32Array, ListArray, StringArray, StructArray, UInt32Array}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + use arrow_schema::{Field, Fields}; + use lance_index::scalar::inverted::{DOC_INDEX_COL, DocumentGranularity, InvertedIndexParams}; + + use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; #[test] fn test_lsm_scanner_builder() { @@ -907,6 +950,22 @@ mod tests { assert!(shard_snapshots.is_empty()); } + #[test] + fn point_lookup_extraction_requires_normalizable_literals() { + use datafusion::prelude::{col, lit}; + + let filters = [ + col("id").in_list(vec![lit(1i32), lit("not an integer")], false), + col("id").in_list(vec![lit(1i32), col("other")], false), + ]; + for filter in filters { + assert!( + extract_pk_point_keys(&filter, "id", &DataType::Int32).is_none(), + "unsupported point-lookup filter must use the scan path: {filter}" + ); + } + } + #[test] fn test_shard_snapshot_construction() { use super::super::data_source::ShardSnapshot; @@ -915,13 +974,13 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_spec_id(1) .with_current_generation(5) - .with_flushed_generation(1, "path/gen_1".to_string()) - .with_flushed_generation(2, "path/gen_2".to_string()); + .with_sstable(1, "path/gen_1".to_string()) + .with_sstable(2, "path/gen_2".to_string()); assert_eq!(snapshot.shard_id, shard_id); assert_eq!(snapshot.spec_id, 1); assert_eq!(snapshot.current_generation, 5); - assert_eq!(snapshot.flushed_generations.len(), 2); + assert_eq!(snapshot.sstables.len(), 2); } #[test] @@ -973,44 +1032,69 @@ mod tests { } #[tokio::test] - async fn invalid_overfetch_factor_is_rejected() { - let shard = Uuid::new_v4(); - let scanner = LsmScanner::without_base_table( + async fn overfetch_factor_only_applies_to_searches() { + // Plain scans refill exactly via LocalLimitExec, so overfetch_factor is + // search-only and must not affect scan planning. + let scan = LsmScanner::without_base_table( pk_schema(), - "memory://t", + "memory://scan", vec![], vec!["id".to_string()], ) - .with_in_memory_memtables( - shard, - InMemoryMemTables { - active: mk_pk_memtable(&[1, 2], 2), - frozen: vec![], - }, + .with_overfetch_factor(0.5) + .try_into_batch() + .await + .unwrap(); + assert_eq!(scan.num_rows(), 0); + + let vector_schema = pk_schema_with(arrow_schema::Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)), + 4, + ), + false, + )); + let vector_search = LsmScanner::without_base_table( + vector_schema, + "memory://vector", + vec![], + vec!["id".to_string()], + ) + .nearest( + "vector", + &arrow_array::Float32Array::from(vec![0.0f32, 0.0, 0.0, 0.0]), + 1, ) + .unwrap() .with_overfetch_factor(0.5); - - let Err(err) = scanner.try_into_stream().await else { - panic!("invalid overfetch factor should fail planning"); + let Err(err) = vector_search.try_into_stream().await else { + panic!("invalid overfetch factor should fail vector search planning"); }; assert!( err.to_string().contains("overfetch_factor"), "unexpected error for invalid overfetch factor: {err}" ); - let empty_scanner = LsmScanner::without_base_table( - pk_schema(), - "memory://empty", + let fts_search = LsmScanner::without_base_table( + pk_schema_with(arrow_schema::Field::new("text", DataType::Utf8, true)), + "memory://fts", vec![], vec!["id".to_string()], ) + .full_text_search( + FullTextSearchQuery::new("lance".to_string()) + .with_column("text".to_string()) + .unwrap(), + ) + .unwrap() .with_overfetch_factor(0.5); - let Err(err) = empty_scanner.try_into_stream().await else { - panic!("invalid overfetch factor should fail even when there are no sources"); + let Err(err) = fts_search.try_into_stream().await else { + panic!("invalid overfetch factor should fail full-text search planning"); }; assert!( err.to_string().contains("overfetch_factor"), - "unexpected error for invalid empty-source overfetch factor: {err}" + "unexpected error for invalid overfetch factor: {err}" ); } @@ -1778,6 +1862,103 @@ mod tests { ); } + #[tokio::test] + async fn full_text_search_supports_nested_list_element_path() { + let doc_fields = Fields::from(vec![Field::new("content", DataType::Utf8, true)]); + let doc_values = StructArray::new( + doc_fields.clone(), + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "alpha"]))], + None, + ); + let doc_item = Arc::new(Field::new("item", DataType::Struct(doc_fields), true)); + let docs = ListArray::new( + doc_item.clone(), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 1, 3])), + Arc::new(doc_values), + None, + ); + let group_fields = Fields::from(vec![Field::new("docs", DataType::List(doc_item), true)]); + let group_values = StructArray::new(group_fields.clone(), vec![Arc::new(docs)], None); + let group_item = Arc::new(Field::new("item", DataType::Struct(group_fields), true)); + let groups = ListArray::new( + group_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2])), + Arc::new(group_values), + None, + ); + let schema = pk_schema_with(Field::new("groups", groups.data_type().clone(), true)); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![7])), Arc::new(groups)], + ) + .unwrap(); + + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + indexes + .add_fts_with_params( + "content_list_element_fts".to_string(), + 1, + "groups.docs.content".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let (batch_position, row_offset, _) = batch_store.append(batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&batch, row_offset, Some(batch_position)) + .unwrap(); + + let scanner = LsmScanner::without_base_table( + schema.clone(), + "memory://nested_fts", + vec![], + vec!["id".to_string()], + ) + .with_in_memory_memtables( + Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(indexes), + schema, + generation: 1, + }, + frozen: vec![], + }, + ) + .project(&["id"]) + .unwrap() + .full_text_search( + FullTextSearchQuery::new("alpha".to_string()) + .with_column("groups.docs.content".to_string()) + .unwrap(), + ) + .unwrap(); + + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let coordinates = batch[DOC_INDEX_COL] + .as_any() + .downcast_ref::() + .unwrap(); + let mut hits = (0..batch.num_rows()) + .map(|row| { + let coordinate = coordinates + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + (ids.value(row), coordinate) + }) + .collect::>(); + hits.sort_unstable(); + assert_eq!(hits, vec![(7, vec![0, 0]), (7, vec![1, 1])]); + } + /// Empty search results must still preserve the execution plan schema. #[tokio::test] async fn try_into_batch_empty_fts_keeps_score_schema() { @@ -1924,6 +2105,48 @@ mod tests { ); assert_eq!(count(plan).await, 2); + // Duplicate literals retain predicate semantics: each matching row is + // emitted once, including when a limit would otherwise hide a distinct + // match behind the duplicate. + let plan = scanner() + .filter_expr(col("id").in_list(vec![lit(1i32), lit(1i32), lit(2i32)], false)) + .limit(Some(2), None) + .unwrap() + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![1, 2]); + + // Distinct literal types can coerce to the same primary-key value and + // must share one deduplication identity before the limit is applied. + let plan = scanner() + .filter_expr(col("id").in_list(vec![lit(1i32), lit(1i64), lit(2i32)], false)) + .limit(Some(2), None) + .unwrap() + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![1, 2]); + + // Coercible literals use the per-key fallback and must have the same + // set semantics as exact-type keys. + let plan = scanner() + .filter_expr(col("id").in_list(vec![lit(1i64), lit(1i64), lit(3i64)], false)) + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![1, 3]); + + // NULL literals cannot match, and duplicate NULLs must not affect the + // non-NULL matches. + let null = lit(ScalarValue::Int32(None)); + let plan = scanner() + .filter_expr(col("id").in_list(vec![null.clone(), lit(2i32), null], false)) + .create_plan() + .await + .unwrap(); + assert_eq!(collect_ids(plan).await, vec![2]); + // A range filter is NOT a point lookup → falls through to the scan path. let plan = scanner() .filter_expr(col("id").gt(lit(2i32))) diff --git a/rust/lance/src/dataset/mem_wal/scanner/collector.rs b/rust/lance/src/dataset/mem_wal/scanner/collector.rs index 6645f159b12..a5d0ecd0b65 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/collector.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/collector.rs @@ -29,6 +29,41 @@ pub struct InMemoryMemTableRef { pub generation: u64, } +impl InMemoryMemTableRef { + /// Row-data bytes: the buffered batches. + /// + /// This is the **flush unit**, not the memtable's footprint — it drives the + /// `max_memtable_size` seal trigger, so it stays a function of the rows in + /// it. Use [`Self::resident_bytes`] to budget memory. + pub fn row_bytes(&self) -> usize { + self.batch_store.row_bytes() + } + + /// Heap held by this memtable's auxiliary lookup structures: its in-memory + /// indexes plus the fixed PK bloom filter. + /// + /// Usually the term that explains an indexed table's footprint — an HNSW + /// index pre-allocates its whole graph on the first insert — so it can dwarf + /// row bytes while [`Self::row_bytes`] reads near zero. + pub fn index_bytes(&self) -> usize { + self.index_store.resident_bytes() + + crate::dataset::mem_wal::memtable::pk_bloom_filter_bytes() + } + + /// Heap the buffered batches keep alive, which is not [`Self::row_bytes`] + /// once any batch is a zero-copy slice: a slice's window is a fraction of + /// the parent buffer it pins. See [`BatchStore::retained_bytes`]. + pub fn retained_row_bytes(&self) -> usize { + self.batch_store.retained_bytes() + } + + /// Total resident heap bytes. This, not [`Self::row_bytes`], is what a + /// memory ceiling must be built on. + pub fn resident_bytes(&self) -> usize { + self.retained_row_bytes() + self.index_bytes() + } +} + /// Back-compat alias; prefer [`InMemoryMemTableRef`]. pub type ActiveMemTableRef = InMemoryMemTableRef; @@ -49,18 +84,18 @@ pub struct InMemoryMemTables { /// /// This collector gathers all data sources that need to be scanned /// for a query, including: -/// - The base table (merged data) — optional; omit for fresh-tier-only scans -/// - Flushed MemTables from each shard +/// - The base table (compacted data) — optional; omit for fresh-tier-only scans +/// - SSTables from each shard /// - In-memory memtables per shard (active + frozen-awaiting-flush) /// /// When the base table is omitted (see [`Self::without_base_table`]), `collect` -/// returns only flushed-generation and active-memtable sources. This is used +/// returns only SSTable and active-memtable sources. This is used /// by callers that own the base read path elsewhere and only need the WAL's -/// fresh tier (active memtable ∪ L0 flushed generations). +/// fresh tier (active memtable ∪ L0 SSTables). pub struct LsmDataSourceCollector { /// Base Lance table (None when scanning only the fresh tier). base_table: Option>, - /// Base path for resolving relative flushed-generation paths. + /// Base path for resolving relative SSTable paths. base_path: String, /// Shard snapshots from MemWAL index. shard_snapshots: Vec, @@ -73,7 +108,7 @@ impl LsmDataSourceCollector { /// /// # Arguments /// - /// * `base_table` - The base Lance table (merged data) + /// * `base_table` - The base Lance table (compacted data) /// * `shard_snapshots` - Snapshots of shard states from MemWAL index pub fn new(base_table: Arc, shard_snapshots: Vec) -> Self { // Use the dataset's URI as base path for resolving relative paths. @@ -89,8 +124,8 @@ impl LsmDataSourceCollector { /// Create a collector without a base table (fresh-tier scan only). /// - /// The collector emits only flushed-generation and active-memtable sources. - /// `base_path` is the table-root URI used to resolve relative flushed paths + /// The collector emits only SSTable and active-memtable sources. + /// `base_path` is the table-root URI used to resolve relative SSTable paths /// (typically the same URI that would have been the base dataset's URI). pub fn without_base_table( base_path: impl Into, @@ -147,16 +182,12 @@ impl LsmDataSourceCollector { &self.in_memory_memtables } - /// Whether the collector has any on-disk source (base table or a flushed - /// generation). The point-lookup fast path uses this to decide, after + /// Whether the collector has any on-disk source (base table or an + /// SSTable). The point-lookup fast path uses this to decide, after /// missing every in-memory memtable, between "definitely absent" (`false`) /// and "must consult disk via the plan path" (`true`). Cheap: no allocation. pub fn has_on_disk_sources(&self) -> bool { - self.base_table.is_some() - || self - .shard_snapshots - .iter() - .any(|s| !s.flushed_generations.is_empty()) + self.base_table.is_some() || self.shard_snapshots.iter().any(|s| !s.sstables.is_empty()) } /// The in-memory memtables (active + frozen across all shards) as @@ -233,10 +264,10 @@ impl LsmDataSourceCollector { /// frozen memtable. During the post-flush grace window a generation is both /// committed to the manifest (a flushed source) and held in memory (an /// in-memory source); it must be served only from memory — which preserves - /// the per-batch boundaries the flushed dataset has lost, so as-of reads + /// the per-batch boundaries the SSTable dataset has lost, so as-of reads /// stay snapshot-bounded — and its on-disk copy skipped to avoid scanning /// the generation twice. See `ShardWriterConfig::frozen_memtable_grace`. - fn flushed_gen_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool { + fn sstable_pinned_in_memory(&self, shard_id: &Uuid, generation: u64) -> bool { self.in_memory_memtables .get(shard_id) .is_some_and(|mems| mems.frozen.iter().any(|f| f.generation == generation)) @@ -246,7 +277,7 @@ impl LsmDataSourceCollector { /// /// Returns sources in a consistent order: /// 1. Base table (gen=0), if configured - /// 2. Flushed MemTables per shard, ordered by generation + /// 2. SSTables per shard, ordered by generation /// 3. In-memory memtables per shard (active + frozen-awaiting-flush) pub fn collect(&self) -> Result> { let mut sources = Vec::new(); @@ -258,15 +289,15 @@ impl LsmDataSourceCollector { } for snapshot in &self.shard_snapshots { - for flushed in &snapshot.flushed_generations { - if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) { + for sstable in &snapshot.sstables { + if self.sstable_pinned_in_memory(&snapshot.shard_id, sstable.generation) { continue; } - let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path); - sources.push(LsmDataSource::FlushedMemTable { + let path = self.resolve_sstable_path(&snapshot.shard_id, &sstable.path); + sources.push(LsmDataSource::SsTable { path, shard_id: snapshot.shard_id, - generation: LsmGeneration::memtable(flushed.generation), + generation: LsmGeneration::memtable(sstable.generation), }); } } @@ -299,15 +330,15 @@ impl LsmDataSourceCollector { continue; } - for flushed in &snapshot.flushed_generations { - if self.flushed_gen_pinned_in_memory(&snapshot.shard_id, flushed.generation) { + for sstable in &snapshot.sstables { + if self.sstable_pinned_in_memory(&snapshot.shard_id, sstable.generation) { continue; } - let path = self.resolve_flushed_path(&snapshot.shard_id, &flushed.path); - sources.push(LsmDataSource::FlushedMemTable { + let path = self.resolve_sstable_path(&snapshot.shard_id, &sstable.path); + sources.push(LsmDataSource::SsTable { path, shard_id: snapshot.shard_id, - generation: LsmGeneration::memtable(flushed.generation), + generation: LsmGeneration::memtable(sstable.generation), }); } } @@ -325,25 +356,21 @@ impl LsmDataSourceCollector { /// Get the total number of data sources. pub fn num_sources(&self) -> usize { - let flushed_count: usize = self - .shard_snapshots - .iter() - .map(|s| s.flushed_generations.len()) - .sum(); + let sstable_count: usize = self.shard_snapshots.iter().map(|s| s.sstables.len()).sum(); let base_count = if self.base_table.is_some() { 1 } else { 0 }; let in_memory_count: usize = self .in_memory_memtables .values() .map(|m| 1 + m.frozen.len()) .sum(); - base_count + flushed_count + in_memory_count + base_count + sstable_count + in_memory_count } - /// Resolve a flushed MemTable path to an absolute path. + /// Resolve an SSTable path to an absolute path. /// - /// Flushed MemTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}` - /// The `folder_name` is what's stored in `FlushedGeneration.path`. - fn resolve_flushed_path(&self, shard_id: &Uuid, folder_name: &str) -> String { + /// SSTables are stored at: `{base_path}/_mem_wal/{shard_id}/{folder_name}` + /// The `folder_name` is what's stored in `SsTable.path`. + fn resolve_sstable_path(&self, shard_id: &Uuid, folder_name: &str) -> String { format!("{}/_mem_wal/{}/{}", self.base_path, shard_id, folder_name) } } @@ -351,7 +378,7 @@ impl LsmDataSourceCollector { #[cfg(test)] mod tests { use super::*; - use crate::dataset::mem_wal::scanner::data_source::FlushedGeneration; + use crate::dataset::mem_wal::scanner::data_source::SsTable; fn create_test_snapshots() -> Vec { let shard_a = Uuid::new_v4(); @@ -362,12 +389,12 @@ mod tests { shard_id: shard_a, spec_id: 1, current_generation: 3, - flushed_generations: vec![ - FlushedGeneration { + sstables: vec![ + SsTable { generation: 1, path: "abc_gen_1".to_string(), }, - FlushedGeneration { + SsTable { generation: 2, path: "def_gen_2".to_string(), }, @@ -377,7 +404,7 @@ mod tests { shard_id: shard_b, spec_id: 1, current_generation: 2, - flushed_generations: vec![FlushedGeneration { + sstables: vec![SsTable { generation: 1, path: "xyz_gen_1".to_string(), }], @@ -390,8 +417,8 @@ mod tests { let snapshots = create_test_snapshots(); // 1 base table + 2 flushed from shard_a + 1 flushed from shard_b = 4 // Using a mock dataset is complex, so we just test the counting logic - assert_eq!(snapshots[0].flushed_generations.len(), 2); - assert_eq!(snapshots[1].flushed_generations.len(), 1); + assert_eq!(snapshots[0].sstables.len(), 2); + assert_eq!(snapshots[1].sstables.len(), 1); } #[test] @@ -466,10 +493,10 @@ mod tests { /// During the post-flush grace window a generation is both committed to the /// manifest (a flushed source) and still pinned in memory (a frozen /// source). The collector must emit it once, from memory — so as-of reads - /// keep batch-resolved membership — and skip the on-disk copy. Flushed - /// generations NOT pinned in memory are still emitted from disk. + /// keep batch-resolved membership — and skip the on-disk copy. SSTables + /// NOT pinned in memory are still emitted from disk. #[test] - fn test_collect_suppresses_flushed_gen_pinned_in_memory() { + fn test_collect_suppresses_sstable_pinned_in_memory() { let shard = Uuid::new_v4(); // Manifest lists gens 1 and 2 as flushed; gen 2 is still pinned in // memory (just flushed, within grace), gen 1 has been swept. @@ -477,12 +504,12 @@ mod tests { shard_id: shard, spec_id: 0, current_generation: 3, - flushed_generations: vec![ - FlushedGeneration { + sstables: vec![ + SsTable { generation: 1, path: "gen_1".to_string(), }, - FlushedGeneration { + SsTable { generation: 2, path: "gen_2".to_string(), }, @@ -498,7 +525,7 @@ mod tests { let sources = collector.collect().unwrap(); // gen 1: on-disk (not pinned). gen 2: in-memory only (pinned, disk // copy suppressed). gen 3: active. No duplicate gen 2. - let flushed: Vec = sources + let sstable_gens: Vec = sources .iter() .filter(|s| !s.is_active_memtable()) .map(|s| s.generation().as_u64()) @@ -508,7 +535,11 @@ mod tests { .filter(|s| s.is_active_memtable()) .map(|s| s.generation().as_u64()) .collect(); - assert_eq!(flushed, vec![1], "only the unpinned flushed gen from disk"); + assert_eq!( + sstable_gens, + vec![1], + "only the unpinned SSTable gen from disk" + ); assert_eq!(in_memory, vec![2, 3], "pinned gen 2 served from memory"); } } diff --git a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs index 0d5f3fdc925..86e6041142b 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/data_source.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/data_source.rs @@ -44,7 +44,7 @@ pub struct FreshTierWatermark { pub struct LsmGeneration(u64); impl LsmGeneration { - /// Generation for the base table (merged data). + /// Generation for the base table (compacted data). pub const BASE_TABLE: Self = Self(0); /// Create a generation for a MemTable. @@ -93,12 +93,12 @@ impl Default for LsmGeneration { } } -/// A flushed generation with its storage path. +/// An SSTable with its storage path. #[derive(Debug, Clone)] -pub struct FlushedGeneration { +pub struct SsTable { /// Generation number. pub generation: u64, - /// Path to the flushed MemTable directory (relative to table root). + /// Path to the SSTable directory (relative to table root). pub path: String, } @@ -114,8 +114,8 @@ pub struct ShardSnapshot { pub spec_id: u32, /// Current generation being written (next flush will be this generation). pub current_generation: u64, - /// List of flushed generations and their paths. - pub flushed_generations: Vec, + /// List of SSTables and their paths. + pub sstables: Vec, } impl ShardSnapshot { @@ -125,7 +125,7 @@ impl ShardSnapshot { shard_id, spec_id: 0, current_generation: 1, - flushed_generations: Vec::new(), + sstables: Vec::new(), } } @@ -141,10 +141,9 @@ impl ShardSnapshot { self } - /// Add a flushed generation. - pub fn with_flushed_generation(mut self, generation: u64, path: String) -> Self { - self.flushed_generations - .push(FlushedGeneration { generation, path }); + /// Add an SSTable. + pub fn with_sstable(mut self, generation: u64, path: String) -> Self { + self.sstables.push(SsTable { generation, path }); self } } @@ -156,9 +155,9 @@ pub enum LsmDataSource { /// The base dataset. dataset: Arc, }, - /// Flushed MemTable stored as Lance table on disk. - FlushedMemTable { - /// Absolute path to the flushed MemTable directory. + /// SSTable stored as Lance table on disk. + SsTable { + /// Absolute path to the SSTable directory. path: String, /// Shard this MemTable belongs to. shard_id: Uuid, @@ -185,7 +184,7 @@ impl LsmDataSource { pub fn generation(&self) -> LsmGeneration { match self { Self::BaseTable { .. } => LsmGeneration::BASE_TABLE, - Self::FlushedMemTable { generation, .. } => *generation, + Self::SsTable { generation, .. } => *generation, Self::ActiveMemTable { generation, .. } => *generation, } } @@ -194,7 +193,7 @@ impl LsmDataSource { pub fn shard_id(&self) -> Option { match self { Self::BaseTable { .. } => None, - Self::FlushedMemTable { shard_id, .. } => Some(*shard_id), + Self::SsTable { shard_id, .. } => Some(*shard_id), Self::ActiveMemTable { shard_id, .. } => Some(*shard_id), } } @@ -213,7 +212,7 @@ impl LsmDataSource { pub fn display_name(&self) -> String { match self { Self::BaseTable { .. } => "base_table".to_string(), - Self::FlushedMemTable { + Self::SsTable { shard_id, generation, .. @@ -279,14 +278,14 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_spec_id(1) .with_current_generation(5) - .with_flushed_generation(1, "abc123_gen_1".to_string()) - .with_flushed_generation(2, "def456_gen_2".to_string()); + .with_sstable(1, "abc123_gen_1".to_string()) + .with_sstable(2, "def456_gen_2".to_string()); assert_eq!(snapshot.shard_id, shard_id); assert_eq!(snapshot.spec_id, 1); assert_eq!(snapshot.current_generation, 5); - assert_eq!(snapshot.flushed_generations.len(), 2); - assert_eq!(snapshot.flushed_generations[0].generation, 1); - assert_eq!(snapshot.flushed_generations[1].generation, 2); + assert_eq!(snapshot.sstables.len(), 2); + assert_eq!(snapshot.sstables[0].generation, 1); + assert_eq!(snapshot.sstables[1].generation, 2); } } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec.rs b/rust/lance/src/dataset/mem_wal/scanner/exec.rs index 1498c5f60ea..9c47c893d8d 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec.rs @@ -10,12 +10,14 @@ //! - [`BloomFilterGuardExec`]: Guards child execution with bloom filter check //! - [`CoalesceFirstExec`]: Returns first non-empty result with short-circuit //! - [`PkBlockFilterExec`]: Drops rows whose PK was superseded by a newer generation (the cross-generation block-list) +//! - [`SchemaRelabelExec`]: Re-labels batches to an exact schema (the logical/storage nullability boundary) mod bloom_guard; mod coalesce_first; mod generation_tag; mod pk; mod pk_block_filter; +mod schema_relabel; pub use bloom_guard::{BloomFilterGuardExec, compute_pk_hash_from_scalars}; pub use coalesce_first::CoalesceFirstExec; @@ -25,3 +27,4 @@ pub use pk::{ validate_pk_types, }; pub use pk_block_filter::PkBlockFilterExec; +pub use schema_relabel::SchemaRelabelExec; diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs index 632b08a753f..e3a710bd367 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/bloom_guard.rs @@ -5,7 +5,6 @@ //! //! Used in point lookup queries to skip generations that definitely don't contain the key. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -134,10 +133,6 @@ impl ExecutionPlan for BloomFilterGuardExec { "BloomFilterGuardExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs index 9e158c86b4a..c212b47c013 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/coalesce_first.rs @@ -5,7 +5,6 @@ //! //! Used in point lookup queries to stop searching after finding the first match. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -111,10 +110,6 @@ impl ExecutionPlan for CoalesceFirstExec { "CoalesceFirstExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs index ba9d565316f..9c1d0060a78 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/generation_tag.rs @@ -3,7 +3,6 @@ //! MemTable generation tagging execution node. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -100,10 +99,6 @@ impl ExecutionPlan for MemtableGenTagExec { "MemtableGenTagExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs index 89dbd7adc61..ef2dc695dfa 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/pk_block_filter.rs @@ -5,7 +5,7 @@ //! //! Drops a row when any newer generation's membership ([`GenMembership`]) //! contains its primary key — in-memory generations probe their PK index by -//! value, flushed generations probe their on-disk PK BTree. Each generation is +//! value, SSTables probe their on-disk PK BTree. Each generation is //! probed once per batch (see the perf note below). Used both as the KNN //! post-filter (vector search, with over-fetch) and the cross-generation scan //! filter (`k = 0`). @@ -22,12 +22,11 @@ //! `BTreeIndex::contains_keys` (one page pass, no per-key `SearchResult` //! allocation); the in-memory arm maps a sync PK lookup over the keys. Probes //! are not disk-bound in steady state: the opened index and its (small, -//! memtable-sized) pages are held by the injected `FlushedMemTableCache` / +//! memtable-sized) pages are held by the injected `SsTableCache` / //! `LanceCache`, so after the first touch every probe is memory-resident. //! Already-blocked rows are dropped from the key set before probing older //! generations, preserving the per-row short-circuit. -use std::any::Any; use std::fmt; use std::pin::Pin; use std::sync::Arc; @@ -109,10 +108,6 @@ impl ExecutionPlan for PkBlockFilterExec { "PkBlockFilterExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.input.schema() } @@ -185,8 +180,8 @@ struct PkBlockFilterStream { warned: bool, } -/// Keep only the rows no newer-gen membership contains. Async because flushed -/// generations are probed against their on-disk PK BTree. +/// Keep only the rows no newer-gen membership contains. Async because SSTables +/// are probed against their on-disk PK BTree. async fn filter_batch(batch: RecordBatch, config: Arc) -> DFResult { let FilterConfig { pk_columns, @@ -322,7 +317,7 @@ mod tests { let (bp, off, _) = store.append(b.clone()).unwrap(); index.insert_with_batch_position(&b, off, Some(bp)).unwrap(); } - let max_visible_row = store.max_visible_row(index.max_visible_batch_position()); + let max_visible_row = store.max_visible_row(index.visible_count()); GenMembership::InMemory { index_store: Arc::new(index), max_visible_row, diff --git a/rust/lance/src/dataset/mem_wal/scanner/exec/schema_relabel.rs b/rust/lance/src/dataset/mem_wal/scanner/exec/schema_relabel.rs new file mode 100644 index 00000000000..88691f99d63 --- /dev/null +++ b/rust/lance/src/dataset/mem_wal/scanner/exec/schema_relabel.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Schema re-labeling execution node. + +use std::fmt; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; + +use arrow_array::{RecordBatch, RecordBatchOptions}; +use arrow_schema::SchemaRef; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::TaskContext; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + SendableRecordBatchStream, +}; +use futures::{Stream, StreamExt}; + +/// Re-labels every batch to an exact target schema, leaving the arrays +/// untouched. `ProjectionExec` cannot: DataFusion derives output nullability +/// from the expressions, not from the schema the planner intended. +/// +/// **Widening** makes a shard's storage schema (see `relax_non_pk_nullability`) +/// agree with the base-table arm before `UnionExec` / `CoalesceFirstExec`. +/// **Narrowing** restores the logical schema at the scan's output boundary and +/// doubles as the tombstone-leak check, since `RecordBatch` validation rejects +/// a null in a non-nullable column. +#[derive(Debug)] +pub struct SchemaRelabelExec { + input: Arc, + schema: SchemaRef, + properties: Arc, +} + +impl SchemaRelabelExec { + /// Wrap `input` so its batches are re-labeled to `schema`: same column + /// count, order, and data types; only names, nullability, and metadata may + /// differ. A mismatch surfaces per batch at execution time, not plan time. + pub fn new(input: Arc, schema: SchemaRef) -> Self { + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + input.output_partitioning().clone(), + input.pipeline_behavior(), + input.boundedness(), + )); + Self { + input, + schema, + properties, + } + } +} + +impl DisplayAs for SchemaRelabelExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + match t { + DisplayFormatType::Default + | DisplayFormatType::Verbose + | DisplayFormatType::TreeRender => { + write!(f, "SchemaRelabelExec") + } + } + } +} + +impl ExecutionPlan for SchemaRelabelExec { + fn name(&self) -> &str { + "SchemaRelabelExec" + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal( + "SchemaRelabelExec requires exactly one child".to_string(), + )); + } + Ok(Arc::new(Self::new( + children[0].clone(), + self.schema.clone(), + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DFResult { + Ok(Box::pin(SchemaRelabelStream { + input: self.input.execute(partition, context)?, + schema: self.schema.clone(), + })) + } +} + +struct SchemaRelabelStream { + input: SendableRecordBatchStream, + schema: SchemaRef, +} + +impl Stream for SchemaRelabelStream { + type Item = DFResult; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.input.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(batch))) => { + // Carry the row count explicitly: `try_new` infers it from the + // first column, which a column-less batch does not have. + let relabeled = RecordBatch::try_new_with_options( + self.schema.clone(), + batch.columns().to_vec(), + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .map_err(|e| DataFusionError::ArrowError(Box::new(e), None)); + Poll::Ready(Some(relabeled)) + } + other => other, + } + } +} + +impl datafusion::physical_plan::RecordBatchStream for SchemaRelabelStream { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int32Array, StringArray}; + use arrow_schema::{DataType, Field, Schema}; + use datafusion::prelude::SessionContext; + use datafusion_physical_plan::test::TestMemoryExec; + use futures::TryStreamExt; + + fn schema_with(nullable: bool) -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, nullable), + ])) + } + + fn source(batch: RecordBatch) -> Arc { + TestMemoryExec::try_new_exec(&[vec![batch.clone()]], batch.schema(), None).unwrap() + } + + fn batch(schema: SchemaRef, names: Vec>) -> RecordBatch { + let ids: Vec = (0..names.len() as i32).collect(); + RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(ids)), + Arc::new(StringArray::from(names)), + ], + ) + .unwrap() + } + + async fn run(plan: Arc) -> DFResult> { + let ctx = SessionContext::new(); + plan.execute(0, ctx.task_ctx())?.try_collect().await + } + + #[tokio::test] + async fn widening_preserves_rows_and_reports_target_schema() { + let input = source(batch(schema_with(false), vec![Some("a"), Some("b")])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(true))); + + assert_eq!(relabeled.schema(), schema_with(true)); + let out = run(relabeled).await.unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].schema(), schema_with(true)); + assert_eq!(out[0].num_rows(), 2); + } + + #[tokio::test] + async fn narrowing_succeeds_when_no_nulls_remain() { + // Post-tombstone-filter: `name` is nullable in storage, but every + // surviving row has a value. + let input = source(batch(schema_with(true), vec![Some("a"), Some("b")])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(false))); + + let out = run(relabeled).await.unwrap(); + assert_eq!(out[0].schema(), schema_with(false)); + assert_eq!(out[0].num_rows(), 2); + } + + #[tokio::test] + async fn narrowing_rejects_a_surviving_null() { + // A tombstone that escaped its filter must error here, not reach the + // caller as a row of nulls. + let input = source(batch(schema_with(true), vec![Some("a"), None])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(false))); + + let error = run(relabeled).await.unwrap_err().to_string(); + assert!( + error.contains("non-nullable") && error.contains("name"), + "expected a nullability error naming the column, got: {error}" + ); + } + + #[tokio::test] + async fn empty_batch_is_relabeled() { + let input = source(batch(schema_with(true), vec![])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, schema_with(false))); + + let out = run(relabeled).await.unwrap(); + assert!(out.iter().all(|b| b.num_rows() == 0)); + assert!(out.iter().all(|b| b.schema() == schema_with(false))); + } + + #[tokio::test] + async fn empty_batch_is_still_checked_against_the_target_schema() { + let input = source(batch(schema_with(true), vec![])); + let mistyped = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Int32, false), + ])); + let relabeled = Arc::new(SchemaRelabelExec::new(input, mistyped)); + + let error = run(relabeled).await.unwrap_err().to_string(); + assert!( + error.contains("column types must match"), + "expected a data type error, got: {error}" + ); + } +} diff --git a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs index d783540bf6b..12f58db6354 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/fts_search.rs @@ -4,7 +4,7 @@ //! Full-text search planner for LSM scanner (local scoring). //! //! Builds an execution plan that scores an FTS query across the base -//! table, flushed memtable generations, and the active/frozen-undrained +//! table, SSTable generations, and the active/frozen-undrained //! in-memory memtables, returning rows ordered by BM25 `_score` DESC. //! //! # Scoring @@ -22,17 +22,17 @@ //! benchmark in this PR shows it carries a real latency penalty, so the //! local path lands first and the global option is optimized separately. //! -//! Staleness: within a flushed generation, the deletion vector written +//! Staleness: within an SSTable, the deletion vector written //! at flush time (see #6929) already masks rows superseded by a newer //! generation, so per-source results are clean within each tier. The -//! same primary key can still appear across tiers (active vs flushed) +//! same primary key can still appear across tiers (active vs SSTable) //! when an updated row sits in the active memtable while the older -//! copy lives in a flushed generation; cross-tier deduplication is +//! copy lives in an SSTable; cross-tier deduplication is //! left to the caller in local mode. //! //! Everything here is contained in the `mem_wal` module — it reuses the //! existing per-source FTS read paths (`scanner.full_text_search` for -//! base/flushed Lance datasets, `MemTableScanner` for the active +//! base/SSTable Lance datasets, `MemTableScanner` for the active //! memtable) and requires no changes to `lance-index`. use std::sync::Arc; @@ -48,16 +48,19 @@ use datafusion::prelude::Expr; use lance_core::{Error, Result, is_system_column}; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::query::{FtsQuery as IndexFtsQuery, Operator}; +use lance_index::scalar::inverted::{DOC_INDEX_COL, DOC_INDEX_FIELD, DocumentGranularity}; use tracing::instrument; use super::block_list::compute_source_block_lists; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; use super::exec::PkBlockFilterExec; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{project_to_canonical, validate_projection_names}; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::dataset::mem_wal::memtable::scanner::MemTableScanner; +use crate::index::scalar::inverted::{indexed_fts_document_granularities, resolve_fts_field}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// `_score` column name in FTS results — kept aligned with /// `lance_index::scalar::inverted::SCORE_COL` so this module doesn't @@ -69,6 +72,140 @@ pub const SCORE_COLUMN: &str = "_score"; /// so a blocked source still yields `k` live rows after the block-list filter. const DEFAULT_OVERFETCH_FACTOR: f64 = 1.0; +fn requested_query_document_granularity( + query: &IndexFtsQuery, +) -> Result> { + fn merge( + current: &mut Option, + requested: Option, + ) -> Result<()> { + let Some(requested) = requested else { + return Ok(()); + }; + if let Some(current) = current + && *current != requested + { + return Err(Error::invalid_input( + "FTS queries cannot mix Row and ListElement document granularities".to_string(), + )); + } + *current = Some(requested); + Ok(()) + } + + fn visit(query: &IndexFtsQuery, current: &mut Option) -> Result<()> { + match query { + IndexFtsQuery::Match(query) => merge(current, query.document_granularity), + IndexFtsQuery::Phrase(query) => merge(current, query.document_granularity), + IndexFtsQuery::Boost(query) => { + visit(&query.positive, current)?; + visit(&query.negative, current) + } + IndexFtsQuery::Boolean(query) => { + for child in query + .must + .iter() + .chain(&query.should) + .chain(&query.must_not) + { + visit(child, current)?; + } + Ok(()) + } + IndexFtsQuery::MultiMatch(query) => { + for child in &query.match_queries { + merge(current, child.document_granularity)?; + } + Ok(()) + } + } + } + + let mut requested = None; + visit(query, &mut requested)?; + Ok(requested) +} + +fn set_query_document_granularity( + query: &mut IndexFtsQuery, + document_granularity: DocumentGranularity, +) { + match query { + IndexFtsQuery::Match(query) => { + query.document_granularity = Some(document_granularity); + } + IndexFtsQuery::Phrase(query) => { + query.document_granularity = Some(document_granularity); + } + IndexFtsQuery::Boost(query) => { + set_query_document_granularity(&mut query.positive, document_granularity); + set_query_document_granularity(&mut query.negative, document_granularity); + } + IndexFtsQuery::Boolean(query) => { + for child in query + .must + .iter_mut() + .chain(&mut query.should) + .chain(&mut query.must_not) + { + set_query_document_granularity(child, document_granularity); + } + } + IndexFtsQuery::MultiMatch(query) => { + for child in &mut query.match_queries { + child.document_granularity = Some(document_granularity); + } + } + } +} + +fn query_document_granularity(query: &FullTextSearchQuery) -> Result { + requested_query_document_granularity(&query.query)?.ok_or_else(|| { + Error::internal("LSM FTS query document granularity was not resolved".to_string()) + }) +} + +fn resolve_document_granularity_from_candidates( + column: &str, + requested: Option, + mut available: Vec, +) -> Result { + available.sort_by_key(|document_granularity| match document_granularity { + DocumentGranularity::Row => 0, + DocumentGranularity::ListElement => 1, + }); + available.dedup(); + match requested { + Some(requested) if available.is_empty() || available.contains(&requested) => Ok(requested), + Some(requested) => Err(Error::invalid_input(format!( + "FTS query for field '{column}' requested {requested:?} document granularity, but \ + the MemWAL sources use a different indexed granularity: {available:?}" + ))), + None if available.is_empty() => Ok(DocumentGranularity::Row), + None if available.len() == 1 => Ok(available[0]), + None => Err(Error::invalid_input(format!( + "FTS query for field '{column}' is ambiguous because Row and ListElement indexes \ + coexist across MemWAL sources; specify document_granularity" + ))), + } +} + +fn validate_source_document_granularities( + column: &str, + resolved: DocumentGranularity, + source_granularities: &[Vec], +) -> Result<()> { + for granularities in source_granularities { + if !granularities.is_empty() && !granularities.contains(&resolved) { + return Err(Error::invalid_input(format!( + "FTS query for field '{column}' resolved to {resolved:?}, but a MemWAL source has \ + only incompatible indexed granularities: {granularities:?}" + ))); + } + } + Ok(()) +} + fn validate_lsm_fts_query(query: &FullTextSearchQuery) -> Result<()> { match &query.query { IndexFtsQuery::Match(m) => { @@ -86,7 +223,11 @@ fn validate_lsm_fts_query(query: &FullTextSearchQuery) -> Result<()> { } } -fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool { +fn active_source_can_execute_fts( + source: &LsmDataSource, + column: &str, + document_granularity: DocumentGranularity, +) -> bool { match source { LsmDataSource::ActiveMemTable { batch_store, @@ -94,10 +235,10 @@ fn active_source_can_execute_fts(source: &LsmDataSource, column: &str) -> bool { .. } => { index_store - .get_fts_by_column(column) + .get_fts_by_column_and_granularity(column, document_granularity) .is_some_and(|index| !index.is_empty()) && batch_store - .max_visible_row(index_store.max_visible_batch_position()) + .max_visible_row(index_store.visible_count()) .is_some() } _ => false, @@ -109,16 +250,18 @@ pub struct LsmFtsSearchPlanner { collector: LsmDataSourceCollector, pk_columns: Vec, base_schema: SchemaRef, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Over-fetch multiple for blocked sources. overfetch_factor: f64, /// Optional prefilter predicate applied to every source arm so FTS hits - /// failing the predicate are dropped. Base/flushed arms use the dataset + /// failing the predicate are dropped. Base/SSTable arms use the dataset /// scanner's native filter; memtable arms filter the materialized hits. filter: Option, } @@ -135,7 +278,8 @@ impl LsmFtsSearchPlanner { pk_columns, base_schema, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, overfetch_factor: DEFAULT_OVERFETCH_FACTOR, filter: None, @@ -144,7 +288,7 @@ impl LsmFtsSearchPlanner { /// Attach an optional prefilter predicate. Every source arm restricts its /// FTS hits to rows matching the predicate, matching a normal filtered - /// full-text scan over base ∪ flushed ∪ in-memory data. + /// full-text scan over base ∪ SSTables ∪ in-memory data. pub fn with_filter(mut self, filter: Option) -> Self { self.filter = filter; self @@ -158,22 +302,27 @@ impl LsmFtsSearchPlanner { self } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); + self + } + + /// Inject a cache of opened SSTable datasets, making repeated /// searches against the same generation a pure `Arc::clone`. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -184,7 +333,7 @@ impl LsmFtsSearchPlanner { /// /// * `column` — text column to search. /// * `query` — the FTS query (match / phrase / boolean / fuzzy for - /// base/flushed Lance sources; the active memtable currently + /// base/SSTable Lance sources; the active memtable currently /// supports `MatchQuery`). /// * `limit` — optional global top-k to return. /// * `projection` — user columns to project. PK columns are @@ -202,19 +351,68 @@ impl LsmFtsSearchPlanner { pub async fn plan_search( &self, column: &str, - query: FullTextSearchQuery, + mut query: FullTextSearchQuery, limit: Option, projection: Option<&[String]>, ) -> Result> { let sources = self.collector.collect()?; + let requested = requested_query_document_granularity(&query.query)?; + let mut available = Vec::new(); + let mut source_granularities = Vec::with_capacity(sources.len()); + for source in &sources { + let granularities = match source { + LsmDataSource::BaseTable { dataset } => { + indexed_fts_document_granularities(dataset, column) + .await? + .into_iter() + .map(|(_, document_granularity)| document_granularity) + .collect::>() + } + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( + path, + self.session.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), + self.warmer.as_ref(), + ) + .await?; + indexed_fts_document_granularities(&dataset, column) + .await? + .into_iter() + .map(|(_, document_granularity)| document_granularity) + .collect::>() + } + LsmDataSource::ActiveMemTable { index_store, .. } => { + index_store.fts_document_granularities_by_column(column) + } + }; + available.extend(granularities.iter().copied()); + source_granularities.push(granularities); + } + let document_granularity = + resolve_document_granularity_from_candidates(column, requested, available)?; + validate_source_document_granularities( + column, + document_granularity, + &source_granularities, + )?; + let schema = lance_core::datatypes::Schema::try_from(self.base_schema.as_ref())?; + resolve_fts_field(&schema, column, document_granularity)?; + set_query_document_granularity(&mut query.query, document_granularity); if sources .iter() - .any(|source| active_source_can_execute_fts(source, column)) + .any(|source| active_source_can_execute_fts(source, column, document_granularity)) { validate_lsm_fts_query(&query)?; } - validate_projection_names(projection, &self.base_schema, &[SCORE_COLUMN])?; - let target_schema = self.canonical_fts_schema(projection); + let allowed_system_columns: &[&str] = if document_granularity.is_list_element() { + &[SCORE_COLUMN, DOC_INDEX_COL] + } else { + &[SCORE_COLUMN] + }; + validate_projection_names(projection, &self.base_schema, allowed_system_columns)?; + let target_schema = self.canonical_fts_schema(projection, document_granularity); let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?; if sources.is_empty() { @@ -228,7 +426,8 @@ impl LsmFtsSearchPlanner { let block_lists = Box::pin(compute_source_block_lists( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), )) .await?; @@ -370,11 +569,12 @@ impl LsmFtsSearchPlanner { scanner.full_text_search(bound_query)?; scanner.create_plan().await } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -402,8 +602,10 @@ impl LsmFtsSearchPlanner { schema, .. } => { - if !active_source_can_execute_fts(source, column) { - return self.empty_plan(&self.canonical_fts_schema(projection)); + let document_granularity = query_document_granularity(query)?; + if !active_source_can_execute_fts(source, column, document_granularity) { + return self + .empty_plan(&self.canonical_fts_schema(projection, document_granularity)); } validate_lsm_fts_query(query)?; let mut scanner = @@ -460,7 +662,11 @@ impl LsmFtsSearchPlanner { } /// Canonical FTS output: user-projected cols + PK + `_score`. - fn canonical_fts_schema(&self, user_projection: Option<&[String]>) -> SchemaRef { + fn canonical_fts_schema( + &self, + user_projection: Option<&[String]>, + document_granularity: DocumentGranularity, + ) -> SchemaRef { let mut ordered: Vec = if let Some(p) = user_projection { p.to_vec() } else { @@ -475,6 +681,9 @@ impl LsmFtsSearchPlanner { ordered.push(pk.clone()); } } + if document_granularity.is_list_element() && !ordered.iter().any(|c| c == DOC_INDEX_COL) { + ordered.push(DOC_INDEX_COL.to_string()); + } if !ordered.iter().any(|c| c == SCORE_COLUMN) { ordered.push(SCORE_COLUMN.to_string()); } @@ -483,6 +692,8 @@ impl LsmFtsSearchPlanner { .filter_map(|name| { if name == SCORE_COLUMN { Some(Arc::new(Field::new(SCORE_COLUMN, DataType::Float32, true))) + } else if name == DOC_INDEX_COL { + Some(Arc::new(DOC_INDEX_FIELD.clone())) } else if is_system_column(name) { Some(Arc::new(Field::new(name.clone(), DataType::UInt64, true))) } else { @@ -508,9 +719,14 @@ mod tests { use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; use crate::dataset::{Dataset, WriteParams}; - use arrow_array::{BooleanArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{ + Array, BooleanArray, Int32Array, ListArray, RecordBatch, RecordBatchIterator, StringArray, + UInt32Array, + }; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use futures::TryStreamExt; + use lance_index::scalar::inverted::query::MatchQuery; use std::collections::HashMap; fn fts_schema() -> Arc { @@ -608,10 +824,321 @@ mod tests { ); } + #[test] + fn resolves_document_granularity_from_memwal_indexes() { + assert_eq!( + resolve_document_granularity_from_candidates("tags", None, vec![]).unwrap(), + DocumentGranularity::Row + ); + assert_eq!( + resolve_document_granularity_from_candidates( + "tags", + None, + vec![DocumentGranularity::ListElement], + ) + .unwrap(), + DocumentGranularity::ListElement + ); + assert!( + resolve_document_granularity_from_candidates( + "tags", + Some(DocumentGranularity::Row), + vec![DocumentGranularity::ListElement], + ) + .unwrap_err() + .to_string() + .contains("different indexed granularity") + ); + let both = vec![DocumentGranularity::Row, DocumentGranularity::ListElement]; + assert!( + resolve_document_granularity_from_candidates("tags", None, both.clone()) + .unwrap_err() + .to_string() + .contains("ambiguous") + ); + assert_eq!( + resolve_document_granularity_from_candidates( + "tags", + Some(DocumentGranularity::ListElement), + both, + ) + .unwrap(), + DocumentGranularity::ListElement + ); + assert!( + validate_source_document_granularities( + "tags", + DocumentGranularity::ListElement, + &[ + vec![DocumentGranularity::ListElement], + vec![DocumentGranularity::Row], + ], + ) + .unwrap_err() + .to_string() + .contains("incompatible indexed granularities") + ); + } + + #[tokio::test] + async fn memwal_rejects_list_element_without_a_list_path() { + let schema = fts_schema(); + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema); + let query = FullTextSearchQuery::new_query(IndexFtsQuery::Match( + MatchQuery::new("lance".to_string()) + .with_document_granularity(DocumentGranularity::ListElement), + )); + + let error = planner + .plan_search("text", query, Some(1), None) + .await + .unwrap_err(); + assert!(error.to_string().contains("has no List layer"), "{error}"); + } + + #[tokio::test] + async fn active_element_document_search_returns_physical_ordinals() { + use lance_index::scalar::inverted::InvertedIndexParams; + + let mut id_meta = HashMap::new(); + id_meta.insert( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + ); + let id_field = Field::new("id", DataType::Int32, false).with_metadata(id_meta); + + let mut tags = ListBuilder::new(StringBuilder::new()); + tags.values().append_value("alpha"); + tags.values().append_value("beta gamma"); + tags.values().append_null(); + tags.values().append_value("beta"); + tags.append(true); + tags.append(true); + tags.values().append_value("beta"); + tags.append(true); + let tags = tags.finish(); + let schema = Arc::new(ArrowSchema::new(vec![ + id_field, + Field::new("tags", tags.data_type().clone(), true), + ])); + let active_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3])), Arc::new(tags)], + ) + .unwrap(); + + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + indexes + .add_fts_with_params( + "tags_list_element_fts".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&active_batch, row_offset, Some(batch_position)) + .unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]) + .with_in_memory_memtables( + uuid::Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(indexes), + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema.clone()); + let projection = vec!["id".to_string()]; + let plan = planner + .plan_search( + "tags", + FullTextSearchQuery::new_query(IndexFtsQuery::Match(MatchQuery::new( + "beta".to_string(), + ))), + Some(10), + Some(&projection), + ) + .await + .unwrap(); + + let ctx = datafusion::prelude::SessionContext::new(); + let stream = plan.execute(0, ctx.task_ctx()).unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + let mut hits = Vec::new(); + for batch in batches { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let doc_indices = batch[DOC_INDEX_COL] + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + let coordinate = doc_indices + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + hits.push((ids.value(row), coordinate)); + } + } + hits.sort_unstable(); + assert_eq!(hits, vec![(1, 1), (1, 3), (3, 0)]); + } + + #[tokio::test] + async fn element_document_search_unions_base_and_active_sources() { + use crate::index::DatasetIndexExt; + use lance_index::IndexType; + use lance_index::scalar::inverted::InvertedIndexParams; + + let mut id_meta = HashMap::new(); + id_meta.insert( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + ); + let id_field = Field::new("id", DataType::Int32, false).with_metadata(id_meta); + let list_type = DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))); + let schema = Arc::new(ArrowSchema::new(vec![ + id_field, + Field::new("tags", list_type, true), + ])); + + let mut base_tags = ListBuilder::new(StringBuilder::new()); + base_tags.values().append_value("beta"); + base_tags.values().append_value("other"); + base_tags.append(true); + let base_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1])), + Arc::new(base_tags.finish()), + ], + ) + .unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", tmp.path().to_str().unwrap()); + let mut base_ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(base_batch)], schema.clone()), + &base_uri, + None, + ) + .await + .unwrap(); + base_ds + .create_index( + &["tags"], + IndexType::Inverted, + Some("tags_list_element_fts".to_string()), + &InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + false, + ) + .await + .unwrap(); + let base_ds = Arc::new(Dataset::open(&base_uri).await.unwrap()); + + let mut active_tags = ListBuilder::new(StringBuilder::new()); + active_tags.values().append_value("other"); + active_tags.values().append_value("beta"); + active_tags.append(true); + let active_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![2])), + Arc::new(active_tags.finish()), + ], + ) + .unwrap(); + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut indexes = IndexStore::new(); + indexes.enable_pk_index(&[("id".to_string(), 0)]); + indexes + .add_fts_with_params( + "tags_list_element_fts".to_string(), + 1, + "tags".to_string(), + InvertedIndexParams::default() + .document_granularity(DocumentGranularity::ListElement), + ) + .unwrap(); + let (_, row_offset, batch_position) = batch_store.append(active_batch.clone()).unwrap(); + indexes + .insert_with_batch_position(&active_batch, row_offset, Some(batch_position)) + .unwrap(); + + let collector = LsmDataSourceCollector::new(base_ds, vec![]).with_in_memory_memtables( + uuid::Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(indexes), + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ); + let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema); + let projection = vec!["id".to_string()]; + let plan = planner + .plan_search( + "tags", + FullTextSearchQuery::new_query(IndexFtsQuery::Match(MatchQuery::new( + "beta".to_string(), + ))), + Some(10), + Some(&projection), + ) + .await + .unwrap(); + + let ctx = datafusion::prelude::SessionContext::new(); + let batches: Vec = plan + .execute(0, ctx.task_ctx()) + .unwrap() + .try_collect() + .await + .unwrap(); + let mut hits = Vec::new(); + for batch in batches { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let coordinates = batch[DOC_INDEX_COL] + .as_any() + .downcast_ref::() + .unwrap(); + for row in 0..batch.num_rows() { + let coordinate = coordinates + .value(row) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + hits.push((ids.value(row), coordinate)); + } + } + hits.sort_unstable(); + assert_eq!(hits, vec![(1, 0), (2, 1)]); + } + #[tokio::test] async fn local_mode_unions_base_and_active_with_consistent_score_schema() { // Regression for the `_score` nullability mismatch between - // FtsIndexExec (active arm) and FTS_SCHEMA (base/flushed). The + // FtsIndexExec (active arm) and FTS_SCHEMA (base/SSTable). The // active-only test below would not catch this — UnionExec rejects // schema-inequality, so we need at least one base + one active // source to exercise that code path. @@ -818,12 +1345,12 @@ mod tests { ); } - /// The flushed arm must apply the filter as a true FTS prefilter, and that + /// The SSTable arm must apply the filter as a true FTS prefilter, and that /// prefiltered candidate set must compose with cross-generation block-list /// filtering plus over-fetch. Gen 1's best predicate-matching hit (id=3) is /// superseded by gen 2; with over-fetch, gen 1 should still contribute id=4. #[tokio::test] - async fn prefilter_on_flushed_composes_with_block_list() { + async fn prefilter_on_sstable_composes_with_block_list() { use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::index::DatasetIndexExt; use datafusion::prelude::{col, lit}; @@ -837,7 +1364,7 @@ mod tests { // Gen 1: id=1 matches strongly but fails the predicate. id=3 matches // strongly but is stale (blocked by gen 2). id=4 is the next live - // predicate match that only survives if the flushed arm prefilters and + // predicate match that only survives if the SSTable arm prefilters and // over-fetches before the block-list drops id=3. let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let mut gen1 = write_dataset( @@ -875,8 +1402,8 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]); let planner = LsmFtsSearchPlanner::new(collector, vec!["id".to_string()], schema) @@ -890,7 +1417,7 @@ mod tests { None, ) .await - .expect("planner should produce a filtered flushed plan"); + .expect("planner should produce a filtered SSTable plan"); let ctx = datafusion::prelude::SessionContext::new(); let stream = plan.execute(0, ctx.task_ctx()).unwrap(); @@ -911,7 +1438,7 @@ mod tests { assert_eq!( ids, vec![4], - "flushed FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}" + "SSTable FTS prefilter should return live id=4 after stale id=3 is blocked; got {ids:?}" ); } diff --git a/rust/lance/src/dataset/mem_wal/scanner/planner.rs b/rust/lance/src/dataset/mem_wal/scanner/planner.rs index ec13da1df66..2a83447fb17 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/planner.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/planner.rs @@ -18,16 +18,17 @@ use crate::dataset::mem_wal::TOMBSTONE; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; use super::exec::{MEMTABLE_GEN_COLUMN, MemtableGenTagExec, PkBlockFilterExec, ROW_ADDRESS_COLUMN}; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{ build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical, validate_projection_names, }; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Combine the user filter (if any) with `NOT _tombstone` so tombstone rows are /// dropped from a WAL-arm scan. Used only for sources whose schema carries the -/// column (active / flushed generations written since deletes existed). +/// column (active / SSTables written since deletes existed). fn fold_not_tombstone(filter: Option<&Expr>) -> Expr { let live = !col(TOMBSTONE); match filter { @@ -44,25 +45,14 @@ pub struct LsmScanPlanner { pk_columns: Vec, /// Schema of the base table. base_schema: SchemaRef, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, - /// Over-fetch multiple for the per-source limit pushdown: block-listed - /// sources scan `(offset + limit) * factor` rows so cross-gen dedup drops - /// still leave enough live rows. Clamped to `>= 1.0`. - /// - /// This headroom must also absorb deletes: a tombstone shadows the older - /// real row without emitting a replacement (shadow-without-replace), so it - /// is pure subtraction from a block-listed source. If delete density inside - /// a source's fetch window exceeds `factor - 1`, that source can deliver - /// `< k` live rows (a recall shortfall, not wrong content — see the - /// under-fetch `warn!` in `PkBlockFilterExec`). Steady-state this is - /// self-limiting because L0→base compaction drains tombstones from the - /// fresh tier; the exposure is a delete-heavy burst between compactions. - overfetch_factor: f64, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, } impl LsmScanPlanner { @@ -77,37 +67,34 @@ impl LsmScanPlanner { pk_columns, base_schema, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, - overfetch_factor: 1.0, } } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated - /// queries against the same generation a pure `Arc::clone`. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { - self.warmer = Some(warmer); + /// Inject a cache of opened SSTable datasets, making repeated + /// queries against the same generation a pure `Arc::clone`. + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Set the over-fetch multiple for the per-source limit pushdown - /// (see the field docs). Values below `1.0` are rejected by - /// [`Self::plan_scan`]. - pub fn with_overfetch_factor(mut self, factor: f64) -> Self { - self.overfetch_factor = factor; + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { + self.warmer = Some(warmer); self } @@ -154,7 +141,6 @@ impl LsmScanPlanner { // 1. Collect all data sources let sources = self.collector.collect()?; - let overfetch = super::validate_overfetch_factor(self.overfetch_factor)?; if sources.is_empty() { // Return empty plan @@ -167,7 +153,8 @@ impl LsmScanPlanner { let block_lists = Box::pin(super::block_list::compute_source_block_lists( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), )) .await?; @@ -177,14 +164,13 @@ impl LsmScanPlanner { let sources: Vec<_> = sources.into_iter().rev().collect(); // Per-source limit pushdown: an unordered LIMIT needs only - // `offset + limit` live rows from EACH source to fill the global - // limit after dedup (any-N semantics), so cap every on-disk source - // instead of scanning whole generations and trimming above the - // union. Block-listed sources over-fetch by `overfetch_factor` so - // cross-gen dedup drops still leave `n_needed` live rows; the - // PkBlockFilter warns when that was not enough. The active memtable - // is in-memory and within-gen append duplicates are resolved by its - // own dedup, so it is never capped here. + // `offset + limit` live rows from each source to fill the global limit + // (any-N semantics). However, a source with a cross-generation block + // list can lose any number of rows after its scan, so a finite fetch + // before that filter is not safe. Leave those scans unbounded and let + // the LocalLimitExec below pull until it sees `n_needed` live rows or + // reaches EOF. Sources without a block filter can still push the limit + // down safely. The active memtable is in-memory and is never capped. let n_needed = limit.map(|l| l.saturating_add(offset.unwrap_or(0))); let mut source_plans = Vec::new(); @@ -194,12 +180,9 @@ impl LsmScanPlanner { let blocked = block_lists .get(&(source.shard_id(), source.generation())) .cloned(); - let fetch = match (n_needed, is_active) { - (Some(n), false) => Some(if blocked.is_some() && !self.pk_columns.is_empty() { - ((n as f64) * overfetch).ceil() as usize - } else { - n - }), + let has_block_filter = blocked.is_some() && !self.pk_columns.is_empty(); + let fetch = match (n_needed, is_active, has_block_filter) { + (Some(n), false, false) => Some(n), _ => None, }; let scan = self @@ -207,14 +190,14 @@ impl LsmScanPlanner { .await?; // Drop cross-generation stale rows (PKs superseded by a newer gen). - // With a limit, `k = n_needed` arms the under-fetch warning; with - // no limit `k = 0` keeps it silent. + // Plain scans refill exactly, so keep the approximate-search + // under-fetch warning disabled with k = 0. let scan = match blocked { Some(set) if !self.pk_columns.is_empty() => Arc::new(PkBlockFilterExec::new( scan, self.pk_columns.clone(), set, - n_needed.unwrap_or(0), + 0, )) as Arc, _ => scan, @@ -338,11 +321,12 @@ impl LsmScanPlanner { scanner.create_plan().await } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -368,7 +352,7 @@ impl LsmScanPlanner { if let Some(expr) = effective { scanner.filter_expr(expr.clone()); } - // Per-source limit pushdown: flushed generations are + // Per-source limit pushdown: SSTables are // within-gen live (dedup-on-flush deletion vectors), so any // `fetch` post-filter rows are valid contributions. if let Some(fetch) = fetch { @@ -472,10 +456,10 @@ mod tests { let shard_id = uuid::Uuid::new_v4(); let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(5) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); - assert_eq!(snapshot.flushed_generations.len(), 2); + assert_eq!(snapshot.sstables.len(), 2); assert_eq!(snapshot.current_generation, 5); } } @@ -535,7 +519,7 @@ mod integration_tests { } /// Create a dataset at the given URI with the provided batches. Also writes - /// the standalone PK sidecar (on `id`) so a flushed-generation source can be + /// the standalone PK sidecar (on `id`) so an SSTable source can be /// probed by the block-list; harmless for a base table (never probed). async fn create_dataset(uri: &str, batches: Vec) -> Dataset { let schema = batches[0].schema(); @@ -568,8 +552,8 @@ mod integration_tests { /// Setup a multi-level LSM structure with: /// - Base table: ids 1-5 with "base" prefix - /// - Flushed gen1: ids 3,4 (updates) with "gen1" prefix - /// - Flushed gen2: ids 4,5 (updates) + id 6 (new) with "gen2" prefix + /// - SSTable gen1: ids 3,4 (updates) with "gen1" prefix + /// - SSTable gen2: ids 4,5 (updates) + id 6 (new) with "gen2" prefix /// - Active memtable: ids 5,6 (updates) + id 7 (new) with "active" prefix /// /// Expected deduplication results: @@ -596,13 +580,13 @@ mod integration_tests { let base_batch = create_test_batch(&schema, &[1, 2, 3, 4, 5], "base"); let base_dataset = Arc::new(create_dataset(&base_uri, vec![base_batch]).await); - // Create flushed gen1 as a separate dataset + // Create SSTable gen1 as a separate dataset let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1"); create_dataset(&gen1_uri, vec![gen1_batch]).await; - // Create flushed gen2 as a separate dataset + // Create SSTable gen2 as a separate dataset let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id); let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2"); create_dataset(&gen2_uri, vec![gen2_batch]).await; @@ -610,8 +594,8 @@ mod integration_tests { // Build shard snapshot let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); // Create active memtable let (batch_store, index_store) = @@ -835,11 +819,11 @@ mod integration_tests { } /// Regression for the concurrent-read-vs-flush hole: a sealed - /// (frozen-awaiting-flush) memtable is not yet recorded as a flushed - /// generation, but its rows must still be in the scan's read union and + /// (frozen-awaiting-flush) memtable is not yet recorded as an + /// SSTable, but its rows must still be in the scan's read union and /// dedup correctly by generation across the active/frozen seam. /// - /// Layout: base(0) ids 1-5, flushed gen1 ids 3,4, flushed gen2 ids + /// Layout: base(0) ids 1-5, SSTable gen1 ids 3,4, SSTable gen2 ids /// 4,5,6, frozen memtable gen3 ids 6,7, active memtable gen4 ids 7,8. #[tokio::test] async fn test_lsm_scan_frozen_memtable_in_read_union() { @@ -868,8 +852,8 @@ mod integration_tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(4) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); // Frozen gen3 (sealed, NOT in the manifest) and active gen4. let (frozen_store, frozen_index) = @@ -928,7 +912,7 @@ mod integration_tests { assert_eq!(results.get(&3), Some(&"gen1_3".to_string())); assert_eq!(results.get(&4), Some(&"gen2_4".to_string())); assert_eq!(results.get(&5), Some(&"gen2_5".to_string())); - // id=6: in flushed gen2 AND frozen gen3 -> frozen wins. This is the + // id=6: in SSTable gen2 AND frozen gen3 -> frozen wins. This is the // bug: pre-fix the frozen memtable fell out of the read union and // id=6 resolved to "gen2_6". assert_eq!(results.get(&6), Some(&"frozen_6".to_string())); @@ -996,6 +980,68 @@ mod integration_tests { assert_eq!(total_rows, 3, "Should have 3 rows due to limit"); } + #[tokio::test] + async fn test_lsm_scan_limit_offset_refills_after_update_shadow_across_fragments() { + let schema = create_pk_schema(); + let temp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", temp.path().to_str().unwrap()); + let base_batch = create_test_batch(&schema, &[1, 2, 3, 4], "base"); + let reader = RecordBatchIterator::new([Ok(base_batch)], schema.clone()); + let base = Arc::new( + Dataset::write( + reader, + &base_uri, + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + assert_eq!( + base.get_fragments().len(), + 2, + "the stale prefix and live rows must occupy separate fragments" + ); + + // The newest values for ids 1 and 2 no longer match the predicate, so + // their matching base values are shadowed. The second base fragment + // still contains the live matches that must fill offset + limit. + let (batch_store, index_store) = + pk_indexed(&[create_test_batch(&schema, &[1, 2], "active")]); + let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()]) + .with_in_memory_memtables( + Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store, + schema, + generation: 1, + }, + frozen: vec![], + }, + ) + .filter("name LIKE 'base%'") + .unwrap() + .limit(Some(1), Some(1)) + .unwrap(); + + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + ids.iter().collect::>(), + vec![Some(4)], + "offset=1, limit=1 must skip id=3 after shadow filtering" + ); + } + #[tokio::test] async fn test_lsm_scan_with_offset_without_limit() { let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) = @@ -1030,7 +1076,7 @@ mod integration_tests { let (base_dataset, _, _, pk_columns, _temp_path) = setup_multi_level_lsm().await; // Create scanner with only base table (no shard snapshots or active memtable) - let scanner = LsmScanner::new(base_dataset, vec![], pk_columns); + let scanner = LsmScanner::new(base_dataset.clone(), vec![], pk_columns.clone()); let plan = scanner.create_plan().await.unwrap(); @@ -1044,6 +1090,22 @@ mod integration_tests { .await .unwrap(); + // A base-only source has no cross-generation block filter, so its + // finite limit remains safe to push into the physical Lance read. + let scanner = LsmScanner::new(base_dataset, vec![], pk_columns) + .limit(Some(3), None) + .unwrap(); + let plan = scanner.create_plan().await.unwrap(); + assert_plan_node_equals( + plan, + "GlobalLimitExec: skip=0, fetch=3 + ProjectionExec:... + LocalLimitExec: fetch=3 + LanceRead:...base/data...range_before=Some(0..3)...refine_filter=--", + ) + .await + .unwrap(); + // Execute and verify all 5 base rows are returned let scanner = LsmScanner::new( Arc::new( @@ -1067,7 +1129,7 @@ mod integration_tests { } #[tokio::test] - async fn test_lsm_scan_flushed_only_no_active() { + async fn test_lsm_scan_sstable_only_no_active() { let (base_dataset, shard_snapshots, _, pk_columns, _temp_path) = setup_multi_level_lsm().await; @@ -1229,7 +1291,7 @@ mod integration_tests { /// /// Similar to setup_multi_level_lsm but: /// - Active memtable has a BTree index on the `id` column - /// - Flushed datasets have BTree index created (enabling ScalarIndexQuery) + /// - SSTables have BTree index created (enabling ScalarIndexQuery) async fn setup_multi_level_lsm_with_btree_index() -> ( Arc, Vec, @@ -1259,7 +1321,7 @@ mod integration_tests { // Reload dataset to pick up the index let base_dataset = Arc::new(Dataset::open(&base_uri).await.unwrap()); - // Create flushed gen1 with BTree index + // Create SSTable gen1 with BTree index let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let gen1_batch = create_test_batch(&schema, &[3, 4], "gen1"); @@ -1268,7 +1330,7 @@ mod integration_tests { .await .unwrap(); - // Create flushed gen2 with BTree index + // Create SSTable gen2 with BTree index let gen2_uri = format!("{}/_mem_wal/{}/gen_2", base_uri, shard_id); let gen2_batch = create_test_batch(&schema, &[4, 5, 6], "gen2"); let mut gen2_dataset = create_dataset(&gen2_uri, vec![gen2_batch]).await; @@ -1279,8 +1341,8 @@ mod integration_tests { // Build shard snapshot let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); // Create active memtable with BTree index let batch_store = Arc::new(BatchStore::with_capacity(100)); @@ -1615,7 +1677,7 @@ mod integration_tests { let (base_dataset, shard_snapshots, active_memtable, pk_columns, _temp_path) = setup_multi_level_lsm().await; - // Use the same base URI the flushed generations were created under, so + // Use the same base URI the SSTables were created under, so // relative `gen_N` folders resolve to real datasets on disk. let base_uri = base_dataset.uri().to_string(); let arrow_schema: arrow_schema::Schema = base_dataset.schema().into(); @@ -1640,7 +1702,7 @@ mod integration_tests { ); assert!( plan_str.contains("gen_1") && plan_str.contains("gen_2"), - "Plan must scan flushed generations, got: {}", + "Plan must scan SSTables, got: {}", plan_str ); assert!( @@ -1747,8 +1809,8 @@ mod integration_tests { } #[tokio::test] - async fn test_lsm_scan_without_base_table_no_flushed_no_active() { - // No base, no flushed, no active → empty result, valid plan. + async fn test_lsm_scan_without_base_table_no_sstable_no_active() { + // No base, no SSTable, no active → empty result, valid plan. let schema = create_pk_schema(); let scanner = LsmScanner::without_base_table( schema, @@ -2082,8 +2144,48 @@ mod integration_tests { } #[tokio::test] - async fn test_lsm_scan_flushed_tombstone_masks_base() { - // A tombstone living in a flushed generation masks the older base row by + async fn test_lsm_scan_limit_refills_after_active_tombstone_shadows_base() { + let base_schema = create_pk_schema(); + let mem_schema = ts_pk_schema(); + let temp = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", temp.path().to_str().unwrap()); + let base = Arc::new( + create_dataset( + &base_uri, + vec![create_test_batch(&base_schema, &[1, 2, 3], "base")], + ) + .await, + ); + + let active_batch = ts_batch(&mem_schema, &[(1, None, true)]); + let (batch_store, index_store) = pk_indexed(&[active_batch]); + let scanner = LsmScanner::new(base, vec![], vec!["id".to_string()]) + .with_in_memory_memtables( + Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store, + schema: mem_schema, + generation: 1, + }, + frozen: vec![], + }, + ) + .limit(Some(2), None) + .unwrap(); + + let batch = scanner.try_into_batch().await.unwrap(); + assert_eq!( + collect_sorted_ids(&[batch]), + vec![2, 3], + "the base scan must continue past the shadowed row to fill the limit" + ); + } + + #[tokio::test] + async fn test_lsm_scan_sstable_tombstone_masks_base() { + // A tombstone living in an SSTable masks the older base row by // PK presence (block-list) and is itself dropped by the folded predicate. let base_schema = create_pk_schema(); let mem_schema = ts_pk_schema(); @@ -2098,14 +2200,14 @@ mod integration_tests { .await, ); - // Flushed gen 1 holds only a tombstone for id=2 (written with the - // `_tombstone` schema, so the flushed arm folds `NOT _tombstone`). + // SSTable gen 1 holds only a tombstone for id=2 (written with the + // `_tombstone` schema, so the SSTable arm folds `NOT _tombstone`). let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); create_dataset(&gen1_uri, vec![ts_batch(&mem_schema, &[(2, None, true)])]).await; let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let scanner = LsmScanner::new(base, vec![shard_snapshot], vec!["id".to_string()]); let batches: Vec = scanner @@ -2118,13 +2220,13 @@ mod integration_tests { assert_eq!( collect_sorted_ids(&batches), vec![1, 3], - "id=2 deleted via flushed-generation tombstone" + "id=2 deleted via an SSTable tombstone" ); } #[tokio::test] async fn test_lsm_scan_tombstone_does_not_consume_limit() { - // A single (newest) flushed generation holds both tombstones and live + // A single (newest) SSTable holds both tombstones and live // rows. With LIMIT 2 the folded `NOT _tombstone` runs *before* the // per-source pushdown limit, so the limit counts only live rows — we get // 2 live rows, not 0 (which is what a post-limit tombstone filter, or a @@ -2152,7 +2254,7 @@ mod integration_tests { .await; let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let scanner = LsmScanner::without_base_table( base_schema, diff --git a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs index 4f7c5f093f6..9aea4e9f4e7 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/point_lookup.rs @@ -27,19 +27,20 @@ use lance_core::{Result, is_system_column}; use lance_datafusion::exec::OneShotExec; use tracing::instrument; -use crate::dataset::mem_wal::TOMBSTONE; -use crate::dataset::mem_wal::index::IndexStore; +use crate::dataset::mem_wal::index::{IndexStore, MemTableVisibility}; use crate::dataset::mem_wal::memtable::batch_store::BatchStore; +use crate::dataset::mem_wal::{TOMBSTONE, relax_non_pk_nullability}; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; use super::exec::{BloomFilterGuardExec, CoalesceFirstExec, compute_pk_hash_from_scalars}; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{ - DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns, + DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, force_schema, null_columns, project_to_canonical, validate_projection_names, wants_row_address, wants_row_id, }; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Plans point lookup queries over LSM data. /// @@ -87,12 +88,14 @@ pub struct LsmPointLookupPlanner { /// Bloom filters for each memtable generation. /// Map: generation -> bloom filter bloom_filters: std::collections::HashMap>, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Precomputed canonical output schema for the no-projection case, so the /// hot `lookup(.., None)` path clones an `Arc` instead of rebuilding the /// schema on every call. @@ -102,6 +105,9 @@ pub struct LsmPointLookupPlanner { /// on the plan fallback path (the part of point-lookup latency that doesn't /// scale with generation count). task_ctx: Arc, + /// Prefix of the in-memory memtables this planner reads. Applies to the fast + /// BTree probe and the plan fallback alike, so both resolve a key the same. + visibility: MemTableVisibility, } impl LsmPointLookupPlanner { @@ -124,32 +130,46 @@ impl LsmPointLookupPlanner { base_schema, bloom_filters: std::collections::HashMap::new(), session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, none_target, task_ctx: SessionContext::new().task_ctx(), + visibility: MemTableVisibility::Published, } } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Read the in-memory memtables at `visibility`. See + /// [`MemTableVisibility::Indexed`] for when a wider bound is sound. + pub fn with_visibility(mut self, visibility: MemTableVisibility) -> Self { + self.visibility = visibility; + self + } + + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); + self + } + + /// Inject a cache of opened SSTable datasets, making repeated /// lookups against the same generation a pure `Arc::clone`. Populate it up /// front during scan setup via /// [`DatasetMemWalExt::prewarm_mem_wal`](crate::dataset::mem_wal::DatasetMemWalExt::prewarm_mem_wal) /// so the first gen-key lookup does not pay the dataset open. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -312,7 +332,7 @@ impl LsmPointLookupPlanner { ) -> Result { let canonical = canonical_output_schema(projection, &self.base_schema, &self.pk_columns, false); - let target = carry_schema(&canonical); + let target = carry_schema(&canonical, &self.pk_columns); let mut out: Vec = Vec::with_capacity(keys.len()); for key in keys { if let Some(b) = self.lookup_keep_tombstone(key, projection).await? { @@ -332,7 +352,7 @@ impl LsmPointLookupPlanner { /// For a single-column primary key this probes the in-memory memtables' /// BTree index directly — no DataFusion plan — newest generation first, and /// returns on the first hit. Only when the lookup must consult an on-disk - /// source (a flushed generation or the base table), a memtable lacks a + /// source (an SSTable or the base table), a memtable lacks a /// BTree on the key, the key is multi-column, or the projection requests /// system columns does it fall back to [`Self::plan_lookup`]. The result is /// identical to executing `plan_lookup` and taking the first row; the fast @@ -390,6 +410,7 @@ impl LsmPointLookupPlanner { &self.pk_columns[0], &pk_values[0], target, + self.visibility, )? { Probe::Hit(batch) => Ok(Some(FastOutcome::Hit(batch))), Probe::Deleted => Ok(Some(FastOutcome::Deleted)), @@ -407,7 +428,7 @@ impl LsmPointLookupPlanner { None => { // Every in-memory memtable missed. If there is no // on-disk source, the key does not exist; otherwise the - // plan path consults the base table / flushed gens. + // plan path consults the base table / SSTables. if !self.collector.has_on_disk_sources() { return Ok(None); } @@ -498,7 +519,8 @@ impl LsmPointLookupPlanner { for key in keys { let mut resolved = false; for (ri, m) in refs.iter().enumerate() { - match probe_position(&m.batch_store, &m.index_store, pk_col, key)? { + match probe_position(&m.batch_store, &m.index_store, pk_col, key, self.visibility)? + { ProbePos::Found { batch_idx, row } => { // Newest version is a tombstone → the key is deleted: // resolve it as a miss (emit nothing) and do not fall @@ -645,13 +667,18 @@ impl LsmPointLookupPlanner { scanner.with_row_address(); } scanner.filter_expr(filter.clone()); - scanner.create_plan().await? + // Box at the call site: `create_plan`'s inlined async layout exceeds + // rustc's depth limit up this point-lookup chain, and boxing inside + // `create_plan` instead triggers a `Box: Send` solver overflow + // (E0275 downstream). Same for the other arms below. + Box::pin(scanner.create_plan()).await? } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -662,7 +689,7 @@ impl LsmPointLookupPlanner { let cols = cols_with_tombstone(&cols, dataset.schema().field(TOMBSTONE).is_some()); scanner.project(&cols.iter().map(|s| s.as_str()).collect::>())?; scanner.filter_expr(filter.clone()); - scanner.create_plan().await? + Box::pin(scanner.create_plan()).await? } LsmDataSource::ActiveMemTable { batch_store, @@ -672,8 +699,12 @@ impl LsmPointLookupPlanner { } => { use crate::dataset::mem_wal::memtable::scanner::MemTableScanner; - let mut scanner = - MemTableScanner::new(batch_store.clone(), index_store.clone(), schema.clone()); + let mut scanner = MemTableScanner::new_at_visibility( + batch_store.clone(), + index_store.clone(), + schema.clone(), + self.visibility, + ); // Carry `_tombstone` through so the post-coalesce filter can drop // a deleted key; it survives the sort below. let cols = cols_with_tombstone(&cols, schema.column_with_name(TOMBSTONE).is_some()); @@ -685,7 +716,7 @@ impl LsmPointLookupPlanner { // over insert-ordered scan would return the *oldest* of // multiple rows sharing the target primary key. scanner.with_row_id(); - let raw = scanner.create_plan().await?; + let raw = Box::pin(scanner.create_plan()).await?; // The filter already restricts to the exact PK value, so the // scan yields that key's insert history. Within the active // memtable larger `_rowid` = newer insert, so sorting `_rowid` @@ -714,7 +745,7 @@ impl LsmPointLookupPlanner { // Output carries `_tombstone` (canonical + the marker) so it survives // the union/coalesce to the post-coalesce filter; base / legacy sources // that lack the column get a synthesized `false`. - project_to_carry(scan, &target) + project_to_carry(scan, &target, &self.pk_columns) } /// Create an empty execution plan with the canonical output schema. @@ -742,11 +773,18 @@ fn cols_with_tombstone(cols: &[String], present: bool) -> Vec { out } -/// Carry schema = canonical output + a trailing non-nullable `_tombstone` -/// Boolean. Non-nullable so the base arm's synthesized `Literal(false)` matches -/// the WAL arms' real column under `CoalesceFirstExec`'s exact-schema check. -fn carry_schema(canonical: &SchemaRef) -> SchemaRef { - let mut fields: Vec> = canonical.fields().iter().cloned().collect(); +/// Carry schema = canonical output widened to the storage schema's +/// nullability, plus a trailing non-nullable `_tombstone` Boolean. +/// +/// Widened because tombstone rows — null in every non-PK column — are still in +/// flight; [`filter_tombstones_after_coalesce`] drops them past +/// `CoalesceFirstExec`, and only then does the plan narrow back to the logical +/// schema. `_tombstone` stays non-nullable so the base arm's synthesized +/// `Literal(false)` matches the WAL arms' real column under +/// `CoalesceFirstExec`'s exact-schema check. +fn carry_schema(canonical: &SchemaRef, pk_columns: &[String]) -> SchemaRef { + let widened = relax_non_pk_nullability(canonical, pk_columns); + let mut fields: Vec> = widened.fields().iter().cloned().collect(); fields.push(Arc::new(Field::new(TOMBSTONE, DataType::Boolean, false))); Arc::new(Schema::new(fields)) } @@ -758,9 +796,10 @@ fn carry_schema(canonical: &SchemaRef) -> SchemaRef { fn project_to_carry( plan: Arc, canonical: &SchemaRef, + pk_columns: &[String], ) -> Result> { let input = plan.schema(); - let carry = carry_schema(canonical); + let carry = carry_schema(canonical, pk_columns); let mut project_exprs: Vec<(Arc, String)> = Vec::with_capacity(carry.fields().len()); for field in carry.fields() { @@ -784,11 +823,11 @@ fn project_to_carry( }; project_exprs.push((expr, name.clone())); } - Ok(Arc::new( - ProjectionExec::try_new(project_exprs, plan).map_err(|e| { - lance_core::Error::internal(format!("Failed to build carry ProjectionExec: {}", e)) - })?, - )) + let projected = Arc::new(ProjectionExec::try_new(project_exprs, plan).map_err(|e| { + lance_core::Error::internal(format!("Failed to build carry ProjectionExec: {}", e)) + })?); + // `CoalesceFirstExec` panics unless every arm lands on exactly `carry`. + Ok(force_schema(projected, &carry)) } /// Drop tombstone rows after `CoalesceFirstExec` has already picked the newest @@ -873,6 +912,7 @@ fn probe_position( index_store: &IndexStore, pk_column: &str, pk_value: &ScalarValue, + visibility: MemTableVisibility, ) -> Result { // Visible batches are the committed prefix [0, last_visible_idx]; each // `StoredBatch` carries its cumulative `row_offset`, so visibility and the @@ -881,7 +921,12 @@ fn probe_position( if len == 0 { return Ok(ProbePos::Miss); } - let last_visible_idx = index_store.max_visible_batch_position().min(len - 1); + // The cursor is an exclusive count, so the last readable batch sits at + // `count - 1`. A count of 0 means nothing is readable yet — not "batch 0". + let readable_count = index_store.prefix_count(visibility).min(len); + let Some(last_visible_idx) = readable_count.checked_sub(1) else { + return Ok(ProbePos::Miss); + }; let last = batch_store.get(last_visible_idx).ok_or_else(|| { lance_core::Error::internal("point-lookup: visible batch index out of range") })?; @@ -981,8 +1026,9 @@ fn probe_memtable( pk_column: &str, pk_value: &ScalarValue, target: &SchemaRef, + visibility: MemTableVisibility, ) -> Result { - match probe_position(batch_store, index_store, pk_column, pk_value)? { + match probe_position(batch_store, index_store, pk_column, pk_value, visibility)? { ProbePos::NoIndex => Ok(Probe::NoIndex), ProbePos::Miss => Ok(Probe::Miss), ProbePos::Found { batch_idx, row } => { @@ -1008,6 +1054,7 @@ mod tests { use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray}; use arrow_schema::{DataType, Field, Schema as ArrowSchema}; use datafusion::physical_plan::displayable; + use rstest::rstest; use std::collections::HashMap; use uuid::Uuid; @@ -1100,7 +1147,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); // Create collector let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot]); @@ -1181,10 +1228,10 @@ mod tests { let base_path = temp_dir.path().to_str().unwrap(); // No base dataset is created. We still need a base URI so the collector - // can resolve flushed-generation paths. + // can resolve SSTable paths. let base_uri = format!("{}/base", base_path); - // Create a flushed generation under {base_uri}/_mem_wal/{shard}/gen_1 + // Create an SSTable under {base_uri}/_mem_wal/{shard}/gen_1 let shard_id = Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let gen1_batch = create_test_batch(&schema, &[2, 3], "gen1"); @@ -1192,12 +1239,12 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]); let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema); - // id=3 lives in the flushed generation + // id=3 lives in the SSTable let pk_values = vec![ScalarValue::Int32(Some(3))]; let plan = planner.plan_lookup(&pk_values, None).await.unwrap(); @@ -1378,8 +1425,9 @@ mod tests { let batch_store = Arc::new(BatchStore::with_capacity(16)); let mut index_store = IndexStore::new(); - // BTree on the PK so that `max_visible_batch_position` advances as - // we insert, otherwise the scanner sees no batches at all. + // BTree on the PK: the point lookup resolves keys through the indexed PK + // path, which this exercises. (`indexed_count`/`visible_count` advance + // from the batch position regardless of whether any index is configured.) index_store.add_btree("id_idx".to_string(), 0, "id".to_string()); // Two writes to pk=1, then an unrelated pk=2. The "new" row goes @@ -1519,11 +1567,94 @@ mod tests { ); } + /// The writer's own read at [`MemTableVisibility::Indexed`] resolves a row + /// whose WAL append is still outstanding, while the default `Published` + /// bound does not. The projection selects which read path runs: the fast + /// BTree probe, or the `MemTableScanner` plan fallback (a system column in + /// the output disqualifies the probe). Both must resolve the key alike. + #[rstest] + #[case::fast_btree_probe(None)] + #[case::scanner_fallback(Some(vec![ + "id".to_string(), + "name".to_string(), + "_rowid".to_string(), + ]))] + #[tokio::test] + async fn test_indexed_visibility_reads_the_undurable_prefix( + #[case] projection: Option>, + ) { + use crate::dataset::mem_wal::index::MemTableVisibility; + use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; + use crate::dataset::mem_wal::wal::WriterCursors; + use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; + + let schema = create_pk_schema(); + let temp_dir = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap()); + + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut index_store = IndexStore::new(); + index_store.enable_pk_index(&[("id".to_string(), 0)]); + // A writer whose durability cursor never advances: the batch indexes, + // but its append stays outstanding, so it never publishes. + index_store.set_durability(Arc::new(WriterCursors::new(true)), 0); + + let batch = create_test_batch(&schema, &[1], "pending"); + let (bp, off, _) = batch_store.append(batch.clone()).unwrap(); + index_store + .insert_with_batch_position(&batch, off, Some(bp)) + .unwrap(); + assert_eq!(index_store.indexed_count(), 1); + assert_eq!(index_store.visible_count(), 0, "the append is outstanding"); + let index_store = Arc::new(index_store); + + let shard_id = Uuid::new_v4(); + let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![]) + .with_in_memory_memtables( + shard_id, + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store, + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ); + let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema); + let key = [ScalarValue::Int32(Some(1))]; + + assert!( + planner + .lookup(&key, projection.as_deref()) + .await + .unwrap() + .is_none(), + "a row whose append is outstanding must stay invisible at Published" + ); + + let planner = planner.with_visibility(MemTableVisibility::Indexed); + let hit = planner + .lookup(&key, projection.as_deref()) + .await + .unwrap() + .expect("the writer must read its own indexed prefix"); + assert_eq!(hit.num_rows(), 1); + let name = hit + .column_by_name("name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(name.value(0), "pending_1"); + } + #[tokio::test] - async fn test_point_lookup_flushed_memtable_returns_newest_duplicate() { - // Regression / invariant pin: when a flushed memtable contains two + async fn test_point_lookup_sstable_returns_newest_duplicate() { + // Regression / invariant pin: when an SSTable contains two // rows for the same PK, the lookup must return the newer one. The - // flushed dataset is reverse-written (newest at the smallest + // SSTable dataset is reverse-written (newest at the smallest // physical position), so we simulate that here by writing the // dataset with the new row first. The point-lookup plan today // returns the first match (smallest `_rowid`) under reverse-write, @@ -1544,7 +1675,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]); let planner = LsmPointLookupPlanner::new(collector, vec!["id".to_string()], schema); @@ -1564,7 +1695,7 @@ mod tests { assert_eq!( name_arr.value(0), "new_1", - "flushed-arm lookup must return the row at the smallest _rowid (newest under reverse-write)" + "SSTable-arm lookup must return the row at the smallest _rowid (newest under reverse-write)" ); } diff --git a/rust/lance/src/dataset/mem_wal/scanner/projection.rs b/rust/lance/src/dataset/mem_wal/scanner/projection.rs index 0ec482aebf8..fd52ac90c0b 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/projection.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/projection.rs @@ -6,8 +6,8 @@ //! //! `MemTableScanner::project()` only special-cases `_rowid`; passing other //! system columns through it errors. And cross-LSM values for system -//! columns aren't comparable (a `_rowid` of 5 in the base and in a flushed -//! memtable refer to different rows). +//! columns aren't comparable (a `_rowid` of 5 in the base and in an +//! SSTable refer to different rows). //! //! - [`build_scanner_projection`] — strips system / `_distance` cols, appends PKs. //! - [`canonical_output_schema`] — final schema honoring user order; system @@ -25,6 +25,8 @@ use datafusion::physical_plan::projection::ProjectionExec; use datafusion::scalar::ScalarValue; use lance_core::{ROW_ADDR, ROW_ID, Result, is_system_column}; +use super::exec::SchemaRelabelExec; + /// Column name for distance in vector search results. pub const DISTANCE_COLUMN: &str = "_distance"; @@ -189,9 +191,26 @@ pub fn null_columns( Ok(Arc::new(projection_exec)) } +/// Force `plan` to report exactly `target_schema`; a no-op when they agree. +/// +/// `ProjectionExec` derives its nullability from the expressions, so the +/// storage schema's widened columns leave the WAL arms disagreeing with the +/// base arm — which `CoalesceFirstExec` and `concat_batches` both reject. +pub(super) fn force_schema( + plan: Arc, + target_schema: &SchemaRef, +) -> Arc { + if plan.schema() == *target_schema { + return plan; + } + Arc::new(SchemaRelabelExec::new(plan, target_schema.clone())) +} + /// Wrap `plan` to emit exactly `target_schema`. Source columns are /// forwarded by name; system / `_distance` cols missing from the source /// are NULL-filled. Other missing columns are an internal error. +/// +/// Reports `target_schema` exactly, nullability included — see [`force_schema`]. pub fn project_to_canonical( plan: Arc, target_schema: &SchemaRef, @@ -222,13 +241,15 @@ pub fn project_to_canonical( let projection_exec = ProjectionExec::try_new(project_exprs, plan).map_err(|e| { lance_core::Error::internal(format!("Failed to build canonical ProjectionExec: {}", e)) })?; - Ok(Arc::new(projection_exec)) + Ok(force_schema(Arc::new(projection_exec), target_schema)) } #[cfg(test)] mod tests { use super::*; + use arrow_array::RecordBatch; use arrow_schema::Schema as ArrowSchema; + use datafusion_physical_plan::test::TestMemoryExec; fn schema() -> SchemaRef { Arc::new(ArrowSchema::new(vec![ @@ -238,6 +259,20 @@ mod tests { ])) } + /// [`schema`] as `relax_non_pk_nullability` would leave it: `id` widened. + fn widened_schema() -> SchemaRef { + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, true), + Field::new("name", DataType::Utf8, true), + Field::new("vector", DataType::Float32, true), + ])) + } + + fn plan_emitting(schema: SchemaRef) -> Arc { + let batch = RecordBatch::new_empty(schema.clone()); + TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap() + } + #[test] fn scanner_projection_strips_system_and_distance() { let s = schema(); @@ -320,4 +355,30 @@ mod tests { // _distance dropped because include_distance=false (e.g. point lookup / scan). assert_eq!(names, vec!["vector", "id"]); } + + #[test] + fn force_schema_leaves_a_matching_plan_alone() { + let plan = plan_emitting(schema()); + let forced = force_schema(plan.clone(), &schema()); + assert!( + Arc::ptr_eq(&plan, &forced), + "a plan already reporting the target schema must not be wrapped" + ); + } + + #[test] + fn force_schema_relabels_a_nullability_mismatch() { + let forced = force_schema(plan_emitting(widened_schema()), &schema()); + assert_eq!(forced.name(), "SchemaRelabelExec"); + assert_eq!(forced.schema(), schema()); + } + + #[test] + fn project_to_canonical_reports_the_target_schema() { + // The ProjectionExec alone would follow its input and report `id` as + // nullable; the relabel is what pins the output to the target. + let target = schema(); + let plan = project_to_canonical(plan_emitting(widened_schema()), &target).unwrap(); + assert_eq!(plan.schema(), target); + } } diff --git a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs b/rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs similarity index 75% rename from rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs rename to rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs index 7a5280bedb8..9bb8dadcf02 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/flushed_cache.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/sstable_cache.rs @@ -1,9 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Cache of opened flushed-generation datasets for the LSM scanner. +//! Cache of opened SSTable datasets for the LSM scanner. //! -//! Flushed generations are written exactly once to a globally-unique, +//! SSTables are written exactly once to a globally-unique, //! content-addressed path (see `memtable/flush.rs`): a fresh random hash per //! flush invocation means the same path always maps to the same immutable //! bytes. A cached `Arc` therefore can never go stale and needs no @@ -11,11 +11,11 @@ //! optimization driven by the consumer at compaction time. //! //! ```text -//! query ──> open_flushed_dataset(path, session, cache) +//! query ──> open_sstable(path, session, cache) //! │ //! cache.is_some() ──────┤────── cache.is_none() //! │ │ -//! FlushedMemTableCache::get_or_open DatasetBuilder::from_uri +//! SsTableCache::get_or_open DatasetBuilder::from_uri //! (single-flight, shared Arc) (cold open every call) //! ``` @@ -24,31 +24,41 @@ use std::sync::Arc; use async_trait::async_trait; use lance_core::{Error, Result}; +use lance_io::object_store::ObjectStoreParams; use crate::dataset::{Dataset, DatasetBuilder}; use crate::session::Session; -/// Cache of opened flushed-generation datasets, keyed by resolved path. +/// Cache of opened SSTable datasets, keyed by resolved path. /// -/// Flushed generations live at a globally-unique, immutable path, so cached +/// SSTables live at a globally-unique, immutable path, so cached /// entries are never stale and require no TTL. Intended to be held by a /// long-lived owner (one per process or per table) and injected into /// per-request scanners via [`crate::dataset::mem_wal::scanner::LsmScanner`] /// (and the point-lookup / vector-search planners). /// -/// The key is the resolved absolute flushed path +/// The key is the resolved absolute SSTable path /// (`{base}/_mem_wal/{shard}/{folder}`), which is globally unique, so a single /// cache can safely span multiple tables. -pub struct FlushedMemTableCache { +/// +/// `store_params` is deliberately *not* part of the key: the first caller to +/// open a path binds the store that every later hit reuses. Credential rotation +/// still works — a vended-credential store holds the live +/// `StorageOptionsAccessor` and re-resolves per request, so a cached handle +/// never carries expired credentials. What this does assume is that a given +/// path is only ever served under one store configuration. Serving one table +/// through a single cache under two different `ObjectStoreParams` would hand +/// every caller the store the first one opened with. +pub struct SsTableCache { // `moka`'s async cache gives a bounded size plus single-flight // `try_get_with`, so concurrent first-queries on a just-flushed - // generation open the dataset exactly once. The opened dataset carries the + // SSTable open the dataset exactly once. The opened dataset carries the // session index cache, which also backs each generation's standalone PK // dedup index (see `block_list::open_pk_index`) — no separate cache path. inner: moka::future::Cache>, } -impl FlushedMemTableCache { +impl SsTableCache { /// Create a cache holding at most `max_entries` opened datasets. /// /// Eviction is size-only (no TTL): an evicted-then-re-requested generation @@ -66,16 +76,14 @@ impl FlushedMemTableCache { } /// Get the dataset for `path`, opening it (exactly once) on a miss. - /// - /// `session` is threaded into the open so the first open populates the - /// shared index / file-metadata caches; subsequent hits are a pure - /// `Arc::clone` with zero object-store I/O. Concurrent callers for the - /// same path share a single open via `moka`'s single-flight - /// `try_get_with`. + /// Concurrent callers share a single open via `moka`'s single-flight + /// `try_get_with`; hits are a pure `Arc::clone`. `session` / `store_params` + /// configure the open. pub async fn get_or_open( &self, path: &str, session: Option>, + store_params: Option, ) -> Result> { self.inner .try_get_with(path.to_string(), async move { @@ -83,6 +91,9 @@ impl FlushedMemTableCache { if let Some(session) = session { builder = builder.with_session(session); } + if let Some(store_params) = store_params { + builder = builder.with_store_params(store_params); + } builder.load().await.map(Arc::new) }) .await @@ -108,36 +119,46 @@ impl FlushedMemTableCache { } } -impl std::fmt::Debug for FlushedMemTableCache { +impl std::fmt::Debug for SsTableCache { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("FlushedMemTableCache") + f.debug_struct("SsTableCache") .field("entry_count", &self.inner.entry_count()) .finish() } } -/// Caching of opened flushed-generation datasets, keyed by immutable path. The +/// Caching of opened SSTable datasets, keyed by immutable path. The /// opened dataset carries the session index cache, which also backs each /// generation's secondary indexes and its PK dedup sidecar (see /// `block_list::open_pk_index`) — so a single `get_or_open` is the -/// whole caching surface. Implemented by [`FlushedMemTableCache`]; a -/// [`GenerationWarmer`] composes one to warm through it, and a consumer may +/// whole caching surface. Implemented by [`SsTableCache`]; a +/// [`SsTableWarmer`] composes one to warm through it, and a consumer may /// supply its own implementation. #[async_trait] pub trait DatasetCache: Send + Sync + std::fmt::Debug { - async fn get_or_open(&self, path: &str, session: Option>) -> Result>; + async fn get_or_open( + &self, + path: &str, + session: Option>, + store_params: Option, + ) -> Result>; /// Drop cached entries whose path is not in `live_paths`. Async so an /// implementation can evict retired generations' index objects (e.g. /// `Session::invalidate_index_prefix`) without a later breaking signature - /// change; [`FlushedMemTableCache`]'s own eviction is synchronous. + /// change; [`SsTableCache`]'s own eviction is synchronous. async fn retain_paths(&self, live_paths: &HashSet); } #[async_trait] -impl DatasetCache for FlushedMemTableCache { - async fn get_or_open(&self, path: &str, session: Option>) -> Result> { - Self::get_or_open(self, path, session).await +impl DatasetCache for SsTableCache { + async fn get_or_open( + &self, + path: &str, + session: Option>, + store_params: Option, + ) -> Result> { + Self::get_or_open(self, path, session, store_params).await } async fn retain_paths(&self, live_paths: &HashSet) { @@ -145,7 +166,7 @@ impl DatasetCache for FlushedMemTableCache { } } -/// Proactively warms a flushed generation into the shared caches: open the +/// Proactively warms an SSTable into the shared caches: open the /// dataset and pre-load its secondary indexes and PK dedup sidecar so the first /// query sees no cold reads. This is the **seam** the flush and read paths fire /// — lance defines it; the consumer (e.g. the WAL pod) implements it. `None` => @@ -161,34 +182,42 @@ impl DatasetCache for FlushedMemTableCache { /// and cheap when the path is already warm** (e.g. dedup in-flight and /// completed paths) — a redundant call must not re-do work or fail. #[async_trait] -pub trait GenerationWarmer: Send + Sync + std::fmt::Debug { +pub trait SsTableWarmer: Send + Sync + std::fmt::Debug { async fn warm(&self, path: &str) -> Result<()>; } -/// Open a flushed-generation dataset, shared by all three LSM open sites +/// Open an SSTable dataset, shared by all three LSM open sites /// (scan, point lookup, vector search). /// /// - `cache` present: route through a [`DatasetCache`] (e.g. -/// [`FlushedMemTableCache`]: single-flight, shared `Arc`, manifest read +/// [`SsTableCache`]: single-flight, shared `Arc`, manifest read /// amortized across queries). /// - `cache` absent: cold open via [`DatasetBuilder`]. Passing `session` /// still reuses the shared index / metadata caches; `None`/`None` /// reproduces the original per-query cold-open behavior exactly. /// - `warmer` present: fire a fire-and-forget warm-on-open backstop behind the /// returned handle (the warmer dedups already-warm paths). `None` => no warming. -pub async fn open_flushed_dataset( +pub async fn open_sstable( path: &str, session: Option<&Arc>, + store_params: Option<&ObjectStoreParams>, cache: Option<&Arc>, - warmer: Option<&Arc>, + warmer: Option<&Arc>, ) -> Result> { let dataset = match cache { - Some(cache) => cache.get_or_open(path, session.cloned()).await?, + Some(cache) => { + cache + .get_or_open(path, session.cloned(), store_params.cloned()) + .await? + } None => { let mut builder = DatasetBuilder::from_uri(path); if let Some(session) = session { builder = builder.with_session(session.clone()); } + if let Some(store_params) = store_params { + builder = builder.with_store_params(store_params.clone()); + } Arc::new(builder.load().await?) } }; @@ -238,9 +267,9 @@ mod tests { let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap()); write_dataset(&uri, &[1, 2, 3]).await; - let cache = FlushedMemTableCache::new(8); - let first = cache.get_or_open(&uri, None).await.unwrap(); - let second = cache.get_or_open(&uri, None).await.unwrap(); + let cache = SsTableCache::new(8); + let first = cache.get_or_open(&uri, None, None).await.unwrap(); + let second = cache.get_or_open(&uri, None, None).await.unwrap(); assert!( Arc::ptr_eq(&first, &second), @@ -261,7 +290,7 @@ mod tests { let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap()); write_dataset(&uri, &[1, 2, 3]).await; - let cache = Arc::new(FlushedMemTableCache::new(8)); + let cache = Arc::new(SsTableCache::new(8)); let calls = Arc::new(AtomicUsize::new(0)); let mut handles = Vec::new(); @@ -271,7 +300,7 @@ mod tests { let calls = calls.clone(); handles.push(tokio::spawn(async move { calls.fetch_add(1, Ordering::SeqCst); - cache.get_or_open(&uri, None).await.unwrap() + cache.get_or_open(&uri, None, None).await.unwrap() })); } @@ -298,9 +327,9 @@ mod tests { write_dataset(&keep_uri, &[1]).await; write_dataset(&drop_uri, &[2]).await; - let cache = FlushedMemTableCache::new(8); - cache.get_or_open(&keep_uri, None).await.unwrap(); - cache.get_or_open(&drop_uri, None).await.unwrap(); + let cache = SsTableCache::new(8); + cache.get_or_open(&keep_uri, None, None).await.unwrap(); + cache.get_or_open(&drop_uri, None, None).await.unwrap(); cache.inner.run_pending_tasks().await; assert_eq!(cache.inner.entry_count(), 2); @@ -314,15 +343,15 @@ mod tests { } #[tokio::test] - async fn test_open_flushed_dataset_no_cache_matches_direct_open() { + async fn test_open_sstable_no_cache_matches_direct_open() { // The `None`/`None` path must reproduce a plain cold open: same data, // independent Arc per call (no caching). let temp_dir = tempfile::tempdir().unwrap(); let uri = format!("{}/gen_1", temp_dir.path().to_str().unwrap()); write_dataset(&uri, &[7, 8, 9]).await; - let a = open_flushed_dataset(&uri, None, None, None).await.unwrap(); - let b = open_flushed_dataset(&uri, None, None, None).await.unwrap(); + let a = open_sstable(&uri, None, None, None, None).await.unwrap(); + let b = open_sstable(&uri, None, None, None, None).await.unwrap(); assert!( !Arc::ptr_eq(&a, &b), "no-cache path must cold-open each call" @@ -330,11 +359,11 @@ mod tests { assert_eq!(a.count_rows(None).await.unwrap(), 3); // With a cache, the second call is a shared clone. - let cache: Arc = Arc::new(FlushedMemTableCache::new(8)); - let c = open_flushed_dataset(&uri, None, Some(&cache), None) + let cache: Arc = Arc::new(SsTableCache::new(8)); + let c = open_sstable(&uri, None, None, Some(&cache), None) .await .unwrap(); - let d = open_flushed_dataset(&uri, None, Some(&cache), None) + let d = open_sstable(&uri, None, None, Some(&cache), None) .await .unwrap(); assert!(Arc::ptr_eq(&c, &d), "cached path must reuse the Arc"); @@ -348,7 +377,7 @@ mod tests { } #[async_trait] - impl GenerationWarmer for NotifyingWarmer { + impl SsTableWarmer for NotifyingWarmer { async fn warm(&self, _path: &str) -> Result<()> { self.calls.fetch_add(1, Ordering::SeqCst); self.notify.notify_one(); @@ -357,7 +386,7 @@ mod tests { } #[tokio::test] - async fn test_open_flushed_dataset_fires_warm_on_open() { + async fn test_open_sstable_fires_warm_on_open() { // The warm-on-open backstop fires the warmer (fire-and-forget) when a // generation is opened, so generations the flusher never warmed still // get warmed lazily on first read. @@ -367,12 +396,12 @@ mod tests { let calls = Arc::new(AtomicUsize::new(0)); let notify = Arc::new(tokio::sync::Notify::new()); - let warmer: Arc = Arc::new(NotifyingWarmer { + let warmer: Arc = Arc::new(NotifyingWarmer { calls: calls.clone(), notify: notify.clone(), }); - let ds = open_flushed_dataset(&uri, None, None, Some(&warmer)) + let ds = open_sstable(&uri, None, None, None, Some(&warmer)) .await .unwrap(); assert_eq!(ds.count_rows(None).await.unwrap(), 3); diff --git a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs index 59f721aa08c..1363efa8f7e 100644 --- a/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs +++ b/rust/lance/src/dataset/mem_wal/scanner/vector_search.rs @@ -28,19 +28,20 @@ use crate::io::exec::TakeExec; use super::collector::LsmDataSourceCollector; use super::data_source::LsmDataSource; -use super::flushed_cache::{DatasetCache, GenerationWarmer, open_flushed_dataset}; use super::projection::{ DISTANCE_COLUMN, build_scanner_projection, canonical_output_schema, null_columns, project_to_canonical, validate_projection_names, wants_row_id, }; +use super::sstable_cache::{DatasetCache, SsTableWarmer, open_sstable}; use crate::session::Session; +use lance_io::object_store::ObjectStoreParams; /// Plans vector search queries over LSM data. /// /// Each source is independently newest-per-PK before the union — the active /// memtable via exact brute-force KNN when PK rewrites or a filter require it /// (append-only active data can still use HNSW), -/// flushed generations via their within-generation deletion vector — and the +/// SSTables via their within-generation deletion vector — and the /// cross-generation block-list ([`super::exec::PkBlockFilterExec`]) drops any /// PK superseded by a newer generation. So each PK reaches the union from /// exactly one source and a distance-ordered merge yields the global top-k; no @@ -58,9 +59,9 @@ use crate::session::Session; /// MemTableBruteForceVectorExec or VectorIndexExec: active memtable KNN /// ProjectionExec (canonical output schema) /// ProjectionExec (null_columns _rowid) -/// PkBlockFilterExec: block-list (flushed) -/// KNNExec: flushed gen N, fetch=ceil(k*overfetch) (fast_search) -/// … one per flushed gen … +/// PkBlockFilterExec: block-list (SSTable) +/// KNNExec: SSTable gen N, fetch=ceil(k*overfetch) (fast_search) +/// … one per SSTable gen … /// ProjectionExec (canonical output schema) /// PkBlockFilterExec: block-list (base) /// KNNExec: base table, k (fast_search)[.refine()?] @@ -68,11 +69,11 @@ use crate::session::Session; /// /// # Index-Only Search (fast_search) /// -/// For base table and flushed memtables we use `fast_search()` to only +/// For base table and SSTables we use `fast_search()` to only /// search indexed data. This is correct because: -/// - Each flushed memtable has its own vector index built during flush. +/// - Each SSTable has its own vector index built during flush. /// - The active memtable covers any unindexed data. -/// - Searching unindexed data in base/flushed would be redundant. +/// - Searching unindexed data in base/SSTable would be redundant. pub struct LsmVectorSearchPlanner { /// Data source collector. collector: LsmDataSourceCollector, @@ -89,17 +90,22 @@ pub struct LsmVectorSearchPlanner { /// the per-source KNN output. Memtable rows already carry all columns; /// the take only fetches additional data for base rows (real `_rowid`). dataset: Option>, - /// Session threaded into flushed-generation opens (shared caches). + /// Session threaded into SSTable opens (shared caches). session: Option>, - /// Cache of opened flushed-generation datasets. - flushed_cache: Option>, - /// Optional warmer fired on first open of a flushed generation. - warmer: Option>, + /// Store params for opening SSTables, reusing the base dataset's store. + store_params: Option, + /// Cache of opened SSTable datasets. + sstable_cache: Option>, + /// Optional warmer fired on first open of an SSTable. + warmer: Option>, /// Optional prefilter predicate applied to every source arm before its KNN /// search, so rows failing the predicate never enter the top-k. Base and - /// flushed arms use the dataset scanner's native prefilter; memtable arms + /// SSTable arms use the dataset scanner's native prefilter; memtable arms /// route to a filtered brute-force scan. filter: Option, + /// Optional `lower <= _distance < upper` bound, applied inside every source + /// arm's KNN so an out-of-range row never consumes a top-k slot. + distance_range: (Option, Option), } impl LsmVectorSearchPlanner { @@ -127,36 +133,52 @@ impl LsmVectorSearchPlanner { distance_type, dataset: None, session: None, - flushed_cache: None, + store_params: None, + sstable_cache: None, warmer: None, filter: None, + distance_range: (None, None), } } /// Attach an optional prefilter predicate. Every source arm restricts its /// KNN to rows matching the predicate (true prefilter), so results match a - /// normal filtered vector scan over base ∪ flushed ∪ in-memory data. + /// normal filtered vector scan over base ∪ SSTables ∪ in-memory data. pub fn with_filter(mut self, filter: Option) -> Self { self.filter = filter; self } - /// Thread a session into flushed-generation opens so the first open - /// populates the shared index / file-metadata caches. + /// Attach an optional distance range, `lower <= _distance < upper` — the + /// same half-open semantics as [`crate::dataset::scanner::Scanner::distance_range`]. + /// Every source arm applies it before its own top-k cut, so an out-of-range + /// row can't displace an in-range one. + pub fn with_distance_range(mut self, lower: Option, upper: Option) -> Self { + self.distance_range = (lower, upper); + self + } + + /// Set the session used to open SSTables. pub fn with_session(mut self, session: Arc) -> Self { self.session = Some(session); self } - /// Inject a cache of opened flushed-generation datasets, making repeated + /// Set the store params used to open SSTables. + pub fn with_store_params(mut self, store_params: ObjectStoreParams) -> Self { + self.store_params = Some(store_params); + self + } + + /// Inject a cache of opened SSTable datasets, making repeated /// searches against the same generation a pure `Arc::clone`. - pub fn with_flushed_cache(mut self, cache: Arc) -> Self { - self.flushed_cache = Some(cache); + pub fn with_sstable_cache(mut self, cache: Arc) -> Self { + self.sstable_cache = Some(cache); self } - /// Inject the warmer fired on first open of a flushed generation. - pub fn with_warmer(mut self, warmer: Arc) -> Self { + /// Inject the warmer fired on first open of an SSTable. + pub fn with_warmer(mut self, warmer: Arc) -> Self { self.warmer = Some(warmer); self } @@ -235,7 +257,8 @@ impl LsmVectorSearchPlanner { let block_lists = Box::pin(super::block_list::compute_source_block_lists( &sources, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), )) .await?; @@ -296,8 +319,8 @@ impl LsmVectorSearchPlanner { // * active: append-only memtables can use HNSW directly; once a // PK rewrite is observed, `MemTableBruteForceVectorExec` drops // superseded versions before the top-k cut. - // * flushed/base: drop cross-gen superseded rows via the - // block-list (within-gen is handled by the flushed DV). + // * SSTable/base: drop cross-gen superseded rows via the + // block-list (within-gen is handled by the SSTable DV). let knn = match blocked { Some(_) if self.pk_columns.is_empty() => knn, Some(set) => Arc::new(super::exec::PkBlockFilterExec::new( @@ -431,6 +454,7 @@ impl LsmVectorSearchPlanner { } let query_arr = single_query_array(query_vector); scanner.nearest(&self.vector_column, query_arr.as_ref(), k)?; + scanner.distance_range(self.distance_range.0, self.distance_range.1); scanner.nprobes(nprobes); scanner.distance_metric(self.distance_type); // Memtables cover unindexed rows; only search indexed data here. @@ -442,11 +466,12 @@ impl LsmVectorSearchPlanner { } scanner.create_plan().await } - LsmDataSource::FlushedMemTable { path, .. } => { - let dataset = open_flushed_dataset( + LsmDataSource::SsTable { path, .. } => { + let dataset = open_sstable( path, self.session.as_ref(), - self.flushed_cache.as_ref(), + self.store_params.as_ref(), + self.sstable_cache.as_ref(), self.warmer.as_ref(), ) .await?; @@ -463,6 +488,7 @@ impl LsmVectorSearchPlanner { // No `with_row_id/address`: per-source IDs would collide with base. let query_arr = single_query_array(query_vector); scanner.nearest(&self.vector_column, query_arr.as_ref(), k)?; + scanner.distance_range(self.distance_range.0, self.distance_range.1); scanner.nprobes(nprobes); scanner.distance_metric(self.distance_type); scanner.fast_search(); @@ -492,6 +518,7 @@ impl LsmVectorSearchPlanner { scanner.filter_expr(filter.clone()); } scanner.nearest(&self.vector_column, query_vector, k)?; + scanner.distance_range(self.distance_range.0, self.distance_range.1); scanner.nprobes(nprobes); scanner.distance_metric(self.distance_type); scanner.create_plan().await @@ -669,7 +696,7 @@ mod tests { let dataset = Dataset::write(reader, uri, Some(WriteParams::default())) .await .unwrap(); - // Also write the standalone PK sidecar (on `id`) so a flushed-generation + // Also write the standalone PK sidecar (on `id`) so an SSTable // source can be probed by the block-list (harmless for a base table). if has_id { crate::dataset::mem_wal::scanner::block_list::write_pk_sidecar(uri, &batches, &["id"]) @@ -847,7 +874,7 @@ mod tests { out_cols ); // Internal columns must not leak: `_rowid` (added by Lance's fast_search - // in the base/flushed arms) and `_memtable_gen` (added by the LSM merge + // in the base/SSTable arms) and `_memtable_gen` (added by the LSM merge // when bloom filters are present) are bookkeeping, not API. assert!( out_schema.field_with_name("_rowid").is_err(), @@ -1060,6 +1087,117 @@ mod tests { ); } + /// `distance_range` must bound the search itself, not its result. + /// + /// Vectors are `id -> [id*0.1, ..]` and the query is id=1's vector, so L2^2 + /// distances are id=1: 0.0, id=0 and id=2: 0.04, id=3: 0.16, id=4: 0.36. + /// + /// The lower-bound probe is the sharp one. It excludes the *nearest* rows, + /// which `VectorIndexExec` cannot honor: its HNSW search cuts to k first, so + /// a `k = 2` search returns id=1 and id=0/id=2 and the bound then drops both, + /// yielding nothing. Only the brute-force arm — which filters the complete + /// candidate set before its cut — gets this right, so a lower bound must + /// route there (see `MemTableScanner::plan_vector_search`). Regression for + /// that routing guard. + #[tokio::test] + async fn test_vector_search_distance_range_bounds_the_search() { + use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; + use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; + use datafusion::prelude::SessionContext; + use futures::TryStreamExt; + + let schema = create_vector_schema(); + let temp_dir = tempfile::tempdir().unwrap(); + let base_uri = format!("{}/base", temp_dir.path().to_str().unwrap()); + // Base rows are far and unindexed, so `fast_search` contributes nothing; + // the test isolates the memtable arms. + let base_dataset = Arc::new( + create_dataset(&base_uri, vec![create_test_batch(&schema, &[100, 200])]).await, + ); + + let build_collector = || { + let batch_store = Arc::new(BatchStore::with_capacity(16)); + let mut index_store = IndexStore::new(); + index_store.enable_pk_index(&[("id".to_string(), 0)]); + // An HNSW index must exist, or the arm falls back to brute force for + // an unrelated reason and the routing guard goes untested. + index_store.add_hnsw( + "vector_hnsw".to_string(), + 1, + "vector".to_string(), + lance_linalg::distance::DistanceType::L2, + 64, + 8, + ); + let batch = create_test_batch(&schema, &[0, 1, 2, 3, 4]); + batch_store.append(batch.clone()).unwrap(); + index_store + .insert_with_batch_position(&batch, 0, Some(0)) + .unwrap(); + LsmDataSourceCollector::new(base_dataset.clone(), vec![]).with_in_memory_memtables( + uuid::Uuid::new_v4(), + InMemoryMemTables { + active: InMemoryMemTableRef { + batch_store, + index_store: Arc::new(index_store), + schema: schema.clone(), + generation: 1, + }, + frozen: vec![], + }, + ) + }; + + let run = async |lower: Option, upper: Option, k: usize| -> Vec { + let planner = LsmVectorSearchPlanner::new( + build_collector(), + vec!["id".to_string()], + schema.clone(), + "vector".to_string(), + lance_linalg::distance::DistanceType::L2, + ) + .with_distance_range(lower, upper); + let plan = planner + .plan_search(&create_query_vector(), k, 1, None, false, 1.0) + .await + .expect("planner should produce a bounded plan"); + let stream = plan.execute(0, SessionContext::new().task_ctx()).unwrap(); + let batches: Vec = stream.try_collect().await.unwrap(); + let mut ids: Vec = Vec::new(); + for b in &batches { + let col = b + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..b.num_rows() { + ids.push(col.value(i)); + } + } + ids.sort(); + ids + }; + + // `_distance >= 0.1` keeps only id=3 (0.16) and id=4 (0.36). A top-k cut + // taken before the bound would have returned id=1/id=0/id=2 and then + // filtered them all away, leaving nothing. + assert_eq!( + run(Some(0.1), None, 2).await, + vec![3, 4], + "a lower bound must restrict the search: the two nearest in-range \ + rows are id=3 and id=4, not an empty result" + ); + + // `_distance < 0.1` keeps id=0, id=1, id=2. Safe on the HNSW arm — it + // trims the far tail the top-k would have dropped anyway. + assert_eq!( + run(None, Some(0.1), 10).await, + vec![0, 1, 2], + "an upper bound must drop id=3 (0.16) and id=4 (0.36)" + ); + } + #[tokio::test] async fn test_vector_search_filtered_active_without_pk_keeps_all_matching_rows() { use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; @@ -1215,13 +1353,13 @@ mod tests { ); } - /// The flushed arm must also apply the filter as a true prefilter, and that + /// The SSTable arm must also apply the filter as a true prefilter, and that /// prefiltered candidate set must compose with cross-generation block-list /// filtering plus over-fetch. Gen 1's closest predicate-matching row (id=3) /// is superseded by gen 2; with over-fetch, gen 1 should still contribute /// the next live predicate match (id=4). #[tokio::test] - async fn test_vector_search_flushed_prefilter_composes_with_block_list() { + async fn test_vector_search_sstable_prefilter_composes_with_block_list() { use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::index::DatasetIndexExt; use crate::index::vector::VectorIndexParams; @@ -1260,8 +1398,8 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]); let planner = LsmVectorSearchPlanner::new( @@ -1286,7 +1424,7 @@ mod tests { assert_eq!(rows.len(), 1, "expected one result, got {:?}", rows); assert_eq!( rows[0].0, 4, - "flushed prefilter should return live id=4 after stale id=3 is blocked; got {:?}", + "SSTable prefilter should return live id=4 after stale id=3 is blocked; got {:?}", rows ); } @@ -1996,14 +2134,14 @@ mod tests { #[tokio::test] async fn test_vector_search_dedup_across_generations() { // Regression: same primary key inserted into two sources (older - // flushed gen and newer active memtable) with different vectors. - // Without the cross-source PK dedup the older flushed row would + // SSTable gen and newer active memtable) with different vectors. + // Without the cross-source PK dedup the older SSTable row would // still appear in top-k. The newer-generation row must win. // - // We simulate a "flushed gen 1" by writing a tiny Lance dataset + // We simulate a "SSTable gen 1" by writing a tiny Lance dataset // under {base_uri}/_mem_wal/{shard}/gen_1 and pointing the // collector at it. Real flush would reverse-write, but for this - // test we only have one row in the flushed gen so order is moot. + // test we only have one row in the SSTable gen so order is moot. use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::dataset::mem_wal::write::{BatchStore, IndexStore}; @@ -2015,7 +2153,7 @@ mod tests { let base_path = temp_dir.path().to_str().unwrap(); let base_uri = format!("{}/base", base_path); - // Flushed gen 1 holds an older version of pk=1 with a "wrong" vector. + // SSTable gen 1 holds an older version of pk=1 with a "wrong" vector. let shard_id = uuid::Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); let old_pk1 = create_test_batch_with_vector(&schema, 1, [9.0, 9.0, 9.0, 9.0]); @@ -2048,7 +2186,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![shard_snapshot]) .with_in_memory_memtables( shard_id, @@ -2104,7 +2242,7 @@ mod tests { async fn test_vector_search_system_columns_real_only_for_base() { // Covers three properties of the per-source system columns: // 1. base-hit `_rowid`/`_rowaddr` carry real values - // 2. flushed-memtable arm runs without erroring + // 2. SSTable arm runs without erroring // 3. `_rowaddr` symmetry with `_rowid` (same code path, both are // surfaced when requested and NULL'd outside the base arm) use crate::dataset::mem_wal::scanner::collector::{InMemoryMemTableRef, InMemoryMemTables}; @@ -2131,7 +2269,7 @@ mod tests { .unwrap(); let base_dataset = Arc::new(base_dataset); - // Flushed memtable: id=2 (a separate Lance dataset under + // SSTable: id=2 (a separate Lance dataset under // {base_uri}/_mem_wal/{shard}/gen_1) with its own vector index. let shard_id = uuid::Uuid::new_v4(); let gen1_uri = format!("{}/_mem_wal/{}/gen_1", base_uri, shard_id); @@ -2163,7 +2301,7 @@ mod tests { let shard_snapshot = ShardSnapshot::new(shard_id) .with_current_generation(2) - .with_flushed_generation(1, "gen_1".to_string()); + .with_sstable(1, "gen_1".to_string()); let collector = LsmDataSourceCollector::new(base_dataset, vec![shard_snapshot]) .with_in_memory_memtables( @@ -2237,10 +2375,10 @@ mod tests { "`_rowaddr` is incompatible with vector_search's fast_search; must be NULL" ); - // id=2 (flushed): both NULL — per-source values would collide with base. - let (rid_null, raddr_null) = seen.get(&2).expect("flushed row id=2 missing"); - assert!(rid_null, "flushed row `_rowid` must be NULL"); - assert!(raddr_null, "flushed row `_rowaddr` must be NULL"); + // id=2 (SSTable): both NULL — per-source values would collide with base. + let (rid_null, raddr_null) = seen.get(&2).expect("SSTable row id=2 missing"); + assert!(rid_null, "SSTable row `_rowid` must be NULL"); + assert!(raddr_null, "SSTable row `_rowaddr` must be NULL"); // id=3 (active): both NULL — BatchStore position is not a Lance row id. let (rid_null, raddr_null) = seen.get(&3).expect("active row id=3 missing"); @@ -2909,9 +3047,9 @@ mod tests { } #[tokio::test] - async fn test_vector_search_flushed_superseded_by_newer_flushed() { - // An older flushed generation's stale row must be suppressed by a newer - // flushed generation (cross-flushed blocking, no base/active involved). + async fn test_vector_search_sstable_superseded_by_newer_sstable() { + // An older SSTable's stale row must be suppressed by a newer + // SSTable (cross-SSTable blocking, no base/active involved). use crate::dataset::mem_wal::scanner::data_source::ShardSnapshot; use crate::index::DatasetIndexExt; use crate::index::vector::VectorIndexParams; @@ -2946,8 +3084,8 @@ mod tests { let snapshot = ShardSnapshot::new(shard_id) .with_current_generation(3) - .with_flushed_generation(1, "gen_1".to_string()) - .with_flushed_generation(2, "gen_2".to_string()); + .with_sstable(1, "gen_1".to_string()) + .with_sstable(2, "gen_2".to_string()); let collector = LsmDataSourceCollector::without_base_table(base_uri, vec![snapshot]); let planner = LsmVectorSearchPlanner::new( diff --git a/rust/lance/src/dataset/mem_wal/test_util.rs b/rust/lance/src/dataset/mem_wal/test_util.rs index 43a3861e686..abafbb7c68f 100644 --- a/rust/lance/src/dataset/mem_wal/test_util.rs +++ b/rust/lance/src/dataset/mem_wal/test_util.rs @@ -1,12 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -//! Test-only object store that injects WAL-write failures, for exercising the -//! WAL persistence-failure fencing path. +//! Test-only object store that injects WAL-write failures (for the WAL +//! persistence-failure fencing path) and records the paths it serves (for +//! asserting which opens actually resolved through a given `ObjectStoreParams`). use std::fmt::{Debug, Display, Formatter}; use std::ops::Range; use std::sync::Arc; +use std::sync::Mutex as StdMutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use bytes::Bytes; @@ -33,6 +35,10 @@ pub struct FailControls { simulate_lost_ack: AtomicBool, /// WAL-entry `put_opts` attempts observed, for assertions. wal_put_attempts: AtomicUsize, + /// Every location written through this store. + put_paths: StdMutex>, + /// Every location read through this store. + get_paths: StdMutex>, } impl FailControls { @@ -48,6 +54,26 @@ impl FailControls { pub fn attempts(&self) -> usize { self.wal_put_attempts.load(Ordering::SeqCst) } + + /// Did any write land on a path containing `needle`? An open that resolved + /// its store from other params never reaches this store, so a `false` here + /// means the params under test did not reach that open. + pub fn wrote_under(&self, needle: &str) -> bool { + self.put_paths + .lock() + .unwrap() + .iter() + .any(|p| p.contains(needle)) + } + + /// Did any read land on a path containing `needle`? See [`Self::wrote_under`]. + pub fn read_under(&self, needle: &str) -> bool { + self.get_paths + .lock() + .unwrap() + .iter() + .any(|p| p.contains(needle)) + } } /// Wraps the inner store with [`FailingObjectStore`] at construction. @@ -63,6 +89,15 @@ impl WrappingObjectStore for FailingWrapper { controls: self.controls.clone(), }) } + + // Injects behaviour into every request, so a listing must not go around it. + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } /// Delegates everything to `inner`, failing WAL-entry PUTs per [`FailControls`]. @@ -101,6 +136,11 @@ impl OSObjectStore for FailingObjectStore { payload: PutPayload, opts: PutOptions, ) -> OSResult { + self.controls + .put_paths + .lock() + .unwrap() + .push(location.to_string()); if Self::is_wal_entry(location) { self.controls .wal_put_attempts @@ -124,14 +164,30 @@ impl OSObjectStore for FailingObjectStore { location: &Path, opts: PutMultipartOptions, ) -> OSResult> { + // Data files (`*.lance`) are written multipart, not via `put_opts`. + self.controls + .put_paths + .lock() + .unwrap() + .push(location.to_string()); self.inner.put_multipart_opts(location, opts).await } async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult { + self.controls + .get_paths + .lock() + .unwrap() + .push(location.to_string()); self.inner.get_opts(location, options).await } async fn get_ranges(&self, location: &Path, ranges: &[Range]) -> OSResult> { + self.controls + .get_paths + .lock() + .unwrap() + .push(location.to_string()); self.inner.get_ranges(location, ranges).await } @@ -167,9 +223,11 @@ impl OSObjectStore for FailingObjectStore { } } -/// Build an in-memory `ObjectStore` whose WAL-entry writes can be failed on -/// demand. Returns the store, its base path, and the shared controls. -pub async fn failing_memory_store() -> (Arc, Path, Arc) { +/// `ObjectStoreParams` carrying the observable store wrapper, plus the controls +/// to drive and inspect it. Open a dataset with these and every store resolved +/// *from these params* — the base and any derived URI they are threaded to — +/// reports its traffic back through the controls. +pub fn observable_store_params() -> (ObjectStoreParams, Arc) { let controls = Arc::new(FailControls::default()); let params = ObjectStoreParams { object_store_wrapper: Some(Arc::new(FailingWrapper { @@ -177,6 +235,13 @@ pub async fn failing_memory_store() -> (Arc, Path, Arc (Arc, Path, Arc) { + let (params, controls) = observable_store_params(); let (store, base) = ObjectStore::from_uri_and_params( Arc::new(ObjectStoreRegistry::default()), "memory:///", diff --git a/rust/lance/src/dataset/mem_wal/util.rs b/rust/lance/src/dataset/mem_wal/util.rs index 3f5090f6b40..a51f269cb67 100644 --- a/rust/lance/src/dataset/mem_wal/util.rs +++ b/rust/lance/src/dataset/mem_wal/util.rs @@ -3,6 +3,7 @@ //! Utility functions for MemWAL operations. +use lance_io::object_store::ObjectStoreParams; use object_store::path::Path; use uuid::Uuid; @@ -91,7 +92,8 @@ impl WatchableOnceCellReader { /// optimizing S3 throughput by spreading sequential writes across internal partitions. /// /// # Example -/// ```ignore +/// ``` +/// # use lance::dataset::mem_wal::util::bit_reverse_u64; /// // 5 in binary: 000...101 /// // Reversed: 101...000 /// assert_eq!(bit_reverse_u64(5), 0xa000000000000000); @@ -129,6 +131,26 @@ pub fn parse_bit_reversed_filename(filename: &str) -> Option { Some(bit_reverse_u64(reversed)) } +/// Adapt the store params a base dataset was opened with for use on a URI +/// *derived* from it (an SSTable under `_mem_wal/`). +/// +/// The deprecated `object_store` binding pins a store to one location: given +/// `Some((store, url))`, both `ObjectStore::from_uri_and_params` and +/// `DatasetBuilder::build_object_store` take the path from `url` and ignore the +/// URI they were asked to open. Carried onto a generation URI it would silently +/// redirect the open — and, on the flush path, the write — at the base table +/// itself. Drop it so the generation URI resolves its own store; everything +/// else (storage options, wrapper, credentials, block size) still carries over. +/// +/// Only the base's *own* URI may reuse the params verbatim. +pub(crate) fn derived_store_params(params: &ObjectStoreParams) -> ObjectStoreParams { + #[allow(deprecated)] + ObjectStoreParams { + object_store: None, + ..params.clone() + } +} + /// Path to the MemWAL root directory. /// /// Returns: `{base_path}/_mem_wal/` @@ -157,29 +179,24 @@ pub fn shard_manifest_path(base_path: &Path, shard_id: &Uuid) -> Path { shard_base_path(base_path, shard_id).join("manifest") } -/// Path to a flushed MemTable directory. +/// Path to an SSTable directory. /// /// Returns: `{base_path}/_mem_wal/{shard_id}/{random_hash}_gen_{generation}/` -pub fn flushed_memtable_path( - base_path: &Path, - shard_id: &Uuid, - random_hash: &str, - generation: u64, -) -> Path { +pub fn sstable_path(base_path: &Path, shard_id: &Uuid, random_hash: &str, generation: u64) -> Path { shard_base_path(base_path, shard_id).join(format!("{}_gen_{}", random_hash, generation)) } -/// Subdirectory of a flushed generation holding its standalone primary-key +/// Subdirectory of an SSTable holding its standalone primary-key /// dedup index (a sidecar BTree, not registered in the manifest). Both the /// flush writer and the block-list probe join this onto the generation path. pub const PK_INDEX_DIR: &str = "_pk_index"; -/// Path to a flushed generation's standalone primary-key dedup index. +/// Path to an SSTable's standalone primary-key dedup index. pub fn pk_index_path(gen_path: &Path) -> Path { gen_path.clone().join(PK_INDEX_DIR) } -/// Generate an 8-character random hex string for flushed MemTable directories. +/// Generate an 8-character random hex string for SSTable directories. pub fn generate_random_hash() -> String { let bytes: [u8; 4] = rand::random(); format!( @@ -288,7 +305,7 @@ mod tests { ); assert_eq!( - flushed_memtable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(), + sstable_path(&base_path, &shard_id, "a1b2c3d4", 5).as_ref(), "my/dataset/_mem_wal/550e8400-e29b-41d4-a716-446655440000/a1b2c3d4_gen_5" ); @@ -372,4 +389,40 @@ mod tests { drop(cell); assert_eq!(handle.await.unwrap(), None); } + + /// The path-bound store binding is the only thing dropped — credentials and + /// storage options must still reach the generation's store. + #[test] + fn test_derived_store_params_drops_only_the_path_bound_store() { + let accessor = lance_io::object_store::StorageOptionsAccessor::with_static_options( + std::collections::HashMap::from([("access_key_id".to_string(), "key".to_string())]), + ); + #[allow(deprecated)] + let params = ObjectStoreParams { + object_store: Some(( + std::sync::Arc::new(object_store::memory::InMemory::new()), + url::Url::parse("memory:///base").unwrap(), + )), + block_size: Some(1234), + storage_options_accessor: Some(std::sync::Arc::new(accessor)), + ..Default::default() + }; + + let derived = derived_store_params(¶ms); + + #[allow(deprecated)] + { + assert!( + derived.object_store.is_none(), + "a store pinned to the base path must not be reused for a generation URI" + ); + } + assert_eq!(derived.block_size, Some(1234)); + assert_eq!( + derived + .storage_options() + .and_then(|o| o.get("access_key_id")), + Some(&"key".to_string()), + ); + } } diff --git a/rust/lance/src/dataset/mem_wal/wal.rs b/rust/lance/src/dataset/mem_wal/wal.rs index b4caff180e1..eaac7af3526 100644 --- a/rust/lance/src/dataset/mem_wal/wal.rs +++ b/rust/lance/src/dataset/mem_wal/wal.rs @@ -9,7 +9,8 @@ use std::io::Cursor; use std::sync::Arc; use std::sync::Mutex as StdMutex; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::MutexGuard as StdMutexGuard; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::time::Instant; use std::time::Duration; @@ -24,7 +25,6 @@ use lance_core::{Error, FenceReason, Result}; use lance_io::object_store::ObjectStore; use object_store::ObjectStoreExt; use object_store::path::Path; -use object_store::{PutMode, PutOptions}; use tokio::sync::{Mutex, mpsc, watch}; use tracing::instrument; @@ -87,68 +87,218 @@ impl WalFlushFailure { } } -/// Watcher for batch durability using watermark-based tracking. +/// The writer's cursors, shared by the WAL-append task, the index-apply task, +/// every memtable's `IndexStore`, and every `put` waiting to become visible. /// -/// Uses a shared watch channel that broadcasts the durable watermark. -/// The watcher waits until the watermark reaches or exceeds its target batch ID. -#[derive(Clone)] -pub struct BatchDurableWatcher { - /// Watch receiver for the durable watermark. - rx: watch::Receiver, - /// Target batch ID to wait for. - target_batch_position: usize, - /// Terminal flush failure shared with the flusher. When set, the watermark - /// can never reach the target, so `wait` returns this typed error instead of - /// blocking forever. +/// Two cursors are stored, one view is derived: +/// +/// - `durable` — writer-global count of WAL-durable batches, advanced by the +/// WAL-append task. Exclusive; 0 means none. +/// - `indexed` — per-memtable count of indexed batches, advanced by the +/// index-apply task. It lives on the memtable's own `IndexStore`, not here, +/// because each memtable has its own indexes. +/// - `visible` — **derived, never stored**: a batch is visible once it is +/// indexed and, under `durable_write`, also durable. +/// +/// Deriving `visible` rather than caching it is deliberate. A cached +/// `min(indexed, durable)` recomputed by two independent tasks is the classic +/// store-buffer race: with `Release`/`Acquire` each task can read the other's +/// *pre-store* value, so both compute a minimum below the true one, and a +/// max-clamped publish then leaves the cached value permanently short. A `put` +/// blocked on it would hang until some unrelated write happened to move a cursor +/// again. With nothing cached there is nothing to leave stale — `notify` is a +/// bare wake-up and every waiter recomputes from the cursors themselves. +pub struct WriterCursors { + durable: AtomicUsize, + /// Bumped whenever a cursor advances or the writer poisons. Carries no + /// value; it exists only to wake waiters, which then recompute. + notify_tx: watch::Sender, + notify_rx: watch::Receiver, + /// First terminal failure. Shared so a poisoned writer wakes every waiter + /// with the typed error, instead of leaving it blocked on a cursor that can + /// never advance again. terminal_error: Arc>>, + /// Whether durability is part of visibility. Per-writer, not per-write: + /// `visible` is a writer-wide definition, so a mix would need two visibility + /// views over one memtable. + durable_write: bool, +} + +impl WriterCursors { + pub fn new(durable_write: bool) -> Self { + let (notify_tx, notify_rx) = watch::channel(0); + Self { + durable: AtomicUsize::new(0), + notify_tx, + notify_rx, + terminal_error: Arc::new(StdMutex::new(None)), + durable_write, + } + } + + /// Writer-global count of WAL-durable batches. + pub fn durable(&self) -> usize { + self.durable.load(Ordering::Acquire) + } + + pub fn durable_write(&self) -> bool { + self.durable_write + } + + /// Advance the durability cursor. Monotonic, so an out-of-order completion + /// can never walk it backwards. + pub(crate) fn advance_durable(&self, global_count: usize) { + self.durable.fetch_max(global_count, Ordering::AcqRel); + self.wake(); + } + + /// Wake every waiter so it recomputes. Called after any cursor advances, and + /// after the writer poisons. + pub(crate) fn wake(&self) { + self.notify_tx.send_modify(|version| *version += 1); + } + + /// The visible prefix of one memtable, given its indexed prefix and its + /// writer-global coordinate. + pub fn visible_count(&self, indexed_count: usize, global_offset: usize) -> usize { + if !self.durable_write { + return indexed_count; + } + indexed_count.min(self.durable().saturating_sub(global_offset)) + } + + /// Lock `terminal_error`, ignoring mutex poisoning. + /// + /// A panic under this lock cannot tear the `Option` it guards: the sole + /// writer builds the value first and assigns it whole, so a panic mid-section + /// leaves the slot exactly as it was. Surfacing poison as an error instead + /// would mask the latched `FenceReason` behind an unrelated "mutex poisoned" + /// precisely when a caller needs the real reason — and would strand + /// `mark_terminal_failure`, which has no error to return. + fn lock_terminal_error(&self) -> StdMutexGuard<'_, Option> { + self.terminal_error + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + pub(crate) fn check_poisoned(&self) -> Result<()> { + if let Some(failure) = self.lock_terminal_error().clone() { + return Err(failure.into_error()); + } + Ok(()) + } + + pub(crate) fn mark_terminal_failure(&self, error: &Error) { + { + let mut slot = self.lock_terminal_error(); + if slot.is_none() { + *slot = Some(WalFlushFailure::from_error(error)); + } + } + // Wake waiters without advancing anything: each re-checks `terminal_error` + // and returns the error rather than blocking on a cursor that can no + // longer move. + self.wake(); + } +} + +/// Blocks a `put` until its batches are visible: indexed, and — in durable mode +/// — WAL-durable too. +/// +/// Recomputes the condition on every wake instead of comparing against a cached +/// watermark. `notify` only says "something moved"; this decides what that means. +pub struct BatchDurableWatcher { + cursors: Arc, + rx: watch::Receiver, + /// The memtable the batches landed in; its `indexed_count` is the apply + /// cursor being waited on. `None` in WAL-only mode, which has no indexes. + indexes: Option>, + /// Local exclusive count this write needs indexed. + target_indexed: usize, + /// Writer-global exclusive count this write needs durable. Batch positions + /// restart at 0 in every memtable while the durability cursor spans the + /// writer's whole life, so the caller must globalize this. + target_durable: usize, } impl BatchDurableWatcher { - /// Create a new watcher for a specific batch ID. pub fn new( - rx: watch::Receiver, - target_batch_position: usize, - terminal_error: Arc>>, + cursors: Arc, + indexes: Option>, + target_indexed: usize, + target_durable: usize, ) -> Self { + let rx = cursors.notify_rx.clone(); Self { + cursors, rx, - target_batch_position, - terminal_error, + indexes, + target_indexed, + target_durable, } } - /// Wait until the batch is durable. - /// - /// Returns Ok(()) when `durable_watermark >= target_batch_position`, or - /// Err if a terminal flush failure (e.g. a fence) means the watermark can - /// never reach the target. + /// Whether the write's batches are indexed — the weaker half of + /// [`Self::is_visible`], with the append possibly still outstanding. + fn is_indexed(&self) -> bool { + // WAL-only mode has no indexes, so there is nothing to index-wait on. + let indexed = match &self.indexes { + Some(indexes) => indexes.indexed_count(), + None => self.target_indexed, + }; + indexed >= self.target_indexed + } + + /// Whether the write is readable yet. + fn is_visible(&self) -> bool { + self.is_indexed() + && (!self.cursors.durable_write() || self.cursors.durable() >= self.target_durable) + } + + /// Wait until the write is visible, or until the writer poisons — in which + /// case no cursor will ever reach the target, so surface the typed error + /// rather than blocking forever. pub async fn wait(&mut self) -> Result<()> { + self.wait_until(Self::is_visible).await + } + + /// Wait until the write is indexed, leaving durability outstanding. Pairs + /// with [`MemTableVisibility::Indexed`](crate::dataset::mem_wal::MemTableVisibility::Indexed) + /// on the read side. + /// + /// Not an acknowledgement: a caller promising durability must still await + /// [`Self::wait`]. + pub async fn wait_indexed(&mut self) -> Result<()> { + self.wait_until(Self::is_indexed).await + } + + async fn wait_until(&mut self, reached: fn(&Self) -> bool) -> Result<()> { loop { - if let Some(failure) = self.terminal_error.lock().unwrap().clone() { - return Err(failure.into_error()); - } - let current = *self.rx.borrow(); - if current >= self.target_batch_position { + // Mark the current version seen *before* testing, so a wake-up landing + // between the test and `changed()` below is not lost. + self.rx.borrow_and_update(); + self.cursors.check_poisoned()?; + if reached(self) { return Ok(()); } self.rx .changed() .await - .map_err(|_| Error::io("Durable watermark channel closed"))?; + .map_err(|_| Error::io("Writer cursor channel closed"))?; } } - /// Check if the batch is already durable (non-blocking). + /// Non-blocking check. pub fn is_durable(&self) -> bool { - *self.rx.borrow() >= self.target_batch_position + self.is_visible() } } impl std::fmt::Debug for BatchDurableWatcher { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("BatchDurableWatcher") - .field("target_batch_position", &self.target_batch_position) - .field("current_watermark", &*self.rx.borrow()) + .field("target_indexed", &self.target_indexed) + .field("target_durable", &self.target_durable) .finish() } } @@ -164,20 +314,15 @@ pub struct WalEntry { pub num_batches: usize, } -/// Result of a parallel WAL flush with index update. +/// Result of a WAL flush. Append-only: index application runs on its own task +/// (see `apply_index_range`), which records its own stats, so this no longer +/// carries index-update timing or row counts. #[derive(Debug, Clone)] pub struct WalFlushResult { /// WAL entry that was written (if any). pub entry: Option, /// Duration of WAL I/O operation. pub wal_io_duration: std::time::Duration, - /// Overall wall-clock duration of the index update operation. - /// This includes any overhead from thread scheduling and context switching. - pub index_update_duration: std::time::Duration, - /// Per-index update durations. Key is index name, value is duration. - pub index_update_duration_breakdown: std::collections::HashMap, - /// Number of rows indexed. - pub rows_indexed: usize, /// Size of WAL data written in bytes. pub wal_bytes: usize, } @@ -185,32 +330,142 @@ pub struct WalFlushResult { /// Source for a WAL flush — either a `BatchStore` range (MemTable mode) or /// a drainable in-memory pending queue (WAL-only mode). pub enum WalFlushSource { - /// MemTable mode: read a `[max_flushed+1, end_batch_position)` range - /// from a `BatchStore`. Indexes are updated in parallel with the WAL - /// append. - BatchStore { - batch_store: Arc, - indexes: Option>, - }, + /// MemTable mode: append the `[durable, end_batch_position)` range of a + /// `BatchStore` to the WAL. Append-only — the index apply runs on its own + /// task, so a failed append can no longer publish rows through it. + BatchStore { batch_store: Arc }, + /// Timer-driven: append whichever store still owes the WAL an append, + /// resolved when the message is *handled*, not when it is enqueued. + /// + /// Carries no store on purpose. `MessageFactory` is synchronous and cannot + /// take the async state lock, and capturing an `Arc` once at + /// handler construction would pin the first memtable forever. + /// + /// Resolution walks the live stores **oldest first** and takes the first with + /// `global_end() > durable`. It must not simply take "the active memtable": + /// a tick enqueued before a freeze is handled *after* it, would resolve to + /// the new memtable, and would append its batches ahead of the outgoing + /// memtable's tail. WAL entry positions are assigned in append-call order, + /// replay walks them ascending, row positions follow, and primary-key recency + /// is "newest visible row position wins" — so an out-of-order append silently + /// inverts dedup after a crash: the stale row wins. A full scan looks fine. + NextPending, /// WAL-only mode: drain all pending batches from the shared /// `WalOnlyState`. There are no in-memory indexes to update. WalOnly { state: Arc }, } impl WalFlushSource { - fn pending_count(&self) -> usize { + fn kind(&self) -> &'static str { match self { - Self::BatchStore { batch_store, .. } => batch_store.pending_wal_flush_count(), - Self::WalOnly { state } => state - .pending - .lock() - .ok() - .map(|p| p.batches.len()) - .unwrap_or(0), + Self::BatchStore { .. } => "BatchStore", + Self::WalOnly { .. } => "WalOnly", + Self::NextPending => "NextPending", } } } +/// Message to trigger an index apply. +/// +/// Carries the store the batches actually landed in, captured on the put path +/// *before* any freeze can rotate the memtable — pairing a new store with an old +/// store's end position would index the wrong range. +#[derive(Clone)] +pub struct TriggerIndexApply { + pub batch_store: Arc, + pub indexes: Arc, + /// Local exclusive end of the range to cover. + pub end_batch_position: usize, +} + +impl std::fmt::Debug for TriggerIndexApply { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TriggerIndexApply") + .field("end_batch_position", &self.end_batch_position) + .finish() + } +} + +/// What an `apply_index_range` call actually indexed. `rows_indexed == 0` +/// marks a routine coalesced no-op (the range was already covered) that must +/// not be recorded as an index update. +#[derive(Debug, Default, Clone, Copy)] +pub struct IndexApplyStats { + pub rows_indexed: usize, + /// Wall-clock time spent applying the range, including the thread-scheduling + /// overhead of the blocking hand-off. + pub duration: std::time::Duration, +} + +/// Apply a contiguous range of batches to a memtable's in-memory indexes. +/// +/// Runs on its own task, as the single sequential consumer of its own channel. +/// Being a single consumer is what makes it safe: it guarantees in-order, +/// contiguous ranges, which is exactly what `HnswGraph::insert_batch` requires — +/// it hard-rejects any range whose start is not `indexed_len`. Ordering comes +/// from the task, not from the flush interval, so triggering per-put is exactly +/// as safe as triggering on a timer. +/// +/// It has its own channel rather than sharing the WAL flusher's, because +/// `TaskDispatcher::run` awaits `handle()` inline: a shared channel would put a +/// ~100ms S3 PUT in front of every latency-sensitive index apply. +pub async fn apply_index_range( + cursors: &Arc, + message: TriggerIndexApply, +) -> Result { + let TriggerIndexApply { + batch_store, + indexes, + end_batch_position, + } = message; + + // Self-batching: a message handled while more puts queue behind it covers + // everything committed so far, so the ones behind it find their range already + // applied. Redundant messages are therefore *routine*, not exceptional, and + // must be a clean no-op — never a call into `insert_batches`, whose HNSW arm + // hard-rejects a non-contiguous start, and which is now terminal for the + // writer. + let start = indexes.indexed_count(); + if end_batch_position <= start { + return Ok(IndexApplyStats::default()); + } + + // Every position in the range must exist. `get` returns `None` only for a + // position past `committed_len`, so a hole means the caller asked to index a + // batch the store never committed. Silently skipping it would let + // `insert_batches` advance `indexed_count` past a never-indexed batch — + // rows counted visible but absent from every index. Fail loudly; the handler + // poisons and reopen rebuilds the indexes from the WAL. + let stored: Vec = (start..end_batch_position) + .map(|position| { + batch_store.get(position).cloned().ok_or_else(|| { + Error::internal(format!( + "index apply range [{start}, {end_batch_position}) is missing batch \ + position {position}; batch_store committed_len is {}", + batch_store.len() + )) + }) + }) + .collect::>()?; + + // `insert_batches` advances `indexed_count` itself, once every index has + // taken the batch. Time the whole hand-off so the recorded latency reflects + // the blocking-pool scheduling too, matching the old inline-flush measurement. + let rows_indexed: usize = stored.iter().map(|b| b.num_rows).sum(); + let apply_start = Instant::now(); + tokio::task::spawn_blocking(move || indexes.insert_batches(&stored)) + .await + .map_err(|e| Error::internal(format!("Index apply task panicked: {e}")))??; + let duration = apply_start.elapsed(); + + // Wake anything waiting to become visible. + cursors.wake(); + Ok(IndexApplyStats { + rows_indexed, + duration, + }) +} + /// Message to trigger a WAL flush. /// /// Carries a `source` describing where to read batches from (BatchStore range @@ -218,7 +473,7 @@ impl WalFlushSource { pub struct TriggerWalFlush { pub source: WalFlushSource, /// End batch position (exclusive). For `BatchStore`, flush batches after - /// `max_flushed_batch_position` up to this. For `WalOnly`, indicates the + /// the writer-global durable cursor up to this. For `WalOnly`, indicates the /// position the durability watermark must reach for callers waiting on /// this flush. Use `usize::MAX` to flush all pending batches. pub end_batch_position: usize, @@ -232,7 +487,7 @@ pub struct TriggerWalFlush { impl std::fmt::Debug for TriggerWalFlush { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TriggerWalFlush") - .field("pending_batches", &self.source.pending_count()) + .field("source", &self.source.kind()) .field("end_batch_position", &self.end_batch_position) .finish() } @@ -333,7 +588,7 @@ impl WalOnlyState { } /// Pending bytes (for size-based flush trigger). - pub fn estimated_size(&self) -> usize { + pub fn queue_bytes(&self) -> usize { self.pending .lock() .ok() @@ -361,11 +616,9 @@ impl WalOnlyState { /// shared `WalAppender`. The flusher delegates the actual WAL write to the /// appender, optionally running a parallel index update in MemTable mode. pub struct WalFlusher { - /// Watch channel sender for durable watermark. - /// Broadcasts the highest batch_position that is now durable. - durable_watermark_tx: watch::Sender, - /// Watch channel receiver for creating new watchers. - durable_watermark_rx: watch::Receiver, + /// The writer's cursors. Shared with the index-apply task and every + /// memtable's `IndexStore`, so all three agree on what is visible. + cursors: Arc, /// Underlying WAL append primitive — owns object store, epoch, and /// position discovery. wal_appender: Arc, @@ -377,35 +630,37 @@ pub struct WalFlusher { /// Created at construction and recreated after each flush. /// Used by backpressure to wait for WAL flushes. wal_flush_cell: std::sync::Mutex>>, - /// First terminal flush failure, shared with every `BatchDurableWatcher`. It - /// wakes durability waiters (the watermark never advances) and is read by - /// `check_poisoned` so the write path fails fast. - terminal_error: Arc>>, } impl WalFlusher { /// Create a new WAL flusher backed by an existing `WalAppender`. /// /// The appender owns object store, epoch, and position state. The - /// flusher adds the durability watermark, trigger channel, and - /// completion cell on top. + /// flusher adds the trigger channel and completion cell on top, and shares + /// the writer's cursors. pub fn new(wal_appender: Arc) -> Self { + // Defaults to durable visibility; the writer replaces this with cursors + // built from its own config. + Self::with_cursors(wal_appender, Arc::new(WriterCursors::new(true))) + } + + pub fn with_cursors(wal_appender: Arc, cursors: Arc) -> Self { let shard_id = wal_appender.shard_id(); - // Initialize durable watermark at 0 (no batches durable yet) - let (durable_watermark_tx, durable_watermark_rx) = watch::channel(0); - // Create initial WAL flush cell for backpressure let wal_flush_cell = WatchableOnceCell::new(); Self { - durable_watermark_tx, - durable_watermark_rx, + cursors, wal_appender, shard_id, flush_tx: None, wal_flush_cell: std::sync::Mutex::new(Some(wal_flush_cell)), - terminal_error: Arc::new(StdMutex::new(None)), } } + /// The writer's cursors, for the index-apply task and the memtables. + pub fn cursors(&self) -> &Arc { + &self.cursors + } + /// Set the flush channel for background flush handler. pub fn set_flush_channel(&mut self, tx: mpsc::UnboundedSender) { self.flush_tx = Some(tx); @@ -420,45 +675,65 @@ impl WalFlusher { /// /// Returns a `BatchDurableWatcher` that can be awaited for durability. /// The actual batch data is stored in the BatchStore. - pub fn track_batch(&self, batch_position: usize) -> BatchDurableWatcher { - // Return a watcher that waits for this batch to become durable - // batch_position is 0-indexed, so we wait for watermark > batch_position (i.e., >= batch_position + 1) + /// Watch a write until it becomes visible. + /// + /// `target_indexed` is a **memtable-local** exclusive count; `target_durable` + /// is a **writer-global** one. They are different coordinate spaces on + /// purpose: batch positions restart at 0 in every memtable, while the + /// durability cursor spans the writer's whole life. A local durable target + /// would already be satisfied by a *previous* memtable's appends, and would + /// ack a write that never reached the WAL. Callers globalize via + /// `BatchStore::global_offset`. + pub fn track_batch( + &self, + indexes: Option>, + target_indexed: usize, + target_durable: usize, + ) -> BatchDurableWatcher { BatchDurableWatcher::new( - self.durable_watermark_rx.clone(), - batch_position + 1, - Arc::clone(&self.terminal_error), + Arc::clone(&self.cursors), + indexes, + target_indexed, + target_durable, ) } - /// Latch a terminal flush failure and wake every durability waiter (the - /// watermark never advances, so they must observe the error, not block). + /// The writer-global WAL durability cursor: how many batches of this + /// writer's batch sequence are durable. Exclusive count; 0 means none. + pub fn durable(&self) -> usize { + self.cursors.durable() + } + + /// Advance the durability cursor and wake waiters. + pub(crate) fn advance_durable(&self, global_count: usize) { + self.cursors.advance_durable(global_count); + } + + /// Latch a terminal flush failure and wake every waiter (no cursor will + /// advance again, so they must observe the error rather than block). /// Idempotent: only the first failure is retained. fn mark_terminal_failure(&self, error: &Error) { - { - let mut slot = self.terminal_error.lock().unwrap(); - if slot.is_none() { - *slot = Some(WalFlushFailure::from_error(error)); - } - } - // Wake `wait`ers without advancing the watermark; each re-checks - // `terminal_error` and returns the error. - self.durable_watermark_tx.send_modify(|_| {}); + self.cursors.mark_terminal_failure(error); + } + + /// Latch a terminal failure from outside the flush path (the index-apply + /// task). Same effect: reads and writes fail fast, waiters wake with the + /// typed error, and recovery is reopen -> replay. + pub(crate) fn poison(&self, error: &Error) { + self.cursors.mark_terminal_failure(error); } /// Fail fast with the typed error if this writer has been fenced (by a peer - /// or its own persistence failure). The write path calls this before touching - /// the memtable so a poisoned writer can't diverge further. Recovery is to - /// reopen the shard (replay the WAL). + /// or its own persistence failure). Both the read and write paths call this + /// so a poisoned writer can neither diverge further nor serve a snapshot + /// that replay will not reproduce. Recovery is to reopen and replay. pub fn check_poisoned(&self) -> Result<()> { - if let Some(failure) = self.terminal_error.lock().unwrap().clone() { - return Err(failure.into_error()); - } - Ok(()) + self.cursors.check_poisoned() } /// Get the current durable watermark. pub fn durable_watermark(&self) -> usize { - *self.durable_watermark_rx.borrow() + self.cursors.durable() } /// Get a watcher for WAL flush completion. @@ -527,14 +802,16 @@ impl WalFlusher { end_batch_position: usize, ) -> Result { let result = match source { - WalFlushSource::BatchStore { - batch_store, - indexes, - } => { - self.flush_from_batch_store(batch_store, indexes.clone(), end_batch_position) + WalFlushSource::BatchStore { batch_store } => { + self.flush_from_batch_store(batch_store, end_batch_position) .await } WalFlushSource::WalOnly { state } => self.flush_from_wal_only(state).await, + // The handler resolves a tick to a concrete store before it gets + // here, because only it can take the async state lock. + WalFlushSource::NextPending => Err(Error::internal( + "WalFlushSource::NextPending must be resolved by the flush handler", + )), }; // A terminal failure means the watermark can never advance; latch the // poison so waiters wake with the typed error and later writes fail fast. @@ -546,80 +823,64 @@ impl WalFlusher { result } + /// Append this store's un-appended suffix to the WAL. **Append-only.** + /// + /// The index apply used to run here, concurrently, under a `tokio::join!`. + /// That was the source of the dirty read: `join!` runs both arms to + /// completion and does not cancel the index arm when the append fails, so a + /// failed append still advanced the cursor readers keyed off. The two + /// operations have nothing in common — one is an in-memory microsecond write, + /// the other a ~100ms S3 PUT billed per call — so they now run on separate + /// tasks with separate cursors, and this one only ever touches the WAL. async fn flush_from_batch_store( &self, batch_store: &BatchStore, - indexes: Option>, end_batch_position: usize, ) -> Result { - // Get current flush position from per-memtable watermark (inclusive) - // start_batch_position is the first batch to flush - let start_batch_position = batch_store - .max_flushed_batch_position() - .map(|w| w + 1) - .unwrap_or(0); - - // If we've already flushed past this end, nothing to do + // Where this store's un-appended suffix begins, derived from the + // writer-global durability cursor. `local_end` clamps a cursor that + // predates this memtable to 0 and one past its end to `committed_len`. + let start_batch_position = batch_store.local_end(self.durable()); + + // Already appended past this end: nothing to do. Redundant triggers are + // routine (a put and the freeze can both target the same range), so this + // is the common case, not an error. if start_batch_position >= end_batch_position { return Ok(empty_flush_result()); } - // Collect batches in range [start_batch_position, end_batch_position) - let mut stored_batches: Vec = - Vec::with_capacity(end_batch_position - start_batch_position); - - for batch_position in start_batch_position..end_batch_position { - if let Some(stored) = batch_store.get(batch_position) { - stored_batches.push(stored.clone()); - } - } - - if stored_batches.is_empty() { - return Ok(empty_flush_result()); - } + // Every position in the range must exist. `get` returns `None` only for a + // position past `committed_len`, so a hole means we were asked to append a + // batch the store never committed. Silently skipping it while still + // advancing durability to `end_batch_position` (below) would mark an + // un-appended batch durable and lose it on replay — a divergence that + // survives the crash and hides from a full scan. Return a terminal error + // so `flush` poisons the writer; reopen replays the WAL. + let stored_batches: Vec = (start_batch_position..end_batch_position) + .map(|batch_position| { + batch_store.get(batch_position).cloned().ok_or_else(|| { + Error::writer_poisoned(format!( + "WAL flush range [{start_batch_position}, {end_batch_position}) is \ + missing batch position {batch_position}; batch_store committed_len is {}", + batch_store.len() + )) + }) + }) + .collect::>()?; - let rows_to_index: usize = stored_batches.iter().map(|b| b.num_rows).sum(); let record_batches: Vec = stored_batches.iter().map(|s| s.data.clone()).collect(); - let appender = self.wal_appender.clone(); - let (append_result, index_result) = if let Some(idx_registry) = indexes { - let wal_future = async move { - let start = Instant::now(); - let r = appender.append(record_batches).await?; - Ok::<_, Error>((r, start.elapsed())) - }; - let index_future = async { - let start = Instant::now(); - let per_index = tokio::task::spawn_blocking(move || { - idx_registry.insert_batches_parallel(&stored_batches) - }) - .await - .map_err(|e| Error::internal(format!("Index update task panicked: {}", e)))??; - Ok::<_, Error>((start.elapsed(), per_index)) - }; - tokio::join!(wal_future, index_future) - } else { - let wal_future = async move { - let start = Instant::now(); - let r = appender.append(record_batches).await?; - Ok::<_, Error>((r, start.elapsed())) - }; - ( - wal_future.await, - Ok((std::time::Duration::ZERO, std::collections::HashMap::new())), - ) - }; - - let (append_result, wal_io_duration) = append_result?; - let (index_update_duration, index_update_duration_breakdown) = index_result?; - - // Update per-memtable watermark (inclusive: last batch ID that was flushed) - batch_store.set_max_flushed_batch_position(end_batch_position - 1); + let start = Instant::now(); + let append_result = self.wal_appender.append(record_batches).await?; + let wal_io_duration = start.elapsed(); - // Notify durability waiters (global channel) - let _ = self.durable_watermark_tx.send(end_batch_position); - // Signal WAL flush completion for backpressure waiters + // Advance the writer-global durability cursor and wake waiters. The range + // just appended is `[start, end)` *local to this store*, so it must be + // lifted into the writer's coordinate space before it is published — + // otherwise a fresh memtable's small local end would be compared against a + // cursor carrying a previous memtable's larger one. + self.advance_durable(batch_store.global_offset() + end_batch_position); self.signal_wal_flush_complete(); Ok(WalFlushResult { @@ -629,9 +890,6 @@ impl WalFlusher { num_batches: append_result.num_batches, }), wal_io_duration, - index_update_duration, - index_update_duration_breakdown, - rows_indexed: rows_to_index, wal_bytes: append_result.wal_bytes, }) } @@ -649,13 +907,15 @@ impl WalFlusher { let append_result = self.wal_appender.append(snapshot.batches).await?; let wal_io_duration = start.elapsed(); - // Append succeeded — remove the flushed batches from the front of - // the queue. Note: WAL-only mode does not use the global durability - // watermark (`durable_watermark_tx`) — `put_wal_only` waits on the - // per-trigger `done` cell instead — so we don't advance it here. - // Same for the wal-flush-completion cell, which is only consulted - // by MemTable-mode backpressure waiters. + // Append succeeded — remove the flushed batches from the front of the + // queue, then advance the writer-global durability cursor so durable + // `put_wal_only` callers waiting on their `BatchDurableWatcher` wake. + // The queue is strict FIFO with contiguous positions and its front + // always sits at the current watermark, so the newly-durable suffix + // ends at `durable + count`. Flushes are serialized on the single + // handler task, so this read-then-advance cannot race another flush. state.commit_flushed(snapshot.count); + self.advance_durable(self.durable() + snapshot.count); Ok(WalFlushResult { entry: Some(WalEntry { @@ -664,9 +924,6 @@ impl WalFlusher { num_batches: append_result.num_batches, }), wal_io_duration, - index_update_duration: std::time::Duration::ZERO, - index_update_duration_breakdown: std::collections::HashMap::new(), - rows_indexed: 0, wal_bytes: append_result.wal_bytes, }) } @@ -699,9 +956,6 @@ pub fn empty_flush_result() -> WalFlushResult { WalFlushResult { entry: None, wal_io_duration: std::time::Duration::ZERO, - index_update_duration: std::time::Duration::ZERO, - index_update_duration_breakdown: std::collections::HashMap::new(), - rows_indexed: 0, wal_bytes: 0, } } @@ -774,7 +1028,7 @@ impl WalEntryData { /// First valid WAL entry position. Positions are 1-based so that a /// `ShardManifest::replay_after_wal_entry_position` of 0 unambiguously means /// "no flush has ever stamped the cursor" — replay then starts at position 1 -/// without needing to consult `flushed_generations`, which an external +/// without needing to consult `sstables`, which an external /// compactor may legitimately drain back to empty. const FIRST_WAL_ENTRY_POSITION: u64 = 1; const MAX_APPEND_CREATE_CONFLICTS: usize = 1024; @@ -1105,7 +1359,7 @@ impl WalAppender { } async fn discover_next_position(&self) -> Result { - if let Ok(Some(manifest)) = self.manifest_store.read_latest().await { + if let Ok(Some(manifest)) = self.manifest_store.latest().await { let hint = manifest.wal_entry_position_last_seen; if let Some(tip) = probe_forward_from( self.object_store.as_ref(), @@ -1128,14 +1382,18 @@ impl WalAppender { /// hint for `next_position()`, probing forward from the hint to find the true /// tip before falling back to a full directory listing. /// -/// Successful `read_entry` calls asynchronously update -/// `wal_entry_position_last_seen` in the shard manifest (fire-and-forget). +/// The highest position read is tracked in memory, so `next_position()` costs +/// nothing after the first entry. Publishing that cursor for other processes is +/// the epoch holder's job — a tailer holds no claim and writes no manifests. #[derive(Debug, Clone)] pub struct WalTailer { object_store: Arc, wal_dir: Path, manifest_store: Arc, shard_id: Uuid, + /// Highest entry position this tailer has read; 0 until it reads one. + /// Shared across clones so they pool what they have seen. + highest_read: Arc, } impl WalTailer { @@ -1152,12 +1410,12 @@ impl WalTailer { wal_dir: shard_wal_path(&base_path, &shard_id), manifest_store, shard_id, + highest_read: Arc::new(AtomicU64::new(0)), } } /// Read a WAL entry at the given position. Returns `None` if no entry exists. - /// On success, asynchronously updates `wal_entry_position_last_seen` in the - /// shard manifest as a best-effort cursor hint for future readers. + /// On success, records the position as this tailer's cursor. pub async fn read_entry(&self, entry_position: u64) -> Result> { let path = self .wal_dir @@ -1181,10 +1439,8 @@ impl WalTailer { })?; let (writer_epoch, batches) = deserialize_appender_batches(bytes)?; - let ms = self.manifest_store.clone(); - tokio::spawn(async move { - let _ = best_effort_cursor_update(&ms, entry_position).await; - }); + self.highest_read + .fetch_max(entry_position, Ordering::Relaxed); Ok(Some(WalReadEntry { shard_id: self.shard_id, @@ -1196,7 +1452,7 @@ impl WalTailer { /// Find the next append position (one past the latest entry). pub async fn next_position(&self) -> Result { - if let Some(hint) = self.manifest_cursor_hint().await + if let Some(hint) = self.cursor_hint().await && let Some(tip) = self.probe_forward(hint).await? { return Ok(tip); @@ -1209,9 +1465,16 @@ impl WalTailer { scan_first_position(self.object_store.as_ref(), &self.wal_dir, self.shard_id).await } - async fn manifest_cursor_hint(&self) -> Option { - let manifest = self.manifest_store.read_latest().await.ok()??; - Some(manifest.wal_entry_position_last_seen) + /// Where to start probing for the WAL tip: what this tailer has already + /// read, or the cursor a previous process published. + async fn cursor_hint(&self) -> Option { + match self.highest_read.load(Ordering::Relaxed) { + 0 => { + let manifest = self.manifest_store.latest().await.ok()??; + Some(manifest.wal_entry_position_last_seen) + } + read => Some(read), + } } async fn probe_forward(&self, hint: u64) -> Result> { @@ -1342,53 +1605,17 @@ async fn atomic_put( bytes: Bytes, ) -> std::result::Result<(), AtomicPutError> { let path = dir.clone().join(filename); - if object_store.is_local() { - let temp = dir - .clone() - .join(format!("{}.tmp.{}", filename, Uuid::new_v4())); - object_store - .inner - .put(&temp, bytes.into()) - .await - .map_err(|e| { - AtomicPutError::Other(Error::io(format!("failed to write temp file: {}", e))) - })?; - match object_store.inner.rename_if_not_exists(&temp, &path).await { - Ok(()) => Ok(()), - Err(object_store::Error::AlreadyExists { .. }) => { - let _ = object_store.delete(&temp).await; - Err(AtomicPutError::AlreadyExists) - } - Err(e) => { - let _ = object_store.delete(&temp).await; - Err(AtomicPutError::Other(Error::io(format!( - "failed to create {} atomically: {}", - path, e - )))) - } - } - } else { - object_store - .inner - .put_opts( - &path, - bytes.into(), - PutOptions { - mode: PutMode::Create, - ..Default::default() - }, - ) - .await - .map_err(|e| match e { - object_store::Error::AlreadyExists { .. } - | object_store::Error::Precondition { .. } => AtomicPutError::AlreadyExists, - _ => AtomicPutError::Other(Error::io(format!( - "failed to create {} atomically: {}", - path, e - ))), - })?; - Ok(()) - } + object_store + .put_if_absent(&path, bytes.into()) + .await + .map_err(|error| match error { + object_store::Error::AlreadyExists { .. } + | object_store::Error::Precondition { .. } => AtomicPutError::AlreadyExists, + _ => AtomicPutError::Other(Error::io(format!( + "failed to create {} atomically: {}", + path, error + ))), + }) } /// Probe forward from a hint position to find the next unwritten position. @@ -1478,22 +1705,10 @@ async fn scan_first_position( Ok(min_position.unwrap_or(FIRST_WAL_ENTRY_POSITION)) } -async fn best_effort_cursor_update(manifest_store: &ShardManifestStore, entry_position: u64) { - let Ok(Some(manifest)) = manifest_store.read_latest().await else { - return; - }; - if entry_position <= manifest.wal_entry_position_last_seen { - return; - } - let mut updated = manifest; - updated.version += 1; - updated.wal_entry_position_last_seen = entry_position; - let _ = manifest_store.write(&updated).await; -} - #[cfg(test)] mod tests { use super::*; + use crate::dataset::mem_wal::index::MemTableVisibility; use crate::dataset::mem_wal::test_util::failing_memory_store; use arrow_array::{Int32Array, StringArray}; use arrow_schema::{DataType, Field, Schema}; @@ -1555,18 +1770,22 @@ mod tests { fn batch_store_source(batch_store: &Arc) -> WalFlushSource { WalFlushSource::BatchStore { batch_store: batch_store.clone(), - indexes: None, } } - fn batch_store_source_with_indexes( - batch_store: &Arc, - indexes: &Arc, - ) -> WalFlushSource { - WalFlushSource::BatchStore { - batch_store: batch_store.clone(), - indexes: Some(indexes.clone()), - } + /// Run the index-apply task's body, as the writer's index task would. + async fn apply_all(batch_store: &Arc, indexes: &Arc) -> Result<()> { + let cursors = Arc::new(WriterCursors::new(true)); + apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len(), + }, + ) + .await + .map(|_| ()) } #[tokio::test] @@ -1576,7 +1795,7 @@ mod tests { let buffer = build_test_flusher(store, &base_path, shard_id, 1); // Track a batch - let watcher = buffer.track_batch(0); + let watcher = buffer.track_batch(None, 0, 1); // Watcher should not be durable yet assert!(!watcher.is_durable()); @@ -1595,7 +1814,7 @@ mod tests { let batch_store = Arc::new(BatchStore::with_capacity(10)); batch_store.append(create_test_batch(&schema, 10)).unwrap(); - let mut watcher = flusher.track_batch(0); + let mut watcher = flusher.track_batch(None, 0, 1); // wait() must NOT resolve before the flush happens let result = @@ -1628,13 +1847,13 @@ mod tests { batch_store.append(batch2).unwrap(); // Track batch IDs in WAL flusher - let mut watcher1 = buffer.track_batch(0); - let mut watcher2 = buffer.track_batch(1); + let mut watcher1 = buffer.track_batch(None, 0, 1); + let mut watcher2 = buffer.track_batch(None, 0, 2); // Verify initial state assert!(!watcher1.is_durable()); assert!(!watcher2.is_durable()); - assert!(batch_store.max_flushed_batch_position().is_none()); + assert_eq!(buffer.durable(), 0); // Flush all pending batches let source = batch_store_source(&batch_store); @@ -1646,8 +1865,8 @@ mod tests { assert_eq!(entry.position, FIRST_WAL_ENTRY_POSITION); assert_eq!(entry.writer_epoch, 1); assert_eq!(entry.num_batches, 2); - // After flushing 2 batches (positions 0 and 1), max flushed position is 1 (inclusive) - assert_eq!(batch_store.max_flushed_batch_position(), Some(1)); + // Two batches appended => the writer-global durable count is 2 (exclusive). + assert_eq!(buffer.durable(), 2); // Watchers should be notified watcher1.wait().await.unwrap(); @@ -1658,12 +1877,19 @@ mod tests { // Regression test for the visibility-cursor bug: with an empty IndexStore // (the common case for WAL-managed tables that mirror an index-less base - // dataset), a WAL flush must still advance `max_visible_batch_position` so - // scanners can see every batch up to the durable position — not just - // batch 0. Before the fix, the cursor stayed at 0 for the lifetime of the - // memtable and scanners returned only the first row. + /// The index apply and the WAL append are separate tasks with separate + /// cursors. Appending makes a range durable; it does not index it. Indexing + /// makes it indexed; it does not make it durable. Only both together make it + /// visible. + /// + /// This also covers the empty-registry case (a memtable with no configured + /// indexes), which used to be skipped entirely by the flush's index arm and + /// so left the cursor stuck at 0 for the memtable's whole life. + #[rstest::rstest] + #[case::no_indexes(false)] + #[case::btree_index(true)] #[tokio::test] - async fn test_wal_flush_advances_visibility_with_empty_indexes() { + async fn test_append_and_index_advance_separate_cursors(#[case] with_btree: bool) { let (store, base_path, _temp_dir) = create_local_store().await; let shard_id = Uuid::new_v4(); let flusher = build_test_flusher(store, &base_path, shard_id, 1); @@ -1674,28 +1900,149 @@ mod tests { batch_store.append(create_test_batch(&schema, 5)).unwrap(); } - // Empty registry, mimicking a memtable with `index_configs = []`. - let indexes = Arc::new(IndexStore::new()); - assert_eq!(indexes.max_visible_batch_position(), 0); + let mut idx = IndexStore::new(); + if with_btree { + idx.add_btree("id_idx".to_string(), 0, "id".to_string()); + } + let indexes = Arc::new(idx); - let source = batch_store_source_with_indexes(&batch_store, &indexes); - flusher.flush(&source, batch_store.len()).await.unwrap(); + // The append alone makes the range durable and indexes nothing. + flusher + .flush(&batch_store_source(&batch_store), batch_store.len()) + .await + .unwrap(); + assert_eq!(flusher.durable(), 3); + assert_eq!(indexes.indexed_count(), 0); - // Cursor must advance to the highest flushed batch position (2), - // making all three batches visible to scanners. - assert_eq!(indexes.max_visible_batch_position(), 2); - assert_eq!(batch_store.max_flushed_batch_position(), Some(2)); + // The index apply alone advances the index cursor. + apply_all(&batch_store, &indexes).await.unwrap(); + assert_eq!(indexes.indexed_count(), 3); } - // Regression guard for the indexed path: with at least one BTree index - // configured, the cursor advance still fires (this was already working - // before the fix — keeping the test to lock in the behavior). + /// A WAL flush asked to cover a range past the store's committed length must + /// fail terminally, not silently short-append. The store is append-only, so + /// `get` returns `None` only past `committed_len`; skipping that position + /// while still advancing durability to `end_batch_position` would mark an + /// un-appended batch durable and lose it on replay. The flush must poison + /// instead, and durability must not move. #[tokio::test] - async fn test_wal_flush_advances_visibility_with_btree_index() { + async fn test_flush_rejects_range_past_committed_len() { let (store, base_path, _temp_dir) = create_local_store().await; let shard_id = Uuid::new_v4(); let flusher = build_test_flusher(store, &base_path, shard_id, 1); + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 5)).unwrap(); + batch_store.append(create_test_batch(&schema, 5)).unwrap(); + + // Two batches committed (positions 0, 1); ask to flush through position 2. + let err = flusher + .flush(&batch_store_source(&batch_store), batch_store.len() + 1) + .await + .unwrap_err(); + assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure)); + assert!( + err.to_string().contains("missing batch position 2"), + "unexpected error: {err}" + ); + + // Terminal: the flusher is poisoned and durability never advanced past + // what was actually appended. + assert!(flusher.check_poisoned().is_err()); + assert_eq!(flusher.durable(), 0); + } + + /// The index-apply path has the same invariant: a range past the committed + /// length must error rather than silently under-index and advance the cursor + /// past a batch that was never inserted. + #[tokio::test] + async fn test_index_apply_rejects_range_past_committed_len() { + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 5)).unwrap(); + + let indexes = Arc::new(IndexStore::new()); + let cursors = Arc::new(WriterCursors::new(true)); + + // One batch committed (position 0); ask to index through position 1. + let err = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len() + 1, + }, + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("missing batch position 1"), + "unexpected error: {err}" + ); + // The cursor did not advance past the hole. + assert_eq!(indexes.indexed_count(), 0); + } + + #[tokio::test] + async fn test_writer_cursors_advance_and_visibility() { + // The durable cursor starts at zero and advances to the value passed. + let cursors = WriterCursors::new(true); + assert_eq!(cursors.durable(), 0); + cursors.advance_durable(5); + assert_eq!(cursors.durable(), 5); + + // fetch_max: a lower value never walks the cursor backwards. + cursors.advance_durable(3); + assert_eq!(cursors.durable(), 5); + + // durable_write = true: visibility is clamped by the writer-global durable + // cursor, offset by this memtable's global coordinate. + assert!(cursors.durable_write()); + // global_offset 0: min(indexed = 10, durable = 5) = 5. + assert_eq!(cursors.visible_count(10, 0), 5); + // global_offset 2: min(indexed = 10, durable 5 - 2 = 3) = 3. + assert_eq!(cursors.visible_count(10, 2), 3); + // A memtable that starts past the durable cursor sees nothing. + assert_eq!(cursors.visible_count(10, 8), 0); + // Indexing, not durability, is the tighter bound here. + assert_eq!(cursors.visible_count(2, 0), 2); + + // durable_write = false: durability is not part of visibility, so the + // indexed count passes through unchanged regardless of the cursor. + let non_durable = WriterCursors::new(false); + assert!(!non_durable.durable_write()); + assert_eq!(non_durable.visible_count(7, 0), 7); + non_durable.advance_durable(1); + assert_eq!(non_durable.visible_count(7, 0), 7); + } + + #[tokio::test] + async fn test_writer_cursors_advance_is_monotonic() { + let cursors = Arc::new(WriterCursors::new(true)); + let mut handles = Vec::new(); + for target in [4usize, 1, 9, 3, 7, 2] { + let cursors = cursors.clone(); + handles.push(tokio::spawn(async move { + let before = cursors.durable(); + cursors.advance_durable(target); + // An advance can only move the cursor forward, never back — even + // when a smaller target races a larger one. + assert!(cursors.durable() >= before); + })); + } + for handle in handles { + handle.await.unwrap(); + } + // Whatever order the concurrent advances landed in, the cursor ends at the + // largest target. + assert_eq!(cursors.durable(), 9); + } + + /// The happy path of the index-apply task: a valid range advances the index + /// cursor to its end and reports exactly the rows it covered. + #[tokio::test] + async fn test_index_apply_advances_cursor_and_counts_rows() { let schema = create_test_schema(); let batch_store = Arc::new(BatchStore::with_capacity(10)); for _ in 0..3 { @@ -1705,12 +2052,38 @@ mod tests { let mut idx = IndexStore::new(); idx.add_btree("id_idx".to_string(), 0, "id".to_string()); let indexes = Arc::new(idx); - - let source = batch_store_source_with_indexes(&batch_store, &indexes); - flusher.flush(&source, batch_store.len()).await.unwrap(); - - assert_eq!(indexes.max_visible_batch_position(), 2); - assert_eq!(batch_store.max_flushed_batch_position(), Some(2)); + let cursors = Arc::new(WriterCursors::new(true)); + + let stats = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len(), + }, + ) + .await + .unwrap(); + + // Three batches of five rows each were indexed, and the cursor advanced to + // cover them. + assert_eq!(indexes.indexed_count(), 3); + assert_eq!(stats.rows_indexed, 15); + + // Re-applying the same range is a coalesced no-op: the cursor holds and no + // rows are recounted. + let repeat = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: batch_store.len(), + }, + ) + .await + .unwrap(); + assert_eq!(indexes.indexed_count(), 3); + assert_eq!(repeat.rows_indexed, 0); } #[tokio::test] @@ -1726,8 +2099,8 @@ mod tests { batch_store.append(create_test_batch(&schema, 5)).unwrap(); // Track batch IDs and flush all pending batches - let _watcher1 = buffer.track_batch(0); - let _watcher2 = buffer.track_batch(1); + let _watcher1 = buffer.track_batch(None, 0, 1); + let _watcher2 = buffer.track_batch(None, 0, 2); let source = batch_store_source(&batch_store); let result = buffer.flush(&source, batch_store.len()).await.unwrap(); let entry = result.entry.unwrap(); @@ -1905,7 +2278,7 @@ mod tests { // A durable put on the predecessor: stage a batch and track it. let batch_store = Arc::new(BatchStore::with_capacity(10)); batch_store.append(create_test_batch(&schema, 1)).unwrap(); - let mut watcher = flusher.track_batch(0); + let mut watcher = flusher.track_batch(None, 0, 1); // Flushing collides with the sentinel and fences. Both the flush result // and the watcher must report the fence — and the watcher must resolve @@ -1944,7 +2317,7 @@ mod tests { } #[tokio::test] - async fn test_wal_tailer_uses_manifest_cursor_hint() { + async fn test_wal_tailer_hints_from_memory_without_writing() { let (store, base_path, _temp_dir) = create_local_store().await; let shard_id = Uuid::new_v4(); let appender = WalAppender::open(store.clone(), base_path.clone(), shard_id, 0) @@ -1959,26 +2332,33 @@ mod tests { .unwrap(); } - let tailer = WalTailer::new(store.clone(), base_path.clone(), shard_id); + let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + let before = manifest_store + .refresh_latest() + .await + .unwrap() + .unwrap() + .version; + + let tailer = WalTailer::new(store, base_path, shard_id); let entry = tailer.read_entry(1).await.unwrap().unwrap(); assert_eq!(entry.entry_position, 1); - // Best-effort cursor update is async; poll briefly until it lands. - let manifest_store = ShardManifestStore::new(store, &base_path, shard_id, 2); - let mut hint = 0u64; - for _ in 0..50 { - if let Some(m) = manifest_store.read_latest().await.unwrap() { - hint = m.wal_entry_position_last_seen; - if hint >= 1 { - break; - } - } - tokio::time::sleep(std::time::Duration::from_millis(20)).await; - } - assert!(hint >= 1, "cursor hint never updated, last={hint}"); + // A tailer holds no claim, so it must not touch the manifest. Publishing + // the cursor is the epoch holder's job, on the replay path. + assert_eq!( + manifest_store + .refresh_latest() + .await + .unwrap() + .unwrap() + .version, + before, + "a tailer must not write manifests" + ); - // next_position must still resolve to one past the last appended entry. - // Three entries from a fresh shard land at 1, 2, 3, so next is 4. + // The hint now comes from what this tailer has read. Three entries from + // a fresh shard land at 1, 2, 3, so next is 4. assert_eq!(tailer.next_position().await.unwrap(), 4); } @@ -2067,6 +2447,198 @@ mod tests { ); } + /// A failed WAL append must not make rows visible, even when the index apply + /// has already taken them. + /// + /// The two now run on separate tasks, so an index apply that lands while the + /// append is failing advances `indexed_count` — which is fine, and + /// unavoidable: indexes are derived state and replay rebuilds them. What must + /// not happen is for that to make the rows *readable*, because they are not in + /// the WAL and replay will not reproduce them. + /// + /// Visibility is derived, not published, so this holds by construction: with + /// `durable_write`, `visible = min(indexed, durable)`, and a failed append + /// leaves `durable` at 0. + #[tokio::test] + async fn test_failed_append_indexes_but_stays_invisible() { + let (store, base, controls) = failing_memory_store().await; + let shard_id = Uuid::new_v4(); + controls.fail_wal_puts(usize::MAX); + let manifest_store = Arc::new(ShardManifestStore::new(store.clone(), &base, shard_id, 2)); + let (epoch, _) = manifest_store.claim_epoch(0).await.unwrap(); + let appender = Arc::new(WalAppender::with_claimed_epoch( + store, + base, + shard_id, + manifest_store, + epoch, + 0, + WalRetryConfig { + max_retries: 1, + base_delay: Duration::from_millis(1), + }, + )); + let cursors = Arc::new(WriterCursors::new(true)); + let flusher = WalFlusher::with_cursors(appender, Arc::clone(&cursors)); + + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 1)).unwrap(); + + let mut idx = IndexStore::new(); + idx.add_btree("id_idx".to_string(), 0, "id".to_string()); + idx.set_durability(Arc::clone(&cursors), 0); + let indexes = Arc::new(idx); + + // The index apply succeeds. + apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: 1, + }, + ) + .await + .unwrap(); + assert_eq!(indexes.indexed_count(), 1); + + // The append does not. + let err = flusher + .flush(&batch_store_source(&batch_store), batch_store.len()) + .await + .expect_err("the WAL PUT is failing, so the append must fail"); + assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure)); + + // Indexed, but not durable — so not visible. + assert_eq!(flusher.durable(), 0); + assert_eq!( + indexes.visible_count(), + 0, + "a row whose WAL append failed must never become readable" + ); + } + + /// `wait_indexed` clears on the index apply alone; `wait` still needs the + /// append. + #[tokio::test] + async fn test_wait_indexed_clears_before_durable() { + let cursors = Arc::new(WriterCursors::new(true)); + + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 1)).unwrap(); + + let mut idx = IndexStore::new(); + idx.add_btree("id_idx".to_string(), 0, "id".to_string()); + idx.set_durability(Arc::clone(&cursors), 0); + let indexes = Arc::new(idx); + + apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: 1, + }, + ) + .await + .unwrap(); + + // Indexed but not durable: the two bounds diverge. + assert_eq!(indexes.indexed_count(), 1); + assert_eq!(indexes.visible_count(), 0); + assert_eq!(indexes.prefix_count(MemTableVisibility::Published), 0); + assert_eq!(indexes.prefix_count(MemTableVisibility::Indexed), 1); + + let mut watcher = + BatchDurableWatcher::new(Arc::clone(&cursors), Some(indexes.clone()), 1, 1); + watcher + .wait_indexed() + .await + .expect("the index apply has landed"); + assert!( + tokio::time::timeout(Duration::from_millis(50), watcher.wait()) + .await + .is_err(), + "durability is still outstanding, so `wait` must not return" + ); + + // The append lands: now both clear. + cursors.advance_durable(1); + watcher.wait().await.expect("the append has landed"); + assert_eq!(indexes.visible_count(), 1); + } + + /// A poisoned writer wakes an index waiter with the typed error: its rows + /// may be indexed, but they are never going to exist. + #[tokio::test] + async fn test_wait_indexed_surfaces_a_poisoned_writer() { + let cursors = Arc::new(WriterCursors::new(true)); + let mut watcher = BatchDurableWatcher::new(Arc::clone(&cursors), None, 1, 1); + + cursors.mark_terminal_failure(&Error::io("the WAL PUT failed")); + + watcher + .wait_indexed() + .await + .expect_err("a poisoned writer must not hand back a clean index wait"); + } + + /// An index-apply failure poisons the writer. + /// + /// A partial apply cannot be rolled back — `insert_batches` joins every index + /// thread unconditionally, so a failure leaves the others fully applied, and + /// none of HNSW, FTS or BTree has a delete. Continuing would re-cover the + /// range on the next attempt and corrupt the indexes that *did* succeed. So + /// the failure is terminal: reads and writes fail fast, and recovery is + /// reopen -> replay, which rebuilds the indexes from the WAL. + #[tokio::test] + async fn test_index_failure_poisons_the_writer() { + let (store, base_path, _temp_dir) = create_local_store().await; + let shard_id = Uuid::new_v4(); + let flusher = build_test_flusher(store, &base_path, shard_id, 1); + let cursors = Arc::clone(flusher.cursors()); + + let schema = create_test_schema(); + let batch_store = Arc::new(BatchStore::with_capacity(10)); + batch_store.append(create_test_batch(&schema, 1)).unwrap(); + + // An HNSW index on `id`, which is an Int32 and not a vector, so every + // insert of this batch fails deterministically. `validate_index_configs` + // rejects this at shard open — that is what makes poison-and-replay + // terminating — so the store has to be built by hand to reach it at all. + let mut idx = IndexStore::new(); + idx.add_hnsw( + "bad_hnsw".to_string(), + 0, + "id".to_string(), + lance_linalg::distance::DistanceType::L2, + 128, + 8, + ); + let indexes = Arc::new(idx); + + let err = apply_index_range( + &cursors, + TriggerIndexApply { + batch_store: batch_store.clone(), + indexes: indexes.clone(), + end_batch_position: 1, + }, + ) + .await + .expect_err("indexing an Int32 column as a vector must fail"); + + // The index task latches it, exactly as `IndexApplyHandler` does. + flusher.poison(&err); + assert!( + flusher.check_poisoned().is_err(), + "the writer must be poisoned rather than limp on with a corrupt index" + ); + assert_eq!(indexes.visible_count(), 0); + } + // A persistence failure during flush latches the poison: the flush result, // `check_poisoned`, and the durability watcher all report the typed error // (rather than the watcher hanging on a watermark that never advances). @@ -2094,7 +2666,7 @@ mod tests { let schema = create_test_schema(); let batch_store = Arc::new(BatchStore::with_capacity(10)); batch_store.append(create_test_batch(&schema, 1)).unwrap(); - let mut watcher = flusher.track_batch(0); + let mut watcher = flusher.track_batch(None, 0, 1); let source = batch_store_source(&batch_store); let flush_err = flusher.flush(&source, batch_store.len()).await.unwrap_err(); @@ -2114,4 +2686,33 @@ mod tests { .expect_err("watcher must surface the poison"); assert_eq!(waited.fence_reason(), Some(FenceReason::PersistenceFailure)); } + + // A panic under the `terminal_error` lock must not cost the writer its + // latched failure. Recovery is reopen -> replay, which is driven by the real + // `FenceReason`; reporting the mutex poisoning instead would bury it. + #[tokio::test] + async fn test_poisoned_terminal_error_mutex_still_reports_typed_failure() { + let cursors = Arc::new(WriterCursors::new(true)); + cursors.mark_terminal_failure(&Error::writer_poisoned("injected persistence failure")); + + let terminal_error = Arc::clone(&cursors.terminal_error); + let panicked = std::thread::spawn(move || { + let _guard = terminal_error.lock().unwrap(); + panic!("poison the terminal error mutex"); + }) + .join(); + assert!(panicked.is_err()); + assert!(cursors.terminal_error.is_poisoned()); + + let error = cursors.check_poisoned().unwrap_err(); + assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure)); + assert!(error.to_string().contains("injected persistence failure")); + + // The latch still takes writes, and still keeps the first failure. + cursors.mark_terminal_failure(&Error::fenced_by_peer("later peer fence")); + assert_eq!( + cursors.check_poisoned().unwrap_err().fence_reason(), + Some(FenceReason::PersistenceFailure) + ); + } } diff --git a/rust/lance/src/dataset/mem_wal/write.rs b/rust/lance/src/dataset/mem_wal/write.rs index 1b505354813..fdf92d03afa 100644 --- a/rust/lance/src/dataset/mem_wal/write.rs +++ b/rust/lance/src/dataset/mem_wal/write.rs @@ -18,6 +18,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock as StdRwLock}; use std::time::{Duration, Instant}; +use arc_swap::ArcSwap; use arrow_array::{ArrayRef, BooleanArray, RecordBatch, new_null_array}; use arrow_schema::Schema as ArrowSchema; use async_trait::async_trait; @@ -25,7 +26,7 @@ use lance_core::datatypes::Schema; use lance_core::{Error, Result}; use lance_index::mem_wal::ShardManifest; use lance_index::vector::hnsw::builder::HnswBuildParams; -use lance_io::object_store::ObjectStore; +use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use log::{debug, error, info, warn}; use object_store::path::Path; use tokio::sync::{RwLock, mpsc}; @@ -37,6 +38,7 @@ use uuid::Uuid; pub use super::index::{ BTreeIndexConfig, BTreeMemIndex, FtsIndexConfig, HnswIndexConfig, IndexStore, MemIndexConfig, + MemIndexKind, validate_index_configs, }; pub use super::memtable::CacheConfig; pub use super::memtable::MemTable; @@ -47,12 +49,15 @@ pub use super::util::{WatchableOnceCell, WatchableOnceCellReader}; pub use super::wal::{WalEntry, WalEntryData, WalFlushFailure, WalFlushResult, WalFlusher}; use super::memtable::flush::TriggerMemTableFlush; -use super::scanner::GenerationWarmer; +use super::observer::WalObserver; +use super::scanner::InMemoryMemTableRef; +use super::scanner::SsTableWarmer; use super::wal::{ - BatchDurableWatcher, TriggerWalFlush, WalAppender, WalFlushSource, WalOnlyState, - WalRetryConfig, WalTailer, empty_flush_result, + BatchDurableWatcher, TriggerIndexApply, TriggerWalFlush, WalAppender, WalFlushSource, + WalOnlyState, WalRetryConfig, WalTailer, WriterCursors, apply_index_range, empty_flush_result, }; -use super::{TOMBSTONE, schema_with_tombstone}; +use super::{TOMBSTONE, relax_non_pk_nullability, schema_with_tombstone}; +use crate::session::Session; use super::manifest::ShardManifestStore; @@ -83,17 +88,6 @@ pub struct ShardWriterConfig { /// - Lower latency, batched S3 operations pub durable_write: bool, - /// Whether to update indexes synchronously on each write. - /// - /// When true: - /// - Newly written data is immediately searchable via indexes - /// - Higher latency due to index update overhead - /// - /// When false: - /// - Index updates are deferred - /// - New data may not appear in index-accelerated queries immediately - pub sync_indexed_write: bool, - /// Maximum WAL buffer size in bytes before triggering a flush. /// /// This is a soft threshold - write batches are atomic and won't be split. @@ -127,8 +121,9 @@ pub struct ShardWriterConfig { /// Maximum number of rows in a MemTable. /// - /// Used to pre-allocate the in-memory HNSW graph and vector storage - /// capacity. When the memtable reaches capacity, it will be flushed. + /// Sizes the in-memory index pre-allocation. The memtable seals before a + /// write that would carry it past this, and a single write larger than the + /// cap is rejected. /// Default: 100,000 rows pub max_memtable_rows: usize, @@ -165,23 +160,6 @@ pub struct ShardWriterConfig { /// Default: 30 seconds pub backpressure_log_interval: Duration, - /// Maximum rows to buffer before flushing to async indexes. - /// - /// Only applies when `sync_indexed_write` is false. Larger values enable - /// better vectorization but increase memory usage and latency before data - /// becomes searchable. - /// - /// Default: 10,000 rows - pub async_index_buffer_rows: usize, - - /// Maximum time to buffer before flushing to async indexes. - /// - /// Only applies when `sync_indexed_write` is false. Ensures bounded latency - /// for data to become searchable even during low write throughput. - /// - /// Default: 1 second - pub async_index_interval: Duration, - /// Interval for periodic stats logging. /// /// Stats (write throughput, backpressure events, memtable size) are logged @@ -191,14 +169,14 @@ pub struct ShardWriterConfig { pub stats_log_interval: Option, /// How long a frozen memtable lingers in memory after its flush commits, - /// before it is evicted and served only from the on-disk flushed dataset. + /// before it is evicted and served only from the on-disk SSTable dataset. /// /// `Duration::ZERO` (the default) disables retention: evict on commit, no /// sweep ticker. Correct for single-shot queries, which can't observe a /// generation evicted mid-read. /// /// A non-zero value is required only for queries split across reads (e.g. - /// fresh tier and base table read separately, then deduped): the flushed + /// fresh tier and base table read separately, then deduped): the SSTable /// dataset loses the per-batch boundaries that bound as-of membership /// (see [`crate::dataset::mem_wal::scanner::FreshTierWatermark`]), so a /// generation evicted between a query's reads can serve a stale row. Set it @@ -223,8 +201,7 @@ pub struct ShardWriterConfig { /// `durable_write` settings as MemTable mode. /// /// MemTable-tied tunables (`max_memtable_size`, `max_memtable_rows`, - /// `max_memtable_batches`, `sync_indexed_write`, `async_index_buffer_rows`, - /// `async_index_interval`) are ignored when `enable_memtable == false`. + /// `max_memtable_batches`) are ignored when `enable_memtable == false`. /// /// For raw single-entry synchronous atomic appends with no buffering and /// no background tasks, use `WalAppender` directly — it is a strictly @@ -237,7 +214,7 @@ pub struct ShardWriterConfig { /// These control the in-memory HNSW graph this writer builds for its /// MemTable (and, on flush, the on-disk graph serialized from it). They are /// a property of the writer that builds the MemTable, not of the index - /// definition: each flushed generation is independent, so different writers + /// definition: each SSTable is independent, so different writers /// may use different parameters. An index without an entry uses the default /// build parameters. `num_edges` is the HNSW graph degree (level 0 retains /// `2 * num_edges`), equivalent to FAISS's `M`. @@ -248,7 +225,36 @@ pub struct ShardWriterConfig { /// Optional warmer fired pre-commit for each new generation (zero cold reads /// on first query). Wired to the flusher; supplied by the consumer (e.g. the /// WAL pod). Default: `None`. - pub warmer: Option>, + pub warmer: Option>, + + /// Optional sink for write-path events, currently flush latency. Wired to + /// the flush handlers; supplied by the consumer (e.g. the WAL pod), which + /// owns the aggregation Lance would otherwise have to pick for it. + /// Default: `None`. + pub observer: Option>, + + /// Store params the base dataset was opened with, reused for the flusher's + /// opens + writes (base + generations). Injected by `mem_wal_writer`; set + /// these to the params of the dataset at `base_uri`, not to params bound to + /// some other path — generation URIs are derived from them. + /// Default: `None` (open by URI alone). + pub store_params: Option, + + /// Session for those opens, injected alongside `store_params`. + /// Default: `None`. + pub session: Option>, + + /// Admission control for every `put`, **replacing** lance's built-in + /// per-shard valve ([`LocalBackpressureController`]). + /// + /// For embedders whose budgets lance cannot see: a process-wide memtable + /// total across shards, a page-cache working set. Because it replaces + /// rather than layers, the injected controller owns the whole policy — a + /// per-shard ceiling included, for which it is handed [`ShardMemory`]. + /// Unlike the built-in valve it may also reject: see + /// [`Error::Backpressure`]. + /// Default: `None` (use the built-in valve). + pub backpressure: Option>, } impl Default for ShardWriterConfig { @@ -257,7 +263,6 @@ impl Default for ShardWriterConfig { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: true, max_wal_buffer_size: 10 * 1024 * 1024, // 10MB max_wal_flush_interval: Some(Duration::from_millis(100)), // 100ms max_wal_persist_retries: 3, @@ -268,13 +273,15 @@ impl Default for ShardWriterConfig { manifest_scan_batch_size: 2, max_unflushed_memtable_bytes: 1024 * 1024 * 1024, // 1GB backpressure_log_interval: Duration::from_secs(30), - async_index_buffer_rows: 10_000, - async_index_interval: Duration::from_secs(1), stats_log_interval: Some(Duration::from_secs(60)), // 1 minute frozen_memtable_grace: Duration::ZERO, enable_memtable: true, hnsw_params: HashMap::new(), warmer: None, + observer: None, + store_params: None, + session: None, + backpressure: None, } } } @@ -300,12 +307,6 @@ impl ShardWriterConfig { self } - /// Set indexed writes mode. - pub fn with_sync_indexed_write(mut self, indexed: bool) -> Self { - self.sync_indexed_write = indexed; - self - } - /// Set maximum WAL buffer size. pub fn with_max_wal_buffer_size(mut self, size: usize) -> Self { self.max_wal_buffer_size = size; @@ -362,21 +363,16 @@ impl ShardWriterConfig { self } - /// Set backpressure log interval. - pub fn with_backpressure_log_interval(mut self, interval: Duration) -> Self { - self.backpressure_log_interval = interval; - self - } - - /// Set async index buffer rows. - pub fn with_async_index_buffer_rows(mut self, rows: usize) -> Self { - self.async_index_buffer_rows = rows; + /// Replace the built-in per-shard valve with `controller`. See + /// [`Self::backpressure`]. + pub fn with_backpressure(mut self, controller: Arc) -> Self { + self.backpressure = Some(controller); self } - /// Set async index interval. - pub fn with_async_index_interval(mut self, interval: Duration) -> Self { - self.async_index_interval = interval; + /// Set backpressure log interval. + pub fn with_backpressure_log_interval(mut self, interval: Duration) -> Self { + self.backpressure_log_interval = interval; self } @@ -386,7 +382,7 @@ impl ShardWriterConfig { self } - /// Set how long a flushed memtable lingers in memory before eviction. MUST + /// Set how long an SSTable lingers in memory before eviction. MUST /// exceed the maximum query elapsed time — see `frozen_memtable_grace`. pub fn with_frozen_memtable_grace(mut self, grace: Duration) -> Self { self.frozen_memtable_grace = grace; @@ -453,7 +449,13 @@ impl TaskDispatcher { let mut ticker_intervals: Vec<(Interval, MessageFactory)> = tickers .into_iter() .map(|(duration, factory)| { - let interval = interval_at(tokio::time::Instant::now() + duration, duration); + let mut interval = interval_at(tokio::time::Instant::now() + duration, duration); + // `Burst` (the default) replays every tick missed while `handle()` + // was running. A WAL append can easily outlast its own interval, so + // the missed ticks pile up, the ticker arm below is always ready, + // and — being `biased` — it starves `rx` indefinitely: freeze + // completion cells and `close()`'s final append never get handled. + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); (interval, factory) }) .collect(); @@ -496,12 +498,12 @@ impl TaskDispatcher { debug!("Task '{}' received cancellation", self.name); break Ok(()); } - _ = first_interval.tick() => { - let message = (ticker_intervals[0].1)(); - if let Err(e) = self.handler.handle(message).await { - error!("Task '{}' error handling ticker message: {}", self.name, e); - } - } + // Explicit messages outrank the ticker. A tick is a backstop + // — it only ever *adds* an append that a real trigger would + // have made anyway — whereas a message may be a freeze's + // completion cell or `close()`'s final append, which nothing + // else will deliver. Polling the ticker first (as this did) + // lets a ready tick starve them. msg = self.rx.recv() => { match msg { Some(message) => { @@ -515,6 +517,12 @@ impl TaskDispatcher { } } } + _ = first_interval.tick() => { + let message = (ticker_intervals[0].1)(); + if let Err(e) = self.handler.handle(message).await { + error!("Task '{}' error handling ticker message: {}", self.name, e); + } + } } } }; @@ -559,20 +567,39 @@ impl TaskExecutor { Ok(()) } + /// Cancel and join every handler registered by [`Self::add_handler`]. + /// + /// Cancellation causes each handler's dispatcher to stop accepting messages and call + /// [`MessageHandler::cleanup`]. This method waits for every dispatcher to finish, even if + /// cleanup fails or a dispatcher panics, and then returns the first such failure. It returns + /// `Ok(())` only after every registered handler has been cleaned up successfully. pub async fn shutdown_all(&self) -> Result<()> { info!("Shutting down all tasks"); self.cancellation_token.cancel(); let tasks = std::mem::take(&mut *self.tasks.write().unwrap()); + let mut first_error = None; for (name, handle) in tasks { match handle.await { Ok(Ok(())) => debug!("Task '{}' completed successfully", name), - Ok(Err(e)) => warn!("Task '{}' completed with error: {}", name, e), - Err(e) => error!("Task '{}' panicked: {}", name, e), + Ok(Err(e)) => { + warn!("Task '{}' completed with error: {}", name, e); + if first_error.is_none() { + first_error = Some(e); + } + } + Err(e) => { + error!("Task '{}' panicked: {}", name, e); + if first_error.is_none() { + first_error = Some(Error::internal(format!( + "Task '{name}' panicked during shutdown: {e}" + ))); + } + } } } - Ok(()) + first_error.map_or(Ok(()), Err) } } @@ -631,10 +658,12 @@ pub type DurabilityCell = WatchableOnceCell; /// Statistics for backpressure monitoring. #[derive(Debug, Default)] pub struct BackpressureStats { - /// Total number of times backpressure was applied. + /// Total number of *completed* waits. total_count: AtomicU64, - /// Total time spent waiting on backpressure (in milliseconds). + /// Total time completed waits spent parked (in milliseconds). total_wait_ms: AtomicU64, + /// Writers parked in `maybe_apply_backpressure` right now. + active_count: AtomicU64, } impl BackpressureStats { @@ -643,18 +672,26 @@ impl BackpressureStats { Self::default() } - /// Record a backpressure event. + /// Record a completed backpressure wait. pub fn record(&self, wait_ms: u64) { self.total_count.fetch_add(1, Ordering::Relaxed); self.total_wait_ms.fetch_add(wait_ms, Ordering::Relaxed); } - /// Get the total backpressure count. + /// Count a writer as parked until the returned guard drops. Drop-based + /// because the caller's future can be cancelled mid-wait, which would + /// otherwise strand a waiter that never returns. + pub fn begin_wait(&self) -> BackpressureWaitGuard<'_> { + self.active_count.fetch_add(1, Ordering::Relaxed); + BackpressureWaitGuard(self) + } + + /// Get the completed-wait count. pub fn count(&self) -> u64 { self.total_count.load(Ordering::Relaxed) } - /// Get the total time spent waiting on backpressure. + /// Get the total time completed waits spent parked. pub fn total_wait_ms(&self) -> u64 { self.total_wait_ms.load(Ordering::Relaxed) } @@ -664,32 +701,355 @@ impl BackpressureStats { BackpressureStatsSnapshot { total_count: self.total_count.load(Ordering::Relaxed), total_wait_ms: self.total_wait_ms.load(Ordering::Relaxed), + active_count: self.active_count.load(Ordering::Relaxed), } } } +/// Keeps its writer counted in `active_count` for as long as it is held. +#[derive(Debug)] +pub struct BackpressureWaitGuard<'a>(&'a BackpressureStats); + +impl Drop for BackpressureWaitGuard<'_> { + fn drop(&mut self) { + self.0.active_count.fetch_sub(1, Ordering::Relaxed); + } +} + /// Snapshot of backpressure statistics. #[derive(Debug, Clone, Default)] pub struct BackpressureStatsSnapshot { - /// Total number of times backpressure was applied. + /// Number of waits that have *finished*. A wait in progress is not counted + /// here, and a cancelled one never is. pub total_count: u64, - /// Total time spent waiting on backpressure (in milliseconds). + /// Total time finished waits spent parked (in milliseconds), on the same + /// denominator as `total_count`. pub total_wait_ms: u64, + /// Writers parked right now. This is the field that answers "am I being + /// throttled at this instant" — the totals only move once a wait ends, so + /// they read zero throughout a first, still-ongoing stall. + pub active_count: u64, } -/// Backpressure controller for managing write flow. -pub struct BackpressureController { - /// Configuration. - config: ShardWriterConfig, - /// Stats for monitoring. +/// A **live** view of what one shard is holding in memory. +/// +/// The single place a shard's byte totals are computed. Everything that wants +/// them — the admission controller, [`ShardWriter::memory`], an operator gauge +/// — goes through this, so there is no second implementation to drift from it. +/// +/// Re-read it on every poll rather than reading once: a controller that delays +/// is waiting for exactly these numbers to fall, so a captured copy would never +/// observe the drain and the wait would never end. A read is one `ArcSwap` load +/// and a sum over the live memtables, so polling is cheap. +#[derive(Clone)] +pub struct ShardMemory(ShardMemorySource); + +/// Where a [`ShardMemory`] reads from. A dispatch over the two write modes, not +/// a second accounting: every arm is a field read, and the arithmetic that +/// combines them lives once, in `ShardMemory`. +#[derive(Clone)] +enum ShardMemorySource { + /// Memtable mode: the published set of resident memtables. + MemTables(Arc>), + /// WAL-only mode has no memtable; the pending queue is the whole pool, and + /// there is no flush to await, so a waiter falls back to a short sleep. + Queue(Arc), + /// Test-only: synthetic unflushed bytes, re-read each poll, so a controller + /// can be driven without standing up a writer. Carries no watcher — a + /// waiter falls back to its sleep, which keeps one poll to one call so a + /// test can count them. + #[cfg(test)] + Fake(Arc usize + Send + Sync>), +} + +impl ShardMemory { + fn memtables(tables: Arc>) -> Self { + Self(ShardMemorySource::MemTables(tables)) + } + + fn queue(state: Arc) -> Self { + Self(ShardMemorySource::Queue(state)) + } + + /// Resident bytes of the active memtable — row data plus its in-memory + /// indexes. In WAL-only mode, the pending queue's bytes. + pub fn active_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => t + .load() + .active + .as_ref() + .map_or(0, InMemoryMemTableRef::resident_bytes), + ShardMemorySource::Queue(q) => q.queue_bytes(), + #[cfg(test)] + ShardMemorySource::Fake(f) => f(), + } + } + + /// Row-data bytes of the active memtable: the flush unit, summed over the + /// windows the batches read through. In WAL-only mode, the pending queue's + /// bytes. + /// + /// Deliberately *not* `active_bytes() - index_bytes()`. That difference is + /// what the batches pin — whole parent buffers, unbounded above this figure + /// once any batch is a zero-copy slice — and is what the ceiling is built + /// on. Use this to reason about when a memtable seals, not about what it + /// costs. + pub fn row_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => t + .load() + .active + .as_ref() + .map_or(0, InMemoryMemTableRef::row_bytes), + ShardMemorySource::Queue(q) => q.queue_bytes(), + #[cfg(test)] + ShardMemorySource::Fake(f) => f(), + } + } + + /// The part of [`Self::active_bytes`] held by the active memtable's + /// in-memory indexes (its PK bloom filter included). A **subset** of that + /// figure, not another term to add. `0` in WAL-only mode, which has none. + /// + /// Broken out because it does not behave like row data and is usually what + /// explains a shard near its ceiling with few rows in it: an HNSW graph is + /// pre-allocated in full on the first insert. + pub fn index_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => t + .load() + .active + .as_ref() + .map_or(0, InMemoryMemTableRef::index_bytes), + ShardMemorySource::Queue(_) => 0, + #[cfg(test)] + ShardMemorySource::Fake(_) => 0, + } + } + + /// Resident bytes of sealed memtables whose flush has not committed. + /// Always `0` in WAL-only mode. + pub fn frozen_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => t + .load() + .frozen + .iter() + .map(InMemoryMemTableRef::resident_bytes) + .sum(), + ShardMemorySource::Queue(_) => 0, + #[cfg(test)] + ShardMemorySource::Fake(_) => 0, + } + } + + /// Resident bytes of sealed memtables that have flushed and are lingering + /// out `frozen_memtable_grace` so in-flight as-of reads stay batch-resolved. + /// + /// Real memory, but no flush reclaims it — the sweeper does, on a timer. A + /// waiter that blocks on this is waiting for the clock, not for a flush. + /// `0` in WAL-only mode, and `0` under the default zero grace. + pub fn grace_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => t + .load() + .grace + .iter() + .map(InMemoryMemTableRef::resident_bytes) + .sum(), + ShardMemorySource::Queue(_) => 0, + #[cfg(test)] + ShardMemorySource::Fake(_) => 0, + } + } + + /// Bytes only a flush can reclaim: the pool to bound against OOM. + /// + /// One load covers both terms, so this cannot cross a freeze and + /// double-count or lose a memtable the way two separate reads could — which + /// is why this is not `active_bytes() + frozen_bytes()`. + pub fn unflushed_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => { + let tables = t.load(); + tables + .active + .as_ref() + .map_or(0, InMemoryMemTableRef::resident_bytes) + + tables + .frozen + .iter() + .map(InMemoryMemTableRef::resident_bytes) + .sum::() + } + ShardMemorySource::Queue(q) => q.queue_bytes(), + #[cfg(test)] + ShardMemorySource::Fake(f) => f(), + } + } + + /// Every resident byte the shard is holding: [`Self::unflushed_bytes`] plus + /// the generations already flushed but still inside `frozen_memtable_grace`. + /// + /// This is the figure a process-wide budget wants. `unflushed_bytes` is the + /// narrower one — what a flush can still reclaim — and is what the per-shard + /// valve throttles on, because throttling on grace-retained memory would + /// stall the writer waiting for a sweeper tick. The two differ only when a + /// grace is configured; under the default zero grace they are equal. + pub fn retained_bytes(&self) -> usize { + match &self.0 { + ShardMemorySource::MemTables(t) => { + let tables = t.load(); + tables + .active + .as_ref() + .map_or(0, InMemoryMemTableRef::resident_bytes) + + tables + .frozen + .iter() + .chain(tables.grace.iter()) + .map(InMemoryMemTableRef::resident_bytes) + .sum::() + } + ShardMemorySource::Queue(q) => q.queue_bytes(), + #[cfg(test)] + ShardMemorySource::Fake(f) => f(), + } + } + + /// What could reclaim memory here while a writer waits on it. + /// + /// A blocking controller must consult this rather than assume that waiting + /// eventually works: a shard can be over its ceiling with nothing running + /// that would bring it back down. See [`Drain`]. + pub fn drain(&self) -> Drain { + match &self.0 { + ShardMemorySource::MemTables(t) => { + let tables = t.load(); + match tables.oldest_flush.clone() { + Some(flush) => Drain::Flush(flush), + // The sweeper will drop these once their grace elapses. + None if !tables.grace.is_empty() => Drain::Background, + None => Drain::Stalled, + } + } + // The WAL flusher drains the queue on its own schedule. + ShardMemorySource::Queue(_) => Drain::Background, + #[cfg(test)] + ShardMemorySource::Fake(_) => Drain::Background, + } + } +} + +/// What can bring a shard back under its ceiling while a writer waits. +/// +/// Returned by [`ShardMemory::drain`] so a blocking controller can tell a wait +/// that ends from one that cannot. The distinction is not academic: a flush +/// that fails leaves its generation resident and charged with nothing queued to +/// retry it, and index memory is charged to the ceiling while the seal trigger +/// measures row bytes — either can put a shard over budget with no flush in +/// flight, and only a new write would start one. +#[derive(Debug)] +pub enum Drain { + /// Park on this flush. Completing it retires a whole generation. + Flush(DurabilityWatcher), + /// Nothing to park on, but something is draining the shard on its own + /// schedule — the WAL flusher, or the grace sweeper. Poll. + Background, + /// Nothing outstanding. Only a new write would start a flush, and a waiter + /// here is precisely what is keeping writes out, so waiting cannot end. + Stalled, +} + +impl Debug for ShardMemory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ShardMemory") + .field("active_bytes", &self.active_bytes()) + .field("frozen_bytes", &self.frozen_bytes()) + .field("grace_bytes", &self.grace_bytes()) + .finish() + } +} + +/// Admission control for [`ShardWriter::put`], consulted before each write. +/// +/// Two implementations ship in-tree and exactly one runs per writer: +/// [`LocalBackpressureController`] by default, or whatever an embedder installs +/// via [`ShardWriterConfig::backpressure`]. They do not layer — the injected +/// one owns the whole policy, per-shard rules included, which is what +/// [`ShardMemory`] is for. +/// +/// An embedder replaces the default when it has budgets lance cannot see: a +/// process-wide memtable total across shards, a page-cache working set. Because +/// [`ShardMemory`] is self-sufficient, a replacement that also wants the +/// built-in per-shard behaviour can call [`LocalBackpressureController`] from +/// inside its own implementation rather than reimplementing it. +#[async_trait::async_trait] +pub trait BackpressureController: Send + Sync + Debug { + /// Decide whether to admit a write into a shard currently holding `shard`. + /// + /// May await to delay the writer, or return [`Error::Backpressure`] to + /// refuse it. Any other error is a real failure. + /// + /// Deliberately not told the incoming batch's size. Batches are already + /// decoded and resident by the time this runs, so there is nothing to + /// reserve against — refusing does not un-allocate them. Bounding a single + /// write's memory is the ingress's job, not this one's. + async fn maybe_apply_backpressure(&self, shard: ShardMemory) -> Result<()>; + + /// Throttling counters for [`ShardWriter::backpressure_stats`]. An injected + /// controller keeps its own metrics, so the default reports zeros rather + /// than requiring it to maintain lance's. + fn stats_snapshot(&self) -> BackpressureStatsSnapshot { + BackpressureStatsSnapshot::default() + } +} + +/// The controller guarding this writer: the embedder's if one was injected, +/// otherwise lance's own [`LocalBackpressureController`]. +fn resolve_backpressure(config: &ShardWriterConfig) -> Arc { + match &config.backpressure { + Some(injected) => injected.clone(), + None => Arc::new(LocalBackpressureController::new(config)), + } +} + +/// Poll cadence while a shard is over its ceiling and something other than a +/// flush is expected to bring it back down — there is no watcher to park on. +const DRAIN_POLL_INTERVAL: Duration = Duration::from_millis(10); + +/// How long a shard must keep reading as [`Drain::Stalled`] before the valve +/// gives up on it rather than waiting. +/// +/// The classification is momentarily wrong under concurrency: another writer +/// holding the state lock has already grown the active memtable past the +/// ceiling but has not yet reached the `freeze_memtable` that publishes a +/// watcher, so a waiter sampling in between sees over-budget with nothing +/// outstanding. That window is one locked section of in-memory work, orders of +/// magnitude under this. A real stall never closes. +const STALL_GRACE: Duration = Duration::from_secs(1); + +/// The per-shard memtable valve: lance's default when nothing is injected. +/// +/// Soft and blocking — it stalls the producer until a flush drains the pool. +/// It refuses a write only when the pool *cannot* drain (see [`Drain`]); +/// waiting there would park the writer for good. Replaced wholesale by +/// [`ShardWriterConfig::backpressure`], so an embedder that injects a +/// controller takes on the per-shard ceiling this provides (see +/// [`ShardMemory`]). +#[derive(Debug)] +pub struct LocalBackpressureController { + max_unflushed_memtable_bytes: usize, + log_interval: Duration, stats: Arc, } -impl BackpressureController { - /// Create a new backpressure controller. - pub fn new(config: ShardWriterConfig) -> Self { +impl LocalBackpressureController { + fn new(config: &ShardWriterConfig) -> Self { Self { - config, + max_unflushed_memtable_bytes: config.max_unflushed_memtable_bytes, + log_interval: config.backpressure_log_interval, stats: Arc::new(BackpressureStats::new()), } } @@ -698,31 +1058,44 @@ impl BackpressureController { pub fn stats(&self) -> &Arc { &self.stats } +} - /// Check and apply backpressure if needed. - /// - /// This method blocks if the system is under memory pressure, waiting for - /// frozen memtables to be flushed to storage until under threshold. - /// - /// Backpressure is applied when: - /// - `unflushed_memtable_bytes` >= `max_unflushed_memtable_bytes` - /// - /// # Arguments - /// - `get_state`: Closure that returns current (unflushed_memtable_bytes, oldest_memtable_watcher) +#[async_trait::async_trait] +impl BackpressureController for LocalBackpressureController { + fn stats_snapshot(&self) -> BackpressureStatsSnapshot { + self.stats.snapshot() + } + + /// Blocks while this shard's unflushed bytes are at or above + /// `max_unflushed_memtable_bytes`, waiting on the oldest flush. /// - /// The closure is called in a loop to get fresh state after each wait. - pub async fn maybe_apply_backpressure(&self, mut get_state: F) -> Result<()> - where - F: FnMut() -> (usize, Option), - { + /// Returns [`Error::Backpressure`] in the one case where blocking would + /// never end: over the ceiling with no flush outstanding and nothing + /// draining in the background, for `STALL_GRACE` running. Only a write + /// starts a flush, and this valve is what holds writes out, so the wait + /// would have no event to end on. The + /// error names the breakdown, because the two ways to get there — a flush + /// that failed and left its generation charged, or index memory carrying a + /// memtable past the ceiling while the seal trigger still sees small row + /// bytes — are both configuration or operational conditions an operator has + /// to act on rather than wait out. + async fn maybe_apply_backpressure(&self, shard: ShardMemory) -> Result<()> { let start = std::time::Instant::now(); let mut iteration = 0u32; + // Held for the whole stall so an operator polling mid-wait sees it; the + // totals below cannot, since they only move once the wait ends. + let mut active_wait = None; + // When the shard first read as un-drainable, cleared the moment it + // stops. See `STALL_GRACE`. + let mut stalled_since: Option = None; loop { - let (unflushed_memtable_bytes, oldest_watcher) = get_state(); + // Re-read every iteration: this loop is waiting for exactly these + // bytes to fall, so a value captured once would never see the drain. + let unflushed_memtable_bytes = shard.unflushed_bytes(); // Check if under threshold - if unflushed_memtable_bytes < self.config.max_unflushed_memtable_bytes { + if unflushed_memtable_bytes < self.max_unflushed_memtable_bytes { if iteration > 0 { let wait_ms = start.elapsed().as_millis() as u64; self.stats.record(wait_ms); @@ -731,28 +1104,55 @@ impl BackpressureController { } iteration += 1; + if active_wait.is_none() { + active_wait = Some(self.stats.begin_wait()); + } debug!( "Backpressure triggered: unflushed_bytes={}, max={}, iteration={}", - unflushed_memtable_bytes, self.config.max_unflushed_memtable_bytes, iteration + unflushed_memtable_bytes, self.max_unflushed_memtable_bytes, iteration ); - // Wait for oldest memtable to flush - if let Some(mut mem_watcher) = oldest_watcher { - tokio::select! { - _ = mem_watcher.await_value() => {} - _ = tokio::time::sleep(self.config.backpressure_log_interval) => { - warn!( - "Backpressure wait timeout, continuing to wait: unflushed_bytes={}, interval={}s, iteration={}", - unflushed_memtable_bytes, - self.config.backpressure_log_interval.as_secs(), - iteration - ); + match shard.drain() { + Drain::Flush(mut mem_watcher) => { + stalled_since = None; + tokio::select! { + _ = mem_watcher.await_value() => {} + _ = tokio::time::sleep(self.log_interval) => { + warn!( + "Backpressure wait timeout, continuing to wait: unflushed_bytes={}, interval={}s, iteration={}", + unflushed_memtable_bytes, + self.log_interval.as_secs(), + iteration + ); + } } } - } else { - // No watcher available - sleep briefly to avoid busy loop - tokio::time::sleep(std::time::Duration::from_millis(10)).await; + // Someone else is draining on a schedule of their own; poll + // rather than busy-loop. + Drain::Background => { + stalled_since = None; + tokio::time::sleep(DRAIN_POLL_INTERVAL).await + } + Drain::Stalled => { + let since = *stalled_since.get_or_insert_with(std::time::Instant::now); + if since.elapsed() < STALL_GRACE { + tokio::time::sleep(DRAIN_POLL_INTERVAL).await; + continue; + } + return Err(Error::backpressure(format!( + "shard is at its memtable ceiling with no flush outstanding, so waiting \ + cannot drain it: unflushed_bytes={}, max={}, active_bytes={} (of which \ + index_bytes={}), frozen_bytes={}. The active memtable seals on resident \ + bytes, so reaching here means a flush failed and left its generation \ + charged with nothing queued to retry it", + unflushed_memtable_bytes, + self.max_unflushed_memtable_bytes, + shard.active_bytes(), + shard.index_bytes(), + shard.frozen_bytes(), + ))); + } } } } @@ -775,14 +1175,90 @@ struct FrozenMemTable { flushed_at_ms: Option, } +/// What one shard holds in memory: detached size handles, plus the flush a +/// waiter should park on. +/// +/// Data only. The byte arithmetic lives on [`ShardMemory`], which is the single +/// place it exists — this is just what gets published. +/// +/// Lives outside [`WriterState`] on purpose. A caller deciding whether to admit +/// a write must read this *while* the writer holds the write lock, which is +/// exactly when a `try_read()` on that lock fails — and tokio's `RwLock` is +/// write-preferring, so it fails whenever a writer is merely queued. Reading +/// through the lock would therefore report zero precisely under load. +/// +/// Replaced wholesale by [`publish_memory`] at the moments the memtable set +/// changes, and **derived** from `WriterState` each time rather than adjusted. +/// So there is no counter to keep paired with anything, nothing on the per-put +/// path, and no way for this and [`ShardWriter::memtable_stats`] to disagree +/// about what the shard holds: they read the same memtables through the same +/// filter. +/// +/// The handles stay live, so byte totals track a memtable that is still growing +/// without anyone republishing. +#[derive(Default)] +struct ResidentMemTables { + /// The active memtable, or `None` before the first publish. + active: Option, + /// Sealed memtables whose flush has not committed — including any left + /// resident by a *failed* flush, which are the ones most worth metering. + frozen: Vec, + /// Sealed memtables whose flush *did* commit, lingering out + /// `frozen_memtable_grace` before `SweepExpired` drops them. + /// + /// Held apart from `frozen` rather than dropped from the view: no flush can + /// reclaim these, so metering the flush valve on them would throttle against + /// memory that is going away on a timer. They are still resident, though, + /// and a process-wide budget has to see them — hence + /// [`ShardMemory::retained_bytes`] alongside + /// [`ShardMemory::unflushed_bytes`]. + grace: Vec, + /// The oldest flush still outstanding, or `None` when none is. + /// + /// Deliberately not backfilled with the active memtable's watcher. That + /// watcher only fires when the active memtable is sealed, which only a put + /// does — so offering it to a waiter that is itself holding puts out names + /// an event that cannot arrive. `None` is the honest answer, and + /// [`ShardMemory::drain`] turns it into one. + oldest_flush: Option, +} + +/// Re-derive a shard's resident-memtable set from its writer state. +/// +/// Call under the write lock after any change to that set: `open`, +/// `freeze_memtable`, a flush commit, and `SweepExpired` — which changes it by +/// evicting grace-expired generations that are still counted until it runs. +/// +/// Cheap: two `Arc` clones per live memtable, no byte walk — the totals are +/// computed on read. +fn publish_memory(memory: &ArcSwap, state: &WriterState) { + let (grace, frozen) = state + .frozen_memtables + .iter() + .partition::, _>(|frozen| frozen.flushed_at_ms.is_some()); + let refs = |tables: Vec<&FrozenMemTable>| { + tables + .into_iter() + .map(|frozen| in_memory_ref(&frozen.memtable)) + .collect() + }; + memory.store(Arc::new(ResidentMemTables { + active: Some(in_memory_ref(&state.memtable)), + frozen: refs(frozen), + grace: refs(grace), + // Oldest first, so the front of the queue is what a waiter parks on. + oldest_flush: state.frozen_flush_watchers.front().cloned(), + })); +} + /// ShardWriter state shared across tasks. struct WriterState { memtable: MemTable, last_flushed_wal_entry_position: u64, - /// Total size of frozen memtables (for backpressure). - frozen_memtable_bytes: usize, - /// Flush watchers for frozen memtables (for backpressure). - frozen_flush_watchers: VecDeque<(usize, DurabilityWatcher)>, + /// Flush watchers for frozen memtables, oldest first. Carries no byte + /// count: sizes are read live off `frozen_memtables` (see + /// [`ResidentMemTables`]), so there is nothing here to keep paired. + frozen_flush_watchers: VecDeque, /// Sealed memtables, kept queryable so a concurrent reader sees no hole /// between `freeze_memtable` and the flush task's manifest commit, and for /// `frozen_memtable_grace` beyond it so as-of reads stay batch-resolved. @@ -798,11 +1274,15 @@ struct WriterState { last_wal_flush_trigger_time: u64, } -/// Capture a point-in-time scan handle to one in-memory memtable (active -/// or frozen — same shape). Shared by `active_memtable_ref` and -/// `in_memory_memtable_refs` so both stamp identical fields. -fn in_memory_ref(mt: &MemTable) -> crate::dataset::mem_wal::scanner::InMemoryMemTableRef { - crate::dataset::mem_wal::scanner::InMemoryMemTableRef { +/// Capture a point-in-time handle to one in-memory memtable (active or frozen +/// — same shape). +/// +/// The single projection of a memtable in this crate: the read path scans +/// through it, and [`ShardMemory`] sizes through it. Everything it holds is an +/// `Arc` or a copy, so a handle is cheap and stays live as the memtable grows — +/// which is what lets the memory view be read without the writer lock. +fn in_memory_ref(mt: &MemTable) -> InMemoryMemTableRef { + InMemoryMemTableRef { batch_store: mt.batch_store(), index_store: mt .indexes_arc() @@ -822,7 +1302,7 @@ fn now_millis() -> u64 { start_time().elapsed().as_millis() as u64 } -/// Replay WAL entries written after the last successfully-flushed generation +/// Replay WAL entries written after the last successfully-flushed SSTable /// into the freshly-built MemTable. Updates any in-memory indexes attached to /// the MemTable so replayed rows are immediately searchable. /// @@ -836,32 +1316,63 @@ fn now_millis() -> u64 { /// Aborts with an error if any replayed entry's `writer_epoch` is strictly /// greater than `our_epoch` — that indicates a successor writer claimed the /// shard between our `claim_epoch` and this replay, fencing us. +/// Outcome of replaying a shard's WAL into memory. +struct ReplayResult { + /// The active memtable — the final, partial one replay left unsealed. A fresh + /// shard yields an empty one; every sealed memtable was flushed to a Lance + /// generation during replay and is not returned. + active: MemTable, + /// One past the highest WAL entry position observed — the next write position. + next_wal_position: u64, +} + +/// Replay a shard's WAL into memory, flushing sealed memtables as the batch store +/// fills. +/// +/// A single memtable holds at most `max_memtable_batches` batches, but a WAL is +/// unbounded — so replay must rotate exactly as the live write path does. It +/// seals a full memtable and, because the data is already durable, flushes it to +/// a Lance generation right here (the same `MemTableFlusher::flush` the live path +/// uses), rather than holding every sealed memtable in memory until open +/// finishes. That bounds resident memory to ~two memtables and truncates the WAL +/// as it goes, so a later reopen replays only the unflushed tail. Only the final +/// partial memtable is returned, as the active one. +/// +/// `make_memtable(generation, global_offset)` builds a fresh, cursor-bound +/// memtable. Rotation happens at WAL-entry boundaries, never mid-entry, so each +/// sealed memtable covers a clean range of complete entries and stamps the last +/// one as its SSTable's `replay_after_wal_entry_position`. +#[allow(clippy::too_many_arguments)] async fn replay_memtable_from_wal( object_store: Arc, base_path: Path, shard_id: Uuid, our_epoch: u64, manifest: &ShardManifest, - memtable: &mut MemTable, -) -> Result { + base_generation: u64, + mut make_memtable: impl FnMut(u64, usize) -> Result, + flusher: &MemTableFlusher, + wal_flusher: &WalFlusher, + index_configs: &[MemIndexConfig], + max_memtable_size: usize, + max_memtable_rows: usize, + max_resident_bytes: usize, +) -> Result { // WAL positions are 1-based (see `FIRST_WAL_ENTRY_POSITION`), so a // cursor of 0 means "no flush has ever stamped this shard" and replay // starts at position 1. After flushing position N the cursor holds N // and replay starts at N+1. The arithmetic collapses to a single // saturating_add(1) in both cases — we deliberately do not consult - // `flushed_generations` here, since an external compactor may + // `sstables` here, since an external compactor may // legitimately drain that vector back to empty after merging its // contents into the base table. let start_position = manifest.replay_after_wal_entry_position.saturating_add(1); - // The MemTable is always freshly built before this function runs, so - // any existing BatchStore entries can only have come from this replay - // pass. We index everything in `[0, batch_count)` at the end. - debug_assert_eq!(memtable.batch_count(), 0); - let tailer = WalTailer::new(object_store, base_path, shard_id); let mut position = start_position; + let mut active = make_memtable(base_generation, 0)?; + loop { match tailer.read_entry(position).await? { // The first NotFound proves the WAL tip is at `position`, which @@ -877,16 +1388,57 @@ async fn replay_memtable_from_wal( // Fence sentinels deserialize to zero batches and are skipped // here — they carry only a position, no rows. if !entry.batches.is_empty() { - // Entries written before deletes existed lack `_tombstone`; - // inject `false` so they match the extended memtable schema. - // Normal entries already carry it and pass through unchanged. - let target_schema = memtable.schema().clone(); + // Re-label to the current storage schema; entries written + // before deletes existed also need `_tombstone = false`. + let storage_schema = active.schema().clone(); let batches = entry .batches .into_iter() - .map(|b| ensure_tombstone_column(b, &target_schema)) + .map(|b| ensure_tombstone_column(b, &storage_schema)) .collect::>>()?; - memtable.insert_batches_only(batches).await?; + + // Seal + flush on the same criteria the live path uses, measured + // against this whole entry, so no entry is split across two + // memtables and each sealed one covers a clean range of complete + // entries. An empty memtable is never rotated: a fresh one holds + // an oversized entry no better, left to the insert below to + // surface. + let entry_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + if !active.batch_store().is_empty() + && memtable_reached_flush_threshold( + &active, + max_memtable_size, + max_memtable_rows, + max_resident_bytes, + batches.len(), + entry_rows, + ) + { + let store = active.batch_store(); + // The last entry this memtable fully absorbed is the one + // before the entry about to be inserted. + let covered = position.saturating_sub(1); + let generation = active.generation() + 1; + let global_end = store.global_end(); + + // The sealed data is already durable in the WAL — mark it + // so the flush's `all_flushed_to_wal` precondition holds and + // no WAL re-append is attempted. + wal_flusher.advance_durable(global_end); + flush_replayed_memtable( + flusher, + &active, + our_epoch, + covered, + global_end, + index_configs, + ) + .await?; + + active = make_memtable(generation, global_end)?; + } + + active.insert_batches_only(batches).await?; } position = position.checked_add(1).ok_or_else(|| { Error::io(format!( @@ -898,21 +1450,18 @@ async fn replay_memtable_from_wal( } } - // Update in-memory indexes with the replayed batches so readers see them - // through the index path (matching what would have happened on the - // pre-crash writer's WAL flush). Indexes from the previous writer don't - // persist; this rebuilds them from the WAL. - if let Some(indexes) = memtable.indexes_arc() { - let batches_after = memtable.batch_count(); - if batches_after > 0 { - let store = memtable.batch_store(); - let mut stored: Vec = Vec::with_capacity(batches_after); - for pos in 0..batches_after { - if let Some(s) = store.get(pos) { - stored.push(s.clone()); - } - } - tokio::task::spawn_blocking(move || indexes.insert_batches_parallel(&stored)) + // Rebuild the active memtable's in-memory indexes from the batches just + // replayed, so readers see them through the index path — matching what the + // pre-crash writer's flush would have done. Sealed memtables needed no + // in-memory index build: they were flushed straight to disk and are gone. + if let Some(indexes) = active.indexes_arc() { + let batch_count = active.batch_count(); + if batch_count > 0 { + let store = active.batch_store(); + let stored: Vec = (0..batch_count) + .filter_map(|pos| store.get(pos).cloned()) + .collect(); + tokio::task::spawn_blocking(move || indexes.insert_batches(&stored)) .await .map_err(|e| { Error::internal(format!("WAL replay index update task panicked: {}", e)) @@ -920,7 +1469,85 @@ async fn replay_memtable_from_wal( } } - Ok(position) + Ok(ReplayResult { + active, + next_wal_position: position, + }) +} + +/// Whether a memtable has reached the threshold at which it should be sealed and +/// flushed. +/// +/// The single source of truth for the flush trigger, shared by the live put path +/// (`maybe_trigger_memtable_flush`) and by replay so the two cannot drift. +/// +/// `incoming_batches` / `incoming_rows` are what is about to be inserted. +/// Pre-insert callers pass the real counts; post-insert callers pass `(1, 1)`, +/// asking whether there is room for one more batch. +/// +/// Four arms, each answering a different question: +/// +/// - **Row window** against `max_memtable_size`. The knob an operator sizes: it +/// measures what a flush actually writes, so a generation stays a predictable +/// fragment in the base dataset. Deliberately the *only* thing charged to this +/// threshold — index memory and buffer padding are not, or fragment size would +/// start depending on index configuration and Arrow's allocator. +/// - **Resident total** against `max_resident_bytes`, the backpressure ceiling. +/// The row window bounds neither what the batches *pin* (a one-row slice holds +/// its whole parent) nor what the indexes hold, so without this arm a memtable +/// can carry a shard past its ceiling with no seal reachable — and the valve, +/// finding nothing outstanding to wait on, would refuse writes that can never +/// succeed. This is the drain path that makes the ceiling live rather than a +/// trap. +/// - **Batch-store capacity**, room for `incoming_batches` more. +/// - **Row count** against `max_memtable_rows`, room for `incoming_rows` more. +/// A hard capacity rather than a target: the in-memory indexes are +/// pre-allocated to exactly this many rows, so an overshoot fails the index +/// apply. The live path checks this arm pre-insert as well. +fn memtable_reached_flush_threshold( + memtable: &MemTable, + max_memtable_size: usize, + max_memtable_rows: usize, + max_resident_bytes: usize, + incoming_batches: usize, + incoming_rows: usize, +) -> bool { + let store = memtable.batch_store(); + store.row_bytes() >= max_memtable_size + || memtable_resident_bytes(memtable) >= max_resident_bytes + || store.remaining_capacity() < incoming_batches + || store.total_rows().saturating_add(incoming_rows) > max_memtable_rows +} + +/// What this memtable holds in memory: the heap its batches pin plus its +/// in-memory indexes. The same quantity [`ShardMemory`] reports for the active +/// memtable, so the seal trigger and the ceiling that gates writes measure the +/// same thing. +fn memtable_resident_bytes(memtable: &MemTable) -> usize { + memtable.batch_store().retained_bytes() + + memtable.indexes().map_or(0, IndexStore::resident_bytes) + + super::memtable::pk_bloom_filter_bytes() +} + +/// Flush a sealed replay memtable to a Lance generation, choosing the indexed +/// path when secondary indexes are configured (mirroring the live memtable-flush +/// handler). Commits the manifest, stamping `covered` as the generation's +/// `replay_after_wal_entry_position` so a later reopen skips these entries. +async fn flush_replayed_memtable( + flusher: &MemTableFlusher, + memtable: &MemTable, + epoch: u64, + covered: u64, + durable: usize, + index_configs: &[MemIndexConfig], +) -> Result<()> { + if index_configs.is_empty() { + flusher.flush(memtable, epoch, covered, durable).await?; + } else { + Box::pin(flusher.flush_with_indexes(memtable, epoch, index_configs, covered, durable)) + .await?; + } + Ok(()) } /// Pair each primary-key column name with its field id (both derived from the @@ -933,46 +1560,44 @@ fn pk_index_columns(pk_columns: &[String], pk_field_ids: &[i32]) -> Vec<(String, .collect() } -/// Ensure `batch` carries the `_tombstone` column required by the extended -/// memtable schema, injecting `false` for every row when it is absent. +/// Re-label `batch` to the storage schema, injecting `_tombstone = false` when +/// absent — callers pass logical-shaped batches, and WAL entries written before +/// deletes existed lack the column. /// -/// Used on the normal write path ([`ShardWriter::put`]) where callers pass -/// base-shaped batches, and on WAL replay of entries written before deletes -/// existed (legacy entries lack the column). A batch that already carries -/// `_tombstone` (a normal replayed entry) is returned unchanged. +/// A batch that already carries `_tombstone` is re-labeled too, so an entry +/// written under an older storage schema replays into the current one. fn ensure_tombstone_column( batch: RecordBatch, - target_schema: &Arc, + storage_schema: &Arc, ) -> Result { - if batch.schema().column_with_name(TOMBSTONE).is_some() { - return Ok(batch); - } let n = batch.num_rows(); let mut columns: Vec = batch.columns().to_vec(); - columns.push(Arc::new(BooleanArray::from(vec![false; n]))); - RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| { + if batch.schema().column_with_name(TOMBSTONE).is_none() { + columns.push(Arc::new(BooleanArray::from(vec![false; n]))); + } + RecordBatch::try_new(storage_schema.clone(), columns).map_err(|e| { Error::invalid_input(format!( - "failed to inject _tombstone column (does the batch match the base schema?): {}", + "failed to inject _tombstone column (does the batch match the base table schema?): {}", e )) }) } -/// Build a tombstone batch from a key-only `keys` batch: the primary key -/// columns are carried through, `_tombstone` is set to `true`, and every other -/// column in the memtable schema is null. +/// Build a tombstone batch from a key-only `keys` batch: primary keys carried +/// through, `_tombstone` true, every other column null. /// -/// Errors if `keys` is missing a primary key column, or if a non-PK column is -/// non-nullable (a tombstone must null it) — surfaced via the `RecordBatch` -/// validation. +/// Non-PK columns are nullable in the storage schema however the base table +/// declares them — that is what lets a strict table have tombstones at all. +/// Primary keys are not, so the validation below still rejects a null, +/// mistyped, or missing key. fn build_tombstone_batch( keys: &RecordBatch, - target_schema: &Arc, + storage_schema: &Arc, pk_columns: &[String], ) -> Result { let n = keys.num_rows(); - let mut columns: Vec = Vec::with_capacity(target_schema.fields().len()); - for field in target_schema.fields() { + let mut columns: Vec = Vec::with_capacity(storage_schema.fields().len()); + for field in storage_schema.fields() { let name = field.name(); if name == TOMBSTONE { columns.push(Arc::new(BooleanArray::from(vec![true; n]))); @@ -988,9 +1613,9 @@ fn build_tombstone_batch( columns.push(new_null_array(field.data_type(), n)); } } - RecordBatch::try_new(target_schema.clone(), columns).map_err(|e| { + RecordBatch::try_new(storage_schema.clone(), columns).map_err(|e| { Error::invalid_input(format!( - "failed to build tombstone batch (is every non-primary-key column nullable?): {}", + "failed to build tombstone batch (do the delete keys match the primary key?): {}", e )) }) @@ -998,9 +1623,15 @@ fn build_tombstone_batch( /// Shared state for writer operations. struct SharedWriterState { - state: Arc>, + /// Detached size handles for every memtable this shard holds, shared with + /// the memtable flush handler (which re-derives them on commit). + memory: Arc>, wal_flusher: Arc, wal_flush_tx: mpsc::UnboundedSender, + /// The index-apply task's channel. Separate from the WAL flusher's on + /// purpose: `TaskDispatcher::run` awaits `handle()` inline, so sharing one + /// would put a ~100ms S3 PUT in front of every latency-sensitive index apply. + index_apply_tx: mpsc::UnboundedSender, memtable_flush_tx: mpsc::UnboundedSender, config: ShardWriterConfig, schema: Arc, @@ -1016,9 +1647,10 @@ struct SharedWriterState { impl SharedWriterState { #[allow(clippy::too_many_arguments)] fn new( - state: Arc>, + memory: Arc>, wal_flusher: Arc, wal_flush_tx: mpsc::UnboundedSender, + index_apply_tx: mpsc::UnboundedSender, memtable_flush_tx: mpsc::UnboundedSender, config: ShardWriterConfig, schema: Arc, @@ -1029,9 +1661,10 @@ impl SharedWriterState { index_configs: Vec, ) -> Self { Self { - state, + memory, wal_flusher, wal_flush_tx, + index_apply_tx, memtable_flush_tx, config, schema, @@ -1043,80 +1676,144 @@ impl SharedWriterState { } } + /// Ask the index-apply task to cover `[indexed, end_batch_position)` of this + /// store. Cheap and idempotent: a range already covered is a no-op, which is + /// the common case under load, since one apply coalesces the puts queued + /// behind it. + fn trigger_index_apply( + &self, + batch_store: Arc, + indexes: Arc, + end_batch_position: usize, + ) -> Result<()> { + self.index_apply_tx + .send(TriggerIndexApply { + batch_store, + indexes, + end_batch_position, + }) + .map_err(|_| Error::io("index apply channel closed")) + } + /// Freeze the current memtable and send it to the flush handler. /// /// Takes `&mut WriterState` directly since caller already holds the lock. fn freeze_memtable(&self, state: &mut WriterState) -> Result { - let pending_wal_range = state.memtable.batch_store().pending_wal_flush_range(); + let durable = self.wal_flusher.durable(); + let pending_wal_range = state + .memtable + .batch_store() + .pending_wal_flush_range(durable); let last_wal_entry_position = state.last_flushed_wal_entry_position; let old_batch_store = state.memtable.batch_store(); - let old_indexes = state.memtable.indexes_arc(); let next_generation = state.memtable.generation() + 1; - let mut new_memtable = MemTable::with_capacity( + // The incoming memtable's batch 0 continues the writer's batch sequence + // where the outgoing one ends. Without this coordinate, local positions + // (which restart at 0 every rotation) cannot be mapped onto the + // writer-global durability cursor. + let next_global_offset = old_batch_store.global_end(); + let mut new_memtable = MemTable::with_capacity_at( self.schema.clone(), next_generation, self.pk_field_ids.clone(), CacheConfig::default(), self.max_memtable_batches, + next_global_offset, )?; - // Build an IndexStore when there are user indexes *or* a primary key: - // the PK dedup index (and its flushed on-disk sidecar) is required for - // cross-generation dedup even when no secondary index is configured. - if !self.index_configs.is_empty() || !self.pk_columns.is_empty() { - let mut indexes = IndexStore::from_configs( - &self.index_configs, - self.max_memtable_rows, - self.max_memtable_batches, - )?; + // Always build and bind an IndexStore, even with no user indexes and no + // primary key. It is what carries the memtable's `indexed_count`, and + // binding it to the writer's cursors is what lets a reader derive the + // visible prefix — so an index-less memtable that skipped this would fall + // back to `visible == indexed` and publish rows before they were durable. + // (A PK memtable also needs the PK dedup index and its flushed sidecar.) + let mut indexes = IndexStore::from_configs( + &self.index_configs, + self.max_memtable_rows, + self.max_memtable_batches, + )?; + if !self.pk_columns.is_empty() { indexes.enable_pk_index(&pk_index_columns(&self.pk_columns, &self.pk_field_ids)); - new_memtable.set_indexes_arc(Arc::new(indexes)); } + indexes.set_durability(Arc::clone(self.wal_flusher.cursors()), next_global_offset); + new_memtable.set_indexes_arc(Arc::new(indexes)); let mut old_memtable = std::mem::replace(&mut state.memtable, new_memtable); old_memtable.freeze(last_wal_entry_position); + + // Set up completion tracking on the outgoing table before it is retained + // and before any fallible dispatch, so the retained table already carries + // its cells and a failed send below can poison-and-return without leaving + // partial state to unwind. let _memtable_flush_watcher = old_memtable.create_memtable_flush_completion(); - if pending_wal_range.is_some() { + // The outgoing memtable may still owe an index apply — the puts that + // filled it triggered one, but the task need not have drained yet, and + // this is the last chance to name that store. Its L0 flush is gated on + // the WAL append (below), not on indexing, so without this its tail could + // stay unindexed and invisible for the rest of its life. + let pending_index_apply = match old_memtable.indexes_arc() { + Some(old_indexes) if old_indexes.indexed_count() < old_batch_store.len() => { + Some((old_batch_store.clone(), old_indexes, old_batch_store.len())) + } + _ => None, + }; + + let pending_wal_flush = if pending_wal_range.is_some() { let completion_cell: WatchableOnceCell< std::result::Result, > = WatchableOnceCell::new(); - let completion_reader = completion_cell.reader(); - old_memtable.set_wal_flush_completion(completion_reader); - - let end_batch_position = old_batch_store.len(); - self.wal_flusher.trigger_flush( - WalFlushSource::BatchStore { - batch_store: old_batch_store, - indexes: old_indexes, - }, - end_batch_position, - Some(completion_cell), - )?; - } - - let frozen_size = old_memtable.estimated_size(); - state.frozen_memtable_bytes += frozen_size; + old_memtable.set_wal_flush_completion(completion_cell.reader()); + Some((old_batch_store.len(), completion_cell)) + } else { + None + }; let flush_watcher = old_memtable .get_memtable_flush_watcher() .expect("Flush watcher should exist after create_memtable_flush_completion"); - state - .frozen_flush_watchers - .push_back((frozen_size, flush_watcher)); + state.frozen_flush_watchers.push_back(flush_watcher); let frozen_memtable = Arc::new(old_memtable); - // Keep this generation queryable past its manifest commit (swept after - // the grace by `SweepExpired`). Arc refcount, not a copy — the flush - // task holds it alive for the whole drain anyway. + // Retain the outgoing table in the read view *before* the fallible + // dispatches below. `state.memtable` was already replaced, so a failed + // send that returned here without this push would drop the table and its + // accepted rows would silently vanish from every scan. Keep it queryable + // past its manifest commit too (swept after the grace by `SweepExpired`); + // Arc refcount, not a copy — the flush task holds it alive anyway. state.frozen_memtables.push_back(FrozenMemTable { memtable: frozen_memtable.clone(), flushed_at_ms: None, }); + // The memtable set changed: re-derive. Before the fallible dispatches + // below, so a poisoned writer still reports the bytes it is holding. + publish_memory(&self.memory, state); + + // Dispatch can only fail if a background task's channel is already closed, + // i.e. the writer is being torn down. Poison so the read path fails fast + // with the typed error instead of serving the retained-but-never-durable + // tail, then return — the table stays in the read view. + if let Some((batch_store, indexes, end_batch_position)) = pending_index_apply { + self.trigger_index_apply(batch_store, indexes, end_batch_position) + .inspect_err(|e| self.wal_flusher.poison(e))?; + } + + if let Some((end_batch_position, completion_cell)) = pending_wal_flush { + self.wal_flusher + .trigger_flush( + WalFlushSource::BatchStore { + batch_store: old_batch_store, + }, + end_batch_position, + Some(completion_cell), + ) + .inspect_err(|e| self.wal_flusher.poison(e))?; + } + debug!( "Frozen memtable generation {}, pending_count = {}", next_generation - 1, @@ -1131,21 +1828,50 @@ impl SharedWriterState { Ok(next_generation) } - /// Track batch for WAL durability. - fn track_batch_for_wal(&self, batch_position: usize) -> super::wal::BatchDurableWatcher { - self.wal_flusher.track_batch(batch_position) + /// Watch for a write to become visible: indexed, and — in durable mode — + /// WAL-durable too. + /// + /// `target_indexed` is memtable-local; `target_durable` is writer-global. + /// Different coordinate spaces on purpose — see `WalFlusher::track_batch`. + fn track_batch_for_wal( + &self, + indexes: Option>, + target_indexed: usize, + target_durable: usize, + ) -> super::wal::BatchDurableWatcher { + self.wal_flusher + .track_batch(indexes, target_indexed, target_durable) } /// Check if memtable flush is needed and trigger if so. /// + /// `incoming_batches` / `incoming_rows`: see [`memtable_reached_flush_threshold`]. + /// /// Takes `&mut WriterState` directly since caller already holds the lock. - fn maybe_trigger_memtable_flush(&self, state: &mut WriterState) -> Result<()> { + fn maybe_trigger_memtable_flush( + &self, + state: &mut WriterState, + incoming_batches: usize, + incoming_rows: usize, + ) -> Result<()> { if state.flush_requested { return Ok(()); } - let should_flush = state.memtable.estimated_size() >= self.config.max_memtable_size - || state.memtable.is_batch_store_full(); + // An empty memtable has nothing to seal, and freezing one would spin: its + // indexes alone can sit above the ceiling. + if state.memtable.batch_count() == 0 { + return Ok(()); + } + + let should_flush = memtable_reached_flush_threshold( + &state.memtable, + self.config.max_memtable_size, + self.config.max_memtable_rows, + self.config.max_unflushed_memtable_bytes, + incoming_batches, + incoming_rows, + ); if should_flush { state.flush_requested = true; @@ -1162,12 +1888,11 @@ impl SharedWriterState { let threshold = self.config.max_wal_buffer_size; let batch_count = state.memtable.batch_count(); - let total_bytes = state.memtable.estimated_size(); + let total_bytes = state.memtable.batch_store().row_bytes(); let batch_store = state.memtable.batch_store(); - let indexes = state.memtable.indexes_arc(); // Check if there are any unflushed batches - let has_pending = batch_store.pending_wal_flush_count() > 0; + let has_pending = batch_store.pending_wal_flush_count(self.wal_flusher.durable()) > 0; // Check time-based trigger first let time_trigger = if let Some(interval) = self.config.max_wal_flush_interval { @@ -1196,10 +1921,7 @@ impl SharedWriterState { // If time trigger fired, send a flush message if time_trigger.is_some() { let _ = self.wal_flush_tx.send(TriggerWalFlush { - source: WalFlushSource::BatchStore { - batch_store, - indexes, - }, + source: WalFlushSource::BatchStore { batch_store }, end_batch_position: batch_count, done: None, }); @@ -1224,7 +1946,6 @@ impl SharedWriterState { let _ = self.wal_flush_tx.send(TriggerWalFlush { source: WalFlushSource::BatchStore { batch_store: batch_store.clone(), - indexes: indexes.clone(), }, end_batch_position: batch_count, done: None, @@ -1233,34 +1954,6 @@ impl SharedWriterState { } } -impl SharedWriterState { - fn unflushed_memtable_bytes(&self) -> usize { - // Total unflushed bytes = active memtable + all frozen memtables - self.state - .try_read() - .ok() - .map(|s| { - let active = s.memtable.estimated_size(); - active + s.frozen_memtable_bytes - }) - .unwrap_or(0) - } - - fn oldest_memtable_watcher(&self) -> Option { - // Return a watcher for the oldest frozen memtable's flush completion. - // If no frozen memtables, return the active memtable's watcher since it will - // eventually be frozen and flushed. - self.state.try_read().ok().and_then(|s| { - // First try frozen memtable watchers - s.frozen_flush_watchers - .front() - .map(|(_, watcher)| watcher.clone()) - // If no frozen memtables, use active memtable's watcher - .or_else(|| s.memtable.get_memtable_flush_watcher()) - }) - } -} - /// Trigger-tracking state for WAL-only mode (no MemTable). /// /// MemTable mode keeps these counters inside `WriterState`. WAL-only mode @@ -1290,7 +1983,7 @@ enum WriterMode { MemTable { state: Arc>, writer_state: Arc, - backpressure: BackpressureController, + backpressure: Arc, }, /// WAL-only mode: drainable pending-batch queue + WAL pipeline. No /// MemTable, no indexes, no Lance file flushing. @@ -1298,7 +1991,7 @@ enum WriterMode { state: Arc, wal_flush_tx: mpsc::UnboundedSender, trigger: StdRwLock, - backpressure: BackpressureController, + backpressure: Arc, }, } @@ -1311,6 +2004,12 @@ pub struct ShardWriter { manifest_store: Arc, stats: SharedWriteStats, mode: WriterMode, + /// The base table's schema as the caller passed it — no `_tombstone`, + /// nullability untouched. Caller input is held to it (see + /// [`Self::validate_against_logical_schema`]) and the scan narrows back to + /// it; the memtable, WAL, and SSTables carry the widened storage schema + /// ([`relax_non_pk_nullability`]) instead. + logical_schema: Arc, } impl ShardWriter { @@ -1334,10 +2033,26 @@ impl ShardWriter { )); } - // Callers pass the base schema; lance owns the `_tombstone` column and - // appends it here so the memtable/generation schema = base + tombstone. - // Idempotent, so a reopen that already extended the schema is a no-op. - let schema = schema_with_tombstone(&schema); + // A durable writer needs a flush ticker to make progress, in either + // mode. With `durable_write` on, a put becomes durable only once its WAL + // append lands, and neither mode self-triggers that append per put — the + // background ticker drives it. Without an interval (or with a zero one, + // which tokio cannot schedule), a small put that never fills the + // size-triggered buffer would block until close. Reject the config here + // rather than let a put hang. + if config.durable_write && config.max_wal_flush_interval.is_none_or(|d| d.is_zero()) { + return Err(Error::invalid_input( + "durable_write requires a positive max_wal_flush_interval: with no \ + flush ticker a durable put has nothing to drive its WAL append and \ + would block until close", + )); + } + + // The caller's schema is the shard's logical schema; the storage schema + // is derived below, once the primary key is known. lance owns + // `_tombstone` and appends it here — idempotent across reopens. + let logical_schema = schema; + let tombstoned = schema_with_tombstone(&logical_schema); let base_uri = base_uri.into(); let shard_id = config.shard_id; @@ -1348,6 +2063,90 @@ impl ShardWriter { config.manifest_scan_batch_size, )); + // Derive PK metadata and run every side-effect-free validation *before* + // claiming the epoch. `claim_epoch` durably bumps the stored epoch and, + // for a successor, `write_fence_sentinel` fences the predecessor — so an + // open doomed by purely local input (an index config that disagrees with + // the schema) must fail here, before it can knock the healthy incumbent off + // the shard. Memtable-only: WAL-only mode has no indexes to validate. + let memtable_validation = if config.enable_memtable { + let lance_schema = Schema::try_from(tombstoned.as_ref())?; + let pk_fields = lance_schema.unenforced_primary_key(); + let pk_field_ids: Vec = pk_fields.iter().map(|f| f.id).collect(); + let pk_columns: Vec = pk_fields.iter().map(|f| f.name.clone()).collect(); + + // Reject an index config that disagrees with the schema *before* a + // single row is accepted. Such a config fails deterministically on + // every insert, including inserts replayed from the WAL — so once a row + // is durable the shard can never reopen. Fail the open instead. + validate_index_configs( + &index_configs, + tombstoned.as_ref(), + &lance_schema, + &pk_columns, + )?; + + // An HNSW graph reserves its whole capacity before the first insert, + // but the seal trigger only measures row bytes. A reservation with no + // room left under the ceiling puts the shard over budget at zero rows + // — nothing to seal, so nothing to flush, so every put stalls and then + // fails as `Error::Backpressure`, which is supposed to mean "retry + // later". Reject the config instead; only the built-in valve reads + // this ceiling. + if config.backpressure.is_none() { + // The headroom the check below reserves for rows. At zero it + // reserves nothing, so a fresh memtable's index reservation may + // *equal* the ceiling — over budget before its first row, with an + // empty memtable there is nothing to seal, and the writer refuses + // its first write forever. A zero threshold is degenerate anyway: + // it seals every memtable at every insert. + if config.max_memtable_size == 0 { + return Err(Error::invalid_input( + "max_memtable_size must be greater than zero: it is both the \ + seal threshold for row data and the headroom reserved for rows \ + under max_unflushed_memtable_bytes, and at zero a writer with \ + in-memory indexes can be at its ceiling before its first row", + )); + } + + // Built the way `make_bound_memtable` builds it below, so this is + // the figure the controller will actually read. + let mut indexes = IndexStore::from_configs( + &index_configs, + config.max_memtable_rows, + config.max_memtable_batches, + )?; + if !pk_columns.is_empty() { + indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids)); + } + let reserved = indexes.resident_bytes() + super::memtable::pk_bloom_filter_bytes(); + // Room for a full memtable of rows on top, or the ceiling is + // crossed before `max_memtable_size` can seal. + let needed = reserved.saturating_add(config.max_memtable_size); + if needed > config.max_unflushed_memtable_bytes { + return Err(Error::invalid_input(format!( + "in-memory indexes reserve {reserved} bytes at \ + max_memtable_rows={}, and max_memtable_size={} must fit alongside them, \ + needing {needed} bytes; max_unflushed_memtable_bytes={} is below that, \ + so the active memtable would cross the backpressure ceiling before \ + accruing enough row bytes to seal, stalling every write. Raise \ + max_unflushed_memtable_bytes to at least {needed}, or lower \ + max_memtable_rows / max_memtable_size", + config.max_memtable_rows, + config.max_memtable_size, + config.max_unflushed_memtable_bytes, + ))); + } + } + + // Widen only now that the primary key is known — a tombstone nulls + // every non-PK column, and PK detection needs the strict schema. + let storage_schema = relax_non_pk_nullability(&tombstoned, &pk_columns); + Some((pk_field_ids, pk_columns, storage_schema)) + } else { + None + }; + // Claim the shard (epoch-based fencing) — done once, then shared // with the WalAppender via `with_claimed_epoch`. let (epoch, manifest) = manifest_store.claim_epoch(config.shard_spec_id).await?; @@ -1389,7 +2188,12 @@ impl ShardWriter { } // Create WAL flusher backed by the shared appender. - let mut wal_flusher = WalFlusher::new(wal_appender); + // Build the cursors from *this writer's* config. `durable_write` is what + // decides whether durability is part of visibility, so a flusher that + // defaulted it would leave a non-durable put waiting on a durability + // cursor nothing ever advances. + let cursors = Arc::new(WriterCursors::new(config.durable_write)); + let mut wal_flusher = WalFlusher::with_cursors(wal_appender, cursors); let (wal_flush_tx, wal_flush_rx) = mpsc::unbounded_channel(); wal_flusher.set_flush_channel(wal_flush_tx.clone()); @@ -1399,11 +2203,15 @@ impl ShardWriter { let task_executor = Arc::new(TaskExecutor::new()); let mode = if config.enable_memtable { + let (pk_field_ids, pk_columns, storage_schema) = memtable_validation + .expect("memtable_validation is Some when enable_memtable is true"); Self::open_memtable_mode( &config, - &schema, + &storage_schema, &manifest, &index_configs, + pk_field_ids, + pk_columns, wal_flusher.clone(), wal_flush_tx, wal_flush_rx, @@ -1436,6 +2244,7 @@ impl ShardWriter { manifest_store, stats, mode, + logical_schema, }) } @@ -1445,6 +2254,8 @@ impl ShardWriter { schema: &Arc, manifest: &ShardManifest, index_configs: &[MemIndexConfig], + pk_field_ids: Vec, + pk_columns: Vec, wal_flusher: Arc, wal_flush_tx: mpsc::UnboundedSender, wal_flush_rx: mpsc::UnboundedReceiver, @@ -1457,50 +2268,118 @@ impl ShardWriter { stats: SharedWriteStats, task_executor: &Arc, ) -> Result { - // Create MemTable with primary key field IDs from schema - let lance_schema = Schema::try_from(schema.as_ref())?; - let pk_fields = lance_schema.unenforced_primary_key(); - let pk_field_ids: Vec = pk_fields.iter().map(|f| f.id).collect(); - let pk_columns: Vec = pk_fields.iter().map(|f| f.name.clone()).collect(); - let mut memtable = MemTable::with_capacity( - schema.clone(), - manifest.current_generation, - pk_field_ids.clone(), - CacheConfig::default(), - config.max_memtable_batches, - )?; - - // Create indexes if configured and set them on the MemTable. The - // PK-position index is enabled before any WAL replay below so replayed - // rows are recorded in it. A primary key alone (no secondary index) - // still needs the PK index so flush writes its on-disk dedup sidecar. - if !index_configs.is_empty() || !pk_columns.is_empty() { + // PK metadata and index/interval validation were resolved in `open` + // before the epoch was claimed (a doomed open must not fence the + // incumbent first). + + // Build a fresh, cursor-bound memtable at a given generation and + // writer-global coordinate. Replay calls this for the first memtable and + // after every rotation. Always builds and binds an `IndexStore`, even + // with no user indexes and no primary key — see the note in + // `freeze_memtable` for why an index-less memtable still needs one. + let make_bound_memtable = |generation: u64, global_offset: usize| -> Result { + let mut memtable = MemTable::with_capacity_at( + schema.clone(), + generation, + pk_field_ids.clone(), + CacheConfig::default(), + config.max_memtable_batches, + global_offset, + )?; let mut indexes = IndexStore::from_configs( index_configs, config.max_memtable_rows, config.max_memtable_batches, )?; - indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids)); + if !pk_columns.is_empty() { + indexes.enable_pk_index(&pk_index_columns(&pk_columns, &pk_field_ids)); + } + indexes.set_durability(Arc::clone(wal_flusher.cursors()), global_offset); memtable.set_indexes_arc(Arc::new(indexes)); - } + Ok(memtable) + }; + + // The flusher writes sealed memtables to Lance generations — both the + // ones replay seals below and the ones the live path freezes later. + let flusher = Arc::new( + MemTableFlusher::new( + object_store.clone(), + base_path.clone(), + base_uri.clone(), + shard_id, + manifest_store.clone(), + ) + .with_warmer(config.warmer.clone()) + .with_storage_context(config.store_params.clone(), config.session.clone()), + ); // Replay any WAL entries written after the last successfully-flushed - // generation. Each entry's writer_epoch is checked against ours; an - // entry with a strictly greater epoch indicates a successor writer - // claimed the shard between our `claim_epoch` and replay, so we - // abort the open with a fence error. The replay walked the tailer - // up to the WAL tip, so we hand the discovered next-write position - // straight to the appender — its first append skips the - // discover_next_position probe entirely. - let next_wal_position = replay_memtable_from_wal( + // SSTable, flushing sealed memtables to Lance SSTables as the batch + // store fills. Each entry's writer_epoch is checked against ours; an entry + // with a strictly greater epoch means a successor claimed the shard + // between our `claim_epoch` and replay, so we abort with a fence error. + // Replay walks the tailer to the WAL tip and returns the discovered + // next-write position, so the appender's first append skips the + // discover_next_position probe. + let ReplayResult { + active: memtable, + next_wal_position, + } = replay_memtable_from_wal( object_store.clone(), base_path.clone(), shard_id, epoch, manifest, - &mut memtable, + manifest.current_generation, + make_bound_memtable, + &flusher, + &wal_flusher, + index_configs, + config.max_memtable_size, + config.max_memtable_rows, + config.max_unflushed_memtable_bytes, ) .await?; + + // Publish the read cursor once, now that replay knows the tip. The + // tailer used to write this per entry, but a tailer holds no claim; + // here it rides the epoch holder's normal commit path. Best-effort: + // it is a hint for other readers, and `position_hint_seed` already + // tolerates it lagging behind `replay_after_wal_entry_position`. + let replayed_through = next_wal_position.saturating_sub(1); + if replayed_through > manifest.wal_entry_position_last_seen + && let Err(error) = manifest_store + .commit_update(epoch, |current| ShardManifest { + version: current.next_version(), + wal_entry_position_last_seen: current + .wal_entry_position_last_seen + .max(replayed_through), + ..current.clone() + }) + .await + { + warn!( + "failed to publish WAL read cursor {} for shard {}: {}", + replayed_through, shard_id, error + ); + } + + // Mark the active memtable's replayed batches durable. They came *from* + // the WAL, and replay has already re-derived its indexes over them. + // + // Without this the durability cursor stays at the last sealed generation, + // so the next WAL flush re-covers the active tail: it re-appends the + // already-durable rows *and* re-inserts every replayed row into the + // indexes. None of the three in-memory indexes is idempotent (HNSW mints + // fresh node ids for the same row, FTS increments doc_count/df rather than + // recomputing them, BTree is a multiset), so a full scan keeps looking + // healthy while every index-accelerated query silently returns duplicates + // — and it compounds, because the WAL now holds those rows twice. + // + // `global_end()` is the writer-global batch count through this memtable, + // since its coordinate continues where the last sealed generation ended. + wal_flusher.advance_durable(memtable.batch_store().global_end()); + wal_flusher .wal_appender() .seed_next_position(next_wal_position) @@ -1512,35 +2391,45 @@ impl ShardWriter { // it is durably reflected in this writer's memtable. We can't // seed from `manifest.wal_entry_position_last_seen` — that field // is bumped on every successful tailer read by other readers, so - // it may sit above what's actually covered by any flushed - // generation. Subtracting 1 from a fresh shard's `next_wal_position` + // it may sit above what's actually covered by any + // SSTable. Subtracting 1 from a fresh shard's `next_wal_position` // of `FIRST_WAL_ENTRY_POSITION` (= 1) yields 0, which correctly // means "no entry covered yet." let initial_covered_wal_entry_position = next_wal_position.saturating_sub(1); - let state = Arc::new(RwLock::new(WriterState { + let memory = Arc::new(ArcSwap::::default()); + + let state = WriterState { memtable, last_flushed_wal_entry_position: initial_covered_wal_entry_position, - frozen_memtable_bytes: 0, frozen_flush_watchers: VecDeque::new(), frozen_memtables: VecDeque::new(), flush_requested: false, wal_flush_trigger_count: 0, last_wal_flush_trigger_time: 0, - })); + }; + // Seed before the first freeze: replay above may already have filled the + // memtable, and nothing else publishes until it seals. + publish_memory(&memory, &state); + let state = Arc::new(RwLock::new(state)); let (memtable_flush_tx, memtable_flush_rx) = mpsc::unbounded_channel(); let flusher = Arc::new( MemTableFlusher::new(object_store, base_path, base_uri, shard_id, manifest_store) - .with_warmer(config.warmer.clone()), + .with_warmer(config.warmer.clone()) + .with_storage_context(config.store_params.clone(), config.session.clone()), ); - let backpressure = BackpressureController::new(config.clone()); - // Background WAL flush handler — parallel WAL I/O + index updates. - let wal_handler = - WalFlushHandler::new(wal_flusher.clone(), Some(state.clone()), stats.clone()); + let wal_handler = WalFlushHandler::new( + wal_flusher.clone(), + Some(state.clone()), + None, + config.max_wal_flush_interval, + stats.clone(), + config.observer.clone(), + ); task_executor.add_handler( "wal_flusher".to_string(), Box::new(wal_handler), @@ -1548,13 +2437,16 @@ impl ShardWriter { )?; // Background MemTable flush handler — frozen memtable to Lance file. - // It rebuilds the same secondary indexes on each flushed generation. + // It rebuilds the same secondary indexes on each SSTable. let memtable_handler = MemTableFlushHandler::new( state.clone(), + memory.clone(), flusher, + wal_flusher.clone(), epoch, index_configs.to_vec(), - stats, + stats.clone(), + config.observer.clone(), config.frozen_memtable_grace, ); task_executor.add_handler( @@ -1563,11 +2455,27 @@ impl ShardWriter { memtable_flush_rx, )?; + // The index-apply task. Its own channel and its own dispatcher: the + // dispatcher awaits `handle()` inline, so sharing the WAL flusher's + // channel would queue every index apply behind a ~100ms S3 PUT. + let (index_apply_tx, index_apply_rx) = mpsc::unbounded_channel(); + let index_handler = IndexApplyHandler { + cursors: Arc::clone(wal_flusher.cursors()), + wal_flusher: wal_flusher.clone(), + stats, + }; + task_executor.add_handler( + "index_applier".to_string(), + Box::new(index_handler), + index_apply_rx, + )?; + // Shared state used by `put()` to dispatch trigger checks. let writer_state = Arc::new(SharedWriterState::new( - state.clone(), + memory, wal_flusher, wal_flush_tx, + index_apply_tx, memtable_flush_tx, config.clone(), schema.clone(), @@ -1578,6 +2486,8 @@ impl ShardWriter { index_configs.to_vec(), )); + let backpressure = resolve_backpressure(config); + Ok(WriterMode::MemTable { state, writer_state, @@ -1593,24 +2503,44 @@ impl ShardWriter { stats: SharedWriteStats, task_executor: &Arc, ) -> Result { + // The pending queue is shared with the flush handler so a background + // tick can resolve to the batches still owed an append — the WAL-only + // analog of resolving a tick against the durability cursor in MemTable + // mode. A durable put waits on the durability cursor the handler's + // append advances, so the ticker must run (`open` rejects durable + + // no-interval); a non-durable writer may pass no interval and rely on + // the size/close triggers alone. + let state = Arc::new(WalOnlyState::default()); + // Background WAL flush handler — no MemTable state to consult, so - // pass `None` for the frozen-vs-active detection. - let wal_handler = WalFlushHandler::new(wal_flusher, None, stats); + // pass `None` for the frozen-vs-active detection; the pending queue and + // the flush interval drive the background append instead. + let wal_handler = WalFlushHandler::new( + wal_flusher, + None, + Some(state.clone()), + config.max_wal_flush_interval, + stats, + config.observer.clone(), + ); task_executor.add_handler( "wal_flusher".to_string(), Box::new(wal_handler), wal_flush_rx, )?; - // Reuse `BackpressureController` (which is keyed off - // `max_unflushed_memtable_bytes`) as the WAL-only backpressure - // budget. WAL-only callers feed it `WalOnlyState::estimated_size()`. - // Keeps the config knob meaningful in WAL-only mode and prevents - // the pending queue from growing unbounded under non-durable writes. - let backpressure = BackpressureController::new(config.clone()); + // Reuse the memtable valve (keyed off `max_unflushed_memtable_bytes`) + // as the WAL-only budget, fed `WalOnlyState::queue_bytes()`. Keeps + // the config knob meaningful in WAL-only mode and prevents the pending + // queue from growing unbounded under non-durable writes. + // + // The *same* `state` the flush handler was given above: a second + // `WalOnlyState` here would leave writes queuing on one and the + // background append draining the other, forever empty. + let backpressure = resolve_backpressure(config); Ok(WriterMode::WalOnly { - state: Arc::new(WalOnlyState::default()), + state, wal_flush_tx, trigger: StdRwLock::new(WalOnlyTriggerState::default()), backpressure, @@ -1639,6 +2569,7 @@ impl ShardWriter { #[instrument(name = "sw_put", level = "info", skip_all, fields(batch_count = batches.len(), shard_id = %self.config.shard_id))] pub async fn put(&self, batches: Vec) -> Result { Self::validate_non_empty(&batches)?; + self.validate_against_logical_schema(&batches)?; match &self.mode { WriterMode::MemTable { @@ -1646,9 +2577,8 @@ impl ShardWriter { writer_state, backpressure, } => { - // Inject `_tombstone = false` so the batch matches the - // extended memtable schema; callers only ever pass base-shaped - // batches and never name the column. + // Callers pass logical-shaped batches and never name + // `_tombstone`. let batches = batches .into_iter() .map(|b| ensure_tombstone_column(b, &writer_state.schema)) @@ -1677,9 +2607,8 @@ impl ShardWriter { /// its key: it wins newest-per-PK resolution (suppressing the older real /// row) and is then dropped from query results. /// - /// Only supported in memtable mode. Because a tombstone nulls every non-PK - /// column, those columns must be nullable in the base schema; a delete - /// against a schema with a non-nullable non-PK column errors. + /// Only supported in memtable mode. Works against non-nullable base columns: + /// tombstones live in the storage schema, which widens them to nullable. /// /// ``` /// # use lance::Result; @@ -1766,6 +2695,7 @@ impl ShardWriter { batches: Vec, ) -> Result<(WriteResult, Option)> { Self::validate_non_empty(&batches)?; + self.validate_against_logical_schema(&batches)?; match &self.mode { WriterMode::MemTable { @@ -1773,8 +2703,7 @@ impl ShardWriter { writer_state, backpressure, } => { - // Inject `_tombstone = false` to match the extended memtable - // schema, mirroring `put`. + // Mirrors `put`. let batches = batches .into_iter() .map(|b| ensure_tombstone_column(b, &writer_state.schema)) @@ -1788,6 +2717,49 @@ impl ShardWriter { } } + /// Reject caller input that violates the logical schema: wrong column names, + /// order, count, or types, or a null where the base table declares + /// non-nullable. + /// + /// The *only* gate on that contract — the storage schema accepts the null, + /// append and `merge_insert` compare with `NullabilityComparison::Ignore`, + /// and the encoder takes validity from the array, not the field — so a null + /// that gets past here reaches the base table silently. + /// + /// Runs before the WAL append: a batch rejected only afterwards would fail + /// identically on every replay, leaving the shard unable to reopen. + fn validate_against_logical_schema(&self, batches: &[RecordBatch]) -> Result<()> { + for (i, batch) in batches.iter().enumerate() { + // Everything downstream matches columns by position, so a swapped + // pair of same-typed columns would be stored under each other's + // names unless caught here. + for (col, (expected, actual)) in self + .logical_schema + .fields() + .iter() + .zip(batch.schema().fields()) + .enumerate() + { + if expected.name() != actual.name() { + return Err(Error::invalid_input(format!( + "batch {i} column {col} is named '{}', but the base table schema \ + declares '{}' at that position", + actual.name(), + expected.name() + ))); + } + } + RecordBatch::try_new(self.logical_schema.clone(), batch.columns().to_vec()).map_err( + |e| { + Error::invalid_input(format!( + "batch {i} does not match the base table schema: {e}" + )) + }, + )?; + } + Ok(()) + } + fn validate_non_empty(batches: &[RecordBatch]) -> Result<()> { if batches.is_empty() { return Err(Error::invalid_input("Cannot write empty batch list")); @@ -1805,7 +2777,7 @@ impl ShardWriter { batches: Vec, state_lock: &Arc>, writer_state: &Arc, - backpressure: &BackpressureController, + backpressure: &Arc, ) -> Result { let (result, watcher) = self .put_memtable_no_wait(batches, state_lock, writer_state, backpressure) @@ -1825,20 +2797,46 @@ impl ShardWriter { batches: Vec, state_lock: &Arc>, writer_state: &Arc, - backpressure: &BackpressureController, + backpressure: &Arc, ) -> Result<(WriteResult, Option)> { // Reject writes on a fenced writer before mutating the memtable, so a // poisoned writer can't drift further from the durable WAL. self.wal_flusher.check_poisoned()?; + // A write lands whole in one memtable, so one larger than the cap fits + // nowhere — a fresh memtable overflows the same way. Deletes arrive here + // as tombstone rows and are bounded the same way. + let incoming_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + if incoming_rows > self.config.max_memtable_rows { + return Err(Error::invalid_input(format!( + "write of {incoming_rows} rows across {} batches exceeds \ + max_memtable_rows={}: a write is never split across memtables, and the \ + in-memory indexes are sized to that cap. Split the write, or raise \ + max_memtable_rows", + batches.len(), + self.config.max_memtable_rows, + ))); + } + + // The seal check runs inside the lock immediately after an insert, but the + // index apply that follows it runs *outside* — and replay hands back a + // memtable whose indexes were built after its last check too. Either way + // the ceiling can be reached with nothing sealed, and the valve below + // would then refuse a write that a seal would have admitted. Re-run the + // check here so the wait has a flush to end on. + // + // The read is two relaxed loads; the lock is taken only when the shard is + // actually at its ceiling, which is the path already about to block. + if ShardMemory::memtables(writer_state.memory.clone()).unflushed_bytes() + >= self.config.max_unflushed_memtable_bytes + { + let mut state = state_lock.write().await; + writer_state.maybe_trigger_memtable_flush(&mut state, 1, 1)?; + } + // Apply backpressure if needed (before acquiring main lock) backpressure - .maybe_apply_backpressure(|| { - ( - writer_state.unflushed_memtable_bytes(), - writer_state.oldest_memtable_watcher(), - ) - }) + .maybe_apply_backpressure(ShardMemory::memtables(writer_state.memory.clone())) .await?; let start = std::time::Instant::now(); @@ -1847,49 +2845,81 @@ impl ShardWriter { let (batch_positions, durable_watcher, batch_store, indexes) = { let mut state = state_lock.write().await; + // 0. Seal first if this put would not fit: the row cap is a hard + // index capacity, so an overshoot cannot be undone afterwards. + writer_state.maybe_trigger_memtable_flush(&mut state, batches.len(), incoming_rows)?; + // 1. Insert all batches into memtable atomically let results = state.memtable.insert_batches_only(batches).await?; - // Get batch position range + // 2. Capture the store the batches actually landed in, *before* step + // 6 below can freeze and swap the active memtable. Reading it + // afterwards hands the flush trigger the **new** store paired with + // the **old** store's end position, so the new store's watermark + // jumps past batches that were never appended. + let batch_store = state.memtable.batch_store(); + let indexes = state.memtable.indexes_arc(); + let start_pos = results.first().map(|(pos, _, _)| *pos).unwrap_or(0); let end_pos = results.last().map(|(pos, _, _)| pos + 1).unwrap_or(0); let batch_positions = start_pos..end_pos; - // 2. Track last batch for WAL durability - let durable_watcher = writer_state.track_batch_for_wal(end_pos.saturating_sub(1)); + // 4. Watch for this write to become *visible*: indexed, and — under + // `durable_write` — WAL-durable too. + // + // The two targets live in different coordinate spaces. `end_pos` + // is memtable-local, which is what the index apply works in. The + // durability cursor is writer-global, because batch positions + // restart at 0 in every memtable while that cursor spans the + // writer's whole life — a local durable target would already be + // satisfied by a *previous* memtable's appends, so the first N + // puts into every post-rotation memtable would ack as durable with + // no WAL append ever happening. + let durable_watcher = writer_state.track_batch_for_wal( + indexes.clone(), + end_pos, + batch_store.global_offset() + end_pos, + ); - // 3. Check if WAL flush should be triggered + // 5. Check if WAL flush should be triggered writer_state.maybe_trigger_wal_flush(&mut state); - // 4. Check if memtable flush is needed - if let Err(e) = writer_state.maybe_trigger_memtable_flush(&mut state) { + // 6. Check if memtable flush is needed (may freeze and rotate) + if let Err(e) = writer_state.maybe_trigger_memtable_flush(&mut state, 1, 1) { warn!("Failed to trigger memtable flush: {}", e); } - // Get batch_store and indexes while we have the lock (for durable_write case) - let batch_store = state.memtable.batch_store(); - let indexes = state.memtable.indexes_arc(); - (batch_positions, durable_watcher, batch_store, indexes) }; // Lock released here self.stats.record_put(start.elapsed()); - // Trigger the flush here (outside the lock) so the watcher can resolve; - // only the `wait()` is the caller's to schedule. - let watcher = if self.config.durable_write { - self.wal_flusher.trigger_flush( - WalFlushSource::BatchStore { - batch_store, - indexes, - }, - batch_positions.end, - None, - )?; - Some(durable_watcher) - } else { - None - }; + // Trigger the index apply, in **both** modes. This is what makes reads + // read-your-writes regardless of `durable_write`: skipping durability now + // costs the caller durability only, not visibility. It is cheap + // (in-memory, ~ms), so there is no reason to batch it onto the WAL's + // schedule — that schedule exists to bound S3 API cost, which an + // in-memory index apply does not incur. + if let Some(indexes) = indexes { + writer_state.trigger_index_apply(batch_store, indexes, batch_positions.end)?; + } + + // The WAL append is *not* triggered here. It happens on the background + // ticker (and on the size trigger, and at freeze/close), which is the only + // way the flush interval can mean anything: while every durable put + // triggered its own append, the interval could add a redundant trigger but + // never delay or batch one. + // + // The cost is real and accepted: a single client's sequential *durable* + // throughput drops from ~10 writes/sec (one PUT round-trip) to roughly one + // per tick. That is a policy choice — the interval should mean what it + // says, and S3 API cost should be bounded. Latency-sensitive callers want + // `durable_write: false`, which now costs them durability only, not + // visibility. + + // The watcher is returned in both modes now. A non-durable put still + // waits — for its index apply (~ms), not for an S3 PUT (~100ms). + let watcher = Some(durable_watcher); Ok((WriteResult { batch_positions }, watcher)) } @@ -1900,7 +2930,7 @@ impl ShardWriter { state: &Arc, wal_flush_tx: &mpsc::UnboundedSender, trigger: &StdRwLock, - backpressure: &BackpressureController, + backpressure: &Arc, ) -> Result { // Reject writes on a fenced writer before enqueuing — see // `put_memtable_no_wait`. @@ -1912,7 +2942,7 @@ impl ShardWriter { // shape as MemTable mode. WAL-only mode has no per-frozen-MemTable // watcher, so the backpressure loop falls back to its short sleep. backpressure - .maybe_apply_backpressure(|| (state.estimated_size(), None)) + .maybe_apply_backpressure(ShardMemory::queue(state.clone())) .await?; let start = std::time::Instant::now(); @@ -1920,54 +2950,48 @@ impl ShardWriter { // Push batches into the pending queue and capture the assigned // [start, end) range. `next_batch_position` is monotonic across the // writer's lifetime; positions are not BatchStore indices but they - // are used the same way for durability tracking. + // are used the same way for durability tracking. Because the queue is + // strict FIFO with contiguous positions and its front always sits at + // the durability cursor, `end` is exactly the writer-global durable + // count this put reaches once its append lands — no globalizing offset + // as in MemTable mode, which restarts positions per generation. let batch_positions = state.push(batches); - // Time- and size-based triggers, mirroring MemTable mode but reading - // pending bytes from `WalOnlyState` instead of an active MemTable. - // Only fires for non-durable writes; durable writes go through the - // explicit done-cell path below so flush errors (e.g., fence) reach - // the caller. - if !self.config.durable_write { - let target_position = batch_positions.end; - let pending_bytes = state.estimated_size(); - self.maybe_trigger_wal_flush_wal_only( - state, - wal_flush_tx, - trigger, - target_position, - pending_bytes, - ); - } + // Under `durable_write` the put becomes durable only once its WAL + // append lands. Track it on the writer-global durability cursor *before* + // triggering, so a flush that completes between here and the wait is not + // missed: the watcher recomputes visibility from the cursor rather than + // latching a one-shot wake. WAL-only mode has no indexes, so the + // index-visibility half of the watcher is a no-op (`None`, target 0). + let durable_watcher = self + .config + .durable_write + .then(|| self.wal_flusher.track_batch(None, 0, batch_positions.end)); + + // Time- and size-based triggers on the write path, for durable and + // non-durable puts alike — mirroring MemTable mode's + // `maybe_trigger_wal_flush`. The background ticker drives the append + // too; whichever fires first wins, and a redundant trigger is a cheap + // no-op because the flush snapshot/commit is idempotent. + self.maybe_trigger_wal_flush_wal_only( + state, + wal_flush_tx, + trigger, + batch_positions.end, + state.queue_bytes(), + ); self.stats.record_put(start.elapsed()); - // For durable writes, trigger an immediate flush and wait for the - // done cell. Using the done cell instead of the durability watermark - // watcher ensures flush errors (e.g., the WalAppender returning a - // fence error) propagate back to `put` instead of hanging. - if self.config.durable_write { - let done = WatchableOnceCell::new(); - let reader = done.reader(); - self.wal_flusher.trigger_flush( - WalFlushSource::WalOnly { - state: state.clone(), - }, - batch_positions.end, - Some(done), - )?; - let mut reader = reader; - match reader.await_value().await { - Some(Ok(_)) => {} - // Rebuild the typed error (peer fence vs. persistence-failure - // self-fence) so a WAL-only durable caller can tell them apart. - Some(Err(failure)) => return Err(failure.into_error()), - None => { - return Err(Error::io( - "WAL flush handler exited before reporting durability", - )); - } - } + // Durable writes wait on the durability cursor, advanced by the append + // the ticker (or the trigger above) drives — the same watermark path + // MemTable mode uses. A terminal flush failure (a peer fence or an + // exhausted-retry persistence self-fence) poisons the writer and wakes + // the waiter with that typed error instead of leaving it to hang. This + // is why `open` rejects `durable_write` with no flush ticker: nothing + // else would advance the cursor a small put is parked on. + if let Some(mut watcher) = durable_watcher { + watcher.wait().await?; } Ok(WriteResult { batch_positions }) @@ -2045,9 +3069,41 @@ impl ShardWriter { self.stats.clone() } + /// What this writer is holding in memory, in both write modes. + /// + /// The same [`ShardMemory`] the admission controller is handed, so an + /// embedder ranking shards by size and the controller gating a write read + /// one implementation rather than two that can disagree. + /// + /// Only a flush reclaims `unflushed_bytes`, so that is the pool to bound + /// against OOM; `retained_bytes` adds what is resident but already flushed + /// and waiting out `frozen_memtable_grace`. Either can far exceed + /// `max_memtable_size`, which gates row data alone. + pub fn memory(&self) -> ShardMemory { + match &self.mode { + WriterMode::MemTable { writer_state, .. } => { + ShardMemory::memtables(writer_state.memory.clone()) + } + WriterMode::WalOnly { state, .. } => ShardMemory::queue(state.clone()), + } + } + /// Get the current shard manifest. + /// + /// Served from this writer's own last manifest commit, so a peer's commit + /// may not appear. Fencing does not rely on this — [`Self::check_fenced`] + /// always reads storage. pub async fn manifest(&self) -> Result> { - self.manifest_store.read_latest().await + self.manifest_store.latest().await + } + + /// The shard's manifest store. + /// + /// Embedders must commit through this instance: a second + /// `ShardManifestStore` over the same shard keeps its own cache, and + /// neither would see the other's commits. + pub fn manifest_store(&self) -> Arc { + self.manifest_store.clone() } /// Get the writer's epoch. @@ -2067,36 +3123,52 @@ impl ShardWriter { /// Get current MemTable statistics. Returns an error in WAL-only mode /// (no MemTable exists). + /// + /// Deliberately does *not* `check_poisoned`, unlike the read and write + /// paths: a poisoned writer is exactly when an operator most needs to see + /// its state, and the caller deciding whether to evict reads these stats. pub async fn memtable_stats(&self) -> Result { let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; let batch_store = state.memtable.batch_store(); - let pending_wal = batch_store.pending_wal_flush_stats(); + let durable = self.wal_flusher.durable(); + let pending_wal = batch_store.pending_wal_flush_stats(durable); Ok(MemTableStats { row_count: state.memtable.row_count(), batch_count: state.memtable.batch_count(), - estimated_size: state.memtable.estimated_size(), generation: state.memtable.generation(), max_buffered_batch_position: batch_store.max_buffered_batch_position(), - max_flushed_batch_position: batch_store.max_flushed_batch_position(), + durable_batch_count: durable, + global_offset: batch_store.global_offset(), pending_wal_start_batch_position: pending_wal.start_batch_position, pending_wal_end_batch_position: pending_wal.end_batch_position, pending_wal_batch_count: pending_wal.batch_count, pending_wal_row_count: pending_wal.row_count, pending_wal_estimated_bytes: pending_wal.estimated_bytes, + frozen_count: state.frozen_memtables.len(), }) } + /// Snapshot of the backpressure counters. Both writer modes answer, so a + /// caller asking "am I throttled" need not know which mode it is in. + pub fn backpressure_stats(&self) -> BackpressureStatsSnapshot { + match &self.mode { + WriterMode::MemTable { backpressure, .. } + | WriterMode::WalOnly { backpressure, .. } => backpressure.stats_snapshot(), + } + } + /// Create a scanner for querying the current MemTable data. /// /// The scanner provides read access to all data currently in the MemTable, /// with optional filtering, projection, and index support. /// - /// The scanner captures the current `max_visible_batch_position` from the + /// The scanner captures the current `visible_count` from the /// `IndexStore` at construction time to ensure consistent visibility. /// - /// Returns an error in WAL-only mode. + /// Returns an error in WAL-only mode, or if the writer is poisoned. pub async fn scan(&self) -> Result { + self.wal_flusher.check_poisoned()?; let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; Ok(state.memtable.scan()) @@ -2106,10 +3178,11 @@ impl ShardWriter { /// Prefer [`Self::in_memory_memtable_refs`] on the read path — it also /// carries frozen-awaiting-flush generations. /// - /// Returns an error in WAL-only mode. + /// Returns an error in WAL-only mode, or if the writer is poisoned. pub async fn active_memtable_ref( &self, ) -> Result { + self.wal_flusher.check_poisoned()?; let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; Ok(in_memory_ref(&state.memtable)) @@ -2121,10 +3194,11 @@ impl ShardWriter { /// path uses this instead of [`Self::active_memtable_ref`] so a /// concurrent reader sees no hole while a flush drains. /// - /// Returns an error in WAL-only mode. + /// Returns an error in WAL-only mode, or if the writer is poisoned. pub async fn in_memory_memtable_refs( &self, ) -> Result { + self.wal_flusher.check_poisoned()?; let state_lock = self.memtable_state_lock()?; let state = state_lock.read().await; Ok(crate::dataset::mem_wal::scanner::InMemoryMemTables { @@ -2155,10 +3229,16 @@ impl ShardWriter { } } - /// Seal the active memtable so it's queued for L0 flush. No-op when - /// the active memtable is empty. Errors in WAL-only mode or if this - /// writer has been fenced by a successor. Pair with - /// [`Self::wait_for_flush_drain`] to wait for the queued flush. + /// Seal the active memtable so it's queued for L0 flush. Errors in + /// WAL-only mode or if this writer has been fenced by a successor. + /// + /// The returned [`SealFence`] is what makes a *bounded* wait possible: + /// a caller that needs "everything written before my seal is in L0" + /// awaits [`SealFence::wait`], which covers the flushes outstanding at + /// seal time and nothing else. [`Self::wait_for_flush_drain`] instead + /// loops until the frozen set is *empty*, re-collecting it each round, + /// so it also waits on every memtable frozen *while it waits*. Under + /// sustained writes that set may never empty. /// /// Beyond test setup where deterministic flush points are required, /// this is the primary lever for callers that need to drive flushes @@ -2168,7 +3248,7 @@ impl ShardWriter { /// the next epoch starts with no replayable entries from the old /// layout. #[instrument(name = "sw_force_seal_active", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] - pub async fn force_seal_active(&self) -> Result<()> { + pub async fn force_seal_active(&self) -> Result { match &self.mode { WriterMode::MemTable { state, @@ -2178,11 +3258,24 @@ impl ShardWriter { self.check_fenced().await?; self.wal_flusher.check_poisoned()?; let mut state = state.write().await; - if state.memtable.batch_count() == 0 { - return Ok(()); - } - writer_state.freeze_memtable(&mut state)?; - Ok(()) + let sealed_generation = if state.memtable.batch_count() == 0 { + None + } else { + let generation = state.memtable.generation(); + writer_state.freeze_memtable(&mut state)?; + Some(generation) + }; + // Capture the outstanding set under the same lock that froze, + // so no freeze can slip between the two and widen the fence. + // It covers more than the generation just sealed: a + // size/interval trigger freezes generations asynchronously, so + // an empty active memtable does *not* mean every pre-call write + // already reached L0. Watchers are popped as flushes settle, so + // whatever remains here is exactly what is still owed. + Ok(SealFence { + sealed_generation, + watchers: state.frozen_flush_watchers.iter().cloned().collect(), + }) } WriterMode::WalOnly { .. } => Err(Error::invalid_input( "force_seal_active not available in WAL-only mode (no MemTable)", @@ -2196,12 +3289,10 @@ impl ShardWriter { /// want everything-on-disk. Errors in WAL-only mode, or if any /// awaited flush reports `DurabilityResult::Failed`. /// - /// Useful in tests for deterministic post-flush assertions, and in - /// production wherever a caller needs a synchronous fence after - /// [`Self::force_seal_active`] — e.g. trimming memtable residency - /// across shards in a multi-table WAL writer, or ensuring the WAL - /// is fully drained to Lance storage before rolling to a new - /// format/epoch. + /// Useful in tests for deterministic post-flush assertions. In + /// production prefer [`SealFence::wait`] from the seal itself: this + /// drains the queue to empty, including memtables frozen after the + /// call, so under sustained writes it may not return. #[instrument(name = "sw_wait_for_flush_drain", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] pub async fn wait_for_flush_drain(&self) -> Result<()> { let state_lock = match &self.mode { @@ -2216,10 +3307,7 @@ impl ShardWriter { loop { let watchers: Vec = { let st = state_lock.read().await; - st.frozen_flush_watchers - .iter() - .map(|(_, w)| w.clone()) - .collect() + st.frozen_flush_watchers.iter().cloned().collect() }; if watchers.is_empty() { return Ok(()); @@ -2268,12 +3356,65 @@ impl ShardWriter { Ok(()) } + /// Send the close-time final WAL flush and await its completion. + /// + /// Sends directly on the flush channel rather than via + /// [`WalFlusher::trigger_flush`]: the latter silently returns `Ok` when the + /// flusher's `flush_tx` is unset, which would let close report success + /// without ever persisting the final WAL entry. A closed send channel must + /// surface as an error here so close never acknowledges durability it did + /// not achieve. + async fn flush_final_wal( + wal_flush_tx: &mpsc::UnboundedSender, + source: WalFlushSource, + end_batch_position: usize, + ) -> Result<()> { + let done = WatchableOnceCell::new(); + let mut reader = done.reader(); + if wal_flush_tx + .send(TriggerWalFlush { + source, + end_batch_position, + done: Some(done), + }) + .is_err() + { + return Err(Error::io("WAL flush channel closed during close")); + } + + match reader.await_value().await { + Some(Ok(_)) => Ok(()), + Some(Err(failure)) => Err(failure.into_error()), + None => Err(Error::io( + "WAL flush handler exited before reporting durability during close", + )), + } + } + + fn merge_close_stage( + close_result: Result<()>, + stage: &str, + stage_result: Result<()>, + ) -> Result<()> { + if let (Err(_), Err(stage_error)) = (&close_result, &stage_result) { + warn!("Close stage '{stage}' also failed: {stage_error}"); + } + close_result.and(stage_result) + } + /// Close the writer gracefully. /// /// Flushes pending data and shuts down background tasks. + /// + /// # Errors + /// + /// Returns an error if pending WAL data cannot be persisted, an active or + /// frozen MemTable cannot be flushed, a flush handler exits before reporting + /// completion, or background tasks cannot be shut down. #[instrument(name = "sw_close", level = "info", skip_all, fields(shard_id = %self.config.shard_id, epoch = self.epoch))] pub async fn close(self) -> Result<()> { info!("Closing ShardWriter for shard {}", self.config.shard_id); + let mut close_result: Result<()> = Ok(()); match &self.mode { WriterMode::MemTable { @@ -2281,31 +3422,46 @@ impl ShardWriter { writer_state, .. } => { - // Send final WAL flush message and wait for completion + // Drain *both* tasks against the active memtable. The index apply + // and the WAL append are independent now, so closing has to + // settle both: the L0 flush below turns this memtable into a + // Lance generation, and a generation whose indexes never saw the + // tail is a generation with a hole in it. let st = state.read().await; let batch_store = st.memtable.batch_store(); let indexes = st.memtable.indexes_arc(); let batch_count = st.memtable.batch_count(); drop(st); + if batch_count > 0 + && let Some(indexes) = indexes + && indexes.indexed_count() < batch_count + { + let mut watcher = self.wal_flusher.track_batch( + Some(Arc::clone(&indexes)), + batch_count, + 0, // durability is settled by the WAL flush below + ); + writer_state.trigger_index_apply( + Arc::clone(&batch_store), + indexes, + batch_count, + )?; + watcher.wait().await?; + } + if batch_count > 0 { - let done = WatchableOnceCell::new(); - let reader = done.reader(); - if writer_state - .wal_flush_tx - .send(TriggerWalFlush { - source: WalFlushSource::BatchStore { - batch_store, - indexes, - }, - end_batch_position: batch_count, - done: Some(done), - }) - .is_ok() - { - let mut reader = reader; - let _ = reader.await_value().await; - } + // Append-only source: on this branch the index apply is a + // separate task (drained above), so the final WAL flush carries + // no indexes. #7769's failure propagation still applies. + let stage_result = Self::flush_final_wal( + &writer_state.wal_flush_tx, + WalFlushSource::BatchStore { batch_store }, + batch_count, + ) + .await; + close_result = + Self::merge_close_stage(close_result, "final WAL flush", stage_result); } // Freeze the active memtable (if any rows) so it joins the @@ -2321,22 +3477,34 @@ impl ShardWriter { // Propagate any freeze error: at close time the caller // has explicitly asked for full durability, so silently // dropping a freeze failure would lose data without any - // signal. If freeze fails, surface the error rather than - // continuing on to drain only the pre-existing frozen - // memtables (whose flushes can still be waited on, but - // the caller now knows the close was incomplete). + // signal. If freeze fails, its error is recorded as the + // first causal failure, but close still drains any + // pre-existing frozen MemTable watchers so a successor + // failure is logged without replacing the first error. let watchers: Vec<_> = { let mut st = state.write().await; if st.memtable.row_count() > 0 { - writer_state.freeze_memtable(&mut st)?; + let freeze_result = writer_state.freeze_memtable(&mut st).map(|_| ()); + close_result = Self::merge_close_stage( + close_result, + "active MemTable freeze", + freeze_result, + ); } - st.frozen_flush_watchers - .iter() - .map(|(_, w)| w.clone()) - .collect() + st.frozen_flush_watchers.iter().cloned().collect() }; - for mut w in watchers { - let _ = w.await_value().await; + for mut watcher in watchers { + let stage_result = match watcher.await_value().await { + Some(durability) => durability.into_result(), + None => Err(Error::io( + "MemTable flush handler exited before reporting completion during close", + )), + }; + close_result = Self::merge_close_stage( + close_result, + "frozen MemTable flush watcher", + stage_result, + ); } } WriterMode::WalOnly { @@ -2349,47 +3517,62 @@ impl ShardWriter { let pending = state.batch_count(); let end_position = state.next_batch_position(); if pending > 0 { - let done = WatchableOnceCell::new(); - let reader = done.reader(); - if wal_flush_tx - .send(TriggerWalFlush { - source: WalFlushSource::WalOnly { - state: state.clone(), - }, - end_batch_position: end_position, - done: Some(done), - }) - .is_ok() - { - let mut reader = reader; - let _ = reader.await_value().await; - } + let stage_result = Self::flush_final_wal( + wal_flush_tx, + WalFlushSource::WalOnly { + state: state.clone(), + }, + end_position, + ) + .await; + close_result = + Self::merge_close_stage(close_result, "final WAL flush", stage_result); } } } // Shutdown background tasks - self.task_executor.shutdown_all().await?; - - info!("ShardWriter closed for shard {}", self.config.shard_id); - Ok(()) + let shutdown_result = self.task_executor.shutdown_all().await; + let close_result = Self::merge_close_stage(close_result, "task shutdown", shutdown_result); + + match &close_result { + Ok(()) => info!("ShardWriter closed for shard {}", self.config.shard_id), + Err(error) => warn!( + "ShardWriter close for shard {} failed: {error}", + self.config.shard_id + ), + } + close_result } } -/// MemTable statistics. +/// MemTable statistics: rows, generation, and what the memtable still owes the +/// WAL. +/// +/// Deliberately carries **no byte totals**. [`ShardWriter::memory`] is the one +/// way to ask what a shard is holding — it answers without the writer lock, and +/// a second set of byte fields here would be a second implementation of the +/// same filter and sum, free to disagree with the gate. #[derive(Debug, Clone)] pub struct MemTableStats { pub row_count: usize, pub batch_count: usize, - pub estimated_size: usize, pub generation: u64, pub max_buffered_batch_position: Option, - pub max_flushed_batch_position: Option, + /// Writer-global count of WAL-durable batches. Exclusive: 0 means none. + /// Compare against `global_offset + batch_count` to see what this memtable + /// still owes the WAL. + pub durable_batch_count: usize, + /// Writer-global coordinate of this memtable's batch 0. + pub global_offset: usize, pub pending_wal_start_batch_position: Option, pub pending_wal_end_batch_position: Option, pub pending_wal_batch_count: usize, pub pending_wal_row_count: usize, pub pending_wal_estimated_bytes: usize, + /// Frozen memtables in the read view: sealed-awaiting-flush, plus flushed + /// ones still inside `frozen_memtable_grace`. + pub frozen_count: usize, } /// WAL statistics. @@ -2399,35 +3582,179 @@ pub struct WalStats { pub next_wal_entry_position: u64, } -/// Background handler for WAL flush operations. +/// The L0 flushes outstanding when [`ShardWriter::force_seal_active`] +/// returned — the exact predicate for "everything written before that call +/// is in L0". +/// +/// The set is fixed at seal time, so [`Self::wait`] is bounded no matter how +/// many memtables freeze while it waits, and each entry reports the outcome of +/// its own flush. That is why the fence is a captured watcher set and not a +/// generation number compared against `ShardManifest::current_generation`: the +/// manifest advances to `generation + 1` on every committed flush without +/// checking for a gap, so a later generation's success moves it past a +/// generation that failed and never reached L0 — the watermark would report +/// durability that does not exist. +#[derive(Debug)] +pub struct SealFence { + sealed_generation: Option, + watchers: Vec, +} + +impl SealFence { + /// The generation the seal froze, or `None` when the active memtable + /// was empty (a no-op seal). Independent of what the fence covers: an + /// empty active memtable says nothing about generations frozen earlier + /// and still awaiting flush, which the fence still waits on. + pub fn sealed_generation(&self) -> Option { + self.sealed_generation + } + + /// Block until every flush this fence covers has landed in L0. + /// + /// Errors if any of them reports `DurabilityResult::Failed`, or if the + /// flush handler exited without reporting. + pub async fn wait(self) -> Result<()> { + for mut watcher in self.watchers { + match watcher.await_value().await { + Some(durability) => durability.into_result()?, + None => { + return Err(Error::io( + "MemTable flush handler exited before reporting completion", + )); + } + } + } + Ok(()) + } +} + +/// The oldest store that still owes the WAL an append, or `None` when everything +/// is durable. +/// +/// Ordering is the whole point. WAL entry positions are assigned in append-call +/// order; replay walks them ascending; row positions follow; and primary-key +/// recency is "newest visible row position wins". So appending a newer memtable +/// ahead of an older one's tail silently inverts dedup after a crash. +/// +/// Taking "the active memtable" would do exactly that, because a timer tick +/// enqueued before a freeze is handled after it and would resolve to the incoming +/// memtable. Selecting by cursor instead makes the target a function of what is +/// actually durable, not of when the timer happened to fire. +fn next_pending_store( + frozen: impl Iterator>, + active: Arc, + durable: usize, +) -> Option> { + frozen + .chain(std::iter::once(active)) + .find(|store| store.global_end() > durable) +} + +/// The index-apply task: one sequential consumer of the index-apply channel. /// -/// This handler does parallel WAL I/O + index updates during flush. -/// Indexes are passed through the TriggerWalFlush message. +/// Sequential consumption is the safety property. `HnswGraph::insert_batch` hard- +/// rejects any range whose start is not its `indexed_len`, so the apply must see +/// contiguous, in-order ranges — and a single consumer guarantees that +/// regardless of how many putters race behind it. Ordering comes from the task, +/// not from a flush interval, which is why triggering per-put is exactly as safe +/// as triggering on a timer, and lets a put become visible in milliseconds +/// instead of waiting on an S3 round-trip. +struct IndexApplyHandler { + cursors: Arc, + wal_flusher: Arc, + stats: SharedWriteStats, +} + +#[async_trait] +impl MessageHandler for IndexApplyHandler { + async fn handle(&mut self, message: TriggerIndexApply) -> Result<()> { + match apply_index_range(&self.cursors, message).await { + Ok(applied) => { + // A coalesced no-op indexes nothing (`rows_indexed == 0`); + // recording it would inflate the count and skew avg latency. + if applied.rows_indexed > 0 { + self.stats + .record_index_update(applied.duration, applied.rows_indexed); + } + Ok(()) + } + // An index apply cannot be partially rolled back, so a failure is + // terminal: poison, and let reopen rebuild the indexes from the WAL. + // See the note in `WalFlusher::flush_from_batch_store`. + Err(e) => { + self.wal_flusher.poison(&e); + Err(e) + } + } + } +} + struct WalFlushHandler { wal_flusher: Arc, /// MemTable-mode writer state, used to detect "frozen vs active" flushes /// via Arc::ptr_eq on the active batch_store. `None` when running in /// WAL-only mode (no MemTable, no frozen-vs-active distinction). memtable_state: Option>>, + /// WAL-only-mode pending queue, so a background tick can resolve to the + /// batches still owed an append. `None` in MemTable mode. Exactly one of + /// `memtable_state` / `wal_only_state` is `Some`. + wal_only_state: Option>, + /// How often to append in the background. `None` disables the ticker, leaving + /// the append size-triggered (and freeze/close-triggered) only. + flush_interval: Option, stats: SharedWriteStats, + observer: Option>, } impl WalFlushHandler { fn new( wal_flusher: Arc, memtable_state: Option>>, + wal_only_state: Option>, + flush_interval: Option, stats: SharedWriteStats, + observer: Option>, ) -> Self { Self { wal_flusher, memtable_state, + wal_only_state, + flush_interval, stats, + observer, } } } #[async_trait] impl MessageHandler for WalFlushHandler { + /// Append periodically in the background. + /// + /// This is what the flush interval was always supposed to mean. It routed to + /// a timer that was only ever *evaluated on the write path*, so it could add + /// a redundant trigger but never delay or batch one — with every durable put + /// triggering its own append, tuning the knob did nothing at all. + /// + /// The ticker exists to bound S3 API cost: an append is a PUT, billed per + /// call, and it is the only thing on this schedule. The index apply is not — + /// it is in-memory and free to batch, so it runs per-put on its own task. + fn tickers(&mut self) -> Vec<(Duration, MessageFactory)> { + // No interval => no ticker. A zero interval would panic in tokio. + let Some(interval) = self.flush_interval.filter(|d| !d.is_zero()) else { + return vec![]; + }; + // The tick names no store: `MessageFactory` is synchronous and cannot take + // the async state lock. `handle()` resolves it against the cursor. + vec![( + interval, + Box::new(|| TriggerWalFlush { + source: WalFlushSource::NextPending, + end_batch_position: 0, + done: None, + }), + )] + } + async fn handle(&mut self, message: TriggerWalFlush) -> Result<()> { let TriggerWalFlush { source, @@ -2435,6 +3762,16 @@ impl MessageHandler for WalFlushHandler { done, } = message; + // A timer tick names no store — resolve it now, at handle time. + let (source, end_batch_position) = match source { + WalFlushSource::NextPending => match self.resolve_next_pending().await { + Some(resolved) => resolved, + // Everything is already durable; the tick has nothing to do. + None => return Ok(()), + }, + other => (other, end_batch_position), + }; + let result = self.do_flush(source, end_batch_position).await; // Propagate the just-appended WAL entry position back into the @@ -2465,6 +3802,64 @@ impl MessageHandler for WalFlushHandler { } impl WalFlushHandler { + /// Pick the store the WAL still owes an append, oldest first. + /// + /// **WAL entries must be appended in global batch-position order for the + /// writer's lifetime.** `WalAppender::append` assigns each entry's position + /// from its own counter, in call order; replay walks those positions + /// ascending and assigns row positions in that order; and primary-key recency + /// is "newest visible row position wins". So append order fixes dedup order. + /// Append two memtables out of order and replay silently hands the dedup to + /// the *stale* row — corruption that survives the crash that caused it, and + /// that a full scan cannot see. + /// + /// Resolving to "the active memtable" would break exactly that: a tick + /// enqueued before a freeze is handled after it, resolves to the incoming + /// memtable, and appends its batches ahead of the outgoing memtable's tail. + /// So the target is a function of `durable`, not of when the timer fired. + /// + /// Safe to walk the frozen list because a store that still owes an append + /// cannot be swept: its L0 flush is blocked on the completion cell that only + /// that append fires. + /// + /// In WAL-only mode there is a single FIFO pending queue and no memtable + /// rotation, so the ordering hazard above cannot arise: the tick resolves to + /// the queue whenever it holds un-appended batches. + async fn resolve_next_pending(&self) -> Option<(WalFlushSource, usize)> { + if let Some(state_lock) = self.memtable_state.as_ref() { + let state = state_lock.read().await; + let durable = self.wal_flusher.durable(); + + return next_pending_store( + state + .frozen_memtables + .iter() + .map(|frozen| frozen.memtable.batch_store()), + state.memtable.batch_store(), + durable, + ) + .map(|store| { + let end = store.len(); + (WalFlushSource::BatchStore { batch_store: store }, end) + }); + } + + let state = self.wal_only_state.as_ref()?; + if state.batch_count() == 0 { + // Everything already appended; the tick has nothing to do. + return None; + } + // `flush_from_wal_only` snapshots the whole queue, so the end position + // is informational here; carry the next position for symmetry. + let end = state.next_batch_position(); + Some(( + WalFlushSource::WalOnly { + state: Arc::clone(state), + }, + end, + )) + } + /// Unified flush method for both active and frozen memtables and for /// WAL-only mode. /// @@ -2484,18 +3879,6 @@ impl WalFlushHandler { ) -> Result { let start = Instant::now(); - // Whether this flush actually updates any in-memory indexes — only - // a BatchStore source carrying a non-empty `IndexStore` does. Used - // to gate the `record_index_update` stat so WAL-only flushes don't - // pollute the index-update counters. - let has_indexes = matches!( - &source, - WalFlushSource::BatchStore { - indexes: Some(_), - .. - } - ); - // Early-out for BatchStore sources where the watermark already // covers the requested end position. Detection of "frozen flush" // requires the active memtable's batch_store; WAL-only handlers @@ -2503,8 +3886,7 @@ impl WalFlushHandler { // BatchStore source, so the early-out simplifies to the watermark // comparison. if let WalFlushSource::BatchStore { batch_store, .. } = &source { - let max_flushed = batch_store.max_flushed_batch_position(); - let flushed_up_to = max_flushed.map(|p| p + 1).unwrap_or(0); + let flushed_up_to = batch_store.local_end(self.wal_flusher.durable()); let is_frozen_flush = if let Some(state_lock) = &self.memtable_state { let state = state_lock.read().await; !Arc::ptr_eq(batch_store, &state.memtable.batch_store()) @@ -2526,14 +3908,13 @@ impl WalFlushHandler { .unwrap_or(0); if batches_flushed > 0 { - self.stats - .record_wal_flush(start.elapsed(), flush_result.wal_bytes); + // One reading for both sinks, so the cumulative total and the + // per-flush observation cannot disagree. + let elapsed = start.elapsed(); + self.stats.record_wal_flush(elapsed, flush_result.wal_bytes); self.stats.record_wal_io(flush_result.wal_io_duration); - if has_indexes { - self.stats.record_index_update( - flush_result.index_update_duration, - flush_result.rows_indexed, - ); + if let Some(observer) = &self.observer { + observer.on_wal_flush(elapsed, flush_result.wal_bytes); } } @@ -2549,35 +3930,49 @@ impl WalFlushHandler { /// handler flushes in the background. struct MemTableFlushHandler { state: Arc>, + /// Shared with `SharedWriterState`; this handler re-derives it once a + /// flush commits and the memtable set changes. + memory: Arc>, flusher: Arc, + /// Source of the writer-global durability cursor, which the L0 flush asserts + /// covers the whole frozen memtable before it writes a generation. + wal_flusher: Arc, epoch: u64, - /// Secondary index configs to rebuild on each flushed generation. When + /// Secondary index configs to rebuild on each SSTable. When /// non-empty the handler flushes via [`MemTableFlusher::flush_with_indexes`] - /// so queries over flushed generations use index lookups instead of full + /// so queries over SSTables use index lookups instead of full /// scans — and so vector search's index-only `fast_search` can see the data /// at all. index_configs: Vec, stats: SharedWriteStats, + observer: Option>, /// How long a frozen memtable lingers in memory after its flush commits /// before `SweepExpired` evicts it. See `ShardWriterConfig::frozen_memtable_grace`. grace: Duration, } impl MemTableFlushHandler { + #[allow(clippy::too_many_arguments)] fn new( state: Arc>, + memory: Arc>, flusher: Arc, + wal_flusher: Arc, epoch: u64, index_configs: Vec, stats: SharedWriteStats, + observer: Option>, grace: Duration, ) -> Self { Self { state, + memory, flusher, + wal_flusher, epoch, index_configs, stats, + observer, grace, } } @@ -2588,12 +3983,18 @@ impl MemTableFlushHandler { let now = now_millis(); let grace_ms = self.grace.as_millis() as u64; let mut state = self.state.write().await; + let before = state.frozen_memtables.len(); state .frozen_memtables .retain(|frozen| match frozen.flushed_at_ms { Some(flushed_at) => now.saturating_sub(flushed_at) < grace_ms, None => true, }); + // Eviction is the only thing that reclaims a grace-retained generation, + // so this is where its bytes leave the memory view. + if state.frozen_memtables.len() != before { + publish_memory(&self.memory, &state); + } } } @@ -2645,7 +4046,6 @@ impl MemTableFlushHandler { memtable: Arc, ) -> Result { let start = Instant::now(); - let memtable_size = memtable.estimated_size(); let flush_result = async { // Step 1: Wait for WAL flush completion (already queued at freeze time). @@ -2670,6 +4070,28 @@ impl MemTableFlushHandler { None }; + // Step 1b: Wait until index application covers this whole memtable. + // + // Freeze queues the apply and this flush on separate channels, so + // without waiting the export can run while the indexes are still + // behind the batch store. The generation's vector index would then + // be short of rows its own SSTable holds, and SSTable vector search + // is index-only -- `fast_search`, no brute-force scan -- so those + // rows stop answering once the frozen memtable retires. + // + // `batch_count` is fixed at freeze, so this waits for a target that + // cannot move, and the watcher surfaces a poisoned writer rather + // than blocking on a cursor that will never arrive. + if !self.index_configs.is_empty() + && let Some(indexes) = memtable.indexes_arc() + { + let target_indexed = memtable.batch_count(); + self.wal_flusher + .track_batch(Some(indexes), target_indexed, 0) + .wait() + .await?; + } + // Step 2: Flush the memtable to Lance storage. The covered WAL // entry position is either the one we just appended (per-memtable, // from the completion cell — authoritative even when concurrent @@ -2681,14 +4103,20 @@ impl MemTableFlushHandler { let covered_wal_entry_position = wal_flushed_position .or_else(|| memtable.frozen_at_wal_entry_position()) .unwrap_or(0); - // Rebuild secondary indexes on the flushed generation so later + // Rebuild secondary indexes on the SSTable so later // queries hit an index instead of scanning. Skip the extra // dataset open when there are no indexes to build. The indexed // path's future is boxed to keep this async block's nesting // under the type-layout recursion limit. + // Read the durability cursor *after* the WAL-append completion above, + // not before: the append that makes this memtable durable is the very + // thing we just waited on, so a cursor sampled earlier would still be + // short of it and trip the flush precondition. + let durable = self.wal_flusher.durable(); + if self.index_configs.is_empty() { self.flusher - .flush(&memtable, self.epoch, covered_wal_entry_position) + .flush(&memtable, self.epoch, covered_wal_entry_position, durable) .await } else { Box::pin(self.flusher.flush_with_indexes( @@ -2696,6 +4124,7 @@ impl MemTableFlushHandler { self.epoch, &self.index_configs, covered_wal_entry_position, + durable, )) .await } @@ -2713,11 +4142,11 @@ impl MemTableFlushHandler { { let mut state = self.state.write().await; // Backpressure drain: unconditional so `wait_for_flush_drain` - // sees the watcher's error signal, not a dropped channel. - if let Some((_size, _watcher)) = state.frozen_flush_watchers.pop_front() { - state.frozen_memtable_bytes = - state.frozen_memtable_bytes.saturating_sub(memtable_size); - } + // sees the watcher's error signal, not a dropped channel. Which + // entry comes off the front does not matter — flushes complete out + // of order, and the snapshot re-derived at the end of this block + // reads the frozen queue itself rather than tracking a charge. + state.frozen_flush_watchers.pop_front(); // Retire the frozen handle on commit success, keyed by generation // (non-FIFO completion is fine). Zero grace evicts here; otherwise // stamp the grace clock so it lingers for multi-part as-of reads @@ -2725,30 +4154,38 @@ impl MemTableFlushHandler { // the read union until a later flush or WAL replay, else a transient // error reopens the hole. if flush_result.is_ok() { - let flushed_generation = memtable.generation(); + let sstable = memtable.generation(); if self.grace.is_zero() { state .frozen_memtables - .retain(|frozen| frozen.memtable.generation() != flushed_generation); + .retain(|frozen| frozen.memtable.generation() != sstable); } else { let now = now_millis(); for frozen in state.frozen_memtables.iter_mut() { - if frozen.memtable.generation() == flushed_generation { + if frozen.memtable.generation() == sstable { frozen.flushed_at_ms = Some(now); } } } } + // Re-derive after both branches above. A *failed* flush leaves its + // memtable un-stamped and so still counted, which is the point: its + // bytes are resident and only another flush can reclaim them. + publish_memory(&self.memory, &state); } let result = flush_result?; + let elapsed = start.elapsed(); self.stats - .record_memtable_flush(start.elapsed(), result.rows_flushed); + .record_memtable_flush(elapsed, result.rows_flushed); + if let Some(observer) = &self.observer { + observer.on_memtable_flush(elapsed, result.rows_flushed); + } info!( "Flushed frozen memtable generation {} ({} rows in {:?})", - result.generation.generation, + result.sstable.generation, result.rows_flushed, start.elapsed() ); @@ -2935,11 +4372,7 @@ impl WriteStatsSnapshot { /// Get average WAL flush size in bytes. pub fn avg_wal_flush_bytes(&self) -> Option { - if self.wal_flush_count > 0 { - Some(self.wal_flush_bytes / self.wal_flush_count) - } else { - None - } + self.wal_flush_bytes.checked_div(self.wal_flush_count) } /// Get WAL write throughput (bytes per second based on WAL flush time). @@ -2971,11 +4404,7 @@ impl WriteStatsSnapshot { /// Get average rows per index update. pub fn avg_index_update_rows(&self) -> Option { - if self.index_update_count > 0 { - Some(self.index_update_rows / self.index_update_count) - } else { - None - } + self.index_update_rows.checked_div(self.index_update_count) } /// Get average MemTable flush latency. @@ -2989,11 +4418,8 @@ impl WriteStatsSnapshot { /// Get average MemTable flush size in rows. pub fn avg_memtable_flush_rows(&self) -> Option { - if self.memtable_flush_count > 0 { - Some(self.memtable_flush_rows / self.memtable_flush_count) - } else { - None - } + self.memtable_flush_rows + .checked_div(self.memtable_flush_count) } /// Log stats summary using tracing (for structured telemetry). @@ -3049,9 +4475,11 @@ pub fn new_shared_stats() -> SharedWriteStats { mod tests { use super::*; use crate::dataset::mem_wal::test_util::failing_memory_store; - use arrow_array::{Int32Array, StringArray}; + use arrow_array::{FixedSizeListArray, Float32Array, Int32Array, StringArray}; use arrow_schema::{DataType, Field}; use lance_core::FenceReason; + use rstest::rstest; + use std::sync::atomic::AtomicUsize; use tempfile::TempDir; async fn create_local_store() -> (Arc, Path, String, TempDir) { @@ -3061,6 +4489,26 @@ mod tests { (store, path, uri, temp_dir) } + #[test] + fn test_merge_close_stage_preserves_first_error() { + let result = ShardWriter::merge_close_stage( + Err(Error::io("primary close error")), + "secondary close stage", + Err(Error::io("secondary close error")), + ); + + let error = result.expect_err("close must preserve the first error"); + assert!(matches!(&error, Error::IO { .. })); + assert!( + error.to_string().contains("primary close error"), + "unexpected error: {error}" + ); + assert!( + !error.to_string().contains("secondary close error"), + "secondary error replaced the primary error: {error}" + ); + } + /// Base schema with `id` marked as the unenforced primary key (delete needs /// a PK). `name` is nullable so a tombstone can null it. fn create_pk_test_schema() -> Arc { @@ -3076,6 +4524,17 @@ mod tests { ])) } + /// [`create_pk_test_schema`] with a non-nullable `name` — the shape that + /// used to make `delete` fail. + fn create_strict_pk_test_schema() -> Arc { + let fields: Vec = create_pk_test_schema() + .fields() + .iter() + .map(|f| f.as_ref().clone().with_nullable(false)) + .collect(); + Arc::new(ArrowSchema::new(fields)) + } + fn id_only_keys(ids: &[i32]) -> RecordBatch { RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![Field::new( @@ -3088,12 +4547,61 @@ mod tests { .unwrap() } + /// Two same-typed columns handed over swapped pass every positional check + /// downstream, so the logical-schema gate has to catch them by name. + #[tokio::test] + async fn test_put_rejects_swapped_same_typed_columns() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("name", DataType::Utf8, true), + Field::new("email", DataType::Utf8, true), + ])); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + ShardWriterConfig { + shard_id: Uuid::new_v4(), + ..Default::default() + }, + schema, + vec![], + ) + .await + .unwrap(); + + let swapped = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("email", DataType::Utf8, true), + Field::new("name", DataType::Utf8, true), + ])), + vec![ + Arc::new(StringArray::from(vec!["a@example.com"])), + Arc::new(StringArray::from(vec!["a"])), + ], + ) + .unwrap(); + + let error = writer.put(vec![swapped]).await.unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + let message = error.to_string(); + assert!( + message.contains("column 0") && message.contains("email") && message.contains("name"), + "error should name the position and both columns: {message}" + ); + + writer.close().await.unwrap(); + } + #[test] fn test_ensure_tombstone_column_injects_false() { let base = create_test_schema(); - let target = schema_with_tombstone(&base); - let out = ensure_tombstone_column(create_test_batch(&base, 0, 3), &target).unwrap(); - assert_eq!(out.schema(), target); + let storage = schema_with_tombstone(&base); + let out = ensure_tombstone_column(create_test_batch(&base, 0, 3), &storage).unwrap(); + assert_eq!(out.schema(), storage); let ts = out .column_by_name(TOMBSTONE) .unwrap() @@ -3105,16 +4613,16 @@ mod tests { "put injects _tombstone = false" ); // Idempotent: a batch already carrying the column passes through. - let again = ensure_tombstone_column(out.clone(), &target).unwrap(); + let again = ensure_tombstone_column(out.clone(), &storage).unwrap(); assert_eq!(again.schema(), out.schema()); } #[test] fn test_build_tombstone_batch_shape() { - let target = schema_with_tombstone(&create_test_schema()); + let storage = schema_with_tombstone(&create_test_schema()); let tomb = - build_tombstone_batch(&id_only_keys(&[5, 7]), &target, &["id".to_string()]).unwrap(); - assert_eq!(tomb.schema(), target); + build_tombstone_batch(&id_only_keys(&[5, 7]), &storage, &["id".to_string()]).unwrap(); + assert_eq!(tomb.schema(), storage); assert_eq!(tomb.num_rows(), 2); let ids = tomb .column_by_name("id") @@ -3139,7 +4647,7 @@ mod tests { #[test] fn test_build_tombstone_batch_missing_pk_errors() { - let target = schema_with_tombstone(&create_test_schema()); + let storage = schema_with_tombstone(&create_test_schema()); let keys = RecordBatch::try_new( Arc::new(ArrowSchema::new(vec![Field::new( "other", @@ -3149,18 +4657,62 @@ mod tests { vec![Arc::new(Int32Array::from(vec![1]))], ) .unwrap(); - assert!(build_tombstone_batch(&keys, &target, &["id".to_string()]).is_err()); + assert!(build_tombstone_batch(&keys, &storage, &["id".to_string()]).is_err()); + } + + #[test] + fn test_build_tombstone_batch_nulls_non_nullable_base_column() { + // The point of the storage schema: a tombstone nulls `v` even though the + // base table declares it non-nullable. + let pk = ["id".to_string()]; + let base = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let storage = relax_non_pk_nullability(&schema_with_tombstone(&base), &pk); + + let batch = build_tombstone_batch(&id_only_keys(&[1]), &storage, &pk).unwrap(); + + assert!(batch["v"].is_null(0), "the tombstone must null `v`"); + assert!(!batch["id"].is_null(0), "the primary key survives"); + assert!( + batch[TOMBSTONE] + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + ); } #[test] - fn test_build_tombstone_batch_non_nullable_nonpk_errors() { - // A tombstone must null every non-PK column; a non-nullable one fails. + fn test_build_tombstone_batch_rejects_null_primary_key() { + // Primary keys are never relaxed, so the storage schema still rejects a + // null key — the delete path needs no separate check for it. + let pk = ["id".to_string()]; let base = Arc::new(ArrowSchema::new(vec![ Field::new("id", DataType::Int32, false), Field::new("v", DataType::Int32, false), ])); - let target = schema_with_tombstone(&base); - assert!(build_tombstone_batch(&id_only_keys(&[1]), &target, &["id".to_string()]).is_err()); + let storage = relax_non_pk_nullability(&schema_with_tombstone(&base), &pk); + let keys = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int32, + true, + )])), + vec![Arc::new(Int32Array::from(vec![None::]))], + ) + .unwrap(); + + let error = build_tombstone_batch(&keys, &storage, &pk).unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + assert!( + error.to_string().contains("non-nullable"), + "error should name the nullability violation: {error}" + ); } #[tokio::test] @@ -3229,6 +4781,159 @@ mod tests { writer.close().await.unwrap(); } + /// Delete works against a base table with non-nullable non-PK columns, and + /// survivors come back through the narrowing egress relabel intact. + #[tokio::test] + async fn test_delete_against_non_nullable_base_column_round_trip() { + use crate::dataset::mem_wal::scanner::LsmScanner; + use futures::TryStreamExt; + + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = create_strict_pk_test_schema(); + assert!( + !schema.field_with_name("name").unwrap().is_nullable(), + "the point of this test is a non-nullable non-PK column" + ); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + durable_write: true, + ..Default::default() + }; + let shard_id = config.shard_id; + let writer = ShardWriter::open( + store, + base_path, + base_uri.clone(), + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + writer.delete(vec![id_only_keys(&[2])]).await.unwrap(); + + let refs = writer.in_memory_memtable_refs().await.unwrap(); + let scanner = LsmScanner::without_base_table( + schema.clone(), + base_uri, + vec![], + vec!["id".to_string()], + ) + .with_in_memory_memtables(shard_id, refs); + let batches: Vec = scanner + .try_into_stream() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + + let mut rows: Vec<(i32, String)> = Vec::new(); + for b in &batches { + assert!( + !b.schema().field_with_name("name").unwrap().is_nullable(), + "egress must narrow back to the logical schema" + ); + let ids = b["id"].as_any().downcast_ref::().unwrap(); + let names = b["name"].as_any().downcast_ref::().unwrap(); + rows.extend((0..ids.len()).map(|i| (ids.value(i), names.value(i).to_string()))); + } + rows.sort_unstable(); + + assert_eq!( + rows, + vec![ + (0, "name_0".to_string()), + (1, "name_1".to_string()), + (3, "name_3".to_string()), + (4, "name_4".to_string()), + ], + "id=2 deleted; every survivor keeps its non-nullable value" + ); + + writer.close().await.unwrap(); + } + + /// The storage schema no longer rejects a caller's null, so `put` is the + /// only thing standing between a null and a non-nullable base column. + #[tokio::test] + async fn test_put_rejects_null_in_non_nullable_base_column() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = create_strict_pk_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + ShardWriterConfig { + shard_id: Uuid::new_v4(), + ..Default::default() + }, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + let error = writer.put(vec![null_name_batch()]).await.unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + assert!( + error.to_string().contains("base table schema"), + "error should point at the schema contract: {error}" + ); + + writer.close().await.unwrap(); + } + + /// WAL-only mode validates too — it has no memtable, so before this gate + /// nothing checked its input at all. + #[tokio::test] + async fn test_wal_only_put_rejects_null_in_non_nullable_base_column() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = create_strict_pk_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + wal_only_config(Uuid::new_v4()), + schema.clone(), + vec![], + ) + .await + .unwrap(); + + let error = writer.put(vec![null_name_batch()]).await.unwrap_err(); + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected InvalidInput, got {error:?}" + ); + + writer.close().await.unwrap(); + } + + /// A caller-shaped batch that declares `name` nullable and carries a null — + /// legal Arrow, illegal against a base table that declares it non-nullable. + fn null_name_batch() -> RecordBatch { + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), + ])), + vec![ + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(StringArray::from(vec![Some("a"), None])), + ], + ) + .unwrap() + } + /// `delete_no_wait` lands the tombstone in the in-memory tier (visible at /// the batch-store level the instant it returns) and hands back the /// durability watcher *without* awaiting it. Index-driven LSM read @@ -3322,7 +5027,7 @@ mod tests { /// await), but the tombstone still lands in the in-memory tier. The delete /// analog of `test_put_no_wait_non_durable_returns_no_watcher`. #[tokio::test] - async fn test_shard_writer_delete_no_wait_non_durable_returns_no_watcher() { + async fn test_non_durable_delete_is_read_your_writes() { let (store, base_path, base_uri, _temp) = create_local_store().await; let schema = create_pk_test_schema(); let config = ShardWriterConfig { @@ -3343,7 +5048,10 @@ mod tests { .delete_no_wait(vec![id_only_keys(&[2])]) .await .unwrap(); - assert!(watcher.is_none(), "non-durable delete has nothing to await"); + // As with a put: a non-durable delete awaits its index apply, not an S3 + // round-trip. It is read-your-writes, just not durable. + let mut watcher = watcher.expect("a non-durable delete awaits its index apply"); + watcher.wait().await.unwrap(); // Tombstone landed in the in-memory tier (5 rows + 1 tombstone). assert_eq!(writer.memtable_stats().await.unwrap().row_count, 6); @@ -3356,7 +5064,7 @@ mod tests { /// with an optional filter. Mirrors how a query reads a WAL table after a /// flush — the path the wallop fuzz exercised when it caught a deleted row /// resurfacing. - async fn read_flushed_ids_via_lsm( + async fn read_sstable_ids_via_lsm( writer: &ShardWriter, schema: Arc, base_uri: &str, @@ -3369,8 +5077,8 @@ mod tests { let manifest = writer.manifest().await.unwrap().unwrap(); let mut snapshot = ShardSnapshot::new(shard_id).with_current_generation(manifest.current_generation); - for fg in &manifest.flushed_generations { - snapshot = snapshot.with_flushed_generation(fg.generation, fg.path.clone()); + for sstable in &manifest.sstables { + snapshot = snapshot.with_sstable(sstable.generation, sstable.path.clone()); } let mut scanner = LsmScanner::without_base_table( schema, @@ -3406,14 +5114,13 @@ mod tests { ShardWriterConfig { shard_id, durable_write: false, - sync_indexed_write: true, manifest_scan_batch_size: 2, ..Default::default() } } /// Delete a key, then flush: the tombstone and the live row land in the - /// *same* flushed generation, so flush-time dedup must keep the tombstone + /// *same* SSTable, so flush-time dedup must keep the tombstone /// (newest) and the read must fold it away. Regression for the wallop /// phantom (deleted row resurfacing in a filtered read after flush). #[tokio::test] @@ -3442,14 +5149,14 @@ mod tests { writer.wait_for_flush_drain().await.unwrap(); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, vec![0, 1, 3, 4], - "id=2 deleted before flush; tombstone must not surface in a flushed-gen scan" + "id=2 deleted before flush; tombstone must not surface in an SSTable scan" ); // The filtered read path (folds NOT _tombstone into the predicate) must // also drop it — this is the exact wallop failure shape (`id < 3`). assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3")) + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 3")) .await, vec![0, 1], "filtered read after flush must not resurface deleted id=2" @@ -3463,7 +5170,7 @@ mod tests { /// mask the older row by PK. This is the wallop scenario (seed flushed, /// then delete, then flush). #[tokio::test] - async fn test_shard_writer_delete_across_flushed_generations() { + async fn test_shard_writer_delete_across_sstables() { let (store, base_path, base_uri, _temp) = create_local_store().await; let schema = create_pk_test_schema(); let shard_id = Uuid::new_v4(); @@ -3492,12 +5199,12 @@ mod tests { writer.wait_for_flush_drain().await.unwrap(); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, vec![1, 2, 3, 4], "id=0 tombstoned in a newer gen must mask the older gen's live row" ); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) .await, Vec::::new(), "filtered read 'id < 1' must not resurface cross-gen deleted id=0" @@ -3506,13 +5213,13 @@ mod tests { writer.close().await.unwrap(); } - /// Same as the cross-generation case, but the flushed generations carry a + /// Same as the cross-generation case, but the SSTables carry a /// BTree index on `id` (as every wallop table does). A filtered read /// `id < 1` resolves through the scalar index; the `NOT _tombstone` residual /// must still be applied or the deleted row leaks. This is the exact wallop /// failure (BTree id + `FilteredRead 'id < 1'` resurfacing deleted id=0). #[tokio::test] - async fn test_shard_writer_delete_across_flushed_generations_indexed() { + async fn test_shard_writer_delete_across_sstables_indexed() { let (store, base_path, base_uri, _temp) = create_local_store().await; let schema = create_pk_test_schema(); let shard_id = Uuid::new_v4(); @@ -3544,12 +5251,12 @@ mod tests { writer.wait_for_flush_drain().await.unwrap(); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, None).await, vec![1, 2, 3, 4], "indexed cross-gen: full scan must mask deleted id=0" ); assert_eq!( - read_flushed_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) + read_sstable_ids_via_lsm(&writer, schema.clone(), &base_uri, shard_id, Some("id < 1")) .await, Vec::::new(), "indexed filtered read 'id < 1' must not resurface deleted id=0 (wallop repro)" @@ -3731,9 +5438,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3774,9 +5480,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3803,7 +5508,7 @@ mod tests { } #[tokio::test] - async fn test_put_no_wait_non_durable_returns_no_watcher() { + async fn test_non_durable_put_is_read_your_writes() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); @@ -3811,9 +5516,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3826,7 +5530,21 @@ mod tests { let batch = create_test_batch(&schema, 0, 10); let (result, watcher) = writer.put_no_wait(vec![batch]).await.unwrap(); assert_eq!(result.batch_positions, 0..1); - assert!(watcher.is_none(), "non-durable put has nothing to await"); + + // A non-durable put still has something to await: its *index apply*. + // Skipping durability now costs the caller durability only — not + // visibility. Before the index apply was split off the WAL flush, a + // non-durable write was not read-your-writes at all: the row stayed + // invisible until some later flush happened to index it. + let mut watcher = watcher.expect("a non-durable put awaits its index apply"); + watcher.wait().await.unwrap(); + + let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap(); + assert_eq!( + scanned.num_rows(), + 10, + "a non-durable put must be readable as soon as it returns" + ); let stats = writer.memtable_stats().await.unwrap(); assert_eq!(stats.row_count, 10); @@ -3843,9 +5561,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3878,9 +5595,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: true, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3914,12 +5630,12 @@ mod tests { } /// End-to-end check that the background flush handler rebuilds secondary - /// indexes on every flushed generation. Before this, the handler flushed - /// via plain `flush`, leaving flushed generations unindexed — point + /// indexes on every SSTable. Before this, the handler flushed + /// via plain `flush`, leaving SSTables unindexed — point /// lookups had to full-scan and vector search's index-only `fast_search` /// couldn't see the data at all. #[tokio::test] - async fn test_flushed_generation_is_indexed() { + async fn test_sstable_is_indexed() { use crate::index::DatasetIndexExt; let (store, base_path, base_uri, _temp_dir) = create_local_store().await; @@ -3930,9 +5646,8 @@ mod tests { shard_id, shard_spec_id: 0, durable_write: false, - sync_indexed_write: true, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -3964,22 +5679,18 @@ mod tests { writer.force_seal_active().await.unwrap(); writer.wait_for_flush_drain().await.unwrap(); - // Resolve the flushed generation recorded in the manifest. + // Resolve the SSTable recorded in the manifest. let manifest = writer.manifest().await.unwrap().unwrap(); - assert_eq!( - manifest.flushed_generations.len(), - 1, - "expected exactly one flushed generation" - ); + assert_eq!(manifest.sstables.len(), 1, "expected exactly one SSTable"); let gen_uri = format!( "{}/_mem_wal/{}/{}", - base_uri, shard_id, manifest.flushed_generations[0].path + base_uri, shard_id, manifest.sstables[0].path ); - // The flushed generation must carry the BTree index built during flush. + // The SSTable must carry the BTree index built during flush. let dataset = crate::Dataset::open(&gen_uri).await.unwrap(); let indices = dataset.load_indices().await.unwrap(); - assert_eq!(indices.len(), 1, "flushed generation should have one index"); + assert_eq!(indices.len(), 1, "SSTable should have one index"); assert_eq!(indices[0].name, "id_idx"); // A PK filter over it must resolve through the index, not a full scan. @@ -4019,9 +5730,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 1024, // Very small - will trigger flush quickly manifest_scan_batch_size: 2, ..Default::default() @@ -4052,53 +5762,153 @@ mod tests { writer.close().await.unwrap(); } - /// Regression for #6713: a single failing `handle()` must not kill - /// the dispatcher. Earlier the loop would `break Err(e)` on the - /// first message error, dropping the rx side and stranding - /// subsequent senders. The flusher tasks need to survive transient - /// errors so the writer keeps making forward progress. + /// `MemTableStats::frozen_count` and `ShardMemory::frozen_bytes` count + /// different things on purpose: bytes drop on flush commit, the handle + /// lingers for `frozen_memtable_grace`. A long grace therefore leaves count + /// non-zero with bytes back at zero — and pins that the two surfaces are + /// answering different questions, not disagreeing about one. #[tokio::test] - async fn test_task_dispatcher_survives_handle_error() { - use std::sync::atomic::{AtomicUsize, Ordering}; - - struct FlakyHandler { - call_count: Arc, - } + async fn test_memtable_stats_frozen_count_outlives_frozen_bytes() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); - #[async_trait] - impl MessageHandler for FlakyHandler { - async fn handle(&mut self, message: u32) -> Result<()> { - let n = self.call_count.fetch_add(1, Ordering::SeqCst); - if n == 0 { - Err(Error::io("first message intentionally fails")) - } else { - let _ = message; - Ok(()) - } - } - } + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 1024, // small enough to seal within the loop below + frozen_memtable_grace: Duration::from_secs(600), + manifest_scan_batch_size: 2, + ..Default::default() + }; - let executor = TaskExecutor::new(); - let call_count = Arc::new(AtomicUsize::new(0)); - let (tx, rx) = mpsc::unbounded_channel::(); - executor - .add_handler( - "flaky".to_string(), - Box::new(FlakyHandler { - call_count: call_count.clone(), - }), - rx, - ) + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await .unwrap(); - // Send three messages: the first errors, the next two should - // still be delivered to the (still-alive) handler. - tx.send(1).unwrap(); - tx.send(2).unwrap(); - tx.send(3).unwrap(); + let fresh = writer.memtable_stats().await.unwrap(); + assert_eq!(fresh.frozen_count, 0); + assert_eq!(writer.memory().frozen_bytes(), 0); - for _ in 0..50 { - if call_count.load(Ordering::SeqCst) >= 3 { + for i in 0..20 { + let batch = create_test_batch(&schema, i * 10, 10); + writer.put(vec![batch]).await.unwrap(); + } + writer.wait_for_flush_drain().await.unwrap(); + + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.frozen_count > 0, + "flushed memtables must stay in the read view for the grace window" + ); + let memory = writer.memory(); + assert_eq!( + memory.frozen_bytes(), + 0, + "every seal flushed, so nothing is owed to flush" + ); + + // Owing nothing to flush is not the same as holding nothing. Those + // generations stay resident for the whole grace window, and a + // process-wide budget has to see them even though the per-shard flush + // valve deliberately does not meter them. + assert!( + memory.grace_bytes() > 0, + "flushed-but-retained generations hold real memory" + ); + assert_eq!( + memory.retained_bytes(), + memory.unflushed_bytes() + memory.grace_bytes(), + "retained is the whole footprint; unflushed is only what a flush can reclaim" + ); + + writer.close().await.unwrap(); + } + + /// The controller sits in a private `WriterMode` variant, reachable only + /// through the writer. Both modes must answer. + #[rstest::rstest] + #[case::memtable(true)] + #[case::wal_only(false)] + #[tokio::test] + async fn test_backpressure_stats_reachable_in_both_modes(#[case] enable_memtable: bool) { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + enable_memtable, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + // A writer well under the threshold has never throttled. + let stats = writer.backpressure_stats(); + assert_eq!(stats.total_count, 0); + assert_eq!(stats.total_wait_ms, 0); + assert_eq!(stats.active_count, 0); + + writer.close().await.unwrap(); + } + + /// Regression for #6713: a single failing `handle()` must not kill + /// the dispatcher. Earlier the loop would `break Err(e)` on the + /// first message error, dropping the rx side and stranding + /// subsequent senders. The flusher tasks need to survive transient + /// errors so the writer keeps making forward progress. + #[tokio::test] + async fn test_task_dispatcher_survives_handle_error() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FlakyHandler { + call_count: Arc, + } + + #[async_trait] + impl MessageHandler for FlakyHandler { + async fn handle(&mut self, message: u32) -> Result<()> { + let n = self.call_count.fetch_add(1, Ordering::SeqCst); + if n == 0 { + Err(Error::io("first message intentionally fails")) + } else { + let _ = message; + Ok(()) + } + } + } + + let executor = TaskExecutor::new(); + let call_count = Arc::new(AtomicUsize::new(0)); + let (tx, rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "flaky".to_string(), + Box::new(FlakyHandler { + call_count: call_count.clone(), + }), + rx, + ) + .unwrap(); + + // Send three messages: the first errors, the next two should + // still be delivered to the (still-alive) handler. + tx.send(1).unwrap(); + tx.send(2).unwrap(); + tx.send(3).unwrap(); + + for _ in 0..50 { + if call_count.load(Ordering::SeqCst) >= 3 { break; } tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; @@ -4110,11 +5920,117 @@ mod tests { call_count.load(Ordering::SeqCst) ); - executor.shutdown_all().await.ok(); + executor + .shutdown_all() + .await + .expect("dispatcher should shut down successfully"); + } + + #[tokio::test] + async fn test_task_executor_shutdown_propagates_cleanup_error_and_joins_all_tasks() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct CleanupHandler { + cleanup_count: Arc, + error_message: Option<&'static str>, + } + + #[async_trait] + impl MessageHandler for CleanupHandler { + async fn handle(&mut self, _message: u32) -> Result<()> { + Ok(()) + } + + async fn cleanup(&mut self, _shutdown_ok: bool) -> Result<()> { + self.cleanup_count.fetch_add(1, Ordering::SeqCst); + match self.error_message { + Some(message) => Err(Error::io(message)), + None => Ok(()), + } + } + } + + let executor = TaskExecutor::new(); + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let (_failing_tx, failing_rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "failing-cleanup".to_string(), + Box::new(CleanupHandler { + cleanup_count: cleanup_count.clone(), + error_message: Some("intentional cleanup failure"), + }), + failing_rx, + ) + .unwrap(); + let (_successful_tx, successful_rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "successful-cleanup".to_string(), + Box::new(CleanupHandler { + cleanup_count: cleanup_count.clone(), + error_message: None, + }), + successful_rx, + ) + .unwrap(); + + let error = executor + .shutdown_all() + .await + .expect_err("shutdown must propagate the handler cleanup failure"); + assert!(matches!(&error, Error::IO { .. })); + assert!( + error.to_string().contains("intentional cleanup failure"), + "unexpected error: {error}" + ); + assert_eq!( + cleanup_count.load(Ordering::SeqCst), + 2, + "shutdown must join and clean up every task after the first failure" + ); + assert!(executor.tasks.read().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_task_executor_shutdown_propagates_task_panic() { + struct PanickingCleanupHandler; + + #[async_trait] + impl MessageHandler for PanickingCleanupHandler { + async fn handle(&mut self, _message: u32) -> Result<()> { + Ok(()) + } + + async fn cleanup(&mut self, _shutdown_ok: bool) -> Result<()> { + panic!("intentional cleanup panic"); + } + } + + let executor = TaskExecutor::new(); + let (_tx, rx) = mpsc::unbounded_channel::(); + executor + .add_handler( + "panicking-cleanup".to_string(), + Box::new(PanickingCleanupHandler), + rx, + ) + .unwrap(); + + let error = executor + .shutdown_all() + .await + .expect_err("shutdown must propagate the task panic"); + assert!(matches!(&error, Error::Internal { .. })); + assert!( + error.to_string().contains("panicking-cleanup") + && error.to_string().contains("panicked during shutdown"), + "unexpected error: {error}" + ); } - /// Same as the local-fs test but against memory:// — closer to S3 - /// semantics (conditional PUT, list-prefix consistency). + /// Regression for #6713 against memory://, which is closer to S3 + /// semantics (conditional PUT and list-prefix consistency). #[tokio::test] async fn test_shard_writer_auto_flush_repeatedly_memory_store() { let base_uri = "memory:///bench_test_flush"; @@ -4126,9 +6042,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64, manifest_scan_batch_size: 2, ..Default::default() @@ -4140,17 +6055,20 @@ mod tests { let initial_gen = writer.memtable_stats().await.unwrap().generation; - for i in 0..1000 { + // The bug appeared on the second generation because repeated flushes + // reused the generation-1 path. Queue several flushes before draining + // so both path uniqueness and background sequencing are exercised. + for i in 0..8 { let batch = create_test_batch(&schema, i * 10, 10); writer.put(vec![batch]).await.unwrap(); } - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + writer.wait_for_flush_drain().await.unwrap(); let stats = writer.memtable_stats().await.unwrap(); assert!( - stats.generation >= initial_gen + 50, - "expected many flushes; generation went {} → {}", + stats.generation >= initial_gen + 3, + "expected repeated successful flushes; generation went {} → {}", initial_gen, stats.generation ); @@ -4163,7 +6081,7 @@ mod tests { /// hit "Dataset already exists: …_gen_1" once the second flush /// started. #[tokio::test] - async fn test_shard_writer_auto_flush_repeatedly_stress() { + async fn test_shard_writer_auto_flush_repeatedly_local_store() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); @@ -4171,9 +6089,8 @@ mod tests { shard_id: Uuid::new_v4(), shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), // Tiny size threshold — every batch crosses it. max_memtable_size: 64, manifest_scan_batch_size: 2, @@ -4186,21 +6103,20 @@ mod tests { let initial_gen = writer.memtable_stats().await.unwrap().generation; - // Every put crosses the size threshold, so each one queues a - // freeze. We want to catch any bug where two flushes collide on - // path/generation. Drive 1000 puts so we get ≥ 100 flushes — - // enough rope for the bug to show up. - for i in 0..1000 { + // Queue multiple generations before waiting. The original failure was + // deterministic on the second flush, so three committed generations + // are sufficient to prove that paths and generation IDs advance. + for i in 0..8 { let batch = create_test_batch(&schema, i * 10, 10); writer.put(vec![batch]).await.unwrap(); } - tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; + writer.wait_for_flush_drain().await.unwrap(); let stats = writer.memtable_stats().await.unwrap(); assert!( - stats.generation >= initial_gen + 50, - "expected many successful auto-flushes; generation went {} → {}", + stats.generation >= initial_gen + 3, + "expected repeated successful flushes; generation went {} → {}", initial_gen, stats.generation ); @@ -4238,9 +6154,8 @@ mod tests { shard_id, shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: usize::MAX, max_unflushed_memtable_bytes: usize::MAX, manifest_scan_batch_size: 2, @@ -4305,119 +6220,588 @@ mod tests { reopened.close().await.unwrap(); } - /// Regression: the memtable flush should successfully fire many - /// times in a row. A bug where every flush wrote the same path was - /// caught by lance-format/lance#6713. + #[rstest] + #[case::memtable(true)] + #[case::wal_only(false)] #[tokio::test] - async fn test_shard_writer_auto_flush_repeatedly() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + async fn test_close_propagates_final_wal_persistence_failure(#[case] enable_memtable: bool) { + let (store, base_path, controls) = failing_memory_store().await; + let base_uri = "memory:///"; let schema = create_test_schema(); - - // durable_write=true matches the LSM `merge_insert` defaults and - // is the configuration that surfaced #6713 in the wild. let config = ShardWriterConfig { shard_id: Uuid::new_v4(), shard_spec_id: 0, - durable_write: true, - sync_indexed_write: false, - max_wal_buffer_size: 1024 * 1024, + durable_write: false, + enable_memtable, + max_wal_buffer_size: usize::MAX, max_wal_flush_interval: None, - // Tiny size threshold so a few batches cross it. - max_memtable_size: 1024, + max_wal_persist_retries: 0, + max_memtable_size: usize::MAX, + max_unflushed_memtable_bytes: usize::MAX, manifest_scan_batch_size: 2, ..Default::default() }; - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) .await .unwrap(); + let task_executor = writer.task_executor.clone(); - let initial_gen = writer.memtable_stats().await.unwrap().generation; - - // Drive enough write traffic to trigger several auto-flushes. - // durable_write=true means each put waits for the WAL flush, so - // we don't need explicit yields between puts. - for i in 0..200 { - let batch = create_test_batch(&schema, i * 10, 10); - writer.put(vec![batch]).await.unwrap(); - } - - // Wait for the background memtable flushes to drain. - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + controls.fail_wal_puts(usize::MAX); - // Generation should have advanced by at least 3 — i.e. we want to - // confirm multiple flushes succeeded back to back, not just one. - let stats = writer.memtable_stats().await.unwrap(); + let error = writer + .close() + .await + .expect_err("close must propagate the final WAL persistence failure"); + assert_eq!(error.fence_reason(), Some(FenceReason::PersistenceFailure)); assert!( - stats.generation >= initial_gen + 3, - "expected ≥ 3 successful auto-flushes; generation went {} → {}", - initial_gen, - stats.generation + error + .to_string() + .contains("injected transient WAL put failure"), + "unexpected error: {error}" + ); + assert!( + task_executor.tasks.read().unwrap().is_empty(), + "close must join background tasks before returning an error" ); - - writer.close().await.unwrap(); } - #[tokio::test] - async fn test_no_backpressure_when_under_threshold() { - let config = ShardWriterConfig::default().with_max_unflushed_memtable_bytes(1024 * 1024); // 1MB - - let controller = BackpressureController::new(config); - - // Should return immediately - well under threshold (100 bytes < 1MB) - controller - .maybe_apply_backpressure(|| (100, None)) - .await - .unwrap(); + /// Recompute what the shard is holding straight from `WriterState`, the + /// way [`ShardWriter::memtable_stats`] reads it — independent of the + /// publish mechanism, which is the thing that can drift. + async fn ground_truth(writer: &ShardWriter) -> (usize, usize) { + let state = writer.memtable_state_lock().unwrap().read().await; + let active = in_memory_ref(&state.memtable).resident_bytes(); + let frozen = state + .frozen_memtables + .iter() + .filter(|frozen| frozen.flushed_at_ms.is_none()) + .map(|frozen| in_memory_ref(&frozen.memtable).resident_bytes()) + .sum(); + (active, frozen) + } - assert_eq!(controller.stats().count(), 0); + async fn assert_no_drift(writer: &ShardWriter, after: &str) { + let (active, frozen) = ground_truth(writer).await; + let memory = writer.memory(); + assert_eq!( + (memory.active_bytes(), memory.frozen_bytes()), + (active, frozen), + "published memory drifted from the writer state after {after}" + ); + assert_eq!( + memory.unflushed_bytes(), + active + frozen, + "unflushed must be the sum of the two terms after {after}" + ); } + /// The keystone invariant: the published snapshot is **derived** from + /// `WriterState`, never adjusted, so it cannot drift from it — across every + /// event that changes the memtable set. + /// + /// This is what replaced a pair of incremented counters. A counter has no + /// way back: one missed decrement is permanent, and the pod eventually + /// refuses every write with a memtable that reads empty. So this walks the + /// writer through open, puts, a seal, a flush commit, and puts into the + /// fresh memtable, checking the invariant after each. + #[rstest] + #[case::grace_keeps_handles(Duration::from_secs(600))] + #[case::zero_grace_evicts_on_commit(Duration::ZERO)] #[tokio::test] - async fn test_backpressure_loops_until_under_threshold() { - use std::sync::atomic::AtomicUsize; - use std::time::Duration; + async fn test_memory_snapshot_never_drifts_from_writer_state(#[case] grace: Duration) { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + durable_write: false, + max_wal_flush_interval: Some(Duration::from_millis(10)), + // Small enough that the loop below seals several times. + max_memtable_size: 2048, + frozen_memtable_grace: grace, + ..Default::default() + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); - let config = ShardWriterConfig::default() - .with_max_unflushed_memtable_bytes(100) // Very low threshold - .with_backpressure_log_interval(Duration::from_millis(50)); + // Seeded at open, before anything has been written. + assert_no_drift(&writer, "open").await; - let controller = BackpressureController::new(config); + for i in 0..24 { + writer + .put(vec![create_test_batch(&schema, i * 10, 10)]) + .await + .unwrap(); + assert_no_drift(&writer, &format!("put {i}")).await; + } - // Simulate: starts at 1000 bytes, drops by 400 each call (simulating memtable flushes) - let call_count = Arc::new(AtomicUsize::new(0)); - let call_count_clone = call_count.clone(); + // Seals happened above; make sure at least one did, or this test proves + // nothing about the freeze path. + assert!( + writer.memory().frozen_bytes() > 0 + || writer.memtable_stats().await.unwrap().frozen_count > 0, + "the loop must have sealed at least once for this to cover freeze" + ); - controller - .maybe_apply_backpressure(move || { - let count = call_count_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - // 1000 -> 600 -> 200 -> under threshold (need 3 iterations) - let unflushed = 1000usize.saturating_sub(count * 400); - (unflushed, None) - }) - .await - .unwrap(); + writer.wait_for_flush_drain().await.unwrap(); + assert_no_drift(&writer, "flush drain").await; - // Should have called get_state 4 times (initial + 3 waits until under 100) - assert_eq!(call_count.load(std::sync::atomic::Ordering::Relaxed), 4); - // Should have recorded backpressure wait time (waited 3 times) - assert_eq!(controller.stats().count(), 1); - } + // A long grace keeps the flushed handles in the read view and a zero + // grace evicts them on commit — two different publish paths. Either way + // they are reclaimable, so neither may keep metering. + assert_eq!( + writer.memory().frozen_bytes(), + 0, + "flushed memtables are reclaimable and must stop metering (grace {grace:?})" + ); - #[test] - fn test_record_put() { - let stats = WriteStats::new(); - stats.record_put(Duration::from_millis(10)); - stats.record_put(Duration::from_millis(20)); + for i in 24..32 { + writer + .put(vec![create_test_batch(&schema, i * 10, 10)]) + .await + .unwrap(); + assert_no_drift(&writer, &format!("post-flush put {i}")).await; + } - let snapshot = stats.snapshot(); - assert_eq!(snapshot.put_count, 2); - assert_eq!(snapshot.put_time, Duration::from_millis(30)); - assert_eq!(snapshot.avg_put_latency(), Some(Duration::from_millis(15))); + writer.close().await.unwrap(); } - #[test] - fn test_record_wal_flush() { + /// The bug the whole design exists to avoid: the old accounting read + /// `WriterState` through `try_read()`, which fails while a writer holds the + /// lock — and, because tokio's `RwLock` is write-preferring, also while one + /// is merely queued. It therefore reported **zero** on exactly the shards + /// taking writes, so a pod under load looked empty. + /// + /// Holding the write lock outright is the deterministic version of that + /// race. + #[tokio::test] + async fn test_memory_is_readable_while_a_writer_holds_the_lock() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + ShardWriterConfig { + shard_id: Uuid::new_v4(), + durable_write: false, + ..Default::default() + }, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 100)]) + .await + .unwrap(); + let unlocked = writer.memory().unflushed_bytes(); + assert!(unlocked > 0, "rows are resident, so this cannot be zero"); + + let state_lock = writer.memtable_state_lock().unwrap().clone(); + let held = state_lock.write().await; + assert_eq!( + writer.memory().unflushed_bytes(), + unlocked, + "a writer holding the lock must not zero the memory view" + ); + // The drain classification comes off the same published snapshot, so it + // answers under the lock as well. Nothing is frozen here, so the honest + // answer is that no flush is outstanding. + assert!( + matches!(writer.memory().drain(), Drain::Stalled), + "the drain classification is published too, so it reads under the lock" + ); + drop(held); + + writer.close().await.unwrap(); + } + + /// Two seal predicates exist — `MemTable::should_flush` and + /// `memtable_reached_flush_threshold` — and their *row-window* arms must trip + /// at the same byte. They did not: one counted the PK bloom filter and the + /// other did not, so they disagreed by a fixed offset on every memtable. + /// + /// Only that arm is shared. `memtable_reached_flush_threshold` also seals on + /// resident bytes and row count, which `should_flush` knows nothing about, so + /// both are held out of range below to compare like with like. + #[tokio::test] + async fn test_both_seal_predicates_share_one_byte_arm() { + let schema = create_test_schema(); + let mut memtable = + MemTable::with_capacity(schema.clone(), 1, vec![], CacheConfig::default(), 64).unwrap(); + memtable + .insert(create_test_batch(&schema, 0, 50)) + .await + .unwrap(); + + let at = memtable.batch_store().row_bytes(); + assert!(at > 0, "the memtable must hold something to be a test"); + // Owned batches, so the pinned footprint sits just under the window sum + // (no per-batch header in the allocation walk). Asserted rather than + // assumed: if it ever exceeded `at`, the retained arm would fire and this + // would be comparing something other than the row arm. + + // `incoming_batches` of 1 against a capacity of 64 keeps the batch-count + // arm out of it, and the two `usize::MAX` limits keep the row-count and + // resident arms out, so only the row-window arms are being compared. + for (bytes, expected) in [(at, true), (at + 1, false)] { + assert_eq!( + memtable.should_flush(bytes), + expected, + "should_flush at {bytes}" + ); + assert_eq!( + memtable_reached_flush_threshold(&memtable, bytes, usize::MAX, usize::MAX, 1, 1), + expected, + "the two seal predicates disagree at {bytes}; a bloom-sized offset \ + between them makes every memtable seal early on one path" + ); + } + } + + /// A handle taken before any rows exist must still see them arrive: the + /// memory view is read without the writer lock, so a handle that captured a + /// value instead of a live counter would gate writes on a stale reading. + /// Index heap is counted, row bytes are not inflated by it. + #[tokio::test] + async fn test_memory_handle_tracks_the_live_memtable() { + let schema = create_test_schema(); + let mut memtable = + MemTable::with_capacity(schema.clone(), 1, vec![], CacheConfig::default(), 8).unwrap(); + + let handle = in_memory_ref(&memtable); + let empty_rows = handle.row_bytes(); + for _ in 0..3 { + memtable + .insert(create_test_batch(&schema, 0, 10)) + .await + .unwrap(); + } + + assert!( + handle.row_bytes() > empty_rows, + "a handle taken before the writes must still see them" + ); + assert_eq!( + handle.row_bytes(), + in_memory_ref(&memtable).row_bytes(), + "a handle and a freshly taken one must agree" + ); + // The bloom filter is fixed-size and lives in the index term, so it + // never moves the flush unit. + assert_eq!( + handle.row_bytes(), + memtable.batch_store().row_bytes(), + "row bytes are the flush unit: batches only" + ); + assert_eq!( + handle.index_bytes(), + super::super::memtable::pk_bloom_filter_bytes(), + "an unindexed memtable still holds its PK bloom filter" + ); + // The ceiling is built on what the batches keep alive, not on the flush + // unit. The two row measures are close but never identical even for + // owned batches — buffer capacity is padded, and `row_bytes` charges a + // `RecordBatch` header per batch that an allocation walk does not see. + // How far they diverge for zero-copy slices, which is the case that + // matters, is pinned in `BatchStore`'s own tests. + assert_eq!( + handle.resident_bytes(), + handle.retained_row_bytes() + handle.index_bytes() + ); + } + + /// A `ShardMemory` backed by a closure instead of a live writer, re-read on + /// every poll exactly as the real one is. + fn fake_memory(read: impl Fn() -> usize + Send + Sync + 'static) -> ShardMemory { + ShardMemory(ShardMemorySource::Fake(Arc::new(read))) + } + + fn fixed_memory(unflushed: usize) -> ShardMemory { + fake_memory(move || unflushed) + } + + fn empty_shard_memory() -> ShardMemory { + fixed_memory(0) + } + + #[tokio::test] + async fn test_no_backpressure_when_under_threshold() { + let config = ShardWriterConfig::default().with_max_unflushed_memtable_bytes(1024 * 1024); // 1MB + + let controller = LocalBackpressureController::new(&config); + + // Should return immediately - well under threshold (100 bytes < 1MB) + controller + .maybe_apply_backpressure(fixed_memory(100)) + .await + .unwrap(); + + assert_eq!(controller.stats().count(), 0); + } + + #[tokio::test] + async fn test_backpressure_loops_until_under_threshold() { + use std::sync::atomic::AtomicUsize; + use std::time::Duration; + + let config = ShardWriterConfig::default() + .with_max_unflushed_memtable_bytes(100) // Very low threshold + .with_backpressure_log_interval(Duration::from_millis(50)); + + // Simulate: starts at 1000 bytes, drops by 400 each call (simulating memtable flushes) + let call_count = Arc::new(AtomicUsize::new(0)); + let call_count_clone = call_count.clone(); + + let controller = LocalBackpressureController::new(&config); + let draining = fake_memory(move || { + let count = call_count_clone.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + // 1000 -> 600 -> 200 -> under threshold (need 3 iterations) + 1000usize.saturating_sub(count * 400) + }); + + controller.maybe_apply_backpressure(draining).await.unwrap(); + + // Should have read the shard 4 times (initial + 3 waits until under 100) + assert_eq!(call_count.load(std::sync::atomic::Ordering::Relaxed), 4); + // Should have recorded backpressure wait time (waited 3 times) + assert_eq!(controller.stats().count(), 1); + assert_eq!( + controller.stats().snapshot().active_count, + 0, + "the wait is over, so nobody is parked" + ); + } + + /// The totals only move when a wait ends, so a first stall would be + /// invisible to a poller if `active_count` did not report it live. + #[tokio::test] + async fn test_backpressure_in_progress_wait_is_observable() { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering as AtomicOrdering; + use std::time::Duration; + + let config = ShardWriterConfig::default() + .with_max_unflushed_memtable_bytes(100) + .with_backpressure_log_interval(Duration::from_millis(50)); + + let unflushed = Arc::new(AtomicUsize::new(1000)); + let release = unflushed.clone(); + + let controller = LocalBackpressureController::new(&config); + let stats = controller.stats().clone(); + + let parked = controller + .maybe_apply_backpressure(fake_memory(move || unflushed.load(AtomicOrdering::Relaxed))); + + let observer = async { + // Bounded so a regression that never publishes the park fails here + // instead of hanging the suite. + let deadline = Instant::now() + Duration::from_secs(5); + while stats.snapshot().active_count == 0 { + assert!( + Instant::now() < deadline, + "an ongoing wait was never published" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + let mid = stats.snapshot(); + assert_eq!(mid.active_count, 1); + assert_eq!( + mid.total_count, 0, + "a wait still in progress is not a completed one" + ); + assert_eq!(mid.total_wait_ms, 0); + + release.store(0, AtomicOrdering::Relaxed); + }; + + let (result, ()) = tokio::join!(parked, observer); + result.unwrap(); + + let after = stats.snapshot(); + assert_eq!(after.active_count, 0, "the guard drops when the wait ends"); + assert_eq!(after.total_count, 1); + } + + /// A `put` whose caller times out drops the wait future mid-park. The gauge + /// must come back down, or a cancelled writer is throttled forever on paper. + #[tokio::test] + async fn test_backpressure_cancelled_wait_does_not_leak_active_count() { + use std::time::Duration; + + let config = ShardWriterConfig::default() + .with_max_unflushed_memtable_bytes(100) + .with_backpressure_log_interval(Duration::from_millis(50)); + + let controller = LocalBackpressureController::new(&config); + + // Never drops below the threshold, so the timeout is what ends the wait. + assert!( + tokio::time::timeout( + Duration::from_millis(50), + controller.maybe_apply_backpressure(fixed_memory(1000)), + ) + .await + .is_err() + ); + + let after = controller.stats().snapshot(); + assert_eq!(after.active_count, 0, "cancellation must release the guard"); + assert_eq!( + after.total_count, 0, + "a cancelled wait never completed, so it is not a completed wait" + ); + } + + /// Records what it saw and answers with a fixed verdict. + #[derive(Debug)] + struct SpyController { + seen: Arc>>, + reject: bool, + } + + #[async_trait::async_trait] + impl BackpressureController for SpyController { + async fn maybe_apply_backpressure(&self, shard: ShardMemory) -> Result<()> { + self.seen + .write() + .unwrap() + .push((shard.active_bytes(), shard.frozen_bytes())); + if self.reject { + return Err(Error::backpressure("full")); + } + Ok(()) + } + } + + /// An injected controller *replaces* the built-in valve rather than + /// stacking on it, and is handed what the calling shard holds, split the + /// way relief cares about — everything it needs to own the whole policy, + /// per-shard rules included. + #[tokio::test] + async fn test_injected_controller_replaces_default_and_sees_shard_memory() { + let seen = Arc::new(StdRwLock::new(Vec::new())); + let spy = Arc::new(SpyController { + seen: seen.clone(), + reject: false, + }); + // A budget the built-in valve would trip on instantly, to prove it is + // not the thing being consulted. + let config = ShardWriterConfig::default() + .with_max_unflushed_memtable_bytes(1) + .with_backpressure(spy); + + let controller = resolve_backpressure(&config); + controller + .maybe_apply_backpressure(fixed_memory(1000)) + .await + .unwrap(); + + assert_eq!( + *seen.read().unwrap(), + vec![(1000, 0)], + "the injected controller ran, and the built-in valve did not park the write" + ); + } + + /// `ShardMemory` must stay live across polls: a controller that delays is + /// waiting for these bytes to fall, so a captured copy would spin forever. + #[tokio::test] + async fn test_shard_memory_reflects_drain_across_polls() { + #[derive(Debug)] + struct DrainWaiter { + polls: AtomicUsize, + } + + #[async_trait::async_trait] + impl BackpressureController for DrainWaiter { + async fn maybe_apply_backpressure(&self, shard: ShardMemory) -> Result<()> { + while shard.unflushed_bytes() > 0 { + self.polls.fetch_add(1, Ordering::Relaxed); + tokio::task::yield_now().await; + } + Ok(()) + } + } + + let resident = Arc::new(AtomicUsize::new(4_096)); + let drain = resident.clone(); + let view = fake_memory(move || resident.load(Ordering::Relaxed)); + + let controller = Arc::new(DrainWaiter { + polls: AtomicUsize::new(0), + }); + let gate = controller.clone(); + let waiting = tokio::spawn(async move { gate.maybe_apply_backpressure(view).await }); + + // Let the gate observe the full pool, then drain it as a flush commit + // would. A stale copy would never see this and the task would hang. + tokio::task::yield_now().await; + drain.store(0, Ordering::Relaxed); + + waiting.await.unwrap().unwrap(); + assert!(controller.polls.load(Ordering::Relaxed) > 0); + } + + /// With nothing injected, lance keeps its own per-shard valve. + #[tokio::test] + async fn test_default_backpressure_is_used_when_none_injected() { + let config = ShardWriterConfig::default().with_max_unflushed_memtable_bytes(100); + let controller = resolve_backpressure(&config); + + let polls = Arc::new(AtomicUsize::new(0)); + let polls_clone = polls.clone(); + controller + .maybe_apply_backpressure(fake_memory(move || { + polls_clone.fetch_add(1, Ordering::Relaxed); + 0 + })) + .await + .unwrap(); + + assert_eq!(polls.load(Ordering::Relaxed), 1); + } + + /// A rejecting controller surfaces as `Error::Backpressure`, which is + /// distinguishable from a real failure without matching on the message. + #[tokio::test] + async fn test_injected_controller_rejects_with_backpressure_error() { + let spy = Arc::new(SpyController { + seen: Arc::new(StdRwLock::new(Vec::new())), + reject: true, + }); + let config = ShardWriterConfig::default().with_backpressure(spy); + let controller = resolve_backpressure(&config); + + let err = controller + .maybe_apply_backpressure(empty_shard_memory()) + .await + .unwrap_err(); + assert!(err.is_backpressure(), "expected backpressure, got {err:?}"); + } + + #[test] + fn test_record_put() { + let stats = WriteStats::new(); + stats.record_put(Duration::from_millis(10)); + stats.record_put(Duration::from_millis(20)); + + let snapshot = stats.snapshot(); + assert_eq!(snapshot.put_count, 2); + assert_eq!(snapshot.put_time, Duration::from_millis(30)); + assert_eq!(snapshot.avg_put_latency(), Some(Duration::from_millis(15))); + } + + #[test] + fn test_record_wal_flush() { let stats = WriteStats::new(); stats.record_wal_flush(Duration::from_millis(100), 1024); stats.record_wal_flush(Duration::from_millis(200), 2048); @@ -4462,7 +6846,7 @@ mod tests { durable_write: true, enable_memtable: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), manifest_scan_batch_size: 2, ..Default::default() } @@ -4487,8 +6871,9 @@ mod tests { .await .unwrap(); - // Two durable puts → two WAL entries (durable_write triggers an - // explicit flush per put). + // Two durable puts → two WAL entries. Each `put` awaits its own append + // (driven by the background ticker) before returning, so the second + // batch is only pushed after the first is durable — they never coalesce. let r1 = writer .put(vec![create_test_batch(&schema, 0, 4)]) .await @@ -4519,6 +6904,51 @@ mod tests { assert!(e0.writer_epoch >= 1); } + /// A durable WAL-only put is driven by the background ticker, not an inline + /// per-put flush: `put().await` must not return until the ticker's append + /// advances the durability watermark it waits on. With a long interval and a + /// buffer the write cannot cross, the ticker is the *only* thing that can + /// make the put durable — so if the WAL entry is present the instant `put` + /// returns (before any `close()`), the watermark-wait is doing its job. + #[tokio::test] + async fn test_wal_only_durable_put_waits_for_ticker_append() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let shard_id = Uuid::new_v4(); + + let mut config = wal_only_config(shard_id); + config.max_wal_flush_interval = Some(Duration::from_millis(100)); + config.max_wal_buffer_size = 100 * 1024 * 1024; // never crossed + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 4)]) + .await + .unwrap(); + + // `put` returned, so the batch must already be durable — read the WAL + // directly, without closing the writer. + let tailer = WalTailer::new(store, base_path, shard_id); + assert_eq!(tailer.next_position().await.unwrap(), 2); + let entry = tailer.read_entry(1).await.unwrap().unwrap(); + assert_eq!(entry.batches.len(), 1); + assert_eq!(entry.batches[0].num_rows(), 4); + + writer.close().await.unwrap(); + } + #[tokio::test] async fn test_wal_only_rejects_index_configs() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; @@ -4753,9 +7183,8 @@ mod tests { shard_id, shard_spec_id: 0, durable_write: true, - sync_indexed_write: false, max_wal_buffer_size: 1024 * 1024, - max_wal_flush_interval: None, + max_wal_flush_interval: Some(Duration::from_millis(10)), max_memtable_size: 64 * 1024 * 1024, manifest_scan_batch_size: 2, ..Default::default() @@ -4779,8 +7208,9 @@ mod tests { } // A durable write whose WAL PUT keeps failing poisons the writer with a - // typed persistence failure; the next write fails fast with the same reason; - // and once storage heals, reopening replays the WAL and writes resume. + // typed persistence failure; the next write *and every read* fail fast with + // the same reason; and once storage heals, reopening replays the WAL and + // writes resume. #[tokio::test] async fn test_writer_poisons_on_persistence_failure_and_recovers_on_reopen() { let (store, base_path, controls) = failing_memory_store().await; @@ -4819,6 +7249,31 @@ mod tests { .await .unwrap_err(); assert_eq!(err.fence_reason(), Some(FenceReason::PersistenceFailure)); + + // ...and rejects *reads* too. Batch 0 was committed to the BatchStore + // before its WAL PUT failed, so a poisoned writer that still served + // reads would hand out a row that is not durable and that replay will + // not reproduce — a divergent snapshot. Mirrors SlateDB's + // `check_closed()` at the top of every read. + for reason in [ + writer.scan().await.err().and_then(|e| e.fence_reason()), + writer + .active_memtable_ref() + .await + .err() + .and_then(|e| e.fence_reason()), + writer + .in_memory_memtable_refs() + .await + .err() + .and_then(|e| e.fence_reason()), + ] { + assert_eq!(reason, Some(FenceReason::PersistenceFailure)); + } + + // Stats stay readable: this is what an operator (and the eviction path) + // inspects to decide what to do about the poisoned shard. + writer.memtable_stats().await.unwrap(); drop(writer); // Storage heals: reopening replays the WAL and accepts writes again. @@ -4832,208 +7287,324 @@ mod tests { .unwrap(); } - /// Replay-on-open recovers durable WAL entries that were never flushed - /// to a Lance generation. Setup: writer A durably writes batches, drops - /// without close (so MemTable freeze never runs); writer B reopens and - /// must see A's rows in its MemTable scan. + /// A doomed open must fail on local validation *before* it claims the epoch, + /// so it cannot fence the healthy writer already serving the shard. The + /// index-config check used to run *after* `claim_epoch` (and, for a successor, + /// after `write_fence_sentinel`), so a rejected open still bumped the stored + /// epoch and fenced the incumbent. #[tokio::test] - async fn test_memtable_replay_recovers_unflushed_writes() { + async fn test_doomed_open_does_not_fence_incumbent() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = schema_with_pk(); let shard_id = Uuid::new_v4(); - // Writer A: write two durable batches, then drop without close. - // The WAL files persist; the in-memory MemTable does not. - { - let writer_a = ShardWriter::open( - store.clone(), - base_path.clone(), - base_uri.clone(), - memtable_config_with_pk(shard_id), - schema.clone(), - vec![], - ) - .await - .unwrap(); - writer_a - .put(vec![create_test_batch(&schema, 0, 5)]) - .await - .unwrap(); - writer_a - .put(vec![create_test_batch(&schema, 100, 3)]) - .await - .unwrap(); - // intentionally drop without close() - } - - // Writer B reopens. Replay must rehydrate A's two batches into the - // active MemTable. - let writer_b = ShardWriter::open( - store, - base_path, - base_uri, + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), memtable_config_with_pk(shard_id), - schema, + schema.clone(), vec![], ) .await .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 1)]) + .await + .unwrap(); - let stats = writer_b.memtable_stats().await.unwrap(); - assert_eq!( - stats.row_count, 8, - "expected replay to insert 5 + 3 = 8 rows, got {}", - stats.row_count - ); - assert_eq!( - stats.batch_count, 2, - "expected replay to insert 2 batches, got {}", - stats.batch_count + // An index config that disagrees with the schema (FTS on the Int32 `id` + // column) is rejected on local validation. On the old path this rejection + // landed only after the epoch had already been claimed. + let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new( + "bad_fts".to_string(), + 0, + "id".to_string(), + )); + let err = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![bad_fts], + ) + .await + .map(|_| ()) + .expect_err("an FTS index on a non-Utf8 column must be rejected"); + assert!( + err.to_string().contains("bad_fts") && err.to_string().contains("Utf8"), + "unexpected error: {err}" ); - writer_b.close().await.unwrap(); + // The incumbent is untouched: not fenced, still accepting writes. + writer_a.check_fenced().await.unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 1, 1)]) + .await + .unwrap(); + writer_a.close().await.unwrap(); } - /// Replay is a no-op on a fresh shard: the MemTable starts empty. + /// A failed dispatch during `freeze_memtable` must not drop the outgoing + /// table's rows from the read view. The active memtable is replaced before + /// the WAL-flush and index-apply sends; a send that failed (background tasks + /// gone) used to return before the outgoing table was retained in + /// `frozen_memtables`, so its accepted rows silently vanished — a scan + /// returned 0 rows with no error. The writer must instead retain the table + /// and poison, so reads fail fast rather than serve a divergent snapshot. #[tokio::test] - async fn test_memtable_replay_no_op_on_fresh_shard() { + async fn test_freeze_dispatch_failure_retains_rows_and_poisons() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = schema_with_pk(); let shard_id = Uuid::new_v4(); - let writer = ShardWriter::open( - store, - base_path, - base_uri, - memtable_config_with_pk(shard_id), - schema, - vec![], - ) - .await - .unwrap(); - let stats = writer.memtable_stats().await.unwrap(); - assert_eq!(stats.row_count, 0); - assert_eq!(stats.batch_count, 0); - writer.close().await.unwrap(); + // Non-durable + no ticker: the put is read-your-writes (waits for its + // index apply) but nothing is WAL-flushed, so the freeze below still owes + // a WAL append. + let config = ShardWriterConfig { + durable_write: false, + max_wal_flush_interval: None, + ..memtable_config_with_pk(shard_id) + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + assert_eq!(writer.memtable_stats().await.unwrap().row_count, 10); + + // Tear the background tasks down out from under the writer, so the + // freeze's dispatch sends hit closed channels. + writer.abort().await.unwrap(); + + let err = writer + .force_seal_active() + .await + .expect_err("force_seal_active must surface the failed dispatch"); + assert!( + err.to_string().contains("channel closed"), + "unexpected error: {err}" + ); + + // The failure poisoned the writer: reads fail fast instead of returning a + // silent zero-row snapshot of a shard whose rows were dropped. + assert!( + writer.scan().await.is_err(), + "a poisoned writer must reject reads, not serve a divergent snapshot" + ); + assert!(writer.in_memory_memtable_refs().await.is_err()); } - /// Regression for the OSS-WAL compactor-drain bug: after a flush - /// records its generation in the manifest and an external compactor - /// later drains `flushed_generations` back to empty (the legitimate - /// outcome of merging the generation into the base table), reopening - /// the writer must not re-replay the already-flushed WAL entry into - /// the active memtable. + /// A WAL holding more batches than one memtable's capacity must reopen. /// - /// Under the pre-fix logic, replay disambiguated "fresh shard" from - /// "flushed-then-compacted" with `flushed_generations.is_empty()`, - /// which collapsed both cases into start-at-0. With 1-based WAL - /// positions and a default cursor of 0 meaning "no flush stamped", - /// the flush-then-drain sequence leaves `replay_after_wal_entry_position` - /// pinned at the flushed position, so replay correctly starts past it. + /// One memtable holds at most `max_memtable_batches` batches, but a WAL is + /// unbounded, so replay has to rotate — seal the full memtable, start a fresh + /// one — exactly as the live write path does. Before, replay stuffed + /// everything into a single memtable and `open()` failed outright with + /// "MemTable batch store is full", leaving the shard permanently unopenable. #[tokio::test] - async fn test_memtable_replay_skips_entries_after_external_compaction() { - use crate::dataset::mem_wal::ShardManifestStore; - + async fn test_replay_rotates_when_wal_exceeds_one_memtable() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = schema_with_pk(); let shard_id = Uuid::new_v4(); - // Writer A: write 5 rows, close (forces a flush of the active - // memtable). The manifest now records a flushed generation and - // pins `replay_after_wal_entry_position` to the covered WAL entry. + const N: i32 = 8; + + // Writer A has a *large* capacity, so its eight one-batch puts all land in + // a single memtable and it never freezes or flushes a generation of its + // own. Dropping it without close leaves an eight-entry WAL and no + // generations — a WAL that no single small memtable could hold. + let writer_a_config = ShardWriterConfig { + max_memtable_batches: 1000, + ..memtable_config_with_pk(shard_id) + }; + // Writer B has a *two-batch* capacity, so replaying that eight-entry WAL is + // exactly what must rotate. Keeping the configs distinct isolates replay + // rotation from the live rotation writer A would otherwise do concurrently. + let config = ShardWriterConfig { + max_memtable_batches: 2, + ..memtable_config_with_pk(shard_id) + }; + { let writer_a = ShardWriter::open( store.clone(), base_path.clone(), base_uri.clone(), - memtable_config_with_pk(shard_id), + writer_a_config, schema.clone(), vec![], ) .await .unwrap(); - writer_a - .put(vec![create_test_batch(&schema, 0, 5)]) - .await - .unwrap(); - writer_a.close().await.unwrap(); + for id in 0..N { + writer_a + .put(vec![create_test_batch(&schema, id, 1)]) + .await + .unwrap(); + } + // Drop without close: only the WAL survives, and it holds more batches + // than writer B's memtable can. } - // Simulate an external compactor merging the flushed generation - // into the base table: drain `flushed_generations` to empty via a - // direct manifest commit. The cursor stays where the flush put it. - let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); - let pre = manifest_store.read_latest().await.unwrap().unwrap(); + // Total rows across the active memtable plus every SSTable. + // Distinct ids, so no cross-generation dedup — a plain sum is exact. + async fn total_rows(writer: &ShardWriter, base_uri: &str, shard_id: Uuid) -> usize { + let mut rows = writer.memtable_stats().await.unwrap().row_count; + let manifest = writer.manifest().await.unwrap().unwrap(); + for sstable in &manifest.sstables { + let gen_uri = format!("{}/_mem_wal/{}/{}", base_uri, shard_id, sstable.path); + let dataset = crate::Dataset::open(&gen_uri).await.unwrap(); + rows += dataset.count_rows(None).await.unwrap(); + } + rows + } + + // Reopen. This used to fail with a full-batch-store error. + let writer_b = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + config.clone(), + schema.clone(), + vec![], + ) + .await + .expect("a WAL larger than one memtable must still reopen"); + + // Rotation produced sealed memtables, and replay flushed each to a Lance + // generation rather than holding it in memory or leaving it in the WAL. + let manifest = writer_b.manifest().await.unwrap().unwrap(); assert!( - !pre.flushed_generations.is_empty(), - "writer A's close() should have stamped a flushed generation" + !manifest.sstables.is_empty(), + "replay must have sealed and flushed at least one full memtable" ); - let cursor_at_flush = pre.replay_after_wal_entry_position; - assert!( - cursor_at_flush >= 1, - "expected cursor to land on a 1-based WAL position after flush, got {cursor_at_flush}" + + // Every row survived, split between the SSTables and the + // active (partial) memtable. + assert_eq!( + total_rows(&writer_b, &base_uri, shard_id).await as i32, + N, + "every replayed row must be durable, across generations and the active memtable" ); - // Bump the epoch (claim_epoch) so we can commit_update without - // being fenced; this also mirrors how a compactor process would - // hold its own writer claim. - let (compactor_epoch, _) = manifest_store.claim_epoch(pre.shard_spec_id).await.unwrap(); - manifest_store - .commit_update(compactor_epoch, |current| ShardManifest { - version: current.version + 1, - flushed_generations: vec![], - ..current.clone() - }) + writer_b.close().await.unwrap(); + + // Because the sealed memtables were flushed, the manifest's replay cursor + // advanced past their WAL entries — so a second reopen replays only the + // tail and still accounts for every row. The WAL truncates across reopens + // rather than growing without bound. + let writer_c = + ShardWriter::open(store, base_path, base_uri.clone(), config, schema, vec![]) + .await + .unwrap(); + assert_eq!(total_rows(&writer_c, &base_uri, shard_id).await as i32, N); + writer_c.close().await.unwrap(); + } + + /// The same rotation, driven by `max_memtable_rows` instead of the batch cap. + /// + /// Replay builds the final memtable's indexes itself, so a WAL holding more + /// rows than one memtable's capacity has to rotate for `open()` to succeed. + #[tokio::test] + async fn test_replay_rotates_when_wal_exceeds_the_row_cap() { + use lance_arrow::FixedSizeListArrayExt; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let dim = 8; + let schema = hnsw_schema(dim); + let shard_id = Uuid::new_v4(); + let cap = 8; + + let vector_batch = |start: i32, rows: usize| { + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from( + (0..rows * dim as usize) + .map(|v| v as f32 * 0.01) + .collect::>(), + ), + dim, + ) + .unwrap(); + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from( + (start..start + rows as i32).collect::>(), + )), + Arc::new(vectors), + ], + ) + .unwrap() + }; + + // Writer A's row cap is far above what it writes, so all 32 rows land in + // one memtable and dropping it without close leaves them all in the WAL. + let writer_a_config = ShardWriterConfig { + max_memtable_rows: 10_000, + ..memtable_config_with_pk(shard_id) + }; + // Writer B caps a memtable at 8 rows — and sizes its HNSW graph to match. + let config = ShardWriterConfig { + max_memtable_rows: cap, + ..memtable_config_with_pk(shard_id) + }; + + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + writer_a_config, + schema.clone(), + hnsw_configs(), + ) .await .unwrap(); - let post = manifest_store.read_latest().await.unwrap().unwrap(); + for round in 0..8i32 { + writer_a + .put(vec![vector_batch(round * 4, 4)]) + .await + .unwrap(); + } + } + + // Replay has 32 rows to place into memtables capped at 8. + let writer_b = + ShardWriter::open(store, base_path, base_uri, config, schema, hnsw_configs()) + .await + .expect("a WAL holding more rows than the cap must still reopen"); + + let manifest = writer_b.manifest().await.unwrap().unwrap(); assert!( - post.flushed_generations.is_empty(), - "compactor drain should have left flushed_generations empty" - ); - assert_eq!( - post.replay_after_wal_entry_position, cursor_at_flush, - "compactor must not touch the replay cursor" + !manifest.sstables.is_empty(), + "replay must have sealed and flushed the memtables it filled" ); - - // Writer B reopens. Pre-fix: replay saw flushed_generations empty, - // restarted at WAL position 0, and re-inserted writer A's rows. - // Post-fix: replay starts at cursor + 1, finds no entry, and the - // memtable stays empty. - let writer_b = ShardWriter::open( - store, - base_path, - base_uri, - memtable_config_with_pk(shard_id), - schema, - vec![], - ) - .await - .unwrap(); let stats = writer_b.memtable_stats().await.unwrap(); - assert_eq!( - stats.row_count, 0, - "memtable must not re-replay compacted WAL entries; got {} rows", + assert!( + stats.row_count <= cap, + "replay left {} rows in a memtable capped at {cap}", stats.row_count ); - assert_eq!(stats.batch_count, 0); + writer_b.close().await.unwrap(); } - /// Replay aborts the open with a clear fence error if it encounters a - /// WAL entry written with an epoch strictly greater than ours. Simulate - /// the race where another writer wrote an entry with a higher epoch - /// between our `claim_epoch` and our replay by injecting a high-epoch - /// entry directly via `WalAppender::with_claimed_epoch` (which - /// bypasses `claim_epoch` and so does not bump the manifest). + /// Replay-on-open recovers durable WAL entries that were never flushed + /// to a Lance generation. Setup: writer A durably writes batches, drops + /// without close (so MemTable freeze never runs); writer B reopens and + /// must see A's rows in its MemTable scan. #[tokio::test] - async fn test_memtable_replay_fenced_aborts_open() { - use crate::dataset::mem_wal::ShardManifestStore; - + async fn test_memtable_replay_recovers_unflushed_writes() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = schema_with_pk(); let shard_id = Uuid::new_v4(); - // Writer A: write one durable batch (claims epoch 1, writes entry at position 1). + // Writer A: write two durable batches, then drop without close. + // The WAL files persist; the in-memory MemTable does not. { let writer_a = ShardWriter::open( store.clone(), @@ -5046,597 +7617,2063 @@ mod tests { .await .unwrap(); writer_a - .put(vec![create_test_batch(&schema, 0, 1)]) + .put(vec![create_test_batch(&schema, 0, 5)]) .await .unwrap(); - // drop without close + writer_a + .put(vec![create_test_batch(&schema, 100, 3)]) + .await + .unwrap(); + // intentionally drop without close() } - // Inject a WAL entry written with epoch 100 — far above whatever - // claim_epoch will hand the next opener. The manifest is not - // updated since we use `with_claimed_epoch` directly. - let manifest_store = Arc::new(ShardManifestStore::new( - store.clone(), - &base_path, - shard_id, - 2, - )); - let high_epoch_appender = WalAppender::with_claimed_epoch( - store.clone(), - base_path.clone(), + // Writer B reopens. Replay must rehydrate A's two batches into the + // active MemTable. + let writer_b = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![], + ) + .await + .unwrap(); + + let stats = writer_b.memtable_stats().await.unwrap(); + assert_eq!( + stats.row_count, 8, + "expected replay to insert 5 + 3 = 8 rows, got {}", + stats.row_count + ); + assert_eq!( + stats.batch_count, 2, + "expected replay to insert 2 batches, got {}", + stats.batch_count + ); + + writer_b.close().await.unwrap(); + } + + /// Replayed batches are already WAL-durable, so the first flush after a + /// reopen must not re-append them to the WAL or re-insert them into the + /// indexes. Before replay stamped the durability cursor it stayed at + /// "nothing flushed", so the next flush re-covered `[0, end)`: it appended + /// the already-durable rows a second time *and* re-indexed them. None of + /// the in-memory indexes is idempotent, so an indexed PK lookup returned + /// the row twice while a full scan still looked healthy. + #[tokio::test] + async fn test_replay_does_not_reappend_or_reindex() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Writer A: two durable batches (5 + 3 = 8 rows), dropped without close. + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 100, 3)]) + .await + .unwrap(); + } + + // Writer B reopens and replays A's two batches. + let writer_b = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + + let stats = writer_b.memtable_stats().await.unwrap(); + assert_eq!(stats.batch_count, 2); + assert_eq!( + stats.durable_batch_count, 2, + "replayed batches came from the WAL, so the durability cursor must already cover them" + ); + + // One more durable put. Its flush must cover only the new batch. + writer_b + .put(vec![create_test_batch(&schema, 200, 2)]) + .await + .unwrap(); + + let tailer = WalTailer::new(store, base_path, shard_id); + let first = tailer.first_position().await.unwrap(); + let next = tailer.next_position().await.unwrap(); + let mut wal_rows = 0; + for position in first..next { + if let Some(entry) = tailer.read_entry(position).await.unwrap() { + wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::(); + } + } + assert_eq!( + wal_rows, 10, + "WAL must hold 8 replayed + 2 new rows; a re-covering flush re-appends the replayed 8" + ); + + // The indexed arm must not see a replayed row twice. + let mut scanner = writer_b.scan().await.unwrap(); + scanner.filter("id = 0").unwrap(); + let hit = scanner.try_into_batch().await.unwrap(); + assert_eq!( + hit.num_rows(), + 1, + "indexed PK lookup returned the replayed row more than once" + ); + + let all = writer_b + .scan() + .await + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(all.num_rows(), 10); + + writer_b.close().await.unwrap(); + } + + /// A non-durable put is readable through the **index-backed** arms the moment + /// it returns, not just through a full scan. + /// + /// This is what splitting the index apply off the WAL flush buys. Before, the + /// index apply only ran as one arm of the flush, so with `durable_write: + /// false` nothing triggered it on the put path at all: the row sat in the + /// batch store, unindexed, until some later flush happened along. A full scan + /// (which reads the batch store directly) could still find it, while every + /// index-accelerated query could not — the tiers disagreed. Now the apply is + /// triggered per-put in both modes, so `put` returning means "indexed". + #[tokio::test] + async fn test_non_durable_put_is_visible_through_the_index() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + let config = ShardWriterConfig { + durable_write: false, + ..memtable_config_with_pk(shard_id) + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + + // The indexed PK lookup must find it — this is the arm that saw nothing + // before, because the index apply had never run. + let mut scanner = writer.scan().await.unwrap(); + scanner.filter("id = 3").unwrap(); + let hit = scanner.try_into_batch().await.unwrap(); + assert_eq!( + hit.num_rows(), + 1, + "an index-backed lookup must see a non-durable put as soon as it returns" + ); + + // ...and so must the unindexed full scan, i.e. the tiers agree. + let all = writer.scan().await.unwrap().try_into_batch().await.unwrap(); + assert_eq!(all.num_rows(), 5); + + writer.close().await.unwrap(); + } + + /// The durability cursor is writer-global, so a put into a *post-rotation* + /// memtable must still wait for its own WAL append. + /// + /// Batch positions restart at 0 in every memtable, but the durability watch + /// channel spans the writer's whole life and is never reset. When the put + /// path targeted a memtable-local position, the first N puts into every + /// memtable after the first were already "satisfied" by the *previous* + /// memtable's N appends: they acked instantly, with no WAL append, and + /// `durable_write: true` silently degraded to non-durable. Worse, the next + /// append then sent a *smaller* value, walking the watermark backwards and + /// hanging any watcher still waiting on the old, higher one. + /// + /// The cursor is now a writer-global exclusive count and every target is + /// lifted through the store's `global_offset`, so it only ever moves forward + /// and a post-rotation put can only be acked by its own append. + #[tokio::test] + async fn test_durable_ack_after_rotation_requires_its_own_wal_append() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // A two-batch memtable, so the third put forces a freeze + rotation. + let config = ShardWriterConfig { + max_memtable_batches: 2, + ..memtable_config_with_pk(shard_id) + }; + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Fill and rotate the first memtable. + for i in 0..3 { + writer + .put(vec![create_test_batch(&schema, i * 5, 5)]) + .await + .unwrap(); + } + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.global_offset > 0, + "expected a rotation; the active memtable is still the writer's first" + ); + + // Every batch acked so far must be genuinely durable, and the cursor must + // cover the active memtable's entire prefix rather than lagging inside it. + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.durable_batch_count >= stats.global_offset + stats.batch_count, + "durable_write acked a put the WAL never received: durable={} but the active \ + memtable spans [{}, {})", + stats.durable_batch_count, + stats.global_offset, + stats.global_offset + stats.batch_count + ); + + // And the WAL really holds every row we acked (3 puts x 5 rows). + writer.close().await.unwrap(); + let tailer = WalTailer::new(store, base_path, shard_id); + let first = tailer.first_position().await.unwrap(); + let next = tailer.next_position().await.unwrap(); + let mut wal_rows = 0; + for position in first..next { + if let Some(entry) = tailer.read_entry(position).await.unwrap() { + wal_rows += entry.batches.iter().map(|b| b.num_rows()).sum::(); + } + } + assert_eq!( + wal_rows, 15, + "every acked row must be in the WAL; a post-rotation put that acked without an \ + append would leave rows missing" + ); + } + + /// A background tick must append the **oldest** store that still owes the WAL, + /// never "whatever memtable is active". + /// + /// A tick carries no store: it is enqueued by a timer and resolved when it is + /// handled. So a tick enqueued before a freeze is handled *after* it, and + /// resolving to the active memtable would append the incoming memtable's + /// batches ahead of the outgoing memtable's tail. WAL entry positions are + /// assigned in append-call order, replay walks them ascending, row positions + /// follow, and primary-key recency is "newest visible row position wins" — so + /// that inverts dedup after a crash, handing the key to the stale row. It + /// survives the crash that caused it, and a full scan cannot see it. + #[test] + fn test_next_pending_store_picks_the_oldest_owing_an_append() { + let schema = create_test_schema(); + + // A frozen store of 2 batches at coordinate 0, and the active store that + // rotated in behind it at coordinate 2. + let frozen = Arc::new(BatchStore::with_capacity(4)); + frozen.append(create_test_batch(&schema, 0, 1)).unwrap(); + frozen.append(create_test_batch(&schema, 1, 1)).unwrap(); + let active = Arc::new(BatchStore::with_capacity_at(4, 2)); + active.append(create_test_batch(&schema, 2, 1)).unwrap(); + + let frozen_list = || std::iter::once(Arc::clone(&frozen)); + + // Nothing durable: the frozen store owes the oldest append, so it wins — + // even though the active memtable also has un-appended batches. + let picked = next_pending_store(frozen_list(), Arc::clone(&active), 0).unwrap(); + assert!( + Arc::ptr_eq(&picked, &frozen), + "the outgoing memtable's tail must be appended before the incoming one's head" + ); + + // Still true partway through the frozen store. + let picked = next_pending_store(frozen_list(), Arc::clone(&active), 1).unwrap(); + assert!(Arc::ptr_eq(&picked, &frozen)); + + // Once the frozen store is fully durable, the active one is next. + let picked = next_pending_store(frozen_list(), Arc::clone(&active), 2).unwrap(); + assert!(Arc::ptr_eq(&picked, &active)); + + // Everything durable: nothing to do. + assert!(next_pending_store(frozen_list(), Arc::clone(&active), 3).is_none()); + } + + /// A durable writer with no flush ticker cannot make progress in either + /// mode — the ticker is the only thing that drives the WAL append the put + /// waits on — so `open()` rejects it rather than letting a put block forever. + #[rstest] + #[case::memtable(true)] + #[case::wal_only(false)] + #[tokio::test] + async fn test_open_rejects_durable_write_without_a_ticker(#[case] enable_memtable: bool) { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + let config = ShardWriterConfig { + durable_write: true, + enable_memtable, + max_wal_flush_interval: None, + ..memtable_config_with_pk(shard_id) + }; + + let Err(err) = ShardWriter::open(store, base_path, base_uri, config, schema, vec![]).await + else { + panic!("durable_write with no ticker must be rejected"); + }; + assert!( + err.to_string().contains("max_wal_flush_interval"), + "the error must name the knob, got: {err}" + ); + } + + /// WAL entries must be appended in global batch-position order across a + /// memtable rotation, because append order *is* primary-key recency order. + /// + /// `WalAppender::append` assigns each entry's position from its own counter, + /// in call order. Replay walks those positions ascending and assigns row + /// positions in that order. Primary-key recency is "newest visible row + /// position wins". So an out-of-order append silently inverts dedup after a + /// crash — the stale row wins — and a full scan cannot see it. + /// + /// The hazard is the background ticker. If it resolved to "whatever memtable + /// is active" rather than to the oldest store still owing an append, a tick + /// enqueued before a freeze but handled after it would append the *incoming* + /// memtable's batches ahead of the outgoing memtable's tail. So the target is + /// resolved from the durability cursor, not from wall-clock timing. + #[tokio::test] + async fn test_wal_append_order_preserves_pk_recency_across_rotation() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Two batches per memtable, so the second put fills it and rotates. + let config = ShardWriterConfig { + max_memtable_batches: 2, + ..memtable_config_with_pk(shard_id) + }; + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Memtable 1: ids 7 then 20 (the second fills it and triggers the freeze). + writer + .put(vec![create_test_batch(&schema, 7, 1)]) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 20, 1)]) + .await + .unwrap(); + // Memtable 2 overwrites id=7 with a newer row. Its append must land in the + // WAL *after* memtable 1's, or replay would resolve id=7 to the stale copy. + writer + .put(vec![create_test_batch(&schema, 30, 1)]) + .await + .unwrap(); + writer.close().await.unwrap(); + + // Walk the WAL in entry order and collect the ids as replay would see them. + let tailer = WalTailer::new(store, base_path, shard_id); + let first = tailer.first_position().await.unwrap(); + let next = tailer.next_position().await.unwrap(); + let mut ids: Vec = Vec::new(); + for position in first..next { + let Some(entry) = tailer.read_entry(position).await.unwrap() else { + continue; + }; + for batch in &entry.batches { + let column = batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + ids.extend((0..column.len()).map(|i| column.value(i))); + } + } + + assert_eq!( + ids, + vec![7, 20, 30], + "WAL entries must follow global batch-position order; memtable 1's rows must \ + precede memtable 2's, or replay inverts primary-key recency" + ); + } + + /// An index config that disagrees with the schema fails `open()` outright. + /// It must not be allowed to accept writes: the insert would fail + /// deterministically on every batch, including batches replayed from the + /// WAL, so once a row was durable the shard could never reopen. Before this + /// check, an FTS index on a non-Utf8 column silently indexed nothing and the + /// shard reported healthy. + #[tokio::test] + async fn test_open_rejects_index_config_that_disagrees_with_schema() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // `id` is Int32, not a string column. + let bad_fts = MemIndexConfig::Fts(FtsIndexConfig::new( + "bad_fts".to_string(), + 0, + "id".to_string(), + )); + + let Err(err) = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![bad_fts], + ) + .await + else { + panic!("open must reject an FTS index on a non-Utf8 column"); + }; + + let message = err.to_string(); + assert!( + message.contains("bad_fts") && message.contains("Utf8"), + "error must name the index and the constraint, got: {message}" + ); + } + + /// Replay is a no-op on a fresh shard: the MemTable starts empty. + #[tokio::test] + async fn test_memtable_replay_no_op_on_fresh_shard() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + let writer = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![], + ) + .await + .unwrap(); + let stats = writer.memtable_stats().await.unwrap(); + assert_eq!(stats.row_count, 0); + assert_eq!(stats.batch_count, 0); + writer.close().await.unwrap(); + } + + /// Regression for the OSS-WAL compactor-drain bug: after a flush + /// records its generation in the manifest and an external compactor + /// later drains `sstables` back to empty (the legitimate + /// outcome after compacting the SSTable into the base table), reopening + /// the writer must not re-replay the already-flushed WAL entry into + /// the active memtable. + /// + /// Under the pre-fix logic, replay disambiguated "fresh shard" from + /// "flushed-then-compacted" with `sstables.is_empty()`, + /// which collapsed both cases into start-at-0. With 1-based WAL + /// positions and a default cursor of 0 meaning "no flush stamped", + /// the flush-then-drain sequence leaves `replay_after_wal_entry_position` + /// pinned at the flushed position, so replay correctly starts past it. + #[tokio::test] + async fn test_memtable_replay_skips_entries_after_external_compaction() { + use crate::dataset::mem_wal::ShardManifestStore; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Writer A: write 5 rows, close (forces a flush of the active + // memtable). The manifest now records an SSTable and + // pins `replay_after_wal_entry_position` to the covered WAL entry. + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 5)]) + .await + .unwrap(); + writer_a.close().await.unwrap(); + } + + // Simulate an external compactor compacting the SSTable. + // into the base table: drain `sstables` to empty via a + // direct manifest commit. The cursor stays where the flush put it. + let manifest_store = ShardManifestStore::new(store.clone(), &base_path, shard_id, 2); + let pre = manifest_store.latest().await.unwrap().unwrap(); + assert!( + !pre.sstables.is_empty(), + "writer A's close() should have stamped an SSTable" + ); + let cursor_at_flush = pre.replay_after_wal_entry_position; + assert!( + cursor_at_flush >= 1, + "expected cursor to land on a 1-based WAL position after flush, got {cursor_at_flush}" + ); + // Bump the epoch (claim_epoch) so we can commit_update without + // being fenced; this also mirrors how a compactor process would + // hold its own writer claim. + let (compactor_epoch, _) = manifest_store.claim_epoch(pre.shard_spec_id).await.unwrap(); + manifest_store + .commit_update(compactor_epoch, |current| ShardManifest { + version: current.next_version(), + sstables: vec![], + ..current.clone() + }) + .await + .unwrap(); + let post = manifest_store.latest().await.unwrap().unwrap(); + assert!( + post.sstables.is_empty(), + "compactor drain should have left sstables empty" + ); + assert_eq!( + post.replay_after_wal_entry_position, cursor_at_flush, + "compactor must not touch the replay cursor" + ); + + // Writer B reopens. Pre-fix: replay saw sstables empty, + // restarted at WAL position 0, and re-inserted writer A's rows. + // Post-fix: replay starts at cursor + 1, finds no entry, and the + // memtable stays empty. + let writer_b = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![], + ) + .await + .unwrap(); + let stats = writer_b.memtable_stats().await.unwrap(); + assert_eq!( + stats.row_count, 0, + "memtable must not re-replay compacted WAL entries; got {} rows", + stats.row_count + ); + assert_eq!(stats.batch_count, 0); + writer_b.close().await.unwrap(); + } + + /// Replay aborts the open with a clear fence error if it encounters a + /// WAL entry written with an epoch strictly greater than ours. Simulate + /// the race where another writer wrote an entry with a higher epoch + /// between our `claim_epoch` and our replay by injecting a high-epoch + /// entry directly via `WalAppender::with_claimed_epoch` (which + /// bypasses `claim_epoch` and so does not bump the manifest). + #[tokio::test] + async fn test_memtable_replay_fenced_aborts_open() { + use crate::dataset::mem_wal::ShardManifestStore; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = schema_with_pk(); + let shard_id = Uuid::new_v4(); + + // Writer A: write one durable batch (claims epoch 1, writes entry at position 1). + { + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + memtable_config_with_pk(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_a + .put(vec![create_test_batch(&schema, 0, 1)]) + .await + .unwrap(); + // drop without close + } + + // Inject a WAL entry written with epoch 100 — far above whatever + // claim_epoch will hand the next opener. The manifest is not + // updated since we use `with_claimed_epoch` directly. + let manifest_store = Arc::new(ShardManifestStore::new( + store.clone(), + &base_path, + shard_id, + 2, + )); + let high_epoch_appender = WalAppender::with_claimed_epoch( + store.clone(), + base_path.clone(), + shard_id, + manifest_store, + 100, + // hint seed irrelevant; the real position counter is discovered + // lazily on the first append. + 0, + WalRetryConfig::default(), + ); + high_epoch_appender + .append(vec![create_test_batch(&schema, 999, 1)]) + .await + .unwrap(); + + // Writer B opens. claim_epoch returns 2 (manifest's writer_epoch + // was 1 before this open). Replay reads the injected entry, sees + // epoch 100 > 2, and aborts with a fence error. + let result = ShardWriter::open( + store, + base_path, + base_uri, + memtable_config_with_pk(shard_id), + schema, + vec![], + ) + .await; + let Err(err) = result else { + panic!("expected open to fail with fence error during replay"); + }; + // Assert the *typed* fence reason, not just the message: a regression + // reverting this to `Error::io` would still carry a message containing + // "fenced" and slip past a string check, but must not report a + // `FenceReason`. + assert_eq!( + err.fence_reason(), + Some(FenceReason::PeerClaimedEpoch), + "replay must abort with a typed peer-fence error, got: {err}" + ); + let msg = err.to_string(); + assert!( + msg.contains("WAL replay aborted") && msg.contains("fenced"), + "unexpected error: {msg}" + ); + } + + /// Regression: `wal_stats().next_wal_entry_position` must reflect the + /// post-recovery cursor immediately on reopen, not 0 until the first + /// append discovers the tip. Pre-fix the appender's hint was seeded at + /// 0 and only updated after the first successful append, so external + /// monitors saw 0 between open and first put on a shard with prior + /// entries. + #[tokio::test] + async fn test_wal_stats_seeded_from_manifest_on_reopen() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let shard_id = Uuid::new_v4(); + + // First writer creates a shard, writes one entry, closes. + let writer1 = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + wal_only_config(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer1 + .put(vec![create_test_batch(&schema, 0, 1)]) + .await + .unwrap(); + writer1.close().await.unwrap(); + + // Reopen: stats must reflect the post-recovery cursor immediately, + // before any put has happened on this writer. + let writer2 = ShardWriter::open( + store, + base_path, + base_uri, + wal_only_config(shard_id), + schema, + vec![], + ) + .await + .unwrap(); + let next = writer2.wal_stats().next_wal_entry_position; + assert!( + next >= 1, + "expected wal_stats to reflect post-recovery cursor (>= 1) on reopen, got {next}" + ); + + writer2.close().await.unwrap(); + } + + /// Regression test for the size-based trigger after a drain. + /// + /// Earlier the WAL-only size trigger used a monotonic counter + /// (`wal_flush_trigger_count`) which never reset across drains. After + /// the first crossing the counter was >= 1 and `pending_bytes / threshold` + /// could never grow past 1 (because pending_bytes resets on drain), so + /// the size trigger silently stopped firing. This test pushes batches + /// to cross the size threshold multiple times across drains and asserts + /// every crossing produces a WAL entry. + #[tokio::test] + async fn test_wal_only_size_trigger_fires_repeatedly() { + use crate::dataset::mem_wal::wal::WalTailer; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + let mut config = wal_only_config(Uuid::new_v4()); + let shard_id = config.shard_id; + // Non-durable so puts don't auto-flush. Time trigger off so only + // the size trigger drives flushes. + config.durable_write = false; + config.max_wal_flush_interval = None; + // Pick a tiny threshold so a single batch crosses it. + config.max_wal_buffer_size = 1; + + let writer = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri, + config, + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Three puts, each large enough to cross the (1-byte) threshold. + // Without the fix, only the first would trigger; the rest would + // sit in the queue until close(). + for i in 0..3 { + writer + .put(vec![create_test_batch(&schema, i * 10, 10)]) + .await + .unwrap(); + // Yield, then sleep, so the background flush handler can + // drain the trigger queue before the next push — otherwise + // multiple pending triggers can coalesce into a single drain. + // 50ms historically failed on slow Windows CI runners; 250ms + // gives a comfortable margin without making the suite slow. + tokio::task::yield_now().await; + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } + + writer.close().await.unwrap(); + + // Each put should have produced its own WAL entry — three crossings, + // three entries. Without the regression fix, all three batches end + // up in a single entry written by `close()`. + let tailer = WalTailer::new(store, base_path, shard_id); + let next = tailer.next_position().await.unwrap(); + assert!( + next >= 3, + "expected at least 3 WAL entries (one per crossing), got next_position = {next}" + ); + } + + /// Regression test for concurrent durable WAL-only puts on a fenced + /// writer. Earlier `flush_from_wal_only` did a destructive `drain()` + /// before calling `wal_appender.append`. If the append failed (e.g. + /// fence), the drained batches were dropped — the next concurrent put + /// would then see an empty pending queue and spuriously return Ok, + /// hiding the data loss. With the snapshot/commit fix, the failed flush + /// leaves the batches in the queue for retry, and — because the flush is + /// terminal — it poisons the writer, waking *both* parked durability + /// waiters with the typed fence error instead of either one hanging. + #[tokio::test] + async fn test_wal_only_fenced_concurrent_puts_do_not_silently_succeed() { + use std::sync::Arc; + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let shard_id = Uuid::new_v4(); + + // Writer A claims epoch 1, writes one entry (takes WAL position 1, + // caches its next-position as 2 internally). + let writer_a = Arc::new( + ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + wal_only_config(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(), + ); + writer_a + .put(vec![create_test_batch(&schema, 0, 1)]) + .await + .unwrap(); + + // Writer B claims epoch 2 and writes its own entry (takes WAL + // position 1). A is now fenced: A's next put will attempt WAL + // position 1 (its cached next), collide with B's entry, and + // surface a "Writer fenced" error from `check_fenced`. + let writer_b = ShardWriter::open( + store, + base_path, + base_uri, + wal_only_config(shard_id), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer_b + .put(vec![create_test_batch(&schema, 1, 1)]) + .await + .unwrap(); + + // Two concurrent durable puts on the (now-fenced) writer A. With + // the destructive-drain bug, the first flush would consume both + // pending batches into a failing append; the second flush would + // see an empty queue and return spurious success, silently losing + // the second put's data. Now both puts park on the durability + // watermark; the ticker's append fails with the fence, poisons the + // writer, and both waiters wake with the fence error — batches intact + // in the queue. + let a1 = writer_a.clone(); + let a2 = writer_a.clone(); + let schema1 = schema.clone(); + let schema2 = schema.clone(); + let h1 = tokio::spawn(async move { a1.put(vec![create_test_batch(&schema1, 2, 1)]).await }); + let h2 = tokio::spawn(async move { a2.put(vec![create_test_batch(&schema2, 3, 1)]).await }); + + let r1 = h1.await.unwrap(); + let r2 = h2.await.unwrap(); + + assert!( + r1.is_err() && r2.is_err(), + "expected both concurrent puts on a fenced writer to fail, got r1={r1:?} r2={r2:?}", + ); + + writer_b.close().await.unwrap(); + } + + #[tokio::test] + async fn test_wal_only_stats_no_memtable_flush() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + let writer = ShardWriter::open( + store, + base_path, + base_uri, + wal_only_config(Uuid::new_v4()), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 0, 1)]) + .await + .unwrap(); + + let stats_handle = writer.stats_handle(); + writer.close().await.unwrap(); + + let snapshot = stats_handle.snapshot(); + assert!(snapshot.put_count >= 1, "expected at least one put"); + assert!( + snapshot.wal_flush_count >= 1, + "expected at least one WAL flush" + ); + assert_eq!( + snapshot.memtable_flush_count, 0, + "WAL-only mode must never trigger a memtable flush" + ); + assert_eq!( + snapshot.index_update_count, 0, + "WAL-only mode must never trigger an index update" + ); + } + + #[tokio::test] + async fn test_memtable_stats_record_index_update() { + // MemTable mode with a BTree index: index application runs on its own + // task and must record an index-update stat. Regression for the stat + // silently reading zero after index apply moved off the WAL-flush path. + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_pk_test_schema(); + let index_configs = vec![MemIndexConfig::BTree(BTreeIndexConfig { + name: "id_idx".to_string(), + field_id: 0, + column: "id".to_string(), + })]; + + let writer = ShardWriter::open( + store, + base_path, + base_uri, + flush_test_config(Uuid::new_v4()), + schema.clone(), + index_configs, + ) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 0, 3)]) + .await + .unwrap(); + + let stats_handle = writer.stats_handle(); + // `close()` drains the index-apply task, so the apply is settled here. + writer.close().await.unwrap(); + + let snapshot = stats_handle.snapshot(); + assert!( + snapshot.index_update_count >= 1, + "the index apply must record an index-update stat, got {}", + snapshot.index_update_count + ); + assert_eq!( + snapshot.index_update_rows, 3, + "every indexed row must be counted exactly once, got {}", + snapshot.index_update_rows + ); + assert!( + snapshot.avg_index_update_latency().is_some(), + "a recorded index update must expose an average latency" + ); + } + + #[tokio::test] + async fn test_force_seal_active_and_wait_for_flush_drain() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + // Thresholds high enough that auto-flush won't fire; the seal is + // the only thing that should rotate the memtable. + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + let initial_gen = writer.memtable_stats().await.unwrap().generation; + let flushed_before = writer + .manifest() + .await + .unwrap() + .map(|m| m.sstables.len()) + .unwrap_or(0); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + let fence = writer.force_seal_active().await.unwrap(); + assert_eq!(fence.sealed_generation(), Some(initial_gen)); + fence.wait().await.unwrap(); + + let stats = writer.memtable_stats().await.unwrap(); + assert_eq!(stats.generation, initial_gen + 1); + assert_eq!(stats.batch_count, 0); + + let manifest = writer + .manifest() + .await + .unwrap() + .expect("manifest should exist after flush"); + assert_eq!(manifest.sstables.len(), flushed_before + 1); + + writer.close().await.unwrap(); + } + + /// A durable put returns only once its WAL flush landed, and the seal + /// fence resolves only once the sealed memtable reached L0 — so both + /// callbacks have fired by the time this asserts, without sleeping. + #[tokio::test] + async fn test_observer_sees_both_flush_kinds() { + #[derive(Debug, Default)] + struct CountingObserver { + wal_flushes: AtomicU64, + wal_bytes: AtomicU64, + memtable_flushes: AtomicU64, + memtable_rows: AtomicU64, + } + + impl WalObserver for CountingObserver { + fn on_wal_flush(&self, _duration: Duration, bytes: usize) { + self.wal_flushes.fetch_add(1, Ordering::Relaxed); + self.wal_bytes.fetch_add(bytes as u64, Ordering::Relaxed); + } + + fn on_memtable_flush(&self, _duration: Duration, rows: usize) { + self.memtable_flushes.fetch_add(1, Ordering::Relaxed); + self.memtable_rows.fetch_add(rows as u64, Ordering::Relaxed); + } + } + + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + let observer = Arc::new(CountingObserver::default()); + let sink: Arc = observer.clone(); + let config = ShardWriterConfig { + observer: Some(sink), + ..seal_fence_test_config(Uuid::new_v4()) + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + writer + .force_seal_active() + .await + .unwrap() + .wait() + .await + .unwrap(); + + assert!(observer.wal_flushes.load(Ordering::Relaxed) > 0); + assert!(observer.wal_bytes.load(Ordering::Relaxed) > 0); + assert_eq!(observer.memtable_flushes.load(Ordering::Relaxed), 1); + assert_eq!(observer.memtable_rows.load(Ordering::Relaxed), 10); + + writer.close().await.unwrap(); + } + + /// Durable writes so `put` returns only once the row is indexed and + /// WAL-durable. Both fence tests tear the background tasks down before + /// freezing, and a freeze still owing an index apply or a WAL append + /// would dispatch onto a closed channel and poison the writer. + fn seal_fence_test_config(shard_id: Uuid) -> ShardWriterConfig { + ShardWriterConfig { shard_id, - manifest_store, - 100, - // hint seed irrelevant; the real position counter is discovered - // lazily on the first append. - 0, - WalRetryConfig::default(), - ); - high_epoch_appender - .append(vec![create_test_batch(&schema, 999, 1)]) + shard_spec_id: 0, + durable_write: true, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() + } + } + + /// An empty active memtable does not mean every pre-call write is in + /// L0: a size/interval trigger swaps generation N for an empty N+1 + /// while N's flush is still in flight. The seal must fence that + /// generation anyway — reporting "nothing sealed, nothing to wait for" + /// lets a caller acknowledge a flush that has not happened. + #[tokio::test] + async fn test_force_seal_active_fences_pending_generation_when_active_is_empty() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + seal_fence_test_config(Uuid::new_v4()), + schema.clone(), + vec![], + ) + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - // Writer B opens. claim_epoch returns 2 (manifest's writer_epoch - // was 1 before this open). Replay reads the injected entry, sees - // epoch 100 > 2, and aborts with a fence error. - let result = ShardWriter::open( + // Stop the flush tasks, then freeze by hand: this is the post-rotation + // state held still — generation N frozen and unflushed, N+1 active and + // empty. + writer.task_executor.shutdown_all().await.unwrap(); + let pending_generation = match &writer.mode { + WriterMode::MemTable { + state, + writer_state, + .. + } => { + let mut state = state.write().await; + writer_state.freeze_memtable(&mut state).unwrap() - 1 + } + WriterMode::WalOnly { .. } => unreachable!("opened in memtable mode"), + }; + + let fence = writer.force_seal_active().await.unwrap(); + assert_eq!( + fence.sealed_generation(), + None, + "the active memtable was empty, so this seal froze nothing" + ); + assert!( + tokio::time::timeout(Duration::from_millis(200), fence.wait()) + .await + .is_err(), + "generation {pending_generation} is still awaiting flush; the fence must not be satisfied" + ); + } + + /// The fence is tied to each flush's own outcome, never to + /// `ShardManifest::current_generation`. The manifest advances to + /// `generation + 1` on every committed flush without checking for a gap, + /// so a later generation's success moves it past one that failed — a + /// watermark comparison would report durability that does not exist. + #[tokio::test] + async fn test_force_seal_active_fence_ignores_manifest_generation_advance() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let writer = ShardWriter::open( store, base_path, base_uri, - memtable_config_with_pk(shard_id), - schema, + seal_fence_test_config(Uuid::new_v4()), + schema.clone(), vec![], ) - .await; - let Err(err) = result else { - panic!("expected open to fail with fence error during replay"); + .await + .unwrap(); + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + + // No flush task, so the sealed generation never reaches L0. + writer.task_executor.shutdown_all().await.unwrap(); + let fence = writer.force_seal_active().await.unwrap(); + let sealed = fence + .sealed_generation() + .expect("the active memtable held rows"); + + // Advance the manifest past the sealed generation, as a later + // generation's successful flush would. + writer + .manifest_store + .commit_update(writer.epoch(), |current| ShardManifest { + version: current.next_version(), + current_generation: sealed + 2, + ..current.clone() + }) + .await + .unwrap(); + + assert!( + tokio::time::timeout(Duration::from_millis(200), fence.wait()) + .await + .is_err(), + "generation {sealed} never reached L0; a manifest advance past it must not satisfy its fence" + ); + } + + /// `abort` tears down the background flush tasks WITHOUT flushing — + /// buffered memtable rows are discarded, not sealed into an L0 + /// generation the way `close` would. Idempotent on a second call. + #[tokio::test] + async fn test_abort_discards_without_flushing_and_is_idempotent() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + + // Thresholds high enough that nothing auto-flushes; the rows stay + // in the active memtable until abort discards them. + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() }; - // Assert the *typed* fence reason, not just the message: a regression - // reverting this to `Error::io` would still carry a message containing - // "fenced" and slip past a string check, but must not report a - // `FenceReason`. + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + let flushed_before = writer + .manifest() + .await + .unwrap() + .map(|m| m.sstables.len()) + .unwrap_or(0); + + writer.abort().await.unwrap(); + + // No generation was sealed — contrast with `close`, which flushes + // the 10 buffered rows into a new L0 generation. + let flushed_after = writer + .manifest() + .await + .unwrap() + .map(|m| m.sstables.len()) + .unwrap_or(0); assert_eq!( - err.fence_reason(), - Some(FenceReason::PeerClaimedEpoch), - "replay must abort with a typed peer-fence error, got: {err}" + flushed_after, flushed_before, + "abort must not flush a new L0 generation" ); - let msg = err.to_string(); + + // Idempotent: re-cancels the already-cancelled token, joins an + // already-emptied task set. + writer.abort().await.unwrap(); + } + + /// On a successful flush commit the sealed generation's rows land in the + /// manifest immediately, but the in-memory handle is NOT dropped — it + /// lingers for `frozen_memtable_grace` (so in-flight as-of reads keep + /// batch-resolved membership), then is swept by the `SweepExpired` ticker. + #[tokio::test] + async fn test_frozen_retained_during_grace_then_swept() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + // Short grace so the sweep is observable without a slow test. + frozen_memtable_grace: Duration::from_millis(50), + ..Default::default() + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + let initial_gen = writer.memtable_stats().await.unwrap().generation; + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + writer.force_seal_active().await.unwrap(); + writer.wait_for_flush_drain().await.unwrap(); + + // Recorded in the manifest at commit time. + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); assert!( - msg.contains("WAL replay aborted") && msg.contains("fenced"), - "unexpected error: {msg}" + manifest + .sstables + .iter() + .any(|g| g.generation == initial_gen), + "SSTable must be recorded in the manifest" + ); + + // Still queryable in memory immediately after commit (within grace). + let refs = writer.in_memory_memtable_refs().await.unwrap(); + assert_eq!(refs.active.generation, initial_gen + 1); + assert!( + refs.frozen.iter().any(|f| f.generation == initial_gen), + "SSTable must stay queryable during the grace window" + ); + + // After the grace elapses (plus a sweep tick) the handle is evicted. + tokio::time::sleep(Duration::from_millis(250)).await; + let refs = writer.in_memory_memtable_refs().await.unwrap(); + assert!( + refs.frozen.is_empty(), + "frozen handle must be swept once the grace elapses" + ); + + writer.close().await.unwrap(); + } + + /// With zero grace (the default) a frozen handle is evicted synchronously on + /// flush commit — no sweep tick, no lingering window. + #[tokio::test] + async fn test_frozen_evicted_immediately_with_zero_grace() { + let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + let schema = create_test_schema(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + shard_spec_id: 0, + durable_write: false, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + frozen_memtable_grace: Duration::ZERO, + ..Default::default() + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + let initial_gen = writer.memtable_stats().await.unwrap().generation; + writer + .put(vec![create_test_batch(&schema, 0, 10)]) + .await + .unwrap(); + writer.force_seal_active().await.unwrap(); + writer.wait_for_flush_drain().await.unwrap(); + + // Rows are durably in the manifest... + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + assert!( + manifest + .sstables + .iter() + .any(|g| g.generation == initial_gen), + "SSTable must be recorded in the manifest" + ); + + // ...and the in-memory handle is already gone, no sweep tick needed. + let refs = writer.in_memory_memtable_refs().await.unwrap(); + assert!( + refs.frozen.is_empty(), + "frozen handle must be evicted on commit when grace is zero" ); + + writer.close().await.unwrap(); } - /// Regression: `wal_stats().next_wal_entry_position` must reflect the - /// post-recovery cursor immediately on reopen, not 0 until the first - /// append discovers the tip. Pre-fix the appender's hint was seeded at - /// 0 and only updated after the first successful append, so external - /// monitors saw 0 between open and first put on a shard with prior - /// entries. #[tokio::test] - async fn test_wal_stats_seeded_from_manifest_on_reopen() { + async fn test_close_propagates_frozen_memtable_flush_failure() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; - let schema = create_test_schema(); + let schema = schema_with_pk(); let shard_id = Uuid::new_v4(); - - // First writer creates a shard, writes one entry, closes. - let writer1 = ShardWriter::open( + let writer_a = ShardWriter::open( store.clone(), base_path.clone(), base_uri.clone(), - wal_only_config(shard_id), + memtable_config_with_pk(shard_id), schema.clone(), vec![], ) .await .unwrap(); - writer1 - .put(vec![create_test_batch(&schema, 0, 1)]) + writer_a + .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - writer1.close().await.unwrap(); - // Reopen: stats must reflect the post-recovery cursor immediately, - // before any put has happened on this writer. - let writer2 = ShardWriter::open( + let writer_b = ShardWriter::open( store, base_path, base_uri, - wal_only_config(shard_id), + memtable_config_with_pk(shard_id), schema, vec![], ) .await .unwrap(); - let next = writer2.wal_stats().next_wal_entry_position; + assert!(writer_b.epoch() > writer_a.epoch()); + + let error = writer_a + .close() + .await + .expect_err("close must propagate the fenced MemTable flush"); assert!( - next >= 1, - "expected wal_stats to reflect post-recovery cursor (>= 1) on reopen, got {next}" + matches!(error, Error::IO { .. }), + "unexpected error: {error}" + ); + assert!( + error.to_string().contains("Writer fenced"), + "unexpected error: {error}" ); - writer2.close().await.unwrap(); + writer_b.close().await.unwrap(); } - /// Regression test for the size-based trigger after a drain. - /// - /// Earlier the WAL-only size trigger used a monotonic counter - /// (`wal_flush_trigger_count`) which never reset across drains. After - /// the first crossing the counter was >= 1 and `pending_bytes / threshold` - /// could never grow past 1 (because pending_bytes resets on drain), so - /// the size trigger silently stopped firing. This test pushes batches - /// to cross the size threshold multiple times across drains and asserts - /// every crossing produces a WAL entry. + /// Regression: a transient flush failure must NOT reopen the + /// concurrent-read-vs-flush hole. The sealed generation stays in the + /// queryable set (rows intact) until a later flush or WAL replay. + /// Failure is induced deterministically by fencing the writer with a + /// successor before the seal, so the flush's `check_fenced` rejects it. #[tokio::test] - async fn test_wal_only_size_trigger_fires_repeatedly() { - use crate::dataset::mem_wal::wal::WalTailer; - + async fn test_frozen_retained_after_failed_flush() { let (store, base_path, base_uri, _temp_dir) = create_local_store().await; let schema = create_test_schema(); + let shard_id = Uuid::new_v4(); - let mut config = wal_only_config(Uuid::new_v4()); - let shard_id = config.shard_id; - // Non-durable so puts don't auto-flush. Time trigger off so only - // the size trigger drives flushes. - config.durable_write = false; - config.max_wal_flush_interval = None; - // Pick a tiny threshold so a single batch crosses it. - config.max_wal_buffer_size = 1; - - let writer = ShardWriter::open( + let writer_a = ShardWriter::open( store.clone(), base_path.clone(), - base_uri, - config, + base_uri.clone(), + memtable_config_with_pk(shard_id), schema.clone(), vec![], ) .await .unwrap(); - // Three puts, each large enough to cross the (1-byte) threshold. - // Without the fix, only the first would trigger; the rest would - // sit in the queue until close(). - for i in 0..3 { - writer - .put(vec![create_test_batch(&schema, i * 10, 10)]) - .await - .unwrap(); - // Yield, then sleep, so the background flush handler can - // drain the trigger queue before the next push — otherwise - // multiple pending triggers can coalesce into a single drain. - // 50ms historically failed on slow Windows CI runners; 250ms - // gives a comfortable margin without making the suite slow. - tokio::task::yield_now().await; - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } - - writer.close().await.unwrap(); - - // Each put should have produced its own WAL entry — three crossings, - // three entries. Without the regression fix, all three batches end - // up in a single entry written by `close()`. - let tailer = WalTailer::new(store, base_path, shard_id); - let next = tailer.next_position().await.unwrap(); - assert!( - next >= 3, - "expected at least 3 WAL entries (one per crossing), got next_position = {next}" - ); - } - - /// Regression test for concurrent durable WAL-only puts on a fenced - /// writer. Earlier `flush_from_wal_only` did a destructive `drain()` - /// before calling `wal_appender.append`. If the append failed (e.g. - /// fence), the drained batches were dropped — the next concurrent put - /// would then see an empty pending queue and spuriously return Ok, - /// hiding the data loss. With the snapshot/commit fix, the failed flush - /// leaves the batches in the queue, and the concurrent put gets a clean - /// fence error too (when its own flush attempts the same WAL position). - #[tokio::test] - async fn test_wal_only_fenced_concurrent_puts_do_not_silently_succeed() { - use std::sync::Arc; - - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; - let schema = create_test_schema(); - let shard_id = Uuid::new_v4(); - - // Writer A claims epoch 1, writes one entry (takes WAL position 1, - // caches its next-position as 2 internally). - let writer_a = Arc::new( - ShardWriter::open( - store.clone(), - base_path.clone(), - base_uri.clone(), - wal_only_config(shard_id), - schema.clone(), - vec![], - ) - .await - .unwrap(), - ); + let initial_gen = writer_a.memtable_stats().await.unwrap().generation; writer_a - .put(vec![create_test_batch(&schema, 0, 1)]) + .put(vec![create_test_batch(&schema, 0, 10)]) .await .unwrap(); - // Writer B claims epoch 2 and writes its own entry (takes WAL - // position 1). A is now fenced: A's next put will attempt WAL - // position 1 (its cached next), collide with B's entry, and - // surface a "Writer fenced" error from `check_fenced`. + // Successor claims a higher epoch, fencing A. let writer_b = ShardWriter::open( store, base_path, base_uri, - wal_only_config(shard_id), + memtable_config_with_pk(shard_id), schema.clone(), vec![], ) .await .unwrap(); - writer_b - .put(vec![create_test_batch(&schema, 1, 1)]) - .await - .unwrap(); + assert!(writer_b.epoch() > writer_a.epoch()); - // Two concurrent durable puts on the (now-fenced) writer A. With - // the destructive-drain bug, the first flush would consume both - // pending batches into a failing append; the second flush would - // see an empty queue and return spurious success, silently losing - // the second put's data. With the snapshot/commit fix, the failed - // append leaves both batches in the queue and the second flush - // also fails with the fence error. - let a1 = writer_a.clone(); - let a2 = writer_a.clone(); - let schema1 = schema.clone(); - let schema2 = schema.clone(); - let h1 = tokio::spawn(async move { a1.put(vec![create_test_batch(&schema1, 2, 1)]).await }); - let h2 = tokio::spawn(async move { a2.put(vec![create_test_batch(&schema2, 3, 1)]).await }); + // `force_seal_active` would reject up-front on a fenced writer; + // freeze directly so the failure surfaces at flush-commit time — + // exactly the freeze/flush race the fix guards. + match &writer_a.mode { + WriterMode::MemTable { + state, + writer_state, + .. + } => { + let mut st = state.write().await; + writer_state.freeze_memtable(&mut st).unwrap(); + } + WriterMode::WalOnly { .. } => unreachable!("opened in memtable mode"), + } - let r1 = h1.await.unwrap(); - let r2 = h2.await.unwrap(); + // The fenced flush fails; the drain surfaces that error. + assert!( + writer_a.wait_for_flush_drain().await.is_err(), + "fenced flush should fail the drain" + ); + // The hole did not reopen: the sealed generation is still queryable + // with its rows, alongside the new (empty) active generation. + let refs = writer_a.in_memory_memtable_refs().await.unwrap(); + assert_eq!(refs.frozen.len(), 1, "sealed generation must be retained"); + assert_eq!(refs.frozen[0].generation, initial_gen); assert!( - r1.is_err() && r2.is_err(), - "expected both concurrent puts on a fenced writer to fail, got r1={r1:?} r2={r2:?}", + !refs.frozen[0].batch_store.is_empty(), + "retained sealed memtable must still hold its rows" + ); + assert_eq!(refs.active.generation, initial_gen + 1); + + // Nor did it vanish from the accounting. A failed flush leaves its + // memtable un-stamped, so it stays in the owed-to-flush set and keeps + // metering — those bytes are resident and only another flush reclaims + // them. This is also what an operator reads to decide whether to evict. + let stats = writer_a.memtable_stats().await.unwrap(); + assert_eq!(stats.frozen_count, 1); + let frozen_bytes = writer_a.memory().frozen_bytes(); + assert!( + frozen_bytes >= refs.frozen[0].batch_store.row_bytes(), + "a failed flush must keep owing its resident bytes, got {frozen_bytes}" + ); + + // Charging those bytes must not become a trap. Nothing retries the + // failed generation, and its watcher came off the queue when the flush + // reported, so there is no event left that would drain the pool — while + // the valve itself is what keeps the puts that could seal a new + // generation from arriving. Waiting here would never end, so the + // controller refuses instead. + assert!( + matches!(writer_a.memory().drain(), Drain::Stalled), + "a failed flush leaves nothing outstanding to wait on" + ); + let controller = LocalBackpressureController::new(&ShardWriterConfig { + max_unflushed_memtable_bytes: writer_a.memory().unflushed_bytes(), + ..Default::default() + }); + let refused = tokio::time::timeout( + STALL_GRACE * 10, + controller.maybe_apply_backpressure(writer_a.memory()), + ) + .await + .expect("a shard nothing can drain must refuse, not park the writer"); + assert!( + refused.is_err_and(|e| e.is_backpressure()), + "the refusal must be the retryable backpressure signal" ); writer_b.close().await.unwrap(); } + /// A one-row slice pins its whole parent, so a memtable can hold megabytes + /// while its row window reads a few dozen bytes. With only the row arm the + /// shard crossed its ceiling with nothing sealed and no seal reachable — the + /// valve then found `Drain::Stalled` and refused every put, permanently. + /// The resident arm is what makes that shard drain instead. #[tokio::test] - async fn test_wal_only_stats_no_memtable_flush() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + async fn test_pinned_parents_seal_before_the_ceiling_traps_the_writer() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + DataType::Int32, + false, + )])); + + let chunk = 1_000_000; // ~4MB parent per put + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + // Far above anything the one-row windows will ever sum to, so the row + // arm cannot be what seals here. + max_memtable_size: 1024 * 1024, + // Never filled, so the capacity arm cannot be it either. + max_memtable_batches: 1024, + max_unflushed_memtable_bytes: 8 * 1024 * 1024, + ..Default::default() + }; + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) + .await + .unwrap(); + + for i in 0..6i32 { + let parent = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from( + (0..chunk).map(|v| v + i).collect::>(), + ))], + ) + .unwrap(); + tokio::time::timeout(STALL_GRACE * 10, writer.put(vec![parent.slice(0, 1)])) + .await + .expect("a shard the resident arm can seal must not park forever") + .unwrap_or_else(|e| { + panic!("put {i} was refused, so the ceiling is still a trap: {e}") + }); + } + + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.generation > 0, + "the pinned parents must have sealed a generation; row bytes never \ + came close to max_memtable_size" + ); + assert!( + writer.memory().row_bytes() < 1024, + "the row window must still be tiny — otherwise the row arm did the \ + sealing and this proves nothing" + ); + + writer.close().await.unwrap(); + } + + /// Index memory is not charged to `max_memtable_size`, so it is the resident + /// arm that has to notice a memtable whose indexes rather than its rows are + /// filling the ceiling. + #[tokio::test] + async fn test_resident_arm_seals_on_index_memory() { let schema = create_test_schema(); + let mut memtable = + MemTable::with_capacity(schema.clone(), 1, vec![], CacheConfig::default(), 64).unwrap(); + memtable + .insert(create_test_batch(&schema, 0, 50)) + .await + .unwrap(); - let writer = ShardWriter::open( - store, - base_path, - base_uri, - wal_only_config(Uuid::new_v4()), - schema.clone(), - vec![], - ) - .await - .unwrap(); - writer - .put(vec![create_test_batch(&schema, 0, 1)]) + let resident = memtable_resident_bytes(&memtable); + let rows = memtable.batch_store().row_bytes(); + assert!( + resident > rows, + "the fixture needs non-row memory to be measuring anything: \ + resident {resident} vs rows {rows}" + ); + + // Row arm way out of range; only the resident arm can fire. + assert!( + memtable_reached_flush_threshold(&memtable, usize::MAX, usize::MAX, resident, 1, 1), + "resident bytes at the ceiling must seal" + ); + assert!( + !memtable_reached_flush_threshold( + &memtable, + usize::MAX, + usize::MAX, + resident + 1, + 1, + 1 + ), + "and must not seal below it" + ); + } + + /// The row arm answers for the rows about to arrive, not the rows already + /// inserted: the cap is a hard index capacity. + #[tokio::test] + async fn test_row_arm_seals_on_max_memtable_rows() { + let schema = create_test_schema(); + let mut memtable = + MemTable::with_capacity(schema.clone(), 1, vec![], CacheConfig::default(), 64).unwrap(); + memtable + .insert(create_test_batch(&schema, 0, 50)) .await .unwrap(); - let stats_handle = writer.stats_handle(); - writer.close().await.unwrap(); - - let snapshot = stats_handle.snapshot(); - assert!(snapshot.put_count >= 1, "expected at least one put"); + // Byte and resident arms out of range; only the row arm can fire. + let row_arm = |cap, incoming| { + memtable_reached_flush_threshold(&memtable, usize::MAX, cap, usize::MAX, 1, incoming) + }; + assert!(!row_arm(50, 0), "50 rows under a cap of 50 must not seal"); + assert!(row_arm(50, 1), "no room for one more row must seal"); assert!( - snapshot.wal_flush_count >= 1, - "expected at least one WAL flush" + !row_arm(60, 10), + "a put that exactly fills the cap must not seal" ); - assert_eq!( - snapshot.memtable_flush_count, 0, - "WAL-only mode must never trigger a memtable flush" - ); - assert_eq!( - snapshot.index_update_count, 0, - "WAL-only mode must never trigger an index update" + assert!( + row_arm(60, 11), + "a put that would overflow the cap must seal" ); } + /// A memtable stays within `max_memtable_rows` with the byte and batch arms + /// far out of reach, so only the row arm can seal it. #[tokio::test] - async fn test_force_seal_active_and_wait_for_flush_drain() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + async fn test_put_seals_on_max_memtable_rows() { + let (store, base_path, base_uri, _t) = create_local_store().await; let schema = create_test_schema(); - - // Thresholds high enough that auto-flush won't fire; the seal is - // the only thing that should rotate the memtable. + let cap = 64; let config = ShardWriterConfig { shard_id: Uuid::new_v4(), - shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, - max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, + max_memtable_rows: cap, + // Both far out of reach, so only the row arm can seal. max_memtable_size: 64 * 1024 * 1024, - manifest_scan_batch_size: 2, + max_memtable_batches: 8_000, ..Default::default() }; - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) .await .unwrap(); - let initial_gen = writer.memtable_stats().await.unwrap().generation; - let flushed_before = writer - .manifest() - .await - .unwrap() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); - - writer - .put(vec![create_test_batch(&schema, 0, 10)]) - .await - .unwrap(); - writer.force_seal_active().await.unwrap(); - writer.wait_for_flush_drain().await.unwrap(); - - let stats = writer.memtable_stats().await.unwrap(); - assert_eq!(stats.generation, initial_gen + 1); - assert_eq!(stats.batch_count, 0); + for round in 0..10i32 { + writer + .put(vec![create_test_batch(&schema, round * 10, 10)]) + .await + .unwrap(); + let stats = writer.memtable_stats().await.unwrap(); + assert!( + stats.row_count <= cap, + "the active memtable holds {} rows, past the cap of {cap}", + stats.row_count + ); + } - let manifest = writer - .manifest() - .await - .unwrap() - .expect("manifest should exist after flush"); - assert_eq!(manifest.flushed_generations.len(), flushed_before + 1); + assert!( + writer.memtable_stats().await.unwrap().generation > 1, + "100 rows under a cap of {cap} must have rotated at least once" + ); writer.close().await.unwrap(); } - /// `abort` tears down the background flush tasks WITHOUT flushing — - /// buffered memtable rows are discarded, not sealed into an L0 - /// generation the way `close` would. Idempotent on a second call. + /// A put is never split across memtables, so one larger than the cap is + /// rejected as invalid input rather than overflowing a fresh memtable. #[tokio::test] - async fn test_abort_discards_without_flushing_and_is_idempotent() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; + async fn test_put_rejects_more_rows_than_a_memtable_holds() { + let (store, base_path, base_uri, _t) = create_local_store().await; let schema = create_test_schema(); - - // Thresholds high enough that nothing auto-flushes; the rows stay - // in the active memtable until abort discards them. let config = ShardWriterConfig { shard_id: Uuid::new_v4(), - shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, - max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, - max_memtable_size: 64 * 1024 * 1024, - manifest_scan_batch_size: 2, + max_memtable_rows: 8, ..Default::default() }; - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) .await .unwrap(); - writer - .put(vec![create_test_batch(&schema, 0, 10)]) - .await - .unwrap(); - let flushed_before = writer - .manifest() - .await - .unwrap() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); - - writer.abort().await.unwrap(); - - // No generation was sealed — contrast with `close`, which flushes - // the 10 buffered rows into a new L0 generation. - let flushed_after = writer - .manifest() + // Split across two batches: the cap is on the put, not on one batch. + let err = writer + .put(vec![ + create_test_batch(&schema, 0, 5), + create_test_batch(&schema, 5, 4), + ]) .await - .unwrap() - .map(|m| m.flushed_generations.len()) - .unwrap_or(0); - assert_eq!( - flushed_after, flushed_before, - "abort must not flush a new L0 generation" + .expect_err("a put of 9 rows under a cap of 8 must be rejected"); + assert!( + matches!(err, Error::InvalidInput { .. }), + "an oversized put is caller error, not a writer fault: {err}" + ); + assert!( + err.to_string().contains("max_memtable_rows=8"), + "the error must name the knob and its value, got: {err}" ); - // Idempotent: re-cancels the already-cancelled token, joins an - // already-emptied task set. - writer.abort().await.unwrap(); + // Exactly the cap still goes through, and the writer is unharmed. + writer + .put(vec![create_test_batch(&schema, 0, 8)]) + .await + .unwrap(); + writer.close().await.unwrap(); } - /// On a successful flush commit the sealed generation's rows land in the - /// manifest immediately, but the in-memory handle is NOT dropped — it - /// lingers for `frozen_memtable_grace` (so in-flight as-of reads keep - /// batch-resolved membership), then is swept by the `SweepExpired` ticker. + /// An HNSW graph is sized to `max_memtable_rows`, so a shard that writes past + /// the cap has to keep sealing for the graph to never see a row it cannot + /// hold. #[tokio::test] - async fn test_frozen_retained_during_grace_then_swept() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; - let schema = create_test_schema(); + async fn test_hnsw_index_survives_a_shard_that_outgrows_the_row_cap() { + use lance_arrow::FixedSizeListArrayExt; + + let (store, base_path, base_uri, _t) = create_local_store().await; + let dim = 8; + let schema = hnsw_schema(dim); + let cap = 64; let config = ShardWriterConfig { shard_id: Uuid::new_v4(), - shard_spec_id: 0, durable_write: false, - sync_indexed_write: false, - max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, + max_memtable_rows: cap, max_memtable_size: 64 * 1024 * 1024, - manifest_scan_batch_size: 2, - // Short grace so the sweep is observable without a slow test. - frozen_memtable_grace: Duration::from_secs(1), + max_memtable_batches: 8_000, ..Default::default() }; - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) - .await + let writer = ShardWriter::open( + store, + base_path, + base_uri, + config, + schema.clone(), + hnsw_configs(), + ) + .await + .unwrap(); + + for round in 0..20i32 { + let ids: Vec = (0..10).map(|v| v + round * 10).collect(); + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from( + (0..10 * dim as usize) + .map(|v| v as f32 * 0.01) + .collect::>(), + ), + dim, + ) + .unwrap(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(ids)), Arc::new(vectors)], + ) .unwrap(); + writer + .put(vec![batch]) + .await + .unwrap_or_else(|e| panic!("put {round} was refused: {e}")); + } - let initial_gen = writer.memtable_stats().await.unwrap().generation; - writer - .put(vec![create_test_batch(&schema, 0, 10)]) + // The index apply runs outside the put, so a poisoned writer surfaces here + // even when every put returned Ok. + writer.close().await.unwrap(); + } + + /// The post-insert seal check runs inside the writer lock; the index apply + /// that follows it runs outside. So a put's index growth is invisible to the + /// only check that put makes, and the *next* put is gated by the valve before + /// it can insert and check again — leaving a shard over its ceiling with + /// nothing sealed and every write refused. Replay reaches the same state by + /// building its final memtable's indexes after its last check. + #[tokio::test] + async fn test_index_growth_after_the_seal_check_still_drains() { + let (store, base_path, base_uri, _t) = create_local_store().await; + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("text", DataType::Utf8, true), + ])); + // Keys long enough to spill out of the skiplist nodes, so the index heap + // grows with the column instead of staying a fixed reservation. + let btree = vec![MemIndexConfig::BTree(BTreeIndexConfig { + name: "text_idx".to_string(), + field_id: 1, + column: "text".to_string(), + })]; + + let rows = 2_000usize; + let width = 512usize; + let payload = rows * width; + let ceiling = payload + payload / 2; + + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + // As high as the open-time check allows, keeping the row arm out of + // reach of the rows below so only the resident arm can seal. + max_memtable_size: ceiling - 64 * 1024, + max_memtable_batches: 1024, + // Above the rows alone, below rows plus the index heap they build. + max_unflushed_memtable_bytes: ceiling, + ..Default::default() + }; + + let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), btree) .await .unwrap(); - writer.force_seal_active().await.unwrap(); - writer.wait_for_flush_drain().await.unwrap(); - // Recorded in the manifest at commit time. - let manifest = writer.manifest().await.unwrap().expect("manifest exists"); - assert!( - manifest - .flushed_generations + for i in 0..3i32 { + let ids: Vec = (0..rows as i32).map(|v| v + i * rows as i32).collect(); + let texts: Vec = ids .iter() - .any(|g| g.generation == initial_gen), - "flushed generation must be recorded in the manifest" - ); + .map(|v| format!("{v:07}{}", "z".repeat(width - 7))) + .collect(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(ids)), + Arc::new(StringArray::from( + texts.iter().map(|t| Some(t.as_str())).collect::>(), + )), + ], + ) + .unwrap(); - // Still queryable in memory immediately after commit (within grace). - let refs = writer.in_memory_memtable_refs().await.unwrap(); - assert_eq!(refs.active.generation, initial_gen + 1); - assert!( - refs.frozen.iter().any(|f| f.generation == initial_gen), - "flushed generation must stay queryable during the grace window" - ); + tokio::time::timeout(STALL_GRACE * 10, writer.put(vec![batch])) + .await + .expect("a shard with a sealable memtable must not park forever") + .unwrap_or_else(|e| { + panic!("put {i} was refused; index growth outran the seal check: {e}") + }); + } - // After the grace elapses (plus a sweep tick) the handle is evicted. - tokio::time::sleep(Duration::from_millis(1_500)).await; - let refs = writer.in_memory_memtable_refs().await.unwrap(); assert!( - refs.frozen.is_empty(), - "frozen handle must be swept once the grace elapses" + writer.memory().index_bytes() > writer.memory().row_bytes(), + "the fixture must be index-dominated, or it is not testing this path" ); writer.close().await.unwrap(); } - /// With zero grace (the default) a frozen handle is evicted synchronously on - /// flush commit — no sweep tick, no lingering window. + /// `max_memtable_size` is the headroom the reservation check reserves for + /// rows. At zero it reserves none, so a fresh memtable's index reservation + /// could equal the ceiling exactly — admissible by the check, yet over budget + /// before its first row, with an empty memtable offering nothing to seal. The + /// writer refused its first write and never recovered. #[tokio::test] - async fn test_frozen_evicted_immediately_with_zero_grace() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; - let schema = create_test_schema(); + async fn test_open_rejects_a_zero_row_headroom() { + let sizing = ShardWriterConfig { + max_memtable_rows: 2_000, + ..Default::default() + }; + let reserved = reserved_index_bytes(&sizing, &hnsw_configs()); + let (store, base_path, base_uri, _t) = create_local_store().await; let config = ShardWriterConfig { shard_id: Uuid::new_v4(), - shard_spec_id: 0, - durable_write: false, - sync_indexed_write: false, - max_wal_buffer_size: 64 * 1024 * 1024, - max_wal_flush_interval: None, - max_memtable_size: 64 * 1024 * 1024, - manifest_scan_batch_size: 2, - frozen_memtable_grace: Duration::ZERO, + max_memtable_rows: sizing.max_memtable_rows, + max_memtable_size: 0, + // Exactly the fresh reservation: the boundary the old `>` admitted. + max_unflushed_memtable_bytes: reserved, ..Default::default() }; - let writer = ShardWriter::open(store, base_path, base_uri, config, schema.clone(), vec![]) - .await - .unwrap(); - let initial_gen = writer.memtable_stats().await.unwrap().generation; - writer - .put(vec![create_test_batch(&schema, 0, 10)]) - .await - .unwrap(); - writer.force_seal_active().await.unwrap(); - writer.wait_for_flush_drain().await.unwrap(); + let err = ShardWriter::open( + store, + base_path, + base_uri, + config, + hnsw_schema(8), + hnsw_configs(), + ) + .await + .err() + .expect("a zero row headroom must be rejected at open"); - // Rows are durably in the manifest... - let manifest = writer.manifest().await.unwrap().expect("manifest exists"); assert!( - manifest - .flushed_generations - .iter() - .any(|g| g.generation == initial_gen), - "flushed generation must be recorded in the manifest" + err.to_string() + .contains("max_memtable_size must be greater than zero"), + "the error must name the knob, got: {err}" ); + } - // ...and the in-memory handle is already gone, no sweep tick needed. - let refs = writer.in_memory_memtable_refs().await.unwrap(); + /// Schema for the reservation tests: `vector` is field 1, the id 0 is `id`. + fn hnsw_schema(dim: i32) -> Arc { + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim), + true, + ), + ])) + } + + fn hnsw_configs() -> Vec { + vec![MemIndexConfig::Hnsw(Box::new(HnswIndexConfig::new( + "vec_idx".to_string(), + 1, + "vector".to_string(), + lance_linalg::distance::DistanceType::L2, + )))] + } + + /// What the configured indexes owe before a single row arrives — the figure + /// `open` validates against, computed the way the memtable will build them. + fn reserved_index_bytes(config: &ShardWriterConfig, configs: &[MemIndexConfig]) -> usize { + IndexStore::from_configs( + configs, + config.max_memtable_rows, + config.max_memtable_batches, + ) + .unwrap() + .resident_bytes() + + super::super::memtable::pk_bloom_filter_bytes() + } + + /// An HNSW graph is charged from `max_memtable_rows` before the first + /// insert, while only row bytes seal a memtable. Sized past the ceiling it + /// would put the shard over budget at zero rows with nothing to seal and so + /// nothing to flush — every put stalling, then failing as `Backpressure`, + /// which means "retry later" and never comes true. That is a config error, + /// so it has to land at `open`, not on put #1. + #[tokio::test] + async fn test_open_rejects_indexes_that_cannot_fit_under_the_ceiling() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + max_memtable_rows: 100_000, + max_memtable_size: 1024 * 1024, + max_unflushed_memtable_bytes: 1024 * 1024, + ..Default::default() + }; + let reserved = reserved_index_bytes(&config, &hnsw_configs()); + assert!( + reserved > config.max_unflushed_memtable_bytes, + "the fixture must actually over-subscribe the ceiling, got {reserved}" + ); + + let err = ShardWriter::open( + store, + base_path, + base_uri, + config, + hnsw_schema(32), + hnsw_configs(), + ) + .await + .err() + .expect("an over-subscribed ceiling must be rejected at open"); + + let message = err.to_string(); + for fragment in [ + "in-memory indexes reserve", + "max_unflushed_memtable_bytes", + "stalling every write", + ] { + assert!( + message.contains(fragment), + "the error must name {fragment}, got: {message}" + ); + } assert!( - refs.frozen.is_empty(), - "frozen handle must be evicted on commit when grace is zero" + !err.is_backpressure(), + "a config error must not masquerade as the retryable busy signal" ); - - writer.close().await.unwrap(); } - /// Regression: a transient flush failure must NOT reopen the - /// concurrent-read-vs-flush hole. The sealed generation stays in the - /// queryable set (rows intact) until a later flush or WAL replay. - /// Failure is induced deterministically by fencing the writer with a - /// successor before the seal, so the flush's `check_fenced` rejects it. + /// The other side of the gate: a ceiling with room for the reservation *and* + /// a full memtable of rows on top opens, and keeps taking writes across a + /// seal — the frozen generation gives the valve a flush to park on, so the + /// wait ends instead of refusing. #[tokio::test] - async fn test_frozen_retained_after_failed_flush() { - let (store, base_path, base_uri, _temp_dir) = create_local_store().await; - let schema = create_test_schema(); - let shard_id = Uuid::new_v4(); - - let writer_a = ShardWriter::open( - store.clone(), - base_path.clone(), - base_uri.clone(), - memtable_config_with_pk(shard_id), - schema.clone(), - vec![], - ) - .await - .unwrap(); + async fn test_writer_with_indexes_under_the_ceiling_keeps_accepting_writes() { + let (store, base_path, base_uri, _temp) = create_local_store().await; + let dim = 8; + let sizing = ShardWriterConfig { + max_memtable_rows: 2_000, + ..Default::default() + }; + let reserved = reserved_index_bytes(&sizing, &hnsw_configs()); + let max_memtable_size = 4 * 1024; - let initial_gen = writer_a.memtable_stats().await.unwrap().generation; - writer_a - .put(vec![create_test_batch(&schema, 0, 10)]) - .await - .unwrap(); + let config = ShardWriterConfig { + shard_id: Uuid::new_v4(), + max_memtable_rows: sizing.max_memtable_rows, + max_memtable_size, + // Two generations' worth, so a seal does not immediately re-park the + // writer on a ceiling it cannot clear. + max_unflushed_memtable_bytes: 2 * (reserved + max_memtable_size), + ..Default::default() + }; - // Successor claims a higher epoch, fencing A. - let writer_b = ShardWriter::open( + let schema = hnsw_schema(dim); + let writer = ShardWriter::open( store, base_path, base_uri, - memtable_config_with_pk(shard_id), + config, schema.clone(), - vec![], + hnsw_configs(), ) .await - .unwrap(); - assert!(writer_b.epoch() > writer_a.epoch()); + .expect("a reservation that leaves room under the ceiling must open"); + + // Enough rows to carry row bytes past `max_memtable_size` several times + // over, so the run spans seals rather than sitting in one memtable. + for round in 0..8i32 { + let rows = 64; + let ids: Vec = (0..rows).map(|i| round * rows + i).collect(); + let values: Vec = (0..rows * dim).map(|i| i as f32).collect(); + let vectors = FixedSizeListArray::try_new( + Arc::new(Field::new("item", DataType::Float32, true)), + dim, + Arc::new(Float32Array::from(values)), + None, + ) + .unwrap(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(ids)), Arc::new(vectors)], + ) + .unwrap(); - // `force_seal_active` would reject up-front on a fenced writer; - // freeze directly so the failure surfaces at flush-commit time — - // exactly the freeze/flush race the fix guards. - match &writer_a.mode { - WriterMode::MemTable { - state, - writer_state, - .. - } => { - let mut st = state.write().await; - writer_state.freeze_memtable(&mut st).unwrap(); - } - WriterMode::WalOnly { .. } => unreachable!("opened in memtable mode"), + tokio::time::timeout(STALL_GRACE * 10, writer.put(vec![batch])) + .await + .expect("a drainable shard must not park the writer indefinitely") + .unwrap_or_else(|e| panic!("put in round {round} was refused: {e}")); } - // The fenced flush fails; the drain surfaces that error. - assert!( - writer_a.wait_for_flush_drain().await.is_err(), - "fenced flush should fail the drain" - ); - - // The hole did not reopen: the sealed generation is still queryable - // with its rows, alongside the new (empty) active generation. - let refs = writer_a.in_memory_memtable_refs().await.unwrap(); - assert_eq!(refs.frozen.len(), 1, "sealed generation must be retained"); - assert_eq!(refs.frozen[0].generation, initial_gen); + // Without a seal this would only prove one memtable fits, which is not + // the case that stalls. assert!( - !refs.frozen[0].batch_store.is_empty(), - "retained sealed memtable must still hold its rows" + writer.memtable_stats().await.unwrap().generation > 0, + "the run must cross a seal for the drain path to have been exercised" ); - assert_eq!(refs.active.generation, initial_gen + 1); - writer_b.close().await.unwrap(); + writer.close().await.unwrap(); } } @@ -5786,9 +9823,7 @@ mod shard_writer_tests { Some("100") ); // Every tunable field is present. - assert!(defaults.contains_key("sync_indexed_write")); assert!(defaults.contains_key("enable_memtable")); - assert!(defaults.contains_key("async_index_interval_ms")); // add_writer_config_default records arbitrary keys. assert_eq!( defaults.get("custom_knob").map(String::as_str), @@ -5811,7 +9846,12 @@ mod shard_writer_tests { let vector_dim = 32; let schema = create_test_schema(vector_dim); - let uri = format!("memory://test_multi_segment_index_{}", Uuid::new_v4()); + // The generation flusher reopens by URI, so this independent open must + // resolve to the same in-memory backend. The unique authority isolates the test. + let uri = format!( + "shared-memory://multi-segment-index-{}/", + Uuid::new_v4().simple() + ); // Initial fragment + an IVF vector index covering it. let initial = create_test_batch(&schema, 0, 256, vector_dim); @@ -5949,7 +9989,7 @@ mod shard_writer_tests { // The tombstone-only generation still flushed (data without an HNSW index). let manifest = writer.manifest().await.unwrap().expect("manifest exists"); assert_eq!( - manifest.flushed_generations.len(), + manifest.sstables.len(), 1, "the all-tombstone generation must still flush" ); @@ -5995,9 +10035,7 @@ mod shard_writer_tests { .expect("Failed to initialize MemWAL"); let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(true) - .with_sync_indexed_write(true); + let config = ShardWriterConfig::new(shard_id).with_durable_write(true); let writer = dataset .mem_wal_writer(shard_id, config) .await @@ -6014,26 +10052,26 @@ mod shard_writer_tests { let manifest_store = super::super::manifest::ShardManifestStore::new(store, &base_path, shard_id, 2); let manifest = manifest_store - .read_latest() + .latest() .await .expect("Failed to read manifest") .expect("Manifest should exist"); - assert_eq!(manifest.flushed_generations.len(), 1); + assert_eq!(manifest.sstables.len(), 1); - let flushed = &manifest.flushed_generations[0]; - let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, flushed.path); - let flushed_dataset = Dataset::open(&gen_uri) + let sstable = &manifest.sstables[0]; + let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, sstable.path); + let sstable = Dataset::open(&gen_uri) .await - .expect("Failed to open flushed generation"); - let flushed_indices = flushed_dataset.load_indices().await.unwrap(); - assert_eq!(flushed_indices.len(), 1); - assert_eq!(flushed_indices[0].name, "text_fts"); + .expect("Failed to open SSTable"); + let sstable_indices = sstable.load_indices().await.unwrap(); + assert_eq!(sstable_indices.len(), 1); + assert_eq!(sstable_indices[0].name, "text_fts"); assert_eq!( - flushed_indices[0].index_version, 1, + sstable_indices[0].index_version, 1, "maintained v1 FTS index must flush as v1" ); - let results = flushed_dataset + let results = sstable .scan() .full_text_search(FullTextSearchQuery::new("Sample".to_owned())) .unwrap() @@ -6049,7 +10087,12 @@ mod shard_writer_tests { let vector_dim = 32; let schema = create_test_schema(vector_dim); - let uri = format!("memory://test_writer_hnsw_params_{}", Uuid::new_v4()); + // The generation flusher reopens by URI, so this independent open must + // resolve to the same in-memory backend. The unique authority isolates the test. + let uri = format!( + "shared-memory://writer-hnsw-params-{}/", + Uuid::new_v4().simple() + ); let initial = create_test_batch(&schema, 0, 256, vector_dim); let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); @@ -6330,9 +10373,7 @@ mod shard_writer_tests { // Create shard writer let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(false) - .with_sync_indexed_write(false); + let config = ShardWriterConfig::new(shard_id).with_durable_write(false); let writer = dataset .mem_wal_writer(shard_id, config) @@ -6357,7 +10398,12 @@ mod shard_writer_tests { let target_id = 1_000i64 + 37; let schema = create_test_schema(vector_dim); - let uri = format!("memory://test_shard_writer_hnsw_{}", Uuid::new_v4()); + // The generation flusher reopens by URI, so this independent open must + // resolve to the same in-memory backend. The unique authority isolates the test. + let uri = format!( + "shared-memory://shard-writer-hnsw-{}/", + Uuid::new_v4().simple() + ); let initial_batch = create_test_batch(&schema, 0, 256, vector_dim); let batches = RecordBatchIterator::new([Ok(initial_batch)], schema.clone()); @@ -6385,9 +10431,7 @@ mod shard_writer_tests { .expect("Failed to initialize MemWAL"); let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(true) - .with_sync_indexed_write(true); + let config = ShardWriterConfig::new(shard_id).with_durable_write(true); let writer = dataset .mem_wal_writer(shard_id, config) @@ -6526,9 +10570,7 @@ mod shard_writer_tests { // Create shard writer with default config let shard_id = Uuid::new_v4(); - let config = ShardWriterConfig::new(shard_id) - .with_durable_write(false) - .with_sync_indexed_write(false); + let config = ShardWriterConfig::new(shard_id).with_durable_write(false); let writer = dataset .mem_wal_writer(shard_id, config) @@ -6553,7 +10595,7 @@ mod shard_writer_tests { /// 2. File system layout is correct (WAL files, manifest, generation directories) /// 3. WAL entries contain expected data /// 4. Data can be read after each flush cycle - /// 5. Manifest tracks flushed generations correctly + /// 5. Manifest tracks SSTables correctly /// /// Run with: cargo test -p lance shard_writer_tests::test_shard_writer_e2e_correctness -- --nocapture #[tokio::test] @@ -6605,7 +10647,6 @@ mod shard_writer_tests { let shard_id = Uuid::new_v4(); let config = ShardWriterConfig::new(shard_id) .with_durable_write(true) // Ensure WAL files are written - .with_sync_indexed_write(true) .with_max_memtable_size(50 * 1024) // 50KB - triggers flush after ~8 batches .with_max_wal_buffer_size(10 * 1024) // 10KB WAL buffer .with_max_wal_flush_interval(Duration::from_millis(50)); // Fast flush @@ -6677,29 +10718,29 @@ mod shard_writer_tests { let manifest_store = super::super::manifest::ShardManifestStore::new(store, &base_path, shard_id, 2); let manifest = manifest_store - .read_latest() + .latest() .await .expect("Failed to read manifest") .expect("Manifest should exist"); - // Verify flushed generations exist on disk + // Verify SSTables exist on disk assert!( - !manifest.flushed_generations.is_empty(), - "Should have at least one flushed generation" + !manifest.sstables.is_empty(), + "Should have at least one SSTable" ); - for flushed_gen in &manifest.flushed_generations { + for sstable in &manifest.sstables { // The path stored in manifest is relative to the shard directory // Construct full path: temp_dir/_mem_wal/shard_id/generation_folder let gen_path = temp_dir .path() .join("_mem_wal") .join(shard_id.to_string()) - .join(&flushed_gen.path); + .join(&sstable.path); // The generation directory should exist assert!( gen_path.exists(), - "Flushed generation directory should exist at {:?}", + "SSTable directory should exist at {:?}", gen_path ); @@ -6729,9 +10770,13 @@ mod shard_writer_tests { // Re-open dataset and create new writer to verify recovery let dataset = Dataset::open(&uri).await.expect("Failed to reopen dataset"); let new_shard_id = Uuid::new_v4(); - let new_config = ShardWriterConfig::new(new_shard_id) - .with_durable_write(false) - .with_sync_indexed_write(true); + // `durable_write(true)` so the put waits for its flush, which is what + // currently publishes the rows. A non-durable put is *not* yet + // read-your-writes: the index apply is welded to the WAL flush, so the + // rows stay invisible until the next flush. This test used to pass with + // `durable_write(false)` only because an un-advanced cursor of 0 was + // misread as "batch 0 is visible" — it was asserting the dirty read. + let new_config = ShardWriterConfig::new(new_shard_id).with_durable_write(true); let new_writer = dataset .mem_wal_writer(new_shard_id, new_config) @@ -6754,4 +10799,347 @@ mod shard_writer_tests { .await .expect("Failed to close new writer"); } + + /// Regression: a base opened with a *path-bound* store binding (the + /// deprecated `ObjectStoreParams::object_store`) must still flush and read + /// generations at their own paths. + /// + /// The binding pins a store to one location, and both + /// `ObjectStore::from_uri_and_params` and `DatasetBuilder::build_object_store` + /// take the path from it while ignoring the URI they were handed. Reusing the + /// base's params verbatim therefore aimed every generation write and open at + /// the base table itself: the flush failed ("dataset already exists") and any + /// derived open returned base rows as generation rows. + #[tokio::test] + async fn test_flush_and_read_with_path_bound_object_store() { + use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; + use futures::TryStreamExt; + use lance_io::object_store::ObjectStoreParams; + use tempfile::TempDir; + + let vector_dim = 8; + let schema = create_test_schema(vector_dim); + let temp_dir = TempDir::new().unwrap(); + let uri = format!("file://{}", temp_dir.path().display()); + + let initial = create_test_batch(&schema, 0, 16, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + dataset + .initialize_mem_wal() + .execute() + .await + .expect("Failed to initialize MemWAL"); + + // Re-bind the base to a store pinned at the base's own path — what + // `DatasetBuilder::with_object_store` leaves on an opened dataset. + #[allow(deprecated)] + let store_params = ObjectStoreParams { + object_store: Some(( + Arc::new(object_store::local::LocalFileSystem::new()), + url::Url::parse(&uri).unwrap(), + )), + ..Default::default() + }; + let dataset = dataset.with_object_store(dataset.object_store.clone(), Some(store_params)); + + let shard_id = Uuid::new_v4(); + let writer = dataset + .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id)) + .await + .expect("Failed to create writer"); + writer + .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) + .await + .expect("Failed to write"); + writer.force_seal_active().await.unwrap(); + writer + .wait_for_flush_drain() + .await + .expect("flush must not be redirected at the base table"); + + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + assert_eq!(manifest.sstables.len(), 1); + let sstable = manifest.sstables[0].clone(); + + // The generation landed under `_mem_wal/`, and the base table is untouched. + let gen_uri = format!("{}/_mem_wal/{}/{}", uri, shard_id, sstable.path); + let generation = Dataset::open(&gen_uri) + .await + .expect("generation must exist at its own path"); + assert_eq!(generation.count_rows(None).await.unwrap(), 8); + let base = Dataset::open(&uri).await.unwrap(); + assert_eq!( + base.count_rows(None).await.unwrap(), + 16, + "the generation write must not land in the base table" + ); + + // The read path resolves the generation, not the base: 16 base + 8 flushed. + // Opening the base instead would dedup back down to 16 rows. + let snapshot = ShardSnapshot::new(shard_id) + .with_current_generation(manifest.current_generation) + .with_sstable(sstable.generation, sstable.path.clone()); + let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]); + let rows: usize = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .expect("scan must open the generation, not the base") + .iter() + .map(|batch| batch.num_rows()) + .sum(); + assert_eq!(rows, 24); + + writer.close().await.unwrap(); + } + + /// The store params a base was opened with must reach every *derived* open: + /// the flush that writes a generation and the scan that reads it back. + /// + /// This is the point of threading them at all. A namespace-vended store + /// exists only on the params (credentials, endpoint, wrapper), so a + /// generation resolved by URI alone would silently sign with the ambient + /// identity instead — succeeding against a local store and failing against + /// the vended one. Asserting on the *generation folder* rather than + /// `_mem_wal/` is what makes this bite: WAL entries are written through the + /// base dataset's own store, so they would show up here either way. + #[tokio::test] + async fn test_store_params_reach_generation_write_and_read() { + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; + use crate::dataset::mem_wal::test_util::observable_store_params; + use futures::TryStreamExt; + use tempfile::TempDir; + + let vector_dim = 8; + let schema = create_test_schema(vector_dim); + let temp_dir = TempDir::new().unwrap(); + let uri = format!("file://{}", temp_dir.path().display()); + + let initial = create_test_batch(&schema, 0, 16, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + + // Open the base through an observable store, exactly as a namespace + // client would hand in a vended-credential store. + let (store_params, controls) = observable_store_params(); + let mut dataset = DatasetBuilder::from_uri(&uri) + .with_store_params(store_params) + .load() + .await + .expect("Failed to open dataset"); + dataset + .initialize_mem_wal() + .execute() + .await + .expect("Failed to initialize MemWAL"); + + let shard_id = Uuid::new_v4(); + let writer = dataset + .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id)) + .await + .expect("Failed to create writer"); + writer + .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) + .await + .expect("Failed to write"); + writer.force_seal_active().await.unwrap(); + writer.wait_for_flush_drain().await.expect("flush failed"); + + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + assert_eq!(manifest.sstables.len(), 1); + let sstable = manifest.sstables[0].clone(); + + // The generation's own Lance manifest is the signal to key on. Keying on + // the generation folder alone would pass vacuously: sidecars like + // `{gen}/bloom_filter.bin` are written through the *base* dataset's + // store, which is observable no matter what the params do. And the + // fragments can't be used either — `ObjectStore::create` writes local + // files through `tokio::fs`, bypassing the object store entirely, so + // `{gen}/data/` never reaches a wrapper under `file://`. The manifest + // goes through `put_opts`, and only the flusher's `Dataset::write` / + // `open_generation` writes it — both of which must carry the params. + let gen_manifest = format!("{}/_versions", sstable.path); + + assert!( + controls.wrote_under(&gen_manifest), + "the flush must write the generation through the base's store params, \ + not a store resolved from the generation URI alone" + ); + + // And the read path must resolve the generation through them too. + let snapshot = ShardSnapshot::new(shard_id) + .with_current_generation(manifest.current_generation) + .with_sstable(sstable.generation, sstable.path.clone()); + let scanner = LsmScanner::new(Arc::new(dataset), vec![snapshot], vec!["id".to_string()]); + let rows: usize = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .expect("scan failed") + .iter() + .map(|batch| batch.num_rows()) + .sum(); + assert_eq!(rows, 24); + + // Reads key on the data files, not the manifest: the flusher already + // pulled the generation's manifest into the shared session cache, so the + // scan's open serves it from memory and never touches the store. The + // fragments are read through it (reads have no local bypass), as is the + // generation's standalone PK index. + assert!( + controls.read_under(&format!("{}/data/", sstable.path)), + "the scan must read the generation through the base's store params" + ); + + writer.close().await.unwrap(); + } + + /// A fresh-tier-only scanner reaches its store params through + /// `with_store_params`, not `new()`, so the setter must strip the path-bound + /// store binding too. Left raw, it redirects the generation open at the base + /// table and the scan silently returns base rows as WAL rows. + #[tokio::test] + async fn test_fresh_tier_scan_with_path_bound_object_store() { + use crate::dataset::mem_wal::scanner::{LsmScanner, ShardSnapshot}; + use futures::TryStreamExt; + use lance_io::object_store::ObjectStoreParams; + use tempfile::TempDir; + + let vector_dim = 8; + let schema = create_test_schema(vector_dim); + let temp_dir = TempDir::new().unwrap(); + let uri = format!("file://{}", temp_dir.path().display()); + + // 16 base rows with ids 0..16; the WAL gets 8 rows with ids 1000..1008, + // so a redirected generation open is unambiguous in the output. + let initial = create_test_batch(&schema, 0, 16, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + dataset + .initialize_mem_wal() + .execute() + .await + .expect("Failed to initialize MemWAL"); + + let shard_id = Uuid::new_v4(); + let writer = dataset + .mem_wal_writer(shard_id, ShardWriterConfig::new(shard_id)) + .await + .expect("Failed to create writer"); + writer + .put(vec![create_test_batch(&schema, 1_000, 8, vector_dim)]) + .await + .expect("Failed to write"); + writer.force_seal_active().await.unwrap(); + writer.wait_for_flush_drain().await.expect("flush failed"); + + let manifest = writer.manifest().await.unwrap().expect("manifest exists"); + let sstable = manifest.sstables[0].clone(); + let snapshot = ShardSnapshot::new(shard_id) + .with_current_generation(manifest.current_generation) + .with_sstable(sstable.generation, sstable.path.clone()); + + // What `DatasetBuilder::with_object_store` leaves on an opened dataset: + // a store pinned at the base's own path. + #[allow(deprecated)] + let store_params = ObjectStoreParams { + object_store: Some(( + Arc::new(object_store::local::LocalFileSystem::new()), + url::Url::parse(&uri).unwrap(), + )), + ..Default::default() + }; + + let arrow_schema: Arc = schema.clone(); + let batches = LsmScanner::without_base_table( + arrow_schema, + uri.clone(), + vec![snapshot], + vec!["id".to_string()], + ) + .with_session(dataset.session()) + .with_store_params(store_params) + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .expect("scan must open the generation, not the base"); + + let rows: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!( + rows, 8, + "fresh tier holds only the 8 WAL rows; 16 means the generation open \ + was redirected at the base table" + ); + let ids: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + assert!( + ids.iter().all(|id| (1_000..1_008).contains(id)), + "expected the WAL's own rows, got {ids:?}" + ); + + writer.close().await.unwrap(); + } + + /// The other paths now prevent a nullable key, so this forges one to stand + /// in for a table written before they were closed. + #[tokio::test] + async fn test_initialize_mem_wal_rejects_a_nullable_primary_key() { + let vector_dim = 128; + let schema = create_append_only_schema(vector_dim); + let uri = format!("memory://test_mem_wal_nullable_pk_{}", Uuid::new_v4()); + + let initial_batch = create_test_batch(&schema, 0, 100, vector_dim); + let batches = RecordBatchIterator::new([Ok(initial_batch)], schema.clone()); + let mut dataset = Dataset::write(batches, &uri, Some(WriteParams::default())) + .await + .expect("Failed to create dataset"); + + { + let manifest = Arc::make_mut(&mut dataset.manifest); + let id_field = manifest + .schema + .fields + .iter_mut() + .find(|field| field.name == "id") + .expect("schema has an id column"); + id_field.unenforced_primary_key_position = Some(1); + id_field.nullable = true; + } + + let err = dataset + .initialize_mem_wal() + .unsharded() + .execute() + .await + .expect_err("MemWAL must not enable on a nullable primary key"); + assert!( + err.to_string().contains("must not be nullable"), + "unexpected error: {err}" + ); + } } diff --git a/rust/lance/src/dataset/metadata.rs b/rust/lance/src/dataset/metadata.rs index 21d92100871..7ae1bd03514 100644 --- a/rust/lance/src/dataset/metadata.rs +++ b/rust/lance/src/dataset/metadata.rs @@ -189,6 +189,25 @@ mod tests { }; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + /// A dataset whose columns are all **non-nullable**, so a primary key can be + /// installed on one. `gen_batch` produces nullable columns, and a primary + /// key column must not be nullable. + async fn dataset_with_non_nullable_columns(uri: &str, names: &[&str]) -> Dataset { + let schema = Arc::new(ArrowSchema::new( + names + .iter() + .map(|name| ArrowField::new(*name, DataType::Int32, false)) + .collect::>(), + )); + let columns: Vec = names + .iter() + .map(|_| Arc::new(Int32Array::from((0..10).collect::>())) as ArrayRef) + .collect(); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(reader, uri, None).await.unwrap() + } + #[rstest] #[tokio::test] async fn test_update_config() { @@ -548,10 +567,7 @@ mod tests { let tmp_dir = lance_core::utils::tempfile::TempStrDir::default(); let uri = tmp_dir.as_str(); - let data = gen_batch() - .col("a", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, uri, None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns(uri, &["a"]).await; assert!(dataset.schema().unenforced_primary_key().is_empty()); dataset @@ -578,10 +594,7 @@ mod tests { use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY; for truthy in ["true", "1", "yes", "TRUE", "Yes"] { - let data = gen_batch() - .col("a", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns("memory://", &["a"]).await; dataset .update_field_metadata() .replace("a", [(LANCE_UNENFORCED_PRIMARY_KEY, truthy)]) @@ -606,10 +619,7 @@ mod tests { LANCE_UNENFORCED_PRIMARY_KEY, LANCE_UNENFORCED_PRIMARY_KEY_POSITION, }; - let data = gen_batch() - .col("a", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns("memory://", &["a"]).await; dataset .update_field_metadata() .replace( @@ -633,11 +643,7 @@ mod tests { // alters the set of primary key columns, is rejected. use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION; - let data = gen_batch() - .col("a", array::step::()) - .col("b", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns("memory://", &["a", "b"]).await; // The first install of the primary key is allowed. dataset @@ -682,6 +688,31 @@ mod tests { assert_eq!(pk[0].name, "a"); } + /// Installing the key by field metadata skips the Arrow-schema conversion + /// that validates one, so a nullable target must be rejected here. + #[tokio::test] + async fn test_unenforced_primary_key_rejects_a_nullable_column() { + use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION; + + // `gen_batch` columns are nullable. + let data = gen_batch() + .col("a", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + + let err = dataset + .update_field_metadata() + .update("a", [(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "1")]) + .unwrap() + .await + .unwrap_err(); + assert!( + err.to_string().contains("must not be nullable"), + "got {err:?}" + ); + assert!(dataset.schema().unenforced_primary_key().is_empty()); + } + #[tokio::test] async fn test_unenforced_primary_key_rejects_invalid_marker() { // Writing a reserved primary key metadata key with a value that is not @@ -689,10 +720,7 @@ mod tests { // silently ignored. use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY; - let data = gen_batch() - .col("a", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns("memory://", &["a"]).await; for invalid in ["no", "false", "0", "anything-else"] { let err = dataset @@ -741,10 +769,7 @@ mod tests { let tmp_dir = lance_core::utils::tempfile::TempStrDir::default(); let uri = tmp_dir.as_str(); - let data = gen_batch() - .col("a", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, uri, None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns(uri, &["a"]).await; assert!(dataset.schema().unenforced_clustering_key().is_empty()); dataset @@ -856,10 +881,7 @@ mod tests { // not a valid position is rejected rather than silently ignored. use lance_core::datatypes::LANCE_UNENFORCED_CLUSTERING_KEY_POSITION; - let data = gen_batch() - .col("a", array::step::()) - .into_reader_rows(RowCount::from(10), BatchCount::from(1)); - let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + let mut dataset = dataset_with_non_nullable_columns("memory://", &["a"]).await; for invalid in ["not-a-number", "", "1.5"] { let err = dataset @@ -877,4 +899,58 @@ mod tests { assert!(dataset.schema().unenforced_clustering_key().is_empty()); } } + + /// A table that already carries a nullable primary key must stay writable, + /// including through the delete that repairs it. + /// + /// Released versions could install a key on a nullable column through this + /// metadata path, so such tables exist. Validating every manifest write + /// would make them read-only on upgrade and leave a full overwrite as the + /// only repair. + #[tokio::test] + async fn nullable_primary_key_stays_repairable() { + let test_dir = lance_core::utils::tempfile::TempStrDir::default(); + let uri: &str = &test_dir; + + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int32, + true, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![Some(1), None]))], + ) + .unwrap(); + let mut dataset = Dataset::write(RecordBatchIterator::new([Ok(batch)], schema), uri, None) + .await + .unwrap(); + + // Forge the state a released version could persist: a key on a + // nullable column, without passing the checks that now prevent it. + { + let manifest = Arc::make_mut(&mut dataset.manifest); + let field = manifest + .schema + .fields + .iter_mut() + .find(|f| f.name == "id") + .unwrap(); + field.unenforced_primary_key_position = Some(1); + } + + // Unrelated writes must still go through. + dataset + .update_config([("unrelated".to_string(), "value".to_string())]) + .await + .expect("an unrelated write must not be blocked by a pre-existing bad key"); + + // And so must the delete that removes the offending rows -- without it + // there is no way to tighten the column afterwards. + dataset + .delete("id IS NULL") + .await + .expect("the repairing delete must not be blocked"); + assert_eq!(dataset.count_rows(None).await.unwrap(), 1); + } } diff --git a/rust/lance/src/dataset/optimize.rs b/rust/lance/src/dataset/optimize.rs index 656f0163ce6..81f01882e55 100644 --- a/rust/lance/src/dataset/optimize.rs +++ b/rust/lance/src/dataset/optimize.rs @@ -82,51 +82,63 @@ //! they can be committed in any order. use lance_core::utils::row_addr_remap::{GroupInput, RowAddrRemap}; use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::ops::{AddAssign, Range}; use std::sync::Arc; use super::fragment::FileFragment; -use super::index::DatasetIndexRemapperOptions; +use super::index::{DatasetIndexRemapperOptions, load_indices_for_remapping}; use super::rowids::load_row_id_sequences; use super::transaction::{ Operation, RewriteGroup, RewrittenIndex, Transaction, TransactionBuilder, }; use super::utils::make_rowid_capture_stream; -use super::{WriteMode, WriteParams, cleanup_data_fragments, write_fragments_internal}; +use super::versions; +use super::{ + WriteMode, WriteParams, cleanup_data_fragments, + write::write_fragments_internal_with_file_row_counts, +}; use crate::Dataset; use crate::Result; use crate::dataset::utils::CapturedRowIds; -use crate::index::DatasetIndexExt; -use crate::io::commit::{commit_transaction, migrate_fragments}; +use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, index_is_usable, load_all_indices}; +use crate::io::commit::{DEFAULT_COMMIT_RETRY_TIMEOUT, commit_transaction, migrate_fragments}; use arrow::array::AsArray; use arrow::datatypes::{UInt8Type, UInt32Type, UInt64Type}; -use arrow_array::Array; -use arrow_array::RecordBatch; -use arrow_array::StructArray; use arrow_array::builder::{LargeBinaryBuilder, PrimitiveBuilder, StringBuilder}; -use arrow_buffer::NullBuffer; +use arrow_array::{ + Array, ArrayRef, GenericListArray, OffsetSizeTrait, RecordBatch, StructArray, UInt32Array, +}; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{ + DataType as ArrowDataType, Field as ArrowField, Schema as ArrowSchema, SchemaRef, +}; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use futures::{StreamExt, TryStreamExt}; +use futures::future::BoxFuture; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use lance_arrow::{list::ListArrayExt, r#struct::StructArrayExt}; use lance_core::Error; -use lance_core::datatypes::{BlobHandling, BlobKind}; +use lance_core::datatypes::{ + BLOB_V2_LOGICAL_FIELDS, BLOB_V2_LOGICAL_TYPE, BlobHandling, BlobKind, BlobV2Layout, + Field as LanceField, +}; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::utils::tracing::{DATASET_COMPACTING_EVENT, TRACE_DATASET_EVENTS}; -use lance_index::frag_reuse::FragReuseGroup; +use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseGroup}; use lance_index::is_system_index; -use lance_table::format::{Fragment, RowIdMeta}; +use lance_index::metrics::NoOpMetricsCollector; +use lance_table::format::{Fragment, IndexMetadata, RowIdMeta}; use roaring::{RoaringBitmap, RoaringTreemap}; use serde::{Deserialize, Serialize}; use tracing::{info, warn}; -mod binary_copy; +pub(super) mod binary_copy; pub mod remapping; use crate::index::frag_reuse::build_new_frag_reuse_index; use crate::io::deletion::read_dataset_deletion_file; -use binary_copy::rewrite_files_binary_copy; pub use remapping::{IgnoreRemap, IndexRemapper, IndexRemapperOptions, RemappedIndex}; /// Controls how data is rewritten during compaction. @@ -262,6 +274,38 @@ pub struct CompactionOptions { /// fragments at a time). /// Defaults to `None` (no limit, all eligible fragments are compacted). pub max_source_fragments: Option, + /// Maximum number of source rows to compact in a single run. Rows are + /// counted as live rows (physical rows minus soft-deleted rows). When + /// set, tasks are included in the plan until adding the next task would + /// exceed this limit. + /// Defaults to `None` (no limit). + pub max_source_rows: Option, + /// Maximum number of source bytes to compact in a single run, measured as + /// the total size of the source fragments' data and overlay files. When + /// set, tasks are included in the plan until adding the next task would + /// exceed this limit. + /// Blob v2 payloads live in separate blob files and are not counted, so + /// this is not a cap on total compaction I/O for datasets with blob + /// columns. + /// Defaults to `None` (no limit). + pub max_source_bytes: Option, + /// Fragment IDs to exclude from compaction planning. + /// + /// Excluded fragments act as boundaries between adjacent compaction candidates, + /// so fragments on opposite sides of an exclusion are never combined into the + /// same task. IDs that are duplicated or absent from the dataset are ignored. + /// Defaults to an empty list. + #[serde(default)] + pub excluded_fragment_ids: Vec, + /// Maximum number of data overlay files a fragment may carry before it is + /// fully compacted. When set, any fragment with more than this many overlays + /// is rewritten into a fresh fragment with its overlays (and deletions) + /// materialized into the base data, dropping the fragment from any index + /// left stale by those overlays. + /// Defaults to `Some(10)`. Set to `Some(0)` to compact every fragment that + /// carries any overlay, or `None` to disable the overlay-count trigger + /// entirely. + pub max_overlays_per_fragment: Option, /// Transaction properties to store with this commit. /// /// These key-value pairs are stored in the transaction file @@ -291,6 +335,10 @@ impl Default for CompactionOptions { enable_binary_copy_force: false, binary_copy_read_batch_bytes: Some(16 * 1024 * 1024), max_source_fragments: None, + max_source_rows: None, + max_source_bytes: None, + excluded_fragment_ids: Vec::new(), + max_overlays_per_fragment: Some(10), transaction_properties: None, } } @@ -317,6 +365,9 @@ impl CompactionOptions { /// - `lance.compaction.compaction_mode` /// - `lance.compaction.binary_copy_read_batch_bytes` /// - `lance.compaction.max_source_fragments` + /// - `lance.compaction.max_source_rows` + /// - `lance.compaction.max_source_bytes` + /// - `lance.compaction.max_overlays_per_fragment` pub fn from_dataset_config(config: &HashMap) -> Result { let mut opts = Self::default(); opts.apply_dataset_config(config)?; @@ -427,6 +478,35 @@ impl CompactionOptions { )) })?); } + "max_source_rows" => { + self.max_source_rows = Some(value.parse().map_err(|_| { + Error::invalid_input(format!( + "Invalid value for {}: '{}' (expected a non-negative integer)", + key, value + )) + })?); + } + "max_source_bytes" => { + self.max_source_bytes = Some(value.parse().map_err(|_| { + Error::invalid_input(format!( + "Invalid value for {}: '{}' (expected a non-negative integer)", + key, value + )) + })?); + } + "max_overlays_per_fragment" => { + // The default is `Some(10)`, so an explicit "none" is the only + // way to disable the trigger through the manifest config. + self.max_overlays_per_fragment = match value.to_ascii_lowercase().as_str() { + "none" => None, + _ => Some(value.parse().map_err(|_| { + Error::invalid_input(format!( + "Invalid value for {}: '{}' (expected a non-negative integer or 'none')", + key, value + )) + })?), + }; + } _ => { warn!("Ignoring unknown compaction config key: {}", key); } @@ -435,11 +515,28 @@ impl CompactionOptions { Ok(()) } - pub fn validate(&mut self) { + pub fn validate(&mut self) -> Result<()> { // If threshold is 100%, same as turning off deletion materialization. if self.materialize_deletions && self.materialize_deletions_threshold >= 1.0 { self.materialize_deletions = false; } + + for (name, value) in [ + ( + "max_source_fragments", + self.max_source_fragments.map(|v| v as u64), + ), + ("max_source_rows", self.max_source_rows.map(|v| v as u64)), + ("max_source_bytes", self.max_source_bytes), + ] { + if value == Some(0) { + return Err(Error::invalid_input(format!( + "CompactionOptions::{} must be greater than 0 (use None for no limit)", + name + ))); + } + } + Ok(()) } /// Returns the effective [`CompactionMode`], preferring the new @@ -473,14 +570,16 @@ impl CompactionOptions { /// - All data files share identical Lance file versions /// - No fragment has a deletion file /// TODO: Need to support schema evolution case like add column and drop column -/// - All data files share identical schema mappings (`fields`, `column_indices`) +/// - All data files use an identical schema mapping (`fields`, `column_indices`) in dataset schema +/// order /// - Input data files must not contain extra global buffers (beyond schema / file descriptor) async fn can_use_binary_copy( dataset: &Dataset, options: &CompactionOptions, fragments: &[Fragment], ) -> bool { - can_use_binary_copy_impl(dataset, options, fragments) + let version = dataset.manifest.data_storage_format.lance_file_format(); + versions::can_use_binary_copy(version, dataset, options, fragments) .await .unwrap_or_else(|err| { log::warn!("Binary copy disabled due to error: {}", err); @@ -488,13 +587,12 @@ async fn can_use_binary_copy( }) } -async fn can_use_binary_copy_impl( +pub(super) async fn can_use_binary_copy_current( dataset: &Dataset, options: &CompactionOptions, fragments: &[Fragment], ) -> Result { use lance_file::reader::FileReader as LFReader; - use lance_file::version::LanceFileVersion; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; if matches!(options.compaction_mode(), CompactionMode::Reencode) { @@ -511,28 +609,11 @@ async fn can_use_binary_copy_impl( return Ok(false); } - let storage_ok = dataset - .manifest - .data_storage_format - .lance_file_version() - .map(|v| !matches!(v.resolve(), LanceFileVersion::Legacy)) - .unwrap_or(false); - if !storage_ok { - log::debug!("Binary copy disabled: dataset uses legacy storage format"); - return Ok(false); - } - if fragments.is_empty() { log::debug!("Binary copy disabled: no fragments to compact"); return Ok(false); } - let storage_file_version = dataset - .manifest - .data_storage_format - .lance_file_version()? - .resolve(); - if fragments[0].files.is_empty() { log::debug!( "Binary copy disabled: fragment {} has no data files", @@ -542,8 +623,17 @@ async fn can_use_binary_copy_impl( } let ref_fields = &fragments[0].files[0].fields; let ref_cols = &fragments[0].files[0].column_indices; - let mut is_same_version = true; - + let version = dataset.manifest.data_storage_format.lance_file_format(); + let (schema_fields, schema_column_indices) = + lance_file::versions::data_file_columns(version, dataset.schema()); + if ref_fields.as_ref() != schema_fields.as_slice() + || ref_cols.as_ref() != schema_column_indices.as_slice() + { + log::debug!( + "Binary copy disabled: data files do not use the dataset schema's physical column order" + ); + return Ok(false); + } for fragment in fragments { if fragment.deletion_file.is_some() { log::debug!( @@ -554,16 +644,6 @@ async fn can_use_binary_copy_impl( } for data_file in &fragment.files { - let version_ok = LanceFileVersion::try_from_major_minor( - data_file.file_major_version, - data_file.file_minor_version, - ) - .map(|v| v.resolve()) - .is_ok_and(|v| v == storage_file_version); - - if !version_ok { - is_same_version = false; - } if data_file.fields != *ref_fields || data_file.column_indices != *ref_cols { return Ok(false); } @@ -600,11 +680,6 @@ async fn can_use_binary_copy_impl( } } - if !is_same_version { - log::debug!("Binary copy disabled: data files use different file versions"); - return Ok(false); - } - Ok(true) } @@ -659,12 +734,17 @@ pub trait CompactionPlanner: Send + Sync { #[derive(Debug, Clone, Default)] pub struct DefaultCompactionPlanner { options: CompactionOptions, + excluded_fragment_ids: RoaringBitmap, } impl DefaultCompactionPlanner { - pub fn new(mut options: CompactionOptions) -> Self { - options.validate(); - Self { options } + pub fn new(mut options: CompactionOptions) -> Result { + options.validate()?; + let excluded_fragment_ids = options.excluded_fragment_ids.iter().copied().collect(); + Ok(Self { + options, + excluded_fragment_ids, + }) } } @@ -688,11 +768,44 @@ impl CompactionPlanner for DefaultCompactionPlanner { fragments.windows(2).all(|w| w[0].id() < w[1].id()), "fragments in manifest are not sorted" ); + // Without stable row ids a rewrite moves every row address, so the + // indices over the rewritten fragments have to be remapped in the same + // commit. An index this build cannot open cannot be remapped, so its + // fragments join the caller's own exclusions and are left uncompacted - + // taking the same path, which terminates the current bin rather than + // letting the candidates on either side of the gap be planned together. + let mut excluded_fragment_ids = self.excluded_fragment_ids.clone(); + if !dataset.manifest.uses_stable_row_ids() && !self.options.defer_index_remap { + let unremappable = unremappable_index_coverage(dataset) + .await? + .into_iter() + .fold(RoaringBitmap::new(), |mut covered, (_, fragments)| { + covered |= fragments; + covered + }); + if !unremappable.is_empty() { + // Otherwise a compaction that plans nothing looks like a + // compaction that found nothing to do. + log::info!( + "holding {} fragment(s) back from compaction: they are covered by an index \ + this build cannot read, and so cannot be remapped here", + unremappable.len(), + ); + } + excluded_fragment_ids |= unremappable; + } + let excluded_fragment_ids = &excluded_fragment_ids; + let mut fragment_metrics = futures::stream::iter(fragments) .map(|fragment| async move { - match collect_metrics(&fragment).await { - Ok(metrics) => Ok((fragment.metadata, metrics)), - Err(e) => Err(e), + if u32::try_from(fragment.id()) + .is_ok_and(|fragment_id| excluded_fragment_ids.contains(fragment_id)) + { + Ok(None) + } else { + collect_metrics(&fragment) + .await + .map(|metrics| Some((fragment.metadata, metrics))) } }) .buffered(dataset.object_store.as_ref().io_parallelism()); @@ -712,9 +825,27 @@ impl CompactionPlanner for DefaultCompactionPlanner { let mut i = 0; while let Some(res) = fragment_metrics.next().await { - let (fragment, metrics) = res?; + let Some((fragment, metrics)) = res? else { + // Exclusions preserve adjacency semantics: they terminate the + // current bin instead of allowing candidates on either side to + // be planned together. + if let Some(bin) = current_bin.take() { + candidate_bins.push(bin); + } + i += 1; + continue; + }; + + let over_overlay_limit = self + .options + .max_overlays_per_fragment + .is_some_and(|max| fragment.overlays.len() > max); - let candidacy = if self.options.materialize_deletions + let candidacy = if over_overlay_limit { + // Too many overlays: fully compact this fragment on its own, + // regardless of its size or deletion count. + Some(CompactionCandidacy::CompactItself) + } else if self.options.materialize_deletions && metrics.deletion_percentage() > self.options.materialize_deletions_threshold { Some(CompactionCandidacy::CompactItself) @@ -776,27 +907,22 @@ impl CompactionPlanner for DefaultCompactionPlanner { candidate_bins.push(bin); } - let all_tasks: Vec = candidate_bins + let all_tasks: Vec<(TaskData, usize)> = candidate_bins .into_iter() .filter(|bin| !bin.is_noop()) .flat_map(|bin| bin.split_for_size(self.options.target_rows_per_fragment)) - .map(|bin| TaskData { - fragments: bin.fragments, + .map(|bin| { + let live_rows = bin.row_counts.iter().sum(); + ( + TaskData { + fragments: bin.fragments, + }, + live_rows, + ) }) .collect(); - let tasks = if let Some(max_frags) = self.options.max_source_fragments { - let mut total_frags = 0; - all_tasks - .into_iter() - .take_while(|task| { - total_frags += task.fragments.len(); - total_frags <= max_frags - }) - .collect() - } else { - all_tasks - }; + let tasks = limit_tasks_to_source_budget(&self.options, dataset.schema(), all_tasks)?; let mut compaction_plan = CompactionPlan::new(dataset.manifest.version, self.options.clone()); @@ -822,7 +948,7 @@ pub async fn compact_files( remap_options: Option>, // These will be deprecated later ) -> Result { info!(target: TRACE_DATASET_EVENTS, event=DATASET_COMPACTING_EVENT, uri = &dataset.uri); - let planner = DefaultCompactionPlanner::new(options); + let planner = DefaultCompactionPlanner::new(options)?; compact_files_with_planner(dataset, remap_options, &planner).await } @@ -840,7 +966,7 @@ pub async fn compact_files_with_planner( let dataset_ref = &dataset.clone(); - let result_stream = futures::stream::iter(compaction_plan.tasks.into_iter()) + let result_stream = futures::stream::iter(compaction_plan.tasks) .map(|task| rewrite_files(Cow::Borrowed(dataset_ref), task, &compaction_plan.options)) .buffer_unordered( compaction_plan @@ -898,6 +1024,111 @@ async fn collect_metrics(fragment: &FileFragment) -> Result { }) } +/// Truncates a planned task list to the configured per-run source budgets +/// (`max_source_fragments`, `max_source_rows`, `max_source_bytes`). +/// +/// All configured budgets apply together: tasks are kept, in order, until +/// adding the next task would exceed any one of them. The budgets are hard +/// upper bounds, so if the first task already exceeds one of them the +/// returned plan is empty and a warning is logged, since compaction would +/// otherwise stall silently. +/// +/// Each task is paired with the number of live rows in its source fragments. +fn limit_tasks_to_source_budget( + options: &CompactionOptions, + schema: &lance_core::datatypes::Schema, + all_tasks: Vec<(TaskData, usize)>, +) -> Result> { + if options.max_source_fragments.is_none() + && options.max_source_rows.is_none() + && options.max_source_bytes.is_none() + { + return Ok(all_tasks.into_iter().map(|(task, _)| task).collect()); + } + + // Only needed for the bytes budget: files whose fields are all absent + // from the current schema only back dropped columns, which compaction + // does not read. + let schema_field_ids: HashSet = if options.max_source_bytes.is_some() { + schema.field_ids().into_iter().collect() + } else { + HashSet::new() + }; + + let num_candidate_tasks = all_tasks.len(); + let mut total_fragments = 0_usize; + let mut total_rows = 0_usize; + let mut total_bytes = 0_u64; + let mut tasks = Vec::with_capacity(all_tasks.len()); + for (task, live_rows) in all_tasks { + total_fragments += task.fragments.len(); + total_rows = total_rows.saturating_add(live_rows); + if options.max_source_bytes.is_some() { + total_bytes = total_bytes.saturating_add(task_source_bytes(&task, &schema_field_ids)?); + } + + let over_budget = options + .max_source_fragments + .is_some_and(|max| total_fragments > max) + || options.max_source_rows.is_some_and(|max| total_rows > max) + || options + .max_source_bytes + .is_some_and(|max| total_bytes > max); + if over_budget { + break; + } + + tasks.push(task); + } + + if tasks.is_empty() && num_candidate_tasks > 0 { + warn!( + "Compaction plan is empty: the first of {} candidate tasks already exceeds a source \ + budget (max_source_fragments={:?}, max_source_rows={:?}, max_source_bytes={:?}); \ + compaction cannot make progress until the budget is raised", + num_candidate_tasks, + options.max_source_fragments, + options.max_source_rows, + options.max_source_bytes + ); + } + + Ok(tasks) +} + +/// Returns the total size in bytes of a task's source data and overlay files. +/// +/// Files whose fields are all absent from `schema_field_ids` only back +/// dropped columns; compaction does not read them, so they are neither +/// counted nor required to have a recorded size. +/// Only sizes recorded in the manifest are used: a missing size is an error +/// rather than a metadata request against object storage, which would turn +/// planning into one round trip per file. Deletion files are not counted. +fn task_source_bytes(task: &TaskData, schema_field_ids: &HashSet) -> Result { + let mut total_bytes = 0_u64; + for fragment in &task.fragments { + let overlay_files = fragment.overlays.iter().map(|overlay| &overlay.data_file); + for data_file in fragment.files.iter().chain(overlay_files) { + if !data_file + .fields + .iter() + .any(|field_id| schema_field_ids.contains(field_id)) + { + continue; + } + let size = data_file.file_size_bytes.get().ok_or_else(|| { + Error::invalid_input(format!( + "max_source_bytes is set but file '{}' of fragment {} has no size recorded \ + in the manifest; unset max_source_bytes to compact this dataset", + data_file.path, fragment.id + )) + })?; + total_bytes = total_bytes.saturating_add(size.get()); + } + } + Ok(total_bytes) +} + /// A plan for what groups of fragments to compact. /// /// See [plan_compaction()] for more details. @@ -938,7 +1169,7 @@ impl CompactionPlan { /// Classification for one blob v2 row during compaction. /// -/// - `Null`: NULL row or Inline blob with position=0 and size=0. +/// - `Null`: NULL row. /// - `External`: External blob referenced by URI. /// - `DataBlob`: Inline/Packed/Dedicated blob stored in Lance files. enum RowClass { @@ -947,25 +1178,6 @@ enum RowClass { DataBlob, } -/// Check if a row is a null Inline blob. -/// -/// This matches `BlobV2StructuralEncoder`'s behavior of encoding null rows as -/// Inline with position=0 and size=0, and `collect_blob_entries_v2`'s behavior -/// of skipping them. -fn is_inline_null_blob( - kind: BlobKind, - position_col: &arrow::array::UInt64Array, - size_col: &arrow::array::UInt64Array, - index: usize, -) -> bool { - if kind != BlobKind::Inline { - return false; - } - let position_is_empty = position_col.is_null(index) || position_col.value(index) == 0; - let size_is_empty = size_col.is_null(index) || size_col.value(index) == 0; - position_is_empty && size_is_empty -} - /// Column views for the 5 fields in a blob v2 descriptor struct. struct BlobV2Descriptor<'a> { kind_col: &'a arrow::array::UInt8Array, @@ -978,6 +1190,14 @@ struct BlobV2Descriptor<'a> { impl<'a> BlobV2Descriptor<'a> { /// Extract the 5 descriptor arrays from a blob v2 descriptor struct array. fn try_from_struct(struct_arr: &'a StructArray, column_name: &str) -> Result { + if BlobV2Layout::classify(struct_arr.fields()) != Some(BlobV2Layout::Descriptor) { + let actual = BlobV2Layout::classify(struct_arr.fields()) + .map(|layout| layout.to_string()) + .unwrap_or_else(|| format!("unrecognized ({:?})", struct_arr.fields())); + return Err(Error::invalid_input(format!( + "Blob v2 column '{column_name}' has {actual} layout; expected descriptor layout before conversion to logical" + ))); + } let kind_col = struct_arr .column_by_name("kind") .ok_or_else(|| { @@ -1033,22 +1253,21 @@ impl<'a> BlobV2Descriptor<'a> { } } -/// Result of row classification for blob v2 compaction. +/// Result of row classification for a blob v2 rewrite. struct RowClassification { row_classes: Vec, - blob_read_addrs: Vec, + data_blob_indices: Vec, } /// Classify each row of a blob v2 column as Null, External, or DataBlob. fn classify_rows( struct_arr: &StructArray, descriptor: &BlobV2Descriptor<'_>, - row_addrs: &arrow::array::UInt64Array, column_name: &str, ) -> Result { let num_rows = struct_arr.len(); let mut row_classes = Vec::with_capacity(num_rows); - let mut blob_read_addrs = Vec::with_capacity(num_rows); + let mut data_blob_indices = Vec::with_capacity(num_rows); for i in 0..num_rows { if struct_arr.is_null(i) || descriptor.kind_col.is_null(i) { @@ -1062,41 +1281,101 @@ fn classify_rows( })?; if kind == BlobKind::External { row_classes.push(RowClass::External); - } else if is_inline_null_blob(kind, descriptor.position_col, descriptor.size_col, i) { - row_classes.push(RowClass::Null); } else { row_classes.push(RowClass::DataBlob); - blob_read_addrs.push(row_addrs.value(i)); + data_blob_indices.push(i); } } } Ok(RowClassification { row_classes, - blob_read_addrs, + data_blob_indices, }) } -/// Build a blob v2 user-view struct array from classification and descriptor. +/// Convert a blob v2 descriptor into the logical writer representation. /// /// Reads blob data lazily using row addresses to avoid materializing all blob /// payloads in memory at once. -async fn build_user_view_struct( +async fn descriptor_to_logical_blob_array( dataset: &Arc, + blob_field_id: u32, + struct_arr: &StructArray, descriptor: &BlobV2Descriptor<'_>, classification: &RowClassification, - column_name: &str, - num_rows: usize, - null_buffer: Option, + row_addrs: &[u64], + field_name: &str, ) -> Result { - let blob_files = if classification.blob_read_addrs.is_empty() { + if struct_arr.len() != row_addrs.len() { + return Err(Error::internal(format!( + "Blob v2 field '{}' row count {} did not match row address count {}", + field_name, + struct_arr.len(), + row_addrs.len() + ))); + } + let blob_files = if classification.data_blob_indices.is_empty() { Vec::new() } else { - super::blob::take_blobs_by_addresses(dataset, &classification.blob_read_addrs, column_name) - .await? + let indices = classification + .data_blob_indices + .iter() + .map(|index| { + u32::try_from(*index).map_err(|_| { + Error::internal(format!( + "Blob v2 row index {} in field '{}' does not fit in u32", + index, field_name + )) + }) + }) + .collect::>>()?; + let indices = UInt32Array::from(indices); + let descriptions = arrow_select::take::take(struct_arr, &indices, None)?; + let descriptions = descriptions.as_struct(); + let data_row_addrs = classification + .data_blob_indices + .iter() + .map(|index| row_addrs[*index]) + .collect::>(); + super::blob::collect_blob_v2_descriptor_files( + dataset, + blob_field_id, + descriptions, + &data_row_addrs, + ) + .await? }; - let mut data_builder = LargeBinaryBuilder::with_capacity(num_rows, 0); + let data_capacity = + classification + .data_blob_indices + .iter() + .try_fold(0usize, |data_capacity, row_idx| { + if descriptor.size_col.is_null(*row_idx) { + return Err(Error::internal(format!( + "Non-null blob row {} in field '{}' is missing its size", + row_idx, field_name + ))); + } + let size = usize::try_from(descriptor.size_col.value(*row_idx)).map_err(|_| { + Error::internal(format!( + "Blob size {} at row {} in field '{}' does not fit in usize", + descriptor.size_col.value(*row_idx), + row_idx, + field_name + )) + })?; + data_capacity.checked_add(size).ok_or_else(|| { + Error::internal(format!( + "Total blob size in field '{}' exceeds usize", + field_name + )) + }) + })?; + + let num_rows = struct_arr.len(); + let mut data_builder = LargeBinaryBuilder::with_capacity(num_rows, data_capacity); let mut uri_builder = StringBuilder::with_capacity(num_rows, 0); let mut out_position_builder = PrimitiveBuilder::::with_capacity(num_rows); let mut out_size_builder = PrimitiveBuilder::::with_capacity(num_rows); @@ -1121,25 +1400,36 @@ async fn build_user_view_struct( let base = dataset.manifest().base_paths.get(&base_id).ok_or_else(|| { Error::internal(format!( "External blob in column '{}' references unknown base_id {}", - column_name, base_id + field_name, base_id )) })?; let absolute_uri = format!("{}/{}", base.path.trim_end_matches('/'), uri_val); uri_builder.append_value(&absolute_uri); } - if descriptor.position_col.is_null(i) { + let position = + (!descriptor.position_col.is_null(i)).then(|| descriptor.position_col.value(i)); + let size = (!descriptor.size_col.is_null(i)).then(|| descriptor.size_col.value(i)); + if position == Some(0) && size == Some(0) { + // Stable descriptors use (0, 0) for the complete external object. + // Logical input represents the same value by omitting the range. out_position_builder.append_null(); - } else { - out_position_builder.append_value(descriptor.position_col.value(i)); - } - if descriptor.size_col.is_null(i) { out_size_builder.append_null(); } else { - out_size_builder.append_value(descriptor.size_col.value(i)); + out_position_builder.append_option(position); + out_size_builder.append_option(size); } } RowClass::DataBlob => { - let data = blob_files[blob_file_idx].read().await?; + let blob_file = blob_files + .get(blob_file_idx) + .and_then(Option::as_ref) + .ok_or_else(|| { + Error::internal(format!( + "Non-null blob row {} in field '{}' resolved to null", + i, field_name + )) + })?; + let data = blob_file.read().await?; blob_file_idx += 1; data_builder.append_value(data.as_ref()); uri_builder.append_null(); @@ -1150,110 +1440,456 @@ async fn build_user_view_struct( } Ok(StructArray::try_new( - lance_core::datatypes::BLOB_V2_USER_FIELDS.clone(), + BLOB_V2_LOGICAL_FIELDS.clone(), vec![ Arc::new(data_builder.finish()), Arc::new(uri_builder.finish()), Arc::new(out_position_builder.finish()), Arc::new(out_size_builder.finish()), ], - null_buffer, + struct_arr.nulls().cloned(), )?) } -async fn transform_blob_v2_batch( - dataset: &Arc, - schema: &lance_core::datatypes::Schema, - batch: RecordBatch, -) -> Result { - let row_addr_idx = batch - .schema() - .column_with_name(lance_core::ROW_ADDR) - .ok_or_else(|| { - Error::internal(format!( - "_rowaddr column missing from batch for blob v2 compaction, columns: {:?}", - batch - .schema() - .fields() - .iter() - .map(|f| f.name()) - .collect::>() - )) - })? - .0; - let row_addrs = batch.column(row_addr_idx).as_primitive::(); +fn transformed_arrow_field(field: &LanceField, data_type: ArrowDataType) -> Arc { + let arrow_field = ArrowField::from(field); + arrow_field_with_data_type(&arrow_field, data_type) +} - let mut new_columns: Vec> = Vec::new(); - let mut new_fields: Vec> = Vec::new(); +fn arrow_field_with_data_type(field: &ArrowField, data_type: ArrowDataType) -> Arc { + Arc::new( + ArrowField::new(field.name(), data_type, field.is_nullable()) + .with_metadata(field.metadata().clone()), + ) +} - let batch_schema = batch.schema(); - for (col_idx, field) in batch_schema.fields().iter().enumerate() { - if field.name() == lance_core::ROW_ADDR { - continue; - } +pub(crate) fn field_contains_blob_v2(field: &LanceField) -> bool { + field.is_blob_v2() || field.children.iter().any(field_contains_blob_v2) +} - let lance_field = schema.field(field.name()); - let is_blob_v2 = lance_field.is_some_and(|f| f.is_blob_v2()); +enum BlobV2FieldRewritePlan { + Passthrough { + output_field: Arc, + }, + Blob { + field_id: u32, + field_name: String, + output_field: Arc, + }, + Struct { + field_name: String, + output_field: Arc, + children: Vec, + }, + List { + field_name: String, + output_field: Arc, + child: Box, + }, + LargeList { + field_name: String, + output_field: Arc, + child: Box, + }, +} - if !is_blob_v2 { - new_columns.push(batch.column(col_idx).clone()); - new_fields.push(field.clone()); - continue; +impl BlobV2FieldRewritePlan { + fn passthrough(field: &ArrowField) -> Self { + Self::Passthrough { + output_field: Arc::new(field.clone()), + } + } + + fn try_new(field: &LanceField, input_field: &ArrowField) -> Result { + if !field_contains_blob_v2(field) { + return Ok(Self::passthrough(input_field)); } - let struct_arr = batch - .column(col_idx) - .as_any() - .downcast_ref::() - .ok_or_else(|| { + if field.is_blob_v2() { + let field_id = u32::try_from(field.id).map_err(|_| { Error::internal(format!( - "Blob v2 column '{}' expected StructArray, got {:?}", - field.name(), - batch.column(col_idx).data_type() + "Blob v2 field id {} for '{}' does not fit in u32", + field.id, field.name )) })?; + let ArrowDataType::Struct(input_children) = input_field.data_type() else { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has non-struct input type {:?}", + field.name, + input_field.data_type() + ))); + }; + let output_field = match BlobV2Layout::classify(input_children) { + Some(BlobV2Layout::Logical) => Arc::new(input_field.clone()), + Some(BlobV2Layout::Descriptor) => { + transformed_arrow_field(field, BLOB_V2_LOGICAL_TYPE.clone()) + } + Some(actual) => { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has {actual} input layout; expected logical or descriptor layout during rewrite", + field.name + ))); + } + None => { + return Err(Error::invalid_input(format!( + "Blob v2 field '{}' has unrecognized input layout {:?}; expected logical or descriptor layout during rewrite", + field.name, input_children + ))); + } + }; + return Ok(Self::Blob { + field_id, + field_name: field.name.clone(), + output_field, + }); + } + + match (field.data_type(), input_field.data_type()) { + (ArrowDataType::Struct(_), ArrowDataType::Struct(input_children)) => { + if field.children.len() != input_children.len() { + return Err(Error::internal(format!( + "Struct field '{}' expected {} children in blob rewrite plan, got {}", + field.name, + field.children.len(), + input_children.len() + ))); + } + let children = field + .children + .iter() + .zip(input_children.iter()) + .map(|(child, input_child)| Self::try_new(child, input_child)) + .collect::>>()?; + let output_children = children + .iter() + .map(|child| child.output_field().clone()) + .collect::>(); + Ok(Self::Struct { + field_name: field.name.clone(), + output_field: arrow_field_with_data_type( + input_field, + ArrowDataType::Struct(output_children.into()), + ), + children, + }) + } + (ArrowDataType::List(_), ArrowDataType::List(input_child)) => { + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "List field '{}' is missing its child in blob rewrite plan", + field.name + )) + })?; + let child = Box::new(Self::try_new(child, input_child)?); + Ok(Self::List { + field_name: field.name.clone(), + output_field: arrow_field_with_data_type( + input_field, + ArrowDataType::List(child.output_field().clone()), + ), + child, + }) + } + (ArrowDataType::LargeList(_), ArrowDataType::LargeList(input_child)) => { + let child = field.children.first().ok_or_else(|| { + Error::internal(format!( + "Large list field '{}' is missing its child in blob rewrite plan", + field.name + )) + })?; + let child = Box::new(Self::try_new(child, input_child)?); + Ok(Self::LargeList { + field_name: field.name.clone(), + output_field: arrow_field_with_data_type( + input_field, + ArrowDataType::LargeList(child.output_field().clone()), + ), + child, + }) + } + (logical_type, input_type) => Err(Error::invalid_input(format!( + "Field '{}' contains blob v2 descendants but logical type {:?} and input type {:?} do not form a supported rewrite container", + field.name, logical_type, input_type + ))), + } + } - let column_name = field.name(); - let descriptor = BlobV2Descriptor::try_from_struct(struct_arr, column_name)?; - let classification = classify_rows(struct_arr, &descriptor, row_addrs, column_name)?; - let num_rows = struct_arr.len(); + fn output_field(&self) -> &Arc { + match self { + Self::Passthrough { output_field } + | Self::Blob { output_field, .. } + | Self::Struct { output_field, .. } + | Self::List { output_field, .. } + | Self::LargeList { output_field, .. } => output_field, + } + } - let new_struct = build_user_view_struct( - dataset, - &descriptor, - &classification, - column_name, - num_rows, - struct_arr.nulls().cloned(), - ) - .await?; + fn transform<'a>( + &'a self, + dataset: &'a Arc, + array: ArrayRef, + row_addrs: Arc<[u64]>, + ) -> BoxFuture<'a, Result> { + async move { + match self { + Self::Passthrough { .. } => Ok(array), + Self::Blob { + field_id, + field_name, + .. + } => { + let struct_arr = array.as_struct(); + match BlobV2Layout::classify(struct_arr.fields()) { + Some(BlobV2Layout::Logical) => Ok(array), + Some(BlobV2Layout::Descriptor) => { + let descriptor = + BlobV2Descriptor::try_from_struct(struct_arr, field_name)?; + let classification = + classify_rows(struct_arr, &descriptor, field_name)?; + let logical = descriptor_to_logical_blob_array( + dataset, + *field_id, + struct_arr, + &descriptor, + &classification, + row_addrs.as_ref(), + field_name, + ) + .await?; + Ok(Arc::new(logical) as ArrayRef) + } + Some(actual) => Err(Error::invalid_input(format!( + "Blob v2 field '{}' has {actual} layout; expected logical or descriptor layout during rewrite", + field_name + ))), + None => Err(Error::invalid_input(format!( + "Blob v2 field '{}' has unrecognized layout {:?}; expected logical or descriptor layout during rewrite", + field_name, + struct_arr.fields() + ))), + } + } + Self::Struct { + field_name, + output_field, + children, + } => { + let struct_arr = array.as_struct().normalize_slicing()?; + let parent_nulls = struct_arr.nulls().cloned(); + let struct_arr = struct_arr.pushdown_nulls()?; + if children.len() != struct_arr.num_columns() { + return Err(Error::internal(format!( + "Struct field '{}' expected {} children during blob rewrite, got {}", + field_name, + children.len(), + struct_arr.num_columns() + ))); + } + let mut child_arrays = Vec::with_capacity(children.len()); + for (child, child_array) in children.iter().zip(struct_arr.columns()) { + child_arrays.push( + child + .transform(dataset, child_array.clone(), row_addrs.clone()) + .await?, + ); + } + let ArrowDataType::Struct(output_fields) = output_field.data_type() else { + return Err(Error::internal(format!( + "Struct field '{}' rewrite plan has non-struct output type {:?}", + field_name, + output_field.data_type() + ))); + }; + Ok(Arc::new(StructArray::try_new( + output_fields.clone(), + child_arrays, + parent_nulls, + )?) as ArrayRef) + } + Self::List { + field_name, child, .. + } => { + transform_blob_v2_list_array::( + dataset, + field_name, + child, + array.as_list::(), + row_addrs, + ) + .await + } + Self::LargeList { + field_name, child, .. + } => { + transform_blob_v2_list_array::( + dataset, + field_name, + child, + array.as_list::(), + row_addrs, + ) + .await + } + } + } + .boxed() + } +} - new_columns.push(Arc::new(new_struct)); - let logical_field = arrow_schema::Field::from(lance_field.ok_or_else(|| { +async fn transform_blob_v2_list_array( + dataset: &Arc, + field_name: &str, + child: &BlobV2FieldRewritePlan, + list_array: &GenericListArray, + row_addrs: Arc<[u64]>, +) -> Result { + let list_array = if list_array.null_count() > 0 { + list_array.filter_garbage_nulls() + } else { + list_array.clone() + }; + let offsets = list_array.value_offsets(); + let values_start = offsets[0].as_usize(); + let values_end = offsets[list_array.len()].as_usize(); + if values_end < values_start { + return Err(Error::internal(format!( + "List field '{}' has invalid offsets during blob rewrite", + field_name + ))); + } + + let values_len = values_end - values_start; + let mut normalized_offsets = Vec::with_capacity(list_array.len() + 1); + normalized_offsets.push(O::usize_as(0)); + let mut child_row_addrs = Vec::with_capacity(values_len); + for row_idx in 0..list_array.len() { + let start = offsets[row_idx].as_usize(); + let end = offsets[row_idx + 1].as_usize(); + if end < start { + return Err(Error::internal(format!( + "List field '{}' has decreasing offsets during blob rewrite", + field_name + ))); + } + let row_addr = row_addrs.get(row_idx).copied().ok_or_else(|| { Error::internal(format!( - "Blob v2 column '{}' missing from dataset schema during compaction", - field.name() + "List field '{}' row address count {} did not match row count {}", + field_name, + row_addrs.len(), + list_array.len() )) - })?); - new_fields.push(Arc::new( - arrow_schema::Field::new( - field.name(), - lance_core::datatypes::BLOB_V2_USER_TYPE.clone(), - field.is_nullable(), - ) - .with_metadata(logical_field.metadata().clone()), - )); + })?; + child_row_addrs.extend(std::iter::repeat_n(row_addr, end - start)); + normalized_offsets.push(O::usize_as(end - values_start)); + } + + let values = list_array.values().slice(values_start, values_len); + let values = child + .transform(dataset, values, Arc::<[u64]>::from(child_row_addrs)) + .await?; + let list_array = GenericListArray::::try_new( + child.output_field().clone(), + OffsetBuffer::new(ScalarBuffer::from(normalized_offsets)), + values, + list_array.nulls().cloned(), + )?; + Ok(Arc::new(list_array)) +} + +pub(crate) struct BlobV2BatchRewritePlan { + row_addr_idx: usize, + columns: Vec<(usize, BlobV2FieldRewritePlan)>, + output_schema: SchemaRef, +} + +impl BlobV2BatchRewritePlan { + pub(crate) fn try_new( + schema: &lance_core::datatypes::Schema, + input_schema: &ArrowSchema, + keep_row_addr: bool, + ) -> Result { + let row_addr_idx = input_schema + .column_with_name(lance_core::ROW_ADDR) + .ok_or_else(|| { + Error::internal(format!( + "_rowaddr column missing from batch for blob v2 rewrite, columns: {:?}", + input_schema + .fields() + .iter() + .map(|f| f.name()) + .collect::>() + )) + })? + .0; + let mut columns = Vec::with_capacity(input_schema.fields().len()); + let mut output_fields = Vec::with_capacity(input_schema.fields().len()); + for (column_idx, input_field) in input_schema.fields().iter().enumerate() { + if input_field.name() == lance_core::ROW_ADDR && !keep_row_addr { + continue; + } + let field_plan = if let Some(field) = schema.field(input_field.name()) { + BlobV2FieldRewritePlan::try_new(field, input_field)? + } else { + BlobV2FieldRewritePlan::passthrough(input_field) + }; + output_fields.push(field_plan.output_field().as_ref().clone()); + columns.push((column_idx, field_plan)); + } + + Ok(Self { + row_addr_idx, + columns, + output_schema: Arc::new(ArrowSchema::new_with_metadata( + output_fields, + input_schema.metadata().clone(), + )), + }) + } + + pub(crate) fn output_schema(&self) -> &SchemaRef { + &self.output_schema } - let new_schema = Arc::new(arrow_schema::Schema::new_with_metadata( - new_fields + pub(crate) async fn transform_batch( + &self, + dataset: &Arc, + batch: RecordBatch, + ) -> Result { + let row_addrs: Arc<[u64]> = batch + .column(self.row_addr_idx) + .as_primitive::() + .values() .iter() - .map(|f| f.as_ref().clone()) - .collect::>(), - batch_schema.metadata().clone(), - )); + .copied() + .collect::>() + .into(); + let mut output_columns = Vec::with_capacity(self.columns.len()); + for (column_idx, field_plan) in &self.columns { + output_columns.push( + field_plan + .transform( + dataset, + batch.column(*column_idx).clone(), + row_addrs.clone(), + ) + .await?, + ); + } + Ok(RecordBatch::try_new( + self.output_schema.clone(), + output_columns, + )?) + } +} - Ok(RecordBatch::try_new(new_schema, new_columns)?) +pub(crate) async fn transform_blob_v2_batch( + dataset: &Arc, + schema: &lance_core::datatypes::Schema, + batch: RecordBatch, + keep_row_addr: bool, +) -> Result { + let plan = BlobV2BatchRewritePlan::try_new(schema, batch.schema().as_ref(), keep_row_addr)?; + plan.transform_batch(dataset, batch).await } /// Build a scan reader for rewrite and optionally capture row IDs. @@ -1417,64 +2053,182 @@ impl CandidateBin { } /// Split into one or more bins with at least `min_num_rows` in them. - fn split_for_size(mut self, min_num_rows: usize) -> Vec { - let mut bins = Vec::new(); - - loop { - let mut bin_len = 0; - let mut bin_row_count = 0; - while bin_row_count < min_num_rows && bin_len < self.row_counts.len() { - bin_row_count += self.row_counts[bin_len]; - bin_len += 1; + fn split_for_size(self, min_num_rows: usize) -> Vec { + let total_rows = self.row_counts.iter().sum::(); + let mut remaining_rows = total_rows; + let mut current_rows = 0; + let mut current_len = 0; + let mut split_lengths = Vec::new(); + + for row_count in &self.row_counts { + current_rows += *row_count; + current_len += 1; + remaining_rows -= *row_count; + + // Only split once the current bin is large enough and there is + // enough left over to form another worthwhile non-empty bin. + if current_rows >= min_num_rows && remaining_rows > 0 && remaining_rows >= min_num_rows + { + split_lengths.push(current_len); + current_rows = 0; + current_len = 0; } + } - // If there's enough remaining to make another worthwhile bin, then - // push what we have as a bin. - if self.row_counts[bin_len..].iter().sum::() >= min_num_rows { - bins.push(Self { - fragments: self.fragments.drain(0..bin_len).collect(), - pos_range: self.pos_range.start..(self.pos_range.start + bin_len), - candidacy: self.candidacy.drain(0..bin_len).collect(), - row_counts: self.row_counts.drain(0..bin_len).collect(), - // By the time we are splitting for size we are done considering indices - indices: Vec::new(), - }); - self.pos_range.start += bin_len; - } else { - // Otherwise, just push the remaining fragments into the last bin - bins.push(self); - break; - } + if split_lengths.is_empty() { + return vec![self]; } + let mut bins = Vec::with_capacity(split_lengths.len() + 1); + let mut fragments = self.fragments.into_iter(); + let mut candidacy = self.candidacy.into_iter(); + let mut row_counts = self.row_counts.into_iter(); + let mut pos_start = self.pos_range.start; + + for bin_len in split_lengths { + bins.push(Self { + fragments: fragments.by_ref().take(bin_len).collect(), + pos_range: pos_start..(pos_start + bin_len), + candidacy: candidacy.by_ref().take(bin_len).collect(), + row_counts: row_counts.by_ref().take(bin_len).collect(), + // By the time we are splitting for size we are done considering indices + indices: Vec::new(), + }); + pos_start += bin_len; + } + + bins.push(Self { + fragments: fragments.collect(), + pos_range: pos_start..self.pos_range.end, + candidacy: candidacy.collect(), + row_counts: row_counts.collect(), + indices: self.indices, + }); + bins } } async fn load_index_fragmaps(dataset: &Dataset) -> Result> { - let indices = dataset.load_indices().await?; + // Coverage, not usability: these bitmaps decide the rewrite groups. Under + // stable row ids `Transaction::recalculate_fragment_bitmap` then rejects any + // group that splits an index's coverage, and it walks every index the new + // manifest carries - including the ones this build cannot read. Binning from + // the filtered view fails that check outright on a dataset holding an index + // written by a newer Lance. The same bitmaps also decide, through + // `any_group_indexed`, whether a deferred compaction writes the + // fragment-reuse index that a build which can read that index needs to + // repair its coverage. + let indices = load_all_indices(dataset).await?; let mut index_fragmaps = Vec::with_capacity(indices.len()); // System indices (fragment-reuse, mem-wal) don't define data coverage and // aren't remapped per rewrite group, so they must not constrain compaction // bins -- otherwise deferred compaction's fragment-reuse index repeatedly // splits the small-fragment run and they never coalesce. for index in indices.iter().filter(|idx| !is_system_index(idx)) { - if let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() { - index_fragmaps.push(fragment_bitmap.clone()); - } else { - let dataset_at_index = dataset.checkout_version(index.dataset_version).await?; - let frags = 0..dataset_at_index.manifest.max_fragment_id.unwrap_or(0); - index_fragmaps.push(RoaringBitmap::from_sorted_iter(frags).unwrap()); - } + index_fragmaps.push(index_fragment_coverage(dataset, index).await?); } Ok(index_fragmaps) } +/// The fragments an index segment covers, reconstructing the coverage of a +/// legacy segment that predates the bitmap from the dataset it was written +/// against. +async fn index_fragment_coverage( + dataset: &Dataset, + index: &IndexMetadata, +) -> Result { + if let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() { + return Ok(fragment_bitmap.clone()); + } + let dataset_at_index = dataset.checkout_version(index.dataset_version).await?; + // max_fragment_id is inclusive (the highest id); +1 for an exclusive + // upper bound so the last fragment is covered (None => empty range). + let frags = 0..dataset_at_index + .manifest + .max_fragment_id + .map_or(0, |m| m + 1); + let mut coverage = RoaringBitmap::from_sorted_iter(frags).unwrap(); + // Reconstructed in the id space of the version the index was written + // against, which a later compaction has already moved on from. + // `load_all_indices` puts a stored bitmap into the current space by running + // it through the fragment-reuse index and leaves a `None` one alone, so a + // reconstruction has to take that step itself. Skipping it names the + // fragments a deferred compaction moved these rows out of, which is a set + // no rewrite can intersect - the guards below then wave through the rewrite + // of the fragment the rows actually live in. + if let Some(frag_reuse_index) = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await? { + frag_reuse_index.remap_fragment_bitmap(&mut coverage)?; + } + Ok(coverage) +} + +/// Each index this build has no reader for, by name, and the fragments it covers. +/// +/// A rewrite moves every row address in the fragments it touches, and putting an +/// index back in step means opening it. A build that cannot open one cannot +/// remap it, so compacting the fragments it covers would leave it addressing +/// rows that are gone -- worse than the erase this whole path exists to prevent. +/// They are held out of the plan instead, and the rest of the table still +/// compacts. +/// +/// Only the eager remap path needs this. Stable row ids keep the addresses +/// across a rewrite, and `defer_index_remap` hands the repair to a build that +/// can read the index, through the fragment-reuse index it writes. +async fn unremappable_index_coverage(dataset: &Dataset) -> Result> { + let mut coverage = Vec::new(); + for index in load_all_indices(dataset).await?.iter() { + if index_is_usable(index) { + continue; + } + coverage.push(( + index.name.clone(), + index_fragment_coverage(dataset, index).await?, + )); + } + Ok(coverage) +} + +/// Refuse a plan that rewrites fragments an index this build cannot read covers. +/// +/// [`DefaultCompactionPlanner`] keeps those fragments out of the plan, but +/// nothing forces a caller through it: `compact_files_with_planner` takes any +/// planner, [`CompactionPlan`] is public and serializable, and a distributed +/// driver hands [`commit_compaction`] results planned elsewhere. Committing such +/// a plan strands the index on fragment ids the rewrite deleted, so the commit +/// boundary refuses it rather than the planner alone. +async fn reject_unremappable_rewrite( + dataset: &Dataset, + completed_tasks: &[RewriteResult], +) -> Result<()> { + let rewritten = completed_tasks + .iter() + .flat_map(|task| task.original_fragments.iter()) + .filter_map(|fragment| u32::try_from(fragment.id).ok()) + .collect::(); + + for (name, covered) in unremappable_index_coverage(dataset).await? { + let blocked = covered & &rewritten; + if !blocked.is_empty() { + return Err(Error::invalid_input(format!( + "compaction would rewrite fragment(s) {:?}, which index {:?} covers. This build \ + has no reader for that index, so it cannot be remapped onto the rewritten \ + fragments and the commit would leave it addressing rows that no longer exist. \ + Plan with DefaultCompactionPlanner, which holds those fragments back, set \ + defer_index_remap, or compact from a build that can read the index.", + blocked.iter().collect::>(), + name, + ))); + } + } + Ok(()) +} + pub async fn plan_compaction( dataset: &Dataset, options: &CompactionOptions, ) -> Result { - let planner = DefaultCompactionPlanner::new(options.clone()); + let planner = DefaultCompactionPlanner::new(options.clone())?; planner.plan(dataset).await } @@ -1519,6 +2273,7 @@ async fn reserve_fragment_ids( &transaction, &Default::default(), &Default::default(), + DEFAULT_COMMIT_RETRY_TIMEOUT, dataset.manifest_location.naming_scheme, None, ) @@ -1570,8 +2325,13 @@ async fn rewrite_files( .iter() .map(|f| f.physical_rows.unwrap() as u64) .sum::(); - // If we aren't using stable row ids, then we need to remap indices. - let needs_remapping = !dataset.manifest.uses_stable_row_ids(); + // Capturing row addresses is only useful if something will consume them: + // an index to remap now, or a deferred remap through the FRI. + let capture_row_addrs = !dataset.manifest.uses_stable_row_ids() + && (options.defer_index_remap + || load_indices_for_remapping(dataset.as_ref()) + .await? + .is_some()); let mut new_fragments: Vec; let task_id = uuid::Uuid::new_v4(); log::info!( @@ -1597,7 +2357,7 @@ async fn rewrite_files( options.batch_size, options.io_buffer_size, true, - needs_remapping, + capture_row_addrs, ) .await?; row_ids_rx = rx_initial; @@ -1616,46 +2376,23 @@ async fn rewrite_files( if has_blob_v2_columns { let dataset_arc = Arc::new(dataset.as_ref().clone()); - let dataset_schema = dataset.schema().clone(); + let rewrite_plan = Arc::new(BlobV2BatchRewritePlan::try_new( + dataset.schema(), + schema.as_ref(), + false, + )?); + let transformed_schema = rewrite_plan.output_schema.clone(); let transformed = reader_with_progress.then(move |batch_result| { let dataset = dataset_arc.clone(); - let schema = dataset_schema.clone(); + let rewrite_plan = rewrite_plan.clone(); async move { let batch = batch_result?; - transform_blob_v2_batch(&dataset, &schema, batch) + rewrite_plan + .transform_batch(&dataset, batch) .await .map_err(|e| datafusion::error::DataFusionError::External(Box::new(e))) } }); - let transformed_schema = { - let mut fields: Vec> = Vec::new(); - for field in schema.fields().iter() { - if field.name() == lance_core::ROW_ADDR { - continue; - } - let lance_field = dataset.schema().field(field.name()); - if let Some(lance_field) = lance_field.filter(|f| f.is_blob_v2()) { - let logical_field = arrow_schema::Field::from(lance_field); - fields.push(Arc::new( - arrow_schema::Field::new( - field.name(), - lance_core::datatypes::BLOB_V2_USER_TYPE.clone(), - field.is_nullable(), - ) - .with_metadata(logical_field.metadata().clone()), - )); - } else { - fields.push(field.clone()); - } - } - Arc::new(arrow_schema::Schema::new_with_metadata( - fields - .iter() - .map(|f| f.as_ref().clone()) - .collect::>(), - schema.metadata().clone(), - )) - }; reader = Some(Box::pin(RecordBatchStreamAdapter::new( transformed_schema, transformed, @@ -1668,8 +2405,54 @@ async fn rewrite_files( } } + let surviving_rows = fragments.iter().try_fold(0_u64, |total, fragment| { + let fragment_rows = fragment.num_rows().ok_or_else(|| { + Error::internal(format!( + "Fragment {} is missing row count metadata after migration", + fragment.id + )) + })?; + total.checked_add(fragment_rows as u64).ok_or_else(|| { + Error::internal("Compaction task surviving row count overflowed u64".to_string()) + }) + })?; + + // Planner-sized tasks may exceed the target, but should remain one output + // instead of producing a target-sized fragment plus a stranded tail. For + // genuinely oversized tasks, choose a target-scale output count and spread + // the tail across those outputs. + let target_rows_per_fragment = options.target_rows_per_fragment as u64; + let output_fragment_count = surviving_rows + .checked_div(target_rows_per_fragment) + .unwrap_or(1) + .max(1); + let output_fragment_count_usize = usize::try_from(output_fragment_count).map_err(|_| { + Error::internal(format!( + "Compaction output fragment count {output_fragment_count} does not fit in usize" + )) + })?; + let base_rows_per_file = surviving_rows / output_fragment_count; + let larger_file_count = + usize::try_from(surviving_rows % output_fragment_count).map_err(|_| { + Error::internal("Compaction larger output fragment count does not fit in usize") + })?; + let file_row_counts = if surviving_rows == 0 { + Vec::new() + } else { + (0..output_fragment_count_usize) + .map(|file_index| { + let file_rows = base_rows_per_file + u64::from(file_index < larger_file_count); + usize::try_from(file_rows).map_err(|_| { + Error::internal(format!( + "Compaction output row count {file_rows} does not fit in usize" + )) + }) + }) + .collect::>>()? + }; + let max_rows_per_file = file_row_counts.first().copied().unwrap_or(1); let mut params = WriteParams { - max_rows_per_file: options.target_rows_per_fragment, + max_rows_per_file, max_rows_per_group: options.max_rows_per_group, mode: WriteMode::Append, // External blobs may reference URIs outside the dataset's base_paths @@ -1687,7 +2470,9 @@ async fn rewrite_files( } if can_binary_copy { - new_fragments = rewrite_files_binary_copy( + let version = dataset.manifest.data_storage_format.lance_file_format(); + new_fragments = versions::rewrite_files_binary_copy( + version, dataset.as_ref(), &fragments, ¶ms, @@ -1701,7 +2486,7 @@ async fn rewrite_files( )); } - if needs_remapping { + if capture_row_addrs { let (tx, rx) = std::sync::mpsc::channel(); let mut addrs = RoaringTreemap::new(); for frag in &fragments { @@ -1720,7 +2505,8 @@ async fn rewrite_files( row_ids_rx = Some(rx); } } else { - let (frags, _) = write_fragments_internal( + let (frags, _) = write_fragments_internal_with_file_row_counts( + dataset.manifest.data_storage_format.lance_file_format(), Some(dataset.as_ref()), dataset.object_store.clone(), &dataset.base, @@ -1728,6 +2514,7 @@ async fn rewrite_files( reader.expect("reader must be prepared for non-binary-copy path"), params, None, + Some(file_row_counts), ) .await?; new_fragments = frags; @@ -1742,7 +2529,11 @@ async fn rewrite_files( let captured_ids = row_ids_rx .try_recv() .map_err(|err| Error::internal(format!("Failed to receive row ids: {}", err)))?; - let row_addrs = captured_ids.row_addrs(None).into_owned(); + let mut row_addrs = captured_ids.row_addrs(None)?.into_owned(); + // Compaction reads whole fragments, so the captured addresses are + // dense per-fragment ranges; run containers (standard roaring + // format) shrink the persisted blob from O(rows) to O(runs) bytes. + row_addrs.optimize(); let mut serialized = Vec::with_capacity(row_addrs.serialized_size()); row_addrs.serialize_into(&mut serialized)?; Ok(Some(serialized)) @@ -1850,7 +2641,7 @@ async fn rechunk_stable_row_ids( for (fragment, sequence) in new_fragments.iter_mut().zip(new_sequences) { // TODO: if large enough, serialize to separate file let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } Ok(()) @@ -1941,8 +2732,8 @@ async fn recalc_versions_for_rewritten_fragments( // Set both version metadata on new fragments for ((fragment, last_updated_seq), created_at_seq) in new_fragments .iter_mut() - .zip(new_last_updated_sequences.into_iter()) - .zip(new_created_at_sequences.into_iter()) + .zip(new_last_updated_sequences) + .zip(new_created_at_sequences) { fragment.last_updated_at_version_meta = Some( lance_table::format::RowDatasetVersionMeta::from_sequence(&last_updated_seq).unwrap(), @@ -1971,8 +2762,25 @@ pub async fn commit_compaction( return Ok(CompactionMetrics::default()); } - // If we aren't using stable row ids, then we need to remap indices. - let needs_remapping = !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap; + // Before anything is written or committed. The condition is the planner's, + // not `has_address_style`: a dataset whose only index is one this build + // cannot read captures no row addresses at all, which is exactly the plan + // that has to be refused here. + if !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap { + reject_unremappable_rewrite(dataset, &completed_tasks).await?; + } + + let has_address_style = completed_tasks.iter().any(|t| t.row_addrs.is_some()); + // Address-style results require immediate index remapping unless it is deferred. + let needs_remapping = + !dataset.manifest.uses_stable_row_ids() && !options.defer_index_remap && has_address_style; + + // Confirm there is a remapper before materializing the potentially very large row address map. + let index_remapper = if needs_remapping { + remap_options.create_remapper(dataset).await? + } else { + None + }; // Determine the earliest version at which compaction tasks were planned/executed. // @@ -1995,15 +2803,26 @@ pub async fn commit_compaction( let mut completed_tasks = completed_tasks; + // Collect the rewritten fragments' file paths up front so every failure + // path below can clean them up (or deliberately keep them). Fragment ids + // may still be reassigned by reserve_fragment_ids; cleanup only needs the + // file paths, which never change. + let all_new_fragments: Vec = completed_tasks + .iter() + .flat_map(|t| t.new_fragments.iter().cloned()) + .collect(); + // Single reserve_fragment_ids for all address-style tasks - let has_address_style = completed_tasks.iter().any(|t| t.row_addrs.is_some()); if has_address_style { let frags: Vec<&mut Fragment> = completed_tasks .iter_mut() .filter(|t| t.row_addrs.is_some()) .flat_map(|t| t.new_fragments.iter_mut()) .collect(); - reserve_fragment_ids(dataset, frags.into_iter()).await?; + if let Err(e) = reserve_fragment_ids(dataset, frags.into_iter()).await { + cleanup_compaction_files_after_reservation_failure(dataset, &all_new_fragments).await; + return Err(e); + } } let mut rewrite_groups = Vec::with_capacity(completed_tasks.len()); @@ -2014,6 +2833,31 @@ pub async fn commit_compaction( let mut frag_reuse_groups: Vec = Vec::new(); let mut new_fragment_bitmap: RoaringBitmap = RoaringBitmap::new(); + // Write an FRI only when the compaction touches data an index must later + // remap: a rewrite group covered by a data index, or by the existing FRI's new + // fragments (the composed remap chain). Compacting only not-yet-indexed data + // needs no FRI (one written for it is un-drainable). Decide all-or-nothing per + // compaction, never per group -- a partial FRI is unsound: a concurrent reindex + // can make a skipped fragment indexed and the conflict resolver's FRI-present + // path won't re-check it. + let indexed_frags: RoaringBitmap = if options.defer_index_remap { + let mut covered = RoaringBitmap::new(); + for bm in load_index_fragmaps(dataset).await? { + covered |= bm; + } + if let Some(bm) = dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await? + .and_then(|fri| fri.fragment_bitmap) + { + covered |= bm; + } + covered + } else { + RoaringBitmap::new() + }; + let mut any_group_indexed = false; + for task in completed_tasks { metrics += task.metrics; let rewrite_group = RewriteGroup { @@ -2021,7 +2865,7 @@ pub async fn commit_compaction( new_fragments: task.new_fragments.clone(), }; - if needs_remapping { + if index_remapper.is_some() { if let Some(row_addrs_bytes) = task.row_addrs { let row_addrs = RoaringTreemap::deserialize_from(&mut Cursor::new(&row_addrs_bytes))?; @@ -2045,7 +2889,19 @@ pub async fn commit_compaction( f.id )) })?; - Ok((f.id as u32, physical_rows as u32)) + let fragment_id = u32::try_from(f.id).map_err(|_| { + Error::invalid_input(format!( + "compacted fragment id {} is outside the row-address range", + f.id + )) + })?; + let physical_rows = u32::try_from(physical_rows).map_err(|_| { + Error::invalid_input(format!( + "compacted fragment {} has physical_rows={} outside the row-address range", + f.id, physical_rows + )) + })?; + Ok((fragment_id, physical_rows)) }) .collect::>>()?; @@ -2054,14 +2910,29 @@ pub async fn commit_compaction( old_frag_ids: task .original_fragments .iter() - .map(|f| f.id as u32) - .collect(), + .map(|f| { + u32::try_from(f.id).map_err(|_| { + Error::invalid_input(format!( + "compacted source fragment id {} is outside the row-address range", + f.id + )) + }) + }) + .collect::>>()?, new_frags, }); } } } } else if options.defer_index_remap { + // Record every group; track whether any touches indexed/chain data. + if task + .original_fragments + .iter() + .any(|f| indexed_frags.contains(f.id as u32)) + { + any_group_indexed = true; + } let changed_row_addrs = task.row_addrs.ok_or_else(|| { Error::internal( "defer_index_remap requires row_addrs but none were provided".to_string(), @@ -2080,8 +2951,7 @@ pub async fn commit_compaction( rewrite_groups.push(rewrite_group); } - let rewritten_indices = if needs_remapping { - let index_remapper = remap_options.create_remapper(dataset)?; + let rewritten_indices = if let Some(index_remapper) = index_remapper { let affected_ids = rewrite_groups .iter() .flat_map(|group| group.old_fragments.iter().map(|frag| frag.id)) @@ -2110,25 +2980,27 @@ pub async fn commit_compaction( .iter_mut() .flat_map(|group| group.new_fragments.iter_mut()) .collect::>(); - reserve_fragment_ids(dataset, new_fragments.into_iter()).await?; + if let Err(e) = reserve_fragment_ids(dataset, new_fragments.into_iter()).await { + cleanup_compaction_files_after_reservation_failure(dataset, &all_new_fragments).await; + return Err(e); + } Vec::new() } else { Vec::new() }; - let frag_reuse_index = if options.defer_index_remap { + // No indexed/chain data touched -> no FRI (all-or-nothing, see above). + let frag_reuse_index = if options.defer_index_remap && any_group_indexed { Some(build_new_frag_reuse_index(dataset, frag_reuse_groups, new_fragment_bitmap).await?) } else { + if options.defer_index_remap { + log::debug!( + "skipping fragment-reuse index: no rewritten fragments were covered by an index" + ); + } None }; - // Collect new fragment paths before moving rewrite_groups into the transaction, - // so we can clean them up if the commit fails. - let all_new_fragments: Vec = rewrite_groups - .iter() - .flat_map(|g| g.new_fragments.iter().cloned()) - .collect(); - let transaction = TransactionBuilder::new( // Use the version at which the compaction tasks were *planned*, not the // version of the dataset handle passed to this function. In distributed @@ -2151,19 +3023,36 @@ pub async fn commit_compaction( .apply_commit(transaction, &Default::default(), &Default::default()) .await { - cleanup_data_fragments( - &dataset.object_store, - &dataset.base, - None, - &all_new_fragments, - ) - .await; + // RewriteResult is serializable and may be retried after an earlier + // ambiguous success. A conflict on this call therefore does not prove + // that the rewritten files are unreferenced. Leave them for dataset GC. + log::warn!( + "Compaction commit failed; leaving {} rewritten fragment(s) in place for GC: {}", + all_new_fragments.len(), + e + ); return Err(e); } Ok(metrics) } +/// Remove rewritten files after fragment-id reservation fails. Reservation +/// commits do not reference the rewritten files, so they are still owned by +/// this uncommitted compaction attempt and are safe to delete. +async fn cleanup_compaction_files_after_reservation_failure( + dataset: &Dataset, + all_new_fragments: &[Fragment], +) { + cleanup_data_fragments( + &dataset.object_store, + &dataset.base, + None, + all_new_fragments, + ) + .await; +} + #[cfg(test)] mod tests { @@ -2191,6 +3080,7 @@ mod tests { use lance_core::utils::tempfile::TempStrDir; use lance_datagen::Dimension; use lance_file::version::LanceFileVersion; + use lance_index::frag_reuse::CompactFragReuseIndexHandle; use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; use lance_index::scalar::{ BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams, @@ -2221,6 +3111,7 @@ mod tests { let fragment = Fragment { id: 0, files: vec![], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(0), @@ -2261,6 +3152,33 @@ mod tests { assert_eq!(split[0].pos_range, 0..2); assert_eq!(split[1].pos_range, 2..5); assert_eq!(split[2].pos_range, 5..8); + + let zero_min_split_bin = CandidateBin { + fragments: std::iter::repeat_n( + Fragment { + id: 0, + files: vec![], + overlays: vec![], + deletion_file: None, + row_id_meta: None, + physical_rows: Some(0), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }, + 3, + ) + .collect(), + pos_range: 0..3, + candidacy: std::iter::repeat_n(CompactionCandidacy::CompactItself, 3).collect(), + row_counts: vec![100, 200, 300], + indices: vec![], + }; + let split = zero_min_split_bin.split_for_size(0); + assert_eq!(split.len(), 3); + assert!(split.iter().all(|bin| !bin.fragments.is_empty())); + assert_eq!(split[0].pos_range, 0..1); + assert_eq!(split[1].pos_range, 1..2); + assert_eq!(split[2].pos_range, 2..3); } fn sample_data() -> RecordBatch { @@ -2273,6 +3191,20 @@ mod tests { .unwrap() } + /// Build (or, with `replace`, rebuild) a scalar index named "scalar" on `col`. + async fn create_scalar_index(dataset: &mut Dataset, col: &str, replace: bool) { + dataset + .create_index( + &[col], + IndexType::Scalar, + Some("scalar".into()), + &ScalarIndexParams::default(), + replace, + ) + .await + .unwrap(); + } + #[derive(Debug, Default, Clone, PartialEq)] struct MockIndexRemapperExpectation { expected: HashMap>, @@ -2355,9 +3287,10 @@ mod tests { } } + #[async_trait] impl IndexRemapperOptions for MockIndexRemapper { - fn create_remapper(&self, _: &Dataset) -> Result> { - Ok(Box::new(self.clone())) + async fn create_remapper(&self, _: &Dataset) -> Result>> { + Ok(Some(Box::new(self.clone()))) } } @@ -2447,6 +3380,169 @@ mod tests { assert_eq!(plan.tasks().len(), 0); } + fn list_data_files(uri: &str) -> std::collections::BTreeSet { + std::fs::read_dir(std::path::Path::new(uri).join("data")) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default() + } + + async fn execute_compaction_plan( + dataset: &Dataset, + options: &CompactionOptions, + ) -> Vec { + let plan = plan_compaction(dataset, options).await.unwrap(); + assert!(!plan.tasks.is_empty()); + let snapshot = dataset.clone(); + futures::stream::iter(plan.tasks) + .map(|task| rewrite_files(Cow::Borrowed(&snapshot), task, options)) + .buffer_unordered(1) + .try_collect() + .await + .unwrap() + } + + /// When the compaction commit's status is unknown (the commit errored and + /// verification was unavailable), the rewritten files must NOT be deleted: + /// if the commit landed they are referenced by the new version, and + /// deleting them would corrupt the table. + #[tokio::test] + async fn test_compaction_retry_after_ambiguous_success_preserves_live_files() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let data = sample_data(); + let num_rows = data.num_rows(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let write_params = WriteParams { + max_rows_per_file: 3_000, + enable_stable_row_ids: true, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, test_uri, Some(write_params)) + .await + .unwrap(); + let files_before = list_data_files(test_uri); + let options = CompactionOptions::default(); + let completed = execute_compaction_plan(&dataset, &options).await; + let serialized = serde_json::to_vec(&completed).unwrap(); + let retry_results: Vec = serde_json::from_slice(&serialized).unwrap(); + let rewritten_files = list_data_files(test_uri) + .difference(&files_before) + .cloned() + .collect::>(); + assert!(!rewritten_files.is_empty()); + + handler.fail_next_rewrite(AmbiguousFailure::LandAndError); + handler + .fail_resolve + .store(true, std::sync::atomic::Ordering::SeqCst); + let err = commit_compaction( + &mut dataset, + completed, + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .expect_err("unknown commit status must surface as an error"); + assert!( + err.is_commit_status_unknown(), + "expected CommitStatusUnknown, got: {:?}", + err + ); + + handler + .fail_resolve + .store(false, std::sync::atomic::Ordering::SeqCst); + + // Retrying the same serialized RewriteResult now conflicts with the + // Rewrite that already landed. The retry must not delete those files. + let retry_error = commit_compaction( + &mut dataset, + retry_results, + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .expect_err("the replayed rewrite must conflict with its landed predecessor"); + assert!( + matches!(retry_error, Error::RetryableCommitConflict { .. }), + "expected RetryableCommitConflict, got: {retry_error:?}" + ); + let files_after_retry = list_data_files(test_uri); + assert!( + rewritten_files + .iter() + .all(|file| files_after_retry.contains(file)), + "a replayed RewriteResult must not delete files referenced by the landed rewrite" + ); + + let ds = Dataset::open(test_uri).await.unwrap(); + assert_eq!(ds.count_rows(None).await.unwrap(), num_rows); + let scanned = ds.scan().try_into_batch().await.unwrap(); + assert_eq!(scanned.num_rows(), num_rows); + } + + /// A failed ReserveFragments commit cannot reference the rewritten files, + /// so both stable-row-id reservation paths must still clean them up. + #[tokio::test] + async fn test_compaction_cleans_up_files_when_fragment_reservation_fails() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let data = sample_data(); + let num_rows = data.num_rows(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let write_params = WriteParams { + max_rows_per_file: 3_000, + enable_stable_row_ids: true, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, test_uri, Some(write_params)) + .await + .unwrap(); + let files_before = list_data_files(test_uri); + let options = CompactionOptions::default(); + let completed = execute_compaction_plan(&dataset, &options).await; + assert!(list_data_files(test_uri).len() > files_before.len()); + + handler.fail_next_reserve(AmbiguousFailure::FailOutright); + let err = commit_compaction( + &mut dataset, + completed, + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .expect_err("fragment reservation that did not land must fail"); + assert!( + !err.is_commit_status_unknown(), + "a verified-absent reservation is a definite failure, got: {:?}", + err + ); + + let files_after = list_data_files(test_uri); + let leftover: Vec<_> = files_after.difference(&files_before).collect(); + assert!( + leftover.is_empty(), + "rewritten files must be cleaned up after reservation fails; leftover: {:?}", + leftover + ); + let ds = Dataset::open(test_uri).await.unwrap(); + assert_eq!(ds.count_rows(None).await.unwrap(), num_rows); + } + #[tokio::test] async fn test_compact_blob_columns() { let test_dir = TempStrDir::default(); @@ -2493,7 +3589,7 @@ mod tests { .unwrap(); assert_eq!(blobs.len(), expected_payload.len()); for (blob, expected) in blobs.iter().zip(expected_payload.iter()) { - let bytes = blob.read().await.unwrap(); + let bytes = blob.as_ref().unwrap().read().await.unwrap(); assert_eq!(bytes.as_ref(), expected.as_slice()); } } @@ -2588,49 +3684,44 @@ mod tests { .unwrap(); let first_new_frag_idx = 7; - // Predicting the remap is difficult. One task will remap to fragments 7/8 and the other - // will remap to fragments 9/10 but we don't know which is which and so we just allow ourselves - // to expect both possibilities. + // The tasks execute concurrently, so either one may reserve the first + // output fragment id. let remap_a = expect_remap( &[ vec![ - // 3 small fragments are rewritten to frags 7 & 8 + // 3 small fragments are rewritten to frag 7 (row_addrs(0, 0..400), true), (row_addrs(1, 0..400), true), - (row_addrs(2, 0..200), true), + (row_addrs(2, 0..400), true), ], - vec![(row_addrs(2, 200..400), true)], // frag 3 is skipped since it does not have enough missing data - // Frags 4, 5, and 6 are rewritten to frags 9 & 10 + // Frags 4, 5, and 6 are rewritten to frag 8 vec![ - // Only 800 of the 1000 rows taken from frag 4 (row_addrs(4, 0..200), true), (row_addrs(4, 200..400), false), (row_addrs(4, 400..1000), true), - // frags 5 compacted with frag 4 - (row_addrs(5, 0..200), true), + (row_addrs(5, 0..300), true), + (row_addrs(6, 0..300), true), ], - vec![(row_addrs(5, 200..300), true), (row_addrs(6, 0..300), true)], ], first_new_frag_idx, ); let remap_b = expect_remap( &[ - // Frags 4, 5, and 6 are rewritten to frags 7 & 8 + // Frags 4, 5, and 6 are rewritten to frag 7 vec![ (row_addrs(4, 0..200), true), (row_addrs(4, 200..400), false), (row_addrs(4, 400..1000), true), - (row_addrs(5, 0..200), true), + (row_addrs(5, 0..300), true), + (row_addrs(6, 0..300), true), ], - vec![(row_addrs(5, 200..300), true), (row_addrs(6, 0..300), true)], - // 3 small fragments rewritten to frags 9 & 10 + // 3 small fragments rewritten to frag 8 vec![ (row_addrs(0, 0..400), true), (row_addrs(1, 0..400), true), - (row_addrs(2, 0..200), true), + (row_addrs(2, 0..400), true), ], - vec![(row_addrs(2, 200..400), true)], ], first_new_frag_idx, ); @@ -2671,16 +3762,155 @@ mod tests { // Assert on metrics assert_eq!(metrics.fragments_removed, 6); - assert_eq!(metrics.fragments_added, 4); + assert_eq!(metrics.fragments_added, 2); assert_eq!(metrics.files_removed, 7); // 6 data files + 1 deletion file - assert_eq!(metrics.files_added, 4); + assert_eq!(metrics.files_added, 2); let fragment_ids = dataset .get_fragments() .iter() .map(|f| f.id()) .collect::>(); - assert_eq!(fragment_ids, vec![3, 7, 8, 9, 10]); + assert_eq!(fragment_ids, vec![3, 7, 8]); + } + + #[rstest] + #[tokio::test] + async fn test_compaction_does_not_strand_small_remainders( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) { + let test_dir = TempStrDir::default(); + let data = sample_data().slice(0, 2_000); + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let mut dataset = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + max_rows_per_file: 200, + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await + .unwrap(); + + let options = CompactionOptions { + target_rows_per_fragment: 500, + ..Default::default() + }; + let metrics = compact_files(&mut dataset, options.clone(), None) + .await + .unwrap(); + + assert_eq!(metrics.fragments_removed, 10); + assert_eq!(metrics.fragments_added, 3); + let mut fragment_sizes = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.metadata.physical_rows.unwrap()) + .collect::>(); + fragment_sizes.sort_unstable(); + assert_eq!(fragment_sizes, vec![600, 600, 800]); + + let second_metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert_eq!(second_metrics, CompactionMetrics::default()); + } + + #[rstest] + #[case::legacy(LanceFileVersion::Legacy)] + #[case::stable(LanceFileVersion::Stable)] + #[tokio::test] + async fn test_compaction_rebalances_oversized_task( + #[case] data_storage_version: LanceFileVersion, + ) { + let test_dir = TempStrDir::default(); + let data = sample_data().slice(0, 5_100); + let reader = RecordBatchIterator::new(vec![Ok(data.slice(0, 5_000))], data.schema()); + let mut dataset = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + max_rows_per_file: 5_000, + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(data.slice(5_000, 100))], data.schema()); + dataset.append(reader, None).await.unwrap(); + + dataset.delete("a < 1000").await.unwrap(); + + let options = CompactionOptions { + target_rows_per_fragment: 1_000, + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert_eq!(plan.tasks.len(), 1); + assert_eq!(plan.tasks[0].fragments.len(), 2); + + let metrics = compact_files(&mut dataset, options.clone(), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 2); + assert_eq!(metrics.fragments_added, 4); + assert_eq!( + dataset + .get_fragments() + .iter() + .map(|fragment| fragment.metadata.physical_rows.unwrap()) + .collect::>(), + vec![1_025; 4] + ); + + let second_metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert_eq!(second_metrics, CompactionMetrics::default()); + } + + #[tokio::test] + async fn test_compaction_balances_non_divisible_stable_task() { + let test_dir = TempStrDir::default(); + let data = sample_data().slice(0, 121); + let reader = RecordBatchIterator::new(vec![Ok(data.clone())], data.schema()); + let mut dataset = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + max_rows_per_file: 121, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.delete("a < 20").await.unwrap(); + + let options = CompactionOptions { + target_rows_per_fragment: 10, + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert_eq!(plan.tasks.len(), 1); + assert_eq!(plan.tasks[0].fragments.len(), 1); + + let metrics = compact_files(&mut dataset, options.clone(), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 1); + assert_eq!(metrics.fragments_added, 10); + assert_eq!( + dataset + .get_fragments() + .iter() + .map(|fragment| fragment.metadata.physical_rows.unwrap()) + .collect::>(), + [vec![11], vec![10; 9]].concat() + ); + + let second_metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert_eq!(second_metrics, CompactionMetrics::default()); } #[rstest] @@ -2886,9 +4116,82 @@ mod tests { } } + #[async_trait] impl IndexRemapperOptions for IgnoreRemap { - fn create_remapper(&self, _: &Dataset) -> Result> { - Ok(Box::new(Self {})) + async fn create_remapper(&self, _: &Dataset) -> Result>> { + Ok(None) + } + } + + #[rstest] + #[case::without_index(false)] + #[case::with_index(true)] + #[tokio::test] + async fn test_row_addrs_only_used_with_remappable_index(#[case] has_index: bool) { + let data = sample_data(); + let reader = RecordBatchIterator::new(vec![Ok(data.slice(0, 9_000))], data.schema()); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: 3_000, + data_storage_version: Some(LanceFileVersion::Legacy), + ..Default::default() + }), + ) + .await + .unwrap(); + + if has_index { + create_scalar_index(&mut dataset, "a", false).await; + } + + let options = CompactionOptions { + target_rows_per_fragment: 9_000, + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert_eq!(plan.tasks().len(), 1); + + let mut result = rewrite_files(Cow::Borrowed(&dataset), plan.tasks()[0].clone(), &options) + .await + .unwrap(); + assert_eq!(result.row_addrs.is_some(), has_index); + + if has_index { + let row_addrs_bytes = result + .row_addrs + .as_ref() + .expect("indexed compaction should capture row addresses"); + let row_addrs = + RoaringTreemap::deserialize_from(&mut Cursor::new(row_addrs_bytes)).unwrap(); + assert_eq!(row_addrs.len(), 9_000); + // The captured addresses are contiguous per-fragment ranges, so the + // persisted blob must be run-optimized: O(fragments) bytes, not + // O(rows). Without run containers this serializes at ~2 bytes per + // address (~18 KB here), so under one byte per address proves the + // run form was written. + assert!( + row_addrs_bytes.len() < row_addrs.len() as usize, + "serialized row addrs ({} bytes for {} addresses) should be \ + run-optimized before persisting", + row_addrs_bytes.len(), + row_addrs.len() + ); + } else { + // Simulate a stale worker result that captured row addresses before the + // dataset no longer needed a remapper. Invalid bytes ensure the commit + // does not attempt to deserialize or materialize the unused map. + result.row_addrs = Some(b"not a roaring treemap".to_vec()); + commit_compaction( + &mut dataset, + vec![result], + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 1); } } @@ -3076,6 +4379,77 @@ mod tests { assert_eq!(before_scalar_result, after_scalar_result); } + /// Regression test for https://github.com/lance-format/lance/issues/8076 + /// + /// A zone map or bloom filter index reports matches as physical row addresses, so + /// compaction invalidates it even under stable row ids. Reusing it for the rewritten + /// fragments made a filtered scan fail with an internal error (a fragment referenced + /// by the index no longer existed) or, once translation tolerated that, silently drop + /// every match. + #[rstest] + #[case::zone_map(BuiltinIndexType::ZoneMap, IndexType::ZoneMap)] + #[case::bloom_filter(BuiltinIndexType::BloomFilter, IndexType::BloomFilter)] + #[tokio::test] + async fn test_addr_domain_index_after_compaction_with_stable_row_ids( + #[case] builtin: BuiltinIndexType, + #[case] index_type: IndexType, + ) { + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned()))); + let mut dataset = Dataset::write( + data_gen.batch(200), + "memory://test/table", + Some(WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 100, // 2 fragments, so compaction has something to merge + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["i"], + index_type, + None, + &ScalarIndexParams::for_builtin(builtin), + false, + ) + .await + .unwrap(); + + compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + + // The index only knows the pre-compaction fragments, so it must not claim to + // cover the fragment they were rewritten into. + let live_fragments: RoaringBitmap = + dataset.fragments().iter().map(|f| f.id as u32).collect(); + let index = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|index| index.fields == vec![0]) + .expect("index must survive compaction") + .clone(); + assert!( + index + .effective_fragment_bitmap(&live_fragments) + .is_none_or(|covered| covered.is_empty()), + "compaction must not point an address-domain index at the fragments it wrote" + ); + + // Every fragment therefore falls back to a full scan, and the filter is answered + // in full. + let mut scanner = dataset.scan(); + scanner.filter("i > 0").unwrap(); + let matched = scanner.try_into_batch().await.unwrap(); + assert_eq!(matched.num_rows(), 199); + } + // Regression test for https://github.com/lancedb/lance/issues/6161 // When FragReuseIndexDetails exceeds 204800 bytes it is written to an external // file. Previously the file was silently dropped (temp file deleted) because @@ -3114,6 +4488,10 @@ mod tests { assert_eq!(dataset.get_fragments().len(), num_fragments); + // An FRI is only written for compactions that touch indexed data, so + // index the column being compacted. + create_scalar_index(&mut dataset, "i", false).await; + // Delete a few rows from each fragment so compaction has something to do. dataset.delete("i % 1000 = 0").await.unwrap(); @@ -3225,17 +4603,10 @@ mod tests { dataset.delete("i < 500").await.unwrap(); dataset2.delete("i < 500").await.unwrap(); - // Create a scalar index to check this is not touched - dataset - .create_index( - &["i"], - IndexType::Scalar, - Some("scalar".into()), - &ScalarIndexParams::default(), - false, - ) - .await - .unwrap(); + // Create the same scalar index on both datasets so deferred and immediate + // remapping are compared under the same conditions. + create_scalar_index(&mut dataset, "i", false).await; + create_scalar_index(&mut dataset2, "i", false).await; // Verify the initial state - no fragment reuse index should exist let initial_indices = dataset.load_indices().await.unwrap(); @@ -3369,7 +4740,9 @@ mod tests { open_frag_reuse_index(frag_reuse_index_meta.uuid, frag_reuse_details.as_ref()) .await .unwrap(); - let stats = frag_reuse_index.statistics().unwrap(); + let stats = CompactFragReuseIndexHandle(Arc::new(frag_reuse_index.clone())) + .statistics() + .unwrap(); assert_eq!( serde_json::to_string(&stats).unwrap(), dataset @@ -3427,6 +4800,52 @@ mod tests { assert_eq!(current_scalar_index.uuid, original_scalar_uuid); } + #[tokio::test] + async fn test_defer_index_remap_skips_fri_when_no_indexed_data() { + // A deferred compaction touching no indexed data must write no FRI -- + // such a version is un-drainable (remap no-ops, trim retains it forever). + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned()))); + + let mut dataset = Dataset::write( + data_gen.batch(600), + "memory://test/noindex", + Some(WriteParams { + max_rows_per_file: 100, // 6 small files -> compaction has work + ..Default::default() + }), + ) + .await + .unwrap(); + + // No index at all: nothing covers any fragment. + assert!(dataset.load_indices().await.unwrap().is_empty()); + let fragments_before = dataset.get_fragments().len(); + assert!(fragments_before > 1, "need multiple fragments to compact"); + + let options = CompactionOptions { + target_rows_per_fragment: 100_000, + defer_index_remap: true, + ..Default::default() + }; + compact_files(&mut dataset, options, None).await.unwrap(); + + // Compaction actually ran... + assert!( + dataset.get_fragments().len() < fragments_before, + "compaction should have merged fragments" + ); + // ...but no fragment-reuse index was created. + assert!( + dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await + .unwrap() + .is_none(), + "deferred compaction with no indexed data must not create an FRI" + ); + } + #[tokio::test] async fn test_defer_index_remap_multiple_compactions() { let mut data_gen = BatchGenerator::new() @@ -3446,6 +4865,10 @@ mod tests { .await .unwrap(); + // FRI is written only for compactions touching indexed data; index "i" so + // the successive deferred compactions build a chained fragment-reuse index. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -3485,26 +4908,113 @@ mod tests { .await .unwrap(); - // Verify the index has one version with the correct dataset version - assert_eq!( - frag_reuse_index - .details - .versions - .iter() - .map(|v| v.dataset_version) - .collect::>(), - compact_read_versions + // Verify the index has one version with the correct dataset version + assert_eq!( + frag_reuse_index + .details + .versions + .iter() + .map(|v| v.dataset_version) + .collect::>(), + compact_read_versions + ); + } + } + + #[tokio::test] + async fn test_defer_index_remap_mixed_records_all_groups() { + // All-or-nothing: a compaction touching any indexed data records the full + // FRI, including the unindexed group (a per-group filter would drop it). + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("i".to_owned()))); + let mut dataset = Dataset::write( + data_gen.batch(300), + "memory://test/mixed", + Some(WriteParams { + max_rows_per_file: 100, // 3 fragments + ..Default::default() + }), + ) + .await + .unwrap(); + + // Index the initial fragments, then append more that stay unindexed. + create_scalar_index(&mut dataset, "i", false).await; + Dataset::write( + data_gen.batch(300), + WriteDestination::Dataset(Arc::new(dataset.clone())), + Some(WriteParams { + max_rows_per_file: 100, // 3 more, unindexed + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + + // Fragments not covered by the scalar index are the "unindexed" ones. + let indexed: HashSet = dataset + .load_index_by_name("scalar") + .await + .unwrap() + .unwrap() + .fragment_bitmap + .unwrap() + .iter() + .collect(); + let unindexed_frags: Vec = dataset + .fragments() + .iter() + .map(|f| f.id) + .filter(|id| !indexed.contains(&(*id as u32))) + .collect(); + assert!( + !unindexed_frags.is_empty(), + "expected some unindexed fragments" + ); + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 100_000, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + + // All-or-nothing: because indexed fragments were compacted, the FRI is + // written AND records the unindexed group too (a per-group filter would + // have dropped it). + let fri_meta = dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await + .unwrap() + .expect("mixed compaction must write an FRI"); + let details = load_frag_reuse_index_details(&dataset, &fri_meta) + .await + .unwrap(); + let recorded_old: HashSet = details + .versions + .iter() + .flat_map(|v| v.old_frag_ids()) + .collect(); + for f in &unindexed_frags { + assert!( + recorded_old.contains(f), + "unindexed fragment {f} must be recorded in the FRI (all-or-nothing)" ); } } #[tokio::test] async fn test_deferred_compaction_not_split_by_frag_reuse_index() { - // A deferred compaction creates a fragment-reuse index covering its - // output. Later small fragments must still compact together with that - // (FRI-covered) output: the FRI is a system index and must not split the - // compaction bin. Without the fix the FRI-covered fragment is isolated, - // so only the new fragments merge and the count never returns to one. + // The fragment-reuse index is a system index and must be excluded from + // compaction bin planning; otherwise its covered fragment is isolated and + // the small fragments never coalesce back to one. let data = sample_data(); let test_dir = TempStrDir::default(); let test_uri = &test_dir; @@ -3526,6 +5036,11 @@ mod tests { ) .await .unwrap(); + + // Index "a" so the deferred compaction records an FRI (only written for + // compactions touching indexed data). The FRI is a system index and must + // still not split later compaction bins -- the property this test guards. + create_scalar_index(&mut dataset, "a", false).await; compact_files(&mut dataset, options.clone(), None) .await .unwrap(); @@ -3553,11 +5068,16 @@ mod tests { .unwrap(); assert_eq!(dataset.get_fragments().len(), 3); + // Reindex so every fragment is data-indexed -- then the FRI (a system + // index, correctly excluded from bin planning) is the only thing that + // could split the bin. + create_scalar_index(&mut dataset, "a", true).await; + compact_files(&mut dataset, options, None).await.unwrap(); assert_eq!( dataset.get_fragments().len(), 1, - "FRI-covered fragment must compact together with the new fragments" + "FRI (a system index) must not split the compaction bin; all fragments coalesce" ); } @@ -3881,6 +5401,9 @@ mod tests { .await .unwrap(); + // Index "i" so the deferred compaction touches indexed data and writes an FRI. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -3977,9 +5500,13 @@ mod tests { .unwrap(); let new_frags3 = frag_reuse_details3.versions.last().unwrap().new_frag_ids(); - // Concurrently commit a frag_reuse_index cleanup operation. - // Because there is no index, it should remove the first version. - // but after rebase it should contain the new compaction versions. + // Concurrently commit a frag_reuse_index cleanup operation. dataset_clone + // only knows the first reuse version; catch its index up so the cleanup + // removes that version. After rebase onto the other compactions it should + // contain the new compaction versions. + remapping::remap_column_index(&mut dataset_clone, &["i"], Some("scalar".into())) + .await + .unwrap(); cleanup_frag_reuse_index(&mut dataset_clone).await.unwrap(); // Load and verify the fragment reuse index content @@ -4019,6 +5546,9 @@ mod tests { .await .unwrap(); + // Index "i" so the deferred compaction touches indexed data and writes an FRI. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -4057,8 +5587,12 @@ mod tests { .unwrap(); assert_eq!(frag_reuse_details.versions.len(), 1); - // First commit the frag_reuse_index cleanup - // Because there is no index, it should remove the first version. + // Catch the index up to the compaction (on `dataset` only; `dataset_clone` + // keeps the un-caught-up index for the concurrent rewrite below), then + // clean up: with the index caught up the trim removes the first version. + remapping::remap_column_index(&mut dataset, &["i"], Some("scalar".into())) + .await + .unwrap(); cleanup_frag_reuse_index(&mut dataset).await.unwrap(); // Load and verify the fragment reuse index content @@ -4129,6 +5663,9 @@ mod tests { .await .unwrap(); + // Index "i" so the deferred compaction touches indexed data and writes an FRI. + create_scalar_index(&mut dataset, "i", false).await; + let options = CompactionOptions { target_rows_per_fragment: 2_000, defer_index_remap: true, @@ -4282,6 +5819,164 @@ mod tests { ); } + #[tokio::test] + async fn test_read_bloom_filter_index_with_defer_index_remap() { + let mut dataset = lance_datagen::gen_batch() + .col("id", lance_datagen::array::step::()) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(3)) + .await + .unwrap(); + + dataset + .create_index( + &["id"], + IndexType::BloomFilter, + Some("id_idx".into()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::BloomFilter), + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 512, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!(metrics.fragments_removed > 0); + assert!(metrics.fragments_added > 0); + + assert_eq!( + dataset.count_rows(Some("id = 2".to_owned())).await.unwrap(), + 1 + ); + + let mut scanner = dataset.scan(); + scanner.filter("id = 2").unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ScalarIndexQuery: query=[id = 2]@id_idx(BloomFilter)"), + "Expected BloomFilter index query in plan: {plan}" + ); + } + + #[tokio::test] + async fn test_read_zonemap_index_with_defer_index_remap() { + let batch = arrow_array::record_batch!( + ("id", Int32, (0..12).collect::>()), + ( + "value", + Int64, + [ + Some(0), + None, + Some(20), + Some(30), + Some(40), + None, + Some(60), + Some(70), + Some(80), + None, + Some(100), + Some(110) + ] + ) + ) + .unwrap(); + let reader = RecordBatchIterator::new([Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: 4, + max_rows_per_group: 4, + enable_stable_row_ids: false, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 3); + + dataset + .create_index( + &["value"], + IndexType::ZoneMap, + Some("value_idx".into()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 512, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 3); + assert_eq!(metrics.fragments_added, 1); + + async fn scan_ids(dataset: &Dataset, filter: &str, use_scalar_index: bool) -> Vec { + let mut scanner = dataset.scan(); + scanner.filter(filter).unwrap(); + scanner.project(&["id"]).unwrap(); + scanner.use_scalar_index(use_scalar_index); + scanner.try_into_batch().await.unwrap()["id"] + .as_primitive::() + .values() + .to_vec() + } + + for (filter, expected) in [ + ("value IS NULL", vec![1, 5, 9]), + ("value = 20", vec![2]), + ("value > 90", vec![10, 11]), + ] { + assert_eq!(scan_ids(&dataset, filter, false).await, expected); + assert_eq!(scan_ids(&dataset, filter, true).await, expected); + } + + let merged = dataset + .merge_existing_index_segments(dataset.load_indices_by_name("value_idx").await.unwrap()) + .await + .unwrap(); + dataset + .commit_existing_index_segments("value_idx", "value", vec![merged]) + .await + .unwrap(); + + for (filter, expected) in [ + ("value IS NULL", vec![1, 5, 9]), + ("value = 20", vec![2]), + ("value > 90", vec![10, 11]), + ] { + assert_eq!(scan_ids(&dataset, filter, false).await, expected); + assert_eq!(scan_ids(&dataset, filter, true).await, expected); + } + + let mut scanner = dataset.scan(); + scanner.filter("value IS NULL").unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ScalarIndexQuery: query=[value IS NULL]@value_idx(ZoneMap)"), + "Expected ZoneMap index query in plan: {plan}" + ); + } + #[tokio::test] async fn test_read_btree_index_with_defer_index_remap() { // Create a dataset with an incremental ID column @@ -5744,14 +7439,14 @@ mod tests { use arrow_array::types::{Float32Type, Int32Type}; use lance_datagen::Dimension; - const DIM: u32 = 32; + const DIM: u32 = 8; let mut dataset = lance_datagen::gen_batch() .col("id", lance_datagen::array::step::()) .col( "vec", lance_datagen::array::rand_vec::(Dimension::from(DIM)), ) - .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(1000)) + .into_ram_dataset(FragmentCount::from(6), FragmentRowCount::from(64)) .await .unwrap(); dataset @@ -5791,7 +7486,7 @@ mod tests { } } } - let step = (rows.len() / 16).max(1); + let step = (rows.len() / 4).max(1); let queries: Vec> = rows.iter().step_by(step).cloned().collect(); let mut baseline: Vec> = Vec::new(); for q in &queries { @@ -5802,7 +7497,7 @@ mod tests { let metrics = compact_files( &mut dataset, CompactionOptions { - target_rows_per_fragment: 2_000, + target_rows_per_fragment: 128, defer_index_remap: true, ..Default::default() }, @@ -5946,10 +7641,15 @@ mod tests { let params = VectorIndexParams::with_ivf_hnsw_pq_params( DistanceType::L2, small_ivf(), - HnswBuildParams::default(), + HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16), PQBuildParams { max_iters: 2, num_sub_vectors: 2, + num_bits: 4, + sample_rate: 2, ..Default::default() }, ); @@ -6067,7 +7767,7 @@ mod tests { ..Default::default() }; - let planner = DefaultCompactionPlanner::new(options); + let planner = DefaultCompactionPlanner::new(options).unwrap(); let plan = planner.plan(&dataset).await.unwrap(); // Should create tasks to compact small fragments @@ -6124,6 +7824,18 @@ mod tests { "lance.compaction.index_remap_mode".to_string(), "compact".to_string(), ), + ( + "lance.compaction.max_source_fragments".to_string(), + "20".to_string(), + ), + ( + "lance.compaction.max_source_rows".to_string(), + "1000000".to_string(), + ), + ( + "lance.compaction.max_source_bytes".to_string(), + "1073741824".to_string(), + ), ]); let opts = CompactionOptions::from_dataset_config(&config).unwrap(); @@ -6139,6 +7851,9 @@ mod tests { assert_eq!(opts.binary_copy_read_batch_bytes, Some(8_388_608)); // A non-default value proves the config string was actually parsed. assert_eq!(opts.index_remap_mode, IndexRemapMode::Compact); + assert_eq!(opts.max_source_fragments, Some(20)); + assert_eq!(opts.max_source_rows, Some(1_000_000)); + assert_eq!(opts.max_source_bytes, Some(1_073_741_824)); } #[test] @@ -6264,6 +7979,29 @@ mod tests { assert!(err_msg.contains("invalid_mode")); } + #[test] + fn test_from_dataset_config_max_overlays_per_fragment() { + let key = "lance.compaction.max_overlays_per_fragment".to_string(); + + // An integer sets the threshold. + let config = HashMap::from([(key.clone(), "3".to_string())]); + let opts = CompactionOptions::from_dataset_config(&config).unwrap(); + assert_eq!(opts.max_overlays_per_fragment, Some(3)); + + // "none" (case-insensitive) disables the trigger, overriding the Some(10) default. + let config = HashMap::from([(key.clone(), "None".to_string())]); + let opts = CompactionOptions::from_dataset_config(&config).unwrap(); + assert_eq!(opts.max_overlays_per_fragment, None); + + // Anything else is rejected. + let config = HashMap::from([(key, "not_a_number".to_string())]); + let err_msg = CompactionOptions::from_dataset_config(&config) + .unwrap_err() + .to_string(); + assert!(err_msg.contains("max_overlays_per_fragment")); + assert!(err_msg.contains("not_a_number")); + } + #[test] fn test_apply_dataset_config_overrides() { let config = HashMap::from([( @@ -6303,37 +8041,7 @@ mod tests { #[tokio::test] async fn test_max_source_fragments() { let test_dir = TempStrDir::default(); - let test_uri = &test_dir; - - let data = sample_data(); - let schema = data.schema(); - - // Create 10 small fragments (100 rows each) via 10 appends - let write_params = WriteParams { - max_rows_per_file: 100, - ..Default::default() - }; - Dataset::write( - RecordBatchIterator::new(vec![Ok(data.slice(0, 100))], schema.clone()), - test_uri, - Some(write_params.clone()), - ) - .await - .unwrap(); - for i in 1..10 { - let mut append_params = write_params.clone(); - append_params.mode = WriteMode::Append; - Dataset::write( - RecordBatchIterator::new(vec![Ok(data.slice(i * 100, 100))], schema.clone()), - test_uri, - Some(append_params), - ) - .await - .unwrap(); - } - - let dataset = Dataset::open(test_uri).await.unwrap(); - assert_eq!(dataset.get_fragments().len(), 10); + let dataset = dataset_with_ten_small_fragments(&test_dir).await; // Plan without limit - all 10 fragments should be candidates. // Use a target that splits the 10 fragments into multiple tasks. @@ -6390,20 +8098,194 @@ mod tests { "expected partial compaction (not fully compacted), got {after_first}" ); - // Run again to make more progress - let opts_bounded = CompactionOptions { + // Run again to make more progress + let opts_bounded = CompactionOptions { + target_rows_per_fragment: 250, + max_source_fragments: Some(4), + ..Default::default() + }; + compact_files(&mut dataset, opts_bounded, None) + .await + .unwrap(); + let after_second = dataset.get_fragments().len(); + assert!( + after_second <= after_first, + "expected progress: {after_second} should be <= {after_first}" + ); + } + + /// Writes `sample_data` as 10 fragments of 100 rows each, deletes half of + /// fragment 0's rows, and puts a one-cell data overlay on it, so every + /// `max_source_*` planning budget is exercised against a dataset that + /// carries deleted rows and overlay files. + async fn dataset_with_ten_small_fragments(test_uri: &str) -> Dataset { + let data = sample_data(); + let schema = data.schema(); + let write_params = WriteParams { + max_rows_per_file: 100, + ..Default::default() + }; + Dataset::write( + RecordBatchIterator::new(vec![Ok(data.slice(0, 100))], schema.clone()), + test_uri, + Some(write_params.clone()), + ) + .await + .unwrap(); + for i in 1..10 { + let mut append_params = write_params.clone(); + append_params.mode = WriteMode::Append; + Dataset::write( + RecordBatchIterator::new(vec![Ok(data.slice(i * 100, 100))], schema.clone()), + test_uri, + Some(append_params), + ) + .await + .unwrap(); + } + let mut dataset = Dataset::open(test_uri).await.unwrap(); + assert_eq!(dataset.get_fragments().len(), 10); + // Fragment 0 keeps 100 physical rows but only 50 live rows. + dataset.delete("a < 50").await.unwrap(); + // The overlay shadows one surviving base value (offset 60, kept clear + // of the delete predicate) without adding any rows. + let dataset = commit_overlay( + dataset, + 0, + &[0], + OverlayCoverage::dense(bitmap([60])), + vec![Arc::new(Int64Array::from(vec![60_000_i64]))], + ) + .await; + assert_eq!( + dataset.get_fragment(0).unwrap().metadata().overlays.len(), + 1 + ); + dataset + } + + #[tokio::test] + async fn test_excluded_fragments_are_planning_boundaries() { + let test_dir = TempStrDir::default(); + let mut dataset = dataset_with_ten_small_fragments(&test_dir).await; + let excluded_fragment_id = 4; + let options = CompactionOptions { + target_rows_per_fragment: 250, + excluded_fragment_ids: vec![excluded_fragment_id, excluded_fragment_id, u32::MAX], + ..Default::default() + }; + + let plan = plan_compaction(&dataset, &options).await.unwrap(); + let planned_fragment_ids = plan + .tasks() + .iter() + .map(|task| { + task.fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect::>() + }) + .collect::>(); + + assert_eq!( + planned_fragment_ids, + vec![vec![0, 1, 2, 3], vec![5, 6, 7, 8, 9]] + ); + assert!( + planned_fragment_ids + .iter() + .flatten() + .all(|fragment_id| *fragment_id != excluded_fragment_id) + ); + + let metrics = compact_files(&mut dataset, options, None).await.unwrap(); + assert_eq!(metrics.fragments_removed, 9); + let remaining_fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + assert!(remaining_fragment_ids.contains(&excluded_fragment_id)); + } + + #[tokio::test] + async fn test_max_source_rows() { + let test_dir = TempStrDir::default(); + let dataset = dataset_with_ten_small_fragments(&test_dir).await; + + // The first task covers fragments 0..=2: 250 live rows (fragment 0 + // keeps 50 after the deletes, and its overlay adds none) but 300 + // physical rows. A budget between the two admits it and only it, + // proving the budget is enforced against live rows. + let opts = CompactionOptions { target_rows_per_fragment: 250, - max_source_fragments: Some(4), + max_source_rows: Some(270), ..Default::default() }; - compact_files(&mut dataset, opts_bounded, None) - .await - .unwrap(); - let after_second = dataset.get_fragments().len(); - assert!( - after_second <= after_first, - "expected progress: {after_second} should be <= {after_first}" - ); + let plan = plan_compaction(&dataset, &opts).await.unwrap(); + assert_eq!(plan.num_tasks(), 1); + let physical_rows: usize = plan + .tasks() + .iter() + .flat_map(|t| &t.fragments) + .map(|f| f.physical_rows.unwrap()) + .sum(); + assert_eq!(physical_rows, 300); + } + + #[tokio::test] + async fn test_max_source_bytes() { + let test_dir = TempStrDir::default(); + let dataset = dataset_with_ten_small_fragments(&test_dir).await; + + let first_task_base_bytes: u64 = dataset.get_fragments()[..3] + .iter() + .flat_map(|f| f.metadata.files.iter()) + .map(|df| df.file_size_bytes.get().unwrap().get()) + .sum(); + let overlay_bytes = dataset.get_fragment(0).unwrap().metadata().overlays[0] + .data_file + .file_size_bytes + .get() + .unwrap() + .get(); + assert!(first_task_base_bytes > 0 && overlay_bytes > 0); + + // The first task covers fragments 0..=2 (target 250 rows). A budget of + // exactly their base bytes is exceeded once fragment 0's overlay file + // is counted, so the plan is empty: the budget is a hard upper bound, + // overlay bytes are part of a task's source, and deletion files are + // never counted. + let opts = CompactionOptions { + target_rows_per_fragment: 250, + max_source_bytes: Some(first_task_base_bytes), + ..Default::default() + }; + let plan = plan_compaction(&dataset, &opts).await.unwrap(); + assert_eq!(plan.num_tasks(), 0); + + // Widening the budget by the overlay's bytes admits exactly the first + // task and nothing more. + let budget = first_task_base_bytes + overlay_bytes; + let opts = CompactionOptions { + target_rows_per_fragment: 250, + max_source_bytes: Some(budget), + ..Default::default() + }; + let plan = plan_compaction(&dataset, &opts).await.unwrap(); + assert_eq!(plan.num_tasks(), 1); + let source_bytes: u64 = plan + .tasks() + .iter() + .flat_map(|t| &t.fragments) + .flat_map(|f| { + f.files + .iter() + .chain(f.overlays.iter().map(|o| &o.data_file)) + }) + .map(|df| df.file_size_bytes.get().unwrap().get()) + .sum(); + assert_eq!(source_bytes, budget); } #[tokio::test] @@ -6698,12 +8580,12 @@ mod tests { count_all_files_in(&data_dir).unwrap_or(0) } - /// Site 2 in PR #6320: when `commit_compaction` fails to apply the commit - /// after `rewrite_files` has already written new data files, those files - /// must be cleaned up. We force the commit failure by injecting an error on - /// writes to the `_transactions/` directory. + /// Once `commit_compaction` reaches the Rewrite commit, its input may be a + /// replay of a result whose earlier commit landed ambiguously. A failure on + /// this call must therefore leave data files for GC instead of deleting + /// files that an existing version may reference. #[tokio::test] - async fn test_commit_compaction_cleans_up_data_on_commit_failure() { + async fn test_commit_compaction_leaves_data_for_gc_on_commit_failure() { use crate::dataset::builder::DatasetBuilder; use crate::utils::test::FailingProxyStore; use lance_io::object_store::ObjectStoreParams; @@ -6722,10 +8604,6 @@ mod tests { &routed_uri, Some(WriteParams { max_rows_per_file: 100, - // Stable row IDs lets `commit_compaction` skip the - // `reserve_fragment_ids` pre-commit (which would otherwise fail - // *before* the new data files exist), isolating the failure to - // the `apply_commit` call we want to test. enable_stable_row_ids: true, ..Default::default() }), @@ -6771,15 +8649,14 @@ mod tests { "Compaction should fail when transaction commit fails" ); - assert_eq!( - count_data_files_in(test_uri), - baseline_files, - "Compaction data files should be cleaned up when commit fails" + assert!( + count_data_files_in(test_uri) > baseline_files, + "Compaction data files should be retained for GC after the Rewrite commit fails" ); } #[tokio::test] - async fn test_commit_compaction_cleans_up_blob_v2_sidecars_on_commit_failure() { + async fn test_commit_compaction_leaves_blob_v2_sidecars_for_gc_on_commit_failure() { use crate::BlobArrayBuilder; use crate::dataset::builder::DatasetBuilder; use crate::utils::test::FailingProxyStore; @@ -6852,10 +8729,9 @@ mod tests { "Compaction should fail when transaction commit fails" ); - assert_eq!( - count_data_files_in(test_uri), - baseline_files, - "Blob v2 sidecars should be cleaned up when commit fails" + assert!( + count_data_files_in(test_uri) > baseline_files, + "Blob v2 sidecars should be retained for GC after the Rewrite commit fails" ); } @@ -6885,16 +8761,182 @@ mod tests { let row_id = row_ids.value(i); let id = ids.value(i); let blobs = dataset.take_blobs(&[row_id], column).await.unwrap(); - if blobs.is_empty() { - result.push((id, None)); - } else { - let data = blobs[0].read().await.unwrap(); - result.push((id, Some(data.to_vec()))); + match blobs.into_iter().next().flatten() { + Some(blob) => { + let data = blob.read().await.unwrap(); + result.push((id, Some(data.to_vec()))); + } + None => result.push((id, None)), } } result } + fn mixed_blob_values() -> Vec<(i32, Option>)> { + vec![ + (0, Some(vec![b'0'; 80])), + (1, None), + (2, Some(Vec::new())), + (3, Some(vec![b'3'; 80])), + (4, Some(vec![b'4'; 80])), + (5, Some(vec![b'5'; 80])), + ] + } + + async fn assert_compaction_preserves_blob_values( + mut dataset: Dataset, + expected: &[(i32, Option>)], + ) { + assert_eq!(dataset.get_fragments().len(), 3); + + let mut before = read_blob_bytes_by_index(&Arc::new(dataset.clone()), "blob").await; + before.sort_by_key(|(id, _)| *id); + assert_eq!(before, expected); + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 1024 * 1024, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + + assert_eq!(dataset.get_fragments().len(), 1); + + let mut after = read_blob_bytes_by_index(&Arc::new(dataset), "blob").await; + after.sort_by_key(|(id, _)| *id); + assert_eq!(after, expected); + } + + #[test] + fn test_blob_v2_rewrite_plan_skips_non_blob_list_subtree() { + let items_field = Field::new( + "items", + DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + true, + ); + let info_field = Field::new( + "info", + DataType::Struct( + vec![ + Arc::new(items_field), + Arc::new(crate::blob_field("blob", true)), + ] + .into(), + ), + true, + ); + let logical_arrow_schema = Schema::new(vec![info_field]); + let logical_schema = + lance_core::datatypes::Schema::try_from(&logical_arrow_schema).unwrap(); + let mut input_schema = logical_schema.clone(); + input_schema.fields[0].unload_blobs_recursive(); + let input_field = Field::from(&input_schema.fields[0]); + + let plan = + BlobV2FieldRewritePlan::try_new(&logical_schema.fields[0], &input_field).unwrap(); + let BlobV2FieldRewritePlan::Struct { children, .. } = &plan else { + panic!("nested blob field should produce a struct rewrite plan"); + }; + assert!(matches!( + children[0], + BlobV2FieldRewritePlan::Passthrough { .. } + )); + assert!(matches!(children[1], BlobV2FieldRewritePlan::Blob { .. })); + + let DataType::Struct(output_children) = plan.output_field().data_type() else { + panic!("nested blob rewrite plan should declare a struct output"); + }; + let DataType::Struct(input_children) = input_field.data_type() else { + panic!("nested blob input should be a struct"); + }; + assert_eq!(output_children[0], input_children[0]); + let DataType::Struct(blob_children) = output_children[1].data_type() else { + panic!("blob rewrite plan should declare a struct blob output"); + }; + assert_eq!( + BlobV2Layout::classify(blob_children), + Some(BlobV2Layout::Logical) + ); + } + + #[tokio::test] + async fn test_compact_blob_v1_preserves_null_empty_and_payload_order() { + let test_dir = TempStrDir::default(); + let expected = mixed_blob_values(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("blob", DataType::LargeBinary, true) + .with_metadata([(BLOB_META_KEY.to_string(), "true".to_string())].into()), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..expected.len() as i32)), + Arc::new(LargeBinaryArray::from_iter( + expected.iter().map(|(_, value)| value.as_deref()), + )), + ], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_0), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + + assert_compaction_preserves_blob_values(dataset, &expected).await; + } + + #[tokio::test] + async fn test_compact_blob_v2_preserves_null_empty_and_payload_order() { + use crate::BlobArrayBuilder; + + let test_dir = TempStrDir::default(); + let expected = mixed_blob_values(); + let mut blob_builder = BlobArrayBuilder::new(expected.len()); + for (_, value) in &expected { + match value { + Some(value) => blob_builder.push_bytes(value).unwrap(), + None => blob_builder.push_null().unwrap(), + } + } + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + crate::blob_field("blob", true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..expected.len() as i32)), + blob_builder.finish().unwrap(), + ], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + + assert_compaction_preserves_blob_values(dataset, &expected).await; + } + #[tokio::test] async fn test_compact_blob_v2_preserves_external_references() { use crate::BlobArrayBuilder; @@ -7885,4 +9927,337 @@ mod tests { ] ); } + // ---- `max_overlays_per_fragment` compaction trigger ---- + // + // Tests for the trigger that fully compacts a fragment carrying too many data + // overlay files into a fresh fragment with the overlays (and deletions) + // materialized into the base data. + use arrow_array::record_batch; + use lance_file::writer::FileWriterOptions; + use lance_io::utils::CachedFileSize; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use std::collections::BTreeMap; + + use crate::dataset::DATA_DIR; + use crate::dataset::transaction::DataOverlayGroup; + + /// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `val` (field 1) = + /// id * 10, six rows per fragment (fragments 0 and 1). + async fn create_base_dataset(uri: &str) -> Dataset { + let batch = record_batch!( + ("id", Int32, (0..12).collect::>()), + ("val", Int32, (0..12).map(|v| v * 10).collect::>()) + ) + .unwrap(); + let schema = batch.schema(); + let write_params = WriteParams { + max_rows_per_file: 6, + max_rows_per_group: 6, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, uri, Some(write_params)) + .await + .unwrap() + } + + fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) + } + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + /// Write a dense overlay covering `fields` of `fragment_id` with `columns` + /// as the per-field value columns, then commit it as a `DataOverlay`. + async fn commit_overlay( + dataset: Dataset, + fragment_id: u64, + fields: &[i32], + coverage: OverlayCoverage, + columns: Vec, + ) -> Dataset { + let read_version = dataset.version().version; + let overlay_schema = dataset.schema().project_by_ids(fields, true); + let filename = format!("{}.lance", Uuid::new_v4()); + let path = dataset.base.clone().join(DATA_DIR).join(filename.as_str()); + let obj_writer = dataset.object_store.create(&path).await.unwrap(); + let file_version = LanceFileVersion::Stable.resolve(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + for (column_index, array) in columns.into_iter().enumerate() { + writer.write_column(column_index, array).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(f, _)| *f as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, c)| *c as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file, + coverage, + committed_version: 0, + }], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap() + } + + /// Commit `n` distinct single-cell overlays to fragment 0 (offset `i`, val + /// column set to `1000 + i`), so the fragment ends up with `n` overlays. The + /// `1000 +` offset keeps overlaid values clear of the base `id * 10` values. + async fn commit_n_overlays(mut dataset: Dataset, n: u32) -> Dataset { + for i in 0..n { + dataset = commit_overlay( + dataset, + 0, + &[1], + OverlayCoverage::dense(bitmap([i])), + vec![i32_array([Some(1000 + i as i32)])], + ) + .await; + } + dataset + } + + /// Options whose only compaction trigger is the overlay limit: base + /// fragments here are far below the default 1M-row target, which would + /// otherwise make them size-based compaction candidates on their own. + fn overlay_only_options(max_overlays_per_fragment: usize) -> CompactionOptions { + CompactionOptions { + max_overlays_per_fragment: Some(max_overlays_per_fragment), + target_rows_per_fragment: 6, + ..Default::default() + } + } + + /// Scan `id` and `val` and return an `id -> val` map (order-independent). + async fn id_val_map(dataset: &Dataset) -> BTreeMap> { + let mut scanner = dataset.scan(); + scanner.project(&["id", "val"]).unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let mut out = BTreeMap::new(); + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let vals = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + let v = if vals.is_null(i) { + None + } else { + Some(vals.value(i)) + }; + out.insert(ids.value(i), v); + } + out + } + + #[tokio::test] + async fn test_max_overlays_triggers_full_compaction() { + // Fragment 0 gets 3 overlays; fragment 1 stays clean. + let dataset = create_base_dataset("memory://").await; + let mut dataset = commit_n_overlays(dataset, 3).await; + assert_eq!( + dataset.get_fragment(0).unwrap().metadata().overlays.len(), + 3 + ); + + // Threshold 2: only fragment 0 (3 > 2) is compacted. + let metrics = compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 1); + assert_eq!(metrics.fragments_added, 1); + + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + // The compacted fragment is a fresh single-data-file fragment with no + // overlays; fragment 1 is untouched. + let compacted = fragments + .iter() + .find(|f| f.id() != 1) + .expect("a new fragment id was assigned"); + assert!(compacted.metadata().overlays.is_empty()); + assert_eq!(compacted.metadata().files.len(), 1); + + // The overlaid values were materialized: id i in 0..3 -> 1000 + i. + let values = id_val_map(&dataset).await; + let expected: BTreeMap> = (0..12) + .map(|id| { + let v = if id < 3 { 1000 + id } else { id * 10 }; + (id, Some(v)) + }) + .collect(); + assert_eq!(values, expected); + } + + #[tokio::test] + async fn test_below_threshold_is_a_noop() { + let dataset = create_base_dataset("memory://").await; + let mut dataset = commit_n_overlays(dataset, 2).await; + + // 2 overlays, threshold 2: `overlays > max` is false, so no compaction. + let metrics = compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 0); + assert_eq!(metrics.fragments_added, 0); + assert_eq!( + dataset.get_fragment(0).unwrap().metadata().overlays.len(), + 2 + ); + } + + #[tokio::test] + async fn test_overlay_compaction_materializes_deletions() { + let dataset = create_base_dataset("memory://").await; + let mut dataset = commit_n_overlays(dataset, 3).await; + // Delete a row from the overlaid fragment (id 2 is at offset 2). + dataset.delete("id = 2").await.unwrap(); + assert!( + dataset + .get_fragment(0) + .unwrap() + .metadata() + .deletion_file + .is_some() + ); + + compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + + // The deletion was materialized: no deletion file remains and id 2 is gone. + for fragment in dataset.get_fragments() { + assert!(fragment.metadata().deletion_file.is_none()); + assert!(fragment.metadata().overlays.is_empty()); + } + let values = id_val_map(&dataset).await; + assert!(!values.contains_key(&2)); + // Surviving overlaid cells still carry their materialized values. + assert_eq!(values.get(&0), Some(&Some(1000))); + assert_eq!(values.get(&1), Some(&Some(1001))); + } + + #[tokio::test] + async fn test_overlay_compaction_reconciles_stale_index() { + let mut dataset = create_base_dataset("memory://").await; + // Index `val` before any overlay -> the index is stale once val is overlaid. + dataset + .create_index( + &["val"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Overlay val[0] 0 -> 100 (committed after the index) and push fragment 0 + // over the overlay limit. + let mut dataset = commit_n_overlays(dataset, 3).await; + + let val_index_before = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|i| i.fields == vec![1]) + .expect("val index present") + .clone(); + assert!( + val_index_before + .fragment_bitmap + .as_ref() + .unwrap() + .contains(0) + ); + + compact_files(&mut dataset, overlay_only_options(2), None) + .await + .unwrap(); + + // The stale val index no longer covers the compacted fragment, so its + // rows fall back to a flat scan instead of serving stale values. + let indices = dataset.load_indices().await.unwrap(); + let val_index = indices + .iter() + .find(|i| i.fields == vec![1]) + .expect("val index present"); + let compacted_id = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .find(|id| *id != 1) + .unwrap(); + assert!( + !val_index + .fragment_bitmap + .as_ref() + .unwrap() + .contains(compacted_id), + "stale index must drop the compacted fragment from its coverage" + ); + + // The indexed query is correct: the materialized value is found and the + // stale pre-overlay value is gone. + let mut scanner = dataset.scan(); + scanner + .filter("val = 1000") + .unwrap() + .project(&["id"]) + .unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(ids.len(), 1); + assert_eq!(ids.value(0), 0); + + let mut scanner = dataset.scan(); + scanner.filter("val = 0").unwrap().project(&["id"]).unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 0, "stale value 0 must no longer match"); + } } diff --git a/rust/lance/src/dataset/optimize/binary_copy.rs b/rust/lance/src/dataset/optimize/binary_copy.rs index f22737fff89..c76e0ea300f 100644 --- a/rust/lance/src/dataset/optimize/binary_copy.rs +++ b/rust/lance/src/dataset/optimize/binary_copy.rs @@ -7,90 +7,44 @@ use crate::dataset::DATA_DIR; use crate::dataset::WriteParams; use crate::dataset::fragment::write::generate_random_filename; use crate::datatypes::Schema; -use lance_arrow::DataTypeExt; use lance_core::Error; -use lance_encoding::decoder::{ColumnInfo, PageEncoding, PageInfo as DecPageInfo}; -use lance_encoding::version::LanceFileVersion; -use lance_file::format::pbfile; +use lance_encoding::decoder::{ColumnInfo, PageInfo as DecPageInfo}; use lance_file::reader::FileReader as LFReader; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions as file_versions; use lance_file::writer::{FileWriter, FileWriterOptions}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; -use lance_io::traits::Writer; use lance_table::format::{DataFile, Fragment}; use prost::Message; use prost_types::Any; use std::ops::Range; use std::sync::Arc; -use tokio::io::AsyncWriteExt; - -const ALIGN: usize = 64; - -/// Apply 64-byte alignment padding for V2.1+ files. -/// -/// For V2.1+, writes padding bytes to align the current position to a 64-byte boundary. -/// For V2.0 and earlier, no padding is applied as alignment is not required. -/// -/// Returns the new position after padding (if any). -async fn apply_alignment_padding( - writer: &mut dyn Writer, - current_pos: u64, - version: LanceFileVersion, -) -> Result { - if version >= LanceFileVersion::V2_1 { - static ZERO_BUFFER: std::sync::OnceLock> = std::sync::OnceLock::new(); - let zero_buf = ZERO_BUFFER.get_or_init(|| vec![0u8; ALIGN]); - - let pad = (ALIGN - (current_pos as usize % ALIGN)) % ALIGN; - if pad != 0 { - writer.write_all(&zero_buf[..pad]).await?; - return Ok(current_pos + pad as u64); - } - } - Ok(current_pos) -} async fn init_writer_if_necessary( dataset: &Dataset, - current_writer: &mut Option>, + version: ConcreteFileVersion, + current_writer: &mut Option, current_filename: &mut Option, ) -> Result { if current_writer.is_none() { let filename = format!("{}.lance", generate_random_filename()); let path = dataset.base.clone().join(DATA_DIR).join(filename.as_str()); - let writer = dataset.object_store.create(&path).await?; - *current_writer = Some(writer); + let object_writer = dataset.object_store.create(&path).await?; + *current_writer = Some(file_versions::create_lazy_writer( + version, + object_writer, + FileWriterOptions::default(), + )?); *current_filename = Some(filename); return Ok(true); } Ok(false) } -/// v2_0 vs v2_1+ field-to-column index mapping -/// - v2_1+ stores only leaf columns; non-leaf fields get `-1` in the mapping -/// - v2_0 includes structural headers as columns; non-leaf fields map to a concrete index -fn compute_field_column_indices( - schema: &Schema, - full_field_ids_len: usize, - version: LanceFileVersion, -) -> Vec { - let is_structural = version >= LanceFileVersion::V2_1; - let mut field_column_indices: Vec = Vec::with_capacity(full_field_ids_len); - let mut curr_col_idx: i32 = 0; - for field in schema.fields_pre_order() { - if field.is_packed_struct() || field.is_leaf() || !is_structural { - field_column_indices.push(curr_col_idx); - curr_col_idx += 1; - } else { - field_column_indices.push(-1); - } - } - field_column_indices -} - /// Finalize the current output file and return it as a single [Fragment]. /// - Ensures an output writer / filename is present (creates a new file if needed). /// - Converts the in-memory `col_pages` / `col_buffers` into `ColumnInfo` metadata, draining them. -/// - Applies v2_0 structural header rules (single page, normalized `num_rows` and `priority`). +/// - Lets the exact file version normalize copied column metadata. /// - Writes the Lance footer via [flush_footer] and registers the resulting [DataFile] in a [Fragment]. /// /// PAY ATTENTION current function will: @@ -99,28 +53,24 @@ fn compute_field_column_indices( #[allow(clippy::too_many_arguments)] async fn finalize_current_output_file( schema: &Schema, - full_field_ids: &[i32], - current_writer: &mut Option>, + version: ConcreteFileVersion, + current_writer: &mut Option, current_filename: &mut Option, current_page_table: &[ColumnInfo], col_pages: &mut [Vec], col_buffers: &mut [Vec<(u64, u64)>], - is_non_leaf_column: &[bool], total_rows_in_current: u64, - version: LanceFileVersion, ) -> Result { let mut final_cols: Vec> = Vec::with_capacity(current_page_table.len()); for (i, column_info) in current_page_table.iter().enumerate() { let mut pages_vec = std::mem::take(&mut col_pages[i]); - // For v2_0 struct headers, force a single page and set num_rows to total - if version == LanceFileVersion::V2_0 - && is_non_leaf_column.get(i).copied().unwrap_or(false) - && !pages_vec.is_empty() - { - pages_vec[0].num_rows = total_rows_in_current; - pages_vec[0].priority = 0; - pages_vec.truncate(1); - } + file_versions::finalize_external_metadata_column( + version, + schema, + i, + &mut pages_vec, + total_rows_in_current, + )?; let pages_arc = Arc::from(pages_vec.into_boxed_slice()); let buffers_vec = std::mem::take(&mut col_buffers[i]); final_cols.push(Arc::new(ColumnInfo::new( @@ -130,15 +80,19 @@ async fn finalize_current_output_file( column_info.encoding.clone(), ))); } - let writer = current_writer.take().unwrap(); - flush_footer(writer, schema, &final_cols, total_rows_in_current, version).await?; + let mut writer = current_writer + .take() + .ok_or_else(|| Error::internal("binary copy output writer was not initialized"))?; + flush_footer(&mut writer, schema, &final_cols, total_rows_in_current).await?; // Register the newly closed output file as a fragment data file - let (maj, min) = version.to_numbers(); let mut fragment = Fragment::new(0); - let field_column_indices = compute_field_column_indices(schema, full_field_ids.len(), version); - let mut data_file = DataFile::new_unstarted(current_filename.take().unwrap(), maj, min); - data_file.fields = full_field_ids.to_vec().into(); + let (field_ids, field_column_indices) = file_versions::data_file_columns(version, schema); + let filename = current_filename + .take() + .ok_or_else(|| Error::internal("binary copy output filename was not initialized"))?; + let mut data_file = DataFile::new_unstarted(filename, version); + data_file.fields = field_ids.into(); data_file.column_indices = field_column_indices.into(); fragment.files.push(data_file); fragment.physical_rows = Some(total_rows_in_current as usize); @@ -157,11 +111,9 @@ async fn finalize_current_output_file( /// └── final flush for remaining rows /// /// Behavior highlights: -/// - Assumes all input files share the same Lance file version; version drives column-count -/// calculation (v2.0 includes structural headers, v2.1+ only leaf columns). +/// - Assumes all input files share the same Lance file version. /// - Preserves stable row ids by concatenating row-id sequences when enabled. -/// - Enforces 64-byte alignment for page and buffer writes in V2.1+ files (V2.0 does not require alignment). -/// - For v2.0, preserves single-page structural headers and normalizes their row counts/priority. +/// - Delegates physical-column mapping and copied metadata normalization to the exact file version. /// - Flushes an output file once `max_rows_per_file` rows are accumulated, then repeats. /// /// Parameters: @@ -170,6 +122,7 @@ async fn finalize_current_output_file( /// - `params`: write parameters (uses `max_rows_per_file`). /// - `read_batch_bytes_opt`: optional I/O batch size when coalescing page reads. pub async fn rewrite_files_binary_copy( + version: ConcreteFileVersion, dataset: &Dataset, fragments: &[Fragment], params: &WriteParams, @@ -185,54 +138,19 @@ pub async fn rewrite_files_binary_copy( // - Reads page and buffer regions directly from source files in bounded batches // - Appends them to a new output file with alignment, updating offsets // - Recomputes page priorities by adding the cumulative row count to preserve order - // - For v2_0, enforces single-page structural header columns when closing a file // - Writes a new footer (schema descriptor, column metadata, offset tables, version) // - Optionally carries forward stable row ids and persists them inline in fragment metadata // Merge small Lance files into larger ones by page-level binary copy. let schema = dataset.schema().clone(); - let full_field_ids = schema.field_ids(); - - // The previous checks have ensured that the file versions of all files are consistent. - let version = LanceFileVersion::try_from_major_minor( - fragments[0].files[0].file_major_version, - fragments[0].files[0].file_minor_version, - ) - .unwrap() - .resolve(); - // v2.0 and v2.1+ handle structural headers differently during file writing: - // - v2_0 materializes ALL fields in pre-order traversal (leaf fields + non-leaf struct headers), - // which means the ColumnInfo set includes all fields in pre-order traversal. - // - v2_1+ materializes fields that are either leaf columns OR packed structs. Non-leaf structural - // headers (unpacked structs with children) are not stored as columns. - // As a result, the ColumnInfo set contains leaf fields and packed structs. - // To correctly align copy layout, we derive `column_count` by version: - // - v2_0: use total number of fields in pre-order (leaf + non-leaf headers) - // - v2_1+: use only the number of leaf fields plus packed structs - let column_count = if version == LanceFileVersion::V2_0 { - schema.fields_pre_order().count() - } else { - schema - .fields_pre_order() - .filter(|f| f.is_packed_struct() || f.is_leaf()) - .count() - }; - - // v2_0 compatibility: build a map to identify non-leaf structural header columns - // - In v2_0 these headers exist as columns and must have a single page - // - In v2_1+ these headers are not stored as columns and this map is unused - let mut is_non_leaf_column: Vec = vec![false; column_count]; - if version == LanceFileVersion::V2_0 { - for (col_idx, field) in schema.fields_pre_order().enumerate() { - // Only mark non-packed Struct fields (lists remain as leaf data carriers) - let is_non_leaf = field.data_type().is_struct() && !field.is_packed_struct(); - is_non_leaf_column[col_idx] = is_non_leaf; - } - } + let column_count = schema + .fields + .iter() + .map(|field| file_versions::physical_column_count(version, field)) + .sum(); let mut out: Vec = Vec::new(); - let mut current_writer: Option> = None; + let mut current_writer: Option = None; let mut current_filename: Option = None; - let mut current_pos: u64 = 0; let mut current_page_table: Vec = Vec::new(); // Baseline column encodings captured from the first source file; all subsequent // files must match per-column to safely concatenate column-level buffers. @@ -279,38 +197,34 @@ pub async fn rewrite_files_binary_copy( .collect(); baseline_col_encoding_bytes = src_column_infos .iter() - .map(|ci| Any::from_msg(&ci.encoding).unwrap().encode_to_vec()) - .collect(); + .map(|ci| Ok(Any::from_msg(&ci.encoding)?.encode_to_vec())) + .collect::>>()?; } // Iterate through each column of the current data file of the current fragment for (col_idx, src_column_info) in src_column_infos.iter().enumerate() { - // v2_0 compatibility: special handling for non-leaf structural header columns - // - v2_0 expects structural header columns to have a SINGLE page; they carry layout - // metadata only and are not true data carriers. - // - When merging multiple input files via binary copy, naively appending pages would - // yield multiple pages for the same structural header column, violating v2_0 rules. - // - To preserve v2_0 invariants, we skip pages beyond the first one for these columns. - // - During finalization we also normalize the single remaining page’s `num_rows` to the - // total number of rows in the output file and reset `priority` to 0. - // - For v2_1+ this logic does not apply because non-leaf headers are not stored as columns. - let is_non_leaf = col_idx < is_non_leaf_column.len() && is_non_leaf_column[col_idx]; - if is_non_leaf && !col_pages[col_idx].is_empty() { - continue; - } - - if init_writer_if_necessary(dataset, &mut current_writer, &mut current_filename) - .await? - { - current_pos = 0; - } + let has_existing_pages = !col_pages[col_idx].is_empty(); + file_versions::copy_external_metadata_column( + version, + &schema, + col_idx, + has_existing_pages, + || async { + init_writer_if_necessary( + dataset, + version, + &mut current_writer, + &mut current_filename, + ) + .await?; - let read_batch_bytes: u64 = read_batch_bytes_opt.unwrap_or(16 * 1024 * 1024) as u64; + let read_batch_bytes: u64 = + read_batch_bytes_opt.unwrap_or(16 * 1024 * 1024) as u64; - let mut page_index = 0; + let mut page_index = 0; - // Iterate through each page of the current column in the current data file of the current fragment - while page_index < src_column_info.page_infos.len() { + // Iterate through each page of the current column in the current data file of the current fragment + while page_index < src_column_info.page_infos.len() { let mut batch_ranges: Vec> = Vec::new(); let mut batch_counts: Vec = Vec::new(); let mut batch_bytes: u64 = 0; @@ -368,49 +282,42 @@ pub async fn rewrite_files_binary_copy( for (buffer_idx, (_, size)) in page.buffer_offsets_and_sizes.iter().enumerate() { - let writer = current_writer.as_mut().unwrap().as_mut(); - current_pos = - apply_alignment_padding(writer, current_pos, version).await?; - let start = current_pos; - if *size == 0 { - new_offsets.push((start, 0)); + let writer = current_writer.as_mut().ok_or_else(|| { + Error::internal("binary copy output writer was not initialized") + })?; + let bytes = if *size == 0 { + None } else { - let bytes = bytes_iter.next().ok_or_else(|| { + Some(bytes_iter.next().ok_or_else(|| { Error::execution(format!( "binary copy: missing page buffer bytes while rewriting data file \ (column {col_idx}, page {page_idx}, buffer {buffer_idx}, expected size {size})", )) - })?; - writer.write_all(&bytes).await?; - current_pos += bytes.len() as u64; - new_offsets.push((start, bytes.len() as u64)); - } + })?) + }; + let (start, written) = writer + .write_external_buffer(bytes.as_deref().unwrap_or_default()) + .await?; + new_offsets.push((start, written)); } - // manual clone encoding - let encoding = if page.encoding.is_structural() { - PageEncoding::Structural(page.encoding.as_structural().clone()) - } else { - PageEncoding::Legacy(page.encoding.as_legacy().clone()) - }; // `priority` acts as the global row offset for this page, ensuring // downstream iterators maintain the correct logical order across // merged inputs. let new_page_info = DecPageInfo { num_rows: page.num_rows, priority: page.priority + total_rows_in_current, - encoding, + encoding: page.encoding.clone(), buffer_offsets_and_sizes: Arc::from(new_offsets.into_boxed_slice()), }; col_pages[col_idx].push(new_page_info); } - } // finished scheduling & copying pages for this column in the current source file + } // finished scheduling & copying pages for this column in the current source file - if !src_column_info.buffer_offsets_and_sizes.is_empty() { + if !src_column_info.buffer_offsets_and_sizes.is_empty() { // Validate column-level encoding compatibility before copying buffers - let src_col_encoding_bytes = Any::from_msg(&src_column_info.encoding) - .unwrap() - .encode_to_vec(); + let src_col_encoding_bytes = + Any::from_msg(&src_column_info.encoding)?.encode_to_vec(); let baseline_bytes = &baseline_col_encoding_bytes[col_idx]; if src_col_encoding_bytes != *baseline_bytes { return Err(Error::execution(format!( @@ -434,24 +341,29 @@ pub async fn rewrite_files_binary_copy( for (buffer_idx, (_, size)) in src_column_info.buffer_offsets_and_sizes.iter().enumerate() { - let writer = current_writer.as_mut().unwrap().as_mut(); - current_pos = apply_alignment_padding(writer, current_pos, version).await?; - let start = current_pos; - if *size == 0 { - col_buffers[col_idx].push((start, 0)); + let writer = current_writer.as_mut().ok_or_else(|| { + Error::internal("binary copy output writer was not initialized") + })?; + let bytes = if *size == 0 { + None } else { - let bytes = bytes_iter.next().ok_or_else(|| { + Some(bytes_iter.next().ok_or_else(|| { Error::execution(format!( "binary copy: missing column buffer bytes while rewriting data file \ (column {col_idx}, buffer {buffer_idx}, expected size {size})", )) - })?; - writer.write_all(&bytes).await?; - current_pos += bytes.len() as u64; - col_buffers[col_idx].push((start, bytes.len() as u64)); - } + })?) + }; + let (start, written) = writer + .write_external_buffer(bytes.as_deref().unwrap_or_default()) + .await?; + col_buffers[col_idx].push((start, written)); } - } + } + Ok(()) + }, + ) + .await?; } // finished all columns in the current source file // Accumulate rows for the current output file and flush when reaching the threshold @@ -459,21 +371,18 @@ pub async fn rewrite_files_binary_copy( if total_rows_in_current >= max_rows_per_file { let fragment_out = finalize_current_output_file( &schema, - &full_field_ids, + version, &mut current_writer, &mut current_filename, ¤t_page_table, &mut col_pages, &mut col_buffers, - &is_non_leaf_column, total_rows_in_current, - version, ) .await?; // Reset state for next output file current_writer = None; - current_pos = 0; current_page_table.clear(); for v in col_pages.iter_mut() { v.clear(); @@ -489,18 +398,17 @@ pub async fn rewrite_files_binary_copy( if total_rows_in_current > 0 { // Flush remaining rows as a final output file - init_writer_if_necessary(dataset, &mut current_writer, &mut current_filename).await?; + init_writer_if_necessary(dataset, version, &mut current_writer, &mut current_filename) + .await?; let frag = finalize_current_output_file( &schema, - &full_field_ids, + version, &mut current_writer, &mut current_filename, ¤t_page_table, &mut col_pages, &mut col_buffers, - &is_non_leaf_column, total_rows_in_current, - version, ) .await?; out.push(frag); @@ -512,9 +420,7 @@ pub async fn rewrite_files_binary_copy( /// /// This function does not manually craft the footer. Instead it: /// - Pads the current `ObjectWriter` position to a 64‑byte boundary (required for v2_1+ readers). -/// - Converts the collected per‑column info (`final_cols`) into `ColumnMetadata`. -/// - Constructs a `lance_file::writer::FileWriter` with the active `schema`, column metadata, -/// and `total_rows_in_current`. +/// - Initializes the active `FileWriter` from the collected column metadata. /// - Calls `FileWriter::finish()` to emit column metadata, offset tables, global buffers /// (schema descriptor), version, and to close the writer. /// @@ -522,81 +428,14 @@ pub async fn rewrite_files_binary_copy( /// - All page data and column‑level buffers referenced by `final_cols` have already been written /// to `writer`; otherwise offsets in the footer will be invalid. /// -/// Version notes: -/// - v2_0 structural single‑page enforcement is handled when building `final_cols`; this function -/// only performs consistent finalization. async fn flush_footer( - mut writer: Box, + writer: &mut FileWriter, schema: &Schema, final_cols: &[Arc], total_rows_in_current: u64, - version: LanceFileVersion, ) -> Result<()> { - let pos = writer.tell().await? as u64; - let _new_pos = apply_alignment_padding(writer.as_mut(), pos, version).await?; - - let mut col_metadatas = Vec::with_capacity(final_cols.len()); - for col in final_cols { - let pages = col - .page_infos - .iter() - .map(|page_info| { - let encoded_encoding = match &page_info.encoding { - PageEncoding::Legacy(array_encoding) => { - Any::from_msg(array_encoding)?.encode_to_vec() - } - PageEncoding::Structural(page_layout) => { - Any::from_msg(page_layout)?.encode_to_vec() - } - }; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = page_info - .buffer_offsets_and_sizes - .as_ref() - .iter() - .cloned() - .unzip(); - Ok(pbfile::column_metadata::Page { - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct( - pbfile::DirectEncoding { - encoding: encoded_encoding, - }, - )), - }), - length: page_info.num_rows, - priority: page_info.priority, - }) - }) - .collect::>>()?; - let (buffer_offsets, buffer_sizes): (Vec<_>, Vec<_>) = - col.buffer_offsets_and_sizes.iter().cloned().unzip(); - let encoded_col_encoding = Any::from_msg(&col.encoding)?.encode_to_vec(); - let column = pbfile::ColumnMetadata { - pages, - buffer_offsets, - buffer_sizes, - encoding: Some(pbfile::Encoding { - location: Some(pbfile::encoding::Location::Direct(pbfile::DirectEncoding { - encoding: encoded_col_encoding, - })), - }), - }; - col_metadatas.push(column); - } - let mut file_writer = FileWriter::new_lazy( - writer, - FileWriterOptions { - format_version: Some(version), - ..Default::default() - }, - ); - file_writer.initialize_with_external_metadata( - schema.clone(), - col_metadatas, - total_rows_in_current, - ); - file_writer.finish().await?; + writer.write_external_buffer(&[]).await?; + writer.initialize_with_external_columns(schema.clone(), final_cols, total_rows_in_current)?; + writer.finish().await?; Ok(()) } diff --git a/rust/lance/src/dataset/optimize/remapping.rs b/rust/lance/src/dataset/optimize/remapping.rs index aef2cd231fc..8a9c8898cb6 100644 --- a/rust/lance/src/dataset/optimize/remapping.rs +++ b/rust/lance/src/dataset/optimize/remapping.rs @@ -61,8 +61,13 @@ pub trait IndexRemapper: Send + Sync { /// /// Currently we don't have any options but we may need options in the future and so we /// want to keep a placeholder +#[async_trait] pub trait IndexRemapperOptions: Send + Sync { - fn create_remapper(&self, dataset: &Dataset) -> Result>; + /// Creates a remapper when the dataset has indices that need row address remapping. + /// + /// Returns `None` when no remappable indices exist, allowing compaction to avoid + /// materializing an unused row address map. + async fn create_remapper(&self, dataset: &Dataset) -> Result>>; } #[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] @@ -75,9 +80,10 @@ impl IndexRemapper for IgnoreRemap { } } +#[async_trait] impl IndexRemapperOptions for IgnoreRemap { - fn create_remapper(&self, _: &Dataset) -> Result> { - Ok(Box::new(Self {})) + async fn create_remapper(&self, _: &Dataset) -> Result>> { + Ok(None) } } @@ -197,7 +203,7 @@ pub fn transpose_row_ids_from_digest( /// If the frag reuse index does not exist, the operation fails with [Error::NotSupported] /// If the frag reuse index exists but is empty, the operation succeeds without a commit. async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { - let indices = dataset.load_indices().await.unwrap(); + let indices = dataset.load_indices().await?; let frag_reuse_index_meta = match indices.iter().find(|idx| idx.name == FRAG_REUSE_INDEX_NAME) { None => Err(Error::not_supported_source( "Fragment reuse index not found, cannot remap an index post compaction".into(), @@ -205,15 +211,11 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { Some(frag_reuse_index_meta) => Ok(frag_reuse_index_meta), }?; - let frag_reuse_details = load_frag_reuse_index_details(dataset, frag_reuse_index_meta) - .await - .unwrap(); + let frag_reuse_details = load_frag_reuse_index_details(dataset, frag_reuse_index_meta).await?; let frag_reuse_index = - open_frag_reuse_index(frag_reuse_index_meta.uuid, frag_reuse_details.as_ref()) - .await - .unwrap(); + open_frag_reuse_index(frag_reuse_index_meta.uuid, frag_reuse_details.as_ref()).await?; - if frag_reuse_index.row_id_maps.is_empty() { + if frag_reuse_index.is_empty() { return Ok(()); } @@ -243,7 +245,8 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { // coverage-remapped + persisted before the data was remapped (e.g. while // remapping a *sibling* index). let baseline_version = curr_index_meta.dataset_version; - let (should_remap, bitmap_after_remap) = match curr_index_meta.fragment_bitmap.clone() { + let has_unknown_coverage = curr_index_meta.fragment_bitmap.is_none(); + let (should_remap, mut bitmap_after_remap) = match curr_index_meta.fragment_bitmap.clone() { Some(mut index_frag_bitmap) => { let mut should_remap = false; for version in frag_reuse_index.details.versions.iter() { @@ -285,8 +288,6 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { } (should_remap, Some(index_frag_bitmap)) } - // if there is no fragment bitmap for the index, - // we attempt remapping but will not update the fragment bitmap. None => (true, None), }; @@ -294,38 +295,55 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { return Ok(()); } - // Compose the row-address remap across all versions. `remap_row_id` already - // chains every version (and passes through addresses a version does not - // touch), so mapping the union of all versions' keys yields a single - // baseline -> final address map applied in one rebuild. - // - // Map every old address; do NOT filter by the current `fragment_bitmap`. In - // the sibling-coverage-remap case the bitmap was already advanced onto the - // new fragments while the index data still holds old addresses, so filtering - // by it would drop exactly the keys this index needs and leave its data - // stale (an empty map makes `index::remap_index` return `Keep`). The map is - // bounded by the rows the reuse index touched; addresses this index does not - // store are simply never looked up. - let composed_row_id_map: HashMap> = frag_reuse_index - .row_id_maps - .iter() - .flat_map(|row_id_map| row_id_map.keys().copied()) - .map(|old_addr| (old_addr, frag_reuse_index.remap_row_id(old_addr))) - .collect(); - - let remapper = RowAddrRemap::direct(composed_row_id_map); - let remap_result = index::remap_index(dataset, index_id, &remapper).await?; + // Apply the compact version chain directly while rebuilding the index. The + // remapper passes intermediate moved addresses into later FRI versions and + // leaves missing mappings unchanged, so no composed per-row map is needed. + // This also handles the sibling-coverage-remap case: remapping is driven by + // the row addresses stored in the index, not by its already-advanced bitmap. + let remap_result = + index::remap_index(dataset, index_id, frag_reuse_index.row_addr_remap()).await?; + + // Remapping advances the index watermark for fragment-reuse cleanup, but it + // does not incorporate overlays committed after the source index was built. + // Exclude those fragments so queries scan their current values instead. + if let Some(fragment_bitmap) = &mut bitmap_after_remap { + for fragment in dataset.manifest.fragments.iter() { + let has_newer_indexed_overlay = fragment.overlays.iter().any(|overlay| { + overlay.committed_version > curr_index_meta.dataset_version + && overlay + .data_file + .fields + .iter() + .any(|field_id| curr_index_meta.fields.contains(field_id)) + }); + if has_newer_indexed_overlay { + fragment_bitmap.remove(fragment.id as u32); + } + } + } + let new_dataset_version = if has_unknown_coverage { + curr_index_meta.dataset_version + } else { + dataset.manifest.version + }; let new_index_meta = match remap_result { - // The composed remap emptied the index (every row deleted). Matching the - // prior per-version behavior, leave the existing index untouched and - // commit nothing -- there is no remap to apply. + // Nothing to commit: either the composed remap emptied the index (every + // row deleted), matching the prior per-version behavior, or + // `index::remap_index` withdrew a covered index it cannot carry payload + // through. Either way the existing entry is left untouched. + // + // The withdrawal case is unreachable here today: the only caller is + // `remap_column_index`, which refuses a covered index first. Compaction + // reaches that withdrawal through `DatasetIndexRemapper`, which handles + // `RemapResult::Drop` in `dataset/index.rs` rather than through here. RemapResult::Drop => return Ok(()), RemapResult::Keep(new_id) => IndexMetadata { uuid: new_id, name: curr_index_meta.name.clone(), fields: curr_index_meta.fields.clone(), - dataset_version: dataset.manifest.version, + covering_fields: curr_index_meta.covering_fields.clone(), + dataset_version: new_dataset_version, fragment_bitmap: bitmap_after_remap, index_details: curr_index_meta.index_details.clone(), index_version: curr_index_meta.index_version, @@ -337,7 +355,8 @@ async fn remap_index(dataset: &mut Dataset, index_id: &Uuid) -> Result<()> { uuid: remapped_index.new_id, name: curr_index_meta.name.clone(), fields: curr_index_meta.fields.clone(), - dataset_version: dataset.manifest.version, + covering_fields: curr_index_meta.covering_fields.clone(), + dataset_version: new_dataset_version, fragment_bitmap: bitmap_after_remap, index_details: Some(Arc::new(remapped_index.index_details)), index_version: remapped_index.index_version as i32, @@ -391,10 +410,26 @@ pub async fn remap_column_index( ))); } Some(index) => { - if index.fields != [field.id] { + // The real question is "does this index belong to this column", + // i.e. its one keyed field is `field.id`. Carried fields are + // irrelevant here, same as in `index::remap_index`. + if index.keyed_field() != Some(field.id) { + Err(Error::index(format!( + "Index name {} already exists with fields {:?} (carried fields {:?}); \ + expected a single keyed field {}", + index_name, index.fields, index.covering_fields, field.id + ))) + } else if !index.covering_fields.is_empty() { + // Same rule as `optimize_indices`, and for the same reason: no + // index type carries the declared payload through a remap, so the + // result would still claim values its storage does not hold. The + // caller named this index, so refuse out loud -- compaction + // withdraws instead only because it must not block a table-level + // operation over one index it cannot remap. Err(Error::index(format!( - "Index name {} already exists with different fields", - index_name + "Remapping index '{}' is not supported: it declares covering \ + fields {:?}, which no index builder writes or preserves yet", + index_name, index.covering_fields, ))) } else { Ok(index) @@ -411,7 +446,7 @@ mod tests { #[test] fn test_compact_matches_transpose() { - use lance_core::utils::row_addr_remap::GroupInput; + use lance_core::utils::row_addr_remap::GroupInputWithLayout; // Ascending old fragments (compaction's scan order), with deletions. let old = vec![ FragDigest { @@ -462,9 +497,12 @@ mod tests { ]; let expected = transpose_row_ids_from_digest(addrs.clone(), &old, &new); - let compact = RowAddrRemap::compact([GroupInput { + let compact = RowAddrRemap::compact_with_layout([GroupInputWithLayout { rewritten_old_row_addrs: addrs, - old_frag_ids: old.iter().map(|f| f.id as u32).collect(), + old_frags: old + .iter() + .map(|f| (f.id as u32, f.physical_rows as u32)) + .collect(), new_frags: new .iter() .map(|f| (f.id as u32, f.physical_rows as u32)) @@ -565,4 +603,102 @@ mod tests { .collect::>(); assert_eq!(result, expected); } + + /// A *physical* remap through the real production entry point, + /// `remap_column_index`, must refuse a covered index rather than quietly do + /// nothing. No index type carries the declared payload through a remap, and + /// the caller named this index, so a silent no-op would hand back an index + /// that covers nothing with no indication why. + /// + /// Asserting on committed metadata here would prove nothing: the flow is + /// `remap_column_index` -> the private `remap_index` -> `index::remap_index`, + /// which withdraws a covered index before the fully-deleted `Keep` check, and + /// the `Drop` arm returns without committing. The `Keep`/`Remapped` arms are + /// therefore unreachable for a covered index, so a "declaration preserved" + /// assertion would pass even if those arms stopped preserving it. Assert the + /// refusal instead. + #[tokio::test] + async fn test_remap_column_index_refuses_a_covered_index() { + use crate::dataset::index::DatasetIndexRemapperOptions; + use crate::dataset::optimize::{ + CompactionOptions, commit_compaction, plan_compaction, rewrite_files, + }; + use crate::utils::test::covering; + use lance_core::utils::tempfile::TempStrDir; + use std::borrow::Cow; + + let test_uri = TempStrDir::default(); + // Two fragments, so compaction has something to merge. + let mut dataset = covering::write_vector_payload_dataset(&test_uri).await; + covering::append_vector_payload_rows(&mut dataset, covering::ROWS_PER_FRAGMENT).await; + covering::create_ivf_pq_index(&mut dataset, "vec").await; + + let (_, id_field_id) = covering::declare_covering(&mut dataset, "vec", "payload").await; + let index_name = dataset.load_indices().await.unwrap()[0].name.clone(); + + // Delete some (not all) rows so the remap has real work to do and does + // not take the all-fragments-deleted `RemapResult::Keep` shortcut. + dataset.delete("payload < 100").await.unwrap(); + + let options = CompactionOptions { + defer_index_remap: true, + ..Default::default() + }; + let plan = plan_compaction(&dataset, &options).await.unwrap(); + assert!( + !plan.tasks().is_empty(), + "compaction plan must have work to do, or this test proves nothing" + ); + for task in plan.tasks().iter() { + let rewrite_result = rewrite_files(Cow::Borrowed(&dataset), task.clone(), &options) + .await + .unwrap(); + commit_compaction( + &mut dataset, + Vec::from([rewrite_result]), + Arc::new(DatasetIndexRemapperOptions::default()), + &options, + ) + .await + .unwrap(); + } + + let before = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| idx.name == index_name) + .cloned() + .expect("precondition: the covered index exists"); + + // `remap_column_index` is user-directed -- the caller named this index -- + // so it must refuse rather than no-op. Refusing is also what makes this + // test meaningful: the `Keep`/`Remapped` arms below are unreachable for a + // covered index, so asserting on the committed metadata instead would + // pass whether or not those arms preserved `covering_fields`. + let error = remap_column_index(&mut dataset, &["vec"], Some(index_name.clone())) + .await + .expect_err("remapping a covered index must be refused"); + assert!( + error.to_string().contains("declares covering fields"), + "unexpected message: {error}" + ); + + // Refused, not half-applied. + let after = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| idx.name == index_name) + .cloned() + .expect("a refused remap must leave the index in place"); + assert_eq!( + after.uuid, before.uuid, + "a refused remap replaced the index" + ); + assert_eq!(after.covering_fields, vec![id_field_id]); + assert_eq!(after.fragment_bitmap, before.fragment_bitmap); + } } diff --git a/rust/lance/src/dataset/optimize/tests/binary_copy.rs b/rust/lance/src/dataset/optimize/tests/binary_copy.rs index 08a4fe024ce..174baf6d028 100644 --- a/rust/lance/src/dataset/optimize/tests/binary_copy.rs +++ b/rust/lance/src/dataset/optimize/tests/binary_copy.rs @@ -2,10 +2,18 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use super::*; +use arrow_array::{Decimal128Array, UInt64Array}; + +const NON_LEGACY_VERSIONS: [LanceFileVersion; 4] = [ + LanceFileVersion::V2_0, + LanceFileVersion::V2_1, + LanceFileVersion::V2_2, + LanceFileVersion::V2_3, +]; #[tokio::test] async fn test_binary_copy_merge_small_files() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_test_binary_copy_merge_small_files(version).await; } } @@ -45,9 +53,146 @@ async fn do_test_binary_copy_merge_small_files(version: LanceFileVersion) { assert_eq!(before, after); } +#[tokio::test] +async fn test_binary_copy_falls_back_for_non_schema_column_order() { + let decimal_type = DataType::Decimal128(38, 10); + let dataset_schema = Arc::new(Schema::new(vec![ + Field::new("v_dec", decimal_type.clone(), true), + Field::new("v_u64", DataType::UInt64, true), + ])); + let write_params = WriteParams { + max_rows_per_file: 1, + data_storage_version: Some(LanceFileVersion::V2_3), + ..Default::default() + }; + let test_dir = TempStrDir::default(); + let empty_batch = RecordBatch::new_empty(dataset_schema.clone()); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(empty_batch)], dataset_schema), + &test_dir, + Some(write_params.clone()), + ) + .await + .unwrap(); + + let decimal_values = Decimal128Array::from_iter_values([ + 201_000_000_000_000_000_000_000_i128, + 202_000_000_000_000_000_000_000_i128, + ]) + .with_precision_and_scale(38, 10) + .unwrap(); + let swapped_schema = Arc::new(Schema::new(vec![ + Field::new("v_u64", DataType::UInt64, true), + Field::new("v_dec", decimal_type, true), + ])); + let swapped_batch = RecordBatch::try_new( + swapped_schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![201, 202])), + Arc::new(decimal_values), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(swapped_batch)], swapped_schema), + Some(write_params), + ) + .await + .unwrap(); + + let fragments: Vec = dataset + .get_fragments() + .into_iter() + .map(Into::into) + .collect(); + assert_eq!(fragments.len(), 2); + for fragment in &fragments { + assert_eq!(fragment.files[0].fields.as_ref(), &[1, 0]); + assert_eq!(fragment.files[0].column_indices.as_ref(), &[0, 1]); + } + + let options = CompactionOptions { + target_rows_per_fragment: 8, + compaction_mode: Some(CompactionMode::TryBinaryCopy), + ..Default::default() + }; + assert!(!can_use_binary_copy(&dataset, &options, &fragments).await); + let before = dataset.scan().try_into_batch().await.unwrap(); + + compact_files(&mut dataset, options, None).await.unwrap(); + + let after = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!(before, after); + let compacted_file = &dataset.manifest.fragments[0].files[0]; + assert_eq!(compacted_file.fields.as_ref(), &[0, 1]); + assert_eq!(compacted_file.column_indices.as_ref(), &[0, 1]); +} + +#[tokio::test] +async fn test_binary_copy_packed_struct_column_mapping() { + for version in NON_LEGACY_VERSIONS { + do_test_binary_copy_packed_struct_column_mapping(version).await; + } +} + +async fn do_test_binary_copy_packed_struct_column_mapping(version: LanceFileVersion) { + use arrow_array::StructArray; + use arrow_schema::Fields; + use std::collections::HashMap; + + let packed_fields = Fields::from(vec![Field::new("child", DataType::Int32, true)]); + let packed_field = Field::new("packed", DataType::Struct(packed_fields.clone()), true) + .with_metadata(HashMap::from([("packed".to_string(), "true".to_string())])); + let schema = Arc::new(Schema::new(vec![ + packed_field, + Field::new("tail", DataType::Int32, true), + ])); + let packed: ArrayRef = Arc::new(StructArray::new( + packed_fields, + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4]))], + None, + )); + let tail: ArrayRef = Arc::new(Int32Array::from(vec![10, 20, 30, 40])); + let batch = RecordBatch::try_new(schema.clone(), vec![packed, tail]).unwrap(); + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(version), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + let before = dataset.scan().try_into_batch().await.unwrap(); + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 100_000, + compaction_mode: Some(CompactionMode::ForceBinaryCopy), + ..Default::default() + }, + None, + ) + .await + .unwrap(); + let after = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!(before, after); + + let data_file = &dataset.manifest.fragments[0].files[0]; + assert_eq!(data_file.fields.len(), 2); + assert_eq!(data_file.column_indices.as_ref(), &[0, 1]); +} + #[tokio::test] async fn test_binary_copy_empty_string_scalar_index() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_test_binary_copy_empty_string_scalar_index(version).await; } } @@ -115,7 +260,7 @@ async fn do_test_binary_copy_empty_string_scalar_index(version: LanceFileVersion #[tokio::test] async fn test_binary_copy_with_defer_remap() { - for version in LanceFileVersion::iter_non_legacy() { + for version in NON_LEGACY_VERSIONS { do_test_binary_copy_with_defer_remap(version).await; } } @@ -185,11 +330,14 @@ async fn do_test_binary_copy_with_defer_remap(version: LanceFileVersion) { assert_eq!(before_batch, after_batch); } +#[rstest::rstest] +#[case(LanceFileVersion::V2_0)] +#[case(LanceFileVersion::V2_1)] +#[case(LanceFileVersion::V2_2)] +#[case(LanceFileVersion::V2_3)] #[tokio::test] -async fn test_binary_copy_preserves_stable_row_ids() { - for version in LanceFileVersion::iter_non_legacy() { - do_binary_copy_preserves_stable_row_ids(version).await; - } +async fn test_binary_copy_preserves_stable_row_ids(#[case] version: LanceFileVersion) { + do_binary_copy_preserves_stable_row_ids(version).await; } async fn do_binary_copy_preserves_stable_row_ids(version: LanceFileVersion) { @@ -201,17 +349,18 @@ async fn do_binary_copy_preserves_stable_row_ids(version: LanceFileVersion) { .col(Box::new(IncrementingInt32::new().named("i".to_owned()))); let mut dataset = Dataset::write( - data_gen.batch(4_000), + data_gen.batch(1_024), format!("memory://test/binary_copy_stable_row_ids_{}", version).as_str(), Some(WriteParams { enable_stable_row_ids: true, data_storage_version: Some(version), - max_rows_per_file: 500, + max_rows_per_file: 256, ..Default::default() }), ) .await .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); dataset .create_index( @@ -276,7 +425,7 @@ async fn do_binary_copy_preserves_stable_row_ids(version: LanceFileVersion) { .unwrap(); let options = CompactionOptions { - target_rows_per_fragment: 2_000, + target_rows_per_fragment: 512, compaction_mode: Some(CompactionMode::ForceBinaryCopy), ..Default::default() }; @@ -318,11 +467,14 @@ async fn do_binary_copy_preserves_stable_row_ids(version: LanceFileVersion) { assert_eq!(before, after); } +#[rstest::rstest] +#[case(LanceFileVersion::V2_0)] +#[case(LanceFileVersion::V2_1)] +#[case(LanceFileVersion::V2_2)] +#[case(LanceFileVersion::V2_3)] #[tokio::test] -async fn test_binary_copy_remaps_unstable_row_ids() { - for version in LanceFileVersion::iter_non_legacy() { - do_binary_copy_remaps_unstable_row_ids(version).await; - } +async fn test_binary_copy_remaps_unstable_row_ids(#[case] version: LanceFileVersion) { + do_binary_copy_remaps_unstable_row_ids(version).await; } async fn do_binary_copy_remaps_unstable_row_ids(version: LanceFileVersion) { @@ -333,17 +485,18 @@ async fn do_binary_copy_remaps_unstable_row_ids(version: LanceFileVersion) { .col(Box::new(IncrementingInt32::new().named("i".to_owned()))); let mut dataset = Dataset::write( - data_gen.batch(4_000), - "memory://test/binary_copy_no_stable", + data_gen.batch(1_024), + format!("memory://test/binary_copy_no_stable_{version}").as_str(), Some(WriteParams { enable_stable_row_ids: false, data_storage_version: Some(version), - max_rows_per_file: 500, + max_rows_per_file: 256, ..Default::default() }), ) .await .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); dataset .create_index( @@ -395,7 +548,7 @@ async fn do_binary_copy_remaps_unstable_row_ids(version: LanceFileVersion) { .unwrap(); let options = CompactionOptions { - target_rows_per_fragment: 2_000, + target_rows_per_fragment: 512, compaction_mode: Some(CompactionMode::ForceBinaryCopy), ..Default::default() }; @@ -685,7 +838,8 @@ async fn test_can_use_binary_copy_version_mismatch() { ); // Simulate mixed file versions by marking the second fragment as v2.1. - let (v21_major, v21_minor) = LanceFileVersion::V2_1.to_numbers(); + let (v21_major, v21_minor) = + lance_file::version::ConcreteFileVersion::V2_1.to_data_file_numbers(); for file in &mut frags[1].files { file.file_major_version = v21_major; file.file_minor_version = v21_minor; @@ -721,11 +875,27 @@ async fn test_can_use_binary_copy_reject_deletions() { assert!(!can_use_binary_copy(&dataset, &options, &frags).await); } +#[rstest::rstest] +#[case(LanceFileVersion::V2_0)] +#[case(LanceFileVersion::V2_1)] +#[case(LanceFileVersion::V2_2)] +#[case(LanceFileVersion::V2_3)] #[tokio::test] -async fn test_binary_copy_compaction_with_complex_schema() { - for version in LanceFileVersion::iter_non_legacy() { - do_test_binary_copy_compaction_with_complex_schema(version).await; - } +async fn test_binary_copy_compaction_with_complex_schema(#[case] version: LanceFileVersion) { + do_test_binary_copy_compaction_with_complex_schema(version).await; +} + +#[test] +fn test_binary_copy_complex_schema_covers_every_non_legacy_version() { + assert_eq!( + [ + LanceFileVersion::V2_0, + LanceFileVersion::V2_1, + LanceFileVersion::V2_2, + LanceFileVersion::V2_3 + ], + NON_LEGACY_VERSIONS + ); } async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVersion) { @@ -733,7 +903,10 @@ async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVe use lance_core::utils::tempfile::TempStrDir; use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch}; - let row_num = 1_000; + let row_num: u64 = 1_000; + let batches: u32 = 10; + const NUM_FRAGMENTS: usize = 100; + let rows_per_file = (row_num as usize * batches as usize) / NUM_FRAGMENTS; let inner_fields = Fields::from(vec![ Field::new("x", DataType::UInt32, true), @@ -794,7 +967,7 @@ async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVe "events", array::rand_list_any(array::rand_struct(event_fields.clone()), true), ) - .into_reader_rows(RowCount::from(row_num), BatchCount::from(10)); + .into_reader_rows(RowCount::from(row_num), BatchCount::from(batches)); let full_dir = TempStrDir::default(); let mut dataset = Dataset::write( @@ -803,13 +976,19 @@ async fn do_test_binary_copy_compaction_with_complex_schema(version: LanceFileVe Some(WriteParams { enable_stable_row_ids: true, data_storage_version: Some(version), - max_rows_per_file: (row_num / 100) as usize, + max_rows_per_file: rows_per_file, ..Default::default() }), ) .await .unwrap(); + assert_eq!( + dataset.get_fragments().len(), + NUM_FRAGMENTS, + "compaction must have many input fragments to merge" + ); + let opt_full = CompactionOptions { compaction_mode: Some(CompactionMode::Reencode), ..Default::default() diff --git a/rust/lance/src/dataset/overlay.rs b/rust/lance/src/dataset/overlay.rs new file mode 100644 index 00000000000..1f55c545559 --- /dev/null +++ b/rust/lance/src/dataset/overlay.rs @@ -0,0 +1,1071 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Resolution of data overlay files on read. +//! +//! An overlay supplies replacement values for some `(row, field)` cells without +//! rewriting the base data. Resolving a read means, for each row we return, +//! deciding whether its value comes from the base column or from an overlay. +//! +//! Three coordinate spaces show up throughout this module; keeping them straight +//! is most of the work: +//! +//! - `offset_in_frag`: a row's physical position in the fragment (0-based over all +//! physical rows, ignoring deletions). This is how a cell is addressed on disk +//! and in an overlay's coverage bitmap. +//! - `offset_in_batch`: a row's position within the batch we are currently +//! assembling (0-based). The output column is indexed by this. +//! - `offset_in_overlay`: the position of a value in an overlay's value column. +//! An overlay stores its values densely — one per covered cell, in ascending +//! `offset_in_frag` order — so a covered cell's value is found by counting how +//! many covered cells come before it. (That count is what a roaring bitmap calls +//! the cell's "rank".) +//! +//! For a given field, the overlays covering it are consulted newest to oldest: the +//! first overlay that covers a row wins, and its value is read at that row's +//! `offset_in_overlay`. A row that no overlay covers keeps its base value. +//! +//! The rows to resolve are passed in as a list of `offset_in_frag` (one per output +//! row), so a single code path serves both scans (a contiguous range of offsets) +//! and `take` (arbitrary offsets). +//! +//! Deletions win over overlays, but nothing here handles that: the merge runs on +//! physical rows *before* deletions are applied, so an overlay value computed for a +//! deleted row is simply dropped along with the row. This matches the spec with no +//! special casing. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; + +use arrow_array::{Array, ArrayRef, RecordBatch, StructArray}; +use arrow_select::interleave::interleave; +use futures::StreamExt; +use lance_core::datatypes::{Field, Schema}; +use lance_core::{Error, Result}; +use roaring::RoaringBitmap; + +use lance_table::format::DataFile; +use lance_table::utils::stream::ReadBatchFut; + +use crate::dataset::fragment::{FileFragment, FragReadConfig, GenericFileReader}; + +// Deciding which rows an overlay makes stale needs only fragment and index metadata, +// so it lives at the table layer; this module resolves the reads that consume it. +pub use lance_table::format::overlay::staleness::{ + collect_overlay_stale_frags, collect_overlay_stale_rows_for_segment, overlaid_fragments, +}; + +/// The plan for merging one field's overlays into one batch: which source (base or +/// a particular overlay) supplies each output row, and which overlay values must be +/// fetched to do it. +/// +/// Built by [`route_overlays`] from the coverage bitmaps alone — before any value +/// column is read — so the caller can fetch only the overlay values it will +/// actually use (its `offsets_in_overlay`) rather than whole columns, then build the +/// merged column with [`assemble_overlay_column`]. +struct OverlayRouting { + /// One `(source, position)` pair per output row, ready to hand to `interleave`. + /// Source `0` is the base column, with `position` = the row's `offset_in_batch`; + /// source `k + 1` is overlay `k`'s fetched values, with `position` = the row's + /// index into those fetched values. + indices: Vec<(usize, usize)>, + /// Per overlay (newest-first): the sorted, deduplicated `offset_in_overlay` + /// values this batch needs from that overlay — i.e. exactly which entries of its + /// value column to fetch. + offsets_in_overlay: Vec>, + /// Whether any row is covered by an overlay at all (false ⇒ every row falls + /// through to the base column, so the base is already the answer and no overlay + /// values need to be read). + any_overlay: bool, +} + +/// For each row in `offsets_in_frag`, decide whether its value comes from the base +/// column or from an overlay — and if from an overlay, at which `offset_in_overlay`. +/// +/// Only the coverage bitmaps are consulted (newest-first), so this runs before any +/// value column is read and reports exactly which overlay values the caller must +/// fetch. +/// +/// A scan asks for a contiguous, ascending range of offsets, which enables a faster +/// bitmap-driven path ([`route_contiguous`]); `take` asks for arbitrary offsets and +/// uses the general path ([`route_arbitrary`]). Both produce identical routing. +fn route_overlays( + offsets_in_frag: &[u32], + coverages_newest_first: &[&RoaringBitmap], +) -> OverlayRouting { + match contiguous_frag_start(offsets_in_frag) { + Some(frag_start) => { + route_contiguous(frag_start, offsets_in_frag.len(), coverages_newest_first) + } + None => route_arbitrary(offsets_in_frag, coverages_newest_first), + } +} + +/// If `offsets_in_frag` is a contiguous ascending run `[start, start + 1, ...]`, +/// return `start`; otherwise `None` (including when empty). +fn contiguous_frag_start(offsets_in_frag: &[u32]) -> Option { + let start = *offsets_in_frag.first()?; + offsets_in_frag + .iter() + .enumerate() + .all(|(i, &offset)| offset as u64 == start as u64 + i as u64) + .then_some(start) +} + +/// Fast path for a scan, where the batch is a contiguous run of offsets starting at +/// `frag_start`. Because the offsets are contiguous, a row's `offset_in_batch` is +/// just `offset_in_frag - frag_start`, so a coverage's set bits map straight to +/// output rows — no need to test each row against each coverage. +/// +/// For each coverage we intersect it with the batch's offset range. Roaring does +/// this a block at a time, so a coverage that does not overlap the batch (e.g. a +/// scan batch past the last cell this overlay touches) is skipped cheaply without +/// inspecting individual bits. +/// +/// Within the batch a coverage's cells appear in ascending order, so their +/// `offset_in_overlay` values are consecutive: the first in-batch cell sits at +/// `offset_in_overlay = ` (a single +/// `rank` lookup), and each following cell is one more. Coverages are applied +/// newest-first, and the first overlay to claim a row wins. +fn route_contiguous( + frag_start: u32, + len: usize, + coverages_newest_first: &[&RoaringBitmap], +) -> OverlayRouting { + let mut offsets_in_overlay: Vec> = vec![Vec::new(); coverages_newest_first.len()]; + // Indexed by offset_in_batch: which (overlay, fetch position) supplies the row. + let mut routed: Vec> = vec![None; len]; + let range_end = (frag_start as u64 + len as u64).min(u32::MAX as u64) as u32; + let mut batch_range = RoaringBitmap::new(); + batch_range.insert_range(frag_start..range_end); + + for (k, coverage) in coverages_newest_first.iter().enumerate() { + let covered_in_batch = *coverage & &batch_range; + if covered_in_batch.is_empty() { + continue; + } + // offset_in_overlay of this coverage's first in-batch cell = the number of + // its cells that lie before the batch. + let first_offset_in_overlay = if frag_start == 0 { + 0 + } else { + coverage.rank(frag_start - 1) as u32 + }; + for (nth_in_batch, offset_in_frag) in covered_in_batch.iter().enumerate() { + let offset_in_batch = (offset_in_frag - frag_start) as usize; + if routed[offset_in_batch].is_none() { + routed[offset_in_batch] = Some((k, offsets_in_overlay[k].len())); + offsets_in_overlay[k].push(first_offset_in_overlay + nth_in_batch as u32); + } + } + } + + let mut any_overlay = false; + let indices = routed + .into_iter() + .enumerate() + .map(|(offset_in_batch, routed)| match routed { + None => (0, offset_in_batch), + Some((k, fetch_pos)) => { + any_overlay = true; + (k + 1, fetch_pos) + } + }) + .collect(); + + OverlayRouting { + indices, + offsets_in_overlay, + any_overlay, + } +} + +/// General path for arbitrary offsets (e.g. `take`): test each row's +/// `offset_in_frag` against the coverages newest-first. `take` batches are small, +/// so this `O(rows * overlays)` probing is not a concern. +fn route_arbitrary( + offsets_in_frag: &[u32], + coverages_newest_first: &[&RoaringBitmap], +) -> OverlayRouting { + // Per overlay: the distinct offset_in_overlay values this batch needs, sorted. + let mut offset_sets: Vec> = vec![BTreeSet::new(); coverages_newest_first.len()]; + // Per output row: the (overlay, offset_in_overlay) that supplies it, if any. + let mut routed_per_row: Vec> = Vec::with_capacity(offsets_in_frag.len()); + for &offset_in_frag in offsets_in_frag { + let mut routed = None; + for (k, coverage) in coverages_newest_first.iter().enumerate() { + if coverage.contains(offset_in_frag) { + // offset_in_overlay = number of covered cells before this one. + let offset_in_overlay = coverage.rank(offset_in_frag) as u32 - 1; + offset_sets[k].insert(offset_in_overlay); + routed = Some((k, offset_in_overlay)); + break; + } + } + routed_per_row.push(routed); + } + + let offsets_in_overlay: Vec> = offset_sets + .iter() + .map(|offsets| offsets.iter().copied().collect()) + .collect(); + // For each overlay, map an offset_in_overlay to its position in the fetched + // (sorted, deduplicated) value list. + let fetch_positions: Vec> = offsets_in_overlay + .iter() + .map(|offsets| { + offsets + .iter() + .enumerate() + .map(|(pos, &o)| (o, pos)) + .collect() + }) + .collect(); + + let mut any_overlay = false; + let indices = routed_per_row + .into_iter() + .enumerate() + .map(|(offset_in_batch, routed)| match routed { + None => (0, offset_in_batch), + Some((k, offset_in_overlay)) => { + any_overlay = true; + (k + 1, fetch_positions[k][&offset_in_overlay]) + } + }) + .collect(); + + OverlayRouting { + indices, + offsets_in_overlay, + any_overlay, + } +} + +/// Build the merged column from `base` and the overlay values fetched for the +/// `offset_in_overlay` values [`route_overlays`] asked for. +/// +/// `fetched_newest_first[k]` holds overlay `k`'s values for `routing`'s +/// `offsets_in_overlay[k]`, in that order. The result has the same length and +/// type as `base`. A covered row whose overlay value is NULL resolves **to** NULL +/// (distinct from a fall-through, which keeps the base value). +fn assemble_overlay_column( + base: &ArrayRef, + routing: &OverlayRouting, + fetched_newest_first: &[ArrayRef], +) -> Result { + if !routing.any_overlay { + return Ok(base.clone()); + } + if fetched_newest_first.len() != routing.offsets_in_overlay.len() { + return Err(Error::invalid_input(format!( + "overlay assembly got {} value columns but routing expects {}", + fetched_newest_first.len(), + routing.offsets_in_overlay.len() + ))); + } + for (k, values) in fetched_newest_first.iter().enumerate() { + if values.len() != routing.offsets_in_overlay[k].len() { + return Err(Error::invalid_input(format!( + "overlay value column {} has {} values but {} were requested", + k, + values.len(), + routing.offsets_in_overlay[k].len() + ))); + } + } + + let mut sources: Vec<&dyn Array> = Vec::with_capacity(fetched_newest_first.len() + 1); + sources.push(base.as_ref()); + for values in fetched_newest_first { + sources.push(values.as_ref()); + } + interleave(&sources, &routing.indices).map_err(Error::from) +} + +/// One overlay's contribution to one projected atomic field, with its file reader opened. +#[derive(Debug, Clone)] +struct LoadedAtomicFieldOverlay { + /// The `offset_in_frag` cells this overlay covers for the atomic field. + coverage: Arc, + /// Reader over the overlay data file, projected to the covered atomic fields; shared + /// across the atomic fields that the same file covers. + reader: Arc, +} + +/// The overlays that apply to a single projected atomic field — a per-row field an overlay +/// can replace as a unit (a primitive leaf, or a whole list/map field; structs are +/// recursed through, not treated as atomic fields). Ordered newest-first, with readers opened +/// and pruned to a specific read. Produced by [`resolve_overlays`] and consumed by +/// [`merge_overlay_batch`]. +#[derive(Debug, Clone)] +pub struct LoadedAtomicField { + /// The top-level output column the atomic field lives in (its name locates the batch + /// column; its field tree drives the descend/splice into that column). + top_field: Arc, + /// Child field ids from `top_field` down to the atomic field (empty when the atomic + /// field *is* the top-level column). Drives descending to, and splicing back, the + /// atomic field. + ancestor_ids: Vec, + /// Projection of exactly the atomic field (its ancestor path pruned to the atomic + /// field subtree), used to fetch the atomic field's values from the overlay file. + fetch_projection: Arc, + overlays_newest_first: Vec, +} + +/// One overlay file that may contribute to a read, before it is opened. Opened +/// lazily by [`resolve_overlays`], and only if the read actually touches it. +#[derive(Debug, Clone)] +struct PlannedOverlayFile { + data_file: DataFile, + /// The covered ∩ projected atomic fields to project when the file is opened, so a single + /// reader serves every atomic field the file contributes to. + open_projection: Arc, +} + +/// One overlay's contribution to one projected atomic field, before the file is opened. +#[derive(Debug, Clone)] +struct PlannedAtomicFieldOverlay { + /// Index into [`OverlayReadPlanner::files`] of the file that supplies the value. + file: usize, + coverage: Arc, +} + +/// The overlays that apply to a single projected atomic field, ordered newest-first, before +/// any file is opened. +#[derive(Debug, Clone)] +struct PlannedAtomicField { + top_field: Arc, + ancestor_ids: Vec, + fetch_projection: Arc, + overlays_newest_first: Vec, +} + +/// A fragment's overlay-resolution plan for a projection, derived from coverage +/// metadata alone — no file opened, no IO. [`resolve_overlays`] turns it into opened +/// [`LoadedAtomicField`]s for one specific read, opening only the files whose cells +/// the read's rows actually touch. +#[derive(Debug, Clone)] +pub struct OverlayReadPlanner { + files: Vec, + atomic_fields: Vec, +} + +impl OverlayReadPlanner { + /// True when no projected atomic field has any overlay, so there is nothing to resolve. + pub fn is_empty(&self) -> bool { + self.atomic_fields.is_empty() + } +} + +/// Plan `fragment`'s overlay resolution for a projection from coverage metadata +/// alone. No files are opened here (see [`resolve_overlays`]) — this only reads the +/// already-parsed coverage bitmaps, so it is cheap enough to run on every open. +/// +/// Overlays are stored oldest-first (sorted newest-last on load, see +/// `sort_overlays_newest_last`), so walking them in reverse gives newest-first +/// precedence. +/// +/// Resolution is per *atomic field* — a per-row field that an overlay replaces as a unit: a +/// primitive leaf, or a whole list/map field. Structs are internal nodes, so each +/// leaf of a struct is its own atomic field and can be overlaid independently of its +/// siblings. An overlay is written against the leaf ids it stores (the V2_1 +/// structural encoding records only leaves), so an overlay contributes to a projected +/// atomic field when any id in its `data_file.fields` falls in that atomic field's leaf +/// set. At merge time the atomic field's value is fetched and spliced into its output +/// column, so an overlay on a sub-field never disturbs the column's other leaves. Each +/// contributing overlay *file* appears once in `files`, shared by every atomic field it +/// covers. +pub fn plan_overlays(fragment: &FileFragment, projection: &Schema) -> Result { + let overlays = &fragment.metadata.overlays; + debug_assert!( + overlays + .windows(2) + .all(|w| w[0].committed_version <= w[1].committed_version), + "overlays must be sorted newest-last (see sort_overlays_newest_last)" + ); + + // The projection's atomic fields, and a leaf-id -> atomic-field-index map so an + // overlay's stored leaf ids resolve to the atomic field they belong to in O(1). + struct AtomicFieldInfo<'a> { + top_field: &'a Field, + ancestor_ids: Vec, + atomic_field_id: i32, + } + let mut atomic_field_infos: Vec = Vec::new(); + let mut leaf_to_atomic_field: HashMap = HashMap::new(); + for top in &projection.fields { + for (atomic_field, ancestor_ids) in enumerate_atomic_fields(top) { + let idx = atomic_field_infos.len(); + let mut value_leaf_ids = Vec::new(); + collect_leaf_ids(atomic_field, &mut value_leaf_ids); + for leaf in value_leaf_ids { + leaf_to_atomic_field.insert(leaf, idx); + } + atomic_field_infos.push(AtomicFieldInfo { + top_field: top, + ancestor_ids, + atomic_field_id: atomic_field.id, + }); + } + } + + // Walk overlays newest-first. For each overlay, find the atomic fields it covers and push + // (newest-first, for free) into their per-atomic field overlay lists. + let mut files = Vec::new(); + let mut atomic_field_overlays: Vec> = + vec![Vec::new(); atomic_field_infos.len()]; + for overlay in overlays.iter().rev() { + // atomic field index -> the `data_file.fields` position whose coverage to read. An + // overlay writes one value per row per atomic field, so its leaves share a coverage; + // the first leaf of each atomic field to appear wins. + let mut covered: HashMap = HashMap::new(); + for (field_pos, &field_id) in overlay.data_file.fields.iter().enumerate() { + if let Some(&atomic_field_idx) = leaf_to_atomic_field.get(&field_id) { + covered.entry(atomic_field_idx).or_insert(field_pos); + } + } + if covered.is_empty() { + continue; + } + let file = files.len(); + let covered_ids: Vec = covered + .keys() + .map(|&i| atomic_field_infos[i].atomic_field_id) + .collect(); + files.push(PlannedOverlayFile { + data_file: overlay.data_file.clone(), + open_projection: Arc::new(projection.project_by_ids(&covered_ids, true)), + }); + for (atomic_field_idx, field_pos) in covered { + atomic_field_overlays[atomic_field_idx].push(PlannedAtomicFieldOverlay { + file, + coverage: overlay.coverage_for_field(field_pos)?, + }); + } + } + + // Emit one PlannedAtomicField per projected atomic field that has overlays, in + // atomic field order. + let mut atomic_fields = Vec::new(); + for (idx, info) in atomic_field_infos.iter().enumerate() { + let overlays_newest_first = std::mem::take(&mut atomic_field_overlays[idx]); + if overlays_newest_first.is_empty() { + continue; + } + atomic_fields.push(PlannedAtomicField { + top_field: Arc::new(info.top_field.clone()), + ancestor_ids: info.ancestor_ids.clone(), + fetch_projection: Arc::new(projection.project_by_ids(&[info.atomic_field_id], true)), + overlays_newest_first, + }); + } + Ok(OverlayReadPlanner { + files, + atomic_fields, + }) +} + +/// The per-row atomic fields of a projected top-level field, each with the child-id path from +/// the top-level field down to it. Structs are recursed through; a primitive leaf or a +/// whole list/map field is an atomic field (values are one-per-row). A top-level primitive or +/// list yields a single atomic field with an empty path. +fn enumerate_atomic_fields(top: &Field) -> Vec<(&Field, Vec)> { + fn recurse<'a>(field: &'a Field, path: &mut Vec, out: &mut Vec<(&'a Field, Vec)>) { + if field.logical_type.is_struct() { + for child in &field.children { + path.push(child.id); + recurse(child, path, out); + path.pop(); + } + } else { + out.push((field, path.clone())); + } + } + let mut out = Vec::new(); + let mut path = Vec::new(); + recurse(top, &mut path, &mut out); + out +} + +/// Collect the leaf field ids in `field`'s subtree — the ids an overlay stores for +/// this atomic field (its own id if primitive; its item leaves if a list/map). +fn collect_leaf_ids(field: &Field, out: &mut Vec) { + if field.children.is_empty() { + out.push(field.id); + } else { + for child in &field.children { + collect_leaf_ids(child, out); + } + } +} + +/// Follow a path of child field ids from `field` down through nested structs, taking +/// the corresponding child array at each step. Returns the array at the end of the +/// path (the whole `array` when `ancestor_ids` is empty). +fn descend_by_ids(array: &ArrayRef, field: &Field, ancestor_ids: &[i32]) -> Result { + let mut arr = array.clone(); + let mut fld = field; + for &id in ancestor_ids { + let child_pos = fld + .children + .iter() + .position(|c| c.id == id) + .ok_or_else(|| { + Error::invalid_input(format!( + "overlay descend: field id {id} not found under '{}'", + fld.name + )) + })?; + let structs = arr.as_any().downcast_ref::().ok_or_else(|| { + Error::invalid_input(format!( + "overlay descend: expected a struct at '{}'", + fld.name + )) + })?; + arr = structs.column(child_pos).clone(); + fld = &fld.children[child_pos]; + } + Ok(arr) +} + +/// Rebuild `array` with the array at `ancestor_ids` replaced by `new_atomic_field`, cloning +/// the struct spine along the path and preserving each struct's null buffer and other +/// children. With an empty path this is just `new_atomic_field` (whole-column replacement). +fn splice_by_ids( + array: &ArrayRef, + field: &Field, + ancestor_ids: &[i32], + new_atomic_field: ArrayRef, +) -> Result { + let Some((&id, rest)) = ancestor_ids.split_first() else { + return Ok(new_atomic_field); + }; + let child_pos = field + .children + .iter() + .position(|c| c.id == id) + .ok_or_else(|| { + Error::invalid_input(format!( + "overlay splice: field id {id} not found under '{}'", + field.name + )) + })?; + let structs = array + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input(format!( + "overlay splice: expected a struct at '{}'", + field.name + )) + })?; + let len = structs.len(); + let (fields, mut children, nulls) = structs.clone().into_parts(); + children[child_pos] = splice_by_ids( + &children[child_pos], + &field.children[child_pos], + rest, + new_atomic_field, + )?; + Ok(Arc::new(StructArray::try_new_with_length( + fields, children, nulls, len, + )?)) +} + +/// Open the overlay readers a specific read needs and return the per-field plans to +/// merge, pruned to that read. +/// +/// `offsets_in_frag` are the rows the read will return. An overlay whose coverage is +/// disjoint from those rows contributes nothing, so it is dropped and its file is +/// never opened — a `take` that misses an overlay's cells pays no IO for it. Each +/// surviving file is opened once, concurrently, projected to the covered fields; the +/// value bytes are still not read here (the per-batch [`merge_overlay_batch`] fetches +/// only the values it needs). +pub async fn resolve_overlays( + planner: &OverlayReadPlanner, + offsets_in_frag: &[u32], + fragment: &FileFragment, + read_config: &FragReadConfig, +) -> Result> { + let read_offsets = read_offsets_bitmap(offsets_in_frag); + + // A file is opened only if some atomic field it covers has cells among the requested rows. + // This is the row-selection pruning: overlays outside the read are skipped. + let mut file_needed = vec![false; planner.files.len()]; + for atomic_field in &planner.atomic_fields { + for overlay in &atomic_field.overlays_newest_first { + if !overlay.coverage.is_disjoint(&read_offsets) { + file_needed[overlay.file] = true; + } + } + } + + // Open each needed file once, concurrently. The reader is shared (via `Arc`) by + // every atomic field that file covers. + // + // These reads use priority 0 (highest): they are issued only when a ready + // consumer polls the batch task (see `merge_overlay_batch`), so we have already + // committed to reading this batch and the overlay reads cannot clog the + // backpressure queue ahead of work we are not ready for. (A future optimization + // could start the overlay fetches earlier to fill compute bubbles, which would + // want a priority tied to the base read.) + let opened: Vec>> = + futures::future::try_join_all(planner.files.iter().enumerate().map(|(i, file)| { + let needed = file_needed[i]; + async move { + if !needed { + return Ok::<_, Error>(None); + } + Ok(fragment + .open_reader(&file.data_file, Some(&file.open_projection), read_config) + .await? + .map(Arc::from)) + } + })) + .await?; + + let mut plans = Vec::new(); + for atomic_field in &planner.atomic_fields { + let mut overlays_newest_first = Vec::new(); + for overlay in &atomic_field.overlays_newest_first { + let Some(reader) = &opened[overlay.file] else { + continue; // pruned: coverage disjoint from the read + }; + overlays_newest_first.push(LoadedAtomicFieldOverlay { + coverage: overlay.coverage.clone(), + reader: reader.clone(), + }); + } + if !overlays_newest_first.is_empty() { + plans.push(LoadedAtomicField { + top_field: atomic_field.top_field.clone(), + ancestor_ids: atomic_field.ancestor_ids.clone(), + fetch_projection: atomic_field.fetch_projection.clone(), + overlays_newest_first, + }); + } + } + Ok(plans) +} + +/// The set of `offset_in_frag` a read will return, as a bitmap for cheap +/// intersection against overlay coverages. Contiguous scans build a single range; +/// arbitrary `take` offsets (small batches) are inserted individually. +fn read_offsets_bitmap(offsets_in_frag: &[u32]) -> RoaringBitmap { + let mut bitmap = RoaringBitmap::new(); + match contiguous_frag_start(offsets_in_frag) { + Some(start) => { + let end = (start as u64 + offsets_in_frag.len() as u64).min(u32::MAX as u64) as u32; + bitmap.insert_range(start..end); + } + None => bitmap.extend(offsets_in_frag.iter().copied()), + } + bitmap +} + +/// Resolve overlays for one base batch: route each projected atomic field against the batch's +/// `offsets_in_frag`, fetch only the overlay values the batch needs (concurrently with +/// the base read), assemble the merged atomic field, and splice it into its output column. +/// AtomicFields with no covered rows, and columns with no plan, pass through. +pub async fn merge_overlay_batch( + base: ReadBatchFut, + offsets_in_frag: &[u32], + plans: &[LoadedAtomicField], +) -> Result { + let atomic_field_work = futures::future::try_join_all(plans.iter().map(|plan| async move { + let coverages: Vec<&RoaringBitmap> = plan + .overlays_newest_first + .iter() + .map(|overlay| overlay.coverage.as_ref()) + .collect(); + let routing = route_overlays(offsets_in_frag, &coverages); + if !routing.any_overlay { + return Ok::<_, Error>((plan, None)); + } + // Fetch each overlay's values and descend to the atomic field array. The fetch is + // projected to the atomic field's ancestor path, so the fetched column is the pruned + // top-level column; `descend_by_ids` walks it down to the atomic field. + let atomic_field = &plan.fetch_projection.fields[0]; + let fetched = futures::future::try_join_all( + plan.overlays_newest_first + .iter() + .zip(&routing.offsets_in_overlay) + .map(|(overlay, offsets_in_overlay)| async move { + let column = fetch_overlay_values( + overlay.reader.as_ref(), + plan.fetch_projection.clone(), + offsets_in_overlay, + ) + .await?; + descend_by_ids(&column, atomic_field, &plan.ancestor_ids) + }), + ) + .await?; + Ok((plan, Some((routing, fetched)))) + })); + + // The base read and every overlay value read proceed concurrently. + let (batch, resolved) = futures::future::try_join(base, atomic_field_work).await?; + + let schema = batch.schema(); + let mut columns = batch.columns().to_vec(); + for (plan, work) in resolved { + let Some((routing, fetched)) = work else { + continue; + }; + let Some(idx) = schema.index_of(&plan.top_field.name).ok() else { + // The plan's column is not in this batch's projection; skip it. + continue; + }; + let base_atomic_field = descend_by_ids(&columns[idx], &plan.top_field, &plan.ancestor_ids)?; + let merged_atomic_field = assemble_overlay_column(&base_atomic_field, &routing, &fetched)?; + columns[idx] = splice_by_ids( + &columns[idx], + &plan.top_field, + &plan.ancestor_ids, + merged_atomic_field, + )?; + } + Ok(RecordBatch::try_new(schema, columns)?) +} + +/// Fetch one overlay's values at the given `offsets_in_overlay` (sorted, unique): +/// the corresponding entries of its value column, as the top-level column pruned to +/// `projection`. Returns `offsets_in_overlay.len()` rows in the same order; empty +/// input reads nothing and returns an empty column. +async fn fetch_overlay_values( + reader: &dyn GenericFileReader, + projection: Arc, + offsets_in_overlay: &[u32], +) -> Result { + if offsets_in_overlay.is_empty() { + return Ok(arrow_array::new_empty_array( + &projection.fields[0].data_type(), + )); + } + let mut tasks = reader + .take_all_tasks( + offsets_in_overlay, + offsets_in_overlay.len() as u32, + projection, + None, + ) + .await?; + let mut chunks: Vec = Vec::new(); + while let Some(task) = tasks.next().await { + let batch = task.task.await?; + chunks.push(batch.column(0).clone()); + } + let chunk_refs: Vec<&dyn arrow_array::Array> = chunks.iter().map(|a| a.as_ref()).collect(); + Ok(arrow_select::concat::concat(&chunk_refs)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow_array::{Int32Array, StringArray, UInt32Array}; + use std::sync::Arc; + + fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) + } + + fn bitmap(offsets: impl IntoIterator) -> RoaringBitmap { + RoaringBitmap::from_iter(offsets) + } + + /// Physical offsets for a contiguous range `[start, start + len)`. + fn offsets(start: u32, len: usize) -> Vec { + (start..start + len as u32).collect() + } + + /// Drive the production flow purely in memory: route against the coverage + /// bitmaps, then fetch just the requested `offset_in_overlay` entries from each + /// overlay's *full* value column (exactly what the value-pushdown `take` does on + /// disk), then assemble. `overlays_newest_first` holds each overlay's + /// `(coverage, full value column indexed by offset_in_overlay)`. + fn resolve( + base: &ArrayRef, + offsets: &[u32], + overlays_newest_first: &[(RoaringBitmap, ArrayRef)], + ) -> ArrayRef { + let coverages: Vec<&RoaringBitmap> = overlays_newest_first.iter().map(|(c, _)| c).collect(); + let routing = route_overlays(offsets, &coverages); + let fetched: Vec = overlays_newest_first + .iter() + .zip(&routing.offsets_in_overlay) + .map(|((_, full), offsets_in_overlay)| { + let indices = UInt32Array::from(offsets_in_overlay.clone()); + arrow_select::take::take(full.as_ref(), &indices, None).unwrap() + }) + .collect(); + assemble_overlay_column(base, &routing, &fetched).unwrap() + } + + fn assert_i32_eq(actual: &ArrayRef, expected: impl IntoIterator>) { + let actual = actual.as_any().downcast_ref::().unwrap(); + assert_eq!(actual, &Int32Array::from_iter(expected)); + } + + #[test] + fn test_no_overlays_returns_base() { + let base = i32_array([Some(1), Some(2), Some(3)]); + let resolved = resolve(&base, &offsets(0, 3), &[]); + assert_i32_eq(&resolved, [Some(1), Some(2), Some(3)]); + } + + #[test] + fn test_single_overlay_value_offset() { + // Base ages [30, 25, 40, 22]; overlay sets offset_in_frag 1 -> 26, whose + // value sits at offset_in_overlay 0. + let base = i32_array([Some(30), Some(25), Some(40), Some(22)]); + let overlay = (bitmap([1]), i32_array([Some(26)])); + let resolved = resolve(&base, &offsets(0, 4), &[overlay]); + assert_i32_eq(&resolved, [Some(30), Some(26), Some(40), Some(22)]); + } + + #[test] + fn test_value_offsets_multiple_cells() { + // Coverage {0, 2, 3} -> values at offset_in_overlay 0, 1, 2. + let base = i32_array([Some(10), Some(11), Some(12), Some(13)]); + let overlay = ( + bitmap([0, 2, 3]), + i32_array([Some(100), Some(120), Some(130)]), + ); + let resolved = resolve(&base, &offsets(0, 4), &[overlay]); + assert_i32_eq(&resolved, [Some(100), Some(11), Some(120), Some(130)]); + } + + #[test] + fn test_newest_overlay_wins() { + // Two overlays both cover offset_in_frag 1; the newest (first in the slice) + // wins. + let base = i32_array([Some(0), Some(1), Some(2)]); + let newest = (bitmap([1]), i32_array([Some(999)])); + let older = (bitmap([1, 2]), i32_array([Some(111), Some(222)])); + let resolved = resolve(&base, &offsets(0, 3), &[newest, older]); + // offset 1 -> newest (999); offset 2 -> only older covers it (222). + assert_i32_eq(&resolved, [Some(0), Some(999), Some(222)]); + } + + #[test] + fn test_null_override_vs_fall_through() { + // A covered offset with a NULL value overrides the cell to NULL; an + // absent offset falls through to the base. + let base = i32_array([Some(1), Some(2), Some(3)]); + let overlay = (bitmap([0]), i32_array([None])); + let resolved = resolve(&base, &offsets(0, 3), &[overlay]); + assert_i32_eq(&resolved, [None, Some(2), Some(3)]); + } + + #[test] + fn test_physical_start_offset() { + // The batch covers physical rows [10, 13); the overlay covers offset 11. + let base = i32_array([Some(0), Some(0), Some(0)]); + let overlay = (bitmap([11]), i32_array([Some(7)])); + let resolved = resolve(&base, &offsets(10, 3), &[overlay]); + assert_i32_eq(&resolved, [Some(0), Some(7), Some(0)]); + } + + #[test] + fn test_string_column_merge() { + let base: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + let overlay = ( + bitmap([0, 2]), + Arc::new(StringArray::from(vec!["A", "C"])) as ArrayRef, + ); + let resolved = resolve(&base, &offsets(0, 3), &[overlay]); + let expected: ArrayRef = Arc::new(StringArray::from(vec!["A", "b", "C"])); + assert_eq!(&resolved, &expected); + } + + #[test] + fn test_non_contiguous_offsets() { + // `take` supplies arbitrary, non-contiguous offsets_in_frag. The base rows + // correspond to offsets 5, 1, 8 (in that order); the overlay covers offsets + // {1, 8}, whose values sit at offset_in_overlay 0, 1. + let base = i32_array([Some(50), Some(10), Some(80)]); + let overlay = (bitmap([1, 8]), i32_array([Some(11), Some(88)])); + let resolved = resolve(&base, &[5, 1, 8], &[overlay]); + // offset 5 uncovered -> base 50; offset 1 -> offset_in_overlay 0 (11); + // offset 8 -> offset_in_overlay 1 (88). + assert_i32_eq(&resolved, [Some(50), Some(11), Some(88)]); + } + + #[test] + fn test_routing_dedups_repeated_offsets() { + // A `take` may request the same offset twice; both rows must route to the + // same overlay value, and that value is fetched only once. + let coverage = bitmap([2, 5]); + let routing = route_overlays(&[5, 2, 5], &[&coverage]); + // offset_in_frag 5 is offset_in_overlay 1, offset_in_frag 2 is + // offset_in_overlay 0: distinct values {0, 1}, sorted. + assert_eq!(routing.offsets_in_overlay, vec![vec![0, 1]]); + let full = i32_array([Some(20), Some(50)]); // values at offset_in_overlay 0, 1 + let fetched = vec![ + arrow_select::take::take( + full.as_ref(), + &UInt32Array::from(routing.offsets_in_overlay[0].clone()), + None, + ) + .unwrap(), + ]; + let base = i32_array([Some(0), Some(0), Some(0)]); + let resolved = assemble_overlay_column(&base, &routing, &fetched).unwrap(); + assert_i32_eq(&resolved, [Some(50), Some(20), Some(50)]); + } + + #[test] + fn test_assemble_value_count_mismatch_errors() { + let coverage = bitmap([0, 1]); + let routing = route_overlays(&[0, 1], &[&coverage]); + let base = i32_array([Some(1), Some(2)]); + // One value supplied for two requested offsets is a caller bug. + let fetched = vec![i32_array([Some(9)])]; + assert!(assemble_overlay_column(&base, &routing, &fetched).is_err()); + } + + #[test] + fn test_contiguous_fast_path_matches_general() { + // The contiguous fast path must produce byte-for-byte identical routing to + // the general offset-major path for any contiguous batch. Fuzz a range of + // fragment starts, lengths, overlay counts, and coverage densities — + // including bits outside the batch range — and compare both paths. + let mut state = 0x9e3779b97f4a7c15u64; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (state >> 33) as u32 + }; + for _ in 0..500 { + let frag_start = next() % 64; + let len = (next() % 48 + 1) as usize; + let num_overlays = (next() % 5) as usize; + let coverages: Vec = (0..num_overlays) + .map(|_| { + let density = next() % 101; + let mut b = RoaringBitmap::new(); + for off in frag_start.saturating_sub(3)..frag_start + len as u32 + 3 { + if next() % 100 < density { + b.insert(off); + } + } + b + }) + .collect(); + let refs: Vec<&RoaringBitmap> = coverages.iter().collect(); + let contiguous_offsets: Vec = (frag_start..frag_start + len as u32).collect(); + + let fast = route_contiguous(frag_start, len, &refs); + let general = route_arbitrary(&contiguous_offsets, &refs); + assert_eq!(fast.indices, general.indices, "indices differ"); + assert_eq!( + fast.offsets_in_overlay, general.offsets_in_overlay, + "offsets_in_overlay differ" + ); + assert_eq!(fast.any_overlay, general.any_overlay, "any_overlay differs"); + } + } + + /// `outer { middle { a, b } }` for exercising the descend/splice helpers. + fn nested_struct() -> (Schema, ArrayRef) { + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + let mid = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let outer_fields = + Fields::from(vec![ArrowField::new("middle", DataType::Struct(mid), true)]); + let arrow_schema = ArrowSchema::new(vec![ArrowField::new( + "outer", + DataType::Struct(outer_fields), + true, + )]); + let mut schema = Schema::try_from(&arrow_schema).unwrap(); + schema.set_field_id(None); + let middle = StructArray::from(vec![ + ( + Arc::new(ArrowField::new("a", DataType::Int32, true)), + i32_array([Some(1), Some(2), Some(3)]), + ), + ( + Arc::new(ArrowField::new("b", DataType::Int32, true)), + i32_array([Some(10), Some(20), Some(30)]), + ), + ]); + let outer: ArrayRef = Arc::new(StructArray::from(vec![( + Arc::new(ArrowField::new("middle", middle.data_type().clone(), true)), + Arc::new(middle) as ArrayRef, + )])); + (schema, outer) + } + + #[test] + fn test_descend_and_splice_roundtrip() { + let (schema, outer_arr) = nested_struct(); + let outer_field = &schema.fields[0]; + let middle_id = outer_field.children[0].id; + let a_id = outer_field.children[0].children[0].id; + let path = [middle_id, a_id]; + + // Descend to the deep leaf `outer.middle.a`. + let a = descend_by_ids(&outer_arr, outer_field, &path).unwrap(); + assert_i32_eq(&a, [Some(1), Some(2), Some(3)]); + + // Splice a replacement in; only `a` changes, `b` is preserved. + let spliced = splice_by_ids( + &outer_arr, + outer_field, + &path, + i32_array([Some(7), Some(8), Some(9)]), + ) + .unwrap(); + let middle = spliced + .as_any() + .downcast_ref::() + .unwrap() + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .clone(); + assert_i32_eq(&middle.column(0).clone(), [Some(7), Some(8), Some(9)]); + assert_i32_eq(&middle.column(1).clone(), [Some(10), Some(20), Some(30)]); + } + + #[test] + fn test_splice_preserves_struct_nulls() { + use arrow_buffer::NullBuffer; + let (schema, base) = nested_struct(); + let outer_field = &schema.fields[0]; + // Rebuild `outer` with a null at row 1 (a null struct value). + let base = base.as_any().downcast_ref::().unwrap(); + let (fields, children, _) = base.clone().into_parts(); + let outer_arr: ArrayRef = Arc::new( + StructArray::try_new( + fields, + children, + Some(NullBuffer::from(vec![true, false, true])), + ) + .unwrap(), + ); + let path = [ + outer_field.children[0].id, + outer_field.children[0].children[0].id, + ]; + let spliced = splice_by_ids( + &outer_arr, + outer_field, + &path, + i32_array([Some(7), Some(8), Some(9)]), + ) + .unwrap(); + let spliced = spliced.as_any().downcast_ref::().unwrap(); + // The outer struct's null buffer survives the splice. + assert!(!spliced.is_null(0)); + assert!(spliced.is_null(1)); + assert!(!spliced.is_null(2)); + } +} diff --git a/rust/lance/src/dataset/refs.rs b/rust/lance/src/dataset/refs.rs index 98b4f0cbc0a..79380a0799c 100644 --- a/rust/lance/src/dataset/refs.rs +++ b/rust/lance/src/dataset/refs.rs @@ -8,7 +8,7 @@ use futures::stream::{StreamExt, TryStreamExt}; use itertools::Itertools; use lance_io::object_store::ObjectStore; use lance_table::io::commit::CommitHandler; -use object_store::path::Path; +use object_store::{Error as ObjectStoreError, path::Path}; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -21,7 +21,6 @@ use std::cmp::Ordering; use std::collections::HashMap; use std::fmt; use std::fmt::Formatter; -use std::io::ErrorKind; use uuid::Uuid; pub const MAIN_BRANCH: &str = "main"; @@ -145,6 +144,25 @@ impl Branches<'_> { } } +async fn put_ref_if_absent( + object_store: &ObjectStore, + path: &Path, + contents: Vec, + conflict_message: String, +) -> Result<()> { + object_store + .put_if_absent(path, contents.into()) + .await + .map_err(|error| match error { + ObjectStoreError::AlreadyExists { .. } | ObjectStoreError::Precondition { .. } => { + Error::RefConflict { + message: conflict_message, + } + } + error => error.into(), + }) +} + impl Tags<'_> { pub async fn fetch_tags(&self) -> Result> { let root_location = self.refs.root()?; @@ -218,23 +236,18 @@ impl Tags<'_> { let root_location = self.refs.root()?; let tag_file = tag_path(&root_location.path, tag); - if self.object_store().exists(&tag_file).await? { - return Err(Error::RefConflict { - message: format!("tag {} already exists", tag), - }); - } let now = utc_now(); let tag_contents = self .build_tag_content_by_ref(reference, Some(now), Some(now)) .await?; - self.object_store() - .put( - &tag_file, - serde_json::to_string_pretty(&tag_contents)?.as_bytes(), - ) - .await - .map(|_| ()) + put_ref_if_absent( + self.object_store(), + &tag_file, + serde_json::to_vec_pretty(&tag_contents)?, + format!("tag {} already exists", tag), + ) + .await } pub async fn delete(&self, tag: &str) -> Result<()> { @@ -453,11 +466,6 @@ impl Branches<'_> { let source_branch = source_branch.and_then(standardize_branch); let root_location = self.refs.root()?; let branch_file = branch_contents_path(&root_location.path, branch_name); - if self.object_store().exists(&branch_file).await? { - return Err(Error::RefConflict { - message: format!("branch {} already exists", branch_name), - }); - } let branch_location = self .refs @@ -476,7 +484,7 @@ impl Branches<'_> { if !self.object_store().exists(&manifest_file.path).await? { return Err(Error::VersionNotFound { - message: format!("Manifest file {} does not exist", &manifest_file.path), + message: format!("Manifest file {} does not exist", manifest_file.path), }); }; @@ -508,13 +516,13 @@ impl Branches<'_> { metadata: HashMap::new(), }; - self.object_store() - .put( - &branch_file, - serde_json::to_string_pretty(&branch_contents)?.as_bytes(), - ) - .await - .map(|_| ()) + put_ref_if_absent( + self.object_store(), + &branch_file, + serde_json::to_vec_pretty(&branch_contents)?, + format!("branch {} already exists", branch_name), + ) + .await } pub async fn replace_metadata( @@ -574,6 +582,28 @@ impl Branches<'_> { log::warn!("BranchContents of {} does not exist", branch); } + // Tags identify snapshots by (branch, version). Deleting a branch removes its entire version chain, + // so any tag whose branch matches the deletion target blocks the operation, regardless of the tagged + // version. + let referenced_tags = self + .refs + .tags() + .fetch_tags() + .await? + .into_iter() + .filter_map(|(name, contents)| { + (contents.branch.as_deref() == Some(branch)).then_some((name, contents.version)) + }) + .collect_vec(); + if !referenced_tags.is_empty() { + return Err(Error::RefConflict { + message: format!( + "Branch {} is referenced by tags {:?}, can not delete", + branch, referenced_tags + ), + }); + } + let root_location = self.refs.root()?; let branch_file = branch_contents_path(&root_location.path, branch); if self.object_store().exists(&branch_file).await? { @@ -611,16 +641,8 @@ impl Branches<'_> { && let Err(e) = self.refs.object_store.remove_dir_all(delete_path).await { match &e { - Error::IO { source, .. } => { - if let Some(io_err) = source.downcast_ref::() { - if io_err.kind() == ErrorKind::NotFound { - log::debug!("Branch directory already deleted: {}", io_err); - } else { - return Err(e); - } - } else { - return Err(e); - } + Error::NotFound { .. } => { + log::debug!("Branch directory already deleted"); } _ => return Err(e), } diff --git a/rust/lance/src/dataset/rowids.rs b/rust/lance/src/dataset/rowids.rs index c57d75386db..b7f68751fe8 100644 --- a/rust/lance/src/dataset/rowids.rs +++ b/rust/lance/src/dataset/rowids.rs @@ -1,40 +1,46 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +mod validate; + use super::Dataset; use crate::session::caches::{RowIdIndexKey, RowIdSequenceKey}; use crate::{Error, Result}; use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; -use lance_core::utils::deletion::DeletionVector; +use lance_core::utils::{address::RowAddress, deletion::DeletionVector}; +use lance_select::{RowAddrSelection, RowAddrTreeMap}; use lance_table::{ format::{Fragment, RowIdMeta}, rowids::{FragmentRowIdIndex, RowIdIndex, RowIdSequence, read_row_ids}, }; use std::sync::Arc; +pub(super) use validate::validate_stable_row_ids; + /// Load a row id sequence from the given dataset and fragment. pub async fn load_row_id_sequence( dataset: &Dataset, fragment: &Fragment, ) -> Result> { - // Virtual path to prevent collisions in the cache. match &fragment.row_id_meta { None => Err(Error::internal("Missing row id meta")), - Some(RowIdMeta::Inline(data)) => { + Some(row_id_meta @ RowIdMeta::Inline(data)) => { let data = data.clone(); let key = RowIdSequenceKey { fragment_id: fragment.id, + row_id_meta, }; dataset .metadata_cache .get_or_insert_with_key(key, || async move { read_row_ids(&data) }) .await } - Some(RowIdMeta::External(file_slice)) => { + Some(row_id_meta @ RowIdMeta::External(file_slice)) => { let file_slice = file_slice.clone(); let dataset_clone = dataset.clone(); let key = RowIdSequenceKey { fragment_id: fragment.id, + row_id_meta, }; dataset .metadata_cache @@ -87,6 +93,146 @@ pub async fn get_row_id_index( } } +/// Map a set of physical row addresses to their stable row ids +/// +/// A fragment's [`RowIdSequence`] holds one id per physical row in offset order; +/// deletions are tracked by the deletion vector and do not compact the sequence, +/// so a physical offset indexes the sequence directly (see [`RowIdIndex`], which +/// maps ids to `start_address + position` and filters deletions separately). +/// Addresses that no longer have a live counterpart are dropped, which is correct: +/// those rows are not part of the answer. That covers both a deleted row and a +/// whole fragment that a maintenance operation replaced — an address-domain index +/// is not remapped by compaction or `update` (neither the zone map nor the bloom +/// filter index supports remap), so its results outlive the fragments they address. +/// The replacement fragments are absent from the index's fragment bitmap, so the +/// scanner routes them to a full scan and no match is lost. +pub(crate) async fn translate_addr_treemap_to_row_ids( + dataset: &Dataset, + addrs: &RowAddrTreeMap, +) -> Result { + let mut row_ids = RowAddrTreeMap::new(); + for (fragment_id, selection) in addrs.iter() { + let Some(file_fragment) = dataset.get_fragment(*fragment_id as usize) else { + continue; + }; + let sequence = load_row_id_sequence(dataset, file_fragment.metadata()).await?; + + match selection { + RowAddrSelection::Full => { + // The whole fragment is selected: every live row's id qualifies. + row_ids |= RowAddrTreeMap::from(sequence.as_ref()); + } + RowAddrSelection::Partial(offsets) => { + let deletion_vector = file_fragment.get_deletion_vector().await?; + for physical_offset in offsets.iter() { + // A deletion does not compact the row id sequence; the deleted row keeps + // its slot (the deletion vector tracks it separately). So a physical offset + // is a direct index into the sequence, regardless of any deletions below it. + // A stale offset that points at a deleted row has no live counterpart and is + // not part of any answer, so it contributes no id to the block/take set. + let deleted = deletion_vector + .as_ref() + .is_some_and(|dv| dv.contains(physical_offset)); + if deleted { + continue; + } + // A selected offset with no sequence entry points past the fragment's rows. + // Silently dropping it would let a stale index result escape masking, so + // treat the mismatch as corruption. + let id = sequence.get(physical_offset as usize).ok_or_else(|| { + Error::internal(format!( + "fragment_id={fragment_id} row-id sequence has no entry at \ + physical_offset={physical_offset} (sequence len={})", + sequence.len() + )) + })?; + row_ids.insert(id); + } + } + } + } + Ok(row_ids) +} + +/// Resolve row addresses to row ids, positionally: `addr = (fragment << 32) | offset` +/// looks up `offset` in the fragment's [`RowIdSequence`]. The inverse companion of +/// [`get_row_id_index`]. +/// +/// Returns one entry per input address, in order. Addresses that no longer +/// resolve — a missing fragment, an out-of-range offset, or a `None` input — +/// yield `None`. On datasets without stable row ids, addresses are the row +/// ids, so the input is returned unchanged. +pub async fn row_addrs_to_row_ids( + dataset: &Dataset, + addrs: impl IntoIterator>, +) -> Result>> { + row_addrs_to_row_ids_impl(dataset, addrs, DeletedRowBehavior::Include).await +} + +/// Resolve physical row addresses to stable row ids only for live physical slots. +/// +/// When stable row ids are enabled, missing fragments, out-of-range offsets, +/// deleted physical slots, and `None` inputs yield `None`. This prevents a +/// deleted slot's stable id from selecting a replacement row written by an +/// update. Without stable row ids, the input is returned unchanged. +pub(crate) async fn live_row_addrs_to_row_ids( + dataset: &Dataset, + addrs: impl IntoIterator>, +) -> Result>> { + row_addrs_to_row_ids_impl(dataset, addrs, DeletedRowBehavior::Exclude).await +} + +#[derive(Clone, Copy)] +enum DeletedRowBehavior { + Include, + Exclude, +} + +async fn row_addrs_to_row_ids_impl( + dataset: &Dataset, + addrs: impl IntoIterator>, + deleted_row_behavior: DeletedRowBehavior, +) -> Result>> { + let addrs: Vec> = addrs.into_iter().collect(); + if !dataset.manifest.uses_stable_row_ids() { + return Ok(addrs); + } + + let mut positions_by_fragment: std::collections::HashMap> = + std::collections::HashMap::new(); + for (position, addr) in addrs.iter().enumerate() { + if let Some(addr) = addr { + positions_by_fragment + .entry(RowAddress::from(*addr).fragment_id()) + .or_default() + .push(position); + } + } + + let mut ids: Vec> = vec![None; addrs.len()]; + for (fragment_id, positions) in positions_by_fragment { + let Some(fragment) = dataset.get_fragment(fragment_id as usize) else { + continue; + }; + let sequence = load_row_id_sequence(dataset, fragment.metadata()).await?; + let deletion_vector = match deleted_row_behavior { + DeletedRowBehavior::Include => None, + DeletedRowBehavior::Exclude => fragment.get_deletion_vector().await?, + }; + for position in positions { + let offset = RowAddress::from(addrs[position].expect("grouped from Some")).row_offset(); + if deletion_vector + .as_ref() + .is_some_and(|deletions| deletions.contains(offset)) + { + continue; + } + ids[position] = sequence.get(offset as usize); + } + } + Ok(ids) +} + async fn load_row_id_index(dataset: &Dataset) -> Result { let sequences = load_row_id_sequences(dataset, &dataset.manifest.fragments) .try_collect::>() @@ -105,10 +251,14 @@ async fn load_row_id_index(dataset: &Dataset) -> Result dv, - Ok(None) | Err(_) => Arc::new(DeletionVector::default()), - } + fragment_clone + .get_deletion_vector() + .await? + .ok_or_else(|| { + Error::internal(format!( + "fragment_id={fragment_id} has deletion-file metadata but no deletion vector" + )) + })? } else { Arc::new(DeletionVector::default()) }; @@ -133,13 +283,16 @@ async fn load_row_id_index(dataset: &Dataset) -> Result>(); let expected_addresses = (0..num_rows) .map(|i| { @@ -243,6 +397,104 @@ mod test { assert_eq!(dataset.manifest().next_row_id, num_rows); } + #[tokio::test] + async fn test_row_id_index_propagates_deletion_vector_read_error() { + let batch = sequence_batch(0..10); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let test_uri = temp_dir.as_str(); + let path_prefix = if test_uri.starts_with('/') { "" } else { "/" }; + let routed_uri = format!("file-object-store://{path_prefix}{test_uri}"); + let mut dataset = Dataset::write( + reader, + &routed_uri, + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.delete("id = 3").await.unwrap(); + drop(dataset); + + let failing_store = Arc::new(FailingProxyStore::new()); + failing_store.fail_when( + "get_opts", + "_deletions", + "injected deletion-vector read failure", + ); + let dataset = DatasetBuilder::from_uri(&routed_uri) + .with_read_params(ReadParams { + store_options: Some(ObjectStoreParams { + object_store_wrapper: Some(failing_store.clone()), + ..Default::default() + }), + ..Default::default() + }) + .load() + .await + .unwrap(); + + let error = get_row_id_index(&dataset).await.unwrap_err(); + assert!(matches!(&error, Error::IO { .. })); + assert!( + error + .to_string() + .contains("injected deletion-vector read failure") + ); + + failing_store.clear_fail_when("get_opts", "_deletions"); + let index = get_row_id_index(&dataset).await.unwrap().unwrap(); + assert!(index.get(2).unwrap().is_some()); + assert!(index.get(3).unwrap().is_none()); + } + + #[tokio::test] + async fn test_row_addrs_to_row_ids() { + let num_rows = 25u64; + let batch = sequence_batch(0..num_rows as i32); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let write_params = WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 10, + ..Default::default() + }; + let dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Sequential assignment: row n lives at (n / 10, n % 10) with id n + let addr = |frag: u64, offset: u64| Some((frag << 32) | offset); + let addrs = vec![ + addr(2, 1), // id 21 + addr(0, 3), // id 3 + None, // null stays null + addr(0, 3), // duplicates allowed + addr(9, 0), // missing fragment + addr(1, 100), // out-of-range offset + ]; + let ids = row_addrs_to_row_ids(&dataset, addrs).await.unwrap(); + assert_eq!(ids, vec![Some(21), Some(3), None, Some(3), None, None]); + + // Without stable row ids, addresses are the ids: input passes through + let batch = sequence_batch(0..num_rows as i32); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let plain = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: 10, + ..Default::default() + }), + ) + .await + .unwrap(); + let addrs = vec![addr(1, 2), None]; + let ids = row_addrs_to_row_ids(&plain, addrs.clone()).await.unwrap(); + assert_eq!(ids, addrs); + } + #[tokio::test] async fn test_row_ids_overwrite() { // Validate we don't re-use after overwriting @@ -273,10 +525,128 @@ mod test { // Overwriting should NOT reset the row id counter. assert_eq!(dataset.manifest().next_row_id, 2 * num_rows); + // Nor the fragment id counter: ids are a high water mark, so the + // overwritten fragment cannot alias the one it replaced. + assert_eq!(dataset.manifest.fragments[0].id, 1); let index = get_row_id_index(&dataset).await.unwrap().unwrap(); - assert!(index.get(0).is_none()); - assert!(index.get(num_rows).is_some()); + assert!(index.get(0).unwrap().is_none()); + assert!(index.get(num_rows).unwrap().is_some()); + } + + /// Fragment ids are a high water mark within one dataset, but a dataset + /// dropped and recreated at the same URI restarts them at 0 while sharing the + /// cache namespace of its predecessor (see #7645). The two generations must + /// still be told apart. + #[tokio::test] + async fn test_row_ids_recreate_at_same_uri() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let tmp_path = &temp_dir; + // Shared so the cache stays warm across the drop, as it would for a host + // that keeps one session open for the lifetime of the process. + let session = Arc::new(Session::default()); + let write = |rows: Range| { + let batch = sequence_batch(rows); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let params = WriteParams { + enable_stable_row_ids: true, + session: Some(session.clone()), + ..Default::default() + }; + async move { + Dataset::write(reader, tmp_path, Some(params)) + .await + .unwrap() + } + }; + + let dataset = write(0..100).await; + let sequence = load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + assert_eq!(sequence.len(), 100); + + // Reloading the unchanged fragment must still hit: keying on contents has + // to leave the sequence cacheable, not just make it distinguishable. + let hits_before = session.metadata_cache_stats().await.hits; + load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + assert_eq!(session.metadata_cache_stats().await.hits, hits_before + 1); + + drop(dataset); + std::fs::remove_dir_all(tmp_path.as_str()).unwrap(); + + // Shorter than the dataset it replaces, so a stale hit is observable: an + // equal-length sequence would be byte-identical and harmless. + let dataset = write(0..60).await; + assert_eq!(dataset.manifest.fragments[0].id, 0); + + let sequence = load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + assert_eq!( + sequence.iter().collect::>(), + (0..60).collect::>() + ); + } + + #[tokio::test] + async fn test_compaction_after_recreate() { + // Compaction rechunks the row id sequences of the fragments it merges, so + // a sequence cached for a dropped dataset does not just misreport ids, it + // writes ids the fragments do not hold back into the manifest. + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let tmp_path = &temp_dir; + let session = Arc::new(Session::default()); + let params = WriteParams { + enable_stable_row_ids: true, + session: Some(session.clone()), + ..Default::default() + }; + + let write = |rows: Range, mode: WriteMode| { + let batch = sequence_batch(rows); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let params = WriteParams { + mode, + ..params.clone() + }; + async move { + Dataset::write(reader, tmp_path, Some(params)) + .await + .unwrap() + } + }; + + let dataset = write(0..100, WriteMode::Create).await; + load_row_id_sequence(&dataset, &dataset.manifest.fragments[0]) + .await + .unwrap(); + + drop(dataset); + std::fs::remove_dir_all(tmp_path.as_str()).unwrap(); + + // The recreated dataset holds 60 rows in fragment 0, not the 100 cached + // above, and a second fragment so compaction has something to merge. + write(0..60, WriteMode::Create).await; + let mut dataset = write(0..100, WriteMode::Append).await; + + compact(&mut dataset, 1024).await; + + assert_eq!(dataset.count_rows(None).await.unwrap(), 160); + let mut scan = dataset.scan(); + scan.with_row_id(); + let batch = scan.try_into_batch().await.unwrap(); + let row_ids = batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + // A stale 100-long sequence would rechunk 0..100 onto the 60 rows of + // fragment 0 and shift everything after it. + assert_eq!(row_ids, (0..160).collect::>()); } #[tokio::test] @@ -313,8 +683,8 @@ mod test { assert_eq!(dataset.manifest().next_row_id, 60); let index = get_row_id_index(&dataset).await.unwrap().unwrap(); - assert!(index.get(0).is_some()); - assert!(index.get(60).is_none()); + assert!(index.get(0).unwrap().is_some()); + assert!(index.get(60).unwrap().is_none()); } #[tokio::test] @@ -460,11 +830,61 @@ mod test { let dataset = update_result.new_dataset; let index = get_row_id_index(&dataset).await.unwrap().unwrap(); - assert!(index.get(0).is_some()); + assert!(index.get(0).unwrap().is_some()); // the updated row ids mapping to new address - assert_eq!(index.get(3), Some(RowAddress::new_from_parts(1, 0))); + assert_eq!( + index.get(3).unwrap(), + Some(RowAddress::new_from_parts(1, 0)) + ); // there is no new row id - assert_eq!(index.get(5), None); + assert_eq!(index.get(5).unwrap(), None); + } + + /// 100 sequential rows across 4 fragments with every third row deleted. + async fn deleted_thirds_dataset(enable_stable_row_ids: bool) -> Dataset { + let batch = sequence_batch(0..100); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let write_params = WriteParams { + enable_stable_row_ids, + max_rows_per_file: 25, + ..Default::default() + }; + let mut dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + dataset.delete("id % 3 = 0").await.unwrap(); + dataset + } + + #[rstest::rstest] + #[case::interleaved((0..100).collect(), (0..100).filter(|id| id % 3 != 0).collect())] + #[case::all_deleted(vec![0, 3, 9], vec![])] + #[case::all_live(vec![1, 2, 50, 98], vec![1, 2, 50, 98])] + #[tokio::test] + async fn test_filter_deleted_ids_with_stable_row_ids( + #[case] ids: Vec, + #[case] expected: Vec, + ) { + // Regression test for https://github.com/lance-format/lance/issues/7701: + // filter_deleted_ids must return exactly the live ids, in order. + // Sequential inserts, so stable row id == id column value. + let dataset = deleted_thirds_dataset(true).await; + assert_eq!(dataset.filter_deleted_ids(&ids).await.unwrap(), expected); + } + + #[tokio::test] + async fn test_filter_deleted_ids_without_stable_row_ids() { + // Non-stable: ids are row addresses; only deleted rows drop out. + let dataset = deleted_thirds_dataset(false).await; + + // Row addresses: fragment (id / 25) << 32 | offset (id % 25). + let addr = |id: u64| (id / 25) << 32 | (id % 25); + let ids = (0..100).map(addr).collect::>(); + let expected = (0..100) + .filter(|id| id % 3 != 0) + .map(addr) + .collect::>(); + assert_eq!(dataset.filter_deleted_ids(&ids).await.unwrap(), expected); } fn build_rowid_to_i_map(row_ids: &UInt64Array, i_array: &Int32Array) -> HashMap { @@ -489,7 +909,7 @@ mod test { build_rowid_to_i_map(row_ids, i) } - async fn compact(dataset: &mut Dataset, target_rows: usize) { + pub(super) async fn compact(dataset: &mut Dataset, target_rows: usize) { let options = CompactionOptions { target_rows_per_fragment: target_rows, ..Default::default() @@ -497,7 +917,7 @@ mod test { let _ = compact_files(dataset, options, None).await.unwrap(); } - async fn delete(dataset: &mut Dataset, expr: &str) { + pub(super) async fn delete(dataset: &mut Dataset, expr: &str) { dataset.delete(expr).await.unwrap(); } diff --git a/rust/lance/src/dataset/rowids/validate.rs b/rust/lance/src/dataset/rowids/validate.rs new file mode 100644 index 00000000000..27bed6491ac --- /dev/null +++ b/rust/lance/src/dataset/rowids/validate.rs @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Integrity checks for the stable row id invariants that the row id index and the write +//! paths rely on. Reached through [`Dataset::validate`]. + +use super::load_row_id_sequence; +use crate::dataset::Dataset; +use crate::dataset::fragment::FileFragment; +use crate::{Error, Result}; +use futures::{StreamExt, TryStreamExt}; +use lance_core::utils::deletion::DeletionVector; +use lance_table::format::RowDatasetVersionMeta; +use lance_table::rowids::RowIdSequence; +use roaring::RoaringTreemap; +use std::sync::Arc; + +/// The per-fragment state a stable row id invariant is checked against. +type FragmentRowIds = ( + FileFragment, + Arc, + Option>, +); + +/// Check the invariants that the stable row id machinery relies on. +/// +/// A no-op on datasets that don't use stable row ids. See [`Dataset::validate`]. +pub async fn validate_stable_row_ids(dataset: &Dataset) -> Result<()> { + if !dataset.manifest.uses_stable_row_ids() { + return Ok(()); + } + + let corrupt = |message: String| Error::corrupt_file(dataset.base.clone(), message); + + for fragment in dataset.manifest.fragments.iter() { + if fragment.row_id_meta.is_none() { + return Err(corrupt(format!( + "Fragment {} has no row id metadata, but dataset {:?} uses stable row ids", + fragment.id, dataset.base + ))); + } + } + + // `buffered` preserves manifest order, which the uniqueness check below relies on to + // report the earlier of the two fragments holding a duplicate id. + let fragments: Vec = futures::stream::iter(dataset.get_fragments()) + .map(|fragment| async move { + let sequence = load_row_id_sequence(dataset, fragment.metadata()).await?; + let deletion_vector = fragment.get_deletion_vector().await?; + Result::Ok((fragment, sequence, deletion_vector)) + }) + .buffered(dataset.object_store.io_parallelism()) + .try_collect() + .await?; + + let mut live_row_ids = RoaringTreemap::new(); + let mut max_row_id: Option = None; + + for (index, (fragment, sequence, deletion_vector)) in fragments.iter().enumerate() { + let metadata = fragment.metadata(); + let physical_rows = metadata.physical_rows.ok_or_else(|| { + corrupt(format!( + "Fragment {} has an unknown physical row count, but dataset {:?} uses stable row ids", + metadata.id, dataset.base + )) + })? as u64; + + if sequence.len() != physical_rows { + return Err(corrupt(format!( + "Fragment {} has {} row ids, but {} physical rows, in dataset {:?}", + metadata.id, + sequence.len(), + physical_rows, + dataset.base + ))); + } + + for (name, meta) in [ + ("created_at", &metadata.created_at_version_meta), + ("last_updated_at", &metadata.last_updated_at_version_meta), + ] { + // Only inline version metadata can be read back; nothing writes the + // external form yet. + if let Some(meta @ RowDatasetVersionMeta::Inline(_)) = meta { + let versions = meta.load_sequence()?.len(); + if versions != physical_rows { + return Err(corrupt(format!( + "Fragment {} has {} {} versions, but {} physical rows, in dataset {:?}", + metadata.id, versions, name, physical_rows, dataset.base + ))); + } + } + } + + // The bounding range covers tombstoned slots too: their ids are retired and must + // not be handed out again either. + if let Some(range) = sequence.row_id_range() { + max_row_id = max_row_id.max(Some(*range.end())); + } + + for (offset, row_id) in sequence.iter().enumerate() { + // An update rewrites a row into a new fragment under the same id and tombstones + // the original slot, so only live ids are required to be unique. + if deletion_vector + .as_ref() + .is_some_and(|deletions| deletions.contains(offset as u32)) + { + continue; + } + if !live_row_ids.insert(row_id) { + return Err(corrupt(format!( + "Row id {} is live in both {} and fragment {} at offset {} in dataset {:?}", + row_id, + describe_first_live_slot(&fragments[..=index], row_id), + metadata.id, + offset, + dataset.base + ))); + } + } + } + + if let Some(max_row_id) = max_row_id + && dataset.manifest.next_row_id <= max_row_id + { + return Err(corrupt(format!( + "Dataset {:?} will hand out row id {} next, but row id {} is already in use", + dataset.base, dataset.manifest.next_row_id, max_row_id + ))); + } + + Ok(()) +} + +/// Describe where `row_id` is first live, so a uniqueness failure can name both of the +/// slots involved. Only runs on the failure path, hence the linear rescan. +fn describe_first_live_slot(fragments: &[FragmentRowIds], row_id: u64) -> String { + for (fragment, sequence, deletion_vector) in fragments { + // A bounding range that excludes the id rules the whole fragment out. + if sequence + .row_id_range() + .is_none_or(|range| !range.contains(&row_id)) + { + continue; + } + for (offset, candidate) in sequence.iter().enumerate() { + if candidate == row_id + && !deletion_vector + .as_ref() + .is_some_and(|deletions| deletions.contains(offset as u32)) + { + return format!("fragment {} at offset {}", fragment.id(), offset); + } + } + } + "an earlier fragment".to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + // Shared with the row id tests next door, which cover the same operations. + use super::super::test::{compact, delete}; + + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::{ + MergeInsertBuilder, UpdateBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams, + }; + use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + use arrow_array::RecordBatchIterator; + use arrow_array::types::Int32Type; + use arrow_schema::Schema as ArrowSchema; + use lance_table::format::{Fragment, RowIdMeta, pb}; + use lance_table::rowids::version::RowDatasetVersionSequence; + use lance_table::rowids::write_row_ids; + use prost::Message; + use rstest::rstest; + + fn encode_segments(segments: Vec) -> Vec { + pb::RowIdSequence { + segments: segments + .into_iter() + .map(|segment| pb::U64Segment { + segment: Some(segment), + }) + .collect(), + } + .encode_to_vec() + } + + fn range_segment(start: u64, end: u64) -> pb::u64_segment::Segment { + pb::u64_segment::Segment::Range(pb::u64_segment::Range { start, end }) + } + + fn empty_encoded_array() -> pb::EncodedU64Array { + pb::EncodedU64Array { + array: Some(pb::encoded_u64_array::Array::U64Array( + pb::encoded_u64_array::U64Array { values: Vec::new() }, + )), + } + } + + fn empty_array_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::Array(empty_encoded_array()) + } + + fn empty_sorted_array_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::SortedArray(empty_encoded_array()) + } + + fn empty_range_with_holes_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::RangeWithHoles(pb::u64_segment::RangeWithHoles { + start: 0, + end: 0, + holes: Some(empty_encoded_array()), + }) + } + + fn empty_range_with_bitmap_segment() -> pb::u64_segment::Segment { + pb::u64_segment::Segment::RangeWithBitmap(pb::u64_segment::RangeWithBitmap { + start: 0, + end: 0, + bitmap: Vec::new(), + }) + } + + /// Two `Range` segments that each fit a `usize` but whose lengths sum past `u64`. + /// Decoding accepts them, so `validate()` must report corruption rather than + /// overflowing while measuring the sequence. + #[tokio::test] + async fn test_validate_rejects_row_id_count_overflow() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + let encoded = encode_segments(vec![range_segment(0, u64::MAX), range_segment(0, u64::MAX)]); + edit_fragments(&mut dataset, |fragments| { + fragments[1].row_id_meta = Some(RowIdMeta::Inline(encoded.into())); + }); + + assert_invalid(&dataset, "total length exceeding u64::MAX").await; + } + + /// An empty segment carries no minimum or maximum, and decoding accepts an empty + /// encoding of every variant. Neither the cardinality check nor the `next_row_id` + /// bound may unwind on one. + #[rstest] + #[case::array(empty_array_segment())] + #[case::sorted_array(empty_sorted_array_segment())] + #[case::range_with_holes(empty_range_with_holes_segment())] + #[case::range_with_bitmap(empty_range_with_bitmap_segment())] + #[tokio::test] + async fn test_validate_tolerates_empty_segment(#[case] empty: pb::u64_segment::Segment) { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + // Ten ids across two segments matches the fragment's ten physical rows, and the + // ids stay clear of fragment 0's, so this dataset is well-formed. + let encoded = encode_segments(vec![range_segment(10, 20), empty]); + edit_fragments(&mut dataset, |fragments| { + fragments[1].row_id_meta = Some(RowIdMeta::Inline(encoded.into())); + }); + + dataset.validate().await.unwrap(); + } + + /// Write a two-fragment dataset with stable row ids to `uri`, then reopen it with a + /// fresh session so the row id sequence cache is empty. Tests that doctor a + /// fragment's `row_id_meta` need that: the cache is keyed by fragment id alone, so a + /// sequence loaded before the mutation would shadow the doctored one. + async fn validation_fixture(uri: &str) -> Dataset { + lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .into_dataset_with_params( + uri, + FragmentCount::from(2), + FragmentRowCount::from(10), + Some(WriteParams { + max_rows_per_file: 10, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + DatasetBuilder::from_uri(uri).load().await.unwrap() + } + + fn edit_fragments(dataset: &mut Dataset, edit: impl FnOnce(&mut Vec)) { + let mut manifest = dataset.manifest.as_ref().clone(); + let mut fragments = manifest.fragments.as_ref().clone(); + edit(&mut fragments); + manifest.fragments = Arc::new(fragments); + dataset.manifest = Arc::new(manifest); + } + + async fn assert_invalid(dataset: &Dataset, expected_message: &str) { + let err = dataset.validate().await.unwrap_err(); + assert!( + matches!(err, Error::CorruptFile { .. }), + "expected a corrupt file error, got {err:?}" + ); + let message = err.to_string(); + assert!( + message.contains(expected_message), + "expected {expected_message:?} in {message:?}" + ); + } + + #[tokio::test] + async fn test_validate_rejects_missing_row_id_meta() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| fragments[1].row_id_meta = None); + + assert_invalid(&dataset, "Fragment 1 has no row id metadata").await; + } + + #[tokio::test] + async fn test_validate_rejects_row_id_sequence_length_mismatch() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| { + let short = RowIdSequence::from(&[100u64, 101, 102][..]); + fragments[1].row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&short).into())); + }); + + assert_invalid(&dataset, "Fragment 1 has 3 row ids, but 10 physical rows").await; + } + + #[tokio::test] + async fn test_validate_rejects_duplicate_live_row_ids() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| { + fragments[1].row_id_meta = fragments[0].row_id_meta.clone(); + }); + + assert_invalid( + &dataset, + "Row id 0 is live in both fragment 0 at offset 0 and fragment 1 at offset 0", + ) + .await; + } + + #[tokio::test] + async fn test_validate_rejects_reused_next_row_id() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + let mut manifest = dataset.manifest.as_ref().clone(); + manifest.next_row_id = 19; + dataset.manifest = Arc::new(manifest); + + assert_invalid( + &dataset, + "will hand out row id 19 next, but row id 19 is already in use", + ) + .await; + } + + #[tokio::test] + async fn test_validate_rejects_misaligned_version_sequence() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let mut dataset = validation_fixture(&temp_dir).await; + edit_fragments(&mut dataset, |fragments| { + let too_long = RowDatasetVersionSequence::from_uniform_row_count(11, 1); + fragments[1].created_at_version_meta = + Some(RowDatasetVersionMeta::from_sequence(&too_long).unwrap()); + }); + + assert_invalid( + &dataset, + "Fragment 1 has 11 created_at versions, but 10 physical rows", + ) + .await; + } + + /// Number of sequence entries whose row id already appeared in an earlier fragment, + /// tombstoned slots included. + async fn count_repeated_row_ids(dataset: &Dataset) -> usize { + let mut seen = RoaringTreemap::new(); + let mut repeated = 0; + for fragment in dataset.get_fragments() { + let sequence = load_row_id_sequence(dataset, fragment.metadata()) + .await + .unwrap(); + repeated += sequence.iter().filter(|id| !seen.insert(*id)).count(); + } + repeated + } + + /// A tombstoned slot keeps its row id, so after an update the same id lives in the + /// rewritten fragment and lingers, deleted, in the original. Only live ids must be + /// unique — this asserts the checks agree with that. + #[tokio::test] + async fn test_validate_across_write_operations() { + let temp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let dataset = validation_fixture(&temp_dir).await; + dataset.validate().await.unwrap(); + + let batch = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step_custom::(20, 1)) + .into_batch_rows(lance_datagen::RowCount::from(10)) + .unwrap(); + let arrow_schema = Arc::new(ArrowSchema::from(dataset.schema())); + let reader = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema); + let mut dataset = Dataset::write( + reader, + &temp_dir, + Some(WriteParams { + mode: WriteMode::Append, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.validate().await.unwrap(); + + delete(&mut dataset, "i = 4 or i = 12").await; + dataset.validate().await.unwrap(); + + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("i >= 15") + .unwrap() + .set("i", "i + 1000") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + dataset.validate().await.unwrap(); + assert!( + count_repeated_row_ids(&dataset).await > 0, + "update should leave rewritten row ids behind in tombstoned slots" + ); + + let merge_source = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step_custom::(8, 1)) + .into_batch_rows(lance_datagen::RowCount::from(6)) + .unwrap(); + let schema = merge_source.schema(); + let merge_job = MergeInsertBuilder::try_new(dataset.clone(), vec!["i".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let source = lance_datafusion::utils::reader_to_stream(Box::new(RecordBatchIterator::new( + [Ok(merge_source)], + schema, + ))); + let (dataset, _stats) = merge_job.execute(source).await.unwrap(); + dataset.validate().await.unwrap(); + + let mut dataset = dataset.as_ref().clone(); + compact(&mut dataset, 20).await; + dataset.validate().await.unwrap(); + } +} diff --git a/rust/lance/src/dataset/scanner.rs b/rust/lance/src/dataset/scanner.rs index 671a2c24333..dbf57bf009f 100644 --- a/rust/lance/src/dataset/scanner.rs +++ b/rust/lance/src/dataset/scanner.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use datafusion::config::ConfigOptions; use lance_select::result::IndexExprResultWireFormat; @@ -17,7 +17,8 @@ use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema, SchemaR use arrow_select::concat::concat_batches; use async_recursion::async_recursion; use chrono::Utc; -use datafusion::common::{DFSchema, JoinType, NullEquality, SchemaExt, exec_datafusion_err}; +use datafusion::catalog::Session; +use datafusion::common::{DFSchema, JoinType, NullEquality, exec_datafusion_err}; use datafusion::functions_aggregate; use datafusion::logical_expr::{Expr, ScalarUDF, col, lit}; use datafusion::physical_expr::PhysicalSortExpr; @@ -51,7 +52,7 @@ use futures::{FutureExt, TryStreamExt}; use lance_arrow::floats::{FloatType, coerce_float_vector}; use lance_arrow::{DataTypeExt, SchemaExt as ArrowSchemaExt}; use lance_core::datatypes::{ - BlobHandling, Field, OnMissing, Projection, escape_field_path_for_project, format_field_path, + BlobHandling, Field, OnMissing, Projection, escape_field_path_for_project, }; use lance_core::error::LanceOptionExt; use lance_core::utils::address::RowAddress; @@ -65,43 +66,65 @@ use lance_datafusion::expr::safe_coerce_scalar; use lance_datafusion::projection::ProjectionPlan; use lance_file::reader::FileReaderOptions; use lance_index::IndexCriteria; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::pbold::InvertedIndexDetails; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::expression::PlannerIndexExt; use lance_index::scalar::expression::ScalarIndexExpr; use lance_index::scalar::inverted::query::{ - FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, PhraseQuery, fill_fts_query_column, + FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, Operator, PhraseQuery, + fill_fts_query_column, }; -use lance_index::scalar::inverted::{SCORE_COL, SCORE_FIELD}; +use lance_index::scalar::inverted::{ + DOC_INDEX_COL, DOC_INDEX_FIELD, DocumentGranularity, INVERTED_INDEX_VERSION_V2, + INVERTED_INDEX_VERSION_V3, SCORE_COL, SCORE_FIELD, fts_schema, +}; +use lance_index::scalar::registry::VALUE_COLUMN_NAME; use lance_index::vector::{ApproxMode, DEFAULT_QUERY_PARALLELISM, DIST_COL, Query}; -use lance_index::{metrics::NoOpMetricsCollector, scalar::inverted::FTS_SCHEMA}; use lance_io::stream::RecordBatchStream; use lance_linalg::distance::MetricType; -use lance_select::{IndexExprResult, RowAddrMask, RowAddrTreeMap}; +use lance_select::IndexExprResult; +// Re-exported so callers of `Scanner::with_row_addr_prefilter` can name the mask +// type without depending on `lance-select` directly. +pub use lance_select::{RowAddrMask, RowAddrTreeMap}; use lance_table::format::{Fragment, IndexMetadata}; +use prost::Message; use roaring::RoaringBitmap; use tracing::{Span, info_span, instrument}; use uuid::Uuid; use super::Dataset; +use super::versions; +use crate::dataset::overlay::{collect_overlay_stale_rows_for_segment, overlaid_fragments}; use crate::dataset::row_offsets_to_row_addresses; +use crate::dataset::rowids::{live_row_addrs_to_row_ids, translate_addr_treemap_to_row_ids}; use crate::dataset::utils::SchemaAdapter; use crate::index::DatasetIndexInternalExt; -use crate::index::scalar::inverted::{load_segment_details, load_segments}; -use crate::index::scalar_logical::scalar_index_fragment_bitmap; +use crate::index::scalar::fetch_index_details; +use crate::index::scalar::inverted::{ + fts_index_fragment_bitmap, load_segment_details, load_segments, normalize_inverted_details, + resolve_fts_field, resolve_query_document_granularity, +}; +use crate::index::scalar_logical::{load_named_scalar_segments, scalar_index_fragment_bitmap}; use crate::index::vector::utils::{ default_distance_type_for, get_vector_dim, get_vector_type, validate_distance_type_for, }; -use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions}; +use crate::io::exec::filtered_read::{ + FilteredReadExec, FilteredReadOptions, FilteredReadThreadingMode, +}; use crate::io::exec::fts::{ - BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, PhraseQueryExec, + BoostQueryExec, CompoundQueryExec, CrossColumnCompoundQueryExec, FlatMatchFilterExec, + FlatMatchQueryExec, FtsDocumentExec, HybridCompoundQueryExec, MatchQueryExec, PhraseQueryExec, + SharedFtsScorer, }; use crate::io::exec::knn::MultivectorScoringExec; use crate::io::exec::scalar_index::{MaterializeIndexExec, ScalarIndexExec}; use crate::io::exec::{ AddRowAddrExec, FilterPlan as ExprFilterPlan, KNNVectorDistanceExec, LancePushdownScanExec, - LanceScanExec, Planner, PreFilterSource, ScanConfig, TakeExec, + LanceScanExec, Planner, PreFilterSource, RowAddrMaskFilterExec, ScanConfig, TakeExec, knn::{ - KnnBatchParams, QUERY_INDEX_COL, knn_empty_result_schema, new_knn_exec, query_index_field, + KnnBatchParams, QUERY_INDEX_COL, knn_empty_result_schema, new_knn_batch_exec, new_knn_exec, + query_index_field, }, project, }; @@ -112,14 +135,337 @@ use crate::io::exec::{ use crate::{Error, Result}; use crate::{ datatypes::Schema, - io::exec::fts::{BoolSlot, BooleanQueryExec, build_boolean_query_children}, + io::exec::fts::{BoolSlot, BooleanQueryExec, build_boolean_query_children_with_schema}, }; pub use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts}; #[cfg(feature = "substrait")] use lance_datafusion::substrait::parse_substrait; -pub(crate) const BATCH_SIZE_FALLBACK: usize = 8192; +/// Rows per output batch when neither the scan options nor +/// `LANCE_DEFAULT_BATCH_SIZE` specify one. +pub const BATCH_SIZE_FALLBACK: usize = 8192; + +pub(crate) fn validate_batch_size(batch_size: usize) -> Result { + let validated = u32::try_from(batch_size).map_err(|_| { + Error::invalid_input(format!( + "batch_size must be between 1 and {}, got {batch_size}", + u32::MAX + )) + })?; + if validated == 0 { + return Err(Error::invalid_input(format!( + "batch_size must be between 1 and {}, got {batch_size}", + u32::MAX + ))); + } + Ok(validated) +} + +enum FtsOverlayPlan { + Unchanged(Option>), + RowLevel { + stale_rows: HashMap, + segments: Vec, + }, + FullScan, +} + +fn collect_fts_columns_in_order(query: &FtsQuery) -> Vec { + fn visit(query: &FtsQuery, columns: &mut Vec, seen: &mut HashSet) { + match query { + FtsQuery::Match(query) => { + if let Some(column) = &query.column + && seen.insert(column.clone()) + { + columns.push(column.clone()); + } + } + FtsQuery::Phrase(query) => { + if let Some(column) = &query.column + && seen.insert(column.clone()) + { + columns.push(column.clone()); + } + } + FtsQuery::Boost(query) => { + visit(&query.positive, columns, seen); + visit(&query.negative, columns, seen); + } + FtsQuery::MultiMatch(query) => { + for match_query in &query.match_queries { + if let Some(column) = &match_query.column + && seen.insert(column.clone()) + { + columns.push(column.clone()); + } + } + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(child, columns, seen); + } + } + } + } + + let mut columns = Vec::new(); + let mut seen = HashSet::new(); + visit(query, &mut columns, &mut seen); + columns +} + +fn collect_phrase_columns(query: &FtsQuery, columns: &mut HashSet) { + match query { + FtsQuery::Phrase(query) => { + if let Some(column) = &query.column { + columns.insert(column.clone()); + } + } + FtsQuery::Boost(query) => { + collect_phrase_columns(&query.positive, columns); + collect_phrase_columns(&query.negative, columns); + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + collect_phrase_columns(child, columns); + } + } + FtsQuery::Match(_) | FtsQuery::MultiMatch(_) => {} + } +} + +async fn load_physical_fts_details( + dataset: &Dataset, + column: &str, + segment: &IndexMetadata, +) -> Result { + let details = fetch_index_details(dataset, column, segment).await?; + let details = InvertedIndexDetails::decode(details.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {err}" + )) + })?; + normalize_inverted_details(segment, details) +} + +fn supports_compound_scorer(query: &FtsQuery) -> bool { + fn supports_shape(query: &FtsQuery) -> bool { + match query { + FtsQuery::Match(_) | FtsQuery::Phrase(_) | FtsQuery::MultiMatch(_) => true, + FtsQuery::Boolean(query) => { + (!query.should.is_empty() || !query.must.is_empty()) + && query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .all(supports_shape) + } + FtsQuery::Boost(query) => { + supports_shape(&query.positive) && supports_shape(&query.negative) + } + } + } + + if matches!(query, FtsQuery::Match(_) | FtsQuery::Phrase(_)) || !supports_shape(query) { + return false; + } + let columns = collect_fts_columns_in_order(query); + !columns.is_empty() && (!matches!(query, FtsQuery::MultiMatch(_)) || columns.len() == 1) +} + +fn supports_indexed_stats_residual_compound(query: &FtsQuery) -> bool { + match query { + FtsQuery::Match(query) => query.fuzziness == Some(0), + // MemWAL phrase matching currently collapses tokenizer position gaps. + // Keep phrase queries on the established fallback until it can retain + // those gaps exactly (notably when stop words are configured). + FtsQuery::Phrase(_) => false, + FtsQuery::Boost(query) => { + supports_indexed_stats_residual_compound(&query.positive) + && supports_indexed_stats_residual_compound(&query.negative) + } + FtsQuery::MultiMatch(query) => query + .match_queries + .iter() + .all(|query| query.fuzziness == Some(0)), + FtsQuery::Boolean(query) => query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .all(supports_indexed_stats_residual_compound), + } +} + +const MAX_QUERY_LOCAL_RESIDUAL_ROWS: usize = 100_000; + +fn has_bounded_query_local_residual_rows(fragments: &[Fragment]) -> bool { + fragments + .iter() + .try_fold(0usize, |total, fragment| { + total.checked_add(fragment.physical_rows?) + }) + .is_some_and(|total| total <= MAX_QUERY_LOCAL_RESIDUAL_ROWS) +} + +fn has_complete_hybrid_fts_coverage( + segments: &[IndexMetadata], + residual_fragments: &[Fragment], + target_fragments: &[Fragment], +) -> bool { + let Some(target) = target_fragments + .iter() + .map(|fragment| u32::try_from(fragment.id).ok()) + .collect::>() + else { + return false; + }; + let Some(residual) = residual_fragments + .iter() + .map(|fragment| u32::try_from(fragment.id).ok()) + .collect::>() + else { + return false; + }; + let mut indexed = RoaringBitmap::new(); + for segment in segments { + let Some(coverage) = segment.fragment_bitmap.as_ref() else { + return false; + }; + if !indexed.is_disjoint(coverage) { + return false; + } + indexed |= coverage; + } + if !indexed.is_subset(&target) || !indexed.is_disjoint(&residual) { + return false; + } + indexed | residual == target +} + +fn validate_fts_query_contract(query: &FtsQuery) -> Result<()> { + fn validate_multiplier(name: &str, value: f32) -> Result<()> { + if value.is_finite() && value >= 0.0 { + Ok(()) + } else { + Err(Error::invalid_input(format!( + "{name} must be finite and non-negative, got {value}" + ))) + } + } + + match query { + FtsQuery::Match(query) => validate_multiplier("MatchQuery boost", query.boost), + FtsQuery::Phrase(_) => Ok(()), + FtsQuery::Boost(query) => { + validate_multiplier("BoostQuery negative_boost", query.negative_boost)?; + validate_fts_query_contract(&query.positive)?; + validate_fts_query_contract(&query.negative) + } + FtsQuery::MultiMatch(query) => { + for match_query in &query.match_queries { + validate_multiplier("MultiMatchQuery boost", match_query.boost)?; + } + Ok(()) + } + FtsQuery::Boolean(query) => { + if query.should.is_empty() && query.must.is_empty() { + return Err(Error::invalid_input( + "boolean query must have at least one should/must query", + )); + } + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + validate_fts_query_contract(child)?; + } + Ok(()) + } + } +} + +fn normalize_fts_zero_boosts(query: &mut FtsQuery) { + fn normalize_zero(value: &mut f32) { + if *value == 0.0 { + *value = 0.0; + } + } + + match query { + FtsQuery::Match(query) => normalize_zero(&mut query.boost), + FtsQuery::Phrase(_) => {} + FtsQuery::Boost(query) => { + normalize_zero(&mut query.negative_boost); + normalize_fts_zero_boosts(&mut query.positive); + normalize_fts_zero_boosts(&mut query.negative); + } + FtsQuery::MultiMatch(query) => { + for match_query in &mut query.match_queries { + normalize_zero(&mut match_query.boost); + } + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter_mut() + .chain(&mut query.must) + .chain(&mut query.must_not) + { + normalize_fts_zero_boosts(child); + } + } + } +} + +/// Keep AUTO fuzziness exact at the public dataset-planning boundary. +/// +/// Low-level index preparation already understands `fuzziness=None`, but a +/// partial dataset plan must prepare one vocabulary across indexed and current +/// unindexed rows. AUTO activation is deferred until OSS-2105 lands that +/// current-row preparation atomically. Until then, recursively rewrite AUTO to +/// exact while preserving explicit positive fuzziness. +fn apply_dataset_planner_auto_fuzziness_compatibility_gate(query: &mut FtsQuery) { + match query { + FtsQuery::Match(query) => { + query.fuzziness.get_or_insert(0); + } + FtsQuery::Phrase(_) => {} + FtsQuery::Boost(query) => { + apply_dataset_planner_auto_fuzziness_compatibility_gate(&mut query.positive); + apply_dataset_planner_auto_fuzziness_compatibility_gate(&mut query.negative); + } + FtsQuery::MultiMatch(query) => { + for match_query in &mut query.match_queries { + match_query.fuzziness.get_or_insert(0); + } + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter_mut() + .chain(&mut query.must) + .chain(&mut query.must_not) + { + apply_dataset_planner_auto_fuzziness_compatibility_gate(child); + } + } + } +} /// Parse an environment variable as a specific type, logging a warning on parse failure. fn parse_env_var(env_var_name: &str, default_val: &str) -> Option @@ -286,10 +632,10 @@ impl MaterializationStyle { } #[derive(Debug)] -struct PlannedFilteredScan { - plan: Arc, - limit_pushed_down: bool, - filter_pushed_down: bool, +pub(super) struct PlannedFilteredScan { + pub(super) plan: Arc, + pub(super) limit_pushed_down: bool, + pub(super) filter_pushed_down: bool, } pub struct FilterPlan { @@ -347,7 +693,7 @@ impl FilterPlan { if self.refine_query_filter { match &self.query_filter { Some(QueryFilter::Fts(fts_query)) => { - let cols = if fts_query.columns().is_empty() { + let cols = if fts_query.query.is_missing_column() { let indexed_columns = fts_indexed_columns(dataset.clone()).await?; let q = fill_fts_query_column(&fts_query.query, &indexed_columns, false)?; q.columns() @@ -375,6 +721,7 @@ impl FilterPlan { &self, input: Arc, scanner: &Scanner, + session: Option<&dyn Session>, ) -> Result> { let mut plan = input; @@ -391,9 +738,12 @@ impl FilterPlan { } if let Some(refine_expr) = &self.expr_filter_plan.refine_expr { - // We create a new planner specific to the node's schema, since - // physical expressions reference column by index rather than by name. - plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?); + plan = Arc::new(match session { + Some(session) => { + LanceFilterExec::try_new_with_session(refine_expr.clone(), plan, session)? + } + None => LanceFilterExec::try_new(refine_expr.clone(), plan)?, + }); } Ok(plan) @@ -734,6 +1084,14 @@ pub struct Scanner { /// If true then the filter will be applied before an index scan prefilter: bool, + /// Optional external allow/block mask keyed in `_rowid` space. On a vector + /// search it is combined with the index-side prefilter and applied to the + /// flat branch for fragments not covered by the index; on a plain scan it is + /// the row source (see `use_external_mask`). Held behind an Arc so cloning it + /// into the ANN sub-plans and the flat-branch filter is cheap regardless of + /// mask size. + external_row_mask: Option>, + /// Materialization style controls when columns are fetched materialization_style: MaterializationStyle, @@ -747,10 +1105,11 @@ pub struct Scanner { batch_size: Option, /// If set, the scanner will produce batches whose total size in bytes - /// is approximately this value, overriding the row-based `batch_size`. + /// is approximately this value. When both limits are set, the scanner uses + /// the smaller row count selected by either limit. batch_size_bytes: Option, - /// Number of batches to prefetch + /// Number of batches to decode concurrently batch_readahead: usize, /// Number of fragments to read concurrently @@ -759,6 +1118,9 @@ pub struct Scanner { /// Number of bytes to allow to queue up in the I/O buffer io_buffer_size: Option, + /// Total bytes reserved by asynchronously materialized blob v2 batches + materialization_readahead_bytes: Option, + limit: Option, offset: Option, @@ -1032,6 +1394,7 @@ impl Scanner { projection_plan, blob_handling: BlobHandling::default(), prefilter: false, + external_row_mask: None, materialization_style: MaterializationStyle::Heuristic, filter: LanceFilter::default(), full_text_query: None, @@ -1040,6 +1403,7 @@ impl Scanner { batch_readahead: get_num_compute_intensive_cpus(), fragment_readahead: None, io_buffer_size: None, + materialization_readahead_bytes: None, limit: None, offset: None, ordering: None, @@ -1196,6 +1560,47 @@ impl Scanner { self } + /// Set an external [`RowAddrMask`] allow/block prefilter. + /// + /// Build the mask with [`RowAddrMask::from_allowed`] to keep only the listed + /// rows or [`RowAddrMask::from_block`] to drop them. On a vector + /// ([`nearest`](Self::nearest)) search the mask is combined with any + /// filter-derived prefilter on the index branch and applied to the flat + /// branch for fragments not covered by the vector index. On a + /// [`full_text_search`](Self::full_text_search) (match or phrase query) the + /// mask is combined into the FTS prefilter so BM25 top-k is computed over + /// masked rows, and the flat branch that scores unindexed fragments + /// (plan_flat_match_query) is masked with RowAddrMaskFilterExec. On a plain + /// scan the mask is used directly as the row source, with any + /// [`filter`](Self::filter) applied as a refine on top. + /// + /// The mask is keyed in the dataset's `_rowid` space, so build it from the + /// same dataset you query. That space is the row address when stable row ids + /// are disabled and the stable row id when they are enabled; both are handled + /// (index prefilter and filtered read branch on `uses_stable_row_ids`), so no + /// caller-side translation is needed either way. + /// + /// # Example + /// + /// ```no_run + /// # use lance::dataset::Dataset; + /// # async fn example(dataset: &Dataset) -> lance::Result<()> { + /// use lance::dataset::scanner::{RowAddrMask, RowAddrTreeMap}; + /// + /// // Restrict the scan to rows whose _rowid is 0, 2, or 4. + /// let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([0u64, 2, 4])); + /// let mut scanner = dataset.scan(); + /// scanner.with_row_addr_prefilter(mask); + /// let batch = scanner.try_into_batch().await?; + /// # let _ = batch; + /// # Ok(()) + /// # } + /// ``` + pub fn with_row_addr_prefilter(&mut self, mask: RowAddrMask) -> &mut Self { + self.external_row_mask = Some(Arc::new(mask)); + self + } + /// Set the callback to be called after the scan with summary statistics pub fn scan_stats_callback(&mut self, callback: ExecutionStatsCallback) -> &mut Self { self.scan_stats_callback = Some(callback); @@ -1275,15 +1680,6 @@ impl Scanner { /// .into_stream(); /// ``` pub fn full_text_search(&mut self, query: FullTextSearchQuery) -> Result<&mut Self> { - let fields = query.columns(); - if !fields.is_empty() { - for field in fields.iter() { - if self.dataset.schema().field(field).is_none() { - return Err(Error::invalid_input(format!("Column {} not found", field))); - } - } - } - self.full_text_query = Some(query); Ok(self) } @@ -1315,9 +1711,11 @@ impl Scanner { /// Set the maximum number of rows per batch. /// - /// Note: this can be overridden by [`Self::batch_size_bytes`] or by a dataset-level - /// `batch_size_bytes` set via [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options). When a byte-based - /// batch size is active, the row-based batch size is used only as an initial estimate. + /// The batch size must be between 1 and [`u32::MAX`], inclusive. + /// + /// When a byte limit is also configured through [`Self::batch_size_bytes`] or + /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options), + /// both limits apply and the one reached first determines the batch size. pub fn batch_size(&mut self, batch_size: usize) -> &mut Self { self.batch_size = Some(batch_size); self @@ -1326,7 +1724,10 @@ impl Scanner { /// Set the target batch size in bytes. /// /// When set, the scanner will produce batches whose total size in bytes - /// is approximately this value, overriding the row-based `batch_size`. + /// is approximately this value. When a row-based `batch_size` is also set, + /// both limits apply and the one reached first determines the batch size. + /// This cannot be combined with [`Self::strict_batch_size`] because strict + /// row batching can merge batches beyond the byte limit. /// /// This can also be configured at the dataset level via /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options). A scanner-level setting takes @@ -1371,16 +1772,43 @@ impl Scanner { self } - /// Set the prefetch size. - /// Ignored in v2 and newer format + /// Set the memory budget for asynchronous blob v2 materialization. + /// + /// Blob descriptors are decoded before their payloads are fetched. When this + /// budget is set, payload materialization may run ahead while the aggregate + /// descriptor arrays, output offsets, and payload bytes awaiting ordered + /// emission stay within `size`. Admission follows output order, and each + /// reservation is retained until its batch is emitted. A single oversized + /// batch is admitted when no other materialization is reserved, which + /// guarantees forward progress. External descriptors without a stored size + /// resolve the complete object length before admission. + /// + /// This budget is separate from [`Self::io_buffer_size`], which controls the + /// storage I/O scheduler, and [`Self::batch_size_bytes`], which targets the + /// size of individual decoded batches. If this setting is not provided, + /// Blob v2 materialization has no independent memory bound. A size of zero + /// is rejected when the scan plan is built. + pub fn materialization_readahead_bytes(&mut self, size: u64) -> &mut Self { + self.materialization_readahead_bytes = Some(size); + self + } + + /// Set the number of batches to decode concurrently. + /// + /// This bounds the decode fan-out of the scan: at most this many batch-decode + /// tasks run in flight at once. Defaults to `get_num_compute_intensive_cpus()`. + /// + /// `nbatches` must be greater than zero. pub fn batch_readahead(&mut self, nbatches: usize) -> &mut Self { self.batch_readahead = nbatches; self } - /// Set the fragment readahead. + /// Set the number of fragments whose reads may be scheduled concurrently. /// - /// This is only used if ``scan_in_order`` is set to false. + /// This applies to both ordered and unordered scans. [`Self::scan_in_order`] + /// controls result ordering, not whether fragment I/O overlaps. Set this to + /// `1` to read one fragment at a time. pub fn fragment_readahead(&mut self, nfragments: usize) -> &mut Self { self.fragment_readahead = Some(nfragments); self @@ -1430,6 +1858,10 @@ impl Scanner { /// By default, this is False and output batches are allowed to have fewer than `batch_size` rows /// Setting this to True will require us to merge batches, incurring a data copy, for a minor performance /// penalty. + /// + /// This cannot be enabled when a byte limit is configured through + /// [`Self::batch_size_bytes`] or + /// [`ReadParams::file_reader_options`](crate::dataset::ReadParams::file_reader_options). pub fn strict_batch_size(&mut self, strict_batch_size: bool) -> &mut Self { self.strict_batch_size = strict_batch_size; self @@ -1742,7 +2174,7 @@ impl Scanner { .field(&column.column_name) .ok_or(Error::invalid_input(format!( "Column {} not found", - &column.column_name + column.column_name )))?; } } @@ -1760,8 +2192,9 @@ impl Scanner { /// Configure the speed / accuracy tradeoff for approximate vector search. /// - /// This setting is currently only used by RQ-quantized indexes, such as - /// IVF_RQ. Other index types ignore this setting. + /// This setting is currently used by RQ-quantized indexes (such as + /// IVF_RQ) and by prefiltered search on HNSW indexes, where `Fast` + /// enables the ACORN traversal. Other index types ignore this setting. pub fn approx_mode(&mut self, approx_mode: ApproxMode) -> &mut Self { if let Some(q) = self.nearest.as_mut() { q.approx_mode = approx_mode; @@ -1839,9 +2272,7 @@ impl Scanner { .or_else(|| self.dataset.file_reader_options.clone()); match (base, self.batch_size_bytes) { (Some(mut opts), Some(bsb)) => { - if opts.batch_size_bytes.is_none() { - opts.batch_size_bytes = Some(bsb); - } + opts.batch_size_bytes = Some(bsb); Some(opts) } (Some(opts), None) => Some(opts), @@ -1899,6 +2330,38 @@ impl Scanner { } } + /// Ensure `input` exposes `column_name` as a top-level column. + /// + /// Nested FTS flat-search paths read the projected struct column from storage + /// but the FTS executor consumes a single document column by name. + fn ensure_column_alias( + &self, + input: Arc, + column_name: &str, + ) -> Result> { + let input_schema = input.schema(); + if input_schema.column_with_name(column_name).is_some() { + return Ok(input); + } + + let mut projection_exprs = Vec::with_capacity(input_schema.fields().len() + 1); + for field in input_schema.fields() { + projection_exprs.push(( + Arc::new(Column::new_with_schema( + field.name(), + input_schema.as_ref(), + )?) as Arc, + field.name().clone(), + )); + } + projection_exprs.push(( + Self::create_column_expr(column_name, self.dataset.as_ref(), input_schema.as_ref())?, + column_name.to_string(), + )); + + Ok(Arc::new(ProjectionExec::try_new(projection_exprs, input)?)) + } + /// Set whether to use statistics to optimize the scan (default: true) /// /// This is used for debugging or benchmarking purposes. @@ -1931,7 +2394,11 @@ impl Scanner { } } - fn add_extra_columns(&self, schema: Schema) -> Result { + fn add_extra_columns( + &self, + schema: Schema, + fts_document_granularity: Option, + ) -> Result { let mut extra_columns = vec![ArrowField::new(ROW_OFFSET, DataType::UInt64, true)]; if self.nearest.as_ref().is_some() { @@ -1943,6 +2410,15 @@ impl Scanner { if self.full_text_query.is_some() { extra_columns.push(ArrowField::new(SCORE_COL, DataType::Float32, true)); + if fts_document_granularity + .map(DocumentGranularity::is_list_element) + // `get_expr_filter` is synchronous, so an omitted granularity + // cannot be resolved from index metadata here. Include the + // column conservatively; `create_plan` uses the resolved value. + .unwrap_or(true) + { + extra_columns.push(DOC_INDEX_FIELD.clone()); + } } schema.merge(&ArrowSchema::new(extra_columns)) @@ -1953,6 +2429,17 @@ impl Scanner { /// This is the schema of the dataset, any metadata columns like _rowid or _rowaddr /// and any extra columns like _distance or _score fn filterable_schema(&self) -> Result> { + let fts_document_granularity = self + .full_text_query + .as_ref() + .and_then(|query| self.fts_document_granularity(&query.query).ok()); + self.filterable_schema_with_fts_granularity(fts_document_granularity) + } + + fn filterable_schema_with_fts_granularity( + &self, + fts_document_granularity: Option, + ) -> Result> { let base_schema = Projection::full(self.dataset.clone()) .with_row_id() .with_row_addr() @@ -1960,7 +2447,10 @@ impl Scanner { .with_row_created_at_version() .to_schema(); - Ok(Arc::new(self.add_extra_columns(base_schema)?)) + Ok(Arc::new(self.add_extra_columns( + base_schema, + fts_document_granularity, + )?)) } /// This takes the current output, and the user's requested projection, and calculates the @@ -1975,6 +2465,16 @@ impl Scanner { // of all available columns if the user did not specify a projection) let mut output_expr = self.projection_plan.to_physical_exprs(current_schema)?; + if self.full_text_query.is_some() + && current_schema.field_with_name(DOC_INDEX_COL).is_ok() + && output_expr.iter().all(|(_, name)| name != DOC_INDEX_COL) + { + output_expr.push(( + expressions::col(DOC_INDEX_COL, current_schema)?, + DOC_INDEX_COL.to_string(), + )); + } + // Make sure _distance and _score are _always_ in the output unless user has opted out of the legacy // projection behavior if self.autoproject_scoring_columns { @@ -2186,6 +2686,9 @@ impl Scanner { } #[allow(clippy::type_complexity)] + // TODO(datafusion-54): migrate off the deprecated + // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder. + #[allow(deprecated)] fn build_physical_aggregate_expr( &self, expr: &Expr, @@ -2310,11 +2813,12 @@ impl Scanner { MaterializationStyle::AllLate => false, MaterializationStyle::AllEarlyExcept(ref cols) => !cols.contains(&(field.id as u32)), MaterializationStyle::Heuristic => { - if field.is_blob() { - // By default, blobs are loaded as descriptions, and so should be early - // - // TODO: Once we make blob handling configurable, we should use the blob - // handling setting here. + if field.is_blob() && self.blob_handling.returns_description(field) { + // A blob returned as a description (offset + size) is tiny, so it is + // cheaper to read eagerly. When blob_handling materializes the full + // binary value instead (e.g. `all_binary`), fall through to the + // width-based heuristic so a selective filter can late-materialize it + // rather than reading the whole column. return true; } @@ -2362,6 +2866,35 @@ impl Scanner { } fn validate_options(&self) -> Result<()> { + if self.batch_readahead == 0 { + return Err(Error::invalid_input_source( + "batch_readahead must be greater than 0, got 0".into(), + )); + } + + if self.materialization_readahead_bytes == Some(0) { + return Err(Error::invalid_input_source( + "materialization_readahead_bytes must be greater than 0, got 0".into(), + )); + } + + if let Some(batch_size) = self.batch_size { + validate_batch_size(batch_size)?; + } + + if self.strict_batch_size + && let Some(batch_size_bytes) = self + .resolved_file_reader_options() + .and_then(|options| options.batch_size_bytes) + { + return Err(Error::invalid_input_source( + format!( + "strict_batch_size=true cannot be combined with batch_size_bytes={batch_size_bytes}; strict row batching can merge batches beyond the byte limit" + ) + .into(), + )); + } + if self.include_deleted_rows && !self.projection_plan.physical_projection.with_row_id { return Err(Error::invalid_input_source( "include_deleted_rows is set but with_row_id is false".into(), @@ -2392,8 +2925,14 @@ impl Scanner { Ok(()) } - async fn create_filter_plan(&self, use_scalar_index: bool) -> Result { - let filter_schema = self.filterable_schema()?; + async fn create_filter_plan( + &self, + use_scalar_index: bool, + query_filter: Option, + fts_document_granularity: Option, + ) -> Result { + let filter_schema = + self.filterable_schema_with_fts_granularity(fts_document_granularity)?; let planner = Planner::new(Arc::new(filter_schema.as_ref().into())); // Check expr filter @@ -2423,15 +2962,15 @@ impl Scanner { // fallback to a non-indexed filter let filter_plan = planner.create_filter_plan(expr.clone(), &index_info, false)?; - FilterPlan::new(self.filter.query_filter.clone(), filter_plan) + FilterPlan::new(query_filter.clone(), filter_plan) } else { - FilterPlan::new(self.filter.query_filter.clone(), filter_plan) + FilterPlan::new(query_filter.clone(), filter_plan) } } else { - FilterPlan::new(self.filter.query_filter.clone(), filter_plan) + FilterPlan::new(query_filter.clone(), filter_plan) } } else { - FilterPlan::new(self.filter.query_filter.clone(), ExprFilterPlan::default()) + FilterPlan::new(query_filter, ExprFilterPlan::default()) }; // Check query filter @@ -2539,18 +3078,50 @@ impl Scanner { /// 3. Sort /// 4. Limit / Offset /// 5. Take remaining columns / Projection + pub fn create_plan(&self) -> BoxFuture<'_, Result>> { + Box::pin(self.create_plan_impl(None)) + } + + pub(crate) fn create_plan_with_session<'a>( + &'a self, + session: &'a dyn Session, + ) -> BoxFuture<'a, Result>> { + Box::pin(self.create_plan_impl(Some(session))) + } + #[instrument(level = "debug", skip_all)] - pub async fn create_plan(&self) -> Result> { + async fn create_plan_impl( + &self, + session: Option<&dyn Session>, + ) -> Result> { log::trace!("creating scanner plan"); self.validate_options()?; + let full_text_query = match &self.full_text_query { + Some(query) => Some(self.resolve_full_text_search_query(query).await?), + None => None, + }; + let query_filter = match &self.filter.query_filter { + Some(QueryFilter::Fts(query)) => Some(QueryFilter::Fts( + self.resolve_full_text_search_query(query).await?, + )), + Some(QueryFilter::Vector(query)) => Some(QueryFilter::Vector(query.clone())), + None => None, + }; + let fts_document_granularity = full_text_query + .as_ref() + .map(|query| self.fts_document_granularity(&query.query)) + .transpose()?; + // Scalar indices are only used when prefiltering let use_scalar_index = self.use_scalar_index && (self.prefilter || self.nearest.is_none()); - let mut filter_plan = self.create_filter_plan(use_scalar_index).await?; + let mut filter_plan = self + .create_filter_plan(use_scalar_index, query_filter, fts_document_granularity) + .await?; let mut use_limit_node = true; // Source: either a (K|A)NN search, full text search, or a (full|indexed) scan - let mut plan: Arc = match (&self.nearest, &self.full_text_query) { + let mut plan: Arc = match (&self.nearest, &full_text_query) { (Some(_), None) => self.vector_search_source(&mut filter_plan).await?, (None, Some(query)) => self.fts_search_source(&mut filter_plan, query).await?, (None, None) => { @@ -2585,7 +3156,7 @@ impl Scanner { self.take_source(take_op).await? } else { let planned_read = self - .filtered_read_source(&mut filter_plan.expr_filter_plan) + .filtered_read_source(&mut filter_plan.expr_filter_plan, session) .await?; if planned_read.limit_pushed_down { use_limit_node = false; @@ -2630,7 +3201,7 @@ impl Scanner { plan = self.take(plan, pre_filter_projection)?; // Filter - plan = filter_plan.refine_filter(plan, self).await?; + plan = filter_plan.refine_filter(plan, self, session).await?; // Aggregate (if set, applies aggregate and returns early) if let Some(agg) = &self.aggregate { @@ -2746,7 +3317,7 @@ impl Scanner { // Do not call this directly, use filtered_read instead // // First return value is the plan, second is whether the limit was pushed down - async fn legacy_filtered_read( + pub(super) async fn legacy_filtered_read( &self, filter_plan: &ExprFilterPlan, projection: Projection, @@ -2847,19 +3418,51 @@ impl Scanner { } } + // A plain-scan external row mask is fed as the FilteredReadExec row source so + // only masked rows are read, with any SQL filter applied as a refine on top. + // Vector and full-text searches apply the mask via their own prefilter paths + // (KNN external_mask / FTS build_prefilter), so this plain-scan source is + // scoped to scans that are neither. FTS in particular has nearest.is_none(), + // so excluding it here keeps the FTS prefilter's own filtered read unmasked. + fn use_external_mask(&self) -> bool { + self.nearest.is_none() && self.full_text_query.is_none() && self.external_row_mask.is_some() + } + + // The filter plan actually handed to the filtered read. With an external mask + // active the mask is the row source, so any SQL filter is demoted to a refine + // on top of it; otherwise the plan is used as-is. Projection and scan-range + // planning must be done against this, not the raw filter_plan, so refine + // columns are retained and limit/offset is not pushed down before masking. + fn effective_filter_plan(&self, filter_plan: &ExprFilterPlan) -> ExprFilterPlan { + if self.use_external_mask() { + match filter_plan.full_expr.clone() { + Some(expr) => ExprFilterPlan::new_refine_only(expr), + None => ExprFilterPlan::default(), + } + } else { + filter_plan.clone() + } + } + // Helper function for filtered_read // // Do not call this directly, use filtered_read instead - async fn new_filtered_read( + pub(super) async fn new_filtered_read( &self, filter_plan: &ExprFilterPlan, projection: Projection, make_deletions_null: bool, fragments: Option>>, scan_range: Option>, + session: Option<&dyn Session>, ) -> Result> { + // Kept for the overlay stale-Take path below, which re-evaluates blocked stale rows. + let user_projection = projection.clone(); + let use_external_mask = self.use_external_mask(); + let effective_filter = self.effective_filter_plan(filter_plan); + let mut read_options = FilteredReadOptions::basic_full_read(&self.dataset) - .with_filter_plan(filter_plan.clone()) + .with_filter_plan(effective_filter) .with_projection(projection); if let Some(fragments) = fragments { @@ -2871,9 +3474,14 @@ impl Scanner { } if let Some(batch_size) = self.batch_size { - read_options = read_options.with_batch_size(batch_size as u32); + read_options = read_options.with_batch_size(validate_batch_size(batch_size)?); } + // Bound the decode fan-out by `batch_readahead`. + read_options = read_options.with_threading_mode( + FilteredReadThreadingMode::OnePartitionMultipleThreads(self.batch_readahead), + ); + if let Some(file_reader_options) = self.resolved_file_reader_options() { read_options = read_options.with_file_reader_options(file_reader_options); } @@ -2890,72 +3498,156 @@ impl Scanner { read_options = read_options.with_io_buffer_size(io_buffer_size_bytes); } + if let Some(materialization_readahead_bytes) = self.materialization_readahead_bytes { + read_options = + read_options.with_materialization_readahead_bytes(materialization_readahead_bytes); + } + if self.fast_search && filter_plan.has_index_query() { read_options = read_options.with_only_indexed_fragments(); } + if let Some(session) = session { + read_options = read_options.with_physical_filters(session)?; + } + + // Mask data overlay files: a row with an overlay committed after an index it relies on + // touched an indexed field can no longer be trusted to that index. Block just those rows + // from the index result (their fragments stay indexed, so non-stale rows keep the index) + // and re-evaluate them on a targeted take path below — O(stale_rows), not O(fragment). + let mut overlay_stale_rows: HashMap = HashMap::new(); + if let Some(index_query) = filter_plan.index_query.as_ref() { + let candidate_frags = read_options + .fragments + .clone() + .unwrap_or_else(|| self.dataset.fragments().clone()); + overlay_stale_rows = self + .overlay_stale_index_rows(index_query, &candidate_frags) + .await?; + if let Some(block) = self.stale_rows_block_mask(&overlay_stale_rows).await? { + read_options = read_options.with_overlay_block(block); + } + } + let result_format = self.index_expr_result_format(); - let index_input = filter_plan.index_query.clone().map(|index_query| { - Arc::new(ScalarIndexExec::new( - self.dataset.clone(), - index_query, - result_format, - )) as Arc - }); + let index_input = match self.external_row_mask.as_deref() { + Some(mask) if use_external_mask => Some(self.mask_as_take_input(mask.clone())?), + _ => filter_plan.index_query.clone().map(|index_query| { + Arc::new(ScalarIndexExec::new( + self.dataset.clone(), + index_query, + result_format, + )) as Arc + }), + }; - Ok(Arc::new(FilteredReadExec::try_new( + let plan: Arc = Arc::new(FilteredReadExec::try_new( self.dataset.clone(), read_options, index_input, + )?); + + if overlay_stale_rows.is_empty() { + return Ok(plan); + } + + // Stale-Take path: take the stale rows' current (overlay-merged) values and re-apply the + // full filter, then union with the indexed read. These rows were blocked from the index + // result above, so this is the only path that can surface them. + let filter = filter_plan.full_expr.as_ref().expect_ok()?; + let filter_cols = Planner::column_names_in_expr(filter); + let take_projection = user_projection.union_columns(filter_cols, OnMissing::Error)?; + + let stale_node = self + .stale_rows_take(&overlay_stale_rows, take_projection) + .await?; + let planner = Planner::new(stale_node.schema()); + let optimized_filter = planner.optimize_expr(filter.clone())?; + let filtered = Arc::new(match session { + Some(session) => { + LanceFilterExec::try_new_with_session(optimized_filter, stale_node, session)? + } + None => LanceFilterExec::try_new(optimized_filter, stale_node)?, + }); + let stale_path: Arc = + Arc::new(project(filtered, plan.schema().as_ref())?); + + let unioned = UnionExec::try_new(vec![plan, stale_path])?; + Ok(Arc::new(RepartitionExec::try_new( + unioned, + datafusion::physical_plan::Partitioning::RoundRobinBatch(1), )?)) } // Helper function for filtered read // // Delegates to legacy or new filtered read based on dataset storage version - async fn filtered_read( - &self, - filter_plan: &ExprFilterPlan, + #[allow(clippy::too_many_arguments)] + fn filtered_read<'a>( + &'a self, + filter_plan: &'a ExprFilterPlan, projection: Projection, make_deletions_null: bool, fragments: Option>>, scan_range: Option>, is_prefilter: bool, - ) -> Result { - // Use legacy path if dataset uses legacy storage format - if self.dataset.is_legacy_storage() { - self.legacy_filtered_read( - filter_plan, - projection, - make_deletions_null, - fragments, - scan_range, - is_prefilter, - ) - .await - } else { - let limit_pushed_down = scan_range.is_some(); - let plan = self - .new_filtered_read( - filter_plan, - projection, - make_deletions_null, - fragments, - scan_range, - ) - .await?; - Ok(PlannedFilteredScan { - filter_pushed_down: true, - limit_pushed_down, - plan, - }) + session: Option<&'a dyn Session>, + ) -> BoxFuture<'a, Result> { + // The plain-scan mask path lives in new_filtered_read; legacy_filtered_read + // has no equivalent, so a masked plain scan there would silently drop the + // mask and return every row. Fail loudly instead. Vector and full-text + // searches apply the mask via their own prefilter paths (ANN prefilter / + // FTS build_prefilter) plus the RowAddrMaskFilterExec flat wrap, so they + // are unaffected -- use_external_mask() is false for them. + let is_legacy = self + .dataset + .manifest() + .data_storage_format + .lance_file_format() + == lance_file::version::ConcreteFileVersion::V1; + if is_legacy && self.use_external_mask() { + return std::future::ready(Err(Error::not_supported( + "with_row_addr_prefilter is not supported for plain scans on \ + legacy-storage datasets", + ))) + .boxed(); } + versions::filtered_read( + self.dataset + .manifest() + .data_storage_format + .lance_file_format(), + self, + filter_plan, + projection, + make_deletions_null, + fragments, + scan_range, + is_prefilter, + session, + ) + .boxed() + } + + fn row_ids_as_take_input(&self, row_ids: RowAddrTreeMap) -> Result> { + self.mask_as_take_input(RowAddrMask::from_allowed(row_ids)) } - fn u64s_as_take_input(&self, u64s: Vec) -> Result> { - let row_addrs = RowAddrTreeMap::from_iter(u64s); - let row_addr_mask = RowAddrMask::from_allowed(row_addrs); - let index_result = IndexExprResult::exact(row_addr_mask); + // Wrap a row-address mask as a one-shot index input for FilteredReadExec, so a + // plain scan reads only the rows the mask selects. + // + // Every take-shaped row source funnels through here: plain takes, the + // _rowid/_rowaddr predicate shortcut, and the overlay stale-row replay under + // both scan and ANN. Intersecting the caller's mask once at this boundary is + // what keeps the invariant on all of them; applying it per branch is how + // branches get missed. Idempotent, so the branch that passes the external + // mask itself is unaffected. + fn mask_as_take_input(&self, mask: RowAddrMask) -> Result> { + let mask = match self.external_row_mask.as_deref() { + Some(external) => mask.intersect(external.clone()), + None => mask, + }; + let index_result = IndexExprResult::exact(mask); let fragments_covered = self.dataset.fragment_bitmap.as_ref().clone(); let format = self.index_expr_result_format(); let batch = index_result.serialize(&fragments_covered, format)?; @@ -2965,19 +3657,27 @@ impl Scanner { Ok(Arc::new(OneShotExec::new(stream))) } + async fn row_addrs_as_take_input(&self, row_addrs: Vec) -> Result> { + let row_ids = + live_row_addrs_to_row_ids(&self.dataset, row_addrs.into_iter().map(Some)).await?; + self.row_ids_as_take_input(RowAddrTreeMap::from_iter(row_ids.into_iter().flatten())) + } + async fn take_source(&self, take_op: TakeOperation) -> Result> { // We generally assume that late materialization does not make sense for take operations // so we can just use the physical projection let projection = self.projection_plan.physical_projection.clone(); let input = match take_op { - TakeOperation::RowIds(ids) => self.u64s_as_take_input(ids), - TakeOperation::RowAddrs(addrs) => self.u64s_as_take_input(addrs), + TakeOperation::RowIds(ids) => { + self.row_ids_as_take_input(RowAddrTreeMap::from_iter(ids)) + } + TakeOperation::RowAddrs(addrs) => self.row_addrs_as_take_input(addrs).await, TakeOperation::RowOffsets(offsets) => { let mut addrs = row_offsets_to_row_addresses(&self.dataset.get_fragments(), &offsets).await?; addrs.retain(|addr| *addr != RowAddress::TOMBSTONE_ROW); - self.u64s_as_take_input(addrs) + self.row_addrs_as_take_input(addrs).await } }?; @@ -2997,6 +3697,7 @@ impl Scanner { async fn filtered_read_source( &self, filter_plan: &mut ExprFilterPlan, + session: Option<&dyn Session>, ) -> Result { log::trace!("source is a filtered read"); @@ -3018,11 +3719,15 @@ impl Scanner { self.projection_plan.physical_projection.clone() }; - let mut projection = if filter_plan.has_refine() { + // Plan against the effective filter: with an external mask the SQL filter + // becomes a refine, so its columns must be retained even when the original + // plan resolved to an exact scalar-index query (has_refine() == false). + let effective_filter = self.effective_filter_plan(filter_plan); + let mut projection = if effective_filter.has_refine() { // If the filter plan has two steps (a scalar indexed portion and a refine portion) then // it makes sense to grab cheap columns during the first step to avoid taking them for // the second step. - self.calc_eager_projection(filter_plan, &effective_projection)? + self.calc_eager_projection(&effective_filter, &effective_projection)? .with_row_id() } else { // If the filter plan only has one step then we just do a filtered read of all the @@ -3036,7 +3741,11 @@ impl Scanner { projection.with_row_addr = true; } - let scan_range = if filter_plan.is_empty() { + // An external mask is applied as the row source inside new_filtered_read, so + // limit/offset must not be pushed down as a pre-mask range (that would limit + // rows before masking). Leaving scan_range None keeps limit_pushed_down false + // so the limit is applied by a node above the masked source instead. + let scan_range = if filter_plan.is_empty() && !self.use_external_mask() { log::trace!("pushing scan_range into filtered_read"); self.get_scan_range(filter_plan).await? } else { @@ -3050,6 +3759,7 @@ impl Scanner { self.fragments.clone().map(Arc::new), scan_range, /*is_prefilter= */ false, + session, ) .await } @@ -3107,7 +3817,10 @@ impl Scanner { // If we are prefiltering then the ann / knn node will take care of the filter let source: Arc = match &filter_plan.fts_filter() { Some(fts_query) => { - let fts_plan = self.fts(&filter_plan.expr_filter_plan, fts_query).await?; + let mut fts_plan = self.fts(&filter_plan.expr_filter_plan, fts_query).await?; + if fts_plan.schema().field_with_name(DOC_INDEX_COL).is_ok() { + fts_plan = self.deduplicate_fts_filter_rows(fts_plan)?; + } let projection = self .dataset .empty_projection() @@ -3133,25 +3846,45 @@ impl Scanner { } } + /// Convert element-document hits into the row-selection semantics required + /// when FTS is used as a filter for another query. + fn deduplicate_fts_filter_rows( + &self, + input: Arc, + ) -> Result> { + let schema = input.schema(); + let group_expr = vec![( + expressions::col(ROW_ID, schema.as_ref())?, + ROW_ID.to_string(), + )]; + let input = Arc::new(RepartitionExec::try_new( + input, + Partitioning::RoundRobinBatch(1), + )?); + Ok(Arc::new(AggregateExec::try_new( + AggregateMode::Single, + PhysicalGroupBy::new_single(group_expr), + Vec::new(), + Vec::new(), + input, + schema, + )?)) + } + async fn fragments_covered_by_fts_leaf( &self, column: &str, + document_granularity: DocumentGranularity, accum: &mut RoaringBitmap, ) -> Result { - let index = self - .dataset - .load_scalar_index(IndexCriteria::default().for_column(column).supports_fts()) - .await?; - match index { - Some(index) => match &index.fragment_bitmap { - Some(fragmap) => { - *accum |= fragmap; - Ok(true) - } - None => Ok(false), - }, - None => Ok(false), - } + let Some(fragment_bitmap) = + fts_index_fragment_bitmap(&self.dataset, column, document_granularity).await? + else { + return Ok(false); + }; + *accum |= fragment_bitmap; + + Ok(true) } #[async_recursion] @@ -3162,10 +3895,14 @@ impl Scanner { ) -> Result { match query { FtsQuery::Match(match_query) => { + let document_granularity = match_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; self.fragments_covered_by_fts_leaf( match_query.column.as_ref().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?, + document_granularity, accum, ) .await @@ -3178,11 +3915,17 @@ impl Scanner { .await?), FtsQuery::MultiMatch(multi_match) => { for mq in &multi_match.match_queries { + let document_granularity = mq.document_granularity.ok_or_else(|| { + Error::internal( + "FTS MultiMatch query granularity was not resolved".to_string(), + ) + })?; if !self .fragments_covered_by_fts_leaf( mq.column.as_ref().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?, + document_granularity, accum, ) .await? @@ -3193,24 +3936,25 @@ impl Scanner { Ok(true) } FtsQuery::Phrase(phrase_query) => { + let document_granularity = phrase_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Phrase query granularity was not resolved".to_string()) + })?; self.fragments_covered_by_fts_leaf( phrase_query.column.as_ref().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?, + document_granularity, accum, ) .await } FtsQuery::Boolean(bool_query) => { - for query in bool_query.must.iter() { - if !self - .fragments_covered_by_fts_query_helper(query, accum) - .await? - { - return Ok(false); - } - } - for query in &bool_query.should { + for query in bool_query + .must + .iter() + .chain(&bool_query.should) + .chain(&bool_query.must_not) + { if !self .fragments_covered_by_fts_query_helper(query, accum) .await? @@ -3239,13 +3983,293 @@ impl Scanner { } } + fn fts_document_granularity(&self, query: &FtsQuery) -> Result { + #[derive(Default)] + struct TargetState { + granularity: Option, + list_element_field_id: Option, + } + + fn add_leaf( + schema: &lance_core::datatypes::Schema, + column: Option<&str>, + document_granularity: Option, + state: &mut TargetState, + ) -> Result<()> { + let document_granularity = document_granularity.ok_or_else(|| { + Error::internal("FTS query document granularity was not resolved".to_string()) + })?; + if let Some(existing) = state.granularity + && existing != document_granularity + { + return Err(Error::invalid_input( + "FTS queries cannot mix Row and ListElement document granularities".to_string(), + )); + } + state.granularity = Some(document_granularity); + + let Some(column) = column else { + if document_granularity.is_list_element() { + return Err(Error::invalid_input( + "ListElement FTS queries must explicitly specify a field path".to_string(), + )); + } + return Ok(()); + }; + let resolved = resolve_fts_field(schema, column, document_granularity)?; + if document_granularity.is_list_element() { + if let Some(existing) = state.list_element_field_id + && existing != resolved.final_field_id + { + return Err(Error::invalid_input( + "all leaves in a ListElement FTS query must use the same final field" + .to_string(), + )); + } + state.list_element_field_id = Some(resolved.final_field_id); + } + Ok(()) + } + + fn visit( + schema: &lance_core::datatypes::Schema, + query: &FtsQuery, + state: &mut TargetState, + ) -> Result<()> { + match query { + FtsQuery::Match(query) => add_leaf( + schema, + query.column.as_deref(), + query.document_granularity, + state, + ), + FtsQuery::Phrase(query) => add_leaf( + schema, + query.column.as_deref(), + query.document_granularity, + state, + ), + FtsQuery::Boost(query) => { + visit(schema, &query.positive, state)?; + visit(schema, &query.negative, state) + } + FtsQuery::Boolean(query) => { + for child in query + .must + .iter() + .chain(&query.should) + .chain(&query.must_not) + { + visit(schema, child, state)?; + } + Ok(()) + } + FtsQuery::MultiMatch(query) => { + for child in &query.match_queries { + if child + .document_granularity + .is_some_and(|value| value.is_list_element()) + { + return Err(Error::not_supported( + "MultiMatch does not support ListElement document granularity" + .to_string(), + )); + } + add_leaf( + schema, + child.column.as_deref(), + child.document_granularity, + state, + )?; + } + Ok(()) + } + } + } + + let mut state = TargetState::default(); + visit(self.dataset.schema(), query, &mut state)?; + state.granularity.ok_or_else(|| { + Error::invalid_input("FTS query must contain at least one leaf query".to_string()) + }) + } + + #[async_recursion] + async fn resolve_fts_query_document_granularity(&self, query: FtsQuery) -> Result { + match query { + FtsQuery::Match(mut query) => { + let column = query.column.as_deref().ok_or_else(|| { + Error::invalid_input("the column must be specified in the query".to_string()) + })?; + query.document_granularity = Some( + resolve_query_document_granularity( + self.dataset.as_ref(), + column, + query.document_granularity, + ) + .await?, + ); + Ok(FtsQuery::Match(query)) + } + FtsQuery::Phrase(mut query) => { + let column = query.column.as_deref().ok_or_else(|| { + Error::invalid_input("the column must be specified in the query".to_string()) + })?; + query.document_granularity = Some( + resolve_query_document_granularity( + self.dataset.as_ref(), + column, + query.document_granularity, + ) + .await?, + ); + Ok(FtsQuery::Phrase(query)) + } + FtsQuery::Boost(mut query) => { + query.positive = Box::new( + self.resolve_fts_query_document_granularity(*query.positive) + .await?, + ); + query.negative = Box::new( + self.resolve_fts_query_document_granularity(*query.negative) + .await?, + ); + Ok(FtsQuery::Boost(query)) + } + FtsQuery::Boolean(mut query) => { + let mut must = Vec::with_capacity(query.must.len()); + for child in query.must { + must.push(self.resolve_fts_query_document_granularity(child).await?); + } + let mut should = Vec::with_capacity(query.should.len()); + for child in query.should { + should.push(self.resolve_fts_query_document_granularity(child).await?); + } + let mut must_not = Vec::with_capacity(query.must_not.len()); + for child in query.must_not { + must_not.push(self.resolve_fts_query_document_granularity(child).await?); + } + query.must = must; + query.should = should; + query.must_not = must_not; + Ok(FtsQuery::Boolean(query)) + } + FtsQuery::MultiMatch(mut query) => { + for child in &mut query.match_queries { + let column = child.column.as_deref().ok_or_else(|| { + Error::invalid_input( + "the column must be specified in the query".to_string(), + ) + })?; + child.document_granularity = Some( + resolve_query_document_granularity( + self.dataset.as_ref(), + column, + child.document_granularity, + ) + .await?, + ); + } + Ok(FtsQuery::MultiMatch(query)) + } + } + } + + fn set_missing_query_granularity( + query: &mut FtsQuery, + document_granularity: DocumentGranularity, + ) { + match query { + FtsQuery::Match(query) => { + query + .document_granularity + .get_or_insert(document_granularity); + } + FtsQuery::Phrase(query) => { + query + .document_granularity + .get_or_insert(document_granularity); + } + FtsQuery::Boost(query) => { + Self::set_missing_query_granularity(&mut query.positive, document_granularity); + Self::set_missing_query_granularity(&mut query.negative, document_granularity); + } + FtsQuery::Boolean(query) => { + for child in query + .must + .iter_mut() + .chain(&mut query.should) + .chain(&mut query.must_not) + { + Self::set_missing_query_granularity(child, document_granularity); + } + } + FtsQuery::MultiMatch(query) => { + for child in &mut query.match_queries { + child + .document_granularity + .get_or_insert(document_granularity); + } + } + } + } + + fn query_requests_list_element(query: &FtsQuery) -> bool { + match query { + FtsQuery::Match(query) => query + .document_granularity + .is_some_and(DocumentGranularity::is_list_element), + FtsQuery::Phrase(query) => query + .document_granularity + .is_some_and(DocumentGranularity::is_list_element), + FtsQuery::Boost(query) => { + Self::query_requests_list_element(&query.positive) + || Self::query_requests_list_element(&query.negative) + } + FtsQuery::Boolean(query) => query + .must + .iter() + .chain(&query.should) + .chain(&query.must_not) + .any(Self::query_requests_list_element), + FtsQuery::MultiMatch(query) => query.match_queries.iter().any(|query| { + query + .document_granularity + .is_some_and(DocumentGranularity::is_list_element) + }), + } + } + + async fn resolve_full_text_search_query( + &self, + query: &FullTextSearchQuery, + ) -> Result { + let mut resolved = query.clone(); + normalize_fts_zero_boosts(&mut resolved.query); + if resolved.query.is_missing_column() { + if Self::query_requests_list_element(&resolved.query) { + return Err(Error::invalid_input( + "ListElement FTS queries must explicitly specify a field path".to_string(), + )); + } + let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; + resolved.query = fill_fts_query_column(&resolved.query, &indexed_columns, false)?; + Self::set_missing_query_granularity(&mut resolved.query, DocumentGranularity::Row); + } + apply_dataset_planner_auto_fuzziness_compatibility_gate(&mut resolved.query); + resolved.query = self + .resolve_fts_query_document_granularity(resolved.query) + .await?; + self.fts_document_granularity(&resolved.query)?; + Ok(resolved) + } + // Create an execution plan to do full text search async fn fts( &self, filter_plan: &ExprFilterPlan, query: &FullTextSearchQuery, ) -> Result> { - let columns = query.columns(); let mut params = query.params(); if params.limit.is_none() { let search_limit = match (self.limit, self.offset) { @@ -3256,14 +4280,8 @@ impl Scanner { }; params = params.with_limit(search_limit); } - let query = if columns.is_empty() { - // the field is not specified, - // try to search over all indexed fields including nested ones - let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; - fill_fts_query_column(&query.query, &indexed_columns, false)? - } else { - query.query.clone() - }; + let query = &query.query; + validate_fts_query_contract(query)?; // TODO: Could maybe walk the query here to find all the indices that will be // involved in the query to calculate a more accuarate required_fragments than @@ -3271,15 +4289,259 @@ impl Scanner { let prefilter_source = self .prefilter_source( filter_plan, - self.fragments_covered_by_fts_query(&query).await?, + self.fragments_covered_by_fts_query(query).await?, ) .await?; + // Data overlay masking blocks stale rows from indexed leaves and re-evaluates only those + // rows from their current values on the flat-text path. let fts_exec = self - .plan_fts(&query, ¶ms, filter_plan, &prefilter_source) + .plan_fts(query, ¶ms, filter_plan, &prefilter_source) .await?; Ok(fts_exec) } + async fn plan_compound_scorer( + &self, + query: &FtsQuery, + params: &FtsSearchParams, + filter_plan: &ExprFilterPlan, + prefilter_source: &PreFilterSource, + document_granularity: DocumentGranularity, + ) -> Result>> { + let columns = collect_fts_columns_in_order(query); + if columns.is_empty() { + return Ok(None); + } + let cross_column = columns.len() > 1; + if cross_column && params.limit.is_none() { + // Candidate-driven cross-column execution requires a bounded top-k + // collector. The existing DataFusion plan remains the exact path + // for callers that request the full result set. + return Ok(None); + } + + let target_fragments: &[Fragment] = self + .fragments + .as_deref() + .unwrap_or_else(|| self.dataset.fragments()); + if target_fragments.is_empty() { + return Ok(None); + } + let mut phrase_columns = HashSet::new(); + collect_phrase_columns(query, &mut phrase_columns); + // Query-local residual scoring intentionally reuses committed-index + // BM25 statistics. Matching remains exact for the supported leaf + // shapes, but ranking is approximate until the appended rows are + // incorporated into a persistent index. + let allow_indexed_stats_residual = !cross_column + && !self.fast_search + && self.fragments.is_none() + && filter_plan.is_empty() + && self.external_row_mask.is_none() + && params.limit.is_some() + && document_granularity == DocumentGranularity::Row + && target_fragments + .iter() + .all(|fragment| fragment.deletion_file.is_none()) + && supports_indexed_stats_residual_compound(query); + + let segment_groups = futures::future::try_join_all(columns.into_iter().map(|column| { + let phrase_columns = &phrase_columns; + async move { + let index = self + .dataset + .load_scalar_index( + IndexCriteria::default() + .for_column(&column) + .supports_fts() + .with_fts_document_granularity(document_granularity), + ) + .await?; + let Some(index) = index else { + return Ok(None); + }; + + let (unindexed_fragments, overlay_plan) = futures::future::try_join( + self.dataset.unindexed_fragments(&index.name), + self.fts_overlay_plan(&column, document_granularity, target_fragments), + ) + .await?; + let unindexed_fragments = self.retain_target_fragments(unindexed_fragments); + let has_bounded_residual = allow_indexed_stats_residual + && has_bounded_query_local_residual_rows(&unindexed_fragments); + if !unindexed_fragments.is_empty() + && (!self.fast_search || unindexed_fragments.len() == target_fragments.len()) + && !(has_bounded_residual + && unindexed_fragments.len() < target_fragments.len()) + { + // Flat and posting-backed leaves do not share a document + // domain, so preserve the exact fallback for partial index + // coverage. Fast search deliberately excludes unindexed + // fragments, so its indexed-only domain remains valid for + // the compound scorer when at least one target fragment is + // indexed. + return Ok(None); + } + let segments = match overlay_plan { + FtsOverlayPlan::Unchanged(Some(segments)) => segments, + FtsOverlayPlan::Unchanged(None) => { + load_segments(&self.dataset, &column, document_granularity) + .await? + .ok_or_else(|| { + Error::invalid_input(format!( + "No Inverted index found for column {column}" + )) + })? + } + FtsOverlayPlan::RowLevel { .. } | FtsOverlayPlan::FullScan => return Ok(None), + }; + if has_bounded_residual && !unindexed_fragments.is_empty() { + if !has_complete_hybrid_fts_coverage( + &segments, + &unindexed_fragments, + target_fragments, + ) { + return Ok(None); + } + if segments.is_empty() { + return Err(Error::internal( + "hybrid compound FTS requires one indexed segment", + )); + } + // Preserve the established semantic mismatch error before + // constructing query-local postings with the same tokenizer. + load_segment_details(&self.dataset, &column, &segments).await?; + } + + if cross_column { + let details = futures::future::try_join_all( + segments.iter().map(|segment| { + load_physical_fts_details(&self.dataset, &column, segment) + }), + ) + .await?; + if phrase_columns.contains(&column) + && details.iter().any(|details| !details.with_position) + { + return Err(Error::invalid_input( + "position is not found but required for phrase queries, try recreating the index with position" + .to_string(), + )); + } + let all_modern = details.iter().all(|details| { + matches!( + details.posting_format_version, + Some(INVERTED_INDEX_VERSION_V2 | INVERTED_INDEX_VERSION_V3) + ) + }); + if !all_modern { + return Ok(None); + } + } else if phrase_columns.contains(&column) { + let details = load_segment_details(&self.dataset, &column, &segments).await?; + if !details.with_position { + return Err(Error::invalid_input( + "position is not found but required for phrase queries, try recreating the index with position" + .to_string(), + )); + } + } + + Ok(Some((column, segments, unindexed_fragments))) + } + })) + .await?; + let Some(segment_groups) = segment_groups.into_iter().collect::>>() else { + return Ok(None); + }; + + if !cross_column { + let (column, segments, unindexed_fragments) = + segment_groups.into_iter().next().ok_or_else(|| { + Error::internal("compound scorer requires one column".to_string()) + })?; + if allow_indexed_stats_residual && !unindexed_fragments.is_empty() { + let resolved = + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let scan_projection = self + .dataset + .empty_projection() + .with_row_id() + .union_columns(&[scan_column], OnMissing::Error)?; + let PlannedFilteredScan { plan, .. } = self + .filtered_read( + &ExprFilterPlan::default(), + scan_projection, + /* make_deletions_null */ false, + Some(Arc::new(unindexed_fragments)), + None, + /* is_prefilter */ true, + None, + ) + .await?; + return Ok(Some(Arc::new(HybridCompoundQueryExec::new( + self.dataset.clone(), + query.clone(), + params.clone(), + column, + segments, + plan, + )))); + } + return Ok(Some(Arc::new( + CompoundQueryExec::new_with_segments( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segments, + ) + .with_external_mask(self.external_row_mask.clone()), + ))); + } + + let mut coverage_groups = segment_groups.iter(); + let Some((_, _, first_unindexed_fragments)) = coverage_groups.next() else { + return Ok(None); + }; + let first_unindexed_fragment_ids = first_unindexed_fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect::(); + if coverage_groups.any(|(_, _, unindexed_fragments)| { + unindexed_fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect::() + != first_unindexed_fragment_ids + }) { + // The cross-column scorer builds one shared prefilter. If column + // coverage differs, that prefilter's union can re-admit stale + // postings from a fragment invalidated only for another column. + // Keep the field-local fallback, which preserves each column's + // own index domain. + return Ok(None); + } + let segment_groups = segment_groups + .into_iter() + .map(|(column, segments, _)| (column, segments)) + .collect(); + let exec = CrossColumnCompoundQueryExec::new_with_segments( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segment_groups, + )? + .with_external_mask(self.external_row_mask.clone()); + Ok(Some(Arc::new(exec))) + } + async fn plan_fts( &self, query: &FtsQuery, @@ -3287,13 +4549,31 @@ impl Scanner { filter_plan: &ExprFilterPlan, prefilter_source: &PreFilterSource, ) -> Result> { + let document_granularity = self.fts_document_granularity(query)?; + if !document_granularity.is_list_element() + && supports_compound_scorer(query) + && let Some(plan) = self + .plan_compound_scorer( + query, + params, + filter_plan, + prefilter_source, + document_granularity, + ) + .await? + { + return Ok(plan); + } + + // Unsupported, unbounded, partial-index, and overlay-backed cross-column + // shapes retain the exact DataFusion fallback. let plan: Arc = match query { FtsQuery::Match(query) => { self.plan_match_query(query, params, filter_plan, prefilter_source) .await? } FtsQuery::Phrase(query) => { - self.plan_phrase_query(query, params, prefilter_source) + self.plan_phrase_query(query, params, filter_plan, prefilter_source) .await? } @@ -3325,15 +4605,53 @@ impl Scanner { } FtsQuery::MultiMatch(query) => { - let mut children = Vec::with_capacity(query.match_queries.len()); - for match_query in &query.match_queries { - let child = - self.plan_match_query(match_query, params, filter_plan, prefilter_source); - children.push(child); - } - let children = futures::future::try_join_all(children).await?; - - let schema = children[0].schema(); + // A top-level cross-column MultiMatch scores each field independently and takes + // the maximum score for each row. A field's bounded compound top-k is therefore + // sufficient to determine the global top-k independently of the other fields. + // Preserve that bounded plan for every eligible field, while planning only + // partial-index and overlay-backed fields through the exhaustive leaf fallback. + let unlimited_params = params.clone().with_limit(None); + let can_use_bounded_compound = + !document_granularity.is_list_element() && params.limit.is_some(); + let field_prefilter_sources = + prefilter_source.shared_for_multimatch_fields(query.match_queries.len()); + let children = futures::future::try_join_all( + query + .match_queries + .iter() + .zip(field_prefilter_sources.iter()) + .map(|(match_query, field_prefilter_source)| { + let unlimited_params = &unlimited_params; + async move { + if can_use_bounded_compound { + let child_query = FtsQuery::Match(match_query.clone()); + if let Some(plan) = self + .plan_compound_scorer( + &child_query, + params, + filter_plan, + field_prefilter_source, + document_granularity, + ) + .await? + { + return Ok(plan); + } + } + + self.plan_match_query( + match_query, + unlimited_params, + filter_plan, + field_prefilter_source, + ) + .await + } + }), + ) + .await?; + + let schema = children[0].schema(); let group_expr = vec![( expressions::col(ROW_ID, schema.as_ref())?, ROW_ID.to_string(), @@ -3361,18 +4679,26 @@ impl Scanner { fts_node, schema, )?); - let sort_expr = PhysicalSortExpr { - expr: expressions::col(SCORE_COL, fts_node.schema().as_ref())?, - options: SortOptions { - descending: true, - nulls_first: false, + let sort_exprs = [ + PhysicalSortExpr { + expr: expressions::col(SCORE_COL, fts_node.schema().as_ref())?, + options: SortOptions { + descending: true, + nulls_first: false, + }, }, - }; + PhysicalSortExpr { + expr: expressions::col(ROW_ID, fts_node.schema().as_ref())?, + options: SortOptions { + descending: false, + nulls_first: false, + }, + }, + ]; - Arc::new( - SortExec::new([sort_expr].into(), fts_node) - .with_fetch(self.limit.map(|l| l as usize)), - ) + // `params.limit` is the recursive planning contract. Compound + // parents pass `None` when they require every candidate. + Arc::new(SortExec::new(sort_exprs.into(), fts_node).with_fetch(params.limit)) } FtsQuery::Boolean(query) => { // TODO: rewrite the query for better performance @@ -3418,11 +4744,32 @@ impl Scanner { ); } - let should = build_boolean_query_children(BoolSlot::Should, should)? - .expect("Should slot always returns Some"); - let must = build_boolean_query_children(BoolSlot::Must, must)?; - let must_not = build_boolean_query_children(BoolSlot::MustNot, must_not)? - .expect("MustNot slot always returns Some"); + let boolean_schema = fts_schema(document_granularity); + let should = build_boolean_query_children_with_schema( + BoolSlot::Should, + should, + boolean_schema.clone(), + )? + .ok_or_else(|| { + Error::internal( + "boolean should planning returned no execution plan".to_string(), + ) + })?; + let must = build_boolean_query_children_with_schema( + BoolSlot::Must, + must, + boolean_schema.clone(), + )?; + let must_not = build_boolean_query_children_with_schema( + BoolSlot::MustNot, + must_not, + boolean_schema, + )? + .ok_or_else(|| { + Error::internal( + "boolean must-not planning returned no execution plan".to_string(), + ) + })?; if query.should.is_empty() && must.is_none() { return Err(Error::invalid_input( @@ -3447,31 +4794,164 @@ impl Scanner { &self, query: &PhraseQuery, params: &FtsSearchParams, + filter_plan: &ExprFilterPlan, prefilter_source: &PreFilterSource, ) -> Result> { let column = query.column.clone().ok_or(Error::invalid_input( "the column must be specified in the query".to_string(), ))?; + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::internal("FTS Phrase query granularity was not resolved".to_string()) + })?; + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let output_schema = fts_schema(document_granularity); + let index = self + .dataset + .load_scalar_index( + IndexCriteria::default() + .for_column(&column) + .supports_fts() + .with_fts_document_granularity(document_granularity), + ) + .await?; + let target_fragments: &[Fragment] = self + .fragments + .as_deref() + .unwrap_or_else(|| self.dataset.fragments()); + if self.fragments.as_ref().is_some_and(Vec::is_empty) { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_query = MatchQuery::new(query.terms.clone()) + .with_column(Some(column.clone())) + .with_operator(Operator::And) + .with_document_granularity(document_granularity); + let flat_params = params.clone().with_phrase_slop(Some(query.slop)); - let segments = load_segments(&self.dataset, &column) - .await? - .ok_or(Error::invalid_input(format!( - "No Inverted index found for column {}", - column - )))?; - let details = load_segment_details(&self.dataset, &column, &segments).await?; + let (phrase_plan, flat_phrase_plan) = match &index { + Some(index) => { + let unindexed_fragments = self + .retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?); + if !target_fragments.is_empty() + && unindexed_fragments.len() == target_fragments.len() + { + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_phrase_plan = self + .plan_flat_match_query( + unindexed_fragments, + HashMap::new(), + &flat_query, + &flat_params, + filter_plan, + None, + ) + .await?; + return Self::combine_fts_leaf_plans(None, Some(flat_phrase_plan), params); + } - if !details.with_position { - return Err(Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position" - .to_string())); - } + let (stale_rows, preset_segments) = match self + .fts_overlay_plan(&column, document_granularity, target_fragments) + .await? + { + FtsOverlayPlan::Unchanged(segments) => (HashMap::new(), segments), + FtsOverlayPlan::RowLevel { + stale_rows, + segments, + } => (stale_rows, Some(segments)), + FtsOverlayPlan::FullScan => { + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_phrase_plan = self + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + &flat_query, + &flat_params, + filter_plan, + None, + ) + .await?; + return Self::combine_fts_leaf_plans(None, Some(flat_phrase_plan), params); + } + }; + let overlay_block = self.stale_rows_block_mask(&stale_rows).await?; + let segments = match preset_segments { + Some(segments) => segments, + None => load_segments(&self.dataset, &column, document_granularity) + .await? + .ok_or_else(|| { + Error::internal(format!( + "FTS metadata routed column {column} without loadable segments" + )) + })?, + }; + let details = load_segment_details(&self.dataset, &column, &segments).await?; + if !details.with_position { + return Err(Error::invalid_input("position is not found but required for phrase queries, try recreating the index with position" + .to_string())); + } - Ok(Arc::new(PhraseQueryExec::new( - self.dataset.clone(), - query.clone(), - params.clone(), - prefilter_source.clone(), - ))) + let has_flat_path = !self.fast_search + && (!unindexed_fragments.is_empty() || !stale_rows.is_empty()); + let shared_scorer = (has_flat_path && document_granularity.is_list_element()) + .then(|| Arc::new(SharedFtsScorer::new())); + let mut phrase_exec = PhraseQueryExec::new_with_segments_and_document_granularity( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segments, + document_granularity, + ); + if let Some(overlay_block) = overlay_block { + phrase_exec = phrase_exec.with_overlay_block(overlay_block); + } + if let Some(shared_scorer) = &shared_scorer { + phrase_exec = phrase_exec.with_shared_scorer(shared_scorer.clone()); + } + phrase_exec = phrase_exec.with_external_mask(self.external_row_mask.clone()); + let phrase_plan = Some(Arc::new(phrase_exec) as Arc); + let flat_phrase_plan = if has_flat_path { + Some( + self.plan_flat_match_query( + unindexed_fragments, + stale_rows, + &flat_query, + &flat_params, + filter_plan, + shared_scorer, + ) + .await?, + ) + } else { + None + }; + (phrase_plan, flat_phrase_plan) + } + None => { + if target_fragments.is_empty() { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_phrase_plan = self + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + &flat_query, + &flat_params, + filter_plan, + None, + ) + .await?; + (None, Some(flat_phrase_plan)) + } + }; + + Self::combine_fts_leaf_plans(phrase_plan, flat_phrase_plan, params) } async fn plan_match_query( @@ -3488,96 +4968,195 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let output_schema = fts_schema(document_granularity); let index = self .dataset - .load_scalar_index(IndexCriteria::default().for_column(&column).supports_fts()) + .load_scalar_index( + IndexCriteria::default() + .for_column(&column) + .supports_fts() + .with_fts_document_granularity(document_granularity), + ) .await?; // Get target fragments - let target_fragments = self + let target_fragments: &[Fragment] = self .fragments - .clone() - .unwrap_or_else(|| self.dataset.fragments().to_vec()); + .as_deref() + .unwrap_or_else(|| self.dataset.fragments()); + if self.fragments.as_ref().is_some_and(Vec::is_empty) { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } let (match_plan, flat_match_plan) = match &index { Some(index) => { - // Get unindexed fragments and filter to target fragments let unindexed_fragments = self .retain_target_fragments(self.dataset.unindexed_fragments(&index.name).await?); - - // If all target fragments are unindexed, skip index entirely - if unindexed_fragments.len() == target_fragments.len() { + if !target_fragments.is_empty() + && unindexed_fragments.len() == target_fragments.len() + { if self.fast_search { - return Ok(Arc::new(EmptyExec::new(FTS_SCHEMA.clone()))); + return Ok(Arc::new(EmptyExec::new(output_schema))); } let flat_match_plan = self - .plan_flat_match_query(unindexed_fragments, query, params, filter_plan) + .plan_flat_match_query( + unindexed_fragments, + HashMap::new(), + query, + params, + filter_plan, + None, + ) .await?; - return Ok(flat_match_plan); + return Self::combine_fts_leaf_plans(None, Some(flat_match_plan), params); } - // Mixed case: use index + flat search for unindexed - let match_plan: Arc = Arc::new(MatchQueryExec::new( - self.dataset.clone(), - query.clone(), - params.clone(), - prefilter_source.clone(), - )); - - if self.fast_search || unindexed_fragments.is_empty() { - (Some(match_plan), None) - } else { - let flat_match_plan = self - .plan_flat_match_query(unindexed_fragments, query, params, filter_plan) - .await?; - (Some(match_plan), Some(flat_match_plan)) + let (stale_rows, preset_segments) = match self + .fts_overlay_plan(&column, document_granularity, target_fragments) + .await? + { + FtsOverlayPlan::Unchanged(segments) => (HashMap::new(), segments), + FtsOverlayPlan::RowLevel { + stale_rows, + segments, + } => (stale_rows, Some(segments)), + FtsOverlayPlan::FullScan => { + if self.fast_search { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } + let flat_match_plan = self + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + query, + params, + filter_plan, + None, + ) + .await?; + return Self::combine_fts_leaf_plans(None, Some(flat_match_plan), params); + } + }; + let overlay_block = self.stale_rows_block_mask(&stale_rows).await?; + let has_flat_path = !self.fast_search + && (!unindexed_fragments.is_empty() || !stale_rows.is_empty()); + let shared_scorer = (has_flat_path && document_granularity.is_list_element()) + .then(|| Arc::new(SharedFtsScorer::new())); + let mut match_exec = match preset_segments { + Some(segments) => MatchQueryExec::new_with_segments_and_document_granularity( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + segments, + document_granularity, + ), + None => MatchQueryExec::new_with_document_granularity( + self.dataset.clone(), + query.clone(), + params.clone(), + prefilter_source.clone(), + document_granularity, + ), + }; + if let Some(overlay_block) = overlay_block { + match_exec = match_exec.with_overlay_block(overlay_block); + } + if let Some(shared_scorer) = &shared_scorer { + match_exec = match_exec.with_shared_scorer(shared_scorer.clone()); } + match_exec = match_exec.with_external_mask(self.external_row_mask.clone()); + let match_plan = Some(Arc::new(match_exec) as Arc); + let flat_match_plan = if has_flat_path { + Some( + self.plan_flat_match_query( + unindexed_fragments, + stale_rows, + query, + params, + filter_plan, + shared_scorer, + ) + .await?, + ) + } else { + None + }; + (match_plan, flat_match_plan) } None => { + if target_fragments.is_empty() { + return Ok(Arc::new(EmptyExec::new(output_schema))); + } if self.fast_search { - return Ok(Arc::new(EmptyExec::new(FTS_SCHEMA.clone()))); + return Ok(Arc::new(EmptyExec::new(output_schema))); } // No index: flat search all target fragments let flat_match_plan = self - .plan_flat_match_query(target_fragments.clone(), query, params, filter_plan) + .plan_flat_match_query( + target_fragments.to_vec(), + HashMap::new(), + query, + params, + filter_plan, + None, + ) .await?; (None, Some(flat_match_plan)) } }; - // Combine plans - let plan = match (match_plan, flat_match_plan) { - (Some(match_plan), Some(flat_match_plan)) => { - let match_plan = UnionExec::try_new(vec![match_plan, flat_match_plan])?; - let match_plan = Arc::new(RepartitionExec::try_new( - match_plan, - Partitioning::RoundRobinBatch(1), - )?); - let sort_expr = PhysicalSortExpr { - expr: expressions::col(SCORE_COL, match_plan.schema().as_ref())?, - options: SortOptions { - descending: true, - nulls_first: false, - }, - }; - Arc::new(SortExec::new([sort_expr].into(), match_plan).with_fetch(params.limit)) + Self::combine_fts_leaf_plans(match_plan, flat_match_plan, params) + } + + fn combine_fts_leaf_plans( + indexed_plan: Option>, + flat_plan: Option>, + params: &FtsSearchParams, + ) -> Result> { + let plan = match (indexed_plan, flat_plan) { + (Some(indexed_plan), Some(flat_plan)) => { + UnionExec::try_new(vec![indexed_plan, flat_plan])? + } + (Some(indexed_plan), None) => return Ok(indexed_plan), + (None, Some(flat_plan)) if params.limit.is_none() => return Ok(flat_plan), + (None, Some(flat_plan)) => flat_plan, + (None, None) => { + return Err(Error::internal( + "FTS leaf planning produced neither an indexed nor a flat plan".to_string(), + )); } - (Some(match_plan), None) => match_plan, - (None, Some(flat_match_plan)) => flat_match_plan, - (None, None) => unreachable!(), }; - - Ok(plan) + let plan = Arc::new(RepartitionExec::try_new( + plan, + Partitioning::RoundRobinBatch(1), + )?); + let sort_expr = PhysicalSortExpr { + expr: expressions::col(SCORE_COL, plan.schema().as_ref())?, + options: SortOptions { + descending: true, + nulls_first: false, + }, + }; + Ok(Arc::new( + SortExec::new([sort_expr].into(), plan).with_fetch(params.limit), + )) } /// Plan match query on unindexed fragments async fn plan_flat_match_query( &self, fragments: Vec, + stale_rows: HashMap, query: &MatchQuery, params: &FtsSearchParams, filter_plan: &ExprFilterPlan, + shared_scorer: Option>, ) -> Result> { let column = query .column @@ -3586,10 +5165,28 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); - - let mut columns = vec![column]; - if let Some(refine_expr) = filter_plan.refine_expr.as_ref() { - columns.extend(Planner::column_names_in_expr(refine_expr)); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + let resolved = resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let document_column = if resolved.has_lists() { + VALUE_COLUMN_NAME.to_string() + } else { + resolved.canonical_path.clone() + }; + let mut columns = vec![scan_column.clone()]; + let filter_expr = if stale_rows.is_empty() { + filter_plan.refine_expr.as_ref() + } else { + filter_plan.full_expr.as_ref() + }; + if let Some(filter_expr) = filter_expr { + columns.extend(Planner::column_names_in_expr(filter_expr)); } let scan_projection = self .dataset @@ -3597,27 +5194,68 @@ impl Scanner { .with_row_id() .union_columns(&columns, OnMissing::Error)?; - let PlannedFilteredScan { mut plan, .. } = self - .filtered_read( - filter_plan, - scan_projection, - /*make_deletions_null=*/ false, - Some(Arc::new(fragments)), - None, - /*is_prefilter=*/ true, - ) - .await?; + let mut inputs = Vec::with_capacity(2); + if !fragments.is_empty() { + let PlannedFilteredScan { mut plan, .. } = self + .filtered_read( + filter_plan, + scan_projection.clone(), + /*make_deletions_null=*/ false, + Some(Arc::new(fragments)), + None, + /*is_prefilter=*/ true, + None, + ) + .await?; + if let Some(refine_expr) = filter_plan.refine_expr.as_ref() { + plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?); + } + inputs.push(plan); + } - if let Some(refine_expr) = filter_plan.refine_expr.as_ref() { - plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?); + if !stale_rows.is_empty() { + let mut plan = self.stale_rows_take(&stale_rows, scan_projection).await?; + if let Some(filter) = filter_plan.full_expr.as_ref() { + let planner = Planner::new(plan.schema()); + let filter = planner.optimize_expr(filter.clone())?; + plan = Arc::new(LanceFilterExec::try_new(filter, plan)?); + } + inputs.push(plan); } - let flat_match_plan = Arc::new(FlatMatchQueryExec::new( + let mut plan: Arc = match inputs.len() { + 0 => { + return Err(Error::internal( + "flat FTS input requires unindexed fragments or stale rows", + )); + } + 1 => inputs.pop().unwrap(), + _ => UnionExec::try_new(inputs)?, + }; + if resolved.has_lists() { + plan = Arc::new(FtsDocumentExec::new(plan, resolved.clone())); + } else { + plan = self.ensure_column_alias(plan, &document_column)?; + } + let mut flat_match_plan = FlatMatchQueryExec::new_with_document_granularity( self.dataset.clone(), query.clone(), params.clone(), plan, - )); + document_granularity, + document_column, + ); + if let Some(shared_scorer) = shared_scorer { + flat_match_plan = flat_match_plan.with_shared_scorer(shared_scorer); + } + let flat_match_plan: Arc = Arc::new(flat_match_plan); + // Unindexed fragments and stale rows never reach the index-side prefilter, + // so apply the external row-address mask to the flat FTS results here + // (mirrors the ANN flat branch). Applied before the caller's top-k so + // masked-out rows do not consume result slots. + if let Some(mask) = self.external_row_mask.clone() { + return Ok(Arc::new(RowAddrMaskFilterExec::new(flat_match_plan, mask))); + } Ok(flat_match_plan) } @@ -3666,7 +5304,7 @@ impl Scanner { if requested_index_segments .iter() - .any(|idx| !idx.fields.contains(&column_id)) + .any(|idx| idx.fields.first() != Some(&column_id)) { return Err(Error::invalid_input(format!( "with_index_segments contained a segment that does not belong to vector column '{}'", @@ -3718,7 +5356,16 @@ impl Scanner { None } } - } else if let Some(index) = indices.iter().find(|i| i.fields.contains(&column_id)) { + } + // An index can only answer a query on the column it is keyed on, which is + // always `fields[0]`. Not `contains`: `fields` also lists columns the index + // merely carries values for, which it cannot search. Not a boundary derived + // from `covering_fields` either -- that is computed from a field older + // writers drop, so it would widen to the carried columns exactly when the + // declaration is lost. + else if let Some(index) = indices.iter().find(|i| { + i.fields.first() == Some(&column_id) && crate::index::index_type_is_known(i) + }) { // Try to get metric type from index metadata first (fast path for newer indices) let index_metric = if let Some(metric) = crate::index::vector::details::metric_type_from_index_metadata(index) @@ -3771,7 +5418,16 @@ impl Scanner { if let Some((index_name, index_segments, index_metric)) = index_and_segments { if self.is_batch_nearest { - return self.batch_indexed_vector_search(filter_plan, &q).await; + validate_distance_type_for(index_metric, &element_type)?; + return self + .batch_indexed_vector_search( + filter_plan, + &q, + &index_name, + &index_segments, + index_metric, + ) + .await; } log::trace!("index found for vector search"); @@ -3784,9 +5440,24 @@ impl Scanner { "Refine factor cannot be zero".to_string(), )); } + // Mask data overlay files: compute which row addresses within each segment have + // been updated by a newer overlay so their ANN entries may be stale. + // These stale rows are blocked from ANN results via the prefilter and re-scored + // on the targeted flat path below — only the specific stale rows, not the whole + // fragment, so sparse overlays incur near-zero overhead. + let stale_rows = self.overlay_stale_vector_rows(&index_segments)?; + // Build a prefilter block mask for stale rows (empty = no-op fast path). + let overlay_block = self.stale_rows_block_mask(&stale_rows).await?; + let ann_node = match vector_type { - DataType::FixedSizeList(_, _) => self.ann(&q, &index_segments, filter_plan).await?, - DataType::List(_) => self.multivec_ann(&q, &index_segments, filter_plan).await?, + DataType::FixedSizeList(_, _) => { + self.ann(&q, &index_segments, filter_plan, overlay_block.clone()) + .await? + } + DataType::List(_) => { + self.multivec_ann(&q, &index_segments, filter_plan, overlay_block.clone()) + .await? + } _ => unreachable!(), }; @@ -3804,7 +5475,14 @@ impl Scanner { if !self.fast_search { knn_node = self - .knn_combined(&q, &index_name, &index_segments, knn_node, filter_plan) + .knn_combined( + &q, + &index_name, + &index_segments, + &stale_rows, + knn_node, + filter_plan, + ) .await?; } @@ -3843,21 +5521,192 @@ impl Scanner { self.fragments.clone().map(Arc::new), None, /*is_prefilter= */ true, + None, ) .await?; if let Some(refine_expr) = &filter_plan.refine_expr { plan = Arc::new(LanceFilterExec::try_new(refine_expr.clone(), plan)?); } + // The flat branch never reaches the index-side prefilter, so apply + // the external row-address mask here against the scanned _rowid. + if let Some(mask) = self.external_row_mask.clone() { + plan = Arc::new(RowAddrMaskFilterExec::new(plan, mask)); + } Ok(self.flat_knn(plan, &q)?) } } + /// Whether a batch (multi-query) vector search can use the shared-scan + /// indexed fast path ([`new_knn_batch_exec`]) instead of running one indexed + /// search per query vector. + /// + /// Requires all of: + /// - no refine step (the batch path does not yet rerank); + /// - fixed nprobes (`minimum_nprobes == maximum_nprobes`) — see below; + /// - every segment an IVF index with a flat-style sub-index (i.e. not HNSW); + /// - every target fragment covered by the *selected* `index_segments` (or + /// `fast_search`, which searches only the selected segments anyway). + /// + /// The fixed-nprobes requirement is a *correctness* gate, not just an + /// optimization. The shared-scan path searches exactly `minimum_nprobes` + /// partitions per query, but the single-query path is adaptive: it applies a + /// k-dependent `early_pruning` floor and then expands probes up to + /// `maximum_nprobes` (late search) when a query has fewer than `k` results. + /// When `minimum_nprobes == maximum_nprobes` neither adjustment can fire + /// (pruning is capped at the maximum, and the late-search range is empty), so + /// the batch result is provably identical to repeated single-query search. + /// With adaptive nprobes the two would diverge, so we fall back to the + /// per-query loop, which reuses the real adaptive search and stays exact. + /// + /// Extending the shared-scan path to adaptive nprobes (a batched early/late + /// search) is left as a follow-up. + async fn batch_index_search_supported( + &self, + index_name: &str, + index_segments: &[IndexMetadata], + q: &Query, + ) -> Result { + // Any refine factor sends the query onto a reranking path that the + // shared batch scan does not implement: the single-query path reranks + // with the original vectors even when the factor is 1, and rejects a + // factor of 0 outright (`Refine factor cannot be zero`). The batch path + // does neither, so fall back to the per-query loop for every `Some(_)`. + if q.refine_factor.is_some() { + return Ok(false); + } + // Only fixed nprobes is provably equivalent to single-query search; see + // the method docs. Adaptive nprobes falls back to the per-query loop. + if q.maximum_nprobes != Some(q.minimum_nprobes) { + return Ok(false); + } + // `nprobes(0)` is not rejected by the query builder, so `min == max == 0` + // slips past the fixed-nprobes check above. The single-query path probes + // nothing and returns an empty result, whereas the batch node would probe + // one partition's worth of neighbors — a silent divergence. Fall back so + // the per-query loop defines the semantics of `nprobes(0)`. + if q.minimum_nprobes == 0 { + return Ok(false); + } + // The per-query path threads a caller-supplied external row-address mask + // (`with_row_addr_prefilter`) into each query's prefilter via + // `with_external_mask`; the shared batch path builds one prefilter across + // the batch and does not carry that mask. Rather than silently returning + // masked-out rows, fall back to the per-query loop whenever a mask is set. + if self.external_row_mask.is_some() { + return Ok(false); + } + // Decide from the index metadata (no I/O) rather than opening the index + // to call `supports_batch_partition_search()`: this is a planning-time + // gate and the single-query path likewise avoids opening the index here. + // An IVF index with a flat-style sub-index (i.e. not HNSW) is exactly the + // set for which `supports_batch_partition_search()` is true; the exec + // re-checks that trait as a defensive invariant. Legacy segments without + // details fall back. + let all_ivf_flat_style = index_segments.iter().all(|index| { + index + .index_details + .as_ref() + .filter(|details| !details.value.is_empty()) + .map(|details| { + let index_type = + crate::index::vector::details::derive_vector_index_type(details); + index_type.starts_with("IVF_") && !index_type.contains("HNSW") + }) + .unwrap_or(false) + }); + if !all_ivf_flat_style { + return Ok(false); + } + // The batch node searches only the index's own entries; unlike the + // single-query path it does not reconcile data overlays (which block + // overlay-stale rows from the ANN result and re-score them on a flat + // take path — see `overlay_stale_vector_rows` in `vector_search`). If any + // indexed row was updated by a newer overlay, the batch path would return + // that row's stale index entry, so fall back to the per-query loop. This + // must precede the `fast_search` shortcut below because the single-query + // path applies the overlay block even in fast-search mode. Cheap in the + // common case: returns an empty map when no target fragment has overlays. + if !self.overlay_stale_vector_rows(index_segments)?.is_empty() { + return Ok(false); + } + if self.fast_search { + return Ok(true); + } + // The batch node only searches the selected `index_segments`, so any + // target fragment those segments do not cover would silently drop rows + // (the single-query path re-scores such fragments on a flat fallback in + // `knn_combined`). Measure coverage against the selected segments -- not + // the whole logical index -- so a subset selected via + // `with_index_segments` cannot hide a fragment that an unselected + // segment happens to cover; fall back whenever any remain. + let uncovered_fragments = self + .fragments_missing_from_index_segments(index_name, index_segments) + .await?; + Ok(uncovered_fragments.is_empty()) + } + + /// Target fragments the given `index_segments` do not cover. + /// + /// The ANN scan reads only the selected segments' partitions, so these are + /// exactly the fragments the single-query path re-scores on a flat fallback + /// in [`Self::knn_combined`]. Coverage is measured against the *selected* + /// segments rather than every segment of the logical index (which + /// `Dataset::unindexed_fragments` would do): a caller may select a subset + /// via [`with_index_segments`](Self::with_index_segments) while another, + /// unselected segment covers one of the requested fragments. + async fn fragments_missing_from_index_segments( + &self, + index_name: &str, + index_segments: &[IndexMetadata], + ) -> Result> { + if let Some(target_fragments) = &self.fragments { + let indexed_fragments = self.get_indexed_frags(index_segments); + Ok(target_fragments + .iter() + .filter(|fragment| !indexed_fragments.contains(fragment.id as u32)) + .cloned() + .collect()) + } else if self.index_segments.is_some() { + // An explicit segment selection with no fragment restriction searches + // exactly those segments; there is nothing to fall back for. + Ok(Vec::new()) + } else { + self.dataset.unindexed_fragments(index_name).await + } + } + async fn batch_indexed_vector_search( &self, filter_plan: &ExprFilterPlan, q: &Query, + index_name: &str, + index_segments: &[IndexMetadata], + index_metric: MetricType, ) -> Result> { + // Fast path: when every index segment is an IVF index with a flat-style + // sub-index (IVF_FLAT/PQ/SQ/RQ), search all query vectors in a single + // pass that reads each partition's storage once and shares the prefilter + // across the batch. HNSW, refine, and mixed indexed/unindexed scans fall + // back to the per-query loop below, which never regresses behavior. + if self + .batch_index_search_supported(index_name, index_segments, q) + .await? + { + let mut batch_query = q.clone(); + batch_query.metric_type = Some(index_metric); + let prefilter_source = self + .prefilter_source(filter_plan, self.get_indexed_frags(index_segments)) + .await?; + return new_knn_batch_exec( + self.dataset.clone(), + index_segments, + &batch_query, + self.nearest_query_count, + prefilter_source, + ); + } + let query_dim = q.key.len() / self.nearest_query_count; let mut query_plans = Vec::with_capacity(self.nearest_query_count); @@ -3939,46 +5788,49 @@ impl Scanner { q: &Query, index_name: &str, indexed_segments: &[IndexMetadata], + stale_rows: &HashMap, mut knn_node: Arc, filter_plan: &ExprFilterPlan, ) -> Result> { - let fallback_fragments = if let Some(target_fragments) = &self.fragments { - let indexed_fragments = self.get_indexed_frags(indexed_segments); - target_fragments - .iter() - .filter(|fragment| !indexed_fragments.contains(fragment.id as u32)) - .cloned() - .collect::>() - } else if self.index_segments.is_some() { - Vec::new() - } else { - self.dataset.unindexed_fragments(index_name).await? - }; + let fallback_fragments = self + .fragments_missing_from_index_segments(index_name, indexed_segments) + .await?; - if !fallback_fragments.is_empty() { - let q = q.clone(); - debug_assert!(q.metric_type.is_some()); + let has_fallback = !fallback_fragments.is_empty(); + let has_stale = !stale_rows.is_empty(); - // If the vector column is not present, we need to take the vector column, so - // that the distance value is comparable with the flat search ones. - if knn_node.schema().column_with_name(&q.column).is_none() { - let vector_projection = self - .dataset - .empty_projection() - .union_column(&q.column, OnMissing::Error) - .unwrap(); - knn_node = self.take(knn_node, vector_projection)?; - } + if !has_fallback && !has_stale { + return Ok(knn_node); + } - let mut columns = vec![q.column.clone()]; - if let Some(expr) = filter_plan.full_expr.as_ref() { - let filter_columns = Planner::column_names_in_expr(expr); - columns.extend(filter_columns); - } - let vector_scan_projection = Arc::new(self.dataset.schema().project(&columns).unwrap()); - // Note: we could try and use the scalar indices here to reduce the scope of this scan but the - // most common case is that fragments that are newer than the vector index are going to be newer - // than the scalar indices anyways + let q = q.clone(); + debug_assert!(q.metric_type.is_some()); + + // Ensure the vector column is present for distance computation. + if knn_node.schema().column_with_name(&q.column).is_none() { + let vector_projection = self + .dataset + .empty_projection() + .union_column(&q.column, OnMissing::Error)?; + knn_node = self.take(knn_node, vector_projection)?; + } + + let mut columns = vec![q.column.clone()]; + if let Some(expr) = filter_plan.full_expr.as_ref() { + let filter_columns = Planner::column_names_in_expr(expr); + columns.extend(filter_columns); + } + + // Collect flat-path plans; union order matches original (flat before ANN) so test snapshots + // and downstream plan analyses remain stable. + let mut flat_inputs: Vec> = Vec::new(); + + // Flat KNN for unindexed (new-data) fragments. + if has_fallback { + let vector_scan_projection = Arc::new(self.dataset.schema().project(&columns)?); + // Note: we could try and use the scalar indices here to reduce the scope of this scan + // but the most common case is that fragments newer than the vector index are also + // newer than the scalar indices. let mut scan_node = self.scan_fragments( true, false, @@ -3989,39 +5841,57 @@ impl Scanner { Arc::new(fallback_fragments), // Can't pushdown limit/offset in an ANN search None, - // We are re-ordering anyways, so no need to get data in data - // in a deterministic order. + // We are re-ordering anyways, so no need to get data in a deterministic order. false, ); - if let Some(expr) = filter_plan.full_expr.as_ref() { - // If there is a prefilter we need to manually apply it to the new data scan_node = Arc::new(LanceFilterExec::try_new(expr.clone(), scan_node)?); } - // first we do flat search on just the new data - let topk_appended = self.flat_knn(scan_node, &q)?; - - // To do a union, we need to make the schemas match. Right now - // knn_node: _distance, _rowid, vector - // topk_appended: vector, , _rowid, _distance - let topk_appended = project(topk_appended, knn_node.schema().as_ref())?; - assert!( - topk_appended - .schema() - .equivalent_names_and_types(&knn_node.schema()) - ); - // union - let unioned = UnionExec::try_new(vec![Arc::new(topk_appended), knn_node])?; - // Enforce only 1 partition. - let unioned = RepartitionExec::try_new( - unioned, - datafusion::physical_plan::Partitioning::RoundRobinBatch(1), - )?; - // then we do a flat search on KNN(new data) + ANN(indexed data) - return self.flat_knn(Arc::new(unioned), &q); + // Appended fragments are not covered by the index, so the external + // row-address mask must be applied to them here. + let scan_node = match self.external_row_mask.clone() { + Some(mask) => Arc::new(RowAddrMaskFilterExec::new(scan_node, mask)) as _, + None => scan_node, + }; + let topk_fallback = self.flat_knn(scan_node, &q)?; + let topk_fallback: Arc = + Arc::new(project(topk_fallback, knn_node.schema().as_ref())?); + flat_inputs.push(topk_fallback); + } + + // Flat KNN for stale rows only (row-level precision). + // Only specific row addresses need re-scoring, not the whole fragment, so sparse overlays + // incur near-zero overhead. + if has_stale { + // Fetch vector + filter columns for the stale rows. `flat_knn` sorts by row id, so the + // take must carry it (the fallback scan above gets it via `scan_fragments`). + let mut take_proj = self + .dataset + .empty_projection() + .with_row_id() + .union_column(&q.column, OnMissing::Error)?; + if let Some(expr) = filter_plan.full_expr.as_ref() { + let filter_columns = Planner::column_names_in_expr(expr); + take_proj = take_proj.union_columns(filter_columns, OnMissing::Error)?; + } + let mut stale_node = self.stale_rows_take(stale_rows, take_proj).await?; + if let Some(expr) = filter_plan.full_expr.as_ref() { + stale_node = Arc::new(LanceFilterExec::try_new(expr.clone(), stale_node)?); + } + let topk_stale = self.flat_knn(stale_node, &q)?; + let topk_stale: Arc = + Arc::new(project(topk_stale, knn_node.schema().as_ref())?); + flat_inputs.push(topk_stale); } - Ok(knn_node) + // Union: flat paths first (matching original order), then ANN results. + flat_inputs.push(knn_node); + let unioned = UnionExec::try_new(flat_inputs)?; + let unioned = RepartitionExec::try_new( + unioned, + datafusion::physical_plan::Partitioning::RoundRobinBatch(1), + )?; + self.flat_knn(Arc::new(unioned), &q) } #[async_recursion] @@ -4052,29 +5922,239 @@ impl Scanner { } } - /// Given an index query, split the fragments into two sets + /// Given an index query, split the fragments into two groups and collect per-row stale data. /// - /// The first set is the relevant fragments, which are covered by ALL indices in the query - /// The second set is the missing fragments, which are missed by at least one index + /// - `relevant_frags`: covered by ALL indices. Stale rows within them are returned separately + /// so callers can block them from `MaterializeIndexExec` and re-score via a targeted take. + /// - `missing_frags`: not covered by at least one index; fall back to full scan + filter. + /// - `stale_rows`: per-fragment row offsets whose indexed values are stale due to a data + /// overlay committed after the index was built (field-aware, version-gated). Empty when no + /// overlays are present. /// - /// There is no point in handling the case where a fragment is covered by some (but not all) - /// of the indices. If we have to do a full scan of the fragment then we do it + /// There is no point in partially indexing a fragment (some indices cover it, others do not). + /// If we have to do a full scan of a fragment for any reason, we do it entirely. async fn partition_frags_by_coverage( &self, index_expr: &ScalarIndexExpr, fragments: Arc>, - ) -> Result<(Vec, Vec)> { + ) -> Result<(Vec, Vec, HashMap)> { let covered_frags = self.fragments_covered_by_index_query(index_expr).await?; + let stale_rows = self + .overlay_stale_index_rows(index_expr, &fragments) + .await?; let mut relevant_frags = Vec::with_capacity(fragments.len()); let mut missing_frags = Vec::with_capacity(fragments.len()); for fragment in fragments.iter() { if covered_frags.contains(fragment.id as u32) { + // Indexed fragments stay on the indexed path. Stale rows within them are blocked + // from the index result and re-evaluated separately via a targeted take. relevant_frags.push(fragment.clone()); } else { missing_frags.push(fragment.clone()); } } - Ok((relevant_frags, missing_frags)) + Ok((relevant_frags, missing_frags, stale_rows)) + } + + /// Per-row stale offsets for each fragment whose indexed values may be stale because an + /// overlay committed *after* an index was built touches a field that index covers. + /// + /// The check is field-aware (an overlay touching only unindexed fields excludes nothing) and + /// version-gated (an overlay with `committed_version <= index.dataset_version` is already + /// incorporated by the index), via + /// [`lance_table::format::overlay::staleness::overlay_exclusion_offsets`]. + async fn overlay_stale_index_rows( + &self, + index_expr: &ScalarIndexExpr, + fragments: &[Fragment], + ) -> Result> { + // Overlays are rare; skip all index loading when none of the candidate fragments has one. + let overlaid_frags = overlaid_fragments(fragments); + if overlaid_frags.is_empty() { + return Ok(HashMap::new()); + } + + // Walk the (boolean) index expression tree to collect leaf searches. + let mut searches = Vec::new(); + let mut stack = vec![index_expr]; + while let Some(expr) = stack.pop() { + match expr { + ScalarIndexExpr::Not(inner) => stack.push(inner), + ScalarIndexExpr::And(lhs, rhs) | ScalarIndexExpr::Or(lhs, rhs) => { + stack.push(lhs); + stack.push(rhs); + } + ScalarIndexExpr::Query(search) => searches.push(search), + } + } + + // `load_named_scalar_segments` returns cached index metadata — no disk I/O on the hot + // path. Even without the cache, this code is only reached when at least one fragment has + // overlays (rare), so the per-leaf cost is acceptable. + let mut stale: HashMap = HashMap::new(); + for search in searches { + let segments = load_named_scalar_segments( + self.dataset.as_ref(), + &search.column, + &search.index_name, + ) + .await?; + for segment in &segments { + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut stale, + self.dataset.schema(), + )?; + } + } + Ok(stale) + } + + /// Compute per-row stale data for a vector index's segments. + /// + /// Returns a map from fragment_id to the set of row offsets within that fragment that are stale + /// (their vector values have been updated by a newer overlay since the index was built). An + /// empty map means no stale rows — the fast path where no masking is needed. + fn overlay_stale_vector_rows( + &self, + segments: &[IndexMetadata], + ) -> Result> { + // Scope to the query's target fragments (all dataset fragments if unscoped). + let dataset_frags = self.dataset.fragments(); + let fragments: &[Fragment] = match self.fragments.as_ref() { + Some(f) => f.as_slice(), + None => dataset_frags.as_slice(), + }; + let overlaid_frags = overlaid_fragments(fragments); + if overlaid_frags.is_empty() { + return Ok(HashMap::new()); + } + let mut stale: HashMap = HashMap::new(); + for segment in segments { + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut stale, + self.dataset.schema(), + )?; + } + Ok(stale) + } + + /// Plan FTS overlay handling at row granularity. + /// + /// Modern segments remain searchable while their overlay-stale rows are blocked and + /// re-evaluated from current values. A legacy segment without fragment coverage falls back + /// to a full target scan when a relevant overlay exists because its indexed row set is + /// unknown. + async fn fts_overlay_plan( + &self, + column: &str, + document_granularity: DocumentGranularity, + target_fragments: &[Fragment], + ) -> Result { + if target_fragments.iter().all(|f| f.overlays.is_empty()) { + return Ok(FtsOverlayPlan::Unchanged(None)); + } + + let Some(segments) = load_segments(&self.dataset, column, document_granularity).await? + else { + return Ok(FtsOverlayPlan::Unchanged(None)); + }; + + let overlaid_frags = overlaid_fragments(target_fragments); + let mut stale_rows = HashMap::new(); + for segment in &segments { + if segment.fragment_bitmap.is_none() { + let mut legacy_stale_rows = HashMap::new(); + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut legacy_stale_rows, + self.dataset.schema(), + )?; + if !legacy_stale_rows.is_empty() { + return Ok(FtsOverlayPlan::FullScan); + } + } else { + collect_overlay_stale_rows_for_segment( + segment, + &overlaid_frags, + &mut stale_rows, + self.dataset.schema(), + )?; + } + } + + if stale_rows.is_empty() { + Ok(FtsOverlayPlan::Unchanged(Some(segments))) + } else { + Ok(FtsOverlayPlan::RowLevel { + stale_rows, + segments, + }) + } + } + + /// Collect the stale rows into a [`RowAddrTreeMap`] in the domain the index results use. + /// + /// Index results are in the row-id domain (see `ScalarQuery::evaluate_nullable`), and a + /// physical row address equals its row id only when the dataset does not use stable row ids. + /// Under stable row ids the addresses are translated to their row ids so the result lines up + /// with the index output it is combined with (block mask) or taken against. + async fn stale_rows_in_id_domain( + &self, + stale_rows: &HashMap, + ) -> Result { + let mut tree_map = RowAddrTreeMap::new(); + for (&frag_id, offsets) in stale_rows { + tree_map.insert_bitmap(frag_id, offsets.clone()); + } + if self.dataset.manifest.uses_stable_row_ids() { + tree_map = translate_addr_treemap_to_row_ids(&self.dataset, &tree_map).await?; + } + Ok(tree_map) + } + + /// Build a block-list mask over stale rows, or `None` when there are none. + /// + /// The mask removes these rows from an index result so the index never emits them; they + /// are re-evaluated against their current (overlay-merged) values on a targeted take path + /// (see [`Self::stale_rows_take`]). + async fn stale_rows_block_mask( + &self, + stale_rows: &HashMap, + ) -> Result> { + if stale_rows.is_empty() { + return Ok(None); + } + let tree_map = self.stale_rows_in_id_domain(stale_rows).await?; + Ok(Some(RowAddrMask::from_block(tree_map))) + } + + /// Take the stale rows by physical address, projecting `projection`, to re-evaluate only + /// those rows (rather than their whole fragments) against their current overlay-merged values. + /// + /// The rows are identified by an address allow list routed through `FilteredReadExec`, not a + /// `_rowid` column (`_rowid` and row address diverge under stable row ids — see + /// [`Self::stale_rows_in_id_domain`]). + async fn stale_rows_take( + &self, + stale_rows: &HashMap, + projection: Projection, + ) -> Result> { + let take_id_map = self.stale_rows_in_id_domain(stale_rows).await?; + let index_input = self.row_ids_as_take_input(take_id_map)?; + let mut read_options = FilteredReadOptions::new(projection); + if let Some(fragments) = self.fragments.as_ref() { + read_options = read_options.with_fragments(Arc::new(fragments.clone())); + } + Ok(Arc::new(FilteredReadExec::try_new( + self.dataset.clone(), + read_options, + Some(index_input), + )?)) } // First perform a lookup in a scalar index for ids and then perform a take on the @@ -4096,16 +6176,24 @@ impl Scanner { let needs_recheck = index_expr.needs_recheck(); - // Figure out which fragments are covered by ALL indices - let (relevant_frags, missing_frags) = self + // Figure out which fragments are covered by ALL indices, and which rows within + // covered fragments are stale due to data overlay files. + let (relevant_frags, missing_frags, stale_rows) = self .partition_frags_by_coverage(index_expr, fragments) .await?; - let mut plan: Arc = Arc::new(MaterializeIndexExec::new( + // Build the MaterializeIndexExec, blocking stale row addresses so the index never + // emits them. Stale rows are re-scored separately via a targeted take below. + let mat_exec = MaterializeIndexExec::new( self.dataset.clone(), index_expr.clone(), Arc::new(relevant_frags), - )); + ); + let mat_exec = match self.stale_rows_block_mask(&stale_rows).await? { + Some(block) => mat_exec.with_overlay_block(block), + None => mat_exec, + }; + let mut plan: Arc = Arc::new(mat_exec); let refine_expr = filter_plan.refine_expr.as_ref(); @@ -4151,6 +6239,21 @@ impl Scanner { plan = Arc::new(AddRowAddrExec::try_new(plan, self.dataset.clone(), 0)?); } + // Both the missing-fragments path (full scan) and the stale-rows path (targeted take) + // need the user's projection extended with any filter columns. Compute it once. + let fallback_projection: Option = + if !missing_frags.is_empty() || !stale_rows.is_empty() { + let filter = filter_plan.full_expr.as_ref().expect_ok()?; + let filter_cols = Planner::column_names_in_expr(filter); + Some( + projection + .clone() + .union_columns(filter_cols, OnMissing::Error)?, + ) + } else { + None + }; + let new_data_path: Option> = if !missing_frags.is_empty() { log::trace!( "scalar_indexed_scan will need full scan of {} missing fragments", @@ -4169,10 +6272,8 @@ impl Scanner { // If there were no extra columns then we still need the project // because Materialize -> Take puts the row id at the left and // Scan puts the row id at the right - let filter = filter_plan.full_expr.as_ref().unwrap(); - let filter_cols = Planner::column_names_in_expr(filter); - let scan_projection = projection.union_columns(filter_cols, OnMissing::Error)?; - + let scan_projection = fallback_projection.clone().expect_ok()?; + let filter = filter_plan.full_expr.as_ref().expect_ok()?; let scan_schema = Arc::new(scan_projection.to_bare_schema()); let scan_arrow_schema = Arc::new(scan_schema.as_ref().into()); let planner = Planner::new(scan_arrow_schema); @@ -4201,16 +6302,37 @@ impl Scanner { None }; - if let Some(new_data_path) = new_data_path { - let unioned = UnionExec::try_new(vec![plan, new_data_path])?; - // Enforce only 1 partition. - let unioned = Arc::new(RepartitionExec::try_new( - unioned, - datafusion::physical_plan::Partitioning::RoundRobinBatch(1), - )?); - Ok(unioned) + // Stale-Take path: re-evaluate only the stale row addresses against the full filter + // (row-level optimization). These rows were blocked from the index result above; + // here we take their current (overlay-merged) values and re-apply the predicate. + // The schema matches `plan` via `project(…, plan.schema())`. + let stale_take_path: Option> = if stale_rows.is_empty() { + None } else { + let filter = filter_plan.full_expr.as_ref().expect_ok()?; + let take_projection = fallback_projection.expect_ok()?; + + let stale_node = self.stale_rows_take(&stale_rows, take_projection).await?; + + let planner = Planner::new(stale_node.schema()); + let optimized_filter = planner.optimize_expr(filter.clone())?; + let filtered = Arc::new(LanceFilterExec::try_new(optimized_filter, stale_node)?); + Some(Arc::new(project(filtered, plan.schema().as_ref())?)) + }; + + let extra_paths: Vec> = [new_data_path, stale_take_path] + .into_iter() + .flatten() + .collect(); + if extra_paths.is_empty() { Ok(plan) + } else { + let all_paths = std::iter::once(plan).chain(extra_paths).collect(); + let unioned = UnionExec::try_new(all_paths)?; + Ok(Arc::new(RepartitionExec::try_new( + unioned, + datafusion::physical_plan::Partitioning::RoundRobinBatch(1), + )?)) } } @@ -4276,6 +6398,7 @@ impl Scanner { batch_readahead: self.batch_readahead, fragment_readahead: self.fragment_readahead, io_buffer_size: self.get_io_buffer_size(), + materialization_readahead_bytes: self.materialization_readahead_bytes, with_row_id, with_row_address, with_row_last_updated_at_version, @@ -4328,8 +6451,8 @@ impl Scanner { )?)) } - /// Here we use a full text search as a post-filter. Any rows that - /// do not contain at least one query token are removed. + /// Here we use a full text search as a post-filter. Rows are retained + /// according to the match query's token operator. /// /// Only valid (currently) for match queries. async fn flat_fts_filter( @@ -4337,14 +6460,9 @@ impl Scanner { input: Arc, q: &FullTextSearchQuery, ) -> Result> { - let fts_query = if q.columns().is_empty() { - let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; - fill_fts_query_column(&q.query, &indexed_columns, false)? - } else { - q.query.clone() - }; + let fts_query = &q.query; - match &fts_query { + match fts_query { FtsQuery::Match(match_query) => { let schema = Arc::new((input.schema()).try_with_column(SCORE_FIELD.clone())?); @@ -4355,21 +6473,37 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); - let input = if schema.column_with_name(&column).is_none() { + let document_granularity = match_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + let resolved = + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let input = if schema.column_with_name(&scan_column).is_none() { let projection = self .dataset .empty_projection() - .union_column(&column, OnMissing::Error)?; - self.take(input, projection)? + .union_column(&scan_column, OnMissing::Error)?; + let input = self.take(input, projection)?; + if resolved.has_lists() { + input + } else { + self.ensure_column_alias(input, &scan_column)? + } } else { input }; - Ok(Arc::new(FlatMatchFilterExec::new( + Ok(Arc::new(FlatMatchFilterExec::new_with_resolved_field( input, self.dataset.clone(), match_query.clone(), q.params(), + resolved, ))) } _ => Err(Error::not_supported( @@ -4387,14 +6521,9 @@ impl Scanner { input: Arc, q: &FullTextSearchQuery, ) -> Result> { - let fts_query = if q.columns().is_empty() { - let indexed_columns = fts_indexed_columns(self.dataset.clone()).await?; - fill_fts_query_column(&q.query, &indexed_columns, false)? - } else { - q.query.clone() - }; + let fts_query = &q.query; - match &fts_query { + match fts_query { FtsQuery::Match(match_query) => { let schema = Arc::new((input.schema()).try_with_column(SCORE_FIELD.clone())?); @@ -4405,21 +6534,48 @@ impl Scanner { "the column must be specified in the query".to_string(), ))? .clone(); - let input = if schema.column_with_name(&column).is_none() { + let document_granularity = match_query.document_granularity.ok_or_else(|| { + Error::internal("FTS Match query granularity was not resolved".to_string()) + })?; + let resolved = + resolve_fts_field(self.dataset.schema(), &column, document_granularity)?; + let scan_column = if resolved.has_lists() { + resolved.root_column.clone() + } else { + resolved.canonical_path.clone() + }; + let document_column = if resolved.has_lists() { + VALUE_COLUMN_NAME.to_string() + } else { + resolved.canonical_path.clone() + }; + let input = if schema.column_with_name(&scan_column).is_none() { let projection = self .dataset .empty_projection() - .union_column(&column, OnMissing::Error)?; - self.take(input, projection)? + .union_column(&scan_column, OnMissing::Error)?; + let input = self.take(input, projection)?; + if resolved.has_lists() { + input + } else { + self.ensure_column_alias(input, &document_column)? + } + } else { + input + }; + let input = if resolved.has_lists() { + Arc::new(FtsDocumentExec::new(input, resolved)) as Arc } else { input }; - Ok(Arc::new(FlatMatchQueryExec::new( + Ok(Arc::new(FlatMatchQueryExec::new_with_document_granularity( self.dataset.clone(), match_query.clone(), q.params(), input, + document_granularity, + document_column, ))) } _ => { @@ -4642,11 +6798,19 @@ impl Scanner { q: &Query, index: &[IndexMetadata], filter_plan: &ExprFilterPlan, + overlay_block: Option, ) -> Result> { let prefilter_source = self .prefilter_source(filter_plan, self.get_indexed_frags(index)) .await?; - let inner_fanout_search = new_knn_exec(self.dataset.clone(), index, q, prefilter_source)?; + let inner_fanout_search = new_knn_exec( + self.dataset.clone(), + index, + q, + prefilter_source, + overlay_block, + self.external_row_mask.clone(), + )?; let sort_expr = PhysicalSortExpr { expr: expressions::col(DIST_COL, inner_fanout_search.schema().as_ref())?, options: SortOptions { @@ -4673,6 +6837,7 @@ impl Scanner { q: &Query, index: &[IndexMetadata], filter_plan: &ExprFilterPlan, + overlay_block: Option, ) -> Result> { // we split the query procedure into two steps: // 1. collect the candidates by vector searching on each query vector @@ -4705,6 +6870,8 @@ impl Scanner { index, &query, prefilter_source.clone(), + overlay_block.clone(), + self.external_row_mask.clone(), )?; let sort_expr = PhysicalSortExpr { expr: expressions::col(DIST_COL, ann_node.schema().as_ref())?, @@ -4785,11 +6952,14 @@ impl Scanner { // are not in the fragments we are scanning. if filter_plan.is_exact_index_search() && self.fragments.is_none() { let index_query = filter_plan.index_query.as_ref().expect_ok()?; - let (_, missing_frags) = self + let (_, missing_frags, stale_rows) = self .partition_frags_by_coverage(index_query, fragments.clone()) .await?; - if missing_frags.is_empty() || self.fast_search { + // Overlay-stale rows must never reach the direct ScalarIndexExec path: it would hand + // ANN/FTS a selection vector containing rows whose indexed values are now stale. When + // any exist, fall through to the filtered-read prefilter, which masks them. + if stale_rows.is_empty() && (missing_frags.is_empty() || self.fast_search) { log::trace!("prefilter entirely satisfied by exact index search"); let result_format = self.index_expr_result_format(); // We can only avoid materializing the index for a prefilter if: @@ -4818,17 +6988,103 @@ impl Scanner { Some(fragments), None, /*is_prefilter= */ true, + None, ) .await?; Ok(PreFilterSource::FilteredRowIds(plan)) } /// Take row indices produced by input plan from the dataset (with projection) + /// + /// Planned as a [`FilteredReadExec`] row-stream read; legacy (v1) storage + /// keeps using [`TakeExec`]. #[allow(deprecated)] fn take( &self, input: Arc, output_projection: Projection, + ) -> Result> { + let fields_to_take = output_projection + .clone() + .subtract_arrow_schema(input.schema().as_ref(), OnMissing::Ignore)?; + if !fields_to_take.has_data_fields() + && !fields_to_take.with_row_id + && !fields_to_take.with_row_addr + { + // No new columns needed + return Ok(input); + } + + versions::take( + self.dataset + .manifest() + .data_storage_format + .lance_file_format(), + self, + input, + output_projection, + ) + } + + pub(super) fn take_current( + &self, + input: Arc, + output_projection: Projection, + ) -> Result> { + let input_schema = input.schema(); + let has_row_id = input_schema.column_with_name(ROW_ID).is_some(); + let has_row_addr = input_schema.column_with_name(ROW_ADDR).is_some(); + if has_row_id || has_row_addr { + // Pass the full (un-subtracted) target so a rebuild against a + // different child re-derives what to fetch, and preserve carried + // identity columns (downstream nodes may key off them; the final + // ProjectionExec trims for free) + let mut projection = output_projection; + projection.with_row_id |= has_row_id; + projection.with_row_addr |= has_row_addr; + let mut read_options = FilteredReadOptions::new(projection); + if self.include_deleted_rows { + // Forwarded so the row-stream read rejects it: deleted rows + // carry a null row id, which the take would silently drop + read_options = read_options.with_deleted_rows()?; + } + if let Some(batch_size) = self.batch_size { + read_options = read_options.with_batch_size(validate_batch_size(batch_size)?); + } + if let Some(fragments) = &self.fragments { + read_options = read_options.with_fragments(Arc::new(fragments.clone())); + } + read_options = read_options.with_threading_mode( + FilteredReadThreadingMode::OnePartitionMultipleThreads(self.batch_readahead), + ); + if let Some(file_reader_options) = self.resolved_file_reader_options() { + read_options = read_options.with_file_reader_options(file_reader_options); + } + if let Some(fragment_readahead) = self.fragment_readahead { + read_options = read_options.with_fragment_readahead(fragment_readahead); + } + if let Some(io_buffer_size_bytes) = self.io_buffer_size { + read_options = read_options.with_io_buffer_size(io_buffer_size_bytes); + } + if let Some(materialization_readahead_bytes) = self.materialization_readahead_bytes { + read_options = read_options + .with_materialization_readahead_bytes(materialization_readahead_bytes); + } + return Ok(Arc::new(FilteredReadExec::try_new( + self.dataset.clone(), + read_options, + Some(input), + )?)); + } + + self.take_legacy(input, output_projection) + } + + #[allow(deprecated)] + pub(super) fn take_legacy( + &self, + input: Arc, + output_projection: Projection, ) -> Result> { let coalesced = Arc::new(CoalesceBatchesExec::new( input.clone(), @@ -4868,7 +7124,12 @@ impl Scanner { #[instrument(level = "info", skip(self))] pub async fn explain_plan(&self, verbose: bool) -> Result { - let plan = self.create_plan().await?; + // Box the plan-building future at the call site: `create_plan`'s inlined async + // layout otherwise exceeds rustc's depth limit here. It has to be boxed at the + // call site rather than inside `create_plan` — boxing internally turns the + // future's `Send` check into a `Box: Send` trait obligation that + // overflows the solver through the cache types (E0275 in downstream crates). + let plan = Box::pin(self.create_plan()).await?; let display = DisplayableExecutionPlan::new(plan.as_ref()); Ok(format!("{}", display.indent(verbose))) @@ -4896,47 +7157,42 @@ impl Scanner { } // Search over all indexed fields including nested ones, collecting columns that have an -// inverted index +// inverted index. Automatic discovery is intentionally restricted to Row +// granularity because ListElement queries require an explicit field path. async fn fts_indexed_columns(dataset: Arc) -> Result> { let mut indexed_columns = Vec::new(); - for field in dataset.schema().fields_pre_order() { - // Check if this field is a string type that could have an inverted index - let is_string_field = match field.data_type() { - DataType::Utf8 | DataType::LargeUtf8 => true, - DataType::List(inner_field) | DataType::LargeList(inner_field) => { - matches!( - inner_field.data_type(), - DataType::Utf8 | DataType::LargeUtf8 - ) - } - _ => false, + for index in dataset.load_indices().await?.iter() { + let Some(field_id) = index.fields.first().copied() else { + continue; }; - - if is_string_field { - // Build the full field path for nested fields - let column_path = - if let Some(ancestors) = dataset.schema().field_ancestry_by_id(field.id) { - let field_refs: Vec<&str> = ancestors.iter().map(|f| f.name.as_str()).collect(); - format_field_path(&field_refs) - } else { - continue; // Skip if we can't find the field ancestry - }; - - // Check if this field has an inverted index - let has_fts_index = dataset - .load_scalar_index( - IndexCriteria::default() - .for_column(&column_path) - .supports_fts(), - ) - .await? - .is_some(); - - if has_fts_index { - indexed_columns.push(column_path); - } + let Ok(preliminary_path) = dataset.schema().field_path(field_id) else { + continue; + }; + let details = + crate::index::scalar::fetch_index_details(dataset.as_ref(), &preliminary_path, index) + .await?; + if !details.type_url.ends_with("InvertedIndexDetails") { + continue; + } + let details = lance_index::pbold::InvertedIndexDetails::decode(details.value.as_slice()) + .map_err(|error| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {error}" + )) + })?; + if DocumentGranularity::try_from(details.document_granularity)? != DocumentGranularity::Row + { + continue; } + let resolved = crate::index::scalar::inverted::resolve_fts_field_by_id( + dataset.schema(), + field_id, + DocumentGranularity::Row, + )?; + indexed_columns.push(resolved.canonical_path); } + indexed_columns.sort(); + indexed_columns.dedup(); Ok(indexed_columns) } @@ -5009,8 +7265,10 @@ pub mod test_dataset { IndexType, scalar::{ScalarIndexParams, inverted::tokenizer::InvertedIndexParams}, vector::{ + hnsw::builder::HnswBuildParams, ivf::IvfBuildParams, kmeans::{KMeansParams, train_kmeans}, + sq::builder::SQBuildParams, }, }; use lance_linalg::distance::DistanceType; @@ -5112,7 +7370,30 @@ pub mod test_dataset { } pub async fn make_vector_index(&mut self) -> Result<()> { - let params = VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2); + self.make_vector_index_with_metric(MetricType::L2).await + } + + pub async fn make_vector_index_with_metric(&mut self, metric: MetricType) -> Result<()> { + let params = VectorIndexParams::ivf_pq(2, 8, 2, metric, 2); + self.dataset + .create_index( + &["vec"], + IndexType::Vector, + Some("idx".to_string()), + ¶ms, + true, + ) + .await?; + Ok(()) + } + + pub async fn make_ivf_hnsw_index(&mut self) -> Result<()> { + let params = VectorIndexParams::with_ivf_hnsw_sq_params( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams::default(), + SQBuildParams::default(), + ); self.dataset .create_index( &["vec"], @@ -5195,14 +7476,43 @@ pub mod test_dataset { Ok(()) } + fn fts_index_params() -> InvertedIndexParams { + // These scanner tests search for the token "s" (from the `s-{N}` + // column values) to exercise fragment/append coverage, and "s" is + // in the full English stop-word list. Keep the token searchable; + // stop-word behavior itself is covered by the tokenizer tests. + InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(false) + } + pub async fn make_fts_index(&mut self) -> Result<()> { - let params = InvertedIndexParams::default().with_position(true); + let params = Self::fts_index_params(); self.dataset .create_index(&["s"], IndexType::Inverted, None, ¶ms, true) .await?; Ok(()) } + pub async fn make_segmented_fts_index(&mut self) -> Result<()> { + let params = Self::fts_index_params(); + let fragments = self.dataset.get_fragments(); + let mut segments = Vec::with_capacity(fragments.len()); + for fragment in fragments { + let segment = self + .dataset + .create_index_builder(&["s"], IndexType::Inverted, ¶ms) + .name("s_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await?; + segments.push(segment); + } + self.dataset + .commit_existing_index_segments("s_idx", "s", segments) + .await + } + pub async fn append_new_data(&mut self) -> Result<()> { self.append_data_with_range(400, 410).await } @@ -5263,7 +7573,9 @@ mod test { }; use lance_file::version::LanceFileVersion; use lance_index::optimize::OptimizeOptions; - use lance_index::scalar::inverted::query::{MatchQuery, PhraseQuery}; + use lance_index::scalar::inverted::query::{ + BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Occur, PhraseQuery, + }; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::pq::PQBuildParams; @@ -5279,43 +7591,274 @@ mod test { use super::*; use crate::dataset::WriteMode; - use crate::dataset::WriteParams; use crate::dataset::optimize::{CompactionOptions, compact_files}; use crate::dataset::scanner::test_dataset::TestVectorDataset; + use crate::dataset::{NewColumnTransform, WriteParams}; use crate::index::vector::{StageParams, VectorIndexParams}; use crate::utils::test::{ DatagenExt, FragmentCount, FragmentRowCount, ThrottledStoreWrapper, assert_plan_node_equals, }; #[test] - fn test_env_var_parsing() { - // Test that invalid environment variable values don't panic - - // Test invalid LANCE_DEFAULT_BATCH_SIZE - unsafe { - std::env::set_var("LANCE_DEFAULT_BATCH_SIZE", "not_a_number"); - } - let result = get_default_batch_size(); - assert_eq!(result, None, "Should return None for invalid batch size"); + fn test_fts_query_contract_rejects_invalid_values() { + let negative_match = FtsQuery::Match(MatchQuery::new("hello".to_string()).with_boost(-1.0)); + let error = validate_fts_query_contract(&negative_match).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("finite and non-negative")); + + let empty_boolean = FtsQuery::Boolean(BooleanQuery::new(Vec::<(Occur, FtsQuery)>::new())); + let error = validate_fts_query_contract(&empty_boolean).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("at least one should/must query")); + + let infinite_boost = FtsQuery::Boost(BoostQuery::new( + MatchQuery::new("hello".to_string()).into(), + MatchQuery::new("world".to_string()).into(), + Some(f32::INFINITY), + )); + let error = validate_fts_query_contract(&infinite_boost).unwrap_err(); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!(error.to_string().contains("BoostQuery negative_boost")); + } - // Test valid LANCE_DEFAULT_BATCH_SIZE - unsafe { - std::env::set_var("LANCE_DEFAULT_BATCH_SIZE", "2048"); - } - let result = get_default_batch_size(); - assert_eq!(result, Some(2048), "Should parse valid batch size"); + #[test] + fn test_query_local_residual_row_bound() { + let fragment_with_rows = |id, physical_rows| { + let mut fragment = Fragment::new(id); + fragment.physical_rows = physical_rows; + fragment + }; - // Test unset LANCE_DEFAULT_BATCH_SIZE - unsafe { - std::env::remove_var("LANCE_DEFAULT_BATCH_SIZE"); - } - let result = get_default_batch_size(); - assert_eq!(result, None, "Should return None when env var is not set"); + assert!(has_bounded_query_local_residual_rows(&[ + fragment_with_rows(0, Some(40_000)), + fragment_with_rows(1, Some(60_000)), + ])); + assert!(!has_bounded_query_local_residual_rows(&[ + fragment_with_rows(0, Some(40_000)), + fragment_with_rows(1, Some(60_001)), + ])); + assert!(!has_bounded_query_local_residual_rows(&[ + fragment_with_rows(0, Some(1)), + fragment_with_rows(1, None), + ])); } #[test] - fn test_parse_env_var() { - // Test parse_env_var with different types to ensure full coverage + fn test_normalize_fts_zero_boosts_recurses_and_preserves_nonzero_values() { + fn boost_bits(query: &FtsQuery) -> Vec { + match query { + FtsQuery::Match(query) => vec![query.boost.to_bits()], + FtsQuery::Phrase(_) => Vec::new(), + FtsQuery::Boost(query) => std::iter::once(query.negative_boost.to_bits()) + .chain(boost_bits(&query.positive)) + .chain(boost_bits(&query.negative)) + .collect(), + FtsQuery::MultiMatch(query) => query + .match_queries + .iter() + .map(|query| query.boost.to_bits()) + .collect(), + FtsQuery::Boolean(query) => query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .flat_map(boost_bits) + .collect(), + } + } + + let match_query = + |terms: &str, boost| MatchQuery::new(terms.to_string()).with_boost(boost).into(); + let multi_match = MultiMatchQuery::try_new( + "needle".to_string(), + vec!["title".to_string(), "body".to_string()], + ) + .unwrap() + .try_with_boosts(vec![-0.0, 2.5]) + .unwrap(); + let negative = BooleanQuery::new([ + (Occur::Should, multi_match.into()), + (Occur::Must, match_query("required", 3.5)), + (Occur::MustNot, match_query("blocked", -0.0)), + ]); + let boost = BoostQuery::new(match_query("positive", -0.0), negative.into(), Some(-0.0)); + let mut query: FtsQuery = BooleanQuery::new([ + (Occur::Should, match_query("outer", -0.0)), + (Occur::Must, boost.into()), + (Occur::MustNot, match_query("unchanged", 4.5)), + ]) + .into(); + + let nz = (-0.0_f32).to_bits(); + let pz = 0.0_f32.to_bits(); + let b2 = 2.5_f32.to_bits(); + let b3 = 3.5_f32.to_bits(); + let b4 = 4.5_f32.to_bits(); + assert_eq!(boost_bits(&query), vec![nz, nz, nz, nz, b2, b3, nz, b4]); + normalize_fts_zero_boosts(&mut query); + assert_eq!(boost_bits(&query), vec![pz, pz, pz, pz, b2, b3, pz, b4]); + } + + #[test] + fn test_dataset_planner_defers_auto_fuzziness_recursively() { + fn collect_fuzziness(query: &FtsQuery, values: &mut Vec>) { + match query { + FtsQuery::Match(query) => values.push(query.fuzziness), + FtsQuery::Phrase(_) => {} + FtsQuery::Boost(query) => { + collect_fuzziness(&query.positive, values); + collect_fuzziness(&query.negative, values); + } + FtsQuery::MultiMatch(query) => { + values.extend(query.match_queries.iter().map(|query| query.fuzziness)); + } + FtsQuery::Boolean(query) => { + for child in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + collect_fuzziness(child, values); + } + } + } + } + + let auto_match = |terms: &str| { + MatchQuery::new(terms.to_owned()) + .with_fuzziness(None) + .into() + }; + let mut multi_match = MultiMatchQuery::try_new( + "multi".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap(); + multi_match.match_queries[0].fuzziness = None; + multi_match.match_queries[1].fuzziness = Some(1); + let boost = BoostQuery::new( + auto_match("positive"), + MatchQuery::new("negative".to_owned()) + .with_fuzziness(Some(0)) + .into(), + None, + ); + let mut query: FtsQuery = BooleanQuery::new([ + (Occur::Should, auto_match("root")), + (Occur::Must, FtsQuery::MultiMatch(multi_match)), + (Occur::MustNot, boost.into()), + ]) + .into(); + + apply_dataset_planner_auto_fuzziness_compatibility_gate(&mut query); + + let mut fuzziness = Vec::new(); + collect_fuzziness(&query, &mut fuzziness); + assert_eq!( + fuzziness, + [Some(0), Some(0), Some(1), Some(0), Some(0)], + "AUTO must become exact without changing explicit fuzzy or exact leaves" + ); + } + + #[test] + fn test_compound_scorer_shape_supports_cross_column_boolean_queries() { + let query = FtsQuery::Boolean(BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("alpha".to_string()) + .with_column(Some("title".to_string())) + .into(), + ), + ( + Occur::Must, + MatchQuery::new("beta".to_string()) + .with_column(Some("body".to_string())) + .into(), + ), + ( + Occur::MustNot, + MatchQuery::new("gamma".to_string()) + .with_column(Some("summary".to_string())) + .into(), + ), + ])); + + assert!(supports_compound_scorer(&query)); + assert_eq!( + collect_fts_columns_in_order(&query), + ["title", "body", "summary"] + ); + } + + #[test] + fn test_compound_scorer_leaves_top_level_cross_column_multi_match_on_existing_path() { + let single_column = FtsQuery::MultiMatch( + MultiMatchQuery::try_new("alpha".to_string(), vec!["title".to_string()]).unwrap(), + ); + let cross_column = FtsQuery::MultiMatch( + MultiMatchQuery::try_new( + "alpha".to_string(), + vec!["title".to_string(), "body".to_string()], + ) + .unwrap(), + ); + + assert!(supports_compound_scorer(&single_column)); + assert!(!supports_compound_scorer(&cross_column)); + } + + #[test] + fn test_collect_phrase_columns_traverses_prohibited_subtrees() { + let phrase = + PhraseQuery::new("exact phrase".to_string()).with_column(Some("body".to_string())); + let query = FtsQuery::Boolean(BooleanQuery::new([ + ( + Occur::Must, + MatchQuery::new("alpha".to_string()) + .with_column(Some("title".to_string())) + .into(), + ), + (Occur::MustNot, phrase.into()), + ])); + let mut columns = HashSet::new(); + + collect_phrase_columns(&query, &mut columns); + + assert_eq!(columns, HashSet::from(["body".to_string()])); + } + + #[test] + fn test_env_var_parsing() { + // Test that invalid environment variable values don't panic + + // Test invalid LANCE_DEFAULT_BATCH_SIZE + unsafe { + std::env::set_var("LANCE_DEFAULT_BATCH_SIZE", "not_a_number"); + } + let result = get_default_batch_size(); + assert_eq!(result, None, "Should return None for invalid batch size"); + + // Test valid LANCE_DEFAULT_BATCH_SIZE + unsafe { + std::env::set_var("LANCE_DEFAULT_BATCH_SIZE", "2048"); + } + let result = get_default_batch_size(); + assert_eq!(result, Some(2048), "Should parse valid batch size"); + + // Test unset LANCE_DEFAULT_BATCH_SIZE + unsafe { + std::env::remove_var("LANCE_DEFAULT_BATCH_SIZE"); + } + let result = get_default_batch_size(); + assert_eq!(result, None, "Should return None when env var is not set"); + } + + #[test] + fn test_parse_env_var() { + // Test parse_env_var with different types to ensure full coverage // Test with a unique env var name to avoid conflicts let test_var = "LANCE_TEST_PARSE_ENV_VAR_USIZE"; @@ -5472,151 +8015,699 @@ mod test { } } + fn batch_row_ids(batch: &RecordBatch) -> Vec { + batch + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::() + .values() + .to_vec() + } + + #[rstest] + #[case::without_stable_row_ids(false)] + #[case::with_stable_row_ids(true)] #[tokio::test] - async fn test_strict_batch_size() { - let dataset = lance_datagen::gen_batch() - .col("x", array::step::()) - .anon_col(array::step::()) - .into_ram_dataset(FragmentCount::from(7), FragmentRowCount::from(6)) + async fn row_addr_mask_plain_scan_allow_block_refine(#[case] stable_row_ids: bool) { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, stable_row_ids) .await .unwrap(); + let ds = &test_ds.dataset; - let mut scan = dataset.scan(); - scan.batch_size(10) - .strict_batch_size(true) - .filter("x % 2 == 0") - .unwrap(); - - let batches = scan - .try_into_stream() - .await + let mut scan = ds.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + let all_set: BTreeSet = all_ids.iter().copied().collect(); + let allow: Vec = all_ids.iter().copied().step_by(2).collect(); + let allow_set: BTreeSet = allow.iter().copied().collect(); + + // Allow-mask plain scan returns exactly the allowed rows. + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allow.iter().copied(), + ))); + scan.with_row_id(); + let got: BTreeSet = batch_row_ids(&scan.try_into_batch().await.unwrap()) + .into_iter() + .collect(); + assert_eq!(got, allow_set); + + // Block-mask plain scan returns every row except the blocked ones, which + // also exercises FilteredReadExec index-input serialization of a BlockList. + let block: Vec = all_ids.iter().copied().step_by(3).collect(); + let block_set: BTreeSet = block.iter().copied().collect(); + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter( + block.iter().copied(), + ))); + scan.with_row_id(); + let got: BTreeSet = batch_row_ids(&scan.try_into_batch().await.unwrap()) + .into_iter() + .collect(); + let expected: BTreeSet = all_set.difference(&block_set).copied().collect(); + assert_eq!(got, expected); + + // With a SQL refine, the result is the allowed rows that also match the filter. + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allow.iter().copied(), + ))); + scan.filter("i >= 200").unwrap(); + scan.project(&["i"]).unwrap(); + scan.with_row_id(); + let refined = scan.try_into_batch().await.unwrap(); + let refined_ids: BTreeSet = batch_row_ids(&refined).into_iter().collect(); + assert!(refined_ids.is_subset(&allow_set) && !refined_ids.is_empty()); + let is = refined + .column_by_name("i") .unwrap() - .try_collect::>() + .as_primitive::(); + assert!(is.values().iter().all(|v| *v >= 200)); + } + + #[tokio::test] + async fn row_addr_mask_plain_scan_rejected_on_legacy() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Legacy, false) .await .unwrap(); - - let batch_sizes = batches.iter().map(|b| b.num_rows()).collect::>(); - assert_eq!(batch_sizes, vec![10, 10, 1]); + let ds = &test_ds.dataset; + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([0u64]))); + let Err(err) = scan.try_into_stream().await else { + panic!("expected legacy-storage masked plain scan to be rejected"); + }; + assert!( + err.to_string().contains("legacy-storage"), + "unexpected: {err}" + ); } + #[rstest] + #[case::without_stable_row_ids(false)] + #[case::with_stable_row_ids(true)] #[tokio::test] - async fn test_column_not_exist() { - let dataset = lance_datagen::gen_batch() - .col("x", array::step::()) - .into_ram_dataset(FragmentCount::from(7), FragmentRowCount::from(6)) + async fn row_addr_mask_ann_search_only_allowed(#[case] stable_row_ids: bool) { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, stable_row_ids) .await .unwrap(); + test_ds.make_vector_index().await.unwrap(); + // Append after indexing so the appended fragment is unindexed (flat branch). + test_ds.append_new_data().await.unwrap(); + let ds = &test_ds.dataset; - let check_err_msg = |r: Result| { - let Err(err) = r else { - panic!( - "Expected an error to be raised saying column y is not found but got no error" - ) - }; - + let mut scan = ds.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + let allow: Vec = all_ids.iter().copied().step_by(3).collect(); + let allow_set: BTreeSet = allow.iter().copied().collect(); + + let key: Float32Array = (0..32).map(|v| v as f32).collect(); + let mut scan = ds.scan(); + scan.nearest("vec", &key, 15).unwrap(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allow.iter().copied(), + ))); + scan.with_row_id(); + let got = batch_row_ids(&scan.try_into_batch().await.unwrap()); + assert!(!got.is_empty()); + for id in got { assert!( - err.to_string().contains("No field named y"), - "Expected error to contain 'No field named y' but got {}", - err + allow_set.contains(&id), + "returned _rowid {id} not in allowlist" ); - }; - - let mut scan = dataset.scan(); - scan.project(&["x", "y"]).unwrap(); - check_err_msg(scan.try_into_stream().await); - - let mut scan = dataset.scan(); - scan.project(&["y"]).unwrap(); - check_err_msg(scan.try_into_stream().await); - - // This represents a query like `SELECT 1 AS foo` which we could _technically_ satisfy - // but it is not supported today - let mut scan = dataset.scan(); - scan.project_with_transform(&[("foo", "1")]).unwrap(); - match scan.try_into_stream().await { - Ok(_) => panic!("Expected an error to be raised saying not supported"), - Err(e) => { - assert!( - e.to_string().contains("Received only dynamic expressions"), - "Expected error to contain 'Received only dynamic expressions' but got {}", - e - ); - } } } - #[cfg(not(windows))] #[tokio::test] - async fn test_local_object_store() { - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, true), - ArrowField::new("s", DataType::Utf8, true), - ])); + async fn row_addr_mask_plain_scan_with_limit() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) + .await + .unwrap(); + let ds = &test_ds.dataset; - let batches: Vec = (0..5) - .map(|i| { - RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20)), - Arc::new(StringArray::from_iter_values( - (i * 20..(i + 1) * 20).map(|v| format!("s-{}", v)), - )), - ], - ) - .unwrap() - }) - .collect(); + let mut scan = ds.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + let allow: Vec = all_ids.iter().copied().step_by(2).collect(); + let allow_set: BTreeSet = allow.iter().copied().collect(); + + // limit must apply AFTER masking: 5 rows, all from the allowlist. + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allow.iter().copied(), + ))); + scan.limit(Some(5), None).unwrap(); + scan.with_row_id(); + let got = batch_row_ids(&scan.try_into_batch().await.unwrap()); + assert_eq!(got.len(), 5, "masked limit should yield 5 masked rows"); + for id in &got { + assert!(allow_set.contains(id), "returned {id} not allowed"); + } + } - let test_dir = TempStrDir::default(); - let test_uri = &test_dir; - let write_params = WriteParams { - max_rows_per_file: 40, - max_rows_per_group: 10, - ..Default::default() - }; - let batches = RecordBatchIterator::new(batches.clone().into_iter().map(Ok), schema.clone()); - Dataset::write(batches, test_uri, Some(write_params)) + #[tokio::test] + async fn row_addr_mask_plain_scan_filter_unprojected_column() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) .await .unwrap(); + let ds = &test_ds.dataset; - let dataset = Dataset::open(&format!("file-object-store://{}", test_uri)) + let mut scan = ds.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + + // Allow everything; filter on `i` but project only `s` (unrelated column). + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + all_ids.iter().copied(), + ))); + scan.filter("i >= 200").unwrap(); + scan.project(&["s"]).unwrap(); + let out = scan.try_into_batch().await.unwrap(); + assert_eq!(out.num_rows(), 200, "expected 200 rows with i>=200"); + } + + #[tokio::test] + async fn row_addr_mask_plain_scan_exact_index_filter_unprojected_column() { + // A scalar index on `i` turns `i >= 200` into an exact index query with no + // refine. Under an external mask that predicate is demoted to a refine over + // the masked rows, so `i` must still be projected for the read even though + // the user only asked for `s`. + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) .await .unwrap(); - let mut builder = dataset.scan(); - builder.batch_size(8); - let mut stream = builder.try_into_stream().await.unwrap(); - let mut rows_read = 0; - while let Some(next) = stream.next().await { - let next = next.unwrap(); - let expected = 8.min(100 - rows_read); - assert_eq!(next.num_rows(), expected); - rows_read += next.num_rows(); - } + test_ds.make_scalar_index().await.unwrap(); + let ds = &test_ds.dataset; + + let mut scan = ds.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + + let mut scan = ds.scan(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + all_ids.iter().copied(), + ))); + scan.filter("i >= 200").unwrap(); + scan.project(&["s"]).unwrap(); + let out = scan.try_into_batch().await.unwrap(); + assert_eq!(out.num_rows(), 200, "expected 200 rows with i>=200"); } + /// A `_rowid` predicate is recognized as a TakeOperation and short-circuits + /// straight to `take_source`, which used to skip the mask entirely. #[tokio::test] - async fn test_filter_parsing() -> Result<()> { - let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false).await?; - let dataset = &test_ds.dataset; + async fn row_addr_mask_take_shortcut_respects_mask() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) + .await + .unwrap(); + let ds = &test_ds.dataset; - let mut scan = dataset.scan(); - assert!(scan.filter.is_none()); + let mut scan = ds.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + let target = all_ids[0]; - scan.filter("i > 50")?; - assert_eq!(scan.get_expr_filter().unwrap(), Some(col("i").gt(lit(50)))); + // Sanity: unmasked, the shortcut returns the row. + let mut scan = ds.scan(); + scan.with_row_id(); + scan.filter(&format!("_rowid = {target}")).unwrap(); + assert_eq!(scan.try_into_batch().await.unwrap().num_rows(), 1); - for use_stats in [false, true] { - let batches = scan - .project(&["s"])? - .use_stats(use_stats) - .try_into_stream() - .await? - .try_collect::>() - .await?; - let batch = concat_batches(&batches[0].schema(), &batches)?; + // Masked to nothing, it must return nothing. + let mut scan = ds.scan(); + scan.with_row_id(); + scan.filter(&format!("_rowid = {target}")).unwrap(); + scan.with_row_addr_prefilter(RowAddrMask::allow_nothing()); + assert_eq!( + scan.try_into_batch().await.unwrap().num_rows(), + 0, + "the take shortcut must not return rows the mask excludes" + ); - let expected_batch = RecordBatch::try_new( + // And an allow-list restricts it rather than being ignored. + let mut scan = ds.scan(); + scan.with_row_id(); + scan.filter(&format!("_rowid = {target}")).unwrap(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([ + target, + ]))); + assert_eq!(scan.try_into_batch().await.unwrap().num_rows(), 1); + } + + /// A same-column compound query (Boost here) is optimized into + /// CompoundFtsScorer, a scorer that built its prefilter without the mask. + #[tokio::test] + async fn row_addr_mask_compound_fts_respects_mask() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) + .await + .unwrap(); + test_ds.make_fts_index().await.unwrap(); + let ds = &test_ds.dataset; + + let compound = || { + let positive = MatchQuery::new("4".to_owned()).with_column(Some("s".to_owned())); + let negative = MatchQuery::new("9".to_owned()).with_column(Some("s".to_owned())); + FullTextSearchQuery::new_query( + BoostQuery::new(positive.into(), negative.into(), Some(1.0)).into(), + ) + }; + + let mut scan = ds.scan(); + scan.full_text_search(compound()).unwrap(); + scan.with_row_id(); + let plan = scan.explain_plan(true).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "expected the compound scorer path, got:\n{plan}" + ); + let base = batch_row_ids(&scan.try_into_batch().await.unwrap()); + assert!(!base.is_empty(), "compound query matched nothing"); + + let mut scan = ds.scan(); + scan.full_text_search(compound()).unwrap(); + scan.with_row_id(); + scan.with_row_addr_prefilter(RowAddrMask::allow_nothing()); + assert_eq!( + scan.try_into_batch().await.unwrap().num_rows(), + 0, + "the compound scorer must not return rows the mask excludes" + ); + + // Allow exactly one baseline hit; only that one may come back. + let keep = base[0]; + let mut scan = ds.scan(); + scan.full_text_search(compound()).unwrap(); + scan.with_row_id(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([keep]))); + assert_eq!( + batch_row_ids(&scan.try_into_batch().await.unwrap()), + vec![keep] + ); + } + + /// A cross-column boolean query plans into CrossColumnCompoundFtsScorer, + /// which is a different exec from the same-column CompoundFtsScorer and + /// builds its own prefilter, so it needs the mask threaded separately. + #[tokio::test] + async fn row_addr_mask_cross_column_fts_respects_mask() { + use lance_index::scalar::inverted::tokenizer::InvertedIndexParams; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("title", DataType::Utf8, true), + ArrowField::new("body", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from_iter_values( + (0..64).map(|v| format!("alpha title {v}")), + )), + Arc::new(StringArray::from_iter_values( + (0..64).map(|v| format!("alpha body {v}")), + )), + ], + ) + .unwrap(); + + let path = TempStrDir::default(); + let reader = RecordBatchIterator::new([Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, &path, None).await.unwrap(); + let params = InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(false); + for column in ["title", "body"] { + dataset + .create_index(&[column], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + } + + // Two leaves on different columns is what selects the cross-column + // scorer; a bounded limit is required by that exec. + let cross_column = || { + FullTextSearchQuery::new_query(FtsQuery::Boolean(BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("title".to_string()) + .with_column(Some("title".to_string())) + .into(), + ), + ( + Occur::Should, + MatchQuery::new("body".to_string()) + .with_column(Some("body".to_string())) + .into(), + ), + ]))) + .limit(Some(10)) + }; + + let mut scan = dataset.scan(); + scan.full_text_search(cross_column()).unwrap(); + scan.with_row_id(); + let plan = scan.explain_plan(true).await.unwrap(); + assert!( + plan.contains("CrossColumnCompoundFtsScorer"), + "expected the cross-column compound scorer path, got:\n{plan}" + ); + let base = batch_row_ids(&scan.try_into_batch().await.unwrap()); + assert!(!base.is_empty(), "cross-column query matched nothing"); + + let mut scan = dataset.scan(); + scan.full_text_search(cross_column()).unwrap(); + scan.with_row_id(); + scan.with_row_addr_prefilter(RowAddrMask::allow_nothing()); + assert_eq!( + scan.try_into_batch().await.unwrap().num_rows(), + 0, + "the cross-column scorer must not return rows the mask excludes" + ); + + let keep = base[0]; + let mut scan = dataset.scan(); + scan.full_text_search(cross_column()).unwrap(); + scan.with_row_id(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([keep]))); + assert_eq!( + batch_row_ids(&scan.try_into_batch().await.unwrap()), + vec![keep] + ); + } + + #[tokio::test] + async fn row_addr_mask_fts_search_only_allowed() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) + .await + .unwrap(); + test_ds.make_fts_index().await.unwrap(); + // Re-append the low-i rows AFTER indexing so token "4" matches both an + // indexed row (index prefilter path) and an unindexed one (flat FTS branch). + test_ds.append_data_with_range(0, 10).await.unwrap(); + let ds = &test_ds.dataset; + + // Baseline: the row ids an unmasked FTS query matches. + let mut scan = ds.scan(); + scan.full_text_search(FullTextSearchQuery::new("4".into())) + .unwrap(); + scan.with_row_id(); + let base_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + let base_set: BTreeSet = base_ids.iter().copied().collect(); + assert!( + base_ids.len() >= 2, + "expected indexed + unindexed matches for token 4, got {base_ids:?}" + ); + + // Allow only every other matching row; the mask must prefilter BM25 so the + // result is exactly the allowed subset of the baseline matches. + let allow: Vec = base_ids.iter().copied().step_by(2).collect(); + let allow_set: BTreeSet = allow.iter().copied().collect(); + + let mut scan = ds.scan(); + scan.full_text_search(FullTextSearchQuery::new("4".into())) + .unwrap(); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allow.iter().copied(), + ))); + scan.with_row_id(); + let got: BTreeSet = batch_row_ids(&scan.try_into_batch().await.unwrap()) + .into_iter() + .collect(); + let expected: BTreeSet = base_set.intersection(&allow_set).copied().collect(); + assert_eq!(got, expected, "masked FTS must return allowed matches only"); + assert!(!got.is_empty()); + + // Block every match -> empty, proving the mask actually filters FTS results + // on both the indexed and flat branches. + let mut scan = ds.scan(); + scan.full_text_search(FullTextSearchQuery::new("4".into())) + .unwrap(); + scan.with_row_addr_prefilter(RowAddrMask::from_block(RowAddrTreeMap::from_iter( + base_ids.iter().copied(), + ))); + scan.with_row_id(); + let blocked = batch_row_ids(&scan.try_into_batch().await.unwrap()); + assert!(blocked.is_empty(), "block-mask must drop all FTS matches"); + } + + #[tokio::test] + async fn test_batch_size_bytes_across_data_files() { + let num_rows = 300; + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(0..num_rows))], + ) + .unwrap(); + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + max_rows_per_file: num_rows as usize + 1, + ..Default::default() + }), + ) + .await + .unwrap(); + + let wide_value = "abcdefghij".repeat(6); + let wide_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "wide", + DataType::Utf8, + false, + )])); + let wide_batch = RecordBatch::try_new( + wide_schema.clone(), + vec![Arc::new(StringArray::from_iter_values( + (0..num_rows).map(|_| wide_value.as_str()), + ))], + ) + .unwrap(); + dataset + .add_columns( + NewColumnTransform::Reader(Box::new(RecordBatchIterator::new( + [Ok(wide_batch)], + wide_schema, + ))), + None, + None, + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragment(0).unwrap().num_data_files(), 2); + + let target_bytes = 8 * 1024; + let mut scan = dataset.scan(); + scan.project(&["id", "wide"]) + .unwrap() + .batch_size_bytes(target_bytes); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + num_rows as usize + ); + for batch in &batches { + let wide = batch["wide"].as_string::(); + let values_bytes = (*wide.value_offsets().last().unwrap() + - *wide.value_offsets().first().unwrap()) as usize; + let logical_bytes = batch.num_rows() * std::mem::size_of::() + + std::mem::size_of_val(wide.value_offsets()) + + values_bytes; + assert!( + logical_bytes <= target_bytes as usize, + "batch has {logical_bytes} logical bytes, target is {target_bytes}" + ); + } + + let mut scan = dataset.scan(); + scan.project(&["id", "wide"]) + .unwrap() + .batch_size(10) + .batch_size_bytes(target_bytes); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert!(batches.iter().all(|batch| batch.num_rows() <= 10)); + + let mut scan = dataset.scan(); + scan.project(&["id", "wide"]) + .unwrap() + .batch_size(200) + .batch_size_bytes(target_bytes) + .strict_batch_size(true); + let error = scan + .try_into_stream() + .await + .err() + .expect("strict row and byte batch limits should be rejected"); + assert!( + error + .to_string() + .contains("strict_batch_size=true cannot be combined with batch_size_bytes=8192"), + "unexpected error: {error}" + ); + } + + #[tokio::test] + async fn test_strict_batch_size() { + let dataset = lance_datagen::gen_batch() + .col("x", array::step::()) + .anon_col(array::step::()) + .into_ram_dataset(FragmentCount::from(7), FragmentRowCount::from(6)) + .await + .unwrap(); + + let mut scan = dataset.scan(); + scan.batch_size(10) + .strict_batch_size(true) + .filter("x % 2 == 0") + .unwrap(); + + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + let batch_sizes = batches.iter().map(|b| b.num_rows()).collect::>(); + assert_eq!(batch_sizes, vec![10, 10, 1]); + } + + #[tokio::test] + async fn test_column_not_exist() { + let dataset = lance_datagen::gen_batch() + .col("x", array::step::()) + .into_ram_dataset(FragmentCount::from(7), FragmentRowCount::from(6)) + .await + .unwrap(); + + let check_err_msg = |r: Result| { + let Err(err) = r else { + panic!( + "Expected an error to be raised saying column y is not found but got no error" + ) + }; + + assert!( + err.to_string().contains("No field named y"), + "Expected error to contain 'No field named y' but got {}", + err + ); + }; + + let mut scan = dataset.scan(); + scan.project(&["x", "y"]).unwrap(); + check_err_msg(scan.try_into_stream().await); + + let mut scan = dataset.scan(); + scan.project(&["y"]).unwrap(); + check_err_msg(scan.try_into_stream().await); + + // This represents a query like `SELECT 1 AS foo` which we could _technically_ satisfy + // but it is not supported today + let mut scan = dataset.scan(); + scan.project_with_transform(&[("foo", "1")]).unwrap(); + match scan.try_into_stream().await { + Ok(_) => panic!("Expected an error to be raised saying not supported"), + Err(e) => { + assert!( + e.to_string().contains("Received only dynamic expressions"), + "Expected error to contain 'Received only dynamic expressions' but got {}", + e + ); + } + } + } + + #[cfg(not(windows))] + #[tokio::test] + async fn test_local_object_store() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, true), + ArrowField::new("s", DataType::Utf8, true), + ])); + + let batches: Vec = (0..5) + .map(|i| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(i * 20..(i + 1) * 20)), + Arc::new(StringArray::from_iter_values( + (i * 20..(i + 1) * 20).map(|v| format!("s-{}", v)), + )), + ], + ) + .unwrap() + }) + .collect(); + + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + let write_params = WriteParams { + max_rows_per_file: 40, + max_rows_per_group: 10, + ..Default::default() + }; + let batches = RecordBatchIterator::new(batches.clone().into_iter().map(Ok), schema.clone()); + Dataset::write(batches, test_uri, Some(write_params)) + .await + .unwrap(); + + let dataset = Dataset::open(&format!("file-object-store://{}", test_uri)) + .await + .unwrap(); + let mut builder = dataset.scan(); + builder.batch_size(8); + let mut stream = builder.try_into_stream().await.unwrap(); + let mut rows_read = 0; + while let Some(next) = stream.next().await { + let next = next.unwrap(); + let expected = 8.min(100 - rows_read); + assert_eq!(next.num_rows(), expected); + rows_read += next.num_rows(); + } + } + + #[tokio::test] + async fn test_filter_parsing() -> Result<()> { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false).await?; + let dataset = &test_ds.dataset; + + let mut scan = dataset.scan(); + assert!(scan.filter.is_none()); + + scan.filter("i > 50")?; + assert_eq!(scan.get_expr_filter().unwrap(), Some(col("i").gt(lit(50)))); + + for use_stats in [false, true] { + let batches = scan + .project(&["s"])? + .use_stats(use_stats) + .try_into_stream() + .await? + .try_collect::>() + .await?; + let batch = concat_batches(&batches[0].schema(), &batches)?; + + let expected_batch = RecordBatch::try_new( // Projected just "s" Arc::new(test_ds.schema.project(&[1])?), vec![Arc::new(StringArray::from_iter_values( @@ -5628,6 +8719,149 @@ mod test { Ok(()) } + // Regression for #6580: a scan with `filter` + `project` of a + // `(Large)List` column used to panic in `merge_with_schema` + // (called from `TakeStream::map_batch`) because the filtered batch arrived + // as a sliced view of a larger batch and the cloned list offsets did not + // start at zero. The trigger requires (a) a `(Large)List` + // projection where the struct is split across `filtered_read` and + // `TakeExec` and (b) a sparse-tail selectivity pattern so the trailing + // filter result lands deep inside the values buffer of its source batch. + // Parametrized over `List`/`LargeList` since the fix touches both offset + // widths in `merge_with_schema`. + #[rstest] + #[tokio::test] + async fn test_filter_project_list_struct_sparse_tail( + // The panic is specific to v2.x storage; the legacy reader takes a + // different code path. V2_0 and V2_2 are the versions called out in + // the original report. + #[values( + LanceFileVersion::V2_0, + LanceFileVersion::Stable, + LanceFileVersion::V2_2 + )] + data_storage_version: LanceFileVersion, + #[values(false, true)] large_list: bool, + ) { + use arrow_array::{LargeListArray, ListArray, UInt16Array}; + use arrow_buffer::{OffsetBuffer, ScalarBuffer}; + + let struct_fields = Fields::from(vec![ + Arc::new(ArrowField::new("a", DataType::Int32, true)), + Arc::new(ArrowField::new("b", DataType::Int32, true)), + ]); + let item_field = Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields.clone()), + true, + )); + let items_dtype = if large_list { + DataType::LargeList(item_field.clone()) + } else { + DataType::List(item_field.clone()) + }; + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("grp", DataType::UInt16, false), + ArrowField::new("items", items_dtype, false), + ])); + + let make_batch = |start: i32, n: usize, group: u16| -> RecordBatch { + let ids = Int32Array::from_iter_values(start..start + n as i32); + let groups = UInt16Array::from(vec![group; n]); + + let mut offsets = Vec::with_capacity(n + 1); + let mut a_vals: Vec = Vec::new(); + let mut b_vals: Vec = Vec::new(); + offsets.push(0i64); + for i in 0..n { + // Variable-length lists (1..=18) so offsets don't land on + // batch-row boundaries. + let len = 1 + (i % 18); + for j in 0..len { + a_vals.push(j as i32); + b_vals.push(-(j as i32)); + } + offsets.push(a_vals.len() as i64); + } + let struct_arr = Arc::new(StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int32Array::from(a_vals)) as ArrayRef, + Arc::new(Int32Array::from(b_vals)) as ArrayRef, + ], + None, + )); + let items: ArrayRef = if large_list { + Arc::new(LargeListArray::new( + item_field.clone(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + struct_arr, + None, + )) + } else { + let offsets_i32: Vec = offsets.iter().map(|&o| o as i32).collect(); + Arc::new(ListArray::new( + item_field.clone(), + OffsetBuffer::new(ScalarBuffer::from(offsets_i32)), + struct_arr, + None, + )) + }; + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(ids) as ArrayRef, + Arc::new(groups) as ArrayRef, + items, + ], + ) + .unwrap() + }; + + // Sparse-tail selectivity (matching the original report's shape at a + // smaller scale): a large leading block of matches, a large gap of + // non-matches, then a small trailing match. Single fragment. + let batches = vec![ + make_batch(0, 20_000, 7), + make_batch(20_000, 80_000, 1), + make_batch(100_000, 1_500, 7), + ]; + + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + let dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: 1_000_000, + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Force a column split inside the `items` struct by marking `items.b` + // as a late-materialized field: `filtered_read` returns the batch with + // `items.a`, and `TakeExec` adds `items.b`. `merge_with_schema` then + // takes its `List` branch, which is where the panic was. + let items_b_field_id = dataset + .schema() + .field("items") + .unwrap() + .child("item") + .unwrap() + .child("b") + .unwrap() + .id as u32; + let mut scan = dataset.scan(); + scan.filter("grp = 7").unwrap(); + scan.project(&["id", "items"]).unwrap(); + scan.materialization_style(MaterializationStyle::AllEarlyExcept(vec![items_b_field_id])); + let result = scan.try_into_batch().await.unwrap(); + assert_eq!(result.num_rows(), 21_500); + } + #[tokio::test] async fn test_scan_regexp_match_and_non_empty_captions() { // Build a small dataset with three Utf8 columns and verify the full @@ -5845,7 +9079,7 @@ mod test { // Make the store slow so that if we don't cancel the scan, it will take a loooong time. let throttled = Arc::new(ThrottledStoreWrapper { config: ThrottleConfig { - wait_get_per_call: Duration::from_secs(1), + wait_get_per_call: Duration::from_millis(100), ..Default::default() }, }); @@ -5884,8 +9118,8 @@ mod test { // This test is a timing test, which is unfortunate, as it may be flaky. I'm hoping // we have enough wiggle room here. The failure case is 30s on my machine and the pass - // case is 2-3s. - assert!(duration < Duration::from_secs(10)); + // case is a few hundred milliseconds. + assert!(duration < Duration::from_secs(3)); } #[rstest] @@ -6079,6 +9313,7 @@ mod test { k: usize, use_index: bool, distance_range: Option<(Option, Option)>, + nprobes: Option, ) { let query_count = query_values.len() / 32; assert_eq!(batch.num_rows(), query_count * k); @@ -6089,6 +9324,12 @@ mod test { let mut scan = dataset.scan(); scan.nearest("vec", &query, k).unwrap(); scan.use_index(use_index); + // Pin nprobes to match the batch query: the single-query indexed path + // otherwise adaptively expands nprobes, which would make equivalence + // depend on data distribution rather than be guaranteed. + if let Some(nprobes) = nprobes { + scan.nprobes(nprobes); + } if let Some((lower, upper)) = distance_range { scan.distance_range(lower, upper); } @@ -6114,471 +9355,1160 @@ mod test { } #[tokio::test] - async fn test_batch_knn_flat() { + async fn test_batch_knn_flat() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + let dataset = &test_ds.dataset; + let k = 2; + + let (queries, query_values) = batch_knn_two_queries(); + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + scan.use_index(false); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("KNNVectorDistance: queries=2"), + "expected flat batch KNN plan, got:\n{}", + plan + ); + assert!( + !plan.contains("ANNSubIndex"), + "flat batch KNN should not use ANN index, got:\n{}", + plan + ); + assert!( + !plan.contains("SortExec: TopK(fetch="), + "batch flat KNN must not truncate to k rows globally, got:\n{}", + plan + ); + + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_knn_output_has_no_vector(&batch, "vec"); + assert_eq!( + batch.num_rows(), + 2 * k, + "batch flat KNN must return k rows per query vector" + ); + assert_eq!( + batch[QUERY_INDEX_COL].as_primitive::().values(), + &[0, 0, 1, 1] + ); + let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); + for query_index in 0..2 { + let rows_for_query = query_indices + .iter() + .filter(|value| *value == Some(query_index)) + .count(); + assert_eq!( + rows_for_query, k, + "query_index {query_index} should have exactly {k} rows" + ); + } + assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None, None) + .await; + + let mut scan_with_vec = dataset.scan(); + scan_with_vec.nearest("vec", &queries, k).unwrap(); + scan_with_vec.use_index(false); + scan_with_vec.project(&["i", "vec"]).unwrap(); + let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + assert!( + batch_with_vec.schema().column_with_name("vec").is_some(), + "batch flat KNN should return vector column when projected" + ); + assert_batch_matches_single_queries( + dataset, + &batch_with_vec, + &query_values, + k, + false, + None, + None, + ) + .await; + + let query_values_one = (32..64).map(|v| v as f32).collect::>(); + let queries_one = FixedSizeListArray::try_new_from_values( + Float32Array::from(query_values_one.clone()), + 32, + ) + .unwrap(); + let mut scan = dataset.scan(); + scan.nearest("vec", &queries_one, k).unwrap(); + scan.use_index(false); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("KNNVectorDistance: queries=1"), + "single-vector batch query should use batch KNN path, got:\n{}", + plan + ); + assert!( + !plan.contains("SortExec: TopK(fetch="), + "batch KNN must not apply per-query SortExec top-k, got:\n{}", + plan + ); + + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_knn_output_has_no_vector(&batch, "vec"); + assert_eq!( + batch[QUERY_INDEX_COL].as_primitive::().values(), + &[0, 0] + ); + } + + #[tokio::test] + async fn test_batch_knn_flat_omits_vector_without_projection() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + let dataset = &test_ds.dataset; + let k = 2; + let (queries, query_values) = batch_knn_two_queries(); + + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + scan.use_index(false); + scan.project(&["i"]).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + assert_batch_knn_output_has_no_vector(&batch, "vec"); + assert_query_index_field(&batch); + assert!(batch.schema().column_with_name("i").is_some()); + assert!(batch.schema().column_with_name(DIST_COL).is_some()); + assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None, None) + .await; + + let mut scan_rowid_only = dataset.scan(); + scan_rowid_only.nearest("vec", &queries, k).unwrap(); + scan_rowid_only.use_index(false); + scan_rowid_only.project(&[ROW_ID]).unwrap(); + let batch_rowid_only = scan_rowid_only.try_into_batch().await.unwrap(); + assert_batch_knn_output_has_no_vector(&batch_rowid_only, "vec"); + assert!(batch_rowid_only.schema().column_with_name(ROW_ID).is_some()); + assert!(batch_rowid_only.schema().column_with_name("i").is_none()); + + let mut scan_with_vec = dataset.scan(); + scan_with_vec.nearest("vec", &queries, k).unwrap(); + scan_with_vec.use_index(false); + scan_with_vec.project(&["vec"]).unwrap(); + let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + assert!( + batch_with_vec.schema().column_with_name("vec").is_some(), + "batch flat KNN must include vector column when vec is projected" + ); + } + + #[tokio::test] + async fn test_batch_knn_flat_filter_keeps_non_vector_columns() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + let dataset = &test_ds.dataset; + let k = 2; + let (queries, query_values) = batch_knn_two_queries(); + + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + scan.use_index(false); + scan.filter("i >= 0").unwrap(); + scan.project(&["i"]).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + + assert_query_index_field(&batch); + assert_batch_knn_output_has_no_vector(&batch, "vec"); + assert!(batch.schema().column_with_name("i").is_some()); + + let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); + for query_index in 0..2 { + let query = + Float32Array::from(query_values[query_index * 32..(query_index + 1) * 32].to_vec()); + let mut single = dataset.scan(); + single.nearest("vec", &query, k).unwrap(); + single.use_index(false); + single.filter("i >= 0").unwrap(); + single.project(&["i"]).unwrap(); + let single_batch = single.try_into_batch().await.unwrap(); + + let mask = BooleanArray::from_iter( + query_indices + .iter() + .map(|value| value.map(|value| value == query_index as i32)), + ); + let batch_slice = arrow::compute::filter_record_batch(&batch, &mask).unwrap(); + assert_eq!( + batch_slice["i"].as_primitive::().values(), + single_batch["i"].as_primitive::().values() + ); + } + } + + #[tokio::test] + async fn test_batch_knn_flat_nested_vector_projection() { + const VECTOR_COLUMN: &str = "payload.vec"; + let (_tmp, dataset) = nested_vector_test_dataset(32).await; + let k = 2; + let (queries, _query_values) = batch_knn_two_queries(); + + let mut scan = dataset.scan(); + scan.nearest(VECTOR_COLUMN, &queries, k).unwrap(); + scan.use_index(false); + scan.project(&["i"]).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_knn_output_has_no_vector(&batch, VECTOR_COLUMN); + assert_eq!(batch.num_rows(), 2 * k); + assert!(batch.schema().column_with_name("i").is_some()); + + let mut scan_with_vec = dataset.scan(); + scan_with_vec.nearest(VECTOR_COLUMN, &queries, k).unwrap(); + scan_with_vec.use_index(false); + scan_with_vec.project(&[VECTOR_COLUMN]).unwrap(); + let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + assert!( + batch_with_vec + .schema() + .column_with_name(VECTOR_COLUMN) + .is_some(), + "batch flat KNN must include nested vector column when projected; columns: {:?}", + batch_with_vec.schema().field_names() + ); + } + + #[tokio::test] + async fn test_batch_knn_flat_escaped_nested_vector_projection() { + const VECTOR_COLUMN: &str = "payload.`vec.with.dot`"; + let (_tmp, dataset) = escaped_nested_vector_test_dataset(32).await; + let k = 2; + let (queries, _) = batch_knn_two_queries(); + + let mut scan = dataset.scan(); + scan.nearest(VECTOR_COLUMN, &queries, k).unwrap(); + scan.use_index(false); + scan.project(&["i"]).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_knn_output_has_no_vector(&batch, VECTOR_COLUMN); + assert_eq!(batch.num_rows(), 2 * k); + assert!(batch.schema().column_with_name("i").is_some()); + + let mut scan_with_vec = dataset.scan(); + scan_with_vec.nearest(VECTOR_COLUMN, &queries, k).unwrap(); + scan_with_vec.use_index(false); + scan_with_vec.project(&[VECTOR_COLUMN]).unwrap(); + let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + assert!( + batch_with_vec + .schema() + .column_with_name(VECTOR_COLUMN) + .is_some(), + "batch flat KNN must include escaped nested vector column when projected; columns: {:?}", + batch_with_vec.schema().field_names() + ); + } + + #[tokio::test] + async fn test_batch_knn_flat_projects_row_id_and_row_addr_without_vector() { let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); let dataset = &test_ds.dataset; let k = 2; + let (queries, _) = batch_knn_two_queries(); - let (queries, query_values) = batch_knn_two_queries(); let mut scan = dataset.scan(); scan.nearest("vec", &queries, k).unwrap(); scan.use_index(false); - scan.project(&["i"]).unwrap(); + scan.project(&[ROW_ID]).unwrap(); + scan.with_row_address(); - let plan = scan.explain_plan(false).await.unwrap(); - assert!( - plan.contains("KNNVectorDistance: queries=2"), - "expected flat batch KNN plan, got:\n{}", - plan - ); - assert!( - !plan.contains("ANNSubIndex"), - "flat batch KNN should not use ANN index, got:\n{}", - plan + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_knn_output_has_no_vector(&batch, "vec"); + assert_eq!(batch.num_rows(), 2 * k); + assert!(batch.schema().column_with_name(ROW_ID).is_some()); + assert!(batch.schema().column_with_name(ROW_ADDR).is_some()); + assert!(batch.schema().column_with_name(DIST_COL).is_some()); + assert_eq!( + batch[ROW_ADDR].as_primitive::().null_count(), + 0, + "row addresses should be materialized for all top-k rows" ); + } + + #[tokio::test] + async fn test_primitive_query_length_multiple_of_dim_is_rejected() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + let dataset = &test_ds.dataset; + let q: Float32Array = (32..96).map(|v| v as f32).collect(); + + let err = match dataset.scan().nearest("vec", &q, 2) { + Err(err) => err.to_string(), + Ok(_) => panic!("expected primitive query length mismatch error"), + }; assert!( - !plan.contains("SortExec: TopK(fetch="), - "batch flat KNN must not truncate to k rows globally, got:\n{}", - plan + err.contains("query dim(64) doesn't match the column vec vector dim(32)"), + "unexpected error: {err}" ); + } - let batch = scan.try_into_batch().await.unwrap(); - assert_query_index_field(&batch); - assert_batch_knn_output_has_no_vector(&batch, "vec"); + async fn dataset_with_query_index_column() -> (TempStrDir, Dataset) { + let path = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, true), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 32, + ), + true, + ), + ArrowField::new(QUERY_INDEX_COL, DataType::UInt32, true), + ])); + let vector_values: Float32Array = (0..32 * 80).map(|v| v as f32).collect(); + let vectors = FixedSizeListArray::try_new_from_values(vector_values, 32).unwrap(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..80)), + Arc::new(vectors), + Arc::new(UInt32Array::from_iter((0..80).map(|v| v as u32))), + ], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(std::iter::once(Ok(batch)), schema.clone()), + &path, + None, + ) + .await + .unwrap(); + (path, dataset) + } + + #[tokio::test] + async fn test_batch_knn_rejects_dataset_query_index_column() { + let (_tmp, dataset) = dataset_with_query_index_column().await; + let (queries, _) = batch_knn_two_queries(); + let err = match dataset.scan().nearest("vec", &queries, 2) { + Err(err) => err.to_string(), + Ok(_) => panic!("expected reserved query_index column error"), + }; + assert!(err.contains(QUERY_INDEX_COL), "unexpected error: {err}"); + } + + #[tokio::test] + async fn test_single_knn_projects_dataset_query_index_column() { + let (_tmp, dataset) = dataset_with_query_index_column().await; + let q: Float32Array = (32..64).map(|v| v as f32).collect(); + + let mut scan = dataset.scan(); + scan.nearest("vec", &q, 2).unwrap(); + scan.use_index(false); + scan.project(&["i"]).unwrap(); + let without_query_index = scan.try_into_batch().await.unwrap(); + + let mut scan = dataset.scan(); + scan.nearest("vec", &q, 2).unwrap(); + scan.use_index(false); + scan.project(&["i", QUERY_INDEX_COL]).unwrap(); + let with_query_index = scan.try_into_batch().await.unwrap(); + + assert_eq!(without_query_index.num_rows(), 2); assert_eq!( - batch.num_rows(), - 2 * k, - "batch flat KNN must return k rows per query vector" + without_query_index["i"] + .as_primitive::() + .values(), + with_query_index["i"].as_primitive::().values() ); assert_eq!( - batch[QUERY_INDEX_COL].as_primitive::().values(), - &[0, 0, 1, 1] + with_query_index[QUERY_INDEX_COL] + .as_primitive::() + .null_count(), + 0 ); - let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); - for query_index in 0..2 { - let rows_for_query = query_indices - .iter() - .filter(|value| *value == Some(query_index)) - .count(); - assert_eq!( - rows_for_query, k, - "query_index {query_index} should have exactly {k} rows" - ); - } - assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None).await; + } - let mut scan_with_vec = dataset.scan(); - scan_with_vec.nearest("vec", &queries, k).unwrap(); - scan_with_vec.use_index(false); - scan_with_vec.project(&["i", "vec"]).unwrap(); - let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); - assert!( - batch_with_vec.schema().column_with_name("vec").is_some(), - "batch flat KNN should return vector column when projected" + #[tokio::test] + async fn test_batch_knn_flat_respects_distance_range() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + let dataset = &test_ds.dataset; + let (queries, query_values) = batch_knn_two_queries(); + + let batch = dataset + .scan() + .nearest("vec", &queries, 2) + .unwrap() + .use_index(false) + .distance_range(Some(1.0), None) + .project(&["i"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + + assert_eq!( + batch[QUERY_INDEX_COL].as_primitive::().values(), + &[0, 0, 1, 1] ); assert_batch_matches_single_queries( dataset, - &batch_with_vec, + &batch, &query_values, - k, + 2, false, + Some((Some(1.0), None)), None, ) .await; + } + + #[tokio::test] + async fn test_batch_knn_indexed() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + test_ds.make_vector_index().await.unwrap(); + let dataset = &test_ds.dataset; + let (queries, query_values) = batch_knn_two_queries(); - let query_values_one = (32..64).map(|v| v as f32).collect::>(); - let queries_one = FixedSizeListArray::try_new_from_values( - Float32Array::from(query_values_one.clone()), - 32, - ) - .unwrap(); let mut scan = dataset.scan(); - scan.nearest("vec", &queries_one, k).unwrap(); - scan.use_index(false); + scan.nearest("vec", &queries, 2).unwrap(); + // Probe both partitions (minimum == maximum) so the per-query top-k is + // merged across multiple partitions and the batch result is + // deterministically equivalent to repeated single-query search (which + // would otherwise adaptively expand nprobes). + scan.nprobes(2); scan.project(&["i"]).unwrap(); let plan = scan.explain_plan(false).await.unwrap(); assert!( - plan.contains("KNNVectorDistance: queries=1"), - "single-vector batch query should use batch KNN path, got:\n{}", + plan.contains("ANNIvfBatch"), + "IVF batch KNN should use the shared-scan batch node, got:\n{}", plan ); assert!( - !plan.contains("SortExec: TopK(fetch="), - "batch KNN must not apply per-query SortExec top-k, got:\n{}", + !plan.contains("ANNSubIndex"), + "IVF batch KNN should not fall back to per-query ANN search, got:\n{}", + plan + ); + assert!( + !plan.contains("KNNVectorDistance: queries=2"), + "indexed batch KNN should not force the flat batch path, got:\n{}", plan ); + // The batch node loads each probed partition once and scores every query + // that probes it, so it must report the *distinct* partitions read: with + // 2 partitions and nprobes(2), both queries probe both partitions, so the + // union is 2 -- not the per-query sum (2 queries x 2 = 4), and never 0 + // (which is what a dropped metric would show). This guards the observed + // `partitions_searched` against silently regressing to either. + let analyzed = scan.analyze_plan().await.unwrap(); + let batch_line = analyzed + .lines() + .find(|line| line.contains("ANNIvfBatch")) + .expect("analyzed plan should contain the ANNIvfBatch node"); + assert!( + batch_line.contains("partitions_searched=2"), + "batch node must report the distinct partitions searched, got:\n{}", + batch_line + ); + let batch = scan.try_into_batch().await.unwrap(); assert_query_index_field(&batch); - assert_batch_knn_output_has_no_vector(&batch, "vec"); assert_eq!( batch[QUERY_INDEX_COL].as_primitive::().values(), - &[0, 0] + &[0, 0, 1, 1] ); + // Shared-scan batch search must return the same rows/distances as + // issuing the queries one at a time against the index. + assert_batch_matches_single_queries(dataset, &batch, &query_values, 2, true, None, Some(2)) + .await; + + let batch = dataset + .scan() + .nearest("vec", &queries, 2) + .unwrap() + .nprobes(2) + .distance_range(Some(1.0), None) + .project(&["i"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_batch_matches_single_queries( + dataset, + &batch, + &query_values, + 2, + true, + Some((Some(1.0), None)), + Some(2), + ) + .await; } - #[tokio::test] - async fn test_batch_knn_flat_omits_vector_without_projection() { - let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + /// End-to-end contract: equal-distance neighbors come back in a canonical, + /// deterministic order — ascending row id within a distance tie — and the + /// shared-scan batch path returns exactly what repeated single-query search + /// does. A single-partition exact (IVF_FLAT) index queried with a vector that + /// matches a row duplicated once per fragment yields five neighbors tied at + /// distance 0; `k = 5` returns all of them, so their order is fixed solely by + /// the tie-break, which orders ties by ascending row id (here ascending `i`, + /// since the data is written in `i` order). + /// + /// This pins the user-visible ordering guarantee; it does not isolate a + /// single internal sort. The final `(distance, row_id)` order is enforced by + /// the downstream consumer, and the stable partition scan order in + /// `search_partitions_batch` only changes *which* tied row survives when a tie + /// is truncated across partitions — which this single-partition, + /// all-ties-fit-within-`k` case deliberately does not exercise. + #[rstest] + #[tokio::test] + async fn test_batch_knn_indexed_orders_ties_by_row_id( + #[values(false, true)] stable_row_ids: bool, + ) { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, stable_row_ids) + .await + .unwrap(); + // Single partition + exact (flat) storage: distances are exact, so the + // vectors duplicated across fragments tie at distance 0, and both the + // batch and single-query paths scan the one partition. That isolates the + // tie-break as the only thing determining the emitted order. + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); + test_ds + .dataset + .create_index( + &["vec"], + IndexType::Vector, + Some("idx".to_string()), + ¶ms, + true, + ) .await .unwrap(); let dataset = &test_ds.dataset; - let k = 2; - let (queries, query_values) = batch_knn_two_queries(); + let (queries, query_values) = batch_knn_two_queries(); + // Each query exactly matches a vector that appears once per 80-row + // fragment (5 copies), all at distance 0. `k = 5` returns every tied + // copy, so no truncation can hide the ordering. + let k = 5; let mut scan = dataset.scan(); scan.nearest("vec", &queries, k).unwrap(); - scan.use_index(false); + scan.nprobes(1); scan.project(&["i"]).unwrap(); - let batch = scan.try_into_batch().await.unwrap(); - assert_batch_knn_output_has_no_vector(&batch, "vec"); - assert_query_index_field(&batch); - assert!(batch.schema().column_with_name("i").is_some()); - assert!(batch.schema().column_with_name(DIST_COL).is_some()); - assert_batch_matches_single_queries(dataset, &batch, &query_values, k, false, None).await; - let mut scan_rowid_only = dataset.scan(); - scan_rowid_only.nearest("vec", &queries, k).unwrap(); - scan_rowid_only.use_index(false); - scan_rowid_only.project(&[ROW_ID]).unwrap(); - let batch_rowid_only = scan_rowid_only.try_into_batch().await.unwrap(); - assert_batch_knn_output_has_no_vector(&batch_rowid_only, "vec"); - assert!(batch_rowid_only.schema().column_with_name(ROW_ID).is_some()); - assert!(batch_rowid_only.schema().column_with_name("i").is_none()); + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ANNIvfBatch"), + "single-partition IVF batch KNN should use the shared-scan batch node, got:\n{}", + plan + ); - let mut scan_with_vec = dataset.scan(); - scan_with_vec.nearest("vec", &queries, k).unwrap(); - scan_with_vec.use_index(false); - scan_with_vec.project(&["vec"]).unwrap(); - let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + // The batch node must return the same rows/distances as issuing each + // query on its own against the index. + assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, Some(1)) + .await; + + // Query 0 matches vector index 1, stored at i = 1, 81, 161, 241, 321 + // (once per fragment). All tie at distance 0, so the canonical + // (distance, row_id) order surfaces them by ascending row id, which here + // is ascending `i`. + let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); + let q0 = arrow::compute::filter_record_batch( + &batch, + &BooleanArray::from_iter(query_indices.iter().map(|value| Some(value == Some(0)))), + ) + .unwrap(); + assert_eq!( + q0["i"].as_primitive::().values(), + &[1, 81, 161, 241, 321] + ); + let q0_dists = q0[DIST_COL].as_primitive::(); assert!( - batch_with_vec.schema().column_with_name("vec").is_some(), - "batch flat KNN must include vector column when vec is projected" + q0_dists + .values() + .iter() + .all(|dist| *dist == q0_dists.value(0)), + "the five duplicated neighbors must be genuine ties, got distances {:?}", + q0_dists.values() ); } + /// Any `refine_factor` sends the query onto a reranking path the shared-scan + /// batch node does not implement, so the scanner must fall back to the + /// per-query indexed loop and still produce correctly grouped results. + /// + /// All of `refine(0)`, `refine(1)`, and `refine(2)` must fall back: + /// `refine(1)` still reranks on the single-query path (a factor of 1 is not + /// a no-op), and `refine(0)` is rejected there with `Refine factor cannot be + /// zero` — the batch path would instead return empty results. Covering the + /// boundary factors guards the `refine_factor.is_some()` gate against + /// regressing back to a `> 1` check. #[tokio::test] - async fn test_batch_knn_flat_filter_keeps_non_vector_columns() { - let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + async fn test_batch_knn_indexed_refine_falls_back() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); + test_ds.make_vector_index().await.unwrap(); let dataset = &test_ds.dataset; - let k = 2; - let (queries, query_values) = batch_knn_two_queries(); - - let mut scan = dataset.scan(); - scan.nearest("vec", &queries, k).unwrap(); - scan.use_index(false); - scan.filter("i >= 0").unwrap(); - scan.project(&["i"]).unwrap(); - let batch = scan.try_into_batch().await.unwrap(); - - assert_query_index_field(&batch); - assert_batch_knn_output_has_no_vector(&batch, "vec"); - assert!(batch.schema().column_with_name("i").is_some()); + let (queries, _query_values) = batch_knn_two_queries(); - let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); - for query_index in 0..2 { - let query = - Float32Array::from(query_values[query_index * 32..(query_index + 1) * 32].to_vec()); - let mut single = dataset.scan(); - single.nearest("vec", &query, k).unwrap(); - single.use_index(false); - single.filter("i >= 0").unwrap(); - single.project(&["i"]).unwrap(); - let single_batch = single.try_into_batch().await.unwrap(); + for refine_factor in [1u32, 2] { + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, 2).unwrap(); + scan.refine(refine_factor); + scan.project(&["i"]).unwrap(); - let mask = BooleanArray::from_iter( - query_indices - .iter() - .map(|value| value.map(|value| value == query_index as i32)), + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + !plan.contains("ANNIvfBatch"), + "refine({refine_factor}) must not use the shared-scan batch node, got:\n{plan}" ); - let batch_slice = arrow::compute::filter_record_batch(&batch, &mask).unwrap(); + assert!( + plan.contains("ANNSubIndex"), + "refine({refine_factor}) batch search should fall back to the per-query \ + indexed loop, got:\n{plan}" + ); + + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); assert_eq!( - batch_slice["i"].as_primitive::().values(), - single_batch["i"].as_primitive::().values() + batch[QUERY_INDEX_COL].as_primitive::().values(), + &[0, 0, 1, 1], + "refine({refine_factor}) should still group results per query" ); } - } - - #[tokio::test] - async fn test_batch_knn_flat_nested_vector_projection() { - const VECTOR_COLUMN: &str = "payload.vec"; - let (_tmp, dataset) = nested_vector_test_dataset(32).await; - let k = 2; - let (queries, _query_values) = batch_knn_two_queries(); + // refine(0) is rejected on the fallback (per-query) path; the batch path + // must not silently accept it and return empty results instead. let mut scan = dataset.scan(); - scan.nearest(VECTOR_COLUMN, &queries, k).unwrap(); - scan.use_index(false); + scan.nearest("vec", &queries, 2).unwrap(); + scan.refine(0); scan.project(&["i"]).unwrap(); - let batch = scan.try_into_batch().await.unwrap(); - assert_query_index_field(&batch); - assert_batch_knn_output_has_no_vector(&batch, VECTOR_COLUMN); - assert_eq!(batch.num_rows(), 2 * k); - assert!(batch.schema().column_with_name("i").is_some()); - - let mut scan_with_vec = dataset.scan(); - scan_with_vec.nearest(VECTOR_COLUMN, &queries, k).unwrap(); - scan_with_vec.use_index(false); - scan_with_vec.project(&[VECTOR_COLUMN]).unwrap(); - let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + let result = scan.try_into_batch().await; assert!( - batch_with_vec - .schema() - .column_with_name(VECTOR_COLUMN) - .is_some(), - "batch flat KNN must include nested vector column when projected; columns: {:?}", - batch_with_vec.schema().field_names() + result.is_err(), + "refine(0) must error rather than fall through to an empty batch result" ); } + /// Without pinned nprobes the shared-scan fast path is not equivalent to + /// single-query search (the single-query path applies an adaptive + /// `early_pruning` floor and late-search expansion that the batch path does + /// not), so the scanner must fall back to the per-query loop, which reuses + /// the real adaptive search and stays exact. #[tokio::test] - async fn test_batch_knn_flat_escaped_nested_vector_projection() { - const VECTOR_COLUMN: &str = "payload.`vec.with.dot`"; - let (_tmp, dataset) = escaped_nested_vector_test_dataset(32).await; + async fn test_batch_knn_indexed_adaptive_nprobes_falls_back() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + test_ds.make_vector_index().await.unwrap(); + let dataset = &test_ds.dataset; + let (queries, query_values) = batch_knn_two_queries(); let k = 2; - let (queries, _) = batch_knn_two_queries(); let mut scan = dataset.scan(); - scan.nearest(VECTOR_COLUMN, &queries, k).unwrap(); - scan.use_index(false); + scan.nearest("vec", &queries, k).unwrap(); + // No nprobes() call: adaptive (minimum_nprobes=1, maximum_nprobes=None). scan.project(&["i"]).unwrap(); - let batch = scan.try_into_batch().await.unwrap(); - assert_query_index_field(&batch); - assert_batch_knn_output_has_no_vector(&batch, VECTOR_COLUMN); - assert_eq!(batch.num_rows(), 2 * k); - assert!(batch.schema().column_with_name("i").is_some()); - let mut scan_with_vec = dataset.scan(); - scan_with_vec.nearest(VECTOR_COLUMN, &queries, k).unwrap(); - scan_with_vec.use_index(false); - scan_with_vec.project(&[VECTOR_COLUMN]).unwrap(); - let batch_with_vec = scan_with_vec.try_into_batch().await.unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); assert!( - batch_with_vec - .schema() - .column_with_name(VECTOR_COLUMN) - .is_some(), - "batch flat KNN must include escaped nested vector column when projected; columns: {:?}", - batch_with_vec.schema().field_names() + !plan.contains("ANNIvfBatch"), + "adaptive nprobes must not use the shared-scan batch node, got:\n{}", + plan + ); + assert!( + plan.contains("ANNSubIndex"), + "adaptive nprobes batch search should fall back to the per-query loop, got:\n{}", + plan ); - } + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + // The fallback runs real single-query searches, so it stays exact even + // with adaptive nprobes. + assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, None) + .await; + } + + /// `nprobes(0)` is not rejected by the query builder, so `minimum_nprobes == + /// maximum_nprobes == 0` slips past the fixed-nprobes gate. The single-query + /// path then probes nothing and returns an empty result, whereas the batch + /// node would clamp `nprobes` up to one partition — a silent divergence. The + /// scanner must fall back so the per-query loop defines the semantics of + /// `nprobes(0)`, and the grouped batch result must equal repeated single-query + /// search (both empty here). #[tokio::test] - async fn test_batch_knn_flat_projects_row_id_and_row_addr_without_vector() { - let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + async fn test_batch_knn_indexed_zero_nprobes_falls_back() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); + test_ds.make_vector_index().await.unwrap(); let dataset = &test_ds.dataset; + let (queries, query_values) = batch_knn_two_queries(); let k = 2; - let (queries, _) = batch_knn_two_queries(); let mut scan = dataset.scan(); scan.nearest("vec", &queries, k).unwrap(); - scan.use_index(false); - scan.project(&[ROW_ID]).unwrap(); - scan.with_row_address(); + scan.nprobes(0); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + !plan.contains("ANNIvfBatch"), + "nprobes(0) must not use the shared-scan batch node (which would clamp \ + to one partition), got:\n{plan}" + ); + assert!( + plan.contains("ANNSubIndex"), + "nprobes(0) batch search should fall back to the per-query indexed loop, got:\n{plan}" + ); + // The fallback runs the real single-query path per query, so the grouped + // batch result must match issuing each query on its own with nprobes(0). let batch = scan.try_into_batch().await.unwrap(); assert_query_index_field(&batch); - assert_batch_knn_output_has_no_vector(&batch, "vec"); - assert_eq!(batch.num_rows(), 2 * k); - assert!(batch.schema().column_with_name(ROW_ID).is_some()); - assert!(batch.schema().column_with_name(ROW_ADDR).is_some()); - assert!(batch.schema().column_with_name(DIST_COL).is_some()); - assert_eq!( - batch[ROW_ADDR].as_primitive::().null_count(), - 0, - "row addresses should be materialized for all top-k rows" - ); + let query_count = query_values.len() / 32; + for query_index in 0..query_count { + let query = + Float32Array::from(query_values[query_index * 32..(query_index + 1) * 32].to_vec()); + let mut single_scan = dataset.scan(); + single_scan.nearest("vec", &query, k).unwrap(); + single_scan.nprobes(0); + single_scan.project(&["i"]).unwrap(); + let single = single_scan.try_into_batch().await.unwrap(); + + let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); + let mask = BooleanArray::from_iter( + query_indices + .iter() + .map(|value| value.map(|value| value == query_index as i32)), + ); + let batch_slice = arrow::compute::filter_record_batch(&batch, &mask).unwrap(); + assert_eq!( + batch_slice["i"].as_primitive::().values(), + single["i"].as_primitive::().values(), + "nprobes(0) query {query_index}: batch rows must match single-query rows" + ); + } } + /// A caller-supplied external row-address mask (`with_row_addr_prefilter`) is + /// applied per query on the single-query prefilter path (`with_external_mask`) + /// but is not carried by the shared batch scan. An otherwise batch-eligible + /// query must therefore fall back to the per-query loop when a mask is present, + /// and every returned row must honor the mask — otherwise the batch path would + /// silently return masked-out rows. #[tokio::test] - async fn test_primitive_query_length_multiple_of_dim_is_rejected() { - let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + async fn test_batch_knn_indexed_external_mask_falls_back() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); + test_ds.make_vector_index().await.unwrap(); let dataset = &test_ds.dataset; - let q: Float32Array = (32..96).map(|v| v as f32).collect(); - - let err = match dataset.scan().nearest("vec", &q, 2) { - Err(err) => err.to_string(), - Ok(_) => panic!("expected primitive query length mismatch error"), - }; + let (queries, _query_values) = batch_knn_two_queries(); + let k = 15; + + // Same query shape as `test_batch_knn_indexed`: without a mask it is + // batch-eligible, so the mask is the only thing that forces the fallback. + let mut unmasked = dataset.scan(); + unmasked.nearest("vec", &queries, k).unwrap(); + unmasked.nprobes(2); + unmasked.project(&["i"]).unwrap(); + let unmasked_plan = unmasked.explain_plan(false).await.unwrap(); assert!( - err.contains("query dim(64) doesn't match the column vec vector dim(32)"), - "unexpected error: {err}" + unmasked_plan.contains("ANNIvfBatch"), + "without a mask this query should use the shared-scan batch node, got:\n{unmasked_plan}" ); - } - async fn dataset_with_query_index_column() -> (TempStrDir, Dataset) { - let path = TempStrDir::default(); - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, true), - ArrowField::new( - "vec", - DataType::FixedSizeList( - Arc::new(ArrowField::new("item", DataType::Float32, true)), - 32, - ), - true, - ), - ArrowField::new(QUERY_INDEX_COL, DataType::UInt32, true), - ])); - let vector_values: Float32Array = (0..32 * 80).map(|v| v as f32).collect(); - let vectors = FixedSizeListArray::try_new_from_values(vector_values, 32).unwrap(); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..80)), - Arc::new(vectors), - Arc::new(UInt32Array::from_iter((0..80).map(|v| v as u32))), - ], - ) - .unwrap(); - let dataset = Dataset::write( - RecordBatchIterator::new(std::iter::once(Ok(batch)), schema.clone()), - &path, - None, - ) - .await - .unwrap(); - (path, dataset) - } + // Build an allowlist from the dataset's row addresses (freshly created + // single fragment, so _rowid == row address). + let mut scan = dataset.scan(); + scan.with_row_id(); + let all_ids = batch_row_ids(&scan.try_into_batch().await.unwrap()); + let allow: Vec = all_ids.iter().copied().step_by(2).collect(); + let allow_set: BTreeSet = allow.iter().copied().collect(); + + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + scan.nprobes(2); + scan.with_row_addr_prefilter(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter( + allow.iter().copied(), + ))); + scan.with_row_id(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + !plan.contains("ANNIvfBatch"), + "an external row mask must not use the shared-scan batch node, which \ + does not carry the mask, got:\n{plan}" + ); + assert!( + plan.contains("ANNSubIndex"), + "a masked batch query should fall back to the per-query indexed loop, got:\n{plan}" + ); - #[tokio::test] - async fn test_batch_knn_rejects_dataset_query_index_column() { - let (_tmp, dataset) = dataset_with_query_index_column().await; - let (queries, _) = batch_knn_two_queries(); - let err = match dataset.scan().nearest("vec", &queries, 2) { - Err(err) => err.to_string(), - Ok(_) => panic!("expected reserved query_index column error"), - }; - assert!(err.contains(QUERY_INDEX_COL), "unexpected error: {err}"); + // The fallback must honor the mask: every returned row is in the allowlist. + let got = batch_row_ids(&scan.try_into_batch().await.unwrap()); + assert!( + !got.is_empty(), + "masked batch KNN should still return allowed rows" + ); + for id in got { + assert!( + allow_set.contains(&id), + "returned _rowid {id} not in allowlist" + ); + } } + /// The shared-scan fast path is only equivalent to repeated single-query + /// search when the selected `index_segments` cover every requested fragment. + /// With one segment per fragment, requesting both fragments but selecting + /// only the first segment leaves fragment 1 covered solely by the + /// *unselected* segment: the batch node would search just the selected + /// segment and silently drop it. Eligibility must be computed from the + /// selected segments' coverage (as `knn_combined` does), not the whole + /// logical index, so the scanner falls back to the per-query loop, which + /// re-scores the uncovered fragment on the flat path and returns every row. #[tokio::test] - async fn test_single_knn_projects_dataset_query_index_column() { - let (_tmp, dataset) = dataset_with_query_index_column().await; - let q: Float32Array = (32..64).map(|v| v as f32).collect(); + async fn test_batch_knn_indexed_partial_segment_selection_falls_back() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, false) + .await + .unwrap(); + // One segment per fragment: segment_ids[0] covers fragment 0 (i=0..200), + // segment_ids[1] covers fragment 1 (i=200..400). + let segment_ids = test_ds.make_segmented_vector_index().await.unwrap(); + let dataset = &test_ds.dataset; + let fragments = dataset.fragments(); + assert_eq!(fragments.len(), 2, "base dataset should have two fragments"); + + let (queries, _query_values) = batch_knn_two_queries(); + // k covers every row in both requested fragments (200 each), so a + // complete search returns 400 rows per query. + let k = 400; let mut scan = dataset.scan(); - scan.nearest("vec", &q, 2).unwrap(); - scan.use_index(false); + scan.nearest("vec", &queries, k).unwrap(); + scan.nprobes(2); + // Request both indexed fragments but select only the segment covering + // fragment 0; fragment 1 is covered only by the unselected segment. + scan.with_fragments(vec![fragments[0].clone(), fragments[1].clone()]); + scan.with_index_segments(vec![segment_ids[0]]).unwrap(); scan.project(&["i"]).unwrap(); - let without_query_index = scan.try_into_batch().await.unwrap(); - let mut scan = dataset.scan(); - scan.nearest("vec", &q, 2).unwrap(); - scan.use_index(false); - scan.project(&["i", QUERY_INDEX_COL]).unwrap(); - let with_query_index = scan.try_into_batch().await.unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + !plan.contains("ANNIvfBatch"), + "a requested fragment outside the selected segments must force a fallback, \ + not a shared scan that drops it, got:\n{plan}" + ); - assert_eq!(without_query_index.num_rows(), 2); + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); assert_eq!( - without_query_index["i"] - .as_primitive::() - .values(), - with_query_index["i"].as_primitive::().values() + batch.num_rows(), + 2 * k, + "each query must return all 400 rows across both requested fragments" ); - assert_eq!( - with_query_index[QUERY_INDEX_COL] - .as_primitive::() - .null_count(), - 0 + let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); + for query_index in 0..2 { + let rows_for_query = query_indices + .iter() + .filter(|value| *value == Some(query_index)) + .count(); + assert_eq!( + rows_for_query, k, + "query_index {query_index} must cover both fragments (got {rows_for_query})" + ); + } + // Fragment 0 (i in 0..200) comes from the selected segment; fragment 1 + // (i in 200..400) must appear via the flat fallback. + let i_array = batch["i"].as_primitive::(); + assert!( + i_array + .iter() + .any(|v| v.is_some_and(|val| (0..200).contains(&val))) + && i_array + .iter() + .any(|v| v.is_some_and(|val| (200..400).contains(&val))), + "results must include rows from both the selected segment and the flat-fallback fragment" ); } + /// A wide batch probes more distinct partitions than one streaming chunk holds + /// (`STREAMING_SEARCH_BATCH_SIZE` = 16), so `search_partitions_batch` scores + /// them in several `spawn_cpu` dispatches, threading the per-query top-k heaps + /// across chunk boundaries. The small indexes in the other tests fit in a + /// single chunk and never exercise that seam; here an exact (flat) index with + /// more partitions than the chunk size, probed in full, pins the multi-chunk + /// path to repeated single-query search. #[tokio::test] - async fn test_batch_knn_flat_respects_distance_range() { - let test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + async fn test_batch_knn_indexed_streams_multiple_chunks() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); - let dataset = &test_ds.dataset; - let (queries, query_values) = batch_knn_two_queries(); - - let batch = dataset - .scan() - .nearest("vec", &queries, 2) - .unwrap() - .use_index(false) - .distance_range(Some(1.0), None) - .project(&["i"]) - .unwrap() - .try_into_batch() + // More partitions than one streaming chunk so scoring spans multiple + // chunks; exact (flat) storage with every partition probed keeps the batch + // result an exact match for single-query search. + let num_partitions = 20; + let params = VectorIndexParams::ivf_flat(num_partitions, MetricType::L2); + test_ds + .dataset + .create_index( + &["vec"], + IndexType::Vector, + Some("idx".to_string()), + ¶ms, + true, + ) .await .unwrap(); + let dataset = &test_ds.dataset; - assert_eq!( - batch[QUERY_INDEX_COL].as_primitive::().values(), - &[0, 0, 1, 1] + // Guard the premise of this test: `nprobes(num_partitions)` probes every + // partition, so the batch spans multiple streaming chunks only if the + // partition count exceeds the chunk size. If the default chunk size is + // ever raised past `num_partitions`, fail loudly here rather than let the + // test silently collapse to a single chunk and stop covering the seam. + let chunk_size = *crate::index::vector::ivf::v2::STREAMING_SEARCH_BATCH_SIZE; + assert!( + num_partitions > chunk_size, + "test needs more partitions ({num_partitions}) than the streaming chunk size \ + ({chunk_size}) to span multiple chunks", + ); + + let (queries, query_values) = batch_knn_two_queries(); + let k = 2; + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + // Probe every partition so both paths are exact regardless of centroid + // proximity, and so the batch spans multiple streaming chunks. + scan.nprobes(num_partitions); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ANNIvfBatch"), + "wide IVF batch KNN should use the shared-scan batch node, got:\n{plan}" ); + + let batch = scan.try_into_batch().await.unwrap(); assert_batch_matches_single_queries( dataset, &batch, &query_values, - 2, - false, - Some((Some(1.0), None)), + k, + true, + None, + Some(num_partitions), ) .await; } + /// IVF_HNSW is an unsupported index type for the shared-scan batch path (its + /// graph sub-index has no global top-k heap), so batch search must fall back + /// to the per-query indexed loop and still produce correct grouped results. #[tokio::test] - async fn test_batch_knn_indexed() { + async fn test_batch_knn_indexed_hnsw_falls_back() { let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); - test_ds.make_vector_index().await.unwrap(); + test_ds.make_ivf_hnsw_index().await.unwrap(); let dataset = &test_ds.dataset; let (queries, query_values) = batch_knn_two_queries(); + let k = 2; let mut scan = dataset.scan(); - scan.nearest("vec", &queries, 2).unwrap(); + scan.nearest("vec", &queries, k).unwrap(); + scan.nprobes(2); scan.project(&["i"]).unwrap(); let plan = scan.explain_plan(false).await.unwrap(); + assert!( + !plan.contains("ANNIvfBatch"), + "HNSW batch search must not use the shared-scan batch node, got:\n{}", + plan + ); assert!( plan.contains("ANNSubIndex"), - "batch KNN should use the vector index when available, got:\n{}", + "HNSW batch search should fall back to the per-query indexed loop, got:\n{}", + plan + ); + + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, Some(2)) + .await; + } + + /// Regression test for cosine batch search: each query vector must be + /// normalized independently. The two queries below have very different + /// magnitudes, so normalizing the concatenated batch key by a single global + /// norm (the bug) would scale them unequally and diverge from per-query + /// single search. + #[tokio::test] + async fn test_batch_knn_indexed_cosine_normalizes_per_query() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + test_ds + .make_vector_index_with_metric(MetricType::Cosine) + .await + .unwrap(); + let dataset = &test_ds.dataset; + + // q0: small-magnitude constant direction; q1: large-magnitude ramp. + let mut query_values = vec![0.05f32; 32]; + query_values.extend((1..=32).map(|v| v as f32 * 3.0)); + let queries = + FixedSizeListArray::try_new_from_values(Float32Array::from(query_values.clone()), 32) + .unwrap(); + + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, 2).unwrap(); + scan.nprobes(2); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ANNIvfBatch"), + "cosine IVF batch KNN should use the shared-scan batch node, got:\n{}", plan ); + + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_matches_single_queries(dataset, &batch, &query_values, 2, true, None, Some(2)) + .await; + } + + /// Batch indexed search builds a single shared prefilter for all queries; + /// results must match per-query single search with the same prefilter. + #[tokio::test] + async fn test_batch_knn_indexed_with_prefilter() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) + .await + .unwrap(); + test_ds.make_vector_index().await.unwrap(); + let dataset = &test_ds.dataset; + let (queries, query_values) = batch_knn_two_queries(); + let k = 2; + + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + scan.nprobes(2); + scan.filter("i > 100").unwrap(); + scan.prefilter(true); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); assert!( - !plan.contains("KNNVectorDistance: queries=2"), - "indexed batch KNN should not force the flat batch path, got:\n{}", + plan.contains("ANNIvfBatch"), + "prefiltered IVF batch KNN should use the shared-scan batch node, got:\n{}", plan ); let batch = scan.try_into_batch().await.unwrap(); assert_query_index_field(&batch); - assert_eq!( - batch[QUERY_INDEX_COL].as_primitive::().values(), - &[0, 0, 1, 1] + // The shared prefilter must exclude i <= 100 for every query. + assert!( + batch["i"] + .as_primitive::() + .values() + .iter() + .all(|i| *i > 100), + "shared prefilter should remove rows with i <= 100" ); - let batch = dataset - .scan() - .nearest("vec", &queries, 2) - .unwrap() - .distance_range(Some(1.0), None) - .project(&["i"]) - .unwrap() - .try_into_batch() + let query_indices = batch[QUERY_INDEX_COL].as_primitive::(); + for query_index in 0..2 { + let query = + Float32Array::from(query_values[query_index * 32..(query_index + 1) * 32].to_vec()); + let single = dataset + .scan() + .nearest("vec", &query, k) + .unwrap() + .nprobes(2) + .filter("i > 100") + .unwrap() + .prefilter(true) + .project(&["i"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mask = BooleanArray::from_iter( + query_indices + .iter() + .map(|v| v.map(|v| v == query_index as i32)), + ); + let slice = arrow::compute::filter_record_batch(&batch, &mask).unwrap(); + assert_eq!( + slice["i"].as_primitive::().values(), + single["i"].as_primitive::().values(), + "prefiltered batch query {query_index} should match single-query search" + ); + } + } + + /// Batch indexed search must merge each query's top-k across multiple delta + /// indices, not just within a single delta. + #[tokio::test] + async fn test_batch_knn_indexed_multiple_deltas() { + let mut test_ds = TestVectorDataset::new(LanceFileVersion::Stable, true) .await .unwrap(); - assert_batch_matches_single_queries( - dataset, - &batch, - &query_values, - 2, - true, - Some((Some(1.0), None)), - ) - .await; + test_ds.make_vector_index().await.unwrap(); + // Append new data and optimize with `append` to add a second delta + // index (rather than merging into the existing one). + test_ds.append_data_with_range(400, 480).await.unwrap(); + test_ds + .dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + let dataset = &test_ds.dataset; + let segments = dataset.load_indices_by_name("idx").await.unwrap(); + assert!( + segments.len() >= 2, + "expected multiple delta index segments to exercise cross-delta merge, got {}", + segments.len() + ); + + let (queries, query_values) = batch_knn_two_queries(); + let k = 3; + let mut scan = dataset.scan(); + scan.nearest("vec", &queries, k).unwrap(); + scan.nprobes(2); + scan.project(&["i"]).unwrap(); + + let plan = scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ANNIvfBatch"), + "multi-delta IVF batch KNN should use the shared-scan batch node, got:\n{}", + plan + ); + + let batch = scan.try_into_batch().await.unwrap(); + assert_query_index_field(&batch); + assert_batch_matches_single_queries(dataset, &batch, &query_values, k, true, None, Some(2)) + .await; } #[tokio::test] @@ -6878,6 +10808,57 @@ mod test { assert_eq!(expected_i, actual_i); } + #[tokio::test(flavor = "multi_thread")] + async fn test_flat_knn_large_limit_preserves_global_order() { + // Regression test for https://github.com/lance-format/lance/issues/7865. + // + // An exact (flat, no vector index) KNN search with a limit larger than one + // output batch (BATCH_SIZE_FALLBACK = 8192 rows) used to be able to return + // results in the wrong global order: `execute_plan` coalesced the + // partitions the physical optimizer parallelizes above the top-k `SortExec` + // with a plain `CoalescePartitionsExec`, which does not preserve order. + // This only reproduces at real (> 1) parallelism, which is why the + // plan-shape tests elsewhere (pinned to `target_parallelism(1)`) never + // caught it. + let dim = 16u32; + let frag_count = 4u32; + let rows_per_fragment = 5_000u32; + let k = 12_000usize; // > BATCH_SIZE_FALLBACK, so results span multiple batches + + let dataset = gen_batch() + .col("vec", array::rand_vec::(Dimension::from(dim))) + .into_ram_dataset( + FragmentCount::from(frag_count), + FragmentRowCount::from(rows_per_fragment), + ) + .await + .unwrap(); + + let query = Float32Array::from(vec![0.0_f32; dim as usize]); + + // The bug is a scheduling race between parallel partitions, so run + // several iterations to reliably catch it if the ordering guarantee + // regresses. + for _ in 0..10 { + let mut scan = dataset.scan(); + scan.nearest("vec", &query, k).unwrap(); + scan.target_parallelism(8); + + let batch = scan.try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), k); + + let distances = batch[DIST_COL].as_primitive::(); + for pair in distances.values().windows(2) { + assert!( + pair[0] <= pair[1], + "flat KNN results must be globally sorted by distance, found {} before {}", + pair[0], + pair[1] + ); + } + } + } + #[rstest] #[tokio::test] async fn test_refine_factor( @@ -7107,6 +11088,30 @@ mod test { assert_eq!(expected_row_ids, actual_row_ids); } + #[tokio::test] + async fn test_filter_legacy_dataset_with_stable_row_ids() { + let test_ds = TestVectorDataset::new(LanceFileVersion::Legacy, true) + .await + .unwrap(); + + let batch = test_ds + .dataset + .scan() + .batch_readahead(get_num_compute_intensive_cpus()) + .project(&["vec"]) + .unwrap() + .with_row_id() + .filter_expr(col("vec").is_not_null()) + .try_into_batch() + .await + .unwrap(); + + let row_ids = batch[ROW_ID].as_primitive::(); + let unique_row_ids = row_ids.values().iter().copied().collect::>(); + assert_eq!(unique_row_ids.len(), 400); + assert_eq!(row_ids.len(), unique_row_ids.len()); + } + #[tokio::test] async fn test_scan_unordered_with_row_id() { // This test doesn't make sense for v2 files, there is no way to get an out-of-order scan @@ -7439,12 +11444,44 @@ mod test { #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] data_storage_version: LanceFileVersion, #[values(false, true)] stable_row_ids: bool, + #[values(ApproxMode::Normal, ApproxMode::Fast)] approx_mode: ApproxMode, #[values( - VectorIndexParams::ivf_pq(2, 8, 2, MetricType::L2, 2), + VectorIndexParams::ivf_pq(2, 4, 2, MetricType::L2, 2), + VectorIndexParams::ivf_hnsw( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16) + ), + VectorIndexParams::with_ivf_hnsw_pq_params( + MetricType::L2, + IvfBuildParams { + num_partitions: Some(2), + max_iters: 2, + sample_rate: 2, + ..Default::default() + }, + HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16), + PQBuildParams { + num_sub_vectors: 2, + num_bits: 4, + max_iters: 2, + sample_rate: 2, + ..Default::default() + } + ), VectorIndexParams::with_ivf_hnsw_sq_params( MetricType::L2, IvfBuildParams::new(2), - HnswBuildParams::default(), + HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16), SQBuildParams::default() ) )] @@ -7460,13 +11497,13 @@ mod test { ArrowField::new("vector", fixed_size_list_type(2, DataType::Float32), true), ])); - let vector_values = Float32Array::from_iter_values((0..600).map(|x| x as f32)); + let vector_values = Float32Array::from_iter_values((0..64).map(|x| x as f32)); let batches = vec![ RecordBatch::try_new( schema.clone(), vec![ - Arc::new(Int32Array::from_iter_values(0..300)), + Arc::new(Int32Array::from_iter_values(0..32)), Arc::new(FixedSizeListArray::try_new_from_values(vector_values, 2).unwrap()), ], ) @@ -7475,7 +11512,7 @@ mod test { let write_params = WriteParams { data_storage_version: Some(data_storage_version), - max_rows_per_file: 300, // At least two files to make sure stable row ids make a difference + max_rows_per_file: 16, // At least two files to make sure stable row ids make a difference enable_stable_row_ids: stable_row_ids, ..Default::default() }; @@ -7493,8 +11530,9 @@ mod test { let mut scan = dataset.scan(); scan.filter("filterable > 5").unwrap(); scan.nearest("vector", query_key.as_ref(), 1).unwrap(); - scan.minimum_nprobes(100); - scan.ef(100); + scan.minimum_nprobes(2); + scan.ef(16); + scan.approx_mode(approx_mode); scan.with_row_id(); let batches = scan @@ -8387,6 +12425,14 @@ mod test { } } + fn uses_legacy_scan(&self) -> bool { + self.dataset + .manifest() + .data_storage_format + .lance_file_format() + == lance_file::version::ConcreteFileVersion::V1 + } + async fn check_vector_scalar_indexed_and_refine(&self, params: &ScalarTestParams) { let (query_plan, batch) = self .run_query( @@ -8396,7 +12442,7 @@ mod test { ) .await; // Materialization is always required if there is a refine - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { assert!(query_plan.contains("MaterializeIndex")); } // The result should not include the sample query @@ -8428,7 +12474,7 @@ mod test { let (query_plan, batch) = self .run_query("indexed != 50", Some(self.sample_query()), params) .await; - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { if params.use_index { // An ANN search whose prefilter is fully satisfied by the index should be // able to use a ScalarIndexQuery @@ -8477,7 +12523,7 @@ mod test { async fn check_simple_indexed_only(&self, params: &ScalarTestParams) { let (query_plan, batch) = self.run_query("indexed != 50", None, params).await; // Materialization is always required for non-vector search - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { assert!(query_plan.contains("MaterializeIndex")); } else { assert!(query_plan.contains("LanceRead")); @@ -8518,7 +12564,7 @@ mod test { params ).await; // Materialization is always required for non-vector search - if self.dataset.is_legacy_storage() { + if self.uses_legacy_scan() { assert!(query_plan.contains("MaterializeIndex")); } else { assert!(query_plan.contains("LanceRead")); @@ -8562,6 +12608,8 @@ mod test { #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] data_storage_version: LanceFileVersion, #[values(false, true)] use_stable_row_ids: bool, + #[values(false, true)] use_index: bool, + #[values(false, true)] use_projection: bool, ) { let fixture = Box::pin(ScalarIndexTestFixture::new( data_storage_version, @@ -8569,42 +12617,36 @@ mod test { )) .await; - for use_index in [false, true] { - for use_projection in [false, true] { - for use_deleted_data in [false, true] { - for use_new_data in [false, true] { - // Don't test compaction in conjunction with deletion and new data, it's too - // many combinations with no clear benefit. Feel free to update if there is - // a need - // TODO: enable compaction for stable row id once supported. - let compaction_choices = - if use_deleted_data || use_new_data || use_stable_row_ids { - vec![false] - } else { - vec![false, true] + for use_deleted_data in [false, true] { + for use_new_data in [false, true] { + // Don't test compaction in conjunction with deletion and new data, it's too + // many combinations with no clear benefit. Feel free to update if there is + // a need + // TODO: enable compaction for stable row id once supported. + let compaction_choices = if use_deleted_data || use_new_data || use_stable_row_ids { + vec![false] + } else { + vec![false, true] + }; + for use_compaction in compaction_choices { + let updated_choices = if use_deleted_data || use_new_data || use_compaction { + vec![false] + } else { + vec![false, true] + }; + for use_updated in updated_choices { + for with_row_id in [false, true] { + let params = ScalarTestParams { + use_index, + use_projection, + use_deleted_data, + use_new_data, + with_row_id, + use_compaction, + use_updated, }; - for use_compaction in compaction_choices { - let updated_choices = - if use_deleted_data || use_new_data || use_compaction { - vec![false] - } else { - vec![false, true] - }; - for use_updated in updated_choices { - for with_row_id in [false, true] { - let params = ScalarTestParams { - use_index, - use_projection, - use_deleted_data, - use_new_data, - with_row_id, - use_compaction, - use_updated, - }; - fixture.check_vector_queries(¶ms).await; - fixture.check_simple_queries(¶ms).await; - } - } + fixture.check_vector_queries(¶ms).await; + fixture.check_simple_queries(¶ms).await; } } } @@ -8669,7 +12711,7 @@ mod test { .col("ngram", array::rand_utf8(ByteCount::from(5), false)) .col("exact", array::rand_type(&DataType::UInt32)) .col("no_index", array::rand_type(&DataType::UInt32)) - .into_reader_rows(RowCount::from(1000), BatchCount::from(5)); + .into_reader_rows(RowCount::from(32), BatchCount::from(2)); let mut dataset = Dataset::write(data, "memory://test", None).await.unwrap(); dataset @@ -8871,6 +12913,56 @@ mod test { "expected ngram index usage for `{filter}`, got plan:\n{plan_str}" ); } + + // contains with >= 3 characters (uses index) + assert_eq!( + matched(&dataset, "contains(s, 'rhino')").await, + ["rhino", "rhino horn", "rhinos nose"] + ); + assert_eq!( + matched(&dataset, "contains(s, 'cat')").await, + ["cat", "cat dog", "catalog", "category", "scatter"] + ); + + // contains with < 3 characters (must NOT use index, i.e., falls back to full scan, and returns correct results) + assert_eq!( + matched(&dataset, "contains(s, 'ca')").await, + ["cat", "cat dog", "catalog", "category", "scatter"] + ); + assert_eq!( + matched(&dataset, "contains(s, 'a')").await, + [ + "cat", "cat dog", "catalog", "category", "dogma", "elephant", "scatter" + ] + ); + + // Verify index is used for contains >= 3 characters, but NOT used for < 3 characters + for filter in ["contains(s, 'rhino')", "contains(s, 'cat')"] { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + let plan_str = format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + assert!( + plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"), + "expected ngram index usage for `{filter}`, got plan:\n{plan_str}" + ); + } + for filter in ["contains(s, 'ca')", "contains(s, 'a')"] { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + let plan_str = format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ); + assert!( + !plan_str.contains("ScalarIndexQuery"), + "expected NO ngram index usage for `{filter}`, got plan:\n{plan_str}" + ); + } } #[tokio::test] @@ -8926,6 +13018,85 @@ mod test { assert_eq!(count(&dataset, "regexp_like(text, 'a.b')").await, 20); } + #[tokio::test] + async fn test_ngram_contains_untokenizable_needle() { + // A needle the tokenizer cannot turn into a trigram must fall back to a + // full recheck rather than silently matching nothing. Byte length is + // not a usable proxy for this: "éé" is two characters but four bytes. + let unit = ["ééb", "aéé", "abc", "dog", "éab"]; + let values: Vec<&str> = unit.iter().copied().cycle().take(60).collect(); + let array = StringArray::from_iter_values(values); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "text", + DataType::Utf8, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(array)]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + max_rows_per_file: 20, // 60 rows -> 3 fragments + ..Default::default() + }; + let mut dataset = Dataset::write( + reader, + "memory://test_ngram_contains_utf8", + Some(write_params), + ) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::NGram, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + async fn count(dataset: &Dataset, filter: &str) -> usize { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let batches = scan + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches.iter().map(|b| b.num_rows()).sum() + } + + async fn plan_of(dataset: &Dataset, filter: &str) -> String { + let mut scan = dataset.scan(); + scan.filter(filter).unwrap(); + let plan = scan.create_plan().await.unwrap(); + format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ) + } + + // Each unit value appears 12 times in the 60 rows. + // Two characters (four bytes): shorter than a trigram, so the index is + // bypassed and "ééb" / "aéé" are still found. + assert_eq!(count(&dataset, "contains(text, 'éé')").await, 24); + let plan_str = plan_of(&dataset, "contains(text, 'éé')").await; + assert!( + !plan_str.contains("ScalarIndexQuery"), + "expected NO ngram index usage for a two character needle, got plan:\n{plan_str}" + ); + + // Three characters: long enough to tokenize, so the index is used. + assert_eq!(count(&dataset, "contains(text, 'ééb')").await, 12); + let plan_str = plan_of(&dataset, "contains(text, 'ééb')").await; + assert!( + plan_str.contains("ScalarIndexQuery") && plan_str.contains("NGram"), + "expected ngram index usage for a three character needle, got plan:\n{plan_str}" + ); + } + #[tokio::test] async fn test_like_prefix_with_btree_index() { // Create dataset with string data that has various prefixes @@ -9978,6 +14149,30 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") ); } + /// A physical deleted-row scan cannot late-materialize: the take would + /// silently drop the tombstone rows (null row id), so the row-stream + /// read must reject the forwarded flag at plan time + #[tokio::test] + async fn test_include_deleted_rows_rejects_late_materialization() { + let data = gen_batch() + .col("i", array::step::()) + .col("payload", array::step::()) + .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://test", None).await.unwrap(); + dataset.delete("i = 5").await.unwrap(); + + let mut scan = dataset.scan(); + scan.project(&["payload"]) + .unwrap() + .filter("i > 2") + .unwrap() + .with_row_id() + .include_deleted_rows() + .materialization_style(MaterializationStyle::AllLate); + let err = scan.create_plan().await.unwrap_err(); + assert!(err.to_string().contains("with_deleted_rows"), "{err}"); + } + #[rstest] #[tokio::test] async fn test_late_materialization( @@ -10079,6 +14274,154 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") assert_io_lt!(io_stats, read_bytes, index_scan_bytes); } + #[tokio::test] + async fn test_blob_all_binary_late_materialization() { + // A selective filter that projects a blob column with `blob_handling=all_binary` + // must late-materialize the blob (take only the matched rows) rather than eagerly + // reading the whole column. Blobs returned as descriptions stay eager (they are + // tiny), but full binary values should follow the width-based heuristic like any + // other wide column. + use lance_io::assert_io_lt; + use lance_table::io::commit::RenameCommitHandler; + + // 8KB stays under the 64KB inline threshold, so the blob is a normal column in + // the data file rather than a dedicated blob file. + let blob_meta = HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]); + let blobs = array::rand_fixedbin(ByteCount::from(8 * 1024), true).with_metadata(blob_meta); + let data = gen_batch() + .col("filterme", array::step::()) + .col("blobs", blobs) + .into_reader_rows(RowCount::from(500), BatchCount::from(8)); + + let dataset = Dataset::write( + data, + "memory://test", + Some(WriteParams { + commit_handler: Some(Arc::new(RenameCommitHandler)), + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Baseline: read the blob column as binary for the whole table. + let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset + dataset + .scan() + .project(&["blobs"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .try_into_batch() + .await + .unwrap(); + let full_scan_bytes = dataset + .object_store + .as_ref() + .io_stats_incremental() + .read_bytes; + + // A filter matching a single row out of 4000 should read far less than the whole + // column: only the filter leaf plus the one materialized blob. + dataset + .scan() + .project(&["blobs"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .filter("filterme = 100") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let io_stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!(io_stats, read_bytes, full_scan_bytes); + } + + #[tokio::test] + async fn test_nested_blob_all_binary_late_materialization() { + // Same as above, but the blob is a leaf *inside* a struct and the filter is on a + // sibling leaf. Materialization is decided per leaf (fields_pre_order), so the + // nested blob must late-materialize under `all_binary` just like a top-level one. + use lance_io::assert_io_lt; + use lance_table::io::commit::RenameCommitHandler; + + let blob_meta = HashMap::from([("lance-encoding:blob".to_string(), "true".to_string())]); + let a_field = ArrowField::new("a", DataType::Int32, false); + let blob_field = + ArrowField::new("blob", DataType::LargeBinary, false).with_metadata(blob_meta); + let struct_fields: Fields = vec![a_field, blob_field].into(); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "s", + DataType::Struct(struct_fields.clone()), + false, + )])); + + let rows_per_batch = 500usize; + let batches: Vec = (0..8) + .map(|b| { + let base = (b * rows_per_batch) as i32; + let a = Arc::new(Int32Array::from_iter_values( + base..base + rows_per_batch as i32, + )); + // Vary the payload per row so it does not collapse under compression. + let blobs: Vec> = (0..rows_per_batch) + .map(|r| { + let seed = (base as usize + r).wrapping_mul(2654435761); + (0usize..8 * 1024) + .map(|i| (i.wrapping_mul(31).wrapping_add(seed) & 0xff) as u8) + .collect() + }) + .collect(); + let blob = Arc::new(arrow_array::LargeBinaryArray::from_iter_values( + blobs.iter().map(|v| v.as_slice()), + )); + let s = StructArray::new(struct_fields.clone(), vec![a, blob as ArrayRef], None); + RecordBatch::try_new(schema.clone(), vec![Arc::new(s)]).unwrap() + }) + .collect(); + + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + let dataset = Dataset::write( + reader, + "memory://test", + Some(WriteParams { + commit_handler: Some(Arc::new(RenameCommitHandler)), + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }), + ) + .await + .unwrap(); + + let _ = dataset.object_store.as_ref().io_stats_incremental(); // reset + dataset + .scan() + .project(&["s"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .try_into_batch() + .await + .unwrap(); + let full_scan_bytes = dataset + .object_store + .as_ref() + .io_stats_incremental() + .read_bytes; + + dataset + .scan() + .project(&["s"]) + .unwrap() + .blob_handling(BlobHandling::AllBinary) + .filter("s.a = 100") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let io_stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!(io_stats, read_bytes, full_scan_bytes); + } + #[rstest] #[tokio::test] async fn test_project_nested( @@ -10185,9 +14528,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[i], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[s@2 as s] - Take: columns=\"i, _rowid, (s)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: ..., projection=[i], num_fragments=2, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10) AND i < Int32(20), refine_filter=i > Int32(10) AND i < Int32(20)" + LanceRead: uri=..., projection=[s], source=stream(_rowid) + LanceRead: ..., projection=[i], num_fragments=2, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10) AND i < Int32(20), refine_filter=i > Int32(10) AND i < Int32(20)" }; assert_plan_equals( &dataset.dataset, @@ -10211,10 +14553,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[i, s], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[i@0 as i, s@1 as s, vec@3 as vec] - Take: columns=\"i, s, _rowid, (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[i, s], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + LanceRead: uri=..., projection=[i, s], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" }; assert_plan_equals( &dataset.dataset, @@ -10254,10 +14595,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[i@2 as i, s@0 as s, vec@3 as vec] - Take: columns=\"s, _rowid, (i), (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[s], num_fragments=2, range_before=None, \ - range_after=None, row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" + LanceRead: uri=..., projection=[i, vec], source=stream(_rowid) + LanceRead: uri=..., projection=[s], num_fragments=2, range_before=None, \ + range_after=None, row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" }; assert_plan_equals( &dataset.dataset, @@ -10279,10 +14619,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri..., projection=[s, vec], row_id=true, row_addr=false, ordered=true, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@0 as s, vec@1 as vec] - Take: columns=\"s, vec, _rowid, (i)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[s, vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" + LanceRead: uri=..., projection=[i], source=stream(_rowid) + LanceRead: uri=..., projection=[s, vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=s IS NOT NULL, refine_filter=s IS NOT NULL" }; assert_plan_equals( &dataset.dataset, @@ -10325,13 +14664,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance] - Take: columns=\"vec, _rowid, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: _distance@2 IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=--, refine_filter=--" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@2 IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=--, refine_filter=--" }; assert_plan_equals( &dataset.dataset, @@ -10355,14 +14693,13 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance] - Take: columns=\"vec, _rowid, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - GlobalLimitExec: skip=0, fetch=1 - FilterExec: _distance@2 IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=--, refine_filter=--" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + GlobalLimitExec: skip=0, fetch=1 + FilterExec: _distance@2 IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=--, refine_filter=--" }; assert_plan_equals( &dataset.dataset, @@ -10375,13 +14712,20 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") // --------------------------------------------------------------------- dataset.make_vector_index().await?; log::info!("Test case: Basic ANN"); - let expected = + let expected = if data_storage_version == LanceFileVersion::Legacy { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] Take: columns=\"_distance, _rowid, (i), (s), (vec)\" CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=42), expr=... ANNSubIndex: name=..., k=42, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=42), expr=... + ANNSubIndex: name=..., k=42, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| scan.nearest("vec", &q, 42), @@ -10390,7 +14734,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: ANN with refine"); - let expected = + let expected = if data_storage_version == LanceFileVersion::Legacy { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 @@ -10401,7 +14745,18 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=40), expr=... ANNSubIndex: name=..., k=40, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=10), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=40), expr=... + ANNSubIndex: name=..., k=40, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| Ok(scan.nearest("vec", &q, 10)?.refine(4)), @@ -10421,13 +14776,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@0 as vec, _distance@2 as _distance] - Take: columns=\"vec, _rowid, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: _distance@... IS NOT NULL - SortExec: TopK(fetch=13), expr=... - KNNVectorDistance: metric=l2 - LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=--, refine_filter=--" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=13), expr=... + KNNVectorDistance: metric=l2 + LanceRead: uri=..., projection=[vec], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=--, refine_filter=--" }; assert_plan_equals( &dataset.dataset, @@ -10437,7 +14791,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: ANN with postfilter"); - let expected = "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid] Take: columns=\"_distance, _rowid, i, (s), (vec)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: i@2 > 10 @@ -10445,7 +14800,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=17), expr=... ANNSubIndex: name=..., k=17, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[s@3 as s, vec@4 as vec, _distance@0 as _distance, _rowid@1 as _rowid] + LanceRead: uri=..., projection=[s, vec], source=stream(_rowid) + FilterExec: i@2 > 10 + LanceRead: uri=..., projection=[i], source=stream(_rowid) + SortExec: TopK(fetch=17), expr=... + ANNSubIndex: name=..., k=17, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10471,13 +14835,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] - Take: columns=\"_distance, _rowid, (i), (s), (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=17), expr=... - ANNSubIndex: name=..., k=17, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10) + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=17), expr=... + ANNSubIndex: name=..., k=17, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10) " }; assert_plan_equals( @@ -10494,7 +14857,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") dataset.append_new_data().await?; log::info!("Test case: Combined KNN/ANN"); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: _distance@... IS NOT NULL @@ -10511,7 +14875,25 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=6), expr=... ANNSubIndex: name=..., k=6, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=6), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=6), expr=... + KNNVectorDistance: metric=l2 + LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=6), expr=... + ANNSubIndex: name=..., k=6, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| scan.nearest("vec", &q, 6), @@ -10523,7 +14905,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") // new data and with filter log::info!("Test case: Combined KNN/ANN with postfilter"); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, i, (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: i@3 > 10 @@ -10543,7 +14926,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=15), expr=... ANNSubIndex: name=..., k=15, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1"; + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + FilterExec: i@3 > 10 + LanceRead: uri=..., projection=[i], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=15), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@2 as _distance, _rowid@1 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=15), expr=... + KNNVectorDistance: metric=l2 + LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=15), expr=... + ANNSubIndex: name=..., k=15, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1" + }; assert_plan_equals( &dataset.dataset, |scan| scan.nearest("vec", &q, 15)?.filter("i > 10"), @@ -10577,26 +14980,24 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] - Take: columns=\"_rowid, vec, _distance, (i), (s)\" - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: _distance@... IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - CoalescePartitionsExec - UnionExec - ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] - FilterExec: _distance@... IS NOT NULL - SortExec: TopK(fetch=5), expr=... - KNNVectorDistance: metric=l2 - FilterExec: i@1 > 10 - LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None - Take: columns=\"_distance, _rowid, (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=5), expr=... - ANNSubIndex: name=..., k=5, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ - row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=5), expr=... + KNNVectorDistance: metric=l2 + FilterExec: i@1 > 10 + LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=5), expr=... + ANNSubIndex: name=..., k=5, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + LanceRead: uri=..., projection=[], num_fragments=2, range_before=None, range_after=None, \ + row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" }; assert_plan_equals( &dataset.dataset, @@ -10619,14 +15020,22 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") dataset.make_scalar_index().await?; log::info!("Test case: ANN with scalar index"); - let expected = + let expected = if data_storage_version == LanceFileVersion::Legacy { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] Take: columns=\"_distance, _rowid, (i), (s), (vec)\" CoalesceBatchesExec: target_batch_size=8192 SortExec: TopK(fetch=5), expr=... ANNSubIndex: name=..., k=5, deltas=1, metric=L2 ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"; + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + } else { + "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=5), expr=... + ANNSubIndex: name=..., k=5, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10651,13 +15060,12 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None" } else { "ProjectionExec: expr=[i@2 as i, s@3 as s, vec@4 as vec, _distance@0 as _distance] - Take: columns=\"_distance, _rowid, (i), (s), (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=5), expr=... - ANNSubIndex: name=..., k=5, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - LanceRead: uri=..., projection=[], num_fragments=3, range_before=None, \ - range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" + LanceRead: uri=..., projection=[i, s, vec], source=stream(_rowid) + SortExec: TopK(fetch=5), expr=... + ANNSubIndex: name=..., k=5, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + LanceRead: uri=..., projection=[], num_fragments=3, range_before=None, \ + range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)" }; assert_plan_equals( &dataset.dataset, @@ -10675,7 +15083,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") dataset.append_new_data().await?; log::info!("Test case: Combined KNN/ANN with scalar index"); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: _distance@... IS NOT NULL @@ -10694,7 +15103,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: TopK(fetch=8), expr=... ANNSubIndex: name=..., k=8, deltas=1, metric=L2 ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"; + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=8), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=8), expr=... + KNNVectorDistance: metric=l2 + FilterExec: i@1 > 10 + LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=8), expr=... + ANNSubIndex: name=..., k=8, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10711,7 +15140,8 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") log::info!( "Test case: Combined KNN/ANN with updated scalar index and outdated vector index" ); - let expected = "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + let expected = if data_storage_version == LanceFileVersion::Legacy { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] Take: columns=\"_rowid, vec, _distance, (i), (s)\" CoalesceBatchesExec: target_batch_size=8192 FilterExec: _distance@... IS NOT NULL @@ -10730,7 +15160,27 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: TopK(fetch=11), expr=... ANNSubIndex: name=..., k=11, deltas=1, metric=L2 ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"; + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + } else { + "ProjectionExec: expr=[i@3 as i, s@4 as s, vec@1 as vec, _distance@2 as _distance] + LanceRead: uri=..., projection=[i, s], source=stream(_rowid) + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=11), expr=... + KNNVectorDistance: metric=l2 + CoalescePartitionsExec + UnionExec + ProjectionExec: expr=[_distance@3 as _distance, _rowid@2 as _rowid, vec@0 as vec] + FilterExec: _distance@... IS NOT NULL + SortExec: TopK(fetch=11), expr=... + KNNVectorDistance: metric=l2 + FilterExec: i@1 > 10 + LanceScan: uri=..., projection=[vec, i], row_id=true, row_addr=false, ordered=false, range=None + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=11), expr=... + ANNSubIndex: name=..., k=11, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1 + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)" + }; dataset.make_scalar_index().await?; assert_plan_equals( &dataset.dataset, @@ -10776,10 +15226,9 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .filter("i > 10") }, "ProjectionExec: expr=[s@2 as s] - Take: columns=\"i, _rowid, (s)\" - CoalesceBatchesExec: target_batch_size=8192 - LanceRead: uri=..., projection=[i], num_fragments=4, range_before=None, \ - range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)", + LanceRead: uri=..., projection=[s], source=stream(_rowid) + LanceRead: uri=..., projection=[i], num_fragments=4, range_before=None, \ + range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=i > Int32(10)", ) .await?; } @@ -10888,10 +15337,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") // All rows are indexed dataset.make_fts_index().await?; log::info!("Test case: Full text search (match query)"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello"#; + MatchQuery: column=s, query=[hello]"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + MatchQuery: column=s, query=[hello]"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10904,10 +15359,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: Full text search (phrase query)"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - PhraseQuery: column=s, query=hello world"#; + PhraseQuery: column=s, query=hello world"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + PhraseQuery: column=s, query=hello world"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10921,12 +15382,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: Full text search (boost query)"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - BoostQuery: negative_boost=1 - MatchQuery: column=s, query=hello - MatchQuery: column=s, query=world"#; + CompoundFtsScorer: query=Boosting(positive=Match(MatchQuery { column: Some("s"), terms: "hello", ... }), negative=Match(MatchQuery { column: Some("s"), terms: "world", ... }), negative_boost=1)"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + CompoundFtsScorer: query=Boosting(positive=Match(MatchQuery { column: Some("s"), terms: "hello", ... }), negative=Match(MatchQuery { column: Some("s"), terms: "world", ... }), negative_boost=1)"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -10948,7 +15413,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello + MatchQuery: column=s, query=[hello] CoalescePartitionsExec UnionExec MaterializeIndex: query=[i > 10]@i_idx(BTree) @@ -10957,11 +15422,10 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i], row_id=true, row_addr=false, ordered=false, range=None"# } else { r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] - Take: columns="_rowid, _score, (s)" - CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# + LanceRead: uri=..., projection=[s], source=stream(_rowid) + MatchQuery: column=s, query=[hello] + LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# }; assert_plan_equals( &dataset.dataset, @@ -10988,19 +15452,18 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] CoalescePartitionsExec UnionExec - MatchQuery: column=s, query=hello + MatchQuery: column=s, query=[hello] FlatMatchQuery: column=s, query=hello LanceScan: uri=..., projection=[s], row_id=true, row_addr=false, ordered=true, range=None"# } else { r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] - Take: columns="_rowid, _score, (s)" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - UnionExec - MatchQuery: column=s, query=hello - FlatMatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"# + LanceRead: uri=..., projection=[s], source=stream(_rowid) + SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] + CoalescePartitionsExec + UnionExec + MatchQuery: column=s, query=[hello] + FlatMatchQuery: column=s, query=hello + LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=--, refine_filter=--"# }; dataset.append_new_data().await?; assert_plan_equals( @@ -11015,10 +15478,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await?; log::info!("Test case: Full text search with unindexed rows and fast_search"); - let expected = r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + let expected = if data_storage_version == LanceFileVersion::Legacy { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] Take: columns="_rowid, _score, (s)" CoalesceBatchesExec: target_batch_size=8192 - MatchQuery: column=s, query=hello"#; + MatchQuery: column=s, query=[hello]"# + } else { + r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] + LanceRead: uri=..., projection=[s], source=stream(_rowid) + MatchQuery: column=s, query=[hello]"# + }; assert_plan_equals( &dataset.dataset, |scan| { @@ -11045,7 +15514,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] CoalescePartitionsExec UnionExec - MatchQuery: column=s, query=hello + MatchQuery: column=s, query=[hello] CoalescePartitionsExec UnionExec MaterializeIndex: query=[i > 10]@i_idx(BTree) @@ -11063,17 +15532,16 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") LanceScan: uri=..., projection=[i, s], row_id=true, row_addr=false, ordered=false, range=None"# } else { r#"ProjectionExec: expr=[s@2 as s, _score@1 as _score, _rowid@0 as _rowid] - Take: columns="_rowid, _score, (s)" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - UnionExec - MatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- - ScalarIndexQuery: query=[i > 10]@i_idx(BTree) - FlatMatchQuery: column=s, query=hello - LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- - ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# + LanceRead: uri=..., projection=[s], source=stream(_rowid) + SortExec: expr=[_score@1 DESC NULLS LAST], preserve_partitioning=[false] + CoalescePartitionsExec + UnionExec + MatchQuery: column=s, query=[hello] + LanceRead: uri=..., projection=[], num_fragments=5, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- + ScalarIndexQuery: query=[i > 10]@i_idx(BTree) + FlatMatchQuery: column=s, query=hello + LanceRead: uri=..., projection=[s], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=false, full_filter=i > Int32(10), refine_filter=-- + ScalarIndexQuery: query=[i > 10]@i_idx(BTree)"# }; assert_plan_equals( &dataset.dataset, @@ -11150,11 +15618,10 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") SortExec: TopK(fetch=34), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]... KNNVectorDistance: metric=l2 LanceScan: uri=..., projection=[vec], row_id=true, row_addr=false, ordered=false, range=None - Take: columns=\"_distance, _rowid, (vec)\" - CoalesceBatchesExec: target_batch_size=8192 - SortExec: TopK(fetch=34), expr=[_distance@0 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]... - ANNSubIndex: name=idx, k=34, deltas=1, metric=L2 - ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1", + LanceRead: uri=..., projection=[vec], source=stream(_rowid) + SortExec: TopK(fetch=34), expr=[_distance@0 ASC NULLS LAST, _rowid@1 ASC NULLS LAST]... + ANNSubIndex: name=idx, k=34, deltas=1, metric=L2 + ANNIvfPartition: uuid=..., minimum_nprobes=1, maximum_nprobes=None, deltas=1", ) .await .unwrap(); @@ -11890,6 +16357,111 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await; } + #[tokio::test] + async fn test_filter_to_take_with_stable_row_ids() { + let ds = lance_datagen::gen_batch() + .col("idx", array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(3), + FragmentRowCount::from(4), + Some(WriteParams { + max_rows_per_file: 4, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let row_addrs = [ + (u64::from(RowAddress::new_from_parts(0, 1)), 1), + (u64::from(RowAddress::new_from_parts(1, 1)), 5), + (u64::from(RowAddress::new_from_parts(2, 1)), 9), + ]; + for (row_addr, expected_idx) in row_addrs { + let batch = ds + .scan() + .filter(&format!("{ROW_ADDR} = {row_addr}")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + batch["idx"].as_primitive::().values(), + &[expected_idx] + ); + } + + let batch = ds + .scan() + .filter(&format!( + "{ROW_ADDR} IN ({}, {})", + row_addrs[1].0, row_addrs[2].0 + )) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch["idx"].as_primitive::().values(), &[5, 9]); + + let batch = ds + .scan() + .filter(&format!("{ROW_ADDR} = 5")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch.num_rows(), 0); + + let batch = ds + .scan() + .filter(&format!("{ROW_OFFSET} IN (5, 9)")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch["idx"].as_primitive::().values(), &[5, 9]); + } + + #[tokio::test] + async fn test_stale_row_address_does_not_follow_stable_id_after_update() { + let ds = lance_datagen::gen_batch() + .col("idx", array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let old_row_addr = u64::from(RowAddress::new_from_parts(0, 1)); + let ds = crate::dataset::UpdateBuilder::new(Arc::new(ds)) + .update_where("idx = 1") + .unwrap() + .set("idx", "101") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + + let batch = ds + .scan() + .filter(&format!("{ROW_ADDR} = {old_row_addr}")) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(batch.num_rows(), 0); + } + #[tokio::test] async fn test_filter_to_take() { let mut ds = lance_datagen::gen_batch() @@ -12334,7 +16906,7 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") } fn find_filtered_read(plan: &dyn ExecutionPlan) -> Option<&FilteredReadExec> { - if let Some(f) = plan.as_any().downcast_ref::() { + if let Some(f) = plan.downcast_ref::() { return Some(f); } for child in plan.children() { @@ -12371,6 +16943,84 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") assert_eq!(filtered.options().io_buffer_size_bytes, Some(7777)); } + #[tokio::test] + async fn test_materialization_readahead_bytes_propagated() { + let data = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_reader_rows(RowCount::from(8), BatchCount::from(1)); + let dataset = Dataset::write(data, "memory://test_materialization_readahead_bytes", None) + .await + .unwrap(); + + let mut scanner = dataset.scan(); + scanner.materialization_readahead_bytes(7777); + let plan = scanner.create_plan().await.unwrap(); + let filtered = find_filtered_read(plan.as_ref()) + .expect("expected a FilteredReadExec in the scan plan"); + assert_eq!( + filtered.options().materialization_readahead_bytes, + Some(7777) + ); + } + + #[tokio::test] + async fn test_zero_materialization_readahead_bytes_rejected() { + let data = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_reader_rows(RowCount::from(8), BatchCount::from(1)); + let dataset = Dataset::write(data, "memory://test_zero_materialization_budget", None) + .await + .unwrap(); + let mut scanner = dataset.scan(); + scanner.materialization_readahead_bytes(0); + let err = scanner.create_plan().await.unwrap_err(); + assert!( + err.to_string() + .contains("materialization_readahead_bytes must be greater than 0") + ); + } + + #[tokio::test] + async fn test_batch_readahead_bounds_decode_concurrency() { + let data = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_reader_rows(RowCount::from(8), BatchCount::from(1)); + let dataset = Dataset::write(data, "memory://test_batch_readahead_concurrency", None) + .await + .unwrap(); + + // Default: threading mode falls back to get_num_compute_intensive_cpus(). + let plan = dataset.scan().create_plan().await.unwrap(); + let filtered = find_filtered_read(plan.as_ref()) + .expect("expected a FilteredReadExec in the scan plan"); + assert_eq!( + filtered.options().threading_mode, + FilteredReadThreadingMode::OnePartitionMultipleThreads(get_num_compute_intensive_cpus()), + ); + + // Explicit batch_readahead(N) bounds the decode fan-out to N. + let mut scanner = dataset.scan(); + scanner.batch_readahead(3); + let plan = scanner.create_plan().await.unwrap(); + let filtered = find_filtered_read(plan.as_ref()) + .expect("expected a FilteredReadExec in the scan plan"); + assert_eq!( + filtered.options().threading_mode, + FilteredReadThreadingMode::OnePartitionMultipleThreads(3), + ); + + let mut scanner = dataset.scan(); + scanner.batch_readahead(0); + let Err(Error::InvalidInput { source, .. }) = scanner.create_plan().await else { + panic!("expected batch_readahead=0 to be rejected"); + }; + assert!( + source + .to_string() + .contains("batch_readahead must be greater than 0") + ); + } + // The env var key scopes serial_test's lock so this test only blocks others // that touch LANCE_DEFAULT_IO_BUFFER_SIZE — unrelated tests still run in // parallel. @@ -12753,8 +17403,20 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") .await .unwrap(); - // Create FTS index on first 2 fragments - test_ds.make_fts_index().await.unwrap(); + // Create one FTS physical segment per indexed fragment. + test_ds.make_segmented_fts_index().await.unwrap(); + let expected_index_coverage = test_ds + .dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::(); + let fragment_bitmap = + fts_index_fragment_bitmap(&test_ds.dataset, "s", DocumentGranularity::Row) + .await + .unwrap() + .expect("segmented FTS index"); + assert_eq!(fragment_bitmap, expected_index_coverage); // Append two more unindexed fragments test_ds.append_data_with_range(400, 410).await.unwrap(); @@ -12766,6 +17428,41 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") assert_eq!(fragments.len(), 4); // "s-5" matches: s-5, s-50..s-59, s-150..s-159 (frag 0), s-250..s-259, s-350..s-359 (frag 1), s-405 (frag 2), s-415 (frag 3) + async fn fts_ids(dataset: &Dataset, fragments: Option>) -> Vec { + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new("s-5".into())) + .unwrap(); + if let Some(fragments) = fragments { + scanner.with_fragments(fragments); + } + let batch = scanner.try_into_batch().await.unwrap(); + let mut ids = batch + .column_by_name("i") + .unwrap() + .as_primitive::() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + let global_ids = fts_ids(&test_ds.dataset, None).await; + let mut fragmented_ids = Vec::with_capacity(global_ids.len()); + for (fragment, expected_range) in + fragments.iter().zip([0..200, 200..400, 400..410, 410..420]) + { + let ids = fts_ids(&test_ds.dataset, Some(vec![fragment.clone()])).await; + assert!( + !ids.is_empty() && ids.iter().all(|id| expected_range.contains(id)), + "fragment {} should return only matching rows in {expected_range:?}", + fragment.id, + ); + fragmented_ids.extend(ids); + } + fragmented_ids.sort_unstable(); + assert_eq!(fragmented_ids, global_ids); + test_fragment_list_filtering(&test_ds, fragments, |dataset| { let mut scanner = dataset.scan(); scanner @@ -12774,5 +17471,24 @@ full_filter=name LIKE Utf8(\"test%2\"), refine_filter=name LIKE Utf8(\"test%2\") scanner }) .await; + for (fragment, (phrase, expected_i)) in + fragments[..2].iter().zip([("s 5", 5), ("s 205", 205)]) + { + let mut scanner = test_ds.dataset.scan(); + scanner.with_fragments(vec![fragment.clone()]); + scanner + .full_text_search(FullTextSearchQuery::new_query( + PhraseQuery::new(phrase.to_string()) + .with_column(Some("s".to_string())) + .into(), + )) + .unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let i_array = batch + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_array.values(), &[expected_i]); + } } } diff --git a/rust/lance/src/dataset/schema_evolution.rs b/rust/lance/src/dataset/schema_evolution.rs index ce32362f324..af0e1ab2622 100644 --- a/rust/lance/src/dataset/schema_evolution.rs +++ b/rust/lance/src/dataset/schema_evolution.rs @@ -12,7 +12,7 @@ use super::{ transaction::{Operation, Transaction}, write::cleanup_data_fragments, }; -use crate::index::DatasetIndexExt; +use crate::index::load_all_indices; use crate::{Error, Result, io::exec::Planner}; use arrow::compute::CastOptions; use arrow::compute::can_cast_types; @@ -23,16 +23,16 @@ use datafusion::execution::SendableRecordBatchStream; use futures::stream::{StreamExt, TryStreamExt}; use lance_arrow::SchemaExt; use lance_core::datatypes::{Field, Schema}; +use lance_core::utils::parse::str_is_truthy; use lance_datafusion::utils::StreamingWriteSource; use lance_encoding::constants::{PACKED_STRUCT_LEGACY_META_KEY, PACKED_STRUCT_META_KEY}; -use lance_encoding::version::LanceFileVersion; -use lance_table::format::Fragment; +#[cfg(test)] +use lance_file::version::ConcreteFileVersion; +use lance_table::format::{Fragment, overlay::TOMBSTONE_FIELD_ID}; -mod optimize; +pub mod optimize; -use optimize::{ - ChainedNewColumnTransformOptimizer, NewColumnTransformOptimizer, SqlToAllNullsOptimizer, -}; +use optimize::{ChainedNewColumnTransformOptimizer, NewColumnTransformOptimizer}; async fn validate_no_nulls_before_making_non_nullable(dataset: &Dataset, path: &str) -> Result<()> { let field = dataset.schema().field(path).ok_or_else(|| { @@ -148,58 +148,33 @@ impl ColumnAlteration { } } -/// Limit casts to same type. This is mostly to filter out weird casts like -/// casting a string to a boolean or float to string. -fn is_upcast_downcast(from_type: &DataType, to_type: &DataType, version: LanceFileVersion) -> bool { - use DataType::*; - match (from_type, to_type) { - // Legacy storage cannot materialize a fresh Dictionary column via - // alter because the writer expects `field.dictionary` metadata to be - // pre-populated, which the alter pipeline does not compute. - (_, Dictionary(_, _)) if matches!(version, LanceFileVersion::Legacy) => false, - // These need to be in front - (Dictionary(_, from_value_type), _) => { - is_upcast_downcast(from_value_type, to_type, version) - } - (_, Dictionary(_, to_value_type)) => is_upcast_downcast(from_type, to_value_type, version), - (from, to) if from.is_integer() => to.is_integer(), - (from, to) if from.is_floating() => to.is_floating(), - (from, to) if from.is_temporal() => to.is_temporal(), - (Boolean, to) => matches!(to, Boolean), - (Utf8 | LargeUtf8, to) => matches!(to, Utf8 | LargeUtf8), - (Binary | LargeBinary, to) => matches!(to, Binary | LargeBinary), - (Decimal128(_, _) | Decimal256(_, _), to) => { - matches!(to, Decimal128(_, _) | Decimal256(_, _)) - } - (List(from_field) | LargeList(from_field) | FixedSizeList(from_field, _), to) => match to { - List(to_field) | LargeList(to_field) | FixedSizeList(to_field, _) => { - is_upcast_downcast(from_field.data_type(), to_field.data_type(), version) - } - _ => false, - }, - - _ => false, - } -} - trait ArrowFieldExt { fn is_packed(&self) -> bool; } +#[cfg(test)] +fn is_upcast_downcast( + from_type: &DataType, + to_type: &DataType, + version: ConcreteFileVersion, +) -> bool { + super::versions::is_upcast_downcast(version, from_type, to_type) +} + impl ArrowFieldExt for ArrowField { fn is_packed(&self) -> bool { let metadata = self.metadata(); metadata .get(PACKED_STRUCT_LEGACY_META_KEY) - .map(|v| v == "true") + .map(|v| str_is_truthy(v)) .unwrap_or(metadata.contains_key(PACKED_STRUCT_META_KEY)) } } -fn check_field_conflict( +pub fn check_field_conflict_with( left: &ArrowField, right: &ArrowField, - version: &LanceFileVersion, + validate_nested_column_add: fn(&ArrowField) -> Result<()>, ) -> Result<()> { if left.name() != right.name() { return Ok(()); @@ -207,13 +182,7 @@ fn check_field_conflict( match (left.data_type(), right.data_type()) { (DataType::Struct(fl), DataType::Struct(fr)) => { - if !version.support_add_sub_column() { - return Err(Error::invalid_input(format!( - "Column {} is a struct col, add sub column is not supported in Lance file version {}", - left.name(), - version - ))); - } + validate_nested_column_add(left)?; if left.is_packed() || right.is_packed() { return Err(Error::invalid_input(format!( @@ -224,15 +193,19 @@ fn check_field_conflict( for l_field in fl.iter() { if let Some((_, r_field)) = fr.find(l_field.name()) { - check_field_conflict(l_field, r_field, version)?; + check_field_conflict_with(l_field, r_field, validate_nested_column_add)?; } } Ok(()) } - (DataType::List(fl), DataType::List(fr)) => check_field_conflict(fl, fr, version), - (DataType::LargeList(fl), DataType::LargeList(fr)) => check_field_conflict(fl, fr, version), + (DataType::List(fl), DataType::List(fr)) => { + check_field_conflict_with(fl, fr, validate_nested_column_add) + } + (DataType::LargeList(fl), DataType::LargeList(fr)) => { + check_field_conflict_with(fl, fr, validate_nested_column_add) + } (DataType::FixedSizeList(fl, _), DataType::FixedSizeList(fr, _)) => { - check_field_conflict(fl, fr, version) + check_field_conflict_with(fl, fr, validate_nested_column_add) } (l_type, r_type) if l_type == r_type => Err(Error::invalid_input(format!( "Column {} already exists in the dataset", @@ -248,21 +221,30 @@ fn check_field_conflict( } } +#[cfg(test)] +fn check_field_conflict( + left: &ArrowField, + right: &ArrowField, + version: &ConcreteFileVersion, +) -> Result<()> { + super::versions::check_field_conflict(*version, left, right) +} + pub(super) async fn add_columns_to_fragments( dataset: &Dataset, transforms: NewColumnTransform, read_columns: Option>, fragments: &[FileFragment], batch_size: Option, -) -> Result<(Vec, Schema, Vec)> { +) -> Result<(Vec, Schema, Vec, bool)> { // Check names early (before calling add_columns_impl) to avoid extra work if // the names are wrong. - let version = dataset.manifest.data_storage_format.lance_file_version()?; + let version = dataset.manifest.data_storage_format.lance_file_format(); let check_names = |output_schema: &ArrowSchema| { for field in &dataset.schema().fields { if let Ok(out_field) = output_schema.field_with_name(&field.name) { let ds_field = ArrowField::from(field); - check_field_conflict(&ds_field, out_field, &version)?; + super::versions::check_field_conflict(version, &ds_field, out_field)?; } } Ok::<(), Error>(()) @@ -270,10 +252,7 @@ pub(super) async fn add_columns_to_fragments( // Optimize the transforms let mut optimizer = ChainedNewColumnTransformOptimizer::new(vec![]); - // ALlNull transform can not performed on legacy files - if !dataset.is_legacy_storage() { - optimizer.add_optimizer(Box::new(SqlToAllNullsOptimizer::new())); - } + super::versions::configure_new_column_optimizers(version, &mut optimizer); let transforms = optimizer.optimize(dataset, transforms)?; let (output_schema, new_fragments, fragments_to_cleanup) = match transforms { @@ -392,15 +371,7 @@ pub(super) async fn add_columns_to_fragments( .map(|f| f.metadata.clone()) .collect::>(); - // Check if any of the fragment's files are using the legacy dataset version if so, we - // can't add all-null columns as a metadata-only operation. The reason is because we - // use the NullReader for fragments that have missing columns and we can't mix legacy - // and non-legacy readers when reading the fragment. - if dataset.is_legacy_storage() { - return Err(Error::not_supported_source( - "Cannot add all-null columns to legacy dataset version.".into(), - )); - } + super::versions::validate_metadata_only_null_columns(version)?; Ok((output_schema, fragments, Vec::new())) } @@ -415,7 +386,58 @@ pub(super) async fn add_columns_to_fragments( }; schema.set_field_id(Some(dataset.manifest.max_field_id())); - Ok((new_fragments, schema, fragments_to_cleanup)) + let preserves_nullability = !merge_introduces_required_field(dataset.schema(), &schema); + + Ok(( + new_fragments, + schema, + fragments_to_cleanup, + preserves_nullability, + )) +} + +/// Whether `merged` introduces a field that data staged against `old` cannot +/// safely omit. The first new node on each path decides: a non-nullable new +/// field beneath an existing ancestor reads as unmasked null for stale rows, +/// which do supply the ancestor, while a nullable new field masks its whole +/// subtree whatever the nullability inside, the same rule the AllNulls +/// transform enforces at the top level. +/// +/// A new node under a non-nullable top-level column claims even when the node +/// itself is nullable: the reader synthesizes missing subcolumns against the +/// column's declared nullability, so a stale fragment cannot be read at all +/// under such a column, nullable child or not. +pub(super) fn merge_introduces_required_field(old: &Schema, merged: &Schema) -> bool { + /// (any node in `merged` is new, any first-new node is non-nullable) + fn subtree_new_nodes(old: &[Field], merged: &[Field]) -> (bool, bool) { + let mut any_new = false; + let mut any_required = false; + for field in merged { + match old.iter().find(|o| o.name == field.name) { + Some(old_field) => { + let (new, required) = subtree_new_nodes(&old_field.children, &field.children); + any_new |= new; + any_required |= required; + } + None => { + any_new = true; + any_required |= !field.nullable; + } + } + } + (any_new, any_required) + } + + merged.fields.iter().any( + |field| match old.fields.iter().find(|o| o.name == field.name) { + Some(old_field) => { + let (any_new, any_required) = + subtree_new_nodes(&old_field.children, &field.children); + any_required || (any_new && !field.nullable) + } + None => !field.nullable, + }, + ) } pub(super) async fn add_columns( @@ -424,27 +446,29 @@ pub(super) async fn add_columns( read_columns: Option>, batch_size: Option, ) -> Result<()> { - let (fragments, schema, fragments_to_cleanup) = add_columns_to_fragments( - dataset, - transforms, - read_columns, - &dataset.get_fragments(), - batch_size, - ) - .await?; + let (fragments, schema, _fragments_to_cleanup, preserves_nullability) = + add_columns_to_fragments( + dataset, + transforms, + read_columns, + &dataset.get_fragments(), + batch_size, + ) + .await?; - let operation = Operation::Merge { fragments, schema }; + let operation = Operation::Merge { + fragments, + schema, + preserves_nullability, + }; let transaction = Transaction::new(dataset.manifest.version, operation, None); - match dataset + // Once the manifest commit has been attempted, an error does not prove + // that the new files are unreferenced: the commit may have landed and only + // its response (or a post-commit callback) may have failed. Leave files + // from failed attempts for dataset GC instead of risking live-data loss. + dataset .apply_commit(transaction, &Default::default(), &Default::default()) .await - { - Ok(()) => Ok(()), - Err(e) => { - cleanup_new_column_data_files(&dataset.get_fragments(), &fragments_to_cleanup).await; - Err(e) - } - } } async fn cleanup_new_column_data_files(fragments: &[FileFragment], new_fragments: &[Fragment]) { @@ -539,7 +563,7 @@ async fn add_columns_impl( } let mut updater = match fragment - .updater(read_columns_ref, schemas.clone(), batch_size) + .updater(read_columns_ref, schemas.clone(), batch_size, None) .await { Ok(updater) => updater, @@ -617,7 +641,7 @@ async fn add_columns_from_stream( let mut last_seen_batch: Option = None; for fragment in fragments { let mut updater = match fragment - .updater::(Some(&[]), schemas.clone(), batch_size) + .updater::(Some(&[]), schemas.clone(), batch_size, None) .await { Ok(updater) => updater, @@ -711,9 +735,10 @@ pub(super) async fn alter_columns( // Mapping of old to new fields that need to be casted. let mut cast_fields: Vec<(Field, Field)> = Vec::new(); + let mut tightens_nullability = false; let mut next_field_id = dataset.manifest.max_field_id() + 1; - let version = dataset.manifest.data_storage_format.lance_file_version()?; + let version = dataset.manifest.data_storage_format.lance_file_format(); for alteration in alterations { let field_src = dataset.schema().field(&alteration.path).ok_or_else(|| { @@ -728,6 +753,9 @@ pub(super) async fn alter_columns( && !nullable { validate_no_nulls_before_making_non_nullable(dataset, &alteration.path).await?; + // A write since this version can falsify it, so withhold the + // preserves_nullability assertion from the transaction. + tightens_nullability = true; } let field_dest = new_schema.mut_field_by_id(field_src.id).unwrap(); @@ -740,7 +768,7 @@ pub(super) async fn alter_columns( if let Some(data_type) = &alteration.data_type { if !(can_cast_types(&field_src.data_type(), data_type) - && is_upcast_downcast(&field_src.data_type(), data_type, version)) + && super::versions::is_upcast_downcast(version, &field_src.data_type(), data_type)) { return Err(Error::invalid_input(format!( "Cannot cast column \"{}\" from {:?} to {:?}", @@ -763,15 +791,19 @@ pub(super) async fn alter_columns( } new_schema.validate()?; + new_schema.verify_primary_key()?; // If any column being cast has an attached index, fail fast. Cast operations // rewrite the underlying column data and silently invalidate any index on the // affected column(s). The current behavior is to drop such indices without // warning, which has caused production incidents where vector search silently // regressed to brute-force scan. We require users to explicitly drop the - // index before altering the column type, so the action is never silent. + // index before altering the column type, so the action is never silent. That + // includes an index this build has no reader for: the cast reassigns the + // field id, so carrying it forward is impossible and staying quiet about it + // is the silent drop this guard exists to abolish. if !cast_fields.is_empty() { - let indices = dataset.load_indices().await?; + let indices = load_all_indices(dataset).await?; let affected: Vec<&lance_table::format::IndexMetadata> = indices .iter() .filter(|idx| { @@ -799,56 +831,119 @@ pub(super) async fn alter_columns( } } + if tightens_nullability && !cast_fields.is_empty() { + return Err(Error::invalid_input( + "cannot make a column non-nullable and cast columns in the same call: \ + apply the cast first, then the nullability change", + )); + } + // If we aren't casting a column, we don't need to touch the fragments. let transaction = if cast_fields.is_empty() { Transaction::new( dataset.manifest.version, - Operation::Project { schema: new_schema }, + Operation::Project { + schema: new_schema, + preserves_nullability: !tightens_nullability, + }, // TODO: Make it possible to alter blob columns /*blob_op= */ None, ) } else { // Otherwise, we need to re-write the relevant fields. - let read_columns = cast_fields + let field_order = dataset + .schema() + .fields_pre_order() + .enumerate() + .map(|(position, field)| (field.id, position)) + .collect::>(); + let mut ordered_cast_fields = cast_fields .iter() - .map(|(old, _new)| { - let parts = dataset.schema().field_ancestry_by_id(old.id).unwrap(); - let part_names = parts.iter().map(|p| p.name.clone()).collect::>(); - part_names.join(".") + .map(|(old, new)| { + let position = field_order.get(&old.id).copied().ok_or_else(|| { + Error::internal(format!( + "Could not find field id {} for column {} while casting", + old.id, old.name + )) + })?; + Ok((position, old, new)) }) - .collect::>(); + .collect::>>()?; + ordered_cast_fields.sort_by_key(|(position, _, _)| *position); + + let read_columns = ordered_cast_fields + .iter() + .map(|(_, old, _)| dataset.schema().field_path_minimal(old.id)) + .collect::>>()?; - let new_ids = cast_fields + let new_ids = ordered_cast_fields .iter() - .map(|(_old, new)| new.id) + .map(|(_, _, new)| new.id) .collect::>(); // This schema contains the exact field ids we want to write the new fields with. let new_col_schema = new_schema.project_by_ids(&new_ids, true); + let output_schema = Arc::new(ArrowSchema::from(&new_col_schema)); + + // A cast rewrites the column under a new field id, so data staged + // against the pre-cast schema omits that id. A required recast field + // reads as unmasked null. Even when a nested field is nullable, a + // required top-level ancestor cannot safely synthesize the missing + // child, following the same rule as `merge_introduces_required_field`. + let cast_touches_required = cast_fields.iter().try_fold( + false, + |touches_required, (_old, new)| -> Result { + if touches_required || !new.nullable { + return Ok(true); + } + let top_level = new_schema + .field_ancestry_by_id(new.id) + .and_then(|ancestry| ancestry.first().copied()) + .ok_or_else(|| { + Error::internal(format!( + "Could not find field id {} for column {} while determining cast nullability", + new.id, new.name + )) + })?; + Ok(!top_level.nullable) + }, + )?; let mapper = move |batch: &RecordBatch| { - let mut fields = Vec::with_capacity(cast_fields.len()); - let mut columns = Vec::with_capacity(batch.num_columns()); - for (old, new) in &cast_fields { - let old_column = batch[&old.name].clone(); - let new_column = cast_with_options( - &old_column, - &new.data_type(), - // Safe: false means it will error if the cast is lossy. - &CastOptions { - safe: false, - ..Default::default() - }, - )?; - columns.push(new_column); - fields.push(Arc::new(ArrowField::from(new))); + if batch.num_columns() != output_schema.fields().len() { + return Err(Error::internal(format!( + "Expected {} columns while casting dataset fields, got {}", + output_schema.fields().len(), + batch.num_columns() + ))); } - let schema = Arc::new(ArrowSchema::new(fields)); - Ok(RecordBatch::try_new(schema, columns)?) + + let columns = batch + .columns() + .iter() + .zip(output_schema.fields()) + .map(|(old_column, new_field)| { + cast_with_options( + old_column, + new_field.data_type(), + // Safe: false means it will error if the cast is lossy. + &CastOptions { + safe: false, + ..Default::default() + }, + ) + }) + .collect::, _>>()?; + Ok(RecordBatch::try_new(output_schema.clone(), columns)?) }; let mapper = Box::new(mapper); + let source_fragments = dataset.get_fragments(); + let original_file_counts = source_fragments + .iter() + .map(|fragment| (fragment.id() as u64, fragment.metadata.files.len())) + .collect::>(); let result = add_columns_impl( - &dataset.get_fragments(), + &source_fragments, Some(read_columns), mapper, None, @@ -864,20 +959,50 @@ pub(super) async fn alter_columns( .fragments .into_iter() .map(|mut frag| { + let original_file_count = + original_file_counts.get(&frag.id).copied().ok_or_else(|| { + Error::internal(format!( + "Could not find source fragment {} after casting columns", + frag.id + )) + })?; + let rewritten_field_ids = frag + .files + .iter() + .skip(original_file_count) + .flat_map(|file| file.fields.iter().copied()) + .collect::>(); + // V1 files record struct ancestor ids, so a child rewrite also + // supersedes those ancestor entries in the original file. + for file in frag.files.iter_mut().take(original_file_count) { + file.fields = file + .fields + .iter() + .map(|field_id| { + if rewritten_field_ids.contains(field_id) { + TOMBSTONE_FIELD_ID + } else { + *field_id + } + }) + .collect::>() + .into(); + } frag.files.retain(|f| { f.fields .iter() .any(|field| schema_field_ids.contains(field)) }); - frag + Ok(frag) }) - .collect::>(); + .collect::>>()?; Transaction::new( dataset.manifest.version, Operation::Merge { schema: new_schema, fragments, + preserves_nullability: !cast_touches_required, }, /*blob_op= */ None, ) @@ -909,9 +1034,10 @@ pub(super) async fn drop_columns(dataset: &mut Dataset, columns: &[&str]) -> Res } } - let version = dataset.manifest.data_storage_format.lance_file_version()?; + let version = dataset.manifest.data_storage_format.lance_file_format(); let columns_to_remove = dataset.manifest.schema.project(columns)?; - let new_schema = exclude(&dataset.manifest.schema, &columns_to_remove, &version)?; + let new_schema = + super::versions::exclude_schema(version, &dataset.manifest.schema, &columns_to_remove)?; if new_schema.fields.is_empty() { return Err(Error::invalid_input( @@ -921,7 +1047,10 @@ pub(super) async fn drop_columns(dataset: &mut Dataset, columns: &[&str]) -> Res let transaction = Transaction::new( dataset.manifest.version, - Operation::Project { schema: new_schema }, + Operation::Project { + schema: new_schema, + preserves_nullability: true, + }, /*blob_op= */ None, ); @@ -932,17 +1061,19 @@ pub(super) async fn drop_columns(dataset: &mut Dataset, columns: &[&str]) -> Res Ok(()) } -/// Exclude the fields from `other` Schema, and returns a new Schema. -pub fn exclude(source: &Schema, other: &Schema, version: &LanceFileVersion) -> Result { +/// Exclude the fields from `other` Schema using the selected nested-field rule. +pub fn exclude_with( + source: &Schema, + other: &Schema, + exclude_nested_field: fn(&Field, &Field) -> Option, +) -> Result { let other: Schema = other.try_into().map_err(|_| { Error::schema("The other schema is not compatible with this schema".to_string()) })?; let mut fields = vec![]; for field in source.fields.iter() { if let Some(other_field) = other.field(&field.name) { - if version.support_remove_sub_column(field) - && let Some(f) = field.exclude(other_field) - { + if let Some(f) = exclude_nested_field(field, other_field) { fields.push(f) } } else { @@ -955,10 +1086,97 @@ pub fn exclude(source: &Schema, other: &Schema, version: &LanceFileVersion) -> R }) } +#[cfg(test)] +fn exclude(source: &Schema, other: &Schema, version: &ConcreteFileVersion) -> Result { + super::versions::exclude_schema(*version, source, other) +} + #[cfg(test)] mod test { use std::{collections::HashMap, fs, num::NonZero, path::Path as StdPath, sync::Mutex}; + use crate::index::DatasetIndexExt; + + #[test] + fn test_merge_introduces_required_field() { + let schema = |fields: Vec| Schema::try_from(&ArrowSchema::new(fields)).unwrap(); + let strukt = |name: &str, nullable: bool, children: Vec| { + ArrowField::new( + name, + DataType::Struct(ArrowFields::from(children)), + nullable, + ) + }; + let int = |name: &str, nullable: bool| ArrowField::new(name, DataType::Int32, nullable); + + let old = schema(vec![ + strukt("s", true, vec![int("a", true)]), + strukt("r", false, vec![int("a", true)]), + ]); + // The first new node on each path decides, at any depth; any new node + // under a non-nullable top-level column claims regardless. + for (merged, expected) in [ + // A nullable new child under a non-nullable top-level column: the + // reader cannot synthesize the missing subcolumn, so claim. + ( + schema(vec![ + strukt("s", true, vec![int("a", true)]), + strukt("r", false, vec![int("a", true), int("b", true)]), + ]), + true, + ), + // Required new child under an existing parent: stale rows supply + // the parent, so the child would read as unmasked null. + ( + schema(vec![strukt( + "s", + true, + vec![int("a", true), int("b", false)], + )]), + true, + ), + ( + schema(vec![strukt( + "s", + true, + vec![int("a", true), int("b", true)], + )]), + false, + ), + // A wholly new nullable container masks its required inside. + ( + schema(vec![ + strukt("s", true, vec![int("a", true)]), + strukt("t", true, vec![int("c", false)]), + ]), + false, + ), + // Same, when the new container hangs under an existing parent. + ( + schema(vec![strukt( + "s", + true, + vec![int("a", true), strukt("t", true, vec![int("c", false)])], + )]), + false, + ), + ( + schema(vec![ + strukt("s", true, vec![int("a", true)]), + int("b", false), + ]), + true, + ), + (schema(vec![strukt("s", true, vec![int("a", true)])]), false), + ] { + assert_eq!( + merge_introduces_required_field(&old, &merged), + expected, + "merged={merged:?}" + ); + } + } + use crate::dataset::WriteParams; use arrow_array::{ ArrayRef, Int32Array, ListArray, RecordBatchIterator, StringArray, StructArray, @@ -967,7 +1185,7 @@ mod test { use super::*; use arrow_schema::Fields as ArrowFields; use lance_core::utils::tempfile::TempStrDir; - use lance_file::version::LanceFileVersion; + use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_table::format::{BasePath, DataFile}; use rstest::rstest; @@ -1101,15 +1319,88 @@ mod test { } #[tokio::test] - async fn test_add_columns_with_fully_deleted_batch() -> Result<()> { - // Regression test: when an entire read batch has been deleted, the - // updater yields a 0-row batch. The inner loop then never runs and - // `batches` stays empty, so `concat_batches(&batches[0]..)` used to - // panic with "index out of bounds: the len is 0 but the index is 0". - // - // A single fragment holds 105 rows; deleting the trailing 5 rows means - // that, when read with batch_size=50, the third batch [100..105) is - // fully filtered out and produces an empty batch. + async fn test_add_columns_preserves_files_when_commit_status_is_unknown() -> Result<()> { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let num_rows = 5; + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..num_rows))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }), + ) + .await?; + let files_before = data_file_paths_in(test_uri); + + handler.fail_next(AmbiguousFailure::LandAndError); + handler + .fail_resolve + .store(true, std::sync::atomic::Ordering::SeqCst); + let error = dataset + .add_columns( + NewColumnTransform::SqlExpressions(vec![("double_id".into(), "2 * id".into())]), + None, + None, + ) + .await + .expect_err("unverifiable commit outcome must be reported"); + assert!( + error.is_commit_status_unknown(), + "expected CommitStatusUnknown, got: {error:?}" + ); + assert!(data_file_paths_in(test_uri).len() > files_before.len()); + + handler + .fail_resolve + .store(false, std::sync::atomic::Ordering::SeqCst); + let reopened = Dataset::open(test_uri).await?; + let data = reopened.scan().try_into_batch().await?; + let double_id = data + .column_by_name("double_id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(double_id, &Int32Array::from(vec![0, 2, 4, 6, 8])); + + Ok(()) + } + + /// Regression test: when an entire read batch has been deleted, the updater + /// yields a 0-row batch and the deleted rows must still be restored, because + /// every data file in a fragment has to keep the same physical row count. + /// + /// A single fragment holds 150 rows and 50 consecutive rows are deleted. Read + /// with batch_size=50 the deleted run lines up exactly with one read batch, + /// which therefore arrives empty. The run is placed at the start, in the + /// middle, and at the end because the restorer treats those positions + /// differently: a deleted run that trails a live batch is greedily appended to + /// it, while a run starting at row 0 has no preceding batch to absorb it. + #[rstest] + #[case::leading("i < 50", (50..150).collect::>())] + #[case::middle("i >= 50 AND i < 100", (0..50).chain(100..150).collect::>())] + #[case::trailing("i >= 100", (0..100).collect::>())] + #[tokio::test] + async fn test_add_columns_with_fully_deleted_batch( + #[case] delete_predicate: &str, + #[case] expected_live_ids: Vec, + #[values(true, false)] new_column_nullable: bool, + ) -> Result<()> { let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( "i", DataType::Int32, @@ -1117,7 +1408,7 @@ mod test { )])); let batch = RecordBatch::try_new( schema.clone(), - vec![Arc::new(Int32Array::from_iter_values(0..105))], + vec![Arc::new(Int32Array::from_iter_values(0..150))], )?; let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); @@ -1133,14 +1424,13 @@ mod test { ) .await?; - // Delete the entire trailing batch [100..105). - dataset.delete("i >= 100").await?; + dataset.delete(delete_predicate).await?; assert_eq!(dataset.count_rows(None).await?, 100); let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( "j", DataType::Int32, - false, + new_column_nullable, )])); let new_batch = RecordBatch::try_new( new_schema.clone(), @@ -1148,13 +1438,18 @@ mod test { )?; let reader = RecordBatchIterator::new(vec![Ok(new_batch)], new_schema.clone()); - // Read with batch_size=50 so the deleted trailing rows form a full empty batch. + // Read with batch_size=50 so the deleted rows form a full empty batch. dataset .add_columns(NewColumnTransform::Reader(Box::new(reader)), None, Some(50)) .await?; + dataset.validate().await?; let data = dataset.scan().try_into_batch().await?; assert_eq!(data.num_rows(), 100); + assert_eq!( + data.column_by_name("i").unwrap().as_ref(), + &Int32Array::from(expected_live_ids) + ); assert_eq!( data.column_by_name("j").unwrap().as_ref(), &Int32Array::from_iter_values(0..100) @@ -1163,6 +1458,73 @@ mod test { Ok(()) } + /// A legacy fragment whose trailing row group is entirely deleted cannot defer its + /// blanks: that batch reaches `add_blanks` with no live row to copy, so the update + /// is refused rather than writing a data file short of the deleted rows. Deferring + /// is what a v2 fragment does instead, which + /// `test_add_columns_with_fully_deleted_batch`'s trailing case covers. + #[tokio::test] + async fn test_add_columns_legacy_trailing_deleted_batch_errors() -> Result<()> { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..105))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 200, + max_rows_per_group: 50, + data_storage_version: Some(LanceFileVersion::Legacy), + ..Default::default() + }), + ) + .await?; + + // The last row group is [100, 105); deleting all of it leaves a trailing read + // batch with no live rows, which legacy files cannot defer past. + dataset.delete("i >= 100").await?; + + let new_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "j", + DataType::Int32, + true, + )])); + let new_batch = RecordBatch::try_new( + new_schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..100))], + )?; + let reader = RecordBatchIterator::new(vec![Ok(new_batch)], new_schema.clone()); + + let err = dataset + .add_columns(NewColumnTransform::Reader(Box::new(reader)), None, None) + .await + .unwrap_err(); + + assert!( + matches!(err, Error::NotSupported { .. }), + "expected NotSupported, got {err:?}" + ); + // Match add_blanks' own wording, not the shared "run compaction" tail: the + // stream-ended error in Updater::next carries that tail too, and this case + // fails before the stream ever runs out. + assert!( + err.to_string().contains("missing too many rows in merge"), + "expected the add_blanks rejection, got: {err}" + ); + + Ok(()) + } + #[rstest] #[tokio::test] async fn test_add_columns_cleans_up_blob_v2_data_on_stream_error( @@ -1295,8 +1657,7 @@ mod test { "checkpointed.lance", vec![dataset.manifest.max_field_id() + 1], vec![0], - 2, - 2, + ConcreteFileVersion::V2_2, NonZero::new(17), None, )); @@ -1957,6 +2318,7 @@ mod test { Ok(Some(Fragment { files: vec![], id: 0, + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(50), @@ -2458,7 +2820,7 @@ mod test { let schema = Schema::try_from(&arrow_schema).unwrap(); let projection = schema.project(&["a", "b.f2", "b.f3"]).unwrap(); - let excluded = exclude(&schema, &projection, &LanceFileVersion::V2_2).unwrap(); + let excluded = exclude(&schema, &projection, &ConcreteFileVersion::V2_2).unwrap(); let expected_arrow_schema = ArrowSchema::new(vec![ ArrowField::new( @@ -3004,6 +3366,114 @@ mod test { Ok(()) } + #[rstest] + #[tokio::test] + async fn test_cast_columns_reversed_order( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) -> Result<()> { + use arrow_array::Int64Array; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(Int32Array::from(vec![10, 20])), + ], + )?; + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(data_storage_version), + max_rows_per_file: 1, + ..Default::default() + }), + ) + .await?; + assert_eq!(dataset.fragments().len(), 2); + + dataset + .alter_columns(&[ + ColumnAlteration::new("b".into()).cast_to(DataType::Int64), + ColumnAlteration::new("a".into()).cast_to(DataType::Int64), + ]) + .await?; + dataset.validate().await?; + + let data = dataset.scan().try_into_batch().await?; + assert_eq!(data["a"].as_ref(), &Int64Array::from(vec![1, 2])); + assert_eq!(data["b"].as_ref(), &Int64Array::from(vec![10, 20])); + + Ok(()) + } + + #[rstest] + #[tokio::test] + async fn test_cast_nested_column( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, + ) -> Result<()> { + use arrow_array::{Int64Array, cast::AsArray}; + + let child_field = Arc::new(ArrowField::new("c", DataType::Int32, false)); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(ArrowFields::from(vec![child_field.clone()])), + false, + )])); + let struct_array = StructArray::try_new( + ArrowFields::from(vec![child_field]), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + None, + )?; + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(struct_array)])?; + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(data_storage_version), + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await?; + assert_eq!(dataset.fragments().len(), 2); + + dataset + .alter_columns(&[ColumnAlteration::new("b.c".into()).cast_to(DataType::Int64)]) + .await?; + dataset.validate().await?; + + let expected_schema = ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(ArrowFields::from(vec![ArrowField::new( + "c", + DataType::Int64, + false, + )])), + false, + )]); + assert_eq!(&ArrowSchema::from(dataset.schema()), &expected_schema); + + let data = dataset.scan().try_into_batch().await?; + let struct_array = data["b"].as_struct(); + assert_eq!( + struct_array.column_by_name("c").unwrap().as_ref(), + &Int64Array::from(vec![1, 2, 3]) + ); + + Ok(()) + } + /// Cast on a column with an attached index must fail fast rather than /// silently dropping the index. This guards against the historical behavior /// where cast would rewrite column data and the index would vanish without @@ -3054,8 +3524,9 @@ mod test { ) .await?; - // Build an IVF_PQ index on the vector column. - let params = VectorIndexParams::ivf_pq(4, 8, 8, MetricType::L2, 50); + // Any attached vector index blocks the cast; IVF_FLAT exercises that + // ownership contract without unrelated quantizer training. + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); dataset .create_index(&["vec"], IndexType::Vector, None, ¶ms, false) .await?; @@ -3126,8 +3597,8 @@ mod test { let dict_i16_utf8 = Dictionary(Box::new(Int16), Box::new(Utf8)); let dict_i32_large_utf8 = Dictionary(Box::new(Int32), Box::new(LargeUtf8)); let dict_i32_int64 = Dictionary(Box::new(Int32), Box::new(Int64)); - let stable = LanceFileVersion::Stable; - let legacy = LanceFileVersion::Legacy; + let stable = LanceFileVersion::Stable.resolve(); + let legacy = LanceFileVersion::Legacy.resolve(); // Dict(_, Utf8) -> Utf8 / LargeUtf8 (decode direction): both versions. assert!(is_upcast_downcast(&dict_i32_utf8, &Utf8, stable)); @@ -3623,7 +4094,7 @@ mod test { DataType::Struct(vec![ArrowField::new("a", DataType::Int32, false)].into()), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // different struct let field1 = ArrowField::new( @@ -3636,7 +4107,7 @@ mod test { DataType::Struct(vec![ArrowField::new("b", DataType::Int32, false)].into()), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // same nested struct let inner_struct1 = ArrowField::new( @@ -3651,22 +4122,22 @@ mod test { ); let field1 = ArrowField::new("test", DataType::Struct(vec![inner_struct1].into()), false); let field2 = ArrowField::new("test", DataType::Struct(vec![inner_struct2].into()), false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // basic type with different name let field1 = ArrowField::new("test1", DataType::Int32, false); let field2 = ArrowField::new("test2", DataType::Int32, false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // basic type with same name let field1 = ArrowField::new("test", DataType::Int32, false); let field2 = ArrowField::new("test", DataType::Int32, false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // different basic type let field1 = ArrowField::new("test", DataType::Int32, false); let field2 = ArrowField::new("test", DataType::Float64, false); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // partial conflict let field1 = ArrowField::new( @@ -3691,7 +4162,7 @@ mod test { ), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // same list let field1 = ArrowField::new( @@ -3704,7 +4175,7 @@ mod test { DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // list with struct let field1 = ArrowField::new( @@ -3725,7 +4196,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // list with different struct let field1 = ArrowField::new( @@ -3746,7 +4217,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // list of struct and basic let field1 = ArrowField::new( @@ -3763,7 +4234,7 @@ mod test { DataType::List(Arc::new(ArrowField::new("item", DataType::Int32, true))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // FixedSizeList with struct let field1 = ArrowField::new( @@ -3790,7 +4261,7 @@ mod test { ), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // FixedSizeList with different struct let field1 = ArrowField::new( @@ -3817,7 +4288,7 @@ mod test { ), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // LargeList with struct let field1 = ArrowField::new( @@ -3838,7 +4309,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_err()); // LargeList with different struct let field1 = ArrowField::new( @@ -3859,7 +4330,7 @@ mod test { ))), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); // packed struct let mut packed_meta = HashMap::new(); @@ -3878,7 +4349,7 @@ mod test { DataType::Struct(vec![ArrowField::new("b", DataType::Int32, false)].into()), false, ); - assert!(check_field_conflict(&field1, &field2, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field2, &ConcreteFileVersion::V2_2).is_ok()); let new_packed_field = ArrowField::new( "new_packed", @@ -3891,7 +4362,7 @@ mod test { DataType::Struct(vec![new_packed_field].into()), false, ); - assert!(check_field_conflict(&field1, &field3, &LanceFileVersion::V2_2).is_ok()); + assert!(check_field_conflict(&field1, &field3, &ConcreteFileVersion::V2_2).is_ok()); let conflict_field = ArrowField::new( "packed", @@ -3900,6 +4371,54 @@ mod test { ) .with_metadata(packed_meta); let field4 = ArrowField::new("test", DataType::Struct(vec![conflict_field].into()), false); - assert!(check_field_conflict(&field1, &field4, &LanceFileVersion::V2_2).is_err()); + assert!(check_field_conflict(&field1, &field4, &ConcreteFileVersion::V2_2).is_err()); + } + + /// Table creation rejects a nullable primary key; altering one afterwards + /// reached the same state without passing that check. + #[tokio::test] + async fn test_alter_columns_cannot_make_a_primary_key_nullable() -> Result<()> { + let pk = ArrowField::new("id", DataType::Int32, false).with_metadata( + [( + "lance-schema:unenforced-primary-key:position".to_string(), + "1".to_string(), + )] + .into(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + pk, + ArrowField::new("value", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(Int32Array::from(vec![10, 20])), + ], + )?; + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + None, + ) + .await?; + + let err = dataset + .alter_columns(&[ColumnAlteration::new("id".into()).set_nullable(true)]) + .await + .expect_err("making a primary key nullable must be rejected"); + assert!( + err.to_string().contains("must not be nullable"), + "unexpected error: {err}" + ); + + // Specific to the key: other columns may still be altered. + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).rename("val".into())]) + .await?; + assert!(!dataset.schema().unenforced_primary_key()[0].nullable); + + Ok(()) } } diff --git a/rust/lance/src/dataset/schema_evolution/optimize.rs b/rust/lance/src/dataset/schema_evolution/optimize.rs index cdfdb82b87b..5be44626821 100644 --- a/rust/lance/src/dataset/schema_evolution/optimize.rs +++ b/rust/lance/src/dataset/schema_evolution/optimize.rs @@ -14,7 +14,7 @@ use crate::Result; use super::NewColumnTransform; /// Optimizes a `NewColumnTransform` into -pub(super) trait NewColumnTransformOptimizer: Send + Sync { +pub trait NewColumnTransformOptimizer: Send + Sync { /// Optimize the passed `NewColumnTransform` to a more efficient form. fn optimize( &self, @@ -24,16 +24,16 @@ pub(super) trait NewColumnTransformOptimizer: Send + Sync { } /// A `NewColumnTransformOptimizer` that chains multiple `NewColumnTransformOptimizer`s together. -pub(super) struct ChainedNewColumnTransformOptimizer { +pub struct ChainedNewColumnTransformOptimizer { optimizers: Vec>, } impl ChainedNewColumnTransformOptimizer { - pub(super) fn new(optimizers: Vec>) -> Self { + pub fn new(optimizers: Vec>) -> Self { Self { optimizers } } - pub(super) fn add_optimizer(&mut self, optimizer: Box) { + pub fn add_optimizer(&mut self, optimizer: Box) { self.optimizers.push(optimizer); } } @@ -59,10 +59,10 @@ impl NewColumnTransformOptimizer for ChainedNewColumnTransformOptimizer { /// would be optimized to /// `NewColumnTransform::AllNulls(Schema::new(vec![Field::new("new_col", DataType::Int)]))`. /// -pub(super) struct SqlToAllNullsOptimizer; +pub struct SqlToAllNullsOptimizer; impl SqlToAllNullsOptimizer { - pub(super) fn new() -> Self { + pub fn new() -> Self { Self } @@ -70,7 +70,7 @@ impl SqlToAllNullsOptimizer { match expr { Expr::Cast(cast) => { if matches!(cast.expr.as_ref(), Expr::Literal(ScalarValue::Null, _)) { - let data_type = cast.data_type.clone(); + let data_type = cast.field.data_type().clone(); AllNullsResult::AllNulls(data_type) } else { AllNullsResult::NotAllNulls diff --git a/rust/lance/src/dataset/sql.rs b/rust/lance/src/dataset/sql.rs index 8a1ccda2df6..2d2391a468c 100644 --- a/rust/lance/src/dataset/sql.rs +++ b/rust/lance/src/dataset/sql.rs @@ -3,12 +3,19 @@ use crate::Dataset; use crate::datafusion::LanceTableProvider; +use crate::dataset::scanner::validate_batch_size; use crate::dataset::utils::SchemaAdapter; use arrow_array::RecordBatch; use datafusion::dataframe::DataFrame; use datafusion::execution::SendableRecordBatchStream; -use datafusion::prelude::SessionContext; +use datafusion::logical_expr::{Expr as LogicalExpr, LogicalPlan}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion::sql::{ + parser::Statement as DFStatement, + sqlparser::ast::{Expr, Ident, SelectItem, SetExpr, Statement}, +}; use futures::TryStreamExt; +use lance_core::{ROW_ADDR, ROW_ID, datatypes::BlobHandling}; use lance_datafusion::udf::register_functions; use std::sync::Arc; @@ -29,6 +36,15 @@ pub struct SqlQueryBuilder { /// If true, the query result will include the internal row address pub(crate) with_row_addr: bool, + + /// Override how blob columns are materialized for this query. + pub(crate) blob_handling: Option, + + /// Override the maximum number of rows in each scan batch. + pub(crate) batch_size: Option, + + /// Override the approximate maximum bytes in each scan batch. + pub(crate) batch_size_bytes: Option, } impl SqlQueryBuilder { @@ -39,6 +55,9 @@ impl SqlQueryBuilder { table_name: "dataset".to_string(), with_row_id: false, with_row_addr: false, + blob_handling: None, + batch_size: None, + batch_size_bytes: None, } } @@ -53,6 +72,10 @@ impl SqlQueryBuilder { /// Specify if the query result should include the internal row id. /// If true, the query result will include an additional column named "_rowid". + /// + /// The column is appended only when output rows map one-to-one to dataset + /// rows. For other queries (DISTINCT, GROUP BY, aggregates, ...) it is not + /// appended, but can still be referenced explicitly in the SQL text. pub fn with_row_id(mut self, row_id: bool) -> Self { self.with_row_id = row_id; self @@ -60,29 +83,196 @@ impl SqlQueryBuilder { /// Specify if the query result should include the internal row address. /// If true, the query result will include an additional column named "_rowaddr". + /// + /// The column is appended only when output rows map one-to-one to dataset + /// rows. For other queries (DISTINCT, GROUP BY, aggregates, ...) it is not + /// appended, but can still be referenced explicitly in the SQL text. pub fn with_row_addr(mut self, row_addr: bool) -> Self { self.with_row_addr = row_addr; self } + /// Override how blob columns are materialized for this query. + /// + /// When unset, the underlying dataset scan uses its default + /// [`BlobHandling::BlobsDescriptions`] policy. + pub fn blob_handling(mut self, blob_handling: BlobHandling) -> Self { + self.blob_handling = Some(blob_handling); + self + } + + /// Set the maximum number of rows produced by each query batch. + /// + /// The batch size must be between 1 and [`u32::MAX`], inclusive. + /// + /// When [`Self::batch_size_bytes`] is also set, both limits apply and the + /// one reached first determines the scan batch size. + pub fn batch_size(mut self, batch_size: usize) -> Self { + self.batch_size = Some(batch_size); + self + } + + /// Set the approximate maximum number of bytes produced by each scan batch. + /// + /// When [`Self::batch_size`] is also set, both limits apply and the one + /// reached first determines the scan batch size. + pub fn batch_size_bytes(mut self, batch_size_bytes: u64) -> Self { + self.batch_size_bytes = Some(batch_size_bytes); + self + } + pub async fn build(self) -> lance_core::Result { - let ctx = SessionContext::new(); + if let Some(batch_size) = self.batch_size { + validate_batch_size(batch_size)?; + } + + let ctx = if let Some(batch_size) = self.batch_size { + SessionContext::new_with_config(SessionConfig::new().with_batch_size(batch_size)) + } else { + SessionContext::new() + }; let row_id = self.with_row_id; let row_addr = self.with_row_addr; - ctx.register_table( - self.table_name, - Arc::new(LanceTableProvider::new( - self.dataset.clone(), - row_id, - row_addr, - )), - )?; + let mut provider = LanceTableProvider::new(self.dataset.clone(), row_id, row_addr); + if let Some(blob_handling) = self.blob_handling { + provider = provider.with_blob_handling(blob_handling); + } + if let Some(batch_size) = self.batch_size { + provider = provider.with_batch_size(batch_size); + } + if let Some(batch_size_bytes) = self.batch_size_bytes { + provider = provider.with_batch_size_bytes(batch_size_bytes); + } + ctx.register_table(self.table_name, Arc::new(provider))?; register_functions(&ctx); - let df = ctx.sql(&self.sql).await?; + let state = ctx.state(); + let dialect = state.config_options().sql_parser.dialect; + let statement = state.sql_to_statement(&self.sql, &dialect)?; + let mut projected = statement.clone(); + let columns = [(self.with_row_id, ROW_ID), (self.with_row_addr, ROW_ADDR)]; + let plan = state.statement_to_plan(statement).await?; + let plan = if safe_to_inject_system_columns(&plan, &columns) + && project_system_columns(&mut projected, &columns) + { + // Fall back to the original plan when the rewritten statement + // fails to plan (e.g. another expression aliased to a system + // column name), so the query still runs without the extra columns. + state.statement_to_plan(projected).await.unwrap_or(plan) + } else { + plan + }; + let df = ctx.execute_logical_plan(plan).await?; Ok(SqlQuery::new(df)) } } +/// Returns true when appending the enabled system columns to the query's +/// top-level SELECT list is provably safe: +/// +/// 1. Row identity: every output row maps to exactly one scanned source row +/// (whitelist of row-preserving operators; aggregates, DISTINCT, joins, +/// unions, ... collapse, duplicate, or synthesize rows), so the injection +/// cannot change the other columns' values or cardinality. +/// 2. Name lineage: no intermediate projection redefines an enabled system +/// column name (e.g. `SELECT (_rowid + 1) AS _rowid` in a subquery), so +/// the injected identifiers can only bind to the real scan columns. +fn safe_to_inject_system_columns(plan: &LogicalPlan, columns: &[(bool, &str)]) -> bool { + match plan { + LogicalPlan::TableScan(_) => true, + LogicalPlan::Projection(projection) => { + let shadows_system_column = projection + .schema + .fields() + .iter() + .zip(&projection.expr) + .filter(|(field, _)| { + columns + .iter() + .any(|&(enabled, name)| enabled && field.name().as_str() == name) + }) + .any(|(field, expr)| { + let mut expr = expr; + while let LogicalExpr::Alias(alias) = expr { + expr = &alias.expr; + } + !matches!(expr, LogicalExpr::Column(column) if &column.name == field.name()) + }); + !shadows_system_column && safe_to_inject_system_columns(&projection.input, columns) + } + LogicalPlan::Filter(_) + | LogicalPlan::Sort(_) + | LogicalPlan::Limit(_) + | LogicalPlan::SubqueryAlias(_) => plan + .inputs() + .iter() + .all(|input| safe_to_inject_system_columns(input, columns)), + _ => false, + } +} + +/// Appends each enabled system column in `columns` to the statement's SELECT +/// list unless the query already projects it (directly or via a wildcard). +/// Returns true if the statement was modified. +/// +/// Only rewrites top-level `SELECT` statements; the caller must separately +/// verify that the injection is safe (see [`safe_to_inject_system_columns`]) +/// before planning the rewritten statement. +fn project_system_columns(statement: &mut DFStatement, columns: &[(bool, &str)]) -> bool { + let DFStatement::Statement(statement) = statement else { + return false; + }; + let Statement::Query(query) = statement.as_mut() else { + return false; + }; + let SetExpr::Select(select) = query.body.as_mut() else { + return false; + }; + + let mut changed = false; + for &(enabled, name) in columns { + if !enabled { + continue; + } + let already_projected = select + .projection + .iter() + .any(|item| projects_column(item, name)); + if already_projected { + continue; + } + select + .projection + .push(SelectItem::UnnamedExpr(Expr::Identifier(Ident::new(name)))); + changed = true; + } + changed +} + +/// Returns true if the SELECT item already yields the column `name`, either +/// as a bare/qualified identifier (e.g. `_rowid`, `t._rowid`) or through a +/// wildcard (`*`, `t.*`), so injecting it again would duplicate the column. +/// +/// Expressions that merely reference the column (e.g. `_rowid + 1`, aliases) +/// intentionally don't count: they produce a different output column. +fn projects_column(item: &SelectItem, name: &str) -> bool { + match item { + SelectItem::Wildcard(_) | SelectItem::QualifiedWildcard(_, _) => true, + SelectItem::UnnamedExpr(Expr::Identifier(ident)) => ident_matches(ident, name), + SelectItem::UnnamedExpr(Expr::CompoundIdentifier(idents)) => idents + .last() + .is_some_and(|ident| ident_matches(ident, name)), + _ => false, + } +} + +fn ident_matches(ident: &Ident, name: &str) -> bool { + if ident.quote_style.is_some() { + ident.value == name + } else { + ident.value.eq_ignore_ascii_case(name) + } +} + pub struct SqlQuery { dataframe: DataFrame, } @@ -123,19 +313,28 @@ impl SqlQuery { #[cfg(test)] mod tests { use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, assert_string_matches}; + use crate::{BlobArrayBuilder, blob_field}; use std::collections::HashMap; use std::sync::Arc; - use crate::Dataset; + use crate::dataset::ReadParams; + use crate::dataset::builder::DatasetBuilder; + use crate::dataset::write::WriteParams; + use crate::{Dataset, Error}; use all_asserts::assert_true; use arrow_array::cast::AsArray; use arrow_array::types::{Int32Type, Int64Type, UInt64Type}; use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray}; use arrow_schema::Schema as ArrowSchema; use arrow_schema::{DataType, Field}; - use lance_arrow::ARROW_EXT_NAME_KEY; use lance_arrow::json::ARROW_JSON_EXT_NAME; + use lance_arrow::{ARROW_EXT_NAME_KEY, SchemaExt}; + use lance_core::datatypes::BlobHandling; + use lance_core::utils::tempfile::TempStrDir; use lance_datagen::{array, gen_batch}; + use lance_file::reader::FileReaderOptions; + use lance_file::version::LanceFileVersion; + use rstest::rstest; #[tokio::test] async fn test_sql_execute() { @@ -186,6 +385,319 @@ mod tests { assert_true!(results.column(3).as_primitive::().value(0) > 100); } + /// Requested system columns are appended after the user's columns when + /// injection is safe, are not duplicated when already projected under any + /// accepted spelling, and are skipped when a subquery alias shadows them + /// (the injected identifiers would bind to the derived expressions and + /// return arbitrary values as row metadata). + #[rstest] + #[case::plain("SELECT x FROM dataset", vec!["x", "_rowid", "_rowaddr"], vec![0, 1])] + #[case::filter_sort_limit( + "SELECT x FROM dataset WHERE x >= 0 ORDER BY x DESC LIMIT 2", + vec!["x", "_rowid", "_rowaddr"], + vec![1, 0] + )] + #[case::wildcard("SELECT * FROM dataset", vec!["x", "_rowid", "_rowaddr"], vec![0, 1])] + #[case::already_projected( + "SELECT x, _rowid, _rowaddr FROM dataset", + vec!["x", "_rowid", "_rowaddr"], + vec![0, 1] + )] + #[case::unquoted_uppercase( + "SELECT x, _ROWID, _ROWADDR FROM dataset", + vec!["x", "_rowid", "_rowaddr"], + vec![0, 1] + )] + #[case::quoted( + r#"SELECT x, "_rowid", "_rowaddr" FROM dataset"#, + vec!["x", "_rowid", "_rowaddr"], + vec![0, 1] + )] + #[case::table_qualified( + "SELECT x, dataset._rowid, dataset._rowaddr FROM dataset", + vec!["x", "_rowid", "_rowaddr"], + vec![0, 1] + )] + #[case::expression_reference( + "SELECT _rowid + 1 AS y FROM dataset", + vec!["y", "_rowid", "_rowaddr"], + vec![0, 1] + )] + #[case::system_columns_only("SELECT _rowid FROM dataset", vec!["_rowid", "_rowaddr"], vec![0, 1])] + #[case::passthrough_subquery( + "SELECT x FROM (SELECT x, _rowid, _rowaddr FROM dataset) s", + vec!["x", "_rowid", "_rowaddr"], + vec![0, 1] + )] + #[case::shadowed_subquery( + "SELECT x FROM (SELECT x, (_rowid + 1) AS _rowid, (_rowaddr + 1) AS _rowaddr FROM dataset) s", + vec!["x"], + vec![] + )] + #[tokio::test] + async fn test_sql_system_column_injection( + #[case] sql: &str, + #[case] expected_columns: Vec<&str>, + #[case] expected_row_ids: Vec, + ) { + let ds = gen_batch() + .col("x", array::step::()) + .into_dataset( + "memory://test_sql_system_column_injection", + FragmentCount::from(1), + FragmentRowCount::from(2), + ) + .await + .unwrap(); + + let batches = ds + .sql(sql) + .with_row_id(true) + .with_row_addr(true) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + + let batch = &batches[0]; + assert_eq!(batch.schema().field_names(), expected_columns); + for name in ["_rowid", "_rowaddr"] { + if expected_columns.contains(&name) { + assert_eq!( + batch[name].as_primitive::().values().as_ref(), + expected_row_ids.as_slice(), + "unexpected values for column {name}", + ); + } + } + } + + /// System columns must never be injected into queries whose output rows + /// are not one-to-one with dataset rows: under GROUP BY ALL or DISTINCT + /// the injected columns would become extra grouping/dedup keys and change + /// the relational results. + #[rstest] + #[case::group_by_all("SELECT x % 1 AS k, COUNT(*) AS n FROM dataset GROUP BY ALL ORDER BY k")] + #[case::group_by_expr("SELECT x % 1 AS k, COUNT(*) AS n FROM dataset GROUP BY k ORDER BY k")] + #[case::distinct("SELECT DISTINCT x % 1 AS k FROM dataset ORDER BY k")] + #[case::distinct_in_subquery( + "SELECT k FROM (SELECT DISTINCT x % 1 AS k FROM dataset) ORDER BY k" + )] + #[case::bare_aggregate("SELECT COUNT(*) AS n FROM dataset")] + #[tokio::test] + async fn test_sql_system_columns_skip_cardinality_changing_queries(#[case] sql: &str) { + let ds = gen_batch() + .col("x", array::step::()) + .into_dataset( + "memory://test_sql_system_columns_cardinality", + FragmentCount::from(1), + FragmentRowCount::from(2), + ) + .await + .unwrap(); + + let baseline = ds + .sql(sql) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + + let with_system_columns = ds + .sql(sql) + .with_row_id(true) + .with_row_addr(true) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + + pretty_assertions::assert_eq!(with_system_columns, baseline); + } + + #[tokio::test] + async fn test_sql_batch_size() { + let ds = gen_batch() + .col("x", array::step::()) + .into_dataset( + "memory://test_sql_batch_size", + FragmentCount::from(2), + FragmentRowCount::from(25), + ) + .await + .unwrap(); + + let batches = ds + .sql("SELECT x FROM dataset") + .batch_size(7) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 50); + assert!(batches.iter().all(|batch| batch.num_rows() <= 7)); + } + + #[tokio::test] + async fn test_sql_rejects_invalid_batch_size() { + let ds = gen_batch() + .col("x", array::step::()) + .into_dataset( + "memory://test_sql_rejects_invalid_batch_size", + FragmentCount::from(1), + FragmentRowCount::from(3), + ) + .await + .unwrap(); + + for batch_size in [0, u32::MAX as usize + 1] { + let error = ds + .sql("SELECT x FROM dataset") + .batch_size(batch_size) + .build() + .await + .err() + .expect("invalid batch size should be rejected"); + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains(&format!("batch_size must be between 1 and {}", u32::MAX)) + ); + } + } + + #[tokio::test] + async fn test_sql_batch_size_bytes_overrides_dataset_default() { + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "x", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..1000))], + ) + .unwrap(); + let test_dir = TempStrDir::default(); + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(&test_dir) + .with_read_params(ReadParams { + file_reader_options: Some(FileReaderOptions { + batch_size_bytes: Some(8_000), + ..Default::default() + }), + ..Default::default() + }) + .load() + .await + .unwrap(); + + let batches = dataset + .sql("SELECT x FROM dataset") + .batch_size_bytes(64) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 1000 + ); + assert!(batches.iter().all(|batch| batch.num_rows() <= 16)); + } + + #[tokio::test] + async fn test_sql_blob_all_binary() { + let schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", true)])); + let mut blobs = BlobArrayBuilder::new(2); + blobs.push_bytes(b"foo").unwrap(); + blobs.push_bytes(b"bar").unwrap(); + let batch = RecordBatch::try_new(schema.clone(), vec![blobs.finish().unwrap()]).unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://test_sql_blob_all_binary", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_3), + ..Default::default() + }), + ) + .await + .unwrap(); + + let batches = dataset + .sql("SELECT blob FROM dataset") + .blob_handling(BlobHandling::AllBinary) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + let blobs = batches[0].column(0).as_binary::(); + assert_eq!(blobs.value(0), b"foo"); + assert_eq!(blobs.value(1), b"bar"); + + // Expressions over the blob column require the planner to see the + // materialized LargeBinary type instead of the blob descriptor struct. + let batches = dataset + .sql("SELECT blob = X'666f6f' FROM dataset") + .blob_handling(BlobHandling::AllBinary) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + let is_foo = batches[0].column(0).as_boolean(); + assert!(is_foo.value(0)); + assert!(!is_foo.value(1)); + + let batches = dataset + .sql("SELECT blob, _rowid, _rowaddr FROM dataset") + .with_row_id(true) + .with_row_addr(true) + .blob_handling(BlobHandling::AllBinary) + .build() + .await + .unwrap() + .into_batch_records() + .await + .unwrap(); + let batch = &batches[0]; + let blobs = batch.column(0).as_binary::(); + assert_eq!(blobs.value(0), b"foo"); + assert_eq!(blobs.value(1), b"bar"); + let row_ids = batch.column(1).as_primitive::(); + assert_eq!(row_ids.value(0), 0); + assert_eq!(row_ids.value(1), 1); + let row_addrs = batch.column(2).as_primitive::(); + assert_eq!(row_addrs.value(0), 0); + assert_eq!(row_addrs.value(1), 1); + } + #[tokio::test] async fn test_sql_count() { let ds = gen_batch() diff --git a/rust/lance/src/dataset/statistics.rs b/rust/lance/src/dataset/statistics.rs index 627ccfc1081..749c58e04d9 100644 --- a/rust/lance/src/dataset/statistics.rs +++ b/rust/lance/src/dataset/statistics.rs @@ -13,7 +13,8 @@ use lance_index::scalar::zonemap::ZoneMapIndex; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use roaring::RoaringBitmap; -use super::{Dataset, fragment::FileFragment}; +use super::overlay::{collect_overlay_stale_frags, overlaid_fragments}; +use super::{Dataset, fragment::FileFragment, versions}; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt}; /// Statistics about a single field in the dataset @@ -52,32 +53,12 @@ impl DatasetStatisticsExt for Dataset { }, ) })); - if !self.is_legacy_storage() { - let scan_scheduler = ScanScheduler::new( - self.object_store.clone(), - SchedulerConfig::max_bandwidth(self.object_store.as_ref()), - ); - let schema = self.schema().clone(); - let dataset = self.clone(); - let fragments = self.fragments().as_ref().clone(); - futures::stream::iter(fragments) - .map(|fragment| { - let file_fragment = FileFragment::new(dataset.clone(), fragment); - let schema = schema.clone(); - let scan_scheduler = scan_scheduler.clone(); - async move { file_fragment.storage_stats(&schema, scan_scheduler).await } - }) - .buffer_unordered(self.object_store.io_parallelism()) - .try_for_each(|fragment_stats| { - for (field_id, bytes) in fragment_stats { - if let Some(stats) = field_stats.get_mut(&field_id) { - stats.bytes_on_disk += bytes; - } - } - futures::future::ready(Ok(())) - }) - .await?; - } + versions::collect_data_stats( + self.manifest().data_storage_format.lance_file_format(), + self, + &mut field_stats, + ) + .await?; let field_stats = field_ids .into_iter() .map(|id| field_stats.remove(&(id as u32)).unwrap()) @@ -88,6 +69,35 @@ impl DatasetStatisticsExt for Dataset { } } +pub(super) async fn collect_current_data_stats( + dataset: &Arc, + field_stats: &mut HashMap, +) -> Result<()> { + let scan_scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(dataset.object_store.as_ref()), + ); + let schema = dataset.schema().clone(); + let fragments = dataset.fragments().as_ref().clone(); + futures::stream::iter(fragments) + .map(|fragment| { + let file_fragment = FileFragment::new(dataset.clone(), fragment); + let schema = schema.clone(); + let scan_scheduler = scan_scheduler.clone(); + async move { file_fragment.storage_stats(&schema, scan_scheduler).await } + }) + .buffer_unordered(dataset.object_store.io_parallelism()) + .try_for_each(|fragment_stats| { + for (field_id, bytes) in fragment_stats { + if let Some(stats) = field_stats.get_mut(&field_id) { + stats.bytes_on_disk += bytes; + } + } + futures::future::ready(Ok(())) + }) + .await +} + /// A read-only handle for cheap, index-derived statistics about a [`Dataset`]. /// /// Obtained via [`Dataset::statistics`]. Groups statistics accessors behind one @@ -108,8 +118,9 @@ impl<'a> DatasetStatistics<'a> { /// /// `None` unless the column's index segments *jointly* cover every live /// fragment and the column can be soundly bounded — fragments appended after - /// the index was built, or a NaN-bearing column, yield `None`. The disjoint - /// segments of a multi-segment index are folded together. + /// the index was built, a data overlay committed after a segment was built, + /// or a NaN-bearing column all yield `None`. The disjoint segments of a + /// multi-segment index are folded together. /// /// When `Some`, the range is a superset of live values, conservative under /// deletion vectors: safe to prune with. See [`ScalarIndex::value_range`]. @@ -134,7 +145,12 @@ impl<'a> DatasetStatistics<'a> { let indices = dataset.load_indices().await?; let segments: Vec<_> = indices .iter() - .filter(|idx| matches!(idx.fields.as_slice(), [only] if *only == field_id)) + .filter(|idx| { + // A covered index still answers for its keyed column; only + // the keyed prefix decides whether this index matches, not + // the full `fields` vector including carried columns. + idx.keyed_field() == Some(field_id) + }) .filter(|idx| { idx.index_details .as_ref() @@ -159,6 +175,21 @@ impl<'a> DatasetStatistics<'a> { return Ok(None); } + // Soundness: a data overlay committed after a segment was built can move a value + // outside that segment's summaries without the ZoneMap ever seeing it, so the fold + // would no longer bound the live values. There is no way to widen the range without + // reading the overlay, so report "unknown" instead. + let overlaid = overlaid_fragments(&dataset.manifest.fragments); + if !overlaid.is_empty() { + let mut stale = RoaringBitmap::new(); + for idx in &segments { + collect_overlay_stale_frags(idx, &overlaid, &mut stale, dataset.schema())?; + } + if !stale.is_disjoint(dataset.fragment_bitmap.as_ref()) { + return Ok(None); + } + } + // Keep the opened indices alive so the `ZoneMapIndex` refs we fold over // stay borrowed. let mut opened = Vec::with_capacity(segments.len()); diff --git a/rust/lance/src/dataset/take.rs b/rust/lance/src/dataset/take.rs index 5d42e80473b..ddb76a0f720 100644 --- a/rust/lance/src/dataset/take.rs +++ b/rust/lance/src/dataset/take.rs @@ -228,7 +228,7 @@ async fn do_take_rows( .with_row_created_at_version(with_row_created_at_version_in_projection) .with_row_last_updated_at_version(with_row_last_updated_at_version_in_projection); let reader = fragment.open(&physical_schema, read_config).await?; - reader.legacy_read_range_as_batch(range).await + reader.read_range_as_batch(range).await } else if row_addr_stats.sorted { // Don't need to re-arrange data, just concatenate let mut batches: Vec<_> = Vec::new(); @@ -299,7 +299,10 @@ async fn do_take_rows( .or_insert_with(|| vec![offset]); }); - let fragments = builder.dataset.get_fragments(); + let addressed_ids: Vec = row_addrs_per_fragment.keys().copied().collect(); + let fragments = builder + .dataset + .get_existing_fragments_from_ids(&addressed_ids); let fragment_and_indices = fragments.into_iter().filter_map(|f| { let row_offset = row_addrs_per_fragment.remove(&(f.id() as u32))?; Some((f, row_offset)) @@ -478,6 +481,14 @@ pub struct TakeBuilder { row_addrs: Option>, projection: Arc, with_row_address: bool, + missing_row_policy: MissingRowPolicy, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) enum MissingRowPolicy { + #[default] + Ignore, + Error, } impl TakeBuilder { @@ -493,6 +504,7 @@ impl TakeBuilder { projection: Arc::new(projection.into_projection_plan(dataset.clone())?), dataset, with_row_address: false, + missing_row_policy: MissingRowPolicy::default(), }) } @@ -508,6 +520,7 @@ impl TakeBuilder { projection, dataset, with_row_address: false, + missing_row_policy: MissingRowPolicy::default(), }) } @@ -517,6 +530,11 @@ impl TakeBuilder { self } + pub(super) fn with_missing_row_policy(mut self, policy: MissingRowPolicy) -> Self { + self.missing_row_policy = policy; + self + } + /// Execute the take operation and return a single batch pub async fn execute(self) -> Result { take_rows(self).await @@ -537,8 +555,20 @@ impl TakeBuilder { .as_ref() .expect("row_ids must be set if row_addrs is not"); let addrs = if let Some(row_id_index) = get_row_id_index(&self.dataset).await? { - row_id_index - .get_many(row_ids) + let resolved = row_id_index.get_many(row_ids)?; + if self.missing_row_policy == MissingRowPolicy::Error + && let Some(first_missing_index) = + resolved.iter().position(|address| address.is_none()) + { + let missing_count = resolved.iter().filter(|address| address.is_none()).count(); + return Err(Error::invalid_input(format!( + "Could not resolve all requested row IDs: requested {}, resolved {}; first missing row ID {} was deleted or not found", + row_ids.len(), + row_ids.len() - missing_count, + row_ids[first_missing_index] + ))); + } + resolved .into_iter() .filter_map(|opt| opt.map(|address| address.into())) .collect::>() diff --git a/rust/lance/src/dataset/tests/data_file_part.rs b/rust/lance/src/dataset/tests/data_file_part.rs new file mode 100644 index 00000000000..c15c37c928f --- /dev/null +++ b/rust/lance/src/dataset/tests/data_file_part.rs @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::{collections::HashMap, fs, ops::Range, sync::Arc}; + +use arrow::array::AsArray; +use arrow_array::{ + ArrayRef, LargeBinaryArray, RecordBatch, RecordBatchIterator, StringArray, StructArray, + UInt64Array, types::Int32Type, +}; +use arrow_schema::{DataType, Field, Schema as ArrowSchema}; +use bytes::Bytes; +use futures::stream; +use lance_arrow::{ARROW_EXT_NAME_KEY, BLOB_V2_EXT_NAME}; +use lance_core::{ + datatypes::{BLOB_V2_LOGICAL_FIELDS, BlobHandling}, + utils::tempfile::TempDir, +}; +use lance_file::concat::EncodedFileInput; +use lance_file::version::LanceFileVersion; +use lance_io::{ + scheduler::{ScanScheduler, SchedulerConfig}, + utils::CachedFileSize, +}; +use lance_table::format::BasePath; + +use crate::blob::{BlobArrayBuilder, BlobDescriptorArrayBuilder, blob_field}; +use crate::dataset::fragment::FileFragment; +use crate::dataset::transaction::{DataReplacementGroup, Operation}; +use crate::dataset::write::WriteParams; +use crate::dataset::{DataFilePart, DataFileTarget, WriteDestination}; +use crate::{Dataset, Result}; + +async fn dataset_of(batch: RecordBatch, version: LanceFileVersion) -> Dataset { + let schema = batch.schema(); + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + Some(WriteParams { + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap() +} + +fn complete_logical_blob_batch(uri: &str, position: u64, size: u64) -> RecordBatch { + let field = Field::new( + "blob", + DataType::Struct(BLOB_V2_LOGICAL_FIELDS.clone()), + true, + ) + .with_metadata(HashMap::from([( + ARROW_EXT_NAME_KEY.to_string(), + BLOB_V2_EXT_NAME.to_string(), + )])); + let array = StructArray::try_new( + BLOB_V2_LOGICAL_FIELDS.clone(), + vec![ + Arc::new(LargeBinaryArray::from(vec![None::<&[u8]>])) as ArrayRef, + Arc::new(StringArray::from(vec![Some(uri)])) as ArrayRef, + Arc::new(UInt64Array::from(vec![Some(position)])) as ArrayRef, + Arc::new(UInt64Array::from(vec![Some(size)])) as ArrayRef, + ], + None, + ) + .unwrap(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![field])), + vec![Arc::new(array)], + ) + .unwrap() +} + +fn only_fragment(dataset: &Dataset) -> FileFragment { + dataset.get_fragments().into_iter().next().unwrap() +} + +async fn write_part( + dataset: &Dataset, + target: &DataFileTarget, + staging_name: &str, + blob_ids: Option>, + batch: RecordBatch, +) -> DataFilePart { + let path = dataset.data_dir().join(staging_name); + let output = dataset.object_store.create(&path).await.unwrap(); + let summary = dataset + .write_data_file_part(target, output, blob_ids.clone(), stream::iter([Ok(batch)])) + .await + .unwrap(); + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::default_for_testing(), + ); + let file = scheduler + .open_file(&path, &CachedFileSize::new(summary.size_bytes)) + .await + .unwrap(); + target + .open_part( + EncodedFileInput::new(file).with_expected_num_rows(summary.num_rows), + blob_ids, + ) + .await + .unwrap() +} + +async fn commit(dataset: &Dataset, replacement: DataReplacementGroup) -> Result { + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset.clone())), + Operation::DataReplacement { + replacements: vec![replacement], + }, + Some(dataset.version_id()), + None, + None, + Arc::new(Default::default()), + false, + ) + .await +} + +#[tokio::test] +async fn concatenates_parts_in_caller_order_without_reusing_staging_files() { + let original = arrow_array::record_batch!(("id", Int32, [0, 1, 2, 3])).unwrap(); + let dataset = dataset_of(original, LanceFileVersion::V2_1).await; + let target = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + let first = write_part( + &dataset, + &target, + "part-1.lance", + None, + arrow_array::record_batch!(("id", Int32, [10, 11])).unwrap(), + ) + .await; + let second = write_part( + &dataset, + &target, + "part-2.lance", + None, + arrow_array::record_batch!(("id", Int32, [12, 13])).unwrap(), + ) + .await; + + let replacement = only_fragment(&dataset) + .write_columns_from_parts(&target, &[second, first]) + .await + .unwrap(); + assert_eq!(replacement.1.path, target.file_name()); + let dataset = commit(&dataset, replacement).await.unwrap(); + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!( + batch["id"].as_primitive::().values(), + &[12, 13, 10, 11] + ); +} + +#[tokio::test] +async fn fragment_adapter_rejects_incomplete_physical_coverage() { + let original = arrow_array::record_batch!(("id", Int32, [0, 1, 2])).unwrap(); + let dataset = dataset_of(original, LanceFileVersion::V2_1).await; + let target = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + let part = write_part( + &dataset, + &target, + "short-part.lance", + None, + arrow_array::record_batch!(("id", Int32, [10, 11])).unwrap(), + ) + .await; + let error = only_fragment(&dataset) + .write_columns_from_parts(&target, &[part]) + .await + .unwrap_err(); + assert!(error.to_string().contains("2 physical rows"), "{error}"); + assert!(error.to_string().contains("contains 3"), "{error}"); +} + +#[tokio::test] +async fn target_uses_an_ordinary_generated_data_file_name() { + let original = arrow_array::record_batch!(("id", Int32, [0, 1])).unwrap(); + let dataset = dataset_of(original, LanceFileVersion::V2_1).await; + let first = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + let second = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + + assert_ne!(first.file_name(), second.file_name()); + assert_eq!(first.file_name().len(), 56); + assert!(first.file_name().ends_with(".lance")); + assert!(!first.file_name().contains('/')); +} + +#[tokio::test] +async fn blob_part_requires_an_id_lease_before_writing() { + let schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", true)])); + let mut blobs = BlobArrayBuilder::new(1); + blobs.push_bytes(b"old").unwrap(); + let original = RecordBatch::try_new(schema.clone(), vec![blobs.finish().unwrap()]).unwrap(); + let dataset = dataset_of(original, LanceFileVersion::V2_2).await; + let target = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + let output = dataset + .object_store + .create(&dataset.data_dir().join("missing-lease-part.lance")) + .await + .unwrap(); + let mut replacement = BlobArrayBuilder::new(1); + replacement.push_bytes(b"new").unwrap(); + let batch = RecordBatch::try_new(schema, vec![replacement.finish().unwrap()]).unwrap(); + + let error = dataset + .write_data_file_part(&target, output, None, stream::iter([Ok(batch)])) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("requires a non-empty Blob ID range"), + "{error}" + ); +} + +#[tokio::test] +async fn data_file_part_rejects_non_empty_file_relative_inline_blob() { + let schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", true)])); + let mut blobs = BlobArrayBuilder::new(1); + blobs.push_bytes(b"ordinary-inline").unwrap(); + let batch = RecordBatch::try_new(schema, vec![blobs.finish().unwrap()]).unwrap(); + let dataset = dataset_of(batch, LanceFileVersion::V2_2).await; + let data_file = &only_fragment(&dataset).metadata.files[0]; + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::default_for_testing(), + ); + let file = scheduler + .open_file( + &dataset.data_dir().join(data_file.path.as_str()), + &data_file.file_size_bytes, + ) + .await + .unwrap(); + + let error = DataFilePart::open(EncodedFileInput::new(file), None, None) + .await + .unwrap_err(); + assert!(error.to_string().contains("non-empty Inline"), "{error}"); +} + +#[tokio::test] +async fn complete_logical_blob_schema_and_external_range_survive_assembly() { + let test_dir = TempDir::default(); + let dataset_path = test_dir.std_path().join("dataset"); + let external_base = test_dir.std_path().join("external"); + let external_objects = external_base.join("objects"); + fs::create_dir_all(&external_objects).unwrap(); + let external_path = external_objects.join("blob.bin"); + fs::write(&external_path, b"prefix-selected-suffix").unwrap(); + let external_uri = format!("file://{}", external_path.display()); + let external_base_uri = format!("file://{}", external_base.display()); + let original = complete_logical_blob_batch(&external_uri, 7, 8); + let schema = original.schema(); + let dataset = Dataset::write( + RecordBatchIterator::new([Ok(original)], schema), + dataset_path.to_str().unwrap(), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + initial_bases: Some(vec![BasePath { + id: 1, + name: Some("external".to_string()), + path: external_base_uri, + is_dataset_root: false, + }]), + ..Default::default() + }), + ) + .await + .unwrap(); + let target = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + assert_eq!( + target.schema().fields[0] + .children + .iter() + .map(|child| child.name.as_str()) + .collect::>(), + ["data", "uri", "position", "size"] + ); + + let part = write_part( + &dataset, + &target, + "complete-logical-part.lance", + Some(1..10), + complete_logical_blob_batch(&external_uri, 7, 8), + ) + .await; + assert_eq!(part.num_rows(), 1); + let replacement = only_fragment(&dataset) + .write_columns_from_parts(&target, &[part]) + .await + .unwrap(); + let dataset = commit(&dataset, replacement).await.unwrap(); + assert_eq!(dataset.schema().fields[0].children.len(), 4); + + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let batch = scanner.try_into_batch().await.unwrap(); + let values = batch["blob"].as_binary::(); + assert_eq!(values.value(0), b"selected"); +} + +#[tokio::test] +async fn blob_parts_write_sidecars_in_final_namespace_and_concat_descriptors() { + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + blob_field("blob", true), + ])); + let make_batch = |ids: Vec, values: Vec<&'static [u8]>| { + let mut blobs = BlobArrayBuilder::new(values.len()); + for value in values { + blobs.push_bytes(value).unwrap(); + } + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int32Array::from(ids)), + blobs.finish().unwrap(), + ], + ) + .unwrap() + }; + let make_prepared_batch = |id: i32, value: &'static [u8]| { + let mut blobs = BlobDescriptorArrayBuilder::new("blob"); + blobs.push_inline(Bytes::from_static(value)).unwrap(); + let (blob_field, blob_array) = blobs.finish().unwrap().into_parts(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + blob_field, + ])), + vec![ + Arc::new(arrow_array::Int32Array::from(vec![id])), + blob_array, + ], + ) + .unwrap() + }; + let dataset = dataset_of( + make_batch(vec![0, 1], vec![b"old-0", b"old-1"]), + LanceFileVersion::V2_2, + ) + .await; + let target = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + let first = write_part( + &dataset, + &target, + "blob-part-1.lance", + Some(1..10), + make_prepared_batch(10, b"replacement-0"), + ) + .await; + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::default_for_testing(), + ); + let file = scheduler + .open_file( + &dataset.data_dir().join("blob-part-1.lance"), + &CachedFileSize::unknown(), + ) + .await + .unwrap(); + let error = target + .open_part(EncodedFileInput::new(file), Some(20..30)) + .await + .unwrap_err(); + assert!( + error.to_string().contains("outside declared range"), + "{error}" + ); + let second = write_part( + &dataset, + &target, + "blob-part-2.lance", + Some(10..20), + make_batch(vec![11], vec![b"replacement-1"]), + ) + .await; + + let other_target = DataFileTarget::new( + None, + Arc::new(dataset.schema().clone()), + dataset.manifest.data_storage_format.lance_file_format(), + ) + .unwrap(); + let error = dataset + .concat_data_file_parts(&other_target, &[first.clone(), second.clone()]) + .await + .unwrap_err(); + assert!(error.to_string().contains("Blob target ID"), "{error}"); + assert!( + !dataset + .object_store + .exists(&dataset.data_dir().join(other_target.file_name())) + .await + .unwrap() + ); + + let replacement = only_fragment(&dataset) + .write_columns_from_parts(&target, &[first, second]) + .await + .unwrap(); + let dataset = commit(&dataset, replacement).await.unwrap(); + let mut scanner = dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let batch = scanner.try_into_batch().await.unwrap(); + let values = batch["blob"].as_binary::(); + assert_eq!(values.value(0), b"replacement-0"); + assert_eq!(values.value(1), b"replacement-1"); +} diff --git a/rust/lance/src/dataset/tests/dataset_aggregate.rs b/rust/lance/src/dataset/tests/dataset_aggregate.rs index 5e55c860f5d..81aa945527d 100644 --- a/rust/lance/src/dataset/tests/dataset_aggregate.rs +++ b/rust/lance/src/dataset/tests/dataset_aggregate.rs @@ -22,7 +22,7 @@ use datafusion_substrait::substrait::proto::{ reference_segment::{self, StructField}, }, extensions::{ - SimpleExtensionDeclaration, SimpleExtensionUri, + SimpleExtensionDeclaration, SimpleExtensionUrn, simple_extension_declaration::{ExtensionFunction, MappingType}, }, function_argument::ArgType, @@ -95,17 +95,6 @@ fn create_aggregate_rel( git_hash: String::new(), producer: "lance-test".to_string(), }), - #[allow(deprecated)] - extension_uris: vec![ - SimpleExtensionUri { - extension_uri_anchor: 1, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), - }, - SimpleExtensionUri { - extension_uri_anchor: 2, - uri: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(), - }, - ], extensions, relations: vec![PlanRel { rel_type: Some(datafusion_substrait::substrait::proto::plan_rel::RelType::Root( @@ -117,7 +106,16 @@ fn create_aggregate_rel( }], advanced_extensions: None, expected_type_urls: vec![], - extension_urns: vec![], + extension_urns: vec![ + SimpleExtensionUrn { + extension_urn_anchor: 1, + urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_aggregate_generic.yaml".to_string(), + }, + SimpleExtensionUrn { + extension_urn_anchor: 2, + urn: "https://github.com/substrait-io/substrait/blob/main/extensions/functions_arithmetic.yaml".to_string(), + }, + ], parameter_bindings: vec![], type_aliases: vec![], }; @@ -129,9 +127,7 @@ fn create_aggregate_rel( fn agg_extension(anchor: u32, name: &str) -> SimpleExtensionDeclaration { SimpleExtensionDeclaration { mapping_type: Some(MappingType::ExtensionFunction(ExtensionFunction { - #[allow(deprecated)] - extension_uri_reference: 1, - extension_urn_reference: 0, + extension_urn_reference: 1, function_anchor: anchor, name: name.to_string(), })), @@ -1602,7 +1598,7 @@ async fn test_scanner_count_rows_with_fts() { assert_plan_node_equals( plan.clone(), "AggregateExec: mode=Single, gby=[], aggr=[count(Int32(1))] - MatchQuery: column=text, query=document", + MatchQuery: column=text, query=[document]", ) .await .unwrap(); diff --git a/rust/lance/src/dataset/tests/dataset_concurrency_store.rs b/rust/lance/src/dataset/tests/dataset_concurrency_store.rs index a9c2aa44c38..92bfc8f1b19 100644 --- a/rust/lance/src/dataset/tests/dataset_concurrency_store.rs +++ b/rust/lance/src/dataset/tests/dataset_concurrency_store.rs @@ -192,7 +192,7 @@ async fn test_add_bases() { use std::sync::Arc; // Create a test dataset - let test_uri = "memory://add_bases_test"; + let test_uri = "shared-memory://add_bases_test/primary"; let mut data_gen = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -213,13 +213,13 @@ async fn test_add_bases() { let new_bases = vec![ BasePath::new( 0, - "memory://bucket1".to_string(), + "shared-memory://add_bases_test/bucket1".to_string(), Some("bucket1".to_string()), false, ), BasePath::new( 0, - "memory://bucket2".to_string(), + "shared-memory://add_bases_test/bucket2".to_string(), Some("bucket2".to_string()), true, ), @@ -243,9 +243,9 @@ async fn test_add_bases() { .find(|bp| bp.name == Some("bucket2".to_string())) .expect("bucket2 not found"); - assert_eq!(bucket1.path, "memory://bucket1"); + assert_eq!(bucket1.path, "shared-memory://add_bases_test/bucket1"); assert!(!bucket1.is_dataset_root); - assert_eq!(bucket2.path, "memory://bucket2"); + assert_eq!(bucket2.path, "shared-memory://add_bases_test/bucket2"); assert!(bucket2.is_dataset_root); let updated_dataset = Arc::new(updated_dataset); @@ -253,7 +253,7 @@ async fn test_add_bases() { // Test conflict detection - try to add a base with the same name let conflicting_bases = vec![BasePath::new( 0, - "memory://bucket3".to_string(), + "shared-memory://add_bases_test/bucket3".to_string(), Some("bucket1".to_string()), false, )]; @@ -270,7 +270,7 @@ async fn test_add_bases() { // Test conflict detection - try to add a base with the same path let conflicting_bases = vec![BasePath::new( 0, - "memory://bucket1".to_string(), + "shared-memory://add_bases_test/bucket1".to_string(), Some("bucket3".to_string()), false, )]; @@ -292,7 +292,7 @@ async fn test_concurrent_add_bases_conflict() { use std::sync::Arc; // Create a test dataset - let test_uri = "memory://concurrent_add_bases_test"; + let test_uri = "shared-memory://concurrent_add_bases_test/primary"; let mut data_gen = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -314,7 +314,7 @@ async fn test_concurrent_add_bases_conflict() { // First transaction adds base1 let new_bases1 = vec![BasePath::new( 0, - "memory://bucket1".to_string(), + "shared-memory://concurrent_add_bases_test/bucket1".to_string(), Some("base1".to_string()), false, )]; @@ -325,7 +325,7 @@ async fn test_concurrent_add_bases_conflict() { // This should succeed as there's no conflict let new_bases2 = vec![BasePath::new( 0, - "memory://bucket2".to_string(), + "shared-memory://concurrent_add_bases_test/bucket2".to_string(), Some("base2".to_string()), false, )]; @@ -360,7 +360,7 @@ async fn test_concurrent_add_bases_name_conflict() { use std::sync::Arc; // Create a test dataset - let test_uri = "memory://concurrent_name_conflict_test"; + let test_uri = "shared-memory://concurrent_name_conflict_test/primary"; let mut data_gen = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -383,7 +383,7 @@ async fn test_concurrent_add_bases_name_conflict() { // First transaction adds base with name "shared_base" let new_bases1 = vec![BasePath::new( 0, - "memory://bucket1".to_string(), + "shared-memory://concurrent_name_conflict_test/bucket1".to_string(), Some("shared_base".to_string()), false, )]; @@ -394,7 +394,7 @@ async fn test_concurrent_add_bases_name_conflict() { // This should fail due to name conflict let new_bases2 = vec![BasePath::new( 0, - "memory://bucket2".to_string(), + "shared-memory://concurrent_name_conflict_test/bucket2".to_string(), Some("shared_base".to_string()), false, )]; @@ -416,7 +416,7 @@ async fn test_concurrent_add_bases_path_conflict() { use std::sync::Arc; // Create a test dataset - let test_uri = "memory://concurrent_path_conflict_test"; + let test_uri = "shared-memory://concurrent_path_conflict_test/primary"; let mut data_gen = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -436,10 +436,10 @@ async fn test_concurrent_add_bases_path_conflict() { let dataset = Arc::new(dataset); let dataset_clone = Arc::new(dataset_clone); - // First transaction adds base with path "memory://shared_path" + // First transaction adds a base at the shared path let new_bases1 = vec![BasePath::new( 0, - "memory://shared_path".to_string(), + "shared-memory://concurrent_path_conflict_test/shared_path".to_string(), Some("base1".to_string()), false, )]; @@ -450,7 +450,7 @@ async fn test_concurrent_add_bases_path_conflict() { // This should fail due to path conflict let new_bases2 = vec![BasePath::new( 0, - "memory://shared_path".to_string(), + "shared-memory://concurrent_path_conflict_test/shared_path".to_string(), Some("base2".to_string()), false, )]; @@ -472,7 +472,7 @@ async fn test_concurrent_add_bases_with_data_write() { use std::sync::Arc; // Create a test dataset - let test_uri = "memory://concurrent_write_test"; + let test_uri = "shared-memory://concurrent_write_test/primary"; let mut data_gen = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -494,7 +494,7 @@ async fn test_concurrent_add_bases_with_data_write() { // First transaction adds a new base let new_bases = vec![BasePath::new( 0, - "memory://bucket1".to_string(), + "shared-memory://concurrent_write_test/bucket1".to_string(), Some("base1".to_string()), false, )]; diff --git a/rust/lance/src/dataset/tests/dataset_index.rs b/rust/lance/src/dataset/tests/dataset_index.rs index 961b381e452..19673e83ae1 100644 --- a/rust/lance/src/dataset/tests/dataset_index.rs +++ b/rust/lance/src/dataset/tests/dataset_index.rs @@ -2,15 +2,21 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use std::vec; use crate::dataset::ROW_ID; +use crate::dataset::WriteDestination; use crate::dataset::builder::DatasetBuilder; use crate::dataset::tests::dataset_migrations::scan_dataset; use crate::dataset::tests::dataset_transactions::{assert_results, execute_sql}; +use crate::dataset::transaction::{DataReplacementGroup, Operation, Transaction}; use crate::index::vector::VectorIndexParams; use crate::session::Session; +use crate::utils::test::covering; use crate::{Dataset, Error, Result}; use lance_arrow::FixedSizeListArrayExt; @@ -19,28 +25,32 @@ use crate::index::DatasetIndexExt; use arrow::array::{AsArray, GenericListBuilder, GenericStringBuilder}; use arrow::datatypes::UInt64Type; use arrow_array::RecordBatch; -use arrow_array::{Array, GenericStringArray, StructArray, UInt64Array}; +use arrow_array::{Array, GenericStringArray, LargeListArray, ListArray, StructArray, UInt64Array}; use arrow_array::{ ArrayRef, Float32Array, Int32Array, RecordBatchIterator, StringArray, builder::StringDictionaryBuilder, - types::{Float32Type, Int32Type}, + types::{Float32Type, Int32Type, Int64Type}, }; use arrow_schema::{ DataType, Field as ArrowField, Field, Fields as ArrowFields, Schema as ArrowSchema, }; use lance_arrow::ARROW_EXT_NAME_KEY; -use lance_core::cache::LanceCache; +use lance_core::cache::{ + CacheBackend, CacheCodec, CacheEntry, InternalCacheKey, LanceCache, QuickCacheBackend, +}; use lance_core::utils::tempfile::TempStrDir; +use lance_datafusion::exec::ExecutionSummaryCounts; +use lance_datafusion::utils::PARTITIONS_SEARCHED_METRIC; use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch}; use lance_file::reader::{FileReader, FileReaderOptions}; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_index::optimize::OptimizeOptions; -use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::{ - InvertedListFormatVersion, - query::{BooleanQuery, MatchQuery, Occur, Operator, PhraseQuery}, + DocumentGranularity, InvertedListFormatVersion, SCORE_COL, + query::{BooleanQuery, BoostQuery, MatchQuery, Occur, Operator, PhraseQuery}, tokenizer::InvertedIndexParams, }; +use lance_index::scalar::{FullTextSearchQuery, ScalarIndex}; use lance_index::{FtsPrewarmOptions, PrewarmOptions}; use lance_index::{IndexType, scalar::ScalarIndexParams, vector::DIST_COL}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; @@ -52,6 +62,7 @@ use futures::{StreamExt, TryStreamExt}; use itertools::Itertools; use lance_arrow::json::ARROW_JSON_EXT_NAME; use lance_index::scalar::inverted::query::{FtsQuery, MultiMatchQuery}; +use lance_table::format::BasePath; use lance_testing::datagen::generate_random_array; use rand::Rng; use rstest::rstest; @@ -169,6 +180,280 @@ async fn test_create_index( assert!(fragment_bitmap.contains(0)); } +/// An index that merely *carries* a vector column must not be selected to +/// answer an ANN query against that column. +/// +/// Two traps this test is written to avoid: +/// - `early_pruning` raises `minimum_nprobes`, which can mask selection +/// differences behind a full scan of the partitions. +/// - `num_partitions = 1` means the probe path is never reached at all (the +/// fixture uses [`covering::NUM_PARTITIONS`]). +/// Both make a broken selection rule look correct, so this asserts on the +/// plan the scanner actually built, never on query results. +#[tokio::test] +async fn test_covered_vector_column_is_not_selected_for_ann() { + let test_uri = TempStrDir::default(); + let mut dataset = covering::write_two_vector_column_dataset(&test_uri).await; + covering::create_ivf_pq_index(&mut dataset, "vec").await; + covering::declare_covering(&mut dataset, "vec", "payload_vec").await; + + let query = generate_random_array(covering::DIMENSION as usize); + + // An ANN query on the carried column must fall back to a flat scan. + let mut scan = dataset.scan(); + // `use_index` stays on: the point is that selection declines this + // index, not that the caller disabled indexing. + scan.nearest("payload_vec", &query, 10).unwrap(); + let plan = scan.explain_plan(true).await.unwrap(); + assert!( + plan.contains("KNNVectorDistance"), + "expected a flat KNN plan for the covered column, got:\n{plan}" + ); + assert!( + !plan.contains("ANNIvfPartition"), + "the covered column must not be served by the ANN index:\n{plan}" + ); + + // The keyed column still uses the index -- without this, the test would + // pass just as well against an index that was never selected at all. + let mut scan = dataset.scan(); + scan.nearest("vec", &query, 10).unwrap(); + let plan = scan.explain_plan(true).await.unwrap(); + assert!( + plan.contains("ANNIvfPartition"), + "the keyed column must still use the index:\n{plan}" + ); +} + +/// A filtered `describe_indices` must still find a covered index by its keyed +/// column. The matcher compares the caller's resolved field slice against the one +/// column named by `for_column`, so a caller that passes all of `index.fields` -- +/// carried columns included -- pushes that slice past length one and gets a silent +/// "no match" for every covered index. +#[tokio::test] +async fn test_describe_indices_filters_a_covered_index_by_its_keyed_field() { + use lance_index::IndexCriteria; + + let test_uri = TempStrDir::default(); + let mut dataset = covering::write_vector_payload_dataset(&test_uri).await; + covering::create_ivf_pq_index(&mut dataset, "vec").await; + + // Baseline: the plain index is found, so a later miss is about covering. + let found = dataset + .describe_indices(Some(IndexCriteria::default().for_column("vec"))) + .await + .unwrap(); + assert_eq!(found.len(), 1, "precondition: the plain index is findable"); + + let (vec_id, _) = covering::declare_covering(&mut dataset, "vec", "payload").await; + + let found = dataset + .describe_indices(Some(IndexCriteria::default().for_column("vec"))) + .await + .unwrap(); + assert_eq!( + found.len(), + 1, + "a covered index must still be findable by its keyed column" + ); + assert_eq!( + found[0].field_ids(), + &[vec_id as u32], + "and must advertise only the keyed column" + ); + + // The carried column is not searchable, so it must not match. + let carried = dataset + .describe_indices(Some(IndexCriteria::default().for_column("payload"))) + .await + .unwrap(); + assert!( + carried.is_empty(), + "a carried column must not advertise an index" + ); +} + +/// An unfiltered `optimize_indices()` is a table-wide request, so a covered index +/// must not abort it -- that would block optimization of every other index on the +/// table over one this build merely cannot rebuild. It is skipped with a warning. +/// Only a caller that names the covered index gets an error (see +/// `test_optimize_indices_rejects_a_covered_index`). +/// +/// Both the current and the stale case are covered: erroring on the stale one +/// aborts the loop before the replacements accumulated for the other groups are +/// committed, leaving an unrelated index stale too. +#[rstest] +#[case::current(false)] +#[case::stale(true)] +#[tokio::test] +async fn test_optimize_skips_a_covered_index_without_blocking_others(#[case] stale: bool) { + let test_uri = TempStrDir::default(); + let mut dataset = covering::write_vector_payload_dataset(&test_uri).await; + + // A covered vector index, and a plain scalar index that optimize may touch. + covering::create_ivf_pq_index(&mut dataset, "vec").await; + covering::create_btree_index(&mut dataset, "payload", Some("payload_idx")).await; + + let (_, payload_id) = covering::declare_covering(&mut dataset, "vec", "payload").await; + let covered_uuid = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| !idx.covering_fields.is_empty()) + .expect("the covered index should exist") + .uuid; + + if stale { + // Now *both* groups have an unindexed fragment, so the covered group + // would genuinely be rebuilt -- this is where the refusal used to fire. + covering::append_vector_payload_rows(&mut dataset, 256).await; + } + + dataset + .optimize_indices(&OptimizeOptions::default()) + .await + .expect("a covered index must not abort an unfiltered optimize"); + + let after = dataset.load_indices().await.unwrap(); + assert!( + after + .iter() + .any(|idx| idx.uuid == covered_uuid && idx.covering_fields == vec![payload_id]), + "the covered index must be left exactly as it was" + ); + + if stale { + let payload_idx = after + .iter() + .filter(|idx| idx.name == "payload_idx") + .filter_map(|idx| idx.fragment_bitmap.as_ref()) + .fold(roaring::RoaringBitmap::new(), |mut acc, bitmap| { + acc |= bitmap; + acc + }); + assert!( + payload_idx.contains(1), + "the unrelated index must still have been optimized onto the new fragment, got {payload_idx:?}" + ); + } else { + assert_eq!(after.len(), 2, "both indices must survive"); + } +} + +/// The same skip, for a *scalar* covered index. The rule does not branch on index +/// type, but scalar groups take their own no-work path a few lines below the +/// covering gate, so a covered scalar index reaching that gate first is worth +/// pinning separately from the vector case above. +/// +/// The append is what gives this test teeth. Without it the covered group has no +/// work either way, so the scalar no-work path below the gate produces the same +/// observable outcome as the gate itself and the test passes with the gate +/// removed entirely. +#[tokio::test] +async fn test_optimize_skips_a_stale_covered_scalar_index() { + let test_uri = TempStrDir::default(); + let mut dataset = covering::write_three_int_column_dataset(&test_uri).await; + covering::create_btree_index(&mut dataset, "a", None).await; + covering::create_btree_index(&mut dataset, "b", None).await; + + let (_, carried_id) = covering::declare_covering(&mut dataset, "a", "carried").await; + let covered_uuid = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| !idx.covering_fields.is_empty()) + .expect("the covered index should exist") + .uuid; + + // Both scalar groups now have an unindexed fragment, so the covered one + // would genuinely be rebuilt if the gate did not skip it first. + covering::append_three_int_column_rows(&mut dataset, 64).await; + + dataset + .optimize_indices(&OptimizeOptions::default()) + .await + .expect("a stale covered scalar index must not abort optimize"); + + let after = dataset.load_indices().await.unwrap(); + assert!( + after + .iter() + .any(|idx| idx.uuid == covered_uuid && idx.covering_fields == vec![carried_id]), + "the covered scalar index must be left exactly as it was, not rebuilt" + ); + // The unrelated scalar index was still maintained, so the skip is scoped to + // the covered group rather than aborting the loop. + let b_id = dataset.schema().field_id("b").unwrap(); + let b_coverage = after + .iter() + .filter(|idx| idx.fields == vec![b_id]) + .filter_map(|idx| idx.fragment_bitmap.as_ref()) + .fold(roaring::RoaringBitmap::new(), |mut acc, bitmap| { + acc |= bitmap; + acc + }); + assert!( + b_coverage.contains(1), + "the unrelated scalar index must still have been optimized onto the new fragment, got {b_coverage:?}" + ); +} + +/// A caller that names the covered index asked for it specifically, so the +/// refusal is loud rather than a skip. +#[tokio::test] +async fn test_optimize_indices_rejects_a_covered_index() { + let test_uri = TempStrDir::default(); + let mut dataset = covering::write_vector_payload_dataset(&test_uri).await; + covering::create_ivf_pq_index(&mut dataset, "vec").await; + + // Nothing writes carried values, so the storage does not contain `payload` + // -- which is exactly why optimize must refuse: it rebuilds from a scan + // projecting the keyed field and `_rowid` only, and would republish the + // declaration on a segment that still has no payload. + let (vec_id, payload_id) = covering::declare_covering(&mut dataset, "vec", "payload").await; + + // Append AFTER declaring covering, so the group really would be rebuilt. + // Without it this would assert the refusal against an index optimize had no + // work for, and would keep passing if the refusal moved behind a no-work + // check. + covering::append_vector_payload_rows(&mut dataset, 256).await; + + let before = dataset.load_indices().await.unwrap(); + let before_uuid = before[0].uuid; + let covered_name = before[0].name.clone(); + + // Name the covered index: the refusal is reserved for a caller that targeted + // it. An unfiltered call skips it instead, which + // `test_optimize_skips_a_covered_index_without_blocking_others` covers. + let err = dataset + .optimize_indices(&OptimizeOptions::default().index_names(vec![covered_name])) + .await + .expect_err("optimizing a targeted covered index must be refused"); + assert!( + err.to_string().contains("declares covering fields"), + "unexpected message: {err}" + ); + + // Refused, not partially applied: the index is exactly as it was. + let after = dataset.load_indices().await.unwrap(); + assert_eq!(after.len(), 1); + assert_eq!( + after[0].uuid, before_uuid, + "a refused optimize must not replace the index" + ); + assert_eq!(after[0].covering_fields, vec![payload_id]); + assert_eq!(after[0].fields, vec![vec_id, payload_id]); + + // The appended data above is unindexed, so this really is a case optimize + // would otherwise have merged -- the refusal is not the no-op path. + assert!( + after.iter().all(|idx| !idx.covering_fields.is_empty()), + "precondition: the only index is still the covered one" + ); +} + #[rstest] #[tokio::test] async fn test_create_scalar_index( @@ -215,6 +500,90 @@ async fn test_create_scalar_index( dataset.index_statistics(&index_name).await.unwrap(); } +#[tokio::test] +async fn test_btree_nullable_filters_match_unindexed_scan() { + let test_uri = TempStrDir::default(); + let num_rows = 10_000u64; + let values: Int32Array = (0..num_rows).map(|id| (id % 5 == 0).then_some(7)).collect(); + let ids = UInt64Array::from_iter_values(0..num_rows); + let batch = RecordBatch::try_from_iter(vec![ + ("value", Arc::new(values) as ArrayRef), + ("id", Arc::new(ids) as ArrayRef), + ]) + .unwrap(); + let schema = batch.schema(); + let reader = RecordBatchIterator::new([Ok(batch)], schema); + let mut dataset = Dataset::write( + reader, + &test_uri, + Some(WriteParams { + max_rows_per_file: 2_500, + ..Default::default() + }), + ) + .await + .unwrap(); + assert!(dataset.get_fragments().len() > 1); + + dataset + .create_index( + &["value"], + IndexType::BTree, + Some("value_btree".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + for predicate in [ + "value = 7", + "value IN (7, 99)", + "NOT (value = 99)", + "NOT (value = 7)", + "NOT (value = 99 OR value = 7)", + "NOT (NOT (value = 7))", + "value = 99 OR value = 7", + ] { + let mut indexed_scan = dataset.scan(); + indexed_scan + .filter(predicate) + .unwrap() + .project(&["id"]) + .unwrap(); + let plan = indexed_scan.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ScalarIndexQuery") && plan.contains("BTree"), + "Expected BTree scalar index query for {predicate}:\n{plan}" + ); + let indexed = indexed_scan.try_into_batch().await.unwrap(); + + let mut baseline_scan = dataset.scan(); + baseline_scan.use_scalar_index(false); + baseline_scan + .filter(predicate) + .unwrap() + .project(&["id"]) + .unwrap(); + let baseline = baseline_scan.try_into_batch().await.unwrap(); + + let sorted_ids = |batch: &RecordBatch| { + let mut ids = batch + .column(0) + .as_primitive::() + .values() + .to_vec(); + ids.sort_unstable(); + ids + }; + assert_eq!( + sorted_ids(&indexed), + sorted_ids(&baseline), + "indexed result differs for {predicate}" + ); + } +} + async fn create_bad_file(data_storage_version: LanceFileVersion) -> Result { let test_uri = TempStrDir::default(); @@ -849,18 +1218,2529 @@ async fn test_fts_on_multiple_columns() { .unwrap(); assert_eq!(results.num_rows(), 1); - let results = dataset - .scan() - .full_text_search( - FullTextSearchQuery::new("common".to_owned()) - .with_column("content".to_owned()) - .unwrap(), - ) - .unwrap() - .try_into_batch() - .await - .unwrap(); - assert_eq!(results.num_rows(), 1); + let results = dataset + .scan() + .full_text_search( + FullTextSearchQuery::new("common".to_owned()) + .with_column("content".to_owned()) + .unwrap(), + ) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(results.num_rows(), 1); +} + +async fn create_fragmented_fts_index(dataset: &mut Dataset, column: &str, with_position: bool) { + create_fragmented_fts_index_with_order(dataset, column, with_position, false).await; +} + +async fn create_fragmented_fts_index_with_order( + dataset: &mut Dataset, + column: &str, + with_position: bool, + reverse_segments: bool, +) { + let mut fragment_groups = dataset + .get_fragments() + .iter() + .map(|fragment| vec![fragment.id() as u32]) + .collect::>(); + if reverse_segments { + fragment_groups.reverse(); + } + create_fragmented_fts_index_with_groups(dataset, column, with_position, fragment_groups).await; +} + +async fn create_fragmented_fts_index_with_groups( + dataset: &mut Dataset, + column: &str, + with_position: bool, + fragment_groups: Vec>, +) { + let index_name = format!("{column}_idx"); + let columns = [column]; + let params = InvertedIndexParams::default().with_position(with_position); + let expected_segments = fragment_groups.len(); + let mut segments = Vec::with_capacity(expected_segments); + for fragment_ids in fragment_groups { + let mut builder = dataset + .create_index_builder(&columns, IndexType::Inverted, ¶ms) + .name(index_name.clone()) + .fragments(fragment_ids); + segments.push(builder.execute_uncommitted().await.unwrap()); + } + dataset + .commit_existing_index_segments(&index_name, column, segments) + .await + .unwrap(); + + let segments = + crate::index::scalar::inverted::load_segments(dataset, column, DocumentGranularity::Row) + .await + .unwrap() + .unwrap(); + assert_eq!(segments.len(), expected_segments); +} + +fn compound_multimatch_query() -> FtsQuery { + MultiMatchQuery::try_new( + "common".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .try_with_boosts(vec![10.0, 1.0]) + .unwrap() + .into() +} + +fn compound_match_query(term: &str, column: &str, boost: f32) -> FtsQuery { + MatchQuery::new(term.to_owned()) + .with_column(Some(column.to_owned())) + .with_boost(boost) + .into() +} + +async fn compound_fts_results( + dataset: &Dataset, + query: FtsQuery, + limit: Option, +) -> Vec<(u64, f32)> { + let mut scan = dataset.scan(); + scan.with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + if let Some(limit) = limit { + scan.limit(Some(limit), None).unwrap(); + } + let batch = scan.try_into_batch().await.unwrap(); + let row_ids = batch[ROW_ID].as_primitive::().values(); + let scores = batch[SCORE_COL].as_primitive::().values(); + row_ids + .iter() + .copied() + .zip(scores.iter().copied()) + .collect() +} + +fn scored_row_bits(rows: &[(u64, f32)]) -> Vec<(u64, u32)> { + rows.iter() + .map(|(row_id, score)| (*row_id, score.to_bits())) + .collect() +} + +fn compound_fts_result_bits(batch: &RecordBatch) -> Vec<(u64, u32)> { + let row_ids = batch[ROW_ID].as_primitive::().values(); + let scores = batch[SCORE_COL].as_primitive::().values(); + row_ids + .iter() + .copied() + .zip(scores.iter().map(|score| score.to_bits())) + .collect() +} + +async fn assert_compound_fts_top_k(dataset: &Dataset, query: FtsQuery, limit: usize) { + let exhaustive = compound_fts_results(dataset, query.clone(), None).await; + assert!( + exhaustive.len() > limit, + "the exhaustive result must contain candidates beyond k" + ); + let limited = compound_fts_results(dataset, query, Some(limit as i64)).await; + assert_eq!(limited, exhaustive[..limit]); +} + +fn expected_must_score_sum(left: Vec<(u64, f32)>, right: Vec<(u64, f32)>) -> Vec<(u64, f32)> { + let right = right.into_iter().collect::>(); + let mut expected = left + .into_iter() + .filter_map(|(row_id, left_score)| { + right + .get(&row_id) + .map(|right_score| (row_id, left_score + right_score)) + }) + .collect::>(); + expected.sort_unstable_by(|(left_row_id, left_score), (right_row_id, right_score)| { + right_score + .total_cmp(left_score) + .then_with(|| left_row_id.cmp(right_row_id)) + }); + expected +} + +const CROSS_COLUMN_COMPOUND_FTS_SCORER: &str = "CrossColumnCompoundFtsScorer"; + +fn independent_compound_fts_oracle<'a>( + dataset: &'a Dataset, + query: &'a FtsQuery, +) -> Pin> + Send + 'a>> { + Box::pin(async move { + match query { + FtsQuery::Match(_) | FtsQuery::Phrase(_) => { + compound_fts_results(dataset, query.clone(), None) + .await + .into_iter() + .collect() + } + FtsQuery::MultiMatch(query) => { + let mut result = HashMap::new(); + for match_query in &query.match_queries { + let leaf = FtsQuery::Match(match_query.clone()); + for (row_id, score) in independent_compound_fts_oracle(dataset, &leaf).await { + result + .entry(row_id) + .and_modify(|current| { + if score.total_cmp(current).is_gt() { + *current = score; + } + }) + .or_insert(score); + } + } + result + } + FtsQuery::Boost(query) => { + let mut result = + independent_compound_fts_oracle(dataset, query.positive.as_ref()).await; + let negative = + independent_compound_fts_oracle(dataset, query.negative.as_ref()).await; + for (row_id, negative_score) in negative { + if let Some(score) = result.get_mut(&row_id) { + *score -= query.negative_boost * negative_score; + } + } + result + } + FtsQuery::Boolean(query) => { + let mut required = None::>; + for clause in &query.must { + let clause = independent_compound_fts_oracle(dataset, clause).await; + if let Some(required) = required.as_mut() { + required.retain(|row_id, score| { + clause.get(row_id).is_some_and(|clause_score| { + *score += *clause_score; + true + }) + }); + } else { + required = Some(clause); + } + } + + let has_required = required.is_some(); + let mut result = required.unwrap_or_default(); + for clause in &query.should { + let clause = independent_compound_fts_oracle(dataset, clause).await; + for (row_id, clause_score) in clause { + if has_required { + if let Some(score) = result.get_mut(&row_id) { + *score += clause_score; + } + } else { + *result.entry(row_id).or_insert(0.0) += clause_score; + } + } + } + + for clause in &query.must_not { + for row_id in independent_compound_fts_oracle(dataset, clause) + .await + .keys() + { + result.remove(row_id); + } + } + result + } + } + }) +} + +fn sorted_compound_fts_oracle(result: HashMap) -> Vec<(u64, f32)> { + result + .into_iter() + .sorted_unstable_by(|(left_row_id, left_score), (right_row_id, right_score)| { + right_score + .total_cmp(left_score) + .then_with(|| left_row_id.cmp(right_row_id)) + }) + .collect() +} + +fn assert_scored_rows_close(case_name: &str, actual: &[(u64, f32)], expected: &[(u64, f32)]) { + assert_eq!( + actual.len(), + expected.len(), + "{case_name} returned a different number of rows" + ); + for ((actual_row_id, actual_score), (expected_row_id, expected_score)) in + actual.iter().zip(expected) + { + assert_eq!( + actual_row_id, expected_row_id, + "{case_name} returned rows in the wrong order" + ); + let tolerance = 1.0e-5 * expected_score.abs().max(1.0); + assert!( + (actual_score - expected_score).abs() <= tolerance, + "{case_name} returned score {actual_score} for row {actual_row_id}, expected {expected_score}" + ); + } +} + +async fn assert_compound_matches_independent_oracle( + dataset: &Dataset, + case_name: &str, + query: &FtsQuery, + limit: usize, +) -> Vec<(u64, f32)> { + let mut expected = + sorted_compound_fts_oracle(independent_compound_fts_oracle(dataset, query).await); + assert!( + expected.len() > limit, + "{case_name} must have candidates beyond k" + ); + expected.truncate(limit); + let actual = compound_fts_results(dataset, query.clone(), Some(limit as i64)).await; + assert_scored_rows_close(case_name, &actual, &expected); + actual +} + +async fn compound_fts_plan(dataset: &Dataset, query: FtsQuery, limit: usize) -> String { + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + scanner.limit(Some(limit as i64), None).unwrap(); + scanner.explain_plan(false).await.unwrap() +} + +async fn write_cross_column_compound_dataset() -> Dataset { + let batch = arrow_array::record_batch!( + ( + "title", + Utf8, + [ + "alpha quick brown fox", + "alpha quick fox brown", + "quick brown fox", + "tie", + "alpha blocked", + "noise", + "alpha quick brown", + "tie", + "alpha", + "noise" + ] + ), + ( + "body", + Utf8, + [ + "gamma", + "gamma gamma optional", + "gamma", + "tiebody", + "gamma blocked", + "gamma optional", + "noise", + "tiebody", + "optional", + "blocked" + ] + ), + ("id", Int32, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + ) + .unwrap(); + let schema = batch.schema(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 5, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + dataset +} + +#[tokio::test] +async fn test_cross_column_compound_scorer_matches_independent_leaf_oracle() { + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + + let phrase = || { + PhraseQuery::new("quick brown".to_owned()) + .with_column(Some("title".to_owned())) + .into() + }; + let phrase_approximation: FtsQuery = MatchQuery::new("quick brown".to_owned()) + .with_column(Some("title".to_owned())) + .with_operator(Operator::And) + .into(); + let phrase_query = phrase(); + let phrase_matches = independent_compound_fts_oracle(&dataset, &phrase_query).await; + let approximation_matches = + independent_compound_fts_oracle(&dataset, &phrase_approximation).await; + assert!( + approximation_matches.len() > phrase_matches.len(), + "the fixture must include an approximation hit rejected by phrase confirmation" + ); + + let nested_required: FtsQuery = BooleanQuery::new([ + (Occur::Should, compound_match_query("alpha", "title", 2.0)), + (Occur::Should, compound_match_query("gamma", "body", 3.0)), + ]) + .into(); + let staged_required_optional: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("alpha", "title", 1.0)), + (Occur::Should, compound_match_query("optional", "body", 4.0)), + ]) + .into(); + let cases: Vec<(&str, FtsQuery, usize)> = vec![ + ( + "must_sum", + BooleanQuery::new([ + (Occur::Must, compound_match_query("alpha", "title", 2.0)), + (Occur::Must, compound_match_query("gamma", "body", 3.0)), + ]) + .into(), + 2, + ), + ( + "should_sum", + BooleanQuery::new([ + (Occur::Should, compound_match_query("alpha", "title", 2.0)), + (Occur::Should, compound_match_query("gamma", "body", 3.0)), + ]) + .into(), + 3, + ), + ("required_optional", staged_required_optional.clone(), 3), + ( + "must_not", + BooleanQuery::new([ + (Occur::Must, compound_match_query("gamma", "body", 3.0)), + ( + Occur::MustNot, + compound_match_query("blocked", "title", 1_000_000.0), + ), + ]) + .into(), + 3, + ), + ( + "phrase_two_phase", + BooleanQuery::new([ + (Occur::Must, phrase()), + (Occur::Must, compound_match_query("gamma", "body", 1.0)), + ]) + .into(), + 1, + ), + ( + "boost", + BoostQuery::new( + compound_match_query("gamma", "body", 3.0), + compound_match_query("alpha", "title", 2.0), + Some(0.5), + ) + .into(), + 3, + ), + ( + "nested", + BooleanQuery::new([ + (Occur::Must, nested_required), + (Occur::Should, phrase()), + (Occur::MustNot, compound_match_query("blocked", "body", 1.0)), + ]) + .into(), + 3, + ), + ]; + + let mut plans = Vec::with_capacity(cases.len()); + for (case_name, query, limit) in cases { + assert_compound_matches_independent_oracle(&dataset, case_name, &query, limit).await; + plans.push((case_name, compound_fts_plan(&dataset, query, limit).await)); + } + + for (case_name, plan) in plans { + assert!( + plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "{case_name} should use the cross-column scorer:\n{plan}" + ); + assert!( + !plan.contains("HashJoinExec"), + "{case_name} should not materialize an intermediate hash join:\n{plan}" + ); + } + + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query( + staged_required_optional.clone(), + )) + .unwrap(); + scanner.limit(Some(3), None).unwrap(); + let staged_results = compound_fts_result_bits(&scanner.try_into_batch().await.unwrap()); + + dataset + .prewarm_index_with_options( + "title_idx", + &PrewarmOptions::Fts(FtsPrewarmOptions::default()), + ) + .await + .unwrap(); + dataset + .prewarm_index_with_options( + "body_idx", + &PrewarmOptions::Fts(FtsPrewarmOptions::default()), + ) + .await + .unwrap(); + + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(staged_required_optional)) + .unwrap(); + scanner.limit(Some(3), None).unwrap(); + let resident_results = compound_fts_result_bits(&scanner.try_into_batch().await.unwrap()); + assert_eq!( + resident_results, staged_results, + "resident and staged cross-column scans must return identical ordered row ids and score bits" + ); +} + +#[tokio::test] +async fn test_top_level_cross_column_multimatch_uses_field_local_compound_scorers() { + const LIMIT: usize = 2; + + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + + let explicit_query: FtsQuery = MultiMatchQuery::try_new( + "noise".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .into(); + let explicit_oracle = sorted_compound_fts_oracle( + independent_compound_fts_oracle(&dataset, &explicit_query).await, + ); + assert_eq!( + explicit_oracle[1].1, explicit_oracle[2].1, + "the fixture should exercise an equal-score tie at the top-k boundary" + ); + assert!(explicit_oracle[1].0 < explicit_oracle[2].0); + let explicit_results = assert_compound_matches_independent_oracle( + &dataset, + "top_level_cross_column_multimatch", + &explicit_query, + LIMIT, + ) + .await; + let explicit_plan = compound_fts_plan(&dataset, explicit_query.clone(), LIMIT).await; + assert!( + !explicit_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "top-level MultiMatch should keep field scoring independent:\n{explicit_plan}" + ); + assert!( + explicit_plan.matches("CompoundFtsScorer").count() >= 2, + "each indexed field should use its own bounded compound scorer:\n{explicit_plan}" + ); + let inferred_query = FtsQuery::Match(MatchQuery::new("noise".to_owned())); + let inferred_results = + compound_fts_results(&dataset, inferred_query.clone(), Some(LIMIT as i64)).await; + assert_eq!( + inferred_results, explicit_results, + "a fieldless Match expanded across all FTS columns should match an explicit MultiMatch" + ); + let inferred_plan = compound_fts_plan(&dataset, inferred_query, LIMIT).await; + assert!( + !inferred_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "a fieldless Match expanded to MultiMatch should keep field scoring independent:\n{inferred_plan}" + ); + assert!( + inferred_plan.matches("CompoundFtsScorer").count() >= 2, + "each inferred field should use its own bounded compound scorer:\n{inferred_plan}" + ); + + let blocked_query = |boosts: Vec| -> FtsQuery { + MultiMatchQuery::try_new( + "blocked".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .try_with_boosts(boosts) + .unwrap() + .into() + }; + let signed_zero = compound_fts_results(&dataset, blocked_query(vec![-0.0, 0.0]), Some(1)).await; + let normalized = compound_fts_results(&dataset, blocked_query(vec![0.0, 0.0]), None).await; + assert_eq!(signed_zero, normalized[..1]); + assert_eq!(signed_zero[0].0, 4); + assert_eq!(signed_zero[0].1.to_bits(), 0.0_f32.to_bits()); + + let unbounded_results = compound_fts_results(&dataset, explicit_query.clone(), None).await; + assert_scored_rows_close( + "unbounded_top_level_cross_column_multimatch", + &unbounded_results, + &explicit_oracle, + ); + let mut unbounded_scanner = dataset.scan(); + unbounded_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(explicit_query.clone())) + .unwrap(); + let unbounded_plan = unbounded_scanner.explain_plan(false).await.unwrap(); + assert!( + !unbounded_plan.contains("CompoundFtsScorer"), + "an unbounded MultiMatch should retain exhaustive leaf planning:\n{unbounded_plan}" + ); + + let mut partial_dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut partial_dataset, "body", true).await; + let appended = arrow_array::record_batch!( + ("title", Utf8, ["noise"]), + ("body", Utf8, ["noise"]), + ("id", Int32, [10]) + ) + .unwrap(); + let schema = appended.schema(); + partial_dataset + .append( + RecordBatchIterator::new(vec![appended].into_iter().map(Ok), schema), + None, + ) + .await + .unwrap(); + // Index only the title after the append so it can retain a bounded plan + // while the partially covered body uses a query-local hybrid scorer. + create_fragmented_fts_index(&mut partial_dataset, "title", true).await; + partial_dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + let partial_results = + compound_fts_results(&partial_dataset, explicit_query.clone(), Some(LIMIT as i64)).await; + let partial_plan = compound_fts_plan(&partial_dataset, explicit_query.clone(), LIMIT).await; + assert!( + !partial_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "top-level MultiMatch should keep field scoring independent:\n{partial_plan}" + ); + assert_eq!( + partial_plan.matches("CompoundFtsScorer").count(), + 2, + "both fields should retain field-local bounded compound scorers:\n{partial_plan}" + ); + assert_eq!( + partial_plan.matches("HybridCompoundFtsScorer").count(), + 1, + "only the partially covered body should use a query-local hybrid scorer:\n{partial_plan}" + ); + assert!( + !partial_plan.contains("FlatMatchQuery"), + "the hybrid body scorer should replace the indexed-plus-flat fallback:\n{partial_plan}" + ); + + let mut fast_scanner = partial_dataset.scan(); + fast_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(explicit_query.clone())) + .unwrap() + .fast_search(); + fast_scanner.limit(Some(LIMIT as i64), None).unwrap(); + let fast_plan = fast_scanner.explain_plan(false).await.unwrap(); + assert!( + !fast_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "top-level MultiMatch should keep field scoring independent:\n{fast_plan}" + ); + assert_eq!( + fast_plan.matches("CompoundFtsScorer").count(), + 2, + "fast search should use a field-local compound scorer for both fields:\n{fast_plan}" + ); + assert!( + !fast_plan.contains("FlatMatchQuery"), + "fast search must skip the partially covered body's flat path:\n{fast_plan}" + ); + + assert_eq!( + partial_results.len(), + LIMIT, + "the approximate residual path must still return a bounded top-k" + ); + assert!( + partial_results.iter().all(|(_, score)| score.is_finite()), + "committed-index statistics must produce finite residual scores" + ); +} + +#[rstest] +#[tokio::test] +async fn test_multimatch_shared_prefilter(#[values(false, true)] use_scalar_index: bool) { + const FILTER: &str = "id IN (0, 2, 5, 6, 8)"; + const LIMIT: usize = 3; + + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + let query: FtsQuery = MultiMatchQuery::try_new( + "noise".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .into(); + + let mut allowed_scan = dataset.scan(); + allowed_scan.use_scalar_index(false); + allowed_scan.with_row_id().filter(FILTER).unwrap(); + let allowed = allowed_scan.try_into_batch().await.unwrap(); + let allowed_row_ids = allowed[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let mut expected = independent_compound_fts_oracle(&dataset, &query).await; + expected.retain(|row_id, _| allowed_row_ids.contains(row_id)); + let mut expected = sorted_compound_fts_oracle(expected); + expected.truncate(LIMIT); + + let mut scanner = dataset.scan(); + scanner + .prefilter(true) + .use_scalar_index(use_scalar_index) + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .filter(FILTER) + .unwrap() + .limit(Some(LIMIT as i64), None) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert_eq!( + plan.matches("CompoundFtsScorer").count(), + 2, + "both fields should keep their bounded scorer:\n{plan}" + ); + assert_eq!( + plan.matches("ScalarIndexQuery").count(), + usize::from(use_scalar_index) * 2, + "each field should declare the shared prefilter dependency:\n{plan}" + ); + + let batch = scanner.try_into_batch().await.unwrap(); + let actual = batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .zip( + batch[SCORE_COL] + .as_primitive::() + .values() + .iter() + .copied(), + ) + .collect::>(); + assert_scored_rows_close("shared_multimatch_prefilter", &actual, &expected); +} + +#[tokio::test] +async fn test_multimatch_shared_prefilter_preserves_deletes() { + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + dataset.delete("id = 1").await.unwrap(); + let query: FtsQuery = MultiMatchQuery::try_new( + "noise".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .into(); + let mut expected = + sorted_compound_fts_oracle(independent_compound_fts_oracle(&dataset, &query).await); + expected.truncate(3); + + let mut scanner = dataset.scan(); + scanner + .prefilter(true) + .use_scalar_index(false) + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .filter("id >= 0") + .unwrap() + .limit(Some(3), None) + .unwrap(); + let actual = scanner.try_into_batch().await.unwrap(); + let actual = actual[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .zip( + actual[SCORE_COL] + .as_primitive::() + .values() + .iter() + .copied(), + ) + .collect::>(); + assert_scored_rows_close("shared_prefilter_deletes", &actual, &expected); +} + +#[tokio::test] +async fn test_multimatch_shared_prefilter_when_first_field_is_flat_only() { + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut dataset, "title", true).await; + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + let query: FtsQuery = MultiMatchQuery::try_new( + "noise".to_owned(), + vec!["body".to_owned(), "title".to_owned()], + ) + .unwrap() + .into(); + let mut scanner = dataset.scan(); + scanner + .prefilter(true) + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .filter("id >= 0") + .unwrap() + .limit(Some(3), None) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("FlatMatchQuery") && plan.contains("SharedMultiMatchPrefilter"), + "the later indexed field must retain the declared shared source:\n{plan}" + ); + scanner.try_into_batch().await.unwrap(); +} + +#[tokio::test] +async fn test_multimatch_fields_have_independent_fuzzy_expansion_budgets() { + let batch = arrow_array::record_batch!( + ("title", Utf8, ["alpha", "nothing"]), + ("body", Utf8, ["nothing", "alphi"]), + ("id", Int32, [0, 1]) + ) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + None, + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + + let fuzzy_multimatch = || { + let mut query = MultiMatchQuery::try_new( + "alphx".to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap(); + for field in &mut query.match_queries { + field.fuzziness = Some(1); + field.max_expansions = 1; + } + query + }; + + let top_level: FtsQuery = fuzzy_multimatch().into(); + let top_level_plan = compound_fts_plan(&dataset, top_level.clone(), 10).await; + assert!(top_level_plan.matches("CompoundFtsScorer").count() >= 2); + assert!(!top_level_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER)); + let top_level_results = compound_fts_results(&dataset, top_level, Some(10)).await; + assert_eq!(top_level_results.len(), 2); + + let nested: FtsQuery = + BooleanQuery::new([(Occur::Must, FtsQuery::MultiMatch(fuzzy_multimatch()))]).into(); + let nested_plan = compound_fts_plan(&dataset, nested.clone(), 10).await; + assert!(nested_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER)); + let nested_results = compound_fts_results(&dataset, nested, Some(10)).await; + assert_scored_rows_close( + "multimatch_independent_fuzzy_budgets", + &nested_results, + &top_level_results, + ); +} + +#[tokio::test] +async fn test_dataset_planner_defers_auto_fuzziness_for_partial_indices() { + let indexed = arrow_array::record_batch!( + ("title", Utf8, ["alpha"]), + ("body", Utf8, ["alpha"]), + ("id", Int32, [0]) + ) + .unwrap(); + let schema = indexed.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![indexed].into_iter().map(Ok), schema), + "memory://", + None, + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + + let unindexed = arrow_array::record_batch!( + ("title", Utf8, ["alpha"]), + ("body", Utf8, ["alpha"]), + ("id", Int32, [1]) + ) + .unwrap(); + let schema = unindexed.schema(); + dataset + .append( + RecordBatchIterator::new(vec![unindexed].into_iter().map(Ok), schema), + None, + ) + .await + .unwrap(); + + let auto_match = |terms: &str| -> FtsQuery { + MatchQuery::new(terms.to_owned()) + .with_column(Some("title".to_owned())) + .with_fuzziness(None) + .into() + }; + let auto_multimatch = |terms: &str| { + let mut query = MultiMatchQuery::try_new( + terms.to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap(); + for match_query in &mut query.match_queries { + match_query.fuzziness = None; + } + query + }; + + for (case_name, query, expected_ids) in [ + ("match_exact", auto_match("ALPHA"), &[0, 1][..]), + ("match_typo", auto_match("alphx"), &[][..]), + ( + "multimatch_exact", + FtsQuery::MultiMatch(auto_multimatch("ALPHA")), + &[0, 1][..], + ), + ( + "multimatch_typo", + FtsQuery::MultiMatch(auto_multimatch("alphx")), + &[][..], + ), + ( + "nested_multimatch_exact", + BooleanQuery::new([(Occur::Must, FtsQuery::MultiMatch(auto_multimatch("ALPHA")))]) + .into(), + &[0, 1][..], + ), + ( + "nested_multimatch_typo", + BooleanQuery::new([(Occur::Must, FtsQuery::MultiMatch(auto_multimatch("alphx")))]) + .into(), + &[][..], + ), + ] { + let batch = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(query).limit(Some(10))) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + batch["id"].as_primitive::().values(), + expected_ids, + "indexed and unindexed rows diverged for {case_name}" + ); + } +} + +#[tokio::test] +async fn test_field_local_match_wand_exactness_certificates() { + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index_with_order(&mut dataset, "title", true, true).await; + create_fragmented_fts_index_with_order(&mut dataset, "body", true, true).await; + + let field_local_query = |term: &str| -> FtsQuery { + MultiMatchQuery::try_new(term.to_owned(), vec!["title".to_owned(), "body".to_owned()]) + .unwrap() + .into() + }; + + let strict_query = field_local_query("alpha"); + let strict_plan = compound_fts_plan(&dataset, strict_query.clone(), 1).await; + assert!( + strict_plan.matches("CompoundFtsScorer").count() >= 2 + && !strict_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "certificate coverage must execute through field-local compound children:\n{strict_plan}" + ); + let strict_oracle = + sorted_compound_fts_oracle(independent_compound_fts_oracle(&dataset, &strict_query).await); + assert!(strict_oracle[0].1.total_cmp(&strict_oracle[1].1).is_gt()); + let strict = compound_fts_results(&dataset, strict_query, Some(1)).await; + assert_scored_rows_close("wand_certificate_strict", &strict, &strict_oracle[..1]); + + let exhaustive_query = field_local_query("tiebody"); + let exhaustive_oracle = sorted_compound_fts_oracle( + independent_compound_fts_oracle(&dataset, &exhaustive_query).await, + ); + let exhaustive = compound_fts_results(&dataset, exhaustive_query, Some(3)).await; + assert_scored_rows_close( + "wand_certificate_exhaustive", + &exhaustive, + &exhaustive_oracle, + ); + + let tied_query = field_local_query("tie"); + let tied_oracle = + sorted_compound_fts_oracle(independent_compound_fts_oracle(&dataset, &tied_query).await); + assert_eq!(tied_oracle.len(), 2); + assert_eq!(tied_oracle[0].1, tied_oracle[1].1); + assert!(tied_oracle[0].0 < tied_oracle[1].0); + let tied = compound_fts_results(&dataset, tied_query, Some(1)).await; + assert_scored_rows_close("wand_certificate_tie_completion", &tied, &tied_oracle[..1]); + + let mixed_query = field_local_query("noise"); + let mixed_oracle = + sorted_compound_fts_oracle(independent_compound_fts_oracle(&dataset, &mixed_query).await); + let mixed = compound_fts_results(&dataset, mixed_query, Some(1)).await; + assert_scored_rows_close("wand_certificate_mixed_fields", &mixed, &mixed_oracle[..1]); + + for (case_name, exact_term, fuzzy_term, limit) in [ + ("strict", "alpha", "alphx", 1), + ("exhaustive", "tiebody", "tiebodx", 3), + ("ambiguous", "tie", "tix", 1), + ] { + let exact = + compound_fts_results(&dataset, field_local_query(exact_term), Some(limit)).await; + let mut fuzzy = MultiMatchQuery::try_new( + fuzzy_term.to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap(); + for query in &mut fuzzy.match_queries { + query.fuzziness = Some(1); + } + let actual = compound_fts_results(&dataset, fuzzy.into(), Some(limit)).await; + assert_scored_rows_close( + &format!("fuzzy_wand_certificate_{case_name}"), + &actual, + &exact, + ); + } +} + +async fn write_wand_tie_dataset(num_ties: usize) -> Dataset { + let reader = gen_batch() + .col("title", array::fill_utf8("token".to_owned())) + .col("body", array::fill_utf8("unrelated".to_owned())) + .into_reader_rows( + RowCount::from(u64::try_from(num_ties).unwrap()), + BatchCount::from(1), + ); + let dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + max_rows_per_file: num_ties.div_ceil(2), + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + dataset +} + +fn wand_tie_multimatch(terms: &str, boost: f32, fuzziness: Option) -> FtsQuery { + let mut query = MultiMatchQuery::try_new( + terms.to_owned(), + vec!["title".to_owned(), "body".to_owned()], + ) + .unwrap() + .try_with_boosts(vec![boost, 1.0]) + .unwrap(); + for match_query in &mut query.match_queries { + match_query.fuzziness = fuzziness; + } + query.into() +} + +#[tokio::test] +async fn test_wand_tie_completion_recovers_reversed_segment_row_id() { + let mut dataset = write_wand_tie_dataset(6).await; + create_fragmented_fts_index_with_order(&mut dataset, "title", true, true).await; + create_fragmented_fts_index_with_order(&mut dataset, "body", true, true).await; + let query = wand_tie_multimatch("token", 1.0, Some(0)); + let oracle = compound_fts_results(&dataset, query.clone(), None).await; + assert_eq!(oracle.len(), 6); + assert_eq!(oracle[0].0, 0); + + let actual = compound_fts_results(&dataset, query, Some(1)).await; + + assert_scored_rows_close( + "wand_tie_completion_reversed_segments", + &actual, + &oracle[..1], + ); +} + +#[tokio::test] +async fn test_wand_tie_overflow_uses_seeded_fuzzy_boosted_fallback() { + const NUM_TIES: usize = 131; + let mut dataset = write_wand_tie_dataset(NUM_TIES).await; + create_fragmented_fts_index_with_order(&mut dataset, "title", true, true).await; + create_fragmented_fts_index_with_order(&mut dataset, "body", true, true).await; + let query = wand_tie_multimatch("tiken", 2.5, Some(1)); + let oracle = compound_fts_results(&dataset, query.clone(), None).await; + assert_eq!(oracle.len(), NUM_TIES); + assert_eq!(oracle[0].0, 0); + + let actual = compound_fts_results(&dataset, query, Some(1)).await; + + assert_scored_rows_close("wand_tie_seeded_fuzzy_boost", &actual, &oracle[..1]); +} + +#[tokio::test] +async fn test_cross_column_compound_uses_one_scalar_prefilter_mask() { + const FILTER: &str = "id IN (0, 2, 5, 6, 8)"; + const LIMIT: usize = 3; + + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index(&mut dataset, "title", true).await; + create_fragmented_fts_index(&mut dataset, "body", true).await; + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + let query: FtsQuery = BooleanQuery::new([ + (Occur::Should, compound_match_query("alpha", "title", 2.0)), + (Occur::Should, compound_match_query("gamma", "body", 3.0)), + ]) + .into(); + let mut allowed_scan = dataset.scan(); + allowed_scan.use_scalar_index(false); + allowed_scan.with_row_id().filter(FILTER).unwrap(); + let allowed_batch = allowed_scan.try_into_batch().await.unwrap(); + let allowed_row_ids = allowed_batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let mut expected = independent_compound_fts_oracle(&dataset, &query).await; + expected.retain(|row_id, _| allowed_row_ids.contains(row_id)); + let mut expected = sorted_compound_fts_oracle(expected); + assert!(expected.len() > LIMIT); + expected.truncate(LIMIT); + + let mut scanner = dataset.scan(); + scanner + .prefilter(true) + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .filter(FILTER) + .unwrap() + .limit(Some(LIMIT as i64), None) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "filtered cross-column search should use the cross-column scorer:\n{plan}" + ); + assert!( + plan.contains("ScalarIndexQuery") && plan.contains("BTree"), + "the shared prefilter should be built from the BTree scalar index:\n{plan}" + ); + assert_eq!( + plan.matches("ScalarIndexQuery").count(), + 1, + "the cross-column scorer should have one shared scalar prefilter:\n{plan}" + ); + + let actual = scanner.try_into_batch().await.unwrap(); + let actual = actual[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .zip( + actual[SCORE_COL] + .as_primitive::() + .values() + .iter() + .copied(), + ) + .collect::>(); + assert_scored_rows_close("scalar_prefilter", &actual, &expected); +} + +#[tokio::test] +async fn test_cross_column_compound_tie_uses_final_row_id() { + let mut dataset = write_cross_column_compound_dataset().await; + create_fragmented_fts_index_with_order(&mut dataset, "title", true, true).await; + create_fragmented_fts_index_with_order(&mut dataset, "body", true, true).await; + + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("tie", "title", 1.0)), + (Occur::Must, compound_match_query("tiebody", "body", 1.0)), + ]) + .into(); + let expected = + sorted_compound_fts_oracle(independent_compound_fts_oracle(&dataset, &query).await); + assert_eq!(expected.len(), 2); + assert_eq!(expected[0].1, expected[1].1); + assert!(expected[0].0 < expected[1].0); + + let actual = compound_fts_results(&dataset, query.clone(), Some(1)).await; + assert_scored_rows_close("equal_score_row_id_tie", &actual, &expected[..1]); + let plan = compound_fts_plan(&dataset, query, 1).await; + assert!( + plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "equal-score cross-column search should use the cross-column scorer:\n{plan}" + ); +} + +async fn assert_cross_column_layout_uses_fast_path(dataset: &Dataset, case_name: &str) { + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("alpha", "title", 2.0)), + (Occur::Must, compound_match_query("gamma", "body", 3.0)), + ]) + .into(); + assert_compound_matches_independent_oracle(dataset, case_name, &query, 2).await; + let plan = compound_fts_plan(dataset, query, 2).await; + assert!( + plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "{case_name} should align independent segment layouts by row address:\n{plan}" + ); +} + +#[tokio::test] +async fn test_cross_column_compound_handles_independent_segment_layouts() { + let mut reordered = write_cross_column_compound_dataset().await; + let fragment_ids = reordered + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + create_fragmented_fts_index_with_groups( + &mut reordered, + "title", + true, + vec![vec![fragment_ids[0]], vec![fragment_ids[1]]], + ) + .await; + create_fragmented_fts_index_with_groups( + &mut reordered, + "body", + true, + vec![vec![fragment_ids[1]], vec![fragment_ids[0]]], + ) + .await; + assert_cross_column_layout_uses_fast_path(&reordered, "reordered_segments").await; + + let mut differently_split = write_cross_column_compound_dataset().await; + let fragment_ids = differently_split + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + create_fragmented_fts_index_with_groups( + &mut differently_split, + "title", + true, + vec![vec![fragment_ids[0]], vec![fragment_ids[1]]], + ) + .await; + create_fragmented_fts_index_with_groups( + &mut differently_split, + "body", + true, + vec![fragment_ids], + ) + .await; + assert_cross_column_layout_uses_fast_path(&differently_split, "differently_split_segments") + .await; +} + +#[tokio::test] +async fn test_cross_column_compound_incomplete_coverage_uses_exact_fallback() { + let initial = arrow_array::record_batch!( + ("title", Utf8, ["old alpha", "old noise"]), + ("body", Utf8, ["old gamma", "old noise"]) + ) + .unwrap(); + let schema = initial.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![initial].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "body", true).await; + + let appended = arrow_array::record_batch!( + ("title", Utf8, ["fresh alpha"]), + ("body", Utf8, ["fresh gamma"]) + ) + .unwrap(); + let schema = appended.schema(); + dataset + .append( + RecordBatchIterator::new(vec![appended].into_iter().map(Ok), schema), + None, + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "title", true).await; + + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("fresh", "title", 1.0)), + (Occur::Must, compound_match_query("fresh", "body", 1.0)), + ]) + .into(); + let expected = + sorted_compound_fts_oracle(independent_compound_fts_oracle(&dataset, &query).await); + assert_eq!(expected.len(), 1, "the appended row must be the only hit"); + let actual = compound_fts_results(&dataset, query.clone(), Some(1)).await; + assert_scored_rows_close("incomplete_coverage", &actual, &expected); + + let plan = compound_fts_plan(&dataset, query.clone(), 1).await; + assert!( + !plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "incomplete column coverage must not use the cross-column scorer:\n{plan}" + ); + assert!( + plan.contains("BooleanQuery"), + "incomplete column coverage should retain the exact fallback:\n{plan}" + ); + + let mut fast_scanner = dataset.scan(); + fast_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .fast_search(); + fast_scanner.limit(Some(1), None).unwrap(); + let fast_plan = fast_scanner.explain_plan(false).await.unwrap(); + assert!( + !fast_plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "different per-column coverage must retain field-local masking:\n{fast_plan}" + ); + assert!( + fast_plan.contains("BooleanQuery"), + "different per-column coverage should retain the field-local fallback:\n{fast_plan}" + ); + assert_eq!( + fast_scanner.try_into_batch().await.unwrap().num_rows(), + 0, + "the only hit is unindexed in body and must be excluded by fast search" + ); +} + +#[tokio::test] +async fn test_same_column_compound_fast_search_excludes_unindexed_rows() { + let initial = arrow_array::record_batch!( + ("text", Utf8, ["fresh alpha", "old noise"]), + ("id", Int32, [0, 1]) + ) + .unwrap(); + let schema = initial.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![initial].into_iter().map(Ok), schema), + "memory://", + None, + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "text", true).await; + + let appended = + arrow_array::record_batch!(("text", Utf8, ["fresh alpha"]), ("id", Int32, [2])).unwrap(); + let schema = appended.schema(); + dataset + .append( + RecordBatchIterator::new(vec![appended].into_iter().map(Ok), schema), + None, + ) + .await + .unwrap(); + + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("fresh", "text", 1.0)), + (Occur::Must, compound_match_query("alpha", "text", 1.0)), + ]) + .into(); + + let mut hybrid_scanner = dataset.scan(); + hybrid_scanner + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap(); + hybrid_scanner.limit(Some(2), None).unwrap(); + let hybrid_plan = hybrid_scanner.explain_plan(false).await.unwrap(); + assert!( + hybrid_plan.contains("HybridCompoundFtsScorer"), + "partial coverage should build one indexed-statistics query-local residual index:\n{hybrid_plan}" + ); + assert!( + !hybrid_plan.contains("FlatMatchQuery"), + "hybrid compound scoring must not scan the residual once per leaf:\n{hybrid_plan}" + ); + let hybrid = hybrid_scanner.try_into_batch().await.unwrap(); + assert_eq!( + hybrid["id"].as_primitive::().values(), + &[0, 2], + "approximate residual search should include the appended hit" + ); + + let empty_terms_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("", "text", 1.0)), + (Occur::Should, compound_match_query(" ", "text", 1.0)), + ]) + .into(); + let empty_terms_plan = compound_fts_plan(&dataset, empty_terms_query.clone(), 2).await; + assert!( + empty_terms_plan.contains("HybridCompoundFtsScorer"), + "the empty analyzed-term case must exercise the hybrid short circuit:\n{empty_terms_plan}" + ); + let empty_results = compound_fts_results(&dataset, empty_terms_query, Some(2)).await; + assert!(empty_results.is_empty()); + + let mut filtered_scanner = dataset.scan(); + filtered_scanner + .with_row_id() + .filter("id >= 0") + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap(); + filtered_scanner.prefilter(true); + filtered_scanner.limit(Some(2), None).unwrap(); + let filtered_plan = filtered_scanner.explain_plan(false).await.unwrap(); + assert!( + !filtered_plan.contains("HybridCompoundFtsScorer"), + "prefiltered residual scoring must retain the exact fallback:\n{filtered_plan}" + ); + + let phrase_query: FtsQuery = BooleanQuery::new([ + ( + Occur::Must, + PhraseQuery::new("fresh alpha".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + (Occur::Must, compound_match_query("fresh", "text", 1.0)), + ]) + .into(); + let phrase_plan = compound_fts_plan(&dataset, phrase_query, 2).await; + assert!( + !phrase_plan.contains("HybridCompoundFtsScorer"), + "phrase position gaps are not yet supported by the residual index:\n{phrase_plan}" + ); + + let mut fast_scanner = dataset.scan(); + fast_scanner + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap() + .fast_search(); + fast_scanner.limit(Some(2), None).unwrap(); + let fast_plan = fast_scanner.explain_plan(false).await.unwrap(); + assert!( + fast_plan.contains("CompoundFtsScorer"), + "fast search should keep the same-column compound scorer:\n{fast_plan}" + ); + assert!( + !fast_plan.contains("FlatMatchQuery"), + "fast search must not plan a flat scan for unindexed rows:\n{fast_plan}" + ); + let fast = fast_scanner.try_into_batch().await.unwrap(); + assert_eq!( + fast["id"].as_primitive::().values(), + &[0], + "fast search should return only the indexed hit" + ); + + let unindexed_fragment = dataset.get_fragments().last().unwrap().clone(); + let mut unindexed_only_scanner = dataset.scan(); + unindexed_only_scanner + .with_fragments(vec![unindexed_fragment.into()]) + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .fast_search(); + unindexed_only_scanner.limit(Some(2), None).unwrap(); + let unindexed_only_plan = unindexed_only_scanner.explain_plan(false).await.unwrap(); + assert!( + !unindexed_only_plan.contains("CompoundFtsScorer"), + "an entirely unindexed target must not build a compound scorer:\n{unindexed_only_plan}" + ); + assert_eq!( + unindexed_only_scanner + .try_into_batch() + .await + .unwrap() + .num_rows(), + 0 + ); +} + +#[tokio::test] +async fn test_partial_compound_hybrid_prunes_same_path_different_base_rewrite() { + let primary = TempStrDir::default(); + let base_one = TempStrDir::default(); + let base_two = TempStrDir::default(); + let initial = arrow_array::record_batch!( + ("text", Utf8, ["stable alpha", "stale alpha"]), + ("id", Int32, [0, 1]) + ) + .unwrap(); + let schema = initial.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![initial].into_iter().map(Ok), schema.clone()), + &primary, + Some(WriteParams { + max_rows_per_file: 1, + initial_bases: Some(vec![ + BasePath::new(1, base_one.to_string(), None, false), + BasePath::new(2, base_two.to_string(), None, false), + ]), + target_bases: Some(vec![1]), + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + assert!( + dataset + .get_fragments() + .iter() + .all(|fragment| { fragment.metadata().files[0].base_id == Some(1) }) + ); + let segment = dataset + .create_index_builder( + &["text"], + IndexType::Inverted, + &InvertedIndexParams::default().with_position(true), + ) + .name("text_idx".to_string()) + .execute_uncommitted() + .await + .unwrap(); + + let relative_path = dataset.get_fragment(1).unwrap().metadata().files[0] + .path + .clone(); + let replacement = + arrow_array::record_batch!(("text", Utf8, ["current beta"]), ("id", Int32, [1])).unwrap(); + let replacement_path = dataset + .data_file_dir_for_base(Some(2)) + .unwrap() + .join(relative_path.as_str()); + let object_writer = dataset + .object_store(Some(2)) + .await + .unwrap() + .create(&replacement_path) + .await + .unwrap(); + let mut writer = lance_file::versions::v2_1::create_writer( + object_writer, + schema.as_ref().try_into().unwrap(), + Default::default(), + ) + .unwrap(); + writer.write_batch(&replacement).await.unwrap(); + writer.finish().await.unwrap(); + let replacement_file = dataset + .create_data_file(&relative_path, Some(2)) + .await + .unwrap(); + assert_eq!(replacement_file.path, relative_path); + assert_eq!(replacement_file.base_id, Some(2)); + + let read_version = dataset.manifest.version; + let mut dataset = Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(1, replacement_file)], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + dataset + .commit_existing_index_segments("text_idx", "text", vec![segment]) + .await + .unwrap(); + let committed = dataset + .load_index_by_name("text_idx") + .await + .unwrap() + .unwrap(); + let coverage = committed.fragment_bitmap.as_ref().unwrap(); + assert!( + coverage.contains(0), + "the unchanged physical file must remain covered" + ); + assert!( + !coverage.contains(1), + "the same path on a different registered base must be pruned" + ); + + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("beta", "text", 1.0)), + (Occur::MustNot, compound_match_query("alpha", "text", 1.0)), + ]) + .into(); + let mut scanner = dataset.scan(); + scanner + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("HybridCompoundFtsScorer"), + "the physically pruned fragment should use hybrid residual scoring:\n{plan}" + ); + let results = scanner.try_into_batch().await.unwrap(); + assert_eq!( + results["id"].as_primitive::().values(), + &[1], + "the current beta row must be visible without leaking stale alpha membership" + ); + assert!( + results[SCORE_COL] + .as_primitive::() + .values() + .iter() + .all(|score| score.is_finite()) + ); +} + +#[tokio::test] +async fn test_partial_compound_hybrid_uses_mixed_approximate_statistics() { + let initial = arrow_array::record_batch!( + ("text", Utf8, ["fresh alpha", "blocked fresh alpha"]), + ("id", Int32, [0, 1]) + ) + .unwrap(); + let schema = initial.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![initial].into_iter().map(Ok), schema), + "memory://", + None, + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "text", true).await; + + let appended = arrow_array::record_batch!( + ( + "text", + Utf8, + [ + "fresh alpha", + "fresh beta", + "fresh alpha", + "fresh beta beta", + "fresh beta blocked" + ] + ), + ("id", Int32, [2, 3, 4, 5, 6]) + ) + .unwrap(); + let schema = appended.schema(); + dataset + .append( + RecordBatchIterator::new(vec![appended].into_iter().map(Ok), schema), + Some(WriteParams { + // Keep the residual rows in separate fragments; execution may + // rechunk their scan batches before query-local indexing. + max_rows_per_file: 1, + ..Default::default() + }), + ) + .await + .unwrap(); + + let positive: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("fresh", "text", 1.0)), + (Occur::Should, compound_match_query("alpha", "text", 1.0)), + (Occur::MustNot, compound_match_query("blocked", "text", 1.0)), + ]) + .into(); + let boost_query: FtsQuery = BoostQuery::new( + positive, + compound_match_query("alpha", "text", 1.0), + Some(0.25), + ) + .into(); + let partial_boost = compound_fts_results(&dataset, boost_query.clone(), Some(10)).await; + assert_eq!( + partial_boost.len(), + 5, + "MUST_NOT must exclude the blocked row" + ); + let multimatch_query: FtsQuery = MultiMatchQuery::try_new( + "fresh alpha".to_string(), + vec!["text".to_string(), "text".to_string()], + ) + .unwrap() + .try_with_boosts(vec![1.0, 2.0]) + .unwrap() + .into(); + for (query_name, query) in [ + ("Boost", boost_query.clone()), + ("MultiMatch", multimatch_query.clone()), + ] { + let mut scanner = dataset.scan(); + scanner + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + scanner.limit(Some(10), None).unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::().values(); + let scores = batch[SCORE_COL].as_primitive::().values(); + let score_bits = ids + .iter() + .copied() + .zip(scores.iter().map(|score| score.to_bits())) + .collect::>(); + let positions = ids + .iter() + .enumerate() + .map(|(position, row_id)| (*row_id, position)) + .collect::>(); + assert_eq!( + score_bits.get(&2), + score_bits.get(&4), + "{query_name} must preserve equal scores within the residual arm" + ); + assert!( + positions[&2] < positions[&4], + "{query_name} must preserve the row-id tie break within the residual arm" + ); + } + + let mut indexed_only_scanner = dataset.scan(); + indexed_only_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(boost_query.clone())) + .unwrap() + .fast_search(); + indexed_only_scanner.limit(Some(10), None).unwrap(); + let indexed_only = + compound_fts_result_bits(&indexed_only_scanner.try_into_batch().await.unwrap()) + .into_iter() + .collect::>(); + let partial_boost_bits = scored_row_bits(&partial_boost) + .into_iter() + .collect::>(); + for (row_id, score) in indexed_only { + assert_eq!( + partial_boost_bits.get(&row_id), + Some(&score), + "hybrid scoring must preserve committed-index scores for indexed row {row_id}" + ); + } + // The residual arm intentionally uses committed + query-local statistics, + // so its scores are not expected to equal either indexed-arm scores or a + // fully rebuilt index's exact global scores. + + let residual_only_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_match_query("beta", "text", 1.0)), + (Occur::MustNot, compound_match_query("blocked", "text", 1.0)), + ]) + .into(); + let mut residual_only_scanner = dataset.scan(); + residual_only_scanner + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new_query(residual_only_query)) + .unwrap(); + residual_only_scanner.limit(Some(10), None).unwrap(); + let residual_only_plan = residual_only_scanner.explain_plan(false).await.unwrap(); + assert!( + residual_only_plan.contains("HybridCompoundFtsScorer"), + "residual-only term membership must use the indexed-statistics hybrid path:\n{residual_only_plan}" + ); + let residual_only = residual_only_scanner.try_into_batch().await.unwrap(); + assert_eq!( + residual_only["id"].as_primitive::().values(), + &[5, 3], + "residual beta TF must rank id=5 first while MUST_NOT excludes id=6" + ); + let residual_only_scores = residual_only[SCORE_COL] + .as_primitive::() + .values(); + assert!( + residual_only_scores.iter().all(|score| score.is_finite()) + && residual_only_scores[0] > residual_only_scores[1], + "residual-only terms must retain membership and use query-local TF/DF scoring" + ); +} + +#[tokio::test] +async fn test_boolean_must_scores_sum_across_execution_paths() { + let batch = arrow_array::record_batch!( + ( + "title", + Utf8, + [ + "alpha beta delta", + "alpha alpha beta delta delta", + "alpha delta", + "beta delta", + "alpha beta beta delta delta delta" + ] + ), + ( + "body", + Utf8, + ["gamma", "gamma gamma", "gamma", "gamma", "other"] + ) + ) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 3, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + create_fragmented_fts_index(&mut dataset, "title", false).await; + create_fragmented_fts_index(&mut dataset, "body", false).await; + const LIMIT: usize = 2; + + let match_query = |term: &str, column: &str, boost: f32| -> FtsQuery { + MatchQuery::new(term.to_owned()) + .with_column(Some(column.to_owned())) + .with_boost(boost) + .into() + }; + + let same_column_left = match_query("alpha", "title", 2.0); + let same_column_right = match_query("beta", "title", 3.0); + let expected = expected_must_score_sum( + compound_fts_results(&dataset, same_column_left.clone(), None).await, + compound_fts_results(&dataset, same_column_right.clone(), None).await, + ); + assert!(expected.len() > LIMIT); + let same_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, same_column_left.clone()), + (Occur::Must, same_column_right.clone()), + ]) + .into(); + let actual = + compound_fts_results(&dataset, same_column_query.clone(), Some(LIMIT as i64)).await; + assert_eq!(actual, expected[..LIMIT]); + let reversed_same_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, same_column_right), + (Occur::Must, same_column_left), + ]) + .into(); + assert_eq!( + compound_fts_results(&dataset, reversed_same_column_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + + let nested_left = match_query("alpha", "title", 2.0); + let nested_middle = match_query("beta", "title", 3.0); + let nested_right = match_query("delta", "title", 5.0); + let expected = expected_must_score_sum( + expected_must_score_sum( + compound_fts_results(&dataset, nested_left.clone(), None).await, + compound_fts_results(&dataset, nested_middle.clone(), None).await, + ), + compound_fts_results(&dataset, nested_right.clone(), None).await, + ); + assert!(expected.len() > LIMIT); + let nested_pair: FtsQuery = + BooleanQuery::new([(Occur::Must, nested_left), (Occur::Must, nested_middle)]).into(); + let nested_query: FtsQuery = + BooleanQuery::new([(Occur::Must, nested_pair), (Occur::Must, nested_right)]).into(); + assert_eq!( + compound_fts_results(&dataset, nested_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + let reversed_nested_pair: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("beta", "title", 3.0)), + (Occur::Must, match_query("alpha", "title", 2.0)), + ]) + .into(); + let reversed_nested_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("delta", "title", 5.0)), + (Occur::Must, reversed_nested_pair), + ]) + .into(); + assert_eq!( + compound_fts_results(&dataset, reversed_nested_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(same_column_query)) + .unwrap(); + scanner.limit(Some(LIMIT as i64), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "same-column MUST should exercise the composable scorer:\n{plan}" + ); + + let cross_column_left = match_query("alpha", "title", 2.0); + let cross_column_right = match_query("gamma", "body", 3.0); + let expected = expected_must_score_sum( + compound_fts_results(&dataset, cross_column_left.clone(), None).await, + compound_fts_results(&dataset, cross_column_right.clone(), None).await, + ); + assert!(expected.len() > LIMIT); + let cross_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, cross_column_left.clone()), + (Occur::Must, cross_column_right.clone()), + ]) + .into(); + let actual = + compound_fts_results(&dataset, cross_column_query.clone(), Some(LIMIT as i64)).await; + assert_eq!(actual, expected[..LIMIT]); + let reversed_cross_column_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, cross_column_right), + (Occur::Must, cross_column_left), + ]) + .into(); + assert_eq!( + compound_fts_results(&dataset, reversed_cross_column_query, Some(LIMIT as i64)).await, + expected[..LIMIT] + ); + + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(cross_column_query)) + .unwrap(); + scanner.limit(Some(LIMIT as i64), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "cross-column MUST should exercise the cross-column scorer:\n{plan}" + ); + assert!( + !plan.contains("HashJoinExec"), + "cross-column MUST should not materialize an intermediate hash join:\n{plan}" + ); +} + +#[tokio::test] +async fn test_nested_multimatch_limit_propagation() { + let batch = arrow_array::record_batch!( + ( + "title", + Utf8, + [ + "common", + "common filler filler filler filler filler filler filler", + "irrelevant", + "common tie", + "common tie", + "irrelevant" + ] + ), + ( + "body", + Utf8, + [ + "penalty", + "special", + "common", + "neutral", + "neutral", + "common filler filler filler penalty" + ] + ) + ) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 3); + create_fragmented_fts_index(&mut dataset, "title", false).await; + create_fragmented_fts_index(&mut dataset, "body", false).await; + + let must_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, compound_multimatch_query()), + ( + Occur::Should, + compound_match_query("special", "body", 100.0), + ), + ]) + .into(); + let must_results = compound_fts_results(&dataset, must_query.clone(), None).await; + assert!( + must_results + .windows(2) + .any(|rows| rows[0].1 == rows[1].1 && rows[0].0 < rows[1].0), + "the exhaustive result should include a deterministic score tie" + ); + assert_compound_matches_independent_oracle(&dataset, "nested_multimatch_must", &must_query, 2) + .await; + let mut staged_scanner = dataset.scan(); + staged_scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(must_query.clone())) + .unwrap(); + staged_scanner.limit(Some(2), None).unwrap(); + staged_scanner.try_into_batch().await.unwrap(); + + let should_query: FtsQuery = BooleanQuery::new([ + (Occur::Should, compound_multimatch_query()), + ( + Occur::Should, + compound_match_query("special", "body", 100.0), + ), + ]) + .into(); + assert_compound_matches_independent_oracle( + &dataset, + "nested_multimatch_should", + &should_query, + 2, + ) + .await; + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(should_query)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains(CROSS_COLUMN_COMPOUND_FTS_SCORER), + "cross-column compound FTS should use the cross-column scorer:\n{plan}" + ); + assert!( + !plan.contains("HashJoinExec"), + "cross-column compound FTS should not materialize intermediate joins:\n{plan}" + ); + + let boost_query: FtsQuery = BoostQuery::new( + compound_multimatch_query(), + compound_match_query("penalty", "body", 100.0), + Some(1.0), + ) + .into(); + assert_compound_matches_independent_oracle( + &dataset, + "nested_multimatch_boost", + &boost_query, + 2, + ) + .await; + + let multimatch_query = compound_multimatch_query(); + assert_compound_matches_independent_oracle( + &dataset, + "cross_column_multimatch", + &multimatch_query, + 1, + ) + .await; +} + +#[tokio::test] +async fn test_same_column_compound_scorer_is_exact_and_bounded() { + let batch = arrow_array::record_batch!(( + "text", + Utf8, + [ + "common", + "common filler filler filler", + "irrelevant", + "common tie", + "common tie", + "common filler" + ] + )) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + create_fragmented_fts_index(&mut dataset, "text", true).await; + + let match_query = |term: &str| { + MatchQuery::new(term.to_owned()) + .with_column(Some("text".to_owned())) + .into() + }; + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("common")), + ( + Occur::Should, + BoostQuery::new(match_query("tie"), match_query("filler"), Some(0.5)).into(), + ), + ( + Occur::Should, + PhraseQuery::new("common tie".to_owned()) + .with_column(Some("text".to_owned())) + .into(), + ), + (Occur::MustNot, match_query("irrelevant")), + ]) + .into(); + + assert_compound_fts_top_k(&dataset, query.clone(), 2).await; + + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "same-column compound FTS should use the scorer tree:\n{plan}" + ); + assert!( + !plan.contains("HashJoinExec"), + "same-column compound FTS should not materialize intermediate joins:\n{plan}" + ); + + let same_column_multimatch: FtsQuery = MultiMatchQuery::try_new( + "common".to_owned(), + vec!["text".to_owned(), "text".to_owned()], + ) + .unwrap() + .try_with_boosts(vec![1.0, 0.5]) + .unwrap() + .into(); + assert_compound_fts_top_k(&dataset, same_column_multimatch.clone(), 2).await; + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(same_column_multimatch)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "bounded same-column MultiMatch should use posting-backed scorers:\n{plan}" + ); +} + +#[tokio::test] +async fn test_pure_should_maxscore_is_exact_across_fragments() { + let batch = arrow_array::record_batch!(( + "text", + Utf8, + [ + "alpha beta rare blocked", + "alpha beta rare", + "alpha beta", + "alpha gamma", + "beta gamma", + "alpha beta rare", + "alpha beta", + "gamma", + "alpha beta rare", + "alpha beta", + "beta", + "alpha" + ] + )) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 3, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); + create_fragmented_fts_index_with_order(&mut dataset, "text", true, true).await; + + let match_query = |term: &str, boost: f32| { + MatchQuery::new(term.to_owned()) + .with_column(Some("text".to_owned())) + .with_boost(boost) + .into() + }; + let query: FtsQuery = BooleanQuery::new([ + (Occur::Should, match_query("alpha", 0.25)), + (Occur::Should, match_query("beta", 0.25)), + (Occur::Should, match_query("rare", 4.0)), + ( + Occur::Should, + PhraseQuery::new("alpha beta".to_owned()) + .with_column(Some("text".to_owned())) + .into(), + ), + (Occur::MustNot, match_query("blocked", 1.0)), + ]) + .into(); + + let exhaustive = compound_fts_results(&dataset, query.clone(), None).await; + assert!(!exhaustive.iter().any(|(row_id, _)| *row_id == 0)); + assert!(exhaustive.len() >= 3); + assert!( + exhaustive[..3] + .iter() + .all(|(_, score)| *score == exhaustive[0].1) + && exhaustive[..3].windows(2).all(|rows| rows[0].0 < rows[1].0), + "the top three identical rows should tie in ascending row-id order" + ); + let limited = compound_fts_results(&dataset, query.clone(), Some(2)).await; + assert_eq!(limited, exhaustive[..2]); + + let collected_stats = Arc::new(Mutex::new(None::)); + let stats_setter = collected_stats.clone(); + let mut scanner = dataset.scan(); + scanner + .scan_stats_callback(Arc::new(move |stats| { + *stats_setter.lock().unwrap() = Some(stats.clone()); + })) + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + scanner.try_into_batch().await.unwrap(); + let stats = collected_stats.lock().unwrap().take().unwrap(); + assert_eq!( + stats.all_counts.get(PARTITIONS_SEARCHED_METRIC), + Some(&(4 * 5)), + "four index partitions should be searched once for each of five query leaves" + ); + + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap(); + scanner.limit(Some(2), None).unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("CompoundFtsScorer"), + "same-column pure SHOULD should use the compound scorer:\n{plan}" + ); +} + +#[tokio::test] +async fn test_compound_phrase_confirmation_short_circuit_is_exact() { + let texts = (0..100) + .map(|row| { + if row % 10 == 0 { + "high cost phrase check cheap reject bonus" + } else if row % 5 == 0 { + "high cost phrase check cheap reject" + } else { + "high cost phrase check cheap filler reject" + } + }) + .collect::>(); + let batch = arrow_array::record_batch!(("text", Utf8, texts)).unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 25, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); + create_fragmented_fts_index(&mut dataset, "text", true).await; + + let phrase_query = |terms: &str| -> FtsQuery { + PhraseQuery::new(terms.to_owned()) + .with_column(Some("text".to_owned())) + .into() + }; + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, phrase_query("high cost phrase check")), + (Occur::Must, phrase_query("cheap reject")), + ]) + .into(); + assert_compound_fts_top_k(&dataset, query.clone(), 10).await; + + let nested: FtsQuery = BooleanQuery::new([ + (Occur::Must, query.clone()), + (Occur::Should, compound_match_query("bonus", "text", 1.0)), + ]) + .into(); + assert_compound_fts_top_k(&dataset, nested, 10).await; +} + +#[tokio::test] +async fn test_compound_tie_uses_resolved_row_id() { + let batch = arrow_array::record_batch!(("text", Utf8, vec!["common"; 384])).unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 256, + ..Default::default() + }), + ) + .await + .unwrap(); + create_fragmented_fts_index_with_order(&mut dataset, "text", false, true).await; + + let query: FtsQuery = MultiMatchQuery::try_new( + "common".to_owned(), + vec!["text".to_owned(), "text".to_owned()], + ) + .unwrap() + .into(); + let mut scanner = dataset.scan(); + scanner + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap(); + scanner.limit(Some(1), None).unwrap(); + let limited = scanner.try_into_batch().await.unwrap(); + let limited_row_id = limited[ROW_ID].as_primitive::().value(0); + + let exhaustive = compound_fts_results(&dataset, query.clone(), None).await; + assert_eq!(limited_row_id, exhaustive[0].0); + assert_eq!(exhaustive.len(), 384); +} + +fn nested_fts_batch( + ids: Vec, + a_values: Vec>, + b_values: Vec>, +) -> RecordBatch { + let a_values = Arc::new(StringArray::from(a_values)) as ArrayRef; + let b_values = Arc::new(StringArray::from(b_values)) as ArrayRef; + let struct_array = StructArray::from(vec![ + ( + Arc::new(Field::new("a", DataType::Utf8, true)), + a_values.clone(), + ), + ( + Arc::new(Field::new("b", DataType::Utf8, true)), + b_values.clone(), + ), + ]); + let struct_type = struct_array.data_type().clone(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("s", struct_type, true), + ])), + vec![ + Arc::new(UInt64Array::from(ids)) as ArrayRef, + Arc::new(struct_array) as ArrayRef, + ], + ) + .unwrap() +} + +async fn nested_fts_result_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let batch = dataset + .scan() + .full_text_search(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = batch["id"].as_primitive::().values().to_vec(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn test_fts_on_nested_fields() { + let batch = nested_fts_batch( + vec![0, 1, 2, 3], + vec![ + Some("lance nested alpha"), + Some("plain text"), + None, + Some("phrase target here"), + ], + vec![ + Some("metadata only"), + Some("database nested beta"), + Some("lance beta"), + Some("other"), + ], + ); + let schema = batch.schema(); + let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let test_uri = TempStrDir::default(); + let mut dataset = Dataset::write(batches, &test_uri, None).await.unwrap(); + + dataset + .create_index( + &["s.a"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().with_position(true), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["s.b"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + let indexed_fields = indices + .iter() + .map(|index| dataset.schema().field_path(index.fields[0]).unwrap()) + .collect::>(); + assert_eq!( + indexed_fields, + HashSet::from(["s.a".to_string(), "s.b".to_string()]) + ); + + let query = FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("alpha".to_owned()).with_column(Some("s.a".to_owned())), + )); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0]); + + let query = FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("beta".to_owned()).with_column(Some("s.b".to_owned())), + )); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![1, 2]); + + assert_eq!( + nested_fts_result_ids(&dataset, FullTextSearchQuery::new("lance".to_owned())).await, + vec![0, 2] + ); + + let query = FullTextSearchQuery::new_query(FtsQuery::MultiMatch(MultiMatchQuery { + match_queries: vec![ + MatchQuery::new("nested".to_owned()).with_column(Some("s.a".to_owned())), + MatchQuery::new("nested".to_owned()).with_column(Some("s.b".to_owned())), + ], + })); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![0, 1]); + + let query = FullTextSearchQuery::new_query( + PhraseQuery::new("phrase target".to_owned()) + .with_column(Some("s.a".to_owned())) + .into(), + ); + assert_eq!(nested_fts_result_ids(&dataset, query).await, vec![3]); + + let append_batch = nested_fts_batch( + vec![4, 5], + vec![Some("fresh lance append"), Some("plain append")], + vec![Some("other"), Some("fresh beta append")], + ); + let schema = append_batch.schema(); + let batches = RecordBatchIterator::new(vec![append_batch].into_iter().map(Ok), schema); + dataset.append(batches, None).await.unwrap(); + + assert_eq!( + nested_fts_result_ids(&dataset, FullTextSearchQuery::new("fresh".to_owned())).await, + vec![4, 5] + ); } #[tokio::test] @@ -1803,6 +4683,182 @@ async fn test_fts_index_with_large_string() { test_fts_index::(true).await; } +#[tokio::test] +async fn test_fts_list_index_uses_row_level_documents() { + let tempdir = TempStrDir::default(); + let uri = tempdir.to_owned(); + drop(tempdir); + + let mut list_col = GenericListBuilder::::new(GenericStringBuilder::::new()); + list_col.values().append_value("lance"); + list_col.values().append_value("lance database"); + list_col.append(true); + list_col.values().append_value("database"); + list_col.append(true); + list_col.append(true); + list_col.values().append_null(); + list_col.append(true); + list_col.append(false); + + let docs = Arc::new(list_col.finish()) as ArrayRef; + let ids = Arc::new(UInt64Array::from_iter_values(0..docs.len() as u64)) as ArrayRef; + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("doc", docs.data_type().clone(), true), + ArrowField::new("id", DataType::UInt64, false), + ])), + vec![docs, ids], + ) + .unwrap(); + let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(batches, &uri, None).await.unwrap(); + + dataset + .create_index( + &["doc"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new("lance".to_owned()).limit(Some(10))) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result["id"].as_primitive::().values(), &[0]); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new("database".to_owned()).limit(Some(10))) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = result["id"] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1], "{:?}", result); +} + +#[tokio::test] +async fn test_fts_list_phrase_query_can_cross_elements() { + assert_fts_list_phrase_query_can_cross_elements::().await; +} + +#[tokio::test] +async fn test_fts_large_list_phrase_query_can_cross_elements() { + assert_fts_list_phrase_query_can_cross_elements::().await; +} + +async fn assert_fts_list_phrase_query_can_cross_elements() { + let tempdir = TempStrDir::default(); + let uri = tempdir.to_owned(); + drop(tempdir); + + let mut list_col = GenericListBuilder::::new(GenericStringBuilder::::new()); + let rows: &[&[&str]] = &[ + &["alpha", "beta"], + &["want the", "apple"], + &["want", "apple"], + ]; + for values in rows.iter().copied() { + for value in values { + list_col.values().append_value(value); + } + list_col.append(true); + } + + let docs = Arc::new(list_col.finish()) as ArrayRef; + let ids = Arc::new(UInt64Array::from(vec![0u64, 1, 2])) as ArrayRef; + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("doc", docs.data_type().clone(), true), + ArrowField::new("id", DataType::UInt64, false), + ])), + vec![docs, ids], + ) + .unwrap(); + let batches = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(batches, &uri, None).await.unwrap(); + + let cases: [(&str, &[u64]); 3] = [ + ("alpha beta", &[0]), + ("want the apple", &[1]), + ("want apple", &[2]), + ]; + let mut flat_results = Vec::with_capacity(cases.len()); + for (terms, expected) in cases { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search( + FullTextSearchQuery::new_query( + PhraseQuery::new(terms.to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + ) + .limit(Some(10)), + ) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result["id"].as_primitive::().values(), expected); + flat_results.push(result); + } + + let params = InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(true); + dataset + .create_index(&["doc"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + + for ((terms, expected), flat_result) in cases.into_iter().zip(flat_results) { + let indexed_result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search( + FullTextSearchQuery::new_query( + PhraseQuery::new(terms.to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + ) + .limit(Some(10)), + ) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + indexed_result["id"].as_primitive::().values(), + expected + ); + assert_eq!( + indexed_result["id"].as_primitive::().values(), + flat_result["id"].as_primitive::().values(), + "query={terms}" + ); + } +} + #[tokio::test] async fn test_fts_accented_chars() { let ds = create_fts_dataset::(false, false, InvertedIndexParams::default()).await; @@ -1869,11 +4925,11 @@ async fn test_fts_phrase_query() { let words = ["lance", "full", "text", "search"]; let mut lance_search_count = 0; let mut full_text_count = 0; - let mut doc_array = (0..4096) + let mut doc_array = (0..256) .map(|_| { let mut rng = rand::rng(); - let mut text = String::with_capacity(512); - let len = rng.random_range(127..512); + let mut text = String::with_capacity(128); + let len = rng.random_range(31..128); for i in 0..len { if i > 0 { text.push(' '); @@ -2167,122 +5223,288 @@ async fn test_prewarm_index_with_position_validation() { ); } -/// Cache backend that exercises the serialization codec on every insert and -/// returns deserialized entries on every get. Items without a codec fall -/// through to an in-memory passthrough so that non-FTS cache traffic still -/// works during the test. -/// -/// Mirrors the helper in `rust/lance/src/index/vector/ivf/v2.rs` tests; if a -/// third user appears, lift this into a shared test utility. -mod fts_serializing_backend { - use std::collections::HashMap; - use std::pin::Pin; - - use futures::Future; - use lance_core::Result; - use lance_core::cache::{ - CacheBackend, CacheCodec, CacheEntry, InternalCacheKey, MokaCacheBackend, - }; +#[tokio::test] +async fn test_fts_best_effort_prewarm_result_reports_dataset_partial_residency() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + drop(tmpdir); + + let doc_col: Arc = Arc::new(GenericStringArray::::from_iter_values( + (0..4096).map(|row| format!("cache pressure token {row}")), + )); + let ids = UInt64Array::from_iter_values(0..doc_col.len() as u64); + let batch = RecordBatch::try_new( + arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("doc", doc_col.data_type().to_owned(), true), + arrow_schema::Field::new("id", DataType::UInt64, false), + ]) + .into(), + vec![Arc::new(doc_col) as ArrayRef, Arc::new(ids) as ArrayRef], + ) + .unwrap(); + let schema = batch.schema(); + let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let mut dataset = Dataset::write(batches, &uri, None).await.unwrap(); + dataset + .create_index( + &["doc"], + IndexType::Inverted, + Some("fts_idx".to_owned()), + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); - type SerializedEntry = (bytes::Bytes, CacheCodec, usize); + let session = Arc::new(Session::with_index_cache_backend( + Arc::new(QuickCacheBackend::with_capacity(8 * 1024)), + 8 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = DatasetBuilder::from_uri(&uri) + .with_session(session) + .load() + .await + .unwrap(); + let options = PrewarmOptions::Fts(FtsPrewarmOptions::default().best_effort()); - #[derive(Debug)] - pub struct SerializingBackend { - serialized: tokio::sync::Mutex>, - passthrough: MokaCacheBackend, - } + dataset + .prewarm_index_with_options("fts_idx", &options) + .await + .unwrap(); + let result = dataset + .prewarm_index_with_options_result("fts_idx", &options) + .await + .unwrap(); - impl SerializingBackend { - pub fn new() -> Self { - Self { - serialized: tokio::sync::Mutex::new(HashMap::new()), - passthrough: MokaCacheBackend::with_capacity(256 * 1024 * 1024), - } - } + assert!( + !result.fully_resident, + "tiny cache should make best-effort dataset prewarm report partial residency" + ); + let diagnostics = result + .diagnostics + .expect("partial dataset prewarm should return aggregate diagnostics"); + assert!(diagnostics.partition_count > 0); + assert!(!diagnostics.failing_segments.is_empty() || !diagnostics.failing_partitions.is_empty()); + assert!( + diagnostics + .failing_partitions + .iter() + .all(|partition| partition.segment_id.is_some()), + "dataset aggregation should attach segment ids to partition diagnostics" + ); +} + +#[derive(Debug)] +struct SingleScalarContainerCacheBackend { + inner: QuickCacheBackend, + scalar_container_inserts: AtomicUsize, +} - pub async fn serialized_entry_count(&self) -> usize { - self.serialized.lock().await.len() +impl SingleScalarContainerCacheBackend { + fn new(capacity: usize) -> Self { + Self { + inner: QuickCacheBackend::with_capacity(capacity), + scalar_container_inserts: AtomicUsize::new(0), } } - #[async_trait::async_trait] - impl CacheBackend for SerializingBackend { - async fn get( - &self, - key: &InternalCacheKey, - codec: Option, - ) -> Option { - let guard = self.serialized.lock().await; - if let Some((bytes, stored_codec, _)) = guard.get(key) { - return stored_codec.deserialize(&bytes.clone()).hit(); - } - drop(guard); - self.passthrough.get(key, codec).await - } + fn rejects_scalar_container(entry: &CacheEntry, codec: Option<&CacheCodec>) -> bool { + codec.is_none() && entry.as_ref().is::>() + } +} - async fn insert( - &self, - key: &InternalCacheKey, - entry: CacheEntry, - size_bytes: usize, - codec: Option, - ) { - if let Some(codec) = codec { - let mut bytes = Vec::new(); - codec - .serialize(&entry, &mut bytes) - .expect("serialization should succeed"); - self.serialized - .lock() - .await - .insert(key.clone(), (bytes::Bytes::from(bytes), codec, size_bytes)); - } else { - self.passthrough.insert(key, entry, size_bytes, None).await; - } +#[async_trait::async_trait] +impl CacheBackend for SingleScalarContainerCacheBackend { + async fn get(&self, key: &InternalCacheKey, codec: Option) -> Option { + self.inner.get(key, codec).await + } + + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + codec: Option, + ) { + if Self::rejects_scalar_container(&entry, codec.as_ref()) + && self + .scalar_container_inserts + .fetch_add(1, Ordering::Relaxed) + > 0 + { + return; } + self.inner.insert(key, entry, size_bytes, codec).await; + } - async fn get_or_insert<'a>( - &self, - key: &InternalCacheKey, - loader: Pin> + Send + 'a>>, - codec: Option, - ) -> Result<(CacheEntry, bool)> { - if let Some(entry) = self.get(key, codec).await { + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + codec: Option, + ) -> Result<(CacheEntry, bool)> { + if codec.is_none() { + if let Some(entry) = self.inner.get(key, None).await { return Ok((entry, true)); } - let (entry, size) = loader.await?; - self.insert(key, entry.clone(), size, codec).await; - Ok((entry, false)) + let (entry, size_bytes) = loader.await?; + if Self::rejects_scalar_container(&entry, None) + && self + .scalar_container_inserts + .fetch_add(1, Ordering::Relaxed) + > 0 + { + return Ok((entry, false)); + } + self.inner + .insert(key, entry.clone(), size_bytes, codec) + .await; + return Ok((entry, false)); } + self.inner.get_or_insert(key, loader, codec).await + } - async fn invalidate_prefix(&self, prefix: &str) { - self.serialized - .lock() - .await - .retain(|k, _| !k.starts_with(prefix)); - self.passthrough.invalidate_prefix(prefix).await; - } + async fn clear(&self) { + self.inner.clear().await; + } - async fn clear(&self) { - self.serialized.lock().await.clear(); - self.passthrough.clear().await; - } + async fn num_entries(&self) -> usize { + self.inner.num_entries().await + } + + async fn size_bytes(&self) -> usize { + self.inner.size_bytes().await + } + + fn approx_num_entries(&self) -> usize { + self.inner.approx_num_entries() + } + + fn approx_size_bytes(&self) -> usize { + self.inner.approx_size_bytes() + } +} + +async fn two_segment_fts_dataset(uri: &str) -> Dataset { + let schema = Arc::new(arrow_schema::Schema::new(vec![ + arrow_schema::Field::new("doc", DataType::Utf8, true), + arrow_schema::Field::new("id", DataType::UInt64, false), + ])); + let make_batch = |fragment: u64| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(GenericStringArray::::from_iter_values( + (0..32u64).map(|row| format!("segment {fragment} token {row}")), + )) as ArrayRef, + Arc::new(UInt64Array::from_iter_values( + (0..32u64).map(|row| fragment * 32 + row), + )) as ArrayRef, + ], + ) + .unwrap() + }; + + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![make_batch(0)].into_iter().map(Ok), schema.clone()), + uri, + None, + ) + .await + .unwrap(); + dataset + .create_index( + &["doc"], + IndexType::Inverted, + Some("fts_idx".to_owned()), + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![make_batch(1)].into_iter().map(Ok), schema), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + assert_eq!( + dataset.load_indices_by_name("fts_idx").await.unwrap().len(), + 2 + ); + dataset +} + +async fn open_with_single_scalar_container_cache(uri: &str) -> Dataset { + let session = Arc::new(Session::with_index_cache_backend( + Arc::new(SingleScalarContainerCacheBackend::new(128 * 1024 * 1024)), + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + DatasetBuilder::from_uri(uri) + .with_session(session) + .load() + .await + .unwrap() +} + +#[tokio::test] +async fn test_fts_best_effort_prewarm_reports_missing_scalar_container() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + two_segment_fts_dataset(&uri).await; + let dataset = open_with_single_scalar_container_cache(&uri).await; + let options = PrewarmOptions::Fts(FtsPrewarmOptions::default().best_effort()); + + let result = dataset + .prewarm_index_with_options_result("fts_idx", &options) + .await + .unwrap(); + + assert!( + !result.fully_resident, + "dataset prewarm must be partial when a selected segment's scalar index \ + container is not cache-resident" + ); + let diagnostics = result + .diagnostics + .expect("missing scalar container should produce aggregate diagnostics"); + assert_eq!( + diagnostics.failing_segments.len(), + 1, + "only the rejected scalar container should be reported as missing" + ); + assert!( + diagnostics.failing_partitions.is_empty(), + "a missing scalar container should not be represented as a partition failure" + ); + let failure = &diagnostics.failing_segments[0]; + assert!(!failure.scalar_index_container_resident); + assert!(!failure.scalar_index_container_matches_prewarmed); +} - async fn num_entries(&self) -> usize { - self.serialized.lock().await.len() + self.passthrough.num_entries().await - } +#[tokio::test] +async fn test_fts_strict_prewarm_fails_missing_scalar_container() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + two_segment_fts_dataset(&uri).await; + let dataset = open_with_single_scalar_container_cache(&uri).await; + let options = PrewarmOptions::Fts(FtsPrewarmOptions::default()); - async fn size_bytes(&self) -> usize { - let serialized: usize = self - .serialized - .lock() - .await - .values() - .map(|(_, _, s)| *s) - .sum(); - serialized + self.passthrough.size_bytes().await - } - } + let err = dataset + .prewarm_index_with_options_result("fts_idx", &options) + .await + .expect_err("strict prewarm should fail after final scalar-container audit"); + assert!( + err.to_string().contains("resident scalar index container"), + "strict error should describe the missing scalar container: {err}" + ); } /// Validates the OSS-741 contract: after FTS prewarm through a serializing @@ -2295,7 +5517,7 @@ mod fts_serializing_backend { async fn test_fts_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2334,7 +5556,7 @@ async fn test_fts_prewarm_with_serializing_backend_serves_query_with_no_io() { // Re-open the dataset on a session whose cache backend serializes every // entry through its codec. Set a generous capacity so nothing is evicted // before we query. - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2413,7 +5635,7 @@ async fn test_fts_prewarm_with_serializing_backend_serves_query_with_no_io() { async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2449,7 +5671,7 @@ async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { // Re-open on a session whose cache backend serializes every entry through // its codec, with a generous capacity so nothing is evicted before we query. - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2476,9 +5698,32 @@ async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { but the serializing store was empty" ); - // After prewarm, an indexed-filter query must reconstruct the index and - // every page it touches from the cache, deserializing via the codec, with - // no disk IO. Project only `_rowid` so the scan does not read a data column. + drop(dataset); + let backend = Arc::new(backend.restart()); + assert_eq!( + backend.l1_entry_count().await, + 0, + "restarting must discard the in-memory L1" + ); + assert_eq!( + backend.serialized_entry_count().await, + serialized_after_prewarm, + "restarting must retain only the serialized entries" + ); + let session = Arc::new(Session::with_index_cache_backend( + backend, + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = DatasetBuilder::from_uri(&uri) + .with_session(session) + .load() + .await + .unwrap(); + + // After recreating the backend, an indexed-filter query must reconstruct + // the index and every page it touches from serialized bytes, with no disk + // IO. Project only `_rowid` so the scan does not read a data column. dataset.object_store.as_ref().io_stats_incremental(); let result = dataset @@ -2516,7 +5761,7 @@ async fn test_btree_prewarm_with_serializing_backend_serves_query_with_no_io() { async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2550,7 +5795,7 @@ async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() .await .unwrap(); - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2599,6 +5844,95 @@ async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() ); } +#[rstest] +#[case::list(false)] +#[case::large_list(true)] +#[tokio::test] +async fn test_label_list_index_types(#[case] large_list: bool) { + let test_uri = TempStrDir::default(); + let label_values = vec![ + Some(vec![Some(1), Some(2)]), + Some(vec![Some(2)]), + Some(vec![Some(1)]), + Some(vec![Some(3)]), + ]; + let labels: ArrayRef = if large_list { + Arc::new(LargeListArray::from_iter_primitive::( + label_values, + )) + } else { + Arc::new(ListArray::from_iter_primitive::( + label_values, + )) + }; + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("labels", labels.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![0, 1, 2, 3])), labels], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(reader, &test_uri, None).await.unwrap(); + + let expected = dataset + .scan() + .project(&["id"]) + .unwrap() + .filter("array_has_any(labels, [1])") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let expected_ids = expected + .column(0) + .as_primitive::() + .values() + .to_vec(); + assert_eq!(expected_ids, vec![0, 2]); + + dataset + .create_index( + &["labels"], + IndexType::LabelList, + Some("labels_idx".to_owned()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .filter("array_has_any(labels, [1])") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let result_ids = result + .column(0) + .as_primitive::() + .values() + .to_vec(); + assert_eq!(result_ids, expected_ids); + + let plan = dataset + .scan() + .filter("array_has_any(labels, [1])") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery") && plan.contains("LabelList"), + "Expected LabelList scalar index query in plan: {plan}" + ); +} + /// LabelList analogue: after prewarming, an `array_has_any` query against a /// `LabelList` index serves results without any further IO. Exercises the /// `LabelListIndexState` codec (which embeds the inner bitmap state and the @@ -2607,7 +5941,7 @@ async fn test_bitmap_prewarm_with_serializing_backend_serves_query_with_no_io() async fn test_label_list_prewarm_with_serializing_backend_serves_query_with_no_io() { use lance_io::assert_io_eq; - use fts_serializing_backend::SerializingBackend; + use crate::utils::test::serializing_cache::SerializingCacheBackend; let tmpdir = TempStrDir::default(); let uri = tmpdir.to_owned(); @@ -2651,7 +5985,7 @@ async fn test_label_list_prewarm_with_serializing_backend_serves_query_with_no_i "test dataset must contain at least one row whose labels include 3" ); - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -2759,6 +6093,130 @@ async fn test_fts_phrase_query_with_removed_stop_words() { } } +#[tokio::test] +async fn test_fts_without_index_on_zero_fragment_dataset_is_empty() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::UInt64, false), + ArrowField::new("doc", DataType::Utf8, true), + ])); + let empty_reader = RecordBatchIterator::new(vec![], schema); + let dataset = Dataset::write(empty_reader, "memory://", None) + .await + .unwrap(); + assert!(dataset.fragments().is_empty()); + + let mut scan = dataset.scan(); + scan.project(&["id"]).unwrap(); + scan.full_text_search(FullTextSearchQuery::new_query( + MatchQuery::new("alpha".to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + )) + .unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); + assert!(plan.contains("EmptyExec"), "unexpected plan: {plan}"); + assert_eq!(scan.try_into_batch().await.unwrap().num_rows(), 0); + + let mut phrase_scan = dataset.scan(); + phrase_scan.project(&["id"]).unwrap(); + phrase_scan + .full_text_search(FullTextSearchQuery::new_query( + PhraseQuery::new("alpha beta".to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + )) + .unwrap(); + let phrase_plan = phrase_scan.explain_plan(false).await.unwrap(); + assert!( + phrase_plan.contains("EmptyExec"), + "unexpected phrase plan: {phrase_plan}" + ); + assert_eq!(phrase_scan.try_into_batch().await.unwrap().num_rows(), 0); +} + +#[tokio::test] +async fn test_fts_phrase_query_normalizes_leading_stop_word_position() { + let tmpdir = TempStrDir::default(); + let uri = tmpdir.to_owned(); + drop(tmpdir); + + let initial = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(UInt64Array::from(vec![99])) as ArrayRef), + ( + "doc", + Arc::new(StringArray::from(vec!["placeholder"])) as ArrayRef, + ), + ]) + .unwrap(); + let initial_reader = RecordBatchIterator::new(vec![Ok(initial.clone())], initial.schema()); + let mut dataset = Dataset::write(initial_reader, &uri, None).await.unwrap(); + let index_params = InvertedIndexParams::default() + .with_position(true) + .remove_stop_words(true); + dataset + .create_index(&["doc"], IndexType::Inverted, None, &index_params, true) + .await + .unwrap(); + + let appended = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(UInt64Array::from(vec![0, 1, 2])) as ArrayRef), + ( + "doc", + Arc::new(StringArray::from(vec![ + "alpha beta", + "the alpha beta", + "alpha gap beta", + ])) as ArrayRef, + ), + ]) + .unwrap(); + let appended_reader = RecordBatchIterator::new(vec![Ok(appended.clone())], appended.schema()); + dataset = Dataset::write( + appended_reader, + Arc::new(dataset), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let appended_fragment = dataset.fragments().last().unwrap().clone(); + let query = FullTextSearchQuery::new_query( + PhraseQuery::new("the alpha beta".to_owned()) + .with_column(Some("doc".to_owned())) + .into(), + ); + + let mut flat_scan = dataset.scan(); + flat_scan.with_fragments(vec![appended_fragment.clone()]); + flat_scan.project(&["id"]).unwrap(); + flat_scan.full_text_search(query.clone()).unwrap(); + let flat_result = flat_scan.try_into_batch().await.unwrap(); + let mut flat_ids = flat_result["id"] + .as_primitive::() + .values() + .to_vec(); + flat_ids.sort_unstable(); + assert_eq!(flat_ids, vec![0, 1]); + + dataset + .create_index(&["doc"], IndexType::Inverted, None, &index_params, true) + .await + .unwrap(); + let mut indexed_scan = dataset.scan(); + indexed_scan.with_fragments(vec![appended_fragment]); + indexed_scan.project(&["id"]).unwrap(); + indexed_scan.full_text_search(query).unwrap(); + let indexed_result = indexed_scan.try_into_batch().await.unwrap(); + let mut indexed_ids = indexed_result["id"] + .as_primitive::() + .values() + .to_vec(); + indexed_ids.sort_unstable(); + assert_eq!(indexed_ids, flat_ids); +} + #[tokio::test] async fn test_fts_phrase_query_preserves_stop_word_gaps() { let tmpdir = TempStrDir::default(); @@ -2818,6 +6276,140 @@ async fn test_fts_phrase_query_preserves_stop_word_gaps() { assert!(!ids.contains(&3), "ids={ids:?}"); } +fn json_batch(values: Vec<&str>) -> RecordBatch { + let mut metadata = HashMap::new(); + metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("json", DataType::Utf8, false).with_metadata(metadata), + ])); + RecordBatch::try_new(schema, vec![Arc::new(StringArray::from(values))]).unwrap() +} + +async fn json_btree_dataset(initial_values: Vec<&str>) -> Dataset { + let initial = json_batch(initial_values); + let initial_schema = initial.schema(); + let reader = RecordBatchIterator::new([Ok(initial)], initial_schema); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + let params = ScalarIndexParams::new("json".to_string()).with_params(&serde_json::json!({ + "target_index_type": "btree", + "path": "val", + })); + dataset + .create_index( + &["json"], + IndexType::Scalar, + Some("json_idx".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + dataset +} + +#[tokio::test] +async fn test_json_btree_index_statistics() { + let dataset = json_btree_dataset(vec![ + r#"{"val": 1000}"#, + r#"{"val": 2000}"#, + r#"{"val": 3000}"#, + ]) + .await; + + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("json_idx").await.unwrap()).unwrap(); + + assert_eq!(stats["name"], "json_idx"); + assert_eq!(stats["num_indices"], 1); + assert_eq!(stats["num_indexed_rows"], 3); + assert_eq!(stats["num_unindexed_rows"], 0); + assert_eq!(stats["indices"][0]["min"], "1000"); + assert_eq!(stats["indices"][0]["max"], "3000"); +} + +#[rstest] +#[case::merge(false)] +#[case::append_rebuild(true)] +#[tokio::test] +async fn test_optimize_json_btree_index(#[case] append_rebuild: bool) { + let mut dataset = json_btree_dataset(vec![r#"{"val": 1000}"#]).await; + + for values in [ + vec![r#"{"val": null}"#, r#"{"val": 2000}"#], + vec![r#"{"other": 1}"#, r#"{"val": 3000}"#], + ] { + let batch = json_batch(values); + let schema = batch.schema(); + dataset + .append(RecordBatchIterator::new([Ok(batch)], schema), None) + .await + .unwrap(); + } + + let options = if append_rebuild { + OptimizeOptions::append() + } else { + OptimizeOptions::default() + }; + dataset.optimize_indices(&options).await.unwrap(); + + let indexed_fragments = dataset + .load_indices_by_name("json_idx") + .await + .unwrap() + .iter() + .flat_map(|index| index.fragment_bitmap.as_ref().unwrap().iter()) + .collect::>(); + assert_eq!(indexed_fragments, HashSet::from([0, 1, 2])); + + let result = dataset + .scan() + .filter("json_get_int(json, 'val') >= 2000") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 2); +} + +#[tokio::test] +async fn test_optimize_append_json_btree_preserves_float_type() { + let mut dataset = json_btree_dataset(vec![r#"{"val": 1.5}"#]).await; + let appended = json_batch(vec![r#"{"val": 2}"#]); + let schema = appended.schema(); + dataset + .append(RecordBatchIterator::new([Ok(appended)], schema), None) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let predicate = "json_get_float(json, 'val') = 2.0"; + let indexed = dataset + .scan() + .filter(predicate) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut baseline_scan = dataset.scan(); + baseline_scan.use_scalar_index(false); + let baseline = baseline_scan + .filter(predicate) + .unwrap() + .try_into_batch() + .await + .unwrap(); + + assert_eq!(baseline.num_rows(), 1); + assert_eq!(indexed.num_rows(), baseline.num_rows()); +} + async fn prepare_json_dataset() -> (Dataset, String) { let text_col = Arc::new(StringArray::from(vec![ r#"{ @@ -3631,7 +7223,7 @@ async fn test_index_inherits_dataset_file_version() { // Verify that the index file uses the same version as the dataset assert_eq!( index_reader.metadata().version(), - dataset_version, + dataset_version.resolve(), "Index file should use the same format version as the dataset" ); @@ -3660,7 +7252,7 @@ async fn test_index_inherits_dataset_file_version() { assert_eq!( aux_reader.metadata().version(), - dataset_version, + dataset_version.resolve(), "Auxiliary index file should use the same format version as the dataset" ); } @@ -3739,7 +7331,7 @@ async fn test_legacy_dataset_uses_v2_0_for_indexes() { // Verify that the index file uses V2_0 (not legacy) assert_eq!( index_reader.metadata().version(), - LanceFileVersion::V2_0, + ConcreteFileVersion::V2_0, "Index files should never use legacy format, even for legacy datasets" ); } @@ -3808,3 +7400,148 @@ async fn test_manifest_read_recovers_from_stale_size() { assert_eq!(indices.len(), 1); assert_eq!(indices[0].name, "id_idx"); } + +/// `load_segment_params` must match the fully opened segment's params, +/// including `custom_stop_words` — the field `InvertedIndexDetails` loses. +#[tokio::test] +async fn test_load_segment_params_full_fidelity() { + use crate::index::DatasetIndexInternalExt; + use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::inverted::{DocumentGranularity, InvertedIndex}; + + let batch = RecordBatch::try_new( + arrow_schema::Schema::new(vec![Field::new("text", DataType::Utf8, false)]).into(), + vec![Arc::new(StringArray::from(vec![ + "the quick brown fox", + "lazy dogs sleep", + ]))], + ) + .unwrap(); + let schema = batch.schema(); + let stream = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let mut dataset = Dataset::write(stream, "memory://test/segment_params", None) + .await + .unwrap(); + + let params = InvertedIndexParams::default().custom_stop_words(Some(vec!["quick".to_string()])); + dataset + .create_index(&["text"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + + let segments = crate::index::scalar::load_segments(&dataset, "text", DocumentGranularity::Row) + .await + .unwrap() + .expect("FTS index segments"); + let read = crate::index::scalar::load_segment_params(&dataset, &segments[0]) + .await + .unwrap(); + + let generic = dataset + .open_generic_index("text", &segments[0].uuid, &NoOpMetricsCollector) + .await + .unwrap(); + let opened = generic + .as_any() + .downcast_ref::() + .expect("inverted index"); + assert_eq!(&read, opened.params()); +} + +/// Compaction of a covered index must succeed. `remap_index` rejected any +/// index with more than one field, so this failed outright -- a covered +/// dataset could be created but never compacted. +#[tokio::test] +async fn test_compaction_withdraws_a_covered_index_without_failing() { + use crate::dataset::optimize::{CompactionOptions, compact_files}; + + let test_uri = TempStrDir::default(); + let dimension = 16; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + dimension, + ), + false, + ), + ArrowField::new("payload", DataType::Int32, false), + ])); + + let make_batch = |offset: i32| { + let vectors = Arc::new( + ::try_new_from_values( + generate_random_array(256 * dimension as usize), + dimension, + ) + .unwrap(), + ); + let payload = Arc::new(Int32Array::from_iter_values(offset..offset + 256)); + RecordBatch::try_new(schema.clone(), vec![vectors, payload]).unwrap() + }; + + // Two fragments, so compaction has something to compact. + let reader = RecordBatchIterator::new(vec![Ok(make_batch(0))], schema.clone()); + let mut dataset = Dataset::write(reader, &test_uri, None).await.unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(make_batch(256))], schema.clone()); + dataset.append(reader, None).await.unwrap(); + + let params = VectorIndexParams::ivf_pq(4, 8, 2, MetricType::L2, 50); + dataset + .create_index(&["vec"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + + let vec_id = dataset.schema().field_id("vec").unwrap(); + let payload_id = dataset.schema().field_id("payload").unwrap(); + let current = dataset.load_indices().await.unwrap(); + let mut covered = current[0].clone(); + covered.fields = vec![vec_id, payload_id]; + covered.covering_fields = vec![payload_id]; + + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![covered], + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let fragments_before: Vec = dataset.fragments().iter().map(|f| f.id).collect(); + assert!( + fragments_before.len() > 1, + "precondition: there must be something to compact" + ); + + // Compaction of the table must not be blocked by an index it cannot remap. + compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .expect("compaction of a covered index must succeed"); + + let fragments_after: Vec = dataset.fragments().iter().map(|f| f.id).collect(); + assert_ne!( + fragments_after, fragments_before, + "compaction rewrote nothing, so the remap path never ran" + ); + + // The entry survives untouched -- withdrawal skips remapping rather than + // deleting metadata -- but it now covers none of the rewritten fragments, so + // no query can be answered from a payload the storage never held. + let after = dataset.load_indices().await.unwrap(); + assert_eq!(after.len(), 1); + assert_eq!(after[0].covering_fields, vec![payload_id]); + let live: roaring::RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect(); + let effective = after[0].effective_fragment_bitmap(&live); + assert!( + effective.is_none_or(|bitmap| bitmap.is_empty()), + "a withdrawn covered index must stop covering fragments, got {:?}", + after[0].fragment_bitmap + ); +} diff --git a/rust/lance/src/dataset/tests/dataset_io.rs b/rust/lance/src/dataset/tests/dataset_io.rs index 1f8c7226bf2..3904d48c820 100644 --- a/rust/lance/src/dataset/tests/dataset_io.rs +++ b/rust/lance/src/dataset/tests/dataset_io.rs @@ -2,7 +2,10 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; use std::vec; use super::dataset_common::{create_file, require_send}; @@ -10,8 +13,15 @@ use super::dataset_common::{create_file, require_send}; use crate::dataset::WriteDestination; use crate::dataset::WriteMode::Overwrite; use crate::dataset::builder::DatasetBuilder; -use crate::dataset::{ManifestWriteConfig, write_manifest_file}; +use crate::dataset::mem_wal::DatasetMemWalExt; +use crate::dataset::schema_evolution::ColumnAlteration; +use crate::dataset::transaction::Operation; +use crate::dataset::{ + ManifestWriteConfig, deep_clone_copy_parallelism, parse_deep_clone_stream_concurrency, + validate_dataset_root_for_drop, write_manifest_file, +}; use crate::session::Session; +use crate::session::caches::ManifestKey; use crate::{Dataset, Error, Result}; use lance_table::format::DataStorageFormat; @@ -34,13 +44,14 @@ use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY}; use lance_core::utils::tempfile::{TempStdDir, TempStrDir}; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; use lance_file::{ - version::LanceFileVersion, - writer::{FileWriter, FileWriterOptions}, + version::{ConcreteFileVersion, LanceFileVersion}, + writer::FileWriterOptions, }; use lance_io::assert_io_eq; use lance_table::feature_flags; -use lance_table::format::BasePath; +use lance_table::format::{BasePath, Fragment, pb}; use object_store::ObjectStoreExt; +use prost::Message; use crate::index::DatasetIndexExt; use futures::TryStreamExt; @@ -60,6 +71,40 @@ fn file_object_store_uri(path: &std::path::Path) -> String { format!("file-object-store://{path_prefix}{path}") } +#[rstest] +#[case::empty("")] +#[case::zero("0")] +#[case::negative("-1")] +#[case::not_a_number("many")] +fn test_parse_deep_clone_stream_concurrency_rejects_invalid_values(#[case] value: &str) { + let error = parse_deep_clone_stream_concurrency(value).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("LANCE_DEEP_CLONE_STREAM_CONCURRENCY")); + assert!(message.contains(&format!("{value:?}"))); +} + +#[test] +fn test_parse_deep_clone_stream_concurrency_accepts_positive_value() { + assert_eq!(parse_deep_clone_stream_concurrency("17").unwrap(), 17); +} + +#[rstest] +#[case::direct_local_copy(64, false, None, 64)] +#[case::streaming_default_cap(64, true, None, 4)] +#[case::streaming_configured_below_cap(2, true, None, 2)] +#[case::streaming_override(64, true, Some(17), 17)] +fn test_deep_clone_copy_parallelism( + #[case] configured: usize, + #[case] uses_streaming_copy: bool, + #[case] stream_override: Option, + #[case] expected: usize, +) { + assert_eq!( + deep_clone_copy_parallelism(configured, uses_streaming_copy, stream_override), + expected + ); +} + #[tokio::test] async fn test_truncate_table() { let tmpdir = tempfile::tempdir().unwrap(); @@ -448,13 +493,10 @@ async fn test_create_data_file_rejects_nested_schema_mismatch() { .create(&dataset.data_dir().join(file_name)) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_2::create_writer( object_writer, crate::datatypes::Schema::try_from(schema.as_ref()).unwrap(), - FileWriterOptions { - format_version: Some(LanceFileVersion::V2_2), - ..Default::default() - }, + FileWriterOptions::default(), ) .unwrap(); writer.write_batch(&batch).await.unwrap(); @@ -498,24 +540,86 @@ async fn test_create_data_file_rejects_nested_schema_mismatch() { ); } -#[tokio::test] -async fn test_shallow_clone_base_artifacts_use_base_object_store() { - let source_dir = tempfile::tempdir().unwrap(); - let clone_dir = tempfile::tempdir().unwrap(); - let source_uri = file_object_store_uri(source_dir.path()); - let clone_uri = file_object_store_uri(clone_dir.path()); - +async fn write_multi_fragment_source(uri: &str) -> Dataset { let batch = gen_batch() .col("id", array::step::()) .into_batch_rows(RowCount::from(64)) .unwrap(); - let mut source = Dataset::write( + Dataset::write( RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), - &source_uri, - None, + uri, + Some(WriteParams { + max_rows_per_file: 8, + ..Default::default() + }), ) .await - .unwrap(); + .unwrap() +} + +async fn tag_and_shallow_clone(source: &mut Dataset, clone_uri: &str) -> Dataset { + source + .tags() + .create("to_clone", source.version().version) + .await + .unwrap(); + source + .shallow_clone(clone_uri, "to_clone", None) + .await + .unwrap() +} + +fn registry_attempts(dataset: &Dataset) -> u64 { + let stats = dataset.session.store_registry().stats(); + stats.hits + stats.misses +} + +#[derive(Debug, Default)] +struct CountingObjectStoreWrapper { + wraps: AtomicUsize, +} + +impl WrappingObjectStore for CountingObjectStoreWrapper { + fn wrap( + &self, + _store_prefix: &str, + original: Arc, + ) -> Arc { + self.wraps.fetch_add(1, Ordering::Relaxed); + original + } + + // Passes requests straight through, so a listing may keep going around it. Only `wrap` is + // counted, since the count is what the caching assertions are written against. + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } +} + +impl CountingObjectStoreWrapper { + fn wraps(&self) -> usize { + self.wraps.load(Ordering::Relaxed) + } +} + +fn first_base_id(dataset: &Dataset) -> u32 { + dataset.get_fragments()[0].metadata().files[0] + .base_id + .expect("shallow clone data files must reference the source base") +} + +#[tokio::test] +async fn test_shallow_clone_base_artifacts_use_base_object_store() { + let source_dir = tempfile::tempdir().unwrap(); + let clone_dir = tempfile::tempdir().unwrap(); + let source_uri = file_object_store_uri(source_dir.path()); + let clone_uri = file_object_store_uri(clone_dir.path()); + + let mut source = write_multi_fragment_source(&source_uri).await; source .create_index( &["id"], @@ -527,16 +631,7 @@ async fn test_shallow_clone_base_artifacts_use_base_object_store() { .await .unwrap(); source.delete("id < 4").await.unwrap(); - source - .tags() - .create("with_artifacts", source.version().version) - .await - .unwrap(); - - let cloned = source - .shallow_clone(&clone_uri, "with_artifacts", None) - .await - .unwrap(); + let cloned = tag_and_shallow_clone(&mut source, &clone_uri).await; let base = cloned .manifest() .base_paths @@ -580,6 +675,171 @@ async fn test_shallow_clone_base_artifacts_use_base_object_store() { assert!(tracker.incremental_stats().read_iops > 0); } +#[tokio::test] +async fn test_shallow_clone_reuses_base_object_store() { + let source_dir = tempfile::tempdir().unwrap(); + let clone_dir = tempfile::tempdir().unwrap(); + let source_uri = file_object_store_uri(source_dir.path()); + let clone_uri = file_object_store_uri(clone_dir.path()); + + let mut source = write_multi_fragment_source(&source_uri).await; + let cloned = tag_and_shallow_clone(&mut source, &clone_uri).await; + let base_id = first_base_id(&cloned); + + let first = cloned.object_store(Some(base_id)).await.unwrap(); + let second = cloned.object_store(Some(base_id)).await.unwrap(); + assert!( + Arc::ptr_eq(&first, &second), + "repeated lookups must reuse the cached base object store" + ); + + let third = cloned.clone().object_store(Some(base_id)).await.unwrap(); + assert!( + Arc::ptr_eq(&first, &third), + "dataset clones must share the base object store cache" + ); + + let tracker = Arc::new(IOTracker::default()); + let wrapped = + cloned.with_object_store_wrappers(vec![tracker.clone() as Arc]); + let wrapped_store = wrapped.object_store(Some(base_id)).await.unwrap(); + assert!( + !Arc::ptr_eq(&first, &wrapped_store), + "the wrapped clone must not serve the undecorated store" + ); + let _ = tracker.incremental_stats(); + wrapped.scan().try_into_batch().await.unwrap(); + assert!( + tracker.incremental_stats().read_iops > 0, + "reads on the wrapped clone must go through the wrapper" + ); + + let fresh = DatasetBuilder::from_uri(&clone_uri) + .with_session(Arc::new(Session::default())) + .load() + .await + .unwrap(); + let attempts_before = registry_attempts(&fresh); + let stores = futures::future::try_join_all((0..8).map(|_| fresh.object_store(Some(base_id)))) + .await + .unwrap(); + assert!( + stores.iter().all(|store| Arc::ptr_eq(store, &stores[0])), + "concurrent resolutions must share one store" + ); + assert_eq!( + registry_attempts(&fresh) - attempts_before, + 1, + "concurrent resolutions must resolve the store exactly once" + ); + + // A caller can derive wrapper-scoped datasets by applying the same shared + // wrapper to a cached dataset. Each derived dataset must retain one base + // store for its own lifetime without sharing it with another scope. + let shared_wrapper = Arc::new(CountingObjectStoreWrapper::default()); + let scope_a = + fresh.with_object_store_wrappers([shared_wrapper.clone() as Arc]); + let scope_b = + fresh.with_object_store_wrappers([shared_wrapper.clone() as Arc]); + let wraps_before = shared_wrapper.wraps(); + let attempts_before = registry_attempts(&fresh); + + let scope_a_first = scope_a.object_store(Some(base_id)).await.unwrap(); + let scope_a_second = scope_a.object_store(Some(base_id)).await.unwrap(); + assert!( + Arc::ptr_eq(&scope_a_first, &scope_a_second), + "one wrapper scope must reuse its resolved base store" + ); + + let scope_b_first = scope_b.object_store(Some(base_id)).await.unwrap(); + let scope_b_second = scope_b.object_store(Some(base_id)).await.unwrap(); + assert!( + Arc::ptr_eq(&scope_b_first, &scope_b_second), + "one wrapper scope must reuse its resolved base store" + ); + assert!( + !Arc::ptr_eq(&scope_a_first, &scope_b_first), + "separate wrapper scopes must not share a stateful base store" + ); + assert!( + !Arc::ptr_eq(&scope_a_first.inner, &scope_b_first.inner), + "separate wrapper scopes must not share provider-local layers" + ); + assert_eq!( + registry_attempts(&fresh) - attempts_before, + 0, + "wrapper-scoped base stores must bypass the global registry cache" + ); + assert_eq!( + shared_wrapper.wraps() - wraps_before, + 2, + "each wrapper scope must build its base store exactly once" + ); + + let read = cloned.scan().try_into_batch().await.unwrap(); + assert_eq!(read.num_rows(), 64); +} + +#[tokio::test] +async fn test_base_object_store_cache_invalidation() { + let source_dir = tempfile::tempdir().unwrap(); + let clone_dir = tempfile::tempdir().unwrap(); + let extra_dir = tempfile::tempdir().unwrap(); + let source_uri = file_object_store_uri(source_dir.path()); + let clone_uri = file_object_store_uri(clone_dir.path()); + + let mut source = write_multi_fragment_source(&source_uri).await; + let mut cloned = tag_and_shallow_clone(&mut source, &clone_uri).await; + let base_id = first_base_id(&cloned); + let store = cloned.object_store(Some(base_id)).await.unwrap(); + + let rebound = cloned.with_object_store( + cloned.object_store.clone(), + Some(ObjectStoreParams { + block_size: Some(32 * 1024), + ..Default::default() + }), + ); + let rebound_store = rebound.object_store(Some(base_id)).await.unwrap(); + assert!( + !Arc::ptr_eq(&store, &rebound_store), + "changed store params must not serve the previously cached base store" + ); + + cloned.delete("id = 0").await.unwrap(); + let attempts_before = registry_attempts(&cloned); + let retained = cloned.object_store(Some(base_id)).await.unwrap(); + assert!( + Arc::ptr_eq(&store, &retained), + "commits that keep base_paths must keep the cache" + ); + assert_eq!( + registry_attempts(&cloned) - attempts_before, + 0, + "commits that keep base_paths must not re-resolve the store" + ); + + let with_extra = Arc::new(cloned) + .add_bases( + vec![lance_table::format::BasePath::new( + 0, + file_object_store_uri(extra_dir.path()), + Some("extra".to_string()), + true, + )], + None, + ) + .await + .unwrap(); + let attempts_before = registry_attempts(&with_extra); + with_extra.object_store(Some(base_id)).await.unwrap(); + assert_eq!( + registry_attempts(&with_extra) - attempts_before, + 1, + "base_paths changes must reset the cache and re-resolve the store" + ); +} + #[cfg(feature = "azure")] #[tokio::test] async fn test_object_store_uses_runtime_base_store_params() { @@ -871,6 +1131,76 @@ async fn test_load_manifest_iops() { assert_io_eq!(io_stats, read_iops, 1); } +#[tokio::test] +async fn test_checkout_removed_version_not_served_from_cache() { + let test_uri = TempStrDir::default(); + let session = Arc::new(Session::default()); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10_i32))], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + &test_uri, + Some(WriteParams { + session: Some(session.clone()), + ..Default::default() + }), + ) + .await + .unwrap(); + + let version = dataset.manifest().version; + let location = dataset.manifest_location().clone(); + let cache = session.metadata_cache.for_dataset(&dataset.uri); + + assert!( + cache + .get_with_key(&ManifestKey { + version, + e_tag: location.e_tag.as_deref(), + }) + .await + .is_some(), + "manifest should be cached after the write" + ); + dataset.checkout_version(version).await.unwrap(); + + // Remove the version from storage, as cleanup (or a manual delete) would. + dataset.object_store.delete(&location.path).await.unwrap(); + + let resolved = dataset + .commit_handler + .resolve_version_location(&dataset.base, version, &dataset.object_store.inner) + .await + .unwrap(); + assert!( + resolved.size.is_none(), + "resolving a removed version must fall back to a size-less location, got {:?}", + resolved.size + ); + + cache + .insert_with_key( + &ManifestKey { + version, + e_tag: None, + }, + Arc::new(dataset.manifest().clone()), + ) + .await; + assert!( + dataset.checkout_version(version).await.is_err(), + "checkout of a version removed from storage must not be served from cache" + ); +} + #[rstest] #[tokio::test] async fn test_write_params( @@ -914,15 +1244,18 @@ async fn test_write_params( assert_eq!(dataset.count_fragments(), 10); for fragment in &fragments { assert_eq!(fragment.count_rows(None).await.unwrap(), 100); - let reader = fragment - .open(dataset.schema(), FragReadConfig::default()) - .await - .unwrap(); // No group / batch concept in v2 if data_storage_version == LanceFileVersion::Legacy { - assert_eq!(reader.legacy_num_batches(), 10); - for i in 0..reader.legacy_num_batches() as u32 { - assert_eq!(reader.legacy_num_rows_in_batch(i).unwrap(), 10); + let reader = crate::dataset::versions::open_v1_fragment_reader( + fragment, + dataset.schema(), + &FragReadConfig::default(), + ) + .await + .unwrap(); + assert_eq!(reader.num_batches(), 10); + for i in 0..reader.num_batches() as u32 { + assert_eq!(reader.num_rows_in_batch(i).unwrap(), 10); } } } @@ -931,7 +1264,11 @@ async fn test_write_params( #[rstest] #[tokio::test] async fn test_write_manifest( - #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + #[values( + LanceFileVersion::Legacy, + LanceFileVersion::Stable, + LanceFileVersion::Next + )] data_storage_version: LanceFileVersion, ) { use lance_table::feature_flags::FLAG_UNKNOWN; @@ -980,8 +1317,12 @@ async fn test_write_manifest( assert_eq!( manifest.data_storage_format, - DataStorageFormat::new(data_storage_version) + DataStorageFormat::new(data_storage_version.resolve()) ); + assert!(!matches!( + manifest.data_storage_format.version.to_manifest_string(), + "stable" | "next" + )); assert_eq!(manifest.reader_feature_flags, 0); // Create one with deletions @@ -1027,9 +1368,12 @@ async fn test_write_manifest( use_legacy_format: None, storage_format: None, disable_transaction_file: false, + migration_next_row_id: None, }, dataset.manifest_location.naming_scheme, None, + // Previously classified from a None inline copy, which validated. + true, ) .await .unwrap(); @@ -1062,73 +1406,201 @@ async fn test_write_manifest( } #[tokio::test] -async fn test_rle_v2_v23_write_and_append() { +async fn test_restore_rejects_unknown_target_flags() { let test_uri = TempStrDir::default(); - let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( - "i", - DataType::Int32, - false, - )])); - let batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(Int32Array::from(vec![7; 1000]))], - ) - .unwrap(); + let data = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let dataset = Dataset::write(data, &test_uri, None).await.unwrap(); - let batches = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone()); - let mut dataset = Dataset::write( - batches, - &test_uri, - Some(WriteParams { - data_storage_version: Some(LanceFileVersion::V2_3), - ..Default::default() - }), + let write_config = ManifestWriteConfig { + auto_set_feature_flags: false, + ..Default::default() + }; + let mut unknown_manifest = dataset.manifest.as_ref().clone(); + unknown_manifest.version = 2; + unknown_manifest.reader_feature_flags |= feature_flags::FLAG_UNKNOWN; + unknown_manifest.writer_feature_flags |= feature_flags::FLAG_UNKNOWN; + write_manifest_file( + dataset.object_store.as_ref(), + dataset.commit_handler.as_ref(), + &dataset.base, + &mut unknown_manifest, + None, + &write_config, + dataset.manifest_location.naming_scheme, + None, + // No inline transaction to classify from, so validate. + true, ) .await .unwrap(); - let manifest = read_manifest( + let mut supported_manifest = dataset.manifest.as_ref().clone(); + supported_manifest.version = 3; + write_manifest_file( dataset.object_store.as_ref(), - &dataset - .commit_handler - .resolve_latest_location(&dataset.base, dataset.object_store.as_ref()) - .await - .unwrap() - .path, + dataset.commit_handler.as_ref(), + &dataset.base, + &mut supported_manifest, None, + &write_config, + dataset.manifest_location.naming_scheme, + None, + // No inline transaction to classify from, so validate. + true, ) .await .unwrap(); - assert_eq!( - manifest.data_storage_format.lance_file_version().unwrap(), - LanceFileVersion::V2_3 - ); - let append_batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(Int32Array::from(vec![9; 1000]))], - ) - .unwrap(); - let append_batches = - RecordBatchIterator::new(vec![Ok(append_batch)].into_iter(), schema.clone()); - dataset = Dataset::write( - append_batches, + let error = Dataset::commit( &test_uri, - Some(WriteParams { - mode: WriteMode::Append, + Operation::Restore { version: 2 }, + Some(3), + None, + None, + Default::default(), + false, + ) + .await + .unwrap_err(); + + assert!(matches!(error, Error::NotSupported { .. }), "{error}"); +} + +#[tokio::test] +async fn test_checkout_latest_rejects_unsupported_reader_before_caching() { + let test_uri = TempStrDir::default(); + let data = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let mut dataset = Dataset::write(data, &test_uri, None).await.unwrap(); + let original_version = dataset.version().version; + + let mut unsupported_manifest = dataset.manifest.as_ref().clone(); + unsupported_manifest.version += 1; + unsupported_manifest.reader_feature_flags |= feature_flags::FLAG_UNKNOWN; + unsupported_manifest.writer_feature_flags |= feature_flags::FLAG_UNKNOWN; + let location = write_manifest_file( + dataset.object_store.as_ref(), + dataset.commit_handler.as_ref(), + &dataset.base, + &mut unsupported_manifest, + None, + &ManifestWriteConfig { + auto_set_feature_flags: false, + ..Default::default() + }, + dataset.manifest_location.naming_scheme, + None, + // No inline transaction to classify from, so validate. + true, + ) + .await + .unwrap(); + + let error = dataset.checkout_latest().await.unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. }), "{error}"); + assert_eq!(dataset.version().version, original_version); + assert!( + dataset + .metadata_cache + .get_with_key(&ManifestKey { + version: location.version, + e_tag: location.e_tag.as_deref(), + }) + .await + .is_none(), + "unsupported manifest must not be cached" + ); +} + +#[tokio::test] +async fn test_serialized_manifest_rejects_unsupported_reader() { + let test_uri = TempStrDir::default(); + let data = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let dataset = Dataset::write(data, &test_uri, None).await.unwrap(); + + let mut unsupported_manifest = dataset.manifest.as_ref().clone(); + unsupported_manifest.reader_feature_flags |= feature_flags::FLAG_UNKNOWN; + unsupported_manifest.writer_feature_flags |= feature_flags::FLAG_UNKNOWN; + let serialized_manifest = pb::Manifest::from(&unsupported_manifest).encode_to_vec(); + + let error = DatasetBuilder::from_uri(&test_uri) + .with_serialized_manifest(&serialized_manifest) + .unwrap() + .load() + .await + .unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. }), "{error}"); +} + +#[tokio::test] +async fn test_rle_v2_v23_write_and_append() { + let test_uri = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![7; 1000]))], + ) + .unwrap(); + + let batches = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema.clone()); + let mut dataset = Dataset::write( + batches, + &test_uri, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_3), ..Default::default() }), ) .await .unwrap(); + let manifest = read_manifest( + dataset.object_store.as_ref(), + &dataset + .commit_handler + .resolve_latest_location(&dataset.base, dataset.object_store.as_ref()) + .await + .unwrap() + .path, + None, + ) + .await + .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::V2_3 + manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 + ); + + let append_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![9; 1000]))], + ) + .unwrap(); + let append_batches = + RecordBatchIterator::new(vec![Ok(append_batch)].into_iter(), schema.clone()); + dataset = Dataset::write( + append_batches, + &test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + assert_eq!( + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); let actual = dataset.scan().try_into_batch().await.unwrap(); @@ -1170,12 +1642,8 @@ async fn test_rle_v2_uncommitted_create_commits_v23_storage() { .await .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::V2_3 + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); } @@ -1210,12 +1678,8 @@ async fn test_rle_v2_shallow_clone_preserves_v23_storage() { .await .unwrap(); assert_eq!( - clone - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::V2_3 + clone.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V2_3 ); } @@ -1453,6 +1917,227 @@ async fn test_deep_clone( assert_eq!(count_files(store, &dst_root, "_deletions").await, 0); } +#[tokio::test] +async fn test_deep_clone_rejects_unsupported_writer_before_copying() { + let test_dir = TempStdDir::default(); + let source_dir = test_dir.join("source"); + let target_dir = test_dir.join("target"); + let mut source = Dataset::write( + gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(32), BatchCount::from(1)), + source_dir.to_str().unwrap(), + None, + ) + .await + .unwrap(); + + let mut unsupported_manifest = source.manifest.as_ref().clone(); + unsupported_manifest.version += 1; + unsupported_manifest.writer_feature_flags |= feature_flags::FLAG_UNKNOWN << 1; + write_manifest_file( + source.object_store.as_ref(), + source.commit_handler.as_ref(), + &source.base, + &mut unsupported_manifest, + None, + &ManifestWriteConfig { + auto_set_feature_flags: false, + ..Default::default() + }, + source.manifest_location.naming_scheme, + None, + // No inline transaction to classify from, so validate. + true, + ) + .await + .unwrap(); + + let error = source + .deep_clone( + target_dir.to_str().unwrap(), + unsupported_manifest.version, + None, + ) + .await + .unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. })); + assert!(!target_dir.exists()); +} + +#[tokio::test] +async fn test_shallow_clone_rejects_unsupported_writer_before_writing_target() { + let test_dir = TempStdDir::default(); + let source_dir = test_dir.join("source"); + let target_dir = test_dir.join("target"); + let mut source = Dataset::write( + gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(32), BatchCount::from(1)), + source_dir.to_str().unwrap(), + None, + ) + .await + .unwrap(); + + let mut unsupported_manifest = source.manifest.as_ref().clone(); + unsupported_manifest.version += 1; + unsupported_manifest.writer_feature_flags |= feature_flags::FLAG_UNKNOWN << 1; + write_manifest_file( + source.object_store.as_ref(), + source.commit_handler.as_ref(), + &source.base, + &mut unsupported_manifest, + None, + &ManifestWriteConfig { + auto_set_feature_flags: false, + ..Default::default() + }, + source.manifest_location.naming_scheme, + None, + // No inline transaction to classify from, so validate. + true, + ) + .await + .unwrap(); + + let error = source + .shallow_clone( + target_dir.to_str().unwrap(), + unsupported_manifest.version, + None, + ) + .await + .unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. }), "{error}"); + assert!(!target_dir.exists()); +} + +#[tokio::test] +async fn test_deep_clone_recognizes_ambiguous_commit_as_own() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let test_dir = TempStdDir::default(); + let source_dir = test_dir.join("source"); + let source_uri = source_dir.to_str().unwrap(); + let target_dir = test_dir.join("target"); + let target_uri = target_dir.to_str().unwrap(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let data_reader = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(32), BatchCount::from(1)); + let mut source = Dataset::write( + data_reader, + source_uri, + Some(WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }), + ) + .await + .unwrap(); + let source_transaction_file = source.manifest().transaction_file.clone(); + + handler.fail_next(AmbiguousFailure::LandAndConflict); + let cloned = source + .deep_clone(target_uri, source.version().version, None) + .await + .expect("readback must identify the deep-clone transaction that landed"); + + assert_eq!(cloned.count_rows(None).await.unwrap(), 32); + assert_ne!(cloned.manifest().transaction_file, source_transaction_file); + assert!(cloned.manifest().transaction_section.is_some()); +} + +// Uses an in-memory source store to force a cross-store copy. The in-memory store has +// known platform-specific quirks on Windows (it reads back empty there; see the note in +// tests/resource_tests.rs), so this test is gated to non-Windows. The local write side is +// covered on Windows by `test_deep_clone`, and streaming copies against real cloud stores +// use platform-agnostic std/tokio I/O. +#[cfg(not(windows))] +#[rstest] +#[tokio::test] +async fn test_deep_clone_cross_store( + #[values(LanceFileVersion::Legacy, LanceFileVersion::Stable)] + data_storage_version: LanceFileVersion, +) { + // Source lives in an in-memory store while the target is a local directory. Their + // different `store_prefix`es exercise separate source and destination implementations. + let session = Arc::new(Session::default()); + let test_dir = TempStdDir::default(); + let clone_dir = test_dir.join("clone_ds"); + let cloned_uri = clone_dir.to_str().unwrap(); + + // 64 rows across 4 files exercises the multi-fragment copy path. + let data_reader = gen_batch() + .col("id", array::step::()) + .col("val", array::fill_utf8("deep".to_string())) + .into_reader_rows(RowCount::from(64), BatchCount::from(1)); + + let mut dataset = Dataset::write( + data_reader, + "memory://cross_store_src", + Some(WriteParams { + max_rows_per_file: 16, + max_rows_per_group: 16, + data_storage_version: Some(data_storage_version), + session: Some(session.clone()), + ..Default::default() + }), + ) + .await + .unwrap(); + assert_ne!(dataset.object_store.store_prefix, ""); + + // Create a scalar index so the index files and the manifest index section are also + // copied across stores (the index section is read through the source store at commit). + dataset + .create_index( + &["id"], + IndexType::Scalar, + Some("id_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Delete some rows so a deletion file is also streamed across stores. + dataset.delete("id < 10").await.unwrap(); + let cloned_dataset = dataset + .deep_clone(cloned_uri, dataset.version().version, None) + .await + .unwrap(); + + // The clone targets a local store, distinct from the in-memory source. + assert_ne!( + cloned_dataset.object_store.store_prefix, + dataset.object_store.store_prefix + ); + assert!(cloned_dataset.manifest().base_paths.is_empty()); + + // Re-open the clone from a fresh session to prove the files were physically copied + // into the target store and the clone is fully independent of the source store. + let reopened = DatasetBuilder::from_uri(cloned_uri).load().await.unwrap(); + let batches = reopened + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 54); // 64 rows - 10 deletions + + // The scalar index must have been copied and resolve against the target store, with its + // base reference normalized to local (no external base_paths). + let cloned_indices = reopened.load_indices().await.unwrap(); + assert_eq!(cloned_indices.len(), 1); + assert_eq!(cloned_indices.first().unwrap().name, "id_idx"); + assert!(cloned_indices.iter().all(|idx| idx.base_id.is_none())); +} + // Helper: count files under a dataset directory (data/_indices/_deletions) async fn count_files(store: &ObjectStore, root: &Path, prefix: &str) -> usize { use futures::StreamExt; @@ -1926,6 +2611,53 @@ async fn append_dictionary( } } +#[rstest] +#[case::appended_null_value( + (0..=i8::MAX) + .map(|value| Some(format!("value-{value}"))) + .collect(), + (0..=i8::MAX).map(Some).chain([None]).collect() +)] +#[case::existing_unaddressable_null_value( + (0..130) + .map(|value| (value != 129).then(|| format!("value-{value}"))) + .collect(), + vec![Some(0_i8), None] +)] +#[tokio::test] +async fn write_rejects_dictionary_null_index_outside_declared_key_range( + #[case] values: Vec>, + #[case] indices: Vec>, +) { + let dictionary = Arc::new(StringArray::from(values)); + let indices = Int8Array::from(indices); + let dictionary = Int8DictionaryArray::try_new(indices, dictionary).unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "d", + dictionary.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(dictionary)]).unwrap(); + + let error = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_0), + ..Default::default() + }), + ) + .await + .expect_err("the widened indices cannot be represented by Int8"); + + assert!(matches!(error, Error::InvalidInput { .. })); + assert!( + error + .to_string() + .contains("dictionary indices use 32 bits but the declared Int8 key type uses 8 bits") + ); +} + #[rstest] #[tokio::test] async fn overwrite_dataset( @@ -1985,9 +2717,10 @@ async fn overwrite_dataset( let fragments = dataset.get_fragments(); assert_eq!(fragments.len(), 1); - // Fragment ids reset after overwrite. - assert_eq!(fragments[0].id(), 0); - assert_eq!(dataset.manifest.max_fragment_id(), Some(0)); + // Fragment ids continue from the dataset's high water mark after an + // overwrite; they are never reused. + assert_eq!(fragments[0].id(), 1); + assert_eq!(dataset.manifest.max_fragment_id(), Some(1)); let actual_ds = Dataset::open(&test_uri).await.unwrap(); assert_eq!(actual_ds.version().version, 2); @@ -2254,12 +2987,8 @@ async fn test_overwrite_mixed_version() { .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::Legacy + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V1 ); let reader = RecordBatchIterator::new(vec![data].into_iter().map(Ok), schema); @@ -2275,12 +3004,8 @@ async fn test_overwrite_mixed_version() { .unwrap(); assert_eq!( - dataset - .manifest - .data_storage_format - .lance_file_version() - .unwrap(), - LanceFileVersion::Legacy + dataset.manifest.data_storage_format.lance_file_format(), + ConcreteFileVersion::V1 ); } @@ -2415,3 +3140,559 @@ async fn test_open_dataset_non_not_found_error_is_not_masked() { err, ); } + +#[tokio::test] +async fn test_get_fragment_by_id() { + // 4 fragments of 10 rows each, ids 0..=3. + let data = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(4)); + let mut dataset = Dataset::write( + data, + "memory://", + Some(WriteParams { + max_rows_per_file: 10, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.fragments().len(), 4); + + for id in 0..4 { + let fragment = dataset.get_fragment(id).unwrap(); + assert_eq!(fragment.id(), id); + } + assert!(dataset.get_fragment(4).is_none()); + assert!(dataset.get_fragment(usize::MAX).is_none()); + + // Deleting all rows of fragment 1 leaves a hole in the id space. + dataset.delete("i >= 10 AND i < 20").await.unwrap(); + assert_eq!(dataset.fragments().len(), 3); + assert!(dataset.get_fragment(1).is_none()); + for id in [0, 2, 3] { + let fragment = dataset.get_fragment(id).unwrap(); + assert_eq!(fragment.id(), id); + } +} + +/// Replace the manifest fragments, rebuilding the derived state exactly as +/// opening a dataset does, so the lookups see a manifest Lance would read off +/// disk rather than one it just built. +fn install_fragments(dataset: &mut Dataset, fragments: Vec) { + let mut manifest = dataset.manifest.as_ref().clone(); + manifest.fragments = Arc::new(fragments); + dataset.manifest = Arc::new(manifest); + dataset.fragment_bitmap = Arc::new( + dataset + .manifest + .fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect(), + ); +} + +/// Manifests written before fragments were forced into id order (Lance 0.10 and +/// earlier) and manifests with duplicate fragment ids (Lance 0.16 and earlier) +/// are still readable -- neither is rejected on open. A lookup that trusted the +/// sorted-by-id invariant would hand back a different fragment's data. +#[rstest] +#[case::unsorted(vec![3, 1, 2, 0])] +#[case::duplicate_ids(vec![0, 0, 2, 3])] +#[tokio::test] +async fn test_get_fragment_on_legacy_manifest(#[case] ids: Vec) { + let data = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(4)); + let mut dataset = Dataset::write( + data, + "memory://", + Some(WriteParams { + max_rows_per_file: 10, + ..Default::default() + }), + ) + .await + .unwrap(); + + let by_id: HashMap = dataset + .manifest + .fragments + .iter() + .map(|fragment| (fragment.id, fragment.clone())) + .collect(); + let fragments: Vec = ids.iter().map(|id| by_id[id].clone()).collect(); + install_fragments(&mut dataset, fragments); + + for id in ids.iter().map(|id| *id as usize) { + let fragment = dataset.get_fragment(id).unwrap(); + assert_eq!( + fragment.id(), + id, + "get_fragment({id}) returned the wrong fragment" + ); + assert_eq!( + fragment.count_rows(None).await.unwrap(), + 10, + "get_fragment({id}) returned unreadable metadata" + ); + } + assert!(dataset.get_fragment(4).is_none()); +} + +async fn write_tiny_dataset(uri: &str) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]).unwrap(); + Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap() +} + +/// `drop` deletes whatever path it is handed, so the guard must accept a real dataset +/// even when the user keeps unmanaged files beside it. +#[rstest] +#[case::committed_dataset(&[], true)] +// `cleanup_preserves_unmanaged_dirs_and_files` establishes that a dataset may sit +// alongside unmanaged files, so those must not block a drop either. +#[case::dataset_with_unmanaged_files(&["images/clip.mp4", "misc/notes.txt"], true)] +#[tokio::test] +async fn test_validate_dataset_root_for_drop_accepts_committed_dataset( + #[case] extra_entries: &[&str], + #[case] expected_ok: bool, +) { + let test_dir = TempStdDir::default(); + let dataset_dir = test_dir.as_ref().join("t.lance"); + write_tiny_dataset(dataset_dir.to_str().unwrap()).await; + for entry in extra_entries { + let path = dataset_dir.join(entry); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"unmanaged").unwrap(); + } + + let (object_store, base) = ObjectStore::from_uri(dataset_dir.to_str().unwrap()) + .await + .unwrap(); + let result = validate_dataset_root_for_drop(&object_store, &base).await; + + assert_eq!( + result.is_ok(), + expected_ok, + "unexpected outcome: {result:?}" + ); +} + +/// A manifest under a detached version name is still a manifest, so a root holding only +/// one is a dataset root. +#[tokio::test] +async fn test_validate_dataset_root_for_drop_accepts_detached_manifest() { + let source_dir = TempStdDir::default(); + let source = source_dir.as_ref().join("t.lance"); + let dataset = write_tiny_dataset(source.to_str().unwrap()).await; + let manifest_name = dataset + .manifest_location() + .path + .filename() + .unwrap() + .to_string(); + let manifest_bytes = std::fs::read(source.join("_versions").join(manifest_name)).unwrap(); + + let test_dir = TempStdDir::default(); + let detached_dir = test_dir.as_ref().join("_versions"); + std::fs::create_dir_all(&detached_dir).unwrap(); + std::fs::write( + detached_dir.join("d9223372036854775808.manifest"), + &manifest_bytes, + ) + .unwrap(); + + let (object_store, base) = ObjectStore::from_uri(test_dir.to_str().unwrap()) + .await + .unwrap(); + validate_dataset_root_for_drop(&object_store, &base) + .await + .unwrap(); +} + +/// A namespace may reserve a table name without ever writing a manifest, and dropping +/// that reservation has to keep working. +#[rstest] +#[case::declared(".lance-reserved")] +#[case::deregistered(".lance-deregistered")] +#[tokio::test] +async fn test_validate_dataset_root_for_drop_accepts_namespace_marker(#[case] marker: &str) { + let test_dir = TempStdDir::default(); + std::fs::write(test_dir.as_ref().join(marker), b"table t").unwrap(); + + let (object_store, base) = ObjectStore::from_uri(test_dir.to_str().unwrap()) + .await + .unwrap(); + validate_dataset_root_for_drop(&object_store, &base) + .await + .unwrap(); +} + +/// Anything short of a readable manifest must fail closed, because the delete this +/// guards is recursive and unrecoverable. A file merely sitting under `_versions/`, or +/// merely named like a manifest, is not evidence: either is trivial to end up with in a +/// storage root that holds irreplaceable data. +#[rstest] +#[case::unrelated_file_under_versions(&["_versions/README", "reports/q1.csv"])] +#[case::unreadable_manifest(&["_versions/1.manifest", "reports/q1.csv"])] +#[case::unreadable_v2_manifest(&["_versions/00000000000000000001.manifest", "reports/q1.csv"])] +#[case::unreadable_detached_manifest(&["_versions/d9223372036854775808.manifest"])] +#[case::staged_manifest(&["_versions/1.manifest-2e1f0c3a", "data/0.lance"])] +#[case::data_files_only(&["data/0.lance"])] +#[case::layout_dirs_only(&["data/0.lance", "tree/branch/data/0.lance"])] +#[case::home_directory(&["data/0.lance", "notes.txt"])] +#[case::unrelated_directory(&["reports/q1.csv"])] +#[tokio::test] +async fn test_validate_dataset_root_for_drop_rejects_paths_without_a_readable_manifest( + #[case] entries: &[&str], +) { + let test_dir = TempStdDir::default(); + for entry in entries { + let path = test_dir.as_ref().join(entry); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"not a manifest").unwrap(); + } + + let (object_store, base) = ObjectStore::from_uri(test_dir.to_str().unwrap()) + .await + .unwrap(); + let err = validate_dataset_root_for_drop(&object_store, &base) + .await + .expect_err("must not authorize a recursive delete without a readable manifest"); + + assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}"); + assert!( + err.to_string().contains("no readable Lance manifest"), + "{err}" + ); +} + +/// A zero-length manifest reads as corrupt rather than as I/O failure, so it must be +/// reported as "not a dataset root" like any other unreadable manifest. +#[tokio::test] +async fn test_validate_dataset_root_for_drop_rejects_empty_manifest() { + let test_dir = TempStdDir::default(); + let versions = test_dir.as_ref().join("_versions"); + std::fs::create_dir_all(&versions).unwrap(); + std::fs::write(versions.join("1.manifest"), b"").unwrap(); + + let (object_store, base) = ObjectStore::from_uri(test_dir.to_str().unwrap()) + .await + .unwrap(); + let err = validate_dataset_root_for_drop(&object_store, &base) + .await + .expect_err("an empty manifest is not evidence of a dataset"); + + assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}"); +} + +/// The parent of a dataset is not a dataset, however Lance-looking its children are. +#[tokio::test] +async fn test_validate_dataset_root_for_drop_rejects_warehouse_root() { + let test_dir = TempStdDir::default(); + let warehouse = test_dir.as_ref().join("warehouse"); + write_tiny_dataset(warehouse.join("t.lance").to_str().unwrap()).await; + + let (object_store, base) = ObjectStore::from_uri(warehouse.to_str().unwrap()) + .await + .unwrap(); + let err = validate_dataset_root_for_drop(&object_store, &base) + .await + .expect_err("a warehouse root must not be droppable"); + + assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}"); +} + +/// A path that does not exist must stay a not-found error from the delete itself +/// rather than becoming a validation error, so callers keep the error kind they +/// already handle. +#[tokio::test] +async fn test_validate_dataset_root_for_drop_allows_missing_path() { + let test_dir = TempStdDir::default(); + let missing = test_dir.as_ref().join("does_not_exist"); + + let (object_store, base) = ObjectStore::from_uri(missing.to_str().unwrap()) + .await + .unwrap(); + + validate_dataset_root_for_drop(&object_store, &base) + .await + .unwrap(); +} + +/// Restore and clone rebuild a manifest from a stored one, never passing +/// through the Arrow-schema conversion that validates a primary key. Both write +/// through `write_manifest_file`, so the invariant is enforced there — a schema +/// that reached a manifest before the write paths were validated cannot be +/// carried forward into a new version. +#[tokio::test] +async fn write_manifest_file_rejects_a_nullable_primary_key() { + use lance_core::utils::tempfile::TempStrDir; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int32, + false, + )])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2]))]).unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + None, + ) + .await + .unwrap(); + + // Stand in for a manifest stored before the write paths were validated. + let mut manifest = dataset.manifest.as_ref().clone(); + let id_field = manifest + .schema + .fields + .iter_mut() + .find(|field| field.name == "id") + .expect("schema has an id column"); + id_field.unenforced_primary_key_position = Some(1); + id_field.nullable = true; + manifest.version += 1; + + let err = write_manifest_file( + dataset.object_store.as_ref(), + dataset.commit_handler.as_ref(), + &dataset.base, + &mut manifest, + None, + &ManifestWriteConfig { + auto_set_feature_flags: false, + timestamp: None, + use_stable_row_ids: false, + use_legacy_format: None, + storage_format: None, + disable_transaction_file: false, + migration_next_row_id: None, + }, + dataset.manifest_location.naming_scheme, + None, + // Previously classified from a None inline copy, which validated. + true, + ) + .await + .expect_err("a nullable primary key must not reach a manifest"); + assert!( + format!("{err:?}").contains("must not be nullable"), + "unexpected error: {err:?}" + ); +} + +/// Stand in for a table written by a released version, where the metadata path +/// could install a primary key on a column that permits nulls. The forging goes +/// through `write_manifest_file` classified as schema-preserving, which is +/// precisely the hole those versions had, so the resulting manifest is the one +/// an upgrade actually finds on disk. +/// +/// The two rows are `[1, NULL]`, so the null the key must not hold is really +/// present and the repairing delete has something to remove. +async fn write_dataset_with_a_legacy_nullable_primary_key(uri: &str) -> Dataset { + forge_legacy_nullable_primary_key(uri, false).await +} + +/// The originally reported sequence initialised MemWAL *before* the key was +/// installed, so `initialize_mem_wal`'s own check never saw it. That ordering +/// matters: a MemWAL transaction carries mem-table state, so it is far more +/// likely to outgrow the inline limit than a bare config update. +async fn forge_legacy_nullable_primary_key(uri: &str, with_mem_wal: bool) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int32, + true, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![Some(1), None]))], + ) + .unwrap(); + let mut dataset = Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap(); + + if with_mem_wal { + dataset + .initialize_mem_wal() + .unsharded() + .execute() + .await + .expect("MemWAL initialises while the key is still valid"); + } + + // Carried forward explicitly: passing None here would drop the MemWAL index + // the fixture just installed. + let indices = dataset.load_indices().await.unwrap().as_ref().clone(); + + let mut manifest = dataset.manifest.as_ref().clone(); + let id_field = manifest + .schema + .fields + .iter_mut() + .find(|field| field.name == "id") + .expect("schema has an id column"); + id_field.unenforced_primary_key_position = Some(1); + manifest.version += 1; + + // Committed below `write_manifest_file` on purpose. Going through it would + // make building the fixture depend on the very validation these tests + // exercise, so a regression would show up as a fixture that cannot be + // built rather than as the behaviour under test changing. + manifest.set_timestamp(crate::dataset::timestamp_to_nanos(None)); + manifest.update_max_fragment_id(); + dataset + .commit_handler + .commit( + &mut manifest, + (!indices.is_empty()).then_some(indices), + &dataset.base, + dataset.object_store.as_ref(), + lance_table::io::commit::write_manifest_file_to_path, + dataset.manifest_location.naming_scheme, + None, + ) + .await + .expect("forging a legacy manifest must not itself be blocked"); + + Dataset::open(uri).await.unwrap() +} + +/// An unrelated config update leaves the key exactly as it found it, so it is +/// exempt -- and must stay exempt no matter how large its payload is. The +/// disposition comes from the operation; only the inline copy depends on size. +#[tokio::test] +async fn an_exempt_operation_stays_exempt_when_its_transaction_spills() { + use crate::io::commit::MAX_INLINE_TRANSACTION_BYTES; + use lance_core::utils::tempfile::TempStrDir; + + let test_dir = TempStrDir::default(); + let mut dataset = write_dataset_with_a_legacy_nullable_primary_key(&test_dir).await; + + dataset + .update_config([("unrelated".to_string(), "small".to_string())]) + .await + .expect("a small unrelated config update must be exempt"); + let after_small = dataset.version().version; + + dataset + .update_config([( + "large-unrelated".to_string(), + "x".repeat(2 * MAX_INLINE_TRANSACTION_BYTES), + )]) + .await + .expect("the same update must stay exempt once its bytes stop inlining"); + + assert!( + dataset.version().version > after_small, + "the spilling update must have committed a new version" + ); +} + +/// The repair path: drop the offending rows, then tighten the column. Both have +/// to be reachable on a table that already carries the bad key, or the only +/// remaining fix is a full overwrite. +#[tokio::test] +async fn a_legacy_nullable_primary_key_can_be_repaired_in_place() { + use lance_core::utils::tempfile::TempStrDir; + + let test_dir = TempStrDir::default(); + let mut dataset = write_dataset_with_a_legacy_nullable_primary_key(&test_dir).await; + assert_eq!(dataset.count_rows(None).await.unwrap(), 2); + + dataset + .delete("id IS NULL") + .await + .expect("removing the offending rows must not be blocked"); + assert_eq!(dataset.count_rows(None).await.unwrap(), 1); + + dataset + .alter_columns(&[ColumnAlteration::new("id".into()).set_nullable(false)]) + .await + .expect("tightening the column completes the repair"); + + let id_field = dataset + .schema() + .fields + .iter() + .find(|field| field.name == "id") + .expect("schema has an id column"); + assert!( + !id_field.nullable, + "the key column must end up non-nullable" + ); +} + +/// The MemWAL variant of the repair path. This is the state the original report +/// was about, and the one a size-coupled gate blocks: its transactions carry +/// An overlay attaches files to existing fragments; it carries no schema, so a +/// dataset that already holds a nullable primary key must still be able to +/// commit one. `DataOverlay` was missing from the exempt classifier, which +/// closed that path for exactly the legacy datasets this validation is meant to +/// leave repairable. +#[tokio::test] +async fn an_overlay_commits_on_a_legacy_nullable_primary_key() { + use lance_core::utils::tempfile::TempStrDir; + use lance_table::transaction::Operation; + + let test_dir = TempStrDir::default(); + let dataset = write_dataset_with_a_legacy_nullable_primary_key(&test_dir).await; + let read_version = dataset.manifest.version; + + let dataset = Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { groups: vec![] }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .expect("an overlay leaves the schema alone, so the legacy key must not block it"); + + assert_eq!(dataset.manifest.version, read_version + 1); + assert_eq!(dataset.count_rows(None).await.unwrap(), 2); +} + +/// mem-table state, so they stop inlining long before a config update does. +#[tokio::test] +async fn a_legacy_nullable_primary_key_can_be_repaired_under_mem_wal() { + use lance_core::utils::tempfile::TempStrDir; + + let test_dir = TempStrDir::default(); + let mut dataset = forge_legacy_nullable_primary_key(&test_dir, true).await; + assert!( + dataset + .load_indices() + .await + .unwrap() + .iter() + .any(|index| index.name == lance_index::mem_wal::MEM_WAL_INDEX_NAME), + "the fixture must really have MemWAL initialised" + ); + + dataset + .update_config([("unrelated".to_string(), "small".to_string())]) + .await + .expect("an unrelated config update must be exempt"); + + dataset + .delete("id IS NULL") + .await + .expect("removing the offending rows must not be blocked under MemWAL"); + assert_eq!(dataset.count_rows(None).await.unwrap(), 1); +} diff --git a/rust/lance/src/dataset/tests/dataset_merge_update.rs b/rust/lance/src/dataset/tests/dataset_merge_update.rs index 4b81f62dc6c..610538c2502 100644 --- a/rust/lance/src/dataset/tests/dataset_merge_update.rs +++ b/rust/lance/src/dataset/tests/dataset_merge_update.rs @@ -6,29 +6,34 @@ use std::sync::Arc; use std::time::Duration; use std::vec; +use crate::dataset::CommitBuilder; use crate::dataset::ROW_ID; use crate::dataset::WriteDestination; +use crate::dataset::builder::DatasetBuilder; use crate::dataset::optimize::{CompactionOptions, compact_files}; +use crate::dataset::schema_evolution::ColumnAlteration; use crate::dataset::transaction::{DataReplacementGroup, Operation}; use crate::dataset::{AutoCleanupParams, MergeInsertBuilder, ProjectionRequest, UpdateBuilder}; use crate::index::DatasetIndexExt; use crate::{Dataset, Error}; -use lance_core::ROW_ADDR; +use lance_core::{ROW_ADDR, ROW_LAST_UPDATED_AT_VERSION}; use lance_index::IndexType; use lance_index::optimize::OptimizeOptions; use lance_index::scalar::FullTextSearchQuery; -use lance_index::scalar::ScalarIndexParams; +use lance_index::scalar::inverted::query::{BooleanQuery, FtsQuery, MatchQuery, Occur}; use lance_index::scalar::inverted::tokenizer::InvertedIndexParams; +use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use mock_instant::thread_local::MockClock; use crate::dataset::write::{InsertBuilder, WriteMode, WriteParams}; use arrow::array::AsArray; +use arrow::array::builder::{LargeListBuilder, LargeStringBuilder}; use arrow::compute::concat_batches; use arrow_array::RecordBatch; -use arrow_array::{Array, LargeBinaryArray, StructArray}; +use arrow_array::{Array, LargeBinaryArray, MapArray, StructArray}; use arrow_array::{ - ArrayRef, Float32Array, Int32Array, ListArray, RecordBatchIterator, StringArray, - types::Int32Type, + ArrayRef, Float32Array, Int32Array, ListArray, RecordBatchIterator, StringArray, UInt64Array, + types::{Int32Type, UInt64Type}, }; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; use lance_arrow::BLOB_META_KEY; @@ -36,7 +41,7 @@ use lance_core::utils::tempfile::{TempDir, TempStrDir}; use lance_datafusion::utils::reader_to_stream; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; use lance_file::version::LanceFileVersion; -use lance_file::writer::FileWriter; + use lance_io::utils::CachedFileSize; use lance_table::format::{BasePath, DataFile, Fragment}; @@ -745,7 +750,7 @@ async fn test_datafile_replacement() { .create(&Path::from("data/test.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, schema.as_ref().try_into().unwrap(), Default::default(), @@ -830,6 +835,7 @@ async fn test_datafile_partial_replacement() { Operation::Merge { fragments: vec![fragment], schema: extended_schema.as_ref().try_into().unwrap(), + preserves_nullability: true, }, Some(2), None, @@ -852,7 +858,7 @@ async fn test_datafile_partial_replacement() { .create(&Path::from("data/test.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, partial_schema.as_ref().try_into().unwrap(), Default::default(), @@ -864,7 +870,7 @@ async fn test_datafile_partial_replacement() { writer.write_batch(&batch).await.unwrap(); writer.finish().await.unwrap(); - let (major, minor) = lance_file::version::LanceFileVersion::Stable.to_numbers(); + let (major, minor) = LanceFileVersion::Stable.resolve().to_data_file_numbers(); // find the datafile we want to replace let new_data_file = DataFile { @@ -1016,6 +1022,7 @@ async fn test_datafile_replacement_error() { Operation::Merge { fragments: vec![fragment], schema: extended_schema.as_ref().try_into().unwrap(), + preserves_nullability: true, }, Some(2), None, @@ -1520,15 +1527,16 @@ async fn test_issue_4429_nested_struct_encoding_v2_1_with_over_65k_structs() { /// Regression test for https://github.com/lancedb/lance/issues/5321 /// -/// merge_insert with reordered columns triggers the RewriteColumns path, -/// which prunes the index bitmap. After compact + optimize_indices, the old -/// stale B-tree data was being merged back in, causing "non-existent fragment" -/// errors on subsequent queries. +/// A partial merge_insert triggers the RewriteColumns path, which prunes the +/// index bitmap. After compact + optimize_indices, the old stale B-tree data +/// was being merged back in, causing "non-existent fragment" errors on +/// subsequent queries. #[tokio::test] async fn test_merge_insert_with_reordered_columns_and_index() { let schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", DataType::Int32, false), ArrowField::new("value", DataType::Utf8, true), + ArrowField::new("untouched", DataType::Utf8, true), ])); // Step 1: Create dataset with one row {id: 1, value: "a"} @@ -1537,6 +1545,7 @@ async fn test_merge_insert_with_reordered_columns_and_index() { vec![ Arc::new(Int32Array::from(vec![0, 1])), Arc::new(StringArray::from(vec!["x", "a"])), + Arc::new(StringArray::from(vec!["u", "v"])), ], ) .unwrap(); @@ -1564,8 +1573,9 @@ async fn test_merge_insert_with_reordered_columns_and_index() { .await .unwrap(); - // Step 3: merge_insert with reversed column order (value, id) - // This triggers the RewriteColumns path, which prunes the index bitmap + // Step 3: merge_insert with a partial schema in reversed column order + // (value, id). Omitting `untouched` triggers the RewriteColumns path, + // which prunes the index bitmap. let reversed_schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("value", DataType::Utf8, true), ArrowField::new("id", DataType::Int32, false), @@ -1610,6 +1620,7 @@ async fn test_merge_insert_with_reordered_columns_and_index() { vec![ Arc::new(Int32Array::from(vec![1])), Arc::new(StringArray::from(vec!["d"])), + Arc::new(StringArray::from(vec!["v"])), ], ) .unwrap(); @@ -1629,6 +1640,396 @@ async fn test_merge_insert_with_reordered_columns_and_index() { final_dataset.validate().await.unwrap(); } +/// Reordered merge_insert sources invalidate LabelList coverage for both full-row +/// and in-place column rewrites. Complete sources are also canonicalized before +/// reaching both the current and frozen Legacy writers. +/// +/// Regression test for https://github.com/lance-format/lance/issues/8502. +#[rstest] +#[case::legacy_full(LanceFileVersion::Legacy, true, 2)] +#[case::stable_full(LanceFileVersion::Stable, true, 2)] +#[case::v2_1_partial(LanceFileVersion::V2_1, false, 1)] +#[tokio::test] +async fn test_merge_insert_reordered_schema_invalidates_label_list_index( + #[case] data_storage_version: LanceFileVersion, + #[case] is_full_schema: bool, + #[case] expected_fragments: usize, +) { + let list_field = ArrowField::new( + "labels", + DataType::LargeList(Arc::new(ArrowField::new("item", DataType::LargeUtf8, true))), + true, + ); + let untouched_field = ArrowField::new("untouched", DataType::Utf8, true); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::UInt64, false), + list_field.clone(), + untouched_field.clone(), + ])); + + let make_labels = |values: &[&str]| { + let mut builder = LargeListBuilder::new(LargeStringBuilder::new()) + .with_field(Arc::new(ArrowField::new("item", DataType::LargeUtf8, true))); + for value in values { + builder.values().append_value(value); + builder.append(true); + } + Arc::new(builder.finish()) as ArrayRef + }; + + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![1, 2])) as ArrayRef, + make_labels(&["a", "b"]), + Arc::new(StringArray::from(vec!["u", "v"])) as ArrayRef, + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new([Ok(initial)], schema.clone()); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(data_storage_version), + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["id"], + IndexType::Bitmap, + Some("id_idx".to_owned()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["labels"], + IndexType::LabelList, + Some("labels_idx".to_owned()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::LabelList), + true, + ) + .await + .unwrap(); + + let mut update_fields = vec![list_field, ArrowField::new("id", DataType::UInt64, false)]; + let mut update_columns = vec![ + make_labels(&["z"]), + Arc::new(UInt64Array::from(vec![2])) as ArrayRef, + ]; + if is_full_schema { + update_fields.push(untouched_field); + update_columns.push(Arc::new(StringArray::from(vec!["v"])) as ArrayRef); + } + let reordered_schema = Arc::new(ArrowSchema::new(update_fields)); + let update = RecordBatch::try_new(reordered_schema.clone(), update_columns).unwrap(); + let reader = RecordBatchIterator::new([Ok(update)], reordered_schema); + let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_owned()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let (dataset, _) = merge_job + .execute(reader_to_stream(Box::new(reader))) + .await + .unwrap(); + assert_eq!( + dataset.get_fragments().len(), + expected_fragments, + "the source width must select the expected rewrite path" + ); + + async fn matching_ids(dataset: &Dataset, use_scalar_index: bool) -> Vec { + let mut scanner = dataset.scan(); + scanner.project(&["id"]).unwrap(); + scanner.filter("array_has(labels, 'z')").unwrap(); + scanner.use_scalar_index(use_scalar_index); + let batch = scanner.try_into_batch().await.unwrap(); + batch["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + } + + assert_eq!(matching_ids(&dataset, false).await, vec![2]); + assert_eq!(matching_ids(&dataset, true).await, vec![2]); + + let row = dataset + .scan() + .filter("id = 2") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + row["untouched"].as_string::().value(0), + "v", + "a partial rewrite must preserve omitted columns" + ); +} + +/// A complete source is matched recursively by field name before it reaches +/// either merge execution path, so reordered struct children keep their values. +#[rstest] +#[tokio::test] +async fn test_merge_insert_nested_reorder_preserves_values(#[values(false, true)] use_index: bool) { + let target_struct_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, false), + ]); + let target_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("s", DataType::Struct(target_struct_fields.clone()), false), + ])); + let initial_struct = StructArray::new( + target_struct_fields, + vec![ + Arc::new(Int32Array::from(vec![100, 200])) as ArrayRef, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + ], + None, + ); + let initial = RecordBatch::try_new( + target_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + Arc::new(initial_struct) as ArrayRef, + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new([Ok(initial)], target_schema); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + if use_index { + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_owned()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + } + + let source_struct_fields = Fields::from(vec![ + ArrowField::new("b", DataType::Int32, false), + ArrowField::new("a", DataType::Int32, false), + ]); + let source_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("s", DataType::Struct(source_struct_fields.clone()), false), + ArrowField::new("id", DataType::Int32, false), + ])); + let source_struct = StructArray::new( + source_struct_fields, + vec![ + Arc::new(Int32Array::from(vec![400])) as ArrayRef, + Arc::new(Int32Array::from(vec![300])) as ArrayRef, + ], + None, + ); + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + Arc::new(source_struct) as ArrayRef, + Arc::new(Int32Array::from(vec![2])) as ArrayRef, + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new([Ok(source)], source_schema); + let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_owned()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap(); + let (dataset, _) = merge_job + .execute(reader_to_stream(Box::new(reader))) + .await + .unwrap(); + + let row = dataset + .scan() + .filter("id = 2") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(row.num_rows(), 1); + let values = row["s"].as_struct(); + assert_eq!( + values + .column_by_name("a") + .unwrap() + .as_primitive::() + .value(0), + 300 + ); + assert_eq!( + values + .column_by_name("b") + .unwrap() + .as_primitive::() + .value(0), + 400 + ); +} + +/// Map entries participate in the same recursive name-based merge contract as +/// structs and lists, including when the map value is itself a struct. +#[rstest] +#[tokio::test] +async fn test_merge_insert_map_value_reorder_preserves_values( + #[values(false, true)] use_index: bool, +) { + let map_field = |value_fields: &Fields| { + let entry_fields = Fields::from(vec![ + ArrowField::new("key", DataType::Utf8, false), + ArrowField::new("value", DataType::Struct(value_fields.clone()), true), + ]); + let entries = ArrowField::new("entries", DataType::Struct(entry_fields), false); + ArrowField::new("m", DataType::Map(Arc::new(entries), false), false) + }; + let map_array = |value_fields: &Fields, keys: Vec<&str>, a: Vec, b: Vec| { + let value_columns = value_fields + .iter() + .map(|field| { + let values = if field.name() == "a" { + a.clone() + } else { + b.clone() + }; + Arc::new(Int32Array::from(values)) as ArrayRef + }) + .collect(); + let values = StructArray::new(value_fields.clone(), value_columns, None); + let entry_fields = Fields::from(vec![ + ArrowField::new("key", DataType::Utf8, false), + ArrowField::new("value", DataType::Struct(value_fields.clone()), true), + ]); + let entries = StructArray::new( + entry_fields.clone(), + vec![ + Arc::new(StringArray::from(keys)) as ArrayRef, + Arc::new(values) as ArrayRef, + ], + None, + ); + let offsets = (0..=entries.len() as i32).collect::>(); + Arc::new(MapArray::new( + Arc::new(ArrowField::new( + "entries", + DataType::Struct(entry_fields), + false, + )), + arrow_buffer::OffsetBuffer::new(offsets.into()), + entries, + None, + false, + )) as ArrayRef + }; + + let target_value_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, false), + ]); + let target_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + map_field(&target_value_fields), + ])); + let initial = RecordBatch::try_new( + target_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + map_array( + &target_value_fields, + vec!["k1", "k2"], + vec![100, 200], + vec![10, 20], + ), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new([Ok(initial)], target_schema); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + + if use_index { + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_owned()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + } + + let source_value_fields = Fields::from(vec![ + ArrowField::new("b", DataType::Int32, false), + ArrowField::new("a", DataType::Int32, false), + ]); + let source_schema = Arc::new(ArrowSchema::new(vec![ + map_field(&source_value_fields), + ArrowField::new("id", DataType::Int32, false), + ])); + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + map_array(&source_value_fields, vec!["k2"], vec![300], vec![400]), + Arc::new(Int32Array::from(vec![2])) as ArrayRef, + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new([Ok(source)], source_schema); + let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_owned()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap(); + let (dataset, _) = merge_job + .execute(reader_to_stream(Box::new(reader))) + .await + .unwrap(); + + let row = dataset + .scan() + .filter("id = 2") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(row.num_rows(), 1); + let map = row["m"].as_map(); + assert_eq!(map.value_length(0), 1); + let entries = map.value(0); + let values = entries["value"].as_struct(); + assert_eq!(values["a"].as_primitive::().value(0), 300); + assert_eq!(values["b"].as_primitive::().value(0), 400); +} + /// With stable row ids, updating a top-level struct column keeps a scalar index on a /// nested child field correct. The update API rejects nested column references, so a /// nested field can only be changed by setting its whole struct column; that update must @@ -1980,6 +2381,18 @@ async fn test_merge_insert_nested_index_stable_row_id() { ) .await .unwrap(); + // Force the indexed slow merge path whose rewrite metadata must include + // nested leaf ids as well as top-level fields. + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_owned()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); // Sanity: index finds id=2 for s.x = 20. let pre = dataset @@ -1991,17 +2404,32 @@ async fn test_merge_insert_nested_index_stable_row_id() { .unwrap(); assert_eq!(pre.num_rows(), 1, "precondition: s.x=20 should match id=2"); - // Full-row merge_insert update of id=2 changing s.x 20 -> 999 (pure rewrite-rows fragment). + // Full-row merge_insert update of id=2 changing s.x 20 -> 999. Supplying + // `(s, id)` also verifies reordered complete sources stay on RewriteRows. let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::DoNothing) .try_build() .unwrap(); - let reader = Box::new(RecordBatchIterator::new( - vec![Ok(make_batch(vec![2], vec![999]))], - schema.clone(), - )); + let reordered_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("s", DataType::Struct(struct_fields.clone()), false), + ArrowField::new("id", DataType::Int32, false), + ])); + let updated_struct = StructArray::new( + struct_fields, + vec![Arc::new(Int32Array::from(vec![999])) as ArrayRef], + None, + ); + let source = RecordBatch::try_new( + reordered_schema.clone(), + vec![ + Arc::new(updated_struct) as ArrayRef, + Arc::new(Int32Array::from(vec![2])) as ArrayRef, + ], + ) + .unwrap(); + let reader = Box::new(RecordBatchIterator::new(vec![Ok(source)], reordered_schema)); let (dataset, _stats) = merge_job.execute(reader_to_stream(reader)).await.unwrap(); // The rewritten fragment must NOT be covered by the nested `s.x` index, so @@ -2143,25 +2571,106 @@ async fn test_merge_insert_flat_index_stable_row_id_multiple_indexes() { ); } -/// DataReplacement should invalidate index fragment bitmaps for replaced fields. #[tokio::test] -async fn test_data_replacement_invalidates_index_bitmap() { - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("a", DataType::Int32, true), - ArrowField::new("b", DataType::Int32, true), - ])); +async fn test_data_replacement_advances_row_lineage() { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "value", + DataType::Int32, + true, + )])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1, 2]))]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); - // Create dataset with 2 columns - let batch = RecordBatch::try_new( + let replacement = RecordBatch::try_new( schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(Int32Array::from(vec![10, 20, 30])), - ], + vec![Arc::new(Int32Array::from(vec![10, 20]))], ) .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); - let mut dataset = Dataset::write(reader, "memory://test_replacement_idx", None) + let object_writer = dataset + .object_store + .create(&Path::from("data/lineage_replacement.lance")) + .await + .unwrap(); + let mut writer = lance_file::versions::v2_1::create_writer( + object_writer, + schema.as_ref().try_into().unwrap(), + Default::default(), + ) + .unwrap(); + writer.write_batch(&replacement).await.unwrap(); + writer.finish().await.unwrap(); + + let frag = dataset.get_fragment(0).unwrap(); + let mut new_data_file = frag.data_file_for_field(0).unwrap().clone(); + new_data_file.path = "lineage_replacement.lance".to_string(); + + let read_version = dataset.version().version; + let dataset = Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(0, new_data_file)], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + assert_eq!(dataset.version().version, 2); + + // The rows read differently now, so their last-updated stamp has to name + // the version that changed them or get_updated_rows will never see them. + let batch = dataset + .scan() + .project(&["value", ROW_LAST_UPDATED_AT_VERSION]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + batch["value"].as_primitive::().values(), + &[10, 20] + ); + assert_eq!( + batch[ROW_LAST_UPDATED_AT_VERSION] + .as_primitive::() + .values(), + &[2, 2] + ); +} + +/// DataReplacement should invalidate index fragment bitmaps for replaced fields. +#[tokio::test] +async fn test_data_replacement_invalidates_index_bitmap() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ])); + + // Create dataset with 2 columns + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int32Array::from(vec![10, 20, 30])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, "memory://test_replacement_idx", None) .await .unwrap(); @@ -2199,7 +2708,7 @@ async fn test_data_replacement_invalidates_index_bitmap() { .create(&Path::from("data/replacement.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, single_col_schema.as_ref().try_into().unwrap(), Default::default(), @@ -2312,7 +2821,7 @@ fn build_overlay_frag(prev: &Fragment, field_id: i32, new_file: &str) -> Fragmen new_file, vec![field_id], vec![0], - &LanceFileVersion::default(), + LanceFileVersion::default().resolve(), None, ); overlay @@ -2377,7 +2886,7 @@ async fn test_merge_rewriting_indexed_column_keeps_index_consistent() { .unwrap(); let new_a_path = dataset.data_dir().join("merge_new_a.lance"); let object_writer = dataset.object_store.create(&new_a_path).await.unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, a_only.as_ref().try_into().unwrap(), Default::default(), @@ -2396,6 +2905,7 @@ async fn test_merge_rewriting_indexed_column_keeps_index_consistent() { Operation::Merge { fragments: vec![overlay], schema: schema.as_ref().try_into().unwrap(), + preserves_nullability: true, }, Some(read_version), None, @@ -2425,7 +2935,6 @@ async fn test_merge_rewriting_indexed_column_keeps_index_consistent() { /// stale index entries are blocked at query time. #[tokio::test] async fn test_data_replacement_populates_invalidated_bitmap() { - use lance_file::writer::FileWriter; use object_store::path::Path; let schema = Arc::new(ArrowSchema::new(vec![ @@ -2481,7 +2990,7 @@ async fn test_data_replacement_populates_invalidated_bitmap() { .create(&Path::from("data/replacement_inv.lance")) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, value_schema.as_ref().try_into().unwrap(), Default::default(), @@ -2606,7 +3115,7 @@ async fn test_fts_stale_entries_after_data_replacement() { .create(&replacement_path) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, schema.as_ref().try_into().unwrap(), Default::default(), @@ -2679,6 +3188,140 @@ async fn test_fts_stale_entries_after_data_replacement() { assert_eq!(results.num_rows(), 1); } +/// Cross-column compound fast search must not combine different column-local +/// fragment domains after a partial data replacement. +#[tokio::test] +async fn test_cross_column_fast_search_blocks_column_local_stale_postings() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("title", DataType::Utf8, false), + ArrowField::new("body", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![0, 1])), + Arc::new(StringArray::from(vec!["noise", "target"])), + Arc::new(StringArray::from(vec!["noise", "stale"])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write( + reader, + "memory://cross_column_fast_search_replacement", + Some(WriteParams { + max_rows_per_file: 1, + ..Default::default() + }), + ) + .await + .unwrap(); + + for column in ["title", "body"] { + dataset + .create_index( + &[column], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + } + + let body_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "body", + DataType::Utf8, + false, + )])); + let replacement = RecordBatch::try_new( + body_schema.clone(), + vec![Arc::new(StringArray::from(vec!["fresh"]))], + ) + .unwrap(); + let replacement_path = dataset.data_dir().join("body_replacement.lance"); + let object_writer = dataset + .object_store + .create(&replacement_path) + .await + .unwrap(); + let mut writer = lance_file::versions::v2_1::create_writer( + object_writer, + body_schema.as_ref().try_into().unwrap(), + Default::default(), + ) + .unwrap(); + writer.write_batch(&replacement).await.unwrap(); + writer.finish().await.unwrap(); + + let (file_major_version, file_minor_version) = + LanceFileVersion::Stable.resolve().to_data_file_numbers(); + let replacement_file = DataFile { + path: "body_replacement.lance".to_string(), + fields: Arc::from([2]), + column_indices: Arc::from([0]), + file_major_version, + file_minor_version, + file_size_bytes: CachedFileSize::unknown(), + base_id: None, + }; + let read_version = dataset.version().version; + let dataset = Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(1, replacement_file)], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + + let match_query = |term: &str, column: &str| { + MatchQuery::new(term.to_owned()) + .with_column(Some(column.to_owned())) + .into() + }; + let query: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("target", "title")), + (Occur::Must, match_query("stale", "body")), + ]) + .into(); + + let mut exact_scanner = dataset.scan(); + exact_scanner + .full_text_search(FullTextSearchQuery::new_query(query.clone())) + .unwrap(); + exact_scanner.limit(Some(10), None).unwrap(); + assert_eq!( + exact_scanner.try_into_batch().await.unwrap().num_rows(), + 0, + "the current body value must not match the stale term" + ); + + let mut fast_scanner = dataset.scan(); + fast_scanner + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .fast_search(); + fast_scanner.limit(Some(10), None).unwrap(); + let fast_plan = fast_scanner.explain_plan(false).await.unwrap(); + assert!( + !fast_plan.contains("CrossColumnCompoundFtsScorer"), + "different per-column coverage must retain field-local masking:\n{fast_plan}" + ); + assert_eq!( + fast_scanner.try_into_batch().await.unwrap().num_rows(), + 0, + "the title index must not re-admit the body's stale physical posting" + ); +} + /// Same scenario as test_fts_index_incremental_reindex_after_in_place_update /// but with a vector (IVF_PQ) index instead of FTS. #[tokio::test] @@ -2783,7 +3426,7 @@ async fn test_vector_index_after_data_replacement() { .create(&replacement_path) .await .unwrap(); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::v2_1::create_writer( object_writer, schema.as_ref().try_into().unwrap(), Default::default(), @@ -2847,6 +3490,7 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() { let schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", DataType::Int32, false), ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("untouched", DataType::Utf8, true), ])); // Step 1: Create dataset with 2 rows in separate fragments @@ -2858,6 +3502,7 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() { "the quick brown fox", "the lazy dog", ])), + Arc::new(StringArray::from(vec!["u", "v"])), ], ) .unwrap(); @@ -2890,9 +3535,9 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() { .unwrap(); assert_eq!(results.num_rows(), 1); - // Step 3: merge_insert with reversed column order (text, id) - // This triggers the RewriteColumns/DataReplacement path, which prunes the - // index fragment bitmap for the 'text' column. + // Step 3: merge_insert with a partial schema in reversed column order + // (text, id). Omitting `untouched` triggers the RewriteColumns/DataReplacement + // path, which prunes the index fragment bitmap for the 'text' column. let reversed_schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("text", DataType::Utf8, true), ArrowField::new("id", DataType::Int32, false), @@ -2996,6 +3641,7 @@ async fn test_fts_index_stale_data_after_merge_insert_compact_optimize() { vec![ Arc::new(Int32Array::from(vec![1])), Arc::new(StringArray::from(vec!["final text"])), + Arc::new(StringArray::from(vec!["v"])), ], ) .unwrap(); @@ -3035,6 +3681,7 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() { let schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("id", DataType::Int32, false), ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("untouched", DataType::Utf8, true), ])); // Step 1: Create dataset with 2 rows in separate fragments @@ -3046,6 +3693,7 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() { "the quick brown fox", "the lazy dog", ])), + Arc::new(StringArray::from(vec!["u", "v"])), ], ) .unwrap(); @@ -3086,9 +3734,9 @@ async fn test_fts_index_incremental_reindex_after_in_place_update() { .unwrap(); assert_eq!(results.num_rows(), 1); - // Step 3: merge_insert with reversed column order to trigger - // RewriteColumns/DataReplacement path, which prunes the index - // fragment bitmap for the updated fragment. + // Step 3: merge_insert with a partial schema in reversed column order. + // Omitting `untouched` triggers the RewriteColumns/DataReplacement path, + // which prunes the index fragment bitmap for the updated fragment. // Update id=1 ("the lazy dog" -> "a speedy cat") let reversed_schema = Arc::new(ArrowSchema::new(vec![ ArrowField::new("text", DataType::Utf8, true), @@ -4294,3 +4942,582 @@ async fn test_merge_insert_target_all_bases() { assert!(all_rows.contains(&row), "missing row {:?}", row); } } + +/// A write landing between the tightening scan and its commit falsifies the +/// claim, leaving a table that validates but cannot be scanned. +#[rstest] +#[case::tightening_conflicts(true, true)] +#[case::rename_does_not(false, false)] +#[tokio::test] +async fn test_alter_columns_conflicts_only_when_asserting( + #[case] tighten: bool, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + // Leave the first handle a version behind, so the alteration commits stale. + let appended = InsertBuilder::new(WriteDestination::Dataset(Arc::new(dataset.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![ + arrow_array::record_batch!(("value", Int32, [3])).unwrap(), + ]) + .await + .unwrap(); + assert_eq!(appended.version().version, 2); + + let mut stale = dataset; + let alteration = if tighten { + ColumnAlteration::new("value".into()).set_nullable(false) + } else { + ColumnAlteration::new("value".into()).rename("renamed".into()) + }; + let result = stale.alter_columns(&[alteration]).await; + assert_eq!( + result.is_err(), + expect_conflict, + "tighten={tighten}: got {result:?}" + ); +} + +/// read_version is the version the data was validated against. Declaring it +/// honestly puts a later tightening inside the conflict window; declaring a +/// later version skips the checks, which is a caller bug, not a guarantee. +#[rstest] +#[case::honest_read_version_conflicts(1, true)] +#[case::misdeclared_read_version_commits(2, false)] +#[tokio::test] +async fn test_stale_append_protection_follows_read_version( + #[case] declared: u64, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new(Dataset::write(reader, "memory://", None).await.unwrap()); + + let mut tightened = dataset.schema().clone(); + tightened.fields[0].nullable = false; + let tightened = Dataset::commit( + WriteDestination::Dataset(dataset.clone()), + Operation::Project { + schema: tightened, + preserves_nullability: false, + }, + Some(dataset.version().version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap(); + assert_eq!(tightened.version().version, 2); + + let mut append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + append.read_version = declared; + + let result = CommitBuilder::new(Arc::new(tightened)) + .execute(append) + .await; + assert_eq!( + result.is_err(), + expect_conflict, + "declared={declared}: got {result:?}" + ); +} + +/// A field added and tightened after the write snapshot: the tightening's +/// claim is in the honest conflict window, so the operation-wide barrier +/// rejects the stale append. Added-but-nullable commits, since synthesized +/// nulls are legal there. +#[rstest] +#[case::added_then_tightened(true, true)] +#[case::added_still_nullable(false, false)] +#[tokio::test] +async fn test_stale_append_vs_field_added_since( + #[case] tighten: bool, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new(Dataset::write(reader, "memory://", None).await.unwrap()); + + let append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + + let mut latest = dataset.as_ref().clone(); + latest + .add_columns( + crate::dataset::NewColumnTransform::SqlExpressions(vec![( + "new_value".to_string(), + "value".to_string(), + )]), + None, + None, + ) + .await + .unwrap(); + if tighten { + latest + .alter_columns(&[ColumnAlteration::new("new_value".into()).set_nullable(false)]) + .await + .unwrap(); + } + + let result = CommitBuilder::new(Arc::new(latest)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "tighten={tighten}: got {result:?}" + ); +} + +#[tokio::test] +async fn test_alter_columns_rejects_cast_with_tightening() { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + let err = dataset + .alter_columns(&[ColumnAlteration::new("value".into()) + .set_nullable(false) + .cast_to(DataType::Int64)]) + .await + .unwrap_err(); + assert!(err.to_string().contains("same call"), "got: {err}"); + + // Separately, both succeed. + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).cast_to(DataType::Int64)]) + .await + .unwrap(); + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).set_nullable(false)]) + .await + .unwrap(); +} + +/// A merge introducing a non-nullable column claims non-null, so a stale +/// append, whose fragments omit the column and would read as null, conflicts. +/// A nullable column keeps the long-standing behavior: the append commits and +/// its rows legally read as null. +#[rstest] +#[case::required_column_conflicts("1", true)] +#[case::nullable_column_commits("value", false)] +#[tokio::test] +async fn test_stale_append_vs_column_added_by_merge( + #[case] expression: &str, + #[case] expect_conflict: bool, +) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new(Dataset::write(reader, "memory://", None).await.unwrap()); + + let append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + + // A literal is non-nullable; a projection of a nullable column is nullable. + let mut latest = dataset.as_ref().clone(); + latest + .add_columns( + crate::dataset::NewColumnTransform::SqlExpressions(vec![( + "new_value".to_string(), + expression.to_string(), + )]), + None, + None, + ) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(latest)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "expression={expression}: got {result:?}" + ); + if let Ok(committed) = result { + committed.scan().try_into_batch().await.unwrap(); + } +} + +/// A cast rewrites the column under a new field id, so a stale append omits it +/// and its rows read as null. Casting a non-nullable column therefore claims +/// non-null and conflicts; casting a nullable one does not. +#[rstest] +#[case::cast_of_required_conflicts(true, true)] +#[case::cast_of_nullable_commits(false, false)] +#[tokio::test] +async fn test_stale_append_vs_cast(#[case] tighten_first: bool, #[case] expect_conflict: bool) { + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + if tighten_first { + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).set_nullable(false)]) + .await + .unwrap(); + } + + // Stage against the pre-cast schema, so the cast lands inside the window. + let staged = Arc::new(dataset.clone()); + let append = InsertBuilder::new(WriteDestination::Dataset(staged.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![batch]) + .await + .unwrap(); + + dataset + .alter_columns(&[ColumnAlteration::new("value".into()).cast_to(DataType::Int64)]) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(dataset)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "tighten_first={tighten_first}: got {result:?}" + ); + if let Ok(committed) = result { + committed.scan().try_into_batch().await.unwrap(); + } +} + +/// A nested cast assigns a new field id to the child. An append staged before +/// the cast still writes the old id, and transaction rebasing does not rewrite +/// its data through the cast. Under a required parent, accepting that append +/// would make the replacement child unreadable, so it must conflict. Supporting +/// that case requires smarter rebasing, not a file-format change. A nullable +/// parent can mask the missing child, so that append remains compatible. +#[rstest] +#[case::nullable_child_required_parent_conflicts(false, true)] +#[case::nullable_child_nullable_parent_commits(true, false)] +#[tokio::test] +async fn test_stale_append_vs_nested_cast( + #[case] parent_nullable: bool, + #[case] expect_conflict: bool, +) { + let child = Arc::new(ArrowField::new("c", DataType::Int32, true)); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![child.clone()])), + parent_nullable, + )])); + let struct_batch = |values: Vec| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StructArray::from(vec![( + child.clone(), + Arc::new(Int32Array::from(values)) as ArrayRef, + )]))], + ) + .unwrap() + }; + + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(struct_batch(vec![1, 2]))], schema.clone()), + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }), + ) + .await + .unwrap(); + + let append = InsertBuilder::new(WriteDestination::Dataset(Arc::new(dataset.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![struct_batch(vec![3])]) + .await + .unwrap(); + + dataset + .alter_columns(&[ColumnAlteration::new("b.c".into()).cast_to(DataType::Int64)]) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(dataset)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "parent_nullable={parent_nullable}: got {result:?}" + ); + if let Ok(committed) = result { + let batch = committed.scan().try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 3); + assert_eq!( + batch["b"] + .as_struct() + .column_by_name("c") + .unwrap() + .null_count(), + 1 + ); + } +} + +/// A subcolumn addition (V2.2+) merges a new child into an existing struct. +/// Stale rows supply the parent, so a required new child would read as +/// unmasked null: the merge claims and the stale append conflicts. A nullable +/// new child under a nullable parent masks itself and keeps appends flowing. +/// Under a non-nullable parent even a nullable child claims: the reader +/// synthesizes missing subcolumns against the column's declared nullability, +/// so the stale fragment could not be read at all. +#[rstest] +#[case::required_child_conflicts(true, false, true)] +#[case::required_child_nullable_parent_conflicts(true, true, true)] +#[case::nullable_child_nullable_parent_commits(false, true, false)] +#[case::nullable_child_required_parent_conflicts(false, false, true)] +#[tokio::test] +async fn test_stale_append_vs_sub_column_added_by_merge( + #[case] child_required: bool, + #[case] parent_nullable: bool, + #[case] expect_conflict: bool, +) { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![ArrowField::new( + "c", + DataType::Int32, + true, + )])), + parent_nullable, + )])); + let struct_batch = |values: Vec| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StructArray::from(vec![( + Arc::new(ArrowField::new("c", DataType::Int32, true)), + Arc::new(Int32Array::from(values)) as ArrayRef, + )]))], + ) + .unwrap() + }; + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(struct_batch(vec![1, 2]))], schema.clone()), + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let append = InsertBuilder::new(WriteDestination::Dataset(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(vec![struct_batch(vec![3])]) + .await + .unwrap(); + + let new_child = Arc::new(ArrowField::new("d", DataType::Int32, !child_required)); + let sub_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![new_child.as_ref().clone()])), + parent_nullable, + )])); + let sub_batch = RecordBatch::try_new( + sub_schema.clone(), + vec![Arc::new(StructArray::from(vec![( + new_child, + Arc::new(Int32Array::from(vec![10, 20])) as ArrayRef, + )]))], + ) + .unwrap(); + let mut latest = dataset.as_ref().clone(); + latest + .add_columns( + crate::dataset::NewColumnTransform::Reader(Box::new(RecordBatchIterator::new( + vec![Ok(sub_batch)], + sub_schema, + ))), + None, + None, + ) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(latest)).execute(append).await; + assert_eq!( + result.is_err(), + expect_conflict, + "child_required={child_required} parent_nullable={parent_nullable}: got {result:?}" + ); + if let Ok(committed) = result { + // The stale rows keep their parent values; the new child reads null. + let batch = committed.scan().try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 3); + let parent = batch["b"].as_struct(); + assert_eq!(parent.null_count(), 0); + assert_eq!(parent.column_by_name("d").unwrap().null_count(), 1); + } +} + +/// The barrier is operation-wide, so a tightening of a nested field conflicts +/// with a concurrent write exactly like a top-level one. +#[tokio::test] +async fn test_alter_columns_nested_tightening_conflicts() { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "b", + DataType::Struct(Fields::from(vec![ArrowField::new( + "c", + DataType::Int32, + true, + )])), + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StructArray::from(vec![( + Arc::new(ArrowField::new("c", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![1, 2])) as ArrayRef, + )]))], + ) + .unwrap(); + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone()), + "memory://", + None, + ) + .await + .unwrap(); + + // Leave the first handle a version behind, so the tightening commits stale. + InsertBuilder::new(WriteDestination::Dataset(Arc::new(dataset.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![batch]) + .await + .unwrap(); + + let mut stale = dataset; + stale + .alter_columns(&[ColumnAlteration::new("b.c".into()).set_nullable(false)]) + .await + .unwrap_err(); +} + +/// The invariant every piece of the tightening barrier serves: no interleaving +/// of honest writers and schema changes may commit a dataset that validates +/// but cannot be scanned. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_concurrent_tightening_stress() { + let dir = TempStrDir::default(); + let uri = dir.as_str().to_string(); + let batch = arrow_array::record_batch!(("value", Int32, [1, 2])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + Dataset::write(reader, uri.as_str(), None).await.unwrap(); + + let mut tasks = tokio::task::JoinSet::new(); + + // Appenders: honest read versions, half the batches carry nulls. A null + // append must either land while the column is nullable or be refused -- + // by the writer against a non-null schema, or by the claim barrier when a + // tightening won the race after the write. + for a in 0..4u8 { + let uri = uri.clone(); + tasks.spawn(async move { + let mut outcomes = [0u32; 2]; + for i in 0..12u32 { + let with_null = (a as u32 + i).is_multiple_of(2); + let batch = if with_null { + arrow_array::record_batch!(("value", Int32, [None, Some(3)])).unwrap() + } else { + arrow_array::record_batch!(("value", Int32, [4, 5])).unwrap() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let result = Dataset::write( + reader, + uri.as_str(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await; + outcomes[result.is_ok() as usize] += 1; + } + outcomes + }); + } + + // The tightener alternates NOT NULL and back. Either step may lose to + // concurrent writes; losing is an acceptable outcome, corruption is not. + { + let uri = uri.clone(); + tasks.spawn(async move { + let mut outcomes = [0u32; 2]; + for i in 0..10u32 { + let Ok(mut dataset) = DatasetBuilder::from_uri(uri.as_str()).load().await else { + continue; + }; + let result = dataset + .alter_columns( + &[ColumnAlteration::new("value".into()).set_nullable(i % 2 == 1)], + ) + .await; + outcomes[result.is_ok() as usize] += 1; + } + outcomes + }); + } + + let mut totals = [0u32; 2]; + while let Some(res) = tasks.join_next().await { + let [err, ok] = res.unwrap(); + totals[0] += err; + totals[1] += ok; + } + + // The oracle: whatever interleaving happened, the final dataset must be + // internally consistent -- validation and scanning agree. + let dataset = DatasetBuilder::from_uri(uri.as_str()).load().await.unwrap(); + dataset.validate().await.unwrap(); + let scanned = dataset.scan().try_into_batch().await.unwrap(); + assert!(scanned.num_rows() >= 2); + // And every historical version must scan too: a corrupt intermediate + // commit would have been the bug even if later commits papered over it. + for version in 1..=dataset.version().version { + let at = dataset.checkout_version(version).await.unwrap(); + at.validate().await.unwrap(); + at.scan().try_into_batch().await.unwrap(); + } + assert!(totals[1] > 0, "nothing succeeded: {totals:?}"); +} diff --git a/rust/lance/src/dataset/tests/dataset_migrations.rs b/rust/lance/src/dataset/tests/dataset_migrations.rs index d71a65bfa69..60f735a4d2d 100644 --- a/rust/lance/src/dataset/tests/dataset_migrations.rs +++ b/rust/lance/src/dataset/tests/dataset_migrations.rs @@ -6,16 +6,19 @@ use std::vec; use crate::dataset::InsertBuilder; use crate::dataset::optimize::{CompactionOptions, compact_files}; +use crate::index::DatasetIndexExt; use crate::utils::test::copy_test_data_to_tmp; use crate::{Dataset, Result}; -use lance_table::format::IndexMetadata; +use lance_index::{IndexCriteria, IndexType, scalar::ScalarIndexParams}; +use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; +use lance_table::format::{Fragment, IndexMetadata, RowIdMeta}; +use lance_table::rowids::read_row_ids; use crate::dataset::write::{WriteMode, WriteParams}; -use crate::index::DatasetIndexExt; use arrow::compute::concat_batches; use arrow_array::RecordBatch; -use arrow_array::{Float32Array, Int64Array, RecordBatchIterator}; -use arrow_schema::Schema as ArrowSchema; +use arrow_array::{Array, Float32Array, Int64Array, ListArray, RecordBatchIterator, UInt32Array}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use lance_file::version::LanceFileVersion; use futures::{StreamExt, TryStreamExt}; @@ -278,6 +281,43 @@ async fn test_v0_8_14_invalid_index_fragment_bitmap( assert_eq!(row_count, 1900); } +/// The repair above is triggered by the writer version of the manifest being +/// committed *from*, and a successful commit stamps the current one. So a +/// commit that cannot open the index has exactly one chance at the corrupt +/// bitmap, and carrying it through unverified would hand a later build a +/// bitmap that looks migrated. +#[tokio::test] +async fn test_v0_8_14_invalid_index_fragment_bitmap_repair_is_not_lost() { + let test_dir = copy_test_data_to_tmp("v0.8.14/corrupt_index").unwrap(); + let test_uri = test_dir.path_str(); + + let indices_dir = test_dir.std_path().join("_indices"); + let stashed_dir = test_dir.std_path().join("_indices_stashed"); + std::fs::rename(&indices_dir, &stashed_dir).unwrap(); + + let mut dataset = Dataset::open(&test_uri).await.unwrap(); + dataset.delete("false").await.unwrap(); + + for idx in dataset.load_indices().await.unwrap().iter() { + assert_eq!( + idx.fragment_bitmap, None, + "a bitmap the migration could not verify must be recorded as unknown" + ); + } + + std::fs::rename(&stashed_dir, &indices_dir).unwrap(); + + let mut dataset = Dataset::open(&test_uri).await.unwrap(); + dataset.delete("false").await.unwrap(); + + for idx in dataset.load_indices().await.unwrap().iter() { + assert!( + idx.fragment_bitmap.as_ref().unwrap().contains(0), + "the first build that can open the index must repair the coverage" + ); + } +} + #[tokio::test] async fn test_fix_v0_10_5_corrupt_schema() { // Schemas could be corrupted by successive calls to `add_columns` and @@ -351,6 +391,97 @@ async fn test_fix_v0_21_0_corrupt_fragment_bitmap() { assert_eq!(get_bitmap(&indices[1]), vec![1]); } +/// Unlike the pre-0.8.15 trigger, an overlap is re-derived from the index +/// metadata on every commit, so it asks to be recalculated again on its own. A +/// commit that cannot open the index has nothing to preserve and must leave the +/// coverage alone: `None` is a state modern indices are not built to recover +/// from, since `calculate_included_frags` exists only for old manifests. +#[tokio::test] +async fn test_v0_21_0_corrupt_fragment_bitmap_kept_when_index_cannot_be_opened() { + let test_dir = copy_test_data_to_tmp("v0.21.0/bad_index_fragment_bitmap").unwrap(); + let test_uri = test_dir.path_str(); + + std::fs::rename( + test_dir.std_path().join("_indices"), + test_dir.std_path().join("_indices_stashed"), + ) + .unwrap(); + + fn coverage(indices: &[IndexMetadata]) -> Vec<(String, Option>)> { + let mut coverage = indices + .iter() + .map(|idx| { + ( + idx.uuid.to_string(), + idx.fragment_bitmap + .as_ref() + .map(|bitmap| bitmap.iter().collect()), + ) + }) + .collect::>(); + coverage.sort(); + coverage + } + + let mut dataset = Dataset::open(&test_uri).await.unwrap(); + let before = coverage(&dataset.load_indices().await.unwrap()); + + dataset.delete("false").await.unwrap(); + + assert_eq!( + coverage(&dataset.load_indices().await.unwrap()), + before, + "coverage the overlap check will ask about again must be left as it stands" + ); +} + +#[tokio::test] +async fn test_v8_decimal_zonemap_missing_extrema() { + async fn query_ids( + dataset: &Dataset, + predicate: &str, + use_scalar_index: bool, + ) -> (String, Vec) { + let mut scan = dataset.scan(); + scan.project(&["id"]) + .unwrap() + .use_scalar_index(use_scalar_index) + .filter(predicate) + .unwrap(); + let plan = scan.explain_plan(false).await.unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + let ids = batch["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + (plan, ids) + } + + let test_dir = copy_test_data_to_tmp("v8.0.0/decimal_zonemap").unwrap(); + let dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + + for predicate in [ + "value = arrow_cast(2.00, 'Decimal128(10, 2)')", + "value >= arrow_cast(2.00, 'Decimal128(10, 2)') AND \ + value < arrow_cast(3.00, 'Decimal128(10, 2)')", + "value IN (arrow_cast(2.00, 'Decimal128(10, 2)'), \ + arrow_cast(4.00, 'Decimal128(10, 2)'))", + ] { + let (indexed_plan, indexed_ids) = query_ids(&dataset, predicate, true).await; + let (flat_plan, flat_ids) = query_ids(&dataset, predicate, false).await; + + assert!(indexed_plan.contains("ScalarIndexQuery"), "{indexed_plan}"); + assert!(!flat_plan.contains("ScalarIndexQuery"), "{flat_plan}"); + assert_eq!( + indexed_ids, flat_ids, + "indexed query diverged for {predicate}" + ); + assert_eq!(flat_ids, vec![2]); + } +} + #[tokio::test] async fn test_max_fragment_id_migration() { // v0.5.9 and earlier did not store the max fragment id in the manifest. @@ -398,6 +529,27 @@ async fn test_index_without_file_sizes() { index.files.is_none() || index.files.as_ref().unwrap().is_empty(), "Index should not have file size info (created with old version)" ); + // A manifest predating `covering_fields` decodes it as empty, so the keyed + // prefix is the whole of `fields` -- exactly what selection assumed before + // covering existed. + assert!( + index.covering_fields.is_empty(), + "Index from old version should declare no covered columns" + ); + + // Selection derives the keyed count as `fields.len() - covering_fields.len()`. + // On this old metadata it must still resolve to the one indexed column; + // otherwise the filter below would quietly fall back to a full scan and + // still return the right row. + let selected = dataset + .load_scalar_index(IndexCriteria::default().for_column("values")) + .await + .unwrap(); + assert_eq!( + selected.map(|idx| idx.name), + Some("values_idx".to_string()), + "an old single-field index must still be selected for its column" + ); // Verify the index still works - scan with a filter that uses the index let batch = dataset @@ -511,3 +663,333 @@ async fn test_list_struct_field_reorder_issue_5702() { // Verify schema has expected columns assert_eq!(batch.schema().fields().len(), 3); // id, data, extra } + +/// Regression test for issue #6936: v6.0.1 truncated a miniblock's structural +/// level count to u16 while retaining the complete RLE payload. +#[tokio::test] +async fn test_v6_0_1_miniblock_level_count_overflow() { + let test_dir = copy_test_data_to_tmp("v6.0.1/miniblock_level_count_overflow.lance").unwrap(); + let test_uri = test_dir.path_str(); + let dataset = Dataset::open(&test_uri).await.unwrap(); + + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!(batch.num_rows(), 66_049); + + let captions = batch["captions"] + .as_any() + .downcast_ref::() + .unwrap(); + let values = captions + .values() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.len(), 16_416); + assert_eq!(values.values().as_ref(), (0..16_416).collect::>()); + + let offsets = captions.value_offsets(); + assert_eq!(offsets[513], 16_416); + assert!(offsets[513..].iter().all(|offset| *offset == 16_416)); + assert_eq!(captions.null_count(), 32_768); + assert!(captions.is_valid(513)); + assert_eq!(captions.value(513).len(), 0); + assert!(captions.is_null(514)); + assert!(captions.is_valid(66_047)); + assert_eq!(captions.value(66_047).len(), 0); + assert!(captions.is_null(66_048)); +} + +// Helper: create a simple dataset with one fragment of `n` rows at the given URI. +async fn make_simple_dataset(uri: &str, n: i64) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(0..n))], + ) + .unwrap(); + Dataset::write(RecordBatchIterator::new(vec![Ok(batch)], schema), uri, None) + .await + .unwrap() +} + +#[tokio::test] +async fn test_migrate_to_stable_row_ids_basic() { + // Create a dataset without stable row IDs (the default). + let mut dataset = make_simple_dataset("memory://migrate_basic", 10).await; + assert!( + !dataset.manifest.uses_stable_row_ids(), + "should not have stable row IDs yet" + ); + + // Append a second batch using InsertBuilder so we share the same object store. + let schema = Arc::new(ArrowSchema::from(dataset.schema())); + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(10..20))], + ) + .unwrap(); + dataset = InsertBuilder::new(Arc::new(dataset)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![batch2]) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + // Run the migration. + dataset.migrate_to_stable_row_ids().await.unwrap(); + + // FLAG_STABLE_ROW_IDS must be set in both reader and writer flags. + assert_ne!( + dataset.manifest.reader_feature_flags & FLAG_STABLE_ROW_IDS, + 0, + "reader_feature_flags should have FLAG_STABLE_ROW_IDS" + ); + assert_ne!( + dataset.manifest.writer_feature_flags & FLAG_STABLE_ROW_IDS, + 0, + "writer_feature_flags should have FLAG_STABLE_ROW_IDS" + ); + assert!(dataset.manifest.uses_stable_row_ids()); + + // All fragments must have row_id_meta set. + for frag in dataset.manifest.fragments.iter() { + assert!( + frag.row_id_meta.is_some(), + "fragment {} should have row_id_meta after migration", + frag.id + ); + } + + // next_row_id should equal the total number of physical rows (10 + 10 = 20). + assert_eq!(dataset.manifest.next_row_id, 20); + + // Appending after migration should correctly assign row IDs from next_row_id. + let batch3 = RecordBatch::try_new( + Arc::new(ArrowSchema::from(dataset.schema())), + vec![Arc::new(Int64Array::from_iter_values(20..25))], + ) + .unwrap(); + let dataset_after_append = InsertBuilder::new(Arc::new(dataset.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![batch3]) + .await + .unwrap(); + + // The new fragment should also have row_id_meta. + let new_frag = dataset_after_append.manifest.fragments.last().unwrap(); + assert!( + new_frag.row_id_meta.is_some(), + "new fragment after migration should have row_id_meta" + ); + // next_row_id should have advanced by the 5 newly appended rows. + assert_eq!(dataset_after_append.manifest.next_row_id, 25); + + dataset.validate().await.unwrap(); +} + +#[tokio::test] +async fn test_migrate_to_stable_row_ids_already_migrated() { + // Create a dataset that already uses stable row IDs. + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(0..5))], + ) + .unwrap(); + let write_params = WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + "memory://already_migrated", + Some(write_params), + ) + .await + .unwrap(); + + assert!(dataset.manifest.uses_stable_row_ids()); + let version_before = dataset.manifest.version; + + // Calling migrate on an already-migrated dataset should be a no-op. + dataset.migrate_to_stable_row_ids().await.unwrap(); + + // Version must not have changed. + assert_eq!( + dataset.manifest.version, version_before, + "migrate should be a no-op when already migrated" + ); + assert!(dataset.manifest.uses_stable_row_ids()); +} + +#[tokio::test] +async fn test_migrate_to_stable_row_ids_empty() { + // Create an empty dataset (schema-only, no fragments). + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "id", + DataType::Int64, + false, + )])); + let empty_reader = RecordBatchIterator::new( + std::iter::empty::>(), + schema.clone(), + ); + let mut dataset = Dataset::write(empty_reader, "memory://migrate_empty", None) + .await + .unwrap(); + + assert!(!dataset.manifest.uses_stable_row_ids()); + assert_eq!(dataset.get_fragments().len(), 0); + + // Migration on an empty dataset should succeed without error. + dataset.migrate_to_stable_row_ids().await.unwrap(); + + assert!(dataset.manifest.uses_stable_row_ids()); + assert_eq!(dataset.manifest.next_row_id, 0); + dataset.validate().await.unwrap(); +} + +#[tokio::test] +async fn test_migrate_to_stable_row_ids_with_deletions() { + // Create a single-fragment dataset of 10 rows then soft-delete 3 of them. + let mut dataset = make_simple_dataset("memory://migrate_deletions", 10).await; + dataset.delete("id < 3").await.unwrap(); + + assert_eq!(dataset.count_rows(None).await.unwrap(), 7); + assert_eq!(dataset.count_deleted_rows().await.unwrap(), 3); + + // physical_rows counts the pre-deletion slots; row IDs must cover all of + // them so that the deleted rows' IDs are never reused. + let physical_rows = dataset.get_fragments()[0].metadata.physical_rows.unwrap(); + assert_eq!(physical_rows, 10); + + dataset.migrate_to_stable_row_ids().await.unwrap(); + + assert!(dataset.manifest.uses_stable_row_ids()); + assert!(dataset.manifest.fragments[0].row_id_meta.is_some()); + + // next_row_id must equal physical_rows (10), not logical rows (7). + assert_eq!(dataset.manifest.next_row_id, 10); + + dataset.validate().await.unwrap(); +} + +#[tokio::test] +async fn test_migrate_to_stable_row_ids_blocked_by_index() { + // Create a 2-fragment dataset and build a BTree index on it. + let mut dataset = make_simple_dataset("memory://btree_blocked", 10).await; + let schema = Arc::new(ArrowSchema::from(dataset.schema())); + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from_iter_values(10..20))], + ) + .unwrap(); + dataset = InsertBuilder::new(Arc::new(dataset)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![batch2]) + .await + .unwrap(); + + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("my_btree".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Migration must be rejected because the BTree index exists. + let err = dataset + .migrate_to_stable_row_ids() + .await + .expect_err("migration should fail when indexes exist"); + + assert!( + err.to_string().contains("my_btree"), + "error should name the blocking index, got: {err}" + ); + + // After dropping the index the migration succeeds. + dataset.drop_index("my_btree").await.unwrap(); + dataset.migrate_to_stable_row_ids().await.unwrap(); + assert!(dataset.manifest.uses_stable_row_ids()); + + // Re-create the index and verify it works correctly. + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + let results = dataset + .scan() + .filter("id = 15") + .unwrap() + .try_into_batch() + .await + .unwrap(); + + assert_eq!(results.num_rows(), 1); + let id_col = results["id"].as_any().downcast_ref::().unwrap(); + assert_eq!(id_col.value(0), 15); +} + +/// The migration numbers from the mark it is handed, so any manifest carrying a +/// non-zero one cannot reissue ids the earlier versions still hold. +#[rstest] +#[case::fresh(0, vec![0..4, 4..10])] +#[case::carried_mark(30, vec![30..34, 34..40])] +fn test_migration_allocates_from_the_given_mark( + #[case] start: u64, + #[case] expected: Vec>, +) { + let mut fragments: Vec = [4usize, 6] + .iter() + .enumerate() + .map(|(i, rows)| { + let mut f = Fragment::new(i as u64); + f.physical_rows = Some(*rows); + f + }) + .collect(); + + let next = Dataset::assign_stable_row_ids_for_migration(&mut fragments, start).unwrap(); + assert_eq!(next, expected.last().unwrap().end); + + let sequences: Vec> = fragments + .iter() + .map(|f| { + let RowIdMeta::Inline(data) = f.row_id_meta.as_ref().unwrap() else { + panic!("migration writes inline row id meta"); + }; + read_row_ids(data).unwrap().iter().collect() + }) + .collect(); + let expected: Vec> = expected.into_iter().map(|r| r.collect()).collect(); + assert_eq!(sequences, expected); +} diff --git a/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs b/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs new file mode 100644 index 00000000000..6695940ccfb --- /dev/null +++ b/rust/lance/src/dataset/tests/dataset_overlay_index_masking.rs @@ -0,0 +1,2347 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! End-to-end tests for data-overlay index masking: a scalar index masks data overlay files so that +//! queries stay correct while overlays remain (stale index hits are dropped and new +//! matches are added by re-evaluating overlay-covered rows on the flat path). + +use std::sync::Arc; + +use futures::TryStreamExt; + +use arrow_array::builder::{ListBuilder, StringBuilder}; +use arrow_array::cast::AsArray; +use arrow_array::types::Int32Type; +use arrow_array::{ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use lance_index::IndexType; +use lance_index::optimize::OptimizeOptions; +use lance_index::scalar::BuiltinIndexType; +use lance_index::scalar::FullTextSearchQuery; +use lance_index::scalar::ScalarIndexParams; +use lance_index::scalar::inverted::query::{ + BooleanQuery, FtsQuery, MatchQuery, MultiMatchQuery, Occur, PhraseQuery, +}; +use lance_index::scalar::inverted::{DocumentGranularity, InvertedIndexParams}; +use lance_io::utils::CachedFileSize; +use lance_linalg::distance::MetricType; +use lance_table::format::DataFile; +use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; +use roaring::RoaringBitmap; +use rstest::rstest; + +use lance_file::writer::FileWriterOptions; + +use crate::Dataset; +use crate::dataset::optimize::{CompactionOptions, compact_files, remapping}; +use crate::dataset::transaction::{DataOverlayGroup, Operation}; +use crate::dataset::{WriteDestination, WriteParams}; +use crate::index::vector::VectorIndexParams; +use crate::index::{CreateIndexBuilder, DatasetIndexExt}; +use crate::io::exec::filtered_read::FilteredReadExec; +use crate::io::exec::fts::FlatMatchQueryExec; + +/// Two-fragment Int32 dataset: `id` (field 0) = 0..12 and `age` (field 1) = id * 10, +/// six rows per file (fragments 0 and 1). In-memory store so overlay files can be written +/// with a store-relative `data/.lance` path and committed against the dataset. +async fn create_base_dataset() -> Dataset { + create_base_dataset_with(false).await +} + +async fn create_base_dataset_with(stable_row_ids: bool) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("age", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(Int32Array::from_iter_values((0..12).map(|v| v * 10))), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap() +} + +async fn build_age_index(dataset: &mut Dataset) { + dataset + .create_index( + &["age"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); +} + +/// Write an overlay file covering `fields` of `fragment_id` with `coverage` and the given +/// per-field value columns, then commit it as a `DataOverlay` transaction. `name` makes +/// the overlay file unique. +async fn commit_overlay( + dataset: Dataset, + name: &str, + fragment_id: u64, + fields: &[i32], + coverage: OverlayCoverage, + columns: Vec, +) -> Dataset { + let read_version = dataset.version().version; + let overlay_schema = dataset.schema().project_by_ids(fields, true); + + let filename = format!("{name}.lance"); + // Use dataset.base so the path is absolute for file:// stores. + // to_local_path() prepends '/' to the object_store path, so a bare + // "data/foo.lance" would resolve to /data/foo.lance (root fs). With + // base we get e.g. tmp/lance-bench/data/foo.lance → /tmp/lance-bench/data/foo.lance. + // For memory:// stores base is empty so the result is the same as before. + let path = dataset.base.clone().join("data").join(filename.as_str()); + let obj_writer = dataset.object_store.create(&path).await.unwrap(); + let file_version = dataset.manifest.data_storage_format.lance_file_format(); + let mut writer = lance_file::versions::create_writer( + file_version, + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + + for (i, array) in columns.into_iter().enumerate() { + writer.write_column(i, array).await.unwrap(); + } + let summary = writer.finish().await.unwrap(); + + let mut data_file = DataFile::new_unstarted(filename, file_version); + data_file.fields = writer + .field_id_to_column_indices() + .iter() + .map(|(field_id, _)| *field_id as i32) + .collect::>() + .into(); + data_file.column_indices = writer + .field_id_to_column_indices() + .iter() + .map(|(_, column_index)| *column_index as i32) + .collect::>() + .into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + + let overlay = DataOverlayFile { + data_file, + coverage, + committed_version: 0, + }; + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset)), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![overlay], + }], + }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await + .unwrap() +} + +/// Sorted `id` values returned by a filtered scan. +async fn ids_matching(dataset: &Dataset, filter: &str) -> Vec { + ids_matching_opts(dataset, filter, false).await +} + +/// Like [`ids_matching`] but lets a test enable `fast_search()`, which skips unindexed +/// fragments. Overlay masking on indexed fragments must still apply regardless. +async fn ids_matching_opts(dataset: &Dataset, filter: &str, fast_search: bool) -> Vec { + let mut scanner = dataset.scan(); + scanner.filter(filter).unwrap().project(&["id"]).unwrap(); + if fast_search { + scanner.fast_search(); + } + let batch = scanner.try_into_batch().await.unwrap(); + let mut ids = ids_from_batches(std::slice::from_ref(&batch)); + ids.sort_unstable(); + ids +} + +/// Concatenate the `id` (Int32) column from each batch, in batch order. +fn ids_from_batches(batches: &[RecordBatch]) -> Vec { + batches + .iter() + .flat_map(|b| { + b.column_by_name("id") + .unwrap() + .as_primitive::() + .values() + .to_vec() + }) + .collect() +} + +fn i32_array(values: impl IntoIterator>) -> ArrayRef { + Arc::new(Int32Array::from_iter(values)) +} + +fn string_lists(rows: &[&[&str]]) -> ArrayRef { + let mut builder = ListBuilder::new(StringBuilder::new()); + for row in rows { + for value in *row { + builder.values().append_value(value); + } + builder.append(true); + } + Arc::new(builder.finish()) +} + +fn fsl(rows: Vec>, dim: i32) -> ArrayRef { + let flat: Vec = rows.into_iter().flatten().collect(); + let item = Arc::new(ArrowField::new("item", DataType::Float32, true)); + Arc::new( + arrow_array::FixedSizeListArray::try_new( + item, + dim, + Arc::new(arrow_array::Float32Array::from(flat)), + None, + ) + .unwrap(), + ) +} + +/// A newer overlay on the indexed field drops stale index hits (the old value no longer +/// matches) and surfaces new matches (the new value is found even though the index never +/// saw it). Mirrors the spec's Bob 25 -> 26 worked example. +/// +/// Parametrized over `stable_row_ids` to cover the address-based stale-Take path under both +/// row-id schemes. +#[rstest] +#[tokio::test] +async fn test_overlay_stale_drop_and_new_match(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Fragment 0, offset 1 is id=1, age=10. The overlay (committed after the index) + // changes its age to 999. + let dataset = commit_overlay( + dataset, + "age_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Stale-drop: the index still holds age=10 for id=1, but its current value is 999, + // so it must not be returned. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + // New-match: the index never saw age=999, but re-evaluation finds it. + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + // An untouched indexed value is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); +} + +/// Row-level BTree precision: when one row in a covered fragment is stale, only that row is +/// blocked from the index result and re-evaluated on the stale-Take path. Non-stale rows in +/// the same fragment (including one that matches the predicate) remain on the indexed path. +/// +/// Setup: fragment 0 has id=5 → age=50 (not stale). Overlay id=1 → age=50 (stale). +/// After the overlay two rows in fragment 0 have age=50. The row-level optimization must +/// return both: id=5 from the index and id=1 from the stale-Take path. +/// +/// Parametrized over `stable_row_ids`: with stable row ids enabled the stale-Take path must +/// identify rows by physical address, not `_rowid`, or it would take the wrong rows. +#[rstest] +#[tokio::test] +async fn test_btree_overlay_row_level_precision(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Fragment 0: ids 0-5, ages 0,10,20,30,40,50. Overlay offset 1 (id=1): age 10→50. + // After this both id=1 and id=5 have age=50, in the same fragment. + let dataset = commit_overlay( + dataset, + "age_row_level", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(50)])], + ) + .await; + + // Stale drop: id=1's old age=10 entry must not appear. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + + // id=5 via index + id=1 via stale-Take path — both in fragment 0. + assert_eq!(ids_matching(&dataset, "age = 50").await, vec![1, 5]); + + // Non-stale rows in the same fragment still return correctly. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); + assert_eq!(ids_matching(&dataset, "age = 30").await, vec![3]); +} + +/// `fast_search` skips *unindexed fragments*, but overlay masking on indexed fragments must +/// still apply: the drop-stale block and the stale-Take re-eval both run regardless of +/// `fast_search` on the scalar path. A regression that gated overlay masking behind +/// `!fast_search` would leak id=1's stale age=10 hit here. +#[tokio::test] +async fn test_btree_overlay_masked_under_fast_search() { + let mut dataset = create_base_dataset().await; + build_age_index(&mut dataset).await; + + // Fragment 0, offset 1 is id=1, age=10. Overlay (committed after the index) → age=999. + let dataset = commit_overlay( + dataset, + "age_fast_search", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Stale hit dropped even under fast_search — the block is not gated by fast_search. + assert_eq!( + ids_matching_opts(&dataset, "age = 10", true).await, + Vec::::new() + ); + // The scalar re-eval path is likewise not gated, so the new value is still surfaced. + assert_eq!( + ids_matching_opts(&dataset, "age = 999", true).await, + vec![1] + ); + // An untouched indexed value on the same fragment is unaffected. + assert_eq!(ids_matching_opts(&dataset, "age = 20", true).await, vec![2]); +} + +/// An overlay touching only a non-indexed field excludes nothing from the index on `age`. +#[tokio::test] +async fn test_overlay_on_unrelated_field_excludes_nothing() { + let mut dataset = create_base_dataset().await; + build_age_index(&mut dataset).await; + + // Overlay field 0 (`id`), not the indexed `age`. The age index stays fully trusted. + let dataset = commit_overlay( + dataset, + "id_overlay", + 0, + &[0], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(777)])], + ) + .await; + + // The age index is still trusted: age=10 finds the offset-1 row, whose id now reads + // through the overlay as 777. The fragment was not routed to the flat path on account + // of an overlay that touches no indexed field. + assert_eq!(ids_matching(&dataset, "age = 10").await, vec![777]); + // An untouched row is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); + // The overlaid id is the new value on read, and the old one is gone. + assert_eq!(ids_matching(&dataset, "id = 777").await, vec![777]); + assert_eq!(ids_matching(&dataset, "id = 1").await, Vec::::new()); +} + +/// An overlay whose `committed_version <= index.dataset_version` is already incorporated by +/// the index (the index was built reading merged values) and is not excluded. +#[tokio::test] +async fn test_overlay_older_than_index_not_excluded() { + let dataset = create_base_dataset().await; + + // Commit the overlay first (age of id=1 becomes 999), then build the index on top. + let mut dataset = commit_overlay( + dataset, + "age_overlay_old", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + build_age_index(&mut dataset).await; + + // The index incorporates the overlay, so it returns the merged value directly. + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); +} + +/// A covered offset whose overlay value is NULL overrides the cell to NULL, so the stale +/// index hit for its old value is dropped. +#[tokio::test] +async fn test_overlay_null_override() { + let mut dataset = create_base_dataset().await; + build_age_index(&mut dataset).await; + + // id=1 (age=10) is overridden to NULL. + let dataset = commit_overlay( + dataset, + "age_overlay_null", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([None])], + ) + .await; + + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age IS NULL").await, vec![1]); +} + +/// Overlays on a non-first fragment are masked correctly, and a query spanning both +/// fragments returns the right rows. +/// +/// Parametrized over `stable_row_ids`, and crucially overlays fragment 1 (ids 6..12), where a +/// physical address diverges from the stable row id — so this exercises the address-vs-row-id +/// distinction that a fragment-0 overlay cannot. +#[rstest] +#[tokio::test] +async fn test_overlay_multi_fragment(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Fragment 1 holds ids 6..12 (ages 60..110). Offset 2 within fragment 1 is id=8, + // age=80; change it to 60 (a value that also legitimately exists at id=6). + let dataset = commit_overlay( + dataset, + "age_overlay_frag1", + 1, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([2])), + vec![i32_array([Some(60)])], + ) + .await; + + // id=8 no longer has age=80 (stale-drop on fragment 1). + assert_eq!(ids_matching(&dataset, "age = 80").await, Vec::::new()); + // Both id=6 (base) and id=8 (overlay) now have age=60 (new-match added to base hit). + assert_eq!(ids_matching(&dataset, "age = 60").await, vec![6, 8]); + // A value in the untouched fragment 0 is still served correctly. + assert_eq!(ids_matching(&dataset, "age = 30").await, vec![3]); +} + +/// A deletion below an overlaid row must not corrupt the physical-offset → stable-row-id +/// translation used to build the overlay block mask. +/// +/// Under stable row ids the stale-row block/take set is computed by mapping each stale +/// *physical offset* to its stable row id via the fragment's `RowIdSequence`. The sequence +/// keeps one entry per physical row (deleted rows are tracked separately by the deletion +/// vector, not compacted out), so the correct mapping is `sequence.get(offset)`. A regression +/// that instead advanced a `sequence.iter()` cursor only for non-deleted offsets desynced the +/// cursor after any deletion at an offset *below* the stale one, blocking/taking the wrong row +/// id: the stale index hit then leaked and the new value was never surfaced. +/// +/// Setup (stable row ids): fragment 1 holds ids 6..12 at offsets 0..6. Delete id=6 (offset 0), +/// then overlay offset 2 (id=8, age 80 → 999). The deletion at offset 0 sits below the stale +/// offset 2, so a cursor-based translation would map offset 2 to id=7 instead of id=8. +/// +/// Parametrized over `stable_row_ids`: only the stable-row-id path translates offsets to row +/// ids, so the bug is specific to it; the non-stable case (addresses are row ids) is a control. +#[rstest] +#[tokio::test] +async fn test_btree_overlay_stale_row_with_prior_deletion( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_base_dataset_with(stable_row_ids).await; + build_age_index(&mut dataset).await; + + // Delete id=6 (fragment 1, offset 0) — a deletion hole below the row the overlay marks stale. + dataset.delete("id = 6").await.unwrap(); + + // Fragment 1, offset 2 is id=8 (age 80). The overlay (committed after the index) → age 999. + let dataset = commit_overlay( + dataset, + "age_overlay_del", + 1, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([2])), + vec![i32_array([Some(999)])], + ) + .await; + + // Stale-drop: id=8's old age=80 index entry must not be returned. + assert_eq!(ids_matching(&dataset, "age = 80").await, Vec::::new()); + // New-match: id=8's current age=999 is found by re-evaluating the stale row. + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![8]); + // A non-stale row in the same deletion-bearing fragment is still served by the index. + assert_eq!(ids_matching(&dataset, "age = 70").await, vec![7]); + // The deleted row is gone. + assert_eq!(ids_matching(&dataset, "age = 60").await, Vec::::new()); +} + +const VEC_DIM: i32 = 8; + +fn vec_query() -> Vec { + vec![1.0_f32, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +} + +/// 64-row two-fragment vector dataset with a single-partition IVF_FLAT index and no overlay. +/// id=35 equals the query; every other base vector is orthogonal to and far from the query. +async fn create_vector_index_dataset(stable_row_ids: bool) -> Dataset { + let query = vec_query(); + + let mut vectors: Vec> = Vec::with_capacity(64); + for i in 0..64 { + if i == 35 { + vectors.push(query.clone()); + } else { + let mut v = vec![0.0_f32; VEC_DIM as usize]; + v[1] = (i + 2) as f32; // orthogonal to the query, distinct, far + vectors.push(v); + } + } + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + VEC_DIM, + ), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..64)), + fsl(vectors, VEC_DIM), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 32, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap(); + + // Single-partition IVF_FLAT: the ANN searches every indexed row with exact distances. + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); + dataset + .create_index(&["vec"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + dataset +} + +/// [`create_vector_index_dataset`] plus an overlay on fragment 1 that moves id=35 (offset 3) +/// onto `far` (away from the query) and id=40 (offset 8) onto the query. Built before the +/// overlay, the index still believes id=35 is the query and has never seen id=40 near it. +/// +/// Overlaying fragment 1 (ids 32..64) is deliberate: a physical address diverges from the +/// stable row id there, so both the ANN prefilter block and the flat re-score take must operate +/// in the row-id domain when `stable_row_ids` is enabled. +async fn create_vector_overlay_dataset(stable_row_ids: bool) -> Dataset { + let query = vec_query(); + let far = vec![0.0_f32, 100.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; + let dataset = create_vector_index_dataset(stable_row_ids).await; + + commit_overlay( + dataset, + "vec_overlay", + 1, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([3, 8])), + vec![fsl(vec![far, query], VEC_DIM)], + ) + .await +} + +/// Run a top-`k` ANN search for the standard query vector and return the returned `id`s, +/// optionally with `fast_search()` enabled. +async fn vector_query_ids(dataset: &Dataset, k: usize, fast_search: bool) -> Vec { + let mut scanner = dataset.scan(); + scanner + .nearest("vec", &arrow_array::Float32Array::from(vec_query()), k) + .unwrap() + .minimum_nprobes(1) + .project(&["id"]) + .unwrap(); + if fast_search { + scanner.fast_search(); + } + let results = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + ids_from_batches(&results) +} + +/// A vector index masks overlays: a row whose vector was moved (by a newer overlay) away +/// from the query is dropped from results, and a row moved *onto* the query is found by +/// re-scoring its current vector on the flat path — even though the index never saw it. +/// +/// Parametrized over `stable_row_ids` to cover the row-id domain for both block and re-score. +#[rstest] +#[tokio::test] +async fn test_vector_index_rescore_on_overlay(#[values(false, true)] stable_row_ids: bool) { + let dataset = create_vector_overlay_dataset(stable_row_ids).await; + let ids = vector_query_ids(&dataset, 3, false).await; + + // id=40 was moved onto the query and is found by re-scoring (new-match recall). + assert!( + ids.contains(&40), + "expected id=40 (re-scored to query) in {ids:?}" + ); + // id=35's stale index entry (the query) must not resurface: its current vector is far. + assert!( + !ids.contains(&35), + "stale vector for id=35 should be dropped, got {ids:?}" + ); +} + +/// The ANN prefilter block that drops stale overlay rows runs regardless of `fast_search`; +/// only the flat re-score is gated by it. So under `fast_search` id=35's stale hit must still +/// be dropped, while id=40 (moved onto the query) is intentionally not re-scored — the same +/// recall tradeoff `fast_search` already makes for unindexed data. A regression that moved the +/// `overlay_block` computation inside the `!fast_search` guard would leak id=35's stale vector. +#[tokio::test] +async fn test_vector_overlay_stale_dropped_under_fast_search() { + let dataset = create_vector_overlay_dataset(false).await; + let ids = vector_query_ids(&dataset, 3, true).await; + + // Correctness: the stale index hit is dropped even though the re-score is skipped. + assert!( + !ids.contains(&35), + "stale vector for id=35 must be dropped under fast_search, got {ids:?}" + ); + // Recall tradeoff: fast_search skips the flat re-score, so the moved-on match is not surfaced. + assert!( + !ids.contains(&40), + "fast_search skips re-score, so id=40 should be absent, got {ids:?}" + ); +} + +/// A batch (multi-vector) nearest query must fall back to the per-query indexed loop when a +/// data overlay makes indexed vector rows stale. The shared-scan batch node (`ANNIvfBatch`) +/// does not apply the overlay block or re-score moved rows, so `batch_index_search_supported` +/// returns false and the per-query loop — which reconciles the overlay exactly as single-query +/// search does — runs instead. +/// +/// The no-overlay control confirms the shared-scan node *is* chosen otherwise (nprobes is +/// pinned so no other gate fires), so the fallback is attributable to the overlay alone. +#[rstest] +#[tokio::test] +async fn test_vector_batch_falls_back_on_overlay(#[values(false, true)] stable_row_ids: bool) { + // Two copies of the standard query, packed as a batch (multi-vector) nearest input. + let queries = fsl(vec![vec_query(), vec_query()], VEC_DIM); + + // Control: without an overlay the shared-scan batch node handles the batch query. + let base = create_vector_index_dataset(stable_row_ids).await; + let mut scanner = base.scan(); + scanner + .nearest("vec", queries.as_ref(), 3) + .unwrap() + .nprobes(1) + .project(&["id"]) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + plan.contains("ANNIvfBatch"), + "without an overlay the batch query should use the shared-scan node, got:\n{plan}" + ); + + // With an overlay on indexed vector rows the gate must fall back to the per-query loop. + let dataset = create_vector_overlay_dataset(stable_row_ids).await; + let mut scanner = dataset.scan(); + scanner + .nearest("vec", queries.as_ref(), 3) + .unwrap() + .nprobes(1) + .project(&["id"]) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + assert!( + !plan.contains("ANNIvfBatch"), + "an overlay on indexed rows must disable the shared-scan node, got:\n{plan}" + ); + assert!( + plan.contains("ANNSubIndex"), + "the batch query should fall back to the per-query indexed loop, got:\n{plan}" + ); + + // Correctness: the fallback reconciles the overlay for the batch — id=40 (moved onto the + // query) is found and id=35 (moved away) is dropped, just like single-query search. + let results = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let ids = ids_from_batches(&results); + assert!( + ids.contains(&40), + "overlay-moved id=40 should be found via the fallback path, got {ids:?}" + ); + assert!( + !ids.contains(&35), + "stale id=35 should be dropped via the fallback path, got {ids:?}" + ); +} + +/// A compound boolean predicate (age AND id) exercises the ScalarIndexExpr tree-walk in +/// `overlay_stale_index_rows`. An overlay on `age` marks fragment 0 stale from the `age` +/// index's perspective, so the compound query must re-evaluate fragment 0 on the flat path. +#[tokio::test] +async fn test_overlay_stale_with_compound_index_expression() { + let mut dataset = create_base_dataset().await; + // Build BTree indexes on both columns so a compound filter can use both. + build_age_index(&mut dataset).await; + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Fragment 0 covers id=0..5, age=0..50. Overlay changes id=1's age from 10 to 999. + let dataset = commit_overlay( + dataset, + "age_compound", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Compound query: both the `age` and `id` index are involved. The overlay on `age` + // makes fragment 0 stale for the `age` index; it falls to the flat path, which uses + // the merged (overlay) value. Result: the stale age=10 hit is gone, age=999 appears. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + // A pure `id` query on an unaffected fragment still works correctly. + assert_eq!(ids_matching(&dataset, "id = 2").await, vec![2]); +} + +/// A `RewriteRows` update (under stable row ids) that touches only a *non-indexed* column moves +/// the matched rows to a new fragment and, because the scalar index's field was not modified, +/// extends that index's fragment coverage onto the new fragment +/// (`register_pure_rewrite_rows_update_frags_in_indices`) so its existing entries are reused. +/// +/// That reuse is unsound when a moved row carried a data overlay on the *indexed* field: the +/// update materializes the overlay's current value into the new fragment, but the reused index +/// entry still holds the stale pre-overlay value, and the new fragment (now marked covered) no +/// longer falls to the flat path that previously served the correct value via overlay masking. +/// +/// Here `age` is indexed and overlaid (id=1: age 10 -> 999); the update sets the non-indexed +/// `id` column on that row. After it, `age = 10` must stay dropped and `age = 999` must still +/// find the row — otherwise the stale index entry has resurfaced. +#[tokio::test] +async fn test_update_nonindexed_column_preserves_overlay_masking() { + use crate::dataset::UpdateBuilder; + + let mut dataset = create_base_dataset_with(true).await; + build_age_index(&mut dataset).await; + + // Overlay fragment 0, offset 1 (id=1): age 10 -> 999, committed after the index. + let dataset = commit_overlay( + dataset, + "age_update", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Masking works before the update. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + + // Update only the non-indexed `id` column of the overlaid row. This is a rewrite-rows move: + // the row (with age materialized to 999) is written to a new fragment and deleted from + // fragment 0, keeping its stable row id. + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("id = 1") + .unwrap() + .set("id", "100") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + + // Still masked: the stale age=10 entry must stay dropped and the overlaid age=999 value must + // still be found (now on the moved row, whose id is 100). + assert_eq!( + ids_matching(&dataset, "age = 10").await, + Vec::::new(), + "stale index entry age=10 resurfaced after updating a non-indexed column" + ); + assert_eq!( + ids_matching(&dataset, "age = 999").await, + vec![100], + "overlaid value age=999 lost after updating a non-indexed column" + ); + // A row untouched by the overlay is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); +} + +/// Text dataset: two fragments, 6 rows each. Schema: id (Int32), text (Utf8). +/// Texts are unique tokens so each row can be identified by its term. +async fn create_text_dataset(stable_row_ids: bool) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("text", DataType::Utf8, true), + ])); + let texts: Vec<&str> = vec![ + "apple pie", + "apple banana", // row 1, fragment 0 — will be overlaid in tests + "cherry cake", + "banana split", + "orange juice", + "grape vine", + "mango sorbet", // fragment 1 starts here + "pear tart", + "lemon curd", + "peach cobbler", + "plum pudding", + "fig newton", + ]; + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(StringArray::from(texts)), + ], + ) + .unwrap(); + let write_params = WriteParams { + max_rows_per_file: 6, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + Dataset::write(reader, "memory://", Some(write_params)) + .await + .unwrap() +} + +async fn build_text_fts_index(dataset: &mut Dataset) { + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); +} + +/// FTS index with token positions stored, required for phrase queries. +async fn build_text_fts_index_with_positions(dataset: &mut Dataset) { + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().with_position(true), + true, + ) + .await + .unwrap(); +} + +/// Collect sorted IDs of rows returned by an FTS query on `text`. +async fn fts_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let results = dataset + .scan() + .full_text_search(query) + .unwrap() + .project(&["id"]) + .unwrap() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut ids = ids_from_batches(&results); + ids.sort_unstable(); + ids +} + +async fn fts_ids_matching(dataset: &Dataset, term: &str) -> Vec { + fts_ids(dataset, FullTextSearchQuery::new(term.to_owned())).await +} + +#[tokio::test] +async fn test_ngram_optimize_preserves_overlay_staleness() { + let mut dataset = create_text_dataset(false).await; + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + let fragment_ids = dataset + .get_fragments() + .into_iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + let mut segments = Vec::with_capacity(fragment_ids.len()); + for fragment_id in fragment_ids { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::NGram, ¶ms) + .name("text_ngram".to_string()) + .fragments(vec![fragment_id]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + let source_version = segments[0].dataset_version; + dataset + .commit_existing_index_segments("text_ngram", "text", segments) + .await + .unwrap(); + + let mut dataset = commit_overlay( + dataset, + "ngram_text_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + + let committed = dataset.load_indices_by_name("text_ngram").await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].dataset_version, source_version); + assert_eq!( + ids_matching(&dataset, "contains(text, 'apple')").await, + vec![0] + ); + assert_eq!( + ids_matching(&dataset, "contains(text, 'mango')").await, + vec![1, 6] + ); +} + +#[tokio::test] +async fn test_btree_physical_merge_preserves_overlay_staleness() { + let mut dataset = create_base_dataset().await; + let params = ScalarIndexParams::default(); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["age"], IndexType::BTree, ¶ms) + .name("age_btree".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + let source_version = segments[0].dataset_version; + let mut dataset = commit_overlay( + dataset, + "btree_before_merge", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + let merged = dataset + .merge_existing_index_segments(segments) + .await + .unwrap(); + assert_eq!(merged.dataset_version, source_version); + dataset + .commit_existing_index_segments("age_btree", "age", vec![merged]) + .await + .unwrap(); + + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); +} + +#[tokio::test] +async fn test_ngram_remap_excludes_newer_overlay_fragments() { + let mut dataset = create_text_dataset(false).await; + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + dataset + .create_index( + &["text"], + IndexType::NGram, + Some("text_ngram".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + let source_version = + dataset.load_indices_by_name("text_ngram").await.unwrap()[0].dataset_version; + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 12, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + let compacted_fragment_id = dataset.get_fragments()[0].id(); + let mut dataset = commit_overlay( + dataset, + "ngram_after_compaction", + compacted_fragment_id as u64, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + remapping::remap_column_index(&mut dataset, &["text"], Some("text_ngram".to_string())) + .await + .unwrap(); + + let committed = dataset.load_indices_by_name("text_ngram").await.unwrap(); + assert_eq!(committed.len(), 1); + assert!(committed[0].dataset_version > source_version); + assert!( + !committed[0] + .fragment_bitmap + .as_ref() + .unwrap() + .contains(compacted_fragment_id as u32) + ); + assert_eq!( + ids_matching(&dataset, "contains(text, 'apple')").await, + vec![0] + ); + assert_eq!( + ids_matching(&dataset, "contains(text, 'mango')").await, + vec![1, 6] + ); +} + +async fn fts_phrase_ids_matching(dataset: &Dataset, phrase: &str) -> Vec { + use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery}; + + let query = FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(phrase.to_owned()).with_column(Some("text".to_owned())), + )); + fts_ids(dataset, query).await +} + +/// An overlay committed after the FTS index is built replaces a row's text. Searching for +/// the old term must not return the stale row; searching for the new term must find it. +#[rstest] +#[tokio::test] +async fn test_fts_overlay_stale_drop_and_new_match(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index(&mut dataset).await; + + // fragment 0, row offset 1 (id=1): "apple banana" → "cherry mango" + // field ID 1 is the `text` column. + let dataset = commit_overlay( + dataset, + "text_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + // "apple" now matches only id=0 ("apple pie"); id=1's stale index entry must be dropped. + assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0]); + + // "banana" matched id=1 and id=3 before; after overlay id=1's stale entry must be gone. + assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3]); + + // "cherry" now matches id=1 (via flat path on stale fragment) and id=2 ("cherry cake"). + let cherry_ids = fts_ids_matching(&dataset, "cherry").await; + assert!( + cherry_ids.contains(&1), + "id=1 overlay→cherry mango should be found: {cherry_ids:?}" + ); + assert!( + cherry_ids.contains(&2), + "id=2 cherry cake should still be found: {cherry_ids:?}" + ); + + // "mango" now matches id=1 (overlay) and id=6 ("mango sorbet" in fragment 1). + let mango_ids = fts_ids_matching(&dataset, "mango").await; + assert!( + mango_ids.contains(&1), + "id=1 overlay→cherry mango should be found: {mango_ids:?}" + ); + assert!( + mango_ids.contains(&6), + "id=6 mango sorbet should still be found: {mango_ids:?}" + ); +} + +#[tokio::test] +async fn test_multimatch_shared_prefilter_preserves_field_overlay_masks() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("text_a", DataType::Utf8, false), + ArrowField::new("text_b", DataType::Utf8, false), + ])); + // Row 0 matches text_a, row 1 matches both fields before its overlay, + // and row 2 is a negative control. + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![0, 1, 2])), + Arc::new(StringArray::from(vec!["apple", "apple", "none"])), + Arc::new(StringArray::from(vec!["none", "apple", "none"])), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + "memory://", + None, + ) + .await + .unwrap(); + for column in ["text_a", "text_b"] { + dataset + .create_index( + &[column], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + } + // Only text_a is stale for row 1. text_b must retain its indexed match, + // while text_a's stale posting is blocked and re-evaluated separately. + let dataset = commit_overlay( + dataset, + "multimatch_text_a_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec!["none"]))], + ) + .await; + let query: FtsQuery = MultiMatchQuery::try_new( + "apple".to_owned(), + vec!["text_a".to_owned(), "text_b".to_owned()], + ) + .unwrap() + .into(); + let mut scanner = dataset.scan(); + scanner + .prefilter(true) + .use_scalar_index(false) + .full_text_search(FullTextSearchQuery::new_query(query)) + .unwrap() + .filter("id >= 0") + .unwrap() + .project(&["id"]) + .unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + let mut ids = batch["id"].as_primitive::().values().to_vec(); + ids.sort_unstable(); + assert_eq!(ids, vec![0, 1]); +} + +/// A phrase query must drop stale indexed positions and re-evaluate the current +/// overlay value on the flat phrase path. +#[rstest] +#[tokio::test] +async fn test_fts_phrase_overlay_stale_drop(#[values(false, true)] stable_row_ids: bool) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index_with_positions(&mut dataset).await; + + // Before any overlay the phrase "apple banana" matches only id=1. + assert_eq!( + fts_phrase_ids_matching(&dataset, "apple banana").await, + vec![1] + ); + + // Overlay id=1's text (field 1) so the phrase no longer applies to its current value. + let dataset = commit_overlay( + dataset, + "phrase_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + // The stale inverted-index positions for "apple banana" on id=1 must not be returned. + assert_eq!( + fts_phrase_ids_matching(&dataset, "apple banana").await, + Vec::::new() + ); + assert_eq!( + fts_phrase_ids_matching(&dataset, "cherry mango").await, + vec![1] + ); +} + +#[tokio::test] +async fn test_fts_empty_fragment_selection_is_empty() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let mut match_scan = dataset.scan(); + match_scan.with_fragments(Vec::new()); + match_scan + .full_text_search(FullTextSearchQuery::new("apple".to_owned())) + .unwrap(); + match_scan.project(&["id"]).unwrap(); + let match_plan = match_scan.explain_plan(false).await.unwrap(); + assert!( + match_plan.contains("EmptyExec"), + "explicit empty fragment selection should produce EmptyExec: {match_plan}" + ); + assert_eq!(match_scan.try_into_batch().await.unwrap().num_rows(), 0); + + let mut phrase_scan = dataset.scan(); + phrase_scan.with_fragments(Vec::new()); + phrase_scan + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("apple pie".to_owned()).with_column(Some("text".to_owned())), + ))) + .unwrap(); + phrase_scan.project(&["id"]).unwrap(); + let phrase_plan = phrase_scan.explain_plan(false).await.unwrap(); + assert!( + phrase_plan.contains("EmptyExec"), + "explicit empty fragment selection should produce EmptyExec: {phrase_plan}" + ); + assert_eq!(phrase_scan.try_into_batch().await.unwrap().num_rows(), 0); +} + +#[rstest] +#[tokio::test] +async fn test_fts_combines_indexed_overlay_stale_and_unindexed_rows( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let batch = + arrow_array::record_batch!(("id", Int32, [12]), ("text", Utf8, ["cherry mango"])).unwrap(); + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Dataset::write( + reader, + Arc::new(dataset), + Some(WriteParams { + mode: crate::dataset::write::WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let dataset = commit_overlay( + dataset, + "fts_combined_flat_paths", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + assert_eq!(fts_ids_matching(&dataset, "mango").await, vec![1, 6, 12]); + assert_eq!( + fts_phrase_ids_matching(&dataset, "cherry mango").await, + vec![1, 12] + ); +} + +#[rstest] +#[tokio::test] +async fn test_fts_overlay_row_level_masking_under_fast_search( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let dataset = commit_overlay( + dataset, + "fts_row_level_fast_search", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + let mut match_scan = dataset.scan(); + match_scan + .full_text_search(FullTextSearchQuery::new("apple".to_owned())) + .unwrap(); + match_scan.project(&["id"]).unwrap(); + match_scan.fast_search(); + let match_result = match_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&match_result)), + vec![0] + ); + + let mut indexed_phrase_scan = dataset.scan(); + indexed_phrase_scan + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("apple pie".to_owned()).with_column(Some("text".to_owned())), + ))) + .unwrap(); + indexed_phrase_scan.project(&["id"]).unwrap(); + indexed_phrase_scan.fast_search(); + let indexed_phrase_result = indexed_phrase_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&indexed_phrase_result)), + vec![0] + ); + + let mut new_phrase_scan = dataset.scan(); + new_phrase_scan + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("cherry mango".to_owned()).with_column(Some("text".to_owned())), + ))) + .unwrap(); + new_phrase_scan.project(&["id"]).unwrap(); + new_phrase_scan.fast_search(); + assert_eq!( + new_phrase_scan.try_into_batch().await.unwrap().num_rows(), + 0 + ); + + let match_query = |terms: &str| { + MatchQuery::new(terms.to_owned()) + .with_column(Some("text".to_owned())) + .into() + }; + let compound_query: FtsQuery = BooleanQuery::new([ + (Occur::Must, match_query("apple")), + (Occur::Should, match_query("pie")), + ]) + .into(); + let mut compound_scan = dataset.scan(); + compound_scan + .full_text_search(FullTextSearchQuery::new_query(compound_query)) + .unwrap(); + compound_scan.project(&["id"]).unwrap(); + compound_scan.fast_search(); + compound_scan.limit(Some(10), None).unwrap(); + let compound_plan = compound_scan.explain_plan(false).await.unwrap(); + assert!( + !compound_plan.contains("CompoundFtsScorer"), + "overlay-stale rows must keep compound fast search on the masked fallback:\n{compound_plan}" + ); + let compound_result = compound_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&compound_result)), + vec![0] + ); +} + +#[rstest] +#[tokio::test] +async fn test_fts_overlay_flat_path_takes_only_stale_rows( + #[values(false, true)] stable_row_ids: bool, +) { + let mut dataset = create_text_dataset(stable_row_ids).await; + build_text_fts_index(&mut dataset).await; + + let dataset = commit_overlay( + dataset, + "fts_targeted_take", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + let mut scan = dataset.scan(); + scan.full_text_search(FullTextSearchQuery::new("cherry".to_owned())) + .unwrap(); + scan.project(&["id"]).unwrap(); + let plan = scan.create_plan().await.unwrap(); + + let mut nodes = vec![plan]; + let mut flat_path_uses_targeted_take = false; + while let Some(node) = nodes.pop() { + if node.downcast_ref::().is_some() { + let mut flat_nodes = node.children().into_iter().cloned().collect::>(); + while let Some(flat_node) = flat_nodes.pop() { + if flat_node + .downcast_ref::() + .is_some_and(|read| read.index_input().is_some()) + { + flat_path_uses_targeted_take = true; + break; + } + flat_nodes.extend(flat_node.children().into_iter().cloned()); + } + } + nodes.extend(node.children().into_iter().cloned()); + } + + assert!( + flat_path_uses_targeted_take, + "overlay-stale FTS rows must be re-evaluated through a targeted take" + ); +} + +#[tokio::test] +async fn test_fts_phrase_searches_unindexed_fragments_unless_fast_search() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let batch = arrow_array::record_batch!( + ("id", Int32, [12, 13, 14]), + ( + "text", + Utf8, + [ + "kiwi berry", + "kiwi berry filling", + "kiwi berry filling filling" + ] + ) + ) + .unwrap(); + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Dataset::write( + reader, + Arc::new(dataset), + Some(WriteParams { + mode: crate::dataset::write::WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let query = FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("kiwi berry".to_owned()).with_column(Some("text".to_owned())), + )); + assert_eq!(fts_ids(&dataset, query.clone()).await, vec![12, 13, 14]); + + let mut limited_scan = dataset.scan(); + limited_scan.with_fragments(vec![dataset.fragments().last().unwrap().clone()]); + limited_scan + .full_text_search(query.clone().limit(Some(1))) + .unwrap(); + limited_scan.project(&["id"]).unwrap(); + let limited_result = limited_scan.try_into_batch().await.unwrap(); + assert_eq!( + ids_from_batches(std::slice::from_ref(&limited_result)), + vec![12] + ); + + let mut filtered_scan = dataset.scan(); + filtered_scan.full_text_search(query.clone()).unwrap(); + filtered_scan.project(&["id"]).unwrap(); + filtered_scan.filter("id < 12").unwrap(); + assert_eq!(filtered_scan.try_into_batch().await.unwrap().num_rows(), 0); + + let mut fast_scan = dataset.scan(); + fast_scan.full_text_search(query).unwrap(); + fast_scan.project(&["id"]).unwrap(); + fast_scan.fast_search(); + assert_eq!(fast_scan.try_into_batch().await.unwrap().num_rows(), 0); +} + +#[tokio::test] +async fn test_fts_phrase_stale_rows_honor_query_limit() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + let dataset = commit_overlay( + dataset, + "phrase_limit_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some( + "apple banana filling filling", + )]))], + ) + .await; + + let query = FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("apple".to_owned()).with_column(Some("text".to_owned())), + )) + .limit(Some(1)); + let mut scan = dataset.scan(); + scan.with_fragments(vec![dataset.fragments()[0].clone()]); + scan.full_text_search(query).unwrap(); + scan.project(&["id"]).unwrap(); + + let result = scan.try_into_batch().await.unwrap(); + assert_eq!(ids_from_batches(std::slice::from_ref(&result)), vec![0]); +} + +/// Overlay routing must select FTS segments by both field and document +/// granularity when Row and ListElement indexes coexist. +#[tokio::test] +async fn test_list_element_fts_overlay_uses_exact_index_and_flat_fallback() { + let ids = Arc::new(Int32Array::from_iter_values(0..4)) as ArrayRef; + let tags = string_lists(&[ + &["old phrase", "keep"], + &["other"], + &["new phrase"], + &["unrelated"], + ]); + let batch = RecordBatch::try_from_iter(vec![("id", ids), ("tags", tags)]).unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["tags"], + IndexType::Inverted, + None, + &InvertedIndexParams::default() + .with_position(true) + .document_granularity(DocumentGranularity::ListElement), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["tags"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().with_position(true), + true, + ) + .await + .unwrap(); + + let dataset = commit_overlay( + dataset, + "list_element_text_overlay", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([0])), + vec![string_lists(&[&["new phrase", "keep"]])], + ) + .await; + + let list_element_match = |terms: &str| { + FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new(terms.to_owned()) + .with_column(Some("tags".to_owned())) + .with_document_granularity(DocumentGranularity::ListElement), + )) + }; + let list_element_phrase = |terms: &str| { + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(terms.to_owned()) + .with_column(Some("tags".to_owned())) + .with_document_granularity(DocumentGranularity::ListElement), + )) + }; + + assert_eq!( + fts_ids(&dataset, list_element_match("old")).await, + Vec::::new() + ); + assert_eq!( + fts_ids(&dataset, list_element_match("new")).await, + vec![0, 2] + ); + assert_eq!( + fts_ids(&dataset, list_element_phrase("new phrase")).await, + vec![0, 2] + ); +} + +/// An overlay on a non-FTS field must not exclude the fragment from phrase search. +#[tokio::test] +async fn test_fts_phrase_overlay_unrelated_field_not_excluded() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index_with_positions(&mut dataset).await; + + // Overlay field 0 (`id`), not the FTS-indexed `text` column: phrase coverage is untouched. + let dataset = commit_overlay( + dataset, + "id_overlay", + 0, + &[0], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(777)])], + ) + .await; + + assert_eq!( + fts_phrase_ids_matching(&dataset, "apple banana").await, + vec![777] + ); +} + +/// An overlay on a field the FTS index does NOT cover must not exclude anything. +#[tokio::test] +async fn test_fts_overlay_unrelated_field_not_excluded() { + let mut dataset = create_text_dataset(false).await; + build_text_fts_index(&mut dataset).await; + + // Overlay field 0 (id) — not covered by the FTS index on `text`. + let dataset = commit_overlay( + dataset, + "id_overlay_for_fts", + 0, + &[0], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // FTS coverage must be unchanged — both rows containing "apple" are still returned. + // The `id` overlay changes row offset 1's id from 1 to 999, so the projected id column + // reflects the overlay even though the FTS index correctly returned that row. + assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0, 999]); + assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3, 999]); +} + +/// Benchmark: measure query latency for BTree, FTS, and vector ANN with 0/4/16 overlay layers. +/// +/// Run with: +/// cargo test -p lance --lib --profile release-with-debug -- overlay_index_masking::bench --ignored --nocapture +#[tokio::test] +#[ignore = "benchmark"] +#[allow(clippy::print_stdout)] +async fn bench_index_query_overlay_overhead() { + use std::time::Instant; + + use arrow_array::Float32Array; + + const DIM: i32 = 32; + const ROWS: i32 = 1_000_000; + const ROWS_PER_FRAG: i32 = 100_000; // 10 fragments + const ITERS: u32 = 10; // large scans — 10 is enough for stable averages + + // Fixed disk path so timings are comparable across runs. Deleted and recreated fresh. + let uri = "/tmp/lance-bench-overlay-oss1325"; + if std::path::Path::new(uri).exists() { + std::fs::remove_dir_all(uri).unwrap(); + } + + // --- Build 1M-row dataset on local disk -------------------------------- + // Schema: id, age, vec, text. Resolve field IDs from the Lance schema instead of + // assuming how nested Arrow child fields are numbered. + + println!("Building {ROWS}-row dataset at {uri} (this takes ~30 s)..."); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("age", DataType::Int32, false), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + DIM, + ), + false, + ), + ArrowField::new("text", DataType::Utf8, false), + ])); + + let row_ids: Vec = (0..ROWS).collect(); + let ages: Vec = row_ids.iter().map(|&i| i * 10).collect(); + // Build the 128 MB flat float array directly (avoids 1M per-row Vec allocations). + let flat_vecs: Vec = (0..(ROWS as usize * DIM as usize)) + .map(|j| (j / DIM as usize) as f32 % 1000.0) + .collect(); + let vec_col = Arc::new( + arrow_array::FixedSizeListArray::try_new( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + DIM, + Arc::new(Float32Array::from(flat_vecs)), + None, + ) + .unwrap(), + ); + let text_col = Arc::new(StringArray::from_iter_values( + (0..ROWS).map(|row| if row == 42 { "needle" } else { "common" }), + )); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(row_ids)), + Arc::new(Int32Array::from(ages)), + vec_col, + text_col, + ], + ) + .unwrap(); + + let write_params = WriteParams { + max_rows_per_file: ROWS_PER_FRAG as usize, + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, Some(write_params)) + .await + .unwrap(); + let text_field_id = dataset.schema().field_id("text").unwrap(); + + println!("Building BTree index on age..."); + dataset + .create_index( + &["age"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + println!("Building IVF_FLAT(1 partition) index on vec..."); + dataset + .create_index( + &["vec"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + println!("Building FTS index on text..."); + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + println!("Indexes built.\n"); + + // --- Timing helper --------------------------------------------------- + + async fn timeit(iters: u32, mut f: F) -> f64 + where + F: FnMut() -> Fut, + Fut: std::future::Future, + { + f().await; // warmup + let t0 = Instant::now(); + for _ in 0..iters { + f().await; + } + t0.elapsed().as_secs_f64() * 1000.0 / iters as f64 + } + + // === Scenario A: BTree query overhead ================================ + // + // Overlay on `age` (field 1), covering only offset 0 of fragment 0. The stale row is + // blocked from the BTree and re-evaluated by targeted take. + // + // btree_same_fragment: `age = 420` → id=42 → in fragment 0 (rows 0..99999). + // The matching row stays indexed even though another row in the fragment is stale. + // + // btree_other_fragment: `age = 1000420` → id=100042 → in fragment 1. + // This isolates the index-lookup baseline outside the overlaid fragment. + println!("=== Scenario A: BTree (one stale row in fragment 0) ==="); + println!( + "{:>10} {:>14} {:>14}", + "overlays", "same_frag_ms", "other_frag_ms" + ); + + let mut committed_a = 0u32; + for num_overlays in [0u32, 1, 4, 16] { + // Commit only the delta since the last iteration. + for layer in committed_a..num_overlays { + dataset = commit_overlay( + dataset, + &format!("age_ol{layer}"), + 0, // fragment 0 + &[1], // field 1 = age + OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + vec![i32_array([Some(999)])], + ) + .await; + } + committed_a = num_overlays; + + let ds = Arc::new(dataset.clone()); + + // Same-fragment indexed match plus targeted re-evaluation of the stale row. + let ds2 = ds.clone(); + let same_fragment_ms = timeit(ITERS, || { + let ds = ds2.clone(); + async move { + ds.scan() + .filter("age = 420") + .unwrap() + .project(&["age"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + } + }) + .await; + + // Fragment 1 never has a stale row and stays entirely index-served. + let ds2 = ds.clone(); + let other_fragment_ms = timeit(ITERS, || { + let ds = ds2.clone(); + async move { + ds.scan() + .filter("age = 1000420") + .unwrap() + .project(&["age"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + } + }) + .await; + + println!("{num_overlays:>10} {same_fragment_ms:>14.1} {other_fragment_ms:>14.1}"); + } + + // === Scenario B: Vector ANN overhead ================================= + // + // Overlay on `vec` (field 2), covering only offset 0 of fragment 0. + // The field-aware check means the 16 age overlays from Scenario A do NOT affect + // the vector index (they touch field 1, not field 2). Only a vec overlay (field 2) + // marks fragment 0 stale for the vector index. + // + // With a vec overlay, only the stale row is excluded from ANN and re-scored exactly. + println!("\n=== Scenario B: Vector ANN (one stale row re-scored) ==="); + println!("{:>12} {:>10}", "vec_overlays", "ann_ms"); + + let query_vec = Float32Array::from(vec![0.5f32; DIM as usize]); + + for num_vec_overlays in [0u32, 1] { + if num_vec_overlays == 1 { + dataset = commit_overlay( + dataset, + "vec_ol0", + 0, // fragment 0 + &[2], // field 2 = vec (FixedSizeList top-level field) + OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + vec![fsl(vec![vec![0.0f32; DIM as usize]], DIM)], + ) + .await; + } + + let ds = Arc::new(dataset.clone()); + let ds2 = ds.clone(); + let qv = query_vec.clone(); + let ann_ms = timeit(ITERS, || { + let ds = ds2.clone(); + let q = qv.clone(); + async move { + ds.scan() + .nearest("vec", &q, 10) + .unwrap() + .minimum_nprobes(1) + .project(&["id"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + } + }) + .await; + + println!("{num_vec_overlays:>12} {ann_ms:>10.1}"); + } + + // === Scenario C: FTS overhead ======================================== + // + // The FTS index has one segment spanning all 10 fragments. An overlay on one text row must + // keep that segment indexed, block just the stale row, and re-evaluate that row by targeted + // take. `needle` belongs to an unaffected row in the same fragment as the stale row. + println!("\n=== Scenario C: FTS (one stale row in a 1M-row segment) ==="); + println!("{:>13} {:>10}", "text_overlays", "fts_ms"); + + for num_text_overlays in [0u32, 1] { + if num_text_overlays == 1 { + dataset = commit_overlay( + dataset, + "text_ol0", + 0, + &[text_field_id], + OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + vec![Arc::new(StringArray::from(vec![Some("updated")]))], + ) + .await; + } + + let ds = Arc::new(dataset.clone()); + let ds2 = ds.clone(); + let fts_ms = timeit(ITERS, || { + let ds = ds2.clone(); + async move { + let result = ds + .scan() + .full_text_search(FullTextSearchQuery::new("needle".to_owned())) + .unwrap() + .project(&["id"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 1); + } + }) + .await; + + println!("{num_text_overlays:>13} {fts_ms:>10.1}"); + } +} + +async fn append_age_fragment(dataset: &mut Dataset, ids: std::ops::Range) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("age", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(ids.clone())), + Arc::new(Int32Array::from_iter_values(ids.map(|v| v * 10))), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); +} + +// `OptimizeIndices` merges an index's delta segments without re-reading data overlays, so the +// merged segment carries the old segments' pre-overlay entries. What keeps those entries masked +// is that the merge stamps the new segment with the *oldest* merged segment's `dataset_version` +// rather than the current one, leaving the mask's version gate +// (`overlay.committed_version > segment.dataset_version`) on. +// +// That invariant is easy to break by accident -- stamping the current version un-masks every +// carried-over entry -- and nothing else asserts it end to end. The following tests pin it down +// for each index type. + +/// Scalar (BTree, Bitmap, ZoneMap): a range query over the indexed column after an overlay + +/// optimize must still drop the stale value and surface the overlaid one. +/// +/// ZoneMap is the case worth having: it ignores `OldIndexDataFilter`, so nothing scrubs the +/// pre-overlay zone summaries out of the merged segment. Breaking the version gate shows up here +/// as a false *negative* -- the stale zone prunes the overlaid value -- rather than the resurfaced +/// stale entry the other two produce. +#[rstest] +#[case::btree(IndexType::BTree)] +#[case::bitmap(IndexType::Bitmap)] +#[case::zonemap(IndexType::ZoneMap)] +#[tokio::test] +async fn test_optimize_preserves_scalar_overlay_masking(#[case] index_type: IndexType) { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + use lance_index::scalar::BuiltinIndexType; + + let params = match index_type { + IndexType::Bitmap => ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap), + IndexType::ZoneMap => ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + _ => ScalarIndexParams::default(), + }; + let mut dataset = create_base_dataset().await; + dataset + .create_index(&["age"], index_type, None, ¶ms, true) + .await + .unwrap(); + + // Overlay fragment 0, offset 1 (id=1): age 10 -> 999, committed after the index. + let mut dataset = commit_overlay( + dataset, + "age_opt", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + // Masking works before optimize. + assert_eq!(ids_matching(&dataset, "age = 10").await, Vec::::new()); + assert_eq!(ids_matching(&dataset, "age = 999").await, vec![1]); + + // Append an unindexed fragment so the merge does real work, then merge all deltas. + append_age_fragment(&mut dataset, 12..18).await; + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + // Still masked: the stale age=10 entry stays dropped and the overlaid age=999 value stays + // visible, because the merged segment kept the old segment's `dataset_version`. + assert_eq!( + ids_matching(&dataset, "age = 10").await, + Vec::::new(), + "stale index entry age=10 for id=1 resurfaced after optimize" + ); + assert_eq!( + ids_matching(&dataset, "age = 999").await, + vec![1], + "overlaid value age=999 dropped after optimize" + ); + // A row untouched by the overlay is unaffected. + assert_eq!(ids_matching(&dataset, "age = 20").await, vec![2]); +} + +/// ZoneMap seed path: an *unindexed* fragment carrying an overlay gets folded into the merged +/// segment from its data file's seed buffer. A seed is a zone summary captured while the base +/// data file was written, so it describes pre-overlay values -- the merged segment ends up +/// holding zones that never saw the overlay. +/// +/// That is only safe because the merge keeps the old `dataset_version`, so the mask still covers +/// those rows. Stamp the current version (or teach the seed path to claim freshness) and the +/// overlaid value becomes unfindable, since its stale zone prunes it. +/// +/// `name` is Utf8, for which seeds are on by default (`default_use_seeds`). +#[tokio::test] +async fn test_optimize_seed_path_respects_overlay() { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + use lance_index::scalar::BuiltinIndexType; + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("name", DataType::Utf8, true), + ])); + let names: Vec = (0..12).map(|i| format!("n{i:02}")).collect(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..12)), + Arc::new(StringArray::from(names)), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + "memory://", + Some(WriteParams { + max_rows_per_file: 6, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["name"], + IndexType::ZoneMap, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + true, + ) + .await + .unwrap(); + + // Append fragment 2. The index already exists, so the write emits a zone-map seed buffer + // into the new data file. + let appended: Vec = (12..18).map(|i| format!("n{i:02}")).collect(); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(12..18)), + Arc::new(StringArray::from(appended)), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); + + // Overlay fragment 2, offset 1 (id=13): "n13" -> "zzz", well outside the seed's zone range. + let mut dataset = commit_overlay( + dataset, + "name_seed", + 2, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("zzz")]))], + ) + .await; + + // Fragment 2 is unindexed, so the overlaid value is visible via the flat path. + assert_eq!(ids_matching(&dataset, "name = 'zzz'").await, vec![13]); + assert_eq!( + ids_matching(&dataset, "name = 'n13'").await, + Vec::::new() + ); + + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + assert_eq!( + ids_matching(&dataset, "name = 'zzz'").await, + vec![13], + "overlaid value dropped after optimize: the merged segment took pre-overlay zones \ + from fragment 2's seed buffer" + ); + assert_eq!( + ids_matching(&dataset, "name = 'n13'").await, + Vec::::new(), + "stale pre-overlay value resurfaced after optimize" + ); + // Rows the overlay never touched keep working through the index. + assert_eq!(ids_matching(&dataset, "name = 'n14'").await, vec![14]); + assert_eq!(ids_matching(&dataset, "name = 'n03'").await, vec![3]); +} + +/// `DatasetStatistics::column_value_range` folds ZoneMap summaries into a global `[min, max]` +/// that callers may prune with, so it must be a superset of the live values. An overlay can +/// move a value outside the summarised range, and the ZoneMap never saw it. +#[tokio::test] +async fn test_column_value_range_none_under_overlay() { + use crate::index::DatasetIndexExt; + use datafusion::scalar::ScalarValue; + use lance_index::scalar::BuiltinIndexType; + + let mut dataset = create_base_dataset().await; + dataset + .create_index( + &["age"], + IndexType::ZoneMap, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap), + true, + ) + .await + .unwrap(); + + assert_eq!( + dataset + .statistics() + .column_value_range("age") + .await + .unwrap(), + Some((ScalarValue::Int32(Some(0)), ScalarValue::Int32(Some(110)))) + ); + + // age 10 -> 999 on fragment 0, committed after the index: 999 is outside [0, 110]. + let dataset = commit_overlay( + dataset, + "age_range", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![i32_array([Some(999)])], + ) + .await; + + assert_eq!( + dataset + .statistics() + .column_value_range("age") + .await + .unwrap(), + None, + "ZoneMap range must not be reported once an overlay may have moved a value outside it" + ); +} + +/// FTS: after an overlay replaces a row's text and the index is optimized, searching for the old +/// terms must not return the stale row, and the new terms must find it. +#[tokio::test] +async fn test_optimize_preserves_fts_overlay_masking() { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + + let mut dataset = create_text_dataset(false).await; + build_text_fts_index(&mut dataset).await; + + // fragment 0, offset 1 (id=1): "apple banana" -> "cherry mango". + let mut dataset = commit_overlay( + dataset, + "text_opt", + 0, + &[1], + OverlayCoverage::dense(RoaringBitmap::from_iter([1])), + vec![Arc::new(StringArray::from(vec![Some("cherry mango")]))], + ) + .await; + + // Masking works before optimize: id=1 no longer matches "banana"/"apple". + assert_eq!(fts_ids_matching(&dataset, "banana").await, vec![3]); + assert_eq!(fts_ids_matching(&dataset, "apple").await, vec![0]); + + // Append an unindexed fragment of new text, then merge all deltas. + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("text", DataType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(12..18)), + Arc::new(StringArray::from(vec![ + "kiwi", "melon", "date", "guava", "papaya", "lychee", + ])), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + // Still masked: id=1's stale "apple"/"banana" postings stay dropped. + assert_eq!( + fts_ids_matching(&dataset, "banana").await, + vec![3], + "stale FTS posting for id=1 (banana) resurfaced after optimize" + ); + assert_eq!( + fts_ids_matching(&dataset, "apple").await, + vec![0], + "stale FTS posting for id=1 (apple) resurfaced after optimize" + ); + // The overlaid terms are found via the flat path. + assert!(fts_ids_matching(&dataset, "cherry").await.contains(&1)); + assert!(fts_ids_matching(&dataset, "mango").await.contains(&1)); +} + +/// Vector (IVF): after an overlay moves a row's vector and the index is optimized, the ANN must +/// not resurface the stale vector, and the moved-onto-query row is found by flat re-scoring. +#[tokio::test] +async fn test_optimize_preserves_vector_overlay_masking() { + use crate::index::DatasetIndexExt; + use lance_index::optimize::OptimizeOptions; + + // Overlay on fragment 1 moves id=35 away from the query and id=40 onto it. + let mut dataset = create_vector_overlay_dataset(false).await; + + // Masking works before optimize. + let before = vector_query_ids(&dataset, 3, false).await; + assert!( + !before.contains(&35), + "pre-optimize id=35 should be dropped: {before:?}" + ); + assert!( + before.contains(&40), + "pre-optimize id=40 should be found: {before:?}" + ); + + // Append an unindexed fragment of far vectors, then merge all deltas. + let far_vecs: Vec> = (0..32) + .map(|i| { + let mut v = vec![0.0_f32; VEC_DIM as usize]; + v[1] = (i + 200) as f32; + v + }) + .collect(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + VEC_DIM, + ), + true, + ), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(64..96)), + fsl(far_vecs, VEC_DIM), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + None, + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + + // Still masked: id=35's stale vector stays dropped and id=40 is still found via re-scoring. + let after = vector_query_ids(&dataset, 3, false).await; + assert!( + !after.contains(&35), + "stale index vector for id=35 resurfaced after optimize: {after:?}" + ); + assert!( + after.contains(&40), + "overlaid vector for id=40 dropped after optimize: {after:?}" + ); +} diff --git a/rust/lance/src/dataset/tests/dataset_scanner.rs b/rust/lance/src/dataset/tests/dataset_scanner.rs index 363698d9966..97cecaa6245 100644 --- a/rust/lance/src/dataset/tests/dataset_scanner.rs +++ b/rust/lance/src/dataset/tests/dataset_scanner.rs @@ -11,11 +11,11 @@ use lance_arrow::{ARROW_EXT_NAME_KEY, FixedSizeListArrayExt}; use crate::index::DatasetIndexExt; use arrow::compute::concat_batches; -use arrow_array::UInt64Array; use arrow_array::cast::AsArray; -use arrow_array::{Array, FixedSizeListArray, ListArray, StructArray}; +use arrow_array::{Array, ArrayRef, FixedSizeListArray, LargeListArray, ListArray, StructArray}; use arrow_array::{Float32Array, Int32Array, RecordBatch, RecordBatchIterator, StringArray}; -use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_array::{Int64Array, UInt64Array}; +use arrow_buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema, SchemaRef}; use futures::TryStreamExt; use lance_arrow::SchemaExt; @@ -25,7 +25,9 @@ use lance_file::reader::{FileReader, FileReaderOptions, describe_encoding}; use lance_file::version::LanceFileVersion; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::{ - SCORE_FIELD, query::PhraseQuery, tokenizer::InvertedIndexParams, + SCORE_FIELD, + query::{FtsQuery, MatchQuery, Operator, PhraseQuery}, + tokenizer::InvertedIndexParams, }; use lance_index::{IndexType, vector::DIST_COL}; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; @@ -33,14 +35,155 @@ use lance_io::utils::CachedFileSize; use lance_linalg::distance::MetricType; use uuid::Uuid; -use crate::Dataset; +use crate::dataset::NewColumnTransform; use crate::dataset::scanner::{DatasetRecordBatchStream, QueryFilter}; use crate::dataset::write::WriteParams; -use lance_index::scalar::inverted::query::FtsQuery; +use crate::{Dataset, Error}; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::pq::PQBuildParams; use lance_index::vector::{DEFAULT_QUERY_PARALLELISM, Query}; use pretty_assertions::assert_eq; +use rstest::rstest; + +/// A null struct must not read back as a valid struct with null children. +/// +/// A scan merges the per-column batches with `lance_arrow::merge`, which used to read an all-null +/// validity buffer as "this side carries no validity" and drop it. A filter that selects only null +/// rows leaves exactly that shape, so the scan reported those rows as valid while `IS NULL` +/// counted them as null. The dataset is created empty and then appended to because that is the +/// path this was found on, and the version is pinned because 2.0 does not encode struct validity. +#[tokio::test] +async fn test_filtered_scan_preserves_nullable_struct_validity() { + let struct_fields = Fields::from(vec![ + ArrowField::new("a", DataType::Int64, true), + ArrowField::new("b", DataType::Utf8, true), + ]); + let item_field = Arc::new(ArrowField::new( + "item", + DataType::Struct(struct_fields.clone()), + true, + )); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::UInt64, false), + ArrowField::new("s", DataType::Struct(struct_fields.clone()), true), + ArrowField::new("l", DataType::List(item_field.clone()), true), + ])); + + let empty = RecordBatch::new_empty(schema.clone()); + let reader = RecordBatchIterator::new([Ok(empty)], schema.clone()); + let mut dataset = Dataset::write( + reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..WriteParams::default() + }), + ) + .await + .unwrap(); + + // Rows 100 and 177 are null in both nested columns while their children still carry values, + // so losing the top-level validity turns them into valid values instead of null ones. + let validity = NullBuffer::from(vec![false, true, false]); + let structs = StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int64Array::from(vec![Some(10), Some(11), Some(12)])), + Arc::new(StringArray::from(vec![Some("x"), Some("y"), Some("z")])), + ], + Some(validity.clone()), + ); + let items = StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int64Array::from(vec![Some(20), Some(21), Some(22)])), + Arc::new(StringArray::from(vec![Some("p"), Some("q"), Some("r")])), + ], + None, + ); + let lists = ListArray::new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(vec![0, 1, 2, 3])), + Arc::new(items), + Some(validity), + ); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt64Array::from(vec![100, 116, 177])), + Arc::new(structs), + Arc::new(lists), + ], + ) + .unwrap(); + dataset + .append( + Box::new(RecordBatchIterator::new([Ok(batch)], schema)), + None, + ) + .await + .unwrap(); + + for (id, expected_is_null) in [(100, true), (116, false), (177, true)] { + let mut scan = dataset.scan(); + scan.filter(&format!("id = {id}")).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + let structs = batch["s"].as_struct(); + assert_eq!(structs.is_null(0), expected_is_null, "s, row id {id}"); + // A null struct masks its children, so both levels have to agree. + assert_eq!( + structs.column(0).is_null(0), + expected_is_null, + "s.a, row id {id}" + ); + assert_eq!( + batch["l"].as_list::().is_null(0), + expected_is_null, + "l, row id {id}" + ); + } + + assert_eq!( + dataset + .count_rows(Some("s IS NULL".to_owned())) + .await + .unwrap(), + 2 + ); + assert_eq!( + dataset + .count_rows(Some("l IS NULL".to_owned())) + .await + .unwrap(), + 2 + ); + + // A struct column added as all-nulls reaches the same merge through schema evolution, where + // every row is null and there is no other side to recover the validity from. + dataset + .add_columns( + NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(vec![ArrowField::new( + "t", + DataType::Struct(struct_fields), + true, + )]))), + None, + None, + ) + .await + .unwrap(); + let mut scan = dataset.scan(); + scan.filter("id = 116").unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + assert!(batch["t"].as_struct().is_null(0)); + assert_eq!( + dataset + .count_rows(Some("t IS NULL".to_owned())) + .await + .unwrap(), + 3 + ); +} #[tokio::test] async fn test_scan_wide_fixed_size_list_at_batch_boundary() { @@ -401,6 +544,212 @@ async fn test_fts_filter_vector_search() { assert!(stream.is_err()); } +#[rstest] +#[case::list(false)] +#[case::large_list(true)] +#[tokio::test] +async fn test_fts_list_postfilter_vector_search(#[case] is_large_list: bool) { + async fn indexed_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = result["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + async fn postfilter_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let query_vector = Float32Array::from(vec![0.0, 0.0]); + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &query_vector, 5) + .unwrap() + .prefilter(false) + .filter_query(QueryFilter::Fts(query)) + .unwrap(); + let plan = scanner.explain_plan(false).await.unwrap(); + let post_filter_position = plan + .find("FlatMatchFilter: column=docs") + .expect("expected FTS to run as a flat match filter"); + let vector_search_position = plan + .find("ANNSubIndex") + .expect("expected the query to use the vector index"); + assert!( + post_filter_position < vector_search_position, + "expected FTS to wrap the vector search as a post-filter, got:\n{plan}" + ); + let result = scanner.try_into_batch().await.unwrap(); + let mut ids = result["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + fn match_query(terms: &str, operator: Operator) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new(terms.to_owned()) + .with_column(Some("docs".to_owned())) + .with_operator(operator), + )) + } + + let item_field = Arc::new(ArrowField::new("item", DataType::Utf8, true)); + let values = Arc::new(StringArray::from(vec![ + Some("target"), + Some("alpha"), + Some("beta"), + Some(""), + None, + Some("target"), + ])) as ArrayRef; + let validity = Some(NullBuffer::from(vec![true, true, true, true, false])); + let docs: ArrayRef = if is_large_list { + Arc::new(LargeListArray::new( + item_field.clone(), + OffsetBuffer::new(ScalarBuffer::from(vec![0_i64, 2, 3, 3, 6, 6])), + values, + validity, + )) + } else { + Arc::new(ListArray::new( + item_field, + OffsetBuffer::new(ScalarBuffer::from(vec![0_i32, 2, 3, 3, 6, 6])), + values, + validity, + )) + }; + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![0.0, 0.0, 1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0]), + 2, + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("vector", vectors.data_type().clone(), false), + ArrowField::new("docs", docs.data_type().clone(), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..5)), + Arc::new(vectors), + docs, + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 3); + + dataset + .create_index( + &["docs"], + IndexType::Inverted, + None, + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + let query = match_query("target", Operator::Or); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [0, 3]); + assert_eq!(postfilter_ids(&dataset, query).await, [0, 3]); + + let query = match_query("target alpha", Operator::And); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [0]); + assert_eq!(postfilter_ids(&dataset, query).await, [0]); + + let query = match_query("target missing", Operator::And); + assert!(indexed_ids(&dataset, query.clone()).await.is_empty()); + assert!(postfilter_ids(&dataset, query).await.is_empty()); + + dataset + .create_index( + &["docs"], + IndexType::Inverted, + None, + &InvertedIndexParams::default().base_tokenizer("raw".to_owned()), + true, + ) + .await + .unwrap(); + let query = match_query("target", Operator::Or); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [3]); + assert_eq!(postfilter_ids(&dataset, query).await, [3]); + + dataset + .create_index( + &["docs"], + IndexType::Inverted, + None, + &InvertedIndexParams::code() + .split_identifiers(true) + .preserve_original(true), + true, + ) + .await + .unwrap(); + let query = match_query("targetAlpha", Operator::And); + assert_eq!(indexed_ids(&dataset, query.clone()).await, [0]); + assert_eq!(postfilter_ids(&dataset, query).await, [0]); + + let fuzzy_query = FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("targets".to_owned()) + .with_column(Some("docs".to_owned())) + .with_fuzziness(Some(1)), + )); + let query_vector = Float32Array::from(vec![0.0, 0.0]); + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &query_vector, 5) + .unwrap() + .prefilter(false) + .filter_query(QueryFilter::Fts(fuzzy_query)) + .unwrap(); + let error = scanner.try_into_batch().await.unwrap_err(); + assert!(matches!(&error, Error::NotSupported { .. })); + assert!( + error + .to_string() + .contains("Fuzzy MatchQuery is not supported when FTS is used as a post-filter"), + "unexpected error: {error}" + ); +} + #[tokio::test] async fn test_scan_limit_offset_preserves_json_extension_metadata() { let schema = Arc::new(ArrowSchema::new(vec![ diff --git a/rust/lance/src/dataset/tests/dataset_schema_evolution.rs b/rust/lance/src/dataset/tests/dataset_schema_evolution.rs index d311d62f555..c00f3c4cdaa 100644 --- a/rust/lance/src/dataset/tests/dataset_schema_evolution.rs +++ b/rust/lance/src/dataset/tests/dataset_schema_evolution.rs @@ -10,7 +10,7 @@ use arrow_array::{ use arrow_schema::{ DataType, Field as ArrowField, Field, Fields as ArrowFields, Fields, Schema as ArrowSchema, }; -use lance_encoding::version::LanceFileVersion; +use lance_file::version::LanceFileVersion; use rstest::rstest; use std::collections::HashMap; use std::sync::Arc; diff --git a/rust/lance/src/dataset/tests/dataset_transactions.rs b/rust/lance/src/dataset/tests/dataset_transactions.rs index 3e2a4caa3b3..572d13524cd 100644 --- a/rust/lance/src/dataset/tests/dataset_transactions.rs +++ b/rust/lance/src/dataset/tests/dataset_transactions.rs @@ -6,20 +6,37 @@ use std::sync::Arc; use std::vec; use crate::dataset::builder::DatasetBuilder; -use crate::dataset::transaction::{Operation, Transaction}; -use crate::dataset::{ManifestWriteConfig, TRANSACTIONS_DIR, write_manifest_file}; +use crate::dataset::transaction::{Operation, Transaction, UpdateMode, UpdatedFragmentOffsets}; +use crate::dataset::{ + ColumnAlteration, ManifestWriteConfig, NewColumnTransform, TRANSACTIONS_DIR, + write_manifest_file, +}; use crate::io::ObjectStoreParams; use crate::session::Session; use crate::{Dataset, Result}; +use lance_file::version::LanceFileVersion; +use lance_table::feature_flags::FLAG_COVERED_INDEX_METADATA; +use lance_table::format::IndexMetadata; use lance_table::io::commit::ManifestNamingScheme; +use roaring::RoaringBitmap; +use uuid::Uuid; use crate::dataset::write::{CommitBuilder, InsertBuilder, WriteMode, WriteParams}; use crate::index::DatasetIndexExt; use arrow_array::Array; use arrow_array::RecordBatch; -use arrow_array::{Int32Array, RecordBatchIterator, StringArray, types::Int32Type}; -use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_array::{ + Int32Array, RecordBatchIterator, StringArray, StructArray, + types::{Int32Type, Int64Type}, +}; +use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; +use lance_core::Error; +use lance_core::datatypes::{Field as LanceCoreField, LogicalType, Schema as LanceSchema}; +use lance_core::utils::address::RowAddress; use lance_core::utils::tempfile::{TempDir, TempStrDir}; +use lance_core::{ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_LAST_UPDATED_AT_VERSION}; use lance_datagen::{BatchCount, RowCount, array}; use crate::datafusion::LanceTableProvider; @@ -208,6 +225,35 @@ async fn test_session_store_registry() { assert_eq!(registry.active_stores().len(), 0); } +#[test] +fn test_decode_inline_transaction_tolerates_unknown_operations() { + use crate::dataset::decode_inline_transaction; + use lance_table::format::pb; + use prost::Message; + + // A transaction written by a newer version of Lance may carry an operation + // this version cannot decode; prost surfaces it as a missing oneof. This + // must not fail (it would prevent opening the dataset), only skip caching. + let unknown_operation = pb::Transaction { + read_version: 1, + uuid: "test".to_string(), + ..Default::default() + }; + assert!(decode_inline_transaction(&unknown_operation.encode_to_vec(), 42).is_none()); + + // Corrupt bytes are likewise tolerated. + assert!(decode_inline_transaction(&[0xff, 0xff, 0xff], 42).is_none()); + + // A decodable transaction is returned. + let known = pb::Transaction::from(&Transaction::new( + 1, + Operation::Append { fragments: vec![] }, + None, + )); + let decoded = decode_inline_transaction(&known.encode_to_vec(), 42).unwrap(); + assert!(matches!(decoded.operation, Operation::Append { .. })); +} + #[tokio::test] async fn test_migrate_v2_manifest_paths() { let test_uri = TempStrDir::default(); @@ -272,6 +318,39 @@ pub(super) fn assert_results( ) } +fn gen_rows() -> impl arrow_array::RecordBatchReader + Send + 'static { + lance_datagen::gen_batch() + .col("key", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)) +} + +/// Write a dataset with `versions` versions of 10 rows each. +async fn write_versions(uri: &str, versions: usize, enable_v2_manifest_paths: bool) -> Dataset { + let mut ds = Dataset::write( + gen_rows(), + uri, + Some(WriteParams { + enable_v2_manifest_paths, + ..Default::default() + }), + ) + .await + .unwrap(); + for _ in 1..versions { + ds.append( + gen_rows(), + Some(WriteParams { + mode: WriteMode::Append, + enable_v2_manifest_paths, + ..Default::default() + }), + ) + .await + .unwrap(); + } + ds +} + #[tokio::test] async fn test_inline_transaction() { use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator}; @@ -349,16 +428,19 @@ async fn test_inline_transaction() { // Case 3: manifest does not contain inline transaction, read should fall back to external transaction file let ds = create_dataset(2).await; let tx = make_tx(ds.manifest().version); - let tx_file = - crate::io::commit::write_transaction_file(ds.object_store.as_ref(), &ds.base, &tx) - .await - .unwrap(); + let tx_file = crate::io::commit::write_transaction_file( + ds.object_store.as_ref(), + &ds.base, + &lance_table::format::pb::Transaction::from(&tx), + ) + .await + .unwrap(); let (mut manifest, indices) = tx .build_manifest( Some(ds.manifest.as_ref()), ds.load_indices().await.unwrap().as_ref().clone(), &tx_file, - &ManifestWriteConfig::default(), + &ManifestWriteConfig::default().to_build_config(), ) .unwrap(); let location = write_manifest_file( @@ -374,6 +456,8 @@ async fn test_inline_transaction() { &ManifestWriteConfig::default(), ds.manifest_location.naming_scheme, None, + // Previously classified from a None inline copy, which validated. + true, ) .await .unwrap(); @@ -382,6 +466,197 @@ async fn test_inline_transaction() { assert!(ds_new.manifest.transaction_file.is_some()); let read_tx = ds_new.read_transaction().await.unwrap().unwrap(); assert_eq!(read_tx, tx); + + // The direct read takes the same external-file fallback. + let version_transaction = ds_new + .read_version_transaction(location.version) + .await + .unwrap(); + assert_eq!(version_transaction.transaction, Some(tx)); +} + +#[tokio::test] +async fn test_read_version_transaction_does_not_populate_caches() { + use lance_index::IndexType; + use lance_index::scalar::ScalarIndexParams; + + let test_uri = TempStrDir::default(); + let mut dataset = write_versions(&test_uri, 1, true).await; + // Index the table so historical manifests carry an IndexSection that a + // caching read path would decode. + dataset + .create_index( + &["key"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); // version 2 + for _ in 0..18 { + dataset + .append( + gen_rows(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + } + let latest_version = dataset.version().version; + assert_eq!(latest_version, 20); + + // Fresh session so any cache insertion by the API under test shows as growth. + let session = Arc::new(Session::default()); + let dataset = DatasetBuilder::from_uri(&test_uri) + .with_session(session.clone()) + .load() + .await + .unwrap(); + + let metadata_stats_before = session.metadata_cache_stats().await; + let index_stats_before = session.index_cache_stats().await; + + let mut actual = Vec::with_capacity(latest_version as usize); + for version in 1..=latest_version { + let version_transaction = dataset.read_version_transaction(version).await.unwrap(); + assert_eq!(version_transaction.version, version); + actual.push(version_transaction); + } + + let metadata_stats_after = session.metadata_cache_stats().await; + let index_stats_after = session.index_cache_stats().await; + assert_eq!( + metadata_stats_after.num_entries, + metadata_stats_before.num_entries + ); + assert_eq!( + metadata_stats_after.size_bytes, + metadata_stats_before.size_bytes + ); + assert_eq!( + index_stats_after.num_entries, + index_stats_before.num_entries + ); + assert_eq!(index_stats_after.size_bytes, index_stats_before.size_bytes); + + // Results match a full checkout. + for version_transaction in &actual { + let checked_out = dataset + .checkout_version(version_transaction.version) + .await + .unwrap(); + assert_eq!( + version_transaction.transaction, + checked_out.read_transaction().await.unwrap() + ); + assert_eq!( + version_transaction.timestamp, + checked_out.version().timestamp + ); + assert!(version_transaction.transaction.is_some()); + } + + // A missing (e.g. cleaned up) version errors as DatasetNotFound, matching + // the historical checkout_version-based contract of the public API. + let err = dataset.read_version_transaction(9999).await.unwrap_err(); + assert!( + matches!(err, crate::Error::DatasetNotFound { .. }), + "expected DatasetNotFound for a missing version, got {err:?}" + ); +} + +#[tokio::test] +async fn test_read_transaction_recovers_from_stale_manifest_size() { + let test_uri = TempStrDir::default(); + let ds = write_versions(&test_uri, 1, true).await; + let manifest = ds.manifest().clone(); + // Only meaningful for the inline path; a plain write inlines the transaction. + assert!(manifest.transaction_section.is_some()); + + // A size at/under the transaction offset makes the first read_message fail + // "file size is too small"; only the retry at the true size can recover. + let mut stale = ds.manifest_location().clone(); + stale.size = Some(1); + let recovered = ds + .read_transaction_from_storage(&manifest, &stale) + .await + .unwrap(); + assert_eq!(recovered, ds.read_transaction().await.unwrap()); + assert!(recovered.is_some()); +} + +#[tokio::test] +async fn test_read_version_transaction_v1_manifest_naming() { + let test_uri = TempStrDir::default(); + let ds = write_versions(&test_uri, 3, false).await; + assert_eq!( + ds.manifest_location().naming_scheme, + ManifestNamingScheme::V1 + ); + + for version in 1..=3 { + let version_transaction = ds.read_version_transaction(version).await.unwrap(); + let checked_out = ds.checkout_version(version).await.unwrap(); + assert_eq!( + version_transaction.transaction, + checked_out.read_transaction().await.unwrap() + ); + assert_eq!( + version_transaction.timestamp, + checked_out.version().timestamp + ); + } +} + +#[tokio::test] +async fn test_read_version_transaction_on_branch() { + let test_uri = TempStrDir::default(); + let mut main_ds = write_versions(&test_uri, 1, true).await; + let branch_ds = main_ds.create_branch("dev", 1, None).await.unwrap(); + + // Commit on the branch. + let branch_ds = Dataset::write( + gen_rows(), + branch_ds.uri(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(branch_ds.manifest().branch.as_deref(), Some("dev")); + + // Versions resolve against the branch chain and match a full checkout. + for version in branch_ds.versions().await.unwrap() { + let version_transaction = branch_ds + .read_version_transaction(version.version) + .await + .unwrap(); + assert_eq!(version_transaction.version, version.version); + assert_eq!(version_transaction.timestamp, version.timestamp); + let checked_out = branch_ds.checkout_version(version.version).await.unwrap(); + assert_eq!(checked_out.manifest().branch.as_deref(), Some("dev")); + assert_eq!( + version_transaction.transaction, + checked_out.read_transaction().await.unwrap() + ); + } + + // The append on the branch is the branch's own transaction. + let latest = branch_ds.version().version; + let version_transaction = branch_ds.read_version_transaction(latest).await.unwrap(); + assert!(matches!( + version_transaction.transaction, + Some(Transaction { + operation: Operation::Append { .. }, + .. + }) + )); } #[tokio::test] @@ -445,6 +720,16 @@ async fn test_list_detached_manifests() { // Now there should be one detached manifest let detached = dataset.list_detached_manifests().await.unwrap(); assert_eq!(detached.len(), 1); + assert_eq!( + dataset + .version_refs() + .await + .unwrap() + .iter() + .map(|version| version.version) + .collect::>(), + vec![1] + ); // The detached version should have the high bit set let detached_version = detached[0].version; @@ -502,3 +787,939 @@ async fn test_list_detached_manifests() { assert_eq!(versions.len(), 1); assert_eq!(versions[0].version, 1); } + +/// Transaction properties large enough to push the transaction over the +/// inline threshold. +fn large_props(key: &str) -> Option>> { + use crate::io::commit::MAX_INLINE_TRANSACTION_BYTES; + let mut props = HashMap::new(); + props.insert( + key.to_string(), + "x".repeat(2 * MAX_INLINE_TRANSACTION_BYTES), + ); + Some(Arc::new(props)) +} + +fn spill_test_batch() -> (Arc, RecordBatch) { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from_iter_values(0..10))], + ) + .unwrap(); + (schema, batch) +} + +/// Load the dataset with a fresh session so assertions hit storage, not caches. +async fn reopen(uri: &str) -> Dataset { + DatasetBuilder::from_uri(uri) + .with_session(Arc::new(Session::default())) + .load() + .await + .unwrap() +} + +#[tokio::test] +async fn test_large_transaction_spills_to_external_file() { + use crate::io::commit::MAX_INLINE_TRANSACTION_BYTES; + + let (schema, batch) = spill_test_batch(); + let test_uri = TempStrDir::default(); + + // New-dataset commit path: a transaction too large to inline is written + // only to the external transaction file. + Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], schema.clone()), + &test_uri, + Some(WriteParams { + transaction_properties: large_props("payload"), + ..Default::default() + }), + ) + .await + .unwrap(); + let ds = reopen(&test_uri).await; + assert!(ds.manifest.transaction_section.is_none()); + assert!(matches!(ds.manifest.transaction_file.as_deref(), Some(f) if !f.is_empty())); + let tx = ds.read_transaction().await.unwrap().unwrap(); + assert_eq!( + tx.transaction_properties + .unwrap() + .get("payload") + .unwrap() + .len(), + 2 * MAX_INLINE_TRANSACTION_BYTES + ); + + // Normal commit path: a small append is still inlined. + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let ds = reopen(&test_uri).await; + assert!(ds.manifest.transaction_section.is_some()); +} + +#[tokio::test] +async fn test_spilled_restore_and_deep_clone_read_own_transaction() { + // Restore and deep clone both rebuild the new manifest from an existing + // manifest file, inheriting its inline transaction offset and external + // transaction file name. When the new transaction is too large to inline, + // readers fall back to exactly those fields, so the stale inherited values + // must not leak into the new manifest. + use crate::dataset::transaction::TransactionBuilder; + + let (schema, batch) = spill_test_batch(); + let source_uri = TempStrDir::default(); + Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], schema.clone()), + &source_uri, + None, + ) + .await + .unwrap(); + let ds = Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + &source_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + // The manifests restored from / cloned from below carry an inline + // transaction offset and their own transaction file. + assert!(ds.manifest.transaction_section.is_some()); + let source_version = ds.manifest().version; + let source_ref_path = ds.uri().to_string(); + let source_tx_file = ds.manifest.transaction_file.clone(); + assert!(matches!(source_tx_file.as_deref(), Some(f) if !f.is_empty())); + + // Restore with a spilled transaction: the stale inline offset must be + // cleared and the transaction read back from the external file. + let restore_tx = TransactionBuilder::new(source_version, Operation::Restore { version: 1 }) + .transaction_properties(large_props("payload")) + .build(); + CommitBuilder::new(Arc::new(ds)) + .execute(restore_tx.clone()) + .await + .unwrap(); + let restored = reopen(&source_uri).await; + assert!(restored.manifest.transaction_section.is_none()); + assert_eq!( + restored.read_transaction().await.unwrap().unwrap(), + restore_tx + ); + + // Deep clone with a spilled transaction: the manifest must reference the + // clone's own transaction file, not the source's. + let clone_tx = TransactionBuilder::new( + source_version, + Operation::Clone { + is_shallow: false, + ref_name: None, + ref_version: source_version, + ref_path: source_ref_path, + branch_name: None, + }, + ) + .transaction_properties(large_props("payload")) + .build(); + let clone_uri = TempStrDir::default(); + CommitBuilder::new(&clone_uri) + .execute(clone_tx.clone()) + .await + .unwrap(); + let cloned = reopen(&clone_uri).await; + assert!(cloned.manifest.transaction_section.is_none()); + assert_ne!(cloned.manifest.transaction_file, source_tx_file); + assert_eq!(cloned.read_transaction().await.unwrap().unwrap(), clone_tx); +} + +/// Partial RewriteColumns refresh in `build_manifest`: only matched physical +/// rows get `last_updated_at_version` bumped; same-fragment unmatched rows and +/// untouched fragments keep both version sequences. +#[tokio::test] +async fn test_build_manifest_partial_last_updated_rewrite_columns_stable_row_ids() { + let dir = TempStrDir::default(); + let uri = dir.as_str(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("x", DataType::Int32, false), + ])); + let batch0 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..8)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader0 = RecordBatchIterator::new(vec![Ok(batch0)], schema.clone()); + let write_params = WriteParams { + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::Stable), + ..Default::default() + }; + let mut dataset = Dataset::write(reader0, uri, Some(write_params)) + .await + .unwrap(); + + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(100..108)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader1 = RecordBatchIterator::new(vec![Ok(batch1)], schema.clone()); + dataset.append(reader1, None).await.unwrap(); + + let frags = dataset.get_fragments(); + assert_eq!( + frags.len(), + 2, + "expected two fragments (append creates a new fragment)" + ); + + async fn scan_row_versions(ds: &Dataset) -> HashMap<(u32, u32), (u64, u64)> { + let mut scanner = ds.scan(); + scanner + .project(&[ + ROW_ADDR, + ROW_LAST_UPDATED_AT_VERSION, + ROW_CREATED_AT_VERSION, + ]) + .unwrap(); + let batches = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut out = HashMap::new(); + for batch in batches { + let addrs = batch + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + let last = batch + .column_by_name(ROW_LAST_UPDATED_AT_VERSION) + .unwrap() + .as_primitive::(); + let created = batch + .column_by_name(ROW_CREATED_AT_VERSION) + .unwrap() + .as_primitive::(); + for row in 0..batch.num_rows() { + let addr = RowAddress::from(addrs.value(row)); + out.insert( + (addr.fragment_id(), addr.row_offset()), + (last.value(row), created.value(row)), + ); + } + } + out + } + + let before = scan_row_versions(&dataset).await; + assert_eq!(before.len(), 16); + + // Update only rows i in {2, 4, 6} within fragment 0 (physical offsets 2, 4, 6). + let update_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("x", DataType::Int32, false), + ])); + let update_batch = RecordBatch::try_new( + update_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![2, 4, 6])), + Arc::new(Int32Array::from(vec![99, 99, 99])), + ], + ) + .unwrap(); + let right: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)].into_iter(), + update_schema, + )); + + let mut frag0 = dataset.get_fragment(0).unwrap(); + let u = frag0 + .update_columns_with_offsets(right, "i", "i") + .await + .unwrap(); + assert_eq!(u.matched_offsets.iter().count(), 3); + for off in [2_u32, 4, 6] { + assert!(u.matched_offsets.contains(off)); + } + + let updated_fragment_offsets = Some(UpdatedFragmentOffsets(HashMap::from([( + u.fragment.id, + u.matched_offsets, + )]))); + + let op = Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![u.fragment], + new_fragments: vec![], + fields_modified: u.fields_modified, + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets, + }; + + let read_v = dataset.version().version; + let dataset = Dataset::commit( + uri, + op, + Some(read_v), + None, + None, + Arc::new(Session::default()), + true, + ) + .await + .unwrap(); + + let new_v = dataset.version().version; + assert_eq!(new_v, read_v + 1); + + let after = scan_row_versions(&dataset).await; + for off in 0..8_u32 { + let key = (0, off); + let (last_before, created_before) = before[&key]; + let (last_after, created_after) = after[&key]; + assert_eq!(created_after, created_before); + if off == 2 || off == 4 || off == 6 { + assert_eq!( + last_after, new_v, + "matched row offset {off} should advance last_updated to new version" + ); + } else { + assert_eq!( + last_after, last_before, + "unmatched row offset {off} in fragment 0 should keep last_updated" + ); + } + } + + for off in 0..8_u32 { + let key = (1, off); + assert_eq!( + after[&key], before[&key], + "fragment 1 row offset {off}: both version columns unchanged" + ); + } +} + +/// Repro shape for issue 7700: write a, b, c; drop one column; add d. The +/// dropped id stays referenced by the data files and max_field_id stays 3. +async fn dataset_with_dropped_column(uri: &str, dropped: &str) -> Dataset { + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ArrowField::new("c", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(Int32Array::from(vec![10, 20])), + Arc::new(Int32Array::from(vec![100, 200])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema); + let mut dataset = Dataset::write(reader, uri, None).await.unwrap(); + dataset.drop_columns(&[dropped]).await.unwrap(); + dataset + .add_columns( + NewColumnTransform::SqlExpressions(vec![("d".into(), "CAST(5 AS INT)".into())]), + None, + None, + ) + .await + .unwrap(); + let dropped_id = ["a", "b", "c"].iter().position(|c| *c == dropped).unwrap() as i32; + let mut expected_ids: Vec = (0..3).filter(|id| *id != dropped_id).collect(); + expected_ids.push(3); + assert_eq!(dataset.schema().field_ids(), expected_ids); + assert_eq!(dataset.manifest.max_field_id(), 3); + dataset +} + +/// Expected values of every column surviving `dropped`, plus d. +fn surviving_columns(dropped: &str) -> Vec<(&'static str, [i32; 2])> { + [ + ("a", [1, 2]), + ("b", [10, 20]), + ("c", [100, 200]), + ("d", [5, 5]), + ] + .into_iter() + .filter(|(name, _)| *name != dropped) + .collect() +} + +fn assert_columns(batch: &RecordBatch, cols: &[(&str, [i32; 2])]) { + for (name, expected) in cols { + let col = &batch[*name]; + assert_eq!( + col.as_primitive::().values(), + expected, + "column {}", + name + ); + } +} + +async fn commit_merge(dataset: &Dataset, schema: LanceSchema) -> Result { + let fragments = dataset + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + Dataset::commit( + Arc::new(dataset.clone()), + Operation::Merge { + fragments, + schema, + preserves_nullability: true, + }, + Some(dataset.manifest.version), + None, + None, + dataset.session(), + false, + ) + .await +} + +// Which clause rejects the lossy round-trip depends on the hole's +// position: a hole before the last field remaps a shared id, while a +// hole at the end reuses the dropped id for the new field. +#[rstest::rstest] +#[case::drop_a_remaps_shared_id("a", "remaps field id 1 from \"b\" to \"c\"")] +#[case::drop_b_remaps_shared_id("b", "remaps field id 2 from \"c\" to \"d\"")] +#[case::drop_c_reuses_dropped_id("c", "assigns id 2 to new field \"d\"")] +#[tokio::test] +async fn test_merge_rejects_renumbered_field_ids(#[case] dropped: &str, #[case] expected: &str) { + let dataset = dataset_with_dropped_column("memory://", dropped).await; + + let arrow_schema = ArrowSchema::from(dataset.schema()); + let renumbered = LanceSchema::try_from(&arrow_schema).unwrap(); + assert_eq!(renumbered.field_ids(), vec![0, 1, 2]); + + let err = commit_merge(&dataset, renumbered).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + let message = err.to_string(); + assert!(message.contains(expected), "unexpected error: {}", message); +} + +#[tokio::test] +async fn test_merge_rejects_dropped_field_id_reuse() { + // Deliberate reuse of a tombstoned id, as opposed to the renumbering + // accident covered above. + let dataset = dataset_with_dropped_column("memory://", "b").await; + + let mut schema = dataset.schema().clone(); + let mut field = LanceCoreField::try_from(&ArrowField::new("e", DataType::Int32, true)).unwrap(); + field.id = 1; + schema.fields.push(field); + + let err = commit_merge(&dataset, schema).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + let message = err.to_string(); + assert!( + message.contains("assigns id 1 to new field \"e\"") + && message.contains("must use ids of at least 4"), + "unexpected error: {}", + message + ); +} + +#[tokio::test] +async fn test_merge_rejects_renumbered_nested_field_ids() { + // A hole inside a struct shifts a nested leaf's id onto a field + // outside the struct on renumbering; the full-path comparison must + // catch the cross-parent remap. + let struct_fields = Fields::from(vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ]); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("s", DataType::Struct(struct_fields.clone()), true), + ArrowField::new("z", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + arrow_schema.clone(), + vec![ + Arc::new(StructArray::new( + struct_fields, + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(Int32Array::from(vec![10, 20])), + ], + None, + )), + Arc::new(Int32Array::from(vec![100, 200])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], arrow_schema); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + dataset.drop_columns(&["s.x"]).await.unwrap(); + assert_eq!(dataset.schema().field_ids(), vec![0, 2, 3]); + + let arrow_schema = ArrowSchema::from(dataset.schema()); + let renumbered = LanceSchema::try_from(&arrow_schema).unwrap(); + assert_eq!(renumbered.field_ids(), vec![0, 1, 2]); + + let err = commit_merge(&dataset, renumbered).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + let message = err.to_string(); + assert!( + message.contains("remaps field id 2 from \"s.y\" to \"z\""), + "unexpected error: {}", + message + ); +} + +#[rstest::rstest] +#[case::drop_a("a")] +#[case::drop_b("b")] +#[case::drop_c("c")] +#[tokio::test] +async fn test_merge_allows_id_preserving_schema_change(#[case] dropped: &str) { + let dataset = dataset_with_dropped_column("memory://", dropped).await; + + let survivors = surviving_columns(dropped); + let first_id = dataset.schema().field(survivors[0].0).unwrap().id; + let mut schema = dataset.schema().clone(); + schema + .mut_field_by_id(first_id) + .unwrap() + .metadata + .insert("wm".into(), "42".into()); + + let dataset = commit_merge(&dataset, schema).await.unwrap(); + assert_eq!( + dataset + .schema() + .field(survivors[0].0) + .unwrap() + .metadata + .get("wm"), + Some(&"42".to_string()) + ); + + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_columns(&batch, &survivors); +} + +#[rstest::rstest] +#[case::drop_a("a")] +#[case::drop_b("b")] +#[case::drop_c("c")] +#[tokio::test] +async fn test_merge_allows_dropping_field(#[case] dropped: &str) { + let dataset = dataset_with_dropped_column("memory://", dropped).await; + + let mut survivors = surviving_columns(dropped); + let omitted = survivors.remove(0); + let names: Vec<&str> = survivors.iter().map(|(n, _)| *n).collect(); + let schema = dataset.schema().project(&names).unwrap(); + + let dataset = commit_merge(&dataset, schema).await.unwrap(); + assert!(dataset.schema().field(omitted.0).is_none()); + + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_columns(&batch, &survivors); +} + +#[tokio::test] +async fn test_merge_rejects_schema_only_path_remap() { + let dataset = dataset_with_dropped_column("memory://", "c").await; + let prior_id = dataset.schema().field("a").unwrap().id; + let fresh_id = dataset.manifest.max_field_id() + 1; + + let mut schema = dataset.schema().clone(); + schema.mut_field_by_id(prior_id).unwrap().id = fresh_id; + + let err = commit_merge(&dataset, schema).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + let message = err.to_string(); + assert!( + message.contains(&format!( + "remaps existing field \"a\" from id {} to id {}", + prior_id, fresh_id + )) && message.contains("base data file"), + "unexpected error: {}", + message + ); +} + +#[rstest::rstest] +#[case::logical_type(DataType::Float32, true, "logical type")] +#[case::nullability(DataType::Int32, false, "nullable")] +#[tokio::test] +async fn test_merge_rejects_shared_id_type_or_nullability_change( + #[case] data_type: DataType, + #[case] nullable: bool, + #[case] expected: &str, +) { + let dataset = dataset_with_dropped_column("memory://", "c").await; + let field_id = dataset.schema().field("a").unwrap().id; + + let mut schema = dataset.schema().clone(); + let field = schema.mut_field_by_id(field_id).unwrap(); + field.logical_type = LogicalType::try_from(&data_type).unwrap(); + field.nullable = nullable; + + let err = commit_merge(&dataset, schema).await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "got {:?}", err); + let message = err.to_string(); + assert!( + message.contains(&format!("changes field id {} (\"a\")", field_id)) + && message.contains(expected), + "unexpected error: {}", + message + ); +} + +/// The pure schema/fragment-shape half of this check lives in +/// `lance_table::transaction`'s `test_merge_allows_rewritten_fresh_field_id`; +/// this covers the `Dataset`-level effect of a rewrite that assigns a fresh +/// field id. +#[tokio::test] +async fn test_alter_columns_materializes_fresh_field_id_in_every_fragment() { + let mut dataset = dataset_with_dropped_column("memory://", "c").await; + let prior_id = dataset.schema().field("a").unwrap().id; + dataset + .alter_columns(&[ColumnAlteration::new("a".into()).cast_to(DataType::Int64)]) + .await + .unwrap(); + let new_id = dataset.schema().field("a").unwrap().id; + assert_ne!(new_id, prior_id); + assert!( + dataset.get_fragments().iter().all(|fragment| { + fragment + .metadata() + .files + .iter() + .any(|file| file.fields.contains(&new_id)) + }), + "alter_columns must materialize the fresh id in every fragment base file" + ); + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!(batch["a"].as_primitive::().values(), &[1, 2]); +} + +/// A covering declaration that is not a suffix of `fields` must be refused +/// at commit, not silently accepted and later misread as a keyed column. +#[tokio::test] +async fn test_create_index_rejects_non_suffix_covering_fields() { + let dir = TempStrDir::default(); + let uri = dir.as_str(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("x", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..8)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, None).await.unwrap(); + + // fields = [0, 1] with covering = [0]: field 0 is the leading entry, + // not the trailing one, so this claims the keyed column is covered. + let bad_index = IndexMetadata { + uuid: Uuid::new_v4(), + name: "bad_idx".to_string(), + fields: vec![0, 1], + covering_fields: vec![0], + dataset_version: dataset.manifest.version, + fragment_bitmap: Some(RoaringBitmap::from_iter([0u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![bad_index], + removed_indices: vec![], + }, + None, + ); + + let err = dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .expect_err("a non-suffix covering declaration must not commit"); + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected InvalidInput, got {err:?}" + ); + assert!( + err.to_string().contains("must come last"), + "unexpected message: {err}" + ); +} + +/// A shallow clone copies the index metadata wholesale, `covering_fields` +/// included, but `Manifest::shallow_clone` builds the new manifest directly -- +/// `Operation::Clone` is refused by `build_manifest` -- so the fence is not +/// recomputed there and has to be carried explicitly. Without that, a clone of +/// a covered table comes back unfenced and a build predating covering can open +/// it and read carried columns as keyed ones. +#[tokio::test] +async fn test_shallow_clone_preserves_the_covering_fence() { + let dir = TempStrDir::default(); + let uri = dir.as_str(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("x", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..8)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, None).await.unwrap(); + let index = IndexMetadata { + uuid: Uuid::new_v4(), + name: "covered_idx".to_string(), + fields: vec![0, 1], + covering_fields: vec![1], + dataset_version: dataset.manifest.version, + fragment_bitmap: Some(RoaringBitmap::from_iter([0u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index], + removed_indices: vec![], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + assert_ne!( + dataset.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "precondition: the source table is fenced" + ); + + let clone_dir = TempStrDir::default(); + let cloned = dataset + .shallow_clone(clone_dir.as_str(), dataset.version().version, None) + .await + .unwrap(); + + // The clone really does carry the covering declaration, so it really does + // need the fence -- assert that first, or the flag check below could pass + // for a clone that simply dropped the index. + let cloned_indices = cloned.load_indices().await.unwrap(); + assert_eq!( + cloned_indices + .iter() + .find(|i| i.name == "covered_idx") + .map(|i| i.covering_fields.clone()), + Some(vec![1]), + "precondition: the clone carries the covering declaration" + ); + assert_ne!( + cloned.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "a clone of a covered table must stay fenced for readers" + ); + assert_ne!( + cloned.manifest.writer_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "a clone of a covered table must stay fenced for writers" + ); +} + +/// Covering redefines what `fields` means, so a build that predates it would +/// select a vector index by membership of `fields` and answer a query on a +/// merely-carried column with an index keyed on a different one. The fence is +/// the feature flag: a covering commit must set it in both words so such a +/// build refuses the table outright, and dropping the last covering index +/// must clear it again rather than fence the table forever. +#[tokio::test] +async fn test_covering_commit_fences_the_table_with_a_feature_flag() { + let dir = TempStrDir::default(); + let uri = dir.as_str(); + + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("x", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..8)), + Arc::new(Int32Array::from(vec![0_i32; 8])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, uri, None).await.unwrap(); + + assert_eq!( + dataset.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "precondition: a plain dataset carries no covering fence" + ); + + let index = IndexMetadata { + uuid: Uuid::new_v4(), + name: "covered_idx".to_string(), + fields: vec![0, 1], + covering_fields: vec![1], + dataset_version: dataset.manifest.version, + fragment_bitmap: Some(RoaringBitmap::from_iter([0u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let uuid = index.uuid; + + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index.clone()], + removed_indices: vec![], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + // Both words: a reader would select the wrong index, a writer would + // mismaintain it. + assert_ne!( + dataset.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "a covering commit must fence readers" + ); + assert_ne!( + dataset.manifest.writer_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "a covering commit must fence writers" + ); + + // The flag has to survive the reload, not just the in-memory manifest: + // `apply_feature_flags` runs a second time in `write_manifest_file` and + // resets both words. + let reopened = Dataset::open(uri).await.unwrap(); + assert_ne!( + reopened.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "the fence must be persisted, not only set in memory" + ); + + // An ordinary commit that has nothing to do with indices must not drop the + // fence. `Manifest::new_from_previous` zeroes both flag words, so the bit + // survives only because `build_manifest` re-derives it from the surviving + // index list on every commit rather than inheriting it -- an append is the + // cheapest way to pin that. + let mut dataset = reopened; + let more = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(8..16)), + Arc::new(Int32Array::from(vec![1_i32; 8])), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(more)], schema.clone()), + None, + ) + .await + .unwrap(); + assert_ne!( + dataset.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "an ordinary append must not lift the covering fence" + ); + assert_ne!( + dataset.manifest.writer_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "an ordinary append must not lift the writer half of the fence" + ); + + // Dropping the last covering index lifts the fence. Nothing clears the bit + // explicitly -- the words start zeroed and it is simply not set again -- so + // this is the pin against someone making the fence sticky via an inherit + // step, the way MemWAL catch-up is. + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![], + removed_indices: vec![IndexMetadata { uuid, ..index }], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + assert_eq!( + dataset.manifest.reader_feature_flags & FLAG_COVERED_INDEX_METADATA, + 0, + "dropping the last covering index must clear the fence" + ); +} diff --git a/rust/lance/src/dataset/tests/dataset_versioning.rs b/rust/lance/src/dataset/tests/dataset_versioning.rs index c04dd0f3183..eb9a350935a 100644 --- a/rust/lance/src/dataset/tests/dataset_versioning.rs +++ b/rust/lance/src/dataset/tests/dataset_versioning.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::collections::HashMap; use std::sync::Arc; use std::vec; @@ -14,14 +15,18 @@ use lance_table::io::commit::ManifestNamingScheme; use crate::dataset::write::{CommitBuilder, WriteMode, WriteParams}; use arrow_array::RecordBatch; use arrow_array::RecordBatchReader; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; use arrow_array::{RecordBatchIterator, UInt32Array, types::Int32Type}; use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; use lance_core::utils::tempfile::{TempDir, TempStdDir, TempStrDir}; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; use lance_file::version::LanceFileVersion; use mock_instant::thread_local::MockClock; +use tokio::sync::Barrier; use crate::dataset::refs::branch_contents_path; +use crate::utils::test::copy_test_data_to_tmp; use futures::TryStreamExt; use lance_core::Error; use object_store::path::Path; @@ -46,6 +51,23 @@ fn assert_all_manifests_use_scheme(test_dir: &TempStdDir, scheme: ManifestNaming ); } +#[tokio::test] +async fn test_list_manifest_locations_rejects_explicit_refs() { + let test_dir = TempStdDir::default(); + let test_uri = test_dir.to_str().unwrap(); + let builders = [ + DatasetBuilder::from_uri(test_uri).with_version(1), + DatasetBuilder::from_uri(test_uri).with_branch("dev", None), + DatasetBuilder::from_uri(test_uri).with_tag("release"), + ]; + + for builder in builders { + let err = builder.list_manifest_locations().await.unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. })); + assert!(err.to_string().contains("does not support an explicit")); + } +} + #[tokio::test] async fn test_v2_manifest_path_create() { // Can create a dataset, using V2 paths @@ -282,6 +304,94 @@ async fn test_stale_checks_cover_fast_successor_and_latest_version( assert!(historical.has_successor_version().await.unwrap()); } +/// All row ids visible in `dataset`, in scan order. +async fn scan_row_ids(dataset: &Dataset) -> Vec { + let batch = dataset + .scan() + .with_row_id() + .project(&["i"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + batch["_rowid"] + .as_primitive::() + .values() + .to_vec() +} + +fn u32_batch(values: std::ops::Range) -> RecordBatch { + arrow_array::record_batch!(("i", UInt32, values.collect::>())).unwrap() +} + +/// Restoring past activation would turn stable row ids off, putting row +/// addresses back into a namespace this table has already issued ids from. +#[tokio::test] +async fn test_restore_rejects_crossing_stable_id_activation() { + let test_uri = TempStrDir::default(); + let batch = u32_batch(0..10); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], batch.schema()), + test_uri.as_str(), + None, + ) + .await + .unwrap(); + dataset.migrate_to_stable_row_ids().await.unwrap(); + + let mut restored = dataset.checkout_version(1).await.unwrap(); + let err = restored.restore().await.unwrap_err(); + assert!( + err.to_string() + .contains("stable row ids were enabled after"), + "{err}" + ); +} + +/// A restore must not rewind the row-id high-water mark, or the next append reuses old ids. +#[tokio::test] +async fn test_restore_preserves_row_id_high_water_mark() { + let test_uri = TempStrDir::default(); + let write = |values: std::ops::Range, mode| { + let uri = test_uri.as_str().to_string(); + async move { + let batch = u32_batch(values); + Dataset::write( + RecordBatchIterator::new([Ok(batch.clone())], batch.schema()), + &uri, + Some(WriteParams { + mode, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap() + } + }; + + write(0..10, WriteMode::Create).await; + let appended = write(10..20, WriteMode::Append).await; + let mark = appended.manifest.next_row_id; + + let mut restored = appended.checkout_version(1).await.unwrap(); + restored.restore().await.unwrap(); + assert!( + restored.manifest.next_row_id >= mark, + "restore rewound the row id high-water mark: {} < {mark}", + restored.manifest.next_row_id + ); + + // The rows appended after the restore must not reuse the dropped rows' ids. + let reused = write(20..30, WriteMode::Append).await; + let ids = scan_row_ids(&reused).await; + assert_eq!( + ids.iter().filter(|id| **id >= mark).count(), + 10, + "appended rows did not all take fresh ids past {mark}: {ids:?}" + ); +} + #[rstest] #[tokio::test] async fn test_restore( @@ -506,6 +616,61 @@ async fn test_tag( assert_eq!(dataset.manifest.version, 1); } +#[tokio::test] +async fn test_concurrent_tag_creation_conflict() { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::UInt32, + false, + )])); + let data = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(UInt32Array::from_iter_values(0..10))], + ) + .unwrap(); + let test_uri = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(data)], schema), + &test_uri, + None, + ) + .await + .unwrap(); + dataset.delete("i >= 5").await.unwrap(); + + let dataset = Arc::new(dataset); + let concurrency = 32; + let barrier = Arc::new(Barrier::new(concurrency)); + let handles = (0..concurrency) + .map(|attempt| { + let dataset = dataset.clone(); + let barrier = barrier.clone(); + let version = (attempt % 2 + 1) as u64; + tokio::spawn(async move { + barrier.wait().await; + (version, dataset.tags().create("race", version).await) + }) + }) + .collect::>(); + + let mut successful_version = None; + let mut conflicts = 0; + for handle in handles { + let (version, result) = handle.await.unwrap(); + match result { + Ok(()) => successful_version = Some(version), + Err(Error::RefConflict { .. }) => conflicts += 1, + Err(error) => panic!("unexpected tag creation error: {error}"), + } + } + + assert_eq!(conflicts, concurrency - 1); + assert_eq!( + dataset.tags().get_version("race").await.unwrap(), + successful_version.unwrap() + ); +} + #[rstest] #[tokio::test] async fn test_fragment_id_zero_not_reused() { @@ -636,6 +801,152 @@ async fn test_fragment_id_never_reset() { assert_eq!(dataset.manifest.max_fragment_id(), Some(4)); } +#[tokio::test] +async fn test_overwrite_does_not_reuse_fragment_ids() { + // An overwrite replaces every fragment, but the ids it hands out must still + // continue from the dataset's high water mark: an id that named one set of + // rows must never name another. + let test_dir = TempStrDir::default(); + let write = |mode: WriteMode, rows: u64| { + let reader = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(rows), BatchCount::from(1)); + let params = WriteParams { + mode, + max_rows_per_file: 10, + ..Default::default() + }; + Dataset::write(reader, test_dir.as_str(), Some(params)) + }; + let fragment_ids = |dataset: &Dataset| { + dataset + .get_fragments() + .iter() + .map(|f| f.id()) + .collect::>() + }; + + let dataset = write(WriteMode::Create, 30).await.unwrap(); + assert_eq!(fragment_ids(&dataset), vec![0, 1, 2]); + + let dataset = write(WriteMode::Overwrite, 20).await.unwrap(); + assert_eq!(fragment_ids(&dataset), vec![3, 4]); + assert_eq!(dataset.manifest.max_fragment_id(), Some(4)); + + let dataset = write(WriteMode::Append, 10).await.unwrap(); + assert_eq!(fragment_ids(&dataset), vec![3, 4, 5]); +} + +#[rstest] +#[tokio::test] +async fn test_overwrite_rejects_fragment_with_deletion_file( + // Every form of the operation must be rejected: upserting config alongside + // the overwrite does not make renumbering the fragment any safer. + #[values(None, Some(HashMap::from([("key".to_string(), "value".to_string())])))] + config_upsert_values: Option>, +) { + // A deletion file's path embeds the fragment id, so it cannot follow its + // fragment to the fresh id an overwrite assigns. Such a fragment belongs to + // the dataset being replaced, so the operation is rejected rather than + // silently losing the deletions. + let test_dir = TempStrDir::default(); + let reader = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + dataset.delete("i < 3").await.unwrap(); + + let fragment = dataset.manifest.fragments[0].clone(); + assert!(fragment.deletion_file.is_some()); + let schema = dataset.schema().clone(); + let read_version = dataset.manifest.version; + + let err = CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + read_version, + Operation::Overwrite { + fragments: vec![fragment], + schema, + config_upsert_values, + initial_bases: None, + }, + None, + )) + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected invalid input, got {err:?}" + ); + assert!( + err.to_string().contains("must be newly written"), + "unexpected message: {err}" + ); +} + +#[tokio::test] +async fn test_commit_rejects_duplicate_fragment_ids() { + // Append honors an id a fragment arrives with, so a caller can still hand in + // one that an existing fragment already uses. Committing that would leave two + // fragments sharing everything keyed by the id. + let test_dir = TempStrDir::default(); + let write = |mode: WriteMode| { + let reader = gen_batch() + .col("i", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let params = WriteParams { + mode, + max_rows_per_file: 5, + ..Default::default() + }; + Dataset::write(reader, test_dir.as_str(), Some(params)) + }; + let dataset = write(WriteMode::Create).await.unwrap(); + let existing = dataset.manifest.fragments[1].clone(); + assert_eq!(existing.id, 1); + + let err = CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + 1, + Operation::Append { + fragments: vec![existing], + }, + None, + )) + .await + .unwrap_err(); + assert!( + err.to_string().contains("two fragments with id 1"), + "unexpected message: {err}" + ); +} + +#[tokio::test] +async fn test_commit_on_dataset_with_mixed_file_versions() { + // A v0.16 dataset that has both v1 and v2 files also has two fragments with + // id 1, because the id allocation of that era could hand out an id a caller + // had already supplied. The mixture is the more actionable diagnosis, so the + // duplicate check must not preempt it. + let test_dir = copy_test_data_to_tmp("v0.16.0/wrong_data_version_no_fix.lance").unwrap(); + let mut dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + let ids = dataset + .manifest + .fragments + .iter() + .map(|f| f.id) + .collect::>(); + assert_eq!(ids, vec![0, 1, 1, 2]); + + let err = dataset.delete("false").await.unwrap_err(); + assert!( + err.to_string() + .contains("The dataset contains a mixture of file versions"), + "unexpected message: {err}" + ); +} + /// create_branch and shallow_clone must read the SOURCE ref's chain, not the /// receiver's. Both chains get a version 2 with diverged row counts so a clone /// that wrongly resolves the version under the receiver succeeds silently with @@ -695,6 +1006,30 @@ async fn test_create_branch_and_shallow_clone_from_other_branch() { ); } +#[tokio::test] +async fn test_cannot_delete_branch_referenced_by_tag() { + let tempdir = TempDir::default(); + let test_uri = tempdir.path_str(); + let data = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(1), BatchCount::from(1)); + let mut dataset = Dataset::write(data, &test_uri, None).await.unwrap(); + let branch = dataset + .create_branch("dev", /*version=*/ 1, /*store_params=*/ None) + .await + .unwrap(); + dataset + .tags() + .create("keep-dev", ("dev", branch.version().version)) + .await + .unwrap(); + + let error = dataset.delete_branch("dev").await.unwrap_err(); + assert!(matches!(&error, Error::RefConflict { .. })); + assert!(error.to_string().contains("keep-dev")); + dataset.checkout_version("keep-dev").await.unwrap(); +} + #[tokio::test] async fn test_branch() { let tempdir = TempDir::default(); @@ -821,12 +1156,39 @@ async fn test_branch() { let (main_rows, _) = collect_rows(&main_dataset).await; assert_eq!(main_rows, 50); // only batch1 assert_eq!(main_dataset.version().version, 1); + let main_versions = main_dataset.version_refs().await.unwrap(); + assert_eq!( + main_versions + .iter() + .map(|version| version.version) + .collect::>(), + vec![1] + ); + assert_eq!( + main_dataset.latest_version_id().await.unwrap(), + main_versions.last().unwrap().version + ); // branch1 has data 1 + 2 (80 rows) let updated_branch1 = Dataset::open(branch1_dataset.uri()).await.unwrap(); let (branch1_rows, _) = collect_rows(&updated_branch1).await; assert_eq!(branch1_rows, 80); // batch1+batch2 assert_eq!(updated_branch1.version().version, 2); + let _ = updated_branch1.object_store.as_ref().io_stats_incremental(); + let branch1_versions = updated_branch1.version_refs().await.unwrap(); + let io_stats = updated_branch1.object_store.as_ref().io_stats_incremental(); + assert_eq!(io_stats.read_bytes, 0); + assert_eq!( + branch1_versions + .iter() + .map(|version| version.version) + .collect::>(), + vec![1, 2] + ); + assert_eq!( + updated_branch1.latest_version_id().await.unwrap(), + branch1_versions.last().unwrap().version + ); // branch2 has data 1 + 2 + 3 (100 rows) let updated_branch2 = Dataset::open(branch2_dataset.uri()).await.unwrap(); @@ -1026,6 +1388,7 @@ async fn test_branch() { let cleaned_path = Path::parse(format!("{}/tree/feature", test_uri)).unwrap(); assert!(!dataset.object_store.exists(&cleaned_path).await.unwrap()); + dataset.tags().delete("tag1").await.unwrap(); dataset.delete_branch("dev/branch2").await.unwrap(); dataset.delete_branch("branch1").await.unwrap(); diff --git a/rust/lance/src/dataset/tests/fragment_validate_tombstones.rs b/rust/lance/src/dataset/tests/fragment_validate_tombstones.rs new file mode 100644 index 00000000000..41c672ff5e6 --- /dev/null +++ b/rust/lance/src/dataset/tests/fragment_validate_tombstones.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use arrow_array::{RecordBatchIterator, record_batch}; +use lance_file::version::LanceFileVersion; +use rstest::rstest; + +use crate::Dataset; +use crate::dataset::WriteParams; +use crate::dataset::fragment::FileFragment; + +/// Rewriting a column with `update_columns` tombstones the field in the file +/// that held it, leaving `-2` behind while a new file answers for it. +/// Validation has to accept that: a tombstone marks a superseded field, not a +/// corrupt one. Legacy files are checked by a separate sort rule, so both +/// storage versions are covered. +#[rstest] +#[case::legacy(LanceFileVersion::Legacy)] +#[case::stable(LanceFileVersion::Stable)] +#[tokio::test] +async fn test_validate_accepts_tombstoned_fields(#[case] version: LanceFileVersion) { + let batch = record_batch!(("i", Int32, [1, 2]), ("v", Int64, [10, 20])).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let dataset = Arc::new( + Dataset::write( + reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(version), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let update = record_batch!(("i1", Int32, [1, 2]), ("v", Int64, [99, 99])).unwrap(); + let right = RecordBatchIterator::new(vec![Ok(update.clone())], update.schema()); + + let mut fragment = dataset.get_fragments().into_iter().next().unwrap(); + let updated = fragment + .update_columns_with_offsets(right, "i", "i1") + .await + .unwrap(); + + let layout: Vec> = updated + .fragment + .files + .iter() + .map(|file| file.fields.as_ref().to_vec()) + .collect(); + assert_eq!(layout, vec![vec![0, -2], vec![1]]); + + FileFragment::new(dataset, updated.fragment) + .validate() + .await + .unwrap(); +} diff --git a/rust/lance/src/dataset/tests/fragment_write_columns.rs b/rust/lance/src/dataset/tests/fragment_write_columns.rs new file mode 100644 index 00000000000..606a972bceb --- /dev/null +++ b/rust/lance/src/dataset/tests/fragment_write_columns.rs @@ -0,0 +1,1025 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Per-fragment column writes: staging columns' data as a standalone file with +//! `FileFragment::write_columns`, and committing it as a `DataReplacement` +//! whose coverage may not line up with any single file -- the case a computed +//! column reaches once compaction folds it into a shared base file. + +use std::sync::Arc; + +use arrow::array::AsArray; +use arrow_array::types::{Int32Type, UInt64Type}; +use arrow_array::{ + Array, ArrayRef, FixedSizeListArray, Int32Array, ListArray, MapArray, RecordBatch, + RecordBatchIterator, StringArray, StructArray, +}; +use arrow_buffer::{NullBuffer, OffsetBuffer}; +use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; +use futures::{StreamExt, TryStreamExt, stream}; +use lance_core::datatypes::Schema as LanceSchema; +use lance_core::utils::tempfile::TempStrDir; +use lance_core::{Error, ROW_ID, ROW_LAST_UPDATED_AT_VERSION}; +use lance_encoding::constants::PACKED_STRUCT_META_KEY; +use lance_file::version::LanceFileVersion; +use rstest::rstest; + +use crate::dataset::optimize::{CompactionOptions, compact_files}; +use crate::dataset::schema_evolution::NewColumnTransform; +use crate::dataset::transaction::{DataReplacementGroup, Operation}; +use crate::dataset::write::WriteParams; +use crate::dataset::{WriteDestination, fragment::FileFragment}; +use crate::{Dataset, Result}; + +fn batch_of(fields: Vec, columns: Vec) -> RecordBatch { + RecordBatch::try_new(Arc::new(ArrowSchema::new(fields)), columns).unwrap() +} + +fn ints(values: Vec) -> ArrayRef { + Arc::new(Int32Array::from(values)) as ArrayRef +} + +async fn dataset_of(batch: RecordBatch, version: Option) -> Dataset { + let schema = batch.schema(); + let params = version.map(|data_storage_version| WriteParams { + data_storage_version: Some(data_storage_version), + ..Default::default() + }); + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + params, + ) + .await + .unwrap() +} + +/// A one-fragment dataset holding a single non-null `id` column of `[1, 2]`. +async fn id_dataset() -> Dataset { + id_dataset_of(2, 1024).await +} + +fn only_fragment(dataset: &Dataset) -> FileFragment { + dataset.get_fragments().into_iter().next().unwrap() +} + +/// Lance schema for a column the dataset does not define, with a fresh id. +fn new_column_schema(dataset: &Dataset, name: &str) -> LanceSchema { + let mut schema = LanceSchema::try_from(&ArrowSchema::new(vec![ArrowField::new( + name, + DataType::Int32, + true, + )])) + .unwrap(); + schema.fields[0].id = dataset.manifest.max_field_id() + 1; + schema +} + +/// Lance schema naming just the declared column `name`. +fn declared_schema(dataset: &Dataset, name: &str) -> LanceSchema { + LanceSchema { + fields: vec![dataset.schema().field(name).unwrap().clone()], + metadata: Default::default(), + } +} + +async fn stage( + dataset: &Dataset, + batch: RecordBatch, + schema: &LanceSchema, +) -> Result { + only_fragment(dataset) + .write_columns(stream::iter([Ok(batch)]), schema) + .await +} + +async fn commit(dataset: &Dataset, replacements: Vec) -> Result { + let read_version = dataset.manifest.version; + Dataset::commit( + WriteDestination::Dataset(Arc::new(dataset.clone())), + Operation::DataReplacement { replacements }, + Some(read_version), + None, + None, + Arc::new(Default::default()), + false, + ) + .await +} + +/// A multi-fragment dataset of `rows` sequential ids, with stable row ids so +/// replacements can be checked against row lineage. +async fn id_dataset_of(rows: i32, max_rows_per_file: usize) -> Dataset { + let batch = batch_of( + vec![ArrowField::new("id", DataType::Int32, false)], + vec![ints((1..=rows).collect())], + ); + let schema = batch.schema(); + Dataset::write( + RecordBatchIterator::new([Ok(batch)], schema), + "memory://", + Some(WriteParams { + max_rows_per_file, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap() +} + +async fn declare_all_null(dataset: &mut Dataset, name: &str) { + let arrow = Arc::new(ArrowSchema::new(vec![ArrowField::new( + name, + DataType::Int32, + true, + )])); + dataset + .add_columns(NewColumnTransform::AllNulls(arrow), None, None) + .await + .unwrap(); +} + +/// Stage `values` for an existing `column` of one fragment. +async fn stage_column( + dataset: &Dataset, + fragment_id: u64, + column: &str, + values: Vec, +) -> DataReplacementGroup { + let schema = declared_schema(dataset, column); + let batch = batch_of( + vec![ArrowField::new(column, DataType::Int32, true)], + vec![ints(values)], + ); + dataset + .get_fragments() + .into_iter() + .find(|fragment| fragment.id() as u64 == fragment_id) + .expect("fragment to stage for") + .write_columns(stream::iter([Ok(batch)]), &schema) + .await + .unwrap() +} + +fn values(batch: &RecordBatch, name: &str) -> Vec> { + let col = batch[name].as_primitive::(); + (0..batch.num_rows()) + .map(|i| (!col.is_null(i)).then(|| col.value(i))) + .collect() +} + +/// A `point` struct of two non-null Int32 children, packed or not. +fn point_schema(packed: bool) -> Arc { + let mut point = ArrowField::new("point", DataType::Struct(point_children()), false); + if packed { + point.set_metadata([(PACKED_STRUCT_META_KEY.to_string(), "true".to_string())].into()); + } + Arc::new(ArrowSchema::new(vec![point])) +} + +fn point_children() -> Fields { + Fields::from(vec![ + ArrowField::new("x", DataType::Int32, false), + ArrowField::new("y", DataType::Int32, false), + ]) +} + +/// `xs` and `ys` are matched to `schema`'s children by name, so a schema that +/// orders them y-then-x still receives each child's own values. +fn points(schema: &Arc, xs: [i32; 2], ys: [i32; 2]) -> RecordBatch { + let DataType::Struct(children) = schema.field(0).data_type().clone() else { + unreachable!("point schema is a struct") + }; + let columns = children + .iter() + .map(|child| ints(if child.name() == "x" { xs } else { ys }.to_vec())) + .collect(); + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StructArray::new(children, columns, None)) as ArrayRef], + ) + .unwrap() +} + +/// Commit `group` and read the `point` column back as its two children. +async fn committed_points(dataset: &Dataset, group: DataReplacementGroup) -> (Vec, Vec) { + let batch = commit(dataset, vec![group]) + .await + .unwrap() + .scan() + .try_into_batch() + .await + .unwrap(); + let child = |i: usize| { + batch + .column(0) + .as_struct() + .column(i) + .as_primitive::() + .values() + .to_vec() + }; + (child(0), child(1)) +} + +#[rstest] +#[tokio::test] +async fn test_records_writer_layout( + #[values(LanceFileVersion::V2_0, LanceFileVersion::V2_1)] version: LanceFileVersion, +) { + let mut dataset = dataset_of( + arrow_array::record_batch!(("id", Int32, [1, 2])).unwrap(), + Some(version), + ) + .await; + declare_all_null(&mut dataset, "value").await; + let schema = declared_schema(&dataset, "value"); + let fragment = only_fragment(&dataset); + + // Streamed as two batches: the DataFile must record the writer's + // field/column layout and the dataset's file version. + let DataReplacementGroup(replaced, data_file) = fragment + .write_columns( + stream::iter([ + Ok(arrow_array::record_batch!(("value", Int32, [1])).unwrap()), + Ok(arrow_array::record_batch!(("value", Int32, [2])).unwrap()), + ]), + &schema, + ) + .await + .unwrap(); + + assert_eq!(replaced, fragment.id() as u64); + assert_eq!(data_file.fields.as_ref(), &[schema.fields[0].id]); + assert_eq!(data_file.fields.len(), data_file.column_indices.len()); + assert!(data_file.path.ends_with(".lance")); + assert_eq!( + (data_file.file_major_version, data_file.file_minor_version), + version.resolve().to_data_file_numbers() + ); +} + +/// Input `write_columns` turns down before anything can be committed. The +/// container cases matter twice over: projection reorders by name but downcasts +/// by shape, so an unchecked batch is dropped silently or panics. +#[rstest] +#[case::too_few_rows("short", "physical rows")] +#[case::too_many_rows("long", "physical rows")] +#[case::unrequested_column("extra", "unexpected=[unrequested]")] +#[case::wrong_container("struct", "should have type int32 but type was struct")] +#[case::reserved_system_name("rowid", "reserved column")] +// The commit publishes data files, never schema, so a field the manifest does +// not define would commit as coverage no live field answers for. +#[case::undeclared_field("undeclared", "does not define")] +// The reader takes type, nullability and nested layout from the manifest, so a +// staged field reusing an id but differing in any of them would be decoded as +// the manifest's version rather than rejected -- `validate()` would not notice. +#[case::field_type_mismatch("wrong_type", "does not match dataset field id")] +#[case::field_nullability_mismatch("wrong_nullability", "does not match dataset field id")] +// Projection picks children by name, so a duplicate makes the choice arbitrary. +// The schema check compares name sets and cannot see one. +#[case::duplicate_column("duplicate", "appears twice")] +#[tokio::test] +async fn test_rejects_bad_input(#[case] shape: &str, #[case] expected: &str) { + let mut dataset = id_dataset().await; + declare_all_null(&mut dataset, "value").await; + let value = ArrowField::new("value", DataType::Int32, true); + let mut schema = declared_schema(&dataset, "value"); + + let values = match shape { + "short" => batch_of(vec![value], vec![ints(vec![7])]), + "long" => batch_of(vec![value], vec![ints(vec![7, 8, 9])]), + "extra" => batch_of( + vec![value, ArrowField::new("unrequested", DataType::Int32, true)], + vec![ints(vec![1, 2]), ints(vec![3, 4])], + ), + "struct" => { + let inner = Fields::from(vec![ArrowField::new("x", DataType::Int32, true)]); + batch_of( + vec![ArrowField::new( + "value", + DataType::Struct(inner.clone()), + true, + )], + vec![Arc::new(StructArray::new(inner, vec![ints(vec![1, 2])], None)) as ArrayRef], + ) + } + "rowid" => { + schema = new_column_schema(&dataset, ROW_ID); + batch_of( + vec![ArrowField::new(ROW_ID, DataType::Int32, true)], + vec![ints(vec![1, 2])], + ) + } + "undeclared" => { + schema = new_column_schema(&dataset, "novel"); + batch_of( + vec![ArrowField::new("novel", DataType::Int32, true)], + vec![ints(vec![1, 2])], + ) + } + "duplicate" => batch_of( + vec![value.clone(), value], + vec![ints(vec![1, 2]), ints(vec![3, 4])], + ), + "wrong_type" | "wrong_nullability" => { + let existing = dataset.schema().field("id").unwrap(); + let staged = if shape == "wrong_type" { + ArrowField::new("id", DataType::Float32, existing.nullable) + } else { + ArrowField::new("id", DataType::Int32, !existing.nullable) + }; + schema = LanceSchema::try_from(&ArrowSchema::new(vec![staged])).unwrap(); + schema.fields[0].id = existing.id; + batch_of( + vec![ArrowField::new("id", DataType::Int32, true)], + vec![ints(vec![1, 2])], + ) + } + other => unreachable!("unknown case {other}"), + }; + + let err = stage(&dataset, values, &schema).await.unwrap_err(); + assert!( + err.to_string().contains(expected), + "expected '{expected}' in error, got: {err}" + ); +} + +/// Nested containers the projector would otherwise reshape or unwind on: a +/// fixed-size list of the wrong width silently becomes a different row count, +/// and nulls under a required item panic instead of erroring. +#[rstest] +#[case::fixed_size_list_reshape( + true, + "fixed_size_list:int32:2 but type was fixed_size_list:int32:4" +)] +#[case::nulls_under_required_item(false, "non-null")] +#[tokio::test] +async fn test_rejects_bad_nested_input(#[case] reshape: bool, #[case] expected: &str) { + let item = |nullable| Arc::new(ArrowField::new("item", DataType::Int32, nullable)); + let nest = |kind: DataType, values: ArrayRef| { + batch_of(vec![ArrowField::new("v", kind, true)], vec![values]) + }; + // Fixed-size list: the same eight values, four rows of two against two of + // four. List: four values, one of them null under a required item. + let fsl = |width: i32| { + let array = FixedSizeListArray::new(item(true), width, ints((1..=8).collect()), None); + nest( + DataType::FixedSizeList(item(true), width), + Arc::new(array) as ArrayRef, + ) + }; + let list = |values: Vec>, nullable| { + let array = ListArray::new( + item(nullable), + OffsetBuffer::new(vec![0, 2, 4].into()), + Arc::new(Int32Array::from(values)) as ArrayRef, + None, + ); + nest(DataType::List(item(nullable)), Arc::new(array) as ArrayRef) + }; + let (seed, staged) = if reshape { + (fsl(2), fsl(4)) + } else { + ( + list(vec![Some(1), Some(2), Some(3), Some(4)], false), + list(vec![Some(10), None, Some(30), Some(40)], true), + ) + }; + + let dataset = dataset_of(seed, None).await; + let schema = dataset.schema().clone(); + let err = stage(&dataset, staged, &schema).await.unwrap_err(); + assert!( + err.to_string().contains(expected), + "expected '{expected}' in error, got: {err}" + ); +} + +/// Requesting the same declared field twice must be rejected inside the +/// staging contract: the per-field identity check cannot see it, and the +/// set-based batch comparison would match one column against both copies. +#[tokio::test] +async fn test_rejects_duplicate_requested_field() { + let mut dataset = id_dataset().await; + declare_all_null(&mut dataset, "value").await; + let mut schema = declared_schema(&dataset, "value"); + schema.fields.push(schema.fields[0].clone()); + + let batch = batch_of( + vec![ArrowField::new("value", DataType::Int32, true)], + vec![ints(vec![1, 2])], + ); + let before = count_files(&dataset).await; + let err = stage(&dataset, batch, &schema).await.unwrap_err(); + assert!(err.to_string().contains("more than once"), "got: {err}"); + assert_eq!(count_files(&dataset).await, before); +} + +/// An empty stream fails the row-count gate and leaves nothing staged, +/// including the footer-only file the eagerly-opened writer creates. +#[tokio::test] +async fn test_rejects_empty_stream() { + let mut dataset = id_dataset().await; + declare_all_null(&mut dataset, "value").await; + let schema = declared_schema(&dataset, "value"); + + let before = count_files(&dataset).await; + let err = only_fragment(&dataset) + .write_columns(stream::iter(Vec::>::new()), &schema) + .await + .unwrap_err(); + assert!(err.to_string().contains("physical rows"), "got: {err}"); + assert_eq!(count_files(&dataset).await, before); +} + +/// A visible null under a required child -- the parent is valid there, so the +/// slot is a value of the field -- is still rejected at the writer. +#[tokio::test] +async fn test_rejects_visible_null_under_required_child() { + let dataset = dataset_of(points(&point_schema(false), [1, 2], [10, 20]), None).await; + + // The batch declares its children nullable, which staging tolerates; the + // manifest's non-null rule is enforced against the data instead. + let staged_children = Fields::from(vec![ + ArrowField::new("x", DataType::Int32, true), + ArrowField::new("y", DataType::Int32, true), + ]); + let staged = batch_of( + vec![ArrowField::new( + "point", + DataType::Struct(staged_children.clone()), + false, + )], + vec![Arc::new(StructArray::new( + staged_children, + vec![ + Arc::new(Int32Array::from(vec![Some(1), None])) as ArrayRef, + ints(vec![10, 20]), + ], + None, + )) as ArrayRef], + ); + + let schema = dataset.schema().clone(); + let err = stage(&dataset, staged, &schema).await.unwrap_err(); + assert!( + err.to_string().contains("non-null"), + "expected a nullability rejection, got: {err}" + ); +} + +/// Field metadata decides physical layout -- a packed struct is one column, an +/// unpacked one a column per child -- so a caller's metadata must not be able to +/// stage a file whose coverage describes a different field set. +#[tokio::test] +async fn test_takes_layout_from_manifest() { + let arrow_schema = point_schema(true); + let dataset = dataset_of( + points(&arrow_schema, [1, 2], [10, 20]), + Some(LanceFileVersion::V2_1), + ) + .await; + let packed_field_id = dataset.schema().field("point").unwrap().id; + + // Identical to the manifest field but for the packed marker, which the + // field-identity comparison does not look at. + let mut staged_schema = dataset.schema().clone(); + staged_schema.fields[0] + .metadata + .remove(PACKED_STRUCT_META_KEY); + assert!(!staged_schema.fields[0].is_packed_struct()); + + let group = stage( + &dataset, + points(&arrow_schema, [3, 4], [30, 40]), + &staged_schema, + ) + .await + .unwrap(); + // Unpacked, the file would cover x and y instead, and DataReplacement would + // see coverage the packed field never had. + assert_eq!(group.1.fields.as_ref(), &[packed_field_id]); + + assert_eq!( + committed_points(&dataset, group).await, + (vec![3, 4], vec![30, 40]) + ); +} + +/// Struct encoders consume children positionally, so a batch whose children are +/// ordered differently from the manifest would be written under the wrong field +/// ids. Batches are matched by name at every level instead. +#[tokio::test] +async fn test_reorders_struct_children_by_name() { + let dataset = dataset_of(points(&point_schema(false), [1, 2], [10, 20]), None).await; + + // Names its children y-then-x: written positionally, y's values land in x. + let reordered = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "point", + DataType::Struct(point_children().into_iter().rev().cloned().collect()), + false, + )])); + let schema = dataset.schema().clone(); + let group = stage(&dataset, points(&reordered, [30, 40], [300, 400]), &schema) + .await + .unwrap(); + + assert_eq!( + committed_points(&dataset, group).await, + (vec![30, 40], vec![300, 400]), + "each child keeps its own values" + ); +} + +/// A Map is projected as a whole: its entries field carries metadata Lance +/// preserves in the schema, and a struct value's children may arrive +/// name-reordered. Projection must rebuild the map -- entries metadata intact, +/// children reordered by name -- rather than reject it. +#[tokio::test] +async fn test_stages_map_with_reordered_value_children() { + let value_children = Fields::from(vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("b", DataType::Int32, true), + ]); + let entry_fields = |value_children: &Fields| { + Fields::from(vec![ + ArrowField::new("key", DataType::Utf8, false), + ArrowField::new("value", DataType::Struct(value_children.clone()), true), + ]) + }; + let entries_field = |value_children: &Fields| { + ArrowField::new( + "entries", + DataType::Struct(entry_fields(value_children)), + false, + ) + .with_metadata([("entry-semantic".to_string(), "kept".to_string())].into()) + }; + let map_batch = |value_children: &Fields, + a: [i32; 2], + b: [i32; 2], + offsets: Vec, + nulls: Option| { + let children: Vec = value_children + .iter() + .map(|child| ints(if child.name() == "a" { a } else { b }.to_vec())) + .collect(); + let value = StructArray::new(value_children.clone(), children, None); + let entries = StructArray::new( + entry_fields(value_children), + vec![ + Arc::new(StringArray::from(vec!["k0", "k1"])) as ArrayRef, + Arc::new(value) as ArrayRef, + ], + None, + ); + let map = MapArray::new( + Arc::new(entries_field(value_children)), + OffsetBuffer::new(offsets.into()), + entries, + nulls, + false, + ); + batch_of( + vec![ArrowField::new( + "m", + DataType::Map(Arc::new(entries_field(value_children)), false), + true, + )], + vec![Arc::new(map) as ArrayRef], + ) + }; + + // Pinned to 2.2, the first version whose encoders accept Map. + let dataset = dataset_of( + map_batch(&value_children, [1, 2], [10, 20], vec![0, 1, 2], None), + Some(LanceFileVersion::V2_2), + ) + .await; + + // The staged batch orders the value's children b-then-a, holds both + // entries in slot 0, and leaves slot 1 null: map validity has to + // survive the rebuild alongside the reordering. + let reordered: Fields = value_children.iter().rev().cloned().collect(); + let schema = dataset.schema().clone(); + let group = stage( + &dataset, + map_batch( + &reordered, + [3, 4], + [30, 40], + vec![0, 2, 2], + Some(NullBuffer::from(vec![true, false])), + ), + &schema, + ) + .await + .unwrap(); + let dataset = commit(&dataset, vec![group]).await.unwrap(); + + let batch = dataset.scan().try_into_batch().await.unwrap(); + let map = batch.column(0).as_map(); + assert!(map.is_valid(0), "slot 0 keeps its entries"); + assert!(map.is_null(1), "slot 1 stays null"); + let value = map.entries().column(1).as_struct(); + let child = |name: &str| { + value + .column_by_name(name) + .unwrap() + .as_primitive::() + .values() + .to_vec() + }; + assert_eq!(child("a"), vec![3, 4], "each child keeps its own values"); + assert_eq!(child("b"), vec![30, 40]); +} + +/// Blob columns arrive logical and must be prepared into sidecars and +/// descriptors before the V2.2+ structural encoders accept them, which is what +/// the per-version update writer does. +#[tokio::test] +async fn test_stages_blob_column() { + use crate::blob::{BlobArrayBuilder, blob_field}; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", true)])); + let blobs = |values: [&[u8]; 2]| { + let mut builder = BlobArrayBuilder::new(2); + for value in values { + builder.push_bytes(value).unwrap(); + } + RecordBatch::try_new(arrow_schema.clone(), vec![builder.finish().unwrap()]).unwrap() + }; + let dataset = dataset_of(blobs([b"one", b"two"]), Some(LanceFileVersion::V2_2)).await; + + let schema = dataset.schema().clone(); + let group = stage(&dataset, blobs([b"three", b"four"]), &schema) + .await + .unwrap(); + assert!( + !group.1.fields.as_ref().is_empty(), + "staged file must cover the blob field" + ); +} + +/// The computed-column lifecycle: declare all null, backfill, compact, and +/// refresh again. The refresh after compaction is the case that previously +/// failed with "no changes were made". +#[tokio::test] +async fn test_replacement_survives_compaction() { + let mut dataset = id_dataset_of(4, 2).await; + declare_all_null(&mut dataset, "v").await; + let v_id = dataset.schema().field("v").unwrap().id; + + let frag_ids: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u64) + .collect(); + let mut replacements = Vec::new(); + for (i, frag_id) in frag_ids.iter().enumerate() { + let base = i as i32 * 100; + replacements.push(stage_column(&dataset, *frag_id, "v", vec![base + 1, base + 2]).await); + } + let mut dataset = commit(&dataset, replacements).await.unwrap(); + + compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + let files = dataset.get_fragments()[0].metadata().files.clone(); + assert_eq!(files.len(), 1, "compaction folded the column into one file"); + assert!(files[0].fields.len() > 1); + + // Refresh the compacted fragment, repeatedly: every round must land its own + // values and reuse the appended file rather than stacking another one. + let fragment_id = dataset.get_fragments()[0].id() as u64; + let rows = dataset.get_fragments()[0].physical_rows().await.unwrap(); + for round in 0..3i32 { + let refreshed: Vec = (0..rows as i32).map(|r| round * 1000 + r).collect(); + let replacement = stage_column(&dataset, fragment_id, "v", refreshed.clone()).await; + dataset = commit(&dataset, vec![replacement]).await.unwrap(); + dataset.validate().await.unwrap(); + + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!( + values(&batch, "v"), + refreshed.iter().map(|v| Some(*v)).collect::>() + ); + assert_eq!( + values(&batch, "id"), + (1..=rows as i32).map(Some).collect::>(), + "round {round} disturbed a sibling column of the tombstoned file" + ); + assert_eq!( + dataset.get_fragments()[0].metadata().files.len(), + 2, + "round {round} changed the file count" + ); + } + assert_eq!( + dataset.schema().field("v").unwrap().id, + v_id, + "field id preserved" + ); + + let files = dataset.get_fragments()[0].metadata().files.clone(); + let covering: Vec<&[i32]> = files + .iter() + .filter(|f| f.fields.contains(&v_id)) + .map(|f| f.fields.as_ref()) + .collect(); + assert_eq!(covering.as_slice(), &[[v_id].as_slice()]); + + // Tombstoning into a wider file has to advance row lineage like any other + // replacement, or a delta consumer never learns the refresh happened. + let version = dataset.version().version; + let batch = dataset + .scan() + .project(&["v", ROW_LAST_UPDATED_AT_VERSION]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!( + batch[ROW_LAST_UPDATED_AT_VERSION] + .as_primitive::() + .values(), + vec![version; rows].as_slice() + ); +} + +/// The existing uncovered (all-null backfill) and exact-match paths must be +/// unchanged. +#[tokio::test] +async fn test_existing_paths_unchanged() { + // Uncovered -> push. + let mut dataset = id_dataset_of(2, 1024).await; + declare_all_null(&mut dataset, "v").await; + let frag_id = dataset.get_fragments()[0].id() as u64; + let r = stage_column(&dataset, frag_id, "v", vec![10, 20]).await; + let dataset = commit(&dataset, vec![r]).await.unwrap(); + dataset.validate().await.unwrap(); + assert_eq!( + values(&dataset.scan().try_into_batch().await.unwrap(), "v"), + vec![Some(10), Some(20)] + ); + let files_after_first = dataset.get_fragments()[0].metadata().files.len(); + + // Exact match -> in-place swap, no new file. + let r = stage_column(&dataset, frag_id, "v", vec![30, 40]).await; + let dataset = commit(&dataset, vec![r]).await.unwrap(); + dataset.validate().await.unwrap(); + assert_eq!( + values(&dataset.scan().try_into_batch().await.unwrap(), "v"), + vec![Some(30), Some(40)] + ); + assert_eq!( + dataset.get_fragments()[0].metadata().files.len(), + files_after_first, + "exact-match replacement swaps in place rather than appending" + ); +} + +/// Dropping a sibling that shares the wider file must not leave that file +/// answering for dead ids only: every data file has to share at least one +/// field with the dataset schema, or validate() reports it as corrupt and +/// cleanup can never collect it. +#[tokio::test] +async fn test_replacement_after_sibling_drop_stays_valid() { + let batch = batch_of( + vec![ + ArrowField::new("a", DataType::Int32, true), + ArrowField::new("v", DataType::Int32, true), + ], + vec![ints(vec![1, 2]), ints(vec![10, 20])], + ); + let mut dataset = dataset_of(batch, None).await; + dataset.drop_columns(&["a"]).await.unwrap(); + + let frag_id = dataset.get_fragments()[0].id() as u64; + let r = stage_column(&dataset, frag_id, "v", vec![30, 40]).await; + let dataset = commit(&dataset, vec![r]).await.unwrap(); + + dataset.validate().await.unwrap(); + assert_eq!( + values(&dataset.scan().try_into_batch().await.unwrap(), "v"), + vec![Some(30), Some(40)] + ); +} + +/// The mirror ordering: a stale handle drops the column after the +/// replacement has committed. The projection rebases over the replacement, +/// wins by commit order, and its pruning must leave no file behind that +/// answers only for the dropped field. +#[tokio::test] +async fn test_stale_column_drop_prunes_committed_replacement() { + let mut dataset = id_dataset_of(2, 1024).await; + declare_all_null(&mut dataset, "v").await; + let frag_id = dataset.get_fragments()[0].id() as u64; + let r = stage_column(&dataset, frag_id, "v", vec![10, 20]).await; + commit(&dataset, vec![r]).await.unwrap(); + + // The stale handle predates the replacement; its commit rebases over it. + dataset.drop_columns(&["v"]).await.unwrap(); + + dataset.validate().await.unwrap(); + assert!(dataset.schema().field("v").is_none()); + let live_ids: Vec = dataset.schema().fields.iter().map(|f| f.id).collect(); + for file in &dataset.get_fragments()[0].metadata().files { + assert!( + file.fields.iter().any(|f| live_ids.contains(f)), + "file {} answers for no live field: {:?}", + file.path, + file.fields + ); + } +} + +/// The staged file is positionally aligned with physical rows, so a fragment +/// with deletions takes a value for every physical slot and the deletion +/// vector keeps masking the deleted ones afterwards. +#[tokio::test] +async fn test_replacement_preserves_deletions() { + let mut dataset = id_dataset_of(4, 1024).await; + declare_all_null(&mut dataset, "v").await; + dataset.delete("id = 2").await.unwrap(); + let frag_id = dataset.get_fragments()[0].id() as u64; + + // Physical row count is still 4: the staged data covers deleted slots too. + let r = stage_column(&dataset, frag_id, "v", vec![10, 20, 30, 40]).await; + let dataset = commit(&dataset, vec![r]).await.unwrap(); + dataset.validate().await.unwrap(); + + let batch = dataset.scan().try_into_batch().await.unwrap(); + assert_eq!(values(&batch, "id"), vec![Some(1), Some(3), Some(4)]); + assert_eq!(values(&batch, "v"), vec![Some(10), Some(30), Some(40)]); +} + +/// A projection that drops the staged column between staging and commit must +/// fail the commit: rebased over the drop, the staged file would answer for no +/// live schema field. +#[tokio::test] +async fn test_concurrent_column_drop_fails_commit() { + let mut dataset = id_dataset_of(2, 1024).await; + declare_all_null(&mut dataset, "v").await; + let frag_id = dataset.get_fragments()[0].id() as u64; + let staged = stage_column(&dataset, frag_id, "v", vec![10, 20]).await; + + // Lands after our snapshot: at commit time the field is gone. + let mut dropper = dataset.clone(); + dropper.drop_columns(&["v"]).await.unwrap(); + + let err = commit(&dataset, vec![staged]).await.unwrap_err(); + assert!( + err.to_string().contains("dropped by concurrent"), + "expected a field-dropped conflict, got: {err}" + ); +} + +/// The legacy reader pairs a fragment's files by batch boundary, so a staged +/// file chunked to the caller's batches would leave the fragment unreadable. +#[tokio::test] +async fn test_rejects_legacy_format() { + let dataset = dataset_of( + arrow_array::record_batch!(("id", Int32, [1, 2])).unwrap(), + Some(LanceFileVersion::Legacy), + ) + .await; + let schema = declared_schema(&dataset, "id"); + let batch = batch_of( + vec![ArrowField::new("id", DataType::Int32, true)], + vec![ints(vec![1, 2])], + ); + let err = stage(&dataset, batch, &schema).await.unwrap_err(); + assert!( + err.to_string().contains("legacy file format"), + "expected a legacy-format rejection, got: {err}" + ); +} + +/// Blob v2 spills sidecars into `data//`; a rejected stage that +/// leaves them behind orphans arbitrarily large objects. +#[tokio::test] +async fn test_discards_blob_sidecars_on_failure() { + use crate::blob::{BlobArrayBuilder, blob_field}; + + let arrow_schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", true)])); + let blobs = |count: usize| { + let mut builder = BlobArrayBuilder::new(count); + for _ in 0..count { + builder.push_bytes(vec![7u8; 128 * 1024]).unwrap(); + } + RecordBatch::try_new(arrow_schema.clone(), vec![builder.finish().unwrap()]).unwrap() + }; + let test_uri = TempStrDir::default(); + let dataset = Dataset::write( + RecordBatchIterator::new([Ok(blobs(2))], arrow_schema.clone()), + &test_uri, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + + let before = count_files(&dataset).await; + // Three rows against a two-row fragment: rejected only after the sidecars + // have been spilled. + let schema = dataset.schema().clone(); + stage(&dataset, blobs(3), &schema).await.unwrap_err(); + assert_eq!( + count_files(&dataset).await, + before, + "a rejected stage must not leave sidecars behind" + ); +} + +/// A stream error after a batch was already written exits through the same +/// cleanup as a rejected batch: the staged data file and any Blob sidecars a +/// finished pack already spilled are discarded, not orphaned. The pack-file +/// threshold is pinned to one blob's size so the first batch finalizes packs +/// before the error arrives. +#[tokio::test] +async fn test_discards_staged_artifacts_on_stream_error() { + use crate::blob::{BlobArrayBuilder, blob_field}; + use lance_arrow::BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY; + + let field = blob_field("blob", true); + let mut metadata = field.metadata().clone(); + metadata.insert( + BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY.to_string(), + (128 * 1024).to_string(), + ); + let arrow_schema = Arc::new(ArrowSchema::new(vec![field.with_metadata(metadata)])); + let blobs = |count: usize| { + let mut builder = BlobArrayBuilder::new(count); + for _ in 0..count { + builder.push_bytes(vec![7u8; 128 * 1024]).unwrap(); + } + RecordBatch::try_new(arrow_schema.clone(), vec![builder.finish().unwrap()]).unwrap() + }; + let test_uri = TempStrDir::default(); + let dataset = Dataset::write( + RecordBatchIterator::new([Ok(blobs(2))], arrow_schema.clone()), + &test_uri, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(); + + let before = count_files(&dataset).await; + let schema = dataset.schema().clone(); + let err = only_fragment(&dataset) + .write_columns( + stream::iter([ + Ok(blobs(2)), + Err(Error::invalid_input("stream failed".to_string())), + ]), + &schema, + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("stream failed"), "got: {err}"); + assert_eq!( + count_files(&dataset).await, + before, + "a stream error must not leave staged artifacts behind" + ); +} + +#[tokio::test] +async fn test_physical_slice_read_preserves_deleted_positions() { + let mut dataset = id_dataset_of(4, 1024).await; + dataset.delete("id = 2").await.unwrap(); + let fragment = only_fragment(&dataset); + let schema = dataset.schema().clone(); + let batches = fragment + .read_physical_slice(0..4, &schema, 2) + .await + .unwrap() + .buffered(1) + .try_collect::>() + .await + .unwrap(); + let batch = + arrow::compute::concat_batches(&Arc::new(ArrowSchema::from(&schema)), &batches).unwrap(); + assert_eq!( + batch["id"].as_primitive::().values(), + &[1, 2, 3, 4] + ); +} + +async fn count_files(dataset: &Dataset) -> usize { + dataset + .object_store + .read_dir_all(&dataset.data_dir(), None) + .try_fold(0usize, |count, _| async move { Ok(count + 1) }) + .await + .unwrap() +} diff --git a/rust/lance/src/dataset/tests/mod.rs b/rust/lance/src/dataset/tests/mod.rs index ecc64587b0c..1204f352966 100644 --- a/rust/lance/src/dataset/tests/mod.rs +++ b/rust/lance/src/dataset/tests/mod.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +mod data_file_part; #[cfg(feature = "substrait")] mod dataset_aggregate; mod dataset_common; @@ -11,7 +12,10 @@ mod dataset_index; mod dataset_io; mod dataset_merge_update; mod dataset_migrations; +mod dataset_overlay_index_masking; mod dataset_scanner; mod dataset_schema_evolution; mod dataset_transactions; mod dataset_versioning; +mod fragment_validate_tombstones; +mod fragment_write_columns; diff --git a/rust/lance/src/dataset/transaction.rs b/rust/lance/src/dataset/transaction.rs deleted file mode 100644 index 3261f9300c4..00000000000 --- a/rust/lance/src/dataset/transaction.rs +++ /dev/null @@ -1,6189 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright The Lance Authors - -//! Transaction definitions for updating datasets -//! -//! Prior to creating a new manifest, a transaction must be created representing -//! the changes being made to the dataset. By representing them as incremental -//! changes, we can detect whether concurrent operations are compatible with -//! one another. We can also rebuild manifests when retrying committing a -//! manifest. -//! -//! For more details please refer to the -//! [Transaction Specification](https://lance.org/format/table/transaction/#transaction-types). - -use super::ManifestWriteConfig; -use super::write::merge_insert::inserted_rows::KeyExistenceFilter; -use crate::dataset::transaction::UpdateMode::{RewriteColumns, RewriteRows}; -use crate::index::mem_wal::update_mem_wal_index_merged_generations; -use crate::utils::temporal::timestamp_to_nanos; -use lance_core::datatypes::{ - LANCE_UNENFORCED_CLUSTERING_KEY_POSITION, LANCE_UNENFORCED_PRIMARY_KEY, - LANCE_UNENFORCED_PRIMARY_KEY_POSITION, -}; -use lance_core::deepsize::DeepSizeOf; -use lance_core::{Error, Result, datatypes::Schema}; -use lance_file::{datatypes::Fields, version::LanceFileVersion}; -use lance_index::mem_wal::MergedGeneration; -use lance_index::{frag_reuse::FRAG_REUSE_INDEX_NAME, is_system_index}; -use lance_io::object_store::ObjectStore; -use lance_table::feature_flags::{FLAG_STABLE_ROW_IDS, apply_feature_flags}; -use lance_table::rowids::read_row_ids; -use lance_table::{ - format::{ - BasePath, DataFile, DataStorageFormat, Fragment, IndexFile, IndexMetadata, Manifest, - RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, pb, - }, - io::{ - commit::CommitHandler, - manifest::{read_manifest, read_manifest_indexes}, - }, - rowids::{RowIdSequence, segment::U64Segment, version::build_version_meta, write_row_ids}, -}; -use object_store::path::Path; -use roaring::RoaringBitmap; -use std::cmp::Ordering; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; -use uuid::Uuid; - -/// Fallback version for rows whose original creation version cannot be determined. -/// Version 1 is the initial dataset version in the Lance format. -const UNKNOWN_CREATED_AT_VERSION: u64 = 1; - -/// Look up the `created_at` version for a single UPDATE-branch row ID. -/// -/// Callers must only call this for row IDs that are confirmed to be present in -/// `row_id_to_source` (i.e. UPDATE branch rows whose source exists in an existing -/// fragment). INSERT branch rows (no source) must use `new_version` directly and -/// must not call this function. -/// -/// Uses `row_id_to_source` to find the originating fragment and row offset, then -/// performs a O(K) random-access lookup via [`RowDatasetVersionSequence::version_at`] -/// on the pre-decoded sequence in `version_cache` (keyed by fragment ID). -/// -/// Returns [`UNKNOWN_CREATED_AT_VERSION`] if the source fragment has no -/// `created_at_version_meta` (missing or failed to decode) or the offset is -/// out of range. -fn resolve_created_at_version( - row_id: u64, - row_id_to_source: &HashMap, - version_cache: &HashMap, -) -> u64 { - let Some((orig_frag, row_offset)) = row_id_to_source.get(&row_id) else { - return UNKNOWN_CREATED_AT_VERSION; - }; - let Some(seq) = version_cache.get(&orig_frag.id) else { - return UNKNOWN_CREATED_AT_VERSION; - }; - seq.version_at(*row_offset) - .unwrap_or(UNKNOWN_CREATED_AT_VERSION) -} - -/// For each new fragment produced by an update, set `created_at_version_meta` -/// (preserved from the original rows) and `last_updated_at_version_meta`. -fn resolve_update_version_metadata( - existing_fragments: &[Fragment], - new_fragments: &mut [Fragment], - new_version: u64, -) -> Result<()> { - // Collect only the row IDs we actually need to resolve, those appearing in new_fragments - // with inline metadata. This bounds the lookup map to O(updated rows) instead of O(all dataset rows) - let needed_row_ids: HashSet = new_fragments - .iter() - .filter_map(|f| match &f.row_id_meta { - Some(RowIdMeta::Inline(data)) => read_row_ids(data).ok(), - _ => None, - }) - .flat_map(|seq| seq.iter().collect::>()) - .collect(); - - let mut row_id_to_source: HashMap = HashMap::new(); - - if !needed_row_ids.is_empty() { - // Compute the bounding range of the needed set once. Any fragment whose - // entire row-id range lies outside [needed_min, needed_max] cannot contain - // any needed ID and can be skipped before the inner per-row loop. - let needed_min = *needed_row_ids.iter().min().unwrap(); - let needed_max = *needed_row_ids.iter().max().unwrap(); - - // Stable row IDs must be globally unique among *live* rows, but after a rewrite-style - // update the same stable ID can appear twice in `existing_fragments`: once in an older - // fragment's inline `row_id_meta` at the original row offset (rows may be soft-deleted - // via a deletion vector) and again in a newer fragment holding rewritten data. For - // `created_at` we need the mapping from the original fragment/offset; that is always the - // first occurrence when fragments are processed in ascending `id` order. - let mut sorted_frags: Vec<&Fragment> = existing_fragments.iter().collect(); - sorted_frags.sort_by_key(|f| f.id); - for frag in sorted_frags { - if let Some(RowIdMeta::Inline(data)) = &frag.row_id_meta - && let Ok(seq) = read_row_ids(data) - { - // Range pre-filter: skip the per-row inner loop when the fragment's - // bounding row-id range has no overlap with [needed_min, needed_max]. - // row_id_range() returns None for empty sequences, which are also skipped. - // This is a conservative check (may produce false positives for sparse - // segments) but never skips a fragment that actually contains a needed ID. - if seq - .row_id_range() - .is_none_or(|r| *r.end() < needed_min || *r.start() > needed_max) - { - continue; - } - - for (offset, rid) in seq.iter().enumerate() { - if needed_row_ids.contains(&rid) { - row_id_to_source.entry(rid).or_insert((frag, offset)); - } - } - } - } - } - - // Pre-decode the `created_at` version sequence for each source fragment exactly - // once. Without this cache, resolve_created_at_version would call load_sequence() - // (a protobuf decode) for every single updated row, even when many rows originate - // from the same fragment. - let source_frag_ids: HashSet = row_id_to_source.values().map(|(f, _)| f.id).collect(); - let version_cache: HashMap = existing_fragments - .iter() - .filter(|f| source_frag_ids.contains(&f.id)) - .filter_map(|frag| { - let seq = frag - .created_at_version_meta - .as_ref()? - .load_sequence() - .ok()?; - Some((frag.id, seq)) - }) - .collect(); - - for fragment in new_fragments.iter_mut() { - let row_ids = match &fragment.row_id_meta { - Some(RowIdMeta::Inline(data)) => read_row_ids(data).ok(), - Some(RowIdMeta::External(_)) => { - log::warn!( - "Fragment {} has external row ID metadata; \ - version tracking will use defaults", - fragment.id, - ); - None - } - None => None, - }; - - if let Some(row_ids) = row_ids { - let physical_rows = fragment.physical_rows.unwrap_or(0); - let created_at_versions: Vec = row_ids - .iter() - .map(|rid| { - if row_id_to_source.contains_key(&rid) { - // UPDATE branch: stable row ID resolves to a source row in an - // existing fragment. Copy created_at from the original row so - // the row's first-appearance version is preserved across rewrites. - resolve_created_at_version(rid, &row_id_to_source, &version_cache) - } else { - // INSERT branch: stable row ID has no source in existing fragments - // (e.g. NOT MATCHED arm of MERGE INTO). The row first appears in - // this commit, so created_at equals the new commit version. - new_version - } - }) - .collect(); - debug_assert_eq!(created_at_versions.len(), physical_rows); - - let runs = encode_version_runs(&created_at_versions); - let created_at_seq = RowDatasetVersionSequence { runs }; - fragment.created_at_version_meta = Some( - RowDatasetVersionMeta::from_sequence(&created_at_seq).map_err(|e| { - Error::internal(format!( - "Failed to create created_at version metadata: {}", - e - )) - })?, - ); - - fragment.last_updated_at_version_meta = build_version_meta(fragment, new_version); - } else { - let version_meta = build_version_meta(fragment, new_version); - fragment.last_updated_at_version_meta = version_meta.clone(); - fragment.created_at_version_meta = version_meta; - } - } - Ok(()) -} - -/// Run-length encode a sequence of per-row versions into [`RowDatasetVersionRun`]s. -fn encode_version_runs(versions: &[u64]) -> Vec { - if versions.is_empty() { - return Vec::new(); - } - let mut runs = Vec::new(); - let mut current_version = versions[0]; - let mut run_start = 0u64; - for (i, &version) in versions.iter().enumerate().skip(1) { - if version != current_version { - runs.push(RowDatasetVersionRun { - span: U64Segment::Range(run_start..i as u64), - version: current_version, - }); - current_version = version; - run_start = i as u64; - } - } - runs.push(RowDatasetVersionRun { - span: U64Segment::Range(run_start..versions.len() as u64), - version: current_version, - }); - runs -} - -/// A change to a dataset that can be retried -/// -/// This contains enough information to be able to build the next manifest, -/// given the current manifest. -#[derive(Debug, Clone, DeepSizeOf, PartialEq)] -pub struct Transaction { - /// The version of the table this transaction is based off of. If this is - /// the first transaction, this should be 0. - pub read_version: u64, - pub uuid: String, - pub operation: Operation, - pub tag: Option, - pub transaction_properties: Option>>, -} - -#[derive(Debug, Clone, DeepSizeOf, PartialEq)] -pub struct DataReplacementGroup(pub u64, pub DataFile); - -/// An entry for a map update. If value is None, the key will be removed from the map. -#[derive(Debug, Clone, DeepSizeOf, PartialEq)] -pub struct UpdateMapEntry { - /// The key of the map entry to update. - pub key: String, - /// The value to set for the key. - pub value: Option, -} - -impl From<(String, Option)> for UpdateMapEntry { - fn from((key, value): (String, Option)) -> Self { - Self { key, value } - } -} - -impl From<(String, String)> for UpdateMapEntry { - fn from((key, value): (String, String)) -> Self { - Self::from((key, Some(value))) - } -} - -impl From<(&str, Option<&str>)> for UpdateMapEntry { - fn from((key, value): (&str, Option<&str>)) -> Self { - Self { - key: key.to_string(), - value: value.map(str::to_owned), - } - } -} - -impl From<(&str, &str)> for UpdateMapEntry { - fn from((key, value): (&str, &str)) -> Self { - Self::from((key, Some(value))) - } -} - -/// Represents updates to a map (either incremental or replacement) -#[derive(Debug, Clone, DeepSizeOf, PartialEq)] -pub struct UpdateMap { - pub update_entries: Vec, - /// If true, the map will be replaced entirely with the new entries. - /// If false, the new entries will be merged with the existing map. - pub replace: bool, -} - -/// An operation on a dataset. -#[derive(Debug, Clone, DeepSizeOf)] -pub enum Operation { - /// Adding new fragments to the dataset. The fragments contained within - /// haven't yet been assigned a final ID. - Append { fragments: Vec }, - /// Updated fragments contain those that have been modified with new deletion - /// files. The deleted fragment IDs are those that should be removed from - /// the manifest. - Delete { - updated_fragments: Vec, - deleted_fragment_ids: Vec, - predicate: String, - }, - /// Overwrite the entire dataset with the given fragments. This is also - /// used when initially creating a table. - Overwrite { - fragments: Vec, - schema: Schema, - config_upsert_values: Option>, - initial_bases: Option>, - }, - /// A new index has been created. - CreateIndex { - /// The new secondary indices, - /// any existing indices with the same name will be replaced. - new_indices: Vec, - /// The indices that have been modified. - removed_indices: Vec, - }, - /// Data is rewritten but *not* modified. This is used for things like - /// compaction or re-ordering. Contains the old fragments and the new - /// ones that have been replaced. - /// - /// This operation will modify the row addresses of existing rows and - /// so any existing index covering a rewritten fragment will need to be - /// remapped. - Rewrite { - /// Groups of fragments that have been modified - groups: Vec, - /// Indices that have been updated with the new row addresses - rewritten_indices: Vec, - /// The fragment reuse index to be created or updated to - frag_reuse_index: Option, - }, - /// Replace data in a column in the dataset with new data. This is used for - /// null column population where we replace an entirely null column with a - /// new column that has data. - /// - /// This operation will only allow replacing files that contain the same schema - /// e.g. if the original files contain columns A, B, C and the new files contain - /// only columns A, B then the operation is not allowed. As we would need to split - /// the original files into two files, one with column A, B and the other with column C. - /// - /// Corollary to the above: the operation will also not allow replacing files unless the - /// affected columns all have the same datafile layout across the fragments being replaced. - /// - /// e.g. if fragments being replaced contain files with different schema layouts on - /// the column being replaced, the operation is not allowed. - /// say `frag_1: [A] [B, C]` and `frag_2: [A, B] [C]` and we are trying to replace column A - /// with a new column A, the operation is not allowed. - DataReplacement { - replacements: Vec, - }, - /// Merge a new column in - /// 'fragments' is the final fragments include all data files, the new fragments must align with old ones at rows. - /// 'schema' is not forced to include existed columns, which means we could use Merge to drop column data - Merge { - fragments: Vec, - schema: Schema, - }, - /// Restore an old version of the database - Restore { version: u64 }, - /// Reserves fragment ids for future use - /// This can be used when row ids need to be known before a transaction - /// has been committed. It is used during a rewrite operation to allow - /// indices to be remapped to the new row ids as part of the operation. - ReserveFragments { num_fragments: u32 }, - - /// Update values in the dataset. - /// - /// Updates are generally vertical or horizontal. - /// - /// A vertical update adds new rows. In this case, the updated_fragments - /// will only have existing rows deleted and will not have any new fields added. - /// All new data will be contained in new_fragments. - /// This is what is used by a merge_insert that matches the whole schema and what - /// is used by the dataset updater. - /// - /// A horizontal update adds new columns. In this case, the updated fragments - /// may have fields removed or added. It is even possible for a field to be tombstoned - /// and then added back in the same update. (which is a field modification). If any - /// fields are modified in this way then they need to be added to the fields_modified list. - /// This way we can correctly update the indices. - /// This is what is used by a merge insert that does not match the whole schema. - Update { - /// Ids of fragments that have been moved - removed_fragment_ids: Vec, - /// Fragments that have been updated - updated_fragments: Vec, - /// Fragments that have been added - new_fragments: Vec, - /// The fields that have been modified - fields_modified: Vec, - /// List of MemWAL region generations to mark as merged after this transaction - merged_generations: Vec, - /// The fields that used to judge whether to preserve the new frag's id into - /// the frag bitmap of the specified indices. - fields_for_preserving_frag_bitmap: Vec, - /// The mode of update - update_mode: Option, - /// Optional filter for detecting conflicts on inserted row keys. - /// Only tracks keys from INSERT operations during merge insert, not updates. - inserted_rows_filter: Option, - /// Physical row offsets (per fragment) that matched `update_columns` for RewriteColumns. - /// `None` means callers did not supply offsets; `build_manifest` skips partial refresh then. - updated_fragment_offsets: Option, - }, - - /// Project to a new schema. This only changes the schema, not the data. - Project { schema: Schema }, - - /// Update the dataset configuration. - UpdateConfig { - config_updates: Option, - table_metadata_updates: Option, - schema_metadata_updates: Option, - field_metadata_updates: HashMap, - }, - /// Update merged generations in MemWAL index. - /// This is used during merge-insert to atomically record which - /// generations have been merged to the base table. - UpdateMemWalState { - merged_generations: Vec, - }, - - /// Clone a dataset. - Clone { - is_shallow: bool, - ref_name: Option, - ref_version: u64, - ref_path: String, - branch_name: Option, - }, - - // Update base paths in the dataset (currently only supports adding new bases). - UpdateBases { - /// The new base paths to add to the manifest. - new_bases: Vec, - }, -} - -#[derive(Debug, Clone, PartialEq, DeepSizeOf)] -pub enum UpdateMode { - /// rows are deleted in current fragments and rewritten in new fragments. - /// This is most optimal when the majority of columns are being rewritten - /// or only a few rows are being updated. - RewriteRows, - - /// within each fragment, columns are fully rewritten and inserted as new data files. - /// Old versions of columns are tombstoned. This is most optimal when most rows are affected - /// but a small subset of columns are affected. - RewriteColumns, -} - -/// Matched physical row offsets per fragment for a partial [`UpdateMode::RewriteColumns`] update. -/// -/// Used with stable row IDs so `build_manifest` can refresh row-level version -/// metadata only for rows that were rewritten. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct UpdatedFragmentOffsets(pub HashMap); - -impl DeepSizeOf for UpdatedFragmentOffsets { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.0.iter().fold(0_usize, |acc, (frag_id, bitmap)| { - acc + frag_id.deep_size_of_children(context) - + (bitmap.len() as usize).saturating_mul(std::mem::size_of::()) - }) - } -} - -impl std::fmt::Display for Operation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Append { .. } => write!(f, "Append"), - Self::Delete { .. } => write!(f, "Delete"), - Self::Overwrite { .. } => write!(f, "Overwrite"), - Self::CreateIndex { .. } => write!(f, "CreateIndex"), - Self::Rewrite { .. } => write!(f, "Rewrite"), - Self::Merge { .. } => write!(f, "Merge"), - Self::Restore { .. } => write!(f, "Restore"), - Self::ReserveFragments { .. } => write!(f, "ReserveFragments"), - Self::Update { .. } => write!(f, "Update"), - Self::Project { .. } => write!(f, "Project"), - Self::UpdateConfig { .. } => write!(f, "UpdateConfig"), - Self::DataReplacement { .. } => write!(f, "DataReplacement"), - Self::Clone { .. } => write!(f, "Clone"), - Self::UpdateMemWalState { .. } => write!(f, "UpdateMemWalState"), - Self::UpdateBases { .. } => write!(f, "UpdateBases"), - } - } -} - -impl From<&Transaction> for lance_table::format::Transaction { - fn from(value: &Transaction) -> Self { - let pb_transaction: pb::Transaction = value.into(); - Self { - inner: pb_transaction, - } - } -} - -impl PartialEq for Operation { - fn eq(&self, other: &Self) -> bool { - // Many of the operations contain `Vec` where the order of the - // elements don't matter. So we need to compare them in a way that - // ignores the order of the elements. - // TODO: we can make it so the vecs are always constructed in order. - // Then we can use `==` instead of `compare_vec`. - fn compare_vec(a: &[T], b: &[T]) -> bool { - a.len() == b.len() && a.iter().all(|f| b.contains(f)) - } - match (self, other) { - (Self::Append { fragments: a }, Self::Append { fragments: b }) => compare_vec(a, b), - ( - Self::Clone { - is_shallow: a_is_shallow, - ref_name: a_ref_name, - ref_version: a_ref_version, - ref_path: a_source_path, - branch_name: a_branch_name, - }, - Self::Clone { - is_shallow: b_is_shallow, - ref_name: b_ref_name, - ref_version: b_ref_version, - ref_path: b_source_path, - branch_name: b_branch_name, - }, - ) => { - a_is_shallow == b_is_shallow - && a_ref_name == b_ref_name - && a_ref_version == b_ref_version - && a_source_path == b_source_path - && a_branch_name == b_branch_name - } - ( - Self::Delete { - updated_fragments: a_updated, - deleted_fragment_ids: a_deleted, - predicate: a_predicate, - }, - Self::Delete { - updated_fragments: b_updated, - deleted_fragment_ids: b_deleted, - predicate: b_predicate, - }, - ) => { - compare_vec(a_updated, b_updated) - && compare_vec(a_deleted, b_deleted) - && a_predicate == b_predicate - } - ( - Self::Overwrite { - fragments: a_fragments, - schema: a_schema, - config_upsert_values: a_config, - initial_bases: a_initial, - }, - Self::Overwrite { - fragments: b_fragments, - schema: b_schema, - config_upsert_values: b_config, - initial_bases: b_initial, - }, - ) => { - compare_vec(a_fragments, b_fragments) - && a_schema == b_schema - && a_config == b_config - && a_initial == b_initial - } - ( - Self::CreateIndex { - new_indices: a_new, - removed_indices: a_removed, - }, - Self::CreateIndex { - new_indices: b_new, - removed_indices: b_removed, - }, - ) => compare_vec(a_new, b_new) && compare_vec(a_removed, b_removed), - ( - Self::Rewrite { - groups: a_groups, - rewritten_indices: a_indices, - frag_reuse_index: a_frag_reuse_index, - }, - Self::Rewrite { - groups: b_groups, - rewritten_indices: b_indices, - frag_reuse_index: b_frag_reuse_index, - }, - ) => { - compare_vec(a_groups, b_groups) - && compare_vec(a_indices, b_indices) - && a_frag_reuse_index == b_frag_reuse_index - } - ( - Self::Merge { - fragments: a_fragments, - schema: a_schema, - }, - Self::Merge { - fragments: b_fragments, - schema: b_schema, - }, - ) => compare_vec(a_fragments, b_fragments) && a_schema == b_schema, - (Self::Restore { version: a }, Self::Restore { version: b }) => a == b, - ( - Self::ReserveFragments { num_fragments: a }, - Self::ReserveFragments { num_fragments: b }, - ) => a == b, - ( - Self::Update { - removed_fragment_ids: a_removed, - updated_fragments: a_updated, - new_fragments: a_new, - fields_modified: a_fields, - merged_generations: a_merged_generations, - fields_for_preserving_frag_bitmap: a_fields_for_preserving_frag_bitmap, - update_mode: a_update_mode, - inserted_rows_filter: a_inserted_rows_filter, - updated_fragment_offsets: a_updated_fragment_offsets, - }, - Self::Update { - removed_fragment_ids: b_removed, - updated_fragments: b_updated, - new_fragments: b_new, - fields_modified: b_fields, - merged_generations: b_merged_generations, - fields_for_preserving_frag_bitmap: b_fields_for_preserving_frag_bitmap, - update_mode: b_update_mode, - inserted_rows_filter: b_inserted_rows_filter, - updated_fragment_offsets: b_updated_fragment_offsets, - }, - ) => { - compare_vec(a_removed, b_removed) - && compare_vec(a_updated, b_updated) - && compare_vec(a_new, b_new) - && compare_vec(a_fields, b_fields) - && compare_vec(a_merged_generations, b_merged_generations) - && compare_vec( - a_fields_for_preserving_frag_bitmap, - b_fields_for_preserving_frag_bitmap, - ) - && a_update_mode == b_update_mode - && a_inserted_rows_filter == b_inserted_rows_filter - && a_updated_fragment_offsets == b_updated_fragment_offsets - } - (Self::Project { schema: a }, Self::Project { schema: b }) => a == b, - ( - Self::UpdateConfig { - config_updates: a_config, - table_metadata_updates: a_table_metadata, - schema_metadata_updates: a_schema, - field_metadata_updates: a_field, - }, - Self::UpdateConfig { - config_updates: b_config, - table_metadata_updates: b_table_metadata, - schema_metadata_updates: b_schema, - field_metadata_updates: b_field, - }, - ) => { - a_config == b_config - && a_table_metadata == b_table_metadata - && a_schema == b_schema - && a_field == b_field - } - ( - Self::DataReplacement { replacements: a }, - Self::DataReplacement { replacements: b }, - ) => a.len() == b.len() && a.iter().all(|r| b.contains(r)), - // Handle all remaining combinations. - // We spell out all combinations explicitly to prevent - // us accidentally handling a new case in the wrong way. - (Self::Append { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Append { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Delete { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Overwrite { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::CreateIndex { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Rewrite { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Merge { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Restore { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::ReserveFragments { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Update { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Project { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::UpdateConfig { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::DataReplacement { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::UpdateMemWalState { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - ( - Self::UpdateMemWalState { - merged_generations: a_merged, - }, - Self::UpdateMemWalState { - merged_generations: b_merged, - }, - ) => compare_vec(a_merged, b_merged), - (Self::Clone { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::UpdateBases { new_bases: a }, Self::UpdateBases { new_bases: b }) => { - compare_vec(a, b) - } - - (Self::UpdateBases { .. }, Self::Append { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Delete { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Overwrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::CreateIndex { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Rewrite { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Merge { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Restore { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::ReserveFragments { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Update { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Project { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::UpdateConfig { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::DataReplacement { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::UpdateMemWalState { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateBases { .. }, Self::Clone { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - - (Self::Append { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Delete { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Overwrite { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::CreateIndex { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Rewrite { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Merge { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Restore { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::ReserveFragments { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Update { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Project { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateConfig { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::DataReplacement { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::UpdateMemWalState { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - (Self::Clone { .. }, Self::UpdateBases { .. }) => { - std::mem::discriminant(self) == std::mem::discriminant(other) - } - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct RewrittenIndex { - pub old_id: Uuid, - pub new_id: Uuid, - pub new_index_details: prost_types::Any, - pub new_index_version: u32, - /// Files in the new index with their sizes. - /// Empty list from older writers that didn't persist this field. - pub new_index_files: Option>, -} - -impl DeepSizeOf for RewrittenIndex { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - self.new_index_details - .type_url - .deep_size_of_children(context) - + self.new_index_details.value.deep_size_of_children(context) - } -} - -#[derive(Debug, Clone, DeepSizeOf)] -pub struct RewriteGroup { - pub old_fragments: Vec, - pub new_fragments: Vec, -} - -impl PartialEq for RewriteGroup { - fn eq(&self, other: &Self) -> bool { - fn compare_vec(a: &[T], b: &[T]) -> bool { - a.len() == b.len() && a.iter().all(|f| b.contains(f)) - } - compare_vec(&self.old_fragments, &other.old_fragments) - && compare_vec(&self.new_fragments, &other.new_fragments) - } -} - -impl Operation { - /// Returns the config keys that have been upserted by this operation. - fn get_upsert_config_keys(&self) -> Vec { - match self { - Self::Overwrite { - config_upsert_values: Some(upsert_values), - .. - } => { - let vec: Vec = upsert_values.keys().cloned().collect(); - vec - } - Self::UpdateConfig { - config_updates: Some(config_updates), - .. - } => config_updates - .update_entries - .iter() - .filter_map(|entry| { - if entry.value.is_some() { - Some(entry.key.clone()) - } else { - None - } - }) - .collect(), - _ => Vec::::new(), - } - } - - /// Returns the config keys that have been deleted by this operation. - fn get_delete_config_keys(&self) -> Vec { - match self { - Self::UpdateConfig { - config_updates: Some(config_updates), - .. - } => config_updates - .update_entries - .iter() - .filter_map(|entry| { - if entry.value.is_none() { - Some(entry.key.clone()) - } else { - None - } - }) - .collect(), - _ => Vec::::new(), - } - } - - pub(crate) fn modifies_same_metadata(&self, other: &Self) -> bool { - match (self, other) { - ( - Self::UpdateConfig { - table_metadata_updates, - schema_metadata_updates, - field_metadata_updates, - .. - }, - Self::UpdateConfig { - table_metadata_updates: other_table_metadata, - schema_metadata_updates: other_schema_metadata, - field_metadata_updates: other_field_metadata, - .. - }, - ) => { - if Self::update_maps_conflict( - table_metadata_updates.as_ref(), - other_table_metadata.as_ref(), - ) { - return true; - } - if schema_metadata_updates.is_some() && other_schema_metadata.is_some() { - return true; - } - if !field_metadata_updates.is_empty() && !other_field_metadata.is_empty() { - for field in field_metadata_updates.keys() { - if other_field_metadata.contains_key(field) { - return true; - } - } - } - false - } - _ => false, - } - } - - fn update_maps_conflict(left: Option<&UpdateMap>, right: Option<&UpdateMap>) -> bool { - let (Some(left), Some(right)) = (left, right) else { - return false; - }; - if left.replace || right.replace { - return true; - } - let left_keys = left - .update_entries - .iter() - .map(|entry| entry.key.as_str()) - .collect::>(); - right - .update_entries - .iter() - .any(|entry| left_keys.contains(entry.key.as_str())) - } - - /// Check whether another operation upserts a key that is referenced by another operation - pub(crate) fn upsert_key_conflict(&self, other: &Self) -> bool { - let self_upsert_keys = self.get_upsert_config_keys(); - let other_upsert_keys = other.get_upsert_config_keys(); - - let self_delete_keys = self.get_delete_config_keys(); - let other_delete_keys = other.get_delete_config_keys(); - - self_upsert_keys - .iter() - .any(|x| other_upsert_keys.contains(x) || other_delete_keys.contains(x)) - || other_upsert_keys - .iter() - .any(|x| self_upsert_keys.contains(x) || self_delete_keys.contains(x)) - } - - pub fn name(&self) -> &str { - match self { - Self::Append { .. } => "Append", - Self::Delete { .. } => "Delete", - Self::Overwrite { .. } => "Overwrite", - Self::CreateIndex { .. } => "CreateIndex", - Self::Rewrite { .. } => "Rewrite", - Self::Merge { .. } => "Merge", - Self::ReserveFragments { .. } => "ReserveFragments", - Self::Restore { .. } => "Restore", - Self::Update { .. } => "Update", - Self::Project { .. } => "Project", - Self::UpdateConfig { .. } => "UpdateConfig", - Self::DataReplacement { .. } => "DataReplacement", - Self::UpdateMemWalState { .. } => "UpdateMemWalState", - Self::Clone { .. } => "Clone", - Self::UpdateBases { .. } => "UpdateBases", - } - } -} - -/// Helper function to apply UpdateMap changes to a HashMap -fn apply_update_map( - target: &mut std::collections::HashMap, - update_map: &UpdateMap, -) { - if update_map.replace { - // Full replacement - clear existing and replace with new entries that have values - target.clear(); - for entry in &update_map.update_entries { - if let Some(value) = &entry.value { - target.insert(entry.key.clone(), value.clone()); - } - } - } else { - // Incremental update - merge entries - for entry in &update_map.update_entries { - if let Some(value) = &entry.value { - target.insert(entry.key.clone(), value.clone()); - } else { - target.remove(&entry.key); - } - } - } -} - -/// Helper function to translate old-style config updates to new UpdateMap format -pub fn translate_config_updates( - upsert_values: &std::collections::HashMap, - delete_keys: &[String], -) -> UpdateMap { - let mut update_entries = Vec::new(); - - // Add upsert entries (with values) - for (key, value) in upsert_values { - update_entries.push(UpdateMapEntry { - key: key.clone(), - value: Some(value.clone()), - }); - } - - // Add delete entries (without values) - for key in delete_keys { - update_entries.push(UpdateMapEntry { - key: key.clone(), - value: None, - }); - } - - UpdateMap { - update_entries, - replace: false, // Old style was always incremental - } -} - -/// Helper function to translate old-style schema metadata to new UpdateMap format -pub fn translate_schema_metadata_updates( - schema_metadata: &std::collections::HashMap, -) -> UpdateMap { - let update_entries = schema_metadata - .iter() - .map(|(key, value)| UpdateMapEntry { - key: key.clone(), - value: Some(value.clone()), - }) - .collect(); - - UpdateMap { - update_entries, - replace: true, // Old style schema metadata was full replacement - } -} - -impl From<&UpdateMap> for pb::transaction::UpdateMap { - fn from(update_map: &UpdateMap) -> Self { - Self { - update_entries: update_map - .update_entries - .iter() - .map(|entry| pb::transaction::UpdateMapEntry { - key: entry.key.clone(), - value: entry.value.clone(), - }) - .collect(), - replace: update_map.replace, - } - } -} - -impl From<&pb::transaction::UpdateMap> for UpdateMap { - fn from(pb_update_map: &pb::transaction::UpdateMap) -> Self { - Self { - update_entries: pb_update_map - .update_entries - .iter() - .map(|entry| UpdateMapEntry { - key: entry.key.clone(), - value: entry.value.clone(), - }) - .collect(), - replace: pb_update_map.replace, - } - } -} - -/// Add TransactionBuilder for flexibly setting option without using `mut` -pub struct TransactionBuilder { - read_version: u64, - // uuid is optional for builder since it can autogenerate - uuid: Option, - operation: Operation, - tag: Option, - transaction_properties: Option>>, -} - -impl TransactionBuilder { - pub fn new(read_version: u64, operation: Operation) -> Self { - Self { - read_version, - uuid: None, - operation, - tag: None, - transaction_properties: None, - } - } - - pub fn uuid(mut self, uuid: String) -> Self { - self.uuid = Some(uuid); - self - } - - pub fn tag(mut self, tag: Option) -> Self { - self.tag = tag; - self - } - - pub fn transaction_properties( - mut self, - transaction_properties: Option>>, - ) -> Self { - self.transaction_properties = transaction_properties; - self - } - - pub fn build(self) -> Transaction { - let uuid = self - .uuid - .unwrap_or_else(|| Uuid::new_v4().hyphenated().to_string()); - Transaction { - read_version: self.read_version, - uuid, - operation: self.operation, - tag: self.tag, - transaction_properties: self.transaction_properties, - } - } -} - -impl Transaction { - pub fn new_from_version(read_version: u64, operation: Operation) -> Self { - TransactionBuilder::new(read_version, operation).build() - } - - pub fn new(read_version: u64, operation: Operation, tag: Option) -> Self { - TransactionBuilder::new(read_version, operation) - .tag(tag) - .build() - } - - fn fragments_with_ids<'a, T>( - new_fragments: T, - fragment_id: &'a mut u64, - ) -> impl Iterator + 'a - where - T: IntoIterator + 'a, - { - new_fragments.into_iter().map(move |mut f| { - if f.id == 0 { - f.id = *fragment_id; - *fragment_id += 1; - } - f - }) - } - - fn data_storage_format_from_files( - fragments: &[Fragment], - user_requested: Option, - ) -> Result { - if let Some(file_version) = Fragment::try_infer_version(fragments)? { - // Ensure user-requested matches data files - if let Some(user_requested) = user_requested - && user_requested != file_version - { - return Err(Error::invalid_input(format!( - "User requested data storage version ({}) does not match version in data files ({})", - user_requested, file_version - ))); - } - Ok(DataStorageFormat::new(file_version)) - } else { - // If no files use user-requested or default - Ok(user_requested - .map(DataStorageFormat::new) - .unwrap_or_default()) - } - } - - pub(crate) async fn restore_old_manifest( - object_store: &ObjectStore, - commit_handler: &dyn CommitHandler, - base_path: &Path, - version: u64, - config: &ManifestWriteConfig, - tx_path: &str, - current_manifest: &Manifest, - ) -> Result<(Manifest, Vec)> { - let location = commit_handler - .resolve_version_location(base_path, version, &object_store.inner) - .await?; - let mut manifest = read_manifest(object_store, &location.path, location.size).await?; - manifest.set_timestamp(timestamp_to_nanos(config.timestamp)); - manifest.transaction_file = Some(tx_path.to_string()); - let indices = read_manifest_indexes(object_store, &location, &manifest).await?; - manifest.max_fragment_id = manifest - .max_fragment_id - .max(current_manifest.max_fragment_id); - Ok((manifest, indices)) - } - - /// Create a new manifest from the current manifest and the transaction. - /// - /// `current_manifest` should only be None if the dataset does not yet exist. - pub(crate) fn build_manifest( - &self, - current_manifest: Option<&Manifest>, - current_indices: Vec, - transaction_file_path: &str, - config: &ManifestWriteConfig, - ) -> Result<(Manifest, Vec)> { - if config.use_stable_row_ids - && current_manifest - .map(|m| !m.uses_stable_row_ids()) - .unwrap_or_default() - { - return Err(Error::not_supported_source( - "Cannot enable stable row ids on existing dataset".into(), - )); - } - let mut reference_paths = match current_manifest { - Some(m) => m.base_paths.clone(), - None => HashMap::new(), - }; - - if let Operation::Overwrite { - initial_bases: Some(initial_bases), - .. - } = &self.operation - { - if current_manifest.is_none() { - // CREATE mode: registering base paths - // Base IDs should have been assigned during write operation - // Validate uniqueness and insert them into the manifest - for base_path in initial_bases.iter() { - if reference_paths.contains_key(&base_path.id) { - return Err(Error::invalid_input(format!( - "Duplicate base path ID {} detected. Base path IDs must be unique.", - base_path.id - ))); - } - reference_paths.insert(base_path.id, base_path.clone()); - } - } else { - // OVERWRITE mode with initial_bases should have been rejected by validation - // This branch should never be reached - return Err(Error::invalid_input( - "OVERWRITE mode cannot register new bases. This should have been caught by validation.", - )); - } - } - - // Get the schema and the final fragment list - let schema = match self.operation { - Operation::Overwrite { ref schema, .. } => schema.clone(), - Operation::Merge { ref schema, .. } => schema.clone(), - Operation::Project { ref schema, .. } => schema.clone(), - _ => { - if let Some(current_manifest) = current_manifest { - current_manifest.schema.clone() - } else { - return Err(Error::internal( - "Cannot create a new dataset without a schema".to_string(), - )); - } - } - }; - - let mut fragment_id = if matches!(self.operation, Operation::Overwrite { .. }) { - 0 - } else { - current_manifest - .and_then(|m| m.max_fragment_id()) - .map(|id| id + 1) - .unwrap_or(0) - }; - let mut final_fragments = Vec::new(); - let mut final_indices = current_indices; - - let mut next_row_id = { - // Only use row ids if the feature flag is set already or - match (current_manifest, config.use_stable_row_ids) { - (Some(manifest), _) if manifest.reader_feature_flags & FLAG_STABLE_ROW_IDS != 0 => { - Some(manifest.next_row_id) - } - (None, true) => Some(0), - (_, false) => None, - (Some(_), true) => { - return Err(Error::not_supported_source( - "Cannot enable stable row ids on existing dataset".into(), - )); - } - } - }; - - let maybe_existing_fragments = - current_manifest - .map(|m| m.fragments.as_ref()) - .ok_or_else(|| { - Error::internal(format!( - "No current manifest was provided while building manifest for operation {}", - self.operation.name() - )) - }); - - match &self.operation { - Operation::Clone { .. } => { - return Err(Error::internal( - "Clone operation should not enter build_manifest.".to_string(), - )); - } - Operation::Append { fragments } => { - final_fragments.extend(maybe_existing_fragments?.clone()); - let mut new_fragments = - Self::fragments_with_ids(fragments.clone(), &mut fragment_id) - .collect::>(); - if let Some(next_row_id) = &mut next_row_id { - Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; - // Add version metadata for all new fragments - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); - for fragment in new_fragments.iter_mut() { - let version_meta = build_version_meta(fragment, new_version); - fragment.last_updated_at_version_meta = version_meta.clone(); - fragment.created_at_version_meta = version_meta; - } - } - final_fragments.extend(new_fragments); - } - Operation::Delete { - updated_fragments, - deleted_fragment_ids, - .. - } => { - // Remove the deleted fragments - final_fragments.extend(maybe_existing_fragments?.clone()); - final_fragments.retain(|f| !deleted_fragment_ids.contains(&f.id)); - final_fragments.iter_mut().for_each(|f| { - for updated in updated_fragments { - if updated.id == f.id { - *f = updated.clone(); - } - } - }); - Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) - } - Operation::Update { - removed_fragment_ids, - updated_fragments, - new_fragments, - fields_modified, - merged_generations, - fields_for_preserving_frag_bitmap, - update_mode, - updated_fragment_offsets, - .. - } => { - // Extract existing fragments once for reuse - let existing_fragments = maybe_existing_fragments?; - - // Apply updates to existing fragments - let updated_frags: Vec = existing_fragments - .iter() - .filter_map(|f| { - if removed_fragment_ids.contains(&f.id) { - return None; - } - if let Some(updated) = updated_fragments.iter().find(|uf| uf.id == f.id) { - Some(updated.clone()) - } else { - Some(f.clone()) - } - }) - .collect(); - - // Update version metadata for updated fragments if stable row IDs are enabled - // Note: We don't update version metadata for fragments with deletion vectors - // because the version sequences are indexed by physical row position, not logical position. - // Version metadata for deleted rows will be filtered out during scan using the deletion vector. - if next_row_id.is_some() { - // Version metadata will be properly set during compaction when deletions are materialized - } - - final_fragments.extend(updated_frags); - - if next_row_id.is_some() - && matches!(update_mode, Some(RewriteColumns)) - && let Some(UpdatedFragmentOffsets(off_map)) = updated_fragment_offsets - && !off_map.is_empty() - { - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); - let prev_version = current_manifest.map(|m| m.version).unwrap_or(0); - for fragment in final_fragments.iter_mut() { - let Some(bitmap) = off_map.get(&fragment.id) else { - continue; - }; - if bitmap.is_empty() { - continue; - } - // Skip fragments with no existing version metadata: the helper - // would fill unmatched rows with prev_version, fabricating a - // last_updated stamp for rows that never had one. - if fragment.last_updated_at_version_meta.is_none() { - continue; - } - let offsets: Vec = bitmap.iter().map(|o| o as usize).collect(); - lance_table::rowids::version::refresh_row_latest_update_meta_for_partial_frag_rewrite_cols( - fragment, - &offsets, - new_version, - prev_version, - )?; - } - } - - // If we updated any fields, remove those fragments from indices covering those fields - Self::prune_updated_fields_from_indices( - &mut final_indices, - updated_fragments, - fields_modified, - ); - - let mut new_fragments = - Self::fragments_with_ids(new_fragments.clone(), &mut fragment_id) - .collect::>(); - - // Assign row IDs to any fragments that don't have them yet - // (e.g., inserted rows from merge_insert operations) - if let Some(next_row_id) = &mut next_row_id { - Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; - } - - if next_row_id.is_some() { - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); - resolve_update_version_metadata( - existing_fragments, - new_fragments.as_mut_slice(), - new_version, - )?; - } - - if config.use_stable_row_ids - && update_mode.is_some() - && *update_mode == Some(RewriteRows) - { - let pure_updated_frag_ids = - Self::collect_pure_rewrite_row_update_frags_ids(&new_fragments)?; - - // collect all the original frag ids that contains the updated rows - let original_fragment_ids: Vec = removed_fragment_ids - .iter() - .chain(updated_fragments.iter().map(|f| &f.id)) - .copied() - .collect(); - - Self::register_pure_rewrite_rows_update_frags_in_indices( - &mut final_indices, - &pure_updated_frag_ids, - &original_fragment_ids, - fields_for_preserving_frag_bitmap, - ); - } - - if let Some(next_row_id) = &mut next_row_id { - Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; - // Note: Version metadata is already set above (lines 1627-1755) - // for Update operations, preserving created_at from original fragments. - // Don't overwrite it here. - } - // Identify fragments that were updated or newly created in this update - let mut target_ids: HashSet = HashSet::new(); - target_ids.extend(new_fragments.iter().map(|f| f.id)); - final_fragments.extend(new_fragments); - Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments); - - if !merged_generations.is_empty() { - update_mem_wal_index_merged_generations( - &mut final_indices, - current_manifest.map_or(1, |m| m.version + 1), - merged_generations.clone(), - )?; - } - } - Operation::Overwrite { fragments, .. } => { - let mut new_fragments = - Self::fragments_with_ids(fragments.clone(), &mut fragment_id) - .collect::>(); - if let Some(next_row_id) = &mut next_row_id { - Self::assign_row_ids(next_row_id, new_fragments.as_mut_slice())?; - // Add version metadata for all new fragments - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); - for fragment in new_fragments.iter_mut() { - let version_meta = build_version_meta(fragment, new_version); - fragment.last_updated_at_version_meta = version_meta.clone(); - fragment.created_at_version_meta = version_meta; - } - } - final_fragments.extend(new_fragments); - final_indices = Vec::new(); - } - Operation::Rewrite { - groups, - rewritten_indices, - frag_reuse_index, - } => { - final_fragments.extend(maybe_existing_fragments?.clone()); - let current_version = current_manifest.map(|m| m.version).unwrap_or_default(); - Self::handle_rewrite_fragments( - &mut final_fragments, - groups, - &mut fragment_id, - current_version, - next_row_id.as_ref(), - )?; - - if next_row_id.is_some() { - // We can re-use indices, but need to rewrite the fragment bitmaps - debug_assert!(rewritten_indices.is_empty()); - for index in final_indices.iter_mut() { - if let Some(fragment_bitmap) = &mut index.fragment_bitmap { - *fragment_bitmap = - Self::recalculate_fragment_bitmap(fragment_bitmap, groups)?; - } - } - } else { - Self::handle_rewrite_indices(&mut final_indices, rewritten_indices, groups)?; - } - - if let Some(frag_reuse_index) = frag_reuse_index { - final_indices.retain(|idx| idx.name != frag_reuse_index.name); - final_indices.push(frag_reuse_index.clone()); - } - } - Operation::CreateIndex { - new_indices, - removed_indices, - } => { - final_fragments.extend(maybe_existing_fragments?.clone()); - let removed_uuids = removed_indices - .iter() - .map(|old_index| old_index.uuid) - .collect::>(); - let new_uuids = new_indices - .iter() - .map(|new_index| new_index.uuid) - .collect::>(); - final_indices.retain(|existing_index| { - !removed_uuids.contains(&existing_index.uuid) - && !new_uuids.contains(&existing_index.uuid) - }); - final_indices.extend(new_indices.clone()); - } - Operation::ReserveFragments { .. } | Operation::UpdateConfig { .. } => { - final_fragments.extend(maybe_existing_fragments?.clone()); - } - Operation::Merge { fragments, .. } => { - let existing_fragments = maybe_existing_fragments?; - let mut merged_fragments = fragments.clone(); - if next_row_id.is_some() { - let new_version = current_manifest.map(|m| m.version + 1).unwrap_or(1); - let prev_by_id: HashMap = - existing_fragments.iter().map(|f| (f.id, f)).collect(); - for fragment in merged_fragments.iter_mut() { - match prev_by_id.get(&fragment.id) { - Some(prev) => { - if merge_fragment_physically_rewritten(prev, fragment) { - lance_table::rowids::version::refresh_row_latest_update_meta_for_full_frag_rewrite_cols( - fragment, - new_version, - )?; - } - } - None => { - // Brand-new fragment ID not present in the previous manifest. - // Set both last_updated and created version meta, consistent - // with Append/Overwrite for genuinely new fragments. - lance_table::rowids::version::refresh_row_latest_update_meta_for_full_frag_rewrite_cols( - fragment, - new_version, - )?; - fragment.created_at_version_meta = - fragment.last_updated_at_version_meta.clone(); - } - } - } - } - final_fragments.extend(merged_fragments); - - // A Merge can rewrite a column's data file in place; the field stays - // in the schema, so the index is retained -- prune its now-stale - // entries for the rewritten fragments. - Self::prune_merge_rewritten_fields_from_indices( - &mut final_indices, - existing_fragments, - fragments, - ); - - // Some fields that have indices may have been removed, so we should - // remove those indices as well. - Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) - } - Operation::Project { .. } => { - final_fragments.extend(maybe_existing_fragments?.clone()); - - // We might have removed all fields for certain data files, so - // we should remove the data files that are no longer relevant. - let remaining_field_ids = schema - .fields_pre_order() - .map(|f| f.id) - .collect::>(); - for fragment in final_fragments.iter_mut() { - fragment.files.retain(|file| { - file.fields - .iter() - .any(|field_id| remaining_field_ids.contains(field_id)) - }); - } - - // Some fields that have indices may have been removed, so we should - // remove those indices as well. - Self::retain_relevant_indices(&mut final_indices, &schema, &final_fragments) - } - Operation::Restore { .. } => { - unreachable!() - } - Operation::DataReplacement { replacements } => { - log::warn!( - "Building manifest with DataReplacement operation. This operation is not stable yet, please use with caution." - ); - - let (old_fragment_ids, new_datafiles): (Vec<&u64>, Vec<&DataFile>) = replacements - .iter() - .map(|DataReplacementGroup(fragment_id, new_file)| (fragment_id, new_file)) - .unzip(); - - // 1. make sure the new files all have the same fields / or empty - // NOTE: arguably this requirement could be relaxed in the future - // for the sake of simplicity, we require the new files to have the same fields - if new_datafiles - .iter() - .map(|f| f.fields.clone()) - .collect::>() - .len() - > 1 - { - let field_info = new_datafiles - .iter() - .enumerate() - .map(|(id, f)| (id, f.fields.clone())) - .fold("".to_string(), |acc, (id, fields)| { - format!("{}File {}: {:?}\n", acc, id, fields) - }); - - return Err(Error::invalid_input(format!( - "All new data files must have the same fields, but found different fields:\n{field_info}" - ))); - } - - let existing_fragments = maybe_existing_fragments?; - - // Collect replaced field IDs before consuming new_datafiles - let replaced_fields: Vec = new_datafiles - .first() - .map(|f| { - f.fields - .iter() - .filter(|&&id| id >= 0) - .map(|&id| id as u32) - .collect() - }) - .unwrap_or_default(); - - // 2. check that the fragments being modified have isomorphic layouts along the columns being replaced - // 3. add modified fragments to final_fragments - for (frag_id, new_file) in old_fragment_ids.iter().zip(new_datafiles) { - let frag = existing_fragments - .iter() - .find(|f| f.id == **frag_id) - .ok_or_else(|| { - Error::invalid_input( - "Fragment being replaced not found in existing fragments", - ) - })?; - let mut new_frag = frag.clone(); - - // TODO(rmeng): check new file and fragment are the same length - - let mut columns_covered = HashSet::new(); - for file in &mut new_frag.files { - if file.fields == new_file.fields - && file.file_major_version == new_file.file_major_version - && file.file_minor_version == new_file.file_minor_version - { - // assign the new file path / size / base to the fragment - file.path = new_file.path.clone(); - file.file_size_bytes = new_file.file_size_bytes.clone(); - file.base_id = new_file.base_id; - } - columns_covered.extend(file.fields.iter()); - } - // SPECIAL CASE: if the column(s) being replaced are not covered by the fragment - // Then it means it's a all-NULL column that is being replaced with real data - // just add it to the final fragments. Push the DataFile as - // given so every field (including base_id) is preserved. - if columns_covered.is_disjoint(&new_file.fields.iter().collect()) { - LanceFileVersion::try_from_major_minor( - new_file.file_major_version, - new_file.file_minor_version, - ) - .expect("Expected valid file version"); - new_frag.files.push(new_file.clone()); - } - - // Nothing changed in the current fragment, which is not expected -- error out - if &new_frag == frag { - return Err(Error::invalid_input( - "Expected to modify the fragment but no changes were made. This means the new data files does not align with any exiting datafiles. Please check if the schema of the new data files matches the schema of the old data files including the file major and minor versions", - )); - } - final_fragments.push(new_frag); - } - - let fragments_changed = old_fragment_ids - .iter() - .cloned() - .cloned() - .collect::>(); - - // 4. push fragments that didn't change back to final_fragments - let unmodified_fragments = existing_fragments - .iter() - .filter(|f| !fragments_changed.contains(&f.id)) - .cloned() - .collect::>(); - - final_fragments.extend(unmodified_fragments); - - // 5. Invalidate index bitmaps for replaced fields - let modified_fragments: Vec = final_fragments - .iter() - .filter(|f| fragments_changed.contains(&f.id)) - .cloned() - .collect(); - - Self::prune_updated_fields_from_indices( - &mut final_indices, - &modified_fragments, - &replaced_fields, - ); - } - Operation::UpdateMemWalState { merged_generations } => { - update_mem_wal_index_merged_generations( - &mut final_indices, - current_manifest.map_or(1, |m| m.version + 1), - merged_generations.clone(), - )?; - } - Operation::UpdateBases { .. } => { - // UpdateBases operation doesn't modify fragments or indices - // Base paths are handled in the manifest creation section below - final_fragments.extend(maybe_existing_fragments?.clone()); - } - }; - - // If a fragment was reserved then it may not belong at the end of the fragments list. - final_fragments.sort_by_key(|frag| frag.id); - - // Clean up data files that only contain tombstoned fields - Self::remove_tombstoned_data_files(&mut final_fragments); - - let user_requested_version = match (&config.storage_format, config.use_legacy_format) { - (Some(storage_format), _) => Some(storage_format.lance_file_version()?), - (None, Some(true)) => Some(LanceFileVersion::Legacy), - (None, Some(false)) => Some(LanceFileVersion::V2_0), - (None, None) => None, - }; - - let mut manifest = if let Some(current_manifest) = current_manifest { - // OVERWRITE with initial_bases on existing dataset is not allowed (caught by validation) - // So we always use new_from_previous which preserves base_paths - let mut prev_manifest = - Manifest::new_from_previous(current_manifest, schema, Arc::new(final_fragments)); - - if let (Some(user_requested_version), Operation::Overwrite { .. }) = - (user_requested_version, &self.operation) - { - // If this is an overwrite operation and the user has requested a specific version - // then overwrite with that version. Otherwise, if the user didn't request a specific - // version, then overwrite with whatever version we had before. - prev_manifest.data_storage_format = DataStorageFormat::new(user_requested_version); - } - - prev_manifest - } else { - let data_storage_format = - Self::data_storage_format_from_files(&final_fragments, user_requested_version)?; - Manifest::new( - schema, - Arc::new(final_fragments), - data_storage_format, - reference_paths, - ) - }; - - manifest.tag.clone_from(&self.tag); - - if config.auto_set_feature_flags { - // Internal operations (e.g. CreateIndex) use ManifestWriteConfig::default() - // which has use_stable_row_ids = false. Without inheriting from the previous - // manifest, apply_feature_flags would clear FLAG_STABLE_ROW_IDS. - let inherited = current_manifest - .map(|m| m.uses_stable_row_ids()) - .unwrap_or(false); - let use_stable_row_ids = config.use_stable_row_ids || inherited; - apply_feature_flags( - &mut manifest, - use_stable_row_ids, - config.disable_transaction_file, - )?; - } - manifest.set_timestamp(timestamp_to_nanos(config.timestamp)); - - manifest.update_max_fragment_id(); - - match &self.operation { - Operation::Overwrite { - config_upsert_values: Some(tm), - .. - } => { - manifest.config_mut().extend(tm.clone()); - } - Operation::UpdateConfig { - config_updates, - table_metadata_updates, - schema_metadata_updates, - field_metadata_updates, - } => { - if let Some(config_updates) = config_updates { - let mut config = manifest.config.clone(); - apply_update_map(&mut config, config_updates); - manifest.config = config; - } - if let Some(table_metadata_updates) = table_metadata_updates { - let mut table_metadata = manifest.table_metadata.clone(); - apply_update_map(&mut table_metadata, table_metadata_updates); - manifest.table_metadata = table_metadata; - } - if let Some(schema_metadata_updates) = schema_metadata_updates { - let mut schema_metadata = manifest.schema.metadata.clone(); - apply_update_map(&mut schema_metadata, schema_metadata_updates); - manifest.schema.metadata = schema_metadata; - } - // The unenforced primary and clustering keys are reserved - // schema properties: each is immutable once set, and its - // reserved metadata keys cannot be written with an invalid - // value. Capture the prior keys, and whether this transaction - // writes a reserved key, before applying the updates so - // violations can be rejected below. This runs on every apply, - // including conflict-rebase, so it also rejects the - // concurrent-writer race. - let primary_key_before: Vec = manifest - .schema - .unenforced_primary_key() - .iter() - .map(|field| field.id) - .collect(); - let writes_primary_key = field_metadata_updates.values().any(|update| { - update.update_entries.iter().any(|entry| { - entry.key == LANCE_UNENFORCED_PRIMARY_KEY - || entry.key == LANCE_UNENFORCED_PRIMARY_KEY_POSITION - }) - }); - let clustering_key_before: Vec = manifest - .schema - .unenforced_clustering_key() - .iter() - .map(|field| field.id) - .collect(); - let writes_clustering_key = field_metadata_updates.values().any(|update| { - update - .update_entries - .iter() - .any(|entry| entry.key == LANCE_UNENFORCED_CLUSTERING_KEY_POSITION) - }); - for (field_id, field_metadata_update) in field_metadata_updates { - if let Some(field) = manifest.schema.field_by_id_mut(*field_id) { - apply_update_map(&mut field.metadata, field_metadata_update); - // Also set unenforced primary key based on updated field metadata. - field.unenforced_primary_key_position = field - .metadata - .get(LANCE_UNENFORCED_PRIMARY_KEY_POSITION) - .and_then(|s| s.parse::().ok()) - .or_else(|| { - field - .metadata - .get(LANCE_UNENFORCED_PRIMARY_KEY) - .filter(|s| { - matches!(s.to_lowercase().as_str(), "true" | "1" | "yes") - }) - .map(|_| 0) - }); - // Also set unenforced clustering key based on updated - // field metadata. - field.unenforced_clustering_key_position = field - .metadata - .get(LANCE_UNENFORCED_CLUSTERING_KEY_POSITION) - .and_then(|s| s.parse::().ok()); - } else { - return Err(Error::invalid_input_source( - format!("Field with id {} does not exist", field_id).into(), - )); - } - } - let primary_key_after: Vec = manifest - .schema - .unenforced_primary_key() - .iter() - .map(|field| field.id) - .collect(); - if !primary_key_before.is_empty() { - // The primary key is already set: reject any change to it, - // and any write that touches a reserved primary key. - if writes_primary_key || primary_key_after != primary_key_before { - return Err(Error::invalid_input( - "the unenforced primary key is a reserved key and cannot be changed once set", - )); - } - } else if writes_primary_key && primary_key_after.is_empty() { - // A reserved primary key was written but did not install a - // valid primary key (e.g. a non-marker flag value or a - // non-numeric position). - return Err(Error::invalid_input( - "the unenforced primary key is a reserved key and cannot be set to an invalid value", - )); - } - let clustering_key_after: Vec = manifest - .schema - .unenforced_clustering_key() - .iter() - .map(|field| field.id) - .collect(); - if !clustering_key_before.is_empty() { - // The clustering key is already set: reject any change to - // it, and any write that touches the reserved key. - if writes_clustering_key || clustering_key_after != clustering_key_before { - return Err(Error::invalid_input( - "the unenforced clustering key is a reserved key and cannot be changed once set", - )); - } - } else if writes_clustering_key && clustering_key_after.is_empty() { - // The reserved clustering key was written but did not - // install a valid clustering key (e.g. a non-numeric - // position value). - return Err(Error::invalid_input( - "the unenforced clustering key is a reserved key and cannot be set to an invalid value", - )); - } - } - _ => {} - } - - // Handle UpdateBases operation to update manifest base_paths - if let Operation::UpdateBases { new_bases } = &self.operation { - // Validate and add new base paths to the manifest - for new_base in new_bases { - // Check for conflicts with existing base paths - if let Some(existing_base) = manifest - .base_paths - .values() - .find(|bp| bp.name == new_base.name || bp.path == new_base.path) - { - return Err(Error::invalid_input(format!( - "Conflict detected: Base path with name '{:?}' or path '{}' already exists. Existing: name='{:?}', path='{}'", - new_base.name, new_base.path, existing_base.name, existing_base.path - ))); - } - - // Assign a new ID if not already assigned - let mut base_to_add = new_base.clone(); - if base_to_add.id == 0 { - let next_id = manifest - .base_paths - .keys() - .max() - .map(|&id| id + 1) - .unwrap_or(1); - base_to_add.id = next_id; - } - - manifest.base_paths.insert(base_to_add.id, base_to_add); - } - } - - if let Operation::ReserveFragments { num_fragments } = self.operation { - manifest.max_fragment_id = Some(manifest.max_fragment_id.unwrap_or(0) + num_fragments); - } - - manifest.transaction_file = Some(transaction_file_path.to_string()); - - if let Some(next_row_id) = next_row_id { - manifest.next_row_id = next_row_id; - } - - Ok((manifest, final_indices)) - } - - fn register_pure_rewrite_rows_update_frags_in_indices( - indices: &mut [IndexMetadata], - pure_update_frag_ids: &[u64], - original_fragment_ids: &[u64], - fields_for_preserving_frag_bitmap: &[u32], - ) { - if pure_update_frag_ids.is_empty() { - return; - } - - let value_updated_field_set = fields_for_preserving_frag_bitmap - .iter() - .collect::>(); - - for index in indices.iter_mut() { - let index_covers_modified_field = index.fields.iter().any(|field_id| { - value_updated_field_set.contains(&u32::try_from(*field_id).unwrap()) - }); - - if !index_covers_modified_field - && let Some(fragment_bitmap) = &mut index.fragment_bitmap - { - // check if all the original fragments contains the updating rows are covered - // by the index(index fragment bitmap contains these frag ids). - // if not, that means not all the updating rows are indexed, so we could not - // index them. - let index_covers_all_original_fragments = original_fragment_ids - .iter() - .all(|&fragment_id| fragment_bitmap.contains(fragment_id as u32)); - - if index_covers_all_original_fragments { - for fragment_id in pure_update_frag_ids.iter().map(|f| *f as u32) { - fragment_bitmap.insert(fragment_id); - } - } - } - } - } - - /// If an operation modifies one or more fields in a fragment then we need to remove - /// that fragment from any indices that cover one of the modified fields. - fn prune_updated_fields_from_indices( - indices: &mut [IndexMetadata], - updated_fragments: &[Fragment], - fields_modified: &[u32], - ) { - if fields_modified.is_empty() { - return; - } - - // If we modified any fields in the fragments then we need to remove those fragments - // from the index if the index covers one of those modified fields. - let fields_modified_set = fields_modified.iter().collect::>(); - for index in indices.iter_mut() { - if index - .fields - .iter() - .any(|field_id| fields_modified_set.contains(&u32::try_from(*field_id).unwrap())) - && let Some(fragment_bitmap) = &mut index.fragment_bitmap - { - for fragment_id in updated_fragments.iter().map(|f| f.id as u32) { - fragment_bitmap.remove(fragment_id); - } - } - } - } - - /// Map each (non-tombstoned) field id in a fragment to the path of the data - /// file that backs it. - fn fragment_field_paths(frag: &Fragment) -> HashMap { - let mut map = HashMap::new(); - for file in &frag.files { - for &field_id in file.fields.iter() { - if field_id >= 0 { - map.insert(field_id, file.path.as_str()); - } - } - } - map - } - - /// A `Merge` can rewrite a column's data *in place* -- the field stays in the - /// schema but its backing data file changes (the overlay fragment carries a new - /// file for the field and tombstones its old field id). `retain_relevant_indices` - /// only drops indices for *removed* fields, so without this the index keeps - /// covering the rewritten fragments with stale entries. Remove each such fragment - /// from any index covering a field whose backing data file changed. - fn prune_merge_rewritten_fields_from_indices( - indices: &mut [IndexMetadata], - prev_fragments: &[Fragment], - new_fragments: &[Fragment], - ) { - let prev_by_id: HashMap = - prev_fragments.iter().map(|f| (f.id, f)).collect(); - for new_frag in new_fragments { - let Some(prev) = prev_by_id.get(&new_frag.id) else { - continue; // brand-new fragment: nothing stale to prune - }; - let prev_paths = Self::fragment_field_paths(prev); - let new_paths = Self::fragment_field_paths(new_frag); - // Fields still present whose backing file path changed == rewritten data. - let changed: Vec = prev_paths - .iter() - .filter(|(field_id, prev_path)| { - new_paths - .get(*field_id) - .is_some_and(|new_path| new_path != *prev_path) - }) - .map(|(field_id, _)| *field_id as u32) - .collect(); - if changed.is_empty() { - continue; - } - Self::prune_updated_fields_from_indices( - indices, - std::slice::from_ref(new_frag), - &changed, - ); - } - } - - fn is_vector_index(index: &IndexMetadata) -> bool { - if let Some(details) = &index.index_details { - details.type_url.ends_with("VectorIndexDetails") - } else { - false - } - } - - /// Remove data files that only contain tombstoned fields (-2) - /// These files no longer contain any live data and can be safely dropped - fn remove_tombstoned_data_files(fragments: &mut [Fragment]) { - for fragment in fragments { - fragment.files.retain(|file| { - // Keep file if it has at least one non-tombstoned field - file.fields.iter().any(|&field_id| field_id != -2) - }); - } - } - - fn retain_relevant_indices( - indices: &mut Vec, - schema: &Schema, - fragments: &[Fragment], - ) { - let field_ids = schema - .fields_pre_order() - .map(|f| f.id) - .collect::>(); - - // Remove indices for fields no longer in schema - indices.retain(|existing_index| { - existing_index - .fields - .iter() - .all(|field_id| field_ids.contains(field_id)) - || is_system_index(existing_index) - }); - - // Fragment bitmaps record which fragments the index was originally built for. - // Operations like updates and data replacement prune these bitmaps, and - // effective_fragment_bitmap intersects with existing fragments at query time. - - // Apply retention logic for indices with empty bitmaps per index name - // (except for fragment reuse indices which are always kept) - let mut indices_by_name: std::collections::HashMap> = - std::collections::HashMap::new(); - - // Group indices by name - for index in indices.iter() { - if index.name != FRAG_REUSE_INDEX_NAME { - indices_by_name - .entry(index.name.clone()) - .or_default() - .push(index); - } - } - - // Build a set of UUIDs to keep based on retention rules - let mut uuids_to_keep = std::collections::HashSet::new(); - - let existing_fragments = fragments - .iter() - .map(|f| f.id as u32) - .collect::(); - - // For each group of indices with the same name - for (_, same_name_indices) in indices_by_name { - if same_name_indices.len() > 1 { - // Separate empty and non-empty indices - let (empty_indices, non_empty_indices): (Vec<_>, Vec<_>) = - same_name_indices.iter().partition(|index| { - index - .effective_fragment_bitmap(&existing_fragments) - .as_ref() - .is_none_or(|bitmap| bitmap.is_empty()) - }); - - if non_empty_indices.is_empty() { - // All indices are empty - for scalar indices, keep only the first (oldest) one - // For vector indices, remove all of them - let mut sorted_indices = empty_indices; - sorted_indices.sort_by_key(|index: &&IndexMetadata| index.dataset_version); // Sort by ascending dataset_version - - // Keep only the first (oldest) if it's not a vector index - if let Some(oldest) = sorted_indices.first() - && !Self::is_vector_index(oldest) - { - uuids_to_keep.insert(oldest.uuid); - } - } else { - // At least one index has non-empty bitmap - keep all non-empty indices - for index in non_empty_indices { - uuids_to_keep.insert(index.uuid); - } - } - } else { - // Single index - keep it unless it's an empty vector index - if let Some(index) = same_name_indices.first() { - let is_empty = index - .effective_fragment_bitmap(&existing_fragments) - .as_ref() - .is_none_or(|bitmap| bitmap.is_empty()); - let is_vector = Self::is_vector_index(index); - - // Keep the index unless it's an empty vector index - if !is_empty || !is_vector { - uuids_to_keep.insert(index.uuid); - } - } - } - } - - // Use Vec::retain to safely remove indices - indices.retain(|index| { - index.name == FRAG_REUSE_INDEX_NAME || uuids_to_keep.contains(&index.uuid) - }); - } - - fn recalculate_fragment_bitmap( - old: &RoaringBitmap, - groups: &[RewriteGroup], - ) -> Result { - let mut new_bitmap = old.clone(); - for group in groups { - let any_in_index = group - .old_fragments - .iter() - .any(|frag| old.contains(frag.id as u32)); - let all_in_index = group - .old_fragments - .iter() - .all(|frag| old.contains(frag.id as u32)); - // Any rewrite group may or may not be covered by the index. However, if any fragment - // in a rewrite group was previously covered by the index then all fragments in the rewrite - // group must have been previously covered by the index. plan_compaction takes care of - // this for us so this should be safe to assume. - if any_in_index { - if all_in_index { - for frag_id in group.old_fragments.iter().map(|frag| frag.id as u32) { - new_bitmap.remove(frag_id); - } - new_bitmap.extend(group.new_fragments.iter().map(|frag| frag.id as u32)); - } else { - return Err(Error::invalid_input( - "The compaction plan included a rewrite group that was a split of indexed and non-indexed data", - )); - } - } - } - Ok(new_bitmap) - } - - fn handle_rewrite_indices( - indices: &mut [IndexMetadata], - rewritten_indices: &[RewrittenIndex], - groups: &[RewriteGroup], - ) -> Result<()> { - let mut modified_indices = HashSet::new(); - - for rewritten_index in rewritten_indices { - if !modified_indices.insert(rewritten_index.old_id) { - return Err(Error::invalid_input(format!( - "An invalid compaction plan must have been generated because multiple tasks modified the same index: {}", - rewritten_index.old_id - ))); - } - - // Skip indices that no longer exist (may have been removed by concurrent operation) - let Some(index) = indices - .iter_mut() - .find(|idx| idx.uuid == rewritten_index.old_id) - else { - continue; - }; - - index.fragment_bitmap = Some(Self::recalculate_fragment_bitmap( - index.fragment_bitmap.as_ref().ok_or_else(|| { - Error::invalid_input(format!( - "Cannot rewrite index {} which did not store fragment bitmap", - index.uuid - )) - })?, - groups, - )?); - index.uuid = rewritten_index.new_id; - // Update file sizes to match the new index files. When not available - // (e.g., from older writers), clear the old file sizes to avoid - // using stale sizes from the pre-remap index. - index.files = rewritten_index.new_index_files.clone(); - } - Ok(()) - } - - fn handle_rewrite_fragments( - final_fragments: &mut Vec, - groups: &[RewriteGroup], - fragment_id: &mut u64, - version: u64, - _next_row_id: Option<&u64>, - ) -> Result<()> { - for group in groups { - // If the old fragments are contiguous, find the range - let replace_range = { - let start = final_fragments - .iter() - .enumerate() - .find(|(_, f)| f.id == group.old_fragments[0].id) - .ok_or_else(|| { - Error::commit_conflict_source( - version, - format!( - "dataset does not contain a fragment a rewrite operation wants to replace: id={}", - group.old_fragments[0].id - ) - .into(), - ) - })? - .0; - - // Verify old_fragments matches contiguous range - let mut i = 1; - loop { - if i == group.old_fragments.len() { - break Some(start..start + i); - } - if final_fragments[start + i].id != group.old_fragments[i].id { - break None; - } - i += 1; - } - }; - - let new_fragments = Self::fragments_with_ids(group.new_fragments.clone(), fragment_id) - .collect::>(); - - // Version metadata for rewritten fragments is handled by the compaction code - // (recalc_versions_for_rewritten_fragments) which preserves version information - // from the original fragments. We don't modify it here. - - if let Some(replace_range) = replace_range { - // Efficiently path using slice - final_fragments.splice(replace_range, new_fragments); - } else { - // Slower path for non-contiguous ranges - for fragment in group.old_fragments.iter() { - final_fragments.retain(|f| f.id != fragment.id); - } - final_fragments.extend(new_fragments); - } - } - Ok(()) - } - - /// collect the pure(the num of row IDs are equal to the physical rows) "rewrite rows" updated fragment ids - fn collect_pure_rewrite_row_update_frags_ids(fragments: &[Fragment]) -> Result> { - let mut pure_update_frag_ids = Vec::new(); - - for fragment in fragments { - let physical_rows = fragment - .physical_rows - .ok_or_else(|| Error::internal("Fragment does not have physical rows"))? - as u64; - - if let Some(row_id_meta) = &fragment.row_id_meta { - let existing_row_count = match row_id_meta { - RowIdMeta::Inline(data) => { - let sequence = read_row_ids(data)?; - sequence.len() as u64 - } - _ => 0, - }; - - // only filter the fragments that match: all the rows have row id, - // which means it does not contain inserted rows in this fragment - if existing_row_count == physical_rows { - pure_update_frag_ids.push(fragment.id); - } - } - } - - Ok(pure_update_frag_ids) - } - - fn assign_row_ids(next_row_id: &mut u64, fragments: &mut [Fragment]) -> Result<()> { - for fragment in fragments { - let physical_rows = fragment - .physical_rows - .ok_or_else(|| Error::internal("Fragment does not have physical rows"))? - as u64; - - if fragment.row_id_meta.is_some() { - // we may meet merge insert case, it only has partial row ids. - // so here, we need to check if the row ids match the physical rows - // if yes, continue - // if not, fill the remaining row ids to the physical rows, then update row_id_meta - - // Check if existing row IDs match the physical rows count - let existing_row_count = match &fragment.row_id_meta { - Some(RowIdMeta::Inline(data)) => { - // Parse the serialized row ID sequence to get the count - let sequence = read_row_ids(data)?; - sequence.len() as u64 - } - _ => 0, - }; - - match existing_row_count.cmp(&physical_rows) { - Ordering::Equal => { - // Row IDs already match physical rows, continue to next fragment - continue; - } - Ordering::Less => { - // Partial row IDs - need to fill the remaining ones - let remaining_rows = physical_rows - existing_row_count; - let new_row_ids = *next_row_id..(*next_row_id + remaining_rows); - - // Merge existing and new row IDs - let combined_sequence = match &fragment.row_id_meta { - Some(RowIdMeta::Inline(data)) => read_row_ids(data)?, - _ => { - return Err(Error::internal( - "Failed to deserialize existing row ID sequence", - )); - } - }; - - let mut row_ids: Vec = combined_sequence.iter().collect(); - for row_id in new_row_ids { - row_ids.push(row_id); - } - let combined_sequence = RowIdSequence::from(row_ids.as_slice()); - - let serialized = write_row_ids(&combined_sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); - *next_row_id += remaining_rows; - } - Ordering::Greater => { - // More row IDs than physical rows - this shouldn't happen - return Err(Error::internal(format!( - "Fragment has more row IDs ({}) than physical rows ({})", - existing_row_count, physical_rows - ))); - } - } - } else { - let row_ids = *next_row_id..(*next_row_id + physical_rows); - let sequence = RowIdSequence::from(row_ids); - // TODO: write to a separate file if large. Possibly share a file with other fragments. - let serialized = write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); - *next_row_id += physical_rows; - } - } - Ok(()) - } -} - -impl From<&DataReplacementGroup> for pb::transaction::DataReplacementGroup { - fn from(DataReplacementGroup(fragment_id, new_file): &DataReplacementGroup) -> Self { - Self { - fragment_id: *fragment_id, - new_file: Some(new_file.into()), - } - } -} - -/// Convert a protobug DataReplacementGroup to a rust native DataReplacementGroup -/// this is unfortunately TryFrom instead of From because of the Option in the pb::DataReplacementGroup -impl TryFrom for DataReplacementGroup { - type Error = Error; - - fn try_from(message: pb::transaction::DataReplacementGroup) -> Result { - Ok(Self( - message.fragment_id, - message - .new_file - .ok_or(Error::invalid_input( - "DataReplacementGroup must have a new_file", - ))? - .try_into()?, - )) - } -} - -impl TryFrom for Transaction { - type Error = Error; - - fn try_from(message: pb::Transaction) -> Result { - let operation = match message.operation { - Some(pb::transaction::Operation::Append(pb::transaction::Append { fragments })) => { - Operation::Append { - fragments: fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - } - } - Some(pb::transaction::Operation::Clone(pb::transaction::Clone { - is_shallow, - ref_name, - ref_version, - ref_path, - branch_name, - })) => Operation::Clone { - is_shallow, - ref_name, - ref_version, - ref_path, - branch_name, - }, - Some(pb::transaction::Operation::Delete(pb::transaction::Delete { - updated_fragments, - deleted_fragment_ids, - predicate, - })) => Operation::Delete { - updated_fragments: updated_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - deleted_fragment_ids, - predicate, - }, - Some(pb::transaction::Operation::Overwrite(pb::transaction::Overwrite { - fragments, - schema, - schema_metadata: _schema_metadata, // TODO: handle metadata - config_upsert_values, - initial_bases, - })) => { - let config_upsert_option = if config_upsert_values.is_empty() { - None - } else { - Some(config_upsert_values) - }; - - Operation::Overwrite { - fragments: fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - schema: Schema::from(&Fields(schema)), - config_upsert_values: config_upsert_option, - initial_bases: if initial_bases.is_empty() { - None - } else { - Some(initial_bases.into_iter().map(BasePath::from).collect()) - }, - } - } - Some(pb::transaction::Operation::ReserveFragments( - pb::transaction::ReserveFragments { num_fragments }, - )) => Operation::ReserveFragments { num_fragments }, - Some(pb::transaction::Operation::Rewrite(pb::transaction::Rewrite { - old_fragments, - new_fragments, - groups, - rewritten_indices, - })) => { - let groups = if !groups.is_empty() { - groups - .into_iter() - .map(RewriteGroup::try_from) - .collect::>()? - } else { - vec![RewriteGroup { - old_fragments: old_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - new_fragments: new_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - }] - }; - let rewritten_indices = rewritten_indices - .iter() - .map(RewrittenIndex::try_from) - .collect::>()?; - - Operation::Rewrite { - groups, - rewritten_indices, - frag_reuse_index: None, - } - } - Some(pb::transaction::Operation::CreateIndex(pb::transaction::CreateIndex { - new_indices, - removed_indices, - })) => Operation::CreateIndex { - new_indices: new_indices - .into_iter() - .map(IndexMetadata::try_from) - .collect::>()?, - removed_indices: removed_indices - .into_iter() - .map(IndexMetadata::try_from) - .collect::>()?, - }, - Some(pb::transaction::Operation::Merge(pb::transaction::Merge { - fragments, - schema, - schema_metadata: _schema_metadata, // TODO: handle metadata - })) => Operation::Merge { - fragments: fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - schema: Schema::from(&Fields(schema)), - }, - Some(pb::transaction::Operation::Restore(pb::transaction::Restore { version })) => { - Operation::Restore { version } - } - Some(pb::transaction::Operation::Update(pb::transaction::Update { - removed_fragment_ids, - updated_fragments, - new_fragments, - fields_modified, - merged_generations, - fields_for_preserving_frag_bitmap, - update_mode, - inserted_rows, - updated_fragment_offsets, - })) => Operation::Update { - removed_fragment_ids, - updated_fragments: updated_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - new_fragments: new_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - fields_modified, - merged_generations: merged_generations - .into_iter() - .map(|m| MergedGeneration::try_from(m).unwrap()) - .collect(), - fields_for_preserving_frag_bitmap, - update_mode: match update_mode { - 0 => Some(UpdateMode::RewriteRows), - 1 => Some(UpdateMode::RewriteColumns), - _ => Some(UpdateMode::RewriteRows), - }, - inserted_rows_filter: inserted_rows - .map(|ik| KeyExistenceFilter::try_from(&ik)) - .transpose()?, - updated_fragment_offsets: { - let m: HashMap = updated_fragment_offsets - .into_iter() - .filter(|(_, list)| !list.values.is_empty()) - .map(|(id, list)| (id, RoaringBitmap::from_iter(list.values))) - .collect(); - if m.is_empty() { - None - } else { - Some(UpdatedFragmentOffsets(m)) - } - }, - }, - Some(pb::transaction::Operation::Project(pb::transaction::Project { schema })) => { - Operation::Project { - schema: Schema::from(&Fields(schema)), - } - } - Some(pb::transaction::Operation::UpdateConfig(update_config)) => { - // Check if new-style fields are present - let has_new_fields = update_config.config_updates.is_some() - || update_config.table_metadata_updates.is_some() - || update_config.schema_metadata_updates.is_some() - || !update_config.field_metadata_updates.is_empty(); - - // Check if old-style fields are present - let has_old_fields = !update_config.upsert_values.is_empty() - || !update_config.delete_keys.is_empty() - || !update_config.schema_metadata.is_empty() - || !update_config.field_metadata.is_empty(); - - // Error if both are present - if has_new_fields && has_old_fields { - return Err(Error::invalid_input_source( - "Cannot mix old and new style UpdateConfig fields".into(), - )); - } - - if has_old_fields { - // Translate old-style to new-style - let config_updates = if !update_config.upsert_values.is_empty() - || !update_config.delete_keys.is_empty() - { - Some(translate_config_updates( - &update_config.upsert_values, - &update_config.delete_keys, - )) - } else { - None - }; - - let schema_metadata_updates = if !update_config.schema_metadata.is_empty() { - Some(translate_schema_metadata_updates( - &update_config.schema_metadata, - )) - } else { - None - }; - - let field_metadata_updates = update_config - .field_metadata - .into_iter() - .map(|(field_id, field_meta_update)| { - ( - field_id as i32, - translate_schema_metadata_updates(&field_meta_update.metadata), - ) - }) - .collect(); - - Operation::UpdateConfig { - config_updates, - table_metadata_updates: None, - schema_metadata_updates, - field_metadata_updates, - } - } else { - // Use new-style fields directly (convert from protobuf) - Operation::UpdateConfig { - config_updates: update_config.config_updates.as_ref().map(UpdateMap::from), - table_metadata_updates: update_config - .table_metadata_updates - .as_ref() - .map(UpdateMap::from), - schema_metadata_updates: update_config - .schema_metadata_updates - .as_ref() - .map(UpdateMap::from), - field_metadata_updates: update_config - .field_metadata_updates - .iter() - .map(|(field_id, pb_update_map)| { - (*field_id, UpdateMap::from(pb_update_map)) - }) - .collect(), - } - } - } - Some(pb::transaction::Operation::DataReplacement( - pb::transaction::DataReplacement { replacements }, - )) => Operation::DataReplacement { - replacements: replacements - .into_iter() - .map(DataReplacementGroup::try_from) - .collect::>>()?, - }, - Some(pb::transaction::Operation::UpdateMemWalState( - pb::transaction::UpdateMemWalState { merged_generations }, - )) => Operation::UpdateMemWalState { - merged_generations: merged_generations - .into_iter() - .map(|m| MergedGeneration::try_from(m).unwrap()) - .collect(), - }, - Some(pb::transaction::Operation::UpdateBases(pb::transaction::UpdateBases { - new_bases, - })) => Operation::UpdateBases { - new_bases: new_bases.into_iter().map(BasePath::from).collect(), - }, - None => { - return Err(Error::internal( - "Transaction message did not contain an operation".to_string(), - )); - } - }; - Ok(Self { - read_version: message.read_version, - uuid: message.uuid.clone(), - operation, - tag: if message.tag.is_empty() { - None - } else { - Some(message.tag.clone()) - }, - transaction_properties: if message.transaction_properties.is_empty() { - None - } else { - Some(Arc::new(message.transaction_properties)) - }, - }) - } -} - -impl TryFrom<&pb::transaction::rewrite::RewrittenIndex> for RewrittenIndex { - type Error = Error; - - fn try_from(message: &pb::transaction::rewrite::RewrittenIndex) -> Result { - Ok(Self { - old_id: message - .old_id - .as_ref() - .map(Uuid::try_from) - .ok_or_else(|| { - Error::invalid_input("required field (old_id) missing from message".to_string()) - })??, - new_id: message - .new_id - .as_ref() - .map(Uuid::try_from) - .ok_or_else(|| { - Error::invalid_input("required field (new_id) missing from message".to_string()) - })??, - new_index_details: message - .new_index_details - .as_ref() - .ok_or_else(|| { - Error::invalid_input("new_index_details is a required field".to_string()) - })? - .clone(), - new_index_version: message.new_index_version, - new_index_files: if message.new_index_files.is_empty() { - None - } else { - Some( - message - .new_index_files - .iter() - .map(|f| IndexFile { - path: f.path.clone(), - size_bytes: f.size_bytes, - }) - .collect(), - ) - }, - }) - } -} - -impl TryFrom for RewriteGroup { - type Error = Error; - - fn try_from(message: pb::transaction::rewrite::RewriteGroup) -> Result { - Ok(Self { - old_fragments: message - .old_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - new_fragments: message - .new_fragments - .into_iter() - .map(Fragment::try_from) - .collect::>>()?, - }) - } -} - -impl From<&Transaction> for pb::Transaction { - fn from(value: &Transaction) -> Self { - let operation = match &value.operation { - Operation::Append { fragments } => { - pb::transaction::Operation::Append(pb::transaction::Append { - fragments: fragments.iter().map(pb::DataFragment::from).collect(), - }) - } - Operation::Clone { - is_shallow, - ref_name, - ref_version, - ref_path, - branch_name, - } => pb::transaction::Operation::Clone(pb::transaction::Clone { - is_shallow: *is_shallow, - ref_name: ref_name.clone(), - ref_version: *ref_version, - ref_path: ref_path.clone(), - branch_name: branch_name.clone(), - }), - Operation::Delete { - updated_fragments, - deleted_fragment_ids, - predicate, - } => pb::transaction::Operation::Delete(pb::transaction::Delete { - updated_fragments: updated_fragments - .iter() - .map(pb::DataFragment::from) - .collect(), - deleted_fragment_ids: deleted_fragment_ids.clone(), - predicate: predicate.clone(), - }), - Operation::Overwrite { - fragments, - schema, - config_upsert_values, - initial_bases, - } => { - pb::transaction::Operation::Overwrite(pb::transaction::Overwrite { - fragments: fragments.iter().map(pb::DataFragment::from).collect(), - schema: Fields::from(schema).0, - schema_metadata: Default::default(), // TODO: handle metadata - config_upsert_values: config_upsert_values - .clone() - .unwrap_or(Default::default()), - initial_bases: initial_bases - .as_ref() - .map(|paths| { - paths - .iter() - .cloned() - .map(|bp: BasePath| -> pb::BasePath { bp.into() }) - .collect::>() - }) - .unwrap_or_default(), - }) - } - Operation::ReserveFragments { num_fragments } => { - pb::transaction::Operation::ReserveFragments(pb::transaction::ReserveFragments { - num_fragments: *num_fragments, - }) - } - Operation::Rewrite { - groups, - rewritten_indices, - frag_reuse_index: _, - } => pb::transaction::Operation::Rewrite(pb::transaction::Rewrite { - groups: groups - .iter() - .map(pb::transaction::rewrite::RewriteGroup::from) - .collect(), - rewritten_indices: rewritten_indices - .iter() - .map(|rewritten| rewritten.into()) - .collect(), - ..Default::default() - }), - Operation::CreateIndex { - new_indices, - removed_indices, - } => pb::transaction::Operation::CreateIndex(pb::transaction::CreateIndex { - new_indices: new_indices.iter().map(pb::IndexMetadata::from).collect(), - removed_indices: removed_indices - .iter() - .map(pb::IndexMetadata::from) - .collect(), - }), - Operation::Merge { fragments, schema } => { - pb::transaction::Operation::Merge(pb::transaction::Merge { - fragments: fragments.iter().map(pb::DataFragment::from).collect(), - schema: Fields::from(schema).0, - schema_metadata: Default::default(), // TODO: handle metadata - }) - } - Operation::Restore { version } => { - pb::transaction::Operation::Restore(pb::transaction::Restore { version: *version }) - } - Operation::Update { - removed_fragment_ids, - updated_fragments, - new_fragments, - fields_modified, - merged_generations, - fields_for_preserving_frag_bitmap, - update_mode, - inserted_rows_filter, - updated_fragment_offsets, - } => pb::transaction::Operation::Update(pb::transaction::Update { - removed_fragment_ids: removed_fragment_ids.clone(), - updated_fragments: updated_fragments - .iter() - .map(pb::DataFragment::from) - .collect(), - new_fragments: new_fragments.iter().map(pb::DataFragment::from).collect(), - fields_modified: fields_modified.clone(), - merged_generations: merged_generations - .iter() - .map(pb::MergedGeneration::from) - .collect(), - fields_for_preserving_frag_bitmap: fields_for_preserving_frag_bitmap.clone(), - update_mode: update_mode - .as_ref() - .map(|mode| match mode { - UpdateMode::RewriteRows => 0, - UpdateMode::RewriteColumns => 1, - }) - .unwrap_or(0), - inserted_rows: inserted_rows_filter.as_ref().map(|ik| ik.into()), - updated_fragment_offsets: updated_fragment_offsets - .as_ref() - .map(|UpdatedFragmentOffsets(m)| { - m.iter() - .filter(|(_, b)| !b.is_empty()) - .map(|(frag_id, b)| { - let values: Vec = b.iter().collect(); - (*frag_id, pb::transaction::UInt32List { values }) - }) - .collect::>() - }) - .unwrap_or_default(), - }), - Operation::Project { schema } => { - pb::transaction::Operation::Project(pb::transaction::Project { - schema: Fields::from(schema).0, - }) - } - Operation::UpdateConfig { - config_updates, - table_metadata_updates, - schema_metadata_updates, - field_metadata_updates, - } => pb::transaction::Operation::UpdateConfig(pb::transaction::UpdateConfig { - config_updates: config_updates - .as_ref() - .map(pb::transaction::UpdateMap::from), - table_metadata_updates: table_metadata_updates - .as_ref() - .map(pb::transaction::UpdateMap::from), - schema_metadata_updates: schema_metadata_updates - .as_ref() - .map(pb::transaction::UpdateMap::from), - field_metadata_updates: field_metadata_updates - .iter() - .map(|(field_id, update_map)| { - (*field_id, pb::transaction::UpdateMap::from(update_map)) - }) - .collect(), - // Leave old fields empty - we only write new-style fields - upsert_values: Default::default(), - delete_keys: Default::default(), - schema_metadata: Default::default(), - field_metadata: Default::default(), - }), - Operation::DataReplacement { replacements } => { - pb::transaction::Operation::DataReplacement(pb::transaction::DataReplacement { - replacements: replacements - .iter() - .map(pb::transaction::DataReplacementGroup::from) - .collect(), - }) - } - Operation::UpdateMemWalState { merged_generations } => { - pb::transaction::Operation::UpdateMemWalState(pb::transaction::UpdateMemWalState { - merged_generations: merged_generations - .iter() - .map(pb::MergedGeneration::from) - .collect::>(), - }) - } - Operation::UpdateBases { new_bases } => { - pb::transaction::Operation::UpdateBases(pb::transaction::UpdateBases { - new_bases: new_bases - .iter() - .cloned() - .map(|bp: BasePath| -> pb::BasePath { bp.into() }) - .collect::>(), - }) - } - }; - - let transaction_properties = value - .transaction_properties - .as_ref() - .map(|arc| arc.as_ref().clone()) - .unwrap_or_default(); - Self { - read_version: value.read_version, - uuid: value.uuid.clone(), - operation: Some(operation), - tag: value.tag.clone().unwrap_or("".to_string()), - transaction_properties, - } - } -} - -impl From<&RewrittenIndex> for pb::transaction::rewrite::RewrittenIndex { - fn from(value: &RewrittenIndex) -> Self { - Self { - old_id: Some((&value.old_id).into()), - new_id: Some((&value.new_id).into()), - new_index_details: Some(value.new_index_details.clone()), - new_index_version: value.new_index_version, - new_index_files: value - .new_index_files - .as_ref() - .map(|files| { - files - .iter() - .map(|f| pb::IndexFile { - path: f.path.clone(), - size_bytes: f.size_bytes, - }) - .collect() - }) - .unwrap_or_default(), - } - } -} - -impl From<&RewriteGroup> for pb::transaction::rewrite::RewriteGroup { - fn from(value: &RewriteGroup) -> Self { - Self { - old_fragments: value - .old_fragments - .iter() - .map(pb::DataFragment::from) - .collect(), - new_fragments: value - .new_fragments - .iter() - .map(pb::DataFragment::from) - .collect(), - } - } -} - -/// Validate the operation is valid for the given manifest. -pub fn validate_operation(manifest: Option<&Manifest>, operation: &Operation) -> Result<()> { - let manifest = match (manifest, operation) { - ( - None, - Operation::Overwrite { - fragments, schema, .. - }, - ) => { - // Validate here because we are going to return early. - schema_fragments_valid(None, schema, fragments)?; - - return Ok(()); - } - (None, Operation::Clone { .. }) => return Ok(()), - (Some(manifest), _) => manifest, - (None, _) => { - return Err(Error::invalid_input(format!( - "Cannot apply operation {} to non-existent dataset", - operation.name() - ))); - } - }; - - match operation { - Operation::Append { fragments } => { - // Fragments must contain all fields in the schema - schema_fragments_valid(Some(manifest), &manifest.schema, fragments) - } - Operation::Project { schema } => { - schema_fragments_valid(Some(manifest), schema, manifest.fragments.as_ref()) - } - Operation::Merge { fragments, schema } => { - merge_fragments_valid(manifest, fragments)?; - schema_fragments_valid(Some(manifest), schema, fragments) - } - Operation::Overwrite { - fragments, - schema, - config_upsert_values: None, - initial_bases: _, - } => { - // Pass None for manifest because Overwrite replaces all fragments. - // The old manifest's storage format is irrelevant for validating - // the new fragments (e.g., LEGACY→STABLE transitions). - schema_fragments_valid(None, schema, fragments) - } - Operation::Update { - updated_fragments, - new_fragments, - .. - } => { - schema_fragments_valid(Some(manifest), &manifest.schema, updated_fragments)?; - schema_fragments_valid(Some(manifest), &manifest.schema, new_fragments) - } - _ => Ok(()), - } -} - -fn schema_fragments_valid( - manifest: Option<&Manifest>, - schema: &Schema, - fragments: &[Fragment], -) -> Result<()> { - if let Some(manifest) = manifest - && manifest.data_storage_format.lance_file_version()? == LanceFileVersion::Legacy - { - return schema_fragments_legacy_valid(schema, fragments); - } - // validate that each data file at least contains one field. - for fragment in fragments { - for data_file in &fragment.files { - if data_file.fields.iter().len() == 0 { - return Err(Error::invalid_input(format!( - "Datafile {} does not contain any fields", - data_file.path - ))); - } - } - } - Ok(()) -} - -/// Check that each fragment contains all fields in the schema. -/// It is not required that the schema contains all fields in the fragment. -/// There may be masked fields. -fn schema_fragments_legacy_valid(schema: &Schema, fragments: &[Fragment]) -> Result<()> { - // TODO: add additional validation. Consider consolidating with various - // validate() methods in the codebase. - for fragment in fragments { - for field in schema.fields_pre_order() { - if !fragment - .files - .iter() - .flat_map(|f| f.fields.iter()) - .any(|f_id| f_id == &field.id) - { - return Err(Error::invalid_input(format!( - "Fragment {} does not contain field {:?}", - fragment.id, field - ))); - } - } - } - Ok(()) -} - -/// Returns true if Operation::Merge rewrote this fragment's column data files (Fragment::files -/// changed versus the previous manifest). Used to bump last_updated_at_version_meta only when -/// new column values were materialized to disk. -/// -/// Deletion file changes alone are not treated as rewrites: tombstones remove rows but -/// survivors did not receive new column bytes; stamping last_updated for those rows would be -/// incorrect for CDF. -#[inline] -fn merge_fragment_physically_rewritten(prev: &Fragment, merged: &Fragment) -> bool { - debug_assert_eq!(prev.id, merged.id); - if prev.files.len() != merged.files.len() { - return true; - } - // Compare identity fields only. file_size_bytes is an AtomicU64 cache that - // concurrent scans can populate in place on the manifest's DataFile, so it - // must not be part of the rewrite check. - prev.files.iter().zip(merged.files.iter()).any(|(p, m)| { - p.path != m.path - || p.fields != m.fields - || p.column_indices != m.column_indices - || p.file_major_version != m.file_major_version - || p.file_minor_version != m.file_minor_version - || p.base_id != m.base_id - }) -} - -/// Validate that Merge operations preserve all original fragments. -/// Merge operations should only add columns or rows, not reduce fragments. -/// This ensures fragments correspond at one-to-one with the original fragment list. -fn merge_fragments_valid(manifest: &Manifest, new_fragments: &[Fragment]) -> Result<()> { - let original_fragments = manifest.fragments.as_ref(); - - // Additional validation: ensure we're not accidentally reducing the fragment count - if new_fragments.len() < original_fragments.len() { - return Err(Error::invalid_input(format!( - "Merge operation reduced fragment count from {} to {}. \ - Merge operations should only add columns, not reduce fragments.", - original_fragments.len(), - new_fragments.len() - ))); - } - - // Collect new fragment IDs - let new_fragment_map: HashMap = - new_fragments.iter().map(|f| (f.id, f)).collect(); - - // Check that all original fragments are preserved in the new fragments list - // Validate that each original fragment's metadata is preserved - let mut missing_fragments: Vec = Vec::new(); - for original_fragment in original_fragments { - if let Some(new_fragment) = new_fragment_map.get(&original_fragment.id) { - // Validate physical_rows (row count) hasn't changed - if original_fragment.physical_rows != new_fragment.physical_rows { - return Err(Error::invalid_input(format!( - "Merge operation changed row count for fragment {}. \ - Original: {:?}, New: {:?}. \ - Merge operations should preserve fragment row counts and only add new columns.", - original_fragment.id, - original_fragment.physical_rows, - new_fragment.physical_rows - ))); - } - } else { - missing_fragments.push(original_fragment.id); - } - } - - if !missing_fragments.is_empty() { - return Err(Error::invalid_input(format!( - "Merge operation is missing original fragments: {:?}. \ - Merge operations should preserve all original fragments and only add new columns. \ - Expected fragments: {:?}, but got: {:?}", - missing_fragments, - original_fragments.iter().map(|f| f.id).collect::>(), - new_fragment_map.keys().copied().collect::>() - ))); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use arrow_array::cast::AsArray; - use arrow_array::types::UInt64Type; - use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator}; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use chrono::Utc; - use futures::TryStreamExt; - use lance_core::datatypes::Schema as LanceSchema; - use lance_core::utils::address::RowAddress; - use lance_core::utils::tempfile::TempStrDir; - use lance_core::{ROW_ADDR, ROW_CREATED_AT_VERSION, ROW_LAST_UPDATED_AT_VERSION}; - use lance_file::version::LanceFileVersion; - use lance_io::utils::CachedFileSize; - use lance_table::format::{ - RowDatasetVersionMeta, RowDatasetVersionRun, RowDatasetVersionSequence, RowIdMeta, - }; - use lance_table::rowids::segment::U64Segment; - use lance_table::rowids::write_row_ids; - use std::collections::HashMap; - use std::sync::Arc; - use uuid::Uuid; - - use crate::Dataset; - use crate::dataset::write::WriteParams; - use crate::session::Session; - - fn sample_manifest() -> Manifest { - let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); - Manifest::new( - LanceSchema::try_from(&schema).unwrap(), - Arc::new(vec![Fragment::new(0)]), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ) - } - - fn sample_index_metadata(name: &str) -> IndexMetadata { - IndexMetadata { - uuid: Uuid::new_v4(), - fields: vec![0], - name: name.to_string(), - dataset_version: 0, - fragment_bitmap: Some([0].into_iter().collect()), - index_details: None, - index_version: 1, - created_at: Some(Utc::now()), - base_id: None, - files: None, - } - } - - #[test] - fn test_rewrite_fragments() { - let existing_fragments: Vec = (0..10).map(Fragment::new).collect(); - - let mut final_fragments = existing_fragments; - let rewrite_groups = vec![ - // Since these are contiguous, they will be put in the same location - // as 1 and 2. - RewriteGroup { - old_fragments: vec![Fragment::new(1), Fragment::new(2)], - // These two fragments were previously reserved - new_fragments: vec![Fragment::new(15), Fragment::new(16)], - }, - // These are not contiguous, so they will be inserted at the end. - RewriteGroup { - old_fragments: vec![Fragment::new(5), Fragment::new(8)], - // We pretend this id was not reserved. Does not happen in practice today - // but we want to leave the door open. - new_fragments: vec![Fragment::new(0)], - }, - ]; - - let mut fragment_id = 20; - let version = 0; - - Transaction::handle_rewrite_fragments( - &mut final_fragments, - &rewrite_groups, - &mut fragment_id, - version, - None, - ) - .unwrap(); - - assert_eq!(fragment_id, 21); - - let expected_fragments: Vec = vec![ - Fragment::new(0), - Fragment::new(15), - Fragment::new(16), - Fragment::new(3), - Fragment::new(4), - Fragment::new(6), - Fragment::new(7), - Fragment::new(9), - Fragment::new(20), - ]; - - assert_eq!(final_fragments, expected_fragments); - } - - #[test] - fn test_merge_fragments_valid() { - // Create a simple schema for testing - let schema = ArrowSchema::new(vec![ - ArrowField::new("id", DataType::Int32, false), - ArrowField::new("name", DataType::Utf8, false), - ]); - - // Create original fragments - let original_fragments = vec![Fragment::new(1), Fragment::new(2), Fragment::new(3)]; - - // Create a manifest with original fragments - let manifest = Manifest::new( - LanceSchema::try_from(&schema).unwrap(), - Arc::new(original_fragments), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ); - - // Test 1: Empty fragments should fail - let empty_fragments = vec![]; - let result = merge_fragments_valid(&manifest, &empty_fragments); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("reduced fragment count") - ); - - // Test 2: Missing original fragments should fail - let missing_fragments = vec![ - Fragment::new(1), - Fragment::new(2), - // Fragment 3 is missing - Fragment::new(4), // New fragment - ]; - let result = merge_fragments_valid(&manifest, &missing_fragments); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("missing original fragments") - ); - - // Test 3: Reduced fragment count should fail - let reduced_fragments = vec![ - Fragment::new(1), - Fragment::new(2), - // Fragment 3 is missing, no new fragments added - ]; - let result = merge_fragments_valid(&manifest, &reduced_fragments); - assert!(result.is_err()); - assert!( - result - .unwrap_err() - .to_string() - .contains("reduced fragment count") - ); - - // Test 4: Valid merge with all original fragments plus new ones should succeed - let valid_fragments = vec![ - Fragment::new(1), - Fragment::new(2), - Fragment::new(3), - Fragment::new(4), // New fragment - Fragment::new(5), // Another new fragment - ]; - let result = merge_fragments_valid(&manifest, &valid_fragments); - assert!(result.is_ok()); - - // Test 5: Same fragments (no new ones) should succeed - let same_fragments = vec![Fragment::new(1), Fragment::new(2), Fragment::new(3)]; - let result = merge_fragments_valid(&manifest, &same_fragments); - assert!(result.is_ok()); - } - - #[test] - fn test_create_index_build_manifest_keeps_unremoved_same_name_indices() { - let manifest = sample_manifest(); - let first_index = sample_index_metadata("vector_idx"); - let second_index = sample_index_metadata("vector_idx"); - let third_index = sample_index_metadata("vector_idx"); - - let transaction = Transaction::new( - manifest.version, - Operation::CreateIndex { - new_indices: vec![third_index.clone()], - removed_indices: vec![second_index.clone()], - }, - None, - ); - - let (_, final_indices) = transaction - .build_manifest( - Some(&manifest), - vec![first_index.clone(), second_index.clone()], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert_eq!(final_indices.len(), 2); - assert!(final_indices.iter().any(|idx| idx.uuid == first_index.uuid)); - assert!(final_indices.iter().any(|idx| idx.uuid == third_index.uuid)); - assert!( - !final_indices - .iter() - .any(|idx| idx.uuid == second_index.uuid) - ); - } - - #[test] - fn test_create_index_build_manifest_deduplicates_relisted_indices_by_uuid() { - let manifest = sample_manifest(); - let first_index = sample_index_metadata("vector_idx"); - let second_index = sample_index_metadata("vector_idx"); - let third_index = sample_index_metadata("vector_idx"); - - let transaction = Transaction::new( - manifest.version, - Operation::CreateIndex { - new_indices: vec![first_index.clone(), third_index.clone()], - removed_indices: vec![second_index.clone()], - }, - None, - ); - - let (_, final_indices) = transaction - .build_manifest( - Some(&manifest), - vec![first_index.clone(), second_index.clone()], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert_eq!(final_indices.len(), 2); - assert_eq!( - final_indices - .iter() - .filter(|idx| idx.uuid == first_index.uuid) - .count(), - 1 - ); - assert!(final_indices.iter().any(|idx| idx.uuid == third_index.uuid)); - assert!( - !final_indices - .iter() - .any(|idx| idx.uuid == second_index.uuid) - ); - } - - #[test] - fn test_remove_tombstoned_data_files() { - // Create a fragment with mixed data files: some normal, some fully tombstoned - let mut fragment = Fragment::new(1); - - // Add a normal data file with valid field IDs - fragment.files.push(DataFile { - path: "normal.lance".to_string(), - fields: Arc::from([1, 2, 3]), - column_indices: Arc::from([]), - file_major_version: 2, - file_minor_version: 0, - file_size_bytes: CachedFileSize::new(1000), - base_id: None, - }); - - // Add a data file with all fields tombstoned - fragment.files.push(DataFile { - path: "all_tombstoned.lance".to_string(), - fields: Arc::from([-2, -2, -2]), - column_indices: Arc::from([]), - file_major_version: 2, - file_minor_version: 0, - file_size_bytes: CachedFileSize::new(500), - base_id: None, - }); - - // Add a data file with mixed tombstoned and valid fields - fragment.files.push(DataFile { - path: "mixed.lance".to_string(), - fields: Arc::from([4, -2, 5]), - column_indices: Arc::from([]), - file_major_version: 2, - file_minor_version: 0, - file_size_bytes: CachedFileSize::new(750), - base_id: None, - }); - - // Add another fully tombstoned file - fragment.files.push(DataFile { - path: "another_tombstoned.lance".to_string(), - fields: Arc::from([-2_i32]), - column_indices: Arc::from([]), - file_major_version: 2, - file_minor_version: 0, - file_size_bytes: CachedFileSize::new(250), - base_id: None, - }); - - let mut fragments = vec![fragment]; - - // Apply the cleanup - Transaction::remove_tombstoned_data_files(&mut fragments); - - // Should have removed the two fully tombstoned files - assert_eq!(fragments[0].files.len(), 2); - assert_eq!(fragments[0].files[0].path, "normal.lance"); - assert_eq!(fragments[0].files[1].path, "mixed.lance"); - } - - #[test] - fn test_assign_row_ids_new_fragment() { - // Test assigning row IDs to a fragment without existing row IDs - let mut fragments = vec![Fragment { - id: 1, - physical_rows: Some(100), - row_id_meta: None, - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }]; - let mut next_row_id = 0; - - Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); - - assert_eq!(next_row_id, 100); - assert!(fragments[0].row_id_meta.is_some()); - - if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { - let sequence = read_row_ids(data).unwrap(); - assert_eq!(sequence.len(), 100); - let row_ids: Vec = sequence.iter().collect(); - assert_eq!(row_ids, (0..100).collect::>()); - } else { - panic!("Expected inline row ID metadata"); - } - } - - #[test] - fn test_assign_row_ids_existing_complete() { - // Test with fragment that already has complete row IDs - let existing_sequence = RowIdSequence::from(0..50); - let serialized = write_row_ids(&existing_sequence); - - let mut fragments = vec![Fragment { - id: 1, - physical_rows: Some(50), - row_id_meta: Some(RowIdMeta::Inline(serialized)), - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }]; - let mut next_row_id = 100; - - Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); - - // next_row_id should not change - assert_eq!(next_row_id, 100); - - if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { - let sequence = read_row_ids(data).unwrap(); - assert_eq!(sequence.len(), 50); - let row_ids: Vec = sequence.iter().collect(); - assert_eq!(row_ids, (0..50).collect::>()); - } else { - panic!("Expected inline row ID metadata"); - } - } - - #[test] - fn test_assign_row_ids_partial_existing() { - // Test with fragment that has partial row IDs (merge insert case) - let existing_sequence = RowIdSequence::from(0..30); - let serialized = write_row_ids(&existing_sequence); - - let mut fragments = vec![Fragment { - id: 1, - physical_rows: Some(50), // More physical rows than existing row IDs - row_id_meta: Some(RowIdMeta::Inline(serialized)), - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }]; - let mut next_row_id = 100; - - Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); - - // next_row_id should advance by 20 (50 - 30) - assert_eq!(next_row_id, 120); - - if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { - let sequence = read_row_ids(data).unwrap(); - assert_eq!(sequence.len(), 50); - let row_ids: Vec = sequence.iter().collect(); - // Should contain original 0-29 plus new 100-119 - let mut expected = (0..30).collect::>(); - expected.extend(100..120); - assert_eq!(row_ids, expected); - } else { - panic!("Expected inline row ID metadata"); - } - } - - #[test] - fn test_assign_row_ids_excess_row_ids() { - // Test error case where fragment has more row IDs than physical rows - let existing_sequence = RowIdSequence::from(0..60); - let serialized = write_row_ids(&existing_sequence); - - let mut fragments = vec![Fragment { - id: 1, - physical_rows: Some(50), // Less physical rows than existing row IDs - row_id_meta: Some(RowIdMeta::Inline(serialized)), - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }]; - let mut next_row_id = 100; - - let result = Transaction::assign_row_ids(&mut next_row_id, &mut fragments); - - assert!(result.is_err()); - if let Err(Error::Internal { message, .. }) = result { - assert!(message.contains("more row IDs (60) than physical rows (50)")); - } else { - panic!("Expected Internal error about excess row IDs"); - } - } - - #[test] - fn test_assign_row_ids_multiple_fragments() { - // Test with multiple fragments, some with existing row IDs, some without - let existing_sequence = RowIdSequence::from(500..520); - let serialized = write_row_ids(&existing_sequence); - - let mut fragments = vec![ - Fragment { - id: 1, - physical_rows: Some(30), // No existing row IDs - row_id_meta: None, - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }, - Fragment { - id: 2, - physical_rows: Some(25), // Partial existing row IDs - row_id_meta: Some(RowIdMeta::Inline(serialized)), - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }, - ]; - let mut next_row_id = 1000; - - Transaction::assign_row_ids(&mut next_row_id, &mut fragments).unwrap(); - - // Should advance by 30 (first fragment) + 5 (second fragment partial) - assert_eq!(next_row_id, 1035); - - // Check first fragment - if let Some(RowIdMeta::Inline(data)) = &fragments[0].row_id_meta { - let sequence = read_row_ids(data).unwrap(); - assert_eq!(sequence.len(), 30); - let row_ids: Vec = sequence.iter().collect(); - assert_eq!(row_ids, (1000..1030).collect::>()); - } else { - panic!("Expected inline row ID metadata for first fragment"); - } - - // Check second fragment - if let Some(RowIdMeta::Inline(data)) = &fragments[1].row_id_meta { - let sequence = read_row_ids(data).unwrap(); - assert_eq!(sequence.len(), 25); - let row_ids: Vec = sequence.iter().collect(); - // Should contain original 500-519 plus new 1030-1034 - let mut expected = (500..520).collect::>(); - expected.extend(1030..1035); - assert_eq!(row_ids, expected); - } else { - panic!("Expected inline row ID metadata for second fragment"); - } - } - - #[test] - fn test_assign_row_ids_missing_physical_rows() { - // Test error case where fragment doesn't have physical_rows set - let mut fragments = vec![Fragment { - id: 1, - physical_rows: None, - row_id_meta: None, - files: vec![], - deletion_file: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }]; - let mut next_row_id = 0; - - let result = Transaction::assign_row_ids(&mut next_row_id, &mut fragments); - - assert!(result.is_err()); - if let Err(Error::Internal { message, .. }) = result { - assert!(message.contains("Fragment does not have physical rows")); - } else { - panic!("Expected Internal error about missing physical rows"); - } - } - - // Helper functions for retain_relevant_indices tests - fn create_test_index( - name: &str, - field_id: i32, - dataset_version: u64, - fragment_bitmap: Option, - is_vector: bool, - ) -> IndexMetadata { - use prost_types::Any; - use std::sync::Arc; - use uuid::Uuid; - - let index_details = if is_vector { - Some(Arc::new(Any { - type_url: "type.googleapis.com/lance.index.VectorIndexDetails".to_string(), - value: vec![], - })) - } else { - Some(Arc::new(Any { - type_url: "type.googleapis.com/lance.index.ScalarIndexDetails".to_string(), - value: vec![], - })) - }; - - IndexMetadata { - uuid: Uuid::new_v4(), - fields: vec![field_id], - name: name.to_string(), - dataset_version, - fragment_bitmap, - index_details, - index_version: 1, - created_at: None, - base_id: None, - files: None, - } - } - - fn create_system_index(name: &str, field_id: i32) -> IndexMetadata { - use prost_types::Any; - use std::sync::Arc; - use uuid::Uuid; - - IndexMetadata { - uuid: Uuid::new_v4(), - fields: vec![field_id], - name: name.to_string(), - dataset_version: 1, - fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2])), - index_details: Some(Arc::new(Any { - type_url: "type.googleapis.com/lance.index.SystemIndexDetails".to_string(), - value: vec![], - })), - index_version: 1, - created_at: None, - base_id: None, - files: None, - } - } - - fn create_test_schema(field_ids: &[i32]) -> Schema { - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; - use lance_core::datatypes::Schema as LanceSchema; - - let fields: Vec = field_ids - .iter() - .map(|id| ArrowField::new(format!("field_{}", id), DataType::Int32, false)) - .collect(); - - let arrow_schema = ArrowSchema::new(fields); - let mut lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // Assign field IDs - for (i, field_id) in field_ids.iter().enumerate() { - lance_schema.mut_field_by_id(i as i32).unwrap().id = *field_id; - } - - lance_schema - } - - #[test] - fn test_retain_indices_removes_missing_fields() { - let schema = create_test_schema(&[1, 2]); - let fragments = vec![Fragment::new(1), Fragment::new(2)]; - - let mut indices = vec![ - create_test_index("idx1", 1, 1, Some(RoaringBitmap::from_iter([1])), false), - create_test_index("idx2", 2, 1, Some(RoaringBitmap::from_iter([1])), false), - create_test_index("idx3", 99, 1, Some(RoaringBitmap::from_iter([1])), false), // Field doesn't exist - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - assert_eq!(indices.len(), 2); - assert!(indices.iter().all(|idx| idx.fields[0] != 99)); - } - - #[test] - fn test_retain_indices_keeps_system_indices() { - use lance_index::mem_wal::MEM_WAL_INDEX_NAME; - - let schema = create_test_schema(&[1, 2]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_system_index(FRAG_REUSE_INDEX_NAME, 99), // Field doesn't exist but should be kept - create_system_index(MEM_WAL_INDEX_NAME, 99), // Field doesn't exist but should be kept - create_test_index("regular_idx", 99, 1, Some(RoaringBitmap::new()), false), // Should be removed - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - assert_eq!(indices.len(), 2); - assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); - assert!(indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME)); - } - - #[test] - fn test_retain_indices_keeps_fragment_reuse_index() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_system_index(FRAG_REUSE_INDEX_NAME, 1), - create_test_index("other_idx", 1, 1, Some(RoaringBitmap::new()), false), - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Fragment reuse index should always be kept - assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); - } - - #[test] - fn test_retain_single_empty_scalar_index() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![create_test_index( - "scalar_idx", - 1, - 1, - Some(RoaringBitmap::new()), // Empty bitmap - false, - )]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Single empty scalar index should be kept - assert_eq!(indices.len(), 1); - } - - #[test] - fn test_retain_single_empty_vector_index() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![create_test_index( - "vector_idx", - 1, - 1, - Some(RoaringBitmap::new()), // Empty bitmap - true, - )]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Single empty vector index should be removed - assert_eq!(indices.len(), 0); - } - - #[test] - fn test_retain_single_nonempty_index() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut scalar_indices = vec![create_test_index( - "scalar_idx", - 1, - 1, - Some(RoaringBitmap::from_iter([1])), - false, - )]; - - let mut vector_indices = vec![create_test_index( - "vector_idx", - 1, - 1, - Some(RoaringBitmap::from_iter([1])), - true, - )]; - - Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); - Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); - - // Both should be kept - assert_eq!(scalar_indices.len(), 1); - assert_eq!(vector_indices.len(), 1); - } - - #[test] - fn test_retain_single_index_with_none_bitmap() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut scalar_indices = vec![create_test_index("scalar_idx", 1, 1, None, false)]; - let mut vector_indices = vec![create_test_index("vector_idx", 1, 1, None, true)]; - - Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); - Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); - - // Scalar should be kept, vector should be removed - assert_eq!(scalar_indices.len(), 1); - assert_eq!(vector_indices.len(), 0); - } - - #[test] - fn test_retain_multiple_empty_scalar_indices_keeps_oldest() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_test_index("idx", 1, 3, Some(RoaringBitmap::new()), false), - create_test_index("idx", 1, 1, Some(RoaringBitmap::new()), false), // Oldest - create_test_index("idx", 1, 2, Some(RoaringBitmap::new()), false), - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Should keep only the oldest (dataset_version = 1) - assert_eq!(indices.len(), 1); - assert_eq!(indices[0].dataset_version, 1); - } - - #[test] - fn test_retain_multiple_empty_vector_indices_removes_all() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_test_index("vec_idx", 1, 1, Some(RoaringBitmap::new()), true), - create_test_index("vec_idx", 1, 2, Some(RoaringBitmap::new()), true), - create_test_index("vec_idx", 1, 3, Some(RoaringBitmap::new()), true), - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // All empty vector indices should be removed - assert_eq!(indices.len(), 0); - } - - #[test] - fn test_retain_mixed_empty_nonempty_keeps_nonempty() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_test_index("idx", 1, 1, Some(RoaringBitmap::new()), false), // Empty - create_test_index("idx", 1, 2, Some(RoaringBitmap::from_iter([1])), false), // Non-empty - create_test_index("idx", 1, 3, Some(RoaringBitmap::new()), false), // Empty - create_test_index("idx", 1, 4, Some(RoaringBitmap::from_iter([1])), false), // Non-empty - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Should keep only non-empty indices - assert_eq!(indices.len(), 2); - assert!( - indices - .iter() - .all(|idx| idx.dataset_version == 2 || idx.dataset_version == 4) - ); - } - - #[test] - fn test_retain_mixed_empty_nonempty_vector_keeps_nonempty() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_test_index("vec_idx", 1, 1, Some(RoaringBitmap::new()), true), // Empty - create_test_index("vec_idx", 1, 2, Some(RoaringBitmap::from_iter([1])), true), // Non-empty - create_test_index("vec_idx", 1, 3, Some(RoaringBitmap::new()), true), // Empty - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Should keep only non-empty index - assert_eq!(indices.len(), 1); - assert_eq!(indices[0].dataset_version, 2); - } - - #[test] - fn test_retain_fragment_bitmap_with_nonexistent_fragments() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1), Fragment::new(2)]; // Only fragments 1 and 2 exist - - let mut indices = vec![create_test_index( - "idx", - 1, - 1, - Some(RoaringBitmap::from_iter([1, 2, 3, 4])), // References non-existent fragments 3, 4 - false, - )]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Should still keep the index (effective bitmap will be intersection with existing) - assert_eq!(indices.len(), 1); - // Original bitmap should be unchanged - assert_eq!( - indices[0].fragment_bitmap.as_ref().unwrap(), - &RoaringBitmap::from_iter([1, 2, 3, 4]) - ); - } - - #[test] - fn test_retain_effective_empty_bitmap_single_index() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(5), Fragment::new(6)]; - - // Bitmap references fragments that don't exist, so effective bitmap is empty - let mut scalar_indices = vec![create_test_index( - "scalar_idx", - 1, - 1, - Some(RoaringBitmap::from_iter([1, 2, 3])), - false, - )]; - - let mut vector_indices = vec![create_test_index( - "vector_idx", - 1, - 1, - Some(RoaringBitmap::from_iter([1, 2, 3])), - true, - )]; - - Transaction::retain_relevant_indices(&mut scalar_indices, &schema, &fragments); - Transaction::retain_relevant_indices(&mut vector_indices, &schema, &fragments); - - // Scalar should be kept (single index, even if effective bitmap is empty) - // Vector should be removed (empty effective bitmap) - assert_eq!(scalar_indices.len(), 1); - assert_eq!(vector_indices.len(), 0); - } - - #[test] - fn test_retain_different_index_names() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_test_index("idx_a", 1, 1, Some(RoaringBitmap::new()), false), - create_test_index("idx_b", 1, 1, Some(RoaringBitmap::new()), true), - create_test_index("idx_c", 1, 1, Some(RoaringBitmap::from_iter([1])), false), - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // idx_a (empty scalar) should be kept, idx_b (empty vector) removed, idx_c (non-empty) kept - assert_eq!(indices.len(), 2); - assert!(indices.iter().any(|idx| idx.name == "idx_a")); - assert!(indices.iter().any(|idx| idx.name == "idx_c")); - assert!(!indices.iter().any(|idx| idx.name == "idx_b")); - } - - #[test] - fn test_retain_empty_indices_vec() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices: Vec = vec![]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - assert_eq!(indices.len(), 0); - } - - #[test] - fn test_retain_all_indices_removed() { - let schema = create_test_schema(&[1]); - let fragments = vec![Fragment::new(1)]; - - let mut indices = vec![ - create_test_index("vec1", 1, 1, Some(RoaringBitmap::new()), true), - create_test_index("vec2", 1, 1, Some(RoaringBitmap::new()), true), - create_test_index("idx3", 99, 1, Some(RoaringBitmap::from_iter([1])), false), // Bad field - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - assert_eq!(indices.len(), 0); - } - - #[test] - fn test_retain_complex_scenario() { - let schema = create_test_schema(&[1, 2]); - let fragments = vec![Fragment::new(1), Fragment::new(2)]; - - let mut indices = vec![ - // System index - should always be kept - create_system_index(FRAG_REUSE_INDEX_NAME, 1), - // Group "idx_a" - all empty scalars, keep oldest - create_test_index("idx_a", 1, 3, Some(RoaringBitmap::new()), false), - create_test_index("idx_a", 1, 1, Some(RoaringBitmap::new()), false), // Oldest - create_test_index("idx_a", 1, 2, Some(RoaringBitmap::new()), false), - // Group "vec_b" - all empty vectors, remove all - create_test_index("vec_b", 1, 1, Some(RoaringBitmap::new()), true), - create_test_index("vec_b", 1, 2, Some(RoaringBitmap::new()), true), - // Group "idx_c" - mixed empty/non-empty, keep non-empty - create_test_index("idx_c", 2, 1, Some(RoaringBitmap::new()), false), - create_test_index("idx_c", 2, 2, Some(RoaringBitmap::from_iter([1])), false), // Keep - create_test_index("idx_c", 2, 3, Some(RoaringBitmap::from_iter([2])), false), // Keep - // Single non-empty - keep - create_test_index("idx_d", 1, 1, Some(RoaringBitmap::from_iter([1, 2])), false), - // Index with bad field - remove - create_test_index("idx_e", 99, 1, Some(RoaringBitmap::from_iter([1])), false), - ]; - - Transaction::retain_relevant_indices(&mut indices, &schema, &fragments); - - // Expected: frag_reuse, idx_a (oldest), idx_c (2 non-empty), idx_d = 5 total - assert_eq!(indices.len(), 5); - - // Verify system index kept - assert!(indices.iter().any(|idx| idx.name == FRAG_REUSE_INDEX_NAME)); - - // Verify idx_a kept oldest only - let idx_a_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "idx_a").collect(); - assert_eq!(idx_a_indices.len(), 1); - assert_eq!(idx_a_indices[0].dataset_version, 1); - - // Verify vec_b all removed - assert!(!indices.iter().any(|idx| idx.name == "vec_b")); - - // Verify idx_c kept non-empty only - let idx_c_indices: Vec<_> = indices.iter().filter(|idx| idx.name == "idx_c").collect(); - assert_eq!(idx_c_indices.len(), 2); - assert!( - idx_c_indices - .iter() - .all(|idx| idx.dataset_version == 2 || idx.dataset_version == 3) - ); - - // Verify idx_d kept - assert!(indices.iter().any(|idx| idx.name == "idx_d")); - - // Verify idx_e removed (bad field) - assert!(!indices.iter().any(|idx| idx.name == "idx_e")); - } - - #[test] - fn test_handle_rewrite_indices_skips_missing_index() { - use uuid::Uuid; - - // Create an empty indices list - let mut indices = vec![]; - - // Create rewritten_indices referring to a non-existent index - let rewritten_indices = vec![RewrittenIndex { - old_id: Uuid::new_v4(), - new_id: Uuid::new_v4(), - new_index_details: prost_types::Any { - type_url: String::new(), - value: vec![], - }, - new_index_version: 1, - new_index_files: None, - }]; - - // Should succeed (skip missing index) instead of error - let result = Transaction::handle_rewrite_indices(&mut indices, &rewritten_indices, &[]); - assert!(result.is_ok()); - assert!(indices.is_empty()); - } - - /// When a fragment has no existing last_updated_at_version_meta (None), a - /// partial RewriteColumns refresh must leave it as None rather than fabricating - /// prev_version for unmatched rows. - #[test] - fn test_partial_rewrite_skips_fragment_with_no_version_meta() { - let row_ids = RowIdSequence::from([10u64, 11, 12, 13, 14].as_slice()); - let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids))); - - let (major, minor) = lance_file::version::LanceFileVersion::Stable.to_numbers(); - let data_file = DataFile::new("data.lance", vec![0], vec![0], major, minor, None, None); - - let fragment = Fragment { - id: 1, - files: vec![data_file], - deletion_file: None, - row_id_meta, - physical_rows: Some(5), - last_updated_at_version_meta: None, - created_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![fragment.clone()]); - - // Simulate a RewriteColumns update that matched offsets 1 and 3 - let off_map = HashMap::from([(1u64, RoaringBitmap::from_iter([1u32, 3]))]); - let tx = Transaction::new( - manifest.version, - Operation::Update { - removed_fragment_ids: vec![], - updated_fragments: vec![fragment], - new_fragments: vec![], - fields_modified: vec![], - merged_generations: vec![], - fields_for_preserving_frag_bitmap: vec![], - update_mode: Some(UpdateMode::RewriteColumns), - inserted_rows_filter: None, - updated_fragment_offsets: Some(UpdatedFragmentOffsets(off_map)), - }, - None, - ); - - let (out, _) = tx - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert!( - out.fragments[0].last_updated_at_version_meta.is_none(), - "fragment with no prior version metadata must not have fabricated prev_version stamped on unmatched rows" - ); - } - - /// Partial RewriteColumns refresh in `build_manifest`: only matched physical - /// rows get `last_updated_at_version` bumped; same-fragment unmatched rows and - /// untouched fragments keep both version sequences. - #[tokio::test] - async fn test_build_manifest_partial_last_updated_rewrite_columns_stable_row_ids() { - let dir = TempStrDir::default(); - let uri = dir.as_str(); - - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, false), - ArrowField::new("x", DataType::Int32, false), - ])); - let batch0 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..8)), - Arc::new(Int32Array::from(vec![0_i32; 8])), - ], - ) - .unwrap(); - let reader0 = RecordBatchIterator::new(vec![Ok(batch0)], schema.clone()); - let write_params = WriteParams { - enable_stable_row_ids: true, - data_storage_version: Some(LanceFileVersion::Stable), - ..Default::default() - }; - let mut dataset = Dataset::write(reader0, uri, Some(write_params)) - .await - .unwrap(); - - let batch1 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(100..108)), - Arc::new(Int32Array::from(vec![0_i32; 8])), - ], - ) - .unwrap(); - let reader1 = RecordBatchIterator::new(vec![Ok(batch1)], schema.clone()); - dataset.append(reader1, None).await.unwrap(); - - let frags = dataset.get_fragments(); - assert_eq!( - frags.len(), - 2, - "expected two fragments (append creates a new fragment)" - ); - - async fn scan_row_versions(ds: &Dataset) -> HashMap<(u32, u32), (u64, u64)> { - let mut scanner = ds.scan(); - scanner - .project(&[ - ROW_ADDR, - ROW_LAST_UPDATED_AT_VERSION, - ROW_CREATED_AT_VERSION, - ]) - .unwrap(); - let batches = scanner - .try_into_stream() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - let mut out = HashMap::new(); - for batch in batches { - let addrs = batch - .column_by_name(ROW_ADDR) - .unwrap() - .as_primitive::(); - let last = batch - .column_by_name(ROW_LAST_UPDATED_AT_VERSION) - .unwrap() - .as_primitive::(); - let created = batch - .column_by_name(ROW_CREATED_AT_VERSION) - .unwrap() - .as_primitive::(); - for row in 0..batch.num_rows() { - let addr = RowAddress::from(addrs.value(row)); - out.insert( - (addr.fragment_id(), addr.row_offset()), - (last.value(row), created.value(row)), - ); - } - } - out - } - - let before = scan_row_versions(&dataset).await; - assert_eq!(before.len(), 16); - - // Update only rows i in {2, 4, 6} within fragment 0 (physical offsets 2, 4, 6). - let update_schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("i", DataType::Int32, false), - ArrowField::new("x", DataType::Int32, false), - ])); - let update_batch = RecordBatch::try_new( - update_schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![2, 4, 6])), - Arc::new(Int32Array::from(vec![99, 99, 99])), - ], - ) - .unwrap(); - let right: Box = Box::new( - RecordBatchIterator::new(vec![Ok(update_batch)].into_iter(), update_schema), - ); - - let mut frag0 = dataset.get_fragment(0).unwrap(); - let u = frag0 - .update_columns_with_offsets(right, "i", "i") - .await - .unwrap(); - assert_eq!(u.matched_offsets.iter().count(), 3); - for off in [2_u32, 4, 6] { - assert!(u.matched_offsets.contains(off)); - } - - let updated_fragment_offsets = Some(UpdatedFragmentOffsets(HashMap::from([( - u.fragment.id, - u.matched_offsets, - )]))); - - let op = Operation::Update { - removed_fragment_ids: vec![], - updated_fragments: vec![u.fragment], - new_fragments: vec![], - fields_modified: u.fields_modified, - merged_generations: Vec::new(), - fields_for_preserving_frag_bitmap: vec![], - update_mode: Some(UpdateMode::RewriteColumns), - inserted_rows_filter: None, - updated_fragment_offsets, - }; - - let read_v = dataset.version().version; - let dataset = Dataset::commit( - uri, - op, - Some(read_v), - None, - None, - Arc::new(Session::default()), - true, - ) - .await - .unwrap(); - - let new_v = dataset.version().version; - assert_eq!(new_v, read_v + 1); - - let after = scan_row_versions(&dataset).await; - for off in 0..8_u32 { - let key = (0, off); - let (last_before, created_before) = before[&key]; - let (last_after, created_after) = after[&key]; - assert_eq!(created_after, created_before); - if off == 2 || off == 4 || off == 6 { - assert_eq!( - last_after, new_v, - "matched row offset {off} should advance last_updated to new version" - ); - } else { - assert_eq!( - last_after, last_before, - "unmatched row offset {off} in fragment 0 should keep last_updated" - ); - } - } - - for off in 0..8_u32 { - let key = (1, off); - assert_eq!( - after[&key], before[&key], - "fragment 1 row offset {off}: both version columns unchanged" - ); - } - } - - /// Regression test for https://github.com/lance-format/lance/issues/6417 - /// - /// When overwriting a LEGACY dataset with STABLE-format fragments, the - /// validation should not use the old manifest's format. STABLE fragments - /// omit struct parent fields, which the strict legacy check rejects. - #[test] - fn test_overwrite_legacy_to_stable_with_struct_fields() { - use arrow_schema::Fields; - - // Schema: id (field 0), name (field 1), address (field 2, struct parent), - // city (field 3), country (field 4) - let arrow_schema = ArrowSchema::new(vec![ - ArrowField::new("id", DataType::Int32, false), - ArrowField::new("name", DataType::Utf8, false), - ArrowField::new( - "address", - DataType::Struct(Fields::from(vec![ - ArrowField::new("city", DataType::Utf8, false), - ArrowField::new("country", DataType::Utf8, false), - ])), - false, - ), - ]); - let schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // Old manifest is LEGACY format - let legacy_manifest = Manifest::new( - schema.clone(), - Arc::new(vec![Fragment::new(0)]), - DataStorageFormat::new(LanceFileVersion::Legacy), - HashMap::new(), - ); - - // New fragments in STABLE format omit struct parent field (id=2), - // only including leaf fields: id=0, name=1, city=3, country=4 - let stable_fragment = Fragment { - id: 0, - files: vec![DataFile::new( - "data.lance", - vec![0, 1, 3, 4], // no field 2 (struct parent) - vec![0, 1, 2, 3], - lance_file::format::MAJOR_VERSION as u32, - lance_file::format::MINOR_VERSION as u32, - None, - None, - )], - physical_rows: Some(10), - deletion_file: None, - row_id_meta: None, - last_updated_at_version_meta: None, - created_at_version_meta: None, - }; - - let operation = Operation::Overwrite { - fragments: vec![stable_fragment], - schema, - config_upsert_values: None, - initial_bases: None, - }; - - // This should succeed — the old manifest's LEGACY format should not - // cause strict validation of the new STABLE fragments. - validate_operation(Some(&legacy_manifest), &operation).unwrap(); - } - - /// Existing fragments use id >= 1 to avoid collision with `Fragment::new(0)` - /// used by `sample_manifest`. New (updated) fragments use id = 10. - fn make_stable_row_id_manifest(fragments: Vec) -> Manifest { - let schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); - let mut manifest = Manifest::new( - LanceSchema::try_from(&schema).unwrap(), - Arc::new(fragments), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ); - manifest.reader_feature_flags = FLAG_STABLE_ROW_IDS; - manifest.next_row_id = 1000; - manifest.version = 4; - manifest - } - - fn update_txn(new_fragments: Vec) -> Transaction { - Transaction::new( - 4, - Operation::Update { - removed_fragment_ids: vec![], - updated_fragments: vec![], - new_fragments, - fields_modified: vec![], - merged_generations: vec![], - fields_for_preserving_frag_bitmap: vec![], - update_mode: None, - inserted_rows_filter: None, - updated_fragment_offsets: None, - }, - None, - ) - } - - fn created_at_versions(manifest: &Manifest, frag_id: u64) -> Vec { - let frag = manifest.fragments.iter().find(|f| f.id == frag_id).unwrap(); - let seq = frag - .created_at_version_meta - .as_ref() - .unwrap() - .load_sequence() - .unwrap(); - seq.versions().collect() - } - - fn last_updated_at_versions(manifest: &Manifest, frag_id: u64) -> Vec { - let frag = manifest.fragments.iter().find(|f| f.id == frag_id).unwrap(); - let seq = frag - .last_updated_at_version_meta - .as_ref() - .unwrap() - .load_sequence() - .unwrap(); - seq.versions().collect() - } - - #[test] - fn merge_build_manifest_refreshes_last_updated_when_data_files_change_stable_row_ids() { - use lance_file::version::LanceFileVersion; - use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let mk_file = |path: &str| DataFile::new(path, vec![0], vec![0], major, minor, None, None); - - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - let row_ids = RowIdSequence::from([100u64, 101, 102, 103, 104].as_slice()); - let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids))); - - let prev_fragment = Fragment { - id: 0, - files: vec![mk_file("before.lance")], - deletion_file: None, - row_id_meta, - physical_rows: Some(5), - last_updated_at_version_meta: None, - created_at_version_meta: None, - }; - - let mut manifest = Manifest::new( - lance_schema.clone(), - Arc::new(vec![prev_fragment.clone()]), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ); - manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; - manifest.next_row_id = 100; - - let merged_fragment = Fragment { - files: vec![mk_file("after.lance")], - ..prev_fragment - }; - - let tx = Transaction::new( - manifest.version, - Operation::Merge { - fragments: vec![merged_fragment], - schema: lance_schema, - }, - None, - ); - - let (out, _) = tx - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert_eq!(out.version, 2); - let frag = &out.fragments[0]; - let seq = frag - .last_updated_at_version_meta - .as_ref() - .unwrap() - .load_sequence() - .unwrap(); - assert_eq!(seq.version_at(0).unwrap(), 2); - assert_eq!(seq.version_at(4).unwrap(), 2); - } - - #[test] - fn merge_build_manifest_skips_refresh_when_carry_forward_stable_row_ids() { - use lance_file::version::LanceFileVersion; - use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - use lance_table::rowids::version::{RowDatasetVersionMeta, RowDatasetVersionSequence}; - - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let data_file = DataFile::new("same.lance", vec![0], vec![0], major, minor, None, None); - - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - let row_ids = RowIdSequence::from([200u64, 201, 202, 203, 204].as_slice()); - let row_id_meta = Some(RowIdMeta::Inline(write_row_ids(&row_ids))); - - let uniform_v1 = RowDatasetVersionSequence::from_uniform_row_count(5, 1); - let meta_v1 = RowDatasetVersionMeta::from_sequence(&uniform_v1).unwrap(); - - let prev_fragment = Fragment { - id: 0, - files: vec![data_file.clone()], - deletion_file: None, - row_id_meta: row_id_meta.clone(), - physical_rows: Some(5), - last_updated_at_version_meta: Some(meta_v1.clone()), - created_at_version_meta: None, - }; - - let mut manifest = Manifest::new( - lance_schema.clone(), - Arc::new(vec![prev_fragment]), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ); - manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; - manifest.next_row_id = 100; - - let merged_fragment = Fragment { - id: 0, - files: vec![data_file], - deletion_file: None, - row_id_meta, - physical_rows: Some(5), - last_updated_at_version_meta: Some(meta_v1), - created_at_version_meta: None, - }; - - let tx = Transaction::new( - manifest.version, - Operation::Merge { - fragments: vec![merged_fragment], - schema: lance_schema, - }, - None, - ); - - let (out, _) = tx - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - let seq = out.fragments[0] - .last_updated_at_version_meta - .as_ref() - .unwrap() - .load_sequence() - .unwrap(); - assert_eq!(seq.version_at(0).unwrap(), 1); - assert_eq!(seq.version_at(4).unwrap(), 1); - } - - #[test] - fn merge_build_manifest_no_last_updated_refresh_without_stable_row_ids() { - use lance_file::version::LanceFileVersion; - use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let mk_file = |path: &str| DataFile::new(path, vec![0], vec![0], major, minor, None, None); - - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - let prev_fragment = Fragment { - id: 0, - files: vec![mk_file("before.lance")], - deletion_file: None, - row_id_meta: None, - physical_rows: Some(5), - last_updated_at_version_meta: None, - created_at_version_meta: None, - }; - - let manifest = Manifest::new( - lance_schema.clone(), - Arc::new(vec![prev_fragment.clone()]), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ); - assert_eq!( - manifest.reader_feature_flags & FLAG_STABLE_ROW_IDS, - 0, - "manifest must not use stable row IDs for this guard test" - ); - - let merged_fragment = Fragment { - files: vec![mk_file("after.lance")], - ..prev_fragment - }; - - let tx = Transaction::new( - manifest.version, - Operation::Merge { - fragments: vec![merged_fragment], - schema: lance_schema, - }, - None, - ); - - let (out, _) = tx - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert!( - out.fragments[0].last_updated_at_version_meta.is_none(), - "without stable row IDs, Merge must not populate per-row last_updated metadata" - ); - } - - #[test] - fn merge_build_manifest_sets_both_version_meta_for_new_fragment_id_stable_row_ids() { - use lance_file::version::LanceFileVersion; - use lance_table::feature_flags::FLAG_STABLE_ROW_IDS; - - let (major, minor) = LanceFileVersion::Stable.to_numbers(); - let mk_file = |path: &str| DataFile::new(path, vec![0], vec![0], major, minor, None, None); - - let arrow_schema = ArrowSchema::new(vec![ArrowField::new("id", DataType::Int32, false)]); - let lance_schema = LanceSchema::try_from(&arrow_schema).unwrap(); - - // Existing fragment (id=0) with stable row IDs - let row_ids_0 = RowIdSequence::from([10u64, 11, 12].as_slice()); - let existing_fragment = Fragment { - id: 0, - files: vec![mk_file("existing.lance")], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_0))), - physical_rows: Some(3), - last_updated_at_version_meta: None, - created_at_version_meta: None, - }; - - let mut manifest = Manifest::new( - lance_schema.clone(), - Arc::new(vec![existing_fragment.clone()]), - DataStorageFormat::new(LanceFileVersion::V2_0), - HashMap::new(), - ); - manifest.reader_feature_flags |= FLAG_STABLE_ROW_IDS; - manifest.next_row_id = 100; - manifest.version = 1; - - // New fragment (id=1) not present in prev manifest — exercises the None branch - let row_ids_1 = RowIdSequence::from([20u64, 21, 22, 23].as_slice()); - let new_fragment = Fragment { - id: 1, - files: vec![mk_file("new.lance")], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&row_ids_1))), - physical_rows: Some(4), - last_updated_at_version_meta: None, - created_at_version_meta: None, - }; - - let tx = Transaction::new( - manifest.version, - Operation::Merge { - fragments: vec![existing_fragment, new_fragment], - schema: lance_schema, - }, - None, - ); - - let (out, _) = tx - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert_eq!(out.version, 2); - - let new_frag = out.fragments.iter().find(|f| f.id == 1).unwrap(); - - // last_updated_at_version must be set to the commit version - let last_updated_seq = new_frag - .last_updated_at_version_meta - .as_ref() - .expect("new fragment must have last_updated_at_version_meta") - .load_sequence() - .unwrap(); - assert_eq!(last_updated_seq.version_at(0).unwrap(), 2); - assert_eq!(last_updated_seq.version_at(3).unwrap(), 2); - - // created_at_version must also be set — must not be None - let created_seq = new_frag - .created_at_version_meta - .as_ref() - .expect("new fragment must have created_at_version_meta") - .load_sequence() - .unwrap(); - assert_eq!(created_seq.version_at(0).unwrap(), 2); - assert_eq!(created_seq.version_at(3).unwrap(), 2); - } - - #[test] - fn test_update_version_tracking_preserves_created_at() { - let existing_seq = RowIdSequence::from([100u64, 101, 102].as_slice()); - let created_at_seq = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 5, - }], - }; - let existing_fragment = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), - physical_rows: Some(3), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&created_at_seq).unwrap(), - ), - last_updated_at_version_meta: None, - }; - - let new_seq = RowIdSequence::from([100u64, 102].as_slice()); - let new_fragment = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![existing_fragment]); - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - assert_eq!(created_at_versions(&result, 10), vec![5, 5]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); - } - - #[test] - fn test_update_version_tracking_mixed_origins() { - let frag_a_seq = RowIdSequence::from([10u64, 11].as_slice()); - let frag_a_created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 2, - }], - }; - let frag_b_seq = RowIdSequence::from([20u64, 21, 22].as_slice()); - let frag_b_created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 3, - }], - }; - - let manifest = make_stable_row_id_manifest(vec![ - Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_a_seq))), - physical_rows: Some(2), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&frag_a_created).unwrap(), - ), - last_updated_at_version_meta: None, - }, - Fragment { - id: 2, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&frag_b_seq))), - physical_rows: Some(3), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&frag_b_created).unwrap(), - ), - last_updated_at_version_meta: None, - }, - ]); - - // New fragment has rows from both original fragments: row 11 from frag_a, row 20 from frag_b - let new_seq = RowIdSequence::from([11u64, 20].as_slice()); - let new_fragment = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Row 11 came from frag_a (offset 1, version 2), row 20 came from frag_b (offset 0, version 3) - assert_eq!(created_at_versions(&result, 10), vec![2, 3]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); - } - - #[test] - fn test_update_version_tracking_insert_branch_gets_new_version() { - // Simulates the INSERT branch (NOT MATCHED) of a MERGE INTO commit: - // the new fragment contains a mix of rewritten rows (UPDATE branch, row ID - // present in existing fragments) and freshly inserted rows (INSERT branch, - // row ID not present in any existing fragment). - // - // UPDATE branch row (10): created_at must be copied from the source fragment. - // INSERT branch row (999): created_at must equal new_version (the merge commit - // version), because the row first appeared in this commit. - let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); - let existing_created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 5, - }], - }; - let existing_fragment = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), - physical_rows: Some(2), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&existing_created).unwrap(), - ), - last_updated_at_version_meta: None, - }; - - // New fragment has row 10 (UPDATE branch) and row 999 (INSERT branch) - let new_seq = RowIdSequence::from([10u64, 999].as_slice()); - let new_fragment = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - // update_txn uses read_version 4 → new_version is 5 - let manifest = make_stable_row_id_manifest(vec![existing_fragment]); - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Row 10 (UPDATE branch): created_at copied from source (version 5). - // Row 999 (INSERT branch): created_at == new_version (5). - assert_eq!(created_at_versions(&result, 10), vec![5, 5]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); - } - - #[test] - fn test_update_version_tracking_merge_into_distinguishes_insert_and_update_branch() { - // Verifies the MERGE INTO correctness contract when UPDATE branch rows and INSERT - // branch rows have *different* source created_at values, so we can distinguish - // which row got which value. - // - // Existing fragment (id=1): row IDs [10, 11], created_at = version 3. - // New fragment (id=20): row IDs [10, 500, 11, 501]. - // - Rows 10 and 11: UPDATE branch (present in existing fragment) → created_at = 3. - // - Rows 500 and 501: INSERT branch (no source) → created_at = new_version = 5. - let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); - let existing_created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 3, - }], - }; - let existing_fragment = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), - physical_rows: Some(2), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&existing_created).unwrap(), - ), - last_updated_at_version_meta: None, - }; - - let new_seq = RowIdSequence::from([10u64, 500, 11, 501].as_slice()); - let new_fragment = Fragment { - id: 20, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(4), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - // update_txn uses read_version 4 → new_version is 5 - let manifest = make_stable_row_id_manifest(vec![existing_fragment]); - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // UPDATE branch rows (10, 11): created_at preserved from source (version 3). - // INSERT branch rows (500, 501): created_at == new_version (5). - assert_eq!(created_at_versions(&result, 20), vec![3, 5, 3, 5]); - // All rows in the new fragment get last_updated == new_version. - assert_eq!(last_updated_at_versions(&result, 20), vec![5, 5, 5, 5]); - } - - #[test] - fn test_update_version_tracking_source_fragment_no_created_at_defaults_to_1() { - // Source fragment has row_id_meta but no created_at_version_meta. - // The row IS found in the lookup, but the version defaults to 1. - let existing_seq = RowIdSequence::from([50u64, 51].as_slice()); - let existing_fragment = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let new_seq = RowIdSequence::from([50u64].as_slice()); - let new_fragment = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(1), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![existing_fragment]); - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Row 50 is found in source but source has no created_at_version_meta → default 1 - assert_eq!(created_at_versions(&result, 10), vec![1]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5]); - } - - #[test] - fn test_update_version_tracking_no_row_id_meta_fallback() { - let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); - let existing_fragment = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let new_fragment = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: None, - physical_rows: Some(3), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![existing_fragment]); - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Fragment starts with no row_id_meta → assign_row_ids gives it fresh IDs → - // those IDs have no source in existing fragments (INSERT branch) → - // created_at == new_version (5) for each row. - assert_eq!(created_at_versions(&result, 10), vec![5, 5, 5]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5, 5]); - } - - #[test] - fn test_update_version_tracking_corrupt_created_at_defaults_to_1() { - let existing_seq = RowIdSequence::from([10u64, 11].as_slice()); - let existing_fragment = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&existing_seq))), - physical_rows: Some(2), - created_at_version_meta: Some(RowDatasetVersionMeta::Inline(Arc::from( - vec![0xFFu8; 8].as_slice(), - ))), - last_updated_at_version_meta: None, - }; - - let new_seq = RowIdSequence::from([10u64].as_slice()); - let new_fragment = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(1), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![existing_fragment]); - let (result, _) = update_txn(vec![new_fragment]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Corrupt metadata causes decode to fail → falls back to UNKNOWN_CREATED_AT_VERSION (1) - assert_eq!(created_at_versions(&result, 10), vec![1]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5]); - } - - // --- Proposal 1: range pre-filter --- - - /// Fragments whose row-ID range lies entirely outside the needed set must not - /// affect the result. Here fragment 1 has IDs [1000, 1001] which are far above - /// the needed range [10, 11]; it is skipped by the range pre-filter and its - /// created_at version (version 99) must never appear in the output. - #[test] - fn test_update_version_tracking_range_filter_skips_non_overlapping_fragment() { - // Fragment in range – IDs [10, 11], created_at = 5 - let in_range_seq = RowIdSequence::from([10u64, 11].as_slice()); - let in_range_created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 5, - }], - }; - let in_range_frag = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&in_range_seq))), - physical_rows: Some(2), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&in_range_created).unwrap(), - ), - last_updated_at_version_meta: None, - }; - - // Fragment outside range – IDs [1000, 1001], created_at = 99 (must never appear) - let out_of_range_seq = RowIdSequence::from([1000u64, 1001].as_slice()); - let out_of_range_created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..2), - version: 99, - }], - }; - let out_of_range_frag = Fragment { - id: 2, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&out_of_range_seq))), - physical_rows: Some(2), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&out_of_range_created).unwrap(), - ), - last_updated_at_version_meta: None, - }; - - // New fragment rewrites both rows from the in-range fragment - let new_seq = RowIdSequence::from([10u64, 11].as_slice()); - let new_frag = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![in_range_frag, out_of_range_frag]); - let (result, _) = update_txn(vec![new_frag]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Both rows originate from the in-range fragment (version 5). - // The out-of-range fragment's version 99 must not appear. - assert_eq!(created_at_versions(&result, 10), vec![5, 5]); - assert_eq!(last_updated_at_versions(&result, 10), vec![5, 5]); - } - - /// When the needed row IDs fall exactly at the boundary of a fragment's range, - /// the range pre-filter must NOT skip the fragment (boundary values are inclusive). - #[test] - fn test_update_version_tracking_range_filter_boundary_inclusive() { - // Fragment IDs [10, 11, 12], created_at = 7 - let seq = RowIdSequence::from([10u64, 11, 12].as_slice()); - let created = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 7, - }], - }; - let existing = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq))), - physical_rows: Some(3), - created_at_version_meta: Some(RowDatasetVersionMeta::from_sequence(&created).unwrap()), - last_updated_at_version_meta: None, - }; - - // New fragment takes the boundary IDs: 10 (min) and 12 (max) - let new_seq = RowIdSequence::from([10u64, 12].as_slice()); - let new_frag = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![existing]); - let (result, _) = update_txn(vec![new_frag]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Boundary IDs must be found and resolved correctly - assert_eq!(created_at_versions(&result, 10), vec![7, 7]); - } - - // --- Proposal 2: version sequence cache --- - - /// When multiple updated rows all originate from the same source fragment, - /// the created_at version sequence for that fragment must be decoded exactly - /// once (not once per row). The observable correctness requirement is that - /// all rows get the right version regardless of how many there are. - #[test] - fn test_update_version_tracking_many_rows_same_source_fragment() { - // Source fragment: 100 rows with IDs 0..100, mixed versions (2 runs). - // First 50 rows at version 3, next 50 rows at version 4. - let src_ids: Vec = (0u64..100).collect(); - let src_seq = RowIdSequence::from(src_ids.as_slice()); - let src_created = RowDatasetVersionSequence { - runs: vec![ - RowDatasetVersionRun { - span: U64Segment::Range(0..50), - version: 3, - }, - RowDatasetVersionRun { - span: U64Segment::Range(0..50), - version: 4, - }, - ], - }; - let src_frag = Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&src_seq))), - physical_rows: Some(100), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&src_created).unwrap(), - ), - last_updated_at_version_meta: None, - }; - - // New fragment rewrites all 100 rows preserving their stable IDs. - let new_seq = RowIdSequence::from(src_ids.as_slice()); - let new_frag = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(100), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let manifest = make_stable_row_id_manifest(vec![src_frag]); - let (result, _) = update_txn(vec![new_frag]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - let versions = created_at_versions(&result, 10); - assert_eq!(versions.len(), 100); - // First 50 rows came from version 3, next 50 from version 4 - assert!(versions[..50].iter().all(|&v| v == 3)); - assert!(versions[50..].iter().all(|&v| v == 4)); - } - - /// Rows originating from multiple distinct source fragments must each get - /// the version from their own source, even when all cached together. - #[test] - fn test_update_version_tracking_cache_multiple_source_fragments() { - let seq_a = RowIdSequence::from([10u64, 11, 12].as_slice()); - let created_a = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 2, - }], - }; - let seq_b = RowIdSequence::from([20u64, 21, 22].as_slice()); - let created_b = RowDatasetVersionSequence { - runs: vec![RowDatasetVersionRun { - span: U64Segment::Range(0..3), - version: 8, - }], - }; - - let manifest = make_stable_row_id_manifest(vec![ - Fragment { - id: 1, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_a))), - physical_rows: Some(3), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&created_a).unwrap(), - ), - last_updated_at_version_meta: None, - }, - Fragment { - id: 2, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&seq_b))), - physical_rows: Some(3), - created_at_version_meta: Some( - RowDatasetVersionMeta::from_sequence(&created_b).unwrap(), - ), - last_updated_at_version_meta: None, - }, - ]); - - // New fragment takes rows from both sources: 12 (frag A, offset 2) and 20 (frag B, offset 0) - let new_seq = RowIdSequence::from([12u64, 20].as_slice()); - let new_frag = Fragment { - id: 10, - files: vec![], - deletion_file: None, - row_id_meta: Some(RowIdMeta::Inline(write_row_ids(&new_seq))), - physical_rows: Some(2), - created_at_version_meta: None, - last_updated_at_version_meta: None, - }; - - let (result, _) = update_txn(vec![new_frag]) - .build_manifest( - Some(&manifest), - vec![], - "txn", - &ManifestWriteConfig::default(), - ) - .unwrap(); - - // Row 12 → frag A offset 2 → version 2; row 20 → frag B offset 0 → version 8 - assert_eq!(created_at_versions(&result, 10), vec![2, 8]); - } - - #[test] - fn test_encode_version_runs_empty() { - let runs = encode_version_runs(&[]); - assert!(runs.is_empty()); - } - - #[test] - fn test_encode_version_runs_single_run() { - let runs = encode_version_runs(&[3, 3, 3]); - assert_eq!(runs.len(), 1); - assert_eq!(runs[0].version, 3); - } - - #[test] - fn test_encode_version_runs_alternating() { - let runs = encode_version_runs(&[1, 2, 1, 2]); - assert_eq!(runs.len(), 4); - assert_eq!(runs[0].version, 1); - assert_eq!(runs[1].version, 2); - assert_eq!(runs[2].version, 1); - assert_eq!(runs[3].version, 2); - } - - fn table_metadata_update(entries: Vec<(&str, Option<&str>)>, replace: bool) -> Operation { - Operation::UpdateConfig { - config_updates: None, - table_metadata_updates: Some(UpdateMap { - update_entries: entries.into_iter().map(UpdateMapEntry::from).collect(), - replace, - }), - schema_metadata_updates: None, - field_metadata_updates: HashMap::new(), - } - } - - #[test] - fn test_table_metadata_conflicts_on_same_key() { - let left = table_metadata_update(vec![("key", Some("1"))], false); - let same_key = table_metadata_update(vec![("key", Some("2"))], false); - let different_key = table_metadata_update(vec![("other", Some("2"))], false); - let replace = table_metadata_update(vec![("other", Some("2"))], true); - - assert!(left.modifies_same_metadata(&same_key)); - assert!(!left.modifies_same_metadata(&different_key)); - assert!(left.modifies_same_metadata(&replace)); - } -} diff --git a/rust/lance/src/dataset/udtf.rs b/rust/lance/src/dataset/udtf.rs index 75c0388bc24..2144b108859 100644 --- a/rust/lance/src/dataset/udtf.rs +++ b/rust/lance/src/dataset/udtf.rs @@ -13,7 +13,6 @@ use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD}; use lance_index::scalar::FullTextSearchQuery; use lance_index::scalar::inverted::parser::from_json; use serde_json::Value; -use std::any::Any; use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; @@ -61,10 +60,6 @@ impl FtsTableProvider { #[async_trait] impl TableProvider for FtsTableProvider { - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.full_schema.clone() } diff --git a/rust/lance/src/dataset/updater.rs b/rust/lance/src/dataset/updater.rs index 752a479e5b0..a504a07b9e7 100644 --- a/rust/lance/src/dataset/updater.rs +++ b/rust/lance/src/dataset/updater.rs @@ -12,7 +12,8 @@ use lance_table::utils::stream::ReadBatchFutStream; use super::Dataset; use super::fragment::FragmentReader; use super::scanner::get_default_batch_size; -use super::write::{GenericWriter, cleanup_data_fragments, open_update_writer}; +use super::versions; +use super::write::{GenericWriter, cleanup_data_fragments}; use crate::dataset::FileFragment; use crate::dataset::utils::SchemaAdapter; @@ -46,6 +47,8 @@ pub struct Updater { /// The adapter to convert the logical data to physical data. schema_adapter: Option, + allow_external_blob_outside_bases: bool, + finished: bool, deletion_restorer: DeletionRestorer, @@ -72,7 +75,13 @@ impl Updater { (None, None) }; - let legacy_batch_size = reader.legacy_num_rows_in_batch(0); + let storage_version = fragment + .dataset() + .manifest() + .data_storage_format + .lance_file_format(); + let legacy_batch_size = + versions::row_group_size_for_rewrite(storage_version, &fragment).await?; let batch_size = match (&legacy_batch_size, batch_size) { // If this is a v1 dataset we must use the row group size of the file @@ -95,6 +104,7 @@ impl Updater { // The schema adapter needs the data schema, not the logical schema, so it can't be // created until after the first batch is read. schema_adapter: None, + allow_external_blob_outside_bases: false, finished: false, deletion_restorer: DeletionRestorer::new(deletion_vector, legacy_batch_size), }) @@ -109,6 +119,10 @@ impl Updater { } /// Returns the next [`RecordBatch`] as input for updater. + /// + /// Every batch this hands out must be passed back to [`Self::update`] before the + /// next call: the deletion restorer advances there, so skipping it would leave + /// deleted rows unaccounted for and fail the stream at its end. pub async fn next(&mut self) -> Result> { if self.finished { return Ok(None); @@ -117,9 +131,26 @@ impl Updater { match batch { None => { if !self.deletion_restorer.is_exhausted() { - // This can happen only if there is a batch size (e.g. v1 file) and the - // last batch(es) are entirely deleted. - return Err(Error::not_supported_source("Missing too many rows in merge, run compaction to materialize deletions first".into())); + // The stream cannot supply rows the restorer still needs. In + // practice that means the deletion vector points at rows the + // stream never produced — an id past the fragment's physical row + // count, or fewer rows read than the fragment claims to have. + // + // Deferred blanks can also be outstanding here, but only if no + // batch after the deferral had a live row, i.e. the whole fragment + // is deleted; `write_deletions` drops such a fragment before it + // reaches an updater, so that path is defensive. A legacy file + // cannot defer at all — its fully deleted batch is refused + // earlier, by `add_blanks`. + // + // Don't name a count: the deletion-vector case owes no blanks yet, + // so a number here would read as zero rows owed. + return Err(Error::not_supported(format!( + "Fragment Updater: the input stream for fragment {} ended while \ + deleted rows were still unaccounted for, run compaction to \ + materialize deletions first", + self.fragment.id(), + ))); } self.finished = true; Ok(None) @@ -144,9 +175,21 @@ impl Updater { .dataset() .manifest() .data_storage_format - .lance_file_version()?; + .lance_file_format(); + + versions::open_update_writer( + data_storage_version, + self.dataset(), + &schema, + self.allow_external_blob_outside_bases, + ) + .await + } - open_update_writer(self.dataset(), &schema, data_storage_version).await + /// Allow trusted existing external blob references to pass through an update rewrite. + /// Callers must separately validate any newly supplied references before writing. + pub(super) fn allow_external_blob_outside_bases(&mut self) { + self.allow_external_blob_outside_bases = true; } /// Update one batch. @@ -229,12 +272,22 @@ impl Updater { } let mut fragment = Fragment::new(self.fragment.id() as u64); + let storage_version = self + .dataset() + .manifest() + .data_storage_format + .lance_file_format(); // cleanup_data_fragments only needs path/base_id to remove the unfinished // data file and any blob sidecars. Build a minimal synthetic fragment so // we can reuse the shared cleanup path without fabricating full metadata. - fragment - .files - .push(DataFile::new(path, vec![], vec![], 0, 0, None, base_id)); + fragment.files.push(DataFile::new( + path, + vec![], + vec![], + storage_version, + None, + base_id, + )); cleanup_data_fragments( &self.dataset().object_store, &self.dataset().base, @@ -264,6 +317,10 @@ impl Updater { /// /// To do this we scan through the deletion vector in sorted order, merging deleted rows /// in as appropriate. +/// +/// Any method returning an error leaves the restorer mid-batch: the deletion vector +/// has been walked past rows that never made it into an output batch. Drop it and +/// start over rather than calling it again. struct DeletionRestorer { current_row_id: u32, @@ -273,6 +330,12 @@ struct DeletionRestorer { deletion_vector_iter: Option + Send>>, last_deleted_row_id: Option, + + /// Blank rows owed to batches that had no live row to copy a placeholder from + /// + /// See [`Self::restore`] for why they are deferred instead of materialized. + /// Only ever non-zero for non-legacy files, which are the only ones that defer. + pending_blank_rows: u32, } impl DeletionRestorer { @@ -282,11 +345,12 @@ impl DeletionRestorer { legacy_batch_size, deletion_vector_iter: Some(deletion_vector.into_sorted_iter()), last_deleted_row_id: None, + pending_blank_rows: 0, } } fn is_exhausted(&self) -> bool { - self.deletion_vector_iter.is_none() + self.deletion_vector_iter.is_none() && self.pending_blank_rows == 0 } fn is_full(batch_size: Option, num_rows: u32) -> bool { @@ -329,11 +393,14 @@ impl DeletionRestorer { let deletion_vector_iter = self.deletion_vector_iter.as_mut().unwrap(); // Now we need to walk through our deletion vector and figure out where to insert blanks - let mut next_deleted_id = if self.last_deleted_row_id.is_some() { - self.last_deleted_row_id - } else { - deletion_vector_iter.next() - }; + // Take the stashed id rather than peeking at it: leaving a consumed id in the + // field relies on the early return above to never read it again. `or_else` has + // to stay lazy — `or` would pull from the iterator even when a stash is waiting, + // silently dropping a deleted row. + let mut next_deleted_id = self + .last_deleted_row_id + .take() + .or_else(|| deletion_vector_iter.next()); loop { if let Some(next_deleted_id) = next_deleted_id { if next_deleted_id > last_row_id @@ -353,17 +420,65 @@ impl DeletionRestorer { } else { // Deleted row ids iterator is exhausted self.deletion_vector_iter = None; + // `is_exhausted` reads these two together, so a stash left behind here + // would make it report exhaustion while a deleted row is still owed. + debug_assert!(self.last_deleted_row_id.is_none()); return deleted; } next_deleted_id = deletion_vector_iter.next(); } } + /// Restore the deleted rows for one batch of live rows. + /// + /// Blanks are materialized by copying the batch's first live row (see + /// [`add_blanks`]), so a batch with no live rows has nothing to copy from. That + /// happens when a deleted run starts at physical row 0: there is no preceding + /// batch for [`Self::deleted_batch_offsets_in_range`] to append the run to, so + /// the run arrives as an empty batch carrying every one of its offsets. + /// + /// Rather than invent placeholder values for an arbitrary schema, we remember + /// how many blanks we owe and prepend them to the next batch that does have a + /// live row. Deleted rows sort before the live rows that follow them, so the + /// physical row order is preserved either way. fn restore(&mut self, batch: RecordBatch) -> Result { + // Holds by construction today — deferring is the only thing that sets + // pending_blank_rows and it is gated on non-legacy — so this documents the + // invariant the legacy row-count check below depends on rather than guarding + // against a state we can reach. + debug_assert!(self.pending_blank_rows == 0 || self.legacy_batch_size.is_none()); + // Because of deleted rows, the number of row ids in the batch might not // match the length. let deleted_batch_offsets = self.deleted_batch_offsets_in_range(batch.num_rows() as u32); - let batch = add_blanks(batch, &deleted_batch_offsets)?; + + // Legacy files must reproduce the original row group size, which deferring + // would break, so they keep reporting the pre-existing error instead. + if batch.num_rows() == 0 && self.legacy_batch_size.is_none() { + let deferred = deleted_batch_offsets.len() as u32; + self.pending_blank_rows += deferred; + self.current_row_id += deferred; + return Ok(batch); + } + + let pending_blank_rows = self.pending_blank_rows; + let batch_offsets = if pending_blank_rows == 0 { + deleted_batch_offsets + } else { + // The deferred blanks take the front of the batch, pushing the offsets + // computed for this batch back by that many rows. + let mut batch_offsets = + Vec::with_capacity(pending_blank_rows as usize + deleted_batch_offsets.len()); + batch_offsets.extend(0..pending_blank_rows); + batch_offsets.extend( + deleted_batch_offsets + .iter() + .map(|offset| offset + pending_blank_rows), + ); + batch_offsets + }; + + let batch = add_blanks(batch, &batch_offsets)?; if let Some(batch_size) = self.legacy_batch_size { // validation just in case, when the input has a fixed batch size then the @@ -378,12 +493,23 @@ impl DeletionRestorer { } } - self.current_row_id += batch.num_rows() as u32; + // The deferred blanks were counted when they were deferred. + self.current_row_id += batch.num_rows() as u32 - pending_blank_rows; + self.pending_blank_rows = 0; Ok(batch) } } /// Add blank rows where there are deleted rows +/// +/// `batch_offsets` must be strictly increasing, and no offset may require more +/// live rows before it than the batch has left: an offset is the position a blank +/// takes in the output, so either kind of violation asks for an impossible number +/// of live rows in between. +/// +/// Blanks copy the batch's first row, so the batch must have at least one row. +/// [`DeletionRestorer::restore`] defers blanks past an empty batch to keep that +/// true; only legacy files, which cannot defer, can still reach the error below. pub(crate) fn add_blanks(batch: RecordBatch, batch_offsets: &[u32]) -> Result { // Fast early return if batch_offsets.is_empty() { @@ -391,18 +517,38 @@ pub(crate) fn add_blanks(batch: RecordBatch, batch_offsets: &[u32]) -> Result::with_capacity(batch.num_rows() + batch_offsets.len()); let mut batch_pos = 0; let mut next_id = 0; - for batch_offset in batch_offsets { - let num_rows = *batch_offset - next_id; + for (idx, batch_offset) in batch_offsets.iter().enumerate() { + // A non-increasing offset panics in debug and wraps in release; reject it + // up front so the error names the real problem. + let num_rows = batch_offset.checked_sub(next_id).ok_or_else(|| { + Error::internal(format!( + "Fragment Updater: blank offsets must be strictly increasing, but offset \ + {batch_offset} (entry {idx} of {}) is below the expected minimum {next_id}", + batch_offsets.len() + )) + })?; + // An offset needing more live rows than remain would index past the batch: + // `take` runs unchecked below, so catch it here rather than letting it + // panic inside arrow or, worse, read the wrong rows. + if num_rows > num_live_rows - batch_pos { + return Err(Error::internal(format!( + "Fragment Updater: blank offset {batch_offset} (entry {idx} of \ + {}) needs {num_rows} more live rows before it, but {} of the batch's \ + {num_live_rows} are still unused", + batch_offsets.len(), + num_live_rows - batch_pos + ))); + } selection_vector.extend(batch_pos..batch_pos + num_rows); // For simplicity, we just use the first value for deleted rows // TODO: optimize this to use small value for each column. @@ -410,7 +556,7 @@ pub(crate) fn add_blanks(batch: RecordBatch, batch_offsets: &[u32]) -> Result Result()) + .into_batch_rows(RowCount::from(0)) + .unwrap(); + + // Assert the source, not just is_err: the batch-size check further down + // returns Internal, and the two are different failures. + let err = restorer.restore(empty).unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. }), "{err:?}"); + } + + /// The v2 side of the same deletion vector: blanks owed by a batch with no live + /// row are deferred to a later batch that has one to copy. + #[test] + fn test_restore_deletes_leading_empty_batch() { + let mut restorer = super::DeletionRestorer::new((0..10).chain([15]).collect(), None); + + let empty = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(0)) + .unwrap(); + + // Nothing is written for the fully deleted batch itself. + assert_eq!(restorer.restore(empty.clone()).unwrap().num_rows(), 0); + assert!(!restorer.is_exhausted()); + + // A second empty batch must carry the debt through untouched: row 15 is + // out of its range, so it defers nothing of its own. + let restored = restorer.restore(empty).unwrap(); + assert_eq!(restored.num_rows(), 0); + assert!(!restorer.is_exhausted()); + + // The next batch covers row ids 10..15, so it owes the 10 deferred blanks + // in front of its own rows and one more for row 15 at the end. That last + // one is what pins the offset shift: without it the offsets would not be + // increasing. + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(5)) + .unwrap(); + let restored = restorer.restore(batch).unwrap(); + + assert_eq!(restored.num_rows(), 16); + let values = restored.column(0).as_primitive::(); + // Blanks copy the batch's first live row rather than inventing a value, + // which is what lets a non-nullable column through. + for i in 0..10 { + assert_eq!(values.value(i), 0); + } + for i in 0..5 { + assert_eq!(values.value(10 + i), i as i32); + } + assert_eq!(values.value(15), 0); + assert!(restorer.is_exhausted()); + } + + /// The debt itself has to keep the restorer from reporting exhaustion, not just + /// the deletion vector. With no row past the deleted run the iterator empties on + /// the first call, so only `pending_blank_rows` can hold `is_exhausted` back — + /// and it must, or `Updater::next` would accept a data file short by ten rows. + #[test] + fn test_restore_deletes_owes_blanks_after_vector_drains() { + let mut restorer = super::DeletionRestorer::new((0..10).collect(), None); + + let empty = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(0)) + .unwrap(); + + assert_eq!(restorer.restore(empty).unwrap().num_rows(), 0); + assert!(!restorer.is_exhausted()); + + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(5)) + .unwrap(); + assert_eq!(restorer.restore(batch).unwrap().num_rows(), 15); + assert!(restorer.is_exhausted()); + } + + /// A deletion vector naming a row the fragment does not have leaves the restorer + /// unexhausted with no blanks owed: the id stays stashed, so the iterator is never + /// drained. `Updater::next` relies on this to refuse rather than write a data file + /// missing that row, and the error must not claim a blank count for it. + #[test] + fn test_restore_deletes_not_exhausted_when_deletion_vector_overruns() { + let mut restorer = super::DeletionRestorer::new([100].into_iter().collect(), None); + + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(5)) + .unwrap(); + + // Row 100 is past this batch, so it is stashed rather than consumed and + // nothing is restored. No blanks are owed either — which is why the error in + // `Updater::next` cannot name a count. + assert_eq!(restorer.restore(batch).unwrap().num_rows(), 5); + assert!(!restorer.is_exhausted()); + } + + /// Deferred blanks are counted into `current_row_id` when they are deferred, so + /// consuming them must not count them again. A later deleted row is what makes + /// the double count observable: it lands at the wrong offset once the restorer + /// thinks the fragment is further along than it is. + #[test] + fn test_restore_deletes_does_not_double_count_deferred_blanks() { + let mut restorer = super::DeletionRestorer::new((0..10).chain([22]).collect(), None); + + let empty = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(0)) + .unwrap(); + assert_eq!(restorer.restore(empty).unwrap().num_rows(), 0); + + // Rows 10..20 are live, so this batch pays off the ten blanks and nothing + // else: row 22 is past its range and stays stashed. + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(10)) + .unwrap(); + assert_eq!(restorer.restore(batch).unwrap().num_rows(), 20); + assert!(!restorer.is_exhausted()); + + // Row 22 falls inside this batch's range, but only if current_row_id sits at + // 20. Counting the deferred blanks twice would have pushed it to 30, putting + // row 22 behind the batch and dropping its blank. + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(5)) + .unwrap(); + let restored = restorer.restore(batch).unwrap(); + + // Physical rows 20..25 arrive with row 22 deleted, so the blank lands third. + assert_eq!(restored.num_rows(), 6); + let values = restored.column(0).as_primitive::(); + assert_eq!(values.value(0), 0); + assert_eq!(values.value(1), 1); + assert_eq!(values.value(2), 0); + for i in 2..5 { + assert_eq!(values.value(1 + i), i as i32); + } + assert!(restorer.is_exhausted()); + } + #[test] fn test_add_blanks() { let batch = lance_datagen::gen_batch() @@ -506,4 +806,55 @@ mod tests { } assert_eq!(values.value(11), 0); } + + /// The ways a caller can hand `add_blanks` offsets it cannot satisfy. The + /// message keyword matters as much as the variant: most of these return + /// `Internal`, so matching only the variant would let one check stand in for + /// the other. + #[rstest] + #[case::empty_batch(0, &[0, 1, 2], "missing too many rows in merge")] + #[case::non_increasing(5, &[3, 1], "strictly increasing")] + #[case::equal_offsets(5, &[1, 1], "strictly increasing")] + #[case::past_end(5, &[100], "more live rows before it, but")] + // Rejected at the second offset with only three live rows left, so this is the + // only case that exercises the `- batch_pos` term: without it the remaining + // count reads as five and this offset slips through. + #[case::past_end_after_live_rows(5, &[2, 7], "more live rows before it, but")] + fn test_add_blanks_rejects_invalid_offsets( + #[case] num_rows: u64, + #[case] batch_offsets: &[u32], + #[case] expected_message: &str, + ) { + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(num_rows)) + .unwrap(); + + let err = add_blanks(batch, batch_offsets).unwrap_err(); + let message = err.to_string(); + assert!( + message.contains(expected_message), + "expected {expected_message:?} in {message:?}" + ); + } + + /// An offset equal to the batch length is the trailing-deletion shape: every + /// live row comes first, then the blanks. It has to be accepted, which is what + /// pins the bounds check to `>` rather than `>=`. + #[test] + fn test_add_blanks_at_end_of_batch() { + let batch = lance_datagen::gen_batch() + .col("x", lance_datagen::array::step::()) + .into_batch_rows(RowCount::from(5)) + .unwrap(); + + let with_blanks = add_blanks(batch, &[5]).unwrap(); + + assert_eq!(with_blanks.num_rows(), 6); + let values = with_blanks.column(0).as_primitive::(); + for i in 0..5 { + assert_eq!(values.value(i), i as i32); + } + assert_eq!(values.value(5), 0); + } } diff --git a/rust/lance/src/dataset/utils.rs b/rust/lance/src/dataset/utils.rs index c9770a3167b..6c61592aeb9 100644 --- a/rust/lance/src/dataset/utils.rs +++ b/rust/lance/src/dataset/utils.rs @@ -117,18 +117,23 @@ impl CapturedRowIds { } } - pub fn row_addrs(&self, index: Option<&RowIdIndex>) -> Cow<'_, RoaringTreemap> { + pub fn row_addrs(&self, index: Option<&RowIdIndex>) -> Result> { match self { - Self::AddressStyle(addrs) => Cow::Borrowed(addrs), + Self::AddressStyle(addrs) => Ok(Cow::Borrowed(addrs)), Self::SequenceStyle(sequence) => { let mut treemap = RoaringTreemap::new(); let Some(index) = index else { panic!("RowIdIndex required for sequence style row ids") }; for row_id in sequence.iter() { - treemap.insert(index.get(row_id).expect("row id missing from index").into()); + treemap.insert( + index + .get(row_id)? + .expect("row id missing from index") + .into(), + ); } - Cow::Owned(treemap) + Ok(Cow::Owned(treemap)) } } } diff --git a/rust/lance/src/dataset/versions/mod.rs b/rust/lance/src/dataset/versions/mod.rs new file mode 100644 index 00000000000..120a983b92a --- /dev/null +++ b/rust/lance/src/dataset/versions/mod.rs @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Dataset policies that differ across exact Lance file versions. +//! +//! File grammar belongs to `lance_file::versions`. This module contains only +//! operation-level dataset choices whose behavior actually differs by version. + +use std::{ + collections::{HashMap, HashSet}, + ops::Range, + sync::Arc, +}; + +use arrow_schema::{DataType, Field as ArrowField}; +use datafusion::catalog::Session; +use datafusion::execution::SendableRecordBatchStream; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use futures::{StreamExt, TryStreamExt}; +use lance_arrow::DataTypeExt; +use lance_core::{ + Error, Result, + datatypes::{Field, Projection, Schema, SchemaCompareOptions}, +}; +use lance_datafusion::chunker::{ + break_stream, break_stream_with_sizes, chunk_stream, chunk_stream_with_sizes, +}; +use lance_file::{ + version::ConcreteFileVersion, + versions as file_versions, + writer::{FileWriter, FileWriterOptions}, +}; +use lance_index::scalar::seed::IndexSeedWriter; +use lance_io::object_store::ObjectStore; +use lance_io::traits::Writer as ObjectWriter; +use lance_table::format::{DataFile, DataStorageFormat, Fragment, Manifest}; +use object_store::path::Path; + +use super::Dataset; +use super::fragment::{ + FileFragment, FragReadConfig, GenericFileReader, MetadataMode, V1FragmentReader, + write::FragmentCreateBuilder, +}; +use super::optimize::CompactionOptions; +use super::scanner::{PlannedFilteredScan, Scanner}; +use super::schema_evolution::optimize::{ + ChainedNewColumnTransformOptimizer, SqlToAllNullsOptimizer, +}; +use super::statistics::FieldStatistics; +use super::utils::SchemaAdapter; +use super::write::{self, GenericWriter, TargetBaseInfo, WriteParams, WriterOptions}; +use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions}; +use crate::io::exec::{ + AddRowAddrExec, FilterPlan as ExprFilterPlan, LanceScanConfig, LanceStream, TakeExec, +}; + +#[allow(clippy::too_many_arguments)] +pub fn create_scan_stream( + version: ConcreteFileVersion, + dataset: Arc, + fragments: Arc>, + offsets: Option>, + projection: Arc, + config: LanceScanConfig, + metrics: &ExecutionPlanMetricsSet, + partition: usize, +) -> datafusion::error::Result { + match version { + ConcreteFileVersion::V1 => LanceStream::try_new_v1( + dataset, fragments, offsets, projection, config, metrics, partition, + ), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => LanceStream::try_new_v2( + dataset, fragments, offsets, projection, config, metrics, partition, + ), + } +} + +pub fn schema_compare_options(version: ConcreteFileVersion) -> SchemaCompareOptions { + match version { + ConcreteFileVersion::V1 => SchemaCompareOptions { + compare_dictionary: true, + ..Default::default() + }, + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => SchemaCompareOptions::default(), + } +} + +async fn create_seed_writers( + version: ConcreteFileVersion, + dataset: Option<&Dataset>, + params: &WriteParams, +) -> Result>> { + match version { + ConcreteFileVersion::V1 => Ok(Vec::new()), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => write::create_seed_writers_current(dataset, params).await, + } +} + +fn create_current_file_writer( + version: ConcreteFileVersion, + object_writer: Box, + schema: Schema, + filename: String, + base_id: Option, +) -> Result<(FileWriter, DataFile)> { + let writer = + file_versions::create_writer(version, object_writer, schema, FileWriterOptions::default())?; + let mut data_file = DataFile::new_unstarted(filename, version); + data_file.base_id = base_id; + Ok((writer, data_file)) +} + +#[allow(clippy::too_many_arguments)] +pub async fn write_fragments( + version: ConcreteFileVersion, + dataset: Option<&Dataset>, + object_store: Arc, + base_dir: &Path, + normalized_schema: Schema, + data: SendableRecordBatchStream, + params: WriteParams, + target_bases_info: Option>, + file_row_counts: Option>, +) -> Result<(Vec, Schema)> { + let version_name = format!("{version:?}"); + let schema = write::prepare_write_schema( + dataset, + normalized_schema, + ¶ms, + schema_compare_options(version), + )?; + match version { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => { + write::validate_legacy_blob_write_schema(&schema, &version_name)?; + } + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::validate_blob_v2_write_schema(&schema)?; + } + } + let seed_writers = create_seed_writers(version, dataset, ¶ms).await?; + let fragments = write_fragments_direct( + version, + dataset, + object_store, + base_dir, + &schema, + data, + params, + target_bases_info, + seed_writers, + file_row_counts, + ) + .await?; + Ok((fragments, schema)) +} + +#[allow(clippy::too_many_arguments)] +pub async fn write_fragments_direct( + version: ConcreteFileVersion, + dataset: Option<&Dataset>, + object_store: Arc, + base_dir: &Path, + schema: &Schema, + data: SendableRecordBatchStream, + params: WriteParams, + target_bases_info: Option>, + seed_writers: Vec>, + file_row_counts: Option>, +) -> Result> { + let adapter = SchemaAdapter::new(data.schema()); + let data = adapter.to_physical_stream(data); + let buffered_reader = if let Some(file_row_counts) = file_row_counts.as_ref() { + if file_row_counts.contains(&0) { + return Err(Error::invalid_input( + "File row counts must be greater than zero", + )); + } + match version { + ConcreteFileVersion::V1 => { + if params.max_rows_per_group == 0 { + return Err(Error::invalid_input( + "max_rows_per_group must be greater than zero when file row counts are specified", + )); + } + let max_rows_per_group = params.max_rows_per_group; + let batch_row_counts = + file_row_counts + .clone() + .into_iter() + .flat_map(move |file_rows| { + (0..file_rows) + .step_by(max_rows_per_group) + .map(move |offset| (file_rows - offset).min(max_rows_per_group)) + }); + chunk_stream_with_sizes(data, batch_row_counts) + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => break_stream_with_sizes(data, file_row_counts.clone()), + } + } else { + match version { + ConcreteFileVersion::V1 => chunk_stream(data, params.max_rows_per_group), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => break_stream(data, params.max_rows_per_file) + .map_ok(|batch| vec![batch]) + .boxed(), + } + }; + let external_base_resolver = match version { + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::blob_v2_external_base_resolver(dataset, ¶ms, schema).await? + } + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => None, + }; + write::do_write_fragments_impl( + dataset, + object_store, + base_dir, + schema, + buffered_reader, + params, + move |object_store, schema, base_dir, options| async move { + open_writer(version, &object_store, &schema, &base_dir, options).await + }, + external_base_resolver, + target_bases_info, + seed_writers, + file_row_counts, + ) + .await +} + +fn binary_copy_files_match(fragments: &[Fragment], expected: ConcreteFileVersion) -> Result { + for fragment in fragments { + for data_file in &fragment.files { + if data_file.file_version()? != expected { + return Ok(false); + } + } + } + Ok(true) +} + +pub async fn can_use_binary_copy( + version: ConcreteFileVersion, + dataset: &Dataset, + options: &CompactionOptions, + fragments: &[Fragment], +) -> Result { + match version { + ConcreteFileVersion::V1 => Ok(false), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + if !binary_copy_files_match(fragments, version)? { + return Ok(false); + } + super::optimize::can_use_binary_copy_current(dataset, options, fragments).await + } + } +} + +pub async fn rewrite_files_binary_copy( + version: ConcreteFileVersion, + dataset: &Dataset, + fragments: &[Fragment], + params: &WriteParams, + read_batch_bytes: Option, +) -> Result> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported( + "binary-copy compaction is not supported for Lance file version 1".to_string(), + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + super::optimize::binary_copy::rewrite_files_binary_copy( + version, + dataset, + fragments, + params, + read_batch_bytes, + ) + .await + } + } +} + +pub fn check_manifest_storage_version(manifest: &mut Manifest) -> Result<()> { + let version = manifest.data_storage_format.lance_file_format(); + match version { + ConcreteFileVersion::V1 => repair_legacy_manifest_storage(manifest), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => validate_exact_manifest_storage(manifest, version), + } +} + +pub fn validate_column_indices(manifest: &Manifest) -> Result<()> { + match manifest.data_storage_format.lance_file_format() { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 => Ok(()), + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + validate_leaf_column_indices(manifest) + } + } +} + +fn validate_leaf_column_indices(manifest: &Manifest) -> Result<()> { + let mut fields_by_id: HashMap = HashMap::new(); + for field in manifest.schema.fields_pre_order() { + let needs_column = field.is_leaf() || field.is_packed_struct() || field.is_blob(); + fields_by_id + .entry(field.id) + .or_insert((field, needs_column)); + } + + let mut validated_lists: HashSet<(usize, usize)> = HashSet::new(); + + for fragment in manifest.fragments.iter() { + for data_file in &fragment.files { + let file_version = data_file.file_version()?; + if file_version == ConcreteFileVersion::V1 || data_file.column_indices.is_empty() { + continue; + } + if data_file.fields.len() != data_file.column_indices.len() { + return Err(Error::invalid_input(format!( + "Data file '{}' (fragment {}) has {} field ids but {} column indices. These must be the same length.", + data_file.path, + fragment.id, + data_file.fields.len(), + data_file.column_indices.len() + ))); + } + if file_version == ConcreteFileVersion::V2_0 { + continue; + } + let list_key = ( + data_file.fields.as_ptr() as usize, + data_file.column_indices.as_ptr() as usize, + ); + if !validated_lists.insert(list_key) { + continue; + } + for (field_id, column_index) in + data_file.fields.iter().zip(data_file.column_indices.iter()) + { + let Some((field, needs_column)) = fields_by_id.get(field_id).copied() else { + continue; + }; + if needs_column && *column_index == -1 { + return Err(Error::invalid_input(format!( + "Field '{}' (id={}) in data file '{}' (fragment {}) has column_index=-1, but leaf fields, packed structs, and blob fields must have a valid column index in file format 2.1+.", + field.name, field_id, data_file.path, fragment.id + ))); + } + if !needs_column && *column_index != -1 { + return Err(Error::invalid_input(format!( + "Non-leaf field '{}' (id={}) in data file '{}' (fragment {}) has column_index={}, but non-leaf fields should have column_index=-1 in file format 2.1+.", + field.name, field_id, data_file.path, fragment.id, column_index + ))); + } + } + } + } + Ok(()) +} + +pub async fn write_fragment( + version: ConcreteFileVersion, + builder: &FragmentCreateBuilder<'_>, + stream: SendableRecordBatchStream, + schema: Schema, + id: u64, +) -> Result { + match version { + ConcreteFileVersion::V1 => builder.write_v1_impl(stream, schema, id).await, + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + builder + .write_current_impl( + move |object_writer, schema, filename| { + create_current_file_writer(version, object_writer, schema, filename, None) + }, + stream, + schema, + id, + ) + .await + } + } +} + +pub async fn open_writer( + version: ConcreteFileVersion, + object_store: &ObjectStore, + schema: &Schema, + base_dir: &Path, + options: WriterOptions, +) -> Result> { + match version { + ConcreteFileVersion::V1 => { + write::open_v1_writer(object_store, schema, base_dir, options).await + } + ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => { + write::open_current_writer( + move |object_writer, schema, filename, base_id| { + create_current_file_writer(version, object_writer, schema, filename, base_id) + }, + object_store, + schema, + base_dir, + options, + ) + .await + } + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::open_current_blob_v2_writer( + move |object_writer, schema, filename, base_id| { + create_current_file_writer(version, object_writer, schema, filename, base_id) + }, + object_store, + schema, + base_dir, + options, + ) + .await + } + } +} + +pub async fn open_update_writer( + version: ConcreteFileVersion, + dataset: &Dataset, + schema: &Schema, + allow_external_blob_outside_bases: bool, +) -> Result> { + let external_base_resolver = match version { + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => { + write::blob_v2_external_base_resolver(Some(dataset), &WriteParams::default(), schema) + .await? + } + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => None, + }; + open_writer( + version, + &dataset.object_store, + schema, + &dataset.base, + WriterOptions::update( + dataset.session.store_registry(), + external_base_resolver, + allow_external_blob_outside_bases, + ), + ) + .await +} + +pub async fn create_fragment_from_file( + file_version: ConcreteFileVersion, + dataset_version: ConcreteFileVersion, + filename: &str, + dataset: &Dataset, + fragment_id: usize, + physical_rows: Option, +) -> Result { + if file_version != dataset_version { + return Err(Error::invalid_input(format!( + "File version mismatch. Dataset version: {:?} Fragment version: {:?}", + dataset_version, file_version + ))); + } + match file_version { + ConcreteFileVersion::V1 => { + FileFragment::create_from_v1_file(filename, dataset, fragment_id, physical_rows).await + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + FileFragment::create_from_current_file(filename, dataset, fragment_id).await + } + } +} + +pub fn index_file_version(version: ConcreteFileVersion) -> ConcreteFileVersion { + match version { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 => ConcreteFileVersion::V2_0, + ConcreteFileVersion::V2_1 => ConcreteFileVersion::V2_1, + ConcreteFileVersion::V2_2 => ConcreteFileVersion::V2_2, + ConcreteFileVersion::V2_3 => ConcreteFileVersion::V2_3, + } +} + +pub async fn open_file_reader( + version: ConcreteFileVersion, + fragment: &FileFragment, + data_file: &DataFile, + projection: Option<&Schema>, + read_config: &FragReadConfig, + metadata_mode: MetadataMode, +) -> Result>> { + match version { + ConcreteFileVersion::V1 => fragment.open_v1_file_reader(data_file, projection).await, + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + fragment + .open_current_file_reader(data_file, projection, read_config, metadata_mode) + .await + } + } +} + +pub async fn open_v1_fragment_reader( + fragment: &FileFragment, + projection: &Schema, + read_config: &FragReadConfig, +) -> Result { + for data_file in &fragment.metadata().files { + let actual = data_file.file_version()?; + if actual != ConcreteFileVersion::V1 { + return Err(Error::invalid_input(format!( + "Cannot open file {} with the v1 reader because it has version {}", + data_file.path, actual + ))); + } + } + fragment + .open_v1_fragment_reader(projection, read_config) + .await +} + +pub async fn row_group_size_for_rewrite( + version: ConcreteFileVersion, + fragment: &FileFragment, +) -> Result> { + match version { + ConcreteFileVersion::V1 => { + let reader = open_v1_fragment_reader( + fragment, + fragment.dataset().schema(), + &FragReadConfig::default(), + ) + .await?; + Ok(reader.num_rows_in_batch(0)) + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(None), + } +} + +pub fn is_upcast_downcast( + version: ConcreteFileVersion, + from_type: &DataType, + to_type: &DataType, +) -> bool { + is_upcast_downcast_impl( + from_type, + to_type, + !matches!(version, ConcreteFileVersion::V1), + ) +} + +fn is_upcast_downcast_impl( + from_type: &DataType, + to_type: &DataType, + dictionary_materialization: bool, +) -> bool { + use DataType::*; + match (from_type, to_type) { + (_, Dictionary(_, _)) if !dictionary_materialization => false, + (Dictionary(_, from_value_type), _) => { + is_upcast_downcast_impl(from_value_type, to_type, dictionary_materialization) + } + (_, Dictionary(_, to_value_type)) => { + is_upcast_downcast_impl(from_type, to_value_type, dictionary_materialization) + } + (from, to) if from.is_integer() => to.is_integer(), + (from, to) if from.is_floating() => to.is_floating(), + (from, to) if from.is_temporal() => to.is_temporal(), + (Boolean, to) => matches!(to, Boolean), + (Utf8 | LargeUtf8, to) => matches!(to, Utf8 | LargeUtf8), + (Binary | LargeBinary, to) => matches!(to, Binary | LargeBinary), + (Decimal128(_, _) | Decimal256(_, _), to) => { + matches!(to, Decimal128(_, _) | Decimal256(_, _)) + } + (List(from_field) | LargeList(from_field) | FixedSizeList(from_field, _), to_type) => { + match to_type { + List(to_field) | LargeList(to_field) | FixedSizeList(to_field, _) => { + is_upcast_downcast_impl( + from_field.data_type(), + to_field.data_type(), + dictionary_materialization, + ) + } + _ => false, + } + } + _ => false, + } +} + +pub fn validate_nulls( + version: ConcreteFileVersion, + datatype: &DataType, + has_nulls: bool, +) -> Result<()> { + let supported = match version { + ConcreteFileVersion::V1 => matches!( + datatype, + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Binary + | DataType::List(_) + | DataType::FixedSizeBinary(_) + | DataType::FixedSizeList(_, _) + ), + ConcreteFileVersion::V2_0 => !matches!(datatype, DataType::Struct(..)), + ConcreteFileVersion::V2_1 | ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => true, + }; + if has_nulls && !supported { + return Err(Error::invalid_input(format!( + "Join produced null values for type: {:?}, but storing nulls for this data type is not supported by the dataset's current Lance file format version: {:?}. This can be caused by an explicit null in the new data.", + datatype, version + ))); + } + Ok(()) +} + +fn reject_nested_column_add(field: &ArrowField, version: ConcreteFileVersion) -> Result<()> { + Err(Error::invalid_input(format!( + "Column {} is a struct col, add sub column is not supported in Lance file version {}", + field.name(), + version + ))) +} + +fn reject_nested_v1(field: &ArrowField) -> Result<()> { + reject_nested_column_add(field, ConcreteFileVersion::V1) +} + +fn reject_nested_v2_0(field: &ArrowField) -> Result<()> { + reject_nested_column_add(field, ConcreteFileVersion::V2_0) +} + +fn reject_nested_v2_1(field: &ArrowField) -> Result<()> { + reject_nested_column_add(field, ConcreteFileVersion::V2_1) +} + +fn allow_nested(_field: &ArrowField) -> Result<()> { + Ok(()) +} + +pub fn check_field_conflict( + version: ConcreteFileVersion, + left: &ArrowField, + right: &ArrowField, +) -> Result<()> { + let validate = match version { + ConcreteFileVersion::V1 => reject_nested_v1, + ConcreteFileVersion::V2_0 => reject_nested_v2_0, + ConcreteFileVersion::V2_1 => reject_nested_v2_1, + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => allow_nested, + }; + super::schema_evolution::check_field_conflict_with(left, right, validate) +} + +fn exclude_struct_field(field: &Field, other: &Field) -> Option { + field + .data_type() + .is_struct() + .then(|| field.exclude(other)) + .flatten() +} + +fn exclude_nested_field(field: &Field, other: &Field) -> Option { + field + .data_type() + .is_nested() + .then(|| field.exclude(other)) + .flatten() +} + +pub fn exclude_schema( + version: ConcreteFileVersion, + source: &Schema, + other: &Schema, +) -> Result { + let exclude = match version { + ConcreteFileVersion::V1 | ConcreteFileVersion::V2_0 | ConcreteFileVersion::V2_1 => { + exclude_struct_field + } + ConcreteFileVersion::V2_2 | ConcreteFileVersion::V2_3 => exclude_nested_field, + }; + super::schema_evolution::exclude_with(source, other, exclude) +} + +pub fn configure_new_column_optimizers( + version: ConcreteFileVersion, + optimizer: &mut ChainedNewColumnTransformOptimizer, +) { + match version { + ConcreteFileVersion::V1 => {} + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + optimizer.add_optimizer(Box::new(SqlToAllNullsOptimizer::new())); + } + } +} + +pub fn validate_metadata_only_null_columns(version: ConcreteFileVersion) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported_source( + "Cannot add all-null columns to legacy dataset version.".into(), + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(()), + } +} + +#[allow(clippy::too_many_arguments)] +pub(in crate::dataset) async fn filtered_read( + version: ConcreteFileVersion, + scanner: &Scanner, + filter_plan: &ExprFilterPlan, + projection: Projection, + make_deletions_null: bool, + fragments: Option>>, + scan_range: Option>, + is_prefilter: bool, + session: Option<&dyn Session>, +) -> Result { + match version { + ConcreteFileVersion::V1 => { + scanner + .legacy_filtered_read( + filter_plan, + projection, + make_deletions_null, + fragments, + scan_range, + is_prefilter, + ) + .await + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + let limit_pushed_down = scan_range.is_some(); + let plan = scanner + .new_filtered_read( + filter_plan, + projection, + make_deletions_null, + fragments, + scan_range, + session, + ) + .await?; + Ok(PlannedFilteredScan { + filter_pushed_down: true, + limit_pushed_down, + plan, + }) + } + } +} + +pub fn take( + version: ConcreteFileVersion, + scanner: &Scanner, + input: Arc, + output_projection: Projection, +) -> Result> { + match version { + ConcreteFileVersion::V1 => scanner.take_legacy(input, output_projection), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => scanner.take_current(input, output_projection), + } +} + +pub async fn collect_data_stats( + version: ConcreteFileVersion, + dataset: &Arc, + field_stats: &mut HashMap, +) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Ok(()), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + super::statistics::collect_current_data_stats(dataset, field_stats).await + } + } +} + +pub fn merge_insert_indexed_take( + version: ConcreteFileVersion, + dataset: Arc, + mut index_mapper: Arc, + projection: Projection, + add_row_addr: bool, +) -> Result> { + match version { + ConcreteFileVersion::V1 => { + if add_row_addr { + let position = index_mapper.schema().fields().len(); + index_mapper = Arc::new(AddRowAddrExec::try_new( + index_mapper, + dataset.clone(), + position, + )?); + } + Ok(Arc::new( + TakeExec::try_new(dataset, index_mapper, projection)?.ok_or_else(|| { + Error::internal("merge-insert legacy take unexpectedly needed no columns") + })?, + )) + } + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => { + let mut projection = projection.with_row_id(); + if add_row_addr { + projection = projection.with_row_addr(); + } + Ok(Arc::new(FilteredReadExec::try_new( + dataset, + FilteredReadOptions::new(projection), + Some(index_mapper), + )?)) + } + } +} + +pub fn validate_row_stream_read(version: ConcreteFileVersion) -> Result<()> { + match version { + ConcreteFileVersion::V1 => Err(Error::not_supported_source( + "taking rows through FilteredReadExec requires the v2 storage format" + .to_string() + .into(), + )), + ConcreteFileVersion::V2_0 + | ConcreteFileVersion::V2_1 + | ConcreteFileVersion::V2_2 + | ConcreteFileVersion::V2_3 => Ok(()), + } +} + +fn repair_legacy_manifest_storage(manifest: &mut Manifest) -> Result<()> { + let declared = manifest.data_storage_format.lance_file_format(); + if let Some(actual) = Fragment::try_infer_version(&manifest.fragments) + .map_err(|error| { + Error::internal(format!( + "The dataset contains a mixture of file versions. You will need to rollback to an earlier version: {error}" + )) + })? + && actual != ConcreteFileVersion::V1 + { + log::warn!( + "Data storage version {} is less than the actual file version {}. This has been automatically updated.", + declared, + actual + ); + manifest.data_storage_format = DataStorageFormat::new(actual); + } + Ok(()) +} + +fn validate_exact_manifest_storage( + manifest: &Manifest, + expected: ConcreteFileVersion, +) -> Result<()> { + if let Some(actual) = Fragment::try_infer_version(&manifest.fragments)? + && actual != expected + { + return Err(Error::internal(format!( + "The operation added files with version {}. However, the data storage version is {}.", + actual, expected + ))); + } + Ok(()) +} diff --git a/rust/lance/src/dataset/write.rs b/rust/lance/src/dataset/write.rs index 80afa49ffdd..559b6c6c486 100644 --- a/rust/lance/src/dataset/write.rs +++ b/rust/lance/src/dataset/write.rs @@ -2,51 +2,51 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use arrow_array::RecordBatch; +use bytes::Bytes; use chrono::TimeDelta; use datafusion::physical_plan::SendableRecordBatchStream; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::StreamExt; use lance_arrow::{ ARROW_EXT_NAME_KEY, BLOB_DEDICATED_SIZE_THRESHOLD_META_KEY, BLOB_INLINE_SIZE_THRESHOLD_META_KEY, BLOB_META_KEY, BLOB_PACK_FILE_SIZE_THRESHOLD_META_KEY, BLOB_V2_EXT_NAME, }; -use lance_core::datatypes::{ - NullabilityComparison, OnMissing, OnTypeMismatch, SchemaCompareOptions, +use lance_core::datatypes::{NullabilityComparison, OnMissing, OnTypeMismatch}; +use lance_core::utils::tracing::{ + AUDIT_MODE_CREATE, AUDIT_MODE_DELETE, AUDIT_TYPE_DATA, TRACE_FILE_AUDIT, }; -use lance_core::error::LanceOptionExt; -use lance_core::utils::tempfile::TempDir; -use lance_core::utils::tracing::{AUDIT_MODE_CREATE, AUDIT_TYPE_DATA, TRACE_FILE_AUDIT}; use lance_core::{Error, Result, datatypes::Schema}; -use lance_datafusion::chunker::{break_stream, chunk_stream}; -use lance_datafusion::spill::{SpillReceiver, SpillSender, create_replay_spill}; use lance_datafusion::utils::StreamingWriteSource; -use lance_file::previous::writer::{ - FileWriter as PreviousFileWriter, ManifestProvider as PreviousManifestProvider, +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; +use lance_file::versions::v1::writer::{ + FileWriter as V1FileWriter, ManifestProvider as V1ManifestProvider, }; -use lance_file::version::LanceFileVersion; -use lance_file::writer::{self as current_writer, FileWriterOptions}; +use lance_file::writer::{self as current_writer}; use lance_io::object_store::{ ObjectStore, ObjectStoreParams, ObjectStoreRegistry, parse_base_scoped_key, }; -use lance_table::format::{BasePath, DataFile, Fragment}; +use lance_io::traits::Writer; +use lance_table::format::{BasePath, DataFile, Fragment, IndexMetadata}; use lance_table::io::commit::{CommitHandler, commit_handler_from_url}; use lance_table::io::manifest::ManifestDescribing; use object_store::path::Path; use std::borrow::Cow; -use std::collections::{BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; +use std::future::Future; use std::num::NonZero; use std::sync::Arc; use std::sync::atomic::AtomicUsize; use tracing::{info, instrument}; use crate::Dataset; -use crate::blob::normalize_prepared_blob_schema; +use crate::blob::prepared_to_logical_blob_schema; use crate::dataset::blob::{ BlobPreprocessor, ExternalBaseCandidate, ExternalBaseResolver, blob_dedicated_threshold_from_metadata, blob_inline_threshold_from_metadata, blob_pack_file_threshold_from_metadata, preprocess_blob_batches, }; +use crate::index::DatasetIndexExt; +use crate::index::scalar::{IndexDetails, fetch_index_details}; use crate::session::Session; use super::DATA_DIR; @@ -54,6 +54,7 @@ use super::fragment::write::generate_random_filename; use super::progress::{NoopFragmentWriteProgress, WriteFragmentProgress}; use super::transaction::Transaction; use super::utils::SchemaAdapter; +use super::versions; mod commit; pub mod delete; @@ -454,8 +455,8 @@ impl WriteParams { } } - pub fn storage_version_or_default(&self) -> LanceFileVersion { - self.data_storage_version.unwrap_or_default() + pub fn storage_version_or_default(&self) -> ConcreteFileVersion { + self.data_storage_version.unwrap_or_default().resolve() } pub fn store_registry(&self) -> Arc { @@ -594,40 +595,62 @@ pub async fn write_fragments( .await } +fn take_batch_rows(batches: &mut VecDeque, max_rows: usize) -> Vec { + let mut output = Vec::with_capacity(batches.len()); + let mut rows_remaining = max_rows; + + while rows_remaining > 0 { + let Some(batch) = batches.pop_front() else { + break; + }; + let batch_rows = batch.num_rows(); + if batch_rows == 0 { + continue; + } + if batch_rows <= rows_remaining { + rows_remaining -= batch_rows; + output.push(batch); + } else { + output.push(batch.slice(0, rows_remaining)); + batches.push_front(batch.slice(rows_remaining, batch_rows - rows_remaining)); + rows_remaining = 0; + } + } + + output +} + +fn balanced_row_counts(total_rows: usize, max_rows_per_file: usize) -> VecDeque { + if total_rows == 0 { + return VecDeque::new(); + } + + let file_count = total_rows.div_ceil(max_rows_per_file); + let base_rows_per_file = total_rows / file_count; + let larger_file_count = total_rows % file_count; + (0..file_count) + .map(|file_index| base_rows_per_file + usize::from(file_index < larger_file_count)) + .collect() +} + #[allow(clippy::too_many_arguments)] -pub async fn do_write_fragments( +pub(super) async fn do_write_fragments_impl( dataset: Option<&Dataset>, object_store: Arc, base_dir: &Path, schema: &Schema, - data: SendableRecordBatchStream, + mut buffered_reader: futures::stream::BoxStream<'static, Result>>, params: WriteParams, - storage_version: LanceFileVersion, + open_writer: OpenWriter, + external_base_resolver: Option>, target_bases_info: Option>, -) -> Result> { - let adapter = SchemaAdapter::new(data.schema()); - let data = adapter.to_physical_stream(data); - - let mut buffered_reader = if storage_version == LanceFileVersion::Legacy { - // In v1 we split the stream into row group sized batches - chunk_stream(data, params.max_rows_per_group) - } else { - // In v2 we don't care about group size but we do want to break - // the stream on file boundaries - break_stream(data, params.max_rows_per_file) - .map_ok(|batch| vec![batch]) - .boxed() - }; - - let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields_pre_order().any(|field| field.is_blob_v2()) - { - Some(Arc::new( - build_external_base_resolver(dataset, ¶ms).await?, - )) - } else { - None - }; + mut seed_writers: Vec>, + file_row_counts: Option>, +) -> Result> +where + OpenWriter: Fn(Arc, Schema, Path, WriterOptions) -> OpenWriterFuture + Send + Sync, + OpenWriterFuture: Future>> + Send, +{ let source_store_registry = dataset .map(|ds| ds.session.store_registry()) .unwrap_or_else(|| params.store_registry()); @@ -639,7 +662,7 @@ pub async fn do_write_fragments( object_store.clone(), base_dir, schema, - storage_version, + open_writer, target_bases_info, external_base_resolver, params.allow_external_blob_outside_bases, @@ -654,59 +677,177 @@ pub async fn do_write_fragments( let mut bytes_completed: u64 = 0; let mut rows_completed: u64 = 0; let mut files_written: u32 = 0; + let has_file_row_counts = file_row_counts.is_some(); + let max_planned_file_rows = file_row_counts + .as_ref() + .and_then(|row_counts| row_counts.iter().copied().max()); + let mut planned_rows_remaining = file_row_counts + .as_ref() + .map(|row_counts| { + row_counts.iter().try_fold(0_usize, |total, &row_count| { + total + .checked_add(row_count) + .ok_or_else(|| Error::internal("Planned file row count total overflowed usize")) + }) + }) + .transpose()?; + let mut file_row_counts = file_row_counts.map(VecDeque::from); + let mut rows_remaining_in_planned_file = file_row_counts.as_mut().and_then(VecDeque::pop_front); // Wrap the loop in an async block so `?` returns into `loop_result` and we // can run cleanup before propagating the error. let loop_result: Result<()> = async { while let Some(batch_chunk) = buffered_reader.next().await { - let batch_chunk = batch_chunk?; - - if writer.is_none() { - let (new_writer, new_fragment) = writer_generator.new_writer().await?; - params.progress.begin(&new_fragment).await?; - writer = Some(new_writer); - fragments.push(new_fragment); - } - - writer.as_mut().unwrap().write(&batch_chunk).await?; - for batch in &batch_chunk { - num_rows_in_current_file += batch.num_rows() as u32; - } + let mut pending_batches = VecDeque::from(batch_chunk?); + + while !pending_batches.is_empty() { + let rows_to_take = if has_file_row_counts { + rows_remaining_in_planned_file.ok_or_else(|| { + Error::internal( + "Writer received rows after all planned file boundaries were consumed", + ) + })? + } else { + usize::MAX + }; + let batch_chunk = take_batch_rows(&mut pending_batches, rows_to_take); + if batch_chunk.is_empty() { + continue; + } - if let Some(cb) = ¶ms.write_progress { - let current_bytes = writer.as_mut().unwrap().tell().await?; - cb.call(WriteStats { - bytes_written: bytes_completed + current_bytes, - rows_written: rows_completed + num_rows_in_current_file as u64, - files_written, - }); - } + if writer.is_none() { + let (new_writer, new_fragment) = writer_generator.new_writer().await?; + params.progress.begin(&new_fragment).await?; + writer = Some(new_writer); + fragments.push(new_fragment); + } - if num_rows_in_current_file >= params.max_rows_per_file as u32 - || writer.as_mut().unwrap().tell().await? >= params.max_bytes_per_file as u64 - { - let (num_rows, data_file) = writer.take().unwrap().finish().await?; - info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path); - debug_assert_eq!(num_rows, num_rows_in_current_file); - bytes_completed += data_file.file_size_bytes.get().map_or(0, |s| s.get()); - rows_completed += num_rows as u64; - files_written += 1; - let last_fragment = fragments.last_mut().unwrap(); - last_fragment.physical_rows = Some(num_rows as usize); - last_fragment.files.push(data_file); - // Notify after pushing the data file so it's tracked for cleanup - // if the callback fails. - params.progress.complete(fragments.last().unwrap()).await?; + let active_writer = writer.as_mut().ok_or_else(|| { + Error::internal("Writer was not initialized before writing a batch") + })?; + active_writer.write(&batch_chunk).await?; + for seed_writer in seed_writers.iter_mut() { + let col_name = seed_writer.column_name().to_owned(); + for batch in &batch_chunk { + if let Some(col) = batch.column_by_name(&col_name) { + seed_writer.observe_batch(col)?; + } + } + } + let batch_chunk_rows = + batch_chunk.iter().map(RecordBatch::num_rows).sum::(); + num_rows_in_current_file += batch_chunk_rows as u32; + + let reached_planned_file_boundary = if has_file_row_counts { + let rows_remaining = + rows_remaining_in_planned_file.as_mut().ok_or_else(|| { + Error::internal( + "Writer received rows without an active planned file boundary", + ) + })?; + *rows_remaining = rows_remaining.checked_sub(batch_chunk_rows).ok_or_else(|| { + Error::internal(format!( + "Writer chunk of {batch_chunk_rows} rows crossed a planned file boundary with {rows_remaining} rows remaining" + )) + })?; + let total_remaining = planned_rows_remaining.as_mut().ok_or_else(|| { + Error::internal("Writer lost the planned row count total") + })?; + *total_remaining = + total_remaining.checked_sub(batch_chunk_rows).ok_or_else(|| { + Error::internal(format!( + "Writer consumed {batch_chunk_rows} rows after the planned row count total was exhausted" + )) + })?; + *rows_remaining == 0 + } else { + false + }; + + let current_file_bytes = writer + .as_mut() + .ok_or_else(|| Error::internal("Writer disappeared after writing a batch"))? + .tell() + .await?; if let Some(cb) = ¶ms.write_progress { cb.call(WriteStats { - bytes_written: bytes_completed, - rows_written: rows_completed, + bytes_written: bytes_completed + current_file_bytes, + rows_written: rows_completed + num_rows_in_current_file as u64, files_written, }); } - num_rows_in_current_file = 0; + + let reached_row_limit = if has_file_row_counts { + reached_planned_file_boundary + } else { + num_rows_in_current_file >= params.max_rows_per_file as u32 + }; + let reached_byte_limit = current_file_bytes >= params.max_bytes_per_file as u64; + + if reached_row_limit || reached_byte_limit { + if has_file_row_counts { + if reached_planned_file_boundary { + rows_remaining_in_planned_file = file_row_counts + .as_mut() + .and_then(VecDeque::pop_front); + } else { + // A byte-driven close is an extra physical boundary. Rebalance all + // unwritten rows under the original maximum instead of preserving a + // tiny abandoned remainder or rolling it into an oversized tail. + let total_remaining = planned_rows_remaining.ok_or_else(|| { + Error::internal("Writer lost the planned row count total") + })?; + let max_rows_per_file = max_planned_file_rows.ok_or_else(|| { + Error::internal( + "Writer cannot replan byte-limited files without a maximum planned row count", + ) + })?; + let mut replanned_counts = + balanced_row_counts(total_remaining, max_rows_per_file); + rows_remaining_in_planned_file = replanned_counts.pop_front(); + file_row_counts = Some(replanned_counts); + } + } + + let mut w = writer.take().ok_or_else(|| { + Error::internal("Writer disappeared before completing a file") + })?; + flush_seed_writers(w.as_mut(), &mut seed_writers).await?; + let (num_rows, data_file) = w.finish().await?; + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path); + debug_assert_eq!(num_rows, num_rows_in_current_file); + bytes_completed += data_file.file_size_bytes.get().map_or(0, |s| s.get()); + rows_completed += num_rows as u64; + files_written += 1; + let last_fragment = fragments.last_mut().ok_or_else(|| { + Error::internal("Writer completed a file without a pending fragment") + })?; + last_fragment.physical_rows = Some(num_rows as usize); + last_fragment.files.push(data_file); + // Notify after pushing the data file so it's tracked for cleanup + // if the callback fails. + let completed_fragment = fragments.last().ok_or_else(|| { + Error::internal("Writer completed a file without a fragment") + })?; + params.progress.complete(completed_fragment).await?; + if let Some(cb) = ¶ms.write_progress { + cb.call(WriteStats { + bytes_written: bytes_completed, + rows_written: rows_completed, + files_written, + }); + } + num_rows_in_current_file = 0; + } } } + + if has_file_row_counts && planned_rows_remaining != Some(0) { + return Err(Error::internal(format!( + "Writer input ended with {} planned rows remaining", + planned_rows_remaining.unwrap_or_default() + ))); + } Ok(()) } .await; @@ -727,6 +868,17 @@ pub async fn do_write_fragments( // Complete the final writer if let Some(mut writer) = writer.take() { + if let Err(e) = flush_seed_writers(writer.as_mut(), &mut seed_writers).await { + drop(writer); + cleanup_data_fragments( + &object_store, + base_dir, + cleanup_bases.as_deref(), + &fragments, + ) + .await; + return Err(e); + } match writer.finish().await { Ok((num_rows, data_file)) => { info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_CREATE, r#type=AUDIT_TYPE_DATA, path = &data_file.path); @@ -761,6 +913,23 @@ pub async fn do_write_fragments( Ok(fragments) } +/// Flush all seed writers into the given file writer, embedding seed buffers +/// and schema metadata before `finish()` is called. +async fn flush_seed_writers( + writer: &mut dyn GenericWriter, + seed_writers: &mut [Box], +) -> Result<()> { + for seed_writer in seed_writers.iter_mut() { + if let Some(bytes) = seed_writer.finish()? { + let buf_index = writer.add_global_buffer(bytes).await?; + let key = seed_writer.schema_metadata_key(); + let value = seed_writer.schema_metadata_value(buf_index); + writer.add_schema_metadata(key, value); + } + } + Ok(()) +} + /// Best-effort cleanup of data files for fragments that were written but not committed. /// /// Contract: @@ -781,6 +950,11 @@ pub(crate) async fn cleanup_data_fragments( let data_dir = base_dir.clone().join(DATA_DIR); let mut skipped_external = 0usize; for fragment in fragments { + // Deliberately not `referenced_lance_files()`: callers decide which + // files belong to the failed write. `schema_evolution` passes a live + // fragment whose `files` it narrowed to the newly written ones while + // leaving `overlays` untouched, so including overlays here would delete + // live data. for file in &fragment.files { let (store, file_dir) = if let Some(base_id) = file.base_id { match target_bases.and_then(|bases| bases.iter().find(|b| b.base_id == base_id)) { @@ -802,8 +976,13 @@ pub(crate) async fn cleanup_data_fragments( }; let path = file_dir.clone().join(file.path.as_str()); - if let Err(e) = store.delete(&path).await { - log::warn!("Failed to clean up orphaned data file '{}': {}", path, e); + match store.delete(&path).await { + Ok(()) => { + info!(target: TRACE_FILE_AUDIT, mode=AUDIT_MODE_DELETE, r#type=AUDIT_TYPE_DATA, path = file.path.as_str()); + } + Err(e) => { + log::warn!("Failed to clean up orphaned data file '{}': {}", path, e); + } } // Clean up any blob v2 sidecars that might exist for this data file. @@ -1220,6 +1399,20 @@ async fn build_external_base_resolver( Ok(ExternalBaseResolver::new(candidates, store_registry)) } +pub(super) async fn blob_v2_external_base_resolver( + dataset: Option<&Dataset>, + params: &WriteParams, + schema: &Schema, +) -> Result>> { + if schema.fields_pre_order().any(|field| field.is_blob_v2()) { + Ok(Some(Arc::new( + build_external_base_resolver(dataset, params).await?, + ))) + } else { + Ok(None) + } +} + /// Writes the given data to the dataset and returns fragments. /// /// NOTE: the fragments have not yet been assigned an ID. That must be done @@ -1229,8 +1422,39 @@ async fn build_external_base_resolver( /// This is a private variant that takes a `SendableRecordBatchStream` instead /// of a reader. We don't expose the stream at our interface because it is a /// DataFusion type. +/// +/// The caller must resolve `storage_version` once for the operation. Operations +/// that also select a commit format must reuse the same value when committing. +#[allow(clippy::too_many_arguments)] #[instrument(level = "debug", skip_all)] pub async fn write_fragments_internal( + storage_version: ConcreteFileVersion, + dataset: Option<&Dataset>, + object_store: Arc, + base_dir: &Path, + schema: Schema, + data: SendableRecordBatchStream, + params: WriteParams, + target_bases_info: Option>, +) -> Result<(Vec, Schema)> { + write_fragments_internal_with_file_row_counts( + storage_version, + dataset, + object_store, + base_dir, + schema, + data, + params, + target_bases_info, + None, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +#[instrument(level = "debug", skip_all)] +pub(crate) async fn write_fragments_internal_with_file_row_counts( + storage_version: ConcreteFileVersion, dataset: Option<&Dataset>, object_store: Arc, base_dir: &Path, @@ -1238,6 +1462,7 @@ pub async fn write_fragments_internal( data: SendableRecordBatchStream, params: WriteParams, target_bases_info: Option>, + file_row_counts: Option>, ) -> Result<(Vec, Schema)> { let mut params = params; let adapter = SchemaAdapter::new(data.schema()); @@ -1256,91 +1481,116 @@ pub async fn write_fragments_internal( // Make sure the max rows per group is not larger than the max rows per file params.max_rows_per_group = std::cmp::min(params.max_rows_per_group, params.max_rows_per_file); validate_external_blob_write_params(¶ms)?; - let normalized_converted_schema = normalize_prepared_blob_schema(&converted_schema)?; - - let (schema, storage_version) = if let Some(dataset) = dataset { - match params.mode { - WriteMode::Append | WriteMode::Create => { - // Append mode, so we need to check compatibility - normalized_converted_schema.check_compatible( - dataset.schema(), - &SchemaCompareOptions { - // We don't care if the user claims their data is nullable / non-nullable. We will - // verify against the actual data. - compare_nullability: NullabilityComparison::Ignore, - allow_missing_if_nullable: true, - ignore_field_order: true, - compare_dictionary: dataset.is_legacy_storage(), - ..Default::default() - }, - )?; - validate_blob_threshold_metadata_for_append( - &normalized_converted_schema, - dataset.schema(), - )?; - let write_schema = dataset.schema().project_by_schema( - &normalized_converted_schema, - OnMissing::Error, - OnTypeMismatch::Error, - )?; - // Use the storage version from the dataset, ignoring any version from the user. - let data_storage_version = dataset - .manifest() - .data_storage_format - .lance_file_version()?; - (write_schema, data_storage_version) - } - WriteMode::Overwrite => { - // Overwrite, use the schema from the data. If the user specified - // a storage version use that. Otherwise use the version from the - // dataset. - let data_storage_version = params.data_storage_version.unwrap_or( - dataset - .manifest() - .data_storage_format - .lance_file_version()?, - ); - (normalized_converted_schema, data_storage_version) - } - } + let normalized_converted_schema = prepared_to_logical_blob_schema(&converted_schema)?; + + versions::write_fragments( + storage_version, + dataset, + object_store, + base_dir, + normalized_converted_schema, + data, + params, + target_bases_info, + file_row_counts, + ) + .await +} + +pub(super) fn prepare_write_schema( + dataset: Option<&Dataset>, + normalized_converted_schema: Schema, + params: &WriteParams, + mut schema_compare_options: lance_core::datatypes::SchemaCompareOptions, +) -> Result { + let schema = if let Some(dataset) = dataset + && matches!(params.mode, WriteMode::Append | WriteMode::Create) + { + schema_compare_options.compare_nullability = NullabilityComparison::Ignore; + schema_compare_options.allow_missing_if_nullable = true; + schema_compare_options.ignore_field_order = true; + normalized_converted_schema.check_compatible(dataset.schema(), &schema_compare_options)?; + validate_blob_threshold_metadata_for_append( + &normalized_converted_schema, + dataset.schema(), + )?; + dataset.schema().project_by_schema( + &normalized_converted_schema, + OnMissing::Error, + OnTypeMismatch::Error, + )? } else { - // Brand new dataset, use the schema from the data and the storage version - // from the user or the default. - ( - normalized_converted_schema, - params.storage_version_or_default(), - ) + normalized_converted_schema }; + Ok(schema) +} - if storage_version < LanceFileVersion::V2_2 && schema.fields_pre_order().any(|f| f.is_blob_v2()) - { +pub(super) fn validate_legacy_blob_write_schema( + schema: &Schema, + version_debug: &str, +) -> Result<()> { + if schema.fields_pre_order().any(|field| field.is_blob_v2()) { return Err(Error::invalid_input(format!( - "Blob v2 requires file version >= 2.2 (got {:?})", - storage_version + "Blob v2 requires file version >= 2.2 (got {version_debug})" ))); } + Ok(()) +} - if storage_version >= LanceFileVersion::V2_2 - && let Some(blob_field_path) = legacy_blob_field_path(&schema) - { +pub(super) fn validate_blob_v2_write_schema(schema: &Schema) -> Result<()> { + if let Some(blob_field_path) = legacy_blob_field_path(schema) { return Err(Error::invalid_input(format!( "Legacy blob columns (field metadata key {BLOB_META_KEY:?}) are not supported for file version >= 2.2. Found legacy blob field: {blob_field_path}. Use the blob v2 extension type (ARROW:extension:name = \"lance.blob.v2\") and the new blob APIs (e.g. lance::blob::blob_field / lance::blob::BlobArrayBuilder)." ))); } + Ok(()) +} - let fragments = do_write_fragments( - dataset, - object_store, - base_dir, - &schema, - data, - params, - storage_version, - target_bases_info, - ) - .await?; +pub(crate) async fn create_seed_writers_current( + dataset: Option<&Dataset>, + params: &WriteParams, +) -> Result>> { + // Seeds only make sense when appending to an existing dataset. + if !matches!(params.mode, WriteMode::Append) { + return Ok(Vec::new()); + } + let Some(dataset) = dataset else { + return Ok(Vec::new()); + }; + + let indices: Arc> = dataset.load_indices().await?; + let mut writers: Vec> = Vec::new(); - Ok((fragments, schema)) + for index in indices.iter() { + // A covered index lists its carried columns in `fields` too; the seed + // writer keys on the single keyed column. System indices commit no + // fields at all, so this also skips them. + let Some(field_id) = index.keyed_field() else { + continue; + }; + let Ok(field_path) = dataset.schema().field_path(field_id) else { + continue; + }; + let Some(data_type) = dataset.schema().field(&field_path).map(|f| f.data_type()) else { + continue; + }; + + let Ok(index_details) = fetch_index_details(dataset, &field_path, index).await else { + continue; + }; + let details = IndexDetails(index_details.clone()); + let Ok(plugin) = details.get_plugin() else { + continue; + }; + if let Some(writer) = plugin + .create_seed_writer(&field_path, &data_type, &index_details) + .await? + { + writers.push(writer); + } + } + + Ok(writers) } fn legacy_blob_field_path(schema: &Schema) -> Option { @@ -1367,13 +1617,23 @@ pub trait GenericWriter: Send { async fn tell(&mut self) -> Result; /// Finish writing the file (flush the remaining data and write footer) async fn finish(&mut self) -> Result<(u32, DataFile)>; + + /// Add a global buffer to the current file. Returns the 1-based buffer index. + /// Must be called before `finish`. No-op on legacy (V1) files (returns `Ok(1)`). + async fn add_global_buffer(&mut self, _buffer: Bytes) -> Result { + Ok(1) + } + + /// Add a key-value pair to the file's schema metadata. + /// Must be called before `finish`. No-op on legacy (V1) files. + fn add_schema_metadata(&mut self, _key: String, _value: String) {} } struct V1WriterAdapter where - M: PreviousManifestProvider + Send + Sync, + M: V1ManifestProvider + Send + Sync, { - writer: PreviousFileWriter, + writer: V1FileWriter, path: String, base_id: Option, } @@ -1381,7 +1641,7 @@ where #[async_trait::async_trait] impl GenericWriter for V1WriterAdapter where - M: PreviousManifestProvider + Send + Sync, + M: V1ManifestProvider + Send + Sync, { async fn write(&mut self, batches: &[RecordBatch]) -> Result<()> { self.writer.write(batches).await @@ -1408,8 +1668,7 @@ where struct V2WriterAdapter { writer: current_writer::FileWriter, - path: String, - base_id: Option, + data_file: Option, preprocessor: Option, } @@ -1429,7 +1688,10 @@ impl GenericWriter for V2WriterAdapter { Ok(()) } fn data_file_path(&self) -> (&str, Option) { - (&self.path, self.base_id) + self.data_file + .as_ref() + .map(|data_file| (data_file.path.as_str(), data_file.base_id)) + .unwrap_or(("", None)) } async fn tell(&mut self) -> Result { Ok(self.writer.tell().await?) @@ -1450,75 +1712,28 @@ impl GenericWriter for V2WriterAdapter { .iter() .map(|(_, column_index)| *column_index as i32) .collect::>(); - let (major, minor) = self.writer.version().to_numbers(); let write_summary = self.writer.finish().await?; - let data_file = DataFile::new( - std::mem::take(&mut self.path), - field_ids, - column_indices, - major, - minor, - NonZero::new(write_summary.size_bytes), - self.base_id, - ); + let mut data_file = self + .data_file + .take() + .ok_or_else(|| Error::internal("current writer was already finished"))?; + data_file.fields = field_ids.into(); + data_file.column_indices = column_indices.into(); + data_file.file_size_bytes = NonZero::new(write_summary.size_bytes).into(); Ok((write_summary.num_rows as u32, data_file)) } -} -pub async fn open_writer( - object_store: &ObjectStore, - schema: &Schema, - base_dir: &Path, - storage_version: LanceFileVersion, -) -> Result> { - open_writer_with_options( - object_store, - schema, - base_dir, - storage_version, - WriterOptions { - add_data_dir: true, - ..Default::default() - }, - ) - .await -} - -pub(super) async fn open_update_writer( - dataset: &Dataset, - schema: &Schema, - storage_version: LanceFileVersion, -) -> Result> { - // add_columns / alter_columns reuse the normal writer stack, but they do not - // flow through WriteParams. Rebuild the external base resolver here so blob - // v2 reference columns can resolve dataset-registered external URIs. - let external_base_resolver = if storage_version >= LanceFileVersion::V2_2 - && schema.fields_pre_order().any(|f| f.is_blob_v2()) - { - Some(Arc::new( - build_external_base_resolver(Some(dataset), &WriteParams::default()).await?, - )) - } else { - None - }; + async fn add_global_buffer(&mut self, buffer: Bytes) -> Result { + self.writer.add_global_buffer(buffer).await + } - open_writer_with_options( - &dataset.object_store, - schema, - &dataset.base, - storage_version, - WriterOptions { - add_data_dir: true, - external_base_resolver, - source_store_registry: dataset.session.store_registry(), - ..Default::default() - }, - ) - .await + fn add_schema_metadata(&mut self, key: String, value: String) { + self.writer.add_schema_metadata(key, value); + } } #[derive(Default)] -struct WriterOptions { +pub(crate) struct WriterOptions { add_data_dir: bool, base_id: Option, external_base_resolver: Option>, @@ -1529,13 +1744,94 @@ struct WriterOptions { blob_pack_file_size_threshold: Option, } -async fn open_writer_with_options( +impl WriterOptions { + pub(super) fn update( + source_store_registry: Arc, + external_base_resolver: Option>, + allow_external_blob_outside_bases: bool, + ) -> Self { + Self { + add_data_dir: true, + external_base_resolver, + allow_external_blob_outside_bases, + source_store_registry, + ..Default::default() + } + } +} + +pub(crate) async fn open_v1_writer( object_store: &ObjectStore, schema: &Schema, base_dir: &Path, - storage_version: LanceFileVersion, options: WriterOptions, ) -> Result> { + let WriterOptions { + add_data_dir, + base_id, + .. + } = options; + let (_data_file_key, filename, _data_dir, full_path) = + prepare_data_file_path(base_dir, add_data_dir); + Ok(Box::new(V1WriterAdapter { + writer: V1FileWriter::::try_new( + object_store, + &full_path, + schema.clone(), + &Default::default(), + ) + .await?, + path: filename, + base_id, + })) +} + +pub(in crate::dataset) async fn open_current_writer( + create_file_writer: F, + object_store: &ObjectStore, + schema: &Schema, + base_dir: &Path, + options: WriterOptions, +) -> Result> +where + F: FnOnce( + Box, + Schema, + String, + Option, + ) -> Result<(current_writer::FileWriter, DataFile)>, +{ + let WriterOptions { + add_data_dir, + base_id, + .. + } = options; + let (_data_file_key, filename, _data_dir, full_path) = + prepare_data_file_path(base_dir, add_data_dir); + let writer = object_store.create(&full_path).await?; + let (file_writer, data_file) = create_file_writer(writer, schema.clone(), filename, base_id)?; + Ok(Box::new(V2WriterAdapter { + writer: file_writer, + data_file: Some(data_file), + preprocessor: None, + })) +} + +pub(in crate::dataset) async fn open_current_blob_v2_writer( + create_file_writer: F, + object_store: &ObjectStore, + schema: &Schema, + base_dir: &Path, + options: WriterOptions, +) -> Result> +where + F: FnOnce( + Box, + Schema, + String, + Option, + ) -> Result<(current_writer::FileWriter, DataFile)>, +{ let WriterOptions { add_data_dir, base_id, @@ -1546,66 +1842,39 @@ async fn open_writer_with_options( source_store_params, blob_pack_file_size_threshold, } = options; + let (data_file_key, filename, data_dir, full_path) = + prepare_data_file_path(base_dir, add_data_dir); + let writer = object_store.create(&full_path).await?; + let (file_writer, data_file) = create_file_writer(writer, schema.clone(), filename, base_id)?; + let preprocessor = BlobPreprocessor::new( + object_store.clone(), + data_dir, + data_file_key, + schema, + external_base_resolver, + allow_external_blob_outside_bases, + external_blob_mode, + source_store_registry, + source_store_params, + blob_pack_file_size_threshold, + )?; + Ok(Box::new(V2WriterAdapter { + writer: file_writer, + data_file: Some(data_file), + preprocessor: Some(preprocessor), + })) +} +fn prepare_data_file_path(base_dir: &Path, add_data_dir: bool) -> (String, String, Path, Path) { let data_file_key = generate_random_filename(); let filename = format!("{}.lance", data_file_key); - let data_dir = if add_data_dir { base_dir.clone().join(DATA_DIR) } else { base_dir.clone() }; - let full_path = data_dir.clone().join(filename.as_str()); - - let writer = if storage_version == LanceFileVersion::Legacy { - Box::new(V1WriterAdapter { - writer: PreviousFileWriter::::try_new( - object_store, - &full_path, - schema.clone(), - &Default::default(), - ) - .await?, - path: filename, - base_id, - }) - } else { - let writer = object_store.create(&full_path).await?; - let enable_blob_v2 = storage_version >= LanceFileVersion::V2_2; - let file_writer = current_writer::FileWriter::try_new( - writer, - schema.clone(), - FileWriterOptions { - format_version: Some(storage_version), - ..Default::default() - }, - )?; - let preprocessor = if enable_blob_v2 { - Some(BlobPreprocessor::new( - object_store.clone(), - data_dir.clone(), - data_file_key.clone(), - schema, - external_base_resolver, - allow_external_blob_outside_bases, - external_blob_mode, - source_store_registry, - source_store_params, - blob_pack_file_size_threshold, - )?) - } else { - None - }; - let writer_adapter = V2WriterAdapter { - writer: file_writer, - path: filename, - base_id, - preprocessor, - }; - Box::new(writer_adapter) as Box - }; - Ok(writer) + (data_file_key, filename, data_dir, full_path) } /// Reserved base id that refers to the dataset's primary storage in @@ -1630,13 +1899,13 @@ pub struct TargetBaseInfo { pub is_dataset_root: bool, } -struct WriterGenerator { +struct WriterGenerator { /// Default object store (used when no target bases specified) object_store: Arc, /// Default base directory (used when no target bases specified) base_dir: Path, schema: Schema, - storage_version: LanceFileVersion, + open_writer: OpenWriter, /// Target base information (if writing to specific bases) target_bases_info: Option>, external_base_resolver: Option>, @@ -1649,13 +1918,17 @@ struct WriterGenerator { next_base_index: AtomicUsize, } -impl WriterGenerator { +impl WriterGenerator +where + OpenWriter: Fn(Arc, Schema, Path, WriterOptions) -> OpenWriterFuture + Send + Sync, + OpenWriterFuture: Future>> + Send, +{ #[allow(clippy::too_many_arguments)] pub fn new( object_store: Arc, base_dir: &Path, schema: &Schema, - storage_version: LanceFileVersion, + open_writer: OpenWriter, target_bases_info: Option>, external_base_resolver: Option>, allow_external_blob_outside_bases: bool, @@ -1668,7 +1941,7 @@ impl WriterGenerator { object_store, base_dir: base_dir.clone(), schema: schema.clone(), - storage_version, + open_writer, target_bases_info, external_base_resolver, allow_external_blob_outside_bases, @@ -1696,11 +1969,10 @@ impl WriterGenerator { let fragment = Fragment::new(0); let writer = if let Some(base_info) = self.select_target_base() { - open_writer_with_options( - &base_info.object_store, - &self.schema, - &base_info.base_dir, - self.storage_version, + (self.open_writer)( + base_info.object_store.clone(), + self.schema.clone(), + base_info.base_dir.clone(), WriterOptions { add_data_dir: base_info.is_dataset_root, // Primary-storage slots stamp no base id, like a write @@ -1716,11 +1988,10 @@ impl WriterGenerator { ) .await? } else { - open_writer_with_options( - &self.object_store, - &self.schema, - &self.base_dir, - self.storage_version, + (self.open_writer)( + self.object_store.clone(), + self.schema.clone(), + self.base_dir.clone(), WriterOptions { add_data_dir: true, base_id: None, @@ -1771,123 +2042,80 @@ async fn resolve_commit_handler( } } -/// Create an iterator of record batch streams from the given source. -/// -/// If `enable_retries` is true, then the source will be saved either in memory -/// or spilled to disk to allow replaying the source in case of a failure. The -/// source will be kept in memory if either (1) the size hint shows that -/// there is only one batch or (2) the stream contains less than 100MB of -/// data. Otherwise, the source will be spilled to a temporary file on disk. -/// -/// This is used to support retries on write operations. -async fn new_source_iter( - source: SendableRecordBatchStream, - enable_retries: bool, -) -> Result + Send + 'static>> { - if enable_retries { - let schema = source.schema(); - - // If size hint shows there is only one batch, spilling has no benefit, just keep that - // in memory. (This is a pretty common case.) - let size_hint = source.size_hint(); - if size_hint.0 == 1 && size_hint.1 == Some(1) { - let batches: Vec = source.try_collect().await?; - Ok(Box::new(std::iter::repeat_with(move || { - Box::pin(RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(batches.clone().into_iter().map(Ok)), - )) as SendableRecordBatchStream - }))) - } else { - // Allow buffering up to 100MB in memory before spilling to disk. - Ok(Box::new( - SpillStreamIter::try_new(source, 100 * 1024 * 1024).await?, - )) - } - } else { - Ok(Box::new(std::iter::once(source))) - } -} - -struct SpillStreamIter { - receiver: SpillReceiver, - _sender_handle: tokio::task::JoinHandle, - // This temp dir is used to store the spilled data. It is kept alive by - // this struct. When this struct is dropped, the Drop implementation of - // tempfile::TempDir will delete the temp dir. - _tmp_dir: TempDir, -} - -impl SpillStreamIter { - pub async fn try_new( - mut source: SendableRecordBatchStream, - memory_limit: usize, - ) -> Result { - let tmp_dir = tokio::task::spawn_blocking(|| { - TempDir::try_new() - .map_err(|e| Error::invalid_input(format!("Failed to create temp dir: {}", e))) - }) - .await - .ok() - .expect_ok()??; - - let tmp_path = tmp_dir.std_path().join("spill.arrows"); - let (mut sender, receiver) = create_replay_spill(tmp_path, source.schema(), memory_limit); - - let sender_handle = tokio::task::spawn(async move { - while let Some(res) = source.next().await { - match res { - Ok(batch) => match sender.write(batch).await { - Ok(_) => {} - Err(e) => { - sender.send_error(e); - break; - } - }, - Err(e) => { - sender.send_error(e); - break; - } - } - } - - if let Err(err) = sender.finish().await { - sender.send_error(err); - } - sender - }); - - Ok(Self { - receiver, - _tmp_dir: tmp_dir, - _sender_handle: sender_handle, - }) - } -} - -impl Iterator for SpillStreamIter { - type Item = SendableRecordBatchStream; - - fn next(&mut self) -> Option { - Some(self.receiver.read()) - } -} - #[cfg(test)] mod tests { use super::*; use std::collections::HashMap; + #[cfg(windows)] + use std::path::{Component, Prefix}; - use arrow_array::{Int32Array, RecordBatchIterator, RecordBatchReader, StructArray}; + use arrow_array::{ + Int32Array, LargeBinaryArray, RecordBatchIterator, RecordBatchReader, StructArray, + }; use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; use datafusion::{error::DataFusionError, physical_plan::stream::RecordBatchStreamAdapter}; use datafusion_physical_plan::RecordBatchStream; use futures::TryStreamExt; + use lance_datafusion::chunker::chunk_stream; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; - use lance_file::previous::reader::FileReader as PreviousFileReader; + use lance_file::version::ConcreteFileVersion; + use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_io::object_store::StorageOptionsAccessor; use lance_io::traits::Reader; use lance_table::format::BasePath; + use rstest::rstest; + + async fn open_v2_1_test_writer( + object_store: Arc, + schema: Schema, + base_dir: Path, + options: WriterOptions, + ) -> Result> { + open_current_writer( + |object_writer, schema, filename, base_id| { + let writer = lance_file::versions::v2_1::create_writer( + object_writer, + schema, + lance_file::writer::FileWriterOptions::default(), + )? + .into(); + let mut data_file = DataFile::new_unstarted(filename, ConcreteFileVersion::V2_1); + data_file.base_id = base_id; + Ok((writer, data_file)) + }, + &object_store, + &schema, + &base_dir, + options, + ) + .await + } + + async fn scan_sorted_ids(dataset: &Dataset) -> Vec { + let batches = dataset + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut ids: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + ids.sort_unstable(); + ids + } #[test] fn test_auto_cleanup_disabled_by_default() { @@ -1899,6 +2127,51 @@ mod tests { assert!(!params.skip_auto_cleanup); } + #[cfg(windows)] + #[tokio::test] + async fn test_create_and_reopen_from_unc_uri() { + let tempdir = tempfile::tempdir().unwrap(); + let dataset_path = tempdir.path().join("dataset with spaces"); + let mut components = dataset_path.components(); + let drive_letter = match components.next() { + Some(Component::Prefix(prefix)) => match prefix.kind() { + Prefix::Disk(letter) | Prefix::VerbatimDisk(letter) => letter, + other => panic!("expected a disk path, found {other:?}"), + }, + other => panic!("expected a disk path, found {other:?}"), + }; + assert!(matches!(components.next(), Some(Component::RootDir))); + + // The administrative disk share provides a real loopback UNC path without + // requiring an external SMB service. + let computer_name = std::env::var("COMPUTERNAME").unwrap(); + let mut dataset_uri = url::Url::parse(&format!("file://{computer_name}/")).unwrap(); + let share = format!("{}$", char::from(drive_letter)); + { + let mut segments = dataset_uri.path_segments_mut().unwrap(); + segments.pop_if_empty().push(&share); + for component in components { + let Component::Normal(segment) = component else { + panic!("unexpected dataset path component: {component:?}"); + }; + segments.push(segment.to_str().unwrap()); + } + } + assert!(dataset_uri.as_str().contains("%20")); + + let reader = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(3), BatchCount::from(1)); + let dataset = Dataset::write(reader, dataset_uri.as_str(), None) + .await + .unwrap(); + assert_eq!(dataset.count_rows(None).await.unwrap(), 3); + drop(dataset); + + let reopened = Dataset::open(dataset_uri.as_str()).await.unwrap(); + assert_eq!(reopened.count_rows(None).await.unwrap(), 3); + } + #[tokio::test] async fn test_chunking_large_batches() { // Create a stream of 3 batches of 10 rows @@ -2004,6 +2277,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test"), @@ -2033,6 +2307,138 @@ mod tests { assert_eq!(fragments.len(), 2); } + #[rstest] + #[case::rebalance_pending_remainder( + &[9_999, 10_001], + &[10_000, 10_000], + 2 * 1024, + &[9_999, 5_001, 5_000] + )] + #[case::replan_pending_boundary( + &[9_999, 1, 10_000, 10_000], + &[20_000, 10_000], + 100 * 1024, + &[9_999, 10_001, 10_000] + )] + #[tokio::test] + async fn test_planned_file_boundary_with_byte_limit( + #[case] input_batch_sizes: &[usize], + #[case] file_row_counts: &[usize], + #[case] max_bytes_per_file: usize, + #[case] expected_file_rows: &[usize], + ) { + let value = vec![0_u8; 1024]; + let arrow_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "a", + DataType::LargeBinary, + false, + )])); + let total_rows = input_batch_sizes.iter().sum::(); + let data = RecordBatch::try_new( + arrow_schema.clone(), + vec![Arc::new(LargeBinaryArray::from_iter_values( + (0..total_rows).map(|_| value.as_slice()), + ))], + ) + .unwrap(); + let mut offset = 0; + let batches = input_batch_sizes + .iter() + .map(|&batch_rows| { + let batch = data.slice(offset, batch_rows); + offset += batch_rows; + Ok::<_, DataFusionError>(batch) + }) + .collect::>(); + let stream = + RecordBatchStreamAdapter::new(arrow_schema.clone(), futures::stream::iter(batches)); + let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); + let object_store = Arc::new(ObjectStore::memory()); + + let (fragments, _) = write_fragments_internal_with_file_row_counts( + ConcreteFileVersion::V2_0, + None, + object_store, + &Path::from("planned_byte_boundary"), + schema, + Box::pin(stream), + WriteParams { + max_rows_per_file: file_row_counts[0], + max_bytes_per_file, + mode: WriteMode::Create, + ..Default::default() + }, + None, + Some(file_row_counts.to_vec()), + ) + .await + .unwrap(); + + assert_eq!( + fragments + .iter() + .map(|fragment| fragment.physical_rows.unwrap()) + .collect::>(), + expected_file_rows + ); + } + + #[tokio::test] + async fn test_repeated_byte_closes_rebalance_planned_rows() { + let large_value = vec![0_u8; 16 * 1024 * 1024]; + let mut values = Vec::with_capacity(15); + values.extend(std::iter::repeat_n(large_value.as_slice(), 2)); + values.extend(std::iter::repeat_n(&[][..], 13)); + let arrow_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "a", + DataType::LargeBinary, + false, + )])); + let data = RecordBatch::try_new( + arrow_schema.clone(), + vec![Arc::new(LargeBinaryArray::from_iter_values(values))], + ) + .unwrap(); + let input_batch_sizes = [1, 1, 3, 5, 5]; + let mut offset = 0; + let batches = input_batch_sizes.map(|batch_rows| { + let batch = data.slice(offset, batch_rows); + offset += batch_rows; + Ok::<_, DataFusionError>(batch) + }); + let stream = + RecordBatchStreamAdapter::new(arrow_schema.clone(), futures::stream::iter(batches)); + let schema = Schema::try_from(arrow_schema.as_ref()).unwrap(); + let object_store = Arc::new(ObjectStore::memory()); + + let (fragments, _) = write_fragments_internal_with_file_row_counts( + ConcreteFileVersion::V2_0, + None, + object_store, + &Path::from("repeated_planned_byte_boundaries"), + schema, + Box::pin(stream), + WriteParams { + max_rows_per_file: 5, + max_bytes_per_file: 100 * 1024, + mode: WriteMode::Create, + ..Default::default() + }, + None, + Some(vec![5, 5, 5]), + ) + .await + .unwrap(); + + assert_eq!( + fragments + .iter() + .map(|fragment| fragment.physical_rows.unwrap()) + .collect::>(), + [1, 1, 5, 4, 4] + ); + } + #[tokio::test] async fn test_max_rows_per_file() { let reader_to_frags = |data_reader: Box| { @@ -2057,6 +2463,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test"), @@ -2118,6 +2525,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test"), @@ -2214,7 +2622,8 @@ mod tests { LanceFileVersion::Next, ]; for version in versions { - let (major, minor) = version.to_numbers(); + let (major, minor) = version.resolve().to_data_file_numbers(); + let write_params = WriteParams { data_storage_version: Some(version), // This parameter should be ignored @@ -2231,6 +2640,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); let (fragments, _) = write_fragments_internal( + version.resolve(), None, object_store, &Path::from("test"), @@ -2306,6 +2716,7 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); let base_path = Path::from("test"); let (fragments, _) = write_fragments_internal( + ConcreteFileVersion::V1, None, object_store.clone(), &base_path, @@ -2327,7 +2738,7 @@ mod tests { .join(DATA_DIR) .join(fragment.files[0].path.as_str()); let file_reader: Arc = object_store.open(&path).await.unwrap().into(); - let reader = PreviousFileReader::try_new_from_reader( + let reader = V1FileReader::try_new_from_reader( &path, file_reader, None, @@ -2515,7 +2926,7 @@ mod tests { object_store.clone(), &base_dir, &schema, - LanceFileVersion::Stable, + open_v2_1_test_writer, Some(target_bases), None, false, @@ -2561,11 +2972,11 @@ mod tests { let object_store = Arc::new(ObjectStore::memory()); let base_dir = Path::from("test/bucket2"); - let mut inner_writer = open_writer_with_options( + let mut inner_writer = versions::open_writer( + LanceFileVersion::Stable.resolve(), &object_store, &schema, &base_dir, - LanceFileVersion::Stable, WriterOptions { add_data_dir: false, // Don't add /data ..Default::default() @@ -2633,7 +3044,7 @@ mod tests { Arc::new(ObjectStore::memory()), &Path::from("default"), &schema, - LanceFileVersion::Stable, + open_v2_1_test_writer, Some(target_bases), None, false, @@ -2785,7 +3196,7 @@ mod tests { use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; // Create dataset with multi-base configuration - let test_uri = "memory://multi_base_test"; + let test_uri = "shared-memory://multi_base_test"; let primary_uri = format!("{}/primary", test_uri); let base1_uri = format!("{}/base1", test_uri); let base2_uri = format!("{}/base2", test_uri); @@ -2852,6 +3263,10 @@ mod tests { ); } + assert_eq!(scan_sorted_ids(&dataset).await, (0..5).collect::>()); + let reopened = Dataset::open(&primary_uri).await.unwrap(); + assert_eq!(scan_sorted_ids(&reopened).await, (0..5).collect::>()); + // Test validation: cannot specify both target_bases and target_base_names_or_paths let mut data_gen2 = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -2962,7 +3377,7 @@ mod tests { use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; // Create initial dataset - let test_uri = "memory://overwrite_test"; + let test_uri = "shared-memory://overwrite_test"; let primary_uri = format!("{}/primary", test_uri); let base1_uri = format!("{}/base1", test_uri); let base2_uri = format!("{}/base2", test_uri); @@ -3045,6 +3460,9 @@ mod tests { .all(|f| f.metadata.files.iter().all(|file| file.base_id == Some(2))) ); + let reopened = Dataset::open(&primary_uri).await.unwrap(); + assert_eq!(scan_sorted_ids(&reopened).await, (0..2).collect::>()); + // Test validation: cannot specify initial_bases in OVERWRITE mode let mut data_gen3 = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -3080,7 +3498,7 @@ mod tests { use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; // Create initial dataset with multi-base configuration - let test_uri = "memory://append_test"; + let test_uri = "shared-memory://append_test"; let primary_uri = format!("{}/primary", test_uri); let base1_uri = format!("{}/base1", test_uri); let base2_uri = format!("{}/base2", test_uri); @@ -3167,6 +3585,11 @@ mod tests { assert!(has_base1_data, "Should have data in base1"); assert!(has_base2_data, "Should have data in base2"); + let mut expected: Vec = (0..3).chain(0..2).chain(0..4).collect(); + expected.sort_unstable(); + let reopened = Dataset::open(&primary_uri).await.unwrap(); + assert_eq!(scan_sorted_ids(&reopened).await, expected); + // Test validation: cannot specify initial_bases in APPEND mode let mut data_gen4 = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -3467,6 +3890,7 @@ mod tests { }; let result = write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test_empty"), @@ -3691,6 +4115,7 @@ mod tests { // Attempt to write data - should fail with IO error due to disk full let result = write_fragments_internal( + write_params.storage_version_or_default(), None, object_store, &Path::from("test_disk_full"), @@ -3733,6 +4158,7 @@ mod tests { async fn test_write_interruption_recovery() { use super::commit::CommitBuilder; use arrow_array::record_batch; + use lance_core::utils::tempfile::TempDir; // Create a temporary directory for testing let temp_dir = TempDir::default(); @@ -3871,14 +4297,16 @@ mod tests { futures::stream::iter(items), )); - let result = do_write_fragments( + let result = versions::write_fragments_direct( + ConcreteFileVersion::V2_1, None, object_store.clone(), &base_dir, &schema, stream, WriteParams::default(), - LanceFileVersion::V2_1, + None, + Vec::new(), None, ) .await; @@ -3928,7 +4356,8 @@ mod tests { futures::stream::iter(items), )); - let result = do_write_fragments( + let result = versions::write_fragments_direct( + ConcreteFileVersion::V2_1, None, object_store.clone(), &base_dir, @@ -3938,7 +4367,8 @@ mod tests { max_rows_per_file: 3, ..Default::default() }, - LanceFileVersion::V2_1, + None, + Vec::new(), None, ) .await; @@ -3974,12 +4404,14 @@ mod tests { // Sanity check: file is on disk. assert_eq!(count_data_files(test_uri), 1); - let mut external_file = DataFile::new_unstarted("external.lance", 2, 1); + let mut external_file = + DataFile::new_unstarted("external.lance", ConcreteFileVersion::V2_1); external_file.base_id = Some(42); - let local_file = DataFile::new_unstarted(local_filename, 2, 1); + let local_file = DataFile::new_unstarted(local_filename, ConcreteFileVersion::V2_1); let fragments = vec![Fragment { id: 0, files: vec![external_file, local_file], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(0), @@ -4051,15 +4483,16 @@ mod tests { assert_eq!(count_data_files(base1_dir.as_str()), 1); assert_eq!(count_plain_files(base2_dir.as_str()), 1); - let mut base1_file = DataFile::new_unstarted("one.lance", 2, 1); + let mut base1_file = DataFile::new_unstarted("one.lance", ConcreteFileVersion::V2_1); base1_file.base_id = Some(1); - let mut base2_file = DataFile::new_unstarted("two.lance", 2, 1); + let mut base2_file = DataFile::new_unstarted("two.lance", ConcreteFileVersion::V2_1); base2_file.base_id = Some(2); - let mut unknown_file = DataFile::new_unstarted("unknown.lance", 2, 1); + let mut unknown_file = DataFile::new_unstarted("unknown.lance", ConcreteFileVersion::V2_1); unknown_file.base_id = Some(42); let fragments = vec![Fragment { id: 0, files: vec![base1_file, base2_file, unknown_file], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(0), @@ -4152,7 +4585,8 @@ mod tests { is_dataset_root: true, }]; - let result = do_write_fragments( + let result = versions::write_fragments_direct( + ConcreteFileVersion::V2_1, None, object_store.clone(), &base_dir, @@ -4162,8 +4596,9 @@ mod tests { max_rows_per_file: 3, ..Default::default() }, - LanceFileVersion::V2_1, Some(target_bases), + vec![], + None, ) .await; @@ -4182,7 +4617,7 @@ mod tests { async fn test_multi_base_target_primary_and_bases() { use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; - let test_uri = "memory://primary_slot_test"; + let test_uri = "shared-memory://primary_slot_test"; let primary_uri = format!("{}/primary", test_uri); let base1_uri = format!("{}/base1", test_uri); let base2_uri = format!("{}/base2", test_uri); @@ -4274,6 +4709,11 @@ mod tests { assert_eq!(file_bases, vec![None, Some(2)]); assert_eq!(dataset.count_rows(None).await.unwrap(), 21); + + let mut expected: Vec = (0..6).chain(0..9).chain(0..6).collect(); + expected.sort_unstable(); + let reopened = Dataset::open(&primary_uri).await.unwrap(); + assert_eq!(scan_sorted_ids(&reopened).await, expected); } /// `target_all_bases` resolves to every registered base at execution @@ -4282,7 +4722,7 @@ mod tests { async fn test_multi_base_target_all_bases() { use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; - let test_uri = "memory://all_bases_test"; + let test_uri = "shared-memory://all_bases_test"; let primary_uri = format!("{}/primary", test_uri); let base1_uri = format!("{}/base1", test_uri); let base2_uri = format!("{}/base2", test_uri); @@ -4364,6 +4804,11 @@ mod tests { .collect(); assert_eq!(file_bases, vec![Some(1), Some(2)]); + let mut expected: Vec = (0..3).chain(0..9).chain(0..6).collect(); + expected.sort_unstable(); + let reopened = Dataset::open(&primary_uri).await.unwrap(); + assert_eq!(scan_sorted_ids(&reopened).await, expected); + // Cannot be combined with explicit target bases. let mut data_gen4 = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); @@ -4389,7 +4834,7 @@ mod tests { // On a dataset with no registered bases: include_primary=true is a // no-op rotation over primary, false is rejected. - let plain_uri = "memory://all_bases_plain"; + let plain_uri = "shared-memory://all_bases_plain/primary"; let mut data_gen5 = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); let plain = Dataset::write(data_gen5.batch(3), plain_uri, None) @@ -4440,12 +4885,13 @@ mod tests { // CREATE mode: initial_bases join the rotation before their ids are // committed to a manifest. - let create_uri = "memory://all_bases_create"; + let create_root = "shared-memory://all_bases_create"; + let create_uri = format!("{}/primary", create_root); let mut data_gen8 = BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("id".to_owned()))); let dataset = Dataset::write( data_gen8.batch(9), - create_uri, + &create_uri, Some( WriteParams { mode: WriteMode::Create, @@ -4455,13 +4901,13 @@ mod tests { id: 0, name: Some("base1".to_string()), is_dataset_root: true, - path: format!("{}/base1", create_uri), + path: format!("{}/base1", create_root), }, BasePath { id: 0, name: Some("base2".to_string()), is_dataset_root: false, - path: format!("{}/base2", create_uri), + path: format!("{}/base2", create_root), }, ]), ..Default::default() @@ -4478,5 +4924,198 @@ mod tests { .flat_map(|f| f.metadata.files.iter().map(|file| file.base_id)) .collect(); assert_eq!(file_bases, vec![None, Some(1), Some(2)]); + let reopened = Dataset::open(&create_uri).await.unwrap(); + assert_eq!(scan_sorted_ids(&reopened).await, (0..9).collect::>()); + } + + #[tokio::test] + async fn test_zone_map_seeds_used_during_update() { + use crate::Dataset; + use crate::index::DatasetIndexExt; + use crate::index::scalar::open_scalar_index; + use arrow::datatypes::Int32Type; + use lance_datagen::{BatchCount, RowCount}; + use lance_datagen::{array, gen_batch}; + use lance_file::reader::FileReaderOptions; + use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::seed::SEED_META_KEY_PREFIX; + use lance_index::{IndexType, scalar::ScalarIndexParams}; + use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; + use lance_io::utils::CachedFileSize; + + let tmpdir = lance_core::utils::tempfile::TempStrDir::default(); + let uri = tmpdir.as_str(); + + // Step 1: Create initial dataset + let reader = gen_batch() + .col("val", array::step::()) + .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, uri, None).await.unwrap(); + + // Step 2: Create a zone map index with seeds explicitly enabled (Int32 defaults to off). + let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::ZoneMap) + .with_params(&serde_json::json!({"use_seeds": true})); + dataset + .create_index(&["val"], IndexType::ZoneMap, None, ¶ms, false) + .await + .unwrap(); + // Step 3: Append new data - seeds should be written automatically + let reader = gen_batch() + .col("val", array::step::()) + .into_reader_rows(RowCount::from(50), BatchCount::from(1)); + let dataset = Dataset::write( + reader, + uri, + Some(WriteParams { + mode: WriteMode::Append, + data_storage_version: Some(lance_file::version::LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Step 4: Verify that the newly appended fragment has a seed embedded + let fragments = dataset.fragments(); + let new_fragment = fragments.last().unwrap(); + let data_file = new_fragment.files.first().unwrap(); + + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + let path = dataset + .base + .clone() + .join(super::DATA_DIR) + .join(data_file.path.as_str()); + let file_scheduler = scheduler + .open_file(&path, &CachedFileSize::unknown()) + .await + .unwrap(); + let reader = lance_file::reader::FileReader::try_open( + file_scheduler, + None, + Default::default(), + &dataset.metadata_cache.file_metadata_cache(&path), + FileReaderOptions::default(), + ) + .await + .unwrap(); + + let meta_key = format!("{}val", SEED_META_KEY_PREFIX); + let has_seed = reader + .metadata() + .file_schema + .metadata + .contains_key(&meta_key); + assert!( + has_seed, + "Newly appended fragment should have a zone map seed in metadata" + ); + + // Step 5: Optimize the index (should use seeds) + let mut dataset = Dataset::open(uri).await.unwrap(); + dataset.optimize_indices(&Default::default()).await.unwrap(); + + // Step 6: Query the updated index to verify it's correct + let dataset = Dataset::open(uri).await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + assert!( + !indices.is_empty(), + "Dataset should still have an index after optimization" + ); + + // Verify the index is a ZoneMap and covers all fragments + let index = indices.iter().find(|i| i.name.contains("val")).unwrap(); + let scalar_index = open_scalar_index(&dataset, "val", index, &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!( + scalar_index.index_type(), + IndexType::ZoneMap, + "Index should still be a ZoneMap after optimization" + ); + let frags = scalar_index.calculate_included_frags().await.unwrap(); + assert_eq!(frags.len(), 2, "Index should cover both fragments"); + } + + /// A covered scalar index must still get a seed writer. The loop skipped any + /// index with more than one field, silently dropping seed writing for it. + #[tokio::test] + async fn test_seed_writers_for_a_covered_index() { + use crate::dataset::transaction::{Operation, Transaction}; + use lance_core::utils::tempfile::TempStrDir; + use lance_index::{IndexType, scalar::ScalarIndexParams}; + + let test_uri = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + ArrowField::new("payload", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..256)), + Arc::new(Int32Array::from_iter_values(0..256)), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, &test_uri, None).await.unwrap(); + + // BTree (the default `ScalarIndexParams`) never writes seeds -- only ZoneMap + // does, and only when `use_seeds` is explicitly requested for a fixed-width + // type like Int32 (see `test_zone_map_seeds_used_during_update` above). + let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::ZoneMap) + .with_params(&serde_json::json!({"use_seeds": true})); + dataset + .create_index(&["id"], IndexType::ZoneMap, None, ¶ms, true) + .await + .unwrap(); + + let append_params = WriteParams { + mode: WriteMode::Append, + ..Default::default() + }; + + let baseline = create_seed_writers_current(Some(&dataset), &append_params) + .await + .unwrap(); + assert!( + !baseline.is_empty(), + "a plain scalar index should produce a seed writer; if this is empty \ + the rest of the test proves nothing" + ); + + // Declare `payload` as carried, then re-check. + let id_field = dataset.schema().field_id("id").unwrap(); + let payload_field = dataset.schema().field_id("payload").unwrap(); + let current = dataset.load_indices().await.unwrap(); + let mut covered = current[0].clone(); + covered.fields = vec![id_field, payload_field]; + covered.covering_fields = vec![payload_field]; + + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![covered], + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let covered_writers = create_seed_writers_current(Some(&dataset), &append_params) + .await + .unwrap(); + assert_eq!( + covered_writers.len(), + baseline.len(), + "a covered index must still produce a seed writer" + ); } } diff --git a/rust/lance/src/dataset/write/commit.rs b/rust/lance/src/dataset/write/commit.rs index d76c2049873..876673596ed 100644 --- a/rust/lance/src/dataset/write/commit.rs +++ b/rust/lance/src/dataset/write/commit.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; use lance_io::object_store::{ObjectStore, ObjectStoreParams}; use lance_select::RowAddrTreeMap; use lance_table::{ @@ -13,6 +13,7 @@ use lance_table::{ io::commit::{CommitConfig, CommitHandler, ManifestNamingScheme}, }; +use crate::io::commit::DEFAULT_COMMIT_RETRY_TIMEOUT; use crate::{ Dataset, Error, Result, dataset::{ @@ -39,16 +40,21 @@ pub struct CommitBuilder<'a> { dest: WriteDestination<'a>, use_stable_row_ids: Option, enable_v2_manifest_paths: bool, - storage_format: Option, + storage_format: Option, commit_handler: Option>, store_params: Option, object_store: Option>, + source_store: Option>, + source_commit_handler: Option>, session: Option>, detached: bool, commit_config: CommitConfig, + retry_timeout: Duration, affected_rows: Option, transaction_properties: Option>>, timeout: Option, + /// When `Some`, this commit is the second step of `migrate_to_stable_row_ids`. + migration_next_row_id: Option, } /// Default timeout applied to [`CommitBuilder::execute`] when none is set. @@ -64,12 +70,16 @@ impl<'a> CommitBuilder<'a> { commit_handler: None, store_params: None, object_store: None, + source_store: None, + source_commit_handler: None, session: None, detached: false, commit_config: Default::default(), + retry_timeout: DEFAULT_COMMIT_RETRY_TIMEOUT, affected_rows: None, transaction_properties: None, timeout: Some(DEFAULT_COMMIT_TIMEOUT), + migration_next_row_id: None, } } @@ -93,6 +103,12 @@ impl<'a> CommitBuilder<'a> { /// All data files must use the same storage format as the existing dataset. /// If a different format is passed, an error will be returned. pub fn with_storage_format(mut self, storage_format: LanceFileVersion) -> Self { + self.storage_format = Some(storage_format.resolve()); + + self + } + + pub(crate) fn with_exact_storage_format(mut self, storage_format: ConcreteFileVersion) -> Self { self.storage_format = Some(storage_format); self } @@ -103,6 +119,29 @@ impl<'a> CommitBuilder<'a> { self } + /// Pass the object store of the dataset being cloned from. + /// + /// Only used by `Operation::Clone`: the source manifest is read through this store + /// while the new dataset is written through the destination store. This lets a clone + /// cross object stores/accounts (e.g. between two Azure accounts), where the source + /// is not reachable with the destination's credentials. Defaults to the destination + /// store when not set, preserving same-store behavior. + pub fn with_source_store(mut self, source_store: Arc) -> Self { + self.source_store = Some(source_store); + self + } + + /// Pass the dataset being cloned from. + /// + /// Only used by `Operation::Clone`: the source manifest is resolved through + /// the dataset's commit handler and read through its object store. This is + /// required when the source and destination use different manifest stores. + pub fn with_source_dataset(mut self, source: &Dataset) -> Self { + self.source_store = Some(source.object_store.clone()); + self.source_commit_handler = Some(source.commit_handler.clone()); + self + } + /// Pass a commit handler to use for the dataset. /// /// Takes precedence over the destination dataset's own handler. If not @@ -167,6 +206,26 @@ impl<'a> CommitBuilder<'a> { self } + /// Set the wall-clock budget used by commit conflict backoff. + /// + /// The first commit attempt is always allowed to complete. If it conflicts, + /// each backoff sleep is bounded by the time remaining in this budget. The + /// default is 30 seconds. + /// + /// # Examples + /// + /// ``` + /// use std::time::Duration; + /// use lance::dataset::CommitBuilder; + /// + /// let _builder = CommitBuilder::new("memory://dataset") + /// .with_retry_timeout(Duration::from_secs(10)); + /// ``` + pub fn with_retry_timeout(mut self, retry_timeout: Duration) -> Self { + self.retry_timeout = retry_timeout; + self + } + pub fn with_skip_auto_cleanup(mut self, skip_auto_cleanup: bool) -> Self { self.commit_config.skip_auto_cleanup = skip_auto_cleanup; self @@ -209,6 +268,17 @@ impl<'a> CommitBuilder<'a> { self } + /// Configure this commit as the second step of a stable row ID migration. + /// + /// Sets `use_stable_row_ids = true` and supplies the `next_row_id` that was + /// computed during the first migration commit. This bypasses the normal + /// "cannot enable stable row IDs on an existing dataset" check so that the + /// flag can be activated without creating the dataset from scratch. + pub(crate) fn with_stable_row_id_migration_activation(mut self, next_row_id: u64) -> Self { + self.migration_next_row_id = Some(next_row_id); + self + } + pub async fn execute(self, transaction: Transaction) -> Result { let timeout = self.timeout; if let Some(t) = timeout @@ -241,6 +311,10 @@ impl<'a> CommitBuilder<'a> { .or_else(|| self.dest.dataset().map(|ds| ds.session.clone())) .unwrap_or_default(); + // Store and handler used to read the source manifest for a clone. + let source_store = self.source_store.clone(); + let source_commit_handler = self.source_commit_handler.clone(); + let (object_store, base_path, commit_handler) = match &self.dest { WriteDestination::Dataset(dataset) => ( dataset.object_store.clone(), @@ -340,7 +414,11 @@ impl<'a> CommitBuilder<'a> { ManifestNamingScheme::V1 }; - let use_stable_row_ids = if let Some(ds) = dest.dataset() { + let use_stable_row_ids = if self.migration_next_row_id.is_some() { + // Migration activation always enables stable row IDs regardless of + // the current dataset state. + true + } else if let Some(ds) = dest.dataset() { ds.manifest.uses_stable_row_ids() } else { self.use_stable_row_ids.unwrap_or(false) @@ -364,6 +442,7 @@ impl<'a> CommitBuilder<'a> { let manifest_config = ManifestWriteConfig { use_stable_row_ids, storage_format: self.storage_format.map(DataStorageFormat::new), + migration_next_row_id: self.migration_next_row_id, ..Default::default() }; @@ -381,6 +460,7 @@ impl<'a> CommitBuilder<'a> { &transaction, &manifest_config, &self.commit_config, + self.retry_timeout, ) .await? } else { @@ -391,6 +471,7 @@ impl<'a> CommitBuilder<'a> { &transaction, &manifest_config, &self.commit_config, + self.retry_timeout, manifest_naming_scheme, self.affected_rows.as_ref(), ) @@ -404,6 +485,8 @@ impl<'a> CommitBuilder<'a> { } else { commit_new_dataset( object_store.as_ref(), + source_store.as_deref(), + source_commit_handler.as_deref(), commit_handler.as_ref(), &base_path, &transaction, @@ -428,13 +511,21 @@ impl<'a> CommitBuilder<'a> { let fragment_bitmap = Arc::new(manifest.fragments.iter().map(|f| f.id as u32).collect()); match &self.dest { - WriteDestination::Dataset(dataset) => Ok(Dataset { - manifest: Arc::new(manifest), - manifest_location, - session, - fragment_bitmap, - ..dataset.as_ref().clone() - }), + WriteDestination::Dataset(dataset) => { + let base_object_stores = if manifest.base_paths == dataset.manifest.base_paths { + dataset.base_object_stores.clone() + } else { + Default::default() + }; + Ok(Dataset { + manifest: Arc::new(manifest), + manifest_location, + session, + fragment_bitmap, + base_object_stores, + ..dataset.as_ref().clone() + }) + } WriteDestination::Uri(uri) => { let refs = Refs::new( object_store.clone(), @@ -461,6 +552,7 @@ impl<'a> CommitBuilder<'a> { file_reader_options: None, store_params: self.store_params.clone().map(Box::new), base_store_params: None, + base_object_stores: Default::default(), }) } } @@ -525,7 +617,12 @@ mod tests { use lance_io::utils::CachedFileSize; use lance_io::{assert_io_eq, assert_io_gt}; - use lance_table::format::{DataFile, Fragment}; + use lance_table::format::{ + DataFile, Fragment, IndexMetadata, Manifest, Transaction as TableTransaction, + }; + use lance_table::io::commit::{ + CommitError, ConditionalPutCommitHandler, ManifestLocation, ManifestWriter, + }; use std::time::Duration; use object_store::throttle::ThrottleConfig; @@ -537,7 +634,9 @@ mod tests { use super::*; fn sample_fragment() -> Fragment { - let (major_version, minor_version) = LanceFileVersion::Stable.to_numbers(); + let (major_version, minor_version) = + LanceFileVersion::Stable.resolve().to_data_file_numbers(); + Fragment { id: 0, files: vec![DataFile { @@ -549,6 +648,7 @@ mod tests { file_size_bytes: CachedFileSize::new(100), base_id: None, }], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(10), @@ -569,6 +669,114 @@ mod tests { } } + #[derive(Debug)] + struct SlowConflictingCommitHandler; + + #[async_trait::async_trait] + impl CommitHandler for SlowConflictingCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + + async fn commit( + &self, + _manifest: &mut Manifest, + _indices: Option>, + _base_path: &object_store::path::Path, + _object_store: &ObjectStore, + _manifest_writer: ManifestWriter, + _naming_scheme: ManifestNamingScheme, + _transaction: Option, + ) -> std::result::Result { + tokio::time::sleep(Duration::from_millis(100)).await; + Err(CommitError::CommitConflict) + } + } + + #[derive(Debug)] + struct DestinationOnlyCommitHandler; + + #[async_trait::async_trait] + impl CommitHandler for DestinationOnlyCommitHandler { + async fn resolve_version_location( + &self, + _base_path: &object_store::path::Path, + _version: u64, + _object_store: &dyn object_store::ObjectStore, + ) -> Result { + Err(Error::invalid_input( + "destination commit handler cannot resolve source versions", + )) + } + + async fn commit( + &self, + manifest: &mut Manifest, + indices: Option>, + base_path: &object_store::path::Path, + object_store: &ObjectStore, + manifest_writer: ManifestWriter, + naming_scheme: ManifestNamingScheme, + transaction: Option, + ) -> std::result::Result { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await + } + } + + #[tokio::test] + async fn test_clone_uses_source_dataset_commit_handler() { + let session = Arc::new(Session::default()); + let batch = RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])), + vec![Arc::new(Int32Array::from_iter_values(0..10_i32))], + ) + .unwrap(); + let source = InsertBuilder::new("memory://clone-source-handler/source") + .with_params(&WriteParams { + session: Some(session.clone()), + ..Default::default() + }) + .execute(vec![batch]) + .await + .unwrap(); + let version = source.version().version; + let transaction = Transaction::new( + version, + Operation::Clone { + is_shallow: true, + ref_name: None, + ref_version: version, + ref_path: source.uri().to_string(), + branch_name: None, + }, + None, + ); + + let cloned = CommitBuilder::new("memory://clone-source-handler/target") + .with_session(session) + .with_commit_handler(Arc::new(DestinationOnlyCommitHandler)) + .with_source_dataset(&source) + .execute(transaction) + .await + .unwrap(); + + assert_eq!(cloned.count_rows(None).await.unwrap(), 10); + } + #[tokio::test] async fn test_reuse_session() { // Need to use in-memory for accurate IOPS tracking. @@ -629,8 +837,11 @@ mod tests { assert_eq!(new_ds.manifest().version, 7); // Session should still be re-used // However, the dataset needs to be loaded and the read version checked out. + // The read version's manifest body is served from the session cache (it + // was cached when v1 was first created), so the checkout only pays the + // version-resolution head, not a manifest read. let io_stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_eq!(io_stats, read_iops, 4, "load dataset + check version"); + assert_io_eq!(io_stats, read_iops, 3, "load dataset + check version"); assert_io_eq!(io_stats, write_iops, 2, "write txn + manifest"); // Commit transaction with URI and new session. Re-use the store @@ -780,6 +991,13 @@ mod tests { assert_eq!(DEFAULT_COMMIT_TIMEOUT, Duration::from_secs(1800)); } + #[test] + fn test_commit_retry_timeout_default_is_thirty_seconds() { + let builder = CommitBuilder::new("memory://default-retry-timeout"); + assert_eq!(builder.retry_timeout, DEFAULT_COMMIT_RETRY_TIMEOUT); + assert_eq!(DEFAULT_COMMIT_RETRY_TIMEOUT, Duration::from_secs(30)); + } + #[tokio::test] async fn test_commit_timeout_zero_rejected() { let dataset = Arc::new( @@ -808,7 +1026,7 @@ mod tests { ); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_commit_timeout_triggers() { let throttled = Arc::new(ThrottledStoreWrapper { config: ThrottleConfig { @@ -847,7 +1065,7 @@ mod tests { assert!(matches!(&err, Error::Timeout { .. }), "got {err:?}"); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_commit_timeout_applies_to_execute_batch() { let throttled = Arc::new(ThrottledStoreWrapper { config: ThrottleConfig { @@ -891,7 +1109,7 @@ mod tests { /// `with_timeout(None)` must let a commit run unbounded. Uses a throttled /// store so the commit takes real wall-clock time — long enough that the /// 50ms timeout in `test_commit_timeout_triggers` would have fired. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_commit_timeout_none_disables() { let throttled = Arc::new(ThrottledStoreWrapper { config: ThrottleConfig { @@ -930,6 +1148,38 @@ mod tests { assert_eq!(new_ds.manifest.version, 2); } + #[tokio::test] + async fn test_commit_retry_timeout_interrupts_conflict_backoff() { + let dataset = InsertBuilder::new("memory://retry-timeout") + .execute(vec![ + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ArrowField::new( + "i", + DataType::Int32, + false, + )])), + vec![Arc::new(Int32Array::from_iter_values(0..10_i32))], + ) + .unwrap(), + ]) + .await + .unwrap(); + + let result = CommitBuilder::new(Arc::new(dataset)) + .with_commit_handler(Arc::new(SlowConflictingCommitHandler)) + .with_max_retries(3) + .with_retry_timeout(Duration::from_millis(150)) + .with_timeout(None) + .execute(sample_transaction(1)) + .await; + + let error = result.expect_err("conflict backoff should respect retry timeout"); + assert!( + matches!(&error, Error::TooMuchWriteContention { message, .. } if message.contains("failed on retry_timeout")), + "got {error:?}" + ); + } + #[tokio::test] async fn test_commit_batch() { // Create a dataset @@ -963,7 +1213,7 @@ mod tests { new_fragments: vec![], removed_fragment_ids: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -1001,7 +1251,7 @@ mod tests { /// On non-lexically-ordered stores (e.g. S3 Express) a commit should use the /// version hint (a few HEAD probes, O(k)) instead of a full O(n) listing. - #[tokio::test] + #[tokio::test(start_paused = true)] async fn test_commit_uses_version_hint_on_non_lexical_store() { // Make `list` artificially slow per entry so a full listing would be // obvious; HEAD/GET/PUT stay fast. diff --git a/rust/lance/src/dataset/write/delete.rs b/rust/lance/src/dataset/write/delete.rs index a063d28ad7b..ee542541099 100644 --- a/rust/lance/src/dataset/write/delete.rs +++ b/rust/lance/src/dataset/write/delete.rs @@ -326,7 +326,7 @@ impl RetryExecutor for DeleteJob { Error::internal(format!("Failed to receive row ids: {}", err)) })?; let row_id_index = get_row_id_index(&self.dataset).await?; - let removed_row_addrs = removed_row_ids.row_addrs(row_id_index.as_deref()); + let removed_row_addrs = removed_row_ids.row_addrs(row_id_index.as_deref())?; let (fragments, deleted_ids) = apply_deletions(&self.dataset, &removed_row_addrs).await?; diff --git a/rust/lance/src/dataset/write/insert.rs b/rust/lance/src/dataset/write/insert.rs index 6e1db342f9c..b5dfd4b2953 100644 --- a/rust/lance/src/dataset/write/insert.rs +++ b/rust/lance/src/dataset/write/insert.rs @@ -7,11 +7,12 @@ use std::sync::Arc; use arrow_array::{RecordBatch, RecordBatchIterator}; use datafusion::execution::SendableRecordBatchStream; use humantime::format_duration; -use lance_core::datatypes::{NullabilityComparison, Schema, SchemaCompareOptions}; +use lance_core::datatypes::{NullabilityComparison, Schema}; +use lance_core::is_system_column; use lance_core::utils::tracing::{DATASET_WRITING_EVENT, TRACE_DATASET_EVENTS}; -use lance_core::{ROW_ADDR, ROW_ID, ROW_OFFSET}; use lance_datafusion::utils::StreamingWriteSource; -use lance_file::version::LanceFileVersion; +use lance_file::version::{ConcreteFileVersion, LanceFileVersion}; + use lance_io::object_store::ObjectStore; use lance_table::feature_flags::can_write_dataset; use lance_table::format::Fragment; @@ -19,7 +20,7 @@ use lance_table::io::commit::CommitHandler; use object_store::path::Path; use crate::Dataset; -use crate::blob::normalize_prepared_blob_schema; +use crate::blob::prepared_to_logical_blob_schema; use crate::dataset::ReadParams; use crate::dataset::builder::DatasetBuilder; use crate::dataset::transaction::{Operation, Transaction, TransactionBuilder}; @@ -136,7 +137,7 @@ impl<'a> InsertBuilder<'a> { async fn do_commit(context: &WriteContext<'_>, transaction: Transaction) -> Result { let mut commit_builder = CommitBuilder::new(context.dest.clone()) .use_stable_row_ids(context.params.enable_stable_row_ids) - .with_storage_format(context.storage_version) + .with_exact_storage_format(context.storage_version) .enable_v2_manifest_paths(context.params.enable_v2_manifest_paths) .with_commit_handler(context.commit_handler.clone()) .with_object_store(context.object_store.clone()) @@ -214,6 +215,7 @@ impl<'a> InsertBuilder<'a> { .await?; let (written_fragments, written_schema) = write_fragments_internal( + context.storage_version, context.dest.dataset(), context.object_store.clone(), &context.base_path, @@ -316,21 +318,18 @@ impl<'a> InsertBuilder<'a> { context.params.enable_stable_row_ids = dataset.manifest.uses_stable_row_ids(); } - let schema_cmp_opts = SchemaCompareOptions { - compare_dictionary: dataset.manifest.should_use_legacy_format(), - compare_nullability: NullabilityComparison::Ignore, - allow_missing_if_nullable: true, - ignore_field_order: true, - ..Default::default() - }; + let version = dataset.manifest.data_storage_format.lance_file_format(); + let mut schema_cmp_opts = crate::dataset::versions::schema_compare_options(version); + schema_cmp_opts.compare_nullability = NullabilityComparison::Ignore; + schema_cmp_opts.allow_missing_if_nullable = true; + schema_cmp_opts.ignore_field_order = true; - let normalized_data_schema = normalize_prepared_blob_schema(data_schema)?; + let normalized_data_schema = prepared_to_logical_blob_schema(data_schema)?; normalized_data_schema.check_compatible(dataset.schema(), &schema_cmp_opts)?; } - // Make sure we aren't using any reserved column names for field in data_schema.fields.iter() { - if field.name == ROW_ID || field.name == ROW_ADDR || field.name == ROW_OFFSET { + if is_system_column(&field.name) { return Err(Error::invalid_input_source( format!( "The column {} is a reserved name and cannot be used in a Lance dataset", @@ -365,7 +364,10 @@ impl<'a> InsertBuilder<'a> { WriteDestination::Dataset(dataset) => ( dataset.object_store.clone(), dataset.base.clone(), - dataset.commit_handler.clone(), + params + .commit_handler + .clone() + .unwrap_or_else(|| dataset.commit_handler.clone()), ), WriteDestination::Uri(uri) => { let registry = params @@ -413,15 +415,15 @@ impl<'a> InsertBuilder<'a> { (WriteMode::Overwrite, WriteDestination::Dataset(dataset)) => { // If overwriting an existing dataset, allow the user to specify but use // the existing version if they don't - params.data_storage_version.map(Ok).unwrap_or_else(|| { - let m = dataset.manifest.as_ref(); - m.data_storage_format.lance_file_version() - })? + params + .data_storage_version + .map(LanceFileVersion::resolve) + .unwrap_or_else(|| dataset.manifest.data_storage_format.lance_file_format()) } (_, WriteDestination::Dataset(dataset)) => { // If appending to an existing dataset, always use the dataset version let m = dataset.manifest.as_ref(); - m.data_storage_format.lance_file_version()? + m.data_storage_format.lance_file_format() } // Otherwise (no existing dataset) fallback to the default if the user didn't specify (_, WriteDestination::Uri(_)) => params.storage_version_or_default(), @@ -445,7 +447,7 @@ struct WriteContext<'a> { object_store: Arc, base_path: Path, commit_handler: Arc, - storage_version: LanceFileVersion, + storage_version: ConcreteFileVersion, } #[cfg(test)] @@ -455,6 +457,7 @@ mod test { use arrow_array::{ArrayRef, BinaryArray, Int32Array, RecordBatchReader, StructArray}; use arrow_schema::{ArrowError, DataType, Field, Schema}; use lance_arrow::BLOB_META_KEY; + use lance_table::io::commit::{RenameCommitHandler, commit_handler_from_url}; use crate::session::Session; @@ -506,6 +509,74 @@ mod test { ); } + #[tokio::test] + async fn dataset_destination_honors_explicit_commit_handler() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + let initial_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]) + .unwrap(); + let mut dataset = InsertBuilder::new("memory://") + .execute_stream(RecordBatchIterator::new( + vec![Ok(initial_batch)], + schema.clone(), + )) + .await + .unwrap(); + dataset.commit_handler = commit_handler_from_url("cos://bucket/dataset", &None) + .await + .unwrap(); + + let append_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![2]))]) + .unwrap(); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(Arc::new(RenameCommitHandler)), + ..Default::default() + }; + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(append_batch)], schema), + Arc::new(dataset), + Some(params), + ) + .await + .expect("the explicit per-write commit handler should override the dataset handler"); + + assert_eq!(dataset.count_rows(None).await.unwrap(), 2); + } + + #[rstest::rstest] + #[case::row_id("_rowid")] + #[case::row_addr("_rowaddr")] + #[case::row_offset("_rowoffset")] + #[case::row_created_at_version("_row_created_at_version")] + #[case::row_last_updated_at_version("_row_last_updated_at_version")] + #[tokio::test] + async fn rejects_reserved_system_column_names(#[case] reserved_name: &str) { + // Every system column name must be rejected on write. The row-version + // columns (`_row_created_at_version`, `_row_last_updated_at_version`) are + // computed at read time and appended by `Projection::to_schema`; a user + // data column sharing one of those names would otherwise pass ingest and + // later collide with the appended field. + let schema = Arc::new(Schema::new(vec![Field::new( + reserved_name, + DataType::Int32, + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(vec![1]))]) + .unwrap(); + + let result = InsertBuilder::new("memory://") + .execute_stream(RecordBatchIterator::new(vec![Ok(batch)], schema.clone())) + .await; + + let err = result.expect_err("writing a reserved system column name should fail"); + assert!( + err.to_string().contains("reserved name"), + "unexpected error for {reserved_name}: {err}" + ); + } + #[tokio::test] async fn allow_overwrite_to_v2_2_without_blob_upgrade() { let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); diff --git a/rust/lance/src/dataset/write/merge_insert.rs b/rust/lance/src/dataset/write/merge_insert.rs index a356b8c1d26..7234e8baa0b 100644 --- a/rust/lance/src/dataset/write/merge_insert.rs +++ b/rust/lance/src/dataset/write/merge_insert.rs @@ -55,13 +55,14 @@ use crate::{ Dataset, datafusion::dataframe::SessionContextExt, dataset::{ - fragment::{FileFragment, FragReadConfig}, - transaction::{Operation, Transaction}, - write::{merge_insert::logical_plan::MergeInsertPlanner, open_writer}, + fragment::FileFragment, + transaction::{Operation, Transaction, UpdatedFragmentOffsets}, + versions, + write::merge_insert::logical_plan::{MergeInsertPlanner, WriteSink}, }, index::DatasetIndexInternalExt, io::exec::{ - AddRowAddrExec, Planner, TakeExec, project, + Planner, project, scalar_index::{IndexLookup, MapIndexExec}, utils::ReplayExec, }, @@ -70,12 +71,14 @@ use arrow_array::{ BooleanArray, RecordBatch, RecordBatchIterator, StructArray, UInt32Array, UInt64Array, cast::AsArray, types::UInt64Type, }; -use arrow_schema::{DataType, Field, Schema}; +use arrow_schema::{ArrowError, DataType, Field, Schema}; use arrow_select::take::take_record_batch; use datafusion::common::NullEquality; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; use datafusion::{ + catalog::{TableProvider, streaming::StreamingTable}, + datasource::MemTable, execution::{ context::{SessionConfig, SessionContext}, memory_pool::MemoryConsumer, @@ -89,6 +92,7 @@ use datafusion::{ repartition::RepartitionExec, sorts::sort::SortExec, stream::RecordBatchStreamAdapter, + streaming::PartitionStream, union::UnionExec, }, physical_planner::{DefaultPhysicalPlanner, PhysicalPlanner}, @@ -114,25 +118,28 @@ use lance_datafusion::{ chunker::chunk_stream, dataframe::BatchStreamGrouper, exec::{ - HardCapBatchSizeExec, LanceExecutionOptions, OneShotExec, analyze_plan, execute_plan, - get_session_context, + HardCapBatchSizeExec, LanceExecutionOptions, OneShotExec, OneShotPartitionStream, + analyze_plan, execute_plan, get_session_context, provider_to_stream, }, + spill::spilling_table_provider, utils::{StreamingWriteSource, reader_to_stream}, }; +#[cfg(test)] use lance_file::version::LanceFileVersion; use lance_index::IndexCriteria; -use lance_index::mem_wal::MergedGeneration; +use lance_index::mem_wal::CompactedSsTable; use lance_select::RowAddrTreeMap; use lance_table::format::{Fragment, IndexMetadata, RowIdMeta}; use log::info; -use roaring::RoaringTreemap; +use roaring::{RoaringBitmap, RoaringTreemap}; use snafu::ResultExt; use std::collections::HashMap; use std::{ collections::{BTreeMap, HashSet}, + iter::Peekable, sync::{ Arc, Mutex, - atomic::{AtomicU32, Ordering}, + atomic::{AtomicU32, AtomicU64, Ordering}, }, time::Duration, }; @@ -143,6 +150,142 @@ mod assign_action; mod exec; mod logical_plan; +/// Build a source schema in target field order while preserving the source's +/// logical leaf types. The latter matters for extension columns such as Arrow +/// JSON, whose write input is Utf8 while the dataset's physical type is binary. +pub(crate) fn canonical_source_schema( + source: &Schema, + target: &Schema, +) -> std::result::Result { + fn canonical_field(source: &Field, target: &Field) -> std::result::Result { + let data_type = match (source.data_type(), target.data_type()) { + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + let fields = target_fields + .iter() + .map(|target_field| { + let source_field = source_fields + .iter() + .find(|field| field.name() == target_field.name()) + .ok_or_else(|| { + ArrowError::SchemaError(format!( + "field {} does not exist in source struct {}", + target_field.name(), + source.name() + )) + })?; + canonical_field(source_field, target_field).map(Arc::new) + }) + .collect::, _>>()?; + DataType::Struct(fields.into()) + } + (DataType::List(source_item), DataType::List(target_item)) => { + DataType::List(Arc::new(canonical_field(source_item, target_item)?)) + } + (DataType::LargeList(source_item), DataType::LargeList(target_item)) => { + DataType::LargeList(Arc::new(canonical_field(source_item, target_item)?)) + } + ( + DataType::FixedSizeList(source_item, size), + DataType::FixedSizeList(target_item, _), + ) => { + DataType::FixedSizeList(Arc::new(canonical_field(source_item, target_item)?), *size) + } + (DataType::Map(source_entries, sorted), DataType::Map(target_entries, _)) => { + DataType::Map( + Arc::new(canonical_field(source_entries, target_entries)?), + *sorted, + ) + } + _ => source.data_type().clone(), + }; + Ok(source.clone().with_data_type(data_type)) + } + + let fields = target + .fields() + .iter() + .map(|target_field| { + let source_field = source.field_with_name(target_field.name()).map_err(|_| { + ArrowError::SchemaError(format!( + "field {} does not exist in source schema", + target_field.name() + )) + })?; + canonical_field(source_field, target_field).map(Arc::new) + }) + .collect::, _>>()?; + Ok(Schema::new_with_metadata(fields, source.metadata().clone())) +} + +struct UpdatedRowAddrReconciler +where + I: Iterator, +{ + updated_rows: Peekable, +} + +impl UpdatedRowAddrReconciler +where + I: Iterator, +{ + fn new(updated_rows: I) -> Self { + Self { + updated_rows: updated_rows.peekable(), + } + } + + fn reconcile_batch(&mut self, original_row_addrs: &[u64]) -> Result> { + let mut indices = Vec::with_capacity(original_row_addrs.len()); + + for (original_offset, original_row_addr) in original_row_addrs.iter().enumerate() { + match self.updated_rows.peek().copied() { + Some((updated_row_addr, updated_row_index)) + if updated_row_addr == *original_row_addr => + { + self.updated_rows.next(); + indices.push(updated_row_index); + } + Some((updated_row_addr, _)) if updated_row_addr < *original_row_addr => { + return Err(Self::missing_row_error( + updated_row_addr, + Some(*original_row_addr), + )); + } + _ => indices.push((0, original_offset)), + } + } + + Ok(indices) + } + + fn finish(mut self) -> Result<()> { + if let Some((updated_row_addr, _)) = self.updated_rows.next() { + Err(Self::missing_row_error(updated_row_addr, None)) + } else { + Ok(()) + } + } + + fn missing_row_error(updated_row_addr: u64, next_original_row_addr: Option) -> Error { + let updated_row_addr = RowAddress::from(updated_row_addr); + let position = next_original_row_addr.map_or_else( + || "no target rows remain".to_string(), + |row_addr| format!("next target row address is {}", RowAddress::from(row_addr)), + ); + Error::internal(format!( + "Merge insert update row address {updated_row_addr} is missing from the target fragment; {position}" + )) + } +} + +/// Whether `field` is a blob or has one anywhere beneath it. +/// +/// `Field::children` is populated for structs, lists, and maps alike, so this +/// covers a blob reached through any nesting — not just a direct struct member. +fn subtree_has_blob(field: &lance_core::datatypes::Field) -> bool { + field.is_blob() || field.children.iter().any(subtree_has_blob) +} + // "update if" expressions typically compare fields from the source table to the target table. // These tables have the same schema and so filter expressions need to differentiate. To do that // we wrap the left side and the right side in a struct and make a single "combined schema" @@ -227,6 +370,86 @@ pub fn create_duplicate_row_error( )))) } +/// Tracks non-null join keys for source rows that will be inserted. +/// +/// NULL join keys are deliberately not tracked because merge insert uses SQL +/// equality, where a key containing NULL does not equal another such key. +#[derive(Debug, Default)] +struct InsertedKeyTracker { + keys: HashSet>, +} + +impl InsertedKeyTracker { + /// Returns true when the row has a new key or a key containing NULL. + fn insert( + &mut self, + batch: &RecordBatch, + row_idx: usize, + on_columns: &[String], + ) -> datafusion::common::Result { + let mut key = Vec::with_capacity(on_columns.len()); + for column_name in on_columns { + let column = batch.column_by_name(column_name).ok_or_else(|| { + DataFusionError::Internal(format!( + "merge insert key column '{}' not found in source batch", + column_name + )) + })?; + let value = ScalarValue::try_from_array(column, row_idx)?; + if value.is_null() { + return Ok(true); + } + key.push(value); + } + Ok(self.keys.insert(key)) + } +} + +/// How a merge insert writes the merged rows to disk. +/// +/// A partial-schema update (the source omits some dataset columns) can be +/// written two ways, and which one writes fewer bytes depends on the fraction of +/// each fragment's rows the source matches, which is only known once the join +/// has run. So the mode is chosen by the caller rather than guessed here. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum MergeInsertWriteMode { + /// Let the engine choose. + /// + /// Currently always rewrites whole rows, except on the one path that + /// predates this enum: a partial-schema update whose join key carries a + /// scalar index patches columns, as it has since before there was a way to + /// ask for either. + // TODO: choose per fragment inside the write sink, where the matched row + // count for that fragment is known. That needs a transaction-format change, + // because `UpdateMode` is per-commit and the conflict resolver treats + // `RewriteColumns` as proof that no row moved. + #[default] + Auto, + /// Delete the matched rows and write whole rows into new fragments. + /// + /// Cost scales with the number of matched rows, so this is the cheaper mode + /// when few rows match or when most columns are being replaced anyway. + /// + /// For a partial-schema update this gives up the scalar-index probe on the + /// join key if there is one, because the indexed path only ever patches + /// columns. + RewriteRows, + /// Attach new data files holding the source columns to the fragments that + /// already hold the matched rows, and tombstone the old versions of those + /// columns. + /// + /// The columns the source omits are neither read nor written, so this is the + /// cheaper mode for a narrow update of a wide table. The replacement column + /// file covers every row of each fragment it touches, so the bytes written + /// barely fall as fewer rows match. + /// + /// Errors if the merge cannot be expressed this way: it must update matched + /// rows only (no inserts, no matched deletes, no delete-by-source) with a + /// source that omits at least one dataset column, carries at least one + /// column besides the join key, and carries no blob column. + RewriteColumns, +} + /// Describes how rows should be handled when there is no matching row in the source table /// /// These are old rows which do not match any new data @@ -316,16 +539,21 @@ pub enum WhenNotMatched { DoNothing, } -/// Describes how to handle duplicate source rows that match the same target row. +/// Describes how to handle duplicate source rows. /// /// If the source contains duplicates and `FirstSeen` behavior doesn't match your needs, /// sort the source data before passing it to the merge insert operation. +/// Rows whose join keys contain NULL are not duplicates because merge insert uses SQL +/// equality, where NULL does not equal NULL. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] pub enum SourceDedupeBehavior { - /// Fail the operation if duplicates are found (default) + /// Fail if multiple source rows match the same target row (default) #[default] Fail, - /// Keep the first seen value and skip subsequent duplicates + /// Keep the first row for each join key and skip subsequent rows + /// + /// This applies both to rows that match a target row and to unmatched rows that + /// would otherwise insert the same non-null join key more than once. FirstSeen, } @@ -340,9 +568,15 @@ struct MergeInsertParams { // Controls whether data that is not matched by the source is deleted or not delete_not_matched_by_source: WhenNotMatchedBySource, conflict_retries: u32, + // When the source is a one-shot stream and `conflict_retries > 0`, the source + // is spilled (memory, then disk) so it can be replayed on each retry. Set to + // false to fail fast on contention instead of buffering the stream. Has no + // effect on re-scannable sources (materialized batches, files), which never + // spill. + spill_for_retry: bool, retry_timeout: Duration, - // List of MemWAL region generations to mark as merged when this commit succeeds. - merged_generations: Vec, + // MemWAL SSTables to mark as compacted when this commit succeeds. + compacted_sstables: Vec, // If true, skip auto cleanup during commits. This should be set to true // for high frequency writes to improve performance. This is also useful // if the writer does not have delete permissions and the clean up would @@ -351,6 +585,9 @@ struct MergeInsertParams { // Controls whether to use indices for the merge operation. Default is true. // Setting to false forces a full table scan even if an index exists. use_index: bool, + // How the merged rows are written to disk. Default is `Auto`, which today + // always rewrites whole rows. + write_mode: MergeInsertWriteMode, // Controls how to handle duplicate source rows that match the same target row. source_dedupe_behavior: SourceDedupeBehavior, // Number of inner commit retries for manifest version conflicts. Default is 20. @@ -365,6 +602,39 @@ struct MergeInsertParams { target_all_bases: Option, } +/// Where the per-fragment patch tasks in +/// [`MergeInsertJob::update_fragments`] deposit their results. +#[derive(Debug, Default)] +struct PatchSink { + /// Fragments that gained a data file, one entry per task. + fragments: Mutex>, + /// Physical offsets each fragment had patched. Only populated under stable + /// row ids, which is the only thing that reads the row-version metadata + /// these correct. + offsets: Mutex>, +} + +/// What [`MergeInsertJob::update_fragments`] wrote. +#[derive(Debug)] +pub(super) struct PatchedFragments { + /// Existing fragments that gained a data file for the patched columns. + pub updated_fragments: Vec, + /// Fragments written for rows that carried no target address. + pub new_fragments: Vec, + /// Ids of the fields written, so the caller can prune indices covering them. + pub fields_modified: Vec, + /// Physical row offsets patched in each updated fragment. + /// + /// `update_fragments` stamps row-level version metadata with the version it + /// was given, which is only a guess: a compatible transaction can commit + /// first, making the real commit version later. Handing these offsets to + /// `Operation::Update`'s `updated_fragment_offsets` lets `build_manifest` + /// re-stamp exactly the patched rows with the version the commit actually + /// got. Empty when the dataset does not use stable row ids, since nothing + /// reads the metadata then. + pub matched_offsets: UpdatedFragmentOffsets, +} + /// A MergeInsertJob inserts new rows, deletes old rows, and updates existing rows all as /// part of a single transaction. #[derive(Clone)] @@ -477,10 +747,12 @@ impl MergeInsertBuilder { insert_not_matched: true, delete_not_matched_by_source: WhenNotMatchedBySource::Keep, conflict_retries: 10, + spill_for_retry: true, retry_timeout: Duration::from_secs(30), - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), skip_auto_cleanup: false, use_index: true, + write_mode: MergeInsertWriteMode::default(), source_dedupe_behavior: SourceDedupeBehavior::Fail, commit_retries: None, target_bases: None, @@ -527,6 +799,27 @@ impl MergeInsertBuilder { self } + /// Controls whether a one-shot stream source is spilled so it can be replayed + /// across retries. + /// + /// When the source is a one-shot stream (e.g. [`MergeInsertJob::execute`]) and + /// `conflict_retries > 0`, the source is buffered in memory and spilled to disk + /// so each retry can re-read it. Set this to `false` to skip that buffering and + /// fail fast with a contention error instead of writing the stream to disk. + /// + /// This has no effect on re-scannable sources (materialized batches via + /// [`MergeInsertJob::execute_batches`], or a [`TableProvider`] via + /// [`MergeInsertJob::execute_provider`]), which are replayed directly and never + /// spill. + /// + /// Default is true. + /// + /// [`TableProvider`]: datafusion::catalog::TableProvider + pub fn spill_for_retry(&mut self, spill: bool) -> &mut Self { + self.params.spill_for_retry = spill; + self + } + /// Set the timeout used to limit retries. /// /// This is the maximum time to spend on the operation before giving up. At @@ -557,10 +850,34 @@ impl MergeInsertBuilder { self } - /// Specify how to handle duplicate source rows that match the same target row. + /// Selects how the merged rows are written to disk. + /// + /// For a partial-schema update the two modes have different cost shapes: + /// [`MergeInsertWriteMode::RewriteColumns`] never reads or writes the columns + /// the source omits, but its replacement column file covers every row of each + /// fragment it touches, so the bytes written barely fall as fewer rows match. + /// [`MergeInsertWriteMode::RewriteRows`] instead scales with the matched rows. + /// Patching columns wins once the fraction of rows matched exceeds roughly the + /// fraction of each row's bytes the source columns occupy: for a KB-scale + /// update of a MB-per-row table that is nearly always, and for a table whose + /// columns are all narrow it may never be. + /// + /// The per-fragment matched row count that decides this is only known once + /// the join has run, so the caller picks rather than the planner guessing. + /// + /// Default is [`MergeInsertWriteMode::Auto`], which rewrites whole rows. + pub fn write_mode(&mut self, mode: MergeInsertWriteMode) -> &mut Self { + self.params.write_mode = mode; + self + } + + /// Specify how to handle duplicate source rows. /// - /// Default is `Fail` which errors on duplicates. - /// Use `FirstSeen` to keep the first encountered row and skip duplicates. + /// Default is `Fail`, which errors when multiple source rows match one target row. + /// Use `FirstSeen` to keep the first encountered row for each non-null join key, + /// including unmatched keys that will be inserted, and skip subsequent rows. + /// Join keys containing NULL are not deduplicated because merge insert uses SQL + /// equality, where NULL does not equal NULL. /// /// If the source contains duplicates and `FirstSeen` behavior doesn't match your needs, /// sort the source data before passing it to the merge insert operation. @@ -569,10 +886,19 @@ impl MergeInsertBuilder { self } - /// Mark MemWAL region generations as merged when this commit succeeds. - /// This updates the merged_generations in the MemWAL Index atomically with the data commit. - pub fn mark_generations_as_merged(&mut self, generations: Vec) -> &mut Self { - self.params.merged_generations.extend(generations); + /// Mark MemWAL SSTables as compacted when this commit succeeds. + /// + /// This updates `compacted_sstables` in the MemWAL index atomically with + /// the data commit. + /// + /// **For multi-pass compaction, call this only on the final successful + /// data-changing pass.** Intermediate passes must not carry compaction + /// progress. Lance cannot tell whether a caller has another pass planned, + /// so it cannot enforce this: if a delete pass carried the marker and the + /// process then died before the matching upsert, the recorded generation + /// would claim rows were copied in that never were. + pub fn mark_sstables_as_compacted(&mut self, sstables: Vec) -> &mut Self { + self.params.compacted_sstables.extend(sstables); self } @@ -698,6 +1024,114 @@ enum SchemaComparison { Subschema, } +/// Wrap a one-shot stream in a non-replayable [`StreamingTable`] provider. +/// +/// The provider can only be scanned once (its single partition hands out the +/// underlying stream), so it must not be used where retries may re-scan it. +fn one_shot_provider(stream: SendableRecordBatchStream) -> Result> { + let schema = stream.schema(); + let partition = Arc::new(OneShotPartitionStream::new(stream)); + Ok(Arc::new(StreamingTable::try_new(schema, vec![partition])?)) +} + +/// Scans source partitions sequentially and removes duplicate non-null keys. +/// +/// Deduplicating before the join fixes the `FirstSeen` winner at the source +/// boundary, before DataFusion can reorder rows. The tracker retains only keys, +/// so this stays streaming without buffering source batches. +#[derive(Debug)] +struct DeduplicatingSourcePartitionStream { + input: Arc, + schema: Arc, + on_columns: Vec, + skipped_duplicates: Arc, +} + +impl DeduplicatingSourcePartitionStream { + fn new( + input: Arc, + on_columns: Vec, + skipped_duplicates: Arc, + ) -> Self { + let schema = input.schema(); + Self { + input, + schema, + on_columns, + skipped_duplicates, + } + } +} + +impl PartitionStream for DeduplicatingSourcePartitionStream { + fn schema(&self) -> &Arc { + &self.schema + } + + fn execute( + &self, + context: Arc, + ) -> SendableRecordBatchStream { + let input = self.input.clone(); + let partition_count = input.properties().output_partitioning().partition_count(); + let partition_streams = stream::iter(0..partition_count) + .map(move |partition| input.execute(partition, context.clone())) + .try_flatten(); + + let mut tracker = InsertedKeyTracker::default(); + let on_columns = self.on_columns.clone(); + let skipped_duplicates = self.skipped_duplicates.clone(); + skipped_duplicates.store(0, Ordering::Relaxed); + let deduplicated = partition_streams.map(move |batch| { + let batch = batch?; + let mut keep = Vec::with_capacity(batch.num_rows()); + let mut num_skipped = 0_u64; + for row_idx in 0..batch.num_rows() { + let is_first = tracker.insert(&batch, row_idx, &on_columns)?; + keep.push(is_first); + if !is_first { + num_skipped = num_skipped.checked_add(1).ok_or_else(|| { + DataFusionError::Execution( + "source duplicate count overflowed u64".to_string(), + ) + })?; + } + } + + let mut current = skipped_duplicates.load(Ordering::Relaxed); + loop { + let updated = current.checked_add(num_skipped).ok_or_else(|| { + DataFusionError::Execution(format!( + "source duplicate count overflow at {} with batch count {}", + current, num_skipped + )) + })?; + match skipped_duplicates.compare_exchange_weak( + current, + updated, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(actual) => current = actual, + } + } + + if num_skipped == 0 { + Ok(batch) + } else { + arrow::compute::filter_record_batch(&batch, &BooleanArray::from(keep)) + .map_err(DataFusionError::from) + } + }); + + Box::pin(RecordBatchStreamAdapter::new( + self.schema.clone(), + deduplicated, + )) + } +} + impl MergeInsertJob { pub async fn execute_reader( self, @@ -711,11 +1145,16 @@ impl MergeInsertJob { let lance_schema: lance_core::datatypes::Schema = schema.try_into()?; let target_schema = self.dataset.schema(); - let mut options = SchemaCompareOptions { - compare_dictionary: self.dataset.is_legacy_storage(), - compare_nullability: NullabilityComparison::Ignore, - ..Default::default() - }; + let version = self + .dataset + .manifest() + .data_storage_format + .lance_file_format(); + let mut options = versions::schema_compare_options(version); + options.compare_nullability = NullabilityComparison::Ignore; + // Merge columns are matched by name, so a complete source remains a + // full-schema merge even when the caller orders its fields differently. + options.ignore_field_order = true; // Try full schema match first. if lance_schema @@ -727,7 +1166,6 @@ impl MergeInsertJob { // If full match fails, try subschema match. options.allow_subschema = true; - options.ignore_field_order = true; // Subschema matching should typically ignore order. lance_schema .check_compatible(target_schema, &options) @@ -825,30 +1263,28 @@ impl MergeInsertJob { .iter() .map(|(col, idx)| IndexLookup::new(col.clone(), idx.name.clone())) .collect::>(); - let mut index_mapper: Arc = Arc::new(MapIndexExec::new_multi( + let index_mapper: Arc = Arc::new(MapIndexExec::new_multi( self.dataset.clone(), lookups, index_mapper_input, )); - // If requested, add row addresses to the output - if add_row_addr { - let pos = index_mapper.schema().fields().len(); // Add to end - index_mapper = Arc::new(AddRowAddrExec::try_new( - index_mapper, - self.dataset.clone(), - pos, - )?); - } - - // 4 - Take the mapped row ids + // 4 - Take the mapped row ids (TakeExec stays for legacy storage: + // the v1 reader cannot serve a FilteredReadExec) let projection = self .dataset .empty_projection() .union_arrow_schema(schema.as_ref(), OnMissing::Error)?; - let mut target = - Arc::new(TakeExec::try_new(self.dataset.clone(), index_mapper, projection)?.unwrap()) - as Arc; + let mut target = versions::merge_insert_indexed_take( + self.dataset + .manifest() + .data_storage_format + .lance_file_format(), + self.dataset.clone(), + index_mapper, + projection, + add_row_addr, + )?; // 5 - Take puts the row id and row addr at the beginning. A full scan (used when there is // no scalar index) puts the row id and addr at the end. We need to match these up so @@ -1084,12 +1520,18 @@ impl MergeInsertJob { self.create_full_table_joined_stream(source).await } - async fn update_fragments( + /// Patches the columns carried by `source` into the fragments that hold the + /// rows it names, and writes the rows with no target address as new + /// fragments. + /// + /// `source` must carry `_rowaddr` plus the columns to write. A null + /// `_rowaddr` routes the row to a new fragment. + pub(super) async fn update_fragments( dataset: Arc, source: SendableRecordBatchStream, current_version: u64, target_bases_info: Option>, - ) -> Result<(Vec, Vec, Vec)> { + ) -> Result { // Shared across the per-group tasks spawned below; only new fragments // are routed to target bases, column patches stay in primary storage. let target_bases_info = Arc::new(target_bases_info); @@ -1113,7 +1555,7 @@ impl MergeInsertJob { // sort node so each input batch fits in the memory pool. let capped_plan = sorted_plan .transform_down(|node| { - if node.as_any().downcast_ref::().is_some() { + if node.downcast_ref::().is_some() { let children = node.children(); let new_children: Vec> = children .into_iter() @@ -1133,7 +1575,7 @@ impl MergeInsertJob { let mut group_stream = BatchStreamGrouper::new(capped_stream, "_fragment_id".into()); // Can update the fragments in parallel. - let updated_fragments = Arc::new(Mutex::new(Vec::new())); + let patched = Arc::new(PatchSink::default()); let new_fragments = Arc::new(Mutex::new(Vec::new())); let mut tasks = JoinSet::new(); let task_limit = dataset.object_store.as_ref().io_parallelism(); @@ -1177,7 +1619,7 @@ impl MergeInsertJob { fragment: FileFragment, mut metadata: Fragment, mut batches: Vec, - updated_fragments: Arc>>, + patched: Arc, reservation_size: usize, current_version: u64, ) -> Result { @@ -1194,21 +1636,65 @@ impl MergeInsertJob { )?; let updated_rows: usize = batches.iter().map(|batch| batch.num_rows()).sum(); - if Some(updated_rows) == metadata.physical_rows { - // All rows have been updated and there are no deletions. So we - // don't need to merge in existing values. - // Also, because we already sorted by row address, the rows - // will be in the correct order. - - let data_storage_version = dataset - .manifest() - .data_storage_format - .lance_file_version()?; - let mut writer = open_writer( + + // This function is here to help rustc with lifetimes. + fn get_row_addr_iter( + batches: &[RecordBatch], + ) -> impl Iterator + '_ + Send { + batches.iter().enumerate().flat_map(|(batch_idx, batch)| { + // The index in source batches will be one more. + let batch_idx = batch_idx + 1; + let row_addrs = batch + .column_by_name(ROW_ADDR) + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + row_addrs + .values() + .iter() + .enumerate() + .map(move |(offset, row_addr)| (*row_addr, (batch_idx, offset))) + }) + } + + let has_full_fragment_coverage = metadata.deletion_file.is_none() + && Some(updated_rows) == metadata.physical_rows + && get_row_addr_iter(&batches) + .map(|(row_addr, _)| row_addr) + .eq(RowAddress::address_range(metadata.id as u32).take(updated_rows)); + + // Record which offsets this fragment patched before the write + // paths below consume `_rowaddr`. Only stable row ids read the + // row-version metadata these drive, so skip the work otherwise. + if dataset.manifest.uses_stable_row_ids() { + let offsets: RoaringBitmap = get_row_addr_iter(&batches) + .map(|(row_addr, _)| RowAddress::from(row_addr).row_offset()) + .collect(); + patched + .offsets + .lock() + .unwrap() + .entry(metadata.id) + .or_default() + .extend(offsets); + } + + if has_full_fragment_coverage { + // Exact, deletion-free coverage can be written directly because the + // batches are sorted by row address. + + let data_storage_version = + dataset.manifest().data_storage_format.lance_file_format(); + let mut writer = versions::open_writer( + data_storage_version, &dataset.object_store, &write_schema, &dataset.base, - data_storage_version, + super::WriterOptions { + add_data_dir: true, + ..Default::default() + }, ) .await?; @@ -1237,16 +1723,12 @@ impl MergeInsertJob { } } - if data_storage_version == LanceFileVersion::Legacy { + if let Some(batch_size) = + versions::row_group_size_for_rewrite(data_storage_version, &fragment) + .await? + { // Need to match the existing batch size exactly, otherwise // we'll get errors. - let reader = fragment - .open( - dataset.schema(), - FragReadConfig::default().with_row_address(true), - ) - .await?; - let batch_size = reader.legacy_num_rows_in_batch(0).unwrap(); let stream = stream::iter(batches.into_iter().map(Ok)); let stream = Box::pin(RecordBatchStreamAdapter::new( Arc::new((&write_schema).into()), @@ -1272,7 +1754,7 @@ impl MergeInsertJob { )?; } - updated_fragments.lock().unwrap().push(metadata); + patched.fragments.lock().unwrap().push(metadata); } else { // TODO: we could skip scanning row addresses we don't need. let update_schema = batches[0].schema(); @@ -1282,6 +1764,7 @@ impl MergeInsertJob { Some(&read_columns), Some((write_schema, dataset.schema().clone())), None, + None, ) .await?; @@ -1289,33 +1772,20 @@ impl MergeInsertJob { // will be the original source data, and all subsequent batches // will be updates. let mut source_batches = Vec::with_capacity(batches.len() + 1); - source_batches.push(batches[0].clone()); // placeholder for source data + // Convert Arrow JSON columns (Utf8) to Lance JSON (LargeBinary) so every + // batch is in physical format, matching what the updater reads from the + // fragment. `convert_json_columns` is a no-op clone when there is nothing + // to convert, so it can be applied unconditionally. The first entry is a + // placeholder for the source data (overwritten each iteration below); it + // must be converted too, otherwise its schema would diverge from the rest. + source_batches.push(convert_json_columns(&batches[0]).map_err(Error::from)?); for batch in &batches { - source_batches.push(batch.drop_column(ROW_ADDR)?); + let dropped = batch.drop_column(ROW_ADDR)?; + source_batches.push(convert_json_columns(&dropped).map_err(Error::from)?); } - // This function is here to help rustc with lifetimes. - fn get_row_addr_iter( - batches: &[RecordBatch], - ) -> impl Iterator + '_ + Send - { - batches.iter().enumerate().flat_map(|(batch_idx, batch)| { - // The index in source batches will be one more. - let batch_idx = batch_idx + 1; - let row_addrs = batch - .column_by_name(ROW_ADDR) - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - row_addrs - .values() - .iter() - .enumerate() - .map(move |(offset, row_addr)| (*row_addr, (batch_idx, offset))) - }) - } - let mut updated_row_addr_iter = get_row_addr_iter(&batches).peekable(); + let mut updated_rows = + UpdatedRowAddrReconciler::new(get_row_addr_iter(&batches)); while let Some(batch) = updater.next().await? { source_batches[0] = @@ -1327,34 +1797,13 @@ impl MergeInsertJob { .as_any() .downcast_ref::() .unwrap(); - let indices = original_row_addrs - .values() - .into_iter() - .enumerate() - .map(|(original_offset, row_addr)| { - match updated_row_addr_iter.peek() { - Some((updated_row_addr, _)) - if *updated_row_addr == *row_addr => - { - updated_row_addr_iter.next().unwrap().1 - } - // If we have passed the next updated row address, something went wrong. - Some((updated_row_addr, _)) => { - debug_assert!( - *updated_row_addr > *row_addr, - "Got updated row address that is not in the original batch" - ); - (0, original_offset) - } - _ => (0, original_offset), - } - }) - .collect::>(); + let indices = updated_rows.reconcile_batch(original_row_addrs.values())?; let updated_batch = interleave_batches(&source_batches, &indices)?; updater.update(updated_batch).await?; } + updated_rows.finish()?; let mut updated_fragment = updater.finish().await?; @@ -1387,7 +1836,7 @@ impl MergeInsertJob { )?; } - updated_fragments.lock().unwrap().push(updated_fragment); + patched.fragments.lock().unwrap().push(updated_fragment); } Ok(reservation_size) } @@ -1423,6 +1872,7 @@ impl MergeInsertJob { )?; let (fragments, _) = write_fragments_internal( + dataset.manifest.data_storage_format.lance_file_format(), Some(dataset.as_ref()), dataset.object_store.clone(), &dataset.base, @@ -1508,7 +1958,7 @@ impl MergeInsertJob { fragment, metadata, batches, - updated_fragments.clone(), + patched.clone(), memory_size, current_version, ); @@ -1545,10 +1995,12 @@ impl MergeInsertJob { } } } - let mut updated_fragments = Arc::try_unwrap(updated_fragments) - .unwrap() - .into_inner() - .unwrap(); + let PatchSink { + fragments: updated_fragments, + offsets: matched_offsets, + } = Arc::try_unwrap(patched).unwrap(); + let mut updated_fragments = updated_fragments.into_inner().unwrap(); + let matched_offsets = matched_offsets.into_inner().unwrap(); // We keep track of all fields that are updated so we can prune the indices. // We could maybe be more precise since some fields are not modified in some @@ -1578,36 +2030,162 @@ impl MergeInsertJob { } } + // Data files record physical leaf fields, while an index can be attached to + // a logical parent such as a list. Include every affected ancestor so index + // coverage is pruned for the complete logical column that was rewritten. + let directly_updated_fields = all_fields_updated.iter().copied().collect::>(); + for field_id in directly_updated_fields { + if let Some(ancestry) = dataset.schema().field_ancestry_by_id(field_id as i32) { + all_fields_updated.extend(ancestry.into_iter().map(|field| field.id as u32)); + } + } + let new_fragments = Arc::try_unwrap(new_fragments) .unwrap() .into_inner() .unwrap(); - Ok(( + Ok(PatchedFragments { updated_fragments, new_fragments, - all_fields_updated.into_iter().collect(), - )) + fields_modified: all_fields_updated.into_iter().collect(), + matched_offsets: UpdatedFragmentOffsets(matched_offsets), + }) } - /// Executes the merge insert job + /// Executes the merge insert job from a one-shot stream source. /// /// This will take in the source, merge it with the existing target data, and insert new - /// rows, update existing rows, and delete existing rows + /// rows, update existing rows, and delete existing rows. + /// + /// A stream can only be read once, so when `conflict_retries > 0` the stream is + /// spilled (in memory, then to disk) so it can be replayed on each retry. See + /// [`MergeInsertBuilder::spill_for_retry`] to fail fast instead, and + /// [`Self::execute_batches`] / [`Self::execute_provider`] for re-scannable + /// sources that never spill. pub async fn execute( self, source: SendableRecordBatchStream, ) -> Result<(Arc, MergeStats)> { - let source_iter = super::new_source_iter(source, self.params.conflict_retries > 0).await?; + let (provider, replayable) = self.stream_source_to_provider(source).await?; + self.execute_inner(provider, replayable).await + } + + /// Executes the merge insert job from a re-scannable [`TableProvider`]. + /// + /// This is the canonical entry point: [`Self::execute`] and + /// [`Self::execute_batches`] are thin wrappers that build a provider and call + /// this method. Because a provider can be scanned repeatedly, retries re-read + /// the source directly and never spill to disk. The provider's reported + /// statistics (e.g. from a [`MemTable`] or file source) also let DataFusion + /// optimize the merge join. + /// + /// [`MemTable`]: datafusion::datasource::MemTable + pub async fn execute_provider( + self, + provider: Arc, + ) -> Result<(Arc, MergeStats)> { + // A genuine TableProvider is re-scannable by contract, so retries are safe. + self.execute_inner(provider, true).await + } + + /// Executes the merge insert job from materialized record batches. + /// + /// The batches are wrapped in an in-memory [`MemTable`], which is re-scannable + /// (retries replay from memory, never spilling) and reports exact statistics to + /// the merge join. This is the preferred entry point when the full source is + /// already in memory. + pub async fn execute_batches( + self, + batches: Vec, + ) -> Result<(Arc, MergeStats)> { + let provider = self.batches_to_provider(batches)?; + self.execute_inner(provider, true).await + } + + /// Like [`Self::execute_batches`] but returns the uncommitted transaction. + /// + /// Use [`CommitBuilder`] to commit the returned transaction. + pub async fn execute_uncommitted_batches( + self, + batches: Vec, + ) -> Result { + let provider = self.batches_to_provider(batches)?; + self.execute_uncommitted_impl(provider).await + } + + /// Wrap materialized batches in a multi-partition in-memory [`MemTable`]. + fn batches_to_provider(&self, batches: Vec) -> Result> { + let schema = batches + .first() + .map(|batch| batch.schema()) + .unwrap_or_else(|| Arc::new(Schema::from(self.dataset.schema()))); + // FirstSeen needs a defined encounter order. Keep materialized batches in + // their caller-provided order; other modes retain parallel source scans. + let partitions = if self.params.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen { + vec![batches] + } else { + Self::batches_into_partitions(batches) + }; + Ok(Arc::new(MemTable::try_new(schema, partitions)?)) + } + + /// Distribute batches round-robin across up to `num_compute_intensive_cpus` + /// partitions, so a [`MemTable`] built from them can be scanned in parallel. + /// Always returns at least one (possibly empty) partition so an empty source + /// still produces a valid provider. + fn batches_into_partitions(batches: Vec) -> Vec> { + let num_partitions = batches.len().min(get_num_compute_intensive_cpus()).max(1); + let mut partitions = vec![Vec::new(); num_partitions]; + for (idx, batch) in batches.into_iter().enumerate() { + partitions[idx % num_partitions].push(batch); + } + partitions + } + + /// Wrap a one-shot stream source in a provider, returning whether it can be + /// replayed across retries. + /// + /// With retries enabled and spilling allowed, the stream is drained into a + /// replayable spill (memory up to 100MB, then disk). Otherwise the stream is + /// wrapped in a non-replayable one-shot provider and any conflict fails fast. + async fn stream_source_to_provider( + &self, + source: SendableRecordBatchStream, + ) -> Result<(Arc, bool)> { + if self.params.conflict_retries > 0 && self.params.spill_for_retry { + // Allow buffering up to 100MB in memory before spilling to disk. + let provider = spilling_table_provider(source, 100 * 1024 * 1024).await?; + Ok((provider, true)) + } else { + Ok((one_shot_provider(source)?, false)) + } + } + + /// Run the retry loop against a provider, re-scanning it on each attempt. + /// + /// `replayable` indicates whether the provider can be scanned more than once. + /// When it cannot (a one-shot stream that was not spilled), retries are + /// disabled so we never scan it twice; the operation runs once and surfaces any + /// commit conflict directly. + async fn execute_inner( + self, + provider: Arc, + replayable: bool, + ) -> Result<(Arc, MergeStats)> { let dataset = self.dataset.clone(); let config = RetryConfig { - max_retries: self.params.conflict_retries, + max_retries: if replayable { + self.params.conflict_retries + } else { + 0 + }, retry_timeout: self.params.retry_timeout, }; - let wrapper = MergeInsertJobWithIterator { + let wrapper = MergeInsertJobWithProvider { job: self, - source_iter: Arc::new(Mutex::new(source_iter)), + provider, attempt_count: Arc::new(AtomicU32::new(0)), }; @@ -1622,7 +2200,8 @@ impl MergeInsertJob { source: impl StreamingWriteSource, ) -> Result { let stream = source.into_stream(); - self.execute_uncommitted_impl(stream).await + self.execute_uncommitted_impl(one_shot_provider(stream)?) + .await } fn create_plan_join_type(&self) -> JoinType { @@ -1640,27 +2219,152 @@ impl MergeInsertJob { } } - async fn create_plan( - self, - source: SendableRecordBatchStream, - ) -> Result> { + /// Resolves the caller's [`MergeInsertWriteMode`] against this operation. + /// + /// [`WriteSink::RewriteColumns`] only ever replaces column data within a + /// fragment, so it cannot express anything that moves or removes rows: + /// inserting unmatched source rows, deleting matched rows, and deleting + /// target rows unmatched by the source all need [`WriteSink::RewriteRows`]. + /// Nor is it worth using when there is nothing to save, which is why a + /// source covering every dataset column or carrying nothing but the join key + /// is left on the row-rewrite path. + /// + /// Asking for it explicitly on an operation it cannot express is an error + /// rather than a silent fallback: the fallback would write orders of + /// magnitude more bytes than the caller asked for, with no signal. + fn select_write_sink(&self, source_schema: &Schema) -> Result { + // Every merge insert wrote whole rows before column patching existed, so + // that is what `Auto` still resolves to. + if self.params.write_mode != MergeInsertWriteMode::RewriteColumns { + return Ok(WriteSink::RewriteRows); + } + + let mut blockers: Vec<&str> = Vec::new(); + + if !self + .dataset + .schema() + .fields + .iter() + .any(|field| source_schema.column_with_name(&field.name).is_none()) + { + blockers.push("the source covers every dataset column, so there is nothing to skip"); + } + // A source of nothing but the join key has no new values to write: the + // patch would reproduce the key column byte for byte, and still cost a + // full-fragment column file plus the invalidation of every index over + // the key. Row-rewrite writes more bytes for it, but it does not + // invalidate those indices. + if !source_schema + .fields() + .iter() + .any(|field| !self.params.on.iter().any(|key| key == field.name())) + { + blockers.push("the source carries no column besides the join key"); + } + if !matches!( + self.params.when_matched, + WhenMatched::UpdateAll | WhenMatched::UpdateIf(_) | WhenMatched::UpdateIfExpr(_) + ) { + blockers.push("when_matched must update rather than delete or do nothing"); + } + if self.params.insert_not_matched { + blockers.push("inserting unmatched source rows adds rows, which patching cannot do"); + } + if !matches!( + self.params.delete_not_matched_by_source, + WhenNotMatchedBySource::Keep + ) { + blockers.push("deleting target rows unmatched by the source removes rows"); + } + // Patching a blob column would have to go through the fragment updater, + // which reads blobs in their stored descriptor form rather than the + // logical one the source provides. Leave those on the row-rewrite path, + // which already converts between the two representations. + // + // The source can only name top-level columns, so the check is rooted at + // the top-level fields it carries and then descends: a blob nested + // anywhere under one of them (struct member, list item, map value) is + // still patched by writing that whole top-level column. + if self + .dataset + .schema() + .fields + .iter() + .filter(|field| source_schema.column_with_name(&field.name).is_some()) + .any(subtree_has_blob) + { + blockers.push("the source carries a blob column, whose stored form differs from the one it provides"); + } + + if blockers.is_empty() { + Ok(WriteSink::RewriteColumns) + } else { + Err(Error::invalid_input(format!( + "MergeInsertWriteMode::RewriteColumns cannot express this merge insert: {}. \ + Use MergeInsertWriteMode::Auto or RewriteRows instead.", + blockers.join("; ") + ))) + } + } + + async fn create_plan(self, provider: Arc) -> Result> { // Goal: we shouldn't manually have to specify which columns to scan. // DataFusion's optimizer should be able to automatically perform // projection pushdown for us. // Goal: we shouldn't have to add new branches in this code to handle // indexed vs non-indexed cases. That should be handled by optimizer rules. - let session_config = SessionConfig::default(); - let session_ctx = SessionContext::new_with_config(session_config); - let scan = session_ctx.read_lance_unordered(self.dataset.clone(), true, true)?; - // Wrap column names in double quotes to preserve case (DataFusion lowercases unquoted identifiers) - let on_cols = self - .params - .on - .iter() - .map(|name| format!("\"{}\"", name)) - .collect::>(); + let write_sink = self.select_write_sink(provider.schema().as_ref())?; + let session_ctx = SessionContext::new(); + let binary_blob_field_ids = self + .dataset + .schema() + .fields_pre_order() + .filter(|field| field.is_blob() && !field.is_blob_v2()) + .map(|field| field.id as u32) + .collect(); + let target_provider = Arc::new( + crate::datafusion::dataframe::LanceTableProvider::new_with_ordering( + self.dataset.clone(), + true, + true, + false, + ) + .with_blob_handling(lance_core::datatypes::BlobHandling::SomeBlobsBinary( + binary_blob_field_ids, + )), + ); + let scan = session_ctx.read_table(target_provider)?; + // Wrap column names in double quotes to preserve case (DataFusion lowercases unquoted identifiers) + let on_cols = self + .params + .on + .iter() + .map(|name| format!("\"{}\"", name)) + .collect::>(); let on_cols_refs = on_cols.iter().map(|s| s.as_str()).collect::>(); - let source_df = session_ctx.read_one_shot(source)?; + // FirstSeen must observe the caller's source order even though the join can + // reorder batches. Deduplicating source partitions sequentially before + // the join fixes the winner at that contract boundary. Other modes plan + // directly against the provider so its statistics reach the optimizer. + let deduplicate_source = + self.params.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen; + let source_skipped_duplicates = Arc::new(AtomicU64::new(0)); + let source_df = if deduplicate_source { + let source_plan = provider.scan(&session_ctx.state(), None, &[], None).await?; + let deduplicated_partition = Arc::new(DeduplicatingSourcePartitionStream::new( + source_plan, + self.params.on.clone(), + source_skipped_duplicates.clone(), + )); + let deduplicated_provider = Arc::new(StreamingTable::try_new( + deduplicated_partition.schema().clone(), + vec![deduplicated_partition], + )?); + session_ctx.read_table(deduplicated_provider)? + } else { + session_ctx.read_table(provider)? + }; // Capture the source field names *before* aliasing / joining so we // can tell which dataset columns are missing from the source and // need to be filled from the target side of the join below. @@ -1705,12 +2409,19 @@ impl MergeInsertJob { // // We iterate the dataset schema in order so that the resulting // physical plan is deterministic and easy to inspect in tests. - for field in dataset_schema.fields() { - if !source_field_names.contains(field.name()) { - df = df.with_column( - field.name(), - logical_expr::col(format!("target.\"{}\"", field.name())), - )?; + // + // `RewriteColumns` patches the source columns into the fragments that + // already hold the matched rows, so the missing columns keep their + // stored values and must not be filled here. Skipping the fill is also + // what keeps them out of the target scan's projection. + if write_sink == WriteSink::RewriteRows { + for field in dataset_schema.fields() { + if !source_field_names.contains(field.name()) { + df = df.with_column( + field.name(), + logical_expr::col(format!("target.\"{}\"", field.name())), + )?; + } } } @@ -1720,6 +2431,8 @@ impl MergeInsertJob { logical_plan, self.dataset.clone(), self.params.clone(), + source_skipped_duplicates, + write_sink, ); let logical_plan = LogicalPlan::Extension(Extension { node: Arc::new(write_node), @@ -1739,14 +2452,14 @@ impl MergeInsertJob { async fn execute_uncommitted_v2( self, - source: SendableRecordBatchStream, + provider: Arc, ) -> Result<( Transaction, MergeStats, Option, Option, )> { - let plan = self.create_plan(source).await?; + let plan = self.create_plan(provider).await?; // Execute the plan // Assert that we have exactly one partition since we're designed for single-partition execution @@ -1780,8 +2493,7 @@ impl MergeInsertJob { // Extract merge stats from the execution plan let (stats, transaction, affected_rows, inserted_rows_filter) = if let Some(full_exec) = - plan.as_any() - .downcast_ref::() + plan.downcast_ref::() { let stats = full_exec.merge_stats().ok_or_else(|| { Error::internal("Merge stats not available - execution may not have completed") @@ -1792,10 +2504,18 @@ impl MergeInsertJob { let affected_rows = full_exec.affected_rows().map(RowAddrTreeMap::from); let inserted_rows_filter = full_exec.inserted_rows_filter(); (stats, transaction, affected_rows, inserted_rows_filter) - } else if let Some(delete_exec) = plan - .as_any() - .downcast_ref::() - { + } else if let Some(in_place_exec) = plan.downcast_ref::() { + let stats = in_place_exec.merge_stats().ok_or_else(|| { + Error::internal("Merge stats not available - execution may not have completed") + })?; + let transaction = in_place_exec.transaction().ok_or_else(|| { + Error::internal("Transaction not available - execution may not have completed") + })?; + // An in-place patch rewrites column data rather than only touching + // deletion files, so the affected rows cannot be expressed as a row + // address set (same reasoning as the legacy in-place path). + (stats, transaction, None, None) + } else if let Some(delete_exec) = plan.downcast_ref::() { let stats = delete_exec.merge_stats().ok_or_else(|| { Error::internal("Merge stats not available - execution may not have completed") })?; @@ -1806,7 +2526,7 @@ impl MergeInsertJob { (stats, transaction, affected_rows, None) } else { return Err(Error::internal( - "Expected FullSchemaMergeInsertExec or DeleteOnlyMergeInsertExec", + "Expected FullSchemaMergeInsertExec, InPlaceMergeInsertExec or DeleteOnlyMergeInsertExec", )); }; @@ -1884,8 +2604,24 @@ impl MergeInsertJob { && self.params.insert_not_matched && matches!(self.params.when_matched, WhenMatched::Delete); + // For a partial-schema update the indexed-scan path patches columns and + // has no branch that rewrites whole rows, so an explicit `RewriteRows` + // can only be honored by falling through to the v2 plan, giving up the + // index probe on the join. Naming the write sink wins over keeping the + // probe: the sink decides how many bytes are written, the probe only how + // the matched rows are found. Merges that write nothing (no matched + // update) or write whole rows on both paths (a full-schema source) are + // unaffected, so they keep the index. + let write_mode_needs_v2 = self.params.write_mode == MergeInsertWriteMode::RewriteRows + && is_subset_schema + && matches!( + self.params.when_matched, + WhenMatched::UpdateAll | WhenMatched::UpdateIf(_) | WhenMatched::UpdateIfExpr(_) + ); + let would_use_scalar_index = if self.params.use_index && !is_partial_delete_with_insert + && !write_mode_needs_v2 && matches!( self.params.delete_not_matched_by_source, WhenNotMatchedBySource::Keep @@ -1935,14 +2671,21 @@ impl MergeInsertJob { async fn execute_uncommitted_impl( self, - source: SendableRecordBatchStream, + provider: Arc, ) -> Result { + // Resolve the write mode before the path fork. The v2 plan resolves it + // again in `create_plan`, but the legacy path below does not go through + // that, and its partial-schema branch patches columns whatever the mode + // says. Resolving here means an unexpressible `RewriteColumns` is + // rejected on either path rather than only on one. + self.select_write_sink(provider.schema().as_ref())?; + // Check if we can use the fast path - let can_use_fast_path = self.can_use_create_plan(source.schema().as_ref()).await?; + let can_use_fast_path = self.can_use_create_plan(provider.schema().as_ref()).await?; if can_use_fast_path { let (transaction, stats, affected_rows, inserted_rows_filter) = - self.execute_uncommitted_v2(source).await?; + self.execute_uncommitted_v2(provider).await?; return Ok(UncommittedMergeInsert { transaction, affected_rows, @@ -1953,6 +2696,8 @@ impl MergeInsertJob { let target_bases_info = resolve_target_bases(&self.dataset, &self.params).await?; + // The slow path consumes a single stream; adapt the provider back into one. + let source = provider_to_stream(provider).await?; let source_schema = source.schema(); let lance_schema = lance_core::datatypes::Schema::try_from(source_schema.as_ref())?; let full_schema = self.dataset.schema(); @@ -1962,9 +2707,32 @@ impl MergeInsertJob { compare_metadata: false, // Allow nullable source fields for non-nullable targets. compare_nullability: NullabilityComparison::Ignore, + // Keep this classification consistent with `can_use_create_plan` + // and `check_compatible_schema`: merge columns match by name. + ignore_field_order: true, ..Default::default() }, ); + let source = if is_full_schema { + let target_schema = Schema::from(full_schema); + let canonical_schema = Arc::new(canonical_source_schema( + source_schema.as_ref(), + &target_schema, + )?); + let projection_schema = canonical_schema.clone(); + let projected = source.map(move |batch| { + batch.and_then(|batch| { + batch + .project_by_schema(projection_schema.as_ref()) + .map_err(DataFusionError::from) + }) + }); + Box::pin(RecordBatchStreamAdapter::new(canonical_schema, projected)) + as SendableRecordBatchStream + } else { + source + }; + let source_schema = source.schema(); let joined = self.create_joined_stream(source).await?; let merger = Merger::try_new( self.params.clone(), @@ -2007,14 +2775,17 @@ impl MergeInsertJob { let removed_row_ids = Arc::into_inner(deleted_rows).unwrap().into_inner().unwrap(); let removed_row_addr_vec = if let Some(row_id_index) = get_row_id_index(&self.dataset).await? { - removed_row_ids - .iter() - .filter_map(|id| row_id_index.get(*id).map(|address| address.into())) - .collect::>() + let mut addresses = Vec::with_capacity(removed_row_ids.len()); + for id in &removed_row_ids { + if let Some(address) = row_id_index.get(*id)? { + addresses.push(address.into()); + } + } + addresses } else { removed_row_ids }; - let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec.into_iter()); + let removed_row_addrs = RoaringTreemap::from_iter(removed_row_addr_vec); let (updated_fragments, removed_fragment_ids) = Self::apply_deletions(&self.dataset, &removed_row_addrs).await?; @@ -2024,10 +2795,9 @@ impl MergeInsertJob { updated_fragments, new_fragments: vec![], fields_modified: vec![], - merged_generations: self.params.merged_generations.clone(), + compacted_sstables: self.params.compacted_sstables.clone(), fields_for_preserving_frag_bitmap: full_schema - .fields - .iter() + .fields_pre_order() .map(|f| f.id as u32) .collect(), update_mode: Some(RewriteRows), @@ -2050,7 +2820,12 @@ impl MergeInsertJob { // We will have a different commit path here too, as we are modifying // fragments rather than writing new ones - let (updated_fragments, new_fragments, fields_modified) = Self::update_fragments( + let PatchedFragments { + updated_fragments, + new_fragments, + fields_modified, + matched_offsets, + } = Self::update_fragments( self.dataset.clone(), Box::pin(stream), self.dataset.manifest.version + 1, @@ -2063,11 +2838,14 @@ impl MergeInsertJob { updated_fragments, new_fragments, fields_modified, - merged_generations: self.params.merged_generations.clone(), + compacted_sstables: self.params.compacted_sstables.clone(), fields_for_preserving_frag_bitmap: vec![], // in-place update do not affect preserving frag bitmap update_mode: Some(RewriteColumns), inserted_rows_filter: None, // not implemented for v1 - updated_fragment_offsets: None, + // The version stamped above is a guess; carry the patched offsets + // so `build_manifest` can re-stamp them at the real commit + // version after a rebase. + updated_fragment_offsets: Some(matched_offsets), }; // We have rewritten the fragments, not just the deletion files, so // we can't use affected rows here. @@ -2075,6 +2853,10 @@ impl MergeInsertJob { } else { let cleanup_bases = target_bases_info.clone(); let (mut new_fragments, _) = write_fragments_internal( + self.dataset + .manifest + .data_storage_format + .lance_file_format(), Some(&self.dataset), self.dataset.object_store.clone(), &self.dataset.base, @@ -2107,7 +2889,7 @@ impl MergeInsertJob { for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) { let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } } @@ -2116,16 +2898,18 @@ impl MergeInsertJob { let removed_row_addr_vec = if let Some(row_id_index) = get_row_id_index(&self.dataset).await? { - let addresses: Vec = removed_row_ids - .iter() - .filter_map(|id| row_id_index.get(*id).map(|address| address.into())) - .collect::>(); + let mut addresses: Vec = Vec::with_capacity(removed_row_ids.len()); + for id in &removed_row_ids { + if let Some(address) = row_id_index.get(*id)? { + addresses.push(address.into()); + } + } addresses } else { removed_row_ids }; - Ok(RoaringTreemap::from_iter(removed_row_addr_vec.into_iter())) + Ok(RoaringTreemap::from_iter(removed_row_addr_vec)) } .await; let removed_row_addrs = match post_write_result { @@ -2165,10 +2949,9 @@ impl MergeInsertJob { // On this path we only make deletions against updated_fragments and will not // modify any field values. fields_modified: vec![], - merged_generations: self.params.merged_generations.clone(), + compacted_sstables: self.params.compacted_sstables.clone(), fields_for_preserving_frag_bitmap: full_schema - .fields - .iter() + .fields_pre_order() .map(|f| f.id as u32) .collect(), update_mode: Some(RewriteRows), @@ -2253,6 +3036,12 @@ impl MergeInsertJob { /// * `schema` - Optional schema of the source data. If None, uses the dataset's schema /// * `verbose` - If true, provides more detailed information in the plan output /// + /// A schema says nothing about how the source would be wrapped, so this always + /// reports the streaming shape: the source is stood in for by an empty one-shot + /// stream. The wrapping affects the plan, so use [`Self::analyze_plan_batches`] + /// or [`Self::analyze_plan_provider`] when that matters. Those execute the merge + /// to collect metrics and may write data files; this method writes nothing. + /// /// # Errors /// /// Returns Error::NotSupported if the merge insert configuration doesn't support @@ -2266,7 +3055,7 @@ impl MergeInsertJob { // Check if we can use create_plan if !self.can_use_create_plan(&schema).await? { - return Err(Error::not_supported_source("This merge insert configuration does not support explain_plan. Only full-schema merge insert operations without a scalar-index execution path are currently supported.".into())); + return Err(Error::not_supported_source("This merge insert configuration does not support explain_plan: either the source schema is not one the plan path accepts, or the join takes the scalar-index execution path.".into())); } // Create an empty batch with the provided schema to pass to create_plan @@ -2278,7 +3067,9 @@ impl MergeInsertJob { // Clone self since create_plan consumes the job let cloned_job = self.clone(); - let plan = cloned_job.create_plan(Box::pin(stream)).await?; + let plan = cloned_job + .create_plan(one_shot_provider(Box::pin(stream))?) + .await?; let display = DisplayableExecutionPlan::new(plan.as_ref()); Ok(format!("{}", display.indent(verbose))) @@ -2299,19 +3090,69 @@ impl MergeInsertJob { /// /// * `source` - The source data stream that would be used in the merge insert /// + /// A stream reports no statistics, so the plan this returns is the streaming + /// one. Callers holding materialized data or a source that reports statistics + /// should use [`Self::analyze_plan_batches`] or [`Self::analyze_plan_provider`], + /// which report the plan those sources actually run. + /// /// # Errors /// - /// Returns Error::NotSupported if the merge insert configuration doesn't support - /// the fast path required for plan generation. + /// See [`Self::analyze_plan_provider`], which this delegates to. pub async fn analyze_plan(&self, source: SendableRecordBatchStream) -> Result { + self.analyze_plan_provider(one_shot_provider(source)?).await + } + + /// [`Self::analyze_plan`] for materialized batches. + /// + /// Mirrors [`Self::execute_batches`]: the batches are wrapped in a + /// [`MemTable`], so the reported plan is the one an in-memory source actually + /// runs. That plan can differ from the streaming one, because the join picks + /// its collected side from the statistics each source reports. + /// + /// Under [`SourceDedupeBehavior::FirstSeen`] the source is deduplicated ahead + /// of the join and re-wrapped in a stream, so the reported plan is the + /// streaming one and the in-memory node does not appear in it. + /// + /// An empty `batches` still reports an in-memory source, but it carries no + /// schema for the provider to use, so the support check runs against the + /// dataset's; see [`Self::analyze_plan_provider`]. + /// + /// [`MemTable`]: datafusion::datasource::MemTable + pub async fn analyze_plan_batches(&self, batches: Vec) -> Result { + self.analyze_plan_provider(self.batches_to_provider(batches)?) + .await + } + + /// [`Self::analyze_plan`] from a re-scannable [`TableProvider`]. + /// + /// Mirrors [`Self::execute_provider`]. Under + /// [`SourceDedupeBehavior::FirstSeen`] the provider is re-wrapped in a stream + /// before the join, so its own node does not appear in the reported plan. + /// + /// The support check runs against `provider.schema()`, which is the source + /// schema the caller supplied. For a provider built from an empty batch list + /// that schema is the dataset's, so a source whose declared schema the dataset + /// does not have is reported rather than rejected. `execute_batches` builds its + /// provider the same way, so the two agree. + /// + /// # Errors + /// + /// * `Error::NotSupported` when the configuration cannot use the plan path. + /// `can_use_create_plan` decides that, and its own doc comment lists the + /// source shapes it accepts. + /// * `Error::invalid_input` from the support check, e.g. a non-nullable dataset + /// column the source does not supply. + /// * Any error from building or executing the plan. This method runs the merge + /// to collect metrics, so I/O and source-deduplication failures surface here. + pub async fn analyze_plan_provider(&self, provider: Arc) -> Result { // Check if we can use create_plan - if !self.can_use_create_plan(source.schema().as_ref()).await? { - return Err(Error::not_supported_source("This merge insert configuration does not support analyze_plan. Only full-schema merge insert operations without a scalar-index execution path are currently supported.".into())); + if !self.can_use_create_plan(provider.schema().as_ref()).await? { + return Err(Error::not_supported_source("This merge insert configuration does not support plan reporting: either the source schema is not one the plan path accepts, or the join takes the scalar-index execution path.".into())); } // Clone self since create_plan consumes the job let cloned_job = self.clone(); - let plan = cloned_job.create_plan(source).await?; + let plan = cloned_job.create_plan(provider).await?; // Use the analyze_plan function from lance_datafusion, but strip out the wrapper lines let options = LanceExecutionOptions::default(); @@ -2361,15 +3202,15 @@ pub struct UncommittedMergeInsert { pub inserted_rows_filter: Option, } -/// Wrapper struct that combines MergeInsertJob with the source iterator for retry functionality +/// Wrapper struct that combines MergeInsertJob with the source provider for retry functionality #[derive(Clone)] -struct MergeInsertJobWithIterator { +struct MergeInsertJobWithProvider { job: MergeInsertJob, - source_iter: Arc + Send + 'static>>>, + provider: Arc, attempt_count: Arc, } -impl RetryExecutor for MergeInsertJobWithIterator { +impl RetryExecutor for MergeInsertJobWithProvider { type Data = UncommittedMergeInsert; type Result = (Arc, MergeStats); @@ -2377,10 +3218,11 @@ impl RetryExecutor for MergeInsertJobWithIterator { // Increment attempt counter self.attempt_count.fetch_add(1, Ordering::SeqCst); - // We need to get a fresh stream for each retry attempt - // The source_iter provides unlimited streams from the same source data - let stream = self.source_iter.lock().unwrap().next().unwrap(); - self.job.clone().execute_uncommitted_impl(stream).await + // Re-scan the provider on each retry attempt. + self.job + .clone() + .execute_uncommitted_impl(self.provider.clone()) + .await } async fn commit(&self, dataset: Arc, mut data: Self::Data) -> Result { @@ -2464,6 +3306,8 @@ struct Merger { enable_stable_row_ids: bool, /// Set to track processed row IDs to detect duplicates processed_row_ids: Arc>>, + /// Set to track non-null keys of rows inserted by FirstSeen mode + processed_insert_keys: Arc>, } impl Merger { @@ -2531,6 +3375,7 @@ impl Merger { output_schema, enable_stable_row_ids, processed_row_ids: Arc::new(Mutex::new(HashSet::new())), + processed_insert_keys: Arc::new(Mutex::new(InsertedKeyTracker::default())), }) } @@ -2570,7 +3415,6 @@ impl Merger { &self, combined_batch: &RecordBatch, right_offset: usize, - num_keys: usize, ) -> Result<(BooleanArray, BooleanArray, BooleanArray)> { // The outer join distinguishes its three cases by which side's join // keys were NULL-padded: a present row always has non-null keys, while @@ -2580,14 +3424,18 @@ impl Merger { // column (e.g. an all-null vector) at position 0, and checking // positions [0, num_keys) there misreads an all-null leading payload // column as an absent join side, silently dropping every matched row - // (https://github.com/lancedb/lancedb/issues/3515). The target half - // carries the same columns in the same order, offset by `right_offset`. + // (https://github.com/lancedb/lancedb/issues/3515). The target half is + // resolved independently because the indexed path keeps dataset-schema + // order even when the source fields are reordered. Restrict that lookup + // to the target half because a valid source field can have the same + // `target_`-prefixed name. + let combined_schema = combined_batch.schema(); let source_key_cols = self .params .on .iter() .map(|key| { - combined_batch.schema().index_of(key).map_err(|_| { + combined_schema.index_of(key).map_err(|_| { Error::internal(format!( "merge insert key column '{}' not found in joined batch", key @@ -2595,11 +3443,26 @@ impl Merger { }) }) .collect::>>()?; - debug_assert_eq!(source_key_cols.len(), num_keys); - let target_key_cols = source_key_cols + let target_key_cols = self + .params + .on .iter() - .map(|c| c + right_offset) - .collect::>(); + .map(|key| { + let target_key = format!("target_{key}"); + combined_schema + .fields() + .iter() + .enumerate() + .skip(right_offset) + .find_map(|(index, field)| (field.name() == &target_key).then_some(index)) + .ok_or_else(|| { + Error::internal(format!( + "merge insert target key column '{}' not found in joined batch", + target_key + )) + }) + }) + .collect::>>()?; let in_left = Self::not_all_null(combined_batch, &source_key_cols)?; let in_right = Self::not_all_null(combined_batch, &target_key_cols)?; @@ -2635,14 +3498,11 @@ impl Merger { (num_fields - 2, Some(num_fields - 1), (num_fields - 2) / 2) }; - let num_keys = self.params.on.len(); - let left_cols = Vec::from_iter(0..right_offset); let right_cols_with_id = Vec::from_iter(right_offset..num_fields); let mut batches = Vec::with_capacity(2); - let (left_only, in_both, right_only) = - self.extract_selections(&batch, right_offset, num_keys)?; + let (left_only, in_both, right_only) = self.extract_selections(&batch, right_offset)?; // There is no contention on this mutex. We're only using it to bypass the rust // borrow checker (the stream needs to be `sync` since it crosses an await point) @@ -2792,7 +3652,24 @@ impl Merger { } } if self.params.insert_not_matched { - let not_matched = arrow::compute::filter_record_batch(&batch, &left_only)?; + let mut not_matched = arrow::compute::filter_record_batch(&batch, &left_only)?; + if self.params.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen { + let mut processed_insert_keys = self.processed_insert_keys.lock().unwrap(); + let mut keep_indices = Vec::with_capacity(not_matched.num_rows()); + for row_idx in 0..not_matched.num_rows() { + if processed_insert_keys.insert(¬_matched, row_idx, &self.params.on)? { + keep_indices.push(row_idx as u32); + } else { + merge_statistics.num_skipped_duplicates += 1; + } + } + drop(processed_insert_keys); + + if keep_indices.len() != not_matched.num_rows() { + not_matched = + take_record_batch(¬_matched, &UInt32Array::from(keep_indices))?; + } + } let left_cols_with_id = left_cols .into_iter() .chain(row_addr_col) @@ -2867,7 +3744,7 @@ mod tests { use arrow_array::types::Float32Type; use arrow_array::{ Array, FixedSizeListArray, Float32Array, Float64Array, Int32Array, Int64Array, ListArray, - RecordBatchIterator, RecordBatchReader, StringArray, StructArray, UInt32Array, + NullArray, RecordBatchIterator, RecordBatchReader, StringArray, StructArray, UInt32Array, types::{Int32Type, UInt32Type}, }; use arrow_array::{RecordBatch, record_batch}; @@ -2896,6 +3773,88 @@ mod tests { t } + #[test] + fn test_inserted_key_tracker_preserves_logical_nulls() { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("id", DataType::Null, true)])), + vec![Arc::new(NullArray::new(2))], + ) + .unwrap(); + let mut tracker = InsertedKeyTracker::default(); + let on_columns = ["id".to_string()]; + + assert!(tracker.insert(&batch, 0, &on_columns).unwrap()); + assert!(tracker.insert(&batch, 1, &on_columns).unwrap()); + } + + #[test] + fn test_updated_row_addr_missing_between_target_rows() { + let row_addr = |offset| u64::from(RowAddress::new_from_parts(3, offset)); + let mut updated_rows = UpdatedRowAddrReconciler::new([(row_addr(1), (1, 0))].into_iter()); + + let error = updated_rows + .reconcile_batch(&[row_addr(0), row_addr(2)]) + .unwrap_err(); + + assert!(matches!(error, Error::Internal { .. })); + let message = error.to_string(); + assert!(message.contains("update row address (3, 1) is missing")); + assert!(message.contains("next target row address is (3, 2)")); + } + + #[test] + fn test_updated_row_addr_missing_after_target_rows() { + let row_addr = |offset| u64::from(RowAddress::new_from_parts(7, offset)); + let mut updated_rows = UpdatedRowAddrReconciler::new([(row_addr(2), (1, 0))].into_iter()); + + assert_eq!( + updated_rows + .reconcile_batch(&[row_addr(0), row_addr(1)]) + .unwrap(), + vec![(0, 0), (0, 1)] + ); + let error = updated_rows.finish().unwrap_err(); + + assert!(matches!(error, Error::Internal { .. })); + let message = error.to_string(); + assert!(message.contains("update row address (7, 2) is missing")); + assert!(message.contains("no target rows remain")); + } + + #[tokio::test] + async fn test_updated_row_addr_missing_in_full_fragment_update() { + let initial = record_batch!(("value", Int32, [10, 20])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + let fragment_id = dataset.get_fragments()[0].id() as u32; + let row_addr = |offset| u64::from(RowAddress::new_from_parts(fragment_id, offset)); + let updates = record_batch!( + (ROW_ADDR, UInt64, [row_addr(0), row_addr(2)]), + ("value", Int32, [100, 200]) + ) + .unwrap(); + let update_stream = + RecordBatchStreamAdapter::new(updates.schema(), futures::stream::iter([Ok(updates)])); + + let error = MergeInsertJob::update_fragments( + dataset.clone(), + Box::pin(update_stream), + dataset.manifest().version + 1, + None, + ) + .await + .unwrap_err(); + + assert!(matches!(error, Error::Internal { .. })); + let message = error.to_string(); + assert!(message.contains("update row address (0, 2) is missing")); + assert!(message.contains("no target rows remain")); + } + // An update-style merge_insert leaves the source and new fragments with // overlapping id ranges; a scattered delete punches holes in that range. A // filtered `with_row_id` scan must still resolve every id (round-tripped via take). @@ -3850,6 +4809,85 @@ mod tests { ); } + #[rstest::rstest] + #[tokio::test] + async fn test_multi_batch_upsert_preserves_stable_row_ids( + #[values(true, false)] use_index: bool, + ) { + let mut dataset = (*create_test_dataset( + "memory://test_multi_batch_upsert_row_ids", + LanceFileVersion::default(), + true, + ) + .await) + .clone(); + dataset + .create_index( + &["key"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + let dataset = Arc::new(dataset); + + let initial_keys = [1, 2, 3, 4, 5, 6]; + let initial_row_ids = get_row_ids_for_keys(&dataset, &initial_keys).await; + let initial_row_ids = initial_row_ids + .values() + .iter() + .copied() + .collect::>(); + let updated_keys = [2, 4]; + let updated_row_ids_before = get_row_ids_for_keys(&dataset, &updated_keys).await; + + // Put inserts in the first source batch and updates in the second. Stable + // row-id assignment still relies on the join emitting all updates first. + let insert_batch = record_batch!( + ("key", UInt32, [7, 8]), + ("value", UInt32, [70, 80]), + ("filterme", Utf8, ["inserted", "inserted"]) + ) + .unwrap(); + let update_batch = record_batch!( + ("key", UInt32, [2, 4]), + ("value", UInt32, [20, 40]), + ("filterme", Utf8, ["updated", "updated"]) + ) + .unwrap(); + let source_schema = insert_batch.schema(); + let source = Box::new(RecordBatchIterator::new( + [Ok(insert_batch), Ok(update_batch)], + source_schema, + )); + + let (dataset, stats) = MergeInsertBuilder::try_new(dataset, vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .use_index(use_index) + .try_build() + .unwrap() + .execute_reader(source) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 2); + assert_eq!(stats.num_inserted_rows, 2); + let updated_row_ids_after = get_row_ids_for_keys(&dataset, &updated_keys).await; + assert_eq!(updated_row_ids_after, updated_row_ids_before); + + let inserted_row_ids = get_row_ids_for_keys(&dataset, &[7, 8]).await; + assert!( + inserted_row_ids + .values() + .iter() + .all(|row_id| !initial_row_ids.contains(row_id)) + ); + } + #[rstest::rstest] #[tokio::test] async fn test_row_id_stability_across_update_and_merge_insert( @@ -4134,42 +5172,172 @@ mod tests { assert_eq!(n_indexed, UPD, "expected {UPD} rows flipped to 'indexed'"); } + #[rstest::rstest] + #[case::reordered_null_payload(false, Some(42))] + #[case::target_name_collision(true, None)] #[tokio::test] - async fn test_indexed_merge_insert() { - let test_dir = TempStrDir::default(); - let test_uri = &test_dir; + async fn test_indexed_partial_merge_with_reordered_source( + #[case] has_target_name_collision: bool, + #[case] expected_payload: Option, + ) { + let (target, source, payload_column) = if has_target_name_collision { + ( + record_batch!( + ("id", UInt64, [1]), + ("target_id", Int32, [9]), + ("b", Int32, [7]) + ) + .unwrap(), + record_batch!(("target_id", Int32, [None]), ("id", UInt64, [1])).unwrap(), + "target_id", + ) + } else { + ( + record_batch!(("id", UInt64, [1]), ("a", Int32, [None]), ("b", Int32, [7])) + .unwrap(), + record_batch!(("a", Int32, [42]), ("id", UInt64, [1])).unwrap(), + "a", + ) + }; + let mut dataset = InsertBuilder::new("memory://") + .execute(vec![target]) + .await + .unwrap(); + dataset + .create_index( + &["id"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(1)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); - let schema = data.schema(); + let source_schema = source.schema(); + let (dataset, stats) = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap() + .execute_reader(RecordBatchIterator::new([Ok(source)], source_schema)) + .await + .unwrap(); - // Create an input dataset with a scalar index on key - let mut ds = Dataset::write(data, test_uri, None).await.unwrap(); - let index_params = ScalarIndexParams::default(); - ds.create_index(&["key"], IndexType::Scalar, None, &index_params, false) + assert_eq!(stats.num_inserted_rows, 0); + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_deleted_rows, 0); + let result = dataset + .scan() + .project(&[payload_column]) + .unwrap() + .try_into_batch() .await .unwrap(); + let payload = result[payload_column].as_primitive::(); + let actual_payload = (!payload.is_null(0)).then(|| payload.value(0)); + assert_eq!(actual_payload, expected_payload); + } - // Create some new (unindexed) data - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(2)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(8)); - let ds = Dataset::write( - data, - test_uri, - Some(WriteParams { - mode: WriteMode::Append, - ..Default::default() - }), + /// merge_insert matches keys by bit pattern, in the indexed probe and in the + /// hash join behind it alike, so a source key of `+0.0` updates only the + /// `+0.0` row. Filters answer zero comparisons per IEEE 754 now, and this + /// pins that the two are still allowed to disagree: making key matching agree + /// needs the unindexed join fixed too, and DataFusion 54 hashes join keys by + /// raw bits. Both settings of `use_index` are exercised; which one the planner + /// picks for a one-row source is not asserted. + #[rstest::rstest] + #[tokio::test] + async fn test_merge_insert_on_float_zero_key(#[values(true, false)] use_index: bool) { + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + + let target = record_batch!( + ( + "key", + Float64, + [Some(-1.0), Some(-0.0), Some(0.0), Some(1.0)] + ), + ("value", Int32, [10, 20, 30, 40]) ) - .await .unwrap(); - + let schema = target.schema(); + let reader = RecordBatchIterator::new(vec![Ok(target)], schema.clone()); + let mut ds = Dataset::write(reader, test_uri, None).await.unwrap(); + ds.create_index( + &["key"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + let source = record_batch!(("key", Float64, [Some(0.0)]), ("value", Int32, [99])).unwrap(); + let source = Box::new(RecordBatchIterator::new(vec![Ok(source)], schema.clone())); + + let (ds, _) = MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + .unwrap() + .when_not_matched(WhenNotMatched::DoNothing) + .when_matched(WhenMatched::UpdateAll) + .use_index(use_index) + .try_build() + .unwrap() + .execute_reader(source) + .await + .unwrap(); + + // Only the +0.0 row is updated. Checking both sides pins which row was + // replaced, not just how many; `value = 20` is the -0.0 row. + assert_eq!(ds.count_rows(None).await.unwrap(), 4); + for (filter, expected) in [("value = 99", 1), ("value = 30", 0), ("value = 20", 1)] { + assert_eq!( + ds.count_rows(Some(filter.to_string())).await.unwrap(), + expected, + "{filter}" + ); + } + } + + #[tokio::test] + async fn test_indexed_merge_insert() { + let test_dir = TempStrDir::default(); + let test_uri = &test_dir; + + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); + let schema = data.schema(); + + // Create an input dataset with a scalar index on key + let mut ds = Dataset::write(data, test_uri, None).await.unwrap(); + let index_params = ScalarIndexParams::default(); + ds.create_index(&["key"], IndexType::Scalar, None, &index_params, false) + .await + .unwrap(); + + // Create some new (unindexed) data + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(2)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(8)); + let ds = Dataset::write( + data, + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let ds = Arc::new(ds); let just_index_col = Schema::new(vec![Field::new("key", DataType::Utf8, false)]); @@ -4353,6 +5521,79 @@ mod tests { assert_eq!(inserted, 1); } + /// A composite index probe can over-match the exact join key, but a target + /// row reached by more than one source batch must still enter the join once. + #[tokio::test] + async fn test_indexed_merge_insert_deduplicates_cross_batch_candidates() { + let initial = record_batch!( + ("a", Int32, [1, 1, 2, 2]), + ("b", Int32, [10, 20, 10, 20]), + ("value", Int32, [100, 200, 300, 400]) + ) + .unwrap(); + let schema = initial.schema(); + + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial)], schema.clone()), + "memory://", + None, + ) + .await + .unwrap(); + + let params = ScalarIndexParams::default(); + dataset + .create_index(&["a"], IndexType::Scalar, None, ¶ms, false) + .await + .unwrap(); + dataset + .create_index(&["b"], IndexType::Scalar, None, ¶ms, false) + .await + .unwrap(); + + let first = record_batch!( + ("a", Int32, [1]), + ("b", Int32, [10]), + ("value", Int32, [901]) + ) + .unwrap(); + // This batch probes `a IN (1, 2) AND b IN (20, 10)`, which reaches + // (1, 10) again even though that tuple is not present in this batch. + let second = record_batch!( + ("a", Int32, [1, 2]), + ("b", Int32, [20, 10]), + ("value", Int32, [902, 903]) + ) + .unwrap(); + + let (dataset, stats) = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["a".to_string(), "b".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + vec![Ok(first), Ok(second)], + schema, + ))) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 3); + assert_eq!(stats.num_inserted_rows, 0); + assert_eq!(dataset.count_rows(None).await.unwrap(), 4); + for (a, b, value) in [(1, 10, 901), (1, 20, 902), (2, 10, 903), (2, 20, 400)] { + assert_eq!( + dataset + .count_rows(Some(format!("a = {a} AND b = {b} AND value = {value}"))) + .await + .unwrap(), + 1, + ); + } + } + /// Composite key merge_insert with no scalar index on any join column /// must keep working via the full-scan fallback. Guards against the /// indexed path becoming a hard requirement after this optimization. @@ -5237,6 +6478,116 @@ mod tests { ); } + /// The probe loop orders join keys by how many distinct values the source + /// batch holds for them, so writing a composite key coarse-to-fine + /// (`["bucket", "id"]`) does not make the coarse probe run first and + /// materialize a candidate set the size of the table. + /// + /// Asserted on the emitted candidate count, which is the only observable + /// that says which probe ran: `IndexMetrics` has no per-probe counter and + /// is shared across lookups. Probing the selective key first leaves 4 + /// candidates and stops, because 4 is already down to the source batch + /// size; following the caller's order instead would probe `bucket` first + /// (8 candidates, no stop), then intersect down to the 2 exact matches. So + /// the larger count is the cheaper plan — one probe instead of two — and + /// the surplus is trimmed by the full-key join downstream. A regression in + /// the ordering shows up here as 2. + /// + /// Sits with the other composite-key merge_insert tests, next to + /// `map_index_exec_multi_lookup_plan_shape`: probe ordering only matters on + /// the composite-key indexed path, and this is where that path is covered. + #[tokio::test] + async fn map_index_exec_probes_most_selective_key_first() { + use crate::io::exec::scalar_index::{IndexLookup, MapIndexExec}; + use arrow_array::types::UInt64Type; + use datafusion::physical_plan::ExecutionPlan; + use lance_datafusion::exec::OneShotExec; + + // `id` is unique; `bucket` takes two values, so a `bucket` probe alone + // reaches half the table while pruning almost nothing. + let initial = record_batch!( + ( + "id", + Int32, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + ), + ( + "bucket", + Int32, + [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1] + ) + ) + .unwrap(); + let schema = initial.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial)], schema), + "memory://", + None, + ) + .await + .unwrap(); + + let params = ScalarIndexParams::default(); + for column in ["id", "bucket"] { + dataset + .create_index( + &[column], + IndexType::Scalar, + Some(format!("{column}_idx")), + ¶ms, + false, + ) + .await + .unwrap(); + } + + // Columns are in lookup order, which is the deliberately bad one: the + // coarse key first. Only ids 0 and 2 also sit in bucket 0, so the exact + // composite match is 2 rows. + let probe = + record_batch!(("bucket", Int32, [0, 0, 0, 0]), ("id", Int32, [0, 1, 2, 3])).unwrap(); + let source_rows = probe.num_rows(); + let plan = MapIndexExec::new_multi( + Arc::new(dataset), + vec![ + IndexLookup::new("bucket", "bucket_idx"), + IndexLookup::new("id", "id_idx"), + ], + Arc::new(OneShotExec::from_batch(probe)), + ); + + let mut stream = plan + .execute(0, Arc::new(datafusion::execution::TaskContext::default())) + .unwrap(); + let mut candidates = Vec::new(); + while let Some(batch) = stream.next().await { + let batch = batch.unwrap(); + candidates.extend( + batch + .column(0) + .as_primitive::() + .values() + .iter() + .copied(), + ); + } + + assert_eq!( + candidates.len(), + source_rows, + "the selective key must be probed first and stop the loop, leaving \ + one candidate per source row; {} means the probes ran in `on` order", + candidates.len() + ); + // Whatever the order, the candidate set has to cover the exact matches. + for row_addr in [0_u64, 2] { + assert!( + candidates.contains(&row_addr), + "candidate set must contain exact match at row {row_addr}: {candidates:?}" + ); + } + } + mod subcols { use super::*; use rstest::rstest; @@ -5468,11 +6819,13 @@ mod tests { } else { // v2 path: partial-schema upserts run through the same // FullSchemaMergeInsertExec as full-schema upserts and - // write brand-new fragments. Fragment 1 is entirely + // write brand-new fragments. In-place column patching is + // opt-in (see the `test_merge_insert_subcols_in_place_*` + // tests), so the default stays here. Fragment 1 is entirely // matched (all 256 rows) so it is removed; fragment 2 is // partially matched so it keeps its id with a deletion - // vector; a new fragment holds the 270 updated rows - // (and the 2 inserted rows when `insert` is set). + // vector; a new fragment holds the 270 updated rows (and the + // 2 inserted rows when `insert` is set). let ids_after: Vec = fragments_after.iter().map(|f| f.id).collect(); assert_eq!( fragments_after.len(), @@ -5639,6 +6992,94 @@ mod tests { ); } + /// The opt-in counterpart of `test_merge_insert_subcols_v2_explain_plan`: + /// with in-place column writes allowed, the columns absent from the + /// source must not appear anywhere in the plan. That absence is the + /// read-side half of the win — the target scan projects only the join + /// key instead of carrying the filled column through the join. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_explain_plan() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; + + let job = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap(); + + let source_schema: Schema = new_data.schema().as_ref().clone(); + let plan = job + .explain_plan(Some(&source_schema), false) + .await + .expect("explain_plan must succeed for partial-schema upsert on v2"); + + assert!( + plan.contains("InPlaceMergeInsert: on=[key]") + && plan.contains("mode=RewriteColumns"), + "expected InPlaceMergeInsert node in plan, got: {}", + plan + ); + assert!( + plan.contains("HashJoinExec"), + "expected HashJoinExec in plan, got: {}", + plan + ); + // `UpdateAll` references no target column, so `other` is absent + // from the plan entirely. An `UpdateIf` whose condition reads an + // omitted column pulls it back into the target scan — see + // `test_merge_insert_subcols_in_place_update_if_reads_omitted_column`. + assert!( + !plan.contains("other"), + "column absent from the source must not be read or projected: {}", + plan + ); + } + + /// The read-side win is conditional: an `UpdateIf` condition that reads + /// a column the source omits pulls that column back into the target + /// scan, because the `__action` expression still references it. The + /// write-side win is unaffected — the column is read but never + /// rewritten. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_update_if_reads_omitted_column() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; + + let job = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::update_if(&ds, "target.other != 'zzzz'").unwrap()) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap(); + + let source_schema: Schema = new_data.schema().as_ref().clone(); + let plan = job + .explain_plan(Some(&source_schema), false) + .await + .expect("explain_plan must succeed for partial-schema upsert on v2"); + + assert!( + plan.contains("InPlaceMergeInsert: on=[key]"), + "the condition must not change the write sink: {plan}" + ); + assert!( + plan.contains("other"), + "a condition over an omitted column must keep it in the plan: {plan}" + ); + + // And the condition is genuinely evaluated against the stored + // values rather than silently seeing nulls: `other` is 4 random + // characters, so it never equals 'zzzz' and every match updates. + let reader = Box::new(RecordBatchIterator::new( + [Ok(new_data.clone())], + new_data.schema(), + )); + let (_, stats) = job.execute_reader(reader).await.unwrap(); + assert_eq!(stats.num_updated_rows, (new_data.num_rows() - 2) as u64); + } + /// Partial-schema upserts with `insert_not_matched=InsertAll` must /// reject non-nullable missing columns at the API boundary instead /// of producing a confusing downstream writer error. The user- @@ -5942,1685 +7383,2464 @@ mod tests { other => panic!("expected Operation::Update, got: {:?}", other), } } - } - // For some reason, Windows isn't able to handle the timeout test. Possibly - // a performance bug in their timer implementation? - #[cfg(not(windows))] - #[rstest::rstest] - #[case::all_success(Duration::from_secs(100_000))] - #[case::timeout(Duration::from_millis(200))] - #[tokio::test] - async fn test_merge_insert_concurrency(#[case] timeout: Duration) { - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false), - Field::new("value", DataType::UInt32, false), - ])); - // To benchmark scaling curve: measure how long to run - // - // And vary `concurrency` to see how it scales. Compare this again `main`. - let concurrency = 10; - let initial_data = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from_iter_values(0..concurrency)), - Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( - 0, - concurrency as usize, - ))), - ], - ) - .unwrap(); + /// An `UpdateIf` condition still routes in place, and rows the + /// condition rejects must keep their stored values rather than being + /// written back or dropped. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_update_if() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; - // Increase likelihood of contention by throttling the store - let throttled = Arc::new(ThrottledStoreWrapper { - config: ThrottleConfig { - // For benchmarking: Increase this to simulate object storage. - wait_list_per_call: Duration::from_millis(20), - wait_get_per_call: Duration::from_millis(20), - wait_put_per_call: Duration::from_millis(20), - ..Default::default() - }, - }); - let session = Arc::new(Session::default()); + // Only rewrite rows whose existing value is below the midpoint of + // the target's `value` range, so the condition rejects part of the + // matched set. + let job = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::update_if(&ds, "target.value < 400").unwrap()) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap(); - let mut dataset = InsertBuilder::new("memory://") - .with_params(&WriteParams { - store_params: Some(ObjectStoreParams { - object_store_wrapper: Some(throttled.clone()), - ..Default::default() - }), - session: Some(session.clone()), - ..Default::default() - }) - .execute(vec![initial_data]) - .await - .unwrap(); + let before = ds + .scan() + .scan_in_order(true) + .try_into_batch() + .await + .unwrap(); + let fragments_before = ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect::>(); - // do merge inserts in parallel based on the concurrency. Each will open the dataset, - // signal they have opened, and then wait for a signal to proceed. Once the signal - // is received, they will do a merge insert and close the dataset. + let reader = Box::new(RecordBatchIterator::new( + [Ok(new_data.clone())], + new_data.schema(), + )); + let (updated_ds, stats) = job.execute_reader(reader).await.unwrap(); - let barrier = Arc::new(Barrier::new(concurrency as usize)); - let mut handles = Vec::new(); - for i in 0..concurrency { - let session_ref = session.clone(); - let schema_ref = schema.clone(); - let barrier_ref = barrier.clone(); - let throttled_ref = throttled.clone(); - let handle = tokio::task::spawn(async move { - let dataset = DatasetBuilder::from_uri("memory://") - .with_read_params(ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(throttled_ref.clone()), - ..Default::default() - }), - session: Some(session_ref.clone()), - ..Default::default() - }) - .load() - .await - .unwrap(); - let dataset = Arc::new(dataset); + // 270 source rows match; the 256 rows of fragment 1 hold values + // 256..512, so only those below 400 are rewritten. + assert!( + stats.num_updated_rows > 0 && stats.num_updated_rows < 270, + "condition should accept some but not all matches, got {}", + stats.num_updated_rows + ); + assert_eq!(stats.num_deleted_rows, 0); - let new_data = RecordBatch::try_new( - schema_ref.clone(), - vec![ - Arc::new(UInt32Array::from(vec![i])), - Arc::new(UInt32Array::from(vec![1])), - ], - ) + let fragments_after = updated_ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect::>(); + assert_eq!( + fragments_before.iter().map(|f| f.id).collect::>(), + fragments_after.iter().map(|f| f.id).collect::>() + ); + + let after = updated_ds + .scan() + .scan_in_order(true) + .try_into_batch() + .await .unwrap(); - let source = Box::new(RecordBatchIterator::new([Ok(new_data)], schema_ref.clone())); + assert_eq!(after.num_rows(), before.num_rows()); - let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) + let index_by_key = |batch: &RecordBatch| { + let keys = batch + .column_by_name("key") .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .conflict_retries(100) - .retry_timeout(timeout) - .try_build() + .as_any() + .downcast_ref::() .unwrap(); - barrier_ref.wait().await; - - job.execute_reader(source) - .await - .map(|(_ds, stats)| stats.num_attempts) - }); - handles.push(handle); - } + let values = batch + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let others = batch + .column_by_name("other") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(|i| { + ( + keys.value(i).to_string(), + (values.value(i), others.value(i).to_string()), + ) + }) + .collect::>() + }; + let before_by_key = index_by_key(&before); + let after_by_key = index_by_key(&after); - let results = try_join_all(handles).await.unwrap(); + let new_keys = new_data + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let new_values = new_data + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); - for attempts in results.iter() { - match attempts { - Ok(attempts) => { - assert!(*attempts <= 10, "Attempt count should be <= 10"); - } - Err(err) => { - // If we get an error, it means the task was cancelled - // due to timeout. This is expected if the timeout is - // set to a low value. - assert!( - matches!(err, Error::TooMuchWriteContention { message, .. } if message.contains("failed on retry_timeout")), - "Expected TooMuchWriteContention error, got: {:?}", - err + let mut rewritten = 0u64; + for i in 0..new_data.num_rows() { + let key = new_keys.value(i).to_string(); + let Some((old_value, old_other)) = before_by_key.get(&key) else { + continue; // one of the two insert-only rows + }; + let (new_value, new_other) = after_by_key + .get(&key) + .unwrap_or_else(|| panic!("key {} disappeared", key)); + assert_eq!( + old_other, new_other, + "`other` is absent from the source and must never change" + ); + if *old_value < 400 { + assert_eq!( + *new_value, + new_values.value(i), + "key {} should be updated", + key + ); + rewritten += 1; + } else { + assert_eq!( + new_value, old_value, + "key {} was rejected by the condition and must keep its value", + key ); } } + assert_eq!(stats.num_updated_rows, rewritten); } - if timeout.as_secs() > 10 { - dataset.checkout_latest().await.unwrap(); - let batches = dataset.scan().try_into_batch().await.unwrap(); + /// A source that duplicates a key matches the same target row twice. + /// The in-place path must apply the same dedupe policy as the + /// row-rewrite one: fail by default, skip when told to keep the first. + #[rstest] + #[tokio::test] + async fn test_merge_insert_subcols_in_place_duplicate_keys( + #[values(false, true)] first_seen: bool, + ) { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; - let values = batches["value"].as_primitive::(); - assert!( - values.values().iter().all(|&v| v == 1), - "All values should be 1 after merge insert. Got: {:?}", - values - ); - } - } + // Repeat the first source row so two source rows carry the same key. + let dup = arrow_select::concat::concat_batches( + &new_data.schema(), + &[new_data.slice(0, 1), new_data.slice(0, 1)], + ) + .unwrap(); + let reader = Box::new(RecordBatchIterator::new([Ok(dup.clone())], dup.schema())); - #[tokio::test] - async fn test_merge_insert_large_concurrent() { - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false), - Field::new("value", DataType::UInt32, false), - ])); - let num_rows = 10; - let initial_data = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from_iter_values(0..num_rows)), - Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( - 0, - num_rows as usize, - ))), - ], - ) - .unwrap(); + let mut builder = + MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]).unwrap(); + builder + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns); + if first_seen { + builder.source_dedupe_behavior(SourceDedupeBehavior::FirstSeen); + } + let result = builder.try_build().unwrap().execute_reader(reader).await; - // Adding latency helps ensure we get contention - let throttled = Arc::new(ThrottledStoreWrapper { - config: ThrottleConfig { - wait_list_per_call: Duration::from_millis(10), - wait_get_per_call: Duration::from_millis(10), - ..Default::default() - }, - }); - let session = Arc::new(Session::default()); + if first_seen { + let (_, stats) = result.expect("FirstSeen must skip the duplicate"); + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_skipped_duplicates, 1); + } else { + let err = result.expect_err("duplicate keys must fail by default"); + assert!( + err.to_string().contains("Ambiguous merge inserts"), + "unexpected error: {err}" + ); + } + } - let dataset = InsertBuilder::new("memory://") - .with_params(&WriteParams { - store_params: Some(ObjectStoreParams { - object_store_wrapper: Some(throttled.clone()), + /// With stable row ids, patching columns in place must stamp + /// `_row_last_updated_at_version` on exactly the rows it rewrote and + /// leave every other row's stamp alone. Covers both branches inside + /// `update_fragments`: a fragment the source covers entirely (the + /// direct-write path) and one it covers partially (the updater path). + #[rstest] + #[tokio::test] + async fn test_merge_insert_subcols_in_place_stable_row_id_versions( + #[values(false, true)] full_fragment: bool, + ) { + use lance_core::ROW_LAST_UPDATED_AT_VERSION; + + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + Field::new("other", DataType::Utf8, true), + ])); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..8)), + Arc::new(StringArray::from(vec!["t"; 8])), + Arc::new(StringArray::from(vec!["o"; 8])), + ], + ) + .unwrap(); + // Two fragments of 4 rows each. + let ds = Dataset::write( + Box::new(RecordBatchIterator::new([Ok(initial)], schema.clone())), + "memory://", + Some(WriteParams { + max_rows_per_file: 4, + enable_stable_row_ids: true, ..Default::default() }), - session: Some(session.clone()), - ..Default::default() - }) - .execute(vec![initial_data]) + ) .await .unwrap(); - let dataset = Arc::new(dataset); + assert_eq!(ds.version().version, 1); + let ds = Arc::new(ds); - // Start one merge insert, but don't commit it yet. - let new_data1 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![1])), - Arc::new(UInt32Array::from(vec![1])), - ], - ) - .unwrap(); - let UncommittedMergeInsert { - transaction: transaction1, - .. - } = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap() - .execute_uncommitted(RecordBatchIterator::new( - vec![Ok(new_data1)], - schema.clone(), - )) - .await + // Either all of fragment 0, or two of its four rows. + let keys: Vec = if full_fragment { + vec![0, 1, 2, 3] + } else { + vec![1, 2] + }; + let source_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + ])); + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + Arc::new(UInt32Array::from(keys.clone())), + Arc::new(StringArray::from(vec!["patched"; keys.len()])), + ], + ) .unwrap(); - // Setup a "large" merge insert, with many batches - let new_data2 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from_iter_values(0..1000)), - Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n(2, 1000))), - ], - ) - .unwrap(); - let notify = Arc::new(Notify::new()); - let source = RecordBatchIterator::new( - (0..10) - .map(|i| { - let batch = new_data2.slice(i * 100, 100); - if i == 9 { - notify.notify_one(); - } - Ok(batch) - }) - .collect::>(), - schema.clone(), - ); - let dataset2 = DatasetBuilder::from_uri("memory://") - .with_read_params(ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(throttled.clone()), + let (updated_ds, stats) = + MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(source)], + source_schema, + ))) + .await + .unwrap(); + assert_eq!(stats.num_updated_rows, keys.len() as u64); + let new_version = updated_ds.version().version; + assert_eq!(new_version, 2); + assert_eq!( + updated_ds.get_fragments().len(), + 2, + "in-place patches must not add fragments" + ); + + let mut scanner = updated_ds.scan(); + scanner + .project(&["key", "tag", ROW_LAST_UPDATED_AT_VERSION]) + .unwrap(); + let result = scanner.try_into_batch().await.unwrap(); + assert_eq!(result.num_rows(), 8); + + let result_keys = result + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let tags = result + .column_by_name("tag") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let versions = result + .column_by_name(ROW_LAST_UPDATED_AT_VERSION) + .expect("stable row ids must expose the last-updated version") + .as_any() + .downcast_ref::() + .unwrap(); + + for i in 0..result.num_rows() { + let key = result_keys.value(i); + let was_patched = keys.contains(&key); + assert_eq!( + tags.value(i), + if was_patched { "patched" } else { "t" }, + "key {key} has the wrong tag" + ); + assert_eq!( + versions.value(i), + if was_patched { new_version } else { 1 }, + "key {key} has the wrong last-updated version" + ); + } + } + + /// The version `update_fragments` stamps into the row-version metadata is + /// only the version this transaction expected at prepare time. A + /// compatible commit can land first, making the real commit version + /// later, so the operation carries the patched offsets and + /// `build_manifest` re-stamps exactly those rows with the version the + /// commit actually got. + #[rstest] + #[tokio::test] + async fn test_merge_insert_subcols_in_place_stable_row_id_rebase( + #[values(false, true)] full_fragment: bool, + ) { + use lance_core::ROW_LAST_UPDATED_AT_VERSION; + + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + Field::new("other", DataType::Utf8, true), + ])); + let rows = |keys: Vec, tag: &str| { + let n = keys.len(); + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(keys)), + Arc::new(StringArray::from(vec![tag; n])), + Arc::new(StringArray::from(vec!["o"; n])), + ], + ) + .unwrap() + }; + let ds = Dataset::write( + Box::new(RecordBatchIterator::new( + [Ok(rows(vec![0, 1], "t"))], + schema.clone(), + )), + "memory://", + Some(WriteParams { + enable_stable_row_ids: true, ..Default::default() }), - session: Some(session.clone()), - ..Default::default() - }) - .load() + ) .await .unwrap(); - let job = MergeInsertBuilder::try_new(Arc::new(dataset2), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap() - .execute_reader(source); - let task = tokio::task::spawn(job); + assert_eq!(ds.version().version, 1); + let ds = Arc::new(ds); - // Right as the large merge insert has finished reading the last batch, - // we will commit the first merge insert. This should trigger a conflict, - // but we should resolve it automatically. - notify.notified().await; - let mut dataset = CommitBuilder::new(dataset) - .execute(transaction1) - .await + // Either both rows of the only fragment (the no-read-back write + // path) or one of them (the updater path). Both stamp the version. + let keys: Vec = if full_fragment { vec![0, 1] } else { vec![0] }; + let source_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + ])); + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + Arc::new(UInt32Array::from(keys.clone())), + Arc::new(StringArray::from(vec!["patched"; keys.len()])), + ], + ) .unwrap(); + let stream = RecordBatchStreamAdapter::new( + source_schema.clone(), + futures::stream::iter([Ok(source)]), + ); - task.await.unwrap().unwrap(); - dataset.checkout_latest().await.unwrap(); - - let batches = dataset.scan().try_into_batch().await.unwrap(); - let values = batches["value"].as_primitive::(); - assert!( - values.values().iter().all(|&v| v == 2), - "All values should be 1 after merge insert. Got: {:?}", - values - ); - } - - #[tokio::test] - async fn test_merge_insert_updates_indices() { - let test_dataset = async || { - let mut dataset = lance_datagen::gen_batch() - .col("id", array::step::()) - .col("value", array::step::()) - .col("other_value", array::step::()) - .into_ram_dataset(FragmentCount::from(4), FragmentRowCount::from(20)) - .await - .unwrap(); + // Prepare against v1 but do not commit yet. + let UncommittedMergeInsert { transaction, .. } = + MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .execute_uncommitted(Box::pin(stream) as SendableRecordBatchStream) + .await + .unwrap(); + match &transaction.operation { + Operation::Update { + updated_fragment_offsets, + .. + } => { + let UpdatedFragmentOffsets(off_map) = updated_fragment_offsets + .as_ref() + .expect("a RewriteColumns update must carry its patched offsets"); + let frag_id = ds.get_fragments()[0].id() as u64; + let offsets = off_map + .get(&frag_id) + .expect("the patched fragment must be present"); + assert_eq!( + offsets.iter().collect::>(), + keys.clone(), + "the offsets must name exactly the patched rows" + ); + } + other => panic!("expected Operation::Update, got: {other:?}"), + } - dataset - .create_index( - &["id"], - IndexType::BTree, + // A concurrent append commits as v2, so the prepared transaction + // lands on v3 rather than the v2 it stamped. + let mut appended = ds.as_ref().clone(); + appended + .append( + RecordBatchIterator::new([Ok(rows(vec![2], "t"))], schema.clone()), None, - &ScalarIndexParams::default(), - false, ) .await .unwrap(); - dataset - .create_index( - &["value"], - IndexType::BTree, - None, - &ScalarIndexParams::default(), - false, - ) + assert_eq!(appended.version().version, 2); + + let committed = CommitBuilder::new(Arc::new(appended)) + .execute(transaction) .await .unwrap(); - dataset - .create_index( - &["other_value"], - IndexType::BTree, - None, - &ScalarIndexParams::default(), - false, - ) - .await + let commit_version = committed.version().version; + assert_eq!(commit_version, 3); + + let mut scanner = committed.scan(); + scanner + .project(&["key", "tag", ROW_LAST_UPDATED_AT_VERSION]) .unwrap(); - Arc::new(dataset) - }; + let result = scanner.try_into_batch().await.unwrap(); + assert_eq!(result.num_rows(), 3); - let check_indices = async |dataset: &Dataset, id_frags: &[u32], value_frags: &[u32]| { - let id_index = dataset - .load_scalar_index(IndexCriteria::default().with_name("id_idx")) - .await + let result_keys = result + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let tags = result + .column_by_name("tag") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let versions = result + .column_by_name(ROW_LAST_UPDATED_AT_VERSION) + .expect("stable row ids must expose the last-updated version") + .as_any() + .downcast_ref::() .unwrap(); - if id_frags.is_empty() { - assert!(id_index.is_none()); - } else { - let id_index = id_index.unwrap(); - let id_frags_bitmap = RoaringBitmap::from_iter(id_frags.iter().copied()); - // Check the effective bitmap (raw bitmap intersected with existing fragments) - let effective_bitmap = id_index - .effective_fragment_bitmap(&dataset.fragment_bitmap) - .unwrap(); - assert_eq!(effective_bitmap, id_frags_bitmap); + for i in 0..result.num_rows() { + let key = result_keys.value(i); + let was_patched = keys.contains(&key); + assert_eq!( + tags.value(i), + if was_patched { "patched" } else { "t" }, + "key {key} has the wrong tag" + ); + let expected = if was_patched { + // Patched by the rebased transaction: the version it landed + // on, not the version it guessed. + commit_version + } else if key == 2 { + // Appended by the concurrent commit. + 2 + } else { + // Untouched by either. + 1 + }; + assert_eq!( + versions.value(i), + expected, + "key {key} has the wrong last-updated version" + ); } + } - let value_index = dataset - .load_scalar_index(IndexCriteria::default().with_name("value_idx")) + /// An in-place column rewrite fills a replacement file covering every row + /// of each fragment it touches, from the snapshot it read, and + /// `build_manifest` then tombstones every overlay for the fields it + /// rewrote. So committing one over a concurrent overlay would drop that + /// overlay's value on a row the merge never matched. The conflict + /// resolver must retry instead, and the retry re-reads the overlaid + /// value. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_rebase_over_overlay() { + use crate::dataset::WriteDestination; + use crate::dataset::transaction::DataOverlayGroup; + use arrow_array::ArrayRef; + use lance_file::writer::FileWriterOptions; + use lance_io::utils::CachedFileSize; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + // Omitted by the source, so `RewriteColumns` has something to skip. + Field::new("other", DataType::Utf8, true), + ])); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0, 1])), + Arc::new(StringArray::from(vec!["base0", "base1"])), + Arc::new(StringArray::from(vec!["o", "o"])), + ], + ) + .unwrap(); + let ds = Arc::new( + Dataset::write( + Box::new(RecordBatchIterator::new([Ok(initial)], schema.clone())), + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) .await - .unwrap(); + .unwrap(), + ); + assert_eq!(ds.version().version, 1); + let tag_field_id = ds.schema().field("tag").unwrap().id; + let fragment_id = ds.get_fragments()[0].id() as u64; - if value_frags.is_empty() { - assert!(value_index.is_none()); - } else { - let value_index = value_index.unwrap(); - let value_frags_bitmap = RoaringBitmap::from_iter(value_frags.iter().copied()); - // Check the effective bitmap (raw bitmap intersected with existing fragments) - let effective_bitmap = value_index - .effective_fragment_bitmap(&dataset.fragment_bitmap) + // Prepare an in-place patch of key 0 against v1, without committing. + let source_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + ])); + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0])), + Arc::new(StringArray::from(vec!["merge0"])), + ], + ) + .unwrap(); + let stream = RecordBatchStreamAdapter::new( + source_schema.clone(), + futures::stream::iter([Ok(source)]), + ); + let UncommittedMergeInsert { transaction, .. } = + MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .execute_uncommitted(Box::pin(stream) as SendableRecordBatchStream) + .await .unwrap(); - assert_eq!(effective_bitmap, value_frags_bitmap); - } - let other_value_index = dataset - .load_scalar_index(IndexCriteria::default().with_name("other_value_idx")) + // A concurrent overlay commits as v2, changing `tag` on key 1, which + // the prepared merge did not match. + let overlay_schema = ds.schema().project_by_ids(&[tag_field_id], true); + let path = ds.base.clone().join("data").join("overlay.lance"); + let obj_writer = ds.object_store.create(&path).await.unwrap(); + let mut writer = lance_file::versions::v2_1::create_writer( + obj_writer, + overlay_schema, + FileWriterOptions::default(), + ) + .unwrap(); + writer + .write_column(0, Arc::new(StringArray::from(vec!["overlay1"])) as ArrayRef) .await - .unwrap() - .unwrap(); - - // The other_value index retains its original bitmap [0,1,2,3] since - // partial merges that don't modify other_value won't prune it. - let effective_bitmap = other_value_index - .effective_fragment_bitmap(&dataset.fragment_bitmap) .unwrap(); - - // The effective bitmap is the intersection of the index's original bitmap - // and the current dataset fragments. Since other_value is not modified by - // partial merges, it retains its validity for fragments it was originally trained on - // that still exist in the dataset. - let index_bitmap = other_value_index.fragment_bitmap.as_ref().unwrap(); - let expected_bitmap = index_bitmap & dataset.fragment_bitmap.as_ref(); - assert_eq!( - effective_bitmap, expected_bitmap, - "other_value index effective bitmap should be intersection. index_bitmap: {:?}, dataset_fragments: {:?}, effective_bitmap: {:?}", - index_bitmap, dataset.fragment_bitmap, effective_bitmap + let summary = writer.finish().await.unwrap(); + let mut data_file = DataFile::new_unstarted( + "overlay.lance".to_string(), + lance_file::version::ConcreteFileVersion::V2_1, ); - }; - - let dataset = test_dataset().await; - - // Sanity test on the initial dataset - check_indices(&dataset, &[0, 1, 2, 3], &[0, 1, 2, 3]).await; - - // Vertical merge insert (full schema), one fragment is deleted and should be removed from - // the index. - let merge_insert = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap(); - - let (dataset, _) = merge_insert - .execute_reader( - lance_datagen::gen_batch() - .col("id", array::step_custom::(50, 1)) - .col("value", array::step_custom::(50, 1)) - .col("other_value", array::step_custom::(50, 1)) - .into_df_stream(RowCount::from(40), BatchCount::from(1)), + data_file.fields = vec![tag_field_id].into(); + data_file.column_indices = vec![0].into(); + data_file.file_size_bytes = CachedFileSize::new(summary.size_bytes); + let overlaid = Dataset::commit( + WriteDestination::Dataset(ds.clone()), + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file, + // Physical offset 1 is key 1. + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([1u32])), + committed_version: 0, + }], + }], + }, + Some(1), + None, + None, + Arc::new(Default::default()), + false, ) .await .unwrap(); + assert_eq!(overlaid.version().version, 2); + let tag_of = |ds: &Dataset, key: u32| { + let ds = ds.clone(); + async move { + let mut scanner = ds.scan(); + scanner + .filter(&format!("key = {key}")) + .unwrap() + .project(&["tag"]) + .unwrap(); + let batch = scanner.try_into_batch().await.unwrap(); + batch + .column_by_name("tag") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + .to_string() + } + }; + assert_eq!(tag_of(&overlaid, 1).await, "overlay1"); - // Fragment 3 removed and correctly removed from the index bitmap. - check_indices(&dataset, &[0, 1, 2], &[0, 1, 2]).await; - - // Now we do the same thing with a partial merge insert (only id and value) - let dataset = test_dataset().await; - - // Vertical merge insert (full schema), one fragment is deleted and should be removed from - // the index. - let merge_insert = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap(); + // The prepared transaction must not be able to commit as-is: doing so + // would tombstone the overlay and resurrect "base1". + let error = CommitBuilder::new(Arc::new(overlaid.clone())) + .with_max_retries(0) + .execute(transaction) + .await + .expect_err("committing over an overlapping overlay must conflict"); + assert!( + matches!(error, Error::RetryableCommitConflict { .. }), + "expected a retryable conflict, got: {error:?}" + ); - let (dataset, _) = merge_insert - .execute_reader( - lance_datagen::gen_batch() - .col("id", array::step_custom::(50, 1)) - .col("value", array::step_custom::(50, 1)) - .into_df_stream(RowCount::from(40), BatchCount::from(1)), + // Re-running the merge against the overlaid dataset patches key 0 and + // leaves the overlay on key 1 intact. + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0])), + Arc::new(StringArray::from(vec!["merge0"])), + ], ) - .await - .unwrap(); - - // Fragment 3 is fully removed. We could keep it technically but today it is removed - // which is also fine. Fragment 2 is partially and must be removed. - // - // TODO: We should not be modifying the id_index here. A merge_insert should not need - // to rewrite the id field. However, it seems we are doing that today. This should be - // fixed in - check_indices(&dataset, &[0, 1], &[0, 1]).await; - - // One more test but this time we touch all fragments which causes the index to be removed - // entirely. - let dataset = test_dataset().await; - - // Vertical merge insert (full schema), one fragment is deleted and should be removed from - // the index. - let merge_insert = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() .unwrap(); + let (retried, _) = + MergeInsertBuilder::try_new(Arc::new(overlaid), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(source)], + source_schema, + ))) + .await + .unwrap(); + assert_eq!(tag_of(&retried, 0).await, "merge0"); + assert_eq!( + tag_of(&retried, 1).await, + "overlay1", + "the overlay on an unmatched row must survive the retried merge" + ); + } - let (dataset, _) = merge_insert - .execute_reader( - lance_datagen::gen_batch() - .col("id", array::step_custom::(10, 1)) - .col("value", array::step_custom::(10, 1)) - .into_df_stream(RowCount::from(80), BatchCount::from(1)), - ) - .await - .unwrap(); + /// The overlay resolution rules require an in-place column rewrite to + /// tombstone any overlay covering the fields it replaced, otherwise the + /// stale overlay value keeps shadowing the freshly written one. + /// `build_manifest` does that off `fields_modified`, so this asserts the + /// in-place path reports them. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_reports_fields_modified() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; - check_indices(&dataset, &[], &[]).await; - } + // `value` is field id 1 in the dataset schema; `key` (the join key) + // is written too because the source carries it. + let value_field_id = ds.schema().field("value").unwrap().id as u32; + let key_field_id = ds.schema().field("key").unwrap().id as u32; + let other_field_id = ds.schema().field("other").unwrap().id as u32; - #[tokio::test] - async fn test_upsert_concurrent_full_frag() { - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false), - Field::new("value", DataType::UInt32, false), - ])); - let initial_data = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![0, 1])), - Arc::new(UInt32Array::from(vec![0, 0])), - ], - ) - .unwrap(); + let stream = RecordBatchStreamAdapter::new( + new_data.schema(), + futures::stream::iter(vec![Ok(new_data.clone())]), + ); + let UncommittedMergeInsert { transaction, .. } = + MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .execute_uncommitted(Box::pin(stream) as SendableRecordBatchStream) + .await + .unwrap(); - // Increase likelihood of contention by throttling the store - let throttled = Arc::new(ThrottledStoreWrapper { - config: ThrottleConfig { - wait_list_per_call: Duration::from_millis(5), - wait_get_per_call: Duration::from_millis(5), - wait_put_per_call: Duration::from_millis(5), - ..Default::default() - }, - }); - let session = Arc::new(Session::default()); + match &transaction.operation { + Operation::Update { + fields_modified, + update_mode, + .. + } => { + assert!(matches!(update_mode, Some(RewriteColumns))); + assert!( + fields_modified.contains(&value_field_id) + && fields_modified.contains(&key_field_id), + "patched fields must be reported, got {fields_modified:?}" + ); + assert!( + !fields_modified.contains(&other_field_id), + "a field the source never provided must not be reported as modified, \ + got {fields_modified:?}" + ); + } + other => panic!("expected Operation::Update, got: {other:?}"), + } + } - let mut dataset = InsertBuilder::new("memory://") - .with_params(&WriteParams { - store_params: Some(ObjectStoreParams { - object_store_wrapper: Some(throttled.clone()), - ..Default::default() - }), - session: Some(session.clone()), + /// Fragments can carry the same column in different data-file layouts: + /// one whose `tag` still lives in the file it was written with, another + /// whose `tag` has already been moved to a patch file by an earlier + /// merge. `Operation::DataReplacement` refuses that case outright, so + /// this asserts `RewriteColumns` does not inherit the restriction — it + /// appends a new data file per fragment rather than swapping one in, so + /// each fragment's existing layout is irrelevant. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_heterogeneous_layouts() { + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + Field::new("other", DataType::Utf8, true), + ])); + let rows = |keys: Vec| { + let n = keys.len(); + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(keys)), + Arc::new(StringArray::from(vec!["t"; n])), + Arc::new(StringArray::from(vec!["o"; n])), + ], + ) + .unwrap() + }; + let write_params = WriteParams { + max_rows_per_file: 2, ..Default::default() - }) - .execute(vec![initial_data]) - .await - .unwrap(); - - // Each merge insert will update one row. Combined, they should delete - // all rows in the first fragment, and it should be dropped. - let barrier = Arc::new(Barrier::new(2)); - let mut handles = Vec::new(); - for i in 0..2 { - let new_data = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![i])), - Arc::new(UInt32Array::from(vec![1])), - ], + }; + let ds = Dataset::write( + Box::new(RecordBatchIterator::new( + [Ok(rows(vec![0, 1]))], + schema.clone(), + )), + "memory://", + Some(write_params.clone()), ) + .await .unwrap(); - let source = Box::new(RecordBatchIterator::new([Ok(new_data)], schema.clone())); - let dataset_ref = Arc::new(dataset.clone()); - let barrier = barrier.clone(); - let handle = tokio::spawn(async move { - barrier.wait().await; - MergeInsertBuilder::try_new(dataset_ref, vec!["id".to_string()]) + let patch_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("tag", DataType::Utf8, true), + ])); + let patch = |keys: Vec, tag: &'static str| { + let n = keys.len(); + RecordBatch::try_new( + patch_schema.clone(), + vec![ + Arc::new(UInt32Array::from(keys)), + Arc::new(StringArray::from(vec![tag; n])), + ], + ) + .unwrap() + }; + async fn run_patch(ds: Arc, batch: RecordBatch) -> (Arc, MergeStats) { + let schema = batch.schema(); + MergeInsertBuilder::try_new(ds, vec!["key".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) .try_build() .unwrap() - .execute_reader(source) + .execute_reader(Box::new(RecordBatchIterator::new([Ok(batch)], schema))) .await - .unwrap(); - }); - handles.push(handle); - } - try_join_all(handles).await.unwrap(); - - dataset.checkout_latest().await.unwrap(); - assert!( - dataset - .get_fragments() - .iter() - .all(|f| f.metadata().num_rows().unwrap() > 0), - "No fragments should have zero rows after upsert" - ); + .unwrap() + } - let batches = dataset.scan().try_into_batch().await.unwrap(); - let values = batches["value"].as_primitive::(); - assert!( - values.values().iter().all(|&v| v == 1), - "All values should be 1 after merge insert. Got: {:?}", - values - ); - } + // Patch fragment 0 so its `tag` moves into a second data file and + // the original file's `tag` field id is tombstoned. + let (ds, _) = run_patch(Arc::new(ds), patch(vec![0, 1], "first")).await; + let layout_of = |ds: &Dataset, frag: usize| { + ds.get_fragments()[frag] + .metadata() + .files + .iter() + .map(|f| f.fields.to_vec()) + .collect::>() + }; + let frag0_layout = layout_of(&ds, 0); + assert_eq!( + frag0_layout.len(), + 2, + "frag 0 should now carry a patch file" + ); - #[tokio::test] - async fn test_plan_upsert() { - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(1)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); - let _schema = data.schema(); + // Append a fresh fragment, whose `tag` is still in its original file. + let mut ds = Arc::unwrap_or_clone(ds); + ds.append( + Box::new(RecordBatchIterator::new( + [Ok(rows(vec![2, 3]))], + schema.clone(), + )), + Some(write_params), + ) + .await + .unwrap(); + let ds = Arc::new(ds); + assert_eq!( + layout_of(&ds, 1).len(), + 1, + "frag 1 should be a single unpatched file" + ); + assert_ne!( + frag0_layout, + layout_of(&ds, 1), + "the two fragments must disagree on where `tag` lives for this test to mean anything" + ); - // Create dataset with initial data - let ds = Dataset::write(data, "memory://", None).await.unwrap(); + // Patch across both layouts in one merge. + let (updated_ds, stats) = run_patch(ds, patch(vec![0, 2], "patched")).await; + assert_eq!(stats.num_updated_rows, 2); + assert_eq!( + updated_ds.get_fragments().len(), + 2, + "in-place patches must not add fragments" + ); - // Create upsert job - let merge_insert_job = - crate::dataset::MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + let result = updated_ds + .scan() + .scan_in_order(true) + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 4); + let keys = result + .column_by_name("key") .unwrap() - .when_matched(crate::dataset::WhenMatched::UpdateAll) - .try_build() + .as_any() + .downcast_ref::() .unwrap(); + let tags = result + .column_by_name("tag") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let others = result + .column_by_name("other") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..result.num_rows() { + let key = keys.value(i); + let expected_tag = match key { + 0 | 2 => "patched", + // Patched in the first merge, untouched by the second. + 1 => "first", + _ => "t", + }; + assert_eq!(tags.value(i), expected_tag, "key {key} has the wrong tag"); + assert_eq!( + others.value(i), + "o", + "key {key}: a column the source never provided must not change" + ); + } + } - // Create new data for upsert - let new_data = lance_datagen::gen_batch() - .with_seed(Seed::from(2)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); - let new_data_stream = reader_to_stream(Box::new(new_data)); - - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); - - // Assert the plan structure using portable plan matching - // The optimized plan should have: - // 1. FullSchemaMergeInsertExec at the top - // 2. ProjectionExec that creates action based on _rowaddr nullness (sentinel is constant - // true so DataFusion folds `sentinel IS NOT NULL` away from the CASE expression) - // 3. HashJoin with projection that includes the sentinel column - // 4. LanceScan that only reads the key column (projection pushdown working!) - // 5. ProjectionExec on the source side that materializes the sentinel literal - assert_plan_node_equals( - plan, - "MergeInsert: on=[key], when_matched=UpdateAll, when_not_matched=InsertAll, when_not_matched_by_source=Keep - CoalescePartitionsExec - ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NULL THEN 2 WHEN _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action] - HashJoinExec: mode=CollectLeft, join_type=Right, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] - LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, \ - row_id=true, row_addr=true, full_filter=--, refine_filter=-- - RepartitionExec: partitioning=RoundRobinBatch(...), input_partitions=1 - ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] - StreamingTableExec: partition_sizes=1, projection=[value, key]" - ).await.unwrap(); - } - - #[tokio::test] - async fn test_fast_path_update_only() { - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(1)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); + /// An `UpdateIf` condition that rejects every matched row leaves the + /// patch stream empty. The in-place path must still commit cleanly — + /// no new fragments (which its `debug_assert` requires), no fragment + /// churn, and no value changed. + #[tokio::test] + async fn test_merge_insert_subcols_in_place_all_rows_rejected() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; - // Create dataset with initial data - let ds = Dataset::write(data, "memory://", None).await.unwrap(); + let before = ds + .scan() + .scan_in_order(true) + .try_into_batch() + .await + .unwrap(); + let fragments_before = ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect::>(); - // Create update-only job (insert_not_matched = false) - let merge_insert_job = - crate::dataset::MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + // `value` is a monotonic step starting at 0, so nothing exceeds + // this bound and every matched row becomes `Action::Nothing`. + let job = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) .unwrap() - .when_matched(crate::dataset::WhenMatched::UpdateAll) - .when_not_matched(crate::dataset::WhenNotMatched::DoNothing) + .when_matched(WhenMatched::update_if(&ds, "target.value > 999999").unwrap()) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) .try_build() .unwrap(); + let reader = Box::new(RecordBatchIterator::new( + [Ok(new_data.clone())], + new_data.schema(), + )); + let (updated_ds, stats) = job + .execute_reader(reader) + .await + .expect("an all-rejected in-place merge must not fail"); - // Create new data for update - let new_data = lance_datagen::gen_batch() - .with_seed(Seed::from(2)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); - let new_data_stream = reader_to_stream(Box::new(new_data)); + assert_eq!(stats.num_updated_rows, 0); + assert_eq!(stats.num_deleted_rows, 0); + assert_eq!(stats.num_inserted_rows, 0); + assert_eq!( + updated_ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect::>(), + fragments_before, + "rejecting every row must leave the fragments untouched" + ); + let after = updated_ds + .scan() + .scan_in_order(true) + .try_into_batch() + .await + .unwrap(); + assert_eq!(before, after, "no value should have changed"); + } - // This should use the fast path (execute_uncommitted_v2) - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); + /// `Auto` and an explicit `RewriteRows` must plan identically, and an + /// operation blocked several ways must have every blocker named rather + /// than only the first one found. + #[tokio::test] + async fn test_merge_insert_write_mode_auto_and_explicit_rows_agree() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; + let source_schema: Schema = new_data.schema().as_ref().clone(); - // The optimized plan should use Inner join instead of Right join since we're not - // inserting unmatched rows. The sentinel IS NOT NULL condition is folded away by - // DataFusion because the sentinel is lit(true), so the CASE only checks _rowaddr. - assert_plan_node_equals( - plan, - "MergeInsert: on=[key], when_matched=UpdateAll, when_not_matched=DoNothing, when_not_matched_by_source=Keep - CoalescePartitionsExec - ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] - LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=-- - RepartitionExec... - ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] - StreamingTableExec: partition_sizes=1, projection=[value, key]" - ).await.unwrap(); - } + let plan_for = |mode: Option| { + let ds = ds.clone(); + let source_schema = source_schema.clone(); + async move { + let mut builder = + MergeInsertBuilder::try_new(ds, vec!["key".to_string()]).unwrap(); + builder + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing); + if let Some(mode) = mode { + builder.write_mode(mode); + } + builder + .try_build() + .unwrap() + .explain_plan(Some(&source_schema), false) + .await + .unwrap() + } + }; + assert_eq!( + plan_for(None).await, + plan_for(Some(MergeInsertWriteMode::RewriteRows)).await, + "Auto must plan the same as an explicit RewriteRows" + ); - #[tokio::test] - async fn test_fast_path_conditional_update() { - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(1)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); + // Inserts and delete-by-source both block column patching, and the + // source here covers every dataset column too. + let full_schema: Schema = ds.schema().into(); + let error = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched_by_source(WhenNotMatchedBySource::Delete) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .explain_plan(Some(&full_schema), false) + .await + .expect_err("a merge blocked several ways must be rejected"); + let message = error.to_string(); + for blocker in ["covers every dataset column", "adds rows", "removes rows"] { + assert!( + message.contains(blocker), + "the error should name {blocker:?}: {message}" + ); + } + } - // Create dataset with initial data - let ds = Dataset::write(data, "memory://", None).await.unwrap(); + /// A scalar index on the join key routes a partial-schema update to the + /// legacy path, which patches columns unconditionally. So an explicit + /// `RewriteRows` has to leave that path to be honored, while `Auto` + /// keeps the index probe and stays on it. + #[rstest] + #[tokio::test] + async fn test_merge_insert_rewrite_rows_leaves_indexed_path( + #[values(None, Some(MergeInsertWriteMode::Auto))] auto: Option, + ) { + let Fixtures { ds, new_data } = Box::pin(setup(true)).await; + let source_schema: Schema = new_data.schema().as_ref().clone(); - // Create conditional update job (WhenMatched::UpdateIf) - let merge_insert_job = crate::dataset::MergeInsertBuilder::try_new( - Arc::new(ds.clone()), - vec!["key".to_string()], - ) - .unwrap() - .when_matched(crate::dataset::WhenMatched::update_if(&ds, "source.value > 20").unwrap()) - .when_not_matched(crate::dataset::WhenNotMatched::DoNothing) - .try_build() - .unwrap(); + let job = |mode: Option| { + let ds = ds.clone(); + async move { + let mut builder = + MergeInsertBuilder::try_new(ds, vec!["key".to_string()]).unwrap(); + builder + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing); + if let Some(mode) = mode { + builder.write_mode(mode); + } + builder.try_build().unwrap() + } + }; - // Create new data for conditional update - let new_data = lance_datagen::gen_batch() - .with_seed(Seed::from(2)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let new_data_reader = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); - let new_data_stream = reader_to_stream(Box::new(new_data_reader)); + // The indexed path has no physical plan, so `explain_plan` rejecting + // the job is how "stayed on the legacy path" is observable. + let error = job(auto) + .await + .explain_plan(Some(&source_schema), false) + .await + .expect_err("Auto must keep the scalar-index route, which has no plan"); + assert!( + error.to_string().contains("does not support explain_plan"), + "unexpected error: {error}" + ); - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); + let plan = job(Some(MergeInsertWriteMode::RewriteRows)) + .await + .explain_plan(Some(&source_schema), false) + .await + .expect("RewriteRows must fall through to the v2 plan"); + assert!( + plan.contains("MergeInsert: on=[key]") && !plan.contains("mode=RewriteColumns"), + "expected the row-rewrite v2 node, got: {plan}" + ); - // The optimized plan should use Inner join and include the UpdateIf condition. - // The sentinel IS NOT NULL condition is folded away (sentinel is lit(true)). - assert_plan_node_equals( - plan, - "MergeInsert: on=[key], when_matched=UpdateIf(source.value > 20), when_not_matched=DoNothing, when_not_matched_by_source=Keep - CoalescePartitionsExec - ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NOT NULL AND value@2 > 20 THEN 1 ELSE 0 END as __action] - HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] - LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=-- - RepartitionExec... - ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] - StreamingTableExec: partition_sizes=1, projection=[value, key]" - ).await.unwrap(); - } + // Patching columns is what the indexed path already does, so asking + // for it explicitly keeps the index probe. + let error = job(Some(MergeInsertWriteMode::RewriteColumns)) + .await + .explain_plan(Some(&source_schema), false) + .await + .expect_err("RewriteColumns must keep the scalar-index route"); + assert!( + error.to_string().contains("does not support explain_plan"), + "unexpected error: {error}" + ); + } - /// Verifies that a default find-or-create merge insert - /// (`WhenMatched::DoNothing` + `WhenNotMatched::InsertAll`) is routed - /// through the v2 `FullSchemaMergeInsertExec` path. Prior to this - /// change, `can_use_create_plan` rejected `DoNothing` outright and the - /// operation fell back to the legacy v1 `Merger`; the assertion below - /// would fail on `main`. See lance-format/lance#6441. - #[tokio::test] - async fn test_fast_path_find_or_create() { - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(1)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); + /// A source carrying nothing but the join key has no values to write, so + /// `RewriteColumns` cannot help it: the patch would reproduce the key + /// column byte for byte at the cost of a full-fragment column file and + /// the invalidation of every index over that key. Asking for it must + /// error rather than quietly writing whole rows instead. + #[tokio::test] + async fn test_merge_insert_rewrite_columns_rejects_key_only_source() { + let Fixtures { ds, new_data } = Box::pin(setup(false)).await; - // Create dataset with initial data - let ds = Dataset::write(data, "memory://", None).await.unwrap(); + let key_only = new_data.project(&[0]).unwrap(); + assert_eq!(key_only.schema().field(0).name(), "key"); + let source_schema: Schema = key_only.schema().as_ref().clone(); - // Default MergeInsertBuilder config is find-or-create: - // when_matched = DoNothing, when_not_matched = InsertAll. - let merge_insert_job = - crate::dataset::MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + let error = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .explain_plan(Some(&source_schema), false) + .await + .expect_err("a key-only source must be rejected, not silently rewritten"); + assert!( + matches!(error, Error::InvalidInput { .. }), + "got: {error:?}" + ); + let message = error.to_string(); + assert!( + message.contains("no column besides the join key"), + "the error must name the blocker: {message}" + ); + + // `Auto` accepts the same merge and writes whole rows. + let plan = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) .try_build() + .unwrap() + .explain_plan(Some(&source_schema), false) + .await .unwrap(); + assert!( + plan.contains("MergeInsert: on=[key]") && !plan.contains("InPlaceMergeInsert"), + "Auto should take the row-rewrite sink: {plan}" + ); + } - // Source data with a mix of already-present and new keys. - let new_data = lance_datagen::gen_batch() - .with_seed(Seed::from(2)) - .col("value", array::step::()) - .col("key", array::rand_pseudo_uuid_hex()); - let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); - let new_data_stream = reader_to_stream(Box::new(new_data)); - - // Should reach the v2 fast path (`create_plan` + FullSchemaMergeInsertExec). - // Dropping to v1 here would return an error from create_plan instead. - let plan = merge_insert_job.create_plan(new_data_stream).await.unwrap(); - - // The join is Right because we keep unmatched source rows (InsertAll) - // but discard unmatched target rows (DoNothing on when_matched, - // Keep on when_not_matched_by_source). The CASE expression simplifies - // to `_rowaddr IS NULL → Insert, else Nothing`. - assert_plan_node_equals( - plan, - "MergeInsert: on=[key], when_matched=DoNothing, when_not_matched=InsertAll, when_not_matched_by_source=Keep - CoalescePartitionsExec - ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NULL THEN 2 ELSE 0 END as __action] - HashJoinExec: mode=CollectLeft, join_type=Right, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] - LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=-- - RepartitionExec... - ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] - StreamingTableExec: partition_sizes=1, projection=[value, key]" - ) - .await - .unwrap(); - } - - #[tokio::test] - async fn test_skip_auto_cleanup() { - let tmpdir = TempStrDir::default(); - let dataset_uri = format!("{}/{}", tmpdir, "test_dataset"); - - // Create initial dataset with auto cleanup interval of 1 version - let data = lance_datagen::gen_batch() - .with_seed(Seed::from(1)) - .col("id", array::step::()) - .into_reader_rows(RowCount::from(100), BatchCount::from(1)); - - let mut auto_cleanup_params = HashMap::new(); - auto_cleanup_params.insert("lance.auto_cleanup.interval".to_string(), "1".to_string()); - auto_cleanup_params.insert( - "lance.auto_cleanup.older_than".to_string(), - "0ms".to_string(), - ); - - let write_params = WriteParams { - mode: WriteMode::Create, - auto_cleanup: Some(crate::dataset::AutoCleanupParams { - interval: 1, - older_than: chrono::TimeDelta::try_milliseconds(0).unwrap(), - }), - ..Default::default() - }; + /// A blob nested inside a struct the source provides cannot be patched: + /// the fragment updater reads blobs in their stored descriptor form, and + /// only the row-rewrite path converts between the two representations. + /// Rooting the check on the top-level fields the source carries, rather + /// than matching nested field names against the source's top-level + /// columns, is what makes the nested case visible. + #[tokio::test] + async fn test_merge_insert_rewrite_columns_rejects_nested_blob() { + use crate::{BlobArrayBuilder, blob_field}; + use arrow_array::{ArrayRef, StructArray}; + use arrow_schema::Fields; + + let info_fields: Fields = vec![ + Field::new("name", DataType::Utf8, false), + blob_field("blob", true), + ] + .into(); + let schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("info", DataType::Struct(info_fields.clone()), true), + Field::new("payload", DataType::Utf8, true), + ])); - // Start at 1 second after epoch - MockClock::set_system_time(std::time::Duration::from_secs(1)); + let mk_info = |n: usize, tag: &str| -> ArrayRef { + let mut builder = BlobArrayBuilder::new(n); + for i in 0..n { + builder + .push_bytes(format!("{tag}-blob-{i}").as_bytes()) + .unwrap(); + } + Arc::new( + StructArray::try_new( + info_fields.clone(), + vec![ + Arc::new(StringArray::from_iter_values( + (0..n).map(|i| format!("{tag}-name-{i}")), + )) as ArrayRef, + Arc::new(builder.finish().unwrap()) as ArrayRef, + ], + None, + ) + .unwrap(), + ) + }; - let dataset = Dataset::write(data, &dataset_uri, Some(write_params)) - .await + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..4)), + mk_info(4, "old"), + Arc::new(StringArray::from(vec!["p"; 4])), + ], + ) .unwrap(); - assert_eq!(dataset.version().version, 1); - - // Advance time - MockClock::set_system_time(std::time::Duration::from_secs(2)); - - // First merge insert WITHOUT skip_auto_cleanup - should trigger cleanup - let new_data = lance_datagen::gen_batch() - .with_seed(Seed::from(2)) - .col("id", array::step::()) - .into_df_stream(RowCount::from(50), BatchCount::from(1)); + let ds = Arc::new( + Dataset::write( + Box::new(RecordBatchIterator::new([Ok(initial)], schema.clone())), + "memory://", + Some(WriteParams { + // Blob v2 requires >= 2.2. + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); - let (dataset2, _) = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap() - .execute(new_data) - .await + // Omits `payload` (so the merge qualifies) but carries `info`, + // whose subtree holds a blob. + let source_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, false), + Field::new("info", DataType::Struct(info_fields.clone()), true), + ])); + let source = RecordBatch::try_new( + source_schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![1u32, 2])), + mk_info(2, "new"), + ], + ) .unwrap(); - assert_eq!(dataset2.version().version, 2); - - // Advance time - MockClock::set_system_time(std::time::Duration::from_secs(3)); - - // Need to do another merge insert for cleanup to take effect since cleanup runs on the old dataset - let new_data_extra = lance_datagen::gen_batch() - .with_seed(Seed::from(4)) - .col("id", array::step::()) - .into_df_stream(RowCount::from(10), BatchCount::from(1)); + let error = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns) + .try_build() + .unwrap() + .explain_plan(Some(source_schema.as_ref()), false) + .await + .expect_err("a nested blob must be rejected, not patched"); + assert!( + matches!(error, Error::InvalidInput { .. }), + "got: {error:?}" + ); + let message = error.to_string(); + assert!( + message.contains("blob column"), + "the error must name the blocker: {message}" + ); - let (dataset2_extra, _) = - MergeInsertBuilder::try_new(dataset2.clone(), vec!["id".to_string()]) + // `Auto` accepts the same merge, takes the row-rewrite sink, and + // leaves the blobs readable. + let plan = MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched(WhenNotMatched::DoNothing) .try_build() .unwrap() - .execute(new_data_extra) + .explain_plan(Some(source_schema.as_ref()), false) .await .unwrap(); + assert!( + plan.contains("MergeInsert: on=[key]") && !plan.contains("InPlaceMergeInsert"), + "Auto should take the row-rewrite sink: {plan}" + ); - assert_eq!(dataset2_extra.version().version, 3); - - // Load the dataset from disk to check versions - let ds_check1 = DatasetBuilder::from_uri(&dataset_uri).load().await.unwrap(); - - // Version 1 should be cleaned up due to auto cleanup (cleanup runs every version) - assert!( - ds_check1.checkout_version(1).await.is_err(), - "Version 1 should have been cleaned up" - ); - // Version 2 should still exist - assert!( - ds_check1.checkout_version(2).await.is_ok(), - "Version 2 should still exist" - ); - - // Advance time - MockClock::set_system_time(std::time::Duration::from_secs(4)); - - // Second merge insert WITH skip_auto_cleanup - should NOT trigger cleanup - let new_data2 = lance_datagen::gen_batch() - .with_seed(Seed::from(3)) - .col("id", array::step::()) - .into_df_stream(RowCount::from(30), BatchCount::from(1)); - - let (dataset3, _) = MergeInsertBuilder::try_new(dataset2_extra, vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .skip_auto_cleanup(true) // Skip auto cleanup - .try_build() - .unwrap() - .execute(new_data2) - .await - .unwrap(); - - assert_eq!(dataset3.version().version, 4); - - // Load the dataset from disk to check versions - let ds_check2 = DatasetBuilder::from_uri(&dataset_uri).load().await.unwrap(); + let (updated_ds, stats) = + MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(source)], + source_schema, + ))) + .await + .expect("nested-blob merge must succeed on the row-rewrite path"); + assert_eq!(stats.num_updated_rows, 2); - // Version 2 should still exist because skip_auto_cleanup was enabled - assert!( - ds_check2.checkout_version(2).await.is_ok(), - "Version 2 should still exist because skip_auto_cleanup was enabled" - ); - // Version 3 should also still exist - assert!( - ds_check2.checkout_version(3).await.is_ok(), - "Version 3 should still exist" - ); + let keys = updated_ds + .scan() + .scan_in_order(true) + .try_into_batch() + .await + .unwrap(); + let key_col = keys + .column_by_name("key") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let blobs = updated_ds + .take_blobs_by_indices(&[0, 1, 2, 3], "info.blob") + .await + .unwrap(); + for (row, blob) in blobs.iter().enumerate() { + let bytes = blob + .as_ref() + .expect("no blob cell should be null") + .read() + .await + .unwrap(); + let text = String::from_utf8(bytes.to_vec()).unwrap(); + let key = key_col.value(row); + let expected_tag = if key == 1 || key == 2 { "new" } else { "old" }; + assert!( + text.starts_with(expected_tag), + "key {key} should hold a {expected_tag}-* blob, got {text}" + ); + } + } } - #[tokio::test] - async fn test_transaction_inserted_rows_filter_roundtrip() { - // Create dataset with unenforced primary key on "id" column + // For some reason, Windows isn't able to handle the timeout test. Possibly + // a performance bug in their timer implementation? + #[cfg(not(windows))] + #[rstest::rstest] + #[case::all_success(Duration::from_secs(100_000))] + #[case::timeout(Duration::from_millis(200))] + #[tokio::test(start_paused = true)] + async fn test_merge_insert_concurrency(#[case] timeout: Duration) { let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false).with_metadata( - vec![( - "lance-schema:unenforced-primary-key".to_string(), - "true".to_string(), - )] - .into_iter() - .collect(), - ), + Field::new("id", DataType::UInt32, false), Field::new("value", DataType::UInt32, false), ])); - let initial = RecordBatch::try_new( + // To benchmark scaling curve: measure how long to run + // + // And vary `concurrency` to see how it scales. Compare this again `main`. + let concurrency = 10; + let initial_data = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(UInt32Array::from(vec![0, 1, 2])), - Arc::new(UInt32Array::from(vec![0, 0, 0])), + Arc::new(UInt32Array::from_iter_values(0..concurrency)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( + 0, + concurrency as usize, + ))), ], ) .unwrap(); - let dataset = InsertBuilder::new("memory://") - .execute(vec![initial]) + + // Increase likelihood of contention by throttling the store + let throttled = Arc::new(ThrottledStoreWrapper { + config: ThrottleConfig { + // For benchmarking: Increase this to simulate object storage. + wait_list_per_call: Duration::from_millis(20), + wait_get_per_call: Duration::from_millis(20), + wait_put_per_call: Duration::from_millis(20), + ..Default::default() + }, + }); + let session = Arc::new(Session::default()); + + let mut dataset = InsertBuilder::new("memory://") + .with_params(&WriteParams { + store_params: Some(ObjectStoreParams { + object_store_wrapper: Some(throttled.clone()), + ..Default::default() + }), + session: Some(session.clone()), + ..Default::default() + }) + .execute(vec![initial_data]) .await .unwrap(); - let dataset = Arc::new(dataset); - // Source with overlapping key 1 - let new_batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![1, 3])), - Arc::new(UInt32Array::from(vec![2, 2])), - ], - ) - .unwrap(); - let stream = RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(vec![Ok(new_batch)]), - ); + // do merge inserts in parallel based on the concurrency. Each will open the dataset, + // signal they have opened, and then wait for a signal to proceed. Once the signal + // is received, they will do a merge insert and close the dataset. - let UncommittedMergeInsert { transaction, .. } = - MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap() - .execute_uncommitted(Box::pin(stream) as SendableRecordBatchStream) - .await + let barrier = Arc::new(Barrier::new(concurrency as usize)); + let mut handles = Vec::new(); + for i in 0..concurrency { + let session_ref = session.clone(); + let schema_ref = schema.clone(); + let barrier_ref = barrier.clone(); + let throttled_ref = throttled.clone(); + let handle = tokio::task::spawn(async move { + let dataset = DatasetBuilder::from_uri("memory://") + .with_read_params(ReadParams { + store_options: Some(ObjectStoreParams { + object_store_wrapper: Some(throttled_ref.clone()), + ..Default::default() + }), + session: Some(session_ref.clone()), + ..Default::default() + }) + .load() + .await + .unwrap(); + let dataset = Arc::new(dataset); + + let new_data = RecordBatch::try_new( + schema_ref.clone(), + vec![ + Arc::new(UInt32Array::from(vec![i])), + Arc::new(UInt32Array::from(vec![1])), + ], + ) .unwrap(); + let source = Box::new(RecordBatchIterator::new([Ok(new_data)], schema_ref.clone())); - // Commit and read back transaction file - let committed = CommitBuilder::new(dataset.clone()) - .execute(transaction) - .await - .unwrap(); - let tx_path = committed.manifest().transaction_file.clone().unwrap(); - let tx_read = read_transaction_file(dataset.object_store.as_ref(), &dataset.base, &tx_path) - .await - .unwrap(); - // Check that inserted_rows_filter is present in the Operation::Update - if let Operation::Update { - inserted_rows_filter, - .. - } = &tx_read.operation - { - assert!(inserted_rows_filter.is_some()); - let filter = inserted_rows_filter.as_ref().unwrap(); - // Field IDs are assigned by Lance schema; check that we tracked exactly 1 key field - assert_eq!(filter.field_ids.len(), 1); - } else { - panic!("Expected Operation::Update"); + let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .conflict_retries(100) + .retry_timeout(timeout) + .try_build() + .unwrap(); + barrier_ref.wait().await; + + job.execute_reader(source) + .await + .map(|(_ds, stats)| stats.num_attempts) + }); + handles.push(handle); + } + + let results = try_join_all(handles).await.unwrap(); + + for attempts in results.iter() { + match attempts { + Ok(attempts) => { + assert!(*attempts <= 10, "Attempt count should be <= 10"); + } + Err(err) => { + // If we get an error, it means the task was cancelled + // due to timeout. This is expected if the timeout is + // set to a low value. + assert!( + matches!(err, Error::TooMuchWriteContention { message, .. } if message.contains("failed on retry_timeout")), + "Expected TooMuchWriteContention error, got: {:?}", + err + ); + } + } + } + + if timeout.as_secs() > 10 { + dataset.checkout_latest().await.unwrap(); + let batches = dataset.scan().try_into_batch().await.unwrap(); + + let values = batches["value"].as_primitive::(); + assert!( + values.values().iter().all(|&v| v == 1), + "All values should be 1 after merge insert. Got: {:?}", + values + ); } } - /// Test that two merge insert operations on the same existing key conflict. - /// First merge insert commits successfully, second one fails with conflict error - /// because both operations updated the same key (detected via bloom filter). #[tokio::test] - async fn test_inserted_rows_filter_bloom_conflict_detection_concurrent() { - // Create schema with unenforced primary key on "id" column + async fn test_merge_insert_large_concurrent() { let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false).with_metadata( - vec![( - "lance-schema:unenforced-primary-key".to_string(), - "true".to_string(), - )] - .into_iter() - .collect(), - ), + Field::new("id", DataType::UInt32, false), Field::new("value", DataType::UInt32, false), ])); - let initial = RecordBatch::try_new( + let num_rows = 10; + let initial_data = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), - Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), + Arc::new(UInt32Array::from_iter_values(0..num_rows)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( + 0, + num_rows as usize, + ))), ], ) .unwrap(); + // Adding latency helps ensure we get contention + let throttled = Arc::new(ThrottledStoreWrapper { + config: ThrottleConfig { + wait_list_per_call: Duration::from_millis(10), + wait_get_per_call: Duration::from_millis(10), + ..Default::default() + }, + }); + let session = Arc::new(Session::default()); + let dataset = InsertBuilder::new("memory://") - .execute(vec![initial]) + .with_params(&WriteParams { + store_params: Some(ObjectStoreParams { + object_store_wrapper: Some(throttled.clone()), + ..Default::default() + }), + session: Some(session.clone()), + ..Default::default() + }) + .execute(vec![initial_data]) .await .unwrap(); let dataset = Arc::new(dataset); - // Both jobs update/insert the same key 2 - let batch1 = RecordBatch::try_new( + // Start one merge insert, but don't commit it yet. + let new_data1 = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(UInt32Array::from(vec![2])), + Arc::new(UInt32Array::from(vec![1])), Arc::new(UInt32Array::from(vec![1])), ], ) .unwrap(); - let batch2 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![2])), - Arc::new(UInt32Array::from(vec![2])), - ], - ) - .unwrap(); - - // Create second merge insert job based on version 1 with 0 retries - let b2 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + let UncommittedMergeInsert { + transaction: transaction1, + .. + } = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) - .conflict_retries(0) .try_build() + .unwrap() + .execute_uncommitted(RecordBatchIterator::new( + vec![Ok(new_data1)], + schema.clone(), + )) + .await .unwrap(); - // First merge insert commits (creates version 2) - let s1 = RecordBatchStreamAdapter::new( + // Setup a "large" merge insert, with many batches + let new_data2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..1000)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n(2, 1000))), + ], + ) + .unwrap(); + let notify = Arc::new(Notify::new()); + let source = RecordBatchIterator::new( + (0..10) + .map(|i| { + let batch = new_data2.slice(i * 100, 100); + if i == 9 { + notify.notify_one(); + } + Ok(batch) + }) + .collect::>(), schema.clone(), - futures::stream::iter(vec![Ok(batch1.clone())]), ); - let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + let dataset2 = DatasetBuilder::from_uri("memory://") + .with_read_params(ReadParams { + store_options: Some(ObjectStoreParams { + object_store_wrapper: Some(throttled.clone()), + ..Default::default() + }), + session: Some(session.clone()), + ..Default::default() + }) + .load() + .await + .unwrap(); + let job = MergeInsertBuilder::try_new(Arc::new(dataset2), vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) .try_build() + .unwrap() + .execute_reader(source); + let task = tokio::task::spawn(job); + + // Right as the large merge insert has finished reading the last batch, + // we will commit the first merge insert. This should trigger a conflict, + // but we should resolve it automatically. + notify.notified().await; + let mut dataset = CommitBuilder::new(dataset) + .execute(transaction1) + .await .unwrap(); - let result1 = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; - assert!(result1.is_ok(), "First merge insert should succeed"); - // Second merge insert tries to commit based on version 1, needs to rebase against version 2 - let s2 = RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(vec![Ok(batch2.clone())]), - ); - let result2 = b2.execute(Box::pin(s2) as SendableRecordBatchStream).await; + task.await.unwrap().unwrap(); + dataset.checkout_latest().await.unwrap(); - // Second merge insert should fail because bloom filters show both updated key 2 + let batches = dataset.scan().try_into_batch().await.unwrap(); + let values = batches["value"].as_primitive::(); assert!( - matches!(result2, Err(crate::Error::TooMuchWriteContention { .. })), - "Expected TooMuchWriteContention (retryable conflict exhausted), got: {:?}", - result2 + values.values().iter().all(|&v| v == 2), + "All values should be 1 after merge insert. Got: {:?}", + values ); } - /// Test that two merge insert operations inserting the same NEW key conflict. - /// First merge insert commits successfully (inserts id=100), second one fails - /// with conflict error because both inserted the same new key (detected via bloom filter). #[tokio::test] - async fn test_concurrent_insert_same_new_key() { - // Create schema with unenforced primary key on "id" column - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false).with_metadata( - vec![( - "lance-schema:unenforced-primary-key".to_string(), - "true".to_string(), - )] - .into_iter() - .collect(), - ), - Field::new("value", DataType::UInt32, false), - ])); - // Initial dataset with ids 0, 1, 2, 3 - NOT containing id=100 - let initial = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), - Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), - ], - ) - .unwrap(); + async fn test_merge_insert_updates_indices() { + let test_dataset = async || { + let mut dataset = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("value", array::step::()) + .col("other_value", array::step::()) + .into_ram_dataset(FragmentCount::from(4), FragmentRowCount::from(20)) + .await + .unwrap(); - let dataset = InsertBuilder::new("memory://") - .execute(vec![initial]) + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + dataset + .create_index( + &["value"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + dataset + .create_index( + &["other_value"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + Arc::new(dataset) + }; + + let check_indices = async |dataset: &Dataset, id_frags: &[u32], value_frags: &[u32]| { + let id_index = dataset + .load_scalar_index(IndexCriteria::default().with_name("id_idx")) + .await + .unwrap(); + + if id_frags.is_empty() { + assert!(id_index.is_none()); + } else { + let id_index = id_index.unwrap(); + let id_frags_bitmap = RoaringBitmap::from_iter(id_frags.iter().copied()); + // Check the effective bitmap (raw bitmap intersected with existing fragments) + let effective_bitmap = id_index + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .unwrap(); + assert_eq!(effective_bitmap, id_frags_bitmap); + } + + let value_index = dataset + .load_scalar_index(IndexCriteria::default().with_name("value_idx")) + .await + .unwrap(); + + if value_frags.is_empty() { + assert!(value_index.is_none()); + } else { + let value_index = value_index.unwrap(); + let value_frags_bitmap = RoaringBitmap::from_iter(value_frags.iter().copied()); + // Check the effective bitmap (raw bitmap intersected with existing fragments) + let effective_bitmap = value_index + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .unwrap(); + assert_eq!(effective_bitmap, value_frags_bitmap); + } + + let other_value_index = dataset + .load_scalar_index(IndexCriteria::default().with_name("other_value_idx")) + .await + .unwrap() + .unwrap(); + + // The other_value index retains its original bitmap [0,1,2,3] since + // partial merges that don't modify other_value won't prune it. + let effective_bitmap = other_value_index + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .unwrap(); + + // The effective bitmap is the intersection of the index's original bitmap + // and the current dataset fragments. Since other_value is not modified by + // partial merges, it retains its validity for fragments it was originally trained on + // that still exist in the dataset. + let index_bitmap = other_value_index.fragment_bitmap.as_ref().unwrap(); + let expected_bitmap = index_bitmap & dataset.fragment_bitmap.as_ref(); + assert_eq!( + effective_bitmap, expected_bitmap, + "other_value index effective bitmap should be intersection. index_bitmap: {:?}, dataset_fragments: {:?}, effective_bitmap: {:?}", + index_bitmap, dataset.fragment_bitmap, effective_bitmap + ); + }; + + let dataset = test_dataset().await; + + // Sanity test on the initial dataset + check_indices(&dataset, &[0, 1, 2, 3], &[0, 1, 2, 3]).await; + + // Vertical merge insert (full schema), one fragment is deleted and should be removed from + // the index. + let merge_insert = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + + let (dataset, _) = merge_insert + .execute_reader( + lance_datagen::gen_batch() + .col("id", array::step_custom::(50, 1)) + .col("value", array::step_custom::(50, 1)) + .col("other_value", array::step_custom::(50, 1)) + .into_df_stream(RowCount::from(40), BatchCount::from(1)), + ) .await .unwrap(); - let dataset = Arc::new(dataset); - // Both jobs try to INSERT the same NEW key id=100 (doesn't exist in initial data) - let batch1 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![100])), // NEW key id=100 - Arc::new(UInt32Array::from(vec![1])), - ], - ) - .unwrap(); - let batch2 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![100])), // Same NEW key id=100 - Arc::new(UInt32Array::from(vec![2])), - ], - ) - .unwrap(); + // Fragment 3 removed and correctly removed from the index bitmap. + check_indices(&dataset, &[0, 1, 2], &[0, 1, 2]).await; - // Create second merge insert job based on version 1 with 0 retries - let b2 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + // Now we do the same thing with a partial merge insert (only id and value) + let dataset = test_dataset().await; + + // Vertical merge insert (full schema), one fragment is deleted and should be removed from + // the index. + let merge_insert = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) - .conflict_retries(0) + .when_not_matched(WhenNotMatched::InsertAll) .try_build() .unwrap(); - // First merge insert commits (creates version 2, inserts id=100) - let s1 = RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(vec![Ok(batch1.clone())]), - ); - let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + let (dataset, _) = merge_insert + .execute_reader( + lance_datagen::gen_batch() + .col("id", array::step_custom::(50, 1)) + .col("value", array::step_custom::(50, 1)) + .into_df_stream(RowCount::from(40), BatchCount::from(1)), + ) + .await + .unwrap(); + + // Fragment 3 is fully removed. We could keep it technically but today it is removed + // which is also fine. Fragment 2 is partially and must be removed. + // + // TODO: We should not be modifying the id_index here. A merge_insert should not need + // to rewrite the id field. However, it seems we are doing that today. This should be + // fixed in + check_indices(&dataset, &[0, 1], &[0, 1]).await; + + // One more test but this time we touch all fragments which causes the index to be removed + // entirely. + let dataset = test_dataset().await; + + // Vertical merge insert (full schema), one fragment is deleted and should be removed from + // the index. + let merge_insert = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched(WhenNotMatched::InsertAll) .try_build() .unwrap(); - let result1 = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; - assert!(result1.is_ok(), "First merge insert should succeed"); - // Second merge insert tries to commit based on version 1, needs to rebase against version 2 - let s2 = RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(vec![Ok(batch2.clone())]), - ); - let result2 = b2.execute(Box::pin(s2) as SendableRecordBatchStream).await; + let (dataset, _) = merge_insert + .execute_reader( + lance_datagen::gen_batch() + .col("id", array::step_custom::(10, 1)) + .col("value", array::step_custom::(10, 1)) + .into_df_stream(RowCount::from(80), BatchCount::from(1)), + ) + .await + .unwrap(); - // Second merge insert should fail because bloom filters show both inserted key 100 - assert!( - matches!(result2, Err(crate::Error::TooMuchWriteContention { .. })), - "Expected TooMuchWriteContention (retryable conflict exhausted), got: {:?}", - result2 - ); + check_indices(&dataset, &[], &[]).await; } - /// Concurrency regression for lance-format/lance#6441: two concurrent - /// find-or-create jobs (`WhenMatched::DoNothing` + `WhenNotMatched::InsertAll`) - /// both try to insert the same fresh key. The second must fail with - /// `TooMuchWriteContention` because the bloom-filter-backed - /// `inserted_rows_filter` detects the overlap during rebase. Before - /// routing find-or-create through v2 this did not work at all: the v1 - /// path returned `inserted_rows_filter=None`, so there was nothing to - /// intersect against during conflict resolution. #[tokio::test] - async fn test_concurrent_find_or_create_same_new_key() { - // Schema with an unenforced primary key on "id" — that is what - // activates bloom-filter conflict detection. + async fn test_upsert_concurrent_full_frag() { let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::UInt32, false).with_metadata( - vec![( - "lance-schema:unenforced-primary-key".to_string(), - "true".to_string(), - )] - .into_iter() - .collect(), - ), + Field::new("id", DataType::UInt32, false), Field::new("value", DataType::UInt32, false), ])); - // Initial dataset with ids 0..=3 — id=100 is not present. - let initial = RecordBatch::try_new( + let initial_data = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), - Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), + Arc::new(UInt32Array::from(vec![0, 1])), + Arc::new(UInt32Array::from(vec![0, 0])), ], ) .unwrap(); - let dataset = InsertBuilder::new("memory://") - .execute(vec![initial]) + // Increase likelihood of contention by throttling the store + let throttled = Arc::new(ThrottledStoreWrapper { + config: ThrottleConfig { + wait_list_per_call: Duration::from_millis(5), + wait_get_per_call: Duration::from_millis(5), + wait_put_per_call: Duration::from_millis(5), + ..Default::default() + }, + }); + let session = Arc::new(Session::default()); + + let mut dataset = InsertBuilder::new("memory://") + .with_params(&WriteParams { + store_params: Some(ObjectStoreParams { + object_store_wrapper: Some(throttled.clone()), + ..Default::default() + }), + session: Some(session.clone()), + ..Default::default() + }) + .execute(vec![initial_data]) .await .unwrap(); - let dataset = Arc::new(dataset); - - // Both jobs try to find-or-create the same new id=100. - let batch1 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![100])), - Arc::new(UInt32Array::from(vec![1])), - ], - ) - .unwrap(); - let batch2 = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![100])), - Arc::new(UInt32Array::from(vec![2])), - ], - ) - .unwrap(); - // b2 is built against version 1 with zero retries, so when it needs - // to rebase against b1's commit the bloom-filter intersection decides - // the outcome directly. - let b2 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::DoNothing) - .when_not_matched(WhenNotMatched::InsertAll) - .conflict_retries(0) - .try_build() + // Each merge insert will update one row. Combined, they should delete + // all rows in the first fragment, and it should be dropped. + let barrier = Arc::new(Barrier::new(2)); + let mut handles = Vec::new(); + for i in 0..2 { + let new_data = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![i])), + Arc::new(UInt32Array::from(vec![1])), + ], + ) .unwrap(); + let source = Box::new(RecordBatchIterator::new([Ok(new_data)], schema.clone())); - // First job commits successfully, producing version 2 with id=100. - let s1 = RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(vec![Ok(batch1.clone())]), - ); - let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::DoNothing) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap(); - let result1 = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; - assert!(result1.is_ok(), "First find-or-create should succeed"); + let dataset_ref = Arc::new(dataset.clone()); + let barrier = barrier.clone(); + let handle = tokio::spawn(async move { + barrier.wait().await; + MergeInsertBuilder::try_new(dataset_ref, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_reader(source) + .await + .unwrap(); + }); + handles.push(handle); + } + try_join_all(handles).await.unwrap(); - // Second job fails because its inserted_rows_filter overlaps b1's. - let s2 = RecordBatchStreamAdapter::new( - schema.clone(), - futures::stream::iter(vec![Ok(batch2.clone())]), + dataset.checkout_latest().await.unwrap(); + assert!( + dataset + .get_fragments() + .iter() + .all(|f| f.metadata().num_rows().unwrap() > 0), + "No fragments should have zero rows after upsert" ); - let result2 = b2.execute(Box::pin(s2) as SendableRecordBatchStream).await; + let batches = dataset.scan().try_into_batch().await.unwrap(); + let values = batches["value"].as_primitive::(); assert!( - matches!(result2, Err(crate::Error::TooMuchWriteContention { .. })), - "Expected TooMuchWriteContention (bloom-filter conflict) for find-or-create, got: {:?}", - result2 + values.values().iter().all(|&v| v == 1), + "All values should be 1 after merge insert. Got: {:?}", + values ); } - #[test] - fn test_concurrent_insert_different_new_list_key() { - // Schema for list(string) key column "tags". - let tags_field = Field::new( - "tags", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - false, - ); - let schema = Arc::new(Schema::new(vec![tags_field])); + #[tokio::test] + async fn test_plan_upsert() { + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); + let _schema = data.schema(); - // Build two batches inserting list key ["a", "b"] and ["c", "d"]. - let mut builder = ListBuilder::new(StringBuilder::new()); - builder.append_value(["a", "b"].iter().copied().map(Some)); - let tags_array1 = builder.finish(); - let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(tags_array1)]).unwrap(); + // Create dataset with initial data + let ds = Dataset::write(data, "memory://", None).await.unwrap(); - let mut builder = ListBuilder::new(StringBuilder::new()); - builder.append_value(["c", "d"].iter().copied().map(Some)); - let tags_array2 = builder.finish(); - let batch2 = RecordBatch::try_new(schema, vec![Arc::new(tags_array2)]).unwrap(); + // Create upsert job + let merge_insert_job = + crate::dataset::MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + .unwrap() + .when_matched(crate::dataset::WhenMatched::UpdateAll) + .try_build() + .unwrap(); - // Build bloom filters for the list keys. - let field_ids = vec![0_i32]; - let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); - let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + // Create new data for upsert + let new_data = lance_datagen::gen_batch() + .with_seed(Seed::from(2)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); + let new_data_stream = reader_to_stream(Box::new(new_data)); - let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("tags")]) - .expect("first batch should produce key"); - let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("tags")]) - .expect("second batch should produce key"); + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); - builder1.insert(key1).unwrap(); - builder2.insert(key2).unwrap(); - let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); - let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + // Assert the plan structure using portable plan matching + // The optimized plan should have: + // 1. FullSchemaMergeInsertExec at the top + // 2. ProjectionExec that creates action based on _rowaddr nullness (sentinel is constant + // true so DataFusion folds `sentinel IS NOT NULL` away from the CASE expression) + // 3. HashJoin with projection that includes the sentinel column + // 4. LanceScan that only reads the key column (projection pushdown working!) + // 5. ProjectionExec on the source side that materializes the sentinel literal + assert_plan_node_equals( + plan, + "MergeInsert: on=[key], when_matched=UpdateAll, when_not_matched=InsertAll, when_not_matched_by_source=Keep + CoalescePartitionsExec + ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NULL THEN 2 WHEN _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action] + HashJoinExec: mode=CollectLeft, join_type=Right, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] + LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, \ + row_id=true, row_addr=true, full_filter=--, refine_filter=-- + RepartitionExec: partitioning=RoundRobinBatch(...), input_partitions=1 + ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] + StreamingTableExec: partition_sizes=1, projection=[value, key]" + ).await.unwrap(); + } - let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + /// #4583 use case 3: which side of the merge_insert hash join gets buffered + /// is decided by the source's statistics, not by the order `create_plan` + /// writes the join in. `create_plan` always puts the target on the left, so + /// without a swap the target is always the build side. + /// + /// The target here is one row past DataFusion's + /// `hash_join_single_partition_threshold_rows`, and `FilteredReadExec` + /// reports no `total_byte_size`, so the target cannot pass the collect + /// threshold. That leaves the source: a materialized one reports exact + /// statistics and fits under the threshold, so `JoinSelection` swaps it onto + /// the build side and rewrites `Right` into `Left`. A one-shot stream reports + /// `Absent` for everything, neither side qualifies for `CollectLeft`, and the + /// plan falls back to a partitioned join whose build side is still the target. + /// + /// The one-shot provider used below stands in for every non-materialized + /// source: `stream_source_to_provider` sends the default path through + /// `spilling_table_provider`, which also hands back a `StreamingTable` and so + /// reports the same absent statistics. + /// + /// This is about which side is buffered, not about how much the target reads. + /// The target scan projects `other` either way, because the row-rewrite fill + /// reads it from the target side of the join. + /// + /// Both expectations characterise DataFusion's choice rather than any Lance + /// logic, and Lance sets no `hash_join_single_partition_threshold*` of its own, + /// so this rides on DataFusion's defaults (1 MiB / 128 Ki rows). A DataFusion + /// upgrade that changes them fails this test without anything in Lance + /// regressing, which is the point: the plan shape is what merge_insert's memory + /// use depends on, so a silent change to it should not go unnoticed. + #[tokio::test] + async fn test_plan_join_build_side_follows_source_statistics() { + fn find_hash_join(plan: &dyn ExecutionPlan) -> Option<&HashJoinExec> { + if let Some(join) = plan.downcast_ref::() { + return Some(join); + } + for child in plan.children() { + if let Some(join) = find_hash_join(child.as_ref()) { + return Some(join); + } + } + None + } + + fn sides(join: &HashJoinExec) -> (String, String) { + let render = |plan: &Arc| { + format!( + "{}", + datafusion::physical_plan::displayable(plan.as_ref()).indent(true) + ) + }; + (render(join.left()), render(join.right())) + } + + // One row past datafusion.optimizer.hash_join_single_partition_threshold_rows. + const TARGET_ROWS: u64 = 128 * 1024 + 1; + + let target = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("key", array::step::()) + .col("value", array::step::()) + .col("other", array::step::()) + .into_reader_rows(RowCount::from(TARGET_ROWS), BatchCount::from(1)); + let ds = Arc::new(Dataset::write(target, "memory://", None).await.unwrap()); + + // Partial schema: the source omits `other`, so the row-rewrite fill makes + // the target scan read it. In the streaming half below, where the target is + // the build side, that means it is held for every buffered row. This test + // asserts which side is the build side, not the projection. + let source = record_batch!( + ("key", UInt32, [0, 1, 2, 3]), + ("value", UInt32, [10, 11, 12, 13]) + ) + .unwrap(); + + let new_job = || { + crate::dataset::MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(crate::dataset::WhenMatched::UpdateAll) + .when_not_matched(crate::dataset::WhenNotMatched::InsertAll) + .try_build() + .unwrap() + }; + + let materialized: Arc = Arc::new( + datafusion::datasource::MemTable::try_new(source.schema(), vec![vec![source.clone()]]) + .unwrap(), + ); + let plan = new_job().create_plan(materialized).await.unwrap(); + let join = + find_hash_join(plan.as_ref()).expect("materialized source must plan a hash join"); + let (build, probe) = sides(join); + assert_eq!( + (*join.partition_mode(), *join.join_type()), + (PartitionMode::CollectLeft, JoinType::Left), + "the target is past the collect threshold and the source is not, so the inputs \ + are swapped and Right is rewritten to Left. build side was:\n{build}" + ); assert!( - !has_intersection, - "Expected bloom filters not intersect for different list(string) keys", + build.contains("DataSourceExec") && !build.contains("LanceRead"), + "the source must be the collected side:\n{build}" ); assert!( - !might_be_fp, - "Bloom filter intersection should be definitively not conflict", + probe.contains("LanceRead"), + "the target must be the probe side, which is the side a hash join offers its \ + dynamic filter to:\n{probe}" + ); + + let reader = RecordBatchIterator::new([Ok(source.clone())], source.schema()); + let stream_plan = new_job() + .create_plan(one_shot_provider(reader_to_stream(Box::new(reader))).unwrap()) + .await + .unwrap(); + let join = + find_hash_join(stream_plan.as_ref()).expect("stream source must plan a hash join"); + let (build, probe) = sides(join); + assert_eq!( + (*join.partition_mode(), *join.join_type()), + (PartitionMode::Partitioned, JoinType::Right), + "the target is past the collect threshold and the source reports no statistics, \ + so neither side qualifies and both are hash-repartitioned. build side was:\n{build}" + ); + assert!( + build.contains("LanceRead"), + "the target stays the build side, so every one of its rows is \ + buffered:\n{build}" + ); + assert!( + probe.contains("StreamingTableExec"), + "the source stays the probe side:\n{probe}" ); } - #[test] - fn test_concurrent_insert_same_new_list_key() { - // Schema for list(string) key column "tags". - let tags_field = Field::new( - "tags", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), - false, - ); - let schema = Arc::new(Schema::new(vec![tags_field])); - - // Build two batches both inserting the same list key ["a", "b"]. - let mut builder = ListBuilder::new(StringBuilder::new()); - builder.append_value(["a", "b"].iter().copied().map(Some)); - let tags_array1 = builder.finish(); - let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(tags_array1)]).unwrap(); - - let mut builder = ListBuilder::new(StringBuilder::new()); - builder.append_value(["a", "b"].iter().copied().map(Some)); - let tags_array2 = builder.finish(); - let batch2 = RecordBatch::try_new(schema, vec![Arc::new(tags_array2)]).unwrap(); - - // Build bloom filters for the list key. - let field_ids = vec![0_i32]; - let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); - let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); - - let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("tags")]) - .expect("first batch should produce key"); - let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("tags")]) - .expect("second batch should produce key"); - - builder1.insert(key1).unwrap(); - builder2.insert(key2).unwrap(); - let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); - let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + /// `analyze_plan` is a diagnostic, so it has to report the plan the source it + /// was handed would actually run. The batches entry point must therefore not + /// fall back to the streaming plan: with the row counts used below the join + /// collects the materialized source and rewrites the join type, and a stream + /// gets neither. Which side wins is a size comparison, not a property of the + /// entry point; see the fixture comment. + #[tokio::test] + async fn test_analyze_plan_reports_the_given_source_shape() { + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("key", array::step::()) + .col("value", array::step::()) + .into_reader_rows(RowCount::from(64), BatchCount::from(1)); + let ds = Arc::new(Dataset::write(data, "memory://", None).await.unwrap()); + + // The source covers the dataset's schema, so nothing is filled from the + // target side. Two rows + // against the target's 64 keeps the source the smaller side, which is what + // makes the join collect it here; both sides are under DataFusion's collect + // threshold, so the choice comes from comparing row counts. Raise the source + // above 64 and the join collects the target instead. + let source = + record_batch!(("key", UInt32, [1, 100]), ("value", UInt32, [999, 999])).unwrap(); + + let new_job = || { + crate::dataset::MergeInsertBuilder::try_new(ds.clone(), vec!["key".to_string()]) + .unwrap() + .when_matched(crate::dataset::WhenMatched::UpdateAll) + .when_not_matched(crate::dataset::WhenNotMatched::InsertAll) + .try_build() + .unwrap() + }; - let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + let materialized = new_job() + .analyze_plan_batches(vec![source.clone()]) + .await + .unwrap(); assert!( - has_intersection, - "Expected bloom filters to intersect for identical list(string) keys", + materialized.contains("DataSourceExec") && !materialized.contains("StreamingTableExec"), + "materialized batches must be reported as an in-memory source:\n{materialized}" ); assert!( - might_be_fp, - "Bloom filter intersection should be treated as potential conflict", + materialized.contains("join_type=Left"), + "collecting the source, which is the smaller side here, rewrites the join type:\n{materialized}" ); - } - - #[test] - fn test_concurrent_insert_same_new_nested_list_key() { - // Build nested list(list(string)) value [["a", "b"], ["c"]] for the "tags" column. - let nested_tags = make_nested_array(&[["a", "b"].as_slice(), ["c"].as_slice()]); - let tags_field = Field::new("tags", nested_tags.data_type().clone(), false); - let nested_tags2 = make_nested_array(&[["a", "b"].as_slice(), ["c"].as_slice()]); - - let schema = Arc::new(Schema::new(vec![tags_field])); - let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_tags)]).unwrap(); - let batch2 = RecordBatch::try_new(schema, vec![Arc::new(nested_tags2)]).unwrap(); - - // Build bloom filters for the nested list key. - let field_ids = vec![0_i32]; - let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); - let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); - - let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("tags")]) - .expect("first batch should produce key"); - let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("tags")]) - .expect("second batch should produce key"); - builder1.insert(key1).unwrap(); - builder2.insert(key2).unwrap(); - let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); - let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); - - let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); - assert!( - has_intersection, - "Expected bloom filters to intersect for identical nested list(list(string)) keys", + // The provider entry is public too, and the batches entry is a thin wrapper + // over it, so pin it directly rather than only through that wrapper. + let provider: Arc = Arc::new( + datafusion::datasource::MemTable::try_new(source.schema(), vec![vec![source.clone()]]) + .unwrap(), ); + let from_provider = new_job().analyze_plan_provider(provider).await.unwrap(); assert!( - might_be_fp, - "Bloom filter intersection should be treated as potential conflict", - ); - } - - #[test] - fn test_concurrent_insert_different_new_struct_key() { - let user_field = Field::new( - "user", - DataType::Struct( - vec![ - Field::new("first", DataType::Utf8, false), - Field::new("last", DataType::Utf8, false), - ] - .into(), - ), - false, + from_provider.contains("DataSourceExec") && from_provider.contains("join_type=Left"), + "a provider with exact statistics reports the same shape as its batches:\n{from_provider}" ); - let schema = Arc::new(Schema::new(vec![user_field])); - - // Build two batches inserting different struct keys. - let struct_array1 = make_struct_array_first_last_name(vec!["alice"], vec!["smith"]); - let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(struct_array1)]).unwrap(); - - let struct_array2 = make_struct_array_first_last_name(vec!["bob"], vec!["jones"]); - let batch2 = RecordBatch::try_new(schema, vec![Arc::new(struct_array2)]).unwrap(); - - // Build bloom filters for the struct key. - let field_ids = vec![0_i32]; - let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); - let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); - let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("user")]) - .expect("first batch should produce key"); - let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("user")]) - .expect("second batch should produce key"); - - builder1.insert(key1).unwrap(); - builder2.insert(key2).unwrap(); - let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); - let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); - - let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + let reader = RecordBatchIterator::new([Ok(source.clone())], source.schema()); + let streaming = new_job() + .analyze_plan(reader_to_stream(Box::new(reader))) + .await + .unwrap(); assert!( - !has_intersection, - "Expected bloom filters not intersect for different struct keys", + streaming.contains("StreamingTableExec") && !streaming.contains("DataSourceExec"), + "a stream must still be reported as a stream:\n{streaming}" ); assert!( - !might_be_fp, - "Bloom filter intersection should be definitively not conflict", + streaming.contains("join_type=Right"), + "nothing is swapped without source statistics:\n{streaming}" ); } - #[test] - fn test_concurrent_insert_same_new_struct_key() { - let user_field = Field::new( - "user", - DataType::Struct( - vec![ - Field::new("first", DataType::Utf8, false), - Field::new("last", DataType::Utf8, false), - ] - .into(), - ), - false, - ); - let schema = Arc::new(Schema::new(vec![user_field])); - - // Build two batches both inserting the same struct key {first: "alice", last: "smith"}. - let struct_array1 = make_struct_array_first_last_name(vec!["alice"], vec!["smith"]); - let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(struct_array1)]).unwrap(); + #[tokio::test] + async fn test_fast_path_update_only() { + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); - let struct_array2 = make_struct_array_first_last_name(vec!["alice"], vec!["smith"]); - let batch2 = RecordBatch::try_new(schema, vec![Arc::new(struct_array2)]).unwrap(); + // Create dataset with initial data + let ds = Dataset::write(data, "memory://", None).await.unwrap(); - // Build bloom filters for the struct key. - let field_ids = vec![0_i32]; - let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); - let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + // Create update-only job (insert_not_matched = false) + let merge_insert_job = + crate::dataset::MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + .unwrap() + .when_matched(crate::dataset::WhenMatched::UpdateAll) + .when_not_matched(crate::dataset::WhenNotMatched::DoNothing) + .try_build() + .unwrap(); - let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("user")]) - .expect("first batch should produce key"); - let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("user")]) - .expect("second batch should produce key"); + // Create new data for update + let new_data = lance_datagen::gen_batch() + .with_seed(Seed::from(2)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); + let new_data_stream = reader_to_stream(Box::new(new_data)); - builder1.insert(key1).unwrap(); - builder2.insert(key2).unwrap(); - let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); - let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + // This should use the fast path (execute_uncommitted_v2) + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); - let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); - assert!( - has_intersection, - "Expected bloom filters to intersect for identical struct keys", - ); - assert!( - might_be_fp, - "Bloom filter intersection should be treated as potential conflict", - ); + // The optimized plan should use Inner join instead of Right join since we're not + // inserting unmatched rows. The sentinel IS NOT NULL condition is folded away by + // DataFusion because the sentinel is lit(true), so the CASE only checks _rowaddr. + assert_plan_node_equals( + plan, + "MergeInsert: on=[key], when_matched=UpdateAll, when_not_matched=DoNothing, when_not_matched_by_source=Keep + CoalescePartitionsExec + ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NOT NULL THEN 1 ELSE 0 END as __action] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] + LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=-- + RepartitionExec... + ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] + StreamingTableExec: partition_sizes=1, projection=[value, key]" + ).await.unwrap(); } - #[test] - fn test_concurrent_insert_same_new_nested_struct_key() { - // Build nested struct value {address: {city: "seattle", zip: 98101}} for the "user" column. - let outer_struct = make_nested_struct_array_city_zip("seattle", 98101); - let user_field = Field::new("user", outer_struct.data_type().clone(), false); - let schema = Arc::new(Schema::new(vec![user_field])); + #[tokio::test] + async fn test_fast_path_conditional_update() { + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); - let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(outer_struct)]).unwrap(); + // Create dataset with initial data + let ds = Dataset::write(data, "memory://", None).await.unwrap(); - let outer_struct2 = make_nested_struct_array_city_zip("seattle", 98101); - let batch2 = RecordBatch::try_new(schema, vec![Arc::new(outer_struct2)]).unwrap(); + // Create conditional update job (WhenMatched::UpdateIf) + let merge_insert_job = crate::dataset::MergeInsertBuilder::try_new( + Arc::new(ds.clone()), + vec!["key".to_string()], + ) + .unwrap() + .when_matched(crate::dataset::WhenMatched::update_if(&ds, "source.value > 20").unwrap()) + .when_not_matched(crate::dataset::WhenNotMatched::DoNothing) + .try_build() + .unwrap(); - // Build bloom filters for the nested struct key. - let field_ids = vec![0_i32]; - let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); - let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + // Create new data for conditional update + let new_data = lance_datagen::gen_batch() + .with_seed(Seed::from(2)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let new_data_reader = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); + let new_data_stream = reader_to_stream(Box::new(new_data_reader)); - let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("user")]) - .expect("first batch should produce key"); - let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("user")]) - .expect("second batch should produce key"); + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); - builder1.insert(key1).unwrap(); - builder2.insert(key2).unwrap(); - let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); - let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); - - let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); - assert!( - has_intersection, - "Expected bloom filters to intersect for identical nested struct keys", - ); - assert!( - might_be_fp, - "Bloom filter intersection should be treated as potential conflict", - ); + // The optimized plan should use Inner join and include the UpdateIf condition. + // The sentinel IS NOT NULL condition is folded away (sentinel is lit(true)). + assert_plan_node_equals( + plan, + "MergeInsert: on=[key], when_matched=UpdateIf(source.value > 20), when_not_matched=DoNothing, when_not_matched_by_source=Keep + CoalescePartitionsExec + ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NOT NULL AND value@2 > 20 THEN 1 ELSE 0 END as __action] + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] + LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=-- + RepartitionExec... + ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] + StreamingTableExec: partition_sizes=1, projection=[value, key]" + ).await.unwrap(); } - /// End-to-end test for merge_insert using a struct-typed key column. + /// Verifies that a default find-or-create merge insert + /// (`WhenMatched::DoNothing` + `WhenNotMatched::InsertAll`) is routed + /// through the v2 `FullSchemaMergeInsertExec` path. Prior to this + /// change, `can_use_create_plan` rejected `DoNothing` outright and the + /// operation fell back to the legacy v1 `Merger`; the assertion below + /// would fail on `main`. See lance-format/lance#6441. #[tokio::test] - async fn test_merge_insert_struct_key_upsert() { - let user_field = Field::new( - "user", - DataType::Struct( - vec![ - Field::new("first", DataType::Utf8, false), - Field::new("last", DataType::Utf8, false), - ] - .into(), - ), - false, - ); - let schema = Arc::new(Schema::new(vec![ - user_field, - Field::new("value", DataType::UInt32, false), - ])); + async fn test_fast_path_find_or_create() { + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let data = data.into_reader_rows(RowCount::from(1024), BatchCount::from(32)); - // Initial dataset: - // (alice, smith) -> 1 - // (bob, jones) -> 1 - // (carla, doe) -> 1 - let user_array = make_struct_array_first_last_name( - vec!["alice", "bob", "carla"], - vec!["smith", "jones", "doe"], - ); - let values = UInt32Array::from(vec![1, 1, 1]); - let initial_batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(user_array), Arc::new(values)]) + // Create dataset with initial data + let ds = Dataset::write(data, "memory://", None).await.unwrap(); + + // Default MergeInsertBuilder config is find-or-create: + // when_matched = DoNothing, when_not_matched = InsertAll. + let merge_insert_job = + crate::dataset::MergeInsertBuilder::try_new(Arc::new(ds), vec!["key".to_string()]) + .unwrap() + .try_build() .unwrap(); - let test_uri = "memory://test_merge_insert_struct_key.lance"; - let dataset = Dataset::write( - RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), - test_uri, - None, + // Source data with a mix of already-present and new keys. + let new_data = lance_datagen::gen_batch() + .with_seed(Seed::from(2)) + .col("value", array::step::()) + .col("key", array::rand_pseudo_uuid_hex()); + let new_data = new_data.into_reader_rows(RowCount::from(512), BatchCount::from(16)); + let new_data_stream = reader_to_stream(Box::new(new_data)); + + // Should reach the v2 fast path (`create_plan` + FullSchemaMergeInsertExec). + // Dropping to v1 here would return an error from create_plan instead. + let plan = merge_insert_job + .create_plan(one_shot_provider(new_data_stream).unwrap()) + .await + .unwrap(); + + // The join is Right because we keep unmatched source rows (InsertAll) + // but discard unmatched target rows (DoNothing on when_matched, + // Keep on when_not_matched_by_source). The CASE expression simplifies + // to `_rowaddr IS NULL → Insert, else Nothing`. + assert_plan_node_equals( + plan, + "MergeInsert: on=[key], when_matched=DoNothing, when_not_matched=InsertAll, when_not_matched_by_source=Keep + CoalescePartitionsExec + ProjectionExec: expr=[_rowid@0 as _rowid, _rowaddr@1 as _rowaddr, value@2 as value, key@3 as key, __merge_source_sentinel@4 as __merge_source_sentinel, CASE WHEN _rowaddr@1 IS NULL THEN 2 ELSE 0 END as __action] + HashJoinExec: mode=CollectLeft, join_type=Right, on=[(key@0, key@1)], projection=[_rowid@1, _rowaddr@2, value@3, key@4, __merge_source_sentinel@5] + LanceRead: uri=..., projection=[key], num_fragments=1, range_before=None, range_after=None, row_id=true, row_addr=true, full_filter=--, refine_filter=-- + RepartitionExec... + ProjectionExec: expr=[value@0 as value, key@1 as key, true as __merge_source_sentinel] + StreamingTableExec: partition_sizes=1, projection=[value, key]" ) .await .unwrap(); - let dataset = Arc::new(dataset); + } - // New data: update alice, insert david - let new_user_array = - make_struct_array_first_last_name(vec!["alice", "david"], vec!["smith", "brown"]); - let new_values = UInt32Array::from(vec![10, 2]); - let new_batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new(new_user_array), Arc::new(new_values)], - ) - .unwrap(); + #[tokio::test] + async fn test_skip_auto_cleanup() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("{}/{}", tmpdir, "test_dataset"); - let reader = RecordBatchIterator::new([Ok(new_batch)], schema.clone()); - let (merged_ds, stats) = MergeInsertBuilder::try_new(dataset, vec!["user".to_string()]) + // Create initial dataset with auto cleanup interval of 1 version + let data = lance_datagen::gen_batch() + .with_seed(Seed::from(1)) + .col("id", array::step::()) + .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + + let mut auto_cleanup_params = HashMap::new(); + auto_cleanup_params.insert("lance.auto_cleanup.interval".to_string(), "1".to_string()); + auto_cleanup_params.insert( + "lance.auto_cleanup.older_than".to_string(), + "0ms".to_string(), + ); + + let write_params = WriteParams { + mode: WriteMode::Create, + auto_cleanup: Some(crate::dataset::AutoCleanupParams { + interval: 1, + older_than: chrono::TimeDelta::try_milliseconds(0).unwrap(), + }), + ..Default::default() + }; + + // Start at 1 second after epoch + MockClock::set_system_time(std::time::Duration::from_secs(1)); + + let dataset = Dataset::write(data, &dataset_uri, Some(write_params)) + .await + .unwrap(); + assert_eq!(dataset.version().version, 1); + + // Advance time + MockClock::set_system_time(std::time::Duration::from_secs(2)); + + // First merge insert WITHOUT skip_auto_cleanup - should trigger cleanup + let new_data = lance_datagen::gen_batch() + .with_seed(Seed::from(2)) + .col("id", array::step::()) + .into_df_stream(RowCount::from(50), BatchCount::from(1)); + + let (dataset2, _) = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) .try_build() .unwrap() - .execute(reader_to_stream(Box::new(reader))) + .execute(new_data) .await .unwrap(); - assert_eq!(stats.num_updated_rows, 1); - assert_eq!(stats.num_inserted_rows, 1); - assert_eq!(stats.num_deleted_rows, 0); - - let result = merged_ds.scan().try_into_batch().await.unwrap(); - let user_col = result - .column_by_name("user") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - let first = user_col - .column(0) - .as_any() - .downcast_ref::() - .unwrap(); - let last = user_col - .column(1) - .as_any() - .downcast_ref::() - .unwrap(); - let values = result - .column_by_name("value") - .unwrap() - .as_primitive::(); + assert_eq!(dataset2.version().version, 2); - let mut rows = Vec::new(); - for i in 0..result.num_rows() { - rows.push(( - first.value(i).to_string(), - last.value(i).to_string(), - values.value(i), - )); - } - rows.sort(); + // Advance time + MockClock::set_system_time(std::time::Duration::from_secs(3)); - assert_eq!( - rows, - vec![ - ("alice".to_string(), "smith".to_string(), 10), - ("bob".to_string(), "jones".to_string(), 1), - ("carla".to_string(), "doe".to_string(), 1), - ("david".to_string(), "brown".to_string(), 2), - ], - ); - } + // Need to do another merge insert for cleanup to take effect since cleanup runs on the old dataset + let new_data_extra = lance_datagen::gen_batch() + .with_seed(Seed::from(4)) + .col("id", array::step::()) + .into_df_stream(RowCount::from(10), BatchCount::from(1)); - fn make_struct_array_first_last_name(first: Vec<&str>, last: Vec<&str>) -> StructArray { - let first = StringArray::from(first); - let last = StringArray::from(last); + let (dataset2_extra, _) = + MergeInsertBuilder::try_new(dataset2.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute(new_data_extra) + .await + .unwrap(); - StructArray::from(vec![ - ( - Arc::new(Field::new("first", DataType::Utf8, false)), - Arc::new(first) as Arc, - ), - ( - Arc::new(Field::new("last", DataType::Utf8, false)), - Arc::new(last) as Arc, - ), - ]) - } + assert_eq!(dataset2_extra.version().version, 3); - fn make_nested_struct_array_city_zip(city: &str, zip: i32) -> StructArray { - let city = StringArray::from(vec![city]); - let zip = Int32Array::from(vec![zip]); + // Load the dataset from disk to check versions + let ds_check1 = DatasetBuilder::from_uri(&dataset_uri).load().await.unwrap(); - let inner_struct = StructArray::from(vec![ - ( - Arc::new(Field::new("city", DataType::Utf8, false)), - Arc::new(city) as Arc, - ), - ( - Arc::new(Field::new("zip", DataType::Int32, false)), - Arc::new(zip) as Arc, - ), - ]); + // Version 1 should be cleaned up due to auto cleanup (cleanup runs every version) + assert!( + ds_check1.checkout_version(1).await.is_err(), + "Version 1 should have been cleaned up" + ); + // Version 2 should still exist + assert!( + ds_check1.checkout_version(2).await.is_ok(), + "Version 2 should still exist" + ); - StructArray::from(vec![( - Arc::new(Field::new( - "address", - inner_struct.data_type().clone(), - false, - )), - Arc::new(inner_struct) as Arc, - )]) + // Advance time + MockClock::set_system_time(std::time::Duration::from_secs(4)); + + // Second merge insert WITH skip_auto_cleanup - should NOT trigger cleanup + let new_data2 = lance_datagen::gen_batch() + .with_seed(Seed::from(3)) + .col("id", array::step::()) + .into_df_stream(RowCount::from(30), BatchCount::from(1)); + + let (dataset3, _) = MergeInsertBuilder::try_new(dataset2_extra, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .skip_auto_cleanup(true) // Skip auto cleanup + .try_build() + .unwrap() + .execute(new_data2) + .await + .unwrap(); + + assert_eq!(dataset3.version().version, 4); + + // Load the dataset from disk to check versions + let ds_check2 = DatasetBuilder::from_uri(&dataset_uri).load().await.unwrap(); + + // Version 2 should still exist because skip_auto_cleanup was enabled + assert!( + ds_check2.checkout_version(2).await.is_ok(), + "Version 2 should still exist because skip_auto_cleanup was enabled" + ); + // Version 3 should also still exist + assert!( + ds_check2.checkout_version(3).await.is_ok(), + "Version 3 should still exist" + ); } - fn make_nested_array(inner_lists: &[&[&str]]) -> ListArray { - let mut inner_builder = ListBuilder::new(StringBuilder::new()); - for inner in inner_lists { - inner_builder.append_value(inner.iter().map(|s| Some(*s))); - } - let inner_list_array = inner_builder.finish(); + #[tokio::test] + async fn test_transaction_inserted_rows_filter_roundtrip() { + // Create dataset with unenforced primary key on "id" column + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false).with_metadata( + vec![( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + )] + .into_iter() + .collect(), + ), + Field::new("value", DataType::UInt32, false), + ])); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0, 1, 2])), + Arc::new(UInt32Array::from(vec![0, 0, 0])), + ], + ) + .unwrap(); + let dataset = InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(); + let dataset = Arc::new(dataset); - let offsets = ScalarBuffer::::from(vec![0, inner_list_array.len() as i32]); - let offsets = OffsetBuffer::new(offsets); - ListArray::new( - Arc::new(Field::new( - "item", - inner_list_array.data_type().clone(), - inner_list_array.nulls().is_some(), - )), - offsets, - Arc::new(inner_list_array), - None, + // Source with overlapping key 1 + let new_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![1, 3])), + Arc::new(UInt32Array::from(vec![2, 2])), + ], ) + .unwrap(); + let stream = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(new_batch)]), + ); + + let UncommittedMergeInsert { transaction, .. } = + MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_uncommitted(Box::pin(stream) as SendableRecordBatchStream) + .await + .unwrap(); + + // Commit and read back transaction file + let committed = CommitBuilder::new(dataset.clone()) + .execute(transaction) + .await + .unwrap(); + let tx_path = committed.manifest().transaction_file.clone().unwrap(); + let tx_read = read_transaction_file(dataset.object_store.as_ref(), &dataset.base, &tx_path) + .await + .unwrap(); + // Check that inserted_rows_filter is present in the Operation::Update + if let Operation::Update { + inserted_rows_filter, + .. + } = &tx_read.operation + { + assert!(inserted_rows_filter.is_some()); + let filter = inserted_rows_filter.as_ref().unwrap(); + // Field IDs are assigned by Lance schema; check that we tracked exactly 1 key field + assert_eq!(filter.field_ids.len(), 1); + } else { + panic!("Expected Operation::Update"); + } } - /// Test that merge_insert with bloom filter fails when committing against - /// an Update transaction that doesn't have a filter. We can't determine if - /// the Update operation conflicted with our inserted rows. + /// Test that two merge insert operations on the same existing key conflict. + /// First merge insert commits successfully, second one fails with conflict error + /// because both operations updated the same key (detected via bloom filter). #[tokio::test] - async fn test_merge_insert_conflict_with_update_without_filter() { - use crate::dataset::UpdateBuilder; - + async fn test_inserted_rows_filter_bloom_conflict_detection_concurrent() { // Create schema with unenforced primary key on "id" column let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::UInt32, false).with_metadata( @@ -7648,17 +9868,26 @@ mod tests { .unwrap(); let dataset = Arc::new(dataset); - // Create merge insert job based on version 1 + // Both jobs update/insert the same key 2 let batch1 = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![2])), Arc::new(UInt32Array::from(vec![1])), ], ) .unwrap(); + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![2])), + Arc::new(UInt32Array::from(vec![2])), + ], + ) + .unwrap(); - let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + // Create second merge insert job based on version 1 with 0 retries + let b2 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) @@ -7666,42 +9895,40 @@ mod tests { .try_build() .unwrap(); - // Regular Update without bloom filter commits first (creates version 2) - let update_result = UpdateBuilder::new(dataset.clone()) - .update_where("id = 0") - .unwrap() - .set("value", "999") - .unwrap() - .build() - .unwrap() - .execute() - .await; - assert!(update_result.is_ok(), "Update should succeed"); - - // Now merge insert tries to commit based on version 1, needs to rebase against version 2 + // First merge insert commits (creates version 2) let s1 = RecordBatchStreamAdapter::new( schema.clone(), futures::stream::iter(vec![Ok(batch1.clone())]), ); - let merge_result = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let result1 = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + assert!(result1.is_ok(), "First merge insert should succeed"); - // Merge insert should fail with retryable conflict because it can't - // determine if Update conflicted (Update has no inserted_rows_filter) + // Second merge insert tries to commit based on version 1, needs to rebase against version 2 + let s2 = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(batch2.clone())]), + ); + let result2 = b2.execute(Box::pin(s2) as SendableRecordBatchStream).await; + + // Second merge insert should fail because bloom filters show both updated key 2 assert!( - matches!( - merge_result, - Err(crate::Error::TooMuchWriteContention { .. }) - ), + matches!(result2, Err(crate::Error::TooMuchWriteContention { .. })), "Expected TooMuchWriteContention (retryable conflict exhausted), got: {:?}", - merge_result + result2 ); } - /// Test that merge_insert with bloom filter fails when committing against - /// an Append operation. We can't determine if the appended rows conflict - /// with our inserted rows. + /// Test that two merge insert operations inserting the same NEW key conflict. + /// First merge insert commits successfully (inserts id=100), second one fails + /// with conflict error because both inserted the same new key (detected via bloom filter). #[tokio::test] - async fn test_merge_insert_conflict_with_append() { + async fn test_concurrent_insert_same_new_key() { // Create schema with unenforced primary key on "id" column let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::UInt32, false).with_metadata( @@ -7714,6 +9941,7 @@ mod tests { ), Field::new("value", DataType::UInt32, false), ])); + // Initial dataset with ids 0, 1, 2, 3 - NOT containing id=100 let initial = RecordBatch::try_new( schema.clone(), vec![ @@ -7729,17 +9957,26 @@ mod tests { .unwrap(); let dataset = Arc::new(dataset); - // Create merge insert job based on version 1 + // Both jobs try to INSERT the same NEW key id=100 (doesn't exist in initial data) let batch1 = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![100])), // NEW key id=100 Arc::new(UInt32Array::from(vec![1])), ], ) .unwrap(); + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100])), // Same NEW key id=100 + Arc::new(UInt32Array::from(vec![2])), + ], + ) + .unwrap(); - let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + // Create second merge insert job based on version 1 with 0 retries + let b2 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::InsertAll) @@ -7747,247 +9984,952 @@ mod tests { .try_build() .unwrap(); - // Append commits first (creates version 2) - let append_batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![50])), - Arc::new(UInt32Array::from(vec![2])), - ], - ) - .unwrap(); - let append_result = InsertBuilder::new(dataset.clone()) - .with_params(&WriteParams { - mode: WriteMode::Append, - ..Default::default() - }) - .execute(vec![append_batch]) - .await; - assert!(append_result.is_ok(), "Append should succeed"); - - // Now merge insert tries to commit based on version 1, needs to rebase against version 2 + // First merge insert commits (creates version 2, inserts id=100) let s1 = RecordBatchStreamAdapter::new( schema.clone(), futures::stream::iter(vec![Ok(batch1.clone())]), ); - let merge_result = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let result1 = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + assert!(result1.is_ok(), "First merge insert should succeed"); - // Merge insert should fail with retryable conflict because it can't - // determine if Append added conflicting keys + // Second merge insert tries to commit based on version 1, needs to rebase against version 2 + let s2 = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(batch2.clone())]), + ); + let result2 = b2.execute(Box::pin(s2) as SendableRecordBatchStream).await; + + // Second merge insert should fail because bloom filters show both inserted key 100 assert!( - matches!( - merge_result, - Err(crate::Error::TooMuchWriteContention { .. }) - ), + matches!(result2, Err(crate::Error::TooMuchWriteContention { .. })), "Expected TooMuchWriteContention (retryable conflict exhausted), got: {:?}", - merge_result + result2 ); } + /// Concurrency regression for lance-format/lance#6441: two concurrent + /// find-or-create jobs (`WhenMatched::DoNothing` + `WhenNotMatched::InsertAll`) + /// both try to insert the same fresh key. The second must fail with + /// `TooMuchWriteContention` because the bloom-filter-backed + /// `inserted_rows_filter` detects the overlap during rebase. Before + /// routing find-or-create through v2 this did not work at all: the v1 + /// path returned `inserted_rows_filter=None`, so there was nothing to + /// intersect against during conflict resolution. #[tokio::test] - async fn test_explain_plan() { - // Set up test data using lance_datagen - let dataset = lance_datagen::gen_batch() - .col("id", lance_datagen::array::step::()) - .col("name", array::cycle_utf8_literals(&["a", "b", "c"])) - .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(3)) - .await - .unwrap(); - - // Create merge insert job - let merge_insert_job = - MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap(); - - // Test explain_plan with default schema (None) - let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); - - // Also validate the full string structure with pattern matching - let expected_pattern = "\ -MergeInsert: on=[id], when_matched=UpdateAll, when_not_matched=InsertAll, when_not_matched_by_source=Keep... - CoalescePartitionsExec... - HashJoinExec... - LanceRead... - StreamingTableExec: partition_sizes=1, projection=[id, name]"; - assert_string_matches(&plan, expected_pattern).unwrap(); - - // Test with explicit schema - let source_schema = arrow_schema::Schema::from(dataset.schema()); - let explicit_plan = merge_insert_job - .explain_plan(Some(&source_schema), false) - .await - .unwrap(); - assert_eq!(plan, explicit_plan); // Should be the same as default - - // Test verbose mode produces different (likely longer) output - let verbose_plan = merge_insert_job.explain_plan(None, true).await.unwrap(); - assert!(verbose_plan.contains("MergeInsert")); - // Verbose should also match the expected pattern - assert_string_matches(&verbose_plan, expected_pattern).unwrap(); - } - - /// Asserts that `explain_plan()` is supported for a default find-or-create - /// configuration (`WhenMatched::DoNothing` + `WhenNotMatched::InsertAll`). - /// Before lance-format/lance#6441 this returned `Error::NotSupported` - /// because the job fell back to the legacy v1 path. - #[tokio::test] - async fn test_explain_plan_find_or_create() { - let dataset = lance_datagen::gen_batch() - .col("id", lance_datagen::array::step::()) - .col("name", array::cycle_utf8_literals(&["a", "b", "c"])) - .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(3)) - .await - .unwrap(); - - // Default builder config == find-or-create. - let merge_insert_job = - MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) - .unwrap() - .try_build() - .unwrap(); - - let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); - - let expected_pattern = "\ -MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_not_matched_by_source=Keep... - CoalescePartitionsExec... - HashJoinExec...join_type=Right... - LanceRead... - StreamingTableExec: partition_sizes=1, projection=[id, name]"; - assert_string_matches(&plan, expected_pattern).unwrap(); - } - - #[tokio::test] - async fn test_explain_plan_full_schema_delete_by_source_with_fsl() { + async fn test_concurrent_find_or_create_same_new_key() { + // Schema with an unenforced primary key on "id" — that is what + // activates bloom-filter conflict detection. let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vec", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - true, + Field::new("id", DataType::UInt32, false).with_metadata( + vec![( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + )] + .into_iter() + .collect(), ), + Field::new("value", DataType::UInt32, false), ])); - - let dataset_batch = RecordBatch::try_new( + // Initial dataset with ids 0..=3 — id=100 is not present. + let initial = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new( - FixedSizeListArray::try_new_from_values( - Float32Array::from(vec![ - 1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3, - ]), - 4, - ) - .unwrap(), - ), + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), ], ) .unwrap(); - let dataset = Dataset::write( - Box::new(RecordBatchIterator::new( - [Ok(dataset_batch)], - schema.clone(), - )), - "memory://test_explain_plan_full_schema_delete_by_source_with_fsl", - None, - ) - .await - .unwrap(); - - let merge_insert_job = - MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .when_not_matched_by_source(WhenNotMatchedBySource::Delete) - .use_index(false) - .try_build() - .unwrap(); - - let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); - assert!(plan.contains("HashJoinExec")); - assert!(plan.contains("join_type=Full")); - assert!(plan.contains("projection=[_rowid")); - assert!( - plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"), - "target-side scan should prune the FSL payload from the join build side: {plan}" - ); - assert!( - !plan.contains("LanceRead: uri=test_explain_plan_full_schema_delete_by_source_with_fsl/data, projection=[id, vec]"), - "target-side scan should not include the FSL payload in the join build side: {plan}" - ); - } - - #[tokio::test] - async fn test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index() { - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vec", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), - true, - ), - ])); + let dataset = InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(); + let dataset = Arc::new(dataset); - let dataset_batch = RecordBatch::try_new( + // Both jobs try to find-or-create the same new id=100. + let batch1 = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new( - FixedSizeListArray::try_new_from_values( - Float32Array::from(vec![ - 1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3, - ]), - 4, - ) - .unwrap(), - ), + Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![1])), ], ) .unwrap(); - - let mut dataset = Dataset::write( - Box::new(RecordBatchIterator::new( - [Ok(dataset_batch)], - schema.clone(), - )), - "memory://test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index", - None, + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![2])), + ], ) - .await .unwrap(); - let scalar_params = ScalarIndexParams::default(); - dataset - .create_index(&["id"], IndexType::Scalar, None, &scalar_params, false) - .await + // b2 is built against version 1 with zero retries, so when it needs + // to rebase against b1's commit the bloom-filter intersection decides + // the outcome directly. + let b2 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::DoNothing) + .when_not_matched(WhenNotMatched::InsertAll) + .conflict_retries(0) + .try_build() .unwrap(); - let merge_insert_job = - MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::InsertAll) - .when_not_matched_by_source(WhenNotMatchedBySource::Delete) - .try_build() - .unwrap(); - - let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); - assert!(plan.contains("HashJoinExec")); - assert!(plan.contains("join_type=Full")); - assert!(plan.contains("projection=[_rowid")); - assert!( - plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"), - "target-side scan should prune the FSL payload from the join build side even when a scalar index exists: {plan}" + // First job commits successfully, producing version 2 with id=100. + let s1 = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(batch1.clone())]), ); - assert!( - !plan.contains( + let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::DoNothing) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let result1 = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + assert!(result1.is_ok(), "First find-or-create should succeed"); + + // Second job fails because its inserted_rows_filter overlaps b1's. + let s2 = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(batch2.clone())]), + ); + let result2 = b2.execute(Box::pin(s2) as SendableRecordBatchStream).await; + + assert!( + matches!(result2, Err(crate::Error::TooMuchWriteContention { .. })), + "Expected TooMuchWriteContention (bloom-filter conflict) for find-or-create, got: {:?}", + result2 + ); + } + + #[test] + fn test_concurrent_insert_different_new_list_key() { + // Schema for list(string) key column "tags". + let tags_field = Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + false, + ); + let schema = Arc::new(Schema::new(vec![tags_field])); + + // Build two batches inserting list key ["a", "b"] and ["c", "d"]. + let mut builder = ListBuilder::new(StringBuilder::new()); + builder.append_value(["a", "b"].iter().copied().map(Some)); + let tags_array1 = builder.finish(); + let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(tags_array1)]).unwrap(); + + let mut builder = ListBuilder::new(StringBuilder::new()); + builder.append_value(["c", "d"].iter().copied().map(Some)); + let tags_array2 = builder.finish(); + let batch2 = RecordBatch::try_new(schema, vec![Arc::new(tags_array2)]).unwrap(); + + // Build bloom filters for the list keys. + let field_ids = vec![0_i32]; + let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); + let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + + let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("tags")]) + .expect("first batch should produce key"); + let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("tags")]) + .expect("second batch should produce key"); + + builder1.insert(key1).unwrap(); + builder2.insert(key2).unwrap(); + let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); + let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + + let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + assert!( + !has_intersection, + "Expected bloom filters not intersect for different list(string) keys", + ); + assert!( + !might_be_fp, + "Bloom filter intersection should be definitively not conflict", + ); + } + + #[test] + fn test_concurrent_insert_same_new_list_key() { + // Schema for list(string) key column "tags". + let tags_field = Field::new( + "tags", + DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + false, + ); + let schema = Arc::new(Schema::new(vec![tags_field])); + + // Build two batches both inserting the same list key ["a", "b"]. + let mut builder = ListBuilder::new(StringBuilder::new()); + builder.append_value(["a", "b"].iter().copied().map(Some)); + let tags_array1 = builder.finish(); + let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(tags_array1)]).unwrap(); + + let mut builder = ListBuilder::new(StringBuilder::new()); + builder.append_value(["a", "b"].iter().copied().map(Some)); + let tags_array2 = builder.finish(); + let batch2 = RecordBatch::try_new(schema, vec![Arc::new(tags_array2)]).unwrap(); + + // Build bloom filters for the list key. + let field_ids = vec![0_i32]; + let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); + let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + + let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("tags")]) + .expect("first batch should produce key"); + let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("tags")]) + .expect("second batch should produce key"); + + builder1.insert(key1).unwrap(); + builder2.insert(key2).unwrap(); + let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); + let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + + let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + assert!( + has_intersection, + "Expected bloom filters to intersect for identical list(string) keys", + ); + assert!( + might_be_fp, + "Bloom filter intersection should be treated as potential conflict", + ); + } + + #[test] + fn test_concurrent_insert_same_new_nested_list_key() { + // Build nested list(list(string)) value [["a", "b"], ["c"]] for the "tags" column. + let nested_tags = make_nested_array(&[["a", "b"].as_slice(), ["c"].as_slice()]); + let tags_field = Field::new("tags", nested_tags.data_type().clone(), false); + let nested_tags2 = make_nested_array(&[["a", "b"].as_slice(), ["c"].as_slice()]); + + let schema = Arc::new(Schema::new(vec![tags_field])); + let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_tags)]).unwrap(); + let batch2 = RecordBatch::try_new(schema, vec![Arc::new(nested_tags2)]).unwrap(); + + // Build bloom filters for the nested list key. + let field_ids = vec![0_i32]; + let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); + let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + + let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("tags")]) + .expect("first batch should produce key"); + let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("tags")]) + .expect("second batch should produce key"); + + builder1.insert(key1).unwrap(); + builder2.insert(key2).unwrap(); + let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); + let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + + let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + assert!( + has_intersection, + "Expected bloom filters to intersect for identical nested list(list(string)) keys", + ); + assert!( + might_be_fp, + "Bloom filter intersection should be treated as potential conflict", + ); + } + + #[test] + fn test_concurrent_insert_different_new_struct_key() { + let user_field = Field::new( + "user", + DataType::Struct( + vec![ + Field::new("first", DataType::Utf8, false), + Field::new("last", DataType::Utf8, false), + ] + .into(), + ), + false, + ); + let schema = Arc::new(Schema::new(vec![user_field])); + + // Build two batches inserting different struct keys. + let struct_array1 = make_struct_array_first_last_name(vec!["alice"], vec!["smith"]); + let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(struct_array1)]).unwrap(); + + let struct_array2 = make_struct_array_first_last_name(vec!["bob"], vec!["jones"]); + let batch2 = RecordBatch::try_new(schema, vec![Arc::new(struct_array2)]).unwrap(); + + // Build bloom filters for the struct key. + let field_ids = vec![0_i32]; + let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); + let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + + let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("user")]) + .expect("first batch should produce key"); + let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("user")]) + .expect("second batch should produce key"); + + builder1.insert(key1).unwrap(); + builder2.insert(key2).unwrap(); + let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); + let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + + let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + assert!( + !has_intersection, + "Expected bloom filters not intersect for different struct keys", + ); + assert!( + !might_be_fp, + "Bloom filter intersection should be definitively not conflict", + ); + } + + #[test] + fn test_concurrent_insert_same_new_struct_key() { + let user_field = Field::new( + "user", + DataType::Struct( + vec![ + Field::new("first", DataType::Utf8, false), + Field::new("last", DataType::Utf8, false), + ] + .into(), + ), + false, + ); + let schema = Arc::new(Schema::new(vec![user_field])); + + // Build two batches both inserting the same struct key {first: "alice", last: "smith"}. + let struct_array1 = make_struct_array_first_last_name(vec!["alice"], vec!["smith"]); + let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(struct_array1)]).unwrap(); + + let struct_array2 = make_struct_array_first_last_name(vec!["alice"], vec!["smith"]); + let batch2 = RecordBatch::try_new(schema, vec![Arc::new(struct_array2)]).unwrap(); + + // Build bloom filters for the struct key. + let field_ids = vec![0_i32]; + let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); + let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + + let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("user")]) + .expect("first batch should produce key"); + let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("user")]) + .expect("second batch should produce key"); + + builder1.insert(key1).unwrap(); + builder2.insert(key2).unwrap(); + let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); + let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + + let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + assert!( + has_intersection, + "Expected bloom filters to intersect for identical struct keys", + ); + assert!( + might_be_fp, + "Bloom filter intersection should be treated as potential conflict", + ); + } + + #[test] + fn test_concurrent_insert_same_new_nested_struct_key() { + // Build nested struct value {address: {city: "seattle", zip: 98101}} for the "user" column. + let outer_struct = make_nested_struct_array_city_zip("seattle", 98101); + let user_field = Field::new("user", outer_struct.data_type().clone(), false); + let schema = Arc::new(Schema::new(vec![user_field])); + + let batch1 = RecordBatch::try_new(schema.clone(), vec![Arc::new(outer_struct)]).unwrap(); + + let outer_struct2 = make_nested_struct_array_city_zip("seattle", 98101); + let batch2 = RecordBatch::try_new(schema, vec![Arc::new(outer_struct2)]).unwrap(); + + // Build bloom filters for the nested struct key. + let field_ids = vec![0_i32]; + let mut builder1 = KeyExistenceFilterBuilder::new(field_ids.clone()); + let mut builder2 = KeyExistenceFilterBuilder::new(field_ids); + + let key1 = extract_key_value_from_batch(&batch1, 0, &[String::from("user")]) + .expect("first batch should produce key"); + let key2 = extract_key_value_from_batch(&batch2, 0, &[String::from("user")]) + .expect("second batch should produce key"); + + builder1.insert(key1).unwrap(); + builder2.insert(key2).unwrap(); + let filter1 = KeyExistenceFilter::from_bloom_filter(&builder1); + let filter2 = KeyExistenceFilter::from_bloom_filter(&builder2); + + let (has_intersection, might_be_fp) = filter1.intersects(&filter2).unwrap(); + assert!( + has_intersection, + "Expected bloom filters to intersect for identical nested struct keys", + ); + assert!( + might_be_fp, + "Bloom filter intersection should be treated as potential conflict", + ); + } + + /// End-to-end test for merge_insert using a struct-typed key column. + #[tokio::test] + async fn test_merge_insert_struct_key_upsert() { + let user_field = Field::new( + "user", + DataType::Struct( + vec![ + Field::new("first", DataType::Utf8, false), + Field::new("last", DataType::Utf8, false), + ] + .into(), + ), + false, + ); + let schema = Arc::new(Schema::new(vec![ + user_field, + Field::new("value", DataType::UInt32, false), + ])); + + // Initial dataset: + // (alice, smith) -> 1 + // (bob, jones) -> 1 + // (carla, doe) -> 1 + let user_array = make_struct_array_first_last_name( + vec!["alice", "bob", "carla"], + vec!["smith", "jones", "doe"], + ); + let values = UInt32Array::from(vec![1, 1, 1]); + let initial_batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(user_array), Arc::new(values)]) + .unwrap(); + + let test_uri = "memory://test_merge_insert_struct_key.lance"; + let dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), + test_uri, + None, + ) + .await + .unwrap(); + let dataset = Arc::new(dataset); + + // New data: update alice, insert david + let new_user_array = + make_struct_array_first_last_name(vec!["alice", "david"], vec!["smith", "brown"]); + let new_values = UInt32Array::from(vec![10, 2]); + let new_batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(new_user_array), Arc::new(new_values)], + ) + .unwrap(); + + let reader = RecordBatchIterator::new([Ok(new_batch)], schema.clone()); + let (merged_ds, stats) = MergeInsertBuilder::try_new(dataset, vec!["user".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute(reader_to_stream(Box::new(reader))) + .await + .unwrap(); + + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_inserted_rows, 1); + assert_eq!(stats.num_deleted_rows, 0); + + let result = merged_ds.scan().try_into_batch().await.unwrap(); + let user_col = result + .column_by_name("user") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let first = user_col + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let last = user_col + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let values = result + .column_by_name("value") + .unwrap() + .as_primitive::(); + + let mut rows = Vec::new(); + for i in 0..result.num_rows() { + rows.push(( + first.value(i).to_string(), + last.value(i).to_string(), + values.value(i), + )); + } + rows.sort(); + + assert_eq!( + rows, + vec![ + ("alice".to_string(), "smith".to_string(), 10), + ("bob".to_string(), "jones".to_string(), 1), + ("carla".to_string(), "doe".to_string(), 1), + ("david".to_string(), "brown".to_string(), 2), + ], + ); + } + + fn make_struct_array_first_last_name(first: Vec<&str>, last: Vec<&str>) -> StructArray { + let first = StringArray::from(first); + let last = StringArray::from(last); + + StructArray::from(vec![ + ( + Arc::new(Field::new("first", DataType::Utf8, false)), + Arc::new(first) as Arc, + ), + ( + Arc::new(Field::new("last", DataType::Utf8, false)), + Arc::new(last) as Arc, + ), + ]) + } + + fn make_nested_struct_array_city_zip(city: &str, zip: i32) -> StructArray { + let city = StringArray::from(vec![city]); + let zip = Int32Array::from(vec![zip]); + + let inner_struct = StructArray::from(vec![ + ( + Arc::new(Field::new("city", DataType::Utf8, false)), + Arc::new(city) as Arc, + ), + ( + Arc::new(Field::new("zip", DataType::Int32, false)), + Arc::new(zip) as Arc, + ), + ]); + + StructArray::from(vec![( + Arc::new(Field::new( + "address", + inner_struct.data_type().clone(), + false, + )), + Arc::new(inner_struct) as Arc, + )]) + } + + fn make_nested_array(inner_lists: &[&[&str]]) -> ListArray { + let mut inner_builder = ListBuilder::new(StringBuilder::new()); + for inner in inner_lists { + inner_builder.append_value(inner.iter().map(|s| Some(*s))); + } + let inner_list_array = inner_builder.finish(); + + let offsets = ScalarBuffer::::from(vec![0, inner_list_array.len() as i32]); + let offsets = OffsetBuffer::new(offsets); + ListArray::new( + Arc::new(Field::new( + "item", + inner_list_array.data_type().clone(), + inner_list_array.nulls().is_some(), + )), + offsets, + Arc::new(inner_list_array), + None, + ) + } + + /// Test that merge_insert with bloom filter fails when committing against + /// an Update transaction that doesn't have a filter. We can't determine if + /// the Update operation conflicted with our inserted rows. + #[tokio::test] + async fn test_merge_insert_conflict_with_update_without_filter() { + use crate::dataset::UpdateBuilder; + + // Create schema with unenforced primary key on "id" column + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false).with_metadata( + vec![( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + )] + .into_iter() + .collect(), + ), + Field::new("value", DataType::UInt32, false), + ])); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), + ], + ) + .unwrap(); + + let dataset = InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(); + let dataset = Arc::new(dataset); + + // Create merge insert job based on version 1 + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![1])), + ], + ) + .unwrap(); + + let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .conflict_retries(0) + .try_build() + .unwrap(); + + // Regular Update without bloom filter commits first (creates version 2) + let update_result = UpdateBuilder::new(dataset.clone()) + .update_where("id = 0") + .unwrap() + .set("value", "999") + .unwrap() + .build() + .unwrap() + .execute() + .await; + assert!(update_result.is_ok(), "Update should succeed"); + + // Now merge insert tries to commit based on version 1, needs to rebase against version 2 + let s1 = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(batch1.clone())]), + ); + let merge_result = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + + // Merge insert should fail with retryable conflict because it can't + // determine if Update conflicted (Update has no inserted_rows_filter) + assert!( + matches!( + merge_result, + Err(crate::Error::TooMuchWriteContention { .. }) + ), + "Expected TooMuchWriteContention (retryable conflict exhausted), got: {:?}", + merge_result + ); + } + + /// Test that merge_insert with bloom filter fails when committing against + /// an Append operation. We can't determine if the appended rows conflict + /// with our inserted rows. + #[tokio::test] + async fn test_merge_insert_conflict_with_append() { + // Create schema with unenforced primary key on "id" column + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false).with_metadata( + vec![( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + )] + .into_iter() + .collect(), + ), + Field::new("value", DataType::UInt32, false), + ])); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), + ], + ) + .unwrap(); + + let dataset = InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(); + let dataset = Arc::new(dataset); + + // Create merge insert job based on version 1 + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![1])), + ], + ) + .unwrap(); + + let b1 = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .conflict_retries(0) + .try_build() + .unwrap(); + + // Append commits first (creates version 2) + let append_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![50])), + Arc::new(UInt32Array::from(vec![2])), + ], + ) + .unwrap(); + let append_result = InsertBuilder::new(dataset.clone()) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![append_batch]) + .await; + assert!(append_result.is_ok(), "Append should succeed"); + + // Now merge insert tries to commit based on version 1, needs to rebase against version 2 + let s1 = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(batch1.clone())]), + ); + let merge_result = b1.execute(Box::pin(s1) as SendableRecordBatchStream).await; + + // Merge insert should fail with retryable conflict because it can't + // determine if Append added conflicting keys + assert!( + matches!( + merge_result, + Err(crate::Error::TooMuchWriteContention { .. }) + ), + "Expected TooMuchWriteContention (retryable conflict exhausted), got: {:?}", + merge_result + ); + } + + #[tokio::test] + async fn test_explain_plan() { + // Set up test data using lance_datagen + let dataset = lance_datagen::gen_batch() + .col("id", lance_datagen::array::step::()) + .col("name", array::cycle_utf8_literals(&["a", "b", "c"])) + .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(3)) + .await + .unwrap(); + + // Create merge insert job + let merge_insert_job = + MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + + // Test explain_plan with default schema (None) + let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); + + // Also validate the full string structure with pattern matching + let expected_pattern = "\ +MergeInsert: on=[id], when_matched=UpdateAll, when_not_matched=InsertAll, when_not_matched_by_source=Keep... + CoalescePartitionsExec... + HashJoinExec... + LanceRead... + StreamingTableExec: partition_sizes=1, projection=[id, name]"; + assert_string_matches(&plan, expected_pattern).unwrap(); + + // Test with explicit schema + let source_schema = arrow_schema::Schema::from(dataset.schema()); + let explicit_plan = merge_insert_job + .explain_plan(Some(&source_schema), false) + .await + .unwrap(); + assert_eq!(plan, explicit_plan); // Should be the same as default + + // Test verbose mode produces different (likely longer) output + let verbose_plan = merge_insert_job.explain_plan(None, true).await.unwrap(); + assert!(verbose_plan.contains("MergeInsert")); + // Verbose should also match the expected pattern + assert_string_matches(&verbose_plan, expected_pattern).unwrap(); + } + + /// Asserts that `explain_plan()` is supported for a default find-or-create + /// configuration (`WhenMatched::DoNothing` + `WhenNotMatched::InsertAll`). + /// Before lance-format/lance#6441 this returned `Error::NotSupported` + /// because the job fell back to the legacy v1 path. + #[tokio::test] + async fn test_explain_plan_find_or_create() { + let dataset = lance_datagen::gen_batch() + .col("id", lance_datagen::array::step::()) + .col("name", array::cycle_utf8_literals(&["a", "b", "c"])) + .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(3)) + .await + .unwrap(); + + // Default builder config == find-or-create. + let merge_insert_job = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .try_build() + .unwrap(); + + let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); + + let expected_pattern = "\ +MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_not_matched_by_source=Keep... + CoalescePartitionsExec... + HashJoinExec...join_type=Right... + LanceRead... + StreamingTableExec: partition_sizes=1, projection=[id, name]"; + assert_string_matches(&plan, expected_pattern).unwrap(); + } + + #[tokio::test] + async fn test_explain_plan_full_schema_delete_by_source_with_fsl() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vec", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + ])); + + let dataset_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![ + 1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3, + ]), + 4, + ) + .unwrap(), + ), + ], + ) + .unwrap(); + + let dataset = Dataset::write( + Box::new(RecordBatchIterator::new( + [Ok(dataset_batch)], + schema.clone(), + )), + "memory://test_explain_plan_full_schema_delete_by_source_with_fsl", + None, + ) + .await + .unwrap(); + + let merge_insert_job = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched_by_source(WhenNotMatchedBySource::Delete) + .use_index(false) + .try_build() + .unwrap(); + + let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); + assert!(plan.contains("HashJoinExec")); + assert!(plan.contains("join_type=Full")); + assert!( + plan.lines().any(|line| line.contains("HashJoinExec") + && line.contains("projection=[") + && line.contains("_rowid")), + "join should push down a projection that retains _rowid: {plan}" + ); + assert!( + plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"), + "target-side scan should prune the FSL payload from the join build side: {plan}" + ); + assert!( + !plan.contains("LanceRead: uri=test_explain_plan_full_schema_delete_by_source_with_fsl/data, projection=[id, vec]"), + "target-side scan should not include the FSL payload in the join build side: {plan}" + ); + } + + #[tokio::test] + async fn test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vec", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + ])); + + let dataset_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![ + 1.0, 1.1, 1.2, 1.3, 2.0, 2.1, 2.2, 2.3, 3.0, 3.1, 3.2, 3.3, + ]), + 4, + ) + .unwrap(), + ), + ], + ) + .unwrap(); + + let mut dataset = Dataset::write( + Box::new(RecordBatchIterator::new( + [Ok(dataset_batch)], + schema.clone(), + )), + "memory://test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index", + None, + ) + .await + .unwrap(); + + let scalar_params = ScalarIndexParams::default(); + dataset + .create_index(&["id"], IndexType::Scalar, None, &scalar_params, false) + .await + .unwrap(); + + let merge_insert_job = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .when_not_matched_by_source(WhenNotMatchedBySource::Delete) + .try_build() + .unwrap(); + + let plan = merge_insert_job.explain_plan(None, false).await.unwrap(); + assert!(plan.contains("HashJoinExec")); + assert!(plan.contains("join_type=Full")); + assert!( + plan.lines().any(|line| line.contains("HashJoinExec") + && line.contains("projection=[") + && line.contains("_rowid")), + "join should push down a projection that retains _rowid: {plan}" + ); + assert!( + plan.contains("LanceRead: uri=") && plan.contains("projection=[id]"), + "target-side scan should prune the FSL payload from the join build side even when a scalar index exists: {plan}" + ); + assert!( + !plan.contains( "LanceRead: uri=test_explain_plan_full_schema_delete_by_source_with_fsl_and_scalar_index/data, projection=[id, vec]" ), "target-side scan should not include the FSL payload in the join build side: {plan}" @@ -8578,6 +11520,95 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n ); } + #[rstest::rstest] + #[case::v2(false)] + #[case::indexed_scan(true)] + #[tokio::test] + async fn test_first_seen_dedupes_unmatched_source_rows(#[case] with_index: bool) { + let initial = + record_batch!(("id", Int32, [Some(1)]), ("value", Int32, [Some(10)])).unwrap(); + let initial = if with_index { + initial + } else { + // Match the reported failure's empty-target setup on the v2 path. + initial.slice(0, 0) + }; + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(initial.clone())], initial.schema()), + "memory://", + None, + ) + .await + .unwrap(); + + if with_index { + dataset + .create_index( + &["id"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + } + + // Split distinct duplicate values across batches to verify that FirstSeen + // preserves source order across the entire stream. On the v2 path, also + // verify that NULL keys remain distinct under SQL equality. + let (first, second, expected_inserted) = if with_index { + ( + record_batch!(("id", Int32, [Some(108)]), ("value", Int32, [Some(1)])).unwrap(), + record_batch!(("id", Int32, [Some(108)]), ("value", Int32, [Some(2)])).unwrap(), + 1, + ) + } else { + ( + record_batch!( + ("id", Int32, [Some(108), None]), + ("value", Int32, [Some(1), Some(3)]) + ) + .unwrap(), + record_batch!( + ("id", Int32, [Some(108), None]), + ("value", Int32, [Some(2), Some(4)]) + ) + .unwrap(), + 3, + ) + }; + + let (dataset, stats) = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .source_dedupe_behavior(SourceDedupeBehavior::FirstSeen) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(first.clone()), Ok(second)], + first.schema(), + ))) + .await + .unwrap(); + + assert_eq!(stats.num_inserted_rows, expected_inserted); + assert_eq!(stats.num_updated_rows, 0); + assert_eq!(stats.num_skipped_duplicates, 1); + + let inserted = dataset + .scan() + .filter("id = 108") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(inserted.num_rows(), 1); + assert_eq!(inserted["value"].as_primitive::().value(0), 1); + } + #[tokio::test] async fn test_merge_insert_use_index() { let data = lance_datagen::gen_batch() @@ -8788,7 +11819,156 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n } #[tokio::test] - async fn test_sub_schema_upsert_fragment_bitmap() { + async fn test_sub_schema_upsert_fragment_bitmap() { + let mut dataset = lance_datagen::gen_batch() + .col("key", array::step_custom::(1, 1)) + .col("value", array::step_custom::(10, 10)) + .col( + "vec", + array::cycle_vec( + array::cycle::(vec![ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, + 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, + ]), + Dimension::from(4), + ), + ) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let scalar_params = ScalarIndexParams::default(); + dataset + .create_index( + &["value"], + IndexType::Scalar, + Some("value_idx".to_string()), + &scalar_params, + true, + ) + .await + .unwrap(); + + let vector_params = VectorIndexParams::ivf_flat(1, MetricType::L2); + dataset + .create_index( + &["vec"], + IndexType::Vector, + Some("vec_idx".to_string()), + &vector_params, + true, + ) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + let value_index = indices.iter().find(|idx| idx.name == "value_idx").unwrap(); + let vec_index = indices.iter().find(|idx| idx.name == "vec_idx").unwrap(); + + assert_eq!( + value_index + .fragment_bitmap + .as_ref() + .unwrap() + .iter() + .collect::>(), + vec![0, 1] + ); + assert_eq!( + vec_index + .fragment_bitmap + .as_ref() + .unwrap() + .iter() + .collect::>(), + vec![0, 1] + ); + + let sub_schema = Arc::new(Schema::new(vec![ + Field::new("key", DataType::UInt32, true), + Field::new( + "vec", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + ])); + + let upsert_keys = UInt32Array::from(vec![2, 5]); + let upsert_vecs = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0]), + 4, + ) + .unwrap(); + + let upsert_batch = RecordBatch::try_new( + sub_schema.clone(), + vec![Arc::new(upsert_keys), Arc::new(upsert_vecs)], + ) + .unwrap(); + + let upsert_stream = RecordBatchStreamAdapter::new( + sub_schema.clone(), + futures::stream::once(async { Ok(upsert_batch) }).boxed(), + ); + + let (updated_dataset, _stats) = + MergeInsertBuilder::try_new(Arc::new(dataset), vec!["key".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .when_not_matched_by_source(WhenNotMatchedBySource::Keep) + .try_build() + .unwrap() + .execute(Box::pin(upsert_stream)) + .await + .unwrap(); + + let fragments = updated_dataset.get_fragments(); + // v2 path: partial-schema upsert goes through FullSchemaMergeInsertExec + // which writes a new fragment containing the updated rows. Fragments + // 0 and 1 keep 2 rows each (with deletion vectors covering the + // matched keys), fragment 2 is the new one holding the 2 updated + // rows. The in-place RewriteColumns alternative is opt-in; see + // `test_sub_schema_upsert_in_place_fragment_bitmap`. + assert_eq!(fragments.len(), 3); + + let updated_indices = updated_dataset.load_indices().await.unwrap(); + // Both indices remain after the v2 upsert. The vector index still + // covers the old fragments' non-deleted rows (the deleted rows are + // filtered by deletion vectors), and queries that need the new + // row values fall back to scanning the unindexed new fragment. + // This is a behavior difference from v1, which eagerly invalidated + // the vec index when any row in a fragment was updated. + assert_eq!(updated_indices.len(), 2); + let updated_value_index = updated_indices + .iter() + .find(|idx| idx.name == "value_idx") + .unwrap(); + + // The scalar index on `value` must still cover fragments 0 and 1 — + // even though those fragments now carry deletion vectors, the + // `value` column itself was not modified, so the existing index + // entries remain valid for the rows that were not deleted. + let value_bitmap = updated_value_index.fragment_bitmap.as_ref().unwrap(); + assert!(value_bitmap.contains(0)); + assert!(value_bitmap.contains(1)); + } + + /// The opt-in counterpart of `test_sub_schema_upsert_fragment_bitmap`. + /// Patching `vec` in place keeps the fragments and their ids, but it does + /// rewrite that column's data, so every index over `vec` must lose the + /// patched fragments while an index over an untouched column keeps them. + /// `fields_modified` is what drives that pruning. + #[tokio::test] + async fn test_sub_schema_upsert_in_place_fragment_bitmap() { let mut dataset = lance_datagen::gen_batch() .col("key", array::step_custom::(1, 1)) .col("value", array::step_custom::(10, 10)) @@ -8825,7 +12005,6 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n ) .await .unwrap(); - let vector_params = VectorIndexParams::ivf_flat(1, MetricType::L2); dataset .create_index( @@ -8838,29 +12017,6 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .await .unwrap(); - let indices = dataset.load_indices().await.unwrap(); - let value_index = indices.iter().find(|idx| idx.name == "value_idx").unwrap(); - let vec_index = indices.iter().find(|idx| idx.name == "vec_idx").unwrap(); - - assert_eq!( - value_index - .fragment_bitmap - .as_ref() - .unwrap() - .iter() - .collect::>(), - vec![0, 1] - ); - assert_eq!( - vec_index - .fragment_bitmap - .as_ref() - .unwrap() - .iter() - .collect::>(), - vec![0, 1] - ); - let sub_schema = Arc::new(Schema::new(vec![ Field::new("key", DataType::UInt32, true), Field::new( @@ -8869,20 +12025,20 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n true, ), ])); - - let upsert_keys = UInt32Array::from(vec![2, 5]); - let upsert_vecs = FixedSizeListArray::try_new_from_values( - Float32Array::from(vec![21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0]), - 4, - ) - .unwrap(); - let upsert_batch = RecordBatch::try_new( sub_schema.clone(), - vec![Arc::new(upsert_keys), Arc::new(upsert_vecs)], + vec![ + Arc::new(UInt32Array::from(vec![2, 5])), + Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0]), + 4, + ) + .unwrap(), + ), + ], ) .unwrap(); - let upsert_stream = RecordBatchStreamAdapter::new( sub_schema.clone(), futures::stream::once(async { Ok(upsert_batch) }).boxed(), @@ -8894,6 +12050,7 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::DoNothing) .when_not_matched_by_source(WhenNotMatchedBySource::Keep) + .write_mode(MergeInsertWriteMode::RewriteColumns) .try_build() .unwrap() .execute(Box::pin(upsert_stream)) @@ -8901,34 +12058,46 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .unwrap(); let fragments = updated_dataset.get_fragments(); - // v2 path: partial-schema upsert goes through FullSchemaMergeInsertExec - // which writes a new fragment containing the updated rows. Fragments - // 0 and 1 keep 2 rows each (with deletion vectors covering the - // matched keys), fragment 2 is the new one holding the 2 updated - // rows. The v1 RewriteColumns optimization (2 fragments, in-place - // rewrite) is tracked separately as issue #4193. - assert_eq!(fragments.len(), 3); + assert_eq!( + fragments.len(), + 2, + "in-place patches must not add or remove fragments" + ); + for fragment in &fragments { + assert!( + fragment.metadata().deletion_file.is_none(), + "in-place patches must not produce deletion vectors" + ); + } let updated_indices = updated_dataset.load_indices().await.unwrap(); - // Both indices remain after the v2 upsert. The vector index still - // covers the old fragments' non-deleted rows (the deleted rows are - // filtered by deletion vectors), and queries that need the new - // row values fall back to scanning the unindexed new fragment. - // This is a behavior difference from v1, which eagerly invalidated - // the vec index when any row in a fragment was updated. assert_eq!(updated_indices.len(), 2); - let updated_value_index = updated_indices + let value_bitmap = updated_indices .iter() .find(|idx| idx.name == "value_idx") - .unwrap(); + .unwrap() + .fragment_bitmap + .as_ref() + .unwrap() + .clone(); + let vec_bitmap = updated_indices + .iter() + .find(|idx| idx.name == "vec_idx") + .unwrap() + .fragment_bitmap + .as_ref() + .unwrap() + .clone(); - // The scalar index on `value` must still cover fragments 0 and 1 — - // even though those fragments now carry deletion vectors, the - // `value` column itself was not modified, so the existing index - // entries remain valid for the rows that were not deleted. - let value_bitmap = updated_value_index.fragment_bitmap.as_ref().unwrap(); + // `value` was not patched, so its index still describes what is stored. assert!(value_bitmap.contains(0)); assert!(value_bitmap.contains(1)); + // `vec` was rewritten in both fragments, so its index no longer does. + assert!( + vec_bitmap.is_empty(), + "index over a patched column must be invalidated for the patched fragments, got {:?}", + vec_bitmap.iter().collect::>() + ); } #[tokio::test] @@ -9559,65 +12728,233 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n assert_eq!(result, expected); } - /// Test WhenMatched::Delete with full schema source data. - /// Source contains all columns (key, value, filterme) but we only use it to identify - /// rows to delete - no data is written back. + /// Test WhenMatched::Delete with full schema source data. + /// Source contains all columns (key, value, filterme) but we only use it to identify + /// rows to delete - no data is written back. + #[rstest::rstest] + #[tokio::test] + async fn test_when_matched_delete_full_schema( + #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, + #[values(true, false)] enable_stable_row_ids: bool, + ) { + let schema = create_test_schema(); + let test_uri = "memory://test_delete_full.lance"; + + // Create dataset with keys 1-6 (value=1) + let ds = create_test_dataset(test_uri, version, enable_stable_row_ids).await; + + // Source data has keys 4, 5, 6, 7, 8, 9 with full schema + // Keys 4, 5, 6 match existing rows and should be deleted + // Keys 7, 8, 9 don't match (and we're not inserting) + let new_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![4, 5, 6, 7, 8, 9])), + Arc::new(UInt32Array::from(vec![2, 2, 2, 2, 2, 2])), + Arc::new(StringArray::from(vec!["A", "B", "C", "A", "B", "C"])), + ], + ) + .unwrap(); + + let keys = vec!["key".to_string()]; + + // First, verify the execution plan structure + // Delete-only should use Inner join and only include key columns (optimization) + // Action 3 = Delete + let plan_job = MergeInsertBuilder::try_new(ds.clone(), keys.clone()) + .unwrap() + .when_matched(WhenMatched::Delete) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap(); + let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( + [Ok(new_batch.clone())], + schema.clone(), + ))); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); + assert_plan_node_equals( + plan, + "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing + ... + HashJoinExec: ...join_type=Inner... + ... + ... + StreamingTableExec: partition_sizes=1, projection=[key]", + ) + .await + .unwrap(); + let job = MergeInsertBuilder::try_new(ds.clone(), keys) + .unwrap() + .when_matched(WhenMatched::Delete) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap(); + + let new_reader = Box::new(RecordBatchIterator::new([Ok(new_batch)], schema.clone())); + let new_stream = reader_to_stream(new_reader); + + let (merged_dataset, merge_stats) = job.execute(new_stream).await.unwrap(); + + // Should have deleted 3 rows (keys 4, 5, 6) + assert_eq!(merge_stats.num_deleted_rows, 3); + assert_eq!(merge_stats.num_inserted_rows, 0); + assert_eq!(merge_stats.num_updated_rows, 0); + + // Verify remaining data - only keys 1, 2, 3 should remain + let batches = merged_dataset + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + let merged = concat_batches(&schema, &batches).unwrap(); + let mut remaining_keys: Vec = merged + .column(0) + .as_primitive::() + .values() + .to_vec(); + remaining_keys.sort(); + assert_eq!(remaining_keys, vec![1, 2, 3]); + } + + /// Test WhenMatched::Delete with ID-only source data (just key column). + /// This is the optimized bulk delete case where we only need key columns for matching. + #[rstest::rstest] + #[tokio::test] + async fn test_when_matched_delete_id_only( + #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, + #[values(true, false)] enable_stable_row_ids: bool, + ) { + let test_uri = "memory://test_delete_id_only.lance"; + + // Create dataset with keys 1-6 (full schema: key, value, filterme) + let ds = create_test_dataset(test_uri, version, enable_stable_row_ids).await; + let id_only_schema = Arc::new(Schema::new(vec![Field::new("key", DataType::UInt32, true)])); + let new_batch = RecordBatch::try_new( + id_only_schema.clone(), + vec![Arc::new(UInt32Array::from(vec![2, 4, 6]))], // Delete keys 2, 4, 6 + ) + .unwrap(); + + let keys = vec!["key".to_string()]; + + // ID-only delete should use Inner join with key-only projection + // on=[(key@0, key@0)] because key is at position 0 in both target and source + let plan_job = MergeInsertBuilder::try_new(ds.clone(), keys.clone()) + .unwrap() + .when_matched(WhenMatched::Delete) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap(); + let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( + [Ok(new_batch.clone())], + id_only_schema.clone(), + ))); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); + assert_plan_node_equals( + plan, + "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing + ... + HashJoinExec: ...join_type=Inner... + ... + ... + StreamingTableExec: partition_sizes=1, projection=[key]", + ) + .await + .unwrap(); + let job = MergeInsertBuilder::try_new(ds.clone(), keys) + .unwrap() + .when_matched(WhenMatched::Delete) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap(); + + let new_reader = Box::new(RecordBatchIterator::new( + [Ok(new_batch)], + id_only_schema.clone(), + )); + let new_stream = reader_to_stream(new_reader); + + let (merged_dataset, merge_stats) = job.execute(new_stream).await.unwrap(); + + // Should have deleted 3 rows (keys 2, 4, 6) + assert_eq!(merge_stats.num_deleted_rows, 3); + assert_eq!(merge_stats.num_inserted_rows, 0); + assert_eq!(merge_stats.num_updated_rows, 0); + + // Verify remaining data - only keys 1, 3, 5 should remain + let full_schema = create_test_schema(); + let batches = merged_dataset + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + + let merged = concat_batches(&full_schema, &batches).unwrap(); + let mut remaining_keys: Vec = merged + .column(0) + .as_primitive::() + .values() + .to_vec(); + remaining_keys.sort(); + assert_eq!(remaining_keys, vec![1, 3, 5]); + } + + /// Test WhenMatched::Delete combined with WhenNotMatched::InsertAll. + /// This replaces existing matching rows with nothing (delete) while inserting new rows. #[rstest::rstest] #[tokio::test] - async fn test_when_matched_delete_full_schema( + async fn test_when_matched_delete_with_insert( #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, - #[values(true, false)] enable_stable_row_ids: bool, ) { let schema = create_test_schema(); - let test_uri = "memory://test_delete_full.lance"; + let test_uri = "memory://test_delete_with_insert.lance"; - // Create dataset with keys 1-6 (value=1) - let ds = create_test_dataset(test_uri, version, enable_stable_row_ids).await; + // Create dataset with keys 1-6 + let ds = create_test_dataset(test_uri, version, false).await; - // Source data has keys 4, 5, 6, 7, 8, 9 with full schema - // Keys 4, 5, 6 match existing rows and should be deleted - // Keys 7, 8, 9 don't match (and we're not inserting) - let new_batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![4, 5, 6, 7, 8, 9])), - Arc::new(UInt32Array::from(vec![2, 2, 2, 2, 2, 2])), - Arc::new(StringArray::from(vec!["A", "B", "C", "A", "B", "C"])), - ], - ) - .unwrap(); + // Source has keys 4, 5, 6 (match - will be deleted) and 7, 8, 9 (new - will be inserted) + let new_batch = create_new_batch(schema.clone()); let keys = vec!["key".to_string()]; - // First, verify the execution plan structure - // Delete-only should use Inner join and only include key columns (optimization) - // Action 3 = Delete + // Delete + Insert should use Right join to see unmatched rows for insertion let plan_job = MergeInsertBuilder::try_new(ds.clone(), keys.clone()) .unwrap() .when_matched(WhenMatched::Delete) - .when_not_matched(WhenNotMatched::DoNothing) + .when_not_matched(WhenNotMatched::InsertAll) .try_build() .unwrap(); let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( [Ok(new_batch.clone())], schema.clone(), ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); assert_plan_node_equals( plan, - "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing - ... - HashJoinExec: ...join_type=Inner... - ... - ... - StreamingTableExec: partition_sizes=1, projection=[key]", - ) - .await - .unwrap(); + "MergeInsert: on=[key], when_matched=Delete, when_not_matched=InsertAll, when_not_matched_by_source=Keep...THEN 2 WHEN...THEN 3 ELSE 0 END as __action]...projection=[key, value, filterme]" + ).await.unwrap(); + + // Delete matched rows, insert unmatched rows let job = MergeInsertBuilder::try_new(ds.clone(), keys) .unwrap() .when_matched(WhenMatched::Delete) - .when_not_matched(WhenNotMatched::DoNothing) + .when_not_matched(WhenNotMatched::InsertAll) .try_build() .unwrap(); @@ -9626,12 +12963,12 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n let (merged_dataset, merge_stats) = job.execute(new_stream).await.unwrap(); - // Should have deleted 3 rows (keys 4, 5, 6) + // Deleted 3 (keys 4, 5, 6), inserted 3 (keys 7, 8, 9) assert_eq!(merge_stats.num_deleted_rows, 3); - assert_eq!(merge_stats.num_inserted_rows, 0); + assert_eq!(merge_stats.num_inserted_rows, 3); assert_eq!(merge_stats.num_updated_rows, 0); - // Verify remaining data - only keys 1, 2, 3 should remain + // Verify: keys 1, 2, 3 (original, not matched), 7, 8, 9 (new inserts) let batches = merged_dataset .scan() .try_into_stream() @@ -9648,32 +12985,60 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .values() .to_vec(); remaining_keys.sort(); - assert_eq!(remaining_keys, vec![1, 2, 3]); + assert_eq!(remaining_keys, vec![1, 2, 3, 7, 8, 9]); + + // Verify values: keys 1, 2, 3 have value=1 (original), keys 7, 8, 9 have value=2 (new) + let keyvals: Vec<(u32, u32)> = merged + .column(0) + .as_primitive::() + .values() + .iter() + .zip( + merged + .column(1) + .as_primitive::() + .values() + .iter(), + ) + .map(|(&k, &v)| (k, v)) + .collect(); + + for (key, value) in keyvals { + if key <= 3 { + assert_eq!(value, 1, "Original keys should have value=1"); + } else { + assert_eq!(value, 2, "New keys should have value=2"); + } + } } - /// Test WhenMatched::Delete with ID-only source data (just key column). - /// This is the optimized bulk delete case where we only need key columns for matching. + /// Test WhenMatched::Delete when source data has no matching keys. + /// This should result in zero deletes and the dataset remains unchanged. #[rstest::rstest] #[tokio::test] - async fn test_when_matched_delete_id_only( + async fn test_when_matched_delete_no_matches( #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, - #[values(true, false)] enable_stable_row_ids: bool, ) { - let test_uri = "memory://test_delete_id_only.lance"; + let schema = create_test_schema(); + let test_uri = "memory://test_delete_no_matches.lance"; - // Create dataset with keys 1-6 (full schema: key, value, filterme) - let ds = create_test_dataset(test_uri, version, enable_stable_row_ids).await; - let id_only_schema = Arc::new(Schema::new(vec![Field::new("key", DataType::UInt32, true)])); - let new_batch = RecordBatch::try_new( - id_only_schema.clone(), - vec![Arc::new(UInt32Array::from(vec![2, 4, 6]))], // Delete keys 2, 4, 6 + // Create dataset with keys 1-6 + let ds = create_test_dataset(test_uri, version, false).await; + + // Source data has keys 100, 200, 300 - none match existing keys 1-6 + let non_matching_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100, 200, 300])), + Arc::new(UInt32Array::from(vec![10, 20, 30])), + Arc::new(StringArray::from(vec!["X", "Y", "Z"])), + ], ) .unwrap(); let keys = vec!["key".to_string()]; - // ID-only delete should use Inner join with key-only projection - // on=[(key@0, key@0)] because key is at position 0 in both target and source + // Even with no matches, the plan structure should be the same let plan_job = MergeInsertBuilder::try_new(ds.clone(), keys.clone()) .unwrap() .when_matched(WhenMatched::Delete) @@ -9681,10 +13046,13 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .try_build() .unwrap(); let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( - [Ok(new_batch.clone())], - id_only_schema.clone(), + [Ok(non_matching_batch.clone())], + schema.clone(), ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); + let plan = plan_job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); assert_plan_node_equals( plan, "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing @@ -9704,20 +13072,19 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .unwrap(); let new_reader = Box::new(RecordBatchIterator::new( - [Ok(new_batch)], - id_only_schema.clone(), + [Ok(non_matching_batch)], + schema.clone(), )); let new_stream = reader_to_stream(new_reader); let (merged_dataset, merge_stats) = job.execute(new_stream).await.unwrap(); - // Should have deleted 3 rows (keys 2, 4, 6) - assert_eq!(merge_stats.num_deleted_rows, 3); + // Should have deleted 0 rows since no keys matched + assert_eq!(merge_stats.num_deleted_rows, 0); assert_eq!(merge_stats.num_inserted_rows, 0); assert_eq!(merge_stats.num_updated_rows, 0); - // Verify remaining data - only keys 1, 3, 5 should remain - let full_schema = create_test_schema(); + // Verify all original data remains unchanged - keys 1-6 should all still be present let batches = merged_dataset .scan() .try_into_stream() @@ -9727,184 +13094,343 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .await .unwrap(); - let merged = concat_batches(&full_schema, &batches).unwrap(); + let merged = concat_batches(&schema, &batches).unwrap(); let mut remaining_keys: Vec = merged .column(0) .as_primitive::() .values() .to_vec(); remaining_keys.sort(); - assert_eq!(remaining_keys, vec![1, 3, 5]); + assert_eq!(remaining_keys, vec![1, 2, 3, 4, 5, 6]); } - /// Test WhenMatched::Delete combined with WhenNotMatched::InsertAll. - /// This replaces existing matching rows with nothing (delete) while inserting new rows. - #[rstest::rstest] + /// Test that MergeInsertPlanner::is_delete_only correctly identifies delete-only operations. + /// + /// Delete-only is true only when: + /// - when_matched = Delete + /// - insert_not_matched = false (WhenNotMatched::DoNothing) + /// - delete_not_matched_by_source = Keep + /// + /// This test iterates through all valid combinations of WhenMatched, WhenNotMatched, + /// and WhenNotMatchedBySource to verify the is_delete_only logic. #[tokio::test] - async fn test_when_matched_delete_with_insert( - #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, - ) { + async fn test_is_delete_only() { + use itertools::iproduct; + + // All variants to test (excluding UpdateIf and DeleteIf because they require expressions) + let when_matched_variants = [ + WhenMatched::UpdateAll, + WhenMatched::DoNothing, + WhenMatched::Fail, + WhenMatched::Delete, + ]; + let when_not_matched_variants = [WhenNotMatched::InsertAll, WhenNotMatched::DoNothing]; + let when_not_matched_by_source_variants = + [WhenNotMatchedBySource::Keep, WhenNotMatchedBySource::Delete]; + let schema = create_test_schema(); - let test_uri = "memory://test_delete_with_insert.lance"; - // Create dataset with keys 1-6 - let ds = create_test_dataset(test_uri, version, false).await; + for (idx, (when_matched, when_not_matched, when_not_matched_by_source)) in iproduct!( + when_matched_variants.iter().cloned(), + when_not_matched_variants.iter().cloned(), + when_not_matched_by_source_variants.iter().cloned() + ) + .enumerate() + { + // Check if this is a valid (non-no-op) combination, since this would fail try_build() + let is_no_op = matches!(when_matched, WhenMatched::DoNothing | WhenMatched::Fail) + && matches!(when_not_matched, WhenNotMatched::DoNothing) + && matches!(when_not_matched_by_source, WhenNotMatchedBySource::Keep); + if is_no_op { + continue; + } + + let test_uri = format!("memory://test_is_delete_only_{}.lance", idx); + let ds = create_test_dataset(&test_uri, LanceFileVersion::V2_0, false).await; + + let new_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![4, 5, 6])), + Arc::new(UInt32Array::from(vec![2, 2, 2])), + Arc::new(StringArray::from(vec!["A", "B", "C"])), + ], + ) + .unwrap(); + + let keys = vec!["key".to_string()]; + + let mut builder = MergeInsertBuilder::try_new(ds.clone(), keys).unwrap(); + builder + .when_matched(when_matched.clone()) + .when_not_matched(when_not_matched.clone()) + .when_not_matched_by_source(when_not_matched_by_source.clone()); + + let job = builder.try_build().unwrap(); + + let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( + [Ok(new_batch)], + schema.clone(), + ))); + let plan = job + .create_plan(one_shot_provider(plan_stream).unwrap()) + .await + .unwrap(); + + let plan_str = datafusion::physical_plan::displayable(plan.as_ref()) + .indent(true) + .to_string(); + + let expected_delete_only = matches!(when_matched, WhenMatched::Delete) + && matches!(when_not_matched, WhenNotMatched::DoNothing) + && matches!(when_not_matched_by_source, WhenNotMatchedBySource::Keep); + + if expected_delete_only { + assert!( + plan_str.contains("DeleteOnlyMergeInsert"), + "Expected DeleteOnlyMergeInsert for ({:?}, {:?}, {:?}), but got:\n{}", + when_matched, + when_not_matched, + when_not_matched_by_source, + plan_str + ); + } else { + assert!( + plan_str.contains("MergeInsert:") + && !plan_str.contains("DeleteOnlyMergeInsert"), + "Expected MergeInsert (not DeleteOnlyMergeInsert) for ({:?}, {:?}, {:?}), but got:\n{}", + when_matched, + when_not_matched, + when_not_matched_by_source, + plan_str + ); + } + } + } + + /// Tests that apply_deletions correctly handles an error when applying the row deletions. + #[tokio::test] + async fn test_apply_deletions_invalid_row_address() { + use super::exec::apply_deletions; + use roaring::RoaringTreemap; + + let test_uri = "memory://test_apply_deletions_error.lance"; + + // Create a dataset with 2 fragments, each with 3 rows + let ds = create_test_dataset(test_uri, LanceFileVersion::V2_0, false).await; + let fragment_id = ds.get_fragments()[0].id() as u32; + + // Create row addresses with invalid row offsets for this fragment + // Row address format: high 32 bits = fragment_id, low 32 bits = row_offset + // Each fragment has only 3 rows (offsets 0, 1, 2). + // + // The error in extend_deletions is triggered when deletion_vector.len() >= physical_rows + // AND at least one row ID is >= physical_rows. + // So we need to add enough deletions (at least 3) with some being invalid (>= 3). + let mut invalid_row_addrs = RoaringTreemap::new(); + let base = (fragment_id as u64) << 32; + // Add 4 deletions: rows 10, 11, 12, 13 (all invalid since only rows 0-2 exist) + for row_offset in 10..14u64 { + invalid_row_addrs.insert(base | row_offset); + } - // Source has keys 4, 5, 6 (match - will be deleted) and 7, 8, 9 (new - will be inserted) - let new_batch = create_new_batch(schema.clone()); + let result = apply_deletions(&ds, &invalid_row_addrs).await; - let keys = vec!["key".to_string()]; + assert!(result.is_err(), "Expected error for invalid row addresses"); + let err = result.unwrap_err(); + assert!( + err.to_string() + .contains("Deletion vector includes rows that aren't in the fragment"), + "Expected 'rows that aren't in the fragment' error, got: {}", + err + ); + } - // Delete + Insert should use Right join to see unmatched rows for insertion - let plan_job = MergeInsertBuilder::try_new(ds.clone(), keys.clone()) - .unwrap() - .when_matched(WhenMatched::Delete) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap(); - let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( - [Ok(new_batch.clone())], - schema.clone(), - ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); - assert_plan_node_equals( - plan, - "MergeInsert: on=[key], when_matched=Delete, when_not_matched=InsertAll, when_not_matched_by_source=Keep...THEN 2 WHEN...THEN 3 ELSE 0 END as __action]...projection=[key, value, filterme]" - ).await.unwrap(); + mod external_error { + use super::*; + use arrow_schema::{ArrowError, Field as ArrowField, Schema as ArrowSchema}; + use std::fmt; - // Delete matched rows, insert unmatched rows - let job = MergeInsertBuilder::try_new(ds.clone(), keys) - .unwrap() - .when_matched(WhenMatched::Delete) - .when_not_matched(WhenNotMatched::InsertAll) - .try_build() - .unwrap(); + #[derive(Debug)] + struct MyTestError { + code: i32, + details: String, + } - let new_reader = Box::new(RecordBatchIterator::new([Ok(new_batch)], schema.clone())); - let new_stream = reader_to_stream(new_reader); + impl fmt::Display for MyTestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "MyTestError({}): {}", self.code, self.details) + } + } - let (merged_dataset, merge_stats) = job.execute(new_stream).await.unwrap(); + impl std::error::Error for MyTestError {} - // Deleted 3 (keys 4, 5, 6), inserted 3 (keys 7, 8, 9) - assert_eq!(merge_stats.num_deleted_rows, 3); - assert_eq!(merge_stats.num_inserted_rows, 3); - assert_eq!(merge_stats.num_updated_rows, 0); + #[tokio::test] + async fn test_merge_insert_execute_reader_preserves_error_message() { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("key", DataType::Int32, false), + ArrowField::new("value", DataType::Int32, false), + ])); - // Verify: keys 1, 2, 3 (original, not matched), 7, 8, 9 (new inserts) - let batches = merged_dataset - .scan() - .try_into_stream() - .await - .unwrap() - .try_collect::>() - .await + // Create initial dataset + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(Int32Array::from(vec![10, 20, 30])), + ], + ) .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let dataset = Arc::new( + Dataset::write(reader, "memory://test_merge_external", None) + .await + .unwrap(), + ); - let merged = concat_batches(&schema, &batches).unwrap(); - let mut remaining_keys: Vec = merged - .column(0) - .as_primitive::() - .values() - .to_vec(); - remaining_keys.sort(); - assert_eq!(remaining_keys, vec![1, 2, 3, 7, 8, 9]); + // Try merge insert with failing source + let error_code = 789; + let iter = std::iter::once(Err(ArrowError::ExternalError(Box::new(MyTestError { + code: error_code, + details: "merge insert failure".to_string(), + })))); + let reader = RecordBatchIterator::new(iter, schema); - // Verify values: keys 1, 2, 3 have value=1 (original), keys 7, 8, 9 have value=2 (new) - let keyvals: Vec<(u32, u32)> = merged - .column(0) - .as_primitive::() - .values() - .iter() - .zip( - merged - .column(1) - .as_primitive::() - .values() - .iter(), - ) - .map(|(&k, &v)| (k, v)) - .collect(); + let result = MergeInsertBuilder::try_new(dataset, vec!["key".to_string()]) + .unwrap() + .try_build() + .unwrap() + .execute_reader(Box::new(reader) as Box) + .await; - for (key, value) in keyvals { - if key <= 3 { - assert_eq!(value, 1, "Original keys should have value=1"); - } else { - assert_eq!(value, 2, "New keys should have value=2"); - } + // The source error is routed through the merge plan, which shares it + // across join partitions, so its concrete type is not recoverable. The + // message must still reach the caller. + let err = result.expect_err("expected the source error to surface"); + assert!( + err.to_string().contains("merge insert failure"), + "source error message should be preserved; got: {err}" + ); } } - /// Test WhenMatched::Delete when source data has no matching keys. - /// This should result in zero deletes and the dataset remains unchanged. - #[rstest::rstest] - #[tokio::test] - async fn test_when_matched_delete_no_matches( - #[values(LanceFileVersion::Legacy, LanceFileVersion::V2_0)] version: LanceFileVersion, - ) { - let schema = create_test_schema(); - let test_uri = "memory://test_delete_no_matches.lance"; + /// Creates a 3-fragment dataset (100 rows each) with columns (id: Utf8, category: Utf8, + /// value_a: Float64, value_b: Float64) and a BTree index on `id`. + /// + /// Fragment 0: id-0000..id-0099 + /// Fragment 1: id-0100..id-0199 + /// Fragment 2: id-0200..id-0299 + async fn create_indexed_3frag_dataset() -> Arc { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new("value_a", DataType::Float64, false), + Field::new("value_b", DataType::Float64, false), + ])); - // Create dataset with keys 1-6 - let ds = create_test_dataset(test_uri, version, false).await; + let make_batch = |frag_idx: usize| { + let start = frag_idx * 100; + let ids: Vec = (start..start + 100).map(|j| format!("id-{j:04}")).collect(); + let categories: Vec<&str> = vec!["A"; 100]; + let value_a: Vec = (0..100) + .map(|i| i as f64 + frag_idx as f64 * 100.0) + .collect(); + let value_b: Vec = (0..100).map(|i| i as f64 * 0.1).collect(); + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(ids)), + Arc::new(StringArray::from(categories)), + Arc::new(Float64Array::from(value_a)), + Arc::new(Float64Array::from(value_b)), + ], + ) + .unwrap() + }; - // Source data has keys 100, 200, 300 - none match existing keys 1-6 - let non_matching_batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![100, 200, 300])), - Arc::new(UInt32Array::from(vec![10, 20, 30])), - Arc::new(StringArray::from(vec!["X", "Y", "Z"])), - ], + // Write first fragment + let batch0 = make_batch(0); + let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); + let mut ds = Dataset::write(reader, "memory://indexed_3frag", None) + .await + .unwrap(); + + // Append fragments 1 and 2 + for frag_idx in 1..3 { + let batch = make_batch(frag_idx); + let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); + ds.append(reader, None).await.unwrap(); + } + + // Create BTree index on id + ds.create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + false, ) + .await .unwrap(); - let keys = vec!["key".to_string()]; + Arc::new(ds) + } - // Even with no matches, the plan structure should be the same - let plan_job = MergeInsertBuilder::try_new(ds.clone(), keys.clone()) - .unwrap() - .when_matched(WhenMatched::Delete) - .when_not_matched(WhenNotMatched::DoNothing) - .try_build() - .unwrap(); - let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( - [Ok(non_matching_batch.clone())], - schema.clone(), - ))); - let plan = plan_job.create_plan(plan_stream).await.unwrap(); - assert_plan_node_equals( - plan, - "DeleteOnlyMergeInsert: on=[key], when_matched=Delete, when_not_matched=DoNothing - ... - HashJoinExec: ...join_type=Inner... - ... - ... - StreamingTableExec: partition_sizes=1, projection=[key]", + /// Perform a partial-schema merge_insert (only id + value_a) targeting specific id ranges. + /// This causes touched fragments to drop from the index bitmap while btree data retains + /// stale entries. + async fn partial_merge_insert( + dataset: Arc, + id_range: std::ops::Range, + value_a_val: f64, + ) -> Arc { + let ids: Vec = id_range.map(|j| format!("id-{j:04}")).collect(); + let n = ids.len(); + let sub_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("value_a", DataType::Float64, false), + ])); + let batch = RecordBatch::try_new( + sub_schema.clone(), + vec![ + Arc::new(StringArray::from(ids)), + Arc::new(Float64Array::from(vec![value_a_val; n])), + ], ) - .await .unwrap(); - let job = MergeInsertBuilder::try_new(ds.clone(), keys) + let reader = Box::new(RecordBatchIterator::new([Ok(batch)], sub_schema)); + + let (ds, _) = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) .unwrap() - .when_matched(WhenMatched::Delete) + .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::DoNothing) .try_build() + .unwrap() + .execute_reader(reader) + .await .unwrap(); + ds + } - let new_reader = Box::new(RecordBatchIterator::new( - [Ok(non_matching_batch)], - schema.clone(), - )); - let new_stream = reader_to_stream(new_reader); + // Regression test: partial-schema merge_insert followed by another partial merge_insert + // on the same rows should not produce "Ambiguous merge inserts" errors. + // + // The bug: the first partial merge_insert drops the touched fragment from the index bitmap + // but leaves stale btree entries. The second merge_insert finds the same rows via both + // the stale btree lookup AND the unindexed fragment scan, causing duplicates. + #[tokio::test] + async fn test_partial_merge_insert_stale_index_ambiguous() { + let dataset = create_indexed_3frag_dataset().await; - let (merged_dataset, merge_stats) = job.execute(new_stream).await.unwrap(); + // Step 2: Partial merge_insert on fragment 1 rows -> fragment 1 drops from bitmap + let dataset = partial_merge_insert(dataset, 100..200, 999.0).await; - // Should have deleted 0 rows since no keys matched - assert_eq!(merge_stats.num_deleted_rows, 0); - assert_eq!(merge_stats.num_inserted_rows, 0); - assert_eq!(merge_stats.num_updated_rows, 0); + // Step 3: Another partial merge_insert on the same rows. + // This should succeed, not fail with "Ambiguous merge inserts". + let dataset = partial_merge_insert(dataset, 100..200, 888.0).await; - // Verify all original data remains unchanged - keys 1-6 should all still be present - let batches = merged_dataset + // Verify correctness: all 300 rows present, updated values correct + let batches = dataset .scan() .try_into_stream() .await @@ -9912,311 +13438,356 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .try_collect::>() .await .unwrap(); + let all_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new("value_a", DataType::Float64, false), + Field::new("value_b", DataType::Float64, false), + ])); + let combined = concat_batches(&all_schema, &batches).unwrap(); + assert_eq!(combined.num_rows(), 300); - let merged = concat_batches(&schema, &batches).unwrap(); - let mut remaining_keys: Vec = merged - .column(0) - .as_primitive::() - .values() - .to_vec(); - remaining_keys.sort(); - assert_eq!(remaining_keys, vec![1, 2, 3, 4, 5, 6]); + // Check the updated rows have value_a = 888.0 + let result = dataset + .scan() + .filter("id >= 'id-0100' AND id < 'id-0200'") + .unwrap() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let result = concat_batches(&all_schema, &result).unwrap(); + assert_eq!(result.num_rows(), 100); + let values = result + .column_by_name("value_a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..100 { + assert_eq!(values.value(i), 888.0, "row {i} should have value_a=888.0"); + } } - /// Test that MergeInsertPlanner::is_delete_only correctly identifies delete-only operations. - /// - /// Delete-only is true only when: - /// - when_matched = Delete - /// - insert_not_matched = false (WhenNotMatched::DoNothing) - /// - delete_not_matched_by_source = Keep - /// - /// This test iterates through all valid combinations of WhenMatched, WhenNotMatched, - /// and WhenNotMatchedBySource to verify the is_delete_only logic. + // Regression test for GitHub issue #6877. + // + // Two sequential full-schema merge_insert UpdateAll calls against the same + // target row, on a dataset with stable_row_ids enabled and a BTREE scalar + // index on the join column, used to fail on the second call with + // "Ambiguous merge inserts are prohibited" — even though each call's + // source had exactly one row per key. + // + // Mechanism: with stable row ids the BTREE stores stable_row_ids (not + // physical addresses). After the first merge_insert, A's stable_row_id is + // preserved but its physical home moves to an unindexed fragment. The + // BTREE-side TakeExec resolves the stable_row_id to A's new location and + // emits a row; the unindexed-fragments scan also covers the new fragment + // and emits the same logical row. Both surface the same `_rowid`, so the + // merge_insert source-dedup HashSet sees a duplicate and aborts. + // + // Fix: thread `restrict_to_fragments` into `do_create_deletion_mask_row_id` + // so the allow-list only contains stable_row_ids whose current physical + // home is inside the index's fragment_bitmap. #[tokio::test] - async fn test_is_delete_only() { - use itertools::iproduct; + async fn test_issue_6877_repeated_merge_insert_stable_row_ids() { + use arrow_array::Int32Array; - // All variants to test (excluding UpdateIf and DeleteIf because they require expressions) - let when_matched_variants = [ - WhenMatched::UpdateAll, - WhenMatched::DoNothing, - WhenMatched::Fail, - WhenMatched::Delete, - ]; - let when_not_matched_variants = [WhenNotMatched::InsertAll, WhenNotMatched::DoNothing]; - let when_not_matched_by_source_variants = - [WhenNotMatchedBySource::Keep, WhenNotMatchedBySource::Delete]; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("value", DataType::Int32, false), + ])); - let schema = create_test_schema(); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["A", "B", "C"])), + Arc::new(Int32Array::from(vec![1, 2, 3])), + ], + ) + .unwrap(); - for (idx, (when_matched, when_not_matched, when_not_matched_by_source)) in iproduct!( - when_matched_variants.iter().cloned(), - when_not_matched_variants.iter().cloned(), - when_not_matched_by_source_variants.iter().cloned() + let mut ds = Dataset::write( + Box::new(RecordBatchIterator::new([Ok(initial)], schema.clone())), + "memory://test_6877", + Some(WriteParams { + mode: WriteMode::Overwrite, + enable_stable_row_ids: true, + ..Default::default() + }), ) - .enumerate() - { - // Check if this is a valid (non-no-op) combination, since this would fail try_build() - let is_no_op = matches!(when_matched, WhenMatched::DoNothing | WhenMatched::Fail) - && matches!(when_not_matched, WhenNotMatched::DoNothing) - && matches!(when_not_matched_by_source, WhenNotMatchedBySource::Keep); - if is_no_op { - continue; - } + .await + .unwrap(); - let test_uri = format!("memory://test_is_delete_only_{}.lance", idx); - let ds = create_test_dataset(&test_uri, LanceFileVersion::V2_0, false).await; + ds.create_index( + &["id"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); - let new_batch = RecordBatch::try_new( + // First merge_insert: A 1 -> 11. + let update_a = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["A"])), + Arc::new(Int32Array::from(vec![11])), + ], + ) + .unwrap(); + let (ds, _) = MergeInsertBuilder::try_new(Arc::new(ds), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(update_a)], schema.clone(), - vec![ - Arc::new(UInt32Array::from(vec![4, 5, 6])), - Arc::new(UInt32Array::from(vec![2, 2, 2])), - Arc::new(StringArray::from(vec!["A", "B", "C"])), - ], - ) + ))) + .await .unwrap(); - let keys = vec!["key".to_string()]; - - let mut builder = MergeInsertBuilder::try_new(ds.clone(), keys).unwrap(); - builder - .when_matched(when_matched.clone()) - .when_not_matched(when_not_matched.clone()) - .when_not_matched_by_source(when_not_matched_by_source.clone()); - - let job = builder.try_build().unwrap(); - - let plan_stream = reader_to_stream(Box::new(RecordBatchIterator::new( - [Ok(new_batch)], + // Second merge_insert: A 11 -> 22. Used to fail before the fix. + let update_a_again = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(vec!["A"])), + Arc::new(Int32Array::from(vec![22])), + ], + ) + .unwrap(); + let (ds, _) = MergeInsertBuilder::try_new(ds, vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() + .unwrap() + .execute_reader(Box::new(RecordBatchIterator::new( + [Ok(update_a_again)], schema.clone(), - ))); - let plan = job.create_plan(plan_stream).await.unwrap(); - - let plan_str = datafusion::physical_plan::displayable(plan.as_ref()) - .indent(true) - .to_string(); - - let expected_delete_only = matches!(when_matched, WhenMatched::Delete) - && matches!(when_not_matched, WhenNotMatched::DoNothing) - && matches!(when_not_matched_by_source, WhenNotMatchedBySource::Keep); - - if expected_delete_only { - assert!( - plan_str.contains("DeleteOnlyMergeInsert"), - "Expected DeleteOnlyMergeInsert for ({:?}, {:?}, {:?}), but got:\n{}", - when_matched, - when_not_matched, - when_not_matched_by_source, - plan_str - ); - } else { - assert!( - plan_str.contains("MergeInsert:") - && !plan_str.contains("DeleteOnlyMergeInsert"), - "Expected MergeInsert (not DeleteOnlyMergeInsert) for ({:?}, {:?}, {:?}), but got:\n{}", - when_matched, - when_not_matched, - when_not_matched_by_source, - plan_str - ); - } - } + ))) + .await + .unwrap(); + + // Sanity check: A's value is now 22. + let batches = ds + .scan() + .filter("id = 'A'") + .unwrap() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let combined = concat_batches(&schema, &batches).unwrap(); + assert_eq!(combined.num_rows(), 1); + let values = combined + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(values.value(0), 22); } - /// Tests that apply_deletions correctly handles an error when applying the row deletions. + // Regression test: partial-schema merge_insert followed by update (deleting all rows + // in a fragment) followed by partial merge_insert should not produce + // "fragment id N does not exist" errors. + // + // The bug: stale btree entries reference the deleted fragment. The deletion mask doesn't + // block those addresses because the fragment isn't in the index bitmap. TakeExec tries + // to read from a non-existent fragment. #[tokio::test] - async fn test_apply_deletions_invalid_row_address() { - use super::exec::apply_deletions; - use roaring::RoaringTreemap; - - let test_uri = "memory://test_apply_deletions_error.lance"; + async fn test_partial_merge_insert_stale_index_fragment_not_exist() { + let dataset = create_indexed_3frag_dataset().await; - // Create a dataset with 2 fragments, each with 3 rows - let ds = create_test_dataset(test_uri, LanceFileVersion::V2_0, false).await; - let fragment_id = ds.get_fragments()[0].id() as u32; + // Step 2: Partial merge_insert on fragment 1 rows -> fragment 1 drops from bitmap + let dataset = partial_merge_insert(dataset, 100..200, 999.0).await; - // Create row addresses with invalid row offsets for this fragment - // Row address format: high 32 bits = fragment_id, low 32 bits = row_offset - // Each fragment has only 3 rows (offsets 0, 1, 2). - // - // The error in extend_deletions is triggered when deletion_vector.len() >= physical_rows - // AND at least one row ID is >= physical_rows. - // So we need to add enough deletions (at least 3) with some being invalid (>= 3). - let mut invalid_row_addrs = RoaringTreemap::new(); - let base = (fragment_id as u64) << 32; - // Add 4 deletions: rows 10, 11, 12, 13 (all invalid since only rows 0-2 exist) - for row_offset in 10..14u64 { - invalid_row_addrs.insert(base | row_offset); - } + // Step 3: Update all rows that were in fragment 1, causing fragment 1 to be + // fully deleted and replaced by a new fragment. + let update_result = crate::dataset::UpdateBuilder::new(Arc::new((*dataset).clone())) + .update_where("id >= 'id-0100' AND id < 'id-0200'") + .unwrap() + .set("category", "'B'") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap(); + let dataset = update_result.new_dataset; - let result = apply_deletions(&ds, &invalid_row_addrs).await; + // Step 4: Partial merge_insert on the same rows. + // This should succeed, not fail with "fragment does not exist". + let dataset = partial_merge_insert(dataset, 100..200, 888.0).await; - assert!(result.is_err(), "Expected error for invalid row addresses"); - let err = result.unwrap_err(); - assert!( - err.to_string() - .contains("Deletion vector includes rows that aren't in the fragment"), - "Expected 'rows that aren't in the fragment' error, got: {}", - err - ); + // Verify correctness + let batches = dataset + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let all_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new("value_a", DataType::Float64, false), + Field::new("value_b", DataType::Float64, false), + ])); + let combined = concat_batches(&all_schema, &batches).unwrap(); + assert_eq!(combined.num_rows(), 300); } - mod external_error { - use super::*; - use arrow_schema::{ArrowError, Field as ArrowField, Schema as ArrowSchema}; - use std::fmt; - - #[derive(Debug)] - struct MyTestError { - code: i32, - details: String, - } - - impl fmt::Display for MyTestError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "MyTestError({}): {}", self.code, self.details) - } - } - - impl std::error::Error for MyTestError {} + // Regression test: partial-schema merge_insert followed by update (deleting SOME rows + // in a fragment) followed by partial merge_insert should not produce + // "RecordBatch size mismatch" errors. + // + // The bug: stale btree entries reference deleted rows in a fragment that still exists. + // The deletion mask doesn't block those addresses (fragment not in bitmap). TakeExec + // reads the fragment but the rows have deletion markers, returning 0 rows where N + // were expected. + #[tokio::test] + async fn test_partial_merge_insert_stale_index_batch_size_mismatch() { + let dataset = create_indexed_3frag_dataset().await; - #[tokio::test] - async fn test_merge_insert_execute_reader_preserves_external_error() { - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("key", DataType::Int32, false), - ArrowField::new("value", DataType::Int32, false), - ])); + // Step 2: Partial merge_insert on fragment 1 rows -> fragment 1 drops from bitmap + let dataset = partial_merge_insert(dataset, 100..200, 999.0).await; - // Create initial dataset - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from(vec![1, 2, 3])), - Arc::new(Int32Array::from(vec![10, 20, 30])), - ], - ) + // Step 3: Update HALF of the rows that were in fragment 1. Fragment 1 survives + // but the updated rows are deleted from it (moved to a new fragment). + let update_result = crate::dataset::UpdateBuilder::new(Arc::new((*dataset).clone())) + .update_where("id >= 'id-0100' AND id < 'id-0150'") + .unwrap() + .set("category", "'B'") + .unwrap() + .build() + .unwrap() + .execute() + .await .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); - let dataset = Arc::new( - Dataset::write(reader, "memory://test_merge_external", None) - .await - .unwrap(), - ); - - // Try merge insert with failing source - let error_code = 789; - let iter = std::iter::once(Err(ArrowError::ExternalError(Box::new(MyTestError { - code: error_code, - details: "merge insert failure".to_string(), - })))); - let reader = RecordBatchIterator::new(iter, schema); + let dataset = update_result.new_dataset; - let result = MergeInsertBuilder::try_new(dataset, vec!["key".to_string()]) - .unwrap() - .try_build() - .unwrap() - .execute_reader(Box::new(reader) as Box) - .await; + // Step 4: Partial merge_insert targeting the rows that were updated (and thus + // deleted from fragment 1). Should succeed, not fail with batch size mismatch. + let dataset = partial_merge_insert(dataset, 100..150, 888.0).await; - match result { - Err(Error::External { source }) => { - let original = source.downcast_ref::().unwrap(); - assert_eq!(original.code, error_code); - } - Err(other) => panic!("Expected External, got: {:?}", other), - Ok(_) => panic!("Expected error"), - } - } + // Verify correctness + let batches = dataset + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let all_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("category", DataType::Utf8, false), + Field::new("value_a", DataType::Float64, false), + Field::new("value_b", DataType::Float64, false), + ])); + let combined = concat_batches(&all_schema, &batches).unwrap(); + assert_eq!(combined.num_rows(), 300); } - /// Creates a 3-fragment dataset (100 rows each) with columns (id: Utf8, category: Utf8, - /// value_a: Float64, value_b: Float64) and a BTree index on `id`. - /// - /// Fragment 0: id-0000..id-0099 - /// Fragment 1: id-0100..id-0199 - /// Fragment 2: id-0200..id-0299 - async fn create_indexed_3frag_dataset() -> Arc { + // Regression test: after a partial-schema merge_insert drops a fragment from the vector + // index bitmap, a vector search should not return duplicate rows. The stale vector index + // data still references the dropped fragment, and the scanner also flat-scans unindexed + // fragments, causing the same rows to appear from both paths. + #[tokio::test] + async fn test_partial_merge_insert_stale_vector_index_duplicates() { + let dim = 4i32; + let rows_per_frag = 10usize; + let num_frags = 3usize; + let total_rows = rows_per_frag * num_frags; + let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Utf8, false), Field::new("category", DataType::Utf8, false), - Field::new("value_a", DataType::Float64, false), - Field::new("value_b", DataType::Float64, false), + Field::new( + "vec", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim), + false, + ), ])); - let make_batch = |frag_idx: usize| { - let start = frag_idx * 100; - let ids: Vec = (start..start + 100).map(|j| format!("id-{j:04}")).collect(); - let categories: Vec<&str> = vec!["A"; 100]; - let value_a: Vec = (0..100) - .map(|i| i as f64 + frag_idx as f64 * 100.0) + let make_batch = |frag_idx: usize, offset: f32| { + let start = frag_idx * rows_per_frag; + let ids: Vec = (start..start + rows_per_frag) + .map(|j| format!("id-{j:04}")) .collect(); - let value_b: Vec = (0..100).map(|i| i as f64 * 0.1).collect(); + let cats: Vec<&str> = vec!["A"; rows_per_frag]; + let values: Vec = (0..rows_per_frag * dim as usize) + .map(|i| (start * dim as usize + i) as f32 + offset) + .collect(); + let vectors = + FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim).unwrap(); RecordBatch::try_new( schema.clone(), vec![ Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(categories)), - Arc::new(Float64Array::from(value_a)), - Arc::new(Float64Array::from(value_b)), + Arc::new(StringArray::from(cats)), + Arc::new(vectors), ], ) .unwrap() }; - // Write first fragment - let batch0 = make_batch(0); + // Write 3 fragments + let batch0 = make_batch(0, 0.0); let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); - let mut ds = Dataset::write(reader, "memory://indexed_3frag", None) + let mut ds = Dataset::write(reader, "memory://vector_stale_test", None) .await .unwrap(); - - // Append fragments 1 and 2 - for frag_idx in 1..3 { - let batch = make_batch(frag_idx); + for frag_idx in 1..num_frags { + let batch = make_batch(frag_idx, 0.0); let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); ds.append(reader, None).await.unwrap(); } - // Create BTree index on id - ds.create_index( - &["id"], - IndexType::BTree, - None, - &ScalarIndexParams::default(), - false, - ) - .await - .unwrap(); + // Create IVF_FLAT vector index on vec + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); + ds.create_index(&["vec"], IndexType::Vector, None, ¶ms, false) + .await + .unwrap(); - Arc::new(ds) - } + let ds = Arc::new(ds); - /// Perform a partial-schema merge_insert (only id + value_a) targeting specific id ranges. - /// This causes touched fragments to drop from the index bitmap while btree data retains - /// stale entries. - async fn partial_merge_insert( - dataset: Arc, - id_range: std::ops::Range, - value_a_val: f64, - ) -> Arc { - let ids: Vec = id_range.map(|j| format!("id-{j:04}")).collect(); - let n = ids.len(); + // Partial merge_insert with (id, vec) on fragment 1 rows - slightly different vectors. + // This drops fragment 1 from the vector index bitmap. + let frag1_start = rows_per_frag; + let ids: Vec = (frag1_start..frag1_start + rows_per_frag) + .map(|j| format!("id-{j:04}")) + .collect(); let sub_schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Utf8, false), - Field::new("value_a", DataType::Float64, false), + Field::new( + "vec", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim), + false, + ), ])); - let batch = RecordBatch::try_new( + let values: Vec = (0..rows_per_frag * dim as usize) + .map(|i| (frag1_start * dim as usize + i) as f32 + 0.5) + .collect(); + let vectors = + FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim).unwrap(); + let update_batch = RecordBatch::try_new( sub_schema.clone(), - vec![ - Arc::new(StringArray::from(ids)), - Arc::new(Float64Array::from(vec![value_a_val; n])), - ], + vec![Arc::new(StringArray::from(ids)), Arc::new(vectors)], ) .unwrap(); - let reader = Box::new(RecordBatchIterator::new([Ok(batch)], sub_schema)); - - let (ds, _) = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) + let reader = Box::new(RecordBatchIterator::new([Ok(update_batch)], sub_schema)); + let (ds, _) = MergeInsertBuilder::try_new(ds, vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::DoNothing) @@ -10225,1090 +13796,1382 @@ MergeInsert: on=[id], when_matched=DoNothing, when_not_matched=InsertAll, when_n .execute_reader(reader) .await .unwrap(); - ds - } - - // Regression test: partial-schema merge_insert followed by another partial merge_insert - // on the same rows should not produce "Ambiguous merge inserts" errors. - // - // The bug: the first partial merge_insert drops the touched fragment from the index bitmap - // but leaves stale btree entries. The second merge_insert finds the same rows via both - // the stale btree lookup AND the unindexed fragment scan, causing duplicates. - #[tokio::test] - async fn test_partial_merge_insert_stale_index_ambiguous() { - let dataset = create_indexed_3frag_dataset().await; - - // Step 2: Partial merge_insert on fragment 1 rows -> fragment 1 drops from bitmap - let dataset = partial_merge_insert(dataset, 100..200, 999.0).await; - // Step 3: Another partial merge_insert on the same rows. - // This should succeed, not fail with "Ambiguous merge inserts". - let dataset = partial_merge_insert(dataset, 100..200, 888.0).await; - - // Verify correctness: all 300 rows present, updated values correct - let batches = dataset + // KNN search with k = total_rows to retrieve all rows + let query: Float32Array = (0..dim) + .map(|i| (frag1_start * dim as usize + i as usize) as f32 + 0.5) + .collect(); + let results = ds .scan() - .try_into_stream() - .await + .nearest("vec", &query, total_rows) .unwrap() - .try_collect::>() + .try_into_batch() .await .unwrap(); - let all_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("category", DataType::Utf8, false), - Field::new("value_a", DataType::Float64, false), - Field::new("value_b", DataType::Float64, false), - ])); - let combined = concat_batches(&all_schema, &batches).unwrap(); - assert_eq!(combined.num_rows(), 300); - // Check the updated rows have value_a = 888.0 - let result = dataset - .scan() - .filter("id >= 'id-0100' AND id < 'id-0200'") - .unwrap() - .try_into_stream() - .await - .unwrap() - .try_collect::>() - .await - .unwrap(); - let result = concat_batches(&all_schema, &result).unwrap(); - assert_eq!(result.num_rows(), 100); - let values = result - .column_by_name("value_a") + // Check no duplicate ids + let ids = results + .column_by_name("id") .unwrap() .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - for i in 0..100 { - assert_eq!(values.value(i), 888.0, "row {i} should have value_a=888.0"); - } + let unique_ids: std::collections::HashSet<&str> = + (0..ids.len()).map(|i| ids.value(i)).collect(); + assert_eq!( + unique_ids.len(), + ids.len(), + "Found duplicate ids in KNN results: {} unique out of {} total", + unique_ids.len(), + ids.len() + ); } - // Regression test for GitHub issue #6877. - // - // Two sequential full-schema merge_insert UpdateAll calls against the same - // target row, on a dataset with stable_row_ids enabled and a BTREE scalar - // index on the join column, used to fail on the second call with - // "Ambiguous merge inserts are prohibited" — even though each call's - // source had exactly one row per key. - // - // Mechanism: with stable row ids the BTREE stores stable_row_ids (not - // physical addresses). After the first merge_insert, A's stable_row_id is - // preserved but its physical home moves to an unindexed fragment. The - // BTREE-side TakeExec resolves the stable_row_id to A's new location and - // emits a row; the unindexed-fragments scan also covers the new fragment - // and emits the same logical row. Both surface the same `_rowid`, so the - // merge_insert source-dedup HashSet sees a duplicate and aborts. - // - // Fix: thread `restrict_to_fragments` into `do_create_deletion_mask_row_id` - // so the allow-list only contains stable_row_ids whose current physical - // home is inside the index's fragment_bitmap. + // Regression test: after a partial-schema merge_insert drops a fragment from the FTS + // index bitmap, a full text search should not return duplicate rows. The stale inverted + // index data still references the dropped fragment, and the scanner also flat-scans + // unindexed fragments, causing the same rows to appear from both paths. #[tokio::test] - async fn test_issue_6877_repeated_merge_insert_stable_row_ids() { - use arrow_array::Int32Array; + async fn test_partial_merge_insert_stale_fts_index_duplicates() { + let rows_per_frag = 10usize; + let num_frags = 3usize; let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Utf8, false), - Field::new("value", DataType::Int32, false), + Field::new("category", DataType::Utf8, false), + Field::new("text", DataType::Utf8, false), ])); - let initial = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(vec!["A", "B", "C"])), - Arc::new(Int32Array::from(vec![1, 2, 3])), - ], - ) - .unwrap(); - - let mut ds = Dataset::write( - Box::new(RecordBatchIterator::new([Ok(initial)], schema.clone())), - "memory://test_6877", - Some(WriteParams { - mode: WriteMode::Overwrite, - enable_stable_row_ids: true, - ..Default::default() - }), - ) - .await - .unwrap(); + let make_batch = |frag_idx: usize| { + let start = frag_idx * rows_per_frag; + let ids: Vec = (start..start + rows_per_frag) + .map(|j| format!("id-{j:04}")) + .collect(); + let cats: Vec<&str> = vec!["A"; rows_per_frag]; + // Every row contains "common" so we can search for it and expect all rows + let texts: Vec = (start..start + rows_per_frag) + .map(|j| format!("common unique{j:04}")) + .collect(); + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(ids)), + Arc::new(StringArray::from(cats)), + Arc::new(StringArray::from(texts)), + ], + ) + .unwrap() + }; - ds.create_index( - &["id"], - IndexType::Scalar, - None, - &ScalarIndexParams::default(), - false, - ) - .await - .unwrap(); + // Write 3 fragments + let batch0 = make_batch(0); + let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); + let mut ds = Dataset::write(reader, "memory://fts_stale_test", None) + .await + .unwrap(); + for frag_idx in 1..num_frags { + let batch = make_batch(frag_idx); + let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); + ds.append(reader, None).await.unwrap(); + } - // First merge_insert: A 1 -> 11. - let update_a = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(vec!["A"])), - Arc::new(Int32Array::from(vec![11])), - ], - ) - .unwrap(); - let (ds, _) = MergeInsertBuilder::try_new(Arc::new(ds), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::DoNothing) - .try_build() - .unwrap() - .execute_reader(Box::new(RecordBatchIterator::new( - [Ok(update_a)], - schema.clone(), - ))) + // Create inverted index on text + let params = InvertedIndexParams::default(); + ds.create_index(&["text"], IndexType::Inverted, None, ¶ms, true) .await .unwrap(); - // Second merge_insert: A 11 -> 22. Used to fail before the fix. - let update_a_again = RecordBatch::try_new( - schema.clone(), + let ds = Arc::new(ds); + + // Partial merge_insert with (id, text) on fragment 1 rows. + // Text still contains "common" so FTS will find them via both paths. + // This drops fragment 1 from the inverted index bitmap. + let frag1_start = rows_per_frag; + let ids: Vec = (frag1_start..frag1_start + rows_per_frag) + .map(|j| format!("id-{j:04}")) + .collect(); + let texts: Vec = (frag1_start..frag1_start + rows_per_frag) + .map(|j| format!("common updated{j:04}")) + .collect(); + let sub_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("text", DataType::Utf8, false), + ])); + let update_batch = RecordBatch::try_new( + sub_schema.clone(), vec![ - Arc::new(StringArray::from(vec!["A"])), - Arc::new(Int32Array::from(vec![22])), + Arc::new(StringArray::from(ids)), + Arc::new(StringArray::from(texts)), ], ) .unwrap(); + let reader = Box::new(RecordBatchIterator::new([Ok(update_batch)], sub_schema)); let (ds, _) = MergeInsertBuilder::try_new(ds, vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) .when_not_matched(WhenNotMatched::DoNothing) .try_build() .unwrap() - .execute_reader(Box::new(RecordBatchIterator::new( - [Ok(update_a_again)], - schema.clone(), - ))) + .execute_reader(reader) .await .unwrap(); - // Sanity check: A's value is now 22. - let batches = ds + // FTS search for "common" — every row should match exactly once + let query = FullTextSearchQuery::new("common".to_string()); + let results = ds .scan() - .filter("id = 'A'") - .unwrap() - .try_into_stream() - .await + .full_text_search(query) .unwrap() - .try_collect::>() + .try_into_batch() .await .unwrap(); - let combined = concat_batches(&schema, &batches).unwrap(); - assert_eq!(combined.num_rows(), 1); - let values = combined - .column_by_name("value") + + // Check no duplicate ids + let ids = results + .column_by_name("id") .unwrap() .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - assert_eq!(values.value(0), 22); + let unique_ids: std::collections::HashSet<&str> = + (0..ids.len()).map(|i| ids.value(i)).collect(); + assert_eq!( + unique_ids.len(), + ids.len(), + "Found duplicate ids in FTS results: {} unique out of {} total", + unique_ids.len(), + ids.len() + ); + // Also verify we got all rows + assert_eq!( + unique_ids.len(), + rows_per_frag * num_frags, + "Expected {} rows but got {}", + rows_per_frag * num_frags, + unique_ids.len() + ); } - // Regression test: partial-schema merge_insert followed by update (deleting all rows - // in a fragment) followed by partial merge_insert should not produce - // "fragment id N does not exist" errors. + // Companion regression test for issue #6877 on the FTS path. // - // The bug: stale btree entries reference the deleted fragment. The deletion mask doesn't - // block those addresses because the fragment isn't in the index bitmap. TakeExec tries - // to read from a non-existent fragment. + // The FTS prefilter shares `do_create_deletion_mask_row_id` with the + // scalar-index path, so the same stable-row-id bypass that produced + // duplicate rows in merge_insert can produce duplicate hits in FTS search + // after a merge_insert moves rows to unindexed fragments. This test pins + // the contract for the FTS consumer. #[tokio::test] - async fn test_partial_merge_insert_stale_index_fragment_not_exist() { - let dataset = create_indexed_3frag_dataset().await; + async fn test_issue_6877_fts_no_duplicates_stable_row_ids() { + let rows_per_frag = 10usize; + let num_frags = 3usize; - // Step 2: Partial merge_insert on fragment 1 rows -> fragment 1 drops from bitmap - let dataset = partial_merge_insert(dataset, 100..200, 999.0).await; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Utf8, false), + Field::new("text", DataType::Utf8, false), + ])); - // Step 3: Update all rows that were in fragment 1, causing fragment 1 to be - // fully deleted and replaced by a new fragment. - let update_result = crate::dataset::UpdateBuilder::new(Arc::new((*dataset).clone())) - .update_where("id >= 'id-0100' AND id < 'id-0200'") - .unwrap() - .set("category", "'B'") - .unwrap() - .build() + let make_batch = |frag_idx: usize| { + let start = frag_idx * rows_per_frag; + let ids: Vec = (start..start + rows_per_frag) + .map(|j| format!("id-{j:04}")) + .collect(); + let texts: Vec = (start..start + rows_per_frag) + .map(|j| format!("common unique{j:04}")) + .collect(); + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(ids)), + Arc::new(StringArray::from(texts)), + ], + ) .unwrap() - .execute() - .await - .unwrap(); - let dataset = update_result.new_dataset; + }; - // Step 4: Partial merge_insert on the same rows. - // This should succeed, not fail with "fragment does not exist". - let dataset = partial_merge_insert(dataset, 100..200, 888.0).await; + let batch0 = make_batch(0); + let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); + let mut ds = Dataset::write( + reader, + "memory://fts_stable_row_id_test", + Some(WriteParams { + mode: WriteMode::Overwrite, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + for frag_idx in 1..num_frags { + let batch = make_batch(frag_idx); + let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); + ds.append(reader, None).await.unwrap(); + } - // Verify correctness - let batches = dataset - .scan() - .try_into_stream() - .await - .unwrap() - .try_collect::>() + let params = InvertedIndexParams::default(); + ds.create_index(&["text"], IndexType::Inverted, None, ¶ms, true) .await .unwrap(); - let all_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("category", DataType::Utf8, false), - Field::new("value_a", DataType::Float64, false), - Field::new("value_b", DataType::Float64, false), - ])); - let combined = concat_batches(&all_schema, &batches).unwrap(); - assert_eq!(combined.num_rows(), 300); - } - - // Regression test: partial-schema merge_insert followed by update (deleting SOME rows - // in a fragment) followed by partial merge_insert should not produce - // "RecordBatch size mismatch" errors. - // - // The bug: stale btree entries reference deleted rows in a fragment that still exists. - // The deletion mask doesn't block those addresses (fragment not in bitmap). TakeExec - // reads the fragment but the rows have deletion markers, returning 0 rows where N - // were expected. - #[tokio::test] - async fn test_partial_merge_insert_stale_index_batch_size_mismatch() { - let dataset = create_indexed_3frag_dataset().await; - - // Step 2: Partial merge_insert on fragment 1 rows -> fragment 1 drops from bitmap - let dataset = partial_merge_insert(dataset, 100..200, 999.0).await; - // Step 3: Update HALF of the rows that were in fragment 1. Fragment 1 survives - // but the updated rows are deleted from it (moved to a new fragment). - let update_result = crate::dataset::UpdateBuilder::new(Arc::new((*dataset).clone())) - .update_where("id >= 'id-0100' AND id < 'id-0150'") - .unwrap() - .set("category", "'B'") + // Full-schema merge_insert rewriting fragment 1's rows. After this, + // the original locations are tombstoned and the new locations live in + // a new (unindexed) fragment; the stable_row_ids are preserved. + let frag1_start = rows_per_frag; + let ids: Vec = (frag1_start..frag1_start + rows_per_frag) + .map(|j| format!("id-{j:04}")) + .collect(); + let texts: Vec = (frag1_start..frag1_start + rows_per_frag) + .map(|j| format!("common updated{j:04}")) + .collect(); + let update_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(StringArray::from(ids)), + Arc::new(StringArray::from(texts)), + ], + ) + .unwrap(); + let reader = Box::new(RecordBatchIterator::new([Ok(update_batch)], schema.clone())); + let (ds, _) = MergeInsertBuilder::try_new(Arc::new(ds), vec!["id".to_string()]) .unwrap() - .build() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() .unwrap() - .execute() + .execute_reader(reader) .await .unwrap(); - let dataset = update_result.new_dataset; - - // Step 4: Partial merge_insert targeting the rows that were updated (and thus - // deleted from fragment 1). Should succeed, not fail with batch size mismatch. - let dataset = partial_merge_insert(dataset, 100..150, 888.0).await; - // Verify correctness - let batches = dataset + // FTS search for "common" — every row should match exactly once. + let query = FullTextSearchQuery::new("common".to_string()); + let results = ds .scan() - .try_into_stream() - .await + .full_text_search(query) .unwrap() - .try_collect::>() + .try_into_batch() .await .unwrap(); - let all_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("category", DataType::Utf8, false), - Field::new("value_a", DataType::Float64, false), - Field::new("value_b", DataType::Float64, false), - ])); - let combined = concat_batches(&all_schema, &batches).unwrap(); - assert_eq!(combined.num_rows(), 300); + + let ids = results + .column_by_name("id") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let unique_ids: std::collections::HashSet<&str> = + (0..ids.len()).map(|i| ids.value(i)).collect(); + assert_eq!( + unique_ids.len(), + ids.len(), + "Found duplicate ids in FTS results: {} unique out of {} total", + unique_ids.len(), + ids.len() + ); + assert_eq!( + unique_ids.len(), + rows_per_frag * num_frags, + "Expected {} rows but got {}", + rows_per_frag * num_frags, + unique_ids.len() + ); } - // Regression test: after a partial-schema merge_insert drops a fragment from the vector - // index bitmap, a vector search should not return duplicate rows. The stale vector index - // data still references the dropped fragment, and the scanner also flat-scans unindexed - // fragments, causing the same rows to appear from both paths. - #[tokio::test] - async fn test_partial_merge_insert_stale_vector_index_duplicates() { - let dim = 4i32; - let rows_per_frag = 10usize; - let num_frags = 3usize; - let total_rows = rows_per_frag * num_frags; + // Regression test: after a partial-schema merge_insert invalidates a fragment, + // compaction should succeed and subsequent searches should return correct results. + // + // The compaction planner separates indexed and unindexed fragments into different + // groups. After invalidating the middle fragment, the indexed fragments on either + // side form separate compactable groups. After compaction the old invalidated + // fragment ID may remain in invalidated_fragment_bitmap but this is harmless + // because the fragment no longer exists and no index results reference it. + #[tokio::test] + async fn test_compaction_after_invalidated_fragment() { + use crate::dataset::optimize::{CompactionOptions, compact_files}; + // Use 5 small fragments so that after invalidating the middle one (fragment 2), + // the planner has enough neighbors to form compactable groups on each side: + // {0,1} (indexed) and {3,4} (indexed), with {2} (unindexed) separate. + let rows_per_frag = 20; + let num_frags = 5; + let total_rows = rows_per_frag * num_frags; let schema = Arc::new(Schema::new(vec![ Field::new("id", DataType::Utf8, false), Field::new("category", DataType::Utf8, false), - Field::new( - "vec", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim), - false, - ), + Field::new("value_a", DataType::Float64, false), + Field::new("value_b", DataType::Float64, false), ])); - let make_batch = |frag_idx: usize, offset: f32| { + let make_batch = |frag_idx: usize| { let start = frag_idx * rows_per_frag; let ids: Vec = (start..start + rows_per_frag) .map(|j| format!("id-{j:04}")) .collect(); - let cats: Vec<&str> = vec!["A"; rows_per_frag]; - let values: Vec = (0..rows_per_frag * dim as usize) - .map(|i| (start * dim as usize + i) as f32 + offset) - .collect(); - let vectors = - FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim).unwrap(); RecordBatch::try_new( schema.clone(), vec![ Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(cats)), - Arc::new(vectors), + Arc::new(StringArray::from(vec!["A"; rows_per_frag])), + Arc::new(Float64Array::from( + (0..rows_per_frag).map(|i| i as f64).collect::>(), + )), + Arc::new(Float64Array::from( + (0..rows_per_frag) + .map(|i| i as f64 * 0.1) + .collect::>(), + )), ], ) .unwrap() }; - // Write 3 fragments - let batch0 = make_batch(0, 0.0); + let batch0 = make_batch(0); let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); - let mut ds = Dataset::write(reader, "memory://vector_stale_test", None) + let mut ds = Dataset::write(reader, "memory://compaction_test", None) .await .unwrap(); for frag_idx in 1..num_frags { - let batch = make_batch(frag_idx, 0.0); + let batch = make_batch(frag_idx); let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); ds.append(reader, None).await.unwrap(); } + ds.create_index( + &["id"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); - // Create IVF_FLAT vector index on vec - let params = VectorIndexParams::ivf_flat(1, MetricType::L2); - ds.create_index(&["vec"], IndexType::Vector, None, ¶ms, false) + let ds = Arc::new(ds); + + // Invalidate fragment 2 (the middle one) + let frag2_start = 2 * rows_per_frag; + let ds = partial_merge_insert(ds, frag2_start..frag2_start + rows_per_frag, 999.0).await; + + // Verify pre-compaction state + let indices = ds.load_indices().await.unwrap(); + let idx = indices.iter().find(|i| i.name == "id_idx").unwrap(); + assert!(!idx.fragment_bitmap.as_ref().unwrap().contains(2)); + + // Run compaction with a target that forces merging of the small fragments. + let mut ds = (*ds).clone(); + let opts = CompactionOptions { + target_rows_per_fragment: total_rows, + ..Default::default() + }; + compact_files(&mut ds, opts, None).await.unwrap(); + + // The indexed fragments (0,1 and 3,4) should be compacted. + // Fragment 2 (unindexed) may or may not be compacted on its own. + // Either way, the old fragment IDs in the bitmap should be replaced. + let indices = ds.load_indices().await.unwrap(); + let idx = indices.iter().find(|i| i.name == "id_idx").unwrap(); + let bitmap = idx.fragment_bitmap.as_ref().unwrap(); + for &old_id in &[0u32, 1, 3, 4] { + assert!( + !bitmap.contains(old_id), + "Old indexed fragment {} should not be in bitmap after compaction", + old_id + ); + } + assert!( + !bitmap.is_empty(), + "Bitmap should have new compacted fragments" + ); + + // The invalidated bitmap may still reference old fragment 2. + // This is harmless — fragment 2 no longer exists (or was compacted into + // a new fragment), so blocking it is a no-op. + + // Verify search works correctly despite stale invalidated entries. + let ds = Arc::new(ds); + let ds = partial_merge_insert(ds, frag2_start..frag2_start + rows_per_frag, 888.0).await; + + // All rows present + let batches = ds + .scan() + .try_into_stream() + .await + .unwrap() + .try_collect::>() .await .unwrap(); + let combined = concat_batches(&schema, &batches).unwrap(); + assert_eq!(combined.num_rows(), total_rows); - let ds = Arc::new(ds); + // Updated rows have correct value + let result = ds + .scan() + .filter(&format!( + "id >= 'id-{:04}' AND id < 'id-{:04}'", + frag2_start, + frag2_start + rows_per_frag + )) + .unwrap() + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let result = concat_batches(&schema, &result).unwrap(); + assert_eq!(result.num_rows(), rows_per_frag); + let values = result + .column_by_name("value_a") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..rows_per_frag { + assert_eq!(values.value(i), 888.0, "row {i} should have value_a=888.0"); + } + } - // Partial merge_insert with (id, vec) on fragment 1 rows - slightly different vectors. - // This drops fragment 1 from the vector index bitmap. - let frag1_start = rows_per_frag; - let ids: Vec = (frag1_start..frag1_start + rows_per_frag) - .map(|j| format!("id-{j:04}")) - .collect(); - let sub_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new( - "vec", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), dim), - false, - ), + fn count_data_files(base_dir: &str) -> usize { + let data_dir = std::path::Path::new(base_dir).join("data"); + if !data_dir.exists() { + return 0; + } + std::fs::read_dir(data_dir) + .unwrap() + .filter(|e| e.as_ref().unwrap().path().is_file()) + .count() + } + + /// Site 3 in PR #6320: when `MergeInsertJob::apply_deletions` fails after + /// the new fragments have been written, the new data files must be cleaned up. + #[tokio::test] + async fn test_merge_insert_cleans_up_data_on_apply_deletions_failure() { + use crate::utils::test::FailingProxyStore; + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, false), ])); - let values: Vec = (0..rows_per_frag * dim as usize) - .map(|i| (frag1_start * dim as usize + i) as f32 + 0.5) - .collect(); - let vectors = - FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim).unwrap(); - let update_batch = RecordBatch::try_new( - sub_schema.clone(), - vec![Arc::new(StringArray::from(ids)), Arc::new(vectors)], + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(0..30)), + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + "foo", 30, + ))), + ], ) .unwrap(); - let reader = Box::new(RecordBatchIterator::new([Ok(update_batch)], sub_schema)); - let (ds, _) = MergeInsertBuilder::try_new(ds, vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::DoNothing) - .try_build() - .unwrap() - .execute_reader(reader) + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + // Prefix `/` so Windows drive letters (e.g. `C:`) don't get parsed as + // the URL authority. + let path_prefix = if test_uri.starts_with('/') { "" } else { "/" }; + let routed_uri = format!("file-object-store://{path_prefix}{test_uri}"); + + let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); + let mut dataset = Dataset::write( + batches, + &routed_uri, + Some(WriteParams { + max_rows_per_file: 10, + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + + // Create a scalar index on the join key. This forces the merge insert + // to take the slow (non-fast) path, which is the path that has the + // post-write cleanup we want to exercise. + dataset + .create_index( + &["id"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) .await .unwrap(); - // KNN search with k = total_rows to retrieve all rows - let query: Float32Array = (0..dim) - .map(|i| (frag1_start * dim as usize + i as usize) as f32 + 0.5) - .collect(); - let results = ds - .scan() - .nearest("vec", &query, total_rows) - .unwrap() - .try_into_batch() + let baseline_files = count_data_files(test_uri); + assert!(baseline_files > 0); + + let failing = Arc::new(FailingProxyStore::new()); + failing.fail_when("put", "_deletions", "injected deletions failure"); + failing.fail_when("put_multipart", "_deletions", "injected deletions failure"); + + let dataset = DatasetBuilder::from_uri(&routed_uri) + .with_read_params(ReadParams { + store_options: Some(ObjectStoreParams { + object_store_wrapper: Some(failing.clone()), + ..Default::default() + }), + ..Default::default() + }) + .load() .await .unwrap(); - // Check no duplicate ids - let ids = results - .column_by_name("id") + // Update existing keys (5..15 already exist) to force the apply_deletions path. + let new_data = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from_iter_values(5..15)), + Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + "bar", 10, + ))), + ], + ) + .unwrap(); + let new_reader = Box::new(RecordBatchIterator::new([Ok(new_data)], schema.clone())); + + let job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) .unwrap() - .as_any() - .downcast_ref::() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .try_build() .unwrap(); - let unique_ids: std::collections::HashSet<&str> = - (0..ids.len()).map(|i| ids.value(i)).collect(); + + let result = job.execute_reader(new_reader).await; + assert!( + result.is_err(), + "Merge insert should fail when deletion-file write fails" + ); + assert_eq!( - unique_ids.len(), - ids.len(), - "Found duplicate ids in KNN results: {} unique out of {} total", - unique_ids.len(), - ids.len() + count_data_files(test_uri), + baseline_files, + "Newly written merge-insert data files should be cleaned up on apply_deletions failure" ); } - // Regression test: after a partial-schema merge_insert drops a fragment from the FTS - // index bitmap, a full text search should not return duplicate rows. The stale inverted - // index data still references the dropped fragment, and the scanner also flat-scans - // unindexed fragments, causing the same rows to appear from both paths. #[tokio::test] - async fn test_partial_merge_insert_stale_fts_index_duplicates() { - let rows_per_frag = 10usize; - let num_frags = 3usize; + async fn test_merge_insert_full_fragment_rewrite_with_json_columns() { + // This test verifies the "all rows updated" fast path in handle_fragment + // correctly converts Arrow JSON (Utf8) to Lance JSON (LargeBinary/JSONB) + // before writing. Without conversion, the file would have Utf8 data (i32 + // offsets) but schema says LargeBinary (i64 offsets), causing decoder panic + // on subsequent reads. + // + // To trigger the fast path we need: + // 1. Subschema update (not all columns) → forces v1 update_fragments path + // 2. ALL rows in a fragment updated → triggers the fast path + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::{ARROW_JSON_EXT_NAME, is_arrow_json_field}; + let test_dir = TempStrDir::default(); + let mut json_metadata = HashMap::new(); + json_metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("category", DataType::Utf8, false), - Field::new("text", DataType::Utf8, false), + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + Field::new("score", DataType::Int64, true), + Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()), ])); - - let make_batch = |frag_idx: usize| { - let start = frag_idx * rows_per_frag; - let ids: Vec = (start..start + rows_per_frag) - .map(|j| format!("id-{j:04}")) - .collect(); - let cats: Vec<&str> = vec!["A"; rows_per_frag]; - // Every row contains "common" so we can search for it and expect all rows - let texts: Vec = (start..start + rows_per_frag) - .map(|j| format!("common unique{j:04}")) - .collect(); - RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(cats)), - Arc::new(StringArray::from(texts)), - ], - ) - .unwrap() + // Small fragment so ALL rows will be updated + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + Arc::new(Int64Array::from(vec![10, 20, 30])), + Arc::new(StringArray::from(vec![ + r#"{"x":1}"#, + r#"{"x":2}"#, + r#"{"x":3}"#, + ])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let write_params = WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() }; + let dataset = Arc::new( + Dataset::write(reader, test_dir.as_ref(), Some(write_params)) + .await + .unwrap(), + ); + assert_eq!(dataset.get_fragments().len(), 1); - // Write 3 fragments - let batch0 = make_batch(0); - let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); - let mut ds = Dataset::write(reader, "memory://fts_stale_test", None) - .await - .unwrap(); - for frag_idx in 1..num_frags { - let batch = make_batch(frag_idx); - let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); - ds.append(reader, None).await.unwrap(); - } - - // Create inverted index on text - let params = InvertedIndexParams::default(); - ds.create_index(&["text"], IndexType::Inverted, None, ¶ms, true) - .await - .unwrap(); - - let ds = Arc::new(ds); - - // Partial merge_insert with (id, text) on fragment 1 rows. - // Text still contains "common" so FTS will find them via both paths. - // This drops fragment 1 from the inverted index bitmap. - let frag1_start = rows_per_frag; - let ids: Vec = (frag1_start..frag1_start + rows_per_frag) - .map(|j| format!("id-{j:04}")) - .collect(); - let texts: Vec = (frag1_start..frag1_start + rows_per_frag) - .map(|j| format!("common updated{j:04}")) - .collect(); - let sub_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("text", DataType::Utf8, false), + // Subschema update: only provide [id, meta] (missing "name" and "score") + // This forces the v1 path (update_fragments) instead of v2 (create_plan). + // Update ALL rows → triggers the "all rows updated" fast path. + let update_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata), ])); let update_batch = RecordBatch::try_new( - sub_schema.clone(), + update_schema.clone(), vec![ - Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(texts)), + Arc::new(Int64Array::from(vec![1, 2, 3])), // all rows + Arc::new(StringArray::from(vec![ + r#"{"updated":true,"id":1}"#, + r#"{"updated":true,"id":2}"#, + r#"{"updated":true,"id":3}"#, + ])), ], ) .unwrap(); - let reader = Box::new(RecordBatchIterator::new([Ok(update_batch)], sub_schema)); - let (ds, _) = MergeInsertBuilder::try_new(ds, vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::DoNothing) - .try_build() - .unwrap() - .execute_reader(reader) - .await - .unwrap(); + let update_reader: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + let stream = reader_to_stream(update_reader); - // FTS search for "common" — every row should match exactly once - let query = FullTextSearchQuery::new("common".to_string()); - let results = ds + // Execute merge_insert with subschema + let mut builder = + MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap(); + builder.when_matched(WhenMatched::UpdateAll); + builder.when_not_matched(WhenNotMatched::DoNothing); + let job = builder.try_build().unwrap(); + let (updated_dataset, stats) = job.execute(stream).await.unwrap(); + + assert_eq!(stats.num_updated_rows, 3); + + // Critical: read the data back. Without the fix, this would PANIC with: + // "the offset of the new Buffer cannot exceed the existing Length: + // slice offset=0 Length=N selfLen=N/2" + let batches = updated_dataset .scan() - .full_text_search(query) + .try_into_stream() + .await .unwrap() - .try_into_batch() + .try_collect::>() .await .unwrap(); + let result = concat_batches(&batches[0].schema(), &batches).unwrap(); + assert_eq!(result.num_rows(), 3); - // Check no duplicate ids - let ids = results - .column_by_name("id") + // Verify JSON column is in Arrow JSON format (Utf8) on read + let result_schema = result.schema(); + let meta_field = result_schema.field_with_name("meta").unwrap(); + assert!( + is_arrow_json_field(meta_field), + "Expected Arrow JSON (Utf8 + arrow.json), got {:?}", + meta_field + ); + + // Verify data correctness + let metas = result + .column_by_name("meta") .unwrap() .as_any() .downcast_ref::() + .expect("meta should be StringArray after read conversion"); + for i in 0..3 { + let val = metas.value(i); + assert!( + val.contains("updated"), + "row {} should have updated meta, got: {}", + i, + val + ); + } + + // Verify non-updated columns are preserved + let scores = result + .column_by_name("score") + .unwrap() + .as_any() + .downcast_ref::() .unwrap(); - let unique_ids: std::collections::HashSet<&str> = - (0..ids.len()).map(|i| ids.value(i)).collect(); - assert_eq!( - unique_ids.len(), - ids.len(), - "Found duplicate ids in FTS results: {} unique out of {} total", - unique_ids.len(), - ids.len() - ); - // Also verify we got all rows - assert_eq!( - unique_ids.len(), - rows_per_frag * num_frags, - "Expected {} rows but got {}", - rows_per_frag * num_frags, - unique_ids.len() + assert_eq!(scores.values(), &[10, 20, 30]); + + // Also verify via take (exercises the take read conversion path) + let take_result = updated_dataset + .take(&[0, 1, 2], updated_dataset.schema().clone()) + .await + .unwrap(); + let take_schema = take_result.schema(); + let take_meta_field = take_schema.field_with_name("meta").unwrap(); + assert!( + is_arrow_json_field(take_meta_field), + "take() should return Arrow JSON, got {:?}", + take_meta_field ); } - // Companion regression test for issue #6877 on the FTS path. - // - // The FTS prefilter shares `do_create_deletion_mask_row_id` with the - // scalar-index path, so the same stable-row-id bypass that produced - // duplicate rows in merge_insert can produce duplicate hits in FTS search - // after a merge_insert moves rows to unindexed fragments. This test pins - // the contract for the FTS consumer. #[tokio::test] - async fn test_issue_6877_fts_no_duplicates_stable_row_ids() { - let rows_per_frag = 10usize; - let num_frags = 3usize; + async fn test_merge_insert_subschema_with_json_columns() { + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::ARROW_JSON_EXT_NAME; + // Create a dataset with an Arrow JSON extension column + let test_dir = TempStrDir::default(); + let mut json_metadata = HashMap::new(); + json_metadata.insert( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + ); let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("text", DataType::Utf8, false), + Field::new("id", DataType::Int64, false), + Field::new("name", DataType::Utf8, true), + Field::new("score", DataType::Int64, true), + Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()), ])); - - let make_batch = |frag_idx: usize| { - let start = frag_idx * rows_per_frag; - let ids: Vec = (start..start + rows_per_frag) - .map(|j| format!("id-{j:04}")) - .collect(); - let texts: Vec = (start..start + rows_per_frag) - .map(|j| format!("common unique{j:04}")) - .collect(); - RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(texts)), - ], - ) - .unwrap() - }; - - let batch0 = make_batch(0); - let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); - let mut ds = Dataset::write( - reader, - "memory://fts_stable_row_id_test", - Some(WriteParams { - mode: WriteMode::Overwrite, - enable_stable_row_ids: true, - ..Default::default() - }), + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![1, 2, 3, 4, 5])), + Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"])), + Arc::new(Int64Array::from(vec![10, 20, 30, 40, 50])), + Arc::new(StringArray::from(vec![ + r#"{"x":1}"#, + r#"{"x":2}"#, + r#"{"x":3}"#, + r#"{"x":4}"#, + r#"{"x":5}"#, + ])), + ], ) - .await .unwrap(); - for frag_idx in 1..num_frags { - let batch = make_batch(frag_idx); - let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); - ds.append(reader, None).await.unwrap(); - } - - let params = InvertedIndexParams::default(); - ds.create_index(&["text"], IndexType::Inverted, None, ¶ms, true) - .await - .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Arc::new( + Dataset::write(reader, test_dir.as_ref(), None) + .await + .unwrap(), + ); - // Full-schema merge_insert rewriting fragment 1's rows. After this, - // the original locations are tombstoned and the new locations live in - // a new (unindexed) fragment; the stable_row_ids are preserved. - let frag1_start = rows_per_frag; - let ids: Vec = (frag1_start..frag1_start + rows_per_frag) - .map(|j| format!("id-{j:04}")) - .collect(); - let texts: Vec = (frag1_start..frag1_start + rows_per_frag) - .map(|j| format!("common updated{j:04}")) - .collect(); + // Perform a subschema merge_insert: only update "meta" column (JSON type) + // This exercises the update_fragments path with interleave_batches + let update_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata), + ])); let update_batch = RecordBatch::try_new( - schema.clone(), + update_schema.clone(), vec![ - Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(texts)), + Arc::new(Int64Array::from(vec![2, 4])), + Arc::new(StringArray::from(vec![ + r#"{"updated":true,"id":2}"#, + r#"{"updated":true,"id":4}"#, + ])), ], ) .unwrap(); - let reader = Box::new(RecordBatchIterator::new([Ok(update_batch)], schema.clone())); - let (ds, _) = MergeInsertBuilder::try_new(Arc::new(ds), vec!["id".to_string()]) - .unwrap() - .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::DoNothing) - .try_build() - .unwrap() - .execute_reader(reader) - .await - .unwrap(); + let update_reader: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + let stream = reader_to_stream(update_reader); - // FTS search for "common" — every row should match exactly once. - let query = FullTextSearchQuery::new("common".to_string()); - let results = ds + // Execute merge_insert with subschema (only id + meta columns) + let mut builder = + MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap(); + builder.when_matched(WhenMatched::UpdateAll); + builder.when_not_matched(WhenNotMatched::DoNothing); + let job = builder.try_build().unwrap(); + let (updated_dataset, stats) = job.execute(stream).await.unwrap(); + + // Verify: the merge should not fail with type mismatch + assert_eq!(stats.num_updated_rows, 2); + + // Read back and verify the JSON column was updated correctly + let batches = updated_dataset .scan() - .full_text_search(query) + .try_into_stream() + .await .unwrap() - .try_into_batch() + .try_collect::>() .await .unwrap(); + let result = concat_batches(&batches[0].schema(), &batches).unwrap(); + assert_eq!(result.num_rows(), 5); - let ids = results + // Verify the "score" column (not in update) is preserved, and "meta" updated + let ids = result .column_by_name("id") .unwrap() .as_any() - .downcast_ref::() + .downcast_ref::() .unwrap(); - let unique_ids: std::collections::HashSet<&str> = - (0..ids.len()).map(|i| ids.value(i)).collect(); - assert_eq!( - unique_ids.len(), - ids.len(), - "Found duplicate ids in FTS results: {} unique out of {} total", - unique_ids.len(), - ids.len() - ); - assert_eq!( - unique_ids.len(), - rows_per_frag * num_frags, - "Expected {} rows but got {}", - rows_per_frag * num_frags, - unique_ids.len() - ); - } - - // Regression test: after a partial-schema merge_insert invalidates a fragment, - // compaction should succeed and subsequent searches should return correct results. - // - // The compaction planner separates indexed and unindexed fragments into different - // groups. After invalidating the middle fragment, the indexed fragments on either - // side form separate compactable groups. After compaction the old invalidated - // fragment ID may remain in invalidated_fragment_bitmap but this is harmless - // because the fragment no longer exists and no index results reference it. - #[tokio::test] - async fn test_compaction_after_invalidated_fragment() { - use crate::dataset::optimize::{CompactionOptions, compact_files}; - - // Use 5 small fragments so that after invalidating the middle one (fragment 2), - // the planner has enough neighbors to form compactable groups on each side: - // {0,1} (indexed) and {3,4} (indexed), with {2} (unindexed) separate. - let rows_per_frag = 20; - let num_frags = 5; - let total_rows = rows_per_frag * num_frags; - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Utf8, false), - Field::new("category", DataType::Utf8, false), - Field::new("value_a", DataType::Float64, false), - Field::new("value_b", DataType::Float64, false), - ])); - - let make_batch = |frag_idx: usize| { - let start = frag_idx * rows_per_frag; - let ids: Vec = (start..start + rows_per_frag) - .map(|j| format!("id-{j:04}")) - .collect(); - RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(StringArray::from(ids)), - Arc::new(StringArray::from(vec!["A"; rows_per_frag])), - Arc::new(Float64Array::from( - (0..rows_per_frag).map(|i| i as f64).collect::>(), - )), - Arc::new(Float64Array::from( - (0..rows_per_frag) - .map(|i| i as f64 * 0.1) - .collect::>(), - )), - ], - ) + let scores = result + .column_by_name("score") .unwrap() - }; - - let batch0 = make_batch(0); - let reader = Box::new(RecordBatchIterator::new([Ok(batch0)], schema.clone())); - let mut ds = Dataset::write(reader, "memory://compaction_test", None) - .await + .as_any() + .downcast_ref::() .unwrap(); - for frag_idx in 1..num_frags { - let batch = make_batch(frag_idx); - let reader = Box::new(RecordBatchIterator::new([Ok(batch)], schema.clone())); - ds.append(reader, None).await.unwrap(); + let metas = result + .column_by_name("meta") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..5 { + let id = ids.value(i); + let score = scores.value(i); + let meta = metas.value(i); + // score = id * 10, regardless of row order + assert_eq!(score, id * 10, "id={} score mismatch", id); + if id == 2 || id == 4 { + assert!( + meta.contains("updated"), + "id={} should have updated meta, got: {}", + id, + meta + ); + } else { + assert!( + meta.contains("\"x\""), + "id={} should have original meta, got: {}", + id, + meta + ); + } } - ds.create_index( - &["id"], - IndexType::BTree, - None, - &ScalarIndexParams::default(), - false, - ) - .await - .unwrap(); + } - let ds = Arc::new(ds); + fn id_value_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("value", DataType::UInt32, false), + ])) + } - // Invalidate fragment 2 (the middle one) - let frag2_start = 2 * rows_per_frag; - let ds = partial_merge_insert(ds, frag2_start..frag2_start + rows_per_frag, 999.0).await; + /// `execute_provider` is the canonical entry point; a `MemTable` source merges + /// the same way a stream does. + #[tokio::test] + async fn test_merge_insert_execute_provider() { + let initial = + record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + // Update id=1, insert id=3. + let new_data = record_batch!(("id", UInt32, [1, 3]), ("value", UInt32, [10, 30])).unwrap(); + let provider: Arc = Arc::new( + datafusion::datasource::MemTable::try_new(new_data.schema(), vec![vec![new_data]]) + .unwrap(), + ); - // Verify pre-compaction state - let indices = ds.load_indices().await.unwrap(); - let idx = indices.iter().find(|i| i.name == "id_idx").unwrap(); - assert!(!idx.fragment_bitmap.as_ref().unwrap().contains(2)); + let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_provider(provider) + .await + .unwrap(); - // Run compaction with a target that forces merging of the small fragments. - let mut ds = (*ds).clone(); - let opts = CompactionOptions { - target_rows_per_fragment: total_rows, - ..Default::default() - }; - compact_files(&mut ds, opts, None).await.unwrap(); + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_inserted_rows, 1); - // The indexed fragments (0,1 and 3,4) should be compacted. - // Fragment 2 (unindexed) may or may not be compacted on its own. - // Either way, the old fragment IDs in the bitmap should be replaced. - let indices = ds.load_indices().await.unwrap(); - let idx = indices.iter().find(|i| i.name == "id_idx").unwrap(); - let bitmap = idx.fragment_bitmap.as_ref().unwrap(); - for &old_id in &[0u32, 1, 3, 4] { - assert!( - !bitmap.contains(old_id), - "Old indexed fragment {} should not be in bitmap after compaction", - old_id - ); - } - assert!( - !bitmap.is_empty(), - "Bitmap should have new compacted fragments" + let batch = merged.scan().try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let values = batch["value"].as_primitive::(); + let merged_rows: HashMap = ids + .values() + .iter() + .zip(values.values().iter()) + .map(|(id, value)| (*id, *value)) + .collect(); + assert_eq!( + merged_rows, + HashMap::from([(0, 0), (1, 10), (2, 0), (3, 30)]) ); + } - // The invalidated bitmap may still reference old fragment 2. - // This is harmless — fragment 2 no longer exists (or was compacted into - // a new fragment), so blocking it is a no-op. + /// `execute_batches` merges materialized batches; multiple batches are spread + /// across partitions and merged correctly. + #[tokio::test] + async fn test_merge_insert_execute_batches() { + let initial = + record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); - // Verify search works correctly despite stale invalidated entries. - let ds = Arc::new(ds); - let ds = partial_merge_insert(ds, frag2_start..frag2_start + rows_per_frag, 888.0).await; + // Two batches: update id=1 (batch 0), insert id=3 (batch 1). + let batch0 = record_batch!(("id", UInt32, [1]), ("value", UInt32, [10])).unwrap(); + let batch1 = record_batch!(("id", UInt32, [3]), ("value", UInt32, [30])).unwrap(); - // All rows present - let batches = ds - .scan() - .try_into_stream() - .await + let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) .unwrap() - .try_collect::>() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap() + .execute_batches(vec![batch0, batch1]) .await .unwrap(); - let combined = concat_batches(&schema, &batches).unwrap(); - assert_eq!(combined.num_rows(), total_rows); - // Updated rows have correct value - let result = ds - .scan() - .filter(&format!( - "id >= 'id-{:04}' AND id < 'id-{:04}'", - frag2_start, - frag2_start + rows_per_frag - )) + assert_eq!(stats.num_updated_rows, 1); + assert_eq!(stats.num_inserted_rows, 1); + + let batch = merged.scan().try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let values = batch["value"].as_primitive::(); + let merged_rows: HashMap = ids + .values() + .iter() + .zip(values.values().iter()) + .map(|(id, value)| (*id, *value)) + .collect(); + assert_eq!( + merged_rows, + HashMap::from([(0, 0), (1, 10), (2, 0), (3, 30)]) + ); + } + + /// An empty batch list still produces a valid (single, empty) partition, so the + /// merge is a no-op and the target is unchanged. + #[tokio::test] + async fn test_merge_insert_execute_batches_empty() { + let initial = + record_batch!(("id", UInt32, [0, 1, 2]), ("value", UInt32, [0, 0, 0])).unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + let (merged, stats) = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) .unwrap() - .try_into_stream() - .await + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() .unwrap() - .try_collect::>() + .execute_batches(vec![]) .await .unwrap(); - let result = concat_batches(&schema, &result).unwrap(); - assert_eq!(result.num_rows(), rows_per_frag); - let values = result - .column_by_name("value_a") - .unwrap() - .as_any() - .downcast_ref::() - .unwrap(); - for i in 0..rows_per_frag { - assert_eq!(values.value(i), 888.0, "row {i} should have value_a=888.0"); - } + + assert_eq!(stats.num_updated_rows, 0); + assert_eq!(stats.num_inserted_rows, 0); + + let batch = merged.scan().try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let values = batch["value"].as_primitive::(); + let merged_rows: HashMap = ids + .values() + .iter() + .zip(values.values().iter()) + .map(|(id, value)| (*id, *value)) + .collect(); + assert_eq!(merged_rows, HashMap::from([(0, 0), (1, 0), (2, 0)])); } - fn count_data_files(base_dir: &str) -> usize { - let data_dir = std::path::Path::new(base_dir).join("data"); - if !data_dir.exists() { - return 0; + fn collect_exact_row_counts(plan: &Arc, out: &mut Vec) { + if let Ok(stats) = plan.partition_statistics(None) + && let datafusion::common::stats::Precision::Exact(n) = stats.num_rows + { + out.push(n); + } + for child in plan.children() { + collect_exact_row_counts(child, out); } - std::fs::read_dir(data_dir) - .unwrap() - .filter(|e| e.as_ref().unwrap().path().is_file()) - .count() } - /// Site 3 in PR #6320: when `MergeInsertJob::apply_deletions` fails after - /// the new fragments have been written, the new data files must be cleaned up. + /// Use case 3: planning against the provider exposes its exact source + /// statistics to the optimizer. #[tokio::test] - async fn test_merge_insert_cleans_up_data_on_apply_deletions_failure() { - use crate::utils::test::FailingProxyStore; + async fn test_merge_insert_source_statistics_in_plan() { + let schema = id_value_schema(); + let target_rows = 1000u32; + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..target_rows)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( + 0, + target_rows as usize, + ))), + ], + ) + .unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); + + // A small source whose exact row count is distinct from the target's. + let source_rows = 10usize; + let new_data = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..source_rows as u32)), + Arc::new(UInt32Array::from_iter_values(std::iter::repeat_n( + 1, + source_rows, + ))), + ], + ) + .unwrap(); + let provider: Arc = + Arc::new(MemTable::try_new(schema.clone(), vec![vec![new_data]]).unwrap()); + + let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .try_build() + .unwrap(); + + // The provider's exact row count reaches the plan's statistics. + let plan = job.create_plan(provider).await.unwrap(); + let mut row_counts = Vec::new(); + collect_exact_row_counts(&plan, &mut row_counts); + assert!( + row_counts.contains(&source_rows), + "source provider's exact row count ({source_rows}) should reach the plan; got {row_counts:?}" + ); + } + /// With a one-shot stream source and `spill_for_retry(false)`, a commit + /// conflict fails fast instead of replaying the stream. The non-replayable + /// one-shot provider must be scanned exactly once (scanning it twice would + /// panic), proving retries are disabled even though `conflict_retries > 0`. + #[tokio::test] + async fn test_merge_insert_spill_for_retry_false_fails_fast() { let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, false), + Field::new("id", DataType::UInt32, false).with_metadata( + vec![( + "lance-schema:unenforced-primary-key".to_string(), + "true".to_string(), + )] + .into_iter() + .collect(), + ), + Field::new("value", DataType::UInt32, false), ])); let initial = RecordBatch::try_new( schema.clone(), vec![ - Arc::new(Int64Array::from_iter_values(0..30)), - Arc::new(StringArray::from_iter_values(std::iter::repeat_n( - "foo", 30, - ))), + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(UInt32Array::from(vec![0, 0, 0, 0])), ], ) .unwrap(); + let dataset = Arc::new( + InsertBuilder::new("memory://") + .execute(vec![initial]) + .await + .unwrap(), + ); - let test_dir = TempStrDir::default(); - let test_uri = test_dir.as_str(); - // Prefix `/` so Windows drive letters (e.g. `C:`) don't get parsed as - // the URL authority. - let path_prefix = if test_uri.starts_with('/') { "" } else { "/" }; - let routed_uri = format!("file-object-store://{path_prefix}{test_uri}"); - - let batches = RecordBatchIterator::new([Ok(initial)], schema.clone()); - let mut dataset = Dataset::write( - batches, - &routed_uri, - Some(WriteParams { - max_rows_per_file: 10, - data_storage_version: Some(LanceFileVersion::V2_1), - ..Default::default() - }), + // Merge insert job based on version 1, with retries enabled but spilling off. + let new_data = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![100])), + Arc::new(UInt32Array::from(vec![1])), + ], ) - .await .unwrap(); + let job = MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .conflict_retries(10) + .spill_for_retry(false) + .try_build() + .unwrap(); - // Create a scalar index on the join key. This forces the merge insert - // to take the slow (non-fast) path, which is the path that has the - // post-write cleanup we want to exercise. - dataset - .create_index( - &["id"], - IndexType::Scalar, - None, - &ScalarIndexParams::default(), - false, - ) + // An append commits first (version 2), so the merge built on version 1 hits + // an unresolvable conflict on commit. + let append_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from(vec![50])), + Arc::new(UInt32Array::from(vec![2])), + ], + ) + .unwrap(); + InsertBuilder::new(dataset.clone()) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![append_batch]) .await .unwrap(); - let baseline_files = count_data_files(test_uri); - assert!(baseline_files > 0); + let source = RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter(vec![Ok(new_data)]), + ); + let merge_result = job + .execute(Box::pin(source) as SendableRecordBatchStream) + .await; - let failing = Arc::new(FailingProxyStore::new()); - failing.fail_when("put", "_deletions", "injected deletions failure"); - failing.fail_when("put_multipart", "_deletions", "injected deletions failure"); + assert!( + matches!( + merge_result, + Err(crate::Error::TooMuchWriteContention { .. }) + ), + "Expected fail-fast TooMuchWriteContention, got: {:?}", + merge_result + ); + } - let dataset = DatasetBuilder::from_uri(&routed_uri) - .with_read_params(ReadParams { - store_options: Some(ObjectStoreParams { - object_store_wrapper: Some(failing.clone()), + #[tokio::test] + async fn test_merge_insert_with_blob_v1_source_provides_blob() { + use arrow_array::LargeBinaryArray; + use arrow_schema::Schema as ArrowSchema; + use lance_arrow::BLOB_META_KEY; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("blobs", DataType::LargeBinary, true).with_metadata(HashMap::from([( + BLOB_META_KEY.to_string(), + "true".to_string(), + )])), + Field::new("id", DataType::Int64, true), + Field::new("other", DataType::Int64, true), + ])); + let make_batch = |blob_values: Vec>, ids, others| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(LargeBinaryArray::from(blob_values)), + Arc::new(Int64Array::from(ids)), + Arc::new(Int64Array::from(others)), + ], + ) + .unwrap() + }; + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new( + vec![Ok(make_batch( + vec![Some(b"foo"), Some(b"bar")], + vec![0, 1], + vec![10, 20], + ))], + schema.clone(), + ), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), ..Default::default() }), - ..Default::default() - }) - .load() + ) .await - .unwrap(); - - // Update existing keys (5..15 already exist) to force the apply_deletions path. - let new_data = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from_iter_values(5..15)), - Arc::new(StringArray::from_iter_values(std::iter::repeat_n( - "bar", 10, - ))), - ], - ) - .unwrap(); - let new_reader = Box::new(RecordBatchIterator::new([Ok(new_data)], schema.clone())); + .unwrap(), + ); + let source = Box::new(RecordBatchIterator::new( + vec![Ok(make_batch( + vec![Some(b"baz"), Some(b"qux")], + vec![1, 2], + vec![200, 300], + ))], + schema, + )); - let job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) .unwrap() .when_matched(WhenMatched::UpdateAll) - .when_not_matched(WhenNotMatched::DoNothing) + .when_not_matched(WhenNotMatched::InsertAll) .try_build() .unwrap(); - - let result = job.execute_reader(new_reader).await; - assert!( - result.is_err(), - "Merge insert should fail when deletion-file write fails" + let (new_dataset, _) = job.execute_reader(source).await.unwrap(); + let blobs = new_dataset + .take_blobs_by_indices(&[0, 1, 2], "blobs") + .await + .unwrap(); + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"foo" ); - assert_eq!( - count_data_files(test_uri), - baseline_files, - "Newly written merge-insert data files should be cleaned up on apply_deletions failure" + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"baz" + ); + assert_eq!( + blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(), + b"qux" ); } #[tokio::test] - async fn test_merge_insert_full_fragment_rewrite_with_json_columns() { - // This test verifies the "all rows updated" fast path in handle_fragment - // correctly converts Arrow JSON (Utf8) to Lance JSON (LargeBinary/JSONB) - // before writing. Without conversion, the file would have Utf8 data (i32 - // offsets) but schema says LargeBinary (i64 offsets), causing decoder panic - // on subsequent reads. - // - // To trigger the fast path we need: - // 1. Subschema update (not all columns) → forces v1 update_fragments path - // 2. ALL rows in a fragment updated → triggers the fast path - use lance_arrow::ARROW_EXT_NAME_KEY; - use lance_arrow::json::{ARROW_JSON_EXT_NAME, is_arrow_json_field}; + async fn test_merge_insert_with_blob_v2_source_provides_blob() { + use crate::{BlobArrayBuilder, blob_field}; + use arrow_schema::Schema as ArrowSchema; let test_dir = TempStrDir::default(); - let mut json_metadata = HashMap::new(); - json_metadata.insert( - ARROW_EXT_NAME_KEY.to_string(), - ARROW_JSON_EXT_NAME.to_string(), - ); - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("name", DataType::Utf8, true), - Field::new("score", DataType::Int64, true), - Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata.clone()), + let schema = Arc::new(ArrowSchema::new(vec![ + blob_field("blobs", true), + Field::new("id", DataType::Int64, true), + Field::new("other", DataType::Int64, true), ])); - // Small fragment so ALL rows will be updated - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1, 2, 3])), - Arc::new(StringArray::from(vec!["a", "b", "c"])), - Arc::new(Int64Array::from(vec![10, 20, 30])), - Arc::new(StringArray::from(vec![ - r#"{"x":1}"#, - r#"{"x":2}"#, - r#"{"x":3}"#, - ])), - ], - ) - .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); - let write_params = WriteParams { - data_storage_version: Some(LanceFileVersion::V2_2), - ..Default::default() + let make_batch = |blob_values: &[&[u8]], ids, others| { + let mut blobs = BlobArrayBuilder::new(blob_values.len()); + for value in blob_values { + blobs.push_bytes(value).unwrap(); + } + RecordBatch::try_new( + schema.clone(), + vec![ + blobs.finish().unwrap(), + Arc::new(Int64Array::from(ids)), + Arc::new(Int64Array::from(others)), + ], + ) + .unwrap() }; let dataset = Arc::new( - Dataset::write(reader, test_dir.as_ref(), Some(write_params)) - .await - .unwrap(), + Dataset::write( + RecordBatchIterator::new( + vec![Ok(make_batch(&[b"foo", b"bar"], vec![0, 1], vec![10, 20]))], + schema.clone(), + ), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), ); - assert_eq!(dataset.get_fragments().len(), 1); - - // Subschema update: only provide [id, meta] (missing "name" and "score") - // This forces the v1 path (update_fragments) instead of v2 (create_plan). - // Update ALL rows → triggers the "all rows updated" fast path. - let update_schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int64, false), - Field::new("meta", DataType::Utf8, true).with_metadata(json_metadata), - ])); - let update_batch = RecordBatch::try_new( - update_schema.clone(), - vec![ - Arc::new(Int64Array::from(vec![1, 2, 3])), // all rows - Arc::new(StringArray::from(vec![ - r#"{"updated":true,"id":1}"#, - r#"{"updated":true,"id":2}"#, - r#"{"updated":true,"id":3}"#, - ])), - ], - ) - .unwrap(); - let update_reader: Box = Box::new(RecordBatchIterator::new( - vec![Ok(update_batch)], - update_schema, + let source = Box::new(RecordBatchIterator::new( + vec![Ok(make_batch( + &[b"baz", b"qux"], + vec![1, 2], + vec![200, 300], + ))], + schema, )); - let stream = reader_to_stream(update_reader); - - // Execute merge_insert with subschema - let mut builder = - MergeInsertBuilder::try_new(dataset.clone(), vec!["id".to_string()]).unwrap(); - builder.when_matched(WhenMatched::UpdateAll); - builder.when_not_matched(WhenNotMatched::DoNothing); - let job = builder.try_build().unwrap(); - let (updated_dataset, stats) = job.execute(stream).await.unwrap(); - - assert_eq!(stats.num_updated_rows, 3); - // Critical: read the data back. Without the fix, this would PANIC with: - // "the offset of the new Buffer cannot exceed the existing Length: - // slice offset=0 Length=N selfLen=N/2" - let batches = updated_dataset - .scan() - .try_into_stream() - .await + let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) .unwrap() - .try_collect::>() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() + .unwrap(); + let (new_dataset, _) = job.execute_reader(source).await.unwrap(); + let blobs = new_dataset + .take_blobs_by_indices(&[0, 1, 2], "blobs") .await .unwrap(); - let result = concat_batches(&batches[0].schema(), &batches).unwrap(); - assert_eq!(result.num_rows(), 3); - - // Verify JSON column is in Arrow JSON format (Utf8) on read - let result_schema = result.schema(); - let meta_field = result_schema.field_with_name("meta").unwrap(); - assert!( - is_arrow_json_field(meta_field), - "Expected Arrow JSON (Utf8 + arrow.json), got {:?}", - meta_field + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"foo" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"baz" + ); + assert_eq!( + blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(), + b"qux" ); + } - // Verify data correctness - let metas = result - .column_by_name("meta") - .unwrap() - .as_any() - .downcast_ref::() - .expect("meta should be StringArray after read conversion"); - for i in 0..3 { - let val = metas.value(i); - assert!( - val.contains("updated"), - "row {} should have updated meta, got: {}", - i, - val + #[tokio::test] + async fn test_merge_insert_with_complete_blob_v2_preserves_schema() { + use arrow_schema::Schema as ArrowSchema; + use lance_arrow::{ARROW_EXT_NAME_KEY, BLOB_V2_EXT_NAME}; + use lance_core::datatypes::BLOB_V2_LOGICAL_FIELDS; + + let test_dir = TempStrDir::default(); + let blob_field = Field::new( + "blobs", + DataType::Struct(BLOB_V2_LOGICAL_FIELDS.clone()), + true, + ) + .with_metadata(HashMap::from([( + ARROW_EXT_NAME_KEY.to_string(), + BLOB_V2_EXT_NAME.to_string(), + )])); + let schema = Arc::new(ArrowSchema::new(vec![ + blob_field, + Field::new("id", DataType::Int64, true), + Field::new("other", DataType::Int64, true), + ])); + let make_batch = |blob_values: &[&[u8]], ids, others| { + let blobs: arrow_array::ArrayRef = Arc::new( + StructArray::try_new( + BLOB_V2_LOGICAL_FIELDS.clone(), + vec![ + Arc::new(arrow_array::LargeBinaryArray::from_iter( + blob_values.iter().map(|value| Some(*value)), + )), + Arc::new(StringArray::from(vec![None::<&str>; blob_values.len()])), + Arc::new(UInt64Array::from(vec![None::; blob_values.len()])), + Arc::new(UInt64Array::from(vec![None::; blob_values.len()])), + ], + None, + ) + .unwrap(), ); - } + RecordBatch::try_new( + schema.clone(), + vec![ + blobs, + Arc::new(Int64Array::from(ids)), + Arc::new(Int64Array::from(others)), + ], + ) + .unwrap() + }; + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new( + vec![Ok(make_batch(&[b"foo", b"bar"], vec![0, 1], vec![10, 20]))], + schema.clone(), + ), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + let source = Box::new(RecordBatchIterator::new( + vec![Ok(make_batch( + &[b"baz", b"qux"], + vec![1, 2], + vec![200, 300], + ))], + schema, + )); - // Verify non-updated columns are preserved - let scores = result - .column_by_name("score") + let job = MergeInsertBuilder::try_new(dataset, vec!["id".to_string()]) .unwrap() - .as_any() - .downcast_ref::() + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::InsertAll) + .try_build() .unwrap(); - assert_eq!(scores.values(), &[10, 20, 30]); - - // Also verify via take (exercises the take read conversion path) - let take_result = updated_dataset - .take(&[0, 1, 2], updated_dataset.schema().clone()) + let (new_dataset, _) = job.execute_reader(source).await.unwrap(); + let dataset_schema = ArrowSchema::from(new_dataset.schema()); + let DataType::Struct(blob_children) = + dataset_schema.field_with_name("blobs").unwrap().data_type() + else { + panic!("expected complete logical blob struct after merge insert"); + }; + assert_eq!(blob_children.as_ref(), BLOB_V2_LOGICAL_FIELDS.as_ref()); + let blobs = new_dataset + .take_blobs_by_indices(&[0, 1, 2], "blobs") .await .unwrap(); - let take_schema = take_result.schema(); - let take_meta_field = take_schema.field_with_name("meta").unwrap(); - assert!( - is_arrow_json_field(take_meta_field), - "take() should return Arrow JSON, got {:?}", - take_meta_field + assert_eq!( + blobs[0].as_ref().unwrap().read().await.unwrap().as_ref(), + b"foo" + ); + assert_eq!( + blobs[1].as_ref().unwrap().read().await.unwrap().as_ref(), + b"baz" + ); + assert_eq!( + blobs[2].as_ref().unwrap().read().await.unwrap().as_ref(), + b"qux" ); } } diff --git a/rust/lance/src/dataset/write/merge_insert/exec.rs b/rust/lance/src/dataset/write/merge_insert/exec.rs index 473051da181..75c74a3078c 100644 --- a/rust/lance/src/dataset/write/merge_insert/exec.rs +++ b/rust/lance/src/dataset/write/merge_insert/exec.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors mod delete; +mod in_place; mod write; use std::collections::BTreeMap; @@ -13,6 +14,7 @@ use lance_table::format::Fragment; use roaring::RoaringTreemap; pub use delete::DeleteOnlyMergeInsertExec; +pub use in_place::InPlaceMergeInsertExec; pub use write::FullSchemaMergeInsertExec; use super::MergeStats; diff --git a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs index 07fad758902..b5df78d851e 100644 --- a/rust/lance/src/dataset/write/merge_insert/exec/delete.rs +++ b/rust/lance/src/dataset/write/merge_insert/exec/delete.rs @@ -1,7 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; use arrow_array::{Array, RecordBatch, UInt8Array, UInt64Array}; use datafusion::common::Result as DFResult; @@ -49,6 +52,7 @@ pub struct DeleteOnlyMergeInsertExec { merge_stats: Arc>>, transaction: Arc>>, affected_rows: Arc>>, + source_skipped_duplicates: Arc, } impl DeleteOnlyMergeInsertExec { @@ -56,6 +60,7 @@ impl DeleteOnlyMergeInsertExec { input: Arc, dataset: Arc, params: MergeInsertParams, + source_skipped_duplicates: Arc, ) -> DFResult { let empty_schema = Arc::new(arrow_schema::Schema::empty()); let properties = Arc::new(PlanProperties::new( @@ -74,6 +79,7 @@ impl DeleteOnlyMergeInsertExec { merge_stats: Arc::new(Mutex::new(None)), transaction: Arc::new(Mutex::new(None)), affected_rows: Arc::new(Mutex::new(None)), + source_skipped_duplicates, }) } @@ -216,10 +222,6 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { "DeleteOnlyMergeInsertExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { Arc::new(arrow_schema::Schema::empty()) } @@ -246,6 +248,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { merge_stats: self.merge_stats.clone(), transaction: self.transaction.clone(), affected_rows: self.affected_rows.clone(), + source_skipped_duplicates: self.source_skipped_duplicates.clone(), })) } @@ -283,9 +286,10 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { let merge_stats_holder = self.merge_stats.clone(); let transaction_holder = self.transaction.clone(); let affected_rows_holder = self.affected_rows.clone(); - let merged_generations = self.params.merged_generations.clone(); + let compacted_sstables = self.params.compacted_sstables.clone(); let source_dedupe_behavior = self.params.source_dedupe_behavior; let on_columns = self.params.on.clone(); + let source_skipped_duplicates = self.source_skipped_duplicates.clone(); let result_stream = futures::stream::once(async move { // Delete-only merges write no data files, but still validate any @@ -308,7 +312,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { updated_fragments, new_fragments: vec![], fields_modified: vec![], - merged_generations, + compacted_sstables, fields_for_preserving_frag_bitmap: dataset .schema() .fields @@ -323,6 +327,13 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { let transaction = Transaction::new(dataset.manifest.version, operation, None); let num_deleted = delete_row_addrs.len(); + let num_skipped_duplicates = (skipped_duplicates.value() as u64) + .checked_add(source_skipped_duplicates.load(Ordering::Relaxed)) + .ok_or_else(|| { + datafusion::error::DataFusionError::Execution( + "merge insert skipped duplicate count overflowed u64".to_string(), + ) + })?; let stats = MergeStats { num_deleted_rows: num_deleted, num_inserted_rows: 0, @@ -330,7 +341,7 @@ impl ExecutionPlan for DeleteOnlyMergeInsertExec { bytes_written: 0, num_files_written: 0, num_attempts: 1, - num_skipped_duplicates: skipped_duplicates.value() as u64, + num_skipped_duplicates, }; if let Ok(mut transaction_guard) = transaction_holder.lock() { diff --git a/rust/lance/src/dataset/write/merge_insert/exec/in_place.rs b/rust/lance/src/dataset/write/merge_insert/exec/in_place.rs new file mode 100644 index 00000000000..fdfad4f4f79 --- /dev/null +++ b/rust/lance/src/dataset/write/merge_insert/exec/in_place.rs @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashSet; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use arrow_array::{Array, RecordBatch, UInt8Array, UInt64Array}; +use arrow_schema::{Schema, SchemaRef}; +use datafusion::common::{DataFusionError, Result as DFResult}; +use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::{ + execution::{SendableRecordBatchStream, TaskContext}, + physical_plan::{ + DisplayAs, ExecutionPlan, PlanProperties, + execution_plan::{Boundedness, EmissionType}, + stream::RecordBatchStreamAdapter, + }, +}; +use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; +use futures::{StreamExt, stream}; +use lance_core::{ROW_ADDR, ROW_ID}; + +use crate::Dataset; +use crate::dataset::transaction::UpdateMode::RewriteColumns; +use crate::dataset::transaction::{Operation, Transaction}; +use crate::dataset::write::merge_insert::assign_action::Action; +use crate::dataset::write::merge_insert::{ + MERGE_ACTION_COLUMN, MERGE_SOURCE_SENTINEL, MergeInsertJob, MergeInsertParams, MergeStats, + PatchedFragments, SourceDedupeBehavior, create_duplicate_row_error, resolve_target_bases, +}; + +use super::MergeInsertMetrics; + +/// Patches the source columns into the existing fragments instead of rewriting +/// whole rows. +/// +/// This is the v2 counterpart of the legacy in-place write path: the columns +/// present in the source are written as new data files attached to the +/// fragments that already hold the matched rows, and the old versions of those +/// columns are tombstoned. Columns absent from the source are never read or +/// written, which is what makes a narrow update of a wide table cheap. +/// +/// Compared to [`super::FullSchemaMergeInsertExec`] this node: +/// - consumes only the source data columns plus `_rowaddr` / `_rowid` / +/// `__action` (the target's other columns never enter the plan, so the +/// target scan does not read them either) +/// - produces no deletion vectors and keeps fragment ids stable +/// - commits [`Operation::Update`] with [`RewriteColumns`] +/// +/// Row placement is resolved by [`MergeInsertJob::update_fragments`], which +/// already sorts by row address, groups by fragment, and fills the rows a +/// fragment did not have an update for. Reusing it keeps a single in-place +/// column-write implementation rather than adding a second one. +#[derive(Debug)] +pub struct InPlaceMergeInsertExec { + input: Arc, + dataset: Arc, + params: MergeInsertParams, + /// Duplicates the source stream dropped before the join, in `FirstSeen` + /// mode. Counted there rather than here, so it has to be folded into the + /// stats this node reports. + source_skipped_duplicates: Arc, + properties: Arc, + metrics: ExecutionPlanMetricsSet, + merge_stats: Arc>>, + transaction: Arc>>, +} + +impl InPlaceMergeInsertExec { + pub fn try_new( + input: Arc, + dataset: Arc, + params: MergeInsertParams, + source_skipped_duplicates: Arc, + ) -> DFResult { + let empty_schema = Arc::new(Schema::empty()); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(empty_schema), + Partitioning::UnknownPartitioning(1), + EmissionType::Final, + Boundedness::Bounded, + )); + + Ok(Self { + input, + dataset, + params, + source_skipped_duplicates, + properties, + metrics: ExecutionPlanMetricsSet::new(), + merge_stats: Arc::new(Mutex::new(None)), + transaction: Arc::new(Mutex::new(None)), + }) + } + + /// Takes the merge statistics if the execution has completed. + pub fn merge_stats(&self) -> Option { + self.merge_stats + .lock() + .ok() + .and_then(|mut guard| guard.take()) + } + + /// Takes the transaction if the execution has completed. + pub fn transaction(&self) -> Option { + self.transaction + .lock() + .ok() + .and_then(|mut guard| guard.take()) + } + + /// Locates the control columns and the source data columns in the input. + /// + /// The output stream carries `_rowaddr` followed by the data columns, which + /// is the schema [`MergeInsertJob::update_fragments`] expects. `_rowid` is + /// read for duplicate detection but not forwarded. Data columns are ordered + /// by the dataset schema so the written file layout does not depend on the + /// order the source happened to provide. + fn prepare_stream_schema( + &self, + input_schema: &SchemaRef, + ) -> DFResult<(usize, usize, usize, Vec, SchemaRef)> { + let index_of = |name: &str| { + input_schema + .column_with_name(name) + .map(|(i, _)| i) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "Expected {name} column in in-place merge insert input" + )) + }) + }; + let rowaddr_idx = index_of(ROW_ADDR)?; + let rowid_idx = index_of(ROW_ID)?; + let action_idx = index_of(MERGE_ACTION_COLUMN)?; + + let mut by_name = std::collections::HashMap::new(); + for (idx, field) in input_schema.fields().iter().enumerate() { + if idx == rowaddr_idx || idx == rowid_idx || idx == action_idx { + continue; + } + let name = field.name().as_str(); + if name == ROW_ADDR + || name == ROW_ID + || name == MERGE_ACTION_COLUMN + || name == MERGE_SOURCE_SENTINEL + { + continue; + } + by_name.insert(name, idx); + } + + let mut data_column_indices = Vec::with_capacity(by_name.len()); + // `_rowaddr` is nullable because that is the schema + // `update_fragments` expects; this node only ever emits non-null + // addresses (see the `Action::UpdateAll` arm in `create_patch_stream`). + let mut output_fields = vec![Arc::new(arrow_schema::Field::new( + ROW_ADDR, + arrow_schema::DataType::UInt64, + true, + ))]; + for dataset_field in self.dataset.schema().fields.iter() { + if let Some(idx) = by_name.remove(dataset_field.name.as_str()) { + data_column_indices.push(idx); + output_fields.push(Arc::new(input_schema.field(idx).clone())); + } + } + + if !by_name.is_empty() { + let mut unknown: Vec<&str> = by_name.into_keys().collect(); + unknown.sort_unstable(); + return Err(DataFusionError::Internal(format!( + "In-place merge insert input carries column(s) {unknown:?} that are not \ + dataset fields" + ))); + } + if data_column_indices.is_empty() { + return Err(DataFusionError::Internal( + "No data columns found in in-place merge insert input".to_string(), + )); + } + + Ok(( + rowaddr_idx, + rowid_idx, + action_idx, + data_column_indices, + Arc::new(Schema::new(output_fields)), + )) + } + + /// Drops the rows that must not be written and projects the rest down to + /// `_rowaddr` + data columns. + fn create_patch_stream( + &self, + input_stream: SendableRecordBatchStream, + metrics: &MergeInsertMetrics, + ) -> DFResult { + let (rowaddr_idx, rowid_idx, action_idx, data_column_indices, output_schema) = + self.prepare_stream_schema(&input_stream.schema())?; + + let dedupe = self.params.source_dedupe_behavior; + let on_columns = self.params.on.clone(); + let updated_rows = metrics.num_updated_rows.clone(); + let skipped_duplicates = metrics.num_skipped_duplicates.clone(); + let mut seen_row_ids = HashSet::new(); + + let schema = output_schema.clone(); + let stream = input_stream.map(move |batch_result| -> DFResult { + let batch = batch_result?; + let row_addrs = downcast_u64(&batch, rowaddr_idx, ROW_ADDR)?; + let row_ids = downcast_u64(&batch, rowid_idx, ROW_ID)?; + let actions = batch + .column(action_idx) + .as_any() + .downcast_ref::() + .ok_or_else(|| { + DataFusionError::Internal(format!( + "Expected UInt8Array for {MERGE_ACTION_COLUMN} column" + )) + })?; + + let mut keep_rows: Vec = Vec::with_capacity(batch.num_rows()); + for row_idx in 0..batch.num_rows() { + let action = Action::try_from(actions.value(row_idx)).map_err(|e| { + DataFusionError::Internal(format!( + "Invalid action code {}: {}", + actions.value(row_idx), + e + )) + })?; + match action { + Action::UpdateAll => { + if row_addrs.is_null(row_idx) { + return Err(DataFusionError::Internal( + "In-place merge insert produced an update without a row address" + .to_string(), + )); + } + if !seen_row_ids.insert(row_ids.value(row_idx)) { + match dedupe { + SourceDedupeBehavior::Fail => { + return Err(create_duplicate_row_error( + &batch, + row_idx, + &on_columns, + )); + } + SourceDedupeBehavior::FirstSeen => { + skipped_duplicates.add(1); + continue; + } + } + } + updated_rows.add(1); + keep_rows.push(row_idx as u32); + } + // Rows the update condition rejected: the target keeps its + // current values, so nothing is written for them. + Action::Nothing => {} + Action::Fail => { + return Err(DataFusionError::Execution(format!( + "Merge insert failed: found matching row with key values: {}", + crate::dataset::write::merge_insert::format_key_values_on_columns( + &batch, + row_idx, + &on_columns + ) + ))); + } + // Eligibility keeps inserts and deletes off this path, so + // reaching here means the routing and this node disagree. + Action::Insert | Action::Delete => { + return Err(DataFusionError::Internal(format!( + "In-place merge insert cannot handle action {action:?}" + ))); + } + } + } + + project_kept_rows( + &batch, + keep_rows, + rowaddr_idx, + &data_column_indices, + schema.clone(), + ) + }); + + Ok(Box::pin(RecordBatchStreamAdapter::new( + output_schema, + stream, + ))) + } +} + +fn downcast_u64<'a>(batch: &'a RecordBatch, idx: usize, name: &str) -> DFResult<&'a UInt64Array> { + batch + .column(idx) + .as_any() + .downcast_ref::() + .ok_or_else(|| DataFusionError::Internal(format!("Expected UInt64Array for {name} column"))) +} + +fn project_kept_rows( + batch: &RecordBatch, + keep_rows: Vec, + rowaddr_idx: usize, + data_column_indices: &[usize], + output_schema: SchemaRef, +) -> DFResult { + let mut source_indices = Vec::with_capacity(data_column_indices.len() + 1); + source_indices.push(rowaddr_idx); + source_indices.extend_from_slice(data_column_indices); + + if keep_rows.is_empty() { + let empty = output_schema + .fields() + .iter() + .map(|field| arrow_array::new_empty_array(field.data_type())) + .collect::>(); + return RecordBatch::try_new(output_schema, empty).map_err(DataFusionError::from); + } + + let indices = arrow_array::UInt32Array::from(keep_rows); + let taken = arrow_select::take::take_record_batch(batch, &indices)?; + let columns = source_indices + .iter() + .map(|&idx| taken.column(idx).clone()) + .collect::>(); + RecordBatch::try_new(output_schema, columns).map_err(DataFusionError::from) +} + +impl DisplayAs for InPlaceMergeInsertExec { + fn fmt_as( + &self, + t: datafusion::physical_plan::DisplayFormatType, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + match t { + datafusion::physical_plan::DisplayFormatType::Default + | datafusion::physical_plan::DisplayFormatType::Verbose => { + let when_matched = match &self.params.when_matched { + crate::dataset::WhenMatched::UpdateAll => "UpdateAll".to_string(), + crate::dataset::WhenMatched::UpdateIf(condition) => { + format!("UpdateIf({})", condition) + } + crate::dataset::WhenMatched::UpdateIfExpr(expr) => { + format!("UpdateIf({})", expr.human_display()) + } + other => format!("{:?}", other), + }; + write!( + f, + "InPlaceMergeInsert: on=[{}], when_matched={}, mode=RewriteColumns", + self.params.on.join(", "), + when_matched + ) + } + datafusion::physical_plan::DisplayFormatType::TreeRender => { + write!(f, "InPlaceMergeInsert[{}]", self.dataset.uri()) + } + } + } +} + +impl ExecutionPlan for InPlaceMergeInsertExec { + fn name(&self) -> &str { + "InPlaceMergeInsertExec" + } + + fn schema(&self) -> SchemaRef { + Arc::new(Schema::empty()) + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DFResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal( + "InPlaceMergeInsertExec requires exactly one child".to_string(), + )); + } + Ok(Arc::new(Self { + input: children[0].clone(), + dataset: self.dataset.clone(), + params: self.params.clone(), + source_skipped_duplicates: self.source_skipped_duplicates.clone(), + properties: self.properties.clone(), + metrics: self.metrics.clone(), + merge_stats: self.merge_stats.clone(), + transaction: self.transaction.clone(), + })) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn supports_limit_pushdown(&self) -> bool { + false + } + + fn required_input_distribution(&self) -> Vec { + vec![datafusion_physical_expr::Distribution::SinglePartition] + } + + fn benefits_from_input_partitioning(&self) -> Vec { + vec![false] + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DFResult { + let _baseline_metrics = BaselineMetrics::new(&self.metrics, partition); + let metrics = MergeInsertMetrics::new(&self.metrics, partition); + + let input_stream = self.input.execute(partition, context)?; + let patch_stream = self.create_patch_stream(input_stream, &metrics)?; + + let dataset = self.dataset.clone(); + let params = self.params.clone(); + let merge_stats_holder = self.merge_stats.clone(); + let transaction_holder = self.transaction.clone(); + let compacted_sstables = self.params.compacted_sstables.clone(); + let source_skipped_duplicates = self.source_skipped_duplicates.clone(); + + let result_stream = stream::once(async move { + let target_bases_info = resolve_target_bases(&dataset, ¶ms).await?; + // A guess: a compatible transaction can commit before this one, in + // which case the real commit version is later. `matched_offsets` + // below is what lets `build_manifest` correct the stamp. + let current_version = dataset.manifest.version + 1; + let PatchedFragments { + updated_fragments, + new_fragments, + fields_modified, + matched_offsets, + } = MergeInsertJob::update_fragments( + dataset.clone(), + patch_stream, + current_version, + target_bases_info, + ) + .await?; + + // Eligibility forbids inserts and the join is therefore an inner + // join, so every row carries a target address and no row is routed + // to a new fragment. + debug_assert!( + new_fragments.is_empty(), + "in-place merge insert produced {} new fragment(s)", + new_fragments.len() + ); + + // Only the files this operation wrote count toward the metrics: an + // updated fragment keeps its pre-existing data files and carries the + // patch as the last one. + for fragment in &updated_fragments { + if let Some(data_file) = fragment.files.last() + && let Some(size) = data_file.file_size_bytes.get() + { + metrics.bytes_written.add(u64::from(size) as usize); + } + metrics.num_files_written.add(1); + } + + let operation = Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments, + new_fragments, + fields_modified, + compacted_sstables, + // In-place patches leave every row where it was, so no index's + // fragment bitmap needs extending. + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(RewriteColumns), + inserted_rows_filter: None, + // Which rows were patched, so `build_manifest` re-stamps their + // `_row_last_updated_at_version` with the version this commit + // actually lands on rather than the one guessed above. + updated_fragment_offsets: Some(matched_offsets), + }; + let transaction = Transaction::new(dataset.manifest.version, operation, None); + + if let Ok(mut guard) = transaction_holder.lock() { + guard.replace(transaction); + } + // `FirstSeen` drops duplicate source rows before the join, so fold + // that count in — this node only sees what survived. + let mut stats = MergeStats::from(&metrics); + stats.num_skipped_duplicates = stats + .num_skipped_duplicates + .checked_add(source_skipped_duplicates.load(Ordering::Relaxed)) + .ok_or_else(|| { + DataFusionError::Execution( + "merge insert skipped duplicate count overflowed u64".to_string(), + ) + })?; + if let Ok(mut guard) = merge_stats_holder.lock() { + guard.replace(stats); + } + + Ok(RecordBatch::new_empty(Arc::new(Schema::empty()))) + }); + + Ok(Box::pin(RecordBatchStreamAdapter::new( + Arc::new(Schema::empty()), + result_stream, + ))) + } +} diff --git a/rust/lance/src/dataset/write/merge_insert/exec/write.rs b/rust/lance/src/dataset/write/merge_insert/exec/write.rs index d5b51b3d97f..5ff115d7280 100644 --- a/rust/lance/src/dataset/write/merge_insert/exec/write.rs +++ b/rust/lance/src/dataset/write/merge_insert/exec/write.rs @@ -2,7 +2,10 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::HashSet; -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; use arrow_array::{Array, RecordBatch, UInt8Array, UInt64Array}; use arrow_schema::Schema; @@ -19,6 +22,7 @@ use datafusion::{ }; use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; use futures::{StreamExt, stream}; +use lance_arrow::RecordBatchExt; use lance_core::{Error, ROW_ADDR, ROW_ID}; use lance_table::format::RowIdMeta; use roaring::RoaringTreemap; @@ -29,8 +33,8 @@ use crate::dataset::write::merge_insert::inserted_rows::{ KeyExistenceFilter, KeyExistenceFilterBuilder, extract_key_value_from_batch, }; use crate::dataset::write::merge_insert::{ - MERGE_SOURCE_SENTINEL, SourceDedupeBehavior, create_duplicate_row_error, - format_key_values_on_columns, resolve_target_bases, + InsertedKeyTracker, MERGE_SOURCE_SENTINEL, SourceDedupeBehavior, canonical_source_schema, + create_duplicate_row_error, format_key_values_on_columns, resolve_target_bases, }; use crate::{ Dataset, @@ -63,6 +67,8 @@ struct MergeState { stable_row_ids: bool, /// Set to track processed row IDs to detect duplicates processed_row_ids: HashSet, + /// Set to track non-null keys of rows inserted by FirstSeen mode + processed_insert_keys: InsertedKeyTracker, /// The "on" column names for merge operation on_columns: Vec, /// How to handle duplicate source rows @@ -84,6 +90,7 @@ impl MergeState { metrics, stable_row_ids, processed_row_ids: HashSet::new(), + processed_insert_keys: InsertedKeyTracker::default(), on_columns, source_dedupe_behavior, } @@ -166,7 +173,15 @@ impl MergeState { Ok(Some(row_idx)) // Keep this row for writing } Action::Insert => { - // Insert action - just insert new data + if self.source_dedupe_behavior == SourceDedupeBehavior::FirstSeen + && !self + .processed_insert_keys + .insert(batch, row_idx, &self.on_columns)? + { + self.metrics.num_skipped_duplicates.add(1); + return Ok(None); + } + // Capture the key value for conflict detection (only for inserts, not updates) if let Some(key_value) = extract_key_value_from_batch(batch, row_idx, &self.on_columns) @@ -211,6 +226,7 @@ pub struct FullSchemaMergeInsertExec { transaction: Arc>>, affected_rows: Arc>>, inserted_rows_filter: Arc>>, + source_skipped_duplicates: Arc, /// Whether the ON columns match the schema's unenforced primary key. /// If true, inserted_rows_filter will be included in the transaction for conflict detection. is_primary_key: bool, @@ -221,6 +237,7 @@ impl FullSchemaMergeInsertExec { input: Arc, dataset: Arc, params: MergeInsertParams, + source_skipped_duplicates: Arc, ) -> DFResult { let empty_schema = Arc::new(arrow_schema::Schema::empty()); let properties = Arc::new(PlanProperties::new( @@ -254,6 +271,7 @@ impl FullSchemaMergeInsertExec { transaction: Arc::new(Mutex::new(None)), affected_rows: Arc::new(Mutex::new(None)), inserted_rows_filter: Arc::new(Mutex::new(None)), + source_skipped_duplicates, is_primary_key, }) } @@ -445,7 +463,8 @@ impl FullSchemaMergeInsertExec { // intended writer schema (which is `dataset.schema()`). Using name // lookup is also a strictly-safer choice for the full-schema path: // it turns an implicit positional assumption into an explicit - // name-based invariant. + // name-based invariant. The filtered batches are recursively projected + // to this schema below so nested children follow the same contract. let mut name_to_idx: std::collections::HashMap<&str, usize> = std::collections::HashMap::with_capacity(input_schema.fields().len()); for (idx, field) in input_schema.fields().iter().enumerate() { @@ -468,8 +487,6 @@ impl FullSchemaMergeInsertExec { let dataset_arrow_schema: arrow_schema::Schema = self.dataset.schema().into(); let dataset_fields = dataset_arrow_schema.fields(); let mut data_column_indices: Vec = Vec::with_capacity(dataset_fields.len()); - let mut output_fields: Vec> = - Vec::with_capacity(dataset_fields.len()); for dataset_field in dataset_fields { let idx = *name_to_idx .get(dataset_field.name().as_str()) @@ -481,7 +498,6 @@ impl FullSchemaMergeInsertExec { )) })?; data_column_indices.push(idx); - output_fields.push(Arc::new(input_schema.field(idx).clone())); } if data_column_indices.is_empty() { @@ -490,7 +506,16 @@ impl FullSchemaMergeInsertExec { )); } - let output_schema = Arc::new(Schema::new(output_fields)); + let source_data_schema = Schema::new( + data_column_indices + .iter() + .map(|idx| input_schema.field(*idx).clone()) + .collect::>(), + ); + let output_schema = Arc::new( + canonical_source_schema(&source_data_schema, &dataset_arrow_schema) + .map_err(datafusion::error::DataFusionError::from)?, + ); Ok(( input_schema, @@ -568,13 +593,12 @@ impl FullSchemaMergeInsertExec { // Take only the rows we want to keep let filtered_batch = arrow_select::take::take_record_batch(batch, &indices)?; - // Project only the data columns - let output_columns: Vec<_> = data_column_indices - .iter() - .map(|&idx| filtered_batch.column(idx).clone()) - .collect(); - - RecordBatch::try_new(output_schema, output_columns) + // First retain the source field layout, then recursively project it into + // the dataset layout. The latter is required for nested structs whose + // children were supplied in a different order. + let projected = filtered_batch.project(data_column_indices)?; + projected + .project_by_schema(output_schema.as_ref()) .map_err(datafusion::error::DataFusionError::from) } @@ -797,10 +821,6 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { "FullSchemaMergeInsertExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { Arc::new(arrow_schema::Schema::empty()) } @@ -828,6 +848,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { transaction: self.transaction.clone(), affected_rows: self.affected_rows.clone(), inserted_rows_filter: self.inserted_rows_filter.clone(), + source_skipped_duplicates: self.source_skipped_duplicates.clone(), is_primary_key: self.is_primary_key, })) } @@ -868,6 +889,39 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { // Execute the input plan to get the merge data stream let input_stream = self.input.execute(partition, context)?; + let has_blob_v2_columns = self + .dataset + .schema() + .fields_pre_order() + .any(|field| field.is_blob_v2()); + let input_stream = if has_blob_v2_columns { + let input_schema = input_stream.schema(); + let rewrite_plan = Arc::new( + crate::dataset::optimize::BlobV2BatchRewritePlan::try_new( + self.dataset.schema(), + input_schema.as_ref(), + true, + ) + .map_err(|error| DataFusionError::External(Box::new(error)))?, + ); + let output_schema = rewrite_plan.output_schema().clone(); + let dataset = self.dataset.clone(); + let transformed = input_stream.then(move |batch_result| { + let dataset = dataset.clone(); + let rewrite_plan = rewrite_plan.clone(); + async move { + let batch = batch_result?; + rewrite_plan + .transform_batch(&dataset, batch) + .await + .map_err(|error| DataFusionError::External(Box::new(error))) + } + }); + Box::pin(RecordBatchStreamAdapter::new(output_schema, transformed)) + as SendableRecordBatchStream + } else { + input_stream + }; // Step 1: Create shared state and streaming processor for row addresses and write data // Get field IDs for the ON columns from the dataset schema @@ -894,7 +948,8 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { let transaction_holder = self.transaction.clone(); let affected_rows_holder = self.affected_rows.clone(); let inserted_rows_filter_holder = self.inserted_rows_filter.clone(); - let merged_generations = self.params.merged_generations.clone(); + let compacted_sstables = self.params.compacted_sstables.clone(); + let source_skipped_duplicates = self.source_skipped_duplicates.clone(); let is_primary_key = self.is_primary_key; let updating_row_ids = { let state = merge_state.lock().unwrap(); @@ -907,6 +962,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { // Keep a copy so failures after the write can clean up routed files. let cleanup_bases = target_bases_info.clone(); let (mut new_fragments, _) = write_fragments_internal( + dataset.manifest.data_storage_format.lance_file_format(), Some(&dataset), dataset.object_store.clone(), &dataset.base, @@ -937,7 +993,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) { let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } } Ok(()) @@ -994,7 +1050,7 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { updated_fragments, new_fragments, fields_modified: vec![], // No fields are modified in schema for upsert - merged_generations, + compacted_sstables, // Use the full pre-order field list (not just top-level `fields`) so // that nested leaf field ids are included. A merge_insert rewrites whole // rows, so every field is potentially modified; omitting nested ids would @@ -1023,7 +1079,15 @@ impl ExecutionPlan for FullSchemaMergeInsertExec { .add(total_files_written); // Get the final stats from the shared state - let stats = MergeStats::from(&merge_state.metrics); + let mut stats = MergeStats::from(&merge_state.metrics); + stats.num_skipped_duplicates = stats + .num_skipped_duplicates + .checked_add(source_skipped_duplicates.load(Ordering::Relaxed)) + .ok_or_else(|| { + DataFusionError::Execution( + "merge insert skipped duplicate count overflowed u64".to_string(), + ) + })?; if let Ok(mut transaction_guard) = transaction_holder.lock() { transaction_guard.replace(transaction); diff --git a/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs b/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs index 805073e75e2..711d796d3d4 100644 --- a/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs +++ b/rust/lance/src/dataset/write/merge_insert/inserted_rows.rs @@ -2,712 +2,11 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors //! Key existence tracking for merge insert conflict detection. +//! +//! The implementation lives in [`lance_table::format::key_existence`] because the +//! filter is serialized into the transaction protobuf. -use std::collections::HashSet; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; - -use arrow_array::cast::AsArray; -use arrow_array::{ - Array, BinaryArray, LargeBinaryArray, LargeListArray, LargeStringArray, ListArray, RecordBatch, - StringArray, StructArray, +pub use lance_table::format::key_existence::{ + BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, BLOOM_FILTER_DEFAULT_PROBABILITY, FilterType, + KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue, extract_key_value_from_batch, }; -use arrow_schema::DataType; -use lance_core::Result; -use lance_core::deepsize::DeepSizeOf; -use lance_core::utils::bloomfilter::sbbf::{Sbbf, SbbfBuilder}; -use lance_table::format::pb; - -// Default bloom filter config: 8192 items @ 0.00057 fpp -> 16KiB filter -pub const BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS: u64 = 8192; -pub const BLOOM_FILTER_DEFAULT_PROBABILITY: f64 = 0.00057; - -/// Key value for conflict detection. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum KeyValue { - String(String), - Int64(i64), - UInt64(u64), - Binary(Vec), - List(Vec), - Struct(Vec), - Composite(Vec), -} - -impl KeyValue { - pub fn to_bytes(&self) -> Vec { - match self { - Self::String(s) => s.as_bytes().to_vec(), - Self::Int64(i) => i.to_le_bytes().to_vec(), - Self::UInt64(u) => u.to_le_bytes().to_vec(), - Self::Binary(b) => b.clone(), - Self::List(values) | Self::Struct(values) | Self::Composite(values) => { - let mut result = Vec::new(); - for value in values { - result.extend_from_slice(&value.to_bytes()); - result.push(0); - } - result - } - } - } - - pub fn hash_value(&self) -> u64 { - let mut hasher = DefaultHasher::new(); - self.to_bytes().hash(&mut hasher); - hasher.finish() - } -} - -/// Builder for KeyExistenceFilter using Split Block Bloom Filter. -#[derive(Debug, Clone)] -pub struct KeyExistenceFilterBuilder { - sbbf: Sbbf, - field_ids: Vec, - item_count: usize, -} - -impl KeyExistenceFilterBuilder { - pub fn new(field_ids: Vec) -> Self { - let sbbf = SbbfBuilder::new() - .expected_items(BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS) - .false_positive_probability(BLOOM_FILTER_DEFAULT_PROBABILITY) - .build() - .expect("Failed to build SBBF"); - Self { - sbbf, - field_ids, - item_count: 0, - } - } - - pub fn insert(&mut self, key: KeyValue) -> Result<()> { - self.sbbf.insert(&key.to_bytes()[..]); - self.item_count += 1; - Ok(()) - } - - pub fn contains(&self, key: &KeyValue) -> bool { - self.sbbf.check(&key.to_bytes()[..]) - } - - pub fn might_intersect(&self, other: &Self) -> Result { - self.sbbf - .might_intersect(&other.sbbf) - .map_err(|e| lance_core::Error::invalid_input(e.to_string())) - } - - pub fn field_ids(&self) -> &[i32] { - &self.field_ids - } - - pub fn estimated_size_bytes(&self) -> usize { - self.sbbf.size_bytes() - } - - pub fn len(&self) -> usize { - self.item_count - } - - pub fn is_empty(&self) -> bool { - self.item_count == 0 - } - - pub fn build(&self) -> KeyExistenceFilter { - KeyExistenceFilter { - field_ids: self.field_ids.clone(), - filter: FilterType::Bloom { - bitmap: self.sbbf.to_bytes(), - num_bits: (self.sbbf.size_bytes() as u32) * 8, - number_of_items: BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, - probability: BLOOM_FILTER_DEFAULT_PROBABILITY, - }, - } - } -} - -impl From<&KeyExistenceFilterBuilder> for pb::transaction::KeyExistenceFilter { - fn from(builder: &KeyExistenceFilterBuilder) -> Self { - Self { - field_ids: builder.field_ids.clone(), - data: Some(pb::transaction::key_existence_filter::Data::Bloom( - pb::transaction::BloomFilter { - bitmap: builder.sbbf.to_bytes(), - num_bits: (builder.sbbf.size_bytes() as u32) * 8, - number_of_items: BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS, - probability: BLOOM_FILTER_DEFAULT_PROBABILITY, - }, - )), - } - } -} - -/// Filter type for key existence data. -#[derive(Debug, Clone, DeepSizeOf, PartialEq)] -pub enum FilterType { - ExactSet(HashSet), - Bloom { - bitmap: Vec, - num_bits: u32, - number_of_items: u64, - probability: f64, - }, -} - -/// Tracks keys of inserted rows for conflict detection. -/// Only created when ON columns match the schema's unenforced primary key. -#[derive(Debug, Clone, DeepSizeOf, PartialEq)] -pub struct KeyExistenceFilter { - pub field_ids: Vec, - pub filter: FilterType, -} - -impl KeyExistenceFilter { - pub fn from_bloom_filter(bloom: &KeyExistenceFilterBuilder) -> Self { - bloom.build() - } - - /// Check if two filters intersect. Returns (has_intersection, might_be_false_positive). - /// Errors if bloom filter configs don't match. - pub fn intersects(&self, other: &Self) -> Result<(bool, bool)> { - match (&self.filter, &other.filter) { - (FilterType::ExactSet(a), FilterType::ExactSet(b)) => { - Ok((a.iter().any(|h| b.contains(h)), false)) - } - (FilterType::ExactSet(_), FilterType::Bloom { .. }) - | (FilterType::Bloom { .. }, FilterType::ExactSet(_)) => { - // Can't compare different hash schemes, assume intersection - Ok((true, true)) - } - ( - FilterType::Bloom { - bitmap: a_bits, - number_of_items: a_num_items, - probability: a_prob, - .. - }, - FilterType::Bloom { - bitmap: b_bits, - number_of_items: b_num_items, - probability: b_prob, - .. - }, - ) => { - if a_num_items != b_num_items || (a_prob - b_prob).abs() > f64::EPSILON { - return Err(lance_core::Error::invalid_input(format!( - "Bloom filter config mismatch: ({}, {}) vs ({}, {})", - a_num_items, a_prob, b_num_items, b_prob - ))); - } - let has = Sbbf::bytes_might_intersect(a_bits, b_bits) - .map_err(|e| lance_core::Error::invalid_input(e.to_string()))?; - Ok((has, has)) - } - } - } -} - -impl From<&KeyExistenceFilter> for pb::transaction::KeyExistenceFilter { - fn from(filter: &KeyExistenceFilter) -> Self { - match &filter.filter { - FilterType::ExactSet(hashes) => Self { - field_ids: filter.field_ids.clone(), - data: Some(pb::transaction::key_existence_filter::Data::Exact( - pb::transaction::ExactKeySetFilter { - key_hashes: hashes.iter().copied().collect(), - }, - )), - }, - FilterType::Bloom { - bitmap, - num_bits, - number_of_items, - probability, - } => Self { - field_ids: filter.field_ids.clone(), - data: Some(pb::transaction::key_existence_filter::Data::Bloom( - pb::transaction::BloomFilter { - bitmap: bitmap.clone(), - num_bits: *num_bits, - number_of_items: *number_of_items, - probability: *probability, - }, - )), - }, - } - } -} - -impl TryFrom<&pb::transaction::KeyExistenceFilter> for KeyExistenceFilter { - type Error = lance_core::Error; - - fn try_from(message: &pb::transaction::KeyExistenceFilter) -> Result { - let filter = match message.data.as_ref() { - Some(pb::transaction::key_existence_filter::Data::Exact(exact)) => { - FilterType::ExactSet(exact.key_hashes.iter().copied().collect()) - } - Some(pb::transaction::key_existence_filter::Data::Bloom(b)) => { - // Use defaults for backwards compatibility - let number_of_items = if b.number_of_items == 0 { - BLOOM_FILTER_DEFAULT_NUMBER_OF_ITEMS - } else { - b.number_of_items - }; - let probability = if b.probability == 0.0 { - BLOOM_FILTER_DEFAULT_PROBABILITY - } else { - b.probability - }; - FilterType::Bloom { - bitmap: b.bitmap.clone(), - num_bits: b.num_bits, - number_of_items, - probability, - } - } - None => FilterType::ExactSet(HashSet::new()), - }; - Ok(Self { - field_ids: message.field_ids.clone(), - filter, - }) - } -} - -/// Extract key value from a batch row. Returns None if null or unsupported type. -pub fn extract_key_value_from_batch( - batch: &RecordBatch, - row_idx: usize, - on_columns: &[String], -) -> Option { - let mut parts: Vec = Vec::with_capacity(on_columns.len()); - - for col_name in on_columns { - let (col_idx, _) = batch.schema().column_with_name(col_name)?; - let column = batch.column(col_idx); - - if column.is_null(row_idx) { - return None; - } - - let key_part = extract_key_value(column, row_idx)?; - parts.push(key_part); - } - - if parts.is_empty() { - None - } else if parts.len() == 1 { - Some(parts.into_iter().next().unwrap()) - } else { - Some(KeyValue::Composite(parts)) - } -} - -fn extract_key_value(array: &dyn Array, row_idx: usize) -> Option { - let v = match array.data_type() { - DataType::Utf8 => { - let arr = array.as_any().downcast_ref::()?; - KeyValue::String(arr.value(row_idx).to_string()) - } - DataType::LargeUtf8 => { - let arr = array.as_any().downcast_ref::()?; - KeyValue::String(arr.value(row_idx).to_string()) - } - DataType::UInt64 => { - let arr = array.as_primitive::(); - KeyValue::UInt64(arr.value(row_idx)) - } - DataType::Int64 => { - let arr = array.as_primitive::(); - KeyValue::Int64(arr.value(row_idx)) - } - DataType::UInt32 => { - let arr = array.as_primitive::(); - KeyValue::UInt64(arr.value(row_idx) as u64) - } - DataType::Int32 => { - let arr = array.as_primitive::(); - KeyValue::Int64(arr.value(row_idx) as i64) - } - DataType::Binary => { - let arr = array.as_any().downcast_ref::()?; - KeyValue::Binary(arr.value(row_idx).to_vec()) - } - DataType::LargeBinary => { - let arr = array.as_any().downcast_ref::()?; - KeyValue::Binary(arr.value(row_idx).to_vec()) - } - DataType::List(_) => { - let list_array = array.as_any().downcast_ref::().unwrap(); - let values = list_array.value(row_idx); - - let mut elements = Vec::with_capacity(values.len()); - for i in 0..values.len() { - if values.is_null(i) { - return None; - } - let element = extract_key_value(&values, i)?; - elements.push(element); - } - KeyValue::List(elements) - } - DataType::LargeList(_) => { - let list_array = array.as_any().downcast_ref::().unwrap(); - let values = list_array.value(row_idx); - - let mut elements = Vec::with_capacity(values.len()); - for i in 0..values.len() { - if values.is_null(i) { - return None; - } - let element = extract_key_value(&values, i)?; - elements.push(element); - } - KeyValue::List(elements) - } - DataType::Struct(_) => { - let struct_array = array.as_any().downcast_ref::()?; - let mut elements = Vec::with_capacity(struct_array.num_columns()); - for i in 0..struct_array.num_columns() { - let child = struct_array.column(i); - if child.is_null(row_idx) { - return None; - } - let field_value = extract_key_value(child.as_ref(), row_idx)?; - elements.push(field_value); - } - KeyValue::Struct(elements) - } - _ => return None, - }; - Some(v) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Arc; - - use arrow_array::builder::{Int32Builder, ListBuilder, StringBuilder}; - use arrow_array::{Int32Array, RecordBatch, StringArray, StructArray}; - use arrow_schema::{Field, Schema}; - - #[test] - fn test_extract_key_value_from_batch_list_int() { - let values_builder = Int32Builder::new(); - let mut list_builder = ListBuilder::new(values_builder); - - list_builder.append_value([Some(1), Some(2)]); - list_builder.append_value([Some(3), Some(4), Some(5)]); - - let list_array = list_builder.finish(); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - list_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) - .expect("batch should be valid"); - - let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("first row should produce a key"); - let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) - .expect("second row should produce a key"); - - match &key0 { - KeyValue::List(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::Int64(1)); - assert_eq!(values[1], KeyValue::Int64(2)); - } - other => panic!("expected list key, got {:?}", other), - } - - match &key1 { - KeyValue::List(values) => { - assert_eq!(values.len(), 3); - assert_eq!(values[0], KeyValue::Int64(3)); - assert_eq!(values[1], KeyValue::Int64(4)); - assert_eq!(values[2], KeyValue::Int64(5)); - } - other => panic!("expected list key, got {:?}", other), - } - - assert_ne!( - key0.hash_value(), - key1.hash_value(), - "different list values should hash differently", - ); - } - - #[test] - fn test_extract_key_value_from_batch_empty_list() { - let values_builder = Int32Builder::new(); - let mut list_builder = ListBuilder::new(values_builder); - - list_builder.append_value(std::iter::empty::>()); - - let list_array = list_builder.finish(); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - list_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) - .expect("batch should be valid"); - - let key = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("empty list should still produce a key"); - - match key { - KeyValue::List(values) => { - assert!(values.is_empty(), "expected empty list"); - } - other => panic!("expected list key, got {:?}", other), - } - } - - #[test] - fn test_extract_key_value_from_batch_list_utf8() { - let values_builder = StringBuilder::new(); - let mut list_builder = ListBuilder::new(values_builder); - - list_builder.append_value([Some("a"), Some("bc")]); - list_builder.append_value([Some("de")]); - - let list_array = list_builder.finish(); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - list_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) - .expect("batch should be valid"); - - let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("first row should produce a key"); - let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) - .expect("second row should produce a key"); - - match &key0 { - KeyValue::List(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::String("a".to_string())); - assert_eq!(values[1], KeyValue::String("bc".to_string())); - } - other => panic!("expected list key, got {:?}", other), - } - - match &key1 { - KeyValue::List(values) => { - assert_eq!(values.len(), 1); - assert_eq!(values[0], KeyValue::String("de".to_string())); - } - other => panic!("expected list key, got {:?}", other), - } - - assert_ne!( - key0.hash_value(), - key1.hash_value(), - "different list values should hash differently", - ); - } - - #[test] - fn test_extract_key_value_from_batch_list_with_null_child() { - let values_builder = Int32Builder::new(); - let mut list_builder = ListBuilder::new(values_builder); - - list_builder.append_value([Some(1), Some(2)]); - list_builder.append_value([Some(3), None]); - - let list_array = list_builder.finish(); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - list_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(list_array)]) - .expect("batch should be valid"); - - let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("first row should produce a key"); - let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]); - - match &key0 { - KeyValue::List(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::Int64(1)); - assert_eq!(values[1], KeyValue::Int64(2)); - } - other => panic!("expected list key, got {:?}", other), - } - - assert!( - key1.is_none(), - "list row with a null child should not produce a key", - ); - } - - #[test] - fn test_extract_key_value_from_batch_struct_int() { - let a_values = Int32Array::from(vec![1, 3]); - let b_values = Int32Array::from(vec![2, 4]); - - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("a", arrow_schema::DataType::Int32, false)), - Arc::new(a_values) as Arc, - ), - ( - Arc::new(Field::new("b", arrow_schema::DataType::Int32, false)), - Arc::new(b_values) as Arc, - ), - ]); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - struct_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]) - .expect("batch should be valid"); - - let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("first row should produce a key"); - let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) - .expect("second row should produce a key"); - - match &key0 { - KeyValue::Struct(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::Int64(1)); - assert_eq!(values[1], KeyValue::Int64(2)); - } - other => panic!("expected struct key, got {:?}", other), - } - - match &key1 { - KeyValue::Struct(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::Int64(3)); - assert_eq!(values[1], KeyValue::Int64(4)); - } - other => panic!("expected struct key, got {:?}", other), - } - - assert_ne!( - key0.hash_value(), - key1.hash_value(), - "different struct values should hash differently", - ); - } - - #[test] - fn test_extract_key_value_from_batch_struct_utf8() { - let first_names = StringArray::from(vec!["alice", "bob"]); - let last_names = StringArray::from(vec!["smith", "jones"]); - - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("first", arrow_schema::DataType::Utf8, false)), - Arc::new(first_names) as Arc, - ), - ( - Arc::new(Field::new("last", arrow_schema::DataType::Utf8, false)), - Arc::new(last_names) as Arc, - ), - ]); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - struct_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]) - .expect("batch should be valid"); - - let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("first row should produce a key"); - let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]) - .expect("second row should produce a key"); - - match &key0 { - KeyValue::Struct(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::String("alice".to_string())); - assert_eq!(values[1], KeyValue::String("smith".to_string())); - } - other => panic!("expected struct key, got {:?}", other), - } - - match &key1 { - KeyValue::Struct(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::String("bob".to_string())); - assert_eq!(values[1], KeyValue::String("jones".to_string())); - } - other => panic!("expected struct key, got {:?}", other), - } - - assert_ne!( - key0.hash_value(), - key1.hash_value(), - "different struct values should hash differently", - ); - } - - #[test] - fn test_extract_key_value_from_batch_struct_with_null_child() { - let a_values = Int32Array::from(vec![Some(1), None]); - let b_values = Int32Array::from(vec![Some(2), Some(3)]); - - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("a", arrow_schema::DataType::Int32, true)), - Arc::new(a_values) as Arc, - ), - ( - Arc::new(Field::new("b", arrow_schema::DataType::Int32, true)), - Arc::new(b_values) as Arc, - ), - ]); - - let schema = Arc::new(Schema::new(vec![Field::new( - "id", - struct_array.data_type().clone(), - false, - )])); - - let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]) - .expect("batch should be valid"); - - let key0 = extract_key_value_from_batch(&batch, 0, &[String::from("id")]) - .expect("first row should produce a key"); - let key1 = extract_key_value_from_batch(&batch, 1, &[String::from("id")]); - - match &key0 { - KeyValue::Struct(values) => { - assert_eq!(values.len(), 2); - assert_eq!(values[0], KeyValue::Int64(1)); - assert_eq!(values[1], KeyValue::Int64(2)); - } - other => panic!("expected struct key, got {:?}", other), - } - - assert!( - key1.is_none(), - "struct row with a null child should not produce a key", - ); - } -} diff --git a/rust/lance/src/dataset/write/merge_insert/logical_plan.rs b/rust/lance/src/dataset/write/merge_insert/logical_plan.rs index 7f67972ff7e..cc4bb6b977c 100644 --- a/rust/lance/src/dataset/write/merge_insert/logical_plan.rs +++ b/rust/lance/src/dataset/write/merge_insert/logical_plan.rs @@ -11,16 +11,35 @@ use datafusion::{ }; use datafusion_expr::{LogicalPlan, UserDefinedLogicalNode, UserDefinedLogicalNodeCore}; use lance_core::{ROW_ADDR, ROW_ID}; -use std::{cmp::Ordering, sync::Arc}; +use std::{ + cmp::Ordering, + sync::{Arc, atomic::AtomicU64}, +}; use crate::Dataset; use crate::dataset::write::merge_insert::exec::{ - DeleteOnlyMergeInsertExec, FullSchemaMergeInsertExec, + DeleteOnlyMergeInsertExec, FullSchemaMergeInsertExec, InPlaceMergeInsertExec, }; use crate::dataset::{WhenMatched, WhenNotMatchedBySource}; use super::{MERGE_ACTION_COLUMN, MERGE_SOURCE_SENTINEL, MergeInsertParams}; +/// Which write half a planned merge insert will use. +/// +/// This is the *resolved* choice, so it has no `Auto`: the caller's +/// [`MergeInsertWriteMode`](super::MergeInsertWriteMode) is resolved against the +/// operation by [`MergeInsertJob::select_write_sink`](super::MergeInsertJob) before +/// the plan is built. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum WriteSink { + /// Delete the matched rows and write whole rows into new fragments. + RewriteRows, + /// Attach new data files for the source columns to the fragments that + /// already hold the matched rows and tombstone the old versions of those + /// columns. + RewriteColumns, +} + /// Logical plan node for merge insert write. /// /// Expects input schema: @@ -36,12 +55,15 @@ pub struct MergeInsertWriteNode { input: LogicalPlan, pub(crate) dataset: Arc, pub(crate) params: MergeInsertParams, + pub(crate) source_skipped_duplicates: Arc, + pub(crate) write_sink: WriteSink, schema: Arc, } impl PartialEq for MergeInsertWriteNode { fn eq(&self, other: &Self) -> bool { self.params == other.params + && self.write_sink == other.write_sink && self.input == other.input && self.dataset.base == other.dataset.base } @@ -52,6 +74,7 @@ impl Eq for MergeInsertWriteNode {} impl std::hash::Hash for MergeInsertWriteNode { fn hash(&self, state: &mut H) { self.params.hash(state); + self.write_sink.hash(state); self.input.hash(state); self.dataset.base.hash(state); } @@ -60,20 +83,31 @@ impl std::hash::Hash for MergeInsertWriteNode { impl PartialOrd for MergeInsertWriteNode { fn partial_cmp(&self, other: &Self) -> Option { match self.params.partial_cmp(&other.params) { - Some(Ordering::Equal) => self.input.partial_cmp(&other.input), + Some(Ordering::Equal) => match self.write_sink.cmp(&other.write_sink) { + Ordering::Equal => self.input.partial_cmp(&other.input), + cmp => Some(cmp), + }, cmp => cmp, } } } impl MergeInsertWriteNode { - pub fn new(input: LogicalPlan, dataset: Arc, params: MergeInsertParams) -> Self { + pub fn new( + input: LogicalPlan, + dataset: Arc, + params: MergeInsertParams, + source_skipped_duplicates: Arc, + write_sink: WriteSink, + ) -> Self { let empty_schema = Arc::new(arrow_schema::Schema::empty()); let schema = Arc::new(DFSchema::try_from(empty_schema).unwrap()); Self { input, dataset, params, + source_skipped_duplicates, + write_sink, schema, } } @@ -121,7 +155,11 @@ impl UserDefinedLogicalNodeCore for MergeInsertWriteNode { f, "MergeInsertWrite: on=[{}], when_matched={}, when_not_matched={}, when_not_matched_by_source={}", on_keys, when_matched, when_not_matched, when_not_matched_by_source - ) + )?; + if self.write_sink == WriteSink::RewriteColumns { + write!(f, ", mode=RewriteColumns")?; + } + Ok(()) } fn with_exprs_and_inputs( @@ -143,6 +181,8 @@ impl UserDefinedLogicalNodeCore for MergeInsertWriteNode { inputs[0].clone(), self.dataset.clone(), self.params.clone(), + self.source_skipped_duplicates.clone(), + self.write_sink, )) } @@ -248,12 +288,21 @@ impl ExtensionPlanner for MergeInsertPlanner { physical_inputs[0].clone(), write_node.dataset.clone(), write_node.params.clone(), + write_node.source_skipped_duplicates.clone(), + )?) + } else if write_node.write_sink == WriteSink::RewriteColumns { + Arc::new(InPlaceMergeInsertExec::try_new( + physical_inputs[0].clone(), + write_node.dataset.clone(), + write_node.params.clone(), + write_node.source_skipped_duplicates.clone(), )?) } else { Arc::new(FullSchemaMergeInsertExec::try_new( physical_inputs[0].clone(), write_node.dataset.clone(), write_node.params.clone(), + write_node.source_skipped_duplicates.clone(), )?) }; Some(exec) diff --git a/rust/lance/src/dataset/write/retry.rs b/rust/lance/src/dataset/write/retry.rs index 1a72c95cec4..ca5bf09b51a 100644 --- a/rust/lance/src/dataset/write/retry.rs +++ b/rust/lance/src/dataset/write/retry.rs @@ -1,17 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; -use either::Either; -use futures::TryFutureExt; -use futures::future::FutureExt; use lance_core::utils::backoff::SlotBackoff; use lance_core::{Error, Result}; use crate::Dataset; +use crate::io::commit::{maybe_timeout, timeout_error}; /// Configuration for retry behavior #[derive(Debug, Clone)] @@ -44,33 +41,6 @@ pub trait RetryExecutor: Clone { fn update_dataset(&mut self, dataset: Arc); } -fn timeout_error(retry_timeout: Duration, attempts: u32) -> Error { - Error::too_much_write_contention(format!( - "Attempted {} times, but failed on retry_timeout of {:.3} seconds.", - attempts, - retry_timeout.as_secs_f32() - )) -} - -fn maybe_timeout( - backoff: &SlotBackoff, - start: Instant, - retry_timeout: Duration, - future: impl Future, -) -> impl Future> { - let attempt = backoff.attempt(); - if attempt == 0 { - // No timeout on first attempt - Either::Left(future.map(|res| Ok(res))) - } else { - let remaining = retry_timeout.saturating_sub(start.elapsed()); - Either::Right( - tokio::time::timeout(remaining, future) - .map_err(move |_| timeout_error(retry_timeout, attempt + 1)), - ) - } -} - /// Execute an operation with retry logic for commit conflicts pub async fn execute_with_retry( executor: E, @@ -86,11 +56,17 @@ pub async fn execute_with_retry( executor_clone.update_dataset(dataset_ref.clone()); let execute_fut = executor_clone.execute_impl(); - let execute_fut = maybe_timeout(&backoff, start, config.retry_timeout, execute_fut); + let execute_fut = + maybe_timeout(backoff.attempt(), start, config.retry_timeout, execute_fut); let data = execute_fut.await??; let commit_future = executor.commit(dataset_ref.clone(), data); - let commit_future = maybe_timeout(&backoff, start, config.retry_timeout, commit_future); + let commit_future = maybe_timeout( + backoff.attempt(), + start, + config.retry_timeout, + commit_future, + ); match commit_future.await? { Ok(result) => return Ok(result), @@ -111,7 +87,8 @@ pub async fn execute_with_retry( } let sleep_fut = tokio::time::sleep(backoff.next_backoff()); - let sleep_fut = maybe_timeout(&backoff, start, config.retry_timeout, sleep_fut); + let sleep_fut = + maybe_timeout(backoff.attempt(), start, config.retry_timeout, sleep_fut); sleep_fut.await?; let mut ds = dataset_ref.as_ref().clone(); diff --git a/rust/lance/src/dataset/write/update.rs b/rust/lance/src/dataset/write/update.rs index a87b3a4260b..7b7b864fbf1 100644 --- a/rust/lance/src/dataset/write/update.rs +++ b/rust/lance/src/dataset/write/update.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -14,17 +14,18 @@ use crate::dataset::transaction::{Operation, Transaction}; use crate::dataset::utils::make_rowid_capture_stream; use crate::{Dataset, io::exec::Planner}; use crate::{Error, Result}; -use arrow_array::RecordBatch; +use arrow_array::{ArrayRef, RecordBatch}; use arrow_schema::{ArrowError, DataType, Schema as ArrowSchema}; use datafusion::common::DFSchema; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::logical_expr::ExprSchemable; -use datafusion::physical_plan::PhysicalExpr; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{PhysicalExpr, SendableRecordBatchStream}; use datafusion::prelude::Expr; use datafusion::scalar::ScalarValue; use futures::StreamExt; use lance_arrow::RecordBatchExt; +use lance_arrow::json::{JsonArray, is_json_field}; use lance_core::datatypes::BlobHandling; use lance_core::error::{InvalidInputSnafu, box_error}; use lance_core::utils::tokio::get_num_compute_intensive_cpus; @@ -133,6 +134,16 @@ impl UpdateBuilder { )) })?; + if crate::dataset::optimize::field_contains_blob_v2(field) { + return Err(Error::not_supported_source( + format!( + "Direct updates to column '{}' containing blob v2 values are not supported", + column.as_ref() + ) + .into(), + )); + } + // TODO: support nested column references. This is mostly blocked on the // ability to insert them into the RecordBatch properly. if column.as_ref().contains('.') { @@ -159,7 +170,16 @@ impl UpdateBuilder { .get_type(&df_schema) .map_err(box_error) .context(InvalidInputSnafu {})?; - if dest_type != src_type { + // A string assigned to a JSON field is logical JSON, not its LargeBinary storage. + // Keep it as UTF-8 here so `apply_updates` can validate and encode it as JSONB. + let is_json_string = schema + .field_with_name(column.as_ref()) + .is_ok_and(is_json_field) + && matches!( + &src_type, + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ); + if dest_type != src_type && !is_json_string { expr = match expr { // TODO: remove this branch once DataFusion supports casting List to FSL // This should happen in Arrow 51.0.0 @@ -280,8 +300,25 @@ impl UpdateJob { async fn execute_impl(self) -> Result { let mut scanner = self.dataset.scan(); + let legacy_blob_ids = self + .dataset + .schema() + .fields_pre_order() + .filter(|field| field.is_blob() && !field.is_blob_v2()) + .filter_map(|field| u32::try_from(field.id).ok()) + .collect::>(); + if !legacy_blob_ids.is_empty() { + scanner.blob_handling(BlobHandling::SomeBlobsBinary(legacy_blob_ids)); + } + let has_blob_v2_columns = self + .dataset + .schema() + .fields_pre_order() + .any(|field| field.is_blob_v2()); + if has_blob_v2_columns { + scanner.with_row_address(); + } scanner.with_row_id(); - scanner.blob_handling(BlobHandling::AllBinary); if let Some(expr) = &self.condition { scanner.filter_expr(expr.clone()); @@ -296,16 +333,77 @@ impl UpdateJob { let (stream, row_id_rx) = make_rowid_capture_stream(stream, self.dataset.manifest.uses_stable_row_ids())?; - let schema = stream.schema(); - - let expected_schema = self.dataset.schema().into(); - if schema.as_ref() != &expected_schema { + let scan_schema = stream.schema(); + let expected_schema: ArrowSchema = self.dataset.schema().into(); + if !has_blob_v2_columns && scan_schema.as_ref() != &expected_schema { return Err(Error::internal(format!( "Expected schema {:?} but got {:?}", - expected_schema, schema + expected_schema, scan_schema ))); } + let stream = if has_blob_v2_columns { + let rewrite_plan = Arc::new(crate::dataset::optimize::BlobV2BatchRewritePlan::try_new( + self.dataset.schema(), + scan_schema.as_ref(), + false, + )?); + let output_schema = rewrite_plan.output_schema().clone(); + let dataset = self.dataset.clone(); + let transformed = stream.then(move |batch_result| { + let dataset = dataset.clone(); + let rewrite_plan = rewrite_plan.clone(); + async move { + let batch = batch_result?; + rewrite_plan + .transform_batch(&dataset, batch) + .await + .map_err(|error| DataFusionError::External(Box::new(error))) + } + }); + Box::pin(RecordBatchStreamAdapter::new(output_schema, transformed)) + as SendableRecordBatchStream + } else { + stream + }; + let schema = stream.schema(); + + let updated_blob_columns = self + .updates + .keys() + .filter(|column_name| { + self.dataset + .schema() + .field(column_name) + .is_some_and(crate::dataset::optimize::field_contains_blob_v2) + }) + .cloned() + .collect::>(); + let updated_blob_column_indices = schema + .fields() + .iter() + .enumerate() + .filter_map(|(column_idx, field)| { + updated_blob_columns + .contains(field.name()) + .then_some(column_idx) + }) + .collect::>(); + let write_params = WriteParams { + allow_external_blob_outside_bases: has_blob_v2_columns, + ..Default::default() + }; + let external_base_resolver = if updated_blob_column_indices.is_empty() { + None + } else { + super::blob_v2_external_base_resolver( + Some(self.dataset.as_ref()), + &write_params, + self.dataset.schema(), + ) + .await? + }; + let updates_ref = self.updates.clone(); let stream = stream .map(move |batch| { @@ -317,21 +415,39 @@ impl UpdateJob { Ok(Ok(batch)) => Ok(batch), Ok(Err(err)) => Err(err), Err(e) => Err(DataFusionError::ExecutionJoin(Box::new(e))), + }) + .then(move |batch_result| { + let external_base_resolver = external_base_resolver.clone(); + let updated_blob_column_indices = updated_blob_column_indices.clone(); + async move { + let batch = batch_result?; + if let Some(resolver) = external_base_resolver.as_deref() { + let updated_blob_batch = batch.project(&updated_blob_column_indices)?; + let selected_rows = vec![true; batch.num_rows()]; + crate::dataset::blob::validate_external_blob_references( + resolver, + &updated_blob_batch, + &selected_rows, + ) + .await + .map_err(|error| DataFusionError::External(Box::new(error)))?; + } + Ok(batch) + } }); let stream = RecordBatchStreamAdapter::new(schema, stream); - let version = self - .dataset - .manifest() - .data_storage_format - .lance_file_version()?; let (mut new_fragments, _) = write_fragments_internal( + self.dataset + .manifest + .data_storage_format + .lance_file_format(), Some(&self.dataset), self.dataset.object_store.clone(), &self.dataset.base, self.dataset.schema().clone(), Box::pin(stream), - WriteParams::with_storage_version(version), + write_params, None, // TODO: support multiple bases for update ) .await?; @@ -357,13 +473,13 @@ impl UpdateJob { })?; for (fragment, sequence) in new_fragments.iter_mut().zip(sequences) { let serialized = lance_table::rowids::write_row_ids(&sequence); - fragment.row_id_meta = Some(RowIdMeta::Inline(serialized)); + fragment.row_id_meta = Some(RowIdMeta::Inline(serialized.into())); } } // Apply deletions let row_id_index = get_row_id_index(&self.dataset).await?; - let row_addrs = removed_row_ids.row_addrs(row_id_index.as_deref()); + let row_addrs = removed_row_ids.row_addrs(row_id_index.as_deref())?; let deletions_result = self.apply_deletions(&row_addrs).await; let (old_fragments, removed_fragment_ids) = match deletions_result { Ok(v) => v, @@ -419,7 +535,7 @@ impl UpdateJob { // are moved(deleted and appended). // so we do not need to handle the frag bitmap of the index about it. fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap, update_mode: Some(RewriteRows), inserted_rows_filter: None, @@ -445,6 +561,34 @@ impl UpdateJob { ) -> DFResult { for (column, expr) in updates.iter() { let new_values = expr.evaluate(&batch)?.into_array(batch.num_rows())?; + let schema = batch.schema(); + let new_values: ArrayRef = if schema.field_with_name(column).is_ok_and(is_json_field) + && matches!( + new_values.data_type(), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) { + let new_values = if new_values.data_type() == &DataType::Utf8View { + arrow_cast::cast(new_values.as_ref(), &DataType::Utf8).map_err(|error| { + DataFusionError::ArrowError( + Box::new(error), + Some(format!( + "convert Utf8View update for JSON column '{column}'" + )), + ) + })? + } else { + new_values + }; + let json_array = JsonArray::try_from(new_values).map_err(|error| { + DataFusionError::ArrowError( + Box::new(error), + Some(format!("encode update for JSON column '{column}'")), + ) + })?; + Arc::new(json_array.into_inner()) + } else { + new_values + }; batch = batch.replace_column_by_name(column.as_str(), new_values)?; } Ok(batch) @@ -538,19 +682,21 @@ mod tests { array::AsArray, datatypes::{Int64Type, UInt32Type}, }; - use arrow_array::types::Float32Type; - use arrow_array::{Int64Array, RecordBatchIterator, StringArray, UInt32Array, UInt64Array}; + use arrow_array::types::{Float32Type, Int32Type}; + use arrow_array::{ + Int64Array, RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, + }; use arrow_schema::{Field, Schema as ArrowSchema}; use arrow_select::concat::concat_batches; use futures::{TryStreamExt, future::try_join_all}; use lance_arrow::ARROW_EXT_NAME_KEY; - use lance_arrow::json::{ARROW_JSON_EXT_NAME, is_arrow_json_field, is_json_field}; + use lance_arrow::json::{ARROW_JSON_EXT_NAME, is_arrow_json_field}; use lance_core::ROW_ID; use lance_core::utils::tempfile::TempStrDir; use lance_datagen::{Dimension, RowCount}; use lance_file::version::LanceFileVersion; use lance_index::IndexType; - use lance_index::scalar::ScalarIndexParams; + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use lance_io::object_store::ObjectStoreParams; use lance_linalg::distance::MetricType; use object_store::throttle::ThrottleConfig; @@ -760,8 +906,11 @@ mod tests { assert_eq!(fragments[2].metadata.physical_rows, Some(15)); } + #[rstest] + #[case::utf8(r#"'{"after": true, "n": 2}'"#)] + #[case::utf8_view(r#"arrow_cast('{"after": true, "n": 2}', 'Utf8View')"#)] #[tokio::test] - async fn test_update_json_and_regular_columns() { + async fn test_update_json_and_regular_columns(#[case] json_expression: &str) { let mut metadata = HashMap::new(); metadata.insert( ARROW_EXT_NAME_KEY.to_string(), @@ -804,7 +953,7 @@ mod tests { .unwrap() .set("name", "'updated'") .unwrap() - .set("meta", r#"jsonb '{"after":true,"n":2}'"#) + .set("meta", json_expression) .unwrap() .build() .unwrap() @@ -833,6 +982,16 @@ mod tests { assert_eq!(names.value(updated_row_idx), "updated"); assert_eq!(metas.value(updated_row_idx), r#"{"after":true,"n":2}"#); + + let filtered_batch = updated_dataset + .scan() + .filter("json_extract(meta, '$.n') = '2'") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(filtered_batch.num_rows(), 1); + assert_eq!(filtered_batch["id"].as_primitive::().value(0), 2); } #[rstest] @@ -952,8 +1111,8 @@ mod tests { // Increase likelihood of contention by throttling the store let throttled = Arc::new(ThrottledStoreWrapper { config: ThrottleConfig { - wait_list_per_call: Duration::from_millis(10), - wait_get_per_call: Duration::from_millis(10), + wait_list_per_call: Duration::from_millis(1), + wait_get_per_call: Duration::from_millis(1), ..Default::default() }, }); @@ -1284,6 +1443,178 @@ mod tests { ); } + #[rstest] + #[case::zone_map(BuiltinIndexType::ZoneMap, "i < 100", 100)] + #[case::bloom_filter(BuiltinIndexType::BloomFilter, "i = 0", 1)] + #[tokio::test] + async fn test_addr_domain_index_does_not_cover_rewritten_update_fragment( + #[case] index_type: BuiltinIndexType, + #[case] query: &str, + #[case] expected_rows: usize, + ) { + let mut dataset = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .col("category", lance_datagen::array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(1), + FragmentRowCount::from(100), + Some(WriteParams { + max_rows_per_file: 100, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["i"], + IndexType::Scalar, + Some("i_idx".to_string()), + &ScalarIndexParams::for_builtin(index_type), + true, + ) + .await + .unwrap(); + + let before = dataset + .scan() + .filter(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(before.num_rows(), expected_rows); + + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("i < 20") + .unwrap() + .set("category", "-1") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + + let indices = dataset.load_indices().await.unwrap(); + let index = indices.iter().find(|index| index.name == "i_idx").unwrap(); + assert_eq!( + index + .fragment_bitmap + .as_ref() + .unwrap() + .iter() + .collect::>(), + vec![0], + "the address-domain index must not cover the rewritten fragment" + ); + + // Regression for https://github.com/lance-format/lance/issues/8278: a later + // update must find rows moved out of the address-domain index's coverage. + let second_update = UpdateBuilder::new(dataset) + .update_where(query) + .unwrap() + .set("category", "-2") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap(); + assert_eq!(second_update.rows_updated, expected_rows as u64); + let dataset = second_update.new_dataset; + + let after = dataset + .scan() + .filter(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(after.num_rows(), expected_rows); + + let updated = dataset + .scan() + .filter("category = -2") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(updated.num_rows(), expected_rows); + } + + /// Regression test for https://github.com/lance-format/lance/issues/8076 + /// + /// A bloom filter index reports matches as physical row addresses. An update that + /// replaces every row of a fragment removes that fragment, but the index keeps the + /// addresses it holds for it, so translating its results to row ids has to tolerate + /// a fragment that is gone rather than fail with an internal error. + #[tokio::test] + async fn test_addr_domain_index_after_update_drops_fragment() { + let mut dataset = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["i"], + IndexType::BloomFilter, + Some("i_idx".to_string()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::BloomFilter), + true, + ) + .await + .unwrap(); + + // Rewrites all of fragment 1 (rows 3, 4, 5), which drops the fragment. + let dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where("i >= 3") + .unwrap() + .set("i", "-1") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset; + assert!(dataset.get_fragments().iter().all(|frag| frag.id() != 1)); + + // The index still holds a block for the dropped fragment, and a bloom filter + // cannot rule out a value it once held, so this query is the one that reaches + // the index with addresses in that fragment. + let matched = dataset + .scan() + .filter("i = 4") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(matched.num_rows(), 0); + + let updated = dataset + .scan() + .filter("i = -1") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(updated.num_rows(), 3); + } + #[tokio::test] async fn test_update_mixed_indexed_unindexed_fragments() { let mut dataset = lance_datagen::gen_batch() @@ -1774,4 +2105,104 @@ mod tests { let idx_foo = ids.values().iter().position(|&x| x == 0).unwrap(); assert_eq!(blobs.value(idx_foo), b"foo"); } + + #[rstest] + #[case::non_empty(0)] + #[case::empty(1)] + #[case::null(2)] + #[tokio::test] + async fn test_update_preserves_blob_v2(#[case] selected_id: i64) { + use crate::{BlobArrayBuilder, blob_field}; + + let make_blobs = || { + let mut builder = BlobArrayBuilder::new(3); + builder.push_bytes(b"one").unwrap(); + builder.push_bytes(b"").unwrap(); + builder.push_null().unwrap(); + builder.finish().unwrap() + }; + let nested_fields = vec![blob_field("blob", true)]; + let nested: Arc = Arc::new( + StructArray::try_new(nested_fields.clone().into(), vec![make_blobs()], None).unwrap(), + ); + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("body", DataType::Utf8, false), + blob_field("payload", true), + Field::new("info", DataType::Struct(nested_fields.into()), true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int64Array::from(vec![0, 1, 2])), + Arc::new(StringArray::from(vec!["body-0", "body-1", "body-2"])), + make_blobs(), + nested, + ], + ) + .unwrap(); + let test_dir = TempStrDir::default(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_2), + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + for column in ["payload", "info"] { + let error = UpdateBuilder::new(dataset.clone()) + .set(column, column) + .unwrap_err(); + assert!(matches!(error, Error::NotSupported { .. })); + assert!( + error.to_string().contains(&format!( + "Direct updates to column '{column}' containing blob v2 values are not supported" + )), + "unexpected error: {error}" + ); + } + + let result = UpdateBuilder::new(dataset) + .update_where(&format!("id = {selected_id}")) + .unwrap() + .set("body", "'updated'") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap(); + assert_eq!(result.rows_updated, 1); + + let mut scanner = result.new_dataset.scan(); + scanner.blob_handling(BlobHandling::AllBinary); + let batch = scanner.try_into_batch().await.unwrap(); + let ids = batch["id"].as_primitive::(); + let bodies = batch["body"].as_string::(); + let payloads = batch["payload"].as_binary::(); + let nested = batch["info"] + .as_struct() + .column_by_name("blob") + .unwrap() + .as_binary::(); + let expected = [Some(b"one".as_slice()), Some(b"".as_slice()), None]; + + for row_idx in 0..batch.num_rows() { + let id = ids.value(row_idx) as usize; + let expected_body = if id as i64 == selected_id { + "updated" + } else { + ["body-0", "body-1", "body-2"][id] + }; + assert_eq!(bodies.value(row_idx), expected_body); + assert_eq!(payloads.iter().nth(row_idx).unwrap(), expected[id]); + assert_eq!(nested.iter().nth(row_idx).unwrap(), expected[id]); + } + } } diff --git a/rust/lance/src/index.rs b/rust/lance/src/index.rs index f5afb48804c..b5803624520 100644 --- a/rust/lance/src/index.rs +++ b/rust/lance/src/index.rs @@ -7,25 +7,28 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, OnceLock}; +use std::time::Instant; use arrow_schema::DataType; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; -use futures::FutureExt; +use futures::{FutureExt, StreamExt, TryStreamExt}; use itertools::Itertools; -use lance_core::cache::CacheKey; +use lance_core::cache::{CacheKey, CacheKeySchema, KeyBuilder}; use lance_core::datatypes::Field; use lance_core::datatypes::Schema as LanceSchema; use lance_core::utils::parse::parse_env_as_bool; use lance_core::utils::tracing::{ IO_TYPE_OPEN_FRAG_REUSE, IO_TYPE_OPEN_MEM_WAL, IO_TYPE_OPEN_VECTOR, TRACE_IO_EVENTS, }; -use lance_file::previous::reader::FileReader as PreviousFileReader; use lance_file::reader::FileReaderOptions; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_index::INDEX_METADATA_SCHEMA_KEY; pub use lance_index::IndexParams; -use lance_index::frag_reuse::{FRAG_REUSE_INDEX_NAME, FragReuseIndex}; -use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex}; +use lance_index::frag_reuse::{ + CompactFragReuseIndex, CompactFragReuseIndexHandle, FRAG_REUSE_INDEX_NAME, +}; +use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexHandle}; use lance_index::optimize::OptimizeOptions; use lance_index::pb::index::Implementation; pub use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress}; @@ -33,7 +36,7 @@ use lance_index::scalar::expression::{IndexInformationProvider, MultiQueryParser use lance_index::scalar::inverted::{InvertedIndex, InvertedIndexPlugin}; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering}; -use lance_index::scalar::{CreatedIndex, ScalarIndex}; +use lance_index::scalar::{CreatedIndex, ScalarIndex, index_files_to_table, table_files_to_index}; use lance_index::vector::bq::builder::RabitQuantizer; use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatQuantizer}; use lance_index::vector::hnsw::HNSW; @@ -41,7 +44,10 @@ use lance_index::vector::pq::ProductQuantizer; use lance_index::vector::quantizer::Quantization; use lance_index::vector::sq::ScalarQuantizer; use lance_index::vector::v3::subindex::IvfSubIndex; -use lance_index::{INDEX_FILE_NAME, Index, IndexType, PrewarmOptions, pb, vector::VectorIndex}; +use lance_index::{ + FtsPrewarmDiagnostics, FtsPrewarmOptions, FtsPrewarmResult, FtsPrewarmSegmentStatus, + INDEX_FILE_NAME, Index, IndexType, PrewarmOptions, pb, vector::VectorIndex, +}; use lance_index::{ IndexCriteria, is_system_index, metrics::{MetricsCollector, NoOpMetricsCollector}, @@ -54,16 +60,17 @@ use lance_io::utils::{ CachedFileSize, read_last_block, read_message, read_message_from_buf, read_metadata_offset, read_version, }; -use lance_table::format::{Fragment, SelfDescribingFileReader}; +use lance_table::format::{DataFile, Fragment, SelfDescribingFileReader}; use lance_table::format::{IndexFile, IndexMetadata, list_index_files_with_sizes}; use lance_table::io::manifest::read_manifest_indexes; use roaring::RoaringBitmap; use scalar::index_matches_criteria; use serde_json::json; -use tracing::{info, instrument}; +use tracing::{info, instrument, warn}; use uuid::Uuid; use vector::details::{ - derive_vector_index_type, infer_missing_vector_details, vector_details_as_json, + derive_vector_index_type, infer_missing_vector_details, needs_vector_details_inference, + vector_details_as_json, }; pub(crate) use vector::details::{vector_index_details, vector_index_details_default}; use vector::ivf::v2::{IVFIndex, IvfStateEntryBox}; @@ -84,14 +91,14 @@ use self::vector::remap_vector_index; use crate::dataset::index::LanceIndexStoreExt; use crate::dataset::optimize::RemappedIndex; use crate::dataset::optimize::remapping::RemapResult; -use crate::dataset::transaction::{Operation, Transaction, TransactionBuilder}; +use crate::dataset::transaction::{Operation, ReadVersionState, Transaction, TransactionBuilder}; pub use crate::index::api::{DatasetIndexExt, IndexSegment, IntoIndexSegment}; use crate::index::frag_reuse::{load_frag_reuse_index_details, open_frag_reuse_index}; use crate::index::mem_wal::open_mem_wal_index; pub use crate::index::prefilter::{FilterLoader, PreFilter}; use crate::index::scalar::{IndexDetails, fetch_index_details, load_training_data}; pub use crate::index::vector::{LogicalIvfView, LogicalVectorIndex}; -use crate::session::index_caches::{FragReuseIndexKey, IndexMetadataKey}; +use crate::session::index_caches::{FragReuseIndexKey, IndexMetadataKey, write_index_identity}; use crate::{Error, Result, dataset::Dataset}; pub use create::CreateIndexBuilder; pub use lance_index::IndexDescription; @@ -130,11 +137,176 @@ fn validate_segment_metadata(index_name: &str, segments: &[IndexMetadata]) -> Re Ok(()) } +fn collect_subtree_field_ids(field: &Field, field_ids: &mut HashSet) { + field_ids.insert(field.id); + for child in &field.children { + collect_subtree_field_ids(child, field_ids); + } +} + +/// Stable identity fields for a physical data file. +/// +/// This mirrors transaction rewrite validation, additionally resolves a +/// registered base to its physical binding, and deliberately excludes +/// `file_size_bytes`, which is a mutable cache rather than file identity. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PhysicalBaseBinding<'a> { + Primary, + Registered { + path: &'a str, + is_dataset_root: bool, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PhysicalDataFileIdentity<'a> { + base_id: Option, + base_binding: PhysicalBaseBinding<'a>, + path: &'a str, + fields: &'a [i32], + column_indices: &'a [i32], + file_major_version: u32, + file_minor_version: u32, +} + +impl<'a> PhysicalDataFileIdentity<'a> { + fn try_new(dataset: &'a Dataset, file: &'a DataFile) -> Option { + let base_binding = match file.base_id { + Some(base_id) => { + let base = dataset.manifest.base_paths.get(&base_id)?; + PhysicalBaseBinding::Registered { + path: &base.path, + is_dataset_root: base.is_dataset_root, + } + } + None => PhysicalBaseBinding::Primary, + }; + Some(Self { + base_id: file.base_id, + base_binding, + path: &file.path, + fields: file.fields.as_ref(), + column_indices: file.column_indices.as_ref(), + file_major_version: file.file_major_version, + file_minor_version: file.file_minor_version, + }) + } +} + +fn fragment_field_files<'a>( + dataset: &'a Dataset, + fragment: &'a Fragment, + indexed_field_ids: &HashSet, +) -> Option>> { + fragment + .files + .iter() + .flat_map(|file| { + file.fields + .iter() + .filter(|field_id| indexed_field_ids.contains(field_id)) + .map(|field_id| { + PhysicalDataFileIdentity::try_new(dataset, file) + .map(|identity| (*field_id, identity)) + }) + }) + .collect() +} + +/// Resolve the field ids a segment's staleness check must consider: the subtree of +/// every field the segment declares, keyed and carried alike (see +/// [`IndexSegment::fields`] and [`IndexSegment::covering_fields`]). A covered +/// segment's carried columns can go stale independently of its keyed column, so +/// checking only the keyed subtree would leave a fragment covered after a carried +/// column was rewritten, and the segment would answer with the obsolete value. +fn segment_indexed_field_ids(dataset: &Dataset, segment: &IndexSegment) -> Result> { + let mut indexed_field_ids = HashSet::new(); + for field_id in segment.fields() { + let field = dataset.schema().field_by_id(*field_id).ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: field id {field_id} does not exist in the current schema" + )) + })?; + collect_subtree_field_ids(field, &mut indexed_field_ids); + } + Ok(indexed_field_ids) +} + +async fn prune_stale_segment_coverage( + dataset: &Dataset, + segments: &mut [IndexSegment], + prune_historically_missing: bool, + prune_newer_overlays: bool, +) -> Result<()> { + let current_fragments = dataset + .fragments() + .iter() + .map(|fragment| (fragment.id as u32, fragment)) + .collect::>(); + let historical_versions = segments + .iter() + .map(IndexSegment::dataset_version) + .filter(|version| *version < dataset.manifest.version) + .collect::>(); + + for version in historical_versions { + let historical = dataset.checkout_version(version).await.map_err(|error| { + Error::invalid_input(format!( + "CreateIndex: cannot validate segment coverage built at dataset version {version}: {error}" + )) + })?; + let historical_fragments = historical + .fragments() + .iter() + .map(|fragment| (fragment.id as u32, fragment)) + .collect::>(); + + for segment in segments + .iter_mut() + .filter(|segment| segment.dataset_version() == version) + { + let indexed_field_ids = segment_indexed_field_ids(dataset, segment)?; + let stale_fragments = segment + .fragment_bitmap() + .iter() + .filter(|fragment_id| { + let Some(historical_fragment) = historical_fragments.get(fragment_id) else { + return prune_historically_missing; + }; + let Some(current_fragment) = current_fragments.get(fragment_id) else { + return true; + }; + let historical_files = + fragment_field_files(&historical, historical_fragment, &indexed_field_ids); + let current_files = + fragment_field_files(dataset, current_fragment, &indexed_field_ids); + let changed_files = + historical_files.is_none() || historical_files != current_files; + let changed_overlays = prune_newer_overlays + && current_fragment.overlays.iter().any(|overlay| { + overlay.committed_version > version + && overlay + .data_file + .fields + .iter() + .any(|field_id| indexed_field_ids.contains(field_id)) + }); + changed_files || changed_overlays + }) + .collect::>(); + for fragment_id in stale_fragments { + segment.fragment_bitmap_mut().remove(fragment_id); + } + } + } + Ok(()) +} + pub(crate) async fn build_index_metadata_from_segments( dataset: &Dataset, index_name: &str, field_id: i32, - segments: Vec, + mut segments: Vec, ) -> Result> { if segments.is_empty() { return Err(Error::invalid_input( @@ -144,7 +316,67 @@ pub(crate) async fn build_index_metadata_from_segments( let mut seen_segment_ids = HashSet::with_capacity(segments.len()); let mut covered_fragments = RoaringBitmap::new(); + // One logical index needs one declaration. The per-segment rules below only + // pin the *keyed* field and its count, so segments that disagree on the + // carried columns -- `fields = [k, a]` beside `fields = [k]`, say -- each pass + // individually. Committing that pair produces an index whose own description + // path refuses it: `IndexDescriptionImpl::try_new` requires `fields` to be + // identical across segments, so `describe_indices` would error on metadata + // this function just wrote. Same rule, and same reason, as + // `merge_existing_index_segments`. + let expected_fields = segments[0].fields().to_vec(); + let expected_covering_fields = segments[0].covering_fields().to_vec(); for segment in &segments { + if segment.dataset_version() > dataset.manifest.version { + return Err(Error::invalid_input(format!( + "CreateIndex: segment {} was built at future dataset version {} (current version {})", + segment.uuid(), + segment.dataset_version(), + dataset.manifest.version + ))); + } + if segment.keyed_field() != Some(field_id) { + return Err(Error::invalid_input(format!( + "CreateIndex: segment {} was built for fields {:?} (carried {:?}), \ + expected keyed field [{}]", + segment.uuid(), + segment.fields(), + segment.covering_fields(), + field_id + ))); + } + // Cardinality alone does not prove `covering_fields` is really the + // trailing slice of `fields`; delegate that to the same rule a + // committed `IndexMetadata` is held to, rather than re-deriving it. + IndexMetadata { + uuid: segment.uuid(), + fields: segment.fields().to_vec(), + covering_fields: segment.covering_fields().to_vec(), + name: index_name.to_string(), + dataset_version: segment.dataset_version(), + fragment_bitmap: None, + index_details: None, + index_version: segment.index_version(), + created_at: None, + base_id: None, + files: None, + } + .validate_covering_fields()?; + if segment.fields() != expected_fields.as_slice() + || segment.covering_fields() != expected_covering_fields.as_slice() + { + return Err(Error::invalid_input(format!( + "CreateIndex: segment {} declares fields {:?} (carried {:?}) but index \ + '{}' declares fields {:?} (carried {:?}); every segment of one index \ + must declare the same columns", + segment.uuid(), + segment.fields(), + segment.covering_fields(), + index_name, + expected_fields, + expected_covering_fields, + ))); + } if !seen_segment_ids.insert(segment.uuid()) { return Err(Error::invalid_input(format!( "CreateIndex: duplicate segment uuid {} for index '{}'", @@ -161,16 +393,26 @@ pub(crate) async fn build_index_metadata_from_segments( covered_fragments |= segment.fragment_bitmap().clone(); } - let mut new_indices = Vec::with_capacity(segments.len()); - for segment in segments { - let (uuid, fragment_bitmap, index_details, index_version) = segment.into_parts(); + prune_stale_segment_coverage(dataset, &mut segments, false, false).await?; + + let new_indices = futures::stream::iter(segments.into_iter().map(|segment| async move { + let ( + uuid, + fragment_bitmap, + fields, + covering_fields, + index_details, + index_version, + dataset_version, + ) = segment.into_parts(); let is_inverted_index = index_details.type_url.ends_with("InvertedIndexDetails"); if is_inverted_index { let metadata = IndexMetadata { uuid, name: index_name.to_string(), - fields: vec![field_id], - dataset_version: dataset.manifest.version, + fields: fields.clone(), + covering_fields: covering_fields.clone(), + dataset_version, fragment_bitmap: Some(fragment_bitmap.clone()), index_details: Some(index_details.clone()), index_version, @@ -181,24 +423,28 @@ pub(crate) async fn build_index_metadata_from_segments( crate::index::scalar::inverted::finalize_segment_files_if_needed(dataset, &metadata) .await?; } - let index_dir = dataset.indices_dir().clone().join(uuid.to_string()); + let index_dir = dataset.indices_dir().join(uuid.to_string()); let mut files = list_index_files_with_sizes(&dataset.object_store, &index_dir).await?; if is_inverted_index { retain_committed_inverted_files(&mut files); } - new_indices.push(IndexMetadata { + Ok::<_, Error>(IndexMetadata { uuid, name: index_name.to_string(), - fields: vec![field_id], - dataset_version: dataset.manifest.version, + fields, + covering_fields, + dataset_version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(index_details), index_version, created_at: Some(chrono::Utc::now()), base_id: None, files: Some(files), - }); - } + }) + })) + .buffered(dataset.object_store.io_parallelism()) + .try_collect::>() + .await?; Ok(new_indices) } @@ -207,6 +453,362 @@ fn retain_committed_inverted_files(files: &mut Vec) { files.retain(|file| !file.path.starts_with("staging/")); } +async fn prewarm_opened_index( + index: Arc, + options: Option<&PrewarmOptions>, +) -> Result<()> { + match options { + None => index.prewarm().await, + Some(PrewarmOptions::Fts(fts_options)) => { + let inverted = index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input(format!( + "FTS prewarm options are only supported for inverted indices, got {:?}", + index.index_type() + )) + })?; + inverted.prewarm_with_options(fts_options).await + } + Some(_) => Err(Error::not_supported( + "unsupported prewarm options for this lance version".to_owned(), + )), + } +} + +struct SegmentPrewarmResult { + partition_count: usize, +} + +struct OpenedSegmentPrewarmResult { + index_uuid: Uuid, + index: Arc, + partition_count: usize, +} + +async fn prewarm_opened_index_result( + index: Arc, + options: &PrewarmOptions, +) -> Result { + match options { + PrewarmOptions::Fts(fts_options) => { + let inverted = index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::invalid_input(format!( + "FTS prewarm options are only supported for inverted indices, got {:?}", + index.index_type() + )) + })?; + inverted.prewarm_with_options_result(fts_options).await?; + Ok(SegmentPrewarmResult { + partition_count: inverted.partition_count(), + }) + } + _ => Err(Error::not_supported( + "unsupported prewarm options for this lance version".to_owned(), + )), + } +} + +async fn aggregate_fts_prewarm_results( + dataset: &Dataset, + segment_results: Vec, + options: Option<&PrewarmOptions>, +) -> Result { + let partition_count = segment_results + .iter() + .map(|result| result.partition_count) + .sum(); + let mut failing_segments = Vec::new(); + let mut failing_partitions = Vec::new(); + let mut best_effort = false; + + for segment in segment_results { + let segment_id = segment.index_uuid.to_string(); + let prewarmed_inverted = segment.index.as_any().downcast_ref::(); + let Some(fts_options) = (match options { + Some(PrewarmOptions::Fts(fts_options)) => Some(fts_options.clone()), + None if prewarmed_inverted.is_some() => Some(FtsPrewarmOptions::default()), + _ => None, + }) else { + continue; + }; + best_effort |= fts_options.mode.is_best_effort(); + + let cached_index = + scalar::cached_scalar_index_container(dataset, &segment.index_uuid).await; + let cached_inverted = cached_index + .as_ref() + .and_then(|index| index.as_any().downcast_ref::()); + let container_resident = cached_inverted.is_some(); + let container_matches_prewarmed = match (prewarmed_inverted, cached_inverted) { + (Some(prewarmed), Some(cached)) => std::ptr::addr_eq(prewarmed, cached), + _ => false, + }; + + if !container_resident || !container_matches_prewarmed { + failing_segments.push(FtsPrewarmSegmentStatus { + segment_id: segment_id.clone(), + scalar_index_container_resident: container_resident, + scalar_index_container_matches_prewarmed: container_matches_prewarmed, + }); + } + + if let Some(cached_inverted) = cached_inverted + && let Some(mut diagnostics) = cached_inverted + .prewarm_residency_result(fts_options.with_position) + .await + .diagnostics + { + failing_segments.append(&mut diagnostics.failing_segments); + failing_partitions.extend(diagnostics.failing_partitions.drain(..).map( + |mut partition| { + partition.segment_id = Some(segment_id.clone()); + partition + }, + )); + } + } + + let diagnostics = FtsPrewarmDiagnostics { + partition_count, + failing_segments, + failing_partitions, + }; + + if diagnostics.fully_resident() { + Ok(FtsPrewarmResult::fully_resident()) + } else if best_effort { + Ok(FtsPrewarmResult::partial(diagnostics)) + } else { + Err(Error::internal(diagnostics.to_string())) + } +} + +fn total_index_segment_size_bytes(indices: &[IndexMetadata]) -> Option { + let mut total = 0u64; + for index_meta in indices { + total += index_meta.total_size_bytes()?; + } + Some(total) +} + +fn cache_size_delta(after: usize, before: usize) -> i64 { + after.saturating_sub(before) as i64 - before.saturating_sub(after) as i64 +} + +fn prewarm_options_fields(options: Option<&PrewarmOptions>) -> (&'static str, bool) { + match options { + None => ("default", false), + Some(PrewarmOptions::Fts(fts_options)) => ("fts", fts_options.with_position), + Some(_) => ("unsupported", false), + } +} + +async fn prewarm_index_segments_by_metadata( + dataset: &Dataset, + name: &str, + indices: Vec, + options: Option<&PrewarmOptions>, + available_segment_count: usize, + requested_segment_count: Option, +) -> Result { + let request_started = Instant::now(); + let selected_segment_count = indices.len(); + let selected_size_bytes = total_index_segment_size_bytes(&indices); + let (prewarm_options, fts_with_position) = prewarm_options_fields(options); + let cache_stats_before = dataset.session.index_cache_stats().await; + info!( + index_name = name, + selected_segment_count, + available_segment_count, + requested_segment_count = requested_segment_count.unwrap_or(0), + segment_filter = requested_segment_count.is_some(), + selected_size_bytes = selected_size_bytes.unwrap_or(0), + selected_size_bytes_known = selected_size_bytes.is_some(), + index_cache_entries_before = cache_stats_before.num_entries, + index_cache_size_bytes_before = cache_stats_before.size_bytes, + prewarm_options, + fts_with_position, + "prewarm index segments started" + ); + + let result = futures::future::try_join_all(indices.into_iter().map(|index_meta| async move { + let index_uuid = index_meta.uuid; + let size_bytes = index_meta.total_size_bytes(); + let fragment_count = index_meta + .fragment_bitmap + .as_ref() + .map(|bitmap| bitmap.len()); + let segment_started = Instant::now(); + info!( + index_name = name, + %index_uuid, + index_version = index_meta.index_version, + dataset_version = index_meta.dataset_version, + fragment_count = fragment_count.unwrap_or(0), + fragment_count_known = fragment_count.is_some(), + size_bytes = size_bytes.unwrap_or(0), + size_bytes_known = size_bytes.is_some(), + "prewarm index segment started" + ); + + let index = match dataset + .open_generic_index(name, &index_uuid, &NoOpMetricsCollector) + .await + { + Ok(index) => { + info!( + index_name = name, + %index_uuid, + index_type = ?index.index_type(), + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "opened index segment for prewarm" + ); + index + } + Err(err) => { + warn!( + index_name = name, + %index_uuid, + error = %err, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "failed to open index segment for prewarm" + ); + return Err(err); + } + }; + + let segment_result = match options { + Some(options) => match prewarm_opened_index_result(index.clone(), options).await { + Ok(result) => result, + Err(err) => { + warn!( + index_name = name, + %index_uuid, + error = %err, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "prewarm index segment failed" + ); + return Err(err); + } + }, + None => { + if let Err(err) = prewarm_opened_index(index.clone(), None).await { + warn!( + index_name = name, + %index_uuid, + error = %err, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "prewarm index segment failed" + ); + return Err(err); + } + SegmentPrewarmResult { partition_count: 0 } + } + }; + + info!( + index_name = name, + %index_uuid, + elapsed_ms = segment_started.elapsed().as_millis() as u64, + "prewarm index segment finished" + ); + Ok(OpenedSegmentPrewarmResult { + index_uuid, + index, + partition_count: segment_result.partition_count, + }) + })) + .await; + + match result { + Ok(segment_results) => { + let cache_stats_after = dataset.session.index_cache_stats().await; + let result = aggregate_fts_prewarm_results(dataset, segment_results, options).await?; + info!( + index_name = name, + selected_segment_count, + fts_fully_resident = result.fully_resident, + index_cache_entries_after = cache_stats_after.num_entries, + index_cache_entries_delta = cache_size_delta( + cache_stats_after.num_entries, + cache_stats_before.num_entries, + ), + index_cache_size_bytes_after = cache_stats_after.size_bytes, + index_cache_size_bytes_delta = + cache_size_delta(cache_stats_after.size_bytes, cache_stats_before.size_bytes,), + elapsed_ms = request_started.elapsed().as_millis() as u64, + "prewarm index segments finished" + ); + Ok(result) + } + Err(err) => { + let cache_stats_after = dataset.session.index_cache_stats().await; + warn!( + index_name = name, + selected_segment_count, + index_cache_entries_after = cache_stats_after.num_entries, + index_cache_entries_delta = cache_size_delta( + cache_stats_after.num_entries, + cache_stats_before.num_entries, + ), + index_cache_size_bytes_after = cache_stats_after.size_bytes, + index_cache_size_bytes_delta = cache_size_delta( + cache_stats_after.size_bytes, + cache_stats_before.size_bytes, + ), + error = %err, + elapsed_ms = request_started.elapsed().as_millis() as u64, + "prewarm index segments failed" + ); + Err(err) + } + } +} + +fn filter_index_segments_by_ids( + name: &str, + indices: Vec, + segment_ids: &[Uuid], +) -> Result> { + if segment_ids.is_empty() { + return Ok(Vec::new()); + } + + let requested = segment_ids.iter().copied().collect::>(); + let mut matched = HashSet::new(); + let filtered = indices + .into_iter() + .filter(|index_meta| { + if requested.contains(&index_meta.uuid) { + matched.insert(index_meta.uuid); + true + } else { + false + } + }) + .collect::>(); + + if matched.len() != requested.len() { + let mut missing = requested + .difference(&matched) + .map(ToString::to_string) + .collect::>(); + missing.sort(); + return Err(Error::index_not_found(format!( + "name={}, segment_ids=[{}]", + name, + missing.join(", ") + ))); + } + + Ok(filtered) +} + fn validate_segment_index_details(index_name: &str, segments: &[IndexMetadata]) -> Result<()> { let mut type_url = None::<&str>; for segment in segments { @@ -247,6 +849,18 @@ fn segment_has_vector_details(segment: &IndexMetadata) -> bool { ) } +/// Whether this build has a reader for the index's declared type. +/// +/// Segments without details predate type URLs and remain readable through the +/// legacy file-based detection in the index open paths. +pub(crate) fn index_type_is_known(index: &IndexMetadata) -> bool { + is_system_index(index) + || index + .index_details + .as_ref() + .is_none_or(|details| IndexDetails(details.clone()).has_reader()) +} + /// Detect FTS / inverted segments from manifest details. /// /// Unlike vector, inverted segment support was added after index details were @@ -265,6 +879,13 @@ fn segment_has_bitmap_details(segment: &IndexMetadata) -> bool { .is_some_and(|details| details.type_url.ends_with("BitmapIndexDetails")) } +fn segment_has_bloomfilter_details(segment: &IndexMetadata) -> bool { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("BloomFilterIndexDetails")) +} + /// Detect BTree segments, preserving a legacy pre-details fallback. fn segment_has_btree_details(segment: &IndexMetadata) -> bool { segment.index_details.as_ref().map_or_else( @@ -299,6 +920,20 @@ fn segment_has_label_list_details(segment: &IndexMetadata) -> bool { .is_some_and(|details| details.type_url.ends_with("LabelListIndexDetails")) } +fn segment_has_rtree_details(segment: &IndexMetadata) -> bool { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("RTreeIndexDetails")) +} + +fn segment_has_ngram_details(segment: &IndexMetadata) -> bool { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("NGramIndexDetails")) +} + // Cache keys for different index types #[derive(Debug, Clone)] pub(crate) struct LegacyVectorIndexCacheKey<'a> { @@ -326,6 +961,14 @@ impl CacheKey for LegacyVectorIndexCacheKey<'_> { fn type_name() -> &'static str { "LegacyVectorIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.legacy-vector-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } } /// Sized cache key for `IvfIndexState`. @@ -333,6 +976,11 @@ impl CacheKey for LegacyVectorIndexCacheKey<'_> { /// Used for v0.3+ indices that support serialization. This key has a codec, /// so custom cache backends can serialize the state to disk/Redis/etc. /// Legacy indices use `LegacyVectorIndexCacheKey` instead (in-memory only). +/// +/// Note: legacy entries hold live readers bound to the object store that +/// opened them, so they are only valid for credential setups that refresh +/// internally (e.g. a credentials provider). Deployments that pass fresh +/// static credentials per dataset open should use v0.3+ index formats. #[derive(Debug, Clone)] pub(crate) struct IvfIndexStateCacheKey<'a> { uuid: &'a Uuid, @@ -360,6 +1008,14 @@ impl CacheKey for IvfIndexStateCacheKey<'_> { } } + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.ivf-state-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } + fn codec() -> Option { Some(lance_core::cache::CacheCodec::from_impl::()) } @@ -383,7 +1039,7 @@ impl<'a> FragReuseIndexCacheKey<'a> { } impl CacheKey for FragReuseIndexCacheKey<'_> { - type ValueType = FragReuseIndex; + type ValueType = CompactFragReuseIndex; fn key(&self) -> std::borrow::Cow<'_, str> { if let Some(fri_uuid) = self.fri_uuid { @@ -396,6 +1052,14 @@ impl CacheKey for FragReuseIndexCacheKey<'_> { fn type_name() -> &'static str { "FragReuseIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.fragment-reuse-cache-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } } #[derive(Debug, Clone)] @@ -424,6 +1088,14 @@ impl CacheKey for MemWalCacheKey<'_> { fn type_name() -> &'static str { "MemWalIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.mem-wal-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + write_index_identity(builder, self.uuid, self.fri_uuid); + } } // Whether to auto-migrate a dataset when we encounter corruption. @@ -493,10 +1165,40 @@ pub(crate) async fn remap_index( .find(|i| i.uuid == *index_id) .ok_or_else(|| Error::index(format!("Index with id {} does not exist", index_id)))?; + // Corrupt metadata fails closed before anything else: a declaration that is + // not a valid suffix of `fields` cannot be reasoned about at all, and the + // withdrawal below would otherwise swallow it as an ordinary covered index. + matched.validate_covering_fields()?; + + // A covered index cannot survive a remap yet. Nothing writes the carried + // values in the first place, and each index type's `remap` rewrites only the + // schema that type knows about, so the remapped segment would still declare + // payload its storage no longer holds. Withdraw it instead, exactly as an + // index whose type reports `can_remap() == false` is withdrawn below: an + // absent index costs a fallback scan, whereas a surviving false declaration + // is answered from data that is not there. Erroring here would instead block + // compaction of the whole table. + if !matched.covering_fields.is_empty() { + log::warn!( + "Index '{}' declares covering fields {:?}, which no index builder \ + writes or preserves yet. Index will be dropped during compaction \ + and must be rebuilt.", + matched.name, + matched.covering_fields, + ); + return Ok(RemapResult::Drop); + } + + // With covering withdrawn above, `fields` is entirely keyed, so more than one + // entry means a genuinely composite index. if matched.fields.len() > 1 { - return Err(Error::index( - "Remapping indices with multiple fields is not supported".to_string(), - )); + return Err(Error::index(format!( + "Remapping index '{}' is not supported: it has {} keyed fields {:?}; \ + only one keyed field is supported", + matched.name, + matched.fields.len(), + matched.fields, + ))); } if let Some(deleted_bitmap) = row_id_map.fully_deleted_fragments() @@ -605,7 +1307,7 @@ pub(crate) async fn remap_index( ) .unwrap(), index_version, - files, + files: table_files_to_index(files), } } _ => { @@ -621,7 +1323,7 @@ pub(crate) async fn remap_index( new_id, index_details: created_index.index_details, index_version: created_index.index_version, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), })) } @@ -720,7 +1422,17 @@ impl IndexDescriptionImpl { "Index fields should be identical across all segments".to_string(), )); } - let field_ids_vec: Vec = field_ids.iter().map(|id| *id as u32).collect(); + // Only the keyed prefix. `fields` also lists the columns the index merely + // carries values for, which it cannot be searched on, and this list is what + // every binding advertises as the index's columns -- Python's + // `_default_vector_index_for_column` matches on membership of it, so + // including a carried column here would hand back the keyed column's IVF + // model for a query about the carried one. + let field_ids_vec: Vec = example_metadata + .keyed_fields() + .iter() + .map(|id| *id as u32) + .collect(); // Index details may be absent on indices created before details were // persisted in the manifest. We describe such indices on a best-effort @@ -781,10 +1493,15 @@ impl IndexDescriptionImpl { let mut missing_fragment_refs = 0u64; for shard in &segments { - let fragment_bitmap = shard - .fragment_bitmap - .as_ref() - .ok_or_else(|| Error::index("Fragment bitmap is required for index description. This index must be retrained to support this method.".to_string()))?; + let Some(fragment_bitmap) = shard.fragment_bitmap.as_ref() else { + // A system index (e.g. __mem_wal) indexes no fragments, so a + // missing bitmap means zero indexed rows. For a data index it + // means unknown coverage — reject rather than fabricate a count. + if is_system_index(shard) { + continue; + } + return Err(Error::index("Fragment bitmap is required for index description. This index must be retrained to support this method.".to_string())); + }; indexed_fragment_refs += fragment_bitmap.len(); for fragment_id in fragment_bitmap.iter() { @@ -879,6 +1596,46 @@ impl IndexDescription for IndexDescriptionImpl { } } +impl Dataset { + /// Whether an otherwise empty commit would record a new MemWAL catch-up + /// position. + /// + /// Dry-runs the derivation rather than restating its conditions: a second + /// copy of "is this index behind" would be one more place to keep in step + /// with the real rule. A no-work optimize publishes no new segment, so the + /// index list it would commit is the one already loaded, and the version it + /// would read is the current one -- which makes the speculative answer the + /// same one the commit reaches. + fn mem_wal_catch_up_would_advance(&self, indices: &[IndexMetadata]) -> Result { + if !indices.iter().any(|index| index.name == MEM_WAL_INDEX_NAME) { + return Ok(false); + } + let catchup_of = |indices: &[IndexMetadata]| -> Result>> { + indices + .iter() + .find(|index| index.name == MEM_WAL_INDEX_NAME) + .cloned() + .map(|index| { + crate::index::mem_wal::load_mem_wal_index_details(index) + .map(|details| details.index_catchup) + }) + .transpose() + }; + + let mut speculative = indices.to_vec(); + Transaction::apply_mem_wal_index_coverage( + &mut speculative, + &Transaction::logical_index_segments(indices), + Some(ReadVersionState { + manifest: &self.manifest, + indices, + }), + self.manifest.version + 1, + )?; + Ok(catchup_of(&speculative)? != catchup_of(indices)?) + } +} + #[async_trait] impl DatasetIndexExt for Dataset { type IndexBuilder<'a> = CreateIndexBuilder<'a>; @@ -948,7 +1705,9 @@ impl DatasetIndexExt for Dataset { } async fn drop_index(&mut self, name: &str) -> Result<()> { - let indices = self.load_indices_by_name(name).await?; + // Removal never opens the index, so an index this build cannot read is + // still droppable - and has to be, since it is otherwise unremovable. + let indices = load_all_indices_by_name(self, name).await?; if indices.is_empty() { return Err(Error::index_not_found(format!("name={}", name))); } @@ -974,49 +1733,93 @@ impl DatasetIndexExt for Dataset { return Err(Error::index_not_found(format!("name={}", name))); } - for index_meta in indices { - let index = self - .open_generic_index(name, &index_meta.uuid, &NoOpMetricsCollector) - .await?; - index.prewarm().await?; - } - - Ok(()) + let available_segment_count = indices.len(); + prewarm_index_segments_by_metadata(self, name, indices, None, available_segment_count, None) + .await + .map(|_| ()) } async fn prewarm_index_with_options(&self, name: &str, options: &PrewarmOptions) -> Result<()> { + self.prewarm_index_with_options_result(name, options) + .await + .map(|_| ()) + } + + async fn prewarm_index_with_options_result( + &self, + name: &str, + options: &PrewarmOptions, + ) -> Result { let indices = self.load_indices_by_name(name).await?; if indices.is_empty() { return Err(Error::index_not_found(format!("name={}", name))); } - for index_meta in indices { - let index = self - .open_generic_index(name, &index_meta.uuid, &NoOpMetricsCollector) - .await?; + let available_segment_count = indices.len(); + prewarm_index_segments_by_metadata( + self, + name, + indices, + Some(options), + available_segment_count, + None, + ) + .await + } - match options { - PrewarmOptions::Fts(fts_options) => { - let inverted = index - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::invalid_input(format!( - "FTS prewarm options are only supported for inverted indices, got {:?}", - index.index_type() - )) - })?; - inverted.prewarm_with_options(fts_options).await?; - } - _ => { - return Err(Error::not_supported( - "unsupported prewarm options for this lance version".to_owned(), - )); - } - } + async fn prewarm_index_segments(&self, name: &str, segment_ids: &[Uuid]) -> Result<()> { + let indices = self.load_indices_by_name(name).await?; + if indices.is_empty() { + return Err(Error::index_not_found(format!("name={}", name))); } + let available_segment_count = indices.len(); + let indices = filter_index_segments_by_ids(name, indices, segment_ids)?; - Ok(()) + prewarm_index_segments_by_metadata( + self, + name, + indices, + None, + available_segment_count, + Some(segment_ids.len()), + ) + .await + .map(|_| ()) + } + + async fn prewarm_index_segments_with_options( + &self, + name: &str, + segment_ids: &[Uuid], + options: &PrewarmOptions, + ) -> Result<()> { + self.prewarm_index_segments_with_options_result(name, segment_ids, options) + .await + .map(|_| ()) + } + + async fn prewarm_index_segments_with_options_result( + &self, + name: &str, + segment_ids: &[Uuid], + options: &PrewarmOptions, + ) -> Result { + let indices = self.load_indices_by_name(name).await?; + if indices.is_empty() { + return Err(Error::index_not_found(format!("name={}", name))); + } + let available_segment_count = indices.len(); + let indices = filter_index_segments_by_ids(name, indices, segment_ids)?; + + prewarm_index_segments_by_metadata( + self, + name, + indices, + Some(options), + available_segment_count, + Some(segment_ids.len()), + ) + .await } async fn describe_indices<'a, 'b>( @@ -1030,8 +1833,13 @@ impl DatasetIndexExt for Dataset { log::warn!("The method describe_indices does not support indexes without index details. Please retrain the index {}", idx.name); return false; } + // Only the keyed prefix. `index_matches_criteria` compares this + // slice against the single column named by `for_column`, so + // passing the carried columns too would push its length past one + // and silently drop every covered index from a filtered + // describe. let fields = idx - .fields + .keyed_fields() .iter() .filter_map(|id| self.schema().field_by_id(*id)) .collect::>(); @@ -1064,100 +1872,86 @@ impl DatasetIndexExt for Dataset { } async fn load_indices(&self) -> Result>> { - let metadata_key = IndexMetadataKey { - version: self.version().version, - }; - let mut indices = match self.index_cache.get_with_key(&metadata_key).await { - Some(indices) => indices, - None => { - let mut loaded_indices = read_manifest_indexes( - &self.object_store, - &self.manifest_location, - &self.manifest, - ) - .await?; - retain_supported_indices(&mut loaded_indices); - let loaded_indices = Arc::new(loaded_indices); - self.index_cache - .insert_with_key(&metadata_key, loaded_indices.clone()) - .await; - loaded_indices - } - }; - - // Infer details for legacy vector indices (once per index name, concurrently). - // This may run on indices that were opportunistically cached during Dataset::open - // before the full Dataset was available for inference. - { - let mut updated = indices.as_ref().clone(); - infer_missing_vector_details(self, &mut updated).await; - if updated != *indices { - indices = Arc::new(updated); - self.index_cache - .insert_with_key(&metadata_key, indices.clone()) - .await; - } - } - - if let Some(frag_reuse_index_meta) = - indices.iter().find(|idx| idx.name == FRAG_REUSE_INDEX_NAME) - { - let fri_key = FragReuseIndexKey { - uuid: &frag_reuse_index_meta.uuid, - }; - let frag_reuse_index = self - .index_cache - .get_or_insert_with_key(fri_key, || async move { - let index_details = - load_frag_reuse_index_details(self, frag_reuse_index_meta).await?; - open_frag_reuse_index(frag_reuse_index_meta.uuid, index_details.as_ref()).await - }) - .await?; - let mut indices = indices.as_ref().clone(); - for idx in indices.iter_mut() { - if let Some(bitmap) = idx.fragment_bitmap.as_mut() { - frag_reuse_index.remap_fragment_bitmap(bitmap)?; - } - } - Ok(Arc::new(indices)) - } else { - Ok(indices) + let indices = load_all_indices(self).await?; + if indices.iter().all(index_is_usable) { + return Ok(indices); } + Ok(Arc::new( + indices + .iter() + .filter(|idx| index_is_usable(idx)) + .cloned() + .collect(), + )) } async fn merge_existing_index_segments( &self, - source_segments: Vec, + mut source_segments: Vec, ) -> Result { validate_segment_metadata("uncommitted", &source_segments)?; + if let Some(segment) = source_segments + .iter() + .find(|segment| segment.dataset_version > self.manifest.version) + { + return Err(Error::invalid_input(format!( + "merge_existing_index_segments: segment {} was built at future dataset version {} (current version {})", + segment.uuid, segment.dataset_version, self.manifest.version + ))); + } + let source_dataset_version = source_segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(self.manifest.version); let field_id = *source_segments[0].fields.first().ok_or_else(|| { Error::invalid_input(format!( "CreateIndex: segment {} is missing field ids", source_segments[0].uuid )) })?; - if source_segments - .iter() - .any(|segment| segment.fields != [field_id]) - { - return Err(Error::invalid_input( - "merge_existing_index_segments requires segments with identical fields".to_string(), - )); + let expected_covering_fields = source_segments[0].covering_fields.clone(); + for segment in &source_segments { + // Same rule as `build_index_metadata_from_segments`: only the + // keyed-field count matters here, not the exact `fields` vector, + // so a covered segment is not rejected outright. + if segment.keyed_field() != Some(field_id) { + return Err(Error::invalid_input(format!( + "merge_existing_index_segments: segment {} was built for fields {:?} \ + (carried {:?}), expected keyed field [{}]", + segment.uuid, segment.fields, segment.covering_fields, field_id + ))); + } + segment.validate_covering_fields()?; + // Merging requires one coherent output declaration, so every + // input must carry the same columns (not merely the same count). + if segment.covering_fields != expected_covering_fields { + return Err(Error::invalid_input( + "merge_existing_index_segments requires segments with identical fields" + .to_string(), + )); + } } let all_vector = source_segments.iter().all(segment_has_vector_details); let all_inverted = source_segments.iter().all(segment_has_inverted_details); let all_bitmap = source_segments.iter().all(segment_has_bitmap_details); + let all_bloomfilter = source_segments.iter().all(segment_has_bloomfilter_details); let all_btree = source_segments.iter().all(segment_has_btree_details); let all_fmindex = source_segments.iter().all(segment_has_fmindex_details); let all_zonemap = source_segments.iter().all(segment_has_zonemap_details); let all_label_list = source_segments.iter().all(segment_has_label_list_details); + let all_rtree = source_segments.iter().all(segment_has_rtree_details); + let all_ngram = source_segments.iter().all(segment_has_ngram_details); if !all_vector && !all_inverted && !all_bitmap + && !all_bloomfilter && !all_btree && !all_fmindex && !all_zonemap && !all_label_list + && !all_rtree + && !all_ngram { return Err(Error::invalid_input( "merge_existing_index_segments requires all segments to have the same supported index type" @@ -1165,28 +1959,52 @@ impl DatasetIndexExt for Dataset { )); } + let merged_dataset_version = if all_rtree { + let mut source_coverage = source_segments + .iter() + .cloned() + .map(IntoIndexSegment::into_index_segment) + .collect::>>()?; + prune_stale_segment_coverage(self, &mut source_coverage, true, true).await?; + for (source, coverage) in source_segments.iter_mut().zip(source_coverage) { + source.fragment_bitmap = Some(coverage.fragment_bitmap().clone()); + } + self.manifest.version + } else { + source_dataset_version + }; + let mut merged_segment = if all_vector { - crate::index::vector::ivf::merge_segments( - self.object_store.as_ref(), - &self.indices_dir(), - source_segments, - ) - .await? + crate::index::vector::ivf::merge_segments(self, source_segments).await? } else if all_inverted { crate::index::scalar::inverted::merge_segments(self, source_segments).await? } else if all_fmindex { crate::index::scalar::fmindex::merge_segments(self, source_segments).await? } else if all_bitmap { crate::index::scalar::bitmap::merge_segments(self, source_segments).await? + } else if all_bloomfilter { + crate::index::scalar::bloomfilter::merge_segments(self, source_segments).await? } else if all_label_list { crate::index::scalar::label_list::merge_segments(self, source_segments).await? } else if all_zonemap { crate::index::scalar::zonemap::merge_segments(self, source_segments).await? + } else if all_ngram { + crate::index::scalar::ngram::merge_segments(self, source_segments).await? + } else if all_rtree { + #[cfg(feature = "geo")] + { + crate::index::scalar::rtree::merge_segments(self, source_segments).await? + } + #[cfg(not(feature = "geo"))] + return Err(Error::not_supported( + "RTree segment merge requires the `geo` feature".to_string(), + )); } else { crate::index::scalar::btree::merge_segments(self, source_segments).await? }; - merged_segment.dataset_version = self.manifest.version; - merged_segment.fields = vec![field_id]; + if !all_ngram && !all_fmindex { + merged_segment.dataset_version = merged_dataset_version; + } Ok(merged_segment) } @@ -1206,6 +2024,34 @@ impl DatasetIndexExt for Dataset { .into_iter() .map(IntoIndexSegment::into_index_segment) .collect::>>()?; + let dataset_fragments = self.fragment_bitmap.as_ref().clone(); + if segments.first().is_some_and(|segment| { + segment + .index_details() + .type_url + .ends_with("NGramIndexDetails") + }) { + let has_retired_coverage = segments + .iter() + .any(|segment| !(segment.fragment_bitmap() - &dataset_fragments).is_empty()); + let frag_reuse_index = self.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let requires_rebuild = frag_reuse_index.as_ref().is_some_and(|frag_reuse_index| { + segments.iter().any(|segment| { + append::fragment_reuse_affects_segment( + frag_reuse_index, + segment.fragment_bitmap(), + segment.dataset_version(), + ) + }) + }); + if has_retired_coverage || requires_rebuild { + return Err(Error::invalid_input( + "CreateIndex: NGram segments built before compaction must be rebuilt or merged \ + with merge_existing_index_segments before commit" + .to_string(), + )); + } + } let new_indices = build_index_metadata_from_segments(self, index_name, field.id, segments).await?; validate_segment_metadata(index_name, &new_indices)?; @@ -1215,39 +2061,74 @@ impl DatasetIndexExt for Dataset { .index_details .as_ref() .map(|details| details.type_url.clone()); - let dataset_fragments = self.fragment_bitmap.as_ref().clone(); let mut incoming_fragments = RoaringBitmap::new(); for segment in &new_indices { - if segment.fields != [field.id] { + // Mirrors `build_index_metadata_from_segments`'s guard, which + // already validated these exact segments. Kept in the same shape + // so this second look does not silently re-reject what that guard + // just accepted. + if segment.keyed_field() != Some(field.id) { return Err(Error::invalid_input(format!( - "CreateIndex: segment {} was built for fields {:?}, expected [{}]", - segment.uuid, segment.fields, field.id + "CreateIndex: segment {} was built for fields {:?} (carried {:?}), \ + expected keyed field [{}]", + segment.uuid, segment.fields, segment.covering_fields, field.id ))); } + segment.validate_covering_fields()?; if let Some(fragment_bitmap) = &segment.fragment_bitmap { incoming_fragments |= fragment_bitmap.clone(); } } - let existing_named_indices = self.load_indices_by_name(index_name).await?; - if existing_named_indices - .iter() - .any(|idx| idx.fields != [field.id]) - { + let existing_named_indices = load_all_indices_by_name(self, index_name).await?; + if existing_named_indices.iter().any(|idx| { + // Same name-collision rule as `CreateIndexBuilder`'s default-name + // loop in create.rs. + idx.keyed_field() != Some(field.id) + }) { return Err(Error::index(format!( "Index name '{index_name}' already exists with different fields, \ please specify a different name" ))); } + let existing_different_type_url = existing_named_indices.iter().find_map(|idx| { + // Legacy metadata may omit index details. Its type cannot be proven + // compatible, so only a full replacement is safe. + let Some(existing_details) = idx.index_details.as_ref() else { + return Some("".to_owned()); + }; + let existing_type_url = existing_details.type_url.as_str(); + (Some(existing_type_url) != incoming_type_url.as_deref()) + .then(|| existing_type_url.to_owned()) + }); + let missing_fragments = &dataset_fragments - &incoming_fragments; + if let Some(existing_type_url) = &existing_different_type_url + && !missing_fragments.is_empty() + { + return Err(Error::invalid_input(format!( + "CreateIndex: cannot change index '{}' from type '{}' to type '{}' with partial fragment coverage; incoming segments are missing current fragments {:?}", + index_name, + existing_type_url, + incoming_type_url.as_deref().unwrap_or(""), + missing_fragments.iter().collect::>() + ))); + } + + let is_index_type_change = existing_different_type_url.is_some(); + // What a retained sibling has to agree with. Every incoming segment + // already carries the same pair: `build_index_metadata_from_segments` + // compares them against each other before this point. + let expected_fields = new_indices[0].fields.clone(); + let expected_covering_fields = new_indices[0].covering_fields.clone(); let removed_indices = existing_named_indices .into_iter() - .filter(|idx| { - idx.index_details - .as_ref() - .zip(incoming_type_url.as_deref()) - .is_none_or(|(details, expected)| details.type_url == expected) - }) .map(|idx| -> Result> { + // A logical index cannot combine segment types. Full current-fragment + // coverage was verified above, so a type change replaces every segment. + if is_index_type_change { + return Ok(Some(idx)); + } + let Some(existing_fragments) = idx.effective_fragment_bitmap(&dataset_fragments) else { if incoming_fragments != dataset_fragments { @@ -1267,6 +2148,28 @@ impl DatasetIndexExt for Dataset { } if existing_fragments.is_disjoint(&incoming_fragments) { + // Retained, so its declaration outlives this commit and has + // to match what is being written: `IndexDescriptionImpl::try_new` + // requires `fields` to be identical across the segments of one + // logical index. Nothing above catches a disagreement, because + // `keyed_fields` is the prefix left after the carried ones -- a + // segment carrying columns keys on exactly what a plain one + // keys on, and both pass the keyed-field guard. + if idx.fields != expected_fields + || idx.covering_fields != expected_covering_fields + { + return Err(Error::invalid_input(format!( + "CreateIndex: incoming segments for '{}' declare fields {:?} and covering_fields {:?}, \ + but retained segment {} declares fields {:?} and covering_fields {:?}; \ + a logical index cannot mix declarations - rebuild every segment in one commit", + index_name, + expected_fields, + expected_covering_fields, + idx.uuid, + idx.fields, + idx.covering_fields + ))); + } return Ok(None); } @@ -1367,9 +2270,14 @@ impl DatasetIndexExt for Dataset { } #[instrument(skip_all)] + async fn optimize_indices(&mut self, options: &OptimizeOptions) -> Result<()> { let dataset = Arc::new(self.clone()); - let indices = self.load_indices().await?; + // Grouped from the complete list so a name's segments are all accounted + // for. A segment this build cannot read is still coverage, and merging + // against a group whose coverage is only partly visible would commit a + // new segment claiming fragments an existing one already holds. + let indices = load_all_indices(self).await?; let indices_to_optimize = options .index_names @@ -1388,7 +2296,73 @@ impl DatasetIndexExt for Dataset { let mut new_indices = vec![]; let mut removed_indices = vec![]; - for deltas in name_to_indices.values() { + for (name, deltas) in name_to_indices.iter() { + if let Some(index) = deltas.iter().find(|idx| !index_type_is_known(idx)) { + let type_url = index + .index_details + .as_ref() + .map(|details| details.type_url.as_str()) + .unwrap_or(""); + log::warn!( + "Skipping optimization of index '{}' because this build does not recognize index type '{}'", + index.name, + type_url + ); + continue; + } + + // Optimizing a covered index would republish its declaration on a + // segment rebuilt without the carried values: `scan_vector_fragments` + // projects the keyed field and `_rowid` only, and the scalar merges + // reconstruct value plus row id. + // + // What decides is the caller's intent, not whether this group is + // stale. An unfiltered `optimize_indices()` is a table-wide + // maintenance request, and erroring aborts the loop before the + // replacements accumulated for the other groups are committed -- so + // one index this build cannot rebuild would leave every other index + // on the table stale. Skip it with a warning instead. + // + // A caller that listed this index in `index_names` asked for it + // specifically, so refuse out loud. The loop is already filtered by + // that list, so reaching here with it set means this group was named. + if let Some(covered) = deltas + .iter() + .find(|index| !index.covering_fields.is_empty()) + { + if options.index_names.is_none() { + log::warn!( + "Skipping index '{}': it declares covering fields {:?}, \ + which no index builder writes or preserves yet.", + covered.name, + covered.covering_fields, + ); + continue; + } + return Err(Error::index(format!( + "Optimizing index '{}' is not supported: it declares \ + covering fields {:?}, which no index builder writes or \ + preserves yet", + covered.name, covered.covering_fields, + ))); + } + + // Optimizing a name means replacing its segments with one that + // covers their union, which this build cannot compute when it + // cannot read one of them: the merged segment would overlap the + // segment left behind, and `Dataset::validate` calls that + // corruption. Leave the whole name to a build that can read it. + if let Some(max_supported_version) = + deltas.iter().find_map(|idx| unsupported_index_version(idx)) + { + log::warn!( + "Index {} has a segment newer than version {}, which this build cannot read; \ + skipping its optimization", + name, + max_supported_version, + ); + continue; + } // Scalar indices have no rebalance concept, so skip them entirely // when every fragment is already covered and the caller hasn't // asked for retrain or an explicit delta merge. Vector indices @@ -1412,7 +2386,8 @@ impl DatasetIndexExt for Dataset { uuid: res.new_uuid, name: last_idx.name.clone(), // Keep the same name fields: last_idx.fields.clone(), - dataset_version: self.manifest.version, + covering_fields: last_idx.covering_fields.clone(), + dataset_version: res.new_dataset_version, fragment_bitmap: Some(res.new_fragment_bitmap), index_details: Some(Arc::new(res.new_index_details)), index_version: res.new_index_version, @@ -1424,7 +2399,14 @@ impl DatasetIndexExt for Dataset { new_indices.push(new_idx); } - if new_indices.is_empty() { + // A no-work optimize still has to commit on a table that requires + // catch-up. Coverage is derived at commit time, so an index that + // already spans the table records its position only if there is a + // commit to record it on -- and that is the ordinary case after a + // remap or a compaction that advanced a generation without changing + // fragments. Returning early there leaves the position missing forever + // and the repair rescheduling itself. + if new_indices.is_empty() && !self.mem_wal_catch_up_would_advance(&indices)? { return Ok(()); } @@ -1473,9 +2455,10 @@ impl DatasetIndexExt for Dataset { if indices.is_empty() { return Err(Error::index_not_found(format!("name={}", index_name))); } - let column = self.schema().field_by_id(indices[0].fields[0]).unwrap(); + let field_id = indices[0].fields[0]; + let field_path = self.schema().field_path(field_id)?; let logical_index = self - .open_logical_vector_index(&column.name, index_name) + .open_logical_vector_index(&field_path, index_name) .await?; logical_index .as_ivf()? @@ -1566,7 +2549,7 @@ async fn index_statistics_frag_reuse(ds: &Dataset) -> Result { .open_frag_reuse_index(&NoOpMetricsCollector) .await? .expect("FragmentReuse index does not exist"); - serialize_index_statistics(&index.statistics()?) + serialize_index_statistics(&CompactFragReuseIndexHandle(index).statistics()?) } async fn index_statistics_mem_wal(ds: &Dataset) -> Result { @@ -1574,7 +2557,7 @@ async fn index_statistics_mem_wal(ds: &Dataset) -> Result { .open_mem_wal_index(&NoOpMetricsCollector) .await? .expect("MemWal index does not exist"); - serialize_index_statistics(&index.statistics()?) + serialize_index_statistics(&MemWalIndexHandle(index).statistics()?) } async fn index_statistics_scalar( @@ -1715,20 +2698,54 @@ async fn gather_fragment_statistics( ))) } -pub(crate) fn retain_supported_indices(indices: &mut Vec) { - indices.retain(|idx| { - let max_supported_version = idx - .index_details - .as_ref() - .map(|details| { - IndexDetails(details.clone()) - .index_version() - // If we don't know how to read the index, it isn't supported - .unwrap_or(i32::MAX as u32) - }) - .unwrap_or_default(); - let is_valid = idx.index_version <= max_supported_version as i32; - if !is_valid { +/// `None` when this build supports the index's version, otherwise the highest +/// version it does support. +/// +/// Only a version bump of a type this build already has a plugin for is caught. +/// Reader availability is checked separately by [`index_is_usable`], because +/// an unknown type has no meaningful maximum version in this build. +pub(crate) fn unsupported_index_version(index: &IndexMetadata) -> Option { + let max_supported_version = index + .index_details + .as_ref() + .map(|details| { + IndexDetails(details.clone()) + .index_version() + .unwrap_or(i32::MAX as u32) + }) + .unwrap_or_default(); + (index.index_version > max_supported_version as i32).then_some(max_supported_version) +} + +/// Whether this build may expose an index through the usable-index view. +/// +/// System indices have dedicated readers rather than scalar plugins. Ordinary +/// indices need both a reader for their exact declared type and a supported +/// format version. +pub(crate) fn index_is_usable(index: &IndexMetadata) -> bool { + index_type_is_known(index) && unsupported_index_version(index).is_none() +} + +/// Name the indices this build has no reader for, once per manifest read. +/// +/// Deliberately not inside the filter in [`DatasetIndexExt::load_indices`]: that +/// runs on every call, and `load_indices` sits on the query-planning path and on +/// merge_insert's per-batch path. Warning there would cost an operator one line +/// per hidden index per query for as long as the dataset carries one. +pub(crate) fn warn_about_unsupported_indices(indices: &[IndexMetadata]) { + for idx in indices { + if !index_type_is_known(idx) { + let type_url = idx + .index_details + .as_ref() + .map(|details| details.type_url.as_str()) + .unwrap_or(""); + log::warn!( + "Index {} has unrecognized type {}, ignoring it", + idx.name, + type_url, + ); + } else if let Some(max_supported_version) = unsupported_index_version(idx) { log::warn!( "Index {} has version {}, which is not supported (<={}), ignoring it", idx.name, @@ -1736,8 +2753,101 @@ pub(crate) fn retain_supported_indices(indices: &mut Vec) { max_supported_version, ); } - is_valid - }) + } +} + +/// Every index the manifest names, including any this build has no reader for. +/// +/// Separate from [`DatasetIndexExt::load_indices`] because the two answer +/// different questions. A reader asks which indices it may *use*, and an index +/// it cannot decode is rightly absent. Everything that decides what the *next* +/// manifest looks like asks a different question, and there the same omission is +/// not a filter but an erasure. `build_manifest` seeds the new index list from +/// what it is handed, so an index left out disappears from the dataset for every +/// build, including the one that could have read it. Index bookkeeping - name +/// reservation, replace and removal selection, explicit drop - answers that +/// second question too: a name it cannot see is a name it will hand out twice. +pub(crate) async fn load_all_indices(dataset: &Dataset) -> Result>> { + let metadata_key = IndexMetadataKey { + version: dataset.version().version, + store_identity: &dataset.object_store.store_prefix, + e_tag: dataset.manifest_location.e_tag.as_deref(), + }; + let mut indices = dataset + .index_cache + .get_or_insert_with_key(metadata_key, || async { + let loaded = read_manifest_indexes( + &dataset.object_store, + &dataset.manifest_location, + &dataset.manifest, + ) + .await?; + warn_about_unsupported_indices(&loaded); + Ok(loaded) + }) + .await?; + + // Infer details for legacy vector indices (once per index name, concurrently). + // This may run on indices that were opportunistically cached during Dataset::open + // before the full Dataset was available for inference. + { + let schema = dataset.schema(); + if indices + .iter() + .any(|idx| needs_vector_details_inference(idx, schema)) + { + let mut updated = indices.as_ref().clone(); + infer_missing_vector_details(dataset, &mut updated).await; + if updated != *indices { + indices = Arc::new(updated); + dataset + .index_cache + .insert_with_key(&metadata_key, indices.clone()) + .await; + } + } + } + + if let Some(frag_reuse_index_meta) = + indices.iter().find(|idx| idx.name == FRAG_REUSE_INDEX_NAME) + { + let fri_key = FragReuseIndexKey { + uuid: &frag_reuse_index_meta.uuid, + }; + let frag_reuse_index = dataset + .index_cache + .get_or_insert_with_key(fri_key, || async move { + let index_details = + load_frag_reuse_index_details(dataset, frag_reuse_index_meta).await?; + open_frag_reuse_index(frag_reuse_index_meta.uuid, index_details.as_ref()).await + }) + .await?; + let mut indices = indices.as_ref().clone(); + for idx in indices.iter_mut() { + if let Some(bitmap) = idx.fragment_bitmap.as_mut() { + frag_reuse_index.remap_fragment_bitmap(bitmap)?; + } + } + Ok(Arc::new(indices)) + } else { + Ok(indices) + } +} + +/// The segments named `name`, including any this build has no reader for. +/// +/// The bookkeeping counterpart to [`DatasetIndexExt::load_indices_by_name`]. See +/// [`load_all_indices`] for which of the two a call site wants. +pub(crate) async fn load_all_indices_by_name( + dataset: &Dataset, + name: &str, +) -> Result> { + Ok(load_all_indices(dataset) + .await? + .iter() + .filter(|idx| idx.name == name) + .cloned() + .collect()) } /// A trait for internal dataset utilities @@ -1777,7 +2887,7 @@ pub trait DatasetIndexInternalExt: DatasetIndexExt { async fn open_frag_reuse_index( &self, metrics: &dyn MetricsCollector, - ) -> Result>>; + ) -> Result>>; /// Opens the MemWAL index async fn open_mem_wal_index( @@ -1834,7 +2944,7 @@ impl DatasetIndexInternalExt for Dataset { let frag_reuse_cache_key = FragReuseIndexCacheKey::new(uuid, frag_reuse_uuid.as_ref()); if let Some(index) = self.index_cache.get_with_key(&frag_reuse_cache_key).await { - return Ok(index.as_index()); + return Ok(Arc::new(CompactFragReuseIndexHandle(index)).as_index()); } // Sometimes we want to open an index and we don't care if it is a scalar or vector index. @@ -1906,7 +3016,7 @@ impl DatasetIndexInternalExt for Dataset { let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref()); if let Some(entry) = self.index_cache.get_with_key(&state_key).await { log::debug!("Found IvfIndexState in cache uuid: {}", uuid); - let partition_cache = self.index_cache.with_key_prefix(&state_key.key()); + let partition_cache = self.index_cache.for_index(uuid, frag_reuse_uuid.as_ref()); let frag_reuse_index = self.open_frag_reuse_index(metrics).await?; return entry .0 @@ -1944,8 +3054,10 @@ impl DatasetIndexInternalExt for Dataset { let tailing_bytes = read_last_block(reader.as_ref()).await?; let (major_version, minor_version) = read_version(&tailing_bytes)?; - // Namespace the index cache by the UUID of the index. - let index_cache = self.index_cache.with_key_prefix(&cache_key.key()); + // Namespace the index cache by the UUID of the index. v2+ partition + // entries are store-free and remain reusable across object-store + // generations alongside their serializable state. + let index_cache = self.index_cache.for_index(uuid, frag_reuse_uuid.as_ref()); // Extract the cacheable state before type-erasing to Arc. fn wrap_ivf( @@ -1985,7 +3097,7 @@ impl DatasetIndexInternalExt for Dataset { (0, 2) => { info!(target: TRACE_IO_EVENTS, index_uuid=%uuid, r#type=IO_TYPE_OPEN_VECTOR, version="0.2", index_type="IVF_PQ"); - let reader = PreviousFileReader::try_new_self_described_from_reader( + let reader = V1FileReader::try_new_self_described_from_reader( reader.clone(), Some(&self.metadata_cache.file_metadata_cache(&index_file)), ) @@ -2003,8 +3115,8 @@ impl DatasetIndexInternalExt for Dataset { (0, 3) | (2, _) => { let scheduler = ScanScheduler::new( - self.object_store.clone(), - SchedulerConfig::max_bandwidth(&self.object_store), + object_store.clone(), + SchedulerConfig::max_bandwidth(&object_store), ); let cached_size = file_sizes .get(INDEX_FILE_NAME) @@ -2038,7 +3150,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_FLAT" => match element_type { DataType::Float16 | DataType::Float32 | DataType::Float64 => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2051,7 +3163,7 @@ impl DatasetIndexInternalExt for Dataset { } DataType::UInt8 => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2070,7 +3182,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_PQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2084,7 +3196,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_SQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2098,8 +3210,8 @@ impl DatasetIndexInternalExt for Dataset { "IVF_RQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), - self.indices_dir(), + object_store.clone(), + index_dir, uuid.to_owned(), frag_reuse_index, self.metadata_cache.as_ref(), @@ -2113,7 +3225,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_HNSW_FLAT" => match element_type { DataType::UInt8 => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2126,7 +3238,7 @@ impl DatasetIndexInternalExt for Dataset { } _ => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2141,7 +3253,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_HNSW_SQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2155,7 +3267,7 @@ impl DatasetIndexInternalExt for Dataset { "IVF_HNSW_PQ" => { let ivf = IVFIndex::::try_new( - self.object_store.clone(), + object_store.clone(), index_dir, uuid.to_owned(), frag_reuse_index, @@ -2190,7 +3302,6 @@ impl DatasetIndexInternalExt for Dataset { io_stats.add_scan_stats(&open_stats); } if let Some(ivf_entry) = ivf_entry { - let state_key = IvfIndexStateCacheKey::new(uuid, frag_reuse_uuid.as_ref()); self.index_cache .insert_with_key(&state_key, Arc::new(ivf_entry)) .await; @@ -2215,7 +3326,7 @@ impl DatasetIndexInternalExt for Dataset { let field_id = self.schema().field_id(column)?; if let Some(invalid_metadata) = metadatas .iter() - .find(|metadata| !metadata.fields.contains(&field_id)) + .find(|metadata| metadata.fields.first() != Some(&field_id)) { return Err(Error::invalid_input(format!( "Logical vector index '{}' contains segment {} that does not belong to column '{}'", @@ -2237,7 +3348,7 @@ impl DatasetIndexInternalExt for Dataset { async fn open_frag_reuse_index( &self, metrics: &dyn MetricsCollector, - ) -> Result>> { + ) -> Result>> { if let Some(frag_reuse_index_meta) = self.load_index_by_name(FRAG_REUSE_INDEX_NAME).await? { let frag_reuse_uuid = frag_reuse_index_meta.uuid; let frag_reuse_key = FragReuseIndexKey { @@ -2247,14 +3358,8 @@ impl DatasetIndexInternalExt for Dataset { let index = self .index_cache .get_or_insert_with_key(frag_reuse_key, || async move { - let index_meta = - self.load_index(&frag_reuse_uuid).await?.ok_or_else(|| { - Error::index(format!( - "Index with id {} does not exist", - frag_reuse_uuid - )) - })?; - let index_details = load_frag_reuse_index_details(self, &index_meta).await?; + let index_details = + load_frag_reuse_index_details(self, &frag_reuse_index_meta).await?; let index = open_frag_reuse_index(frag_reuse_index_meta.uuid, index_details.as_ref()) .await?; @@ -2327,12 +3432,6 @@ impl DatasetIndexInternalExt for Dataset { // so the optimizer treats coverage as unknown. let mut fragment_bitmaps: HashMap<(String, String), Option> = HashMap::new(); for index in indices.iter().filter(|idx| { - let idx_schema = schema.project_by_ids(idx.fields.as_slice(), true); - let is_vector_index = idx_schema - .fields - .iter() - .any(|f| is_vector_field(f.data_type())); - // Check if this is an FTS index by looking at index details let is_fts_index = if let Some(details) = &idx.index_details { IndexDetails(details.clone()).supports_fts() @@ -2346,7 +3445,10 @@ impl DatasetIndexInternalExt for Dataset { !bitmap.is_empty() && !(bitmap & self.fragment_bitmap.as_ref()).is_empty() }); - idx.fields.len() == 1 && !is_vector_index && (has_non_empty_bitmap || is_fts_index) + // Same name-collision rationale as `CreateIndexBuilder`'s + // default-name loop in create.rs: only the keyed prefix decides + // which column this index answers for. + idx.keyed_field().is_some() && (has_non_empty_bitmap || is_fts_index) }) { let field = index.fields[0]; let field = schema.field_by_id(field).ok_or_else(|| { @@ -2491,8 +3593,13 @@ impl DatasetIndexInternalExt for Dataset { )) })?; - let mut field_names = Vec::new(); - for field_id in source_index.fields.iter() { + // Only the keyed prefix matters here. The rebuild below writes fresh, + // non-covered metadata and never reads the carried columns, so requiring + // them to exist and type-match would reject a target that can hold this + // index perfectly well. + let keyed_fields = source_index.keyed_fields(); + let mut field_paths = Vec::with_capacity(keyed_fields.len()); + for field_id in keyed_fields.iter() { let source_field = source_dataset .schema() .field_by_id(*field_id) @@ -2502,33 +3609,35 @@ impl DatasetIndexInternalExt for Dataset { field_id )) })?; + let source_field_path = source_dataset.schema().field_path(*field_id)?; - let target_field = self.schema().field(&source_field.name).ok_or_else(|| { + let target_field = self.schema().field(&source_field_path).ok_or_else(|| { Error::index(format!( "Field '{}' required by index '{}' not found in target dataset", - source_field.name, index_name + source_field_path, index_name )) })?; if source_field.data_type() != target_field.data_type() { return Err(Error::index(format!( "Field '{}' has different types in source ({:?}) and target ({:?}) datasets", - source_field.name, + source_field_path, source_field.data_type(), target_field.data_type() ))); } - field_names.push(source_field.name.as_str()); + field_paths.push(source_field_path); } - if field_names.is_empty() { + if field_paths.is_empty() { return Err(Error::index(format!( "Index '{}' has no fields", index_name ))); } + let field_names = field_paths.iter().map(String::as_str).collect::>(); if let Some(index_details) = &source_index.index_details { let index_details_wrapper = IndexDetails(index_details.clone()); @@ -2636,7 +3745,7 @@ mod tests { use crate::session::Session; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount, copy_test_data_to_tmp}; use arrow::array::AsArray; - use arrow::datatypes::{Float32Type, Int32Type}; + use arrow::datatypes::{Float32Type, Int32Type, Int64Type}; use arrow_array::Int32Array; use arrow_array::{ FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, StringArray, @@ -2647,8 +3756,12 @@ mod tests { use lance_core::utils::tempfile::TempStrDir; use lance_datagen::gen_batch; use lance_datagen::{BatchCount, ByteCount, Dimension, RowCount, array}; - use lance_index::pbold::BTreeIndexDetails; + use lance_index::pbold::{BTreeIndexDetails, InvertedIndexDetails}; use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME; + use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery}; + use lance_index::scalar::inverted::{ + INVERTED_INDEX_VERSION_V1, INVERTED_INDEX_VERSION_V2, INVERTED_INDEX_VERSION_V3, + }; use lance_index::scalar::{ BuiltinIndexType, FullTextSearchQuery, InvertedIndexParams, ScalarIndexParams, }; @@ -2687,6 +3800,7 @@ mod tests { uuid, name: index_name.to_string(), fields: vec![field_id], + covering_fields: vec![], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap.into_iter().collect()), index_details: Some(Arc::new(vector_index_details_default())), @@ -2715,20 +3829,10 @@ mod tests { } fn segment_from_metadata(metadata: &IndexMetadata) -> IndexSegment { - IndexSegment::new( - metadata.uuid, - metadata - .fragment_bitmap - .as_ref() - .expect("test segment metadata should have fragment coverage") - .iter(), - metadata - .index_details - .as_ref() - .expect("test segment metadata should have index details") - .clone(), - metadata.index_version, - ) + metadata + .clone() + .into_index_segment() + .expect("test segment metadata should convert to an index segment") } async fn write_fragmented_vector_dataset(uri: &str, dimension: i32) -> Dataset { @@ -2893,33 +3997,379 @@ mod tests { segment_ids } - #[tokio::test] - async fn test_open_logical_vector_index_single_segment_quality_apis() { - const DIMENSION: i32 = 8; - - let test_dir = tempfile::tempdir().unwrap(); - let test_uri = test_dir.path().to_str().unwrap(); - let mut dataset = write_fragmented_vector_dataset(test_uri, DIMENSION).await; - let params = - VectorIndexParams::with_ivf_flat_params(DistanceType::L2, IvfBuildParams::new(2)); + /// Premise guard, not a regression test: confirms that the broken shape + /// (`fields` truncated to the keyed field while `covering_fields` is + /// inherited unchanged) is actually rejected by `validate_covering_fields`, + /// and that the fixed shape (both fields carried through) is accepted. + /// Both shapes are hand-built here, so this exercises no `merge_segments` + /// code and would keep passing even if every impl regressed to the broken + /// shape. The real regression coverage is + /// `test_merge_existing_index_segments_preserves_covering_bitmap`, + /// `_btree`, `_vector`, and `_rtree` below, which merge real segments + /// through the actual `merge_segments` implementations. + #[test] + fn test_merged_covered_metadata_is_committable() { + let source = IndexMetadata { + uuid: Uuid::new_v4(), + name: "covered".to_string(), + fields: vec![7, 11], + covering_fields: vec![11], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([0u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; - dataset - .create_index( - &["vector"], - IndexType::Vector, - Some("vector_idx".to_string()), - ¶ms, - true, - ) - .await - .unwrap(); + // The shape every merge_segments impl builds: keyed field only, carried + // inherited from the source. + let field_id = *source.fields.first().unwrap(); + let merged = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![field_id], + ..source.clone() + }; + assert!( + merged.validate_covering_fields().is_err(), + "this is the broken shape; if it validates, the premise has changed" + ); - let logical_index = dataset - .open_logical_vector_index("vector", "vector_idx") - .await - .unwrap(); + // What the fix must produce instead. + let fixed = IndexMetadata { + uuid: Uuid::new_v4(), + ..source + }; + fixed + .validate_covering_fields() + .expect("merged covered metadata must be committable"); + assert_eq!(fixed.fields, vec![7, 11]); + assert_eq!(fixed.covering_fields, vec![11]); + } - assert_eq!(logical_index.name(), "vector_idx"); + /// Two-fragment, two-int-column dataset for the scalar-segment covering + /// tests below. Column "id" is keyed, "carried" plays the covered column. + async fn write_two_int_column_dataset(uri: &str, rows_per_fragment: i32) -> Dataset { + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("carried", array::step::()) + .into_reader_rows( + RowCount::from(rows_per_fragment as u64), + BatchCount::from(2), + ); + Dataset::write( + reader, + uri, + Some(WriteParams { + max_rows_per_file: rows_per_fragment as usize, + ..Default::default() + }), + ) + .await + .unwrap() + } + + /// Nothing writes a covering declaration yet, so the declaration below is + /// hand-constructed on top of real single-field segments: merge plumbing + /// never reads covered-column storage, so a declaration naming a real, + /// uninvolved column is a faithful stand-in for a genuinely covered segment. + fn make_covered( + segment: IndexMetadata, + keyed_field_id: i32, + carried_field_id: i32, + ) -> IndexMetadata { + IndexMetadata { + fields: vec![keyed_field_id, carried_field_id], + covering_fields: vec![carried_field_id], + ..segment + } + } + + /// Build one covered segment per named fragment, merge them, and assert the + /// merged declaration survived and is committable. + /// + /// The per-index-type cases below differ only in how their dataset and + /// params are built, so the merge and the assertions live here. + async fn assert_merge_preserves_covering( + dataset: &mut Dataset, + column: &str, + index_type: IndexType, + params: &dyn IndexParams, + fragment_ids: Vec, + keyed_field_id: i32, + carried_field_id: i32, + ) { + let index_name = format!("covered_{column}"); + let mut covered_segments = Vec::new(); + for fragment_id in fragment_ids { + let segment = dataset + .create_index_builder(&[column], index_type, params) + .name(index_name.clone()) + .fragments(vec![fragment_id]) + .execute_uncommitted() + .await + .unwrap(); + covered_segments.push(make_covered(segment, keyed_field_id, carried_field_id)); + } + + let merged = dataset + .merge_existing_index_segments(covered_segments) + .await + .unwrap(); + + merged + .validate_covering_fields() + .expect("merged covering declaration must be committable"); + assert_eq!(merged.fields, vec![keyed_field_id, carried_field_id]); + assert_eq!( + merged.covering_fields, + vec![carried_field_id], + "merge_segments must neither drop the covering declaration nor \ + produce an uncommittable one" + ); + } + + fn all_fragment_ids(dataset: &Dataset) -> Vec { + dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect() + } + + /// Two shapes of the same bug, one per `merge_segments` family: + /// + /// - **struct update** (bitmap, inverted, zonemap, label_list, + /// bloomfilter, fmindex): `fields` was truncated to `vec![field_id]` + /// while `..segments[0].clone()` re-supplied `covering_fields`, + /// producing an *uncommittable* merged segment. + /// - **explicit** (btree, ngram): `fields: vec![field_id], + /// covering_fields: vec![]` was built outright, *silently dropping* the + /// declaration instead. + #[rstest] + #[case::struct_update(BuiltinIndexType::Bitmap, IndexType::Bitmap)] + #[case::explicit(BuiltinIndexType::BTree, IndexType::BTree)] + #[tokio::test] + async fn test_merge_existing_index_segments_preserves_covering_scalar( + #[case] builtin: BuiltinIndexType, + #[case] index_type: IndexType, + ) { + let test_dir = TempStrDir::default(); + let mut dataset = write_two_int_column_dataset(&test_dir, 20).await; + let id_field_id = dataset.schema().field("id").unwrap().id; + let carried_field_id = dataset.schema().field("carried").unwrap().id; + let fragment_ids = all_fragment_ids(&dataset); + + let params = ScalarIndexParams::for_builtin(builtin); + assert_merge_preserves_covering( + &mut dataset, + "id", + index_type, + ¶ms, + fragment_ids, + id_field_id, + carried_field_id, + ) + .await; + } + + /// Segments that disagree on which columns they carry cannot be folded + /// into one coherent output declaration: there is no single + /// `covering_fields` that would describe the merged segment truthfully. + #[tokio::test] + async fn test_merge_existing_index_segments_rejects_mismatched_covering_fields() { + let test_dir = TempStrDir::default(); + let mut dataset = write_two_int_column_dataset(&test_dir, 20).await; + let id_field_id = dataset.schema().field("id").unwrap().id; + let carried_field_id = dataset.schema().field("carried").unwrap().id; + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let mut mismatched_segments = Vec::new(); + for (i, fragment) in dataset.get_fragments().into_iter().enumerate() { + let segment = dataset + .create_index_builder(&["id"], IndexType::Bitmap, ¶ms) + .name("mismatched_bitmap".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + // First segment declares `carried` covered; the second declares + // nothing covered -- a genuine disagreement, not just a + // reordering. + if i == 0 { + mismatched_segments.push(make_covered(segment, id_field_id, carried_field_id)); + } else { + mismatched_segments.push(segment); + } + } + + let err = dataset + .merge_existing_index_segments(mismatched_segments) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("requires segments with identical fields"), + "unexpected error: {err}" + ); + } + + /// Vector shape: `crate::index::vector::ivf::merge_segments` already + /// preserves both `fields` and `covering_fields` via struct update, but + /// `merge_existing_index_segments`'s own tail used to unconditionally + /// overwrite `fields` to `vec![field_id]` afterwards, reproducing the + /// struct-update bug for every index type it dispatches to. + #[tokio::test] + async fn test_merge_existing_index_segments_preserves_covering_vector() { + const DIMENSION: i32 = 4; + let test_dir = TempStrDir::default(); + let mut dataset = write_fragmented_vector_dataset(&test_dir, DIMENSION).await; + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + + let batch = dataset + .scan() + .project(&["vector"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let vectors = batch + .column_by_name("vector") + .expect("vector column should exist") + .as_fixed_size_list(); + let values = vectors.values().as_primitive::(); + let centroids = train_kmeans::( + values, + KMeansParams::new(None, 10, 1, DistanceType::L2), + DIMENSION as usize, + 2, + 2, + ) + .unwrap() + .centroids + .as_primitive::() + .clone(); + let centroids = + Arc::new(FixedSizeListArray::try_new_from_values(centroids, DIMENSION).unwrap()); + let params = VectorIndexParams::with_ivf_flat_params( + DistanceType::L2, + IvfBuildParams::try_with_centroids(2, centroids).unwrap(), + ); + + let fragment_ids = all_fragment_ids(&dataset).into_iter().take(2).collect(); + + assert_merge_preserves_covering( + &mut dataset, + "vector", + IndexType::Vector, + ¶ms, + fragment_ids, + vector_field_id, + id_field_id, + ) + .await; + } + + /// Struct-update shape, sweep-found in `rtree.rs`: same bug as the + /// bitmap/inverted/zonemap/label_list/bloomfilter/fmindex family, not + /// called out in the original brief but reachable from the same + /// `all_rtree` dispatch branch this function tests. + #[cfg(feature = "geo")] + #[tokio::test] + async fn test_merge_existing_index_segments_preserves_covering_rtree() { + use geo_types::line_string; + use geoarrow_array::GeoArrowArray; + use geoarrow_array::builder::LineStringBuilder; + use geoarrow_schema::{Dimension, LineStringType}; + + const ROWS_PER_FRAGMENT: i32 = 20; + let line_string_type = LineStringType::new(Dimension::XY, Default::default()); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + line_string_type.clone().to_field("geometry", true), + ])); + + let batches = (0..2) + .map(|fragment: i32| { + let mut builder = LineStringBuilder::new(line_string_type.clone()); + for row in 0..ROWS_PER_FRAGMENT { + let x = (fragment * ROWS_PER_FRAGMENT + row) as f64; + builder + .push_line_string(Some(&line_string![ + (x: x, y: x), + (x: x + 1.0, y: x + 1.0) + ])) + .unwrap(); + } + let ids = Int32Array::from_iter_values( + fragment * ROWS_PER_FRAGMENT..(fragment + 1) * ROWS_PER_FRAGMENT, + ); + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(ids), builder.finish().to_array_ref()], + ) + }) + .collect::, arrow_schema::ArrowError>>() + .unwrap(); + + let test_dir = TempStrDir::default(); + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema.clone()); + let mut dataset = Dataset::write( + reader, + &test_dir, + Some(WriteParams { + max_rows_per_file: ROWS_PER_FRAGMENT as usize, + ..Default::default() + }), + ) + .await + .unwrap(); + let id_field_id = dataset.schema().field("id").unwrap().id; + let geometry_field_id = dataset.schema().field("geometry").unwrap().id; + let fragment_ids = all_fragment_ids(&dataset); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::RTree); + assert_merge_preserves_covering( + &mut dataset, + "geometry", + IndexType::RTree, + ¶ms, + fragment_ids, + geometry_field_id, + id_field_id, + ) + .await; + } + + #[tokio::test] + async fn test_open_logical_vector_index_single_segment_quality_apis() { + const DIMENSION: i32 = 8; + + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let mut dataset = write_fragmented_vector_dataset(test_uri, DIMENSION).await; + let params = + VectorIndexParams::with_ivf_flat_params(DistanceType::L2, IvfBuildParams::new(2)); + + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + ¶ms, + true, + ) + .await + .unwrap(); + + let logical_index = dataset + .open_logical_vector_index("vector", "vector_idx") + .await + .unwrap(); + + assert_eq!(logical_index.name(), "vector_idx"); assert_eq!(logical_index.column(), "vector"); assert_eq!(logical_index.num_segments(), 1); assert_eq!(logical_index.metadatas().len(), 1); @@ -3367,6 +4817,106 @@ mod tests { assert_eq!(stats["num_indexed_rows"], 512); } + #[tokio::test] + async fn test_v036_scalar_details_are_still_known() { + let test_dir = copy_test_data_to_tmp("0.36.0/btree_in_index_pkg.lance").unwrap(); + let dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + let details = indices[0].index_details.clone().unwrap(); + + assert_eq!(details.type_url, "/lance.index.pb.BTreeIndexDetails"); + assert_eq!(IndexDetails(details).get_plugin().unwrap().name(), "BTree"); + assert!(index_type_is_known(&indices[0])); + } + + #[tokio::test] + async fn test_unknown_index_type_does_not_block_queries_or_optimization() { + let reader = gen_batch() + .col("vector", array::rand_vec::(Dimension::from(8))) + .col("number", array::step::()) + .into_reader_rows(RowCount::from(64), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + dataset + .create_index( + &["number"], + IndexType::BTree, + Some("number_idx".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + let appended = gen_batch() + .col("vector", array::rand_vec::(Dimension::from(8))) + .col("number", array::step::()) + .into_reader_rows(RowCount::from(32), BatchCount::from(1)); + dataset.append(appended, None).await.unwrap(); + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("number_idx").await.unwrap()).unwrap(); + assert_eq!(stats["num_unindexed_rows"], 32); + + let field_id = dataset.schema().field("vector").unwrap().id; + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + let mut foreign_segment = write_vector_segment_metadata( + &dataset, + "foreign_idx", + field_id, + Uuid::new_v4(), + fragment_ids, + b"opaque external index", + ) + .await; + foreign_segment.index_details = Some(Arc::new(prost_types::Any { + type_url: "type.googleapis.com/example.MyVectorIndexDetails".to_string(), + value: Vec::new(), + })); + foreign_segment.index_version = 1; + dataset + .commit_existing_index_segments("foreign_idx", "vector", vec![foreign_segment]) + .await + .unwrap(); + + assert!( + dataset + .load_indices_by_name("foreign_idx") + .await + .unwrap() + .is_empty(), + "an index with no reader must not enter the usable-index view" + ); + assert_eq!( + load_all_indices_by_name(&dataset, "foreign_idx") + .await + .unwrap() + .len(), + 1, + "hiding an unusable index must not erase its manifest metadata" + ); + + let query = Float32Array::from(vec![0.5_f32; 8]); + let mut scanner = dataset.scan(); + scanner.nearest("vector", &query, 5).unwrap(); + assert_eq!(scanner.try_into_batch().await.unwrap().num_rows(), 5); + + dataset.optimize_indices(&Default::default()).await.unwrap(); + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("number_idx").await.unwrap()).unwrap(); + assert_eq!(stats["num_unindexed_rows"], 0); + assert_eq!( + load_all_indices_by_name(&dataset, "foreign_idx") + .await + .unwrap() + .len(), + 1, + "optimizing supported indices must preserve the opaque segment" + ); + } + #[tokio::test] async fn test_optimize_delta_indices() { let dimensions = 16; @@ -3644,6 +5194,53 @@ mod tests { assert_eq!(stats["num_indices"], 1); } + #[rstest] + #[case::v1("v3.0.1/fts_v1", INVERTED_INDEX_VERSION_V1)] + #[case::v2("v4.0.1/fts_v2", INVERTED_INDEX_VERSION_V2)] + #[tokio::test] + async fn test_read_fts_format_fixture( + #[case] fixture_path: &str, + #[case] expected_version: u32, + ) { + async fn search_ids(dataset: &Dataset, query: FullTextSearchQuery) -> Vec { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(query) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids = result["id"].as_primitive::().values().to_vec(); + ids.sort_unstable(); + ids + } + + let test_dir = copy_test_data_to_tmp(fixture_path).unwrap(); + let dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_version, expected_version as i32); + + let match_ids = search_ids( + &dataset, + FullTextSearchQuery::new("compatibility".to_string()), + ) + .await; + assert_eq!(match_ids, (0..300).collect::>()); + + let phrase = + PhraseQuery::new("lance database".to_string()).with_column(Some("text".to_string())); + let phrase_ids = search_ids( + &dataset, + FullTextSearchQuery::new_query(FtsQuery::Phrase(phrase)), + ) + .await; + assert_eq!(phrase_ids, (0..300).step_by(3).collect::>()); + } + #[rstest] #[tokio::test] async fn test_optimize_fts(#[values(false, true)] with_position: bool) { @@ -4103,14 +5700,212 @@ mod tests { } #[tokio::test] - async fn test_remap_empty() { - let data = gen_batch() - .col("int", array::step::()) - .col( - "vector", - array::rand_vec::(Dimension::from(16)), - ) - .into_reader_rows(RowCount::from(256), BatchCount::from(1)); + async fn test_index_metadata_cache_does_not_survive_drop_recreate_same_uri() { + fn batch_with_schema_metadata(metadata_value: String) -> RecordBatch { + let field = Field::new("tag", DataType::Utf8, false); + let schema = Arc::new(Schema::new_with_metadata( + vec![field], + HashMap::from([("large_metadata".to_string(), metadata_value)]), + )); + let array = StringArray::from_iter_values((0..128).map(|i| ["a", "b", "c"][i % 3])); + RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap() + } + + async fn write_indexed_dataset(uri: &str, session: Arc, metadata_value: String) { + let batch = batch_with_schema_metadata(metadata_value); + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let write_params = WriteParams { + session: Some(session), + ..Default::default() + }; + let mut dataset = Dataset::write(reader, uri, Some(write_params)) + .await + .unwrap(); + dataset + .create_index( + &["tag"], + IndexType::Bitmap, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + } + + let qn_session = Arc::new(Session::default()); + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + write_indexed_dataset(test_uri, qn_session.clone(), "old".to_string()).await; + let first_dataset = DatasetBuilder::from_uri(test_uri) + .with_session(qn_session.clone()) + .load() + .await + .unwrap(); + let first_indices = first_dataset.load_indices().await.unwrap(); + let first_uuid = first_indices[0].uuid; + assert_eq!(first_dataset.version().version, 2); + drop(first_dataset); + + std::fs::remove_dir_all(test_uri).unwrap(); + + // Use a different writer session so the QN session keeps the previous + // incarnation's index metadata cache entry. The large schema metadata + // keeps the manifest index section outside the final read block during + // open, so the fresh manifest load cannot opportunistically overwrite + // the stale index metadata entry before load_indices(). + write_indexed_dataset( + test_uri, + Arc::new(Session::default()), + "x".repeat(128 * 1024), + ) + .await; + let second_dataset = DatasetBuilder::from_uri(test_uri) + .with_session(qn_session) + .load() + .await + .unwrap(); + assert_eq!(second_dataset.version().version, 2); + + let raw_second_indices = read_manifest_indexes( + second_dataset.object_store.as_ref(), + &second_dataset.manifest_location, + second_dataset.manifest(), + ) + .await + .unwrap(); + let raw_second_uuid = raw_second_indices[0].uuid; + assert_ne!( + raw_second_uuid, first_uuid, + "the recreated dataset should commit a new physical index UUID" + ); + + let cached_second_indices = second_dataset.load_indices().await.unwrap(); + assert_eq!( + cached_second_indices[0].uuid, raw_second_uuid, + "load_indices should return index metadata from the recreated dataset, not the previous same-URI incarnation" + ); + } + + #[tokio::test] + async fn test_load_indices_singleflights_concurrent_cache_misses() { + let session = Arc::new(Session::default()); + let write_params = WriteParams { + session: Some(session.clone()), + ..Default::default() + }; + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![Field::new("tag", DataType::Utf8, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(vec!["a", "b", "c"]))], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &test_dir, + Some(write_params), + ) + .await + .unwrap(); + dataset + .create_index( + &["tag"], + IndexType::Bitmap, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + session.index_cache.clear().await; + let before = session.index_cache_stats().await; + let results = futures::future::join_all((0..32).map(|_| dataset.load_indices())).await; + assert!(results.iter().all(|result| result.is_ok())); + let after = session.index_cache_stats().await; + + assert_eq!(after.misses - before.misses, 1); + assert_eq!(after.hits - before.hits, 31); + } + + #[tokio::test] + async fn test_open_frag_reuse_index_with_zero_capacity_cache() { + let test_dir = TempStrDir::default(); + let data = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(400), BatchCount::from(1)); + let mut dataset = Dataset::write( + data, + &test_dir, + Some(WriteParams { + max_rows_per_file: 100, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["id"], + IndexType::BTree, + Some("id_idx".to_owned()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 200, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!( + dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await + .unwrap() + .is_some() + ); + + let session = Arc::new(Session::with_index_cache_backend( + Arc::new(lance_core::cache::MokaCacheBackend::no_cache()), + 128 * 1024 * 1024, + Default::default(), + )); + let dataset = DatasetBuilder::from_uri(&test_dir) + .with_session(session) + .load() + .await + .unwrap(); + + let frag_reuse_index = tokio::time::timeout( + std::time::Duration::from_secs(5), + dataset.open_frag_reuse_index(&NoOpMetricsCollector), + ) + .await + .expect("opening the fragment reuse index deadlocked") + .unwrap(); + assert!(frag_reuse_index.is_some()); + } + + #[tokio::test] + async fn test_remap_empty_chain() { + let data = gen_batch() + .col("int", array::step::()) + .col( + "vector", + array::rand_vec::(Dimension::from(16)), + ) + .into_reader_rows(RowCount::from(256), BatchCount::from(1)); let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); let params = VectorIndexParams::ivf_pq(1, 8, 1, DistanceType::L2, 1); @@ -4120,15 +5915,187 @@ mod tests { .unwrap(); let index_uuid = dataset.load_indices().await.unwrap()[0].uuid; - let remap_to_empty = (0..dataset.count_all_rows().await.unwrap()) + let row_count = dataset.count_all_rows().await.unwrap(); + let first_half = (0..row_count / 2) + .map(|i| (i as u64, None)) + .collect::>(); + let second_half = (row_count / 2..row_count) .map(|i| (i as u64, None)) .collect::>(); - let new_uuid = remap_index(&dataset, &index_uuid, &RowAddrRemap::direct(remap_to_empty)) + let remap_to_empty = RowAddrRemap::chained([ + RowAddrRemap::direct(first_half), + RowAddrRemap::direct(second_half), + ]); + let new_uuid = remap_index(&dataset, &index_uuid, &remap_to_empty) .await .unwrap(); assert_eq!(new_uuid, RemapResult::Keep(index_uuid)); } + /// The `fields.len() > 1` rejection in `remap_index`, which had no dedicated + /// coverage. A covered index never reaches it: the withdrawal above returns + /// `RemapResult::Drop` first, which is what + /// `test_compaction_withdraws_a_covered_index_without_failing` in + /// `dataset/tests/dataset_index.rs` exercises, and + /// `test_remap_column_index_refuses_a_covered_index` in + /// `dataset/optimize/remapping.rs` never reaches `remap_index` at all since + /// `remap_column_index` refuses first. So only a genuinely composite index -- + /// two keyed fields, no covering declaration -- arrives here, and it must + /// still be rejected. + #[tokio::test] + async fn test_remap_index_rejects_composite_index() { + let data = gen_batch() + .col("a", array::step::()) + .col("b", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + dataset + .create_index( + &["a"], + IndexType::BTree, + None, + &lance_index::scalar::ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + let a_id = dataset.schema().field("a").unwrap().id; + let b_id = dataset.schema().field("b").unwrap().id; + let current = dataset.load_indices().await.unwrap(); + let mut composite = current[0].clone(); + // No covering declaration at all -- genuinely composite, not covered. + composite.fields = vec![a_id, b_id]; + composite.covering_fields = vec![]; + + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![composite], + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let index_uuid = dataset.load_indices().await.unwrap()[0].uuid; + let error = remap_index(&dataset, &index_uuid, &RowAddrRemap::empty()) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("only one keyed field is supported"), + "{error}" + ); + } + + /// Decoding a manifest validates the declaration, so `matched` cannot be + /// malformed by that route -- but it is not the only route: metadata + /// constructed in-process (segment conversion, a distributed build's output) + /// reaches this guard without ever passing through + /// `TryFrom`. This guard is therefore load-bearing on its + /// own, and must produce a named error rather than either a panic (plain + /// subtraction on `usize` underflows) or a silent + /// saturation that lets corrupt metadata through as an ordinary covered index + /// and gets it quietly withdrawn. The test reaches it by seeding the index + /// cache, the same way the migration-path test in `dataset/index.rs` does. + #[tokio::test] + async fn test_remap_index_rejects_malformed_covering_fields() { + let data = gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + dataset + .create_index( + &["id"], + IndexType::BTree, + None, + &lance_index::scalar::ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + let index_uuid = dataset.load_indices().await.unwrap()[0].uuid; + + // Malform the cached metadata directly: more carried fields than fields + // at all. No commit and no manifest decode can produce this -- both run + // `validate_covering_fields` -- which is exactly why the guard has to + // stand on its own for metadata that reached it by neither route. + let mut indices = dataset.load_indices().await.unwrap().as_ref().clone(); + for idx in &mut indices { + if idx.uuid == index_uuid { + idx.covering_fields = idx + .fields + .iter() + .copied() + .chain(std::iter::once(999)) + .collect(); + } + } + assert!( + indices + .iter() + .any(|idx| idx.uuid == index_uuid && idx.covering_fields.len() > idx.fields.len()), + "test setup should have produced a malformed entry" + ); + let metadata_key = crate::session::index_caches::IndexMetadataKey { + version: dataset.version().version, + store_identity: &dataset.object_store.store_prefix, + e_tag: dataset.manifest_location.e_tag.as_deref(), + }; + dataset + .index_cache + .insert_with_key(&metadata_key, Arc::new(indices)) + .await; + + // The validation runs first in `remap_index`, ahead of the withdrawal and + // of every row-map-dependent branch, so an empty remap reaches it. + let error = remap_index(&dataset, &index_uuid, &RowAddrRemap::empty()) + .await + .expect_err("malformed covering metadata must not be silently accepted"); + assert!( + error.to_string().contains("are not among its fields"), + "expected the validator's message, got: {error}" + ); + } + + /// A covered scalar index still answers for its keyed column: only the + /// keyed prefix of `fields` must be a single field for `scalar_index_info` + /// to make it eligible for filter pushdown, not the full vector including + /// carried columns. Before this fix, a covered index was silently excluded + /// -- no error, no failing query, just a plan that quietly stopped using it. + #[tokio::test] + async fn test_scalar_index_info_includes_covered_index() { + let data = gen_batch() + .col("a", array::step::()) + .col("b", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + dataset + .create_index( + &["a"], + IndexType::BTree, + None, + &lance_index::scalar::ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + crate::utils::test::covering::declare_covering(&mut dataset, "a", "b").await; + + let index_info = dataset.scalar_index_info().await.unwrap(); + assert!( + index_info.get_index("a").is_some(), + "a covered index must still be eligible for filter pushdown on its keyed column" + ); + } + #[tokio::test] async fn test_optimize_ivf_pq_up_to_date() { // https://github.com/lance-format/lance/issues/4016 @@ -4488,6 +6455,8 @@ mod tests { // We commit by doing a delete("false") after replacing the cached indices. let metadata_key = crate::session::index_caches::IndexMetadataKey { version: dataset.version().version, + store_identity: &dataset.object_store.store_prefix, + e_tag: dataset.manifest_location.e_tag.as_deref(), }; dataset .index_cache @@ -4579,6 +6548,7 @@ mod tests { uuid: Uuid::new_v4(), name: "mystery_idx".to_string(), fields: vec![field_id], + covering_fields: vec![], dataset_version: dataset.manifest.version, fragment_bitmap: Some(std::iter::once(0_u32).collect()), index_details: None, @@ -4616,6 +6586,7 @@ mod tests { uuid: Uuid::new_v4(), name: "mystery_idx".to_string(), fields: vec![field_id], + covering_fields: vec![], dataset_version: dataset.manifest.version, fragment_bitmap: Some(std::iter::once(0_u32).collect()), index_details: Some(Arc::new(prost_types::Any { @@ -4715,50 +6686,127 @@ mod tests { ); } - /// Helper function to check if an index is being used in a query plan - fn assert_index_usage(plan: &str, column_name: &str, should_use_index: bool, context: &str) { - let index_used = if column_name == "text" { - // For inverted index, look for MatchQuery which indicates FTS index usage - plan.contains("MatchQuery") - } else { - // For btree/bitmap, look for MaterializeIndex which indicates scalar index usage - plan.contains("ScalarIndexQuery") - }; - - if should_use_index { - assert!( - index_used, - "Query plan should use index {}: {}", - context, plan - ); - } else { - assert!( - !index_used, - "Query plan should NOT use index {}: {}", - context, plan - ); - } - } - - /// Test that scalar indices are retained after deleting all data from a table. - /// - /// This test verifies that when we: - /// 1. Create a table with data - /// 2. Add a scalar index with train=true - /// 3. Delete all data in the table - /// The index remains available on the table. #[rstest] - #[case::btree("i", IndexType::BTree, Box::new(ScalarIndexParams::default()))] - #[case::bitmap("i", IndexType::Bitmap, Box::new(ScalarIndexParams::default()))] - #[case::inverted("text", IndexType::Inverted, Box::new(InvertedIndexParams::default()))] + #[case::block_size_128(128)] + #[case::block_size_256(256)] #[tokio::test] - async fn test_scalar_index_retained_after_delete_all( - #[case] column_name: &str, - #[case] index_type: IndexType, - #[case] params: Box, - ) { - use lance_datagen::{BatchCount, ByteCount, RowCount, array}; - + async fn test_optimize_empty_code_fts_index_preserves_params(#[case] block_size: usize) { + let dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("code", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values([0, 1])), + Arc::new(StringArray::from_iter_values([ + "fn GetUserEmail() {}", + "fn ParseConfig() {}", + ])), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(reader, &dir, None).await.unwrap(); + + let params = InvertedIndexParams::default() + .analyzer("code") + .unwrap() + .block_size(block_size) + .unwrap() + .split_identifiers(true) + .preserve_original(false); + dataset + .create_index_builder(&["code"], IndexType::Inverted, ¶ms) + .name("code_idx".to_string()) + .train(false) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32); + + dataset.optimize_indices(&Default::default()).await.unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_version, INVERTED_INDEX_VERSION_V3 as i32); + let details = indices[0] + .index_details + .as_ref() + .expect("optimized FTS index should retain index details") + .to_msg::() + .unwrap(); + assert_eq!(details.base_tokenizer.as_deref(), Some("code")); + assert_eq!(details.block_size, Some(block_size as u32)); + let code_config = details + .code_config + .expect("optimized code FTS index should retain code configuration"); + assert!(code_config.split_identifiers); + assert_eq!(code_config.preserve_original, Some(false)); + + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("code_idx").await.unwrap()).unwrap(); + assert_eq!(stats["num_unindexed_rows"], 0); + + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .full_text_search(FullTextSearchQuery::new("email".to_string())) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result["id"].as_primitive::().values(), &[0]); + } + + /// Helper function to check if an index is being used in a query plan + fn assert_index_usage(plan: &str, column_name: &str, should_use_index: bool, context: &str) { + let index_used = if column_name == "text" { + // For inverted index, look for MatchQuery which indicates FTS index usage + plan.contains("MatchQuery") + } else { + // For btree/bitmap, look for MaterializeIndex which indicates scalar index usage + plan.contains("ScalarIndexQuery") + }; + + if should_use_index { + assert!( + index_used, + "Query plan should use index {}: {}", + context, plan + ); + } else { + assert!( + !index_used, + "Query plan should NOT use index {}: {}", + context, plan + ); + } + } + + /// Test that scalar indices are retained after deleting all data from a table. + /// + /// This test verifies that when we: + /// 1. Create a table with data + /// 2. Add a scalar index with train=true + /// 3. Delete all data in the table + /// The index remains available on the table. + #[rstest] + #[case::btree("i", IndexType::BTree, Box::new(ScalarIndexParams::default()))] + #[case::bitmap("i", IndexType::Bitmap, Box::new(ScalarIndexParams::default()))] + #[case::inverted("text", IndexType::Inverted, Box::new(InvertedIndexParams::default()))] + #[tokio::test] + async fn test_scalar_index_retained_after_delete_all( + #[case] column_name: &str, + #[case] index_type: IndexType, + #[case] params: Box, + ) { + use lance_datagen::{BatchCount, ByteCount, RowCount, array}; + // Create dataset with initial data let reader = lance_datagen::gen_batch() .col("i", array::step::()) @@ -5750,6 +7798,60 @@ mod tests { ); } + /// `initialize_index` must depend only on the keyed prefix of the source + /// index. The rebuild it drives writes fresh, non-covered metadata and never + /// reads the carried columns, so requiring them to exist and type-match in + /// the target rejects a target that can hold the index perfectly well. + #[tokio::test] + async fn test_initialize_index_ignores_carried_columns() { + use crate::dataset::Dataset; + use lance_index::scalar::ScalarIndexParams; + + let source_data = gen_batch() + .col("a", array::step::()) + .col("carried", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut source = Dataset::write(source_data, "memory://source", None) + .await + .unwrap(); + source + .create_index( + &["a"], + IndexType::BTree, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // No creation API writes a covering declaration yet, so commit one. + crate::utils::test::covering::declare_covering(&mut source, "a", "carried").await; + let index_name = source.load_indices().await.unwrap()[0].name.clone(); + + // The target holds the keyed column only -- exactly what the rebuild + // reads. + let target_data = gen_batch() + .col("a", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut target = Dataset::write(target_data, "memory://target", None) + .await + .unwrap(); + + target.initialize_index(&source, &index_name).await.unwrap(); + + let initialized = target.load_indices().await.unwrap(); + assert_eq!(initialized.len(), 1); + assert_eq!( + initialized[0].fields, + vec![target.schema().field("a").unwrap().id] + ); + assert!( + initialized[0].covering_fields.is_empty(), + "the rebuild carries nothing, so it must not claim to" + ); + } + #[tokio::test] async fn test_initialize_single_index() { use crate::dataset::Dataset; @@ -5857,6 +7959,68 @@ mod tests { ); } + #[rstest] + #[case::simple("value", "data.value")] + #[case::quoted("value.with.dot", "data.`value.with.dot`")] + #[tokio::test] + async fn test_initialize_index_on_nested_field( + #[case] nested_field_name: &str, + #[case] nested_field_path: &str, + ) { + let nested_field = Arc::new(Field::new(nested_field_name, DataType::Int32, false)); + let schema = Arc::new(Schema::new(vec![Field::new( + "data", + DataType::Struct(vec![nested_field.clone()].into()), + false, + )])); + let nested_values = arrow_array::StructArray::from(vec![( + nested_field, + Arc::new(Int32Array::from_iter_values(0..10)) as Arc, + )]); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested_values)]).unwrap(); + + let test_dir = TempStrDir::default(); + let source_uri = format!("{}/source", test_dir); + let target_uri = format!("{}/target", test_dir); + + let source_reader = + RecordBatchIterator::new(vec![batch.clone()].into_iter().map(Ok), schema.clone()); + let mut source_dataset = Dataset::write(source_reader, &source_uri, None) + .await + .unwrap(); + source_dataset + .create_index( + &[nested_field_path], + IndexType::BTree, + Some("nested_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + let source_dataset = Dataset::open(&source_uri).await.unwrap(); + + let target_reader = + RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); + let mut target_dataset = Dataset::write(target_reader, &target_uri, None) + .await + .unwrap(); + target_dataset + .initialize_index(&source_dataset, "nested_idx") + .await + .unwrap(); + + let target_indices = target_dataset.load_indices().await.unwrap(); + assert_eq!(target_indices.len(), 1); + assert_eq!( + target_dataset + .schema() + .field_path(target_indices[0].fields[0]) + .unwrap(), + nested_field_path + ); + } + #[tokio::test] async fn test_vector_index_on_nested_field_with_dots() { let dimensions = 16; @@ -6736,168 +8900,351 @@ mod tests { assert!(err.to_string().contains("at least one index segment")); } + /// A segment may carry extra columns; `build_index_metadata_from_segments` + /// must still commit it, keyed on the field the index is being built for. + /// Calls the guarded function directly (not through + /// `commit_existing_index_segments`) so this test's outcome depends only + /// on this guard, not on the independent duplicate a few lines into + /// `commit_existing_index_segments`. #[tokio::test] - async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() { + async fn test_build_index_metadata_from_segments_accepts_carried_fields() { use lance_datagen::{BatchCount, RowCount, array}; let test_dir = tempfile::tempdir().unwrap(); - let test_uri = test_dir.path().to_str().unwrap(); - let reader = lance_datagen::gen_batch() .col("id", array::step::()) .col( "vector", array::rand_vec::(8.into()), ) - .into_reader_rows(RowCount::from(20), BatchCount::from(2)); - - let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); - let field_id = dataset.schema().field("vector").unwrap().id; - let seg0 = write_vector_segment_metadata( + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + let mut metadata = write_vector_segment_metadata( &dataset, "vector_idx", - field_id, + vector_field_id, Uuid::new_v4(), - [0_u32, 1_u32], - b"seg0", + [0_u32], + b"segment", ) .await; - let seg1 = write_vector_segment_metadata( + // Carry `id` alongside the keyed `vector` field. + metadata.fields = vec![vector_field_id, id_field_id]; + metadata.covering_fields = vec![id_field_id]; + + let new_indices = build_index_metadata_from_segments( &dataset, "vector_idx", - field_id, - Uuid::new_v4(), - [1_u32], - b"seg1", + vector_field_id, + vec![segment_from_metadata(&metadata)], ) - .await; + .await + .unwrap(); - let err = dataset - .commit_existing_index_segments( - "vector_idx", - "vector", - vec![segment_from_metadata(&seg0), segment_from_metadata(&seg1)], - ) - .await - .unwrap_err(); - assert!(err.to_string().contains("overlapping fragment coverage")); + assert_eq!(new_indices.len(), 1); + assert_eq!(new_indices[0].fields, vec![vector_field_id, id_field_id]); + assert_eq!(new_indices[0].covering_fields, vec![id_field_id]); } + /// A segment's carried columns can go stale independently of its keyed column, + /// so the staleness check has to walk every entry of `fields`, not just the + /// keyed subtree. Here the carried column did not exist when the segment was + /// built, so the segment cannot be carrying its values and its coverage of that + /// fragment must be pruned. Walking only the keyed subtree sees no change and + /// leaves the fragment covered, which is the bug. #[tokio::test] - async fn test_commit_existing_index_segments_rejects_mixed_index_detail_types() { + async fn test_prune_stale_coverage_notices_a_changed_carried_column() { + use crate::dataset::NewColumnTransform; use lance_datagen::{BatchCount, RowCount, array}; let test_dir = tempfile::tempdir().unwrap(); - let test_uri = test_dir.path().to_str().unwrap(); - let reader = lance_datagen::gen_batch() .col("id", array::step::()) .col( "vector", array::rand_vec::(8.into()), ) - .into_reader_rows(RowCount::from(20), BatchCount::from(2)); - - let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); - let field_id = dataset.schema().field("vector").unwrap().id; - let seg0 = write_vector_segment_metadata( + let vector_field_id = dataset.schema().field("vector").unwrap().id; + // Built against the schema as it stands now, before `payload` exists. + let metadata = write_vector_segment_metadata( &dataset, "vector_idx", - field_id, + vector_field_id, Uuid::new_v4(), [0_u32], - b"seg0", + b"segment", ) .await; - let seg1 = IndexMetadata { - uuid: Uuid::new_v4(), - name: "vector_idx".to_string(), - fields: vec![field_id], - dataset_version: dataset.manifest.version, - fragment_bitmap: Some(std::iter::once(1_u32).collect()), - index_details: Some(Arc::new( - prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap(), - )), - index_version: IndexType::BTree.version(), - created_at: Some(chrono::Utc::now()), - base_id: None, - files: seg0.files.clone(), - }; - let err = dataset - .commit_existing_index_segments("vector_idx", "vector", vec![seg0, seg1]) + // `payload` lands in a new data file on the same fragment, so the fragment's + // file layout changes for `payload` while `vector`'s file is untouched. + dataset + .add_columns( + NewColumnTransform::SqlExpressions(vec![("payload".into(), "id * 2".into())]), + None, + None, + ) .await - .unwrap_err(); + .unwrap(); + let payload_field_id = dataset.schema().field("payload").unwrap().id; + + let mut covered = metadata.clone(); + covered.fields = vec![vector_field_id, payload_field_id]; + covered.covering_fields = vec![payload_field_id]; + + let new_indices = build_index_metadata_from_segments( + &dataset, + "vector_idx", + vector_field_id, + vec![segment_from_metadata(&covered)], + ) + .await + .unwrap(); + + assert_eq!(new_indices.len(), 1); assert!( - err.to_string() - .contains("mixes incompatible index detail types") + new_indices[0].fragment_bitmap.as_ref().unwrap().is_empty(), + "coverage of a fragment whose carried column changed must be pruned, got {:?}", + new_indices[0].fragment_bitmap + ); + + // Control: with nothing carried, the same segment over the same fragment + // keeps its coverage -- so the assertion above is about the carried column, + // not about `add_columns` invalidating everything. + let plain_indices = build_index_metadata_from_segments( + &dataset, + "vector_idx", + vector_field_id, + vec![segment_from_metadata(&metadata)], + ) + .await + .unwrap(); + assert!( + !plain_indices[0] + .fragment_bitmap + .as_ref() + .unwrap() + .is_empty(), + "a non-covering segment must keep its coverage across the same add_columns" ); } + /// The per-segment rules only pin the keyed field and its count, so two + /// segments can disagree about the carried columns and each still pass. That + /// pair must be refused at commit, because `IndexDescriptionImpl::try_new` + /// requires `fields` to be identical across segments -- committing it would + /// leave `describe_indices` erroring on metadata this call just wrote. #[tokio::test] - async fn test_commit_existing_index_segments_rejects_partial_replacement_of_wider_segment() { + async fn test_build_index_metadata_from_segments_rejects_mixed_covering_declarations() { use lance_datagen::{BatchCount, RowCount, array}; let test_dir = tempfile::tempdir().unwrap(); - let test_uri = test_dir.path().to_str().unwrap(); - let reader = lance_datagen::gen_batch() .col("id", array::step::()) .col( "vector", array::rand_vec::(8.into()), ) - .into_reader_rows(RowCount::from(20), BatchCount::from(2)); + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); - let mut dataset = Dataset::write( - reader, - test_uri, - Some(WriteParams { - max_rows_per_file: 20, - max_rows_per_group: 20, - ..Default::default() - }), + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + + let mut covered = write_vector_segment_metadata( + &dataset, + "vector_idx", + vector_field_id, + Uuid::new_v4(), + [0_u32], + b"covered", + ) + .await; + covered.fields = vec![vector_field_id, id_field_id]; + covered.covering_fields = vec![id_field_id]; + + // Plain: one keyed field, nothing carried. Passes every per-segment rule + // on its own -- `keyed == 1` and `fields[0]` is the keyed field -- and so + // does `covered`; only comparing them catches the disagreement. + let plain = write_vector_segment_metadata( + &dataset, + "vector_idx", + vector_field_id, + Uuid::new_v4(), + [1_u32], + b"plain", + ) + .await; + assert_eq!(plain.fields, vec![vector_field_id]); + assert!(plain.covering_fields.is_empty()); + + let err = build_index_metadata_from_segments( + &dataset, + "vector_idx", + vector_field_id, + vec![ + segment_from_metadata(&covered), + segment_from_metadata(&plain), + ], ) .await - .unwrap(); - assert_eq!(dataset.get_fragments().len(), 2); + .expect_err("segments disagreeing about carried columns must not commit"); + assert!( + err.to_string().contains("must declare the same columns"), + "unexpected message: {err}" + ); + } - let field_id = dataset.schema().field("vector").unwrap().id; - let original = write_vector_segment_metadata( + /// `fields` lists the carried columns too, but a logical description must + /// advertise only what the index can be searched on. Every binding reads this + /// list as "the index's columns" -- Python's `_default_vector_index_for_column` + /// matches on membership -- so a carried column here hands back the keyed + /// column's model for a query about the carried one. + #[tokio::test] + async fn test_index_description_reports_only_keyed_fields() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + let mut metadata = write_vector_segment_metadata( &dataset, "vector_idx", - field_id, + vector_field_id, Uuid::new_v4(), - [0_u32, 1_u32], - b"original", + [0_u32], + b"segment", ) .await; - dataset - .commit_existing_index_segments("vector_idx", "vector", vec![original]) + metadata.fields = vec![vector_field_id, id_field_id]; + metadata.covering_fields = vec![id_field_id]; + + let description = IndexDescriptionImpl::try_new(vec![metadata], &dataset) .await .unwrap(); - let replacement = write_vector_segment_metadata( + assert_eq!( + description.field_ids(), + &[vector_field_id as u32], + "a covered index must advertise only its keyed column" + ); + } + + /// A segment keyed on the wrong field must still be rejected. Calls + /// `build_index_metadata_from_segments` directly so the assertion is + /// evidence for *this* guard specifically -- going through + /// `commit_existing_index_segments` would let the untouched-looking + /// duplicate check there mask this guard's removal. + #[tokio::test] + async fn test_build_index_metadata_from_segments_rejects_wrong_field_provenance() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + let mut metadata = write_vector_segment_metadata( &dataset, "vector_idx", - field_id, + vector_field_id, Uuid::new_v4(), [0_u32], - b"replacement", + b"segment", ) .await; + metadata.fields = vec![id_field_id]; - let err = dataset - .commit_existing_index_segments("vector_idx", "vector", vec![replacement]) + let error = build_index_metadata_from_segments( + &dataset, + "vector_idx", + vector_field_id, + vec![segment_from_metadata(&metadata)], + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("expected keyed field")); + } + + /// Matching cardinality is not enough: `covering_fields` must actually be + /// the trailing slice of `fields`. `fields = [vector, id]`, + /// `covering_fields = [999]` has exactly one keyed field but carries a + /// column that is not even present in `fields`, let alone its suffix. + #[tokio::test] + async fn test_build_index_metadata_from_segments_rejects_non_suffix_covering_fields() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) .await - .unwrap_err(); - assert!(err.to_string().contains("would orphan fragments")); + .unwrap(); + + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + let mut metadata = write_vector_segment_metadata( + &dataset, + "vector_idx", + vector_field_id, + Uuid::new_v4(), + [0_u32], + b"segment", + ) + .await; + metadata.fields = vec![vector_field_id, id_field_id]; + metadata.covering_fields = vec![999]; + + let error = build_index_metadata_from_segments( + &dataset, + "vector_idx", + vector_field_id, + vec![segment_from_metadata(&metadata)], + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("are not among its fields")); } #[tokio::test] - async fn test_commit_existing_index_segments_removes_empty_segment() { + async fn test_commit_existing_index_segments_rejects_wrong_field_provenance() { use lance_datagen::{BatchCount, RowCount, array}; let test_dir = tempfile::tempdir().unwrap(); @@ -6911,59 +9258,78 @@ mod tests { let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) .await .unwrap(); - let field_id = dataset.schema().field("vector").unwrap().id; - let uuid = Uuid::new_v4(); - // Commit a 0-fragment segment, then a real segment covering the dataset. - let empty = write_vector_segment_metadata( + let vector_field_id = dataset.schema().field("vector").unwrap().id; + let mut metadata = write_vector_segment_metadata( &dataset, "vector_idx", - field_id, - uuid, - std::iter::empty::(), - b"empty", + vector_field_id, + Uuid::new_v4(), + [0_u32], + b"segment", ) .await; - dataset + metadata.fields = vec![dataset.schema().field("id").unwrap().id]; + + let error = dataset .commit_existing_index_segments( "vector_idx", "vector", - vec![segment_from_metadata(&empty)], + vec![segment_from_metadata(&metadata)], + ) + .await + .unwrap_err(); + // This end-to-end path is guarded twice (`build_index_metadata_from_segments` + // and a duplicate check in `commit_existing_index_segments` itself), so + // this assertion alone does not prove which one fired -- see + // `test_build_index_metadata_from_segments_rejects_wrong_field_provenance` + // for that. It still proves the public `commit_existing_index_segments` + // entry point rejects this input end-to-end. + assert!(error.to_string().contains("expected keyed field")); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_rejects_future_dataset_version() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col( + "vector", + array::rand_vec::(8.into()), ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) .await .unwrap(); - let seg = write_vector_segment_metadata( + + let field_id = dataset.schema().field("vector").unwrap().id; + let mut metadata = write_vector_segment_metadata( &dataset, "vector_idx", field_id, Uuid::new_v4(), [0_u32], - b"seg", + b"segment", ) .await; - dataset + metadata.dataset_version = dataset.manifest.version + 1; + + let error = dataset .commit_existing_index_segments( "vector_idx", "vector", - vec![segment_from_metadata(&seg)], + vec![segment_from_metadata(&metadata)], ) .await - .unwrap(); - - // The real segment covers the dataset, so the redundant empty one is removed. - let committed = dataset.load_indices_by_name("vector_idx").await.unwrap(); - assert_eq!( - committed.iter().map(|i| i.uuid).collect::>(), - HashSet::from([seg.uuid]), - "empty segment should be removed once a real segment covers the dataset", - ); + .unwrap_err(); + assert!(error.to_string().contains("future dataset version")); } #[tokio::test] - async fn test_resolve_index_column_error_cases() { + async fn test_commit_existing_index_segments_rejects_overlapping_fragment_coverage() { use lance_datagen::{BatchCount, RowCount, array}; - // Create a test dataset let test_dir = tempfile::tempdir().unwrap(); let test_uri = test_dir.path().to_str().unwrap(); @@ -6971,1124 +9337,3029 @@ mod tests { .col("id", array::step::()) .col( "vector", - array::rand_vec::(32.into()), + array::rand_vec::(8.into()), ) - .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + .into_reader_rows(RowCount::from(20), BatchCount::from(2)); let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); - // Create an index - let params = crate::index::vector::VectorIndexParams::ivf_flat( - 4, - lance_linalg::distance::MetricType::L2, - ); - dataset - .create_index( - &["vector"], - IndexType::Vector, - Some("my_index".to_string()), - ¶ms, - false, + let field_id = dataset.schema().field("vector").unwrap().id; + let seg0 = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32, 1_u32], + b"seg0", + ) + .await; + let seg1 = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [1_u32], + b"seg1", + ) + .await; + + let err = dataset + .commit_existing_index_segments( + "vector_idx", + "vector", + vec![segment_from_metadata(&seg0), segment_from_metadata(&seg1)], ) .await - .unwrap(); - - // Reload dataset - let dataset = Dataset::open(test_uri).await.unwrap(); - let indices = dataset.load_indices().await.unwrap(); - let index_meta = &indices[0]; - - // Test: Pass a column that doesn't exist and is not the index name - let result = resolve_index_column(dataset.schema(), index_meta, "nonexistent_column"); - assert!(result.is_err()); - let err_msg = result.unwrap_err().to_string(); - assert!( - err_msg.contains("does not exist in the schema"), - "Error message should mention column doesn't exist, got: {}", - err_msg - ); + .unwrap_err(); + assert!(err.to_string().contains("overlapping fragment coverage")); } #[tokio::test] - async fn test_resolve_index_column_nested_field() { - use arrow_array::{RecordBatch, StructArray}; - use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + async fn test_commit_existing_index_segments_rejects_mixed_index_detail_types() { + use lance_datagen::{BatchCount, RowCount, array}; - // Create a test dataset with nested struct manually let test_dir = tempfile::tempdir().unwrap(); let test_uri = test_dir.path().to_str().unwrap(); - // Create schema with nested structure: data.vector - let vector_field = ArrowField::new( - "vector", - DataType::FixedSizeList( - Arc::new(ArrowField::new("item", DataType::Float32, true)), - 8, - ), - false, - ); - let struct_field = ArrowField::new( - "data", - DataType::Struct(vec![vector_field.clone()].into()), - false, - ); - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new("id", DataType::Int32, false), - struct_field, - ])); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(20), BatchCount::from(2)); - // Create data - let id_array = arrow_array::Int32Array::from(vec![1, 2, 3, 4, 5]); + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); - // Create nested vector data - let mut vector_values = Vec::new(); - for _ in 0..5 { - for _ in 0..8 { - vector_values.push(rand::random::()); - } - } - let vector_array = arrow_array::FixedSizeListArray::try_new_from_values( - arrow_array::Float32Array::from(vector_values), - 8, + let field_id = dataset.schema().field("vector").unwrap().id; + let seg0 = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32], + b"seg0", + ) + .await; + let seg1 = IndexMetadata { + uuid: Uuid::new_v4(), + name: "vector_idx".to_string(), + fields: vec![field_id], + covering_fields: vec![], + dataset_version: dataset.manifest.version, + fragment_bitmap: Some(std::iter::once(1_u32).collect()), + index_details: Some(Arc::new( + prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap(), + )), + index_version: IndexType::BTree.version(), + created_at: Some(chrono::Utc::now()), + base_id: None, + files: seg0.files.clone(), + }; + + let err = dataset + .commit_existing_index_segments("vector_idx", "vector", vec![seg0, seg1]) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("mixes incompatible index detail types") + ); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_rejects_partial_replacement_of_wider_segment() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(20), BatchCount::from(2)); + + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 20, + max_rows_per_group: 20, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + let field_id = dataset.schema().field("vector").unwrap().id; + let original = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32, 1_u32], + b"original", + ) + .await; + dataset + .commit_existing_index_segments("vector_idx", "vector", vec![original]) + .await + .unwrap(); + + let replacement = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32], + b"replacement", + ) + .await; + + let err = dataset + .commit_existing_index_segments("vector_idx", "vector", vec![replacement]) + .await + .unwrap_err(); + assert!(err.to_string().contains("would orphan fragments")); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_replaces_different_index_type() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + + let index_name = "shared_name"; + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let original = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("staged_btree".to_string()) + .execute_uncommitted() + .await + .unwrap(); + dataset + .commit_existing_index_segments(index_name, "id", vec![original]) + .await + .unwrap(); + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .name("staged_bitmap".to_string()) + .execute_uncommitted() + .await + .unwrap(); + let replacement_uuid = replacement.uuid; + let replacement_type_url = replacement.index_details.as_ref().unwrap().type_url.clone(); + + dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap(); + + let committed = dataset + .load_index_by_name(index_name) + .await + .unwrap() + .unwrap(); + assert_eq!(committed.uuid, replacement_uuid); + assert_eq!( + committed.index_details.unwrap().type_url, + replacement_type_url + ); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_rejects_partial_index_type_change() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(2)); + let mut dataset = Dataset::write( + reader, + test_dir.path().to_str().unwrap(), + Some(WriteParams { + max_rows_per_file: 20, + max_rows_per_group: 20, + ..Default::default() + }), + ) + .await + .unwrap(); + + let index_name = "shared_name"; + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let mut original_segments = Vec::with_capacity(fragments.len()); + for fragment in &fragments { + original_segments.push( + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(index_name, "id", original_segments) + .await + .unwrap(); + let committed_version = dataset.manifest.version; + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .fragments(vec![fragments[0].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected invalid input error, got: {error}" + ); + assert!( + error + .to_string() + .contains("cannot change index 'shared_name'") + ); + assert!(error.to_string().contains("missing current fragments [1]")); + assert_eq!(dataset.manifest.version, committed_version); + + let committed = + crate::index::scalar_logical::load_named_scalar_segments(&dataset, "id", index_name) + .await + .unwrap(); + assert_eq!(committed.len(), 2); + assert!(committed.iter().all(|segment| { + segment + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("BTreeIndexDetails")) + })); + } + + /// A covered index still names the same column by its keyed prefix. The + /// name-collision guard in `commit_existing_index_segments` must recognize + /// that, not reject on the full `fields` vector including the carried + /// column -- which would misfire "already exists with different fields" + /// before ever reaching the deeper type-compatibility check this test + /// otherwise shares with + /// `test_commit_existing_index_segments_rejects_partial_index_type_change`. + #[tokio::test] + async fn test_commit_existing_index_segments_recognizes_covered_index_by_name() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("extra", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(2)); + let mut dataset = Dataset::write( + reader, + test_dir.path().to_str().unwrap(), + Some(WriteParams { + max_rows_per_file: 20, + max_rows_per_group: 20, + ..Default::default() + }), ) + .await .unwrap(); - let struct_array = StructArray::from(vec![( - Arc::new(vector_field), - Arc::new(vector_array) as arrow_array::ArrayRef, - )]); + let index_name = "shared_name"; + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let mut original_segments = Vec::with_capacity(fragments.len()); + for fragment in &fragments { + original_segments.push( + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(index_name, "id", original_segments) + .await + .unwrap(); + + // Hand-declare the committed index as covering an extra carried column + // -- there is no producer in this phase, so this is done directly. + let id_field = dataset.schema().field("id").unwrap().id; + let extra_field = dataset.schema().field("extra").unwrap().id; + let current = dataset.load_indices_by_name(index_name).await.unwrap(); + let covered_segments = current + .iter() + .cloned() + .map(|mut idx| { + idx.fields = vec![id_field, extra_field]; + idx.covering_fields = vec![extra_field]; + idx + }) + .collect::>(); + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: covered_segments, + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .fragments(vec![fragments[0].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap_err(); + + // Must reach the deeper type-compatibility error, not misfire the + // shallow "already exists with different fields" guard this test + // targets. + assert!( + error + .to_string() + .contains("cannot change index 'shared_name'"), + "{error}" + ); + assert!( + error.to_string().contains("missing current fragments"), + "{error}" + ); + assert!( + !error + .to_string() + .contains("already exists with different fields"), + "{error}" + ); + } + + #[tokio::test] + async fn test_partial_type_change_with_legacy_missing_details_is_rejected() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + + let index_name = "shared_name"; + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let original = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .execute_uncommitted() + .await + .unwrap(); + dataset + .commit_existing_index_segments(index_name, "id", vec![original]) + .await + .unwrap(); + + let current = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(current.len(), 1); + let original_uuid = current[0].uuid; + let mut legacy = current.clone(); + legacy[0].index_details = None; + legacy[0].index_version = 0; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: legacy, + removed_indices: current, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let append_reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write( + append_reader, + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 2); + + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); + let replacement = dataset + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .fragments(vec![fragments[1].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let version_before = dataset.manifest.version; + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![replacement]) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected invalid input error, got: {error}" + ); + assert!(error.to_string().contains("from type ''")); + assert!(error.to_string().contains("partial fragment coverage")); + assert!(error.to_string().contains("missing current fragments [0]")); + assert_eq!(dataset.manifest.version, version_before); + + let committed = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].uuid, original_uuid); + assert!(committed[0].index_details.is_none()); + assert_eq!(committed[0].index_version, 0); + + let field_id = dataset.schema().field("id").unwrap().id; + let sentinel_segment = IndexSegment::new( + Uuid::new_v4(), + [fragments[1].id() as u32], + [field_id], + Arc::new(prost_types::Any { + type_url: "".to_string(), + value: Vec::new(), + }), + 0, + dataset.manifest.version, + vec![], + ); + let error = dataset + .commit_existing_index_segments(index_name, "id", vec![sentinel_segment]) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidInput { .. }), + "expected invalid input error, got: {error}" + ); + assert!(error.to_string().contains("from type ''")); + assert!(error.to_string().contains("to type ''")); + assert!(error.to_string().contains("missing current fragments [0]")); + assert_eq!(dataset.manifest.version, version_before); + + let committed = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].uuid, original_uuid); + assert!(committed[0].index_details.is_none()); + assert_eq!(committed[0].index_version, 0); + } + + #[tokio::test] + async fn test_commit_existing_index_segments_removes_empty_segment() { + use lance_datagen::{BatchCount, RowCount, array}; + + let test_dir = tempfile::tempdir().unwrap(); + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(8.into()), + ) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_dir.path().to_str().unwrap(), None) + .await + .unwrap(); + let field_id = dataset.schema().field("vector").unwrap().id; + let uuid = Uuid::new_v4(); + + // Commit a 0-fragment segment, then a real segment covering the dataset. + let empty = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + uuid, + std::iter::empty::(), + b"empty", + ) + .await; + dataset + .commit_existing_index_segments( + "vector_idx", + "vector", + vec![segment_from_metadata(&empty)], + ) + .await + .unwrap(); + let seg = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32], + b"seg", + ) + .await; + dataset + .commit_existing_index_segments( + "vector_idx", + "vector", + vec![segment_from_metadata(&seg)], + ) + .await + .unwrap(); + + // The real segment covers the dataset, so the redundant empty one is removed. + let committed = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!( + committed.iter().map(|i| i.uuid).collect::>(), + HashSet::from([seg.uuid]), + "empty segment should be removed once a real segment covers the dataset", + ); + } + + #[tokio::test] + async fn test_resolve_index_column_error_cases() { + use lance_datagen::{BatchCount, RowCount, array}; + + // Create a test dataset + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(32.into()), + ) + .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + + // Create an index + let params = crate::index::vector::VectorIndexParams::ivf_flat( + 4, + lance_linalg::distance::MetricType::L2, + ); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("my_index".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + + // Reload dataset + let dataset = Dataset::open(test_uri).await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + let index_meta = &indices[0]; + + // Test: Pass a column that doesn't exist and is not the index name + let result = resolve_index_column(dataset.schema(), index_meta, "nonexistent_column"); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("does not exist in the schema"), + "Error message should mention column doesn't exist, got: {}", + err_msg + ); + } + + #[tokio::test] + async fn test_resolve_index_column_nested_field() { + use arrow_array::{RecordBatch, StructArray}; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + + // Create a test dataset with nested struct manually + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + // Create schema with nested structure: data.vector + let vector_field = ArrowField::new( + "vector", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 8, + ), + false, + ); + let struct_field = ArrowField::new( + "data", + DataType::Struct(vec![vector_field.clone()].into()), + false, + ); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, false), + struct_field, + ])); + + // Create data + let id_array = arrow_array::Int32Array::from(vec![1, 2, 3, 4, 5]); + + // Create nested vector data + let mut vector_values = Vec::new(); + for _ in 0..5 { + for _ in 0..8 { + vector_values.push(rand::random::()); + } + } + let vector_array = arrow_array::FixedSizeListArray::try_new_from_values( + arrow_array::Float32Array::from(vector_values), + 8, + ) + .unwrap(); + + let struct_array = StructArray::from(vec![( + Arc::new(vector_field), + Arc::new(vector_array) as arrow_array::ArrayRef, + )]); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(id_array), Arc::new(struct_array)], + ) + .unwrap(); + + let reader = Box::new(arrow_array::RecordBatchIterator::new( + vec![Ok(batch)], + schema, + )); + + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + + // Create an index on the nested field + let params = crate::index::vector::VectorIndexParams::ivf_flat( + 2, + lance_linalg::distance::MetricType::L2, + ); + dataset + .create_index( + &["data.vector"], + IndexType::Vector, + Some("nested_vector_index".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + + // Reload dataset to get the index metadata + let dataset = Dataset::open(test_uri).await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + let index_meta = &indices[0]; + + // Test 1: Pass the nested field path directly + let (field_path, field) = + resolve_index_column(dataset.schema(), index_meta, "data.vector").unwrap(); + assert_eq!(field_path, "data.vector"); + assert_eq!(field.name, "vector"); + + // Test 2: Pass the index name, should resolve to the nested field path + let (field_path2, field2) = + resolve_index_column(dataset.schema(), index_meta, "nested_vector_index").unwrap(); + assert_eq!(field_path2, "data.vector"); + assert_eq!(field2.name, "vector"); + + // Verify the field path is correct for nested access + assert!( + field_path2.contains('.'), + "Field path should contain '.' for nested field" + ); + } + + #[tokio::test] + async fn test_scalar_index_file_sizes_captured() { + // Test that file sizes are captured when creating a scalar index + let reader = gen_batch() + .col("id", array::step::()) + .col("values", array::rand_utf8(ByteCount::from(10), false)) + .into_reader_rows(RowCount::from(4), BatchCount::from(1)); + + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + // Create a scalar index + dataset + .create_index( + &["values"], + IndexType::Scalar, + Some("test_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Get index metadata and verify files are populated + let indices = dataset.load_indices().await.unwrap(); + let test_index = indices.iter().find(|idx| idx.name == "test_idx").unwrap(); + + assert!( + test_index.files.is_some(), + "Index should have files populated" + ); + let files = test_index.files.as_ref().unwrap(); + assert!(!files.is_empty(), "Index should have at least one file"); + + // Verify each file has a positive size + for file in files { + assert!( + file.size_bytes > 0, + "File {} should have positive size", + file.path + ); + } + + // Verify total_size_bytes works + let total_size = test_index.total_size_bytes(); + assert!(total_size.is_some(), "total_size_bytes should return Some"); + assert!(total_size.unwrap() > 0, "Total size should be positive"); + } + + #[tokio::test] + async fn test_vector_index_file_sizes_captured() { + // Test that file sizes are captured when creating a vector index + let reader = gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::(4.into()), + ) + .into_reader_rows(RowCount::from(300), BatchCount::from(1)); + + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + // Create vector index + let params = VectorIndexParams::ivf_pq(1, 8, 2, MetricType::L2, 2); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("test_vec_idx".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + + // Get index metadata and verify files are populated + let indices = dataset.load_indices().await.unwrap(); + let test_index = indices + .iter() + .find(|idx| idx.name == "test_vec_idx") + .unwrap(); + + assert!( + test_index.files.is_some(), + "Index should have files populated" + ); + let files = test_index.files.as_ref().unwrap(); + assert!(!files.is_empty(), "Index should have at least one file"); + + // Verify each file has a positive size + for file in files { + assert!( + file.size_bytes > 0, + "File {} should have positive size", + file.path + ); + } + + // Verify total_size_bytes works + let total_size = test_index.total_size_bytes(); + assert!(total_size.is_some(), "total_size_bytes should return Some"); + assert!(total_size.unwrap() > 0, "Total size should be positive"); + } + + #[tokio::test] + async fn test_describe_indices_total_size() { + // Test that describe_indices returns total_size_bytes + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("values", DataType::Utf8, false), + ])); + + let values = StringArray::from_iter_values(["hello", "world", "foo", "bar"]); + let record_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..4)), + Arc::new(values), + ], + ) + .unwrap(); + + let reader = + RecordBatchIterator::new(vec![record_batch].into_iter().map(Ok), schema.clone()); + + let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + + // Create a scalar index + dataset + .create_index( + &["values"], + IndexType::Scalar, + Some("test_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Use describe_indices to get index info + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions.len(), 1); + + let desc = &descriptions[0]; + assert_eq!(desc.name(), "test_idx"); + assert_eq!(desc.rows_indexed(), 4); + + // Verify total_size_bytes is available + let total_size = desc.total_size_bytes(); + assert!(total_size.is_some(), "total_size_bytes should be Some"); + assert!(total_size.unwrap() > 0, "Total size should be positive"); + } + + #[tokio::test] + async fn test_describe_indices_rows_indexed_multi_fragment() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("values", array::rand_utf8(ByteCount::from(8), false)) + .into_reader_rows(RowCount::from(10), BatchCount::from(3)); + + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 10, + max_rows_per_group: 10, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.fragments().len(), 3); + + dataset + .create_index( + &["values"], + IndexType::Scalar, + Some("multi_frag_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions.len(), 1); + assert_eq!(descriptions[0].name(), "multi_frag_idx"); + assert_eq!( + descriptions[0].rows_indexed(), + 30, + "rows_indexed should sum logical rows across all indexed fragments" + ); + } + + #[tokio::test] + async fn test_describe_indices_rows_indexed_with_deletions() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("values", array::rand_utf8(ByteCount::from(8), false)) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + dataset.delete("id >= 15").await.unwrap(); + assert_eq!(dataset.count_rows(None).await.unwrap(), 15); + + dataset + .create_index( + &["values"], + IndexType::Scalar, + Some("deleted_rows_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions.len(), 1); + assert_eq!(descriptions[0].name(), "deleted_rows_idx"); + assert_eq!( + descriptions[0].rows_indexed(), + 15, + "rows_indexed should use logical rows (physical_rows - num_deleted_rows)" + ); + } + + #[tokio::test] + async fn test_describe_indices_rows_indexed_stale_bitmap_fragment() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + let reader = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("vector", array::rand_vec::(8.into())) + .into_reader_rows(RowCount::from(10), BatchCount::from(2)); + + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 10, + max_rows_per_group: 10, + ..Default::default() + }), + ) + .await + .unwrap(); + assert_eq!(dataset.fragments().len(), 2); + + let field_id = dataset.schema().field("vector").unwrap().id; + let stale_segment = write_vector_segment_metadata( + &dataset, + "vector_idx", + field_id, + Uuid::new_v4(), + [0_u32, 1_u32, 999_u32], + b"stale-bitmap-segment", + ) + .await; + + dataset + .commit_existing_index_segments( + "vector_idx", + "vector", + vec![segment_from_metadata(&stale_segment)], + ) + .await + .unwrap(); + + let descriptions = dataset.describe_indices(None).await.unwrap(); + assert_eq!(descriptions.len(), 1); + assert_eq!(descriptions[0].name(), "vector_idx"); + assert_eq!( + descriptions[0].rows_indexed(), + 20, + "stale bitmap entries for missing fragments should be skipped without failing" + ); + } + + /// Helper to assert that all indices have file sizes populated + async fn assert_all_indices_have_files(dataset: &Dataset, context: &str) { + let indices = dataset.load_indices().await.unwrap(); + for index in indices.iter() { + // Skip system indices (mem_wal, frag_reuse) which don't have files + if index.name == lance_index::mem_wal::MEM_WAL_INDEX_NAME + || index.name == lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME + { + continue; + } + assert!( + index.files.is_some(), + "{}: Index '{}' should have files field populated", + context, + index.name + ); + let files = index.files.as_ref().unwrap(); + assert!( + !files.is_empty(), + "{}: Index '{}' should have at least one file", + context, + index.name + ); + for file in files { + assert!( + file.size_bytes > 0, + "{}: Index '{}' file '{}' should have positive size", + context, + index.name, + file.path + ); + } + } + } + + #[tokio::test] + async fn test_scalar_index_create_does_not_list_files() { + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("category", DataType::Int32, false), + ])); + let ids = Int32Array::from_iter_values(0..128); + let categories = Int32Array::from_iter_values((0..128).map(|value| value % 8)); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(categories)]) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + let io_tracker = dataset.object_store.as_ref().io_tracker().clone(); + + io_tracker.incremental_stats(); + dataset + .create_index( + &["category"], + IndexType::Bitmap, + Some("category_bitmap".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + let stats = io_tracker.incremental_stats(); + let list_stats = list_io_stats(&stats); + assert_io_eq!( + list_stats, + read_iops, + 0, + "new scalar index files should be reported by writer return values" + ); + } + + #[tokio::test] + async fn test_vector_index_create_does_not_list_files() { + let test_dir = TempStrDir::default(); + let dimension = 8; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + dimension, + ), + false, + ), + ])); + let ids = Int32Array::from_iter_values(0..256); + let vectors = (0..256) + .map(|row| { + Some( + (0..dimension) + .map(|dim| Some((row * dimension + dim) as f32)) + .collect::>(), + ) + }) + .collect::>(); + let vector_array = + FixedSizeListArray::from_iter_primitive::(vectors, dimension); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vector_array)]) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + let io_tracker = dataset.object_store.as_ref().io_tracker().clone(); + + io_tracker.incremental_stats(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_ivf_flat".to_string()), + &VectorIndexParams::ivf_flat(4, MetricType::L2), + true, + ) + .await + .unwrap(); + + let stats = io_tracker.incremental_stats(); + let list_stats = list_io_stats(&stats); + assert_io_eq!( + list_stats, + read_iops, + 0, + "new V3 vector index files should be reported by builder return values" + ); + } + + #[tokio::test] + async fn test_index_file_sizes_through_lifecycle() { + use crate::dataset::WriteDestination; + use crate::dataset::optimize::{CompactionOptions, compact_files, remapping}; + use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; + + // Create initial dataset with columns for different index types + let data = gen_batch() + .col("int_col", array::step::()) + .col("str_col", array::rand_utf8(8.into(), false)) + .col( + "vec_col", + array::rand_vec::(Dimension::from(32)), + ) + .into_reader_rows(RowCount::from(1000), BatchCount::from(1)); + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + data, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: 200, // Multiple fragments for compaction + ..Default::default() + }), + ) + .await + .unwrap(); + + // Create BTree index + dataset + .create_index( + &["int_col"], + IndexType::BTree, + Some("btree_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Create Bitmap index + dataset + .create_index( + &["int_col"], + IndexType::Bitmap, + Some("bitmap_idx".to_string()), + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Create Inverted index for text search + dataset + .create_index( + &["str_col"], + IndexType::Inverted, + Some("inverted_idx".to_string()), + &InvertedIndexParams::default(), + false, + ) + .await + .unwrap(); + + // Validate files are populated after creation + assert_all_indices_have_files(&dataset, "after initial creation").await; + + // Append more data + let more_data = gen_batch() + .col("int_col", array::step::()) + .col("str_col", array::rand_utf8(8.into(), false)) + .col( + "vec_col", + array::rand_vec::(Dimension::from(32)), + ) + .into_reader_rows(RowCount::from(500), BatchCount::from(1)); + + Dataset::write( + more_data, + WriteDestination::Dataset(Arc::new(dataset.clone())), + Some(WriteParams { + max_rows_per_file: 200, + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + // Optimize indices (triggers update/merge) + dataset + .optimize_indices(&OptimizeOptions::default()) + .await + .unwrap(); + + // Validate files are still populated after optimize + assert_all_indices_have_files(&dataset, "after optimize_indices").await; + + // Run compaction with deferred remap + let options = CompactionOptions { + target_rows_per_fragment: 500, + defer_index_remap: true, + ..Default::default() + }; + + compact_files(&mut dataset, options.clone(), None) + .await + .unwrap(); + + // Check if frag reuse index exists (indicates remap is needed) + if dataset + .load_index_by_name(FRAG_REUSE_INDEX_NAME) + .await + .unwrap() + .is_some() + { + // Remap each index + remapping::remap_column_index( + &mut dataset, + &["int_col"], + Some("btree_idx".to_string()), + ) + .await + .unwrap(); + + remapping::remap_column_index( + &mut dataset, + &["int_col"], + Some("bitmap_idx".to_string()), + ) + .await + .unwrap(); + + remapping::remap_column_index( + &mut dataset, + &["str_col"], + Some("inverted_idx".to_string()), + ) + .await + .unwrap(); + + // Validate files are populated after remap + assert_all_indices_have_files(&dataset, "after remap").await; + } + } + + #[tokio::test] + async fn test_btree_index_iops() { + // Test that querying a BTree index uses minimal IOPs (no HEAD requests) + let test_dir = TempStrDir::default(); + + // Create dataset with a column suitable for BTree index + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Int32, false), + ])); + + let num_rows = 1000; + let ids = Int32Array::from_iter_values(0..num_rows); + let values = Int32Array::from_iter_values((0..num_rows).map(|i| i % 100)); + + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(values)]).unwrap(); + + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + + // Create BTree index + dataset + .create_index( + &["value"], + IndexType::BTree, + Some("btree_idx".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Re-open dataset fresh to avoid cached state + let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + // Reset IO stats before query + let _ = dataset.object_store.as_ref().io_stats_incremental(); + + // Query using the BTree index + let results = dataset + .scan() + .filter("value = 50") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert!(results.num_rows() > 0); + + // Verify IOPs - should be minimal (no HEAD requests) + let stats = dataset.object_store.as_ref().io_stats_incremental(); + // We expect reads for: index metadata + index pages + data files + // The key assertion is that we don't have extra HEAD requests + assert_io_lt!( + stats, + read_iops, + 10, + "BTree index query should use minimal IOPs" + ); + } + + #[tokio::test] + async fn test_bitmap_index_iops() { + // Test that querying a Bitmap index uses minimal IOPs (no HEAD requests) + let test_dir = TempStrDir::default(); + + // Create dataset with low-cardinality column for Bitmap index + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("category", DataType::Int32, false), + ])); + + let num_rows = 1000; + let ids = Int32Array::from_iter_values(0..num_rows); + // Low cardinality - only 10 unique values + let categories = Int32Array::from_iter_values((0..num_rows).map(|i| i % 10)); + + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(categories)]) + .unwrap(); + + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + + // Create Bitmap index + dataset + .create_index( + &["category"], + IndexType::Bitmap, + Some("bitmap_idx".to_string()), + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Re-open dataset fresh + let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + // Reset IO stats before query + let _ = dataset.object_store.as_ref().io_stats_incremental(); + + // Query using the Bitmap index + let results = dataset + .scan() + .filter("category = 5") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert!(results.num_rows() > 0); + + // Verify IOPs + let stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!( + stats, + read_iops, + 10, + "Bitmap index query should use minimal IOPs" + ); + } + + #[tokio::test] + async fn test_inverted_index_iops() { + // Test that querying an Inverted (FTS) index uses minimal IOPs + let test_dir = TempStrDir::default(); + + // Create dataset with text column for Inverted index + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("text", DataType::Utf8, false), + ])); + + let num_rows = 100; + let ids = Int32Array::from_iter_values(0..num_rows); + let texts = StringArray::from_iter_values((0..num_rows).map(|i| { + if i % 3 == 0 { + format!("hello world document {}", i) + } else if i % 3 == 1 { + format!("goodbye universe text {}", i) + } else { + format!("random content item {}", i) + } + })); + + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(texts)]).unwrap(); + + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + + // Create Inverted index + let params = InvertedIndexParams::default(); + dataset + .create_index( + &["text"], + IndexType::Inverted, + Some("inverted_idx".to_string()), + ¶ms, + true, + ) + .await + .unwrap(); + + // Re-open dataset fresh + let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + // Reset IO stats before query + let _ = dataset.object_store.as_ref().io_stats_incremental(); + + // Query using the Inverted index (full-text search) + let results = dataset + .scan() + .full_text_search(FullTextSearchQuery::new("hello".to_string())) + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert!(results.num_rows() > 0); + + // Verify IOPs. The deferred DocSet loads per-doc num_tokens/row_ids on + // first use rather than eagerly at index open, so a cold (un-prewarmed) + // query opens the docs file on demand — a couple more IOPs than the + // eager path, but constant and only on the first query (prewarm or a + // warm cache serve it with zero IO). + let stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!( + stats, + read_iops, + 18, + "Inverted index query should use minimal IOPs" + ); + } + + #[tokio::test] + async fn test_ivf_pq_index_iops() { + // Test that querying an IVF_PQ vector index uses minimal IOPs + let test_dir = TempStrDir::default(); + + // Create dataset with vector column + let dimension = 32; + let num_rows = 1000; + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + dimension, + ), + false, + ), + ])); + + let ids = Int32Array::from_iter_values(0..num_rows); + let vectors: Vec>>> = (0..num_rows) + .map(|i| { + Some( + (0..dimension) + .map(|j| Some((i * dimension + j) as f32 / 1000.0)) + .collect(), + ) + }) + .collect(); + let vector_array = + FixedSizeListArray::from_iter_primitive::(vectors, dimension); + + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vector_array)]) + .unwrap(); + + let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); + let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + .await + .unwrap(); + + // Create IVF_PQ index + let params = VectorIndexParams::ivf_pq(4, 8, 4, MetricType::L2, 50); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("ivf_pq_idx".to_string()), + ¶ms, + true, + ) + .await + .unwrap(); + + // Re-open dataset fresh + let dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + // Do a full scan to warm up data file metadata + let _ = dataset.scan().try_into_batch().await.unwrap(); + + // Reset IO stats before query + let _ = dataset.object_store.as_ref().io_stats_incremental(); + + // Query using the IVF_PQ index (KNN search) + let query_vector: Vec = (0..dimension).map(|i| i as f32 / 1000.0).collect(); + let results = dataset + .scan() + .nearest("vector", &Float32Array::from(query_vector), 10) + .unwrap() + .nprobes(2) + .try_into_batch() + .await + .unwrap(); + assert!(results.num_rows() > 0); + + // Verify IOPs + let stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_lt!( + stats, + read_iops, + 17, + "IVF_PQ index query should use minimal IOPs" + ); + } + + #[tokio::test] + async fn test_describe_indices_returns_correct_vector_index_type() { + const DIM: i32 = 8; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM), + true, + ), + ])); + let data = generate_random_array(256 * DIM as usize); let batch = RecordBatch::try_new( schema.clone(), - vec![Arc::new(id_array), Arc::new(struct_array)], + vec![ + Arc::new(Int32Array::from_iter_values(0..256)), + Arc::new(FixedSizeListArray::try_new_from_values(data, DIM).unwrap()), + ], ) .unwrap(); - let reader = Box::new(arrow_array::RecordBatchIterator::new( - vec![Ok(batch)], - schema, - )); - - let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + let test_dir = TempStrDir::default(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write(reader, &test_dir, None).await.unwrap(); - // Create an index on the nested field - let params = crate::index::vector::VectorIndexParams::ivf_flat( - 2, - lance_linalg::distance::MetricType::L2, - ); + // Create IVF_FLAT index + let params = VectorIndexParams::ivf_flat(2, MetricType::L2); dataset .create_index( - &["data.vector"], + &["vector"], IndexType::Vector, - Some("nested_vector_index".to_string()), + Some("vector_idx".to_string()), ¶ms, - false, + true, ) .await .unwrap(); - // Reload dataset to get the index metadata - let dataset = Dataset::open(test_uri).await.unwrap(); - let indices = dataset.load_indices().await.unwrap(); - assert_eq!(indices.len(), 1); - let index_meta = &indices[0]; - - // Test 1: Pass the nested field path directly - let (field_path, field) = - resolve_index_column(dataset.schema(), index_meta, "data.vector").unwrap(); - assert_eq!(field_path, "data.vector"); - assert_eq!(field.name, "vector"); - - // Test 2: Pass the index name, should resolve to the nested field path - let (field_path2, field2) = - resolve_index_column(dataset.schema(), index_meta, "nested_vector_index").unwrap(); - assert_eq!(field_path2, "data.vector"); - assert_eq!(field2.name, "vector"); + // Reload dataset and call describe_indices + let dataset = Dataset::open(&test_dir).await.unwrap(); + let descriptions = dataset.describe_indices(None).await.unwrap(); - // Verify the field path is correct for nested access - assert!( - field_path2.contains('.'), - "Field path should contain '.' for nested field" - ); + assert_eq!(descriptions.len(), 1); + let desc = &descriptions[0]; + assert_eq!(desc.name(), "vector_idx"); + // This should be "IVF_FLAT", not "Unknown" + assert_eq!(desc.index_type(), "IVF_FLAT"); + assert!(!desc.field_ids().is_empty()); } + /// FRI-straddle corruption (PR #6610) used to panic in `load_indices`. + /// The fixture is a pre-#6610 dataset where a user index's + /// `fragment_bitmap` only partially covers a rewrite group. After the + /// tolerant-load fix `load_indices` returns Ok; affected old-frag IDs + /// are dropped, no new-frag IDs are inserted, and `validate()` succeeds. #[tokio::test] - async fn test_scalar_index_file_sizes_captured() { - // Test that file sizes are captured when creating a scalar index - let reader = gen_batch() - .col("id", array::step::()) - .col("values", array::rand_utf8(ByteCount::from(10), false)) - .into_reader_rows(RowCount::from(4), BatchCount::from(1)); + async fn test_load_indices_tolerates_fri_straddle() { + let tmp = copy_test_data_to_tmp("fri_straddle_pre_6610/fri_straddle_dataset").unwrap(); + let uri = format!("file://{}", tmp.std_path().display()); + let dataset = Dataset::open(&uri).await.unwrap(); - let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + assert!(!indices.is_empty()); + dataset.validate().await.unwrap(); + } - // Create a scalar index - dataset - .create_index( - &["values"], - IndexType::Scalar, - Some("test_idx".to_string()), - &ScalarIndexParams::default(), - false, - ) - .await - .unwrap(); + /// Any commit reseeds indices via `load_indices` → `build_manifest`, + /// so a single no-op write persists the cleaned bitmap to disk. + #[tokio::test] + async fn test_auto_heal_persists_cleaned_bitmap() { + use lance_table::io::manifest::read_manifest_indexes; - // Get index metadata and verify files are populated - let indices = dataset.load_indices().await.unwrap(); - let test_index = indices.iter().find(|idx| idx.name == "test_idx").unwrap(); + let tmp = copy_test_data_to_tmp("fri_straddle_pre_6610/fri_straddle_dataset").unwrap(); + let uri = format!("file://{}", tmp.std_path().display()); + // Sanity: the on-disk fixture has at least one straddling segment. + let pre = Dataset::open(&uri).await.unwrap(); + let raw_pre = + read_manifest_indexes(&pre.object_store, &pre.manifest_location, &pre.manifest) + .await + .unwrap(); + let cleaned = pre.load_indices().await.unwrap(); + let any_changed = raw_pre + .iter() + .zip(cleaned.iter()) + .any(|(r, c)| r.fragment_bitmap != c.fragment_bitmap); assert!( - test_index.files.is_some(), - "Index should have files populated" + any_changed, + "fixture should have at least one segment whose bitmap is cleaned at load" ); - let files = test_index.files.as_ref().unwrap(); - assert!(!files.is_empty(), "Index should have at least one file"); + drop(pre); - // Verify each file has a positive size - for file in files { - assert!( - file.size_bytes > 0, - "File {} should have positive size", - file.path + // No-op delete commits a fresh manifest seeded from cleaned indices. + let mut dataset = Dataset::open(&uri).await.unwrap(); + dataset.delete("false").await.unwrap(); + + // Reopen and read raw manifest indices: cleaned bitmaps now persisted. + let healed = Dataset::open(&uri).await.unwrap(); + let raw_post = read_manifest_indexes( + &healed.object_store, + &healed.manifest_location, + &healed.manifest, + ) + .await + .unwrap(); + let cleaned_post = healed.load_indices().await.unwrap(); + for (r, c) in raw_post.iter().zip(cleaned_post.iter()) { + assert_eq!( + r.fragment_bitmap, c.fragment_bitmap, + "after auto-heal, raw on-disk bitmap should match cleaned bitmap" ); } - - // Verify total_size_bytes works - let total_size = test_index.total_size_bytes(); - assert!(total_size.is_some(), "total_size_bytes should return Some"); - assert!(total_size.unwrap() > 0, "Total size should be positive"); } - #[tokio::test] - async fn test_vector_index_file_sizes_captured() { - // Test that file sizes are captured when creating a vector index - let reader = gen_batch() - .col("id", array::step::()) - .col( - "vector", - array::rand_vec::(4.into()), - ) - .into_reader_rows(RowCount::from(300), BatchCount::from(1)); + fn two_column_reader() -> impl arrow_array::RecordBatchReader + Send + 'static { + lance_datagen::gen_batch() + .col("id", array::step::()) + .col("payload", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)) + } - let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + #[derive(Debug, Clone, Copy)] + enum UnreadableIndexKind { + NewerVersion, + UnknownType, + } - // Create vector index - let params = VectorIndexParams::ivf_pq(1, 8, 2, MetricType::L2, 2); + /// Make `index_name` unreadable to this build, and give it the full fragment + /// coverage a real index of that name would have. + async fn hide_index_as(dataset: &mut Dataset, index_name: &str, kind: UnreadableIndexKind) { + let current = dataset.load_indices_by_name(index_name).await.unwrap(); + assert_eq!(current.len(), 1); + let mut unreadable = current.clone(); + match kind { + UnreadableIndexKind::NewerVersion => { + unreadable[0].index_version = current[0].index_version + 1; + } + UnreadableIndexKind::UnknownType => { + unreadable[0].index_details = Some(Arc::new(prost_types::Any { + type_url: "type.googleapis.com/example.ForeignIndexDetails".to_string(), + value: Vec::new(), + })); + } + } + unreadable[0].fragment_bitmap = Some(dataset.fragment_bitmap.as_ref().clone()); + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: unreadable, + removed_indices: current, + }, + None, + ); dataset - .create_index( - &["vector"], - IndexType::Vector, - Some("test_vec_idx".to_string()), - ¶ms, - false, - ) + .apply_commit(transaction, &Default::default(), &Default::default()) .await .unwrap(); - - // Get index metadata and verify files are populated - let indices = dataset.load_indices().await.unwrap(); - let test_index = indices - .iter() - .find(|idx| idx.name == "test_vec_idx") - .unwrap(); - - assert!( - test_index.files.is_some(), - "Index should have files populated" - ); - let files = test_index.files.as_ref().unwrap(); - assert!(!files.is_empty(), "Index should have at least one file"); - - // Verify each file has a positive size - for file in files { - assert!( - file.size_bytes > 0, - "File {} should have positive size", - file.path - ); - } - - // Verify total_size_bytes works - let total_size = test_index.total_size_bytes(); - assert!(total_size.is_some(), "total_size_bytes should return Some"); - assert!(total_size.unwrap() > 0, "Total size should be positive"); } - #[tokio::test] - async fn test_describe_indices_total_size() { - // Test that describe_indices returns total_size_bytes - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("values", DataType::Utf8, false), - ])); + /// Raise `index_name` past the version this build can read. + async fn hide_index_from_this_build(dataset: &mut Dataset, index_name: &str) { + hide_index_as(dataset, index_name, UnreadableIndexKind::NewerVersion).await; + } - let values = StringArray::from_iter_values(["hello", "world", "foo", "bar"]); - let record_batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..4)), - Arc::new(values), - ], - ) - .unwrap(); + /// The readable companion every fixture below carries over `payload`. + const READABLE_INDEX: &str = "payload_idx"; - let reader = - RecordBatchIterator::new(vec![record_batch].into_iter().map(Ok), schema.clone()); + /// A dataset carrying an unreadable BTree index over `id` beside an ordinary + /// readable BTree index over `payload`. + /// + /// The readable companion is what makes the filter's selectivity visible: + /// with a single entry, "hid the one it cannot read" and "hid everything" + /// produce the same answer to every assertion in this module. + /// + /// Both indices are committed untrained and given their coverage by hand. + /// Nothing here ever reads them, and training one would take a non-spillable + /// 40 MB reservation out of the session's shared 150 MB pool to sort ten + /// rows - three of those in flight at once is all the pool has room for. + async fn dataset_with_an_unreadable_index( + uri: &str, + index_name: &str, + kind: UnreadableIndexKind, + ) -> Dataset { + let mut dataset = Dataset::write(two_column_reader(), uri, None) + .await + .unwrap(); - let mut dataset = Dataset::write(reader, "memory://", None).await.unwrap(); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name(index_name.to_string()) + .train(false) + .await + .unwrap(); + hide_index_as(&mut dataset, index_name, kind).await; - // Create a scalar index dataset - .create_index( - &["values"], - IndexType::Scalar, - Some("test_idx".to_string()), - &ScalarIndexParams::default(), - false, - ) + .create_index_builder(&["payload"], IndexType::BTree, &btree_params) + .name(READABLE_INDEX.to_string()) + .train(false) .await .unwrap(); + dataset + } - // Use describe_indices to get index info - let descriptions = dataset.describe_indices(None).await.unwrap(); - assert_eq!(descriptions.len(), 1); + /// A dataset carrying an index whose version is newer than this build. + async fn dataset_with_an_index_from_a_newer_build(uri: &str, index_name: &str) -> Dataset { + dataset_with_an_unreadable_index(uri, index_name, UnreadableIndexKind::NewerVersion).await + } - let desc = &descriptions[0]; - assert_eq!(desc.name(), "test_idx"); - assert_eq!(desc.rows_indexed(), 4); + /// Indices the manifest itself carries, bypassing the usable-index filter. + async fn raw_manifest_indices(dataset: &Dataset) -> Vec { + lance_table::io::manifest::read_manifest_indexes( + &dataset.object_store, + &dataset.manifest_location, + &dataset.manifest, + ) + .await + .unwrap() + } - // Verify total_size_bytes is available - let total_size = desc.total_size_bytes(); - assert!(total_size.is_some(), "total_size_bytes should be Some"); - assert!(total_size.unwrap() > 0, "Total size should be positive"); + /// The manifest entry named `name`, whole. Compare these, not names: a + /// carried-forward index that kept its name but lost its coverage or gained + /// a new uuid is exactly the corruption this suite exists to catch. + async fn manifest_index(dataset: &Dataset, name: &str) -> IndexMetadata { + raw_manifest_indices(dataset) + .await + .into_iter() + .find(|idx| idx.name == name) + .unwrap_or_else(|| panic!("no index named {name} in the manifest")) + } + + /// Sorted: a commit that replaces an entry appends the replacement, so + /// comparing in manifest order would break on which operation ran rather + /// than on what it did. Order is not meaningless in general - delta merging + /// selects a suffix of it - but no test here asserts on it. + async fn manifest_index_names(dataset: &Dataset) -> Vec { + let mut names = raw_manifest_indices(dataset) + .await + .into_iter() + .map(|idx| idx.name) + .collect::>(); + names.sort(); + names } + /// An index this build cannot read must be hidden, not erased. + /// + /// Every commit rebuilds the index list from what it is handed, so filtering + /// the version there turns "ignore it" into "delete it", and the build that + /// could have read the index never gets the chance. #[tokio::test] - async fn test_describe_indices_rows_indexed_multi_fragment() { + async fn test_unsupported_index_survives_an_unrelated_commit() { let test_dir = tempfile::tempdir().unwrap(); let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - let reader = lance_datagen::gen_batch() - .col("id", array::step::()) - .col("values", array::rand_utf8(ByteCount::from(8), false)) - .into_reader_rows(RowCount::from(10), BatchCount::from(3)); + let dataset = Dataset::open(test_uri).await.unwrap(); + assert_eq!( + dataset + .load_indices() + .await + .unwrap() + .iter() + .map(|idx| idx.name.as_str()) + .collect::>(), + [READABLE_INDEX], + "the filter must hide the index this build cannot read, and only it" + ); + assert_eq!( + manifest_index_names(&dataset).await, + ["id_idx", READABLE_INDEX] + ); + let before = manifest_index(&dataset, "id_idx").await; - let mut dataset = Dataset::write( - reader, + // An unrelated append. Nothing about the index is part of this operation. + let dataset = Dataset::write( + two_column_reader(), test_uri, Some(WriteParams { - max_rows_per_file: 10, - max_rows_per_group: 10, + mode: WriteMode::Append, ..Default::default() }), ) .await .unwrap(); - assert_eq!(dataset.fragments().len(), 3); + assert_eq!( + manifest_index(&dataset, "id_idx").await, + before, + "an unrelated append changed an index this build merely could not read" + ); + } + + /// Carrying an unreadable index forward is not the same as keeping it + /// forever: dropping the column it covers still removes it. + #[tokio::test] + async fn test_unsupported_index_is_dropped_with_its_column() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; + + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let readable_before = manifest_index(&dataset, READABLE_INDEX).await; + + dataset.drop_columns(&["id"]).await.unwrap(); + + assert_eq!( + manifest_index_names(&dataset).await, + [READABLE_INDEX], + "dropping `id` must remove the index over it, and only that one" + ); + assert_eq!( + manifest_index(&dataset, READABLE_INDEX).await, + readable_before, + "the index over the surviving column was rewritten" + ); + } + + /// Carrying it forward must also not drag it through index migration. + /// + /// `migrate_indices` recalculates a missing `fragment_bitmap` by opening the + /// index, which is precisely what this build cannot do - so an unreadable + /// index would fail every later commit instead of riding along. + #[rstest] + #[case::newer_version(UnreadableIndexKind::NewerVersion)] + #[case::unknown_type(UnreadableIndexKind::UnknownType)] + #[tokio::test] + async fn test_unsupported_index_without_a_bitmap_does_not_fail_later_commits( + #[case] kind: UnreadableIndexKind, + ) { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let mut dataset = dataset_with_an_unreadable_index(test_uri, "id_idx", kind).await; + + // Drop the coverage too, so migration would want to rebuild it. + let hidden = manifest_index(&dataset, "id_idx").await; + let without_bitmap = IndexMetadata { + fragment_bitmap: None, + ..hidden.clone() + }; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![without_bitmap.clone()], + removed_indices: vec![hidden], + }, + None, + ); dataset - .create_index( - &["values"], - IndexType::Scalar, - Some("multi_frag_idx".to_string()), - &ScalarIndexParams::default(), - false, - ) + .apply_commit(transaction, &Default::default(), &Default::default()) .await .unwrap(); - let descriptions = dataset.describe_indices(None).await.unwrap(); - assert_eq!(descriptions.len(), 1); - assert_eq!(descriptions[0].name(), "multi_frag_idx"); + let mut dataset = Dataset::open(test_uri).await.unwrap(); + dataset.delete("false").await.unwrap(); + assert_eq!( - descriptions[0].rows_indexed(), - 30, - "rows_indexed should sum logical rows across all indexed fragments" + manifest_index(&dataset, "id_idx").await, + without_bitmap, + "a commit rewrote an index it cannot open instead of carrying it through" ); } + /// A name an unreadable index already owns cannot be handed out again. + /// + /// Nothing in the format stops two entries from sharing a name, and the + /// build that can read both would take them for segments of one index. #[tokio::test] - async fn test_describe_indices_rows_indexed_with_deletions() { + async fn test_unsupported_index_name_is_still_taken() { let test_dir = tempfile::tempdir().unwrap(); let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - let reader = lance_datagen::gen_batch() - .col("id", array::step::()) - .col("values", array::rand_utf8(ByteCount::from(8), false)) - .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let err = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .await + .expect_err("a name is taken by an index this build cannot read"); + assert!( + err.to_string().contains("already exists"), + "expected a name collision, got: {err}" + ); + assert_eq!( + manifest_index_names(&dataset).await, + ["id_idx", READABLE_INDEX] + ); + } - let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); - dataset.delete("id >= 15").await.unwrap(); - assert_eq!(dataset.count_rows(None).await.unwrap(), 15); + /// The generated-name loop reads the same view the collision check does, so + /// a name an unreadable index holds is skipped rather than reused. + #[tokio::test] + async fn test_an_auto_generated_name_skips_an_unsupported_index() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; + // A different index kind on the same column: the loop steps past the + // taken name instead of stopping at the collision check. + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let bitmap_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Bitmap); dataset - .create_index( - &["values"], - IndexType::Scalar, - Some("deleted_rows_idx".to_string()), - &ScalarIndexParams::default(), - false, - ) + .create_index_builder(&["id"], IndexType::Bitmap, &bitmap_params) + .train(false) .await .unwrap(); - let descriptions = dataset.describe_indices(None).await.unwrap(); - assert_eq!(descriptions.len(), 1); - assert_eq!(descriptions[0].name(), "deleted_rows_idx"); assert_eq!( - descriptions[0].rows_indexed(), - 15, - "rows_indexed should use logical rows (physical_rows - num_deleted_rows)" + manifest_index_names(&dataset).await, + ["id_idx", "id_idx_2", READABLE_INDEX], + "the generated name reused one an unreadable index already holds" ); } + /// The multi-segment FM-Index builder reserves names on its own, so it needs + /// the same complete view as the single-segment path. + /// + /// The hidden index is a BTree and the new one an FM index, which is what + /// makes this test specific to the multi-segment builder: its name loop + /// (`index/create.rs`) only steps past a taken name when the *fields* differ, + /// where the single-segment loop also steps past a different index kind. So + /// the single-segment path would quietly settle on `text_idx_2` and succeed; + /// only the multi-segment path keeps `text_idx` and hits the collision. #[tokio::test] - async fn test_describe_indices_rows_indexed_stale_bitmap_fragment() { + async fn test_multi_segment_fmindex_respects_an_unsupported_index_name() { let test_dir = tempfile::tempdir().unwrap(); let test_uri = test_dir.path().to_str().unwrap(); - let reader = lance_datagen::gen_batch() - .col("id", array::step::()) - .col("vector", array::rand_vec::(8.into())) - .into_reader_rows(RowCount::from(10), BatchCount::from(2)); - + let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from(vec!["alpha", "beta", "gamma"]))], + ) + .unwrap(); let mut dataset = Dataset::write( - reader, + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), test_uri, - Some(WriteParams { - max_rows_per_file: 10, - max_rows_per_group: 10, - ..Default::default() - }), + None, ) .await .unwrap(); - assert_eq!(dataset.fragments().len(), 2); - - let field_id = dataset.schema().field("vector").unwrap().id; - let stale_segment = write_vector_segment_metadata( - &dataset, - "vector_idx", - field_id, - Uuid::new_v4(), - [0_u32, 1_u32, 999_u32], - b"stale-bitmap-segment", - ) - .await; + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); dataset - .commit_existing_index_segments( - "vector_idx", - "vector", - vec![segment_from_metadata(&stale_segment)], - ) + .create_index_builder(&["text"], IndexType::BTree, &btree_params) + .name("text_idx".to_string()) + .train(false) .await .unwrap(); + hide_index_from_this_build(&mut dataset, "text_idx").await; - let descriptions = dataset.describe_indices(None).await.unwrap(); - assert_eq!(descriptions.len(), 1); - assert_eq!(descriptions[0].name(), "vector_idx"); - assert_eq!( - descriptions[0].rows_indexed(), - 20, - "stale bitmap entries for missing fragments should be skipped without failing" + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let multi_segment_params = ScalarIndexParams::for_builtin(BuiltinIndexType::Fm) + .with_params(&serde_json::json!({ "num_segments": 2 })); + let err = dataset + .create_index_builder(&["text"], IndexType::Fm, &multi_segment_params) + .train(false) + .await + .expect_err("a name is taken by an index this build cannot read"); + assert!( + err.to_string().contains("already exists"), + "expected a name collision, got: {err}" ); + assert_eq!(manifest_index_names(&dataset).await, ["text_idx"]); } - /// Helper to assert that all indices have file sizes populated - async fn assert_all_indices_have_files(dataset: &Dataset, context: &str) { - let indices = dataset.load_indices().await.unwrap(); - for index in indices.iter() { - // Skip system indices (mem_wal, frag_reuse) which don't have files - if index.name == lance_index::mem_wal::MEM_WAL_INDEX_NAME - || index.name == lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME - { - continue; - } - assert!( - index.files.is_some(), - "{}: Index '{}' should have files field populated", - context, - index.name - ); - let files = index.files.as_ref().unwrap(); - assert!( - !files.is_empty(), - "{}: Index '{}' should have at least one file", - context, - index.name - ); - for file in files { - assert!( - file.size_bytes > 0, - "{}: Index '{}' file '{}' should have positive size", - context, - index.name, - file.path - ); - } - } + /// Being unreadable must not make an index unremovable. + #[tokio::test] + async fn test_unsupported_index_can_be_dropped_by_name() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; + + let mut dataset = Dataset::open(test_uri).await.unwrap(); + dataset.drop_index("id_idx").await.unwrap(); + + assert_eq!( + manifest_index_names(&dataset).await, + [READABLE_INDEX], + "drop_index removed the wrong set of indices" + ); } + /// `replace` has to select the index it replaces from the same complete view + /// the name was reserved against, or it adds a twin instead of replacing. #[tokio::test] - async fn test_scalar_index_create_does_not_list_files() { - let test_dir = TempStrDir::default(); - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("category", DataType::Int32, false), - ])); - let ids = Int32Array::from_iter_values(0..128); - let categories = Int32Array::from_iter_values((0..128).map(|value| value % 8)); - let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(categories)]) - .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); - let mut dataset = Dataset::write(reader, test_dir.as_str(), None) - .await - .unwrap(); - let io_tracker = dataset.object_store.as_ref().io_tracker().clone(); - - io_tracker.incremental_stats(); - dataset - .create_index( - &["category"], - IndexType::Bitmap, - Some("category_bitmap".to_string()), - &ScalarIndexParams::default(), - true, - ) - .await - .unwrap(); + async fn test_replacing_an_unsupported_index_does_not_duplicate_it() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - let stats = io_tracker.incremental_stats(); - let list_stats = list_io_stats(&stats); - assert_io_eq!( - list_stats, - read_iops, - 0, - "new scalar index files should be reported by writer return values" + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .replace(true) + .train(false) + .await + .unwrap(); + + assert_eq!( + manifest_index_names(&dataset).await, + ["id_idx", READABLE_INDEX], + "replace committed a second index under a name already taken" + ); + assert!( + unsupported_index_version(&manifest_index(&dataset, "id_idx").await).is_none(), + "replace kept the unreadable index and discarded the new one" ); } + /// The same selection, through the segment-commit path rather than the + /// builder: full coverage replaces the segments already under that name. #[tokio::test] - async fn test_vector_index_create_does_not_list_files() { - let test_dir = TempStrDir::default(); - let dimension = 8; - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - dimension, - ), - false, - ), - ])); - let ids = Int32Array::from_iter_values(0..256); - let vectors = (0..256) - .map(|row| { - Some( - (0..dimension) - .map(|dim| Some((row * dimension + dim) as f32)) - .collect::>(), - ) - }) - .collect::>(); - let vector_array = - FixedSizeListArray::from_iter_primitive::(vectors, dimension); - let batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vector_array)]) - .unwrap(); - let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); - let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + async fn test_committing_a_segment_beside_an_unsupported_index_replaces_it() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; + + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let mut segment = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .replace(true) + .train(false) + .execute_uncommitted() .await .unwrap(); - let io_tracker = dataset.object_store.as_ref().io_tracker().clone(); - - io_tracker.incremental_stats(); + // Full coverage, so the removal decision goes through the fragment + // overlap branch rather than the empty-bitmap shortcut. Set by hand + // because training it would take 40 MB of the shared pool to sort ten + // rows - see `dataset_with_an_index_from_a_newer_build`. + segment.fragment_bitmap = Some(dataset.fragment_bitmap.as_ref().clone()); dataset - .create_index( - &["vector"], - IndexType::Vector, - Some("vector_ivf_flat".to_string()), - &VectorIndexParams::ivf_flat(4, MetricType::L2), - true, - ) + .commit_existing_index_segments("id_idx", "id", vec![segment]) .await .unwrap(); - let stats = io_tracker.incremental_stats(); - let list_stats = list_io_stats(&stats); - assert_io_eq!( - list_stats, - read_iops, - 0, - "new V3 vector index files should be reported by builder return values" + assert_eq!( + manifest_index_names(&dataset).await, + ["id_idx", READABLE_INDEX], + "the incoming segment was committed beside the unreadable one" + ); + assert!( + unsupported_index_version(&manifest_index(&dataset, "id_idx").await).is_none(), + "the incoming segment did not replace the unreadable one" ); } + /// The retention path itself: a segment on fragments the existing one does + /// not cover is kept beside it, and agreeing declarations are what makes + /// that legal. This is the case the rejection below must not swallow - + /// partial-coverage builds depend on it. #[tokio::test] - async fn test_index_file_sizes_through_lifecycle() { - use crate::dataset::WriteDestination; - use crate::dataset::optimize::{CompactionOptions, compact_files, remapping}; - use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; + async fn test_committing_a_segment_on_disjoint_fragments_keeps_the_existing_one() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); - // Create initial dataset with columns for different index types - let data = gen_batch() - .col("int_col", array::step::()) - .col("str_col", array::rand_utf8(8.into(), false)) - .col( - "vec_col", - array::rand_vec::(Dimension::from(32)), + let mut dataset = Dataset::write(two_column_reader(), test_uri, None) + .await + .unwrap(); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .train(false) + .await + .unwrap(); + // Given by hand: an untrained segment commits with an empty bitmap, which + // takes the zero-coverage removal branch rather than the disjoint one. + // See `dataset_with_an_index_from_a_newer_build` for why nothing here trains. + let untrained = manifest_index(&dataset, "id_idx").await; + let mut existing = untrained.clone(); + existing.fragment_bitmap = Some(dataset.fragment_bitmap.as_ref().clone()); + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![existing.clone()], + removed_indices: vec![untrained], + }, + None, + ), + &Default::default(), + &Default::default(), ) - .into_reader_rows(RowCount::from(1000), BatchCount::from(1)); + .await + .unwrap(); + let covered = existing.fragment_bitmap.clone().unwrap(); + assert!(!covered.is_empty()); - let test_dir = TempStrDir::default(); let mut dataset = Dataset::write( - data, - test_dir.as_str(), + two_column_reader(), + test_uri, Some(WriteParams { - max_rows_per_file: 200, // Multiple fragments for compaction + mode: WriteMode::Append, ..Default::default() }), ) .await .unwrap(); - - // Create BTree index - dataset - .create_index( - &["int_col"], - IndexType::BTree, - Some("btree_idx".to_string()), - &ScalarIndexParams::default(), - false, - ) + let appended = dataset.fragment_bitmap.as_ref() - &covered; + assert!(!appended.is_empty()); + + let mut segment = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .replace(true) + .fragments(appended.iter().collect()) + .train(false) + .execute_uncommitted() .await .unwrap(); + segment.fragment_bitmap = Some(appended); + assert_eq!(segment.fields, existing.fields); + assert_eq!(segment.covering_fields, existing.covering_fields); - // Create Bitmap index dataset - .create_index( - &["int_col"], - IndexType::Bitmap, - Some("bitmap_idx".to_string()), - &ScalarIndexParams::default(), - false, - ) + .commit_existing_index_segments("id_idx", "id", vec![segment]) .await .unwrap(); - // Create Inverted index for text search + let uuids = raw_manifest_indices(&dataset) + .await + .into_iter() + .filter(|idx| idx.name == "id_idx") + .map(|idx| idx.uuid) + .collect::>(); + assert_eq!(uuids.len(), 2, "the disjoint existing segment was dropped"); + assert!(uuids.contains(&existing.uuid)); + } + + /// One logical index needs one declaration, and the complete view is what + /// makes the disagreement reachable: a segment this build cannot read may + /// carry columns, and a plain segment committed beside it on disjoint + /// fragments is retained rather than replaced. Both pass the per-segment + /// rules - `keyed_fields` is the prefix left after the carried ones, so + /// `[id, payload]` carrying `[payload]` keys on `id` exactly as `[id]` does. + /// Committing the pair would leave `describe_indices` erroring on metadata + /// this call just wrote, the same failure + /// `test_build_index_metadata_from_segments_rejects_mixed_covering_declarations` + /// pins for the incoming side. + #[tokio::test] + async fn test_committing_a_plain_segment_beside_a_covered_unsupported_one_is_rejected() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let mut dataset = dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; + + // Give the hidden segment a carried column. A newer build is exactly + // where a covering segment would come from. + let hidden = manifest_index(&dataset, "id_idx").await; + let payload_id = dataset.schema().field("payload").unwrap().id; + let mut hidden_covered = hidden.clone(); + hidden_covered.fields.push(payload_id); + hidden_covered.covering_fields = vec![payload_id]; + assert_eq!(hidden_covered.keyed_field(), hidden.keyed_field()); + let covered_fragments = hidden_covered.fragment_bitmap.clone().unwrap(); dataset - .create_index( - &["str_col"], - IndexType::Inverted, - Some("inverted_idx".to_string()), - &InvertedIndexParams::default(), - false, + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![hidden_covered], + removed_indices: vec![hidden], + }, + None, + ), + &Default::default(), + &Default::default(), ) .await .unwrap(); - // Validate files are populated after creation - assert_all_indices_have_files(&dataset, "after initial creation").await; - - // Append more data - let more_data = gen_batch() - .col("int_col", array::step::()) - .col("str_col", array::rand_utf8(8.into(), false)) - .col( - "vec_col", - array::rand_vec::(Dimension::from(32)), - ) - .into_reader_rows(RowCount::from(500), BatchCount::from(1)); - - Dataset::write( - more_data, - WriteDestination::Dataset(Arc::new(dataset.clone())), + // Fragments the hidden segment does not cover, so the incoming segment + // takes the disjoint branch and the hidden one is retained. + let mut dataset = Dataset::write( + two_column_reader(), + test_uri, Some(WriteParams { - max_rows_per_file: 200, mode: WriteMode::Append, ..Default::default() }), ) .await .unwrap(); - - dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() + let appended = dataset.fragment_bitmap.as_ref() - &covered_fragments; + assert!(!appended.is_empty()); + + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + let mut plain = dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .replace(true) + .fragments(appended.iter().collect()) + .train(false) + .execute_uncommitted() .await .unwrap(); + plain.fragment_bitmap = Some(appended); + assert!(plain.covering_fields.is_empty()); - // Optimize indices (triggers update/merge) - dataset - .optimize_indices(&OptimizeOptions::default()) + let err = dataset + .commit_existing_index_segments("id_idx", "id", vec![plain]) .await - .unwrap(); + .expect_err("a logical index cannot mix covered and plain segment declarations"); + assert!( + err.to_string().contains("covering_fields"), + "unexpected error: {err}" + ); + } - // Validate files are still populated after optimize - assert_all_indices_have_files(&dataset, "after optimize_indices").await; + /// A cast reassigns the field id, so no index on that column can be carried + /// forward. The guard that makes that explicit has to see the hidden ones + /// too, or they get exactly the silent drop it exists to abolish. + #[tokio::test] + async fn test_casting_a_column_with_an_unsupported_index_is_rejected() { + use crate::dataset::ColumnAlteration; - // Run compaction with deferred remap - let options = CompactionOptions { - target_rows_per_fragment: 500, - defer_index_remap: true, - ..Default::default() - }; + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - compact_files(&mut dataset, options.clone(), None) + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let err = dataset + .alter_columns(&[ColumnAlteration::new("id".into()).cast_to(DataType::Int64)]) .await - .unwrap(); + .expect_err("a cast must not silently erase an index it cannot read"); + assert!( + err.to_string().contains("id_idx"), + "the error should name the index, got: {err}" + ); + assert_eq!( + manifest_index_names(&dataset).await, + ["id_idx", READABLE_INDEX], + "the rejected cast still changed the manifest" + ); + } - // Check if frag reuse index exists (indicates remap is needed) - if dataset - .load_index_by_name(FRAG_REUSE_INDEX_NAME) - .await - .unwrap() - .is_some() - { - // Remap each index - remapping::remap_column_index( - &mut dataset, - &["int_col"], - Some("btree_idx".to_string()), - ) - .await - .unwrap(); + /// A cache entry written under the key's previous meaning must cold-miss. + /// + /// v1 of `lance.index.metadata-key` held only the indices the writing build + /// could read. The key fields are identical, so on a persistent backend + /// shared with such a build nothing but the schema version stops this one + /// from reading that filtered list as the complete one. + #[tokio::test] + async fn test_a_pre_rotation_cache_entry_is_not_consulted() { + use lance_core::cache::{CacheCodec, CacheKey, CacheKeySchema, KeyBuilder}; + use std::borrow::Cow; - remapping::remap_column_index( - &mut dataset, - &["int_col"], - Some("bitmap_idx".to_string()), - ) - .await - .unwrap(); + struct PreRotationIndexMetadataKey<'a> { + version: u64, + store_identity: &'a str, + } - remapping::remap_column_index( - &mut dataset, - &["str_col"], - Some("inverted_idx".to_string()), - ) - .await - .unwrap(); + impl CacheKey for PreRotationIndexMetadataKey<'_> { + type ValueType = Vec; - // Validate files are populated after remap - assert_all_indices_have_files(&dataset, "after remap").await; - } - } + fn key(&self) -> Cow<'_, str> { + Cow::Owned(format!( + "{}:{}/{}", + self.store_identity.len(), + self.store_identity, + self.version + )) + } - #[tokio::test] - async fn test_btree_index_iops() { - // Test that querying a BTree index uses minimal IOPs (no HEAD requests) - let test_dir = TempStrDir::default(); + fn type_name() -> &'static str { + "Vec" + } - // Create dataset with a column suitable for BTree index - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("value", DataType::Int32, false), - ])); + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.metadata-key", 1) + } - let num_rows = 1000; - let ids = Int32Array::from_iter_values(0..num_rows); - let values = Int32Array::from_iter_values((0..num_rows).map(|i| i % 100)); + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.store_identity); + builder.write_u64(self.version); + } - let batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(values)]).unwrap(); + fn codec() -> Option { + Some(lance_table::format::index_metadata_codec()) + } + } - let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); - let mut dataset = Dataset::write(reader, test_dir.as_str(), None) - .await - .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - // Create BTree index + let mut dataset = Dataset::open(test_uri).await.unwrap(); + let complete = raw_manifest_indices(&dataset).await; + let as_a_released_build_would_cache_it = complete + .iter() + .filter(|idx| unsupported_index_version(idx).is_none()) + .cloned() + .collect::>(); + assert!( + as_a_released_build_would_cache_it.len() < complete.len(), + "the fixture must give the two key versions different values to cache" + ); dataset - .create_index( - &["value"], - IndexType::BTree, - Some("btree_idx".to_string()), - &ScalarIndexParams::default(), - true, + .index_cache + .insert_with_key( + &PreRotationIndexMetadataKey { + version: dataset.version().version, + store_identity: &dataset.object_store.store_prefix, + }, + Arc::new(as_a_released_build_would_cache_it), ) - .await - .unwrap(); + .await; - // Re-open dataset fresh to avoid cached state - let dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() - .await - .unwrap(); + dataset.delete("false").await.unwrap(); - // Reset IO stats before query - let _ = dataset.object_store.as_ref().io_stats_incremental(); + assert_eq!( + manifest_index_names(&dataset).await, + ["id_idx", READABLE_INDEX], + "a commit read a cache entry written under the key's previous meaning" + ); + } - // Query using the BTree index - let results = dataset - .scan() - .filter("value = 50") - .unwrap() - .try_into_batch() + /// `validate` checks the manifest, so it has to see all of it. An index this + /// build cannot read is no less corrupt for being unreadable, and now that + /// such an index is carried forward the corrupt state is durable rather than + /// gone at the next commit. + #[tokio::test] + async fn test_validate_sees_an_unsupported_index() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let mut dataset = dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; + dataset.validate().await.unwrap(); + + // A second segment under the same name covering the same fragments. Only + // `detect_overlapping_fragments` over the complete list can see it. + let hidden = manifest_index(&dataset, "id_idx").await; + let overlapping = IndexMetadata { + uuid: Uuid::new_v4(), + ..hidden + }; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![overlapping], + removed_indices: vec![], + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) .await .unwrap(); - assert!(results.num_rows() > 0); - // Verify IOPs - should be minimal (no HEAD requests) - let stats = dataset.object_store.as_ref().io_stats_incremental(); - // We expect reads for: index metadata + index pages + data files - // The key assertion is that we don't have extra HEAD requests - assert_io_lt!( - stats, - read_iops, - 10, - "BTree index query should use minimal IOPs" + let err = dataset + .validate() + .await + .expect_err("two segments of one name covering the same fragments is corrupt"); + assert!( + err.to_string().contains("id_idx"), + "the error should name the index, got: {err}" ); } + /// A detached commit builds its manifest through its own code path, and it + /// carries the index list forward exactly as an attached one does. #[tokio::test] - async fn test_bitmap_index_iops() { - // Test that querying a Bitmap index uses minimal IOPs (no HEAD requests) - let test_dir = TempStrDir::default(); - - // Create dataset with low-cardinality column for Bitmap index - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("category", DataType::Int32, false), - ])); + async fn test_a_detached_commit_does_not_erase_an_unsupported_index() { + use crate::dataset::InsertBuilder; + use crate::dataset::write::CommitBuilder; - let num_rows = 1000; - let ids = Int32Array::from_iter_values(0..num_rows); - // Low cardinality - only 10 unique values - let categories = Int32Array::from_iter_values((0..num_rows).map(|i| i % 10)); + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(categories)]) + let batches = two_column_reader() + .collect::, _>>() .unwrap(); - - let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); - let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + let dataset = Arc::new(Dataset::open(test_uri).await.unwrap()); + let before = manifest_index(&dataset, "id_idx").await; + let transaction = InsertBuilder::new(dataset.clone()) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted(batches) + .await + .unwrap(); + let detached = CommitBuilder::new(dataset.clone()) + .with_detached(true) + .execute(transaction) .await .unwrap(); - // Create Bitmap index - dataset - .create_index( - &["category"], - IndexType::Bitmap, - Some("bitmap_idx".to_string()), - &ScalarIndexParams::default(), - true, - ) + assert_eq!( + manifest_index_names(&detached).await, + ["id_idx", READABLE_INDEX], + "a detached commit erased an index this build merely could not read" + ); + assert_eq!( + manifest_index(&detached, "id_idx").await, + before, + "a detached commit rewrote an index this build merely could not read" + ); + assert!(lance_table::format::is_detached_version( + detached.manifest.version + )); + assert_eq!(detached.count_rows(None).await.unwrap(), 20); + } + + /// Compaction bins fragments so that no rewrite group splits an index's + /// coverage, and `recalculate_fragment_bitmap` rejects the plan if one does. + /// Both sides therefore have to count the same indices: planning from the + /// filtered view while the commit carries the complete one makes compaction + /// fail outright on a dataset holding an index from a newer build. + #[tokio::test] + async fn test_compaction_survives_an_unsupported_index() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + + let write_params = WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 5, + ..Default::default() + }; + let mut dataset = Dataset::write(two_column_reader(), test_uri, Some(write_params.clone())) .await .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); - // Re-open dataset fresh - let dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .train(false) .await .unwrap(); + hide_index_from_this_build(&mut dataset, "id_idx").await; - // Reset IO stats before query - let _ = dataset.object_store.as_ref().io_stats_incremental(); + // A fragment the index does not cover, so a bin holding it together with + // the covered ones would split the index's coverage. + let mut dataset = Dataset::write( + two_column_reader(), + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..write_params + }), + ) + .await + .unwrap(); + let before = manifest_index(&dataset, "id_idx").await; - // Query using the Bitmap index - let results = dataset - .scan() - .filter("category = 5") - .unwrap() - .try_into_batch() + let metrics = compact_files(&mut dataset, CompactionOptions::default(), None) .await .unwrap(); - assert!(results.num_rows() > 0); - // Verify IOPs - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_lt!( - stats, - read_iops, - 10, - "Bitmap index query should use minimal IOPs" + // Without a real rewrite the assertions below hold vacuously: an empty + // plan commits nothing and re-reads the manifest it started from. + assert_eq!(metrics.fragments_removed, 4); + assert_eq!(dataset.get_fragments().len(), 2); + + let after = manifest_index(&dataset, "id_idx").await; + assert_eq!(after.uuid, before.uuid); + assert_eq!(after.index_version, before.index_version); + let live = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect::(); + assert!( + !after.fragment_bitmap.unwrap().is_disjoint(&live), + "the surviving index covers only fragments the rewrite deleted" ); } + /// Without stable row ids a rewrite moves every row address, so each index + /// over the rewritten fragments has to be remapped - and remapping one means + /// opening it. A build with no reader for an index cannot remap it, so + /// compacting the fragments it covers would leave it addressing rows that no + /// longer exist. Those fragments are held back from the plan instead; the + /// rest of the table still compacts. + /// + /// The stable-row-id case is the test above: there the fragment-reuse index + /// repairs the coverage afterwards, so nothing has to be held back. + #[rstest] + #[case::newer_version(UnreadableIndexKind::NewerVersion)] + #[case::unknown_type(UnreadableIndexKind::UnknownType)] #[tokio::test] - async fn test_inverted_index_iops() { - // Test that querying an Inverted (FTS) index uses minimal IOPs - let test_dir = TempStrDir::default(); - - // Create dataset with text column for Inverted index - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new("text", DataType::Utf8, false), - ])); - - let num_rows = 100; - let ids = Int32Array::from_iter_values(0..num_rows); - let texts = StringArray::from_iter_values((0..num_rows).map(|i| { - if i % 3 == 0 { - format!("hello world document {}", i) - } else if i % 3 == 1 { - format!("goodbye universe text {}", i) - } else { - format!("random content item {}", i) - } - })); - - let batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(texts)]).unwrap(); + async fn test_compaction_defers_fragments_an_unsupported_index_covers( + #[case] kind: UnreadableIndexKind, + ) { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); - let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); - let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + let write_params = WriteParams { + enable_stable_row_ids: false, + max_rows_per_file: 5, + ..Default::default() + }; + let mut dataset = Dataset::write(two_column_reader(), test_uri, Some(write_params.clone())) .await .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); - // Create Inverted index - let params = InvertedIndexParams::default(); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); dataset - .create_index( - &["text"], - IndexType::Inverted, - Some("inverted_idx".to_string()), - ¶ms, - true, - ) + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .train(false) .await .unwrap(); - - // Re-open dataset fresh - let dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() + hide_index_as(&mut dataset, "id_idx", kind).await; + let covered = manifest_index(&dataset, "id_idx") .await + .fragment_bitmap .unwrap(); - // Reset IO stats before query - let _ = dataset.object_store.as_ref().io_stats_incremental(); + // Two more fragments the hidden index does not cover: they are the ones + // compaction is still free to rewrite. + let mut dataset = Dataset::write( + two_column_reader(), + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..write_params + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); + let before = manifest_index(&dataset, "id_idx").await; - // Query using the Inverted index (full-text search) - let results = dataset - .scan() - .full_text_search(FullTextSearchQuery::new("hello".to_string())) - .unwrap() - .try_into_batch() + let metrics = compact_files(&mut dataset, CompactionOptions::default(), None) .await .unwrap(); - assert!(results.num_rows() > 0); - // Verify IOPs. The deferred DocSet loads per-doc num_tokens/row_ids on - // first use rather than eagerly at index open, so a cold (un-prewarmed) - // query opens the docs file on demand — a couple more IOPs than the - // eager path, but constant and only on the first query (prewarm or a - // warm cache serve it with zero IO). - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_lt!( - stats, - read_iops, - 18, - "Inverted index query should use minimal IOPs" + // The two uncovered fragments coalesce; the two the hidden index covers + // are left alone. Both halves matter: no rewrite at all would satisfy the + // coverage assertion below for the wrong reason. + assert_eq!(metrics.fragments_removed, 2); + assert_eq!(metrics.fragments_added, 1); + let live = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect::(); + assert!( + covered.is_subset(&live), + "a fragment the unreadable index covers was rewritten: covered {covered:?}, live {live:?}" ); - } - - #[tokio::test] - async fn test_ivf_pq_index_iops() { - // Test that querying an IVF_PQ vector index uses minimal IOPs - let test_dir = TempStrDir::default(); - // Create dataset with vector column - let dimension = 32; - let num_rows = 1000; + let after = manifest_index(&dataset, "id_idx").await; + assert_eq!(after.uuid, before.uuid); + assert_eq!(after.fragment_bitmap, before.fragment_bitmap); - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - dimension, - ), - false, - ), - ])); + // Nothing compactable is left outside the held-back set, so the plan is + // empty. That has to be an ordinary no-op: on a table the index covers + // whole - the usual shape - every compaction takes this path. + let metrics = compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 0); + assert_eq!(metrics.fragments_added, 0); + assert_eq!(manifest_index(&dataset, "id_idx").await.uuid, before.uuid); + } - let ids = Int32Array::from_iter_values(0..num_rows); - let vectors: Vec>>> = (0..num_rows) - .map(|i| { - Some( - (0..dimension) - .map(|j| Some((i * dimension + j) as f32 / 1000.0)) - .collect(), - ) - }) - .collect(); - let vector_array = - FixedSizeListArray::from_iter_primitive::(vectors, dimension); + /// Holding back the fragments an unreadable index covers is an optimization + /// in the planner, not the rule: `compact_files_with_planner` takes any + /// planner, `CompactionPlan` is public and serializable, and a distributed + /// driver hands `commit_compaction` results planned on another machine. This + /// takes that last route, so the refusal is pinned to the commit boundary. + /// + /// The other half - that the boundary does not refuse a rewrite the index + /// does not cover - is the test above, which compacts the uncovered + /// fragments of this same shape through `compact_files`. + #[tokio::test] + async fn test_committing_a_compaction_an_unsupported_index_covers_is_rejected() { + use crate::dataset::index::DatasetIndexRemapperOptions; + use crate::dataset::optimize::{CompactionPlan, TaskData, commit_compaction}; - let batch = - RecordBatch::try_new(schema.clone(), vec![Arc::new(ids), Arc::new(vector_array)]) - .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); - let reader = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); - let mut dataset = Dataset::write(reader, test_dir.as_str(), None) + let write_params = WriteParams { + enable_stable_row_ids: false, + max_rows_per_file: 5, + ..Default::default() + }; + let mut dataset = Dataset::write(two_column_reader(), test_uri, Some(write_params.clone())) .await .unwrap(); - // Create IVF_PQ index - let params = VectorIndexParams::ivf_pq(4, 8, 4, MetricType::L2, 50); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); dataset - .create_index( - &["vector"], - IndexType::Vector, - Some("ivf_pq_idx".to_string()), - ¶ms, - true, - ) + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .train(false) .await .unwrap(); - - // Re-open dataset fresh - let dataset = DatasetBuilder::from_uri(test_dir.as_str()) - .load() + hide_index_from_this_build(&mut dataset, "id_idx").await; + let covered = manifest_index(&dataset, "id_idx") .await + .fragment_bitmap .unwrap(); - // Do a full scan to warm up data file metadata - let _ = dataset.scan().try_into_batch().await.unwrap(); + // Two more fragments the hidden index does not cover, so the plan below + // is a genuine selection rather than "every fragment there is". + let mut dataset = Dataset::write( + two_column_reader(), + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..write_params + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 4); + let before = manifest_index(&dataset, "id_idx").await; - // Reset IO stats before query - let _ = dataset.object_store.as_ref().io_stats_incremental(); + let plan = CompactionPlan { + tasks: vec![TaskData { + fragments: dataset + .fragments() + .iter() + .filter(|fragment| covered.contains(fragment.id as u32)) + .cloned() + .collect(), + }], + read_version: dataset.version().version, + options: CompactionOptions::default(), + }; + assert_eq!(plan.tasks[0].fragments.len(), 2); - // Query using the IVF_PQ index (KNN search) - let query_vector: Vec = (0..dimension).map(|i| i as f32 / 1000.0).collect(); - let results = dataset - .scan() - .nearest("vector", &Float32Array::from(query_vector), 10) - .unwrap() - .nprobes(2) - .try_into_batch() - .await - .unwrap(); - assert!(results.num_rows() > 0); + let mut rewrites = Vec::new(); + for task in plan.compaction_tasks() { + rewrites.push(task.execute(&dataset).await.unwrap()); + } - // Verify IOPs - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_lt!( - stats, - read_iops, - 17, - "IVF_PQ index query should use minimal IOPs" + let err = commit_compaction( + &mut dataset, + rewrites, + Arc::new(DatasetIndexRemapperOptions::default()), + &plan.options, + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("id_idx"), + "the refusal has to name the index that blocks the rewrite: {err}" ); + + // Nothing was committed: the fragments the plan named are still there, + // and the index still covers them. + assert_eq!(dataset.get_fragments().len(), 4); + let after = manifest_index(&dataset, "id_idx").await; + assert_eq!(after.uuid, before.uuid); + assert_eq!(after.fragment_bitmap, before.fragment_bitmap); } + /// An index old enough to predate the fragment bitmap has its coverage + /// reconstructed from the version it was written against, so it arrives in + /// that version's fragment-id space. `load_all_indices` moves a *stored* + /// bitmap into the current space through the fragment-reuse index and leaves + /// a `None` one alone, so the reconstruction has to make that move itself. + /// + /// Without it, a deferred compaction is enough to defeat both guards: the + /// coverage still names the fragments the rows moved out of, which no later + /// rewrite can intersect, so the planner stops holding anything back and the + /// commit boundary waves the rewrite through. #[tokio::test] - async fn test_describe_indices_returns_correct_vector_index_type() { - const DIM: i32 = 8; - let schema = Arc::new(Schema::new(vec![ - Field::new("id", DataType::Int32, false), - Field::new( - "vector", - DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), DIM), - true, - ), - ])); + async fn test_reconstructed_coverage_follows_a_deferred_compaction() { + use crate::dataset::index::DatasetIndexRemapperOptions; + use crate::dataset::optimize::{CompactionPlan, TaskData, commit_compaction}; - let data = generate_random_array(256 * DIM as usize); - let batch = RecordBatch::try_new( - schema.clone(), - vec![ - Arc::new(Int32Array::from_iter_values(0..256)), - Arc::new(FixedSizeListArray::try_new_from_values(data, DIM).unwrap()), - ], - ) - .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); - let test_dir = TempStrDir::default(); - let reader = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); - let mut dataset = Dataset::write(reader, &test_dir, None).await.unwrap(); + let write_params = WriteParams { + enable_stable_row_ids: false, + max_rows_per_file: 5, + ..Default::default() + }; + let mut dataset = Dataset::write(two_column_reader(), test_uri, Some(write_params.clone())) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); - // Create IVF_FLAT index - let params = VectorIndexParams::ivf_flat(2, MetricType::L2); + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); dataset - .create_index( - &["vector"], - IndexType::Vector, - Some("vector_idx".to_string()), - ¶ms, - true, - ) + .create_index_builder(&["id"], IndexType::BTree, &btree_params) + .name("id_idx".to_string()) + .train(false) .await .unwrap(); + hide_index_from_this_build(&mut dataset, "id_idx").await; - // Reload dataset and call describe_indices - let dataset = Dataset::open(&test_dir).await.unwrap(); - let descriptions = dataset.describe_indices(None).await.unwrap(); + // Drop the bitmap, which is what an index written before it existed + // looks like: coverage has to be reconstructed from `dataset_version`. + let hidden = manifest_index(&dataset, "id_idx").await; + let legacy = IndexMetadata { + fragment_bitmap: None, + ..hidden.clone() + }; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![legacy], + removed_indices: vec![hidden], + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); - assert_eq!(descriptions.len(), 1); - let desc = &descriptions[0]; - assert_eq!(desc.name(), "vector_idx"); - // This should be "IVF_FLAT", not "Unknown" - assert_eq!(desc.index_type(), "IVF_FLAT"); - assert!(!desc.field_ids().is_empty()); + // Deferred remap is the one compaction a build that cannot read the + // index may run: it hands the repair on through a fragment-reuse index. + // Fragments 0 and 1 become fragment 2, and the index still covers those + // rows - now under a different id. + compact_files( + &mut dataset, + CompactionOptions { + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + let after_defer = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + assert_eq!(after_defer, vec![2]); + + // The commit-boundary half, taken first: the planner half below rewrites + // fragment 2 when the remap is missing, which would leave this nothing + // to ask about and hide whether the guard is sensitive on its own. + let plan = CompactionPlan { + tasks: vec![TaskData { + fragments: dataset.fragments().as_ref().clone(), + }], + read_version: dataset.version().version, + options: CompactionOptions::default(), + }; + let mut rewrites = Vec::new(); + for task in plan.compaction_tasks() { + rewrites.push(task.execute(&dataset).await.unwrap()); + } + let err = commit_compaction( + &mut dataset, + rewrites, + Arc::new(DatasetIndexRemapperOptions::default()), + &plan.options, + ) + .await + .unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("id_idx") && message.contains("[2]"), + "the refusal has to name the index and the fragment in current ids: {message}" + ); + + // Two fragments the index never covered, so the planner half has + // something to compact and cannot pass by finding nothing to do. + let mut dataset = Dataset::write( + two_column_reader(), + test_uri, + Some(WriteParams { + mode: WriteMode::Append, + ..write_params + }), + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 3); + + // The planner half: the appended pair coalesces, fragment 2 is held back. + let metrics = compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + assert_eq!(metrics.fragments_removed, 2); + assert_eq!(metrics.fragments_added, 1); + assert!( + dataset + .get_fragments() + .iter() + .any(|fragment| fragment.id() == 2), + "the fragment the reconstructed coverage maps to was rewritten" + ); } - /// FRI-straddle corruption (PR #6610) used to panic in `load_indices`. - /// The fixture is a pre-#6610 dataset where a user index's - /// `fragment_bitmap` only partially covers a rewrite group. After the - /// tolerant-load fix `load_indices` returns Ok; affected old-frag IDs - /// are dropped, no new-frag IDs are inserted, and `validate()` succeeds. + /// Optimizing a name whose segments this build cannot all read would commit + /// a merged segment overlapping the one it left behind - a state + /// `Dataset::validate` reports as corruption and no later commit can heal. #[tokio::test] - async fn test_load_indices_tolerates_fri_straddle() { - let tmp = copy_test_data_to_tmp("fri_straddle_pre_6610/fri_straddle_dataset").unwrap(); - let uri = format!("file://{}", tmp.std_path().display()); - let dataset = Dataset::open(&uri).await.unwrap(); + async fn test_optimize_skips_an_index_with_an_unsupported_segment() { + let test_dir = tempfile::tempdir().unwrap(); + let test_uri = test_dir.path().to_str().unwrap(); + let mut dataset = dataset_with_an_index_from_a_newer_build(test_uri, "id_idx").await; - let indices = dataset.load_indices().await.unwrap(); - assert!(!indices.is_empty()); + // A readable segment beside the hidden one, under the same name, so the + // group is exactly the mixed case: `id_idx` is now partly readable. + let hidden = manifest_index(&dataset, "id_idx").await; + let readable_sibling = IndexMetadata { + uuid: Uuid::new_v4(), + index_version: hidden.index_version - 1, + fragment_bitmap: Some(RoaringBitmap::new()), + ..hidden + }; + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![readable_sibling], + removed_indices: vec![], + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + let segments_of = |indices: Vec| { + indices + .into_iter() + .filter(|idx| idx.name == "id_idx") + .collect::>() + }; + let before = segments_of(raw_manifest_indices(&dataset).await); + assert_eq!(before.len(), 2, "the fixture must build the mixed group"); + + dataset + .optimize_indices(&OptimizeOptions::default()) + .await + .unwrap(); + + assert_eq!( + segments_of(raw_manifest_indices(&dataset).await), + before, + "optimize touched a name carrying a segment this build cannot read" + ); dataset.validate().await.unwrap(); } - /// Any commit reseeds indices via `load_indices` → `build_manifest`, - /// so a single no-op write persists the cleaned bitmap to disk. #[tokio::test] - async fn test_auto_heal_persists_cleaned_bitmap() { - use lance_table::io::manifest::read_manifest_indexes; + async fn test_optimize_rebuilds_dormant_vector_index_instead_of_merging_stale_rows() { + use crate::dataset::UpdateBuilder; - let tmp = copy_test_data_to_tmp("fri_straddle_pre_6610/fri_straddle_dataset").unwrap(); - let uri = format!("file://{}", tmp.std_path().display()); + let mut dataset = gen_batch() + .col("vec", array::rand_vec::(Dimension::from(4))) + .into_ram_dataset_with_params( + FragmentCount::from(2), + FragmentRowCount::from(3), + Some(WriteParams { + max_rows_per_file: 3, + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + let original = dataset + .scan() + .project(&["vec"]) + .unwrap() + .limit(Some(1), None) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let query = original["vec"].as_fixed_size_list().value(0); - // Sanity: the on-disk fixture has at least one straddling segment. - let pre = Dataset::open(&uri).await.unwrap(); - let raw_pre = - read_manifest_indexes(&pre.object_store, &pre.manifest_location, &pre.manifest) - .await - .unwrap(); - let cleaned = pre.load_indices().await.unwrap(); - let any_changed = raw_pre - .iter() - .zip(cleaned.iter()) - .any(|(r, c)| r.fragment_bitmap != c.fragment_bitmap); + dataset + .create_index( + &["vec"], + IndexType::Vector, + Some("vec_idx".to_string()), + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + // A full rewrite replaces every covered fragment; the definition is + // retained as a dormant segment with empty effective coverage. + let mut updated = UpdateBuilder::new(Arc::new(dataset)) + .set("vec", "array[100.0, 100.0, 100.0, 100.0]") + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset + .as_ref() + .clone(); + let retained = updated.load_indices_by_name("vec_idx").await.unwrap(); + assert_eq!(retained.len(), 1); assert!( - any_changed, - "fixture should have at least one segment whose bitmap is cleaned at load" + retained[0] + .effective_fragment_bitmap(&updated.fragment_bitmap) + .unwrap() + .is_empty() ); - drop(pre); - // No-op delete commits a fresh manifest seeded from cleaned indices. - let mut dataset = Dataset::open(&uri).await.unwrap(); - dataset.delete("false").await.unwrap(); + // Optimize must rebuild from live data and replace the dormant + // segment, not merge its stale postings back in. + updated.optimize_indices(&Default::default()).await.unwrap(); + let rebuilt = updated.load_indices_by_name("vec_idx").await.unwrap(); + assert_eq!(rebuilt.len(), 1); + assert!( + !rebuilt[0] + .effective_fragment_bitmap(&updated.fragment_bitmap) + .unwrap() + .is_empty() + ); - // Reopen and read raw manifest indices: cleaned bitmaps now persisted. - let healed = Dataset::open(&uri).await.unwrap(); - let raw_post = read_manifest_indexes( - &healed.object_store, - &healed.manifest_location, - &healed.manifest, - ) - .await - .unwrap(); - let cleaned_post = healed.load_indices().await.unwrap(); - for (r, c) in raw_post.iter().zip(cleaned_post.iter()) { - assert_eq!( - r.fragment_bitmap, c.fragment_bitmap, - "after auto-heal, raw on-disk bitmap should match cleaned bitmap" - ); - } + let result = updated + .scan() + .nearest("vec", query.as_ref(), 1) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let distance = result["_distance"].as_primitive::().value(0); + assert!( + distance > 1_000.0, + "nearest distance {distance} should reflect the rewritten vectors, not stale postings" + ); } } diff --git a/rust/lance/src/index/api.rs b/rust/lance/src/index/api.rs index f856e9004f3..c17d821b608 100644 --- a/rust/lance/src/index/api.rs +++ b/rust/lance/src/index/api.rs @@ -5,7 +5,9 @@ use std::sync::Arc; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; -use lance_index::{IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions}; +use lance_index::{ + FtsPrewarmResult, IndexParams, IndexType, PrewarmOptions, optimize::OptimizeOptions, +}; use lance_table::format::IndexMetadata; use roaring::RoaringBitmap; use uuid::Uuid; @@ -15,37 +17,52 @@ use crate::{Error, Result}; /// A single physical segment of a logical index. /// /// Each segment is stored independently and will become one manifest entry when committed. -/// The logical index identity (name / target column / dataset version) is provided separately -/// by the commit API. +/// The logical index name is provided separately by the commit API, while physical field and +/// dataset-version provenance travel with the segment. #[derive(Debug, Clone, PartialEq)] pub struct IndexSegment { /// Unique ID of the physical segment. uuid: Uuid, /// The fragments covered by this segment. fragment_bitmap: RoaringBitmap, + /// Field IDs whose physical values are encoded in this segment. + fields: Vec, + /// Field IDs whose values this segment carries but is not keyed on. + /// + /// Always the trailing entries of `fields`. + covering_fields: Vec, /// Metadata specific to the index type. index_details: Arc, /// The on-disk index version for this segment. index_version: i32, + /// Dataset version at which this segment's physical contents were built. + dataset_version: u64, } impl IndexSegment { - /// Create a fully described segment with the given UUID, fragment coverage, and index - /// metadata. - pub fn new( + /// Create a fully described segment with its physical build provenance. + pub fn new( uuid: Uuid, fragment_bitmap: I, + fields: F, index_details: Arc, index_version: i32, + dataset_version: u64, + covering_fields: C, ) -> Self where I: IntoIterator, + F: IntoIterator, + C: IntoIterator, { Self { uuid, fragment_bitmap: fragment_bitmap.into_iter().collect(), + fields: fields.into_iter().collect(), + covering_fields: covering_fields.into_iter().collect(), index_details, index_version, + dataset_version, } } @@ -59,6 +76,37 @@ impl IndexSegment { &self.fragment_bitmap } + pub(crate) fn fragment_bitmap_mut(&mut self) -> &mut RoaringBitmap { + &mut self.fragment_bitmap + } + + /// Return the field IDs whose values are encoded in this segment. + pub fn fields(&self) -> &[i32] { + &self.fields + } + + /// Return the fields whose values this segment carries but is not keyed on. + /// + /// Always the trailing entries of [`Self::fields`]. + pub fn covering_fields(&self) -> &[i32] { + &self.covering_fields + } + + /// Return the single column this segment is keyed on, or `None` when it is + /// keyed on several -- a genuinely composite index -- or on none at all. + /// + /// Mirrors [`IndexMetadata::keyed_field`], including its fail-closed + /// behavior on a declaration longer than [`Self::fields`]: a segment comes + /// from a caller (a distributed build's output, say) that this build never + /// validated. + pub fn keyed_field(&self) -> Option { + let keyed = self.fields.len().saturating_sub(self.covering_fields.len()); + match &self.fields[..keyed] { + [only] => Some(*only), + _ => None, + } + } + /// Return the serialized index details for this segment. pub fn index_details(&self) -> &Arc { &self.index_details @@ -69,13 +117,31 @@ impl IndexSegment { self.index_version } + /// Return the source dataset version for this segment. + pub fn dataset_version(&self) -> u64 { + self.dataset_version + } + /// Consume the segment and return its component parts. - pub fn into_parts(self) -> (Uuid, RoaringBitmap, Arc, i32) { + pub fn into_parts( + self, + ) -> ( + Uuid, + RoaringBitmap, + Vec, + Vec, + Arc, + i32, + u64, + ) { ( self.uuid, self.fragment_bitmap, + self.fields, + self.covering_fields, self.index_details, self.index_version, + self.dataset_version, ) } } @@ -110,8 +176,11 @@ impl IntoIndexSegment for IndexMetadata { Ok(IndexSegment::new( self.uuid, fragment_bitmap.iter(), + self.fields, index_details, self.index_version, + self.dataset_version, + self.covering_fields, )) } } @@ -167,7 +236,55 @@ pub trait DatasetIndexExt { )) } - /// Read all indices of this Dataset version. + /// Prewarm an index by name with additional options and return the structured outcome. + async fn prewarm_index_with_options_result( + &self, + _name: &str, + _options: &PrewarmOptions, + ) -> Result { + Err(Error::not_supported( + "prewarm result reports are not supported by this dataset implementation".to_owned(), + )) + } + + /// Prewarm selected physical segments of an index by name. + async fn prewarm_index_segments(&self, _name: &str, _segment_ids: &[Uuid]) -> Result<()> { + Err(Error::not_supported( + "segment-level prewarm is not supported by this dataset implementation".to_owned(), + )) + } + + /// Prewarm selected physical segments of an index by name with additional options. + async fn prewarm_index_segments_with_options( + &self, + _name: &str, + _segment_ids: &[Uuid], + _options: &PrewarmOptions, + ) -> Result<()> { + Err(Error::not_supported( + "prewarm options are not supported by this dataset implementation".to_owned(), + )) + } + + /// Prewarm selected physical segments with options and return the structured outcome. + async fn prewarm_index_segments_with_options_result( + &self, + _name: &str, + _segment_ids: &[Uuid], + _options: &PrewarmOptions, + ) -> Result { + Err(Error::not_supported( + "prewarm result reports are not supported by this dataset implementation".to_owned(), + )) + } + + /// Read the indices of this Dataset version that this build can use. + /// + /// An index whose declared type has no reader in this build, or whose format + /// version is newer than this build supports, is omitted: it is still in the + /// manifest and still belongs to the dataset, but nothing here can decode + /// it. Code deciding what the *next* manifest should say must not use this + /// list - it would drop what it omits. /// /// The indices are lazy loaded and cached in memory within the `Dataset` instance. /// The cache is invalidated when the dataset version (Manifest) is changed. @@ -254,3 +371,88 @@ pub trait DatasetIndexExt { with_vector: bool, ) -> Result; } + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[test] + fn test_index_metadata_conversion_preserves_provenance() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + name: "test".to_string(), + fields: vec![3, 7], + covering_fields: vec![], + dataset_version: 42, + fragment_bitmap: Some(RoaringBitmap::from_iter([1, 2])), + index_details: Some(Arc::new(prost_types::Any { + type_url: "example.IndexDetails".to_string(), + value: vec![1, 2, 3], + })), + index_version: 5, + created_at: None, + base_id: None, + files: None, + }; + + let segment = metadata.into_index_segment().unwrap(); + assert_eq!(segment.fields(), [3, 7]); + assert_eq!(segment.dataset_version(), 42); + } + + /// Segments arrive from callers this build never validated, so the keyed + /// prefix must fail closed on a declaration longer than `fields` rather than + /// underflow, exactly as [`IndexMetadata::keyed_field`] does. + #[rstest] + #[case::not_covered(vec![7], vec![], Some(7))] + #[case::covered(vec![7, 11], vec![11], Some(7))] + #[case::composite(vec![7, 11], vec![], None)] + #[case::malformed_longer_than_fields(vec![7], vec![11, 13], None)] + fn test_index_segment_keyed_field( + #[case] fields: Vec, + #[case] covering_fields: Vec, + #[case] expected: Option, + ) { + let segment = IndexSegment::new( + Uuid::new_v4(), + [0u32], + fields, + Arc::new(prost_types::Any { + type_url: "test".to_string(), + value: vec![], + }), + 0, + 1, + covering_fields, + ); + + assert_eq!(segment.keyed_field(), expected); + } + + /// A covering declaration must survive the metadata -> segment -> metadata + /// round trip. `IndexSegment` is the hop where it was previously dropped. + #[test] + fn test_index_segment_preserves_covering_fields() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + name: "covered".to_string(), + fields: vec![7, 11], + covering_fields: vec![11], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([0u32])), + index_details: Some(Arc::new(prost_types::Any { + type_url: "test".to_string(), + value: vec![], + })), + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + + let segment = metadata.into_index_segment().unwrap(); + assert_eq!(segment.fields(), &[7, 11]); + assert_eq!(segment.covering_fields(), &[11]); + } +} diff --git a/rust/lance/src/index/append.rs b/rust/lance/src/index/append.rs index d3ecde030c1..8bfb32aec75 100644 --- a/rust/lance/src/index/append.rs +++ b/rust/lance/src/index/append.rs @@ -5,28 +5,41 @@ use std::sync::Arc; use futures::{FutureExt, TryStreamExt}; use lance_core::{Error, Result}; +use lance_file::reader::FileReaderOptions; use lance_index::{ INDEX_FILE_NAME, IndexType, + frag_reuse::CompactFragReuseIndex, metrics::NoOpMetricsCollector, optimize::OptimizeOptions, - progress::NoopIndexBuildProgress, + progress::{IndexBuildProgress, NoopIndexBuildProgress}, scalar::{ - CreatedIndex, OldIndexDataFilter, ScalarIndex, inverted::InvertedIndex, + CreatedIndex, OldIndexDataFilter, ScalarIndex, index_files_to_table, + inverted::InvertedIndex, lance_format::LanceIndexStore, + seed::{FragmentSeed, SEED_META_KEY_PREFIX}, + table_files_to_index, }, }; +use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; +use lance_io::utils::CachedFileSize; use lance_select::{RowAddrTreeMap, RowSetOps}; use lance_table::format::{Fragment, IndexMetadata}; +use prost::Message; use roaring::RoaringBitmap; use uuid::Uuid; -use super::DatasetIndexInternalExt; -use super::vector::LogicalVectorIndex; -use super::vector::ivf::{optimize_vector_indices, select_segment_for_single_rebalance}; +use super::vector::ivf::{ + VectorSegmentCompatibility, index_type_for_segmented_optimize, optimize_vector_indices, + select_segment_for_single_rebalance, vector_segment_compatibility, +}; +use super::vector::{LogicalVectorIndex, fresh_vector_segment_params}; +use super::{CreateIndexBuilder, DatasetIndexInternalExt}; use crate::dataset::Dataset; use crate::dataset::index::LanceIndexStoreExt; use crate::dataset::rowids::load_row_id_sequences; -use crate::index::scalar::load_training_data; +use crate::index::scalar::{ + IndexDetails, fetch_index_details, load_fts_training_data, load_training_data, +}; use crate::index::vector_index_details_default; #[derive(Debug, Clone)] @@ -34,6 +47,7 @@ pub struct IndexMergeResults<'a> { pub new_uuid: Uuid, pub removed_indices: Vec<&'a IndexMetadata>, pub new_fragment_bitmap: RoaringBitmap, + pub new_dataset_version: u64, pub new_index_version: i32, pub new_index_details: prost_types::Any, /// List of files and their sizes for the merged index @@ -147,13 +161,65 @@ pub fn split_segment_coverage<'a>( (effective, deleted) } -/// Build one [`OldIndexDataFilter`] per segment, each derived from that segment's -/// *own* effective (still-live) and retired fragment coverage, plus the union of -/// every segment's still-live coverage. +pub fn fragment_reuse_affects_segments<'a>( + frag_reuse_index: &CompactFragReuseIndex, + segments: impl IntoIterator, +) -> bool { + segments.into_iter().any(|segment| { + let Some(coverage) = segment.fragment_bitmap.as_ref() else { + return false; + }; + fragment_reuse_affects_segment(frag_reuse_index, coverage, segment.dataset_version) + }) +} + +pub fn fragment_reuse_affects_segment( + frag_reuse_index: &CompactFragReuseIndex, + coverage: &RoaringBitmap, + dataset_version: u64, +) -> bool { + frag_reuse_index.details.versions.iter().any(|version| { + version.groups.iter().any(|group| { + if group.changed_row_addrs.is_empty() { + return false; + } + let covers_old = group + .old_frags + .iter() + .any(|fragment| coverage.contains(fragment.id as u32)); + let covers_new = group + .new_frags + .iter() + .any(|fragment| coverage.contains(fragment.id as u32)); + (version.dataset_version >= dataset_version && covers_old) + || (version.dataset_version > dataset_version && covers_new) + }) + }) +} + +/// Build one [`OldIndexDataFilter`] per segment and return their effective coverage. pub async fn build_per_segment_filters( dataset: &Dataset, segments: &[&IndexMetadata], ) -> Result<(RoaringBitmap, Vec>)> { + if dataset.manifest.uses_stable_row_ids() { + let mut effective_union = RoaringBitmap::new(); + let mut filters = Vec::with_capacity(segments.len()); + for segment in segments { + let effective = segment + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing fragment coverage", + segment.uuid + )) + })?; + effective_union |= &effective; + filters.push(build_old_data_filter(dataset, &effective, &RoaringBitmap::new()).await?); + } + return Ok((effective_union, filters)); + } + let mut effective_union = RoaringBitmap::new(); let mut filters = Vec::with_capacity(segments.len()); for segment in segments { @@ -175,6 +241,88 @@ pub async fn build_per_segment_filters( Ok((effective_union, filters)) } +/// Attempt to read seed buffers for `column_name` from `fragments`' data files. +/// +/// Returns `Some(vec)` only if every fragment has a seed entry; returns `None` +/// if any fragment is missing a seed or its data file cannot be opened. +/// Index-type-specific validation (e.g. `rows_per_zone` checks) is left to the +/// caller via [`FragmentSeed::metadata_value`]. +async fn try_harvest_seeds( + dataset: &Dataset, + fragments: &[Fragment], + column_name: &str, +) -> Result>> { + if fragments.is_empty() { + return Ok(Some(Vec::new())); + } + + let meta_key = format!("{}{}", SEED_META_KEY_PREFIX, column_name); + let mut seeds = Vec::with_capacity(fragments.len()); + + let scheduler = ScanScheduler::new( + dataset.object_store.clone(), + SchedulerConfig::max_bandwidth(&dataset.object_store), + ); + + for fragment in fragments { + let Some(data_file) = fragment.files.first() else { + return Ok(None); + }; + + let path = dataset + .base + .clone() + .join(crate::dataset::DATA_DIR) + .join(data_file.path.as_str()); + let Ok(file_scheduler) = scheduler.open_file(&path, &CachedFileSize::unknown()).await + else { + return Ok(None); + }; + + let Ok(reader) = lance_file::reader::FileReader::try_open( + file_scheduler, + None, + Default::default(), + &dataset.metadata_cache.file_metadata_cache(&path), + FileReaderOptions::default(), + ) + .await + else { + return Ok(None); + }; + + let Some(meta_value) = reader + .metadata() + .file_schema + .metadata + .get(&meta_key) + .cloned() + else { + return Ok(None); + }; + + // The buf_index is always the portion before the first ':'. + let Some(buf_index_str) = meta_value.split(':').next() else { + return Ok(None); + }; + let Ok(buf_index) = buf_index_str.parse::() else { + return Ok(None); + }; + + let Ok(bytes) = reader.read_global_buffer(buf_index).await else { + return Ok(None); + }; + + seeds.push(FragmentSeed { + fragment_id: fragment.id, + bytes, + metadata_value: meta_value, + }); + } + + Ok(Some(seeds)) +} + async fn load_unindexed_training_data( dataset: &Dataset, field_path: &str, @@ -293,7 +441,15 @@ async fn merge_scalar_indices<'a>( field_path: &str, column_name: &str, base_unindexed_bitmap: RoaringBitmap, -) -> Result, RoaringBitmap, CreatedIndex)>> { +) -> Result< + Option<( + Uuid, + Vec<&'a IndexMetadata>, + RoaringBitmap, + CreatedIndex, + u64, + )>, +> { if old_indices.is_empty() { return Err(Error::index( "merge_scalar_indices: no previous index found".to_string(), @@ -328,7 +484,12 @@ async fn merge_scalar_indices<'a>( // Scalar Index that expos an N:1 segment-merge primitive reachable without // rescanning the dataset - let has_segment_merge_primitive = matches!(index_type, IndexType::BTree); + let has_segment_merge_primitive = matches!(index_type, IndexType::BTree | IndexType::NGram); + let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let ngram_requires_rebuild = index_type == IndexType::NGram + && frag_reuse_index.as_ref().is_some_and(|frag_reuse_index| { + fragment_reuse_affects_segments(frag_reuse_index, selected_old_indices.iter().copied()) + }); // Merge new data into the existing segment(s) without rebuilding from // scratch, when all hold: @@ -343,53 +504,104 @@ async fn merge_scalar_indices<'a>( // primitive) the index is rebuilt from scratch over `frag_bitmap`. let can_merge_segments = !effective_old_frags.is_empty() && !update_criteria.requires_old_data + && !ngram_requires_rebuild && (has_segment_merge_primitive || selected_old_indices.len() == 1); - let created_index = if !can_merge_segments { - rebuild_scalar_segment( - dataset.as_ref(), - &reference_index, - field_path, - column_name, - new_uuid, - frag_bitmap.iter().collect(), + let (created_index, new_dataset_version) = if !can_merge_segments { + ( + rebuild_scalar_segment( + dataset.as_ref(), + &reference_index, + field_path, + column_name, + new_uuid, + frag_bitmap.iter().collect(), + ) + .await?, + dataset.manifest.version, ) - .await? } else { - let new_data_stream = - load_unindexed_training_data(dataset.as_ref(), field_path, &update_criteria, unindexed) - .await?; let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?; - match index_type { - IndexType::BTree => { - let (_, old_data_filters) = - build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?; - crate::index::scalar::btree::open_and_merge_segments( - dataset.as_ref(), - field_path, - &selected_old_indices, - new_data_stream, - &new_store, - &old_data_filters, - ) - .await? + // Try a seed-based update before falling back to a full column scan. + // Seeds are only available if every unindexed fragment had a seed buffer + // written during its data file write, and the plugin validates that the + // seeds are compatible with the current index configuration. + let index_details = + fetch_index_details(dataset.as_ref(), field_path, reference_idx).await?; + let details = IndexDetails(index_details.clone()); + let plugin = details.get_plugin()?; + // Only open data files looking for seeds when the plugin confirms this + // index type and configuration can actually produce them. + let maybe_created = if plugin.might_use_seeds(&index_details) { + if let Some(seeds) = try_harvest_seeds(dataset.as_ref(), unindexed, column_name).await? + { + plugin + .update_from_seeds(seeds, reference_index.clone(), &index_details, &new_store) + .await? + } else { + None } - // NOTE: IndexType::Inverted never reaches here -- it is handled by the - // dedicated arm in merge_indices_with_unindexed_frags before this - // function is called. - _ => { - let old_data_filter = build_old_data_filter( - dataset.as_ref(), - &effective_old_frags, - &deleted_old_frags, - ) - .await?; - reference_index - .update(new_data_stream, &new_store, old_data_filter) + } else { + None + }; + + let created_index = if let Some(created) = maybe_created { + created + } else { + let new_data_stream = load_unindexed_training_data( + dataset.as_ref(), + field_path, + &update_criteria, + unindexed, + ) + .await?; + + match index_type { + IndexType::BTree => { + let (_, old_data_filters) = + build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?; + crate::index::scalar::btree::open_and_merge_segments( + dataset.as_ref(), + field_path, + &selected_old_indices, + new_data_stream, + &new_store, + &old_data_filters, + ) + .await? + } + IndexType::NGram => { + let (_, old_data_filters) = + build_per_segment_filters(dataset.as_ref(), &selected_old_indices).await?; + crate::index::scalar::ngram::open_and_merge_segments( + dataset.as_ref(), + &selected_old_indices, + Some(new_data_stream), + &new_store, + &old_data_filters, + ) .await? + } + _ => { + let old_data_filter = build_old_data_filter( + dataset.as_ref(), + &effective_old_frags, + &deleted_old_frags, + ) + .await?; + reference_index + .update(new_data_stream, &new_store, old_data_filter) + .await? + } } - } + }; + let source_dataset_version = selected_old_indices + .iter() + .map(|index| index.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + (created_index, source_dataset_version) }; Ok(Some(( @@ -397,6 +609,7 @@ async fn merge_scalar_indices<'a>( selected_old_indices.to_vec(), frag_bitmap, created_index, + new_dataset_version, ))) } @@ -445,6 +658,87 @@ pub async fn merge_indices<'a>( .await } +async fn build_fresh_vector_segment( + dataset: &Dataset, + logical_index: &LogicalVectorIndex, + field_path: &str, + fragment_bitmap: &RoaringBitmap, + progress: Arc, +) -> Result { + let (reference_metadata, reference_index) = logical_index.iter().last().ok_or_else(|| { + Error::index(format!( + "Optimize vector index: logical index '{}' has no physical segments", + logical_index.name() + )) + })?; + let params = fresh_vector_segment_params(reference_metadata, reference_index.as_ref())?; + let mut build_dataset = dataset.clone(); + CreateIndexBuilder::new( + &mut build_dataset, + &[field_path], + IndexType::Vector, + ¶ms, + ) + .name(logical_index.name().to_string()) + .replace(true) + .fragments(fragment_bitmap.iter().collect()) + .progress(progress) + .execute_uncommitted() + .await +} + +async fn scan_vector_fragments( + dataset: &Dataset, + field_path: &str, + column_nullable: bool, + fragments: &[Fragment], +) -> Result { + let mut scanner = dataset.scan(); + scanner + .with_fragments(fragments.to_vec()) + .with_row_id() + .project(&[field_path])?; + if column_nullable { + let column_expr = lance_datafusion::logical_expr::field_path_to_expr(field_path)?; + scanner.filter_expr(column_expr.is_not_null()); + } + scanner.try_into_stream().await +} + +fn fresh_vector_segment_result<'a>( + segment: IndexMetadata, + expected_fragment_bitmap: &RoaringBitmap, + removed_indices: Vec<&'a IndexMetadata>, +) -> Result> { + let fragment_bitmap = segment.fragment_bitmap.ok_or_else(|| { + Error::index( + "Optimize vector index: newly built segment has no fragment bitmap".to_string(), + ) + })?; + if fragment_bitmap != *expected_fragment_bitmap { + return Err(Error::index(format!( + "Optimize vector index: newly built segment covers fragments {:?}, expected {:?}", + fragment_bitmap, expected_fragment_bitmap + ))); + } + let index_details = segment.index_details.ok_or_else(|| { + Error::index("Optimize vector index: newly built segment has no index details".to_string()) + })?; + let files = segment.files.ok_or_else(|| { + Error::index("Optimize vector index: newly built segment has no file metadata".to_string()) + })?; + + Ok(IndexMergeResults { + new_uuid: segment.uuid, + removed_indices, + new_fragment_bitmap: fragment_bitmap, + new_dataset_version: segment.dataset_version, + new_index_version: segment.index_version, + new_index_details: index_details.as_ref().clone(), + files, + }) +} + /// Merge a list of provided unindexed data, with a specific number of previous indices /// into a new index, to improve the query performance. pub async fn merge_indices_with_unindexed_frags<'a>( @@ -467,7 +761,33 @@ pub async fn merge_indices_with_unindexed_frags<'a>( old_indices[0].fields[0] )))?; - let field_path = dataset.schema().field_path(old_indices[0].fields[0])?; + let raw_field_path = dataset.schema().field_path(old_indices[0].fields[0])?; + let first_details = + super::scalar::fetch_index_details(dataset.as_ref(), &raw_field_path, old_indices[0]) + .await?; + let resolved_fts = if first_details.type_url.ends_with("InvertedIndexDetails") { + let details = + lance_index::pbold::InvertedIndexDetails::decode(first_details.value.as_slice()) + .map_err(|error| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {error}" + )) + })?; + let granularity = lance_index::scalar::inverted::DocumentGranularity::try_from( + details.document_granularity, + )?; + Some(crate::index::scalar::inverted::resolve_fts_field_by_id( + dataset.schema(), + old_indices[0].fields[0], + granularity, + )?) + } else { + None + }; + let field_path = resolved_fts + .as_ref() + .map(|resolved| resolved.canonical_path.clone()) + .unwrap_or(raw_field_path); let first_is_vector_index = metadata_is_vector_index(dataset.as_ref(), old_indices[0]).await?; for idx in old_indices.iter().skip(1) { let is_vector_index = metadata_is_vector_index(dataset.as_ref(), idx).await?; @@ -484,61 +804,179 @@ pub async fn merge_indices_with_unindexed_frags<'a>( base_unindexed_bitmap.insert(frag.id as u32); }); - let (new_uuid, removed_indices, new_fragment_bitmap, created_index) = if first_is_vector_index { - let full_logical_index = dataset - .open_logical_vector_index(&field_path, &old_indices[0].name) - .await?; - let mut opened_indices_by_uuid = full_logical_index - .iter() - .map(|(metadata, index)| (metadata.uuid, (metadata.clone(), index.clone()))) - .collect::>(); - let mut selected_metadatas = Vec::with_capacity(old_indices.len()); - let mut selected_indices = Vec::with_capacity(old_indices.len()); - for metadata in old_indices { - let (selected_metadata, selected_index) = opened_indices_by_uuid.remove(&metadata.uuid).ok_or_else(|| { + let (new_uuid, removed_indices, new_fragment_bitmap, created_index, new_dataset_version) = + if first_is_vector_index { + // Segments with stored rows (raw coverage) but no live coverage + // (e.g. a definition retained through a full rewrite) are dormant. + // With stable row ids their stored postings still share ids with + // live rows, so merging them would resurrect stale vectors; they + // may only be replaced. A born-empty segment (deferred build) has + // no stored rows and stays mergeable. + let (live_segments, dormant_segments): (Vec<&IndexMetadata>, Vec<&IndexMetadata>) = + old_indices.iter().copied().partition(|idx| { + let has_stored_rows = idx + .fragment_bitmap + .as_ref() + .is_some_and(|bitmap| !bitmap.is_empty()); + let has_live_coverage = idx + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .is_none_or(|bitmap| !bitmap.is_empty()); + !has_stored_rows || has_live_coverage + }); + if !dormant_segments.is_empty() && !live_segments.is_empty() && !options.retrain { + // Optimize the live segments as usual; the dormant segments + // are superseded by whatever that produces. + let mut merged = Box::pin(merge_indices_with_unindexed_frags( + dataset.clone(), + &live_segments, + unindexed, + options, + )) + .await?; + if let Some(results) = merged.as_mut() { + results.removed_indices.extend(dormant_segments); + } + return Ok(merged); + } + let rebuild_dormant = live_segments.is_empty() && !dormant_segments.is_empty(); + if rebuild_dormant && unindexed.is_empty() { + return Ok(None); + } + + let full_logical_index = dataset + .open_logical_vector_index(&field_path, &old_indices[0].name) + .await?; + let mut opened_indices_by_uuid = full_logical_index + .iter() + .map(|(metadata, index)| (metadata.uuid, (metadata.clone(), index.clone()))) + .collect::>(); + let mut selected_metadatas = Vec::with_capacity(old_indices.len()); + let mut selected_indices = Vec::with_capacity(old_indices.len()); + for metadata in old_indices { + let (selected_metadata, selected_index) = opened_indices_by_uuid.remove(&metadata.uuid).ok_or_else(|| { Error::index(format!( "Append index: logical vector index '{}' does not contain requested segment {}", old_indices[0].name, metadata.uuid )) })?; - selected_metadatas.push(selected_metadata); - selected_indices.push(selected_index); - } - let logical_index = LogicalVectorIndex::try_new( - old_indices[0].name.clone(), - field_path.clone(), - selected_metadatas - .into_iter() - .zip(selected_indices) - .collect(), - )?; - let ivf_view = logical_index.as_ivf()?; - - // Specialized vector no-op: when there is no new data and the caller - // hasn't asked for retrain or an explicit delta merge, the only useful - // work is rebalancing. Bail when no segment needs rebalancing so - // repeated optimize calls don't keep rewriting the same index. This - // matches the scalar gate in `Dataset::optimize_indices`, which also - // treats `OptimizeOptions::append()` (num_indices_to_merge=Some(0)) - // as "no explicit merge requested". - if unindexed.is_empty() - && !options.retrain - && options.num_indices_to_merge.is_none_or(|n| n == 0) - && select_segment_for_single_rebalance(&ivf_view)?.is_none() - { - return Ok(None); - } + selected_metadatas.push(selected_metadata); + selected_indices.push(selected_index); + } + let logical_index = LogicalVectorIndex::try_new( + old_indices[0].name.clone(), + field_path.clone(), + selected_metadatas + .into_iter() + .zip(selected_indices) + .collect(), + )?; - let use_single_segment_rebalance = logical_index.num_segments() > 1 - && options.num_indices_to_merge.is_none_or(|n| n == 0) - && !options.retrain - && unindexed.is_empty(); + if options.retrain || rebuild_dormant { + let fragment_bitmap = dataset.fragment_bitmap.as_ref().clone(); + let segment = build_fresh_vector_segment( + dataset.as_ref(), + &logical_index, + &field_path, + &fragment_bitmap, + options.progress.clone(), + ) + .await?; + return fresh_vector_segment_result( + segment, + &fragment_bitmap, + old_indices.to_vec(), + ) + .map(Some); + } + + let ivf_view = logical_index.as_ivf()?; + let compatibility = + vector_segment_compatibility(&ivf_view, "optimizing logical vector index")?; + let explicit_append = options.num_indices_to_merge == Some(0); + let default_append_for_heterogeneous_models = options.num_indices_to_merge.is_none() + && compatibility == VectorSegmentCompatibility::QueryCompatibleModelsDiffer; + + if !unindexed.is_empty() && (explicit_append || default_append_for_heterogeneous_models) + { + // CreateIndex commits append new segments at the end of the manifest's + // stable index order. Reusing the current suffix model keeps consecutive + // appends directly mergeable without requiring unrelated older segments + // to share that model. + let (reference_metadata, reference_index) = + logical_index.iter().last().ok_or_else(|| { + Error::index(format!( + "Optimize vector index: logical index '{}' has no physical segments", + logical_index.name() + )) + })?; + let reference_logical_index = LogicalVectorIndex::try_new( + logical_index.name().to_string(), + field_path.clone(), + vec![(reference_metadata.clone(), reference_index.clone())], + )?; + let reference_ivf_view = reference_logical_index.as_ivf()?; + let new_data_stream = scan_vector_fragments( + dataset.as_ref(), + &field_path, + column.nullable, + unindexed, + ) + .await?; + let mut append_options = options.clone(); + append_options.num_indices_to_merge = Some(0); + append_options.retrain = false; + let (new_uuid, indices_merged, files) = optimize_vector_indices( + dataset.as_ref().clone(), + Some(new_data_stream), + &field_path, + &reference_ivf_view, + &append_options, + ) + .boxed() + .await?; + if indices_merged != 0 { + return Err(Error::index(format!( + "Optimize vector index append unexpectedly merged {indices_merged} existing segments" + ))); + } + return Ok(Some(IndexMergeResults { + new_uuid, + removed_indices: Vec::new(), + new_fragment_bitmap: base_unindexed_bitmap, + new_dataset_version: dataset.manifest.version, + new_index_version: index_type_for_segmented_optimize(reference_index.as_ref())? + .version(), + new_index_details: reference_metadata + .index_details + .as_deref() + .cloned() + .unwrap_or_else(vector_index_details_default), + files, + })); + } - if use_single_segment_rebalance { - let Some(selected_segment_id) = select_segment_for_single_rebalance(&ivf_view)? else { + // Append is a steady-state no-op when every fragment is already + // indexed. Default optimize may still rebalance one segment. + if unindexed.is_empty() && options.num_indices_to_merge == Some(0) { return Ok(None); - }; - let removed_segment = old_indices + } + if unindexed.is_empty() + && options.num_indices_to_merge.is_none() + && select_segment_for_single_rebalance(&ivf_view)?.is_none() + { + return Ok(None); + } + + let use_single_segment_rebalance = logical_index.num_segments() > 1 + && options.num_indices_to_merge.is_none() + && unindexed.is_empty(); + + if use_single_segment_rebalance { + let Some(selected_segment_id) = select_segment_for_single_rebalance(&ivf_view)? + else { + return Ok(None); + }; + let removed_segment = old_indices .iter() .copied() .find(|metadata| metadata.uuid == selected_segment_id) @@ -548,7 +986,7 @@ pub async fn merge_indices_with_unindexed_frags<'a>( old_indices[0].name, selected_segment_id )) })?; - let (selected_metadata, selected_index) = logical_index + let (selected_metadata, selected_index) = logical_index .iter() .find(|(metadata, _)| metadata.uuid == selected_segment_id) .map(|(metadata, index)| (metadata.clone(), index.clone())) @@ -558,299 +996,366 @@ pub async fn merge_indices_with_unindexed_frags<'a>( selected_segment_id, old_indices[0].name )) })?; - let selected_logical_index = LogicalVectorIndex::try_new( - old_indices[0].name.clone(), - field_path.clone(), - vec![(selected_metadata, selected_index)], - )?; - let selected_ivf_view = selected_logical_index.as_ivf()?; - let (new_uuid, indices_merged, files) = Box::pin(optimize_vector_indices( - dataset.as_ref().clone(), - Option::< - lance_io::stream::RecordBatchStreamAdapter< - futures::stream::Empty>, - >, - >::None, - &field_path, - &selected_ivf_view, - options, - )) - .await?; - if indices_merged == 0 { - return Ok(None); - } - - let new_fragment_bitmap = removed_segment - .effective_fragment_bitmap(&dataset.fragment_bitmap) - .or_else(|| removed_segment.fragment_bitmap.clone()) - .unwrap_or_default(); + let selected_logical_index = LogicalVectorIndex::try_new( + old_indices[0].name.clone(), + field_path.clone(), + vec![(selected_metadata, selected_index)], + )?; + let selected_ivf_view = selected_logical_index.as_ivf()?; + let (new_uuid, indices_merged, files) = Box::pin(optimize_vector_indices( + dataset.as_ref().clone(), + Option::< + lance_io::stream::RecordBatchStreamAdapter< + futures::stream::Empty>, + >, + >::None, + &field_path, + &selected_ivf_view, + options, + )) + .await?; + if indices_merged == 0 { + return Ok(None); + } - Ok(( - new_uuid, - vec![removed_segment], - new_fragment_bitmap, - CreatedIndex { - index_details: vector_index_details_default(), - index_version: lance_index::IndexType::Vector.version() as u32, - files, - }, - )) - } else { - let mut frag_bitmap = base_unindexed_bitmap.clone(); + let new_fragment_bitmap = removed_segment + .effective_fragment_bitmap(&dataset.fragment_bitmap) + .or_else(|| removed_segment.fragment_bitmap.clone()) + .unwrap_or_default(); - let new_data_stream = if unindexed.is_empty() { - None + Ok(( + new_uuid, + vec![removed_segment], + new_fragment_bitmap, + CreatedIndex { + index_details: removed_segment + .index_details + .as_deref() + .cloned() + .unwrap_or_else(vector_index_details_default), + index_version: removed_segment.index_version as u32, + files: table_files_to_index(files), + }, + removed_segment.dataset_version, + )) } else { - let mut scanner = dataset.scan(); - scanner - .with_fragments(unindexed.to_vec()) - .with_row_id() - .project(&[&field_path])?; - if column.nullable { - let column_expr = - lance_datafusion::logical_expr::field_path_to_expr(&field_path)?; - scanner.filter_expr(column_expr.is_not_null()); + let mut frag_bitmap = base_unindexed_bitmap.clone(); + let num_segments_to_merge = options + .num_indices_to_merge + .unwrap_or(logical_index.num_segments()) + .min(logical_index.num_segments()); + let merge_start = logical_index + .num_segments() + .saturating_sub(num_segments_to_merge); + let merge_logical_index = LogicalVectorIndex::try_new( + old_indices[0].name.clone(), + field_path.clone(), + logical_index + .iter() + .skip(merge_start) + .map(|(metadata, index)| (metadata.clone(), index.clone())) + .collect(), + )?; + let merge_ivf_view = merge_logical_index.as_ivf()?; + + let new_data_stream = if unindexed.is_empty() { + None + } else { + Some( + scan_vector_fragments( + dataset.as_ref(), + &field_path, + column.nullable, + unindexed, + ) + .await?, + ) + }; + + let (new_uuid, indices_merged, files) = optimize_vector_indices( + dataset.as_ref().clone(), + new_data_stream, + &field_path, + &merge_ivf_view, + options, + ) + .boxed() + .await?; + + let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec(); + let new_dataset_version = removed_indices + .iter() + .map(|index| index.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + removed_indices.iter().for_each(|idx| { + frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter()); + }); + for removed in removed_indices.iter() { + if let Some(effective) = + removed.effective_fragment_bitmap(&dataset.fragment_bitmap) + { + frag_bitmap |= &effective; + } } - Some(scanner.try_into_stream().await?) - }; - let (new_uuid, indices_merged, files) = optimize_vector_indices( - dataset.as_ref().clone(), - new_data_stream, - &field_path, - &ivf_view, - options, - ) - .boxed() - .await?; + // Metadata must come from an actual merge source. Older incompatible + // segments outside the selected suffix may describe a different model. + let (reference_metadata, reference_index) = + merge_logical_index.iter().last().ok_or_else(|| { + Error::index( + "Optimize vector index merge did not select a reference segment" + .to_string(), + ) + })?; + let index_details = removed_indices + .iter() + .rev() + .filter_map(|idx| idx.index_details.as_ref()) + .find(|d| !d.value.is_empty()) + .map(|d| d.as_ref().clone()) + .or_else(|| { + reference_metadata + .index_details + .as_deref() + .filter(|details| !details.value.is_empty()) + .cloned() + }) + .unwrap_or_else(vector_index_details_default); + let index_version = if let Some(metadata) = removed_indices.first() { + metadata.index_version as u32 + } else { + index_type_for_segmented_optimize(reference_index.as_ref())?.version() as u32 + }; - let removed_indices = old_indices[old_indices.len() - indices_merged..].to_vec(); - removed_indices.iter().for_each(|idx| { - frag_bitmap.extend(idx.fragment_bitmap.as_ref().unwrap().iter()); - }); - for removed in removed_indices.iter() { - if let Some(effective) = removed.effective_fragment_bitmap(&dataset.fragment_bitmap) + Ok(( + new_uuid, + removed_indices, + frag_bitmap, + CreatedIndex { + index_details, + index_version, + files: table_files_to_index(files), + }, + new_dataset_version, + )) + } + } else { + let mut indices = Vec::with_capacity(old_indices.len()); + for idx in old_indices { + match dataset + .open_generic_index(&field_path, &idx.uuid, &NoOpMetricsCollector) + .await { - frag_bitmap |= &effective; + Ok(index) => indices.push(index), + Err(e) => { + log::warn!( + "Cannot open index on column '{}': {}. \ + Skipping index merge for this column.", + field_path, + e + ); + return Ok(None); + } } } - // Carry forward existing index details, preferring the first segment - // that has populated (non-empty) details. - let index_details = old_indices - .iter() - .rev() - .filter_map(|idx| idx.index_details.as_ref()) - .find(|d| !d.value.is_empty()) - .map(|d| d.as_ref().clone()) - .unwrap_or_else(vector_index_details_default); - - Ok(( - new_uuid, - removed_indices, - frag_bitmap, - CreatedIndex { - index_details, - // retain_supported_indices guarantees all old_indices have - // index_version <= our max supported version, so we can safely - // write the current library's version for this index type. - index_version: lance_index::IndexType::Vector.version() as u32, - files, - }, - )) - } - } else { - let mut indices = Vec::with_capacity(old_indices.len()); - for idx in old_indices { - match dataset - .open_generic_index(&field_path, &idx.uuid, &NoOpMetricsCollector) - .await + if indices + .windows(2) + .any(|w| w[0].index_type() != w[1].index_type()) { - Ok(index) => indices.push(index), - Err(e) => { - log::warn!( - "Cannot open index on column '{}': {}. \ - Skipping index merge for this column.", - field_path, - e - ); - return Ok(None); - } + return Err(Error::index(format!( + "Append index: invalid index deltas: {:?}", + old_indices + ))); } - } - if indices - .windows(2) - .any(|w| w[0].index_type() != w[1].index_type()) - { - return Err(Error::index(format!( - "Append index: invalid index deltas: {:?}", - old_indices - ))); - } + let index_type = indices[0].index_type(); + match index_type { + IndexType::Inverted => { + let selected_old_indices = + select_segments_to_merge(dataset.as_ref(), old_indices, options); + if unindexed.is_empty() && selected_old_indices.len() <= 1 { + return Ok(None); + } + let reference_idx = selected_old_indices + .first() + .copied() + .unwrap_or(old_indices[old_indices.len() - 1]); + let reference_index = dataset + .open_scalar_index(&field_path, &reference_idx.uuid, &NoOpMetricsCollector) + .await?; + let update_criteria = reference_index.update_criteria(); + if update_criteria.requires_old_data { + let params = reference_index.derive_index_params()?; + let resolved = resolved_fts.as_ref().ok_or_else(|| { + Error::internal( + "Inverted index metadata did not resolve an FTS field".to_string(), + ) + })?; + let new_data_stream = load_fts_training_data( + dataset.as_ref(), + resolved, + &update_criteria.data_criteria, + None, + true, + None, + ) + .await?; + let new_uuid = Uuid::new_v4(); + let created_index = super::scalar::build_scalar_index( + dataset.as_ref(), + &resolved.canonical_path, + new_uuid, + ¶ms, + true, + None, + Some(new_data_stream), + Arc::new(NoopIndexBuildProgress), + ) + .await?; + return Ok(Some(IndexMergeResults { + new_uuid, + removed_indices: old_indices.to_vec(), + new_fragment_bitmap: dataset.fragment_bitmap.as_ref().clone(), + new_dataset_version: dataset.manifest.version, + new_index_version: created_index.index_version as i32, + new_index_details: created_index.index_details, + files: index_files_to_table(created_index.files), + })); + } - let index_type = indices[0].index_type(); - match index_type { - IndexType::Inverted => { - let selected_old_indices = - select_segments_to_merge(dataset.as_ref(), old_indices, options); - if unindexed.is_empty() && selected_old_indices.len() <= 1 { - return Ok(None); - } - let reference_idx = selected_old_indices - .first() - .copied() - .unwrap_or(old_indices[old_indices.len() - 1]); - let reference_index = dataset - .open_scalar_index(&field_path, &reference_idx.uuid, &NoOpMetricsCollector) - .await?; - let update_criteria = reference_index.update_criteria(); - if update_criteria.requires_old_data { - let params = reference_index.derive_index_params()?; - let new_data_stream = load_training_data( + let fragments = Some(unindexed.to_vec()); + let resolved = resolved_fts.as_ref().ok_or_else(|| { + Error::internal( + "Inverted index metadata did not resolve an FTS field".to_string(), + ) + })?; + let new_data_stream = load_fts_training_data( dataset.as_ref(), - &field_path, + resolved, &update_criteria.data_criteria, - None, + fragments, true, None, ) .await?; - let new_uuid = Uuid::new_v4(); - let created_index = super::scalar::build_scalar_index( - dataset.as_ref(), - column.name.as_str(), - new_uuid, - ¶ms, - true, - None, - Some(new_data_stream), - Arc::new(NoopIndexBuildProgress), - ) - .await?; - return Ok(Some(IndexMergeResults { - new_uuid, - removed_indices: old_indices.to_vec(), - new_fragment_bitmap: dataset.fragment_bitmap.as_ref().clone(), - new_index_version: created_index.index_version as i32, - new_index_details: created_index.index_details, - files: created_index.files, - })); - } - let fragments = Some(unindexed.to_vec()); - let new_data_stream = load_training_data( - dataset.as_ref(), - &field_path, - &update_criteria.data_criteria, - fragments, - true, - None, - ) - .await?; - - let mut frag_bitmap = base_unindexed_bitmap; - let mut effective_old_frags = RoaringBitmap::new(); - let mut selected_indices = Vec::with_capacity(selected_old_indices.len()); - for idx in &selected_old_indices { - if let Some(effective) = idx.effective_fragment_bitmap(&dataset.fragment_bitmap) - { - frag_bitmap |= &effective; - effective_old_frags |= &effective; + let mut frag_bitmap = base_unindexed_bitmap; + let mut effective_old_frags = RoaringBitmap::new(); + let mut selected_indices = Vec::with_capacity(selected_old_indices.len()); + for idx in &selected_old_indices { + if let Some(effective) = + idx.effective_fragment_bitmap(&dataset.fragment_bitmap) + { + frag_bitmap |= &effective; + effective_old_frags |= &effective; + } + let scalar_index = dataset + .open_scalar_index(&field_path, &idx.uuid, &NoOpMetricsCollector) + .await?; + let inverted_index = scalar_index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::index(format!( + "Append index: expected inverted index segment {}, got {:?}", + idx.uuid, + scalar_index.index_type() + )) + })?; + selected_indices.push(Arc::new(inverted_index.clone())); } - let scalar_index = dataset - .open_scalar_index(&field_path, &idx.uuid, &NoOpMetricsCollector) - .await?; - let inverted_index = scalar_index - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::index(format!( - "Append index: expected inverted index segment {}, got {:?}", - idx.uuid, - scalar_index.index_type() - )) - })?; - selected_indices.push(Arc::new(inverted_index.clone())); - } - let old_data_filter = if selected_indices.is_empty() { - None - } else if dataset.manifest.uses_stable_row_ids() { - let valid_old_row_ids = - build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags).await?; - Some(OldIndexDataFilter::RowIds(valid_old_row_ids)) - } else { - Some(OldIndexDataFilter::Fragments { - to_keep: effective_old_frags, - to_remove: RoaringBitmap::new(), - }) - }; + let old_data_filter = if selected_indices.is_empty() { + None + } else if dataset.manifest.uses_stable_row_ids() { + let valid_old_row_ids = + build_stable_row_id_filter(dataset.as_ref(), &effective_old_frags) + .await?; + Some(OldIndexDataFilter::RowIds(valid_old_row_ids)) + } else { + Some(OldIndexDataFilter::Fragments { + to_keep: effective_old_frags, + to_remove: RoaringBitmap::new(), + }) + }; - let new_uuid = Uuid::new_v4(); - let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?; - let created_index = if selected_indices.is_empty() { - let params = reference_index.derive_index_params()?; - super::scalar::build_scalar_index( - dataset.as_ref(), - column.name.as_str(), + let new_uuid = Uuid::new_v4(); + let new_store = LanceIndexStore::from_dataset_for_new(&dataset, &new_uuid)?; + let (created_index, new_dataset_version) = if selected_indices.is_empty() { + ( + super::scalar::build_scalar_index( + dataset.as_ref(), + &resolved.canonical_path, + new_uuid, + &reference_index.derive_index_params()?, + true, + None, + Some(new_data_stream), + Arc::new(NoopIndexBuildProgress), + ) + .await?, + dataset.manifest.version, + ) + } else { + ( + InvertedIndex::merge_segments( + &selected_indices, + new_data_stream, + &new_store, + old_data_filter, + options.progress.clone(), + ) + .await?, + selected_old_indices + .iter() + .map(|index| index.dataset_version) + .min() + .unwrap_or(dataset.manifest.version), + ) + }; + + Ok(( new_uuid, - ¶ms, - true, - None, - Some(new_data_stream), - Arc::new(NoopIndexBuildProgress), - ) - .await? - } else { - InvertedIndex::merge_segments( - &selected_indices, - new_data_stream, - &new_store, - old_data_filter, - options.progress.clone(), + selected_old_indices.to_vec(), + frag_bitmap, + created_index, + new_dataset_version, + )) + } + it if it.is_scalar() => { + let Some(result) = merge_scalar_indices( + dataset.clone(), + old_indices, + unindexed, + options, + it, + &field_path, + column.name.as_str(), + base_unindexed_bitmap, ) .await? - }; - - Ok(( - new_uuid, - selected_old_indices.to_vec(), - frag_bitmap, - created_index, - )) - } - it if it.is_scalar() => { - let Some(result) = merge_scalar_indices( - dataset.clone(), - old_indices, - unindexed, - options, - it, - &field_path, - column.name.as_str(), - base_unindexed_bitmap, - ) - .await? - else { - return Ok(None); - }; - Ok(result) + else { + return Ok(None); + }; + Ok(result) + } + _ => Err(Error::index(format!( + "Append index: invalid index type: {:?}", + indices[0].index_type() + ))), } - _ => Err(Error::index(format!( - "Append index: invalid index type: {:?}", - indices[0].index_type() - ))), - } - }?; + }?; Ok(Some(IndexMergeResults { new_uuid, removed_indices, new_fragment_bitmap, + new_dataset_version, new_index_version: created_index.index_version as i32, new_index_details: created_index.index_details, - files: created_index.files, + files: index_files_to_table(created_index.files), })) } @@ -863,31 +1368,689 @@ mod tests { use arrow::datatypes::{Float32Type, UInt32Type}; use arrow_array::cast::AsArray; use arrow_array::{ - FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt32Array, + Array, ArrayRef, FixedSizeListArray, Float32Array, Int32Array, ListArray, RecordBatch, + RecordBatchIterator, StringArray, StructArray, UInt32Array, UInt64Array, }; + use arrow_buffer::{BooleanBufferBuilder, NullBuffer, OffsetBuffer}; use arrow_schema::{DataType, Field, Schema}; use futures::TryStreamExt; use lance_arrow::FixedSizeListArrayExt; use lance_core::utils::tempfile::TempStrDir; use lance_datafusion::utils::reader_to_stream; use lance_datagen::{Dimension, RowCount, array}; + use lance_file::version::LanceFileVersion; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::sq::builder::SQBuildParams; use lance_index::{ IndexType, - scalar::{BuiltinIndexType, ScalarIndexParams, SearchResult, TextQuery}, + scalar::{ + BuiltinIndexType, InvertedIndexParams, ScalarIndexParams, SearchResult, TextQuery, + }, vector::{ivf::IvfBuildParams, pq::PQBuildParams}, }; use lance_linalg::distance::MetricType; - use lance_testing::datagen::generate_random_array; + use lance_testing::datagen::{generate_random_array, generate_random_array_with_seed}; use rstest::rstest; use crate::dataset::builder::DatasetBuilder; use crate::dataset::optimize::{CompactionOptions, compact_files}; use crate::dataset::{MergeInsertBuilder, WhenMatched, WhenNotMatched, WriteMode, WriteParams}; + use crate::index::CreateIndexBuilder; use crate::index::vector::VectorIndexParams; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + #[test] + fn test_fragment_reuse_at_source_version_affects_segment() { + use lance_index::frag_reuse::{ + FragDigest, FragReuseGroup, FragReuseIndexDetails, FragReuseVersion, + }; + + let segment = IndexMetadata { + uuid: Uuid::new_v4(), + name: "text_ngram".to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 5, + fragment_bitmap: Some(RoaringBitmap::from_iter([1u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let frag_reuse_index = CompactFragReuseIndex::from_row_id_maps( + Uuid::new_v4(), + vec![], + FragReuseIndexDetails { + versions: vec![FragReuseVersion { + dataset_version: 5, + groups: vec![FragReuseGroup { + changed_row_addrs: vec![1], + old_frags: vec![FragDigest { + id: 1, + physical_rows: 1, + num_deleted_rows: 0, + }], + new_frags: vec![FragDigest { + id: 2, + physical_rows: 1, + num_deleted_rows: 0, + }], + }], + }], + }, + ); + + assert!(fragment_reuse_affects_segments( + &frag_reuse_index, + [&segment] + )); + + let rebuilt_segment = IndexMetadata { + dataset_version: 5, + fragment_bitmap: Some(RoaringBitmap::from_iter([2u32])), + ..segment + }; + assert!(!fragment_reuse_affects_segments( + &frag_reuse_index, + [&rebuilt_segment] + )); + + let stale_remapped_segment = IndexMetadata { + dataset_version: 4, + ..rebuilt_segment + }; + assert!(fragment_reuse_affects_segments( + &frag_reuse_index, + [&stale_remapped_segment] + )); + } + + fn clustered_vector_batch( + schema: Arc, + start_id: i32, + rows: usize, + dimension: usize, + center: f32, + ) -> (RecordBatch, Arc) { + let values = (0..rows) + .flat_map(|row| { + (0..dimension) + .map(move |column| center + row as f32 * 0.01 + column as f32 * 0.0001) + }) + .collect::>(); + let vectors = Arc::new( + FixedSizeListArray::try_new_from_values( + arrow_array::Float32Array::from(values), + dimension as i32, + ) + .unwrap(), + ); + let ids = Arc::new(Int32Array::from_iter_values( + start_id..start_id + rows as i32, + )); + let batch = RecordBatch::try_new(schema, vec![ids, vectors.clone() as ArrayRef]).unwrap(); + (batch, vectors) + } + + async fn nearest_id( + dataset: &Dataset, + query: &arrow_array::Float32Array, + num_probes: usize, + ) -> i32 { + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .nearest("vector", query, 1) + .unwrap() + .nprobes(num_probes) + .refine(1) + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 1); + result["id"] + .as_primitive::() + .value(0) + } + + #[rstest] + #[case::append(OptimizeOptions::append(), false)] + #[case::default_with_stable_row_ids_and_delete(OptimizeOptions::default(), true)] + #[tokio::test] + async fn test_vector_append_is_segment_set_native_with_distinct_models( + #[case] options: OptimizeOptions, + #[case] use_stable_row_ids: bool, + ) { + const DIMENSION: usize = 8; + const ROWS_PER_FRAGMENT: usize = 64; + const INDEX_NAME: &str = "vector_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let (first_batch, first_vectors) = + clustered_vector_batch(schema.clone(), 0, ROWS_PER_FRAGMENT, DIMENSION, 0.0); + let (second_batch, _) = clustered_vector_batch( + schema.clone(), + ROWS_PER_FRAGMENT as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 100.0, + ); + let reader = + RecordBatchIterator::new(vec![Ok(first_batch), Ok(second_batch)], schema.clone()); + let mut dataset = Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + enable_stable_row_ids: use_stable_row_ids, + max_rows_per_file: ROWS_PER_FRAGMENT, + ..Default::default() + }), + ) + .await + .unwrap(); + + let initial_fragments = dataset.get_fragments(); + assert_eq!(initial_fragments.len(), 2); + let params = VectorIndexParams::ivf_flat(1, MetricType::L2); + let mut initial_segments = Vec::with_capacity(initial_fragments.len()); + for fragment in &initial_fragments { + initial_segments.push( + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name(INDEX_NAME.to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(INDEX_NAME, "vector", initial_segments) + .await + .unwrap(); + dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + + let old_segments = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(old_segments.len(), 2); + let logical_index = dataset + .open_logical_vector_index("vector", INDEX_NAME) + .await + .unwrap(); + let centroids = logical_index + .as_ivf() + .unwrap() + .segments() + .map(|(_, index)| index.ivf_model().centroids_array().unwrap().to_data()) + .collect::>(); + assert_ne!( + centroids[0], centroids[1], + "test setup must use distinct IVF centroid models" + ); + let version_before_default_optimize = dataset.version().version; + dataset + .optimize_indices(&OptimizeOptions::default()) + .await + .unwrap(); + assert_eq!( + dataset.version().version, + version_before_default_optimize, + "default optimize must leave incompatible steady-state segments unmerged" + ); + assert_eq!( + dataset.load_indices_by_name(INDEX_NAME).await.unwrap(), + old_segments, + "default optimize changed incompatible steady-state segments" + ); + + let appended_start_id = (2 * ROWS_PER_FRAGMENT) as i32; + let (appended_batch, appended_vectors) = clustered_vector_batch( + schema.clone(), + appended_start_id, + ROWS_PER_FRAGMENT, + DIMENSION, + 200.0, + ); + dataset + .append( + RecordBatchIterator::new(vec![Ok(appended_batch)], schema), + None, + ) + .await + .unwrap(); + let appended_fragment_id = dataset.get_fragments().last().unwrap().id() as u32; + if use_stable_row_ids { + dataset + .delete(&format!("id = {appended_start_id}")) + .await + .unwrap(); + } + + dataset.optimize_indices(&options).await.unwrap(); + + let mut dataset = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + let committed = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!( + committed.len(), + 3, + "append must add one segment without rewriting existing segments" + ); + for old in &old_segments { + let retained = committed + .iter() + .find(|segment| segment.uuid == old.uuid) + .expect("old physical segment UUID must remain committed"); + assert_eq!(retained, old, "old physical segment metadata changed"); + } + + let new_segment = committed + .iter() + .find(|segment| { + old_segments + .iter() + .all(|old_segment| old_segment.uuid != segment.uuid) + }) + .expect("one new physical segment must be committed"); + assert_eq!( + new_segment.fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter([appended_fragment_id]), + "the appended segment must cover only the new fragment" + ); + + let logical_index = dataset + .open_logical_vector_index("vector", INDEX_NAME) + .await + .unwrap(); + let latest_old_uuid = old_segments.last().unwrap().uuid; + let latest_old_centroids = logical_index + .iter() + .find(|(metadata, _)| metadata.uuid == latest_old_uuid) + .unwrap() + .1 + .ivf_model() + .centroids_array() + .unwrap() + .to_data(); + let new_centroids = logical_index + .iter() + .find(|(metadata, _)| metadata.uuid == new_segment.uuid) + .unwrap() + .1 + .ivf_model() + .centroids_array() + .unwrap() + .to_data(); + assert_eq!( + new_centroids, latest_old_centroids, + "append must reuse the latest segment's complete IVF model" + ); + + let mut covered = RoaringBitmap::new(); + for segment in &committed { + let segment_coverage = segment.fragment_bitmap.as_ref().unwrap(); + assert!( + covered.is_disjoint(segment_coverage), + "physical vector segment coverage must be pairwise disjoint" + ); + covered |= segment_coverage; + } + assert_eq!( + covered, + dataset.fragment_bitmap.as_ref().clone(), + "physical vector segments must exactly cover indexed fragments" + ); + + let old_query = first_vectors.value(1); + assert_eq!( + nearest_id( + &dataset, + old_query.as_primitive::(), + committed.len() + ) + .await, + 1, + "query must return a representative row from an old segment" + ); + let appended_query_offset = usize::from(use_stable_row_ids); + let appended_query = appended_vectors.value(appended_query_offset); + assert_eq!( + nearest_id( + &dataset, + appended_query.as_primitive::(), + committed.len() + ) + .await, + appended_start_id + appended_query_offset as i32, + "query must return a representative row from the appended segment" + ); + if use_stable_row_ids { + assert_eq!( + dataset + .scan() + .filter(&format!("id = {appended_start_id}")) + .unwrap() + .count_rows() + .await + .unwrap(), + 0, + "deleted stable-row-id data must not reappear through the appended segment" + ); + } + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + let merged_suffix = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!( + merged_suffix.len(), + 2, + "the reference-compatible suffix should merge without rewriting the incompatible base" + ); + assert!( + merged_suffix + .iter() + .any(|segment| segment.uuid == old_segments[0].uuid), + "the incompatible base segment must remain unchanged" + ); + assert!( + merged_suffix + .iter() + .all(|segment| segment.uuid != latest_old_uuid && segment.uuid != new_segment.uuid), + "the compatible suffix segments must be replaced" + ); + + dataset + .optimize_indices(&OptimizeOptions::retrain()) + .await + .unwrap(); + let retrained = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(retrained.len(), 1); + assert_eq!( + retrained[0].fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref(), + "explicit retrain must source-rebuild one segment over all current fragments" + ); + assert_eq!( + nearest_id( + &dataset, + old_query.as_primitive::(), + retrained.len() + ) + .await, + 1 + ); + assert_eq!( + nearest_id( + &dataset, + appended_query.as_primitive::(), + retrained.len() + ) + .await, + appended_start_id + appended_query_offset as i32 + ); + if use_stable_row_ids { + assert_eq!( + dataset + .scan() + .filter(&format!("id = {appended_start_id}")) + .unwrap() + .count_rows() + .await + .unwrap(), + 0, + "explicit retrain must not restore a deleted stable-row-id row" + ); + } + } + + #[rstest] + #[case::metric( + VectorIndexParams::ivf_flat(1, MetricType::L2), + VectorIndexParams::ivf_flat(1, MetricType::Cosine), + "has metric" + )] + #[case::index_family( + VectorIndexParams::ivf_flat(1, MetricType::L2), + VectorIndexParams::ivf_hnsw( + MetricType::L2, + IvfBuildParams::new(1), + HnswBuildParams::default() + ), + "has type" + )] + #[tokio::test] + async fn test_vector_append_validates_logical_query_compatibility( + #[case] first_params: VectorIndexParams, + #[case] second_params: VectorIndexParams, + #[case] expected_error: &str, + ) { + const DIMENSION: usize = 8; + const ROWS_PER_FRAGMENT: usize = 64; + const INDEX_NAME: &str = "vector_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let (first_batch, _) = + clustered_vector_batch(schema.clone(), 0, ROWS_PER_FRAGMENT, DIMENSION, 1.0); + let (second_batch, _) = clustered_vector_batch( + schema.clone(), + ROWS_PER_FRAGMENT as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 100.0, + ); + let reader = + RecordBatchIterator::new(vec![Ok(first_batch), Ok(second_batch)], schema.clone()); + let mut dataset = Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: ROWS_PER_FRAGMENT, + ..Default::default() + }), + ) + .await + .unwrap(); + + let initial_fragments = dataset.get_fragments(); + let mut segments = Vec::with_capacity(initial_fragments.len()); + for (fragment, params) in initial_fragments.iter().zip([first_params, second_params]) { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name(INDEX_NAME.to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + dataset + .commit_existing_index_segments(INDEX_NAME, "vector", segments) + .await + .unwrap(); + + let (appended_batch, _) = clustered_vector_batch( + schema.clone(), + (2 * ROWS_PER_FRAGMENT) as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 200.0, + ); + dataset + .append( + RecordBatchIterator::new(vec![Ok(appended_batch)], schema), + None, + ) + .await + .unwrap(); + + let version_before = dataset.version().version; + let segments_before = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); + let object_store = dataset.object_store.clone(); + let directories_before = object_store + .read_dir(dataset.indices_dir()) + .await + .unwrap() + .into_iter() + .collect::>(); + + let error = dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap_err(); + assert!( + error.to_string().contains(expected_error), + "expected logical query compatibility error containing '{expected_error}', got {error}" + ); + + let latest = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + assert_eq!( + latest.version().version, + version_before, + "incompatible logical segments must fail before committing" + ); + let mut segments_after = latest.load_indices_by_name(INDEX_NAME).await.unwrap(); + let mut segments_before = segments_before; + for segment in segments_after.iter_mut().chain(segments_before.iter_mut()) { + segment.created_at = None; + } + assert_eq!( + segments_after, segments_before, + "incompatible logical segments must remain unchanged" + ); + let directories_after = object_store + .read_dir(latest.indices_dir()) + .await + .unwrap() + .into_iter() + .collect::>(); + assert_eq!( + directories_after, directories_before, + "query compatibility validation must run before staging a new segment" + ); + } + + #[tokio::test] + async fn test_vector_segment_native_append_preserves_commit_conflicts() { + const DIMENSION: usize = 8; + const ROWS_PER_FRAGMENT: usize = 64; + const INDEX_NAME: &str = "vector_idx"; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let (initial_batch, _) = + clustered_vector_batch(schema.clone(), 0, ROWS_PER_FRAGMENT, DIMENSION, 0.0); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()), + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: ROWS_PER_FRAGMENT, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some(INDEX_NAME.to_string()), + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + let old_uuid = dataset.load_indices_by_name(INDEX_NAME).await.unwrap()[0].uuid; + + let (first_append, _) = clustered_vector_batch( + schema.clone(), + ROWS_PER_FRAGMENT as i32, + ROWS_PER_FRAGMENT, + DIMENSION, + 100.0, + ); + dataset + .append( + RecordBatchIterator::new(vec![Ok(first_append)], schema.clone()), + None, + ) + .await + .unwrap(); + let stale_version = dataset.version().version; + let mut first_optimizer = dataset.checkout_version(stale_version).await.unwrap(); + let mut stale_optimizer = dataset.checkout_version(stale_version).await.unwrap(); + first_optimizer + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let error = stale_optimizer + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap_err(); + assert!( + matches!(error, Error::RetryableCommitConflict { .. }), + "stale vector append optimize must retain retryable conflict semantics: {error}" + ); + + let latest = DatasetBuilder::from_uri(test_dir.as_str()) + .load() + .await + .unwrap(); + let committed = latest.load_indices_by_name(INDEX_NAME).await.unwrap(); + assert_eq!(committed.len(), 2); + assert!( + committed.iter().any(|segment| segment.uuid == old_uuid), + "the successful concurrent optimize must retain the old segment" + ); + assert_eq!( + latest.unindexed_fragments(INDEX_NAME).await.unwrap().len(), + 0, + "a failed stale commit must not publish the staged vector segment" + ); + } + #[tokio::test] async fn test_append_index() { const DIM: usize = 64; @@ -958,51 +2121,163 @@ mod tests { .optimize_indices(&OptimizeOptions::append()) .await .unwrap(); - let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); - let indices = dataset.load_indices().await.unwrap(); + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let indices = dataset.load_indices().await.unwrap(); + + assert!( + dataset + .unindexed_fragments(&index.name) + .await + .unwrap() + .is_empty() + ); + + // There should be two indices directories existed. + let object_store = dataset.object_store.as_ref(); + let index_dirs = object_store.read_dir(dataset.indices_dir()).await.unwrap(); + assert_eq!(index_dirs.len(), 2); + + let mut scanner = dataset.scan(); + scanner + .nearest("vector", q.as_primitive::(), 10) + .unwrap(); + let results = scanner + .try_into_stream() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let vectors = &results[0]["vector"]; + // Second batch of vectors should be in the index. + let contained = vectors.as_fixed_size_list().iter().any(|v| { + let vec = v.as_ref().unwrap(); + array.iter().any(|a| a.as_ref().unwrap() == vec) + }); + assert!(contained); + + // Check that the index has all 2000 rows. + let mut num_rows = 0; + for index in indices.iter() { + let index = dataset + .open_vector_index("vector", &index.uuid, &NoOpMetricsCollector) + .await + .unwrap(); + num_rows += index.num_rows(); + } + assert_eq!(num_rows, 2000); + } + + #[tokio::test] + async fn test_optimize_append_preserves_case_sensitive_nullable_vector_column() { + const DIM: usize = 64; + const ROWS: usize = 1000; + + fn make_vectors(rows: usize, dim: usize, include_null: bool) -> FixedSizeListArray { + if include_null { + let mut nulls_builder = BooleanBufferBuilder::new(rows); + for row_idx in 0..rows { + nulls_builder.append(row_idx != 0); + } + let nulls = NullBuffer::new(nulls_builder.finish()); + FixedSizeListArray::try_new( + Arc::new(Field::new("item", DataType::Float32, true)), + dim as i32, + Arc::new(generate_random_array(rows * dim)), + Some(nulls), + ) + .unwrap() + } else { + FixedSizeListArray::try_new_from_values( + generate_random_array(rows * dim), + dim as i32, + ) + .unwrap() + } + } + + fn make_batch( + schema: Arc, + start_id: u32, + vectors: Arc, + ) -> RecordBatch { + let columns: Vec = vec![ + Arc::new(UInt32Array::from_iter_values( + start_id..start_id + ROWS as u32, + )) as ArrayRef, + vectors as ArrayRef, + ]; + RecordBatch::try_new(schema, columns).unwrap() + } + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let vector_type = DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("VECTOR", vector_type, true), + ])); + + let initial_vectors = Arc::new(make_vectors(ROWS, DIM, false)); + let initial_batch = make_batch(schema.clone(), 0, initial_vectors); + let batches = RecordBatchIterator::new(std::iter::once(Ok(initial_batch)), schema.clone()); + let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap(); + + let params = VectorIndexParams::with_ivf_pq_params( + MetricType::L2, + IvfBuildParams::new(2), + PQBuildParams { + num_sub_vectors: 2, + ..Default::default() + }, + ); + dataset + .create_index(&["VECTOR"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + + let appended_vectors = Arc::new(make_vectors(ROWS, DIM, true)); + let query = appended_vectors.value(5); + let appended_batch = make_batch(schema.clone(), ROWS as u32, appended_vectors); + let batches = RecordBatchIterator::new(std::iter::once(Ok(appended_batch)), schema); + dataset.append(batches, None).await.unwrap(); + + let index_name = dataset.load_indices().await.unwrap()[0].name.clone(); + assert!( + !dataset + .unindexed_fragments(&index_name) + .await + .unwrap() + .is_empty() + ); + + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); assert!( dataset - .unindexed_fragments(&index.name) + .unindexed_fragments(&index_name) .await .unwrap() .is_empty() ); - // There should be two indices directories existed. - let object_store = dataset.object_store.as_ref(); - let index_dirs = object_store.read_dir(dataset.indices_dir()).await.unwrap(); - assert_eq!(index_dirs.len(), 2); - let mut scanner = dataset.scan(); scanner - .nearest("vector", q.as_primitive::(), 10) - .unwrap(); - let results = scanner - .try_into_stream() - .await - .unwrap() - .try_collect::>() - .await + .nearest("VECTOR", query.as_primitive::(), 10) .unwrap(); - let vectors = &results[0]["vector"]; - // Second batch of vectors should be in the index. - let contained = vectors.as_fixed_size_list().iter().any(|v| { - let vec = v.as_ref().unwrap(); - array.iter().any(|a| a.as_ref().unwrap() == vec) - }); - assert!(contained); - - // Check that the index has all 2000 rows. - let mut num_rows = 0; - for index in indices.iter() { - let index = dataset - .open_vector_index("vector", &index.uuid, &NoOpMetricsCollector) - .await - .unwrap(); - num_rows += index.num_rows(); - } - assert_eq!(num_rows, 2000); + let results = scanner.try_into_batch().await.unwrap(); + assert_eq!( + results.num_rows(), + 10, + "expected the requested k=10 nearest-neighbor results" + ); } /// Regression: a second `OptimizeOptions::append()` call on a steady-state @@ -1060,7 +2335,7 @@ mod tests { RecordBatchIterator::new(vec![make_batch()].into_iter().map(Ok), schema.clone()); dataset.append(batches, None).await.unwrap(); - // First append: folds the new fragment into a fresh delta segment. + // First append: writes the new fragment into a reference-compatible delta segment. dataset .optimize_indices(&OptimizeOptions::append()) .await @@ -1105,23 +2380,62 @@ mod tests { #[tokio::test] async fn test_query_delta_indices( #[values( - VectorIndexParams::ivf_pq(2, 8, 4, MetricType::L2, 2), + VectorIndexParams::ivf_pq(2, 4, 2, MetricType::L2, 2), + VectorIndexParams::ivf_rq(2, 1, MetricType::L2), + VectorIndexParams::ivf_hnsw( + MetricType::L2, + IvfBuildParams::new(2), + HnswBuildParams { + max_level: 2, + m: 4, + ef_construction: 16, + prefetch_distance: Some(1), + } + ), + VectorIndexParams::with_ivf_hnsw_pq_params( + MetricType::L2, + IvfBuildParams { + num_partitions: Some(2), + max_iters: 2, + sample_rate: 2, + ..Default::default() + }, + HnswBuildParams { + max_level: 2, + m: 4, + ef_construction: 16, + prefetch_distance: Some(1), + }, + PQBuildParams { + num_sub_vectors: 2, + num_bits: 4, + max_iters: 2, + sample_rate: 2, + ..Default::default() + } + ), VectorIndexParams::with_ivf_hnsw_sq_params( MetricType::L2, IvfBuildParams::new(2), - HnswBuildParams::default(), + HnswBuildParams { + max_level: 2, + m: 4, + ef_construction: 16, + prefetch_distance: Some(1), + }, SQBuildParams::default() ) )] index_params: VectorIndexParams, ) { - const DIM: usize = 64; - const TOTAL: usize = 1000; + const DIM: usize = 8; + const INITIAL_ROWS: usize = 64; + const APPENDED_ROWS: usize = 16; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - let vectors = generate_random_array(TOTAL * DIM); + let vectors = generate_random_array_with_seed::(INITIAL_ROWS * DIM, [42; 32]); let schema = Arc::new(Schema::new(vec![ Field::new( @@ -1139,7 +2453,7 @@ mod tests { schema.clone(), vec![ array.clone(), - Arc::new(UInt32Array::from_iter_values(0..TOTAL as u32)), + Arc::new(UInt32Array::from_iter_values(0..INITIAL_ROWS as u32)), ], ) .unwrap(); @@ -1159,9 +2473,9 @@ mod tests { let batch = RecordBatch::try_new( schema.clone(), vec![ - array.clone(), + Arc::new(array.slice(0, APPENDED_ROWS)), Arc::new(UInt32Array::from_iter_values( - TOTAL as u32..(TOTAL * 2) as u32, + INITIAL_ROWS as u32..(INITIAL_ROWS + APPENDED_ROWS) as u32, )), ], ) @@ -1169,6 +2483,7 @@ mod tests { let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema.clone()); dataset.append(batches, None).await.unwrap(); + let appended_fragment_id = dataset.get_fragments().last().unwrap().id() as u32; let stats: serde_json::Value = serde_json::from_str(&dataset.index_statistics("vector_idx").await.unwrap()).unwrap(); assert_eq!(stats["num_indices"], 1); @@ -1179,12 +2494,19 @@ mod tests { .optimize_indices(&OptimizeOptions::append()) .await .unwrap(); - let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let mut dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); let stats: serde_json::Value = serde_json::from_str(&dataset.index_statistics("vector_idx").await.unwrap()).unwrap(); assert_eq!(stats["num_indices"], 2); assert_eq!(stats["num_indexed_fragments"], 2); assert_eq!(stats["num_unindexed_fragments"], 0); + let appended_segments = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert!( + appended_segments + .iter() + .all(|segment| segment.index_version == index_params.index_type().version()), + "append must preserve the storage type's index version" + ); let logical_index = dataset .open_logical_vector_index("vector", "vector_idx") .await @@ -1196,24 +2518,86 @@ mod tests { .into_iter() .map(|(_, num_rows)| num_rows) .sum::(), - 2000 + (INITIAL_ROWS + APPENDED_ROWS) as u64 ); + if matches!( + index_params.index_type(), + IndexType::IvfHnswFlat | IndexType::IvfHnswPq | IndexType::IvfHnswSq + ) { + let hnsw_params = logical_index + .iter() + .map(|(_, index)| index.statistics().unwrap()["sub_index"]["params"].clone()) + .collect::>(); + assert!( + hnsw_params.iter().all(|params| params == &hnsw_params[0]), + "append must preserve the reference segment's HNSW build parameters: {hnsw_params:?}" + ); + } - let results = dataset - .scan() + let mut fanout_scanner = dataset.scan(); + fanout_scanner .project(&["id"]) .unwrap() .nearest("vector", array.value(0).as_primitive::(), 2) .unwrap() .nprobes(2) - .refine(1) - .try_into_batch() + .refine(1); + let fanout_plan = fanout_scanner.explain_plan(true).await.unwrap(); + assert!( + fanout_plan.contains("ANNSubIndex: name=vector_idx, k=2, deltas=2"), + "logical vector query must fan out across both physical segments, plan was:\n{fanout_plan}" + ); + let results = fanout_scanner.try_into_batch().await.unwrap(); + assert_eq!(results.num_rows(), 2); + + for segment in &appended_segments { + let fragment_bitmap = segment + .fragment_bitmap + .as_ref() + .expect("vector segment must record fragment coverage"); + let expected_appended_row = fragment_bitmap.contains(appended_fragment_id); + let mut segment_scanner = dataset.scan(); + segment_scanner + .project(&["id"]) + .unwrap() + .nearest("vector", array.value(0).as_primitive::(), 1) + .unwrap() + .nprobes(2) + .refine(1) + .with_index_segments(vec![segment.uuid]) + .unwrap(); + let segment_result = segment_scanner.try_into_batch().await.unwrap(); + assert_eq!(segment_result.num_rows(), 1); + let id = segment_result["id"].as_primitive::().value(0); + assert_eq!( + id >= INITIAL_ROWS as u32, + expected_appended_row, + "segment {} returned row {id} outside its fragment coverage {:?}", + segment.uuid, + fragment_bitmap + ); + } + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) .await .unwrap(); - assert_eq!(results.num_rows(), 2); - let mut id_arr = results["id"].as_primitive::().values().to_vec(); - id_arr.sort(); - assert_eq!(id_arr, vec![0, 1000]); + let merged = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!( + merged.len(), + 1, + "the reference-compatible append segment must merge with its source" + ); + assert_eq!( + merged[0].index_version, + index_params.index_type().version(), + "merge must preserve the storage type's index version" + ); + assert_eq!( + merged[0].fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref(), + "the compatible merge must preserve exact fragment coverage" + ); } #[tokio::test] @@ -1360,6 +2744,139 @@ mod tests { assert_eq!(results[0].num_rows(), 10); } + #[tokio::test] + async fn test_vector_merge_filters_stable_row_id_replacements() { + const DIMENSION: usize = 4; + + let test_dir = TempStrDir::default(); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIMENSION as i32, + ), + false, + ), + ])); + let initial_values = (0..40) + .flat_map(|row| [if row < 20 { 1.0 } else { 0.0 }; DIMENSION]) + .collect::>(); + let initial = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..40)), + Arc::new( + FixedSizeListArray::try_new_from_values( + arrow_array::Float32Array::from(initial_values), + DIMENSION as i32, + ) + .unwrap(), + ), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new([Ok(initial)], schema.clone()), + test_dir.as_str(), + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("vector_idx".to_string()), + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + + let replacements = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(20..40)), + Arc::new( + FixedSizeListArray::try_new_from_values( + arrow_array::Float32Array::from(vec![10.0; 20 * DIMENSION]), + DIMENSION as i32, + ) + .unwrap(), + ), + ], + ) + .unwrap(); + let merge_job = MergeInsertBuilder::try_new(Arc::new(dataset), vec!["id".to_string()]) + .unwrap() + .when_matched(WhenMatched::UpdateAll) + .try_build() + .unwrap(); + let (dataset, stats) = merge_job + .execute(reader_to_stream(Box::new(RecordBatchIterator::new( + [Ok(replacements)], + schema, + )))) + .await + .unwrap(); + assert_eq!(stats.num_updated_rows, 20); + + let mut dataset = dataset.as_ref().clone(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + assert_eq!( + dataset + .load_indices_by_name("vector_idx") + .await + .unwrap() + .len(), + 2 + ); + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + + let logical_index = dataset + .open_logical_vector_index("vector", "vector_idx") + .await + .unwrap(); + assert_eq!(logical_index.num_segments(), 1); + assert_eq!( + logical_index + .num_rows_per_segment() + .into_iter() + .map(|(_, rows)| rows) + .sum::(), + 40, + "the merged index must contain one current copy of every stable row id" + ); + + let query = arrow_array::Float32Array::from(vec![0.0; DIMENSION]); + let result = dataset + .scan() + .project(&["id"]) + .unwrap() + .nearest("vector", &query, 5) + .unwrap() + .nprobes(1) + .try_into_batch() + .await + .unwrap(); + let ids = result["id"].as_primitive::(); + assert!( + ids.values().iter().all(|id| *id < 20), + "stale pre-update vectors must not survive the optimize merge: {ids:?}" + ); + } + #[tokio::test] async fn test_merge_indices_with_unindexed_frags_vector_subset() { const DIM: usize = 64; @@ -1664,25 +3181,121 @@ mod tests { let logical = crate::index::scalar_logical::open_named_scalar_index( &dataset, "text", - "text_fmindex", + "text_fmindex", + &NoOpMetricsCollector, + ) + .await + .unwrap(); + + for (pattern, expected) in [("old alpha", 1), ("new gamma", 1), ("needle", 2)] { + let query = TextQuery::StringContains(pattern.to_string()); + let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap(); + let row_addrs = match result { + SearchResult::Exact(row_addrs) => row_addrs, + other => panic!("expected exact result for {pattern}, got {other:?}"), + }; + let count = row_addrs.true_rows().row_addrs().unwrap().count(); + assert_eq!( + count, expected, + "expected {expected} matches for {pattern}, got {count}" + ); + } + } + + #[tokio::test] + async fn test_optimize_ngram_merge_remaps_deferred_compaction() { + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, true)])); + let make_batch = |values: &[&str]| { + RecordBatch::try_new( + schema.clone(), + vec![Arc::new(StringArray::from_iter_values( + values.iter().copied(), + ))], + ) + .unwrap() + }; + let reader = RecordBatchIterator::new( + vec![ + Ok(make_batch(&["alpha needle", "beta needle"])), + Ok(make_batch(&["gamma needle", "delta needle"])), + ], + schema.clone(), + ); + let mut dataset = Dataset::write( + reader, + test_uri, + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::NGram, + Some("text_ngram".into()), + &ScalarIndexParams::for_builtin(BuiltinIndexType::NGram), + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 10, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!(metrics.fragments_removed > 0 && metrics.fragments_added > 0); + + let appended = + RecordBatchIterator::new(vec![Ok(make_batch(&["epsilon needle"]))], schema.clone()); + let mut dataset = Dataset::write( + appended, + test_uri, + Some(WriteParams { + max_rows_per_file: 2, + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(1)) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let logical = crate::index::scalar_logical::open_named_scalar_index( + &dataset, + "text", + "text_ngram", &NoOpMetricsCollector, ) .await .unwrap(); - - for (pattern, expected) in [("old alpha", 1), ("new gamma", 1), ("needle", 2)] { - let query = TextQuery::StringContains(pattern.to_string()); - let result = logical.search(&query, &NoOpMetricsCollector).await.unwrap(); - let row_addrs = match result { - SearchResult::Exact(row_addrs) => row_addrs, - other => panic!("expected exact result for {pattern}, got {other:?}"), - }; - let count = row_addrs.true_rows().row_addrs().unwrap().count(); - assert_eq!( - count, expected, - "expected {expected} matches for {pattern}, got {count}" - ); - } + let result = logical + .search( + &TextQuery::StringContains("needle".to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let row_addrs = match result { + SearchResult::AtMost(row_addrs) => row_addrs, + other => panic!("expected AtMost result from ngram, got {other:?}"), + }; + assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 5); } #[tokio::test] @@ -1922,6 +3535,144 @@ mod tests { assert_eq!(rows, 2, "value 'd' lives in appended fragment"); } + #[tokio::test] + async fn test_optimize_append_with_backpressured_multichunk_read() { + const WIDE_DIMENSION: usize = 140_000; + const NARROW_DIMENSION: usize = 4_096; + const SHORT_ROWS: usize = 68; + const LONG_ROWS: usize = 128; + + fn make_batch(schema: Arc, rows: usize, base: usize) -> RecordBatch { + let docs_field = schema.field(1); + let DataType::List(item_field) = docs_field.data_type() else { + unreachable!("docs must be a list"); + }; + let DataType::Struct(doc_fields) = item_field.data_type() else { + unreachable!("docs items must be structs"); + }; + let wide_values = Float32Array::from_iter_values( + (0..rows * WIDE_DIMENSION).map(|index| ((index + base) % 1009) as f32), + ); + let narrow_values = Float32Array::from_iter_values( + (0..rows * NARROW_DIMENSION).map(|index| ((index + base) % 251) as f32), + ); + let docs = StructArray::new( + doc_fields.clone(), + vec![ + Arc::new( + FixedSizeListArray::try_new_from_values(wide_values, WIDE_DIMENSION as i32) + .unwrap(), + ), + Arc::new( + FixedSizeListArray::try_new_from_values( + narrow_values, + NARROW_DIMENSION as i32, + ) + .unwrap(), + ), + Arc::new(StringArray::from_iter_values( + (0..rows).map(|index| format!("document-{}", index + base)), + )), + ], + None, + ); + let docs = ListArray::new( + item_field.clone(), + OffsetBuffer::from_lengths(std::iter::repeat_n(1, rows)), + Arc::new(docs), + None, + ); + let ids = UInt64Array::from_iter_values(base as u64..(base + rows) as u64); + RecordBatch::try_new(schema, vec![Arc::new(ids), Arc::new(docs)]).unwrap() + } + + let vector_item = Arc::new(Field::new("item", DataType::Float32, true)); + let doc_fields = vec![ + Field::new( + "wide", + DataType::FixedSizeList(vector_item.clone(), WIDE_DIMENSION as i32), + true, + ), + Field::new( + "narrow", + DataType::FixedSizeList(vector_item, NARROW_DIMENSION as i32), + true, + ), + Field::new("text", DataType::Utf8, true), + ] + .into(); + let docs_item = Arc::new(Field::new("item", DataType::Struct(doc_fields), true)); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt64, false), + Field::new("docs", DataType::List(docs_item), true), + ])); + + let initial_batch = make_batch(schema.clone(), 1, 1_000); + let initial_reader = RecordBatchIterator::new(vec![Ok(initial_batch)], schema.clone()); + let mut dataset = Dataset::write( + initial_reader, + "memory://", + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + dataset + .create_index( + &["docs.text"], + IndexType::Inverted, + Some("docs_text".to_string()), + &InvertedIndexParams::default(), + true, + ) + .await + .unwrap(); + + let appended_batches = vec![ + Ok(make_batch(schema.clone(), SHORT_ROWS, 0)), + Ok(make_batch(schema.clone(), LONG_ROWS, SHORT_ROWS)), + ]; + let appended_reader = RecordBatchIterator::new(appended_batches, schema); + dataset + .append( + appended_reader, + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::V2_1), + ..Default::default() + }), + ) + .await + .unwrap(); + // The deleted first batch makes the live delta start exactly at the + // second write-batch boundary, matching a merge-insert style update. + dataset.delete("id < 68").await.unwrap(); + + let optimize_options = OptimizeOptions::append(); + // Nested FTS reads the entire `docs` root. Under this budget the wide + // sibling is split into same-priority chunks, then the narrow sibling's + // higher-priority I/O consumes the remaining budget while the wide read + // is awaited. Admitted chunks must continue despite that backpressure. + let optimize = crate::index::scalar::TEST_TRAINING_IO_BUFFER_SIZE.scope( + 70 * 1024 * 1024, + dataset.optimize_indices(&optimize_options), + ); + tokio::time::timeout(std::time::Duration::from_secs(20), optimize) + .await + .expect("incremental nested FTS optimization timed out") + .unwrap(); + + let segments = dataset + .load_indices() + .await + .unwrap() + .iter() + .filter(|index| index.name == "docs_text") + .count(); + assert_eq!(segments, 2, "optimization should add one FTS delta segment"); + } + #[tokio::test] async fn test_optimize_btree_keeps_rows_with_stable_row_ids_after_compaction() { async fn query_id_count(dataset: &Dataset, id: &str) -> usize { @@ -2000,6 +3751,376 @@ mod tests { assert_eq!(query_id_count(&dataset, "song-42").await, 1); } + /// Updating an indexed vector column in place (`update_columns` + + /// `Operation::Update`) keeps the fragment id and the row address, so after the + /// fragment is pruned from the old segment's bitmap that segment still physically + /// holds the pre-update vector. Merging the segment must drop those rows: the merge + /// concatenates its physical rows with the freshly scanned ones and commits the + /// union of both coverages, so a surviving stale row is authorized by the same + /// bitmap as its fresh copy and no read-time filter can separate them. + #[rstest] + // IVF_FLAT keeps the vector verbatim, so the query distance tells which of the two + // copies survived. IVF_PQ re-uses the codebook trained before the update, which + // cannot represent the new value, so it only pins the row count and covers the + // transposed-code path in `take_partition_batches`. + #[case::ivf_flat(VectorIndexParams::ivf_flat(1, MetricType::L2), true)] + #[case::ivf_pq(VectorIndexParams::ivf_pq(1, 8, 16, MetricType::L2, 2), false)] + #[tokio::test] + async fn test_optimize_vector_index_drops_stale_rows_on_merge( + #[case] index_params: VectorIndexParams, + #[case] distance_is_exact: bool, + // The two row-id schemes take different filter branches: an address carries its + // fragment, a stable row id needs the fragments' persisted row-id sequences. + #[values(false, true)] enable_stable_row_ids: bool, + ) { + use crate::dataset::transaction::{Operation, UpdateMode, UpdatedFragmentOffsets}; + use arrow::datatypes::UInt64Type; + use arrow_array::{Float32Array, RecordBatchReader, UInt64Array}; + use lance_core::ROW_ID; + use lance_index::vector::DIST_COL; + use std::collections::HashMap; + + // Must match the sub-vector count in the IVF_PQ case above. + const DIM: usize = 16; + const BULK_ROWS: usize = 1000; + const UPDATED_ID: u32 = 10_000; + const STALE_VALUE: f32 = 2.0; + const FRESH_VALUE: f32 = 10.8; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let vector_type = DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("vector", vector_type.clone(), true), + ])); + let constant_vector = |value: f32| { + FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values(std::iter::repeat_n(value, DIM)), + DIM as i32, + ) + .unwrap() + }; + + // Fragment 0: bulk vectors in [0, 1). None of them is near the query, so a + // surviving stale copy of the updated row ranks well inside top-k. + let bulk = generate_random_array_with_seed::(BULK_ROWS * DIM, [42; 32]); + let bulk_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values(0..BULK_ROWS as u32)), + Arc::new(FixedSizeListArray::try_new_from_values(bulk, DIM as i32).unwrap()), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(bulk_batch)], schema.clone()), + test_uri, + Some(WriteParams { + enable_stable_row_ids, + ..Default::default() + }), + ) + .await + .unwrap(); + + // Fragment 1: the single row that gets updated in place. + let stale_batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values([UPDATED_ID])), + Arc::new(constant_vector(STALE_VALUE)), + ], + ) + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(stale_batch)], schema.clone()), + None, + ) + .await + .unwrap(); + assert_eq!(dataset.get_fragments().len(), 2); + + // One segment covering both fragments. + dataset + .create_index(&["vector"], IndexType::Vector, None, &index_params, true) + .await + .unwrap(); + + let mut fragment = dataset.get_fragment(1).unwrap(); + let mut fragment_scan = fragment.scan(); + fragment_scan.with_row_id(); + let updated_row_id = fragment_scan.try_into_batch().await.unwrap()[ROW_ID] + .as_primitive::() + .value(0); + + let update_schema = Arc::new(Schema::new(vec![ + Field::new(ROW_ID, DataType::UInt64, false), + Field::new("vector", vector_type, true), + ])); + let update_batch = RecordBatch::try_new( + update_schema.clone(), + vec![ + Arc::new(UInt64Array::from_iter_values([updated_row_id])), + Arc::new(constant_vector(FRESH_VALUE)), + ], + ) + .unwrap(); + let right_stream: Box = Box::new(RecordBatchIterator::new( + vec![Ok(update_batch)], + update_schema, + )); + let updated = fragment + .update_columns_with_offsets(right_stream, ROW_ID, ROW_ID) + .await + .unwrap(); + let updated_fragment_id = updated.fragment.id; + let dataset = Dataset::commit( + test_uri, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![updated.fragment], + new_fragments: vec![], + fields_modified: updated.fields_modified, + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: Some(UpdatedFragmentOffsets(HashMap::from([( + updated_fragment_id, + updated.matched_offsets, + )]))), + }, + Some(dataset.version().version), + None, + None, + Default::default(), + true, + ) + .await + .unwrap(); + + let mut dataset = dataset; + dataset + .optimize_indices(&OptimizeOptions::merge(1)) + .await + .unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let segments = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!(segments.len(), 1, "merge must leave a single segment"); + + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &constant_vector(FRESH_VALUE).value(0), 10) + .unwrap() + .with_row_id() + .project(&["id"]) + .unwrap(); + let results = scanner.try_into_batch().await.unwrap(); + let hits = results["id"] + .as_primitive::() + .values() + .iter() + .zip(results[DIST_COL].as_primitive::().values()) + .filter(|(id, _)| **id == UPDATED_ID) + .map(|(_, distance)| *distance) + .collect::>(); + assert_eq!( + hits.len(), + 1, + "updated row returned {} times after merge; row ids = {:?}", + hits.len(), + results[ROW_ID].as_primitive::().values() + ); + if distance_is_exact { + // The pre-update vector would sit at (FRESH - STALE)^2 * DIM from the query, + // so anything but ~0 means the stale copy is the one that survived. + assert!( + hits[0] < 1.0, + "surviving copy is the pre-update one: distance {} to the updated vector", + hits[0] + ); + } + } + + /// The ordinary update path (`UpdateBuilder`, `UpdateMode::RewriteRows`) commits + /// `fields_modified: vec![]`, so the old fragment keeps its place in the segment's + /// bitmap while its physical row is only deletion-marked. Under stable row ids the + /// rewritten copy reuses the same row id, so a merge that keeps every covered row + /// emits that id twice and the read-time filter admits both: the id is live again + /// at its new address. Coverage alone is therefore not a sufficient merge filter. + #[rstest] + // The two optimize modes read existing rows through different code, and both must + // filter: an explicit merge through `take_partition_batches`, and the default + // options over under-sized partitions through `partition_row_ids`, which feeds the + // join that rebuilds them. + #[case::merge(1, OptimizeOptions::merge(1))] + #[case::join(4, OptimizeOptions::new())] + #[tokio::test] + async fn test_optimize_vector_index_drops_rewritten_rows_on_merge( + #[case] num_partitions: usize, + #[case] optimize_options: OptimizeOptions, + // Only the stable-row-id scheme can duplicate: an address-domain rewrite lands + // at a new address, so the stale posting is masked at query time. + #[values(false, true)] enable_stable_row_ids: bool, + ) { + use crate::dataset::UpdateBuilder; + use arrow::datatypes::UInt64Type; + use arrow_array::Float32Array; + use lance_core::ROW_ID; + use lance_index::vector::DIST_COL; + + const DIM: usize = 4; + const BULK_ROWS: usize = 1000; + const UPDATED_ID: u32 = 10_000; + const STALE_VALUE: f32 = 2.0; + const FRESH_VALUE: f32 = 10.8; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ), + true, + ), + ])); + let constant_vector = |value: f32| { + FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values(std::iter::repeat_n(value, DIM)), + DIM as i32, + ) + .unwrap() + }; + + // The updated row shares its fragment with the bulk rows, so the update leaves + // that fragment covered by the segment with one row deletion-marked instead of + // retiring it. The bulk vectors sit in [0, 1), far from the query, so a + // surviving stale copy of the updated row ranks well inside top-k. + let bulk = generate_random_array_with_seed::(BULK_ROWS * DIM, [42; 32]); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values( + (0..BULK_ROWS as u32).chain(std::iter::once(UPDATED_ID)), + )), + Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from_iter_values( + bulk.values() + .iter() + .copied() + .chain(std::iter::repeat_n(STALE_VALUE, DIM)), + ), + DIM as i32, + ) + .unwrap(), + ), + ], + ) + .unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + test_uri, + Some(WriteParams { + enable_stable_row_ids, + ..Default::default() + }), + ) + .await + .unwrap(); + + dataset + .create_index( + &["vector"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(num_partitions, MetricType::L2), + true, + ) + .await + .unwrap(); + + let fresh_literal = format!( + "array[{}]", + std::iter::repeat_n(FRESH_VALUE.to_string(), DIM) + .collect::>() + .join(", ") + ); + let mut dataset = UpdateBuilder::new(Arc::new(dataset)) + .update_where(&format!("id = {UPDATED_ID}")) + .unwrap() + .set("vector", &fresh_literal) + .unwrap() + .build() + .unwrap() + .execute() + .await + .unwrap() + .new_dataset + .as_ref() + .clone(); + dataset.optimize_indices(&optimize_options).await.unwrap(); + + let dataset = DatasetBuilder::from_uri(test_uri).load().await.unwrap(); + let segments = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!(segments.len(), 1, "merge must leave a single segment"); + let partitions_after = dataset + .open_vector_index("vector", &segments[0].uuid, &NoOpMetricsCollector) + .await + .unwrap() + .ivf_model() + .num_partitions(); + // A join is what routes existing rows through `partition_row_ids`; without it + // this case would silently degrade into a second copy of the merge case. + assert!( + num_partitions == 1 || partitions_after < num_partitions, + "expected a join, but the index still has {partitions_after} partitions" + ); + + let mut scanner = dataset.scan(); + scanner + .nearest("vector", &constant_vector(FRESH_VALUE).value(0), 10) + .unwrap() + .with_row_id() + .project(&["id"]) + .unwrap(); + let results = scanner.try_into_batch().await.unwrap(); + let hits = results["id"] + .as_primitive::() + .values() + .iter() + .zip(results[DIST_COL].as_primitive::().values()) + .filter(|(id, _)| **id == UPDATED_ID) + .map(|(_, distance)| *distance) + .collect::>(); + assert_eq!( + hits.len(), + 1, + "rewritten row returned {} times after optimize; row ids = {:?}", + hits.len(), + results[ROW_ID].as_primitive::().values() + ); + // The pre-update vector would sit at (FRESH - STALE)^2 * DIM from the query, so + // anything but ~0 means the stale copy is the one that survived. + assert!( + hits[0] < 1.0, + "surviving copy is the pre-update one: distance {} to the updated vector", + hits[0] + ); + } + /// Under stable row ids, updating an indexed column and then calling /// `optimize_indices` must not leave stale entries (old value -> updated row) /// in the scalar index. An update deletes the old copy of each row and diff --git a/rust/lance/src/index/create.rs b/rust/lance/src/index/create.rs index 702a8c4e49f..8cd006a59a1 100644 --- a/rust/lance/src/index/create.rs +++ b/rust/lance/src/index/create.rs @@ -9,11 +9,11 @@ use crate::{ }, index::{ DatasetIndexExt, DatasetIndexInternalExt, IntoIndexSegment, - build_index_metadata_from_segments, + build_index_metadata_from_segments, load_all_indices, scalar::{build_bitmap_index_segment, build_scalar_index}, vector::{ - LANCE_VECTOR_INDEX, VectorIndexParams, build_distributed_vector_index, - build_empty_vector_index, build_vector_index, + LANCE_VECTOR_INDEX, StageParams, VectorIndexParams, build_distributed_vector_index, + build_empty_vector_index, build_filtered_vector_index, build_vector_index, }, vector_index_details, vector_index_details_default, }, @@ -21,10 +21,15 @@ use crate::{ use futures::{FutureExt, future::BoxFuture}; use lance_core::datatypes::format_field_path; use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress}; -use lance_index::{IndexParams, IndexType, scalar::CreatedIndex}; +use lance_index::{ + IndexParams, IndexType, registry::plugin_name_from_details_url, scalar::CreatedIndex, +}; use lance_index::{ metrics::NoOpMetricsCollector, - scalar::{LANCE_SCALAR_INDEX, ScalarIndexParams, inverted::tokenizer::InvertedIndexParams}, + scalar::{ + LANCE_SCALAR_INDEX, ScalarIndexParams, index_files_to_table, + inverted::tokenizer::InvertedIndexParams, table_files_to_index, + }, }; use lance_table::format::{IndexMetadata, list_index_files_with_sizes}; use std::{collections::HashMap, future::IntoFuture, sync::Arc}; @@ -44,6 +49,23 @@ fn default_index_name(fields: &[&str]) -> String { } } +fn resolved_inverted_params(params: &ScalarIndexParams) -> Result { + let provided = params + .params + .as_deref() + .map(serde_json::from_str::) + .transpose()? + .unwrap_or_else(|| serde_json::json!({})); + provided.as_object().ok_or_else(|| { + Error::invalid_input("inverted index parameters must be a JSON object".to_string()) + })?; + Ok(serde_json::from_value(provided)?) +} + +fn scalar_params_from_inverted(params: &InvertedIndexParams) -> Result { + Ok(ScalarIndexParams::new("inverted".to_string()).with_params(¶ms.to_training_json()?)) +} + pub struct CreateIndexBuilder<'a> { dataset: &'a mut Dataset, columns: Vec, @@ -144,29 +166,103 @@ impl<'a> CreateIndexBuilder<'a> { )); } let column_input = &self.columns[0]; + let scalar_fts_request = self.index_type == IndexType::Scalar + && self + .params + .as_any() + .downcast_ref::() + .is_some_and(|params| { + params.index_type.eq_ignore_ascii_case("inverted") + || params.index_type.eq_ignore_ascii_case("fts") + }); + let inverted_params = if self.index_type == IndexType::Inverted { + if let Some(params) = self + .params + .as_any() + .downcast_ref::() + { + Some(params.clone()) + } else if let Some(params) = + self.params.as_any().downcast_ref::() + { + Some(resolved_inverted_params(params)?) + } else { + return Err(Error::index( + "Inverted index type must take InvertedIndexParams or ScalarIndexParams" + .to_string(), + )); + } + } else if scalar_fts_request { + Some(resolved_inverted_params( + self.params + .as_any() + .downcast_ref::() + .expect("scalar_fts_request verified scalar params"), + )?) + } else { + None + }; + let resolved_fts_field = if let Some(params) = &inverted_params { + Some(crate::index::scalar::inverted::resolve_fts_field( + self.dataset.schema(), + column_input, + params.get_document_granularity(), + )?) + } else { + None + }; + let lookup_column = resolved_fts_field + .as_ref() + .map(|resolved| resolved.root_column.as_str()) + .unwrap_or(column_input); // Use case-insensitive lookup for both simple and nested paths. // resolve_case_insensitive tries exact match first, then falls back to case-insensitive. - let Some(field_path) = self.dataset.schema().resolve_case_insensitive(column_input) else { + let Some(field_path) = self.dataset.schema().resolve_case_insensitive(lookup_column) else { return Err(Error::index(format!( "CreateIndex: column '{column_input}' does not exist" ))); }; - let field = *field_path.last().unwrap(); + let field = if let Some(resolved) = &resolved_fts_field { + self.dataset + .schema() + .field_by_id(resolved.final_field_id) + .ok_or_else(|| { + Error::index(format!( + "CreateIndex: FTS field path '{column_input}' resolved to missing field id {}", + resolved.final_field_id + )) + })? + } else { + *field_path.last().ok_or_else(|| { + Error::index(format!( + "CreateIndex: column '{column_input}' resolved to an empty field path" + )) + })? + }; // Reconstruct the column path with correct case from schema // Use quoted format for SQL parsing (special chars are quoted) let names: Vec<&str> = field_path.iter().map(|f| f.name.as_str()).collect(); let quoted_column: String = format_field_path(&names); - let column = quoted_column.as_str(); - - // If train is true but dataset is empty, automatically set train to false - let train = if self.train { - self.dataset.count_rows(None).await? > 0 - } else { - false - }; + let column = resolved_fts_field + .as_ref() + .map(|resolved| resolved.canonical_path.as_str()) + .unwrap_or(quoted_column.as_str()); + + let vector_fragments_for_validation = + is_builtin_vector_index(self.index_type, self.params) + .then_some(self.fragments.as_deref()) + .flatten(); + let train = should_train_index( + self.dataset, + self.train, + vector_fragments_for_validation, + ) + .await?; - // Load indices from the disk. - let indices = self.dataset.load_indices().await?; + // Load indices from the disk. Names are reserved against every index the + // manifest carries: one this build cannot read still owns its name, and + // handing that name out again commits two indices under it. + let indices = load_all_indices(self.dataset).await?; let fri = self .dataset .open_frag_reuse_index(&NoOpMetricsCollector) @@ -174,16 +270,32 @@ impl<'a> CreateIndexBuilder<'a> { let index_name = if let Some(name) = self.name.take() { name } else { - // Generate default name with collision handling - let column_path = default_index_name(&names); + // Generate default name with collision handling. + // A name is available when there is no existing index with: + // - the same name AND different fields, OR + // - the same name AND same field BUT a different index kind + let column_path = resolved_fts_field + .as_ref() + .map(|resolved| { + if resolved.document_granularity.is_list_element() { + format!("{}_list_element", resolved.canonical_path) + } else { + resolved.canonical_path.clone() + } + }) + .unwrap_or_else(|| default_index_name(&names)); let base_name = format!("{column_path}_idx"); let mut candidate = base_name.clone(); let mut counter = 2; // Start with no suffix, then use _2, _3, ... - // Find unique name by appending numeric suffix if needed - while indices - .iter() - .any(|idx| idx.name == candidate && idx.fields != [field.id]) - { + while indices.iter().any(|idx| { + // A covered index still names the same column by its keyed + // prefix; only that prefix decides whether this is "the same + // field", not the full `fields` vector including carried + // columns. + let different_field = idx.keyed_field() != Some(field.id); + idx.name == candidate + && (different_field || !index_matches_type(idx, self.index_type, self.params)) + }) { candidate = format!("{base_name}_{counter}"); counter += 1; } @@ -193,10 +305,11 @@ impl<'a> CreateIndexBuilder<'a> { .iter() .filter(|idx| idx.name == index_name) .collect::>(); - if existing_named_indices - .iter() - .any(|idx| idx.fields != [field.id]) - { + if existing_named_indices.iter().any(|idx| { + // Same rule as above: the keyed prefix decides identity, not the + // full `fields` vector. + idx.keyed_field() != Some(field.id) + }) { return Err(Error::index(format!( "Index name '{index_name}' already exists with different fields, \ please specify a different name" @@ -237,7 +350,7 @@ impl<'a> CreateIndexBuilder<'a> { let base_params = ScalarIndexParams::for_builtin(self.index_type.try_into()?); // If custom params were provided, extract the params JSON and apply it - let params = if let Some(provided_params) = + let mut params = if let Some(provided_params) = self.params.as_any().downcast_ref::() { if let Some(params_json) = &provided_params.params { @@ -255,6 +368,9 @@ impl<'a> CreateIndexBuilder<'a> { } else { base_params }; + if let Some(inverted_params) = &inverted_params { + params = scalar_params_from_inverted(inverted_params)?; + } let preprocesssed_data = self .preprocessed_data @@ -307,12 +423,18 @@ impl<'a> CreateIndexBuilder<'a> { .downcast_ref::() .ok_or_else(|| { Error::index("Scalar index type must take a ScalarIndexParams".to_string()) - })?; + })? + .clone(); + let params = if let Some(inverted_params) = &inverted_params { + scalar_params_from_inverted(inverted_params)? + } else { + params + }; build_scalar_index( self.dataset, column, index_id, - params, + ¶ms, train, self.fragments.clone(), None, @@ -322,18 +444,11 @@ impl<'a> CreateIndexBuilder<'a> { } (IndexType::Inverted, _) => { // Inverted index params. - let inverted_params = self - .params - .as_any() - .downcast_ref::() - .ok_or_else(|| { - Error::index( - "Inverted index type must take a InvertedIndexParams".to_string(), - ) - })?; + let inverted_params = inverted_params + .as_ref() + .expect("IndexType::Inverted parameters were resolved above"); - let params = ScalarIndexParams::new("inverted".to_string()) - .with_params(&inverted_params.to_training_json()?); + let params = scalar_params_from_inverted(inverted_params)?; build_scalar_index( self.dataset, column, @@ -367,24 +482,40 @@ impl<'a> CreateIndexBuilder<'a> { })?; let index_version = vec_params.index_type().version() as u32; + let effective_fragments = + effective_vector_fragments(self.dataset, self.fragments.as_deref()); let files = if train { // Check if this is distributed indexing (fragment-level) - if let Some(fragments) = &self.fragments { - // For distributed indexing, build only on specified fragments - // This creates temporary index metadata without committing - let (segment_uuid, files) = Box::pin(build_distributed_vector_index( - self.dataset, - column, - &index_name, - index_id, - vec_params, - fri, - fragments, - self.progress.clone(), - )) - .await?; - output_index_uuid = segment_uuid; - files + if let Some(fragments) = effective_fragments.as_deref() { + if vector_params_have_precomputed_ivf(vec_params) { + // For distributed indexing, build only on specified fragments + // This creates temporary index metadata without committing + let (segment_uuid, files) = Box::pin(build_distributed_vector_index( + self.dataset, + column, + &index_name, + index_id, + vec_params, + fri, + fragments, + self.progress.clone(), + )) + .await?; + output_index_uuid = segment_uuid; + files + } else { + Box::pin(build_filtered_vector_index( + self.dataset, + column, + &index_name, + index_id, + vec_params, + fri, + fragments, + self.progress.clone(), + )) + .await? + } } else { // Standard full dataset indexing Box::pin(build_vector_index( @@ -412,7 +543,7 @@ impl<'a> CreateIndexBuilder<'a> { CreatedIndex { index_details: vector_index_details(vec_params), index_version, - files, + files: table_files_to_index(files), } } // Can't use if let Some(...) here because it's not stable yet. @@ -451,7 +582,7 @@ impl<'a> CreateIndexBuilder<'a> { CreatedIndex { index_details: vector_index_details_default(), index_version: self.index_type.version() as u32, - files, + files: table_files_to_index(files), } } (IndexType::FragmentReuse, _) => { @@ -470,6 +601,7 @@ impl<'a> CreateIndexBuilder<'a> { uuid: output_index_uuid, name: index_name, fields: vec![field.id], + covering_fields: vec![], dataset_version: self.dataset.manifest.version, fragment_bitmap: if train { match &self.fragments { @@ -484,7 +616,7 @@ impl<'a> CreateIndexBuilder<'a> { index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }) } .boxed() @@ -503,8 +635,7 @@ impl<'a> CreateIndexBuilder<'a> { let new_idx = self.execute_uncommitted().await?; let index_uuid = new_idx.uuid; let removed_indices = if self.replace { - self.dataset - .load_indices() + load_all_indices(self.dataset) .await? .iter() .filter(|idx| idx.name == new_idx.name) @@ -513,28 +644,15 @@ impl<'a> CreateIndexBuilder<'a> { } else { vec![] }; - let transaction = if uses_segment_commit_path(self.index_type, self.params) { - let dataset_version = new_idx.dataset_version; - TransactionBuilder::new( - dataset_version, - Operation::CreateIndex { - new_indices: vec![new_idx], - removed_indices, - }, - ) - .transaction_properties(self.transaction_properties.clone()) - .build() - } else { - TransactionBuilder::new( - new_idx.dataset_version, - Operation::CreateIndex { - new_indices: vec![new_idx], - removed_indices, - }, - ) - .transaction_properties(self.transaction_properties.clone()) - .build() - }; + let transaction = TransactionBuilder::new( + new_idx.dataset_version, + Operation::CreateIndex { + new_indices: vec![new_idx], + removed_indices, + }, + ) + .transaction_properties(self.transaction_properties.clone()) + .build(); self.dataset .apply_commit(transaction, &Default::default(), &Default::default()) @@ -590,7 +708,7 @@ impl<'a> CreateIndexBuilder<'a> { false }; - let indices = self.dataset.load_indices().await?; + let indices = load_all_indices(self.dataset).await?; let index_name = if let Some(name) = self.name.take() { name } else { @@ -598,10 +716,11 @@ impl<'a> CreateIndexBuilder<'a> { let base_name = format!("{column_path}_idx"); let mut candidate = base_name.clone(); let mut counter = 2; - while indices - .iter() - .any(|idx| idx.name == candidate && idx.fields != [field.id]) - { + while indices.iter().any(|idx| { + // Same name-collision rule as `execute_uncommitted_impl`'s + // default-name loop above in this file. + idx.name == candidate && idx.keyed_field() != Some(field.id) + }) { candidate = format!("{base_name}_{counter}"); counter += 1; } @@ -611,10 +730,11 @@ impl<'a> CreateIndexBuilder<'a> { .iter() .filter(|idx| idx.name == index_name) .collect::>(); - if existing_named_indices - .iter() - .any(|idx| idx.fields != [field.id]) - { + if existing_named_indices.iter().any(|idx| { + // Same rule as above: the keyed prefix decides identity, not the + // full `fields` vector. + idx.keyed_field() != Some(field.id) + }) { return Err(Error::index(format!( "Index name '{index_name}' already exists with different fields, \ please specify a different name" @@ -645,13 +765,14 @@ impl<'a> CreateIndexBuilder<'a> { uuid: segment_uuid, name: index_name.clone(), fields: vec![field.id], + covering_fields: vec![], dataset_version: self.dataset.manifest.version, fragment_bitmap: Some(roaring::RoaringBitmap::new()), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }; let segments = vec![metadata.into_index_segment()?]; let new_indices = @@ -714,13 +835,14 @@ impl<'a> CreateIndexBuilder<'a> { uuid: segment_uuid, name: index_name.clone(), fields: vec![field.id], + covering_fields: vec![], dataset_version: self.dataset.manifest.version, fragment_bitmap: Some(fragment_ids.into_iter().collect()), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }); } @@ -733,8 +855,6 @@ impl<'a> CreateIndexBuilder<'a> { build_index_metadata_from_segments(self.dataset, &index_name, field.id, segments) .await?; - // Collect all same-name indices for removal when replace is set, - // matching the standard execute() path behavior. let removed_indices = if self.replace { existing_named_indices .into_iter() @@ -768,43 +888,33 @@ impl<'a> CreateIndexBuilder<'a> { } } -fn is_btree_scalar_params(params: &dyn IndexParams) -> bool { - params - .as_any() - .downcast_ref::() - .is_some_and(|p| p.index_type.eq_ignore_ascii_case("btree")) -} - -/// Validate that a user-supplied `index_uuid` is permitted for this build. -fn ensure_index_uuid_allowed( +/// Returns true if an existing `IndexMetadata` matches the given type +fn index_matches_type( + idx: &IndexMetadata, index_type: IndexType, params: &dyn IndexParams, - fragments: Option<&Vec>, - index_uuid: Option<&Uuid>, -) -> Result<()> { - let is_btree = index_type == IndexType::BTree - || params - .as_any() - .downcast_ref::() - .map(|params| params.index_type.eq_ignore_ascii_case("btree")) - .unwrap_or(false); - - if index_uuid.is_some() && fragments.is_some_and(|fragments| !fragments.is_empty()) && is_btree +) -> bool { + let Some(d) = &idx.index_details else { + // Fallback for legacy indexes, assume we are not trying to change the type + return true; + }; + // When index_type is Scalar the actual type is carried in ScalarIndexParams as a + // plugin name string (e.g. "zonemap"). The registry uses the same normalization + // for type_url lookup: lowercase the last path segment and strip "indexdetails". + // Compare directly instead of going through IndexType so this path stays valid + // as we move away from the IndexType enum. + if index_type == IndexType::Scalar + && let Some(scalar_params) = params.as_any().downcast_ref::() { - return Err(Error::invalid_input( - "index_uuid is no longer accepted for BTree distributed index builds; segment UUIDs \ - are generated by Lance and returned in the index metadata." - .to_string(), - )); + return scalar_params.index_type.to_lowercase() + == plugin_name_from_details_url(&d.type_url); } - Ok(()) + index_type.matches_details(d) } -fn uses_segment_commit_path(index_type: IndexType, params: &dyn IndexParams) -> bool { - let params_family = params.index_name(); - - if params_family == LANCE_VECTOR_INDEX +fn is_builtin_vector_index(index_type: IndexType, params: &dyn IndexParams) -> bool { + params.index_name() == LANCE_VECTOR_INDEX && matches!( index_type, IndexType::Vector @@ -817,19 +927,84 @@ fn uses_segment_commit_path(index_type: IndexType, params: &dyn IndexParams) -> | IndexType::IvfHnswSq ) && params.as_any().is::() - { - return true; +} + +async fn should_train_index( + dataset: &Dataset, + train: bool, + vector_fragments: Option<&[u32]>, +) -> Result { + if !train { + return Ok(false); } - if params_family == LANCE_SCALAR_INDEX { - match index_type { - IndexType::BTree => return true, - IndexType::Scalar if is_btree_scalar_params(params) => return true, - _ => {} - } + if dataset.fragment_bitmap.is_empty() { + return Ok(false); + } + + if let Some(fragment_ids) = vector_fragments { + dataset.get_fragments_from_ids(fragment_ids)?; + return Ok(true); + } + + Ok(dataset.count_rows(None).await? > 0) +} + +fn vector_params_have_precomputed_ivf(params: &VectorIndexParams) -> bool { + matches!( + params.stages.first(), + Some(StageParams::Ivf(ivf_params)) if ivf_params.centroids.is_some() + ) +} + +fn effective_vector_fragments(dataset: &Dataset, fragments: Option<&[u32]>) -> Option> { + let fragments = Dataset::normalize_fragment_ids(fragments?); + let fragment_bitmap: roaring::RoaringBitmap = fragments.iter().copied().collect(); + (fragment_bitmap != *dataset.fragment_bitmap).then_some(fragments) +} + +/// Validate that a user-supplied `index_uuid` is permitted for this build. +fn ensure_index_uuid_allowed( + index_type: IndexType, + params: &dyn IndexParams, + fragments: Option<&Vec>, + index_uuid: Option<&Uuid>, +) -> Result<()> { + let scalar_index_type = params + .as_any() + .downcast_ref::() + .map(|params| params.index_type.as_str()); + let unsafe_distributed_type = if index_type == IndexType::BTree + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("btree")) + { + Some("BTree") + } else if index_type == IndexType::RTree + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("rtree")) + { + Some("RTree") + } else if index_type == IndexType::NGram + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("ngram")) + { + Some("NGram") + } else if index_type == IndexType::LabelList + || scalar_index_type.is_some_and(|index_type| index_type.eq_ignore_ascii_case("labellist")) + { + Some("LabelList") + } else { + None + }; + + if index_uuid.is_some() + && fragments.is_some_and(|fragments| !fragments.is_empty()) + && let Some(index_type) = unsafe_distributed_type + { + return Err(Error::invalid_input(format!( + "index_uuid is no longer accepted for {index_type} distributed index builds; \ + segment UUIDs are generated by Lance and returned in the index metadata." + ))); } - false + Ok(()) } impl<'a> IntoFuture for CreateIndexBuilder<'a> { @@ -846,6 +1021,7 @@ mod tests { use super::*; use crate::dataset::{WriteMode, WriteParams}; use crate::index::{DatasetIndexExt, IndexSegment}; + use crate::utils::test::covering; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; use arrow::datatypes::{Float32Type, Int32Type, Int64Type}; use arrow_array::cast::AsArray; @@ -859,12 +1035,15 @@ mod tests { use lance_index::optimize::OptimizeOptions; use lance_index::progress::IndexBuildProgress; use lance_index::scalar::{ - FullTextSearchQuery, SargableQuery, SearchResult, inverted::tokenizer::InvertedIndexParams, + BloomFilterQuery, FullTextSearchQuery, SargableQuery, SearchResult, + inverted::tokenizer::InvertedIndexParams, }; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::kmeans::{KMeansParams, train_kmeans}; use lance_linalg::distance::{DistanceType, MetricType}; + use roaring::RoaringBitmap; + use rstest::rstest; use std::{collections::BTreeSet, ops::Bound, sync::Arc}; use uuid::Uuid; @@ -886,6 +1065,24 @@ mod tests { assert_eq!(json.get("num_workers"), Some(&serde_json::Value::from(7))); } + #[rstest] + #[case::omitted(r#"{"base_tokenizer":"ngram"}"#, false)] + #[case::explicit( + r#"{"base_tokenizer":"ngram","stem":true,"remove_stop_words":true}"#, + true + )] + fn test_generic_inverted_params_preserve_ngram_defaults( + #[case] raw_params: &str, + #[case] expected_word_filters: bool, + ) { + let provided: serde_json::Value = serde_json::from_str(raw_params).unwrap(); + let params = ScalarIndexParams::new("inverted".to_string()).with_params(&provided); + let resolved = serde_json::to_value(resolved_inverted_params(¶ms).unwrap()).unwrap(); + assert_eq!(resolved["base_tokenizer"], "ngram"); + assert_eq!(resolved["stem"], expected_word_filters); + assert_eq!(resolved["remove_stop_words"], expected_word_filters); + } + #[test] fn test_default_index_name() { // Single field - preserved as-is @@ -1009,6 +1206,112 @@ mod tests { assert!(err.to_string().contains("already exists")); } + /// A covered index still names the same column by its keyed prefix. Both + /// the auto-naming loop and the explicit-name check must recognize an + /// existing covered index as matching its keyed field, not the full + /// `fields` vector including the carried column -- which would otherwise + /// silently rename around it (auto) or spuriously reject as "different + /// fields" (explicit) instead of reaching the ordinary "already exists, + /// use replace=True" outcome. + #[tokio::test] + async fn test_index_name_collision_recognizes_covered_index() { + let mut dataset = gen_batch() + .col("a", lance_datagen::array::step::()) + .col("b", lance_datagen::array::step::()) + .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(100)) + .await + .unwrap(); + + covering::commit_synthetic_covered_index(&mut dataset, "a_idx", "a", "b").await; + + let params = ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BTree); + + // Default naming (the auto-name loop): must recognize the covered + // "a_idx" as the same field, not silently rename around it to + // "a_idx_2". + let err = CreateIndexBuilder::new(&mut dataset, &["a"], IndexType::BTree, ¶ms) + .execute() + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("already exists, please specify a different name or use replace=True"), + "{err}" + ); + + // Explicit naming (the existing_named_indices check): must not + // misfire "different fields" for the same reason. + let err2 = CreateIndexBuilder::new(&mut dataset, &["a"], IndexType::BTree, ¶ms) + .name("a_idx".to_string()) + .execute() + .await + .unwrap_err(); + assert!( + err2.to_string() + .contains("already exists, please specify a different name or use replace=True"), + "{err2}" + ); + } + + /// The same guard shape, duplicated in `execute_multi_segment_fmindex` + /// (the multi-segment FM-Index build path) under its own default-naming + /// loop and its own explicit-name check. `train(false)` keeps this test + /// fast: the guard runs before any real FM training, so an untrained + /// empty segment is enough to reach it. + #[tokio::test] + async fn test_fmindex_multi_segment_name_collision_recognizes_covered_index() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let batch1 = create_text_batch(0, 10); + let batch2 = create_text_batch(10, 20); + let write_params = WriteParams { + max_rows_per_file: 10, + max_rows_per_group: 5, + ..Default::default() + }; + let batches = RecordBatchIterator::new( + vec![Ok(batch1), Ok(batch2)], + create_text_batch(0, 1).schema(), + ); + let mut dataset = Dataset::write(batches, &dataset_uri, Some(write_params)) + .await + .unwrap(); + + covering::commit_synthetic_covered_index(&mut dataset, "text_idx", "text", "id").await; + + let params = ScalarIndexParams { + index_type: "fm".to_string(), + params: Some(r#"{"num_segments": 2}"#.to_string()), + }; + + // Default naming: must recognize the covered "text_idx" as the same + // field, not silently rename around it to "text_idx_2". + let err = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, ¶ms) + .train(false) + .execute() + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("already exists, please specify a different name or use replace=True"), + "{err}" + ); + + // Explicit naming: must not misfire "different fields" for the same + // reason. + let err2 = CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::Fm, ¶ms) + .name("text_idx".to_string()) + .train(false) + .execute() + .await + .unwrap_err(); + assert!( + err2.to_string() + .contains("already exists, please specify a different name or use replace=True"), + "{err2}" + ); + } + #[tokio::test] async fn test_concurrent_create_index_same_name_returns_retryable_conflict() { let tmpdir = TempStrDir::default(); @@ -1152,6 +1455,50 @@ mod tests { IvfBuildParams::try_with_centroids(4, centroids).unwrap() } + async fn write_vector_fragment_dataset(uri: &str) -> Dataset { + let reader = gen_batch() + .col("id", lance_datagen::array::step::()) + .col( + "vector", + lance_datagen::array::rand_vec::(lance_datagen::Dimension::from(16)), + ) + .into_reader_rows( + lance_datagen::RowCount::from(256), + lance_datagen::BatchCount::from(4), + ); + Dataset::write( + reader, + uri, + Some(WriteParams { + max_rows_per_file: 64, + mode: WriteMode::Overwrite, + ..Default::default() + }), + ) + .await + .unwrap() + } + + #[tokio::test] + async fn test_get_frags_from_ordered_ids_accepts_unsorted_duplicates() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + let first = fragments[0].id() as u32; + let second = fragments[1].id() as u32; + + let resolved = dataset.get_frags_from_ordered_ids(&[second, first, second, u32::MAX]); + + assert_eq!(resolved.len(), 4); + assert_eq!(resolved[0].as_ref().unwrap().id() as u32, second); + assert_eq!(resolved[1].as_ref().unwrap().id() as u32, first); + assert_eq!(resolved[2].as_ref().unwrap().id() as u32, second); + assert!(resolved[3].is_none()); + } + #[tokio::test] async fn test_execute_uncommitted() { // Test the complete workflow that covers the user's specified code pattern: @@ -1553,15 +1900,119 @@ mod tests { } #[tokio::test] - async fn test_range_based_btree_index_create() { - use crate::dataset::scanner::ColumnOrdering; - use futures::TryStreamExt; + async fn test_label_list_distributed_index_uuid_collision_rejected() { + use lance_datagen::{Dimension, array}; + use lance_index::scalar::label_list::BITMAP_LOOKUP_NAME; - let tmpdir = TempStrDir::default(); - let dataset_uri = format!("file://{}", tmpdir.as_str()); + let test_dir = TempStrDir::default(); + let labels = |label| { + array::cycle_vec_var( + array::cycle::(vec![label]), + Dimension::from(1), + Dimension::from(2), + ) + }; + let mut dataset = gen_batch() + .col("labels_a", labels(1)) + .col("labels_b", labels(3)) + .into_dataset( + test_dir.as_str(), + FragmentCount::from(2), + FragmentRowCount::from(4), + ) + .await + .unwrap(); + let params = + ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::LabelList); + let live_index = dataset + .create_index( + &["labels_a"], + IndexType::LabelList, + Some("labels_a_idx".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + let artifact_path = dataset + .indices_dir() + .join(live_index.uuid.to_string()) + .join(BITMAP_LOOKUP_NAME); + let artifact_before = dataset + .object_store + .read_one_all(&artifact_path) + .await + .unwrap(); + let version_before = dataset.version().version; + let fragment_id = dataset.get_fragments()[0].id() as u32; - // Write the dataset with deliberately unsorted ids so the sort step is real. - let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + for index_type in [IndexType::LabelList, IndexType::Scalar] { + let err = dataset + .create_index_builder(&["labels_b"], index_type, ¶ms) + .name("labels_b_idx".to_string()) + .fragments(vec![fragment_id]) + .index_uuid(live_index.uuid) + .execute_uncommitted() + .await + .unwrap_err(); + + assert!( + matches!(err, Error::InvalidInput { .. }), + "expected invalid input error, got: {err}" + ); + assert!( + err.to_string().contains( + "index_uuid is no longer accepted for LabelList distributed index builds" + ), + "unexpected error: {err}" + ); + } + assert_eq!(dataset.version().version, version_before); + let artifact_after = dataset + .object_store + .read_one_all(&artifact_path) + .await + .unwrap(); + assert_eq!(artifact_after, artifact_before); + + let dataset = Dataset::open(test_dir.as_str()).await.unwrap(); + assert_eq!( + dataset + .count_rows(Some("array_has_any(labels_a, [1])".to_string())) + .await + .unwrap(), + 8 + ); + assert_eq!( + dataset + .count_rows(Some("array_has_any(labels_a, [3])".to_string())) + .await + .unwrap(), + 0 + ); + let plan = dataset + .scan() + .filter("array_has_any(labels_a, [1])") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery") && plan.contains("LabelList"), + "expected LabelList scalar index query in plan: {plan}" + ); + } + + #[tokio::test] + async fn test_range_based_btree_index_create() { + use crate::dataset::scanner::ColumnOrdering; + use futures::TryStreamExt; + + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + + // Write the dataset with deliberately unsorted ids so the sort step is real. + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( "id", DataType::Int32, false, @@ -1706,6 +2157,170 @@ mod tests { ); } + #[tokio::test] + async fn test_bloomfilter_distributed_segments_merge_and_query() { + #[derive(serde::Serialize)] + struct BloomParams { + number_of_items: u64, + probability: f64, + } + + let mut dataset = gen_batch() + .col("value", lance_datagen::array::step::()) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(8)) + .await + .unwrap(); + + let params = + ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BloomFilter) + .with_params(&BloomParams { + number_of_items: 4, + probability: 0.000_001, + }); + let fragments = dataset.get_fragments(); + assert_eq!(fragments.len(), 3); + let mut staged = Vec::with_capacity(fragments.len()); + for fragment in &fragments { + staged.push( + CreateIndexBuilder::new(&mut dataset, &["value"], IndexType::BloomFilter, ¶ms) + .name("value_bloom_segments".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + + let mut future_segment = staged[1].clone(); + future_segment.dataset_version = dataset.manifest.version + 1; + let error = dataset + .merge_existing_index_segments(vec![staged[0].clone(), future_segment]) + .await + .unwrap_err(); + assert!( + error.to_string().contains("future dataset version"), + "unexpected merge error: {error}" + ); + + dataset + .commit_existing_index_segments("value_bloom_segments", "value", staged.clone()) + .await + .unwrap(); + let logical = crate::index::scalar_logical::open_named_scalar_index( + &dataset, + "value", + "value_bloom_segments", + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert_eq!( + logical.calculate_included_frags().await.unwrap(), + dataset.fragment_bitmap.as_ref().clone() + ); + + let mut trimmed = staged.clone(); + trimmed[0].fragment_bitmap = Some(RoaringBitmap::new()); + let trimmed_merged = dataset + .merge_existing_index_segments(trimmed) + .await + .unwrap(); + assert_eq!( + trimmed_merged.fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter(fragments[1..].iter().map(|fragment| fragment.id() as u32)) + ); + + let incompatible_params = + ScalarIndexParams::for_builtin(lance_index::scalar::BuiltinIndexType::BloomFilter) + .with_params(&BloomParams { + number_of_items: 8, + probability: 0.01, + }); + let incompatible = CreateIndexBuilder::new( + &mut dataset, + &["value"], + IndexType::BloomFilter, + &incompatible_params, + ) + .name("value_bloom_incompatible".to_string()) + .fragments(vec![fragments[1].id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + let err = dataset + .merge_existing_index_segments(vec![staged[0].clone(), incompatible]) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("different parameters: number_of_items=4"), + "unexpected merge error: {err}" + ); + + let expected_dataset_version = staged + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap(); + let merged = dataset.merge_existing_index_segments(staged).await.unwrap(); + assert_eq!(merged.dataset_version, expected_dataset_version); + assert_eq!( + merged.fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref() + ); + assert!( + merged + .index_details + .as_ref() + .unwrap() + .type_url + .ends_with("BloomFilterIndexDetails") + ); + dataset + .commit_existing_index_segments("value_bloom_merged", "value", vec![merged]) + .await + .unwrap(); + let committed = dataset + .load_indices_by_name("value_bloom_merged") + .await + .unwrap(); + assert_eq!(committed[0].dataset_version, expected_dataset_version); + let merged_logical = crate::index::scalar_logical::open_named_scalar_index( + &dataset, + "value", + "value_bloom_merged", + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let candidates = merged_logical + .search( + &BloomFilterQuery::Equals(ScalarValue::Int32(Some(17))), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert!( + candidates + .row_addrs() + .true_rows() + .row_addrs() + .unwrap() + .map(u64::from) + .any(|row_addr| row_addr == (2_u64 << 32) + 1) + ); + + let result = dataset + .scan() + .filter("value = 17") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(result.num_rows(), 1); + assert_eq!(result["value"].as_primitive::().value(0), 17); + } + #[tokio::test] async fn test_vector_execute_uncommitted_segments_commit_without_staging() { let tmpdir = TempStrDir::default(); @@ -1809,6 +2424,330 @@ mod tests { assert!(result.num_rows() > 0); } + #[tokio::test] + async fn test_vector_explicit_all_fragments_uses_full_build_without_precomputed_ivf() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + assert!(fragment_ids.len() >= 2); + + let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10); + params.version(crate::index::vector::IndexFileVersion::Legacy); + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(fragment_ids) + .execute_uncommitted() + .await + .unwrap(); + + assert_eq!( + segment.fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref() + ); + } + + #[tokio::test] + async fn test_vector_precomputed_ivf_num_partitions_mismatch_errors() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await; + let centroid_count = ivf_params.centroids.as_ref().unwrap().len(); + ivf_params.num_partitions = Some(centroid_count + 1); + let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params); + + let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .execute_uncommitted() + .await + .unwrap_err(); + + assert!( + err.to_string().contains(&format!( + "num_partitions {} does not match precomputed IVF centroids length {}", + centroid_count + 1, + centroid_count + )), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_vector_subset_legacy_ivf_pq_rejects_filtered_build() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + let mut params = VectorIndexParams::ivf_pq(2, 8, 1, MetricType::L2, 10); + params.version(crate::index::vector::IndexFileVersion::Legacy); + + let err = CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![fragments[0].id() as u32]) + .execute_uncommitted() + .await + .unwrap_err(); + + assert!( + err.to_string() + .contains("filtered IVF_PQ builds do not support legacy format"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_vector_subset_fragments_train_without_precomputed_ivf() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 3); + let selected = vec![ + fragments[1].id() as u32, + fragments[0].id() as u32, + fragments[1].id() as u32, + ]; + let expected_bitmap = selected.iter().copied().collect::(); + + let params = VectorIndexParams::ivf_flat(2, DistanceType::L2); + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(selected) + .execute_uncommitted() + .await + .unwrap(); + + assert_eq!(segment.fragment_bitmap.as_ref().unwrap(), &expected_bitmap); + } + + #[tokio::test] + async fn test_vector_merge_rejects_independently_trained_fragment_segments() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + + let params = VectorIndexParams::ivf_flat(2, DistanceType::L2); + let mut segments = Vec::new(); + for fragment in fragments.iter().take(2) { + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + segments.push(segment); + } + + let err = dataset + .merge_existing_index_segments(segments) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("IVF centroids mismatch across shards"), + "unexpected error: {err}" + ); + } + + #[tokio::test] + async fn test_vector_optimize_rejects_independently_trained_fragment_segments() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 2); + + let params = VectorIndexParams::ivf_flat(2, DistanceType::L2); + let mut segments = Vec::new(); + for fragment in fragments.iter().take(2) { + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(); + segments.push(segment); + } + dataset + .commit_existing_index_segments("vector_idx", "vector", segments) + .await + .unwrap(); + + let version_before_optimize = dataset.version().version; + let segments_before_optimize = dataset.load_indices_by_name("vector_idx").await.unwrap(); + let err = dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap_err(); + assert!( + err.to_string() + .contains("vector index segments do not share IVF centroids"), + "unexpected error: {err}" + ); + assert_eq!(dataset.version().version, version_before_optimize); + assert_eq!( + dataset.load_indices_by_name("vector_idx").await.unwrap(), + segments_before_optimize, + "an incompatible merge must not partially commit index metadata" + ); + } + + #[tokio::test] + async fn test_vector_optimize_only_validates_requested_compatible_suffix() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + let fragments = dataset.get_fragments(); + assert!(fragments.len() >= 4); + let tail_start = fragments.len() - 2; + + let mut independent_ivf = IvfBuildParams::new(2); + independent_ivf.max_iters = 7; + let independent_params = + VectorIndexParams::with_ivf_flat_params(DistanceType::L2, independent_ivf); + let independent_segment = CreateIndexBuilder::new( + &mut dataset, + &["vector"], + IndexType::Vector, + &independent_params, + ) + .name("vector_idx".to_string()) + .fragments( + fragments[..tail_start] + .iter() + .map(|fragment| fragment.id() as u32) + .collect(), + ) + .execute_uncommitted() + .await + .unwrap(); + + let mut shared_ivf = prepare_vector_ivf(&dataset, "vector").await; + shared_ivf.max_iters = 13; + let shared_params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, shared_ivf); + let mut compatible_tail = Vec::new(); + for fragment in fragments.iter().skip(tail_start) { + compatible_tail.push( + CreateIndexBuilder::new( + &mut dataset, + &["vector"], + IndexType::Vector, + &shared_params, + ) + .name("vector_idx".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + + let independent_uuid = independent_segment.uuid; + let tail_uuids = compatible_tail + .iter() + .map(|segment| segment.uuid) + .collect::>(); + let expected_merged_fragments = compatible_tail + .iter() + .flat_map(|segment| segment.fragment_bitmap.as_ref().unwrap().iter()) + .collect::(); + let expected_merged_details = compatible_tail.last().unwrap().index_details.clone(); + assert_ne!( + independent_segment.index_details, expected_merged_details, + "test setup must distinguish base and suffix metadata" + ); + let mut segments = vec![independent_segment]; + segments.extend(compatible_tail); + dataset + .commit_existing_index_segments("vector_idx", "vector", segments) + .await + .unwrap(); + + dataset + .optimize_indices(&OptimizeOptions::merge(2)) + .await + .unwrap(); + + let optimized = dataset.load_indices_by_name("vector_idx").await.unwrap(); + assert_eq!(optimized.len(), 2); + assert!( + optimized + .iter() + .any(|segment| segment.uuid == independent_uuid), + "the incompatible base segment must not participate in the requested suffix merge" + ); + assert!( + tail_uuids + .iter() + .all(|uuid| optimized.iter().all(|segment| segment.uuid != *uuid)), + "both compatible suffix segments must be replaced" + ); + let merged = optimized + .iter() + .find(|segment| segment.uuid != independent_uuid) + .unwrap(); + assert_eq!( + merged.fragment_bitmap.as_ref().unwrap(), + &expected_merged_fragments + ); + assert_eq!( + merged.index_details, expected_merged_details, + "merged metadata must come from the selected suffix, not an incompatible base segment" + ); + } + + #[tokio::test] + async fn test_vector_empty_fragments_with_precomputed_ivf_builds_empty_segment() { + let tmpdir = TempStrDir::default(); + let dataset_uri = format!("file://{}", tmpdir.as_str()); + let mut dataset = write_vector_fragment_dataset(&dataset_uri).await; + + let mut ivf_params = prepare_vector_ivf(&dataset, "vector").await; + let expected_partitions = ivf_params.centroids.as_ref().unwrap().len(); + ivf_params.num_partitions = None; + let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params); + let segment = + CreateIndexBuilder::new(&mut dataset, &["vector"], IndexType::Vector, ¶ms) + .name("vector_idx".to_string()) + .fragments(vec![]) + .execute_uncommitted() + .await + .unwrap(); + + assert!(segment.fragment_bitmap.as_ref().unwrap().is_empty()); + dataset + .commit_existing_index_segments("vector_idx", "vector", vec![segment]) + .await + .unwrap(); + let logical_index = dataset + .open_logical_vector_index("vector", "vector_idx") + .await + .unwrap(); + let metadata = logical_index.metadatas().next().unwrap(); + assert_eq!( + logical_index.as_ivf().unwrap().num_partitions_per_segment(), + vec![(metadata.uuid, expected_partitions)] + ); + } + #[tokio::test] async fn test_commit_existing_index_segments_vector_commits_multi_segment_logical_index() { let tmpdir = TempStrDir::default(); @@ -2566,7 +3505,7 @@ mod tests { let mut legacy_segment = segment.clone(); legacy_segment.uuid = legacy_uuid; legacy_segment.index_version = LABEL_LIST_NULLS_MIN_VERSION; - legacy_segment.files = Some(vec![legacy_file]); + legacy_segment.files = Some(index_files_to_table(vec![legacy_file])); let err = dataset .merge_existing_index_segments(vec![legacy_segment]) @@ -2619,6 +3558,7 @@ mod tests { .await .unwrap(); + let source_dataset_version = dataset.manifest.version; dataset .commit_existing_index_segments( "vector_idx", @@ -2626,8 +3566,11 @@ mod tests { vec![IndexSegment::new( uuid, dataset.fragment_bitmap.as_ref().clone(), + [dataset.schema().field("vector").unwrap().id], Arc::new(vector_index_details(¶ms)), IndexType::IvfHnswFlat.version(), + source_dataset_version, + vec![], )], ) .await @@ -2640,6 +3583,28 @@ mod tests { indices[0].fragment_bitmap.as_ref().unwrap(), dataset.fragment_bitmap.as_ref() ); + assert_eq!(indices[0].dataset_version, source_dataset_version); + + let err = dataset + .commit_existing_index_segments( + "future_vector_idx", + "vector", + vec![IndexSegment::new( + Uuid::new_v4(), + dataset.fragment_bitmap.as_ref().clone(), + [dataset.schema().field("vector").unwrap().id], + Arc::new(vector_index_details(¶ms)), + IndexType::IvfHnswFlat.version(), + dataset.manifest.version + 1, + vec![], + )], + ) + .await + .unwrap_err(); + assert!( + err.to_string().contains("future dataset version"), + "unexpected error: {err}" + ); } #[tokio::test] @@ -2690,7 +3655,7 @@ mod tests { } #[tokio::test] - async fn test_create_index_ivf_rq_preserves_index_version_on_segment_commit_path() { + async fn test_create_index_ivf_rq_preserves_index_version() { let tmpdir = TempStrDir::default(); let dataset_uri = format!("file://{}", tmpdir.as_str()); diff --git a/rust/lance/src/index/frag_reuse.rs b/rust/lance/src/index/frag_reuse.rs index 23a8fec5145..e64841012d1 100644 --- a/rust/lance/src/index/frag_reuse.rs +++ b/rust/lance/src/index/frag_reuse.rs @@ -2,20 +2,17 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use crate::Dataset; -use crate::dataset::optimize::remapping::transpose_row_ids_from_digest; use crate::index::DatasetIndexExt; use lance_core::Error; use lance_index::frag_reuse::{ - FRAG_REUSE_DETAILS_FILE_NAME, FRAG_REUSE_INDEX_NAME, FragReuseGroup, FragReuseIndex, + CompactFragReuseIndex, FRAG_REUSE_DETAILS_FILE_NAME, FRAG_REUSE_INDEX_NAME, FragReuseGroup, FragReuseIndexDetails, FragReuseVersion, }; use lance_table::format::IndexMetadata; use lance_table::format::pb::fragment_reuse_index_details::{Content, InlineContent}; use lance_table::format::pb::{ExternalFile, FragmentReuseIndexDetails}; use prost::Message; -use roaring::{RoaringBitmap, RoaringTreemap}; -use std::collections::HashMap; -use std::io::Cursor; +use roaring::RoaringBitmap; use std::sync::Arc; use tokio::io::AsyncWriteExt; use uuid::Uuid; @@ -71,25 +68,8 @@ pub async fn load_frag_reuse_index_details( pub(crate) async fn open_frag_reuse_index( uuid: Uuid, details: &FragReuseIndexDetails, -) -> lance_core::Result { - let mut row_id_maps: Vec>> = - Vec::with_capacity(details.versions.len()); - for version in &details.versions { - let mut row_id_map = HashMap::>::new(); - for group in version.groups.iter() { - let cursor = Cursor::new(&group.changed_row_addrs); - let changed_row_addrs = RoaringTreemap::deserialize_from(cursor).unwrap(); - let group_row_id_map = transpose_row_ids_from_digest( - changed_row_addrs, - &group.old_frags, - &group.new_frags, - ); - row_id_map.extend(group_row_id_map); - } - row_id_maps.push(row_id_map); - } - - Ok(FragReuseIndex::new(uuid, row_id_maps, details.clone())) +) -> lance_core::Result { + CompactFragReuseIndex::try_new(uuid, details.clone()) } pub(crate) async fn build_new_frag_reuse_index( @@ -166,6 +146,7 @@ pub(crate) async fn build_frag_reuse_index_metadata( uuid: index_id, name: FRAG_REUSE_INDEX_NAME.to_string(), fields: vec![], + covering_fields: vec![], dataset_version: dataset.manifest.version, fragment_bitmap: Some(new_fragment_bitmap), index_details: Some(Arc::new(prost_types::Any::from_msg(&proto)?)), diff --git a/rust/lance/src/index/mem_wal.rs b/rust/lance/src/index/mem_wal.rs index de2c70b62d2..f82b0a44e7e 100644 --- a/rust/lance/src/index/mem_wal.rs +++ b/rust/lance/src/index/mem_wal.rs @@ -3,124 +3,25 @@ //! MemWAL Index operations. //! -//! The MemWAL Index stores: -//! - Configuration (sharding_specs, maintained_indexes) -//! - Merge progress (merged_generations per shard) -//! - Shard state snapshots (eventually consistent) -//! -//! Writers no longer update the index on every write. Instead, they update -//! shard manifests directly. This module provides functions to: -//! - Load the MemWAL index -//! - Update merged generations (called during merge-insert commits) - -use std::sync::Arc; - -use lance_core::{Error, Result}; -use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MemWalIndex, MemWalIndexDetails, MergedGeneration}; -use lance_table::format::{IndexMetadata, pb}; -use uuid::Uuid; - -/// Load MemWalIndexDetails from an IndexMetadata. -pub(crate) fn load_mem_wal_index_details(index: IndexMetadata) -> Result { - if let Some(details_any) = index.index_details.as_ref() { - if !details_any.type_url.ends_with("MemWalIndexDetails") { - return Err(Error::index(format!( - "Index details is not for the MemWAL index, but {}", - details_any.type_url - ))); - } - - Ok(MemWalIndexDetails::try_from( - details_any.to_msg::()?, - )?) - } else { - Err(Error::index("Index details not found for the MemWAL index")) - } -} +//! The index data structures and the helpers that read and update the index's +//! `IndexMetadata` entry live in [`lance_table::system_index::mem_wal`]; this +//! module holds the dataset-level operations built on top of them. -/// Open the MemWAL index from its metadata. -pub(crate) fn open_mem_wal_index(index: IndexMetadata) -> Result> { - Ok(Arc::new(MemWalIndex::new(load_mem_wal_index_details( - index, - )?))) -} - -/// Update merged_generations in the MemWAL index. -/// This is called during merge-insert commits to atomically record which -/// generations have been merged to the base table. -pub(crate) fn update_mem_wal_index_merged_generations( - indices: &mut Vec, - dataset_version: u64, - new_merged_generations: Vec, -) -> Result<()> { - if new_merged_generations.is_empty() { - return Ok(()); - } - - let pos = indices - .iter() - .position(|idx| idx.name == MEM_WAL_INDEX_NAME); - - let new_meta = if let Some(pos) = pos { - let current_meta = indices.remove(pos); - let mut details = load_mem_wal_index_details(current_meta)?; - - // Update merged_generations - for each shard, keep the higher generation - for new_mg in new_merged_generations { - if let Some(existing) = details - .merged_generations - .iter_mut() - .find(|mg| mg.shard_id == new_mg.shard_id) - { - if new_mg.generation > existing.generation { - existing.generation = new_mg.generation; - } - } else { - details.merged_generations.push(new_mg); - } - } - - new_mem_wal_index_meta(dataset_version, details)? - } else { - // Create new MemWAL index with just the merged generations - let details = MemWalIndexDetails { - merged_generations: new_merged_generations, - ..Default::default() - }; - new_mem_wal_index_meta(dataset_version, details)? - }; - - indices.push(new_meta); - Ok(()) -} - -/// Create a new MemWAL index metadata entry. -pub(crate) fn new_mem_wal_index_meta( - dataset_version: u64, - details: MemWalIndexDetails, -) -> Result { - Ok(IndexMetadata { - uuid: Uuid::new_v4(), - name: MEM_WAL_INDEX_NAME.to_string(), - fields: vec![], - dataset_version, - fragment_bitmap: None, - index_details: Some(Arc::new(prost_types::Any::from_msg( - &pb::MemWalIndexDetails::from(&details), - )?)), - index_version: 0, - created_at: Some(chrono::Utc::now()), - base_id: None, - // Memory WAL index is inline (no files) - files: None, - }) -} +pub(crate) use lance_table::system_index::mem_wal::{ + load_mem_wal_index_details, new_mem_wal_index_meta, open_mem_wal_index, +}; #[cfg(test)] mod tests { use super::*; + use std::collections::HashMap; + + use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME, MemWalIndexDetails}; + use lance_table::format::IndexMetadata; + use lance_table::system_index::mem_wal::update_mem_wal_index_compacted_sstables; use std::sync::Arc; + use uuid::Uuid; use crate::index::DatasetIndexExt; use arrow_array::{Int32Array, RecordBatch}; @@ -152,18 +53,75 @@ mod tests { .unwrap() } + /// A dataset with `__lance_mem_wal` already installed, as MemWAL + /// initialization leaves it. Recording compaction progress requires the + /// system index to exist, so any test that commits progress needs this + /// rather than a bare [`test_dataset`]. + async fn test_dataset_with_mem_wal() -> crate::Dataset { + let dataset = test_dataset().await; + let mem_wal_index = + new_mem_wal_index_meta(dataset.manifest.version, MemWalIndexDetails::default()) + .unwrap(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![mem_wal_index], + removed_indices: vec![], + }, + None, + ); + CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap() + } + + /// UpdateMemWalState touches indexes, not data, so it must carry the + /// fragment list forward. The operation builds its manifest from scratch, + /// and an unpopulated fragment list is published as an empty one. + #[tokio::test] + async fn test_update_mem_wal_state_preserves_fragments() { + let dataset = test_dataset_with_mem_wal().await; + let rows_before = dataset.count_rows(None).await.unwrap(); + let fragments_before: Vec = dataset.fragments().iter().map(|f| f.id).collect(); + assert!(rows_before > 0, "precondition: the table holds rows"); + + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(Uuid::new_v4(), 1)], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!( + dataset.fragments().iter().map(|f| f.id).collect::>(), + fragments_before, + "UpdateMemWalState dropped fragments" + ); + assert_eq!( + dataset.count_rows(None).await.unwrap(), + rows_before, + "UpdateMemWalState dropped rows" + ); + } + /// Test that UpdateMemWalState with lower generation than committed fails without retry. /// Per spec: If committed_generation >= to_commit_generation, abort without retry. #[tokio::test] async fn test_update_mem_wal_state_conflict_lower_generation_no_retry() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 10 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -177,7 +135,7 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], }, None, ); @@ -193,14 +151,14 @@ mod tests { /// Test that UpdateMemWalState with equal generation as committed fails without retry. #[tokio::test] async fn test_update_mem_wal_state_conflict_equal_generation_no_retry() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 10 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -213,7 +171,7 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -230,14 +188,14 @@ mod tests { /// Per spec: If committed_generation < to_commit_generation, retry is allowed. #[tokio::test] async fn test_update_mem_wal_state_conflict_higher_generation_retryable() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 5 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], }, None, ); @@ -251,7 +209,7 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -267,7 +225,7 @@ mod tests { /// Test that UpdateMemWalState on different shards don't conflict. #[tokio::test] async fn test_update_mem_wal_state_different_shards_no_conflict() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard1 = Uuid::new_v4(); let shard2 = Uuid::new_v4(); @@ -275,7 +233,7 @@ mod tests { let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard1, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard1, 10)], }, None, ); @@ -289,7 +247,7 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard2, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard2, 5)], }, None, ); @@ -312,21 +270,21 @@ mod tests { .unwrap() .clone(); let details = load_mem_wal_index_details(mem_wal_idx).unwrap(); - assert_eq!(details.merged_generations.len(), 2); + assert_eq!(details.compacted_sstables.len(), 2); } /// Test that CreateIndex of MemWalIndex can be rebased against UpdateMemWalState. - /// The merged_generations from UpdateMemWalState should be merged into CreateIndex. + /// The compacted_sstables from UpdateMemWalState should be included in CreateIndex. #[tokio::test] async fn test_create_index_rebase_against_update_mem_wal_state() { - let dataset = test_dataset().await; + let dataset = test_dataset_with_mem_wal().await; let shard = Uuid::new_v4(); // First commit UpdateMemWalState with generation 10 let txn1 = Transaction::new( dataset.manifest.version, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -336,7 +294,7 @@ mod tests { .unwrap(); // CreateIndex of MemWalIndex based on old version (before UpdateMemWalState) - // This should succeed and merge the generations + // This should succeed and combine the compaction progress. let details = MemWalIndexDetails { num_shards: 1, ..Default::default() @@ -359,7 +317,7 @@ mod tests { result ); - // Verify the merged_generations from UpdateMemWalState were merged into CreateIndex + // Verify the compacted_sstables from UpdateMemWalState were included in CreateIndex let dataset = result.unwrap(); let mem_wal_idx = dataset .load_indices() @@ -370,9 +328,9 @@ mod tests { .unwrap() .clone(); let details = load_mem_wal_index_details(mem_wal_idx).unwrap(); - assert_eq!(details.merged_generations.len(), 1); - assert_eq!(details.merged_generations[0].shard_id, shard); - assert_eq!(details.merged_generations[0].generation, 10); + assert_eq!(details.compacted_sstables.len(), 1); + assert_eq!(details.compacted_sstables[0].shard_id, shard); + assert_eq!(details.compacted_sstables[0].generation, 10); assert_eq!(details.num_shards, 1); // Config from CreateIndex preserved } @@ -382,9 +340,9 @@ mod tests { let dataset = test_dataset().await; let shard = Uuid::new_v4(); - // First commit CreateIndex of MemWalIndex with merged_generations + // First commit CreateIndex of MemWalIndex with compacted_sstables let details = MemWalIndexDetails { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], ..Default::default() }; let mem_wal_index = new_mem_wal_index_meta(dataset.manifest.version, details).unwrap(); @@ -406,7 +364,7 @@ mod tests { let txn2 = Transaction::new( dataset.manifest.version - 1, // Based on old version Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], }, None, ); @@ -419,75 +377,664 @@ mod tests { ); } + /// A table with `generation` folded into base. + async fn compacted_dataset(shard: Uuid, generation: u64) -> crate::Dataset { + let dataset = test_dataset_with_mem_wal().await; + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, generation)], + }, + None, + ); + CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap() + } + + /// An index segment spanning `fragments`, as a completed build leaves. + fn index_over(name: &str, fragments: &[u32]) -> IndexMetadata { + IndexMetadata { + uuid: Uuid::new_v4(), + name: name.to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap: Some(roaring::RoaringBitmap::from_iter(fragments.iter().copied())), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + async fn catch_up_generation(dataset: &crate::Dataset, index: &str) -> Option { + let meta = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .unwrap() + .clone(); + load_mem_wal_index_details(meta) + .unwrap() + .index_catchup + .into_iter() + .find(|entry| entry.index_name == index) + .and_then(|entry| entry.caught_up_generations.first().map(|g| g.generation)) + } + + /// The commit path, not the derivation in isolation: `commit_transaction` + /// has to load the read version's indices and hand them down. + #[tokio::test] + async fn an_index_covering_the_table_earns_catch_up_on_commit() { + let shard = Uuid::new_v4(); + let dataset = compacted_dataset(shard, 5).await; + + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(5)); + } + + /// A repair with nothing to rebuild still has to commit. + /// + /// Coverage is derived at commit time, so an index that already spans the + /// table records its position only if there is a commit to record it on. + /// That is the ordinary case after a remap, or after a compaction that + /// advanced a generation without changing which fragments exist: the + /// optimize finds no unindexed fragments and has no new segment to publish. + /// Skipping the commit there leaves the position missing forever, the + /// scheduler repeating the same repair, and the last SSTable unretirable. + #[tokio::test] + async fn a_no_work_repair_still_records_derived_catch_up() { + use lance_index::optimize::OptimizeOptions; + + let shard = Uuid::new_v4(); + let dataset = compacted_dataset(shard, 7).await; + // Already spans the table, so the optimize below has nothing to build. + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + // Compaction advances without adding a fragment, so the index still + // covers and the optimize stays a no-op. + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 9)], + }, + None, + ); + let mut dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + dataset + .optimize_indices(&OptimizeOptions::append().index_names(vec!["idx".to_string()])) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(9)); + + // And once it is current, the next pass must not commit again: + // periodic maintenance would otherwise mint a version forever. + let after_repair = dataset.manifest.version; + dataset + .optimize_indices(&OptimizeOptions::append().index_names(vec!["idx".to_string()])) + .await + .unwrap(); + assert_eq!(dataset.manifest.version, after_repair); + } + + /// A no-op optimize on a table that is not on the protocol must stay a + /// no-op: the early return is what keeps ordinary tables from committing an + /// empty version on every maintenance pass. + #[tokio::test] + async fn a_no_work_optimize_with_nothing_compacted_commits_nothing() { + use lance_index::optimize::OptimizeOptions; + + let dataset = test_dataset_with_mem_wal().await; + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let mut dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + let before = dataset.manifest.version; + + dataset + .optimize_indices(&OptimizeOptions::append().index_names(vec!["idx".to_string()])) + .await + .unwrap(); + + assert_eq!(dataset.manifest.version, before); + } + + /// A table carrying compaction progress but no catch-up entry earns one + /// from an ordinary commit. This is how a table written before catch-up was + /// maintained heals itself: nothing has to be run against it. + #[tokio::test] + async fn a_table_with_no_catchup_entry_earns_one() { + let dataset = test_dataset_with_mem_wal().await; + let shard = Uuid::new_v4(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + let txn = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(5)); + } + + /// What a commit earns is fixed by the version it read, and a rebase does + /// not move it. The builder inspected a one-fragment table with generation + /// 5 folded in; by the time it commits, an append has landed. It still + /// earns 5 -- judged against the table it never saw, it would earn nothing + /// and the SSTables would be retained forever. + #[tokio::test] + async fn credit_is_anchored_to_the_read_version_across_a_rebase() { + let shard = Uuid::new_v4(); + let dataset = compacted_dataset(shard, 5).await; + let read_version = dataset.manifest.version; + + let data = RecordBatch::try_new( + Arc::new(Schema::from(dataset.schema())), + vec![ + Arc::new(Int32Array::from_iter_values(10..20_i32)), + Arc::new(Int32Array::from_iter_values(std::iter::repeat_n(0, 10))), + ], + ) + .unwrap(); + let dataset = InsertBuilder::new(Arc::new(dataset)) + .with_params(&WriteParams { + mode: crate::dataset::WriteMode::Append, + max_rows_per_file: 10, + ..Default::default() + }) + .execute(vec![data]) + .await + .unwrap(); + assert!( + dataset.get_fragments().len() > 1, + "the append must add a fragment the index has not seen" + ); + + // Built against `read_version`, and only ever saw fragment 0. + let txn = Transaction::new( + read_version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + assert_eq!(catch_up_generation(&dataset, "idx").await, Some(5)); + } + + /// A user index build that races a compaction commit is rejected outright + /// rather than rebased -- only the system index may rebase against + /// `UpdateMemWalState`. Anything scheduling catch-up work has to expect the + /// build to be thrown away and retried, so a busy shard needs the two kept + /// apart rather than merely retried. + #[tokio::test] + async fn a_user_index_build_cannot_rebase_past_a_compaction_commit() { + let shard = Uuid::new_v4(); + let dataset = compacted_dataset(shard, 5).await; + let read_version = dataset.manifest.version; + + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 9)], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + let txn = Transaction::new( + read_version, + Operation::CreateIndex { + new_indices: vec![index_over("idx", &[0])], + removed_indices: vec![], + }, + None, + ); + let err = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap_err(); + + assert!( + err.to_string().contains("incompatible"), + "expected an incompatible-transaction error, got {err}" + ); + } + + /// One `__lance_mem_wal` entry carrying `details`, as a real table has. + fn indices_with(details: MemWalIndexDetails) -> Vec { + vec![new_mem_wal_index_meta(1, details).unwrap()] + } + + fn compacted_generation(indices: &[IndexMetadata], shard: Uuid) -> Option { + load_mem_wal_index_details(indices[0].clone()) + .unwrap() + .compacted_sstables + .iter() + .find(|sstable| sstable.shard_id == shard) + .map(|sstable| sstable.generation) + } + #[test] - fn test_update_merged_generations() { - let mut indices = Vec::new(); + fn test_update_compacted_sstables() { let shard1 = Uuid::new_v4(); let shard2 = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails::default()); - // First update - creates new index - update_mem_wal_index_merged_generations( + update_mem_wal_index_compacted_sstables( &mut indices, 1, - vec![MergedGeneration::new(shard1, 5)], + vec![CompactedSsTable::new(shard1, 5)], ) .unwrap(); - assert_eq!(indices.len(), 1); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - assert_eq!(details.merged_generations.len(), 1); - assert_eq!(details.merged_generations[0].shard_id, shard1); - assert_eq!(details.merged_generations[0].generation, 5); + assert_eq!(compacted_generation(&indices, shard1), Some(5)); - // Second update - updates existing shard - update_mem_wal_index_merged_generations( + // Advancing an existing shard. + update_mem_wal_index_compacted_sstables( &mut indices, 2, - vec![MergedGeneration::new(shard1, 10)], + vec![CompactedSsTable::new(shard1, 10)], ) .unwrap(); + assert_eq!(compacted_generation(&indices, shard1), Some(10)); - assert_eq!(indices.len(), 1); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - assert_eq!(details.merged_generations.len(), 1); - assert_eq!(details.merged_generations[0].generation, 10); - - // Third update - adds new shard - update_mem_wal_index_merged_generations( + // A second shard is independent. + update_mem_wal_index_compacted_sstables( &mut indices, 3, - vec![MergedGeneration::new(shard2, 3)], + vec![CompactedSsTable::new(shard2, 3)], ) .unwrap(); + assert_eq!(compacted_generation(&indices, shard1), Some(10)); + assert_eq!(compacted_generation(&indices, shard2), Some(3)); + } - assert_eq!(indices.len(), 1); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - assert_eq!(details.merged_generations.len(), 2); + /// A stale proposal must fail the whole transaction: accepting it while + /// keeping generation 10 would publish the stale worker's rows under a + /// marker they did not produce. + #[test] + fn a_lower_generation_rejects_the_whole_update() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails { + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + ..Default::default() + }); + let before = indices.clone(); - // Fourth update - lower generation should not update - update_mem_wal_index_merged_generations( + let err = update_mem_wal_index_compacted_sstables( &mut indices, - 4, - vec![MergedGeneration::new(shard1, 8)], // lower than 10 + 2, + vec![CompactedSsTable::new(shard, 8)], + ) + .unwrap_err(); + + assert!( + err.to_string().contains("Stale SSTable compaction"), + "{err}" + ); + assert_eq!( + indices[0].uuid, before[0].uuid, + "a rejected update must leave the index list untouched" + ); + assert_eq!(compacted_generation(&indices, shard), Some(10)); + } + + /// Equal is also refused: it reports nothing new, and accepting it would let + /// a retry publish a second set of row mutations under the same marker. + #[test] + fn an_equal_generation_rejects() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails { + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + ..Default::default() + }); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 2, + vec![CompactedSsTable::new(shard, 10)], + ) + .unwrap_err(); + + assert!( + err.to_string().contains("Stale SSTable compaction"), + "{err}" + ); + } + + #[test] + fn duplicate_shards_in_one_update_reject() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails::default()); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 1, + vec![ + CompactedSsTable::new(shard, 5), + CompactedSsTable::new(shard, 6), + ], + ) + .unwrap_err(); + + assert!(err.to_string().contains("Duplicate shard"), "{err}"); + } + + /// Absent metadata must not be materialized: default details describe a + /// table with no MemWAL shards, so the recorded generation would name a + /// shard nothing can corroborate. + #[test] + fn a_missing_mem_wal_index_rejects() { + let mut indices: Vec = Vec::new(); + + let err = update_mem_wal_index_compacted_sstables( + &mut indices, + 1, + vec![CompactedSsTable::new(Uuid::new_v4(), 5)], + ) + .unwrap_err(); + + assert!(err.to_string().contains("does not exist"), "{err}"); + assert!(indices.is_empty(), "nothing should have been created"); + } + + /// Recording progress replaces the entry where it sits, so the index list + /// keeps its order. + #[test] + fn recording_progress_keeps_the_system_index_position() { + let shard = Uuid::new_v4(); + let mut indices = indices_with(MemWalIndexDetails::default()); + // A neighbour to show the entry is replaced in place, not moved. + indices.push(IndexMetadata { + name: "other_index".to_string(), + ..indices[0].clone() + }); + update_mem_wal_index_compacted_sstables( + &mut indices, + 2, + vec![CompactedSsTable::new(shard, 5)], ) .unwrap(); - let details = load_mem_wal_index_details(indices[0].clone()).unwrap(); - let r1_mg = details - .merged_generations - .iter() - .find(|mg| mg.shard_id == shard1) + assert_eq!(indices[0].name, MEM_WAL_INDEX_NAME); + assert_eq!(indices[1].name, "other_index"); + } + + /// Recording progress must not disturb any other saved MemWAL field. + #[test] + fn recording_progress_preserves_unrelated_mem_wal_state() { + let shard = Uuid::new_v4(); + let other_shard = Uuid::new_v4(); + let mut writer_config_defaults = HashMap::new(); + writer_config_defaults.insert("target_size".to_string(), "64MB".to_string()); + let details = MemWalIndexDetails { + snapshot_ts_millis: 12_345, + num_shards: 4, + maintained_indexes: vec!["vector_idx".to_string()], + compacted_sstables: vec![CompactedSsTable::new(other_shard, 7)], + writer_config_defaults: writer_config_defaults.clone(), + ..Default::default() + }; + let mut indices = indices_with(details); + + update_mem_wal_index_compacted_sstables( + &mut indices, + 2, + vec![CompactedSsTable::new(shard, 5)], + ) + .unwrap(); + + let after = load_mem_wal_index_details(indices[0].clone()).unwrap(); + assert_eq!(after.snapshot_ts_millis, 12_345); + assert_eq!(after.num_shards, 4); + assert_eq!(after.maintained_indexes, vec!["vector_idx".to_string()]); + assert_eq!(after.writer_config_defaults, writer_config_defaults); + // The untouched shard keeps its generation. + assert_eq!(compacted_generation(&indices, other_shard), Some(7)); + } + + /// The hole this guards is a worker that refreshed to HEAD before + /// submitting: there is no intervening transaction, so the conflict + /// resolver sees nothing to reject and only apply-time validation stands + /// between a stale generation and a published commit. + /// + /// The row mutation is the point: were the stale generation accepted, this + /// commit would add a fragment while the recorded generation stayed at 10, + /// so the rows and the number describing them would disagree. + #[tokio::test] + async fn a_stale_generation_against_latest_head_publishes_nothing() { + use lance_table::format::Fragment; + + let shard = Uuid::new_v4(); + let dataset = test_dataset_with_mem_wal().await; + let version = dataset.manifest.version; + + // Record generation 10. + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(Transaction::new( + version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], + }, + None, + )) + .await .unwrap(); - assert_eq!(r1_mg.generation, 10); // Should still be 10 + + let version_before = dataset.manifest.version; + let fragments_before = dataset.get_fragments().len(); + + // Built against the LATEST version, so there is nothing stale for the + // conflict resolver to catch, and carrying a real row mutation. + let stale = Transaction::new( + version_before, + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![], + new_fragments: vec![Fragment::new(999)], + fields_modified: vec![], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ); + let mut dataset = dataset; + let result = CommitBuilder::new(Arc::new(dataset.clone())) + .execute(stale) + .await; + + // Asserting the reason, not just the failure: the commit must be + // refused by apply-time generation validation, not by something + // incidental about the fragment. + let err = result.expect_err("a stale generation must fail the whole commit"); + assert!( + err.to_string().contains("Stale SSTable compaction"), + "expected stale-generation rejection, got {err}" + ); + + // Neither the marker nor the fragment may have been published. + dataset.checkout_latest().await.unwrap(); + let latest = dataset; + assert_eq!( + latest.manifest.version, version_before, + "no new table version may be published" + ); + assert_eq!( + latest.get_fragments().len(), + fragments_before, + "the row mutation must not be published" + ); + let details = load_mem_wal_index_details( + latest + .load_indices() + .await + .unwrap() + .iter() + .find(|idx| idx.name == MEM_WAL_INDEX_NAME) + .unwrap() + .clone(), + ) + .unwrap(); + assert_eq!( + details.compacted_sstables[0].generation, 10, + "the recorded generation must be unchanged" + ); } + /// The system index holds the catch-up positions the WAL pod retires SSTables against. + /// Erasing it through the ordinary index API would leave the table claiming + /// nothing was ever compacted while the SSTables are already gone. + #[test] - fn test_empty_merged_generations_noop() { + fn test_empty_compacted_sstables_noop() { let mut indices = Vec::new(); - // Empty update should be a no-op - update_mem_wal_index_merged_generations(&mut indices, 1, vec![]).unwrap(); + // Empty update should be a no-op, even with no MemWAL index present. + update_mem_wal_index_compacted_sstables(&mut indices, 1, vec![]).unwrap(); assert!(indices.is_empty()); } + + /// Regression: a committed `__mem_wal` (legitimately `fragment_bitmap: + /// None`) must not break `describe_indices` — the path behind lancedb's + /// `list_indices`/`wait_for_index`. It's described as zero indexed rows, + /// like `__frag_reuse`. + #[tokio::test] + async fn test_describe_indices_includes_mem_wal_system_index() { + use crate::index::DatasetIndexExt; + use lance_index::IndexType; + use lance_index::scalar::ScalarIndexParams; + + let mut dataset = test_dataset_with_mem_wal().await; + + // A real user index that describe_indices must keep returning. + dataset + .create_index( + &["a"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // Commit a __mem_wal index, as WAL provisioning does in production. + let shard = Uuid::new_v4(); + let txn = Transaction::new( + dataset.manifest.version, + Operation::UpdateMemWalState { + compacted_sstables: vec![CompactedSsTable::new(shard, 1)], + }, + None, + ); + let dataset = CommitBuilder::new(Arc::new(dataset)) + .execute(txn) + .await + .unwrap(); + + // The system index is present with no fragment_bitmap (by design). + let mem_wal = dataset + .load_indices() + .await + .unwrap() + .iter() + .find(|i| i.name == MEM_WAL_INDEX_NAME) + .unwrap() + .clone(); + assert!(mem_wal.fragment_bitmap.is_none()); + + // describe_indices describes the bitmap-less __mem_wal alongside the + // real index instead of erroring. + let descriptions = dataset.describe_indices(None).await.unwrap(); + let mem_wal_desc = descriptions + .iter() + .find(|d| d.name() == MEM_WAL_INDEX_NAME) + .expect("__mem_wal must be described, not skipped"); + assert_eq!( + mem_wal_desc.index_type(), + "MemWal", + "system index type must resolve via infer_system_index_type" + ); + assert_eq!( + mem_wal_desc.rows_indexed(), + 0, + "a bitmap-less system index indexes zero rows" + ); + assert_eq!( + descriptions.len(), + 2, + "both the real scalar index and __mem_wal must be listed" + ); + } } diff --git a/rust/lance/src/index/prefilter.rs b/rust/lance/src/index/prefilter.rs index 071f1b8893d..78dcbd3ad43 100644 --- a/rust/lance/src/index/prefilter.rs +++ b/rust/lance/src/index/prefilter.rs @@ -48,10 +48,14 @@ pub struct DatasetPreFilter { // and allow list at the same time we start searching the query. We will await // these tasks only when we've done as much work as we can without them. pub(super) deleted_ids: Option>>>, - pub(super) filtered_ids: Option>>, + pub(super) filtered_ids: Option>>>, // Fragment IDs whose data is still in the index but has been removed from the dataset. // Used by FTS merge-on-read to prune stale fragments at search time. pub(super) deleted_fragments: Option, + // Row addresses whose index entries are stale due to a newer data overlay committed after + // the index was built. Computed synchronously at plan time and ANDead into the final mask + // so the index never returns those rows. + pub(super) overlay_block: Option, // When the tasks are finished this is the combined filter pub(super) final_mask: Mutex>>, } @@ -61,11 +65,26 @@ impl DatasetPreFilter { dataset: Arc, indices: &[IndexMetadata], filter: Option>, + ) -> Self { + let filter = filter.map(|filter| { + async move { filter.load().await.map(Arc::new) } + .in_current_span() + .boxed() + }); + Self::new_with_filter_future(dataset, indices, filter) + } + + pub(crate) fn new_with_filter_future( + dataset: Arc, + indices: &[IndexMetadata], + filter: Option>>>, ) -> Self { let mut fragments = RoaringBitmap::new(); let all_have_bitmaps = indices.iter().all(|idx| idx.fragment_bitmap.is_some()); if !all_have_bitmaps { - fragments.insert_range(0..dataset.manifest.max_fragment_id.unwrap_or(0)); + if let Some(max_fragment_id) = dataset.manifest.max_fragment_id() { + fragments.insert_range(0..=max_fragment_id as u32); + } } else { indices.iter().for_each(|idx| { fragments |= idx.fragment_bitmap.as_ref().unwrap(); @@ -77,12 +96,12 @@ impl DatasetPreFilter { Self::create_deletion_mask(dataset, fragments) } .map(SharedPrerequisite::spawn); - let filtered_ids = filter - .map(|filtered_ids| SharedPrerequisite::spawn(filtered_ids.load().in_current_span())); + let filtered_ids = filter.map(SharedPrerequisite::spawn); Self { deleted_ids, filtered_ids, deleted_fragments: None, + overlay_block: None, final_mask: Mutex::new(OnceCell::new()), } } @@ -226,6 +245,13 @@ impl DatasetPreFilter { self.deleted_fragments = Some(fragments); } + /// Block specific row addresses from index results because their index entries are stale + /// due to a data overlay committed after the index was built. + pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self { + self.overlay_block = Some(block); + self + } + /// Creates a task to load a mask that filters out deleted rows and, /// when `restrict_to_fragments` is true, also restricts results to only /// the given `fragments`. @@ -371,7 +397,7 @@ impl PreFilter for DatasetPreFilter { final_mask.get_or_init(|| { let mut combined = RowAddrMask::default(); if let Some(filtered_ids) = &self.filtered_ids { - combined = combined & filtered_ids.get_ready(); + combined = combined & filtered_ids.get_ready().as_ref().clone(); } if let Some(deleted_ids) = &self.deleted_ids { combined = combined & (*deleted_ids.get_ready()).clone(); @@ -383,6 +409,9 @@ impl PreFilter for DatasetPreFilter { } combined = combined & RowAddrMask::from_block(block_list); } + if let Some(overlay_block) = &self.overlay_block { + combined = combined & overlay_block.clone(); + } Arc::new(combined) }); @@ -393,6 +422,7 @@ impl PreFilter for DatasetPreFilter { self.deleted_ids.is_none() && self.filtered_ids.is_none() && self.deleted_fragments.is_none() + && self.overlay_block.is_none() } /// Get the row id mask for this prefilter @@ -421,6 +451,7 @@ impl PreFilter for DatasetPreFilter { mod test { use lance_select::RowSetOps; use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; + use rstest::rstest; use crate::dataset::WriteParams; @@ -529,6 +560,38 @@ mod test { assert_eq!(mask.block_list(), Some(&expected)); } + #[rstest] + #[case::stored_high_water_mark(false)] + #[case::computed_high_water_mark(true)] + #[tokio::test] + async fn test_legacy_index_mask_includes_max_fragment(#[case] unset_max_fragment_id: bool) { + let datasets = test_datasets(false).await; + let mut dataset = (*datasets.deletions_missing_frags).clone(); + if unset_max_fragment_id { + Arc::make_mut(&mut dataset.manifest).max_fragment_id = None; + } + let index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: Vec::new(), + covering_fields: vec![], + name: "legacy".to_string(), + dataset_version: dataset.manifest.version, + fragment_bitmap: None, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let prefilter = DatasetPreFilter::new(Arc::new(dataset), &[index], None); + + prefilter.wait_for_ready().await.unwrap(); + + let mut expected = RowAddrTreeMap::from_iter(vec![(2 << 32) + 2]); + expected.insert_fragment(1); + assert_eq!(prefilter.mask().block_list(), Some(&expected)); + } + #[tokio::test] async fn test_deletion_mask_stable_row_id() { // Here, behavior is different. diff --git a/rust/lance/src/index/scalar.rs b/rust/lance/src/index/scalar.rs index b2ee8e0426a..731b8dd14e6 100644 --- a/rust/lance/src/index/scalar.rs +++ b/rust/lance/src/index/scalar.rs @@ -5,13 +5,17 @@ //! pub(crate) mod bitmap; +pub(crate) mod bloomfilter; pub(crate) mod btree; pub(crate) mod fmindex; pub(crate) mod inverted; pub(crate) mod label_list; +pub(crate) mod ngram; +#[cfg(feature = "geo")] +pub(crate) mod rtree; pub(crate) mod zonemap; -pub use inverted::{load_segment_details, load_segments}; +pub use inverted::{load_segment_details, load_segment_params, load_segments}; pub use crate::index::scalar_logical::{LogicalScalarIndex, load_named_scalar_segments}; @@ -35,7 +39,9 @@ use lance_core::datatypes::Field; use lance_core::utils::tracing::{IO_TYPE_OPEN_SCALAR, TRACE_IO_EVENTS}; use lance_core::{Error, ROW_ADDR, ROW_ID, Result}; use lance_datafusion::exec::LanceExecutionOptions; +use lance_index::frag_reuse::CompactFragReuseIndexHandle; use lance_index::metrics::{MetricsCollector, NoOpMetricsCollector}; +use lance_index::pb::VectorIndexDetails; use lance_index::pbold::{ BTreeIndexDetails, BitmapIndexDetails, InvertedIndexDetails, LabelListIndexDetails, }; @@ -47,7 +53,8 @@ use lance_index::scalar::label_list::{ LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION, }; use lance_index::scalar::registry::{ - ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, VALUE_COLUMN_NAME, + ScalarIndexCacheKey, ScalarIndexLoad, ScalarIndexPlugin, TrainingCriteria, TrainingOrdering, + VALUE_COLUMN_NAME, }; use lance_index::scalar::{BuiltinIndexType, CreatedIndex, InvertedIndexParams}; use lance_index::scalar::{ @@ -57,6 +64,7 @@ use lance_index::scalar::{ use lance_index::{IndexCriteria, IndexType}; use lance_table::format::{Fragment, IndexMetadata}; use log::info; +use prost::{Message, Name}; use tracing::instrument; // Log an update every TRAINING_UPDATE_FREQ million rows processed @@ -122,6 +130,12 @@ impl TrainingRequest { } } +#[cfg(test)] +tokio::task_local! { + /// Overrides the scalar training scan's I/O budget without mutating process-wide state. + pub(crate) static TEST_TRAINING_IO_BUFFER_SIZE: u64; +} + pub(crate) async fn scan_training_data( dataset: &Dataset, column: &str, @@ -131,6 +145,10 @@ pub(crate) async fn scan_training_data( let num_rows = dataset.count_all_rows().await?; let mut scan = dataset.scan(); + #[cfg(test)] + if let Ok(io_buffer_size) = TEST_TRAINING_IO_BUFFER_SIZE.try_with(|size| *size) { + scan.io_buffer_size(io_buffer_size); + } // Fragment filtering is now handled in load_training_data function // This function just processes the fragments passed to it @@ -235,6 +253,35 @@ pub(crate) async fn load_training_data( } } +pub(crate) async fn load_fts_training_data( + dataset: &Dataset, + resolved: &inverted::ResolvedFtsField, + criteria: &TrainingCriteria, + fragments: Option>, + train: bool, + fragment_ids: Option>, +) -> Result { + let scan_column = if resolved.has_lists() { + resolved.root_column.as_str() + } else { + resolved.canonical_path.as_str() + }; + let stream = load_training_data( + dataset, + scan_column, + criteria, + fragments, + train, + fragment_ids, + ) + .await?; + if resolved.has_lists() { + inverted::transform_fts_document_stream(stream, resolved.clone()) + } else { + Ok(stream) + } +} + // TODO: Allow users to register their own plugins static SCALAR_INDEX_PLUGIN_REGISTRY: LazyLock> = LazyLock::new(IndexPluginRegistry::with_default_plugins); @@ -258,6 +305,23 @@ impl IndexDetails { SCALAR_INDEX_PLUGIN_REGISTRY.get_plugin_by_details(self.0.as_ref()) } + /// Returns whether this build has a reader for the complete declared type. + pub(crate) fn has_reader(&self) -> bool { + let Some((_, details_type_name)) = self.0.type_url.rsplit_once('/') else { + return false; + }; + if details_type_name.is_empty() || details_type_name.starts_with('.') { + return false; + } + + details_type_name.eq_ignore_ascii_case(&VectorIndexDetails::full_name()) + // MemWAL flush briefly wrote this pre-`pb` package name. Keep that + // exact historical native identity readable without accepting any + // other message that merely shares the VectorIndexDetails suffix. + || details_type_name.eq_ignore_ascii_case("lance.index.VectorIndexDetails") + || SCALAR_INDEX_PLUGIN_REGISTRY.supports_details(self.0.as_ref()) + } + /// Returns the index version pub fn index_version(&self) -> Result { if self.is_vector() { @@ -286,13 +350,40 @@ pub(super) async fn build_scalar_index( preprocessed_data: Option, progress: Arc, ) -> Result { - let field = dataset - .schema() - .field(column) - .ok_or(Error::invalid_input_source( - format!("No column with name {}", column).into(), - ))?; - let field: arrow_schema::Field = field.into(); + let inverted_params = (params.index_type.eq_ignore_ascii_case("inverted") + || params.index_type.eq_ignore_ascii_case("fts")) + .then(|| serde_json::from_str::(params.params.as_deref().unwrap_or("{}"))) + .transpose()?; + let resolved_fts_field = inverted_params + .as_ref() + .map(|params| { + inverted::resolve_fts_field(dataset.schema(), column, params.get_document_granularity()) + }) + .transpose()?; + let field = if let Some(resolved) = &resolved_fts_field { + let source = dataset + .schema() + .field_by_id(resolved.final_field_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "FTS field id {} is missing from the dataset schema", + resolved.final_field_id + )) + })?; + if resolved.has_lists() { + arrow_schema::Field::new(VALUE_COLUMN_NAME, DataType::Utf8, true) + } else { + source.into() + } + } else { + let field = dataset + .schema() + .field(column) + .ok_or(Error::invalid_input_source( + format!("No column with name {}", column).into(), + ))?; + field.into() + }; let index_store = LanceIndexStore::from_dataset_for_new(dataset, &uuid)?; @@ -306,9 +397,20 @@ pub(super) async fn build_scalar_index( trainer.new_training_request(params.params.as_deref().unwrap_or("{}"), &field)?; progress.stage_start("load_data", None, "rows").await?; - let training_data = match preprocessed_data { - Some(preprocessed_data) => preprocessed_data, - None => { + let training_data = match (preprocessed_data, resolved_fts_field.as_ref()) { + (Some(preprocessed_data), _) => preprocessed_data, + (None, Some(resolved)) => { + load_fts_training_data( + dataset, + resolved, + training_request.criteria(), + None, + train, + fragment_ids.clone(), + ) + .await? + } + (None, None) => { load_training_data( dataset, column, @@ -401,6 +503,23 @@ pub async fn fetch_index_details( None => infer_scalar_index_details(dataset, column, index).await?, }; + if index_details.type_url.ends_with("InvertedIndexDetails") { + let details = + InvertedIndexDetails::decode(index_details.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {err}" + )) + })?; + let details = inverted::normalize_inverted_details(index, details)?; + return Ok(Arc::new(prost_types::Any::from_msg(&details).map_err( + |err| { + Error::io(format!( + "failed to encode InvertedIndexDetails payload: {err}" + )) + }, + )?)); + } + Ok(index_details) } @@ -459,8 +578,8 @@ pub async fn open_scalar_index( .index_cache .for_index(&index.uuid, frag_reuse_index.as_ref().map(|f| &f.uuid)); - let frag_reuse_index: Option> = - frag_reuse_index.map(|f| f as Arc); + let frag_reuse_index: Option> = frag_reuse_index + .map(|f| Arc::new(CompactFragReuseIndexHandle(f)) as Arc); // Runs only on a cold miss, and at most once even under concurrent opens // (the plugin coalesces). The compat check lives here because a warm hit was @@ -490,6 +609,17 @@ pub async fn open_scalar_index( .await } +pub(crate) async fn cached_scalar_index_container( + dataset: &Dataset, + uuid: &Uuid, +) -> Option> { + let frag_reuse_uuid = dataset.frag_reuse_index_uuid().await; + let index_cache = dataset + .index_cache + .for_index(uuid, frag_reuse_uuid.as_ref()); + index_cache.get_unsized_with_key(&ScalarIndexCacheKey).await +} + pub(crate) async fn infer_scalar_index_details( dataset: &Dataset, column: &str, @@ -515,11 +645,7 @@ pub(crate) async fn infer_scalar_index_details( let inverted_list_lookup = index_dir.clone().join(METADATA_FILE); let legacy_inverted_list_lookup = index_dir.clone().join(INVERT_LIST_FILE); let object_store = dataset.object_store_for_index(index).await?; - let index_details = if let DataType::List(_) = col.data_type() { - prost_types::Any::from_msg(&LabelListIndexDetails::default()).unwrap() - } else if object_store.exists(&bitmap_page_lookup).await? { - prost_types::Any::from_msg(&BitmapIndexDetails::default()).unwrap() - } else if object_store.exists(&inverted_list_lookup).await? { + let index_details = if object_store.exists(&inverted_list_lookup).await? { // Try to infer inverted index details from metadata file to capture with_position and other params // Fall back to defaults if anything goes wrong let default_details = prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap(); @@ -536,6 +662,10 @@ pub(crate) async fn infer_scalar_index_details( parse_params().await.unwrap_or(default_details) } else if object_store.exists(&legacy_inverted_list_lookup).await? { prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap() + } else if let DataType::List(_) | DataType::LargeList(_) = col.data_type() { + prost_types::Any::from_msg(&LabelListIndexDetails::default()).unwrap() + } else if object_store.exists(&bitmap_page_lookup).await? { + prost_types::Any::from_msg(&BitmapIndexDetails::default()).unwrap() } else { prost_types::Any::from_msg(&BTreeIndexDetails::default()).unwrap() }; @@ -564,24 +694,60 @@ pub fn index_matches_criteria( } if let Some(for_column) = criteria.for_column { - if index.fields.len() != 1 { + // A covered index lists its carried columns in `fields` too. Only the + // keyed prefix decides which column this index answers for, and there + // must be exactly one of it. + if index.keyed_field().is_none() { return Ok(false); } if fields.len() != 1 { - // This should be unreachable since we just verified index.fields.len() == 1 but - // return false just in case + // Callers must resolve `fields` from the keyed prefix alone. A caller + // that passes all of `index.fields` -- carried columns included -- + // lands here for every covered index and silently gets "no match" + // rather than an error, which is how `describe_indices` came to omit + // them. return Ok(false); } - let field = fields[0]; - // Build the full field path for nested fields - let field_path = if let Some(ancestors) = schema.field_ancestry_by_id(field.id) { - let field_refs: Vec<&str> = ancestors.iter().map(|f| f.name.as_str()).collect(); - lance_core::datatypes::format_field_path(&field_refs) + let is_fts_index = index + .index_details + .as_ref() + .is_some_and(|details| details.type_url.ends_with("InvertedIndexDetails")); + if criteria.must_support_fts && is_fts_index { + let requested_granularity = criteria + .fts_document_granularity + .unwrap_or(lance_index::scalar::inverted::DocumentGranularity::Row); + let requested = inverted::resolve_fts_field(schema, for_column, requested_granularity)?; + if index.fields[0] != requested.final_field_id { + return Ok(false); + } + let Some(details_any) = index.index_details.as_ref() else { + return Ok(false); + }; + let details = + InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|err| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {err}" + )) + })?; + let details = inverted::normalize_inverted_details(index, details)?; + let stored_granularity = lance_index::scalar::inverted::DocumentGranularity::try_from( + details.document_granularity, + )?; + if stored_granularity != requested_granularity { + return Ok(false); + } } else { - field.name.clone() - }; - if for_column != field_path { - return Ok(false); + let field = fields[0]; + // Build the full field path for nested fields + let field_path = if let Some(ancestors) = schema.field_ancestry_by_id(field.id) { + let field_refs: Vec<&str> = ancestors.iter().map(|f| f.name.as_str()).collect(); + lance_core::datatypes::format_field_path(&field_refs) + } else { + field.name.clone() + }; + if for_column != field_path { + return Ok(false); + } } } @@ -651,10 +817,17 @@ pub async fn initialize_scalar_index( // Parse the JSON into InvertedIndexParams let inverted_params: InvertedIndexParams = serde_json::from_str(params_json)?; + let resolved = inverted::resolve_fts_field_by_id( + target_dataset.schema(), + *source_index.fields.first().ok_or_else(|| { + Error::index("Inverted index metadata has no indexed field".to_string()) + })?, + inverted_params.get_document_granularity(), + )?; target_dataset .create_index( - &[column_name], + &[&resolved.canonical_path], index_type, Some(source_index.name.clone()), &inverted_params, @@ -726,6 +899,7 @@ mod tests { uuid: uuid::Uuid::new_v4(), name: name.to_string(), fields: vec![field_id], + covering_fields: vec![], dataset_version: 1, fragment_bitmap: None, index_details, @@ -736,12 +910,41 @@ mod tests { } } + #[test] + fn test_has_reader_matches_complete_type_name_case_insensitively() { + let has_reader = |type_url: &str| { + IndexDetails(Arc::new(prost_types::Any { + type_url: type_url.to_string(), + value: Vec::new(), + })) + .has_reader() + }; + + for type_url in [ + "/lance.index.pb.VectorIndexDetails", + "type.googleapis.com/LANCE.INDEX.PB.VECTORINDEXDETAILS", + "type.googleapis.com/lance.index.VectorIndexDetails", + "type.googleapis.com/LANCE.TABLE.BTREEINDEXDETAILS", + ] { + assert!(has_reader(type_url), "expected a reader for {type_url}"); + } + + for type_url in [ + "type.googleapis.com/example.MyVectorIndexDetails", + "type.googleapis.com/example.BTreeIndexDetails", + "VectorIndexDetails", + ] { + assert!(!has_reader(type_url), "unexpected reader for {type_url}"); + } + } + #[test] fn test_index_matches_criteria_vector_index() { let index1 = make_index_metadata("vector_index", 1, Some(IndexType::Vector)); let criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: None, @@ -768,6 +971,7 @@ mod tests { let criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: None, @@ -789,6 +993,7 @@ mod tests { // test for_column let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: Some("mycol"), has_name: None, @@ -805,6 +1010,7 @@ mod tests { // test has_name let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: Some("btree_index"), @@ -827,6 +1033,7 @@ mod tests { // test supports_exact_equality let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: true, for_column: None, has_name: None, @@ -848,6 +1055,7 @@ mod tests { // test multiple indices let mut criteria = IndexCriteria { must_support_fts: false, + fts_document_granularity: None, must_support_exact_equality: false, for_column: None, has_name: None, @@ -867,6 +1075,48 @@ mod tests { assert!(result); } + /// A covered scalar index lists its carried columns in `fields` too. It must + /// still match its keyed column -- rejecting on `fields.len() != 1` would + /// silently stop selecting it, with no error and no failing query, just a + /// plan that quietly stops using the index. + #[test] + fn test_index_matches_criteria_covered_index() { + let mut btree_index = make_index_metadata("btree_index", 1, Some(IndexType::BTree)); + // Keyed on field 1, carrying field 2 -- a valid trailing subset. + btree_index.fields = vec![1, 2]; + btree_index.covering_fields = vec![2]; + + let criteria = IndexCriteria { + must_support_fts: false, + fts_document_granularity: None, + must_support_exact_equality: false, + for_column: Some("mycol"), + has_name: None, + }; + + let field = Field::new_arrow("mycol", DataType::Int32, true).unwrap(); + let schema = lance_core::datatypes::Schema { + fields: vec![field.clone()], + metadata: Default::default(), + }; + + let result = + index_matches_criteria(&btree_index, &criteria, &[&field], false, &schema).unwrap(); + assert!( + result, + "a covered scalar index must still match its keyed column" + ); + + // A genuinely composite index -- two keyed fields, no declaration -- + // stays rejected: `for_column` means the index maps to a single column. + let mut composite = make_index_metadata("composite", 1, Some(IndexType::BTree)); + composite.fields = vec![1, 2]; + composite.covering_fields = vec![]; + let result = + index_matches_criteria(&composite, &criteria, &[&field], false, &schema).unwrap(); + assert!(!result, "a composite index must not match a single column"); + } + /// Regression guard for over-projection of `Map` siblings in /// `Field::apply_projection`. Before the parent-selection guard, /// every `Map` column in a schema survived every projection because @@ -1952,6 +2202,53 @@ mod tests { ); } + /// A covered ZoneMap (`fields=[id, other]`, `other` carried) must still be + /// found by `column_value_range("id")`: matching on `idx.fields` as a whole + /// (rather than its keyed prefix) would silently exclude it and lose range + /// pruning with no error. + #[tokio::test] + async fn test_column_value_range_recognizes_covered_index() { + use crate::index::DatasetIndexExt; + use arrow::datatypes::Int64Type; + use datafusion::scalar::ScalarValue; + use lance_datagen::array; + use lance_index::IndexType; + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + + // 2 fragments x 5 rows: `id` and `other` both step 0..9. + let mut ds = lance_datagen::gen_batch() + .col("id", array::step::()) + .col("other", array::step::()) + .into_ram_dataset(FragmentCount::from(2), FragmentRowCount::from(5)) + .await + .unwrap(); + let other_field_id = ds.schema().field("other").unwrap().id; + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap); + let mut segment = ds + .create_index_builder(&["id"], IndexType::Scalar, ¶ms) + .name("id_idx".to_string()) + .execute_uncommitted() + .await + .unwrap(); + // Nothing writes a covering declaration yet, so it is hand-constructed + // here on top of a real single-field segment: read plumbing never touches + // covered-column storage, so a declaration naming a real, uninvolved + // column is a faithful stand-in for a genuinely covered segment. + segment.fields.push(other_field_id); + segment.covering_fields = vec![other_field_id]; + ds.commit_existing_index_segments("id_idx", "id", vec![segment]) + .await + .unwrap(); + + assert_eq!(ds.load_indices_by_name("id_idx").await.unwrap().len(), 1); + assert_eq!( + ds.statistics().column_value_range("id").await.unwrap(), + Some((ScalarValue::Int64(Some(0)), ScalarValue::Int64(Some(9)))), + "covered ZoneMap must still be matched by its keyed column" + ); + } + #[tokio::test] async fn test_zonemap_index_then_deletion() { // Tests the opposite scenario: create index FIRST, then perform deletions @@ -2304,4 +2601,70 @@ mod tests { "Should have 0 rows with value='banana' after deletion" ); } + + // End-to-end: create index → delete a whole fragment → search (via index) → + // update index → search again. No deleted rows should ever appear. + #[tokio::test] + async fn test_zonemap_search_with_deleted_fragment_before_and_after_update() { + use arrow::datatypes::Int32Type; + use lance_datagen::array; + use lance_index::IndexType; + use lance_index::optimize::OptimizeOptions; + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + + // 3 fragments × 10 rows: id 0-9 (frag 0), 10-19 (frag 1), 20-29 (frag 2). + let mut ds = lance_datagen::gen_batch() + .col("id", array::step::()) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(10)) + .await + .unwrap(); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap); + ds.create_index(&["id"], IndexType::Scalar, None, ¶ms, false) + .await + .unwrap(); + + // Delete the middle fragment entirely. + ds.delete("id >= 10 AND id < 20").await.unwrap(); + + // Helper: run a filter scan and return the sorted id values. + async fn live_ids(ds: &crate::Dataset) -> Vec { + let batch = ds + .scan() + .filter("id >= 0") + .unwrap() + .try_into_batch() + .await + .unwrap(); + let mut ids: Vec = batch["id"] + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec(); + ids.sort_unstable(); + ids + } + + // --- Before index update --- + let ids = live_ids(&ds).await; + assert_eq!(ids.len(), 20, "expected 20 live rows before index update"); + assert!( + ids.iter().all(|&id| !(10..20).contains(&id)), + "deleted fragment rows (id 10-19) must not appear before index update; got: {:?}", + ids + ); + + // Update the zone map index to reflect the deletion. + ds.optimize_indices(&OptimizeOptions::new()).await.unwrap(); + + // --- After index update --- + let ids = live_ids(&ds).await; + assert_eq!(ids.len(), 20, "expected 20 live rows after index update"); + assert!( + ids.iter().all(|&id| !(10..20).contains(&id)), + "deleted fragment rows (id 10-19) must not appear after index update; got: {:?}", + ids + ); + } } diff --git a/rust/lance/src/index/scalar/bitmap.rs b/rust/lance/src/index/scalar/bitmap.rs index 2eb5702ee28..f9b8bf69e92 100644 --- a/rust/lance/src/index/scalar/bitmap.rs +++ b/rust/lance/src/index/scalar/bitmap.rs @@ -3,6 +3,7 @@ use lance_index::metrics::NoOpMetricsCollector; use lance_index::scalar::bitmap::BitmapIndex; +use lance_index::scalar::index_files_to_table; use lance_index::scalar::lance_format::LanceIndexStore; use lance_table::format::IndexMetadata; use roaring::RoaringBitmap; @@ -63,14 +64,13 @@ pub(in crate::index) async fn merge_segments( Ok(IndexMetadata { uuid: new_uuid, - fields: vec![field_id], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar/bloomfilter.rs b/rust/lance/src/index/scalar/bloomfilter.rs new file mode 100644 index 00000000000..baa0decae0f --- /dev/null +++ b/rust/lance/src/index/scalar/bloomfilter.rs @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::bloomfilter::BloomFilterIndex; +use lance_index::scalar::index_files_to_table; +use lance_index::scalar::lance_format::LanceIndexStore; +use lance_table::format::IndexMetadata; +use roaring::RoaringBitmap; +use uuid::Uuid; + +use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt}; + +/// Merge one caller-defined group of source BloomFilter segments into a single segment. +pub(in crate::index) async fn merge_segments( + dataset: &Dataset, + segments: Vec, +) -> Result { + if segments.is_empty() { + return Err(Error::index("No segment metadata was provided".to_string())); + } + + let field_id = *segments[0].fields.first().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing field ids", + segments[0].uuid + )) + })?; + let field_path = dataset.schema().field_path(field_id)?; + let dataset_version = segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + let mut fragment_bitmap = RoaringBitmap::new(); + let dataset_fragments = dataset.fragment_bitmap.as_ref(); + let fragment_filters = segments + .iter() + .map(|segment| { + segment + .effective_fragment_bitmap(dataset_fragments) + .ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing fragment coverage", + segment.uuid + )) + }) + }) + .collect::>>()?; + for effective in &fragment_filters { + fragment_bitmap |= effective; + } + + let mut scalar_indices = Vec::with_capacity(segments.len()); + for (position, (segment, effective)) in segments.iter().zip(&fragment_filters).enumerate() { + if effective.is_empty() && !(fragment_bitmap.is_empty() && position == 0) { + continue; + } + let scalar_index = + super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?; + scalar_indices.push((segment.uuid, scalar_index, effective)); + } + + let mut source_indices = Vec::with_capacity(scalar_indices.len()); + for (segment_uuid, scalar_index, fragment_filter) in &scalar_indices { + let bloomfilter_index = scalar_index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::index(format!( + "merge_existing_index_segments: expected bloom filter segment {}, got {:?}", + segment_uuid, + scalar_index.index_type() + )) + })?; + source_indices.push((bloomfilter_index, *fragment_filter)); + } + + let new_uuid = Uuid::new_v4(); + let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; + let created_index = + lance_index::scalar::bloomfilter::merge_bloomfilter_indices(&source_indices, &new_store) + .await?; + + Ok(IndexMetadata { + uuid: new_uuid, + dataset_version, + fragment_bitmap: Some(fragment_bitmap), + index_details: Some(Arc::new(created_index.index_details)), + index_version: created_index.index_version as i32, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: Some(index_files_to_table(created_index.files)), + ..segments[0].clone() + }) +} diff --git a/rust/lance/src/index/scalar/btree.rs b/rust/lance/src/index/scalar/btree.rs index 4339b8c183b..51f3cfc7cd1 100644 --- a/rust/lance/src/index/scalar/btree.rs +++ b/rust/lance/src/index/scalar/btree.rs @@ -15,7 +15,7 @@ use lance_index::pbold::BTreeIndexDetails; use lance_index::scalar::btree::BTreeIndex; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::VALUE_COLUMN_NAME; -use lance_index::scalar::{CreatedIndex, OldIndexDataFilter}; +use lance_index::scalar::{CreatedIndex, OldIndexDataFilter, index_files_to_table}; use lance_table::format::IndexMetadata; use uuid::Uuid; @@ -154,13 +154,14 @@ pub(crate) async fn merge_segments( Ok(IndexMetadata { uuid: output_uuid, name: segments[0].name.clone(), - fields: vec![field_id], + fields: segments[0].fields.clone(), + covering_fields: segments[0].covering_fields.clone(), dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), }) } diff --git a/rust/lance/src/index/scalar/fmindex.rs b/rust/lance/src/index/scalar/fmindex.rs index 32684ebf9ab..eceae7b2033 100644 --- a/rust/lance/src/index/scalar/fmindex.rs +++ b/rust/lance/src/index/scalar/fmindex.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use lance_index::scalar::index_files_to_table; use lance_table::format::IndexMetadata; use roaring::RoaringBitmap; use std::sync::Arc; @@ -69,14 +70,13 @@ pub(in crate::index) async fn merge_segments( return Ok(IndexMetadata { uuid: new_uuid, - fields: vec![field_id], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }); } @@ -104,14 +104,13 @@ pub(in crate::index) async fn merge_segments( Ok(IndexMetadata { uuid: new_uuid, - fields: vec![field_id], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar/inverted.rs b/rust/lance/src/index/scalar/inverted.rs index 000d2c3139c..c7c521050b4 100644 --- a/rust/lance/src/index/scalar/inverted.rs +++ b/rust/lance/src/index/scalar/inverted.rs @@ -3,15 +3,28 @@ #![allow(clippy::redundant_pub_crate)] -use std::sync::Arc; +use std::{collections::BTreeMap, sync::Arc}; -use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; +use arrow_array::cast::AsArray; +use arrow_array::{ + Array, ArrayRef, LargeStringArray, RecordBatch, StringArray, StringViewArray, UInt32Array, + UInt64Array, +}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use datafusion::error::DataFusionError; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use lance_core::ROW_ID; +use futures::StreamExt; +use lance_core::{ + ROW_ID, + datatypes::{Field, LogicalType, Schema, format_field_path, parse_field_path}, +}; use lance_index::metrics::NoOpMetricsCollector; use lance_index::pbold::InvertedIndexDetails; -use lance_index::scalar::inverted::InvertedIndex; +use lance_index::scalar::index_files_to_table; +use lance_index::scalar::inverted::{ + DocumentGranularity, InvertedIndex, InvertedIndexParams, doc_index_storage_column, +}; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::registry::VALUE_COLUMN_NAME; use lance_table::format::IndexMetadata; @@ -25,6 +38,688 @@ use crate::{ index::{DatasetIndexExt, scalar::fetch_index_details}, }; +#[derive(Debug, Clone)] +enum FtsTraversal { + Text, + Struct { + child_index: usize, + child: Box, + }, + List { + child: Box, + }, +} + +impl FtsTraversal { + fn list_depth(&self) -> usize { + match self { + Self::Text => 0, + Self::Struct { child, .. } => child.list_depth(), + Self::List { child } => 1 + child.list_depth(), + } + } +} + +/// Schema-derived form of an FTS field path. +#[derive(Debug, Clone)] +pub(crate) struct ResolvedFtsField { + pub final_field_id: i32, + pub root_column: String, + pub canonical_path: String, + pub document_granularity: DocumentGranularity, + traversal: FtsTraversal, + list_depth: usize, +} + +/// One logical document extracted from a dataset row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FtsDocument { + pub row_index: usize, + pub text: String, + pub doc_index: Vec, +} + +impl ResolvedFtsField { + pub fn has_lists(&self) -> bool { + self.list_depth > 0 + } + + pub fn coordinate_rank(&self) -> usize { + if self.document_granularity.is_list_element() { + self.list_depth + } else { + 0 + } + } + + pub fn documents_from_batch(&self, batch: &RecordBatch) -> Result> { + let column = batch.column_by_name(&self.root_column).ok_or_else(|| { + Error::invalid_input(format!( + "FTS root column '{}' is missing from the input batch", + self.root_column + )) + })?; + self.documents_from_array(column, batch.num_rows()) + } + + fn documents_from_array(&self, column: &ArrayRef, num_rows: usize) -> Result> { + let mut documents = Vec::new(); + match self.document_granularity { + DocumentGranularity::Row => { + documents.reserve(num_rows); + for row_index in 0..num_rows { + let mut text = String::new(); + append_row_text(&self.traversal, column.as_ref(), row_index, &mut text)?; + documents.push(FtsDocument { + row_index, + text, + doc_index: Vec::new(), + }); + } + } + DocumentGranularity::ListElement => { + for row_index in 0..num_rows { + collect_element_documents( + &self.traversal, + column.as_ref(), + row_index, + row_index, + 0, + self.list_depth, + &mut Vec::with_capacity(self.list_depth), + &mut documents, + )?; + } + } + } + Ok(documents) + } +} + +fn string_value(array: &dyn Array, index: usize) -> Result> { + if array.is_null(index) { + return Ok(None); + } + match array.data_type() { + DataType::Utf8 => Ok(Some( + array + .as_any() + .downcast_ref::() + .expect("Utf8 array type") + .value(index), + )), + DataType::LargeUtf8 => Ok(Some( + array + .as_any() + .downcast_ref::() + .expect("LargeUtf8 array type") + .value(index), + )), + DataType::Utf8View => Ok(Some( + array + .as_any() + .downcast_ref::() + .expect("Utf8View array type") + .value(index), + )), + data_type => Err(Error::internal(format!( + "FTS traversal expected a string array, got {data_type}" + ))), + } +} + +fn append_text(output: &mut String, text: &str) { + if !output.is_empty() { + output.push(' '); + } + output.push_str(text); +} + +fn append_row_text( + traversal: &FtsTraversal, + array: &dyn Array, + index: usize, + output: &mut String, +) -> Result<()> { + match traversal { + FtsTraversal::Text => { + if let Some(text) = string_value(array, index)? { + append_text(output, text); + } + } + FtsTraversal::Struct { child_index, child } => { + if !array.is_null(index) { + let structs = array.as_struct(); + append_row_text(child, structs.column(*child_index).as_ref(), index, output)?; + } + } + FtsTraversal::List { child } => match array.data_type() { + DataType::List(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + let start = offsets[index] as usize; + let end = offsets[index + 1] as usize; + for element_index in start..end { + append_row_text(child, lists.values().as_ref(), element_index, output)?; + } + } + } + DataType::LargeList(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + let start = offsets[index] as usize; + let end = offsets[index + 1] as usize; + for element_index in start..end { + append_row_text(child, lists.values().as_ref(), element_index, output)?; + } + } + } + data_type => { + return Err(Error::internal(format!( + "FTS traversal expected a list array, got {data_type}" + ))); + } + }, + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn collect_element_documents( + traversal: &FtsTraversal, + array: &dyn Array, + index: usize, + row_index: usize, + list_depth: usize, + boundary_depth: usize, + doc_index: &mut Vec, + documents: &mut Vec, +) -> Result<()> { + match traversal { + FtsTraversal::Text => Err(Error::internal( + "ListElement FTS traversal did not encounter its list boundary".to_string(), + )), + FtsTraversal::Struct { child_index, child } => { + if !array.is_null(index) { + let structs = array.as_struct(); + collect_element_documents( + child, + structs.column(*child_index).as_ref(), + index, + row_index, + list_depth, + boundary_depth, + doc_index, + documents, + )?; + } + Ok(()) + } + FtsTraversal::List { child } => { + let mut visit = |values: &ArrayRef, start: usize, end: usize| -> Result<()> { + for (ordinal, element_index) in (start..end).enumerate() { + let ordinal = u32::try_from(ordinal).map_err(|_| { + Error::invalid_input(format!( + "FTS element ordinal overflow for row index {row_index}" + )) + })?; + doc_index.push(ordinal); + if list_depth + 1 == boundary_depth { + let mut text = String::new(); + append_row_text(child, values.as_ref(), element_index, &mut text)?; + documents.push(FtsDocument { + row_index, + text, + doc_index: doc_index.clone(), + }); + } else { + collect_element_documents( + child, + values.as_ref(), + element_index, + row_index, + list_depth + 1, + boundary_depth, + doc_index, + documents, + )?; + } + doc_index.pop(); + } + Ok(()) + }; + match array.data_type() { + DataType::List(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + visit( + lists.values(), + offsets[index] as usize, + offsets[index + 1] as usize, + )?; + } + } + DataType::LargeList(_) => { + let lists = array.as_list::(); + if !lists.is_null(index) { + let offsets = lists.value_offsets(); + visit( + lists.values(), + offsets[index] as usize, + offsets[index + 1] as usize, + )?; + } + } + data_type => { + return Err(Error::internal(format!( + "FTS traversal expected a list array, got {data_type}" + ))); + } + } + Ok(()) + } + } +} + +fn find_child_case_insensitive<'a>(field: &'a Field, name: &str) -> Option<(usize, &'a Field)> { + field + .children + .iter() + .enumerate() + .find(|(_, child)| child.name == name) + .or_else(|| { + field + .children + .iter() + .enumerate() + .find(|(_, child)| child.name.eq_ignore_ascii_case(name)) + }) +} + +fn build_fts_traversal( + field: &Field, + remaining_names: &[String], + canonical_names: &mut Vec, + final_field_id: &mut i32, +) -> Result { + match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => { + let child = field.children.first().ok_or_else(|| { + Error::invalid_input(format!( + "FTS list field '{}' does not have an item field", + field.name + )) + })?; + Ok(FtsTraversal::List { + child: Box::new(build_fts_traversal( + child, + remaining_names, + canonical_names, + final_field_id, + )?), + }) + } + DataType::Struct(_) => { + let Some((name, rest)) = remaining_names.split_first() else { + return Err(Error::invalid_input(format!( + "FTS field path ends at struct '{}'; specify the final text field", + field.name + ))); + }; + let (child_index, child) = + find_child_case_insensitive(field, name).ok_or_else(|| { + Error::index(format!( + "FTS field '{}' does not contain child '{}'", + field.name, name + )) + })?; + canonical_names.push(child.name.clone()); + *final_field_id = child.id; + Ok(FtsTraversal::Struct { + child_index, + child: Box::new(build_fts_traversal( + child, + rest, + canonical_names, + final_field_id, + )?), + }) + } + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View if remaining_names.is_empty() => { + Ok(FtsTraversal::Text) + } + _ if field.logical_type == LogicalType::from("json") && remaining_names.is_empty() => { + Ok(FtsTraversal::Text) + } + data_type if !remaining_names.is_empty() => Err(Error::invalid_input(format!( + "FTS field '{}' has type {data_type} and cannot contain child '{}'", + field.name, remaining_names[0] + ))), + data_type => Err(Error::invalid_input(format!( + "FTS field '{}' must resolve to Utf8, LargeUtf8, Utf8View, or JSON, got {data_type}", + field.name + ))), + } +} + +/// Resolve a public FTS field path and derive all list traversal from schema. +pub(crate) fn resolve_fts_field( + schema: &Schema, + path: &str, + document_granularity: DocumentGranularity, +) -> Result { + let names = parse_field_path(path)?; + let (root_name, remaining_names) = names + .split_first() + .ok_or_else(|| Error::invalid_input("FTS field path cannot be empty".to_string()))?; + let root = schema + .fields + .iter() + .find(|field| field.name == *root_name) + .or_else(|| { + schema + .fields + .iter() + .find(|field| field.name.eq_ignore_ascii_case(root_name)) + }) + .ok_or_else(|| { + Error::index(format!( + "FTS field path '{path}' does not exist in the dataset schema" + )) + })?; + + let mut canonical_names = vec![root.name.clone()]; + let mut final_field_id = root.id; + let traversal = build_fts_traversal( + root, + remaining_names, + &mut canonical_names, + &mut final_field_id, + )?; + let list_depth = traversal.list_depth(); + if document_granularity.is_list_element() && list_depth == 0 { + return Err(Error::invalid_input(format!( + "FTS field path '{}' has no List layer and cannot use ListElement document granularity", + format_field_path( + &canonical_names + .iter() + .map(String::as_str) + .collect::>() + ) + ))); + } + if list_depth > 0 && root.logical_type == LogicalType::from("json") { + return Err(Error::invalid_input( + "nested List traversal is not supported for JSON FTS sources".to_string(), + )); + } + let canonical_path = format_field_path( + &canonical_names + .iter() + .map(String::as_str) + .collect::>(), + ); + Ok(ResolvedFtsField { + final_field_id, + root_column: root.name.clone(), + canonical_path, + document_granularity, + traversal, + list_depth, + }) +} + +fn find_public_path_by_id(field: &Field, field_id: i32, path: &mut Vec) -> bool { + if field.id == field_id { + return true; + } + match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => field + .children + .first() + .is_some_and(|child| find_physical_path_by_id(child, field_id, path)), + DataType::Struct(_) => field.children.iter().any(|child| { + path.push(child.name.clone()); + let found = find_public_path_by_id(child, field_id, path); + if !found { + path.pop(); + } + found + }), + _ => false, + } +} + +fn find_physical_path_by_id(field: &Field, field_id: i32, path: &mut Vec) -> bool { + if field.id == field_id { + return false; + } + match field.data_type() { + DataType::List(_) | DataType::LargeList(_) => field + .children + .first() + .is_some_and(|child| find_physical_path_by_id(child, field_id, path)), + DataType::Struct(_) => field.children.iter().any(|child| { + path.push(child.name.clone()); + let found = find_public_path_by_id(child, field_id, path); + if !found { + path.pop(); + } + found + }), + _ => false, + } +} + +/// Resolve an indexed final field id against the current schema, preserving +/// routing across field renames. +pub(crate) fn resolve_fts_field_by_id( + schema: &Schema, + field_id: i32, + document_granularity: DocumentGranularity, +) -> Result { + for root in &schema.fields { + let mut path = vec![root.name.clone()]; + if find_public_path_by_id(root, field_id, &mut path) { + let path_refs = path.iter().map(String::as_str).collect::>(); + return resolve_fts_field(schema, &format_field_path(&path_refs), document_granularity); + } + } + Err(Error::invalid_input(format!( + "FTS index refers to missing or Arrow-internal field id {field_id}" + ))) +} + +/// Return the persisted FTS document granularity for each logical index on a +/// public field path. Multiple physical segments with the same index name are +/// collapsed after validating that their metadata agrees. +pub(crate) async fn indexed_fts_document_granularities( + dataset: &Dataset, + column: &str, +) -> Result> { + let resolved = resolve_fts_field(dataset.schema(), column, DocumentGranularity::Row)?; + let mut by_name = BTreeMap::new(); + + for index in dataset.load_indices().await?.iter() { + // A covered FTS index still answers for its keyed column; only the + // keyed prefix decides whether this index matches, not the full + // `fields` vector including carried columns. + if index.keyed_field() != Some(resolved.final_field_id) { + continue; + } + let details_any = fetch_index_details(dataset, &resolved.canonical_path, index).await?; + if !details_any.type_url.ends_with("InvertedIndexDetails") { + continue; + } + let details = + InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|error| { + Error::corrupt_file( + dataset.indices_dir().join(index.uuid.to_string()), + format!( + "failed to decode InvertedIndexDetails for FTS index '{}': {error}", + index.name + ), + ) + })?; + let document_granularity = DocumentGranularity::try_from(details.document_granularity)?; + if let Some(existing) = by_name.insert(index.name.clone(), document_granularity) + && existing != document_granularity + { + return Err(Error::corrupt_file( + dataset.indices_dir().join(index.uuid.to_string()), + format!( + "FTS index '{}' has inconsistent document granularity across segments: \ + {existing:?} and {document_granularity:?}", + index.name + ), + )); + } + } + + Ok(by_name.into_iter().collect()) +} + +/// Resolve an optional query granularity against persisted FTS index metadata. +/// +/// A unique indexed granularity is authoritative and also controls flat search +/// over unindexed fragments. An explicit request selects between coexisting +/// row and list-element indexes, but cannot contradict the only indexed +/// granularity. With no index, the established row default is retained. +pub(crate) async fn resolve_query_document_granularity( + dataset: &Dataset, + column: &str, + requested: Option, +) -> Result { + let indices = indexed_fts_document_granularities(dataset, column).await?; + let mut available = indices + .iter() + .map(|(_, document_granularity)| *document_granularity) + .collect::>(); + available.sort_by_key(|document_granularity| match document_granularity { + DocumentGranularity::Row => 0, + DocumentGranularity::ListElement => 1, + }); + available.dedup(); + + let resolved = match requested { + Some(requested) if available.is_empty() || available.contains(&requested) => requested, + Some(requested) => { + let indexed = indices + .iter() + .map(|(name, document_granularity)| format!("'{name}' ({document_granularity:?})")) + .collect::>() + .join(", "); + return Err(Error::invalid_input(format!( + "FTS query for field '{column}' requested {requested:?} document granularity, \ + but the existing FTS index uses a different granularity: {indexed}" + ))); + } + None if available.is_empty() => DocumentGranularity::Row, + None if available.len() == 1 => available[0], + None => { + let indexed = indices + .iter() + .map(|(name, document_granularity)| format!("'{name}' ({document_granularity:?})")) + .collect::>() + .join(", "); + return Err(Error::invalid_input(format!( + "FTS query for field '{column}' is ambiguous because Row and ListElement \ + indexes coexist: {indexed}; specify document_granularity" + ))); + } + }; + + resolve_fts_field(dataset.schema(), column, resolved)?; + Ok(resolved) +} + +pub(crate) fn fts_document_schema(coordinate_rank: usize) -> Arc { + let mut fields = vec![ + ArrowField::new(VALUE_COLUMN_NAME, DataType::Utf8, false), + ArrowField::new(ROW_ID, DataType::UInt64, false), + ]; + fields.extend( + (0..coordinate_rank) + .map(|rank| ArrowField::new(doc_index_storage_column(rank), DataType::UInt32, false)), + ); + Arc::new(ArrowSchema::new(fields)) +} + +pub(crate) fn transform_fts_document_stream( + input: SendableRecordBatchStream, + resolved: ResolvedFtsField, +) -> Result { + let input_schema = input.schema(); + if (input_schema + .column_with_name(&resolved.root_column) + .is_none() + && input_schema.column_with_name(VALUE_COLUMN_NAME).is_none()) + || input_schema.column_with_name(ROW_ID).is_none() + { + return Err(Error::internal( + "FTS document input must contain the root source column and _rowid".to_string(), + )); + } + let output_schema = fts_document_schema(resolved.coordinate_rank()); + let stream_schema = output_schema.clone(); + let stream = input.map(move |batch| { + let batch = batch?; + let source = batch + .column_by_name(&resolved.root_column) + .or_else(|| batch.column_by_name(VALUE_COLUMN_NAME)) + .expect("FTS document input schema was validated"); + let documents = resolved + .documents_from_array(source, batch.num_rows()) + .map_err(DataFusionError::from)?; + let input_row_ids = batch[ROW_ID].as_primitive::(); + let texts = + StringArray::from_iter_values(documents.iter().map(|document| document.text.as_str())); + let row_ids = UInt64Array::from_iter_values( + documents + .iter() + .map(|document| input_row_ids.value(document.row_index)), + ); + let mut columns = vec![Arc::new(texts) as ArrayRef, Arc::new(row_ids) as ArrayRef]; + for rank in 0..resolved.coordinate_rank() { + columns.push(Arc::new(UInt32Array::from_iter_values( + documents.iter().map(|document| document.doc_index[rank]), + )) as ArrayRef); + } + RecordBatch::try_new(stream_schema.clone(), columns).map_err(DataFusionError::from) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + output_schema, + stream, + ))) +} + +/// Fill legacy posting-format metadata while leaving protobuf-default row +/// document granularity untouched. +pub(crate) fn normalize_inverted_details( + index: &IndexMetadata, + mut details: InvertedIndexDetails, +) -> Result { + if !matches!(index.index_version, 0..=3) { + return Err(Error::invalid_input(format!( + "FTS index '{}' has unsupported index_version {}; expected 0, 1, 2, or 3", + index.name, index.index_version + ))); + } + if details.posting_format_version.is_none() { + let posting_format_version = match index.index_version { + 0 | 1 => 1, + 2 => 2, + 3 => 3, + _ => unreachable!("index version was validated above"), + }; + details.posting_format_version = Some(posting_format_version); + } + Ok(details) +} + /// Build an empty update stream for the inverted merge API. /// /// `InvertedIndex::merge_segments` is shaped as "merge old segments plus new @@ -32,18 +727,25 @@ use crate::{ /// and `_rowid` fields. The stream intentionally contains no batches. fn empty_inverted_update_stream( dataset: &Dataset, - field_id: i32, + resolved: &ResolvedFtsField, ) -> Result { - let field = dataset.schema().field_by_id(field_id).ok_or_else(|| { - Error::invalid_input(format!( - "merge_existing_index_segments: field id {} does not exist", - field_id - )) - })?; - let schema = Arc::new(ArrowSchema::new(vec![ - ArrowField::new(VALUE_COLUMN_NAME, field.data_type(), true), - ArrowField::new(ROW_ID, arrow_schema::DataType::UInt64, false), - ])); + let field = dataset + .schema() + .field_by_id(resolved.final_field_id) + .ok_or_else(|| { + Error::invalid_input(format!( + "merge_existing_index_segments: field id {} does not exist", + resolved.final_field_id + )) + })?; + let schema = if resolved.has_lists() { + fts_document_schema(resolved.coordinate_rank()) + } else { + Arc::new(ArrowSchema::new(vec![ + ArrowField::new(VALUE_COLUMN_NAME, field.data_type(), true), + ArrowField::new(ROW_ID, arrow_schema::DataType::UInt64, false), + ])) + }; Ok(Box::pin(RecordBatchStreamAdapter::new( schema, futures::stream::empty(), @@ -90,7 +792,21 @@ pub(crate) async fn merge_segments( segments[0].uuid )) })?; - let field_path = dataset.schema().field_path(field_id)?; + let details = match segments[0].index_details.as_ref() { + Some(details_any) => { + let details = + InvertedIndexDetails::decode(details_any.value.as_slice()).map_err(|error| { + Error::io(format!( + "failed to decode InvertedIndexDetails payload: {error}" + )) + })?; + normalize_inverted_details(&segments[0], details)? + } + None => normalize_inverted_details(&segments[0], InvertedIndexDetails::default())?, + }; + let document_granularity = DocumentGranularity::try_from(details.document_granularity)?; + let resolved = resolve_fts_field_by_id(dataset.schema(), field_id, document_granularity)?; + load_segment_details(dataset, &resolved.canonical_path, &segments).await?; let mut source_indices = Vec::with_capacity(segments.len()); let mut fragment_bitmap = RoaringBitmap::new(); @@ -102,8 +818,19 @@ pub(crate) async fn merge_segments( segment.uuid )) })?; - let scalar_index = - super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?; + if segment.fields != segments[0].fields { + return Err(Error::invalid_input(format!( + "FTS index {} has inconsistent fields across segments", + segments[0].name + ))); + } + let scalar_index = super::open_scalar_index( + dataset, + &resolved.canonical_path, + segment, + &NoOpMetricsCollector, + ) + .await?; let inverted_index = scalar_index .as_any() .downcast_ref::() @@ -121,7 +848,7 @@ pub(crate) async fn merge_segments( let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; let created_index = InvertedIndex::merge_segments( &source_indices, - empty_inverted_update_stream(dataset, field_id)?, + empty_inverted_update_stream(dataset, &resolved)?, &new_store, None, lance_index::progress::noop_progress(), @@ -130,14 +857,13 @@ pub(crate) async fn merge_segments( Ok(IndexMetadata { uuid: new_uuid, - fields: vec![field_id], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } @@ -149,12 +875,17 @@ pub(crate) async fn merge_segments( /// exists, the returned vector contains every committed segment's /// [`IndexMetadata`] (UUID, fragment coverage, index details). All segments /// must share the same indexed fields; mismatched fields return an error. -pub async fn load_segments(dataset: &Dataset, column: &str) -> Result>> { +pub async fn load_segments( + dataset: &Dataset, + column: &str, + document_granularity: DocumentGranularity, +) -> Result>> { let Some(index_meta) = dataset .load_scalar_index( lance_index::IndexCriteria::default() .for_column(column) - .supports_fts(), + .supports_fts() + .with_fts_document_granularity(document_granularity), ) .await? else { @@ -179,13 +910,43 @@ pub async fn load_segments(dataset: &Dataset, column: &str) -> Result Result> { + let Some(segments) = load_segments(dataset, column, document_granularity).await? else { + return Ok(None); + }; + + let fragment_bitmap = + segments + .iter() + .try_fold(RoaringBitmap::new(), |mut coverage, segment| { + coverage |= segment.fragment_bitmap.as_ref()?.clone(); + Some(coverage) + }); + + Ok(fragment_bitmap) +} + /// Load and validate the shared [`InvertedIndexDetails`] across committed /// segments returned by [`load_segments`]. /// -/// All segments are required to agree on their decoded `InvertedIndexDetails` -/// payload (analyzer, tokenizer, position settings, etc.); inconsistent -/// segments return an error. Returns the canonical details that may be used -/// when constructing a tokenizer or running a query against the index. +/// All segments are required to agree on their semantic `InvertedIndexDetails` +/// payload (tokenizer, position settings, etc.); inconsistent +/// segments return an error. Details are canonicalized before comparison so +/// legacy segments that omit default fields remain compatible with newly +/// written text FTS segments. `posting_format_version` is a physical +/// per-segment property and may differ when a legacy FTS v1 segment is +/// combined with a newly written one. Returns the first segment's +/// canonicalized details for tokenizer construction and query planning. pub async fn load_segment_details( dataset: &Dataset, column: &str, @@ -200,8 +961,9 @@ pub async fn load_segment_details( "failed to decode InvertedIndexDetails payload: {err}" )) })?; + let details = canonicalize_inverted_index_details(details)?; match &expected_details { - Some(expected) if expected != &details => { + Some(expected) if !inverted_index_details_semantically_equal(expected, &details) => { return Err(Error::invalid_input(format!( "FTS index {} has inconsistent inverted index details across segments", meta.name @@ -219,10 +981,122 @@ pub async fn load_segment_details( }) } +fn canonicalize_inverted_index_details( + details: InvertedIndexDetails, +) -> Result { + let params = InvertedIndexParams::try_from(&details)?; + InvertedIndexDetails::try_from(¶ms) +} + +/// Compare canonicalized inverted-index details for shared semantic configuration. +/// +/// `posting_format_version` records how a single segment physically stores +/// postings, so mixed-version FTS segments may disagree on it without being +/// incompatible. Every other field remains part of the equality check. +fn inverted_index_details_semantically_equal( + left: &InvertedIndexDetails, + right: &InvertedIndexDetails, +) -> bool { + let mut left = left.clone(); + let mut right = right.clone(); + left.posting_format_version = None; + right.posting_format_version = None; + left == right +} + +/// Read one segment's [`InvertedIndexParams`] +pub async fn load_segment_params( + dataset: &Dataset, + segment: &IndexMetadata, +) -> Result { + let store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?; + InvertedIndex::load_params(&store).await +} + #[cfg(test)] mod tests { use super::*; + /// A covered FTS index (`fields=[tags, id]`, `id` carried) must still be + /// recognized by `indexed_fts_document_granularities`: matching on + /// `idx.fields` as a whole (rather than its keyed prefix) would leave + /// `available` empty and silently fall back to the `Row` default even + /// though a `ListElement` index exists. + #[tokio::test] + async fn test_resolve_query_document_granularity_recognizes_covered_index() { + use arrow_array::builder::{ListBuilder, StringBuilder}; + use arrow_array::{Int32Array, RecordBatchIterator}; + use lance_core::utils::tempfile::TempStrDir; + use lance_index::IndexType; + + let mut tags_builder = ListBuilder::new(StringBuilder::new()); + for tag in ["alpha", "beta", "gamma", "delta"] { + tags_builder.values().append_value(tag); + tags_builder.append(true); + } + let tags: ArrayRef = Arc::new(tags_builder.finish()); + let ids: ArrayRef = Arc::new(Int32Array::from_iter_values(0..4)); + let batch = RecordBatch::try_from_iter(vec![("id", ids), ("tags", tags)]).unwrap(); + let schema = batch.schema(); + + let test_dir = TempStrDir::default(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.as_str(), + None, + ) + .await + .unwrap(); + + let tags_field_id = dataset.schema().field("tags").unwrap().id; + let id_field_id = dataset.schema().field("id").unwrap().id; + + let params = + InvertedIndexParams::default().document_granularity(DocumentGranularity::ListElement); + let segment = dataset + .create_index_builder(&["tags"], IndexType::Inverted, ¶ms) + .name("tags_idx".to_string()) + .execute_uncommitted() + .await + .unwrap(); + + // Nothing writes a covering declaration yet, and read plumbing never + // touches covered-column storage, so declaring the uninvolved `id` + // column as carried is a faithful stand-in for a genuinely covered + // FTS segment. + let covered = IndexMetadata { + fields: vec![tags_field_id, id_field_id], + covering_fields: vec![id_field_id], + ..segment + }; + dataset + .commit_existing_index_segments("tags_idx", "tags", vec![covered]) + .await + .unwrap(); + + let resolved = resolve_query_document_granularity(&dataset, "tags", None) + .await + .unwrap(); + assert_eq!( + resolved, + DocumentGranularity::ListElement, + "a covered ListElement FTS index must still be recognized, not silently \ + skipped in favor of the Row default" + ); + } + + fn fts_test_schema() -> Schema { + let schema = ArrowSchema::new(vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new( + "tags", + DataType::List(Arc::new(ArrowField::new("item", DataType::Utf8, true))), + true, + ), + ]); + Schema::try_from(&schema).unwrap() + } + #[test] fn decode_legacy_inverted_details_type_url() { let mut details_any = prost_types::Any::from_msg(&InvertedIndexDetails::default()).unwrap(); @@ -231,4 +1105,106 @@ mod tests { let decoded = InvertedIndexDetails::decode(details_any.value.as_slice()).unwrap(); assert_eq!(decoded, InvertedIndexDetails::default()); } + + #[test] + fn resolve_element_document_field() { + let schema = fts_test_schema(); + let tags = schema.field("tags").unwrap(); + let resolved = + resolve_fts_field(&schema, "tags", DocumentGranularity::ListElement).unwrap(); + assert_eq!(resolved.final_field_id, tags.id); + assert_eq!(resolved.root_column, "tags"); + assert_eq!(resolved.canonical_path, "tags"); + assert_eq!(resolved.coordinate_rank(), 1); + + let err = resolve_fts_field(&schema, "text", DocumentGranularity::ListElement).unwrap_err(); + assert!(err.to_string().contains("has no List layer"), "{err}"); + + let err = + resolve_fts_field(&schema, "tags[*]", DocumentGranularity::ListElement).unwrap_err(); + assert!(err.to_string().contains("does not exist"), "{err}"); + } + + #[test] + fn normalize_legacy_metadata_as_row_document() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![1], + covering_fields: vec![], + name: "tags_idx".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 3, + created_at: None, + base_id: None, + files: None, + }; + let details = + normalize_inverted_details(&metadata, InvertedIndexDetails::default()).unwrap(); + assert_eq!( + DocumentGranularity::try_from(details.document_granularity).unwrap(), + DocumentGranularity::Row + ); + assert_eq!(details.posting_format_version, Some(3)); + } + + #[test] + fn normalize_rejects_unreleased_v4_metadata() { + let metadata = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![1], + covering_fields: vec![], + name: "tags_idx".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 4, + created_at: None, + base_id: None, + files: None, + }; + let details = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); + + let err = normalize_inverted_details(&metadata, details).unwrap_err(); + + assert!(err.to_string().contains("unsupported index_version 4")); + } + + #[test] + fn canonicalize_inverted_details_accepts_legacy_empty_details() { + let legacy = InvertedIndexDetails::default(); + let current = InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(); + + assert_ne!(legacy, current); + assert_eq!( + canonicalize_inverted_index_details(legacy).unwrap(), + canonicalize_inverted_index_details(current).unwrap() + ); + } + + #[test] + fn inverted_details_equal_when_only_posting_format_version_differs() { + let left = canonicalize_inverted_index_details( + InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(), + ) + .unwrap(); + let mut right = left.clone(); + right.posting_format_version = Some(1); + + assert_ne!(left, right); + assert!(inverted_index_details_semantically_equal(&left, &right)); + } + + #[test] + fn inverted_details_reject_with_position_mismatch() { + let left = canonicalize_inverted_index_details( + InvertedIndexDetails::try_from(&InvertedIndexParams::default()).unwrap(), + ) + .unwrap(); + let mut right = left.clone(); + right.with_position = !left.with_position; + + assert!(!inverted_index_details_semantically_equal(&left, &right)); + } } diff --git a/rust/lance/src/index/scalar/label_list.rs b/rust/lance/src/index/scalar/label_list.rs index 27bc49643bb..98f412b46c9 100644 --- a/rust/lance/src/index/scalar/label_list.rs +++ b/rust/lance/src/index/scalar/label_list.rs @@ -3,6 +3,7 @@ use lance_index::metrics::NoOpMetricsCollector; use lance_index::scalar::IndexStore; +use lance_index::scalar::index_files_to_table; use lance_index::scalar::label_list::{ BITMAP_LOOKUP_NAME, LABEL_LIST_NULLS_METADATA_KEY, LABEL_LIST_NULLS_MIN_VERSION, LabelListIndex, }; @@ -135,14 +136,13 @@ pub(in crate::index) async fn merge_segments( Ok(IndexMetadata { uuid: new_uuid, - fields: vec![field_id], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar/ngram.rs b/rust/lance/src/index/scalar/ngram.rs new file mode 100644 index 00000000000..11f0d5a0f52 --- /dev/null +++ b/rust/lance/src/index/scalar/ngram.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use datafusion::physical_plan::SendableRecordBatchStream; +use lance_index::frag_reuse::CompactFragReuseIndexHandle; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::progress::NoopIndexBuildProgress; +use lance_index::scalar::lance_format::LanceIndexStore; +use lance_index::scalar::ngram::NGramIndex; +use lance_index::scalar::{ + BuiltinIndexType, CreatedIndex, IndexStore, OldIndexDataFilter, RowIdRemapper, + ScalarIndexParams, index_files_to_table, +}; +use lance_table::format::IndexMetadata; +use roaring::RoaringBitmap; +use uuid::Uuid; + +use crate::{ + Dataset, Error, Result, dataset::index::LanceIndexStoreExt, index::DatasetIndexInternalExt, +}; + +async fn collect_ngram_segment_stores( + dataset: &Dataset, + segments: &[IndexMetadata], +) -> Result>> { + let mut stores: Vec> = Vec::with_capacity(segments.len()); + for segment in segments { + let store = LanceIndexStore::from_dataset_for_existing(dataset, segment).await?; + stores.push(Arc::new(store)); + } + Ok(stores) +} + +/// Merge one caller-defined group of source NGram segments into a single segment. +pub(in crate::index) async fn merge_segments( + dataset: &Dataset, + segments: Vec, +) -> Result { + if segments.is_empty() { + return Err(Error::index("No segment metadata was provided".to_string())); + } + + // All source segments must belong to the same column. + let reference_fields = segments[0].fields.as_slice(); + for segment in segments.iter().skip(1) { + if segment.fields.as_slice() != reference_fields { + return Err(Error::invalid_input(format!( + "NGram merge_segments: segment {} has fields {:?}, expected {:?}", + segment.uuid, segment.fields, reference_fields, + ))); + } + } + + let field_id = *segments[0].fields.first().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing field ids", + segments[0].uuid + )) + })?; + let source_dataset_version = segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + let segment_refs = segments.iter().collect::>(); + + let new_uuid = Uuid::new_v4(); + let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let has_retired_coverage = segments.iter().any(|segment| { + segment + .deleted_fragment_bitmap(&dataset.fragment_bitmap) + .is_some_and(|retired| !retired.is_empty()) + }); + let requires_rebuild = frag_reuse_index.as_ref().is_some_and(|frag_reuse_index| { + crate::index::append::fragment_reuse_affects_segments( + frag_reuse_index, + segment_refs.iter().copied(), + ) + }); + if has_retired_coverage && !requires_rebuild { + return Err(Error::invalid_input( + "NGram merge_segments: source segments cover retired fragments but no applicable \ + fragment-reuse mapping is available; rebuild the affected segments from the current \ + dataset" + .to_string(), + )); + } + let (created_index, dataset_version, fragment_bitmap) = if requires_rebuild { + let mut fragment_bitmap = segments + .iter() + .map(|segment| { + segment.fragment_bitmap.as_ref().cloned().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing fragment coverage", + segment.uuid + )) + }) + }) + .collect::>>()? + .into_iter() + .fold(RoaringBitmap::new(), |coverage, segment| coverage | segment); + frag_reuse_index + .as_ref() + .expect("requires_rebuild implies a fragment-reuse index") + .remap_fragment_bitmap(&mut fragment_bitmap)?; + fragment_bitmap &= dataset.fragment_bitmap.as_ref(); + let field_path = dataset.schema().field_path(field_id)?; + ( + super::build_scalar_index( + dataset, + &field_path, + new_uuid, + &ScalarIndexParams::for_builtin(BuiltinIndexType::NGram), + true, + Some(fragment_bitmap.iter().collect()), + None, + Arc::new(NoopIndexBuildProgress), + ) + .await?, + dataset.manifest.version, + fragment_bitmap, + ) + } else { + let (fragment_bitmap, old_data_filters) = + crate::index::append::build_per_segment_filters(dataset, &segment_refs).await?; + let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; + ( + open_and_merge_segments(dataset, &segment_refs, None, &new_store, &old_data_filters) + .await?, + source_dataset_version, + fragment_bitmap, + ) + }; + + Ok(IndexMetadata { + uuid: new_uuid, + name: segments[0].name.clone(), + fields: segments[0].fields.clone(), + covering_fields: segments[0].covering_fields.clone(), + dataset_version, + fragment_bitmap: Some(fragment_bitmap), + index_details: Some(Arc::new(created_index.index_details)), + index_version: created_index.index_version as i32, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: Some(index_files_to_table(created_index.files)), + }) +} + +/// Merge the given NGram segments with optional newly appended data into a single +/// canonical segment. Pass `None` for `new_data` for a pure consolidation. +pub(in crate::index) async fn open_and_merge_segments( + dataset: &Dataset, + segments: &[&IndexMetadata], + new_data: Option, + new_store: &LanceIndexStore, + old_data_filters: &[Option], +) -> Result { + let segments = segments.iter().map(|&s| s.clone()).collect::>(); + let segment_stores = collect_ngram_segment_stores(dataset, &segments).await?; + let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(CompactFragReuseIndexHandle(index)) as Arc); + NGramIndex::merge_segments_with_remapper( + &segment_stores, + new_data, + new_store, + old_data_filters, + frag_reuse_index, + ) + .await +} diff --git a/rust/lance/src/index/scalar/rtree.rs b/rust/lance/src/index/scalar/rtree.rs new file mode 100644 index 00000000000..92639c9274d --- /dev/null +++ b/rust/lance/src/index/scalar/rtree.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::sync::Arc; + +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::lance_format::LanceIndexStore; +use lance_index::scalar::rtree::RTreeIndex; +use lance_index::scalar::{OldIndexDataFilter, index_files_to_table}; +use lance_select::RowSetOps; +use lance_table::format::IndexMetadata; +use uuid::Uuid; + +use crate::{Dataset, Error, Result, dataset::index::LanceIndexStoreExt}; + +fn filter_keeps_nothing(filter: &Option) -> bool { + match filter { + Some(OldIndexDataFilter::Fragments { to_keep, .. }) => to_keep.is_empty(), + Some(OldIndexDataFilter::RowIds(valid)) => valid.is_empty(), + None => false, + } +} + +/// Merge one caller-defined group of source RTree segments into a single segment. +pub(in crate::index) async fn merge_segments( + dataset: &Dataset, + segments: Vec, +) -> Result { + if segments.is_empty() { + return Err(Error::index("No segment metadata was provided".to_string())); + } + + let field_id = *segments[0].fields.first().ok_or_else(|| { + Error::invalid_input(format!( + "CreateIndex: segment {} is missing field ids", + segments[0].uuid + )) + })?; + let field_path = dataset.schema().field_path(field_id)?; + let dataset_version = segments + .iter() + .map(|segment| segment.dataset_version) + .min() + .unwrap_or(dataset.manifest.version); + let segment_refs = segments.iter().collect::>(); + let (fragment_bitmap, old_data_filters) = + crate::index::append::build_per_segment_filters(dataset, &segment_refs).await?; + + let mut source_indices = Vec::with_capacity(segments.len()); + let mut source_filters = Vec::with_capacity(old_data_filters.len()); + let all_keep_nothing = old_data_filters.iter().all(filter_keeps_nothing); + for (position, (segment, filter)) in segments.iter().zip(&old_data_filters).enumerate() { + if filter_keeps_nothing(filter) && !(all_keep_nothing && position == 0) { + continue; + } + let scalar_index = + super::open_scalar_index(dataset, &field_path, segment, &NoOpMetricsCollector).await?; + let rtree_index = scalar_index + .as_any() + .downcast_ref::() + .ok_or_else(|| { + Error::index(format!( + "merge_existing_index_segments: expected RTree segment {}, got {:?}", + segment.uuid, + scalar_index.index_type() + )) + })?; + source_indices.push(Arc::new(rtree_index.clone())); + source_filters.push(filter.clone()); + } + + let new_uuid = Uuid::new_v4(); + let new_store = LanceIndexStore::from_dataset_for_new(dataset, &new_uuid)?; + let created_index = lance_index::scalar::rtree::merge_rtree_indices( + &source_indices, + &new_store, + &source_filters, + ) + .await?; + + Ok(IndexMetadata { + uuid: new_uuid, + dataset_version, + fragment_bitmap: Some(fragment_bitmap), + index_details: Some(Arc::new(created_index.index_details)), + index_version: created_index.index_version as i32, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: Some(index_files_to_table(created_index.files)), + ..segments[0].clone() + }) +} diff --git a/rust/lance/src/index/scalar/zonemap.rs b/rust/lance/src/index/scalar/zonemap.rs index 0cbd98f2c40..6d619fb7984 100644 --- a/rust/lance/src/index/scalar/zonemap.rs +++ b/rust/lance/src/index/scalar/zonemap.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use lance_index::metrics::NoOpMetricsCollector; +use lance_index::scalar::index_files_to_table; use lance_index::scalar::lance_format::LanceIndexStore; use lance_index::scalar::zonemap::ZoneMapIndex; use lance_table::format::IndexMetadata; @@ -73,14 +74,13 @@ pub(in crate::index) async fn merge_segments( Ok(IndexMetadata { uuid: new_uuid, - fields: vec![field_id], dataset_version: dataset.manifest.version, fragment_bitmap: Some(fragment_bitmap), index_details: Some(Arc::new(created_index.index_details)), index_version: created_index.index_version as i32, created_at: Some(chrono::Utc::now()), base_id: None, - files: Some(created_index.files), + files: Some(index_files_to_table(created_index.files)), ..segments[0].clone() }) } diff --git a/rust/lance/src/index/scalar_logical.rs b/rust/lance/src/index/scalar_logical.rs index 4ff0da7f091..4a22aac62e7 100644 --- a/rust/lance/src/index/scalar_logical.rs +++ b/rust/lance/src/index/scalar_logical.rs @@ -12,7 +12,9 @@ use futures::future::try_join_all; use lance_core::deepsize::{Context, DeepSizeOf}; use lance_core::{Error, Result}; use lance_index::metrics::MetricsCollector; -use lance_index::scalar::{AnyQuery, CreatedIndex, ScalarIndex, SearchResult, UpdateCriteria}; +use lance_index::scalar::{ + AnyQuery, CreatedIndex, ScalarIndex, SearchOptions, SearchResult, UpdateCriteria, +}; use lance_index::{Index, IndexType}; use lance_select::NullableRowAddrSet; use lance_table::format::IndexMetadata; @@ -23,6 +25,21 @@ use crate::dataset::Dataset; use crate::index::scalar::fetch_index_details; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt}; +/// Query-time view that exposes several physical scalar index segments as a single [`ScalarIndex`]. +/// +/// A named scalar index can be built incrementally, producing multiple physical segments that +/// each cover a disjoint set of fragments. When such an index is opened, the loader bundles +/// the segments into a `LogicalScalarIndex` so the scanner can treat them as one index: queries +/// are fanned out to every segment in parallel and the row-address results are unioned together. +/// +/// All segments must share the same [`IndexType`]; mixing types is rejected at construction. +/// Per-segment [`SearchResult`] precision is preserved when combining: a union of `Exact` +/// results stays `Exact`, a union containing `AtMost` results yields `AtMost`, and a union +/// containing `AtLeast` results yields `AtLeast`. Combining `AtMost` and `AtLeast` segments in +/// the same query is not supported. +/// +/// This is a read-only wrapper. [`ScalarIndex::remap`] and [`ScalarIndex::update`] both return +/// an error — callers must rebuild the index to consolidate segments before mutating it. #[derive(Debug)] pub struct LogicalScalarIndex { name: String, @@ -126,16 +143,32 @@ impl ScalarIndex for LogicalScalarIndex { &self, query: &dyn AnyQuery, metrics: &dyn MetricsCollector, + ) -> Result { + self.search_with_options(query, SearchOptions::default(), metrics) + .await + } + + async fn search_with_options( + &self, + query: &dyn AnyQuery, + options: SearchOptions, + metrics: &dyn MetricsCollector, ) -> Result { let results = try_join_all( self.segments .iter() - .map(|segment| segment.search(query, metrics)), + .map(|segment| segment.search_with_options(query, options, metrics)), ) .await?; combine_search_results(results) } + fn results_are_row_addresses(&self) -> bool { + // All segments of a logical index share the same underlying index type, + // so they agree on the result domain. + self.segments[0].results_are_row_addresses() + } + fn can_remap(&self) -> bool { false } @@ -274,6 +307,11 @@ fn union_fragment_bitmaps(indices: &[IndexMetadata], index_name: &str) -> Result Ok(combined) } +/// Return the union of fragment bitmaps across every usable segment of a named scalar index. +/// +/// Only segments whose fragment bitmap intersects the dataset's current fragment set are +/// considered. Returns `Ok(None)` when no such segment exists, `Ok(Some(bitmap))` otherwise. +/// Errors if the segments disagree on their underlying index type. pub async fn scalar_index_fragment_bitmap( dataset: &Dataset, column: &str, @@ -290,6 +328,13 @@ pub async fn scalar_index_fragment_bitmap( } } +/// Open a named scalar index, transparently bundling multiple segments when present. +/// +/// Loads every segment registered under `index_name` whose fragment bitmap intersects the +/// dataset. If exactly one usable segment exists it is returned directly; if multiple exist +/// they are wrapped in a [`LogicalScalarIndex`] so the caller sees a single [`ScalarIndex`]. +/// Errors if no usable segment exists (the scanner planned a query against an index that is +/// not present) or if the segments mix incompatible types. pub async fn open_named_scalar_index( dataset: &Dataset, column: &str, @@ -333,11 +378,14 @@ mod tests { use datafusion::scalar::ScalarValue; use lance_core::utils::address::RowAddress; use lance_core::utils::tempfile::TempStrDir; - use lance_datagen::array; + use lance_datagen::{ArrayGeneratorExt, array}; use lance_index::IndexType; use lance_index::metrics::NoOpMetricsCollector; use lance_index::scalar::bitmap::BITMAP_LOOKUP_NAME; - use lance_index::scalar::{BuiltinIndexType, SargableQuery, ScalarIndexParams}; + use lance_index::scalar::{ + BuiltinIndexType, SargableQuery, ScalarIndexParams, SearchOptions, SearchResult, + }; + use lance_select::{RowAddrTreeMap, RowSetOps}; use crate::Dataset; use crate::dataset::WriteParams; @@ -399,7 +447,10 @@ mod tests { async fn test_open_named_scalar_index_uses_all_btree_segments() { let test_dir = TempStrDir::default(); let dataset = lance_datagen::gen_batch() - .col("value", array::step::()) + .col( + "value", + array::fill::(7).with_nulls(&[true, false, true, true]), + ) .into_dataset( test_dir.as_str(), FragmentCount::from(4), @@ -441,6 +492,24 @@ mod tests { dataset.fragment_bitmap.as_ref().clone() ); + let query = SargableQuery::Equals(ScalarValue::Int32(Some(99))); + let tracked = logical.search(&query, &NoOpMetricsCollector).await.unwrap(); + let SearchResult::Exact(tracked) = tracked else { + panic!("BTree search should be exact"); + }; + assert!(tracked.true_rows().is_empty()); + assert!(!tracked.null_rows().is_empty()); + + let untracked = logical + .search_with_options( + &query, + SearchOptions::default().with_track_nulls(false), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + assert_eq!(untracked, SearchResult::exact(RowAddrTreeMap::default())); + let combined_bitmap = scalar_index_fragment_bitmap(&dataset, "value", "value_btree") .await .unwrap() @@ -1462,6 +1531,224 @@ mod tests { ); } + #[tokio::test] + async fn test_ngram_segment_merge_rebuilds_after_deferred_compaction() { + let test_dir = TempStrDir::default(); + let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "text", + arrow_schema::DataType::Utf8, + true, + )])); + let make_batch = |values| { + arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::StringArray::from(values))], + ) + .unwrap() + }; + let reader = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec![Some("alpha needle"), None]))], + schema.clone(), + ); + Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + let appended = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec![ + Some("beta needle"), + Some("gamma stack"), + ]))], + schema.clone(), + ); + let mut dataset = Dataset::write( + appended, + test_dir.as_str(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::NGram, ¶ms) + .name("text_ngram".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + let source_version = segments[0].dataset_version; + dataset + .create_index( + &["text"], + IndexType::NGram, + Some("compaction_guard".to_string()), + ¶ms, + false, + ) + .await + .unwrap(); + + let metrics = compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 10, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + assert!(metrics.fragments_removed > 0 && metrics.fragments_added > 0); + assert!(dataset.version().version > source_version); + + let direct_commit_err = dataset + .commit_existing_index_segments("text_ngram_direct", "text", segments.clone()) + .await + .unwrap_err(); + assert!( + direct_commit_err + .to_string() + .contains("must be rebuilt or merged") + ); + + let rebuild_version = dataset.version().version; + let merged = dataset + .merge_existing_index_segments(segments) + .await + .unwrap(); + assert_eq!(merged.dataset_version, rebuild_version); + dataset + .commit_existing_index_segments("text_ngram", "text", vec![merged]) + .await + .unwrap(); + + let committed = dataset.load_indices_by_name("text_ngram").await.unwrap(); + assert_eq!(committed.len(), 1); + assert_eq!(committed[0].dataset_version, rebuild_version); + assert_eq!( + committed[0].fragment_bitmap.as_ref().unwrap(), + dataset.fragment_bitmap.as_ref() + ); + + let logical = + open_named_scalar_index(&dataset, "text", "text_ngram", &NoOpMetricsCollector) + .await + .unwrap(); + let result = logical + .search( + &lance_index::scalar::TextQuery::StringContains("needle".to_string()), + &NoOpMetricsCollector, + ) + .await + .unwrap(); + let row_addrs = match result { + SearchResult::AtMost(row_addrs) => row_addrs, + other => panic!("expected AtMost result from ngram, got {other:?}"), + }; + assert_eq!(row_addrs.true_rows().row_addrs().unwrap().count(), 2); + } + + #[tokio::test] + async fn test_ngram_segment_merge_rejects_retired_coverage_without_remap() { + let test_dir = TempStrDir::default(); + let schema = Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "text", + arrow_schema::DataType::Utf8, + true, + )])); + let make_batch = |values| { + arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::StringArray::from(values))], + ) + .unwrap() + }; + let reader = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec!["alpha", "beta"]))], + schema.clone(), + ); + Dataset::write( + reader, + test_dir.as_str(), + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(); + let appended = arrow_array::RecordBatchIterator::new( + vec![Ok(make_batch(vec!["gamma", "delta"]))], + schema, + ); + let mut dataset = Dataset::write( + appended, + test_dir.as_str(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::NGram); + let mut segments = Vec::new(); + for fragment in dataset.get_fragments() { + segments.push( + CreateIndexBuilder::new(&mut dataset, &["text"], IndexType::NGram, ¶ms) + .name("text_ngram".to_string()) + .fragments(vec![fragment.id() as u32]) + .execute_uncommitted() + .await + .unwrap(), + ); + } + + compact_files( + &mut dataset, + CompactionOptions { + target_rows_per_fragment: 10, + defer_index_remap: true, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + + let direct_err = dataset + .commit_existing_index_segments("text_ngram", "text", segments.clone()) + .await + .unwrap_err(); + assert!(direct_err.to_string().contains("must be rebuilt or merged")); + + let merge_err = dataset + .merge_existing_index_segments(segments) + .await + .unwrap_err(); + assert!( + merge_err + .to_string() + .contains("no applicable fragment-reuse mapping is available") + ); + } + #[tokio::test] async fn test_fmindex_merge_single_segment_passthrough() { let test_dir = TempStrDir::default(); diff --git a/rust/lance/src/index/vector.rs b/rust/lance/src/index/vector.rs index c9920fb8adf..d9e749d44ad 100644 --- a/rust/lance/src/index/vector.rs +++ b/rust/lance/src/index/vector.rs @@ -8,6 +8,7 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::sync::Arc; use std::{any::Any, collections::HashMap}; +mod bounded_partition_stream; pub mod builder; pub(crate) mod details; pub mod hamming; @@ -19,14 +20,15 @@ pub mod utils; mod fixture_test; use self::{ivf::*, pq::PQIndex}; +use arrow_array::Array; use arrow_schema::{DataType, Schema}; use builder::{IvfIndexBuilder, VectorIndexBuildSummary}; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use futures::stream; use lance_core::utils::tempfile::TempStdDir; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_index::frag_reuse::FragReuseIndex; +use lance_file::versions::v1::reader::FileReader as V1FileReader; +use lance_index::frag_reuse::CompactFragReuseIndex; use lance_index::metrics::NoOpMetricsCollector; use lance_index::optimize::OptimizeOptions; use lance_index::progress::{IndexBuildProgress, noop_progress}; @@ -516,6 +518,7 @@ async fn prepare_vector_segment_build( progress: Arc, mode: &str, require_precomputed_ivf: bool, + fragment_ids: Option<&[u32]>, ) -> Result<(DataType, IndexType, IvfBuildParams, Box)> { let stages = ¶ms.stages; @@ -557,15 +560,29 @@ async fn prepare_vector_segment_build( validate_supported_rq_num_bits(rq_params.num_bits)?; } - let num_rows = dataset.count_rows(None).await?; - let num_partitions = ivf_params0.num_partitions.unwrap_or_else(|| { - recommended_num_partitions( - num_rows, - ivf_params0 - .target_partition_size - .unwrap_or(index_type.target_partition_size()), - ) - }); + let num_partitions = match (ivf_params0.num_partitions, ivf_params0.centroids.as_ref()) { + (Some(num_partitions), Some(centroids)) if num_partitions != centroids.len() => { + return Err(Error::index(format!( + "{mode}: num_partitions {} does not match precomputed IVF centroids length {}", + num_partitions, + centroids.len() + ))); + } + (Some(num_partitions), _) => num_partitions, + (None, Some(centroids)) => centroids.len(), + (None, None) => { + let num_rows = match fragment_ids { + Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await?, + None => dataset.count_rows(None).await?, + }; + recommended_num_partitions( + num_rows, + ivf_params0 + .target_partition_size + .unwrap_or(index_type.target_partition_size()), + ) + } + }; let mut ivf_params = ivf_params0.clone(); ivf_params.num_partitions = Some(num_partitions); @@ -591,7 +608,7 @@ pub(crate) async fn build_distributed_vector_index( _name: &str, uuid: Uuid, params: &VectorIndexParams, - frag_reuse_index: Option>, + frag_reuse_index: Option>, fragment_ids: &[u32], progress: Arc, ) -> Result<(Uuid, Vec)> { @@ -602,6 +619,7 @@ pub(crate) async fn build_distributed_vector_index( progress.clone(), "Build Distributed Vector Index", true, + Some(fragment_ids), ) .await?; let stages = ¶ms.stages; @@ -943,8 +961,57 @@ pub(crate) async fn build_vector_index( name: &str, uuid: Uuid, params: &VectorIndexParams, - frag_reuse_index: Option>, + frag_reuse_index: Option>, progress: Arc, +) -> Result> { + build_vector_index_impl( + dataset, + column, + name, + uuid, + params, + frag_reuse_index, + progress, + None, + ) + .await +} + +/// Build a standalone vector index segment over a subset of fragments. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn build_filtered_vector_index( + dataset: &Dataset, + column: &str, + name: &str, + uuid: Uuid, + params: &VectorIndexParams, + frag_reuse_index: Option>, + fragment_ids: &[u32], + progress: Arc, +) -> Result> { + build_vector_index_impl( + dataset, + column, + name, + uuid, + params, + frag_reuse_index, + progress, + Some(fragment_ids), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn build_vector_index_impl( + dataset: &Dataset, + column: &str, + name: &str, + uuid: Uuid, + params: &VectorIndexParams, + frag_reuse_index: Option>, + progress: Arc, + fragment_ids: Option<&[u32]>, ) -> Result> { let (element_type, index_type, ivf_params, shuffler) = prepare_vector_segment_build( dataset, @@ -953,6 +1020,7 @@ pub(crate) async fn build_vector_index( progress.clone(), "Build Vector Index", false, + fragment_ids, ) .await?; let stages = ¶ms.stages; @@ -971,10 +1039,11 @@ pub(crate) async fn build_vector_index( (), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } DataType::UInt8 => { let summary = IvfIndexBuilder::::new( @@ -988,17 +1057,16 @@ pub(crate) async fn build_vector_index( (), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); - } - _ => { - return Err(Error::index(format!( - "Build Vector Index: invalid data type: {:?}", - element_type - ))); + Ok(summary.files) } + _ => Err(Error::index(format!( + "Build Vector Index: invalid data type: {:?}", + element_type + ))), }, IndexType::IvfPq => { let len = stages.len(); @@ -1011,6 +1079,12 @@ pub(crate) async fn build_vector_index( match params.version { IndexFileVersion::Legacy => { + if fragment_ids.is_some() { + return Err(Error::index( + "Build Vector Index: filtered IVF_PQ builds do not support legacy format" + .to_string(), + )); + } let files = build_ivf_pq_index( dataset, column, @@ -1022,7 +1096,7 @@ pub(crate) async fn build_vector_index( progress.clone(), ) .await?; - return Ok(files); + Ok(files) } IndexFileVersion::V3 => { let mut builder = IvfIndexBuilder::::new( @@ -1039,10 +1113,11 @@ pub(crate) async fn build_vector_index( let summary = builder .with_transpose(!params.skip_transpose) + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } } } @@ -1065,10 +1140,11 @@ pub(crate) async fn build_vector_index( (), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } IndexType::IvfRq => { let StageParams::RQ(rq_params) = &stages[1] else { @@ -1092,10 +1168,11 @@ pub(crate) async fn build_vector_index( let summary = builder .with_transpose(!params.skip_transpose) + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } IndexType::IvfHnswFlat => { let StageParams::Hnsw(hnsw_params) = &stages[1] else { @@ -1117,10 +1194,11 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } _ => { let summary = IvfIndexBuilder::::new( @@ -1134,10 +1212,11 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } } } @@ -1165,10 +1244,11 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); + Ok(summary.files) } IndexType::IvfHnswSq => { let StageParams::Hnsw(hnsw_params) = &stages[1] else { @@ -1194,17 +1274,16 @@ pub(crate) async fn build_vector_index( hnsw_params.clone(), frag_reuse_index, )? + .with_optional_fragment_filter(fragment_ids) .with_progress(progress.clone()) .build() .await?; - return Ok(summary.files); - } - _ => { - return Err(Error::index(format!( - "Build Vector Index: invalid index type: {:?}", - index_type - ))); + Ok(summary.files) } + _ => Err(Error::index(format!( + "Build Vector Index: invalid index type: {:?}", + index_type + ))), } } @@ -1217,7 +1296,7 @@ pub(crate) async fn build_vector_index_incremental( uuid: Uuid, params: &VectorIndexParams, existing_index: Arc, - frag_reuse_index: Option>, + frag_reuse_index: Option>, progress: Arc, ) -> Result { let stages = ¶ms.stages; @@ -1539,7 +1618,7 @@ pub(crate) async fn open_vector_index( uuid: &Uuid, vec_idx: &lance_index::pb::VectorIndex, reader: Arc, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> Result> { let metric_type = pb::VectorMetricType::try_from(vec_idx.metric_type)?.into(); @@ -1633,8 +1712,8 @@ pub(crate) async fn open_vector_index_v2( dataset: Arc, column: &str, uuid: &Uuid, - reader: PreviousFileReader, - frag_reuse_index: Option>, + reader: V1FileReader, + frag_reuse_index: Option>, ) -> Result> { let index_metadata = reader .schema() @@ -1869,6 +1948,7 @@ pub async fn initialize_vector_index( uuid: new_uuid, name: source_index.name.clone(), fields: vec![field.id], + covering_fields: vec![], dataset_version: target_dataset.manifest.version, fragment_bitmap, index_details: source_index.index_details.clone(), @@ -1951,7 +2031,7 @@ fn derive_rabit_params(rabit_quantizer: &RabitQuantizer) -> RQBuildParams { /// Extract HNSW build parameters from the source vector index statistics. /// Returns default parameters if extraction fails. /// TODO: support consistently deriving all the original parameters -fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams { +pub(crate) fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams { let default_params = HnswBuildParams { max_level: 4, m: 20, @@ -1984,18 +2064,124 @@ fn derive_hnsw_params(source_index: &dyn VectorIndex) -> HnswBuildParams { .and_then(|v| v.as_u64()) .map(|v| v as usize) .unwrap_or(100); + let prefetch_distance = params + .get("prefetch_distance") + .and_then(|v| v.as_u64()) + .map(|v| v as usize); return HnswBuildParams { max_level, m, ef_construction, - prefetch_distance: None, + prefetch_distance, }; } default_params } +fn vector_index_type(index: &dyn VectorIndex) -> IndexType { + match index.sub_index_type() { + (SubIndexType::Flat, QuantizationType::Flat | QuantizationType::FlatBin) => { + IndexType::IvfFlat + } + (SubIndexType::Flat, QuantizationType::Product) => IndexType::IvfPq, + (SubIndexType::Flat, QuantizationType::Scalar) => IndexType::IvfSq, + (SubIndexType::Flat, QuantizationType::Rabit) => IndexType::IvfRq, + (SubIndexType::Hnsw, QuantizationType::Flat | QuantizationType::FlatBin) => { + IndexType::IvfHnswFlat + } + (SubIndexType::Hnsw, QuantizationType::Product) => IndexType::IvfHnswPq, + (SubIndexType::Hnsw, QuantizationType::Scalar) => IndexType::IvfHnswSq, + (SubIndexType::Hnsw, QuantizationType::Rabit) => IndexType::Vector, + } +} + +/// Derive structural build parameters for a new, independently trained segment. +/// +/// Learned IVF centroids, quantizer codebooks, and rotations are deliberately +/// omitted so the new segment is valid for its own fragment set. +pub(crate) fn fresh_vector_segment_params( + metadata: &IndexMetadata, + index: &dyn VectorIndex, +) -> Result { + if let Some(params) = metadata + .index_details + .as_deref() + .and_then(details::vector_params_from_details) + && params.metric_type == index.metric_type() + && params.index_type() == vector_index_type(index) + { + return Ok(params); + } + + let mut ivf_params = derive_ivf_params(index.ivf_model()); + ivf_params.centroids = None; + #[allow(deprecated)] + { + ivf_params.retrain = false; + } + + let metric_type = index.metric_type(); + let quantizer = index.quantizer(); + Ok(match index.sub_index_type() { + (SubIndexType::Flat, QuantizationType::Flat | QuantizationType::FlatBin) => { + VectorIndexParams::with_ivf_flat_params(metric_type, ivf_params) + } + (SubIndexType::Flat, QuantizationType::Product) => { + let quantizer: ProductQuantizer = quantizer.try_into()?; + let mut pq_params = derive_pq_params(&quantizer); + pq_params.codebook = None; + VectorIndexParams::with_ivf_pq_params(metric_type, ivf_params, pq_params) + } + (SubIndexType::Flat, QuantizationType::Scalar) => { + let quantizer: ScalarQuantizer = quantizer.try_into()?; + VectorIndexParams::with_ivf_sq_params( + metric_type, + ivf_params, + derive_sq_params(&quantizer), + ) + } + (SubIndexType::Flat, QuantizationType::Rabit) => { + let quantizer: RabitQuantizer = quantizer.try_into()?; + VectorIndexParams::with_ivf_rq_params( + metric_type, + ivf_params, + derive_rabit_params(&quantizer), + ) + } + (SubIndexType::Hnsw, QuantizationType::Flat | QuantizationType::FlatBin) => { + VectorIndexParams::ivf_hnsw(metric_type, ivf_params, derive_hnsw_params(index)) + } + (SubIndexType::Hnsw, QuantizationType::Product) => { + let quantizer: ProductQuantizer = quantizer.try_into()?; + let mut pq_params = derive_pq_params(&quantizer); + pq_params.codebook = None; + VectorIndexParams::with_ivf_hnsw_pq_params( + metric_type, + ivf_params, + derive_hnsw_params(index), + pq_params, + ) + } + (SubIndexType::Hnsw, QuantizationType::Scalar) => { + let quantizer: ScalarQuantizer = quantizer.try_into()?; + VectorIndexParams::with_ivf_hnsw_sq_params( + metric_type, + ivf_params, + derive_hnsw_params(index), + derive_sq_params(&quantizer), + ) + } + (SubIndexType::Hnsw, QuantizationType::Rabit) => { + return Err(Error::index( + "Cannot build a fresh IVF_HNSW_RQ segment: this index type is unsupported" + .to_string(), + )); + } + }) +} + #[cfg(test)] mod tests { use super::*; @@ -2079,25 +2265,27 @@ mod tests { let uri = format!("{}/ds", test_dir.as_str()); let reader = lance_datagen::gen_batch() - .col("vector", array::rand_vec::(32.into())) - .into_reader_rows(RowCount::from(400), BatchCount::from(1)); + .col("vector", array::rand_vec::(8.into())) + .into_reader_rows(RowCount::from(64), BatchCount::from(1)); let mut dataset = Dataset::write(reader, &uri, None).await.unwrap(); let params = VectorIndexParams::with_ivf_hnsw_pq_params( MetricType::L2, IvfBuildParams { - num_partitions: Some(8), + num_partitions: Some(2), + max_iters: 2, + sample_rate: 2, ..Default::default() }, - HnswBuildParams { - max_level: 6, - m: 24, - ef_construction: 120, - prefetch_distance: None, - }, + HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16), PQBuildParams { - num_sub_vectors: 8, - num_bits: 8, + num_sub_vectors: 2, + num_bits: 4, + max_iters: 2, + sample_rate: 2, ..Default::default() }, ); @@ -2747,7 +2935,8 @@ mod tests { .await .unwrap(); let arrow_schema = ArrowSchema::new(vec![Field::new("dummy", ArrowDataType::Int32, true)]); - let mut v2w = lance_file::writer::FileWriter::try_new( + let mut v2w = lance_file::versions::create_writer( + dataset_format_version(&dataset), writer, lance_core::datatypes::Schema::try_from(&arrow_schema).unwrap(), FileWriterOptions::default(), @@ -3220,29 +3409,33 @@ mod tests { let source_uri = format!("{}/source", test_dir.as_str()); let target_uri = format!("{}/target", test_dir.as_str()); - // Create source dataset with vector column (need at least 256 rows for PQ training) + // A 4-bit PQ codebook needs at least 16 training rows. let source_reader = lance_datagen::gen_batch() .col("id", array::step::()) - .col("vector", array::rand_vec::(32.into())) - .into_reader_rows(RowCount::from(400), BatchCount::from(1)); + .col("vector", array::rand_vec::(8.into())) + .into_reader_rows(RowCount::from(64), BatchCount::from(1)); let mut source_dataset = Dataset::write(source_reader, &source_uri, None) .await .unwrap(); // Create IVF_HNSW_PQ index on source with custom HNSW parameters let ivf_params = IvfBuildParams { - num_partitions: Some(8), + num_partitions: Some(2), + max_iters: 2, + sample_rate: 2, ..Default::default() }; let hnsw_params = HnswBuildParams { - max_level: 6, - m: 24, - ef_construction: 120, + max_level: 2, + m: 4, + ef_construction: 16, prefetch_distance: None, }; let pq_params = PQBuildParams { - num_sub_vectors: 8, - num_bits: 8, + num_sub_vectors: 2, + num_bits: 4, + max_iters: 2, + sample_rate: 2, ..Default::default() }; let params = VectorIndexParams::with_ivf_hnsw_pq_params( @@ -3274,8 +3467,8 @@ mod tests { // Create target dataset with same schema let target_reader = lance_datagen::gen_batch() .col("id", array::step::()) - .col("vector", array::rand_vec::(32.into())) - .into_reader_rows(RowCount::from(100), BatchCount::from(1)); + .col("vector", array::rand_vec::(8.into())) + .into_reader_rows(RowCount::from(32), BatchCount::from(1)); let mut target_dataset = Dataset::write(target_reader, &target_uri, None) .await .unwrap(); @@ -3322,8 +3515,8 @@ mod tests { // Check number of partitions assert_eq!( stats.get("num_partitions").and_then(|v| v.as_u64()), - Some(8), - "Should have 8 partitions" + Some(2), + "Should have 2 partitions" ); // Verify centroids are shared between source and target indices @@ -3384,13 +3577,13 @@ mod tests { // Verify PQ parameters assert_eq!( sub_index.get("nbits").and_then(|v| v.as_u64()), - Some(8), - "PQ should use 8 bits" + Some(4), + "PQ should use 4 bits" ); assert_eq!( sub_index.get("num_sub_vectors").and_then(|v| v.as_u64()), - Some(8), - "PQ should have 8 sub vectors" + Some(2), + "PQ should have 2 sub vectors" ); // Verify IVF parameters are correctly derived @@ -3402,8 +3595,8 @@ mod tests { ); assert_eq!( target_ivf_params.num_partitions, - Some(8), - "Should have 8 partitions as configured" + Some(2), + "Should have 2 partitions as configured" ); // Verify PQ parameters are correctly derived @@ -3424,29 +3617,29 @@ mod tests { "PQ num_bits should match" ); assert_eq!( - target_pq_params.num_sub_vectors, 8, - "PQ should have 8 sub vectors" + target_pq_params.num_sub_vectors, 2, + "PQ should have 2 sub vectors" ); - assert_eq!(target_pq_params.num_bits, 8, "PQ should use 8 bits"); + assert_eq!(target_pq_params.num_bits, 4, "PQ should use 4 bits"); // Verify HNSW parameters are extracted and used correctly let derived_hnsw_params = derive_hnsw_params(target_vector_index.as_ref()); assert_eq!( - derived_hnsw_params.max_level, 6, - "HNSW max_level should be extracted as 6 from source index" + derived_hnsw_params.max_level, 2, + "HNSW max_level should be extracted as 2 from source index" ); assert_eq!( - derived_hnsw_params.m, 24, - "HNSW m should be extracted as 24 from source index" + derived_hnsw_params.m, 4, + "HNSW m should be extracted as 4 from source index" ); assert_eq!( - derived_hnsw_params.ef_construction, 120, - "HNSW ef_construction should be extracted as 120 from source index" + derived_hnsw_params.ef_construction, 16, + "HNSW ef_construction should be extracted as 16 from source index" ); // Verify the index is functional let query_vector = lance_datagen::gen_batch() - .anon_col(array::rand_vec::(32.into())) + .anon_col(array::rand_vec::(8.into())) .into_batch_rows(RowCount::from(1)) .unwrap() .column(0) diff --git a/rust/lance/src/index/vector/bounded_partition_stream.rs b/rust/lance/src/index/vector/bounded_partition_stream.rs new file mode 100644 index 00000000000..150a4d72043 --- /dev/null +++ b/rust/lance/src/index/vector/bounded_partition_stream.rs @@ -0,0 +1,693 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::BTreeMap; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::task::{Context, Poll}; + +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::task::AtomicWaker; +use futures::{Stream, StreamExt}; +use lance_core::{Error, Result}; +use tokio::sync::OwnedSemaphorePermit; + +/// Work admitted to [`BoundedPartitionStream`]. +type WeightedJobStarter = Box< + dyn FnOnce(AdmissionPermit) -> BoxFuture<'static, Result<(T, AdmissionPermit)>> + + Send + + 'static, +>; + +pub(super) struct WeightedJob { + weight_bytes: usize, + start: WeightedJobStarter, +} + +impl WeightedJob { + #[cfg(test)] + pub(super) fn new( + weight_bytes: usize, + future: impl Future> + Send + 'static, + ) -> Self { + Self { + weight_bytes, + start: Box::new(move |permit| { + Box::pin(async move { future.await.map(|value| (value, permit)) }) + }), + } + } + + pub(super) fn with_permit(weight_bytes: usize, start: F) -> Self + where + F: FnOnce(AdmissionPermit) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + Self { + weight_bytes, + start: Box::new(move |permit| Box::pin(start(permit))), + } + } +} + +/// A completed job whose admission charge remains held until the result is dropped. +/// +/// Keeping the permit with a result bounds both active builds and results waiting for +/// an earlier partition to finish. +pub(super) struct Budgeted { + pub(super) value: T, + pub(super) permit: Option>, + pub(super) entry_permit: Option, +} + +impl Budgeted { + pub(super) fn untracked(value: T) -> Self { + Self { + value, + permit: None, + entry_permit: None, + } + } +} + +/// Buffers out-of-order partition results and exposes only the next id to write. +pub(super) struct OrderedPartitionResults { + next_partition_id: usize, + num_partitions: usize, + pending: BTreeMap, +} + +impl OrderedPartitionResults { + pub(super) fn new(num_partitions: usize) -> Self { + Self { + next_partition_id: 0, + num_partitions, + pending: BTreeMap::new(), + } + } + + pub(super) fn push(&mut self, partition_id: usize, value: T) -> Result<()> { + if partition_id >= self.num_partitions { + return Err(Error::internal(format!( + "partition build returned out-of-range partition id {} for {} partitions", + partition_id, self.num_partitions + ))); + } + if partition_id < self.next_partition_id + || self.pending.insert(partition_id, value).is_some() + { + return Err(Error::internal(format!( + "partition build returned duplicate partition id {}", + partition_id + ))); + } + Ok(()) + } + + pub(super) fn pop_next(&mut self) -> Option<(usize, T)> { + let partition_id = self.next_partition_id; + let value = self.pending.remove(&partition_id)?; + self.next_partition_id += 1; + Some((partition_id, value)) + } + + pub(super) fn finish(&self) -> Result<()> { + if self.next_partition_id != self.num_partitions { + return Err(Error::internal(format!( + "partition build stream ended before partition {} of {}; buffered partition ids: {:?}", + self.next_partition_id, + self.num_partitions, + self.pending.keys().copied().collect::>() + ))); + } + Ok(()) + } +} + +pub(super) struct AdmissionPermit { + budget: Arc, + charged_bytes: usize, +} + +impl AdmissionPermit { + /// Reconcile the pre-decode admission charge to the materialized size. + /// + /// Oversized values remain charged at the cap. Estimates are conservative, + /// so this normally releases capacity; an underestimate is still reflected + /// in the budget to prevent admitting additional work against stale usage. + pub(super) fn reconcile(&mut self, actual_bytes: usize) { + let charged_bytes = actual_bytes.min(self.budget.max_bytes); + match charged_bytes.cmp(&self.charged_bytes) { + std::cmp::Ordering::Less => { + self.budget + .current_bytes + .fetch_sub(self.charged_bytes - charged_bytes, Ordering::AcqRel); + } + std::cmp::Ordering::Greater => { + let additional_bytes = charged_bytes - self.charged_bytes; + let current_bytes = self + .budget + .current_bytes + .fetch_add(additional_bytes, Ordering::AcqRel) + + additional_bytes; + #[cfg(not(test))] + let _ = current_bytes; + #[cfg(test)] + self.budget + .peak_bytes + .fetch_max(current_bytes, Ordering::AcqRel); + } + std::cmp::Ordering::Equal => {} + } + self.charged_bytes = charged_bytes; + self.budget.waker.wake(); + } +} + +impl Drop for AdmissionPermit { + fn drop(&mut self) { + self.budget + .current_bytes + .fetch_sub(self.charged_bytes, Ordering::AcqRel); + self.budget.current_entries.fetch_sub(1, Ordering::AcqRel); + self.budget.waker.wake(); + } +} + +struct Budget { + max_bytes: usize, + max_entries: usize, + current_bytes: AtomicUsize, + current_entries: AtomicUsize, + waker: AtomicWaker, + #[cfg(test)] + peak_bytes: AtomicUsize, + #[cfg(test)] + peak_entries: AtomicUsize, +} + +impl Budget { + fn can_admit(&self, weight_bytes: usize) -> bool { + let current_bytes = self.current_bytes.load(Ordering::Acquire); + let current_entries = self.current_entries.load(Ordering::Acquire); + if current_entries >= self.max_entries { + return false; + } + if weight_bytes > self.max_bytes { + // An oversized (hotspot) partition is charged at the cap and must run + // alone. It cannot strand the oldest partition behind later work. + return current_entries == 0; + } + current_bytes + .checked_add(weight_bytes) + .is_some_and(|total| total <= self.max_bytes) + } + + fn admit(self: &Arc, weight_bytes: usize) -> AdmissionPermit { + let charged_bytes = weight_bytes.min(self.max_bytes); + let current_bytes = self + .current_bytes + .fetch_add(charged_bytes, Ordering::AcqRel) + + charged_bytes; + let current_entries = self.current_entries.fetch_add(1, Ordering::AcqRel) + 1; + #[cfg(not(test))] + let _ = (current_bytes, current_entries); + #[cfg(test)] + { + self.peak_bytes.fetch_max(current_bytes, Ordering::AcqRel); + self.peak_entries + .fetch_max(current_entries, Ordering::AcqRel); + } + AdmissionPermit { + budget: self.clone(), + charged_bytes, + } + } +} + +/// Runs partition jobs out of order while bounding active and completed work. +/// +/// The input is polled in partition order. A job is admitted only when both its +/// byte charge and the total number of active/completed entries fit. Oversized +/// jobs are charged at the byte cap and admitted only when the budget is empty. +pub(super) struct BoundedPartitionStream { + input: S, + pending: Option>, + in_flight: FuturesUnordered>>>, + budget: Arc, + max_concurrency: usize, + is_input_done: bool, + is_failed: bool, +} + +impl BoundedPartitionStream +where + S: Stream>> + Unpin, +{ + pub(super) fn try_new( + input: S, + max_concurrency: usize, + max_bytes: usize, + max_entries: usize, + ) -> Result { + if max_concurrency == 0 || max_bytes == 0 || max_entries == 0 { + return Err(Error::invalid_input(format!( + "bounded partition stream limits must be non-zero: max_concurrency={}, max_bytes={}, max_entries={}", + max_concurrency, max_bytes, max_entries + ))); + } + Ok(Self { + input, + pending: None, + in_flight: FuturesUnordered::new(), + budget: Arc::new(Budget { + max_bytes, + max_entries, + current_bytes: AtomicUsize::new(0), + current_entries: AtomicUsize::new(0), + waker: AtomicWaker::new(), + #[cfg(test)] + peak_bytes: AtomicUsize::new(0), + #[cfg(test)] + peak_entries: AtomicUsize::new(0), + }), + max_concurrency, + is_input_done: false, + is_failed: false, + }) + } + + fn fail(&mut self) { + self.is_failed = true; + self.pending = None; + self.in_flight = FuturesUnordered::new(); + } + + #[cfg(test)] + fn stats(&self) -> (usize, usize, usize, usize) { + ( + self.budget.current_bytes.load(Ordering::Acquire), + self.budget.peak_bytes.load(Ordering::Acquire), + self.budget.current_entries.load(Ordering::Acquire), + self.budget.peak_entries.load(Ordering::Acquire), + ) + } +} + +impl Stream for BoundedPartitionStream +where + S: Stream>> + Unpin, +{ + type Item = Result>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.is_failed { + return Poll::Ready(None); + } + self.budget.waker.register(cx.waker()); + + loop { + if self.in_flight.len() >= self.max_concurrency { + break; + } + + if let Some(job) = self.pending.take() { + if !self.budget.can_admit(job.weight_bytes) { + self.pending = Some(job); + break; + } + let permit = self.budget.admit(job.weight_bytes); + let future = (job.start)(permit); + self.in_flight.push(Box::pin(async move { + future.await.map(|(value, permit)| Budgeted { + value, + permit: Some(Arc::new(permit)), + entry_permit: None, + }) + })); + continue; + } + + if self.is_input_done { + break; + } + match Pin::new(&mut self.input).poll_next(cx) { + Poll::Ready(Some(Ok(job))) => self.pending = Some(job), + Poll::Ready(Some(Err(error))) => { + self.fail(); + return Poll::Ready(Some(Err(error))); + } + Poll::Ready(None) => self.is_input_done = true, + Poll::Pending => break, + } + } + + match self.in_flight.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(output))) => Poll::Ready(Some(Ok(output))), + Poll::Ready(Some(Err(error))) => { + self.fail(); + Poll::Ready(Some(Err(error))) + } + Poll::Ready(None) if self.is_input_done && self.pending.is_none() => Poll::Ready(None), + Poll::Ready(None) | Poll::Pending => Poll::Pending, + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; + + use futures::stream; + use futures::{StreamExt, TryStreamExt}; + + use super::*; + + #[tokio::test] + async fn slow_head_does_not_block_later_jobs() { + let jobs = (0..4).map(|partition_id| { + Ok(WeightedJob::new(1, async move { + tokio::time::sleep(Duration::from_millis(if partition_id == 0 { + 40 + } else { + 1 + })) + .await; + Ok(partition_id) + })) + }); + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 4, 4, 4).unwrap(); + let first = output.next().await.unwrap().unwrap(); + assert_ne!(first.value, 0); + let mut completed = vec![first.value]; + completed.extend( + output + .map(|result| result.unwrap().value) + .collect::>() + .await, + ); + completed.sort_unstable(); + assert_eq!(completed, vec![0, 1, 2, 3]); + } + + #[tokio::test] + async fn byte_and_entry_caps_include_completed_results() { + let jobs = + (0..5).map(|partition_id| Ok(WeightedJob::new(3, async move { Ok(partition_id) }))); + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 5, 6, 2).unwrap(); + let first = output.next().await.unwrap().unwrap(); + let second = output.next().await.unwrap().unwrap(); + let (_, peak_bytes, current_entries, peak_entries) = output.stats(); + assert_eq!(peak_bytes, 6); + assert_eq!(current_entries, 2); + assert_eq!(peak_entries, 2); + drop((first, second)); + let mut remaining = 0; + while let Some(result) = output.next().await { + drop(result.unwrap()); + remaining += 1; + } + assert_eq!(remaining, 3); + let (current_bytes, peak_bytes, current_entries, peak_entries) = output.stats(); + assert_eq!(current_bytes, 0); + assert_eq!(peak_bytes, 6); + assert_eq!(current_entries, 0); + assert_eq!(peak_entries, 2); + } + + #[tokio::test] + async fn oversized_job_is_exclusive() { + let jobs = vec![ + Ok(WeightedJob::new(2, async { Ok(0) })), + Ok(WeightedJob::new(20, async { Ok(1) })), + Ok(WeightedJob::new(2, async { Ok(2) })), + ]; + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 3, 8, 3).unwrap(); + let first = output.next().await.unwrap().unwrap(); + assert_eq!(first.value, 0); + drop(first); + let oversized = output.next().await.unwrap().unwrap(); + assert_eq!(oversized.value, 1); + let (current_bytes, peak_bytes, current_entries, _) = output.stats(); + assert_eq!(current_bytes, 8); + assert_eq!(peak_bytes, 8); + assert_eq!(current_entries, 1); + drop(oversized); + assert_eq!(output.next().await.unwrap().unwrap().value, 2); + } + + #[tokio::test] + async fn oversized_materialization_starts_only_after_exclusive_admission() { + let oversized_materialized = Arc::new(AtomicBool::new(false)); + let marker = oversized_materialized.clone(); + let jobs = vec![ + Ok(WeightedJob::new(2, async { Ok(0) })), + Ok(WeightedJob::with_permit( + 20, + move |mut admission| async move { + marker.store(true, Ordering::Release); + admission.reconcile(20); + Ok((1, admission)) + }, + )), + ]; + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 2, 8, 2).unwrap(); + + let first = output.next().await.unwrap().unwrap(); + assert_eq!(first.value, 0); + assert!(!oversized_materialized.load(Ordering::Acquire)); + + drop(first); + let oversized = output.next().await.unwrap().unwrap(); + assert_eq!(oversized.value, 1); + assert!(oversized_materialized.load(Ordering::Acquire)); + let (current_bytes, peak_bytes, current_entries, _) = output.stats(); + assert_eq!(current_bytes, 8); + assert_eq!(peak_bytes, 8); + assert_eq!(current_entries, 1); + } + + #[tokio::test] + async fn actual_size_reconciliation_releases_admission_capacity() { + let jobs = vec![ + Ok(WeightedJob::with_permit(8, |mut admission| async move { + admission.reconcile(2); + Ok((0, admission)) + })), + Ok(WeightedJob::new(6, async { Ok(1) })), + ]; + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 2, 8, 2).unwrap(); + + let first = output.next().await.unwrap().unwrap(); + assert_eq!(first.value, 0); + let (current_bytes, peak_bytes, current_entries, peak_entries) = output.stats(); + assert_eq!(current_bytes, 2); + assert_eq!(peak_bytes, 8); + assert_eq!(current_entries, 1); + assert_eq!(peak_entries, 1); + + let second = output.next().await.unwrap().unwrap(); + assert_eq!(second.value, 1); + let (current_bytes, _, current_entries, peak_entries) = output.stats(); + assert_eq!(current_bytes, 8); + assert_eq!(current_entries, 2); + assert_eq!(peak_entries, 2); + drop((first, second)); + } + + #[tokio::test] + async fn expanded_window_results_respect_partition_entry_cap() { + struct ResidentResult(usize, Arc); + + impl ResidentResult { + fn new(value: usize, resident: Arc) -> Self { + resident.fetch_add(1, Ordering::AcqRel); + Self(value, resident) + } + } + + impl Drop for ResidentResult { + fn drop(&mut self) { + self.1.fetch_sub(1, Ordering::AcqRel); + } + } + + let resident = Arc::new(AtomicUsize::new(0)); + let partition_entries = Arc::new(tokio::sync::Semaphore::new(1)); + let resident_for_job = resident.clone(); + let entries_for_job = partition_entries.clone(); + let jobs = vec![Ok(WeightedJob::with_permit( + 1, + move |admission| async move { + let builds = stream::iter((0..8).map(move |value| { + let resident = resident_for_job.clone(); + let partition_entries = entries_for_job.clone(); + async move { + let entry_permit = partition_entries.acquire_owned().await.unwrap(); + Ok::<_, Error>((ResidentResult::new(value, resident), entry_permit)) + } + })) + .buffer_unordered(8) + .boxed(); + Ok((builds, admission)) + }, + ))]; + let windows = BoundedPartitionStream::try_new(stream::iter(jobs), 1, 1, 1).unwrap(); + let mut output = windows + .map_ok(|window| { + let Budgeted { + value: builds, + permit, + entry_permit, + } = window; + assert!(entry_permit.is_none()); + builds.map_ok(move |(value, entry_permit)| Budgeted { + value, + permit: permit.clone(), + entry_permit: Some(entry_permit), + }) + }) + .try_flatten_unordered(Some(1)) + .boxed(); + + let first = output.next().await.unwrap().unwrap(); + assert!(first.value.0 < 8); + assert_eq!(resident.load(Ordering::Acquire), 1); + assert!( + tokio::time::timeout(Duration::from_millis(10), output.next()) + .await + .is_err() + ); + + drop(first); + while let Some(result) = output.next().await { + let result = result.unwrap(); + assert_eq!(resident.load(Ordering::Acquire), 1); + drop(result); + } + assert_eq!(resident.load(Ordering::Acquire), 0); + } + + #[tokio::test] + async fn dropping_a_held_result_wakes_budget_blocked_stream() { + let jobs = vec![ + Ok(WeightedJob::new(1, async { Ok(0) })), + Ok(WeightedJob::new(1, async { Ok(1) })), + ]; + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 2, 1, 1).unwrap(); + let first = output.next().await.unwrap().unwrap(); + let next = output.next(); + tokio::pin!(next); + assert!( + tokio::time::timeout(Duration::from_millis(10), &mut next) + .await + .is_err() + ); + drop(first); + let second = tokio::time::timeout(Duration::from_millis(100), &mut next) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(second.value, 1); + } + + #[tokio::test] + async fn error_drops_pending_work() { + let was_dropped = Arc::new(AtomicBool::new(false)); + struct DropFlag(Arc); + impl Drop for DropFlag { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + + let guard = DropFlag(was_dropped.clone()); + let jobs = vec![ + Ok(WeightedJob::new(1, async { + Err::(Error::internal("build failed")) + })), + Ok(WeightedJob::new(1, async move { + let _guard = guard; + futures::future::pending::<()>().await; + Ok(1) + })), + ]; + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 2, 2, 2).unwrap(); + let Err(error) = output.next().await.unwrap() else { + panic!("expected build failure"); + }; + assert!(error.to_string().contains("build failed")); + assert!(was_dropped.load(Ordering::Acquire)); + assert!(output.next().await.is_none()); + } + + #[tokio::test] + async fn dropping_stream_cancels_in_flight_work() { + let was_dropped = Arc::new(AtomicBool::new(false)); + struct DropFlag(Arc); + impl Drop for DropFlag { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + + let guard = DropFlag(was_dropped.clone()); + let jobs = vec![Ok(WeightedJob::new(1, async move { + let _guard = guard; + futures::future::pending::<()>().await; + Ok(0) + }))]; + let mut output = BoundedPartitionStream::try_new(stream::iter(jobs), 1, 1, 1).unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(10), output.next()) + .await + .is_err() + ); + drop(output); + assert!(was_dropped.load(Ordering::Acquire)); + } + + #[test] + fn ordered_results_drain_in_partition_order_and_include_empty_values() { + let mut results = OrderedPartitionResults::new(4); + results.push(2, Some(2)).unwrap(); + assert!(results.pop_next().is_none()); + results.push(0, Some(0)).unwrap(); + assert_eq!(results.pop_next(), Some((0, Some(0)))); + results.push(1, None).unwrap(); + assert_eq!(results.pop_next(), Some((1, None))); + assert_eq!(results.pop_next(), Some((2, Some(2)))); + results.push(3, Some(3)).unwrap(); + assert_eq!(results.pop_next(), Some((3, Some(3)))); + results.finish().unwrap(); + } + + #[test] + fn ordered_results_reject_duplicate_out_of_range_and_missing() { + let mut pending_duplicate = OrderedPartitionResults::new(2); + pending_duplicate.push(1, 1).unwrap(); + let error = pending_duplicate.push(1, 1).unwrap_err(); + assert!(error.to_string().contains("duplicate partition id 1")); + + let mut written_duplicate = OrderedPartitionResults::new(2); + written_duplicate.push(0, 0).unwrap(); + assert_eq!(written_duplicate.pop_next(), Some((0, 0))); + let error = written_duplicate.push(0, 0).unwrap_err(); + assert!(error.to_string().contains("duplicate partition id 0")); + + let mut out_of_range = OrderedPartitionResults::new(2); + let error = out_of_range.push(2, 2).unwrap_err(); + assert!(error.to_string().contains("out-of-range partition id 2")); + + let mut missing = OrderedPartitionResults::new(3); + missing.push(1, 1).unwrap(); + let error = missing.finish().unwrap_err(); + assert!(error.to_string().contains("ended before partition 0 of 3")); + assert!(error.to_string().contains("[1]")); + } +} diff --git a/rust/lance/src/index/vector/builder.rs b/rust/lance/src/index/vector/builder.rs index bf968d9744f..3a65d4776d1 100644 --- a/rust/lance/src/index/vector/builder.rs +++ b/rust/lance/src/index/vector/builder.rs @@ -27,12 +27,14 @@ use lance_core::datatypes::Schema; use lance_core::utils::tempfile::TempStdDir; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; use lance_core::{Error, ROW_ID_FIELD, Result}; -use lance_encoding::version::LanceFileVersion; -use lance_file::writer::{FileWriter, FileWriterOptions}; -use lance_index::frag_reuse::FragReuseIndex; +use lance_file::version::ConcreteFileVersion; +use lance_file::versions as file_versions; +use lance_file::writer::FileWriterOptions; +use lance_index::frag_reuse::{CompactFragReuseIndex, CompactFragReuseIndexHandle}; use lance_index::metrics::NoOpMetricsCollector; use lance_index::optimize::OptimizeOptions; use lance_index::progress::{IndexBuildProgress, NoopIndexBuildProgress}; +use lance_index::scalar::RowIdRemapper; use lance_index::vector::bq::storage::{RABIT_CODE_COLUMN, unpack_codes}; use lance_index::vector::kmeans::KMeansParams; use lance_index::vector::pq::storage::transpose; @@ -43,7 +45,9 @@ use lance_index::vector::quantizer::{QuantizerMetadata, QuantizerStorage}; use lance_index::vector::shared::{SupportedIvfIndexType, write_unified_ivf_and_index_metadata}; use lance_index::vector::storage::STORAGE_METADATA_KEY; use lance_index::vector::transform::Flatten; -use lance_index::vector::v3::shuffler::{EmptyReader, IvfShufflerReader, create_ivf_shuffler}; +use lance_index::vector::v3::shuffler::{ + DEFAULT_PARTITION_WINDOW_BYTES, EmptyReader, IvfShufflerReader, create_ivf_shuffler, +}; use lance_index::vector::v3::subindex::SubIndexType; use lance_index::vector::{LOSS_METADATA_KEY, PART_ID_COLUMN, PQ_CODE_COLUMN, VectorIndex}; use lance_index::vector::{PART_ID_FIELD, ivf::storage::IvfModel}; @@ -63,7 +67,7 @@ use lance_index::{ }; use lance_index::{ INDEX_METADATA_SCHEMA_KEY, IndexMetadata, IndexType, MAX_PARTITION_SIZE_FACTOR, - MIN_PARTITION_SIZE_PERCENT, + MIN_PARTITION_SIZE_PERCENT, scalar::OldIndexDataFilter, }; use lance_io::local::to_local_path; use lance_io::stream::RecordBatchStream; @@ -74,12 +78,17 @@ use lance_table::format::IndexFile; use log::info; use object_store::path::Path; use prost::Message; +use roaring::RoaringBitmap; +use tokio::sync::{OnceCell, OwnedSemaphorePermit, Semaphore}; use tracing::{Level, instrument, span}; use crate::Dataset; use crate::dataset::ProjectionRequest; use crate::dataset::index::dataset_format_version; -use crate::index::vector::ivf::v2::PartitionEntry; +use crate::index::append::build_old_data_filter; +use crate::index::vector::bounded_partition_stream::{ + BoundedPartitionStream, Budgeted, OrderedPartitionResults, WeightedJob, +}; use crate::index::vector::utils::infer_vector_dim; use super::v2::IVFIndex; @@ -92,6 +101,26 @@ use super::{ const REASSIGN_RANGE: usize = 64; // sample size for kmeans training when splitting a partition (sample_rate * k = 256 * 2) const SPLIT_SAMPLE_SIZE: usize = 512; +/// Maximum decoded input bytes admitted across active builds and completed +/// partitions waiting for their turn to be written. +const PARTITION_BUILD_BUDGET_BYTES: usize = 512 * 1024 * 1024; +/// Bound ready-map overhead even when many consecutive partitions are empty. +const PARTITION_BUILD_ENTRIES_PER_WORKER: usize = 2; + +#[derive(Debug, Clone, Copy)] +struct FreshPartitionBuildLimits { + window_bytes: usize, + decoded_budget_bytes: usize, +} + +impl Default for FreshPartitionBuildLimits { + fn default() -> Self { + Self { + window_bytes: DEFAULT_PARTITION_WINDOW_BYTES, + decoded_budget_bytes: PARTITION_BUILD_BUDGET_BYTES, + } + } +} /// Build a new centroid array that incorporates the results of partition splits. /// @@ -118,6 +147,89 @@ fn apply_centroid_splits( )?) } +/// An index segment an optimize pass reads existing rows from, paired with the rows +/// that segment is still allowed to contribute. +/// +/// A segment's index file keeps every row it was built with, but the rows it may still +/// contribute shrink afterwards: an update rewrites a row and either prunes its +/// fragment from the segment's bitmap (in-place column rewrite) or deletion-marks the +/// old physical row while the rewritten copy reuses its stable row id. Copying such a +/// row into a merged segment would duplicate it, because the merged coverage also spans +/// the fresh copy and nothing downstream can tell the two apart. +/// +/// The filter is resolved on first use rather than up front: under stable row ids +/// building it loads every covered fragment's row-id sequence, and the common optimize +/// pass appends a delta without reading a single existing row. +#[derive(Clone)] +pub struct ExistingIndex { + pub index: Arc, + coverage: Option>, +} + +/// The inputs [`ExistingIndex::old_data_filter`] needs, plus the filter once built. +struct SegmentCoverage { + dataset: Dataset, + effective_frags: RoaringBitmap, + deleted_frags: RoaringBitmap, + filter: OnceCell>, +} + +impl ExistingIndex { + /// An existing index whose rows are all still valid. + pub fn unfiltered(index: Arc) -> Self { + Self { + index, + coverage: None, + } + } + + /// An existing index that may only contribute rows still live in `effective_frags`. + pub fn with_coverage( + index: Arc, + dataset: Dataset, + effective_frags: RoaringBitmap, + deleted_frags: RoaringBitmap, + ) -> Self { + Self { + index, + coverage: Some(Arc::new(SegmentCoverage { + dataset, + effective_frags, + deleted_frags, + filter: OnceCell::new(), + })), + } + } + + /// Whether the filter has already been built. A pass that reads no existing rows + /// must never pay for one. + #[cfg(test)] + pub(crate) fn filter_is_built(&self) -> bool { + self.coverage + .as_deref() + .is_some_and(|coverage| coverage.filter.initialized()) + } + + /// The filter to apply to this segment's stored rows, or `None` when every row it + /// holds is still valid. Built once and shared by all partitions. + pub(crate) async fn old_data_filter(&self) -> Result> { + let Some(coverage) = self.coverage.as_deref() else { + return Ok(None); + }; + let filter = coverage + .filter + .get_or_try_init(|| { + build_old_data_filter( + &coverage.dataset, + &coverage.effective_frags, + &coverage.deleted_frags, + ) + }) + .await?; + Ok(filter.as_ref()) + } +} + // Builder for IVF index // The builder will train the IVF model and quantizer, shuffle the dataset, and build the sub index // for each partition. @@ -148,9 +260,9 @@ pub struct IvfIndexBuilder { shuffle_data_input: Mutex>, // fields for merging indices / remapping - existing_indices: Vec>, + existing_indices: Vec, - frag_reuse_index: Option>, + frag_reuse_index: Option>, // fragments for distributed indexing fragment_filter: Option>, @@ -163,13 +275,60 @@ pub struct IvfIndexBuilder { transpose_codes: bool, // lance file version for writing index files - format_version: LanceFileVersion, + format_version: ConcreteFileVersion, progress: Arc, } type BuildStream = - Pin::Storage, S, f64)>>> + Send>>; + Pin>>> + Send>>; + +type FreshWindowBuildStream = + Pin, OwnedSemaphorePermit)>> + Send>>; +type PartitionInputAdmissionStream = + Pin> + Send>>; + +fn admit_partition_inputs( + inputs: Vec, + entry_permits: Arc, +) -> PartitionInputAdmissionStream { + stream::iter(inputs) + .then(move |input| { + let entry_permits = entry_permits.clone(); + async move { + let entry_permit = entry_permits + .acquire_owned() + .await + .map_err(|_| Error::internal("partition build entry semaphore was closed"))?; + Ok((input, entry_permit)) + } + }) + .boxed() +} + +fn partition_window_entry_limit( + partition_range: &std::ops::Range, + num_partitions: usize, + max_entries: usize, + concurrency: usize, +) -> usize { + if partition_range.start == 0 && partition_range.end == num_partitions { + max_entries + } else { + max_entries.div_ceil(concurrency) + } +} + +struct PartitionBuildResult { + partition_id: usize, + built: Option<(Q::Storage, S, f64)>, +} + +struct FreshPartitionInput { + partition_id: usize, + batches: Vec, + loss: f64, +} type UnindexedStream = Box> + Send + Unpin + 'static>; @@ -189,7 +348,7 @@ impl IvfIndexBuilder ivf_params: Option, quantizer_params: Option, sub_index_params: S::BuildParams, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> Result { let temp_dir = TempStdDir::default(); let temp_dir_path = Path::from_filesystem_path(&temp_dir)?; @@ -230,7 +389,7 @@ impl IvfIndexBuilder distance_type: DistanceType, shuffler: Box, sub_index_params: S::BuildParams, - frag_reuse_index: Option>, + frag_reuse_index: Option>, optimize_options: OptimizeOptions, ) -> Result { let mut builder = Self::new( @@ -278,7 +437,7 @@ impl IvfIndexBuilder quantizer: Some(ivf_index.quantizer().try_into()?), shuffle_reader: None, shuffle_data_input: Mutex::new(None), - existing_indices: vec![index], + existing_indices: vec![ExistingIndex::unfiltered(index)], frag_reuse_index: None, fragment_filter: None, optimize_options: None, @@ -344,29 +503,28 @@ impl IvfIndexBuilder }; log::info!("remap {} partitions", ivf.num_partitions()); - let existing_index = self.existing_indices[0].clone(); + let existing_index = self.existing_indices[0].index.clone(); let mapping = Arc::new(mapping.clone()); - let build_iter = - (0..ivf.num_partitions()).map(move |part_id| { - let existing_index = existing_index.clone(); - let mapping = mapping.clone(); - async move { - let ivf_index = existing_index - .as_any() - .downcast_ref::>() - .ok_or(Error::invalid_input("existing index is not IVF index"))?; - let part = ivf_index - .load_partition(part_id, false, &NoOpMetricsCollector) - .await?; - let part = part.as_any().downcast_ref::>().ok_or( - Error::internal("failed to downcast partition entry".to_string()), - )?; + let build_iter = (0..ivf.num_partitions()).map(move |part_id| { + let existing_index = existing_index.clone(); + let mapping = mapping.clone(); + async move { + let ivf_index = existing_index + .as_any() + .downcast_ref::>() + .ok_or(Error::invalid_input("existing index is not IVF index"))?; + let part = ivf_index + .load_partition(part_id, false, &NoOpMetricsCollector) + .await?; - let storage = part.storage.remap(&mapping)?; - let index = part.index.remap(&mapping, &storage)?; - Result::Ok(Some((storage, index, 0.0))) - } - }); + let storage = part.storage.remap(&mapping)?; + let index = part.index.remap(&mapping, &storage)?; + Result::Ok(Budgeted::untracked(PartitionBuildResult { + partition_id: part_id, + built: Some((storage, index, 0.0)), + })) + } + }); let files = self .merge_partitions( @@ -388,14 +546,29 @@ impl IvfIndexBuilder self } + /// Read existing rows from `indices`, keeping every row they hold. pub fn with_existing_indices(&mut self, indices: Vec>) -> &mut Self { - self.existing_indices = indices; + self.existing_indices = indices.into_iter().map(ExistingIndex::unfiltered).collect(); + self + } + + /// Read existing rows from `sources`, keeping only the rows each segment is still + /// allowed to contribute. See [`ExistingIndex`]. + pub fn with_existing_index_sources(&mut self, sources: Vec) -> &mut Self { + self.existing_indices = sources; self } /// Set fragment filter for distributed indexing pub fn with_fragment_filter(&mut self, fragment_ids: Vec) -> &mut Self { - self.fragment_filter = Some(fragment_ids); + self.fragment_filter = Some(Dataset::normalize_fragment_ids(&fragment_ids)); + self + } + + pub fn with_optional_fragment_filter(&mut self, fragment_ids: Option<&[u32]>) -> &mut Self { + if let Some(fragment_ids) = fragment_ids { + self.fragment_filter = Some(Dataset::normalize_fragment_ids(fragment_ids)); + } self } @@ -453,7 +626,7 @@ impl IvfIndexBuilder )); }; let sample_size_hint = match &self.quantizer_params { - Some(params) => params.sample_size(), + Some(params) => params.try_sample_size()?, None => 256 * 256, // here it must be retrain, let's just set sample size to the default value }; @@ -553,19 +726,11 @@ impl IvfIndexBuilder return Ok(None); }; match &self.fragment_filter { - Some(fragment_ids) => { - let fragments: Vec<_> = dataset - .get_fragments() - .into_iter() - .filter(|f| fragment_ids.contains(&(f.id() as u32))) - .collect(); - let counts = futures::stream::iter(fragments) - .map(|f| async move { f.count_rows(None).await }) - .buffer_unordered(16) // ref: Dataset::count_all_rows() - .try_collect::>() - .await?; - Ok(Some(counts.iter().sum::() as u64)) - } + Some(fragment_ids) => Ok(Some( + dataset + .count_rows_in_existing_fragments(fragment_ids) + .await? as u64, + )), None => Ok(Some(dataset.count_rows(None).await? as u64)), } } @@ -603,14 +768,9 @@ impl IvfIndexBuilder "applying fragment filter for distributed indexing: {:?}", fragment_ids ); - // Filter fragments by converting fragment_ids to Fragment objects - let all_fragments = dataset.fragments(); - let filtered_fragments: Vec<_> = all_fragments - .iter() - .filter(|fragment| fragment_ids.contains(&(fragment.id as u32))) - .cloned() - .collect(); - builder.with_fragments(filtered_fragments); + builder.with_fragments( + dataset.get_existing_fragment_metadata_from_ids(fragment_ids), + ); } let (vector_type, _) = get_vector_type(dataset.schema(), &self.column)?; @@ -896,12 +1056,29 @@ impl IvfIndexBuilder let distance_type = self.distance_type; let column = self.column.clone(); let frag_reuse_index = self.frag_reuse_index.clone(); + if self.optimize_options.is_none() + && self.existing_indices.is_empty() + && partition_adjustment.is_none() + { + let num_partitions = assign_batches.len(); + return Self::build_fresh_partitions_windowed( + reader, + num_partitions, + distance_type, + quantizer, + sub_index_params, + column, + frag_reuse_index, + FreshPartitionBuildLimits::default(), + ); + } let partition_adjustment = Arc::new(partition_adjustment); let build_iter = assign_batches .into_iter() .enumerate() .map(move |(partition, assign_batch)| { + let output_partition_id = partition; let reader = reader.clone(); let indices = merge_indices.clone(); let distance_type = distance_type; @@ -989,7 +1166,10 @@ impl IvfIndexBuilder let num_rows = batches.iter().map(|b| b.num_rows()).sum::(); if num_rows == 0 { - return Ok(None); + return Ok(Budgeted::untracked(PartitionBuildResult { + partition_id: output_partition_id, + built: None, + })); } let (storage, sub_index) = Self::build_index( @@ -1000,7 +1180,10 @@ impl IvfIndexBuilder column, frag_reuse_index, )?; - Ok(Some((storage, sub_index, loss))) + Ok(Budgeted::untracked(PartitionBuildResult { + partition_id: output_partition_id, + built: Some((storage, sub_index, loss)), + })) }) .await } @@ -1010,6 +1193,212 @@ impl IvfIndexBuilder .boxed()) } + #[allow(clippy::too_many_arguments)] + fn build_fresh_partitions_windowed( + reader: Arc, + num_partitions: usize, + distance_type: DistanceType, + quantizer: Q, + sub_index_params: S::BuildParams, + column: String, + frag_reuse_index: Option>, + limits: FreshPartitionBuildLimits, + ) -> Result> { + let concurrency = get_num_compute_intensive_cpus().max(1); + let max_entries = concurrency.saturating_mul(PARTITION_BUILD_ENTRIES_PER_WORKER); + let cpu_permits = Arc::new(Semaphore::new(concurrency)); + let jobs = stream::try_unfold(0usize, move |next_partition_id| { + let reader = reader.clone(); + let quantizer = quantizer.clone(); + let sub_index_params = sub_index_params.clone(); + let column = column.clone(); + let frag_reuse_index = frag_reuse_index.clone(); + let cpu_permits = cpu_permits.clone(); + async move { + if next_partition_id == num_partitions { + return Ok(None); + } + let plan = reader.plan_partition_window( + next_partition_id, + limits.window_bytes, + )?; + if plan.partition_range.start != next_partition_id + || plan.partition_range.end <= plan.partition_range.start + || plan.partition_range.end > num_partitions + { + return Err(Error::internal(format!( + "shuffle reader planned invalid partition window {:?}; expected a non-empty window starting at {} within {} partitions", + plan.partition_range, next_partition_id, num_partitions + ))); + } + let next_partition_id = plan.partition_range.end; + let planned_range = plan.partition_range; + let window_entry_limit = partition_window_entry_limit( + &planned_range, + num_partitions, + max_entries, + concurrency, + ); + let job = WeightedJob::with_permit( + plan.estimated_decoded_bytes, + move |mut admission| async move { + let mut window = reader + .read_partition_window( + planned_range.start, + limits.window_bytes, + ) + .await?; + if window.partition_range != planned_range + || window.partitions.len() != planned_range.len() + { + return Err(Error::internal(format!( + "shuffle reader returned partition window {:?} with {} entries after planning {:?}", + window.partition_range, + window.partitions.len(), + planned_range + ))); + } + for (expected_partition_id, partition) in + planned_range.clone().zip(&window.partitions) + { + if partition.partition_id != expected_partition_id { + return Err(Error::internal(format!( + "shuffle reader window {:?} returned partition id {} at position {}", + planned_range, + partition.partition_id, + expected_partition_id - planned_range.start + ))); + } + } + + let count_stream_bytes = window.materialized_decoded_bytes.is_none(); + let mut decoded_bytes = + window.materialized_decoded_bytes.unwrap_or_default(); + let mut inputs = Vec::with_capacity(window.partitions.len()); + for mut partition in window.partitions.drain(..) { + let mut batches = Vec::new(); + let mut loss = 0.0; + if let Some(mut data) = partition.data.take() { + while let Some(batch) = data.try_next().await? { + loss += batch + .metadata() + .get(LOSS_METADATA_KEY) + .map(|value| value.parse::().unwrap_or(0.0)) + .unwrap_or(0.0); + if count_stream_bytes { + decoded_bytes = batch.columns().iter().try_fold( + decoded_bytes, + |total, array| { + total + .checked_add(array.get_array_memory_size()) + .ok_or_else(|| { + Error::internal(format!( + "decoded byte count overflow for partition {}", + partition.partition_id + )) + }) + }, + )?; + } + batches.push(batch.drop_column(PART_ID_COLUMN)?); + } + } + inputs.push(FreshPartitionInput { + partition_id: partition.partition_id, + batches, + loss, + }); + } + admission.reconcile(decoded_bytes); + + // Multiple windows each own a small FIFO entry budget so a + // later window cannot consume every slot needed for the + // oldest window to make ordered progress. If the whole + // shuffle fits in one window, that window owns the complete + // entry budget and can use the full CPU concurrency. + let entry_permits = Arc::new(Semaphore::new(window_entry_limit)); + let builds = admit_partition_inputs(inputs, entry_permits) + .map_ok(move |(input, entry_permit)| { + let quantizer = quantizer.clone(); + let sub_index_params = sub_index_params.clone(); + let column = column.clone(); + let frag_reuse_index = frag_reuse_index.clone(); + let cpu_permits = cpu_permits.clone(); + async move { + let partition_id = input.partition_id; + let loss = input.loss; + let _cpu_permit = + cpu_permits.acquire_owned().await.map_err(|_| { + Error::internal( + "partition build CPU semaphore was closed", + ) + })?; + let built = spawn_cpu(move || -> Result<_> { + let num_rows = input + .batches + .iter() + .map(|batch| batch.num_rows()) + .sum::(); + if num_rows == 0 { + return Ok(None); + } + let (storage, sub_index) = Self::build_index( + distance_type, + quantizer, + sub_index_params, + input.batches, + column, + frag_reuse_index, + )?; + Ok(Some((storage, sub_index, loss))) + }) + .await?; + Ok::<_, Error>(( + PartitionBuildResult { + partition_id, + built, + }, + entry_permit, + )) + } + }) + .try_buffer_unordered(concurrency) + .boxed(); + Ok::<(FreshWindowBuildStream, _), Error>((builds, admission)) + }, + ); + Ok(Some((job, next_partition_id))) + } + }) + .boxed(); + + let windows = BoundedPartitionStream::try_new( + jobs, + concurrency, + limits.decoded_budget_bytes, + // One admission entry per outstanding window. Together with each + // window's entry limit above, this bounds partition results by + // `max_entries` even after an inner stream has finished. + concurrency, + )?; + Ok(windows + .map_ok(|window| { + let Budgeted { + value: builds, + permit, + entry_permit, + } = window; + debug_assert!(entry_permit.is_none()); + builds.map_ok(move |(value, entry_permit)| Budgeted { + value, + permit: permit.clone(), + entry_permit: Some(entry_permit), + }) + }) + .try_flatten_unordered(Some(concurrency)) + .boxed()) + } + #[instrument(name = "build_index", level = "debug", skip_all)] #[allow(clippy::too_many_arguments)] fn build_index( @@ -1018,10 +1407,13 @@ impl IvfIndexBuilder sub_index_params: S::BuildParams, batches: Vec, column: String, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> Result<(Q::Storage, S)> { - let storage = StorageBuilder::new(column, distance_type, quantizer, frag_reuse_index)? - .build(batches)?; + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(CompactFragReuseIndexHandle(index)) as Arc); + let storage = + StorageBuilder::new_with_remapper(column, distance_type, quantizer, frag_reuse_index)? + .build(batches)?; let sub_index = S::index_vectors(&storage, sub_index_params)?; Ok((storage, sub_index)) @@ -1030,12 +1422,13 @@ impl IvfIndexBuilder #[instrument(name = "take_partition_batches", level = "debug", skip_all)] async fn take_partition_batches( part_id: usize, - existing_indices: &[Arc], + existing_indices: &[ExistingIndex], reader: Option<&dyn ShuffleReader>, ) -> Result<(Vec, f64)> { let mut batches = Vec::new(); - for existing_index in existing_indices.iter() { - let existing_index = existing_index + for source in existing_indices.iter() { + let existing_index = source + .index .as_any() .downcast_ref::>() .ok_or(Error::invalid_input("existing index is not IVF index"))?; @@ -1046,6 +1439,11 @@ impl IvfIndexBuilder continue; } + // Resolved before the partition is decoded: partitions are built + // concurrently, so whichever one builds the filter would otherwise hold a + // decoded partition per in-flight task while the rest wait on it. + let old_data_filter = source.old_data_filter().await?; + let part_storage = existing_index.load_partition_storage(part_id, None).await?; let mut part_batches = part_storage.to_batches()?.collect::>(); // for PQ, the PQ codes are transposed, so we need to transpose them back @@ -1087,6 +1485,18 @@ impl IvfIndexBuilder _ => {} } + // Drop rows this segment may no longer contribute. They are physically + // still in its index file, and the merged segment covers their live copies + // again, so keeping them would emit the same row twice. + if let Some(filter) = old_data_filter { + for batch in part_batches.iter_mut() { + let keep = filter.filter_row_ids(batch[ROW_ID].as_primitive::()); + if keep.true_count() < batch.num_rows() { + *batch = arrow::compute::filter_record_batch(batch, &keep)?; + } + } + } + batches.extend(part_batches); } @@ -1140,23 +1550,22 @@ impl IvfIndexBuilder let storage_path = self.index_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); let index_path = self.index_dir.clone().join(INDEX_FILE_NAME); - let writer_options = FileWriterOptions { - format_version: Some(self.format_version), - ..Default::default() - }; + let writer_options = FileWriterOptions::default(); let mut storage_writer = if is_flat { None } else { let mut fields = vec![ROW_ID_FIELD.clone(), quantizer.field()]; fields.extend(quantizer.extra_fields()); let storage_schema: Schema = (&arrow_schema::Schema::new(fields)).try_into()?; - Some(FileWriter::try_new( + Some(file_versions::create_writer( + self.format_version, self.store.create(&storage_path).await?, storage_schema, writer_options.clone(), )?) }; - let mut index_writer = FileWriter::try_new( + let mut index_writer = file_versions::create_writer( + self.format_version, self.store.create(&index_path).await?, S::schema().as_ref().try_into()?, writer_options.clone(), @@ -1167,96 +1576,112 @@ impl IvfIndexBuilder let mut index_ivf = IvfModel::new(ivf.centroids.clone().unwrap(), ivf.loss); let mut partition_index_metadata = Vec::with_capacity(ivf.num_partitions()); - let mut part_id = 0; + let num_partitions = ivf.num_partitions(); + let mut ordered_results = OrderedPartitionResults::new(num_partitions); let mut total_loss = 0.0; let progress = self.progress.clone(); - log::info!("merging {} partitions", ivf.num_partitions()); - while let Some(part) = build_stream.try_next().await? { - part_id += 1; - progress.stage_progress("merge_partitions", part_id).await?; - let Some((storage, index, loss)) = part else { - log::warn!("partition {} is empty, skipping", part_id); + log::info!("merging {} partitions", num_partitions); + while let Some(result) = build_stream.try_next().await? { + let partition_id = result.value.partition_id; + ordered_results.push(partition_id, result)?; + + while let Some((partition_id, result)) = ordered_results.pop_next() { + let Budgeted { + value: PartitionBuildResult { built: part, .. }, + permit: _permit, + entry_permit: _entry_permit, + } = result; + let completed_partitions = partition_id + 1; + progress + .stage_progress("merge_partitions", completed_partitions as u64) + .await?; + let Some((storage, index, loss)) = part else { + log::warn!("partition {} is empty, skipping", partition_id); - storage_ivf.add_partition(0); - index_ivf.add_partition(0); - partition_index_metadata.push(String::new()); + storage_ivf.add_partition(0); + index_ivf.add_partition(0); + partition_index_metadata.push(String::new()); - continue; - }; - total_loss += loss; + continue; + }; + total_loss += loss; - if storage.len() == 0 { - storage_ivf.add_partition(0); - } else { - for mut batch in storage.to_batches()? { - if is_pq - && !self.transpose_codes - && batch.num_rows() > 0 - && batch.column_by_name(PQ_CODE_COLUMN).is_some() - { - let codes_fsl = batch - .column_by_name(PQ_CODE_COLUMN) - .unwrap() - .as_fixed_size_list(); - let num_rows = batch.num_rows(); - let bytes_per_code = codes_fsl.value_length() as usize; - let codes = codes_fsl.values().as_primitive::(); - let original_codes = transpose(codes, bytes_per_code, num_rows); - let original_fsl = Arc::new(FixedSizeListArray::try_new_from_values( - original_codes, - bytes_per_code as i32, - )?); - batch = batch.replace_column_by_name(PQ_CODE_COLUMN, original_fsl)?; - } + if storage.len() == 0 { + storage_ivf.add_partition(0); + } else { + for mut batch in storage.to_batches()? { + if is_pq + && !self.transpose_codes + && batch.num_rows() > 0 + && batch.column_by_name(PQ_CODE_COLUMN).is_some() + { + let codes_fsl = batch + .column_by_name(PQ_CODE_COLUMN) + .unwrap() + .as_fixed_size_list(); + let num_rows = batch.num_rows(); + let bytes_per_code = codes_fsl.value_length() as usize; + let codes = codes_fsl.values().as_primitive::(); + let original_codes = transpose(codes, bytes_per_code, num_rows); + let original_fsl = Arc::new(FixedSizeListArray::try_new_from_values( + original_codes, + bytes_per_code as i32, + )?); + batch = batch.replace_column_by_name(PQ_CODE_COLUMN, original_fsl)?; + } - if is_rq - && !self.transpose_codes - && batch.num_rows() > 0 - && batch.column_by_name(RABIT_CODE_COLUMN).is_some() - { - let codes_fsl = batch - .column_by_name(RABIT_CODE_COLUMN) - .unwrap() - .as_fixed_size_list(); - let unpacked = Arc::new(unpack_codes(codes_fsl)); - batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, unpacked)?; - } + if is_rq + && !self.transpose_codes + && batch.num_rows() > 0 + && batch.column_by_name(RABIT_CODE_COLUMN).is_some() + { + let codes_fsl = batch + .column_by_name(RABIT_CODE_COLUMN) + .unwrap() + .as_fixed_size_list(); + let unpacked = Arc::new(unpack_codes(codes_fsl)); + batch = batch.replace_column_by_name(RABIT_CODE_COLUMN, unpacked)?; + } - if storage_writer.is_none() { - let storage_schema: Schema = batch.schema_ref().as_ref().try_into()?; - storage_writer = Some(FileWriter::try_new( - self.store.create(&storage_path).await?, - storage_schema, - writer_options.clone(), - )?); + if storage_writer.is_none() { + let storage_schema: Schema = batch.schema_ref().as_ref().try_into()?; + storage_writer = Some(file_versions::create_writer( + self.format_version, + self.store.create(&storage_path).await?, + storage_schema, + writer_options.clone(), + )?); + } + storage_writer + .as_mut() + .expect("storage writer must be initialized before write") + .write_batch(&batch) + .await?; + storage_ivf.add_partition(batch.num_rows() as u32); } - storage_writer - .as_mut() - .expect("storage writer must be initialized before write") - .write_batch(&batch) - .await?; - storage_ivf.add_partition(batch.num_rows() as u32); } - } - let index_batch = index.to_batch()?; - if index_batch.num_rows() == 0 { - index_ivf.add_partition(0); - partition_index_metadata.push(String::new()); - } else { - index_writer.write_batch(&index_batch).await?; - index_ivf.add_partition(index_batch.num_rows() as u32); - partition_index_metadata.push( - index_batch - .schema() - .metadata - .get(S::metadata_key()) - .cloned() - .unwrap_or_default(), - ); + let index_batch = index.to_batch()?; + if index_batch.num_rows() == 0 { + index_ivf.add_partition(0); + partition_index_metadata.push(String::new()); + } else { + index_writer.write_batch(&index_batch).await?; + index_ivf.add_partition(index_batch.num_rows() as u32); + partition_index_metadata.push( + index_batch + .schema() + .metadata + .get(S::metadata_key()) + .cloned() + .unwrap_or_default(), + ); + } } } + ordered_results.finish()?; + match self.shuffle_reader.as_ref() { Some(reader) => { // it's building index, the loss is already calculated in the shuffle reader @@ -1292,7 +1717,8 @@ impl IvfIndexBuilder ), ]); let storage_schema: Schema = (&flat_schema).try_into()?; - storage_writer = Some(FileWriter::try_new( + storage_writer = Some(file_versions::create_writer( + self.format_version, self.store.create(&storage_path).await?, storage_schema, writer_options.clone(), @@ -1453,7 +1879,7 @@ impl IvfIndexBuilder fn check_partition_adjustment( ivf: &IvfModel, reader: &dyn ShuffleReader, - existing_indices: &[Arc], + existing_indices: &[ExistingIndex], ) -> Result<(Vec, Option)> { let index_type = IndexType::try_from( index_type_string(S::name().try_into()?, Q::quantization_type()).as_str(), @@ -1464,8 +1890,8 @@ impl IvfIndexBuilder let mut min_partition_size = usize::MAX; for partition in 0..ivf.num_partitions() { let mut num_rows = reader.partition_size(partition)?; - for index in existing_indices.iter() { - num_rows += index.partition_size(partition); + for source in existing_indices.iter() { + num_rows += source.index.partition_size(partition); } if num_rows > MAX_PARTITION_SIZE_FACTOR * index_type.target_partition_size() { split_partitions.push(partition); @@ -2032,7 +2458,8 @@ impl IvfIndexBuilder async fn partition_row_ids(&self, part_idx: usize) -> Result> { // existing part: read from the existing indices let mut row_ids = Vec::new(); - for index in self.existing_indices.iter() { + for source in self.existing_indices.iter() { + let index = &source.index; if part_idx >= index.ivf_model().num_partitions() { // there was a bug that may cause delta indices have different number of partitions, // it's safe to skip loading the extra partition, and split/join the existing partitions, @@ -2048,8 +2475,21 @@ impl IvfIndexBuilder let mut reader = index .partition_reader(part_idx, false, &NoOpMetricsCollector) .await?; + let old_data_filter = source.old_data_filter().await?; while let Some(batch) = reader.try_next().await? { - row_ids.extend(batch[ROW_ID].as_primitive::().values()); + let batch_row_ids = batch[ROW_ID].as_primitive::(); + match old_data_filter { + // Rows the segment may no longer contribute must not be reassigned + // to a partition; their live copy comes from another source. + Some(filter) => row_ids.extend( + batch_row_ids + .values() + .iter() + .zip(filter.filter_row_ids(batch_row_ids).values().iter()) + .filter_map(|(row_id, keep)| keep.then_some(row_id)), + ), + None => row_ids.extend(batch_row_ids.values()), + } } } @@ -2233,9 +2673,14 @@ pub(crate) fn index_type_string(sub_index: SubIndexType, quantizer: Quantization #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + use super::*; use arrow_array::{Array, Float32Array, NullArray}; use lance_index::vector::flat::index::{FlatIndex, FlatQuantizer}; + use lance_index::vector::v3::shuffler::{ + ShufflePartition, ShufflePartitionWindow, ShufflePartitionWindowPlan, + }; struct SingleBatchReader { batch: RecordBatch, @@ -2272,11 +2717,217 @@ mod tests { } } + struct WindowedBatchReader { + batches: Vec, + windows_read: Arc, + } + + #[async_trait::async_trait] + impl ShuffleReader for WindowedBatchReader { + async fn read_partition( + &self, + partition_id: usize, + ) -> Result>> { + let Some(batch) = self.batches.get(partition_id) else { + return Ok(None); + }; + Ok(Some(Box::new(RecordBatchStreamAdapter::new( + batch.schema(), + stream::iter(vec![Ok(batch.clone())]), + )))) + } + + fn plan_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + if max_decoded_bytes == 0 { + return Err(Error::invalid_input( + "max_decoded_bytes must be greater than 0", + )); + } + if start_partition_id >= self.batches.len() { + return Err(Error::invalid_input(format!( + "start_partition_id={} is out of range [0, {})", + start_partition_id, + self.batches.len() + ))); + } + let end_partition_id = start_partition_id + .saturating_add(max_decoded_bytes) + .min(self.batches.len()); + Ok(ShufflePartitionWindowPlan { + partition_range: start_partition_id..end_partition_id, + estimated_decoded_bytes: end_partition_id - start_partition_id, + }) + } + + async fn read_partition_window( + &self, + start_partition_id: usize, + max_decoded_bytes: usize, + ) -> Result { + let plan = self.plan_partition_window(start_partition_id, max_decoded_bytes)?; + self.windows_read.fetch_add(1, Ordering::Relaxed); + if start_partition_id == 0 { + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while self.windows_read.load(Ordering::Relaxed) < 2 { + tokio::task::yield_now().await; + } + }) + .await + .map_err(|_| Error::internal("second partition window was not admitted"))?; + } + let partitions = plan + .partition_range + .clone() + .map(|partition_id| { + let batch = self.batches[partition_id].clone(); + ShufflePartition { + partition_id, + data: Some(Box::new(RecordBatchStreamAdapter::new( + batch.schema(), + stream::iter(vec![Ok(batch)]), + ))), + } + }) + .collect(); + Ok(ShufflePartitionWindow { + materialized_decoded_bytes: Some(plan.partition_range.len()), + partition_range: plan.partition_range, + partitions, + }) + } + + fn partition_size(&self, partition_id: usize) -> Result { + Ok(self + .batches + .get(partition_id) + .map(RecordBatch::num_rows) + .unwrap_or(0)) + } + + fn total_loss(&self) -> Option { + None + } + } + + fn flat_partition_batch(partition_id: usize) -> RecordBatch { + let vectors = FixedSizeListArray::try_new_from_values( + Float32Array::from(vec![partition_id as f32, partition_id as f32 + 0.5]), + 2, + ) + .unwrap(); + RecordBatch::try_new( + Arc::new(arrow_schema::Schema::new(vec![ + ROW_ID_FIELD.clone(), + Field::new("vector", vectors.data_type().clone(), false), + ])), + vec![ + Arc::new(UInt64Array::from(vec![partition_id as u64])), + Arc::new(vectors), + ], + ) + .unwrap() + } + // Helper to read centroid i from a FixedSizeListArray as a Vec fn centroid_values(arr: &FixedSizeListArray, i: usize) -> Vec { arr.value(i).as_primitive::().values().to_vec() } + #[tokio::test] + async fn partition_entry_admission_preserves_input_order() { + let entry_permits = Arc::new(Semaphore::new(1)); + let held_permit = entry_permits.clone().acquire_owned().await.unwrap(); + let mut admitted = admit_partition_inputs(vec![0, 1, 2], entry_permits); + + let first = admitted.next(); + tokio::pin!(first); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), &mut first) + .await + .is_err() + ); + + drop(held_permit); + let (partition_id, first_permit) = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut first) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(partition_id, 0); + + let second = admitted.next(); + tokio::pin!(second); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), &mut second) + .await + .is_err() + ); + drop(first_permit); + let (partition_id, _second_permit) = + tokio::time::timeout(std::time::Duration::from_millis(100), &mut second) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(partition_id, 1); + } + + #[test] + fn single_partition_window_uses_full_entry_budget() { + assert_eq!(partition_window_entry_limit(&(0..64), 64, 32, 16), 32); + assert_eq!(partition_window_entry_limit(&(0..32), 64, 32, 16), 2); + assert_eq!(partition_window_entry_limit(&(32..64), 64, 32, 16), 2); + } + + #[tokio::test] + async fn fresh_partition_build_runs_multiple_windows_end_to_end() { + let num_partitions = 6; + let windows_read = Arc::new(AtomicUsize::new(0)); + let reader = Arc::new(WindowedBatchReader { + batches: (0..num_partitions).map(flat_partition_batch).collect(), + windows_read: windows_read.clone(), + }); + let mut build_stream = + IvfIndexBuilder::::build_fresh_partitions_windowed( + reader, + num_partitions, + DistanceType::L2, + FlatQuantizer::new(2, DistanceType::L2), + (), + "vector".to_string(), + None, + FreshPartitionBuildLimits { + window_bytes: 2, + decoded_budget_bytes: 4, + }, + ) + .unwrap(); + + let mut ordered_results = OrderedPartitionResults::new(num_partitions); + let mut merged_partition_ids = Vec::with_capacity(num_partitions); + while let Some(result) = build_stream.try_next().await.unwrap() { + ordered_results + .push(result.value.partition_id, result) + .unwrap(); + while let Some((partition_id, result)) = ordered_results.pop_next() { + assert!(result.value.built.is_some()); + merged_partition_ids.push(partition_id); + } + } + ordered_results.finish().unwrap(); + + assert_eq!(windows_read.load(Ordering::Relaxed), 3); + assert_eq!( + merged_partition_ids, + (0..num_partitions).collect::>() + ); + } + #[test] fn apply_centroid_splits_correct_count_and_ordering() { // 4 original centroids at [0,0], [1,1], [2,2], [3,3]. diff --git a/rust/lance/src/index/vector/details.rs b/rust/lance/src/index/vector/details.rs index 63f9375792e..c2aabf9089a 100644 --- a/rust/lance/src/index/vector/details.rs +++ b/rust/lance/src/index/vector/details.rs @@ -364,7 +364,7 @@ pub fn needs_vector_details_inference( ) -> bool { match &index.index_details { Some(d) => d.type_url.ends_with("VectorIndexDetails") && d.value.is_empty(), - None => index.fields.iter().any(|&field_id| { + None => index.fields.first().is_some_and(|&field_id| { schema .field_by_id(field_id) .map(|f| matches!(f.data_type(), arrow_schema::DataType::FixedSizeList(_, _))) @@ -926,6 +926,7 @@ mod tests { let index = IndexMetadata { uuid: uuid::Uuid::new_v4(), fields: vec![0], + covering_fields: vec![], name: "test_index".to_string(), dataset_version: 1, fragment_bitmap: None, @@ -948,6 +949,7 @@ mod tests { let index = IndexMetadata { uuid: uuid::Uuid::new_v4(), fields: vec![0], + covering_fields: vec![], name: "test_index".to_string(), dataset_version: 1, fragment_bitmap: None, @@ -968,6 +970,7 @@ mod tests { let index = IndexMetadata { uuid: uuid::Uuid::new_v4(), fields: vec![0], + covering_fields: vec![], name: "test_index".to_string(), dataset_version: 1, fragment_bitmap: None, @@ -982,6 +985,89 @@ mod tests { assert_eq!(metric, None); } + // Schema with a vector column ("vec", FixedSizeList) and a scalar column + // ("tag", Utf8). Field ids are assigned by `set_field_id` during conversion, + // so look them up by name rather than hardcoding. + fn schema_with_vector_and_scalar() -> lance_core::datatypes::Schema { + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + let arrow = ArrowSchema::new(vec![ + ArrowField::new( + "vec", + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + 8, + ), + true, + ), + ArrowField::new("tag", DataType::Utf8, true), + ]); + lance_core::datatypes::Schema::try_from(&arrow).unwrap() + } + + fn index_over_field(field_id: i32, index_details: Option) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + fields: vec![field_id], + covering_fields: vec![], + name: "idx".to_string(), + dataset_version: 1, + fragment_bitmap: None, + index_details: index_details.map(Arc::new), + index_version: 1, + created_at: None, + base_id: None, + files: None, + } + } + + #[test] + fn test_needs_inference_missing_details_on_vector_field() { + // Oldest legacy case (<=0.19.2): no details, but the indexed field is a + // vector type => must infer. + let schema = schema_with_vector_and_scalar(); + let vec_id = schema.field("vec").unwrap().id; + let index = index_over_field(vec_id, None); + assert!(needs_vector_details_inference(&index, &schema)); + } + + #[test] + fn test_needs_inference_missing_details_on_scalar_field() { + // No details and the indexed field is not a vector type (e.g. an FTS or + // scalar index) => nothing to infer, so the load_indices fast path may + // skip the clone/infer/compare. + let schema = schema_with_vector_and_scalar(); + let tag_id = schema.field("tag").unwrap().id; + let index = index_over_field(tag_id, None); + assert!(!needs_vector_details_inference(&index, &schema)); + } + + #[test] + fn test_needs_inference_empty_vector_details() { + // Newer pre-details case: a VectorIndexDetails type_url with empty value + // bytes => must infer. + let schema = schema_with_vector_and_scalar(); + let vec_id = schema.field("vec").unwrap().id; + let index = index_over_field(vec_id, Some(vector_index_details_default())); + assert!(needs_vector_details_inference(&index, &schema)); + } + + #[test] + fn test_needs_inference_populated_vector_details() { + // Modern vector index with populated details => no inference needed. + let schema = schema_with_vector_and_scalar(); + let vec_id = schema.field("vec").unwrap().id; + let details = make_details( + VectorMetricType::L2, + None, + Some(Compression::Pq(ProductQuantization { + num_bits: 8, + num_sub_vectors: 16, + })), + ); + let index = index_over_field(vec_id, Some(details)); + assert!(!needs_vector_details_inference(&index, &schema)); + } + #[test] fn test_metric_type_from_index_metadata_all_metrics() { // Test all supported metric types. @@ -1009,6 +1095,7 @@ mod tests { let index = IndexMetadata { uuid: uuid::Uuid::new_v4(), fields: vec![0], + covering_fields: vec![], name: "test_index".to_string(), dataset_version: 1, fragment_bitmap: None, diff --git a/rust/lance/src/index/vector/fixture_test.rs b/rust/lance/src/index/vector/fixture_test.rs index a03cc313466..2af020e1df2 100644 --- a/rust/lance/src/index/vector/fixture_test.rs +++ b/rust/lance/src/index/vector/fixture_test.rs @@ -278,6 +278,7 @@ mod test { deleted_ids: None, filtered_ids: None, deleted_fragments: None, + overlay_block: None, final_mask: Mutex::new(OnceCell::new()), }), &NoOpMetricsCollector, diff --git a/rust/lance/src/index/vector/hamming.rs b/rust/lance/src/index/vector/hamming.rs index ba6ea98c42d..3240ab3021b 100644 --- a/rust/lance/src/index/vector/hamming.rs +++ b/rust/lance/src/index/vector/hamming.rs @@ -5,123 +5,229 @@ //! //! This module provides functionality to perform pairwise hamming distance //! computation and clustering on specific partitions of IVF_FLAT indices. - +//! +//! A logical IVF_FLAT index may consist of multiple physical segments (e.g. +//! delta segments created by `optimize_indices` in append mode, or segments +//! committed by distributed index builds). All segments of one logical index +//! are assumed to share the same global IVF centroids, so one partition id +//! refers to the same centroid region in every segment; this is validated and +//! an error is returned if the centroids differ. + +use std::sync::Arc; use std::time::Instant; -use arrow_array::RecordBatchReader; use arrow_array::cast::AsArray; use arrow_array::types::UInt64Type; +use arrow_array::{Array, FixedSizeListArray, RecordBatchReader}; use arrow_schema::DataType; use lance_core::{Error, Result}; use lance_index::metrics::NoOpMetricsCollector; use lance_index::vector::VectorIndex; use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex}; use lance_index::vector::flat::storage::FLAT_COLUMN; +use lance_index::vector::ivf::storage::IvfModel; use lance_index::vector::storage::VectorStore; use lance_linalg::distance::{ - ClusteringResult, cluster_pairwise_result, extract_hashes_from_fixed_list, + BinaryHashValues, ClusteringResult, cluster_pairwise_result, + extract_binary_hashes_from_fixed_list, pairwise_hamming_distance_binary_parallel, pairwise_hamming_distance_parallel, }; +use lance_table::format::IndexMetadata; use rand::rng; use rand::seq::index::sample; +use uuid::Uuid; use crate::dataset::Dataset; -use crate::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, filter_index_segments_by_ids}; use super::ivf::v2::IVFIndex; -/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index. -/// -/// This function loads a specific partition from an IVF_FLAT index on a hash column, -/// computes pairwise hamming distances between all hashes in the partition, -/// filters by threshold, and clusters the results using union-find. -/// -/// # Arguments -/// -/// * `dataset` - The Lance dataset -/// * `index_name` - Name of the IVF_FLAT index on the hash column -/// * `partition_id` - The partition ID within the IVF_FLAT index -/// * `hamming_threshold` - Maximum hamming distance to consider as similar -/// -/// # Returns -/// -/// A `RecordBatchReader` yielding batches with columns: -/// - `representative`: UInt64 - The representative row ID for each cluster -/// - `duplicates`: `List` - List of duplicate row IDs in each cluster -/// -/// # Errors +/// One opened physical segment of a logical IVF_FLAT binary index. +struct HashIndexSegment { + metadata: IndexMetadata, + index: Arc, +} + +impl HashIndexSegment { + fn ivf_flat_bin(&self) -> &IVFIndex { + self.index + .as_any() + .downcast_ref::>() + .expect("segment type validated in open_hash_index_segments") + } +} + +/// Validate that a column stores fixed-width binary hashes as +/// `FixedSizeList`, where `N` is a positive multiple of 8 bytes. +fn validate_hash_column(column: &str, data_type: &DataType) -> Result { + match data_type { + DataType::FixedSizeList(inner, size) if *inner.data_type() == DataType::UInt8 => { + if *size <= 0 || !(*size as usize).is_multiple_of(8) { + return Err(Error::invalid_input(format!( + "Column '{}' must be FixedSizeList where N is a positive \ + multiple of 8 bytes, got FixedSizeList", + column, size + ))); + } + Ok(*size as usize) + } + DataType::FixedSizeList(inner, size) => Err(Error::invalid_input(format!( + "Column '{}' must be FixedSizeList where N is a positive \ + multiple of 8 bytes, got FixedSizeList<{:?}, {}>", + column, + inner.data_type(), + size + ))), + _ => Err(Error::invalid_input(format!( + "Column '{}' must be FixedSizeList where N is a positive \ + multiple of 8 bytes, got {:?}", + column, data_type + ))), + } +} + +/// Validate that every segment of a logical IVF index shares the same global +/// centroids, so one partition id refers to the same centroid region in each +/// segment. Fails if any segment has no centroids or diverging centroids. +fn validate_shared_centroids<'a>( + index_name: &str, + models: impl IntoIterator, +) -> Result<()> { + struct Reference<'a> { + uuid: Uuid, + num_partitions: usize, + centroids: &'a FixedSizeListArray, + } + + let mut reference: Option> = None; + for (uuid, model) in models { + let centroids = model.centroids_array().ok_or_else(|| { + Error::invalid_input(format!( + "Index '{}' segment {} has no IVF centroids; hamming clustering requires \ + segments built from a shared global IVF model", + index_name, uuid + )) + })?; + match &reference { + None => { + reference = Some(Reference { + uuid, + num_partitions: model.num_partitions(), + centroids, + }); + } + Some(reference) => { + if centroids.to_data() != reference.centroids.to_data() { + return Err(Error::invalid_input(format!( + "Index '{}' segments do not share the same global IVF centroids: \ + segment {} ({} partitions) differs from segment {} ({} partitions); \ + retrain the index to merge segments before hamming clustering", + index_name, + uuid, + model.num_partitions(), + reference.uuid, + reference.num_partitions + ))); + } + } + } + } + Ok(()) +} + +/// Open the physical segments of a logical IVF_FLAT binary index. /// -/// Returns an error if: -/// - The index doesn't exist or is not an IVF_FLAT index -/// - The indexed column has wrong type (must be `FixedSizeList`) -/// - The partition ID is out of range -pub async fn hamming_clustering_for_ivf_partition( +/// When `segment_ids` is `None` all segments are opened; otherwise only the +/// requested segments are opened and every requested id must exist. Validates +/// that all selected segments index the same fixed-width binary hash column, +/// are IVF_FLAT indices for binary data, and share the same global centroids. +async fn open_hash_index_segments( dataset: &Dataset, index_name: &str, - partition_id: usize, - hamming_threshold: u32, -) -> Result> { - // Load indices and find the IVF_FLAT index - let indices = dataset.load_indices().await?; - let index_meta = indices - .iter() - .find(|idx| idx.name == index_name) - .ok_or_else(|| { - Error::invalid_input(format!("Index '{}' not found on dataset", index_name)) - })?; + segment_ids: Option<&[Uuid]>, +) -> Result> { + let metadatas = dataset.load_indices_by_name(index_name).await?; + if metadatas.is_empty() { + return Err(Error::invalid_input(format!( + "Index '{}' not found on dataset", + index_name + ))); + } - // Get the column name from the index metadata - let schema = dataset.schema(); - let field_id = index_meta - .fields + let metadatas = match segment_ids { + None => metadatas, + Some(segment_ids) => { + if segment_ids.is_empty() { + return Err(Error::invalid_input(format!( + "Segment selection for index '{}' must not be empty; \ + omit index_segments to use all segments", + index_name + ))); + } + filter_index_segments_by_ids(index_name, metadatas, segment_ids)? + } + }; + + let fields = metadatas[0].fields.clone(); + if let Some(mismatched) = metadatas.iter().find(|meta| meta.fields != fields) { + return Err(Error::invalid_input(format!( + "Index '{}' segments cover different fields: segment {} covers {:?} \ + while segment {} covers {:?}", + index_name, metadatas[0].uuid, fields, mismatched.uuid, mismatched.fields + ))); + } + + let field_id = fields .first() .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?; + let schema = dataset.schema(); let field = schema.field_by_id(*field_id).ok_or_else(|| { Error::invalid_input(format!( "Field with id {} not found in schema for index '{}'", field_id, index_name )) })?; - let column = &field.name; + validate_hash_column(&field.name, &field.data_type())?; - // Check column is FixedSizeList - let data_type = field.data_type(); - match data_type { - DataType::FixedSizeList(inner, 8) => { - if *inner.data_type() != DataType::UInt8 { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got FixedSizeList<{:?}, 8>", - column, - inner.data_type() - ))); - } - } - _ => { + let mut segments = Vec::with_capacity(metadatas.len()); + for metadata in metadatas { + let index = dataset + .open_vector_index(&field.name, &metadata.uuid, &NoOpMetricsCollector) + .await?; + if index + .as_any() + .downcast_ref::>() + .is_none() + { return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got {:?}", - column, data_type + "Index '{}' segment {} is not an IVF_FLAT index for binary data", + index_name, metadata.uuid ))); } + segments.push(HashIndexSegment { metadata, index }); } - // Open the vector index - let index = dataset - .open_vector_index(column, &index_meta.uuid, &NoOpMetricsCollector) - .await?; + validate_shared_centroids( + index_name, + segments + .iter() + .map(|segment| (segment.metadata.uuid, segment.ivf_flat_bin().ivf_model())), + )?; - // Try to downcast to IVFIndex (IVF_FLAT for binary data) - let ivf_index = index - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - Error::invalid_input(format!( - "Index '{}' is not an IVF_FLAT index for binary data", - index_name - )) - })?; + Ok(segments) +} - // Check partition ID is valid - let num_partitions = ivf_index.ivf_model().num_partitions(); +async fn hamming_clustering_for_ivf_partition_impl( + dataset: &Dataset, + index_name: &str, + segment_ids: Option<&[Uuid]>, + partition_id: usize, + hamming_threshold: u32, +) -> Result> { + let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?; + + // All segments share centroids, so the partition count is uniform. + let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions(); if partition_id >= num_partitions { return Err(Error::invalid_input(format!( "Partition ID {} is out of range (0..{})", @@ -129,45 +235,53 @@ pub async fn hamming_clustering_for_ivf_partition( ))); } - // Load the partition storage - let storage = ivf_index.load_partition_storage(partition_id, None).await?; - - // Get row IDs - let row_id_slice: Vec = storage.row_ids().copied().collect(); - - if row_id_slice.is_empty() { - let empty = ClusteringResult { - clusters: Vec::new(), - }; - return Ok(empty.into_reader(None)); + // Concatenate the partition's row ids and hashes across segments; identical + // hashes land in the same partition of every segment, so one pairwise pass + // over the union finds cross-segment duplicates. + let mut all_row_ids: Vec = Vec::new(); + let mut hash_chunks = Vec::new(); + let mut num_hashes = 0; + for segment in &segments { + let storage = segment + .ivf_flat_bin() + .load_partition_storage(partition_id, None) + .await?; + all_row_ids.extend(storage.row_ids().copied()); + for batch in storage.to_batches()? { + let vectors = batch + .column_by_name(FLAT_COLUMN) + .ok_or_else(|| { + Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN)) + })? + .as_fixed_size_list(); + let hashes = extract_binary_hashes_from_fixed_list(vectors)?; + num_hashes += hashes.len(); + hash_chunks.push(hashes); + } + if all_row_ids.len() != num_hashes { + return Err(Error::internal(format!( + "Index '{}' segment {} partition {}: row id count {} does not match hash count {}", + index_name, + segment.metadata.uuid, + partition_id, + all_row_ids.len(), + num_hashes + ))); + } } - // Get vectors from the storage batches - let batches: Vec<_> = storage.to_batches()?.collect(); - if batches.is_empty() { + if all_row_ids.is_empty() { let empty = ClusteringResult { clusters: Vec::new(), }; return Ok(empty.into_reader(None)); } - - // Extract the hash vectors from the FLAT_COLUMN - let mut all_hashes = Vec::new(); - for batch in &batches { - let vectors = batch - .column_by_name(FLAT_COLUMN) - .ok_or_else(|| { - Error::invalid_input(format!("Column '{}' not found in storage", FLAT_COLUMN)) - })? - .as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(vectors)?; - all_hashes.extend(hashes); - } + let all_hashes = BinaryHashValues::concat(&hash_chunks)?; // Compute pairwise hamming distances with threshold filtering - let pairwise_result = pairwise_hamming_distance_parallel( + let pairwise_result = pairwise_hamming_distance_binary_parallel( &all_hashes, - Some(&row_id_slice), + Some(&all_row_ids), Some(hamming_threshold), ); @@ -177,60 +291,125 @@ pub async fn hamming_clustering_for_ivf_partition( Ok(clustering.into_reader(None)) } -/// Get partition statistics for an IVF_FLAT index. -pub async fn get_ivf_partition_info( +/// Perform pairwise hamming distance clustering on a partition of an IVF_FLAT index. +/// +/// This function loads a specific partition from every segment of an IVF_FLAT +/// index on a hash column, computes pairwise hamming distances between all +/// hashes in the combined partition, filters by threshold, and clusters the +/// results using union-find. See [`hamming_clustering_for_ivf_partition_segments`] +/// to restrict the computation to selected segments. +/// +/// # Arguments +/// +/// * `dataset` - The Lance dataset +/// * `index_name` - Name of the IVF_FLAT index on the hash column +/// * `partition_id` - The partition ID within the IVF_FLAT index +/// * `hamming_threshold` - Maximum hamming distance to consider as similar +/// +/// # Returns +/// +/// A `RecordBatchReader` yielding batches with columns: +/// - `representative`: UInt64 - The representative row ID for each cluster +/// - `duplicates`: `List` - List of duplicate row IDs in each cluster +/// +/// # Errors +/// +/// Returns an error if: +/// - The index doesn't exist or is not an IVF_FLAT index +/// - The indexed column has wrong type (must be `FixedSizeList` where +/// `N` is a positive multiple of 8 bytes) +/// - The index segments do not share the same global IVF centroids +/// - The partition ID is out of range +pub async fn hamming_clustering_for_ivf_partition( dataset: &Dataset, index_name: &str, -) -> Result> { - let indices = dataset.load_indices().await?; - let index_meta = indices - .iter() - .find(|idx| idx.name == index_name) - .ok_or_else(|| { - Error::invalid_input(format!("Index '{}' not found on dataset", index_name)) - })?; - - // Get the column name from the index metadata - let schema = dataset.schema(); - let field_id = index_meta - .fields - .first() - .ok_or_else(|| Error::invalid_input(format!("Index '{}' has no fields", index_name)))?; - let field = schema.field_by_id(*field_id).ok_or_else(|| { - Error::invalid_input(format!( - "Field with id {} not found in schema for index '{}'", - field_id, index_name - )) - })?; - let column = &field.name; - - let index = dataset - .open_vector_index(column, &index_meta.uuid, &NoOpMetricsCollector) - .await?; - - let ivf_index = index - .as_any() - .downcast_ref::>() - .ok_or_else(|| { - Error::invalid_input(format!( - "Index '{}' is not an IVF_FLAT index for binary data", - index_name - )) - })?; + partition_id: usize, + hamming_threshold: u32, +) -> Result> { + hamming_clustering_for_ivf_partition_impl( + dataset, + index_name, + None, + partition_id, + hamming_threshold, + ) + .await +} - let num_partitions = ivf_index.ivf_model().num_partitions(); - let mut partition_infos = Vec::with_capacity(num_partitions); +/// Perform pairwise hamming distance clustering on a partition of selected +/// segments of an IVF_FLAT index. +/// +/// Same as [`hamming_clustering_for_ivf_partition`] but only the requested +/// physical segments contribute rows. Segment ids are the index UUIDs reported +/// by index descriptions; every requested id must belong to the named index and +/// the selection must not be empty. +pub async fn hamming_clustering_for_ivf_partition_segments( + dataset: &Dataset, + index_name: &str, + segment_ids: &[Uuid], + partition_id: usize, + hamming_threshold: u32, +) -> Result> { + hamming_clustering_for_ivf_partition_impl( + dataset, + index_name, + Some(segment_ids), + partition_id, + hamming_threshold, + ) + .await +} - for i in 0..num_partitions { - partition_infos.push(PartitionInfo { - partition_id: i, - size: ivf_index.ivf_model().partition_size(i), - }); +async fn get_ivf_partition_info_impl( + dataset: &Dataset, + index_name: &str, + segment_ids: Option<&[Uuid]>, +) -> Result> { + let segments = open_hash_index_segments(dataset, index_name, segment_ids).await?; + + let num_partitions = segments[0].ivf_flat_bin().ivf_model().num_partitions(); + let mut partition_infos: Vec = (0..num_partitions) + .map(|partition_id| PartitionInfo { + partition_id, + size: 0, + }) + .collect(); + for segment in &segments { + // Sizes come from the partition storage; the IVF model of a v3 index + // file does not carry partition lengths. + let index = segment.ivf_flat_bin(); + for info in partition_infos.iter_mut() { + info.size += index.partition_size(info.partition_id); + } } Ok(partition_infos) } +/// Get partition statistics for an IVF_FLAT index. +/// +/// Partition sizes are aggregated across all segments of the logical index. +/// See [`get_ivf_partition_info_segments`] to restrict the statistics to +/// selected segments. +pub async fn get_ivf_partition_info( + dataset: &Dataset, + index_name: &str, +) -> Result> { + get_ivf_partition_info_impl(dataset, index_name, None).await +} + +/// Get partition statistics for selected segments of an IVF_FLAT index. +/// +/// Same as [`get_ivf_partition_info`] but only the requested physical segments +/// contribute to the partition sizes. +pub async fn get_ivf_partition_info_segments( + dataset: &Dataset, + index_name: &str, + segment_ids: &[Uuid], +) -> Result> { + get_ivf_partition_info_impl(dataset, index_name, Some(segment_ids)).await +} + /// Information about an IVF partition. #[derive(Debug, Clone)] pub struct PartitionInfo { @@ -247,7 +426,8 @@ pub struct PartitionInfo { /// # Arguments /// /// * `dataset` - The Lance dataset -/// * `column` - Name of the hash column (must be `FixedSizeList`) +/// * `column` - Name of the hash column (must be `FixedSizeList` +/// where `N` is a positive multiple of 8 bytes) /// * `sample_size` - Number of rows to sample (if None or >= total rows, uses all rows) /// * `hamming_threshold` - Maximum hamming distance to consider as similar /// @@ -267,26 +447,7 @@ pub async fn hamming_clustering_for_sample( let field = schema.field(column).ok_or_else(|| { Error::invalid_input(format!("Column '{}' not found in dataset schema", column)) })?; - - // Check column is FixedSizeList - let data_type = field.data_type(); - match data_type { - DataType::FixedSizeList(inner, 8) => { - if *inner.data_type() != DataType::UInt8 { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got FixedSizeList<{:?}, 8>", - column, - inner.data_type() - ))); - } - } - _ => { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got {:?}", - column, data_type - ))); - } - } + validate_hash_column(column, &field.data_type())?; // Get total row count let total_rows: usize = dataset @@ -326,7 +487,7 @@ pub async fn hamming_clustering_for_sample( Error::invalid_input(format!("Column '{}' not found in result", column)) })?; let hashes_arr = hash_col.as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(hashes_arr)?; + let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?; (hashes, row_id_vec) } else { @@ -348,7 +509,7 @@ pub async fn hamming_clustering_for_sample( Error::invalid_input(format!("Column '{}' not found in result", column)) })?; let hashes_arr = hash_col.as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(hashes_arr)?; + let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?; (hashes, row_id_vec) }; @@ -362,7 +523,7 @@ pub async fn hamming_clustering_for_sample( // Compute pairwise hamming distances let pairwise = - pairwise_hamming_distance_parallel(&hashes, Some(&row_ids), Some(hamming_threshold)); + pairwise_hamming_distance_binary_parallel(&hashes, Some(&row_ids), Some(hamming_threshold)); // Cluster edges let clustering = cluster_pairwise_result(&pairwise); @@ -380,7 +541,8 @@ pub async fn hamming_clustering_for_sample( /// # Arguments /// /// * `dataset` - The Lance dataset -/// * `column` - Name of the hash column (must be `FixedSizeList`) +/// * `column` - Name of the hash column (must be `FixedSizeList` +/// where `N` is a positive multiple of 8 bytes) /// * `fragment_id` - The fragment ID to read from /// * `start_row` - The starting row offset within the fragment /// * `num_rows` - Number of rows to read from the start position @@ -396,7 +558,8 @@ pub async fn hamming_clustering_for_sample( /// /// Returns an error if: /// - The fragment doesn't exist -/// - The column has wrong type (must be `FixedSizeList`) +/// - The column has wrong type (must be `FixedSizeList` where `N` +/// is a positive multiple of 8 bytes) /// - The row range is out of bounds pub async fn hamming_clustering_for_range( dataset: &Dataset, @@ -411,26 +574,7 @@ pub async fn hamming_clustering_for_range( let field = schema.field(column).ok_or_else(|| { Error::invalid_input(format!("Column '{}' not found in dataset schema", column)) })?; - - // Check column is FixedSizeList - let data_type = field.data_type(); - match data_type { - DataType::FixedSizeList(inner, 8) => { - if *inner.data_type() != DataType::UInt8 { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got FixedSizeList<{:?}, 8>", - column, - inner.data_type() - ))); - } - } - _ => { - return Err(Error::invalid_input(format!( - "Column '{}' must be FixedSizeList, got {:?}", - column, data_type - ))); - } - } + validate_hash_column(column, &field.data_type())?; // Get the fragment let fragment = dataset.get_fragment(fragment_id).ok_or_else(|| { @@ -483,7 +627,7 @@ pub async fn hamming_clustering_for_range( .column_by_name(column) .ok_or_else(|| Error::invalid_input(format!("Column '{}' not found in result", column)))?; let hashes_arr = hash_col.as_fixed_size_list(); - let hashes = extract_hashes_from_fixed_list(hashes_arr)?; + let hashes = extract_binary_hashes_from_fixed_list(hashes_arr)?; if hashes.len() < 2 { let empty = ClusteringResult { @@ -493,8 +637,11 @@ pub async fn hamming_clustering_for_range( } // Compute pairwise hamming distances - let pairwise = - pairwise_hamming_distance_parallel(&hashes, Some(&row_id_vec), Some(hamming_threshold)); + let pairwise = pairwise_hamming_distance_binary_parallel( + &hashes, + Some(&row_id_vec), + Some(hamming_threshold), + ); // Cluster edges let clustering = cluster_pairwise_result(&pairwise); @@ -599,6 +746,31 @@ mod tests { clusters } + #[test] + fn test_validate_hash_column_generic_widths() { + use arrow_schema::{DataType, Field}; + use std::sync::Arc; + + fn hash_type(size: i32) -> DataType { + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::UInt8, true)), size) + } + + assert_eq!(validate_hash_column("hash", &hash_type(8)).unwrap(), 8); + assert_eq!(validate_hash_column("hash", &hash_type(16)).unwrap(), 16); + assert_eq!(validate_hash_column("hash", &hash_type(32)).unwrap(), 32); + + let err = validate_hash_column("hash", &hash_type(12)).unwrap_err(); + assert!(err.to_string().contains("12"), "{}", err); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + + let err = validate_hash_column("hash", &hash_type(4)).unwrap_err(); + assert!(err.to_string().contains("4"), "{}", err); + assert!(err.to_string().contains("multiple of 8 bytes"), "{}", err); + + let err = validate_hash_column("hash", &hash_type(-8)).unwrap_err(); + assert!(err.to_string().contains("-8"), "{}", err); + } + #[test] fn test_hamming_clustering_from_hashes_basic() { // Create some test hashes with known distances @@ -764,6 +936,200 @@ mod tests { assert!(err.to_string().contains("not found"), "Error: {}", err); } + #[test] + fn test_validate_shared_centroids() { + use arrow_array::UInt8Array; + use lance_arrow::FixedSizeListArrayExt; + + fn model_from_bytes(bytes: Vec) -> IvfModel { + let centroids = + FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), 8).unwrap(); + IvfModel::new(centroids, None) + } + + let uuid_a = Uuid::new_v4(); + let uuid_b = Uuid::new_v4(); + + let model_a = model_from_bytes(vec![0u8; 16]); + let model_b = model_from_bytes(vec![0u8; 16]); + validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_b)]).unwrap(); + + let mut diverged = vec![0u8; 16]; + diverged[0] = 1; + let model_c = model_from_bytes(diverged); + let err = + validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_c)]).unwrap_err(); + assert!( + err.to_string() + .contains("do not share the same global IVF centroids"), + "{}", + err + ); + assert!(err.to_string().contains(&uuid_b.to_string()), "{}", err); + + let model_d = model_from_bytes(vec![0u8; 24]); + let err = + validate_shared_centroids("idx", [(uuid_a, &model_a), (uuid_b, &model_d)]).unwrap_err(); + assert!(err.to_string().contains("2 partitions"), "{}", err); + assert!(err.to_string().contains("3 partitions"), "{}", err); + + let err = validate_shared_centroids("idx", [(uuid_a, &IvfModel::empty())]).unwrap_err(); + assert!(err.to_string().contains("has no IVF centroids"), "{}", err); + } + + #[tokio::test] + async fn test_hamming_clustering_for_ivf_partition_multi_segment_128_bit() { + use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array}; + use arrow_schema::{Field, Schema}; + use lance_arrow::FixedSizeListArrayExt; + use lance_index::optimize::OptimizeOptions; + use lance_index::vector::ivf::IvfBuildParams; + use std::sync::Arc; + use tempfile::tempdir; + + const HASH_BYTES: i32 = 16; + + fn hash_batch(schema: Arc, values: &[u64]) -> arrow_array::RecordBatch { + let mut bytes = Vec::with_capacity(values.len() * HASH_BYTES as usize); + for value in values { + bytes.extend_from_slice(&value.to_le_bytes()); + let high = value.rotate_left(17) ^ 0xA5A5_A5A5_A5A5_A5A5; + bytes.extend_from_slice(&high.to_le_bytes()); + } + let array = + FixedSizeListArray::try_new_from_values(UInt8Array::from(bytes), HASH_BYTES) + .unwrap(); + arrow_array::RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap() + } + + let schema = Arc::new(Schema::new(vec![Field::new( + "hash", + arrow_schema::DataType::FixedSizeList( + Arc::new(Field::new("item", arrow_schema::DataType::UInt8, true)), + HASH_BYTES, + ), + false, + )])); + + // 25 distinct hash values, two copies each; the same batch is written to + // fragment 0 and appended as fragment 1, so every value has duplicates + // in both fragments. + let values: Vec = (0..50) + .map(|i| ((i / 2) as u64).wrapping_mul(0x9E3779B97F4A7C15)) + .collect(); + let num_values = 25; + + let temp_dir = tempdir().unwrap(); + let uri = temp_dir.path().to_str().unwrap(); + let reader = RecordBatchIterator::new( + vec![Ok(hash_batch(schema.clone(), &values))], + schema.clone(), + ); + let mut dataset = crate::Dataset::write(reader, uri, None).await.unwrap(); + + let params = crate::index::vector::VectorIndexParams::with_ivf_flat_params( + lance_linalg::distance::MetricType::Hamming, + IvfBuildParams::new(4), + ); + dataset + .create_index( + &["hash"], + crate::index::IndexType::Vector, + Some("hash_idx".into()), + ¶ms, + false, + ) + .await + .unwrap(); + + let reader = RecordBatchIterator::new( + vec![Ok(hash_batch(schema.clone(), &values))], + schema.clone(), + ); + dataset.append(reader, None).await.unwrap(); + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + + let segments = dataset.load_indices_by_name("hash_idx").await.unwrap(); + assert_eq!( + segments.len(), + 2, + "expected a delta segment after optimize append" + ); + + // Partition sizes aggregate across both segments. + let infos = get_ivf_partition_info(&dataset, "hash_idx").await.unwrap(); + assert_eq!(infos.len(), 4); + assert_eq!(infos.iter().map(|info| info.size).sum::(), 100); + + // Clustering each partition with threshold 0 must group all four copies + // of every value, including the copies in the appended fragment. + const FRAG1_START: u64 = 1 << 32; + let mut clusters = Vec::new(); + for partition_id in 0..4 { + let reader = + hamming_clustering_for_ivf_partition(&dataset, "hash_idx", partition_id, 0) + .await + .unwrap(); + clusters.extend(collect_clusters(reader)); + } + assert_eq!(clusters.len(), num_values); + for (representative, duplicates) in &clusters { + assert_eq!(duplicates.len(), 3); + assert!(*representative < FRAG1_START); + assert!( + duplicates.iter().any(|row_id| *row_id >= FRAG1_START), + "cluster {} should contain rows from the appended fragment", + representative + ); + } + + // Selecting only the original segment reproduces the single-segment scope. + let first_segment = segments + .iter() + .find(|meta| meta.fragment_bitmap.as_ref().unwrap().contains(0)) + .unwrap(); + let mut old_clusters = Vec::new(); + for partition_id in 0..4 { + let reader = hamming_clustering_for_ivf_partition_segments( + &dataset, + "hash_idx", + &[first_segment.uuid], + partition_id, + 0, + ) + .await + .unwrap(); + old_clusters.extend(collect_clusters(reader)); + } + assert_eq!(old_clusters.len(), num_values); + for (representative, duplicates) in &old_clusters { + assert_eq!(duplicates.len(), 1); + assert!(*representative < FRAG1_START); + assert!(duplicates.iter().all(|row_id| *row_id < FRAG1_START)); + } + let infos = get_ivf_partition_info_segments(&dataset, "hash_idx", &[first_segment.uuid]) + .await + .unwrap(); + assert_eq!(infos.iter().map(|info| info.size).sum::(), 50); + + // Invalid segment selections are rejected. + let err = hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[], 0, 0) + .await + .err() + .unwrap(); + assert!(err.to_string().contains("must not be empty"), "{}", err); + let missing = Uuid::new_v4(); + let err = + hamming_clustering_for_ivf_partition_segments(&dataset, "hash_idx", &[missing], 0, 0) + .await + .err() + .unwrap(); + assert!(err.to_string().contains(&missing.to_string()), "{}", err); + } + #[tokio::test] async fn test_hamming_clustering_for_sample_integration() { use arrow_array::{FixedSizeListArray, RecordBatchIterator, UInt8Array}; diff --git a/rust/lance/src/index/vector/ivf.rs b/rust/lance/src/index/vector/ivf.rs index 9b8db75fde3..2ac0760caf6 100644 --- a/rust/lance/src/index/vector/ivf.rs +++ b/rust/lance/src/index/vector/ivf.rs @@ -4,12 +4,12 @@ //! IVF - Inverted File index. use super::{ - LogicalIvfView, + LogicalIvfView, derive_hnsw_params, pq::{PQIndex, build_pq_model}, utils::{filter_finite_training_data, maybe_sample_training_data}, }; use super::{ - builder::{IvfIndexBuilder, index_type_string}, + builder::{ExistingIndex, IvfIndexBuilder, index_type_string}, utils::PartitionLoadLock, }; use crate::dataset::index::dataset_format_version; @@ -19,7 +19,11 @@ use crate::index::vector::open_index_file; use crate::index::vector::utils::{get_vector_dim, get_vector_type}; use crate::{ dataset::Dataset, - index::{INDEX_FILE_NAME, pb, prefilter::PreFilter, vector::ivf::io::write_pq_partitions}, + index::{ + INDEX_FILE_NAME, pb, + prefilter::PreFilter, + vector::ivf::io::{write_pq_partition_payload, write_pq_partitions}, + }, }; use crate::{dataset::builder::DatasetBuilder, index::vector::IndexFileVersion}; use arrow::array::ArrayData; @@ -47,7 +51,7 @@ use lance_core::deepsize::DeepSizeOf; use lance_core::utils::row_addr_remap::RowAddrRemap; use lance_core::{ Error, ROW_ID_FIELD, Result, - cache::{LanceCache, UnsizedCacheKey, WeakLanceCache}, + cache::{CacheKeySchema, KeyBuilder, LanceCache, UnsizedCacheKey, WeakLanceCache}, traits::DatasetTakeRows, utils::parse::parse_env_as_bool, utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS}, @@ -55,14 +59,14 @@ use lance_core::{ use lance_encoding::decoder::FilterExpression; use lance_file::{ format::MAGIC, - previous::writer::{ - FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions, - }, reader::{FileReader as V2Reader, FileReaderOptions as V2ReaderOptions}, + versions as file_versions, + versions::v1::writer::{FileWriter as V1FileWriter, FileWriterOptions as V1FileWriterOptions}, writer::{FileWriter as V2Writer, FileWriterOptions as V2WriterOptions}, }; use lance_index::metrics::MetricsCollector; use lance_index::metrics::NoOpMetricsCollector; +use lance_index::prefilter::NoFilter; use lance_index::vector::DISTANCE_TYPE_KEY; use lance_index::vector::bq::builder::RabitQuantizer; use lance_index::vector::flat::index::{FlatBinQuantizer, FlatIndex, FlatMetadata, FlatQuantizer}; @@ -102,7 +106,6 @@ use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_io::utils::CachedFileSize; use lance_io::{ ReadBatchParams, - encodings::plain::PlainEncoder, local::to_local_path, object_store::ObjectStore, stream::RecordBatchStream, @@ -110,6 +113,7 @@ use lance_io::{ }; use lance_linalg::distance::{DistanceType, Dot, L2, MetricType}; use lance_linalg::{distance::Normalize, kernels::normalize_fsl_owned}; +use lance_select::RowAddrTreeMap; use lance_table::format::{IndexFile, IndexMetadata as TableIndexMetadata}; use log::{info, warn}; use object_store::path::Path; @@ -123,7 +127,7 @@ use std::{ any::Any, collections::{HashMap, HashSet}, ops::Range, - sync::Arc, + sync::{Arc, OnceLock}, }; use tokio::sync::mpsc; use tracing::instrument; @@ -157,6 +161,14 @@ impl UnsizedCacheKey for LegacyIVFPartitionKey { fn type_name() -> &'static str { "LegacyIVFPartition" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.legacy-ivf-partition-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.partition_id as u64); + } } /// IVF Index. @@ -177,12 +189,20 @@ pub struct IVFIndex { pub metric_type: MetricType, index_cache: WeakLanceCache, + partition_rows: Vec>>, } impl DeepSizeOf for IVFIndex { fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { // `Uuid` is a fixed 16-byte struct with no heap children, so contributes 0. - self.reader.deep_size_of_children(context) + self.sub_index.deep_size_of_children(context) + self.reader.deep_size_of_children(context) + + self.sub_index.deep_size_of_children(context) + + self + .partition_rows + .iter() + .filter_map(OnceLock::get) + .map(|rows| rows.deep_size_of_children(context)) + .sum::() } } @@ -212,9 +232,46 @@ impl IVFIndex { metric_type, partition_locks: PartitionLoadLock::new(num_partitions), index_cache: WeakLanceCache::from(&index_cache), + partition_rows: (0..num_partitions).map(|_| OnceLock::new()).collect(), }) } + fn cache_partition_rows( + &self, + partition_id: usize, + partition: &dyn VectorIndex, + ) -> Result> { + let rows = self.partition_rows.get(partition_id).ok_or_else(|| { + Error::index(format!( + "partition id {partition_id} is out of range of {} partitions", + self.ivf.num_partitions() + )) + })?; + Ok(rows + .get_or_init(|| Arc::new(partition.row_ids().collect())) + .clone()) + } + + fn prefilter_for_partition( + &self, + partition_id: usize, + partition: &dyn VectorIndex, + pre_filter: Arc, + ) -> Result> { + if pre_filter.is_empty() { + return Ok(Arc::new(NoFilter)); + } + if !pre_filter.needs_partition_row_ids() { + return Ok(pre_filter); + } + let rows = self.cache_partition_rows(partition_id, partition)?; + if pre_filter.is_empty_for(rows.as_ref()) { + Ok(Arc::new(NoFilter)) + } else { + Ok(pre_filter) + } + } + /// Load one partition of the IVF sub-index. /// /// Internal API with no stability guarantees. @@ -322,7 +379,7 @@ fn candidate_is_better( } } -fn index_type_for_segmented_optimize(index: &dyn VectorIndex) -> Result { +pub(crate) fn index_type_for_segmented_optimize(index: &dyn VectorIndex) -> Result { let (sub_index_type, quantization_type) = index.sub_index_type(); IndexType::try_from(index_type_string(sub_index_type, quantization_type).as_str()) } @@ -386,6 +443,203 @@ pub(crate) fn select_segment_for_single_rebalance( Ok(selected.map(|candidate| candidate.segment_id)) } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum VectorSegmentCompatibility { + SharedModel, + QueryCompatibleModelsDiffer, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum VectorModelMismatch { + StorageFormat, + IvfCentroids, + QuantizerMetadata, +} + +fn vector_index_dimension(index: &dyn VectorIndex) -> usize { + let ivf_dimension = index.ivf_model().dimension(); + if ivf_dimension != 0 { + return ivf_dimension; + } + + match index.quantizer() { + Quantizer::Flat(quantizer) => quantizer.metadata(None).dim, + Quantizer::FlatBin(quantizer) => quantizer.metadata(None).dim, + Quantizer::Product(quantizer) => quantizer.dimension, + Quantizer::Scalar(quantizer) => quantizer.metadata(None).dim, + Quantizer::Rabit(quantizer) => quantizer.metadata(None).rotated_dim(), + } +} + +fn validate_vector_query_compatibility( + indices: &[Arc], + operation: &str, +) -> Result<()> { + let Some(first) = indices.first() else { + return Ok(()); + }; + + let first_metric = first.metric_type(); + let first_dimension = vector_index_dimension(first.as_ref()); + let first_index_type = first.sub_index_type(); + let first_quantizer = first.quantizer(); + let first_quantizer_type = first_quantizer.quantization_type(); + + for (idx, index) in indices.iter().enumerate().skip(1) { + if index.metric_type() != first_metric { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has metric {:?}, expected {:?}", + index.metric_type(), + first_metric + ))); + } + let dimension = vector_index_dimension(index.as_ref()); + if dimension != first_dimension { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has dimension {dimension}, expected {first_dimension}" + ))); + } + let index_type = index.sub_index_type(); + if std::mem::discriminant(&index_type.0) != std::mem::discriminant(&first_index_type.0) + || index_type.1 != first_index_type.1 + { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has type {:?}, expected {:?}", + index_type, first_index_type + ))); + } + + let quantizer = index.quantizer(); + if quantizer.quantization_type() != first_quantizer_type { + return Err(Error::index(format!( + "{operation}: vector index segment {idx} has quantizer {:?}, expected {:?}", + quantizer.quantization_type(), + first_quantizer_type + ))); + } + } + + Ok(()) +} + +fn vector_model_mismatch(indices: &[Arc]) -> Option { + let first = indices.first()?; + let first_centroids = first.ivf_model().centroids_array(); + let first_quantizer = first.quantizer(); + + for index in indices.iter().skip(1) { + if first.as_any().type_id() != index.as_any().type_id() { + return Some(VectorModelMismatch::StorageFormat); + } + match (first_centroids, index.ivf_model().centroids_array()) { + (Some(expected), Some(actual)) if expected.to_data() != actual.to_data() => { + return Some(VectorModelMismatch::IvfCentroids); + } + (Some(_), None) | (None, Some(_)) => { + return Some(VectorModelMismatch::IvfCentroids); + } + _ => {} + } + + if !shared_quantizer_model(&first_quantizer, &index.quantizer()) { + return Some(VectorModelMismatch::QuantizerMetadata); + } + } + + None +} + +pub(crate) fn vector_segment_compatibility( + logical_index: &LogicalIvfView<'_>, + operation: &str, +) -> Result { + let indices = logical_index.indices().cloned().collect::>(); + validate_vector_query_compatibility(&indices, operation)?; + Ok(if vector_model_mismatch(&indices).is_none() { + VectorSegmentCompatibility::SharedModel + } else { + VectorSegmentCompatibility::QueryCompatibleModelsDiffer + }) +} + +fn validate_shared_vector_model(indices: &[Arc], operation: &str) -> Result<()> { + validate_vector_query_compatibility(indices, operation)?; + match vector_model_mismatch(indices) { + Some(VectorModelMismatch::StorageFormat) => Err(Error::index(format!( + "{operation}: vector index segments do not share a storage format" + ))), + Some(VectorModelMismatch::IvfCentroids) => Err(Error::index(format!( + "{operation}: vector index segments do not share IVF centroids" + ))), + Some(VectorModelMismatch::QuantizerMetadata) => Err(Error::index(format!( + "{operation}: vector index segments do not share quantizer metadata" + ))), + None => Ok(()), + } +} + +fn shared_quantizer_model(left: &Quantizer, right: &Quantizer) -> bool { + match (left, right) { + (Quantizer::Flat(left), Quantizer::Flat(right)) => { + left.metadata(None).dim == right.metadata(None).dim + } + (Quantizer::FlatBin(left), Quantizer::FlatBin(right)) => { + left.metadata(None).dim == right.metadata(None).dim + } + (Quantizer::Product(left), Quantizer::Product(right)) => { + left.num_sub_vectors == right.num_sub_vectors + && left.num_bits == right.num_bits + && left.dimension == right.dimension + && left.distance_type == right.distance_type + && left.codebook.to_data() == right.codebook.to_data() + } + (Quantizer::Scalar(left), Quantizer::Scalar(right)) => { + left.metadata(None) == right.metadata(None) + } + (Quantizer::Rabit(left), Quantizer::Rabit(right)) => { + let left = left.metadata(None); + let right = right.metadata(None); + left.rotation_type == right.rotation_type + && left.code_dim == right.code_dim + && left.num_bits == right.num_bits + && left.packed == right.packed + && left.query_estimator == right.query_estimator + && left.fast_rotation_signs == right.fast_rotation_signs + && match (&left.rotate_mat, &right.rotate_mat) { + (Some(left), Some(right)) => left.to_data() == right.to_data(), + (None, None) => true, + _ => false, + } + } + _ => false, + } +} + +/// Pair every segment with the coverage that decides which of its rows this optimize +/// pass may still copy into the new index. +/// +/// A segment that predates fragment bitmaps has unknown coverage, so it keeps every +/// row it holds. Turning coverage into a filter is deferred to the first partition +/// that actually reads the segment, because under stable row ids it costs a row-id +/// sequence load per covered fragment and most passes only append a delta. +fn existing_index_sources( + dataset: &Dataset, + logical_index: &LogicalIvfView<'_>, +) -> Vec { + logical_index + .segments() + .map(|(metadata, index)| { + let (Some(effective), Some(deleted)) = ( + metadata.effective_fragment_bitmap(&dataset.fragment_bitmap), + metadata.deleted_fragment_bitmap(&dataset.fragment_bitmap), + ) else { + return ExistingIndex::unfiltered(index.clone()); + }; + ExistingIndex::with_coverage(index.clone(), dataset.clone(), effective, deleted) + }) + .collect() +} + // TODO: move to `lance-index` crate. /// /// Returns (new_uuid, num_indices_merged, files) @@ -403,18 +657,14 @@ pub(crate) async fn optimize_vector_indices( "optimizing vector index: no existing index found".to_string(), )); } + validate_shared_vector_model(&existing_indices, "optimizing vector index")?; // try cast to v1 IVFIndex, // fallback to v2 IVFIndex if it's not v1 IVFIndex if !existing_indices[0].as_any().is::() { - return optimize_vector_indices_v2( - &dataset, - unindexed, - vector_column, - &existing_indices, - options, - ) - .await; + let sources = existing_index_sources(&dataset, logical_index); + return optimize_vector_indices_v2(&dataset, unindexed, vector_column, &sources, options) + .await; } let new_uuid = Uuid::new_v4(); @@ -483,7 +733,7 @@ pub(crate) async fn optimize_vector_indices_v2( dataset: &Dataset, unindexed: Option, vector_column: &str, - existing_indices: &[Arc], + existing_indices: &[ExistingIndex], options: &OptimizeOptions, ) -> Result<(Uuid, usize, Vec)> { // Sanity check the indices @@ -496,11 +746,12 @@ pub(crate) async fn optimize_vector_indices_v2( let new_uuid = Uuid::new_v4(); let index_dir = dataset.indices_dir().join(new_uuid.to_string()); - let ivf_model = existing_indices[0].ivf_model(); - let quantizer = existing_indices[0].quantizer(); - let distance_type = existing_indices[0].metric_type(); + let reference_index = &existing_indices[0].index; + let ivf_model = reference_index.ivf_model(); + let quantizer = reference_index.quantizer(); + let distance_type = reference_index.metric_type(); let num_partitions = ivf_model.num_partitions(); - let index_type = existing_indices[0].sub_index_type(); + let index_type = reference_index.sub_index_type(); let frag_reuse_index = dataset.open_frag_reuse_index(&NoOpMetricsCollector).await?; let format_version = dataset_format_version(dataset); @@ -526,7 +777,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -544,7 +795,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -565,7 +816,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -585,7 +836,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -605,7 +856,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -624,7 +875,7 @@ pub(crate) async fn optimize_vector_indices_v2( )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -639,13 +890,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -657,13 +908,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -678,13 +929,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -698,13 +949,13 @@ pub(crate) async fn optimize_vector_indices_v2( index_dir, distance_type, shuffler, - HnswBuildParams::default(), + derive_hnsw_params(reference_index.as_ref()), frag_reuse_index, options.clone(), )? .with_ivf(ivf_model.clone()) .with_quantizer(quantizer.try_into()?) - .with_existing_indices(existing_indices.clone()) + .with_existing_index_sources(existing_indices.clone()) .with_progress(options.progress.clone()) .shuffle_data_input(unindexed) .build() @@ -870,11 +1121,8 @@ async fn optimize_ivf_hnsw_indices( // Prepare the HNSW writer let schema = lance_core::datatypes::Schema::try_from(HNSW::schema().as_ref())?; - let mut writer = PreviousFileWriter::with_object_writer( - writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut writer = + V1FileWriter::with_object_writer(writer, schema, &V1FileWriterOptions::default())?; writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -898,11 +1146,8 @@ async fn optimize_ivf_hnsw_indices( ), ]); let schema = lance_core::datatypes::Schema::try_from(&schema)?; - let mut aux_writer = PreviousFileWriter::with_object_writer( - aux_writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut aux_writer = + V1FileWriter::with_object_writer(aux_writer, schema, &V1FileWriterOptions::default())?; aux_writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -1195,6 +1440,9 @@ impl VectorIndex for IVFIndex { metrics: &dyn MetricsCollector, ) -> Result { let part_index = self.load_partition(partition_id, true, metrics).await?; + pre_filter.wait_for_ready().await?; + let pre_filter = + self.prefilter_for_partition(partition_id, part_index.as_ref(), pre_filter)?; let query = self.preprocess_query(partition_id, query)?; let batch = part_index.search(&query, pre_filter, metrics).await?; @@ -1744,8 +1992,12 @@ impl RemapPageTask { page.pq.code_dim(), page.row_ids.as_ref().unwrap().len(), ); - PlainEncoder::write(writer, &[&original_pq]).await?; - PlainEncoder::write(writer, &[page.row_ids.as_ref().unwrap().as_ref()]).await?; + write_pq_partition_payload( + writer, + &[&original_pq], + &[page.row_ids.as_ref().unwrap().as_ref()], + ) + .await?; Ok(()) } } @@ -1892,7 +2144,7 @@ pub(crate) async fn remap_index_file( let tasks = generate_remap_tasks(&index.ivf.offsets, &index.ivf.lengths)?; - let mut task_stream = stream::iter(tasks.into_iter()) + let mut task_stream = stream::iter(tasks) .map(|task| task.load_and_remap(reader.clone(), index, mapping)) .buffered(object_store.io_parallelism()); @@ -2064,11 +2316,8 @@ async fn write_ivf_hnsw_file( let writer = object_store.create(&path).await?; let schema = lance_core::datatypes::Schema::try_from(HNSW::schema().as_ref())?; - let mut writer = PreviousFileWriter::with_object_writer( - writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut writer = + V1FileWriter::with_object_writer(writer, schema, &V1FileWriterOptions::default())?; writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -2096,11 +2345,8 @@ async fn write_ivf_hnsw_file( ), ]); let schema = lance_core::datatypes::Schema::try_from(&schema)?; - let mut aux_writer = PreviousFileWriter::with_object_writer( - aux_writer, - schema, - &PreviousFileWriterOptions::default(), - )?; + let mut aux_writer = + V1FileWriter::with_object_writer(aux_writer, schema, &V1FileWriterOptions::default())?; aux_writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -2174,14 +2420,35 @@ async fn write_ivf_hnsw_file( /// Merge one caller-defined group of source segments into a single segment. pub(crate) async fn merge_segments( - object_store: &ObjectStore, - indices_dir: &Path, + dataset: &Dataset, segments: Vec, ) -> Result { - merge_segments_with_progress( - object_store, - indices_dir, + let mut row_filters = Vec::with_capacity(segments.len()); + let no_deleted_fragments = RoaringBitmap::new(); + for segment in &segments { + let owned_fragments = segment.fragment_bitmap.as_ref().ok_or_else(|| { + Error::index(format!( + "Segment '{}' is missing fragment coverage", + segment.uuid + )) + })?; + row_filters.push( + crate::index::append::build_old_data_filter( + dataset, + owned_fragments, + &no_deleted_fragments, + ) + .await? + .ok_or_else(|| { + Error::internal("Vector segment ownership filter is missing".to_string()) + })?, + ); + } + merge_segments_with_row_filters( + dataset.object_store.as_ref(), + &dataset.indices_dir(), segments, + row_filters, lance_index::progress::noop_progress(), ) .await @@ -2189,11 +2456,38 @@ pub(crate) async fn merge_segments( /// Merge one caller-defined group of source segments into a single segment and /// report progress through the provided callback. +#[cfg(test)] pub(crate) async fn merge_segments_with_progress( object_store: &ObjectStore, indices_dir: &Path, segments: Vec, progress: Arc, +) -> Result { + let row_filters = segments + .iter() + .map(|segment| { + let to_keep = segment.fragment_bitmap.clone().ok_or_else(|| { + Error::index(format!( + "Segment '{}' is missing fragment coverage", + segment.uuid + )) + })?; + Ok(lance_index::scalar::OldIndexDataFilter::Fragments { + to_keep, + to_remove: RoaringBitmap::new(), + }) + }) + .collect::>>()?; + merge_segments_with_row_filters(object_store, indices_dir, segments, row_filters, progress) + .await +} + +async fn merge_segments_with_row_filters( + object_store: &ObjectStore, + indices_dir: &Path, + segments: Vec, + row_filters: Vec, + progress: Arc, ) -> Result { if segments.is_empty() { return Err(Error::index("No segment metadata was provided".to_string())); @@ -2213,6 +2507,16 @@ pub(crate) async fn merge_segments_with_progress( })?; fragment_bitmap |= source_fragment_bitmap.clone(); } + let mut index_details = crate::index::vector_index_details_default(); + for segment in &segments { + if let Some(details) = segment.index_details.as_deref() { + let details = details.clone(); + if !details.value.is_empty() { + index_details = details; + break; + } + } + } let index_version = infer_source_index_version(&segments)?; let segment_uuid = Uuid::new_v4(); @@ -2222,15 +2526,15 @@ pub(crate) async fn merge_segments_with_progress( indices_dir, &final_dir, &segments, + &row_filters, None, progress, ) .await?; - merged_segment = TableIndexMetadata { uuid: segment_uuid, fragment_bitmap: Some(fragment_bitmap), - index_details: Some(Arc::new(crate::index::vector_index_details_default())), + index_details: Some(Arc::new(index_details)), index_version, created_at: Some(chrono::Utc::now()), base_id: None, @@ -2250,6 +2554,7 @@ async fn merge_segments_to_dir( indices_dir: &Path, final_dir: &Path, segments: &[TableIndexMetadata], + row_filters: &[lance_index::scalar::OldIndexDataFilter], _requested_index_type: Option, progress: Arc, ) -> Result> { @@ -2278,15 +2583,14 @@ async fn merge_segments_to_dir( .join(INDEX_FILE_NAME) }) .collect::>(); - - let auxiliary_file = - lance_index::vector::distributed::index_merger::merge_partial_vector_auxiliary_files( - object_store, - &aux_paths, - final_dir, - progress.clone(), - ) - .await?; + let auxiliary_file = lance_index::vector::distributed::index_merger::merge_partial_vector_auxiliary_files_with_row_filters( + object_store, + &aux_paths, + final_dir, + row_filters, + progress.clone(), + ) + .await?; let index_file = write_root_vector_index_from_auxiliary( object_store, final_dir, @@ -2441,13 +2745,11 @@ async fn write_root_vector_index_from_auxiliary( // Schema for HNSW sub-index: include neighbors/dist fields; empty batch is fine. let arrow_schema = HNSW::schema(); let schema = lance_core::datatypes::Schema::try_from(arrow_schema.as_ref())?; - let mut v2_writer = V2Writer::try_new( + let mut v2_writer = file_versions::create_writer( + format_version, obj_writer, schema, - V2WriterOptions { - format_version: Some(format_version), - ..Default::default() - }, + V2WriterOptions::default(), )?; // For HNSW variants, attach per-partition metadata list; for FLAT-based @@ -4267,19 +4569,26 @@ async fn train_streaming_coreset_ivf_model( let coreset_len = coreset.len(); let (coreset_data, coreset_weights, coreset_losses) = coreset.into_fsl_parts(dimension)?; - let weighted_hierarchical_params = WeightedHierarchicalKMeansParams { - dimension, - target_k: num_partitions, - metric_type: DistanceType::L2, - max_iters: params.max_iters, - on_progress: on_progress.clone(), + // Scope `weighted_hierarchical_params` so the `on_progress` clone it holds + // (which owns a clone of `progress_tx`) is dropped as soon as training + // returns. Otherwise it would outlive the `progress_worker.await` below, + // keeping a channel sender alive so `progress_rx.recv()` never returns + // `None` and the progress worker — and thus this function — hangs forever. + let mut centroids = { + let weighted_hierarchical_params = WeightedHierarchicalKMeansParams { + dimension, + target_k: num_partitions, + metric_type: DistanceType::L2, + max_iters: params.max_iters, + on_progress: on_progress.clone(), + }; + train_weighted_hierarchical_f32_kmeans( + &coreset_data, + &coreset_weights, + &coreset_losses, + &weighted_hierarchical_params, + )? }; - let mut centroids = train_weighted_hierarchical_f32_kmeans( - &coreset_data, - &coreset_weights, - &coreset_losses, - &weighted_hierarchical_params, - )?; let refine_iters = 3; if refine_iters > 0 { let refined = refine_weighted_f32_kmeans( @@ -4559,6 +4868,7 @@ mod tests { use lance_datagen::{ArrayGeneratorExt, BatchCount, Dimension, RowCount, array, gen_batch}; use lance_index::VECTOR_INDEX_VERSION; use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::OldIndexDataFilter; use lance_index::vector::sq::builder::SQBuildParams; use lance_linalg::distance::l2_distance_batch; use lance_testing::datagen::{ @@ -4577,6 +4887,152 @@ mod tests { const DIM: usize = 32; + /// Building a merge filter loads a row-id sequence per covered fragment under + /// stable row ids, and an optimize pass that only appends a delta reads no existing + /// row at all. Such a pass must therefore build no filter, and a pass that does + /// merge must build one and reuse it across partitions. + #[tokio::test] + async fn test_optimize_builds_merge_filters_only_when_merging() { + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let make_batch = || { + gen_batch() + .col( + "vector", + array::rand_vec::(Dimension::from(DIM as u32)), + ) + .into_batch_rows(RowCount::from(256)) + .unwrap() + }; + let batch = make_batch(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), + test_uri, + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + // A single partition keeps `check_partition_adjustment` from selecting a split + // or a join, which would legitimately merge every segment. + dataset + .create_index( + &["vector"], + IndexType::Vector, + None, + &VectorIndexParams::ivf_flat(1, MetricType::L2), + true, + ) + .await + .unwrap(); + dataset + .append( + RecordBatchIterator::new(vec![Ok(make_batch())], schema), + None, + ) + .await + .unwrap(); + + let logical_index = dataset + .open_logical_vector_index("vector", "vector_idx") + .await + .unwrap(); + let ivf_view = logical_index.as_ivf().unwrap(); + let new_data = |fragments| async { + let mut scanner = dataset.scan(); + scanner + .with_fragments(fragments) + .with_row_id() + .project(&["vector"]) + .unwrap(); + scanner.try_into_stream().await.unwrap() + }; + let unindexed = dataset.unindexed_fragments("vector_idx").await.unwrap(); + assert_eq!( + unindexed.len(), + 1, + "the appended fragment must be unindexed" + ); + + // `ExistingIndex` shares its coverage behind an `Arc`, so the sources handed to + // the builder report what the builder actually did with them. + let sources = existing_index_sources(&dataset, &ivf_view); + optimize_vector_indices_v2( + &dataset, + Some(new_data(unindexed.clone()).await), + "vector", + &sources, + &OptimizeOptions::new(), + ) + .await + .unwrap(); + assert!( + sources.iter().all(|source| !source.filter_is_built()), + "a delta append reads no existing row, so it must build no filter" + ); + + let sources = existing_index_sources(&dataset, &ivf_view); + optimize_vector_indices_v2( + &dataset, + Some(new_data(unindexed).await), + "vector", + &sources, + &OptimizeOptions::merge(1), + ) + .await + .unwrap(); + assert!( + sources.iter().all(|source| source.filter_is_built()), + "a merge reads existing rows, so it must build a filter per merged segment" + ); + assert!( + matches!( + sources[0].old_data_filter().await.unwrap(), + Some(OldIndexDataFilter::RowIds(_)) + ), + "a stable-row-id segment must filter on exact row-id membership" + ); + } + + #[test] + fn test_shared_quantizer_model_compares_skipped_payloads() { + let codebook = |offset| { + let values = Float32Array::from_iter_values((0..32).map(|v| v as f32 + offset)); + FixedSizeListArray::try_new_from_values(values, 2).unwrap() + }; + let pq1 = Quantizer::Product(ProductQuantizer::new( + 1, + 4, + 2, + codebook(0.0), + DistanceType::L2, + )); + let pq2 = Quantizer::Product(ProductQuantizer::new( + 1, + 4, + 2, + codebook(100.0), + DistanceType::L2, + )); + assert!(!shared_quantizer_model(&pq1, &pq2)); + + let rq1 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::( + 1, + 8, + lance_index::vector::bq::RQRotationType::Matrix, + )); + let rq2 = Quantizer::Rabit(RabitQuantizer::new_with_rotation::( + 1, + 8, + lance_index::vector::bq::RQRotationType::Matrix, + )); + assert!(!shared_quantizer_model(&rq1, &rq2)); + } + async fn compute_test_ivf_loss(dataset: &Dataset, column: &str, ivf: &IvfModel) -> f64 { let centroids = ivf .centroids_array() @@ -4907,7 +5363,15 @@ mod tests { test_uri: &str, range: Range, ) -> (Dataset, Arc) { - let vectors = generate_random_array_with_range::(1000 * DIM, range); + generate_test_dataset_with_rows(test_uri, range, 1000).await + } + + async fn generate_test_dataset_with_rows( + test_uri: &str, + range: Range, + num_rows: usize, + ) -> (Dataset, Arc) { + let vectors = generate_random_array_with_range::(num_rows * DIM, range); let metadata: HashMap = vec![("test".to_string(), "ivf_pq".to_string())] .into_iter() .collect(); @@ -5063,6 +5527,7 @@ mod tests { uuid, dataset_version: dataset.version().version, fields: vec![field.id], + covering_fields: vec![], name: INDEX_NAME.to_string(), fragment_bitmap: Some(dataset.fragment_bitmap.as_ref().clone()), index_details: Some(Arc::new(vector_index_details_default())), @@ -5102,6 +5567,7 @@ mod tests { uuid, dataset_version: 0, fields: Vec::new(), + covering_fields: vec![], name: INDEX_NAME.to_string(), fragment_bitmap: None, index_details: Some(Arc::new(vector_index_details_default())), @@ -5161,6 +5627,7 @@ mod tests { uuid: new_uuid, dataset_version: dataset_mut.version().version, fields: vec![field.id], + covering_fields: vec![], name: format!("{}_remapped", INDEX_NAME), fragment_bitmap: Some(dataset_mut.fragment_bitmap.as_ref().clone()), index_details: Some(Arc::new(vector_index_details_default())), @@ -5235,6 +5702,32 @@ mod tests { } } + fn fast_ivf_params(num_partitions: usize) -> IvfBuildParams { + IvfBuildParams { + num_partitions: Some(num_partitions), + max_iters: 2, + sample_rate: 2, + ..Default::default() + } + } + + fn fast_pq_params(num_sub_vectors: usize, num_bits: usize) -> PQBuildParams { + PQBuildParams { + num_sub_vectors, + num_bits, + max_iters: 2, + sample_rate: 2, + ..Default::default() + } + } + + fn fast_hnsw_params() -> HnswBuildParams { + HnswBuildParams::default() + .max_level(3) + .num_edges(8) + .ef_construction(32) + } + // Clippy doesn't like that all start with Ivf but we might have some in the future // that _don't_ start with Ivf so I feel it is meaningful to keep the prefix #[allow(clippy::enum_variant_names)] @@ -5273,13 +5766,13 @@ mod tests { num_partitions: 2, metric_type: MetricType::Dot, dimension: 16, - index_type: TestIndexType::IvfHnswPq { pq: TestPqParams::small(), num_edges: 100 }, + index_type: TestIndexType::IvfHnswPq { pq: TestPqParams::small(), num_edges: 4 }, })] #[case::ivf_hnsw_sq(CreateIndexCase { metric_type: MetricType::Dot, num_partitions: 2, dimension: 16, - index_type: TestIndexType::IvfHnswSq { num_edges: 100 }, + index_type: TestIndexType::IvfHnswSq { num_edges: 4 }, })] async fn test_create_index_nulls( #[case] test_case: CreateIndexCase, @@ -5291,36 +5784,37 @@ mod tests { let mut index_params = match test_case.index_type { TestIndexType::IvfPq { pq } => VectorIndexParams::with_ivf_pq_params( test_case.metric_type, - IvfBuildParams::new(test_case.num_partitions), - PQBuildParams::new(pq.num_sub_vectors, pq.num_bits), + fast_ivf_params(test_case.num_partitions), + fast_pq_params(pq.num_sub_vectors, pq.num_bits), ), TestIndexType::IvfHnswPq { pq, num_edges } => { VectorIndexParams::with_ivf_hnsw_pq_params( test_case.metric_type, - IvfBuildParams::new(test_case.num_partitions), - HnswBuildParams::default().num_edges(num_edges), - PQBuildParams::new(pq.num_sub_vectors, pq.num_bits), + fast_ivf_params(test_case.num_partitions), + fast_hnsw_params().num_edges(num_edges), + fast_pq_params(pq.num_sub_vectors, pq.num_bits), ) } - TestIndexType::IvfFlat => { - VectorIndexParams::ivf_flat(test_case.num_partitions, test_case.metric_type) - } + TestIndexType::IvfFlat => VectorIndexParams::with_ivf_flat_params( + test_case.metric_type, + fast_ivf_params(test_case.num_partitions), + ), TestIndexType::IvfHnswSq { num_edges } => VectorIndexParams::with_ivf_hnsw_sq_params( test_case.metric_type, - IvfBuildParams::new(test_case.num_partitions), - HnswBuildParams::default().num_edges(num_edges), + fast_ivf_params(test_case.num_partitions), + fast_hnsw_params().num_edges(num_edges), SQBuildParams::default(), ), }; index_params.version(index_version); - let nrows = 2_000; + let nrows = 512_usize; let data = gen_batch() .col( "vec", array::rand_vec::(Dimension::from(test_case.dimension as u32)), ) - .into_batch_rows(RowCount::from(nrows)) + .into_batch_rows(RowCount::from(nrows as u64)) .unwrap(); // Make every other row null @@ -5353,9 +5847,9 @@ mod tests { .collect::(); let results = dataset .scan() - .nearest("vec", &query, 2_000) + .nearest("vec", &query, nrows) .unwrap() - .ef(100_000) + .ef(nrows) .minimum_nprobes(2) .try_into_batch() .await @@ -5364,10 +5858,10 @@ mod tests { if is_approximate { let recall = results.num_rows() as f32 / num_non_null as f32; assert!( - recall >= 0.99, + recall >= 0.5, "Recall {} below threshold {} ({}/{})", recall, - 0.99, + 0.5, results.num_rows(), num_non_null, ); @@ -5661,6 +6155,94 @@ mod tests { ); } + /// Regression test for a hang in the streaming *coreset* trainer + /// (`train_streaming_coreset_ivf_model`, taken when `num_partitions > 256`). + /// + /// That function spawns a `progress_worker` task that loops on + /// `progress_rx.recv()` and only terminates once every clone of the mpsc + /// sender is dropped. The `on_progress` closure owns a sender clone, and + /// `WeightedHierarchicalKMeansParams` used to retain an `on_progress` clone + /// that outlived the `progress_worker.await` at the end of the function. + /// With a live sender remaining, `recv()` never returned `None`, the worker + /// never finished, and the trainer hung forever after all compute was done. + /// + /// The build is wrapped in a timeout so the regression fails fast rather + /// than hanging the test process indefinitely. + #[tokio::test(flavor = "multi_thread")] + async fn test_streaming_coreset_ivf_training_terminates() { + use lance_index::progress::IndexBuildProgress; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::Duration; + + #[derive(Debug, Default)] + struct CountingProgress { + progress_calls: AtomicU64, + } + + #[async_trait::async_trait] + impl IndexBuildProgress for CountingProgress { + async fn stage_start(&self, _: &str, _: Option, _: &str) -> Result<()> { + Ok(()) + } + async fn stage_progress(&self, _: &str, _: u64) -> Result<()> { + self.progress_calls.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + async fn stage_complete(&self, _: &str) -> Result<()> { + Ok(()) + } + } + + const SMALL_DIM: usize = 8; + + let test_dir = TempStrDir::default(); + let uri = format!("{}/ds", test_dir.as_str()); + let reader = gen_batch() + .col("id", array::step::()) + .col( + "vector", + array::rand_vec::((SMALL_DIM as u32).into()), + ) + .into_reader_rows(RowCount::from(2048), BatchCount::from(4)); + let dataset = Dataset::write(reader, &uri, None).await.unwrap(); + + // > 256 partitions routes through `train_streaming_coreset_ivf_model`. + let mut params = IvfBuildParams::new(257); + params.sample_rate = 8; + params.streaming_sample_rate = Some(4); + params.streaming_refine_passes = 1; + params.max_iters = 2; + + let progress = Arc::new(CountingProgress::default()); + + let ivf_model = tokio::time::timeout( + Duration::from_secs(120), + build_ivf_model( + &dataset, + "vector", + SMALL_DIM, + MetricType::L2, + ¶ms, + None, + progress.clone(), + ), + ) + .await + .expect( + "streaming coreset IVF training hung: progress worker never terminated after training", + ) + .unwrap(); + + assert_eq!(ivf_model.num_partitions(), 257); + assert_eq!(ivf_model.dimension(), SMALL_DIM); + // The progress worker must have processed reports and then joined + // cleanly (proven by `build_ivf_model` returning at all). + assert!( + progress.progress_calls.load(Ordering::Relaxed) > 0, + "expected the progress worker to receive at least one report" + ); + } + #[test] fn test_fixed_training_ranges_are_sorted_and_bounded() { let ranges = generate_fixed_training_ranges(10_000, 1_234, 1_024, 16); @@ -6063,12 +6645,13 @@ mod tests { let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - let nlist = 4; - let (mut dataset, vector_array) = generate_test_dataset(test_uri, 0.0..1.0).await; + let nlist = 2; + let (mut dataset, vector_array) = + generate_test_dataset_with_rows(test_uri, 0.0..1.0, 512).await; - let ivf_params = IvfBuildParams::new(nlist); - let pq_params = PQBuildParams::default(); - let hnsw_params = HnswBuildParams::default(); + let ivf_params = fast_ivf_params(nlist); + let pq_params = fast_pq_params(4, 8); + let hnsw_params = fast_hnsw_params(); let params = VectorIndexParams::with_ivf_hnsw_pq_params( MetricType::L2, ivf_params, @@ -6083,23 +6666,20 @@ mod tests { let query = vector_array.value(0); let query = query.as_primitive::(); - let k = 100; + let k = 20; let results = dataset .scan() .with_row_id() .nearest("vector", query, k) .unwrap() .minimum_nprobes(nlist) - .try_into_stream() - .await - .unwrap() - .try_collect::>() + .ef(64) + .try_into_batch() .await .unwrap(); - assert_eq!(1, results.len()); - assert_eq!(k, results[0].num_rows()); + assert_eq!(k, results.num_rows()); - let row_ids = results[0] + let row_ids = results .column_by_name(ROW_ID) .unwrap() .as_any() @@ -6108,7 +6688,7 @@ mod tests { .iter() .map(|v| v.unwrap() as u32) .collect::>(); - let dists = results[0] + let dists = results .column_by_name("_distance") .unwrap() .as_any() @@ -6122,10 +6702,19 @@ mod tests { let results_set = results.iter().map(|r| r.1).collect::>(); let gt_set = gt.iter().map(|r| r.1).collect::>(); + assert_eq!(results_set.len(), k, "search returned duplicate row ids"); + assert!( + results.iter().all(|(distance, _)| distance.is_finite()), + "search returned a non-finite distance: {results:?}" + ); + assert!( + results.windows(2).all(|pair| pair[0].0 <= pair[1].0), + "search distances are not sorted: {results:?}" + ); let recall = results_set.intersection(>_set).count() as f32 / k as f32; assert!( - recall >= 0.9, + recall >= 0.5, "recall: {}\n results: {:?}\n\ngt: {:?}", recall, results, diff --git a/rust/lance/src/index/vector/ivf/builder.rs b/rust/lance/src/index/vector/ivf/builder.rs index 9bd1ba95803..d1961071315 100644 --- a/rust/lance/src/index/vector/ivf/builder.rs +++ b/rust/lance/src/index/vector/ivf/builder.rs @@ -13,7 +13,8 @@ use futures::{StreamExt, TryStreamExt}; use lance_arrow::{RecordBatchExt, SchemaExt}; use lance_core::utils::address::RowAddress; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; +use lance_file::versions as file_versions; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_file::writer::FileWriterOptions; use lance_index::vector::PART_ID_COLUMN; use lance_index::vector::pq::ProductQuantizer; @@ -35,6 +36,7 @@ use lance_linalg::distance::{DistanceType, MetricType}; use crate::Dataset; use crate::dataset::builder::DatasetBuilder; +use crate::dataset::index::dataset_format_version; use crate::index::vector::ivf::io::write_pq_partitions; use super::io::write_hnsw_quantization_index_partitions; @@ -217,7 +219,11 @@ pub async fn write_vector_storage( data.boxed() }; - let mut writer = lance_file::writer::FileWriter::new_lazy(writer, FileWriterOptions::default()); + let mut writer = file_versions::create_lazy_writer( + dataset_format_version(dataset), + writer, + FileWriterOptions::default(), + )?; let mut transformed_stream = data .map_ok(move |batch| { let ivf_transformer = ivf_transformer.clone(); @@ -242,8 +248,8 @@ pub async fn write_vector_storage( #[instrument(level = "debug", skip(writer, auxiliary_writer, data, ivf, quantizer))] pub(super) async fn build_hnsw_partitions( dataset: Arc, - writer: &mut PreviousFileWriter, - auxiliary_writer: Option<&mut PreviousFileWriter>, + writer: &mut V1FileWriter, + auxiliary_writer: Option<&mut V1FileWriter>, data: impl RecordBatchStream + Unpin + 'static, column: &str, ivf: &mut IvfModel, diff --git a/rust/lance/src/index/vector/ivf/io.rs b/rust/lance/src/index/vector/ivf/io.rs index 612dd162281..fd6c0585ee5 100644 --- a/rust/lance/src/index/vector/ivf/io.rs +++ b/rust/lance/src/index/vector/ivf/io.rs @@ -10,9 +10,10 @@ use super::IVFIndex; use crate::dataset::ROW_ID; use crate::index::vector::pq::{PQIndex, build_pq_storage}; use arrow::compute::concat; -use arrow_array::UInt64Array; use arrow_array::{ - Array, FixedSizeListArray, RecordBatch, UInt32Array, cast::AsArray, types::UInt64Type, + Array, FixedSizeListArray, PrimitiveArray, RecordBatch, UInt32Array, UInt64Array, + cast::AsArray, + types::{ArrowPrimitiveType, UInt8Type, UInt64Type}, }; use futures::stream::Peekable; use futures::{Stream, StreamExt, TryStreamExt}; @@ -22,10 +23,9 @@ use lance_core::datatypes::Schema; use lance_core::traits::DatasetTakeRows; use lance_core::utils::tempfile::TempStdDir; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; -use lance_file::previous::reader::FileReader as PreviousFileReader; -use lance_file::previous::writer::FileWriter as PreviousFileWriter; +use lance_file::versions::v1::reader::FileReader as V1FileReader; +use lance_file::versions::v1::writer::FileWriter as V1FileWriter; use lance_index::metrics::NoOpMetricsCollector; -use lance_index::scalar::IndexWriter; use lance_index::vector::hnsw::HNSW; use lance_index::vector::hnsw::{HnswMetadata, builder::HnswBuildParams}; use lance_index::vector::ivf::storage::IvfModel; @@ -35,7 +35,6 @@ use lance_index::vector::quantizer::{Quantization, Quantizer}; use lance_index::vector::v3::subindex::IvfSubIndex; use lance_index::vector::{PART_ID_COLUMN, PQ_CODE_COLUMN}; use lance_io::ReadBatchParams; -use lance_io::encodings::plain::PlainEncoder; use lance_io::object_store::ObjectStore; use lance_io::traits::Writer; use lance_linalg::distance::{DistanceType, MetricType}; @@ -43,7 +42,9 @@ use lance_linalg::kernels::normalize_fsl; use lance_table::format::SelfDescribingFileReader; use lance_table::io::manifest::ManifestDescribing; use object_store::path::Path; +use tokio::io::AsyncWriteExt; use tokio::sync::Semaphore; +use tokio::task::JoinHandle; use crate::Result; @@ -51,6 +52,75 @@ use crate::Result; static HNSW_PARTITIONS_BUILD_PARALLEL: LazyLock = LazyLock::new(get_num_compute_intensive_cpus); +async fn write_primitive_values( + writer: &mut dyn Writer, + array: &PrimitiveArray, +) -> Result<()> { + let data = array.to_data(); + let byte_width = std::mem::size_of::(); + let start = array.offset() * byte_width; + let end = start + array.len() * byte_width; + writer + .write_all(&data.buffers()[0].as_slice()[start..end]) + .await?; + Ok(()) +} + +async fn write_pq_codes(writer: &mut dyn Writer, arrays: &[&dyn Array]) -> Result<()> { + for array in arrays { + // Loaded legacy partitions expose transposed codes as flat UInt8 arrays, + // while newly shuffled partitions supply FixedSizeList. + if let Some(values) = array.as_any().downcast_ref::>() { + write_primitive_values(writer, values).await?; + } else if let Some(codes) = array.as_any().downcast_ref::() { + let value_offset = codes.value_offset(0) as usize; + let value_len = codes.len() * codes.value_length() as usize; + let values = codes.values().slice(value_offset, value_len); + let values = values + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + Error::index(format!( + "legacy IVF PQ code values must be UInt8, found {}", + values.data_type() + )) + })?; + write_primitive_values(writer, values).await?; + } else { + return Err(Error::index(format!( + "legacy IVF PQ codes must be UInt8 or FixedSizeList, found {}", + array.data_type() + ))); + } + } + Ok(()) +} + +async fn write_row_ids(writer: &mut dyn Writer, arrays: &[&dyn Array]) -> Result<()> { + for array in arrays { + let row_ids = array + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + Error::index(format!( + "legacy IVF row ids must be UInt64, found {}", + array.data_type() + )) + })?; + write_primitive_values(writer, row_ids).await?; + } + Ok(()) +} + +pub(super) async fn write_pq_partition_payload( + writer: &mut dyn Writer, + pq_codes: &[&dyn Array], + row_ids: &[&dyn Array], +) -> Result<()> { + write_pq_codes(writer, pq_codes).await?; + write_row_ids(writer, row_ids).await +} + /// Merge streams with the same partition id and collect PQ codes and row IDs. async fn merge_streams( streams_heap: &mut BinaryHeap<(Reverse, usize)>, @@ -222,10 +292,8 @@ pub(super) async fn write_pq_partitions( ivf.add_partition_with_offset(writer.tell().await?, total_records as u32); if total_records > 0 { let pq_refs = pq_array.iter().map(|a| a.as_ref()).collect::>(); - PlainEncoder::write(writer, &pq_refs).await?; - let row_ids_refs = row_id_array.iter().map(|a| a.as_ref()).collect::>(); - PlainEncoder::write(writer, row_ids_refs.as_slice()).await?; + write_pq_partition_payload(writer, &pq_refs, &row_ids_refs).await?; } log::info!( "Wrote partition {} in {} ms", @@ -242,8 +310,8 @@ pub(super) async fn write_hnsw_quantization_index_partitions( column: &str, distance_type: DistanceType, hnsw_params: &HnswBuildParams, - writer: &mut PreviousFileWriter, - mut auxiliary_writer: Option<&mut PreviousFileWriter>, + writer: &mut V1FileWriter, + mut auxiliary_writer: Option<&mut V1FileWriter>, ivf: &mut IvfModel, quantizer: Quantizer, streams: Option>>>, @@ -278,164 +346,234 @@ pub(super) async fn write_hnsw_quantization_index_partitions( } let object_store = ObjectStore::local(); - let mut part_files = Vec::with_capacity(ivf.num_partitions()); - let mut aux_part_files = Vec::with_capacity(ivf.num_partitions()); - let tmp_part_dir = Path::from_filesystem_path(TempStdDir::default())?; - let mut tasks = Vec::with_capacity(ivf.num_partitions()); - let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL)); - for part_id in 0..ivf.num_partitions() { - part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id))); - aux_part_files.push( - tmp_part_dir - .clone() - .join(format!("hnsw_part_aux_{}", part_id)), - ); + // Partitions are staged in this scratch dir, then merged into the final index. + // Share the guard with every task via `Arc` so its `Drop` removes the dir only + // after the last task finishes -- never while one is still writing. + let tmp_part_dir_guard = Arc::new(TempStdDir::default()); + let tmp_part_dir = Path::from_filesystem_path(&**tmp_part_dir_guard)?; - let mut code_array: Vec> = vec![]; - let mut row_id_array: Vec> = vec![]; + // `Option` per handle so the consume loop can `take()` each one, leaving the + // not-yet-consumed handles for the error-path drain. + let mut tasks: Vec>>> = + Vec::with_capacity(ivf.num_partitions()); - // We don't transform vectors to SQ codes while shuffling, - // so we won't merge SQ codes from the stream. + let build_result: Result<(Vec, IvfModel)> = async { + let mut part_files = Vec::with_capacity(ivf.num_partitions()); + let mut aux_part_files = Vec::with_capacity(ivf.num_partitions()); + let sem = Arc::new(Semaphore::new(*HNSW_PARTITIONS_BUILD_PARALLEL)); + for part_id in 0..ivf.num_partitions() { + part_files.push(tmp_part_dir.clone().join(format!("hnsw_part_{}", part_id))); + aux_part_files.push( + tmp_part_dir + .clone() + .join(format!("hnsw_part_aux_{}", part_id)), + ); - if let Some(&previous_indices) = existing_indices.as_ref() { - for &idx in previous_indices.iter() { - let sub_index = idx - .load_partition(part_id, true, &NoOpMetricsCollector) - .await?; - let row_ids = Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned())); - row_id_array.push(row_ids); - } - } + let mut code_array: Vec> = vec![]; + let mut row_id_array: Vec> = vec![]; - let code_column = match &quantizer { - Quantizer::Product(pq) => Some(pq.column()), - _ => None, - }; - merge_streams( - &mut streams_heap, - &mut new_streams, - part_id as u32, - code_column, - &mut code_array, - &mut row_id_array, - ) - .await?; + // We don't transform vectors to SQ codes while shuffling, + // so we won't merge SQ codes from the stream. - if row_id_array.is_empty() { - tasks.push(tokio::spawn(async { Ok(0) })); - continue; - } + if let Some(&previous_indices) = existing_indices.as_ref() { + for &idx in previous_indices.iter() { + let sub_index = idx + .load_partition(part_id, true, &NoOpMetricsCollector) + .await?; + let row_ids = + Arc::new(UInt64Array::from_iter_values(sub_index.row_ids().cloned())); + row_id_array.push(row_ids); + } + } - let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); - let part_writer = PreviousFileWriter::::try_new( - &object_store, - part_file, - Schema::try_from(writer.schema())?, - &Default::default(), - ) - .await?; + let code_column = match &quantizer { + Quantizer::Product(pq) => Some(pq.column()), + _ => None, + }; + merge_streams( + &mut streams_heap, + &mut new_streams, + part_id as u32, + code_column, + &mut code_array, + &mut row_id_array, + ) + .await?; - let aux_part_writer = match auxiliary_writer.as_ref() { - Some(writer) => Some( - PreviousFileWriter::::try_new( - &object_store, - aux_part_file, - Schema::try_from(writer.schema())?, - &Default::default(), - ) - .await?, - ), - None => None, - }; + if row_id_array.is_empty() { + tasks.push(Some(tokio::spawn(async { Ok(0) }))); + continue; + } - let dataset = dataset.clone(); - let column = column.to_owned(); - let hnsw_params = hnsw_params.clone(); - let quantizer = quantizer.clone(); - let sem = sem.clone(); - tasks.push(tokio::spawn(async move { - let _permit = sem.acquire().await.expect("semaphore error"); - - log::debug!("Building HNSW partition {}", part_id); - let result = build_hnsw_quantization_partition( - dataset, - &column, - distance_type, - hnsw_params, - part_writer, - aux_part_writer, - quantizer, - row_id_array, - code_array, + let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); + let part_writer = V1FileWriter::::try_new( + &object_store, + part_file, + Schema::try_from(writer.schema())?, + &Default::default(), ) - .await; - log::debug!("Finished building HNSW partition {}", part_id); - result - })); - } + .await?; - let mut aux_ivf = IvfModel::empty(); - let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions()); - for (part_id, task) in tasks.into_iter().enumerate() { - let offset = writer.len(); - let num_rows = task.await??; + let aux_part_writer = match auxiliary_writer.as_ref() { + Some(writer) => Some( + V1FileWriter::::try_new( + &object_store, + aux_part_file, + Schema::try_from(writer.schema())?, + &Default::default(), + ) + .await?, + ), + None => None, + }; - if num_rows == 0 { - ivf.add_partition(0); - aux_ivf.add_partition(0); - hnsw_metadata.push(HnswMetadata::default()); - continue; + let dataset = dataset.clone(); + let column = column.to_owned(); + let hnsw_params = hnsw_params.clone(); + let quantizer = quantizer.clone(); + let sem = sem.clone(); + let tmp_part_dir_guard = tmp_part_dir_guard.clone(); + tasks.push(Some(tokio::spawn(async move { + // Hold a guard clone so the scratch dir stays alive while this task writes. + let _tmp_part_dir_guard = tmp_part_dir_guard; + let _permit = sem.acquire().await.map_err(|err| { + Error::io(format!( + "failed to acquire HNSW partition build permit: {err}" + )) + })?; + + log::debug!("Building HNSW partition {}", part_id); + let result = build_hnsw_quantization_partition( + dataset, + &column, + distance_type, + hnsw_params, + part_writer, + aux_part_writer, + quantizer, + row_id_array, + code_array, + ) + .await; + log::debug!("Finished building HNSW partition {}", part_id); + result + }))); } - let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); - let part_reader = - PreviousFileReader::try_new_self_described(&object_store, part_file, None).await?; + let mut aux_ivf = IvfModel::empty(); + let mut hnsw_metadata = Vec::with_capacity(ivf.num_partitions()); + for (part_id, task) in tasks.iter_mut().enumerate() { + let task = task + .take() + .expect("each partition task is consumed exactly once"); + let offset = writer.len(); + let num_rows = task.await??; - let batches = futures::stream::iter(0..part_reader.num_batches()) - .map(|batch_id| { - part_reader.read_batch( - batch_id as i32, - ReadBatchParams::RangeFull, - part_reader.schema(), - ) - }) - .buffered(object_store.io_parallelism()) - .try_collect::>() - .await?; - writer.write(&batches).await?; - - ivf.add_partition((writer.len() - offset) as u32); - hnsw_metadata.push(serde_json::from_str( - part_reader.schema().metadata[HNSW::metadata_key()].as_str(), - )?); - std::mem::drop(part_reader); - object_store.delete(part_file).await?; - - if let Some(aux_writer) = auxiliary_writer.as_mut() { - let aux_part_reader = - PreviousFileReader::try_new_self_described(&object_store, aux_part_file, None) - .await?; + if num_rows == 0 { + ivf.add_partition(0); + aux_ivf.add_partition(0); + hnsw_metadata.push(HnswMetadata::default()); + continue; + } + + let (part_file, aux_part_file) = (&part_files[part_id], &aux_part_files[part_id]); + let part_reader = + V1FileReader::try_new_self_described(&object_store, part_file, None).await?; - let batches = futures::stream::iter(0..aux_part_reader.num_batches()) + let batches = futures::stream::iter(0..part_reader.num_batches()) .map(|batch_id| { - aux_part_reader.read_batch( + part_reader.read_batch( batch_id as i32, ReadBatchParams::RangeFull, - aux_part_reader.schema(), + part_reader.schema(), ) }) .buffered(object_store.io_parallelism()) .try_collect::>() .await?; - std::mem::drop(aux_part_reader); - object_store.delete(aux_part_file).await?; + writer.write(&batches).await?; + + ivf.add_partition((writer.len() - offset) as u32); + hnsw_metadata.push(serde_json::from_str( + part_reader.schema().metadata[HNSW::metadata_key()].as_str(), + )?); + std::mem::drop(part_reader); + object_store.delete(part_file).await?; + + if let Some(aux_writer) = auxiliary_writer.as_mut() { + let aux_part_reader = + V1FileReader::try_new_self_described(&object_store, aux_part_file, None) + .await?; + + let batches = futures::stream::iter(0..aux_part_reader.num_batches()) + .map(|batch_id| { + aux_part_reader.read_batch( + batch_id as i32, + ReadBatchParams::RangeFull, + aux_part_reader.schema(), + ) + }) + .buffered(object_store.io_parallelism()) + .try_collect::>() + .await?; + std::mem::drop(aux_part_reader); + object_store.delete(aux_part_file).await?; + + aux_writer.write(&batches).await?; + aux_ivf.add_partition(num_rows as u32); + } + } - aux_writer.write(&batches).await?; - aux_ivf.add_partition(num_rows as u32); + Ok((hnsw_metadata, aux_ivf)) + } + .await; + + // On error, abort and await the partition builds we never consumed so none + // keep running in the background; see `drain_partition_tasks`. + if build_result.is_err() { + for err in drain_partition_tasks(&mut tasks).await { + log::warn!( + "HNSW partition build task failed while draining after an earlier error: {err}" + ); + } + } + + build_result +} + +/// Abort and await every still-outstanding partition-build task, returning the +/// non-cancellation errors they surfaced. +/// +/// A dropped [`JoinHandle`] detaches its task, so every handle is aborted up front +/// before any is awaited: otherwise a task slow to observe its own cancellation +/// would keep running -- and keep writing into the scratch dir -- while an earlier +/// handle is still being awaited. Awaiting then resolves only once each task has +/// actually stopped. Cancellation errors are the expected result of the abort and +/// dropped; failures and panics from tasks that had already finished before the +/// abort are returned so the caller can surface them (a task still in flight is +/// cancelled, so this best-effort drain only reports errors already produced). +async fn drain_partition_tasks(tasks: &mut [Option>>]) -> Vec { + for task in tasks.iter() { + if let Some(handle) = task.as_ref() { + handle.abort(); } } - Ok((hnsw_metadata, aux_ivf)) + let mut errors = Vec::with_capacity(tasks.len()); + for task in tasks.iter_mut() { + let Some(handle) = task.take() else { + continue; + }; + match handle.await { + Ok(Ok(_)) => {} + Ok(Err(e)) => errors.push(e), + Err(join_err) if join_err.is_cancelled() => {} + Err(join_err) => errors.push(Error::io(format!( + "HNSW partition build task panicked: {join_err}" + ))), + } + } + errors } #[allow(clippy::too_many_arguments)] @@ -444,8 +582,8 @@ async fn build_hnsw_quantization_partition( column: &str, metric_type: MetricType, hnsw_params: Arc, - writer: PreviousFileWriter, - aux_writer: Option>, + writer: V1FileWriter, + aux_writer: Option>, quantizer: Quantizer, row_ids_array: Vec>, code_array: Vec>, @@ -473,24 +611,29 @@ async fn build_hnsw_quantization_partition( let build_hnsw = build_and_write_hnsw(vectors.clone(), (*hnsw_params).clone(), metric_type, writer); + // Build PQ storage as a child future, joined below: it writes `aux_writer`'s + // file into the scratch dir, so it is cancelled together with this task when the + // error-path drain aborts it, and the join surfaces its errors. let build_store = match quantizer { Quantizer::Flat(_) => { return Err(Error::index( "Flat quantizer is not supported for IVF_HNSW".to_string(), )); } - Quantizer::Product(pq) => tokio::spawn(build_and_write_pq_storage( - metric_type, - row_ids, - code_array, - pq, - aux_writer.unwrap(), - )), - - _ => unreachable!("IVF_HNSW_SQ has been moved to v2 index builder"), + Quantizer::Product(pq) => { + let aux_writer = aux_writer.ok_or_else(|| { + Error::index("IVF_HNSW_PQ requires an auxiliary writer for PQ storage".to_string()) + })?; + build_and_write_pq_storage(metric_type, row_ids, code_array, pq, aux_writer) + } + _ => { + return Err(Error::index( + "IVF_HNSW_SQ is not supported in the legacy HNSW partition writer".to_string(), + )); + } }; - let index_rows = futures::join!(build_hnsw, build_store).0?; + let (index_rows, ()) = futures::try_join!(build_hnsw, build_store)?; assert!( index_rows >= num_rows, "index rows {} must be greater than or equal to num rows {}", @@ -504,11 +647,11 @@ async fn build_and_write_hnsw( vectors: Arc, params: HnswBuildParams, distance_type: DistanceType, - mut writer: PreviousFileWriter, + mut writer: V1FileWriter, ) -> Result { let batch = params.build(vectors, distance_type).await?.to_batch()?; let metadata = batch.schema_ref().metadata().clone(); - writer.write_record_batch(batch).await?; + writer.write(&[batch]).await?; Ok(writer.finish_with_metadata(&metadata).await?.num_rows as usize) } @@ -517,11 +660,11 @@ async fn build_and_write_pq_storage( row_ids: Arc, code_array: Vec>, pq: ProductQuantizer, - mut writer: PreviousFileWriter, + mut writer: V1FileWriter, ) -> Result<()> { let storage = spawn_cpu(move || build_pq_storage(metric_type, row_ids, code_array, pq)).await?; - writer.write_record_batch(storage.batch().clone()).await?; + writer.write(&[storage.batch().clone()]).await?; writer.finish().await?; Ok(()) } @@ -530,16 +673,51 @@ async fn build_and_write_pq_storage( mod tests { use super::*; + use std::path::PathBuf; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use crate::Dataset; use crate::index::vector::ivf::v2; use crate::index::{DatasetIndexExt, DatasetIndexInternalExt, vector::VectorIndexParams}; - use arrow_array::RecordBatchIterator; + use arrow_array::{RecordBatchIterator, UInt8Array}; use arrow_schema::{Field, Schema}; - use lance_core::utils::tempfile::TempStrDir; + use lance_core::utils::tempfile::{TempObjFile, TempStrDir}; use lance_index::IndexType; use lance_index::metrics::NoOpMetricsCollector; + use lance_index::vector::ivf::IvfBuildParams; + use lance_index::vector::pq::PQBuildParams; use lance_testing::datagen::generate_random_array; + #[tokio::test] + async fn pq_partition_payload_preserves_sliced_values() { + let codes = + FixedSizeListArray::try_new_from_values(UInt8Array::from_iter_values(0..6), 2).unwrap(); + let codes = codes.slice(1, 2); + let flat_codes = UInt8Array::from_iter_values(6..9).slice(1, 2); + let row_ids = UInt64Array::from_iter_values([10, 11, 12]).slice(1, 2); + let flat_row_id = UInt64Array::from_iter_values([13]); + + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = object_store.create(&path).await.unwrap(); + write_pq_partition_payload( + writer.as_mut(), + &[&codes as &dyn Array, &flat_codes as &dyn Array], + &[&row_ids as &dyn Array, &flat_row_id as &dyn Array], + ) + .await + .unwrap(); + Writer::shutdown(&mut writer).await.unwrap(); + + let reader = object_store.open(&path).await.unwrap(); + let bytes = reader.get_range(0..30).await.unwrap(); + let mut expected = vec![2, 3, 4, 5, 7, 8]; + expected.extend_from_slice(&11_u64.to_le_bytes()); + expected.extend_from_slice(&12_u64.to_le_bytes()); + expected.extend_from_slice(&13_u64.to_le_bytes()); + assert_eq!(bytes.as_ref(), expected); + } + #[tokio::test] async fn test_merge_multiple_indices() { const DIM: usize = 32; @@ -589,4 +767,266 @@ mod tests { //let indices = /ds. } + + /// The scratch dir must outlive every partition task: dropping the caller-side + /// guard while tasks are still in flight must not remove it, because each task + /// still holds an `Arc` clone of the guard. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_scratch_dir_outlives_partition_tasks() { + let tmp_part_dir_guard = Arc::new(TempStdDir::default()); + let scratch_path = tmp_part_dir_guard.to_path_buf(); + + // Park each task until the caller-side guard is dropped, so the dir's + // survival is attributable solely to the clones the tasks still hold. + let running = Arc::new(AtomicUsize::new(0)); + let released = Arc::new(AtomicBool::new(false)); + + const NUM_TASKS: usize = 3; + let mut tasks = Vec::with_capacity(NUM_TASKS); + for _ in 0..NUM_TASKS { + let task_guard = tmp_part_dir_guard.clone(); + let running = running.clone(); + let released = released.clone(); + tasks.push(tokio::spawn(async move { + running.fetch_add(1, Ordering::SeqCst); + while !released.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + // Still holding `task_guard`, so the directory must be live. + task_guard.exists() + })); + } + + // Drop the caller-side guard only once every task is parked holding its clone. + while running.load(Ordering::SeqCst) < NUM_TASKS { + tokio::task::yield_now().await; + } + drop(tmp_part_dir_guard); + assert!( + scratch_path.exists(), + "scratch dir removed while partition tasks still held the guard" + ); + + released.store(true, Ordering::SeqCst); + for task in tasks { + assert!( + task.await.unwrap(), + "a partition task observed its scratch dir already removed" + ); + } + assert!( + !scratch_path.exists(), + "scratch dir was not removed after the last task's guard clone dropped" + ); + } + + /// The drain step must outlive every spawned task: it aborts and awaits each + /// one so that, once it returns, no task is still running against the scratch + /// directory. It must also surface late failures rather than swallow them. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_drain_partition_tasks_waits_and_reports_errors() { + // Scratch dir guard analogous to the one held by + // `write_hnsw_quantization_index_partitions`; tasks read from it, and it + // is dropped only after the drain completes. + let scratch_guard = TempStdDir::default(); + let scratch_path = scratch_guard.to_path_buf(); + + // Count of live task futures. `LiveGuard` decrements on both completion and + // cancellation, so a zero count after the drain proves every task terminated + // rather than being detached. + let live = Arc::new(AtomicUsize::new(0)); + let saw_missing_dir = Arc::new(AtomicBool::new(false)); + + struct LiveGuard(Arc); + impl Drop for LiveGuard { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::SeqCst); + } + } + + const NUM_SLOW: usize = 3; + let mut tasks: Vec>>> = Vec::with_capacity(NUM_SLOW + 1); + + // Tasks that never resolve on their own: the drain's abort is the only thing + // that can stop them, which is exactly what this test exercises. + for _ in 0..NUM_SLOW { + let live = live.clone(); + let saw_missing_dir = saw_missing_dir.clone(); + let scratch_path = scratch_path.clone(); + tasks.push(Some(tokio::spawn(async move { + live.fetch_add(1, Ordering::SeqCst); + let _guard = LiveGuard(live.clone()); + if !scratch_path.exists() { + saw_missing_dir.store(true, Ordering::SeqCst); + } + futures::future::pending::<()>().await; + Ok(0) + }))); + } + + // A task that fails; its error must be returned, not silently dropped. + // The drain aborts every handle up front, and an abort only preserves a + // task's output if the task has already finished -- an in-flight task is + // cancelled and its error lost. So wait until this task has actually + // completed before handing it to the drain; otherwise whether its error + // surfaces would depend on the scheduler and the test would be flaky. + let failing = + tokio::spawn(async move { Err(Error::io("late partition failure".to_string())) }); + while !failing.is_finished() { + tokio::task::yield_now().await; + } + tasks.push(Some(failing)); + + // Ensure all slow tasks are actually running before draining, so the + // drain has to await their cancellation rather than aborting them before + // they ever start. + while live.load(Ordering::SeqCst) < NUM_SLOW { + tokio::task::yield_now().await; + } + + let errors = drain_partition_tasks(&mut tasks).await; + + assert!(tasks.iter().all(Option::is_none), "handles left undrained"); + assert_eq!( + live.load(Ordering::SeqCst), + 0, + "a task was still running after the drain returned" + ); + assert!( + !saw_missing_dir.load(Ordering::SeqCst), + "scratch dir was removed while a task was still running" + ); + assert_eq!(errors.len(), 1, "expected exactly the one late failure"); + assert!( + errors[0].to_string().contains("late partition failure"), + "late failure was not surfaced: {}", + errors[0] + ); + + // The guard, not the drain, removes the scratch dir. + assert!(scratch_path.exists()); + drop(scratch_guard); + assert!(!scratch_path.exists()); + } + + /// `write_hnsw_quantization_index_partitions` stages each partition in a scratch + /// directory owned by a [`TempStdDir`] guard, which must remove it once the build + /// finishes so the OS temp dir does not grow without bound across legacy + /// IVF_HNSW_* builds. + /// + /// The OS temp dir is process-global, so an in-process check can't attribute a + /// leak to our own build. Instead we run the build in a child process with + /// `TMPDIR` pointed at an isolated dir we own, then assert nothing survives. + #[test] + fn test_hnsw_pq_scratch_dir_is_not_leaked() { + // Isolated temp root for the child. Owned here so it -- and anything the + // child leaks into it -- is removed when this guard drops at test end. + let isolated_root = TempStdDir::default(); + + let child_test = "index::vector::ivf::io::tests::build_legacy_hnsw_pq_in_child_process"; + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([child_test, "--exact", "--ignored", "--nocapture"]) + .env("TMPDIR", isolated_root.as_ref()) + .env("LANCE_HNSW_LEAK_TEST_ROOT", isolated_root.as_ref()) + .output() + .expect("failed to spawn child test process"); + assert!( + output.status.success(), + "child build process failed:\n--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + + // Each build stages its partitions in one `.tmp*` dir directly under TMPDIR. + // Every guard removes its dir when it drops, so none should survive. + let leaked: Vec = std::fs::read_dir(&isolated_root) + .expect("read isolated temp root") + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.is_dir() + && path + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(".tmp")) + }) + .collect(); + + assert!( + leaked.is_empty(), + "legacy IVF_HNSW_PQ build leaked {} scratch director{} under the temp dir; \ + the TempStdDir guard should remove each one when it drops: {:?}", + leaked.len(), + if leaked.len() == 1 { "y" } else { "ies" }, + leaked, + ); + } + + /// Child half of [`test_hnsw_pq_scratch_dir_is_not_leaked`]. Ignored so it only + /// runs when the parent spawns it with `TMPDIR` and `LANCE_HNSW_LEAK_TEST_ROOT` + /// pointed at an isolated dir. Builds a few legacy IVF_HNSW_PQ indices; the + /// parent does the leak detection. + #[tokio::test] + #[ignore = "spawned as a child process by test_hnsw_pq_scratch_dir_is_not_leaked"] + async fn build_legacy_hnsw_pq_in_child_process() { + // Only do work when spawned by the parent; a bare `--ignored` run leaves + // the variable unset, so no-op rather than fail. + let Ok(root) = std::env::var("LANCE_HNSW_LEAK_TEST_ROOT") else { + return; + }; + + const DIM: usize = 8; + const ROWS: usize = 256; + const NLIST: usize = 2; + const NUM_BUILDS: usize = 2; + + // Keep the dataset out of the temp dir's `.tmp*` namespace so the parent + // never confuses it with a leaked scratch directory. + let dataset_uri = format!("{root}/dataset"); + let values = generate_random_array(ROWS * DIM); + let fsl = Arc::new(FixedSizeListArray::try_new_from_values(values, DIM as i32).unwrap()); + let schema = Arc::new(Schema::new(vec![Field::new( + "vector", + fsl.data_type().clone(), + false, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![fsl]).unwrap(); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let ds = Dataset::write(batches, &dataset_uri, Default::default()) + .await + .unwrap(); + + let ivf_params = IvfBuildParams { + num_partitions: Some(NLIST), + max_iters: 2, + sample_rate: 2, + ..Default::default() + }; + let hnsw_params = HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16); + let pq_params = PQBuildParams { + num_sub_vectors: 2, + num_bits: 8, + max_iters: 2, + sample_rate: 2, + ..Default::default() + }; + + for _ in 0..NUM_BUILDS { + crate::index::vector::ivf::build_ivf_hnsw_pq_index( + &ds, + "vector", + "idx", + uuid::Uuid::new_v4(), + MetricType::L2, + &ivf_params, + &hnsw_params, + &pq_params, + ) + .await + .unwrap(); + } + } } diff --git a/rust/lance/src/index/vector/ivf/partition_serde.rs b/rust/lance/src/index/vector/ivf/partition_serde.rs index ad737620a94..a1869357980 100644 --- a/rust/lance/src/index/vector/ivf/partition_serde.rs +++ b/rust/lance/src/index/vector/ivf/partition_serde.rs @@ -264,7 +264,7 @@ impl CacheCodecImpl for PartitionEntry { None, )?; - Ok(Self { index, storage }) + Ok(Self::new(index, storage)) } } @@ -308,7 +308,7 @@ impl CacheCodecImpl for PartitionEntry { None, )?; - Ok(Self { index, storage }) + Ok(Self::new(index, storage)) } } @@ -352,7 +352,7 @@ impl CacheCodecImpl for PartitionEntry { None, )?; - Ok(Self { index, storage }) + Ok(Self::new(index, storage)) } } @@ -399,7 +399,7 @@ impl CacheCodecImpl for PartitionEntry { None, )?; - Ok(Self { index, storage }) + Ok(Self::new(index, storage)) } } @@ -478,7 +478,7 @@ impl CacheCodecImpl for PartitionEntry { None, )?; - Ok(Self { index, storage }) + Ok(Self::new(index, storage)) } } @@ -584,10 +584,8 @@ mod tests { let num_rows = 100; let storage = make_test_pq_storage(num_rows, dim, num_sub_vectors); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let serialized = ser_body(&entry); let deserialized = @@ -634,10 +632,8 @@ mod tests { ) .unwrap(); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -653,10 +649,8 @@ mod tests { let dim = 16; let num_sub_vectors = 2; let storage = make_test_pq_storage(0, dim, num_sub_vectors); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let serialized = ser_body(&entry); let deserialized = @@ -669,10 +663,8 @@ mod tests { // Serialize a valid entry, then truncate the bytes and verify that // deserialization fails rather than panicking. let storage = make_test_pq_storage(1, 16, 2); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let mut bytes = ser_body(&entry); bytes.truncate(3); assert!(de_body::>(bytes).is_err()); @@ -708,10 +700,7 @@ mod tests { #[test] fn test_roundtrip_flat_flat() { let storage = make_flat_storage(50, 64); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -736,10 +725,8 @@ mod tests { let values = Float32Array::from(vec![1.0f32; 32]); let vectors = FixedSizeListArray::try_new_from_values(values, 32).unwrap(); let storage = FlatFloatStorage::new(vectors, dt); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); assert_eq!(restored.storage.distance_type(), dt); @@ -749,10 +736,7 @@ mod tests { #[test] fn test_roundtrip_flat_flat_f16() { let storage = make_flat_storage_f16(8, 16); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -771,10 +755,7 @@ mod tests { #[test] fn test_roundtrip_flat_flat_f64() { let storage = make_flat_storage_f64(8, 16); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -824,10 +805,8 @@ mod tests { #[test] fn test_roundtrip_flat_sq() { let storage = make_sq_storage(100, 64, DistanceType::L2); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -852,10 +831,8 @@ mod tests { fn test_sq_distance_types() { for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] { let storage = make_sq_storage(10, 16, dt); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); assert_eq!(restored.storage.distance_type(), dt); @@ -894,10 +871,8 @@ mod tests { .unwrap(); assert_eq!(storage.len(), 30); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -988,10 +963,7 @@ mod tests { let num_rows = 50; let code_dim = 64; let storage = make_rabit_storage_fast(num_rows, code_dim, DistanceType::L2); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -1028,10 +1000,8 @@ mod tests { fn test_rabitq_distance_types() { for dt in [DistanceType::L2, DistanceType::Cosine, DistanceType::Dot] { let storage = make_rabit_storage_fast(10, 32, dt); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = + PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); // The codec round-trips the distance type faithfully. @@ -1057,10 +1027,7 @@ mod tests { storage.metadata().query_estimator, RabitQueryEstimator::RawQuery ); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -1081,10 +1048,7 @@ mod tests { RQRotationType::Matrix, RabitQueryEstimator::ResidualQuery, ); - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage, - }; + let entry = PartitionEntry::::new(FlatIndex::default(), storage); let bytes = ser_body(&entry); let restored = de_body::>(bytes).unwrap(); @@ -1117,10 +1081,10 @@ mod tests { use lance_core::cache::CacheCodec; const ALIGN: usize = 64; - let entry = PartitionEntry:: { - index: FlatIndex::default(), - storage: make_sq_storage(64, 32, DistanceType::L2), - }; + let entry = PartitionEntry::::new( + FlatIndex::default(), + make_sq_storage(64, 32, DistanceType::L2), + ); let codec = CacheCodec::from_impl::>(); let any: Arc = Arc::new(entry); let mut buf = Vec::new(); @@ -1176,7 +1140,6 @@ mod tests { metadata: lance_index::vector::flat::index::FlatMetadata { dim: 2 }, sub_index_type: SubIndexType::Flat, quantization_type: QuantizationType::Flat, - cache_key_prefix: "prefix/".to_string(), index_file_size: 1024, aux_file_size: 512, rq_search_cache: empty_rabit_search_cache_cell(), diff --git a/rust/lance/src/index/vector/ivf/v2.rs b/rust/lance/src/index/vector/ivf/v2.rs index 0faa79dce5f..f1cc373d94a 100644 --- a/rust/lance/src/index/vector/ivf/v2.rs +++ b/rust/lance/src/index/vector/ivf/v2.rs @@ -9,7 +9,10 @@ use std::{ any::Any, borrow::Cow, collections::{BinaryHeap, HashMap}, - sync::{Arc, Mutex}, + sync::{ + Arc, LazyLock, Mutex, OnceLock, + atomic::{AtomicBool, Ordering}, + }, }; use crate::index::vector::{IndexFileVersion, builder::index_type_string}; @@ -27,8 +30,8 @@ use futures::prelude::stream::{self, TryStreamExt}; use futures::{StreamExt, TryFutureExt}; use lance_arrow::RecordBatchExt; use lance_core::cache::{ - CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, LanceCache, - WeakLanceCache, + CacheCodec, CacheCodecImpl, CacheEntryReader, CacheEntryWriter, CacheKey, CacheKeySchema, + KeyBuilder, LanceCache, WeakLanceCache, }; use lance_core::deepsize::DeepSizeOf; use lance_core::utils::tokio::{get_num_compute_intensive_cpus, spawn_cpu}; @@ -36,10 +39,12 @@ use lance_core::utils::tracing::{IO_TYPE_LOAD_VECTOR_PART, TRACE_IO_EVENTS}; use lance_core::{Error, ROW_ID, Result}; use lance_encoding::decoder::{DecoderPlugins, FilterExpression}; use lance_file::LanceEncodingsIo; -use lance_file::reader::{CachedFileMetadata, FileReader, FileReaderOptions}; +use lance_file::reader::{CachedFileMetadata, FileReader, FileReaderOptions, ReaderProjection}; use lance_index::cache_pb::IvfStateHeader; -use lance_index::frag_reuse::FragReuseIndex; +use lance_index::frag_reuse::{CompactFragReuseIndex, CompactFragReuseIndexHandle}; use lance_index::metrics::{LocalMetricsCollector, MetricsCollector, NoOpMetricsCollector}; +use lance_index::prefilter::NoFilter; +use lance_index::scalar::RowIdRemapper; use lance_index::vector::VectorIndexCacheEntry; use lance_index::vector::bq::builder::RabitQuantizer; use lance_index::vector::bq::ex_dot::{blocked_ex_code_bytes, padded_query_len}; @@ -75,6 +80,7 @@ use lance_io::{ ReadBatchParams, object_store::ObjectStore, scheduler::ScanScheduler, traits::Reader, }; use lance_linalg::distance::DistanceType; +use lance_select::RowAddrTreeMap; use object_store::path::Path; use prost::Message; use roaring::RoaringBitmap; @@ -112,8 +118,6 @@ pub(crate) struct IvfIndexState { pub(crate) metadata: ::Metadata, pub(crate) sub_index_type: SubIndexType, pub(crate) quantization_type: QuantizationType, - /// The cache key prefix used by the original index's WeakLanceCache. - pub(crate) cache_key_prefix: String, /// File sizes for the index and auxiliary files, used to avoid HEAD requests /// when reconstructing from cache. pub(crate) index_file_size: u64, @@ -122,6 +126,41 @@ pub(crate) struct IvfIndexState { pub(crate) rq_search_cache: RabitSearchCacheCell, } +/// Number of prepared partitions handed to a single `spawn_cpu` dispatch on the +/// streaming search path. +/// +/// The streaming path deliberately avoids per-partition CPU-task fan-out (a measured +/// 14-30% latency win, see #6475). Searching a batch of partitions per `spawn_cpu` +/// keeps most of that benefit — the per-dispatch overhead is paid once per +/// `STREAMING_SEARCH_BATCH_SIZE` partitions instead of once per partition — while +/// keeping the channel `recv`/`send` in async code so no CPU-pool thread ever parks on +/// a channel (which can deadlock the pool on small hosts, see #7642). `should_stop` is +/// still checked per partition, so early-stop granularity is unchanged. +/// +/// This is a tunable knob: larger batches amortize dispatch overhead further and keep +/// more work on a single CPU thread, at the cost of more prepared partitions held in +/// memory at once. The batch is an upper bound: the search loop greedily drains +/// whatever is already prepared rather than waiting for a full batch, so a slow +/// producer yields small batches (matching the old search-as-it-arrives latency) and +/// only a fast producer fills whole ones. Override with the +/// `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE` environment variable. +pub(crate) const DEFAULT_STREAMING_SEARCH_BATCH_SIZE: usize = 16; + +pub(crate) static STREAMING_SEARCH_BATCH_SIZE: LazyLock = LazyLock::new(|| { + let batch_size = std::env::var("LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE") + .map(|value| { + value + .parse() + .expect("failed to parse LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE") + }) + .unwrap_or(DEFAULT_STREAMING_SEARCH_BATCH_SIZE); + assert!( + batch_size > 0, + "LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE must be greater than 0, got {batch_size}" + ); + batch_size +}); + struct PreparedPartitionSearch { query: Query, pre_filter: Arc, @@ -129,7 +168,7 @@ struct PreparedPartitionSearch { partition_centroid: Option, rq_search_cache: Option>, raw_query_context: Option>, - part_entry: Arc, + part_entry: Arc>, _marker: PhantomData<(S, Q)>, } @@ -209,7 +248,6 @@ impl DeepSizeOf for IvfIndexState { + self.aux_ivf.deep_size_of_children(context) + self.sub_index_metadata.deep_size_of_children(context) + self.metadata.deep_size_of_children(context) - + self.cache_key_prefix.deep_size_of_children(context) + self .rq_search_cache .lock() @@ -234,7 +272,7 @@ pub(crate) trait IvfStateEntry: DeepSizeOf + Send + Sync + 'static { object_store: Arc, file_metadata_cache: &'a LanceCache, index_cache: LanceCache, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> BoxFuture<'a, Result>>; } @@ -321,7 +359,6 @@ impl CacheCodecImpl for IvfStateEntryBox { metadata, sub_index_type, quantization_type, - cache_key_prefix: header.cache_key_prefix, index_file_size: header.index_file_size, aux_file_size: header.aux_file_size, rq_search_cache: empty_rabit_search_cache_cell(), @@ -393,7 +430,6 @@ impl IvfStateEntry for IvfIndexState { sub_index_type: self.sub_index_type.to_string(), quantization_type: self.quantization_type.to_string(), quantizer_metadata_json, - cache_key_prefix: self.cache_key_prefix.clone(), index_file_size: self.index_file_size, aux_file_size: self.aux_file_size, }; @@ -412,7 +448,7 @@ impl IvfStateEntry for IvfIndexState { object_store: Arc, file_metadata_cache: &'a LanceCache, index_cache: LanceCache, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> BoxFuture<'a, Result>> { Box::pin(async move { match self.sub_index_type { @@ -451,43 +487,12 @@ impl CacheKey for FileMetadataCacheKey { fn key(&self) -> std::borrow::Cow<'_, str> { "".into() } -} - -/// Cached open file readers for the index and aux files. -/// -/// Stored in `file_metadata_cache` to avoid re-opening files on every reconstruction. -/// Not serializable (no codec); a cache miss just triggers a re-open. -struct CachedIndexReaders { - index_reader: Arc, - aux_reader: Arc, -} -impl lance_core::deepsize::DeepSizeOf for CachedIndexReaders { - fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { - // FileReader doesn't impl DeepSizeOf. We approximate by counting the - // fixed struct size for each reader plus the Arc - // heap contents. The metadata Arcs are also held by FileMetadataCacheKey - // entries, so this may over-count across cache entries, but - // over-counting is safer than under-counting for eviction purposes. - std::mem::size_of::() * 2 - + self.index_reader.metadata().deep_size_of_children(context) - + self.aux_reader.metadata().deep_size_of_children(context) + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.ivf-file-metadata-key", 1) } -} -struct CachedIndexReadersKey { - uuid: String, -} - -impl CacheKey for CachedIndexReadersKey { - type ValueType = CachedIndexReaders; - fn type_name() -> &'static str { - "CachedIndexReaders" - } - fn key(&self) -> std::borrow::Cow<'_, str> { - self.uuid.as_str().into() - } - // No codec() override → in-memory only + fn write_key(&self, _builder: &mut KeyBuilder) {} } /// Open a FileReader, reusing cached file metadata if available. @@ -517,21 +522,58 @@ async fn open_reader_cached( .await } else { let file_scheduler = scheduler.open_file(path, &cached_size).await?; - FileReader::try_open( + let reader = FileReader::try_open( file_scheduler, None, Arc::::default(), cache, FileReaderOptions::default(), ) - .await + .await?; + // File metadata is store-free, so it outlives the reader opened here: + // cache it to spare later reconstructions the footer read. + file_cache + .insert_with_key(&FileMetadataCacheKey, reader.metadata().clone()) + .await; + Ok(reader) } } -#[derive(Debug, DeepSizeOf)] +#[derive(Debug)] pub struct PartitionEntry { pub index: S, pub storage: Q::Storage, + partition_rows: OnceLock>, + partition_rows_accounted: AtomicBool, +} + +impl PartitionEntry { + pub(super) fn new(index: S, storage: Q::Storage) -> Self { + Self { + index, + storage, + partition_rows: OnceLock::new(), + partition_rows_accounted: AtomicBool::new(false), + } + } + + fn partition_rows(&self) -> Arc { + self.partition_rows + .get_or_init(|| Arc::new(self.storage.row_ids().collect())) + .clone() + } +} + +impl DeepSizeOf for PartitionEntry { + fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize { + self.index.deep_size_of_children(context) + + self.storage.deep_size_of_children(context) + + self + .partition_rows + .get() + .map(|rows| rows.deep_size_of_children(context)) + .unwrap_or_default() + } } impl VectorIndexCacheEntry @@ -566,9 +608,23 @@ impl CacheKey for IVFPartit } fn type_name() -> &'static str { - // Using type_name is safe here: the impl is in the same crate as the - // types, so the monomorphized pointer is consistent. - std::any::type_name::>() + "IVFPartition" + } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.ivf-partition-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(S::name()); + builder.write_variant(match Q::quantization_type() { + QuantizationType::Flat => 0, + QuantizationType::FlatBin => 1, + QuantizationType::Product => 2, + QuantizationType::Scalar => 3, + QuantizationType::Rabit => 4, + }); + builder.write_u64(self.partition_id as u64); } fn codec() -> Option { @@ -590,6 +646,11 @@ pub struct IVFIndex { ivf: IvfModel, reader: FileReader, + /// Narrowed read of the index file, when the sub-index declares that + /// [`IvfSubIndex::load`] consumes only part of what it writes. `None` reads + /// every column. Built once here because it is fallible and only depends on + /// the file schema. + read_projection: Option, sub_index_metadata: Vec, storage: IvfQuantizationStorage, @@ -630,6 +691,58 @@ impl DeepSizeOf for IVFIndex { } impl IVFIndex { + fn read_projection(reader: &FileReader) -> Result> { + S::read_columns() + .map(|columns| { + lance_file::versions::reader_projection_from_column_names( + reader.metadata().version(), + reader.schema(), + columns, + ) + }) + .transpose() + } + + async fn cache_partition_rows( + index_cache: &WeakLanceCache, + partition_id: usize, + partition: &Arc>, + ) -> Result> { + let rows = partition.partition_rows(); + if !partition.partition_rows_accounted.load(Ordering::Acquire) { + let cache_key = IVFPartitionKey::::new(partition_id); + if index_cache + .insert_with_key(&cache_key, partition.clone()) + .await + { + partition + .partition_rows_accounted + .store(true, Ordering::Release); + } + } + Ok(rows) + } + + async fn prefilter_for_partition( + index_cache: &WeakLanceCache, + partition_id: usize, + partition: &Arc>, + pre_filter: Arc, + ) -> Result> { + if pre_filter.is_empty() { + return Ok(Arc::new(NoFilter)); + } + if !pre_filter.needs_partition_row_ids() { + return Ok(pre_filter); + } + let rows = Self::cache_partition_rows(index_cache, partition_id, partition).await?; + if pre_filter.is_empty_for(rows.as_ref()) { + Ok(Arc::new(NoFilter)) + } else { + Ok(pre_filter) + } + } + fn use_query_residual( storage: &IvfQuantizationStorage, distance_type: DistanceType, @@ -712,6 +825,9 @@ impl IVFIndex { self.load_partition(partition_id, true, metrics), pre_filter.wait_for_ready(), )?; + let pre_filter = + Self::prefilter_for_partition(&self.index_cache, partition_id, &part_entry, pre_filter) + .await?; Ok(PreparedPartitionSearch { query: query.clone(), pre_filter, @@ -733,6 +849,9 @@ impl IVFIndex { raw_query_context: Option>, ) -> Result> { let part_entry = self.load_partition(partition_id, true, metrics).await?; + let pre_filter = + Self::prefilter_for_partition(&self.index_cache, partition_id, &part_entry, pre_filter) + .await?; Ok(PreparedPartitionSearch { query: query.clone(), pre_filter, @@ -782,17 +901,11 @@ impl IVFIndex { let param = (&query).into(); let refine_factor = query.refine_factor.unwrap_or(1) as usize; let k = query.k * refine_factor; - let part = part_entry - .as_any() - .downcast_ref::>() - .ok_or(Error::internal( - "failed to downcast partition entry".to_string(), - ))?; - let batch = part.index.search_with_scratch( + let batch = part_entry.index.search_with_scratch( query.key, k, param, - &part.storage, + &part_entry.storage, pre_filter, metrics, residual, @@ -840,17 +953,11 @@ impl IVFIndex { let param = (&query).into(); let refine_factor = query.refine_factor.unwrap_or(1) as usize; let k = query.k * refine_factor; - let part = part_entry - .as_any() - .downcast_ref::>() - .ok_or(Error::internal( - "failed to downcast partition entry".to_string(), - ))?; - part.index.accumulate_topk_with_scratch( + part_entry.index.accumulate_topk_with_scratch( query.key, k, param, - &part.storage, + &part_entry.storage, pre_filter, heap, residual, @@ -970,7 +1077,7 @@ impl IVFIndex { object_store: Arc, index_dir: Path, uuid: Uuid, - frag_reuse_index: Option>, + frag_reuse_index: Option>, file_metadata_cache: &LanceCache, index_cache: LanceCache, file_sizes: HashMap, @@ -1043,8 +1150,11 @@ impl IVFIndex { FileReaderOptions::default(), ) .await?; + let frag_reuse_index = frag_reuse_index + .clone() + .map(|index| Arc::new(CompactFragReuseIndexHandle(index)) as Arc); let storage = - IvfQuantizationStorage::try_new(storage_reader, frag_reuse_index.clone()).await?; + IvfQuantizationStorage::try_new_with_remapper(storage_reader, frag_reuse_index).await?; // Cache file metadata so reconstructions from IvfIndexState can skip // footer reads. @@ -1061,19 +1171,6 @@ impl IVFIndex { .insert_with_key(&FileMetadataCacheKey, storage.reader().metadata().clone()) .await; - // Cache open readers so the first reconstruction also skips file opens. - file_metadata_cache - .insert_with_key( - &CachedIndexReadersKey { - uuid: uuid_str.clone(), - }, - Arc::new(CachedIndexReaders { - index_reader: Arc::new(index_reader.clone()), - aux_reader: Arc::new(storage.reader().clone()), - }), - ) - .await; - let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf, &storage)); let use_query_residual = Self::use_query_residual(&storage, distance_type); let use_residual_scratch = Self::use_residual_scratch(&ivf, use_query_residual); @@ -1085,6 +1182,7 @@ impl IVFIndex { // cumulative stats are exactly the one-time index-open I/O. let open_io_stats = scheduler.stats(); + let read_projection = Self::read_projection(&index_reader)?; Ok(Self { uri: to_local_path(&uri), index_path: uri.as_ref().to_string(), @@ -1095,6 +1193,7 @@ impl IVFIndex { rq_search_cache, ivf, reader: index_reader, + read_projection, storage, sub_index_metadata, distance_type, @@ -1119,11 +1218,12 @@ impl IVFIndex { index_cache: LanceCache, io_parallelism: usize, rq_search_cache: Option>, - ) -> Self { + ) -> Result { let scratch_pool = Arc::new(Self::query_scratch_pool(&ivf, &storage)); let use_query_residual = Self::use_query_residual(&storage, distance_type); let use_residual_scratch = Self::use_residual_scratch(&ivf, use_query_residual); - Self { + let read_projection = Self::read_projection(&reader)?; + Ok(Self { uri, index_path, uuid, @@ -1133,6 +1233,7 @@ impl IVFIndex { rq_search_cache, ivf, reader, + read_projection, storage, sub_index_metadata, distance_type, @@ -1143,7 +1244,7 @@ impl IVFIndex { // and the first open via `try_new` already accounts for it). open_io_stats: ScanStats::default(), _marker: PhantomData, - } + }) } #[instrument(level = "debug", skip(self, metrics))] @@ -1152,7 +1253,7 @@ impl IVFIndex { partition_id: usize, write_cache: bool, metrics: &dyn MetricsCollector, - ) -> Result> { + ) -> Result>> { if partition_id >= self.ivf.num_partitions() { return Err(Error::index(format!( "partition id {} is out of range of {} partitions", @@ -1164,20 +1265,27 @@ impl IVFIndex { let cache_key = IVFPartitionKey::::new(partition_id); if write_cache { - let entry = self + let result = self .index_cache - .get_or_insert_with_key(cache_key, || async { + .get_or_insert_with_key_hit(cache_key, || async { info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id); metrics.record_part_load(); self.load_partition_entry(partition_id, metrics.io_stats()) .await }) - .await?; - Ok(entry as Arc) + .await; + match &result { + Ok((_, true)) => metrics.record_index_cache_hit(), + _ => metrics.record_index_cache_miss(), + } + let (entry, _) = result?; + Ok(entry) } else { if let Some(part_idx) = self.index_cache.get_with_key(&cache_key).await { + metrics.record_index_cache_hit(); return Ok(part_idx); } + metrics.record_index_cache_miss(); info!(target: TRACE_IO_EVENTS, r#type=IO_TYPE_LOAD_VECTOR_PART, index_type="ivf", part_id=partition_id); metrics.record_part_load(); Ok(Arc::new( @@ -1192,7 +1300,14 @@ impl IVFIndex { partition_id: usize, io_stats: Option, ) -> Result> { - let schema = Arc::new(self.reader.schema().as_ref().into()); + // `concat_batches` indexes the batches by this schema's field positions + // without comparing the two, so the schema has to describe exactly what + // was read: the full file schema over a projected read would index past + // the last column. + let schema = Arc::new(match &self.read_projection { + Some(projection) => projection.schema.as_ref().into(), + None => self.reader.schema().as_ref().into(), + }); let batch = match self.reader.metadata().num_rows { 0 => RecordBatch::new_empty(schema), _ => { @@ -1210,16 +1325,26 @@ impl IVFIndex { } None => Cow::Borrowed(&self.reader), }; - let batches = reader - .read_stream( - ReadBatchParams::Range(row_range), - u32::MAX, - 1, - FilterExpression::no_filter(), - ) - .await? - .try_collect::>() - .await?; + let params = ReadBatchParams::Range(row_range); + let stream = match &self.read_projection { + Some(projection) => { + reader + .read_stream_projected( + params, + u32::MAX, + 1, + projection.clone(), + FilterExpression::no_filter(), + ) + .await? + } + None => { + reader + .read_stream(params, u32::MAX, 1, FilterExpression::no_filter()) + .await? + } + }; + let batches = stream.try_collect::>().await?; concat_batches(&schema, batches.iter())? } } @@ -1230,10 +1355,7 @@ impl IVFIndex { )?; let idx = S::load(batch)?; let storage = self.load_partition_storage(partition_id, io_stats).await?; - Ok(PartitionEntry { - index: idx, - storage, - }) + Ok(PartitionEntry::new(idx, storage)) } pub async fn load_partition_storage( @@ -1271,7 +1393,6 @@ impl IVFIndex { metadata: self.storage.metadata().clone(), sub_index_type, quantization_type, - cache_key_prefix: self.index_cache.prefix().to_string(), index_file_size: self.reader.metadata().file_size(), aux_file_size: self.storage.reader().metadata().file_size(), rq_search_cache: rabit_search_cache_cell(self.rq_search_cache.clone()), @@ -1431,6 +1552,9 @@ impl VectorIndex for IVFInd ) -> Result { let part_entry = self.load_partition(partition_id, true, metrics).await?; pre_filter.wait_for_ready().await?; + let pre_filter = + Self::prefilter_for_partition(&self.index_cache, partition_id, &part_entry, pre_filter) + .await?; let partition_centroid = self.ivf.centroid(partition_id); let rq_search_cache = self.rq_search_cache.clone(); @@ -1450,12 +1574,6 @@ impl VectorIndex for IVFInd let refine_factor = query.refine_factor.unwrap_or(1) as usize; let k = query.k * refine_factor; let local_metrics = LocalMetricsCollector::default(); - let part = part_entry - .as_any() - .downcast_ref::>() - .ok_or(Error::internal( - "failed to downcast partition entry".to_string(), - ))?; let rotated_partition_centroid = rotated_partition_centroid_slice(rq_search_cache.as_deref(), partition_id); let residual = Self::query_context_for_scratch( @@ -1467,11 +1585,11 @@ impl VectorIndex for IVFInd raw_query_context.as_deref(), )?; let batch = scratch_pool.with_scratch(|scratch| { - part.index.search_with_scratch( + part_entry.index.search_with_scratch( query.key, k, param, - &part.storage, + &part_entry.storage, pre_filter, &local_metrics, residual, @@ -1622,8 +1740,11 @@ impl VectorIndex for IVFInd ))); } + // The prepared channel holds a full search batch so that partitions prepared + // while the previous batch is being searched are ready for the next greedy + // drain, instead of serializing producer and consumer through a single slot. let (prepared_tx, mut prepared_rx) = - mpsc::channel::>>(1); + mpsc::channel::>>(*STREAMING_SEARCH_BATCH_SIZE); let (batch_tx, batch_rx) = mpsc::channel::>(1); let prepare_index = self.clone(); @@ -1665,61 +1786,139 @@ impl VectorIndex for IVFInd let use_query_residual = self.use_query_residual; let use_residual_scratch = self.use_residual_scratch; let search_metrics = metrics.clone(); - let batch_tx_for_search = batch_tx.clone(); let search_control = control.clone(); let scratch_pool = self.scratch_pool.clone(); + // Search prepared partitions in batches. Each batch is searched in a single + // `spawn_cpu` dispatch (amortizing the per-dispatch overhead the single-worker + // design in #6475 avoided), but the channel `recv`/`send` stay in async code so + // no CPU-pool thread ever parks on a channel — parking one can deadlock the pool + // on small hosts (#7642). `should_stop` is checked per partition, so early-stop + // granularity is unchanged. + // + // Batches are formed greedily: wait for one prepared partition, then drain + // whatever else is already prepared, up to the batch size. Waiting for a full + // batch instead would delay the first search (and the early-stop feedback it + // produces) behind up to a whole batch of prepare I/O, which is significant + // when prepare parallelism is low. tokio::spawn(async move { - let search_result = spawn_cpu(move || -> DataFusionResult<()> { - scratch_pool.with_scratch(|scratch| { - while let Some(prepared) = prepared_rx.blocking_recv() { - let prepared = match prepared { - Ok(prepared) => prepared, - Err(err) => { - let _ = batch_tx_for_search - .blocking_send(Err(DataFusionError::from(err))); - return Ok(()); - } - }; + loop { + // Stop pulling as soon as the search is done — or the receiver of our + // results is gone — so the producer stops preparing partitions we + // would never search. + if search_control + .as_ref() + .is_some_and(|control| control.should_stop()) + || batch_tx.is_closed() + { + return; + } - if search_control - .as_ref() - .is_some_and(|control| control.should_stop()) - { - return Ok(()); + let mut prepared_batch = Vec::with_capacity(*STREAMING_SEARCH_BATCH_SIZE); + let mut prepare_error = None; + let mut producer_done = false; + match prepared_rx.recv().await { + Some(Ok(prepared)) => prepared_batch.push(prepared), + Some(Err(err)) => prepare_error = Some(DataFusionError::from(err)), + None => producer_done = true, + } + while prepare_error.is_none() + && !producer_done + && prepared_batch.len() < *STREAMING_SEARCH_BATCH_SIZE + { + match prepared_rx.try_recv() { + Ok(Ok(prepared)) => prepared_batch.push(prepared), + Ok(Err(err)) => { + prepare_error = Some(DataFusionError::from(err)); } - - let batch = { - Self::run_prepared_partition_search( - use_query_residual, - use_residual_scratch, - prepared, - search_metrics.as_ref(), - scratch, - ) + // Nothing else is prepared yet; search what we have rather + // than waiting for more. + Err(mpsc::error::TryRecvError::Empty) => break, + Err(mpsc::error::TryRecvError::Disconnected) => { + producer_done = true; } - .map_err(DataFusionError::from); - match batch { - Ok(batch) => { - if let Some(control) = search_control.as_ref() { - control.record_batch(&batch); + } + } + + if !prepared_batch.is_empty() { + let scratch_pool = scratch_pool.clone(); + let search_metrics = search_metrics.clone(); + let search_control = search_control.clone(); + // `is_closed` is synchronously callable, so a sender clone lets the + // CPU loop notice a dropped receiver between partitions instead of + // searching out the whole batch for a cancelled query. (A `select!` + // on `closed()` would not help here: `spawn_cpu` closures are not + // cancellable, so abandoning the await leaves the work running.) + let cancel_probe = batch_tx.clone(); + let search_output = spawn_cpu(move || { + let mut outputs: Vec> = + Vec::with_capacity(prepared_batch.len()); + // `stopped` means the whole search should end (an error, an + // early-stop signal, or cancellation), not just this batch. + let mut stopped = false; + scratch_pool.with_scratch(|scratch| { + for prepared in prepared_batch { + if search_control + .as_ref() + .is_some_and(|control| control.should_stop()) + || cancel_probe.is_closed() + { + stopped = true; + break; } - if batch_tx_for_search.blocking_send(Ok(batch)).is_err() { - return Ok(()); + match Self::run_prepared_partition_search( + use_query_residual, + use_residual_scratch, + prepared, + search_metrics.as_ref(), + scratch, + ) + .map_err(DataFusionError::from) + { + Ok(batch) => { + if let Some(control) = search_control.as_ref() { + control.record_batch(&batch); + } + outputs.push(Ok(batch)); + } + Err(err) => { + outputs.push(Err(err)); + stopped = true; + break; + } } } - Err(err) => { - let _ = batch_tx_for_search.blocking_send(Err(err)); - return Ok(()); - } + }); + Ok::<_, DataFusionError>((outputs, stopped)) + }) + .await; + + let (outputs, stopped) = match search_output { + Ok(output) => output, + // Defensive: the closure always returns Ok (search errors are + // captured per partition in `outputs`), so this arm should be + // unreachable. Forward and stop rather than drop silently. + Err(err) => { + let _ = batch_tx.send(Err(err)).await; + return; + } + }; + for output in outputs { + if batch_tx.send(output).await.is_err() { + return; } } - Ok(()) - }) - }) - .await; + if stopped { + return; + } + } - if let Err(err) = search_result { - let _ = batch_tx.send(Err(err)).await; + if let Some(err) = prepare_error { + let _ = batch_tx.send(Err(err)).await; + return; + } + if producer_done { + return; + } } }); @@ -1729,6 +1928,186 @@ impl VectorIndex for IVFInd ))) } + fn supports_batch_partition_search(&self) -> bool { + S::supports_global_topk_heap() + } + + async fn search_partitions_batch( + self: Arc, + query: Query, + partitions_per_query: Vec>, + q_c_dists_per_query: Vec>, + pre_filter: Arc, + metrics: Arc, + ) -> Result> { + if !S::supports_global_topk_heap() { + return Err(Error::not_supported( + "batch partition search requires a global top-k heap sub-index", + )); + } + let query_count = partitions_per_query.len(); + if q_c_dists_per_query.len() != query_count { + return Err(Error::invalid_input(format!( + "batch partition search: {query_count} query partition lists but {} distance lists", + q_c_dists_per_query.len() + ))); + } + if query_count == 0 { + return Ok(Vec::new()); + } + if !query.key.len().is_multiple_of(query_count) { + return Err(Error::invalid_input(format!( + "batch partition search: query key length {} is not divisible by query count {query_count}", + query.key.len() + ))); + } + let dim = query.key.len() / query_count; + + // Per-query immutable search state: the query vector slice and the + // optional Rabit raw-query context both depend only on the query vector, + // so compute them once up front rather than per probed partition. + let mut base_queries = Vec::with_capacity(query_count); + let mut raw_query_contexts = Vec::with_capacity(query_count); + for query_index in 0..query_count { + if partitions_per_query[query_index].len() != q_c_dists_per_query[query_index].len() { + return Err(Error::invalid_input(format!( + "batch partition search: query {query_index} has {} partitions but {} distances", + partitions_per_query[query_index].len(), + q_c_dists_per_query[query_index].len() + ))); + } + let mut single_query = query.clone(); + single_query.key = query.key.slice(query_index * dim, dim); + raw_query_contexts.push(self.prepare_rq_raw_query_context(&single_query.key)?); + base_queries.push(single_query); + } + // Shared across every chunk's scoring dispatch below, so wrap once in an + // `Arc` instead of cloning the whole `Vec` per chunk. + let base_queries = Arc::new(base_queries); + let raw_query_contexts = Arc::new(raw_query_contexts); + + // Invert the per-query partition lists so each distinct partition is + // loaded once and scored against every query that probes it. + let mut assignments: HashMap> = HashMap::new(); + for (query_index, (parts, dists)) in partitions_per_query + .iter() + .zip(q_c_dists_per_query.iter()) + .enumerate() + { + for (part_id, dist_q_c) in parts.values().iter().zip(dists.values().iter()) { + assignments + .entry(*part_id) + .or_default() + .push((query_index, *dist_q_c)); + } + } + + pre_filter.wait_for_ready().await?; + + // Score partitions in a deterministic order. `assignments` is a HashMap, + // so its iteration order (and hence the order partitions accumulate into + // each per-query heap) is otherwise arbitrary. When several rows tie at + // the k-th distance, which one the capped heap keeps depends on insertion + // order, so a stable partition order is what makes the selected top-k + // deterministic across runs. (Any of the tied rows is an equally valid + // k-th neighbor, so this does not affect recall.) + let mut assignment_list: Vec<(u32, Vec<(usize, f32)>)> = assignments.into_iter().collect(); + assignment_list.sort_by_key(|(part_id, _)| *part_id); + + // Load each distinct partition's storage exactly once (the shared I/O + // that batch search exists to save), but *stream* the loaded partitions + // through scoring in chunks rather than materializing them all. A wide + // batch probes up to `min(query_count * nprobes, num_partitions)` distinct + // partitions, so collecting every loaded partition before scoring would + // make peak memory scale with the batch width — up to the whole index. + // Streaming bounds resident partition storage to the load window plus one + // chunk. `buffered` preserves the sorted load order above, so scoring order + // (and thus the k-th-distance tie-break) stays deterministic. + let load_parallelism = get_num_compute_intensive_cpus().max(1); + let load_index = self.clone(); + let load_metrics = metrics.clone(); + let mut loaded_chunks = stream::iter(assignment_list) + .map(move |(part_id, probing_queries)| { + let index = load_index.clone(); + let metrics = load_metrics.clone(); + async move { + let part_entry = index + .load_partition(part_id as usize, true, metrics.as_ref()) + .await?; + Result::Ok((part_id as usize, part_entry, probing_queries)) + } + }) + .buffered(load_parallelism) + .chunks(*STREAMING_SEARCH_BATCH_SIZE); + + let use_query_residual = self.use_query_residual; + let use_residual_scratch = self.use_residual_scratch; + let heap_capacity = query.k * query.refine_factor.unwrap_or(1) as usize; + let mut heaps: Vec>> = (0..query_count) + .map(|_| BinaryHeap::with_capacity(heap_capacity)) + .collect(); + + // Score each chunk on the CPU pool while the next chunk loads. `spawn_cpu` + // dispatches the scoring immediately and only touches CPU-bound state, so + // `join!`-ing it with the next `loaded_chunks` pull keeps partition I/O in + // flight during scoring: the async task, never a CPU-pool thread, does the + // waiting (#7642), and the load stream is not paused (the pairing `spawn_cpu`'s + // docs recommend with `buffered`). Scoring stays sequential across chunks — + // each mutates the same per-query heaps — so a step costs about + // max(load, score) rather than their sum, and a scored chunk's storage is + // dropped before the next is scored, keeping peak memory bounded. + let mut pending = loaded_chunks.next().await; + while let Some(chunk) = pending { + let chunk = chunk.into_iter().collect::>>()?; + let index = self.clone(); + let pre_filter = pre_filter.clone(); + let base_queries = base_queries.clone(); + let raw_query_contexts = raw_query_contexts.clone(); + let scratch_pool = self.scratch_pool.clone(); + let search_metrics = metrics.clone(); + let score = spawn_cpu(move || -> Result>>> { + scratch_pool.with_scratch(|scratch| -> Result<()> { + for (part_id, part_entry, probing_queries) in &chunk { + let partition_centroid = index.ivf.centroid(*part_id); + for (query_index, dist_q_c) in probing_queries { + let mut single_query = base_queries[*query_index].clone(); + single_query.dist_q_c = *dist_q_c; + let prepared = PreparedPartitionSearch:: { + query: single_query, + pre_filter: pre_filter.clone(), + partition_id: *part_id, + partition_centroid: partition_centroid.clone(), + rq_search_cache: index.rq_search_cache.clone(), + raw_query_context: raw_query_contexts[*query_index].clone(), + part_entry: part_entry.clone(), + _marker: PhantomData, + }; + Self::accumulate_prepared_partition_search( + use_query_residual, + use_residual_scratch, + prepared, + &mut heaps[*query_index], + scratch, + search_metrics.as_ref(), + )?; + } + } + Ok(()) + })?; + Ok(heaps) + }); + // Load the next chunk while this one is scored on the CPU pool. + let (scored, next) = futures::join!(score, loaded_chunks.next()); + heaps = scored?; + pending = next; + } + + heaps + .into_iter() + .map(Self::global_heap_to_batch) + .collect::>>() + } + fn is_loadable(&self) -> bool { false } @@ -1753,12 +2132,6 @@ impl VectorIndex for IVFInd metrics: &dyn MetricsCollector, ) -> Result { let partition = self.load_partition(partition_id, false, metrics).await?; - let partition = partition - .as_any() - .downcast_ref::>() - .ok_or(Error::internal( - "failed to downcast partition entry".to_string(), - ))?; let store = &partition.storage; let schema = if with_vector { store.schema().clone() @@ -1833,7 +2206,7 @@ async fn reconstruct_typed( object_store: Arc, file_metadata_cache: &LanceCache, index_cache: LanceCache, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> Result> { let io_parallelism = object_store.io_parallelism(); @@ -1846,45 +2219,29 @@ async fn reconstruct_typed( let dir: Path = parts.into_iter().collect(); let aux_path = dir.clone().join(INDEX_AUXILIARY_FILE_NAME); - let readers_key = CachedIndexReadersKey { - uuid: state.uuid.clone(), - }; - - let (index_reader, aux_reader) = - if let Some(cached) = file_metadata_cache.get_with_key(&readers_key).await { - // Warm path: reuse the cached readers directly, no file opens needed. - ((*cached.index_reader).clone(), (*cached.aux_reader).clone()) - } else { - // Cold path: open files, then cache the readers for future reconstructions. - let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); - let scheduler = ScanScheduler::new(object_store, scheduler_config); - let index_reader = open_reader_cached( - &scheduler, - &index_path, - file_metadata_cache, - state.index_file_size, - ) - .await?; - let aux_reader = open_reader_cached( - &scheduler, - &aux_path, - file_metadata_cache, - state.aux_file_size, - ) - .await?; - file_metadata_cache - .insert_with_key( - &readers_key, - Arc::new(CachedIndexReaders { - index_reader: Arc::new(index_reader.clone()), - aux_reader: Arc::new(aux_reader.clone()), - }), - ) - .await; - (index_reader, aux_reader) - }; + // Readers carry a scheduler bound to an object store, so they cannot be + // shared across dataset opens. Reuse only portable file metadata and bind + // fresh readers to the object store supplied for this reconstruction. + let scheduler_config = SchedulerConfig::max_bandwidth(&object_store); + let scheduler = ScanScheduler::new(object_store, scheduler_config); + let index_reader = open_reader_cached( + &scheduler, + &index_path, + file_metadata_cache, + state.index_file_size, + ) + .await?; + let aux_reader = open_reader_cached( + &scheduler, + &aux_path, + file_metadata_cache, + state.aux_file_size, + ) + .await?; - let storage = IvfQuantizationStorage::from_cached( + let frag_reuse_index = frag_reuse_index + .map(|index| Arc::new(CompactFragReuseIndexHandle(index)) as Arc); + let storage = IvfQuantizationStorage::from_cached_with_remapper( aux_reader, state.aux_ivf.clone(), state.metadata.clone(), @@ -1907,22 +2264,28 @@ async fn reconstruct_typed( index_cache, io_parallelism, rq_search_cache, - ); + )?; Ok(Arc::new(index)) } #[cfg(test)] mod tests { - use std::collections::HashSet; + use std::collections::{HashMap, HashSet}; use std::iter::repeat_n; - use std::{ops::Range, sync::Arc}; + use std::{ + ops::Range, + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + }; - use all_asserts::{assert_ge, assert_le, assert_lt}; + use all_asserts::{assert_ge, assert_lt}; use arrow::datatypes::{Float64Type, UInt8Type, UInt64Type}; use arrow::{array::AsArray, datatypes::Float32Type}; use arrow_array::{ Array, ArrayRef, ArrowPrimitiveType, FixedSizeListArray, Float32Array, Int64Array, - ListArray, RecordBatch, RecordBatchIterator, UInt64Array, + ListArray, PrimitiveArray, RecordBatch, RecordBatchIterator, UInt64Array, }; use arrow_buffer::OffsetBuffer; use arrow_schema::{DataType, Field, Schema, SchemaRef}; @@ -1935,11 +2298,14 @@ mod tests { transform::{EX_ADD_FACTORS_COLUMN, EX_SCALE_FACTORS_COLUMN}, }; use lance_index::vector::storage::VectorStore; + use lance_index::vector::v3::subindex::IvfSubIndex; use crate::dataset::{InsertBuilder, UpdateBuilder, WriteMode, WriteParams}; use crate::index::DatasetIndexExt; use crate::index::DatasetIndexInternalExt; - use crate::index::vector::ivf::v2::IvfPq; + use crate::index::vector::ivf::v2::{ + IVFPartitionKey, IvfFlatIndex, IvfHnswSqIndex, IvfPq, IvfStateEntryBox, PartitionEntry, + }; use crate::utils::test::copy_test_data_to_tmp; use crate::{ Dataset, @@ -1949,20 +2315,27 @@ mod tests { dataset::optimize::{CompactionOptions, compact_files}, index::vector::IndexFileVersion, }; - use lance_core::cache::LanceCache; + use lance_core::cache::{CacheBackend, CacheCodecImpl, LanceCache, WeakLanceCache}; + use lance_core::deepsize::DeepSizeOf; use lance_core::utils::tempfile::TempStrDir; use lance_core::{ROW_ID, Result}; + use lance_datagen::{Dimension, RowCount, Seed, array, gen_batch}; use lance_encoding::decoder::DecoderPlugins; use lance_file::reader::{FileReader, FileReaderOptions}; - use lance_file::writer::FileWriter; use lance_index::IndexType; + use lance_index::optimize::OptimizeOptions; + use lance_index::prefilter::PreFilter; use lance_index::progress::IndexBuildProgress; use lance_index::vector::DIST_COL; + use lance_index::vector::flat::index::{FlatIndex, FlatQuantizer}; + use lance_index::vector::flat::storage::FlatFloatStorage; + use lance_index::vector::hnsw::HNSW; use lance_index::vector::hnsw::builder::HnswBuildParams; use lance_index::vector::ivf::IvfBuildParams; use lance_index::vector::kmeans::{KMeansParams, train_kmeans}; - use lance_index::vector::pq::PQBuildParams; + use lance_index::vector::pq::{PQBuildParams, ProductQuantizer}; use lance_index::vector::quantizer::QuantizerMetadata; + use lance_index::vector::sq::ScalarQuantizer; use lance_index::vector::sq::builder::SQBuildParams; use lance_index::vector::{ pq::storage::ProductQuantizationMetadata, @@ -1970,26 +2343,124 @@ mod tests { storage::STORAGE_METADATA_KEY, }; use lance_index::{INDEX_AUXILIARY_FILE_NAME, metrics::NoOpMetricsCollector}; - use lance_index::{optimize::OptimizeOptions, scalar::IndexReader}; use lance_io::{ - object_store::ObjectStore, + object_store::{ObjectStore, ObjectStoreParams, StorageOptionsAccessor}, scheduler::{ScanScheduler, SchedulerConfig}, utils::CachedFileSize, }; use lance_linalg::distance::{DistanceType, multivec_distance}; use lance_linalg::kernels::normalize_fsl; + use lance_select::{RowAddrMask, RowAddrTreeMap}; use lance_table::format::IndexMetadata; use lance_testing::datagen::{generate_random_array, generate_random_array_with_range}; - use rand::distr::uniform::SampleUniform; + use rand::distr::{Distribution, StandardUniform, uniform::SampleUniform}; use rand::{Rng, SeedableRng, rngs::StdRng}; use rstest::rstest; use uuid::Uuid; const NUM_ROWS: usize = 512; const DIM: usize = 32; + // 8-bit PQ needs at least 256 training vectors; 320 leaves a stable margin + // while 20 neighbors provide a useful recall oracle. + const PQ_MATRIX_NUM_ROWS: usize = 320; + const PQ_MATRIX_K: usize = 20; + // An 8-bit PQ codebook has 256 centroids, so this is the smallest valid + // training fixture shared by the 8-bit and 4-bit runtime cases. + const LIGHTWEIGHT_PQ_ROWS: usize = 256; + const LIGHTWEIGHT_PQ_PARTITIONS: usize = 2; + const LIGHTWEIGHT_PQ_SUB_VECTORS: usize = 4; lance_testing::define_stage_event_progress!(RecordingProgress, IndexBuildProgress, Result<()>); + struct PartitionCoverageTestFilter { + needs_partition_rows: bool, + } + + #[async_trait::async_trait] + impl PreFilter for PartitionCoverageTestFilter { + async fn wait_for_ready(&self) -> Result<()> { + Ok(()) + } + + fn is_empty(&self) -> bool { + false + } + + fn needs_partition_row_ids(&self) -> bool { + self.needs_partition_rows + } + + fn is_empty_for(&self, _rows: &RowAddrTreeMap) -> bool { + true + } + + fn mask(&self) -> Arc { + Arc::new(RowAddrMask::all_rows()) + } + + fn filter_row_ids<'a>(&self, row_ids: Box + 'a>) -> Vec { + row_ids.enumerate().map(|(index, _)| index as u64).collect() + } + } + + #[tokio::test] + async fn test_partition_coverage_is_only_built_for_capable_filters() { + let vectors = + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0_f32; 16]), 4) + .unwrap(); + let entry = Arc::new(PartitionEntry::::new( + FlatIndex::default(), + FlatFloatStorage::new(vectors, DistanceType::L2), + )); + let cache = LanceCache::with_capacity(1 << 20); + cache + .insert_with_key( + &IVFPartitionKey::::new(0), + entry.clone(), + ) + .await; + let weak_cache = WeakLanceCache::from(&cache); + let size_without_coverage = entry.deep_size_of(); + let cache_weight_without_coverage = cache.size_bytes().await; + + let ordinary_filter: Arc = Arc::new(PartitionCoverageTestFilter { + needs_partition_rows: false, + }); + let returned = super::IVFIndex::::prefilter_for_partition( + &weak_cache, + 0, + &entry, + ordinary_filter.clone(), + ) + .await + .unwrap(); + assert!(Arc::ptr_eq(&returned, &ordinary_filter)); + assert!(entry.partition_rows.get().is_none()); + assert_eq!(cache.size_bytes().await, cache_weight_without_coverage); + + let segment_filter: Arc = Arc::new(PartitionCoverageTestFilter { + needs_partition_rows: true, + }); + let returned = super::IVFIndex::::prefilter_for_partition( + &weak_cache, + 0, + &entry, + segment_filter, + ) + .await + .unwrap(); + assert!(returned.is_empty()); + + let first_rows = entry.partition_rows(); + let second_rows = entry.partition_rows(); + assert!(Arc::ptr_eq(&first_rows, &second_rows)); + assert!(entry.deep_size_of() > size_without_coverage); + let cache_weight_with_coverage = cache.size_bytes().await; + assert!(cache_weight_with_coverage > cache_weight_without_coverage); + assert!(cache_weight_with_coverage >= entry.deep_size_of()); + assert!(entry.partition_rows_accounted.load(Ordering::Acquire)); + } + #[test] fn test_rotated_partition_centroid_slice_borrows_cache() { let cache = super::RabitSearchCache { @@ -2277,27 +2748,32 @@ mod tests { fn generate_clustered_multivec_batch( cluster_sizes: &[usize], - offsets: &[f32], + centroids: &[(f32, f32)], vectors_per_row: usize, + start_id: u64, ) -> (RecordBatch, SchemaRef) { assert_eq!( cluster_sizes.len(), - offsets.len(), - "cluster sizes and offsets must match" + centroids.len(), + "cluster sizes and centroids must match" ); const ITEM_FIELD_NAME: &str = "item"; let total_rows: usize = cluster_sizes.iter().sum(); let mut ids = Vec::with_capacity(total_rows); let mut values = Vec::with_capacity(total_rows * vectors_per_row * DIM); let mut rng = StdRng::seed_from_u64(12345); - let mut current_id = 0u64; - for (&rows, &offset) in cluster_sizes.iter().zip(offsets.iter()) { + let mut current_id = start_id; + for (&rows, &(x, y)) in cluster_sizes.iter().zip(centroids.iter()) { for _ in 0..rows { ids.push(current_id); current_id += 1; for _ in 0..vectors_per_row { for dim in 0..DIM { - let base = if dim == 0 { offset } else { 0.0 }; + let base = match dim { + 0 => x, + 1 => y, + _ => 0.0, + }; let noise = (rng.random::() - 0.5) * 0.02; values.push(base + noise); } @@ -2349,6 +2825,23 @@ mod tests { ) } + fn build_centroids_2d(centroids: &[(f32, f32)]) -> Arc { + let mut values = Vec::with_capacity(centroids.len() * DIM); + for &(x, y) in centroids { + for dim in 0..DIM { + values.push(match dim { + 0 => x, + 1 => y, + _ => 0.0, + }); + } + } + Arc::new( + FixedSizeListArray::try_new_from_values(Float32Array::from(values), DIM as i32) + .unwrap(), + ) + } + fn make_fragment_offset_batches( rows_per_fragment: usize, offsets: &[f32], @@ -2419,82 +2912,224 @@ mod tests { .downcast_ref::() .expect("expected IvfPq index") } + + fn ivf_flat(&self) -> &IvfFlatIndex { + self.index + .as_any() + .downcast_ref::() + .expect("expected IvfFlat index") + } } - async fn load_vector_index_context( - dataset: &Dataset, - column: &str, - index_name: &str, - ) -> VectorIndexTestContext { - let stats_json = dataset.index_statistics(index_name).await.unwrap(); - let stats: serde_json::Value = serde_json::from_str(&stats_json).unwrap(); - let uuid_str = stats["indices"][0]["uuid"] - .as_str() - .expect("Index uuid should be present"); - let uuid = Uuid::parse_str(uuid_str).expect("uuid in stats should be a valid UUID"); - let index = dataset - .open_vector_index(column, &uuid, &NoOpMetricsCollector) - .await - .unwrap(); + fn lightweight_pq_params() -> PQBuildParams { + PQBuildParams { + num_sub_vectors: LIGHTWEIGHT_PQ_SUB_VECTORS, + num_bits: 4, + max_iters: 2, + sample_rate: 16, + ..Default::default() + } + } - VectorIndexTestContext { - stats_json, - stats, - index, + fn lightweight_pq_params_with_bits(num_bits: usize) -> PQBuildParams { + let num_sub_vectors = if num_bits == 4 { + // M4 is only a 2-byte code, so random KMeans/HNSW can leave recall near + // the threshold. M32 restores the original 4-bit test capacity. + DIM + } else { + LIGHTWEIGHT_PQ_SUB_VECTORS + }; + PQBuildParams { + num_sub_vectors, + num_bits, + max_iters: 2, + sample_rate: 16, + ..Default::default() } } - async fn verify_partition_split_after_append( - mut dataset: Dataset, - test_uri: &str, - params: VectorIndexParams, - description: &str, + fn lightweight_hnsw_params() -> HnswBuildParams { + HnswBuildParams::default() + .max_level(2) + .num_edges(4) + .ef_construction(16) + } + + fn make_seeded_vector_batch(num_rows: usize) -> (RecordBatch, SchemaRef) { + let batch = lance_datagen::gen_batch() + .with_seed(lance_datagen::Seed::from(42)) + .col("id", lance_datagen::array::step::()) + .col( + "vector", + lance_datagen::array::rand_vec::((DIM as u32).into()), + ) + .into_batch_rows(lance_datagen::RowCount::from(num_rows as u64)) + .unwrap(); + let schema = batch.schema(); + (batch, schema) + } + + async fn search_lightweight_pq_index( + dataset: &Dataset, + query: &dyn Array, + k: usize, + num_partitions: usize, + refine_factor: u32, + ef: usize, + ) -> RecordBatch { + dataset + .scan() + .nearest("vector", query, k) + .unwrap() + .minimum_nprobes(num_partitions) + .ef(ef) + .refine(refine_factor) + .with_row_id() + .try_into_batch() + .await + .unwrap() + } + + async fn assert_lightweight_pq_index( + distance_type: DistanceType, + num_bits: usize, + use_hnsw: bool, ) { - const INDEX_NAME: &str = "vector_idx"; - const APPEND_ROWS: usize = 50_000; + const INDEX_NAME: &str = "test_index"; + const K: usize = 10; + let test_dir = TempStrDir::default(); + let (batch, schema) = make_seeded_vector_batch(LIGHTWEIGHT_PQ_ROWS); + let vectors = batch["vector"].as_fixed_size_list().clone(); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(batches, test_dir.as_str(), None) + .await + .unwrap(); + + let mut ivf_params = IvfBuildParams::new(LIGHTWEIGHT_PQ_PARTITIONS); + ivf_params.max_iters = 2; + ivf_params.sample_rate = 16; + let pq_params = lightweight_pq_params_with_bits(num_bits); + let expected_num_sub_vectors = pq_params.num_sub_vectors; + let params = if use_hnsw { + VectorIndexParams::with_ivf_hnsw_pq_params( + distance_type, + ivf_params, + lightweight_hnsw_params(), + pq_params, + ) + } else { + VectorIndexParams::with_ivf_pq_params(distance_type, ivf_params, pq_params) + }; dataset .create_index( &["vector"], IndexType::Vector, - Some(INDEX_NAME.to_string()), + Some(INDEX_NAME.to_owned()), ¶ms, true, ) .await .unwrap(); - let initial_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + let stats_json = dataset.index_statistics(INDEX_NAME).await.unwrap(); + let stats: serde_json::Value = serde_json::from_str(&stats_json).unwrap(); + let expected_index_type = if use_hnsw { "IVF_HNSW_PQ" } else { "IVF_PQ" }; + let expected_sub_index = if use_hnsw { "HNSW" } else { "PQ" }; + assert_eq!(stats["index_type"], expected_index_type); assert_eq!( - initial_ctx.num_partitions(), - 2, - "Expected {} initial partitions to be 2 before append, got stats: {}", - description, - initial_ctx.stats_json() + stats["indices"][0]["num_partitions"], + LIGHTWEIGHT_PQ_PARTITIONS + ); + assert_eq!( + stats["indices"][0]["sub_index"]["index_type"], + expected_sub_index + ); + assert_eq!(stats["indices"][0]["sub_index"]["nbits"], num_bits); + assert_eq!( + stats["indices"][0]["sub_index"]["num_sub_vectors"], + expected_num_sub_vectors ); + if use_hnsw { + let hnsw_params = &stats["indices"][0]["sub_index"]["params"]; + assert_eq!(hnsw_params["max_level"], 2); + assert_eq!(hnsw_params["m"], 4); + assert_eq!(hnsw_params["ef_construction"], 16); + } - // Append tightly clustered vectors so data flows into the same partition. - append_dataset::(&mut dataset, APPEND_ROWS, 0.0..0.05).await; + let query = vectors.value(0); + let ground_truth = ground_truth(&dataset, "vector", query.as_ref(), K, distance_type).await; + let before_reopen = search_lightweight_pq_index( + &dataset, + query.as_ref(), + K, + LIGHTWEIGHT_PQ_PARTITIONS, + 4, + 64, + ) + .await; + assert_eq!(before_reopen.num_rows(), K); + let row_ids = before_reopen[ROW_ID].as_primitive::().values(); + assert_eq!(row_ids.iter().copied().collect::>().len(), K); + let distances = before_reopen[DIST_COL] + .as_primitive::() + .values(); + assert!(distances.iter().all(|distance| distance.is_finite())); + assert!(distances.windows(2).all(|pair| pair[0] <= pair[1])); + let recall = row_ids + .iter() + .filter(|row_id| ground_truth.contains(row_id)) + .count() as f32 + / K as f32; + assert_ge!(recall, 0.5, "recall: {recall}"); + + drop(dataset); + let reopened = Dataset::open(test_dir.as_str()).await.unwrap(); + let reopened_stats: serde_json::Value = + serde_json::from_str(&reopened.index_statistics(INDEX_NAME).await.unwrap()).unwrap(); + assert_eq!(reopened_stats, stats); + assert_eq!( + search_lightweight_pq_index( + &reopened, + query.as_ref(), + K, + LIGHTWEIGHT_PQ_PARTITIONS, + 4, + 64, + ) + .await, + before_reopen + ); + } - dataset - .optimize_indices(&OptimizeOptions::new()) + async fn load_vector_index_context( + dataset: &Dataset, + column: &str, + index_name: &str, + ) -> VectorIndexTestContext { + let stats_json = dataset.index_statistics(index_name).await.unwrap(); + let stats: serde_json::Value = serde_json::from_str(&stats_json).unwrap(); + let uuid_str = stats["indices"][0]["uuid"] + .as_str() + .expect("Index uuid should be present"); + let uuid = Uuid::parse_str(uuid_str).expect("uuid in stats should be a valid UUID"); + let index = dataset + .open_vector_index(column, &uuid, &NoOpMetricsCollector) .await .unwrap(); - let dataset = Dataset::open(test_uri).await.unwrap(); - let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; - assert!( - final_ctx.num_partitions() >= 3, - "Expected partition split to increase partitions beyond 2 for {}, got stats: {}", - description, - final_ctx.stats_json() - ); + VectorIndexTestContext { + stats_json, + stats, + index, + } } async fn shrink_smallest_partition( dataset: &mut Dataset, index_name: &str, expected_after_join: usize, + next_id: &mut u64, ) -> (usize, usize, usize) { const ROWS_TO_APPEND_FOR_JOIN: usize = 32; let row_count_before = dataset.count_all_rows().await.unwrap(); @@ -2532,7 +3167,13 @@ mod tests { delete_ids(dataset, &ids[1..]).await; compact_after_deletions(dataset).await; - append_constant_vector(dataset, ROWS_TO_APPEND_FOR_JOIN, &template_values).await; + append_constant_vector_with_start_id( + dataset, + ROWS_TO_APPEND_FOR_JOIN, + &template_values, + next_id, + ) + .await; dataset .optimize_indices(&OptimizeOptions::new()) .await @@ -2558,8 +3199,14 @@ mod tests { (deleted_rows, ROWS_TO_APPEND_FOR_JOIN, post_partitions) } - async fn append_constant_vector(dataset: &mut Dataset, rows: usize, template: &[f32]) { - append_constant_vector_with_params(dataset, rows, template, None).await; + async fn append_constant_vector_with_start_id( + dataset: &mut Dataset, + rows: usize, + template: &[f32], + next_id: &mut u64, + ) { + append_constant_vector_batch(dataset, rows, template, *next_id, None).await; + *next_id += rows as u64; } async fn append_partition_templates( @@ -2612,6 +3259,17 @@ mod tests { rows: usize, template: &[f32], write_params: Option, + ) { + let start_id = dataset.count_all_rows().await.unwrap() as u64; + append_constant_vector_batch(dataset, rows, template, start_id, write_params).await; + } + + async fn append_constant_vector_batch( + dataset: &mut Dataset, + rows: usize, + template: &[f32], + start_id: u64, + write_params: Option, ) { assert_eq!( template.len(), @@ -2620,7 +3278,6 @@ mod tests { DIM ); - let start_id = dataset.count_all_rows().await.unwrap() as u64; let ids = Arc::new(UInt64Array::from_iter_values( start_id..start_id + rows as u64, )); @@ -2653,13 +3310,14 @@ mod tests { dataset: &mut Dataset, index_name: &str, template: &[f32], + next_id: &mut u64, rows_to_append: usize, expected_partitions: usize, expected_total_rows: usize, expected_index_count: usize, expect_split: bool, ) { - append_constant_vector(dataset, rows_to_append, template).await; + append_constant_vector_with_start_id(dataset, rows_to_append, template, next_id).await; dataset .optimize_indices(&OptimizeOptions::new()) .await @@ -2757,6 +3415,17 @@ mod tests { .collect() } + async fn load_flat_partition_row_ids(index: &IvfFlatIndex, partition_idx: usize) -> Vec { + index + .storage + .load_partition(partition_idx, None) + .await + .unwrap() + .row_ids() + .copied() + .collect() + } + async fn delete_ids(dataset: &mut Dataset, ids: &[u64]) { if ids.is_empty() { return; @@ -2864,11 +3533,20 @@ mod tests { test_uri: &str, schema: Arc, batches: Vec, + ) -> Dataset { + write_dataset_from_batches_with_max_rows(test_uri, schema, batches, 500).await + } + + async fn write_dataset_from_batches_with_max_rows( + test_uri: &str, + schema: Arc, + batches: Vec, + max_rows_per_file: usize, ) -> Dataset { let batches = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); let write_params = WriteParams { - max_rows_per_file: 500, + max_rows_per_file, mode: WriteMode::Overwrite, ..Default::default() }; @@ -2881,6 +3559,30 @@ mod tests { async fn prepare_global_ivf_pq( dataset: &Dataset, vector_column: &str, + ) -> (IvfBuildParams, PQBuildParams) { + prepare_ivf_pq( + dataset, + vector_column, + TWO_FRAG_DIM, + TWO_FRAG_NUM_PARTITIONS, + TWO_FRAG_NUM_SUBVECTORS, + TWO_FRAG_NUM_BITS, + TWO_FRAG_MAX_ITERS, + TWO_FRAG_SAMPLE_RATE, + ) + .await + } + + #[allow(clippy::too_many_arguments)] + async fn prepare_ivf_pq( + dataset: &Dataset, + vector_column: &str, + expected_dimension: usize, + num_partitions: usize, + num_sub_vectors: usize, + num_bits: usize, + max_iters: u32, + sample_rate: usize, ) -> (IvfBuildParams, PQBuildParams) { let batch = dataset .scan() @@ -2895,39 +3597,33 @@ mod tests { .as_fixed_size_list(); let dim = vectors.value_length() as usize; - assert_eq!(dim, TWO_FRAG_DIM, "unexpected vector dimension"); + assert_eq!(dim, expected_dimension, "unexpected vector dimension"); let values = vectors.values().as_primitive::(); - let kmeans_params = KMeansParams::new(None, TWO_FRAG_MAX_ITERS, 1, DistanceType::L2); - let kmeans = train_kmeans::( - values, - kmeans_params, - dim, - TWO_FRAG_NUM_PARTITIONS, - TWO_FRAG_SAMPLE_RATE, - ) - .unwrap(); + let kmeans_params = KMeansParams::new(None, max_iters, 1, DistanceType::L2); + let kmeans = + train_kmeans::(values, kmeans_params, dim, num_partitions, sample_rate) + .unwrap(); let centroids_flat = kmeans.centroids.as_primitive::().clone(); let centroids_fsl = Arc::new(FixedSizeListArray::try_new_from_values(centroids_flat, dim as i32).unwrap()); let mut ivf_params = - IvfBuildParams::try_with_centroids(TWO_FRAG_NUM_PARTITIONS, centroids_fsl).unwrap(); - ivf_params.max_iters = TWO_FRAG_MAX_ITERS as usize; - ivf_params.sample_rate = TWO_FRAG_SAMPLE_RATE; + IvfBuildParams::try_with_centroids(num_partitions, centroids_fsl).unwrap(); + ivf_params.max_iters = max_iters as usize; + ivf_params.sample_rate = sample_rate; - let mut pq_train_params = PQBuildParams::new(TWO_FRAG_NUM_SUBVECTORS, TWO_FRAG_NUM_BITS); - pq_train_params.max_iters = TWO_FRAG_MAX_ITERS as usize; - pq_train_params.sample_rate = TWO_FRAG_SAMPLE_RATE; + let mut pq_train_params = PQBuildParams::new(num_sub_vectors, num_bits); + pq_train_params.max_iters = max_iters as usize; + pq_train_params.sample_rate = sample_rate; let pq = pq_train_params.build(vectors, DistanceType::L2).unwrap(); let codebook_flat = pq.codebook.values().as_primitive::().clone(); let pq_codebook: ArrayRef = Arc::new(codebook_flat); - let mut pq_params = - PQBuildParams::with_codebook(TWO_FRAG_NUM_SUBVECTORS, TWO_FRAG_NUM_BITS, pq_codebook); - pq_params.max_iters = TWO_FRAG_MAX_ITERS as usize; - pq_params.sample_rate = TWO_FRAG_SAMPLE_RATE; + let mut pq_params = PQBuildParams::with_codebook(num_sub_vectors, num_bits, pq_codebook); + pq_params.max_iters = max_iters as usize; + pq_params.sample_rate = sample_rate; (ivf_params, pq_params) } @@ -3666,9 +4362,21 @@ mod tests { async fn test_merge_existing_hnsw_segments_rebuilds_graph(#[case] expected_index_type: &str) { let test_dir = TempStrDir::default(); let base_uri = test_dir.as_str(); - let (schema, batches) = make_two_fragment_batches(); + let (schema, batches, max_rows_per_file) = if expected_index_type == "IVF_HNSW_PQ" { + let (batch, schema) = make_seeded_vector_batch(LIGHTWEIGHT_PQ_ROWS * 2); + (schema, vec![batch], LIGHTWEIGHT_PQ_ROWS) + } else { + let (schema, batches) = make_two_fragment_batches(); + (schema, batches, 500) + }; let dataset_uri = format!("{}/merge_hnsw_rebuilds_graph", base_uri); - let mut dataset = write_dataset_from_batches(&dataset_uri, schema, batches).await; + let mut dataset = write_dataset_from_batches_with_max_rows( + &dataset_uri, + schema, + batches, + max_rows_per_file, + ) + .await; let fragments = dataset.get_fragments(); assert!(fragments.len() >= 2); @@ -3679,11 +4387,21 @@ mod tests { HnswBuildParams::default(), ), "IVF_HNSW_PQ" => { - let (ivf_params, pq_params) = prepare_global_ivf_pq(&dataset, "vector").await; + let (ivf_params, pq_params) = prepare_ivf_pq( + &dataset, + "vector", + DIM, + LIGHTWEIGHT_PQ_PARTITIONS, + LIGHTWEIGHT_PQ_SUB_VECTORS, + 8, + 2, + 16, + ) + .await; VectorIndexParams::with_ivf_hnsw_pq_params( DistanceType::L2, ivf_params, - HnswBuildParams::default(), + lightweight_hnsw_params(), pq_params, ) } @@ -3816,7 +4534,6 @@ mod tests { ); let expected_rows = fragments[0].physical_rows().await.unwrap() as u64 + fragments[1].physical_rows().await.unwrap() as u64; - let (ivf_params, pq_params) = prepare_global_ivf_pq(&dataset, "vector").await; let params = VectorIndexParams::with_ivf_pq_params(DistanceType::L2, ivf_params, pq_params); let mut segments = Vec::new(); @@ -4013,6 +4730,126 @@ mod tests { } } + fn pq_matrix_batch() -> RecordBatch + where + T: ArrowPrimitiveType + 'static, + T::Native: Copy + 'static, + PrimitiveArray: From> + 'static, + StandardUniform: Distribution, + { + gen_batch() + .with_seed(Seed(42)) + .col("id", array::step::()) + .col("vector", array::rand_vec::(Dimension::from(DIM as u32))) + .into_batch_rows(RowCount::from(PQ_MATRIX_NUM_ROWS as u64)) + .unwrap() + } + + fn pq_matrix_params( + nlist: usize, + distance_type: DistanceType, + version: IndexFileVersion, + ) -> VectorIndexParams { + let mut ivf_params = IvfBuildParams::new(nlist); + ivf_params.max_iters = 2; + ivf_params.sample_rate = PQ_MATRIX_NUM_ROWS; + let pq_params = PQBuildParams { + num_sub_vectors: 4, + num_bits: 8, + max_iters: 2, + sample_rate: 1, + ..Default::default() + }; + let mut params = + VectorIndexParams::with_ivf_pq_params(distance_type, ivf_params, pq_params); + params.version(version); + params + } + + async fn test_pq_matrix_case( + nlist: usize, + distance_type: DistanceType, + version: IndexFileVersion, + ) { + const INDEX_NAME: &str = "pq_matrix"; + + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let batch = pq_matrix_batch::(); + let schema = batch.schema(); + let query = batch["vector"].as_fixed_size_list().value(0); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap(); + let params = pq_matrix_params(nlist, distance_type, version.clone()); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some(INDEX_NAME.to_string()), + ¶ms, + true, + ) + .await + .unwrap(); + + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics(INDEX_NAME).await.unwrap()).unwrap(); + assert_eq!(stats["index_type"], "IVF_PQ"); + let indices = stats["indices"].as_array().unwrap(); + assert_eq!(indices.len(), 1); + let index = &indices[0]; + assert_eq!(index["index_type"], "IVF_PQ"); + assert_eq!(index["metric_type"], distance_type.to_string()); + assert_eq!(index["num_partitions"], nlist); + assert_eq!(index["sub_index"]["index_type"], "PQ"); + assert_eq!( + index["index_file_version"], + match version { + IndexFileVersion::Legacy => "Legacy", + IndexFileVersion::V3 => "V3", + } + ); + + drop(dataset); + let dataset = Dataset::open(test_uri).await.unwrap(); + let ground_truth = ground_truth( + &dataset, + "vector", + query.as_ref(), + PQ_MATRIX_K, + distance_type, + ) + .await; + let result = dataset + .scan() + .nearest("vector", query.as_primitive::(), PQ_MATRIX_K) + .unwrap() + .nprobes(nlist) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + + assert_eq!(result.num_rows(), PQ_MATRIX_K); + let row_ids = result[ROW_ID].as_primitive::().values(); + assert_eq!( + row_ids.iter().copied().collect::>().len(), + PQ_MATRIX_K + ); + let distances = result[DIST_COL].as_primitive::().values(); + assert!(distances.iter().all(|distance| distance.is_finite())); + assert!( + distances.windows(2).all(|pair| pair[0] <= pair[1]), + "distances are not sorted: {distances:?}" + ); + let recall = row_ids + .iter() + .filter(|row_id| ground_truth.contains(row_id)) + .count() as f32 + / PQ_MATRIX_K as f32; + assert_ge!(recall, 0.5, "recall: {recall}, row_ids: {row_ids:?}"); + } + async fn test_index_impl( params: VectorIndexParams, nlist: usize, @@ -4306,78 +5143,75 @@ mod tests { } #[rstest] - #[case(4, DistanceType::L2, 0.9)] - #[case(4, DistanceType::Cosine, 0.9)] - #[case(4, DistanceType::Dot, 0.85)] + #[case::l2(4, DistanceType::L2)] + #[case::cosine(4, DistanceType::Cosine)] + #[case::dot(4, DistanceType::Dot)] #[tokio::test] - async fn test_build_ivf_pq( - #[case] nlist: usize, - #[case] distance_type: DistanceType, - #[case] recall_requirement: f32, - ) { - let ivf_params = IvfBuildParams::new(nlist); - let pq_params = PQBuildParams::default(); - let params = VectorIndexParams::with_ivf_pq_params(distance_type, ivf_params, pq_params) - .version(crate::index::vector::IndexFileVersion::Legacy) - .clone(); - test_index(params.clone(), nlist, recall_requirement, None).await; - if distance_type == DistanceType::Cosine { - test_index_multivec(params.clone(), nlist, recall_requirement).await; - } - test_distance_range(Some(params.clone()), nlist).await; - // PQ performs worse on farther vectors, so if we delete the many nearest vectors, the recall will be lower - // lower the recall requirement in remap case for PQ, because it deletes half of the vectors - test_remap(params, nlist, recall_requirement * 0.9).await; + async fn test_build_ivf_pq(#[case] nlist: usize, #[case] distance_type: DistanceType) { + test_pq_matrix_case(nlist, distance_type, IndexFileVersion::Legacy).await; } #[rstest] - #[case(1, DistanceType::L2, 0.9)] - #[case(1, DistanceType::Cosine, 0.9)] - #[case(1, DistanceType::Dot, 0.85)] - #[case(4, DistanceType::L2, 0.9)] - #[case(4, DistanceType::Cosine, 0.9)] - #[case(4, DistanceType::Dot, 0.85)] + #[case::l2_nlist1(1, DistanceType::L2)] + #[case::cosine_nlist1(1, DistanceType::Cosine)] + #[case::dot_nlist1(1, DistanceType::Dot)] + #[case::l2_nlist4(4, DistanceType::L2)] + #[case::cosine_nlist4(4, DistanceType::Cosine)] + #[case::dot_nlist4(4, DistanceType::Dot)] #[tokio::test] - async fn test_build_ivf_pq_v3( - #[case] nlist: usize, - #[case] distance_type: DistanceType, - #[case] recall_requirement: f32, - ) { - let ivf_params = IvfBuildParams::new(nlist); - let pq_params = PQBuildParams::default(); - let params = VectorIndexParams::with_ivf_pq_params(distance_type, ivf_params, pq_params); - test_index(params.clone(), nlist, recall_requirement, None).await; - if distance_type == DistanceType::Cosine { - test_index_multivec(params.clone(), nlist, recall_requirement).await; - } - test_distance_range(Some(params.clone()), nlist).await; - // PQ performs worse on farther vectors, so if we delete the many nearest vectors, the recall will be lower - // lower the recall requirement in remap case for PQ, because it deletes half of the vectors - test_remap(params.clone(), nlist, recall_requirement * 0.9).await; - test_delete_all_rows(params).await; + async fn test_build_ivf_pq_v3(#[case] nlist: usize, #[case] distance_type: DistanceType) { + test_pq_matrix_case(nlist, distance_type, IndexFileVersion::V3).await; } #[rstest] - // Temporarily disable recall checks for 4-bit PQ. - #[case(4, DistanceType::L2, 0.0)] - #[case(4, DistanceType::Cosine, 0.0)] - #[case(4, DistanceType::Dot, 0.0)] + #[case::legacy(IndexFileVersion::Legacy)] + #[case::v3(IndexFileVersion::V3)] #[tokio::test] - async fn test_build_ivf_pq_4bit( - #[case] nlist: usize, - #[case] distance_type: DistanceType, - #[case] recall_requirement: f32, - ) { - let ivf_params = IvfBuildParams::new(nlist); - let pq_params = PQBuildParams::new(32, 4); - let params = VectorIndexParams::with_ivf_pq_params(distance_type, ivf_params, pq_params); - test_index(params.clone(), nlist, recall_requirement, None).await; - if distance_type == DistanceType::Cosine { - test_index_multivec(params.clone(), nlist, recall_requirement).await; - } - // PQ performs worse on farther vectors, so if we delete the many nearest vectors, the recall will be lower - // lower the recall requirement in remap case for PQ, because it deletes half of the vectors - test_remap(params, nlist, recall_requirement * 0.9).await; + async fn test_ivf_pq_distance_range(#[case] version: IndexFileVersion) { + let params = pq_matrix_params(1, DistanceType::L2, version); + test_distance_range(Some(params), 1).await; + } + + #[rstest] + #[case::legacy(IndexFileVersion::Legacy)] + #[case::v3(IndexFileVersion::V3)] + #[tokio::test] + async fn test_ivf_pq_f64_smoke(#[case] version: IndexFileVersion) { + let test_dir = TempStrDir::default(); + let batch = pq_matrix_batch::(); + let schema = batch.schema(); + let vectors = Arc::new(batch["vector"].as_fixed_size_list().clone()); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(batches, test_dir.as_str(), None) + .await + .unwrap(); + let params = pq_matrix_params(1, DistanceType::L2, version); + dataset + .create_index(&["vector"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + test_recall::(params, 1, 0.5, "vector", &dataset, vectors).await; + } + + #[tokio::test] + async fn test_legacy_ivf_pq_cosine_multivec_smoke() { + let params = pq_matrix_params(1, DistanceType::Cosine, IndexFileVersion::Legacy); + test_index_multivec_impl::(params, 1, 0.5, 0.0..1.0).await; + } + + #[tokio::test] + async fn test_ivf_pq_delete_all_rows_lifecycle() { + let params = pq_matrix_params(1, DistanceType::L2, IndexFileVersion::V3); + test_delete_all_rows(params).await; + } + + #[rstest] + #[case::l2(DistanceType::L2)] + #[case::cosine(DistanceType::Cosine)] + #[case::dot(DistanceType::Dot)] + #[tokio::test] + async fn test_build_ivf_pq_4bit(#[case] distance_type: DistanceType) { + assert_lightweight_pq_index(distance_type, 4, false).await; } #[rstest] @@ -4411,17 +5245,19 @@ mod tests { test_index_impl::(params, nlist, 0.75, -1.0..1.0, None).await; } - // RQ doesn't perform well for random data - // need to verify recall with real-world dataset (e.g. sift1m) + // These queries probe every partition, so recall here measures RaBitQ quantization + // error alone. At 1 bit per dimension it averages ~0.67 on this uniformly random, + // L2-normalized data, and each build draws a fresh random rotation, so no bar worth + // asserting sits clear of the spread. 5 bits lifts recall to ~0.97; its `ex_bits = 4` + // also covers a FastScan ex-code kernel that the multi-bit test below never reaches. #[rstest] - #[case(1, DistanceType::L2, 0.5)] - #[case(1, DistanceType::Cosine, 0.5)] - #[case(1, DistanceType::Dot, 0.5)] - #[case(4, DistanceType::L2, 0.5)] - #[case(4, DistanceType::Cosine, 0.5)] - #[case(4, DistanceType::Dot, 0.5)] + #[case(1, DistanceType::L2, 0.9)] + #[case(1, DistanceType::Cosine, 0.9)] + #[case(1, DistanceType::Dot, 0.9)] + #[case(4, DistanceType::L2, 0.9)] + #[case(4, DistanceType::Cosine, 0.9)] + #[case(4, DistanceType::Dot, 0.9)] #[tokio::test] - // #[ignore = "Temporarily skipping flaky 4-bit IVF_RQ tests"] async fn test_build_ivf_rq( #[case] nlist: usize, #[case] distance_type: DistanceType, @@ -4430,7 +5266,7 @@ mod tests { ) { let _ = env_logger::try_init(); let ivf_params = IvfBuildParams::new(nlist); - let rq_params = RQBuildParams::with_rotation_type(1, rotation_type); + let rq_params = RQBuildParams::with_rotation_type(5, rotation_type); let params = VectorIndexParams::with_ivf_rq_params(distance_type, ivf_params, rq_params); test_index(params.clone(), nlist, recall_requirement, None).await; if distance_type == DistanceType::Cosine { @@ -4600,57 +5436,304 @@ mod tests { } #[rstest] - #[case(4, DistanceType::L2, 0.9)] - #[case(4, DistanceType::Cosine, 0.9)] - #[case(4, DistanceType::Dot, 0.85)] + #[case::l2(DistanceType::L2)] + #[case::cosine(DistanceType::Cosine)] + #[case::dot(DistanceType::Dot)] #[tokio::test] - async fn test_create_ivf_hnsw_pq( - #[case] nlist: usize, - #[case] distance_type: DistanceType, - #[case] recall_requirement: f32, - ) { - let ivf_params = IvfBuildParams::new(nlist); - let pq_params = PQBuildParams::default(); - let hnsw_params = HnswBuildParams::default(); + async fn test_create_ivf_hnsw_pq(#[case] distance_type: DistanceType) { + assert_lightweight_pq_index(distance_type, 8, true).await; + } + + #[rstest] + #[case::l2(DistanceType::L2)] + #[case::cosine(DistanceType::Cosine)] + #[case::dot(DistanceType::Dot)] + #[tokio::test] + async fn test_create_ivf_hnsw_pq_4bit(#[case] distance_type: DistanceType) { + assert_lightweight_pq_index(distance_type, 4, true).await; + } + + #[tokio::test] + async fn test_create_ivf_hnsw_pq_multivec() { + const NUM_ROWS: usize = 64; + const K: usize = 10; + + let test_dir = TempStrDir::default(); + let batch = lance_datagen::gen_batch() + .with_seed(lance_datagen::Seed::from(42)) + .col("id", lance_datagen::array::step::()) + .col( + "vector", + lance_datagen::array::cycle_vec_var( + lance_datagen::array::rand_vec::((DIM as u32).into()), + 3_u32.into(), + 4_u32.into(), + ), + ) + .into_batch_rows(lance_datagen::RowCount::from(NUM_ROWS as u64)) + .unwrap(); + let vectors = batch["vector"].as_list::().clone(); + let schema = batch.schema(); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(batches, test_dir.as_str(), None) + .await + .unwrap(); + + let mut ivf_params = IvfBuildParams::new(1); + ivf_params.max_iters = 2; + ivf_params.sample_rate = 16; let params = VectorIndexParams::with_ivf_hnsw_pq_params( - distance_type, + DistanceType::Cosine, ivf_params, - hnsw_params, - pq_params, + lightweight_hnsw_params(), + lightweight_pq_params(), ); - test_index(params.clone(), nlist, recall_requirement, None).await; - if distance_type == DistanceType::Cosine { - test_index_multivec(params.clone(), nlist, recall_requirement).await; + dataset + .create_index(&["vector"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + + let query = vectors.value(0); + // Three vectors per query amplify the internal candidate k. This + // bounded budget covers all 64 * 3 vector entries in the fixture. + let result = search_lightweight_pq_index(&dataset, query.as_ref(), K, 1, 2, 256).await; + assert_eq!(result.num_rows(), K); + let row_ids = result[ROW_ID].as_primitive::().values(); + assert_eq!(row_ids.iter().copied().collect::>().len(), K); + let distances = result[DIST_COL].as_primitive::().values(); + assert!(distances.iter().all(|distance| distance.is_finite())); + assert!(distances.windows(2).all(|pair| pair[0] <= pair[1])); + + let ground_truth = multivec_ground_truth(&vectors, query.as_ref(), K, DistanceType::Cosine) + .into_iter() + .map(|(_, row_id)| row_id) + .collect::>(); + let recall = row_ids + .iter() + .filter(|row_id| ground_truth.contains(row_id)) + .count() as f32 + / K as f32; + assert_ge!(recall, 0.5, "recall: {recall}"); + } + + // `lance-index` keeps these crate-private; spelling them out here also pins + // the on-disk names, which are part of the index file contract. + const HNSW_VECTOR_ID_COL: &str = "__vector_id"; + const HNSW_NEIGHBORS_COL: &str = "__neighbors"; + + async fn build_ivf_hnsw_sq(test_uri: &str, nlist: usize) -> Dataset { + let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; + let params = VectorIndexParams::with_ivf_hnsw_sq_params( + DistanceType::L2, + IvfBuildParams::new(nlist), + HnswBuildParams::default(), + SQBuildParams::default(), + ); + dataset + .create_index(&["vector"], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); + dataset + } + + async fn open_ivf_hnsw_sq(dataset: &Dataset) -> Arc { + let indices = dataset.load_indices().await.unwrap(); + dataset + .open_vector_index("vector", &indices[0].uuid, &NoOpMetricsCollector) + .await + .unwrap() + } + + async fn assert_hnsw_columns(dataset: &Dataset, context: &str) { + let index = open_ivf_hnsw_sq(dataset).await; + let hnsw = index + .as_any() + .downcast_ref::() + .expect("IVF_HNSW_SQ should open as IvfHnswSqIndex"); + + let written = hnsw + .reader + .schema() + .fields + .iter() + .map(|f| f.name.as_str()) + .collect::>(); + assert_eq!( + written, + vec![HNSW_VECTOR_ID_COL, HNSW_NEIGHBORS_COL, DIST_COL], + "{context}: the written index file must keep every column" + ); + + // Every partition, not just the first: a projection that applied + // unevenly would leave the partition cache holding mixed schemas. + for partition_id in 0..hnsw.ivf.num_partitions() { + let entry = hnsw.load_partition_entry(partition_id, None).await.unwrap(); + let loaded = entry.index.to_batch().unwrap(); + let loaded_schema = loaded.schema(); + let read = loaded_schema + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(); + assert_eq!( + read, + vec![HNSW_VECTOR_ID_COL, HNSW_NEIGHBORS_COL], + "{context}: partition {partition_id} materialized the write-only distance column" + ); } - // PQ performs worse on farther vectors, so if we delete the many nearest vectors, the recall will be lower - // lower the recall requirement in remap case for PQ, because it deletes half of the vectors - test_remap(params, nlist, recall_requirement * 0.9).await; } - #[rstest] - // Temporarily disable recall checks for 4-bit PQ. - #[case(4, DistanceType::L2, 0.0)] - #[case(4, DistanceType::Cosine, 0.0)] - #[case(4, DistanceType::Dot, 0.0)] + /// The index file keeps all three HNSW columns while a loaded partition + /// carries only the two the graph reads. Both halves matter: shrinking the + /// written schema would panic readers older than v8.0.0, and widening the + /// read back would undo the saving. + /// + /// Re-checked after a delta merge, because that is the one path that writes + /// a new index file while an already-projected index is open. #[tokio::test] - async fn test_create_ivf_hnsw_pq_4bit( - #[case] nlist: usize, - #[case] distance_type: DistanceType, - #[case] recall_requirement: f32, - ) { - let ivf_params = IvfBuildParams::new(nlist); - let pq_params = PQBuildParams::new(32, 4); - let hnsw_params = HnswBuildParams::default(); - let params = VectorIndexParams::with_ivf_hnsw_pq_params( - distance_type, - ivf_params, - hnsw_params, - pq_params, + async fn test_hnsw_partition_load_reads_only_graph_columns() { + let test_dir = TempStrDir::default(); + let mut dataset = build_ivf_hnsw_sq(test_dir.as_str(), 4).await; + assert_hnsw_columns(&dataset, "fresh index").await; + + append_dataset::(&mut dataset, 64, 0.0..1.0).await; + dataset + .optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + dataset + .optimize_indices(&OptimizeOptions::merge(10)) + .await + .unwrap(); + assert_hnsw_columns(&dataset, "after delta merge").await; + } + + /// The saving is the point of the change, so pin it: reading a real + /// partition range through the declared projection must move strictly fewer + /// bytes than the full-schema read the index used to perform. + #[tokio::test] + async fn test_hnsw_read_projection_moves_fewer_bytes() { + use futures::TryStreamExt as _; + + let test_dir = TempStrDir::default(); + let dataset = build_ivf_hnsw_sq(test_dir.as_str(), 4).await; + let index = open_ivf_hnsw_sq(&dataset).await; + let hnsw = index + .as_any() + .downcast_ref::() + .expect("IVF_HNSW_SQ should open as IvfHnswSqIndex"); + + let projection = hnsw + .read_projection + .as_ref() + .expect("HNSW declares a read projection"); + assert_eq!(projection.schema.fields.len(), 2); + + let row_range = hnsw.ivf.row_range(0); + assert!(!row_range.is_empty(), "partition 0 should hold rows"); + let store = dataset.object_store.as_ref(); + + let read_bytes_for = async |projection: lance_file::reader::ReaderProjection| { + let _ = store.io_stats_incremental(); + hnsw.reader + .read_stream_projected( + lance_io::ReadBatchParams::Range(row_range.clone()), + u32::MAX, + 1, + projection, + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + store.io_stats_incremental().read_bytes + }; + + // Projected first on purpose: it then pays any first-touch metadata + // cost, so the comparison understates rather than flatters the saving. + let projected_bytes = read_bytes_for(projection.clone()).await; + let full_bytes = read_bytes_for(lance_file::versions::reader_projection_from_whole_schema( + hnsw.reader.schema(), + hnsw.reader.metadata().version(), + )) + .await; + + assert!( + projected_bytes > 0, + "the projected read still has to fetch the graph" ); - test_index(params.clone(), nlist, recall_requirement, None).await; - if distance_type == DistanceType::Cosine { - test_index_multivec(params, nlist, recall_requirement).await; + assert_lt!(projected_bytes, full_bytes); + } + + /// The projection selects columns by field id, and `__neighbors` and + /// `_distance` are both 4-byte-item lists whose child fields share a name, + /// so a wrong column index would reinterpret distances as neighbor ids with + /// no type error to catch it. Compare the columns themselves, not just names. + #[tokio::test] + async fn test_hnsw_projected_read_matches_full_read() { + use futures::TryStreamExt as _; + + let test_dir = TempStrDir::default(); + let dataset = build_ivf_hnsw_sq(test_dir.as_str(), 4).await; + let index = open_ivf_hnsw_sq(&dataset).await; + let hnsw = index + .as_any() + .downcast_ref::() + .expect("IVF_HNSW_SQ should open as IvfHnswSqIndex"); + let projection = hnsw + .read_projection + .as_ref() + .expect("HNSW declares a read projection"); + + let read_range = async |proj: lance_file::reader::ReaderProjection, + range: std::ops::Range| { + let batches = hnsw + .reader + .read_stream_projected( + lance_io::ReadBatchParams::Range(range), + u32::MAX, + 1, + proj, + lance_encoding::decoder::FilterExpression::no_filter(), + ) + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + arrow::compute::concat_batches(&batches[0].schema(), &batches).unwrap() + }; + + let mut compared = 0; + for partition_id in 0..hnsw.ivf.num_partitions() { + let range = hnsw.ivf.row_range(partition_id); + if range.is_empty() { + continue; + } + let full = read_range( + lance_file::versions::reader_projection_from_whole_schema( + hnsw.reader.schema(), + hnsw.reader.metadata().version(), + ), + range.clone(), + ) + .await; + let projected = read_range(projection.clone(), range).await; + + assert_eq!(projected.num_columns(), 2); + assert_eq!(projected.num_rows(), full.num_rows()); + for name in [HNSW_VECTOR_ID_COL, HNSW_NEIGHBORS_COL] { + assert_eq!( + projected.column_by_name(name).unwrap(), + full.column_by_name(name).unwrap(), + "partition {partition_id}: {name} differs between the projected and full read" + ); + } + compared += 1; } + assert!(compared > 0, "no non-empty partition was compared"); } async fn test_index_multivec(params: VectorIndexParams, nlist: usize, recall_requirement: f32) { @@ -4713,14 +5796,13 @@ mod tests { .as_primitive::() .values() .to_vec(); + assert_eq!(row_ids.len(), k); + assert_eq!(row_ids.iter().copied().collect::>().len(), k); let dists = result[DIST_COL] .as_primitive::() .values() .to_vec(); - let results = dists - .into_iter() - .zip(row_ids.clone().into_iter()) - .collect::>(); + let results = dists.into_iter().zip(row_ids.clone()).collect::>(); let row_ids = row_ids.into_iter().collect::>(); let gt = multivec_ground_truth(&vectors, &query, k, params.metric_type); @@ -4855,12 +5937,46 @@ mod tests { let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - let nlist = 500; - let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; + let num_rows = 32; + let num_partitions = num_rows + 2; + let mut vector_values = vec![0.0; num_rows * DIM]; + for row in 0..num_rows { + vector_values[row * DIM + row] = 1.0; + } + let one_hot_vectors = Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from(vector_values.clone()), + DIM as i32, + ) + .unwrap(), + ); + let batch = gen_batch() + .col("id", array::step::()) + .col("vector", array::jitter_centroids(one_hot_vectors, 0.0)) + .into_batch_rows(RowCount::from(num_rows as u64)) + .unwrap(); + let schema = batch.schema(); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut dataset = Dataset::write(batches, test_uri, None).await.unwrap(); - let ivf_params = IvfBuildParams::new(nlist); + // Keep partition 0 empty: stats previously failed when the first partition was empty. + let mut centroid_values = Vec::with_capacity(num_partitions * DIM); + centroid_values.extend(std::iter::repeat_n(2.0, DIM)); + centroid_values.extend(vector_values); + centroid_values.extend(std::iter::repeat_n(-2.0, DIM)); + let centroids = Arc::new( + FixedSizeListArray::try_new_from_values( + Float32Array::from(centroid_values), + DIM as i32, + ) + .unwrap(), + ); + let ivf_params = IvfBuildParams::try_with_centroids(num_partitions, centroids).unwrap(); let sq_params = SQBuildParams::default(); - let hnsw_params = HnswBuildParams::default(); + let hnsw_params = HnswBuildParams::default() + .max_level(1) + .num_edges(4) + .ef_construction(4); let params = VectorIndexParams::with_ivf_hnsw_sq_params( DistanceType::L2, ivf_params, @@ -4883,14 +5999,25 @@ mod tests { let stats: serde_json::Value = serde_json::from_str(stats.as_str()).unwrap(); assert_eq!(stats["index_type"].as_str().unwrap(), "IVF_HNSW_SQ"); - for index in stats["indices"].as_array().unwrap() { - assert_eq!(index["index_type"].as_str().unwrap(), "IVF_HNSW_SQ"); - assert_eq!( - index["num_partitions"].as_number().unwrap(), - &serde_json::Number::from(nlist) - ); - assert_eq!(index["sub_index"]["index_type"].as_str().unwrap(), "HNSW"); - } + let indices = stats["indices"].as_array().unwrap(); + assert_eq!(indices.len(), 1); + let index = &indices[0]; + assert_eq!(index["index_type"].as_str().unwrap(), "IVF_HNSW_SQ"); + assert_eq!( + index["num_partitions"].as_number().unwrap(), + &serde_json::Number::from(num_partitions) + ); + assert_eq!(index["sub_index"]["index_type"].as_str().unwrap(), "HNSW"); + let partition_sizes = index["partitions"] + .as_array() + .unwrap() + .iter() + .map(|partition| partition["size"].as_u64().unwrap()) + .collect::>(); + assert_eq!(partition_sizes.len(), num_partitions); + assert_eq!(partition_sizes.iter().sum::(), num_rows as u64); + assert_eq!(partition_sizes[0], 0); + assert!(partition_sizes.contains(&0)); } async fn test_distance_range(params: Option, nlist: usize) { @@ -5113,10 +6240,7 @@ mod tests { .as_primitive::() .values() .to_vec(); - let results = dists - .into_iter() - .zip(row_ids.into_iter()) - .collect::>(); + let results = dists.into_iter().zip(row_ids).collect::>(); let row_ids = results.iter().map(|(_, id)| *id).collect::>(); assert!(row_ids.len() == k); @@ -5172,11 +6296,25 @@ mod tests { // Rewrite auxiliary file with PQ codebook inlined into schema metadata. let mut metadata = reader.schema().metadata.clone(); - let batch = reader - .read_range(0..reader.num_rows() as usize, None) + let projection = lance_file::versions::reader_projection_from_whole_schema( + reader.schema(), + reader.metadata().version(), + ); + let batches = reader + .read_stream_projected( + lance_io::ReadBatchParams::RangeFull, + u32::MAX, + u32::MAX, + projection, + lance_encoding::decoder::FilterExpression::no_filter(), + ) .await?; + use futures::TryStreamExt as _; + let batches = batches.try_collect::>().await?; + let batch = arrow::compute::concat_batches(&batches[0].schema(), &batches)?; let new_aux_path = new_dir.clone().join(INDEX_AUXILIARY_FILE_NAME); - let mut writer = FileWriter::try_new( + let mut writer = lance_file::versions::create_writer( + reader.metadata().version(), obj_store.create(&new_aux_path).await?, batch.schema_ref().as_ref().try_into()?, Default::default(), @@ -5222,6 +6360,34 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_legacy_non_divisible_pq_search() { + const DIM: usize = 64; + const PERSISTED_DIM: usize = 56; + + let test_dir = copy_test_data_to_tmp("v0.10.15/non_divisible_pq").unwrap(); + let dataset = Dataset::open(&test_dir.path_str()).await.unwrap(); + let query = Float32Array::from( + (1..=DIM) + .map(|value| value as f32 + if value <= PERSISTED_DIM { 1.0 } else { 1_000.0 }) + .collect::>(), + ); + + let result = dataset + .scan() + .nearest("vector", &query, 1) + .unwrap() + .try_into_batch() + .await + .unwrap(); + + assert_eq!(result.num_rows(), 1); + assert_eq!( + result[DIST_COL].as_primitive::().values(), + &[PERSISTED_DIM as f32] + ); + } + #[tokio::test] async fn test_pq_storage_backwards_compat() { let test_dir = copy_test_data_to_tmp("v0.27.1/pq_in_schema").unwrap(); @@ -5335,78 +6501,26 @@ mod tests { } #[tokio::test] - async fn test_create_index_with_many_invalid_vectors() { + async fn test_compaction_remaps_second_delta_with_shared_partition_topology() { + const INDEX_NAME: &str = "vector_idx"; + const BASE_ROWS_PER_PARTITION: usize = 2_200; + const SMALL_APPEND_ROWS: usize = 64; + let offsets = [-50.0, 50.0]; + let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - // we use 8192 batch size by default, so we need to generate 8192 * 3 vectors to get 3 batches - // generate 3 batches, and the first batch's vectors are all with NaN - let num_rows = 8192 * 3; - let mut vectors = Vec::new(); - for i in 0..num_rows { - if i < 8192 { - vectors.extend(std::iter::repeat_n(f32::NAN, DIM)); - } else if i < 8192 * 2 { - vectors.extend(std::iter::repeat_n(rand::random::(), DIM)); - } else { - vectors.extend(std::iter::repeat_n(rand::random::() * 1e20, DIM)); - } - } - let schema = Schema::new(vec![Field::new( - "vector", - DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Float32, true)), - DIM as i32, - ), - true, - )]); - let schema = Arc::new(schema); - let batch = RecordBatch::try_new( - schema.clone(), - vec![Arc::new( - FixedSizeListArray::try_new_from_values(Float32Array::from(vectors), DIM as i32) - .unwrap(), - )], + let (batch, schema) = generate_clustered_batch(BASE_ROWS_PER_PARTITION, offsets); + let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); + let mut dataset = Dataset::write( + batches, + test_uri, + Some(WriteParams { + mode: WriteMode::Overwrite, + ..Default::default() + }), ) - .unwrap(); - let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); - let params = WriteParams { - mode: WriteMode::Overwrite, - ..Default::default() - }; - let mut dataset = Dataset::write(batches, test_uri, Some(params)) - .await - .unwrap(); - - let params = VectorIndexParams::ivf_pq(4, 8, DIM / 8, DistanceType::Dot, 50); - - dataset - .create_index(&["vector"], IndexType::Vector, None, ¶ms, true) - .await - .unwrap(); - } - - #[tokio::test] - async fn test_remap_join_on_second_delta() { - const INDEX_NAME: &str = "vector_idx"; - const BASE_ROWS_PER_PARTITION: usize = 3_000; - const SMALL_APPEND_ROWS: usize = 64; - let offsets = [-50.0, 50.0]; - - let test_dir = TempStrDir::default(); - let test_uri = test_dir.as_str(); - - let (batch, schema) = generate_clustered_batch(BASE_ROWS_PER_PARTITION, offsets); - let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); - let mut dataset = Dataset::write( - batches, - test_uri, - Some(WriteParams { - mode: WriteMode::Overwrite, - ..Default::default() - }), - ) - .await + .await .unwrap(); let centroids = build_centroids_for_offsets(&offsets); @@ -5414,7 +6528,7 @@ mod tests { let params = VectorIndexParams::with_ivf_pq_params( DistanceType::L2, ivf_params, - PQBuildParams::default(), + lightweight_pq_params(), ); dataset .create_index( @@ -5492,7 +6606,7 @@ mod tests { .await .unwrap(); - let mut dataset = Dataset::open(test_uri).await.unwrap(); + let dataset = Dataset::open(test_uri).await.unwrap(); let stats_after_compaction: serde_json::Value = serde_json::from_str(&dataset.index_statistics(INDEX_NAME).await.unwrap()).unwrap(); assert_eq!(stats_after_compaction["num_indices"].as_u64().unwrap(), 2); @@ -5507,48 +6621,21 @@ mod tests { partitions_after, vec![base_partition_count, base_partition_count] ); - - const LARGE_APPEND_ROWS: usize = 40_000; - append_constant_vector(&mut dataset, LARGE_APPEND_ROWS, &template_values).await; - dataset - .optimize_indices(&OptimizeOptions::new()) - .await - .unwrap(); - - let dataset = Dataset::open(test_uri).await.unwrap(); - let stats_after_split: serde_json::Value = - serde_json::from_str(&dataset.index_statistics(INDEX_NAME).await.unwrap()).unwrap(); - assert_eq!(stats_after_split["num_indices"].as_u64().unwrap(), 1); - let final_partition_count = stats_after_split["indices"][0]["num_partitions"] - .as_u64() - .unwrap() as usize; - assert_eq!( - final_partition_count, - base_partition_count + 1, - "expected split to increase partitions beyond {}, got {}", - base_partition_count, - final_partition_count - ); } #[tokio::test] async fn test_spfresh_join_split() { - // Two join cycles followed by three append cycles: - // 1. Each deletion shrinks the smallest partition and verifies the partition count. - // 2. Append #1 (10k rows) creates a delta index without splitting. - // 3. Append #2 and #3 (40k rows each) trigger splits, forcing merges and validating partition sizes. - const INDEX_NAME: &str = "vector_idx"; - const NLIST: usize = 3; - const FIRST_APPEND_ROWS: usize = 10_000; - const SECOND_APPEND_ROWS: usize = 30_000; - const THIRD_APPEND_ROWS: usize = 35_000; + const NLIST: usize = 2; + const NO_SPLIT_APPEND_ROWS: usize = 32; + // The joined base and no-split delta contain 2,265 rows. This append + // takes the single IVF-PQ partition one row past its 32,768-row limit. + const SPLIT_APPEND_ROWS: usize = 30_504; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - // Two small clusters (for joins) and two large clusters (for splits). - let cluster_sizes = [100, 4_000, 4_000]; + let cluster_sizes = [100, 2_200]; let total_rows: usize = cluster_sizes.iter().sum(); let mut centroid_values = Vec::new(); @@ -5610,7 +6697,7 @@ mod tests { let params = VectorIndexParams::with_ivf_pq_params( DistanceType::L2, ivf_params, - PQBuildParams::default(), + lightweight_pq_params(), ); dataset .create_index( @@ -5623,8 +6710,7 @@ mod tests { .await .unwrap(); - // Template vector from the first large cluster for deterministic appends. - let template_id = (cluster_sizes[0] + cluster_sizes[1]) as u64; + let template_id = cluster_sizes[0] as u64; let template_batch = dataset .take_rows(&[template_id], dataset.schema().clone()) .await @@ -5641,63 +6727,37 @@ mod tests { "Template vector should match DIM" ); - let mut expected_partitions = NLIST; + let mut next_id = total_rows as u64; let mut expected_rows = total_rows; - // Two join cycles. - for expected_after in [NLIST - 1, NLIST - 2] { - let (deleted_rows, appended_rows, actual_partitions) = - shrink_smallest_partition(&mut dataset, INDEX_NAME, expected_after).await; - expected_rows = expected_rows - deleted_rows + appended_rows; - assert_eq!( - dataset.count_all_rows().await.unwrap(), - expected_rows, - "Row count mismatch after join" - ); - expected_partitions = actual_partitions; - } + let (deleted_rows, appended_rows, actual_partitions) = + shrink_smallest_partition(&mut dataset, INDEX_NAME, 1, &mut next_id).await; + expected_rows = expected_rows - deleted_rows + appended_rows; + assert_eq!(actual_partitions, 1); + assert_eq!(dataset.count_all_rows().await.unwrap(), expected_rows); - // Append #1: no split, expect a delta index. - let rows = FIRST_APPEND_ROWS; append_and_verify_append_phase( &mut dataset, INDEX_NAME, &template_values, - rows, - expected_partitions, - expected_rows + rows, + &mut next_id, + NO_SPLIT_APPEND_ROWS, + 1, + expected_rows + NO_SPLIT_APPEND_ROWS, 2, false, ) .await; - expected_rows += rows; - - // Append #2: triggers split and merge. - expected_partitions += 1; - let rows = SECOND_APPEND_ROWS; - append_and_verify_append_phase( - &mut dataset, - INDEX_NAME, - &template_values, - rows, - expected_partitions, - expected_rows + rows, - 1, - true, - ) - .await; - expected_rows += rows; + expected_rows += NO_SPLIT_APPEND_ROWS; - // Append #3: triggers another split, remains a single merged index. - expected_partitions += 1; - let rows = THIRD_APPEND_ROWS; append_and_verify_append_phase( &mut dataset, INDEX_NAME, &template_values, - rows, - expected_partitions, - expected_rows + rows, + &mut next_id, + SPLIT_APPEND_ROWS, + 2, + expected_rows + SPLIT_APPEND_ROWS, 1, true, ) @@ -5706,28 +6766,93 @@ mod tests { #[tokio::test] async fn test_partition_split_on_append_multivec() { - // This test verifies that when we append enough multivector data to a partition - // such that it exceeds MAX_PARTITION_SIZE_FACTOR * target_partition_size, - // the partition will be split into 2 partitions. + const INDEX_NAME: &str = "vector_idx"; + const VECTORS_PER_ROW: usize = 3; + // 512 base rows and this append flatten to 33,036 vectors, just over + // the 32,768-vector IVF-PQ split threshold. + const APPEND_ROWS: usize = 10_500; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - // Create initial dataset with multivector data - let (dataset, _) = generate_multivec_test_dataset::(test_uri, 0.0..1.0).await; + let (mut dataset, _) = + generate_multivec_test_dataset::(test_uri, 0.0..1.0).await; + let params = VectorIndexParams::with_ivf_pq_params( + DistanceType::Cosine, + IvfBuildParams::new(1), + lightweight_pq_params(), + ); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some(INDEX_NAME.to_string()), + ¶ms, + true, + ) + .await + .unwrap(); + + let initial_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + assert_eq!(initial_ctx.num_partitions(), 1); + + append_dataset::(&mut dataset, APPEND_ROWS, 0.0..0.05).await; + dataset + .optimize_indices(&OptimizeOptions::new()) + .await + .unwrap(); + + let expected_rows = NUM_ROWS + APPEND_ROWS; + let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + assert_eq!( + final_ctx.num_partitions(), + 2, + "Expected one oversized multivector partition to split, stats: {}", + final_ctx.stats_json() + ); + let partitions = final_ctx.stats()["indices"][0]["partitions"] + .as_array() + .expect("partitions should be present"); + assert_eq!(partitions.len(), 2); + assert_eq!( + partitions + .iter() + .map(|partition| partition["size"].as_u64().unwrap() as usize) + .sum::(), + expected_rows * VECTORS_PER_ROW + ); + assert_eq!(dataset.count_all_rows().await.unwrap(), expected_rows); - // Create an IVF-PQ index with 2 partitions - // For IvfPq, target_partition_size = 8192 - // Split triggers when partition_size > 4 * 8192 = 32,768 - let params = VectorIndexParams::ivf_pq(2, 8, DIM / 8, DistanceType::Cosine, 50); - verify_partition_split_after_append(dataset, test_uri, params, "multivector data").await; + let query_batch = dataset + .scan() + .limit(Some(1), None) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let query = query_batch["vector"].as_list::().value(0); + let results = dataset + .scan() + .with_row_id() + .nearest("vector", &query, 10) + .unwrap() + .distance_metric(DistanceType::Cosine) + .try_into_batch() + .await + .unwrap(); + let mut row_ids = HashSet::new(); + for row_id in results[ROW_ID].as_primitive::().values() { + assert!(row_ids.insert(*row_id), "duplicate row id {row_id}"); + } } #[tokio::test] async fn test_split_multiple_partitions_in_one_optimize() { const INDEX_NAME: &str = "vector_idx"; const BASE_ROWS_PER_PARTITION: usize = 512; - const APPEND_ROWS_PER_PARTITION: usize = 40_000; + // Each IVF-FLAT partition reaches 16,512 rows, just over its 16,384-row + // split threshold. + const APPEND_ROWS_PER_PARTITION: usize = 16_000; let offsets = [-50.0, 50.0]; let test_dir = TempStrDir::default(); @@ -5748,11 +6873,7 @@ mod tests { let centroids = build_centroids_for_offsets(&offsets); let ivf_params = IvfBuildParams::try_with_centroids(2, centroids).unwrap(); - let params = VectorIndexParams::with_ivf_pq_params( - DistanceType::L2, - ivf_params, - PQBuildParams::default(), - ); + let params = VectorIndexParams::with_ivf_flat_params(DistanceType::L2, ivf_params); dataset .create_index( &["vector"], @@ -5766,22 +6887,14 @@ mod tests { let initial_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; assert_eq!(initial_ctx.num_partitions(), 2); - let mut templates = Vec::with_capacity(2); - for partition_idx in 0..2 { - let row_ids = load_partition_row_ids(initial_ctx.ivf(), partition_idx).await; - let template_batch = dataset - .take_rows(&[row_ids[0]], dataset.schema().clone()) - .await - .unwrap(); - templates.push( - template_batch["vector"] - .as_fixed_size_list() - .value(0) - .as_primitive::() - .values() - .to_vec(), - ); - } + let templates = offsets + .iter() + .map(|offset| { + let mut template = vec![0.0; DIM]; + template[0] = *offset; + template + }) + .collect::>(); append_partition_templates(&mut dataset, APPEND_ROWS_PER_PARTITION, &templates).await; @@ -5821,6 +6934,24 @@ mod tests { assert_eq!(total_partition_rows, expected_rows); assert_eq!(dataset.count_all_rows().await.unwrap(), expected_rows); + let mut indexed_row_ids = HashSet::with_capacity(expected_rows); + for partition_idx in 0..final_ctx.num_partitions() { + for row_id in load_flat_partition_row_ids(final_ctx.ivf_flat(), partition_idx).await { + assert!( + indexed_row_ids.insert(row_id), + "row id {row_id} appeared in multiple partitions" + ); + } + } + assert_eq!(indexed_row_ids.len(), expected_rows); + let live_row_ids = dataset.scan().with_row_id().try_into_batch().await.unwrap()[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + assert_eq!(indexed_row_ids, live_row_ids); + let nearest = dataset .scan() .with_row_id() @@ -5838,22 +6969,20 @@ mod tests { #[tokio::test] async fn test_join_partition_on_delete_multivec() { - // This test verifies that IVF index with multivector data handles deletions - // and compaction correctly, and that partition join works when applicable. - // - // Due to the complexity of multivector partition assignment, we use a more - // flexible verification approach that doesn't require specific partition sizes. - + const INDEX_NAME: &str = "vector_idx"; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); const MULTIVEC_PER_ROW: usize = 3; - let cluster_sizes = [4000, 4000, 400]; - let offsets: Vec = vec![0.0, 10.0, 20.0]; - let nlist = offsets.len(); + const APPEND_ROWS: usize = 32; + let cluster_sizes = [800, 800, 400]; + // Multivector indices require cosine distance. Unit centroids in three + // distinct directions avoid the collinear assignment in the old fixture. + let centroids = [(-1.0, 0.0), (0.0, 1.0), (1.0, 0.0)]; + let total_rows = cluster_sizes.iter().sum::(); let mut dataset = { let (batch, schema) = - generate_clustered_multivec_batch(&cluster_sizes, &offsets, MULTIVEC_PER_ROW); + generate_clustered_multivec_batch(&cluster_sizes, ¢roids, MULTIVEC_PER_ROW, 0); let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); Dataset::write( batches, @@ -5867,41 +6996,36 @@ mod tests { .unwrap() }; - const SMALL_APPEND_FOR_JOIN: usize = 32; - let centroids = build_centroids_for_offsets(&offsets); - let ivf_params = IvfBuildParams::try_with_centroids(nlist, centroids).unwrap(); + let ivf_params = + IvfBuildParams::try_with_centroids(centroids.len(), build_centroids_2d(¢roids)) + .unwrap(); let params = VectorIndexParams::with_ivf_pq_params( DistanceType::Cosine, ivf_params, - PQBuildParams::default(), + lightweight_pq_params(), ); dataset .create_index( &["vector"], IndexType::Vector, - Some("vector_idx".to_string()), + Some(INDEX_NAME.to_string()), ¶ms, true, ) .await .unwrap(); - // Verify initial partition count and record it for later comparison. - let index_ctx = load_vector_index_context(&dataset, "vector", "vector_idx").await; - let initial_partitions = index_ctx.num_partitions(); - assert!( - initial_partitions <= nlist && initial_partitions > 1, - "Expected at most {} partitions, got {}", - nlist, - initial_partitions - ); + let index_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + assert_eq!(index_ctx.num_partitions(), 3); - // Find the smallest partition and delete most of its rows - let row_ids = { + let mut logical_row_ids = { let ivf = index_ctx.ivf(); - let mut smallest: Option> = None; + let mut smallest: Option> = None; for i in 0..ivf.ivf.num_partitions() { - let partition_row_ids = load_partition_row_ids(ivf, i).await; + let partition_row_ids = load_partition_row_ids(ivf, i) + .await + .into_iter() + .collect::>(); if partition_row_ids.is_empty() { continue; } @@ -5914,259 +7038,289 @@ mod tests { smallest = Some(partition_row_ids); } } - smallest.unwrap_or_default() + smallest + .expect("expected a non-empty partition") + .into_iter() + .collect::>() }; - - if row_ids.is_empty() { - // All partitions might be large - just verify basic functionality - let (batch, _) = generate_batch::(1, None, 0.0..1.0, true); - let test_vector = batch["vector"].as_list::().value(0); - let result = dataset - .scan() - .nearest("vector", &test_vector, 5) - .unwrap() - .try_into_batch() - .await - .unwrap(); - assert!(result.num_rows() > 0, "Multivector search should work"); - return; - } - - // Keep only a few rows to make partition small - let keep_count = 5.min(row_ids.len()); - let retained_ids: Vec = row_ids.iter().take(keep_count).copied().collect(); - - // Delete all rows except the first keep_count rows - delete_ids(&mut dataset, &row_ids[keep_count..]).await; - - // Compact to potentially trigger partition join + logical_row_ids.sort_unstable(); + assert_eq!(logical_row_ids.len(), cluster_sizes[2]); + let retained_id = logical_row_ids[0]; + delete_ids(&mut dataset, &logical_row_ids[1..]).await; compact_after_deletions(&mut dataset).await; - // Append a tiny batch and optimize incrementally to trigger the join path. - append_dataset::(&mut dataset, SMALL_APPEND_FOR_JOIN, 0.0..0.01).await; + let (append_batch, append_schema) = generate_clustered_multivec_batch( + &[APPEND_ROWS], + ¢roids[2..], + MULTIVEC_PER_ROW, + total_rows as u64, + ); dataset - .optimize_indices(&OptimizeOptions::new()) + .append( + RecordBatchIterator::new(vec![Ok(append_batch)], append_schema), + None, + ) .await .unwrap(); dataset - // A second pass ensures the incremental index sees the reduced - // partition sizes and applies the join. .optimize_indices(&OptimizeOptions::new()) .await .unwrap(); - // Verify partition count decreased after join - let final_ctx = load_vector_index_context(&dataset, "vector", "vector_idx").await; - let final_num_partitions = final_ctx.num_partitions(); - assert_le!( - final_num_partitions, - initial_partitions, - "Partition count should drop after join, was {}, now {}", - initial_partitions, - final_num_partitions + let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + assert_eq!( + final_ctx.num_partitions(), + 2, + "Expected the reduced multivector partition to join, stats: {}", + final_ctx.stats_json() ); + assert_eq!(final_ctx.stats()["num_indices"].as_u64().unwrap(), 1); + let expected_rows = total_rows - cluster_sizes[2] + 1 + APPEND_ROWS; + assert_eq!(dataset.count_all_rows().await.unwrap(), expected_rows); - // Verify that multivector search still works after compaction - // Get a sample row by scanning and filtering - let sample_id = retained_ids[0]; let sample_row = dataset .scan() - .filter(&format!("id = {}", sample_id)) + .with_row_id() + .filter(&format!("id = {retained_id}")) .unwrap() .try_into_batch() .await .unwrap(); - - if sample_row.num_rows() > 0 { - let test_vector = sample_row["vector"].as_list::().value(0); - let result = dataset - .scan() - .nearest("vector", &test_vector, 10) - .unwrap() - .try_into_batch() - .await - .unwrap(); - assert!( - result.num_rows() > 0, - "Multivector search should return results after compaction" - ); + assert_eq!(sample_row.num_rows(), 1); + let retained_row_id = sample_row[ROW_ID].as_primitive::().value(0); + let mut indexed_row_id_counts = HashMap::new(); + for partition_idx in 0..final_ctx.num_partitions() { + for row_id in load_partition_row_ids(final_ctx.ivf(), partition_idx).await { + *indexed_row_id_counts.entry(row_id).or_insert(0usize) += 1; + } } - - // Verify the dataset still has rows after deletions and compaction - let remaining_rows = dataset.count_all_rows().await.unwrap(); + assert_eq!( + indexed_row_id_counts.values().sum::(), + expected_rows * MULTIVEC_PER_ROW + ); + assert_eq!( + indexed_row_id_counts.get(&retained_row_id), + Some(&MULTIVEC_PER_ROW), + "all vectors for the retained logical row should survive the join" + ); assert!( - remaining_rows > 0, - "Dataset should still have rows after deletions and compaction" + indexed_row_id_counts + .values() + .all(|count| *count == MULTIVEC_PER_ROW), + "each logical row should have exactly {MULTIVEC_PER_ROW} indexed vectors" ); + let live_row_ids = dataset.scan().with_row_id().try_into_batch().await.unwrap()[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + assert_eq!(live_row_ids.len(), expected_rows); + assert_eq!( + indexed_row_id_counts + .keys() + .copied() + .collect::>(), + live_row_ids + ); + } - // Verify we can perform multivector search on remaining data - let sample_batch = dataset - .scan() - .limit(Some(1), None) - .unwrap() - .try_into_batch() - .await - .unwrap(); - - if sample_batch.num_rows() > 0 { - let test_vector = sample_batch["vector"].as_list::().value(0); - let search_result = dataset - .scan() - .nearest("vector", &test_vector, 10) - .unwrap() - .try_into_batch() - .await - .unwrap(); - assert!( - search_result.num_rows() > 0, - "Multivector search should return results with remaining data" - ); - } + async fn row_ids_matching(dataset: &Dataset, predicate: &str) -> HashSet { + let mut scan = dataset.scan(); + scan.with_row_id(); + scan.filter(predicate).unwrap(); + let batch = scan.try_into_batch().await.unwrap(); + batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect() } - #[tokio::test] - async fn test_prewarm_ivf_pq() { - use lance_io::assert_io_eq; + struct OptimizeAfterDelete { + deleted_row_ids: HashSet, + index_row_ids: HashSet, + num_partitions_after: usize, + stats_json: String, + } + /// Shared scenario for the issue-7701 regressions: stable-row-id dataset, + /// IVF_FLAT index, scattered delete, optimize. Asserts the invariants both + /// partition adjustments must hold -- no live row lost, no id that never + /// existed -- and returns the state for the mode-specific assertions. + async fn optimize_after_delete( + total_rows: usize, + nlist: usize, + delete_predicate: &str, + keep_predicate: &str, + ) -> OptimizeAfterDelete { + const INDEX_NAME: &str = "vector_idx"; let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); - let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; - let params = VectorIndexParams::with_ivf_pq_params( - DistanceType::L2, - IvfBuildParams::new(4), - PQBuildParams::default(), - ); + let (batch, schema) = generate_batch::(total_rows, None, 0.0..1.0, false); + let batches = RecordBatchIterator::new(vec![batch].into_iter().map(Ok), schema); + let mut dataset = Dataset::write( + batches, + test_uri, + Some(WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + ) + .await + .unwrap(); + + let params = VectorIndexParams::ivf_flat(nlist, DistanceType::L2); dataset .create_index( &["vector"], IndexType::Vector, - Some("my_idx".to_owned()), + Some(INDEX_NAME.to_string()), ¶ms, true, ) .await .unwrap(); - // Reset IO stats after index creation - dataset.object_store.as_ref().io_stats_incremental(); - - // Prewarm should perform IO to load all partitions into cache - dataset.prewarm_index("my_idx").await.unwrap(); - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert!( - stats.read_iops > 0, - "prewarm should have read from disk, but read_iops was 0" - ); + let deleted_row_ids = row_ids_matching(&dataset, delete_predicate).await; + let live_row_ids = row_ids_matching(&dataset, keep_predicate).await; + dataset.delete(delete_predicate).await.unwrap(); - // Can query index without IO - let q = Float32Array::from_iter_values(repeat_n(0.0, DIM)); dataset - .scan() - .nearest("vector", &q, 10) - .unwrap() - .project(&["_rowid"]) - .unwrap() - .try_into_batch() + .optimize_indices(&OptimizeOptions::new()) .await .unwrap(); - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_eq!( - stats, - read_iops, - 0, - "query should not perform IO after prewarm" + + let final_ctx = load_vector_index_context(&dataset, "vector", INDEX_NAME).await; + let num_partitions_after = final_ctx.num_partitions(); + let stats_json = final_ctx.stats_json().to_string(); + let flat = final_ctx + .index + .as_any() + .downcast_ref::() + .expect("expected IvfFlat index"); + let mut index_row_ids = HashSet::new(); + for part in 0..flat.ivf.num_partitions() { + index_row_ids.extend(load_flat_partition_row_ids(flat, part).await); + } + + for row_id in &live_row_ids { + assert!( + index_row_ids.contains(row_id), + "live row id {} missing from index after optimize", + row_id + ); + } + for row_id in &index_row_ids { + assert!( + live_row_ids.contains(row_id) || deleted_row_ids.contains(row_id), + "unexpected row id {} in index after optimize", + row_id + ); + } + + OptimizeAfterDelete { + deleted_row_ids, + index_row_ids, + num_partitions_after, + stats_json, + } + } + + #[tokio::test] + async fn test_optimize_join_after_delete_with_stable_row_ids() { + // Regression test for https://github.com/lance-format/lance/issues/7701: + // every partition (400 rows / 4) is under the IVF_FLAT join threshold, + // so optimize joins the smallest after a scattered delete. + let run = optimize_after_delete(400, 4, "id % 3 = 0", "id % 3 != 0").await; + + assert_eq!( + run.num_partitions_after, 3, + "optimize should have joined the smallest partition, got stats: {}", + run.stats_json ); - // Second prewarm should not need IO (already cached) - dataset.prewarm_index("my_idx").await.unwrap(); - let stats = dataset.object_store.as_ref().io_stats_incremental(); - assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO"); + // The join reads every partition's stored rows through the merge filter, so + // deleted ids are dropped index-wide and not just from the joined partition. + for row_id in &run.deleted_row_ids { + assert!( + !run.index_row_ids.contains(row_id), + "deleted row id {} still in index after join", + row_id + ); + } } #[tokio::test] - async fn test_prewarm_ivf_pq_multiple_deltas() { + async fn test_optimize_split_after_delete_with_stable_row_ids() { + // Regression test for https://github.com/lance-format/lance/issues/7701: + // one partition holds more than 4x the IVF_FLAT target, so optimize + // splits it after a scattered delete. This path reaches + // filter_deleted_ids through reshuffle_partitions, unlike the join + // path's take_vectors. + let run = optimize_after_delete(20_000, 1, "id % 5 = 0", "id % 5 != 0").await; + + assert!( + run.num_partitions_after > 1, + "optimize should have split the oversized partition, got stats: {}", + run.stats_json + ); + + // The split rebuilds the whole partition from live rows: no deleted + // ids remain. + for row_id in &run.deleted_row_ids { + assert!( + !run.index_row_ids.contains(row_id), + "deleted row id {} still in index after split", + row_id + ); + } + } + + #[tokio::test] + async fn test_prewarm_ivf_pq() { use lance_io::assert_io_eq; const INDEX_NAME: &str = "my_idx"; - const BASE_ROWS_PER_PARTITION: usize = 3_000; - const SMALL_APPEND_ROWS: usize = 64; - let offsets = [-50.0, 50.0]; - let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); + let (mut dataset, vectors) = generate_test_dataset::(test_uri, 0.0..1.0).await; - let (batch, schema) = generate_clustered_batch(BASE_ROWS_PER_PARTITION, offsets); - let batches = RecordBatchIterator::new(vec![Ok(batch)], schema.clone()); - let mut dataset = Dataset::write( - batches, - test_uri, - Some(WriteParams { - mode: WriteMode::Overwrite, - ..Default::default() - }), - ) - .await - .unwrap(); - - let centroids = build_centroids_for_offsets(&offsets); - let ivf_params = IvfBuildParams::try_with_centroids(2, centroids).unwrap(); let params = VectorIndexParams::with_ivf_pq_params( DistanceType::L2, - ivf_params, - PQBuildParams::default(), + IvfBuildParams::new(4), + PQBuildParams::new(4, 4), ); dataset .create_index( &["vector"], IndexType::Vector, - Some(INDEX_NAME.to_string()), + Some(INDEX_NAME.to_owned()), ¶ms, true, ) .await .unwrap(); - let template_batch = dataset - .take_rows(&[0], dataset.schema().clone()) - .await - .unwrap(); - let template_values = template_batch["vector"] - .as_fixed_size_list() - .value(0) - .as_primitive::() - .values() - .to_vec(); - let mut append_params = WriteParams { - max_rows_per_file: 32, - max_rows_per_group: 32, - ..Default::default() - }; - append_params.mode = WriteMode::Append; - append_constant_vector_with_params( - &mut dataset, - SMALL_APPEND_ROWS, - &template_values, - Some(append_params), - ) - .await; - + append_dataset::(&mut dataset, 8, 0.0..1.0).await; dataset - .optimize_indices(&OptimizeOptions::new()) + .optimize_indices(&OptimizeOptions::append()) .await .unwrap(); - // Reopen dataset to avoid carrying index state in-memory from index creation. + // Reopen to avoid carrying index state in memory from index creation. let dataset = Dataset::open(test_uri).await.unwrap(); let indices = dataset.load_indices_by_name(INDEX_NAME).await.unwrap(); - assert_eq!(indices.len(), 2, "expected two index deltas for my_idx"); + assert_eq!(indices.len(), 2, "expected two index deltas"); let unique_uuids: HashSet<_> = indices.iter().map(|meta| meta.uuid).collect(); assert_eq!(unique_uuids.len(), 2, "expected two unique index UUIDs"); - // Reset IO stats after index creation + // Reset IO stats after index creation. dataset.object_store.as_ref().io_stats_incremental(); - // Prewarm should perform IO to load all index deltas into cache + // Prewarm should perform IO to load all index deltas into cache. dataset.prewarm_index(INDEX_NAME).await.unwrap(); let stats = dataset.object_store.as_ref().io_stats_incremental(); assert!( @@ -6174,11 +7328,11 @@ mod tests { "prewarm should have read from disk, but read_iops was 0" ); - // Query should not perform IO after prewarm of all deltas - let q = Float32Array::from(template_values.clone()); + // Query should not perform IO after prewarming all deltas. + let q = vectors.value(0); dataset .scan() - .nearest("vector", &q, 10) + .nearest("vector", q.as_primitive::(), 10) .unwrap() .project(&["_rowid"]) .unwrap() @@ -6193,59 +7347,126 @@ mod tests { "query should not perform IO after prewarm" ); - // Second prewarm should not need IO (already cached) + // Second prewarm should not need IO (already cached). dataset.prewarm_index(INDEX_NAME).await.unwrap(); let stats = dataset.object_store.as_ref().io_stats_incremental(); assert_io_eq!(stats, read_iops, 0, "second prewarm should not perform IO"); } - type SerializedEntry = (Vec, lance_core::cache::CacheCodec, usize); - + /// Index-cache backend that can drop partition entries on demand. + /// + /// Used to simulate cache invalidation after a credential rotation: + /// partition keys are opaque digests, so the test supplies the exact + /// [`InternalCacheKey`] set to bypass once the index identity is known + /// (see [`ivf_partition_cache_keys`]). #[derive(Debug)] - struct SerializingBackend { - /// Serialized entries: key -> (bytes, codec, size). - serialized: tokio::sync::Mutex< - std::collections::HashMap, - >, - /// Fallback for entries without a codec. - passthrough: lance_core::cache::MokaCacheBackend, + struct PartitionBypassCacheBackend { + inner: lance_core::cache::MokaCacheBackend, + partition_keys: std::sync::Mutex>, + bypass_partitions: AtomicBool, + partition_hits: AtomicUsize, } - impl SerializingBackend { + impl PartitionBypassCacheBackend { fn new() -> Self { Self { - serialized: tokio::sync::Mutex::new(std::collections::HashMap::new()), - passthrough: lance_core::cache::MokaCacheBackend::with_capacity(256 * 1024 * 1024), + inner: lance_core::cache::MokaCacheBackend::with_capacity(256 * 1024 * 1024), + partition_keys: std::sync::Mutex::new(HashSet::new()), + bypass_partitions: AtomicBool::new(false), + partition_hits: AtomicUsize::new(0), } } - async fn serialized_entry_count(&self) -> usize { - self.serialized.lock().await.len() + fn set_partition_keys(&self, partition_keys: HashSet) { + *self.partition_keys.lock().unwrap() = partition_keys; + } + + fn is_partition(&self, key: &lance_core::cache::InternalCacheKey) -> bool { + self.partition_keys.lock().unwrap().contains(key) } - async fn passthrough_entry_count(&self) -> usize { - use lance_core::cache::CacheBackend; - self.passthrough.num_entries().await + fn set_bypass_partitions(&self, bypass_partitions: bool) { + self.bypass_partitions + .store(bypass_partitions, Ordering::Relaxed); + } + + fn should_bypass(&self, key: &lance_core::cache::InternalCacheKey) -> bool { + self.bypass_partitions.load(Ordering::Relaxed) && self.is_partition(key) + } + + /// Whether the backend currently holds an entry for `key`. + async fn contains(&self, key: &lance_core::cache::InternalCacheKey) -> bool { + self.inner.get(key, None).await.is_some() + } + + fn partition_hits(&self) -> usize { + self.partition_hits.load(Ordering::Relaxed) } } + /// Derive the internal cache keys of the IVF partition entries for an + /// index, replicating the namespace path + /// `dataset URI -> index UUID -> frag-reuse UUID` used when opening the + /// index. V3 partitions use [`IVFPartitionKey`]; legacy (v0.1/v0.2) + /// indices use `LegacyIVFPartitionKey`. + fn ivf_partition_cache_keys( + dataset_uri: &str, + uuid: &uuid::Uuid, + fri_uuid: Option<&uuid::Uuid>, + num_partitions: usize, + index_version: &IndexFileVersion, + ) -> HashSet { + use lance_core::cache::{CacheKey, CacheNamespace, KeyBuilder, UnsizedCacheKey}; + + let mut namespace = CacheNamespace::root().child(dataset_uri); + namespace = namespace.child(uuid.as_hyphenated().to_string().as_str()); + if let Some(fri_uuid) = fri_uuid { + namespace = namespace.child(fri_uuid.as_hyphenated().to_string().as_str()); + } + + (0..num_partitions) + .map(|partition_id| { + if matches!(index_version, IndexFileVersion::V3) { + let cache_key = + IVFPartitionKey::::new(partition_id); + let mut builder = KeyBuilder::new( + namespace, + IVFPartitionKey::::stable_type_id(), + IVFPartitionKey::::schema(), + ); + cache_key.write_key(&mut builder); + builder.finish() + } else { + let cache_key = + crate::index::vector::ivf::LegacyIVFPartitionKey::new(partition_id); + let mut builder = KeyBuilder::new( + namespace, + crate::index::vector::ivf::LegacyIVFPartitionKey::stable_type_id(), + crate::index::vector::ivf::LegacyIVFPartitionKey::schema(), + ); + cache_key.write_key(&mut builder); + builder.finish() + } + }) + .collect() + } + #[async_trait::async_trait] - impl lance_core::cache::CacheBackend for SerializingBackend { + impl lance_core::cache::CacheBackend for PartitionBypassCacheBackend { async fn get( &self, key: &lance_core::cache::InternalCacheKey, codec: Option, ) -> Option { - // Try serialized store first - let guard = self.serialized.lock().await; - if let Some((bytes, stored_codec, _)) = guard.get(key) { - return stored_codec - .deserialize(&bytes::Bytes::copy_from_slice(bytes)) - .hit(); + if self.should_bypass(key) { + None + } else { + let entry = self.inner.get(key, codec).await; + if entry.is_some() && self.is_partition(key) { + self.partition_hits.fetch_add(1, Ordering::Relaxed); + } + entry } - drop(guard); - // Fall through to passthrough - self.passthrough.get(key, codec).await } async fn insert( @@ -6255,17 +7476,8 @@ mod tests { size_bytes: usize, codec: Option, ) { - if let Some(codec) = codec { - let mut bytes = Vec::new(); - codec - .serialize(&entry, &mut bytes) - .expect("serialization should succeed"); - self.serialized - .lock() - .await - .insert(key.clone(), (bytes, codec, size_bytes)); - } else { - self.passthrough.insert(key, entry, size_bytes, None).await; + if !self.should_bypass(key) { + self.inner.insert(key, entry, size_bytes, codec).await; } } @@ -6281,40 +7493,38 @@ mod tests { >, codec: Option, ) -> Result<(lance_core::cache::CacheEntry, bool)> { - if let Some(entry) = self.get(key, codec).await { - return Ok((entry, true)); + if self.should_bypass(key) { + let (entry, _) = loader.await?; + Ok((entry, false)) + } else { + let result = self.inner.get_or_insert(key, loader, codec).await; + if result.as_ref().is_ok_and(|(_, is_cache_hit)| *is_cache_hit) + && self.is_partition(key) + { + self.partition_hits.fetch_add(1, Ordering::Relaxed); + } + result } - let (entry, size) = loader.await?; - self.insert(key, entry.clone(), size, codec).await; - Ok((entry, false)) - } - - async fn invalidate_prefix(&self, prefix: &str) { - self.serialized - .lock() - .await - .retain(|k, _| !k.starts_with(prefix)); - self.passthrough.invalidate_prefix(prefix).await; } async fn clear(&self) { - self.serialized.lock().await.clear(); - self.passthrough.clear().await; + self.inner.clear().await; } async fn num_entries(&self) -> usize { - self.serialized.lock().await.len() + self.passthrough.num_entries().await + self.inner.num_entries().await } async fn size_bytes(&self) -> usize { - let serialized: usize = self - .serialized - .lock() - .await - .values() - .map(|(_, _, s)| *s) - .sum(); - serialized + self.passthrough.size_bytes().await + self.inner.size_bytes().await + } + + fn approx_num_entries(&self) -> usize { + self.inner.approx_num_entries() + } + + fn approx_size_bytes(&self) -> usize { + self.inner.approx_size_bytes() } } @@ -6322,18 +7532,37 @@ mod tests { /// serializing cache backend, then query. Verifies that entries are /// serialized to bytes and that queries produce correct results after /// deserialization. + #[rstest] + #[case::ivf_pq( + VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::new(4), + PQBuildParams::default(), + ), + as CacheCodecImpl>::TYPE_ID + )] + #[case::ivf_hnsw_sq( + VectorIndexParams::with_ivf_hnsw_sq_params( + DistanceType::L2, + IvfBuildParams::new(4), + HnswBuildParams::default(), + SQBuildParams::default(), + ), + as CacheCodecImpl>::TYPE_ID + )] #[tokio::test] - async fn test_prewarm_and_query_with_serializing_backend() { + async fn test_prewarm_and_query_with_serializing_backend( + #[case] params: VectorIndexParams, + #[case] partition_type_id: &'static str, + ) { + use crate::utils::test::serializing_cache::SerializingCacheBackend; + use lance_io::assert_io_eq; + let test_dir = TempStrDir::default(); let test_uri = test_dir.as_str(); // Create dataset with vector index using default cache let (mut dataset, _) = generate_test_dataset::(test_uri, 0.0..1.0).await; - let params = VectorIndexParams::with_ivf_pq_params( - DistanceType::L2, - IvfBuildParams::new(4), - PQBuildParams::default(), - ); dataset .create_index( &["vector"], @@ -6345,8 +7574,11 @@ mod tests { .await .unwrap(); + let q = Float32Array::from_iter_values(repeat_n(0.5, DIM)); + let expected = ground_truth(&dataset, "vector", &q, 10, DistanceType::L2).await; + // Re-open with the serializing backend - let backend = Arc::new(SerializingBackend::new()); + let backend = Arc::new(SerializingCacheBackend::new()); let session = Arc::new(crate::session::Session::with_index_cache_backend( backend.clone(), 128 * 1024 * 1024, @@ -6361,7 +7593,10 @@ mod tests { // Prewarm — this should serialize entries into the backend dataset.prewarm_index("serde_idx").await.unwrap(); let serialized = backend.serialized_entry_count().await; - let passthrough = backend.passthrough_entry_count().await; + let state_type_id = IvfStateEntryBox::TYPE_ID; + let state_inserts = backend.serialized_insert_count(state_type_id).await; + let partition_inserts = backend.serialized_insert_count(partition_type_id).await; + let passthrough = backend.l1_entry_count().await; assert!( serialized > 0, "prewarm should have serialized entries into the backend" @@ -6372,18 +7607,53 @@ mod tests { but found {passthrough} passthrough entries" ); - // Query — the backend will deserialize entries from bytes. - // After prewarm, all entries are in serialized form, so every - // cache hit involves a deserialization round-trip. - let q = Float32Array::from_iter_values(repeat_n(0.5, DIM)); + drop(dataset); + let backend = Arc::new(backend.restart()); + assert_eq!( + backend.l1_entry_count().await, + 0, + "restarting must discard the in-memory L1" + ); + assert_eq!( + backend.serialized_entry_count().await, + serialized, + "restarting must retain the serialized IVF state and partitions" + ); + let session = Arc::new(crate::session::Session::with_index_cache_backend( + backend.clone(), + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = crate::DatasetBuilder::from_uri(test_uri) + .with_session(session) + .load() + .await + .unwrap(); + + // Query — the recreated backend will deserialize entries from bytes. + // All index entries are in serialized form, so every cache hit involves + // a deserialization round-trip. let results = dataset .scan() + .with_row_id() .nearest("vector", &q, 10) .unwrap() .nprobes(4) + .project(&["_rowid"]) + .unwrap() .try_into_batch() .await .unwrap(); + assert_eq!( + backend.serialized_insert_count(state_type_id).await, + state_inserts, + "the first restarted query must reuse the serialized IVF state" + ); + assert_eq!( + backend.serialized_insert_count(partition_type_id).await, + partition_inserts, + "the first restarted query must reuse every serialized IVF partition" + ); assert_eq!(results.num_rows(), 10, "should return 10 nearest neighbors"); // Verify distances are sorted (ascending for L2) @@ -6396,5 +7666,391 @@ mod tests { for w in distances.windows(2) { assert!(w[1] >= w[0], "distances should be sorted ascending"); } + + let row_ids = results[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let recall = row_ids.intersection(&expected).count() as f32 / expected.len() as f32; + assert_ge!( + recall, + 0.5, + "serialized IVF query recall is below threshold: {recall}" + ); + + dataset.object_store.as_ref().io_stats_incremental(); + dataset + .scan() + .nearest("vector", &q, 10) + .unwrap() + .nprobes(4) + .project(&["_rowid"]) + .unwrap() + .try_into_batch() + .await + .unwrap(); + let stats = dataset.object_store.as_ref().io_stats_incremental(); + assert_io_eq!( + stats, + read_iops, + 0, + "warmed IVF query should not perform IO after backend restart" + ); + } + + #[rstest] + #[case::v3(IndexFileVersion::V3)] + #[case::legacy(IndexFileVersion::Legacy)] + #[tokio::test] + async fn test_vector_cache_uses_current_object_store(#[case] index_version: IndexFileVersion) { + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + let (mut dataset, vectors) = generate_test_dataset::(test_uri, 0.0..1.0).await; + append_dataset::(&mut dataset, NUM_ROWS, 0.0..1.0).await; + assert_eq!(dataset.get_fragments().len(), 2); + + let params = VectorIndexParams::with_ivf_pq_params( + DistanceType::L2, + IvfBuildParams::new(4), + PQBuildParams::default(), + ) + .version(index_version.clone()) + .clone(); + dataset + .create_index( + &["vector"], + IndexType::Vector, + Some("credential_rotation_idx".to_owned()), + ¶ms, + true, + ) + .await + .unwrap(); + let index_meta = dataset + .load_indices_by_name("credential_rotation_idx") + .await + .unwrap() + .pop() + .unwrap(); + let query = vectors.value(0); + let ground_truth = ground_truth(&dataset, "vector", &query, 20, DistanceType::L2).await; + + let cache_backend = Arc::new(PartitionBypassCacheBackend::new()); + let session = Arc::new(crate::session::Session::with_index_cache_backend( + cache_backend.clone(), + 128 * 1024 * 1024, + Arc::new(lance_io::object_store::ObjectStoreRegistry::default()), + )); + let dataset = crate::DatasetBuilder::from_uri(test_uri) + .with_session(session) + .load() + .await + .unwrap(); + + let store_params_a = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + "credential_generation".to_owned(), + "secret-generation-a".to_owned(), + )]), + ))), + ..Default::default() + }; + let (store_a, _) = ObjectStore::from_uri_and_params( + dataset.session().store_registry(), + dataset.uri(), + &store_params_a, + ) + .await + .unwrap(); + let dataset_a = dataset.with_object_store(store_a.clone(), Some(store_params_a)); + + let store_params_b = ObjectStoreParams { + storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options( + HashMap::from([( + "credential_generation".to_owned(), + "secret-generation-b".to_owned(), + )]), + ))), + ..Default::default() + }; + let (store_b, _) = ObjectStore::from_uri_and_params( + dataset.session().store_registry(), + dataset.uri(), + &store_params_b, + ) + .await + .unwrap(); + assert!(!Arc::ptr_eq(&store_a, &store_b)); + let dataset_b = dataset.with_object_store(store_b.clone(), Some(store_params_b)); + + let _ = store_a.io_stats_incremental(); + let _ = store_b.io_stats_incremental(); + + dataset_a + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + + let frag_reuse_uuid = dataset_a.frag_reuse_index_uuid().await; + let state_cache_key = + crate::index::IvfIndexStateCacheKey::new(&index_meta.uuid, frag_reuse_uuid.as_ref()); + let cached_state = if matches!(index_version, IndexFileVersion::V3) { + Some( + dataset_a + .index_cache + .get_with_key(&state_cache_key) + .await + .expect("V3 IVF state should be cached"), + ) + } else { + None + }; + let index_path_fragment = format!("_indices/{}", index_meta.uuid); + let first_store_stats = store_a.io_stats_incremental(); + assert!( + first_store_stats + .requests + .iter() + .any(|request| request.path.as_ref().contains(&index_path_fragment)), + "the first query should read the index through the first object store: {first_store_stats:#?}" + ); + let partition_keys = ivf_partition_cache_keys( + dataset.uri(), + &index_meta.uuid, + frag_reuse_uuid.as_ref(), + 4, + &index_version, + ); + cache_backend.set_partition_keys(partition_keys.clone()); + for partition_key in &partition_keys { + assert!( + cache_backend.contains(partition_key).await, + "the first query should populate portable partition entries" + ); + } + let index_entries_after_a = dataset.session().index_cache_stats().await.num_entries; + let metadata_entries_after_a = dataset.session().metadata_cache_stats().await.num_entries; + let _ = store_b.io_stats_incremental(); + + cache_backend.set_bypass_partitions(true); + let results = dataset_b + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let row_ids = results[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let recall = row_ids.intersection(&ground_truth).count() as f32 / 20.0; + assert_ge!(recall, 0.5); + + let old_store_stats = store_a.io_stats_incremental(); + let old_store_index_reads = old_store_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + let new_store_stats = store_b.io_stats_incremental(); + let new_store_index_reads = new_store_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + if matches!(index_version, IndexFileVersion::V3) { + assert_eq!( + old_store_index_reads, 0, + "the new dataset query must not use readers bound to the old object store: {old_store_stats:#?}" + ); + assert!( + new_store_index_reads > 0, + "the new dataset query should read the index through the new object store: {new_store_stats:#?}" + ); + } else { + // Legacy live indices are shared across dataset opens: their + // readers stay bound to the object store that first populated the + // cache, so the second dataset keeps reading through the old store. + assert!( + old_store_index_reads > 0, + "the cached legacy index should keep reading through the original object store: {old_store_stats:#?}" + ); + assert_eq!( + new_store_index_reads, 0, + "the cached legacy index must not reopen through the new object store: {new_store_stats:#?}" + ); + } + if let Some(cached_state) = cached_state { + let state_after_rotation = dataset_b + .index_cache + .get_with_key(&state_cache_key) + .await + .expect("V3 IVF state should remain cached after rotation"); + assert!( + Arc::ptr_eq(&cached_state, &state_after_rotation), + "store-free IVF state should be reused across object-store generations" + ); + } + + // Re-query through the first dataset: V3 portable state is rebound to + // the store supplied by each reconstruction, while the cached legacy + // index keeps reading through its original store. Either way the second + // store must not be touched. + let _ = store_a.io_stats_incremental(); + let _ = store_b.io_stats_incremental(); + dataset_a + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let store_a_stats = store_a.io_stats_incremental(); + let store_a_index_reads = store_a_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + let store_b_stats = store_b.io_stats_incremental(); + let store_b_index_reads = store_b_stats + .requests + .iter() + .filter(|request| request.path.as_ref().contains(&index_path_fragment)) + .count(); + assert!( + store_a_index_reads > 0, + "re-querying the first dataset should read the index through its object store: {store_a_stats:#?}" + ); + assert_eq!( + store_b_index_reads, 0, + "re-querying the first dataset must not use the second object store: {store_b_stats:#?}" + ); + + // Cache keys are opaque digests, so they cannot embed credential + // material by construction. What rotation must not do is mint new + // entries: the same portable state, partitions, and file metadata + // serve both object-store generations. + let index_entries_after_rotation = dataset.session().index_cache_stats().await.num_entries; + let metadata_entries_after_rotation = + dataset.session().metadata_cache_stats().await.num_entries; + assert_eq!( + index_entries_after_rotation, index_entries_after_a, + "credential rotation must not create new index cache entries" + ); + assert_eq!( + metadata_entries_after_rotation, metadata_entries_after_a, + "credential rotation must not create new metadata cache entries" + ); + + cache_backend.set_bypass_partitions(false); + let partition_hits_before = cache_backend.partition_hits(); + dataset_b + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + assert!( + cache_backend.partition_hits() > partition_hits_before, + "the second store should reuse portable partitions populated by the first" + ); + } + + #[tokio::test] + async fn test_shallow_clone_ivf_rq_uses_resolved_index_directory() { + let test_dir = TempStrDir::default(); + let source_uri = format!("{}/source", test_dir.as_str()); + let clone_uri = format!("{}/clone", test_dir.as_str()); + let (mut source, vectors) = + generate_test_dataset::(&source_uri, 0.0..1.0).await; + append_dataset::(&mut source, NUM_ROWS, 0.0..1.0).await; + assert_eq!(source.get_fragments().len(), 2); + + let params = VectorIndexParams::ivf_rq(4, 5, DistanceType::L2); + source + .create_index( + &["vector"], + IndexType::Vector, + Some("ivf_rq_idx".to_owned()), + ¶ms, + true, + ) + .await + .unwrap(); + + let query = vectors.value(0); + let ground_truth = ground_truth(&source, "vector", &query, 20, DistanceType::L2).await; + source + .tags() + .create("with_ivf_rq", source.version().version) + .await + .unwrap(); + let cloned = source + .shallow_clone(&clone_uri, "with_ivf_rq", None) + .await + .unwrap(); + + let index_meta = cloned + .load_indices_by_name("ivf_rq_idx") + .await + .unwrap() + .pop() + .unwrap(); + assert!( + index_meta.base_id.is_some(), + "a shallow-cloned index should reference its source base" + ); + assert_eq!( + cloned.indice_files_dir(&index_meta).unwrap(), + source.indices_dir(), + "the cloned index should resolve its path through the source base" + ); + assert_ne!( + cloned.indice_files_dir(&index_meta).unwrap(), + cloned.indices_dir(), + "the cloned index should not use the clone's primary index directory" + ); + + let cloned = crate::DatasetBuilder::from_uri(&clone_uri) + .with_session(Arc::new(crate::session::Session::default())) + .load() + .await + .unwrap(); + + let results = cloned + .scan() + .nearest("vector", &query, 20) + .unwrap() + .minimum_nprobes(4) + .with_row_id() + .try_into_batch() + .await + .unwrap(); + let row_ids = results[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>(); + let recall = row_ids.intersection(&ground_truth).count() as f32 / 20.0; + assert_ge!(recall, 0.5); } } diff --git a/rust/lance/src/index/vector/pq.rs b/rust/lance/src/index/vector/pq.rs index 217667afbe9..141c8b85f27 100644 --- a/rust/lance/src/index/vector/pq.rs +++ b/rust/lance/src/index/vector/pq.rs @@ -5,11 +5,15 @@ use lance_core::utils::row_addr_remap::RowAddrRemap; use std::any::Any; use std::sync::Arc; -use arrow::compute::concat; +use arrow::{ + array::{ArrayData, make_array}, + compute::concat, +}; use arrow_array::types::UInt64Type; use arrow_array::{ Array, FixedSizeListArray, RecordBatch, UInt8Array, UInt64Array, cast::{AsArray, as_primitive_array}, + new_empty_array, }; use arrow_array::{ArrayRef, Float32Array, UInt32Array}; use arrow_ord::sort::sort_to_indices; @@ -18,12 +22,12 @@ use arrow_select::take::take; use async_trait::async_trait; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use lance_arrow::FixedSizeListArrayExt; +use lance_arrow::{BufferExt, DataTypeExt, FixedSizeListArrayExt}; use lance_core::deepsize::DeepSizeOf; use lance_core::utils::address::RowAddress; use lance_core::utils::tokio::spawn_cpu; use lance_core::{ROW_ID, ROW_ID_FIELD}; -use lance_index::frag_reuse::FragReuseIndex; +use lance_index::frag_reuse::CompactFragReuseIndex; use lance_index::metrics::MetricsCollector; use lance_index::vector::ivf::storage::IvfModel; use lance_index::vector::pq::storage::{ProductQuantizationStorage, transpose}; @@ -33,7 +37,7 @@ use lance_index::{ Index, IndexType, vector::{Query, pq::ProductQuantizer}, }; -use lance_io::{traits::Reader, utils::read_fixed_stride_array}; +use lance_io::traits::Reader; use lance_linalg::distance::{DistanceType, MetricType}; use log::{info, warn}; use roaring::RoaringBitmap; @@ -68,7 +72,37 @@ pub struct PQIndex { /// Metric type. metric_type: MetricType, - frag_reuse_index: Option>, + frag_reuse_index: Option>, +} + +async fn read_legacy_index_values( + reader: &dyn Reader, + data_type: &DataType, + offset: usize, + length: usize, +) -> Result { + if length == 0 { + return Ok(new_empty_array(data_type)); + } + + let byte_length = length + .checked_mul(data_type.byte_width()) + .ok_or_else(|| Error::index("legacy IVF page byte length overflow".to_string()))?; + let end = offset + .checked_add(byte_length) + .ok_or_else(|| Error::index("legacy IVF page offset overflow".to_string()))?; + let bytes = reader.get_range(offset..end).await?; + let buffer = if bytes.len() < byte_length { + arrow_buffer::Buffer::copy_bytes_bytes(bytes, byte_length) + } else { + arrow_buffer::Buffer::from_bytes_bytes(bytes, data_type.byte_width() as u64) + }; + let data = ArrayData::builder(data_type.clone()) + .len(length) + .null_count(0) + .add_buffer(buffer) + .build()?; + Ok(make_array(data)) } impl DeepSizeOf for PQIndex { @@ -116,7 +150,7 @@ impl PQIndex { pub(crate) fn new( pq: ProductQuantizer, metric_type: MetricType, - frag_reuse_index: Option>, + frag_reuse_index: Option>, ) -> Self { Self { code: None, @@ -331,24 +365,14 @@ impl VectorIndex for PQIndex { length: usize, ) -> Result> { let pq_code_length = self.pq.code_dim() * length; - let pq_codes = read_fixed_stride_array( - reader.as_ref(), - &DataType::UInt8, - offset, - pq_code_length, - .., - ) - .await?; + let pq_codes = + read_legacy_index_values(reader.as_ref(), &DataType::UInt8, offset, pq_code_length) + .await?; let row_id_offset = offset + pq_code_length /* *1 */; - let row_ids = read_fixed_stride_array( - reader.as_ref(), - &DataType::UInt64, - row_id_offset, - length, - .., - ) - .await?; + let row_ids = + read_legacy_index_values(reader.as_ref(), &DataType::UInt64, row_id_offset, length) + .await?; let pq_codes = transpose( pq_codes.as_primitive(), @@ -435,8 +459,11 @@ impl VectorIndex for PQIndex { .map_or(0, |row_ids| row_ids.len() as u64) } - fn row_ids(&self) -> Box> { - todo!("this method is for only IVF_HNSW_* index"); + fn row_ids(&self) -> Box + '_> { + match self.row_ids.as_ref() { + Some(row_ids) => Box::new(row_ids.values().iter()), + None => Box::new(std::iter::empty()), + } } async fn remap(&mut self, mapping: &RowAddrRemap) -> Result<()> { @@ -651,8 +678,12 @@ mod tests { use arrow::datatypes::Float32Type; use arrow_array::RecordBatchIterator; use arrow_schema::{Field, Schema}; - use lance_core::utils::tempfile::TempStrDir; + use lance_core::utils::tempfile::{TempObjFile, TempStrDir}; + use lance_io::object_store::ObjectStore; + use lance_io::traits::Writer; use lance_linalg::kernels::normalize_fsl; + use object_store::path::Path; + use tokio::io::AsyncWriteExt; use crate::index::vector::ivf::build_ivf_model; use lance_index::metrics::NoOpMetricsCollector; @@ -664,6 +695,47 @@ mod tests { }; const DIM: usize = 128; + + #[tokio::test] + async fn empty_legacy_ivf_pq_partition_does_not_read_object_store() { + let object_store = ObjectStore::memory(); + let reader = object_store + .open(&Path::from("missing-index")) + .await + .unwrap(); + let codebook_values = Float32Array::from_iter_values((0..256).map(|value| value as f32)); + let codebook = FixedSizeListArray::try_new_from_values(codebook_values, 1).unwrap(); + let index = PQIndex::new( + ProductQuantizer::new(1, 8, 1, codebook, DistanceType::L2), + MetricType::L2, + None, + ); + + let loaded = index.load(reader.into(), 1, 0).await.unwrap(); + + assert_eq!(loaded.num_rows(), 0); + } + + #[tokio::test] + async fn legacy_index_values_are_read_from_the_requested_offset() { + let path = TempObjFile::default(); + let object_store = ObjectStore::local(); + let mut writer = object_store.create(&path).await.unwrap(); + writer.write_all(&[0xFF]).await.unwrap(); + writer.write_all(&11_u64.to_le_bytes()).await.unwrap(); + writer.write_all(&12_u64.to_le_bytes()).await.unwrap(); + Writer::shutdown(writer.as_mut()).await.unwrap(); + + let reader = object_store.open(&path).await.unwrap(); + let values = read_legacy_index_values(reader.as_ref(), &DataType::UInt64, 1, 2) + .await + .unwrap(); + assert_eq!( + values.as_primitive::(), + &UInt64Array::from_iter_values([11, 12]) + ); + } + async fn generate_dataset( test_uri: &str, range: Range, @@ -701,7 +773,11 @@ mod tests { let centroids = generate_random_array_with_range::(4 * DIM, -1.0..1.0); let fsl = FixedSizeListArray::try_new_from_values(centroids, DIM as i32).unwrap(); let ivf = IvfModel::new(fsl, None); - let params = PQBuildParams::new(16, 8); + let params = PQBuildParams { + max_iters: 2, + sample_rate: 4, + ..PQBuildParams::new(16, 8) + }; let pq = build_pq_model(&dataset, "vector", DIM, MetricType::L2, ¶ms, Some(&ivf)) .await .unwrap(); @@ -741,7 +817,11 @@ mod tests { ) .await .unwrap(); - let params = PQBuildParams::new(16, 8); + let params = PQBuildParams { + max_iters: 2, + sample_rate: 4, + ..PQBuildParams::new(16, 8) + }; let pq = build_pq_model( &dataset, "vector", diff --git a/rust/lance/src/index/vector/utils.rs b/rust/lance/src/index/vector/utils.rs index 3046d0f3a83..012d8d3a703 100644 --- a/rust/lance/src/index/vector/utils.rs +++ b/rust/lance/src/index/vector/utils.rs @@ -1,16 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashSet; use std::pin::Pin; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use arrow::array::ArrayData; use arrow::datatypes::DataType; use arrow_array::new_empty_array; use arrow_array::{Array, ArrayRef, FixedSizeListArray, RecordBatch, cast::AsArray}; use arrow_buffer::{Buffer, MutableBuffer}; -use futures::{Stream, StreamExt, TryStreamExt, stream}; +use futures::{Stream, StreamExt, stream}; use lance_arrow::DataTypeExt; use lance_core::datatypes::Schema; use lance_linalg::distance::DistanceType; @@ -18,6 +18,7 @@ use log::{info, warn}; use rand::rngs::SmallRng; use rand::seq::{IteratorRandom, SliceRandom}; use rand::{Rng, SeedableRng}; +use roaring::RoaringTreemap; use tokio::sync::Mutex; use crate::dataset::{Dataset, ProjectionRequest, TakeBuilder, row_offsets_to_row_addresses}; @@ -264,38 +265,7 @@ fn infer_vector_element_type_impl( async fn count_rows(dataset: &Dataset, fragment_ids: Option<&[u32]>) -> Result { match fragment_ids { None => dataset.count_rows(None).await, - Some(fragment_ids) => { - let sorted_ids: Vec; - let sorted_fragment_ids = if fragment_ids.windows(2).all(|w| w[0] <= w[1]) { - fragment_ids - } else { - sorted_ids = { - let mut v = fragment_ids.to_vec(); - v.sort_unstable(); - v - }; - &sorted_ids - }; - let fragments = dataset.get_frags_from_ordered_ids(sorted_fragment_ids); - let valid_fragments = fragments - .into_iter() - .enumerate() - .map(|(i, frag)| { - frag.ok_or_else(|| { - Error::index(format!( - "Unexpectedly missing fragment {}", - sorted_fragment_ids[i] - )) - }) - }) - .collect::>>()?; - let cnts = stream::iter(valid_fragments) - .map(|f| async move { f.count_rows(None).await }) - .buffer_unordered(16) - .try_collect::>() - .await?; - Ok(cnts.iter().sum::()) - } + Some(fragment_ids) => dataset.count_rows_in_fragments(fragment_ids).await, } } @@ -309,8 +279,6 @@ pub async fn maybe_sample_training_data( sample_size_hint: usize, fragment_ids: Option<&[u32]>, ) -> Result { - let num_rows = count_rows(dataset, fragment_ids).await?; - let vector_field = dataset.schema().field(column).ok_or(Error::index(format!( "Sample training data: column {} does not exist in schema", column @@ -328,6 +296,8 @@ pub async fn maybe_sample_training_data( return Ok(new_empty_array(&fsl_type).as_fixed_size_list().clone()); } + let num_rows = count_rows(dataset, fragment_ids).await?; + let is_nullable = vector_field.nullable; let sample_size_hint = match vector_field.data_type() { @@ -486,18 +456,39 @@ async fn sample_training_data( ); return vector_column_to_fsl(&batch, column); } + // Rows the consumer still needs. The fragment producer sizes each + // prefetch round to this outstanding demand, keeping reads bounded by + // the requested sample size. + let still_needed = Arc::new(AtomicUsize::new(sample_size_hint)); let scan = sample_training_data_scan_from_fragments( dataset, column, - sample_size_hint, num_rows, fragment_ids, + still_needed.clone(), )?; return match vector_field.data_type() { DataType::FixedSizeList(_, _) => { - sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await + sample_nullable_fsl( + column, + sample_size_hint, + byte_width, + vector_field, + scan, + Some(still_needed), + ) + .await + } + _ => { + sample_nullable_fallback( + column, + sample_size_hint, + is_nullable, + scan, + Some(still_needed), + ) + .await } - _ => sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await, }; } @@ -516,12 +507,20 @@ async fn sample_training_data( DataType::FixedSizeList(_, _) => { let scan = sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?; - sample_nullable_fsl(column, sample_size_hint, byte_width, vector_field, scan).await + sample_nullable_fsl( + column, + sample_size_hint, + byte_width, + vector_field, + scan, + None, + ) + .await } _ => { let scan = sample_training_data_scan(dataset, column, sample_size_hint, num_rows, byte_width)?; - sample_nullable_fallback(column, sample_size_hint, is_nullable, scan).await + sample_nullable_fallback(column, sample_size_hint, is_nullable, scan, None).await } } } @@ -550,12 +549,19 @@ fn sample_training_data_scan( /// sampling must first map random offsets within the selected fragments to row /// addresses and then `take` those rows. Both nullable FSL and multivector /// paths reuse this stream to avoid duplicating fragment sampling logic. +/// +/// Each round is sized to `still_needed` (the consumer's outstanding demand), +/// so a low-null column reads at most the requested sample. Visited offsets +/// are tracked in a [`RoaringTreemap`] because sparse or all-null columns +/// force the stream to visit most selected rows before it can terminate, and +/// a compressed bitmap keeps that persistent state near `num_rows / 8` bytes +/// even when fully populated. fn sample_training_data_scan_from_fragments( dataset: &Dataset, column: &str, - sample_size_hint: usize, num_rows: usize, fragment_ids: &[u32], + still_needed: Arc, ) -> Result> + Send>>> { if fragment_ids.is_empty() { return Err(Error::invalid_input( @@ -563,21 +569,7 @@ fn sample_training_data_scan_from_fragments( )); } - let mut ordered_ids = fragment_ids.to_vec(); - ordered_ids.sort_unstable(); - ordered_ids.dedup(); - let selected_fragments = dataset - .get_frags_from_ordered_ids(&ordered_ids) - .into_iter() - .zip(ordered_ids.iter()) - .map(|(fragment, fragment_id)| { - fragment.ok_or_else(|| { - Error::invalid_input(format!( - "Unknown fragment id {fragment_id} in training fragment filter" - )) - }) - }) - .collect::>>()?; + let selected_fragments = dataset.get_fragments_from_ids(fragment_ids)?; let dataset = Arc::new(dataset.clone()); let projection = Arc::new( ProjectionRequest::from(dataset.schema().project(&[column])?) @@ -589,19 +581,36 @@ fn sample_training_data_scan_from_fragments( dataset, projection, selected_fragments, - HashSet::::with_capacity(sample_size_hint.min(num_rows)), + RoaringTreemap::new(), SmallRng::from_os_rng(), + still_needed, ), - move |(dataset, projection, selected_fragments, mut seen_offsets, mut rng)| async move { - if seen_offsets.len() >= num_rows { + move |( + dataset, + projection, + selected_fragments, + mut seen_offsets, + mut rng, + still_needed, + )| async move { + if seen_offsets.len() as usize >= num_rows { + return Ok(None); + } + let still = still_needed.load(Ordering::Relaxed); + if still == 0 { return Ok(None); } - let remaining = num_rows.saturating_sub(seen_offsets.len()); - let target = sample_size_hint.saturating_mul(2).min(remaining); + let remaining = num_rows.saturating_sub(seen_offsets.len() as usize); + // Sizing the round to the outstanding demand keeps a low-null + // column's reads bounded by the requested sample, matching the + // non-nullable path. + let target = still.min(remaining); let mut sampled_offsets = if remaining <= target.saturating_mul(4) { + // Few offsets remain unseen, so shuffling the unseen set is + // cheaper than repeatedly rejecting already-sampled offsets. let mut unseen_indices = (0..num_rows as u64) - .filter(|index| !seen_offsets.contains(index)) + .filter(|index| !seen_offsets.contains(*index)) .collect::>(); unseen_indices.shuffle(&mut rng); unseen_indices.truncate(target); @@ -635,7 +644,14 @@ fn sample_training_data_scan_from_fragments( .await?; Ok(Some(( batch, - (dataset, projection, selected_fragments, seen_offsets, rng), + ( + dataset, + projection, + selected_fragments, + seen_offsets, + rng, + still_needed, + ), ))) }, ); @@ -646,22 +662,7 @@ fn resolve_scan_fragments( dataset: &Dataset, fragment_ids: &[u32], ) -> Result> { - let mut ordered_ids = fragment_ids.to_vec(); - ordered_ids.sort_unstable(); - let fragments = dataset.get_frags_from_ordered_ids(&ordered_ids); - if let Some(missing_id) = fragments - .iter() - .zip(ordered_ids.iter()) - .find_map(|(fragment, fragment_id)| fragment.is_none().then_some(*fragment_id)) - { - return Err(Error::invalid_input(format!( - "Unknown fragment id {missing_id} in training fragment filter" - ))); - } - Ok(fragments - .into_iter() - .map(|fragment| fragment.unwrap().metadata().clone()) - .collect()) + dataset.get_fragment_metadata_from_ids(fragment_ids) } /// Build a FixedSizeListArray from raw flat value bytes. @@ -721,6 +722,7 @@ async fn sample_nullable_fsl( byte_width: usize, vector_field: &lance_core::datatypes::Field, mut scan: S, + still_needed: Option>, ) -> Result where S: Stream> + Unpin, @@ -731,6 +733,12 @@ where let mut rows_scanned: usize = 0; while num_non_null < sample_size_hint { + let remaining_rows = sample_size_hint - num_non_null; + // A fragment-limited producer sizes its next prefetch round to this + // outstanding demand. + if let Some(still_needed) = &still_needed { + still_needed.store(remaining_rows, Ordering::Relaxed); + } let Some(batch) = scan.next().await else { break; }; @@ -750,7 +758,16 @@ where continue; } let previous_num_non_null = num_non_null; - accumulate_fsl_values(&mut values_buf, &mut num_non_null, &array, byte_width, true)?; + // `remaining_rows` keeps `values_buf` within its pre-allocated + // `sample_size_hint * byte_width` capacity. + accumulate_fsl_values( + &mut values_buf, + &mut num_non_null, + &array, + byte_width, + true, + remaining_rows, + )?; info!( "Sample training data: batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled after null filtering)", batch_count, @@ -762,6 +779,11 @@ where ); } + // Zero the demand so any further poll of the producer terminates instead + // of reading another round. + if let Some(still_needed) = &still_needed { + still_needed.store(0, Ordering::Relaxed); + } let num_rows_out = num_non_null.min(sample_size_hint); values_buf.truncate(num_rows_out * byte_width); @@ -797,7 +819,14 @@ async fn sample_fsl_uniform( for (chunk_idx, chunk) in indices.chunks(TAKE_CHUNK_SIZE).enumerate() { let batch = dataset.take(chunk, projection.clone()).await?; let array = get_column_from_batch(&batch, column)?; - accumulate_fsl_values(&mut values_buf, &mut total_rows, &array, byte_width, false)?; + accumulate_fsl_values( + &mut values_buf, + &mut total_rows, + &array, + byte_width, + false, + usize::MAX, + )?; info!( "Sample training data: batch {}/{} read {} rows ({}/{} sampled by uniform random sampling)", chunk_idx + 1, @@ -821,13 +850,20 @@ async fn sample_fsl_uniform( /// When `filter_nulls` is false and there are no nulls, copies raw bytes /// directly from the FSL values buffer (accounting for child array offset). /// When `filter_nulls` is true, uses Arrow's `filter` kernel to remove nulls. +/// At most `max_rows` rows are appended so callers can stop copying once their +/// sample is full; otherwise one oversized source batch can grow `values_buf` +/// far beyond the intended cap. fn accumulate_fsl_values( values_buf: &mut MutableBuffer, num_rows: &mut usize, array: &ArrayRef, byte_width: usize, filter_nulls: bool, + max_rows: usize, ) -> Result<()> { + if max_rows == 0 { + return Ok(()); + } let needs_filter = filter_nulls && array.null_count() > 0; if needs_filter { @@ -835,21 +871,29 @@ fn accumulate_fsl_values( let mask = arrow_array::BooleanArray::from(nulls.inner().clone()); let filtered = arrow::compute::filter(array, &mask)?; let fsl = filtered.as_fixed_size_list(); + let take = fsl.len().min(max_rows); + if take == 0 { + return Ok(()); + } let values_data = fsl.values().to_data(); - let value_bytes = &values_data.buffers()[0].as_slice()[..fsl.len() * byte_width]; + let value_bytes = &values_data.buffers()[0].as_slice()[..take * byte_width]; values_buf.extend_from_slice(value_bytes); - *num_rows += fsl.len(); + *num_rows += take; } else { // No nulls: copy raw bytes directly, accounting for child array offset. let fsl = array.as_fixed_size_list(); + let take = fsl.len().min(max_rows); + if take == 0 { + return Ok(()); + } let values = fsl.values(); let values_data = values.to_data(); let elem_size = byte_width / fsl.value_length() as usize; let offset_bytes = values_data.offset() * elem_size; - let total_bytes = fsl.len() * byte_width; + let total_bytes = take * byte_width; let buf = &values_data.buffers()[0].as_slice()[offset_bytes..offset_bytes + total_bytes]; values_buf.extend_from_slice(buf); - *num_rows += fsl.len(); + *num_rows += take; } Ok(()) } @@ -862,6 +906,7 @@ async fn sample_nullable_fallback( sample_size_hint: usize, is_nullable: bool, mut scan: S, + still_needed: Option>, ) -> Result where S: Stream> + Unpin, @@ -873,6 +918,12 @@ where let mut rows_scanned: usize = 0; while num_non_null < sample_size_hint { + let remaining_rows = sample_size_hint - num_non_null; + // A fragment-limited producer sizes its next prefetch round to this + // outstanding demand. + if let Some(still_needed) = &still_needed { + still_needed.store(remaining_rows, Ordering::Relaxed); + } let Some(batch) = scan.next().await else { break; }; @@ -898,7 +949,14 @@ where } else { batch }; - let accepted_rows = batch.num_rows(); + // Slicing to the outstanding demand keeps the retained batches, and + // the post-loop `concat_batches`, bounded by the sample size. + let accepted_rows = batch.num_rows().min(remaining_rows); + let batch = if accepted_rows < batch.num_rows() { + batch.slice(0, accepted_rows) + } else { + batch + }; num_non_null += accepted_rows; info!( "Sample training data (fallback): batch {} read {} rows, accepted {} rows ({} scanned, {}/{} sampled)", @@ -912,6 +970,12 @@ where filtered.push(batch); } + // Zero the demand so any further poll of the producer terminates instead + // of reading another round. + if let Some(still_needed) = &still_needed { + still_needed.store(0, Ordering::Relaxed); + } + let Some(schema) = schema else { return Err(Error::index("No non-null training data found".to_string())); }; @@ -1040,6 +1104,7 @@ mod tests { use crate::dataset::InsertBuilder; use arrow_array::{ArrayRef, Float32Array, types::Float32Type}; + use arrow_buffer::{BooleanBufferBuilder, NullBuffer}; use arrow_schema::{DataType, Field}; use lance_arrow::FixedSizeListArrayExt; use lance_datagen::{ArrayGeneratorExt, Dimension, RowCount, array, gen_batch}; @@ -1201,7 +1266,15 @@ mod tests { let mut buf = MutableBuffer::new(0); let mut num_rows = 0usize; let sliced_ref: ArrayRef = Arc::new(sliced); - accumulate_fsl_values(&mut buf, &mut num_rows, &sliced_ref, byte_width, false).unwrap(); + accumulate_fsl_values( + &mut buf, + &mut num_rows, + &sliced_ref, + byte_width, + false, + usize::MAX, + ) + .unwrap(); assert_eq!(num_rows, 4); let result: &[f32] = @@ -1327,4 +1400,140 @@ mod tests { let result = count_rows(&dataset, Some(&[ids[2], ids[0]])).await.unwrap(); assert_eq!(result, 250); } + + /// Nullable FSL with fragment-limited sampling must fill the requested sample + /// size when enough non-null rows exist, and terminate cleanly when all + /// selected rows are null. + #[tokio::test] + async fn test_maybe_sample_training_data_fsl_nullable_fragment_limited() { + let nrows: usize = 2000; + let dims: u32 = 8; + let sample_size: usize = 500; + + for (case, null_probability, expected_len) in + [("partial_nulls", 0.5, sample_size), ("all_nulls", 1.0, 0)] + { + let col_gen = array::rand_vec::(Dimension::from(dims)) + .with_random_nulls(null_probability); + let data = gen_batch() + .col("vec", col_gen) + .into_batch_rows(RowCount::from(nrows as u64)) + .unwrap(); + + let dataset = InsertBuilder::new("memory://") + .execute(vec![data]) + .await + .unwrap(); + + let fragment_ids: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + + let training_data = + maybe_sample_training_data(&dataset, "vec", sample_size, Some(&fragment_ids)) + .await + .unwrap(); + + assert_eq!(training_data.len(), expected_len, "{case}"); + assert_eq!(training_data.null_count(), 0, "{case}"); + assert_eq!(training_data.value_length(), dims as i32, "{case}"); + } + } + + /// Scan-side regression: each fragment-limited producer round must read at + /// most the consumer's outstanding demand. Driving the producer directly + /// with a fixed `still_needed` and inspecting the raw batch size catches + /// over-reads that a post-truncation output-length check cannot. + #[tokio::test] + async fn test_sample_fragment_scan_round_caps_at_still_needed() { + let nrows: usize = 4000; + let dims: u32 = 8; + let still: usize = 500; + + let col_gen = array::rand_vec::(Dimension::from(dims)).with_random_nulls(0.5); + let data = gen_batch() + .col("vec", col_gen) + .into_batch_rows(RowCount::from(nrows as u64)) + .unwrap(); + + let dataset = InsertBuilder::new("memory://fsl_scan_round_cap_test") + .execute(vec![data]) + .await + .unwrap(); + + let fragment_ids: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + let num_rows = count_rows(&dataset, Some(&fragment_ids)).await.unwrap(); + + // `still_needed` is left large enough that `num_rows` never bounds the + // round, so the batch size reflects the demand cap and nothing else. + let still_needed = Arc::new(AtomicUsize::new(still)); + let mut scan = sample_training_data_scan_from_fragments( + &dataset, + "vec", + num_rows, + &fragment_ids, + still_needed.clone(), + ) + .unwrap(); + + let batch = scan.next().await.unwrap().unwrap(); + assert!( + batch.num_rows() <= still, + "producer round read {} rows but only {} were outstanding", + batch.num_rows(), + still + ); + } + + #[test] + fn test_accumulate_fsl_values_respects_max_rows() { + let dim: usize = 4; + let total_rows: usize = 100; + let max_rows: usize = 16; + let byte_width = dim * std::mem::size_of::(); + + let values: Vec = (0..total_rows * dim).map(|i| i as f32).collect(); + let fsl = FixedSizeListArray::try_new_from_values(Float32Array::from(values), dim as i32) + .unwrap(); + let arr: ArrayRef = Arc::new(fsl); + + let mut buf = MutableBuffer::new(0); + let mut num_rows = 0usize; + accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap(); + + assert_eq!(num_rows, max_rows); + assert_eq!(buf.len(), max_rows * byte_width); + + let values: Vec = (0..total_rows * dim).map(|i| i as f32).collect(); + let item_field = Arc::new(Field::new("item", DataType::Float32, true)); + + // Every other row is null, leaving 50 non-null rows. + let mut nulls_builder = BooleanBufferBuilder::new(total_rows); + for i in 0..total_rows { + nulls_builder.append(i % 2 == 0); + } + let nulls = NullBuffer::new(nulls_builder.finish()); + + let fsl = FixedSizeListArray::try_new( + item_field, + dim as i32, + Arc::new(Float32Array::from(values)), + Some(nulls), + ) + .unwrap(); + let arr: ArrayRef = Arc::new(fsl); + + let mut buf = MutableBuffer::new(0); + let mut num_rows = 0usize; + accumulate_fsl_values(&mut buf, &mut num_rows, &arr, byte_width, true, max_rows).unwrap(); + + assert_eq!(num_rows, max_rows); + assert_eq!(buf.len(), max_rows * byte_width); + } } diff --git a/rust/lance/src/io/commit.rs b/rust/lance/src/io/commit.rs index 6ac6a0c362f..f48913e1513 100644 --- a/rust/lance/src/io/commit.rs +++ b/rust/lance/src/io/commit.rs @@ -20,24 +20,31 @@ //! alternative to [`CommitHandler`]. use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::num::NonZero; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use conflict_resolver::TransactionRebase; use lance_core::utils::backoff::{Backoff, SlotBackoff}; +use lance_core::utils::tracing::{AUDIT_MODE_DELETE, AUDIT_TYPE_TRANSACTION, TRACE_FILE_AUDIT}; +#[cfg(test)] use lance_file::version::LanceFileVersion; + use lance_index::metrics::NoOpMetricsCollector; use lance_io::utils::CachedFileSize; use lance_select::RowAddrTreeMap; +use lance_table::feature_flags::ensure_can_write_manifest; use lance_table::format::{ - DETACHED_VERSION_MASK, DataStorageFormat, DeletionFile, Fragment, IndexMetadata, Manifest, - WriterVersion, is_detached_version, list_index_files_with_sizes, pb, + DETACHED_VERSION_MASK, DeletionFile, Fragment, IndexMetadata, Manifest, WriterVersion, + is_detached_version, list_index_files_with_sizes, operation_may_change_schema, pb, }; use lance_table::io::commit::{ CommitConfig, CommitError, CommitHandler, ManifestLocation, ManifestNamingScheme, }; +use lance_table::io::manifest::read_manifest; use rand::{Rng, rng}; +use roaring::RoaringBitmap; use super::ObjectStore; use crate::Dataset; @@ -48,20 +55,19 @@ use crate::dataset::{ ManifestWriteConfig, NewTransactionResult, TRANSACTIONS_DIR, load_new_transactions, write_manifest_file, }; -use crate::index::DatasetIndexExt; use crate::index::DatasetIndexInternalExt; use crate::index::vector::details::infer_missing_vector_details; +use crate::index::{index_is_usable, load_all_indices}; use crate::io::deletion::read_dataset_deletion_file; use crate::session::Session; use crate::session::caches::DSMetadataCache; use crate::session::index_caches::IndexMetadataKey; use futures::future::Either; -use futures::{StreamExt, TryFutureExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use lance_core::{Error, Result}; use lance_index::is_system_index; use lance_io::object_store::ObjectStoreRegistry; use log; -#[cfg(test)] use object_store::ObjectStoreExt; use object_store::path::Path; use prost::Message; @@ -75,8 +81,36 @@ pub mod namespace_manifest; #[cfg(all(feature = "dynamodb_tests", test))] mod s3_test; +/// Wall-clock budget for conflict retry backoff when callers do not override it. +pub(crate) const DEFAULT_COMMIT_RETRY_TIMEOUT: Duration = Duration::from_secs(30); + +pub(crate) fn timeout_error(retry_timeout: Duration, attempts: u32) -> Error { + Error::too_much_write_contention(format!( + "Attempted {} times, but failed on retry_timeout of {:.3} seconds.", + attempts, + retry_timeout.as_secs_f32() + )) +} + +pub(crate) fn maybe_timeout( + attempt: u32, + start: Instant, + retry_timeout: Duration, + future: impl Future, +) -> impl Future> { + if attempt == 0 { + // The first attempt establishes the observed latency used by SlotBackoff. + Either::Left(future.map(Ok)) + } else { + let remaining = retry_timeout.saturating_sub(start.elapsed()); + Either::Right( + tokio::time::timeout(remaining, future) + .map_err(move |_| timeout_error(retry_timeout, attempt + 1)), + ) + } +} + /// Read the transaction data from a transaction file. -#[cfg(test)] pub(crate) async fn read_transaction_file( object_store: &ObjectStore, base_path: &Path, @@ -97,6 +131,11 @@ pub(crate) async fn read_transaction_file( /// Logs a warning on failure rather than propagating the error, since the /// primary operation has already failed and the orphaned file will eventually /// be removed by GC. +/// +/// Callers must only invoke this for attempts whose commit is confirmed to +/// have NOT landed (see [`verify_commit_outcome`]): a landed manifest +/// references its transaction file by path, so deleting it would corrupt the +/// version. async fn cleanup_transaction_file( object_store: &ObjectStore, base_path: &Path, @@ -109,12 +148,174 @@ async fn cleanup_transaction_file( .clone() .join(TRANSACTIONS_DIR) .join(transaction_file); - if let Err(e) = object_store.delete(&path).await { - log::warn!( - "Failed to clean up orphaned transaction file '{}': {}", - transaction_file, - e - ); + match object_store.delete(&path).await { + Ok(()) => { + tracing::info!( + target: TRACE_FILE_AUDIT, + mode = AUDIT_MODE_DELETE, + r#type = AUDIT_TYPE_TRANSACTION, + path = transaction_file, + ); + } + Err(e) => { + log::warn!( + "Failed to clean up orphaned transaction file '{}': {}", + transaction_file, + e + ); + } + } +} + +/// Who owns the manifest at a version, checked after a failed commit attempt. +#[derive(Debug)] +enum CommitOutcome { + /// The manifest at the version is the one this attempt wrote: the commit + /// actually landed even though the store reported a failure (e.g. the + /// response to a successful conditional PUT was lost and an internal + /// retry surfaced "already exists"). + Ours { + manifest: Box, + location: ManifestLocation, + }, + /// A manifest exists at the version and records a different transaction + /// file: another writer definitely won the version. + Foreign, + /// No manifest exists at the version: this attempt definitely did not + /// land. + Absent, + /// The verification reads themselves kept failing; whether the commit + /// landed cannot be determined. Callers must not run destructive cleanup + /// in this state. + Unknown, +} + +/// Maximum verification read attempts in [`verify_commit_outcome`]. +const COMMIT_VERIFICATION_ATTEMPTS: u32 = 3; + +/// Determine whether a failed commit attempt actually landed, by comparing +/// the complete transaction recorded in the manifest at `version` with this +/// attempt's transaction. +/// +/// Never returns an error. Read failures and non-definitive not-found results +/// are retried briefly, then collapse to [`CommitOutcome::Unknown`]. +async fn verify_commit_outcome( + object_store: &ObjectStore, + commit_handler: &dyn CommitHandler, + base_path: &Path, + version: u64, + transaction: &Transaction, +) -> CommitOutcome { + enum VerificationFailure { + NotFound, + Read(Error), + } + + let mut backoff = Backoff::default(); + let failure = loop { + let failure = match try_read_manifest_at(object_store, commit_handler, base_path, version) + .await + { + Ok(Some((manifest, location))) => { + match read_manifest_transaction(object_store, base_path, &manifest, &location).await + { + Ok(Some(committed_transaction)) => { + return if committed_transaction == *transaction { + CommitOutcome::Ours { + manifest: Box::new(manifest), + location, + } + } else { + CommitOutcome::Foreign + }; + } + Ok(None) => return CommitOutcome::Foreign, + Err(error) => VerificationFailure::Read(error), + } + } + Ok(None) if commit_handler.is_version_not_found_definitive() => { + return CommitOutcome::Absent; + } + Ok(None) => VerificationFailure::NotFound, + Err(error) => VerificationFailure::Read(error), + }; + + if backoff.attempt() + 1 >= COMMIT_VERIFICATION_ATTEMPTS { + break failure; + } + tokio::time::sleep(backoff.next_backoff()).await; + }; + + match failure { + VerificationFailure::NotFound => { + log::warn!( + "The manifest for version {} was not visible after {} commit verification \ + attempts, and the commit handler does not guarantee definitive not-found \ + results; treating the commit status as unknown", + version, + COMMIT_VERIFICATION_ATTEMPTS + ); + CommitOutcome::Unknown + } + VerificationFailure::Read(error) => { + log::warn!( + "Could not verify the outcome of the commit attempt for version {} after {} \ + tries; treating the commit status as unknown: {}", + version, + COMMIT_VERIFICATION_ATTEMPTS, + error + ); + CommitOutcome::Unknown + } + } +} + +async fn read_manifest_transaction( + object_store: &ObjectStore, + base_path: &Path, + manifest: &Manifest, + location: &ManifestLocation, +) -> Result> { + if let Some(position) = manifest.transaction_section { + let reader = if let Some(size) = location.size { + object_store + .open_with_size(&location.path, size as usize) + .await? + } else { + object_store.open(&location.path).await? + }; + let transaction: pb::Transaction = + lance_io::utils::read_message(reader.as_ref(), position).await?; + Transaction::try_from(transaction).map(Some) + } else if let Some(transaction_file) = manifest.transaction_file.as_deref() { + read_transaction_file(object_store, base_path, transaction_file) + .await + .map(Some) + } else { + Ok(None) + } +} + +/// Read the manifest at `version`, distinguishing "no such version" +/// (`Ok(None)`) from transient read failures (`Err`). +async fn try_read_manifest_at( + object_store: &ObjectStore, + commit_handler: &dyn CommitHandler, + base_path: &Path, + version: u64, +) -> Result> { + let location = match commit_handler + .resolve_version_location(base_path, version, &object_store.inner) + .await + { + Ok(location) => location, + Err(Error::NotFound { .. }) => return Ok(None), + Err(e) => return Err(e), + }; + match read_manifest(object_store, &location.path, location.size).await { + Ok(manifest) => Ok(Some((manifest, location))), + Err(Error::NotFound { .. }) => Ok(None), + Err(e) => Err(e), } } @@ -122,7 +323,7 @@ async fn cleanup_transaction_file( pub(crate) async fn write_transaction_file( object_store: &ObjectStore, base_path: &Path, - transaction: &Transaction, + transaction: &pb::Transaction, ) -> Result { let file_name = format!("{}-{}.txn", transaction.read_version, transaction.uuid); let path = base_path @@ -130,16 +331,25 @@ pub(crate) async fn write_transaction_file( .join(TRANSACTIONS_DIR) .join(file_name.as_str()); - let message = pb::Transaction::from(transaction); - let buf = message.encode_to_vec(); + let buf = transaction.encode_to_vec(); object_store.put(&path, &buf).await?; Ok(file_name) } +/// Transactions serialized above this size are not inlined into the manifest. +#[cfg(not(test))] +pub(crate) const MAX_INLINE_TRANSACTION_BYTES: usize = 20 * 1024 * 1024; +/// Smaller threshold for unit tests so spill coverage does not need +/// multi-megabyte payloads. +#[cfg(test)] +pub(crate) const MAX_INLINE_TRANSACTION_BYTES: usize = 64 * 1024; + #[allow(clippy::too_many_arguments)] async fn do_commit_new_dataset( object_store: &ObjectStore, + source_store: Option<&ObjectStore>, + source_commit_handler: Option<&dyn CommitHandler>, commit_handler: &dyn CommitHandler, base_path: &Path, transaction: &Transaction, @@ -148,34 +358,58 @@ async fn do_commit_new_dataset( metadata_cache: &DSMetadataCache, store_registry: Arc, ) -> Result<(Manifest, ManifestLocation)> { - let transaction_file = if !write_config.disable_transaction_file() { - write_transaction_file(object_store, base_path, transaction).await? - } else { - String::new() - }; + let pb_transaction = pb::Transaction::from(transaction); + let inline_transaction = pb_transaction.encoded_len() <= MAX_INLINE_TRANSACTION_BYTES; + // Classified from the operation itself. Reading it back off the inline + // copy would tie the verdict to the payload size instead. + let may_change_schema = operation_may_change_schema(&pb_transaction); - let (mut manifest, indices) = if let Operation::Clone { - is_shallow, - ref_name, + let clone_source = if let Operation::Clone { ref_version, ref_path, - branch_name, .. } = &transaction.operation { + // The source manifest must be read through the source store, which may differ + // from the destination store when cloning across object stores/accounts. Falls + // back to the destination store for same-store clones. + let source_store = source_store.unwrap_or(object_store); + let source_commit_handler = source_commit_handler.unwrap_or(commit_handler); let source_base_path = ObjectStore::extract_path_from_uri(store_registry, ref_path.as_str())?; - let source_manifest_location = commit_handler - .resolve_version_location(&source_base_path, *ref_version, &object_store.inner) + let source_manifest_location = source_commit_handler + .resolve_version_location(&source_base_path, *ref_version, &source_store.inner) .await?; let source_manifest = Dataset::load_manifest( - object_store, + source_store, &source_manifest_location, - base_path.to_string().as_str(), + ref_path.as_str(), &Session::default(), ) .await?; + ensure_can_write_manifest(&source_manifest)?; + Some((source_store, source_manifest_location, source_manifest)) + } else { + None + }; + + let transaction_file = if !write_config.disable_transaction_file() { + write_transaction_file(object_store, base_path, &pb_transaction).await? + } else { + String::new() + }; + let (mut manifest, indices) = if let ( + Operation::Clone { + is_shallow, + ref_name, + ref_path, + branch_name, + .. + }, + Some((source_store, source_manifest_location, source_manifest)), + ) = (&transaction.operation, clone_source) + { if *is_shallow { let new_base_id = source_manifest .base_paths @@ -192,7 +426,7 @@ async fn do_commit_new_dataset( ); let updated_indices = if let Some(index_section_pos) = source_manifest.index_section { - let reader = object_store.open(&source_manifest_location.path).await?; + let reader = source_store.open(&source_manifest_location.path).await?; let section: pb::IndexSection = lance_io::utils::read_message(reader.as_ref(), index_section_pos).await?; section @@ -215,9 +449,11 @@ async fn do_commit_new_dataset( new_manifest.branch = None; new_manifest.tag = None; new_manifest.index_section = None; // will be rewritten below + new_manifest.transaction_file = + (!transaction_file.is_empty()).then_some(transaction_file.clone()); let mut new_frags = new_manifest.fragments.as_ref().clone(); for f in &mut new_frags { - for df in &mut f.files { + for df in f.referenced_lance_files_mut() { df.base_id = None; } if let Some(d) = f.deletion_file.as_mut() { @@ -229,7 +465,7 @@ async fn do_commit_new_dataset( // Indices: keep metadata but normalize base to local let mut updated_indices = Vec::new(); if let Some(index_section_pos) = source_manifest.index_section { - let reader = object_store.open(&source_manifest_location.path).await?; + let reader = source_store.open(&source_manifest_location.path).await?; let section: pb::IndexSection = lance_io::utils::read_message(reader.as_ref(), index_section_pos).await?; updated_indices = section @@ -245,8 +481,12 @@ async fn do_commit_new_dataset( (new_manifest, updated_indices) } } else { - let (manifest, indices) = - transaction.build_manifest(None, vec![], &transaction_file, write_config)?; + let (manifest, indices) = transaction.build_manifest( + None, + vec![], + &transaction_file, + &write_config.to_build_config(), + )?; (manifest, indices) }; @@ -262,7 +502,8 @@ async fn do_commit_new_dataset( }, write_config, manifest_naming_scheme, - Some(transaction), + inline_transaction.then(|| pb_transaction.into()), + may_change_schema, ) .await; @@ -270,36 +511,122 @@ async fn do_commit_new_dataset( // if there is a conflict. match result { Ok(manifest_location) => { - let tx_key = crate::session::caches::TransactionKey { - version: manifest.version, - }; - metadata_cache - .insert_with_key(&tx_key, Arc::new(transaction.clone())) - .await; - - let manifest_key = crate::session::caches::ManifestKey { - version: manifest_location.version, - e_tag: manifest_location.e_tag.as_deref(), - }; - metadata_cache - .insert_with_key(&manifest_key, Arc::new(manifest.clone())) + record_new_dataset_commit(metadata_cache, transaction, &manifest, &manifest_location) .await; Ok((manifest, manifest_location)) } Err(CommitError::CommitConflict) => { + // The dataset may "already exist" because this attempt's own + // manifest write landed but returned an ambiguous error. Verify + // before reporting a conflict (and before deleting the + // transaction file a landed manifest would reference). + match verify_commit_outcome( + object_store, + commit_handler, + base_path, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_new_dataset_commit( + metadata_cache, + transaction, + &committed_manifest, + &location, + ) + .await; + return Ok((committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + "dataset creation reported a conflict but the manifest could not \ + be read back for verification" + .to_string() + .into(), + )); + } + } cleanup_transaction_file(object_store, base_path, &transaction_file).await; Err(crate::Error::dataset_already_exists(base_path.to_string())) } Err(CommitError::OtherError(err)) => { + match verify_commit_outcome( + object_store, + commit_handler, + base_path, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_new_dataset_commit( + metadata_cache, + transaction, + &committed_manifest, + &location, + ) + .await; + if commit_handler.propagate_commit_error_after_success() { + return Err(err); + } + return Ok((committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + Box::new(err), + )); + } + } cleanup_transaction_file(object_store, base_path, &transaction_file).await; Err(err) } } } +/// Cache bookkeeping for a successful new-dataset commit, shared by the +/// direct-success and verified-own-commit paths of `do_commit_new_dataset`. +async fn record_new_dataset_commit( + metadata_cache: &DSMetadataCache, + transaction: &Transaction, + manifest: &Manifest, + location: &ManifestLocation, +) { + let tx_key = crate::session::caches::TransactionKey { + version: manifest.version, + }; + metadata_cache + .insert_with_key(&tx_key, Arc::new(transaction.clone())) + .await; + + let manifest_key = crate::session::caches::ManifestKey { + version: location.version, + e_tag: location.e_tag.as_deref(), + }; + metadata_cache + .insert_with_key(&manifest_key, Arc::new(manifest.clone())) + .await; +} + #[allow(clippy::too_many_arguments)] pub(crate) async fn commit_new_dataset( object_store: &ObjectStore, + source_store: Option<&ObjectStore>, + source_commit_handler: Option<&dyn CommitHandler>, commit_handler: &dyn CommitHandler, base_path: &Path, transaction: &Transaction, @@ -310,6 +637,8 @@ pub(crate) async fn commit_new_dataset( ) -> Result<(Manifest, ManifestLocation)> { do_commit_new_dataset( object_store, + source_store, + source_commit_handler, commit_handler, base_path, transaction, @@ -368,98 +697,31 @@ async fn migrate_manifest( } fn check_storage_version(manifest: &mut Manifest) -> Result<()> { - let data_storage_version = manifest.data_storage_format.lance_file_version()?; - if manifest.data_storage_format.lance_file_version()? == LanceFileVersion::Legacy { - // Due to bugs in 0.16 it is possible the dataset's data storage version does not - // match the file version. As a result, we need to check and see if they are out - // of sync. - if let Some(actual_file_version) = - Fragment::try_infer_version(&manifest.fragments).map_err(|e| Error::internal(format!( - "The dataset contains a mixture of file versions. You will need to rollback to an earlier version: {}", - e - )))? - && actual_file_version > data_storage_version { - log::warn!( - "Data storage version {} is less than the actual file version {}. This has been automatically updated.", - data_storage_version, - actual_file_version - ); - manifest.data_storage_format = DataStorageFormat::new(actual_file_version); - } - } else { - // Otherwise, if we are on 2.0 or greater, we should ensure that the file versions - // match the data storage version. This is a sanity assertion to prevent data corruption. - if let Some(actual_file_version) = Fragment::try_infer_version(&manifest.fragments)? - && actual_file_version != data_storage_version - { - return Err(Error::internal(format!( - "The operation added files with version {}. However, the data storage version is {}.", - actual_file_version, data_storage_version - ))); - } + crate::dataset::versions::check_manifest_storage_version(manifest) +} + +/// Reject a manifest in which two fragments share an id. Per-fragment state is +/// keyed by fragment id — deletion file paths, cached row id sequences, row +/// addresses — so a duplicate makes it ambiguous which rows that state describes. +/// +/// Runs after the legacy fixups above, so a dataset that needs a rollback for some +/// other reason is diagnosed with that first. Relies on `build_manifest` leaving +/// the fragments sorted by id. +fn check_fragment_ids(manifest: &Manifest) -> Result<()> { + if let Some(pair) = manifest.fragments.windows(2).find(|p| p[0].id == p[1].id) { + return Err(Error::invalid_input(format!( + "The commit would produce two fragments with id {}. Fragment ids must be \ + unique. Datasets written by Lance 0.16 and earlier may already contain \ + duplicate ids; those have to be rewritten, or rolled back to a version \ + without the duplicate.", + pair[0].id + ))); } Ok(()) } fn check_column_indices(manifest: &Manifest) -> Result<()> { - let data_storage_version = manifest.data_storage_format.lance_file_version()?; - if data_storage_version < LanceFileVersion::V2_1 { - return Ok(()); - } - - for fragment in manifest.fragments.iter() { - for data_file in &fragment.files { - if data_file.is_legacy_file() || data_file.column_indices.is_empty() { - continue; - } - if data_file.fields.len() != data_file.column_indices.len() { - return Err(Error::invalid_input(format!( - "Data file '{}' (fragment {}) has {} field ids but {} column indices. \ - These must be the same length.", - data_file.path, - fragment.id, - data_file.fields.len(), - data_file.column_indices.len() - ))); - } - let file_version = LanceFileVersion::try_from_major_minor( - data_file.file_major_version, - data_file.file_minor_version, - )?; - if file_version < LanceFileVersion::V2_1 { - continue; - } - for (field_id, column_index) in - data_file.fields.iter().zip(data_file.column_indices.iter()) - { - // Field ids may not exist in the current schema after schema - // evolution (e.g. cast/drop column). Skip those. - let Some(field) = manifest.schema.field_by_id(*field_id) else { - continue; - }; - let needs_column = field.is_leaf() || field.is_packed_struct() || field.is_blob(); - if needs_column && *column_index == -1 { - return Err(Error::invalid_input(format!( - "Field '{}' (id={}) in data file '{}' (fragment {}) \ - has column_index=-1, but leaf fields, packed structs, \ - and blob fields must have a valid column index in \ - file format 2.1+.", - field.name, field_id, data_file.path, fragment.id - ))); - } - if !needs_column && *column_index != -1 { - return Err(Error::invalid_input(format!( - "Non-leaf field '{}' (id={}) in data file '{}' (fragment {}) \ - has column_index={}, but non-leaf fields should have \ - column_index=-1 in file format 2.1+. Only leaf fields, \ - packed structs, and blob fields should have column indices.", - field.name, field_id, data_file.path, fragment.id, column_index - ))); - } - } - } - } - Ok(()) + crate::dataset::versions::validate_column_indices(manifest) } /// Fix schema in case of duplicate field ids. @@ -489,13 +751,12 @@ fn fix_schema(manifest: &mut Manifest) -> Result<()> { } // Now, we need to remap the field ids to be unique. - let mut field_id_seed = manifest.max_field_id() + 1; let mut old_field_id_mapping: HashMap = HashMap::new(); let mut fields_with_duplicate_ids = fields_with_duplicate_ids.into_iter().collect::>(); fields_with_duplicate_ids.sort_unstable(); - for field_id in fields_with_duplicate_ids { + for (field_id_seed, field_id) in (manifest.max_field_id() + 1..).zip(fields_with_duplicate_ids) + { old_field_id_mapping.insert(field_id, field_id_seed); - field_id_seed += 1; } let mut fragments = manifest.fragments.as_ref().clone(); @@ -675,8 +936,14 @@ fn must_recalculate_fragment_bitmap( /// /// Indices might be missing `fragment_bitmap`, so this function will add it. /// Indices might also be missing `files` (file sizes), so this function will collect them. -async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result<()> { +/// +/// Returns the logical indices whose `fragment_bitmap` this replaced. Those are +/// the only changes here that alter what an index covers, and the caller has to +/// withdraw MemWAL catch-up for them: this runs after the coverage derivation, +/// and it keeps the segment's UUID. +async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Result> { infer_missing_vector_details(dataset, indices).await; + let mut recovered_coverage = Vec::new(); let needs_recalculating = match detect_overlapping_fragments(indices) { Ok(()) => vec![], Err(BadFragmentBitmapError { bad_indices }) => { @@ -684,17 +951,76 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re } }; for index in indices.iter_mut() { + // Migration is skipped for an index this build has no reader for: every + // branch below would have to open it to recalculate anything, which is + // exactly what this build cannot do, and failing here would fail an + // unrelated commit. Skipped, not untouched - `load_all_indices` still + // remaps its `fragment_bitmap` through the fragment-reuse index, which + // is what keeps its coverage pointing at the fragments its rows live in. + if !index_is_usable(index) { + continue; + } + // Also true when the bitmap is missing entirely, so the failure path below + // pairs it with `is_some` to mean "written before the 0.8.15 fix". + let bitmap_missing_or_legacy = + must_recalculate_fragment_bitmap(index, dataset.manifest.writer_version.as_ref()); if needs_recalculating.contains(&index.name) - || must_recalculate_fragment_bitmap(index, dataset.manifest.writer_version.as_ref()) - && !is_system_index(index) + || bitmap_missing_or_legacy && !is_system_index(index) { - debug_assert_eq!(index.fields.len(), 1); + // A covered index still has exactly one keyed field; the trailing + // `covering_fields` are carried, not keyed, so counting them + // against `fields.len()` would fail this on a legal covered index. + debug_assert!( + index.keyed_field().is_some(), + "migrate_indices expects a single keyed field, got fields {:?} carrying {:?}", + index.fields, + index.covering_fields, + ); let idx_field = dataset.schema().field_by_id(index.fields[0]).ok_or_else(|| Error::internal(format!("Index with uuid {} referred to field with id {} which did not exist in dataset", index.uuid, index.fields[0])))?; // We need to calculate the fragments covered by the index - let idx = dataset - .open_generic_index(&idx_field.name, &index.uuid, &NoOpMetricsCollector) - .await?; - index.fragment_bitmap = Some(idx.calculate_included_frags().await?); + let recalculated = async { + let idx = dataset + .open_generic_index(&idx_field.name, &index.uuid, &NoOpMetricsCollector) + .await?; + idx.calculate_included_frags().await + } + .await; + match recalculated { + Ok(fragment_bitmap) => { + if index.fragment_bitmap.as_ref() != Some(&fragment_bitmap) { + recovered_coverage.push(index.name.clone()); + } + index.fragment_bitmap = Some(fragment_bitmap); + } + Err(e) => { + // Recalculating means opening the index, and failing here fails + // every commit the dataset takes, since migration runs on all of + // them. A missing bitmap and overlapping segment bitmaps are both + // re-derived from the index metadata, so they ask again on their + // own; the pre-0.8.15 trigger reads the previous manifest's writer + // version, which this commit replaces with the current one, and a + // bitmap left in place would look migrated from here on. + let repair_ends_with_this_commit = + index.fragment_bitmap.is_some() && bitmap_missing_or_legacy; + log::warn!( + "Could not recalculate the fragment bitmap for index {} (uuid: {}): {}. {}", + index.name, + index.uuid, + e, + if repair_ends_with_this_commit { + "Dropping its coverage to unknown so a build that can open the index recalculates it." + } else { + "Leaving the repair to a build that can open the index." + } + ); + if repair_ends_with_this_commit { + index.fragment_bitmap = None; + // Derivation ran before this and may have credited a + // catch-up position off the bitmap being dropped here. + recovered_coverage.push(index.name.clone()); + } + } + } } // We can't reliably recalculate the index type for label_list and bitmap indices and so we can't migrate this field. // However, we still log for visibility and to help potentially diagnose issues in the future if we grow to rely on the field. @@ -739,7 +1065,7 @@ async fn migrate_indices(dataset: &Dataset, indices: &mut [IndexMetadata]) -> Re } } - Ok(()) + Ok(recovered_coverage) } pub(crate) struct BadFragmentBitmapError { @@ -751,17 +1077,28 @@ pub(crate) struct BadFragmentBitmapError { pub(crate) fn detect_overlapping_fragments( indices: &[IndexMetadata], ) -> std::result::Result<(), BadFragmentBitmapError> { - let index_names: HashSet<&str> = indices.iter().map(|i| i.name.as_str()).collect(); + let mut bitmaps_by_name: HashMap<&str, Vec<&RoaringBitmap>> = HashMap::new(); + for index in indices { + if let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() { + bitmaps_by_name + .entry(index.name.as_str()) + .or_default() + .push(fragment_bitmap); + } + } let mut bad_indices = Vec::new(); // (index_name, overlapping_fragments) - for name in index_names { + for (name, fragment_bitmaps) in bitmaps_by_name { + // A single segment (the common case) cannot overlap with itself, so + // skip it before hashing every fragment id it covers. + if fragment_bitmaps.len() < 2 { + continue; + } let mut seen_fragment_ids = HashSet::new(); let mut overlap = Vec::new(); - for index in indices.iter().filter(|i| i.name == name) { - if let Some(fragment_bitmap) = index.fragment_bitmap.as_ref() { - for fragment in fragment_bitmap { - if !seen_fragment_ids.insert(fragment) { - overlap.push(fragment); - } + for fragment_bitmap in fragment_bitmaps { + for fragment in fragment_bitmap { + if !seen_fragment_ids.insert(fragment) { + overlap.push(fragment); } } } @@ -783,17 +1120,31 @@ pub(crate) async fn do_commit_detached_transaction( transaction: &Transaction, write_config: &ManifestWriteConfig, commit_config: &CommitConfig, + retry_timeout: Duration, ) -> Result<(Manifest, ManifestLocation)> { + ensure_can_write_manifest(&dataset.manifest)?; + let pb_transaction = pb::Transaction::from(transaction); + let inline_transaction = pb_transaction.encoded_len() <= MAX_INLINE_TRANSACTION_BYTES; + // Classified from the operation itself. Reading it back off the inline + // copy would tie the verdict to the payload size instead. + let may_change_schema = operation_may_change_schema(&pb_transaction); + // We don't strictly need a transaction file but we go ahead and create one for // record-keeping if nothing else. let transaction_file = if !write_config.disable_transaction_file() { - write_transaction_file(object_store, &dataset.base, transaction).await? + write_transaction_file(object_store, &dataset.base, &pb_transaction).await? } else { String::new() }; + // The inline copy is moved into the first commit attempt; a retry (only on + // a random-version collision) rebuilds it instead of cloning up front. + let mut inline_tx: Option = + inline_transaction.then(|| pb_transaction.into()); + // We still do a loop since we may have conflicts in the random version we pick let mut backoff = Backoff::default(); + let start = Instant::now(); while backoff.attempt() < commit_config.num_retries { // Pick a random u64 with the highest bit set to indicate it is detached let random_version = rng().random::() | DETACHED_VERSION_MASK; @@ -805,7 +1156,7 @@ pub(crate) async fn do_commit_detached_transaction( commit_handler, &dataset.base, version, - write_config, + &write_config.to_build_config(), &transaction_file, &dataset.manifest, ) @@ -813,9 +1164,9 @@ pub(crate) async fn do_commit_detached_transaction( } _ => transaction.build_manifest( Some(dataset.manifest.as_ref()), - dataset.load_indices().await?.as_ref().clone(), + load_all_indices(dataset).await?.as_ref().clone(), &transaction_file, - write_config, + &write_config.to_build_config(), )?, }; @@ -828,7 +1179,15 @@ pub(crate) async fn do_commit_detached_transaction( fix_schema(&mut manifest)?; check_storage_version(&mut manifest)?; check_column_indices(&manifest)?; - migrate_indices(dataset, &mut indices).await?; + check_fragment_ids(&manifest)?; + // Runs after the coverage derivation and can replace a fragment bitmap + // while keeping its UUID, so anything it narrowed loses its position. + let recovered_coverage = migrate_indices(dataset, &mut indices).await?; + Transaction::withdraw_coverage_invalidated_after_build( + &mut indices, + &recovered_coverage, + manifest.version, + )?; // Try to commit the manifest let result = write_manifest_file( @@ -843,7 +1202,8 @@ pub(crate) async fn do_commit_detached_transaction( }, write_config, ManifestNamingScheme::V2, - Some(transaction), + inline_tx.take(), + may_change_schema, ) .await; @@ -852,12 +1212,77 @@ pub(crate) async fn do_commit_detached_transaction( return Ok((manifest, location)); } Err(CommitError::CommitConflict) => { - // We pick a random u64 for the version, so it's possible (though extremely unlikely) - // that we have a conflict. In that case, we just try again. - tokio::time::sleep(backoff.next_backoff()).await; + // Either an (extremely unlikely) random-version collision, or + // our own write landed but returned an ambiguous error. + // Verify before retrying with a new random version. + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + return Ok((*committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + "detached commit reported a conflict but the manifest could \ + not be read back for verification" + .to_string() + .into(), + )); + } + } + if start.elapsed() > retry_timeout { + cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await; + return Err(timeout_error(retry_timeout, backoff.attempt() + 1)); + } + let sleep_fut = tokio::time::sleep(backoff.next_backoff()); + if let Err(error) = + maybe_timeout(backoff.attempt(), start, retry_timeout, sleep_fut).await + { + cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await; + return Err(error); + } + // The inline copy was moved into the failed attempt; rebuild + // it for the retry with a new random version. + inline_tx = inline_transaction.then(|| pb::Transaction::from(transaction).into()); } Err(CommitError::OtherError(err)) => { - // If other error, return + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + manifest.version, + transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + if commit_handler.propagate_commit_error_after_success() { + return Err(err); + } + return Ok((*committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + manifest.version, + Box::new(err), + )); + } + } cleanup_transaction_file(object_store, &dataset.base, &transaction_file).await; return Err(err); } @@ -884,6 +1309,7 @@ pub(crate) async fn commit_detached_transaction( transaction: &Transaction, write_config: &ManifestWriteConfig, commit_config: &CommitConfig, + retry_timeout: Duration, ) -> Result<(Manifest, ManifestLocation)> { do_commit_detached_transaction( dataset, @@ -892,6 +1318,7 @@ pub(crate) async fn commit_detached_transaction( transaction, write_config, commit_config, + retry_timeout, ) .await } @@ -910,6 +1337,57 @@ async fn load_and_sort_new_transactions( Ok((new_ds, txns)) } +/// Success-path bookkeeping shared by the direct-success and +/// verified-own-commit paths of [`commit_transaction`]: populate the session +/// caches and run the auto-cleanup hook. +async fn record_successful_commit( + dataset: &Dataset, + transaction: &Transaction, + manifest: &Manifest, + location: &ManifestLocation, + indices: Vec, + skip_auto_cleanup: bool, +) { + let tx_key = crate::session::caches::TransactionKey { + version: manifest.version, + }; + dataset + .metadata_cache + .insert_with_key(&tx_key, Arc::new(transaction.clone())) + .await; + + let manifest_key = crate::session::caches::ManifestKey { + version: location.version, + e_tag: location.e_tag.as_deref(), + }; + dataset + .metadata_cache + .insert_with_key(&manifest_key, Arc::new(manifest.clone())) + .await; + if !indices.is_empty() { + let key = IndexMetadataKey { + version: manifest.version, + store_identity: &dataset.object_store.store_prefix, + e_tag: location.e_tag.as_deref(), + }; + dataset + .index_cache + .insert_with_key(&key, Arc::new(indices)) + .await; + } + + if !skip_auto_cleanup { + // Note: We're using the old dataset here (before the new manifest is committed). + // This means cleanup runs based on the previous version's state, which may affect + // which versions are available for cleanup. + match auto_cleanup_hook(dataset, manifest).await { + Ok(Some(stats)) => log::info!("Auto cleanup triggered: {:?}", stats), + Err(e) => log::error!("Error encountered during auto_cleanup_hook: {}", e), + _ => {} + }; + } +} + /// Attempt to commit a transaction, with retries and conflict resolution. #[allow(clippy::too_many_arguments)] pub(crate) async fn commit_transaction( @@ -919,6 +1397,7 @@ pub(crate) async fn commit_transaction( transaction: &Transaction, write_config: &ManifestWriteConfig, commit_config: &CommitConfig, + retry_timeout: Duration, manifest_naming_scheme: ManifestNamingScheme, affected_rows: Option<&RowAddrTreeMap>, ) -> Result<(Manifest, ManifestLocation)> { @@ -943,6 +1422,20 @@ pub(crate) async fn commit_transaction( dataset.clone() }; + // The version this transaction read, captured before the retry loop moves + // `dataset` forward. MemWAL index catch-up is derived from it: an index + // covering every fragment live here holds every row compaction had copied + // in by then. + // + // The Arc is kept rather than cloned out: `load_all_indices` returns shared + // cached data, so the common case is a cache hit rather than a read. + let read_version_dataset = dataset.clone(); + let read_version_indices = load_all_indices(&read_version_dataset).await?; + let read_version_state = Some(crate::dataset::transaction::ReadVersionState { + manifest: read_version_dataset.manifest.as_ref(), + indices: read_version_indices.as_slice(), + }); + let mut transaction = transaction.clone(); let num_attempts = std::cmp::max(commit_config.num_retries, 1); @@ -967,6 +1460,8 @@ pub(crate) async fn commit_transaction( if !strict_overwrite { (dataset, other_transactions) = load_and_sort_new_transactions(&dataset).await?; + ensure_can_write_manifest(&dataset.manifest)?; + // See if we can retry the commit. Try to account for all // transactions that have been committed since the read_version. // Use small amount of backoff to handle transactions that all @@ -980,10 +1475,20 @@ pub(crate) async fn commit_transaction( } transaction = rebase.finish(&dataset).await?; + } else { + ensure_can_write_manifest(&dataset.manifest)?; } + // Recomputed every attempt: the rebase above may have rewritten the + // transaction. + let pb_transaction = pb::Transaction::from(&transaction); + let inline_transaction = pb_transaction.encoded_len() <= MAX_INLINE_TRANSACTION_BYTES; + // Classified from the operation itself. Reading it back off the inline + // copy would tie the verdict to the payload size instead. + let may_change_schema = operation_may_change_schema(&pb_transaction); + current_transaction_file = if !write_config.disable_transaction_file() { - write_transaction_file(object_store, &dataset.base, &transaction).await? + write_transaction_file(object_store, &dataset.base, &pb_transaction).await? } else { String::new() }; @@ -1003,17 +1508,18 @@ pub(crate) async fn commit_transaction( commit_handler, &dataset.base, version, - write_config, + &write_config.to_build_config(), transaction_file, &dataset.manifest, ) .await? } - _ => transaction.build_manifest( + _ => transaction.build_manifest_with_read_version( Some(dataset.manifest.as_ref()), - dataset.load_indices().await?.as_ref().clone(), + load_all_indices(&dataset).await?.as_ref().clone(), transaction_file, - write_config, + &write_config.to_build_config(), + read_version_state, )?, }; @@ -1032,8 +1538,16 @@ pub(crate) async fn commit_transaction( check_storage_version(&mut manifest)?; check_column_indices(&manifest)?; + check_fragment_ids(&manifest)?; - migrate_indices(&dataset, &mut indices).await?; + // Runs after the coverage derivation and can replace a fragment bitmap + // while keeping its UUID, so anything it narrowed loses its position. + let recovered_coverage = migrate_indices(&dataset, &mut indices).await?; + Transaction::withdraw_coverage_invalidated_after_build( + &mut indices, + &recovered_coverage, + target_version, + )?; // Try to commit the manifest let result = write_manifest_file( @@ -1048,52 +1562,71 @@ pub(crate) async fn commit_transaction( }, write_config, manifest_naming_scheme, - Some(&transaction), + inline_transaction.then(|| pb_transaction.into()), + may_change_schema, ) .await; match result { Ok(manifest_location) => { - // Cache both the transaction file and manifest - let tx_key = crate::session::caches::TransactionKey { - version: target_version, - }; - dataset - .metadata_cache - .insert_with_key(&tx_key, Arc::new(transaction.clone())) - .await; - - let manifest_key = crate::session::caches::ManifestKey { - version: manifest_location.version, - e_tag: manifest_location.e_tag.as_deref(), - }; - dataset - .metadata_cache - .insert_with_key(&manifest_key, Arc::new(manifest.clone())) - .await; - if !indices.is_empty() { - let key = IndexMetadataKey { - version: target_version, - }; - dataset - .index_cache - .insert_with_key(&key, Arc::new(indices)) - .await; - } - - if !commit_config.skip_auto_cleanup { - // Note: We're using the old dataset here (before the new manifest is committed). - // This means cleanup runs based on the previous version's state, which may affect - // which versions are available for cleanup. - match auto_cleanup_hook(&dataset, &manifest).await { - Ok(Some(stats)) => log::info!("Auto cleanup triggered: {:?}", stats), - Err(e) => log::error!("Error encountered during auto_cleanup_hook: {}", e), - _ => {} - }; - } + record_successful_commit( + &dataset, + &transaction, + &manifest, + &manifest_location, + indices, + commit_config.skip_auto_cleanup, + ) + .await; return Ok((manifest, manifest_location)); } Err(CommitError::CommitConflict) => { + // The store may have applied this attempt's write and still + // reported a conflict (e.g. the response to a successful + // conditional PUT was lost and an internal retry saw + // "already exists"). Verify who owns the version before + // treating the attempt as lost: deleting the artifacts of a + // commit that actually landed corrupts the version. + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + target_version, + &transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_successful_commit( + &dataset, + &transaction, + &committed_manifest, + &location, + indices, + commit_config.skip_auto_cleanup, + ) + .await; + return Ok((committed_manifest, location)); + } + // Confirmed loss: another writer owns the version (or, + // for handlers that detect conflicts before writing, + // the attempt never landed). Proceed with the normal + // rebase-and-retry path. + CommitOutcome::Foreign | CommitOutcome::Absent => {} + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + target_version, + "commit reported a conflict but the manifest at the target \ + version could not be read back for verification" + .to_string() + .into(), + )); + } + } let next_attempt_i = backoff.attempt() + 1; if backoff.attempt() == 0 { @@ -1113,16 +1646,63 @@ pub(crate) async fn commit_transaction( ¤t_transaction_file, ) .await; - tokio::time::sleep(backoff.next_backoff()).await; + if start.elapsed() > retry_timeout { + return Err(timeout_error(retry_timeout, backoff.attempt() + 1)); + } + let sleep_fut = tokio::time::sleep(backoff.next_backoff()); + maybe_timeout(backoff.attempt(), start, retry_timeout, sleep_fut).await?; continue; } else { break; } } Err(CommitError::OtherError(err)) => { - cleanup_transaction_file(object_store, &dataset.base, ¤t_transaction_file) - .await; - return Err(err); + match verify_commit_outcome( + object_store, + commit_handler, + &dataset.base, + target_version, + &transaction, + ) + .await + { + CommitOutcome::Ours { + manifest: committed_manifest, + location, + } => { + let committed_manifest = *committed_manifest; + record_successful_commit( + &dataset, + &transaction, + &committed_manifest, + &location, + indices, + commit_config.skip_auto_cleanup, + ) + .await; + if commit_handler.propagate_commit_error_after_success() { + return Err(err); + } + return Ok((committed_manifest, location)); + } + CommitOutcome::Foreign | CommitOutcome::Absent => { + // The attempt certainly did not land; its + // transaction file is orphaned. + cleanup_transaction_file( + object_store, + &dataset.base, + ¤t_transaction_file, + ) + .await; + return Err(err); + } + CommitOutcome::Unknown => { + return Err(Error::commit_status_unknown_source( + target_version, + Box::new(err), + )); + } + } } } } @@ -1150,11 +1730,13 @@ mod tests { use lance_core::datatypes::{Field, Schema}; use lance_core::utils::tempfile::TempStrDir; use lance_datagen::{BatchCount, RowCount, array, gen_batch}; + use lance_file::version::ConcreteFileVersion; use lance_index::IndexType; use lance_linalg::distance::MetricType; use lance_table::format::{DataFile, DataStorageFormat}; use lance_table::io::commit::{ CommitLease, CommitLock, ManifestWriter, RenameCommitHandler, UnsafeCommitHandler, + commit_handler_from_url, }; use lance_testing::datagen::generate_random_array; @@ -1162,6 +1744,7 @@ mod tests { use crate::Dataset; use crate::dataset::{WriteMode, WriteParams}; + use crate::index::DatasetIndexExt; use crate::index::vector::VectorIndexParams; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; @@ -1302,9 +1885,13 @@ mod tests { Some("hello world".to_string()), ); - let file_name = write_transaction_file(&object_store, &base_path, &transaction) - .await - .unwrap(); + let file_name = write_transaction_file( + &object_store, + &base_path, + &pb::Transaction::from(&transaction), + ) + .await + .unwrap(); let read_transaction = read_transaction_file(&object_store, &base_path, &file_name) .await .unwrap(); @@ -1410,18 +1997,129 @@ mod tests { assert!(dataset.checkout_version(4).await.is_err()); } + /// Every commit runs `migrate_indices`, and recalculating a missing + /// `fragment_bitmap` there means opening the index. An index this build + /// cannot open must not take the write path down with it: the dataset would + /// be unwritable, not merely unreadable, and every later commit would fail + /// the same way. #[tokio::test] - async fn test_load_and_sort_new_transactions() { - // Create a dataset - let mut dataset = lance_datagen::gen_batch() - .col("i", lance_datagen::array::step::()) - .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(10)) - .await - .unwrap(); + async fn test_commit_survives_an_index_it_cannot_open() { + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + use lance_table::io::manifest::read_manifest_indexes; - // Create 100 small UpdateConfig transactions - for i in 0..100 { - dataset + let test_dir = TempStrDir::default(); + let test_uri = test_dir.as_str(); + + let reader = gen_batch() + .col("id", array::step::()) + .col("payload", array::step::()) + .into_reader_rows(RowCount::from(10), BatchCount::from(1)); + let mut dataset = Dataset::write(reader, test_uri, None).await.unwrap(); + + // The readable companion is what makes the difference visible: with a + // single index, "carried through the one it cannot open" and "stopped + // recalculating altogether" answer every assertion below the same way. + let btree_params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + for column in ["id", "payload"] { + dataset + .create_index_builder(&[column], IndexType::BTree, &btree_params) + .name(format!("{column}_idx")) + .await + .unwrap(); + } + + let broken = dataset.load_index_by_name("id_idx").await.unwrap().unwrap(); + dataset + .object_store + .remove_dir_all(dataset.indices_dir().join(broken.uuid.to_string())) + .await + .unwrap(); + + // Reopened so the fixture is judged on what is on disk rather than on + // what this process still holds from building the index. + let mut dataset = Dataset::open(test_uri).await.unwrap(); + assert!( + dataset + .open_generic_index("id", &broken.uuid, &NoOpMetricsCollector) + .await + .is_err(), + "the fixture is supposed to leave an index this build cannot open" + ); + + // Migration recalculates a bitmap that is missing, and no current writer + // emits one - untrained indices get an empty bitmap, not none at all - so + // the state an old manifest arrives in is set here by hand. + let indices = read_manifest_indexes( + &dataset.object_store, + &dataset.manifest_location, + &dataset.manifest, + ) + .await + .unwrap(); + let without_bitmaps = indices + .iter() + .map(|index| IndexMetadata { + fragment_bitmap: None, + ..index.clone() + }) + .collect::>(); + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: without_bitmaps, + removed_indices: indices, + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + // And an unrelated commit after it, since the missing bitmap is now what + // the manifest holds and migration retries on every commit. + dataset.delete("false").await.unwrap(); + + let migrated = read_manifest_indexes( + &dataset.object_store, + &dataset.manifest_location, + &dataset.manifest, + ) + .await + .unwrap(); + let coverage = |name: &str| { + migrated + .iter() + .find(|index| index.name == name) + .unwrap_or_else(|| panic!("no index named {name} in the manifest")) + .fragment_bitmap + .as_ref() + .map(|bitmap| bitmap.iter().collect::>()) + }; + assert_eq!( + coverage("id_idx"), + None, + "an index that cannot be opened must report unknown coverage" + ); + assert_eq!( + coverage("payload_idx"), + Some(vec![0]), + "an index that opens must still have its coverage recalculated" + ); + } + + #[tokio::test] + async fn test_load_and_sort_new_transactions() { + // Create a dataset + let mut dataset = lance_datagen::gen_batch() + .col("i", lance_datagen::array::step::()) + .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(10)) + .await + .unwrap(); + + // Create 100 small UpdateConfig transactions + for i in 0..100 { + dataset .update_config(vec![(format!("key_{}", i), format!("value_{}", i))]) .await .unwrap(); @@ -1691,6 +2389,7 @@ mod tests { DataFile::new_legacy_from_fields("path1", vec![0, 1, 2], None), DataFile::new_legacy_from_fields("unused", vec![9], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1703,6 +2402,7 @@ mod tests { DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None), DataFile::new_legacy_from_fields("path3", vec![2], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1740,6 +2440,7 @@ mod tests { vec![0, 1, 10], None, )], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1752,6 +2453,7 @@ mod tests { DataFile::new_legacy_from_fields("path2", vec![0, 1, 2], None), DataFile::new_legacy_from_fields("path3", vec![10], None), ], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: None, @@ -1770,6 +2472,10 @@ mod tests { #[async_trait::async_trait] impl CommitHandler for FailingCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + true + } + async fn commit( &self, _manifest: &mut Manifest, @@ -1833,6 +2539,372 @@ mod tests { extra = txn_files_after.saturating_sub(txn_files_before), ); } + + #[tokio::test] + async fn test_cos_commit_failure_preserves_error_and_cleans_up_transaction() { + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let batch = simple_batch(&schema, vec![1, 2, 3]); + let commit_handler = commit_handler_from_url("cos://bucket/dataset", &None) + .await + .unwrap(); + let params = WriteParams { + commit_handler: Some(commit_handler), + ..Default::default() + }; + + let error = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(params), + ) + .await + .expect_err("the default Tencent COS handler should reject writes"); + + assert!( + matches!(&error, Error::NotSupported { .. }), + "expected NotSupported, got: {error:?}" + ); + assert!( + error.to_string().contains("distributed commit_lock"), + "unexpected error: {error}" + ); + assert_eq!( + count_txn_files(uri), + 0, + "a definitively failed COS commit must clean up its transaction file" + ); + } + + fn simple_batch(schema: &Arc, values: Vec) -> RecordBatch { + RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(values))]).unwrap() + } + + fn simple_schema() -> Arc { + Arc::new(ArrowSchema::new(vec![ArrowField::new( + "x", + DataType::Int32, + false, + )])) + } + + /// A commit whose manifest lands but is reported as a conflict (the + /// incident shape: successful conditional PUT, response lost, internal + /// retry sees "already exists") must be recognized as our own commit and + /// returned as success — with the rows appearing exactly once and the + /// transaction file left in place. + #[tokio::test] + async fn test_commit_succeeds_when_conflict_is_own_commit() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + assert_eq!( + handler.resolve_calls(), + 0, + "an uncontended commit must not perform verification reads" + ); + let txn_files_before = count_txn_files(uri); + + handler.fail_next(AmbiguousFailure::LandAndConflict); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let ds = Dataset::write(reader, uri, Some(params)) + .await + .expect("a conflict with our own landed commit must be reported as success"); + + assert_eq!(ds.version().version, 2); + assert_eq!( + ds.count_rows(None).await.unwrap(), + 6, + "rows must appear exactly once (no duplicate re-commit)" + ); + assert_eq!( + count_txn_files(uri), + txn_files_before + 1, + "the landed commit's transaction file is referenced by the manifest and must survive" + ); + + // A fresh reader sees the committed version. + let ds2 = Dataset::open(uri).await.unwrap(); + assert_eq!(ds2.version().version, 2); + assert_eq!(ds2.count_rows(None).await.unwrap(), 6); + } + + /// Same as above, but the landed commit is reported as a plain I/O error + /// (e.g. the store's retries all returned 5xx while the first attempt had + /// landed). Verification must still recognize the commit as ours. + #[tokio::test] + async fn test_commit_succeeds_when_landed_with_other_error() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + let txn_files_before = count_txn_files(uri); + + handler.fail_next(AmbiguousFailure::LandAndError); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let ds = Dataset::write(reader, uri, Some(params)) + .await + .expect("an errored commit that actually landed must be reported as success"); + + assert_eq!(ds.version().version, 2); + assert_eq!(ds.count_rows(None).await.unwrap(), 6); + assert_eq!(count_txn_files(uri), txn_files_before + 1); + } + + #[tokio::test] + async fn test_commit_retries_temporarily_invisible_manifest() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + + handler.fail_next(AmbiguousFailure::LandAndError); + handler.fail_next_resolves_with_not_found(2); + let calls_before = handler.resolve_calls(); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = + RecordBatchIterator::new(vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], schema); + let dataset = Dataset::write(reader, uri, Some(params)) + .await + .expect("verification must retry a non-definitive NotFound result"); + + assert_eq!(handler.resolve_calls() - calls_before, 3); + assert_eq!(dataset.count_rows(None).await.unwrap(), 6); + } + + #[tokio::test] + async fn test_commit_verifies_inline_transaction_without_transaction_file() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = + RecordBatchIterator::new(vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], schema); + let dataset = Dataset::write(reader, uri, Some(params)).await.unwrap(); + let transaction = Transaction::new( + dataset.version().version, + Operation::Append { fragments: vec![] }, + None, + ); + let write_config = ManifestWriteConfig::default().with_transaction_file_disabled(); + + handler.fail_next(AmbiguousFailure::LandAndConflict); + let (manifest, _) = commit_transaction( + &dataset, + dataset.object_store.as_ref(), + handler.as_ref(), + &transaction, + &write_config, + &CommitConfig::default(), + DEFAULT_COMMIT_RETRY_TIMEOUT, + dataset.manifest_location.naming_scheme, + None, + ) + .await + .expect("the inline transaction must identify the landed commit"); + + assert_eq!(manifest.version, 2); + assert!(manifest.transaction_file.is_none()); + assert!(manifest.transaction_section.is_some()); + } + + /// A commit that errors without landing keeps today's behavior: + /// verification finds no manifest at the target version, the original + /// error propagates (not status-unknown), and the orphaned transaction + /// file is cleaned up. + #[tokio::test] + async fn test_commit_definite_failure_cleans_up_and_keeps_error() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + let txn_files_before = count_txn_files(uri); + + handler.fail_next(AmbiguousFailure::FailOutright); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let result = Dataset::write(reader, uri, Some(params)).await; + let err = result.expect_err("commit that did not land must fail"); + assert!( + !err.is_commit_status_unknown(), + "a verified-absent commit is a definite failure, got: {:?}", + err + ); + assert_eq!( + count_txn_files(uri), + txn_files_before, + "orphaned transaction file of a definitely-failed commit must be cleaned up" + ); + } + + /// When the commit errors AND verification itself is unavailable, the + /// commit status is unknown: surface `CommitStatusUnknown` and delete + /// nothing. + #[tokio::test] + async fn test_commit_status_unknown_when_verification_unavailable() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + let txn_files_before = count_txn_files(uri); + + // The commit lands but errors, and verification reads fail too. + handler.fail_next(AmbiguousFailure::LandAndError); + handler + .fail_resolve + .store(true, std::sync::atomic::Ordering::SeqCst); + let params = WriteParams { + mode: WriteMode::Append, + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![4, 5, 6]))], + schema.clone(), + ); + let result = Dataset::write(reader, uri, Some(params)).await; + let err = result.expect_err("unknown status must not be reported as success"); + assert!( + err.is_commit_status_unknown(), + "expected CommitStatusUnknown, got: {:?}", + err + ); + assert_eq!( + count_txn_files(uri), + txn_files_before + 1, + "nothing may be deleted while the commit status is unknown" + ); + + // The commit did land: a fresh reader must see a consistent v2. + handler + .fail_resolve + .store(false, std::sync::atomic::Ordering::SeqCst); + let ds = Dataset::open(uri).await.unwrap(); + assert_eq!(ds.version().version, 2); + assert_eq!(ds.count_rows(None).await.unwrap(), 6); + } + + /// Dataset creation whose manifest lands but is reported as a conflict + /// must succeed instead of returning "dataset already exists". + #[tokio::test] + async fn test_create_dataset_succeeds_when_conflict_is_own_commit() { + use crate::utils::test::{AmbiguousCommitHandler, AmbiguousFailure}; + + let tmp = TempStrDir::default(); + let uri = tmp.as_str(); + let schema = simple_schema(); + let handler = Arc::new(AmbiguousCommitHandler::default()); + handler.fail_next(AmbiguousFailure::LandAndConflict); + + let params = WriteParams { + commit_handler: Some(handler.clone()), + ..Default::default() + }; + let reader = RecordBatchIterator::new( + vec![Ok(simple_batch(&schema, vec![1, 2, 3]))], + schema.clone(), + ); + let ds = Dataset::write(reader, uri, Some(params)) + .await + .expect("creation whose commit landed must succeed"); + assert_eq!(ds.version().version, 1); + assert_eq!(ds.count_rows(None).await.unwrap(), 3); + } + /// Helper to build a simple manifest for check_column_indices tests. fn make_manifest_with_file( schema: Schema, @@ -1842,6 +2914,7 @@ mod tests { let fragment = Fragment { id: 0, files: vec![data_file], + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(100), @@ -1851,7 +2924,7 @@ mod tests { Manifest::new( schema, Arc::new(vec![fragment]), - DataStorageFormat::new(data_storage_version), + DataStorageFormat::new(data_storage_version.resolve()), HashMap::new(), ) } @@ -1873,7 +2946,14 @@ mod tests { }; // field ids: struct=0, leaf=1; give struct a real column_index (wrong) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!( @@ -1901,7 +2981,14 @@ mod tests { }; // field ids: list=0, item=1; give list a real column_index (wrong) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!( @@ -1929,7 +3016,14 @@ mod tests { }; // struct=-1 (correct), leaf=0 (correct) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![-1, 0], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![-1, 0], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); assert!(check_column_indices(&manifest).is_ok()); } @@ -1954,7 +3048,14 @@ mod tests { }; // packed struct=0 (allowed), leaf=1 - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); assert!(check_column_indices(&manifest).is_ok()); } @@ -1975,7 +3076,14 @@ mod tests { metadata: Default::default(), }; - let data_file = DataFile::new("data.lance", vec![0, 1], vec![0, 1], 2, 0, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_0, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_0); assert!(check_column_indices(&manifest).is_ok()); } @@ -1992,7 +3100,14 @@ mod tests { }; // 1 field id but 2 column indices - let data_file = DataFile::new("data.lance", vec![0], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!(result.is_err(), "Expected error for mismatched lengths"); @@ -2012,7 +3127,14 @@ mod tests { }; // field id 99 does not exist in the schema — should be skipped - let data_file = DataFile::new("data.lance", vec![0, 99], vec![0, 1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 99], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); assert!(check_column_indices(&manifest).is_ok()); } @@ -2034,7 +3156,14 @@ mod tests { }; // struct=-1 (correct), but leaf=-1 (wrong — leaf must have a real column) - let data_file = DataFile::new("data.lance", vec![0, 1], vec![-1, -1], 2, 1, None, None); + let data_file = DataFile::new( + "data.lance", + vec![0, 1], + vec![-1, -1], + ConcreteFileVersion::V2_1, + None, + None, + ); let manifest = make_manifest_with_file(schema, data_file, LanceFileVersion::V2_1); let result = check_column_indices(&manifest); assert!( @@ -2044,4 +3173,156 @@ mod tests { let msg = result.unwrap_err().to_string(); assert!(msg.contains("must have a valid column index"), "{msg}"); } + + #[test] + fn test_check_column_indices_rejects_after_dedup() { + let mut struct_field = Field::try_from(ArrowField::new( + "s", + DataType::Struct(vec![ArrowField::new("x", DataType::Int32, false)].into()), + false, + )) + .unwrap(); + struct_field.set_id(-1, &mut 0); + + let schema = Schema { + fields: vec![struct_field], + metadata: Default::default(), + }; + + // struct=-1, leaf=0: valid layout; clones share the same Arcs. + let shared_file = DataFile::new( + "shared.lance", + vec![0, 1], + vec![-1, 0], + ConcreteFileVersion::V2_1, + None, + None, + ); + // Wrongly gives the struct a real column index. + let bad_file = DataFile::new( + "bad.lance", + vec![0, 1], + vec![0, 1], + ConcreteFileVersion::V2_1, + None, + None, + ); + let make_fragment = |id: u64, file: DataFile| Fragment { + id, + files: vec![file], + overlays: vec![], + deletion_file: None, + row_id_meta: None, + physical_rows: Some(100), + last_updated_at_version_meta: None, + created_at_version_meta: None, + }; + let manifest = Manifest::new( + schema, + Arc::new(vec![ + make_fragment(0, shared_file.clone()), + make_fragment(1, shared_file), + make_fragment(2, bad_file), + ]), + DataStorageFormat::new(LanceFileVersion::V2_1.resolve()), + HashMap::new(), + ); + let msg = check_column_indices(&manifest).unwrap_err().to_string(); + assert!(msg.contains("Non-leaf field"), "{msg}"); + assert!(msg.contains("bad.lance"), "{msg}"); + } + + /// Reproduces the debug-only panic `migrate_indices`'s fragment-bitmap + /// recalculation guard used to contain: a legal covered index + /// (`fields=[a,b]`, `covering_fields=[b]`) has `fields.len() == 2`, which + /// the old `debug_assert_eq!(index.fields.len(), 1)` rejected outright even + /// though the following line only ever reads `fields[0]`. + /// `must_recalculate_fragment_bitmap` takes this branch whenever + /// `fragment_bitmap` is `None`, so committing with it unset drives the + /// assert during the index's own commit. + #[tokio::test] + async fn test_covered_index_commit_recalculates_fragment_bitmap_without_panicking() { + use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + + let data = gen_batch() + .col("a", array::step::()) + .col("b", array::step::()) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let mut dataset = Dataset::write(data, "memory://", None).await.unwrap(); + dataset + .create_index( + &["a"], + IndexType::BTree, + None, + &ScalarIndexParams::for_builtin(BuiltinIndexType::BTree), + false, + ) + .await + .unwrap(); + + let b_id = dataset.schema().field("b").unwrap().id; + let current = dataset.load_indices().await.unwrap(); + let mut covered = current[0].clone(); + covered.fields.push(b_id); + covered.covering_fields = vec![b_id]; + // Force the fragment-bitmap recalculation branch this guard sits in. + covered.fragment_bitmap = None; + + let transaction = Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![covered], + removed_indices: current.to_vec(), + }, + None, + ); + dataset + .apply_commit(transaction, &Default::default(), &Default::default()) + .await + .unwrap(); + + let recomputed = dataset.load_indices().await.unwrap(); + assert_eq!(recomputed.len(), 1); + assert!( + recomputed[0].fragment_bitmap.is_some(), + "migrate_indices should have recalculated the fragment bitmap for the covered index" + ); + } + + fn index_segment(name: &str, fragment_bitmap: Option) -> IndexMetadata { + IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: name.to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + } + } + + #[test] + fn test_detect_overlapping_fragments() { + let indices = vec![ + index_segment("idx_a", Some(RoaringBitmap::from_iter(0..5))), + index_segment("idx_a", Some(RoaringBitmap::from_iter([3, 4, 10]))), + index_segment("idx_a", None), + index_segment("idx_b", Some(RoaringBitmap::from_iter(0..5))), + ]; + let err = detect_overlapping_fragments(&indices).unwrap_err(); + assert_eq!(err.bad_indices.len(), 1); + let (name, overlapping) = &err.bad_indices[0]; + assert_eq!(name, "idx_a"); + assert_eq!(overlapping, &vec![3, 4]); + + let disjoint = vec![ + index_segment("idx_a", Some(RoaringBitmap::from_iter(0..5))), + index_segment("idx_a", Some(RoaringBitmap::from_iter(5..10))), + ]; + assert!(detect_overlapping_fragments(&disjoint).is_ok()); + } } diff --git a/rust/lance/src/io/commit/conflict_resolver.rs b/rust/lance/src/io/commit/conflict_resolver.rs index d95821dd130..4a824058640 100644 --- a/rust/lance/src/io/commit/conflict_resolver.rs +++ b/rust/lance/src/io/commit/conflict_resolver.rs @@ -7,15 +7,17 @@ use crate::index::mem_wal::{load_mem_wal_index_details, new_mem_wal_index_meta}; use crate::io::deletion::read_dataset_deletion_file; use crate::{ Dataset, - dataset::transaction::{Operation, Transaction, UpdateMode}, + dataset::transaction::{DataOverlayGroup, Operation, Transaction, UpdateMode}, }; use futures::{StreamExt, TryStreamExt}; use lance_core::{Error, Result, utils::deletion::DeletionVector}; use lance_index::frag_reuse::FRAG_REUSE_INDEX_NAME; -use lance_index::mem_wal::{MEM_WAL_INDEX_NAME, MergedGeneration}; +use lance_index::mem_wal::{CompactedSsTable, MEM_WAL_INDEX_NAME}; use lance_select::{RowAddrTreeMap, RowSetOps}; use lance_table::format::IndexMetadata; +use lance_table::format::overlay::OverlayCoverage; use lance_table::{format::Fragment, io::deletion::write_deletion_file}; +use roaring::RoaringBitmap; use std::{ borrow::Cow, collections::{HashMap, HashSet}, @@ -32,9 +34,63 @@ pub struct TransactionRebase<'a> { modified_fragment_ids: HashSet, affected_rows: Option<&'a RowAddrTreeMap>, conflicting_frag_reuse_indices: Vec, - /// Merged generations from conflicting UpdateMemWalState transactions. + /// Compacted SSTables from conflicting UpdateMemWalState transactions. /// Used when rebasing CreateIndex of MemWalIndex. - conflicting_mem_wal_merged_gens: Vec, + conflicting_mem_wal_compacted_sstables: Vec, +} + +/// Whether `operation` may make a nullability-affecting schema change: a +/// projection or merge that does not assert `preserves_nullability`. A +/// tightening projection scanned for nulls at its read version, and a merge +/// may introduce a field that data staged against an earlier schema cannot +/// safely omit; either is falsified by a concurrent value-write. +fn may_alter_nullability(operation: &Operation) -> bool { + matches!( + operation, + Operation::Project { + preserves_nullability: false, + .. + } | Operation::Merge { + preserves_nullability: false, + .. + } + ) +} + +/// Whether `operation` can commit rows that falsify such a change: by writing +/// a null into a scanned field, or by omitting a required column entirely (a +/// stale append's fragments read as null for columns they predate). `Delete` +/// only removes rows and `Rewrite` preserves the values it moves; `Project` +/// already conflicts with projections and merges elsewhere. +fn supplies_values(operation: &Operation) -> bool { + matches!( + operation, + Operation::Append { .. } + | Operation::Update { .. } + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } + ) +} + +/// Whether an operation changes schema-level or per-field metadata. A merge +/// carries the complete schema from its read version, so rebasing either +/// operation over the other can discard one side's metadata changes. +fn updates_schema_or_field_metadata(operation: &Operation) -> bool { + let Operation::UpdateConfig { + schema_metadata_updates, + field_metadata_updates, + .. + } = operation + else { + return false; + }; + + schema_metadata_updates + .as_ref() + .is_some_and(|update| update.replace || !update.update_entries.is_empty()) + || field_metadata_updates + .values() + .any(|update| update.replace || !update.update_entries.is_empty()) } impl<'a> TransactionRebase<'a> { @@ -60,7 +116,7 @@ impl<'a> TransactionRebase<'a> { initial_fragments: HashMap::new(), modified_fragment_ids: HashSet::new(), conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }), Operation::Delete { updated_fragments, @@ -88,20 +144,20 @@ impl<'a> TransactionRebase<'a> { modified_fragment_ids, affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }); } let initial_fragments = initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids) - .await; + .await?; Ok(Self { transaction, affected_rows, initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } Operation::Rewrite { groups, .. } => { @@ -112,14 +168,14 @@ impl<'a> TransactionRebase<'a> { let initial_fragments = initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids) - .await; + .await?; Ok(Self { transaction, affected_rows, initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } Operation::DataReplacement { replacements } => { @@ -127,28 +183,43 @@ impl<'a> TransactionRebase<'a> { replacements.iter().map(|r| r.0).collect::>(); let initial_fragments = initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids) - .await; + .await?; Ok(Self { transaction, affected_rows, initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }) + } + Operation::DataOverlay { groups } => { + let modified_fragment_ids = + groups.iter().map(|g| g.fragment_id).collect::>(); + let initial_fragments = + initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids) + .await?; + Ok(Self { + transaction, + affected_rows, + initial_fragments, + modified_fragment_ids, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } Operation::Merge { fragments, .. } => { let modified_fragment_ids = fragments.iter().map(|f| f.id).collect::>(); let initial_fragments = initial_fragments_for_rebase(dataset, &transaction, &modified_fragment_ids) - .await; + .await?; Ok(Self { transaction, affected_rows, initial_fragments, modified_fragment_ids, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }) } } @@ -197,6 +268,23 @@ impl<'a> TransactionRebase<'a> { ) } + #[track_caller] + fn data_replacement_field_removed_err( + &self, + field_id: i32, + fragment_id: u64, + other_transaction: &Transaction, + other_version: u64, + ) -> Error { + Error::incompatible_transaction_source( + format!( + "DataReplacement target field {} in fragment {} was dropped by concurrent {} at version {}.", + field_id, fragment_id, other_transaction.operation, other_version + ) + .into(), + ) + } + /// Check whether the transaction conflicts with another transaction. /// Mutate the current [TransactionRebase] based on `other_transaction` to be used for /// eventually finishing the rebase process. @@ -204,6 +292,23 @@ impl<'a> TransactionRebase<'a> { /// Will return an error if the transaction is not valid. Otherwise, it will /// return Ok(()). pub fn check_txn(&mut self, other_transaction: &Transaction, other_version: u64) -> Result<()> { + // Either order: the claim was checked without the write's data. + let ours = &self.transaction.operation; + let theirs = &other_transaction.operation; + if (may_alter_nullability(ours) && supplies_values(theirs)) + || (supplies_values(ours) && may_alter_nullability(theirs)) + { + return Err(self.retryable_conflict_err(other_transaction, other_version)); + } + // Merge carries a complete schema from its read version. In either + // commit order, rebasing it with a metadata update can keep one side's + // schema while discarding metadata from the other side. + if (matches!(ours, Operation::Merge { .. }) && updates_schema_or_field_metadata(theirs)) + || (updates_schema_or_field_metadata(ours) && matches!(theirs, Operation::Merge { .. })) + { + return Err(self.retryable_conflict_err(other_transaction, other_version)); + } + let op = &self.transaction.operation; match op { Operation::Delete { .. } => self.check_delete_txn(other_transaction, other_version), @@ -219,6 +324,9 @@ impl<'a> TransactionRebase<'a> { Operation::DataReplacement { .. } => { self.check_data_replacement_txn(other_transaction, other_version) } + Operation::DataOverlay { .. } => { + self.check_data_overlay_txn(other_transaction, other_version) + } Operation::Merge { .. } => self.check_merge_txn(other_transaction, other_version), Operation::Restore { .. } => self.check_restore_txn(other_transaction, other_version), Operation::ReserveFragments { .. } => { @@ -251,6 +359,10 @@ impl<'a> TransactionRebase<'a> { | Operation::Project { .. } | Operation::Append { .. } | Operation::UpdateConfig { .. } + // A concurrent overlay is inert against the rows we delete + // (deletions take precedence over overlays) and otherwise + // preserves physical offsets, so it never conflicts. + | Operation::DataOverlay { .. } | Operation::UpdateBases { .. } => Ok(()), Operation::Rewrite { groups, .. } => { if groups @@ -345,7 +457,11 @@ impl<'a> TransactionRebase<'a> { ) -> Result<()> { if let Operation::Update { inserted_rows_filter: self_inserted_rows_filter, - merged_generations: self_merged_generations, + compacted_sstables: self_compacted_sstables, + new_fragments: self_new_fragments, + update_mode: self_update_mode, + updated_fragments: self_updated_fragments, + fields_modified: self_fields_modified, .. } = &self.transaction.operation { @@ -399,6 +515,79 @@ impl<'a> TransactionRebase<'a> { | Operation::Clone { .. } | Operation::UpdateConfig { .. } | Operation::UpdateBases { .. } => Ok(()), + Operation::DataOverlay { groups } => { + // Our update recomputed rows from the pre-overlay base, so if + // it commits over an overlay it would silently undo the + // overlay's values for any cell it recomputed. + // + // An in-place column rewrite (RewriteColumns) writes a + // replacement file covering *every* row of each fragment it + // touches, filled from the snapshot it read. `build_manifest` + // then tombstones every overlay for the fields it rewrote, so + // an overlay value on a row this update never matched is + // dropped and the stale copied value becomes visible. Retry + // whenever the overlay touches a fragment we rewrote and a + // field we rewrote; the retry re-reads the overlaid values. + if matches!(self_update_mode, Some(UpdateMode::RewriteColumns)) { + let rewritten: HashSet = self_updated_fragments + .iter() + .map(|fragment| fragment.id) + .collect(); + for group in groups { + if !rewritten.contains(&group.fragment_id) { + continue; + } + let overlaps_rewritten_field = group.overlays.iter().any(|overlay| { + overlay.data_file.fields.iter().any(|&field| { + field >= 0 && self_fields_modified.contains(&(field as u32)) + }) + }); + if overlaps_rewritten_field { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + } + } + return Ok(()); + } + + // A row-moving update (RewriteRows) relocates the rows it + // touches out to new fragments; only the rows it actually + // moved lose their overlay, so we conflict only when the + // moved rows intersect the overlay's coverage. + let moves_rows = !self_new_fragments.is_empty() + && matches!(self_update_mode, Some(UpdateMode::RewriteRows) | None); + if !moves_rows { + return Ok(()); + } + // `affected_rows` holds the physical offsets (per fragment) + // this update moved. The overlay's coverage is in the same + // physical-offset space, so we can intersect the two in + // memory. Without affected rows we cannot be precise, so we + // fall back to a fragment-granular conflict. + for group in groups { + if !self.modified_fragment_ids.contains(&group.fragment_id) { + continue; + } + let Some(affected_rows) = self.affected_rows else { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + }; + let Some(moved) = + affected_rows.get_fragment_bitmap(group.fragment_id as u32) + else { + continue; + }; + let coverage = overlay_group_coverage(group); + if !(moved & &coverage).is_empty() { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + } + } + Ok(()) + } Operation::Append { .. } => { // If current transaction has primary key conflict detection, // we can't safely commit against an Append because we don't @@ -487,10 +676,11 @@ impl<'a> TransactionRebase<'a> { Err(self.incompatible_conflict_err(other_transaction, other_version)) } Operation::UpdateMemWalState { - merged_generations: other_merged_generations, - } => self.check_merged_generations_conflict( - other_merged_generations, - self_merged_generations, + compacted_sstables: other_compacted_sstables, + .. + } => self.check_compacted_sstables_conflict( + other_compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ), @@ -509,11 +699,15 @@ impl<'a> TransactionRebase<'a> { new_indices, removed_indices, .. - } = &self.transaction.operation + } = &mut self.transaction.operation { match &other_transaction.operation { Operation::Append { .. } | Operation::Clone { .. } + // An overlay committed after this index's version is newer than + // the index; the query path excludes its covered cells via the + // version gate, so the build does not conflict. + | Operation::DataOverlay { .. } | Operation::UpdateBases { .. } => Ok(()), Operation::CreateIndex { new_indices: created_indices, @@ -552,9 +746,29 @@ impl<'a> TransactionRebase<'a> { } // Although some of the rows we indexed may have been deleted / moved, // row ids are still valid, so we allow this optimistically. - Operation::Delete { .. } | Operation::Update { .. } => Ok(()), - // Merge, reserve, and project don't change row ids, so this should be fine. - Operation::Merge { .. } => Ok(()), + Operation::Delete { .. } => Ok(()), + Operation::Update { + updated_fragments, + fields_modified, + .. + } => { + Transaction::prune_updated_fields_from_indices( + new_indices, + updated_fragments, + fields_modified, + ); + Ok(()) + } + // Merge, reserve, and project don't change row ids. The MemWAL + // index is the exception: its install validates schema-dependent + // state, which a concurrent schema change invalidates. + Operation::Merge { .. } => { + if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } Operation::ReserveFragments { .. } => Ok(()), Operation::Project { .. } => Ok(()), // Should be compatible with rewrite if it didn't move the rows @@ -571,6 +785,27 @@ impl<'a> TransactionRebase<'a> { // triggers a CreateIndex, and it needs to add the new reuse // version created by the rewrite if let Some(committed_fri) = frag_reuse_index { + let ngram_coverage = new_indices + .iter() + .filter(|idx| { + idx.index_details.as_ref().is_some_and(|details| { + details.type_url.ends_with("NGramIndexDetails") + }) + }) + .filter_map(|idx| idx.fragment_bitmap.as_ref()) + .fold(RoaringBitmap::new(), |coverage, fragments| { + coverage | fragments + }); + if groups + .iter() + .flat_map(|group| group.old_fragments.iter()) + .any(|fragment| ngram_coverage.contains(fragment.id as u32)) + { + return Err( + self.retryable_conflict_err(other_transaction, other_version) + ); + } + if new_indices .iter() .any(|idx| idx.name == FRAG_REUSE_INDEX_NAME) @@ -613,15 +848,16 @@ impl<'a> TransactionRebase<'a> { } Operation::UpdateConfig { .. } => Ok(()), Operation::DataReplacement { replacements } => { - // A data replacement only conflicts if it is updating the field that - // is being indexed. - let newly_indexed_fields = new_indices + // A data replacement only conflicts if it is updating a field the + // index depends on -- whether keyed on or merely carried, since + // `fields` lists both (see `IndexMetadata::covering_fields`). + let newly_depended_fields = new_indices .iter() .flat_map(|idx| idx.fields.iter()) .collect::>(); for replacement in replacements { for field in replacement.1.fields.iter() { - if newly_indexed_fields.contains(&field) { + if newly_depended_fields.contains(&field) { return Err( self.retryable_conflict_err(other_transaction, other_version) ); @@ -631,14 +867,15 @@ impl<'a> TransactionRebase<'a> { Ok(()) } Operation::UpdateMemWalState { - merged_generations: other_merged_gens, + compacted_sstables: other_compacted_sstables, + .. } => { // CreateIndex of MemWalIndex is compatible with UpdateMemWalState // as they can be rebased on each other if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { - // Collect merged_generations from UpdateMemWalState for rebasing - self.conflicting_mem_wal_merged_gens - .extend(other_merged_gens.iter().cloned()); + // Collect compacted_sstables from UpdateMemWalState for rebasing + self.conflicting_mem_wal_compacted_sstables + .extend(other_compacted_sstables.iter().cloned()); Ok(()) } else { Err(self.incompatible_conflict_err(other_transaction, other_version)) @@ -695,6 +932,20 @@ impl<'a> TransactionRebase<'a> { Ok(()) } } + Operation::DataOverlay { groups } => { + // Rewriting a fragment changes its physical row addresses, so + // an overlay addressed by physical offset on that fragment is + // invalidated and must be re-applied against the new base. + if groups + .iter() + .map(|g| g.fragment_id) + .any(|id| self.modified_fragment_ids.contains(&id)) + { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } Operation::Rewrite { groups, frag_reuse_index: committed_fri, @@ -874,6 +1125,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::ReserveFragments { .. } @@ -907,7 +1159,8 @@ impl<'a> TransactionRebase<'a> { | Operation::Merge { .. } | Operation::UpdateConfig { .. } | Operation::Clone { .. } - | Operation::DataReplacement { .. } => Ok(()), + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } => Ok(()), } } @@ -922,8 +1175,28 @@ impl<'a> TransactionRebase<'a> { | Operation::Clone { .. } | Operation::UpdateConfig { .. } | Operation::ReserveFragments { .. } - | Operation::Project { .. } + // Both a column replacement and an overlay preserve physical row + // addresses; the overlay is newer and wins its covered cells. + | Operation::DataOverlay { .. } | Operation::UpdateBases { .. } => Ok(()), + Operation::Project { schema, .. } => { + // A project operation can drop fields. If the project + // dropped a field this operation was replacing then + // we have a conflict. + for replacement in replacements { + for field in replacement.1.fields.iter() { + if *field >= 0 && schema.field_by_id(*field).is_none() { + return Err(self.data_replacement_field_removed_err( + *field, + replacement.0, + other_transaction, + other_version, + )); + } + } + } + Ok(()) + } Operation::Merge { .. } => { // Merge rewrites the whole fragment list; always conflict // (symmetric with check_merge_txn). @@ -987,20 +1260,21 @@ impl<'a> TransactionRebase<'a> { Ok(()) } Operation::CreateIndex { new_indices, .. } => { - // A data replacement only conflicts if it is updating the field that - // is being indexed. + // A data replacement only conflicts if it is updating a field the + // index depends on -- whether keyed on or merely carried, since + // `fields` lists both (see `IndexMetadata::covering_fields`). // // TODO: We could potentially just drop the fragments being replaced from // the index's fragment bitmap, which would lead to fewer conflicts. However // this would introduce fragment bitmaps with holes which may not be well tested // yet. For now, we don't allow this case. - let newly_indexed_fields = new_indices + let newly_depended_fields = new_indices .iter() .flat_map(|idx| idx.fields.iter()) .collect::>(); for replacement in replacements { for field in replacement.1.fields.iter() { - if newly_indexed_fields.contains(&field) { + if newly_depended_fields.contains(&field) { return Err( self.retryable_conflict_err(other_transaction, other_version) ); @@ -1055,14 +1329,134 @@ impl<'a> TransactionRebase<'a> { } } - fn check_merge_txn( + /// Conflict checks for our DataOverlay transaction against a concurrent one. + /// + /// Overlays are intentionally permissive (see the Data Overlay Files spec): + /// they stack with other overlays and tolerate appends, index builds, data + /// replacement, deletes, and in-place column rewrites (Update with + /// `RewriteColumns`), because overlay coverage is addressed by physical offset + /// and the version gate keeps indexes correct. A concurrent operation + /// conflicts when it takes precedence over the overlay for cells the overlay + /// covers, dropping the overlay's values: retryably when it rewrites the + /// physical layout of one of our fragments (Rewrite, Merge) or re-creates the + /// covered rows from the pre-overlay base (a row-moving Update — checked + /// row-by-row in `finish_data_overlay`), or removes an overlaid fragment + /// outright (a Delete / Update that drops the fragment); and incompatibly for + /// whole-dataset replacements (Overwrite / Restore) and MemWAL state updates + /// (UpdateMemWalState), which do not rebase against data operations. + fn check_data_overlay_txn( &mut self, other_transaction: &Transaction, other_version: u64, ) -> Result<()> { match &other_transaction.operation { - Operation::CreateIndex { .. } + Operation::Append { .. } + | Operation::CreateIndex { .. } | Operation::ReserveFragments { .. } + | Operation::Project { .. } + | Operation::UpdateConfig { .. } + | Operation::UpdateBases { .. } + | Operation::Clone { .. } + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } => Ok(()), + // A concurrent Delete only tombstones rows via a deletion vector, + // which preserves physical offsets; the overlay value for a deleted + // offset is simply inert. Conflict only if the whole overlaid + // fragment was removed, orphaning the overlay. + Operation::Delete { + deleted_fragment_ids, + .. + } => { + if deleted_fragment_ids + .iter() + .any(|id| self.modified_fragment_ids.contains(id)) + { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } + // A concurrent Update that removed an overlaid fragment orphans the + // overlay outright — conflict. A row-moving update (RewriteRows) + // deletes the rows it touches and re-creates them in new fragments; + // the update took precedence and the re-created rows were computed + // from the pre-overlay base, so the overlay's values for those cells + // are lost. That is a per-row problem, not an offset one: only the + // moved rows are affected. Comparing the moved rows against the + // overlay's coverage needs the update's deletion vectors, so we mark + // the fragment here and verify row-by-row in `finish_data_overlay`. + // An in-place column rewrite (RewriteColumns) preserves rows and just + // tombstones the overlaid fields at build time, so it never conflicts. + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + update_mode, + .. + } => { + let removed_ours = removed_fragment_ids + .iter() + .any(|id| self.modified_fragment_ids.contains(id)); + if removed_ours { + return Err(self.retryable_conflict_err(other_transaction, other_version)); + } + let moves_rows = !new_fragments.is_empty() + && matches!(update_mode, Some(UpdateMode::RewriteRows) | None); + if moves_rows { + for updated in updated_fragments { + if let Some((_, needs_row_check)) = + self.initial_fragments.get_mut(&updated.id) + { + *needs_row_check = true; + } + } + } + Ok(()) + } + Operation::Rewrite { groups, .. } => { + // A rewrite (compaction / fold) of a fragment we are overlaying + // changes its physical row addresses, so our offsets would be + // invalid. Conflict only if it touches one of our fragments. + let touches_our_fragment = groups + .iter() + .flat_map(|g| g.old_fragments.iter()) + .any(|f| self.modified_fragment_ids.contains(&f.id)); + if touches_our_fragment { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } + Operation::Merge { .. } => { + // Merge rewrites the whole fragment list; always conflict. + Err(self.retryable_conflict_err(other_transaction, other_version)) + } + // Overwrite/Restore replace the dataset; UpdateMemWalState does not + // rebase against data operations (mirroring check_update_mem_wal_state_txn, + // which likewise treats a concurrent DataOverlay as incompatible). + Operation::Overwrite { .. } + | Operation::Restore { .. } + | Operation::UpdateMemWalState { .. } => { + Err(self.incompatible_conflict_err(other_transaction, other_version)) + } + } + } + + fn check_merge_txn( + &mut self, + other_transaction: &Transaction, + other_version: u64, + ) -> Result<()> { + match &other_transaction.operation { + // See the MemWAL exception in check_create_index_txn. + Operation::CreateIndex { new_indices, .. } => { + if new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME) { + Err(self.retryable_conflict_err(other_transaction, other_version)) + } else { + Ok(()) + } + } + Operation::ReserveFragments { .. } | Operation::Clone { .. } | Operation::UpdateConfig { .. } | Operation::UpdateBases { .. } => Ok(()), @@ -1072,7 +1466,8 @@ impl<'a> TransactionRebase<'a> { | Operation::Delete { .. } | Operation::Rewrite { .. } | Operation::Merge { .. } - | Operation::DataReplacement { .. } => { + | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } => { Err(self.retryable_conflict_err(other_transaction, other_version)) } Operation::Overwrite { .. } @@ -1096,6 +1491,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::ReserveFragments { .. } @@ -1124,6 +1520,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::ReserveFragments { .. } | Operation::Update { .. } @@ -1148,6 +1545,7 @@ impl<'a> TransactionRebase<'a> { | Operation::UpdateConfig { .. } | Operation::CreateIndex { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Rewrite { .. } | Operation::Clone { .. } | Operation::ReserveFragments { .. } @@ -1212,6 +1610,7 @@ impl<'a> TransactionRebase<'a> { | Operation::CreateIndex { .. } | Operation::Rewrite { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::ReserveFragments { .. } @@ -1230,45 +1629,49 @@ impl<'a> TransactionRebase<'a> { other_transaction: &Transaction, other_version: u64, ) -> Result<()> { + // Activation rebases like any other MemWAL state update; its preconditions + // are re-checked against the rebased index list when the commit applies. if let Operation::UpdateMemWalState { - merged_generations: self_merged_generations, + compacted_sstables: self_compacted_sstables, + .. } = &self.transaction.operation { match &other_transaction.operation { Operation::UpdateMemWalState { - merged_generations: other_merged_generations, + compacted_sstables: other_compacted_sstables, + .. } => { // Two UpdateMemWalState transactions conflict if they're updating - // the same shard's merged_generation - self.check_merged_generations_conflict( - other_merged_generations, - self_merged_generations, + // the same shard's compacted SSTable + self.check_compacted_sstables_conflict( + other_compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ) } Operation::Update { - merged_generations: other_merged_generations, + compacted_sstables: other_compacted_sstables, .. } => { - // Update transactions with merged_generations can conflict - self.check_merged_generations_conflict( - other_merged_generations, - self_merged_generations, + // Update transactions with compacted_sstables can conflict + self.check_compacted_sstables_conflict( + other_compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ) } Operation::CreateIndex { new_indices, .. } => { - // Check if CreateIndex has a MemWalIndex with merged_generations + // Check if CreateIndex has a MemWalIndex with compacted_sstables if let Some(mem_wal_idx) = new_indices .iter() .find(|idx| idx.name == MEM_WAL_INDEX_NAME) { let details = load_mem_wal_index_details(mem_wal_idx.clone())?; - self.check_merged_generations_conflict( - &details.merged_generations, - self_merged_generations, + self.check_compacted_sstables_conflict( + &details.compacted_sstables, + self_compacted_sstables, other_transaction, other_version, ) @@ -1284,6 +1687,7 @@ impl<'a> TransactionRebase<'a> { | Operation::Overwrite { .. } | Operation::Delete { .. } | Operation::DataReplacement { .. } + | Operation::DataOverlay { .. } | Operation::Merge { .. } | Operation::Restore { .. } | Operation::Clone { .. } @@ -1339,10 +1743,10 @@ impl<'a> TransactionRebase<'a> { } } - fn check_merged_generations_conflict( + fn check_compacted_sstables_conflict( &self, - committed: &[MergedGeneration], - to_commit: &[MergedGeneration], + committed: &[CompactedSsTable], + to_commit: &[CompactedSsTable], other_transaction: &Transaction, other_version: u64, ) -> Result<()> { @@ -1351,7 +1755,8 @@ impl<'a> TransactionRebase<'a> { for to_commit_mg in to_commit { if committed_mg.shard_id == to_commit_mg.shard_id { // Same shard being updated - // If committed >= to_commit, data already merged or superseded - abort without retry + // If committed >= to_commit, the SSTable is already compacted + // or superseded, so abort without retry. // If committed < to_commit, can retry with new state if committed_mg.generation >= to_commit_mg.generation { return Err( @@ -1374,6 +1779,7 @@ impl<'a> TransactionRebase<'a> { } Operation::CreateIndex { .. } => self.finish_create_index(dataset).await, Operation::Rewrite { .. } => self.finish_rewrite(dataset).await, + Operation::DataOverlay { .. } => self.finish_data_overlay(dataset).await, Operation::Append { .. } | Operation::Overwrite { .. } | Operation::DataReplacement { .. } @@ -1550,10 +1956,95 @@ impl<'a> TransactionRebase<'a> { } } + /// Verify no concurrent row-moving Update dropped the values of any cell + /// this overlay covers. `check_data_overlay_txn` flags (via the + /// `initial_fragments` needs-check bool) each overlaid fragment on which a + /// concurrent RewriteRows update relocated rows; here we read the deletion + /// vectors and conflict only when the moved rows intersect the overlay's + /// coverage. + /// + /// The moved rows are computed as the current deletion vector minus the + /// read-time one. In the rare case where both a concurrent Delete and a + /// concurrent Update touched the same flagged fragment, the Delete's rows are + /// also counted and may trigger an unnecessary retry — never data loss. Pure + /// concurrent deletes leave the fragment unflagged and are not examined here. + async fn finish_data_overlay(self, dataset: &Dataset) -> Result { + let fragments_to_check: HashSet = self + .initial_fragments + .iter() + .filter_map(|(id, (_, needs_check))| needs_check.then_some(*id)) + .collect(); + if fragments_to_check.is_empty() { + return Ok(Transaction { + read_version: dataset.manifest.version, + ..self.transaction + }); + } + + // Coverage (physical offsets, unioned across fields) per flagged fragment. + let Operation::DataOverlay { groups } = &self.transaction.operation else { + return Err(wrong_operation_err(&self.transaction.operation)); + }; + let mut coverage_by_fragment: HashMap = HashMap::new(); + for group in groups { + if !fragments_to_check.contains(&group.fragment_id) { + continue; + } + *coverage_by_fragment.entry(group.fragment_id).or_default() |= + overlay_group_coverage(group); + } + + for (fragment_id, coverage) in coverage_by_fragment { + let Some(current_fragment) = dataset + .fragments() + .as_slice() + .iter() + .find(|f| f.id == fragment_id) + else { + // The fragment is gone entirely; the overlay is orphaned. + return Err(crate::Error::retryable_commit_conflict_source( + dataset.manifest.version, + format!( + "This {} transaction was preempted: overlaid fragment {} was removed by a concurrent transaction. Please retry.", + self.transaction.uuid, fragment_id + ) + .into(), + )); + }; + let current_deletions = + read_fragment_deletion_bitmap(dataset, current_fragment).await?; + let initial_deletions = match self.initial_fragments.get(&fragment_id) { + Some((initial_fragment, _)) => { + read_fragment_deletion_bitmap(dataset, initial_fragment).await? + } + None => RoaringBitmap::new(), + }; + let moved_rows = ¤t_deletions - &initial_deletions; + let conflicting = &moved_rows & &coverage; + if !conflicting.is_empty() { + let sample: Vec = conflicting.iter().take(5).collect(); + return Err(crate::Error::retryable_commit_conflict_source( + dataset.manifest.version, + format!( + "This {} transaction was preempted by a concurrent update that moved overlaid rows on fragment {} (offsets {:?}). Please retry.", + self.transaction.uuid, fragment_id, sample.as_slice() + ) + .into(), + )); + } + } + + Ok(Transaction { + read_version: dataset.manifest.version, + ..self.transaction + }) + } + async fn finish_create_index(mut self, dataset: &Dataset) -> Result { if let Operation::CreateIndex { new_indices, removed_indices, + .. } = &mut self.transaction.operation { // Handle FRAG_REUSE_INDEX rebasing @@ -1608,33 +2099,35 @@ impl<'a> TransactionRebase<'a> { // Handle MEM_WAL_INDEX rebasing let has_mem_wal = new_indices.iter().any(|idx| idx.name == MEM_WAL_INDEX_NAME); - if has_mem_wal && !self.conflicting_mem_wal_merged_gens.is_empty() { + if has_mem_wal && !self.conflicting_mem_wal_compacted_sstables.is_empty() { let pos = new_indices .iter() .position(|idx| idx.name == MEM_WAL_INDEX_NAME) .unwrap(); - let current_meta = new_indices.remove(pos); - let mut details = load_mem_wal_index_details(current_meta)?; + let mut details = load_mem_wal_index_details(new_indices[pos].clone())?; - // Merge conflicting merged_generations - for each shard, keep higher generation - // We own self so we can consume conflicting_mem_wal_merged_gens directly - for new_mg in self.conflicting_mem_wal_merged_gens { + // Reconcile conflicting compacted_sstables by keeping each shard's higher + // generation. Both sides are already-committed facts here, so the higher one + // is correct; rejecting a stale proposal is the job of apply-time validation + // against the latest state, which runs after this rebase. + // We own self so we can consume conflicting_mem_wal_compacted_sstables directly + for new_sstable in self.conflicting_mem_wal_compacted_sstables { if let Some(existing) = details - .merged_generations + .compacted_sstables .iter_mut() - .find(|mg| mg.shard_id == new_mg.shard_id) + .find(|sstable| sstable.shard_id == new_sstable.shard_id) { - if new_mg.generation > existing.generation { - existing.generation = new_mg.generation; + if new_sstable.generation > existing.generation { + existing.generation = new_sstable.generation; } } else { - details.merged_generations.push(new_mg); + details.compacted_sstables.push(new_sstable); } } - let new_meta = new_mem_wal_index_meta(dataset.manifest.version, details)?; - new_indices.push(new_meta); + // Replaced in place so the index list keeps its order. + new_indices[pos] = new_mem_wal_index_meta(dataset.manifest.version, details)?; } for singleton_name in [FRAG_REUSE_INDEX_NAME, MEM_WAL_INDEX_NAME] { @@ -1747,23 +2240,22 @@ async fn initial_fragments_for_rebase( dataset: &Dataset, transaction: &Transaction, modified_fragment_ids: &HashSet, -) -> HashMap { +) -> Result> { if modified_fragment_ids.is_empty() { - return HashMap::new(); + return Ok(HashMap::new()); } let dataset = if dataset.manifest.version != transaction.read_version { - Cow::Owned( - dataset - .checkout_version(transaction.read_version) - .await - .unwrap(), - ) + // The read version may have been garbage-collected by a concurrent + // `cleanup_old_versions` between the commit attempt and the rebase. + // Propagate the error so the commit fails gracefully instead of + // panicking (which aborts the whole process when `panic = "abort"`). + Cow::Owned(dataset.checkout_version(transaction.read_version).await?) } else { Cow::Borrowed(dataset) }; - dataset + Ok(dataset .fragments() .iter() .filter(|fragment| { @@ -1771,7 +2263,41 @@ async fn initial_fragments_for_rebase( modified_fragment_ids.contains(&fragment.id) }) .map(|fragment| (fragment.id, (fragment.clone(), false))) - .collect::>() + .collect()) +} + +/// Read a fragment's deletion vector as a bitmap of physical offsets, or an +/// empty bitmap when the fragment has no deletion file. +async fn read_fragment_deletion_bitmap( + dataset: &Dataset, + fragment: &Fragment, +) -> Result { + match &fragment.deletion_file { + Some(deletion_file) => { + let dv = read_dataset_deletion_file(dataset, fragment.id, deletion_file).await?; + Ok(RoaringBitmap::from(dv.as_ref())) + } + None => Ok(RoaringBitmap::new()), + } +} + +/// The physical offsets a group's overlays cover, unioned across every overlay +/// and every field. This is the set of cells whose values the overlay supplies, +/// used to test whether a concurrent row-moving Update actually invalidates the +/// overlay. +fn overlay_group_coverage(group: &DataOverlayGroup) -> RoaringBitmap { + let mut union = RoaringBitmap::new(); + for overlay in &group.overlays { + match &overlay.coverage { + OverlayCoverage::Shared(bitmap) => union |= bitmap.as_ref(), + OverlayCoverage::PerField(bitmaps) => { + for bitmap in bitmaps { + union |= bitmap.as_ref(); + } + } + } + } + union } fn wrong_operation_err(op: &Operation) -> Error { @@ -1794,7 +2320,7 @@ mod tests { use lance_table::io::deletion::{deletion_file_path, read_deletion_file}; use super::*; - use crate::dataset::transaction::{DataReplacementGroup, RewriteGroup}; + use crate::dataset::transaction::{DataReplacementGroup, RewriteGroup, UpdateMap}; use crate::dataset::write::WriteMode; use crate::session::caches::DeletionFileKey; use crate::{ @@ -1876,34 +2402,177 @@ mod tests { } } + #[rstest::rstest] + #[case::config(false)] + #[case::table_metadata(true)] #[tokio::test] - async fn test_non_overlapping_rebase_delete_update() { - let dataset = test_dataset(5, 5).await; - let operation = Operation::Update { - updated_fragments: vec![Fragment::new(0)], - removed_fragment_ids: vec![], - new_fragments: vec![], - fields_modified: vec![], - merged_generations: Vec::new(), - fields_for_preserving_frag_bitmap: vec![], - update_mode: None, - inserted_rows_filter: None, - updated_fragment_offsets: None, - }; - let transaction = Transaction::new_from_version(1, operation); - let other_operations = [ - Operation::Update { - updated_fragments: vec![Fragment::new(1)], - removed_fragment_ids: vec![2], - new_fragments: vec![], - fields_modified: vec![], - merged_generations: Vec::new(), - fields_for_preserving_frag_bitmap: vec![], - update_mode: None, - inserted_rows_filter: None, - updated_fragment_offsets: None, + async fn test_merge_preserves_unrelated_update_config_compatibility( + #[case] update_table_metadata: bool, + #[values(true, false)] merge_commits_first: bool, + ) { + let dataset = Arc::new(test_dataset(5, 1).await); + let read_version = dataset.manifest.version; + + let mut merged_schema = dataset.schema().clone(); + merged_schema + .metadata + .insert("merge.schema".to_string(), "preserved".to_string()); + let field_id = merged_schema.fields[0].id; + merged_schema.fields[0] + .metadata + .insert("merge.field".to_string(), "preserved".to_string()); + let merge = Transaction::new_from_version( + read_version, + Operation::Merge { + fragments: dataset.manifest.fragments.as_ref().clone(), + schema: merged_schema, + preserves_nullability: true, }, - Operation::Delete { + ); + + let replacement = UpdateMap { + update_entries: vec![("key", Some("value")).into()], + replace: true, + }; + let update_config = Transaction::new_from_version( + read_version, + Operation::UpdateConfig { + config_updates: (!update_table_metadata).then_some(replacement.clone()), + table_metadata_updates: update_table_metadata.then_some(replacement), + schema_metadata_updates: None, + field_metadata_updates: HashMap::new(), + }, + ); + + let (first, stale) = if merge_commits_first { + (merge, update_config) + } else { + (update_config, merge) + }; + CommitBuilder::new(dataset.clone()) + .execute(first) + .await + .unwrap(); + let latest_dataset = CommitBuilder::new(dataset).execute(stale).await.unwrap(); + + assert_eq!( + latest_dataset + .schema() + .metadata + .get("merge.schema") + .map(String::as_str), + Some("preserved") + ); + assert_eq!( + latest_dataset + .schema() + .field_by_id(field_id) + .unwrap() + .metadata + .get("merge.field") + .map(String::as_str), + Some("preserved") + ); + let updated_map = if update_table_metadata { + &latest_dataset.manifest.table_metadata + } else { + &latest_dataset.manifest.config + }; + assert_eq!(updated_map.get("key").map(String::as_str), Some("value")); + } + + #[rstest::rstest] + #[case::schema_metadata(true)] + #[case::field_metadata(false)] + #[tokio::test] + async fn test_concurrent_merge_and_metadata_update_conflict( + #[case] replace_schema_metadata: bool, + #[values(true, false)] replace: bool, + #[values(true, false)] merge_commits_first: bool, + ) { + let dataset = Arc::new(test_dataset(5, 1).await); + let read_version = dataset.manifest.version; + + let mut merged_schema = dataset.schema().clone(); + merged_schema + .metadata + .insert("merge.schema".to_string(), "coordinated".to_string()); + let field_id = merged_schema.fields[0].id; + merged_schema.fields[0] + .metadata + .insert("merge.field".to_string(), "coordinated".to_string()); + let merge = Transaction::new_from_version( + read_version, + Operation::Merge { + fragments: dataset.manifest.fragments.as_ref().clone(), + schema: merged_schema, + preserves_nullability: true, + }, + ); + + let metadata_update = UpdateMap { + update_entries: vec![("replacement", Some("coordinated")).into()], + replace, + }; + let update_config = Transaction::new_from_version( + read_version, + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: None, + schema_metadata_updates: replace_schema_metadata.then_some(metadata_update.clone()), + field_metadata_updates: if replace_schema_metadata { + HashMap::new() + } else { + HashMap::from_iter([(field_id, metadata_update)]) + }, + }, + ); + + let (first, stale) = if merge_commits_first { + (merge, update_config) + } else { + (update_config, merge) + }; + CommitBuilder::new(dataset.clone()) + .execute(first) + .await + .unwrap(); + let error = CommitBuilder::new(dataset) + .execute(stale) + .await + .unwrap_err(); + + assert!(matches!(error, Error::RetryableCommitConflict { .. })); + } + + #[tokio::test] + async fn test_non_overlapping_rebase_delete_update() { + let dataset = test_dataset(5, 5).await; + let operation = Operation::Update { + updated_fragments: vec![Fragment::new(0)], + removed_fragment_ids: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let transaction = Transaction::new_from_version(1, operation); + let other_operations = [ + Operation::Update { + updated_fragments: vec![Fragment::new(1)], + removed_fragment_ids: vec![2], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + Operation::Delete { deleted_fragment_ids: vec![3], updated_fragments: vec![], predicate: "a > 0".to_string(), @@ -1913,7 +2582,7 @@ mod tests { updated_fragments: vec![Fragment::new(4)], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -1949,6 +2618,78 @@ mod tests { assert_io_eq!(io_stats, write_iops, 0); } + #[tokio::test] + async fn test_rebase_errors_when_read_version_was_cleaned_up() { + // Regression test: `initial_fragments_for_rebase` used to `unwrap()` the + // result of `checkout_version(read_version)`. If a concurrent + // `cleanup_old_versions` removed that version between the conflicting + // commit and the rebase, this panicked (aborting the whole process when + // built with `panic = "abort"`). The rebase should fail with an error + // instead so the commit can be retried. + let tmp_dir = lance_core::utils::tempfile::TempStrDir::default(); + let uri = tmp_dir.as_str().to_string(); + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, true), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..5)), + Arc::new(Int32Array::from_iter_values(std::iter::repeat_n(0, 5))), + ], + ) + .unwrap(); + + // Write version 1, then append version 2. + let write_params = WriteParams { + max_rows_per_file: 1, + ..Default::default() + }; + InsertBuilder::new(&uri) + .with_params(&write_params) + .execute(vec![batch.clone()]) + .await + .unwrap(); + let append_params = WriteParams { + mode: WriteMode::Append, + max_rows_per_file: 1, + ..Default::default() + }; + let dataset = InsertBuilder::new(&uri) + .with_params(&append_params) + .execute(vec![batch]) + .await + .unwrap(); + assert_eq!(dataset.manifest.version, 2); + + // A transaction that read version 1 and modified fragment 0. + let operation = Operation::Update { + updated_fragments: vec![Fragment::new(0)], + removed_fragment_ids: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let transaction = Transaction::new_from_version(1, operation); + + // Simulate a concurrent `cleanup_old_versions` removing version 1. + let naming_scheme = dataset.manifest_location().naming_scheme; + let v1_manifest = naming_scheme.manifest_path(&dataset.base, 1); + dataset.object_store.delete(&v1_manifest).await.unwrap(); + + // Rebasing now needs to check out version 1, which no longer exists. + // This used to panic; it should return `DatasetNotFound` instead. + let err = TransactionRebase::try_new(&dataset, transaction, None) + .await + .unwrap_err(); + assert!(matches!(err, Error::DatasetNotFound { .. })); + } + async fn apply_deletion( delete_rows: &[u32], fragment: &mut Fragment, @@ -2006,7 +2747,7 @@ mod tests { "path1", vec![0], vec![0], - &LanceFileVersion::Stable, + LanceFileVersion::Stable.resolve(), NonZero::new(10), ) .with_physical_rows(3); @@ -2016,7 +2757,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![sample_file.clone()], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2032,7 +2773,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![sample_file], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2149,7 +2890,7 @@ mod tests { "path1", vec![0], vec![0], - &LanceFileVersion::Stable, + LanceFileVersion::Stable.resolve(), NonZero::new(10), ) .with_physical_rows(3); @@ -2162,7 +2903,7 @@ mod tests { removed_fragment_ids: vec![0], new_fragments: vec![sample_file.clone()], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2176,7 +2917,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![sample_file.clone()], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2285,6 +3026,7 @@ mod tests { uuid: uuid::Uuid::new_v4(), name: "test".to_string(), fields: vec![0], + covering_fields: vec![], dataset_version: 1, fragment_bitmap: None, index_details: None, @@ -2313,6 +3055,7 @@ mod tests { Operation::Merge { fragments: vec![fragment0.clone(), fragment2.clone()], schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, }, Operation::Overwrite { fragments: vec![fragment0.clone(), fragment2.clone()], @@ -2337,7 +3080,7 @@ mod tests { updated_fragments: vec![fragment0.clone()], new_fragments: vec![fragment2.clone()], fields_modified: vec![0], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2508,8 +3251,9 @@ mod tests { Operation::Merge { fragments: vec![fragment0.clone(), fragment2.clone()], schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, }, - // Merge conflicts with everything except CreateIndex and ReserveFragments. + // Merge also conflicts with schema and field metadata updates. [ Retryable, // append Compatible, // create index @@ -2519,7 +3263,7 @@ mod tests { Retryable, // rewrite Compatible, // reserve Retryable, // update - Compatible, // update config + Retryable, // update config ], ), ( @@ -2544,7 +3288,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![fragment2], fields_modified: vec![0], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -2667,7 +3411,7 @@ mod tests { Compatible, // append Compatible, // create index Compatible, // delete - Compatible, // merge + Retryable, // merge NotCompatible, // overwrite Compatible, // rewrite Compatible, // reserve @@ -2694,7 +3438,7 @@ mod tests { Compatible, // append Compatible, // create index Compatible, // delete - Compatible, // merge + Retryable, // merge NotCompatible, // overwrite Compatible, // rewrite Compatible, // reserve @@ -2720,7 +3464,7 @@ mod tests { Compatible, // append Compatible, // create index Compatible, // delete - Compatible, // merge + Retryable, // merge NotCompatible, // overwrite Compatible, // rewrite Compatible, // reserve @@ -2730,55 +3474,789 @@ mod tests { ), ]; - for (operation, expected_conflicts) in &cases { - let transaction = Transaction::new(0, operation.clone(), None); + for (operation, expected_conflicts) in &cases { + let transaction = Transaction::new(0, operation.clone(), None); + let mut rebase = TransactionRebase { + transaction, + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(operation).collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + + for (other, expected_conflict) in other_transactions.iter().zip(expected_conflicts) { + match expected_conflict { + Compatible => { + let result = rebase.check_txn(other, 1); + assert!( + result.is_ok(), + "Transaction {:?} should {:?} with {:?}, but was {:?}", + operation, + expected_conflict, + other, + result + ) + } + NotCompatible => { + let result = rebase.check_txn(other, 1); + assert!( + matches!(result, Err(Error::IncompatibleTransaction { .. })), + "Transaction {:?} should be {:?} with {:?}, but was: {:?}", + operation, + expected_conflict, + other, + result + ) + } + Retryable => { + let result = rebase.check_txn(other, 1); + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "Transaction {:?} should be {:?} with {:?}, but was {:?}", + operation, + expected_conflict, + other, + result + ) + } + } + } + } + } + + #[test] + fn test_data_overlay_conflicts() { + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use ConflictResult::*; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + // Our transaction overlays fragment 1. + let overlay_op = |fragment_id: u64| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }; + let update_removing = |removed_fragment_ids: Vec| Operation::Update { + removed_fragment_ids, + updated_fragments: vec![], + new_fragments: vec![], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: None, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let delete = |updated: Vec, deleted: Vec| Operation::Delete { + updated_fragments: updated, + deleted_fragment_ids: deleted, + predicate: "x > 2".to_string(), + }; + // A row-moving update (RewriteRows) relocates the updated rows into + // new_fragments; an in-place column rewrite (RewriteColumns) leaves rows + // where they are. + let update_moving = |updated: Vec, new: Vec| Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: updated, + new_fragments: new, + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteRows), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let update_rewrite_columns = |updated: Vec| Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: updated, + new_fragments: vec![], + fields_modified: vec![0], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let rewrite_of = |old: &Fragment| Operation::Rewrite { + groups: vec![RewriteGroup { + old_fragments: vec![old.clone()], + new_fragments: vec![], + }], + rewritten_indices: vec![], + frag_reuse_index: None, + }; + + let fragment0 = Fragment::new(0); + let fragment1 = Fragment::new(1); + + // Each case is checked against our overlay on fragment 1. + let cases: Vec<(Operation, ConflictResult)> = vec![ + // Permissive: preserves physical offsets / leaves fragment 1 in place. + ( + Operation::Append { + fragments: vec![fragment0.clone()], + }, + Compatible, + ), + ( + Operation::CreateIndex { + new_indices: vec![], + removed_indices: vec![], + }, + Compatible, + ), + ( + Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 1, + DataFile::new_legacy_from_fields("r.lance", vec![0], None), + )], + }, + Compatible, + ), + // Another overlay on the same fragment stacks rather than conflicts. + (overlay_op(1), Compatible), + // A Delete only tombstones rows (deletion vector) on fragment 1, and + // an in-place column rewrite preserves offsets, so both are compatible. + (delete(vec![fragment1.clone()], vec![]), Compatible), + (update_rewrite_columns(vec![fragment1.clone()]), Compatible), + (update_removing(vec![2]), Compatible), + // ...but removing our overlaid fragment 1 orphans the overlay -> conflict. + (delete(vec![], vec![1]), Retryable), + (update_removing(vec![1]), Retryable), + // A row-moving update re-creates the rows it touches from the + // pre-overlay base. Whether that actually drops any overlaid cell is + // a per-row question answered in `finish_data_overlay` (see + // test_data_overlay_finish_conflicts_with_row_moving_update), so the + // check itself defers rather than conflicting; a moving update on any + // fragment is compatible at this stage. + ( + update_moving(vec![fragment1.clone()], vec![fragment0.clone()]), + Compatible, + ), + ( + update_moving(vec![fragment0.clone()], vec![fragment0.clone()]), + Compatible, + ), + // Rewriting fragment 1 invalidates its physical offsets -> conflict; + // a rewrite of a different fragment does not. + (rewrite_of(&fragment1), Retryable), + (rewrite_of(&fragment0), Compatible), + // Merge rewrites the whole fragment list; Restore replaces the dataset. + ( + Operation::Merge { + fragments: vec![fragment1.clone()], + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, + }, + Retryable, + ), + (Operation::Restore { version: 1 }, NotCompatible), + // Overwrite/Restore replace the dataset, and UpdateMemWalState does + // not rebase against data operations — all hard conflicts. + ( + Operation::Overwrite { + fragments: vec![fragment0.clone()], + schema: lance_core::datatypes::Schema::default(), + config_upsert_values: None, + initial_bases: None, + }, + NotCompatible, + ), + ( + Operation::UpdateMemWalState { + compacted_sstables: vec![], + }, + NotCompatible, + ), + ]; + + for (other, expected) in cases { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, overlay_op(1), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&overlay_op(1)) + .collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let other_txn = Transaction::new(0, other.clone(), None); + let result = rebase.check_txn(&other_txn, 1); + match expected { + Compatible => assert!( + result.is_ok(), + "overlay should be compatible with {other:?}, got {result:?}" + ), + Retryable => assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "overlay should retryably conflict with {other:?}, got {result:?}" + ), + NotCompatible => assert!( + matches!(result, Err(Error::IncompatibleTransaction { .. })), + "overlay should be incompatible with {other:?}, got {result:?}" + ), + } + } + } + + #[test] + fn test_rewrite_conflicts_with_data_overlay() { + // Reverse direction of test_data_overlay_conflicts: our transaction is a + // Rewrite and a concurrent DataOverlay has already committed. A rewrite + // changes the physical row addresses of the fragments it touches, so an + // overlay on one of those fragments is invalidated (retryable); an + // overlay on any other fragment is unaffected. + use crate::dataset::transaction::DataOverlayGroup; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let overlay_on = |fragment_id: u64| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }; + // Our transaction rewrites fragment 1. + let rewrite_op = Operation::Rewrite { + groups: vec![RewriteGroup { + old_fragments: vec![Fragment::new(1)], + new_fragments: vec![], + }], + rewritten_indices: vec![], + frag_reuse_index: None, + }; + + for (other, expect_conflict) in [(overlay_on(1), true), (overlay_on(0), false)] { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, rewrite_op.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&rewrite_op).collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let other_txn = Transaction::new(0, other.clone(), None); + let result = rebase.check_txn(&other_txn, 1); + if expect_conflict { + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "rewrite of fragment 1 should retryably conflict with {other:?}, got {result:?}" + ); + } else { + assert!( + result.is_ok(), + "rewrite of fragment 1 should not conflict with {other:?}, got {result:?}" + ); + } + } + } + + #[test] + fn test_update_conflicts_with_data_overlay() { + // Reverse direction of test_data_overlay_conflicts: our transaction is an + // Update and a concurrent DataOverlay has already committed. A row-moving + // update relocates the rows it touches, so an overlay on one of those + // fragments can no longer be applied (retryable); an overlay on any other + // fragment is compatible. An in-place column rewrite preserves rows but + // replaces the whole column from its own snapshot, so it conflicts + // whenever the overlay covers a fragment and field it rewrote. + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let overlay_on_field = |fragment_id: u64, field: i32| Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![field], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }; + let overlay_on = |fragment_id: u64| overlay_on_field(fragment_id, 0); + // Our update always touches fragment 1. + let update = + |update_mode: Option, new_fragments: Vec| Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![Fragment::new(1)], + new_fragments, + fields_modified: vec![0], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode, + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + + // The overlay covers physical offset 0 of its fragment. Row addresses + // pack the fragment id in the high 32 bits and the offset in the low 32. + let rows_on = |fragment_id: u64, offsets: &[u32]| { + let mut map = RowAddrTreeMap::new(); + map.insert_bitmap( + fragment_id as u32, + RoaringBitmap::from_iter(offsets.iter().copied()), + ); + map + }; + + // (update, committed overlay, moved rows the update carries, expect conflict) + let cases = [ + // Row-moving update whose moved rows include the overlaid cell -> the + // update would undo the overlay, so conflict. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(1), + Some(rows_on(1, &[0])), + true, + ), + // ...but if the moved rows miss the overlaid cell, the overlay survives. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(1), + Some(rows_on(1, &[5])), + false, + ), + // An overlay on a fragment the update did not touch is fine. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(0), + Some(rows_on(1, &[0])), + false, + ), + // An in-place column rewrite replaces field 0 across all of fragment + // 1 from its own snapshot, and `build_manifest` tombstones the + // overlay for that field, so the overlay's value would be lost even + // though it sits on a row the update never matched -> conflict. + ( + update(Some(UpdateMode::RewriteColumns), vec![]), + overlay_on(1), + Some(rows_on(1, &[0])), + true, + ), + // ...and the coverage is irrelevant: an overlay on a row the update + // did not match is exactly the case that gets silently dropped. + ( + update(Some(UpdateMode::RewriteColumns), vec![]), + overlay_on(1), + Some(rows_on(1, &[5])), + true, + ), + // An overlay on a field the rewrite did not touch survives the + // tombstoning, so it stays compatible. + ( + update(Some(UpdateMode::RewriteColumns), vec![]), + overlay_on_field(1, 7), + Some(rows_on(1, &[0])), + false, + ), + // So does an overlay on a fragment the rewrite did not touch. + ( + update(Some(UpdateMode::RewriteColumns), vec![]), + overlay_on(0), + Some(rows_on(1, &[0])), + false, + ), + // Without affected rows we cannot be precise, so a row-moving update + // on the overlaid fragment falls back to a conservative conflict. + ( + update(Some(UpdateMode::RewriteRows), vec![Fragment::new(2)]), + overlay_on(1), + None, + true, + ), + ]; + + for (update_op, other, affected_rows, expect_conflict) in cases { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, update_op.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&update_op).collect::>(), + affected_rows: affected_rows.as_ref(), + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let other_txn = Transaction::new(0, other.clone(), None); + let result = rebase.check_txn(&other_txn, 1); + if expect_conflict { + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "update should retryably conflict with {other:?}, got {result:?}" + ); + } else { + assert!( + result.is_ok(), + "update should be compatible with {other:?}, got {result:?}" + ); + } + } + } + + /// An append is the one value-write that rebases across a committed merge, + /// so a merge introducing a required field must claim: the append's + /// fragments omit the new column and its rows would read as null. A merge + /// without the claim keeps the long-standing behavior of appends passing + /// over nullable column adds. The reverse order conflicts regardless of + /// the claim, because a rebasing merge rewrites the whole fragment list. + #[test] + fn test_merge_claim_blocks_stale_append() { + for claims in [true, false] { + let merge = Operation::Merge { + fragments: vec![Fragment::new(0)], + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: !claims, + }; + let append = Operation::Append { + fragments: vec![Fragment::new(1)], + }; + + let mut append_rebase = TransactionRebase { + transaction: Transaction::new(0, append.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = append_rebase.check_txn(&Transaction::new(0, merge.clone(), None), 1); + assert_eq!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + claims, + "append rebasing over merge/claims={claims}: got {result:?}" + ); + + let mut merge_rebase = TransactionRebase { + transaction: Transaction::new(0, merge, None), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::from_iter([0]), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = merge_rebase.check_txn(&Transaction::new(0, append, None), 1); + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "merge rebasing over append/claims={claims}: got {result:?}" + ); + } + } + + /// A claim conflicts with any write that can supply values, either order. + #[test] + fn test_non_null_claim_barrier() { + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let file = || DataFile::new_legacy_from_fields("w.lance", vec![0], None); + let writers = [ + ( + "append", + Operation::Append { + fragments: vec![Fragment::new(1)], + }, + ), + ( + "update", + Operation::Update { + removed_fragment_ids: vec![], + updated_fragments: vec![Fragment::new(0)], + new_fragments: vec![], + fields_modified: vec![0], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + ), + ( + "replacement", + Operation::DataReplacement { + replacements: vec![DataReplacementGroup(0, file())], + }, + ), + ( + "overlay", + Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 0, + overlays: vec![DataOverlayFile { + data_file: file(), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 0, + }], + }], + }, + ), + ]; + + for (writer_name, writer) in &writers { + for claims in [true, false] { + let project = Operation::Project { + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: !claims, + }; + for (order, ours, theirs) in [ + ("project-rebasing", project.clone(), writer.clone()), + ("writer-rebasing", writer.clone(), project.clone()), + ] { + let mut rebase = TransactionRebase { + transaction: Transaction::new(0, ours.clone(), None), + initial_fragments: HashMap::new(), + modified_fragment_ids: modified_fragment_ids(&ours).collect::>(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = rebase.check_txn(&Transaction::new(0, theirs, None), 1); + assert_eq!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + claims, + "{writer_name}/claims={claims}/{order}: got {result:?}" + ); + } + } + } + } + + #[tokio::test] + #[rstest::rstest] + #[case::coverage_overlaps_moved_row(vec![0u32], true)] + #[case::coverage_disjoint_from_moved_row(vec![3u32], false)] + async fn test_data_overlay_finish_conflicts_with_row_moving_update( + #[case] coverage_offsets: Vec, + #[case] expect_conflict: bool, + ) { + // 5 rows in one fragment. A concurrent RewriteRows update moves row 0 out + // to a new fragment (deleting it from fragment 0). Our overlay on fragment + // 0 conflicts only when its coverage includes the moved row; the decision + // is made in finish, which reads the deletion vectors. + use crate::dataset::transaction::{DataOverlayGroup, UpdateMode}; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + use roaring::RoaringBitmap; + + let dataset = test_dataset(5, 1).await; + let mut fragment = dataset.fragments().as_slice()[0].clone(); + + let moved_fragment = Fragment::new(0) + .with_file( + "moved.lance", + vec![0], + vec![0], + LanceFileVersion::Stable.resolve(), + NonZero::new(10), + ) + .with_physical_rows(1); + let update_op = Operation::Update { + updated_fragments: vec![apply_deletion(&[0], &mut fragment, &dataset).await], + removed_fragment_ids: vec![], + new_fragments: vec![moved_fragment], + fields_modified: vec![], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteRows), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }; + let update_txn = Transaction::new_from_version(dataset.manifest.version, update_op); + + let overlay_op = Operation::DataOverlay { + groups: vec![DataOverlayGroup { + fragment_id: 0, + overlays: vec![DataOverlayFile { + data_file: DataFile::new_legacy_from_fields("overlay.lance", vec![0], None), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter(coverage_offsets)), + committed_version: 0, + }], + }], + }; + let overlay_txn = Transaction::new_from_version(dataset.manifest.version, overlay_op); + + // Commit the update so the latest dataset reflects the moved (deleted) row. + let latest_dataset = CommitBuilder::new(Arc::new(dataset.clone())) + .execute(update_txn.clone()) + .await + .unwrap(); + + let mut rebase = TransactionRebase::try_new(&dataset, overlay_txn.clone(), None) + .await + .unwrap(); + // The check defers the row-level decision to finish, flagging fragment 0. + rebase.check_txn(&update_txn, 1).unwrap(); + assert_eq!( + rebase + .initial_fragments + .iter() + .map(|(id, (_, needs_check))| (*id, *needs_check)) + .collect::>(), + vec![(0, true)], + ); + + let res = rebase.finish(&latest_dataset).await; + if expect_conflict { + assert!( + matches!(res, Err(crate::Error::RetryableCommitConflict { .. })), + "overlay covering the moved row should conflict, got {res:?}" + ); + } else { + assert!( + res.is_ok(), + "overlay disjoint from the moved row should succeed, got {res:?}" + ); + } + } + + #[rstest::rstest] + #[test] + #[case::indexed_field_updated(0, vec![0])] + #[case::other_field_updated(1, vec![0, 1])] + fn test_create_index_rebase_prunes_updated_field_coverage( + #[case] field_modified: u32, + #[case] expected_fragment_ids: Vec, + ) { + let index = IndexMetadata { + uuid: Uuid::new_v4(), + name: "test".to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([0, 1])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let mut rebase = TransactionRebase { + transaction: Transaction::new( + 1, + Operation::CreateIndex { + new_indices: vec![index], + removed_indices: vec![], + }, + None, + ), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let update = Transaction::new( + 1, + Operation::Update { + updated_fragments: vec![Fragment::new(1)], + removed_fragment_ids: vec![], + new_fragments: vec![], + fields_modified: vec![field_modified], + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: vec![], + update_mode: Some(UpdateMode::RewriteColumns), + inserted_rows_filter: None, + updated_fragment_offsets: None, + }, + None, + ); + + rebase.check_txn(&update, 2).unwrap(); + + let Operation::CreateIndex { new_indices, .. } = &rebase.transaction.operation else { + panic!("expected CreateIndex operation"); + }; + assert_eq!( + new_indices[0].fragment_bitmap.as_ref().unwrap(), + &RoaringBitmap::from_iter(expected_fragment_ids) + ); + } + + #[test] + fn test_mem_wal_install_conflicts_with_merge() { + let mem_wal_index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: MEM_WAL_INDEX_NAME.to_string(), + fields: vec![], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap: None, + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let column_index = IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: "btree".to_string(), + ..mem_wal_index.clone() + }; + let merge = Transaction::new( + 0, + Operation::Merge { + fragments: vec![], + schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, + }, + None, + ); + + // Install rebasing over a committed Merge conflicts; a column index + // stays compatible. + for (index, conflicts) in [(mem_wal_index.clone(), true), (column_index, false)] { + let txn = Transaction::new( + 0, + Operation::CreateIndex { + new_indices: vec![index], + removed_indices: vec![], + }, + None, + ); let mut rebase = TransactionRebase { - transaction, + transaction: txn, initial_fragments: HashMap::new(), - modified_fragment_ids: modified_fragment_ids(operation).collect::>(), + modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; - - for (other, expected_conflict) in other_transactions.iter().zip(expected_conflicts) { - match expected_conflict { - Compatible => { - let result = rebase.check_txn(other, 1); - assert!( - result.is_ok(), - "Transaction {:?} should {:?} with {:?}, but was {:?}", - operation, - expected_conflict, - other, - result - ) - } - NotCompatible => { - let result = rebase.check_txn(other, 1); - assert!( - matches!(result, Err(Error::IncompatibleTransaction { .. })), - "Transaction {:?} should be {:?} with {:?}, but was: {:?}", - operation, - expected_conflict, - other, - result - ) - } - Retryable => { - let result = rebase.check_txn(other, 1); - assert!( - matches!(result, Err(Error::RetryableCommitConflict { .. })), - "Transaction {:?} should be {:?} with {:?}, but was {:?}", - operation, - expected_conflict, - other, - result - ) - } - } - } + let result = rebase.check_txn(&merge, 1); + assert_eq!(result.is_err(), conflicts, "{result:?}"); } + + // And the reverse: a Merge rebasing over a committed install conflicts. + let install = Transaction::new( + 0, + Operation::CreateIndex { + new_indices: vec![mem_wal_index], + removed_indices: vec![], + }, + None, + ); + let mut rebase = TransactionRebase { + transaction: merge, + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = rebase.check_txn(&install, 1); + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "{result:?}" + ); } #[test] @@ -2787,6 +4265,7 @@ mod tests { uuid: uuid::Uuid::new_v4(), name: "test".to_string(), fields: vec![0], + covering_fields: vec![], dataset_version: 1, fragment_bitmap: None, index_details: None, @@ -2815,7 +4294,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let same_name = Transaction::new( @@ -2853,6 +4332,7 @@ mod tests { uuid: uuid::Uuid::new_v4(), name: "test".to_string(), fields: vec![0], + covering_fields: vec![], dataset_version: 1, fragment_bitmap: None, index_details: None, @@ -2869,7 +4349,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let different_name_result = rebase.check_txn(&different_name, 1); assert!( @@ -2879,6 +4359,81 @@ mod tests { ); } + #[test] + fn test_create_ngram_index_conflicts_with_overlapping_deferred_rewrite() { + let ngram_index = |fragment_id| IndexMetadata { + uuid: Uuid::new_v4(), + name: "text_ngram".to_string(), + fields: vec![0], + covering_fields: vec![], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([fragment_id])), + index_details: Some(Arc::new(prost_types::Any { + type_url: "lance.index.NGramIndexDetails".to_string(), + value: Vec::new(), + })), + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let frag_reuse_index = IndexMetadata { + uuid: Uuid::new_v4(), + name: FRAG_REUSE_INDEX_NAME.to_string(), + fields: vec![], + covering_fields: vec![], + dataset_version: 2, + fragment_bitmap: Some(RoaringBitmap::from_iter([2u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let rewrite = Transaction::new( + 1, + Operation::Rewrite { + groups: vec![RewriteGroup { + old_fragments: vec![Fragment::new(1)], + new_fragments: vec![Fragment::new(2)], + }], + rewritten_indices: vec![], + frag_reuse_index: Some(frag_reuse_index), + }, + None, + ); + + for (covered_fragment, expect_conflict) in [(1u32, true), (3u32, false)] { + let mut rebase = TransactionRebase { + transaction: Transaction::new( + 1, + Operation::CreateIndex { + new_indices: vec![ngram_index(covered_fragment)], + removed_indices: vec![], + }, + None, + ), + initial_fragments: HashMap::new(), + modified_fragment_ids: HashSet::new(), + affected_rows: None, + conflicting_frag_reuse_indices: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), + }; + let result = rebase.check_txn(&rewrite, 2); + if expect_conflict { + assert!( + matches!(result, Err(Error::RetryableCommitConflict { .. })), + "overlapping staged NGram index should conflict, got {result:?}" + ); + } else { + assert!( + result.is_ok(), + "disjoint staged NGram index should remain compatible, got {result:?}" + ); + } + } + } + #[tokio::test] async fn test_add_bases_non_conflicting() { let dataset = test_dataset(10, 2).await; @@ -3067,7 +4622,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -3255,6 +4810,7 @@ mod tests { Operation::DataReplacement { replacements } => { Box::new(replacements.iter().map(|r| r.0)) } + Operation::DataOverlay { groups } => Box::new(groups.iter().map(|g| g.fragment_id)), } } @@ -3359,7 +4915,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![2], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteColumns), inserted_rows_filter: None, @@ -3378,7 +4934,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![Fragment::new(5)], fields_modified: vec![2], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteColumns), inserted_rows_filter: None, @@ -3396,7 +4952,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![], fields_modified: vec![1], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteColumns), inserted_rows_filter: None, @@ -3414,7 +4970,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![Fragment::new(5)], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteRows), inserted_rows_filter: None, @@ -3432,7 +4988,7 @@ mod tests { removed_fragment_ids: vec![0], new_fragments: vec![], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: None, inserted_rows_filter: None, @@ -3450,7 +5006,7 @@ mod tests { removed_fragment_ids: vec![], new_fragments: vec![Fragment::new(5)], fields_modified: vec![], - merged_generations: Vec::new(), + compacted_sstables: Vec::new(), fields_for_preserving_frag_bitmap: vec![], update_mode: Some(RewriteRows), inserted_rows_filter: None, @@ -3491,6 +5047,42 @@ mod tests { Operation::Merge { fragments: vec![Fragment::new(0)], schema: lance_core::datatypes::Schema::default(), + preserves_nullability: true, + }, + Retryable, + ), + ( + // Unlike every other case here, op1 is CreateIndex and op2 is + // DataReplacement. This is deliberate, not an inconsistency: + // `check_txn` dispatches on op1's operation type, so only + // op1 == CreateIndex reaches `check_create_index_txn`'s + // `Operation::DataReplacement` arm, which is the arm this case + // targets. Swapping the order to match the other cases would + // instead exercise the mirrored `check_data_replacement_txn`'s + // `Operation::CreateIndex` arm, leaving the intended arm with + // zero coverage. + "CreateIndex covering a field vs DataReplacement of that field", + Operation::CreateIndex { + new_indices: vec![IndexMetadata { + uuid: Uuid::new_v4(), + name: "covering_idx".to_string(), + fields: vec![0, 3], + covering_fields: vec![3], + dataset_version: 1, + fragment_bitmap: Some(RoaringBitmap::from_iter([0u32])), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }], + removed_indices: vec![], + }, + Operation::DataReplacement { + replacements: vec![DataReplacementGroup( + 0, + DataFile::new_legacy_from_fields("path0_3", vec![3], None), + )], }, Retryable, ), @@ -3506,7 +5098,7 @@ mod tests { modified_fragment_ids: modified_fragment_ids(&op1).collect::>(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&txn2, 1); @@ -3542,7 +5134,7 @@ mod tests { } #[test] - fn test_merged_generations_conflict_lower_generation_fails() { + fn test_compacted_sstables_conflict_lower_generation_fails() { // Test: committed generation >= to_commit generation should be incompatible (no retry) let shard = Uuid::new_v4(); @@ -3550,7 +5142,7 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -3558,7 +5150,7 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], }, None, ); @@ -3569,7 +5161,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3581,14 +5173,14 @@ mod tests { } #[test] - fn test_merged_generations_conflict_equal_generation_fails() { + fn test_compacted_sstables_conflict_equal_generation_fails() { // Test: committed generation == to_commit generation should be incompatible (no retry) let shard = Uuid::new_v4(); let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -3596,7 +5188,7 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -3607,7 +5199,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3619,7 +5211,7 @@ mod tests { } #[test] - fn test_merged_generations_conflict_higher_generation_retryable() { + fn test_compacted_sstables_conflict_higher_generation_retryable() { // Test: committed generation < to_commit generation should be retryable let shard = Uuid::new_v4(); @@ -3627,7 +5219,7 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], }, None, ); @@ -3635,7 +5227,7 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -3646,7 +5238,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3658,7 +5250,7 @@ mod tests { } #[test] - fn test_merged_generations_different_shards_ok() { + fn test_compacted_sstables_different_shards_ok() { // Test: different shards should not conflict let shard1 = Uuid::new_v4(); let shard2 = Uuid::new_v4(); @@ -3666,7 +5258,7 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard1, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard1, 10)], }, None, ); @@ -3674,7 +5266,7 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard2, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard2, 5)], }, None, ); @@ -3685,7 +5277,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3697,15 +5289,15 @@ mod tests { } #[test] - fn test_update_mem_wal_state_vs_create_index_with_merged_generations() { + fn test_update_mem_wal_state_vs_create_index_with_compacted_sstables() { use crate::index::mem_wal::new_mem_wal_index_meta; use lance_index::mem_wal::MemWalIndexDetails; let shard = Uuid::new_v4(); - // Create a MemWalIndex with merged_generations + // Create a MemWalIndex with compacted_sstables let details = MemWalIndexDetails { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], ..Default::default() }; let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap(); @@ -3724,7 +5316,7 @@ mod tests { let to_commit_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 5)], + compacted_sstables: vec![CompactedSsTable::new(shard, 5)], }, None, ); @@ -3735,7 +5327,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result = rebase.check_txn(&committed_txn, 1); @@ -3749,7 +5341,7 @@ mod tests { let to_commit_txn_higher = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 15)], + compacted_sstables: vec![CompactedSsTable::new(shard, 15)], }, None, ); @@ -3760,7 +5352,7 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; let result_higher = rebase_higher.check_txn(&committed_txn, 1); @@ -3778,7 +5370,7 @@ mod tests { let shard = Uuid::new_v4(); - // CreateIndex with MemWalIndex (no merged_generations initially) + // CreateIndex with MemWalIndex (no compacted_sstables initially) let details = MemWalIndexDetails::default(); let mem_wal_index = new_mem_wal_index_meta(1, details).unwrap(); @@ -3795,7 +5387,7 @@ mod tests { let committed_txn = Transaction::new( 0, Operation::UpdateMemWalState { - merged_generations: vec![MergedGeneration::new(shard, 10)], + compacted_sstables: vec![CompactedSsTable::new(shard, 10)], }, None, ); @@ -3806,11 +5398,11 @@ mod tests { modified_fragment_ids: HashSet::new(), affected_rows: None, conflicting_frag_reuse_indices: Vec::new(), - conflicting_mem_wal_merged_gens: Vec::new(), + conflicting_mem_wal_compacted_sstables: Vec::new(), }; // CreateIndex of MemWalIndex should be compatible with UpdateMemWalState - // and should collect the merged_generations for rebasing + // and should collect the compacted_sstables for rebasing let result = rebase.check_txn(&committed_txn, 1); assert!( result.is_ok(), @@ -3818,10 +5410,16 @@ mod tests { result ); - // Verify that merged_generations were collected - assert_eq!(rebase.conflicting_mem_wal_merged_gens.len(), 1); - assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].shard_id, shard); - assert_eq!(rebase.conflicting_mem_wal_merged_gens[0].generation, 10); + // Verify that compacted_sstables were collected + assert_eq!(rebase.conflicting_mem_wal_compacted_sstables.len(), 1); + assert_eq!( + rebase.conflicting_mem_wal_compacted_sstables[0].shard_id, + shard + ); + assert_eq!( + rebase.conflicting_mem_wal_compacted_sstables[0].generation, + 10 + ); } #[tokio::test] diff --git a/rust/lance/src/io/commit/external_manifest.rs b/rust/lance/src/io/commit/external_manifest.rs index 850d10f9a23..ff9b48df11b 100644 --- a/rust/lance/src/io/commit/external_manifest.rs +++ b/rust/lance/src/io/commit/external_manifest.rs @@ -4,20 +4,21 @@ /// Keep the tests in `lance` crate because it has dependency on [Dataset]. #[cfg(test)] mod test { + use std::collections::HashMap; use std::ops::Range; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; - use std::{collections::HashMap, time::Duration}; use async_trait::async_trait; use bytes::Bytes; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt, future::join_all}; use lance_core::{Error, Result}; + use lance_io::object_store::ObjectStore; use lance_table::io::commit::external_manifest::{ ExternalManifestCommitHandler, ExternalManifestStore, }; - use lance_table::io::commit::{CommitHandler, ManifestNamingScheme}; + use lance_table::io::commit::{CommitHandler, ManifestLocation, ManifestNamingScheme}; use lance_testing::datagen::{BatchGenerator, IncrementingInt32}; use object_store::memory::InMemory; use object_store::{ @@ -25,7 +26,7 @@ mod test { ObjectStore as OSObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult, local::LocalFileSystem, path::Path, }; - use tokio::sync::Mutex; + use tokio::sync::{Barrier, Mutex}; use crate::dataset::builder::DatasetBuilder; use crate::{ @@ -34,22 +35,30 @@ mod test { }; use lance_core::utils::tempfile::TempStrDir; - // sleep for 1 second to simulate a slow external store on write #[derive(Debug)] - struct SleepyExternalManifestStore { + struct TestExternalManifestStore { store: Arc>>, + contention: Option<(u64, Arc)>, } - impl SleepyExternalManifestStore { + impl TestExternalManifestStore { fn new() -> Self { Self { store: Arc::new(Mutex::new(HashMap::new())), + contention: None, + } + } + + fn with_contention(version: u64, participants: usize) -> Self { + Self { + contention: Some((version, Arc::new(Barrier::new(participants)))), + ..Self::new() } } } #[async_trait] - impl ExternalManifestStore for SleepyExternalManifestStore { + impl ExternalManifestStore for TestExternalManifestStore { /// Get the manifest path for a given uri and version async fn get(&self, uri: &str, version: u64) -> Result { let store = self.store.lock().await; @@ -85,7 +94,14 @@ mod test { _size: u64, _e_tag: Option, ) -> Result<()> { - tokio::time::sleep(Duration::from_millis(100)).await; + if let Some((contended_version, barrier)) = &self.contention + && version == *contended_version + { + // Every writer reaches the external compare-and-set with the same + // proposed version before any writer can publish it. This forces + // the retry path deterministically instead of relying on sleeps. + barrier.wait().await; + } let mut store = self.store.lock().await; match store.get(&(uri.to_string(), version)) { @@ -109,8 +125,6 @@ mod test { _size: u64, _e_tag: Option, ) -> Result<()> { - tokio::time::sleep(Duration::from_millis(100)).await; - let mut store = self.store.lock().await; match store.get(&(uri.to_string(), version)) { Some(_) => { @@ -125,6 +139,79 @@ mod test { } } + #[derive(Debug)] + struct StaticExternalManifestStore { + location: ManifestLocation, + verify_store: Option>, + } + + #[async_trait] + impl ExternalManifestStore for StaticExternalManifestStore { + async fn get(&self, _uri: &str, version: u64) -> Result { + if version == self.location.version { + Ok(self.location.path.to_string()) + } else { + Err(Error::not_found(format!("version {}", version))) + } + } + + async fn get_manifest_location( + &self, + _base_uri: &str, + version: u64, + ) -> Result { + if version == self.location.version { + Ok(self.location.clone()) + } else { + Err(Error::not_found(format!("version {}", version))) + } + } + + async fn get_latest_version(&self, _uri: &str) -> Result> { + Ok(Some(( + self.location.version, + self.location.path.to_string(), + ))) + } + + async fn get_latest_manifest_location( + &self, + _base_uri: &str, + ) -> Result> { + Ok(Some(self.location.clone())) + } + + async fn put_if_not_exists( + &self, + _uri: &str, + _version: u64, + _path: &str, + _size: u64, + _e_tag: Option, + ) -> Result<()> { + Ok(()) + } + + async fn put_if_exists( + &self, + _uri: &str, + _version: u64, + path: &str, + size: u64, + e_tag: Option, + ) -> Result<()> { + if let Some(store) = &self.verify_store { + let final_meta = store.head(&Path::from(path)).await?; + assert_eq!(size, final_meta.size); + assert_eq!( + e_tag, None, + "the generic workflow must not persist a physical generation" + ); + } + Ok(()) + } + } + fn read_params(handler: Arc) -> ReadParams { ReadParams { commit_handler: Some(handler), @@ -139,6 +226,256 @@ mod test { } } + #[tokio::test] + async fn finalized_external_manifest_location_is_head_checked() { + let sleepy_store = TestExternalManifestStore::new(); + let inner_store = sleepy_store.store.clone(); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(sleepy_store), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let body = b"manifest body"; + + object_store + .inner + .put(&final_path, PutPayload::from_static(body)) + .await + .expect("seed finalized manifest"); + inner_store + .lock() + .await + .insert((base_path.to_string(), version), final_path.to_string()); + + let location = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect("resolve latest finalized manifest"); + + assert_eq!(location.path, final_path); + assert_eq!(location.size, Some(body.len() as u64)); + assert_eq!(location.naming_scheme, ManifestNamingScheme::V2); + } + + #[tokio::test] + async fn finalized_external_manifest_location_falls_back_to_v1() { + let sleepy_store = TestExternalManifestStore::new(); + let inner_store = sleepy_store.store.clone(); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(sleepy_store), + }; + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let missing_v2_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let v1_path = ManifestNamingScheme::V1.manifest_path(&base_path, version); + + object_store + .inner + .put(&v1_path, PutPayload::from_static(b"v1 manifest body")) + .await + .expect("seed V1 manifest"); + inner_store.lock().await.insert( + (base_path.to_string(), version), + missing_v2_path.to_string(), + ); + + let latest_location = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect("resolve latest should fall back to V1"); + assert_eq!(latest_location.path, v1_path); + assert_eq!(latest_location.naming_scheme, ManifestNamingScheme::V1); + + let version_location = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("resolve version should fall back to V1"); + assert_eq!(version_location.path, v1_path); + assert_eq!(version_location.naming_scheme, ManifestNamingScheme::V1); + } + + #[tokio::test] + async fn finalized_external_manifest_location_rejects_size_mismatch() { + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let body = b"manifest body"; + + object_store + .inner + .put(&final_path, PutPayload::from_static(body)) + .await + .expect("seed finalized manifest"); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(StaticExternalManifestStore { + location: ManifestLocation { + version, + path: final_path, + size: Some(body.len() as u64 + 1), + naming_scheme: ManifestNamingScheme::V2, + e_tag: None, + identity: None, + }, + verify_store: None, + }), + }; + + let err = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect_err("stale external manifest size should be rejected"); + assert!(matches!(err, Error::CorruptFile { .. }), "{err:?}"); + assert!(err.to_string().contains("Manifest size mismatch"), "{err}"); + } + + #[tokio::test] + async fn finalized_external_manifest_location_without_stored_etag_uses_current_etag() { + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 7; + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, version); + let body = b"manifest body"; + + object_store + .inner + .put(&final_path, PutPayload::from_static(body)) + .await + .expect("seed finalized manifest"); + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(StaticExternalManifestStore { + location: ManifestLocation { + version, + path: final_path, + size: Some(body.len() as u64), + naming_scheme: ManifestNamingScheme::V2, + e_tag: None, + identity: None, + }, + verify_store: None, + }), + }; + + let resolved = handler + .resolve_latest_location(&base_path, &object_store) + .await + .expect("the canonical manifest should resolve from object storage"); + let current_meta = object_store + .inner + .head(&resolved.path) + .await + .expect("read current object-store metadata"); + assert_eq!(resolved.e_tag, current_meta.e_tag); + } + + #[tokio::test] + async fn external_manifest_store_put_returns_destination_etag() { + let object_store: Arc = Arc::new(InMemory::new()); + let base_path = Path::from("repro"); + let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd"); + object_store + .put(&staging_path, PutPayload::from_static(b"manifest body")) + .await + .expect("seed staging manifest"); + let staging_meta = object_store + .head(&staging_path) + .await + .expect("read staging metadata"); + + let external_store = StaticExternalManifestStore { + location: ManifestLocation { + version: 1, + path: staging_path.clone(), + size: Some(staging_meta.size), + naming_scheme: ManifestNamingScheme::V2, + e_tag: staging_meta.e_tag.clone(), + identity: None, + }, + verify_store: Some(object_store.clone()), + }; + let location = external_store + .put( + &base_path, + 1, + &staging_path, + staging_meta.size, + staging_meta.e_tag.clone(), + object_store.as_ref(), + ManifestNamingScheme::V2, + ) + .await + .expect("finalize manifest"); + let final_meta = object_store + .head(&location.path) + .await + .expect("read finalized metadata"); + + assert_ne!( + staging_meta.e_tag, final_meta.e_tag, + "test store must assign a new ETag to the copied object" + ); + assert_eq!(location.size, Some(final_meta.size)); + assert_eq!( + location.e_tag, final_meta.e_tag, + "the caller must receive the finalized physical generation" + ); + } + + #[tokio::test] + async fn external_manifest_handler_finalize_returns_destination_etag() { + let object_store = ObjectStore::memory(); + let base_path = Path::from("repro"); + let version = 1; + let staging_path = Path::from("repro/_versions/1.manifest.staging-abcd"); + object_store + .inner + .put(&staging_path, PutPayload::from_static(b"manifest body")) + .await + .expect("seed staging manifest"); + let staging_meta = object_store + .inner + .head(&staging_path) + .await + .expect("read staging metadata"); + + let handler = ExternalManifestCommitHandler { + external_manifest_store: Arc::new(StaticExternalManifestStore { + location: ManifestLocation { + version, + path: staging_path, + size: Some(staging_meta.size), + naming_scheme: ManifestNamingScheme::V2, + e_tag: staging_meta.e_tag.clone(), + identity: None, + }, + verify_store: Some(object_store.inner.clone()), + }), + }; + + let location = handler + .resolve_version_location(&base_path, version, object_store.inner.as_ref()) + .await + .expect("finalize manifest"); + let final_meta = object_store + .inner + .head(&location.path) + .await + .expect("read finalized metadata"); + + assert_ne!( + staging_meta.e_tag, final_meta.e_tag, + "test store must assign a new ETag to the copied object" + ); + assert_eq!(location.size, Some(final_meta.size)); + assert_eq!( + location.e_tag, final_meta.e_tag, + "the caller must receive the finalized physical generation" + ); + } + #[tokio::test] async fn test_dataset_can_onboard_external_store() { // First write a dataset WITHOUT external store @@ -150,7 +487,7 @@ mod test { Dataset::write(reader, ds_uri, None).await.unwrap(); // Then try to load the dataset with external store handler set - let sleepy_store = SleepyExternalManifestStore::new(); + let sleepy_store = TestExternalManifestStore::new(); let handler = Arc::new(ExternalManifestCommitHandler { external_manifest_store: Arc::new(sleepy_store), }); @@ -177,7 +514,7 @@ mod test { #[tokio::test] #[cfg(not(windows))] async fn test_can_create_dataset_with_external_store() { - let sleepy_store = SleepyExternalManifestStore::new(); + let sleepy_store = TestExternalManifestStore::new(); let handler = ExternalManifestCommitHandler { external_manifest_store: Arc::new(sleepy_store), }; @@ -204,96 +541,96 @@ mod test { #[cfg(not(windows))] #[tokio::test] async fn test_concurrent_commits_are_okay() { - // Run test 20 times to have a higher chance of catching race conditions - for _ in 0..20 { - let sleepy_store = SleepyExternalManifestStore::new(); - let handler = ExternalManifestCommitHandler { - external_manifest_store: Arc::new(sleepy_store), - }; - let handler = Arc::new(handler); - - let mut data_gen = - BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("x".to_owned()))); - let dir = TempStrDir::default(); - let ds_uri = &dir; - - Dataset::write( - data_gen.batch(10), - ds_uri, - Some(write_params(handler.clone())), - ) - .await - .unwrap(); + const NUM_WRITERS: usize = 5; + const CONTENDED_VERSION: u64 = 2; + let external_store = + TestExternalManifestStore::with_contention(CONTENDED_VERSION, NUM_WRITERS); + let handler = Arc::new(ExternalManifestCommitHandler { + external_manifest_store: Arc::new(external_store), + }); - // we have 5 retries by default, more than this will just fail - let write_futs = (0..5) - .map(|_| data_gen.batch(10)) - .map(|data| { - let mut params = write_params(handler.clone()); - params.mode = WriteMode::Append; - Dataset::write(data, ds_uri, Some(params)) - }) - .collect::>(); + let mut data_gen = + BatchGenerator::new().col(Box::new(IncrementingInt32::new().named("x".to_owned()))); + let dir = TempStrDir::default(); + let ds_uri = &dir; + + Dataset::write( + data_gen.batch(10), + ds_uri, + Some(write_params(handler.clone())), + ) + .await + .unwrap(); - let res = join_all(write_futs).await; + // All writers first attempt version 2. One succeeds and the rest must + // observe the conflict, refresh, and commit distinct later versions. + let write_futs = (0..NUM_WRITERS) + .map(|_| data_gen.batch(10)) + .map(|data| { + let mut params = write_params(handler.clone()); + params.mode = WriteMode::Append; + Dataset::write(data, ds_uri, Some(params)) + }) + .collect::>(); - let errors = res - .into_iter() - .filter(|r| r.is_err()) - .map(|r| r.unwrap_err()) - .collect::>(); + let res = join_all(write_futs).await; - assert!(errors.is_empty(), "{:?}", errors); + let errors = res + .into_iter() + .filter(|r| r.is_err()) + .map(|r| r.unwrap_err()) + .collect::>(); - // load the data and check the content - let ds = DatasetBuilder::from_uri(ds_uri) - .with_read_params(read_params(handler)) - .load() - .await - .unwrap(); - assert_eq!(ds.count_rows(None).await.unwrap(), 60); + assert!(errors.is_empty(), "{:?}", errors); - // No temporary manifests left over - let manifest_path = format!("{}/{}", dir, "_versions/"); - let unexpected_entries = std::fs::read_dir(manifest_path) - .unwrap() - .filter(|entry| { - let entry = entry.as_ref().unwrap(); - !entry - .file_name() - .as_os_str() - .to_string_lossy() - .ends_with(".manifest") - }) - // There is a bug in local fs where concurrent commits can leave behind - // temporary `x.manifest#n` files. This might be a bug in object-store. - // TODO: fix this. - .filter(|entry| { - let entry = entry.as_ref().unwrap(); - !entry - .file_name() - .as_os_str() - .to_string_lossy() - .contains(".manifest#") - }) - // The version hint file is expected to be present. - .filter(|entry| { - let entry = entry.as_ref().unwrap(); - !entry - .file_name() - .as_os_str() - .to_string_lossy() - .starts_with("latest_version_hint") - }) - .collect::>(); - assert!(unexpected_entries.is_empty(), "{:?}", unexpected_entries); - } + let ds = DatasetBuilder::from_uri(ds_uri) + .with_read_params(read_params(handler)) + .load() + .await + .unwrap(); + assert_eq!(ds.count_rows(None).await.unwrap(), 60); + assert_eq!(ds.version().version, (NUM_WRITERS + 1) as u64); + + // No temporary manifests left over. + let manifest_path = format!("{}/{}", dir, "_versions/"); + let unexpected_entries = std::fs::read_dir(manifest_path) + .unwrap() + .filter(|entry| { + let entry = entry.as_ref().unwrap(); + !entry + .file_name() + .as_os_str() + .to_string_lossy() + .ends_with(".manifest") + }) + // There is a bug in local fs where concurrent commits can leave behind + // temporary `x.manifest#n` files. This might be a bug in object-store. + // TODO: fix this. + .filter(|entry| { + let entry = entry.as_ref().unwrap(); + !entry + .file_name() + .as_os_str() + .to_string_lossy() + .contains(".manifest#") + }) + // The version hint file is expected to be present. + .filter(|entry| { + let entry = entry.as_ref().unwrap(); + !entry + .file_name() + .as_os_str() + .to_string_lossy() + .starts_with("latest_version_hint") + }) + .collect::>(); + assert!(unexpected_entries.is_empty(), "{:?}", unexpected_entries); } #[tokio::test] #[cfg(not(windows))] async fn test_out_of_sync_dataset_can_recover() { - let sleepy_store = SleepyExternalManifestStore::new(); + let sleepy_store = TestExternalManifestStore::new(); let inner_store = sleepy_store.store.clone(); let handler = ExternalManifestCommitHandler { external_manifest_store: Arc::new(sleepy_store), @@ -592,10 +929,8 @@ mod test { /// our `CopyCapStore` wrapper rejects with the same `EntityTooLarge` /// error S3 returns in production. /// - /// Today this test is RED: the copy step fails on >5 GB. - /// After `copy_size_aware` lands, it should turn GREEN by falling back - /// to a multipart-equivalent path (option 1: read+rewrite via - /// `ObjectWriter`). + /// The regression verifies that `copy_size_aware` falls back to the + /// multipart-equivalent read+rewrite path instead of calling CopyObject. #[tokio::test] async fn manifest_commit_succeeds_when_staging_exceeds_5gb_copy_cap() { let inner: Arc = Arc::new(InMemory::new()); @@ -616,8 +951,14 @@ mod test { // Spin up an ExternalManifestStore and drive `put` (the same code // path the failing CTAS hits via ExternalManifestCommitHandler). - let external = SleepyExternalManifestStore::new(); + let external = TestExternalManifestStore::new(); let head_meta = capped.head(&staging_path).await.unwrap(); + let final_path = ManifestNamingScheme::V2.manifest_path(&base_path, 1); + // The fixture stores only a tiny body, so its source-size override must + // also apply to the destination metadata. This keeps the fake object + // store internally consistent with the real 14 GB object it models + // when finalization verifies that the copy preserved size. + capped.override_size(&final_path, head_meta.size).await; let location = external .put( @@ -686,7 +1027,7 @@ mod test { // well below the 5 GB cap, so copy_size_aware must take the fast // path. - let external = SleepyExternalManifestStore::new(); + let external = TestExternalManifestStore::new(); let head_meta = capped.head(&staging_path).await.unwrap(); external diff --git a/rust/lance/src/io/commit/namespace_manifest.rs b/rust/lance/src/io/commit/namespace_manifest.rs index f4f012adcca..fb3dffd6c7c 100644 --- a/rust/lance/src/io/commit/namespace_manifest.rs +++ b/rust/lance/src/io/commit/namespace_manifest.rs @@ -177,6 +177,7 @@ impl ExternalManifestStore for LanceNamespaceExternalManifestStore { size: version_info.manifest_size.map(|s| s as u64), naming_scheme, e_tag: version_info.e_tag, + identity: None, }) } diff --git a/rust/lance/src/io/commit/s3_test.rs b/rust/lance/src/io/commit/s3_test.rs index b5b1a09c776..b942e12d40c 100644 --- a/rust/lance/src/io/commit/s3_test.rs +++ b/rust/lance/src/io/commit/s3_test.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use arrow::datatypes::Int32Type; @@ -11,6 +11,7 @@ use crate::{ io::{ObjectStoreParams, StorageOptionsAccessor}, }; use aws_config::{BehaviorVersion, ConfigLoader, Region, SdkConfig}; +use aws_sdk_dynamodb::types::AttributeValue; use aws_sdk_s3::{Client as S3Client, config::Credentials}; use futures::future::try_join_all; use lance_datagen::{RowCount, array, gen_batch}; @@ -149,6 +150,47 @@ impl DynamoDBCommitTable { Self(name.to_string()) } + async fn item_for_version(&self, expected_version: u64) -> HashMap { + let config = aws_config().await; + let client = aws_sdk_dynamodb::Client::new(&config); + let expected_version_string = expected_version.to_string(); + client + .scan() + .table_name(&self.0) + .consistent_read(true) + .send() + .await + .unwrap() + .items + .unwrap_or_default() + .into_iter() + .find(|item| { + item.get("version").and_then(|value| value.as_n().ok()) + == Some(&expected_version_string) + }) + .unwrap_or_else(|| panic!("DynamoDB row for version {expected_version} not found")) + } + + async fn set_legacy_etag(&self, version: u64, e_tag: &str) { + let item = self.item_for_version(version).await; + let base_uri = item + .get("base_uri") + .and_then(|value| value.as_s().ok()) + .expect("DynamoDB row must contain a string base_uri") + .clone(); + let config = aws_config().await; + aws_sdk_dynamodb::Client::new(&config) + .update_item() + .table_name(&self.0) + .key("base_uri", AttributeValue::S(base_uri)) + .key("version", AttributeValue::N(version.to_string())) + .update_expression("SET e_tag = :e_tag") + .expression_attribute_values(":e_tag", AttributeValue::S(e_tag.to_string())) + .send() + .await + .unwrap(); + } + async fn delete_table(client: aws_sdk_dynamodb::Client, name: &str) { match client .delete_table() @@ -304,9 +346,25 @@ async fn test_ddb_open_iops() { // * write staged file // * copy to final file // * delete staged file + // Commit: 2 read IOPs: one to list versions before creating the dataset and + // one to HEAD the canonical manifest after COPY. DynamoDB does not persist + // that ETag, but the freshly committed Dataset needs the observed physical + // generation so downstream caches cannot reuse an older Dataset at the + // same URI and version. let io_stats = committed_ds.object_store.as_ref().io_stats_incremental(); assert_io_eq!(io_stats, write_iops, 4); - assert_io_eq!(io_stats, read_iops, 1); + assert_io_eq!(io_stats, read_iops, 2); + assert!(committed_ds.manifest_location().e_tag.is_some()); + + let committed_row = ddb_table.item_for_version(1).await; + assert!( + !committed_row.contains_key("e_tag"), + "DynamoDB must not persist a physical object generation" + ); + // Simulate a row written by an older Lance version. New DynamoDB readers + // ignore this physical-generation token instead of treating it as logical + // manifest identity. + ddb_table.set_legacy_etag(1, "legacy-stale-etag").await; let dataset = DatasetBuilder::from_uri(&uri) .with_read_params(ReadParams { @@ -316,10 +374,15 @@ async fn test_ddb_open_iops() { .load() .await .unwrap(); + assert_ne!( + dataset.manifest_location().e_tag.as_deref(), + Some("legacy-stale-etag") + ); let io_stats = dataset.object_store.as_ref().io_stats_incremental(); - // Open dataset can be read with 1 IOP, just to read the manifest. - // Looking up latest manifest is handled in dynamodb. - assert_io_eq!(io_stats, read_iops, 1); + // Open dataset can be read with 2 IOPs: HEAD verifies that the + // finalized path returned by DynamoDB still exists, then the manifest + // itself is read. Looking up the latest manifest is handled in DynamoDB. + assert_io_eq!(io_stats, read_iops, 2); assert_io_eq!(io_stats, write_iops, 0); // Append @@ -334,14 +397,19 @@ async fn test_ddb_open_iops() { let io_stats = dataset.object_store.as_ref().io_stats_incremental(); // Append: 5 IOPS: data file, transaction file, 3x manifest file assert_io_eq!(io_stats, write_iops, 5); + // Append reads once to list versions and once to observe the canonical + // generation after COPY. DDB stores only the stable final path and size; + // the returned Dataset retains the observed ETag. // TODO: we can reduce this by implementing a specialized CommitHandler::list_manifest_locations() // for the DDB commit handler. - assert_io_eq!(io_stats, read_iops, 1); + assert_io_eq!(io_stats, read_iops, 2); + assert!(dataset.manifest_location().e_tag.is_some()); // Checkout original version dataset.checkout_version(1).await.unwrap(); let io_stats = dataset.object_store.as_ref().io_stats_incremental(); - // Checkout: 1 IOPS: manifest file + // Checkout: 1 read IOP. Version resolution HEAD-checks the finalized path, + // while the manifest body is served from the Session metadata cache. assert_io_eq!(io_stats, read_iops, 1); assert_io_eq!(io_stats, write_iops, 0); } diff --git a/rust/lance/src/io/exec.rs b/rust/lance/src/io/exec.rs index a477d60d56d..d37b58a238e 100644 --- a/rust/lance/src/io/exec.rs +++ b/rust/lance/src/io/exec.rs @@ -18,6 +18,7 @@ pub(crate) mod knn; mod optimizer; mod projection; mod pushdown_scan; +pub(crate) mod row_addr_mask; mod rowids; pub mod scalar_index; mod scan; @@ -35,7 +36,9 @@ pub use lance_index::scalar::expression::FilterPlan; pub use optimizer::get_physical_optimizer; pub use projection::project; pub use pushdown_scan::{LancePushdownScanExec, ScanConfig}; +pub use row_addr_mask::RowAddrMaskFilterExec; pub use rowids::{AddRowAddrExec, AddRowOffsetExec}; +pub(crate) use scan::LanceStream; pub use scan::{LanceScanConfig, LanceScanExec}; pub use take::TakeExec; pub use utils::PreFilterSource; diff --git a/rust/lance/src/io/exec/count_from_mask.rs b/rust/lance/src/io/exec/count_from_mask.rs index 0b7aeb11111..e2f7f00efb5 100644 --- a/rust/lance/src/io/exec/count_from_mask.rs +++ b/rust/lance/src/io/exec/count_from_mask.rs @@ -395,10 +395,6 @@ impl ExecutionPlan for CountFromMaskExec { "CountFromMaskExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.schema.clone() } @@ -463,11 +459,11 @@ impl ExecutionPlan for CountFromMaskExec { fn partition_statistics( &self, _partition: Option, - ) -> datafusion::error::Result { - Ok(datafusion::physical_plan::Statistics { + ) -> datafusion::error::Result> { + Ok(Arc::new(datafusion::physical_plan::Statistics { num_rows: datafusion::common::stats::Precision::Exact(1), ..datafusion::physical_plan::Statistics::new_unknown(&self.schema) - }) + })) } fn metrics(&self) -> Option { @@ -494,7 +490,6 @@ mod tests { use datafusion::logical_expr::lit; use datafusion::physical_expr::execution_props::ExecutionProps; use datafusion::physical_plan::ExecutionPlan; - use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter; use datafusion::scalar::ScalarValue; use futures::TryStreamExt; use lance_core::utils::tempfile::TempStrDir; @@ -512,8 +507,13 @@ mod tests { use crate::index::DatasetIndexExt; use crate::io::exec::scalar_index::ScalarIndexExec; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + #[allow(deprecated)] + use datafusion::physical_planner::create_aggregate_expr_and_maybe_filter; /// Build an `AggregateFunctionExpr` matching `COUNT(*)`. + // TODO(datafusion-54): migrate off the deprecated + // create_aggregate_expr_and_maybe_filter to LoweredAggregateBuilder. + #[allow(deprecated)] fn count_star_expr(input_schema: &SchemaRef) -> Arc { let expr = functions_aggregate::count::count(lit(1)); let df_schema = DFSchema::try_from(input_schema.as_ref().clone()).unwrap(); diff --git a/rust/lance/src/io/exec/count_pushdown.rs b/rust/lance/src/io/exec/count_pushdown.rs index d5d90b5881a..9da8f4842f1 100644 --- a/rust/lance/src/io/exec/count_pushdown.rs +++ b/rust/lance/src/io/exec/count_pushdown.rs @@ -13,16 +13,17 @@ //! enough to be reused. //! //! Two rewritten shapes are emitted depending on whether the scalar index -//! backing the filter covers every dataset fragment. +//! backing the filter covers every fragment targeted by the scan. //! -//! **Full coverage** (index ⊇ dataset, or no filter at all): +//! **Full coverage** (index ⊇ targeted fragments, or no filter at all): //! //! ```text //! AggregateExec(Final, aggs=[count(...)], group_by=[]) //! └── CountFromMaskExec { prefilter_input = index_input } //! ``` //! -//! **Partial coverage** (index ⊊ dataset — typically appended fragments): +//! **Partial coverage** (index misses some targeted fragments — typically +//! appended fragments): //! //! ```text //! AggregateExec(Final, aggs=[count(...)], group_by=[]) @@ -83,7 +84,7 @@ impl PhysicalOptimizerRule for CountPushdown { ) -> DFResult> { Ok(plan .transform_down(|plan| { - let Some(agg) = plan.as_any().downcast_ref::() else { + let Some(agg) = plan.downcast_ref::() else { return Ok(Transformed::no(plan)); }; if let Some(rewritten) = try_rewrite(agg)? { @@ -147,6 +148,11 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> }; let options = filtered_read.options(); + // We don't currently support count pushdown when the row selector + // is a row stream. + if filtered_read.row_stream_input().is_some() { + return Ok(None); + } // A refine filter is a residual the index couldn't fully evaluate — it // needs column data to apply, which we can't. if options.refine_filter.is_some() { @@ -173,21 +179,38 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> ); return Ok(None); } - // Same story for an explicit fragment subset: legitimate, but unexpected - // alongside an aggregate, and we lose the pushdown opportunity. - if options.fragments.is_some() { - warn!( - "count_pushdown: skipped because the FilteredReadExec was scoped \ - to an explicit fragment subset; the count will be computed via a \ - full scan. Intersecting that subset into the coverage logic would \ - let this query be answered from index metadata." - ); - return Ok(None); - } - let dataset = filtered_read.dataset().clone(); let dataset_fragments: RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect(); + let fragment_scope = if let Some(fragments) = options.fragments.as_ref() { + let fragment_scope = fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect::(); + // A bitmap cannot preserve duplicate fragments, and CountFromMaskExec + // resolves fragment IDs against the current manifest instead of using + // the descriptors supplied to FilteredReadExec. + let has_duplicate_fragments = fragment_scope.len() != fragments.len() as u64; + let descriptors_are_current = fragments.iter().all(|fragment| { + let Ok(fragment_id) = u32::try_from(fragment.id) else { + return false; + }; + if !dataset.fragment_bitmap.contains(fragment_id) { + return false; + } + let fragment_index = dataset.fragment_bitmap.rank(fragment_id) as usize - 1; + dataset.fragments().get(fragment_index) == Some(fragment) + }); + if has_duplicate_fragments || !descriptors_are_current { + return Ok(None); + } + Some(fragment_scope) + } else { + None + }; + let target_fragments = fragment_scope + .clone() + .unwrap_or_else(|| dataset_fragments.clone()); let prefilter_input = filtered_read.index_input().cloned(); // If there is a prefilter, inspect its ScalarIndexExpr leaves: @@ -202,15 +225,12 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> let index_coverage = match &prefilter_input { None => None, Some(input) => { - let scalar_exec = input - .as_any() - .downcast_ref::() - .ok_or_else(|| { - datafusion::error::DataFusionError::Internal( - "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec" - .to_string(), - ) - })?; + let scalar_exec = input.downcast_ref::().ok_or_else(|| { + datafusion::error::DataFusionError::Internal( + "count_pushdown: FilteredReadExec.index_input is not a ScalarIndexExec" + .to_string(), + ) + })?; if scalar_exec.expr().needs_recheck() { return Ok(None); } @@ -226,11 +246,11 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> // Decide on the plan shape. Three cases: // // 1. No prefilter (no filter at all): single pushdown branch over every - // dataset fragment. Always safe. - // 2. Prefilter + index covers every dataset fragment: single pushdown + // targeted fragment. Always safe. + // 2. Prefilter + index covers every targeted fragment: single pushdown // branch, prefilter feeds in directly. - // 3. Prefilter + index covers a strict subset: split into pushdown over - // indexed fragments + parallel scan over unindexed fragments. + // 3. Prefilter + index covers a strict subset of the target: split into + // pushdown over indexed fragments + parallel scan over unindexed fragments. let (partial_stream, partial_state_schema): (Arc, _) = match index_coverage { None => { // No prefilter at all (verified above): nothing to restrict. @@ -238,32 +258,36 @@ fn try_rewrite(agg: &AggregateExec) -> DFResult>> dataset, aggr_exprs.clone(), prefilter_input, - None, + fragment_scope, )?; let schema = exec.schema(); (Arc::new(exec), schema) } - Some(coverage) if (&dataset_fragments - &coverage).is_empty() => { - // Prefilter exists and the index covers every dataset fragment — + Some(coverage) if (&target_fragments - &coverage).is_empty() => { + // Prefilter exists and the index covers every targeted fragment — // safe to push the whole count down. let exec = CountFromMaskExec::try_new_restricted( dataset, aggr_exprs.clone(), prefilter_input, - None, + fragment_scope, )?; let schema = exec.schema(); (Arc::new(exec), schema) } Some(coverage) => { - // Split plan: CountFromMaskExec for the indexed fragments, a - // normal scan + AggregateExec(Partial) for the rest. - let uncovered = &dataset_fragments - &coverage; + // Split plan: CountFromMaskExec for the targeted indexed fragments, + // a normal scan + AggregateExec(Partial) for the targeted remainder. + let covered = &target_fragments & &coverage; + if covered.is_empty() { + return Ok(None); + } + let uncovered = &target_fragments - &coverage; let pushdown_exec = CountFromMaskExec::try_new_restricted( dataset, aggr_exprs.clone(), prefilter_input, - Some(&dataset_fragments & &coverage), + Some(covered), )?; let partial_state_schema = pushdown_exec.schema(); let pushdown_branch: Arc = Arc::new(pushdown_exec); @@ -367,20 +391,21 @@ fn build_scan_branch( fn strip_row_preserving_wrappers(plan: &Arc) -> Option<&FilteredReadExec> { let mut current: &dyn ExecutionPlan = plan.as_ref(); loop { - if let Some(filtered_read) = current.as_any().downcast_ref::() { + if let Some(filtered_read) = current.downcast_ref::() { return Some(filtered_read); } let next: &Arc = - if let Some(inner) = current.as_any().downcast_ref::() { + if let Some(inner) = current.downcast_ref::() { inner.input() } else if let Some(inner) = { #[allow(deprecated)] - current.as_any().downcast_ref::() + current.downcast_ref::() } { inner.input() - } else if let Some(inner) = current.as_any().downcast_ref::() { + } else if let Some(inner) = current.downcast_ref::() { inner.input() - } else if let Some(proj) = current.as_any().downcast_ref::() { + } else { + let proj = current.downcast_ref::()?; // Only walk through projections that are row-preserving: every // output expression is a direct column reference back to the // input. (Empty projections trivially qualify — DataFusion uses @@ -390,7 +415,6 @@ fn strip_row_preserving_wrappers(plan: &Arc) -> Option<&Filte let identity = proj.expr().iter().all(|projection_expr| { projection_expr .expr - .as_any() .downcast_ref::() .is_some_and(|c| c.name() == input_schema.field(c.index()).name()) }); @@ -398,8 +422,6 @@ fn strip_row_preserving_wrappers(plan: &Arc) -> Option<&Filte return None; } proj.input() - } else { - return None; }; current = next.as_ref(); } @@ -434,7 +456,7 @@ fn is_count_star(af: &Arc) -> bool { if args.len() != 1 { return false; } - let Some(lit) = args[0].as_any().downcast_ref::() else { + let Some(lit) = args[0].downcast_ref::() else { return false; }; // `COUNT(NULL)` would always return 0; rule it out so we don't accidentally @@ -497,7 +519,7 @@ mod tests { fn plan_contains_pushdown(plan: &Arc) -> bool { let mut found = false; plan.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { @@ -511,7 +533,7 @@ mod tests { fn plan_contains_union(plan: &Arc) -> bool { let mut found = false; plan.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { @@ -587,6 +609,54 @@ mod tests { ); } + #[tokio::test] + async fn rule_fires_when_filter_is_scoped_to_fragment() { + let fixture = make_fixture().await; + let mut scanner = fixture.dataset.get_fragments()[1].scan(); + scanner.empty_project().unwrap().with_row_id(); + scanner.filter("ordered < 25").unwrap(); + + let (plan, count) = run_count(&mut scanner).await; + + assert_eq!(count, 10); + assert!( + plan_contains_pushdown(&plan), + "expected CountFromMaskExec for a fragment-scoped count: {}", + displayable(plan.as_ref()).indent(true) + ); + assert!( + !plan_contains_union(&plan), + "no union expected when the index covers the requested fragment, got: {}", + displayable(plan.as_ref()).indent(true) + ); + } + + #[tokio::test] + async fn count_matches_scan_for_stale_fragment_descriptor() { + let fixture = make_fixture().await; + let mut dataset = fixture.dataset.as_ref().clone(); + let stale_fragment = dataset.fragments()[0].clone(); + dataset.delete("ordered = 0").await.unwrap(); + let dataset = Arc::new(dataset); + + let mut scan = dataset.scan(); + scan.with_fragments(vec![stale_fragment.clone()]); + scan.filter("ordered < 10").unwrap(); + let scanned_rows = scan.try_into_batch().await.unwrap().num_rows() as i64; + + let mut count_scan = dataset.scan(); + count_scan.with_fragments(vec![stale_fragment]); + count_scan.filter("ordered < 10").unwrap(); + let (plan, count) = run_count(&mut count_scan).await; + + assert_eq!(count, scanned_rows); + assert!( + !plan_contains_pushdown(&plan), + "a stale fragment descriptor must retain the original scan plan: {}", + displayable(plan.as_ref()).indent(true) + ); + } + #[tokio::test] async fn rule_emits_split_plan_for_partial_index_coverage() { // Build index over 4 fragments, then append a 5th — the index now @@ -648,6 +718,43 @@ mod tests { "expected UnionExec for partial-coverage split, got: {}", displayable(plan.as_ref()).indent(true) ); + + let fragments = dataset.fragments(); + let mut indexed_fragment_scanner = dataset.scan(); + indexed_fragment_scanner + .with_fragments(vec![fragments[1].clone()]) + .filter("ordered < 100") + .unwrap(); + let (plan, count) = run_count(&mut indexed_fragment_scanner).await; + assert_eq!(count, 10); + assert!( + plan_contains_pushdown(&plan), + "expected pushdown when the index covers the requested fragment: {}", + displayable(plan.as_ref()).indent(true) + ); + assert!( + !plan_contains_union(&plan), + "unindexed fragments outside the requested scope must not add a scan branch: {}", + displayable(plan.as_ref()).indent(true) + ); + + let mut mixed_fragment_scanner = dataset.scan(); + mixed_fragment_scanner + .with_fragments(vec![fragments[1].clone(), fragments[4].clone()]) + .filter("ordered < 100") + .unwrap(); + let (plan, count) = run_count(&mut mixed_fragment_scanner).await; + assert_eq!(count, 20); + assert!( + plan_contains_pushdown(&plan), + "expected pushdown for the indexed requested fragment: {}", + displayable(plan.as_ref()).indent(true) + ); + assert!( + plan_contains_union(&plan), + "expected a scan branch for the unindexed requested fragment: {}", + displayable(plan.as_ref()).indent(true) + ); } #[tokio::test] @@ -730,6 +837,19 @@ mod tests { "rule should fire under stable row IDs with a filter, got plan: {}", displayable(plan.as_ref()).indent(true) ); + + let mut fragment_scanner = dataset.scan(); + fragment_scanner + .with_fragments(vec![dataset.fragments()[1].clone()]) + .filter("ordered >= 0") + .unwrap(); + let (plan, count) = run_count(&mut fragment_scanner).await; + assert_eq!(count, 9); + assert!( + plan_contains_pushdown(&plan), + "rule should push down a fragment-scoped count under stable row IDs: {}", + displayable(plan.as_ref()).indent(true) + ); } #[tokio::test] diff --git a/rust/lance/src/io/exec/filter.rs b/rust/lance/src/io/exec/filter.rs index 3a36f8d6712..f6d3005065e 100644 --- a/rust/lance/src/io/exec/filter.rs +++ b/rust/lance/src/io/exec/filter.rs @@ -3,7 +3,7 @@ use std::sync::Arc; -use datafusion::{execution::TaskContext, logical_expr::Expr}; +use datafusion::{catalog::Session, execution::TaskContext, logical_expr::Expr}; use datafusion_physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, Statistics, filter::FilterExec, metrics::MetricsSet, @@ -31,6 +31,24 @@ impl LanceFilterExec { pub fn try_new(expr: Expr, input: Arc) -> Result { let planner = Planner::new(input.schema()); let predicate = planner.create_physical_expr(&expr)?; + Self::try_new_with_predicate(expr, predicate, input) + } + + pub fn try_new_with_session( + expr: Expr, + input: Arc, + session: &dyn Session, + ) -> Result { + let planner = Planner::new(input.schema()); + let predicate = planner.create_physical_expr_with_session(&expr, session)?; + Self::try_new_with_predicate(expr, predicate, input) + } + + fn try_new_with_predicate( + expr: Expr, + predicate: Arc, + input: Arc, + ) -> Result { let filter_exec = FilterExec::try_new(predicate.clone(), input)?; Ok(Self { expr, @@ -48,10 +66,6 @@ impl ExecutionPlan for LanceFilterExec { "LanceFilterExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { self.filter.properties() } @@ -71,7 +85,6 @@ impl ExecutionPlan for LanceFilterExec { // Rewrap the result in a LanceFilterExec to preserve the logical expression let new_filter_plan = self.filter.clone().with_new_children(children)?; let new_filter = new_filter_plan - .as_any() .downcast_ref::() .expect("FilterExec::with_new_children should return FilterExec") .clone(); @@ -93,7 +106,7 @@ impl ExecutionPlan for LanceFilterExec { self.filter.metrics() } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn partition_statistics(&self, partition: Option) -> DataFusionResult> { self.filter.partition_statistics(partition) } diff --git a/rust/lance/src/io/exec/filtered_read.rs b/rust/lance/src/io/exec/filtered_read.rs index 059fa4e2b36..5688e77b547 100644 --- a/rust/lance/src/io/exec/filtered_read.rs +++ b/rust/lance/src/io/exec/filtered_read.rs @@ -1,20 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::collections::{BTreeMap, HashMap}; -use std::pin::Pin; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::{ops::Range, sync::Arc}; +use std::task::Poll; +use std::{ + ops::{Range, RangeInclusive}, + sync::Arc, +}; -use arrow_array::RecordBatch; -use arrow_schema::SchemaRef; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_array::{Array, BooleanArray, RecordBatch, RecordBatchOptions, UInt32Array}; +use arrow_schema::{Schema as ArrowSchema, SchemaRef}; +use datafusion::catalog::Session; use datafusion::common::runtime::SpawnedTask; use datafusion::common::stats::Precision; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::stream::{RecordBatchReceiverStream, RecordBatchStreamAdapter}; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, execution_plan::{Boundedness, EmissionType}, @@ -27,11 +32,13 @@ use datafusion_physical_plan::metrics::{BaselineMetrics, Count, MetricsSet, Time use futures::stream::BoxStream; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future}; use lance_arrow::RecordBatchExt; -use lance_core::datatypes::OnMissing; +use lance_core::datatypes::{OnMissing, Schema as LanceSchema}; use lance_core::utils::deletion::DeletionVector; use lance_core::utils::futures::FinallyStreamExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; -use lance_core::{Error, Result, datatypes::Projection}; +use lance_core::{ + Error, ROW_ADDR, ROW_ADDR_FIELD, ROW_ID, ROW_ID_FIELD, Result, datatypes::Projection, +}; use lance_datafusion::planner::Planner; use lance_datafusion::utils::{ ExecutionPlanMetricsSetExt, FRAGMENTS_SCANNED_METRIC, RANGES_SCANNED_METRIC, @@ -41,7 +48,8 @@ use lance_file::reader::FileReaderOptions; use lance_index::scalar::expression::FilterPlan; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_select::{ - IndexExprResult, RowAddrSelection, RowAddrTreeMap, bitmap_to_ranges, ranges_to_bitmap, + IndexExprResult, RowAddrMask, RowAddrSelection, RowAddrTreeMap, bitmap_to_ranges, + ranges_to_bitmap, result::IndexExprResultWireFormat, }; use lance_table::format::Fragment; use lance_table::rowids::RowIdSequence; @@ -51,22 +59,72 @@ use tokio::sync::{Mutex as AsyncMutex, OnceCell}; use tracing::{Instrument, instrument}; use crate::Dataset; +use crate::dataset::blob::{BlobMaterializationContext, MaterializedBlobBatch}; use crate::dataset::fragment::{FileFragment, FragReadConfig}; use crate::dataset::rowids::load_row_id_sequence; use crate::dataset::scanner::{ BATCH_SIZE_FALLBACK, DEFAULT_FRAGMENT_READAHEAD, get_default_batch_size, get_default_io_buffer_size_override, }; +use crate::dataset::versions; use super::utils::IoMetrics; +type MaterializedReadBatchFut = futures::future::BoxFuture<'static, Result>; + +fn public_blob_v2_binary_projection_schema(projection: &Projection) -> SchemaRef { + let schema = projection.to_schema(); + let schema = crate::dataset::blob::public_blob_v2_binary_output_schema(&schema); + let schema: ArrowSchema = (&schema).into(); + Arc::new(schema) +} + #[derive(Debug)] pub struct EvaluatedIndex { index_result: IndexExprResult, applicable_fragments: RoaringBitmap, } +// Keep common selective and dense masks single-pass without allowing highly fragmented stable-ID +// masks to retain unbounded request-scoped range storage before final planning. +const MAX_RETAINED_STABLE_INDEX_RANGE_BYTES: usize = 16 * 1024 * 1024; + +struct StableIndexRouting { + row_id_sequence: Arc, + upper_ranges: Option>>, +} + +enum FragmentMetadataLoad { + Skip, + Load, + LoadWithStableRouting(StableIndexRouting), +} + impl EvaluatedIndex { + fn retain_stable_index_ranges( + upper_ranges: Vec>, + retained_range_bytes: &AtomicUsize, + ) -> Option>> { + let allocated_bytes = upper_ranges + .capacity() + .saturating_mul(std::mem::size_of::>()); + let mut retained_bytes = retained_range_bytes.load(Ordering::Relaxed); + loop { + let new_total = retained_bytes + .checked_add(allocated_bytes) + .filter(|new_total| *new_total <= MAX_RETAINED_STABLE_INDEX_RANGE_BYTES)?; + match retained_range_bytes.compare_exchange_weak( + retained_bytes, + new_total, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return Some(upper_ranges), + Err(actual) => retained_bytes = actual, + } + } + } + /// Get the row id mask representing which rows matched the index filter. pub fn index_result(&self) -> &IndexExprResult { &self.index_result @@ -86,6 +144,68 @@ impl EvaluatedIndex { applicable_fragments, }) } + + /// Block `rows` (stale overlay row addresses) from the index result so the index never + /// emits them. Their fragments stay in the covered set, so non-stale rows keep the index; + /// the blocked rows are re-evaluated against their current (overlay-merged) values on a + /// separate targeted take path built by the scanner. + fn without_rows(mut self, block_list: &RowAddrTreeMap) -> Self { + self.index_result.upper = + std::mem::take(&mut self.index_result.upper).also_block(block_list.clone()); + self.index_result.lower = + std::mem::take(&mut self.index_result.lower).also_block(block_list.clone()); + self + } + + /// Decide whether planning needs full metadata from `fragment`. + /// + /// The upper mask contains every row that might match. For address-style row ids its high + /// 32 bits identify the fragment directly. Stable row ids need the fragment's row-id + /// sequence to make the same decision, but this still avoids opening deletion metadata, + /// row counts, and data files for non-candidate fragments. + async fn fragment_metadata_load( + &self, + dataset: &Dataset, + fragment: &Fragment, + only_indexed_fragments: bool, + retained_range_bytes: &AtomicUsize, + ) -> Result { + let fragment_id = fragment.id as u32; + if !self.applicable_fragments.contains(fragment_id) { + return Ok(if only_indexed_fragments { + FragmentMetadataLoad::Skip + } else { + FragmentMetadataLoad::Load + }); + } + let Some(candidate_rows) = self.index_result.upper.allow_list() else { + // A block-list may select rows from every fragment. + return Ok(FragmentMetadataLoad::Load); + }; + if candidate_rows.iter().next().is_none() { + return Ok(FragmentMetadataLoad::Skip); + } + if dataset.manifest.uses_stable_row_ids() { + let row_id_sequence = load_row_id_sequence(dataset, fragment).await?; + let upper_ranges = row_id_sequence.mask_to_offset_ranges(&self.index_result.upper); + return Ok(if upper_ranges.is_empty() { + FragmentMetadataLoad::Skip + } else { + FragmentMetadataLoad::LoadWithStableRouting(StableIndexRouting { + row_id_sequence, + upper_ranges: Self::retain_stable_index_ranges( + upper_ranges, + retained_range_bytes, + ), + }) + }); + } + Ok(match candidate_rows.get(&fragment_id) { + Some(RowAddrSelection::Full) => FragmentMetadataLoad::Load, + Some(RowAddrSelection::Partial(rows)) if !rows.is_empty() => FragmentMetadataLoad::Load, + Some(RowAddrSelection::Partial(_)) | None => FragmentMetadataLoad::Skip, + }) + } } /// A fragment along with ranges of row offsets to read @@ -99,6 +219,7 @@ struct ScopedFragmentRead { // An in-memory filter to apply after reading the fragment (whatever couldn't be // pushed down into the index query) filter: Option, + physical_filter: Option>, priority: u32, scan_scheduler: Arc, } @@ -123,6 +244,9 @@ impl ScopedFragmentRead { #[derive(Debug, Clone)] struct LoadedFragment { row_id_sequence: Arc, + /// Stable row-ID upper ranges computed while routing index candidates. + /// Reusing them avoids mapping the same mask again during final planning. + index_upper_ranges: Option>>, deletion_vector: Option>, fragment: Arc, // The number of physical rows in the fragment @@ -314,7 +438,7 @@ struct FilteredReadStream { /// The stream of filtered rows, expressed as a stream of tasks (batch futures) /// /// This stream can be shared by multiple partitions - task_stream: Arc>>>, + task_stream: Arc>>>, /// The scan scheduler for the scan scan_scheduler: Arc, /// The global metrics for the scan @@ -328,6 +452,116 @@ struct FilteredReadStream { threading_mode: FilteredReadThreadingMode, /// Range to apply to the result stream if not already pushed down in planning phase scan_range_after_filter: Option>, + /// Fragments planned non-empty, and their total planned rows; the output + /// side uses these to detect take-shaped plans (batch size resolves at + /// execute time, so the detection lives there too) + touched_fragments: usize, + planned_rows: u64, +} + +/// Below this many fragments there are too few handoffs to be worth +/// consolidating +const CONSOLIDATE_MIN_FRAGMENTS: usize = 8; + +/// Above this per-fragment average, batches are big enough to amortize +/// their handoff +const CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT: u64 = 1024; + +/// Pump a take-shaped read on a spawned task, handing the consumer +/// consolidated batches. Inline polling would otherwise execute the +/// per-batch pipeline work on the consumer, which serializes concurrent +/// small reads. +fn consolidated_stream( + inner: SendableRecordBatchStream, + target: usize, +) -> SendableRecordBatchStream { + let mut builder = RecordBatchReceiverStream::builder(inner.schema(), 4); + let tx = builder.tx(); + builder.spawn(async move { + let mut stream = coalesce_batches(inner, target).boxed(); + while let Some(item) = stream.next().await { + if tx.send(item).await.is_err() { + // Receiver dropped: the query was cancelled + break; + } + } + Ok(()) + }); + builder.build() +} + +/// Merge batches up to `target` rows; batches already at the target pass +/// through whole (never split). Order is preserved. +pub fn coalesce_batches( + input: SendableRecordBatchStream, + target: usize, +) -> impl Stream> { + struct Coalescer { + input: SendableRecordBatchStream, + schema: SchemaRef, + target: usize, + buffered: Vec, + buffered_rows: usize, + exhausted: bool, + } + + impl Coalescer { + fn ready_to_emit(&self) -> bool { + self.buffered_rows >= self.target || (self.exhausted && !self.buffered.is_empty()) + } + + fn buffer(&mut self, batch: RecordBatch) { + self.buffered_rows += batch.num_rows(); + self.buffered.push(batch); + } + + fn emit(&mut self) -> DataFusionResult { + self.buffered_rows = 0; + if self.buffered.len() > 1 { + let batch = arrow::compute::concat_batches(&self.schema, self.buffered.iter())?; + self.buffered.clear(); + Ok(batch) + } else { + self.buffered.pop().ok_or_else(|| { + DataFusionError::Internal( + "coalesce_batches emitted with an empty buffer".to_string(), + ) + }) + } + } + } + + let schema = input.schema(); + let coalescer = Coalescer { + input, + schema, + target, + buffered: Vec::new(), + buffered_rows: 0, + exhausted: false, + }; + futures::stream::try_unfold(coalescer, |mut this| async move { + loop { + if this.ready_to_emit() { + return Ok(Some((this.emit()?, this))); + } + if this.exhausted { + return Ok(None); + } + match this.input.try_next().await? { + Some(batch) if batch.num_rows() >= this.target && !this.buffered.is_empty() => { + // Emit the partial buffer on its own; the large batch + // then passes through whole on the next iteration + let out = this.emit()?; + this.buffer(batch); + return Ok(Some((out, this))); + } + Some(batch) if batch.num_rows() > 0 => this.buffer(batch), + Some(_) => {} + None => this.exhausted = true, + } + } + }) } impl std::fmt::Debug for FilteredReadStream { @@ -337,16 +571,25 @@ impl std::fmt::Debug for FilteredReadStream { } impl FilteredReadStream { - /// Create a new FilteredReadStream from a pre-computed internal plan + /// Create a new FilteredReadStream from a pre-computed internal plan. + /// Fragment handles are constructed I/O-free from the manifest + /// descriptors, only for the fragments the plan selects. A `None` + /// scheduler is created here; the row-stream path injects its per-query + /// shared one and a per-batch priority offset. #[instrument(name = "init_filtered_read_stream", skip_all)] - async fn try_new( + #[allow(clippy::too_many_arguments)] + fn try_new( dataset: Arc, options: FilteredReadOptions, - metrics: &ExecutionPlanMetricsSet, + global_metrics: Arc, plan: FilteredReadInternalPlan, - ) -> DataFusionResult { - let global_metrics = Arc::new(FilteredReadGlobalMetrics::new(metrics)); - + scan_scheduler: Option>, + priority_offset: Option, + materialization_context: Arc, + materialize_blob_v2_binary: bool, + ) -> Self { + let scan_scheduler = + scan_scheduler.unwrap_or_else(|| Self::make_scan_scheduler(&dataset, &options)); let threading_mode = options.threading_mode; let io_parallelism = dataset.object_store.io_parallelism(); @@ -355,62 +598,41 @@ impl FilteredReadStream { .unwrap_or_else(|| (*DEFAULT_FRAGMENT_READAHEAD).unwrap_or(io_parallelism * 2)) .max(1); - let fragments = options + let fragment_descriptors = options .fragments .clone() .unwrap_or_else(|| dataset.fragments().clone()); log::debug!( "Filtered read on {} fragments with frag_readahead={} and io_parallelism={}", - fragments.len(), + fragment_descriptors.len(), fragment_readahead, io_parallelism ); - // Ideally we don't need to collect here but if we don't we get "implementation of FnOnce is - // not general enough" false positives from rustc - let frag_futs = fragments - .iter() - .map(|frag| { - Result::Ok(Self::load_fragment( - dataset.clone(), - frag.clone(), - options.with_deleted_rows, - )) - }) - .collect::>(); - let loaded_fragments = futures::stream::iter(frag_futs) - // Cannot use unordered because we need to populate logical_offset based on user-provided order - .try_buffered(io_parallelism) - .try_collect::>() - .await?; - - let output_schema = Arc::new(options.projection.to_arrow_schema()); - - let obj_store = dataset.object_store.clone(); - // Explicit options take precedence; otherwise fall back to the - // LANCE_DEFAULT_IO_BUFFER_SIZE env var if set; otherwise max_bandwidth. - let scheduler_config = if let Some(io_buffer_size_bytes) = options - .io_buffer_size_bytes - .or_else(get_default_io_buffer_size_override) - { - SchedulerConfig::new(io_buffer_size_bytes) + let output_schema = if materialize_blob_v2_binary { + public_blob_v2_binary_projection_schema(&options.projection) } else { - SchedulerConfig::max_bandwidth(obj_store.as_ref()) + Arc::new(ArrowSchema::from( + &crate::dataset::blob::blob_v2_descriptor_schema(&options.projection.to_schema()), + )) }; - let scan_scheduler = ScanScheduler::new(obj_store, scheduler_config); - // Get scan_range_after_filter from the plan let scan_range_after_filter = plan.scan_range_after_filter.clone(); // Convert plan to scoped fragments for I/O - let scoped_fragments = Self::plan_to_scoped_fragments( + let mut scoped_fragments = Self::plan_to_scoped_fragments( &plan, - &loaded_fragments, + &fragment_descriptors, &dataset, &options, scan_scheduler.clone(), ); + if let Some(priority_offset) = priority_offset.filter(|offset| *offset != 0) { + for scoped in &mut scoped_fragments { + scoped.priority = scoped.priority.saturating_add(priority_offset); + } + } let global_metrics_clone = global_metrics.clone(); @@ -420,8 +642,18 @@ impl FilteredReadStream { move |scoped_fragment| { let metrics = global_metrics_clone.clone(); let limit = scan_range_after_filter.as_ref().map(|r| r.end); + let dataset = dataset.clone(); + let materialization_context = materialization_context.clone(); SpawnedTask::spawn( - Self::read_fragment(scoped_fragment, metrics, limit).in_current_span(), + Self::read_fragment( + dataset, + scoped_fragment, + metrics, + limit, + materialization_context, + materialize_blob_v2_binary, + ) + .in_current_span(), ) .map(|thread_result| thread_result.unwrap()) } @@ -429,7 +661,23 @@ impl FilteredReadStream { .buffered(fragment_readahead); let task_stream = fragment_streams.try_flatten().boxed(); - Ok(Self { + // A batch never spans fragments, so a plan touching many fragments + // with few rows each emits one tiny batch per fragment. Fragments + // planned empty produce no batch and don't count. Filtered scans + // stay dense here: their planned rows are a pre-refine upper bound. + let (touched_fragments, planned_rows) = + plan.rows + .values() + .fold((0usize, 0u64), |(fragments, rows), ranges| { + let fragment_rows: u64 = + ranges.iter().map(|range| range.end - range.start).sum(); + if fragment_rows > 0 { + (fragments + 1, rows + fragment_rows) + } else { + (fragments, rows) + } + }); + Self { output_schema, task_stream: Arc::new(AsyncMutex::new(task_stream)), scan_scheduler, @@ -437,13 +685,70 @@ impl FilteredReadStream { active_partitions_counter: Arc::new(AtomicUsize::new(0)), threading_mode, scan_range_after_filter, - }) + touched_fragments, + planned_rows, + } + } + + /// Drain the entire read into batches (used by the row-stream path, + /// which is the stream's only consumer and records metrics per batch) + async fn collect_all(&self, decode_parallelism: usize) -> Result> { + let mut task_stream = self.task_stream.lock().await; + (&mut *task_stream) + .try_buffered(decode_parallelism) + .try_collect() + .await + } + + async fn load_all_fragments( + dataset: &Arc, + options: &FilteredReadOptions, + ) -> Result> { + let io_parallelism = dataset.object_store.io_parallelism(); + let fragments = options + .fragments + .clone() + .unwrap_or_else(|| dataset.fragments().clone()); + // Ideally we don't need to collect here but if we don't we get "implementation of FnOnce is + // not general enough" false positives from rustc + let frag_futs = fragments + .iter() + .map(|frag| { + Result::Ok(Self::load_fragment( + dataset.clone(), + frag.clone(), + options.with_deleted_rows, + None, + )) + }) + .collect::>(); + futures::stream::iter(frag_futs) + // Cannot use unordered because we need to populate logical_offset based on user-provided order + .try_buffered(io_parallelism) + .try_collect::>() + .await + } + + /// Create the I/O scheduler for a read (explicit option → env override → + /// max bandwidth) + fn make_scan_scheduler(dataset: &Dataset, options: &FilteredReadOptions) -> Arc { + let obj_store = dataset.object_store.clone(); + let scheduler_config = if let Some(io_buffer_size_bytes) = options + .io_buffer_size_bytes + .or_else(get_default_io_buffer_size_override) + { + SchedulerConfig::new(io_buffer_size_bytes) + } else { + SchedulerConfig::max_bandwidth(obj_store.as_ref()) + }; + ScanScheduler::new(obj_store, scheduler_config) } async fn load_fragment( dataset: Arc, frag: Fragment, include_deleted_rows: bool, + stable_index_routing: Option, ) -> Result { let file_fragment = FileFragment::new(dataset.clone(), frag.clone()); let deletion_vector = if include_deleted_rows { @@ -453,19 +758,27 @@ impl FilteredReadStream { }; let num_physical_rows = file_fragment.physical_rows().await? as u64; - let (row_id_sequence, num_logical_rows) = if dataset.manifest.uses_stable_row_ids() { - let row_id_sequence = load_row_id_sequence(dataset.as_ref(), &frag).await?; - let num_logical_rows = row_id_sequence.len(); - (row_id_sequence, num_logical_rows) - } else { - let row_ids_start = frag.id << 32; - let row_ids_end = row_ids_start + num_physical_rows; - let num_logical_rows = file_fragment.count_rows(None).await? as u64; - let addrs_as_ids = Arc::new(RowIdSequence::from(row_ids_start..row_ids_end)); - (addrs_as_ids, num_logical_rows) - }; + let (row_id_sequence, num_logical_rows, index_upper_ranges) = + if dataset.manifest.uses_stable_row_ids() { + let (row_id_sequence, index_upper_ranges) = + if let Some(routing) = stable_index_routing { + (routing.row_id_sequence, routing.upper_ranges) + } else { + (load_row_id_sequence(dataset.as_ref(), &frag).await?, None) + }; + let num_logical_rows = row_id_sequence.len(); + (row_id_sequence, num_logical_rows, index_upper_ranges) + } else { + debug_assert!(stable_index_routing.is_none()); + let row_ids_start = frag.id << 32; + let row_ids_end = row_ids_start + num_physical_rows; + let num_logical_rows = file_fragment.count_rows(None).await? as u64; + let addrs_as_ids = Arc::new(RowIdSequence::from(row_ids_start..row_ids_end)); + (addrs_as_ids, num_logical_rows, None) + }; Ok(LoadedFragment { row_id_sequence, + index_upper_ranges, fragment: Arc::new(file_fragment), num_physical_rows, num_logical_rows, @@ -486,7 +799,7 @@ impl FilteredReadStream { // Returns: FilteredReadInternalPlan #[instrument(name = "plan_scan", skip_all)] fn plan_scan( - fragments: &[LoadedFragment], + mut fragments: Vec, evaluated_index: &Option>, options: &FilteredReadOptions, ) -> FilteredReadInternalPlan { @@ -528,11 +841,12 @@ impl FilteredReadStream { let mut range_offset = 0; for LoadedFragment { row_id_sequence, + index_upper_ranges, fragment, num_logical_rows, num_physical_rows, deletion_vector, - } in fragments.iter() + } in fragments.iter_mut() { if let Some(range_before_filter) = &options.scan_range_before_filter && range_offset >= range_before_filter.end @@ -546,11 +860,11 @@ impl FilteredReadStream { if let Some(range_before_filter) = &options.scan_range_before_filter { let range_start = range_offset; let range_end = if options.with_deleted_rows { - range_offset += num_physical_rows; - range_start + num_physical_rows + range_offset += *num_physical_rows; + range_start + *num_physical_rows } else { - range_offset += num_logical_rows; - range_start + num_logical_rows + range_offset += *num_logical_rows; + range_start + *num_logical_rows }; to_read = Self::trim_ranges(to_read, range_start..range_end, range_before_filter); if to_read.is_empty() { @@ -563,6 +877,7 @@ impl FilteredReadStream { evaluated_index, fragment, row_id_sequence, + index_upper_ranges.take(), to_read, &mut to_skip, &mut to_take, @@ -640,10 +955,13 @@ impl FilteredReadStream { } } + /// Handles are constructed here, I/O-free, only for the fragments the + /// plan selects; priority is the fragment's position in the candidate + /// list so a sparse plan keeps the original I/O order. fn plan_to_scoped_fragments( plan: &FilteredReadInternalPlan, - fragments: &[LoadedFragment], - dataset: &Dataset, + fragments: &[Fragment], + dataset: &Arc, options: &FilteredReadOptions, scan_scheduler: Arc, ) -> Vec { @@ -659,7 +977,7 @@ impl FilteredReadStream { let mut scoped_fragments = Vec::new(); for (priority, fragment) in fragments.iter().enumerate() { - let fragment_id = fragment.fragment.id() as u32; + let fragment_id = fragment.id as u32; // Check if this fragment is in the plan if let Some(ranges) = plan.rows.get(&fragment_id) { @@ -669,15 +987,19 @@ impl FilteredReadStream { // Get filter for this fragment (convert Arc back to Expr) let filter = plan.filters.get(&fragment_id).map(|f| (**f).clone()); + let physical_filter = filter + .as_ref() + .and_then(|filter| options.physical_filter(filter)); scoped_fragments.push(ScopedFragmentRead { - fragment: fragment.fragment.clone(), + fragment: Arc::new(FileFragment::new(dataset.clone(), fragment.clone())), ranges: ranges.clone(), projection: projection.clone(), with_deleted_rows: options.with_deleted_rows, batch_size: default_batch_size, file_reader_options: options.file_reader_options.clone(), filter, + physical_filter, priority: priority as u32, scan_scheduler: scan_scheduler.clone(), }); @@ -693,6 +1015,7 @@ impl FilteredReadStream { evaluated_index: &Option>, fragment: &FileFragment, row_id_sequence: &Arc, + index_upper_ranges: Option>>, to_read: Vec>, to_skip: &mut u64, to_take: &mut u64, @@ -709,8 +1032,12 @@ impl FilteredReadStream { let index_result = &evaluated_index.index_result; if index_result.is_exact() { // lower == upper; either side gives the precise answer. - let valid_ranges = row_id_sequence.mask_to_offset_ranges(&index_result.upper); - let mut matched_ranges = Self::intersect_ranges(&to_read, &valid_ranges); + let mut matched_ranges = Self::intersect_index_ranges( + &to_read, + row_id_sequence, + &index_result.upper, + index_upper_ranges, + ); fragments_to_read.insert(fragment_id, matched_ranges.clone()); Self::apply_skip_take_to_ranges(&mut matched_ranges, to_skip, to_take); @@ -718,8 +1045,12 @@ impl FilteredReadStream { } else if index_result.is_at_least() { // upper is universe; lower is the guaranteed-match set // used for the skip/take push-down path. - let valid_ranges = row_id_sequence.mask_to_offset_ranges(&index_result.lower); - let mut guaranteed_ranges = Self::intersect_ranges(&to_read, &valid_ranges); + let mut guaranteed_ranges = Self::intersect_index_ranges( + &to_read, + row_id_sequence, + &index_result.lower, + None, + ); fragments_to_read.insert(fragment_id, guaranteed_ranges.clone()); Self::apply_skip_take_to_ranges(&mut guaranteed_ranges, to_skip, to_take); @@ -737,8 +1068,12 @@ impl FilteredReadStream { // `lower` portion would also be visible up at the // `can_skip_recheck` block — both are deferred. See // TODO(refined-pushdown). - let valid_ranges = row_id_sequence.mask_to_offset_ranges(&index_result.upper); - let matched_ranges = Self::intersect_ranges(&to_read, &valid_ranges); + let matched_ranges = Self::intersect_index_ranges( + &to_read, + row_id_sequence, + &index_result.upper, + index_upper_ranges, + ); fragments_to_read.insert(fragment_id, matched_ranges); } } else { @@ -794,7 +1129,7 @@ impl FilteredReadStream { physical_ranges.truncate(write_idx); } - /// Intersect two sets of sorted ranges + /// Intersect two sets of sorted ranges. fn intersect_ranges(ranges1: &[Range], ranges2: &[Range]) -> Vec> { let mut result = Vec::new(); let mut i = 0; @@ -823,6 +1158,19 @@ impl FilteredReadStream { result } + fn intersect_index_ranges( + to_read: &[Range], + row_id_sequence: &RowIdSequence, + index_mask: &RowAddrMask, + precomputed_ranges: Option>>, + ) -> Vec> { + // Taking routed ranges fragment-by-fragment drops each input as its final intersection is + // produced instead of retaining every routed range vector alongside the completed plan. + let index_ranges = + precomputed_ranges.unwrap_or_else(|| row_id_sequence.mask_to_offset_ranges(index_mask)); + Self::intersect_ranges(to_read, &index_ranges) + } + /// Apply skip and take to ranges and update the counters fn apply_skip_take_to_ranges( to_read: &mut Vec>, @@ -979,16 +1327,16 @@ impl FilteredReadStream { } }); let partition_metrics_clone = partition_metrics.clone(); - let base_batch_stream = - futures_stream - .try_buffered(num_threads) - .try_filter_map(move |batch| { - std::future::ready(Ok(if batch.num_rows() == 0 { - None - } else { - Some(batch) - })) - }); + let base_batch_stream = futures_stream + .try_buffered(num_threads) + .map_ok(MaterializedBlobBatch::into_batch) + .try_filter_map(move |batch| { + std::future::ready(Ok(if batch.num_rows() == 0 { + None + } else { + Some(batch) + })) + }); let batch_stream = if let Some(ref range) = self.scan_range_after_filter { Self::apply_hard_range(base_batch_stream, range.clone()).boxed() @@ -1044,7 +1392,7 @@ impl FilteredReadStream { }; if let Some(task) = maybe_task { let task = task?; - let batch = task.await?; + let batch = task.await?.into_batch(); partition_metrics .baseline_metrics .record_output(batch.num_rows()); @@ -1076,11 +1424,22 @@ impl FilteredReadStream { // Reads a single fragment into a stream of batch tasks #[instrument(name = "read_fragment", level = "debug", skip_all)] async fn read_fragment( + dataset: Arc, mut fragment_read_task: ScopedFragmentRead, global_metrics: Arc, fragment_soft_limit: Option, - ) -> Result>> { - let output_schema = Arc::new(fragment_read_task.projection.to_arrow_schema()); + materialization_context: Arc, + materialize_blob_v2_binary: bool, + ) -> Result>> { + let output_schema = if materialize_blob_v2_binary { + public_blob_v2_binary_projection_schema(fragment_read_task.projection.as_ref()) + } else { + Arc::new(ArrowSchema::from( + &crate::dataset::blob::blob_v2_descriptor_schema( + &fragment_read_task.projection.to_schema(), + ), + )) + }; if let Some(filter) = &fragment_read_task.filter { let filter_cols = Planner::column_names_in_expr(filter); @@ -1095,10 +1454,23 @@ impl FilteredReadStream { } } - let read_schema = fragment_read_task.projection.to_bare_schema(); + let output_read_schema = Arc::new(fragment_read_task.projection.to_schema()); + let bare_read_schema = fragment_read_task.projection.to_bare_schema(); + let has_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(&bare_read_schema); + let materialize_blob_v2_binary = materialize_blob_v2_binary && has_blob_v2_binary; + let read_schema = if has_blob_v2_binary { + crate::dataset::blob::blob_v2_descriptor_schema(&bare_read_schema) + } else { + bare_read_schema + }; + let mut frag_read_config = fragment_read_task.frag_read_config(); + if has_blob_v2_binary { + frag_read_config = frag_read_config.with_row_address(true); + } let mut fragment_reader = fragment_read_task .fragment - .open(&read_schema, fragment_read_task.frag_read_config()) + .open(&read_schema, frag_read_config) .await?; if fragment_read_task.with_deleted_rows { @@ -1109,14 +1481,18 @@ impl FilteredReadStream { // the row ids are not contiguous fragment_read_task.ranges.sort_by_key(|r| r.start); - let physical_filter = fragment_read_task - .filter - .map(|filter| { - let planner = - Planner::new(Arc::new(fragment_read_task.projection.to_arrow_schema())); - planner.create_physical_expr(&filter) - }) - .transpose()?; + let physical_filter = match fragment_read_task.physical_filter { + Some(filter) => Some(filter), + None => fragment_read_task + .filter + .map(|filter| { + let planner = Planner::new(public_blob_v2_binary_projection_schema( + fragment_read_task.projection.as_ref(), + )); + planner.create_physical_expr(&filter) + }) + .transpose()?, + }; // We are going to count the fragment as scanned on the first batch we // read. This might miss empty fragments, but we assume that wouldn't be @@ -1136,7 +1512,7 @@ impl FilteredReadStream { let global_metrics = global_metrics.clone(); let fragment_counted = fragment_counted.clone(); let range_tracker = range_tracker.clone(); - batch_fut + let batch_fut = batch_fut .inspect_ok(move |batch| { let num_rows = batch.num_rows(); global_metrics.rows_scanned.add(num_rows); @@ -1151,7 +1527,27 @@ impl FilteredReadStream { global_metrics.ranges_scanned.add(additional_ranges); } }) - .boxed() + .boxed(); + if materialize_blob_v2_binary { + let dataset = dataset.clone(); + let output_read_schema = output_read_schema.clone(); + let materialization_context = materialization_context.clone(); + let admission = materialization_context.admission(); + batch_fut + .and_then(move |batch| async move { + crate::dataset::blob::materialize_blob_v2_binary_batch_with_admission( + &dataset, + output_read_schema.as_ref(), + batch, + &materialization_context, + admission, + ) + .await + }) + .boxed() + } else { + batch_fut.map_ok(MaterializedBlobBatch::unreserved).boxed() + } }) .zip(futures::stream::repeat(( physical_filter.clone(), @@ -1159,32 +1555,32 @@ impl FilteredReadStream { ))) .map(|(batch_fut, args)| Self::wrap_with_filter(batch_fut, args.0, args.1)); - let result: Pin> + Send>> = - if let Some(limit) = fragment_soft_limit { - Box::pin(Self::apply_soft_limit(fragment_stream, limit)) - } else { - Box::pin(fragment_stream) - }; + let result = if let Some(limit) = fragment_soft_limit { + Self::apply_soft_limit(fragment_stream, limit).boxed() + } else { + fragment_stream.boxed() + }; Ok(result) } fn wrap_with_filter( - batch_fut: ReadBatchFut, + batch_fut: MaterializedReadBatchFut, filter: Option>, output_schema: SchemaRef, - ) -> Result { + ) -> Result { if let Some(filter) = filter { Ok(batch_fut .map(move |batch| { let batch = batch?; - let batch = datafusion_physical_plan::filter::batch_filter(&batch, &filter) - .map_err(|e| { - Error::execution(format!( - "Error applying filter expression to batch: {e}" - )) - })?; + let filtered = + datafusion_physical_plan::filter::batch_filter(batch.batch(), &filter) + .map_err(|e| { + Error::execution(format!( + "Error applying filter expression to batch: {e}" + )) + })?; // Drop any fields loaded purely for the purpose of applying the filter - Ok(batch.project_by_schema(output_schema.as_ref())?) + Ok(batch.with_batch(filtered.project_by_schema(output_schema.as_ref())?)) }) .boxed()) } else { @@ -1192,9 +1588,12 @@ impl FilteredReadStream { } } - fn apply_soft_limit(stream: S, limit: u64) -> impl Stream> + fn apply_soft_limit( + stream: S, + limit: u64, + ) -> impl Stream> where - S: Stream>, + S: Stream>, { let rows_read = Arc::new(AtomicUsize::new(0)); @@ -1209,7 +1608,7 @@ impl FilteredReadStream { batch_fut .map(move |batch_result| { batch_result.inspect(|batch| { - let batch_rows = batch.num_rows(); + let batch_rows = batch.batch().num_rows(); rows_read.fetch_add(batch_rows, Ordering::Relaxed); }) }) @@ -1270,7 +1669,7 @@ pub struct FilteredReadOptions { pub scan_range_before_filter: Option>, /// The range of rows to read after applying the filter. pub scan_range_after_filter: Option>, - /// Include deleted rows in the scan + /// Include deleted rows in the scan; they are returned with a null row id pub with_deleted_rows: bool, /// The maximum number of rows per batch pub batch_size: Option, @@ -1289,12 +1688,21 @@ pub struct FilteredReadOptions { /// result to avoid applying this (and instead only apply the refine filter) but in some cases /// the index result does not cover all fragments or is not exact. pub full_filter: Option, + physical_filters: Vec<(Expr, Arc)>, /// The threading mode to use for the scan pub threading_mode: FilteredReadThreadingMode, /// The size of the I/O buffer to use for the scan pub io_buffer_size_bytes: Option, + /// Total memory budget for asynchronously materialized blob v2 batches + pub materialization_readahead_bytes: Option, /// If true, skip fragments that are not covered by the scalar index result. pub only_indexed_fragments: bool, + /// Row addresses whose index entries may be stale because an overlay committed after the + /// index was built touches an indexed field. They are blocked from the index result so the + /// index never emits them; the scanner re-evaluates just these rows against their current + /// (overlay-merged) values on a targeted take path. Their fragments stay in the covered set, + /// so non-stale rows keep the index. `None` on the common no-overlay fast path. + pub overlay_block: Option, } impl FilteredReadOptions { @@ -1322,14 +1730,24 @@ impl FilteredReadOptions { projection, refine_filter: None, full_filter: None, + physical_filters: Vec::new(), io_buffer_size_bytes: None, + materialization_readahead_bytes: None, only_indexed_fragments: false, + overlay_block: None, threading_mode: FilteredReadThreadingMode::OnePartitionMultipleThreads( get_num_compute_intensive_cpus(), ), } } + /// Block the given stale overlay row addresses (see the `overlay_block` field) from the + /// scalar index result so the index never emits them. + pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self { + self.overlay_block = Some(block); + self + } + /// Include deleted rows in the scan /// /// This is currently only supported if there is no scan_range specified @@ -1447,6 +1865,7 @@ impl FilteredReadOptions { "refine_filter is set but full_filter is not".into(), )); } + self.physical_filters.clear(); self.refine_filter = refine_filter; self.full_filter = full_filter; Ok(self) @@ -1454,17 +1873,54 @@ impl FilteredReadOptions { /// An alternative to [`Self::with_filter`] to set the filters from a FilterPlan if you already have one pub fn with_filter_plan(mut self, filter_plan: FilterPlan) -> Self { + self.physical_filters.clear(); self.refine_filter = filter_plan.refine_expr; self.full_filter = filter_plan.full_expr; self } + /// Plan configured filters with the supplied DataFusion session. + pub(crate) fn with_physical_filters(mut self, session: &dyn Session) -> Result { + for filter in [&self.full_filter, &self.refine_filter] + .into_iter() + .flatten() + { + if self + .physical_filters + .iter() + .any(|(planned_filter, _)| planned_filter == filter) + { + continue; + } + + let filter_columns = Planner::column_names_in_expr(filter); + let projection = self + .projection + .clone() + .union_columns(filter_columns, OnMissing::Error)?; + let schema = public_blob_v2_binary_projection_schema(&projection); + let physical_filter = + Planner::new(schema).create_physical_expr_with_session(filter, session)?; + self.physical_filters + .push((filter.clone(), physical_filter)); + } + Ok(self) + } + + fn physical_filter(&self, filter: &Expr) -> Option> { + self.physical_filters + .iter() + .find(|(planned_filter, _)| planned_filter == filter) + .map(|(_, physical_filter)| physical_filter.clone()) + } + /// Specify the projection to use for the scan /// /// If the row id or row address are requested then they will be placed at the end /// of the output schema. If both are requested then the row id will come before /// the row address. pub fn with_projection(mut self, projection: Projection) -> Self { + self.physical_filters.clear(); self.projection = projection; self } @@ -1477,11 +1933,30 @@ impl FilteredReadOptions { self } + /// Specify the memory budget for asynchronous blob v2 materialization. + pub fn with_materialization_readahead_bytes(mut self, size: u64) -> Self { + self.materialization_readahead_bytes = Some(size); + self + } + /// Only read fragments covered by a scalar index result. pub fn with_only_indexed_fragments(mut self) -> Self { self.only_indexed_fragments = true; self } + + /// Specify the threading mode to use for the scan. + /// + /// This controls how decode work is parallelized. For the default single-partition + /// scan, the parameter of [`FilteredReadThreadingMode::OnePartitionMultipleThreads`] + /// bounds how many batch-decode tasks are buffered in flight (via `try_buffered`). + /// + /// The parallelism must be greater than 0. A value of 0 is rejected by + /// [`FilteredReadExec::try_new`]. + pub fn with_threading_mode(mut self, threading_mode: FilteredReadThreadingMode) -> Self { + self.threading_mode = threading_mode; + self + } } /// A plan node that reads a dataset, applying an optional filter and projection. @@ -1502,9 +1977,10 @@ impl FilteredReadOptions { pub struct FilteredReadExec { dataset: Arc, options: FilteredReadOptions, + materialization_context: Arc, properties: Arc, metrics: ExecutionPlanMetricsSet, - index_input: Option>, + input: RowSelector, // Precomputed internal plan plan: Arc>, // When execute is first called we will initialize the FilteredReadStream. In order to support @@ -1512,6 +1988,53 @@ pub struct FilteredReadExec { running_stream: Arc>>, } +/// Describes which rows a [`FilteredReadExec`] should read +#[derive(Debug)] +enum RowSelector { + /// Every live row of the dataset (no input plan) + AllRows, + /// A set of rows: one serialized [`IndexExprResult`] batch. Output is in + /// storage order and deduplicated. + RowSet(Arc), + /// A stream of rows: record batches with a `_rowid`/`_rowaddr` column + /// and other payload columns (just carried) + RowStream(Arc), +} + +impl RowSelector { + fn row_set_plan(&self) -> Option<&Arc> { + match self { + Self::RowSet(plan) => Some(plan), + _ => None, + } + } + + fn child(&self) -> Option<&Arc> { + match self { + Self::AllRows => None, + Self::RowSet(plan) => Some(plan), + Self::RowStream(source) => Some(&source.plan), + } + } +} + +/// State derived at construction for a row-stream source +#[derive(Debug)] +struct RowStreamSource { + plan: Arc, + /// The stream column identifying rows: [`ROW_ID`] or [`ROW_ADDR`] + key_column: &'static str, + /// Options for the internal fragment read; carries the projection that + /// reflects the actual columns to read (plus the alignment key column) + read_options: FilteredReadOptions, + /// The schema for newly read columns + new_fields_schema: SchemaRef, + /// Descriptor-bearing output before Blob v2 payload materialization + intermediate_output_schema: SchemaRef, + /// Final schema used to materialize one complete row-stream output batch + materialization_output_schema: Option>, +} + /// Public plan for distributed execution - uses bitmap for flexibility #[derive(Clone)] pub struct FilteredReadPlan { @@ -1558,50 +2081,271 @@ impl FilteredReadInternalPlan { } impl FilteredReadExec { + /// Create a new filtered read pub fn try_new( dataset: Arc, - mut options: FilteredReadOptions, - index_input: Option>, + options: FilteredReadOptions, + input: Option>, ) -> Result { - if options.with_deleted_rows { - // Ensure we have the row id column if with_deleted_rows is set - options.projection = options.projection.with_row_id(); - } - - if options.projection.is_empty() { - return Err(Error::invalid_input_source("no columns were selected and with_row_id / with_row_address is false, there is nothing to scan" - .into())); + if options.materialization_readahead_bytes == Some(0) { + return Err(Error::invalid_input_source( + "materialization_readahead_bytes must be greater than 0, got 0".into(), + )); } - - if options.scan_range_after_filter.is_some() { - // Validate that there's a filter when using scan_range_after_filter - if options.full_filter.is_none() - && options.refine_filter.is_none() - && index_input.is_none() - { - return Err(Error::invalid_input_source("scan_range_after_filter requires a filter to be applied. Use scan_range_before_filter for unfiltered scans." - .into())); - } - - // TODO: support multi partition - if matches!( - options.threading_mode, - FilteredReadThreadingMode::MultiplePartitions(_) - ) { - return Err(Error::not_supported_source( - "scan_range_after_filter not yet supported with multiple partitions" - .to_string() - .into(), - )); + match input { + Some(input) if Self::is_index_query_schema(input.schema().as_ref()) => { + Self::try_new_scan(dataset, options, Some(input)) } + Some(input) => Self::try_new_row_stream(dataset, options, input), + None => Self::try_new_scan(dataset, options, None), } - let output_schema = Arc::new(options.projection.to_arrow_schema()); - let num_partitions = match options.threading_mode { - FilteredReadThreadingMode::OnePartitionMultipleThreads(_) => 1, - FilteredReadThreadingMode::MultiplePartitions(n) => n, - }; + } - let properties = Arc::new(PlanProperties::new( + /// Whether `schema` is one of the serialized [`IndexExprResult`] wire + /// layouts (see [`IndexExprResultWireFormat`]) + fn is_index_query_schema(schema: &arrow_schema::Schema) -> bool { + [ + IndexExprResultWireFormat::TwoMask, + IndexExprResultWireFormat::ThreeVariant, + ] + .iter() + .any(|format| schema.fields() == format.schema().fields()) + } + + /// The input columns that carry through to the output: identity columns + /// appear iff their flag is requested, ordinary columns always carry + fn carried_schema(input_schema: &arrow_schema::Schema, projection: &Projection) -> SchemaRef { + Arc::new(arrow_schema::Schema::new( + input_schema + .fields() + .iter() + .filter(|f| { + (f.name() != ROW_ID || projection.with_row_id) + && (f.name() != ROW_ADDR || projection.with_row_addr) + }) + .cloned() + .collect::>(), + )) + } + + /// Construct a read over a row-stream source + fn try_new_row_stream( + dataset: Arc, + options: FilteredReadOptions, + input: Arc, + ) -> Result { + versions::validate_row_stream_read( + dataset.manifest().data_storage_format.lance_file_format(), + )?; + if options.refine_filter.is_some() || options.full_filter.is_some() { + return Err(Error::invalid_input_source( + "filters are not supported when taking rows from an input plan".into(), + )); + } + // A limit is safer to apply upstream, on the cheap keyed rows + if options.scan_range_before_filter.is_some() || options.scan_range_after_filter.is_some() { + return Err(Error::invalid_input_source( + "scan ranges are not supported when taking rows from an input plan".into(), + )); + } + // Row-stream reads do not support deleted rows yet; deleted rows are + // excluded from the output by default + if options.with_deleted_rows || options.only_indexed_fragments { + return Err(Error::invalid_input_source( + "with_deleted_rows / only_indexed_fragments are not supported when taking rows from an input plan".into(), + )); + } + let input_schema = input.schema(); + let key_column = if input_schema.column_with_name(ROW_ID).is_some() { + ROW_ID + } else if input_schema.column_with_name(ROW_ADDR).is_some() { + ROW_ADDR + } else { + return Err(Error::invalid_input_source( + format!( + "a row-stream input plan must have a column named '{}' or '{}'", + ROW_ADDR, ROW_ID + ) + .into(), + )); + }; + + let fields_to_read = options + .projection + .clone() + .subtract_arrow_schema(input_schema.as_ref(), OnMissing::Ignore)?; + let synthesize_row_id = fields_to_read.with_row_id; + let synthesize_row_addr = fields_to_read.with_row_addr; + if !fields_to_read.has_data_fields() && !synthesize_row_id && !synthesize_row_addr { + return Err(Error::invalid_input_source( + "the input plan already contains every projected field; there is nothing to read" + .into(), + )); + } + + let carried_schema = Self::carried_schema(input_schema.as_ref(), &options.projection); + + // Output = carried columns ⊕ fetched fields ⊕ synthesized identity + let materialization_output_schema = super::TakeExec::calculate_output_schema( + dataset.schema(), + carried_schema.as_ref(), + &fields_to_read, + ); + let output_schema = Arc::new(arrow_schema::Schema::from( + &crate::dataset::blob::public_blob_v2_binary_output_schema( + &materialization_output_schema, + ), + )); + + // Partitioning and emission behavior follow the input + let properties = Arc::new( + input + .properties() + .as_ref() + .clone() + .with_eq_properties(EquivalenceProperties::new(output_schema)), + ); + + let bare_lance_schema = fields_to_read.to_bare_schema(); + let materialize_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(&bare_lance_schema); + let read_lance_schema = if materialize_blob_v2_binary { + crate::dataset::blob::blob_v2_descriptor_schema(&bare_lance_schema) + } else { + bare_lance_schema + }; + let bare_schema = arrow_schema::Schema::from(&read_lance_schema); + let mut new_fields = bare_schema.fields().iter().cloned().collect::>(); + if synthesize_row_id { + new_fields.push(Arc::new(ROW_ID_FIELD.clone())); + } + if (synthesize_row_addr || materialize_blob_v2_binary) + && !new_fields.iter().any(|field| field.name() == ROW_ADDR) + { + new_fields.push(Arc::new(ROW_ADDR_FIELD.clone())); + } + let new_fields_schema = Arc::new(arrow_schema::Schema::new(new_fields)); + + let intermediate_lance_schema = if materialize_blob_v2_binary { + crate::dataset::blob::blob_v2_descriptor_schema(&materialization_output_schema) + } else { + materialization_output_schema.clone() + }; + let mut intermediate_fields = arrow_schema::Schema::from(&intermediate_lance_schema) + .fields() + .iter() + .cloned() + .collect::>(); + if materialize_blob_v2_binary + && !intermediate_fields + .iter() + .any(|field| field.name() == ROW_ADDR) + { + intermediate_fields.push(Arc::new(ROW_ADDR_FIELD.clone())); + } + let intermediate_output_schema = Arc::new(arrow_schema::Schema::new(intermediate_fields)); + + let materialization_output_schema = + materialize_blob_v2_binary.then(|| Arc::new(materialization_output_schema)); + + // fields_to_read keeps the synthesis flags; add the key column on top + let mut read_options = options.clone(); + read_options.projection = if key_column == ROW_ID { + fields_to_read.with_row_id() + } else { + fields_to_read.with_row_addr() + }; + if materialize_blob_v2_binary { + read_options.projection = read_options.projection.with_row_addr(); + } + + Ok(Self { + materialization_context: BlobMaterializationContext::new( + options.io_buffer_size_bytes, + options.materialization_readahead_bytes, + ), + dataset, + options, + properties, + metrics: ExecutionPlanMetricsSet::new(), + input: RowSelector::RowStream(Arc::new(RowStreamSource { + plan: input, + key_column, + read_options, + new_fields_schema, + intermediate_output_schema, + materialization_output_schema, + })), + plan: Arc::new(OnceCell::new()), + running_stream: Arc::new(AsyncMutex::new(None)), + }) + } + + fn try_new_scan( + dataset: Arc, + mut options: FilteredReadOptions, + index_input: Option>, + ) -> Result { + let input = match index_input { + Some(plan) => RowSelector::RowSet(plan), + None => RowSelector::AllRows, + }; + if options.with_deleted_rows { + // Ensure we have the row id column if with_deleted_rows is set + options.projection = options.projection.with_row_id(); + } + + if options.projection.is_empty() { + return Err(Error::invalid_input_source("no columns were selected and with_row_id / with_row_address is false, there is nothing to scan" + .into())); + } + + // A parallelism of 0 would cause `try_buffered(0)` to hang forever instead of erroring + match options.threading_mode { + FilteredReadThreadingMode::OnePartitionMultipleThreads(0) => { + return Err(Error::invalid_input_source( + "FilteredReadThreadingMode::OnePartitionMultipleThreads must be greater than 0, got 0" + .into(), + )); + } + FilteredReadThreadingMode::MultiplePartitions(0) => { + return Err(Error::invalid_input_source( + "FilteredReadThreadingMode::MultiplePartitions must be greater than 0, got 0" + .into(), + )); + } + _ => {} + } + + if options.scan_range_after_filter.is_some() { + // Validate that there's a filter when using scan_range_after_filter + if options.full_filter.is_none() + && options.refine_filter.is_none() + && input.row_set_plan().is_none() + { + return Err(Error::invalid_input_source("scan_range_after_filter requires a filter to be applied. Use scan_range_before_filter for unfiltered scans." + .into())); + } + + // TODO: support multi partition + if matches!( + options.threading_mode, + FilteredReadThreadingMode::MultiplePartitions(_) + ) { + return Err(Error::not_supported_source( + "scan_range_after_filter not yet supported with multiple partitions" + .to_string() + .into(), + )); + } + } + let output_schema = public_blob_v2_binary_projection_schema(&options.projection); + let num_partitions = match options.threading_mode { + FilteredReadThreadingMode::OnePartitionMultipleThreads(_) => 1, + FilteredReadThreadingMode::MultiplePartitions(n) => n, + }; + + let properties = Arc::new(PlanProperties::new( EquivalenceProperties::new(output_schema), Partitioning::RoundRobinBatch(num_partitions), EmissionType::Incremental, @@ -1611,12 +2355,16 @@ impl FilteredReadExec { let metrics = ExecutionPlanMetricsSet::new(); Ok(Self { + materialization_context: BlobMaterializationContext::new( + options.io_buffer_size_bytes, + options.materialization_readahead_bytes, + ), dataset, options, properties, running_stream: Arc::new(AsyncMutex::new(None)), metrics, - index_input, + input, plan: Arc::new(OnceCell::new()), }) } @@ -1675,9 +2423,15 @@ impl FilteredReadExec { let index_search_result = index_search.next().await.ok_or_else(|| { Error::internal("Index search did not yield any results".to_string()) })??; - evaluated_index = Some(Arc::new(EvaluatedIndex::try_from_arrow( - &index_search_result, - )?)); + let mut idx = EvaluatedIndex::try_from_arrow(&index_search_result)?; + // `overlay_block` is always constructed as a block list (see + // `Scanner::stale_rows_block_mask`), so `block_list()` is always `Some`. + if let Some(block_list) = + options.overlay_block.as_ref().and_then(|b| b.block_list()) + { + idx = idx.without_rows(block_list); + } + evaluated_index = Some(Arc::new(idx)); } // Load fragments to compute the plan @@ -1688,24 +2442,74 @@ impl FilteredReadExec { .unwrap_or_else(|| dataset.fragments().clone()); let with_deleted_rows = options.with_deleted_rows; + // A range before filtering is expressed in dataset/fragment order. Planning it + // needs every preceding fragment's logical row count, including fragments that + // the index later eliminates. Without that range, discard covered non-candidate + // fragments before opening their full metadata. + let needs_fragment_offsets = options.scan_range_before_filter.is_some(); + let only_indexed_fragments = options.only_indexed_fragments; + let retained_range_bytes = Arc::new(AtomicUsize::new(0)); let frag_futs = fragments .iter() - .map(|frag| { - Result::Ok(FilteredReadStream::load_fragment( - dataset.clone(), - frag.clone(), - with_deleted_rows, - )) + .map(|fragment| { + let dataset = dataset.clone(); + let evaluated_index = evaluated_index.clone(); + let retained_range_bytes = retained_range_bytes.clone(); + let fragment = fragment.clone(); + async move { + let metadata_load = if needs_fragment_offsets { + FragmentMetadataLoad::Load + } else if let Some(index) = evaluated_index { + index + .fragment_metadata_load( + dataset.as_ref(), + &fragment, + only_indexed_fragments, + retained_range_bytes.as_ref(), + ) + .await? + } else if only_indexed_fragments { + FragmentMetadataLoad::Skip + } else { + FragmentMetadataLoad::Load + }; + match metadata_load { + FragmentMetadataLoad::Skip => Ok::<_, Error>(None), + FragmentMetadataLoad::Load => Ok(Some( + FilteredReadStream::load_fragment( + dataset, + fragment, + with_deleted_rows, + None, + ) + .await?, + )), + FragmentMetadataLoad::LoadWithStableRouting(routing) => Ok(Some( + FilteredReadStream::load_fragment( + dataset, + fragment, + with_deleted_rows, + Some(routing), + ) + .await?, + )), + } + } }) .collect::>(); let loaded_fragments = futures::stream::iter(frag_futs) - .try_buffered(io_parallelism) - .try_collect::>() - .await?; + .buffered(io_parallelism) + .try_collect::>>() + .await? + .into_iter() + .flatten() + .collect::>(); - // Plan the scan + // Plan the scan; the metadata loaded here drops when planning + // finishes — stream construction rebuilds I/O-free handles + // from the manifest descriptors Ok(FilteredReadStream::plan_scan( - &loaded_fragments, + loaded_fragments, &evaluated_index, options, )) @@ -1715,11 +2519,18 @@ impl FilteredReadExec { /// Get the existing plan or create it if it doesn't exist pub async fn get_or_create_plan(&self, ctx: Arc) -> Result { + if self.row_stream_input().is_some() { + return Err(Error::not_supported_source( + "a FilteredReadExec with a row-stream source does not have a precomputable plan" + .to_string() + .into(), + )); + } let internal_plan = Self::get_or_create_plan_impl( &self.plan, self.dataset.clone(), &self.options, - self.index_input.as_ref(), + self.input.row_set_plan(), 0, ctx, ) @@ -1746,13 +2557,15 @@ impl FilteredReadExec { n.min(target_partitions).max(1), ); } + let batch_size_rows = options.batch_size; let batch_size_bytes = options .file_reader_options .as_ref() .and_then(|o| o.batch_size_bytes); let metrics = self.metrics.clone(); - let index_input = self.index_input.clone(); + let index_input = self.input.row_set_plan().cloned(); let plan_cell = self.plan.clone(); + let materialization_context = self.materialization_context.clone(); let stream = futures::stream::once(async move { let mut running_stream = running_stream_lock.lock().await; @@ -1769,16 +2582,44 @@ impl FilteredReadExec { ) .await .map_err(|e| DataFusionError::External(e.into()))?; - let new_running_stream = - FilteredReadStream::try_new(dataset, options, &metrics, plan.clone()) - .await - .map_err(|e| DataFusionError::External(e.into()))?; + let new_running_stream = FilteredReadStream::try_new( + dataset, + options, + Arc::new(FilteredReadGlobalMetrics::new(&metrics)), + plan.clone(), + None, + None, + materialization_context, + true, + ); let first_stream = new_running_stream.get_stream(&metrics, partition); *running_stream = Some(new_running_stream); first_stream }; - let stream: SendableRecordBatchStream = match batch_size_bytes { - Some(target) => { + // Only masked reads consolidate; plain scans keep their batch + // boundaries, and the byte-based rechunk merges on its own + let consolidate = if index_input.is_some() && batch_size_bytes.is_none() { + running_stream.as_ref().and_then(|running| { + // Explicit option → lance env default → session batch size + let batch_target_rows = batch_size_rows + .map(|batch_size| batch_size as usize) + .or_else(get_default_batch_size) + .unwrap_or_else(|| context.session_config().batch_size()); + let is_sparse_plan = batch_target_rows > 0 + && running.touched_fragments >= CONSOLIDATE_MIN_FRAGMENTS + && running.planned_rows + < running.touched_fragments as u64 + * CONSOLIDATE_MAX_AVG_PLANNED_ROWS_PER_FRAGMENT; + is_sparse_plan.then_some(batch_target_rows) + }) + } else { + None + }; + drop(running_stream); + + let stream = match (consolidate, batch_size_bytes) { + (Some(target), _) => consolidated_stream(inner, target), + (None, Some(bytes)) => { let schema = inner.schema(); Box::pin(RecordBatchStreamAdapter::new( schema.clone(), @@ -1786,11 +2627,11 @@ impl FilteredReadExec { inner, schema, 0, - target as usize, + bytes as usize, ), )) } - None => inner, + (None, None) => inner, }; DataFusionResult::::Ok(stream) }) @@ -1808,17 +2649,504 @@ impl FilteredReadExec { } pub fn index_input(&self) -> Option<&Arc> { - self.index_input.as_ref() + self.input.row_set_plan() + } + + pub fn row_stream_input(&self) -> Option<&Arc> { + match &self.input { + RowSelector::RowStream(source) => Some(&source.plan), + _ => None, + } } /// Return the pre-computed plan if one exists, without triggering initialization. pub fn plan(&self) -> Option { self.plan.get().map(|p| p.to_external_plan()) } + + fn execute_row_stream( + &self, + source: &Arc, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let input_stream = source.plan.execute(partition, context)?; + let dataset = self.dataset.clone(); + let source = source.clone(); + let carried_schema = + Self::carried_schema(source.plan.schema().as_ref(), &self.options.projection); + let output_schema = self.schema(); + let metrics = self.metrics.clone(); + let materialization_context = self.materialization_context.clone(); + + let lazy_stream = futures::stream::once(async move { + let row_stream_read = Arc::new(RowStreamRead::new( + dataset, + source, + carried_schema, + output_schema, + &metrics, + partition, + materialization_context, + )); + row_stream_read.apply(input_stream) + }) + .flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + lazy_stream, + ))) + } +} + +/// How many batches run concurrently. Each batch's read already carries +/// the full fragment-readahead and decode parallelism, so a shallow pipeline +/// keeps the I/O pipe full; running every batch at once only multiplies that +/// into lock contention +const ROW_STREAM_CONCURRENT_BATCHES: usize = 4; + +/// Fragment metadata, loaded on the first batch and reused afterwards +struct StreamFragments { + /// All dataset (or scoped) fragments, in dataset order + fragments: Vec, + /// Fragment id → position in `fragments` + positions: HashMap, + /// Each fragment's row-id span, for skipping fragments a batch cannot + /// touch (None = empty fragment) + id_spans: Vec>>, +} + +impl StreamFragments { + fn get(&self, fragment_id: u32) -> Option<&LoadedFragment> { + self.positions + .get(&fragment_id) + .map(|position| &self.fragments[*position]) + } +} + +/// Executes a [`FilteredReadExec`] over a row-stream source +struct RowStreamRead { + dataset: Arc, + source: Arc, + /// The input columns that carry through to the output + carried_schema: SchemaRef, + output_schema: SchemaRef, + scan_scheduler: Arc, + materialization_context: Arc, + loaded_fragments: OnceCell, + global_metrics: Arc, + baseline_metrics: BaselineMetrics, +} + +impl RowStreamRead { + fn new( + dataset: Arc, + source: Arc, + carried_schema: SchemaRef, + output_schema: SchemaRef, + metrics: &ExecutionPlanMetricsSet, + partition: usize, + materialization_context: Arc, + ) -> Self { + let scan_scheduler = + FilteredReadStream::make_scan_scheduler(&dataset, &source.read_options); + Self { + dataset, + source, + carried_schema, + output_schema, + scan_scheduler, + materialization_context, + loaded_fragments: OnceCell::new(), + global_metrics: Arc::new(FilteredReadGlobalMetrics::new(metrics)), + baseline_metrics: BaselineMetrics::new(metrics, partition), + } + } + + async fn load_fragments(&self) -> Result<&StreamFragments> { + self.loaded_fragments + .get_or_try_init(|| async { + let fragments = FilteredReadStream::load_all_fragments( + &self.dataset, + &self.source.read_options, + ) + .await?; + let positions = fragments + .iter() + .enumerate() + .map(|(position, fragment)| (fragment.fragment.id() as u32, position)) + .collect(); + let id_spans = fragments + .iter() + .map(|fragment| fragment.row_id_sequence.row_id_range()) + .collect(); + Ok(StreamFragments { + fragments, + positions, + id_spans, + }) + }) + .await + } + + /// Build a batch's read ranges directly from physical row addresses + fn plan_batch_from_addresses( + addrs: &RowAddrTreeMap, + fragments: &StreamFragments, + ) -> FilteredReadInternalPlan { + let mut rows: BTreeMap>> = BTreeMap::new(); + for (fragment_id, requested) in addrs.iter() { + // Unknown fragments (e.g. fully deleted) drop like stale keys + let Some(fragment) = fragments.get(*fragment_id) else { + continue; + }; + let requested = match requested { + RowAddrSelection::Full => vec![0..fragment.num_physical_rows], + RowAddrSelection::Partial(bitmap) => bitmap_to_ranges(bitmap), + }; + let valid = FilteredReadStream::full_frag_range( + fragment.num_physical_rows, + &fragment.deletion_vector, + ); + let matched = FilteredReadStream::intersect_ranges(&valid, &requested); + if !matched.is_empty() { + rows.insert(*fragment_id, matched); + } + } + FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + } + } + + /// Build a batch's read ranges by resolving stable row ids through the + /// fragments' row-id sequences + fn plan_batch_from_row_ids( + ids: RowAddrTreeMap, + keys: &arrow_array::PrimitiveArray, + fragments: &StreamFragments, + ) -> FilteredReadInternalPlan { + let mut rows: BTreeMap>> = BTreeMap::new(); + let (Some(min_key), Some(max_key)) = (arrow::compute::min(keys), arrow::compute::max(keys)) + else { + // Every key is null + return FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + }; + }; + let requested = RowAddrMask::from_allowed(ids); + for (fragment, id_span) in fragments.fragments.iter().zip(&fragments.id_spans) { + // Only fragments whose id span overlaps the batch's key range + // can hold requested rows + let Some(id_span) = id_span else { continue }; + if *id_span.end() < min_key || *id_span.start() > max_key { + continue; + } + let offsets = fragment.row_id_sequence.mask_to_offset_ranges(&requested); + if offsets.is_empty() { + continue; + } + let valid = FilteredReadStream::full_frag_range( + fragment.num_physical_rows, + &fragment.deletion_vector, + ); + let matched = FilteredReadStream::intersect_ranges(&valid, &offsets); + if !matched.is_empty() { + rows.insert(fragment.fragment.id() as u32, matched); + } + } + FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + } + } + + fn key_array<'a>( + &self, + batch: &'a RecordBatch, + producer: &str, + ) -> DataFusionResult<&'a arrow_array::PrimitiveArray> { + let keys = batch + .column_by_name(self.source.key_column) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "the row-stream {} is missing the '{}' column", + producer, self.source.key_column + )) + })?; + keys.as_primitive_opt::().ok_or_else(|| { + DataFusionError::Internal(format!( + "expected the row-stream column '{}' to be UInt64 but it was {}", + self.source.key_column, + keys.data_type() + )) + }) + } + + async fn plan_batch( + &self, + keys: &arrow_array::PrimitiveArray, + ) -> DataFusionResult { + let compute_timer = self.baseline_metrics.elapsed_compute().timer(); + // Null keys are excluded; attach_columns drops their rows + let batch_keys = if keys.null_count() == 0 { + RowAddrTreeMap::from_iter(keys.values().iter().copied()) + } else { + RowAddrTreeMap::from_iter(keys.iter().flatten()) + }; + drop(compute_timer); + + let fragments = self.load_fragments().await?; + // Row ids equal row addresses when the dataset does not use stable + // row ids, so either key resolves directly by position + if self.source.key_column == ROW_ADDR || !self.dataset.manifest.uses_stable_row_ids() { + Ok(Self::plan_batch_from_addresses(&batch_keys, fragments)) + } else { + Ok(Self::plan_batch_from_row_ids(batch_keys, keys, fragments)) + } + } + + /// Read the batch's planned ranges through the same executor as a scan, + /// returning the rows in storage order, deduplicated, with the key + /// column included + async fn read_batch( + &self, + internal_plan: FilteredReadInternalPlan, + batch_index: u32, + ) -> DataFusionResult { + let fragment_count = self.load_fragments().await?.fragments.len(); + // I/O priority: earlier batches strictly first (output emits in batch + // order), fragments keep dataset order within a batch + let priority_offset = batch_index.saturating_mul(fragment_count as u32); + let read = FilteredReadStream::try_new( + self.dataset.clone(), + self.source.read_options.clone(), + self.global_metrics.clone(), + internal_plan, + Some(self.scan_scheduler.clone()), + Some(priority_offset), + self.materialization_context.clone(), + false, + ); + let decode_parallelism = match self.source.read_options.threading_mode { + FilteredReadThreadingMode::OnePartitionMultipleThreads(n) => n, + FilteredReadThreadingMode::MultiplePartitions(n) => n, + }; + let read_batches = read.collect_all(decode_parallelism.max(1)).await?; + Ok(MaterializedBlobBatch::concat( + &read.output_schema, + read_batches, + )?) + } + + /// Align the read rows back to the batch's row order and merge the + /// fetched columns on + fn attach_columns( + &self, + batch: RecordBatch, + read_data: MaterializedBlobBatch, + ) -> DataFusionResult { + let _compute_timer = self.baseline_metrics.elapsed_compute().timer(); + let keys = self.key_array(&batch, "input")?; + let read_keys = self.key_array(read_data.batch(), "read")?; + let output = attach_read_columns( + &batch, + keys, + read_data.batch(), + read_keys, + self.carried_schema.as_ref(), + self.source.new_fields_schema.as_ref(), + &self.source.intermediate_output_schema, + )?; + Ok(read_data.with_batch(output)) + } + + async fn execute_batch( + self: Arc, + batch: RecordBatch, + batch_index: u32, + admission: crate::dataset::blob::BlobMaterializationAdmission, + ) -> DataFusionResult { + if batch.num_rows() == 0 { + return Ok(MaterializedBlobBatch::unreserved(RecordBatch::new_empty( + self.output_schema.clone(), + ))); + } + let internal_plan = self.plan_batch(self.key_array(&batch, "input")?).await?; + let read_data = self.read_batch(internal_plan, batch_index).await?; + let attached = self.attach_columns(batch, read_data)?; + if let Some(output_schema) = &self.source.materialization_output_schema { + Ok( + crate::dataset::blob::materialize_blob_v2_binary_batch_with_admission( + &self.dataset, + output_schema, + attached.into_batch(), + &self.materialization_context, + admission, + ) + .await?, + ) + } else { + drop(admission); + Ok(attached) + } + } + + fn apply( + self: Arc, + input: SendableRecordBatchStream, + ) -> impl Stream> { + let batch_target_rows = self + .source + .read_options + .batch_size + .map(|batch_size| batch_size as usize) + .unwrap_or_else(|| get_default_batch_size().unwrap_or(BATCH_SIZE_FALLBACK)); + let on_result = self.clone(); + let on_done = self.clone(); + coalesce_batches(input, batch_target_rows) + .enumerate() + .map(move |(batch_index, batch)| { + let batch = batch?; + let this = self.clone(); + let admission = this.materialization_context.admission(); + DataFusionResult::Ok( + // SpawnedTask aborts on drop: cancelling the query + // cancels in-flight batches + SpawnedTask::spawn( + this.execute_batch(batch, batch_index as u32, admission) + .in_current_span(), + ) + .map(|res| match res { + Ok(result) => result, + Err(join_error) => Err(DataFusionError::External(Box::new(join_error))), + }), + ) + }) + .boxed() + .try_buffered(ROW_STREAM_CONCURRENT_BATCHES) + .map_ok(MaterializedBlobBatch::into_batch) + .map(move |result| { + on_result + .global_metrics + .io_metrics + .record(&on_result.scan_scheduler); + match on_result + .baseline_metrics + .record_poll(Poll::Ready(Some(result))) + { + Poll::Ready(Some(result)) => result, + _ => unreachable!("record_poll returned a different poll state"), + } + }) + .finally(move || { + on_done.baseline_metrics.done(); + on_done + .global_metrics + .io_metrics + .record(&on_done.scan_scheduler); + }) + } +} + +/// Align `read_data` rows back to a keyed batch's row order and merge the +/// fetched columns on. +/// +/// `keys` is the row-id-space key of each `batch` row (usually the batch's +/// own key column; a caller that keys by address passes the resolved ids +/// instead) and `read_keys` the key of each `read_data` row, which must be +/// unique. Input rows whose key has no read row — null or stale keys — are +/// DROPPED; duplicate input keys re-expand through the gather. Output +/// columns follow `output_schema`: `carried_schema` names come from `batch`, +/// `new_fields_schema` names from `read_data`. +pub fn attach_read_columns( + batch: &RecordBatch, + keys: &arrow_array::PrimitiveArray, + read_data: &RecordBatch, + read_keys: &arrow_array::PrimitiveArray, + carried_schema: &arrow_schema::Schema, + new_fields_schema: &arrow_schema::Schema, + output_schema: &SchemaRef, +) -> DataFusionResult { + // Fast path: one read row per input row with an identical key sequence — + // already aligned, skip the hash map and the permutation + if keys.null_count() == 0 + && read_data.num_rows() == batch.num_rows() + && read_keys.values() == keys.values() + { + let new_data = read_data.project_by_schema(new_fields_schema)?; + let carried = batch.project_by_schema(carried_schema)?; + return Ok(carried.merge_with_schema(&new_data, output_schema.as_ref())?); + } + + let key_to_index: HashMap = read_keys + .values() + .iter() + .enumerate() + .map(|(index, key)| (*key, index as u32)) + .collect(); + + // Sizes differ only when some input keys have no live row (null or + // stale keys): drop those input rows first + let (batch, keys) = if read_data.num_rows() != batch.num_rows() { + let matched: BooleanArray = keys + .iter() + .map(|key| key.map(|key| key_to_index.contains_key(&key))) + .collect(); + let keys = arrow::compute::filter(keys, &matched)? + .as_primitive::() + .clone(); + (arrow::compute::filter_record_batch(batch, &matched)?, keys) + } else { + (batch.clone(), keys.clone()) + }; + if batch.num_rows() == 0 { + return Ok(RecordBatch::new_empty(output_schema.clone())); + } + + // Gather the read rows into input order — every remaining key hits + let indices = UInt32Array::from_iter_values(keys.values().iter().map(|key| key_to_index[key])); + let new_data = arrow_select::take::take_record_batch(read_data, &indices)?; + let new_data = new_data.project_by_schema(new_fields_schema)?; + let carried = batch.project_by_schema(carried_schema)?; + Ok(carried.merge_with_schema(&new_data, output_schema.as_ref())?) } impl DisplayAs for FilteredReadExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let RowSelector::RowStream(source) = &self.input { + let columns = source + .new_fields_schema + .fields + .iter() + .map(|f| f.name().as_str()) + .collect::>() + .join(", "); + return match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!( + f, + "LanceRead: uri={}, projection=[{}], source=stream({})", + self.dataset.data_dir(), + columns, + source.key_column, + ) + } + DisplayFormatType::TreeRender => { + write!( + f, + "LanceRead\nuri={}\nprojection=[{}]\nsource=stream({})", + self.dataset.data_dir(), + columns, + source.key_column, + ) + } + }; + } let columns = self .options .projection @@ -1892,22 +3220,24 @@ impl ExecutionPlan for FilteredReadExec { "FilteredReadExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn properties(&self) -> &Arc { &self.properties } fn children(&self) -> Vec<&Arc> { - if let Some(index_input) = &self.index_input { - vec![index_input] + if let Some(child) = self.input.child() { + vec![child] } else { vec![] } } + fn benefits_from_input_partitioning(&self) -> Vec { + // Partitioning a row-stream read would create multiple I/O schedulers + // (RAM heavy); the other selectors have no row input + vec![false; self.children().len()] + } + fn metrics(&self) -> Option { Some(self.metrics.clone_inner()) } @@ -1915,7 +3245,14 @@ impl ExecutionPlan for FilteredReadExec { fn partition_statistics( &self, partition: Option, - ) -> datafusion::error::Result { + ) -> datafusion::error::Result> { + if let RowSelector::RowStream(source) = &self.input { + // At most one output row per input row + return Ok(Arc::new(Statistics { + num_rows: source.plan.partition_statistics(partition)?.num_rows, + ..Statistics::new_unknown(self.schema().as_ref()) + })); + } let fragments = self .options .fragments @@ -1952,10 +3289,10 @@ impl ExecutionPlan for FilteredReadExec { total_rows }; - return Ok(Statistics { + return Ok(Arc::new(Statistics { num_rows: Precision::Exact(total_rows as usize), ..datafusion::physical_plan::Statistics::new_unknown(self.schema().as_ref()) - }); + })); }; // We could evaluate the indexed filter here but this is still during the planning @@ -1973,10 +3310,12 @@ impl ExecutionPlan for FilteredReadExec { .clone() .union_columns(filter_columns, OnMissing::Error)?; - let read_schema = Arc::new(read_projection.to_arrow_schema()); + let read_schema = public_blob_v2_binary_projection_schema(&read_projection); - let planner = Arc::new(Planner::new(read_schema.clone())); - let physical_filter = planner.create_physical_expr(filter)?; + let physical_filter = match self.options.physical_filter(filter) { + Some(physical_filter) => physical_filter, + None => Planner::new(read_schema.clone()).create_physical_expr(filter)?, + }; let mock_input = Arc::new(Self::try_new( self.dataset.clone(), @@ -1990,7 +3329,7 @@ impl ExecutionPlan for FilteredReadExec { None, )?); let df_filter_exec = FilterExec::try_new(physical_filter, mock_input)?; - let mut df_stats = df_filter_exec.partition_statistics(partition)?; + let mut df_stats = Arc::unwrap_or_clone(df_filter_exec.partition_statistics(partition)?); // If we have an after-filter range, we should apply it to the stats (the before-filter range // is applied in the mock input) @@ -2026,7 +3365,7 @@ impl ExecutionPlan for FilteredReadExec { } }); - Ok(df_stats) + Ok(Arc::new(df_stats)) } fn with_new_children( @@ -2038,18 +3377,12 @@ impl ExecutionPlan for FilteredReadExec { Error::internal("A FilteredReadExec cannot have two children".to_string()).into(), )) } else { - let index_input = children.into_iter().next(); - Ok(Arc::new(Self { - dataset: self.dataset.clone(), - options: self.options.clone(), - properties: self.properties.clone(), - metrics: self.metrics.clone(), - // Seems unlikely this would already be initialized but clear it - // out just in case - running_stream: Arc::new(AsyncMutex::new(None)), - index_input, - plan: Arc::new(OnceCell::new()), - })) + // Rebuild via try_new so the selector and derived state are + // re-derived from the new child's schema + let child = children.into_iter().next(); + let rebuilt = Self::try_new(self.dataset.clone(), self.options.clone(), child) + .map_err(|e| DataFusionError::External(e.into()))?; + Ok(Arc::new(rebuilt)) } } @@ -2058,10 +3391,40 @@ impl ExecutionPlan for FilteredReadExec { partition: usize, context: Arc, ) -> DataFusionResult { - Ok(self.obtain_stream(partition, context)) + let stream = match &self.input { + RowSelector::RowStream(source) => self.execute_row_stream(source, partition, context), + _ => Ok(self.obtain_stream(partition, context)), + }?; + + // Readers can omit the logical schema metadata, while row-stream merges + // can retain metadata from their input. Normalize once at the execution + // boundary so every selector satisfies RecordBatchStream's exact schema + // contract. Rebuilding the batch reuses the arrays without copying them. + let output_schema = self.schema(); + let batch_schema = output_schema.clone(); + let stream = stream.map(move |batch| { + let batch = batch?; + if batch.schema_ref() == &batch_schema { + return Ok(batch); + } + let (_, columns, row_count) = batch.into_parts(); + RecordBatch::try_new_with_options( + batch_schema.clone(), + columns, + &RecordBatchOptions::new().with_row_count(Some(row_count)), + ) + .map_err(|e| DataFusionError::ArrowError(Box::new(e), None)) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + output_schema, + stream, + ))) } fn fetch(&self) -> Option { + if self.row_stream_input().is_some() { + return None; + } if self.options.full_filter.is_none() { self.options .scan_range_before_filter @@ -2076,13 +3439,15 @@ impl ExecutionPlan for FilteredReadExec { } fn supports_limit_pushdown(&self) -> bool { - // This is to push the limit through the node and into an upstream node. - // The only upstream node is the index search and we can't push the limit - // to that node. - false + // A limit pushes through to a row-stream input (one output row per + // input row); the other selectors have no node to push it to + self.row_stream_input().is_some() } fn with_fetch(&self, limit: Option) -> Option> { + if self.row_stream_input().is_some() { + return None; + } // TODO: Support multiple partitions in the future by coordinating limits across partitions if matches!( self.options.threading_mode, @@ -2099,7 +3464,12 @@ impl ExecutionPlan for FilteredReadExec { let mut updated_options = self.options.clone(); if self.options.full_filter.is_none() && self.options.refine_filter.is_none() { - if self.options.scan_range_before_filter.is_some() { + // A before-filter range trims raw scan positions, which is only valid for + // an unindexed full scan. With an index_input (e.g. an external row mask or + // a scalar-index result) the rows are selected by that input, so a pre-range + // would apply before selection and keep the wrong rows; leave the limit to a + // node above the read instead. + if self.options.scan_range_before_filter.is_some() || self.index_input().is_some() { return None; } updated_options.scan_range_before_filter = Some(0..(limit as u64)); @@ -2113,7 +3483,7 @@ impl ExecutionPlan for FilteredReadExec { match Self::try_new( self.dataset.clone(), updated_options, - self.index_input.clone(), + self.input.row_set_plan().cloned(), ) { Ok(exec) => Some(Arc::new(exec)), Err(e) => { @@ -2130,7 +3500,8 @@ impl ExecutionPlan for FilteredReadExec { #[cfg(test)] mod tests { - use std::collections::HashSet; + use std::collections::{HashMap, HashSet}; + use std::sync::atomic::{AtomicUsize, Ordering}; use crate::index::DatasetIndexExt; use arrow::{ @@ -2142,7 +3513,9 @@ mod tests { }; use itertools::Itertools; use lance_core::datatypes::OnMissing; + use lance_core::utils::address::RowAddress; use lance_core::utils::tempfile::TempStrDir; + use lance_datafusion::exec::OneShotExec; use lance_datagen::{BatchCount, Dimension, RowCount, array, gen_batch}; use lance_index::{ IndexType, @@ -2150,6 +3523,12 @@ mod tests { scalar::{ScalarIndexParams, expression::PlannerIndexExt}, }; use lance_select::result::IndexExprResultWireFormat; + use lance_select::{RowAddrMask, RowAddrTreeMap}; + use rstest::rstest; + use tracing_subscriber::{ + Layer, + layer::{Context, SubscriberExt}, + }; use crate::{ dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams}, @@ -2160,6 +3539,53 @@ mod tests { use super::*; + #[derive(Clone, Default)] + struct MaskToOffsetRangesCounter { + count: Arc, + } + + impl Layer for MaskToOffsetRangesCounter + where + S: tracing::Subscriber, + { + fn on_new_span( + &self, + attrs: &tracing::span::Attributes<'_>, + _id: &tracing::span::Id, + _ctx: Context<'_, S>, + ) { + if attrs.metadata().name() == "mask_to_offset_ranges" { + self.count.fetch_add(1, Ordering::Relaxed); + } + } + } + + #[test] + fn test_stable_index_range_retention_is_bounded() { + let ranges = vec![0..1]; + let range_bytes = ranges.capacity() * std::mem::size_of::>(); + let retained_range_bytes = AtomicUsize::new( + MAX_RETAINED_STABLE_INDEX_RANGE_BYTES + .checked_sub(range_bytes) + .unwrap(), + ); + + assert!( + EvaluatedIndex::retain_stable_index_ranges(ranges, &retained_range_bytes).is_some() + ); + assert_eq!( + retained_range_bytes.load(Ordering::Relaxed), + MAX_RETAINED_STABLE_INDEX_RANGE_BYTES + ); + assert!( + EvaluatedIndex::retain_stable_index_ranges(vec![1..2], &retained_range_bytes).is_none() + ); + assert_eq!( + retained_range_bytes.load(Ordering::Relaxed), + MAX_RETAINED_STABLE_INDEX_RANGE_BYTES + ); + } + struct TestFixture { _tmp_path: TempStrDir, dataset: Arc, @@ -2370,12 +3796,399 @@ mod tests { (tmp_path, Arc::new(dataset)) } + #[rstest] + #[case::unfiltered(None)] + #[case::filtered(Some("value > 2"))] + #[tokio::test] + async fn test_output_batches_preserve_schema_metadata(#[case] filter: Option<&str>) { + let tmp_path = TempStrDir::default(); + let metadata = HashMap::from([( + "embedding_functions".to_string(), + "[{\"name\":\"test\"}]".to_string(), + )]); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![arrow_schema::Field::new( + "value", + arrow_schema::DataType::Int32, + true, + )], + metadata.clone(), + )); + let batch = arrow_array::record_batch!(("value", Int32, [1, 2, 3, 4])) + .unwrap() + .with_schema(schema.clone()) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let dataset = Arc::new( + Dataset::write( + reader, + tmp_path.as_str(), + Some(WriteParams { + max_rows_per_file: 2, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + assert_eq!(dataset.get_fragments().len(), 2); + + let mut options = FilteredReadOptions::basic_full_read(&dataset); + if let Some(filter) = filter { + let arrow_schema = Arc::new(ArrowSchema::from(dataset.schema())); + let planner = Planner::new(arrow_schema); + let expr = planner.parse_filter(filter).unwrap(); + options = options.with_filter(Some(expr.clone()), Some(expr)).unwrap(); + } + + let plan = FilteredReadExec::try_new(dataset.clone(), options, None).unwrap(); + let expected_schema = plan.schema(); + assert_eq!(expected_schema.metadata(), &metadata); + let batches = plan + .execute(0, Arc::new(TaskContext::default())) + .unwrap() + .try_collect::>() + .await + .unwrap(); + + assert!(!batches.is_empty()); + assert!( + batches + .iter() + .all(|batch| batch.schema() == expected_schema), + "output schema metadata was not preserved with filter {filter:?}" + ); + } + fn u32s(ranges: Vec>) -> Arc { Arc::new(UInt32Array::from_iter_values( ranges.into_iter().flat_map(|r| r.into_iter()), )) } + async fn metadata_pruning_dataset(uses_stable_row_ids: bool) -> (TempStrDir, Arc) { + let tmp_path = TempStrDir::default(); + let dataset = Arc::new( + gen_batch() + .col("value", array::step::()) + .into_dataset_with_params( + tmp_path.as_str(), + FragmentCount::from(4), + FragmentRowCount::from(10), + Some(WriteParams { + max_rows_per_file: 10, + enable_stable_row_ids: uses_stable_row_ids, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + assert_eq!( + dataset.manifest().uses_stable_row_ids(), + uses_stable_row_ids + ); + (tmp_path, dataset) + } + + fn index_result_input( + result: IndexExprResult, + fragments: &[Fragment], + ) -> Arc { + let covered: RoaringBitmap = fragments + .iter() + .map(|fragment| fragment.id as u32) + .collect(); + index_result_input_with_coverage(result, &covered) + } + + fn index_result_input_with_coverage( + result: IndexExprResult, + covered: &RoaringBitmap, + ) -> Arc { + let batch = result + .serialize(covered, IndexExprResultWireFormat::default()) + .unwrap(); + let schema = batch.schema(); + let index_stream = futures::stream::once(async move { Ok(batch) }); + Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new( + schema, + index_stream, + )))) + } + + /// An exact empty index result should finish without opening metadata for any covered + /// fragment. The invalid file paths are sentinels that make an attempted open fail, turning + /// the metadata-I/O behavior into a deterministic regression assertion. + #[rstest] + #[case::address(false)] + #[case::stable(true)] + #[tokio::test] + async fn test_exact_empty_index_skips_fragment_metadata(#[case] uses_stable_row_ids: bool) { + let (_tmp_path, dataset) = metadata_pruning_dataset(uses_stable_row_ids).await; + + let mut fragments = dataset.fragments().as_ref().clone(); + for fragment in &mut fragments { + fragment.physical_rows = None; + fragment.files[0].path = "must-not-open.lance".to_string(); + if uses_stable_row_ids { + fragment.row_id_meta = None; + } + } + let index_input = index_result_input( + IndexExprResult::exact(RowAddrMask::allow_nothing()), + &fragments, + ); + + let options = + FilteredReadOptions::basic_full_read(&dataset).with_fragments(Arc::new(fragments)); + let plan = FilteredReadExec::try_new(dataset, options, Some(index_input)).unwrap(); + let batches = plan + .execute(0, Arc::new(TaskContext::default())) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert!(batches.is_empty()); + } + + /// A sparse non-empty result should only open full metadata for candidate fragments. Stable + /// row ids still need the inline row-id sequence to route candidates, but must not open the + /// data file or deletion metadata of non-candidate fragments. + #[rstest] + #[case::address(false)] + #[case::stable(true)] + #[tokio::test] + async fn test_sparse_index_skips_non_candidate_fragment_metadata( + #[case] uses_stable_row_ids: bool, + ) { + let (_tmp_path, dataset) = metadata_pruning_dataset(uses_stable_row_ids).await; + + let mut fragments = dataset.fragments().as_ref().clone(); + for fragment in fragments.iter_mut().skip(1) { + fragment.physical_rows = None; + fragment.files[0].path = "must-not-open.lance".to_string(); + } + // The initial stable row ids are 0..40, so row id 3 and address (0, 3) select the same + // physical row in their respective modes. + let candidate = if uses_stable_row_ids { + 3 + } else { + RowAddress::new_from_parts(0, 3).into() + }; + let candidates = RowAddrTreeMap::from_iter([candidate]); + let index_input = index_result_input( + IndexExprResult::exact(RowAddrMask::from_allowed(candidates)), + &fragments, + ); + + let options = + FilteredReadOptions::basic_full_read(&dataset).with_fragments(Arc::new(fragments)); + let plan = FilteredReadExec::try_new(dataset, options, Some(index_input)).unwrap(); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let schema = stream.schema(); + let batches = stream.try_collect::>().await.unwrap(); + let batch = concat_batches(&schema, &batches).unwrap(); + + assert_eq!(batch["value"].as_primitive::().values(), &[3]); + } + + /// Dense stable-ID masks route every fragment. Carry the ranges computed during routing into + /// final planning so this case does not map the same IDs to offsets twice. + #[tokio::test(flavor = "current_thread")] + async fn test_dense_stable_index_reuses_routing_ranges() { + let (_tmp_path, dataset) = metadata_pruning_dataset(true).await; + let fragments = dataset.fragments().as_ref().clone(); + let index_input = index_result_input( + IndexExprResult::exact(RowAddrMask::from_allowed(RowAddrTreeMap::from(0_u64..40))), + &fragments, + ); + let options = FilteredReadOptions::basic_full_read(&dataset); + let read = FilteredReadExec::try_new(dataset, options, Some(index_input)).unwrap(); + let counter = MaskToOffsetRangesCounter::default(); + let subscriber = tracing_subscriber::registry().with(counter.clone()); + let _guard = tracing::subscriber::set_default(subscriber); + + let plan = read + .get_or_create_plan(Arc::new(TaskContext::default())) + .await + .unwrap(); + + assert_eq!(counter.count.load(Ordering::Relaxed), 4); + assert_eq!(plan.rows.iter().count(), 4); + for (_, selection) in plan.rows.iter() { + let RowAddrSelection::Partial(offsets) = selection else { + panic!("dense stable-ID plan should contain explicit offsets"); + }; + assert_eq!(offsets.iter().collect_vec(), (0..10).collect_vec()); + } + } + + /// Pruning must use the upper bound of an inexact result. The lower bound alone contains row + /// 3, while row 13 is only a possible match in the upper bound and must still be read. + #[rstest] + #[case::address(false)] + #[case::stable(true)] + #[tokio::test] + async fn test_refined_index_prunes_from_upper_bound(#[case] uses_stable_row_ids: bool) { + let (_tmp_path, dataset) = metadata_pruning_dataset(uses_stable_row_ids).await; + + let mut fragments = dataset.fragments().as_ref().clone(); + for fragment in fragments.iter_mut().skip(2) { + fragment.physical_rows = None; + fragment.files[0].path = "must-not-open.lance".to_string(); + } + let candidate = |fragment_id, offset, stable_row_id| { + if uses_stable_row_ids { + stable_row_id + } else { + RowAddress::new_from_parts(fragment_id, offset).into() + } + }; + let definite_row = candidate(0, 3, 3); + let possible_row = candidate(1, 3, 13); + let result = IndexExprResult::new( + RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([definite_row])), + RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([definite_row, possible_row])), + ); + let index_input = index_result_input(result, &fragments); + + let options = + FilteredReadOptions::basic_full_read(&dataset).with_fragments(Arc::new(fragments)); + let plan = FilteredReadExec::try_new(dataset, options, Some(index_input)).unwrap(); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let schema = stream.schema(); + let batches = stream.try_collect::>().await.unwrap(); + let batch = concat_batches(&schema, &batches).unwrap(); + + assert_eq!( + batch["value"].as_primitive::().values(), + &[3, 13] + ); + } + + /// An empty result only eliminates fragments covered by the index. Uncovered fragments must + /// still be scanned for a complete query, while fast search intentionally omits them. + #[rstest] + #[case::address_complete(false, false)] + #[case::address_fast(false, true)] + #[case::stable_complete(true, false)] + #[case::stable_fast(true, true)] + #[tokio::test] + async fn test_empty_index_preserves_uncovered_fragments( + #[case] uses_stable_row_ids: bool, + #[case] only_indexed_fragments: bool, + ) { + let (_tmp_path, dataset) = metadata_pruning_dataset(uses_stable_row_ids).await; + + let mut fragments = dataset.fragments().as_ref().clone(); + let covered: RoaringBitmap = fragments + .iter() + .take(3) + .map(|fragment| fragment.id as u32) + .collect(); + for fragment in fragments.iter_mut().take(3) { + fragment.physical_rows = None; + fragment.files[0].path = "must-not-open.lance".to_string(); + if uses_stable_row_ids { + fragment.row_id_meta = None; + } + } + let index_input = index_result_input_with_coverage( + IndexExprResult::exact(RowAddrMask::allow_nothing()), + &covered, + ); + + let mut options = + FilteredReadOptions::basic_full_read(&dataset).with_fragments(Arc::new(fragments)); + if only_indexed_fragments { + options = options.with_only_indexed_fragments(); + } + let plan = FilteredReadExec::try_new(dataset, options, Some(index_input)).unwrap(); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let schema = stream.schema(); + let batches = stream.try_collect::>().await.unwrap(); + let batch = concat_batches(&schema, &batches).unwrap(); + + let expected = if only_indexed_fragments { + UInt32Array::from(Vec::::new()) + } else { + UInt32Array::from((30..40).collect::>()) + }; + assert_eq!(batch["value"].as_ref(), &expected); + } + + /// Take-shaped masked reads consolidate their tiny per-fragment batches; + /// few-fragment and dense masked reads keep per-fragment boundaries. + #[test_log::test(tokio::test)] + async fn test_take_shaped_mask_consolidation() { + // 20 fragments x 2000 rows, value = global row number + let tmp_path = TempStrDir::default(); + let data = gen_batch() + .col("value", array::step::()) + .into_reader_rows(RowCount::from(2000), BatchCount::from(20)); + let dataset = Arc::new( + Dataset::write( + data, + tmp_path.as_str(), + Some(WriteParams { + max_rows_per_file: 2000, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let mask_input = |addrs: Vec| -> Arc { + let covered: RoaringBitmap = dataset.fragments().iter().map(|f| f.id as u32).collect(); + let batch = + IndexExprResult::exact(RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(addrs))) + .serialize(&covered, IndexExprResultWireFormat::default()) + .unwrap(); + let schema = batch.schema(); + let stream = futures::stream::once(async move { Ok(batch) }); + Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new( + schema, stream, + )))) + }; + let run = |input: Arc| { + let dataset = dataset.clone(); + async move { + // Pin the batch size so batch-count assertions don't depend + // on LANCE_DEFAULT_BATCH_SIZE + let options = FilteredReadOptions::basic_full_read(&dataset).with_batch_size(2000); + let plan = + FilteredReadExec::try_new(dataset.clone(), options, Some(input)).unwrap(); + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + stream.try_collect::>().await.unwrap() + } + }; + let addr = |frag: u32, offset: u32| u64::from(RowAddress::new_from_parts(frag, offset)); + + // Take shape: 20 fragments, 2 rows each -> one consolidated batch, + // rows in fragment order + let addrs: Vec = (0..20u32).flat_map(|f| [addr(f, 3), addr(f, 7)]).collect(); + let batches = run(mask_input(addrs)).await; + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 40); + assert_eq!(batches.len(), 1); + let expected = + UInt32Array::from_iter_values((0..20u32).flat_map(|f| [f * 2000 + 3, f * 2000 + 7])); + assert_eq!(batches[0].column(0).as_ref(), &expected); + + // Too few fragments -> inline path, one batch per fragment + let batches = run(mask_input(vec![addr(0, 3), addr(1, 7)])).await; + assert_eq!(batches.len(), 2); + + // Dense (2000 planned rows per fragment) -> inline path + let addrs: Vec = (0..8u32) + .flat_map(|f| (0..2000u32).map(move |o| addr(f, o))) + .collect(); + let batches = run(mask_input(addrs)).await; + assert_eq!(batches.len(), 8); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 16000); + } + /// Round-trip every interval shape through the arrow wire format and /// confirm the endpoints survive. Exercises both /// `IndexExprResult::serialize` and `EvaluatedIndex::try_from_arrow` @@ -2410,15 +4223,15 @@ mod tests { // robust, but the canonical builders preserve representation. assert_eq!( decoded.index_result.lower, original.lower, - "{name}: lower endpoint changed across round-trip", + "{name}: lower endpoint changed across batch-trip", ); assert_eq!( decoded.index_result.upper, original.upper, - "{name}: upper endpoint changed across round-trip", + "{name}: upper endpoint changed across batch-trip", ); assert_eq!( decoded.applicable_fragments, frags, - "{name}: applicable fragments changed across round-trip", + "{name}: applicable fragments changed across batch-trip", ); } } @@ -2765,6 +4578,48 @@ mod tests { assert_eq!(num_rows, 300); } + /// A stale (not rebuilt after a delete) index hit drops on the live view + /// and returns as a null-_rowid tombstone with with_deleted_rows + #[test_log::test(tokio::test)] + async fn test_with_deleted_rows_stale_index() { + let fixture = Arc::new(TestFixture::new().await); + let base_options = FilteredReadOptions::basic_full_read(&fixture.dataset); + + // Row 220 is deletion-vector-deleted but still in the index + let filter_plan = fixture.filter_plan("fully_indexed == 220", true).await; + + // Live view: the stale index hit drops + fixture + .test_plan( + base_options.clone().with_filter_plan(filter_plan), + &u32s(vec![]), + ) + .await; + + // Physical view: the tombstone returns + let filter_plan = fixture.filter_plan("fully_indexed == 220", true).await; + let options = base_options + .with_deleted_rows() + .unwrap() + .with_filter_plan(filter_plan); + let plan = fixture.make_plan(options).await; + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let batches = stream.try_collect::>().await.unwrap(); + let total_rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 1); + let batch = batches.iter().find(|b| b.num_rows() > 0).unwrap(); + let values = batch + .column_by_name("fully_indexed") + .unwrap() + .as_primitive::(); + assert_eq!(values.value(0), 220); + let row_ids = batch + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::(); + assert!(row_ids.is_null(0)); + } + #[test] fn test_dv_to_ranges() { let dv = Arc::new(DeletionVector::from_iter(vec![1])); @@ -2967,10 +4822,7 @@ mod tests { assert_eq!(plan.options().scan_range_before_filter, None); assert_eq!(plan.fetch(), None); let new_plan = plan.with_fetch(Some(100)).unwrap(); - let new_plan = new_plan - .as_any() - .downcast_ref::() - .unwrap(); + let new_plan = new_plan.downcast_ref::().unwrap(); assert_eq!(new_plan.options().scan_range_before_filter, Some(0..100)); assert_eq!(new_plan.fetch(), Some(100)); } @@ -2998,10 +4850,7 @@ mod tests { assert_eq!(plan.options().scan_range_after_filter, None); assert_eq!(plan.fetch(), None); let new_plan = plan.with_fetch(Some(50)).unwrap(); - let new_plan = new_plan - .as_any() - .downcast_ref::() - .unwrap(); + let new_plan = new_plan.downcast_ref::().unwrap(); assert_eq!(new_plan.options().scan_range_after_filter, Some(0..50)); assert_eq!(new_plan.fetch(), Some(50)); } @@ -3039,6 +4888,35 @@ mod tests { let result = plan.with_fetch(None); assert!(result.is_none()); } + + // Case 7: index_input present with no filter (the external-row-mask + // plain-scan shape) - with_fetch must reject before-filter pushdown, since + // the index_input selects the rows and a raw before-filter range would trim + // scan positions before that selection. + { + // Build a real scalar-index input, then attach it to options that carry + // no filter of their own. + let index_filter_plan = fixture.filter_plan("fully_indexed < 200", false).await; + let index_input = fixture + .index_input(&base_options.clone().with_filter_plan(index_filter_plan)) + .await; + assert!(index_input.is_some(), "expected a scalar-index input"); + + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + base_options.clone(), + index_input, + ) + .unwrap(); + assert!(plan.index_input().is_some()); + assert!(plan.options().full_filter.is_none() && plan.options().refine_filter.is_none()); + + let result = plan.with_fetch(Some(100)); + assert!( + result.is_none(), + "with_fetch must reject before-filter pushdown when index_input is present" + ); + } } #[tokio::test] @@ -3051,14 +4929,11 @@ mod tests { let options = base_options.with_filter_plan(filter_plan); let plan = fixture.make_plan(options).await; - assert!(plan.index_input.is_some()); + assert!(plan.index_input().is_some()); assert!(plan.options().refine_filter.is_some()); let limited_plan = plan.with_fetch(Some(10)).unwrap(); - let limited_plan = limited_plan - .as_any() - .downcast_ref::() - .unwrap(); + let limited_plan = limited_plan.downcast_ref::().unwrap(); assert_eq!(limited_plan.options().scan_range_after_filter, Some(0..10)); let stream = limited_plan @@ -3780,7 +5655,7 @@ mod tests { /// Test that direct execution gives the same result as get_plan + execute_with_plan #[test_log::test(tokio::test)] - async fn test_plan_round_trip() { + async fn test_plan_batch_trip() { let fixture = TestFixture::new().await; let ctx = Arc::new(TaskContext::default()); @@ -3893,4 +5768,1003 @@ mod tests { assert_eq!(default_result.num_rows(), capped_result.num_rows()); } + + // Row-stream selector tests + + mod row_stream { + use super::*; + use arrow_array::{Float32Array, LargeBinaryArray, StringArray, UInt64Array}; + use arrow_schema::{DataType, Field as ArrowField, Fields, Schema as ArrowSchema}; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use lance_datafusion::exec::OneShotExec; + use rstest::rstest; + + use crate::blob::{BlobArrayBuilder, blob_field}; + use crate::dataset::{Dataset, WriteParams}; + use crate::utils::test::NoContextTestFixture; + use lance_core::datatypes::BlobHandling; + + struct TakeFixture { + dataset: Arc, + _tmp_dir: TempStrDir, + } + + /// 30 rows across 3 fragments with columns i, s, and struct{x, y} + async fn take_fixture_with_metadata( + stable_row_ids: bool, + metadata: HashMap, + ) -> TakeFixture { + let struct_fields = Fields::from(vec![ + Arc::new(ArrowField::new("x", DataType::Int32, false)), + Arc::new(ArrowField::new("y", DataType::Int32, false)), + ]); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("s", DataType::Utf8, false), + ArrowField::new("struct", DataType::Struct(struct_fields.clone()), false), + ], + metadata, + )); + let batches: Vec = (0..3) + .map(|batch_id| { + let value_range = batch_id * 10..batch_id * 10 + 10; + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(value_range.clone())), + Arc::new(StringArray::from_iter_values( + value_range.clone().map(|v| format!("s-{v}")), + )), + Arc::new(arrow_array::StructArray::new( + struct_fields.clone(), + vec![ + Arc::new(Int32Array::from_iter(value_range.clone())), + Arc::new(Int32Array::from_iter(value_range)), + ], + None, + )), + ], + ) + .unwrap() + }) + .collect(); + + let tmp_dir = TempStrDir::default(); + let uri = tmp_dir.as_str(); + let params = WriteParams { + max_rows_per_file: 10, + enable_stable_row_ids: stable_row_ids, + ..Default::default() + }; + let reader = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + TakeFixture { + dataset: Arc::new(Dataset::open(uri).await.unwrap()), + _tmp_dir: tmp_dir, + } + } + + async fn take_fixture(stable_row_ids: bool) -> TakeFixture { + take_fixture_with_metadata(stable_row_ids, HashMap::new()).await + } + + /// Wrap batches of (payload, key) rows into an input plan + fn rows_input(batches: Vec) -> Arc { + let schema = batches[0].schema(); + let stream = futures::stream::iter(batches.into_iter().map(Ok)); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + Arc::new(OneShotExec::new(stream)) + } + + fn take_plan( + dataset: &Arc, + input: Arc, + columns: &[&str], + ) -> Result { + let projection = dataset + .empty_projection() + .union_columns(columns, OnMissing::Error) + .unwrap(); + FilteredReadExec::try_new( + dataset.clone(), + FilteredReadOptions::new(projection), + Some(input), + ) + } + + fn take_plan_sized( + dataset: &Arc, + input: Arc, + columns: &[&str], + batch_size: u32, + ) -> Result { + let projection = dataset + .empty_projection() + .union_columns(columns, OnMissing::Error) + .unwrap(); + FilteredReadExec::try_new( + dataset.clone(), + FilteredReadOptions::new(projection).with_batch_size(batch_size), + Some(input), + ) + } + + async fn run(plan: &FilteredReadExec) -> Vec { + plan.execute(0, Arc::new(TaskContext::default())) + .unwrap() + .try_collect::>() + .await + .unwrap() + } + + #[rstest] + #[case::aligned(false, HashMap::new())] + #[case::reordered( + true, + HashMap::from([("input_only".to_string(), "true".to_string())]) + )] + #[tokio::test] + async fn row_stream_output_preserves_plan_schema_metadata( + #[case] reordered: bool, + #[case] input_metadata: HashMap, + ) { + let dataset_metadata = HashMap::from([( + "embedding_functions".to_string(), + "[{\"name\":\"test\"}]".to_string(), + )]); + let fixture = take_fixture_with_metadata(false, dataset_metadata.clone()).await; + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys = if reordered { + vec![addr(1, 0), addr(0, 0)] + } else { + vec![addr(0, 0), addr(0, 1)] + }; + let input_schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("payload", DataType::Float32, false), + ArrowField::new(ROW_ADDR, DataType::UInt64, false), + ], + input_metadata, + )); + let input = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(Float32Array::from(vec![0.5, 1.5])), + Arc::new(UInt64Array::from(keys)), + ], + ) + .unwrap(); + + let plan = take_plan(&fixture.dataset, rows_input(vec![input]), &["i"]).unwrap(); + let expected_schema = plan.schema(); + assert_eq!(expected_schema.metadata(), &dataset_metadata); + let batches = run(&plan).await; + + assert!(!batches.is_empty()); + assert!( + batches + .iter() + .all(|batch| batch.schema() == expected_schema), + "row-stream output did not match the plan schema for reordered={reordered}" + ); + } + + /// A sparse plan constructs fragment handles only for the fragments + /// it selects, keeping their candidate-list position as priority — + /// no metadata is loaded or retained for unselected fragments + #[tokio::test] + async fn sparse_plan_scopes_only_selected_fragments() { + let fixture = take_fixture(false).await; + let dataset = &fixture.dataset; + let descriptors = dataset.fragments().clone(); + assert_eq!(descriptors.len(), 3); + + let mut rows = BTreeMap::new(); + rows.insert(2u32, vec![0u64..5]); + let plan = FilteredReadInternalPlan { + rows, + filters: HashMap::new(), + scan_range_after_filter: None, + }; + let options = FilteredReadOptions::basic_full_read(dataset); + let scheduler = FilteredReadStream::make_scan_scheduler(dataset, &options); + + let scoped = FilteredReadStream::plan_to_scoped_fragments( + &plan, + &descriptors, + dataset, + &options, + scheduler, + ); + assert_eq!(scoped.len(), 1); + assert_eq!(scoped[0].fragment.id(), 2); + assert_eq!(scoped[0].priority, 2); + } + + /// Output preserves the input's row order, duplicates, and payload + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_preserves_order_dups_and_payload( + #[case] stable_row_ids: bool, + #[case] key: &str, + ) { + let fixture = take_fixture(stable_row_ids).await; + + // Stable row ids are assigned sequentially on write, so the id of + // row `i` is `i`; without them id == address + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![21, 3, 15, 21, 0] + } else { + vec![ + addr(2, 1), // i = 21 + addr(0, 3), // i = 3 + addr(1, 5), // i = 15 + addr(2, 1), // i = 21 (duplicate) + addr(0, 0), // i = 0 + ] + }; + let expected_i: Vec = vec![21, 3, 15, 21, 0]; + + let input_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("payload", DataType::Float32, false), + ArrowField::new(key, DataType::UInt64, true), + ])); + let payload: Vec = (0..keys.len()).map(|v| v as f32 * 0.5).collect(); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![ + Arc::new(Float32Array::from(payload.clone())), + Arc::new(UInt64Array::from(keys.clone())), + ], + ) + .unwrap(); + let batches = vec![batch.slice(0, 3), batch.slice(3, 2)]; + + let plan = take_plan(&fixture.dataset, rows_input(batches), &["s", "i"]).unwrap(); + assert!(plan.row_stream_input().is_some()); + // Input columns, then new fields; the unrequested key is stripped + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["payload", "i", "s"] + ); + + let result = run(&plan).await; + assert_eq!(result.len(), 1); + assert_eq!(result[0].num_rows(), 5); + let result = concat_batches(&plan.schema(), &result).unwrap(); + + let i_col = result.column_by_name("i").unwrap(); + assert_eq!( + i_col.as_primitive::().values(), + &expected_i[..] + ); + let s_col = result.column_by_name("s").unwrap().as_string::(); + for (row, i) in expected_i.iter().enumerate() { + assert_eq!(s_col.value(row), format!("s-{i}")); + } + let payload_col = result + .column_by_name("payload") + .unwrap() + .as_primitive::(); + assert_eq!(payload_col.values(), &payload[..]); + } + + #[tokio::test] + async fn blob_take_reserves_one_complete_duplicate_expanded_output() { + let tmp_dir = TempStrDir::default(); + let first_payload = vec![0x11; 1024]; + let second_payload = vec![0x22; 1024]; + let mut blobs = BlobArrayBuilder::new(2); + blobs.push_bytes(&first_payload).unwrap(); + blobs.push_bytes(&second_payload).unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![blob_field("blob", false)])); + let batch = + RecordBatch::try_new(schema.clone(), vec![blobs.finish().unwrap()]).unwrap(); + let dataset = Arc::new( + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + tmp_dir.as_str(), + Some(WriteParams { + data_storage_version: Some(lance_file::version::LanceFileVersion::V2_2), + max_rows_per_file: 1, + ..Default::default() + }), + ) + .await + .unwrap(), + ); + + let first = 0_u64; + let second = 1_u64 << 32; + let mut keys = vec![first; 100]; + keys.push(second); + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + false, + )])); + let input = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + let projection = dataset + .empty_projection() + .union_columns(["blob"], OnMissing::Error) + .unwrap() + .with_blob_handling(BlobHandling::AllBinary); + let plan = FilteredReadExec::try_new( + dataset, + FilteredReadOptions::new(projection) + .with_batch_size(101) + .with_materialization_readahead_bytes(512), + Some(rows_input(vec![input])), + ) + .unwrap(); + + let batches = tokio::time::timeout(std::time::Duration::from_secs(10), async { + run(&plan).await + }) + .await + .expect("row-stream blob materialization must make progress"); + let output = concat_batches(&plan.schema(), &batches).unwrap(); + assert!( + plan.materialization_context.peak_reserved_bytes() + >= (101 * first_payload.len()) as u64 + ); + let blobs = output + .column_by_name("blob") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(blobs.len(), 101); + assert!( + blobs + .iter() + .take(100) + .all(|value| value == Some(first_payload.as_slice())) + ); + assert_eq!(blobs.value(100), second_payload.as_slice()); + } + + /// Tiny input batches merge up to the target and oversized ones pass + /// through whole, preserving order across the boundaries + #[tokio::test] + async fn take_coalesces_input_to_batch_size() { + let fixture = take_fixture(false).await; + + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys: Vec = vec![ + addr(2, 3), // i = 23 + addr(0, 1), // i = 1 + addr(1, 4), // i = 14 + addr(0, 7), // i = 7 + addr(2, 0), // i = 20 + addr(1, 1), // i = 11 + addr(0, 2), // i = 2 + ]; + let expected_i: Vec = vec![23, 1, 14, 7, 20, 11, 2]; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new( + input_schema.clone(), + vec![Arc::new(UInt64Array::from(keys.clone()))], + ) + .unwrap(); + + let assert_batches = + |result: Vec, schema: SchemaRef, expected_sizes: Vec| { + assert_eq!( + result.iter().map(|b| b.num_rows()).collect::>(), + expected_sizes + ); + let merged = concat_batches(&schema, &result).unwrap(); + let i_col = merged.column_by_name("i").unwrap(); + assert_eq!( + i_col.as_primitive::().values(), + &expected_i[..] + ); + }; + + // Seven one-row batches merge whenever the buffer reaches 3 rows + let tiny = (0..7).map(|i| batch.slice(i, 1)).collect::>(); + let plan = take_plan_sized(&fixture.dataset, rows_input(tiny), &["i"], 3).unwrap(); + assert_batches(run(&plan).await, plan.schema(), vec![3, 3, 1]); + + // One oversized batch passes through whole — never split + let plan = + take_plan_sized(&fixture.dataset, rows_input(vec![batch.clone()]), &["i"], 3) + .unwrap(); + assert_batches(run(&plan).await, plan.schema(), vec![7]); + + // A large batch flushes the partial buffer and passes through + let mixed = vec![batch.slice(0, 2), batch.slice(2, 5)]; + let plan = take_plan_sized(&fixture.dataset, rows_input(mixed), &["i"], 3).unwrap(); + assert_batches(run(&plan).await, plan.schema(), vec![2, 5]); + } + + /// Storage-ordered input exercises the aligned fast path + #[rstest] + #[case::by_row_addr(ROW_ADDR)] + #[case::by_row_id(ROW_ID)] + #[tokio::test] + async fn take_aligned_input_fast_path(#[case] key: &str) { + let fixture = take_fixture(false).await; + + let addr = |frag: u64, off: u64| (frag << 32) | off; + let keys: Vec = vec![ + addr(0, 0), // i = 0 + addr(0, 3), // i = 3 + addr(1, 5), // i = 15 + addr(2, 1), // i = 21 + addr(2, 9), // i = 29 + ]; + let expected_i: Vec = vec![0, 3, 15, 21, 29]; + + let input_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("payload", DataType::Float32, false), + ArrowField::new(key, DataType::UInt64, true), + ])); + let payload: Vec = (0..keys.len()).map(|v| v as f32 * 0.5).collect(); + let batch = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(Float32Array::from(payload.clone())), + Arc::new(UInt64Array::from(keys)), + ], + ) + .unwrap(); + + let plan = take_plan(&fixture.dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 5); + let i_col = result.column_by_name("i").unwrap(); + assert_eq!( + i_col.as_primitive::().values(), + &expected_i[..] + ); + let payload_col = result + .column_by_name("payload") + .unwrap() + .as_primitive::(); + assert_eq!(payload_col.values(), &payload[..]); + } + + /// A fragment-scoped take reads from the scoped fragments only; keys + /// pointing outside the scope drop like stale rows + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_scoped_to_fragments(#[case] stable_row_ids: bool, #[case] key: &str) { + let fixture = take_fixture(stable_row_ids).await; + let subset = Arc::new(vec![fixture.dataset.fragments()[1].clone()]); + + let addr = |frag: u64, off: u64| (frag << 32) | off; + // i = 12 inside the scoped fragment, i = 3 outside the scope + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![12, 3] + } else { + vec![addr(1, 2), addr(0, 3)] + }; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + key, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection).with_fragments(subset), + Some(rows_input(vec![batch])), + ) + .unwrap(); + + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 1); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.value(0), 12); + } + + /// A batch whose keys span the whole id range but hit only two rows: + /// the span prefilter must not misread coverage as membership + #[tokio::test] + async fn take_stable_ids_wide_key_span() { + let fixture = take_fixture(true).await; + // Last row of the last fragment, first row of the first: every + // fragment's span overlaps, only two rows match + let keys: Vec = vec![29, 0]; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&fixture.dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &[29, 0]); + } + + /// Identity flags: requested-but-missing columns are synthesized, + /// carried ones kept, unrequested carried ones stripped + #[rstest] + #[case::unstable(false)] + #[case::stable(true)] + #[tokio::test] + async fn take_identity_flags(#[case] stable_row_ids: bool) { + let fixture = take_fixture(stable_row_ids).await; + let addr = |frag: u64, off: u64| (frag << 32) | off; + + let ids: Vec = if stable_row_ids { + vec![21, 3] + } else { + vec![addr(2, 1), addr(0, 3)] + }; + let expected_addrs: Vec = vec![addr(2, 1), addr(0, 3)]; + let expected_i: Vec = vec![21, 3]; + let id_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let id_batch = + RecordBatch::try_new(id_schema, vec![Arc::new(UInt64Array::from(ids.clone()))]) + .unwrap(); + + // Keep the carried _rowid and synthesize _rowaddr + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap() + .with_row_id() + .with_row_addr(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![id_batch.clone()])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec![ROW_ID, "i", ROW_ADDR] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let id_col = result + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::(); + assert_eq!(id_col.values(), &ids[..]); + let addr_col = result + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + assert_eq!(addr_col.values(), &expected_addrs[..]); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &expected_i[..]); + + // Synthesize _rowaddr but strip the unrequested carried _rowid + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap() + .with_row_addr(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![id_batch.clone()])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["i", ROW_ADDR] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let addr_col = result + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + assert_eq!(addr_col.values(), &expected_addrs[..]); + + // Address-keyed input, synthesize _rowid (the reverse direction) + let addr_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let addr_batch = RecordBatch::try_new( + addr_schema, + vec![Arc::new(UInt64Array::from(expected_addrs.clone()))], + ) + .unwrap(); + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap() + .with_row_id(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![addr_batch])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec!["i", ROW_ID] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let id_col = result + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::(); + assert_eq!(id_col.values(), &ids[..]); + + // Fetch nothing, synthesize only (the AddRowAddrExec shape) + let projection = fixture + .dataset + .empty_projection() + .with_row_id() + .with_row_addr(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![id_batch])), + ) + .unwrap(); + assert_eq!( + plan.schema() + .fields() + .iter() + .map(|f| f.name().clone()) + .collect::>(), + vec![ROW_ID, ROW_ADDR] + ); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let addr_col = result + .column_by_name(ROW_ADDR) + .unwrap() + .as_primitive::(); + assert_eq!(addr_col.values(), &expected_addrs[..]); + } + + /// New sub-fields merge into an existing struct column + #[tokio::test] + async fn take_merges_nested_struct() { + let fixture = take_fixture(false).await; + + let data = fixture + .dataset + .scan() + .project(&["struct"]) + .unwrap() + .with_row_id() + .try_into_batch() + .await + .unwrap(); + // Rebuild the input with only struct.y so struct.x must be taken + let full_struct = data.column_by_name("struct").unwrap().as_struct(); + let y_only = arrow_array::StructArray::new( + Fields::from(vec![Arc::new(ArrowField::new("y", DataType::Int32, false))]), + vec![full_struct.column_by_name("y").unwrap().clone()], + None, + ); + let input_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("struct", y_only.data_type().clone(), false), + ArrowField::new(ROW_ID, DataType::UInt64, true), + ])); + let data = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(y_only), + data.column_by_name(ROW_ID).unwrap().clone(), + ], + ) + .unwrap(); + + let projection = fixture + .dataset + .empty_projection() + .union_column("struct.x", OnMissing::Error) + .unwrap(); + let plan = FilteredReadExec::try_new( + fixture.dataset.clone(), + FilteredReadOptions::new(projection), + Some(rows_input(vec![data])), + ) + .unwrap(); + + let expected_struct_type = DataType::Struct(Fields::from(vec![ + Arc::new(ArrowField::new("x", DataType::Int32, false)), + Arc::new(ArrowField::new("y", DataType::Int32, false)), + ])); + assert_eq!( + plan.schema().field_with_name("struct").unwrap().data_type(), + &expected_struct_type + ); + + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 30); + let struct_col = result.column_by_name("struct").unwrap().as_struct(); + assert_eq!( + struct_col.column_by_name("x").unwrap(), + struct_col.column_by_name("y").unwrap() + ); + } + + /// Input rows whose key no longer exists (deleted rows) are dropped + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_drops_stale_keys(#[case] stable_row_ids: bool, #[case] key: &str) { + let fixture = take_fixture(stable_row_ids).await; + let mut dataset = fixture.dataset.as_ref().clone(); + dataset.delete("i = 15").await.unwrap(); + let dataset = Arc::new(dataset); + + let addr = |frag: u64, off: u64| (frag << 32) | off; + // The pre-delete identifiers of rows 15 (now deleted) and 16 + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![15, 16] + } else { + vec![addr(1, 5), addr(1, 6)] + }; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + key, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 1); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.value(0), 16); + } + + /// Keys of a fully deleted fragment (gone from the manifest) are + /// dropped like stale rows + #[rstest] + #[case::by_row_addr(false, ROW_ADDR)] + #[case::by_row_id(false, ROW_ID)] + #[case::stable_by_row_addr(true, ROW_ADDR)] + #[case::stable_by_row_id(true, ROW_ID)] + #[tokio::test] + async fn take_drops_keys_of_deleted_fragment( + #[case] stable_row_ids: bool, + #[case] key: &str, + ) { + let fixture = take_fixture(stable_row_ids).await; + let mut dataset = fixture.dataset.as_ref().clone(); + dataset.delete("i >= 10 and i < 20").await.unwrap(); + let dataset = Arc::new(dataset); + + let addr = |frag: u64, off: u64| (frag << 32) | off; + // The pre-delete identifiers of row 15 (fragment 1, now gone + // from the manifest) and row 20 (fragment 2, still live) + let keys: Vec = if key == ROW_ID && stable_row_ids { + vec![15, 20] + } else { + vec![addr(1, 5), addr(2, 0)] + }; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + key, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + assert_eq!(result.num_rows(), 1); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.value(0), 20); + } + + /// After delete + compaction the stable row-id sequences are no + /// longer simple contiguous ranges; ids must still resolve to the + /// moved rows and deleted ids must still drop + #[tokio::test] + async fn take_stable_ids_after_compaction() { + use crate::dataset::optimize::{CompactionOptions, compact_files}; + + let fixture = take_fixture(true).await; + let mut dataset = fixture.dataset.as_ref().clone(); + // Punch holes, then rewrite all fragments into one + dataset.delete("i % 3 = 0").await.unwrap(); + compact_files(&mut dataset, CompactionOptions::default(), None) + .await + .unwrap(); + let dataset = Arc::new(dataset); + assert_eq!(dataset.get_fragments().len(), 1); + + // Survivors in scattered order, plus a compacted-away id (15) + let keys: Vec = vec![25, 1, 15, 14]; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(keys))]) + .unwrap(); + + let plan = take_plan(&dataset, rows_input(vec![batch]), &["i"]).unwrap(); + let result = concat_batches(&plan.schema(), &run(&plan).await).unwrap(); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &[25, 1, 14]); + } + + /// with_deleted_rows is rejected for a row-stream read + #[rstest] + #[case::unstable(false)] + #[case::stable(true)] + #[tokio::test] + async fn take_rejects_with_deleted_rows(#[case] stable_row_ids: bool) { + let fixture = take_fixture(stable_row_ids).await; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ADDR, + DataType::UInt64, + true, + )])); + let batch = + RecordBatch::try_new(input_schema, vec![Arc::new(UInt64Array::from(vec![0_u64]))]) + .unwrap(); + let projection = fixture + .dataset + .empty_projection() + .union_columns(["i"], OnMissing::Error) + .unwrap(); + let err = FilteredReadExec::try_new( + fixture.dataset, + FilteredReadOptions::new(projection) + .with_deleted_rows() + .unwrap(), + Some(rows_input(vec![batch])), + ) + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err}"); + assert!(err.to_string().contains("with_deleted_rows")); + } + + /// Construction errors: no key column, nothing to read + #[tokio::test] + async fn take_construction_errors() { + let fixture = take_fixture(false).await; + let no_key_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "payload", + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new( + no_key_schema, + vec![Arc::new(UInt64Array::from(vec![0_u64]))], + ) + .unwrap(); + let err = take_plan(&fixture.dataset, rows_input(vec![batch]), &["s"]).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err}"); + assert!(err.to_string().contains("must have a column")); + + // Taking fields the input already has: nothing to read + let with_s_schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new(ROW_ADDR, DataType::UInt64, true), + ArrowField::new("s", DataType::Utf8, false), + ])); + let with_s_batch = RecordBatch::try_new( + with_s_schema, + vec![ + Arc::new(UInt64Array::from(vec![0_u64])), + Arc::new(StringArray::from(vec!["x"])), + ], + ) + .unwrap(); + let err = + take_plan(&fixture.dataset, rows_input(vec![with_s_batch]), &["s"]).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err}"); + assert!(err.to_string().contains("nothing to read")); + } + + /// with_new_children re-derives the row-stream source and preserves the schema + #[tokio::test] + async fn take_with_new_children() { + let fixture = take_fixture(false).await; + let input_schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(UInt64Array::from(vec![0_u64, 1]))], + ) + .unwrap(); + let input = rows_input(vec![batch]); + let plan: Arc = + Arc::new(take_plan(&fixture.dataset, input.clone(), &["s"]).unwrap()); + let rebuilt = plan.clone().with_new_children(vec![input]).unwrap(); + assert_eq!(plan.schema(), rebuilt.schema()); + assert!( + rebuilt + .downcast_ref::() + .unwrap() + .row_stream_input() + .is_some() + ); + } + + /// Take-mode nodes can be created and executed without an active + /// tokio runtime (required for DataFusion foreign table providers) + #[test] + fn no_context_take_rows() { + use lance_datafusion::datagen::DatafusionDatagenExt; + use lance_datagen::{BatchCount, RowCount}; + + let fixture = NoContextTestFixture::new(); + let dataset = Arc::new(fixture.dataset); + let input = lance_datagen::gen_batch() + .col(ROW_ID, lance_datagen::array::step::()) + .into_df_exec(RowCount::from(50), BatchCount::from(2)); + let plan = take_plan(&dataset, input, &["text"]).unwrap(); + plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + } + } } diff --git a/rust/lance/src/io/exec/filtered_read_proto.rs b/rust/lance/src/io/exec/filtered_read_proto.rs index 4eb329506aa..d21029c784d 100644 --- a/rust/lance/src/io/exec/filtered_read_proto.rs +++ b/rust/lance/src/io/exec/filtered_read_proto.rs @@ -145,6 +145,11 @@ fn fr_options_to_proto( threading_mode: Some(threading_mode_to_proto(&options.threading_mode)), io_buffer_size_bytes: options.io_buffer_size_bytes, filter_schema_ipc, + materialization_readahead_bytes: options.materialization_readahead_bytes, + batch_size_bytes: options + .file_reader_options + .as_ref() + .and_then(|o| o.batch_size_bytes), }) } @@ -194,6 +199,17 @@ async fn fr_options_from_proto( if let Some(io_buffer) = proto.io_buffer_size_bytes { options = options.with_io_buffer_size(io_buffer); } + if let Some(materialization_readahead_bytes) = proto.materialization_readahead_bytes { + options = options.with_materialization_readahead_bytes(materialization_readahead_bytes); + } + if let Some(batch_size_bytes) = proto.batch_size_bytes { + // Merge the scanner-level byte budget into the dataset's existing + // file-reader options so that distributed execution preserves + // validation and I/O settings such as read_chunk_size. + let mut file_reader_options = dataset.file_reader_options.clone().unwrap_or_default(); + file_reader_options.batch_size_bytes = Some(batch_size_bytes); + options = options.with_file_reader_options(file_reader_options); + } if let Some(mode) = proto.threading_mode { options.threading_mode = threading_mode_from_proto(&mode)?; } @@ -494,6 +510,8 @@ mod tests { use std::collections::HashSet; use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; + use lance_encoding::decoder::DecoderConfig; + use lance_file::reader::FileReaderOptions; #[test] fn test_range_roundtrip() { @@ -588,9 +606,27 @@ mod tests { Arc::new(dataset) } + /// Create a test dataset with non-default file-reader options so that + /// round-trip tests can verify that scanner-level overrides preserve the + /// dataset-level defaults. + async fn make_test_dataset_with_file_reader_options() -> Arc { + let mut dataset = make_test_dataset().await; + if let Some(ds) = Arc::get_mut(&mut dataset) { + ds.file_reader_options = Some(FileReaderOptions { + read_chunk_size: 1234, + decoder_config: DecoderConfig { + validate_on_decode: true, + ..Default::default() + }, + batch_size_bytes: None, + }); + } + dataset + } + #[tokio::test] async fn test_options_roundtrip_basic() { - let dataset = make_test_dataset().await; + let dataset = make_test_dataset_with_file_reader_options().await; let ctx = SessionContext::new(); let state = ctx.state(); let filter_schema = Arc::new(prune_schema_for_substrait(&dataset.schema().into())); @@ -599,8 +635,13 @@ mod tests { .with_scan_range_before_filter(10..90) .unwrap() .with_batch_size(64) + .with_file_reader_options(FileReaderOptions { + batch_size_bytes: Some(4096), + ..Default::default() + }) .with_fragment_readahead(4) - .with_io_buffer_size(1024 * 1024); + .with_io_buffer_size(1024 * 1024) + .with_materialization_readahead_bytes(8 * 1024 * 1024); let proto = fr_options_to_proto(&options, &filter_schema, &state).unwrap(); let back = fr_options_from_proto(proto, &dataset, &state) @@ -614,6 +655,24 @@ mod tests { assert_eq!(options.batch_size, back.batch_size); assert_eq!(options.fragment_readahead, back.fragment_readahead); assert_eq!(options.io_buffer_size_bytes, back.io_buffer_size_bytes); + assert_eq!( + options.materialization_readahead_bytes, + back.materialization_readahead_bytes + ); + assert_eq!( + options + .file_reader_options + .as_ref() + .and_then(|o| o.batch_size_bytes), + back.file_reader_options + .as_ref() + .and_then(|o| o.batch_size_bytes) + ); + // The scanner-level byte budget must be merged into the dataset's + // existing file-reader options, not replace them. + let effective = back.file_reader_options.as_ref().unwrap(); + assert_eq!(effective.read_chunk_size, 1234); + assert!(effective.decoder_config.validate_on_decode); assert_eq!(options.threading_mode, back.threading_mode); assert_eq!(options.with_deleted_rows, back.with_deleted_rows); assert_eq!(options.projection.field_ids, back.projection.field_ids); @@ -727,6 +786,59 @@ mod tests { ); } + /// A row-stream (take) exec serializes like any other: the input plan + /// travels as the node's child through the plan codec, and decoding + /// re-derives the row-stream selector from the child's schema + #[tokio::test] + async fn test_exec_to_proto_roundtrip_row_stream() { + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use lance_core::{ROW_ID, ROW_ID_FIELD}; + use lance_datafusion::exec::OneShotExec; + + fn keys_input() -> Arc { + let schema = Arc::new(ArrowSchema::new(vec![ROW_ID_FIELD.clone()])); + let batch = arrow_array::RecordBatch::try_new( + schema.clone(), + vec![Arc::new(arrow_array::UInt64Array::from(vec![3u64, 1, 4]))], + ) + .unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]); + Arc::new(OneShotExec::new(Box::pin(RecordBatchStreamAdapter::new( + schema, stream, + )))) + } + + let dataset = make_test_dataset().await; + let ctx = SessionContext::new(); + let state = ctx.state(); + + let options = FilteredReadOptions::basic_full_read(&dataset); + let exec = FilteredReadExec::try_new(dataset.clone(), options, Some(keys_input())).unwrap(); + assert!(exec.row_stream_input().is_some()); + + let proto = filtered_read_exec_to_proto(&exec, &state).await.unwrap(); + + // The codec hands the decoded child back; the selector re-derives + // from its schema + let back = + filtered_read_exec_from_proto(proto, Some(dataset.clone()), Some(keys_input()), &state) + .await + .unwrap(); + assert!(back.row_stream_input().is_some()); + assert_eq!(exec.schema(), back.schema()); + assert_eq!( + exec.options().projection.field_ids, + back.options().projection.field_ids + ); + assert!( + back.row_stream_input() + .unwrap() + .schema() + .column_with_name(ROW_ID) + .is_some() + ); + } + #[tokio::test] async fn test_plan_proto_roundtrip() { let dataset = make_test_dataset().await; diff --git a/rust/lance/src/io/exec/fts.rs b/rust/lance/src/io/exec/fts.rs index add864c0ea9..3483a476d52 100644 --- a/rust/lance/src/io/exec/fts.rs +++ b/rust/lance/src/io/exec/fts.rs @@ -1,61 +1,260 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::collections::HashMap; -use std::sync::Arc; +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, OnceLock}; -use arrow::array::{AsArray, BooleanBuilder}; +use arrow::array::{AsArray, BooleanBuilder, ListBuilder, UInt32Builder}; use arrow::datatypes::{Float32Type, UInt64Type}; use arrow_array::{Array, BooleanArray, Float32Array, OffsetSizeTrait, RecordBatch, UInt64Array}; -use arrow_schema::{DataType, SchemaRef}; +use arrow_schema::{DataType, Field, SchemaRef}; use datafusion::common::{NullEquality, Statistics}; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::execution::SendableRecordBatchStream; use datafusion::physical_plan::empty::EmptyExec; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, Gauge, MetricsSet}; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; use datafusion_physical_expr::expressions::Column; -use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning}; +use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning, PhysicalExpr}; +use datafusion_physical_plan::ExecutionPlanProperties; use datafusion_physical_plan::joins::{HashJoinExec, PartitionMode}; -use datafusion_physical_plan::metrics::{BaselineMetrics, Count}; +use datafusion_physical_plan::metrics::{BaselineMetrics, Count, Time}; use futures::future::try_join_all; -use futures::stream::{self}; +use futures::stream::{self, FuturesUnordered}; use futures::{FutureExt, StreamExt, TryStreamExt}; use itertools::Itertools; use lance_core::{ Error, ROW_ID, Result, - utils::{tokio::get_num_compute_intensive_cpus, tracing::StreamTracingExt}, + utils::{ + tokio::{get_num_compute_intensive_cpus, spawn_cpu}, + tracing::StreamTracingExt, + }, }; use lance_datafusion::utils::{ExecutionPlanMetricsSetExt, MetricsExt, PARTITIONS_SEARCHED_METRIC}; +use lance_select::RowAddrMask; use lance_table::format::IndexMetadata; +use rustc_hash::FxHashSet; use super::PreFilterSource; -use super::utils::{IndexMetrics, build_prefilter}; -use crate::index::scalar::inverted::{load_segment_details, load_segments}; -use crate::{Dataset, index::DatasetIndexInternalExt}; -use lance_index::metrics::{ - AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, AND_CANDIDATES_SEEN_METRIC, AND_FULL_SCORES_METRIC, - FREQS_COLLECTED_METRIC, MetricsCollector, +use super::utils::{IndexMetrics, PreFilterMasks, build_prefilter}; +use crate::dataset::mem_wal::index::{QueryLocalFtsIndex, QueryLocalFtsStats}; +use crate::index::scalar::inverted::{ + ResolvedFtsField, fts_document_schema, load_segment_details, load_segments, + transform_fts_document_stream, }; +use crate::{Dataset, index::DatasetIndexInternalExt}; +use lance_index::metrics::MetricsCollector; use lance_index::scalar::inverted::builder::ScoredDoc; use lance_index::scalar::inverted::builder::document_input; use lance_index::scalar::inverted::document_tokenizer::{DocType, JsonTokenizer, LanceTokenizer}; use lance_index::scalar::inverted::query::{ - BoostQuery, FtsSearchParams, MatchQuery, PhraseQuery, Tokens, collect_query_tokens, - has_query_token, + BoostQuery, FtsQuery, FtsQueryNode, FtsSearchParams, MatchQuery, Operator, PhraseQuery, Tokens, + collect_query_tokens, has_query_token, uses_fuzzy_expansion, }; use lance_index::scalar::inverted::tokenizer::document_tokenizer::TextTokenizer; use lance_index::scalar::inverted::{ - FTS_SCHEMA, InvertedIndex, MemBM25Scorer, SCORE_COL, build_global_bm25_scorer, - flat_bm25_search_stream_with_metrics, + DOC_INDEX_COL, DocumentGranularity, FTS_SCHEMA, FlatBm25SearchOptions, InvertedIndex, + MemBM25Scorer, PreparedBm25Query, SCORE_COL, Scorer, build_global_bm25_scorer, compound_search, + compound_search_prepared_match, compound_search_prepared_match_with_score_floor, + compound_search_with_base_scorer, cross_column_compound_search, exclusive_scaled_score_floor, + flat_bm25_search_stream_with_options_and_scorer, fts_schema, materialized_compound_top_k, + prepare_bm25_query, }; use lance_index::{prefilter::PreFilter, scalar::inverted::query::BooleanQuery}; use lance_tokenizer::{SimpleTokenizer, TextAnalyzer}; use tracing::instrument; +use uuid::Uuid; + +/// Maximum number of additional kth-score rows retained before exact replay. +/// One extra probe slot is reserved for the strict lower-score guard. +const WAND_TIE_COMPLETION_BUDGET: usize = 128; + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenWithPosition { + text: String, + position: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenizedQuery(Vec); + +impl TokenizedQuery { + fn from_tokens(tokens: &Tokens) -> Self { + let mut token_positions = Vec::with_capacity(tokens.len()); + for index in 0..tokens.len() { + token_positions.push(TokenWithPosition { + text: tokens.get_token(index).to_string(), + position: tokens.position(index), + }); + } + Self(token_positions) + } +} + +impl std::fmt::Display for TokenizedQuery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (index, token) in self.0.iter().enumerate() { + if index > 0 { + write!(f, ", ")?; + } + write!(f, "({:?}, {})", token.text, token.position)?; + } + write!(f, "]") + } +} + +fn record_tokenized_query(snapshot: &OnceLock, tokens: &Tokens) { + snapshot.get_or_init(|| TokenizedQuery::from_tokens(tokens)); +} + +fn fmt_tokenized_query( + snapshot: &OnceLock, + separator: &str, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if let Some(tokens) = snapshot.get() { + write!(f, "{separator}tokenized_query={tokens}")?; + } + Ok(()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TokenizedLeafKind { + Match, + Phrase, +} + +impl std::fmt::Display for TokenizedLeafKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Match => write!(f, "Match"), + Self::Phrase => write!(f, "Phrase"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenizedQueryLeaf { + kind: TokenizedLeafKind, + column: Option, + tokens: TokenizedQuery, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TokenizedCompoundQuery(Vec); + +impl std::fmt::Display for TokenizedCompoundQuery { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "[")?; + for (index, leaf) in self.0.iter().enumerate() { + if index > 0 { + write!(f, ", ")?; + } + write!( + f, + "{}(column={:?}, tokens={})", + leaf.kind, + leaf.column.as_deref().unwrap_or_default(), + leaf.tokens + )?; + } + write!(f, "]") + } +} + +fn fmt_tokenized_compound_query( + snapshot: &OnceLock, + separator: &str, + f: &mut std::fmt::Formatter<'_>, +) -> std::fmt::Result { + if let Some(tokens) = snapshot.get() { + write!(f, "{separator}tokenized_query={tokens}")?; + } + Ok(()) +} + +/// Expands a schema-derived nested FTS source into one canonical row per +/// logical document before flat search or index building consumes it. +#[derive(Debug)] +pub struct FtsDocumentExec { + input: Arc, + resolved: ResolvedFtsField, + properties: Arc, +} + +impl FtsDocumentExec { + pub(crate) fn new(input: Arc, resolved: ResolvedFtsField) -> Self { + let schema = fts_document_schema(resolved.coordinate_rank()); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema), + input.output_partitioning().clone(), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Self { + input, + resolved, + properties, + } + } +} + +impl DisplayAs for FtsDocumentExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "FtsDocument: column={}, granularity={:?}", + self.resolved.canonical_path, self.resolved.document_granularity + ) + } +} + +impl ExecutionPlan for FtsDocumentExec { + fn name(&self) -> &str { + "FtsDocumentExec" + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal( + "FtsDocumentExec expects one child".to_string(), + )); + } + Ok(Arc::new(Self::new( + children.pop().unwrap(), + self.resolved.clone(), + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + transform_fts_document_stream( + self.input.execute(partition, context)?, + self.resolved.clone(), + ) + .map_err(DataFusionError::from) + } + + fn properties(&self) -> &Arc { + &self.properties + } +} /// Open one FTS segment as an [`InvertedIndex`]. async fn open_fts_segment( @@ -97,15 +296,80 @@ async fn open_fts_segments( .await } +async fn search_prepared_segments( + indices: &[Arc], + prepared: Arc, + pre_filter: Arc, + metrics: Arc, + initial_score_floor: Option, +) -> Result> { + let limit = prepared.params.limit.unwrap_or(usize::MAX); + let mut candidates = std::collections::BinaryHeap::new(); + let searches = indices + .iter() + .map(|index| { + let index = Arc::clone(index); + let prepared = prepared.clone(); + let pre_filter = pre_filter.clone(); + let metrics = metrics.clone(); + async move { + if let Some(initial_score_floor) = initial_score_floor { + index + .bm25_search_prepared_documents_with_score_floor( + prepared.query.clone(), + prepared.params.clone(), + prepared.operator, + pre_filter, + metrics, + initial_score_floor, + ) + .await + } else { + index + .bm25_search_prepared_documents( + prepared.query.clone(), + prepared.params.clone(), + prepared.operator, + pre_filter, + metrics, + ) + .await + } + } + }) + .collect::>(); + let searches = stream::iter(searches).buffer_unordered(get_num_compute_intensive_cpus()); + let mut searches = searches; + + while let Some(documents) = searches.try_next().await? { + for document in documents { + if candidates.len() < limit { + candidates.push(std::cmp::Reverse(document)); + } else if candidates.peek().unwrap().0.score < document.score { + candidates.pop(); + candidates.push(std::cmp::Reverse(document)); + } + } + } + + Ok(candidates + .into_sorted_vec() + .into_iter() + .map(|std::cmp::Reverse(document)| document) + .collect()) +} + +#[allow(clippy::too_many_arguments)] async fn search_segments( indices: &[Arc], tokens: Arc, params: Arc, - operator: lance_index::scalar::inverted::query::Operator, + operator: Operator, pre_filter: Arc, metrics: Arc, base_scorer: Arc, -) -> Result<(Vec, Vec)> { + initial_score_floor: Option, +) -> Result> { let limit = params.limit.unwrap_or(usize::MAX); let mut candidates = std::collections::BinaryHeap::new(); let searches = indices @@ -118,29 +382,43 @@ async fn search_segments( let metrics = metrics.clone(); let base_scorer = base_scorer.clone(); async move { - index - .bm25_search( - tokens, - params, - operator, - pre_filter, - metrics, - Some(base_scorer.as_ref()), - ) - .await + if let Some(initial_score_floor) = initial_score_floor { + index + .bm25_search_documents_with_score_floor( + tokens, + params, + operator, + pre_filter, + metrics, + Some(base_scorer.as_ref()), + initial_score_floor, + ) + .await + } else { + index + .bm25_search_documents( + tokens, + params, + operator, + pre_filter, + metrics, + Some(base_scorer.as_ref()), + ) + .await + } } }) .collect::>(); let searches = stream::iter(searches).buffer_unordered(get_num_compute_intensive_cpus()); let mut searches = searches; - while let Some((doc_ids, scores)) = searches.try_next().await? { - for (row_id, score) in doc_ids.into_iter().zip(scores.into_iter()) { + while let Some(documents) = searches.try_next().await? { + for document in documents { if candidates.len() < limit { - candidates.push(std::cmp::Reverse(ScoredDoc::new(row_id, score))); - } else if candidates.peek().unwrap().0.score.0 < score { + candidates.push(std::cmp::Reverse(document)); + } else if candidates.peek().unwrap().0.score < document.score { candidates.pop(); - candidates.push(std::cmp::Reverse(ScoredDoc::new(row_id, score))); + candidates.push(std::cmp::Reverse(document)); } } } @@ -148,134 +426,2196 @@ async fn search_segments( Ok(candidates .into_sorted_vec() .into_iter() - .map(|std::cmp::Reverse(doc)| (doc.row_id, doc.score.0)) - .unzip()) + .map(|std::cmp::Reverse(document)| document) + .collect()) } -/// Fall back to the default simple tokenizer when no on-disk FTS segment exists. -fn default_text_tokenizer() -> Box { - Box::new(TextTokenizer::new( - TextAnalyzer::builder(SimpleTokenizer::default()).build(), - )) +#[derive(Clone)] +struct PreparedMatch { + query: Arc, + params: Arc, + operator: Operator, } -pub struct FtsIndexMetrics { - index_metrics: IndexMetrics, - partitions_searched: Count, - and_candidates_seen: Count, - and_candidates_pruned_before_return: Count, - and_full_scores: Count, - freqs_collected: Count, - baseline_metrics: BaselineMetrics, +impl PreparedMatch { + async fn new( + indices: &[Arc], + tokens: Tokens, + params: FtsSearchParams, + operator: Operator, + metrics: &FtsIndexMetrics, + base_scorer: Option>, + ) -> Result { + let query = Arc::new( + prepare_bm25_query(indices, tokens, ¶ms, Some(metrics), base_scorer).await?, + ); + Ok(Self { + query, + params: Arc::new(params), + operator, + }) + } } -impl FtsIndexMetrics { - pub fn new(metrics: &ExecutionPlanMetricsSet, partition: usize) -> Self { - Self { - index_metrics: IndexMetrics::new(metrics, partition), - partitions_searched: metrics.new_count(PARTITIONS_SEARCHED_METRIC, partition), - and_candidates_seen: metrics.new_count(AND_CANDIDATES_SEEN_METRIC, partition), - and_candidates_pruned_before_return: metrics - .new_count(AND_CANDIDATES_PRUNED_BEFORE_RETURN_METRIC, partition), - and_full_scores: metrics.new_count(AND_FULL_SCORES_METRIC, partition), - freqs_collected: metrics.new_count(FREQS_COLLECTED_METRIC, partition), - baseline_metrics: BaselineMetrics::new(metrics, partition), +fn scored_documents_batch(schema: SchemaRef, documents: Vec) -> Result { + let row_ids = UInt64Array::from_iter_values(documents.iter().map(|document| document.row_id)); + let scores = Float32Array::from_iter_values(documents.iter().map(|document| document.score.0)); + let mut columns = vec![Arc::new(row_ids) as Arc]; + if schema.field_with_name(DOC_INDEX_COL).is_ok() { + let mut builder = ListBuilder::new(UInt32Builder::new()).with_field(Field::new( + "item", + DataType::UInt32, + false, + )); + for document in &documents { + builder.values().append_slice(&document.doc_index); + builder.append(true); } + columns.push(Arc::new(builder.finish())); } + columns.push(Arc::new(scores)); + Ok(RecordBatch::try_new(schema, columns)?) +} - pub fn record_parts_searched(&self, num_parts: usize) { - self.partitions_searched.add(num_parts); - } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct DocumentKey { + row_id: u64, + doc_index: Vec, } -impl MetricsCollector for FtsIndexMetrics { - fn record_parts_loaded(&self, num_parts: usize) { - self.index_metrics.record_parts_loaded(num_parts); - } +fn batch_document_keys(batch: &RecordBatch) -> Result> { + let row_ids = batch[ROW_ID].as_primitive::(); + let doc_indices = batch + .column_by_name(DOC_INDEX_COL) + .map(|column| column.as_list::()); + (0..batch.num_rows()) + .map(|row| { + let doc_index = if let Some(doc_indices) = doc_indices { + if doc_indices.is_null(row) { + return Err(Error::internal( + "element-document FTS produced a null document coordinate".to_string(), + )); + } + doc_indices + .value(row) + .as_primitive::() + .values() + .to_vec() + } else { + Vec::new() + }; + Ok(DocumentKey { + row_id: row_ids.value(row), + doc_index, + }) + }) + .collect() +} - fn record_index_loads(&self, num_indexes: usize) { - self.index_metrics.record_index_loads(num_indexes); - } +fn batch_scored_document_keys(batch: &RecordBatch) -> Result> { + let keys = batch_document_keys(batch)?; + let scores = batch[SCORE_COL].as_primitive::(); + Ok(keys + .into_iter() + .enumerate() + .map(|(index, key)| (key, scores.value(index))) + .collect()) +} - fn record_comparisons(&self, num_comparisons: usize) { - self.index_metrics.record_comparisons(num_comparisons); - } +fn batch_scored_document_keys_sum_scores(batch: &RecordBatch) -> Result> { + let keys = batch_document_keys(batch)?; + let schema = batch.schema(); + let score_columns = schema + .fields() + .iter() + .enumerate() + .filter(|(_, field)| field.name() == SCORE_COL) + .map(|(index, _)| batch.column(index).as_primitive::()) + .collect::>(); + if score_columns.is_empty() { + return Err(Error::internal(format!( + "Boolean MUST result is missing required {SCORE_COL} columns" + ))); + } + keys.into_iter() + .enumerate() + .map(|(row, key)| { + let score: f32 = score_columns.iter().map(|scores| scores.value(row)).sum(); + if !score.is_finite() { + return Err(Error::internal(format!( + "Boolean MUST score sum must be finite, got {score} for row_id={}", + key.row_id + ))); + } + Ok((key, score)) + }) + .collect() +} - fn record_and_candidates_seen(&self, num_candidates: usize) { - self.and_candidates_seen.add(num_candidates); - } +fn document_key_scores_batch( + schema: SchemaRef, + values: impl IntoIterator, +) -> Result { + scored_documents_batch( + schema, + values + .into_iter() + .map(|(key, score)| ScoredDoc::with_doc_index(key.row_id, key.doc_index, score)) + .collect(), + ) +} + +fn compare_scored_documents( + (left_key, left_score): &(DocumentKey, f32), + (right_key, right_score): &(DocumentKey, f32), +) -> Ordering { + right_score + .total_cmp(left_score) + .then_with(|| left_key.cmp(right_key)) +} - fn record_and_candidates_pruned_before_return(&self, num_candidates: usize) { - self.and_candidates_pruned_before_return.add(num_candidates); +fn count_fts_leaves(query: &FtsQuery) -> usize { + match query { + FtsQuery::Match(_) | FtsQuery::Phrase(_) => 1, + FtsQuery::Boost(query) => { + count_fts_leaves(&query.positive) + count_fts_leaves(&query.negative) + } + FtsQuery::MultiMatch(query) => query.match_queries.len(), + FtsQuery::Boolean(query) => query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .map(count_fts_leaves) + .sum(), } +} + +/// Return every leaf column, including prohibited Boolean leaves. +/// +/// The repeated, ordered list is useful beyond the distinct set returned by +/// `FtsQueryNode::columns`: each leaf contributes its own posting-partition +/// work and must use the tokenizer and statistics of its field. +fn compound_leaf_columns(query: &FtsQuery) -> Result> { + fn visit<'a>(query: &'a FtsQuery, columns: &mut Vec<&'a str>) -> Result<()> { + let required_column = |column: &'a Option, kind: &str| { + column.as_deref().ok_or_else(|| { + Error::invalid_input(format!( + "cross-column compound FTS {kind} leaf is missing its resolved column" + )) + }) + }; - fn record_and_full_scores(&self, num_scores: usize) { - self.and_full_scores.add(num_scores); + match query { + FtsQuery::Match(query) => columns.push(required_column(&query.column, "Match")?), + FtsQuery::Phrase(query) => columns.push(required_column(&query.column, "Phrase")?), + FtsQuery::Boost(query) => { + visit(&query.positive, columns)?; + visit(&query.negative, columns)?; + } + FtsQuery::MultiMatch(query) => { + for query in &query.match_queries { + columns.push(required_column(&query.column, "MultiMatch")?); + } + } + FtsQuery::Boolean(query) => { + for query in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(query, columns)?; + } + } + } + Ok(()) } - fn record_freqs_collected(&self, num_collections: usize) { - self.freqs_collected.add(num_collections); + let mut columns = Vec::with_capacity(count_fts_leaves(query)); + visit(query, &mut columns)?; + Ok(columns) +} + +fn compound_query_uses_fuzzy_expansion(query: &FtsQuery) -> bool { + match query { + FtsQuery::Match(query) => uses_fuzzy_expansion(query.fuzziness), + FtsQuery::Phrase(_) => false, + FtsQuery::Boost(query) => { + compound_query_uses_fuzzy_expansion(&query.positive) + || compound_query_uses_fuzzy_expansion(&query.negative) + } + FtsQuery::MultiMatch(query) => query + .match_queries + .iter() + .any(|query| uses_fuzzy_expansion(query.fuzziness)), + FtsQuery::Boolean(query) => query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + .any(compound_query_uses_fuzzy_expansion), } } +/// One DataFusion boundary around a posting-backed compound scorer tree. #[derive(Debug)] -pub struct MatchQueryExec { +pub struct CompoundQueryExec { dataset: Arc, - query: MatchQuery, + query: FtsQuery, + tokenized_query: Arc>, params: FtsSearchParams, prefilter_source: PreFilterSource, - /// When set, `execute()` skips `build_global_bm25_scorer` and threads this - /// scorer down to `InvertedIndex::bm25_search`. + /// When set, leaf scorers use this instead of building one from the + /// searched segments — see [`MatchQueryExec::with_base_scorer`]. base_scorer: Option>, - /// When set, `execute()` skips `load_segments` and searches exactly these - /// segments. - preset_segments: Option>, - + /// Canonical vocabulary/scorer pair for a root Match query prepared over + /// the complete corpus before this exec was restricted to a segment + /// subset. + prepared_match: Option>, + segment_selection: FtsSegmentSelection, + /// Caller-supplied row-address mask, intersected into the prefilter so the + /// compound scorer ranks only surviving rows (see + /// [`MatchQueryExec::with_external_mask`]). + external_mask: Option>, properties: Arc, metrics: ExecutionPlanMetricsSet, } -impl DisplayAs for MatchQueryExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match t { - DisplayFormatType::Default | DisplayFormatType::Verbose => { - write!( - f, - "MatchQuery: column={}, query={}", - self.query.column.as_deref().unwrap_or_default(), - self.query.terms - ) - } - DisplayFormatType::TreeRender => { - write!( - f, - "MatchQuery\ncolumn={}\nquery={}", - self.query.column.as_deref().unwrap_or_default(), - self.query.terms - ) - } - } +impl CompoundQueryExec { + pub fn new_with_segments( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segments: Vec, + ) -> Self { + Self::new_inner( + dataset, + query, + params, + prefilter_source, + FtsSegmentSelection::ExactResolved(Arc::from(segments)), + ) } -} -impl MatchQueryExec { - /// Merge the fuzzy fields from `query` into `params` so that the stored - /// params reflect what BM25 stat collection and search will actually use. - fn effective_params(query: &MatchQuery, params: FtsSearchParams) -> FtsSearchParams { - params - .with_fuzziness(query.fuzziness) - .with_max_expansions(query.max_expansions) - .with_prefix_length(query.prefix_length) + pub fn new_with_segment_uuids( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segment_uuids: Vec, + ) -> Self { + Self::new_inner( + dataset, + query, + params, + prefilter_source, + FtsSegmentSelection::exact_uuids(segment_uuids), + ) + } + + fn new_inner( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segment_selection: FtsSegmentSelection, + ) -> Self { + Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + base_scorer: None, + prepared_match: None, + segment_selection, + external_mask: None, + properties: Arc::new(PlanProperties::new( + EquivalenceProperties::new(FTS_SCHEMA.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )), + metrics: ExecutionPlanMetricsSet::new(), + } + } + + /// See [`MatchQueryExec::with_external_mask`]. + pub fn with_external_mask(mut self, mask: Option>) -> Self { + self.external_mask = mask; + self + } + + /// Override locally computed BM25 statistics with a corpus-wide scorer. + /// + /// The scorer must cover every token in every query leaf, including fuzzy + /// expansions. Execution returns an error when any required token is absent. + pub fn with_base_scorer(mut self, scorer: Arc) -> Self { + self.base_scorer = Some(scorer); + self.prepared_match = None; + self + } + + /// Override root-Match preparation with one canonical vocabulary/scorer + /// pair built against the complete corpus. + /// + /// This is required for distributed fuzzy execution over a segment subset; + /// a scorer alone cannot preserve the globally capped rewrite. + #[doc(hidden)] + pub fn with_prepared_match(mut self, prepared: Arc) -> Self { + self.prepared_match = Some(prepared); + self.base_scorer = None; + self + } + + pub fn dataset(&self) -> &Arc { + &self.dataset + } + + pub fn query(&self) -> &FtsQuery { + &self.query + } + + pub fn params(&self) -> &FtsSearchParams { + &self.params + } + + pub fn prefilter_source(&self) -> &PreFilterSource { + &self.prefilter_source + } + + pub fn base_scorer(&self) -> Option<&Arc> { + self.base_scorer.as_ref() + } + + /// See [`MatchQueryExec::explicit_segment_uuids`]. + pub fn explicit_segment_uuids(&self) -> Option> { + self.segment_selection.explicit_segment_uuids() + } +} + +#[derive(Debug)] +struct QueryLocalResidualShard { + index: QueryLocalFtsIndex, + stats: QueryLocalFtsStats, +} + +async fn index_query_local_residual_batch( + mut residual: QueryLocalResidualShard, + batch: RecordBatch, + allowed_terms: Arc>, +) -> Result { + spawn_cpu(move || { + let row_ids = batch + .column_by_name(ROW_ID) + .ok_or_else(|| { + Error::invalid_input( + "hybrid compound FTS residual input is missing _rowid".to_string(), + ) + })? + .as_primitive::(); + let stats = residual.index.insert_with_row_ids_for_terms( + &batch, + row_ids, + allowed_terms.as_ref(), + )?; + residual.stats.checked_add_assign(stats)?; + Ok(residual) + }) + .await +} + +/// Build a bounded set of independent residual posting shards. +/// +/// A single [`QueryLocalFtsIndex`] intentionally has one writer. Reusing one +/// index per CPU worker preserves that contract while allowing different scan +/// batches to tokenize in parallel. Completed workers immediately take the +/// next batch, so the entire stream is never collected in memory and the +/// number of live tokenizers/posting maps is bounded by the CPU pool size. +async fn index_query_local_residual( + residual_input: SendableRecordBatchStream, + seed: QueryLocalFtsIndex, + allowed_terms: Arc>, +) -> DataFusionResult> { + // Match flat FTS's CPU-task sizing. Dataset scan batches are normally + // row-bounded (often 8,192 rows), which can leave a small residual with + // only one or two tokenizer tasks. Byte rechunking keeps tasks substantial + // while exposing enough parallelism for variable-width text. + const ACCUMULATE_BYTES: usize = 256 * 1024; + const SLICE_BYTES: usize = 512 * 1024; + let input_schema = residual_input.schema(); + let mut residual_input = Box::pin(lance_arrow::stream::rechunk_stream_by_size( + residual_input, + input_schema, + ACCUMULATE_BYTES, + SLICE_BYTES, + )); + let parallelism = get_num_compute_intensive_cpus().max(1); + let mut initial_batches = Vec::with_capacity(parallelism); + let mut is_input_exhausted = false; + + while initial_batches.len() < parallelism { + let Some(batch) = residual_input.try_next().await? else { + is_input_exhausted = true; + break; + }; + initial_batches.push(batch); + } + + if initial_batches.is_empty() { + return Ok(vec![QueryLocalResidualShard { + index: seed, + stats: QueryLocalFtsStats::default(), + }]); + } + + // Construct every shard from the already-loaded seed before dispatching + // CPU work. This keeps tokenizer model I/O out of `spawn_cpu` closures. + let mut initial_shards = Vec::with_capacity(initial_batches.len()); + for _ in 1..initial_batches.len() { + initial_shards.push(QueryLocalResidualShard { + index: seed.empty_sibling(), + stats: QueryLocalFtsStats::default(), + }); + } + initial_shards.push(QueryLocalResidualShard { + index: seed, + stats: QueryLocalFtsStats::default(), + }); + + let mut in_flight = FuturesUnordered::new(); + for (shard, batch) in initial_shards.into_iter().zip(initial_batches) { + in_flight.push(index_query_local_residual_batch( + shard, + batch, + allowed_terms.clone(), + )); + } + + let mut shards = Vec::with_capacity(parallelism.min(in_flight.len())); + while let Some(shard) = in_flight.try_next().await? { + if is_input_exhausted { + shards.push(shard); + continue; + } + match residual_input.try_next().await? { + Some(batch) => in_flight.push(index_query_local_residual_batch( + shard, + batch, + allowed_terms.clone(), + )), + None => { + is_input_exhausted = true; + shards.push(shard); + } + } + } + Ok(shards) +} + +async fn query_local_residual_leaves( + shards: Vec, + query: FtsQuery, + scorer: Arc, +) -> Result>> { + let shard_leaves = stream::iter(shards.into_iter().map(|shard| { + let query = query.clone(); + let scorer = scorer.clone(); + spawn_cpu(move || shard.index.exact_leaf_results(&query, scorer.as_ref())) + })) + .buffered(get_num_compute_intensive_cpus().max(1)) + .try_collect::>() + .await?; + + let leaf_count = shard_leaves.first().map_or(0, Vec::len); + let mut merged = vec![Vec::new(); leaf_count]; + for leaves in shard_leaves { + if leaves.len() != leaf_count { + return Err(Error::internal(format!( + "hybrid compound FTS residual shards produced inconsistent leaf counts: expected {leaf_count}, got {}", + leaves.len() + ))); + } + for (merged, rows) in merged.iter_mut().zip(leaves) { + merged.extend(rows); + } + } + Ok(merged) +} + +fn residual_bm25_scorer( + committed_scorer: &MemBM25Scorer, + shards: &[QueryLocalResidualShard], +) -> Result { + let mut scorer = committed_scorer.clone(); + for shard in shards { + shard.stats.add_to_scorer(&mut scorer)?; + } + Ok(scorer) +} + +/// Compound FTS over committed postings plus a small append-only residual scan. +/// +/// The residual documents are tokenized once into query-local postings, rather +/// than once for every compound leaf. The indexed arm uses committed-index +/// BM25 statistics. The residual arm extends those statistics with the +/// query-local materialized documents, which matches the established mixed +/// flat-search approximation without rescanning the residual input or rebuilding +/// exact corpus statistics. +#[derive(Debug)] +pub(crate) struct HybridCompoundQueryExec { + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + column: String, + segments: Arc<[IndexMetadata]>, + residual_input: Arc, + properties: Arc, + metrics: ExecutionPlanMetricsSet, +} + +impl HybridCompoundQueryExec { + pub(crate) fn new( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + column: String, + segments: Vec, + residual_input: Arc, + ) -> Self { + Self { + dataset, + query, + params, + column, + segments: Arc::from(segments), + residual_input, + properties: Arc::new(PlanProperties::new( + EquivalenceProperties::new(FTS_SCHEMA.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )), + metrics: ExecutionPlanMetricsSet::new(), + } + } +} + +impl DisplayAs for HybridCompoundQueryExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "HybridCompoundFtsScorer: column={}, query={}", + self.column, self.query + ) + } +} + +impl ExecutionPlan for HybridCompoundQueryExec { + fn name(&self) -> &str { + "HybridCompoundQueryExec" + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.residual_input] + } + + fn required_input_distribution(&self) -> Vec { + vec![Distribution::SinglePartition] + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal(format!( + "hybrid compound FTS expected one residual child, got {}", + children.len() + ))); + } + let residual_input = children.pop().ok_or_else(|| { + DataFusionError::Internal("hybrid compound FTS lost its residual child".to_string()) + })?; + Ok(Arc::new(Self::new( + self.dataset.clone(), + self.query.clone(), + self.params.clone(), + self.column.clone(), + self.segments.to_vec(), + residual_input, + ))) + } + + #[instrument(name = "hybrid_compound_fts_exec", level = "debug", skip_all)] + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let dataset = self.dataset.clone(); + let query = self.query.clone(); + let params = self.params.clone(); + let column = self.column.clone(); + let segments = self.segments.clone(); + let residual_input = self.residual_input.clone(); + let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); + let schema = self.schema(); + + let stream = stream::once(async move { + let _timer = metrics.baseline_metrics.elapsed_compute().timer(); + let indices = + open_fts_segments(&dataset, &column, &segments, &metrics.index_metrics).await?; + let first_index = indices.first().ok_or_else(|| { + DataFusionError::Execution(format!( + "FTS index for column {column} has no committed segments" + )) + })?; + let field_id = dataset.schema().field_id(&column)?; + let tokenizer = first_index.tokenizer(); + let doc_type = tokenizer.doc_type(); + let residual_seed = QueryLocalFtsIndex::try_with_loaded_tokenizer( + field_id, + column.clone(), + first_index.params().clone(), + tokenizer, + )?; + let terms = residual_seed.exact_query_terms(&query)?; + if terms.is_empty() { + metrics.baseline_metrics.record_output(0); + return scored_documents_batch(schema, Vec::new()).map_err(DataFusionError::from); + } + let allowed_terms = Arc::new(terms.iter().cloned().collect::>()); + let query_tokens = Tokens::new(terms.clone(), doc_type); + let exact_params = params + .clone() + .with_fuzziness(Some(0)) + .with_phrase_slop(None); + + let residual_context = context.clone(); + let residual_indexing = async move { + let residual_input = residual_input.execute(partition, residual_context)?; + index_query_local_residual(residual_input, residual_seed, allowed_terms).await + }; + let scorer_build = async { + let scorer = build_global_bm25_scorer( + &indices, + &query_tokens, + &exact_params, + Some(metrics.as_ref()), + ) + .await?; + DataFusionResult::>::Ok(Arc::new(scorer)) + }; + let (residual_shards, committed_scorer) = + futures::future::try_join(residual_indexing, scorer_build).await?; + let residual_scorer = Arc::new(residual_bm25_scorer( + committed_scorer.as_ref(), + &residual_shards, + )?); + let limit = params.limit.ok_or_else(|| { + DataFusionError::Execution( + "hybrid compound FTS requires a bounded result limit".to_string(), + ) + })?; + + let prefilter = build_prefilter( + context, + partition, + &PreFilterSource::None, + dataset, + &segments, + PreFilterMasks { + overlay_block: None, + external_mask: None, + }, + )?; + let indexed_search = compound_search_with_base_scorer( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + committed_scorer, + ); + let residual_query = query.clone(); + let residual_search = async move { + let residual_leaves = query_local_residual_leaves( + residual_shards, + residual_query.clone(), + residual_scorer, + ) + .await?; + spawn_cpu(move || { + materialized_compound_top_k(&residual_query, residual_leaves, limit) + }) + .await + }; + let ((indexed_row_ids, indexed_scores), (residual_row_ids, residual_scores)) = + futures::future::try_join(indexed_search, residual_search).await?; + + let mut documents = indexed_row_ids + .into_iter() + .zip(indexed_scores) + .chain(residual_row_ids.into_iter().zip(residual_scores)) + .map(|(row_id, score)| ScoredDoc::new(row_id, score)) + .collect::>(); + documents.sort_unstable_by(|left, right| { + right + .score + .0 + .total_cmp(&left.score.0) + .then_with(|| left.row_id.cmp(&right.row_id)) + }); + documents.truncate(limit); + metrics.baseline_metrics.record_output(documents.len()); + scored_documents_batch(schema, documents).map_err(DataFusionError::from) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream.stream_in_current_span().boxed(), + ))) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn properties(&self) -> &Arc { + &self.properties + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum WandExactnessCertificate { + Exhaustive, + Strict, + Ambiguous, +} + +/// Classify a globally merged bounded Match WAND result. +/// +/// Sorting before classification is essential: per-segment WAND output is not +/// a final cross-segment ordering. A strict score gap after result k proves +/// that score-only pruning could not have discarded a row-id tie at the final +/// boundary. Returning fewer rows than requested proves exhaustion. Merely +/// observing a lower score during collection is not a proof because other +/// partitions may still contain kth-score ties. +fn classify_wand_exactness_certificate( + documents: &mut [ScoredDoc], + limit: usize, + probe_limit: usize, +) -> WandExactnessCertificate { + if limit == 0 + || probe_limit <= limit + || documents.len() > probe_limit + || documents + .iter() + .any(|document| !document.score.0.is_finite()) + { + return WandExactnessCertificate::Ambiguous; + } + documents.sort_unstable_by(|left, right| { + right + .score + .0 + .total_cmp(&left.score.0) + .then_with(|| left.row_id.cmp(&right.row_id)) + }); + if documents.len() < probe_limit { + WandExactnessCertificate::Exhaustive + } else if documents[limit - 1].score.0.total_cmp( + &documents + .last() + .expect("a full bounded probe has a guard candidate") + .score + .0, + ) == Ordering::Greater + { + WandExactnessCertificate::Strict + } else { + WandExactnessCertificate::Ambiguous + } +} + +fn finish_wand_documents(mut documents: Vec, limit: usize) -> (Vec, Vec) { + documents.truncate(limit); + documents + .into_iter() + .map(|document| (document.row_id, document.score.0)) + .unzip() +} + +async fn exact_prepared_match_fallback( + indices: &[Arc], + query: &FtsQuery, + params: &FtsSearchParams, + prefilter: Arc, + metrics: Arc, + prepared_match: Arc, + score_floor: Option, +) -> Result<(Vec, Vec)> { + if let Some(score_floor) = score_floor { + compound_search_prepared_match_with_score_floor( + indices, + query, + params, + prefilter, + metrics, + prepared_match, + score_floor, + ) + .await + } else { + compound_search_prepared_match(indices, query, params, prefilter, metrics, prepared_match) + .await + } +} + +impl DisplayAs for CompoundQueryExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "CompoundFtsScorer: query={}", self.query)?; + fmt_tokenized_compound_query(&self.tokenized_query, ", ", f) + } + DisplayFormatType::TreeRender => { + write!(f, "CompoundFtsScorer\nquery={}", self.query)?; + fmt_tokenized_compound_query(&self.tokenized_query, "\n", f) + } + } + } +} + +impl ExecutionPlan for CompoundQueryExec { + fn name(&self) -> &str { + "CompoundQueryExec" + } + + fn children(&self) -> Vec<&Arc> { + self.prefilter_source.execution_plan().into_iter().collect() + } + + fn required_input_distribution(&self) -> Vec { + self.children() + .iter() + .map(|_| Distribution::SinglePartition) + .collect() + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + let prefilter_source = match children.len() { + 0 if matches!(self.prefilter_source, PreFilterSource::None) => PreFilterSource::None, + 1 => { + let Some(source) = children.pop() else { + return Err(DataFusionError::Internal( + "compound FTS lost its prefilter child".to_string(), + )); + }; + self.prefilter_source.with_execution_plan(source)? + } + count => { + return Err(DataFusionError::Internal(format!( + "compound FTS expected at most one prefilter child, got {count}" + ))); + } + }; + Ok(Arc::new(Self { + dataset: self.dataset.clone(), + query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), + params: self.params.clone(), + prefilter_source, + base_scorer: self.base_scorer.clone(), + prepared_match: self.prepared_match.clone(), + segment_selection: self.segment_selection.clone(), + external_mask: self.external_mask.clone(), + properties: self.properties.clone(), + metrics: ExecutionPlanMetricsSet::new(), + })) + } + + #[instrument(name = "compound_fts_scorer_exec", level = "debug", skip_all)] + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let dataset = self.dataset.clone(); + let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); + let params = self.params.clone(); + let prefilter_source = self.prefilter_source.clone(); + let preset_base_scorer = self.base_scorer.clone(); + let preset_prepared_match = self.prepared_match.clone(); + let segment_selection = self.segment_selection.clone(); + let external_mask = self.external_mask.clone(); + let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); + + let stream = stream::once(async move { + let _timer = metrics.baseline_metrics.elapsed_compute().timer(); + let columns = query.columns(); + let column = columns.iter().next().ok_or_else(|| { + DataFusionError::Execution( + "compound FTS query does not reference an indexed column".to_string(), + ) + })?; + if columns.len() != 1 { + return Err(DataFusionError::Execution( + "posting-backed compound FTS requires exactly one column".to_string(), + )); + } + let segments = segment_selection + .resolve( + &dataset, + column, + DocumentGranularity::Row, + &metrics.segment_bind_duration, + ) + .await?; + if preset_prepared_match.is_some() && !matches!(&query, FtsQuery::Match(_)) { + return Err(DataFusionError::Execution( + "CompoundQueryExec prepared vocabulary requires a root Match query".to_string(), + )); + } + let scorer_only_fuzzy = preset_prepared_match.is_none() + && compound_query_uses_fuzzy_expansion(&query) + && preset_base_scorer.is_some(); + let scorer_override_covers_all = if scorer_only_fuzzy { + segment_selection + .covers_all_committed(&dataset, column, DocumentGranularity::Row, &segments) + .await? + } else { + true + }; + let _details = load_segment_details(&dataset, column, &segments).await?; + let indices = + open_fts_segments(&dataset, column, &segments, &metrics.index_metrics).await?; + if let Some(first_index) = indices.first() { + tokenized_query + .get_or_init(|| tokenize_compound_query(&query, first_index.as_ref())); + } + let mut prefilter = build_prefilter( + context, + partition, + &prefilter_source, + dataset, + &segments, + PreFilterMasks { + overlay_block: None, + external_mask, + }, + )?; + let deleted_fragments = + indices + .iter() + .fold(roaring::RoaringBitmap::new(), |mut deleted, index| { + deleted |= index.deleted_fragments().clone(); + deleted + }); + if !deleted_fragments.is_empty() { + let prefilter = Arc::get_mut(&mut prefilter).ok_or_else(|| { + DataFusionError::Internal( + "compound FTS prefilter was unexpectedly shared before initialization" + .to_string(), + ) + })?; + prefilter.set_deleted_fragments(deleted_fragments); + } + metrics.record_parts_searched( + indices + .iter() + .map(|index| index.partition_count()) + .sum::() + .saturating_mul(count_fts_leaves(&query)), + ); + let base_scorer = match (preset_prepared_match.is_some(), preset_base_scorer) { + (true, _) => None, + (false, scorer) => scorer, + }; + if base_scorer.is_some() && scorer_only_fuzzy && !scorer_override_covers_all { + return Err(DataFusionError::Execution( + "fuzzy CompoundQueryExec cannot use a scorer-only override over a segment subset; prepare the canonical vocabulary with prepare_bm25_query and pass it with with_prepared_match" + .to_string(), + )); + } + let certificate_limit = match (&query, params.limit) { + (FtsQuery::Match(match_query), Some(limit)) + if limit > 0 + && params.wand_factor == 1.0 + && match_query.boost.is_finite() + && match_query.boost > 0.0 + && base_scorer.is_none() + && indices + .iter() + .all(|index| index.supports_wand_exactness_certificate()) => + { + limit + .checked_add(1) + .map(|wand_limit| (match_query.clone(), limit, wand_limit)) + } + _ => None, + }; + let (row_ids, scores) = if let Some((match_query, limit, wand_limit)) = + certificate_limit + { + let wand_params = MatchQueryExec::effective_params(&match_query, params.clone()) + .with_phrase_slop(None) + .with_limit(Some(wand_limit)); + let prepared = if let Some(prepared_match) = preset_prepared_match.clone() { + Arc::new(PreparedMatch { + query: prepared_match, + params: Arc::new(wand_params), + operator: match_query.operator, + }) + } else { + let first_index = indices.first().ok_or_else(|| { + DataFusionError::Execution(format!( + "FTS index for column {column} has no segments" + )) + })?; + let mut tokenizer = + tokenizer_for_match_query(first_index.as_ref(), match_query.fuzziness); + let tokens = collect_query_tokens(&match_query.terms, &mut tokenizer); + let scorer_start = std::time::Instant::now(); + let prepared = Arc::new( + PreparedMatch::new( + &indices, + tokens, + wand_params, + match_query.operator, + metrics.as_ref(), + None, + ) + .await?, + ); + metrics.record_scorer_build(scorer_start.elapsed()); + prepared + }; + + // Zero-weight terms can match documents without contributing a + // positive score. A short score-only WAND result therefore does + // not prove exhaustion. Preserve exact membership semantics for + // those rare corpora without recording a certificate attempt. + if prepared.query.scorer().token_docs.keys().any(|token| { + let weight = prepared.query.scorer().query_weight(token); + !weight.is_finite() || weight <= 0.0 + }) { + compound_search_prepared_match( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + prepared.query.clone(), + ) + .await? + } else { + prefilter.wait_for_ready().await?; + let mut documents = search_prepared_segments( + &indices, + prepared.clone(), + prefilter.clone(), + metrics.clone(), + None, + ) + .await?; + documents.iter_mut().for_each(|document| { + document.score.0 *= match_query.boost; + }); + match classify_wand_exactness_certificate(&mut documents, limit, wand_limit) { + WandExactnessCertificate::Exhaustive => { + finish_wand_documents(documents, limit) + } + WandExactnessCertificate::Strict => finish_wand_documents(documents, limit), + WandExactnessCertificate::Ambiguous => { + let score_floor = documents + .get(limit - 1) + .map(|document| document.score.0) + .filter(|score| score.is_finite()); + let completion_limit = limit + .checked_add(WAND_TIE_COMPLETION_BUDGET) + .and_then(|limit| limit.checked_add(1)); + if let (Some(score_floor), Some(completion_limit)) = + (score_floor, completion_limit) + { + let completion_prepared = Arc::new(PreparedMatch { + query: prepared.query.clone(), + params: Arc::new( + prepared + .params + .as_ref() + .clone() + .with_limit(Some(completion_limit)), + ), + operator: prepared.operator, + }); + let raw_score_floor = + exclusive_scaled_score_floor(score_floor, match_query.boost); + let mut completion = search_prepared_segments( + &indices, + completion_prepared, + prefilter.clone(), + metrics.clone(), + raw_score_floor, + ) + .await?; + completion.iter_mut().for_each(|document| { + document.score.0 *= match_query.boost; + }); + match classify_wand_exactness_certificate( + &mut completion, + limit, + completion_limit, + ) { + WandExactnessCertificate::Exhaustive => { + finish_wand_documents(completion, limit) + } + WandExactnessCertificate::Strict => { + finish_wand_documents(completion, limit) + } + WandExactnessCertificate::Ambiguous => { + let seeded_floor = completion + .iter() + .all(|document| document.score.0.is_finite()) + .then_some(score_floor); + exact_prepared_match_fallback( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + prepared.query.clone(), + seeded_floor, + ) + .await? + } + } + } else { + exact_prepared_match_fallback( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + prepared.query.clone(), + score_floor, + ) + .await? + } + } + } + } + } else { + match (preset_prepared_match, base_scorer) { + (Some(prepared_match), _) => { + compound_search_prepared_match( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + prepared_match, + ) + .await? + } + (None, Some(base_scorer)) => { + compound_search_with_base_scorer( + &indices, + &query, + ¶ms, + prefilter, + metrics.clone(), + base_scorer, + ) + .await? + } + (None, None) => { + compound_search(&indices, &query, ¶ms, prefilter, metrics.clone()) + .await? + } + } + }; + metrics.baseline_metrics.record_output(row_ids.len()); + Ok::<_, DataFusionError>(RecordBatch::try_new( + FTS_SCHEMA.clone(), + vec![ + Arc::new(UInt64Array::from(row_ids)), + Arc::new(Float32Array::from(scores)), + ], + )?) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream.stream_in_current_span().boxed(), + ))) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn supports_limit_pushdown(&self) -> bool { + false + } +} + +#[derive(Debug, Clone)] +struct CompoundColumnSelection { + column: String, + segment_selection: FtsSegmentSelection, +} + +/// One DataFusion boundary around a cross-column posting-backed scorer tree. +/// +/// Each column keeps its own ordered segment selection and tokenizer. The +/// lower-level scorer joins leaves in the common row-address domain; segment +/// ordinals are deliberately never paired across columns. +#[derive(Debug)] +pub struct CrossColumnCompoundQueryExec { + dataset: Arc, + query: FtsQuery, + tokenized_query: Arc>, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + columns: Arc<[CompoundColumnSelection]>, + /// Combined into the prefilter so only masked rows are scored (see + /// [`MatchQueryExec::with_external_mask`]). + external_mask: Option>, + properties: Arc, + metrics: ExecutionPlanMetricsSet, +} + +impl CrossColumnCompoundQueryExec { + pub fn new_with_segments( + dataset: Arc, + query: FtsQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + columns: Vec<(String, Vec)>, + ) -> Result { + if params.limit.is_none() { + return Err(Error::invalid_input( + "cross-column compound FTS requires a bounded result limit", + )); + } + let leaf_columns = compound_leaf_columns(&query)?; + let query_columns = leaf_columns.iter().copied().collect::>(); + if query_columns.len() < 2 { + return Err(Error::invalid_input(format!( + "cross-column compound FTS requires at least two query columns, got {}", + query_columns.len() + ))); + } + + let mut selected_columns = HashSet::with_capacity(columns.len()); + for (column, segments) in &columns { + if column.is_empty() { + return Err(Error::invalid_input( + "cross-column compound FTS segment selection has an empty column name", + )); + } + if segments.is_empty() { + return Err(Error::invalid_input(format!( + "cross-column compound FTS requires at least one segment for column {column}" + ))); + } + if !selected_columns.insert(column.as_str()) { + return Err(Error::invalid_input(format!( + "cross-column compound FTS has duplicate segment selections for column {column}" + ))); + } + } + + if selected_columns != query_columns { + let mut missing = query_columns + .difference(&selected_columns) + .copied() + .collect::>(); + let mut unexpected = selected_columns + .difference(&query_columns) + .copied() + .collect::>(); + missing.sort_unstable(); + unexpected.sort_unstable(); + return Err(Error::invalid_input(format!( + "cross-column compound FTS segment selections do not match query leaves: \ + missing={missing:?}, unexpected={unexpected:?}" + ))); + } + + let columns = columns + .into_iter() + .map(|(column, segments)| CompoundColumnSelection { + column, + segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)), + }) + .collect::>(); + Ok(Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + columns: Arc::from(columns), + external_mask: None, + properties: Arc::new(PlanProperties::new( + EquivalenceProperties::new(FTS_SCHEMA.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )), + metrics: ExecutionPlanMetricsSet::new(), + }) + } + + /// See [`MatchQueryExec::with_external_mask`]. + pub fn with_external_mask(mut self, mask: Option>) -> Self { + self.external_mask = mask; + self + } + + pub fn dataset(&self) -> &Arc { + &self.dataset + } + + pub fn query(&self) -> &FtsQuery { + &self.query + } + + pub fn params(&self) -> &FtsSearchParams { + &self.params + } + + pub fn prefilter_source(&self) -> &PreFilterSource { + &self.prefilter_source + } +} + +impl DisplayAs for CrossColumnCompoundQueryExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "CrossColumnCompoundFtsScorer: query={}", self.query)?; + fmt_tokenized_compound_query(&self.tokenized_query, ", ", f) + } + DisplayFormatType::TreeRender => { + write!(f, "CrossColumnCompoundFtsScorer\nquery={}", self.query)?; + fmt_tokenized_compound_query(&self.tokenized_query, "\n", f) + } + } + } +} + +impl ExecutionPlan for CrossColumnCompoundQueryExec { + fn name(&self) -> &str { + "CrossColumnCompoundQueryExec" + } + + fn children(&self) -> Vec<&Arc> { + self.prefilter_source.execution_plan().into_iter().collect() + } + + fn required_input_distribution(&self) -> Vec { + self.children() + .iter() + .map(|_| Distribution::SinglePartition) + .collect() + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + let prefilter_source = match children.len() { + 0 if matches!(self.prefilter_source, PreFilterSource::None) => PreFilterSource::None, + 1 => { + let Some(source) = children.pop() else { + return Err(DataFusionError::Internal( + "cross-column compound FTS lost its prefilter child".to_string(), + )); + }; + self.prefilter_source.with_execution_plan(source)? + } + count => { + return Err(DataFusionError::Internal(format!( + "cross-column compound FTS expected at most one prefilter child, got {count}" + ))); + } + }; + + Ok(Arc::new(Self { + dataset: self.dataset.clone(), + query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), + params: self.params.clone(), + prefilter_source, + columns: self.columns.clone(), + external_mask: self.external_mask.clone(), + properties: self.properties.clone(), + metrics: ExecutionPlanMetricsSet::new(), + })) + } + + #[instrument( + name = "cross_column_compound_fts_scorer_exec", + level = "debug", + skip_all + )] + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let dataset = self.dataset.clone(); + let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); + let params = self.params.clone(); + let prefilter_source = self.prefilter_source.clone(); + let columns = self.columns.clone(); + let external_mask = self.external_mask.clone(); + let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); + + let stream = stream::once(async move { + let _timer = metrics.baseline_metrics.elapsed_compute().timer(); + let selected_segments = columns + .iter() + .flat_map(|selection| { + selection + .segment_selection + .preset_segments() + .into_iter() + .flatten() + .cloned() + }) + .collect::>(); + if selected_segments.is_empty() { + return Err(DataFusionError::Internal( + "cross-column compound FTS lost its exact segment selections".to_string(), + )); + } + // DatasetPreFilter starts its deletion and filter prerequisites in + // the background. Construct it before opening index segments so + // both I/O paths can make progress concurrently. + let mut prefilter = build_prefilter( + context, + partition, + &prefilter_source, + dataset.clone(), + &selected_segments, + PreFilterMasks { + overlay_block: None, + external_mask, + }, + )?; + let opened_columns = try_join_all(columns.iter().cloned().map(|selection| { + let dataset = dataset.clone(); + let metrics = metrics.clone(); + async move { + let segments = selection + .segment_selection + .resolve( + &dataset, + &selection.column, + DocumentGranularity::Row, + &metrics.segment_bind_duration, + ) + .await?; + let indices = open_fts_segments( + &dataset, + &selection.column, + &segments, + &metrics.index_metrics, + ) + .await?; + Ok::<_, DataFusionError>((selection.column, indices)) + } + })) + .await?; + + let mut tokenizer_indices = HashMap::with_capacity(opened_columns.len()); + let mut partition_counts = HashMap::with_capacity(opened_columns.len()); + for (column, indices) in &opened_columns { + let first_index = indices.first().ok_or_else(|| { + DataFusionError::Execution(format!( + "cross-column compound FTS opened no segments for column {column}" + )) + })?; + tokenizer_indices.insert(column.as_str(), first_index.as_ref()); + partition_counts.insert( + column.as_str(), + indices + .iter() + .map(|index| index.partition_count()) + .sum::(), + ); + } + let tokens = tokenize_cross_column_compound_query(&query, &tokenizer_indices)?; + tokenized_query.get_or_init(|| tokens); + + let searched_parts = compound_leaf_columns(&query)?.into_iter().try_fold( + 0usize, + |searched, column| { + let column_parts = partition_counts.get(column).copied().ok_or_else(|| { + DataFusionError::Execution(format!( + "cross-column compound FTS has no opened index for query column \ + {column}" + )) + })?; + Ok::<_, DataFusionError>(searched.saturating_add(column_parts)) + }, + )?; + metrics.record_parts_searched(searched_parts); + + let deleted_fragments = opened_columns.iter().flat_map(|(_, indices)| indices).fold( + roaring::RoaringBitmap::new(), + |mut deleted, index| { + deleted |= index.deleted_fragments().clone(); + deleted + }, + ); + if !deleted_fragments.is_empty() { + let prefilter = Arc::get_mut(&mut prefilter).ok_or_else(|| { + DataFusionError::Internal( + "cross-column compound FTS prefilter was unexpectedly shared before \ + initialization" + .to_string(), + ) + })?; + prefilter.set_deleted_fragments(deleted_fragments); + } + + let search_columns = opened_columns; + let (row_ids, scores) = cross_column_compound_search( + &search_columns, + &query, + ¶ms, + prefilter, + metrics.clone(), + ) + .await?; + metrics.baseline_metrics.record_output(row_ids.len()); + Ok::<_, DataFusionError>(RecordBatch::try_new( + FTS_SCHEMA.clone(), + vec![ + Arc::new(UInt64Array::from(row_ids)), + Arc::new(Float32Array::from(scores)), + ], + )?) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream.stream_in_current_span().boxed(), + ))) + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn supports_limit_pushdown(&self) -> bool { + false + } +} + +/// Fall back to the default simple tokenizer when no on-disk FTS segment exists. +fn default_text_tokenizer() -> Box { + Box::new(TextTokenizer::new( + TextAnalyzer::builder(SimpleTokenizer::default()).build(), + )) +} + +fn tokenizer_for_match_query( + index: &InvertedIndex, + fuzziness: Option, +) -> Box { + // Preserve the legacy explicit-fuzzy behavior, while AUTO fuzziness uses + // the index analyzer so its source terms share the indexed vocabulary's + // normalization and filtering. + if !matches!(fuzziness, Some(distance) if distance > 0) { + return index.tokenizer(); + } + + let analyzer = TextAnalyzer::from(SimpleTokenizer::default()); + match index.tokenizer().doc_type() { + DocType::Text => Box::new(TextTokenizer::new(analyzer)), + DocType::Json => Box::new(JsonTokenizer::new(analyzer)), + } +} + +fn tokenize_compound_query(query: &FtsQuery, index: &InvertedIndex) -> TokenizedCompoundQuery { + fn visit(query: &FtsQuery, index: &InvertedIndex, leaves: &mut Vec) { + match query { + FtsQuery::Match(query) => { + let mut tokenizer = tokenizer_for_match_query(index, query.fuzziness); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Match, + column: query.column.clone(), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + FtsQuery::Phrase(query) => { + let mut tokenizer = index.tokenizer(); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Phrase, + column: query.column.clone(), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + FtsQuery::Boost(query) => { + visit(&query.positive, index, leaves); + visit(&query.negative, index, leaves); + } + FtsQuery::MultiMatch(query) => { + for query in &query.match_queries { + let mut tokenizer = tokenizer_for_match_query(index, query.fuzziness); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Match, + column: query.column.clone(), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + } + FtsQuery::Boolean(query) => { + for query in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(query, index, leaves); + } + } + } + } + + let mut leaves = Vec::with_capacity(count_fts_leaves(query)); + visit(query, index, &mut leaves); + TokenizedCompoundQuery(leaves) +} + +fn tokenize_cross_column_compound_query( + query: &FtsQuery, + indices: &HashMap<&str, &InvertedIndex>, +) -> Result { + fn index_for_leaf<'a>( + column: Option<&str>, + kind: &str, + indices: &HashMap<&str, &'a InvertedIndex>, + ) -> Result<(&'a InvertedIndex, String)> { + let column = column.ok_or_else(|| { + Error::invalid_input(format!( + "cross-column compound FTS {kind} leaf is missing its resolved column" + )) + })?; + let index = indices.get(column).copied().ok_or_else(|| { + Error::invalid_input(format!( + "cross-column compound FTS has no opened index for {kind} column {column}" + )) + })?; + Ok((index, column.to_string())) + } + + fn visit( + query: &FtsQuery, + indices: &HashMap<&str, &InvertedIndex>, + leaves: &mut Vec, + ) -> Result<()> { + match query { + FtsQuery::Match(query) => { + let (index, column) = index_for_leaf(query.column.as_deref(), "Match", indices)?; + let mut tokenizer = tokenizer_for_match_query(index, query.fuzziness); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Match, + column: Some(column), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + FtsQuery::Phrase(query) => { + let (index, column) = index_for_leaf(query.column.as_deref(), "Phrase", indices)?; + let mut tokenizer = index.tokenizer(); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Phrase, + column: Some(column), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + FtsQuery::Boost(query) => { + visit(&query.positive, indices, leaves)?; + visit(&query.negative, indices, leaves)?; + } + FtsQuery::MultiMatch(query) => { + for query in &query.match_queries { + let (index, column) = + index_for_leaf(query.column.as_deref(), "MultiMatch", indices)?; + let mut tokenizer = tokenizer_for_match_query(index, query.fuzziness); + let tokens = collect_query_tokens(&query.terms, &mut tokenizer); + leaves.push(TokenizedQueryLeaf { + kind: TokenizedLeafKind::Match, + column: Some(column), + tokens: TokenizedQuery::from_tokens(&tokens), + }); + } + } + FtsQuery::Boolean(query) => { + for query in query + .should + .iter() + .chain(&query.must) + .chain(&query.must_not) + { + visit(query, indices, leaves)?; + } + } + } + Ok(()) + } + + let mut leaves = Vec::with_capacity(count_fts_leaves(query)); + visit(query, indices, &mut leaves)?; + Ok(TokenizedCompoundQuery(leaves)) +} + +type SharedScorerResult = std::result::Result, Arc>; + +/// Coordinates BM25 corpus statistics between the indexed and flat branches +/// of a mixed search. The flat branch extends the indexed statistics with the +/// unindexed documents, then publishes the resulting corpus-wide scorer. +#[derive(Debug)] +pub(crate) struct SharedFtsScorer { + sender: tokio::sync::watch::Sender>, +} + +impl SharedFtsScorer { + pub(crate) fn new() -> Self { + let (sender, _) = tokio::sync::watch::channel(None); + Self { sender } + } + + fn publish(&self, scorer: MemBM25Scorer) { + self.sender.send_replace(Some(Ok(Arc::new(scorer)))); + } + + fn publish_error(&self, error: &DataFusionError) { + self.sender + .send_replace(Some(Err(Arc::from(error.to_string())))); + } + + async fn wait(&self) -> DataFusionResult> { + let mut receiver = self.sender.subscribe(); + loop { + let result = receiver.borrow_and_update().clone(); + if let Some(result) = result { + return result.map_err(|message| DataFusionError::Execution(message.to_string())); + } + receiver.changed().await.map_err(|_| { + DataFusionError::Execution( + "mixed FTS corpus scorer producer stopped before publishing statistics" + .to_string(), + ) + })?; + } + } +} + +struct SharedFtsScorerProducer { + scorer: Arc, + completed: bool, +} + +impl SharedFtsScorerProducer { + fn new(scorer: Arc) -> Self { + Self { + scorer, + completed: false, + } + } + + fn publish(mut self, scorer: MemBM25Scorer) { + self.scorer.publish(scorer); + self.completed = true; + } + + fn publish_error(mut self, error: &DataFusionError) { + self.scorer.publish_error(error); + self.completed = true; + } +} + +impl Drop for SharedFtsScorerProducer { + fn drop(&mut self) { + if !self.completed { + self.scorer.sender.send_replace(Some(Err(Arc::from( + "mixed FTS corpus scorer producer was cancelled before publishing statistics", + )))); + } + } +} + +/// Time spent resolving an exact ordered UUID selection to committed FTS segments. +pub const FTS_SEGMENT_BIND_DURATION_METRIC: &str = "fts_segment_bind_duration"; + +#[derive(Debug, Clone)] +enum FtsSegmentSelection { + AllCommitted, + ExactResolved(Arc<[IndexMetadata]>), + ExactUuids(Arc<[Uuid]>), +} + +impl FtsSegmentSelection { + fn exact_uuids(mut uuids: Vec) -> Self { + let mut seen = HashSet::with_capacity(uuids.len()); + uuids.retain(|uuid| seen.insert(*uuid)); + Self::ExactUuids(Arc::from(uuids)) + } + + fn preset_segments(&self) -> Option<&[IndexMetadata]> { + match self { + Self::ExactResolved(segments) => Some(segments), + Self::AllCommitted | Self::ExactUuids(_) => None, + } + } + + fn searches_all_committed(&self) -> bool { + matches!(self, Self::AllCommitted) + } + + async fn covers_all_committed( + &self, + dataset: &Dataset, + column: &str, + document_granularity: DocumentGranularity, + resolved: &[IndexMetadata], + ) -> DataFusionResult { + if self.searches_all_committed() { + return Ok(true); + } + let Some(committed) = load_segments(dataset, column, document_granularity).await? else { + return Ok(false); + }; + let selected = resolved + .iter() + .map(|segment| segment.uuid) + .collect::>(); + let committed = committed + .iter() + .map(|segment| segment.uuid) + .collect::>(); + Ok(selected == committed) + } + + fn explicit_segment_uuids(&self) -> Option> { + match self { + Self::AllCommitted => None, + Self::ExactResolved(segments) => { + Some(segments.iter().map(|segment| segment.uuid).collect()) + } + Self::ExactUuids(uuids) => Some(uuids.to_vec()), + } + } + + async fn resolve( + &self, + dataset: &Dataset, + column: &str, + document_granularity: DocumentGranularity, + segment_bind_duration: &Time, + ) -> DataFusionResult> { + let segments = match self { + Self::AllCommitted => load_segments(dataset, column, document_granularity) + .await? + .map(Arc::from) + .ok_or_else(|| { + DataFusionError::Execution(format!( + "No Inverted index found for column {}", + column, + )) + }), + Self::ExactResolved(segments) => Ok(segments.clone()), + Self::ExactUuids(uuids) => { + let _timer = segment_bind_duration.timer(); + let dataset_version = dataset.version_id(); + if uuids.is_empty() { + return Err(DataFusionError::Execution(format!( + "Exact FTS segment selection for column {} at dataset version {} \ + requires at least one segment UUID", + column, dataset_version + ))); + } + + let committed_segments = load_segments(dataset, column, document_granularity) + .await? + .ok_or_else(|| { + DataFusionError::Execution(format!( + "Cannot resolve exact FTS segment selection for column {} at dataset \ + version {}: no Inverted index found", + column, dataset_version + )) + })?; + let mut segments_by_uuid = HashMap::with_capacity(committed_segments.len()); + for segment in committed_segments { + let uuid = segment.uuid; + if segments_by_uuid.insert(uuid, segment).is_some() { + return Err(DataFusionError::Execution(format!( + "FTS metadata for column {} at dataset version {} contains duplicate \ + segment UUID {}", + column, dataset_version, uuid + ))); + } + } + + let mut resolved = Vec::with_capacity(uuids.len()); + for uuid in uuids.iter() { + let segment = segments_by_uuid.get(uuid).ok_or_else(|| { + DataFusionError::Execution(format!( + "Requested FTS segment UUID {} for column {} is not committed in \ + dataset version {}", + uuid, column, dataset_version + )) + })?; + resolved.push(segment.clone()); + } + Ok(Arc::from(resolved)) + } + }?; + let details = load_segment_details(dataset, column, &segments).await?; + let indexed_granularity = DocumentGranularity::try_from(details.document_granularity)?; + if indexed_granularity != document_granularity { + return Err(DataFusionError::Execution(format!( + "FTS segments selected for column {column} use {indexed_granularity:?} document \ + granularity, but the query was resolved as {document_granularity:?}" + ))); + } + Ok(segments) + } +} + +pub struct FtsIndexMetrics { + index_metrics: IndexMetrics, + partitions_searched: Count, + /// Wall time (ms) of the exec-local `build_global_bm25_scorer` + /// fallback; zero when a preset base scorer was injected. + scorer_build_ms: Gauge, + segment_bind_duration: Time, + baseline_metrics: BaselineMetrics, +} + +impl FtsIndexMetrics { + pub fn new(metrics: &ExecutionPlanMetricsSet, partition: usize) -> Self { + Self { + index_metrics: IndexMetrics::new(metrics, partition), + partitions_searched: metrics.new_count(PARTITIONS_SEARCHED_METRIC, partition), + scorer_build_ms: metrics.new_gauge("scorer_build_ms", partition), + segment_bind_duration: metrics.new_time(FTS_SEGMENT_BIND_DURATION_METRIC, partition), + baseline_metrics: BaselineMetrics::new(metrics, partition), + } + } + + pub fn record_parts_searched(&self, num_parts: usize) { + self.partitions_searched.add(num_parts); + } + + pub fn record_scorer_build(&self, elapsed: std::time::Duration) { + self.scorer_build_ms.set(elapsed.as_millis() as usize); + } +} + +impl MetricsCollector for FtsIndexMetrics { + fn record_parts_loaded(&self, num_parts: usize) { + self.index_metrics.record_parts_loaded(num_parts); + } + + fn record_index_loads(&self, num_indexes: usize) { + self.index_metrics.record_index_loads(num_indexes); + } + + fn record_comparisons(&self, num_comparisons: usize) { + self.index_metrics.record_comparisons(num_comparisons); + } + + fn record_index_cache_hits(&self, num_hits: usize) { + self.index_metrics.record_index_cache_hits(num_hits); + } + + fn record_index_cache_misses(&self, num_misses: usize) { + self.index_metrics.record_index_cache_misses(num_misses); + } +} + +#[derive(Debug)] +pub struct MatchQueryExec { + dataset: Arc, + query: MatchQuery, + tokenized_query: Arc>, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + /// When set, `execute()` skips `build_global_bm25_scorer` and threads this + /// scorer down to `InvertedIndex::bm25_search`. + base_scorer: Option>, + /// Canonical fuzzy vocabulary and corpus-wide scorer prepared against the + /// complete distributed corpus. Unlike `base_scorer`, this is safe to + /// forward to an exec that searches only a segment subset. + prepared_query: Option>, + /// Corpus-wide scorer published by the flat branch of a mixed search. + shared_scorer: Option>, + segment_selection: FtsSegmentSelection, + /// Rows whose indexed values were superseded by newer data overlays. + overlay_block: Option, + document_granularity: DocumentGranularity, + schema: SchemaRef, + /// Optional external row-address mask combined (logical AND) with the BM25 + /// prefilter so only masked rows are scored (see [`Self::with_external_mask`]). + external_mask: Option>, + + properties: Arc, + metrics: ExecutionPlanMetricsSet, +} + +impl DisplayAs for MatchQueryExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!( + f, + "MatchQuery: column={}, query=[{}]", + self.query.column.as_deref().unwrap_or_default(), + self.query.terms + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) + } + DisplayFormatType::TreeRender => { + write!( + f, + "MatchQuery\ncolumn={}\nquery={}", + self.query.column.as_deref().unwrap_or_default(), + self.query.terms + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) + } + } + } +} + +impl MatchQueryExec { + /// Merge the fuzzy fields from `query` into `params` so that the stored + /// params reflect what BM25 stat collection and search will actually use. + fn effective_params(query: &MatchQuery, params: FtsSearchParams) -> FtsSearchParams { + params + .with_fuzziness(query.fuzziness) + .with_max_expansions(query.max_expansions) + .with_prefix_length(query.prefix_length) + } + + pub fn new( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_document_granularity( + dataset, + query, + params, + prefilter_source, + document_granularity, + )) + } + + pub fn new_with_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + document_granularity: DocumentGranularity, + ) -> Self { + let schema = fts_schema(document_granularity); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )); + let params = Self::effective_params(&query, params); + Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + base_scorer: None, + prepared_query: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::AllCommitted, + overlay_block: None, + document_granularity, + schema, + external_mask: None, + properties, + metrics: ExecutionPlanMetricsSet::new(), + } + } + + /// Construct a `MatchQueryExec` bound to an explicit, pre-resolved set of + /// FTS segments. Unlike [`Self::new`], `execute()` will not call + /// [`load_segments`] — it will search exactly the segments supplied here. + /// + /// Useful when a caller has already enumerated segments and wants to scope + /// this exec to a strict subset — for example, a distributed query that + /// routes per-segment work across hosts, where each per-host leaf should + /// only search its own assigned subset of the dataset's committed + /// segments. + pub fn new_with_segments( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segments: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_segments_and_document_granularity( + dataset, + query, + params, + prefilter_source, + segments, + document_granularity, + )) } - pub fn new( + pub fn new_with_segments_and_document_granularity( dataset: Arc, query: MatchQuery, params: FtsSearchParams, prefilter_source: PreFilterSource, + segments: Vec, + document_granularity: DocumentGranularity, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -284,48 +2624,64 @@ impl MatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: None, + prepared_query: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)), + overlay_block: None, + document_granularity, + schema, + external_mask: None, properties, metrics: ExecutionPlanMetricsSet::new(), } } - /// Construct a `MatchQueryExec` bound to an explicit, pre-resolved set of - /// FTS segments. Unlike [`Self::new`], `execute()` will not call - /// [`load_segments`] — it will search exactly the segments supplied here. + /// Construct a `MatchQueryExec` bound to an exact ordered set of committed + /// FTS segment UUIDs. /// - /// Useful when a caller has already enumerated segments and wants to scope - /// this exec to a strict subset — for example, a distributed query that - /// routes per-segment work across hosts, where each per-host leaf should - /// only search its own assigned subset of the dataset's committed - /// segments. - pub fn new_with_segments( + /// The UUIDs are resolved from this exec's dataset snapshot when the output + /// stream is polled. Duplicate UUIDs are removed while preserving their + /// first-occurrence order. Resolution fails if the list is empty or any UUID + /// is not committed for the query column. + pub fn new_with_segment_uuids( dataset: Arc, query: MatchQuery, params: FtsSearchParams, prefilter_source: PreFilterSource, - segments: Vec, - ) -> Self { + segment_uuids: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, )); let params = Self::effective_params(&query, params); - Self { + Ok(Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: Some(segments), + prepared_query: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::exact_uuids(segment_uuids), + overlay_block: None, + external_mask: None, + document_granularity, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), - } + }) } /// Override the BM25 scorer used by `execute()`. When set, the local @@ -339,9 +2695,45 @@ impl MatchQueryExec { /// routes per-segment work to multiple hosts and aggregates stats /// out-of-band, so each per-host leaf scores against the full corpus /// rather than its local segment subset. See [`build_global_bm25_scorer`] - /// for constructing one. + /// for constructing one. For a fuzzy query over an explicit segment + /// subset, use [`Self::with_prepared_query`] so the globally selected + /// vocabulary travels with the scorer. pub fn with_base_scorer(mut self, scorer: Arc) -> Self { self.base_scorer = Some(scorer); + self.prepared_query = None; + self + } + + /// Override local query preparation with one canonical vocabulary/scorer + /// pair built against the complete corpus. + /// + /// Distributed fuzzy callers must use this instead of + /// [`Self::with_base_scorer`], because worker-local expansion can select a + /// different capped vocabulary from the one used to build the scorer. + #[doc(hidden)] + pub fn with_prepared_query(mut self, query: Arc) -> Self { + self.prepared_query = Some(query); + self.base_scorer = None; + self + } + + pub(crate) fn with_shared_scorer(mut self, scorer: Arc) -> Self { + self.shared_scorer = Some(scorer); + self + } + + /// Exclude rows whose indexed text was superseded by a newer data overlay. + pub(crate) fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self { + self.overlay_block = Some(overlay_block); + self + } + + /// Restrict BM25 scoring to rows selected by an external row-address mask. + /// The mask is combined (logical AND) with the prefilter built by + /// `build_prefilter`, so top-k is computed over masked rows only. No-op when + /// `mask` is `None`. + pub fn with_external_mask(mut self, mask: Option>) -> Self { + self.external_mask = mask; self } @@ -366,7 +2758,16 @@ impl MatchQueryExec { } pub fn preset_segments(&self) -> Option<&[IndexMetadata]> { - self.preset_segments.as_deref() + self.segment_selection.preset_segments() + } + + /// Return the ordered segment UUIDs for an explicit selection. + /// + /// Returns `None` when this exec searches all committed segments. UUID-based + /// selections omit duplicates while preserving first-occurrence order. + /// Pre-resolved selections preserve the supplied metadata order. + pub fn explicit_segment_uuids(&self) -> Option> { + self.segment_selection.explicit_segment_uuids() } } @@ -375,16 +2776,8 @@ impl ExecutionPlan for MatchQueryExec { "MatchQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { - match &self.prefilter_source { - PreFilterSource::None => vec![], - PreFilterSource::FilteredRowIds(src) => vec![&src], - PreFilterSource::ScalarIndexQuery(src) => vec![&src], - } + self.prefilter_source.execution_plan().into_iter().collect() } fn required_input_distribution(&self) -> Vec { @@ -410,37 +2803,39 @@ impl ExecutionPlan for MatchQueryExec { Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source: PreFilterSource::None, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + prepared_query: self.prepared_query.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), + external_mask: self.external_mask.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } } 1 => { let src = children.pop().unwrap(); - let prefilter_source = match &self.prefilter_source { - PreFilterSource::FilteredRowIds(_) => { - PreFilterSource::FilteredRowIds(src.clone()) - } - PreFilterSource::ScalarIndexQuery(_) => { - PreFilterSource::ScalarIndexQuery(src.clone()) - } - PreFilterSource::None => { - return Err(DataFusionError::Internal( - "Unexpected prefilter source".to_string(), - )); - } - }; + let prefilter_source = self.prefilter_source.with_execution_plan(src)?; Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + prepared_query: self.prepared_query.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), + external_mask: self.external_mask.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -461,11 +2856,18 @@ impl ExecutionPlan for MatchQueryExec { context: Arc, ) -> DataFusionResult { let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); let params = self.params.clone(); let ds = self.dataset.clone(); let prefilter_source = self.prefilter_source.clone(); + let external_mask = self.external_mask.clone(); let preset_base_scorer = self.base_scorer.clone(); - let preset_segments = self.preset_segments.clone(); + let preset_prepared_query = self.prepared_query.clone(); + let shared_scorer = self.shared_scorer.clone(); + let segment_selection = self.segment_selection.clone(); + let overlay_block = self.overlay_block.clone(); + let document_granularity = self.document_granularity; + let schema = self.schema.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let column = query.column.ok_or(DataFusionError::Execution(format!( "column not set for MatchQuery {}", @@ -473,21 +2875,38 @@ impl ExecutionPlan for MatchQueryExec { )))?; let stream = stream::once(async move { let _timer = metrics.baseline_metrics.elapsed_compute().timer(); - let segments = match preset_segments { - Some(segments) => segments, - None => load_segments(&ds, &column) + let segments = segment_selection + .resolve( + &ds, + &column, + document_granularity, + &metrics.segment_bind_duration, + ) + .await?; + let scorer_only_fuzzy = preset_prepared_query.is_none() + && uses_fuzzy_expansion(params.fuzziness) + && (preset_base_scorer.is_some() || shared_scorer.is_some()); + let scorer_override_covers_all = if scorer_only_fuzzy { + segment_selection + .covers_all_committed(&ds, &column, document_granularity, &segments) .await? - .ok_or(DataFusionError::Execution(format!( - "No Inverted index found for column {}", - column, - )))?, + } else { + true }; - let _details = load_segment_details(&ds, &column, &segments).await?; let indices = open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?; - let mut pre_filter = - build_prefilter(context.clone(), partition, &prefilter_source, ds, &segments)?; + let mut pre_filter = build_prefilter( + context.clone(), + partition, + &prefilter_source, + ds, + &segments, + PreFilterMasks { + overlay_block, + external_mask, + }, + )?; let deleted_fragments = indices .iter() @@ -503,60 +2922,60 @@ impl ExecutionPlan for MatchQueryExec { metrics .record_parts_searched(indices.iter().map(|index| index.partition_count()).sum()); - let is_fuzzy = matches!(query.fuzziness, Some(n) if n != 0); let first_index = indices.first().ok_or(DataFusionError::Execution(format!( "FTS index for column {} has no segments", column )))?; - let mut tokenizer = match is_fuzzy { - false => first_index.tokenizer(), - true => { - let tokenizer = TextAnalyzer::from(SimpleTokenizer::default()); - match first_index.tokenizer().doc_type() { - DocType::Text => { - Box::new(TextTokenizer::new(tokenizer)) as Box - } - DocType::Json => { - Box::new(JsonTokenizer::new(tokenizer)) as Box - } - } - } - }; + let mut tokenizer = tokenizer_for_match_query(first_index, query.fuzziness); let tokens = collect_query_tokens(&query.terms, &mut tokenizer); - let base_scorer = match preset_base_scorer { - Some(scorer) => scorer, - None => Arc::new( - build_global_bm25_scorer(&indices, &tokens, ¶ms) - .boxed() - .await?, - ), + record_tokenized_query(&tokenized_query, &tokens); + let prepared = if let Some(prepared_query) = preset_prepared_query { + Arc::new(PreparedMatch { + query: prepared_query, + params: Arc::new(params), + operator: query.operator, + }) + } else { + let base_scorer = match (preset_base_scorer, shared_scorer) { + (Some(scorer), _) => Some(scorer), + (None, Some(shared_scorer)) => Some(shared_scorer.wait().await?), + (None, None) => None, + }; + if base_scorer.is_some() && scorer_only_fuzzy && !scorer_override_covers_all { + return Err(DataFusionError::Execution( + "fuzzy MatchQuery cannot use a scorer-only override; prepare the canonical vocabulary with prepare_bm25_query and pass it with with_prepared_query" + .to_string(), + )); + } + let builds_local_scorer = base_scorer.is_none(); + let scorer_start = std::time::Instant::now(); + let prepared = Arc::new( + PreparedMatch::new( + &indices, + tokens, + params, + query.operator, + metrics.as_ref(), + base_scorer, + ) + .await?, + ); + if builds_local_scorer { + metrics.record_scorer_build(scorer_start.elapsed()); + } + prepared }; pre_filter.wait_for_ready().await?; - let tokens = Arc::new(tokens); - let params = Arc::new(params); - let (doc_ids, mut scores) = search_segments( - &indices, - tokens, - params, - query.operator, - pre_filter, - metrics.clone(), - base_scorer, - ) - .await?; - scores.iter_mut().for_each(|s| { - *s *= query.boost; + let mut documents = + search_prepared_segments(&indices, prepared, pre_filter, metrics.clone(), None) + .await?; + documents.iter_mut().for_each(|document| { + document.score.0 *= query.boost; }); - metrics.baseline_metrics.record_output(doc_ids.len()); + metrics.baseline_metrics.record_output(documents.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(doc_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + let batch = scored_documents_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); @@ -579,22 +2998,66 @@ impl ExecutionPlan for MatchQueryExec { } } -/// Filters the input, removing rows that do not share tokens with the query +/// Filters the input according to a match query's token operator. #[derive(Debug)] pub struct FlatMatchFilterExec { dataset: Arc, input: Arc, query: MatchQuery, + tokenized_query: Arc>, params: FtsSearchParams, /// Optional pre-resolved segment list. See /// [`MatchQueryExec::new_with_segments`]. `FlatMatchFilterExec` only /// uses the first segment's tokenizer, but the full list is preserved so /// the field round-trips through `with_new_children`. preset_segments: Option>, + document_column: String, + resolved_field: Option, metrics: ExecutionPlanMetricsSet, } +struct FlatMatchFilterStreamOptions { + dataset: Arc, + query: MatchQuery, + tokenized_query: Arc>, + document_column: String, + preset_segments: Option>, + resolved_field: Option, + metrics_set: ExecutionPlanMetricsSet, +} + +fn document_matches_query( + text: &str, + tokenizer: &mut Box, + query_tokens: &Tokens, + operator: Operator, +) -> bool { + match operator { + Operator::Or => has_query_token(text, tokenizer, query_tokens), + Operator::And => { + let mut remaining_positions = (0..query_tokens.len()) + .map(|index| query_tokens.position(index)) + .collect::>(); + if remaining_positions.is_empty() { + return false; + } + let mut stream = tokenizer.token_stream_for_doc(text); + while let Some(token) = stream.next() { + for index in 0..query_tokens.len() { + if token.text == query_tokens.get_token(index) { + remaining_positions.remove(&query_tokens.position(index)); + } + } + if remaining_positions.is_empty() { + return true; + } + } + false + } + } +} + impl DisplayAs for FlatMatchFilterExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { match t { @@ -604,7 +3067,8 @@ impl DisplayAs for FlatMatchFilterExec { "FlatMatchFilter: column={}, query={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -612,7 +3076,8 @@ impl DisplayAs for FlatMatchFilterExec { "FlatMatchFilter\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } @@ -622,9 +3087,10 @@ impl FlatMatchFilterExec { async fn load_tokenizer( dataset: &Dataset, column: &str, + document_granularity: DocumentGranularity, metrics: &IndexMetrics, ) -> DataFusionResult> { - if let Some(segments) = load_segments(dataset, column).await? { + if let Some(segments) = load_segments(dataset, column, document_granularity).await? { let index_meta = segments.first().ok_or_else(|| { DataFusionError::Execution(format!( "FTS index for column {} has no segments", @@ -657,13 +3123,47 @@ impl FlatMatchFilterExec { dataset: Arc, query: MatchQuery, params: FtsSearchParams, + ) -> Self { + let document_column = query.column.clone().unwrap_or_default(); + Self::new_with_document_column(input, dataset, query, params, document_column) + } + + pub fn new_with_document_column( + input: Arc, + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + document_column: String, + ) -> Self { + Self { + dataset, + input, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + preset_segments: None, + document_column, + resolved_field: None, + metrics: ExecutionPlanMetricsSet::new(), + } + } + + pub(crate) fn new_with_resolved_field( + input: Arc, + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + resolved_field: ResolvedFtsField, ) -> Self { Self { dataset, input, query, + tokenized_query: Arc::new(OnceLock::new()), params, preset_segments: None, + document_column: resolved_field.root_column.clone(), + resolved_field: Some(resolved_field), metrics: ExecutionPlanMetricsSet::new(), } } @@ -678,12 +3178,16 @@ impl FlatMatchFilterExec { params: FtsSearchParams, segments: Vec, ) -> Self { + let document_column = query.column.clone().unwrap_or_default(); Self { dataset, input, query, + tokenized_query: Arc::new(OnceLock::new()), params, preset_segments: Some(segments), + document_column, + resolved_field: None, metrics: ExecutionPlanMetricsSet::new(), } } @@ -708,12 +3212,20 @@ impl FlatMatchFilterExec { text_col: &dyn Array, tokenizer: &mut Box, query_tokens: &Tokens, + operator: Operator, ) -> BooleanArray { let text_col = text_col.as_string::(); let mut predicate = BooleanBuilder::with_capacity(text_col.len()); for idx in 0..text_col.len() { - let value = text_col.value(idx); - predicate.append_value(has_query_token(value, tokenizer, query_tokens)); + predicate.append_value( + !text_col.is_null(idx) + && document_matches_query( + text_col.value(idx), + tokenizer, + query_tokens, + operator, + ), + ); } predicate.finish() } @@ -722,11 +3234,17 @@ impl FlatMatchFilterExec { input: SendableRecordBatchStream, partition: usize, schema: SchemaRef, - dataset: Arc, - query: MatchQuery, - preset_segments: Option>, - metrics_set: ExecutionPlanMetricsSet, + options: FlatMatchFilterStreamOptions, ) -> DataFusionResult { + let FlatMatchFilterStreamOptions { + dataset, + query, + tokenized_query, + document_column, + preset_segments, + resolved_field, + metrics_set, + } = options; let metrics = Arc::new(FtsIndexMetrics::new(&metrics_set, partition)); let column = query .column @@ -735,6 +3253,21 @@ impl FlatMatchFilterExec { "column not set for MatchQuery {}", query.terms )))?; + if uses_fuzzy_expansion(query.fuzziness) { + return Err(DataFusionError::NotImplemented(format!( + "Fuzzy MatchQuery is not supported when FTS is used as a post-filter: column={}, fuzziness={:?}", + column, query.fuzziness + ))); + } + let document_granularity = resolved_field + .as_ref() + .map(|resolved| resolved.document_granularity) + .or(query.document_granularity) + .ok_or_else(|| { + DataFusionError::Execution( + "MatchQuery document granularity was not resolved".to_string(), + ) + })?; let mut tokenizer = match preset_segments { Some(segments) => { Self::load_tokenizer_from_preset_segments( @@ -745,33 +3278,72 @@ impl FlatMatchFilterExec { ) .await? } - None => Self::load_tokenizer(&dataset, &column, &metrics.index_metrics).await?, + None => { + Self::load_tokenizer( + &dataset, + &column, + document_granularity, + &metrics.index_metrics, + ) + .await? + } }; let query_tokens = Arc::new(collect_query_tokens(&query.terms, &mut tokenizer)); + record_tokenized_query(&tokenized_query, &query_tokens); let baseline = BaselineMetrics::new(&metrics_set, partition); let elapsed_compute = baseline.elapsed_compute().clone(); let stream = input.then(move |batch_result| { - let column = column.clone(); + let column = document_column.clone(); let query_tokens = query_tokens.clone(); let mut tokenizer = tokenizer.box_clone(); let elapsed_compute = elapsed_compute.clone(); + let resolved_field = resolved_field.clone(); + let query_operator = query.operator; async move { let batch = batch_result?; let _t = elapsed_compute.timer(); + if let Some(resolved_field) = resolved_field { + let documents = resolved_field + .documents_from_batch(&batch) + .map_err(DataFusionError::from)?; + let mut matches = vec![false; batch.num_rows()]; + for document in documents { + if document_matches_query( + &document.text, + &mut tokenizer, + &query_tokens, + query_operator, + ) { + matches[document.row_index] = true; + } + } + let predicate = BooleanArray::from(matches); + return Ok(arrow::compute::filter_record_batch(&batch, &predicate)?); + } let text_column = batch.column_by_name(&column).ok_or_else(|| { DataFusionError::Execution(format!("Column {} not found in batch", column,)) })?; let predicate = match text_column.data_type() { DataType::Utf8 => { - Self::find_matches::(text_column, &mut tokenizer, &query_tokens) + Self::find_matches::( + text_column, + &mut tokenizer, + &query_tokens, + query_operator, + ) } DataType::LargeUtf8 => { - Self::find_matches::(text_column, &mut tokenizer, &query_tokens) + Self::find_matches::( + text_column, + &mut tokenizer, + &query_tokens, + query_operator, + ) } _ => { return Err(DataFusionError::Execution(format!( - "Column {} is not a string", + "FTS document column {} is not a string; nested List inputs must be expanded before filtering", column, ))); } @@ -795,10 +3367,6 @@ impl ExecutionPlan for FlatMatchFilterExec { "FlatMatchFilterExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { vec![&self.input] } @@ -820,8 +3388,11 @@ impl ExecutionPlan for FlatMatchFilterExec { dataset: self.dataset.clone(), input, query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), preset_segments: self.preset_segments.clone(), + document_column: self.document_column.clone(), + resolved_field: self.resolved_field.clone(), metrics: ExecutionPlanMetricsSet::new(), })) } @@ -838,10 +3409,15 @@ impl ExecutionPlan for FlatMatchFilterExec { input, partition, schema.clone(), - self.dataset.clone(), - self.query.clone(), - self.preset_segments.clone(), - self.metrics.clone(), + FlatMatchFilterStreamOptions { + dataset: self.dataset.clone(), + query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), + document_column: self.document_column.clone(), + preset_segments: self.preset_segments.clone(), + resolved_field: self.resolved_field.clone(), + metrics_set: self.metrics.clone(), + }, ); let stream = stream::once(stream_fut) .try_flatten() @@ -850,7 +3426,7 @@ impl ExecutionPlan for FlatMatchFilterExec { Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn partition_statistics(&self, partition: Option) -> DataFusionResult> { self.input.partition_statistics(partition) } @@ -872,14 +3448,20 @@ impl ExecutionPlan for FlatMatchFilterExec { pub struct FlatMatchQueryExec { dataset: Arc, query: MatchQuery, + tokenized_query: Arc>, params: FtsSearchParams, unindexed_input: Arc, /// Optional override for the BM25 scorer normally built locally inside /// `execute()`. See [`MatchQueryExec::with_base_scorer`]. base_scorer: Option>, + /// Publishes the scorer extended with this flat branch's documents. + shared_scorer: Option>, /// Optional pre-resolved segment list. See /// [`MatchQueryExec::new_with_segments`]. preset_segments: Option>, + document_granularity: DocumentGranularity, + document_column: String, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -894,7 +3476,8 @@ impl DisplayAs for FlatMatchQueryExec { "FlatMatchQuery: column={}, query={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -902,7 +3485,8 @@ impl DisplayAs for FlatMatchQueryExec { "FlatMatchQuery\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } @@ -914,9 +3498,32 @@ impl FlatMatchQueryExec { query: MatchQuery, params: FtsSearchParams, unindexed_input: Arc, + ) -> Result { + let document_column = query.column.clone().unwrap_or_default(); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_document_granularity( + dataset, + query, + params, + unindexed_input, + document_granularity, + document_column, + )) + } + + pub fn new_with_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + unindexed_input: Arc, + document_granularity: DocumentGranularity, + document_column: String, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Incremental, Boundedness::Bounded, @@ -924,10 +3531,15 @@ impl FlatMatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, unindexed_input, base_scorer: None, + shared_scorer: None, preset_segments: None, + document_granularity, + document_column, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -940,9 +3552,34 @@ impl FlatMatchQueryExec { params: FtsSearchParams, unindexed_input: Arc, segments: Vec, + ) -> Result { + let document_column = query.column.clone().unwrap_or_default(); + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("MatchQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_segments_and_document_granularity( + dataset, + query, + params, + unindexed_input, + segments, + document_granularity, + document_column, + )) + } + + pub fn new_with_segments_and_document_granularity( + dataset: Arc, + query: MatchQuery, + params: FtsSearchParams, + unindexed_input: Arc, + segments: Vec, + document_granularity: DocumentGranularity, + document_column: String, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Incremental, Boundedness::Bounded, @@ -950,10 +3587,15 @@ impl FlatMatchQueryExec { Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, unindexed_input, base_scorer: None, + shared_scorer: None, preset_segments: Some(segments), + document_granularity, + document_column, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -965,6 +3607,11 @@ impl FlatMatchQueryExec { self } + pub(crate) fn with_shared_scorer(mut self, scorer: Arc) -> Self { + self.shared_scorer = Some(scorer); + self + } + pub fn query(&self) -> &MatchQuery { &self.query } @@ -991,10 +3638,6 @@ impl ExecutionPlan for FlatMatchQueryExec { "FlatMatchQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { vec![&self.unindexed_input] } @@ -1020,10 +3663,15 @@ impl ExecutionPlan for FlatMatchQueryExec { Ok(Arc::new(Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), unindexed_input, base_scorer: self.base_scorer.clone(), + shared_scorer: self.shared_scorer.clone(), preset_segments: self.preset_segments.clone(), + document_granularity: self.document_granularity, + document_column: self.document_column.clone(), + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1036,12 +3684,17 @@ impl ExecutionPlan for FlatMatchQueryExec { context: Arc, ) -> DataFusionResult { let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); let ds = self.dataset.clone(); let preset_base_scorer = self.base_scorer.clone(); + let shared_scorer_producer = self.shared_scorer.clone().map(SharedFtsScorerProducer::new); let preset_segments = self.preset_segments.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let metrics_clone = metrics.clone(); let target_batch_size = context.session_config().batch_size(); + let document_granularity = self.document_granularity; + let document_column = self.document_column.clone(); + let phrase_slop = self.params.phrase_slop; // CPU time accumulator passed into `flat_bm25_search_stream_with_metrics` // so it can attribute the spawn_cpu tokenize work and synchronous @@ -1054,57 +3707,92 @@ impl ExecutionPlan for FlatMatchQueryExec { "column not set for MatchQuery {}", query.terms )))?; - let unindexed_input = - document_input(self.unindexed_input.execute(partition, context)?, &column)?; + let unindexed_input = document_input( + self.unindexed_input.execute(partition, context)?, + &document_column, + )?; let stream = stream::once(async move { - let segments = match preset_segments { - Some(segments) => Some(segments), - None => load_segments(&ds, &column).await?, - }; - let (tokenizer, base_scorer) = match segments { - Some(segments) => { - let _details = load_segment_details(&ds, &column, &segments).await?; - let indices = - open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?; - metrics.record_parts_searched( - indices.iter().map(|index| index.partition_count()).sum(), - ); - let first_index = indices.first().ok_or(DataFusionError::Execution( - format!("FTS index for column {} has no segments", column), - ))?; - let mut tokenizer = first_index.tokenizer(); - let base_scorer = match preset_base_scorer { - Some(scorer) => (*scorer).clone(), - None => { - let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer); - build_global_bm25_scorer( - &indices, - &query_tokens, - &FtsSearchParams::new(), - ) - .boxed() - .await? - } - }; - (tokenizer, Some(base_scorer)) - } - None => ( - default_text_tokenizer(), - preset_base_scorer.map(|s| (*s).clone()), - ), - }; + let shared_scorer_producer = shared_scorer_producer; + let result = async { + let segments = match preset_segments { + Some(segments) => Some(segments), + None => load_segments(&ds, &column, document_granularity).await?, + }; + let (tokenizer, base_scorer) = match segments { + Some(segments) => { + let _details = load_segment_details(&ds, &column, &segments).await?; + let indices = + open_fts_segments(&ds, &column, &segments, &metrics.index_metrics) + .await?; + metrics.record_parts_searched( + indices.iter().map(|index| index.partition_count()).sum(), + ); + let first_index = indices.first().ok_or(DataFusionError::Execution( + format!("FTS index for column {} has no segments", column), + ))?; + let mut tokenizer = first_index.tokenizer(); + let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer); + record_tokenized_query(&tokenized_query, &query_tokens); + let base_scorer = match preset_base_scorer { + Some(scorer) => (*scorer).clone(), + None => { + let scorer_start = std::time::Instant::now(); + let scorer = build_global_bm25_scorer( + &indices, + &query_tokens, + &FtsSearchParams::new(), + Some(metrics.as_ref()), + ) + .boxed() + .await?; + metrics.record_scorer_build(scorer_start.elapsed()); + scorer + } + }; + (tokenizer, Some(base_scorer)) + } + None => { + let mut tokenizer = default_text_tokenizer(); + let query_tokens = collect_query_tokens(&query.terms, &mut tokenizer); + record_tokenized_query(&tokenized_query, &query_tokens); + (tokenizer, preset_base_scorer.map(|s| (*s).clone())) + } + }; - flat_bm25_search_stream_with_metrics( - unindexed_input, - column, - query.terms, - tokenizer, - base_scorer, - target_batch_size, - Some(elapsed_compute), - ) - .await + flat_bm25_search_stream_with_options_and_scorer( + unindexed_input, + document_column, + query.terms, + tokenizer, + base_scorer, + FlatBm25SearchOptions { + target_batch_size, + elapsed_compute: Some(elapsed_compute), + operator: query.operator, + boost: query.boost, + document_granularity, + phrase_slop, + }, + ) + .await + } + .await; + + match result { + Ok((stream, scorer)) => { + if let Some(producer) = shared_scorer_producer { + producer.publish(scorer); + } + Ok(stream) + } + Err(error) => { + if let Some(producer) = shared_scorer_producer { + producer.publish_error(&error); + } + Err(error) + } + } }) .try_flatten() .map(move |batch| { @@ -1142,14 +3830,22 @@ impl ExecutionPlan for FlatMatchQueryExec { pub struct PhraseQueryExec { dataset: Arc, query: PhraseQuery, + tokenized_query: Arc>, params: FtsSearchParams, prefilter_source: PreFilterSource, /// Optional override for the BM25 scorer normally built locally inside /// `execute()`. See [`MatchQueryExec::with_base_scorer`]. base_scorer: Option>, - /// Optional pre-resolved segment list. See - /// [`MatchQueryExec::new_with_segments`]. - preset_segments: Option>, + /// Corpus-wide scorer published by the flat branch of a mixed search. + shared_scorer: Option>, + segment_selection: FtsSegmentSelection, + /// Rows whose indexed values were superseded by newer data overlays. + overlay_block: Option, + document_granularity: DocumentGranularity, + schema: SchemaRef, + /// Optional external row-address mask combined (logical AND) with the BM25 + /// prefilter so only masked rows are scored (see [`MatchQueryExec::with_external_mask`]). + external_mask: Option>, properties: Arc, metrics: ExecutionPlanMetricsSet, } @@ -1163,7 +3859,8 @@ impl DisplayAs for PhraseQueryExec { "PhraseQuery: column={}, query={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, ", ", f) } DisplayFormatType::TreeRender => { write!( @@ -1171,65 +3868,164 @@ impl DisplayAs for PhraseQueryExec { "PhraseQuery\ncolumn={}\nquery={}", self.query.column.as_deref().unwrap_or_default(), self.query.terms - ) + )?; + fmt_tokenized_query(&self.tokenized_query, "\n", f) } } } -} +} + +impl PhraseQueryExec { + pub fn new( + dataset: Arc, + query: PhraseQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("PhraseQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_document_granularity( + dataset, + query, + params, + prefilter_source, + document_granularity, + )) + } + + pub fn new_with_document_granularity( + dataset: Arc, + query: PhraseQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + document_granularity: DocumentGranularity, + ) -> Self { + let schema = fts_schema(document_granularity); + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )); + let params = params.with_phrase_slop(Some(query.slop)); + + Self { + dataset, + query, + tokenized_query: Arc::new(OnceLock::new()), + params, + prefilter_source, + base_scorer: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::AllCommitted, + overlay_block: None, + document_granularity, + schema, + external_mask: None, + properties, + metrics: ExecutionPlanMetricsSet::new(), + } + } + + /// See [`MatchQueryExec::new_with_segments`]. + pub fn new_with_segments( + dataset: Arc, + query: PhraseQuery, + params: FtsSearchParams, + prefilter_source: PreFilterSource, + segments: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("PhraseQuery document granularity must be resolved".to_string()) + })?; + Ok(Self::new_with_segments_and_document_granularity( + dataset, + query, + params, + prefilter_source, + segments, + document_granularity, + )) + } -impl PhraseQueryExec { - pub fn new( + pub fn new_with_segments_and_document_granularity( dataset: Arc, query: PhraseQuery, - mut params: FtsSearchParams, + params: FtsSearchParams, prefilter_source: PreFilterSource, + segments: Vec, + document_granularity: DocumentGranularity, ) -> Self { + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, )); - params = params.with_phrase_slop(Some(query.slop)); + let params = params.with_phrase_slop(Some(query.slop)); Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: None, + shared_scorer: None, + segment_selection: FtsSegmentSelection::ExactResolved(Arc::from(segments)), + overlay_block: None, + external_mask: None, + document_granularity, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } } - /// See [`MatchQueryExec::new_with_segments`]. - pub fn new_with_segments( + /// Construct a `PhraseQueryExec` bound to an exact ordered set of committed + /// FTS segment UUIDs. + /// + /// The UUIDs are resolved from this exec's dataset snapshot when the output + /// stream is polled. Duplicate UUIDs are removed while preserving their + /// first-occurrence order. Resolution fails if the list is empty or any UUID + /// is not committed for the query column. + pub fn new_with_segment_uuids( dataset: Arc, query: PhraseQuery, mut params: FtsSearchParams, prefilter_source: PreFilterSource, - segments: Vec, - ) -> Self { + segment_uuids: Vec, + ) -> Result { + let document_granularity = query.document_granularity.ok_or_else(|| { + Error::invalid_input("PhraseQuery document granularity must be resolved".to_string()) + })?; + let schema = fts_schema(document_granularity); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, )); params = params.with_phrase_slop(Some(query.slop)); - Self { + Ok(Self { dataset, query, + tokenized_query: Arc::new(OnceLock::new()), params, prefilter_source, base_scorer: None, - preset_segments: Some(segments), + shared_scorer: None, + segment_selection: FtsSegmentSelection::exact_uuids(segment_uuids), + overlay_block: None, + document_granularity, + schema, + external_mask: None, properties, metrics: ExecutionPlanMetricsSet::new(), - } + }) } /// Override the local BM25 scorer; see [`MatchQueryExec::with_base_scorer`]. @@ -1238,6 +4034,23 @@ impl PhraseQueryExec { self } + pub(crate) fn with_shared_scorer(mut self, scorer: Arc) -> Self { + self.shared_scorer = Some(scorer); + self + } + + /// Exclude rows whose indexed text was superseded by a newer data overlay. + pub(crate) fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self { + self.overlay_block = Some(overlay_block); + self + } + + /// See [`MatchQueryExec::with_external_mask`]. + pub fn with_external_mask(mut self, mask: Option>) -> Self { + self.external_mask = mask; + self + } + pub fn query(&self) -> &PhraseQuery { &self.query } @@ -1259,7 +4072,16 @@ impl PhraseQueryExec { } pub fn preset_segments(&self) -> Option<&[IndexMetadata]> { - self.preset_segments.as_deref() + self.segment_selection.preset_segments() + } + + /// Return the ordered segment UUIDs for an explicit selection. + /// + /// Returns `None` when this exec searches all committed segments. UUID-based + /// selections omit duplicates while preserving first-occurrence order. + /// Pre-resolved selections preserve the supplied metadata order. + pub fn explicit_segment_uuids(&self) -> Option> { + self.segment_selection.explicit_segment_uuids() } } @@ -1268,16 +4090,8 @@ impl ExecutionPlan for PhraseQueryExec { "PhraseQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { - match &self.prefilter_source { - PreFilterSource::None => vec![], - PreFilterSource::FilteredRowIds(src) => vec![&src], - PreFilterSource::ScalarIndexQuery(src) => vec![&src], - } + self.prefilter_source.execution_plan().into_iter().collect() } fn required_input_distribution(&self) -> Vec { @@ -1293,38 +4107,45 @@ impl ExecutionPlan for PhraseQueryExec { mut children: Vec>, ) -> DataFusionResult> { let plan = match children.len() { - 0 => Self { - dataset: self.dataset.clone(), - query: self.query.clone(), - params: self.params.clone(), - prefilter_source: PreFilterSource::None, - base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), - properties: self.properties.clone(), - metrics: ExecutionPlanMetricsSet::new(), - }, + 0 => { + if !matches!(self.prefilter_source, PreFilterSource::None) { + return Err(DataFusionError::Internal( + "Unexpected prefilter source".to_string(), + )); + } + Self { + dataset: self.dataset.clone(), + query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), + params: self.params.clone(), + prefilter_source: PreFilterSource::None, + base_scorer: self.base_scorer.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), + external_mask: self.external_mask.clone(), + properties: self.properties.clone(), + metrics: ExecutionPlanMetricsSet::new(), + } + } 1 => { let src = children.pop().unwrap(); - let prefilter_source = match &self.prefilter_source { - PreFilterSource::FilteredRowIds(_) => { - PreFilterSource::FilteredRowIds(src.clone()) - } - PreFilterSource::ScalarIndexQuery(_) => { - PreFilterSource::ScalarIndexQuery(src.clone()) - } - PreFilterSource::None => { - return Err(DataFusionError::Internal( - "Unexpected prefilter source".to_string(), - )); - } - }; + let prefilter_source = self.prefilter_source.with_execution_plan(src)?; Self { dataset: self.dataset.clone(), query: self.query.clone(), + tokenized_query: self.tokenized_query.clone(), params: self.params.clone(), prefilter_source, base_scorer: self.base_scorer.clone(), - preset_segments: self.preset_segments.clone(), + shared_scorer: self.shared_scorer.clone(), + segment_selection: self.segment_selection.clone(), + overlay_block: self.overlay_block.clone(), + document_granularity: self.document_granularity, + schema: self.schema.clone(), + external_mask: self.external_mask.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -1345,11 +4166,17 @@ impl ExecutionPlan for PhraseQueryExec { context: Arc, ) -> DataFusionResult { let query = self.query.clone(); + let tokenized_query = self.tokenized_query.clone(); let params = self.params.clone(); let ds = self.dataset.clone(); let prefilter_source = self.prefilter_source.clone(); + let external_mask = self.external_mask.clone(); let preset_base_scorer = self.base_scorer.clone(); - let preset_segments = self.preset_segments.clone(); + let shared_scorer = self.shared_scorer.clone(); + let segment_selection = self.segment_selection.clone(); + let overlay_block = self.overlay_block.clone(); + let document_granularity = self.document_granularity; + let schema = self.schema.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let stream = stream::once(async move { let _timer = metrics.baseline_metrics.elapsed_compute().timer(); @@ -1357,21 +4184,28 @@ impl ExecutionPlan for PhraseQueryExec { "column not set for PhraseQuery {}", query.terms )))?; - let segments = match preset_segments { - Some(segments) => segments, - None => load_segments(&ds, &column) - .await? - .ok_or(DataFusionError::Execution(format!( - "No Inverted index found for column {}", - column, - )))?, - }; - let _details = load_segment_details(&ds, &column, &segments).await?; + let segments = segment_selection + .resolve( + &ds, + &column, + document_granularity, + &metrics.segment_bind_duration, + ) + .await?; let indices = open_fts_segments(&ds, &column, &segments, &metrics.index_metrics).await?; - let mut pre_filter = - build_prefilter(context.clone(), partition, &prefilter_source, ds, &segments)?; + let mut pre_filter = build_prefilter( + context.clone(), + partition, + &prefilter_source, + ds, + &segments, + PreFilterMasks { + overlay_block, + external_mask, + }, + )?; let deleted_fragments = indices .iter() @@ -1393,19 +4227,31 @@ impl ExecutionPlan for PhraseQueryExec { )))?; let mut tokenizer = first_index.tokenizer(); let tokens = collect_query_tokens(&query.terms, &mut tokenizer); - let base_scorer = match preset_base_scorer { - Some(scorer) => scorer, - None => Arc::new( - build_global_bm25_scorer(&indices, &tokens, ¶ms) + record_tokenized_query(&tokenized_query, &tokens); + let base_scorer = match (preset_base_scorer, shared_scorer) { + (Some(scorer), _) => scorer, + (None, Some(shared_scorer)) => shared_scorer.wait().await?, + (None, None) => { + let scorer_start = std::time::Instant::now(); + let scorer = Arc::new( + build_global_bm25_scorer( + &indices, + &tokens, + ¶ms, + Some(metrics.as_ref()), + ) .boxed() .await?, - ), + ); + metrics.record_scorer_build(scorer_start.elapsed()); + scorer + } }; pre_filter.wait_for_ready().await?; let tokens = Arc::new(tokens); let params = Arc::new(params); - let (doc_ids, scores) = search_segments( + let documents = search_segments( &indices, tokens, params, @@ -1413,16 +4259,11 @@ impl ExecutionPlan for PhraseQueryExec { pre_filter, metrics.clone(), base_scorer, + None, ) .await?; - metrics.baseline_metrics.record_output(doc_ids.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(doc_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + metrics.baseline_metrics.record_output(documents.len()); + let batch = scored_documents_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -1450,6 +4291,7 @@ pub struct BoostQueryExec { params: FtsSearchParams, positive: Arc, negative: Arc, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -1483,8 +4325,9 @@ impl BoostQueryExec { positive: Arc, negative: Arc, ) -> Self { + let schema = positive.schema(); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -1494,6 +4337,7 @@ impl BoostQueryExec { params, positive, negative, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -1521,10 +4365,6 @@ impl ExecutionPlan for BoostQueryExec { "BoostQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { vec![&self.positive, &self.negative] } @@ -1555,6 +4395,7 @@ impl ExecutionPlan for BoostQueryExec { params: self.params.clone(), positive, negative, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1570,6 +4411,7 @@ impl ExecutionPlan for BoostQueryExec { let params = self.params.clone(); let positive = self.positive.execute(partition, context.clone())?; let negative = self.negative.execute(partition, context)?; + let schema = self.schema.clone(); let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); let stream = stream::once(async move { let positive = positive.try_collect::>().await?; @@ -1578,38 +4420,26 @@ impl ExecutionPlan for BoostQueryExec { let _timer = metrics.baseline_metrics.elapsed_compute().timer(); let mut res = HashMap::new(); for batch in positive { - let doc_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - - for (doc_id, score) in std::iter::zip(doc_ids, scores) { - res.insert(*doc_id, *score); + for (key, score) in batch_scored_document_keys(&batch)? { + res.insert(key, score); } } for batch in negative { - let doc_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - - for (doc_id, neg_score) in std::iter::zip(doc_ids, scores) { - if let Some(score) = res.get_mut(doc_id) { + for (key, neg_score) in batch_scored_document_keys(&batch)? { + if let Some(score) = res.get_mut(&key) { *score -= query.negative_boost * neg_score; } } } - let (doc_ids, scores): (Vec<_>, Vec<_>) = res + let documents = res .into_iter() - .sorted_unstable_by(|(_, a), (_, b)| b.total_cmp(a)) + .sorted_unstable_by(compare_scored_documents) .take(params.limit.unwrap_or(usize::MAX)) - .unzip(); - metrics.baseline_metrics.record_output(doc_ids.len()); + .collect::>(); + metrics.baseline_metrics.record_output(documents.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(doc_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + let batch = document_key_scores_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -1655,13 +4485,21 @@ pub enum BoolSlot { /// lookups). Returns `Result>>` so the /// `Must` slot's `None` case is naturally expressible. pub fn build_boolean_query_children( + slot: BoolSlot, + children: Vec>, +) -> Result>> { + build_boolean_query_children_with_schema(slot, children, FTS_SCHEMA.clone()) +} + +pub fn build_boolean_query_children_with_schema( slot: BoolSlot, mut children: Vec>, + schema: SchemaRef, ) -> Result>> { match slot { BoolSlot::Should | BoolSlot::MustNot => { if children.is_empty() { - Ok(Some(Arc::new(EmptyExec::new(FTS_SCHEMA.clone())))) + Ok(Some(Arc::new(EmptyExec::new(schema)))) } else if children.len() == 1 { Ok(Some(children.pop().unwrap())) } else { @@ -1676,13 +4514,20 @@ pub fn build_boolean_query_children( let mut joined: Option> = None; for plan in children { if let Some(left) = joined { + let mut on: Vec<(Arc, Arc)> = vec![( + Arc::new(Column::new_with_schema(ROW_ID, &schema)?), + Arc::new(Column::new_with_schema(ROW_ID, &schema)?), + )]; + if schema.field_with_name(DOC_INDEX_COL).is_ok() { + on.push(( + Arc::new(Column::new_with_schema(DOC_INDEX_COL, &schema)?), + Arc::new(Column::new_with_schema(DOC_INDEX_COL, &schema)?), + )); + } joined = Some(Arc::new(HashJoinExec::try_new( left, plan, - vec![( - Arc::new(Column::new_with_schema(ROW_ID, &FTS_SCHEMA)?), - Arc::new(Column::new_with_schema(ROW_ID, &FTS_SCHEMA)?), - )], + on, None, &datafusion_expr::JoinType::Inner, None, @@ -1706,6 +4551,7 @@ pub struct BooleanQueryExec { should: Arc, must: Option>, must_not: Arc, + schema: SchemaRef, properties: Arc, metrics: ExecutionPlanMetricsSet, @@ -1746,8 +4592,9 @@ impl BooleanQueryExec { must: Option>, must_not: Arc, ) -> Self { + let schema = should.schema(); let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(FTS_SCHEMA.clone()), + EquivalenceProperties::new(schema.clone()), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, @@ -1758,6 +4605,7 @@ impl BooleanQueryExec { must, should, must_not, + schema, properties, metrics: ExecutionPlanMetricsSet::new(), } @@ -1789,10 +4637,6 @@ impl ExecutionPlan for BooleanQueryExec { "BooleanQueryExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn children(&self) -> Vec<&Arc> { match &self.must { Some(must) => vec![&self.should, &self.must_not, must], @@ -1822,6 +4666,7 @@ impl ExecutionPlan for BooleanQueryExec { should, must: None, must_not: self.must_not.clone(), + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1835,6 +4680,7 @@ impl ExecutionPlan for BooleanQueryExec { should, must: None, must_not, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1849,6 +4695,7 @@ impl ExecutionPlan for BooleanQueryExec { should, must: Some(must), must_not, + schema: self.schema.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), })) @@ -1877,6 +4724,7 @@ impl ExecutionPlan for BooleanQueryExec { let mut should = self.should.execute(partition, context.clone())?; let mut must_not = self.must_not.execute(partition, context)?; let metrics = Arc::new(FtsIndexMetrics::new(&self.metrics, partition)); + let schema = self.schema.clone(); let stream = stream::once(async move { let elapsed_time = metrics.baseline_metrics.elapsed_compute(); @@ -1886,25 +4734,17 @@ impl ExecutionPlan for BooleanQueryExec { if let Some(mut must) = must { while let Some(batch) = must.try_next().await? { let _timer = elapsed_time.timer(); - let row_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - res.extend(std::iter::zip( - row_ids.iter().copied(), - scores.iter().copied(), - )); + res.extend(batch_scored_document_keys_sum_scores(&batch)?); } } // add the scores from the should clause while let Some(batch) = should.try_next().await? { let _timer = elapsed_time.timer(); - let row_ids = batch[ROW_ID].as_primitive::().values(); - let scores = batch[SCORE_COL].as_primitive::().values(); - - for (row_id, score) in std::iter::zip(row_ids, scores) { - let entry = res.entry(*row_id).and_modify(|e| *e += score); + for (key, score) in batch_scored_document_keys(&batch)? { + let entry = res.entry(key).and_modify(|value| *value += score); if !has_must { - entry.or_insert(*score); + entry.or_insert(score); } } } @@ -1912,9 +4752,8 @@ impl ExecutionPlan for BooleanQueryExec { // remove the results from the must_not clause while let Some(batch) = must_not.try_next().await? { let _timer = elapsed_time.timer(); - let row_ids = batch[ROW_ID].as_primitive::().values(); - for row_id in row_ids { - res.remove(row_id); + for key in batch_document_keys(&batch)? { + res.remove(&key); } } @@ -1936,19 +4775,13 @@ impl ExecutionPlan for BooleanQueryExec { // sort the results and take the top k let _timer = elapsed_time.timer(); - let (row_ids, scores): (Vec<_>, Vec<_>) = res + let documents = res .into_iter() - .sorted_unstable_by(|(_, a), (_, b)| b.total_cmp(a)) + .sorted_unstable_by(compare_scored_documents) .take(params.limit.unwrap_or(usize::MAX)) - .unzip(); - metrics.baseline_metrics.record_output(row_ids.len()); - let batch = RecordBatch::try_new( - FTS_SCHEMA.clone(), - vec![ - Arc::new(UInt64Array::from(row_ids)), - Arc::new(Float32Array::from(scores)), - ], - )?; + .collect::>(); + metrics.baseline_metrics.record_output(documents.len()); + let batch = document_key_scores_batch(schema, documents)?; Ok::<_, DataFusionError>(batch) }); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -1976,25 +4809,29 @@ mod tests { UInt64Array, }; use arrow_schema::DataType; + use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion::{execution::TaskContext, physical_plan::ExecutionPlan}; use futures::TryStreamExt; - use lance_core::ROW_ID; + use lance_core::{ROW_ID, utils::address::RowAddress}; use lance_datafusion::datagen::DatafusionDatagenExt; use lance_datafusion::exec::{ExecutionStatsCallback, ExecutionSummaryCounts}; use lance_datafusion::utils::PARTITIONS_SEARCHED_METRIC; use lance_datagen::{BatchCount, ByteCount, RowCount}; use lance_index::metrics::NoOpMetricsCollector; + use lance_index::scalar::inverted::builder::ScoredDoc; use lance_index::scalar::inverted::query::{ BooleanQuery, BoostQuery, FtsQuery, FtsSearchParams, MatchQuery, Occur, Operator, PhraseQuery, collect_query_tokens, has_query_token, }; use lance_index::scalar::inverted::{ - FTS_SCHEMA, InvertedIndex, Language, SCORE_COL, build_global_bm25_scorer, + DocumentGranularity, FTS_SCHEMA, InvertedIndex, Language, SCORE_COL, + build_global_bm25_scorer, prepare_bm25_query, }; use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams}; use lance_index::{IndexCriteria, IndexType}; use lance_table::format::IndexMetadata; + use uuid::Uuid; use crate::{ Dataset, @@ -2006,8 +4843,11 @@ mod tests { }; use super::{ - BoolSlot, BoostQueryExec, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, - PhraseQueryExec, build_boolean_query_children, open_fts_segments, + BoolSlot, BoostQueryExec, CompoundQueryExec, CrossColumnCompoundQueryExec, + FTS_SEGMENT_BIND_DURATION_METRIC, FlatMatchFilterExec, FlatMatchQueryExec, MatchQueryExec, + PhraseQueryExec, WAND_TIE_COMPLETION_BUDGET, WandExactnessCertificate, + build_boolean_query_children, classify_wand_exactness_certificate, default_text_tokenizer, + open_fts_segments, tokenizer_for_match_query, }; use crate::io::exec::utils::IndexMetrics; use datafusion::physical_plan::empty::EmptyExec; @@ -2033,6 +4873,315 @@ mod tests { } } + #[test] + fn test_wand_exactness_certificate_classification() { + let documents = |scores: &[f32]| { + scores + .iter() + .enumerate() + .rev() + .map(|(row_id, score)| ScoredDoc::new(row_id as u64, *score)) + .collect::>() + }; + + let mut exhaustive = documents(&[3.0, 2.0]); + assert_eq!( + classify_wand_exactness_certificate(&mut exhaustive, 3, 4), + WandExactnessCertificate::Exhaustive + ); + + let mut strict = documents(&[4.0, 3.0, 3.0, 1.0]); + assert_eq!( + classify_wand_exactness_certificate(&mut strict, 3, 4), + WandExactnessCertificate::Strict + ); + assert_eq!( + strict + .iter() + .map(|document| document.row_id) + .collect::>(), + vec![0, 1, 2, 3], + "ties wholly inside top-k must use row-id ordering without forcing fallback" + ); + + let mut ambiguous = documents(&[4.0, 3.0, 2.0, 2.0]); + assert_eq!( + classify_wand_exactness_certificate(&mut ambiguous, 3, 4), + WandExactnessCertificate::Ambiguous + ); + + let mut non_finite = documents(&[4.0, f32::INFINITY]); + assert_eq!( + classify_wand_exactness_certificate(&mut non_finite, 1, 2), + WandExactnessCertificate::Ambiguous + ); + + let mut zero_limit = documents(&[1.0]); + assert_eq!( + classify_wand_exactness_certificate(&mut zero_limit, 0, 1), + WandExactnessCertificate::Ambiguous + ); + + let mut reversed_segments = vec![ + ScoredDoc::new(99, 2.0), + ScoredDoc::new(50, 3.0), + ScoredDoc::new(1, 2.0), + ]; + assert_eq!( + classify_wand_exactness_certificate(&mut reversed_segments, 2, 4), + WandExactnessCertificate::Exhaustive + ); + assert_eq!( + reversed_segments + .iter() + .map(|document| document.row_id) + .collect::>(), + vec![50, 1, 99], + "completed ties must use final row-id order, not segment arrival order" + ); + + let completion_limit = 1 + WAND_TIE_COMPLETION_BUDGET + 1; + let mut at_budget = (0..=WAND_TIE_COMPLETION_BUDGET) + .rev() + .map(|row_id| ScoredDoc::new(row_id as u64, 2.0)) + .chain(std::iter::once(ScoredDoc::new(u64::MAX, 1.0))) + .collect::>(); + assert_eq!(at_budget.len(), completion_limit); + assert_eq!( + classify_wand_exactness_certificate(&mut at_budget, 1, completion_limit), + WandExactnessCertificate::Strict, + "the completion budget includes a slot for a strict lower-score guard" + ); + assert_eq!(at_budget[0].row_id, 0); + + let mut overflow = (0..completion_limit) + .rev() + .map(|row_id| ScoredDoc::new(row_id as u64, 2.0)) + .collect::>(); + assert_eq!( + classify_wand_exactness_certificate(&mut overflow, 1, completion_limit), + WandExactnessCertificate::Ambiguous, + "a full probe with no lower-score guard must replay exactly" + ); + } + + async fn create_segment_selection_fixture() -> (Arc, Vec, Vec) { + let mut dataset = lance_datagen::gen_batch() + .col( + "text", + lance_datagen::array::cycle_utf8_literals(&["quick brown fox"]), + ) + .col( + "other", + lance_datagen::array::cycle_utf8_literals(&["not indexed"]), + ) + .into_ram_dataset(FragmentCount::from(3), FragmentRowCount::from(2)) + .await + .unwrap(); + let fragment_ids = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect::>(); + assert_eq!(fragment_ids.len(), 3); + + let params = InvertedIndexParams::default().with_position(true); + let mut segments = Vec::with_capacity(fragment_ids.len()); + for fragment_id in &fragment_ids { + let mut builder = dataset + .create_index_builder(&["text"], IndexType::Inverted, ¶ms) + .name("segment_selection_fts".to_string()) + .fragments(vec![*fragment_id]); + segments.push(builder.execute_uncommitted().await.unwrap()); + } + dataset + .commit_existing_index_segments("segment_selection_fts", "text", segments.clone()) + .await + .unwrap(); + + let committed = crate::index::scalar::inverted::load_segments( + &dataset, + "text", + DocumentGranularity::Row, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(committed.len(), fragment_ids.len()); + (Arc::new(dataset), committed, fragment_ids) + } + + fn tokenized_query_index_params() -> InvertedIndexParams { + InvertedIndexParams::new("simple".to_string(), Language::English) + .with_position(true) + .lower_case(true) + .stem(false) + .remove_stop_words(true) + .ascii_folding(false) + } + + async fn create_tokenized_query_fixture(with_unindexed_append: bool) -> Dataset { + let mut dataset = lance_datagen::gen_batch() + .col( + "text", + lance_datagen::array::cycle_utf8_literals(&["first and second"]), + ) + .into_ram_dataset(FragmentCount::from(1), FragmentRowCount::from(2)) + .await + .unwrap(); + dataset + .create_index( + &["text"], + IndexType::Inverted, + None, + &tokenized_query_index_params(), + true, + ) + .await + .unwrap(); + + if with_unindexed_append { + let appended = lance_datagen::gen_batch() + .col( + "text", + lance_datagen::array::cycle_utf8_literals(&["first and second"]), + ) + .into_reader_rows(RowCount::from(2), BatchCount::from(1)); + dataset.append(appended, None).await.unwrap(); + } + dataset + } + + fn find_plan_line<'a>(analysis: &'a str, node: &str) -> &'a str { + analysis + .lines() + .find(|line| line.trim_start().starts_with(node)) + .unwrap_or_else(|| panic!("{node} missing from plan:\n{analysis}")) + } + + fn segment_uuid_for_fragment(segments: &[IndexMetadata], fragment_id: u32) -> Uuid { + segments + .iter() + .find(|segment| { + segment + .fragment_bitmap + .as_ref() + .is_some_and(|fragments| fragments.contains(fragment_id)) + }) + .map(|segment| segment.uuid) + .unwrap() + } + + fn expected_row_ids(fragment_ids: &[u32]) -> Vec { + let mut row_ids = fragment_ids + .iter() + .flat_map(|fragment_id| { + (0..2).map(|offset| u64::from(RowAddress::new_from_parts(*fragment_id, offset))) + }) + .collect::>(); + row_ids.sort_unstable(); + row_ids + } + + async fn execute_results(plan: &dyn ExecutionPlan) -> DataFusionResult> { + let batches: Vec = plan + .execute(0, Arc::new(TaskContext::default()))? + .try_collect() + .await?; + let mut results = Vec::new(); + for batch in batches { + let row_ids = batch[ROW_ID] + .as_any() + .downcast_ref::() + .unwrap(); + let scores = batch[SCORE_COL] + .as_any() + .downcast_ref::() + .unwrap(); + results.extend( + row_ids + .values() + .iter() + .copied() + .zip(scores.values().iter().copied()), + ); + } + results.sort_by_key(|(row_id, _)| *row_id); + Ok(results) + } + + async fn execute_row_ids(plan: &dyn ExecutionPlan) -> DataFusionResult> { + Ok(execute_results(plan) + .await? + .into_iter() + .map(|(row_id, _)| row_id) + .collect()) + } + + fn metric_value(plan: &dyn ExecutionPlan, name: &str) -> usize { + plan.metrics() + .unwrap() + .iter() + .find(|metric| metric.value().name() == name) + .unwrap() + .value() + .as_usize() + } + + fn assert_execution_error(error: DataFusionError, expected_message: &str) { + assert!( + matches!(&error, DataFusionError::Execution(_)), + "expected execution error, got {error:?}" + ); + assert!( + error.to_string().contains(expected_message), + "expected error containing {expected_message:?}, got {error}" + ); + } + + #[test] + fn document_match_filter_respects_document_boundary() { + let mut tokenizer = default_text_tokenizer(); + let query_tokens = collect_query_tokens("alpha", &mut tokenizer); + assert!(super::document_matches_query( + "alpha beta", + &mut tokenizer, + &query_tokens, + Operator::Or, + )); + + let mut tokenizer = default_text_tokenizer(); + let query_tokens = collect_query_tokens("alpha beta", &mut tokenizer); + assert!(!super::document_matches_query( + "alpha", + &mut tokenizer, + &query_tokens, + Operator::And, + )); + assert!(super::document_matches_query( + "alpha beta", + &mut tokenizer, + &query_tokens, + Operator::And, + )); + } + + #[tokio::test] + async fn shared_fts_scorer_reports_cancelled_producer() { + let scorer = Arc::new(super::SharedFtsScorer::new()); + let producer = super::SharedFtsScorerProducer::new(scorer.clone()); + drop(producer); + + let error = tokio::time::timeout(std::time::Duration::from_secs(1), scorer.wait()) + .await + .expect("cancelled producer must wake scorer waiters") + .unwrap_err(); + assert!( + error.to_string().contains("producer was cancelled"), + "{error}" + ); + } + #[test] fn execute_without_context() { // These tests ensure we can create nodes and call execute without a tokio Runtime @@ -2041,10 +5190,13 @@ mod tests { let fixture = NoContextTestFixture::new(); let match_query = MatchQueryExec::new( Arc::new(fixture.dataset.clone()), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), PreFilterSource::None, - ); + ) + .unwrap(); match_query .execute(0, Arc::new(TaskContext::default())) .unwrap(); @@ -2060,10 +5212,13 @@ mod tests { let flat_match_query = FlatMatchQueryExec::new( Arc::new(fixture.dataset.clone()), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), flat_input, - ); + ) + .unwrap(); flat_match_query .execute(0, Arc::new(TaskContext::default())) .unwrap(); @@ -2072,10 +5227,12 @@ mod tests { let phrase_query = PhraseQueryExec::new( Arc::new(fixture.dataset.clone()), - PhraseQuery::new("blah".to_string()), + PhraseQuery::new("blah".to_string()) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::new().with_phrase_slop(Some(0)), PreFilterSource::None, - ); + ) + .unwrap(); phrase_query .execute(0, Arc::new(TaskContext::default())) .unwrap(); @@ -2084,17 +5241,23 @@ mod tests { let boost_input_one = MatchQueryExec::new( Arc::new(fixture.dataset.clone()), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), PreFilterSource::None, - ); + ) + .unwrap(); let boost_input_two = MatchQueryExec::new( Arc::new(fixture.dataset), - MatchQuery::new("blah".to_string()).with_column(Some("text".to_string())), + MatchQuery::new("blah".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row), FtsSearchParams::default(), PreFilterSource::None, - ); + ) + .unwrap(); let boost_query = BoostQueryExec::new( BoostQuery::new( @@ -2129,8 +5292,12 @@ mod tests { let text_col = LargeStringArray::from(vec!["hello world", "no match here", "say hello there"]); - let result = - FlatMatchFilterExec::find_matches::(&text_col, &mut tokenizer, &query_tokens); + let result = FlatMatchFilterExec::find_matches::( + &text_col, + &mut tokenizer, + &query_tokens, + Operator::Or, + ); assert_eq!(result.len(), 3); assert!(result.value(0), "expected match in 'hello world'"); @@ -2182,14 +5349,24 @@ mod tests { .unwrap(); let metrics = IndexMetrics::new(&ExecutionPlanMetricsSet::new(), 0); - let mut tokenizer = FlatMatchFilterExec::load_tokenizer(&dataset, "text", &metrics) - .await - .unwrap(); + let mut tokenizer = FlatMatchFilterExec::load_tokenizer( + &dataset, + "text", + DocumentGranularity::Row, + &metrics, + ) + .await + .unwrap(); let query_tokens = collect_query_tokens("hello", &mut tokenizer); - let mut tokenizer = FlatMatchFilterExec::load_tokenizer(&dataset, "text", &metrics) - .await - .unwrap(); + let mut tokenizer = FlatMatchFilterExec::load_tokenizer( + &dataset, + "text", + DocumentGranularity::Row, + &metrics, + ) + .await + .unwrap(); assert!(has_query_token("hello", &mut tokenizer, &query_tokens)); assert!( !has_query_token("HELLO", &mut tokenizer, &query_tokens), @@ -2260,6 +5437,92 @@ mod tests { assert!(analysis.contains(PARTITIONS_SEARCHED_METRIC)); } + #[tokio::test] + async fn test_analyze_plan_shows_indexed_and_flat_match_tokens() { + let dataset = create_tokenized_query_fixture(true).await; + let query = MatchQuery::new("FIRST and SECOND".to_string()) + .with_column(Some("text".to_string())) + .with_operator(Operator::And); + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + + let explained = scanner.explain_plan(false).await.unwrap(); + assert!( + !explained.contains("tokenized_query="), + "explain_plan should not claim runtime tokenization: {explained}" + ); + + let analysis = scanner.analyze_plan().await.unwrap(); + let expected = r#"tokenized_query=[("first", 0), ("second", 2)]"#; + assert!( + find_plan_line(&analysis, "MatchQuery:").contains(expected), + "indexed MatchQuery is missing token positions:\n{analysis}" + ); + assert!( + find_plan_line(&analysis, "FlatMatchQuery:").contains(expected), + "flat MatchQuery is missing token positions:\n{analysis}" + ); + } + + #[tokio::test] + async fn test_analyze_plan_shows_indexed_and_flat_phrase_tokens() { + let dataset = create_tokenized_query_fixture(true).await; + let query = + PhraseQuery::new("FIRST and SECOND".to_string()).with_column(Some("text".to_string())); + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + + let analysis = scanner.analyze_plan().await.unwrap(); + let expected = r#"tokenized_query=[("first", 0), ("second", 2)]"#; + assert!( + find_plan_line(&analysis, "PhraseQuery:").contains(expected), + "indexed PhraseQuery is missing token positions:\n{analysis}" + ); + assert!( + find_plan_line(&analysis, "FlatMatchQuery:").contains(expected), + "flat phrase path is missing token positions:\n{analysis}" + ); + } + + #[tokio::test] + async fn test_analyze_plan_shows_compound_leaf_tokens() { + let dataset = create_tokenized_query_fixture(false).await; + let query = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("FIRST and SECOND".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ( + Occur::Must, + PhraseQuery::new("SECOND FIRST".to_string()) + .with_column(Some("text".to_string())) + .with_slop(2) + .into(), + ), + ]); + let mut scanner = dataset.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + + let analysis = scanner.analyze_plan().await.unwrap(); + let compound = find_plan_line(&analysis, "CompoundFtsScorer:"); + assert!( + compound.contains(r#"Match(column="text", tokens=[("first", 0), ("second", 2)])"#), + "compound Match leaf is missing token positions: {compound}" + ); + assert!( + compound.contains(r#"Phrase(column="text", tokens=[("second", 0), ("first", 1)])"#), + "compound Phrase leaf is missing token positions: {compound}" + ); + } + #[tokio::test] async fn test_boolean_query_parts_searched_metrics() { let mut dataset = lance_datagen::gen_batch() @@ -2282,49 +5545,434 @@ mod tests { .await .unwrap(); - let index_meta = dataset - .load_scalar_index(IndexCriteria::default().for_column("text").supports_fts()) - .await - .unwrap() - .unwrap(); - let index = dataset - .open_generic_index("text", &index_meta.uuid, &NoOpMetricsCollector) - .await - .unwrap(); - let inverted_index = index.as_any().downcast_ref::().unwrap(); - let expected_parts = inverted_index.partition_count(); + let index_meta = dataset + .load_scalar_index(IndexCriteria::default().for_column("text").supports_fts()) + .await + .unwrap() + .unwrap(); + let index = dataset + .open_generic_index("text", &index_meta.uuid, &NoOpMetricsCollector) + .await + .unwrap(); + let inverted_index = index.as_any().downcast_ref::().unwrap(); + let expected_parts = inverted_index.partition_count(); + + let query = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("hello".to_string()) + .with_operator(Operator::And) + .into(), + ), + ( + Occur::Must, + MatchQuery::new("lance".to_string()) + .with_operator(Operator::And) + .into(), + ), + ]); + let expected_total = expected_parts * 2; + + let mut scanner = dataset.scan(); + scanner + .project(&["text"]) + .unwrap() + .with_row_id() + .full_text_search(FullTextSearchQuery::new_query(query.into())) + .unwrap(); + let analysis = scanner.analyze_plan().await.unwrap(); + let compound_line = analysis + .lines() + .find(|line| line.contains("CompoundFtsScorer")) + .unwrap(); + assert!( + compound_line.contains(&format!("{PARTITIONS_SEARCHED_METRIC}={expected_total}")), + "compound FTS scorer metrics missing partitions_searched: {compound_line}" + ); + } + + #[tokio::test] + async fn test_match_query_exec_segment_selection() { + let (dataset, segments, fragment_ids) = create_segment_selection_fixture().await; + let query = MatchQuery::new("quick".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row); + let params = FtsSearchParams::default().with_limit(Some(20)); + let committed_uuids = segments + .iter() + .map(|segment| segment.uuid) + .collect::>(); + + let all_committed = MatchQueryExec::new( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + ) + .unwrap(); + assert!(all_committed.preset_segments().is_none()); + assert!(all_committed.explicit_segment_uuids().is_none()); + let all_results = execute_results(&all_committed).await.unwrap(); + assert_eq!( + all_results + .iter() + .map(|(row_id, _)| *row_id) + .collect::>(), + expected_row_ids(&fragment_ids) + ); + assert_eq!( + metric_value(&all_committed, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let exact_resolved = MatchQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .unwrap(); + assert_eq!(exact_resolved.preset_segments(), Some(segments.as_slice())); + assert_eq!( + exact_resolved.explicit_segment_uuids(), + Some(committed_uuids.clone()) + ); + assert_eq!(execute_results(&exact_resolved).await.unwrap(), all_results); + assert_eq!( + metric_value(&exact_resolved, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let mismatched_granularity = MatchQueryExec::new_with_segments_and_document_granularity( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + segments.clone(), + DocumentGranularity::ListElement, + ); + assert_execution_error( + execute_row_ids(&mismatched_granularity).await.unwrap_err(), + "use Row document granularity", + ); + + let selected_fragment = fragment_ids[1]; + let selected_uuid = segment_uuid_for_fragment(&segments, selected_fragment); + let unpolled = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + drop( + unpolled + .execute(0, Arc::new(TaskContext::default())) + .unwrap(), + ); + assert_eq!( + metric_value(&unpolled, FTS_SEGMENT_BIND_DURATION_METRIC), + 0, + "UUID binding should not start until the output stream is polled" + ); + + let exact_uuids = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert!(exact_uuids.preset_segments().is_none()); + assert_eq!( + exact_uuids.explicit_segment_uuids(), + Some(vec![selected_uuid]) + ); + assert_eq!( + execute_row_ids(&exact_uuids).await.unwrap(), + expected_row_ids(&[selected_fragment]) + ); + assert!( + metric_value(&exact_uuids, FTS_SEGMENT_BIND_DURATION_METRIC) > 0, + "successful UUID binding should record a duration" + ); + + let input_uuids = vec![ + segment_uuid_for_fragment(&segments, fragment_ids[2]), + segment_uuid_for_fragment(&segments, fragment_ids[0]), + segment_uuid_for_fragment(&segments, fragment_ids[2]), + ]; + let deduplicated_uuids = input_uuids[..2].to_vec(); + let ordered_plan = Arc::new( + MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + input_uuids, + ) + .unwrap(), + ) + .with_new_children(vec![]) + .unwrap(); + let rewritten = ordered_plan.downcast_ref::().unwrap(); + assert_eq!( + rewritten.explicit_segment_uuids(), + Some(deduplicated_uuids.clone()) + ); + assert_eq!( + execute_row_ids(rewritten).await.unwrap(), + expected_row_ids(&[fragment_ids[2], fragment_ids[0]]) + ); + let resolver_metrics_set = ExecutionPlanMetricsSet::new(); + let resolver_metrics = super::FtsIndexMetrics::new(&resolver_metrics_set, 0); + let resolved = rewritten + .segment_selection + .resolve( + &dataset, + "text", + DocumentGranularity::Row, + &resolver_metrics.segment_bind_duration, + ) + .await + .unwrap(); + assert_eq!( + resolved + .iter() + .map(|segment| segment.uuid) + .collect::>(), + deduplicated_uuids + ); + + let empty = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&empty).await.unwrap_err(), + "requires at least one segment UUID", + ); + + let missing_uuid = Uuid::new_v4(); + let missing = MatchQueryExec::new_with_segment_uuids( + dataset.clone(), + query, + params.clone(), + PreFilterSource::None, + vec![missing_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&missing).await.unwrap_err(), + &missing_uuid.to_string(), + ); + + let wrong_column = MatchQueryExec::new_with_segment_uuids( + dataset, + MatchQuery::new("quick".to_string()) + .with_column(Some("other".to_string())) + .with_document_granularity(DocumentGranularity::Row), + params, + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&wrong_column).await.unwrap_err(), + "no Inverted index found", + ); + } + + #[tokio::test] + async fn test_phrase_query_exec_segment_selection() { + let (dataset, segments, fragment_ids) = create_segment_selection_fixture().await; + let query = PhraseQuery::new("quick brown".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row); + let params = FtsSearchParams::default().with_limit(Some(20)); + let committed_uuids = segments + .iter() + .map(|segment| segment.uuid) + .collect::>(); + + let all_committed = PhraseQueryExec::new( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + ) + .unwrap(); + assert!(all_committed.preset_segments().is_none()); + assert!(all_committed.explicit_segment_uuids().is_none()); + let all_results = execute_results(&all_committed).await.unwrap(); + assert_eq!( + all_results + .iter() + .map(|(row_id, _)| *row_id) + .collect::>(), + expected_row_ids(&fragment_ids) + ); + assert_eq!( + metric_value(&all_committed, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let exact_resolved = PhraseQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .unwrap(); + assert_eq!(exact_resolved.preset_segments(), Some(segments.as_slice())); + assert_eq!( + exact_resolved.explicit_segment_uuids(), + Some(committed_uuids) + ); + assert_eq!(execute_results(&exact_resolved).await.unwrap(), all_results); + assert_eq!( + metric_value(&exact_resolved, FTS_SEGMENT_BIND_DURATION_METRIC), + 0 + ); + + let selected_fragment = fragment_ids[1]; + let selected_uuid = segment_uuid_for_fragment(&segments, selected_fragment); + let unpolled = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + drop( + unpolled + .execute(0, Arc::new(TaskContext::default())) + .unwrap(), + ); + assert_eq!( + metric_value(&unpolled, FTS_SEGMENT_BIND_DURATION_METRIC), + 0, + "UUID binding should not start until the output stream is polled" + ); + + let exact_uuids = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert!(exact_uuids.preset_segments().is_none()); + assert_eq!( + exact_uuids.explicit_segment_uuids(), + Some(vec![selected_uuid]) + ); + assert_eq!( + execute_row_ids(&exact_uuids).await.unwrap(), + expected_row_ids(&[selected_fragment]) + ); + assert!( + metric_value(&exact_uuids, FTS_SEGMENT_BIND_DURATION_METRIC) > 0, + "successful UUID binding should record a duration" + ); + + let input_uuids = vec![ + segment_uuid_for_fragment(&segments, fragment_ids[2]), + segment_uuid_for_fragment(&segments, fragment_ids[0]), + segment_uuid_for_fragment(&segments, fragment_ids[2]), + ]; + let deduplicated_uuids = input_uuids[..2].to_vec(); + let ordered_plan = Arc::new( + PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + input_uuids, + ) + .unwrap(), + ) + .with_new_children(vec![]) + .unwrap(); + let rewritten = ordered_plan.downcast_ref::().unwrap(); + assert_eq!( + rewritten.explicit_segment_uuids(), + Some(deduplicated_uuids.clone()) + ); + assert_eq!( + execute_row_ids(rewritten).await.unwrap(), + expected_row_ids(&[fragment_ids[2], fragment_ids[0]]) + ); + let resolver_metrics_set = ExecutionPlanMetricsSet::new(); + let resolver_metrics = super::FtsIndexMetrics::new(&resolver_metrics_set, 0); + let resolved = rewritten + .segment_selection + .resolve( + &dataset, + "text", + DocumentGranularity::Row, + &resolver_metrics.segment_bind_duration, + ) + .await + .unwrap(); + assert_eq!( + resolved + .iter() + .map(|segment| segment.uuid) + .collect::>(), + deduplicated_uuids + ); + + let empty = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&empty).await.unwrap_err(), + "requires at least one segment UUID", + ); - let query = BooleanQuery::new([ - ( - Occur::Should, - MatchQuery::new("hello".to_string()) - .with_operator(Operator::And) - .into(), - ), - ( - Occur::Must, - MatchQuery::new("lance".to_string()) - .with_operator(Operator::And) - .into(), - ), - ]); - let expected_total = expected_parts * 2; + let missing_uuid = Uuid::new_v4(); + let missing = PhraseQueryExec::new_with_segment_uuids( + dataset.clone(), + query, + params.clone(), + PreFilterSource::None, + vec![missing_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&missing).await.unwrap_err(), + &missing_uuid.to_string(), + ); - let mut scanner = dataset.scan(); - scanner - .project(&["text"]) - .unwrap() - .with_row_id() - .full_text_search(FullTextSearchQuery::new_query(query.into())) - .unwrap(); - let analysis = scanner.analyze_plan().await.unwrap(); - let boolean_line = analysis - .lines() - .find(|line| line.contains("BooleanQuery")) - .unwrap(); - assert!( - boolean_line.contains(&format!("{PARTITIONS_SEARCHED_METRIC}={expected_total}")), - "BooleanQuery metrics missing partitions_searched: {boolean_line}" + let wrong_column = PhraseQueryExec::new_with_segment_uuids( + dataset, + PhraseQuery::new("quick brown".to_string()) + .with_column(Some("other".to_string())) + .with_document_granularity(DocumentGranularity::Row), + params, + PreFilterSource::None, + vec![selected_uuid], + ) + .unwrap(); + assert_execution_error( + execute_row_ids(&wrong_column).await.unwrap_err(), + "no Inverted index found", ); } @@ -2354,8 +6002,8 @@ mod tests { ( "text", Arc::new(StringArray::from(vec![ - Some("alpha beta"), - Some("gamma lance"), + Some("lancd alpha"), + Some("lancd lance"), ])) as ArrayRef, ), ]) @@ -2379,7 +6027,7 @@ mod tests { .with_position(false) .lower_case(true) .stem(false) - .remove_stop_words(false) + .remove_stop_words(true) .ascii_folding(false) .max_token_length(None); let fragment_ids = ds @@ -2411,7 +6059,9 @@ mod tests { ); let dataset = Arc::new(ds); - let query = MatchQuery::new("lance".to_string()).with_column(Some("text".to_string())); + let query = MatchQuery::new("lance".to_string()) + .with_column(Some("text".to_string())) + .with_document_granularity(DocumentGranularity::Row); let search_params = FtsSearchParams::default().with_limit(Some(10)); // Baseline: the existing path that builds the global scorer locally. @@ -2420,7 +6070,8 @@ mod tests { query.clone(), search_params.clone(), PreFilterSource::None, - ); + ) + .unwrap(); let baseline_batches: Vec = baseline_exec .execute(0, Arc::new(TaskContext::default())) .unwrap() @@ -2435,10 +6086,14 @@ mod tests { // Override: build the global scorer manually via the public helper, then // construct the exec with the preset segments and the preset scorer. - let preset_segments = crate::index::scalar::inverted::load_segments(&dataset, "text") - .await - .unwrap() - .expect("FTS index just created"); + let preset_segments = crate::index::scalar::inverted::load_segments( + &dataset, + "text", + DocumentGranularity::Row, + ) + .await + .unwrap() + .expect("FTS index just created"); let metrics_set = ExecutionPlanMetricsSet::new(); let metrics = IndexMetrics::new(&metrics_set, 0); let indices = open_fts_segments(&dataset, "text", &preset_segments, &metrics) @@ -2449,10 +6104,50 @@ mod tests { "expected >= 2 segments to exercise global IDF, got {}", indices.len() ); + + let mut auto_tokenizer = tokenizer_for_match_query(&indices[0], None); + let auto_tokens = collect_query_tokens("THE LANCE", &mut auto_tokenizer); + assert_eq!(auto_tokens.len(), 1); + assert_eq!(auto_tokens.get_token(0), "lance"); + let mut explicit_fuzzy_tokenizer = tokenizer_for_match_query(&indices[0], Some(1)); + let explicit_fuzzy_tokens = + collect_query_tokens("THE LANCE", &mut explicit_fuzzy_tokenizer); + assert_eq!(explicit_fuzzy_tokens.len(), 2); + assert_eq!(explicit_fuzzy_tokens.get_token(0), "THE"); + assert_eq!(explicit_fuzzy_tokens.get_token(1), "LANCE"); + + let auto_query = |terms: &str| { + MatchQuery::new(terms.to_owned()) + .with_column(Some("text".to_owned())) + .with_fuzziness(None) + .with_document_granularity(DocumentGranularity::Row) + }; + let lowercase_auto_exec = MatchQueryExec::new( + dataset.clone(), + auto_query("lance"), + search_params.clone(), + PreFilterSource::None, + ) + .unwrap(); + let lowercase_auto_results = execute_results(&lowercase_auto_exec).await.unwrap(); + assert!(!lowercase_auto_results.is_empty()); + let normalized_auto_exec = MatchQueryExec::new( + dataset.clone(), + auto_query("THE LANCE"), + search_params.clone(), + PreFilterSource::None, + ) + .unwrap(); + assert_eq!( + execute_results(&normalized_auto_exec).await.unwrap(), + lowercase_auto_results, + "AUTO fuzzy Match must preserve index lowercase and stop-word analysis" + ); + let mut tokenizer = indices[0].tokenizer(); let tokens = collect_query_tokens(&query.terms, &mut tokenizer); let global_scorer = Arc::new( - build_global_bm25_scorer(&indices, &tokens, &search_params) + build_global_bm25_scorer(&indices, &tokens, &search_params, None) .await .unwrap(), ); @@ -2462,8 +6157,9 @@ mod tests { query.clone(), search_params.clone(), PreFilterSource::None, - preset_segments, + preset_segments.clone(), ) + .unwrap() .with_base_scorer(global_scorer); let override_batches: Vec = override_exec .execute(0, Arc::new(TaskContext::default())) @@ -2506,6 +6202,147 @@ mod tests { ); } + // A distributed fuzzy subset must receive the canonical vocabulary + // together with its scorer. A scorer-only override cannot reproduce a + // globally capped rewrite from worker-local segment vocabularies. + let fuzzy_query = MatchQuery::new("lancx".to_string()) + .with_column(Some("text".to_string())) + .with_fuzziness(Some(1)) + .with_max_expansions(1) + .with_document_granularity(DocumentGranularity::Row); + let fuzzy_params = search_params + .clone() + .with_fuzziness(Some(1)) + .with_max_expansions(1); + let mut tokenizer = tokenizer_for_match_query(&indices[0], fuzzy_query.fuzziness); + let fuzzy_tokens = collect_query_tokens(&fuzzy_query.terms, &mut tokenizer); + let prepared = Arc::new( + prepare_bm25_query(&indices, fuzzy_tokens, &fuzzy_params, None, None) + .await + .unwrap(), + ); + assert_eq!(prepared.tokens().len(), 1); + assert_eq!(prepared.tokens().get_token(0), "lancd"); + + let prepared_full_exec = MatchQueryExec::new_with_segments( + dataset.clone(), + fuzzy_query.clone(), + search_params.clone(), + PreFilterSource::None, + preset_segments.clone(), + ) + .unwrap() + .with_prepared_query(prepared.clone()); + let prepared_full_results = execute_row_ids(&prepared_full_exec).await.unwrap(); + assert_eq!(prepared_full_results.len(), 2); + + let all_committed_scorer_exec = MatchQueryExec::new( + dataset.clone(), + fuzzy_query.clone(), + search_params.clone(), + PreFilterSource::None, + ) + .unwrap() + .with_base_scorer(prepared.scorer().clone()); + assert_eq!( + execute_row_ids(&all_committed_scorer_exec).await.unwrap(), + prepared_full_results + ); + + let explicit_full_scorer_exec = MatchQueryExec::new_with_segments( + dataset.clone(), + fuzzy_query.clone(), + search_params.clone(), + PreFilterSource::None, + preset_segments.clone(), + ) + .unwrap() + .with_base_scorer(prepared.scorer().clone()); + assert_eq!( + execute_row_ids(&explicit_full_scorer_exec).await.unwrap(), + prepared_full_results + ); + + let subset_exec = MatchQueryExec::new_with_segments( + dataset.clone(), + fuzzy_query.clone(), + search_params.clone(), + PreFilterSource::None, + vec![preset_segments[0].clone()], + ) + .unwrap() + .with_prepared_query(prepared.clone()); + assert!(execute_row_ids(&subset_exec).await.unwrap().is_empty()); + + let scorer_only_exec = MatchQueryExec::new_with_segments( + dataset.clone(), + fuzzy_query.clone(), + search_params.clone(), + PreFilterSource::None, + vec![preset_segments[0].clone()], + ) + .unwrap() + .with_base_scorer(prepared.scorer().clone()); + assert_execution_error( + execute_row_ids(&scorer_only_exec).await.unwrap_err(), + "fuzzy MatchQuery cannot use a scorer-only override", + ); + + let compound_query = FtsQuery::Match(fuzzy_query.clone()); + let compound_subset = CompoundQueryExec::new_with_segments( + dataset.clone(), + compound_query.clone(), + search_params.clone(), + PreFilterSource::None, + vec![preset_segments[0].clone()], + ) + .with_prepared_match(prepared.clone()); + assert!(execute_row_ids(&compound_subset).await.unwrap().is_empty()); + + let compound_scorer_only_subset = CompoundQueryExec::new_with_segments( + dataset.clone(), + compound_query.clone(), + search_params.clone(), + PreFilterSource::None, + vec![preset_segments[0].clone()], + ) + .with_base_scorer(prepared.scorer().clone()); + assert_execution_error( + execute_row_ids(&compound_scorer_only_subset) + .await + .unwrap_err(), + "fuzzy CompoundQueryExec cannot use a scorer-only override over a segment subset", + ); + + let compound_full_scorer = CompoundQueryExec::new_with_segments( + dataset.clone(), + compound_query.clone(), + search_params.clone(), + PreFilterSource::None, + preset_segments.clone(), + ) + .with_base_scorer(prepared.scorer().clone()); + assert_eq!( + execute_row_ids(&compound_full_scorer).await.unwrap(), + prepared_full_results + ); + + // With limit=1 the two globally selected `lancd` documents tie. The + // WAND probe is ambiguous, so bounded tie completion must retain the + // same prepared vocabulary instead of rewriting against this exec's + // segments. + let compound_wand_replay = CompoundQueryExec::new_with_segments( + dataset, + compound_query, + search_params.with_limit(Some(1)), + PreFilterSource::None, + preset_segments, + ) + .with_prepared_match(prepared); + assert_eq!( + execute_row_ids(&compound_wand_replay).await.unwrap().len(), + 1 + ); // Locally-bound helper: collect (row_id, score) pairs sorted by score desc. fn concat_score_batches(batches: &[RecordBatch]) -> Vec<(u64, f32)> { let mut out: Vec<(u64, f32)> = Vec::new(); @@ -2532,6 +6369,196 @@ mod tests { } } + #[tokio::test] + async fn test_compound_query_exec_validates_base_scorer() { + let (dataset, segments, _) = create_segment_selection_fixture().await; + let search_params = FtsSearchParams::default().with_limit(Some(10)); + let metrics_set = ExecutionPlanMetricsSet::new(); + let metrics = IndexMetrics::new(&metrics_set, 0); + let indices = open_fts_segments(&dataset, "text", &segments, &metrics) + .await + .unwrap(); + + let query: FtsQuery = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("quick".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ( + Occur::Should, + MatchQuery::new("brown".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ]) + .into(); + + let baseline = CompoundQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + search_params.clone(), + PreFilterSource::None, + segments.clone(), + ); + let baseline_results = execute_results(&baseline).await.unwrap(); + + let mut tokenizer = indices[0].tokenizer(); + let complete_tokens = collect_query_tokens("quick brown", &mut tokenizer); + let complete_scorer = Arc::new( + build_global_bm25_scorer(&indices, &complete_tokens, &search_params, None) + .await + .unwrap(), + ); + let complete_override = CompoundQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + search_params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .with_base_scorer(complete_scorer); + assert_eq!( + execute_results(&complete_override).await.unwrap(), + baseline_results + ); + + let mut tokenizer = indices[0].tokenizer(); + let incomplete_tokens = collect_query_tokens("quick", &mut tokenizer); + let incomplete_scorer = Arc::new( + build_global_bm25_scorer(&indices, &incomplete_tokens, &search_params, None) + .await + .unwrap(), + ); + let incomplete_override = CompoundQueryExec::new_with_segments( + dataset.clone(), + query, + search_params.clone(), + PreFilterSource::None, + segments.clone(), + ) + .with_base_scorer(incomplete_scorer); + + let error = execute_results(&incomplete_override).await.unwrap_err(); + assert!( + error + .to_string() + .contains("injected BM25 scorer is missing compound FTS token 'brown'"), + "unexpected incomplete-scorer error: {error}" + ); + + let mut tokenizer = indices[0].tokenizer(); + let brown_tokens = collect_query_tokens("brown", &mut tokenizer); + let scorer_without_fuzzy_expansion = Arc::new( + build_global_bm25_scorer(&indices, &brown_tokens, &search_params, None) + .await + .unwrap(), + ); + let fuzzy_query = BooleanQuery::new([ + ( + Occur::Should, + MatchQuery::new("quik".to_string()) + .with_column(Some("text".to_string())) + .with_fuzziness(Some(1)) + .into(), + ), + ( + Occur::Should, + MatchQuery::new("brown".to_string()) + .with_column(Some("text".to_string())) + .into(), + ), + ]); + let fuzzy_override = CompoundQueryExec::new_with_segments( + dataset, + fuzzy_query.into(), + search_params, + PreFilterSource::None, + segments, + ) + .with_base_scorer(scorer_without_fuzzy_expansion); + let error = execute_results(&fuzzy_override).await.unwrap_err(); + assert!( + error + .to_string() + .contains("injected BM25 scorer is missing compound FTS token 'quick'"), + "unexpected fuzzy-scorer error: {error}" + ); + } + + #[tokio::test] + async fn test_cross_column_compound_exec_validates_constructor_inputs() { + let (dataset, segments, _) = create_segment_selection_fixture().await; + let query: FtsQuery = BooleanQuery::new([ + ( + Occur::Must, + MatchQuery::new("quick".to_string()) + .with_column(Some("title".to_string())) + .into(), + ), + ( + Occur::MustNot, + MatchQuery::new("blocked".to_string()) + .with_column(Some("body".to_string())) + .into(), + ), + ]) + .into(); + let params = FtsSearchParams::default().with_limit(Some(10)); + + let error = CrossColumnCompoundQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + params.clone(), + PreFilterSource::None, + vec![("title".to_string(), segments.clone())], + ) + .unwrap_err(); + assert!( + error.to_string().contains(r#"missing=["body"]"#), + "unexpected missing-column error: {error}" + ); + + let error = CrossColumnCompoundQueryExec::new_with_segments( + dataset.clone(), + query.clone(), + FtsSearchParams::default(), + PreFilterSource::None, + vec![ + ("title".to_string(), segments.clone()), + ("body".to_string(), segments.clone()), + ], + ) + .unwrap_err(); + assert!( + error + .to_string() + .contains("requires a bounded result limit"), + "unexpected unbounded-query error: {error}" + ); + + let exec = CrossColumnCompoundQueryExec::new_with_segments( + dataset, + query, + params, + PreFilterSource::None, + vec![ + ("title".to_string(), segments.clone()), + ("body".to_string(), segments), + ], + ) + .unwrap(); + let display = format!( + "{}", + datafusion::physical_plan::displayable(&exec).one_line() + ); + assert!( + display.contains("CrossColumnCompoundFtsScorer:"), + "unexpected display name: {display}" + ); + } + fn empty_fts_child() -> Arc { Arc::new(EmptyExec::new(FTS_SCHEMA.clone())) } @@ -2542,7 +6569,7 @@ mod tests { .unwrap() .expect("Should slot always returns Some"); assert!( - plan.as_any().downcast_ref::().is_some(), + plan.downcast_ref::().is_some(), "expected EmptyExec for empty Should slot, got {plan:?}" ); } @@ -2570,12 +6597,10 @@ mod tests { .unwrap() .expect("Should slot always returns Some"); let repartition = plan - .as_any() .downcast_ref::() .expect("multi-child Should should be wrapped in RepartitionExec"); let inner = repartition .input() - .as_any() .downcast_ref::() .expect("RepartitionExec should wrap a UnionExec"); assert_eq!(inner.children().len(), 2); @@ -2614,7 +6639,7 @@ mod tests { // there are N-1 joins. let mut joins = 0usize; let mut current: Arc = plan; - while let Some(join) = current.clone().as_any().downcast_ref::() { + while let Some(join) = current.clone().downcast_ref::() { joins += 1; current = join.children()[0].clone(); } @@ -2630,12 +6655,10 @@ mod tests { .unwrap() .expect("MustNot slot always returns Some"); let repartition = plan - .as_any() .downcast_ref::() .expect("multi-child MustNot should be wrapped in RepartitionExec"); let inner = repartition .input() - .as_any() .downcast_ref::() .expect("RepartitionExec should wrap a UnionExec"); assert_eq!(inner.children().len(), 2); diff --git a/rust/lance/src/io/exec/knn.rs b/rust/lance/src/io/exec/knn.rs index f6332ada94e..e075a787f3f 100644 --- a/rust/lance/src/io/exec/knn.rs +++ b/rust/lance/src/io/exec/knn.rs @@ -3,14 +3,13 @@ #[cfg(test)] use lance_core::utils::row_addr_remap::RowAddrRemap; -use std::any::Any; use std::cmp::Ordering as CmpOrdering; use std::collections::{BinaryHeap, HashMap, HashSet}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, LazyLock, Mutex}; use std::time::Instant; -use arrow::array::{Float32Builder, Int32Builder}; +use arrow::array::{Float32Builder, Int32Builder, UInt64Builder}; use arrow::datatypes::{Float32Type, UInt32Type, UInt64Type}; use arrow_array::{Array, Float32Array, UInt32Array, UInt64Array}; use arrow_array::{ @@ -57,7 +56,9 @@ use lance_index::vector::{ }; use lance_linalg::distance::DistanceType; use lance_linalg::kernels::normalize_arrow; +use lance_select::{RowAddrMask, RowAddrTreeMap}; use lance_table::format::IndexMetadata; +use roaring::RoaringBitmap; use tokio::sync::Notify; use uuid::Uuid; @@ -68,6 +69,7 @@ use crate::index::vector::utils::{get_vector_type, validate_distance_type_for}; use crate::{Error, Result}; use lance_arrow::*; +use super::row_addr_mask::MaskAndLoader; use super::utils::{ FilteredRowIdsToPrefilter, IndexMetrics, InstrumentedRecordBatchStreamAdapter, PreFilterSource, SelectionVectorToPrefilter, @@ -124,6 +126,58 @@ async fn find_partitions_on_cpu( .map_err(|e| DataFusionError::Execution(format!("Failed to find partitions: {}", e))) } +/// Per-query IVF partition rankings: for each query, its probed-partition ids +/// and the corresponding centroid distances, in query order. +type BatchPartitionRankings = (Vec>, Vec>); + +/// Rank every query vector in a batch against the IVF centroids on the CPU +/// runtime. +/// +/// [`VectorIndex::find_partitions`] is pure CPU work, and a wide batch over a +/// large centroid set multiplies it enough to monopolize a Tokio worker and +/// stall unrelated async progress. Dispatch the whole ranking loop as a single +/// `spawn_cpu` job -- mirroring the single-query [`find_partitions_on_cpu`] -- +/// so it stays off the async executor threads. +/// +/// `query.key` holds all `query_count` vectors concatenated and `dim` is the +/// per-vector width. Returns each query's probed-partition list and the +/// corresponding centroid distances, in query order. +async fn find_partitions_batch_on_cpu( + index: Arc, + query: Query, + query_count: usize, + dim: usize, +) -> DataFusionResult { + spawn_cpu(move || -> Result { + let mut partitions_per_query = Vec::with_capacity(query_count); + let mut dists_per_query = Vec::with_capacity(query_count); + for query_index in 0..query_count { + let mut single_query = query.clone(); + single_query.key = query.key.slice(query_index * dim, dim); + // Probe a fixed number of partitions per query. The scanner only + // routes here when `minimum_nprobes == maximum_nprobes` and + // `minimum_nprobes > 0` (see `Scanner::batch_index_search_supported`), + // so this is exactly what the single-query path would search -- no + // adaptive `early_pruning` floor or late-search expansion applies, + // making the batch result identical to repeated single-query search. + // No clamp is needed: the gate rejects `nprobes(0)` (which the + // single-query path treats as "probe nothing") rather than silently + // searching one partition here. + debug_assert!( + single_query.minimum_nprobes > 0, + "batch node reached with nprobes(0); the scanner gate should have fallen back" + ); + single_query.maximum_nprobes = Some(single_query.minimum_nprobes); + let (partitions, q_c_dists) = index.find_partitions(&single_query)?; + partitions_per_query.push(Arc::new(partitions)); + dists_per_query.push(Arc::new(q_c_dists)); + } + Ok((partitions_per_query, dists_per_query)) + }) + .await + .map_err(|e| DataFusionError::Execution(format!("Failed to find partitions: {e}"))) +} + fn normalize_query_for_index(index: &dyn VectorIndex, query: Query) -> DataFusionResult { if index.metric_type() != DistanceType::Cosine { return Ok(query); @@ -137,6 +191,37 @@ fn normalize_query_for_index(index: &dyn VectorIndex, query: Query) -> DataFusio Ok(query) } +/// Normalize a batch query's concatenated key for a cosine index. +/// +/// `query.key` holds `query_count` vectors of length `dim` concatenated, so each +/// vector must be normalized **independently** — normalizing the whole buffer +/// would divide every vector by a single global norm that depends on the other +/// queries in the batch, corrupting per-query cosine distances. Returns the +/// query unchanged for non-cosine metrics. +fn normalize_batch_query_for_index( + index: &dyn VectorIndex, + mut query: Query, + query_count: usize, + dim: usize, +) -> DataFusionResult { + if index.metric_type() != DistanceType::Cosine { + return Ok(query); + } + + let normalized: Vec = (0..query_count) + .map(|i| { + normalize_arrow(&query.key.slice(i * dim, dim)) + .map(|(key, _)| key) + .map_err(|e| DataFusionError::Execution(format!("Failed to normalize query: {e}"))) + }) + .collect::>()?; + let refs: Vec<&dyn Array> = normalized.iter().map(|a| a.as_ref()).collect(); + query.key = arrow_select::concat::concat(&refs).map_err(|e| { + DataFusionError::Execution(format!("Failed to concat normalized query: {e}")) + })?; + Ok(query) +} + /// [ExecutionPlan] compute vector distance from a query vector. /// /// Preconditions: @@ -829,10 +914,6 @@ impl ExecutionPlan for KNNVectorDistanceExec { "KNNVectorDistanceExec" } - fn as_any(&self) -> &dyn Any { - self - } - /// Flat KNN inherits the schema from input node, and add one distance column. fn schema(&self) -> arrow_schema::SchemaRef { self.output_schema.clone() @@ -948,7 +1029,7 @@ impl ExecutionPlan for KNNVectorDistanceExec { Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) } - fn partition_statistics(&self, partition: Option) -> DataFusionResult { + fn partition_statistics(&self, partition: Option) -> DataFusionResult> { let inner_stats = self.input.partition_statistics(partition)?; let input_schema = self.input.schema(); let input_stats_by_name = inner_stats @@ -985,11 +1066,11 @@ impl ExecutionPlan for KNNVectorDistanceExec { } }) .collect::>(); - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows: inner_stats.num_rows, column_statistics, ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { @@ -1104,11 +1185,58 @@ pub static KNN_PARTITION_SCHEMA: LazyLock = LazyLock::new(|| { ])) }); +/// Build the shared [`DatasetPreFilter`] for an ANN search node, executing the +/// prefilter source (if any) for this partition. Used by both the single-query +/// [`ANNIvfSubIndexExec`] and the batch [`ANNIvfBatchExec`] so the prefilter is +/// wired identically (and, for a batch, built once and shared across queries). +/// +/// `overlay_block`, when `Some`, excludes rows whose index entries may be stale +/// due to a newer data overlay (see [`DatasetPreFilter::with_overlay_block`]). +fn build_dataset_prefilter( + dataset: Arc, + indices: &[IndexMetadata], + prefilter_source: &PreFilterSource, + partition: usize, + context: Arc, + overlay_block: Option, + external_mask: Option>, +) -> DataFusionResult> { + let prefilter_loader = match prefilter_source { + PreFilterSource::FilteredRowIds(src_node) => { + let stream = src_node.execute(partition, context)?; + Some(Box::new(FilteredRowIdsToPrefilter(stream)) as Box) + } + PreFilterSource::ScalarIndexQuery(src_node) => { + let stream = src_node.execute(partition, context)?; + Some(Box::new(SelectionVectorToPrefilter(stream)) as Box) + } + PreFilterSource::None => None, + }; + // AND the external row-address mask into whatever the filter produced. + let prefilter_loader = match external_mask { + Some(mask) => { + Some(Box::new(MaskAndLoader::new(mask, prefilter_loader)) as Box) + } + None => prefilter_loader, + }; + let mut pre_filter = DatasetPreFilter::new(dataset, indices, prefilter_loader); + if let Some(overlay_block) = overlay_block { + pre_filter = pre_filter.with_overlay_block(overlay_block); + } + Ok(Arc::new(pre_filter)) +} + +/// Create a new ANN execution node. `overlay_block`, when `Some`, excludes rows whose index +/// entries may be stale due to a newer data overlay (see [`ANNIvfSubIndexExec::with_overlay_block`]). +/// `external_mask`, when `Some`, additionally restricts the scan to a caller-supplied +/// allow/block set (see [`ANNIvfSubIndexExec::with_external_mask`]). pub fn new_knn_exec( dataset: Arc, indices: &[IndexMetadata], query: &Query, prefilter_source: PreFilterSource, + overlay_block: Option, + external_mask: Option>, ) -> Result> { let ivf_node = ANNIvfPartitionExec::try_new( dataset.clone(), @@ -1116,13 +1244,19 @@ pub fn new_knn_exec( query.clone(), )?; - let sub_index = ANNIvfSubIndexExec::try_new( + let mut sub_index = ANNIvfSubIndexExec::try_new( Arc::new(ivf_node), dataset, indices.to_vec(), query.clone(), prefilter_source, )?; + if let Some(overlay_block) = overlay_block { + sub_index = sub_index.with_overlay_block(overlay_block); + } + if external_mask.is_some() { + sub_index = sub_index.with_external_mask(external_mask); + } Ok(Arc::new(sub_index)) } @@ -1225,10 +1359,6 @@ impl ExecutionPlan for ANNIvfPartitionExec { "ANNIVFPartitionExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { KNN_PARTITION_SCHEMA.clone() } @@ -1237,11 +1367,11 @@ impl ExecutionPlan for ANNIvfPartitionExec { &self.properties } - fn partition_statistics(&self, _partition: Option) -> DataFusionResult { - Ok(Statistics { + fn partition_statistics(&self, _partition: Option) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact(self.query.minimum_nprobes), ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { @@ -1386,6 +1516,14 @@ pub struct ANNIvfSubIndexExec { /// Prefiltering input prefilter_source: PreFilterSource, + /// Row addresses whose index entries are stale due to a newer data overlay. Blocked from + /// index results at execution time via [`DatasetPreFilter::with_overlay_block`]. + overlay_block: Option, + + /// Optional external row-address allow/block mask, combined with the + /// prefilter using logical AND. + external_mask: Option>, + /// Datafusion Plan Properties properties: Arc, @@ -1418,11 +1556,27 @@ impl ANNIvfSubIndexExec { indices, query, prefilter_source, + overlay_block: None, + external_mask: None, properties, metrics: ExecutionPlanMetricsSet::new(), }) } + /// Block stale row addresses (see the `overlay_block` field) from index results. + pub fn with_overlay_block(mut self, overlay_block: RowAddrMask) -> Self { + self.overlay_block = Some(overlay_block); + self + } + + /// Restrict the ANN search to a caller-supplied row-address allow/block set. + /// Intersected with the prefilter, so top-k is computed over surviving rows + /// rather than filtered afterwards. No-op when `mask` is `None`. + pub fn with_external_mask(mut self, mask: Option>) -> Self { + self.external_mask = mask; + self + } + /// Returns a reference to the vector query. pub fn query(&self) -> &Query { &self.query @@ -1533,6 +1687,82 @@ impl ANNIvfEarlySearchResults { struct LatePartitionSearchControl { state: Arc, max_results: usize, + seg_mask: Option>, +} + +/// A query prefilter restricted to the fragments owned by one physical index segment. +/// +/// The shared dataset prefilter covers the union of every segment. When an in-place +/// update removes a fragment from an older segment's metadata, this additional mask +/// keeps that segment's stale physical rows out of its local top-k. +struct SegmentPreFilter { + base: Arc, + ownership_mask: Arc, + final_mask: Mutex>>, +} + +impl SegmentPreFilter { + fn new(base: Arc, ownership_mask: Arc) -> Self { + Self { + base, + ownership_mask, + final_mask: Mutex::new(None), + } + } +} + +#[async_trait::async_trait] +impl PreFilter for SegmentPreFilter { + async fn wait_for_ready(&self) -> Result<()> { + self.base.wait_for_ready().await?; + let mut final_mask = self.final_mask.lock().unwrap(); + final_mask.get_or_insert_with(|| { + Arc::new(self.base.mask().as_ref().clone() & self.ownership_mask.as_ref().clone()) + }); + Ok(()) + } + + fn is_empty(&self) -> bool { + false + } + + fn needs_partition_row_ids(&self) -> bool { + self.base.is_empty() + } + + fn is_empty_for(&self, rows: &RowAddrTreeMap) -> bool { + self.base.is_empty() && self.ownership_mask.selects_all(rows) + } + + fn mask(&self) -> Arc { + self.final_mask + .lock() + .unwrap() + .as_ref() + .expect("mask called without call to wait_for_ready") + .clone() + } + + fn filter_row_ids<'a>(&self, row_ids: Box + 'a>) -> Vec { + self.mask().selected_indices(row_ids) + } +} + +async fn prefilter_for_segment( + dataset: Arc, + index: &IndexMetadata, + base: Arc, +) -> Result> { + let Some(owned_fragments) = index.fragment_bitmap.clone() else { + return Ok(base); + }; + let Some(ownership_mask) = + DatasetPreFilter::create_restricted_deletion_mask(dataset, owned_fragments) + else { + return Ok(base); + }; + let ownership_mask = ownership_mask.await?; + Ok(Arc::new(SegmentPreFilter::new(base, ownership_mask))) } impl PartitionSearchControl for LatePartitionSearchControl { @@ -1541,10 +1771,59 @@ impl PartitionSearchControl for LatePartitionSearchControl { } fn record_batch(&self, batch: &RecordBatch) { - self.state.record_late_batch(batch.num_rows()); + // The batch this sees is the raw partition result; the stream applies the + // segment restriction afterwards, so only the rows that survive it may count + // towards the shared budget. + let num_rows = match self.seg_mask.as_ref() { + Some(seg_mask) => num_rows_in_segment(batch, seg_mask), + None => batch.num_rows(), + }; + self.state.record_late_batch(num_rows); } } +/// How many of `batch`'s rows belong to fragments the segment still owns. +fn num_rows_in_segment(batch: &RecordBatch, seg_mask: &RowAddrMask) -> usize { + batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .filter(|&&id| seg_mask.selected(id)) + .count() +} + +/// Drop the rows of `batch` whose fragment the segment no longer owns. +/// +/// A segment's index file can still hold rows for fragments that were pruned from its +/// `fragment_bitmap`, for example after an in-place column update. Once a newer delta +/// owns such a fragment, those rows must not reach the query, and they must not reach +/// the shared search accounting either: a stale row that is counted and only dropped +/// later can satisfy the `k` budget on its own, so the segment that owns the fresh +/// copy stops probing and the query returns fewer than `k` current rows. +fn restrict_to_segment( + batch: RecordBatch, + seg_mask: Option<&RowAddrMask>, +) -> DataFusionResult { + let Some(seg_mask) = seg_mask else { + return Ok(batch); + }; + if batch.num_rows() == 0 { + return Ok(batch); + } + let keep = BooleanArray::from_iter( + batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .map(|&id| Some(seg_mask.selected(id))), + ); + if keep.false_count() == 0 { + return Ok(batch); + } + arrow::compute::filter_record_batch(&batch, &keep) + .map_err(|e| DataFusionError::ArrowError(Box::new(e), None)) +} + fn effective_query_parallelism( query: &Query, index: &dyn VectorIndex, @@ -1579,13 +1858,15 @@ impl ANNIvfSubIndexExec { index: Arc, query: Query, part_id: usize, - pre_filter: Arc, + pre_filter: Arc, metrics: Arc, + seg_mask: Option>, ) -> DataFusionResult { let batch = index .search_in_partition(part_id, &query, pre_filter, &metrics.index_metrics) .map_err(|e| DataFusionError::Execution(format!("Failed to calculate KNN: {}", e))) .await?; + let batch = restrict_to_segment(batch, seg_mask.as_deref())?; metrics.baseline_metrics.record_output(batch.num_rows()); Ok(batch) } @@ -1596,20 +1877,19 @@ impl ANNIvfSubIndexExec { state: Arc, record_initial: bool, record_partition_per_batch: bool, + seg_mask: Option>, ) -> stream::BoxStream<'static, DataFusionResult> { stream .map(move |batch| { - let metrics = metrics.clone(); - let state = state.clone(); - batch.inspect(move |batch| { - if record_partition_per_batch { - metrics.partitions_searched.add(1); - } - metrics.baseline_metrics.record_output(batch.num_rows()); - if record_initial { - state.record_batch(batch); - } - }) + let batch = restrict_to_segment(batch?, seg_mask.as_deref())?; + if record_partition_per_batch { + metrics.partitions_searched.add(1); + } + metrics.baseline_metrics.record_output(batch.num_rows()); + if record_initial { + state.record_batch(&batch); + } + Ok(batch) }) .boxed() } @@ -1620,10 +1900,12 @@ impl ANNIvfSubIndexExec { query: Query, partitions: Arc, q_c_dists: Arc, - prefilter: Arc, + prefilter: Arc, + global_prefilter: Arc, metrics: Arc, state: Arc, target_partitions: usize, + seg_mask: Option>, ) -> impl Stream> { let stream = futures::stream::once(async move { let max_nprobes = query @@ -1631,20 +1913,33 @@ impl ANNIvfSubIndexExec { .unwrap_or(partitions.len()) .min(partitions.len()); let min_nprobes = query.minimum_nprobes.min(max_nprobes); + + // Every delta must reach the barrier, even if it has no partitions left + // to search, so that siblings waiting for the initial search can proceed. + let found_so_far = state.wait_for_minimum_to_finish().await; if max_nprobes <= min_nprobes { // We've already searched all partitions, no late search needed return futures::stream::empty().boxed(); } - let found_so_far = state.wait_for_minimum_to_finish().await; if found_so_far >= query.k { // We found enough results, no need for late search return futures::stream::empty().boxed(); } + if seg_mask + .as_ref() + .is_some_and(|mask| mask.max_len() == Some(0)) + { + // Every fragment this segment used to own now belongs to a newer delta, + // so probing it can neither produce a row nor move the shared budget that + // stops the late search. Skip it instead of scanning to maximum_nprobes. + return futures::stream::empty().boxed(); + } + // We know the prefilter should be ready at this point so we shouldn't // need to call wait_for_ready - let prefilter_mask = prefilter.mask(); + let prefilter_mask = global_prefilter.mask(); let max_results = prefilter_mask.max_len().map(|x| x as usize); @@ -1657,20 +1952,29 @@ impl ANNIvfSubIndexExec { // This next if check should be true, because we wouldn't get max_results otherwise if let Some(iter_addrs) = prefilter_mask.iter_addrs() { - // We only run this on the first delta because the prefilter mask is shared - // by all deltas and we don't want to duplicate the rows. - if state - .took_no_rows_shortcut - .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) - .is_ok() - { + // Emit the prefilter rows that the partition search did not reach. + // + // The prefilter mask is shared by all deltas. When a per-segment + // restriction is in effect (`seg_mask` is `Some`) each delta emits only + // the addresses its own segment owns; the segments partition the + // fragments, so each address is emitted by exactly one delta. Without a + // restriction the mask is global, so only the first delta emits (guarded + // by a shared flag) to avoid duplicating rows across deltas. + let should_emit = seg_mask.is_some() + || state + .took_no_rows_shortcut + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_ok(); + if should_emit { let initial_addrs = state.initial_ids.lock().unwrap(); let found_addrs = HashSet::<_>::from_iter(initial_addrs.iter().copied()); drop(initial_addrs); - let mask_addrs = HashSet::from_iter(iter_addrs.map(u64::from)); - let not_found_addrs = mask_addrs.difference(&found_addrs); - let not_found_addrs = - UInt64Array::from_iter_values(not_found_addrs.copied()); + let not_found_addrs = UInt64Array::from_iter_values( + iter_addrs.map(u64::from).filter(|addr| { + !found_addrs.contains(addr) + && seg_mask.as_ref().is_none_or(|m| m.selected(*addr)) + }), + ); let not_found_distance = Float32Array::from_value(f32::INFINITY, not_found_addrs.len()); let not_found_batch = RecordBatch::try_new( @@ -1680,8 +1984,8 @@ impl ANNIvfSubIndexExec { .unwrap(); return futures::stream::once(async move { Ok(not_found_batch) }).boxed(); } else { - // We meet all the criteria for an early exit, but we aren't first - // delta so we just return an empty stream and skip the late search + // We meet all the criteria for an early exit, but we aren't the first + // delta and the mask is global, so skip to avoid duplicate rows. return futures::stream::empty().boxed(); } } @@ -1711,6 +2015,7 @@ impl ANNIvfSubIndexExec { Some(Arc::new(LatePartitionSearchControl { state: state.clone(), max_results, + seg_mask: seg_mask.clone(), })), index_metrics, ) @@ -1725,6 +2030,7 @@ impl ANNIvfSubIndexExec { state, false, true, + seg_mask, ), ) }) @@ -1741,6 +2047,7 @@ impl ANNIvfSubIndexExec { let pre_filter = prefilter.clone(); let state = state.clone(); let index = index.clone(); + let seg_mask = seg_mask.clone(); async move { metrics.partitions_searched.add(1); let batch = Self::search_partition( @@ -1749,6 +2056,7 @@ impl ANNIvfSubIndexExec { part_id as usize, pre_filter, metrics, + seg_mask, ) .await?; state.record_late_batch(batch.num_rows()); @@ -1771,10 +2079,11 @@ impl ANNIvfSubIndexExec { query: Query, partitions: Arc, q_c_dists: Arc, - prefilter: Arc, + prefilter: Arc, metrics: Arc, state: Arc, target_partitions: usize, + seg_mask: Option>, ) -> impl Stream> { let minimum_nprobes = query.minimum_nprobes.min(partitions.len()); @@ -1808,6 +2117,7 @@ impl ANNIvfSubIndexExec { state, true, false, + seg_mask, ), ) }) @@ -1825,10 +2135,17 @@ impl ANNIvfSubIndexExec { let index = index.clone(); let pre_filter = prefilter.clone(); let state = state.clone(); + let seg_mask = seg_mask.clone(); async move { - let batch = - Self::search_partition(index, query, part_id as usize, pre_filter, metrics) - .await?; + let batch = Self::search_partition( + index, + query, + part_id as usize, + pre_filter, + metrics, + seg_mask, + ) + .await?; state.record_batch(&batch); Ok(batch) } @@ -1843,10 +2160,6 @@ impl ExecutionPlan for ANNIvfSubIndexExec { "ANNSubIndexExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { KNN_INDEX_SCHEMA.clone() } @@ -1893,6 +2206,8 @@ impl ExecutionPlan for ANNIvfSubIndexExec { indices: self.indices.clone(), query: self.query.clone(), prefilter_source, + overlay_block: self.overlay_block.clone(), + external_mask: self.external_mask.clone(), properties: self.properties.clone(), metrics: ExecutionPlanMetricsSet::new(), } @@ -1916,6 +2231,20 @@ impl ExecutionPlan for ANNIvfSubIndexExec { let ds = self.dataset.clone(); let column = self.query.column.clone(); let indices = self.indices.clone(); + // Per-segment fragment restriction, applied to every partition result before + // the shared search accounting sees it. Only enabled when every segment has a + // fragment_bitmap, mirroring the `all_have_bitmaps` gate in + // DatasetPreFilter::new so we never restrict more aggressively than the + // shared prefilter's fallback. + let segment_bitmaps: Arc> = + Arc::new(if indices.iter().all(|idx| idx.fragment_bitmap.is_some()) { + indices + .iter() + .map(|idx| (idx.uuid, idx.fragment_bitmap.clone().unwrap())) + .collect() + } else { + HashMap::new() + }); let prefilter_source = self.prefilter_source.clone(); let metrics = Arc::new(AnnIndexMetrics::new(&self.metrics, partition)); let metrics_clone = metrics.clone(); @@ -1961,23 +2290,22 @@ impl ExecutionPlan for ANNIvfSubIndexExec { async move { DataFusionResult::Ok(stream::iter(plan)) } }) .try_flatten(); - let prefilter_loader = match &prefilter_source { - PreFilterSource::FilteredRowIds(src_node) => { - let stream = src_node.execute(partition, context)?; - Some(Box::new(FilteredRowIdsToPrefilter(stream)) as Box) - } - PreFilterSource::ScalarIndexQuery(src_node) => { - let stream = src_node.execute(partition, context)?; - Some(Box::new(SelectionVectorToPrefilter(stream)) as Box) - } - PreFilterSource::None => None, - }; - - let pre_filter = Arc::new(DatasetPreFilter::new( + let pre_filter = build_dataset_prefilter( ds.clone(), &indices, - prefilter_loader, - )); + &prefilter_source, + partition, + context, + self.overlay_block.clone(), + self.external_mask.clone(), + )?; + let indices_by_uuid = Arc::new( + indices + .iter() + .cloned() + .map(|index| (index.uuid, index)) + .collect::>(), + ); let state = Arc::new(ANNIvfEarlySearchResults::new(indices.len(), query.k)); @@ -1989,35 +2317,73 @@ impl ExecutionPlan for ANNIvfSubIndexExec { let column = column.clone(); let metrics = metrics.clone(); let pre_filter = pre_filter.clone(); + let indices_by_uuid = indices_by_uuid.clone(); let state = state.clone(); + let segment_bitmaps = segment_bitmaps.clone(); let mut query = query.clone(); let pruned_nprobes = early_pruning(q_c_dists.values(), query.k); adjust_probes(&mut query, pruned_nprobes); async move { + let index_metadata = indices_by_uuid.get(&index_uuid).ok_or_else(|| { + DataFusionError::Execution(format!( + "ANNSubIndexExec: input referenced unknown index segment {index_uuid}" + )) + })?; + let segment_pre_filter = prefilter_for_segment( + ds.clone(), + index_metadata, + pre_filter.clone(), + ) + .await?; let raw_index = ds .open_vector_index(&column, &index_uuid, &metrics.index_metrics) .await?; let query = normalize_query_for_index(raw_index.as_ref(), query)?; + // A segment's index file may still physically contain rows for + // fragments that were pruned from its fragment_bitmap (e.g. after an + // in-place column update via update_columns). Once a newer delta + // segment owns such a fragment, the stale rows in this segment must + // not be returned, otherwise the same row is emitted by two segments. + // Build a per-segment restriction mask, reusing the scheme-aware + // helper so it is correct for both row-address and stable-row-id + // datasets. The shared prefilter is built from the union of all + // segment bitmaps and cannot express this per-segment rule. + let seg_mask = match segment_bitmaps.get(&index_uuid).cloned() { + Some(bitmap) => { + match DatasetPreFilter::create_restricted_deletion_mask( + ds.clone(), + bitmap, + ) { + Some(fut) => Some(fut.await?), + None => None, + } + } + None => None, + }; + let early_search = Self::initial_search( raw_index.clone(), query.clone(), part_ids.clone(), q_c_dists.clone(), - pre_filter.clone(), + segment_pre_filter.clone(), metrics.clone(), state.clone(), target_partitions, + seg_mask.clone(), ); let late_search = Self::late_search( raw_index.clone(), query, part_ids, q_c_dists, + segment_pre_filter, pre_filter, metrics, state, target_partitions, + seg_mask, ); DataFusionResult::Ok(early_search.chain(late_search).boxed()) } @@ -2043,8 +2409,8 @@ impl ExecutionPlan for ANNIvfSubIndexExec { fn partition_statistics( &self, partition: Option, - ) -> DataFusionResult { - Ok(Statistics { + ) -> DataFusionResult> { + Ok(Arc::new(Statistics { num_rows: Precision::Exact( self.query.k * self.query.refine_factor.unwrap_or(1) as usize @@ -2056,7 +2422,7 @@ impl ExecutionPlan for ANNIvfSubIndexExec { .unwrap_or(&1), ), ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { @@ -2072,88 +2438,147 @@ impl ExecutionPlan for ANNIvfSubIndexExec { } } -fn adjust_probes(query: &mut Query, pruned_nprobes: usize) { - query.minimum_nprobes = query.minimum_nprobes.max(pruned_nprobes); - if let Some(maximum) = query.maximum_nprobes - && query.minimum_nprobes > maximum - { - query.minimum_nprobes = maximum; - } -} - -fn early_pruning(dists: &[f32], k: usize) -> usize { - if dists.is_empty() { - return 0; - } - - const PRUNING_FACTORS: [f32; 3] = [0.6, 7.0, 81.0]; - let factor = match k { - ..=1 => PRUNING_FACTORS[0], - 2..=10 => PRUNING_FACTORS[1], - 11.. => PRUNING_FACTORS[2], - }; - let dist_threshold = dists[0] * factor; - dists.partition_point(|dist| *dist <= dist_threshold) +/// Build a batch (multi-query) indexed vector search plan. +/// +/// `query.key` must hold all `query_count` query vectors concatenated. +pub fn new_knn_batch_exec( + dataset: Arc, + indices: &[IndexMetadata], + query: &Query, + query_count: usize, + prefilter_source: PreFilterSource, +) -> Result> { + Ok(Arc::new(ANNIvfBatchExec::try_new( + dataset, + indices.to_vec(), + query.clone(), + query_count, + prefilter_source, + )?)) } +/// [ExecutionPlan] for batch (multi-query) IVF vector search. +/// +/// Where the single-query path uses [`ANNIvfPartitionExec`] + +/// [`ANNIvfSubIndexExec`], this node ranks every query vector against the IVF +/// centroids and then asks the index to read each probed partition's storage +/// once, scoring all queries that probe it +/// (via [`VectorIndex::search_partitions_batch`]). The prefilter is built once +/// and shared across all queries. +/// +/// This is a separate node rather than a mode on the two single-query nodes +/// because the two-node pipeline streams one partition-list per delta through a +/// per-query top-k, whereas the shared scan must invert queries onto partitions +/// and keep one heap per query in a single pass. It still reuses the underlying +/// primitives (partition load, prefilter wiring via [`build_dataset_prefilter`], +/// and the per-partition accumulate the index performs). +/// +/// Output schema: `{query_index: Int32, _distance: Float32, _rowid: UInt64}`, +/// sorted by `(query_index, _distance, _rowid)`, with up to `k` rows per query. +/// +/// Per-query nprobes are honored statically from the ranking; the adaptive +/// late-search expansion used by the single-query path is not applied, so recall +/// matches repeated single-query search when `minimum_nprobes == maximum_nprobes`. #[derive(Debug)] -pub struct MultivectorScoringExec { - // the inputs are sorted ANN search results - inputs: Vec>, +pub struct ANNIvfBatchExec { + dataset: Arc, + indices: Vec, + /// Vector query whose `key` holds all `query_count` vectors concatenated. query: Query, + query_count: usize, + prefilter_source: PreFilterSource, properties: Arc, + metrics: ExecutionPlanMetricsSet, } -impl MultivectorScoringExec { - pub fn try_new(inputs: Vec>, query: Query) -> Result { +impl ANNIvfBatchExec { + pub fn try_new( + dataset: Arc, + indices: Vec, + query: Query, + query_count: usize, + prefilter_source: PreFilterSource, + ) -> Result { + if indices.is_empty() { + return Err(Error::index( + "ANNIvfBatchExec: no index found for query".to_string(), + )); + } + if query_count == 0 || !query.key.len().is_multiple_of(query_count) { + return Err(Error::invalid_input(format!( + "ANNIvfBatchExec: query key length {} is not divisible by query count {query_count}", + query.key.len() + ))); + } let properties = Arc::new(PlanProperties::new( - EquivalenceProperties::new(KNN_INDEX_SCHEMA.clone()), + EquivalenceProperties::new(knn_empty_result_schema(true)), Partitioning::RoundRobinBatch(1), EmissionType::Final, Boundedness::Bounded, )); - Ok(Self { - inputs, + dataset, + indices, query, + query_count, + prefilter_source, properties, + metrics: ExecutionPlanMetricsSet::new(), }) } } -impl DisplayAs for MultivectorScoringExec { +impl DisplayAs for ANNIvfBatchExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { match t { DisplayFormatType::Default | DisplayFormatType::Verbose => { - write!(f, "MultivectorScoring: k={}", self.query.k) + write!( + f, + "ANNIvfBatch: query_count={}, k={}, deltas={}", + self.query_count, + self.query.k, + self.indices.len() + ) } DisplayFormatType::TreeRender => { - write!(f, "MultivectorScoring\nk={}", self.query.k) + write!( + f, + "ANNIvfBatch\nquery_count={}\nk={}\ndeltas={}", + self.query_count, + self.query.k, + self.indices.len() + ) } } } } -impl ExecutionPlan for MultivectorScoringExec { +impl ExecutionPlan for ANNIvfBatchExec { fn name(&self) -> &str { - "MultivectorScoringExec" + "ANNIvfBatchExec" } - fn as_any(&self) -> &dyn Any { - self + fn schema(&self) -> SchemaRef { + knn_empty_result_schema(true) } - fn schema(&self) -> arrow_schema::SchemaRef { - KNN_INDEX_SCHEMA.clone() + fn properties(&self) -> &Arc { + &self.properties + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) } fn children(&self) -> Vec<&Arc> { - self.inputs.iter().collect() + match &self.prefilter_source { + PreFilterSource::None => vec![], + PreFilterSource::FilteredRowIds(src) => vec![src], + PreFilterSource::ScalarIndexQuery(src) => vec![src], + } } fn required_input_distribution(&self) -> Vec { - // This node fully consumes and re-orders the input rows. It must be - // run on a single partition. self.children() .iter() .map(|_| Distribution::SinglePartition) @@ -2162,10 +2587,31 @@ impl ExecutionPlan for MultivectorScoringExec { fn with_new_children( self: Arc, - children: Vec>, + mut children: Vec>, ) -> DataFusionResult> { - let plan = Self::try_new(children, self.query.clone())?; - Ok(Arc::new(plan)) + let prefilter_source = match (&self.prefilter_source, children.len()) { + (PreFilterSource::None, 0) => PreFilterSource::None, + (PreFilterSource::FilteredRowIds(_), 1) => { + PreFilterSource::FilteredRowIds(children.pop().expect("length checked")) + } + (PreFilterSource::ScalarIndexQuery(_), 1) => { + PreFilterSource::ScalarIndexQuery(children.pop().expect("length checked")) + } + _ => { + return Err(DataFusionError::Internal( + "ANNIvfBatchExec given an unexpected number of children".to_string(), + )); + } + }; + Ok(Arc::new(Self { + dataset: self.dataset.clone(), + indices: self.indices.clone(), + query: self.query.clone(), + query_count: self.query_count, + prefilter_source, + properties: self.properties.clone(), + metrics: ExecutionPlanMetricsSet::new(), + })) } fn execute( @@ -2173,18 +2619,294 @@ impl ExecutionPlan for MultivectorScoringExec { partition: usize, context: Arc, ) -> DataFusionResult { - let inputs = self - .inputs - .iter() - .map(|input| input.execute(partition, context.clone())) - .collect::>>()?; - - // collect the top k results from each stream, - // and max-reduce for each query, - // records the minimum distance for each query as estimation. - let mut reduced_inputs = stream::select_all(inputs.into_iter().map(|stream| { - stream.map(|batch| { - let batch = batch?; + let schema = self.schema(); + let ds = self.dataset.clone(); + let column = self.query.column.clone(); + let indices = self.indices.clone(); + let query = self.query.clone(); + let query_count = self.query_count; + let metrics = Arc::new(AnnIndexMetrics::new(&self.metrics, partition)); + let metrics_clone = metrics.clone(); + let timer = Instant::now(); + + let pre_filter = build_dataset_prefilter( + ds.clone(), + &indices, + &self.prefilter_source, + partition, + context, + // The batch node has no data overlay to reconcile against, so no + // stale-row block is applied (see `ANNIvfSubIndexExec::overlay_block`). + None, + // The batch node does not support an external row-address mask. + None, + )?; + + let result_schema = schema.clone(); + let fut = async move { + let dim = query.key.len() / query_count; + // Per-query candidate (distance, row_id) pairs accumulated across deltas. + let mut candidates: Vec> = vec![Vec::new(); query_count]; + + for index_meta in &indices { + let index = ds + .open_vector_index(&column, &index_meta.uuid, &metrics.index_metrics) + .await?; + // The scanner's `batch_index_search_supported` gate decides which + // indices reach this node; this check only guards against that + // gate and the index implementation disagreeing (an internal + // invariant, not a user-facing error). + if !index.supports_batch_partition_search() { + return Err(DataFusionError::Internal(format!( + "ANNIvfBatchExec reached for index {} that does not support batch \ + partition search", + index_meta.uuid + ))); + } + // Normalize each query vector independently (cosine only) + // before ranking; see normalize_batch_query_for_index. + let normalized = normalize_batch_query_for_index( + index.as_ref(), + query.clone(), + query_count, + dim, + )?; + + // Rank every query vector against the IVF centroids on the CPU + // runtime rather than inside this async future: the ranking is + // pure CPU and the batch width multiplies it, so a wide batch + // over a large centroid set could otherwise monopolize a Tokio + // worker. See `find_partitions_batch_on_cpu`. + let (partitions_per_query, dists_per_query) = find_partitions_batch_on_cpu( + index.clone(), + normalized.clone(), + query_count, + dim, + ) + .await?; + + // Record the partitions this delta actually reads. The batch + // node loads each probed partition once and scores every query + // that probes it, so the honest "partitions searched" count is + // the union across queries -- the shared I/O this node exists to + // save -- not the per-query sum. Mirrors the single-query + // ANNIvfSubIndexExec, which also records PARTITIONS_SEARCHED. + let distinct_partitions: RoaringBitmap = partitions_per_query + .iter() + .flat_map(|parts| parts.values().iter().copied()) + .collect(); + metrics + .partitions_searched + .add(distinct_partitions.len() as usize); + + let index_metrics: Arc = + Arc::new(metrics.index_metrics.clone()); + let pre_filter: Arc = pre_filter.clone(); + let per_query = index + .search_partitions_batch( + normalized, + partitions_per_query, + dists_per_query, + pre_filter, + index_metrics, + ) + .await?; + + // `search_partitions_batch` must return exactly one result batch + // per input query, in order, so `query_index` lines up with the + // `candidates` slot below. A mismatch means the index disagreed + // with the per-query fan-out and would otherwise silently drop or + // misattribute results (or panic on out-of-bounds indexing). + if per_query.len() != query_count { + return Err(DataFusionError::Internal(format!( + "batch partition search returned {} result batches for {} queries", + per_query.len(), + query_count + ))); + } + for (query_index, batch) in per_query.into_iter().enumerate() { + // Access by name rather than position: the result schema is + // `VECTOR_RESULT_SCHEMA` (`_distance`, `_rowid`), and looking + // up by name keeps this correct if that column order changes. + let dists = batch + .column_by_name(DIST_COL) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "batch partition search result missing '{DIST_COL}' column" + )) + })? + .as_primitive::(); + let row_ids = batch + .column_by_name(ROW_ID) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "batch partition search result missing '{ROW_ID}' column" + )) + })? + .as_primitive::(); + candidates[query_index].extend( + dists + .values() + .iter() + .copied() + .zip(row_ids.values().iter().copied()), + ); + } + } + + // Per-query top-k merge across deltas, tagged with query_index. + let mut query_index_builder = Int32Builder::new(); + let mut distance_builder = Float32Builder::new(); + let mut row_id_builder = UInt64Builder::new(); + for (query_index, cands) in candidates.iter_mut().enumerate() { + cands.sort_by(|a, b| a.0.total_cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + cands.truncate(query.k); + for (distance, row_id) in cands.iter() { + query_index_builder.append_value(query_index as i32); + distance_builder.append_value(*distance); + row_id_builder.append_value(*row_id); + } + } + let batch = RecordBatch::try_new( + result_schema, + vec![ + Arc::new(query_index_builder.finish()), + Arc::new(distance_builder.finish()), + Arc::new(row_id_builder.finish()), + ], + )?; + metrics.baseline_metrics.record_output(batch.num_rows()); + DataFusionResult::Ok(batch) + }; + + let stream = stream::once(fut).finally(move || { + metrics_clone.index_metrics.flush_io(); + metrics_clone + .baseline_metrics + .elapsed_compute() + .add_duration(timer.elapsed()); + metrics_clone.baseline_metrics.done(); + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + schema, + stream.boxed(), + ))) + } + + fn supports_limit_pushdown(&self) -> bool { + false + } +} + +fn adjust_probes(query: &mut Query, pruned_nprobes: usize) { + query.minimum_nprobes = query.minimum_nprobes.max(pruned_nprobes); + if let Some(maximum) = query.maximum_nprobes + && query.minimum_nprobes > maximum + { + query.minimum_nprobes = maximum; + } +} + +fn early_pruning(dists: &[f32], k: usize) -> usize { + if dists.is_empty() { + return 0; + } + + const PRUNING_FACTORS: [f32; 3] = [0.6, 7.0, 81.0]; + let factor = match k { + ..=1 => PRUNING_FACTORS[0], + 2..=10 => PRUNING_FACTORS[1], + 11.. => PRUNING_FACTORS[2], + }; + let dist_threshold = dists[0] * factor; + dists.partition_point(|dist| *dist <= dist_threshold) +} + +#[derive(Debug)] +pub struct MultivectorScoringExec { + // the inputs are sorted ANN search results + inputs: Vec>, + query: Query, + properties: Arc, +} + +impl MultivectorScoringExec { + pub fn try_new(inputs: Vec>, query: Query) -> Result { + let properties = Arc::new(PlanProperties::new( + EquivalenceProperties::new(KNN_INDEX_SCHEMA.clone()), + Partitioning::RoundRobinBatch(1), + EmissionType::Final, + Boundedness::Bounded, + )); + + Ok(Self { + inputs, + query, + properties, + }) + } +} + +impl DisplayAs for MultivectorScoringExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "MultivectorScoring: k={}", self.query.k) + } + DisplayFormatType::TreeRender => { + write!(f, "MultivectorScoring\nk={}", self.query.k) + } + } + } +} + +impl ExecutionPlan for MultivectorScoringExec { + fn name(&self) -> &str { + "MultivectorScoringExec" + } + + fn schema(&self) -> arrow_schema::SchemaRef { + KNN_INDEX_SCHEMA.clone() + } + + fn children(&self) -> Vec<&Arc> { + self.inputs.iter().collect() + } + + fn required_input_distribution(&self) -> Vec { + // This node fully consumes and re-orders the input rows. It must be + // run on a single partition. + self.children() + .iter() + .map(|_| Distribution::SinglePartition) + .collect() + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> DataFusionResult> { + let plan = Self::try_new(children, self.query.clone())?; + Ok(Arc::new(plan)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let inputs = self + .inputs + .iter() + .map(|input| input.execute(partition, context.clone())) + .collect::>>()?; + + // collect the top k results from each stream, + // and max-reduce for each query, + // records the minimum distance for each query as estimation. + let mut reduced_inputs = stream::select_all(inputs.into_iter().map(|stream| { + stream.map(|batch| { + let batch = batch?; let row_ids = batch[ROW_ID].as_primitive::(); let dists = batch[DIST_COL].as_primitive::(); debug_assert_eq!(dists.null_count(), 0); @@ -2290,12 +3012,14 @@ impl ExecutionPlan for MultivectorScoringExec { mod tests { use super::*; + use std::any::Any; + use crate::index::DatasetIndexExt; use arrow::compute::{concat_batches, sort_to_indices, take_record_batch}; use arrow::datatypes::Float32Type; use arrow_array::{ - ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatchIterator, StringArray, - StructArray, + ArrayRef, FixedSizeListArray, Float32Array, Int32Array, RecordBatchIterator, + RecordBatchReader, StringArray, StructArray, }; use arrow_schema::{Field as ArrowField, Schema as ArrowSchema}; use async_trait::async_trait; @@ -2321,6 +3045,7 @@ mod tests { use crate::dataset::{WriteMode, WriteParams}; use crate::index::vector::VectorIndexParams; + use crate::index::vector::ivf::v2::STREAMING_SEARCH_BATCH_SIZE; use crate::io::exec::testing::TestingExec; fn base_query() -> Query { @@ -2405,7 +3130,8 @@ mod tests { prepared_partitions: Arc>>, searched_partitions: Arc>>, search_threads: Arc>>, - row_ids: Vec, + /// The rows each partition returns, indexed by partition id. + row_ids: Vec>, } #[async_trait] @@ -2576,12 +3302,18 @@ mod tests { async fn search_in_partition( &self, - _partition_id: usize, + partition_id: usize, _query: &Query, _pre_filter: Arc, _metrics: &dyn lance_index::metrics::MetricsCollector, ) -> Result { - panic!("sequential prepared path should not call search_in_partition") + // Only the parallel path reaches this. Tests that must stay on the sequential + // path assert that every partition went through prepare_partition_search, + // which this entry point never records. + self.search_prepared_partition( + Box::new(partition_id), + &lance_index::metrics::NoOpMetricsCollector, + ) } async fn prepare_partition_search( @@ -2608,11 +3340,17 @@ mod tests { .unwrap_or("unknown") .to_string(), ); + let row_ids = &self.row_ids[partition_id]; + // Distances stay distinct within a partition so a test can tell whether the + // distance column survived a filter aligned with its row ids. + let dists = (0..row_ids.len()) + .map(|offset| partition_id as f32 + offset as f32 * 0.5) + .collect::>(); Ok(RecordBatch::try_new( KNN_INDEX_SCHEMA.clone(), vec![ - Arc::new(Float32Array::from(vec![partition_id as f32])), - Arc::new(UInt64Array::from(vec![self.row_ids[partition_id]])), + Arc::new(Float32Array::from(dists)), + Arc::new(UInt64Array::from(row_ids.clone())), ], )?) } @@ -2634,7 +3372,6 @@ mod tests { _metrics: Arc, ) -> Result { let (batch_tx, batch_rx) = mpsc::channel(1); - let batch_tx_for_search = batch_tx.clone(); let prepared_partition_ids = (start_idx..end_idx) .map(|idx| partitions.value(idx) as usize) .collect::>(); @@ -2642,42 +3379,74 @@ mod tests { .lock() .unwrap() .extend(prepared_partition_ids.iter().copied()); + // Mirror the production streaming path (v2.rs): search prepared partitions + // in batches of STREAMING_SEARCH_BATCH_SIZE, one `spawn_cpu` per batch, with + // the channel send in async code so no CPU-pool thread parks (#7642). tokio::spawn(async move { - let search_result = spawn_cpu(move || -> DataFusionResult<()> { - for partition_id in prepared_partition_ids { - if control - .as_ref() - .is_some_and(|control| control.should_stop()) - { - return Ok(()); - } - let batch = self - .search_prepared_partition( - Box::new(partition_id), - &lance_index::metrics::NoOpMetricsCollector, - ) - .map_err(datafusion::error::DataFusionError::from); - match batch { - Ok(batch) => { - if let Some(control) = control.as_ref() { - control.record_batch(&batch); + for chunk in prepared_partition_ids.chunks(*STREAMING_SEARCH_BATCH_SIZE) { + if control + .as_ref() + .is_some_and(|control| control.should_stop()) + || batch_tx.is_closed() + { + return; + } + let chunk = chunk.to_vec(); + let index = self.clone(); + let control_for_search = control.clone(); + let cancel_probe = batch_tx.clone(); + let search_output = spawn_cpu(move || { + let mut outputs: Vec> = + Vec::with_capacity(chunk.len()); + let mut stopped = false; + for partition_id in chunk { + if control_for_search + .as_ref() + .is_some_and(|control| control.should_stop()) + || cancel_probe.is_closed() + { + stopped = true; + break; + } + match index + .search_prepared_partition( + Box::new(partition_id), + &lance_index::metrics::NoOpMetricsCollector, + ) + .map_err(datafusion::error::DataFusionError::from) + { + Ok(batch) => { + if let Some(control) = control_for_search.as_ref() { + control.record_batch(&batch); + } + outputs.push(Ok(batch)); } - if batch_tx_for_search.blocking_send(Ok(batch)).is_err() { - return Ok(()); + Err(err) => { + outputs.push(Err(err)); + stopped = true; + break; } } - Err(err) => { - let _ = batch_tx_for_search.blocking_send(Err(err)); - return Ok(()); - } } - } - Ok(()) - }) - .await; + Ok::<_, datafusion::error::DataFusionError>((outputs, stopped)) + }) + .await; - if let Err(err) = search_result { - let _ = batch_tx.send(Err(err)).await; + let (outputs, stopped) = match search_output { + Ok(output) => output, + Err(err) => { + let _ = batch_tx.send(Err(err)).await; + return; + } + }; + for output in outputs { + if batch_tx.send(output).await.is_err() { + return; + } + } + if stopped { + return; + } } }); @@ -2705,11 +3474,11 @@ mod tests { } fn num_rows(&self) -> u64 { - self.row_ids.len() as u64 + self.row_ids.iter().map(|ids| ids.len() as u64).sum() } fn row_ids(&self) -> Box + '_> { - Box::new(self.row_ids.iter()) + Box::new(self.row_ids.iter().flatten()) } async fn remap(&mut self, _mapping: &RowAddrRemap) -> Result<()> { @@ -2778,6 +3547,7 @@ mod tests { let index = IndexMetadata { uuid: uuid::Uuid::new_v4(), fields: vec![], + covering_fields: vec![], name: "test".to_string(), dataset_version: 1, fragment_bitmap: Some(indexed_fragments), @@ -2792,6 +3562,105 @@ mod tests { prefilter } + #[tokio::test] + async fn test_append_only_deltas_keep_empty_prefilter_fast_path() { + let first = lance_datagen::gen_batch() + .col( + "vector", + array::rand_vec::(lance_datagen::Dimension::from(4)), + ) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + let first_schema = first.schema(); + let mut dataset = Dataset::write(first, "memory://", None).await.unwrap(); + let first_version = dataset.manifest.version; + let first_fragments = dataset.fragment_bitmap.as_ref().clone(); + let field_id = dataset.schema().field("vector").unwrap().id; + + let second = lance_datagen::gen_batch() + .col( + "vector", + array::rand_vec::(lance_datagen::Dimension::from(4)), + ) + .into_reader_rows(RowCount::from(20), BatchCount::from(1)); + assert_eq!(second.schema(), first_schema); + dataset.append(second, None).await.unwrap(); + let appended_fragments = dataset.fragment_bitmap.as_ref() - &first_fragments; + let dataset = Arc::new(dataset); + let old_segment = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![field_id], + covering_fields: vec![], + name: "vector_idx".to_string(), + dataset_version: first_version, + fragment_bitmap: Some(first_fragments.clone()), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + let new_segment = IndexMetadata { + uuid: Uuid::new_v4(), + fields: vec![field_id], + covering_fields: vec![], + name: "vector_idx".to_string(), + dataset_version: dataset.manifest.version, + fragment_bitmap: Some(appended_fragments.clone()), + index_details: None, + index_version: 0, + created_at: None, + base_id: None, + files: None, + }; + assert!( + !appended_fragments.is_empty(), + "the append fixture must create at least one new fragment" + ); + let base = Arc::new(DatasetPreFilter::new( + dataset.clone(), + &[old_segment.clone(), new_segment.clone()], + None, + )); + base.wait_for_ready().await.unwrap(); + assert!(base.is_empty(), "the combined delta coverage is unfiltered"); + let segment_prefilter = prefilter_for_segment(dataset.clone(), &old_segment, base) + .await + .unwrap(); + segment_prefilter.wait_for_ready().await.unwrap(); + + assert!( + !segment_prefilter.is_empty(), + "the segment ownership restriction is not globally empty" + ); + assert!(segment_prefilter.needs_partition_row_ids()); + let old_partition_rows = first_fragments + .iter() + .flat_map(|fragment_id| { + (0..20_u64).map(move |offset| (u64::from(fragment_id) << 32) | offset) + }) + .collect::(); + assert!( + segment_prefilter.is_empty_for(&old_partition_rows), + "an append-only segment must preserve the unfiltered partition fast path" + ); + + let appended_fragment_id = appended_fragments.iter().next().unwrap(); + let mut rows_with_unowned_entry = old_partition_rows; + rows_with_unowned_entry.insert(u64::from(appended_fragment_id) << 32); + assert!(!segment_prefilter.is_empty_for(&rows_with_unowned_entry)); + + let ordinary_base = Arc::new( + DatasetPreFilter::new(dataset, &[old_segment, new_segment], None) + .with_overlay_block(RowAddrMask::allow_nothing()), + ); + let ordinary_segment = + SegmentPreFilter::new(ordinary_base, Arc::new(RowAddrMask::all_rows())); + assert!( + !ordinary_segment.needs_partition_row_ids(), + "a user filter cannot take the no-filter fast path, so partition coverage is unused" + ); + } + fn prepared_metrics() -> Arc { Arc::new(AnnIndexMetrics::new(&ExecutionPlanMetricsSet::new(), 0)) } @@ -2803,7 +3672,13 @@ mod tests { Arc>>, ); + /// One partition per row id, each returning that single row. fn prepared_index(row_ids: Vec) -> PreparedIndexState { + prepared_index_multi(row_ids.into_iter().map(|row_id| vec![row_id]).collect()) + } + + /// One partition per entry, each returning the rows in that entry. + fn prepared_index_multi(row_ids: Vec>) -> PreparedIndexState { let prepared_partitions = Arc::new(Mutex::new(Vec::new())); let searched_partitions = Arc::new(Mutex::new(Vec::new())); let search_threads = Arc::new(Mutex::new(Vec::new())); @@ -2871,33 +3746,72 @@ mod tests { ); } + // Batch analogue of `test_find_partitions_runs_on_cpu_runtime`: the batch + // node's per-query centroid ranking multiplies the CPU cost, so it must also + // run on the dedicated cpu runtime rather than a Tokio async worker. + #[tokio::test] + async fn test_find_partitions_batch_runs_on_cpu_runtime() { + let thread_name = Arc::new(Mutex::new(None)); + let index: Arc = Arc::new(ThreadCapturingIndex { + thread_name: thread_name.clone(), + row_ids: Vec::new(), + }); + + // Two query vectors of dim 1 concatenated into one key. + let mut query = base_query(); + query.key = Arc::new(Float32Array::from(vec![0.0f32, 1.0f32])); + let (partitions, dists) = find_partitions_batch_on_cpu(index, query, 2, 1) + .await + .unwrap(); + assert_eq!(partitions.len(), 2, "one partition list per query"); + assert_eq!(dists.len(), 2, "one distance list per query"); + + let thread_name = thread_name.lock().unwrap().clone().unwrap(); + assert!( + thread_name.contains("lance-cpu"), + "expected batch find_partitions to run on the dedicated cpu runtime, got thread {thread_name}", + ); + } + + // All partitions fit in a single search batch, so they are searched in one + // `spawn_cpu` dispatch and therefore share one cpu thread. The partition count + // adapts to the configured batch size so the single-batch property holds under + // any valid `LANCE_IVF_STREAMING_SEARCH_BATCH_SIZE`, including 1. #[tokio::test] async fn test_sequential_initial_search_prepares_all_then_searches_on_one_cpu_thread() { + let num_partitions = 3.min(*STREAMING_SEARCH_BATCH_SIZE); + let row_ids = (0..num_partitions).map(|i| 10 + i as u64).collect(); let (index, prepared_partitions, searched_partitions, search_threads) = - prepared_index(vec![10, 11, 12]); + prepared_index(row_ids); let mut query = base_query(); - query.minimum_nprobes = 3; + query.minimum_nprobes = num_partitions; let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k)); + let partition_idx = (0..num_partitions as u32).collect::>(); + let q_c_dists = (0..num_partitions) + .map(|i| i as f32 * 0.1) + .collect::>(); let batches = ANNIvfSubIndexExec::initial_search( index, query, - Arc::new(UInt32Array::from(vec![0, 1, 2])), - Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3])), + Arc::new(UInt32Array::from(partition_idx)), + Arc::new(Float32Array::from(q_c_dists)), empty_prefilter().await, prepared_metrics(), state, usize::MAX, + None, ) .try_collect::>() .await .unwrap(); - assert_eq!(batches.len(), 3); - assert_eq!(*prepared_partitions.lock().unwrap(), vec![0, 1, 2]); - assert_eq!(*searched_partitions.lock().unwrap(), vec![0, 1, 2]); + let expected: Vec = (0..num_partitions).collect(); + assert_eq!(batches.len(), num_partitions); + assert_eq!(*prepared_partitions.lock().unwrap(), expected); + assert_eq!(*searched_partitions.lock().unwrap(), expected); let search_threads = search_threads.lock().unwrap().clone(); - assert_eq!(search_threads.len(), 3); + assert_eq!(search_threads.len(), num_partitions); assert!( search_threads.iter().all(|name| name.contains("lance-cpu")), "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}", @@ -2908,6 +3822,53 @@ mod tests { ); } + // Regression guard for the batched streaming search (#7642): with more partitions + // than a single batch, the search spans multiple `spawn_cpu` dispatches. Verify that + // every partition is still prepared and searched in order across the batch boundary, + // and that all search work stays on the cpu runtime. + // + // Note: this does not reproduce the single-thread-pool deadlock the async recv/send + // fixes -- that requires a 1-thread CPU pool, which is a process-global singleton and + // impractical to force in a unit test (same limitation noted for the #7423 fix). + #[tokio::test] + async fn test_sequential_search_spans_multiple_cpu_batches() { + let num_partitions = *STREAMING_SEARCH_BATCH_SIZE + 3; + let row_ids = (0..num_partitions).map(|i| i as u64 * 10).collect(); + let (index, prepared_partitions, searched_partitions, search_threads) = + prepared_index(row_ids); + let mut query = base_query(); + query.minimum_nprobes = num_partitions; + let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k)); + + let partition_idx = (0..num_partitions as u32).collect::>(); + let q_c_dists = (0..num_partitions).map(|i| i as f32).collect::>(); + let batches = ANNIvfSubIndexExec::initial_search( + index, + query, + Arc::new(UInt32Array::from(partition_idx.clone())), + Arc::new(Float32Array::from(q_c_dists)), + empty_prefilter().await, + prepared_metrics(), + state, + usize::MAX, + None, + ) + .try_collect::>() + .await + .unwrap(); + + let expected: Vec = (0..num_partitions).collect(); + assert_eq!(batches.len(), num_partitions); + assert_eq!(*prepared_partitions.lock().unwrap(), expected); + assert_eq!(*searched_partitions.lock().unwrap(), expected); + let search_threads = search_threads.lock().unwrap().clone(); + assert_eq!(search_threads.len(), num_partitions); + assert!( + search_threads.iter().all(|name| name.contains("lance-cpu")), + "expected prepared searches to run on the cpu runtime, got threads {search_threads:?}", + ); + } + #[tokio::test] async fn test_sequential_late_search_prepares_all_then_stops_search_early() { let (index, prepared_partitions, searched_partitions, _search_threads) = @@ -2928,15 +3889,18 @@ mod tests { .unwrap(), ); + let prefilter = empty_prefilter().await; let batches = ANNIvfSubIndexExec::late_search( index, query, Arc::new(UInt32Array::from(vec![0, 1, 2])), Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3])), - empty_prefilter().await, + prefilter.clone(), + prefilter, prepared_metrics(), state.clone(), usize::MAX, + None, ) .try_collect::>() .await @@ -2948,6 +3912,257 @@ mod tests { assert_eq!(state.num_results_found.load(Ordering::Relaxed), 2); } + fn row_ids_of(batches: &[RecordBatch]) -> Vec { + batches + .iter() + .flat_map(|batch| { + batch[ROW_ID] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>() + }) + .collect() + } + + fn dists_of(batches: &[RecordBatch]) -> Vec { + batches + .iter() + .flat_map(|batch| { + batch[DIST_COL] + .as_primitive::() + .values() + .iter() + .copied() + .collect::>() + }) + .collect() + } + + /// A probe can reach a row whose fragment the segment no longer owns. Such a row + /// must be dropped before the shared accounting sees it: counting it and only + /// dropping it downstream lets it consume the `k` budget on its own, so the segment + /// stops probing early and the query returns fewer than `k` current rows. + /// + /// Partitions 0 and 2 hold rows this segment no longer owns and 1 and 3 hold rows + /// it does, so the restriction is exercised in both the initial and the late search. + /// The two parallelism settings pick different code paths: the sequential one counts + /// inside the index via `LatePartitionSearchControl`, the parallel one counts in + /// `search_partition`. + #[rstest] + #[tokio::test] + async fn test_unowned_row_does_not_fill_the_shared_budget( + #[values(1, 2)] query_parallelism: i32, + ) { + // Every partition mixes owned and unowned rows differently: partition 0 loses its + // first row, partition 1 its last, partition 2 all of them and partition 3 its + // last, so the restriction has to keep part of a batch rather than all or nothing. + let (index, prepared_partitions, searched_partitions, _search_threads) = + prepared_index_multi(vec![vec![21, 22, 24], vec![25, 26], vec![20], vec![23, 27]]); + let seg_mask = Arc::new(RowAddrMask::from_allowed( + lance_select::RowAddrTreeMap::from_iter([22u64, 23, 24, 25]), + )); + let partitions = Arc::new(UInt32Array::from(vec![0, 1, 2, 3])); + let q_c_dists = Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3, 0.4])); + + let mut query = base_query(); + query.k = 4; + query.minimum_nprobes = 2; + query.maximum_nprobes = Some(4); + query.query_parallelism = query_parallelism; + let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k)); + + let early = ANNIvfSubIndexExec::initial_search( + index.clone(), + query.clone(), + partitions.clone(), + q_c_dists.clone(), + empty_prefilter().await, + prepared_metrics(), + state.clone(), + usize::MAX, + Some(seg_mask.clone()), + ) + .try_collect::>() + .await + .unwrap(); + + assert_eq!( + row_ids_of(&early), + vec![22, 24, 25], + "the initial search must emit the owned rows and only those" + ); + assert_eq!( + dists_of(&early), + vec![0.5, 1.0, 1.0], + "the distance column must stay aligned with the surviving row ids" + ); + assert_eq!( + *state.initial_ids.lock().unwrap(), + vec![22, 24, 25], + "unowned rows must not take up the initial result budget" + ); + + let prefilter = empty_prefilter().await; + let late = ANNIvfSubIndexExec::late_search( + index, + query, + partitions, + q_c_dists, + prefilter.clone(), + prefilter, + prepared_metrics(), + state.clone(), + usize::MAX, + Some(seg_mask), + ) + .try_collect::>() + .await + .unwrap(); + + assert_eq!( + *searched_partitions.lock().unwrap(), + vec![0, 1, 2, 3], + "the all-unowned partition 2 must not stop the late search" + ); + assert_eq!( + row_ids_of(&late), + vec![23], + "the late search must emit only rows the segment owns" + ); + assert_eq!( + state.num_results_found.load(Ordering::Relaxed), + 4, + "only rows that survive the segment restriction may be counted" + ); + // A parallelism setting the cpu pool cannot honour would silently rerun the + // sequential path, leaving `search_partition`'s restriction untested. + let prepared_partitions = prepared_partitions.lock().unwrap(); + if query_parallelism == 1 { + assert_eq!(*prepared_partitions, vec![0, 1, 2, 3]); + } else { + assert!( + prepared_partitions.is_empty(), + "the parallel path must not prepare partitions, got {prepared_partitions:?}", + ); + } + } + + /// Every fragment the segment used to own now belongs to a newer delta, so it can + /// never contribute a row nor move the shared budget that ends the late search. + /// Probing it to `maximum_nprobes` would be pure waste. + #[tokio::test] + async fn test_segment_owning_nothing_skips_the_late_search() { + let (index, _prepared_partitions, searched_partitions, _search_threads) = + prepared_index(vec![21, 22, 23, 24]); + let seg_mask = Arc::new(RowAddrMask::from_allowed( + lance_select::RowAddrTreeMap::new(), + )); + let partitions = Arc::new(UInt32Array::from(vec![0, 1, 2, 3])); + let q_c_dists = Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3, 0.4])); + + let mut query = base_query(); + query.k = 4; + query.minimum_nprobes = 1; + query.maximum_nprobes = Some(4); + let state = Arc::new(ANNIvfEarlySearchResults::new(1, query.k)); + + ANNIvfSubIndexExec::initial_search( + index.clone(), + query.clone(), + partitions.clone(), + q_c_dists.clone(), + empty_prefilter().await, + prepared_metrics(), + state.clone(), + usize::MAX, + Some(seg_mask.clone()), + ) + .try_collect::>() + .await + .unwrap(); + + let prefilter = empty_prefilter().await; + let late = ANNIvfSubIndexExec::late_search( + index, + query, + partitions, + q_c_dists, + prefilter.clone(), + prefilter, + prepared_metrics(), + state.clone(), + usize::MAX, + Some(seg_mask), + ) + .try_collect::>() + .await + .unwrap(); + + assert!(late.is_empty()); + assert_eq!( + *searched_partitions.lock().unwrap(), + vec![0], + "only the initial probe may run; the late search must not probe at all" + ); + } + + #[tokio::test] + async fn test_delta_skipping_late_search_releases_sibling() { + let prefilter = empty_prefilter().await; + let state = Arc::new(ANNIvfEarlySearchResults::new(2, 4)); + + let (index_a, _prepared_a, searched_a, _threads_a) = prepared_index(vec![21]); + let mut query_a = base_query(); + query_a.k = 4; + query_a.minimum_nprobes = 1; + let delta_a = ANNIvfSubIndexExec::late_search( + index_a, + query_a, + Arc::new(UInt32Array::from(vec![0])), + Arc::new(Float32Array::from(vec![0.1])), + prefilter.clone(), + prefilter.clone(), + prepared_metrics(), + state.clone(), + usize::MAX, + None, + ) + .try_collect::>(); + + let (index_b, _prepared_b, searched_b, _threads_b) = prepared_index(vec![31, 32, 33, 34]); + let mut query_b = base_query(); + query_b.k = 4; + query_b.minimum_nprobes = 1; + query_b.maximum_nprobes = Some(4); + let delta_b = ANNIvfSubIndexExec::late_search( + index_b, + query_b, + Arc::new(UInt32Array::from(vec![0, 1, 2, 3])), + Arc::new(Float32Array::from(vec![0.1, 0.2, 0.3, 0.4])), + prefilter.clone(), + prefilter, + prepared_metrics(), + state, + usize::MAX, + None, + ) + .try_collect::>(); + + let (result_a, result_b) = tokio::time::timeout( + std::time::Duration::from_secs(5), + futures::future::join(delta_a, delta_b), + ) + .await + .expect("late search deadlocked because one delta skipped the shared barrier"); + + result_a.unwrap(); + result_b.unwrap(); + assert!(searched_a.lock().unwrap().is_empty()); + assert!(!searched_b.lock().unwrap().is_empty()); + } + #[tokio::test] async fn knn_flat_search() { let schema = Arc::new(ArrowSchema::new(vec![ diff --git a/rust/lance/src/io/exec/optimizer.rs b/rust/lance/src/io/exec/optimizer.rs index 72488f3a14e..528cc2fa709 100644 --- a/rust/lance/src/io/exec/optimizer.rs +++ b/rust/lance/src/io/exec/optimizer.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use super::TakeExec; +use super::filtered_read::FilteredReadExec; use arrow_schema::Schema as ArrowSchema; #[allow(deprecated)] use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; @@ -18,17 +19,38 @@ use datafusion::{ }; use datafusion_physical_expr::{PhysicalExpr, expressions::Column}; -/// Rule that eliminates [TakeExec] nodes that are immediately followed by another [TakeExec]. +/// Rule that eliminates take nodes that are immediately followed by another +/// take node, fetching the union of the columns in a single node instead. +/// +/// A "take" is either a [TakeExec] (legacy storage) or a [FilteredReadExec] +/// with a row-stream source (see `FilteredReadExec::row_stream_input`); the +/// scanner emits stacked takes in some plan shapes (e.g. filter columns then +/// projection columns). #[derive(Debug)] pub struct CoalesceTake; impl CoalesceTake { + /// Whether `plan` is a take node this rule knows how to collapse + fn as_take(plan: &Arc) -> Option<&dyn ExecutionPlan> { + if plan.downcast_ref::().is_some() { + Some(plan.as_ref()) + } else if let Some(filtered_read) = plan.downcast_ref::() { + filtered_read + .row_stream_input() + .is_some() + .then_some(plan.as_ref()) + } else { + None + } + } + fn field_order_differs(old_schema: &ArrowSchema, new_schema: &ArrowSchema) -> bool { - old_schema - .fields - .iter() - .zip(&new_schema.fields) - .any(|(old, new)| old.name() != new.name()) + old_schema.fields.len() != new_schema.fields.len() + || old_schema + .fields + .iter() + .zip(&new_schema.fields) + .any(|(old, new)| old.name() != new.name()) } fn remap_collapsed_output( @@ -47,24 +69,39 @@ impl CoalesceTake { Arc::new(ProjectionExec::try_new(project_exprs, plan).unwrap()) } + /// Collapse two stacked takes into one, or return None when the rebuilt + /// node would not produce every column of the original output (the + /// rebuild re-derives what to fetch from the outer take's projection, so + /// a column only the inner take fetched can go missing if the outer + /// projection doesn't cover it) fn collapse_takes( - inner_take: &TakeExec, - outer_take: &TakeExec, + inner_take: &dyn ExecutionPlan, + outer_take: &dyn ExecutionPlan, outer_exec: Arc, - ) -> Arc { + ) -> Option> { let inner_take_input = inner_take.children()[0].clone(); let old_output_schema = outer_take.schema(); - let collapsed = outer_exec - .with_new_children(vec![inner_take_input]) - .unwrap(); + let collapsed = outer_exec.with_new_children(vec![inner_take_input]).ok()?; let new_output_schema = collapsed.schema(); + if old_output_schema + .fields() + .iter() + .any(|field| new_output_schema.field_with_name(field.name()).is_err()) + { + return None; + } + // It's possible that collapsing the take can change the field order. This disturbs DF's planner and // so we must restore it. if Self::field_order_differs(&old_output_schema, &new_output_schema) { - Self::remap_collapsed_output(&old_output_schema, &new_output_schema, collapsed) + Some(Self::remap_collapsed_output( + &old_output_schema, + &new_output_schema, + collapsed, + )) } else { - collapsed + Some(collapsed) } } } @@ -78,26 +115,23 @@ impl PhysicalOptimizerRule for CoalesceTake { ) -> DFResult> { Ok(plan .transform_down(|plan| { - if let Some(outer_take) = plan.as_any().downcast_ref::() { + if let Some(outer_take) = Self::as_take(&plan) { let child = outer_take.children()[0]; - // Case 1: TakeExec -> TakeExec - if let Some(inner_take) = child.as_any().downcast_ref::() { - return Ok(Transformed::yes(Self::collapse_takes( - inner_take, - outer_take, - plan.clone(), - ))); - // Case 2: TakeExec -> CoalesceBatchesExec -> TakeExec - } else if let Some(exec_child) = - child.as_any().downcast_ref::() - { + // Case 1: take -> take + if let Some(inner_take) = Self::as_take(child) { + if let Some(collapsed) = + Self::collapse_takes(inner_take, outer_take, plan.clone()) + { + return Ok(Transformed::yes(collapsed)); + } + // Case 2: take -> CoalesceBatchesExec -> take + } else if let Some(exec_child) = child.downcast_ref::() { let inner_child = exec_child.children()[0].clone(); - if let Some(inner_take) = inner_child.as_any().downcast_ref::() { - return Ok(Transformed::yes(Self::collapse_takes( - inner_take, - outer_take, - plan.clone(), - ))); + if let Some(inner_take) = Self::as_take(&inner_child) + && let Some(collapsed) = + Self::collapse_takes(inner_take, outer_take, plan.clone()) + { + return Ok(Transformed::yes(collapsed)); } } } @@ -128,7 +162,7 @@ impl PhysicalOptimizerRule for SimplifyProjection { ) -> DFResult> { Ok(plan .transform_down(|plan| { - if let Some(proj) = plan.as_any().downcast_ref::() { + if let Some(proj) = plan.downcast_ref::() { let children = proj.children(); if children.len() != 1 { return Ok(Transformed::no(plan)); @@ -145,7 +179,7 @@ impl PhysicalOptimizerRule for SimplifyProjection { } if proj.expr().iter().enumerate().all(|(index, proj_expr)| { - if let Some(expr) = proj_expr.expr.as_any().downcast_ref::() { + if let Some(expr) = proj_expr.expr.downcast_ref::() { // no renaming, no reordering expr.index() == index && expr.name() == proj_expr.alias } else { @@ -184,3 +218,196 @@ pub fn get_physical_optimizer() -> PhysicalOptimizer { Arc::new(datafusion::physical_optimizer::enforce_distribution::EnforceDistribution::new()), ]) } + +#[cfg(test)] +mod tests { + use super::*; + + use arrow_array::cast::AsArray; + use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt64Array}; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use arrow_select::concat::concat_batches; + use datafusion::execution::TaskContext; + use datafusion::physical_plan::stream::RecordBatchStreamAdapter; + use futures::TryStreamExt; + use lance_core::ROW_ID; + use lance_core::datatypes::OnMissing; + use lance_core::utils::tempfile::TempStrDir; + use lance_datafusion::exec::OneShotExec; + use lance_file::version::LanceFileVersion; + + use crate::dataset::{Dataset, WriteParams}; + use crate::io::exec::filtered_read::{FilteredReadExec, FilteredReadOptions}; + + /// 20 rows, one fragment, columns i (Int32) and s (Utf8) + async fn fixture(storage_version: LanceFileVersion) -> (Arc, TempStrDir) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("i", DataType::Int32, false), + ArrowField::new("s", DataType::Utf8, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from_iter_values(0..20)), + Arc::new(StringArray::from_iter_values( + (0..20).map(|v| format!("s-{v}")), + )), + ], + ) + .unwrap(); + let tmp_dir = TempStrDir::default(); + let uri = tmp_dir.as_str(); + let params = WriteParams { + data_storage_version: Some(storage_version), + ..Default::default() + }; + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(reader, uri, Some(params)).await.unwrap(); + (Arc::new(Dataset::open(uri).await.unwrap()), tmp_dir) + } + + /// An input plan producing one batch of `_rowid` keys + fn keys_input(keys: Vec) -> Arc { + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + ROW_ID, + DataType::UInt64, + true, + )])); + let batch = + RecordBatch::try_new(schema.clone(), vec![Arc::new(UInt64Array::from(keys))]).unwrap(); + let stream = futures::stream::iter(vec![Ok(batch)]); + let stream = Box::pin(RecordBatchStreamAdapter::new(schema, stream)); + Arc::new(OneShotExec::new(stream)) + } + + fn row_stream_take( + dataset: &Arc, + input: Arc, + columns: &[&str], + ) -> Arc { + // Mirror Scanner::take: full target projection, carried identity kept + let mut projection = dataset + .empty_projection() + .union_columns(columns, OnMissing::Error) + .unwrap(); + projection.with_row_id = true; + Arc::new( + FilteredReadExec::try_new( + dataset.clone(), + FilteredReadOptions::new(projection), + Some(input), + ) + .unwrap(), + ) + } + + async fn run(plan: &Arc) -> RecordBatch { + let stream = plan.execute(0, Arc::new(TaskContext::default())).unwrap(); + let schema = stream.schema(); + let batches: Vec<_> = stream.try_collect().await.unwrap(); + concat_batches(&schema, batches.iter()).unwrap() + } + + fn count_takes(plan: &Arc) -> usize { + let self_count = CoalesceTake::as_take(plan).map(|_| 1).unwrap_or(0); + self_count + + plan + .children() + .iter() + .map(|child| count_takes(child)) + .sum::() + } + + /// Stacked row-stream takes collapse into one node fetching both takes' + /// columns, preserving the output schema and values + #[tokio::test] + async fn collapse_row_stream_takes() { + let (dataset, _tmp) = fixture(LanceFileVersion::Stable).await; + + // OneShotExec inputs are single-use: build the plan fresh per run + let build = |dataset: &Arc| { + let inner = row_stream_take(dataset, keys_input(vec![3, 1, 4]), &["s"]); + row_stream_take(dataset, inner, &["i", "s"]) + }; + let outer = build(&dataset); + let expected_schema = outer.schema(); + assert_eq!(count_takes(&outer), 2); + let expected = run(&outer).await; + + let optimized = CoalesceTake + .optimize(build(&dataset), &ConfigOptions::default()) + .unwrap(); + assert_eq!(count_takes(&optimized), 1); + assert_eq!(optimized.schema(), expected_schema); + + let result = run(&optimized).await; + assert_eq!(result, expected); + let i_col = result + .column_by_name("i") + .unwrap() + .as_primitive::(); + assert_eq!(i_col.values(), &[3, 1, 4]); + } + + /// When the outer take's projection does not cover a column the inner + /// take fetched, the collapse is skipped instead of dropping the column + #[tokio::test] + async fn collapse_skipped_when_column_would_drop() { + let (dataset, _tmp) = fixture(LanceFileVersion::Stable).await; + + // Outer target deliberately omits the inner take's "s" + let build = |dataset: &Arc| { + let inner = row_stream_take(dataset, keys_input(vec![3, 1, 4]), &["s"]); + row_stream_take(dataset, inner, &["i"]) + }; + let outer = build(&dataset); + assert_eq!(count_takes(&outer), 2); + let expected = run(&outer).await; + + let optimized = CoalesceTake + .optimize(build(&dataset), &ConfigOptions::default()) + .unwrap(); + assert_eq!(count_takes(&optimized), 2); + assert_eq!(run(&optimized).await, expected); + assert!(expected.column_by_name("s").is_some()); + } + + /// Legacy TakeExec pairs still collapse (through the CoalesceBatchesExec + /// the scanner inserts on that path) + #[tokio::test] + async fn collapse_legacy_takes() { + let (dataset, _tmp) = fixture(LanceFileVersion::Legacy).await; + + let build = |dataset: &Arc| -> Arc { + let inner_proj = dataset + .empty_projection() + .union_columns(["s"], OnMissing::Error) + .unwrap(); + let inner: Arc = Arc::new( + TakeExec::try_new(dataset.clone(), keys_input(vec![3, 1, 4]), inner_proj) + .unwrap() + .unwrap(), + ); + let outer_proj = dataset + .empty_projection() + .union_columns(["i", "s"], OnMissing::Error) + .unwrap(); + Arc::new( + TakeExec::try_new(dataset.clone(), inner, outer_proj) + .unwrap() + .unwrap(), + ) + }; + let outer = build(&dataset); + let expected_schema = outer.schema(); + assert_eq!(count_takes(&outer), 2); + let expected = run(&outer).await; + + let optimized = CoalesceTake + .optimize(build(&dataset), &ConfigOptions::default()) + .unwrap(); + assert_eq!(count_takes(&optimized), 1); + assert_eq!(optimized.schema(), expected_schema); + assert_eq!(run(&optimized).await, expected); + } +} diff --git a/rust/lance/src/io/exec/projection.rs b/rust/lance/src/io/exec/projection.rs index 3106fcfac61..06bc0b8de67 100644 --- a/rust/lance/src/io/exec/projection.rs +++ b/rust/lance/src/io/exec/projection.rs @@ -44,7 +44,7 @@ pub fn project(input: Arc, projection: &ArrowSchema) -> Resul let field_names = projection.fields().iter().map(|f| f.name()).cloned(); - for (name, selection) in field_names.zip(selections.into_iter()) { + for (name, selection) in field_names.zip(selections) { let expr = selection_as_expr(&selection, input_schema.fields(), None); exprs.push((expr, name)); } diff --git a/rust/lance/src/io/exec/pushdown_scan.rs b/rust/lance/src/io/exec/pushdown_scan.rs index b0b0eacded6..0f1d2a2617a 100644 --- a/rust/lance/src/io/exec/pushdown_scan.rs +++ b/rust/lance/src/io/exec/pushdown_scan.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use std::collections::HashMap; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow_array::cast::AsArray; use arrow_array::types::{Int64Type, UInt64Type}; @@ -25,7 +25,8 @@ use datafusion::{ }; use datafusion_functions::core::expr_ext::FieldAccessor; use datafusion_physical_expr::EquivalenceProperties; -use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; +use futures::stream::BoxStream; +use futures::{FutureExt, StreamExt, TryStreamExt}; use lance_arrow::{RecordBatchExt, SchemaExt}; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::{ROW_ADDR, ROW_ADDR_FIELD, ROW_ID_FIELD}; @@ -40,13 +41,14 @@ use crate::{ Dataset, dataset::{ ROW_ID, - fragment::{FileFragment, FragmentReader}, + fragment::{FileFragment, V1FragmentReader}, + versions, }, datatypes::Schema, }; use super::Planner; -use super::utils::InstrumentedRecordBatchStreamAdapter; +use super::utils::{InstrumentedRecordBatchStreamAdapter, buffered_fragment_opens}; #[derive(Debug, Clone)] pub struct ScanConfig { @@ -158,10 +160,6 @@ impl ExecutionPlan for LancePushdownScanExec { "LancePushdownScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -207,23 +205,24 @@ impl ExecutionPlan for LancePushdownScanExec { } }); - let batch_stream = fragment_stream.map(|(exec, fragment)| async move { - let frag_scanner = FragmentScanner::open( - fragment, - exec.dataset, - exec.projection, - exec.predicate_projection, - exec.predicate, - exec.config.clone(), - ) - .await?; - - frag_scanner.scan() - }); + let batch_stream = buffered_fragment_opens( + fragment_stream, + self.config.fragment_readahead, + |(exec, fragment)| async move { + let frag_scanner = FragmentScanner::open( + fragment, + exec.dataset, + exec.projection, + exec.predicate_projection, + exec.predicate, + exec.config.clone(), + ) + .await?; - let batch_stream = batch_stream - .buffered(self.config.fragment_readahead) - .try_flatten(); + frag_scanner.scan() + }, + ) + .try_flatten(); Ok(Box::pin(InstrumentedRecordBatchStreamAdapter::new( self.schema(), @@ -283,7 +282,7 @@ struct FragmentScanner { predicate_projection: Arc, predicate: Expr, config: ScanConfig, - reader: FragmentReader, + reader: V1FragmentReader, stats: Option, } @@ -304,15 +303,14 @@ impl FragmentScanner { if let Some(file_reader_options) = config.file_reader_options.clone() { frag_config = frag_config.with_file_reader_options(file_reader_options); } - let mut reader = fragment.open(dataset.schema(), frag_config).await?; + let mut reader = + versions::open_v1_fragment_reader(&fragment, dataset.schema(), &frag_config).await?; if config.make_deletions_null { reader.with_make_deletions_null(); } // We only need the statistics for the predicate projection. - let stats = reader - .legacy_read_page_stats(Some(&predicate_projection)) - .await?; + let stats = reader.read_page_stats(Some(&predicate_projection)).await?; Ok(Self { fragment, @@ -325,7 +323,7 @@ impl FragmentScanner { }) } - pub fn scan(self) -> Result> + 'static + Send> { + pub fn scan(self) -> Result>> { let batch_readahead = self.config.batch_readahead; let simplified_predicates = self.simplified_predicates()?; let ordered_output = self.config.ordered_output; @@ -381,7 +379,7 @@ impl FragmentScanner { projection_reader.with_row_address(); } let batch = projection_reader - .legacy_read_batch_projected(batch_id, .., &self.projection) + .read_batch_projected(batch_id, .., &self.projection) .await?; let batch = self.final_projection(batch)?; Ok(Some(batch)) @@ -414,7 +412,7 @@ impl FragmentScanner { reader.with_row_address(); let batch = reader - .legacy_read_batch_projected(batch_id, .., &predicate_projection) + .read_batch_projected(batch_id, .., &predicate_projection) .await?; // 2. Evaluate predicate @@ -488,7 +486,7 @@ impl FragmentScanner { self.projection.project_by_ids(&remaining_fields, true); Some( self.reader - .legacy_read_batch_projected( + .read_batch_projected( batch_id, selection.clone(), &remaining_projection, @@ -658,20 +656,20 @@ impl FragmentScanner { } fn simplified_predicates(&self) -> Result> { - let num_batches = self.reader.legacy_num_batches(); + let num_batches = self.reader.num_batches(); if let Some(stats) = &self.stats { let batch_sizes: Vec = (0..num_batches as u32) .map(|batch_id| { self.reader - .legacy_num_rows_in_batch(batch_id) + .num_rows_in_batch(batch_id) .expect("Operation does not yet support v2 fragments") as usize }) .collect(); let schema = Arc::new(ArrowSchema::from(self.predicate_projection.as_ref()).try_into()?); - let context = SimplifyContext::default().with_schema(schema); + let context = SimplifyContext::builder().with_schema(schema).build(); let mut simplifier = ExprSimplifier::new(context); let mut predicates = Vec::with_capacity(num_batches); @@ -700,6 +698,11 @@ impl FragmentScanner { #[cfg(test)] mod test { + use std::collections::HashSet; + use std::fmt::Display; + use std::sync::Mutex; + use std::time::Duration; + use arrow_array::{ ArrayRef, DictionaryArray, FixedSizeListArray, Float32Array, Int32Array, RecordBatchIterator, StringArray, StructArray, TimestampMicrosecondArray, UInt64Array, @@ -708,17 +711,275 @@ mod test { use arrow_ord::sort::sort_to_indices; use arrow_schema::{Field, TimeUnit}; use arrow_select::concat::concat_batches; + use async_trait::async_trait; use datafusion::prelude::{Column, SessionContext, lit}; + use futures::stream::BoxStream; use lance_arrow::{FixedSizeListArrayExt, SchemaExt}; use lance_core::utils::tempfile::TempStrDir; + use lance_datagen::{array, gen_batch}; use lance_file::version::LanceFileVersion; + use lance_io::object_store::WrappingObjectStore; + use object_store::list::PaginatedListStore; + use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, + Result as ObjectStoreResult, path::Path, + }; use pretty_assertions::assert_eq; + use tokio::sync::{Semaphore, mpsc}; use crate::dataset::WriteParams; + use crate::io::exec::{LanceScanConfig, LanceScanExec}; + use crate::utils::test::{DatagenExt, FragmentCount, FragmentRowCount}; use lance_datafusion::logical_expr::ExprExt; use super::*; + #[derive(Debug)] + struct BlockingDataFileReads { + release: Semaphore, + started_paths: Mutex>, + started_tx: mpsc::UnboundedSender<()>, + completed_tx: mpsc::UnboundedSender<()>, + } + + impl BlockingDataFileReads { + async fn wait_for_release(&self, location: &Path) -> bool { + let is_first_data_file_read = location.as_ref().ends_with(".lance") + && self.started_paths.lock().unwrap().insert(location.clone()); + if !is_first_data_file_read { + return false; + } + + self.started_tx.send(()).unwrap(); + self.release + .acquire() + .await + .expect("release semaphore was closed") + .forget(); + true + } + } + + #[derive(Debug, Clone)] + struct BlockingDataFileStoreWrapper { + reads: Arc, + } + + impl WrappingObjectStore for BlockingDataFileStoreWrapper { + fn wrap(&self, _prefix: &str, target: Arc) -> Arc { + Arc::new(BlockingDataFileStore { + target, + reads: self.reads.clone(), + }) + } + + // Only data file reads are blocked, so a listing can keep the pushdown. + fn wrap_paginated( + &self, + _store_prefix: &str, + original: Arc, + ) -> Option> { + Some(original) + } + } + + #[derive(Debug)] + struct BlockingDataFileStore { + target: Arc, + reads: Arc, + } + + impl Display for BlockingDataFileStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "BlockingDataFileStore({})", self.target) + } + } + + #[async_trait] + impl ObjectStore for BlockingDataFileStore { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + options: PutOptions, + ) -> ObjectStoreResult { + self.target.put_opts(location, payload, options).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + options: PutMultipartOptions, + ) -> ObjectStoreResult> { + self.target.put_multipart_opts(location, options).await + } + + async fn get_opts( + &self, + location: &Path, + options: GetOptions, + ) -> ObjectStoreResult { + let is_tracked_read = self.reads.wait_for_release(location).await; + let result = self.target.get_opts(location, options).await; + if is_tracked_read { + self.reads.completed_tx.send(()).unwrap(); + } + result + } + + fn delete_stream( + &self, + locations: BoxStream<'static, ObjectStoreResult>, + ) -> BoxStream<'static, ObjectStoreResult> { + self.target.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, ObjectStoreResult> { + self.target.list(prefix) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&Path>, + ) -> ObjectStoreResult { + self.target.list_with_delimiter(prefix).await + } + + async fn copy_opts( + &self, + from: &Path, + to: &Path, + options: CopyOptions, + ) -> ObjectStoreResult<()> { + self.target.copy_opts(from, to, options).await + } + + async fn rename_opts( + &self, + from: &Path, + to: &Path, + options: RenameOptions, + ) -> ObjectStoreResult<()> { + self.target.rename_opts(from, to, options).await + } + } + + #[derive(Debug)] + enum LegacyScanPath { + Regular, + Pushdown, + } + + #[tokio::test] + #[rstest::rstest] + #[case::regular(LegacyScanPath::Regular)] + #[case::pushdown(LegacyScanPath::Pushdown)] + async fn test_fragment_opens_progress_with_bounded_cancellation( + #[case] scan_path: LegacyScanPath, + ) { + const FRAGMENT_READAHEAD: usize = 2; + const ROWS_PER_FRAGMENT: u32 = 8; + + let dataset = gen_batch() + .col("x", array::step::()) + .into_ram_dataset_with_params( + FragmentCount::from(4), + FragmentRowCount::from(ROWS_PER_FRAGMENT), + Some(WriteParams { + data_storage_version: Some(LanceFileVersion::Legacy), + max_rows_per_file: ROWS_PER_FRAGMENT as usize, + max_rows_per_group: ROWS_PER_FRAGMENT as usize, + ..WriteParams::default() + }), + ) + .await + .unwrap(); + dataset.session().file_metadata_cache().clear().await; + + let (started_tx, mut started_rx) = mpsc::unbounded_channel(); + let (completed_tx, mut completed_rx) = mpsc::unbounded_channel(); + let reads = Arc::new(BlockingDataFileReads { + release: Semaphore::new(0), + started_paths: Mutex::new(HashSet::new()), + started_tx, + completed_tx, + }); + let dataset = Arc::new(dataset.with_object_store_wrappers([Arc::new( + BlockingDataFileStoreWrapper { + reads: reads.clone(), + }, + ) + as Arc])); + let fragments = dataset.fragments().clone(); + let projection = Arc::new(dataset.schema().clone()); + + let exec: Arc = match scan_path { + LegacyScanPath::Regular => Arc::new(LanceScanExec::new( + dataset, + fragments, + None, + projection, + LanceScanConfig { + fragment_readahead: Some(FRAGMENT_READAHEAD), + ordered_output: true, + ..LanceScanConfig::default() + }, + )), + LegacyScanPath::Pushdown => Arc::new( + LancePushdownScanExec::try_new( + dataset, + fragments, + projection, + col("x").gt(lit(-1)), + ScanConfig { + fragment_readahead: FRAGMENT_READAHEAD, + ..ScanConfig::default() + }, + ) + .unwrap(), + ), + }; + let context = SessionContext::new(); + let mut output = exec.execute(0, context.task_ctx()).unwrap(); + + assert!(futures::poll!(output.next()).is_pending()); + for _ in 0..FRAGMENT_READAHEAD { + tokio::time::timeout(Duration::from_secs(5), started_rx.recv()) + .await + .expect("fragment open did not start") + .expect("fragment open start channel closed"); + } + tokio::task::yield_now().await; + assert!( + matches!(started_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "fragment opens exceeded fragment_readahead" + ); + + // The child task must finish this request even though the output stream + // is not polled again. + reads.release.add_permits(1); + tokio::time::timeout(Duration::from_secs(5), completed_rx.recv()) + .await + .expect("fragment open did not progress independently") + .expect("fragment open completion channel closed"); + assert!( + matches!(started_rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "fragment opens exceeded fragment_readahead" + ); + + // Dropping the scan must abort the other in-flight open instead of + // detaching it from the cancelled query. + drop(output); + reads.release.add_permits(FRAGMENT_READAHEAD); + assert!( + tokio::time::timeout(Duration::from_secs(1), completed_rx.recv()) + .await + .is_err(), + "fragment open completed after scan cancellation" + ); + } + // TODO: test pushdown with nested column once https://github.com/apache/arrow-datafusion/pull/8256 // is released. diff --git a/rust/lance/src/io/exec/row_addr_mask.rs b/rust/lance/src/io/exec/row_addr_mask.rs new file mode 100644 index 00000000000..eb7059098bc --- /dev/null +++ b/rust/lance/src/io/exec/row_addr_mask.rs @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! RowAddrMask prefilter wiring for vector search. +//! +//! An externally supplied [`RowAddrMask`] is applied to a KNN search through two +//! pieces, because the two search branches consume a prefilter differently: +//! - [`MaskAndLoader`] folds the mask into the index-side prefilter loader +//! (ANN / IVF branch). The mask, any filter-derived selection vector, and +//! the deletion vector are all combined (logical AND) by DatasetPreFilter. +//! - [`RowAddrMaskFilterExec`] applies the mask to the flat-KNN branch, which +//! scans fragments not covered by the vector index and so never reaches the +//! index-side prefilter. + +use std::sync::Arc; + +use arrow::datatypes::UInt64Type; +use arrow_array::cast::AsArray; +use arrow_array::{BooleanArray, RecordBatch}; +use async_trait::async_trait; +use datafusion::error::DataFusionError; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, SendableRecordBatchStream, +}; +use futures::StreamExt; +use lance_core::error::DataFusionResult; +use lance_core::{ROW_ID, Result}; +use lance_index::prefilter::FilterLoader; +use lance_select::RowAddrMask; + +/// FilterLoader that combines an external RowAddrMask (logical AND) with an +/// optional inner loader. +/// +/// With an inner loader present the two masks are intersected; otherwise the +/// external mask is used alone. DatasetPreFilter later intersects the result +/// with the dataset deletion vector. +pub struct MaskAndLoader { + mask: Arc, + inner: Option>, +} + +impl MaskAndLoader { + pub fn new(mask: Arc, inner: Option>) -> Self { + Self { mask, inner } + } +} + +#[async_trait] +impl FilterLoader for MaskAndLoader { + async fn load(self: Box) -> Result { + match self.inner { + Some(inner) => Ok(Arc::unwrap_or_clone(self.mask) & inner.load().await?), + None => Ok(Arc::unwrap_or_clone(self.mask)), + } + } +} + +/// Execution node that drops rows whose `_rowid` is not selected by `mask`. +/// +/// The key is read from the `_rowid` column, and `mask` is keyed in that same +/// `_rowid` space, so this is consistent whether stable row ids are enabled (the +/// value is the stable row id) or disabled (it is the row address). Schema and +/// ordering are preserved; only the row count changes. +#[derive(Debug)] +pub struct RowAddrMaskFilterExec { + input: Arc, + mask: Arc, + properties: Arc, +} + +impl RowAddrMaskFilterExec { + pub fn new(input: Arc, mask: Arc) -> Self { + // Filtering preserves schema, partitioning and ordering, so the input's + // plan properties carry over unchanged. + let properties = input.properties().clone(); + Self { + input, + mask, + properties, + } + } +} + +impl DisplayAs for RowAddrMaskFilterExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "RowAddrMaskFilter") + } +} + +impl ExecutionPlan for RowAddrMaskFilterExec { + fn name(&self) -> &str { + "RowAddrMaskFilterExec" + } + + fn properties(&self) -> &Arc { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn maintains_input_order(&self) -> Vec { + vec![true] + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + if children.len() != 1 { + return Err(DataFusionError::Internal( + "RowAddrMaskFilterExec must have exactly one child".to_string(), + )); + } + let child = children.pop().ok_or_else(|| { + DataFusionError::Internal("RowAddrMaskFilterExec child unavailable".to_string()) + })?; + Ok(Arc::new(Self::new(child, self.mask.clone()))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> DataFusionResult { + let input_stream = self.input.execute(partition, context)?; + let schema = input_stream.schema(); + let mask = self.mask.clone(); + let stream = input_stream.map(move |batch| apply_mask(&mask, batch?)); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) + } +} + +/// Keep rows whose `_rowid` is selected by the mask (the mask is keyed in the +/// same `_rowid` space). Null ids are dropped; they cannot be in any allow set. +fn apply_mask(mask: &RowAddrMask, batch: RecordBatch) -> DataFusionResult { + let row_id_column = batch.column_by_name(ROW_ID).ok_or_else(|| { + DataFusionError::Internal(format!( + "RowAddrMaskFilterExec input missing {ROW_ID} column" + )) + })?; + let row_ids = row_id_column + .as_primitive_opt::() + .ok_or_else(|| { + DataFusionError::Internal(format!( + "{ROW_ID} column must be UInt64 but was {:?}", + row_id_column.data_type() + )) + })?; + let keep = BooleanArray::from_iter( + row_ids + .iter() + .map(|addr| Some(addr.is_some_and(|addr| mask.selected(addr)))), + ); + arrow::compute::filter_record_batch(&batch, &keep) + .map_err(|e| DataFusionError::ArrowError(Box::new(e), None)) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::datatypes::{DataType, Field, Schema}; + use arrow_array::{Int32Array, UInt64Array}; + use lance_select::RowAddrTreeMap; + + fn batch_with_rowids(ids: Vec>) -> RecordBatch { + let n = ids.len() as i32; + let schema = Arc::new(Schema::new(vec![ + Field::new(ROW_ID, DataType::UInt64, true), + Field::new("v", DataType::Int32, false), + ])); + RecordBatch::try_new( + schema, + vec![ + Arc::new(UInt64Array::from(ids)), + Arc::new(Int32Array::from((0..n).collect::>())), + ], + ) + .unwrap() + } + + fn kept_rowids(batch: &RecordBatch) -> Vec> { + batch + .column_by_name(ROW_ID) + .unwrap() + .as_primitive::() + .iter() + .collect() + } + + #[test] + fn apply_mask_allow_keeps_only_selected() { + let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64, 3, 5])); + let batch = batch_with_rowids(vec![Some(1), Some(2), Some(3), Some(4), Some(5)]); + let out = apply_mask(&mask, batch).unwrap(); + assert_eq!(kept_rowids(&out), vec![Some(1), Some(3), Some(5)]); + } + + #[test] + fn apply_mask_block_drops_selected() { + let mask = RowAddrMask::from_block(RowAddrTreeMap::from_iter([2u64, 4])); + let batch = batch_with_rowids(vec![Some(1), Some(2), Some(3), Some(4), Some(5)]); + let out = apply_mask(&mask, batch).unwrap(); + assert_eq!(kept_rowids(&out), vec![Some(1), Some(3), Some(5)]); + } + + #[test] + fn apply_mask_drops_null_rowids() { + // A null id cannot be in any allow set, so it is dropped. + let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64, 2, 3])); + let batch = batch_with_rowids(vec![Some(1), None, Some(3)]); + let out = apply_mask(&mask, batch).unwrap(); + assert_eq!(kept_rowids(&out), vec![Some(1), Some(3)]); + } + + #[test] + fn apply_mask_missing_rowid_column_errs() { + let schema = Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, false)])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2]))]).unwrap(); + let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64])); + let err = apply_mask(&mask, batch).unwrap_err(); + assert!(matches!(err, DataFusionError::Internal(_)), "got {err:?}"); + let msg = err.to_string(); + assert!( + msg.contains(ROW_ID) && msg.contains("missing"), + "unexpected: {msg}" + ); + } + + #[test] + fn apply_mask_wrong_type_rowid_column_errs() { + // _rowid present but not UInt64 -> Internal error naming the actual type. + let schema = Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::Int32, + false, + )])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1, 2]))]).unwrap(); + let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64])); + let err = apply_mask(&mask, batch).unwrap_err(); + assert!(matches!(err, DataFusionError::Internal(_)), "got {err:?}"); + let msg = err.to_string(); + assert!( + msg.contains("UInt64") && msg.contains("Int32"), + "unexpected: {msg}" + ); + } + + struct FixedLoader(RowAddrMask); + + #[async_trait] + impl FilterLoader for FixedLoader { + async fn load(self: Box) -> Result { + Ok(self.0) + } + } + + #[tokio::test] + async fn mask_and_loader_without_inner_returns_mask() { + let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64, 2, 3])); + let loaded = Box::new(MaskAndLoader::new(Arc::new(mask), None)) + .load() + .await + .unwrap(); + assert!(loaded.selected(2)); + assert!(!loaded.selected(4)); + } + + #[tokio::test] + async fn mask_and_loader_with_inner_intersects() { + // {1,2,3,4} AND inner {2,4,6} = {2,4}. + let mask = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64, 2, 3, 4])); + let inner = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([2u64, 4, 6])); + let loaded = Box::new(MaskAndLoader::new( + Arc::new(mask), + Some(Box::new(FixedLoader(inner))), + )) + .load() + .await + .unwrap(); + assert!(loaded.selected(2)); + assert!(loaded.selected(4)); + assert!(!loaded.selected(1)); + assert!(!loaded.selected(6)); + } + + #[tokio::test] + async fn mask_and_loader_block_and_allow() { + // block{2} AND allow{1,2,3} = allow({1,2,3} - {2}) = allow{1,3}. + let mask = RowAddrMask::from_block(RowAddrTreeMap::from_iter([2u64])); + let inner = RowAddrMask::from_allowed(RowAddrTreeMap::from_iter([1u64, 2, 3])); + let loaded = Box::new(MaskAndLoader::new( + Arc::new(mask), + Some(Box::new(FixedLoader(inner))), + )) + .load() + .await + .unwrap(); + assert!(loaded.selected(1)); + assert!(loaded.selected(3)); + assert!(!loaded.selected(2)); + assert!(!loaded.selected(4)); + } + + #[tokio::test] + async fn mask_and_loader_block_and_block() { + // block{1} AND block{2} = block{1,2}: everything except 1 and 2 is selected. + let mask = RowAddrMask::from_block(RowAddrTreeMap::from_iter([1u64])); + let inner = RowAddrMask::from_block(RowAddrTreeMap::from_iter([2u64])); + let loaded = Box::new(MaskAndLoader::new( + Arc::new(mask), + Some(Box::new(FixedLoader(inner))), + )) + .load() + .await + .unwrap(); + assert!(!loaded.selected(1)); + assert!(!loaded.selected(2)); + assert!(loaded.selected(3)); + } +} diff --git a/rust/lance/src/io/exec/rowids.rs b/rust/lance/src/io/exec/rowids.rs index 837d0b81fa3..094bf5bd91e 100644 --- a/rust/lance/src/io/exec/rowids.rs +++ b/rust/lance/src/io/exec/rowids.rs @@ -136,7 +136,9 @@ impl AddRowAddrExec { let mut builder = arrow::array::UInt64Builder::with_capacity(row_id_values.len()); for rowid in row_id_values.iter() { if let Some(rowid) = rowid { - if let Some(row_addr) = row_id_index.get(rowid) { + if let Some(row_addr) = + row_id_index.get(rowid).map_err(DataFusionError::from)? + { builder.append_value(row_addr.into()); } else { return Err(DataFusionError::Internal(format!( @@ -153,7 +155,9 @@ impl AddRowAddrExec { // Fast path - no branching for null values let mut rowaddrs: Vec = Vec::with_capacity(row_id_values.len()); for rowid in row_id_values.values() { - if let Some(row_addr) = row_id_index.get(*rowid) { + if let Some(row_addr) = + row_id_index.get(*rowid).map_err(DataFusionError::from)? + { rowaddrs.push(row_addr.into()); } else { return Err(DataFusionError::Internal(format!( @@ -242,10 +246,6 @@ impl ExecutionPlan for AddRowAddrExec { "AddRowAddrExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> Arc { self.output_schema.clone() } @@ -291,8 +291,8 @@ impl ExecutionPlan for AddRowAddrExec { fn partition_statistics( &self, partition: Option, - ) -> Result { - let mut stats = self.input.partition_statistics(partition)?; + ) -> Result> { + let mut stats = Arc::unwrap_or_clone(self.input.partition_statistics(partition)?); let row_id_col_stats = stats.column_statistics.get(self.rowid_pos).ok_or_else(|| { DataFusionError::Internal("RowAddrExec: rowid column stats not found".into()) @@ -327,7 +327,7 @@ impl ExecutionPlan for AddRowAddrExec { .column_statistics .insert(self.rowaddr_pos, row_addr_col_stats); - Ok(stats) + Ok(Arc::new(stats)) } fn metrics(&self) -> Option { @@ -442,15 +442,19 @@ impl AddRowOffsetExec { frag_id_to_offset: &HashMap, ) -> Result { let row_addr_values = row_addr.as_primitive::().values(); - let mut row_offsets = Vec::with_capacity(row_addr_values.len()); + let mut row_offsets = vec![0; row_addr_values.len()]; + // The deletion iterator only moves forward, so compute in address order and + // scatter the offsets back into input order. + let mut sorted_row_indices = (0..row_addr_values.len()).collect::>(); + sorted_row_indices.sort_unstable_by_key(|index| row_addr_values[*index]); let mut last_frag_id = u32::MAX; let mut last_frag_offset = 0; let mut last_frag_delete_count = 0; let mut dv_iter = None; - for addr in row_addr_values { - let addr = RowAddress::new_from_u64(*addr); + for row_index in sorted_row_indices { + let addr = RowAddress::new_from_u64(row_addr_values[row_index]); let frag_id = addr.fragment_id(); if frag_id != last_frag_id { last_frag_id = frag_id; @@ -480,10 +484,10 @@ impl AddRowOffsetExec { break; } } - row_offsets - .push(last_frag_offset + row_offset as u64 - last_frag_delete_count as u64); + row_offsets[row_index] = + last_frag_offset + row_offset as u64 - last_frag_delete_count as u64; } else { - row_offsets.push(last_frag_offset + row_offset as u64); + row_offsets[row_index] = last_frag_offset + row_offset as u64; } } @@ -506,10 +510,6 @@ impl ExecutionPlan for AddRowOffsetExec { "AddRowOffsetExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn properties(&self) -> &Arc { &self.properties } @@ -526,7 +526,7 @@ impl ExecutionPlan for AddRowOffsetExec { vec![false] } - fn partition_statistics(&self, partition: Option) -> Result { + fn partition_statistics(&self, partition: Option) -> Result> { self.input.partition_statistics(partition) } @@ -735,4 +735,27 @@ mod test { .fold(0, |acc, col| acc + col.get_array_memory_size()); assert_eq!(stats.total_byte_size, Precision::Exact(actual_byte_size)); } + + #[test] + fn test_row_offsets_with_unsorted_addresses() { + let row_addrs: ArrayRef = Arc::new(UInt64Array::from(vec![ + u64::from(RowAddress::new_from_parts(0, 100)), + u64::from(RowAddress::new_from_parts(0, 50)), + ])); + let frag_id_to_offset = HashMap::from([( + 0, + FragInfo { + row_offset: 1_000, + deletion_vector: Some(Arc::new(DeletionVector::from_iter([10, 60]))), + }, + )]); + + let row_offsets = + AddRowOffsetExec::compute_row_offsets(&row_addrs, &frag_id_to_offset).unwrap(); + + assert_eq!( + row_offsets.as_primitive::().values(), + &[1_098, 1_049] + ); + } } diff --git a/rust/lance/src/io/exec/scalar_index.rs b/rust/lance/src/io/exec/scalar_index.rs index 0f74a13478d..0bd37170c76 100644 --- a/rust/lance/src/io/exec/scalar_index.rs +++ b/rust/lance/src/io/exec/scalar_index.rs @@ -1,22 +1,24 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors +use std::collections::HashSet; use std::sync::{Arc, LazyLock}; use super::utils::{IndexMetrics, InstrumentedRecordBatchStreamAdapter}; use crate::{ Dataset, - dataset::rowids::load_row_id_sequences, + dataset::rowids::{load_row_id_sequences, translate_addr_treemap_to_row_ids}, index::{ prefilter::DatasetPreFilter, scalar_logical::{open_named_scalar_index, scalar_index_fragment_bitmap}, }, }; -use arrow_array::{Array, RecordBatch, UInt64Array}; +use arrow_array::{Array, ArrayRef, RecordBatch, UInt64Array, cast::AsArray, types::UInt64Type}; use arrow_schema::{Schema, SchemaRef}; use async_recursion::async_recursion; use async_trait::async_trait; use datafusion::{ + execution::memory_pool::{MemoryConsumer, MemoryReservation}, physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, execution_plan::{Boundedness, EmissionType}, @@ -27,7 +29,7 @@ use datafusion::{ }; use datafusion_physical_expr::EquivalenceProperties; use futures::{StreamExt, TryFutureExt, TryStreamExt, stream::BoxStream}; -use lance_core::{Error, ROW_ID_FIELD, Result, utils::address::RowAddress}; +use lance_core::{Error, ROW_ID_FIELD, Result, deepsize::DeepSizeOf, utils::address::RowAddress}; use lance_datafusion::{ chunker::break_stream, utils::{ @@ -42,7 +44,8 @@ use lance_index::{ }, }; use lance_select::{ - IndexExprResult, RowAddrMask, RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat, + IndexExprResult, NullableIndexExprResult, NullableRowAddrMask, NullableRowAddrSet, RowAddrMask, + RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat, }; use lance_table::format::Fragment; use roaring::RoaringBitmap; @@ -58,6 +61,50 @@ impl ScalarIndexLoader for Dataset { ) -> Result> { open_named_scalar_index(self, column, index_name, metrics).await } + + async fn row_addr_result_to_row_ids( + &self, + result: NullableIndexExprResult, + ) -> Result { + // Addresses and row ids only diverge under stable row ids; otherwise the + // address is the row id and there is nothing to translate. + if !self.manifest.uses_stable_row_ids() { + return Ok(result); + } + + let NullableIndexExprResult { lower, upper, .. } = result; + let lower = translate_addr_mask_to_row_ids(self, lower).await?; + let upper = translate_addr_mask_to_row_ids(self, upper).await?; + Ok(NullableIndexExprResult::new(lower, upper)) + } +} + +/// Translate an address-domain [`NullableRowAddrMask`] into the row-id domain +/// +/// Address-domain index results are always positive allow-lists (`AtMost`), so +/// a block-list here would mean a boolean op was applied before translation, +/// which is unsupported. +async fn translate_addr_mask_to_row_ids( + dataset: &Dataset, + mask: NullableRowAddrMask, +) -> Result { + match mask { + NullableRowAddrMask::AllowList(set) => Ok(NullableRowAddrMask::AllowList( + translate_addr_set_to_row_ids(dataset, set).await?, + )), + NullableRowAddrMask::BlockList(_) => Err(Error::internal( + "cannot translate a block-list address mask to the row-id domain", + )), + } +} + +async fn translate_addr_set_to_row_ids( + dataset: &Dataset, + set: NullableRowAddrSet, +) -> Result { + let selected = translate_addr_treemap_to_row_ids(dataset, set.selected_rows()).await?; + let nulls = translate_addr_treemap_to_row_ids(dataset, set.null_rows()).await?; + Ok(NullableRowAddrSet::new(selected, nulls)) } /// An execution node that performs a scalar index search @@ -182,10 +229,6 @@ impl ExecutionPlan for ScalarIndexExec { "ScalarIndexExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.result_format.schema().clone() } @@ -233,11 +276,11 @@ impl ExecutionPlan for ScalarIndexExec { fn partition_statistics( &self, _partition: Option, - ) -> datafusion::error::Result { - Ok(datafusion::physical_plan::Statistics { + ) -> datafusion::error::Result> { + Ok(Arc::new(datafusion::physical_plan::Statistics { num_rows: datafusion::common::stats::Precision::Exact(2), ..datafusion::physical_plan::Statistics::new_unknown(self.result_format.schema()) - }) + })) } fn metrics(&self) -> Option { @@ -267,6 +310,82 @@ pub struct IndexLookup { pub index_name: String, } +const MAP_INDEX_CANDIDATES_MEMORY_CONSUMER: &str = "MapIndexExecCandidates"; + +/// Per-address allowance for cooperative pool accounting of the retained map. +/// This conservatively bounds its approximate [`DeepSizeOf`] growth, but is not +/// an allocator-level peak-memory bound. The reservation is shrunk to the +/// measured retained-map size after each input batch. +const ROW_ADDR_INSERT_RESERVATION_BYTES: usize = 256; + +#[derive(Debug)] +struct DistinctRowAddrs { + emitted: RowAddrTreeMap, + reservation: MemoryReservation, +} + +impl DistinctRowAddrs { + fn new(reservation: MemoryReservation) -> Self { + Self { + emitted: RowAddrTreeMap::new(), + reservation, + } + } + + fn retain_unseen(&mut self, row_addrs: &UInt64Array) -> datafusion::error::Result { + let initial_reservation_size = self.reservation.size(); + // The first batch also has to account for the empty map's inline size, + // which is not part of the initially empty reservation. + let retained_size = initial_reservation_size.max(std::mem::size_of::()); + let unaccounted_candidates = row_addrs + .values() + .iter() + .filter(|row_addr| !self.emitted.contains(**row_addr)) + .count(); + let provisional_size = unaccounted_candidates + .checked_mul(ROW_ADDR_INSERT_RESERVATION_BYTES) + .and_then(|additional| retained_size.checked_add(additional)) + .ok_or_else(|| { + datafusion::error::DataFusionError::ResourcesExhausted(format!( + "Candidate memory reservation overflowed for {MAP_INDEX_CANDIDATES_MEMORY_CONSUMER}" + )) + })?; + self.reservation.try_resize(provisional_size)?; + + // Allocate output storage only after the complete retained-state + // reservation succeeds. + let mut unseen = Vec::with_capacity(unaccounted_candidates); + for row_addr in row_addrs.values() { + if self.emitted.insert(*row_addr) { + unseen.push(*row_addr); + } + } + + let measured_size = self.emitted.deep_size_of(); + if measured_size > provisional_size { + self.rollback_batch(&unseen, initial_reservation_size); + return Err(datafusion::error::DataFusionError::ResourcesExhausted( + format!( + "MapIndexExecCandidates batch exceeded its {ROW_ADDR_INSERT_RESERVATION_BYTES}-byte per-candidate reservation" + ), + )); + } + self.reservation.resize(measured_size); + Ok(UInt64Array::from(unseen)) + } + + fn rollback_batch(&mut self, unseen: &[u64], reservation_size: usize) { + for row_addr in unseen { + let is_removed = self.emitted.remove(*row_addr); + debug_assert!( + is_removed, + "a newly inserted MapIndexExec candidate must be removable" + ); + } + self.reservation.resize(reservation_size); + } +} + impl IndexLookup { pub fn new(column: impl Into, index_name: impl Into) -> Self { Self { @@ -282,10 +401,13 @@ impl IndexLookup { /// /// Multiple `(column, index_name)` lookups can be supplied: the operator /// expects one input column per lookup (in matching order) and emits the -/// row addresses where every column's value is present in its respective -/// index — that is, the AND of the per-column index probes. This lets a -/// composite-key join trim the candidate row set with every available -/// scalar index before the downstream take. +/// row addresses that could match on every column. The result is an upper +/// bound, not an exact set — the probes are evaluated one key at a time, +/// most-selective first, and stop as soon as the candidate set is no larger +/// than the input batch, so a caller must still filter on the full key. +/// This lets a composite-key join trim the candidate row set before the +/// downstream take without paying for a probe that prunes nothing. A row +/// address reached by more than one input batch is emitted only once. #[derive(Debug)] pub struct MapIndexExec { dataset: Arc, @@ -332,9 +454,10 @@ impl MapIndexExec { ) } - /// Build a `MapIndexExec` that probes one or more scalar indices and - /// emits the AND of their results. `lookups` must be non-empty and - /// `input` must produce one column per lookup, in the same order. + /// Build a `MapIndexExec` that probes one or more scalar indices and emits + /// an upper bound on the row addresses matching every one of them (see the + /// type docs). `lookups` must be non-empty and `input` must produce one + /// column per lookup, in the same order. pub fn new_multi( dataset: Arc, lookups: Vec, @@ -366,11 +489,21 @@ impl MapIndexExec { lookups: Vec, index_metrics: Arc, metrics_set: ExecutionPlanMetricsSet, + candidate_reservation: MemoryReservation, ) -> datafusion::error::Result { // A row can be found by the composite probe only if it lives in a // fragment covered by *every* index in `lookups`; restrict the // deletion mask to that intersection so we only filter deletes we // could actually see. + // + // This loop must keep covering every lookup even though `map_batch` + // may skip some probes. A skipped probe leaves candidates from + // fragments that its index does not cover, and the restricted mask is + // the only thing that then blocks them. Those fragments are read + // separately by the unindexed-fragment scan in + // `create_indexed_scan_joined_stream`, so letting candidates through + // would feed the same target row into the join twice, which the + // default `SourceDedupeBehavior::Fail` reports as an error. let mut fragment_bitmap: Option = None; for lookup in &lookups { let bm = scalar_index_fragment_bitmap(&dataset, &lookup.column, &lookup.index_name) @@ -412,6 +545,19 @@ impl MapIndexExec { Self::map_batch(lookups, dataset, deletion_mask, batch, metrics).await } }); + let mut distinct_row_addrs = DistinctRowAddrs::new(candidate_reservation); + let stream = stream.and_then(move |batch| { + // Each batch's index result is already a set. Retain first + // occurrences here to make the complete candidate stream a set too. + let row_addrs = batch.column(0).as_primitive::(); + let result = distinct_row_addrs + .retain_unseen(row_addrs) + .and_then(|unseen| { + RecordBatch::try_new(INDEX_LOOKUP_SCHEMA.clone(), vec![Arc::new(unseen)]) + .map_err(datafusion::error::DataFusionError::from) + }); + futures::future::ready(result) + }); let stream = stream.map(move |batch| { let poll = baseline.record_poll(std::task::Poll::Ready(Some(batch))); match poll { @@ -425,32 +571,43 @@ impl MapIndexExec { ))) } - /// Build the AND-of-IsIn `ScalarIndexExpr` describing this batch's - /// composite lookup: each input column contributes one `IsIn` query - /// against its matching index. - fn build_query( - lookups: &[IndexLookup], - batch: &RecordBatch, - ) -> datafusion::error::Result { - let per_column = lookups.iter().enumerate().map(|(idx, lookup)| { - let column = batch.column(idx); - let values = (0..column.len()) - .map(|row| ScalarValue::try_from_array(column, row)) - .collect::>>()?; - Ok::<_, datafusion::error::DataFusionError>(ScalarIndexExpr::Query(ScalarIndexSearch { - column: lookup.column.clone(), - index_name: lookup.index_name.clone(), - // Internal IndexedLookup-style query — type is unknown at this layer - index_type: String::new(), - query: Arc::new(SargableQuery::IsIn(values)), - needs_recheck: false, - fragment_bitmap: None, - })) - }); + /// The values of one input column, deduped when `dedupe` is set. + /// + /// Deduping earns its keep once there is more than one key: the distinct + /// count doubles as the probe-ordering signal in [`Self::map_batch`], and a + /// repeated value only adds an `IsIn` entry that re-selects index pages + /// already selected. With a single key there is nothing to order, so + /// hashing every value would be pure overhead on the most common path. + /// + /// One NULL survives dedupe on purpose: an index reads a NULL in the list + /// as "also match null rows", which is a flag rather than a count. + fn key_values(column: &ArrayRef, dedupe: bool) -> datafusion::error::Result> { + let mut values = Vec::with_capacity(column.len()); + let mut seen = HashSet::with_capacity(if dedupe { column.len() } else { 0 }); + for row in 0..column.len() { + let value = ScalarValue::try_from_array(column, row)?; + if dedupe { + if seen.contains(&value) { + continue; + } + seen.insert(value.clone()); + } + values.push(value); + } + Ok(values) + } - per_column - .reduce(|lhs, rhs| Ok(ScalarIndexExpr::And(Box::new(lhs?), Box::new(rhs?)))) - .expect("MapIndexExec built with no lookups") + /// Build the `IsIn` query for one join key against its matching index. + fn build_key_query(lookup: &IndexLookup, values: Vec) -> ScalarIndexExpr { + ScalarIndexExpr::Query(ScalarIndexSearch { + column: lookup.column.clone(), + index_name: lookup.index_name.clone(), + // Internal IndexedLookup-style query — type is unknown at this layer + index_type: String::new(), + query: Arc::new(SargableQuery::IsIn(values)), + needs_recheck: false, + fragment_bitmap: None, + }) } async fn map_batch( @@ -460,12 +617,89 @@ impl MapIndexExec { batch: RecordBatch, metrics: Arc, ) -> datafusion::error::Result { - let query = Self::build_query(&lookups, &batch)?; - let query_result = query.evaluate(dataset.as_ref(), metrics.as_ref()).await?; - if !query_result.is_exact() { - todo!("Support for non-exact query results as input for merge_insert") + // The operator's contract is one input column per lookup, in order (see + // `new_multi`). Check it here rather than letting `batch.column` panic + // deep inside a DataFusion stream, and check it before the probe loop: + // the loop can skip the offending lookup, which would turn a broken + // plan into a failure that depends on the data. + if lookups.len() != batch.num_columns() { + return Err(datafusion::error::DataFusionError::Internal(format!( + "MapIndexExec has {} lookups but its input produced {} columns", + lookups.len(), + batch.num_columns() + ))); + } + + // Probe the keys one at a time and intersect, rather than evaluating an + // AND of every probe at once. A join key whose values repeat across the + // target (a bucket or status column) matches most of the table, so its + // probe materializes a candidate set the size of the dataset while + // pruning almost nothing: on a 10M-row table, probing a 1024-distinct + // key asked for 2.4 GB of candidates. + // With one key there is nothing to order and nothing a second probe + // could intersect away, so that path stays exactly as it was. + let several_keys = lookups.len() > 1; + let mut values_per_key = Vec::with_capacity(lookups.len()); + for column in batch.columns() { + values_per_key.push(Self::key_values(column, several_keys)?); + } + + // Probe the most selective key first. The key with the most distinct + // source values partitions the target most finely, so its probe is the + // one most likely to leave a candidate set small enough to skip the + // rest. Following the caller's `on` order instead would run the + // expensive probe first, because the natural way to write a composite + // key is coarse-to-fine (`["tenant_id", "row_id"]`). The distinct count + // is a source-side proxy for target-side selectivity, and it only knows + // how many values a probe will look up, not how many target rows each + // of them matches. A skewed batch — many distinct values that each + // match many rows, sitting next to few values that each match few — + // can therefore be ordered worse than the caller wrote it. The floor is + // the old behaviour's probe set — run sequentially, see below — because + // the break only ever removes probes. + let mut probe_order: Vec = (0..lookups.len()).collect(); + if several_keys { + // Stable, so keys with equally distinct values keep `on` order. + probe_order.sort_by_key(|&idx| std::cmp::Reverse(values_per_key[idx].len())); + } + + // Stop once the candidate set is no larger than the source batch. A + // further probe could still remove false positives, but what is left to + // remove is bounded by the source batch, so it cannot save more work + // downstream than the probe itself costs. That bound is the point: per + // batch the emitted set is either the full intersection or at most + // `batch.num_rows()` rows, which is also what keeps the cross-batch + // candidate set in `DistinctRowAddrs` bounded. + // + // Skipping a probe only leaves extra candidates, never drops a match: + // the downstream hash join filters on the full composite key (see + // `create_indexed_scan_joined_stream`), the same reason an unindexed + // `on` column is allowed to prune nothing. Extra candidates stay inside + // the index-covered fragments because the restricted deletion mask + // applied below is built from *every* lookup's fragment bitmap. + // + // The probes run in sequence where the old `ScalarIndexExpr::And` ran + // them concurrently. That is the price of being able to stop, and it + // shows up only when no probe is skipped and the index cache is cold. + let source_rows = batch.num_rows() as u64; + // `all_rows()` is the identity for `intersect`, so the first probe + // needs no special case. + let mut row_addr_mask = RowAddrMask::all_rows(); + for idx in probe_order { + if row_addr_mask + .max_len() + .is_some_and(|len| len <= source_rows) + { + break; + } + let values = std::mem::take(&mut values_per_key[idx]); + let query = Self::build_key_query(&lookups[idx], values); + let query_result = query.evaluate(dataset.as_ref(), metrics.as_ref()).await?; + if !query_result.is_exact() { + todo!("Support for non-exact query results as input for merge_insert") + } + row_addr_mask = row_addr_mask.intersect(query_result.upper); } - let mut row_addr_mask = query_result.upper; if let Some(deletion_mask) = deletion_mask.as_ref() { row_addr_mask = row_addr_mask & deletion_mask.as_ref().clone(); @@ -489,10 +723,6 @@ impl ExecutionPlan for MapIndexExec { "MapIndexExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { INDEX_LOOKUP_SCHEMA.clone() } @@ -523,6 +753,10 @@ impl ExecutionPlan for MapIndexExec { partition: usize, context: Arc, ) -> datafusion::error::Result { + // Cross-batch deduplication retains every emitted candidate until the + // stream ends, so this state is not covered by per-batch reservations. + let candidate_reservation = MemoryConsumer::new(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER) + .register(context.memory_pool()); let input = self.input.execute(partition, context)?; let stream_fut = Self::build_stream( input, @@ -531,6 +765,7 @@ impl ExecutionPlan for MapIndexExec { self.lookups.clone(), Arc::new(IndexMetrics::new(&self.metrics, partition)), self.metrics.clone(), + candidate_reservation, ); let stream = futures::stream::once(stream_fut).try_flatten(); Ok(Box::pin(RecordBatchStreamAdapter::new( @@ -561,6 +796,10 @@ pub struct MaterializeIndexExec { dataset: Arc, expr: ScalarIndexExpr, fragments: Arc>, + /// Row addresses blocked from the index result due to data overlay files committed after the + /// index was built. ANDead into the candidate mask before row ID materialisation so that stale + /// index entries never reach downstream operators. + overlay_block: Option, properties: Arc, metrics: ExecutionPlanMetricsSet, } @@ -633,16 +872,24 @@ impl MaterializeIndexExec { dataset, expr, fragments, + overlay_block: None, properties, metrics: ExecutionPlanMetricsSet::new(), } } + /// Block specific row addresses (see the `overlay_block` field) from the index result. + pub fn with_overlay_block(mut self, block: RowAddrMask) -> Self { + self.overlay_block = Some(block); + self + } + #[instrument(name = "materialize_scalar_index", skip_all, level = "debug")] async fn do_execute( expr: ScalarIndexExpr, dataset: Arc, fragments: Arc>, + overlay_block: Option, metrics: Arc, ) -> Result { let expr_result = expr.evaluate(dataset.as_ref(), metrics.as_ref()); @@ -670,12 +917,15 @@ impl MaterializeIndexExec { } Ok(result.upper) }; - let mask = if let Some(prefilter) = prefilter { + let mut mask = if let Some(prefilter) = prefilter { let (expr_result, prefilter) = futures::try_join!(expr_result, prefilter)?; take_upper(expr_result)? & (*prefilter).clone() } else { take_upper(expr_result.await?)? }; + if let Some(block) = overlay_block { + mask = mask & block; + } let ids = row_ids_for_mask(mask, &dataset, &fragments).await?; let ids = UInt64Array::from(ids); Ok(RecordBatch::try_new( @@ -776,10 +1026,6 @@ impl ExecutionPlan for MaterializeIndexExec { "MaterializeIndexExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { MATERIALIZE_INDEX_SCHEMA.clone() } @@ -811,6 +1057,7 @@ impl ExecutionPlan for MaterializeIndexExec { self.expr.clone(), self.dataset.clone(), self.fragments.clone(), + self.overlay_block.clone(), metrics, ); let stream = futures::stream::iter(vec![batch_fut]) @@ -849,13 +1096,24 @@ mod tests { use crate::index::DatasetIndexExt; use arrow::datatypes::UInt64Type; + use arrow::record_batch::RecordBatchIterator; + use arrow_array::{ArrayRef, Int32Array, RecordBatch, UInt64Array}; use arrow_schema::Schema; use datafusion::{ - execution::TaskContext, physical_plan::ExecutionPlan, prelude::SessionConfig, + error::DataFusionError, + execution::{ + TaskContext, + memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryPool}, + }, + physical_plan::ExecutionPlan, + prelude::SessionConfig, scalar::ScalarValue, }; use futures::TryStreamExt; - use lance_core::utils::tempfile::TempStrDir; + use lance_core::{ + deepsize::DeepSizeOf, + utils::{address::RowAddress, tempfile::TempStrDir}, + }; use lance_datagen::gen_batch; use lance_index::{ IndexType, @@ -864,15 +1122,19 @@ mod tests { expression::{ScalarIndexExpr, ScalarIndexSearch}, }, }; - use lance_select::result::IndexExprResultWireFormat; + use lance_select::{RowAddrTreeMap, RowSetOps, result::IndexExprResultWireFormat}; use crate::{ Dataset, + dataset::WriteParams, io::exec::scalar_index::MaterializeIndexExec, utils::test::{DatagenExt, FragmentCount, FragmentRowCount, NoContextTestFixture}, }; - use super::{MapIndexExec, ScalarIndexExec}; + use super::{ + DistinctRowAddrs, MAP_INDEX_CANDIDATES_MEMORY_CONSUMER, MapIndexExec, + ROW_ADDR_INSERT_RESERVATION_BYTES, ScalarIndexExec, + }; struct TestFixture { dataset: Arc, @@ -910,6 +1172,66 @@ mod tests { } } + #[test] + fn test_map_index_candidates_accept_empty_first_batch() { + let pool: Arc = Arc::new(GreedyMemoryPool::new(1024)); + let reservation = MemoryConsumer::new(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER).register(&pool); + let mut candidates = DistinctRowAddrs::new(reservation); + + let empty = candidates + .retain_unseen(&UInt64Array::from(Vec::::new())) + .unwrap(); + assert!(empty.is_empty()); + assert_eq!(pool.reserved(), RowAddrTreeMap::new().deep_size_of()); + } + + #[test] + fn test_map_index_candidates_respect_memory_pool() { + let mut first_candidate = RowAddrTreeMap::new(); + first_candidate.insert(1); + let first_candidate_size = first_candidate.deep_size_of(); + let pool: Arc = Arc::new(GreedyMemoryPool::new( + ROW_ADDR_INSERT_RESERVATION_BYTES + first_candidate_size, + )); + let reservation = MemoryConsumer::new(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER).register(&pool); + let mut candidates = DistinctRowAddrs::new(reservation); + + let first = candidates + .retain_unseen(&UInt64Array::from(vec![1])) + .unwrap(); + assert_eq!(first.values(), &[1]); + assert_eq!(pool.reserved(), first_candidate_size); + + let second = candidates + .retain_unseen(&UInt64Array::from(vec![2])) + .unwrap(); + assert_eq!(second.values(), &[2]); + let retained_size = pool.reserved(); + assert!(retained_size > first_candidate_size); + + let error = candidates + .retain_unseen(&UInt64Array::from(vec![3])) + .unwrap_err(); + assert!(matches!(error, DataFusionError::ResourcesExhausted(_))); + assert!( + error + .to_string() + .contains(MAP_INDEX_CANDIDATES_MEMORY_CONSUMER) + ); + assert!(candidates.emitted.contains(1)); + assert!(candidates.emitted.contains(2)); + assert!(!candidates.emitted.contains(3)); + assert_eq!(pool.reserved(), retained_size); + + let duplicate = candidates + .retain_unseen(&UInt64Array::from(vec![1, 2])) + .unwrap(); + assert!(duplicate.values().is_empty()); + + drop(candidates); + assert_eq!(pool.reserved(), 0); + } + #[tokio::test] async fn test_materialize_index_exec() { let TestFixture { @@ -928,7 +1250,6 @@ mod tests { needs_recheck: false, fragment_bitmap: None, }); - let fragments = dataset.fragments().clone(); let plan = MaterializeIndexExec::new(dataset, query, fragments); @@ -949,6 +1270,51 @@ mod tests { assert_eq!(batches[0].num_rows(), 5); } + #[tokio::test] + async fn test_translate_addr_treemap_to_stable_row_ids() { + let test_dir = TempStrDir::default(); + let batch = RecordBatch::try_from_iter(vec![( + "id", + Arc::new(Int32Array::from((0..10).collect::>())) as ArrayRef, + )]) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let write_params = WriteParams { + enable_stable_row_ids: true, + max_rows_per_file: 5, + ..Default::default() + }; + let dataset = Dataset::write(reader, test_dir.as_str(), Some(write_params)) + .await + .unwrap(); + let fragment_id = dataset.get_fragments()[1].id() as u32; + + let mut full_fragment = RowAddrTreeMap::new(); + full_fragment.insert_fragment(fragment_id); + let translated = super::translate_addr_treemap_to_row_ids(&dataset, &full_fragment) + .await + .unwrap(); + let row_ids = translated + .get_fragment_bitmap(0) + .unwrap() + .iter() + .collect::>(); + assert_eq!(row_ids, vec![5, 6, 7, 8, 9]); + + let mut partial_fragment = RowAddrTreeMap::new(); + partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 1).into()); + partial_fragment.insert(RowAddress::new_from_parts(fragment_id, 3).into()); + let translated = super::translate_addr_treemap_to_row_ids(&dataset, &partial_fragment) + .await + .unwrap(); + let row_ids = translated + .get_fragment_bitmap(0) + .unwrap() + .iter() + .collect::>(); + assert_eq!(row_ids, vec![6, 8]); + } + /// `ScalarIndexExec::schema()` (and the stream it emits) must advertise /// the same schema the batch actually carries — otherwise downstream /// consumers that trust `ExecutionPlan::schema()` will see a different diff --git a/rust/lance/src/io/exec/scan.rs b/rust/lance/src/io/exec/scan.rs index 3ec63ce04cc..8ad4a4b32b6 100644 --- a/rust/lance/src/io/exec/scan.rs +++ b/rust/lance/src/io/exec/scan.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::any::Any; use std::ops::Range; use std::pin::Pin; use std::sync::Arc; @@ -26,7 +25,10 @@ use futures::{StreamExt, TryStreamExt}; use lance_arrow::SchemaExt; use lance_core::utils::tokio::get_num_compute_intensive_cpus; use lance_core::utils::tracing::StreamTracingExt; -use lance_core::{Error, ROW_ADDR_FIELD, ROW_ID_FIELD}; +use lance_core::{ + Error, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION_FIELD, ROW_ID_FIELD, + ROW_LAST_UPDATED_AT_VERSION_FIELD, +}; use lance_file::reader::FileReaderOptions; use lance_io::scheduler::{ScanScheduler, SchedulerConfig}; use lance_table::format::Fragment; @@ -41,7 +43,7 @@ use crate::dataset::scanner::{ }; use crate::datatypes::Schema; -use super::utils::IoMetrics; +use super::utils::{IoMetrics, buffered_fragment_opens}; async fn open_file( file_fragment: FileFragment, @@ -166,18 +168,10 @@ impl LanceStream { metrics: &ExecutionPlanMetricsSet, partition: usize, ) -> Result { - let is_v2_scan = fragments - .iter() - .filter_map(|frag| frag.files.first().map(|f| !f.is_legacy_file())) - .next() - .unwrap_or(false); - if is_v2_scan { - Self::try_new_v2( - dataset, fragments, offsets, projection, config, metrics, partition, - ) - } else { - Self::try_new_v1(dataset, fragments, projection, config, metrics, partition) - } + let version = dataset.manifest().data_storage_format.lance_file_format(); + crate::dataset::versions::create_scan_stream( + version, dataset, fragments, offsets, projection, config, metrics, partition, + ) } #[allow(clippy::too_many_arguments)] @@ -192,7 +186,36 @@ impl LanceStream { ) -> Result { let scan_metrics = ScanMetrics::new(metrics, partition); let timer = scan_metrics.baseline_metrics.elapsed_compute().timer(); - let project_schema = projection.clone(); + let materialize_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(projection.as_ref()); + let read_projection = if materialize_blob_v2_binary { + Arc::new(crate::dataset::blob::blob_v2_descriptor_schema( + projection.as_ref(), + )) + } else { + projection.clone() + }; + let project_schema = read_projection; + let output_projection = if materialize_blob_v2_binary { + let mut output_projection = projection.as_ref().clone(); + let mut system_fields = Vec::with_capacity(4); + if config.with_row_id { + system_fields.push(ROW_ID_FIELD.clone()); + } + if config.with_row_address { + system_fields.push(ROW_ADDR_FIELD.clone()); + } + if config.with_row_last_updated_at_version { + system_fields.push(ROW_LAST_UPDATED_AT_VERSION_FIELD.clone()); + } + if config.with_row_created_at_version { + system_fields.push(ROW_CREATED_AT_VERSION_FIELD.clone()); + } + output_projection.extend(&system_fields)?; + Arc::new(output_projection) + } else { + projection.clone() + }; let io_parallelism = dataset.object_store.io_parallelism(); // First, use the value specified by the user in the call // Second, use the default from the environment variable, if specified @@ -275,12 +298,18 @@ impl LanceStream { let scan_scheduler_clone = scan_scheduler.clone(); + let materialize_dataset = dataset; + let materialization_context = crate::dataset::blob::BlobMaterializationContext::new( + Some(config.io_buffer_size), + config.materialization_readahead_bytes, + ); let config_for_stream = config.clone(); let batches = stream::iter(file_fragments.into_iter().enumerate()) .map(move |(priority, file_fragment)| { let project_schema = project_schema.clone(); let scan_scheduler = scan_scheduler.clone(); let config = config_for_stream.clone(); + let force_row_address = materialize_blob_v2_binary; #[allow(clippy::type_complexity)] let frag_task: BoxFuture< Result>>>>, @@ -288,7 +317,7 @@ impl LanceStream { (async move { let mut frag_config = FragReadConfig::default() .with_row_id(config.with_row_id) - .with_row_address(config.with_row_address) + .with_row_address(config.with_row_address || force_row_address) .with_row_last_updated_at_version( config.with_row_last_updated_at_version, ) @@ -349,6 +378,31 @@ impl LanceStream { ) .stream_in_current_span() .boxed(); + let inner_stream = if materialize_blob_v2_binary { + inner_stream + .map_ok(move |batch| { + let dataset = materialize_dataset.clone(); + let output_projection = output_projection.clone(); + let materialization_context = materialization_context.clone(); + let admission = materialization_context.admission(); + async move { + crate::dataset::blob::materialize_blob_v2_binary_batch_with_admission( + &dataset, + output_projection.as_ref(), + batch, + &materialization_context, + admission, + ) + .await + .map_err(DataFusionError::from) + } + }) + .try_buffered(config.batch_readahead) + .map_ok(|batch| batch.into_batch()) + .boxed() + } else { + inner_stream + }; timer.done(); Ok(Self { @@ -364,6 +418,7 @@ impl LanceStream { pub fn try_new_v1( dataset: Arc, fragments: Arc>, + _offsets: Option>, projection: Arc, config: LanceScanConfig, metrics: &ExecutionPlanMetricsSet, @@ -390,9 +445,11 @@ impl LanceStream { .collect::>(); let batches = if config.ordered_output { - let readers = stream::iter(file_fragments) - .map(move |file_fragment| { - Ok(open_file( + let readers = buffered_fragment_opens( + stream::iter(file_fragments), + fragment_readahead, + move |file_fragment| { + open_file( file_fragment, project_schema.clone(), FragReadConfig::default() @@ -404,9 +461,9 @@ impl LanceStream { .with_row_created_at_version(config.with_row_created_at_version), config.with_make_deletions_null, None, - )) - }) - .try_buffered(fragment_readahead); + ) + }, + ); let tasks = readers.and_then(move |reader| async move { reader .read_all(config.batch_size as u32) @@ -422,9 +479,11 @@ impl LanceStream { .stream_in_current_span() .boxed() } else { - let readers = stream::iter(file_fragments) - .map(move |file_fragment| { - Ok(open_file( + let readers = buffered_fragment_opens( + stream::iter(file_fragments), + fragment_readahead, + move |file_fragment| { + open_file( file_fragment, project_schema.clone(), FragReadConfig::default() @@ -436,9 +495,9 @@ impl LanceStream { .with_row_created_at_version(config.with_row_created_at_version), config.with_make_deletions_null, None, - )) - }) - .try_buffered(fragment_readahead); + ) + }, + ); let tasks = readers.and_then(move |reader| async move { reader .read_all(config.batch_size as u32) @@ -482,7 +541,9 @@ impl core::fmt::Debug for LanceStream { impl RecordBatchStream for LanceStream { fn schema(&self) -> SchemaRef { - let mut schema: ArrowSchema = self.projection.as_ref().into(); + let output_projection = + crate::dataset::blob::public_blob_v2_binary_output_schema(self.projection.as_ref()); + let mut schema: ArrowSchema = (&output_projection).into(); if self.config.with_row_id { schema = schema.try_with_column(ROW_ID_FIELD.clone()).unwrap(); } @@ -509,6 +570,7 @@ pub struct LanceScanConfig { pub batch_readahead: usize, pub fragment_readahead: Option, pub io_buffer_size: u64, + pub materialization_readahead_bytes: Option, pub with_row_id: bool, pub with_row_address: bool, pub with_row_last_updated_at_version: bool, @@ -530,6 +592,7 @@ impl Default for LanceScanConfig { batch_readahead: get_num_compute_intensive_cpus(), fragment_readahead: None, io_buffer_size: *DEFAULT_IO_BUFFER_SIZE, + materialization_readahead_bytes: None, with_row_id: false, with_row_address: false, with_row_last_updated_at_version: false, @@ -602,7 +665,9 @@ impl LanceScanExec { projection: Arc, config: LanceScanConfig, ) -> Self { - let mut output_schema: ArrowSchema = projection.as_ref().into(); + let output_projection = + crate::dataset::blob::public_blob_v2_binary_output_schema(projection.as_ref()); + let mut output_schema: ArrowSchema = (&output_projection).into(); if config.with_row_id { output_schema = output_schema.try_with_column(ROW_ID_FIELD.clone()).unwrap(); @@ -673,10 +738,6 @@ impl ExecutionPlan for LanceScanExec { "LanceScanExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -727,7 +788,7 @@ impl ExecutionPlan for LanceScanExec { ))) } - fn partition_statistics(&self, _partition: Option) -> Result { + fn partition_statistics(&self, _partition: Option) -> Result> { // Some fragments from older datasets might have the row count stats missing. let (row_count, is_exact) = self.fragments @@ -744,10 +805,10 @@ impl ExecutionPlan for LanceScanExec { false => Precision::Absent, }; - Ok(Statistics { + Ok(Arc::new(Statistics { num_rows, ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn metrics(&self) -> Option { diff --git a/rust/lance/src/io/exec/take.rs b/rust/lance/src/io/exec/take.rs index c3642cdb043..3c84a1df03c 100644 --- a/rust/lance/src/io/exec/take.rs +++ b/rust/lance/src/io/exec/take.rs @@ -38,6 +38,7 @@ use crate::dataset::Dataset; use crate::dataset::fragment::{FragReadConfig, FragmentReader}; use crate::dataset::rowids::get_row_id_index; use crate::datatypes::Schema; +use crate::index::prefilter::DatasetPreFilter; use super::utils::IoMetrics; @@ -70,6 +71,10 @@ struct TakeStream { dataset: Arc, /// The fields to take from the input stream fields_to_take: Arc, + /// The descriptor-view schema used for storage reads when blob payloads + /// must be materialized after take. + read_fields: Arc, + materialize_blob_v2_binary: bool, /// The output schema, needed for us to merge the new columns /// into the input data in the correct order output_schema: SchemaRef, @@ -92,9 +97,20 @@ impl TakeStream { metrics: &ExecutionPlanMetricsSet, partition: usize, ) -> Self { + let materialize_blob_v2_binary = + crate::dataset::blob::schema_has_blob_v2_binary_view(fields_to_take.as_ref()); + let read_fields = if materialize_blob_v2_binary { + Arc::new(crate::dataset::blob::blob_v2_descriptor_schema( + fields_to_take.as_ref(), + )) + } else { + fields_to_take.clone() + }; Self { dataset, fields_to_take, + read_fields, + materialize_blob_v2_binary, output_schema, readers_cache: Arc::new(Mutex::new(HashMap::new())), scan_scheduler, @@ -131,14 +147,12 @@ impl TakeStream { )) })?; - let reader = Arc::new( - fragment - .open( - &self.fields_to_take, - FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone()), - ) - .await?, - ); + let mut read_config = + FragReadConfig::default().with_scan_scheduler(self.scan_scheduler.clone()); + if self.materialize_blob_v2_binary { + read_config = read_config.with_row_address(true); + } + let reader = Arc::new(fragment.open(&self.read_fields, read_config).await?); let mut readers = self.readers_cache.lock().unwrap(); readers.insert(fragment_id, reader.clone()); @@ -160,11 +174,12 @@ impl TakeStream { } /// Returns the row addresses for the given batch, plus an optional validity - /// mask. When stable row IDs are used, some row IDs from stale index results - /// (e.g. FTS matches for deleted rows) may no longer exist in the row ID - /// index. These are excluded from the returned addresses, and the mask - /// indicates which input rows are still valid so the caller can filter the - /// batch to match. + /// mask. Some row IDs from stale index results (e.g. FTS matches for deleted + /// rows) may no longer be valid. For stable row IDs, the row ID index detects + /// these entries. For physical row IDs, the dataset deletion mask detects + /// deleted rows and removed fragments. Invalid entries are excluded from the + /// returned addresses, and the mask indicates which input rows are still + /// valid so the caller can filter the batch to match. async fn get_row_addrs( &self, batch: &RecordBatch, @@ -176,28 +191,48 @@ impl TakeStream { if let Some(row_id_index) = get_row_id_index(&self.dataset).await? { let row_id_array = row_id_array.as_primitive::(); - let mut addresses = Vec::with_capacity(row_id_array.len()); - let mut valid = Vec::with_capacity(row_id_array.len()); - - for id in row_id_array.values().iter() { - if let Some(address) = row_id_index.get(*id) { - addresses.push(u64::from(address)); - valid.push(true); - } else { - valid.push(false); - } + Self::resolve_row_addrs(row_id_array, |id| Ok(row_id_index.get(id)?.map(u64::from))) + } else { + let row_id_array = row_id_array.as_primitive::(); + let fragments = row_id_array + .values() + .iter() + .map(|id| RowAddress::from(*id).fragment_id()) + .collect(); + if let Some(mask) = + DatasetPreFilter::create_deletion_mask(self.dataset.clone(), fragments) + { + let mask = mask.await?; + Self::resolve_row_addrs(row_id_array, |id| Ok(mask.selected(id).then_some(id))) + } else { + Ok((Arc::new(row_id_array.clone()), None)) } + } + } + } - let mask = if addresses.len() < row_id_array.len() { - Some(BooleanArray::from(valid)) - } else { - None - }; - Ok((Arc::new(UInt64Array::from(addresses)), mask)) + fn resolve_row_addrs( + row_ids: &UInt64Array, + mut resolve: impl FnMut(u64) -> Result>, + ) -> Result<(Arc, Option)> { + let mut addresses = Vec::with_capacity(row_ids.len()); + let mut valid = Vec::with_capacity(row_ids.len()); + + for id in row_ids.values().iter() { + if let Some(address) = resolve(*id)? { + addresses.push(address); + valid.push(true); } else { - Ok((row_id_array.clone(), None)) + valid.push(false); } } + + let mask = if addresses.len() < row_ids.len() { + Some(BooleanArray::from(valid)) + } else { + None + }; + Ok((Arc::new(UInt64Array::from(addresses)), mask)) } async fn map_batch( @@ -208,9 +243,9 @@ impl TakeStream { let compute_timer = self.metrics.baseline_metrics.elapsed_compute().timer(); let (row_addrs_arr, validity_mask) = self.get_row_addrs(&batch).await?; - // Filter out rows whose row IDs no longer exist (e.g. stale FTS/vector - // index entries pointing to deleted rows). Without this, the downstream - // merge would fail with a row-count mismatch. + // Filter stale index entries before reading so the input batch and taken + // columns remain aligned. Otherwise, the downstream merge would fail with + // a row-count mismatch. let batch = if let Some(mask) = validity_mask { arrow::compute::filter_record_batch(&batch, &mask)? } else { @@ -355,6 +390,15 @@ impl TakeStream { (None, None) => {} } + if self.materialize_blob_v2_binary { + new_data = crate::dataset::blob::materialize_blob_v2_binary_batch( + &self.dataset, + self.fields_to_take.as_ref(), + new_data, + ) + .await?; + } + Ok(batch.merge_with_schema(&new_data, self.output_schema.as_ref())?) } @@ -487,10 +531,10 @@ impl TakeExec { projection ); - let output_schema = Arc::new(Self::calculate_output_schema( - dataset.schema(), - &input.schema(), - &projection, + let output_schema = + Self::calculate_output_schema(dataset.schema(), &input.schema(), &projection); + let output_schema = Arc::new(crate::dataset::blob::public_blob_v2_binary_output_schema( + &output_schema, )); let output_arrow = Arc::new(ArrowSchema::from(output_schema.as_ref())); let properties = Arc::new( @@ -521,7 +565,7 @@ impl TakeExec { /// /// If this happens the order of the new nested fields will match the order defined in /// the dataset schema. - fn calculate_output_schema( + pub(crate) fn calculate_output_schema( dataset_schema: &Schema, input_schema: &ArrowSchema, projection: &Projection, @@ -576,10 +620,6 @@ impl ExecutionPlan for TakeExec { "TakeExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> SchemaRef { self.output_schema.clone() } @@ -663,11 +703,11 @@ impl ExecutionPlan for TakeExec { fn partition_statistics( &self, partition: Option, - ) -> Result { - Ok(Statistics { + ) -> Result> { + Ok(Arc::new(Statistics { num_rows: self.input.partition_statistics(partition)?.num_rows, ..Statistics::new_unknown(self.schema().as_ref()) - }) + })) } fn properties(&self) -> &Arc { @@ -855,6 +895,70 @@ mod tests { } } + #[tokio::test] + async fn test_take_filters_stale_physical_row_ids() { + let TestFixture { + dataset, + _tmp_dir_guard, + } = test_fixture().await; + let mut dataset = dataset.as_ref().clone(); + dataset.delete("i = 1").await.unwrap(); + let dataset = Arc::new(dataset); + + // Simulate stale index results for a deleted row and a removed fragment. + let missing_fragment_row_id = u64::from(RowAddress::new_from_parts(99, 0)); + let row_ids = Arc::new(UInt64Array::from(vec![ + 0_u64, + 1, + 2, + missing_fragment_row_id, + ])); + let scores = Arc::new(Int32Array::from(vec![10, 11, 12, 13])); + let input_batch = RecordBatch::try_from_iter(vec![ + (ROW_ID, row_ids as ArrayRef), + ("score", scores as ArrayRef), + ]) + .unwrap(); + let schema = input_batch.schema(); + let input_stream = futures::stream::iter(vec![Ok(input_batch)]); + let input_stream = Box::pin(RecordBatchStreamAdapter::new(schema, input_stream)); + let input = Arc::new(OneShotExec::new(input_stream)); + + let projection = dataset + .empty_projection() + .union_column("s", OnMissing::Error) + .unwrap(); + let take_exec = TakeExec::try_new(dataset, input, projection) + .unwrap() + .unwrap(); + let result = take_exec + .execute(0, Arc::new(TaskContext::default())) + .unwrap() + .try_collect::>() + .await + .unwrap(); + + let result = concat_batches(&result[0].schema(), &result).unwrap(); + assert_eq!( + result[ROW_ID] + .as_any() + .downcast_ref::() + .unwrap(), + &UInt64Array::from(vec![0_u64, 2]) + ); + assert_eq!( + result["score"] + .as_any() + .downcast_ref::() + .unwrap(), + &Int32Array::from(vec![10, 12]) + ); + assert_eq!( + result["s"].as_any().downcast_ref::().unwrap(), + &StringArray::from(vec!["str-0", "str-2"]) + ); + } + #[tokio::test(flavor = "current_thread")] async fn test_take_records_output_and_io_metrics() { use datafusion::physical_plan::metrics::MetricValue; diff --git a/rust/lance/src/io/exec/testing.rs b/rust/lance/src/io/exec/testing.rs index 2d5911a4e46..f979c403431 100644 --- a/rust/lance/src/io/exec/testing.rs +++ b/rust/lance/src/io/exec/testing.rs @@ -4,7 +4,6 @@ //! Testing Node //! -use std::any::Any; use std::sync::Arc; use arrow_array::RecordBatch; @@ -51,10 +50,6 @@ impl ExecutionPlan for TestingExec { "TestingExec" } - fn as_any(&self) -> &dyn Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.batches[0].schema() } diff --git a/rust/lance/src/io/exec/utils.rs b/rust/lance/src/io/exec/utils.rs index 6e2d50d3736..fb36f612742 100644 --- a/rust/lance/src/io/exec/utils.rs +++ b/rust/lance/src/io/exec/utils.rs @@ -2,13 +2,15 @@ // SPDX-FileCopyrightText: Copyright The Lance Authors use lance_datafusion::utils::{ - BYTES_READ_METRIC, ExecutionPlanMetricsSetExt, INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, - IOPS_METRIC, PARTS_LOADED_METRIC, REQUESTS_METRIC, + BYTES_READ_METRIC, ExecutionPlanMetricsSetExt, INDEX_CACHE_HITS_METRIC, + INDEX_CACHE_MISSES_METRIC, INDEX_COMPARISONS_METRIC, INDICES_LOADED_METRIC, IOPS_METRIC, + PARTS_LOADED_METRIC, REQUESTS_METRIC, }; use lance_index::metrics::MetricsCollector; use lance_io::scheduler::{IoStats, ScanScheduler, ScanStats}; use lance_table::format::IndexMetadata; use pin_project::pin_project; +use std::collections::HashMap; use std::future::Future; use std::pin::Pin; use std::sync::{Arc, Mutex}; @@ -17,24 +19,53 @@ use std::task::{Context, Poll}; use arrow_array::{RecordBatch, UInt64Array}; use arrow_schema::SchemaRef; use async_trait::async_trait; +use datafusion::common::runtime::SpawnedTask; use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::physical_plan::metrics::{ BaselineMetrics, Count, ExecutionPlanMetricsSet, Gauge, MetricBuilder, MetricValue, }; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, RecordBatchStream, SendableRecordBatchStream, + DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, RecordBatchStream, + SendableRecordBatchStream, }; +use datafusion_physical_expr::{Distribution, EquivalenceProperties, Partitioning}; +use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; +use futures::future::{BoxFuture, Shared}; use futures::stream::FuturesUnordered; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use lance_core::error::{CloneableResult, Error}; use lance_core::utils::futures::{Capacity, SharedStreamExt}; use lance_core::{ROW_ID, Result}; use lance_index::prefilter::FilterLoader; use lance_select::{RowAddrMask, RowAddrTreeMap, result::IndexExprResult}; +use tracing::Instrument; +use super::row_addr_mask::MaskAndLoader; use crate::Dataset; use crate::index::prefilter::DatasetPreFilter; +/// Open fragments on cancellation-safe tasks while preserving the stream's +/// ordering and readahead bound. +pub(crate) fn buffered_fragment_opens( + fragments: S, + fragment_readahead: usize, + mut open: Open, +) -> impl Stream> +where + S: Stream + Send, + Open: FnMut(S::Item) -> OpenFuture + Send, + OpenFuture: Future> + Send + 'static, + Reader: Send + 'static, +{ + fragments + .map(move |fragment| { + SpawnedTask::spawn(open(fragment).in_current_span()).map(|task_result| { + task_result.map_err(|error| DataFusionError::External(Box::new(error)))? + }) + }) + .buffered(fragment_readahead) +} + #[derive(Debug, Clone)] pub enum PreFilterSource { /// The prefilter input is an array of row ids that match the filter condition @@ -45,29 +76,365 @@ pub enum PreFilterSource { None, } +type SharedPreFilterFuture = Shared>>>; + +struct SharedPreFilterEntry { + context: std::sync::Weak, + future: SharedPreFilterFuture, + waiters: usize, + is_complete: bool, + generation: u64, +} + +/// Query-plan-local materialization state for a MultiMatch base prefilter. +/// +/// Entries are keyed by task-context identity and partition. This prevents a +/// reused physical plan from carrying a mask into a later query and keeps an +/// accidental multi-partition execution from sharing across input partitions. +/// The mutex is held only while installing or cloning a future; prefilter +/// execution never runs under it. +struct SharedPreFilterMaterialization { + queries: Mutex>, + next_generation: std::sync::atomic::AtomicU64, +} + +impl std::fmt::Debug for SharedPreFilterMaterialization { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let queries = self + .queries + .lock() + .map(|queries| queries.len()) + .unwrap_or_default(); + f.debug_struct("SharedPreFilterMaterialization") + .field("queries", &queries) + .finish() + } +} + +impl SharedPreFilterMaterialization { + fn new() -> Self { + Self { + queries: Mutex::new(HashMap::new()), + next_generation: std::sync::atomic::AtomicU64::new(0), + } + } +} + +#[derive(Debug)] +struct SharedPreFilterExec { + source: Arc, + materialization: Arc, + properties: Arc, +} + +impl SharedPreFilterExec { + fn new( + source: Arc, + materialization: Arc, + ) -> Self { + Self { + properties: Arc::new(PlanProperties::new( + EquivalenceProperties::new(source.schema()), + Partitioning::UnknownPartitioning(1), + EmissionType::Final, + Boundedness::Bounded, + )), + source, + materialization, + } + } +} + +impl DisplayAs for SharedPreFilterExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "SharedMultiMatchPrefilter") + } +} + +impl ExecutionPlan for SharedPreFilterExec { + fn name(&self) -> &str { + "SharedPreFilterExec" + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.source] + } + + fn required_input_distribution(&self) -> Vec { + self.children() + .iter() + .map(|_| Distribution::SinglePartition) + .collect() + } + + fn with_new_children( + self: Arc, + mut children: Vec>, + ) -> DataFusionResult> { + let source = match children.len() { + 1 => children.pop().ok_or_else(|| { + DataFusionError::Internal( + "shared MultiMatch prefilter lost its source child".to_string(), + ) + })?, + count => { + return Err(DataFusionError::Internal(format!( + "shared MultiMatch prefilter expected one child, got {count}" + ))); + } + }; + Ok(Arc::new(Self::new(source, self.materialization.clone()))) + } + + fn execute( + &self, + _partition: usize, + _context: Arc, + ) -> DataFusionResult { + Err(DataFusionError::Internal( + "shared MultiMatch prefilter must be materialized by its FTS consumer".to_string(), + )) + } + + fn properties(&self) -> &Arc { + &self.properties + } +} + +pub(crate) struct PreFilterMasks { + pub overlay_block: Option, + pub external_mask: Option>, +} + +impl PreFilterSource { + /// Return a plan-local shared form for a MultiMatch with multiple fields. + /// No-filter and already-shared sources retain their existing identity. + pub(crate) fn shared_for_multimatch_fields(&self, field_count: usize) -> Vec { + if field_count <= 1 { + return vec![self.clone(); field_count]; + } + match self { + Self::FilteredRowIds(source) | Self::ScalarIndexQuery(source) => { + let materialization = Arc::new(SharedPreFilterMaterialization::new()); + (0..field_count) + .map(|_| { + let shared = Arc::new(SharedPreFilterExec::new( + source.clone(), + materialization.clone(), + )); + if matches!(self, Self::FilteredRowIds(_)) { + Self::FilteredRowIds(shared) + } else { + Self::ScalarIndexQuery(shared) + } + }) + .collect() + } + Self::None => vec![self.clone(); field_count], + } + } + + pub(crate) fn execution_plan(&self) -> Option<&Arc> { + match self { + Self::FilteredRowIds(source) | Self::ScalarIndexQuery(source) => Some(source), + Self::None => None, + } + } + + pub(crate) fn with_execution_plan( + &self, + source: Arc, + ) -> DataFusionResult { + match self { + Self::FilteredRowIds(_) => Ok(Self::FilteredRowIds(source)), + Self::ScalarIndexQuery(_) => Ok(Self::ScalarIndexQuery(source)), + Self::None => Err(DataFusionError::Internal( + "prefilter source received an unexpected execution-plan child".to_string(), + )), + } + } +} + +struct SharedPreFilterWaiter { + materialization: Arc, + key: (usize, usize), + generation: u64, +} + +impl SharedPreFilterWaiter { + fn mark_complete(&self) { + if let Ok(mut queries) = self.materialization.queries.lock() + && let Some(entry) = queries.get_mut(&self.key) + && entry.generation == self.generation + { + entry.is_complete = true; + } + } +} + +impl Drop for SharedPreFilterWaiter { + fn drop(&mut self) { + let Ok(mut queries) = self.materialization.queries.lock() else { + return; + }; + let should_remove = if let Some(entry) = queries.get_mut(&self.key) + && entry.generation == self.generation + { + let Some(waiters) = entry.waiters.checked_sub(1) else { + debug_assert!(false, "shared prefilter waiter count underflowed"); + return; + }; + entry.waiters = waiters; + entry.waiters == 0 && !entry.is_complete + } else { + false + }; + if should_remove { + queries.remove(&self.key); + } + } +} + +fn shared_prefilter_future( + materialization: Arc, + source: Arc, + is_scalar_index_query: bool, + context: Arc, + partition: usize, +) -> BoxFuture<'static, Result>> { + async move { + let context_id = Arc::as_ptr(&context) as usize; + let key = (context_id, partition); + let (future, generation) = { + let mut queries = materialization.queries.lock().map_err(|_| { + Error::internal("MultiMatch prefilter materialization lock was poisoned") + })?; + queries.retain(|_, entry| entry.context.strong_count() > 0); + if let Some(entry) = queries.get_mut(&key) { + entry.waiters = entry.waiters.checked_add(1).ok_or_else(|| { + Error::internal("MultiMatch prefilter waiter count overflowed") + })?; + (entry.future.clone(), entry.generation) + } else { + let generation = materialization + .next_generation + .fetch_update( + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + |generation| generation.checked_add(1), + ) + .map_err(|_| { + Error::internal("MultiMatch prefilter generation counter overflowed") + })?; + let entry = SharedPreFilterEntry { + context: Arc::downgrade(&context), + future: { + async move { + let result = async move { + let stream = source.execute(partition, context)?; + if is_scalar_index_query { + Box::new(SelectionVectorToPrefilter(stream)).load().await + } else { + Box::new(FilteredRowIdsToPrefilter(stream)).load().await + } + } + .await; + CloneableResult::from(result.map(Arc::new)) + } + .boxed() + .shared() + }, + waiters: 1, + is_complete: false, + generation, + }; + let future = entry.future.clone(); + queries.insert(key, entry); + (future, generation) + } + }; + let waiter = SharedPreFilterWaiter { + materialization, + key, + generation, + }; + let CloneableResult(result) = future.await; + waiter.mark_complete(); + result.map_err(|error| error.0) + } + .boxed() +} + pub(crate) fn build_prefilter( context: Arc, partition: usize, prefilter_source: &PreFilterSource, ds: Arc, index_meta: &[IndexMetadata], + masks: PreFilterMasks, ) -> Result> { + let mut shared_filter = None; let prefilter_loader = match &prefilter_source { PreFilterSource::FilteredRowIds(src_node) => { - let stream = src_node.execute(partition, context)?; - Some(Box::new(FilteredRowIdsToPrefilter(stream)) as Box) + if let Some(shared) = src_node.downcast_ref::() { + shared_filter = Some(shared_prefilter_future( + shared.materialization.clone(), + shared.source.clone(), + false, + context, + partition, + )); + None + } else { + let stream = src_node.execute(partition, context)?; + Some(Box::new(FilteredRowIdsToPrefilter(stream)) as Box) + } } PreFilterSource::ScalarIndexQuery(src_node) => { - let stream = src_node.execute(partition, context)?; - Some(Box::new(SelectionVectorToPrefilter(stream)) as Box) + if let Some(shared) = src_node.downcast_ref::() { + shared_filter = Some(shared_prefilter_future( + shared.materialization.clone(), + shared.source.clone(), + true, + context, + partition, + )); + None + } else { + let stream = src_node.execute(partition, context)?; + Some(Box::new(SelectionVectorToPrefilter(stream)) as Box) + } } PreFilterSource::None => None, }; - Ok(Arc::new(DatasetPreFilter::new( - ds, - index_meta, - prefilter_loader, - ))) + // Combine the external row-address mask (logical AND) with whatever the + // filter produced, so an FTS prefilter restricts BM25 scoring to masked rows + // (mirrors the ANN path). Independent of `overlay_block`, which the prefilter + // applies separately to drop index entries staled by a data overlay. + let mut prefilter = if let Some(shared_filter) = shared_filter { + let shared_filter = match masks.external_mask { + Some(mask) => async move { + Ok(Arc::new( + mask.as_ref().clone() & shared_filter.await?.as_ref().clone(), + )) + } + .boxed(), + None => shared_filter, + }; + DatasetPreFilter::new_with_filter_future(ds, index_meta, Some(shared_filter)) + } else { + let prefilter_loader = match masks.external_mask { + Some(mask) => { + Some(Box::new(MaskAndLoader::new(mask, prefilter_loader)) as Box) + } + None => prefilter_loader, + }; + DatasetPreFilter::new(ds, index_meta, prefilter_loader) + }; + if let Some(overlay_block) = masks.overlay_block { + prefilter = prefilter.with_overlay_block(overlay_block); + } + Ok(Arc::new(prefilter)) } // Utility to convert an input (containing row ids) into a prefilter @@ -421,10 +788,6 @@ impl ExecutionPlan for ReplayExec { "ReplayExec" } - fn as_any(&self) -> &dyn std::any::Any { - self - } - fn schema(&self) -> arrow_schema::SchemaRef { self.input.schema() } @@ -521,6 +884,8 @@ pub struct IndexMetrics { indices_loaded: Count, parts_loaded: Count, index_comparisons: Count, + index_cache_hits: Count, + index_cache_misses: Count, /// Per-query sink that accumulates exact index-file I/O as partitions are /// loaded from storage. Shared by all clones of this `IndexMetrics`, so /// concurrent partition loads all funnel into the same counters. Published @@ -535,6 +900,8 @@ impl IndexMetrics { indices_loaded: metrics.new_count(INDICES_LOADED_METRIC, partition), parts_loaded: metrics.new_count(PARTS_LOADED_METRIC, partition), index_comparisons: metrics.new_count(INDEX_COMPARISONS_METRIC, partition), + index_cache_hits: metrics.new_count(INDEX_CACHE_HITS_METRIC, partition), + index_cache_misses: metrics.new_count(INDEX_CACHE_MISSES_METRIC, partition), io_stats: IoStats::new(), io_metrics: IoMetrics::new(metrics, partition), } @@ -559,6 +926,12 @@ impl MetricsCollector for IndexMetrics { fn record_comparisons(&self, num_comparisons: usize) { self.index_comparisons.add(num_comparisons); } + fn record_index_cache_hits(&self, num_hits: usize) { + self.index_cache_hits.add(num_hits); + } + fn record_index_cache_misses(&self, num_misses: usize) { + self.index_cache_misses.add(num_misses); + } fn io_stats(&self) -> Option { Some(self.io_stats.clone()) } @@ -569,9 +942,10 @@ mod tests { use std::sync::Arc; - use arrow_array::{RecordBatchReader, types::UInt32Type}; - use arrow_schema::SortOptions; + use arrow_array::{RecordBatch, RecordBatchReader, UInt64Array, types::UInt32Type}; + use arrow_schema::{DataType, Field, Schema, SortOptions}; use datafusion::common::NullEquality; + use datafusion::error::{DataFusionError, Result as DataFusionResult}; use datafusion::{ logical_expr::JoinType, physical_expr::expressions::Column, @@ -579,12 +953,295 @@ mod tests { ExecutionPlan, joins::SortMergeJoinExec, stream::RecordBatchStreamAdapter, }, }; - use futures::{StreamExt, TryStreamExt}; - use lance_core::utils::futures::Capacity; + use futures::{StreamExt, TryStreamExt, stream}; + use lance_core::{ROW_ID, utils::futures::Capacity}; use lance_datafusion::exec::OneShotExec; use lance_datagen::{BatchCount, RowCount, array}; + use lance_select::result::IndexExprResultWireFormat; + use lance_select::{RowAddrMask, RowAddrTreeMap, RowSetOps, result::IndexExprResult}; + use roaring::RoaringBitmap; + use rstest::rstest; + + use super::{ + InstrumentedChildInputStream, PreFilterSource, ReplayExec, SharedPreFilterExec, + SharedPreFilterMaterialization, shared_prefilter_future, + }; + + fn prefilter_source(is_scalar_index_query: bool, is_empty: bool) -> PreFilterSource { + let mask = if is_empty { + RowAddrMask::allow_nothing() + } else { + RowAddrMask::from_allowed(RowAddrTreeMap::from_iter(0_u64..4)) + }; + let batch = if is_scalar_index_query { + IndexExprResult::exact(mask) + .serialize( + &RoaringBitmap::from_iter([0_u32]), + IndexExprResultWireFormat::TwoMask, + ) + .unwrap() + } else { + let row_ids = if is_empty { + UInt64Array::from(Vec::::new()) + } else { + UInt64Array::from_iter_values(0_u64..4) + }; + RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])), + vec![Arc::new(row_ids)], + ) + .unwrap() + }; + // A duplicate source execution fails, so successful concurrent + // materialization verifies sharing without production metrics. + let source = Arc::new(OneShotExec::from_batch(batch)); + if is_scalar_index_query { + PreFilterSource::ScalarIndexQuery(source) + } else { + PreFilterSource::FilteredRowIds(source) + } + } + + fn shared_materialization(source: &PreFilterSource) -> Arc { + match source { + PreFilterSource::FilteredRowIds(source) | PreFilterSource::ScalarIndexQuery(source) => { + source + .downcast_ref::() + .expect("expected a shared prefilter source") + .materialization + .clone() + } + _ => panic!("expected a shared prefilter source"), + } + } + + fn shared_source(source: &PreFilterSource) -> Arc { + match source { + PreFilterSource::FilteredRowIds(source) | PreFilterSource::ScalarIndexQuery(source) => { + source + .downcast_ref::() + .expect("expected a shared prefilter source") + .source + .clone() + } + _ => panic!("expected a shared prefilter source"), + } + } + + #[rstest] + #[case::two_fields(2)] + #[case::four_fields(4)] + #[case::eight_fields(8)] + #[tokio::test] + async fn shared_multimatch_prefilter_materializes_once( + #[case] field_count: usize, + #[values(false, true)] is_scalar_index_query: bool, + #[values(false, true)] is_empty: bool, + ) { + let shared_sources = prefilter_source(is_scalar_index_query, is_empty) + .shared_for_multimatch_fields(field_count); + assert_eq!( + shared_sources + .iter() + .filter(|source| source.execution_plan().is_some()) + .count(), + field_count, + "every field must declare its shared source dependency" + ); + let context = Arc::new(datafusion::execution::TaskContext::default()); + let masks = futures::future::try_join_all(shared_sources.iter().map(|source| { + shared_prefilter_future( + shared_materialization(source), + shared_source(source), + is_scalar_index_query, + context.clone(), + 0, + ) + })) + .await + .unwrap(); - use super::{InstrumentedChildInputStream, ReplayExec}; + assert!(masks.windows(2).all(|pair| Arc::ptr_eq(&pair[0], &pair[1]))); + assert_eq!(masks[0].allow_list().unwrap().is_empty(), is_empty); + } + + #[test] + fn no_filter_and_single_field_do_not_install_sharing() { + let no_filter = PreFilterSource::None.shared_for_multimatch_fields(8); + assert!( + no_filter + .iter() + .all(|source| matches!(source, PreFilterSource::None)) + ); + + let single = prefilter_source(false, false).shared_for_multimatch_fields(1); + assert!(matches!( + single.as_slice(), + [PreFilterSource::FilteredRowIds(_)] + )); + } + + #[tokio::test] + async fn shared_multimatch_prefilter_caches_source_error() { + let schema = Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::iter(vec![Err(DataFusionError::Execution( + "shared prefilter failure".to_string(), + ))]), + )); + let source = PreFilterSource::FilteredRowIds(Arc::new(OneShotExec::new(stream))); + let shared_sources = source.shared_for_multimatch_fields(2); + let context = Arc::new(datafusion::execution::TaskContext::default()); + let left = shared_prefilter_future( + shared_materialization(&shared_sources[0]), + shared_source(&shared_sources[0]), + false, + context.clone(), + 0, + ); + let right = shared_prefilter_future( + shared_materialization(&shared_sources[1]), + shared_source(&shared_sources[1]), + false, + context, + 0, + ); + let (left, right) = tokio::join!(left, right); + + assert!( + left.unwrap_err() + .to_string() + .contains("shared prefilter failure") + ); + assert!( + right + .unwrap_err() + .to_string() + .contains("shared prefilter failure") + ); + } + + #[tokio::test] + async fn shared_multimatch_prefilter_survives_waiter_cancellation() { + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])), + vec![Arc::new(UInt64Array::from_iter_values(0_u64..4))], + ) + .unwrap(); + let schema = batch.schema(); + let (started, has_started) = tokio::sync::oneshot::channel::<()>(); + let (release, wait) = tokio::sync::oneshot::channel::<()>(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(async move { + started.send(()).map_err(|_| { + DataFusionError::Execution( + "shared prefilter startup receiver dropped".to_string(), + ) + })?; + wait.await.map_err(|error| { + DataFusionError::Execution(format!( + "shared prefilter release sender dropped: {error}" + )) + })?; + Ok(batch) + }), + )); + let source = PreFilterSource::FilteredRowIds(Arc::new(OneShotExec::new(stream))); + let shared_sources = source.shared_for_multimatch_fields(2); + let materialization = shared_materialization(&shared_sources[0]); + let context = Arc::new(datafusion::execution::TaskContext::default()); + let first = tokio::spawn(shared_prefilter_future( + materialization.clone(), + shared_source(&shared_sources[0]), + false, + context.clone(), + 0, + )); + tokio::time::timeout(std::time::Duration::from_secs(5), has_started) + .await + .expect("shared prefilter source should start") + .expect("shared prefilter startup sender should remain alive"); + let second = tokio::spawn(shared_prefilter_future( + materialization.clone(), + shared_source(&shared_sources[1]), + false, + context, + 0, + )); + loop { + let waiters = materialization + .queries + .lock() + .unwrap() + .values() + .map(|entry| entry.waiters) + .sum::(); + if waiters == 2 { + break; + } + tokio::task::yield_now().await; + } + first.abort(); + release.send(()).unwrap(); + let mask = tokio::time::timeout(std::time::Duration::from_secs(5), second) + .await + .expect("replacement waiter should resume the shared source") + .unwrap() + .unwrap(); + assert_eq!(mask.allow_list().unwrap().len(), Some(4)); + } + + #[tokio::test] + async fn shared_multimatch_prefilter_drops_fully_canceled_query() { + let schema = Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])); + let (started, has_started) = tokio::sync::oneshot::channel::<()>(); + let stream = Box::pin(RecordBatchStreamAdapter::new( + schema, + stream::once(async move { + started.send(()).map_err(|_| { + DataFusionError::Execution( + "shared prefilter startup receiver dropped".to_string(), + ) + })?; + std::future::pending::>().await + }), + )); + let source = PreFilterSource::FilteredRowIds(Arc::new(OneShotExec::new(stream))); + let shared_sources = source.shared_for_multimatch_fields(2); + let materialization = shared_materialization(&shared_sources[0]); + let waiter = tokio::spawn(shared_prefilter_future( + materialization.clone(), + shared_source(&shared_sources[0]), + false, + Arc::new(datafusion::execution::TaskContext::default()), + 0, + )); + tokio::time::timeout(std::time::Duration::from_secs(5), has_started) + .await + .expect("shared prefilter source should start") + .expect("shared prefilter startup sender should remain alive"); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert!(materialization.queries.lock().unwrap().is_empty()); + } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn instrumented_child_input_stream_excludes_child_poll_time() { diff --git a/rust/lance/src/lib.rs b/rust/lance/src/lib.rs index 2a5d0d2c822..66bc876bd88 100644 --- a/rust/lance/src/lib.rs +++ b/rust/lance/src/lib.rs @@ -81,6 +81,8 @@ pub mod datafusion; pub mod dataset; pub mod index; pub mod io; +#[cfg(feature = "metrics")] +pub mod metrics; pub mod session; pub mod table; pub mod utils; diff --git a/rust/lance/src/metrics.md b/rust/lance/src/metrics.md new file mode 100644 index 00000000000..05d9df0c0e3 --- /dev/null +++ b/rust/lance/src/metrics.md @@ -0,0 +1,40 @@ +Lance publishes metrics through the [`metrics`](https://docs.rs/metrics) crate +facade. Install any recorder (Prometheus, OpenTelemetry, etc.) in your +application and Lance will emit into it; when no recorder is installed, emission +is a cheap no-op. Metrics are only emitted when Lance is built with the +`metrics` feature. + +## Object store metrics + +These track I/O against the underlying object store. The `base` label +identifies the store; its cardinality is controlled by the +`LANCE_OBJECT_STORE_METRICS_LABEL` environment variable: + +- `scheme` (default) — the scheme only (`s3`, `gs`, `az`, `file`, `memory`); + low, bounded cardinality. +- `full` — the store's unique prefix (`s3$my-bucket`, `az$container@account` + where Azure's account also matters), so multiple buckets on the same cloud + can be told apart. Cardinality grows with the number of stores accessed. +- `off` — omit the `base` label entirely. + +`operation` is one of `get`, `put`, `put_part`, `head`, `list`, `delete`, +`copy`, `rename`, `complete_multipart`, or `abort_multipart`. + +Request counts are per logical operation: a `list` or `delete` that spans many +objects is one request, matching how backends batch them. + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `lance_object_store_requests_total` | counter | `operation`, `base` | Object store requests issued. | +| `lance_object_store_request_bytes_total` | counter | `operation`, `base` | Bytes transferred by `get`/`put` requests. A `get` is counted once its response body has been fully read. | +| `lance_object_store_request_duration_seconds` | histogram | `operation`, `base` | Per-request latency, in seconds. For `get` this covers the full body transfer, not just time-to-first-byte. | +| `lance_object_store_errors_total` | counter | `operation`, `base` | Requests that returned an error. | +| `lance_object_store_in_flight_requests` | gauge | `operation`, `base` | Requests currently in flight. | +| `lance_object_store_throttle_total` | counter | `status`, `base` | Throttle responses (HTTP 429 / 503) seen at the HTTP layer, counted per attempt including retries. The `status` label is the numeric HTTP status. | +| `lance_object_store_retryable_responses_total` | counter | `status`, `base` | Retryable responses (HTTP 5xx / 429 / 408) seen at the HTTP layer, counted per attempt including retries. A superset of `throttle_total`; 409 (conflict) is excluded so commit conflicts are not counted. | + +`lance_object_store_throttle_total` and +`lance_object_store_retryable_responses_total` are recorded only for the native +cloud stores (S3, GCS, Azure); Opendal-backed stores bypass the HTTP client +where the counters are installed, so they report the other object store metrics +but not throttle/retryable counts. diff --git a/rust/lance/src/metrics.rs b/rust/lance/src/metrics.rs new file mode 100644 index 00000000000..1d48865e859 --- /dev/null +++ b/rust/lance/src/metrics.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Metrics published by Lance. +#![doc = include_str!("metrics.md")] +//! +//! The metrics themselves are emitted from the relevant subsystems (for +//! example object store I/O is instrumented in +//! [`lance_io::object_store::metrics`]); this module exists to document the +//! full catalogue of metric names, types, and labels in one place. diff --git a/rust/lance/src/session.rs b/rust/lance/src/session.rs index c6214298341..8328ba20b22 100644 --- a/rust/lance/src/session.rs +++ b/rust/lance/src/session.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::sync::Arc; -use lance_core::cache::{CacheBackend, CacheKeyIterator, LanceCache}; +use lance_core::cache::{CacheBackend, LanceCache, QuickCacheBackend}; use lance_core::deepsize::DeepSizeOf; use lance_core::{Error, Result}; use lance_index::IndexType; @@ -21,6 +21,17 @@ pub(crate) mod caches; pub mod index_caches; pub(crate) mod index_extension; +/// Cache selection for one session cache tier. +#[derive(Clone, Debug)] +pub enum CacheSpec { + /// Use the Lance-level default capacity for the tier. + Default, + /// Use the default in-memory backend with this capacity. + Size(usize), + /// Use an already constructed backend. + Backend(Arc), +} + /// A user session holds the runtime state for a [`crate::Dataset`] /// /// A session will be created automatically when a Dataset is opened. However, you @@ -95,8 +106,10 @@ impl Session { /// /// Parameters: /// - /// - ***index_cache_size***: the size of the index cache. - /// - ***metadata_cache_size***: the size of the metadata cache. + /// - ***index_cache_size***: the size of the index cache, backed by + /// [`QuickCacheBackend`]. + /// - ***metadata_cache_size***: the size of the metadata cache, backed by + /// [`QuickCacheBackend`]. /// - ***store_registry***: the object store registry to use when opening /// datasets. This determines which schemes are available, and also allows /// re-using object stores. @@ -106,8 +119,12 @@ impl Session { store_registry: Arc, ) -> Self { Self { - index_cache: GlobalIndexCache(LanceCache::with_capacity(index_cache_size)), - metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)), + index_cache: GlobalIndexCache(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(index_cache_size), + ))), + metadata_cache: GlobalMetadataCache(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(metadata_cache_size), + ))), index_extensions: HashMap::new(), store_registry, spill_store: Arc::new(LocalSpillStore::default()), @@ -117,7 +134,7 @@ impl Session { /// Create a session with a custom index cache backend. /// /// The provided backend will be used for caching index data. The metadata - /// cache will use the default Moka-based backend with the given capacity. + /// cache uses a [`QuickCacheBackend`] with the given capacity. pub fn with_index_cache_backend( index_cache_backend: Arc, metadata_cache_size: usize, @@ -125,7 +142,9 @@ impl Session { ) -> Self { Self { index_cache: GlobalIndexCache(LanceCache::with_backend(index_cache_backend)), - metadata_cache: GlobalMetadataCache(LanceCache::with_capacity(metadata_cache_size)), + metadata_cache: GlobalMetadataCache(LanceCache::with_backend(Arc::new( + QuickCacheBackend::with_capacity(metadata_cache_size), + ))), index_extensions: HashMap::new(), store_registry, spill_store: Arc::new(LocalSpillStore::default()), @@ -157,6 +176,64 @@ impl Session { &*self.spill_store } + /// Create a session with custom backends for both caches. + /// + /// Each [`CacheSpec`] controls one tier. [`CacheSpec::Default`] uses that + /// tier's Lance-level default capacity, [`CacheSpec::Size`] uses the + /// default in-memory backend with an explicit capacity, and + /// [`CacheSpec::Backend`] uses a caller-provided backend. This keeps size + /// and backend selection mutually exclusive. + /// + /// This is the recommended constructor when a caller has already resolved + /// backend selection through + /// [`build_from_config`](lance_core::cache::build_from_config) or + /// [`build_from_uri`](lance_core::cache::build_from_uri) — the resulting + /// `Arc` can be plugged in for either or both caches. + /// + /// # Examples + /// + /// ``` + /// # use lance::session::{CacheSpec, Session}; + /// # use lance_core::cache::build_from_uri; + /// # fn example() -> lance_core::Result<()> { + /// let index_backend = build_from_uri("moka://?capacity=1048576")?; + /// let session = Session::with_cache_backends( + /// CacheSpec::Backend(index_backend), + /// CacheSpec::Default, + /// Default::default(), + /// ); + /// # let _ = session; + /// # Ok(()) + /// # } + /// ``` + pub fn with_cache_backends( + index_cache: CacheSpec, + metadata_cache: CacheSpec, + store_registry: Arc, + ) -> Self { + let index_cache = Self::build_cache(index_cache, DEFAULT_INDEX_CACHE_SIZE); + let metadata_cache = Self::build_cache(metadata_cache, DEFAULT_METADATA_CACHE_SIZE); + Self { + index_cache: GlobalIndexCache(index_cache), + metadata_cache: GlobalMetadataCache(metadata_cache), + index_extensions: HashMap::new(), + store_registry, + spill_store: Arc::new(LocalSpillStore::default()), + } + } + + fn build_cache(spec: CacheSpec, default_size: usize) -> LanceCache { + match spec { + CacheSpec::Default => { + LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(default_size))) + } + CacheSpec::Size(size) => { + LanceCache::with_backend(Arc::new(QuickCacheBackend::with_capacity(size))) + } + CacheSpec::Backend(backend) => LanceCache::with_backend(backend), + } + } + /// Register a new index extension. /// /// A name can only be registered once per type of index extension. @@ -239,44 +316,6 @@ impl Session { pub async fn index_cache_stats(&self) -> lance_core::cache::CacheStats { self.index_cache.0.stats().await } - - /// Return an iterator over keys currently held by the index cache. - /// - /// Returns `None` when the index cache backend does not support key - /// inventory. - /// - /// # Examples - /// - /// ``` - /// # use lance::session::Session; - /// # async fn example() { - /// let session = Session::default(); - /// let keys = session.index_cache_keys().await; - /// assert!(keys.is_some()); - /// # } - /// ``` - pub async fn index_cache_keys(&self) -> Option> { - self.index_cache.0.keys().await - } - - /// Return an iterator over keys currently held by the metadata cache. - /// - /// Returns `None` when the metadata cache backend does not support key - /// inventory. - /// - /// # Examples - /// - /// ``` - /// # use lance::session::Session; - /// # async fn example() { - /// let session = Session::default(); - /// let keys = session.metadata_cache_keys().await; - /// assert!(keys.is_some()); - /// # } - /// ``` - pub async fn metadata_cache_keys(&self) -> Option> { - self.metadata_cache.0.keys().await - } } impl Default for Session { @@ -306,7 +345,7 @@ mod tests { } fn type_name() -> &'static str { - "TestVec" + "Test" } } @@ -334,41 +373,72 @@ mod tests { ); } + /// `with_cache_backends` should honor whichever tier the caller + /// provided a backend for and fall back to that tier's default on the + /// other tier. + #[tokio::test] + async fn test_with_cache_backends_uses_provided_and_default() { + use lance_core::cache::build_from_uri; + + let index_backend = build_from_uri("moka://?capacity=1048576").unwrap(); + let session = Session::with_cache_backends( + CacheSpec::Backend(index_backend), + CacheSpec::Default, + Default::default(), + ); + + let value = Arc::new(vec![1, 2, 3]); + session + .index_cache + .insert_with_key(&TestKey("injected-index-backend"), value.clone()) + .await; + assert_eq!( + session + .index_cache + .get_with_key(&TestKey("injected-index-backend")) + .await + .as_deref(), + Some(value.as_ref()) + ); + // Metadata cache fell back to a size-based default. We can only + // sanity-check that the session was constructed without panicking. + let stats = session.metadata_cache.0.stats().await; + assert_eq!(stats.num_entries, 0); + } + #[tokio::test] - async fn test_session_cache_keys() { - let session = Session::new(10_000, 10_000, Default::default()); + async fn test_with_cache_backends_uses_explicit_size() { + let session = Session::with_cache_backends( + CacheSpec::Size(0), + CacheSpec::Size(2048), + Default::default(), + ); session .index_cache - .insert_with_key(&TestKey("index-key"), Arc::new(vec![1])) + .insert_with_key(&TestKey("disabled-index-cache"), Arc::new(vec![1, 2, 3])) .await; + assert!( + session + .index_cache + .get_with_key(&TestKey("disabled-index-cache")) + .await + .is_none() + ); + session .metadata_cache .0 - .insert_with_key(&TestKey("metadata-key"), Arc::new(vec![2])) + .insert_with_key(&TestKey("metadata-cache"), Arc::new(vec![4, 5, 6])) .await; - - let index_keys = session - .index_cache_keys() - .await - .unwrap() - .collect::>(); - assert_eq!(index_keys.len(), 1); - assert_eq!(index_keys[0].prefix(), ""); - assert_eq!(index_keys[0].key(), "index-key"); - assert_eq!(index_keys[0].type_name(), "TestVec"); - - let metadata_keys = session - .metadata_cache_keys() - .await - .unwrap() - .collect::>(); - assert_eq!(metadata_keys.len(), 1); - assert_eq!(metadata_keys[0].prefix(), ""); - assert_eq!(metadata_keys[0].key(), "metadata-key"); - assert_eq!(metadata_keys[0].type_name(), "TestVec"); - - assert_ne!(index_keys, metadata_keys); + assert!( + session + .metadata_cache + .0 + .get_with_key(&TestKey("metadata-cache")) + .await + .is_some() + ); } #[tokio::test] diff --git a/rust/lance/src/session/caches.rs b/rust/lance/src/session/caches.rs index a2dda6069ab..330dbdbd0b0 100644 --- a/rust/lance/src/session/caches.rs +++ b/rust/lance/src/session/caches.rs @@ -14,12 +14,12 @@ use std::{borrow::Cow, ops::Deref}; use lance_core::deepsize::{Context, DeepSizeOf}; use lance_core::{ - cache::{CacheKey, LanceCache}, + cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}, utils::deletion::DeletionVector, }; use lance_select::RowAddrMask; use lance_table::{ - format::{DeletionFile, Manifest}, + format::{DeletionFile, DeletionFileType, Manifest, RowIdMeta}, rowids::{RowIdIndex, RowIdSequence}, }; use object_store::path::Path; @@ -80,6 +80,20 @@ impl CacheKey for ManifestKey<'_> { fn type_name() -> &'static str { "Manifest" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.manifest-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + if let Some(e_tag) = self.e_tag { + builder.write_some(); + builder.write_str(e_tag); + } else { + builder.write_none(); + } + } } #[derive(Debug)] @@ -95,6 +109,14 @@ impl CacheKey for TransactionKey { fn type_name() -> &'static str { "Transaction" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.transaction-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + } } #[derive(Debug)] @@ -117,6 +139,26 @@ impl CacheKey for DeletionFileKey<'_> { fn type_name() -> &'static str { "DeletionVector" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.deletion-file-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.fragment_id); + builder.write_u64(self.deletion_file.read_version); + builder.write_u64(self.deletion_file.id); + builder.write_variant(match &self.deletion_file.file_type { + DeletionFileType::Array => 0, + DeletionFileType::Bitmap => 1, + }); + if let Some(base_id) = self.deletion_file.base_id { + builder.write_some(); + builder.write_u32(base_id); + } else { + builder.write_none(); + } + } } #[derive(Debug)] @@ -139,6 +181,20 @@ impl CacheKey for RowAddrMaskKey { fn type_name() -> &'static str { "RowAddrMask" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.row-address-mask-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + if let Some(restrict_hash) = self.restrict_hash { + builder.write_some(); + builder.write_u64(restrict_hash); + } else { + builder.write_none(); + } + } } #[derive(Debug)] @@ -154,21 +210,68 @@ impl CacheKey for RowIdIndexKey { fn type_name() -> &'static str { "RowIdIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.row-id-index-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.version); + } } #[derive(Debug)] -pub struct RowIdSequenceKey { +pub struct RowIdSequenceKey<'a> { pub fragment_id: u64, + /// Where the sequence is stored. A fragment id alone is not enough: this + /// cache is namespaced by dataset URI only (see + /// [`GlobalMetadataCache::for_dataset`]), and a dataset dropped and + /// recreated at the same URI restarts fragment ids at 0, so a reused id + /// would otherwise be served the earlier generation's sequence (#7645). + /// The `row_id_meta` differentiates generations of dataset fragments. + /// + /// Any operation that changes which row ids a fragment holds also writes it + /// new `row_id_meta`, so generations stay distinct; operations that leave + /// row ids alone (deletes, added columns) leave it untouched and keep + /// hitting the cache. + /// + /// For inline metadata the identity of the sequence *is* its encoded bytes, + /// so the key uses + /// [`InlineRowIds::digest`](lance_table::format::InlineRowIds::digest), + /// which those bytes memoize on first use — an array-encoded sequence is + /// 8 bytes per row, too much to rehash on every lookup. + pub row_id_meta: &'a RowIdMeta, } -impl CacheKey for RowIdSequenceKey { +impl CacheKey for RowIdSequenceKey<'_> { type ValueType = RowIdSequence; + // Only the legacy display form. Identity comes from `write_key` below. fn key(&self) -> Cow<'_, str> { Cow::Owned(format!("row_id_sequence/{}", self.fragment_id)) } fn type_name() -> &'static str { "RowIdSequence" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.dataset.row-id-sequence-key", 2) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_u64(self.fragment_id); + match self.row_id_meta { + RowIdMeta::Inline(data) => { + builder.write_variant(0); + builder.write_fixed_bytes(data.digest()); + } + RowIdMeta::External(file) => { + builder.write_variant(1); + builder.write_str(&file.path); + builder.write_u64(file.offset); + builder.write_u64(file.size); + } + } + } } impl DSMetadataCache { @@ -178,3 +281,121 @@ impl DSMetadataCache { self.0.with_key_prefix(prefix.as_ref()) } } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use lance_table::format::ExternalFile; + use lance_table::rowids::write_row_ids; + + use super::*; + + #[tokio::test] + async fn deletion_file_key_separates_storage_bases() { + let cache = LanceCache::with_capacity(4096); + let deletion_file = DeletionFile { + read_version: 3, + id: 4, + file_type: DeletionFileType::Bitmap, + num_deleted_rows: Some(1), + base_id: None, + }; + cache + .insert_with_key( + &DeletionFileKey { + fragment_id: 2, + deletion_file: &deletion_file, + }, + Arc::new(DeletionVector::NoDeletions), + ) + .await; + + let deletion_file_on_other_base = DeletionFile { + base_id: Some(7), + ..deletion_file + }; + assert!( + cache + .get_with_key(&DeletionFileKey { + fragment_id: 2, + deletion_file: &deletion_file_on_other_base, + }) + .await + .is_none() + ); + } + + #[tokio::test] + async fn row_id_sequence_key_separates_fragment_generations() { + // A dataset dropped and recreated at the same URI restarts fragment ids, + // so the same id must not resolve to the earlier generation's sequence. + let cache = LanceCache::with_capacity(4096); + let first_generation = RowIdMeta::Inline(write_row_ids(&(0..100).into()).into()); + let key = RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &first_generation, + }; + cache + .insert_with_key(&key, Arc::new(RowIdSequence::from(0..100))) + .await; + assert!(cache.get_with_key(&key).await.is_some()); + + let second_generation = RowIdMeta::Inline(write_row_ids(&(100..160).into()).into()); + assert!( + cache + .get_with_key(&RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &second_generation, + }) + .await + .is_none() + ); + } + + #[tokio::test] + async fn row_id_sequence_key_separates_external_slices() { + // External metadata is a read-only legacy shape, but the same slice of + // the same file is the only thing that may share a cache entry. + let cache = LanceCache::with_capacity(4096); + let external = |offset| { + RowIdMeta::External(ExternalFile { + path: "_row_ids/1.rowids".into(), + offset, + size: 16, + }) + }; + let first_slice = external(0); + cache + .insert_with_key( + &RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &first_slice, + }, + Arc::new(RowIdSequence::from(0..100)), + ) + .await; + + let second_slice = external(16); + assert!( + cache + .get_with_key(&RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &second_slice, + }) + .await + .is_none() + ); + // An inline sequence never aliases an external one. + let inline = RowIdMeta::Inline(write_row_ids(&(0..100).into()).into()); + assert!( + cache + .get_with_key(&RowIdSequenceKey { + fragment_id: 0, + row_id_meta: &inline, + }) + .await + .is_none() + ); + } +} diff --git a/rust/lance/src/session/index_caches.rs b/rust/lance/src/session/index_caches.rs index e93b7208b09..23922c7a4b0 100644 --- a/rust/lance/src/session/index_caches.rs +++ b/rust/lance/src/session/index_caches.rs @@ -12,9 +12,9 @@ use std::{borrow::Cow, ops::Deref, sync::Arc}; -use lance_core::cache::{CacheKey, LanceCache}; +use lance_core::cache::{CacheKey, CacheKeySchema, KeyBuilder, LanceCache}; use lance_core::deepsize::{Context, DeepSizeOf}; -use lance_index::frag_reuse::FragReuseIndex; +use lance_index::frag_reuse::CompactFragReuseIndex; use lance_table::format::IndexMetadata; use uuid::Uuid; @@ -64,17 +64,31 @@ impl Deref for DSIndexCache { impl DSIndexCache { /// Create an index-specific cache with the given UUID prefix. pub fn for_index(&self, uuid: &Uuid, fri_uuid: Option<&Uuid>) -> LanceCache { + let mut uuid_buffer = Uuid::encode_buffer(); + let cache = self + .0 + .with_key_prefix(uuid.as_hyphenated().encode_lower(&mut uuid_buffer)); if let Some(fri_uuid) = fri_uuid { // If a FRI UUID is provided, use it to create a more specific cache key. - let cache_key = format!("{}-{}", uuid, fri_uuid); - self.0.with_key_prefix(&cache_key) + let mut fri_uuid_buffer = Uuid::encode_buffer(); + cache.with_key_prefix(fri_uuid.as_hyphenated().encode_lower(&mut fri_uuid_buffer)) } else { // Otherwise, just use the index UUID as the key prefix. - self.0.with_key_prefix(&uuid.to_string()) + cache } } } +pub(crate) fn write_index_identity(builder: &mut KeyBuilder, uuid: &Uuid, fri_uuid: Option<&Uuid>) { + builder.write_fixed_bytes(uuid.as_bytes()); + if let Some(fri_uuid) = fri_uuid { + builder.write_some(); + builder.write_fixed_bytes(fri_uuid.as_bytes()); + } else { + builder.write_none(); + } +} + // Cache key types for type-safe cache access #[derive(Debug)] @@ -83,7 +97,7 @@ pub struct FragReuseIndexKey<'a> { } impl CacheKey for FragReuseIndexKey<'_> { - type ValueType = FragReuseIndex; + type ValueType = CompactFragReuseIndex; fn key(&self) -> Cow<'_, str> { Cow::Owned(format!("frag_reuse/{}", self.uuid)) @@ -92,24 +106,60 @@ impl CacheKey for FragReuseIndexKey<'_> { fn type_name() -> &'static str { "FragReuseIndex" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.fragment-reuse-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_fixed_bytes(self.uuid.as_bytes()); + } } -#[derive(Debug)] -pub struct IndexMetadataKey { +#[derive(Clone, Copy, Debug)] +pub struct IndexMetadataKey<'a> { pub version: u64, + pub store_identity: &'a str, + pub e_tag: Option<&'a str>, } -impl CacheKey for IndexMetadataKey { +impl CacheKey for IndexMetadataKey<'_> { type ValueType = Vec; fn key(&self) -> Cow<'_, str> { - Cow::Owned(self.version.to_string()) + Cow::Owned(format!( + "{}:{}/{}/{}", + self.store_identity.len(), + self.store_identity, + self.version, + self.e_tag.unwrap_or("") + )) } fn type_name() -> &'static str { "Vec" } + fn schema() -> CacheKeySchema { + // v2 holds every index the manifest names; v1 held only the ones the + // writing build could read. The fields are identical, so on a persistent + // backend shared with another release nothing but this version stops each + // build from reading the other's entry as its own meaning. + CacheKeySchema::new("lance.index.metadata-key", 2) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_str(self.store_identity); + builder.write_u64(self.version); + match self.e_tag { + Some(e_tag) => { + builder.write_some(); + builder.write_str(e_tag); + } + None => builder.write_none(), + } + } + fn codec() -> Option { Some(lance_table::format::index_metadata_codec()) } @@ -144,4 +194,49 @@ impl CacheKey for ScalarIndexDetailsKey<'_> { fn type_name() -> &'static str { "ScalarIndexDetails" } + + fn schema() -> CacheKeySchema { + CacheKeySchema::new("lance.index.scalar-details-key", 1) + } + + fn write_key(&self, builder: &mut KeyBuilder) { + builder.write_fixed_bytes(self.uuid.as_bytes()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn index_metadata_key_isolates_object_store_identity() { + let first = IndexMetadataKey { + version: 7, + store_identity: "s3$first-options", + e_tag: Some("manifest-etag"), + }; + let second = IndexMetadataKey { + version: 7, + store_identity: "s3$second-options", + e_tag: Some("manifest-etag"), + }; + + assert_ne!(first.key(), second.key()); + } + + #[test] + fn index_metadata_key_isolates_manifest_generation() { + let first = IndexMetadataKey { + version: 7, + store_identity: "s3$options", + e_tag: Some("first-etag"), + }; + let second = IndexMetadataKey { + version: 7, + store_identity: "s3$options", + e_tag: Some("second-etag"), + }; + + assert_ne!(first.key(), second.key()); + } } diff --git a/rust/lance/src/session/index_extension.rs b/rust/lance/src/session/index_extension.rs index 0f4d2ecf310..c93ae3269ef 100644 --- a/rust/lance/src/session/index_extension.rs +++ b/rust/lance/src/session/index_extension.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use lance_core::Result; use lance_core::deepsize::DeepSizeOf; -use lance_file::previous::reader::FileReader as PreviousFileReader; +use lance_file::versions::v1::reader::FileReader as V1FileReader; use lance_index::{IndexParams, IndexType, vector::VectorIndex}; use uuid::Uuid; @@ -48,7 +48,7 @@ pub trait VectorIndexExtension: IndexExtension { dataset: Arc, column: &str, uuid: &Uuid, - reader: PreviousFileReader, + reader: V1FileReader, ) -> Result>; } @@ -72,10 +72,10 @@ mod test { use arrow_schema::Schema; use datafusion::execution::SendableRecordBatchStream; use lance_core::deepsize::DeepSizeOf; - use lance_file::previous::writer::{ - FileWriter as PreviousFileWriter, FileWriterOptions as PreviousFileWriterOptions, - }; use lance_file::version::LanceFileVersion; + use lance_file::versions::v1::writer::{ + FileWriter as V1FileWriter, FileWriterOptions as V1FileWriterOptions, + }; use lance_index::vector::v3::subindex::SubIndexType; use lance_index::{ INDEX_FILE_NAME, INDEX_METADATA_SCHEMA_KEY, Index, IndexMetadata, IndexType, @@ -270,13 +270,9 @@ mod test { let arrow_schema = Arc::new(Schema::new(vec![VECTOR_ID_FIELD.clone()])); let schema = lance_core::datatypes::Schema::try_from(arrow_schema.as_ref()).unwrap(); - let mut writer: PreviousFileWriter = - PreviousFileWriter::with_object_writer( - writer, - schema, - &PreviousFileWriterOptions::default(), - ) - .unwrap(); + let mut writer: V1FileWriter = + V1FileWriter::with_object_writer(writer, schema, &V1FileWriterOptions::default()) + .unwrap(); writer.add_metadata( INDEX_METADATA_SCHEMA_KEY, json!(IndexMetadata { @@ -304,7 +300,7 @@ mod test { _dataset: Arc, _column: &str, _uuid: &Uuid, - _reader: PreviousFileReader, + _reader: V1FileReader, ) -> Result> { self.load_index_called .store(true, std::sync::atomic::Ordering::Release); diff --git a/rust/lance/src/utils/test.rs b/rust/lance/src/utils/test.rs index 3338eee07a8..9de4b762761 100644 --- a/rust/lance/src/utils/test.rs +++ b/rust/lance/src/utils/test.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The Lance Authors -use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; use lance_core::utils::tempfile::{TempDir, TempStrDir}; @@ -12,7 +13,12 @@ use lance_arrow::RecordBatchExt; use lance_core::datatypes::Schema; use lance_datagen::{BatchCount, BatchGeneratorBuilder, ByteCount, RowCount}; use lance_file::version::LanceFileVersion; -use lance_table::format::Fragment; +use lance_table::format::pb::transaction::Operation as PbOperation; +use lance_table::format::{Fragment, Transaction as TableTransaction}; +use lance_table::io::commit::{ + CommitError, CommitHandler, ConditionalPutCommitHandler, ManifestLocation, + ManifestNamingScheme, ManifestWriter, +}; use rand::prelude::SliceRandom; use rand::{Rng, SeedableRng}; @@ -21,7 +27,9 @@ use crate::dataset::WriteParams; use crate::dataset::fragment::write::FragmentCreateBuilder; use crate::dataset::transaction::Operation; +pub mod covering; mod failing_store; +pub mod serializing_cache; mod throttle_store; pub use failing_store::FailingProxyStore; @@ -243,6 +251,7 @@ impl TestDatasetGenerator { Fragment { id: 0, files, + overlays: vec![], deletion_file: None, row_id_meta: None, physical_rows: Some(batch.num_rows()), @@ -632,17 +641,8 @@ mod tests { .flat_map(|file| file.fields.iter()) .cloned() .collect::>(); - let mut field_ids = schema - .fields_pre_order() - .filter_map(|f| { - if data_storage_version < LanceFileVersion::V2_1 || f.children.is_empty() { - Some(f.id) - } else { - // In 2.1+, struct / list fields don't have their own column - None - } - }) - .collect::>(); + let (mut field_ids, _) = + lance_file::versions::data_file_columns(data_storage_version.resolve(), &schema); field_ids_frags.sort_unstable(); field_ids.sort_unstable(); assert_eq!(field_ids_frags, field_ids); @@ -729,3 +729,201 @@ mod tests { } } } + +/// How [`AmbiguousCommitHandler`] should treat the next commit. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AmbiguousFailure { + /// Apply the commit (the manifest lands), then report a conflict — the + /// store equivalent of a successful conditional PUT whose response was + /// lost and whose internal retry saw "already exists". + LandAndConflict, + /// Apply the commit, then report an I/O error. + LandAndError, + /// Do not apply the commit; report an I/O error. + FailOutright, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AmbiguousFailureTarget { + Any, + Rewrite, + ReserveFragments, +} + +/// A commit handler for tests that can make a commit physically land while +/// reporting a failure, and can make commit-outcome verification unavailable. +/// +/// Delegates real work to [`ConditionalPutCommitHandler`]. +#[derive(Debug)] +pub struct AmbiguousCommitHandler { + /// Failure to inject into the next commit; taken (reset to `None`) when + /// the commit runs. + fail_next_commit: Mutex>, + /// When set, version resolution fails, making commit-outcome verification + /// impossible. + pub fail_resolve: AtomicBool, + /// Number of upcoming resolution calls that should return NotFound. + resolve_not_found_remaining: AtomicUsize, + /// Whether a persistent NotFound proves that the requested version is + /// absent. Tests can disable this to model an eventually visible resolver. + not_found_is_definitive: AtomicBool, + /// Number of version resolutions requested (a proxy for how often commit + /// verification ran). + resolve_calls: AtomicUsize, +} + +impl Default for AmbiguousCommitHandler { + fn default() -> Self { + Self { + fail_next_commit: Mutex::new(None), + fail_resolve: AtomicBool::new(false), + resolve_not_found_remaining: AtomicUsize::new(0), + not_found_is_definitive: AtomicBool::new(true), + resolve_calls: AtomicUsize::new(0), + } + } +} + +impl AmbiguousCommitHandler { + pub fn fail_next(&self, failure: AmbiguousFailure) { + *self.fail_next_commit.lock().unwrap() = Some((failure, AmbiguousFailureTarget::Any)); + } + + /// Arm the failure for the next Rewrite commit only. + pub fn fail_next_rewrite(&self, failure: AmbiguousFailure) { + *self.fail_next_commit.lock().unwrap() = Some((failure, AmbiguousFailureTarget::Rewrite)); + } + + /// Arm the failure for the next ReserveFragments commit only. + pub fn fail_next_reserve(&self, failure: AmbiguousFailure) { + *self.fail_next_commit.lock().unwrap() = + Some((failure, AmbiguousFailureTarget::ReserveFragments)); + } + + pub fn fail_next_resolves_with_not_found(&self, count: usize) { + self.resolve_not_found_remaining + .store(count, Ordering::SeqCst); + self.not_found_is_definitive.store(false, Ordering::SeqCst); + } + + pub fn resolve_calls(&self) -> usize { + self.resolve_calls.load(Ordering::SeqCst) + } +} + +#[async_trait::async_trait] +impl CommitHandler for AmbiguousCommitHandler { + fn is_version_not_found_definitive(&self) -> bool { + self.not_found_is_definitive.load(Ordering::SeqCst) + } + + fn propagate_commit_error_after_success(&self) -> bool { + false + } + + async fn commit( + &self, + manifest: &mut lance_table::format::Manifest, + indices: Option>, + base_path: &object_store::path::Path, + object_store: &lance_io::object_store::ObjectStore, + manifest_writer: ManifestWriter, + naming_scheme: ManifestNamingScheme, + transaction: Option, + ) -> std::result::Result { + let operation = transaction + .as_ref() + .and_then(|t| t.as_pb().operation.as_ref()) + .map(|operation| match operation { + PbOperation::Rewrite(_) => AmbiguousFailureTarget::Rewrite, + PbOperation::ReserveFragments(_) => AmbiguousFailureTarget::ReserveFragments, + _ => AmbiguousFailureTarget::Any, + }); + let failure = { + let mut armed = self.fail_next_commit.lock().unwrap(); + let matches_target = armed.as_ref().is_some_and(|(_, target)| { + *target == AmbiguousFailureTarget::Any || operation == Some(*target) + }); + matches_target.then(|| armed.take().unwrap().0) + }; + match failure { + None => { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await + } + Some(AmbiguousFailure::LandAndConflict) => { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await?; + Err(CommitError::CommitConflict) + } + Some(AmbiguousFailure::LandAndError) => { + ConditionalPutCommitHandler + .commit( + manifest, + indices, + base_path, + object_store, + manifest_writer, + naming_scheme, + transaction, + ) + .await?; + Err(CommitError::OtherError(lance_core::Error::io( + "simulated ambiguous commit failure", + ))) + } + Some(AmbiguousFailure::FailOutright) => Err(CommitError::OtherError( + lance_core::Error::io("simulated outright commit failure"), + )), + } + } + + async fn resolve_version_location( + &self, + base_path: &object_store::path::Path, + version: u64, + object_store: &dyn object_store::ObjectStore, + ) -> lance_core::Result { + self.resolve_calls.fetch_add(1, Ordering::SeqCst); + if self.fail_resolve.load(Ordering::SeqCst) { + return Err(lance_core::Error::io("simulated verification outage")); + } + let mut remaining = self.resolve_not_found_remaining.load(Ordering::SeqCst); + while remaining > 0 { + match self.resolve_not_found_remaining.compare_exchange_weak( + remaining, + remaining - 1, + Ordering::SeqCst, + Ordering::SeqCst, + ) { + Ok(_) => { + return Err(lance_core::Error::not_found( + "simulated temporarily invisible manifest", + )); + } + Err(actual) => remaining = actual, + } + } + ConditionalPutCommitHandler + .resolve_version_location(base_path, version, object_store) + .await + } +} diff --git a/rust/lance/src/utils/test/covering.rs b/rust/lance/src/utils/test/covering.rs new file mode 100644 index 00000000000..bf559663115 --- /dev/null +++ b/rust/lance/src/utils/test/covering.rs @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Fixtures for tests about covering indexes. +//! +//! No index builder writes carried values yet, so there is no API that +//! produces a covering index. Every covering test has to build a plain index +//! and then re-commit its metadata with the declaration attached, which is +//! what [`declare_covering`] does. Once a creation API exists these fixtures +//! collapse into calls to it. + +use std::sync::Arc; + +use arrow_array::{FixedSizeListArray, Int32Array, RecordBatch, RecordBatchIterator}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use lance_arrow::FixedSizeListArrayExt; +use lance_index::IndexType; +use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; +use lance_linalg::distance::MetricType; +use lance_testing::datagen::generate_random_array; + +use crate::Dataset; +use crate::dataset::transaction::{Operation, Transaction}; +use crate::index::DatasetIndexExt; +use crate::index::vector::VectorIndexParams; + +/// Rows written per fragment by the fixtures in this module. +pub const ROWS_PER_FRAGMENT: i32 = 512; + +/// Vector width used by the fixtures in this module. +pub const DIMENSION: i32 = 16; + +/// Partitions used by [`create_ivf_pq_index`]. +/// +/// Four, not one: with a single partition the probe path is never reached, so +/// a selection rule that wrongly picks a covered index still looks correct. +pub const NUM_PARTITIONS: u32 = 4; + +fn vector_field(name: &str) -> ArrowField { + ArrowField::new( + name, + DataType::FixedSizeList( + Arc::new(ArrowField::new("item", DataType::Float32, true)), + DIMENSION, + ), + false, + ) +} + +fn random_vectors(rows: i32) -> Arc { + Arc::new( + FixedSizeListArray::try_new_from_values( + generate_random_array(rows as usize * DIMENSION as usize), + DIMENSION, + ) + .unwrap(), + ) +} + +/// A one-fragment dataset with a `vec` column to key an index on and an +/// `Int32` `payload` column to declare as carried. +pub async fn write_vector_payload_dataset(uri: &str) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + vector_field("vec"), + ArrowField::new("payload", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + random_vectors(ROWS_PER_FRAGMENT), + Arc::new(Int32Array::from_iter_values(0..ROWS_PER_FRAGMENT)), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(reader, uri, None).await.unwrap() +} + +/// A one-fragment dataset whose carried column is itself a vector, for tests +/// about which column an index may be *selected* for. +pub async fn write_two_vector_column_dataset(uri: &str) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + vector_field("vec"), + vector_field("payload_vec"), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + random_vectors(ROWS_PER_FRAGMENT), + random_vectors(ROWS_PER_FRAGMENT), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(reader, uri, None).await.unwrap() +} + +/// Append a fragment to a [`write_vector_payload_dataset`] dataset, leaving +/// every index stale. +/// +/// Tests about maintenance need this so the index group really would be +/// rebuilt; without it they assert against a group the operation had no work +/// for, and keep passing if the behavior under test moves behind a no-work +/// check. +pub async fn append_vector_payload_rows(dataset: &mut Dataset, rows: i32) { + let schema = Arc::new(ArrowSchema::new(vec![ + vector_field("vec"), + ArrowField::new("payload", DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + random_vectors(rows), + Arc::new(Int32Array::from_iter_values( + ROWS_PER_FRAGMENT..ROWS_PER_FRAGMENT + rows, + )), + ], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + dataset.append(reader, None).await.unwrap(); +} + +/// Build an IVF_PQ index on `column`, with [`NUM_PARTITIONS`] partitions so +/// the probe path is genuinely reached. +pub async fn create_ivf_pq_index(dataset: &mut Dataset, column: &str) { + let params = VectorIndexParams::ivf_pq(NUM_PARTITIONS as usize, 8, 2, MetricType::L2, 50); + dataset + .create_index(&[column], IndexType::Vector, None, ¶ms, true) + .await + .unwrap(); +} + +/// Build a BTree index on `column`. +pub async fn create_btree_index(dataset: &mut Dataset, column: &str, name: Option<&str>) { + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index( + &[column], + IndexType::BTree, + name.map(str::to_string), + ¶ms, + true, + ) + .await + .unwrap(); +} + +/// A one-fragment dataset of three `Int32` columns, for tests about covered +/// *scalar* indexes: `a` and `b` are keyed, `carried` plays the covered column. +pub async fn write_three_int_column_dataset(uri: &str) -> Dataset { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, false), + ArrowField::new("carried", DataType::Int32, false), + ])); + let column = || Arc::new(Int32Array::from_iter_values(0..64)) as _; + let batch = RecordBatch::try_new(schema.clone(), vec![column(), column(), column()]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + Dataset::write(reader, uri, None).await.unwrap() +} + +/// Commit a covering declaration for `keyed`/`carried` under `name`, with no +/// index files behind it. +/// +/// Unlike [`declare_covering`], this does not need a real index to exist: +/// guards that only look up an entry in the manifest are reached just as well +/// by a synthetic one, and building a real index first would cost more than it +/// proves. Returns the two field ids. +pub async fn commit_synthetic_covered_index( + dataset: &mut Dataset, + name: &str, + keyed: &str, + carried: &str, +) -> (i32, i32) { + let keyed_id = dataset.schema().field_id(keyed).unwrap(); + let carried_id = dataset.schema().field_id(carried).unwrap(); + + let covered = lance_table::format::IndexMetadata { + uuid: uuid::Uuid::new_v4(), + name: name.to_string(), + fields: vec![keyed_id, carried_id], + covering_fields: vec![carried_id], + dataset_version: dataset.manifest.version, + fragment_bitmap: Some(dataset.fragment_bitmap.as_ref().clone()), + index_details: None, + index_version: 0, + created_at: Some(chrono::Utc::now()), + base_id: None, + files: None, + }; + + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![covered], + removed_indices: vec![], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + (keyed_id, carried_id) +} + +/// Append a fragment to a [`write_three_int_column_dataset`] dataset, leaving +/// every index stale. +pub async fn append_three_int_column_rows(dataset: &mut Dataset, rows: i32) { + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("a", DataType::Int32, false), + ArrowField::new("b", DataType::Int32, false), + ArrowField::new("carried", DataType::Int32, false), + ])); + let column = || Arc::new(Int32Array::from_iter_values(64..64 + rows)) as _; + let batch = RecordBatch::try_new(schema.clone(), vec![column(), column(), column()]).unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + dataset.append(reader, None).await.unwrap(); +} + +/// Re-commit the index keyed on `keyed` so that it declares `carried` as a +/// covering column, and return the two field ids. +/// +/// Only that index is replaced; any other index on the table is left in place, +/// which is what tests about one covered index not blocking the others rely +/// on. +pub async fn declare_covering(dataset: &mut Dataset, keyed: &str, carried: &str) -> (i32, i32) { + let keyed_id = dataset.schema().field_id(keyed).unwrap(); + let carried_id = dataset.schema().field_id(carried).unwrap(); + + let current = dataset.load_indices().await.unwrap(); + let plain = current + .iter() + .find(|idx| idx.fields == vec![keyed_id]) + .cloned() + .unwrap_or_else(|| panic!("no index keyed on '{keyed}' to declare covering on")); + + let covered = lance_table::format::IndexMetadata { + fields: vec![keyed_id, carried_id], + covering_fields: vec![carried_id], + ..plain.clone() + }; + + dataset + .apply_commit( + Transaction::new( + dataset.manifest.version, + Operation::CreateIndex { + new_indices: vec![covered], + removed_indices: vec![plain], + }, + None, + ), + &Default::default(), + &Default::default(), + ) + .await + .unwrap(); + + (keyed_id, carried_id) +} diff --git a/rust/lance/src/utils/test/failing_store.rs b/rust/lance/src/utils/test/failing_store.rs index 07b5f7da92d..ca8b6eca419 100644 --- a/rust/lance/src/utils/test/failing_store.rs +++ b/rust/lance/src/utils/test/failing_store.rs @@ -89,6 +89,12 @@ impl FailingProxyStore { }), ); } + + /// Stop failing calls configured by [`Self::fail_when`]. + pub fn clear_fail_when(&self, method: &str, path_substr: &str) { + let mut policy = self.policy.lock().unwrap(); + policy.clear_before_policy(&format!("fail_{}_{}", method, path_substr)); + } } impl WrappingObjectStore for FailingProxyStore { @@ -99,4 +105,13 @@ impl WrappingObjectStore for FailingProxyStore { ) -> Arc { Arc::new(ProxyObjectStore::new(original, self.policy.clone())) } + + // Injects behaviour into every request, so a listing must not go around it. + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } diff --git a/rust/lance/src/utils/test/serializing_cache.rs b/rust/lance/src/utils/test/serializing_cache.rs new file mode 100644 index 00000000000..89f2b2cbb74 --- /dev/null +++ b/rust/lance/src/utils/test/serializing_cache.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::Future; +use lance_core::Result; +use lance_core::cache::{CacheBackend, CacheCodec, CacheEntry, InternalCacheKey, MokaCacheBackend}; + +#[derive(Debug)] +struct SerializedEntry { + bytes: Bytes, + size_bytes: usize, +} + +#[derive(Debug, Default)] +struct SerializedStore { + entries: tokio::sync::Mutex>, + insert_counts: tokio::sync::Mutex>, +} + +/// Test-only cache backend that forces codec-backed entries through bytes. +/// +/// The serialized store can survive [`restart`](Self::restart), while entries +/// without a codec live only in the in-memory L1. This deliberately models the +/// persistence boundary, not a production cache: +/// - it has no capacity limit or eviction policy for serialized entries; +/// - concurrent serialized misses are not single-flight; +/// - accounting uses the logical size supplied by the cache layer. +#[derive(Debug)] +pub struct SerializingCacheBackend { + serialized: Arc, + l1: MokaCacheBackend, +} + +impl SerializingCacheBackend { + pub fn new() -> Self { + Self { + serialized: Arc::new(SerializedStore::default()), + l1: MokaCacheBackend::with_capacity(256 * 1024 * 1024), + } + } + + /// Recreate the backend over the same serialized bytes and an empty L1. + pub fn restart(&self) -> Self { + Self { + serialized: self.serialized.clone(), + l1: MokaCacheBackend::with_capacity(256 * 1024 * 1024), + } + } + + pub async fn serialized_entry_count(&self) -> usize { + self.serialized.entries.lock().await.len() + } + + pub async fn serialized_insert_count(&self, type_id: &'static str) -> usize { + self.serialized + .insert_counts + .lock() + .await + .get(type_id) + .copied() + .unwrap_or(0) + } + + pub async fn l1_entry_count(&self) -> usize { + self.l1.num_entries().await + } +} + +#[async_trait] +impl CacheBackend for SerializingCacheBackend { + async fn get(&self, key: &InternalCacheKey, codec: Option) -> Option { + let Some(codec) = codec else { + return self.l1.get(key, None).await; + }; + let bytes = self + .serialized + .entries + .lock() + .await + .get(key) + .map(|entry| entry.bytes.clone())?; + codec.deserialize(&bytes).hit() + } + + async fn insert( + &self, + key: &InternalCacheKey, + entry: CacheEntry, + size_bytes: usize, + codec: Option, + ) { + let Some(codec) = codec else { + self.l1.insert(key, entry, size_bytes, None).await; + return; + }; + let mut bytes = Vec::new(); + codec + .serialize(&entry, &mut bytes) + .expect("test cache entry serialization should succeed"); + let mut insert_counts = self.serialized.insert_counts.lock().await; + *insert_counts.entry(codec.type_id()).or_default() += 1; + drop(insert_counts); + self.serialized.entries.lock().await.insert( + *key, + SerializedEntry { + bytes: Bytes::from(bytes), + size_bytes, + }, + ); + } + + async fn get_or_insert<'a>( + &self, + key: &InternalCacheKey, + loader: Pin> + Send + 'a>>, + codec: Option, + ) -> Result<(CacheEntry, bool)> { + if let Some(entry) = self.get(key, codec).await { + return Ok((entry, true)); + } + let (entry, size_bytes) = loader.await?; + self.insert(key, entry.clone(), size_bytes, codec).await; + Ok((entry, false)) + } + + async fn clear(&self) { + self.serialized.entries.lock().await.clear(); + self.l1.clear().await; + } + + async fn num_entries(&self) -> usize { + self.serialized.entries.lock().await.len() + self.l1.num_entries().await + } + + async fn size_bytes(&self) -> usize { + let serialized_size = self + .serialized + .entries + .lock() + .await + .values() + .map(|entry| entry.size_bytes) + .sum::(); + serialized_size.saturating_add(self.l1.size_bytes().await) + } +} + +#[cfg(test)] +mod tests { + use lance_core::cache::{CacheCodecImpl, CacheEntryReader, CacheEntryWriter}; + use lance_core::{Error, Result}; + + use super::*; + + #[derive(Debug)] + struct StoredValue(u32); + + impl CacheCodecImpl for StoredValue { + const TYPE_ID: &'static str = "test.LookupCodec"; + const CURRENT_VERSION: u32 = 1; + + fn serialize(&self, writer: &mut CacheEntryWriter<'_>) -> Result<()> { + writer.write_raw(&self.0.to_le_bytes()) + } + + fn deserialize(_reader: &mut CacheEntryReader<'_>) -> Result { + Err(Error::internal( + "the codec retained at insert time must not be used for lookup", + )) + } + } + + #[derive(Debug, PartialEq)] + struct LookupValue(u32); + + impl CacheCodecImpl for LookupValue { + const TYPE_ID: &'static str = "test.LookupCodec"; + const CURRENT_VERSION: u32 = 1; + + fn serialize(&self, writer: &mut CacheEntryWriter<'_>) -> Result<()> { + writer.write_raw(&self.0.to_le_bytes()) + } + + fn deserialize(reader: &mut CacheEntryReader<'_>) -> Result { + let bytes = reader.read_raw()?; + let value = u32::from_le_bytes( + bytes + .as_ref() + .try_into() + .map_err(|_| Error::internal("invalid test cache value"))?, + ); + Ok(Self(value)) + } + } + + #[tokio::test] + async fn restart_keeps_bytes_uses_lookup_codec_and_discards_l1() { + let backend = SerializingCacheBackend::new(); + let serialized_key = InternalCacheKey::from_bytes([1; 16]); + backend + .insert( + &serialized_key, + Arc::new(StoredValue(42)), + 4, + Some(CacheCodec::from_impl::()), + ) + .await; + let l1_key = InternalCacheKey::from_bytes([2; 16]); + backend.insert(&l1_key, Arc::new(7_u32), 4, None).await; + + let restarted = backend.restart(); + assert_eq!(restarted.serialized_entry_count().await, 1); + assert_eq!( + restarted + .serialized_insert_count(StoredValue::TYPE_ID) + .await, + 1 + ); + assert_eq!(restarted.l1_entry_count().await, 0); + assert!(restarted.get(&l1_key, None).await.is_none()); + + let decoded = restarted + .get( + &serialized_key, + Some(CacheCodec::from_impl::()), + ) + .await + .unwrap() + .downcast::() + .unwrap(); + assert_eq!(*decoded, LookupValue(42)); + assert_eq!( + restarted + .serialized_insert_count(StoredValue::TYPE_ID) + .await, + 1 + ); + } +} diff --git a/rust/lance/src/utils/test/throttle_store.rs b/rust/lance/src/utils/test/throttle_store.rs index 8b4897cb57f..cb8159016d6 100644 --- a/rust/lance/src/utils/test/throttle_store.rs +++ b/rust/lance/src/utils/test/throttle_store.rs @@ -19,4 +19,13 @@ impl WrappingObjectStore for ThrottledStoreWrapper { let throttle_store = ThrottledStore::new(original, self.config); Arc::new(throttle_store) } + + // Injects behaviour into every request, so a listing must not go around it. + fn wrap_paginated( + &self, + _store_prefix: &str, + _original: Arc, + ) -> Option> { + None + } } diff --git a/rust/lance/tests/count_pushdown/mod.rs b/rust/lance/tests/count_pushdown/mod.rs index aaa3f5f539e..d8afa051bca 100644 --- a/rust/lance/tests/count_pushdown/mod.rs +++ b/rust/lance/tests/count_pushdown/mod.rs @@ -81,7 +81,7 @@ fn lance_aware_context(dataset: Arc) -> SessionContext { fn plan_contains_pushdown(plan: &Arc) -> bool { let mut found = false; plan.apply(|node| { - if node.as_any().is::() { + if node.is::() { found = true; Ok(TreeNodeRecursion::Stop) } else { diff --git a/rust/lance/tests/integration_tests.rs b/rust/lance/tests/integration_tests.rs index 7a6d3e71ca4..e6ea4e321ae 100644 --- a/rust/lance/tests/integration_tests.rs +++ b/rust/lance/tests/integration_tests.rs @@ -4,6 +4,7 @@ // NOTE: we only create one integration test binary, to keep compilation overhead down. mod count_pushdown; +mod mem_wal; #[cfg(feature = "slow_tests")] mod query; #[cfg(feature = "slow_tests")] diff --git a/rust/lance/tests/mem_wal/mod.rs b/rust/lance/tests/mem_wal/mod.rs new file mode 100644 index 00000000000..c617c2a1589 --- /dev/null +++ b/rust/lance/tests/mem_wal/mod.rs @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! End-to-end MemWAL tests through the public [`ShardWriter`] surface, as +//! opposed to the lib-level unit tests that can reach internals directly. + +use std::time::Duration; + +use arrow_array::cast::AsArray; +use arrow_array::record_batch; +use arrow_array::types::Int32Type; +use lance::dataset::mem_wal::{ShardWriter, ShardWriterConfig}; +use lance_core::FenceReason; +use lance_io::object_store::ObjectStore; +use uuid::Uuid; + +fn durable_writer_config(shard_id: Uuid) -> ShardWriterConfig { + ShardWriterConfig { + shard_id, + durable_write: true, + max_wal_buffer_size: 64 * 1024 * 1024, + max_wal_flush_interval: Some(Duration::from_millis(10)), + max_memtable_size: 64 * 1024 * 1024, + manifest_scan_batch_size: 2, + ..Default::default() + } +} + +#[tokio::test] +async fn durable_put_is_readable_through_public_scan() { + let base_uri = "memory://".to_string(); + let (store, base_path) = ObjectStore::from_uri(&base_uri).await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2, 3])).unwrap(); + let writer = ShardWriter::open( + store, + base_path, + base_uri, + durable_writer_config(Uuid::new_v4()), + batch.schema(), + vec![], + ) + .await + .unwrap(); + + writer.put(vec![batch]).await.unwrap(); + + let scanned = writer.scan().await.unwrap().try_into_batch().await.unwrap(); + let scanned_ids = scanned["id"].as_primitive::(); + assert_eq!( + scanned_ids.values(), + &[1, 2, 3], + "a durable put must be readable through the public scan API" + ); + + writer.close().await.unwrap(); +} + +/// A durable put into a post-rotation MemTable must be acknowledged only by +/// its own WAL flush, never by a flush from an older generation. +/// +/// Batch positions restart at 0 in every MemTable generation. A durability +/// watch keyed on that generation-local position is satisfied by the previous +/// generation's flush at the same position — so the first durable put after a +/// rotation used to return success before its own WAL append completed. Here +/// a peer writer has claimed a higher epoch in between, so the false success +/// is observable: the put's own flush is fenced, and the put must surface +/// that fence instead of reporting durability. +#[tokio::test] +async fn durable_put_does_not_alias_across_memtable_generations() { + let base_uri = "memory://".to_string(); + let (store, base_path) = ObjectStore::from_uri(&base_uri).await.unwrap(); + let shard_id = Uuid::new_v4(); + let first_batch = record_batch!(("id", Int32, [1])).unwrap(); + let schema = first_batch.schema(); + let config = durable_writer_config(shard_id); + + let writer_a = ShardWriter::open( + store.clone(), + base_path.clone(), + base_uri.clone(), + config.clone(), + schema.clone(), + vec![], + ) + .await + .unwrap(); + + // Write and fully flush generation N, then rotate to generation N+1. + writer_a.put(vec![first_batch]).await.unwrap(); + let first_generation = writer_a.memtable_stats().await.unwrap().generation; + writer_a.force_seal_active().await.unwrap(); + writer_a.wait_for_flush_drain().await.unwrap(); + let current_generation = writer_a.memtable_stats().await.unwrap().generation; + assert_eq!( + current_generation, + first_generation + 1, + "expected rotation from generation {first_generation}, got {current_generation}" + ); + + // A peer claims a higher epoch, fencing writer A's next WAL append. + let writer_b = ShardWriter::open(store, base_path, base_uri, config, schema, vec![]) + .await + .unwrap(); + assert!( + writer_b.epoch() > writer_a.epoch(), + "expected peer epoch to increase: writer_a={}, writer_b={}", + writer_a.epoch(), + writer_b.epoch() + ); + + // Generation N+1's first durable put lands at the same local batch + // position (0..1) that generation N already flushed. It must wait for its + // own flush and therefore surface the fence, not ack from the stale + // generation. + let second_batch = record_batch!(("id", Int32, [2])).unwrap(); + let error = writer_a + .put(vec![second_batch]) + .await + .expect_err("generation-2 durable put must wait for its own WAL flush"); + assert_eq!( + error.fence_reason(), + Some(FenceReason::PeerClaimedEpoch), + "expected a peer-claimed-epoch fence, got: {error}" + ); + assert!( + error.to_string().contains("Writer fenced"), + "expected the writer-fenced error prefix, got: {error}" + ); + + writer_b.close().await.unwrap(); +} diff --git a/rust/lance/tests/query/inverted.rs b/rust/lance/tests/query/inverted.rs index 4392278c0d8..b1db7fd6f2f 100644 --- a/rust/lance/tests/query/inverted.rs +++ b/rust/lance/tests/query/inverted.rs @@ -5,17 +5,29 @@ use std::sync::Arc; use arrow_array::cast::AsArray; use arrow_array::{ - ArrayRef, Int32Array, RecordBatch, RecordBatchIterator, StringArray, UInt32Array, + ArrayRef, FixedSizeListArray, Float32Array, Int32Array, ListArray, RecordBatch, + RecordBatchIterator, StringArray, StructArray, UInt32Array, + builder::{ListBuilder, StringBuilder}, }; +use arrow_buffer::{OffsetBuffer, ScalarBuffer}; +use arrow_schema::{DataType, Field as ArrowField, Fields as ArrowFields}; use lance::Dataset; -use lance::dataset::scanner::ColumnOrdering; -use lance::dataset::{InsertBuilder, WriteParams}; -use lance::index::DatasetIndexExt; +use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::dataset::scanner::{ColumnOrdering, QueryFilter}; +use lance::dataset::{ColumnAlteration, InsertBuilder, WriteMode, WriteParams}; +use lance::index::{DatasetIndexExt, DatasetIndexInternalExt}; +use lance_arrow::FixedSizeListArrayExt; use lance_index::IndexType; -use lance_index::scalar::inverted::Language; -use lance_index::scalar::inverted::query::{FtsQuery, PhraseQuery}; +use lance_index::metrics::NoOpMetricsCollector; +use lance_index::optimize::OptimizeOptions; +use lance_index::prefilter::NoFilter; +use lance_index::scalar::inverted::query::{ + BooleanQuery, BoostQuery, FtsQuery, FtsSearchParams, MatchQuery, MultiMatchQuery, Occur, + Operator, PhraseQuery, collect_query_tokens, +}; +use lance_index::scalar::inverted::{DocumentGranularity, Language}; use lance_index::scalar::{FullTextSearchQuery, InvertedIndexParams}; -use lance_table::format::IndexMetadata; +use lance_table::format::{Fragment, IndexMetadata}; use super::{strip_score_column, test_fts, test_scan, test_take}; use crate::utils::DatasetTestCases; @@ -41,6 +53,46 @@ fn params_for(base_tokenizer: &str, lower_case: bool, with_position: bool) -> In .max_token_length(None) } +fn list_element_params(with_position: bool) -> InvertedIndexParams { + base_inverted_params(with_position).document_granularity(DocumentGranularity::ListElement) +} + +fn list_element_match_node(column: &str, terms: &str) -> MatchQuery { + MatchQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::ListElement) +} + +fn list_element_match(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Match(list_element_match_node(column, terms))) +} + +fn list_element_phrase(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::ListElement), + )) +} + +fn row_match_node(column: &str, terms: &str) -> MatchQuery { + MatchQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::Row) +} + +fn row_match(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Match(row_match_node(column, terms))) +} + +fn row_phrase(column: &str, terms: &str) -> FullTextSearchQuery { + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new(terms.to_string()) + .with_column(Some(column.to_string())) + .with_document_granularity(DocumentGranularity::Row), + )) +} + // Execute a full-text search with optional filter and deterministic id ordering. async fn run_fts(ds: &Dataset, query: FullTextSearchQuery, filter: Option<&str>) -> RecordBatch { let mut scanner = ds.scan(); @@ -56,6 +108,23 @@ async fn run_fts(ds: &Dataset, query: FullTextSearchQuery, filter: Option<&str>) scanner.try_into_batch().await.unwrap() } +async fn boolean_fts_ids_on_fragment( + dataset: &Dataset, + fragment: Fragment, + query: BooleanQuery, +) -> Vec { + let mut scanner = dataset.scan(); + scanner.with_fragments(vec![fragment]); + scanner.project(&["id"]).unwrap(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Boolean(query))) + .unwrap(); + scanner.try_into_batch().await.unwrap()["id"] + .as_primitive::() + .values() + .to_vec() +} + // Run an FTS query and assert results match a deterministic expected batch. async fn assert_fts_expected( original: &RecordBatch, @@ -75,6 +144,971 @@ async fn assert_fts_expected( assert_eq!(&expected, &scanned); } +fn string_lists(values: &[Option>>]) -> ArrayRef { + let mut builder = ListBuilder::new(StringBuilder::new()); + for value in values { + match value { + Some(elements) => { + for element in elements { + match element { + Some(element) => builder.values().append_value(element), + None => builder.values().append_null(), + } + } + builder.append(true); + } + None => builder.append(false), + } + } + Arc::new(builder.finish()) +} + +fn element_hits(batch: &RecordBatch) -> Vec<(i32, Vec)> { + let ids = batch["id"].as_primitive::(); + let coordinates = batch["_doc_index"] + .as_any() + .downcast_ref::() + .unwrap(); + let mut hits = (0..batch.num_rows()) + .map(|row| { + let coordinate = coordinates.value(row); + ( + ids.value(row), + coordinate + .as_primitive::() + .values() + .to_vec(), + ) + }) + .collect::>(); + hits.sort_unstable(); + hits +} + +fn element_scored_hits(batch: &RecordBatch) -> Vec<(i32, Vec, f32)> { + let ids = batch["id"].as_primitive::(); + let coordinates = batch["_doc_index"].as_list::(); + let scores = batch["_score"].as_primitive::(); + let mut hits = (0..batch.num_rows()) + .map(|row| { + ( + ids.value(row), + coordinates + .value(row) + .as_primitive::() + .values() + .to_vec(), + scores.value(row), + ) + }) + .collect::>(); + hits.sort_unstable_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + hits +} + +fn assert_same_element_scores(left: &RecordBatch, right: &RecordBatch) { + let left = element_scored_hits(left); + let right = element_scored_hits(right); + assert_eq!(left.len(), right.len()); + for (left, right) in left.iter().zip(&right) { + assert_eq!((&left.0, &left.1), (&right.0, &right.1)); + assert!((left.2 - right.2).abs() < 1e-5, "{left:?} != {right:?}"); + } +} + +fn assert_element_coordinates_point_to(batch: &RecordBatch, column: &str, term: &str) { + let values = batch[column].as_list::(); + let coordinates = batch["_doc_index"].as_list::(); + for row in 0..batch.num_rows() { + let coordinate = coordinates + .value(row) + .as_primitive::() + .value(0) as usize; + let elements = values.value(row); + let element = elements.as_string::().value(coordinate); + assert!( + element.contains(term), + "coordinate {coordinate} selected {element:?}" + ); + } +} + +fn expected_bm25_score( + num_docs: usize, + token_docs: usize, + total_tokens: u32, + doc_tokens: u32, +) -> f32 { + let num_docs = num_docs as f32; + let idf = ((num_docs - token_docs as f32 + 0.5) / (token_docs as f32 + 0.5) + 1.0).ln(); + let avg_doc_length = total_tokens as f32 / num_docs; + let doc_norm = 1.2 * (1.0 - 0.75 + 0.75 * doc_tokens as f32 / avg_doc_length); + idf * 2.2 / (1.0 + doc_norm) +} + +#[tokio::test] +async fn test_boolean_must_not_uses_all_index_fragment_coverage() { + let initial = arrow_array::record_batch!( + ("id", Int32, [0]), + ("positive_text", Utf8, ["placeholder"]), + ("negative_text", Utf8, ["placeholder"]) + ) + .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(initial.clone())], initial.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + dataset + .create_index( + &["positive_text"], + IndexType::Inverted, + None, + &base_inverted_params(false), + true, + ) + .await + .unwrap(); + + let appended = arrow_array::record_batch!( + ("id", Int32, [1, 2]), + ("positive_text", Utf8, ["include", "include"]), + ("negative_text", Utf8, ["exclude", "keep"]) + ) + .unwrap(); + let appended_reader = RecordBatchIterator::new([Ok(appended.clone())], appended.schema()); + dataset.append(appended_reader, None).await.unwrap(); + let appended_fragment = dataset.fragments().last().unwrap().clone(); + + dataset + .create_index( + &["negative_text"], + IndexType::Inverted, + None, + &base_inverted_params(false), + true, + ) + .await + .unwrap(); + + let query = BooleanQuery::new([ + ( + Occur::Must, + FtsQuery::Match(row_match_node("positive_text", "include")), + ), + ( + Occur::MustNot, + FtsQuery::Match(row_match_node("negative_text", "exclude")), + ), + ]); + assert_eq!( + boolean_fts_ids_on_fragment(&dataset, appended_fragment.clone(), query).await, + vec![2] + ); + + let partially_qualified_query = BooleanQuery::new([ + ( + Occur::Must, + FtsQuery::Match(MatchQuery::new("include".to_string())), + ), + ( + Occur::MustNot, + FtsQuery::Match(row_match_node("negative_text", "exclude")), + ), + ]); + assert_eq!( + boolean_fts_ids_on_fragment(&dataset, appended_fragment, partially_qualified_query).await, + vec![2] + ); +} + +#[tokio::test] +async fn test_row_document_raw_list_is_consistent_across_index_coverage() { + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0])) as ArrayRef), + ("tags", string_lists(&[Some(vec![Some("a"), Some("b")])])), + ]) + .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + ¶ms_for("raw", false, false), + true, + ) + .await + .unwrap(); + + let appended = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![1])) as ArrayRef), + ("tags", string_lists(&[Some(vec![Some("a"), Some("b")])])), + ]) + .unwrap(); + ds = InsertBuilder::new(Arc::new(ds)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![appended]) + .await + .unwrap(); + + let joined = run_fts(&ds, row_match("tags", "a b"), None).await; + assert_eq!( + joined["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + let element = run_fts(&ds, row_match("tags", "a"), None).await; + assert_eq!(element.num_rows(), 0); +} + +#[tokio::test] +async fn test_element_document_fts_flat_indexed_and_mixed() { + let ids = Arc::new(Int32Array::from(vec![0, 1, 2, 3, 4, 5])); + let tags = string_lists(&[ + Some(vec![ + Some("alpha beta"), + Some("gamma alpha"), + None, + Some(""), + Some("delta"), + ]), + Some(vec![Some("beta"), Some("gamma")]), + None, + Some(vec![]), + Some(vec![None, None]), + Some(vec![Some("!!!"), Some("epsilon")]), + ]); + let batch = RecordBatch::try_from_iter(vec![("id", ids as ArrayRef), ("tags", tags)]).unwrap(); + let schema = batch.schema(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + let element_query = list_element_match("tags", "alpha"); + let element_and_query = FullTextSearchQuery::new_query(FtsQuery::Match( + list_element_match_node("tags", "alpha beta").with_operator(Operator::And), + )); + let flat = run_fts(&ds, element_query.clone(), None).await; + assert_eq!(element_hits(&flat), vec![(0, vec![0]), (0, vec![1])]); + let flat_and = run_fts(&ds, element_and_query.clone(), None).await; + assert_eq!(element_hits(&flat_and), vec![(0, vec![0])]); + let flat_phrase = run_fts(&ds, list_element_phrase("tags", "alpha beta"), None).await; + assert_eq!(element_hits(&flat_phrase), vec![(0, vec![0])]); + let flat_cross_element_phrase = + run_fts(&ds, list_element_phrase("tags", "beta gamma"), None).await; + assert_eq!(flat_cross_element_phrase.num_rows(), 0); + let row_flat_phrase = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Phrase( + PhraseQuery::new("beta gamma".to_string()).with_column(Some("tags".to_string())), + )), + None, + ) + .await; + assert_eq!( + row_flat_phrase["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + assert_element_coordinates_point_to(&flat, "tags", "alpha"); + let params = list_element_params(true); + let err = ds + .create_index(&["id"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap_err(); + assert!(err.to_string().contains("must resolve to Utf8"), "{err}"); + let err = ds + .create_index(&["tags[*]"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap_err(); + assert!(err.to_string().contains("tags[*]"), "{err}"); + ds.create_index(&["tags"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + let mut element_only_auto = ds.scan(); + element_only_auto + .full_text_search(FullTextSearchQuery::new("alpha".to_string())) + .unwrap(); + let err = element_only_auto.try_into_batch().await.unwrap_err(); + assert!( + err.to_string() + .contains("unless an INVERTED index has been created"), + "{err}" + ); + let inferred_element = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Match( + MatchQuery::new("alpha".to_string()).with_column(Some("tags".to_string())), + )), + None, + ) + .await; + assert_eq!( + element_hits(&inferred_element), + vec![(0, vec![0]), (0, vec![1])] + ); + let mut mismatched_row = ds.scan(); + mismatched_row + .full_text_search(row_match("tags", "alpha")) + .unwrap(); + let err = mismatched_row.try_into_batch().await.unwrap_err(); + assert!( + err.to_string().contains("requested Row") && err.to_string().contains("ListElement"), + "{err}" + ); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &base_inverted_params(true), + true, + ) + .await + .unwrap(); + let names = ds + .load_indices() + .await + .unwrap() + .iter() + .map(|index| index.name.clone()) + .collect::>(); + assert!(names.contains(&"tags_idx".to_string())); + assert!(names.contains(&"tags_list_element_idx".to_string())); + + let auto_row = run_fts(&ds, FullTextSearchQuery::new("alpha".to_string()), None).await; + assert_eq!(auto_row.num_rows(), 1); + assert!(auto_row.column_by_name("_doc_index").is_none()); + let mut ambiguous = ds.scan(); + ambiguous + .full_text_search( + FullTextSearchQuery::new("alpha".to_string()) + .with_column("tags".to_string()) + .unwrap(), + ) + .unwrap(); + let err = ambiguous.try_into_batch().await.unwrap_err(); + assert!( + err.to_string().contains("ambiguous") + && err.to_string().contains("specify document_granularity"), + "{err}" + ); + let mut row_projection = ds.scan(); + row_projection + .full_text_search(row_match("tags", "alpha")) + .unwrap(); + row_projection.project(&["_doc_index"]).unwrap(); + let err = row_projection.try_into_batch().await.unwrap_err(); + assert!(err.to_string().contains("_doc_index"), "{err}"); + + let indexed = run_fts(&ds, element_query.clone(), None).await; + assert_eq!(element_hits(&indexed), vec![(0, vec![0]), (0, vec![1])]); + let indexed_and = run_fts(&ds, element_and_query, None).await; + assert_eq!(element_hits(&indexed_and), vec![(0, vec![0])]); + assert_element_coordinates_point_to(&indexed, "tags", "alpha"); + let element_index = ds + .load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .pop() + .unwrap(); + assert_eq!( + element_index.index_version, + lance_index::scalar::inverted::INVERTED_INDEX_VERSION_V3 as i32 + ); + let element_index = ds + .open_scalar_index("tags", &element_index.uuid, &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!(element_index.statistics().unwrap()["num_docs"], 11); + let element_index = element_index + .as_any() + .downcast_ref::() + .unwrap(); + let (total_tokens, num_docs, token_docs) = element_index + .bm25_stats_for_terms(&["alpha".to_string()], None) + .await + .unwrap(); + assert_eq!((total_tokens, num_docs, token_docs), (8, 11, vec![2])); + let mut tokenizer = element_index.tokenizer(); + let tokens = Arc::new(collect_query_tokens("alpha", &mut tokenizer)); + let documents = element_index + .bm25_search_documents( + tokens, + Arc::new(FtsSearchParams::default()), + Operator::Or, + Arc::new(NoFilter), + Arc::new(NoOpMetricsCollector), + None, + ) + .await + .unwrap(); + let expected_score = expected_bm25_score(11, 2, 8, 2); + assert!( + documents + .iter() + .all(|document| (document.score.0 - expected_score).abs() < 1e-5), + "{documents:?}" + ); + assert_same_element_scores(&flat, &indexed); + let indexed_phrase = run_fts(&ds, list_element_phrase("tags", "alpha beta"), None).await; + assert_eq!(element_hits(&indexed_phrase), vec![(0, vec![0])]); + let filtered = run_fts(&ds, element_query.clone(), Some("id = 0")).await; + assert_eq!(element_hits(&filtered), vec![(0, vec![0]), (0, vec![1])]); + let ordinal = run_fts(&ds, list_element_match("tags", "delta"), None).await; + assert_eq!(element_hits(&ordinal), vec![(0, vec![4])]); + let punctuation_gap = run_fts(&ds, list_element_match("tags", "epsilon"), None).await; + assert_eq!(element_hits(&punctuation_gap), vec![(5, vec![1])]); + let limited = run_fts(&ds, element_query.clone().limit(Some(1)), None).await; + assert_eq!(limited.num_rows(), 1); + assert_eq!( + limited["id"] + .as_primitive::() + .value(0), + 0 + ); + assert_eq!(element_hits(&limited).len(), 1); + + let element_match = |terms: &str| FtsQuery::Match(list_element_match_node("tags", terms)); + let boolean = BooleanQuery::new([ + (Occur::Must, element_match("alpha")), + (Occur::Must, element_match("beta")), + ]); + let boolean = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Boolean(boolean)), + None, + ) + .await; + assert_eq!(element_hits(&boolean), vec![(0, vec![0])]); + + let boost = BoostQuery::new(element_match("alpha"), element_match("gamma"), Some(0.5)); + let boost = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Boost(boost)), + None, + ) + .await; + assert_eq!(element_hits(&boost), vec![(0, vec![0]), (0, vec![1])]); + + let phrase = PhraseQuery::new("beta gamma".to_string()) + .with_column(Some("tags".to_string())) + .with_document_granularity(DocumentGranularity::ListElement); + let element_phrase = run_fts( + &ds, + FullTextSearchQuery::new_query(FtsQuery::Phrase(phrase)), + None, + ) + .await; + assert_eq!(element_phrase.num_rows(), 0); + + let row_phrase = run_fts(&ds, row_phrase("tags", "beta gamma"), None).await; + assert_eq!( + row_phrase["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + assert!(row_phrase.column_by_name("_doc_index").is_none()); + + let cross_target = BooleanQuery::new([ + (Occur::Must, element_match("alpha")), + (Occur::Must, FtsQuery::Match(row_match_node("tags", "beta"))), + ]); + let mut scanner = ds.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Boolean( + cross_target, + ))) + .unwrap(); + let err = scanner + .try_into_batch() + .await + .expect_err("mixed document granularities must be rejected"); + assert!( + err.to_string() + .contains("cannot mix Row and ListElement document granularities"), + "{err}" + ); + + let mut multi_match = + MultiMatchQuery::try_new("alpha".to_string(), vec!["tags".to_string()]).unwrap(); + multi_match.match_queries[0].document_granularity = Some(DocumentGranularity::ListElement); + let mut scanner = ds.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::MultiMatch( + multi_match, + ))) + .unwrap(); + let err = scanner + .try_into_batch() + .await + .expect_err("ListElement MultiMatch must be rejected"); + assert!( + err.to_string() + .contains("MultiMatch does not support ListElement document granularity"), + "{err}" + ); + + let mut row_multi_match = + MultiMatchQuery::try_new("beta".to_string(), vec!["tags".to_string()]).unwrap(); + row_multi_match.match_queries[0].document_granularity = Some(DocumentGranularity::Row); + let mixed_multi_match = BooleanQuery::new([ + (Occur::Must, element_match("alpha")), + (Occur::Must, FtsQuery::MultiMatch(row_multi_match)), + ]); + let mut scanner = ds.scan(); + scanner + .full_text_search(FullTextSearchQuery::new_query(FtsQuery::Boolean( + mixed_multi_match, + ))) + .unwrap(); + let err = scanner + .try_into_batch() + .await + .expect_err("mixed document granularities must be rejected"); + assert!( + err.to_string() + .contains("cannot mix Row and ListElement document granularities"), + "{err}" + ); + + let appended = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![6])) as ArrayRef), + ( + "tags", + string_lists(&[Some(vec![Some("alpha beta"), Some("alpha again")])]), + ), + ]) + .unwrap(); + ds = InsertBuilder::new(Arc::new(ds)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![appended]) + .await + .unwrap(); + + let mixed = run_fts(&ds, element_query, None).await; + assert_eq!( + element_hits(&mixed), + vec![(0, vec![0]), (0, vec![1]), (6, vec![0]), (6, vec![1])] + ); + assert_element_coordinates_point_to(&mixed, "tags", "alpha"); + let mixed_phrase = run_fts(&ds, list_element_phrase("tags", "alpha beta"), None).await; + assert_eq!( + element_hits(&mixed_phrase), + vec![(0, vec![0]), (6, vec![0])] + ); + let all_data = ds.scan().try_into_batch().await.unwrap(); + let flat_reference_dir = tempfile::tempdir().unwrap(); + let flat_reference_ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(all_data.clone())], all_data.schema()), + flat_reference_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + let flat_reference = run_fts( + &flat_reference_ds, + list_element_match("tags", "alpha"), + None, + ) + .await; + assert_same_element_scores(&mixed, &flat_reference); + let flat_phrase_reference = run_fts( + &flat_reference_ds, + list_element_phrase("tags", "alpha beta"), + None, + ) + .await; + assert_same_element_scores(&mixed_phrase, &flat_phrase_reference); + + ds.optimize_indices(&OptimizeOptions::append()) + .await + .unwrap(); + assert_eq!( + ds.load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .len(), + 2 + ); + let optimized = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_same_element_scores(&optimized, &flat_reference); + assert_element_coordinates_point_to(&optimized, "tags", "alpha"); + + ds.optimize_indices( + &OptimizeOptions::merge(2).index_names(vec!["tags_list_element_idx".to_string()]), + ) + .await + .unwrap(); + assert_eq!( + ds.load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .len(), + 1 + ); + let merged = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_same_element_scores(&merged, &flat_reference); + assert_element_coordinates_point_to(&merged, "tags", "alpha"); + + ds.delete("id = 0").await.unwrap(); + compact_files(&mut ds, CompactionOptions::default(), None) + .await + .unwrap(); + let compacted = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_eq!(element_hits(&compacted), vec![(6, vec![0]), (6, vec![1])]); + + ds.alter_columns(&[ColumnAlteration::new("tags".into()).rename("labels".into())]) + .await + .unwrap(); + let renamed = run_fts(&ds, list_element_match("labels", "alpha"), None).await; + assert_eq!(element_hits(&renamed), vec![(6, vec![0]), (6, vec![1])]); + let mut old_name_scanner = ds.scan(); + old_name_scanner + .full_text_search(list_element_match("tags", "alpha")) + .unwrap(); + let err = old_name_scanner + .try_into_batch() + .await + .expect_err("renamed element target must reject the old path"); + assert!(err.to_string().contains("tags"), "{err}"); +} + +#[tokio::test] +async fn test_element_document_persists_empty_and_zero_token_corpora() { + async fn assert_empty_query(tags: ArrayRef, expected_documents: usize, with_position: bool) { + let ids = Arc::new(Int32Array::from_iter_values(0..tags.len() as i32)); + let batch = + RecordBatch::try_from_iter(vec![("id", ids as ArrayRef), ("tags", tags)]).unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &list_element_params(with_position), + true, + ) + .await + .unwrap(); + + let result = run_fts(&ds, list_element_match("tags", "alpha"), None).await; + assert_eq!(result.num_rows(), 0); + + let metadata = ds + .load_indices_by_name("tags_list_element_idx") + .await + .unwrap() + .pop() + .unwrap(); + let index = ds + .open_scalar_index("tags", &metadata.uuid, &NoOpMetricsCollector) + .await + .unwrap(); + assert_eq!(index.statistics().unwrap()["num_docs"], expected_documents); + } + + for with_position in [false, true] { + assert_empty_query(string_lists(&[None, Some(vec![])]), 0, with_position).await; + assert_empty_query( + string_lists(&[Some(vec![None, Some(""), Some("!!!")])]), + 3, + with_position, + ) + .await; + } +} + +#[tokio::test] +async fn test_element_document_nested_lists_use_deepest_boundary() { + let doc_fields = ArrowFields::from(vec![ArrowField::new("content", DataType::Utf8, true)]); + let doc_values = StructArray::new( + doc_fields.clone(), + vec![Arc::new(StringArray::from(vec![ + Some("alpha"), + Some("beta"), + Some("gamma"), + Some("alpha delta"), + Some("alpha"), + ])) as ArrayRef], + None, + ); + let doc_item = Arc::new(ArrowField::new("item", DataType::Struct(doc_fields), true)); + let docs_type = DataType::List(doc_item.clone()); + let docs = ListArray::new( + doc_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 4, 4, 5])), + Arc::new(doc_values), + None, + ); + let group_fields = ArrowFields::from(vec![ArrowField::new("docs", docs_type, true)]); + let group_values = + StructArray::new(group_fields.clone(), vec![Arc::new(docs) as ArrayRef], None); + let group_item = Arc::new(ArrowField::new( + "item", + DataType::Struct(group_fields), + true, + )); + let groups = ListArray::new( + group_item, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2, 4])), + Arc::new(group_values), + None, + ); + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ("groups", Arc::new(groups) as ArrayRef), + ]) + .unwrap(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + + let path = "groups.docs.content"; + let expected = vec![(0, vec![0, 0]), (0, vec![1, 1]), (1, vec![1, 0])]; + let flat = run_fts(&ds, list_element_match(path, "alpha"), None).await; + assert_eq!(element_hits(&flat), expected); + + let row_flat = run_fts( + &ds, + FullTextSearchQuery::new("alpha".to_string()) + .with_column(path.to_string()) + .unwrap(), + None, + ) + .await; + assert_eq!( + row_flat["id"] + .as_primitive::() + .values(), + &[0, 1] + ); + assert!(row_flat.column_by_name("_doc_index").is_none()); + + ds.create_index( + &[path], + IndexType::Inverted, + None, + &list_element_params(true), + true, + ) + .await + .unwrap(); + ds.create_index( + &[path], + IndexType::Inverted, + None, + &base_inverted_params(true), + true, + ) + .await + .unwrap(); + + let indexed = run_fts(&ds, list_element_match(path, "alpha"), None).await; + assert_eq!(element_hits(&indexed), expected); + let indices = ds.load_indices().await.unwrap(); + let row = indices + .iter() + .find(|index| index.name == "groups.docs.content_idx") + .unwrap(); + let elements = indices + .iter() + .find(|index| index.name == "groups.docs.content_list_element_idx") + .unwrap(); + let groups = ds.schema().field("groups").unwrap(); + let group_item = groups.children.first().unwrap(); + let docs = group_item + .children + .iter() + .find(|field| field.name == "docs") + .unwrap(); + let doc_item = docs.children.first().unwrap(); + let content = doc_item + .children + .iter() + .find(|field| field.name == "content") + .unwrap(); + assert_eq!(row.fields, elements.fields); + assert_eq!(row.fields, vec![content.id]); + assert_ne!(row.fields, vec![docs.children[0].id]); +} + +#[tokio::test] +async fn test_element_document_bm25_uses_element_corpus() { + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ( + "tags", + string_lists(&[ + Some(vec![ + Some("needle"), + Some("filler filler filler filler filler filler filler filler filler"), + ]), + Some(vec![Some("needle filler")]), + ]), + ), + ]) + .unwrap(); + let schema = batch.schema(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + let params = base_inverted_params(false); + ds.create_index(&["tags"], IndexType::Inverted, None, ¶ms, true) + .await + .unwrap(); + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &list_element_params(false), + true, + ) + .await + .unwrap(); + + let row = run_fts(&ds, row_match("tags", "needle"), None).await; + let element = run_fts(&ds, list_element_match("tags", "needle"), None).await; + let row_scores = row["_score"].as_primitive::(); + let element_scores = element["_score"].as_primitive::(); + + let expected_row_long = expected_bm25_score(2, 2, 12, 10); + let expected_row_short = expected_bm25_score(2, 2, 12, 2); + let expected_element_short = expected_bm25_score(3, 2, 12, 1); + let expected_element_long = expected_bm25_score(3, 2, 12, 2); + assert!((row_scores.value(0) - expected_row_long).abs() < 1e-5); + assert!((row_scores.value(1) - expected_row_short).abs() < 1e-5); + assert!((element_scores.value(0) - expected_element_short).abs() < 1e-5); + assert!((element_scores.value(1) - expected_element_long).abs() < 1e-5); + assert!(row_scores.value(1) > row_scores.value(0)); + assert!(element_scores.value(0) > element_scores.value(1)); +} + +#[tokio::test] +async fn test_element_document_fts_vector_prefilter_deduplicates_parent_rows() { + let vectors = + FixedSizeListArray::try_new_from_values(Float32Array::from(vec![0.0, 0.0, 1.0, 1.0]), 2) + .unwrap(); + let batch = RecordBatch::try_from_iter(vec![ + ("id", Arc::new(Int32Array::from(vec![0, 1])) as ArrayRef), + ( + "tags", + string_lists(&[ + Some(vec![Some("alpha"), Some("alpha again")]), + Some(vec![Some("beta")]), + ]), + ), + ("vector", Arc::new(vectors) as ArrayRef), + ]) + .unwrap(); + let schema = batch.schema(); + let test_dir = tempfile::tempdir().unwrap(); + let mut ds = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + test_dir.path().to_str().unwrap(), + None, + ) + .await + .unwrap(); + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &list_element_params(false), + true, + ) + .await + .unwrap(); + + let query_vector = Float32Array::from(vec![0.0, 0.0]); + let mut scanner = ds.scan(); + scanner + .nearest("vector", &query_vector, 10) + .unwrap() + .filter_query(QueryFilter::Fts(list_element_match("tags", "alpha"))) + .unwrap() + .prefilter(true); + let result = scanner.try_into_batch().await.unwrap(); + + assert_eq!( + result["id"] + .as_primitive::() + .values(), + &[0] + ); + assert!(result.column_by_name("_doc_index").is_none()); + + ds.create_index( + &["tags"], + IndexType::Inverted, + None, + &base_inverted_params(false), + true, + ) + .await + .unwrap(); + let mut scanner = ds.scan(); + scanner + .nearest("vector", &query_vector, 10) + .unwrap() + .filter_query(QueryFilter::Fts(FullTextSearchQuery::new( + "alpha".to_string(), + ))) + .unwrap() + .prefilter(true); + let result = scanner.try_into_batch().await.unwrap(); + assert_eq!( + result["id"] + .as_primitive::() + .values(), + &[0] + ); + assert!(result.column_by_name("_doc_index").is_none()); +} + #[tokio::test] // Ensure indexed and non-indexed full-text search return the same ids. async fn test_inverted_basic_equivalence() { diff --git a/rust/lance/tests/query/mod.rs b/rust/lance/tests/query/mod.rs index 9e609b19d0b..9c71765c345 100644 --- a/rust/lance/tests/query/mod.rs +++ b/rust/lance/tests/query/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; -use arrow_array::{RecordBatch, UInt32Array, cast::AsArray}; +use arrow_array::{RecordBatch, UInt32Array, cast::AsArray, types::Int32Type}; use arrow_select::concat::concat_batches; use datafusion::datasource::MemTable; use datafusion::prelude::SessionContext; @@ -97,6 +97,42 @@ async fn test_filter(original: &RecordBatch, ds: &Dataset, predicate: &str) { assert_eq!(&expected, &scanned); } +/// Assert a filtered scan returns exactly `expected_ids`, once through whatever +/// index the dataset has and once with scalar indices turned off. +/// +/// Use this instead of [`test_filter`] for predicates whose correct answer +/// differs from what the pinned DataFusion release computes, so there is no +/// reference implementation to compare against. The un-indexed pass matters +/// because `DatasetTestCases` never actually generates the no-index variant: its +/// combination generator drops the empty combination. +async fn assert_filter_ids(ds: &Dataset, predicate: &str, expected_ids: &[i32]) { + for use_scalar_index in [true, false] { + let mut scanner = ds.scan(); + scanner + .project(&["id"]) + .unwrap() + .filter(predicate) + .unwrap() + .use_scalar_index(use_scalar_index) + .order_by(Some(vec![ColumnOrdering::asc_nulls_first( + "id".to_string(), + )])) + .unwrap(); + let scanned = scanner.try_into_batch().await.unwrap(); + // Collected as options so a NULL id cannot read back as a real id, and so + // a length mismatch fails here rather than silently comparing a prefix. + let ids = scanned["id"] + .as_primitive::() + .iter() + .collect::>(); + let expected = expected_ids.iter().copied().map(Some).collect::>(); + assert_eq!( + ids, expected, + "predicate: {predicate}, index: {use_scalar_index}" + ); + } +} + // Rebuild a batch using only columns present in the schema (drops _score from FTS results). fn strip_score_column(batch: &RecordBatch, schema: &arrow_schema::Schema) -> RecordBatch { let columns = schema diff --git a/rust/lance/tests/query/primitives.rs b/rust/lance/tests/query/primitives.rs index 65fa6f4e4d3..7836a0bae94 100644 --- a/rust/lance/tests/query/primitives.rs +++ b/rust/lance/tests/query/primitives.rs @@ -6,18 +6,20 @@ use std::sync::Arc; use arrow::datatypes::*; use arrow_array::{ ArrayRef, BinaryArray, BinaryViewArray, Float32Array, Float64Array, Int32Array, - LargeBinaryArray, LargeStringArray, RecordBatch, StringArray, StringViewArray, + LargeBinaryArray, LargeStringArray, RecordBatch, RecordBatchIterator, StringArray, + StringViewArray, }; use arrow_schema::DataType; use lance::Dataset; -use lance::dataset::WriteParams; use lance::dataset::optimize::{CompactionOptions, compact_files}; +use lance::dataset::{InsertBuilder, WriteMode, WriteParams}; use lance::index::DatasetIndexExt; use lance_datagen::{ArrayGeneratorExt, RowCount, array, gen_batch}; use lance_index::IndexType; +use lance_index::scalar::ScalarIndexParams; -use super::{test_filter, test_scan, test_take}; +use super::{assert_filter_ids, test_filter, test_scan, test_take}; use crate::utils::DatasetTestCases; #[tokio::test] @@ -172,6 +174,9 @@ async fn test_query_float(#[case] data_type: DataType) { #[tokio::test] #[rstest::rstest] +// Float16 is missing on purpose: `safe_coerce_scalar` has no Float16 arm, so +// `value < 0.0` against a Float16 column fails to resolve the literal long before +// any of this matters. See rust/lance-datafusion/src/expr.rs. #[case::float32(DataType::Float32)] #[case::float64(DataType::Float64)] async fn test_query_float_special_values(#[case] data_type: DataType) { @@ -223,17 +228,144 @@ async fn test_query_float_special_values(#[case] data_type: DataType) { .run(|ds: Dataset, original: RecordBatch| async move { test_scan(&original, &ds).await; test_take(&original, &ds).await; - test_filter(&original, &ds, "value > 0.0").await; - test_filter(&original, &ds, "value < 0.0").await; - test_filter(&original, &ds, "value = 0.0").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; test_filter(&original, &ds, "isnan(value)").await; test_filter(&original, &ds, "not isnan(value)").await; + + // The remaining predicates compare against zero, where DataFusion + // 54 answers by Arrow's total order: it ranks `-0.0` below `+0.0` + // instead of treating the two encodings as one number the way + // IEEE 754 and SQL do. That makes it useless as the reference, so + // assert the rows. Ids are 0: +0.0, 1: -0.0, 2: +inf, 3: -inf, + // 4: NaN, 5: 1.0, 6: -1.0, 7: MIN, 8: MAX, 9: NULL. + for zero in ["0.0", "-0.0"] { + assert_filter_ids(&ds, &format!("value < {zero}"), &[3, 6, 7]).await; + assert_filter_ids(&ds, &format!("value <= {zero}"), &[0, 1, 3, 6, 7]).await; + assert_filter_ids(&ds, &format!("value = {zero}"), &[0, 1]).await; + assert_filter_ids(&ds, &format!("value != {zero}"), &[2, 3, 4, 5, 6, 7, 8]).await; + // NaN is row 4. Arrow sorts it above every other value, so it + // survives `>` and `>=`, which IEEE would reject. That gap is + // not specific to zero and this rewrite leaves it alone. + assert_filter_ids(&ds, &format!("value > {zero}"), &[2, 4, 5, 8]).await; + assert_filter_ids(&ds, &format!("value >= {zero}"), &[0, 1, 2, 4, 5, 8]).await; + // A literal on the left. DataFusion's canonicalizer swaps it back + // before the rewrite runs, so this pins the answer rather than the + // mirroring branch, which `a_literal_on_the_left_mirrors_the_operator` + // owns and which SQL reaches only when the other side is not a + // bare column. + assert_filter_ids(&ds, &format!("{zero} > value"), &[3, 6, 7]).await; + // BETWEEN only works because the simplifier expands it into two + // comparisons before the rewrite runs. + assert_filter_ids(&ds, &format!("value BETWEEN {zero} AND {zero}"), &[0, 1]).await; + assert_filter_ids( + &ds, + &format!("value NOT BETWEEN {zero} AND {zero}"), + &[2, 3, 4, 5, 6, 7, 8], + ) + .await; + // An IN list gains the encoding it does not spell out. + assert_filter_ids(&ds, &format!("value IN ({zero}, 1.0)"), &[0, 1, 5]).await; + assert_filter_ids( + &ds, + &format!("value NOT IN ({zero}, 1.0)"), + &[2, 3, 4, 6, 7, 8], + ) + .await; + // Composed with NULL logic, where this index layer has broken before. + assert_filter_ids( + &ds, + &format!("value != {zero} OR value IS NULL"), + &[2, 3, 4, 5, 6, 7, 8, 9], + ) + .await; + } }) .await } +/// A rewritten zero predicate still has to reach a scalar index. Without this, +/// the rewrite could reshape the predicate into something `maybe_indexed_column` +/// no longer recognizes, and every zero filter would quietly fall back to a full +/// scan plus refine while still returning the right rows. Only the default BTree +/// index is covered here; the other index types are exercised for row equality by +/// `test_query_float_special_values`, not for pushdown. +#[tokio::test] +async fn test_float_zero_predicate_uses_scalar_index() { + let batch = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(0..4)) as ArrayRef, + ), + ( + "value", + Arc::new(Float64Array::from(vec![0.0, -0.0, 1.0, -1.0])) as ArrayRef, + ), + ]) + .unwrap(); + let schema = batch.schema(); + let reader = RecordBatchIterator::new(vec![Ok(batch)], schema); + let mut ds = Dataset::write(reader, "memory://zero_index_pushdown", None) + .await + .unwrap(); + ds.create_index( + &["value"], + IndexType::Scalar, + None, + &ScalarIndexParams::default(), + false, + ) + .await + .unwrap(); + + for predicate in ["value = 0.0", "value < 0.0", "value != 0.0"] { + let plan = ds + .scan() + .filter(predicate) + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery"), + "`{predicate}` should use the scalar index, got plan:\n{plan}" + ); + // The rewrite's output survives a second `optimize_expr`, which the scan + // path does run, so the predicate must not appear twice. + assert_eq!( + plan.matches("value_idx").count(), + 1, + "`{predicate}` should search the index once, got plan:\n{plan}" + ); + } + + // Rows appended after the index is built are answered by the unindexed scan + // while the rest come from the index. Both halves have to agree. + let appended = RecordBatch::try_from_iter(vec![ + ( + "id", + Arc::new(Int32Array::from_iter_values(4..8)) as ArrayRef, + ), + ( + "value", + Arc::new(Float64Array::from(vec![0.0, -0.0, 1.0, -1.0])) as ArrayRef, + ), + ]) + .unwrap(); + let ds = InsertBuilder::new(Arc::new(ds)) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute(vec![appended]) + .await + .unwrap(); + + assert_filter_ids(&ds, "value = 0.0", &[0, 1, 4, 5]).await; + assert_filter_ids(&ds, "value < 0.0", &[3, 7]).await; + assert_filter_ids(&ds, "value >= 0.0", &[0, 1, 2, 4, 5, 6]).await; +} + #[tokio::test] #[rstest::rstest] #[case::date32(DataType::Date32)] @@ -260,7 +392,9 @@ async fn test_query_date(#[case] data_type: DataType) { test_scan(&original, &ds).await; test_take(&original, &ds).await; test_filter(&original, &ds, "value < current_date()").await; - test_filter(&original, &ds, "value > DATE '2024-01-01'").await; + // Mid-range literal: rand_type samples dates from the fixed range + // [2023-01-01, 2024-01-01), so this splits the generated values + test_filter(&original, &ds, "value > DATE '2023-07-01'").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; }) @@ -295,7 +429,9 @@ async fn test_query_timestamp(#[case] data_type: DataType) { test_scan(&original, &ds).await; test_take(&original, &ds).await; test_filter(&original, &ds, "value < current_timestamp()").await; - test_filter(&original, &ds, "value > TIMESTAMP '2024-01-01 00:00:00'").await; + // Mid-range literal: rand_type samples timestamps from the fixed range + // [2023-01-01, 2024-01-01), so this splits the generated values + test_filter(&original, &ds, "value > TIMESTAMP '2023-07-01 00:00:00'").await; test_filter(&original, &ds, "value is null").await; test_filter(&original, &ds, "value is not null").await; }) @@ -513,3 +649,87 @@ async fn test_filtered_scan_after_compact_with_srid() { results.num_rows() ); } + +/// Verifies that a zone map index on a string column is used (ScalarIndexQuery +/// in the plan) for both IS NULL and IS NOT NULL predicate filters. +/// +/// IS NOT NULL must not silently fall back to a full scan when a zone map +/// index exists — both predicates should leverage the index. +#[tokio::test] +async fn test_zone_map_null_index_used() { + // 6 non-null strings and 4 null values across 10 rows. + let string_values = vec![ + Some("alpha"), + None, + Some("beta"), + Some("gamma"), + None, + Some("delta"), + None, + Some("epsilon"), + Some("zeta"), + None, + ]; + let value_array = Arc::new(StringArray::from(string_values)) as ArrayRef; + let id_array = Arc::new(Int32Array::from((0..10).collect::>())) as ArrayRef; + let batch = RecordBatch::try_from_iter(vec![("id", id_array), ("value", value_array)]).unwrap(); + + let mut ds = InsertBuilder::new("memory://") + .execute(vec![batch]) + .await + .unwrap(); + + ds.create_index( + &["value"], + IndexType::ZoneMap, + None, + &lance_index::scalar::ScalarIndexParams::default(), + true, + ) + .await + .unwrap(); + + // IS NULL: the zone map index must appear in the plan. + let plan = ds + .scan() + .filter("value IS NULL") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery"), + "IS NULL should use zone map index, got plan:\n{}", + plan + ); + let null_batch = ds + .scan() + .filter("value IS NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(null_batch.num_rows(), 4); + + // IS NOT NULL: the zone map index must also appear in the plan. + let plan = ds + .scan() + .filter("value IS NOT NULL") + .unwrap() + .explain_plan(false) + .await + .unwrap(); + assert!( + plan.contains("ScalarIndexQuery"), + "IS NOT NULL should use zone map index, got plan:\n{}", + plan + ); + let non_null_batch = ds + .scan() + .filter("value IS NOT NULL") + .unwrap() + .try_into_batch() + .await + .unwrap(); + assert_eq!(non_null_batch.num_rows(), 6); +} diff --git a/rust/lance/tests/scalar_index_spill.rs b/rust/lance/tests/scalar_index_spill.rs new file mode 100644 index 00000000000..595e625477e --- /dev/null +++ b/rust/lance/tests/scalar_index_spill.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The Lance Authors + +//! Regression test: training a BTREE scalar index on a large string column +//! must succeed when the sort spills under a small bounded memory pool. +//! +//! Sorting far more data than the pool holds produces many spilled runs; the +//! external-sort merge phase then needs pool memory on top of the runs. +//! Before the sort spill reservation was sized to the pool (#7675), the merge +//! reservation could exceed the whole `FairSpillPool`, failing index creation +//! with `ResourcesExhausted` unless spilling was bypassed entirely via +//! `LANCE_BYPASS_SPILLING`. +//! +//! This file must stay a single-test integration binary: the training path +//! only reads the pool size from the process-global `LANCE_MEM_POOL_SIZE`, so +//! any sibling test could race the `set_var` or inherit the tiny pool. + +use lance::Dataset; +use lance::dataset::WriteParams; +use lance::index::DatasetIndexExt; +use lance_datafusion::exec::LanceExecutionOptions; +use lance_datagen::{BatchCount, ByteCount, RowCount, array, gen_batch}; +use lance_index::IndexType; +use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; + +const MEM_POOL_SIZE: u64 = 4 * 1024 * 1024; + +#[tokio::test] +async fn test_btree_training_sort_spill_merge_fits_pool() { + // 4 MiB pool vs ~36 MiB of sort input (512K rows of 64-byte strings plus + // row ids) forces many spilled sort runs. + unsafe { + std::env::set_var("LANCE_MEM_POOL_SIZE", MEM_POOL_SIZE.to_string()); + // The historical workaround for this very bug; if it leaks in from the + // environment the bounded pool is skipped and the test checks nothing. + std::env::remove_var("LANCE_BYPASS_SPILLING"); + } + // The training scan builds its execution options from the env vars above; + // fail loudly if that plumbing ever changes, otherwise the sort would run + // against the default (much larger) pool and pass vacuously. + let options = LanceExecutionOptions { + use_spilling: true, + ..Default::default() + }; + assert_eq!(options.mem_pool_size(), MEM_POOL_SIZE); + assert!(options.use_spilling()); + + let data = gen_batch() + .col("value", array::rand_utf8(ByteCount::from(64), false)) + .into_reader_rows(RowCount::from(8192), BatchCount::from(64)); + + let write_params = WriteParams { + max_rows_per_file: 256 * 1024, + ..Default::default() + }; + let mut dataset = Dataset::write(data, "memory://", Some(write_params)) + .await + .unwrap(); + assert!(dataset.get_fragments().len() > 1); + + let params = ScalarIndexParams::for_builtin(BuiltinIndexType::BTree); + dataset + .create_index(&["value"], IndexType::BTree, None, ¶ms, false) + .await + .unwrap(); + + let indices = dataset.load_indices().await.unwrap(); + assert_eq!(indices.len(), 1); +} diff --git a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock index 6fcff711a06..531e90b410f 100644 --- a/test_data/fri_straddle_pre_6610/datagen/Cargo.lock +++ b/test_data/fri_straddle_pre_6610/datagen/Cargo.lock @@ -1039,6 +1039,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1124,9 +1135,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -2541,11 +2552,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -2555,10 +2564,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -4594,14 +4606,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -4675,6 +4688,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -4713,6 +4737,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -4723,6 +4753,15 @@ dependencies = [ "rand 0.9.4", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xoshiro" version = "0.7.0" diff --git a/test_data/readme.md b/test_data/readme.md index 8d041a96db5..544b54fd3b9 100644 --- a/test_data/readme.md +++ b/test_data/readme.md @@ -27,7 +27,19 @@ folder contains a `datagen.py` script that generates one or more lance datasets. correctly, so there are duplicate field ids in the schema. There aren't great workarounds for readers. Writers should make sure to check the field ids in the schema and re-compute them if necessary. +* `v0.10.15/non_divisible_pq`: This dataset has an 8-bit IVF-PQ index whose + 64-dimensional vectors were divided into 14 sub-vectors. Writers at this + version silently omitted the final eight dimensions from the PQ codebook. + Readers should preserve that prefix-only search behavior. * `v0.27.1/pq_in_schema`: This dataset uses the old method of storing the PQ metadata in the schema metadata in the index file. We switched to storing them in a global buffer in https://github.com/lancedb/lance/pull/3829, but still - need to be able to read the old format. \ No newline at end of file + need to be able to read the old format. +* `v3.0.1/fts_v1` and `v4.0.1/fts_v2`: These datasets cover the supported FTS + layouts written by stable Lance releases. Each fixture contains posting lists + that cross the layout's block boundary and positions used by phrase queries. + The v1 fixture also retains the retired `skip_merge` parameter written by Lance + 3.0.1. +* `v8.0.0/decimal_zonemap`: This dataset has a Decimal128 ZoneMap whose non-null + values have null min/max statistics because Decimal extrema were not computed + by Lance 8.0.0. diff --git a/test_data/v0.10.15/datagen.py b/test_data/v0.10.15/datagen.py new file mode 100644 index 00000000000..f864cfa0ee8 --- /dev/null +++ b/test_data/v0.10.15/datagen.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import shutil + +import lance +import numpy as np +import pyarrow as pa + +# To generate the test file, we should be running this version of lance. +assert lance.__version__ == "0.10.15" + +name = "non_divisible_pq" +dimension = 64 +num_sub_vectors = 14 +sub_vector_dimension = dimension // num_sub_vectors + +shutil.rmtree(name, ignore_errors=True) + +vector = np.arange(1, dimension + 1, dtype=np.float32) +data = pa.table( + { + "id": pa.array([0]), + "vector": pa.FixedSizeListArray.from_arrays(pa.array(vector), dimension), + } +) +dataset = lance.write_dataset(data, name) + +ivf_centroids = np.zeros((1, dimension), dtype=np.float32) +persisted_prefix = vector[: num_sub_vectors * sub_vector_dimension].reshape( + num_sub_vectors, sub_vector_dimension +) +pq_codebook = np.repeat(persisted_prefix[:, np.newaxis, :], 256, axis=1) +dataset.create_index( + "vector", + "IVF_PQ", + metric="l2", + num_partitions=1, + ivf_centroids=ivf_centroids, + num_sub_vectors=num_sub_vectors, + pq_codebook=pq_codebook, +) diff --git a/test_data/v0.10.15/non_divisible_pq/_indices/be068df8-322d-4309-8347-51afc73e8d3f/index.idx b/test_data/v0.10.15/non_divisible_pq/_indices/be068df8-322d-4309-8347-51afc73e8d3f/index.idx new file mode 100644 index 00000000000..6af76b538b4 Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/_indices/be068df8-322d-4309-8347-51afc73e8d3f/index.idx differ diff --git a/test_data/v0.10.15/non_divisible_pq/_latest.manifest b/test_data/v0.10.15/non_divisible_pq/_latest.manifest new file mode 100644 index 00000000000..3bb9ca51bc7 Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/_latest.manifest differ diff --git a/test_data/v0.10.15/non_divisible_pq/_transactions/0-573045e8-48b7-4662-941d-d30af281d50f.txn b/test_data/v0.10.15/non_divisible_pq/_transactions/0-573045e8-48b7-4662-941d-d30af281d50f.txn new file mode 100644 index 00000000000..c7a0feb2315 Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/_transactions/0-573045e8-48b7-4662-941d-d30af281d50f.txn differ diff --git a/test_data/v0.10.15/non_divisible_pq/_transactions/1-11123412-ba3d-4e23-8edd-5b0d5a1e84f6.txn b/test_data/v0.10.15/non_divisible_pq/_transactions/1-11123412-ba3d-4e23-8edd-5b0d5a1e84f6.txn new file mode 100644 index 00000000000..4a5161722fc Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/_transactions/1-11123412-ba3d-4e23-8edd-5b0d5a1e84f6.txn differ diff --git a/test_data/v0.10.15/non_divisible_pq/_versions/1.manifest b/test_data/v0.10.15/non_divisible_pq/_versions/1.manifest new file mode 100644 index 00000000000..acb27411508 Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/_versions/1.manifest differ diff --git a/test_data/v0.10.15/non_divisible_pq/_versions/2.manifest b/test_data/v0.10.15/non_divisible_pq/_versions/2.manifest new file mode 100644 index 00000000000..3bb9ca51bc7 Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/_versions/2.manifest differ diff --git a/test_data/v0.10.15/non_divisible_pq/data/2f1a33f2-fdcd-40f7-a361-844659965e1e.lance b/test_data/v0.10.15/non_divisible_pq/data/2f1a33f2-fdcd-40f7-a361-844659965e1e.lance new file mode 100644 index 00000000000..8fda283a059 Binary files /dev/null and b/test_data/v0.10.15/non_divisible_pq/data/2f1a33f2-fdcd-40f7-a361-844659965e1e.lance differ diff --git a/test_data/v3.0.1/datagen.py b/test_data/v3.0.1/datagen.py new file mode 100644 index 00000000000..2815351490c --- /dev/null +++ b/test_data/v3.0.1/datagen.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import os +import shutil +from pathlib import Path + +import lance +import pyarrow as pa +from lance.query import PhraseQuery + +EXPECTED_LANCE_VERSION = "3.0.1" +EXPECTED_FTS_VERSION = 1 +NUM_ROWS = 300 + +assert lance.__version__ == EXPECTED_LANCE_VERSION + +os.environ["LANCE_FTS_FORMAT_VERSION"] = str(EXPECTED_FTS_VERSION) + +dataset_path = Path(__file__).parent / "fts_v1" +shutil.rmtree(dataset_path, ignore_errors=True) + +row_ids = list(range(NUM_ROWS)) +texts = [ + "lance database compatibility shared" + if row_id % 3 == 0 + else "database lance compatibility shared" + for row_id in row_ids +] +dataset = lance.write_dataset(pa.table({"id": row_ids, "text": texts}), dataset_path) +dataset.create_scalar_index( + "text", + "INVERTED", + with_position=True, + skip_merge=True, +) + +index = dataset.describe_indices()[0] +assert index.segments[0].index_version == EXPECTED_FTS_VERSION + +matches = dataset.to_table(full_text_query="compatibility") +assert set(matches["id"].to_pylist()) == set(row_ids) + +phrase_matches = dataset.to_table(full_text_query=PhraseQuery("lance database", "text")) +assert set(phrase_matches["id"].to_pylist()) == set(range(0, NUM_ROWS, 3)) diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance new file mode 100644 index 00000000000..10d6e21f91a Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/metadata.lance differ diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance new file mode 100644 index 00000000000..d7314b6b7c2 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_docs.lance differ diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance new file mode 100644 index 00000000000..609d3cbcbb9 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_invert.lance differ diff --git a/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance new file mode 100644 index 00000000000..614e7e40872 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_indices/450d1cd4-9eb2-4a11-8eac-ab1b64b1ea31/part_1_tokens.lance differ diff --git a/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn b/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn new file mode 100644 index 00000000000..2f01cf7c9b2 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_transactions/0-0e238eef-4390-4c40-9c7e-3c1437b4735b.txn differ diff --git a/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn b/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn new file mode 100644 index 00000000000..333e934e259 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_transactions/1-d331315a-f1a6-4735-a453-90470239bd60.txn differ diff --git a/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest b/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000..d61e99244a2 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_versions/18446744073709551613.manifest differ diff --git a/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest b/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000..7839e3ac728 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/_versions/18446744073709551614.manifest differ diff --git a/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance b/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance new file mode 100644 index 00000000000..96e80c7b9a8 Binary files /dev/null and b/test_data/v3.0.1/fts_v1/data/110110010101010101010101b80a6f4c4d804a62b40de56ffb.lance differ diff --git a/test_data/v4.0.1/datagen.py b/test_data/v4.0.1/datagen.py new file mode 100644 index 00000000000..accaa57cadf --- /dev/null +++ b/test_data/v4.0.1/datagen.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import os +import shutil +from pathlib import Path + +import lance +import pyarrow as pa +from lance.query import PhraseQuery + +EXPECTED_LANCE_VERSION = "4.0.1" +EXPECTED_FTS_VERSION = 2 +NUM_ROWS = 300 + +assert lance.__version__ == EXPECTED_LANCE_VERSION + +os.environ["LANCE_FTS_FORMAT_VERSION"] = str(EXPECTED_FTS_VERSION) + +dataset_path = Path(__file__).parent / "fts_v2" +shutil.rmtree(dataset_path, ignore_errors=True) + +row_ids = list(range(NUM_ROWS)) +texts = [ + "lance database compatibility shared" + if row_id % 3 == 0 + else "database lance compatibility shared" + for row_id in row_ids +] +dataset = lance.write_dataset(pa.table({"id": row_ids, "text": texts}), dataset_path) +dataset.create_scalar_index("text", "INVERTED", with_position=True) + +index = dataset.describe_indices()[0] +assert index.segments[0].index_version == EXPECTED_FTS_VERSION + +matches = dataset.to_table(full_text_query="compatibility") +assert set(matches["id"].to_pylist()) == set(row_ids) + +phrase_matches = dataset.to_table(full_text_query=PhraseQuery("lance database", "text")) +assert set(phrase_matches["id"].to_pylist()) == set(range(0, NUM_ROWS, 3)) diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance new file mode 100644 index 00000000000..1f3d5698f43 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/metadata.lance differ diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance new file mode 100644 index 00000000000..d7314b6b7c2 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_docs.lance differ diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance new file mode 100644 index 00000000000..196ad67f15a Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_invert.lance differ diff --git a/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance new file mode 100644 index 00000000000..614e7e40872 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_indices/1fb8e405-43d0-4707-be37-64eadc3d5e97/part_6_tokens.lance differ diff --git a/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn b/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn new file mode 100644 index 00000000000..f550fa2c193 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_transactions/0-fbd896dd-b250-46fa-b60d-9b1474d7fc7b.txn differ diff --git a/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn b/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn new file mode 100644 index 00000000000..3cbfb8b41cb Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_transactions/1-ebe80000-4421-4ed6-8072-093abeb55eaf.txn differ diff --git a/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest b/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000..6fa1d8bb6e7 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_versions/18446744073709551613.manifest differ diff --git a/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest b/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000..cb8ff78ef70 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/_versions/18446744073709551614.manifest differ diff --git a/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance b/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance new file mode 100644 index 00000000000..96e80c7b9a8 Binary files /dev/null and b/test_data/v4.0.1/fts_v2/data/000100000010011011000110c47acb4569aa4d2c1957105328.lance differ diff --git a/test_data/v6.0.1/datagen.py b/test_data/v6.0.1/datagen.py new file mode 100644 index 00000000000..127eb883f6e --- /dev/null +++ b/test_data/v6.0.1/datagen.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import shutil +from pathlib import Path + +import lance +import pyarrow as pa + +EXPECTED_LANCE_VERSION = "6.0.1" +NUM_DENSE_ROWS = 513 +VALUES_PER_DENSE_ROW = 32 +NUM_TRAILING_ROWS = 65_536 + +assert lance.__version__ == EXPECTED_LANCE_VERSION + +dataset_path = Path(__file__).parent / "miniblock_level_count_overflow.lance" +shutil.rmtree(dataset_path, ignore_errors=True) + +captions = pa.array( + [ + list(range(row * VALUES_PER_DENSE_ROW, (row + 1) * VALUES_PER_DENSE_ROW)) + for row in range(NUM_DENSE_ROWS) + ] + + [([] if row % 2 == 0 else None) for row in range(NUM_TRAILING_ROWS)], + type=pa.list_(pa.uint32()), +) +table = pa.table( + { + "mime": ["image/jpeg"] * len(captions), + "captions": captions, + } +) +lance.write_dataset(table, dataset_path, data_storage_version="2.2") + +# The v6.0.1 writer truncated a miniblock's structural level count to u16 while +# retaining the complete RLE payload. Confirm this generator still captures the defect. +try: + lance.dataset(dataset_path).to_table() +except pa.ArrowInvalid as error: + assert 'StructArray field "captions", expected 8192 got 513' in str(error) +else: + raise AssertionError("expected the v6.0.1 miniblock level-count overflow") diff --git a/test_data/v6.0.1/miniblock_level_count_overflow.lance/_transactions/0-8f7139ed-6394-401a-bad6-d22598894380.txn b/test_data/v6.0.1/miniblock_level_count_overflow.lance/_transactions/0-8f7139ed-6394-401a-bad6-d22598894380.txn new file mode 100644 index 00000000000..8e4a8a56abe Binary files /dev/null and b/test_data/v6.0.1/miniblock_level_count_overflow.lance/_transactions/0-8f7139ed-6394-401a-bad6-d22598894380.txn differ diff --git a/test_data/v6.0.1/miniblock_level_count_overflow.lance/_versions/18446744073709551614.manifest b/test_data/v6.0.1/miniblock_level_count_overflow.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000..284bfccc89e Binary files /dev/null and b/test_data/v6.0.1/miniblock_level_count_overflow.lance/_versions/18446744073709551614.manifest differ diff --git a/test_data/v6.0.1/miniblock_level_count_overflow.lance/data/110111010100010000111011182b1d41469840c6b679aeab55.lance b/test_data/v6.0.1/miniblock_level_count_overflow.lance/data/110111010100010000111011182b1d41469840c6b679aeab55.lance new file mode 100644 index 00000000000..b16a3cc2826 Binary files /dev/null and b/test_data/v6.0.1/miniblock_level_count_overflow.lance/data/110111010100010000111011182b1d41469840c6b679aeab55.lance differ diff --git a/test_data/v8.0.0/datagen.py b/test_data/v8.0.0/datagen.py new file mode 100644 index 00000000000..dea3ca107ec --- /dev/null +++ b/test_data/v8.0.0/datagen.py @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The Lance Authors + +import shutil +from decimal import Decimal +from pathlib import Path + +import lance +import pyarrow as pa +from lance.indices import IndexConfig + +EXPECTED_LANCE_VERSION = "8.0.0" + +assert lance.__version__ == EXPECTED_LANCE_VERSION + +dataset_path = Path(__file__).parent / "decimal_zonemap" +shutil.rmtree(dataset_path, ignore_errors=True) + +values = pa.array( + [Decimal("1.00"), Decimal("2.00"), Decimal("3.00")], + type=pa.decimal128(10, 2), +) +dataset = lance.write_dataset( + pa.table({"id": [1, 2, 3], "value": values}), + dataset_path, +) +dataset.create_scalar_index("value", IndexConfig("zonemap", {})) + +indices = dataset.describe_indices() +assert len(indices) == 1 +assert indices[0].index_type == "ZoneMap" diff --git a/test_data/v8.0.0/decimal_zonemap/_indices/ac25dab3-5657-4e81-90b8-df2daacdb7ab/zonemap.lance b/test_data/v8.0.0/decimal_zonemap/_indices/ac25dab3-5657-4e81-90b8-df2daacdb7ab/zonemap.lance new file mode 100644 index 00000000000..1d993f7c6a9 Binary files /dev/null and b/test_data/v8.0.0/decimal_zonemap/_indices/ac25dab3-5657-4e81-90b8-df2daacdb7ab/zonemap.lance differ diff --git a/test_data/v8.0.0/decimal_zonemap/_transactions/0-6ba4fa06-16d6-41b4-83cf-cf793c636534.txn b/test_data/v8.0.0/decimal_zonemap/_transactions/0-6ba4fa06-16d6-41b4-83cf-cf793c636534.txn new file mode 100644 index 00000000000..d1522a4f8cd Binary files /dev/null and b/test_data/v8.0.0/decimal_zonemap/_transactions/0-6ba4fa06-16d6-41b4-83cf-cf793c636534.txn differ diff --git a/test_data/v8.0.0/decimal_zonemap/_transactions/1-aba0c0c9-b72c-425a-8fb2-d38e1635659c.txn b/test_data/v8.0.0/decimal_zonemap/_transactions/1-aba0c0c9-b72c-425a-8fb2-d38e1635659c.txn new file mode 100644 index 00000000000..ff43f49c149 Binary files /dev/null and b/test_data/v8.0.0/decimal_zonemap/_transactions/1-aba0c0c9-b72c-425a-8fb2-d38e1635659c.txn differ diff --git a/test_data/v8.0.0/decimal_zonemap/_versions/18446744073709551613.manifest b/test_data/v8.0.0/decimal_zonemap/_versions/18446744073709551613.manifest new file mode 100644 index 00000000000..0725e4e5d41 Binary files /dev/null and b/test_data/v8.0.0/decimal_zonemap/_versions/18446744073709551613.manifest differ diff --git a/test_data/v8.0.0/decimal_zonemap/_versions/18446744073709551614.manifest b/test_data/v8.0.0/decimal_zonemap/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000..ee3bc710a99 Binary files /dev/null and b/test_data/v8.0.0/decimal_zonemap/_versions/18446744073709551614.manifest differ diff --git a/test_data/v8.0.0/decimal_zonemap/_versions/latest_version_hint.json b/test_data/v8.0.0/decimal_zonemap/_versions/latest_version_hint.json new file mode 100644 index 00000000000..218abba1699 --- /dev/null +++ b/test_data/v8.0.0/decimal_zonemap/_versions/latest_version_hint.json @@ -0,0 +1 @@ +{"version":2} \ No newline at end of file diff --git a/test_data/v8.0.0/decimal_zonemap/data/010000100111000010010010c5c73f4e94990fd906290a7890.lance b/test_data/v8.0.0/decimal_zonemap/data/010000100111000010010010c5c73f4e94990fd906290a7890.lance new file mode 100644 index 00000000000..5e6a3d6c6c1 Binary files /dev/null and b/test_data/v8.0.0/decimal_zonemap/data/010000100111000010010010c5c73f4e94990fd906290a7890.lance differ